@fiodos/web-core 0.1.10 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/api/backendClient.js +20 -0
- package/dist/cjs/config/types.d.ts +18 -0
- package/dist/cjs/config/types.js +6 -1
- package/dist/cjs/controller/AgentController.d.ts +84 -1
- package/dist/cjs/controller/AgentController.js +515 -75
- package/dist/cjs/index.d.ts +3 -3
- package/dist/cjs/index.js +2 -1
- package/dist/cjs/orb/mountOrb.d.ts +6 -0
- package/dist/cjs/orb/mountOrb.js +229 -33
- package/dist/cjs/orb/orbView.d.ts +2 -0
- package/dist/cjs/orb/publishedConfig.d.ts +18 -0
- package/dist/cjs/orb/publishedConfig.js +6 -0
- package/dist/cjs/speech/bubbleDictation.d.ts +11 -0
- package/dist/cjs/speech/bubbleDictation.js +88 -0
- package/dist/cjs/ui/messages.d.ts +15 -0
- package/dist/cjs/ui/messages.js +12 -0
- package/dist/cjs/version.d.ts +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/api/backendClient.js +20 -0
- package/dist/esm/config/types.d.ts +18 -0
- package/dist/esm/config/types.js +5 -0
- package/dist/esm/controller/AgentController.d.ts +84 -1
- package/dist/esm/controller/AgentController.js +517 -77
- package/dist/esm/index.d.ts +3 -3
- package/dist/esm/index.js +1 -1
- package/dist/esm/orb/mountOrb.d.ts +6 -0
- package/dist/esm/orb/mountOrb.js +230 -34
- package/dist/esm/orb/orbView.d.ts +2 -0
- package/dist/esm/orb/publishedConfig.d.ts +18 -0
- package/dist/esm/orb/publishedConfig.js +6 -0
- package/dist/esm/speech/bubbleDictation.d.ts +11 -0
- package/dist/esm/speech/bubbleDictation.js +84 -0
- package/dist/esm/ui/messages.d.ts +15 -0
- package/dist/esm/ui/messages.js +12 -0
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +1 -1
|
@@ -15,18 +15,19 @@
|
|
|
15
15
|
* deliberate modal tap; a loose "yes" never confirms them.
|
|
16
16
|
* - The agent only ever executes actions declared in the manifest.
|
|
17
17
|
*/
|
|
18
|
-
import { combineContexts, createActionExecutor, createConsentManager, createSafeTelemetry, createSanitizer, createConversationSession, defaultRouteContextText, formatMessage, prepareConversationForTurn, recordConversationExchange, resolveConfirmationLexicon, resolveMessages, validateManifest, } from '@fiodos/core';
|
|
18
|
+
import { combineContexts, createActionExecutor, createConsentManager, createSafeTelemetry, createSanitizer, buildBubbleExchangeWindow, BUBBLE_PAGE_EXCHANGES, conversationRetentionOptions, createConversationSession, persistConversationSession, restoreConversationSession, defaultRouteContextText, formatMessage, prepareConversationForTurn, recordConversationExchange, resolveConfirmationLexicon, resolveMessages, validateManifest, } from '@fiodos/core';
|
|
19
19
|
import { AgentApiError } from '../api/errors.js';
|
|
20
20
|
import { formatAgentTurnError } from '../api/formatTurnError.js';
|
|
21
21
|
import { buildManifestPayload } from '../api/backendClient.js';
|
|
22
22
|
import { createScreenContextStore, } from '../context/screenContextStore.js';
|
|
23
23
|
import { createSpeechSession } from '../speech/speechSession.js';
|
|
24
24
|
import { classifyPendingReply, decideAction, } from '../core/turnEngine.js';
|
|
25
|
-
import { DEFAULT_AGENT_TIMINGS, } from '../config/types.js';
|
|
25
|
+
import { DEFAULT_AGENT_TIMINGS, DEFAULT_AGENT_CHAINING, } from '../config/types.js';
|
|
26
26
|
import { resolveUiMessages } from '../ui/messages.js';
|
|
27
27
|
function randomSessionId() {
|
|
28
28
|
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
|
29
29
|
}
|
|
30
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
30
31
|
export class AgentController {
|
|
31
32
|
constructor(config) {
|
|
32
33
|
this.enabled = true;
|
|
@@ -55,6 +56,21 @@ export class AgentController {
|
|
|
55
56
|
// audio), skip the extra /preview-tts round-trip and speak via device voice.
|
|
56
57
|
this.serverTtsUnavailable = false;
|
|
57
58
|
this.pendingConsentIntent = null;
|
|
59
|
+
/** Parked chain state (set when a chain pauses on a sensitive action). */
|
|
60
|
+
this.chainState = null;
|
|
61
|
+
/** REAL live activity for the panel (replaces the old canned walk-through). */
|
|
62
|
+
this.liveActivity = null;
|
|
63
|
+
this.chainActive = false;
|
|
64
|
+
/** How many exchanges the bubble currently mounts (grows one page at a time). */
|
|
65
|
+
this.bubbleWindowSize = BUBBLE_PAGE_EXCHANGES;
|
|
66
|
+
/** Raw dashboard retention (minutes; 0 = never; null = default). Persisted
|
|
67
|
+
* with the session so a restore applies the same expiry rule. */
|
|
68
|
+
this.conversationRetentionMinutes = null;
|
|
69
|
+
/** Persistence scope: internal orbs keep their memory under a separate key
|
|
70
|
+
* so a public orb sharing the same appId can never read it. */
|
|
71
|
+
this.conversationScope = 'public';
|
|
72
|
+
this.loadingOlderExchanges = false;
|
|
73
|
+
this.loadOlderTimer = null;
|
|
58
74
|
this.listeners = new Set();
|
|
59
75
|
this.unsubscribeSpeech = null;
|
|
60
76
|
// Fail fast on an invalid manifest — same contract as every platform.
|
|
@@ -81,6 +97,13 @@ export class AgentController {
|
|
|
81
97
|
isAuthenticated: config.isAuthenticated,
|
|
82
98
|
});
|
|
83
99
|
this.timings = { ...DEFAULT_AGENT_TIMINGS, ...config.timings };
|
|
100
|
+
// Chaining OFF by default; clamp to the same safe bounds as react/react-native.
|
|
101
|
+
const chainingMerged = { ...DEFAULT_AGENT_CHAINING, ...config.chaining };
|
|
102
|
+
this.chaining = {
|
|
103
|
+
enabled: chainingMerged.enabled === true,
|
|
104
|
+
maxSteps: Math.max(1, Math.min(6, Math.floor(chainingMerged.maxSteps || 6))),
|
|
105
|
+
dwellMs: Math.max(0, Math.min(3000, Math.floor(chainingMerged.dwellMs ?? 450))),
|
|
106
|
+
};
|
|
84
107
|
this.ttsLocale = config.ttsLocale ?? config.sttLocale;
|
|
85
108
|
this.notifyError =
|
|
86
109
|
config.notifyError ??
|
|
@@ -105,12 +128,34 @@ export class AgentController {
|
|
|
105
128
|
});
|
|
106
129
|
// Re-emit combined state whenever the speech machine advances.
|
|
107
130
|
this.unsubscribeSpeech = this.speech.subscribe(() => this.emit());
|
|
131
|
+
// Rehydrate the previous conversation (page reloads): the persisted blob
|
|
132
|
+
// carries its own retention rule, so expiry is decided correctly even
|
|
133
|
+
// before the published config arrives. Best-effort and non-blocking.
|
|
134
|
+
void this.restorePersistedConversation();
|
|
135
|
+
}
|
|
136
|
+
// ── Conversation persistence (survives page reloads) ───────────────────────
|
|
137
|
+
async restorePersistedConversation() {
|
|
138
|
+
const restored = await restoreConversationSession(this.config.storage, this.config.manifest.appId, Date.now(), this.conversationScope);
|
|
139
|
+
if (!restored)
|
|
140
|
+
return;
|
|
141
|
+
// Never clobber turns the user already made while the restore resolved.
|
|
142
|
+
if (this.conversation.turns.length > 0 || this.conversation.uiArchive?.length)
|
|
143
|
+
return;
|
|
144
|
+
this.conversation.turns = restored.turns;
|
|
145
|
+
this.conversation.summary = restored.summary;
|
|
146
|
+
this.conversation.uiArchive = restored.uiArchive;
|
|
147
|
+
this.conversation.lastActivityAt = restored.lastActivityAt;
|
|
148
|
+
this.emit();
|
|
149
|
+
}
|
|
150
|
+
persistConversation() {
|
|
151
|
+
void persistConversationSession(this.config.storage, this.config.manifest.appId, this.conversation, this.conversationRetentionMinutes, this.conversationScope);
|
|
108
152
|
}
|
|
109
153
|
// ── Subscription / state ───────────────────────────────────────────────────
|
|
110
154
|
getState() {
|
|
111
155
|
const speech = this.speech.getSnapshot();
|
|
112
156
|
const showWaveform = (this.phase === 'listening' || this.phase === 'confirming') && speech.state === 'recording';
|
|
113
157
|
const showThinking = this.phase === 'thinking' || (this.phase === 'listening' && speech.state === 'transcribing');
|
|
158
|
+
const bubbleWindow = buildBubbleExchangeWindow(this.conversation, this.lastExchange, this.bubbleWindowSize, this.conversationOptions);
|
|
114
159
|
return {
|
|
115
160
|
phase: this.phase,
|
|
116
161
|
interimText: speech.interimText,
|
|
@@ -120,12 +165,32 @@ export class AgentController {
|
|
|
120
165
|
isBusy: showThinking || this.phase === 'speaking',
|
|
121
166
|
isListening: this.phase === 'listening' && speech.state !== 'idle',
|
|
122
167
|
lastExchange: this.lastExchange,
|
|
168
|
+
recentExchanges: bubbleWindow.exchanges,
|
|
169
|
+
hasOlderExchanges: bubbleWindow.hasOlder,
|
|
170
|
+
loadingOlderExchanges: this.loadingOlderExchanges,
|
|
123
171
|
confirmationMessage: this.confirmationMessage,
|
|
124
172
|
confirmationVoiceHint: this.confirmationVoiceHint,
|
|
125
173
|
consentModalVisible: this.consentModalVisible,
|
|
126
174
|
voiceBlocked: this.voiceBlocked,
|
|
175
|
+
liveActivity: this.liveActivity,
|
|
176
|
+
chainActive: this.chainActive,
|
|
127
177
|
};
|
|
128
178
|
}
|
|
179
|
+
/** Update the REAL live activity and re-render (no-op when unchanged). */
|
|
180
|
+
setActivity(next) {
|
|
181
|
+
const prev = this.liveActivity;
|
|
182
|
+
if (prev === next || (prev && next && prev.kind === next.kind && prev.label === next.label)) {
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
this.liveActivity = next;
|
|
186
|
+
this.emit();
|
|
187
|
+
}
|
|
188
|
+
setChainActive(value) {
|
|
189
|
+
if (this.chainActive === value)
|
|
190
|
+
return;
|
|
191
|
+
this.chainActive = value;
|
|
192
|
+
this.emit();
|
|
193
|
+
}
|
|
129
194
|
/** Whether speech recognition can actually run (API present AND mic not denied). */
|
|
130
195
|
get voiceAvailable() {
|
|
131
196
|
return this.config.voice.isRecognitionAvailable() && !this.voiceBlocked;
|
|
@@ -144,6 +209,53 @@ export class AgentController {
|
|
|
144
209
|
this.voiceBlocked = blocked;
|
|
145
210
|
this.emit();
|
|
146
211
|
}
|
|
212
|
+
/**
|
|
213
|
+
* Page one more block of older session messages into the bubble (scroll-to-top).
|
|
214
|
+
* The mount is deferred behind a short loading state so old messages stream in
|
|
215
|
+
* on demand instead of all rendering at once (keeps the open bubble light).
|
|
216
|
+
*/
|
|
217
|
+
loadOlderExchanges() {
|
|
218
|
+
if (this.loadingOlderExchanges)
|
|
219
|
+
return;
|
|
220
|
+
const win = buildBubbleExchangeWindow(this.conversation, this.lastExchange, this.bubbleWindowSize, this.conversationOptions);
|
|
221
|
+
if (!win.hasOlder)
|
|
222
|
+
return;
|
|
223
|
+
this.loadingOlderExchanges = true;
|
|
224
|
+
this.emit();
|
|
225
|
+
this.loadOlderTimer = setTimeout(() => {
|
|
226
|
+
this.loadOlderTimer = null;
|
|
227
|
+
this.bubbleWindowSize += BUBBLE_PAGE_EXCHANGES;
|
|
228
|
+
this.loadingOlderExchanges = false;
|
|
229
|
+
this.emit();
|
|
230
|
+
}, 350);
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Apply the dashboard's context-retention setting (minutes; 0 = never clear,
|
|
234
|
+
* null/undefined = the 30-minute default). Called when the published config
|
|
235
|
+
* arrives/refreshes, so the developer's choice is live without a reload.
|
|
236
|
+
* `expandedMemory` = INTERNAL orb profile: use the expanded conversation
|
|
237
|
+
* memory window (more literal turns + longer rolling summary per request).
|
|
238
|
+
*/
|
|
239
|
+
setConversationRetention(minutes, expandedMemory = false) {
|
|
240
|
+
this.conversationOptions = conversationRetentionOptions(minutes, expandedMemory);
|
|
241
|
+
this.conversationRetentionMinutes = typeof minutes === 'number' ? minutes : null;
|
|
242
|
+
const scope = expandedMemory ? 'internal' : 'public';
|
|
243
|
+
if (scope !== this.conversationScope) {
|
|
244
|
+
this.conversationScope = scope;
|
|
245
|
+
// The published profile arrived after the constructor's restore attempt:
|
|
246
|
+
// retry under the right storage key (guarded — never clobbers live turns).
|
|
247
|
+
void this.restorePersistedConversation();
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
/** Collapse the bubble history back to the initial page (bubble closed). */
|
|
251
|
+
resetBubbleWindow() {
|
|
252
|
+
if (this.loadOlderTimer) {
|
|
253
|
+
clearTimeout(this.loadOlderTimer);
|
|
254
|
+
this.loadOlderTimer = null;
|
|
255
|
+
}
|
|
256
|
+
this.loadingOlderExchanges = false;
|
|
257
|
+
this.bubbleWindowSize = BUBBLE_PAGE_EXCHANGES;
|
|
258
|
+
}
|
|
147
259
|
subscribe(listener) {
|
|
148
260
|
this.listeners.add(listener);
|
|
149
261
|
return () => {
|
|
@@ -302,6 +414,11 @@ export class AgentController {
|
|
|
302
414
|
if (this.phase === 'confirming')
|
|
303
415
|
this.setPhase('idle');
|
|
304
416
|
this.emit();
|
|
417
|
+
// REAL activity: the confirmed action is executing right now.
|
|
418
|
+
this.setActivity({
|
|
419
|
+
kind: 'executing',
|
|
420
|
+
label: this.actionLabel(pending.intent) ?? pending.intent,
|
|
421
|
+
});
|
|
305
422
|
const result = await this.executor.execute(pending.action, pending.context);
|
|
306
423
|
this.telemetry.logEvent({
|
|
307
424
|
eventType: result.success ? 'action_executed' : 'action_failed',
|
|
@@ -315,6 +432,39 @@ export class AgentController {
|
|
|
315
432
|
this.emit();
|
|
316
433
|
await this.speakReply(result.message, null);
|
|
317
434
|
}
|
|
435
|
+
// RESUME the autonomous chain if this sensitive action paused one — the
|
|
436
|
+
// human confirmation requirement is fully preserved: the chain only
|
|
437
|
+
// advances past a sensitive step AFTER the user confirmed and it executed.
|
|
438
|
+
const chainState = this.chainState;
|
|
439
|
+
if (chainState) {
|
|
440
|
+
this.chainState = null;
|
|
441
|
+
if (result.success && this.chaining.enabled) {
|
|
442
|
+
this.telemetry.logEvent({
|
|
443
|
+
eventType: 'agent_chain_step',
|
|
444
|
+
intentDetected: pending.intent,
|
|
445
|
+
sessionId: this.sessionId,
|
|
446
|
+
chainStep: chainState.step,
|
|
447
|
+
});
|
|
448
|
+
if (chainState.pendingKey)
|
|
449
|
+
chainState.visitedKeys.add(chainState.pendingKey);
|
|
450
|
+
await this.runChainLoop({
|
|
451
|
+
userMessage: chainState.userMessage,
|
|
452
|
+
step: chainState.step,
|
|
453
|
+
lastStep: chainState.pendingStep ?? chainState.lastStep,
|
|
454
|
+
visitedKeys: chainState.visitedKeys,
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
else if (!result.success) {
|
|
458
|
+
this.telemetry.logEvent({
|
|
459
|
+
eventType: 'agent_chain_aborted',
|
|
460
|
+
sessionId: this.sessionId,
|
|
461
|
+
chainAbortReason: 'error',
|
|
462
|
+
});
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
// The confirmed action (and any resumed chain) finished: clear the live row.
|
|
466
|
+
if (!this.pending)
|
|
467
|
+
this.setActivity(null);
|
|
318
468
|
}
|
|
319
469
|
cancelPendingAction() {
|
|
320
470
|
this.clearConfirmWindow();
|
|
@@ -322,9 +472,20 @@ export class AgentController {
|
|
|
322
472
|
this.pending = null;
|
|
323
473
|
this.confirmationMessage = null;
|
|
324
474
|
this.confirmationVoiceHint = null;
|
|
475
|
+
this.setActivity(null);
|
|
325
476
|
if (this.phase === 'confirming')
|
|
326
477
|
this.setPhase('idle');
|
|
327
478
|
this.emit();
|
|
479
|
+
// Declining a sensitive action stops any chain it was part of, cleanly.
|
|
480
|
+
const chainState = this.chainState;
|
|
481
|
+
this.chainState = null;
|
|
482
|
+
if (chainState) {
|
|
483
|
+
this.telemetry.logEvent({
|
|
484
|
+
eventType: 'agent_chain_aborted',
|
|
485
|
+
sessionId: this.sessionId,
|
|
486
|
+
chainAbortReason: 'user_interrupt',
|
|
487
|
+
});
|
|
488
|
+
}
|
|
328
489
|
if (!pending)
|
|
329
490
|
return;
|
|
330
491
|
this.telemetry.logEvent({
|
|
@@ -368,6 +529,286 @@ export class AgentController {
|
|
|
368
529
|
this.beginConfirmationVoiceWindow();
|
|
369
530
|
}
|
|
370
531
|
}
|
|
532
|
+
routeLabel(intent) {
|
|
533
|
+
return this.config.manifest.routes.find((r) => r.intent === intent)?.label;
|
|
534
|
+
}
|
|
535
|
+
actionLabel(intent) {
|
|
536
|
+
return this.config.manifest.actions.find((a) => a.intent === intent)?.label;
|
|
537
|
+
}
|
|
538
|
+
/**
|
|
539
|
+
* Resolve ONE backend turn into user-facing reply/audio: run its action (or
|
|
540
|
+
* stage its confirmation), emit telemetry, and report whether the autonomous
|
|
541
|
+
* chain may take another step. Shared by the first turn and every
|
|
542
|
+
* continuation so both follow the exact same security path — disabled
|
|
543
|
+
* capabilities, confidentiality and sensitive confirmations apply per step.
|
|
544
|
+
*/
|
|
545
|
+
async applyOutcome(turn, routeAtSend, snapshotContext, userMessage) {
|
|
546
|
+
let outReply = turn.reply;
|
|
547
|
+
let outAudio = turn.audioBase64;
|
|
548
|
+
if (outReply)
|
|
549
|
+
this.serverTtsUnavailable = !outAudio;
|
|
550
|
+
const decision = await decideAction({
|
|
551
|
+
action: turn.action,
|
|
552
|
+
manifest: this.config.manifest,
|
|
553
|
+
executor: this.executor,
|
|
554
|
+
context: snapshotContext,
|
|
555
|
+
messages: this.messages,
|
|
556
|
+
userMessage,
|
|
557
|
+
});
|
|
558
|
+
let canContinue = false;
|
|
559
|
+
let lastStep;
|
|
560
|
+
let actionKey;
|
|
561
|
+
let hasPending = false;
|
|
562
|
+
let pendingInProgress = false;
|
|
563
|
+
let pendingStep;
|
|
564
|
+
let pendingKey;
|
|
565
|
+
if (decision.kind === 'executed') {
|
|
566
|
+
const r = decision.result;
|
|
567
|
+
// REAL activity for the panel: this action actually just ran — the
|
|
568
|
+
// label comes from the manifest, never invented.
|
|
569
|
+
if (r.success && turn.action?.type === 'navigate') {
|
|
570
|
+
this.setActivity({
|
|
571
|
+
kind: 'navigating',
|
|
572
|
+
label: this.routeLabel(turn.action.intent ?? '') ?? turn.action.intent ?? '',
|
|
573
|
+
});
|
|
574
|
+
}
|
|
575
|
+
else if (r.success && !r.recoverable && turn.action?.type === 'execute') {
|
|
576
|
+
this.setActivity({
|
|
577
|
+
kind: 'executing',
|
|
578
|
+
label: this.actionLabel(turn.action.intent ?? '') ?? turn.action.intent ?? '',
|
|
579
|
+
});
|
|
580
|
+
}
|
|
581
|
+
if (turn.action?.type === 'navigate') {
|
|
582
|
+
this.telemetry.logEvent({
|
|
583
|
+
eventType: r.success ? 'navigation_executed' : 'navigation_failed',
|
|
584
|
+
intentDetected: turn.action.intent,
|
|
585
|
+
sessionId: this.sessionId,
|
|
586
|
+
currentRoute: routeAtSend ?? undefined,
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
else if (turn.action?.type === 'execute') {
|
|
590
|
+
this.telemetry.logEvent({
|
|
591
|
+
eventType: r.recoverable
|
|
592
|
+
? 'action_needs_context'
|
|
593
|
+
: r.success
|
|
594
|
+
? 'action_executed'
|
|
595
|
+
: 'action_failed',
|
|
596
|
+
intentDetected: turn.action.intent,
|
|
597
|
+
sessionId: this.sessionId,
|
|
598
|
+
actionExecuted: { intent: turn.action.intent, confirmed: false },
|
|
599
|
+
});
|
|
600
|
+
}
|
|
601
|
+
if (r.recoverable) {
|
|
602
|
+
/* keep the assistant's guiding reply; never speak a refusal. */
|
|
603
|
+
}
|
|
604
|
+
else if (r.message) {
|
|
605
|
+
outReply = r.message;
|
|
606
|
+
outAudio = null;
|
|
607
|
+
}
|
|
608
|
+
// Chain signal: only a cleanly EXECUTED, non-recoverable step may advance,
|
|
609
|
+
// and only when the backend asked to continue (task_status=in_progress).
|
|
610
|
+
if (turn.action &&
|
|
611
|
+
(turn.action.type === 'navigate' || turn.action.type === 'execute') &&
|
|
612
|
+
r.success &&
|
|
613
|
+
!r.recoverable) {
|
|
614
|
+
const intent = turn.action.intent ?? '';
|
|
615
|
+
lastStep = {
|
|
616
|
+
type: turn.action.type,
|
|
617
|
+
intent,
|
|
618
|
+
label: turn.action.type === 'navigate' ? this.routeLabel(intent) : this.actionLabel(intent),
|
|
619
|
+
success: true,
|
|
620
|
+
};
|
|
621
|
+
actionKey = `${turn.action.type}:${intent}`;
|
|
622
|
+
canContinue = turn.taskStatus === 'in_progress';
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
else if (decision.kind === 'precheck-resolved') {
|
|
626
|
+
if (decision.result.message && !decision.result.recoverable) {
|
|
627
|
+
outReply = decision.result.message;
|
|
628
|
+
outAudio = null;
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
else if (decision.kind === 'needs-confirmation') {
|
|
632
|
+
// A sensitive action ALWAYS pauses (chaining or not): stage the
|
|
633
|
+
// confirmation and stop advancing. Resumes after the user confirms.
|
|
634
|
+
this.pending = { ...decision.pending, silent: this.silentTurn };
|
|
635
|
+
this.confirmationReasked = false;
|
|
636
|
+
this.confirmationMessage = decision.pending.question;
|
|
637
|
+
this.confirmationVoiceHint =
|
|
638
|
+
decision.pending.voiceMode === 'strict' &&
|
|
639
|
+
decision.pending.strictPhrase &&
|
|
640
|
+
!this.silentTurn
|
|
641
|
+
? formatMessage(this.messages.strictConfirmationModalHint, {
|
|
642
|
+
phrase: decision.pending.strictPhrase,
|
|
643
|
+
})
|
|
644
|
+
: null;
|
|
645
|
+
this.emit();
|
|
646
|
+
this.telemetry.logEvent({
|
|
647
|
+
eventType: 'action_confirmation_requested',
|
|
648
|
+
intentDetected: decision.pending.intent,
|
|
649
|
+
sessionId: this.sessionId,
|
|
650
|
+
});
|
|
651
|
+
// REAL activity: the agent is genuinely waiting for the human now.
|
|
652
|
+
this.setActivity({ kind: 'waiting_confirmation' });
|
|
653
|
+
hasPending = true;
|
|
654
|
+
pendingInProgress = turn.taskStatus === 'in_progress';
|
|
655
|
+
const pIntent = decision.pending.intent || '';
|
|
656
|
+
pendingStep = { type: 'execute', intent: pIntent, label: this.actionLabel(pIntent), success: true };
|
|
657
|
+
pendingKey = `execute:${pIntent}`;
|
|
658
|
+
}
|
|
659
|
+
return {
|
|
660
|
+
reply: outReply,
|
|
661
|
+
audioBase64: outAudio,
|
|
662
|
+
hasPending,
|
|
663
|
+
canContinue,
|
|
664
|
+
lastStep,
|
|
665
|
+
actionKey,
|
|
666
|
+
pendingInProgress,
|
|
667
|
+
pendingStep,
|
|
668
|
+
pendingKey,
|
|
669
|
+
};
|
|
670
|
+
}
|
|
671
|
+
/**
|
|
672
|
+
* The autonomous continuation loop. Extracted so confirmPendingAction can
|
|
673
|
+
* RESUME the SAME chain after a sensitive step is confirmed. Capped at
|
|
674
|
+
* maxSteps, with no-progress detection and sensitive-confirmation pauses.
|
|
675
|
+
*/
|
|
676
|
+
async runChainLoop(state) {
|
|
677
|
+
let step = state.step;
|
|
678
|
+
let lastStep = state.lastStep;
|
|
679
|
+
const visitedKeys = state.visitedKeys;
|
|
680
|
+
// Keep the panel's live row visible for the WHOLE chain (including the
|
|
681
|
+
// dwell between steps), so the real narration never flickers away.
|
|
682
|
+
this.setChainActive(true);
|
|
683
|
+
try {
|
|
684
|
+
while (step < this.chaining.maxSteps) {
|
|
685
|
+
step += 1;
|
|
686
|
+
if (this.chaining.dwellMs > 0)
|
|
687
|
+
await sleep(this.chaining.dwellMs);
|
|
688
|
+
if (this.pending != null)
|
|
689
|
+
return; // paused / interrupted elsewhere
|
|
690
|
+
const routeNow = this.config.navigation.getCurrentRoute();
|
|
691
|
+
const snap = this.screenStore.getSnapshot();
|
|
692
|
+
const rawCtx = combineContexts(this.config.getUserContextText?.() ?? null, this.screenStore.getText(), {
|
|
693
|
+
truncationSuffix: this.messages.contextTruncationSuffix,
|
|
694
|
+
maxChars: this.config.contextMaxChars,
|
|
695
|
+
});
|
|
696
|
+
const san = rawCtx ? this.sanitizer(rawCtx).sanitized : undefined;
|
|
697
|
+
const conv = prepareConversationForTurn(this.conversation, Date.now(), this.conversationOptions);
|
|
698
|
+
this.setPhase('thinking');
|
|
699
|
+
// Honest state: the model is choosing the next step right now.
|
|
700
|
+
this.setActivity({ kind: 'thinking' });
|
|
701
|
+
const controller = new AbortController();
|
|
702
|
+
this.abort = controller;
|
|
703
|
+
let oc;
|
|
704
|
+
try {
|
|
705
|
+
const turn = await this.config.backend.sendTurn({
|
|
706
|
+
message: state.userMessage,
|
|
707
|
+
language: this.config.locale,
|
|
708
|
+
voice: this.getTtsVoice(),
|
|
709
|
+
screenContext: san,
|
|
710
|
+
currentRoute: routeNow ?? undefined,
|
|
711
|
+
conversationHistory: conv.history,
|
|
712
|
+
conversationSummary: conv.summary ?? undefined,
|
|
713
|
+
sessionId: this.sessionId,
|
|
714
|
+
isContinuation: true,
|
|
715
|
+
chainStep: step,
|
|
716
|
+
lastStep,
|
|
717
|
+
...buildManifestPayload(this.config.manifest),
|
|
718
|
+
}, { signal: controller.signal });
|
|
719
|
+
oc = await this.applyOutcome(turn, routeNow, snap?.context ?? {}, state.userMessage);
|
|
720
|
+
}
|
|
721
|
+
catch (e) {
|
|
722
|
+
this.abort = null;
|
|
723
|
+
const interrupted = e instanceof AgentApiError && e.code === 'cancelled';
|
|
724
|
+
if (interrupted) {
|
|
725
|
+
if (this.phase === 'thinking')
|
|
726
|
+
this.setPhase('idle');
|
|
727
|
+
}
|
|
728
|
+
else {
|
|
729
|
+
if (e instanceof AgentApiError && e.code === 'quota_exceeded') {
|
|
730
|
+
this.config.onQuotaExceeded?.();
|
|
731
|
+
}
|
|
732
|
+
const errMsg = formatAgentTurnError(e, this.config.locale);
|
|
733
|
+
this.lastExchange = { userText: state.userMessage, replyText: errMsg };
|
|
734
|
+
if (this.phase === 'thinking')
|
|
735
|
+
this.setPhase('idle');
|
|
736
|
+
this.emit();
|
|
737
|
+
this.notifyError(errMsg);
|
|
738
|
+
}
|
|
739
|
+
this.telemetry.logEvent({
|
|
740
|
+
eventType: 'agent_chain_aborted',
|
|
741
|
+
sessionId: this.sessionId,
|
|
742
|
+
chainStep: step,
|
|
743
|
+
chainAbortReason: interrupted ? 'user_interrupt' : 'error',
|
|
744
|
+
});
|
|
745
|
+
return;
|
|
746
|
+
}
|
|
747
|
+
this.abort = null;
|
|
748
|
+
if (oc.reply) {
|
|
749
|
+
this.lastExchange = { userText: state.userMessage, replyText: oc.reply };
|
|
750
|
+
this.emit();
|
|
751
|
+
recordConversationExchange(this.conversation, state.userMessage, oc.reply, Date.now(), this.conversationOptions);
|
|
752
|
+
this.persistConversation();
|
|
753
|
+
}
|
|
754
|
+
await this.speakReply(oc.reply, oc.audioBase64);
|
|
755
|
+
this.telemetry.logEvent({
|
|
756
|
+
eventType: 'agent_chain_step',
|
|
757
|
+
intentDetected: oc.lastStep?.intent ?? oc.pendingStep?.intent ?? null,
|
|
758
|
+
sessionId: this.sessionId,
|
|
759
|
+
chainStep: step,
|
|
760
|
+
});
|
|
761
|
+
// Sensitive action → ALWAYS pause for confirmation. Park the chain so
|
|
762
|
+
// confirmPendingAction can resume it (only if the model wants to keep
|
|
763
|
+
// going); a rejection stops it cleanly.
|
|
764
|
+
if (oc.hasPending) {
|
|
765
|
+
this.chainState = oc.pendingInProgress
|
|
766
|
+
? {
|
|
767
|
+
userMessage: state.userMessage,
|
|
768
|
+
step,
|
|
769
|
+
lastStep,
|
|
770
|
+
visitedKeys,
|
|
771
|
+
pendingKey: oc.pendingKey,
|
|
772
|
+
pendingStep: oc.pendingStep,
|
|
773
|
+
}
|
|
774
|
+
: null;
|
|
775
|
+
this.beginConfirmationVoiceWindow();
|
|
776
|
+
return;
|
|
777
|
+
}
|
|
778
|
+
if (this.phase !== 'speaking')
|
|
779
|
+
this.setPhase('idle');
|
|
780
|
+
if (oc.actionKey && visitedKeys.has(oc.actionKey)) {
|
|
781
|
+
this.telemetry.logEvent({
|
|
782
|
+
eventType: 'agent_chain_aborted',
|
|
783
|
+
sessionId: this.sessionId,
|
|
784
|
+
chainStep: step,
|
|
785
|
+
chainAbortReason: 'no_progress',
|
|
786
|
+
});
|
|
787
|
+
return;
|
|
788
|
+
}
|
|
789
|
+
if (oc.actionKey)
|
|
790
|
+
visitedKeys.add(oc.actionKey);
|
|
791
|
+
if (!oc.canContinue) {
|
|
792
|
+
this.telemetry.logEvent({
|
|
793
|
+
eventType: 'agent_chain_completed',
|
|
794
|
+
sessionId: this.sessionId,
|
|
795
|
+
chainStep: step,
|
|
796
|
+
});
|
|
797
|
+
return;
|
|
798
|
+
}
|
|
799
|
+
lastStep = oc.lastStep;
|
|
800
|
+
}
|
|
801
|
+
this.telemetry.logEvent({
|
|
802
|
+
eventType: 'agent_chain_aborted',
|
|
803
|
+
sessionId: this.sessionId,
|
|
804
|
+
chainStep: step,
|
|
805
|
+
chainAbortReason: 'max_steps',
|
|
806
|
+
});
|
|
807
|
+
}
|
|
808
|
+
finally {
|
|
809
|
+
this.setChainActive(false);
|
|
810
|
+
}
|
|
811
|
+
}
|
|
371
812
|
// ── Main turn ────────────────────────────────────────────────────────────────
|
|
372
813
|
async handleTranscript(rawText) {
|
|
373
814
|
const text = rawText.trim();
|
|
@@ -377,6 +818,8 @@ export class AgentController {
|
|
|
377
818
|
return;
|
|
378
819
|
}
|
|
379
820
|
this.setPhase('thinking');
|
|
821
|
+
// REAL activity: the request is genuinely in flight from this moment.
|
|
822
|
+
this.setActivity({ kind: 'thinking' });
|
|
380
823
|
this.pendingTurnText = text;
|
|
381
824
|
this.lastExchange = { userText: text, replyText: '' };
|
|
382
825
|
this.emit();
|
|
@@ -390,6 +833,7 @@ export class AgentController {
|
|
|
390
833
|
: 'The response took too long. Please try again.';
|
|
391
834
|
this.lastExchange = { userText, replyText };
|
|
392
835
|
this.pendingTurnText = null;
|
|
836
|
+
this.setActivity(null);
|
|
393
837
|
this.setPhase('idle');
|
|
394
838
|
this.emit();
|
|
395
839
|
}
|
|
@@ -410,11 +854,10 @@ export class AgentController {
|
|
|
410
854
|
maxChars: this.config.contextMaxChars,
|
|
411
855
|
});
|
|
412
856
|
const sanitized = rawContext ? this.sanitizer(rawContext).sanitized : undefined;
|
|
413
|
-
const conv = prepareConversationForTurn(this.conversation);
|
|
857
|
+
const conv = prepareConversationForTurn(this.conversation, Date.now(), this.conversationOptions);
|
|
414
858
|
const controller = new AbortController();
|
|
415
859
|
this.abort = controller;
|
|
416
|
-
let
|
|
417
|
-
let audioBase64 = null;
|
|
860
|
+
let outcome = null;
|
|
418
861
|
try {
|
|
419
862
|
const turn = await this.config.backend.sendTurn({
|
|
420
863
|
message: text,
|
|
@@ -427,79 +870,12 @@ export class AgentController {
|
|
|
427
870
|
sessionId: this.sessionId,
|
|
428
871
|
...buildManifestPayload(this.config.manifest),
|
|
429
872
|
}, { signal: controller.signal });
|
|
430
|
-
|
|
431
|
-
audioBase64 = turn.audioBase64;
|
|
432
|
-
// Learn whether this backend ships server-side TTS: a non-empty reply
|
|
433
|
-
// with no audio means it doesn't — later lines then speak instantly.
|
|
434
|
-
if (reply)
|
|
435
|
-
this.serverTtsUnavailable = !audioBase64;
|
|
436
|
-
const decision = await decideAction({
|
|
437
|
-
action: turn.action,
|
|
438
|
-
manifest: this.config.manifest,
|
|
439
|
-
executor: this.executor,
|
|
440
|
-
context: snapshot?.context ?? {},
|
|
441
|
-
messages: this.messages,
|
|
442
|
-
userMessage: text,
|
|
443
|
-
});
|
|
444
|
-
if (decision.kind === 'executed') {
|
|
445
|
-
const r = decision.result;
|
|
446
|
-
if (turn.action?.type === 'navigate') {
|
|
447
|
-
this.telemetry.logEvent({
|
|
448
|
-
eventType: r.success ? 'navigation_executed' : 'navigation_failed',
|
|
449
|
-
intentDetected: turn.action.intent,
|
|
450
|
-
sessionId: this.sessionId,
|
|
451
|
-
currentRoute,
|
|
452
|
-
});
|
|
453
|
-
}
|
|
454
|
-
else if (turn.action?.type === 'execute') {
|
|
455
|
-
this.telemetry.logEvent({
|
|
456
|
-
eventType: r.recoverable
|
|
457
|
-
? 'action_needs_context'
|
|
458
|
-
: r.success
|
|
459
|
-
? 'action_executed'
|
|
460
|
-
: 'action_failed',
|
|
461
|
-
intentDetected: turn.action.intent,
|
|
462
|
-
sessionId: this.sessionId,
|
|
463
|
-
actionExecuted: { intent: turn.action.intent, confirmed: false },
|
|
464
|
-
});
|
|
465
|
-
}
|
|
466
|
-
if (r.recoverable) {
|
|
467
|
-
/* keep the assistant's guiding reply; never speak a refusal. */
|
|
468
|
-
}
|
|
469
|
-
else if (r.message) {
|
|
470
|
-
reply = r.message;
|
|
471
|
-
audioBase64 = null;
|
|
472
|
-
}
|
|
473
|
-
}
|
|
474
|
-
else if (decision.kind === 'precheck-resolved') {
|
|
475
|
-
if (decision.result.message && !decision.result.recoverable) {
|
|
476
|
-
reply = decision.result.message;
|
|
477
|
-
audioBase64 = null;
|
|
478
|
-
}
|
|
479
|
-
}
|
|
480
|
-
else if (decision.kind === 'needs-confirmation') {
|
|
481
|
-
this.pending = { ...decision.pending, silent: this.silentTurn };
|
|
482
|
-
this.confirmationReasked = false;
|
|
483
|
-
this.confirmationMessage = decision.pending.question;
|
|
484
|
-
this.confirmationVoiceHint =
|
|
485
|
-
decision.pending.voiceMode === 'strict' &&
|
|
486
|
-
decision.pending.strictPhrase &&
|
|
487
|
-
!this.silentTurn
|
|
488
|
-
? formatMessage(this.messages.strictConfirmationModalHint, {
|
|
489
|
-
phrase: decision.pending.strictPhrase,
|
|
490
|
-
})
|
|
491
|
-
: null;
|
|
492
|
-
this.emit();
|
|
493
|
-
this.telemetry.logEvent({
|
|
494
|
-
eventType: 'action_confirmation_requested',
|
|
495
|
-
intentDetected: decision.pending.intent,
|
|
496
|
-
sessionId: this.sessionId,
|
|
497
|
-
});
|
|
498
|
-
}
|
|
873
|
+
outcome = await this.applyOutcome(turn, currentRoute, snapshot?.context ?? {}, text);
|
|
499
874
|
}
|
|
500
875
|
catch (e) {
|
|
501
876
|
this.clearWatchdog();
|
|
502
877
|
this.abort = null;
|
|
878
|
+
this.setActivity(null);
|
|
503
879
|
if (e instanceof AgentApiError && e.code === 'cancelled') {
|
|
504
880
|
if (this.phase === 'thinking')
|
|
505
881
|
this.setPhase('idle');
|
|
@@ -521,10 +897,14 @@ export class AgentController {
|
|
|
521
897
|
this.clearWatchdog();
|
|
522
898
|
this.abort = null;
|
|
523
899
|
this.pendingTurnText = null;
|
|
900
|
+
if (!outcome)
|
|
901
|
+
return;
|
|
902
|
+
const reply = outcome.reply;
|
|
524
903
|
if (reply) {
|
|
525
904
|
this.lastExchange = { userText: text, replyText: reply };
|
|
526
905
|
this.emit();
|
|
527
|
-
recordConversationExchange(this.conversation, text, reply);
|
|
906
|
+
recordConversationExchange(this.conversation, text, reply, Date.now(), this.conversationOptions);
|
|
907
|
+
this.persistConversation();
|
|
528
908
|
}
|
|
529
909
|
else {
|
|
530
910
|
const fallback = this.config.locale.startsWith('es')
|
|
@@ -533,14 +913,53 @@ export class AgentController {
|
|
|
533
913
|
this.lastExchange = { userText: text, replyText: fallback };
|
|
534
914
|
this.emit();
|
|
535
915
|
}
|
|
536
|
-
const hasPending =
|
|
537
|
-
await this.speakReply(reply, audioBase64);
|
|
916
|
+
const hasPending = outcome.hasPending;
|
|
917
|
+
await this.speakReply(reply, outcome.audioBase64);
|
|
538
918
|
if (hasPending) {
|
|
539
919
|
this.beginConfirmationVoiceWindow();
|
|
920
|
+
// First step is a sensitive action AND the model wants to keep going:
|
|
921
|
+
// park the chain so it RESUMES after the user confirms.
|
|
922
|
+
if (this.chaining.enabled && outcome.pendingInProgress) {
|
|
923
|
+
this.telemetry.logEvent({
|
|
924
|
+
eventType: 'agent_chain_started',
|
|
925
|
+
sessionId: this.sessionId,
|
|
926
|
+
chainStep: 1,
|
|
927
|
+
});
|
|
928
|
+
this.chainState = {
|
|
929
|
+
userMessage: text,
|
|
930
|
+
step: 1,
|
|
931
|
+
lastStep: outcome.pendingStep,
|
|
932
|
+
visitedKeys: new Set(),
|
|
933
|
+
pendingKey: outcome.pendingKey,
|
|
934
|
+
pendingStep: outcome.pendingStep,
|
|
935
|
+
};
|
|
936
|
+
}
|
|
540
937
|
}
|
|
541
938
|
else if (this.phase !== 'speaking') {
|
|
542
939
|
this.setPhase('idle');
|
|
543
940
|
}
|
|
941
|
+
// Kick off the autonomous chain when the first step executed cleanly and
|
|
942
|
+
// the backend asked to continue (no sensitive confirmation pending).
|
|
943
|
+
if (this.chaining.enabled && outcome.canContinue && !hasPending) {
|
|
944
|
+
this.telemetry.logEvent({
|
|
945
|
+
eventType: 'agent_chain_started',
|
|
946
|
+
sessionId: this.sessionId,
|
|
947
|
+
chainStep: 1,
|
|
948
|
+
});
|
|
949
|
+
const visitedKeys = new Set();
|
|
950
|
+
if (outcome.actionKey)
|
|
951
|
+
visitedKeys.add(outcome.actionKey);
|
|
952
|
+
await this.runChainLoop({
|
|
953
|
+
userMessage: text,
|
|
954
|
+
step: 1,
|
|
955
|
+
lastStep: outcome.lastStep,
|
|
956
|
+
visitedKeys,
|
|
957
|
+
});
|
|
958
|
+
}
|
|
959
|
+
// Final reply is on screen: the live narrative disappears (unless the
|
|
960
|
+
// agent is genuinely waiting for a confirmation, which stays honest).
|
|
961
|
+
if (!this.pending)
|
|
962
|
+
this.setActivity(null);
|
|
544
963
|
this.config.onTurnCompleted?.();
|
|
545
964
|
}
|
|
546
965
|
// ── Speech session wiring ────────────────────────────────────────────────────
|
|
@@ -653,6 +1072,11 @@ export class AgentController {
|
|
|
653
1072
|
this.emit();
|
|
654
1073
|
this.telemetry.logEvent({ eventType: 'agent_consent_declined', sessionId: this.sessionId });
|
|
655
1074
|
}
|
|
1075
|
+
/**
|
|
1076
|
+
* Cancel everything in flight. If a first turn was still awaiting its reply,
|
|
1077
|
+
* its message is removed from the panel and returned here so the UI can put
|
|
1078
|
+
* it back into the input for editing; null otherwise.
|
|
1079
|
+
*/
|
|
656
1080
|
cancelAll() {
|
|
657
1081
|
this.abort?.abort();
|
|
658
1082
|
this.abort = null;
|
|
@@ -662,10 +1086,22 @@ export class AgentController {
|
|
|
662
1086
|
this.speech.cancel();
|
|
663
1087
|
this.config.voice.stopPlayback();
|
|
664
1088
|
this.pending = null;
|
|
1089
|
+
this.chainState = null;
|
|
665
1090
|
this.confirmationMessage = null;
|
|
666
1091
|
this.confirmationVoiceHint = null;
|
|
1092
|
+
this.liveActivity = null;
|
|
1093
|
+
this.chainActive = false;
|
|
1094
|
+
// A first turn was still in flight: remove its message from the panel and
|
|
1095
|
+
// hand the text back for editing (the turn never completed, so nothing
|
|
1096
|
+
// was recorded in the conversation).
|
|
1097
|
+
const draft = this.pendingTurnText;
|
|
1098
|
+
if (draft) {
|
|
1099
|
+
this.pendingTurnText = null;
|
|
1100
|
+
this.lastExchange = null;
|
|
1101
|
+
}
|
|
667
1102
|
this.setPhase('idle');
|
|
668
1103
|
this.emit();
|
|
1104
|
+
return draft;
|
|
669
1105
|
}
|
|
670
1106
|
/** Best-effort backend warm-up (cold-start mitigation). */
|
|
671
1107
|
warmUp() {
|
|
@@ -673,6 +1109,10 @@ export class AgentController {
|
|
|
673
1109
|
}
|
|
674
1110
|
dispose() {
|
|
675
1111
|
this.cancelAll();
|
|
1112
|
+
if (this.loadOlderTimer) {
|
|
1113
|
+
clearTimeout(this.loadOlderTimer);
|
|
1114
|
+
this.loadOlderTimer = null;
|
|
1115
|
+
}
|
|
676
1116
|
this.unsubscribeSpeech?.();
|
|
677
1117
|
this.unsubscribeSpeech = null;
|
|
678
1118
|
this.speech.dispose();
|