@fiodos/web-core 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/dist/cjs/api/backendClient.js +20 -0
  2. package/dist/cjs/config/types.d.ts +18 -0
  3. package/dist/cjs/config/types.js +6 -1
  4. package/dist/cjs/controller/AgentController.d.ts +84 -1
  5. package/dist/cjs/controller/AgentController.js +515 -75
  6. package/dist/cjs/index.d.ts +3 -3
  7. package/dist/cjs/index.js +2 -1
  8. package/dist/cjs/orb/mountOrb.d.ts +6 -0
  9. package/dist/cjs/orb/mountOrb.js +307 -53
  10. package/dist/cjs/orb/orbView.d.ts +2 -0
  11. package/dist/cjs/orb/publishedConfig.d.ts +18 -0
  12. package/dist/cjs/orb/publishedConfig.js +6 -0
  13. package/dist/cjs/speech/bubbleDictation.d.ts +11 -0
  14. package/dist/cjs/speech/bubbleDictation.js +88 -0
  15. package/dist/cjs/ui/messages.d.ts +15 -0
  16. package/dist/cjs/ui/messages.js +12 -0
  17. package/dist/cjs/version.d.ts +1 -1
  18. package/dist/cjs/version.js +1 -1
  19. package/dist/esm/api/backendClient.js +20 -0
  20. package/dist/esm/config/types.d.ts +18 -0
  21. package/dist/esm/config/types.js +5 -0
  22. package/dist/esm/controller/AgentController.d.ts +84 -1
  23. package/dist/esm/controller/AgentController.js +517 -77
  24. package/dist/esm/index.d.ts +3 -3
  25. package/dist/esm/index.js +1 -1
  26. package/dist/esm/orb/mountOrb.d.ts +6 -0
  27. package/dist/esm/orb/mountOrb.js +308 -54
  28. package/dist/esm/orb/orbView.d.ts +2 -0
  29. package/dist/esm/orb/publishedConfig.d.ts +18 -0
  30. package/dist/esm/orb/publishedConfig.js +6 -0
  31. package/dist/esm/speech/bubbleDictation.d.ts +11 -0
  32. package/dist/esm/speech/bubbleDictation.js +84 -0
  33. package/dist/esm/ui/messages.d.ts +15 -0
  34. package/dist/esm/ui/messages.js +12 -0
  35. package/dist/esm/version.d.ts +1 -1
  36. package/dist/esm/version.js +1 -1
  37. package/package.json +1 -1
@@ -15,18 +15,19 @@
15
15
  * deliberate modal tap; a loose "yes" never confirms them.
16
16
  * - The agent only ever executes actions declared in the manifest.
17
17
  */
18
- import { combineContexts, createActionExecutor, createConsentManager, createSafeTelemetry, createSanitizer, createConversationSession, defaultRouteContextText, formatMessage, prepareConversationForTurn, recordConversationExchange, resolveConfirmationLexicon, resolveMessages, validateManifest, } from '@fiodos/core';
18
+ import { combineContexts, createActionExecutor, createConsentManager, createSafeTelemetry, createSanitizer, buildBubbleExchangeWindow, BUBBLE_PAGE_EXCHANGES, conversationRetentionOptions, createConversationSession, persistConversationSession, restoreConversationSession, defaultRouteContextText, formatMessage, prepareConversationForTurn, recordConversationExchange, resolveConfirmationLexicon, resolveMessages, validateManifest, } from '@fiodos/core';
19
19
  import { AgentApiError } from '../api/errors.js';
20
20
  import { formatAgentTurnError } from '../api/formatTurnError.js';
21
21
  import { buildManifestPayload } from '../api/backendClient.js';
22
22
  import { createScreenContextStore, } from '../context/screenContextStore.js';
23
23
  import { createSpeechSession } from '../speech/speechSession.js';
24
24
  import { classifyPendingReply, decideAction, } from '../core/turnEngine.js';
25
- import { DEFAULT_AGENT_TIMINGS, } from '../config/types.js';
25
+ import { DEFAULT_AGENT_TIMINGS, DEFAULT_AGENT_CHAINING, } from '../config/types.js';
26
26
  import { resolveUiMessages } from '../ui/messages.js';
27
27
  function randomSessionId() {
28
28
  return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
29
29
  }
30
+ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
30
31
  export class AgentController {
31
32
  constructor(config) {
32
33
  this.enabled = true;
@@ -55,6 +56,21 @@ export class AgentController {
55
56
  // audio), skip the extra /preview-tts round-trip and speak via device voice.
56
57
  this.serverTtsUnavailable = false;
57
58
  this.pendingConsentIntent = null;
59
+ /** Parked chain state (set when a chain pauses on a sensitive action). */
60
+ this.chainState = null;
61
+ /** REAL live activity for the panel (replaces the old canned walk-through). */
62
+ this.liveActivity = null;
63
+ this.chainActive = false;
64
+ /** How many exchanges the bubble currently mounts (grows one page at a time). */
65
+ this.bubbleWindowSize = BUBBLE_PAGE_EXCHANGES;
66
+ /** Raw dashboard retention (minutes; 0 = never; null = default). Persisted
67
+ * with the session so a restore applies the same expiry rule. */
68
+ this.conversationRetentionMinutes = null;
69
+ /** Persistence scope: internal orbs keep their memory under a separate key
70
+ * so a public orb sharing the same appId can never read it. */
71
+ this.conversationScope = 'public';
72
+ this.loadingOlderExchanges = false;
73
+ this.loadOlderTimer = null;
58
74
  this.listeners = new Set();
59
75
  this.unsubscribeSpeech = null;
60
76
  // Fail fast on an invalid manifest — same contract as every platform.
@@ -81,6 +97,13 @@ export class AgentController {
81
97
  isAuthenticated: config.isAuthenticated,
82
98
  });
83
99
  this.timings = { ...DEFAULT_AGENT_TIMINGS, ...config.timings };
100
+ // Chaining OFF by default; clamp to the same safe bounds as react/react-native.
101
+ const chainingMerged = { ...DEFAULT_AGENT_CHAINING, ...config.chaining };
102
+ this.chaining = {
103
+ enabled: chainingMerged.enabled === true,
104
+ maxSteps: Math.max(1, Math.min(6, Math.floor(chainingMerged.maxSteps || 6))),
105
+ dwellMs: Math.max(0, Math.min(3000, Math.floor(chainingMerged.dwellMs ?? 450))),
106
+ };
84
107
  this.ttsLocale = config.ttsLocale ?? config.sttLocale;
85
108
  this.notifyError =
86
109
  config.notifyError ??
@@ -105,12 +128,34 @@ export class AgentController {
105
128
  });
106
129
  // Re-emit combined state whenever the speech machine advances.
107
130
  this.unsubscribeSpeech = this.speech.subscribe(() => this.emit());
131
+ // Rehydrate the previous conversation (page reloads): the persisted blob
132
+ // carries its own retention rule, so expiry is decided correctly even
133
+ // before the published config arrives. Best-effort and non-blocking.
134
+ void this.restorePersistedConversation();
135
+ }
136
+ // ── Conversation persistence (survives page reloads) ───────────────────────
137
+ async restorePersistedConversation() {
138
+ const restored = await restoreConversationSession(this.config.storage, this.config.manifest.appId, Date.now(), this.conversationScope);
139
+ if (!restored)
140
+ return;
141
+ // Never clobber turns the user already made while the restore resolved.
142
+ if (this.conversation.turns.length > 0 || this.conversation.uiArchive?.length)
143
+ return;
144
+ this.conversation.turns = restored.turns;
145
+ this.conversation.summary = restored.summary;
146
+ this.conversation.uiArchive = restored.uiArchive;
147
+ this.conversation.lastActivityAt = restored.lastActivityAt;
148
+ this.emit();
149
+ }
150
+ persistConversation() {
151
+ void persistConversationSession(this.config.storage, this.config.manifest.appId, this.conversation, this.conversationRetentionMinutes, this.conversationScope);
108
152
  }
109
153
  // ── Subscription / state ───────────────────────────────────────────────────
110
154
  getState() {
111
155
  const speech = this.speech.getSnapshot();
112
156
  const showWaveform = (this.phase === 'listening' || this.phase === 'confirming') && speech.state === 'recording';
113
157
  const showThinking = this.phase === 'thinking' || (this.phase === 'listening' && speech.state === 'transcribing');
158
+ const bubbleWindow = buildBubbleExchangeWindow(this.conversation, this.lastExchange, this.bubbleWindowSize, this.conversationOptions);
114
159
  return {
115
160
  phase: this.phase,
116
161
  interimText: speech.interimText,
@@ -120,12 +165,32 @@ export class AgentController {
120
165
  isBusy: showThinking || this.phase === 'speaking',
121
166
  isListening: this.phase === 'listening' && speech.state !== 'idle',
122
167
  lastExchange: this.lastExchange,
168
+ recentExchanges: bubbleWindow.exchanges,
169
+ hasOlderExchanges: bubbleWindow.hasOlder,
170
+ loadingOlderExchanges: this.loadingOlderExchanges,
123
171
  confirmationMessage: this.confirmationMessage,
124
172
  confirmationVoiceHint: this.confirmationVoiceHint,
125
173
  consentModalVisible: this.consentModalVisible,
126
174
  voiceBlocked: this.voiceBlocked,
175
+ liveActivity: this.liveActivity,
176
+ chainActive: this.chainActive,
127
177
  };
128
178
  }
179
+ /** Update the REAL live activity and re-render (no-op when unchanged). */
180
+ setActivity(next) {
181
+ const prev = this.liveActivity;
182
+ if (prev === next || (prev && next && prev.kind === next.kind && prev.label === next.label)) {
183
+ return;
184
+ }
185
+ this.liveActivity = next;
186
+ this.emit();
187
+ }
188
+ setChainActive(value) {
189
+ if (this.chainActive === value)
190
+ return;
191
+ this.chainActive = value;
192
+ this.emit();
193
+ }
129
194
  /** Whether speech recognition can actually run (API present AND mic not denied). */
130
195
  get voiceAvailable() {
131
196
  return this.config.voice.isRecognitionAvailable() && !this.voiceBlocked;
@@ -144,6 +209,53 @@ export class AgentController {
144
209
  this.voiceBlocked = blocked;
145
210
  this.emit();
146
211
  }
212
+ /**
213
+ * Page one more block of older session messages into the bubble (scroll-to-top).
214
+ * The mount is deferred behind a short loading state so old messages stream in
215
+ * on demand instead of all rendering at once (keeps the open bubble light).
216
+ */
217
+ loadOlderExchanges() {
218
+ if (this.loadingOlderExchanges)
219
+ return;
220
+ const win = buildBubbleExchangeWindow(this.conversation, this.lastExchange, this.bubbleWindowSize, this.conversationOptions);
221
+ if (!win.hasOlder)
222
+ return;
223
+ this.loadingOlderExchanges = true;
224
+ this.emit();
225
+ this.loadOlderTimer = setTimeout(() => {
226
+ this.loadOlderTimer = null;
227
+ this.bubbleWindowSize += BUBBLE_PAGE_EXCHANGES;
228
+ this.loadingOlderExchanges = false;
229
+ this.emit();
230
+ }, 350);
231
+ }
232
+ /**
233
+ * Apply the dashboard's context-retention setting (minutes; 0 = never clear,
234
+ * null/undefined = the 30-minute default). Called when the published config
235
+ * arrives/refreshes, so the developer's choice is live without a reload.
236
+ * `expandedMemory` = INTERNAL orb profile: use the expanded conversation
237
+ * memory window (more literal turns + longer rolling summary per request).
238
+ */
239
+ setConversationRetention(minutes, expandedMemory = false) {
240
+ this.conversationOptions = conversationRetentionOptions(minutes, expandedMemory);
241
+ this.conversationRetentionMinutes = typeof minutes === 'number' ? minutes : null;
242
+ const scope = expandedMemory ? 'internal' : 'public';
243
+ if (scope !== this.conversationScope) {
244
+ this.conversationScope = scope;
245
+ // The published profile arrived after the constructor's restore attempt:
246
+ // retry under the right storage key (guarded — never clobbers live turns).
247
+ void this.restorePersistedConversation();
248
+ }
249
+ }
250
+ /** Collapse the bubble history back to the initial page (bubble closed). */
251
+ resetBubbleWindow() {
252
+ if (this.loadOlderTimer) {
253
+ clearTimeout(this.loadOlderTimer);
254
+ this.loadOlderTimer = null;
255
+ }
256
+ this.loadingOlderExchanges = false;
257
+ this.bubbleWindowSize = BUBBLE_PAGE_EXCHANGES;
258
+ }
147
259
  subscribe(listener) {
148
260
  this.listeners.add(listener);
149
261
  return () => {
@@ -302,6 +414,11 @@ export class AgentController {
302
414
  if (this.phase === 'confirming')
303
415
  this.setPhase('idle');
304
416
  this.emit();
417
+ // REAL activity: the confirmed action is executing right now.
418
+ this.setActivity({
419
+ kind: 'executing',
420
+ label: this.actionLabel(pending.intent) ?? pending.intent,
421
+ });
305
422
  const result = await this.executor.execute(pending.action, pending.context);
306
423
  this.telemetry.logEvent({
307
424
  eventType: result.success ? 'action_executed' : 'action_failed',
@@ -315,6 +432,39 @@ export class AgentController {
315
432
  this.emit();
316
433
  await this.speakReply(result.message, null);
317
434
  }
435
+ // RESUME the autonomous chain if this sensitive action paused one — the
436
+ // human confirmation requirement is fully preserved: the chain only
437
+ // advances past a sensitive step AFTER the user confirmed and it executed.
438
+ const chainState = this.chainState;
439
+ if (chainState) {
440
+ this.chainState = null;
441
+ if (result.success && this.chaining.enabled) {
442
+ this.telemetry.logEvent({
443
+ eventType: 'agent_chain_step',
444
+ intentDetected: pending.intent,
445
+ sessionId: this.sessionId,
446
+ chainStep: chainState.step,
447
+ });
448
+ if (chainState.pendingKey)
449
+ chainState.visitedKeys.add(chainState.pendingKey);
450
+ await this.runChainLoop({
451
+ userMessage: chainState.userMessage,
452
+ step: chainState.step,
453
+ lastStep: chainState.pendingStep ?? chainState.lastStep,
454
+ visitedKeys: chainState.visitedKeys,
455
+ });
456
+ }
457
+ else if (!result.success) {
458
+ this.telemetry.logEvent({
459
+ eventType: 'agent_chain_aborted',
460
+ sessionId: this.sessionId,
461
+ chainAbortReason: 'error',
462
+ });
463
+ }
464
+ }
465
+ // The confirmed action (and any resumed chain) finished: clear the live row.
466
+ if (!this.pending)
467
+ this.setActivity(null);
318
468
  }
319
469
  cancelPendingAction() {
320
470
  this.clearConfirmWindow();
@@ -322,9 +472,20 @@ export class AgentController {
322
472
  this.pending = null;
323
473
  this.confirmationMessage = null;
324
474
  this.confirmationVoiceHint = null;
475
+ this.setActivity(null);
325
476
  if (this.phase === 'confirming')
326
477
  this.setPhase('idle');
327
478
  this.emit();
479
+ // Declining a sensitive action stops any chain it was part of, cleanly.
480
+ const chainState = this.chainState;
481
+ this.chainState = null;
482
+ if (chainState) {
483
+ this.telemetry.logEvent({
484
+ eventType: 'agent_chain_aborted',
485
+ sessionId: this.sessionId,
486
+ chainAbortReason: 'user_interrupt',
487
+ });
488
+ }
328
489
  if (!pending)
329
490
  return;
330
491
  this.telemetry.logEvent({
@@ -368,6 +529,286 @@ export class AgentController {
368
529
  this.beginConfirmationVoiceWindow();
369
530
  }
370
531
  }
532
+ routeLabel(intent) {
533
+ return this.config.manifest.routes.find((r) => r.intent === intent)?.label;
534
+ }
535
+ actionLabel(intent) {
536
+ return this.config.manifest.actions.find((a) => a.intent === intent)?.label;
537
+ }
538
+ /**
539
+ * Resolve ONE backend turn into user-facing reply/audio: run its action (or
540
+ * stage its confirmation), emit telemetry, and report whether the autonomous
541
+ * chain may take another step. Shared by the first turn and every
542
+ * continuation so both follow the exact same security path — disabled
543
+ * capabilities, confidentiality and sensitive confirmations apply per step.
544
+ */
545
+ async applyOutcome(turn, routeAtSend, snapshotContext, userMessage) {
546
+ let outReply = turn.reply;
547
+ let outAudio = turn.audioBase64;
548
+ if (outReply)
549
+ this.serverTtsUnavailable = !outAudio;
550
+ const decision = await decideAction({
551
+ action: turn.action,
552
+ manifest: this.config.manifest,
553
+ executor: this.executor,
554
+ context: snapshotContext,
555
+ messages: this.messages,
556
+ userMessage,
557
+ });
558
+ let canContinue = false;
559
+ let lastStep;
560
+ let actionKey;
561
+ let hasPending = false;
562
+ let pendingInProgress = false;
563
+ let pendingStep;
564
+ let pendingKey;
565
+ if (decision.kind === 'executed') {
566
+ const r = decision.result;
567
+ // REAL activity for the panel: this action actually just ran — the
568
+ // label comes from the manifest, never invented.
569
+ if (r.success && turn.action?.type === 'navigate') {
570
+ this.setActivity({
571
+ kind: 'navigating',
572
+ label: this.routeLabel(turn.action.intent ?? '') ?? turn.action.intent ?? '',
573
+ });
574
+ }
575
+ else if (r.success && !r.recoverable && turn.action?.type === 'execute') {
576
+ this.setActivity({
577
+ kind: 'executing',
578
+ label: this.actionLabel(turn.action.intent ?? '') ?? turn.action.intent ?? '',
579
+ });
580
+ }
581
+ if (turn.action?.type === 'navigate') {
582
+ this.telemetry.logEvent({
583
+ eventType: r.success ? 'navigation_executed' : 'navigation_failed',
584
+ intentDetected: turn.action.intent,
585
+ sessionId: this.sessionId,
586
+ currentRoute: routeAtSend ?? undefined,
587
+ });
588
+ }
589
+ else if (turn.action?.type === 'execute') {
590
+ this.telemetry.logEvent({
591
+ eventType: r.recoverable
592
+ ? 'action_needs_context'
593
+ : r.success
594
+ ? 'action_executed'
595
+ : 'action_failed',
596
+ intentDetected: turn.action.intent,
597
+ sessionId: this.sessionId,
598
+ actionExecuted: { intent: turn.action.intent, confirmed: false },
599
+ });
600
+ }
601
+ if (r.recoverable) {
602
+ /* keep the assistant's guiding reply; never speak a refusal. */
603
+ }
604
+ else if (r.message) {
605
+ outReply = r.message;
606
+ outAudio = null;
607
+ }
608
+ // Chain signal: only a cleanly EXECUTED, non-recoverable step may advance,
609
+ // and only when the backend asked to continue (task_status=in_progress).
610
+ if (turn.action &&
611
+ (turn.action.type === 'navigate' || turn.action.type === 'execute') &&
612
+ r.success &&
613
+ !r.recoverable) {
614
+ const intent = turn.action.intent ?? '';
615
+ lastStep = {
616
+ type: turn.action.type,
617
+ intent,
618
+ label: turn.action.type === 'navigate' ? this.routeLabel(intent) : this.actionLabel(intent),
619
+ success: true,
620
+ };
621
+ actionKey = `${turn.action.type}:${intent}`;
622
+ canContinue = turn.taskStatus === 'in_progress';
623
+ }
624
+ }
625
+ else if (decision.kind === 'precheck-resolved') {
626
+ if (decision.result.message && !decision.result.recoverable) {
627
+ outReply = decision.result.message;
628
+ outAudio = null;
629
+ }
630
+ }
631
+ else if (decision.kind === 'needs-confirmation') {
632
+ // A sensitive action ALWAYS pauses (chaining or not): stage the
633
+ // confirmation and stop advancing. Resumes after the user confirms.
634
+ this.pending = { ...decision.pending, silent: this.silentTurn };
635
+ this.confirmationReasked = false;
636
+ this.confirmationMessage = decision.pending.question;
637
+ this.confirmationVoiceHint =
638
+ decision.pending.voiceMode === 'strict' &&
639
+ decision.pending.strictPhrase &&
640
+ !this.silentTurn
641
+ ? formatMessage(this.messages.strictConfirmationModalHint, {
642
+ phrase: decision.pending.strictPhrase,
643
+ })
644
+ : null;
645
+ this.emit();
646
+ this.telemetry.logEvent({
647
+ eventType: 'action_confirmation_requested',
648
+ intentDetected: decision.pending.intent,
649
+ sessionId: this.sessionId,
650
+ });
651
+ // REAL activity: the agent is genuinely waiting for the human now.
652
+ this.setActivity({ kind: 'waiting_confirmation' });
653
+ hasPending = true;
654
+ pendingInProgress = turn.taskStatus === 'in_progress';
655
+ const pIntent = decision.pending.intent || '';
656
+ pendingStep = { type: 'execute', intent: pIntent, label: this.actionLabel(pIntent), success: true };
657
+ pendingKey = `execute:${pIntent}`;
658
+ }
659
+ return {
660
+ reply: outReply,
661
+ audioBase64: outAudio,
662
+ hasPending,
663
+ canContinue,
664
+ lastStep,
665
+ actionKey,
666
+ pendingInProgress,
667
+ pendingStep,
668
+ pendingKey,
669
+ };
670
+ }
671
+ /**
672
+ * The autonomous continuation loop. Extracted so confirmPendingAction can
673
+ * RESUME the SAME chain after a sensitive step is confirmed. Capped at
674
+ * maxSteps, with no-progress detection and sensitive-confirmation pauses.
675
+ */
676
+ async runChainLoop(state) {
677
+ let step = state.step;
678
+ let lastStep = state.lastStep;
679
+ const visitedKeys = state.visitedKeys;
680
+ // Keep the panel's live row visible for the WHOLE chain (including the
681
+ // dwell between steps), so the real narration never flickers away.
682
+ this.setChainActive(true);
683
+ try {
684
+ while (step < this.chaining.maxSteps) {
685
+ step += 1;
686
+ if (this.chaining.dwellMs > 0)
687
+ await sleep(this.chaining.dwellMs);
688
+ if (this.pending != null)
689
+ return; // paused / interrupted elsewhere
690
+ const routeNow = this.config.navigation.getCurrentRoute();
691
+ const snap = this.screenStore.getSnapshot();
692
+ const rawCtx = combineContexts(this.config.getUserContextText?.() ?? null, this.screenStore.getText(), {
693
+ truncationSuffix: this.messages.contextTruncationSuffix,
694
+ maxChars: this.config.contextMaxChars,
695
+ });
696
+ const san = rawCtx ? this.sanitizer(rawCtx).sanitized : undefined;
697
+ const conv = prepareConversationForTurn(this.conversation, Date.now(), this.conversationOptions);
698
+ this.setPhase('thinking');
699
+ // Honest state: the model is choosing the next step right now.
700
+ this.setActivity({ kind: 'thinking' });
701
+ const controller = new AbortController();
702
+ this.abort = controller;
703
+ let oc;
704
+ try {
705
+ const turn = await this.config.backend.sendTurn({
706
+ message: state.userMessage,
707
+ language: this.config.locale,
708
+ voice: this.getTtsVoice(),
709
+ screenContext: san,
710
+ currentRoute: routeNow ?? undefined,
711
+ conversationHistory: conv.history,
712
+ conversationSummary: conv.summary ?? undefined,
713
+ sessionId: this.sessionId,
714
+ isContinuation: true,
715
+ chainStep: step,
716
+ lastStep,
717
+ ...buildManifestPayload(this.config.manifest),
718
+ }, { signal: controller.signal });
719
+ oc = await this.applyOutcome(turn, routeNow, snap?.context ?? {}, state.userMessage);
720
+ }
721
+ catch (e) {
722
+ this.abort = null;
723
+ const interrupted = e instanceof AgentApiError && e.code === 'cancelled';
724
+ if (interrupted) {
725
+ if (this.phase === 'thinking')
726
+ this.setPhase('idle');
727
+ }
728
+ else {
729
+ if (e instanceof AgentApiError && e.code === 'quota_exceeded') {
730
+ this.config.onQuotaExceeded?.();
731
+ }
732
+ const errMsg = formatAgentTurnError(e, this.config.locale);
733
+ this.lastExchange = { userText: state.userMessage, replyText: errMsg };
734
+ if (this.phase === 'thinking')
735
+ this.setPhase('idle');
736
+ this.emit();
737
+ this.notifyError(errMsg);
738
+ }
739
+ this.telemetry.logEvent({
740
+ eventType: 'agent_chain_aborted',
741
+ sessionId: this.sessionId,
742
+ chainStep: step,
743
+ chainAbortReason: interrupted ? 'user_interrupt' : 'error',
744
+ });
745
+ return;
746
+ }
747
+ this.abort = null;
748
+ if (oc.reply) {
749
+ this.lastExchange = { userText: state.userMessage, replyText: oc.reply };
750
+ this.emit();
751
+ recordConversationExchange(this.conversation, state.userMessage, oc.reply, Date.now(), this.conversationOptions);
752
+ this.persistConversation();
753
+ }
754
+ await this.speakReply(oc.reply, oc.audioBase64);
755
+ this.telemetry.logEvent({
756
+ eventType: 'agent_chain_step',
757
+ intentDetected: oc.lastStep?.intent ?? oc.pendingStep?.intent ?? null,
758
+ sessionId: this.sessionId,
759
+ chainStep: step,
760
+ });
761
+ // Sensitive action → ALWAYS pause for confirmation. Park the chain so
762
+ // confirmPendingAction can resume it (only if the model wants to keep
763
+ // going); a rejection stops it cleanly.
764
+ if (oc.hasPending) {
765
+ this.chainState = oc.pendingInProgress
766
+ ? {
767
+ userMessage: state.userMessage,
768
+ step,
769
+ lastStep,
770
+ visitedKeys,
771
+ pendingKey: oc.pendingKey,
772
+ pendingStep: oc.pendingStep,
773
+ }
774
+ : null;
775
+ this.beginConfirmationVoiceWindow();
776
+ return;
777
+ }
778
+ if (this.phase !== 'speaking')
779
+ this.setPhase('idle');
780
+ if (oc.actionKey && visitedKeys.has(oc.actionKey)) {
781
+ this.telemetry.logEvent({
782
+ eventType: 'agent_chain_aborted',
783
+ sessionId: this.sessionId,
784
+ chainStep: step,
785
+ chainAbortReason: 'no_progress',
786
+ });
787
+ return;
788
+ }
789
+ if (oc.actionKey)
790
+ visitedKeys.add(oc.actionKey);
791
+ if (!oc.canContinue) {
792
+ this.telemetry.logEvent({
793
+ eventType: 'agent_chain_completed',
794
+ sessionId: this.sessionId,
795
+ chainStep: step,
796
+ });
797
+ return;
798
+ }
799
+ lastStep = oc.lastStep;
800
+ }
801
+ this.telemetry.logEvent({
802
+ eventType: 'agent_chain_aborted',
803
+ sessionId: this.sessionId,
804
+ chainStep: step,
805
+ chainAbortReason: 'max_steps',
806
+ });
807
+ }
808
+ finally {
809
+ this.setChainActive(false);
810
+ }
811
+ }
371
812
  // ── Main turn ────────────────────────────────────────────────────────────────
372
813
  async handleTranscript(rawText) {
373
814
  const text = rawText.trim();
@@ -377,6 +818,8 @@ export class AgentController {
377
818
  return;
378
819
  }
379
820
  this.setPhase('thinking');
821
+ // REAL activity: the request is genuinely in flight from this moment.
822
+ this.setActivity({ kind: 'thinking' });
380
823
  this.pendingTurnText = text;
381
824
  this.lastExchange = { userText: text, replyText: '' };
382
825
  this.emit();
@@ -390,6 +833,7 @@ export class AgentController {
390
833
  : 'The response took too long. Please try again.';
391
834
  this.lastExchange = { userText, replyText };
392
835
  this.pendingTurnText = null;
836
+ this.setActivity(null);
393
837
  this.setPhase('idle');
394
838
  this.emit();
395
839
  }
@@ -410,11 +854,10 @@ export class AgentController {
410
854
  maxChars: this.config.contextMaxChars,
411
855
  });
412
856
  const sanitized = rawContext ? this.sanitizer(rawContext).sanitized : undefined;
413
- const conv = prepareConversationForTurn(this.conversation);
857
+ const conv = prepareConversationForTurn(this.conversation, Date.now(), this.conversationOptions);
414
858
  const controller = new AbortController();
415
859
  this.abort = controller;
416
- let reply = '';
417
- let audioBase64 = null;
860
+ let outcome = null;
418
861
  try {
419
862
  const turn = await this.config.backend.sendTurn({
420
863
  message: text,
@@ -427,79 +870,12 @@ export class AgentController {
427
870
  sessionId: this.sessionId,
428
871
  ...buildManifestPayload(this.config.manifest),
429
872
  }, { signal: controller.signal });
430
- reply = turn.reply;
431
- audioBase64 = turn.audioBase64;
432
- // Learn whether this backend ships server-side TTS: a non-empty reply
433
- // with no audio means it doesn't — later lines then speak instantly.
434
- if (reply)
435
- this.serverTtsUnavailable = !audioBase64;
436
- const decision = await decideAction({
437
- action: turn.action,
438
- manifest: this.config.manifest,
439
- executor: this.executor,
440
- context: snapshot?.context ?? {},
441
- messages: this.messages,
442
- userMessage: text,
443
- });
444
- if (decision.kind === 'executed') {
445
- const r = decision.result;
446
- if (turn.action?.type === 'navigate') {
447
- this.telemetry.logEvent({
448
- eventType: r.success ? 'navigation_executed' : 'navigation_failed',
449
- intentDetected: turn.action.intent,
450
- sessionId: this.sessionId,
451
- currentRoute,
452
- });
453
- }
454
- else if (turn.action?.type === 'execute') {
455
- this.telemetry.logEvent({
456
- eventType: r.recoverable
457
- ? 'action_needs_context'
458
- : r.success
459
- ? 'action_executed'
460
- : 'action_failed',
461
- intentDetected: turn.action.intent,
462
- sessionId: this.sessionId,
463
- actionExecuted: { intent: turn.action.intent, confirmed: false },
464
- });
465
- }
466
- if (r.recoverable) {
467
- /* keep the assistant's guiding reply; never speak a refusal. */
468
- }
469
- else if (r.message) {
470
- reply = r.message;
471
- audioBase64 = null;
472
- }
473
- }
474
- else if (decision.kind === 'precheck-resolved') {
475
- if (decision.result.message && !decision.result.recoverable) {
476
- reply = decision.result.message;
477
- audioBase64 = null;
478
- }
479
- }
480
- else if (decision.kind === 'needs-confirmation') {
481
- this.pending = { ...decision.pending, silent: this.silentTurn };
482
- this.confirmationReasked = false;
483
- this.confirmationMessage = decision.pending.question;
484
- this.confirmationVoiceHint =
485
- decision.pending.voiceMode === 'strict' &&
486
- decision.pending.strictPhrase &&
487
- !this.silentTurn
488
- ? formatMessage(this.messages.strictConfirmationModalHint, {
489
- phrase: decision.pending.strictPhrase,
490
- })
491
- : null;
492
- this.emit();
493
- this.telemetry.logEvent({
494
- eventType: 'action_confirmation_requested',
495
- intentDetected: decision.pending.intent,
496
- sessionId: this.sessionId,
497
- });
498
- }
873
+ outcome = await this.applyOutcome(turn, currentRoute, snapshot?.context ?? {}, text);
499
874
  }
500
875
  catch (e) {
501
876
  this.clearWatchdog();
502
877
  this.abort = null;
878
+ this.setActivity(null);
503
879
  if (e instanceof AgentApiError && e.code === 'cancelled') {
504
880
  if (this.phase === 'thinking')
505
881
  this.setPhase('idle');
@@ -521,10 +897,14 @@ export class AgentController {
521
897
  this.clearWatchdog();
522
898
  this.abort = null;
523
899
  this.pendingTurnText = null;
900
+ if (!outcome)
901
+ return;
902
+ const reply = outcome.reply;
524
903
  if (reply) {
525
904
  this.lastExchange = { userText: text, replyText: reply };
526
905
  this.emit();
527
- recordConversationExchange(this.conversation, text, reply);
906
+ recordConversationExchange(this.conversation, text, reply, Date.now(), this.conversationOptions);
907
+ this.persistConversation();
528
908
  }
529
909
  else {
530
910
  const fallback = this.config.locale.startsWith('es')
@@ -533,14 +913,53 @@ export class AgentController {
533
913
  this.lastExchange = { userText: text, replyText: fallback };
534
914
  this.emit();
535
915
  }
536
- const hasPending = this.pending != null;
537
- await this.speakReply(reply, audioBase64);
916
+ const hasPending = outcome.hasPending;
917
+ await this.speakReply(reply, outcome.audioBase64);
538
918
  if (hasPending) {
539
919
  this.beginConfirmationVoiceWindow();
920
+ // First step is a sensitive action AND the model wants to keep going:
921
+ // park the chain so it RESUMES after the user confirms.
922
+ if (this.chaining.enabled && outcome.pendingInProgress) {
923
+ this.telemetry.logEvent({
924
+ eventType: 'agent_chain_started',
925
+ sessionId: this.sessionId,
926
+ chainStep: 1,
927
+ });
928
+ this.chainState = {
929
+ userMessage: text,
930
+ step: 1,
931
+ lastStep: outcome.pendingStep,
932
+ visitedKeys: new Set(),
933
+ pendingKey: outcome.pendingKey,
934
+ pendingStep: outcome.pendingStep,
935
+ };
936
+ }
540
937
  }
541
938
  else if (this.phase !== 'speaking') {
542
939
  this.setPhase('idle');
543
940
  }
941
+ // Kick off the autonomous chain when the first step executed cleanly and
942
+ // the backend asked to continue (no sensitive confirmation pending).
943
+ if (this.chaining.enabled && outcome.canContinue && !hasPending) {
944
+ this.telemetry.logEvent({
945
+ eventType: 'agent_chain_started',
946
+ sessionId: this.sessionId,
947
+ chainStep: 1,
948
+ });
949
+ const visitedKeys = new Set();
950
+ if (outcome.actionKey)
951
+ visitedKeys.add(outcome.actionKey);
952
+ await this.runChainLoop({
953
+ userMessage: text,
954
+ step: 1,
955
+ lastStep: outcome.lastStep,
956
+ visitedKeys,
957
+ });
958
+ }
959
+ // Final reply is on screen: the live narrative disappears (unless the
960
+ // agent is genuinely waiting for a confirmation, which stays honest).
961
+ if (!this.pending)
962
+ this.setActivity(null);
544
963
  this.config.onTurnCompleted?.();
545
964
  }
546
965
  // ── Speech session wiring ────────────────────────────────────────────────────
@@ -653,6 +1072,11 @@ export class AgentController {
653
1072
  this.emit();
654
1073
  this.telemetry.logEvent({ eventType: 'agent_consent_declined', sessionId: this.sessionId });
655
1074
  }
1075
+ /**
1076
+ * Cancel everything in flight. If a first turn was still awaiting its reply,
1077
+ * its message is removed from the panel and returned here so the UI can put
1078
+ * it back into the input for editing; null otherwise.
1079
+ */
656
1080
  cancelAll() {
657
1081
  this.abort?.abort();
658
1082
  this.abort = null;
@@ -662,10 +1086,22 @@ export class AgentController {
662
1086
  this.speech.cancel();
663
1087
  this.config.voice.stopPlayback();
664
1088
  this.pending = null;
1089
+ this.chainState = null;
665
1090
  this.confirmationMessage = null;
666
1091
  this.confirmationVoiceHint = null;
1092
+ this.liveActivity = null;
1093
+ this.chainActive = false;
1094
+ // A first turn was still in flight: remove its message from the panel and
1095
+ // hand the text back for editing (the turn never completed, so nothing
1096
+ // was recorded in the conversation).
1097
+ const draft = this.pendingTurnText;
1098
+ if (draft) {
1099
+ this.pendingTurnText = null;
1100
+ this.lastExchange = null;
1101
+ }
667
1102
  this.setPhase('idle');
668
1103
  this.emit();
1104
+ return draft;
669
1105
  }
670
1106
  /** Best-effort backend warm-up (cold-start mitigation). */
671
1107
  warmUp() {
@@ -673,6 +1109,10 @@ export class AgentController {
673
1109
  }
674
1110
  dispose() {
675
1111
  this.cancelAll();
1112
+ if (this.loadOlderTimer) {
1113
+ clearTimeout(this.loadOlderTimer);
1114
+ this.loadOlderTimer = null;
1115
+ }
676
1116
  this.unsubscribeSpeech?.();
677
1117
  this.unsubscribeSpeech = null;
678
1118
  this.speech.dispose();