@fiodos/web-core 0.1.10 → 0.1.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/cjs/api/backendClient.js +20 -0
  2. package/dist/cjs/config/types.d.ts +24 -0
  3. package/dist/cjs/config/types.js +6 -1
  4. package/dist/cjs/controller/AgentController.d.ts +105 -2
  5. package/dist/cjs/controller/AgentController.js +572 -75
  6. package/dist/cjs/dropin/createFiodosAgent.d.ts +5 -0
  7. package/dist/cjs/dropin/createFiodosAgent.js +7 -0
  8. package/dist/cjs/index.d.ts +3 -3
  9. package/dist/cjs/index.js +2 -1
  10. package/dist/cjs/orb/mountOrb.d.ts +6 -0
  11. package/dist/cjs/orb/mountOrb.js +229 -33
  12. package/dist/cjs/orb/orbView.d.ts +2 -0
  13. package/dist/cjs/orb/publishedConfig.d.ts +18 -0
  14. package/dist/cjs/orb/publishedConfig.js +6 -0
  15. package/dist/cjs/speech/bubbleDictation.d.ts +11 -0
  16. package/dist/cjs/speech/bubbleDictation.js +88 -0
  17. package/dist/cjs/ui/messages.d.ts +15 -0
  18. package/dist/cjs/ui/messages.js +12 -0
  19. package/dist/cjs/version.d.ts +1 -1
  20. package/dist/cjs/version.js +1 -1
  21. package/dist/esm/api/backendClient.js +20 -0
  22. package/dist/esm/config/types.d.ts +24 -0
  23. package/dist/esm/config/types.js +5 -0
  24. package/dist/esm/controller/AgentController.d.ts +105 -2
  25. package/dist/esm/controller/AgentController.js +574 -77
  26. package/dist/esm/dropin/createFiodosAgent.d.ts +5 -0
  27. package/dist/esm/dropin/createFiodosAgent.js +7 -0
  28. package/dist/esm/index.d.ts +3 -3
  29. package/dist/esm/index.js +1 -1
  30. package/dist/esm/orb/mountOrb.d.ts +6 -0
  31. package/dist/esm/orb/mountOrb.js +230 -34
  32. package/dist/esm/orb/orbView.d.ts +2 -0
  33. package/dist/esm/orb/publishedConfig.d.ts +18 -0
  34. package/dist/esm/orb/publishedConfig.js +6 -0
  35. package/dist/esm/speech/bubbleDictation.d.ts +11 -0
  36. package/dist/esm/speech/bubbleDictation.js +84 -0
  37. package/dist/esm/ui/messages.d.ts +15 -0
  38. package/dist/esm/ui/messages.js +12 -0
  39. package/dist/esm/version.d.ts +1 -1
  40. package/dist/esm/version.js +1 -1
  41. package/package.json +2 -2
@@ -15,18 +15,19 @@
15
15
  * deliberate modal tap; a loose "yes" never confirms them.
16
16
  * - The agent only ever executes actions declared in the manifest.
17
17
  */
18
- import { combineContexts, createActionExecutor, createConsentManager, createSafeTelemetry, createSanitizer, createConversationSession, defaultRouteContextText, formatMessage, prepareConversationForTurn, recordConversationExchange, resolveConfirmationLexicon, resolveMessages, validateManifest, } from '@fiodos/core';
18
+ import { combineContexts, createActionExecutor, createConsentManager, createSafeTelemetry, createSanitizer, buildBubbleExchangeWindow, BUBBLE_PAGE_EXCHANGES, clearPersistedConversation, conversationIdentityKey, conversationRetentionOptions, createConversationSession, persistConversationSession, restoreConversationSession, defaultRouteContextText, formatMessage, prepareConversationForTurn, recordConversationExchange, resolveConfirmationLexicon, resolveMessages, validateManifest, } from '@fiodos/core';
19
19
  import { AgentApiError } from '../api/errors.js';
20
20
  import { formatAgentTurnError } from '../api/formatTurnError.js';
21
21
  import { buildManifestPayload } from '../api/backendClient.js';
22
22
  import { createScreenContextStore, } from '../context/screenContextStore.js';
23
23
  import { createSpeechSession } from '../speech/speechSession.js';
24
24
  import { classifyPendingReply, decideAction, } from '../core/turnEngine.js';
25
- import { DEFAULT_AGENT_TIMINGS, } from '../config/types.js';
25
+ import { DEFAULT_AGENT_TIMINGS, DEFAULT_AGENT_CHAINING, } from '../config/types.js';
26
26
  import { resolveUiMessages } from '../ui/messages.js';
27
27
  function randomSessionId() {
28
28
  return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
29
29
  }
30
+ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
30
31
  export class AgentController {
31
32
  constructor(config) {
32
33
  this.enabled = true;
@@ -39,6 +40,7 @@ export class AgentController {
39
40
  this.consentModalVisible = false;
40
41
  this.sessionId = randomSessionId();
41
42
  this.sessionStarted = false;
43
+ // Replaced wholesale on identity swaps / explicit resets.
42
44
  this.conversation = createConversationSession();
43
45
  this.pending = null;
44
46
  this.confirmationReasked = false;
@@ -55,6 +57,22 @@ export class AgentController {
55
57
  // audio), skip the extra /preview-tts round-trip and speak via device voice.
56
58
  this.serverTtsUnavailable = false;
57
59
  this.pendingConsentIntent = null;
60
+ /** Parked chain state (set when a chain pauses on a sensitive action). */
61
+ this.chainState = null;
62
+ /** REAL live activity for the panel (replaces the old canned walk-through). */
63
+ this.liveActivity = null;
64
+ this.chainActive = false;
65
+ /** How many exchanges the bubble currently mounts (grows one page at a time). */
66
+ this.bubbleWindowSize = BUBBLE_PAGE_EXCHANGES;
67
+ /** Raw dashboard retention (minutes; 0 = never; null = default). Persisted
68
+ * with the session so a restore applies the same expiry rule. */
69
+ this.conversationRetentionMinutes = null;
70
+ /** Persistence scope: internal orbs keep their memory under a separate key
71
+ * so a public orb sharing the same appId can never read it. */
72
+ this.conversationScope = 'public';
73
+ this.removeIdentityListeners = null;
74
+ this.loadingOlderExchanges = false;
75
+ this.loadOlderTimer = null;
58
76
  this.listeners = new Set();
59
77
  this.unsubscribeSpeech = null;
60
78
  // Fail fast on an invalid manifest — same contract as every platform.
@@ -81,6 +99,13 @@ export class AgentController {
81
99
  isAuthenticated: config.isAuthenticated,
82
100
  });
83
101
  this.timings = { ...DEFAULT_AGENT_TIMINGS, ...config.timings };
102
+ // Chaining OFF by default; clamp to the same safe bounds as react/react-native.
103
+ const chainingMerged = { ...DEFAULT_AGENT_CHAINING, ...config.chaining };
104
+ this.chaining = {
105
+ enabled: chainingMerged.enabled === true,
106
+ maxSteps: Math.max(1, Math.min(6, Math.floor(chainingMerged.maxSteps || 6))),
107
+ dwellMs: Math.max(0, Math.min(3000, Math.floor(chainingMerged.dwellMs ?? 450))),
108
+ };
84
109
  this.ttsLocale = config.ttsLocale ?? config.sttLocale;
85
110
  this.notifyError =
86
111
  config.notifyError ??
@@ -105,12 +130,84 @@ export class AgentController {
105
130
  });
106
131
  // Re-emit combined state whenever the speech machine advances.
107
132
  this.unsubscribeSpeech = this.speech.subscribe(() => this.emit());
133
+ this.conversationIdentity = conversationIdentityKey(config.getUserId?.());
134
+ // Rehydrate the previous conversation (page reloads): the persisted blob
135
+ // carries its own retention rule, so expiry is decided correctly even
136
+ // before the published config arrives. Best-effort and non-blocking.
137
+ void this.restorePersistedConversation();
138
+ // Logout/login usually happens on another screen: re-check identity when
139
+ // the tab regains focus or becomes visible again, besides before every turn.
140
+ if (typeof window !== 'undefined') {
141
+ const check = () => void this.syncConversationIdentity();
142
+ window.addEventListener('focus', check);
143
+ document.addEventListener('visibilitychange', check);
144
+ this.removeIdentityListeners = () => {
145
+ window.removeEventListener('focus', check);
146
+ document.removeEventListener('visibilitychange', check);
147
+ };
148
+ }
149
+ }
150
+ // ── Conversation persistence (survives page reloads) ───────────────────────
151
+ async restorePersistedConversation() {
152
+ const identityAtStart = this.conversationIdentity;
153
+ const restored = await restoreConversationSession(this.config.storage, this.config.manifest.appId, Date.now(), this.conversationScope, identityAtStart);
154
+ // The identity may have changed again while the read resolved.
155
+ if (!restored || this.conversationIdentity !== identityAtStart)
156
+ return;
157
+ // Never clobber turns the user already made while the restore resolved.
158
+ if (this.conversation.turns.length > 0 || this.conversation.uiArchive?.length)
159
+ return;
160
+ this.conversation.turns = restored.turns;
161
+ this.conversation.summary = restored.summary;
162
+ this.conversation.uiArchive = restored.uiArchive;
163
+ this.conversation.lastActivityAt = restored.lastActivityAt;
164
+ this.emit();
165
+ }
166
+ persistConversation() {
167
+ void persistConversationSession(this.config.storage, this.config.manifest.appId, this.conversation, this.conversationRetentionMinutes, this.conversationScope, this.conversationIdentity);
168
+ }
169
+ /**
170
+ * Re-evaluates the host identity and, when it changed, swaps conversations:
171
+ * the outgoing identity's session is saved under ITS private key (so a user
172
+ * who logs back in recovers it, honouring "never clear"), all in-memory
173
+ * state and UI are wiped, and the incoming identity's own session (usually
174
+ * empty) is restored. An anonymous visitor after a logout always starts
175
+ * clean — the previous user's transcript is never inherited.
176
+ */
177
+ syncConversationIdentity() {
178
+ const next = conversationIdentityKey(this.config.getUserId?.());
179
+ if (next === this.conversationIdentity)
180
+ return false;
181
+ // 1. Park the outgoing identity's session under its own key.
182
+ this.persistConversation();
183
+ // 2. Wipe every trace from memory and the panel.
184
+ this.conversation = createConversationSession();
185
+ this.conversationIdentity = next;
186
+ this.lastExchange = null;
187
+ this.emit();
188
+ // 3. Bring in the new identity's own session, if any.
189
+ void this.restorePersistedConversation();
190
+ return true;
191
+ }
192
+ /**
193
+ * Wipes the CURRENT identity's conversation: live memory, panel and the
194
+ * persisted blob. Hosts can call it from their logout for a hard wipe (by
195
+ * default a logout only parks the conversation under the user's private
196
+ * local key).
197
+ */
198
+ resetConversation() {
199
+ this.cancelAll();
200
+ this.conversation = createConversationSession();
201
+ this.lastExchange = null;
202
+ void clearPersistedConversation(this.config.storage, this.config.manifest.appId, this.conversationScope, this.conversationIdentity);
203
+ this.emit();
108
204
  }
109
205
  // ── Subscription / state ───────────────────────────────────────────────────
110
206
  getState() {
111
207
  const speech = this.speech.getSnapshot();
112
208
  const showWaveform = (this.phase === 'listening' || this.phase === 'confirming') && speech.state === 'recording';
113
209
  const showThinking = this.phase === 'thinking' || (this.phase === 'listening' && speech.state === 'transcribing');
210
+ const bubbleWindow = buildBubbleExchangeWindow(this.conversation, this.lastExchange, this.bubbleWindowSize, this.conversationOptions);
114
211
  return {
115
212
  phase: this.phase,
116
213
  interimText: speech.interimText,
@@ -120,12 +217,32 @@ export class AgentController {
120
217
  isBusy: showThinking || this.phase === 'speaking',
121
218
  isListening: this.phase === 'listening' && speech.state !== 'idle',
122
219
  lastExchange: this.lastExchange,
220
+ recentExchanges: bubbleWindow.exchanges,
221
+ hasOlderExchanges: bubbleWindow.hasOlder,
222
+ loadingOlderExchanges: this.loadingOlderExchanges,
123
223
  confirmationMessage: this.confirmationMessage,
124
224
  confirmationVoiceHint: this.confirmationVoiceHint,
125
225
  consentModalVisible: this.consentModalVisible,
126
226
  voiceBlocked: this.voiceBlocked,
227
+ liveActivity: this.liveActivity,
228
+ chainActive: this.chainActive,
127
229
  };
128
230
  }
231
+ /** Update the REAL live activity and re-render (no-op when unchanged). */
232
+ setActivity(next) {
233
+ const prev = this.liveActivity;
234
+ if (prev === next || (prev && next && prev.kind === next.kind && prev.label === next.label)) {
235
+ return;
236
+ }
237
+ this.liveActivity = next;
238
+ this.emit();
239
+ }
240
+ setChainActive(value) {
241
+ if (this.chainActive === value)
242
+ return;
243
+ this.chainActive = value;
244
+ this.emit();
245
+ }
129
246
  /** Whether speech recognition can actually run (API present AND mic not denied). */
130
247
  get voiceAvailable() {
131
248
  return this.config.voice.isRecognitionAvailable() && !this.voiceBlocked;
@@ -144,6 +261,53 @@ export class AgentController {
144
261
  this.voiceBlocked = blocked;
145
262
  this.emit();
146
263
  }
264
+ /**
265
+ * Page one more block of older session messages into the bubble (scroll-to-top).
266
+ * The mount is deferred behind a short loading state so old messages stream in
267
+ * on demand instead of all rendering at once (keeps the open bubble light).
268
+ */
269
+ loadOlderExchanges() {
270
+ if (this.loadingOlderExchanges)
271
+ return;
272
+ const win = buildBubbleExchangeWindow(this.conversation, this.lastExchange, this.bubbleWindowSize, this.conversationOptions);
273
+ if (!win.hasOlder)
274
+ return;
275
+ this.loadingOlderExchanges = true;
276
+ this.emit();
277
+ this.loadOlderTimer = setTimeout(() => {
278
+ this.loadOlderTimer = null;
279
+ this.bubbleWindowSize += BUBBLE_PAGE_EXCHANGES;
280
+ this.loadingOlderExchanges = false;
281
+ this.emit();
282
+ }, 350);
283
+ }
284
+ /**
285
+ * Apply the dashboard's context-retention setting (minutes; 0 = never clear,
286
+ * null/undefined = the 30-minute default). Called when the published config
287
+ * arrives/refreshes, so the developer's choice is live without a reload.
288
+ * `expandedMemory` = INTERNAL orb profile: use the expanded conversation
289
+ * memory window (more literal turns + longer rolling summary per request).
290
+ */
291
+ setConversationRetention(minutes, expandedMemory = false) {
292
+ this.conversationOptions = conversationRetentionOptions(minutes, expandedMemory);
293
+ this.conversationRetentionMinutes = typeof minutes === 'number' ? minutes : null;
294
+ const scope = expandedMemory ? 'internal' : 'public';
295
+ if (scope !== this.conversationScope) {
296
+ this.conversationScope = scope;
297
+ // The published profile arrived after the constructor's restore attempt:
298
+ // retry under the right storage key (guarded — never clobbers live turns).
299
+ void this.restorePersistedConversation();
300
+ }
301
+ }
302
+ /** Collapse the bubble history back to the initial page (bubble closed). */
303
+ resetBubbleWindow() {
304
+ if (this.loadOlderTimer) {
305
+ clearTimeout(this.loadOlderTimer);
306
+ this.loadOlderTimer = null;
307
+ }
308
+ this.loadingOlderExchanges = false;
309
+ this.bubbleWindowSize = BUBBLE_PAGE_EXCHANGES;
310
+ }
147
311
  subscribe(listener) {
148
312
  this.listeners.add(listener);
149
313
  return () => {
@@ -302,6 +466,11 @@ export class AgentController {
302
466
  if (this.phase === 'confirming')
303
467
  this.setPhase('idle');
304
468
  this.emit();
469
+ // REAL activity: the confirmed action is executing right now.
470
+ this.setActivity({
471
+ kind: 'executing',
472
+ label: this.actionLabel(pending.intent) ?? pending.intent,
473
+ });
305
474
  const result = await this.executor.execute(pending.action, pending.context);
306
475
  this.telemetry.logEvent({
307
476
  eventType: result.success ? 'action_executed' : 'action_failed',
@@ -315,6 +484,39 @@ export class AgentController {
315
484
  this.emit();
316
485
  await this.speakReply(result.message, null);
317
486
  }
487
+ // RESUME the autonomous chain if this sensitive action paused one — the
488
+ // human confirmation requirement is fully preserved: the chain only
489
+ // advances past a sensitive step AFTER the user confirmed and it executed.
490
+ const chainState = this.chainState;
491
+ if (chainState) {
492
+ this.chainState = null;
493
+ if (result.success && this.chaining.enabled) {
494
+ this.telemetry.logEvent({
495
+ eventType: 'agent_chain_step',
496
+ intentDetected: pending.intent,
497
+ sessionId: this.sessionId,
498
+ chainStep: chainState.step,
499
+ });
500
+ if (chainState.pendingKey)
501
+ chainState.visitedKeys.add(chainState.pendingKey);
502
+ await this.runChainLoop({
503
+ userMessage: chainState.userMessage,
504
+ step: chainState.step,
505
+ lastStep: chainState.pendingStep ?? chainState.lastStep,
506
+ visitedKeys: chainState.visitedKeys,
507
+ });
508
+ }
509
+ else if (!result.success) {
510
+ this.telemetry.logEvent({
511
+ eventType: 'agent_chain_aborted',
512
+ sessionId: this.sessionId,
513
+ chainAbortReason: 'error',
514
+ });
515
+ }
516
+ }
517
+ // The confirmed action (and any resumed chain) finished: clear the live row.
518
+ if (!this.pending)
519
+ this.setActivity(null);
318
520
  }
319
521
  cancelPendingAction() {
320
522
  this.clearConfirmWindow();
@@ -322,9 +524,20 @@ export class AgentController {
322
524
  this.pending = null;
323
525
  this.confirmationMessage = null;
324
526
  this.confirmationVoiceHint = null;
527
+ this.setActivity(null);
325
528
  if (this.phase === 'confirming')
326
529
  this.setPhase('idle');
327
530
  this.emit();
531
+ // Declining a sensitive action stops any chain it was part of, cleanly.
532
+ const chainState = this.chainState;
533
+ this.chainState = null;
534
+ if (chainState) {
535
+ this.telemetry.logEvent({
536
+ eventType: 'agent_chain_aborted',
537
+ sessionId: this.sessionId,
538
+ chainAbortReason: 'user_interrupt',
539
+ });
540
+ }
328
541
  if (!pending)
329
542
  return;
330
543
  this.telemetry.logEvent({
@@ -368,6 +581,286 @@ export class AgentController {
368
581
  this.beginConfirmationVoiceWindow();
369
582
  }
370
583
  }
584
+ routeLabel(intent) {
585
+ return this.config.manifest.routes.find((r) => r.intent === intent)?.label;
586
+ }
587
+ actionLabel(intent) {
588
+ return this.config.manifest.actions.find((a) => a.intent === intent)?.label;
589
+ }
590
+ /**
591
+ * Resolve ONE backend turn into user-facing reply/audio: run its action (or
592
+ * stage its confirmation), emit telemetry, and report whether the autonomous
593
+ * chain may take another step. Shared by the first turn and every
594
+ * continuation so both follow the exact same security path — disabled
595
+ * capabilities, confidentiality and sensitive confirmations apply per step.
596
+ */
597
+ async applyOutcome(turn, routeAtSend, snapshotContext, userMessage) {
598
+ let outReply = turn.reply;
599
+ let outAudio = turn.audioBase64;
600
+ if (outReply)
601
+ this.serverTtsUnavailable = !outAudio;
602
+ const decision = await decideAction({
603
+ action: turn.action,
604
+ manifest: this.config.manifest,
605
+ executor: this.executor,
606
+ context: snapshotContext,
607
+ messages: this.messages,
608
+ userMessage,
609
+ });
610
+ let canContinue = false;
611
+ let lastStep;
612
+ let actionKey;
613
+ let hasPending = false;
614
+ let pendingInProgress = false;
615
+ let pendingStep;
616
+ let pendingKey;
617
+ if (decision.kind === 'executed') {
618
+ const r = decision.result;
619
+ // REAL activity for the panel: this action actually just ran — the
620
+ // label comes from the manifest, never invented.
621
+ if (r.success && turn.action?.type === 'navigate') {
622
+ this.setActivity({
623
+ kind: 'navigating',
624
+ label: this.routeLabel(turn.action.intent ?? '') ?? turn.action.intent ?? '',
625
+ });
626
+ }
627
+ else if (r.success && !r.recoverable && turn.action?.type === 'execute') {
628
+ this.setActivity({
629
+ kind: 'executing',
630
+ label: this.actionLabel(turn.action.intent ?? '') ?? turn.action.intent ?? '',
631
+ });
632
+ }
633
+ if (turn.action?.type === 'navigate') {
634
+ this.telemetry.logEvent({
635
+ eventType: r.success ? 'navigation_executed' : 'navigation_failed',
636
+ intentDetected: turn.action.intent,
637
+ sessionId: this.sessionId,
638
+ currentRoute: routeAtSend ?? undefined,
639
+ });
640
+ }
641
+ else if (turn.action?.type === 'execute') {
642
+ this.telemetry.logEvent({
643
+ eventType: r.recoverable
644
+ ? 'action_needs_context'
645
+ : r.success
646
+ ? 'action_executed'
647
+ : 'action_failed',
648
+ intentDetected: turn.action.intent,
649
+ sessionId: this.sessionId,
650
+ actionExecuted: { intent: turn.action.intent, confirmed: false },
651
+ });
652
+ }
653
+ if (r.recoverable) {
654
+ /* keep the assistant's guiding reply; never speak a refusal. */
655
+ }
656
+ else if (r.message) {
657
+ outReply = r.message;
658
+ outAudio = null;
659
+ }
660
+ // Chain signal: only a cleanly EXECUTED, non-recoverable step may advance,
661
+ // and only when the backend asked to continue (task_status=in_progress).
662
+ if (turn.action &&
663
+ (turn.action.type === 'navigate' || turn.action.type === 'execute') &&
664
+ r.success &&
665
+ !r.recoverable) {
666
+ const intent = turn.action.intent ?? '';
667
+ lastStep = {
668
+ type: turn.action.type,
669
+ intent,
670
+ label: turn.action.type === 'navigate' ? this.routeLabel(intent) : this.actionLabel(intent),
671
+ success: true,
672
+ };
673
+ actionKey = `${turn.action.type}:${intent}`;
674
+ canContinue = turn.taskStatus === 'in_progress';
675
+ }
676
+ }
677
+ else if (decision.kind === 'precheck-resolved') {
678
+ if (decision.result.message && !decision.result.recoverable) {
679
+ outReply = decision.result.message;
680
+ outAudio = null;
681
+ }
682
+ }
683
+ else if (decision.kind === 'needs-confirmation') {
684
+ // A sensitive action ALWAYS pauses (chaining or not): stage the
685
+ // confirmation and stop advancing. Resumes after the user confirms.
686
+ this.pending = { ...decision.pending, silent: this.silentTurn };
687
+ this.confirmationReasked = false;
688
+ this.confirmationMessage = decision.pending.question;
689
+ this.confirmationVoiceHint =
690
+ decision.pending.voiceMode === 'strict' &&
691
+ decision.pending.strictPhrase &&
692
+ !this.silentTurn
693
+ ? formatMessage(this.messages.strictConfirmationModalHint, {
694
+ phrase: decision.pending.strictPhrase,
695
+ })
696
+ : null;
697
+ this.emit();
698
+ this.telemetry.logEvent({
699
+ eventType: 'action_confirmation_requested',
700
+ intentDetected: decision.pending.intent,
701
+ sessionId: this.sessionId,
702
+ });
703
+ // REAL activity: the agent is genuinely waiting for the human now.
704
+ this.setActivity({ kind: 'waiting_confirmation' });
705
+ hasPending = true;
706
+ pendingInProgress = turn.taskStatus === 'in_progress';
707
+ const pIntent = decision.pending.intent || '';
708
+ pendingStep = { type: 'execute', intent: pIntent, label: this.actionLabel(pIntent), success: true };
709
+ pendingKey = `execute:${pIntent}`;
710
+ }
711
+ return {
712
+ reply: outReply,
713
+ audioBase64: outAudio,
714
+ hasPending,
715
+ canContinue,
716
+ lastStep,
717
+ actionKey,
718
+ pendingInProgress,
719
+ pendingStep,
720
+ pendingKey,
721
+ };
722
+ }
723
+ /**
724
+ * The autonomous continuation loop. Extracted so confirmPendingAction can
725
+ * RESUME the SAME chain after a sensitive step is confirmed. Capped at
726
+ * maxSteps, with no-progress detection and sensitive-confirmation pauses.
727
+ */
728
+ async runChainLoop(state) {
729
+ let step = state.step;
730
+ let lastStep = state.lastStep;
731
+ const visitedKeys = state.visitedKeys;
732
+ // Keep the panel's live row visible for the WHOLE chain (including the
733
+ // dwell between steps), so the real narration never flickers away.
734
+ this.setChainActive(true);
735
+ try {
736
+ while (step < this.chaining.maxSteps) {
737
+ step += 1;
738
+ if (this.chaining.dwellMs > 0)
739
+ await sleep(this.chaining.dwellMs);
740
+ if (this.pending != null)
741
+ return; // paused / interrupted elsewhere
742
+ const routeNow = this.config.navigation.getCurrentRoute();
743
+ const snap = this.screenStore.getSnapshot();
744
+ const rawCtx = combineContexts(this.config.getUserContextText?.() ?? null, this.screenStore.getText(), {
745
+ truncationSuffix: this.messages.contextTruncationSuffix,
746
+ maxChars: this.config.contextMaxChars,
747
+ });
748
+ const san = rawCtx ? this.sanitizer(rawCtx).sanitized : undefined;
749
+ const conv = prepareConversationForTurn(this.conversation, Date.now(), this.conversationOptions);
750
+ this.setPhase('thinking');
751
+ // Honest state: the model is choosing the next step right now.
752
+ this.setActivity({ kind: 'thinking' });
753
+ const controller = new AbortController();
754
+ this.abort = controller;
755
+ let oc;
756
+ try {
757
+ const turn = await this.config.backend.sendTurn({
758
+ message: state.userMessage,
759
+ language: this.config.locale,
760
+ voice: this.getTtsVoice(),
761
+ screenContext: san,
762
+ currentRoute: routeNow ?? undefined,
763
+ conversationHistory: conv.history,
764
+ conversationSummary: conv.summary ?? undefined,
765
+ sessionId: this.sessionId,
766
+ isContinuation: true,
767
+ chainStep: step,
768
+ lastStep,
769
+ ...buildManifestPayload(this.config.manifest),
770
+ }, { signal: controller.signal });
771
+ oc = await this.applyOutcome(turn, routeNow, snap?.context ?? {}, state.userMessage);
772
+ }
773
+ catch (e) {
774
+ this.abort = null;
775
+ const interrupted = e instanceof AgentApiError && e.code === 'cancelled';
776
+ if (interrupted) {
777
+ if (this.phase === 'thinking')
778
+ this.setPhase('idle');
779
+ }
780
+ else {
781
+ if (e instanceof AgentApiError && e.code === 'quota_exceeded') {
782
+ this.config.onQuotaExceeded?.();
783
+ }
784
+ const errMsg = formatAgentTurnError(e, this.config.locale);
785
+ this.lastExchange = { userText: state.userMessage, replyText: errMsg };
786
+ if (this.phase === 'thinking')
787
+ this.setPhase('idle');
788
+ this.emit();
789
+ this.notifyError(errMsg);
790
+ }
791
+ this.telemetry.logEvent({
792
+ eventType: 'agent_chain_aborted',
793
+ sessionId: this.sessionId,
794
+ chainStep: step,
795
+ chainAbortReason: interrupted ? 'user_interrupt' : 'error',
796
+ });
797
+ return;
798
+ }
799
+ this.abort = null;
800
+ if (oc.reply) {
801
+ this.lastExchange = { userText: state.userMessage, replyText: oc.reply };
802
+ this.emit();
803
+ recordConversationExchange(this.conversation, state.userMessage, oc.reply, Date.now(), this.conversationOptions);
804
+ this.persistConversation();
805
+ }
806
+ await this.speakReply(oc.reply, oc.audioBase64);
807
+ this.telemetry.logEvent({
808
+ eventType: 'agent_chain_step',
809
+ intentDetected: oc.lastStep?.intent ?? oc.pendingStep?.intent ?? null,
810
+ sessionId: this.sessionId,
811
+ chainStep: step,
812
+ });
813
+ // Sensitive action → ALWAYS pause for confirmation. Park the chain so
814
+ // confirmPendingAction can resume it (only if the model wants to keep
815
+ // going); a rejection stops it cleanly.
816
+ if (oc.hasPending) {
817
+ this.chainState = oc.pendingInProgress
818
+ ? {
819
+ userMessage: state.userMessage,
820
+ step,
821
+ lastStep,
822
+ visitedKeys,
823
+ pendingKey: oc.pendingKey,
824
+ pendingStep: oc.pendingStep,
825
+ }
826
+ : null;
827
+ this.beginConfirmationVoiceWindow();
828
+ return;
829
+ }
830
+ if (this.phase !== 'speaking')
831
+ this.setPhase('idle');
832
+ if (oc.actionKey && visitedKeys.has(oc.actionKey)) {
833
+ this.telemetry.logEvent({
834
+ eventType: 'agent_chain_aborted',
835
+ sessionId: this.sessionId,
836
+ chainStep: step,
837
+ chainAbortReason: 'no_progress',
838
+ });
839
+ return;
840
+ }
841
+ if (oc.actionKey)
842
+ visitedKeys.add(oc.actionKey);
843
+ if (!oc.canContinue) {
844
+ this.telemetry.logEvent({
845
+ eventType: 'agent_chain_completed',
846
+ sessionId: this.sessionId,
847
+ chainStep: step,
848
+ });
849
+ return;
850
+ }
851
+ lastStep = oc.lastStep;
852
+ }
853
+ this.telemetry.logEvent({
854
+ eventType: 'agent_chain_aborted',
855
+ sessionId: this.sessionId,
856
+ chainStep: step,
857
+ chainAbortReason: 'max_steps',
858
+ });
859
+ }
860
+ finally {
861
+ this.setChainActive(false);
862
+ }
863
+ }
371
864
  // ── Main turn ────────────────────────────────────────────────────────────────
372
865
  async handleTranscript(rawText) {
373
866
  const text = rawText.trim();
@@ -376,7 +869,12 @@ export class AgentController {
376
869
  this.setPhase('idle');
377
870
  return;
378
871
  }
872
+ // Identity gate: if the host user changed since the last turn (login,
873
+ // logout, account switch), swap conversations BEFORE recording anything.
874
+ this.syncConversationIdentity();
379
875
  this.setPhase('thinking');
876
+ // REAL activity: the request is genuinely in flight from this moment.
877
+ this.setActivity({ kind: 'thinking' });
380
878
  this.pendingTurnText = text;
381
879
  this.lastExchange = { userText: text, replyText: '' };
382
880
  this.emit();
@@ -390,6 +888,7 @@ export class AgentController {
390
888
  : 'The response took too long. Please try again.';
391
889
  this.lastExchange = { userText, replyText };
392
890
  this.pendingTurnText = null;
891
+ this.setActivity(null);
393
892
  this.setPhase('idle');
394
893
  this.emit();
395
894
  }
@@ -410,11 +909,10 @@ export class AgentController {
410
909
  maxChars: this.config.contextMaxChars,
411
910
  });
412
911
  const sanitized = rawContext ? this.sanitizer(rawContext).sanitized : undefined;
413
- const conv = prepareConversationForTurn(this.conversation);
912
+ const conv = prepareConversationForTurn(this.conversation, Date.now(), this.conversationOptions);
414
913
  const controller = new AbortController();
415
914
  this.abort = controller;
416
- let reply = '';
417
- let audioBase64 = null;
915
+ let outcome = null;
418
916
  try {
419
917
  const turn = await this.config.backend.sendTurn({
420
918
  message: text,
@@ -427,79 +925,12 @@ export class AgentController {
427
925
  sessionId: this.sessionId,
428
926
  ...buildManifestPayload(this.config.manifest),
429
927
  }, { signal: controller.signal });
430
- reply = turn.reply;
431
- audioBase64 = turn.audioBase64;
432
- // Learn whether this backend ships server-side TTS: a non-empty reply
433
- // with no audio means it doesn't — later lines then speak instantly.
434
- if (reply)
435
- this.serverTtsUnavailable = !audioBase64;
436
- const decision = await decideAction({
437
- action: turn.action,
438
- manifest: this.config.manifest,
439
- executor: this.executor,
440
- context: snapshot?.context ?? {},
441
- messages: this.messages,
442
- userMessage: text,
443
- });
444
- if (decision.kind === 'executed') {
445
- const r = decision.result;
446
- if (turn.action?.type === 'navigate') {
447
- this.telemetry.logEvent({
448
- eventType: r.success ? 'navigation_executed' : 'navigation_failed',
449
- intentDetected: turn.action.intent,
450
- sessionId: this.sessionId,
451
- currentRoute,
452
- });
453
- }
454
- else if (turn.action?.type === 'execute') {
455
- this.telemetry.logEvent({
456
- eventType: r.recoverable
457
- ? 'action_needs_context'
458
- : r.success
459
- ? 'action_executed'
460
- : 'action_failed',
461
- intentDetected: turn.action.intent,
462
- sessionId: this.sessionId,
463
- actionExecuted: { intent: turn.action.intent, confirmed: false },
464
- });
465
- }
466
- if (r.recoverable) {
467
- /* keep the assistant's guiding reply; never speak a refusal. */
468
- }
469
- else if (r.message) {
470
- reply = r.message;
471
- audioBase64 = null;
472
- }
473
- }
474
- else if (decision.kind === 'precheck-resolved') {
475
- if (decision.result.message && !decision.result.recoverable) {
476
- reply = decision.result.message;
477
- audioBase64 = null;
478
- }
479
- }
480
- else if (decision.kind === 'needs-confirmation') {
481
- this.pending = { ...decision.pending, silent: this.silentTurn };
482
- this.confirmationReasked = false;
483
- this.confirmationMessage = decision.pending.question;
484
- this.confirmationVoiceHint =
485
- decision.pending.voiceMode === 'strict' &&
486
- decision.pending.strictPhrase &&
487
- !this.silentTurn
488
- ? formatMessage(this.messages.strictConfirmationModalHint, {
489
- phrase: decision.pending.strictPhrase,
490
- })
491
- : null;
492
- this.emit();
493
- this.telemetry.logEvent({
494
- eventType: 'action_confirmation_requested',
495
- intentDetected: decision.pending.intent,
496
- sessionId: this.sessionId,
497
- });
498
- }
928
+ outcome = await this.applyOutcome(turn, currentRoute, snapshot?.context ?? {}, text);
499
929
  }
500
930
  catch (e) {
501
931
  this.clearWatchdog();
502
932
  this.abort = null;
933
+ this.setActivity(null);
503
934
  if (e instanceof AgentApiError && e.code === 'cancelled') {
504
935
  if (this.phase === 'thinking')
505
936
  this.setPhase('idle');
@@ -521,10 +952,14 @@ export class AgentController {
521
952
  this.clearWatchdog();
522
953
  this.abort = null;
523
954
  this.pendingTurnText = null;
955
+ if (!outcome)
956
+ return;
957
+ const reply = outcome.reply;
524
958
  if (reply) {
525
959
  this.lastExchange = { userText: text, replyText: reply };
526
960
  this.emit();
527
- recordConversationExchange(this.conversation, text, reply);
961
+ recordConversationExchange(this.conversation, text, reply, Date.now(), this.conversationOptions);
962
+ this.persistConversation();
528
963
  }
529
964
  else {
530
965
  const fallback = this.config.locale.startsWith('es')
@@ -533,14 +968,53 @@ export class AgentController {
533
968
  this.lastExchange = { userText: text, replyText: fallback };
534
969
  this.emit();
535
970
  }
536
- const hasPending = this.pending != null;
537
- await this.speakReply(reply, audioBase64);
971
+ const hasPending = outcome.hasPending;
972
+ await this.speakReply(reply, outcome.audioBase64);
538
973
  if (hasPending) {
539
974
  this.beginConfirmationVoiceWindow();
975
+ // First step is a sensitive action AND the model wants to keep going:
976
+ // park the chain so it RESUMES after the user confirms.
977
+ if (this.chaining.enabled && outcome.pendingInProgress) {
978
+ this.telemetry.logEvent({
979
+ eventType: 'agent_chain_started',
980
+ sessionId: this.sessionId,
981
+ chainStep: 1,
982
+ });
983
+ this.chainState = {
984
+ userMessage: text,
985
+ step: 1,
986
+ lastStep: outcome.pendingStep,
987
+ visitedKeys: new Set(),
988
+ pendingKey: outcome.pendingKey,
989
+ pendingStep: outcome.pendingStep,
990
+ };
991
+ }
540
992
  }
541
993
  else if (this.phase !== 'speaking') {
542
994
  this.setPhase('idle');
543
995
  }
996
+ // Kick off the autonomous chain when the first step executed cleanly and
997
+ // the backend asked to continue (no sensitive confirmation pending).
998
+ if (this.chaining.enabled && outcome.canContinue && !hasPending) {
999
+ this.telemetry.logEvent({
1000
+ eventType: 'agent_chain_started',
1001
+ sessionId: this.sessionId,
1002
+ chainStep: 1,
1003
+ });
1004
+ const visitedKeys = new Set();
1005
+ if (outcome.actionKey)
1006
+ visitedKeys.add(outcome.actionKey);
1007
+ await this.runChainLoop({
1008
+ userMessage: text,
1009
+ step: 1,
1010
+ lastStep: outcome.lastStep,
1011
+ visitedKeys,
1012
+ });
1013
+ }
1014
+ // Final reply is on screen: the live narrative disappears (unless the
1015
+ // agent is genuinely waiting for a confirmation, which stays honest).
1016
+ if (!this.pending)
1017
+ this.setActivity(null);
544
1018
  this.config.onTurnCompleted?.();
545
1019
  }
546
1020
  // ── Speech session wiring ────────────────────────────────────────────────────
@@ -653,6 +1127,11 @@ export class AgentController {
653
1127
  this.emit();
654
1128
  this.telemetry.logEvent({ eventType: 'agent_consent_declined', sessionId: this.sessionId });
655
1129
  }
1130
+ /**
1131
+ * Cancel everything in flight. If a first turn was still awaiting its reply,
1132
+ * its message is removed from the panel and returned here so the UI can put
1133
+ * it back into the input for editing; null otherwise.
1134
+ */
656
1135
  cancelAll() {
657
1136
  this.abort?.abort();
658
1137
  this.abort = null;
@@ -662,10 +1141,22 @@ export class AgentController {
662
1141
  this.speech.cancel();
663
1142
  this.config.voice.stopPlayback();
664
1143
  this.pending = null;
1144
+ this.chainState = null;
665
1145
  this.confirmationMessage = null;
666
1146
  this.confirmationVoiceHint = null;
1147
+ this.liveActivity = null;
1148
+ this.chainActive = false;
1149
+ // A first turn was still in flight: remove its message from the panel and
1150
+ // hand the text back for editing (the turn never completed, so nothing
1151
+ // was recorded in the conversation).
1152
+ const draft = this.pendingTurnText;
1153
+ if (draft) {
1154
+ this.pendingTurnText = null;
1155
+ this.lastExchange = null;
1156
+ }
667
1157
  this.setPhase('idle');
668
1158
  this.emit();
1159
+ return draft;
669
1160
  }
670
1161
  /** Best-effort backend warm-up (cold-start mitigation). */
671
1162
  warmUp() {
@@ -673,6 +1164,12 @@ export class AgentController {
673
1164
  }
674
1165
  dispose() {
675
1166
  this.cancelAll();
1167
+ this.removeIdentityListeners?.();
1168
+ this.removeIdentityListeners = null;
1169
+ if (this.loadOlderTimer) {
1170
+ clearTimeout(this.loadOlderTimer);
1171
+ this.loadOlderTimer = null;
1172
+ }
676
1173
  this.unsubscribeSpeech?.();
677
1174
  this.unsubscribeSpeech = null;
678
1175
  this.speech.dispose();