@fiodos/web-core 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/api/backendClient.js +20 -0
- package/dist/cjs/config/types.d.ts +18 -0
- package/dist/cjs/config/types.js +6 -1
- package/dist/cjs/controller/AgentController.d.ts +84 -1
- package/dist/cjs/controller/AgentController.js +515 -75
- package/dist/cjs/index.d.ts +3 -3
- package/dist/cjs/index.js +2 -1
- package/dist/cjs/orb/mountOrb.d.ts +6 -0
- package/dist/cjs/orb/mountOrb.js +307 -53
- package/dist/cjs/orb/orbView.d.ts +2 -0
- package/dist/cjs/orb/publishedConfig.d.ts +18 -0
- package/dist/cjs/orb/publishedConfig.js +6 -0
- package/dist/cjs/speech/bubbleDictation.d.ts +11 -0
- package/dist/cjs/speech/bubbleDictation.js +88 -0
- package/dist/cjs/ui/messages.d.ts +15 -0
- package/dist/cjs/ui/messages.js +12 -0
- package/dist/cjs/version.d.ts +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/api/backendClient.js +20 -0
- package/dist/esm/config/types.d.ts +18 -0
- package/dist/esm/config/types.js +5 -0
- package/dist/esm/controller/AgentController.d.ts +84 -1
- package/dist/esm/controller/AgentController.js +517 -77
- package/dist/esm/index.d.ts +3 -3
- package/dist/esm/index.js +1 -1
- package/dist/esm/orb/mountOrb.d.ts +6 -0
- package/dist/esm/orb/mountOrb.js +308 -54
- package/dist/esm/orb/orbView.d.ts +2 -0
- package/dist/esm/orb/publishedConfig.d.ts +18 -0
- package/dist/esm/orb/publishedConfig.js +6 -0
- package/dist/esm/speech/bubbleDictation.d.ts +11 -0
- package/dist/esm/speech/bubbleDictation.js +84 -0
- package/dist/esm/ui/messages.d.ts +15 -0
- package/dist/esm/ui/messages.js +12 -0
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +1 -1
|
@@ -30,6 +30,7 @@ const messages_1 = require("../ui/messages");
|
|
|
30
30
|
function randomSessionId() {
|
|
31
31
|
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
|
32
32
|
}
|
|
33
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
33
34
|
class AgentController {
|
|
34
35
|
constructor(config) {
|
|
35
36
|
this.enabled = true;
|
|
@@ -58,6 +59,21 @@ class AgentController {
|
|
|
58
59
|
// audio), skip the extra /preview-tts round-trip and speak via device voice.
|
|
59
60
|
this.serverTtsUnavailable = false;
|
|
60
61
|
this.pendingConsentIntent = null;
|
|
62
|
+
/** Parked chain state (set when a chain pauses on a sensitive action). */
|
|
63
|
+
this.chainState = null;
|
|
64
|
+
/** REAL live activity for the panel (replaces the old canned walk-through). */
|
|
65
|
+
this.liveActivity = null;
|
|
66
|
+
this.chainActive = false;
|
|
67
|
+
/** How many exchanges the bubble currently mounts (grows one page at a time). */
|
|
68
|
+
this.bubbleWindowSize = core_1.BUBBLE_PAGE_EXCHANGES;
|
|
69
|
+
/** Raw dashboard retention (minutes; 0 = never; null = default). Persisted
|
|
70
|
+
* with the session so a restore applies the same expiry rule. */
|
|
71
|
+
this.conversationRetentionMinutes = null;
|
|
72
|
+
/** Persistence scope: internal orbs keep their memory under a separate key
|
|
73
|
+
* so a public orb sharing the same appId can never read it. */
|
|
74
|
+
this.conversationScope = 'public';
|
|
75
|
+
this.loadingOlderExchanges = false;
|
|
76
|
+
this.loadOlderTimer = null;
|
|
61
77
|
this.listeners = new Set();
|
|
62
78
|
this.unsubscribeSpeech = null;
|
|
63
79
|
// Fail fast on an invalid manifest — same contract as every platform.
|
|
@@ -84,6 +100,13 @@ class AgentController {
|
|
|
84
100
|
isAuthenticated: config.isAuthenticated,
|
|
85
101
|
});
|
|
86
102
|
this.timings = { ...types_1.DEFAULT_AGENT_TIMINGS, ...config.timings };
|
|
103
|
+
// Chaining OFF by default; clamp to the same safe bounds as react/react-native.
|
|
104
|
+
const chainingMerged = { ...types_1.DEFAULT_AGENT_CHAINING, ...config.chaining };
|
|
105
|
+
this.chaining = {
|
|
106
|
+
enabled: chainingMerged.enabled === true,
|
|
107
|
+
maxSteps: Math.max(1, Math.min(6, Math.floor(chainingMerged.maxSteps || 6))),
|
|
108
|
+
dwellMs: Math.max(0, Math.min(3000, Math.floor(chainingMerged.dwellMs ?? 450))),
|
|
109
|
+
};
|
|
87
110
|
this.ttsLocale = config.ttsLocale ?? config.sttLocale;
|
|
88
111
|
this.notifyError =
|
|
89
112
|
config.notifyError ??
|
|
@@ -108,12 +131,34 @@ class AgentController {
|
|
|
108
131
|
});
|
|
109
132
|
// Re-emit combined state whenever the speech machine advances.
|
|
110
133
|
this.unsubscribeSpeech = this.speech.subscribe(() => this.emit());
|
|
134
|
+
// Rehydrate the previous conversation (page reloads): the persisted blob
|
|
135
|
+
// carries its own retention rule, so expiry is decided correctly even
|
|
136
|
+
// before the published config arrives. Best-effort and non-blocking.
|
|
137
|
+
void this.restorePersistedConversation();
|
|
138
|
+
}
|
|
139
|
+
// ── Conversation persistence (survives page reloads) ───────────────────────
|
|
140
|
+
async restorePersistedConversation() {
|
|
141
|
+
const restored = await (0, core_1.restoreConversationSession)(this.config.storage, this.config.manifest.appId, Date.now(), this.conversationScope);
|
|
142
|
+
if (!restored)
|
|
143
|
+
return;
|
|
144
|
+
// Never clobber turns the user already made while the restore resolved.
|
|
145
|
+
if (this.conversation.turns.length > 0 || this.conversation.uiArchive?.length)
|
|
146
|
+
return;
|
|
147
|
+
this.conversation.turns = restored.turns;
|
|
148
|
+
this.conversation.summary = restored.summary;
|
|
149
|
+
this.conversation.uiArchive = restored.uiArchive;
|
|
150
|
+
this.conversation.lastActivityAt = restored.lastActivityAt;
|
|
151
|
+
this.emit();
|
|
152
|
+
}
|
|
153
|
+
persistConversation() {
|
|
154
|
+
void (0, core_1.persistConversationSession)(this.config.storage, this.config.manifest.appId, this.conversation, this.conversationRetentionMinutes, this.conversationScope);
|
|
111
155
|
}
|
|
112
156
|
// ── Subscription / state ───────────────────────────────────────────────────
|
|
113
157
|
getState() {
|
|
114
158
|
const speech = this.speech.getSnapshot();
|
|
115
159
|
const showWaveform = (this.phase === 'listening' || this.phase === 'confirming') && speech.state === 'recording';
|
|
116
160
|
const showThinking = this.phase === 'thinking' || (this.phase === 'listening' && speech.state === 'transcribing');
|
|
161
|
+
const bubbleWindow = (0, core_1.buildBubbleExchangeWindow)(this.conversation, this.lastExchange, this.bubbleWindowSize, this.conversationOptions);
|
|
117
162
|
return {
|
|
118
163
|
phase: this.phase,
|
|
119
164
|
interimText: speech.interimText,
|
|
@@ -123,12 +168,32 @@ class AgentController {
|
|
|
123
168
|
isBusy: showThinking || this.phase === 'speaking',
|
|
124
169
|
isListening: this.phase === 'listening' && speech.state !== 'idle',
|
|
125
170
|
lastExchange: this.lastExchange,
|
|
171
|
+
recentExchanges: bubbleWindow.exchanges,
|
|
172
|
+
hasOlderExchanges: bubbleWindow.hasOlder,
|
|
173
|
+
loadingOlderExchanges: this.loadingOlderExchanges,
|
|
126
174
|
confirmationMessage: this.confirmationMessage,
|
|
127
175
|
confirmationVoiceHint: this.confirmationVoiceHint,
|
|
128
176
|
consentModalVisible: this.consentModalVisible,
|
|
129
177
|
voiceBlocked: this.voiceBlocked,
|
|
178
|
+
liveActivity: this.liveActivity,
|
|
179
|
+
chainActive: this.chainActive,
|
|
130
180
|
};
|
|
131
181
|
}
|
|
182
|
+
/** Update the REAL live activity and re-render (no-op when unchanged). */
|
|
183
|
+
setActivity(next) {
|
|
184
|
+
const prev = this.liveActivity;
|
|
185
|
+
if (prev === next || (prev && next && prev.kind === next.kind && prev.label === next.label)) {
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
this.liveActivity = next;
|
|
189
|
+
this.emit();
|
|
190
|
+
}
|
|
191
|
+
setChainActive(value) {
|
|
192
|
+
if (this.chainActive === value)
|
|
193
|
+
return;
|
|
194
|
+
this.chainActive = value;
|
|
195
|
+
this.emit();
|
|
196
|
+
}
|
|
132
197
|
/** Whether speech recognition can actually run (API present AND mic not denied). */
|
|
133
198
|
get voiceAvailable() {
|
|
134
199
|
return this.config.voice.isRecognitionAvailable() && !this.voiceBlocked;
|
|
@@ -147,6 +212,53 @@ class AgentController {
|
|
|
147
212
|
this.voiceBlocked = blocked;
|
|
148
213
|
this.emit();
|
|
149
214
|
}
|
|
215
|
+
/**
|
|
216
|
+
* Page one more block of older session messages into the bubble (scroll-to-top).
|
|
217
|
+
* The mount is deferred behind a short loading state so old messages stream in
|
|
218
|
+
* on demand instead of all rendering at once (keeps the open bubble light).
|
|
219
|
+
*/
|
|
220
|
+
loadOlderExchanges() {
|
|
221
|
+
if (this.loadingOlderExchanges)
|
|
222
|
+
return;
|
|
223
|
+
const win = (0, core_1.buildBubbleExchangeWindow)(this.conversation, this.lastExchange, this.bubbleWindowSize, this.conversationOptions);
|
|
224
|
+
if (!win.hasOlder)
|
|
225
|
+
return;
|
|
226
|
+
this.loadingOlderExchanges = true;
|
|
227
|
+
this.emit();
|
|
228
|
+
this.loadOlderTimer = setTimeout(() => {
|
|
229
|
+
this.loadOlderTimer = null;
|
|
230
|
+
this.bubbleWindowSize += core_1.BUBBLE_PAGE_EXCHANGES;
|
|
231
|
+
this.loadingOlderExchanges = false;
|
|
232
|
+
this.emit();
|
|
233
|
+
}, 350);
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Apply the dashboard's context-retention setting (minutes; 0 = never clear,
|
|
237
|
+
* null/undefined = the 30-minute default). Called when the published config
|
|
238
|
+
* arrives/refreshes, so the developer's choice is live without a reload.
|
|
239
|
+
* `expandedMemory` = INTERNAL orb profile: use the expanded conversation
|
|
240
|
+
* memory window (more literal turns + longer rolling summary per request).
|
|
241
|
+
*/
|
|
242
|
+
setConversationRetention(minutes, expandedMemory = false) {
|
|
243
|
+
this.conversationOptions = (0, core_1.conversationRetentionOptions)(minutes, expandedMemory);
|
|
244
|
+
this.conversationRetentionMinutes = typeof minutes === 'number' ? minutes : null;
|
|
245
|
+
const scope = expandedMemory ? 'internal' : 'public';
|
|
246
|
+
if (scope !== this.conversationScope) {
|
|
247
|
+
this.conversationScope = scope;
|
|
248
|
+
// The published profile arrived after the constructor's restore attempt:
|
|
249
|
+
// retry under the right storage key (guarded — never clobbers live turns).
|
|
250
|
+
void this.restorePersistedConversation();
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
/** Collapse the bubble history back to the initial page (bubble closed). */
|
|
254
|
+
resetBubbleWindow() {
|
|
255
|
+
if (this.loadOlderTimer) {
|
|
256
|
+
clearTimeout(this.loadOlderTimer);
|
|
257
|
+
this.loadOlderTimer = null;
|
|
258
|
+
}
|
|
259
|
+
this.loadingOlderExchanges = false;
|
|
260
|
+
this.bubbleWindowSize = core_1.BUBBLE_PAGE_EXCHANGES;
|
|
261
|
+
}
|
|
150
262
|
subscribe(listener) {
|
|
151
263
|
this.listeners.add(listener);
|
|
152
264
|
return () => {
|
|
@@ -305,6 +417,11 @@ class AgentController {
|
|
|
305
417
|
if (this.phase === 'confirming')
|
|
306
418
|
this.setPhase('idle');
|
|
307
419
|
this.emit();
|
|
420
|
+
// REAL activity: the confirmed action is executing right now.
|
|
421
|
+
this.setActivity({
|
|
422
|
+
kind: 'executing',
|
|
423
|
+
label: this.actionLabel(pending.intent) ?? pending.intent,
|
|
424
|
+
});
|
|
308
425
|
const result = await this.executor.execute(pending.action, pending.context);
|
|
309
426
|
this.telemetry.logEvent({
|
|
310
427
|
eventType: result.success ? 'action_executed' : 'action_failed',
|
|
@@ -318,6 +435,39 @@ class AgentController {
|
|
|
318
435
|
this.emit();
|
|
319
436
|
await this.speakReply(result.message, null);
|
|
320
437
|
}
|
|
438
|
+
// RESUME the autonomous chain if this sensitive action paused one — the
|
|
439
|
+
// human confirmation requirement is fully preserved: the chain only
|
|
440
|
+
// advances past a sensitive step AFTER the user confirmed and it executed.
|
|
441
|
+
const chainState = this.chainState;
|
|
442
|
+
if (chainState) {
|
|
443
|
+
this.chainState = null;
|
|
444
|
+
if (result.success && this.chaining.enabled) {
|
|
445
|
+
this.telemetry.logEvent({
|
|
446
|
+
eventType: 'agent_chain_step',
|
|
447
|
+
intentDetected: pending.intent,
|
|
448
|
+
sessionId: this.sessionId,
|
|
449
|
+
chainStep: chainState.step,
|
|
450
|
+
});
|
|
451
|
+
if (chainState.pendingKey)
|
|
452
|
+
chainState.visitedKeys.add(chainState.pendingKey);
|
|
453
|
+
await this.runChainLoop({
|
|
454
|
+
userMessage: chainState.userMessage,
|
|
455
|
+
step: chainState.step,
|
|
456
|
+
lastStep: chainState.pendingStep ?? chainState.lastStep,
|
|
457
|
+
visitedKeys: chainState.visitedKeys,
|
|
458
|
+
});
|
|
459
|
+
}
|
|
460
|
+
else if (!result.success) {
|
|
461
|
+
this.telemetry.logEvent({
|
|
462
|
+
eventType: 'agent_chain_aborted',
|
|
463
|
+
sessionId: this.sessionId,
|
|
464
|
+
chainAbortReason: 'error',
|
|
465
|
+
});
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
// The confirmed action (and any resumed chain) finished: clear the live row.
|
|
469
|
+
if (!this.pending)
|
|
470
|
+
this.setActivity(null);
|
|
321
471
|
}
|
|
322
472
|
cancelPendingAction() {
|
|
323
473
|
this.clearConfirmWindow();
|
|
@@ -325,9 +475,20 @@ class AgentController {
|
|
|
325
475
|
this.pending = null;
|
|
326
476
|
this.confirmationMessage = null;
|
|
327
477
|
this.confirmationVoiceHint = null;
|
|
478
|
+
this.setActivity(null);
|
|
328
479
|
if (this.phase === 'confirming')
|
|
329
480
|
this.setPhase('idle');
|
|
330
481
|
this.emit();
|
|
482
|
+
// Declining a sensitive action stops any chain it was part of, cleanly.
|
|
483
|
+
const chainState = this.chainState;
|
|
484
|
+
this.chainState = null;
|
|
485
|
+
if (chainState) {
|
|
486
|
+
this.telemetry.logEvent({
|
|
487
|
+
eventType: 'agent_chain_aborted',
|
|
488
|
+
sessionId: this.sessionId,
|
|
489
|
+
chainAbortReason: 'user_interrupt',
|
|
490
|
+
});
|
|
491
|
+
}
|
|
331
492
|
if (!pending)
|
|
332
493
|
return;
|
|
333
494
|
this.telemetry.logEvent({
|
|
@@ -371,6 +532,286 @@ class AgentController {
|
|
|
371
532
|
this.beginConfirmationVoiceWindow();
|
|
372
533
|
}
|
|
373
534
|
}
|
|
535
|
+
routeLabel(intent) {
|
|
536
|
+
return this.config.manifest.routes.find((r) => r.intent === intent)?.label;
|
|
537
|
+
}
|
|
538
|
+
actionLabel(intent) {
|
|
539
|
+
return this.config.manifest.actions.find((a) => a.intent === intent)?.label;
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* Resolve ONE backend turn into user-facing reply/audio: run its action (or
|
|
543
|
+
* stage its confirmation), emit telemetry, and report whether the autonomous
|
|
544
|
+
* chain may take another step. Shared by the first turn and every
|
|
545
|
+
* continuation so both follow the exact same security path — disabled
|
|
546
|
+
* capabilities, confidentiality and sensitive confirmations apply per step.
|
|
547
|
+
*/
|
|
548
|
+
async applyOutcome(turn, routeAtSend, snapshotContext, userMessage) {
|
|
549
|
+
let outReply = turn.reply;
|
|
550
|
+
let outAudio = turn.audioBase64;
|
|
551
|
+
if (outReply)
|
|
552
|
+
this.serverTtsUnavailable = !outAudio;
|
|
553
|
+
const decision = await (0, turnEngine_1.decideAction)({
|
|
554
|
+
action: turn.action,
|
|
555
|
+
manifest: this.config.manifest,
|
|
556
|
+
executor: this.executor,
|
|
557
|
+
context: snapshotContext,
|
|
558
|
+
messages: this.messages,
|
|
559
|
+
userMessage,
|
|
560
|
+
});
|
|
561
|
+
let canContinue = false;
|
|
562
|
+
let lastStep;
|
|
563
|
+
let actionKey;
|
|
564
|
+
let hasPending = false;
|
|
565
|
+
let pendingInProgress = false;
|
|
566
|
+
let pendingStep;
|
|
567
|
+
let pendingKey;
|
|
568
|
+
if (decision.kind === 'executed') {
|
|
569
|
+
const r = decision.result;
|
|
570
|
+
// REAL activity for the panel: this action actually just ran — the
|
|
571
|
+
// label comes from the manifest, never invented.
|
|
572
|
+
if (r.success && turn.action?.type === 'navigate') {
|
|
573
|
+
this.setActivity({
|
|
574
|
+
kind: 'navigating',
|
|
575
|
+
label: this.routeLabel(turn.action.intent ?? '') ?? turn.action.intent ?? '',
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
else if (r.success && !r.recoverable && turn.action?.type === 'execute') {
|
|
579
|
+
this.setActivity({
|
|
580
|
+
kind: 'executing',
|
|
581
|
+
label: this.actionLabel(turn.action.intent ?? '') ?? turn.action.intent ?? '',
|
|
582
|
+
});
|
|
583
|
+
}
|
|
584
|
+
if (turn.action?.type === 'navigate') {
|
|
585
|
+
this.telemetry.logEvent({
|
|
586
|
+
eventType: r.success ? 'navigation_executed' : 'navigation_failed',
|
|
587
|
+
intentDetected: turn.action.intent,
|
|
588
|
+
sessionId: this.sessionId,
|
|
589
|
+
currentRoute: routeAtSend ?? undefined,
|
|
590
|
+
});
|
|
591
|
+
}
|
|
592
|
+
else if (turn.action?.type === 'execute') {
|
|
593
|
+
this.telemetry.logEvent({
|
|
594
|
+
eventType: r.recoverable
|
|
595
|
+
? 'action_needs_context'
|
|
596
|
+
: r.success
|
|
597
|
+
? 'action_executed'
|
|
598
|
+
: 'action_failed',
|
|
599
|
+
intentDetected: turn.action.intent,
|
|
600
|
+
sessionId: this.sessionId,
|
|
601
|
+
actionExecuted: { intent: turn.action.intent, confirmed: false },
|
|
602
|
+
});
|
|
603
|
+
}
|
|
604
|
+
if (r.recoverable) {
|
|
605
|
+
/* keep the assistant's guiding reply; never speak a refusal. */
|
|
606
|
+
}
|
|
607
|
+
else if (r.message) {
|
|
608
|
+
outReply = r.message;
|
|
609
|
+
outAudio = null;
|
|
610
|
+
}
|
|
611
|
+
// Chain signal: only a cleanly EXECUTED, non-recoverable step may advance,
|
|
612
|
+
// and only when the backend asked to continue (task_status=in_progress).
|
|
613
|
+
if (turn.action &&
|
|
614
|
+
(turn.action.type === 'navigate' || turn.action.type === 'execute') &&
|
|
615
|
+
r.success &&
|
|
616
|
+
!r.recoverable) {
|
|
617
|
+
const intent = turn.action.intent ?? '';
|
|
618
|
+
lastStep = {
|
|
619
|
+
type: turn.action.type,
|
|
620
|
+
intent,
|
|
621
|
+
label: turn.action.type === 'navigate' ? this.routeLabel(intent) : this.actionLabel(intent),
|
|
622
|
+
success: true,
|
|
623
|
+
};
|
|
624
|
+
actionKey = `${turn.action.type}:${intent}`;
|
|
625
|
+
canContinue = turn.taskStatus === 'in_progress';
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
else if (decision.kind === 'precheck-resolved') {
|
|
629
|
+
if (decision.result.message && !decision.result.recoverable) {
|
|
630
|
+
outReply = decision.result.message;
|
|
631
|
+
outAudio = null;
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
else if (decision.kind === 'needs-confirmation') {
|
|
635
|
+
// A sensitive action ALWAYS pauses (chaining or not): stage the
|
|
636
|
+
// confirmation and stop advancing. Resumes after the user confirms.
|
|
637
|
+
this.pending = { ...decision.pending, silent: this.silentTurn };
|
|
638
|
+
this.confirmationReasked = false;
|
|
639
|
+
this.confirmationMessage = decision.pending.question;
|
|
640
|
+
this.confirmationVoiceHint =
|
|
641
|
+
decision.pending.voiceMode === 'strict' &&
|
|
642
|
+
decision.pending.strictPhrase &&
|
|
643
|
+
!this.silentTurn
|
|
644
|
+
? (0, core_1.formatMessage)(this.messages.strictConfirmationModalHint, {
|
|
645
|
+
phrase: decision.pending.strictPhrase,
|
|
646
|
+
})
|
|
647
|
+
: null;
|
|
648
|
+
this.emit();
|
|
649
|
+
this.telemetry.logEvent({
|
|
650
|
+
eventType: 'action_confirmation_requested',
|
|
651
|
+
intentDetected: decision.pending.intent,
|
|
652
|
+
sessionId: this.sessionId,
|
|
653
|
+
});
|
|
654
|
+
// REAL activity: the agent is genuinely waiting for the human now.
|
|
655
|
+
this.setActivity({ kind: 'waiting_confirmation' });
|
|
656
|
+
hasPending = true;
|
|
657
|
+
pendingInProgress = turn.taskStatus === 'in_progress';
|
|
658
|
+
const pIntent = decision.pending.intent || '';
|
|
659
|
+
pendingStep = { type: 'execute', intent: pIntent, label: this.actionLabel(pIntent), success: true };
|
|
660
|
+
pendingKey = `execute:${pIntent}`;
|
|
661
|
+
}
|
|
662
|
+
return {
|
|
663
|
+
reply: outReply,
|
|
664
|
+
audioBase64: outAudio,
|
|
665
|
+
hasPending,
|
|
666
|
+
canContinue,
|
|
667
|
+
lastStep,
|
|
668
|
+
actionKey,
|
|
669
|
+
pendingInProgress,
|
|
670
|
+
pendingStep,
|
|
671
|
+
pendingKey,
|
|
672
|
+
};
|
|
673
|
+
}
|
|
674
|
+
/**
|
|
675
|
+
* The autonomous continuation loop. Extracted so confirmPendingAction can
|
|
676
|
+
* RESUME the SAME chain after a sensitive step is confirmed. Capped at
|
|
677
|
+
* maxSteps, with no-progress detection and sensitive-confirmation pauses.
|
|
678
|
+
*/
|
|
679
|
+
async runChainLoop(state) {
|
|
680
|
+
let step = state.step;
|
|
681
|
+
let lastStep = state.lastStep;
|
|
682
|
+
const visitedKeys = state.visitedKeys;
|
|
683
|
+
// Keep the panel's live row visible for the WHOLE chain (including the
|
|
684
|
+
// dwell between steps), so the real narration never flickers away.
|
|
685
|
+
this.setChainActive(true);
|
|
686
|
+
try {
|
|
687
|
+
while (step < this.chaining.maxSteps) {
|
|
688
|
+
step += 1;
|
|
689
|
+
if (this.chaining.dwellMs > 0)
|
|
690
|
+
await sleep(this.chaining.dwellMs);
|
|
691
|
+
if (this.pending != null)
|
|
692
|
+
return; // paused / interrupted elsewhere
|
|
693
|
+
const routeNow = this.config.navigation.getCurrentRoute();
|
|
694
|
+
const snap = this.screenStore.getSnapshot();
|
|
695
|
+
const rawCtx = (0, core_1.combineContexts)(this.config.getUserContextText?.() ?? null, this.screenStore.getText(), {
|
|
696
|
+
truncationSuffix: this.messages.contextTruncationSuffix,
|
|
697
|
+
maxChars: this.config.contextMaxChars,
|
|
698
|
+
});
|
|
699
|
+
const san = rawCtx ? this.sanitizer(rawCtx).sanitized : undefined;
|
|
700
|
+
const conv = (0, core_1.prepareConversationForTurn)(this.conversation, Date.now(), this.conversationOptions);
|
|
701
|
+
this.setPhase('thinking');
|
|
702
|
+
// Honest state: the model is choosing the next step right now.
|
|
703
|
+
this.setActivity({ kind: 'thinking' });
|
|
704
|
+
const controller = new AbortController();
|
|
705
|
+
this.abort = controller;
|
|
706
|
+
let oc;
|
|
707
|
+
try {
|
|
708
|
+
const turn = await this.config.backend.sendTurn({
|
|
709
|
+
message: state.userMessage,
|
|
710
|
+
language: this.config.locale,
|
|
711
|
+
voice: this.getTtsVoice(),
|
|
712
|
+
screenContext: san,
|
|
713
|
+
currentRoute: routeNow ?? undefined,
|
|
714
|
+
conversationHistory: conv.history,
|
|
715
|
+
conversationSummary: conv.summary ?? undefined,
|
|
716
|
+
sessionId: this.sessionId,
|
|
717
|
+
isContinuation: true,
|
|
718
|
+
chainStep: step,
|
|
719
|
+
lastStep,
|
|
720
|
+
...(0, backendClient_1.buildManifestPayload)(this.config.manifest),
|
|
721
|
+
}, { signal: controller.signal });
|
|
722
|
+
oc = await this.applyOutcome(turn, routeNow, snap?.context ?? {}, state.userMessage);
|
|
723
|
+
}
|
|
724
|
+
catch (e) {
|
|
725
|
+
this.abort = null;
|
|
726
|
+
const interrupted = e instanceof errors_1.AgentApiError && e.code === 'cancelled';
|
|
727
|
+
if (interrupted) {
|
|
728
|
+
if (this.phase === 'thinking')
|
|
729
|
+
this.setPhase('idle');
|
|
730
|
+
}
|
|
731
|
+
else {
|
|
732
|
+
if (e instanceof errors_1.AgentApiError && e.code === 'quota_exceeded') {
|
|
733
|
+
this.config.onQuotaExceeded?.();
|
|
734
|
+
}
|
|
735
|
+
const errMsg = (0, formatTurnError_1.formatAgentTurnError)(e, this.config.locale);
|
|
736
|
+
this.lastExchange = { userText: state.userMessage, replyText: errMsg };
|
|
737
|
+
if (this.phase === 'thinking')
|
|
738
|
+
this.setPhase('idle');
|
|
739
|
+
this.emit();
|
|
740
|
+
this.notifyError(errMsg);
|
|
741
|
+
}
|
|
742
|
+
this.telemetry.logEvent({
|
|
743
|
+
eventType: 'agent_chain_aborted',
|
|
744
|
+
sessionId: this.sessionId,
|
|
745
|
+
chainStep: step,
|
|
746
|
+
chainAbortReason: interrupted ? 'user_interrupt' : 'error',
|
|
747
|
+
});
|
|
748
|
+
return;
|
|
749
|
+
}
|
|
750
|
+
this.abort = null;
|
|
751
|
+
if (oc.reply) {
|
|
752
|
+
this.lastExchange = { userText: state.userMessage, replyText: oc.reply };
|
|
753
|
+
this.emit();
|
|
754
|
+
(0, core_1.recordConversationExchange)(this.conversation, state.userMessage, oc.reply, Date.now(), this.conversationOptions);
|
|
755
|
+
this.persistConversation();
|
|
756
|
+
}
|
|
757
|
+
await this.speakReply(oc.reply, oc.audioBase64);
|
|
758
|
+
this.telemetry.logEvent({
|
|
759
|
+
eventType: 'agent_chain_step',
|
|
760
|
+
intentDetected: oc.lastStep?.intent ?? oc.pendingStep?.intent ?? null,
|
|
761
|
+
sessionId: this.sessionId,
|
|
762
|
+
chainStep: step,
|
|
763
|
+
});
|
|
764
|
+
// Sensitive action → ALWAYS pause for confirmation. Park the chain so
|
|
765
|
+
// confirmPendingAction can resume it (only if the model wants to keep
|
|
766
|
+
// going); a rejection stops it cleanly.
|
|
767
|
+
if (oc.hasPending) {
|
|
768
|
+
this.chainState = oc.pendingInProgress
|
|
769
|
+
? {
|
|
770
|
+
userMessage: state.userMessage,
|
|
771
|
+
step,
|
|
772
|
+
lastStep,
|
|
773
|
+
visitedKeys,
|
|
774
|
+
pendingKey: oc.pendingKey,
|
|
775
|
+
pendingStep: oc.pendingStep,
|
|
776
|
+
}
|
|
777
|
+
: null;
|
|
778
|
+
this.beginConfirmationVoiceWindow();
|
|
779
|
+
return;
|
|
780
|
+
}
|
|
781
|
+
if (this.phase !== 'speaking')
|
|
782
|
+
this.setPhase('idle');
|
|
783
|
+
if (oc.actionKey && visitedKeys.has(oc.actionKey)) {
|
|
784
|
+
this.telemetry.logEvent({
|
|
785
|
+
eventType: 'agent_chain_aborted',
|
|
786
|
+
sessionId: this.sessionId,
|
|
787
|
+
chainStep: step,
|
|
788
|
+
chainAbortReason: 'no_progress',
|
|
789
|
+
});
|
|
790
|
+
return;
|
|
791
|
+
}
|
|
792
|
+
if (oc.actionKey)
|
|
793
|
+
visitedKeys.add(oc.actionKey);
|
|
794
|
+
if (!oc.canContinue) {
|
|
795
|
+
this.telemetry.logEvent({
|
|
796
|
+
eventType: 'agent_chain_completed',
|
|
797
|
+
sessionId: this.sessionId,
|
|
798
|
+
chainStep: step,
|
|
799
|
+
});
|
|
800
|
+
return;
|
|
801
|
+
}
|
|
802
|
+
lastStep = oc.lastStep;
|
|
803
|
+
}
|
|
804
|
+
this.telemetry.logEvent({
|
|
805
|
+
eventType: 'agent_chain_aborted',
|
|
806
|
+
sessionId: this.sessionId,
|
|
807
|
+
chainStep: step,
|
|
808
|
+
chainAbortReason: 'max_steps',
|
|
809
|
+
});
|
|
810
|
+
}
|
|
811
|
+
finally {
|
|
812
|
+
this.setChainActive(false);
|
|
813
|
+
}
|
|
814
|
+
}
|
|
374
815
|
// ── Main turn ────────────────────────────────────────────────────────────────
|
|
375
816
|
async handleTranscript(rawText) {
|
|
376
817
|
const text = rawText.trim();
|
|
@@ -380,6 +821,8 @@ class AgentController {
|
|
|
380
821
|
return;
|
|
381
822
|
}
|
|
382
823
|
this.setPhase('thinking');
|
|
824
|
+
// REAL activity: the request is genuinely in flight from this moment.
|
|
825
|
+
this.setActivity({ kind: 'thinking' });
|
|
383
826
|
this.pendingTurnText = text;
|
|
384
827
|
this.lastExchange = { userText: text, replyText: '' };
|
|
385
828
|
this.emit();
|
|
@@ -393,6 +836,7 @@ class AgentController {
|
|
|
393
836
|
: 'The response took too long. Please try again.';
|
|
394
837
|
this.lastExchange = { userText, replyText };
|
|
395
838
|
this.pendingTurnText = null;
|
|
839
|
+
this.setActivity(null);
|
|
396
840
|
this.setPhase('idle');
|
|
397
841
|
this.emit();
|
|
398
842
|
}
|
|
@@ -413,11 +857,10 @@ class AgentController {
|
|
|
413
857
|
maxChars: this.config.contextMaxChars,
|
|
414
858
|
});
|
|
415
859
|
const sanitized = rawContext ? this.sanitizer(rawContext).sanitized : undefined;
|
|
416
|
-
const conv = (0, core_1.prepareConversationForTurn)(this.conversation);
|
|
860
|
+
const conv = (0, core_1.prepareConversationForTurn)(this.conversation, Date.now(), this.conversationOptions);
|
|
417
861
|
const controller = new AbortController();
|
|
418
862
|
this.abort = controller;
|
|
419
|
-
let
|
|
420
|
-
let audioBase64 = null;
|
|
863
|
+
let outcome = null;
|
|
421
864
|
try {
|
|
422
865
|
const turn = await this.config.backend.sendTurn({
|
|
423
866
|
message: text,
|
|
@@ -430,79 +873,12 @@ class AgentController {
|
|
|
430
873
|
sessionId: this.sessionId,
|
|
431
874
|
...(0, backendClient_1.buildManifestPayload)(this.config.manifest),
|
|
432
875
|
}, { signal: controller.signal });
|
|
433
|
-
|
|
434
|
-
audioBase64 = turn.audioBase64;
|
|
435
|
-
// Learn whether this backend ships server-side TTS: a non-empty reply
|
|
436
|
-
// with no audio means it doesn't — later lines then speak instantly.
|
|
437
|
-
if (reply)
|
|
438
|
-
this.serverTtsUnavailable = !audioBase64;
|
|
439
|
-
const decision = await (0, turnEngine_1.decideAction)({
|
|
440
|
-
action: turn.action,
|
|
441
|
-
manifest: this.config.manifest,
|
|
442
|
-
executor: this.executor,
|
|
443
|
-
context: snapshot?.context ?? {},
|
|
444
|
-
messages: this.messages,
|
|
445
|
-
userMessage: text,
|
|
446
|
-
});
|
|
447
|
-
if (decision.kind === 'executed') {
|
|
448
|
-
const r = decision.result;
|
|
449
|
-
if (turn.action?.type === 'navigate') {
|
|
450
|
-
this.telemetry.logEvent({
|
|
451
|
-
eventType: r.success ? 'navigation_executed' : 'navigation_failed',
|
|
452
|
-
intentDetected: turn.action.intent,
|
|
453
|
-
sessionId: this.sessionId,
|
|
454
|
-
currentRoute,
|
|
455
|
-
});
|
|
456
|
-
}
|
|
457
|
-
else if (turn.action?.type === 'execute') {
|
|
458
|
-
this.telemetry.logEvent({
|
|
459
|
-
eventType: r.recoverable
|
|
460
|
-
? 'action_needs_context'
|
|
461
|
-
: r.success
|
|
462
|
-
? 'action_executed'
|
|
463
|
-
: 'action_failed',
|
|
464
|
-
intentDetected: turn.action.intent,
|
|
465
|
-
sessionId: this.sessionId,
|
|
466
|
-
actionExecuted: { intent: turn.action.intent, confirmed: false },
|
|
467
|
-
});
|
|
468
|
-
}
|
|
469
|
-
if (r.recoverable) {
|
|
470
|
-
/* keep the assistant's guiding reply; never speak a refusal. */
|
|
471
|
-
}
|
|
472
|
-
else if (r.message) {
|
|
473
|
-
reply = r.message;
|
|
474
|
-
audioBase64 = null;
|
|
475
|
-
}
|
|
476
|
-
}
|
|
477
|
-
else if (decision.kind === 'precheck-resolved') {
|
|
478
|
-
if (decision.result.message && !decision.result.recoverable) {
|
|
479
|
-
reply = decision.result.message;
|
|
480
|
-
audioBase64 = null;
|
|
481
|
-
}
|
|
482
|
-
}
|
|
483
|
-
else if (decision.kind === 'needs-confirmation') {
|
|
484
|
-
this.pending = { ...decision.pending, silent: this.silentTurn };
|
|
485
|
-
this.confirmationReasked = false;
|
|
486
|
-
this.confirmationMessage = decision.pending.question;
|
|
487
|
-
this.confirmationVoiceHint =
|
|
488
|
-
decision.pending.voiceMode === 'strict' &&
|
|
489
|
-
decision.pending.strictPhrase &&
|
|
490
|
-
!this.silentTurn
|
|
491
|
-
? (0, core_1.formatMessage)(this.messages.strictConfirmationModalHint, {
|
|
492
|
-
phrase: decision.pending.strictPhrase,
|
|
493
|
-
})
|
|
494
|
-
: null;
|
|
495
|
-
this.emit();
|
|
496
|
-
this.telemetry.logEvent({
|
|
497
|
-
eventType: 'action_confirmation_requested',
|
|
498
|
-
intentDetected: decision.pending.intent,
|
|
499
|
-
sessionId: this.sessionId,
|
|
500
|
-
});
|
|
501
|
-
}
|
|
876
|
+
outcome = await this.applyOutcome(turn, currentRoute, snapshot?.context ?? {}, text);
|
|
502
877
|
}
|
|
503
878
|
catch (e) {
|
|
504
879
|
this.clearWatchdog();
|
|
505
880
|
this.abort = null;
|
|
881
|
+
this.setActivity(null);
|
|
506
882
|
if (e instanceof errors_1.AgentApiError && e.code === 'cancelled') {
|
|
507
883
|
if (this.phase === 'thinking')
|
|
508
884
|
this.setPhase('idle');
|
|
@@ -524,10 +900,14 @@ class AgentController {
|
|
|
524
900
|
this.clearWatchdog();
|
|
525
901
|
this.abort = null;
|
|
526
902
|
this.pendingTurnText = null;
|
|
903
|
+
if (!outcome)
|
|
904
|
+
return;
|
|
905
|
+
const reply = outcome.reply;
|
|
527
906
|
if (reply) {
|
|
528
907
|
this.lastExchange = { userText: text, replyText: reply };
|
|
529
908
|
this.emit();
|
|
530
|
-
(0, core_1.recordConversationExchange)(this.conversation, text, reply);
|
|
909
|
+
(0, core_1.recordConversationExchange)(this.conversation, text, reply, Date.now(), this.conversationOptions);
|
|
910
|
+
this.persistConversation();
|
|
531
911
|
}
|
|
532
912
|
else {
|
|
533
913
|
const fallback = this.config.locale.startsWith('es')
|
|
@@ -536,14 +916,53 @@ class AgentController {
|
|
|
536
916
|
this.lastExchange = { userText: text, replyText: fallback };
|
|
537
917
|
this.emit();
|
|
538
918
|
}
|
|
539
|
-
const hasPending =
|
|
540
|
-
await this.speakReply(reply, audioBase64);
|
|
919
|
+
const hasPending = outcome.hasPending;
|
|
920
|
+
await this.speakReply(reply, outcome.audioBase64);
|
|
541
921
|
if (hasPending) {
|
|
542
922
|
this.beginConfirmationVoiceWindow();
|
|
923
|
+
// First step is a sensitive action AND the model wants to keep going:
|
|
924
|
+
// park the chain so it RESUMES after the user confirms.
|
|
925
|
+
if (this.chaining.enabled && outcome.pendingInProgress) {
|
|
926
|
+
this.telemetry.logEvent({
|
|
927
|
+
eventType: 'agent_chain_started',
|
|
928
|
+
sessionId: this.sessionId,
|
|
929
|
+
chainStep: 1,
|
|
930
|
+
});
|
|
931
|
+
this.chainState = {
|
|
932
|
+
userMessage: text,
|
|
933
|
+
step: 1,
|
|
934
|
+
lastStep: outcome.pendingStep,
|
|
935
|
+
visitedKeys: new Set(),
|
|
936
|
+
pendingKey: outcome.pendingKey,
|
|
937
|
+
pendingStep: outcome.pendingStep,
|
|
938
|
+
};
|
|
939
|
+
}
|
|
543
940
|
}
|
|
544
941
|
else if (this.phase !== 'speaking') {
|
|
545
942
|
this.setPhase('idle');
|
|
546
943
|
}
|
|
944
|
+
// Kick off the autonomous chain when the first step executed cleanly and
|
|
945
|
+
// the backend asked to continue (no sensitive confirmation pending).
|
|
946
|
+
if (this.chaining.enabled && outcome.canContinue && !hasPending) {
|
|
947
|
+
this.telemetry.logEvent({
|
|
948
|
+
eventType: 'agent_chain_started',
|
|
949
|
+
sessionId: this.sessionId,
|
|
950
|
+
chainStep: 1,
|
|
951
|
+
});
|
|
952
|
+
const visitedKeys = new Set();
|
|
953
|
+
if (outcome.actionKey)
|
|
954
|
+
visitedKeys.add(outcome.actionKey);
|
|
955
|
+
await this.runChainLoop({
|
|
956
|
+
userMessage: text,
|
|
957
|
+
step: 1,
|
|
958
|
+
lastStep: outcome.lastStep,
|
|
959
|
+
visitedKeys,
|
|
960
|
+
});
|
|
961
|
+
}
|
|
962
|
+
// Final reply is on screen: the live narrative disappears (unless the
|
|
963
|
+
// agent is genuinely waiting for a confirmation, which stays honest).
|
|
964
|
+
if (!this.pending)
|
|
965
|
+
this.setActivity(null);
|
|
547
966
|
this.config.onTurnCompleted?.();
|
|
548
967
|
}
|
|
549
968
|
// ── Speech session wiring ────────────────────────────────────────────────────
|
|
@@ -656,6 +1075,11 @@ class AgentController {
|
|
|
656
1075
|
this.emit();
|
|
657
1076
|
this.telemetry.logEvent({ eventType: 'agent_consent_declined', sessionId: this.sessionId });
|
|
658
1077
|
}
|
|
1078
|
+
/**
|
|
1079
|
+
* Cancel everything in flight. If a first turn was still awaiting its reply,
|
|
1080
|
+
* its message is removed from the panel and returned here so the UI can put
|
|
1081
|
+
* it back into the input for editing; null otherwise.
|
|
1082
|
+
*/
|
|
659
1083
|
cancelAll() {
|
|
660
1084
|
this.abort?.abort();
|
|
661
1085
|
this.abort = null;
|
|
@@ -665,10 +1089,22 @@ class AgentController {
|
|
|
665
1089
|
this.speech.cancel();
|
|
666
1090
|
this.config.voice.stopPlayback();
|
|
667
1091
|
this.pending = null;
|
|
1092
|
+
this.chainState = null;
|
|
668
1093
|
this.confirmationMessage = null;
|
|
669
1094
|
this.confirmationVoiceHint = null;
|
|
1095
|
+
this.liveActivity = null;
|
|
1096
|
+
this.chainActive = false;
|
|
1097
|
+
// A first turn was still in flight: remove its message from the panel and
|
|
1098
|
+
// hand the text back for editing (the turn never completed, so nothing
|
|
1099
|
+
// was recorded in the conversation).
|
|
1100
|
+
const draft = this.pendingTurnText;
|
|
1101
|
+
if (draft) {
|
|
1102
|
+
this.pendingTurnText = null;
|
|
1103
|
+
this.lastExchange = null;
|
|
1104
|
+
}
|
|
670
1105
|
this.setPhase('idle');
|
|
671
1106
|
this.emit();
|
|
1107
|
+
return draft;
|
|
672
1108
|
}
|
|
673
1109
|
/** Best-effort backend warm-up (cold-start mitigation). */
|
|
674
1110
|
warmUp() {
|
|
@@ -676,6 +1112,10 @@ class AgentController {
|
|
|
676
1112
|
}
|
|
677
1113
|
dispose() {
|
|
678
1114
|
this.cancelAll();
|
|
1115
|
+
if (this.loadOlderTimer) {
|
|
1116
|
+
clearTimeout(this.loadOlderTimer);
|
|
1117
|
+
this.loadOlderTimer = null;
|
|
1118
|
+
}
|
|
679
1119
|
this.unsubscribeSpeech?.();
|
|
680
1120
|
this.unsubscribeSpeech = null;
|
|
681
1121
|
this.speech.dispose();
|