@fiodos/web-core 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/LICENSE +77 -0
  2. package/dist/cjs/adapters/webNavigationAdapter.d.ts +31 -0
  3. package/dist/cjs/adapters/webNavigationAdapter.js +45 -0
  4. package/dist/cjs/adapters/webStorageAdapter.d.ts +13 -0
  5. package/dist/cjs/adapters/webStorageAdapter.js +57 -0
  6. package/dist/cjs/adapters/webVoiceAdapter.d.ts +16 -0
  7. package/dist/cjs/adapters/webVoiceAdapter.js +477 -0
  8. package/dist/cjs/api/backendClient.d.ts +31 -0
  9. package/dist/cjs/api/backendClient.js +182 -0
  10. package/dist/cjs/api/backendTelemetry.d.ts +23 -0
  11. package/dist/cjs/api/backendTelemetry.js +80 -0
  12. package/dist/cjs/api/clientBootstrap.d.ts +44 -0
  13. package/dist/cjs/api/clientBootstrap.js +68 -0
  14. package/dist/cjs/api/errors.d.ts +11 -0
  15. package/dist/cjs/api/errors.js +16 -0
  16. package/dist/cjs/bridge/createBridge.d.ts +13 -0
  17. package/dist/cjs/bridge/createBridge.js +15 -0
  18. package/dist/cjs/config/types.d.ts +63 -0
  19. package/dist/cjs/config/types.js +9 -0
  20. package/dist/cjs/context/screenContextStore.d.ts +26 -0
  21. package/dist/cjs/context/screenContextStore.js +25 -0
  22. package/dist/cjs/controller/AgentController.d.ts +115 -0
  23. package/dist/cjs/controller/AgentController.js +594 -0
  24. package/dist/cjs/core/turnEngine.d.ts +70 -0
  25. package/dist/cjs/core/turnEngine.js +96 -0
  26. package/dist/cjs/dropin/createFiodosAgent.d.ts +60 -0
  27. package/dist/cjs/dropin/createFiodosAgent.js +143 -0
  28. package/dist/cjs/dropin/createFyodosAgent.d.ts +60 -0
  29. package/dist/cjs/dropin/createFyodosAgent.js +143 -0
  30. package/dist/cjs/index.d.ts +45 -0
  31. package/dist/cjs/index.js +67 -0
  32. package/dist/cjs/orb/mountOrb.d.ts +24 -0
  33. package/dist/cjs/orb/mountOrb.js +694 -0
  34. package/dist/cjs/orb/orbView.d.ts +56 -0
  35. package/dist/cjs/orb/orbView.js +426 -0
  36. package/dist/cjs/orb/publishedConfig.d.ts +32 -0
  37. package/dist/cjs/orb/publishedConfig.js +70 -0
  38. package/dist/cjs/package.json +1 -0
  39. package/dist/cjs/speech/speechSession.d.ts +35 -0
  40. package/dist/cjs/speech/speechSession.js +60 -0
  41. package/dist/cjs/ui/messages.d.ts +36 -0
  42. package/dist/cjs/ui/messages.js +41 -0
  43. package/dist/esm/adapters/webNavigationAdapter.d.ts +31 -0
  44. package/dist/esm/adapters/webNavigationAdapter.js +41 -0
  45. package/dist/esm/adapters/webStorageAdapter.d.ts +13 -0
  46. package/dist/esm/adapters/webStorageAdapter.js +54 -0
  47. package/dist/esm/adapters/webVoiceAdapter.d.ts +16 -0
  48. package/dist/esm/adapters/webVoiceAdapter.js +474 -0
  49. package/dist/esm/api/backendClient.d.ts +31 -0
  50. package/dist/esm/api/backendClient.js +178 -0
  51. package/dist/esm/api/backendTelemetry.d.ts +23 -0
  52. package/dist/esm/api/backendTelemetry.js +77 -0
  53. package/dist/esm/api/clientBootstrap.d.ts +44 -0
  54. package/dist/esm/api/clientBootstrap.js +63 -0
  55. package/dist/esm/api/errors.d.ts +11 -0
  56. package/dist/esm/api/errors.js +11 -0
  57. package/dist/esm/bridge/createBridge.d.ts +13 -0
  58. package/dist/esm/bridge/createBridge.js +12 -0
  59. package/dist/esm/config/types.d.ts +63 -0
  60. package/dist/esm/config/types.js +6 -0
  61. package/dist/esm/context/screenContextStore.d.ts +26 -0
  62. package/dist/esm/context/screenContextStore.js +22 -0
  63. package/dist/esm/controller/AgentController.d.ts +115 -0
  64. package/dist/esm/controller/AgentController.js +590 -0
  65. package/dist/esm/core/turnEngine.d.ts +70 -0
  66. package/dist/esm/core/turnEngine.js +92 -0
  67. package/dist/esm/dropin/createFiodosAgent.d.ts +60 -0
  68. package/dist/esm/dropin/createFiodosAgent.js +140 -0
  69. package/dist/esm/dropin/createFyodosAgent.d.ts +60 -0
  70. package/dist/esm/dropin/createFyodosAgent.js +140 -0
  71. package/dist/esm/index.d.ts +45 -0
  72. package/dist/esm/index.js +38 -0
  73. package/dist/esm/orb/mountOrb.d.ts +24 -0
  74. package/dist/esm/orb/mountOrb.js +691 -0
  75. package/dist/esm/orb/orbView.d.ts +56 -0
  76. package/dist/esm/orb/orbView.js +421 -0
  77. package/dist/esm/orb/publishedConfig.d.ts +32 -0
  78. package/dist/esm/orb/publishedConfig.js +67 -0
  79. package/dist/esm/package.json +1 -0
  80. package/dist/esm/speech/speechSession.d.ts +35 -0
  81. package/dist/esm/speech/speechSession.js +57 -0
  82. package/dist/esm/ui/messages.d.ts +36 -0
  83. package/dist/esm/ui/messages.js +38 -0
  84. package/package.json +38 -0
@@ -0,0 +1,590 @@
1
+ /**
2
+ * AgentController — the framework-free orchestrator.
3
+ *
4
+ * This is the vanilla port of @fiodos/react's `useAgent` hook: it assembles the
5
+ * @fiodos/core runtime (executor, consent, sanitizer, telemetry, messages,
6
+ * lexicon, conversation) ONCE, drives the listen → think → (confirm) → speak
7
+ * state machine, and exposes its state through a `subscribe` listener and
8
+ * imperative methods. Every framework binding (Vue, Svelte, Angular) wraps the
9
+ * SAME controller, so the security and reasoning behaviour is identical across
10
+ * frameworks and matches React / React Native.
11
+ *
12
+ * Security invariants (unchanged from the shared core):
13
+ * - The MANIFEST decides what needs confirmation, never the LLM.
14
+ * - Strict (payment-grade) confirmations still require the exact phrase or a
15
+ * deliberate modal tap; a loose "yes" never confirms them.
16
+ * - The agent only ever executes actions declared in the manifest.
17
+ */
18
+ import { combineContexts, createActionExecutor, createConsentManager, createSafeTelemetry, createSanitizer, createConversationSession, formatMessage, prepareConversationForTurn, recordConversationExchange, resolveConfirmationLexicon, resolveMessages, validateManifest, } from '@fiodos/core';
19
+ import { AgentApiError } from '../api/errors.js';
20
+ import { buildManifestPayload } from '../api/backendClient.js';
21
+ import { createScreenContextStore, } from '../context/screenContextStore.js';
22
+ import { createSpeechSession } from '../speech/speechSession.js';
23
+ import { classifyPendingReply, decideAction, } from '../core/turnEngine.js';
24
+ import { DEFAULT_AGENT_TIMINGS, } from '../config/types.js';
25
+ import { resolveUiMessages } from '../ui/messages.js';
26
+ function randomSessionId() {
27
+ return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
28
+ }
29
+ export class AgentController {
30
+ constructor(config) {
31
+ this.enabled = true;
32
+ this.phase = 'idle';
33
+ this.lastExchange = null;
34
+ this.confirmationMessage = null;
35
+ this.confirmationVoiceHint = null;
36
+ this.consentModalVisible = false;
37
+ this.sessionId = randomSessionId();
38
+ this.sessionStarted = false;
39
+ this.conversation = createConversationSession();
40
+ this.pending = null;
41
+ this.confirmationReasked = false;
42
+ this.confirmationVoiceActive = false;
43
+ this.silentTurn = false;
44
+ this.voiceBlocked = false;
45
+ this.abort = null;
46
+ this.watchdog = null;
47
+ this.confirmWindow = null;
48
+ this.pendingConsentIntent = null;
49
+ this.listeners = new Set();
50
+ this.unsubscribeSpeech = null;
51
+ // Fail fast on an invalid manifest — same contract as every platform.
52
+ validateManifest(config.manifest);
53
+ this.config = config;
54
+ this.messages = resolveMessages(config.locale, {
55
+ catalogs: config.messages?.catalogs,
56
+ overrides: config.messages?.overrides,
57
+ });
58
+ this.ui = resolveUiMessages(config.locale, config.uiMessages);
59
+ this.lexicon = config.confirmationLexicon ?? resolveConfirmationLexicon(config.locale);
60
+ this.sanitizer = config.sanitizer ?? createSanitizer();
61
+ this.telemetry = createSafeTelemetry(config.telemetry);
62
+ this.consent = createConsentManager({
63
+ storage: config.storage,
64
+ appId: config.manifest.appId,
65
+ version: config.consentVersion,
66
+ });
67
+ this.executor = createActionExecutor({
68
+ manifest: config.manifest,
69
+ registries: config.registries,
70
+ navigation: config.navigation,
71
+ messages: this.messages,
72
+ isAuthenticated: config.isAuthenticated,
73
+ });
74
+ this.timings = { ...DEFAULT_AGENT_TIMINGS, ...config.timings };
75
+ this.ttsLocale = config.ttsLocale ?? config.sttLocale;
76
+ this.notifyError =
77
+ config.notifyError ??
78
+ ((message) => {
79
+ // eslint-disable-next-line no-console
80
+ console.error(`[fyodos] ${message}`);
81
+ });
82
+ this.screenStore = createScreenContextStore({
83
+ getCurrentRoute: () => config.navigation.getCurrentRoute(),
84
+ routeFallback: config.routeContextFallback,
85
+ });
86
+ this.speech = createSpeechSession({
87
+ adapter: config.voice,
88
+ locale: config.sttLocale,
89
+ maxSeconds: this.timings.maxListeningSeconds,
90
+ onTranscript: (text) => this.onTranscript(text),
91
+ onError: (error) => this.onSpeechError(error),
92
+ });
93
+ // Re-emit combined state whenever the speech machine advances.
94
+ this.unsubscribeSpeech = this.speech.subscribe(() => this.emit());
95
+ }
96
+ // ── Subscription / state ───────────────────────────────────────────────────
97
+ getState() {
98
+ const speech = this.speech.getSnapshot();
99
+ const showWaveform = (this.phase === 'listening' || this.phase === 'confirming') && speech.state === 'recording';
100
+ const showThinking = this.phase === 'thinking' || (this.phase === 'listening' && speech.state === 'transcribing');
101
+ return {
102
+ phase: this.phase,
103
+ interimText: speech.interimText,
104
+ volumeLevel: speech.volumeLevel,
105
+ showWaveform,
106
+ showThinking,
107
+ isBusy: showThinking || this.phase === 'speaking',
108
+ isListening: this.phase === 'listening' && speech.state !== 'idle',
109
+ lastExchange: this.lastExchange,
110
+ confirmationMessage: this.confirmationMessage,
111
+ confirmationVoiceHint: this.confirmationVoiceHint,
112
+ consentModalVisible: this.consentModalVisible,
113
+ voiceBlocked: this.voiceBlocked,
114
+ };
115
+ }
116
+ /** Whether speech recognition can actually run (API present AND mic not denied). */
117
+ get voiceAvailable() {
118
+ return this.config.voice.isRecognitionAvailable() && !this.voiceBlocked;
119
+ }
120
+ /** Unlock audio playback inside a user gesture (orb tap) so the spoken reply
121
+ * is allowed/audible on mobile browsers. Must be called synchronously from
122
+ * the tap handler, before any await. No-op where unnecessary. */
123
+ primeAudio() {
124
+ this.config.voice.primePlayback?.();
125
+ }
126
+ /** Flag the mic as denied/unusable (called on a permission error or upfront
127
+ * permission detection) so the orb routes taps to the keyboard instead. */
128
+ markVoiceBlocked(blocked) {
129
+ if (this.voiceBlocked === blocked)
130
+ return;
131
+ this.voiceBlocked = blocked;
132
+ this.emit();
133
+ }
134
+ subscribe(listener) {
135
+ this.listeners.add(listener);
136
+ return () => {
137
+ this.listeners.delete(listener);
138
+ };
139
+ }
140
+ emit() {
141
+ const state = this.getState();
142
+ for (const l of this.listeners)
143
+ l(state);
144
+ }
145
+ setEnabled(value) {
146
+ this.enabled = value;
147
+ if (!value)
148
+ this.cancelAll();
149
+ }
150
+ getTtsVoice() {
151
+ return typeof this.config.ttsVoice === 'function'
152
+ ? this.config.ttsVoice()
153
+ : this.config.ttsVoice;
154
+ }
155
+ setPhase(next) {
156
+ const prev = this.phase;
157
+ if (prev === next)
158
+ return;
159
+ this.phase = next;
160
+ this.config.onPhaseChange?.(next, prev);
161
+ this.emit();
162
+ }
163
+ clearWatchdog() {
164
+ if (this.watchdog) {
165
+ clearTimeout(this.watchdog);
166
+ this.watchdog = null;
167
+ }
168
+ }
169
+ clearConfirmWindow() {
170
+ if (this.confirmWindow) {
171
+ clearTimeout(this.confirmWindow);
172
+ this.confirmWindow = null;
173
+ }
174
+ this.confirmationVoiceActive = false;
175
+ }
176
+ // ── Speech playback (ONE consistent voice) ─────────────────────────────────
177
+ async speakReply(text, audioBase64) {
178
+ if (this.silentTurn)
179
+ return;
180
+ if (!text && !audioBase64)
181
+ return;
182
+ this.setPhase('speaking');
183
+ try {
184
+ let audio = audioBase64;
185
+ if (!audio && text) {
186
+ audio = await this.config.backend.previewTts(this.getTtsVoice() ?? '', text);
187
+ }
188
+ if (audio) {
189
+ await this.config.voice.playAudioBase64Mp3(audio);
190
+ }
191
+ else if (text) {
192
+ await this.config.voice.speakWithDeviceTts(text, { locale: this.ttsLocale });
193
+ }
194
+ }
195
+ catch {
196
+ /* playback best-effort */
197
+ }
198
+ finally {
199
+ if (this.phase === 'speaking')
200
+ this.setPhase('idle');
201
+ }
202
+ }
203
+ // ── Confirmation resolution ─────────────────────────────────────────────────
204
+ beginConfirmationVoiceWindow() {
205
+ const pending = this.pending;
206
+ if (!pending || pending.silent || pending.voiceMode === 'disabled')
207
+ return;
208
+ if (!this.config.voice.isRecognitionAvailable())
209
+ return;
210
+ this.clearConfirmWindow();
211
+ this.setPhase('confirming');
212
+ this.confirmationVoiceActive = true;
213
+ void this.speech.start();
214
+ this.confirmWindow = setTimeout(() => {
215
+ this.speech.stop();
216
+ this.confirmationVoiceActive = false;
217
+ }, this.timings.confirmationVoiceWindowMs);
218
+ }
219
+ async confirmPendingAction() {
220
+ this.clearConfirmWindow();
221
+ const pending = this.pending;
222
+ if (!pending) {
223
+ this.confirmationMessage = null;
224
+ this.confirmationVoiceHint = null;
225
+ this.emit();
226
+ return;
227
+ }
228
+ // Double confirmation: ask a second time before executing.
229
+ if (pending.remainingConfirmations > 1) {
230
+ pending.remainingConfirmations -= 1;
231
+ this.confirmationReasked = false;
232
+ const second = this.messages.doubleConfirmationQuestion;
233
+ this.confirmationMessage = second;
234
+ if (this.lastExchange)
235
+ this.lastExchange = { ...this.lastExchange, replyText: second };
236
+ this.emit();
237
+ await this.speakReply(second, null);
238
+ if (this.pending)
239
+ this.beginConfirmationVoiceWindow();
240
+ return;
241
+ }
242
+ this.pending = null;
243
+ this.confirmationMessage = null;
244
+ this.confirmationVoiceHint = null;
245
+ if (this.phase === 'confirming')
246
+ this.setPhase('idle');
247
+ this.emit();
248
+ const result = await this.executor.execute(pending.action, pending.context);
249
+ this.telemetry.logEvent({
250
+ eventType: result.success ? 'action_executed' : 'action_failed',
251
+ intentDetected: pending.intent,
252
+ sessionId: this.sessionId,
253
+ actionExecuted: { intent: pending.intent, confirmed: true },
254
+ });
255
+ if (result.message) {
256
+ if (this.lastExchange)
257
+ this.lastExchange = { ...this.lastExchange, replyText: result.message };
258
+ this.emit();
259
+ await this.speakReply(result.message, null);
260
+ }
261
+ }
262
+ cancelPendingAction() {
263
+ this.clearConfirmWindow();
264
+ const pending = this.pending;
265
+ this.pending = null;
266
+ this.confirmationMessage = null;
267
+ this.confirmationVoiceHint = null;
268
+ if (this.phase === 'confirming')
269
+ this.setPhase('idle');
270
+ this.emit();
271
+ if (!pending)
272
+ return;
273
+ this.telemetry.logEvent({
274
+ eventType: 'action_cancelled',
275
+ intentDetected: pending.intent,
276
+ sessionId: this.sessionId,
277
+ });
278
+ if (this.lastExchange)
279
+ this.lastExchange = { ...this.lastExchange, replyText: this.messages.cancelled };
280
+ this.emit();
281
+ void this.speakReply(this.messages.cancelled, null);
282
+ }
283
+ async handleConfirmationTranscript(text) {
284
+ this.confirmationVoiceActive = false;
285
+ this.clearConfirmWindow();
286
+ const pending = this.pending;
287
+ if (this.phase === 'confirming')
288
+ this.setPhase('idle');
289
+ if (!pending)
290
+ return;
291
+ // Tap-only confirmations can never be resolved by a stray transcript.
292
+ if (pending.voiceMode === 'disabled' || pending.silent)
293
+ return;
294
+ const reply = classifyPendingReply(text, pending, this.lexicon, { silent: pending.silent });
295
+ if (reply === 'affirmative') {
296
+ await this.confirmPendingAction();
297
+ return;
298
+ }
299
+ if (reply === 'negative') {
300
+ this.cancelPendingAction();
301
+ return;
302
+ }
303
+ // Ambiguous: re-ask ONCE, then fall back to modal-only.
304
+ if (!this.confirmationReasked) {
305
+ this.confirmationReasked = true;
306
+ const reask = `${this.messages.confirmationNotUnderstood} ${pending.question}`.trim();
307
+ this.confirmationMessage = reask;
308
+ this.emit();
309
+ await this.speakReply(reask, null);
310
+ if (this.pending)
311
+ this.beginConfirmationVoiceWindow();
312
+ }
313
+ }
314
+ // ── Main turn ────────────────────────────────────────────────────────────────
315
+ async handleTranscript(rawText) {
316
+ const text = rawText.trim();
317
+ if (!this.enabled || text.length < this.timings.minTranscriptChars) {
318
+ if (this.phase === 'thinking')
319
+ this.setPhase('idle');
320
+ return;
321
+ }
322
+ this.setPhase('thinking');
323
+ this.clearWatchdog();
324
+ this.watchdog = setTimeout(() => {
325
+ if (this.phase === 'thinking') {
326
+ this.telemetry.logEvent({ eventType: 'agent_watchdog_reset', sessionId: this.sessionId });
327
+ this.setPhase('idle');
328
+ }
329
+ }, this.timings.thinkingWatchdogMs);
330
+ const currentRoute = this.config.navigation.getCurrentRoute();
331
+ if (!this.sessionStarted) {
332
+ this.sessionStarted = true;
333
+ this.telemetry.logEvent({ eventType: 'agent_session_started', sessionId: this.sessionId });
334
+ }
335
+ this.telemetry.logEvent({
336
+ eventType: 'agent_invoked',
337
+ sessionId: this.sessionId,
338
+ currentRoute,
339
+ });
340
+ const snapshot = this.screenStore.getSnapshot();
341
+ const rawContext = combineContexts(this.config.getUserContextText?.() ?? null, this.screenStore.getText(), {
342
+ truncationSuffix: this.messages.contextTruncationSuffix,
343
+ maxChars: this.config.contextMaxChars,
344
+ });
345
+ const sanitized = rawContext ? this.sanitizer(rawContext).sanitized : undefined;
346
+ const conv = prepareConversationForTurn(this.conversation);
347
+ const controller = new AbortController();
348
+ this.abort = controller;
349
+ let reply = '';
350
+ let audioBase64 = null;
351
+ try {
352
+ const turn = await this.config.backend.sendTurn({
353
+ message: text,
354
+ language: this.config.locale,
355
+ voice: this.getTtsVoice(),
356
+ screenContext: sanitized,
357
+ currentRoute: currentRoute ?? undefined,
358
+ conversationHistory: conv.history,
359
+ conversationSummary: conv.summary ?? undefined,
360
+ sessionId: this.sessionId,
361
+ ...buildManifestPayload(this.config.manifest),
362
+ }, { signal: controller.signal });
363
+ reply = turn.reply;
364
+ audioBase64 = turn.audioBase64;
365
+ const decision = await decideAction({
366
+ action: turn.action,
367
+ manifest: this.config.manifest,
368
+ executor: this.executor,
369
+ context: snapshot?.context ?? {},
370
+ messages: this.messages,
371
+ userMessage: text,
372
+ });
373
+ if (decision.kind === 'executed') {
374
+ const r = decision.result;
375
+ if (turn.action?.type === 'navigate') {
376
+ this.telemetry.logEvent({
377
+ eventType: r.success ? 'navigation_executed' : 'navigation_failed',
378
+ intentDetected: turn.action.intent,
379
+ sessionId: this.sessionId,
380
+ currentRoute,
381
+ });
382
+ }
383
+ else if (turn.action?.type === 'execute') {
384
+ this.telemetry.logEvent({
385
+ eventType: r.recoverable
386
+ ? 'action_needs_context'
387
+ : r.success
388
+ ? 'action_executed'
389
+ : 'action_failed',
390
+ intentDetected: turn.action.intent,
391
+ sessionId: this.sessionId,
392
+ actionExecuted: { intent: turn.action.intent, confirmed: false },
393
+ });
394
+ }
395
+ if (r.recoverable) {
396
+ /* keep the assistant's guiding reply; never speak a refusal. */
397
+ }
398
+ else if (r.message) {
399
+ reply = r.message;
400
+ audioBase64 = null;
401
+ }
402
+ }
403
+ else if (decision.kind === 'precheck-resolved') {
404
+ if (decision.result.message && !decision.result.recoverable) {
405
+ reply = decision.result.message;
406
+ audioBase64 = null;
407
+ }
408
+ }
409
+ else if (decision.kind === 'needs-confirmation') {
410
+ this.pending = { ...decision.pending, silent: this.silentTurn };
411
+ this.confirmationReasked = false;
412
+ this.confirmationMessage = decision.pending.question;
413
+ this.confirmationVoiceHint =
414
+ decision.pending.voiceMode === 'strict' &&
415
+ decision.pending.strictPhrase &&
416
+ !this.silentTurn
417
+ ? formatMessage(this.messages.strictConfirmationModalHint, {
418
+ phrase: decision.pending.strictPhrase,
419
+ })
420
+ : null;
421
+ this.emit();
422
+ this.telemetry.logEvent({
423
+ eventType: 'action_confirmation_requested',
424
+ intentDetected: decision.pending.intent,
425
+ sessionId: this.sessionId,
426
+ });
427
+ }
428
+ }
429
+ catch (e) {
430
+ this.clearWatchdog();
431
+ this.abort = null;
432
+ if (e instanceof AgentApiError && e.code === 'cancelled') {
433
+ if (this.phase === 'thinking')
434
+ this.setPhase('idle');
435
+ return;
436
+ }
437
+ if (e instanceof AgentApiError && e.code === 'quota_exceeded') {
438
+ this.config.onQuotaExceeded?.();
439
+ }
440
+ if (this.phase === 'thinking')
441
+ this.setPhase('idle');
442
+ this.notifyError(reply || (e instanceof Error ? e.message : 'error'));
443
+ return;
444
+ }
445
+ this.clearWatchdog();
446
+ this.abort = null;
447
+ if (reply) {
448
+ this.lastExchange = { userText: text, replyText: reply };
449
+ this.emit();
450
+ recordConversationExchange(this.conversation, text, reply);
451
+ }
452
+ const hasPending = this.pending != null;
453
+ await this.speakReply(reply, audioBase64);
454
+ if (hasPending) {
455
+ this.beginConfirmationVoiceWindow();
456
+ }
457
+ else if (this.phase !== 'speaking') {
458
+ this.setPhase('idle');
459
+ }
460
+ this.config.onTurnCompleted?.();
461
+ }
462
+ // ── Speech session wiring ────────────────────────────────────────────────────
463
+ onTranscript(text) {
464
+ if (this.confirmationVoiceActive || this.pending) {
465
+ void this.handleConfirmationTranscript(text);
466
+ return;
467
+ }
468
+ void this.handleTranscript(text);
469
+ }
470
+ onSpeechError(error) {
471
+ this.telemetry.logEvent({ eventType: 'agent_stt_failed', sessionId: this.sessionId });
472
+ if (this.phase === 'listening' || this.phase === 'confirming')
473
+ this.setPhase('idle');
474
+ const msg = error instanceof Error ? error.message : String(error);
475
+ // A permission/hardware error means voice can't run: flag it so taps route
476
+ // to the keyboard (the binding reacts to `voiceBlocked` to open the bubble).
477
+ const lower = msg.toLowerCase();
478
+ if (lower.includes('not-allowed') ||
479
+ lower.includes('not allowed') ||
480
+ lower.includes('service-not-allowed') ||
481
+ lower.includes('audio-capture') ||
482
+ lower.includes('permission')) {
483
+ this.markVoiceBlocked(true);
484
+ return;
485
+ }
486
+ this.notifyError(msg);
487
+ }
488
+ // ── Consent + listening ──────────────────────────────────────────────────────
489
+ async beginListening() {
490
+ if (!this.enabled)
491
+ return;
492
+ if (!this.config.voice.isRecognitionAvailable()) {
493
+ this.telemetry.logEvent({ eventType: 'agent_stt_failed', sessionId: this.sessionId });
494
+ this.notifyError(this.ui.voiceUnavailable);
495
+ return;
496
+ }
497
+ const gate = this.config.gateTurn?.();
498
+ if (gate && !gate.ok) {
499
+ if (gate.message)
500
+ this.notifyError(gate.message);
501
+ return;
502
+ }
503
+ this.config.voice.stopPlayback();
504
+ this.silentTurn = false;
505
+ this.setPhase('listening');
506
+ await this.speech.start();
507
+ }
508
+ async toggleListening() {
509
+ if (!this.enabled)
510
+ return;
511
+ if (this.phase === 'confirming')
512
+ return; // modal owns this phase
513
+ if (this.phase === 'listening') {
514
+ this.speech.stop();
515
+ this.setPhase('thinking');
516
+ return;
517
+ }
518
+ if (this.phase === 'thinking' || this.phase === 'speaking')
519
+ return;
520
+ const consented = await this.consent.hasConsented();
521
+ if (!consented) {
522
+ this.pendingConsentIntent = 'voice';
523
+ this.consentModalVisible = true;
524
+ this.emit();
525
+ return;
526
+ }
527
+ await this.beginListening();
528
+ }
529
+ async submitTextTurn(text, options) {
530
+ if (!this.enabled)
531
+ return;
532
+ const trimmed = text.trim();
533
+ if (!trimmed)
534
+ return;
535
+ const gate = this.config.gateTurn?.();
536
+ if (gate && !gate.ok) {
537
+ if (gate.message)
538
+ this.notifyError(gate.message);
539
+ return;
540
+ }
541
+ // A pending confirmation steals the next text turn (so typed "yes" resolves
542
+ // it instead of starting a fresh turn). Honours the same voice-mode rules.
543
+ if (this.pending && !this.pending.silent) {
544
+ await this.handleConfirmationTranscript(trimmed);
545
+ return;
546
+ }
547
+ this.silentTurn = options?.speak !== true;
548
+ await this.handleTranscript(trimmed);
549
+ }
550
+ async acceptConsent() {
551
+ await this.consent.grant();
552
+ this.consentModalVisible = false;
553
+ this.emit();
554
+ this.telemetry.logEvent({ eventType: 'agent_consent_granted', sessionId: this.sessionId });
555
+ const intent = this.pendingConsentIntent;
556
+ this.pendingConsentIntent = null;
557
+ if (intent === 'voice')
558
+ await this.beginListening();
559
+ }
560
+ declineConsent() {
561
+ this.consentModalVisible = false;
562
+ this.pendingConsentIntent = null;
563
+ this.emit();
564
+ this.telemetry.logEvent({ eventType: 'agent_consent_declined', sessionId: this.sessionId });
565
+ }
566
+ cancelAll() {
567
+ this.abort?.abort();
568
+ this.abort = null;
569
+ this.clearWatchdog();
570
+ this.clearConfirmWindow();
571
+ this.speech.cancel();
572
+ this.config.voice.stopPlayback();
573
+ this.pending = null;
574
+ this.confirmationMessage = null;
575
+ this.confirmationVoiceHint = null;
576
+ this.setPhase('idle');
577
+ this.emit();
578
+ }
579
+ /** Best-effort backend warm-up (cold-start mitigation). */
580
+ warmUp() {
581
+ void this.config.backend.warmUp?.();
582
+ }
583
+ dispose() {
584
+ this.cancelAll();
585
+ this.unsubscribeSpeech?.();
586
+ this.unsubscribeSpeech = null;
587
+ this.speech.dispose();
588
+ this.listeners.clear();
589
+ }
590
+ }
@@ -0,0 +1,70 @@
1
+ /**
2
+ * Framework-free turn decision engine.
3
+ *
4
+ * Concentrates the SECURITY-CRITICAL decision logic of a turn so it can be
5
+ * unit-tested without any framework, and so every web binding (Vue, Svelte,
6
+ * Angular) and React Native share the exact same rules. It does NOT
7
+ * reimplement the engine: it delegates execution, prechecks and confirmation
8
+ * classification to @fiodos/core. It only sequences them.
9
+ *
10
+ * Invariants enforced here (identical to @fiodos/react-native):
11
+ * 1. The source of truth for "needs confirmation" is the MANIFEST
12
+ * (`requireConfirmation`), never the LLM's flag.
13
+ * 2. A sensitive action is NEVER executed by `decideAction`; it returns a
14
+ * PendingConfirmation and waits for an explicit confirmation.
15
+ * 3. Voice confirmation honours the manifest mode: 'disabled' can never be
16
+ * confirmed by voice (tap only); 'strict' requires the exact phrase; a
17
+ * loose "yes" does not confirm a strict action. Negation always wins.
18
+ */
19
+ import { type ActionExecutor, type ActionResult, type AgentAction, type AgentMessages, type AgentScreenContext, type AppManifest, type ConfirmationLexicon, type ConfirmationReply } from '@fiodos/core';
20
+ export type VoiceConfirmationMode = 'standard' | 'strict' | 'disabled';
21
+ export interface PendingConfirmation {
22
+ action: AgentAction;
23
+ context: AgentScreenContext;
24
+ /** Canonical question to show/speak. */
25
+ question: string;
26
+ voiceMode: VoiceConfirmationMode;
27
+ /** Strict-mode exact phrase, or null when not strict. */
28
+ strictPhrase: string | null;
29
+ /** Remaining confirmations (2 for doubleConfirmation, else 1). */
30
+ remainingConfirmations: number;
31
+ /** Original user message that triggered the action (telemetry). */
32
+ userMessage: string;
33
+ intent: string;
34
+ }
35
+ export type ActionDecision = {
36
+ kind: 'executed';
37
+ result: ActionResult;
38
+ } | {
39
+ kind: 'precheck-resolved';
40
+ result: ActionResult;
41
+ } | {
42
+ kind: 'needs-confirmation';
43
+ pending: PendingConfirmation;
44
+ } | {
45
+ kind: 'noop';
46
+ };
47
+ export interface DecideActionParams {
48
+ action: AgentAction | null;
49
+ manifest: AppManifest;
50
+ executor: ActionExecutor;
51
+ /** Structured screen snapshot captured at invocation time. */
52
+ context: AgentScreenContext;
53
+ messages: AgentMessages;
54
+ userMessage: string;
55
+ }
56
+ /**
57
+ * Decides what to do with a backend-proposed action. Sensitive actions are
58
+ * never executed here — they return `needs-confirmation`.
59
+ */
60
+ export declare function decideAction(params: DecideActionParams): Promise<ActionDecision>;
61
+ /**
62
+ * Classifies a spoken reply against a pending confirmation, honouring the
63
+ * manifest's voice mode. Returns 'ambiguous' when voice cannot resolve it
64
+ * (disabled mode, silent turn, or unclear speech) — the caller must then rely
65
+ * on the tactile modal. NEVER returns 'affirmative' for a disabled/silent
66
+ * confirmation.
67
+ */
68
+ export declare function classifyPendingReply(text: string, pending: PendingConfirmation, lexicon: ConfirmationLexicon, opts?: {
69
+ silent?: boolean;
70
+ }): ConfirmationReply;