@fiodos/web-core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +77 -0
- package/dist/cjs/adapters/webNavigationAdapter.d.ts +31 -0
- package/dist/cjs/adapters/webNavigationAdapter.js +45 -0
- package/dist/cjs/adapters/webStorageAdapter.d.ts +13 -0
- package/dist/cjs/adapters/webStorageAdapter.js +57 -0
- package/dist/cjs/adapters/webVoiceAdapter.d.ts +16 -0
- package/dist/cjs/adapters/webVoiceAdapter.js +477 -0
- package/dist/cjs/api/backendClient.d.ts +31 -0
- package/dist/cjs/api/backendClient.js +182 -0
- package/dist/cjs/api/backendTelemetry.d.ts +23 -0
- package/dist/cjs/api/backendTelemetry.js +80 -0
- package/dist/cjs/api/clientBootstrap.d.ts +44 -0
- package/dist/cjs/api/clientBootstrap.js +68 -0
- package/dist/cjs/api/errors.d.ts +11 -0
- package/dist/cjs/api/errors.js +16 -0
- package/dist/cjs/bridge/createBridge.d.ts +13 -0
- package/dist/cjs/bridge/createBridge.js +15 -0
- package/dist/cjs/config/types.d.ts +63 -0
- package/dist/cjs/config/types.js +9 -0
- package/dist/cjs/context/screenContextStore.d.ts +26 -0
- package/dist/cjs/context/screenContextStore.js +25 -0
- package/dist/cjs/controller/AgentController.d.ts +115 -0
- package/dist/cjs/controller/AgentController.js +594 -0
- package/dist/cjs/core/turnEngine.d.ts +70 -0
- package/dist/cjs/core/turnEngine.js +96 -0
- package/dist/cjs/dropin/createFiodosAgent.d.ts +60 -0
- package/dist/cjs/dropin/createFiodosAgent.js +143 -0
- package/dist/cjs/dropin/createFyodosAgent.d.ts +60 -0
- package/dist/cjs/dropin/createFyodosAgent.js +143 -0
- package/dist/cjs/index.d.ts +45 -0
- package/dist/cjs/index.js +67 -0
- package/dist/cjs/orb/mountOrb.d.ts +24 -0
- package/dist/cjs/orb/mountOrb.js +694 -0
- package/dist/cjs/orb/orbView.d.ts +56 -0
- package/dist/cjs/orb/orbView.js +426 -0
- package/dist/cjs/orb/publishedConfig.d.ts +32 -0
- package/dist/cjs/orb/publishedConfig.js +70 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/speech/speechSession.d.ts +35 -0
- package/dist/cjs/speech/speechSession.js +60 -0
- package/dist/cjs/ui/messages.d.ts +36 -0
- package/dist/cjs/ui/messages.js +41 -0
- package/dist/esm/adapters/webNavigationAdapter.d.ts +31 -0
- package/dist/esm/adapters/webNavigationAdapter.js +41 -0
- package/dist/esm/adapters/webStorageAdapter.d.ts +13 -0
- package/dist/esm/adapters/webStorageAdapter.js +54 -0
- package/dist/esm/adapters/webVoiceAdapter.d.ts +16 -0
- package/dist/esm/adapters/webVoiceAdapter.js +474 -0
- package/dist/esm/api/backendClient.d.ts +31 -0
- package/dist/esm/api/backendClient.js +178 -0
- package/dist/esm/api/backendTelemetry.d.ts +23 -0
- package/dist/esm/api/backendTelemetry.js +77 -0
- package/dist/esm/api/clientBootstrap.d.ts +44 -0
- package/dist/esm/api/clientBootstrap.js +63 -0
- package/dist/esm/api/errors.d.ts +11 -0
- package/dist/esm/api/errors.js +11 -0
- package/dist/esm/bridge/createBridge.d.ts +13 -0
- package/dist/esm/bridge/createBridge.js +12 -0
- package/dist/esm/config/types.d.ts +63 -0
- package/dist/esm/config/types.js +6 -0
- package/dist/esm/context/screenContextStore.d.ts +26 -0
- package/dist/esm/context/screenContextStore.js +22 -0
- package/dist/esm/controller/AgentController.d.ts +115 -0
- package/dist/esm/controller/AgentController.js +590 -0
- package/dist/esm/core/turnEngine.d.ts +70 -0
- package/dist/esm/core/turnEngine.js +92 -0
- package/dist/esm/dropin/createFiodosAgent.d.ts +60 -0
- package/dist/esm/dropin/createFiodosAgent.js +140 -0
- package/dist/esm/dropin/createFyodosAgent.d.ts +60 -0
- package/dist/esm/dropin/createFyodosAgent.js +140 -0
- package/dist/esm/index.d.ts +45 -0
- package/dist/esm/index.js +38 -0
- package/dist/esm/orb/mountOrb.d.ts +24 -0
- package/dist/esm/orb/mountOrb.js +691 -0
- package/dist/esm/orb/orbView.d.ts +56 -0
- package/dist/esm/orb/orbView.js +421 -0
- package/dist/esm/orb/publishedConfig.d.ts +32 -0
- package/dist/esm/orb/publishedConfig.js +67 -0
- package/dist/esm/package.json +1 -0
- package/dist/esm/speech/speechSession.d.ts +35 -0
- package/dist/esm/speech/speechSession.js +57 -0
- package/dist/esm/ui/messages.d.ts +36 -0
- package/dist/esm/ui/messages.js +38 -0
- package/package.json +38 -0
|
@@ -0,0 +1,594 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AgentController = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* AgentController — the framework-free orchestrator.
|
|
6
|
+
*
|
|
7
|
+
* This is the vanilla port of @fiodos/react's `useAgent` hook: it assembles the
|
|
8
|
+
* @fiodos/core runtime (executor, consent, sanitizer, telemetry, messages,
|
|
9
|
+
* lexicon, conversation) ONCE, drives the listen → think → (confirm) → speak
|
|
10
|
+
* state machine, and exposes its state through a `subscribe` listener and
|
|
11
|
+
* imperative methods. Every framework binding (Vue, Svelte, Angular) wraps the
|
|
12
|
+
* SAME controller, so the security and reasoning behaviour is identical across
|
|
13
|
+
* frameworks and matches React / React Native.
|
|
14
|
+
*
|
|
15
|
+
* Security invariants (unchanged from the shared core):
|
|
16
|
+
* - The MANIFEST decides what needs confirmation, never the LLM.
|
|
17
|
+
* - Strict (payment-grade) confirmations still require the exact phrase or a
|
|
18
|
+
* deliberate modal tap; a loose "yes" never confirms them.
|
|
19
|
+
* - The agent only ever executes actions declared in the manifest.
|
|
20
|
+
*/
|
|
21
|
+
const core_1 = require("@fiodos/core");
|
|
22
|
+
const errors_1 = require("../api/errors");
|
|
23
|
+
const backendClient_1 = require("../api/backendClient");
|
|
24
|
+
const screenContextStore_1 = require("../context/screenContextStore");
|
|
25
|
+
const speechSession_1 = require("../speech/speechSession");
|
|
26
|
+
const turnEngine_1 = require("../core/turnEngine");
|
|
27
|
+
const types_1 = require("../config/types");
|
|
28
|
+
const messages_1 = require("../ui/messages");
|
|
29
|
+
function randomSessionId() {
|
|
30
|
+
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
|
31
|
+
}
|
|
32
|
+
class AgentController {
|
|
33
|
+
constructor(config) {
|
|
34
|
+
this.enabled = true;
|
|
35
|
+
this.phase = 'idle';
|
|
36
|
+
this.lastExchange = null;
|
|
37
|
+
this.confirmationMessage = null;
|
|
38
|
+
this.confirmationVoiceHint = null;
|
|
39
|
+
this.consentModalVisible = false;
|
|
40
|
+
this.sessionId = randomSessionId();
|
|
41
|
+
this.sessionStarted = false;
|
|
42
|
+
this.conversation = (0, core_1.createConversationSession)();
|
|
43
|
+
this.pending = null;
|
|
44
|
+
this.confirmationReasked = false;
|
|
45
|
+
this.confirmationVoiceActive = false;
|
|
46
|
+
this.silentTurn = false;
|
|
47
|
+
this.voiceBlocked = false;
|
|
48
|
+
this.abort = null;
|
|
49
|
+
this.watchdog = null;
|
|
50
|
+
this.confirmWindow = null;
|
|
51
|
+
this.pendingConsentIntent = null;
|
|
52
|
+
this.listeners = new Set();
|
|
53
|
+
this.unsubscribeSpeech = null;
|
|
54
|
+
// Fail fast on an invalid manifest — same contract as every platform.
|
|
55
|
+
(0, core_1.validateManifest)(config.manifest);
|
|
56
|
+
this.config = config;
|
|
57
|
+
this.messages = (0, core_1.resolveMessages)(config.locale, {
|
|
58
|
+
catalogs: config.messages?.catalogs,
|
|
59
|
+
overrides: config.messages?.overrides,
|
|
60
|
+
});
|
|
61
|
+
this.ui = (0, messages_1.resolveUiMessages)(config.locale, config.uiMessages);
|
|
62
|
+
this.lexicon = config.confirmationLexicon ?? (0, core_1.resolveConfirmationLexicon)(config.locale);
|
|
63
|
+
this.sanitizer = config.sanitizer ?? (0, core_1.createSanitizer)();
|
|
64
|
+
this.telemetry = (0, core_1.createSafeTelemetry)(config.telemetry);
|
|
65
|
+
this.consent = (0, core_1.createConsentManager)({
|
|
66
|
+
storage: config.storage,
|
|
67
|
+
appId: config.manifest.appId,
|
|
68
|
+
version: config.consentVersion,
|
|
69
|
+
});
|
|
70
|
+
this.executor = (0, core_1.createActionExecutor)({
|
|
71
|
+
manifest: config.manifest,
|
|
72
|
+
registries: config.registries,
|
|
73
|
+
navigation: config.navigation,
|
|
74
|
+
messages: this.messages,
|
|
75
|
+
isAuthenticated: config.isAuthenticated,
|
|
76
|
+
});
|
|
77
|
+
this.timings = { ...types_1.DEFAULT_AGENT_TIMINGS, ...config.timings };
|
|
78
|
+
this.ttsLocale = config.ttsLocale ?? config.sttLocale;
|
|
79
|
+
this.notifyError =
|
|
80
|
+
config.notifyError ??
|
|
81
|
+
((message) => {
|
|
82
|
+
// eslint-disable-next-line no-console
|
|
83
|
+
console.error(`[fyodos] ${message}`);
|
|
84
|
+
});
|
|
85
|
+
this.screenStore = (0, screenContextStore_1.createScreenContextStore)({
|
|
86
|
+
getCurrentRoute: () => config.navigation.getCurrentRoute(),
|
|
87
|
+
routeFallback: config.routeContextFallback,
|
|
88
|
+
});
|
|
89
|
+
this.speech = (0, speechSession_1.createSpeechSession)({
|
|
90
|
+
adapter: config.voice,
|
|
91
|
+
locale: config.sttLocale,
|
|
92
|
+
maxSeconds: this.timings.maxListeningSeconds,
|
|
93
|
+
onTranscript: (text) => this.onTranscript(text),
|
|
94
|
+
onError: (error) => this.onSpeechError(error),
|
|
95
|
+
});
|
|
96
|
+
// Re-emit combined state whenever the speech machine advances.
|
|
97
|
+
this.unsubscribeSpeech = this.speech.subscribe(() => this.emit());
|
|
98
|
+
}
|
|
99
|
+
// ── Subscription / state ───────────────────────────────────────────────────
|
|
100
|
+
getState() {
|
|
101
|
+
const speech = this.speech.getSnapshot();
|
|
102
|
+
const showWaveform = (this.phase === 'listening' || this.phase === 'confirming') && speech.state === 'recording';
|
|
103
|
+
const showThinking = this.phase === 'thinking' || (this.phase === 'listening' && speech.state === 'transcribing');
|
|
104
|
+
return {
|
|
105
|
+
phase: this.phase,
|
|
106
|
+
interimText: speech.interimText,
|
|
107
|
+
volumeLevel: speech.volumeLevel,
|
|
108
|
+
showWaveform,
|
|
109
|
+
showThinking,
|
|
110
|
+
isBusy: showThinking || this.phase === 'speaking',
|
|
111
|
+
isListening: this.phase === 'listening' && speech.state !== 'idle',
|
|
112
|
+
lastExchange: this.lastExchange,
|
|
113
|
+
confirmationMessage: this.confirmationMessage,
|
|
114
|
+
confirmationVoiceHint: this.confirmationVoiceHint,
|
|
115
|
+
consentModalVisible: this.consentModalVisible,
|
|
116
|
+
voiceBlocked: this.voiceBlocked,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
/** Whether speech recognition can actually run (API present AND mic not denied). */
|
|
120
|
+
get voiceAvailable() {
|
|
121
|
+
return this.config.voice.isRecognitionAvailable() && !this.voiceBlocked;
|
|
122
|
+
}
|
|
123
|
+
/** Unlock audio playback inside a user gesture (orb tap) so the spoken reply
|
|
124
|
+
* is allowed/audible on mobile browsers. Must be called synchronously from
|
|
125
|
+
* the tap handler, before any await. No-op where unnecessary. */
|
|
126
|
+
primeAudio() {
|
|
127
|
+
this.config.voice.primePlayback?.();
|
|
128
|
+
}
|
|
129
|
+
/** Flag the mic as denied/unusable (called on a permission error or upfront
|
|
130
|
+
* permission detection) so the orb routes taps to the keyboard instead. */
|
|
131
|
+
markVoiceBlocked(blocked) {
|
|
132
|
+
if (this.voiceBlocked === blocked)
|
|
133
|
+
return;
|
|
134
|
+
this.voiceBlocked = blocked;
|
|
135
|
+
this.emit();
|
|
136
|
+
}
|
|
137
|
+
subscribe(listener) {
|
|
138
|
+
this.listeners.add(listener);
|
|
139
|
+
return () => {
|
|
140
|
+
this.listeners.delete(listener);
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
emit() {
|
|
144
|
+
const state = this.getState();
|
|
145
|
+
for (const l of this.listeners)
|
|
146
|
+
l(state);
|
|
147
|
+
}
|
|
148
|
+
setEnabled(value) {
|
|
149
|
+
this.enabled = value;
|
|
150
|
+
if (!value)
|
|
151
|
+
this.cancelAll();
|
|
152
|
+
}
|
|
153
|
+
getTtsVoice() {
|
|
154
|
+
return typeof this.config.ttsVoice === 'function'
|
|
155
|
+
? this.config.ttsVoice()
|
|
156
|
+
: this.config.ttsVoice;
|
|
157
|
+
}
|
|
158
|
+
setPhase(next) {
|
|
159
|
+
const prev = this.phase;
|
|
160
|
+
if (prev === next)
|
|
161
|
+
return;
|
|
162
|
+
this.phase = next;
|
|
163
|
+
this.config.onPhaseChange?.(next, prev);
|
|
164
|
+
this.emit();
|
|
165
|
+
}
|
|
166
|
+
clearWatchdog() {
|
|
167
|
+
if (this.watchdog) {
|
|
168
|
+
clearTimeout(this.watchdog);
|
|
169
|
+
this.watchdog = null;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
clearConfirmWindow() {
|
|
173
|
+
if (this.confirmWindow) {
|
|
174
|
+
clearTimeout(this.confirmWindow);
|
|
175
|
+
this.confirmWindow = null;
|
|
176
|
+
}
|
|
177
|
+
this.confirmationVoiceActive = false;
|
|
178
|
+
}
|
|
179
|
+
// ── Speech playback (ONE consistent voice) ─────────────────────────────────
|
|
180
|
+
async speakReply(text, audioBase64) {
|
|
181
|
+
if (this.silentTurn)
|
|
182
|
+
return;
|
|
183
|
+
if (!text && !audioBase64)
|
|
184
|
+
return;
|
|
185
|
+
this.setPhase('speaking');
|
|
186
|
+
try {
|
|
187
|
+
let audio = audioBase64;
|
|
188
|
+
if (!audio && text) {
|
|
189
|
+
audio = await this.config.backend.previewTts(this.getTtsVoice() ?? '', text);
|
|
190
|
+
}
|
|
191
|
+
if (audio) {
|
|
192
|
+
await this.config.voice.playAudioBase64Mp3(audio);
|
|
193
|
+
}
|
|
194
|
+
else if (text) {
|
|
195
|
+
await this.config.voice.speakWithDeviceTts(text, { locale: this.ttsLocale });
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
catch {
|
|
199
|
+
/* playback best-effort */
|
|
200
|
+
}
|
|
201
|
+
finally {
|
|
202
|
+
if (this.phase === 'speaking')
|
|
203
|
+
this.setPhase('idle');
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
// ── Confirmation resolution ─────────────────────────────────────────────────
|
|
207
|
+
beginConfirmationVoiceWindow() {
|
|
208
|
+
const pending = this.pending;
|
|
209
|
+
if (!pending || pending.silent || pending.voiceMode === 'disabled')
|
|
210
|
+
return;
|
|
211
|
+
if (!this.config.voice.isRecognitionAvailable())
|
|
212
|
+
return;
|
|
213
|
+
this.clearConfirmWindow();
|
|
214
|
+
this.setPhase('confirming');
|
|
215
|
+
this.confirmationVoiceActive = true;
|
|
216
|
+
void this.speech.start();
|
|
217
|
+
this.confirmWindow = setTimeout(() => {
|
|
218
|
+
this.speech.stop();
|
|
219
|
+
this.confirmationVoiceActive = false;
|
|
220
|
+
}, this.timings.confirmationVoiceWindowMs);
|
|
221
|
+
}
|
|
222
|
+
async confirmPendingAction() {
|
|
223
|
+
this.clearConfirmWindow();
|
|
224
|
+
const pending = this.pending;
|
|
225
|
+
if (!pending) {
|
|
226
|
+
this.confirmationMessage = null;
|
|
227
|
+
this.confirmationVoiceHint = null;
|
|
228
|
+
this.emit();
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
// Double confirmation: ask a second time before executing.
|
|
232
|
+
if (pending.remainingConfirmations > 1) {
|
|
233
|
+
pending.remainingConfirmations -= 1;
|
|
234
|
+
this.confirmationReasked = false;
|
|
235
|
+
const second = this.messages.doubleConfirmationQuestion;
|
|
236
|
+
this.confirmationMessage = second;
|
|
237
|
+
if (this.lastExchange)
|
|
238
|
+
this.lastExchange = { ...this.lastExchange, replyText: second };
|
|
239
|
+
this.emit();
|
|
240
|
+
await this.speakReply(second, null);
|
|
241
|
+
if (this.pending)
|
|
242
|
+
this.beginConfirmationVoiceWindow();
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
this.pending = null;
|
|
246
|
+
this.confirmationMessage = null;
|
|
247
|
+
this.confirmationVoiceHint = null;
|
|
248
|
+
if (this.phase === 'confirming')
|
|
249
|
+
this.setPhase('idle');
|
|
250
|
+
this.emit();
|
|
251
|
+
const result = await this.executor.execute(pending.action, pending.context);
|
|
252
|
+
this.telemetry.logEvent({
|
|
253
|
+
eventType: result.success ? 'action_executed' : 'action_failed',
|
|
254
|
+
intentDetected: pending.intent,
|
|
255
|
+
sessionId: this.sessionId,
|
|
256
|
+
actionExecuted: { intent: pending.intent, confirmed: true },
|
|
257
|
+
});
|
|
258
|
+
if (result.message) {
|
|
259
|
+
if (this.lastExchange)
|
|
260
|
+
this.lastExchange = { ...this.lastExchange, replyText: result.message };
|
|
261
|
+
this.emit();
|
|
262
|
+
await this.speakReply(result.message, null);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
cancelPendingAction() {
|
|
266
|
+
this.clearConfirmWindow();
|
|
267
|
+
const pending = this.pending;
|
|
268
|
+
this.pending = null;
|
|
269
|
+
this.confirmationMessage = null;
|
|
270
|
+
this.confirmationVoiceHint = null;
|
|
271
|
+
if (this.phase === 'confirming')
|
|
272
|
+
this.setPhase('idle');
|
|
273
|
+
this.emit();
|
|
274
|
+
if (!pending)
|
|
275
|
+
return;
|
|
276
|
+
this.telemetry.logEvent({
|
|
277
|
+
eventType: 'action_cancelled',
|
|
278
|
+
intentDetected: pending.intent,
|
|
279
|
+
sessionId: this.sessionId,
|
|
280
|
+
});
|
|
281
|
+
if (this.lastExchange)
|
|
282
|
+
this.lastExchange = { ...this.lastExchange, replyText: this.messages.cancelled };
|
|
283
|
+
this.emit();
|
|
284
|
+
void this.speakReply(this.messages.cancelled, null);
|
|
285
|
+
}
|
|
286
|
+
async handleConfirmationTranscript(text) {
|
|
287
|
+
this.confirmationVoiceActive = false;
|
|
288
|
+
this.clearConfirmWindow();
|
|
289
|
+
const pending = this.pending;
|
|
290
|
+
if (this.phase === 'confirming')
|
|
291
|
+
this.setPhase('idle');
|
|
292
|
+
if (!pending)
|
|
293
|
+
return;
|
|
294
|
+
// Tap-only confirmations can never be resolved by a stray transcript.
|
|
295
|
+
if (pending.voiceMode === 'disabled' || pending.silent)
|
|
296
|
+
return;
|
|
297
|
+
const reply = (0, turnEngine_1.classifyPendingReply)(text, pending, this.lexicon, { silent: pending.silent });
|
|
298
|
+
if (reply === 'affirmative') {
|
|
299
|
+
await this.confirmPendingAction();
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
if (reply === 'negative') {
|
|
303
|
+
this.cancelPendingAction();
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
// Ambiguous: re-ask ONCE, then fall back to modal-only.
|
|
307
|
+
if (!this.confirmationReasked) {
|
|
308
|
+
this.confirmationReasked = true;
|
|
309
|
+
const reask = `${this.messages.confirmationNotUnderstood} ${pending.question}`.trim();
|
|
310
|
+
this.confirmationMessage = reask;
|
|
311
|
+
this.emit();
|
|
312
|
+
await this.speakReply(reask, null);
|
|
313
|
+
if (this.pending)
|
|
314
|
+
this.beginConfirmationVoiceWindow();
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
// ── Main turn ────────────────────────────────────────────────────────────────
|
|
318
|
+
async handleTranscript(rawText) {
|
|
319
|
+
const text = rawText.trim();
|
|
320
|
+
if (!this.enabled || text.length < this.timings.minTranscriptChars) {
|
|
321
|
+
if (this.phase === 'thinking')
|
|
322
|
+
this.setPhase('idle');
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
this.setPhase('thinking');
|
|
326
|
+
this.clearWatchdog();
|
|
327
|
+
this.watchdog = setTimeout(() => {
|
|
328
|
+
if (this.phase === 'thinking') {
|
|
329
|
+
this.telemetry.logEvent({ eventType: 'agent_watchdog_reset', sessionId: this.sessionId });
|
|
330
|
+
this.setPhase('idle');
|
|
331
|
+
}
|
|
332
|
+
}, this.timings.thinkingWatchdogMs);
|
|
333
|
+
const currentRoute = this.config.navigation.getCurrentRoute();
|
|
334
|
+
if (!this.sessionStarted) {
|
|
335
|
+
this.sessionStarted = true;
|
|
336
|
+
this.telemetry.logEvent({ eventType: 'agent_session_started', sessionId: this.sessionId });
|
|
337
|
+
}
|
|
338
|
+
this.telemetry.logEvent({
|
|
339
|
+
eventType: 'agent_invoked',
|
|
340
|
+
sessionId: this.sessionId,
|
|
341
|
+
currentRoute,
|
|
342
|
+
});
|
|
343
|
+
const snapshot = this.screenStore.getSnapshot();
|
|
344
|
+
const rawContext = (0, core_1.combineContexts)(this.config.getUserContextText?.() ?? null, this.screenStore.getText(), {
|
|
345
|
+
truncationSuffix: this.messages.contextTruncationSuffix,
|
|
346
|
+
maxChars: this.config.contextMaxChars,
|
|
347
|
+
});
|
|
348
|
+
const sanitized = rawContext ? this.sanitizer(rawContext).sanitized : undefined;
|
|
349
|
+
const conv = (0, core_1.prepareConversationForTurn)(this.conversation);
|
|
350
|
+
const controller = new AbortController();
|
|
351
|
+
this.abort = controller;
|
|
352
|
+
let reply = '';
|
|
353
|
+
let audioBase64 = null;
|
|
354
|
+
try {
|
|
355
|
+
const turn = await this.config.backend.sendTurn({
|
|
356
|
+
message: text,
|
|
357
|
+
language: this.config.locale,
|
|
358
|
+
voice: this.getTtsVoice(),
|
|
359
|
+
screenContext: sanitized,
|
|
360
|
+
currentRoute: currentRoute ?? undefined,
|
|
361
|
+
conversationHistory: conv.history,
|
|
362
|
+
conversationSummary: conv.summary ?? undefined,
|
|
363
|
+
sessionId: this.sessionId,
|
|
364
|
+
...(0, backendClient_1.buildManifestPayload)(this.config.manifest),
|
|
365
|
+
}, { signal: controller.signal });
|
|
366
|
+
reply = turn.reply;
|
|
367
|
+
audioBase64 = turn.audioBase64;
|
|
368
|
+
const decision = await (0, turnEngine_1.decideAction)({
|
|
369
|
+
action: turn.action,
|
|
370
|
+
manifest: this.config.manifest,
|
|
371
|
+
executor: this.executor,
|
|
372
|
+
context: snapshot?.context ?? {},
|
|
373
|
+
messages: this.messages,
|
|
374
|
+
userMessage: text,
|
|
375
|
+
});
|
|
376
|
+
if (decision.kind === 'executed') {
|
|
377
|
+
const r = decision.result;
|
|
378
|
+
if (turn.action?.type === 'navigate') {
|
|
379
|
+
this.telemetry.logEvent({
|
|
380
|
+
eventType: r.success ? 'navigation_executed' : 'navigation_failed',
|
|
381
|
+
intentDetected: turn.action.intent,
|
|
382
|
+
sessionId: this.sessionId,
|
|
383
|
+
currentRoute,
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
else if (turn.action?.type === 'execute') {
|
|
387
|
+
this.telemetry.logEvent({
|
|
388
|
+
eventType: r.recoverable
|
|
389
|
+
? 'action_needs_context'
|
|
390
|
+
: r.success
|
|
391
|
+
? 'action_executed'
|
|
392
|
+
: 'action_failed',
|
|
393
|
+
intentDetected: turn.action.intent,
|
|
394
|
+
sessionId: this.sessionId,
|
|
395
|
+
actionExecuted: { intent: turn.action.intent, confirmed: false },
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
if (r.recoverable) {
|
|
399
|
+
/* keep the assistant's guiding reply; never speak a refusal. */
|
|
400
|
+
}
|
|
401
|
+
else if (r.message) {
|
|
402
|
+
reply = r.message;
|
|
403
|
+
audioBase64 = null;
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
else if (decision.kind === 'precheck-resolved') {
|
|
407
|
+
if (decision.result.message && !decision.result.recoverable) {
|
|
408
|
+
reply = decision.result.message;
|
|
409
|
+
audioBase64 = null;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
else if (decision.kind === 'needs-confirmation') {
|
|
413
|
+
this.pending = { ...decision.pending, silent: this.silentTurn };
|
|
414
|
+
this.confirmationReasked = false;
|
|
415
|
+
this.confirmationMessage = decision.pending.question;
|
|
416
|
+
this.confirmationVoiceHint =
|
|
417
|
+
decision.pending.voiceMode === 'strict' &&
|
|
418
|
+
decision.pending.strictPhrase &&
|
|
419
|
+
!this.silentTurn
|
|
420
|
+
? (0, core_1.formatMessage)(this.messages.strictConfirmationModalHint, {
|
|
421
|
+
phrase: decision.pending.strictPhrase,
|
|
422
|
+
})
|
|
423
|
+
: null;
|
|
424
|
+
this.emit();
|
|
425
|
+
this.telemetry.logEvent({
|
|
426
|
+
eventType: 'action_confirmation_requested',
|
|
427
|
+
intentDetected: decision.pending.intent,
|
|
428
|
+
sessionId: this.sessionId,
|
|
429
|
+
});
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
catch (e) {
|
|
433
|
+
this.clearWatchdog();
|
|
434
|
+
this.abort = null;
|
|
435
|
+
if (e instanceof errors_1.AgentApiError && e.code === 'cancelled') {
|
|
436
|
+
if (this.phase === 'thinking')
|
|
437
|
+
this.setPhase('idle');
|
|
438
|
+
return;
|
|
439
|
+
}
|
|
440
|
+
if (e instanceof errors_1.AgentApiError && e.code === 'quota_exceeded') {
|
|
441
|
+
this.config.onQuotaExceeded?.();
|
|
442
|
+
}
|
|
443
|
+
if (this.phase === 'thinking')
|
|
444
|
+
this.setPhase('idle');
|
|
445
|
+
this.notifyError(reply || (e instanceof Error ? e.message : 'error'));
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
448
|
+
this.clearWatchdog();
|
|
449
|
+
this.abort = null;
|
|
450
|
+
if (reply) {
|
|
451
|
+
this.lastExchange = { userText: text, replyText: reply };
|
|
452
|
+
this.emit();
|
|
453
|
+
(0, core_1.recordConversationExchange)(this.conversation, text, reply);
|
|
454
|
+
}
|
|
455
|
+
const hasPending = this.pending != null;
|
|
456
|
+
await this.speakReply(reply, audioBase64);
|
|
457
|
+
if (hasPending) {
|
|
458
|
+
this.beginConfirmationVoiceWindow();
|
|
459
|
+
}
|
|
460
|
+
else if (this.phase !== 'speaking') {
|
|
461
|
+
this.setPhase('idle');
|
|
462
|
+
}
|
|
463
|
+
this.config.onTurnCompleted?.();
|
|
464
|
+
}
|
|
465
|
+
// ── Speech session wiring ────────────────────────────────────────────────────
|
|
466
|
+
onTranscript(text) {
|
|
467
|
+
if (this.confirmationVoiceActive || this.pending) {
|
|
468
|
+
void this.handleConfirmationTranscript(text);
|
|
469
|
+
return;
|
|
470
|
+
}
|
|
471
|
+
void this.handleTranscript(text);
|
|
472
|
+
}
|
|
473
|
+
onSpeechError(error) {
|
|
474
|
+
this.telemetry.logEvent({ eventType: 'agent_stt_failed', sessionId: this.sessionId });
|
|
475
|
+
if (this.phase === 'listening' || this.phase === 'confirming')
|
|
476
|
+
this.setPhase('idle');
|
|
477
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
478
|
+
// A permission/hardware error means voice can't run: flag it so taps route
|
|
479
|
+
// to the keyboard (the binding reacts to `voiceBlocked` to open the bubble).
|
|
480
|
+
const lower = msg.toLowerCase();
|
|
481
|
+
if (lower.includes('not-allowed') ||
|
|
482
|
+
lower.includes('not allowed') ||
|
|
483
|
+
lower.includes('service-not-allowed') ||
|
|
484
|
+
lower.includes('audio-capture') ||
|
|
485
|
+
lower.includes('permission')) {
|
|
486
|
+
this.markVoiceBlocked(true);
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
489
|
+
this.notifyError(msg);
|
|
490
|
+
}
|
|
491
|
+
// ── Consent + listening ──────────────────────────────────────────────────────
|
|
492
|
+
async beginListening() {
|
|
493
|
+
if (!this.enabled)
|
|
494
|
+
return;
|
|
495
|
+
if (!this.config.voice.isRecognitionAvailable()) {
|
|
496
|
+
this.telemetry.logEvent({ eventType: 'agent_stt_failed', sessionId: this.sessionId });
|
|
497
|
+
this.notifyError(this.ui.voiceUnavailable);
|
|
498
|
+
return;
|
|
499
|
+
}
|
|
500
|
+
const gate = this.config.gateTurn?.();
|
|
501
|
+
if (gate && !gate.ok) {
|
|
502
|
+
if (gate.message)
|
|
503
|
+
this.notifyError(gate.message);
|
|
504
|
+
return;
|
|
505
|
+
}
|
|
506
|
+
this.config.voice.stopPlayback();
|
|
507
|
+
this.silentTurn = false;
|
|
508
|
+
this.setPhase('listening');
|
|
509
|
+
await this.speech.start();
|
|
510
|
+
}
|
|
511
|
+
async toggleListening() {
|
|
512
|
+
if (!this.enabled)
|
|
513
|
+
return;
|
|
514
|
+
if (this.phase === 'confirming')
|
|
515
|
+
return; // modal owns this phase
|
|
516
|
+
if (this.phase === 'listening') {
|
|
517
|
+
this.speech.stop();
|
|
518
|
+
this.setPhase('thinking');
|
|
519
|
+
return;
|
|
520
|
+
}
|
|
521
|
+
if (this.phase === 'thinking' || this.phase === 'speaking')
|
|
522
|
+
return;
|
|
523
|
+
const consented = await this.consent.hasConsented();
|
|
524
|
+
if (!consented) {
|
|
525
|
+
this.pendingConsentIntent = 'voice';
|
|
526
|
+
this.consentModalVisible = true;
|
|
527
|
+
this.emit();
|
|
528
|
+
return;
|
|
529
|
+
}
|
|
530
|
+
await this.beginListening();
|
|
531
|
+
}
|
|
532
|
+
async submitTextTurn(text, options) {
|
|
533
|
+
if (!this.enabled)
|
|
534
|
+
return;
|
|
535
|
+
const trimmed = text.trim();
|
|
536
|
+
if (!trimmed)
|
|
537
|
+
return;
|
|
538
|
+
const gate = this.config.gateTurn?.();
|
|
539
|
+
if (gate && !gate.ok) {
|
|
540
|
+
if (gate.message)
|
|
541
|
+
this.notifyError(gate.message);
|
|
542
|
+
return;
|
|
543
|
+
}
|
|
544
|
+
// A pending confirmation steals the next text turn (so typed "yes" resolves
|
|
545
|
+
// it instead of starting a fresh turn). Honours the same voice-mode rules.
|
|
546
|
+
if (this.pending && !this.pending.silent) {
|
|
547
|
+
await this.handleConfirmationTranscript(trimmed);
|
|
548
|
+
return;
|
|
549
|
+
}
|
|
550
|
+
this.silentTurn = options?.speak !== true;
|
|
551
|
+
await this.handleTranscript(trimmed);
|
|
552
|
+
}
|
|
553
|
+
async acceptConsent() {
|
|
554
|
+
await this.consent.grant();
|
|
555
|
+
this.consentModalVisible = false;
|
|
556
|
+
this.emit();
|
|
557
|
+
this.telemetry.logEvent({ eventType: 'agent_consent_granted', sessionId: this.sessionId });
|
|
558
|
+
const intent = this.pendingConsentIntent;
|
|
559
|
+
this.pendingConsentIntent = null;
|
|
560
|
+
if (intent === 'voice')
|
|
561
|
+
await this.beginListening();
|
|
562
|
+
}
|
|
563
|
+
declineConsent() {
|
|
564
|
+
this.consentModalVisible = false;
|
|
565
|
+
this.pendingConsentIntent = null;
|
|
566
|
+
this.emit();
|
|
567
|
+
this.telemetry.logEvent({ eventType: 'agent_consent_declined', sessionId: this.sessionId });
|
|
568
|
+
}
|
|
569
|
+
cancelAll() {
|
|
570
|
+
this.abort?.abort();
|
|
571
|
+
this.abort = null;
|
|
572
|
+
this.clearWatchdog();
|
|
573
|
+
this.clearConfirmWindow();
|
|
574
|
+
this.speech.cancel();
|
|
575
|
+
this.config.voice.stopPlayback();
|
|
576
|
+
this.pending = null;
|
|
577
|
+
this.confirmationMessage = null;
|
|
578
|
+
this.confirmationVoiceHint = null;
|
|
579
|
+
this.setPhase('idle');
|
|
580
|
+
this.emit();
|
|
581
|
+
}
|
|
582
|
+
/** Best-effort backend warm-up (cold-start mitigation). */
|
|
583
|
+
warmUp() {
|
|
584
|
+
void this.config.backend.warmUp?.();
|
|
585
|
+
}
|
|
586
|
+
dispose() {
|
|
587
|
+
this.cancelAll();
|
|
588
|
+
this.unsubscribeSpeech?.();
|
|
589
|
+
this.unsubscribeSpeech = null;
|
|
590
|
+
this.speech.dispose();
|
|
591
|
+
this.listeners.clear();
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
exports.AgentController = AgentController;
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Framework-free turn decision engine.
|
|
3
|
+
*
|
|
4
|
+
* Concentrates the SECURITY-CRITICAL decision logic of a turn so it can be
|
|
5
|
+
* unit-tested without any framework, and so every web binding (Vue, Svelte,
|
|
6
|
+
* Angular) and React Native share the exact same rules. It does NOT
|
|
7
|
+
* reimplement the engine: it delegates execution, prechecks and confirmation
|
|
8
|
+
* classification to @fiodos/core. It only sequences them.
|
|
9
|
+
*
|
|
10
|
+
* Invariants enforced here (identical to @fiodos/react-native):
|
|
11
|
+
* 1. The source of truth for "needs confirmation" is the MANIFEST
|
|
12
|
+
* (`requireConfirmation`), never the LLM's flag.
|
|
13
|
+
* 2. A sensitive action is NEVER executed by `decideAction`; it returns a
|
|
14
|
+
* PendingConfirmation and waits for an explicit confirmation.
|
|
15
|
+
* 3. Voice confirmation honours the manifest mode: 'disabled' can never be
|
|
16
|
+
* confirmed by voice (tap only); 'strict' requires the exact phrase; a
|
|
17
|
+
* loose "yes" does not confirm a strict action. Negation always wins.
|
|
18
|
+
*/
|
|
19
|
+
import { type ActionExecutor, type ActionResult, type AgentAction, type AgentMessages, type AgentScreenContext, type AppManifest, type ConfirmationLexicon, type ConfirmationReply } from '@fiodos/core';
|
|
20
|
+
export type VoiceConfirmationMode = 'standard' | 'strict' | 'disabled';
|
|
21
|
+
export interface PendingConfirmation {
|
|
22
|
+
action: AgentAction;
|
|
23
|
+
context: AgentScreenContext;
|
|
24
|
+
/** Canonical question to show/speak. */
|
|
25
|
+
question: string;
|
|
26
|
+
voiceMode: VoiceConfirmationMode;
|
|
27
|
+
/** Strict-mode exact phrase, or null when not strict. */
|
|
28
|
+
strictPhrase: string | null;
|
|
29
|
+
/** Remaining confirmations (2 for doubleConfirmation, else 1). */
|
|
30
|
+
remainingConfirmations: number;
|
|
31
|
+
/** Original user message that triggered the action (telemetry). */
|
|
32
|
+
userMessage: string;
|
|
33
|
+
intent: string;
|
|
34
|
+
}
|
|
35
|
+
export type ActionDecision = {
|
|
36
|
+
kind: 'executed';
|
|
37
|
+
result: ActionResult;
|
|
38
|
+
} | {
|
|
39
|
+
kind: 'precheck-resolved';
|
|
40
|
+
result: ActionResult;
|
|
41
|
+
} | {
|
|
42
|
+
kind: 'needs-confirmation';
|
|
43
|
+
pending: PendingConfirmation;
|
|
44
|
+
} | {
|
|
45
|
+
kind: 'noop';
|
|
46
|
+
};
|
|
47
|
+
export interface DecideActionParams {
|
|
48
|
+
action: AgentAction | null;
|
|
49
|
+
manifest: AppManifest;
|
|
50
|
+
executor: ActionExecutor;
|
|
51
|
+
/** Structured screen snapshot captured at invocation time. */
|
|
52
|
+
context: AgentScreenContext;
|
|
53
|
+
messages: AgentMessages;
|
|
54
|
+
userMessage: string;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Decides what to do with a backend-proposed action. Sensitive actions are
|
|
58
|
+
* never executed here — they return `needs-confirmation`.
|
|
59
|
+
*/
|
|
60
|
+
export declare function decideAction(params: DecideActionParams): Promise<ActionDecision>;
|
|
61
|
+
/**
|
|
62
|
+
* Classifies a spoken reply against a pending confirmation, honouring the
|
|
63
|
+
* manifest's voice mode. Returns 'ambiguous' when voice cannot resolve it
|
|
64
|
+
* (disabled mode, silent turn, or unclear speech) — the caller must then rely
|
|
65
|
+
* on the tactile modal. NEVER returns 'affirmative' for a disabled/silent
|
|
66
|
+
* confirmation.
|
|
67
|
+
*/
|
|
68
|
+
export declare function classifyPendingReply(text: string, pending: PendingConfirmation, lexicon: ConfirmationLexicon, opts?: {
|
|
69
|
+
silent?: boolean;
|
|
70
|
+
}): ConfirmationReply;
|