@ateam-ai-mobile/ui 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +325 -21
- package/dist/index.d.ts +325 -21
- package/dist/index.js +1127 -176
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1120 -177
- package/dist/index.mjs.map +1 -1
- package/package.json +3 -3
- package/src/WorkspaceBanner.tsx +71 -0
- package/src/auth/HandoffScreen.tsx +94 -0
- package/src/auth/JoinScreen.tsx +145 -0
- package/src/auth/LoginScreen.tsx +181 -0
- package/src/auth/OpenJoinScreen.tsx +131 -0
- package/src/auth/handoffExchange.ts +49 -0
- package/src/auth/inviteJoin.ts +44 -0
- package/src/auth/login.ts +57 -0
- package/src/auth/openJoin.ts +56 -0
- package/src/chat/MessageBubble.tsx +3 -0
- package/src/index.ts +38 -1
- package/src/plugin/Plugin.tsx +14 -9
- package/src/providers.ts +44 -0
- package/src/voice/VoiceCallSheet.tsx +4 -32
- package/src/voice/VoiceMicIndicator.tsx +99 -0
- package/src/voice/VoiceSessionHost.tsx +77 -0
- package/src/voice/useVoiceCall.ts +40 -231
- package/src/voice/voiceSession.ts +628 -0
|
@@ -0,0 +1,628 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* voiceSession — the ONE live voice session, headless and app-root-owned.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS. The session used to live inside `useVoiceCall`, whose WebView
|
|
5
|
+
* was rendered by VoiceCallSheet. Unmount the sheet and the WebView unmounted
|
|
6
|
+
* with it, so the session died — not by design, just by where the WebView lived.
|
|
7
|
+
* That made the built-in screen the ONLY way to hold a live loop, exactly the
|
|
8
|
+
* state the camera was in before sensing and preview were split.
|
|
9
|
+
*
|
|
10
|
+
* The fix is the same one VisionStreamHost applied to the camera: move the thing
|
|
11
|
+
* that must stay alive to the app root, and make every surface a READER of it.
|
|
12
|
+
* This module is that thing — a singleton that owns session state and drives the
|
|
13
|
+
* WebView; `VoiceSessionHost` mounts the (offscreen) WebView bound to it once, at
|
|
14
|
+
* the root; `useVoiceCall` / `props.voice` are views onto it.
|
|
15
|
+
*
|
|
16
|
+
* It is the provider handed to `setVoiceProvider` (plugin-runtime), so a plugin's
|
|
17
|
+
* gated `props.voice` and the built-in sheet drive the SAME session — one phone,
|
|
18
|
+
* one mic, one Realtime loop, one bill (spec invariant 2).
|
|
19
|
+
*
|
|
20
|
+
* TRUST. The device-tool channel (origin-checked postMessage) is preserved
|
|
21
|
+
* verbatim from the hook: the model's tool calls arrive on THIS phone over the
|
|
22
|
+
* WebView's own WebRTC, and only a message from the voice app's own origin may
|
|
23
|
+
* drive the device. `prompt()` is an AGENT turn, never forged user speech, and it
|
|
24
|
+
* cannot reach the WebView beyond that one injected call.
|
|
25
|
+
*/
|
|
26
|
+
import { getVoiceApiUrl, getToken, getTenant, getUser, refreshToken } from '@ateam-ai-mobile/core';
|
|
27
|
+
import {
|
|
28
|
+
callDeviceTool,
|
|
29
|
+
ensureDeviceToolsLoaded,
|
|
30
|
+
listDeviceTools,
|
|
31
|
+
PLATFORM_VOICE_CALLER,
|
|
32
|
+
type VoiceSessionProvider,
|
|
33
|
+
type VoiceCaller,
|
|
34
|
+
type VoiceSnapshot,
|
|
35
|
+
type VoiceState,
|
|
36
|
+
type VoiceTranscriptEntry,
|
|
37
|
+
type VoiceStartOptions,
|
|
38
|
+
type VoiceEvent,
|
|
39
|
+
type VoiceSubscription,
|
|
40
|
+
} from '@ateam-ai-mobile/plugin-runtime';
|
|
41
|
+
|
|
42
|
+
// Re-export the transcript/state shapes under their historical names so existing
|
|
43
|
+
// importers (useVoiceCall, VoiceCallSheet) keep compiling. The canonical
|
|
44
|
+
// definitions live in plugin-runtime (the layer both ui and plugins depend on).
|
|
45
|
+
export type CallState = VoiceState;
|
|
46
|
+
export type TranscriptEntry = VoiceTranscriptEntry;
|
|
47
|
+
|
|
48
|
+
/** How long to wait for the WebView to reach 'connected' before start() rejects.
|
|
49
|
+
* A hung promise (e.g. no VoiceSessionHost mounted) is worse than a clear error. */
|
|
50
|
+
const CONNECT_TIMEOUT_MS = 30_000;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* The voice WebView bakes the JWT into its URL once at mount and never refreshes
|
|
54
|
+
* it (unlike the SDK REST client, which self-heals on 401). Refresh a stale/near-
|
|
55
|
+
* expiry token BEFORE the WebView mounts so it gets a fresh one. Cheap unless
|
|
56
|
+
* actually near expiry (decoded locally, no network unless needed).
|
|
57
|
+
*/
|
|
58
|
+
const TOKEN_REFRESH_SKEW_MS = 90_000;
|
|
59
|
+
function tokenNeedsRefresh(token: string | null): boolean {
|
|
60
|
+
if (!token) return true;
|
|
61
|
+
try {
|
|
62
|
+
const payloadB64 = token.split('.')[1];
|
|
63
|
+
const padded = payloadB64.replace(/-/g, '+').replace(/_/g, '/');
|
|
64
|
+
const json = typeof atob === 'function'
|
|
65
|
+
? atob(padded)
|
|
66
|
+
: Buffer.from(padded, 'base64').toString('utf8');
|
|
67
|
+
const exp = JSON.parse(json)?.exp;
|
|
68
|
+
if (typeof exp !== 'number') return false;
|
|
69
|
+
return Date.now() >= exp * 1000 - TOKEN_REFRESH_SKEW_MS;
|
|
70
|
+
} catch {
|
|
71
|
+
return false;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Installed BEFORE the voice app loads — captures every audio track that could
|
|
76
|
+
// reach the peer connection so setMuted can flip enabled=false on all of them,
|
|
77
|
+
// however the voice app obtained the track. (Verbatim from useVoiceCall.)
|
|
78
|
+
export const INJECT_TRACK_HOOK = `
|
|
79
|
+
(function() {
|
|
80
|
+
if (window._adaTrackHookInstalled) return;
|
|
81
|
+
window._adaTrackHookInstalled = true;
|
|
82
|
+
window._allAudioTracks = [];
|
|
83
|
+
window._adaPeerConnections = [];
|
|
84
|
+
|
|
85
|
+
function track(t) {
|
|
86
|
+
if (t && t.kind === 'audio' && window._allAudioTracks.indexOf(t) === -1) {
|
|
87
|
+
window._allAudioTracks.push(t);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
try {
|
|
92
|
+
if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
|
|
93
|
+
const origGUM = navigator.mediaDevices.getUserMedia.bind(navigator.mediaDevices);
|
|
94
|
+
navigator.mediaDevices.getUserMedia = async function(constraints) {
|
|
95
|
+
const stream = await origGUM(constraints);
|
|
96
|
+
if (constraints && constraints.audio) {
|
|
97
|
+
try { stream.getAudioTracks().forEach(track); } catch (e) {}
|
|
98
|
+
}
|
|
99
|
+
return stream;
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
} catch (e) {}
|
|
103
|
+
|
|
104
|
+
try {
|
|
105
|
+
if (window.RTCPeerConnection) {
|
|
106
|
+
const origPC = window.RTCPeerConnection;
|
|
107
|
+
const origAddTrack = origPC.prototype.addTrack;
|
|
108
|
+
const origAddStream = origPC.prototype.addStream;
|
|
109
|
+
const origAddTransceiver = origPC.prototype.addTransceiver;
|
|
110
|
+
|
|
111
|
+
origPC.prototype.addTrack = function(t) {
|
|
112
|
+
if (t && t.kind === 'audio') track(t);
|
|
113
|
+
return origAddTrack.apply(this, arguments);
|
|
114
|
+
};
|
|
115
|
+
if (origAddStream) {
|
|
116
|
+
origPC.prototype.addStream = function(stream) {
|
|
117
|
+
try { stream.getAudioTracks().forEach(track); } catch (e) {}
|
|
118
|
+
return origAddStream.apply(this, arguments);
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
if (origAddTransceiver) {
|
|
122
|
+
origPC.prototype.addTransceiver = function(arg) {
|
|
123
|
+
if (arg && arg.kind === undefined && arg.track && arg.track.kind === 'audio') {
|
|
124
|
+
track(arg.track);
|
|
125
|
+
}
|
|
126
|
+
const tx = origAddTransceiver.apply(this, arguments);
|
|
127
|
+
try {
|
|
128
|
+
if (tx && tx.sender && tx.sender.track && tx.sender.track.kind === 'audio') track(tx.sender.track);
|
|
129
|
+
} catch (e) {}
|
|
130
|
+
return tx;
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
} catch (e) {}
|
|
135
|
+
|
|
136
|
+
window._adaSetMuted = function(muted) {
|
|
137
|
+
let n = 0;
|
|
138
|
+
if (Array.isArray(window._allAudioTracks)) {
|
|
139
|
+
window._allAudioTracks.forEach(function(t) {
|
|
140
|
+
try { t.enabled = !muted; n++; } catch (e) {}
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
return n;
|
|
144
|
+
};
|
|
145
|
+
})(); true;
|
|
146
|
+
`;
|
|
147
|
+
|
|
148
|
+
const buildMuteScript = (muted: boolean) => `
|
|
149
|
+
(function(muted) {
|
|
150
|
+
if (typeof window.setMuted === 'function') { window.setMuted(muted); }
|
|
151
|
+
let n = 0;
|
|
152
|
+
if (typeof window._adaSetMuted === 'function') {
|
|
153
|
+
n = window._adaSetMuted(muted);
|
|
154
|
+
}
|
|
155
|
+
if (n === 0 && Array.isArray(window._allAudioTracks)) {
|
|
156
|
+
window._allAudioTracks.forEach(function(t) {
|
|
157
|
+
try { t.enabled = !muted; n++; } catch (e) {}
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
console.log('[ada-mute]', muted ? 'MUTED' : 'UNMUTED', 'tracks=', n);
|
|
161
|
+
})(${muted}); true;
|
|
162
|
+
`;
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* The agent-initiated turn — the guide speaking first. Calls the voice app's
|
|
166
|
+
* `__adasAgentTurn(input)`, which must issue a Realtime `response.create` into
|
|
167
|
+
* the ALREADY-OPEN session (Core owns that function; see the spec §4.2).
|
|
168
|
+
*
|
|
169
|
+
* REFUSE, DON'T NO-OP: an undefined function called through injectJavaScript
|
|
170
|
+
* fails SILENTLY in the WebView, and the author sees a guide that simply never
|
|
171
|
+
* speaks first — indistinguishable from a model that chose silence. So if the
|
|
172
|
+
* hook is absent (an older voice page), we post a notice back that surfaces as
|
|
173
|
+
* `lastError`, without ending the session.
|
|
174
|
+
*/
|
|
175
|
+
const buildAgentTurnScript = (input: string) => `
|
|
176
|
+
(function(input) {
|
|
177
|
+
if (typeof window.__adasAgentTurn === 'function') {
|
|
178
|
+
try { window.__adasAgentTurn(input); }
|
|
179
|
+
catch (e) {
|
|
180
|
+
window.ReactNativeWebView && window.ReactNativeWebView.postMessage(JSON.stringify({
|
|
181
|
+
type: 'agentTurnFailed', message: String(e && e.message || e),
|
|
182
|
+
}));
|
|
183
|
+
}
|
|
184
|
+
} else if (window.ReactNativeWebView) {
|
|
185
|
+
window.ReactNativeWebView.postMessage(JSON.stringify({
|
|
186
|
+
type: 'agentTurnUnsupported',
|
|
187
|
+
message: 'voice app does not support agent turns (__adasAgentTurn missing) — update the voice page',
|
|
188
|
+
}));
|
|
189
|
+
}
|
|
190
|
+
})(${JSON.stringify(input)}); true;
|
|
191
|
+
`;
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* A silent OBSERVATION — tell the model something without asking it to speak.
|
|
195
|
+
* Calls the voice app's `__adasObserve(text)`, which adds a `system` conversation
|
|
196
|
+
* item and does NOT request a response (Core owns that function). This is the
|
|
197
|
+
* cheap channel for a continuous stream — e.g. what the camera sees while the
|
|
198
|
+
* user walks — as opposed to `__adasAgentTurn`, which asks the model to speak.
|
|
199
|
+
*
|
|
200
|
+
* Same refuse-don't-no-op contract as the agent turn: a missing hook posts a
|
|
201
|
+
* notice back rather than failing silently.
|
|
202
|
+
*/
|
|
203
|
+
const buildObserveScript = (text: string) => `
|
|
204
|
+
(function(text) {
|
|
205
|
+
if (typeof window.__adasObserve === 'function') {
|
|
206
|
+
try { window.__adasObserve(text); }
|
|
207
|
+
catch (e) {
|
|
208
|
+
window.ReactNativeWebView && window.ReactNativeWebView.postMessage(JSON.stringify({
|
|
209
|
+
type: 'observeFailed', message: String(e && e.message || e),
|
|
210
|
+
}));
|
|
211
|
+
}
|
|
212
|
+
} else if (window.ReactNativeWebView) {
|
|
213
|
+
window.ReactNativeWebView.postMessage(JSON.stringify({
|
|
214
|
+
type: 'observeUnsupported',
|
|
215
|
+
message: 'voice app does not support observe (__adasObserve missing) — update the voice page',
|
|
216
|
+
}));
|
|
217
|
+
}
|
|
218
|
+
})(${JSON.stringify(text)}); true;
|
|
219
|
+
`;
|
|
220
|
+
|
|
221
|
+
/** Origin of a URL, or null if it can't be parsed. */
|
|
222
|
+
function originOf(url?: string | null): string | null {
|
|
223
|
+
if (!url) return null;
|
|
224
|
+
const m = /^([a-z][a-z0-9+.-]*:\/\/[^/?#]+)/i.exec(url);
|
|
225
|
+
return m ? m[1].toLowerCase() : null;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
interface InternalState {
|
|
229
|
+
callState: CallState;
|
|
230
|
+
sessionId: string | null;
|
|
231
|
+
error: string | null;
|
|
232
|
+
transcript: TranscriptEntry[];
|
|
233
|
+
aiPartial: string;
|
|
234
|
+
userPartial: string;
|
|
235
|
+
muted: boolean;
|
|
236
|
+
/** User VAD, from the mini-app's speechActivity events. */
|
|
237
|
+
userSpeaking: boolean;
|
|
238
|
+
/** Epoch ms of the last speaking:false, or null before the first utterance. */
|
|
239
|
+
lastUserSpeechEndedAt: number | null;
|
|
240
|
+
/** The WebView should be mounted (a session is wanted / live). */
|
|
241
|
+
mounted: boolean;
|
|
242
|
+
/** The WebView source, computed ONCE at start() so state changes (transcript,
|
|
243
|
+
* partials) never remount the WebView and drop the session. */
|
|
244
|
+
webViewSource: { uri: string } | null;
|
|
245
|
+
skillSlug?: string;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
const IDLE: InternalState = {
|
|
249
|
+
callState: 'idle',
|
|
250
|
+
sessionId: null,
|
|
251
|
+
error: null,
|
|
252
|
+
transcript: [],
|
|
253
|
+
aiPartial: '',
|
|
254
|
+
userPartial: '',
|
|
255
|
+
muted: false,
|
|
256
|
+
userSpeaking: false,
|
|
257
|
+
lastUserSpeechEndedAt: null,
|
|
258
|
+
mounted: false,
|
|
259
|
+
webViewSource: null,
|
|
260
|
+
skillSlug: undefined,
|
|
261
|
+
};
|
|
262
|
+
|
|
263
|
+
type Pending = { resolve: (s: VoiceSnapshot) => void; reject: (e: Error) => void; timer: any };
|
|
264
|
+
|
|
265
|
+
class VoiceSessionManager implements VoiceSessionProvider {
|
|
266
|
+
private state: InternalState = { ...IDLE };
|
|
267
|
+
private listeners = new Set<(e: VoiceEvent) => void>();
|
|
268
|
+
private webView: { injectJavaScript?: (js: string) => void } | null = null;
|
|
269
|
+
private pending: Pending[] = [];
|
|
270
|
+
/**
|
|
271
|
+
* WHO owns the one physical session. Set to the caller that started it from
|
|
272
|
+
* idle (a plugin's pluginId, or the PLATFORM symbol for the built-in sheet),
|
|
273
|
+
* and kept after stop until the next start reassigns it — so a plugin can read
|
|
274
|
+
* its own just-ended transcript but another plugin never can.
|
|
275
|
+
*
|
|
276
|
+
* ISOLATION (GPT blocker on 8881225): every access is scoped through this. A
|
|
277
|
+
* different plugin must not read, control, or attach to another plugin's live
|
|
278
|
+
* conversation just because it declared voice.
|
|
279
|
+
*/
|
|
280
|
+
private owner: VoiceCaller | null = null;
|
|
281
|
+
|
|
282
|
+
/** The platform (built-in sheet, mic indicator, session host) sees and
|
|
283
|
+
* controls any session — it is the user's own surface. A plugin caller may
|
|
284
|
+
* only touch a session it owns. */
|
|
285
|
+
private canAccess(caller: VoiceCaller): boolean {
|
|
286
|
+
return caller === PLATFORM_VOICE_CALLER || this.owner === caller;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
private foreignError(): Error {
|
|
290
|
+
const err: any = new Error(
|
|
291
|
+
'voice: another solution is using the microphone — one live session per device.',
|
|
292
|
+
);
|
|
293
|
+
err.code = 'RESTRICTED';
|
|
294
|
+
return err;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
// ── Host wiring (used only by VoiceSessionHost) ──────────────
|
|
298
|
+
|
|
299
|
+
/** The offscreen WebView sets its ref here so the session can inject into it. */
|
|
300
|
+
setWebView(ref: { injectJavaScript?: (js: string) => void } | null): void {
|
|
301
|
+
this.webView = ref;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/** Whole state — the host reads `mounted` + `webViewSource` to decide what to
|
|
305
|
+
* render, and every reader derives its snapshot from this. */
|
|
306
|
+
getInternal(): Readonly<InternalState> {
|
|
307
|
+
return this.state;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
readonly injectedTrackHook = INJECT_TRACK_HOOK;
|
|
311
|
+
|
|
312
|
+
// ── VoiceAPI (what props.voice and useVoiceCall call) ────────
|
|
313
|
+
|
|
314
|
+
async start(opts?: VoiceStartOptions, caller: VoiceCaller = PLATFORM_VOICE_CALLER): Promise<VoiceSnapshot> {
|
|
315
|
+
// Single session: if one is live or connecting, ATTACH — never a second
|
|
316
|
+
// Realtime loop (invariant 2). But ONLY the owner (or the platform) may
|
|
317
|
+
// attach; a different plugin that declared voice is REFUSED, not silently
|
|
318
|
+
// joined to a conversation it does not own (GPT isolation blocker). A
|
|
319
|
+
// different skillSlug does NOT restart it.
|
|
320
|
+
if (this.state.callState === 'connected') {
|
|
321
|
+
if (!this.canAccess(caller)) throw this.foreignError();
|
|
322
|
+
return this.realSnapshot();
|
|
323
|
+
}
|
|
324
|
+
if (this.state.callState === 'connecting') {
|
|
325
|
+
if (!this.canAccess(caller)) throw this.foreignError();
|
|
326
|
+
return this.waitForConnect();
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
// Device tools only exist once their bundle is evaluated; a user who goes
|
|
330
|
+
// straight to voice can otherwise ask for a tool the phone never loaded.
|
|
331
|
+
// Best-effort — a failure just means no device tools this time.
|
|
332
|
+
try {
|
|
333
|
+
await ensureDeviceToolsLoaded();
|
|
334
|
+
} catch (e: any) {
|
|
335
|
+
console.warn('[voiceSession] device tool preload failed:', e?.message || String(e));
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
// Hand the WebView a FRESH token before it mounts (it never refreshes once
|
|
339
|
+
// mounted). Only pays a round-trip when near expiry.
|
|
340
|
+
try {
|
|
341
|
+
if (tokenNeedsRefresh(getToken())) await refreshToken();
|
|
342
|
+
} catch {
|
|
343
|
+
// Proceed with whatever token we have; voice-backend surfaces a clean auth
|
|
344
|
+
// error rather than us blocking the call.
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// Idle → this caller becomes the owner of the new session.
|
|
348
|
+
this.owner = caller;
|
|
349
|
+
const skillSlug = opts?.skillSlug;
|
|
350
|
+
this.setState({
|
|
351
|
+
callState: 'connecting',
|
|
352
|
+
error: null,
|
|
353
|
+
transcript: [],
|
|
354
|
+
aiPartial: '',
|
|
355
|
+
userPartial: '',
|
|
356
|
+
muted: false,
|
|
357
|
+
mounted: true,
|
|
358
|
+
skillSlug,
|
|
359
|
+
webViewSource: { uri: this.buildUri(skillSlug) },
|
|
360
|
+
});
|
|
361
|
+
return this.waitForConnect();
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
stop(caller: VoiceCaller = PLATFORM_VOICE_CALLER): void {
|
|
365
|
+
// A plugin may not hang up another solution's live call. (Idle stop is a
|
|
366
|
+
// harmless no-op for anyone.)
|
|
367
|
+
if (this.state.callState !== 'idle' && !this.canAccess(caller)) throw this.foreignError();
|
|
368
|
+
this.webView?.injectJavaScript?.('window.hangUp && window.hangUp(); true;');
|
|
369
|
+
this.rejectPending(new Error('voice: session stopped before it connected'));
|
|
370
|
+
this.setState({
|
|
371
|
+
...IDLE,
|
|
372
|
+
// keep the transcript of the call that just ended for any reader still up
|
|
373
|
+
transcript: this.state.transcript,
|
|
374
|
+
});
|
|
375
|
+
// owner is intentionally KEPT (not reset to null): the next start() from
|
|
376
|
+
// idle reassigns it, and until then the owner can still read its own
|
|
377
|
+
// just-ended transcript while others cannot.
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
setMuted(muted: boolean, caller: VoiceCaller = PLATFORM_VOICE_CALLER): void {
|
|
381
|
+
if (this.state.callState !== 'idle' && !this.canAccess(caller)) throw this.foreignError();
|
|
382
|
+
this.webView?.injectJavaScript?.(buildMuteScript(muted));
|
|
383
|
+
this.setState({ muted });
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
prompt(input: string, caller: VoiceCaller = PLATFORM_VOICE_CALLER): void {
|
|
387
|
+
if (this.state.callState !== 'connected') {
|
|
388
|
+
throw new Error('voice: session is not connected — call start() and await it before prompt().');
|
|
389
|
+
}
|
|
390
|
+
if (!this.canAccess(caller)) throw this.foreignError();
|
|
391
|
+
this.webView?.injectJavaScript?.(buildAgentTurnScript(input));
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/**
|
|
395
|
+
* Stream a silent observation into the live session — tell the model without
|
|
396
|
+
* asking it to speak. The cheap, high-frequency channel (safe once or twice a
|
|
397
|
+
* second) behind a future props.voice.observe(); today the host uses it to run
|
|
398
|
+
* the real-device stream proof (VOICE_RT_STREAM_DEVICE_PROOF) before the gated
|
|
399
|
+
* plugin API is added. Owner-scoped like every other control.
|
|
400
|
+
*/
|
|
401
|
+
observe(text: string, caller: VoiceCaller = PLATFORM_VOICE_CALLER): void {
|
|
402
|
+
if (this.state.callState !== 'connected') {
|
|
403
|
+
throw new Error('voice: session is not connected — observe() needs a live session.');
|
|
404
|
+
}
|
|
405
|
+
if (!this.canAccess(caller)) throw this.foreignError();
|
|
406
|
+
this.webView?.injectJavaScript?.(buildObserveScript(text));
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
getSnapshot(caller: VoiceCaller = PLATFORM_VOICE_CALLER): VoiceSnapshot {
|
|
410
|
+
// A non-owner sees "you have no session" — which is TRUE for it — and learns
|
|
411
|
+
// nothing about the conversation another solution is holding. This is not a
|
|
412
|
+
// fake idle (invariant 6 is about an UNDECLARED plugin): a declared plugin
|
|
413
|
+
// that does not own the live session genuinely has none of its own.
|
|
414
|
+
return this.canAccess(caller) ? this.realSnapshot() : this.foreignSnapshot();
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
private realSnapshot(): VoiceSnapshot {
|
|
418
|
+
const s = this.state;
|
|
419
|
+
return {
|
|
420
|
+
state: s.callState,
|
|
421
|
+
sessionId: s.sessionId,
|
|
422
|
+
muted: s.muted,
|
|
423
|
+
speaking: !!s.aiPartial,
|
|
424
|
+
listening: !!s.userPartial,
|
|
425
|
+
userSpeaking: s.userSpeaking,
|
|
426
|
+
lastUserSpeechEndedAt: s.lastUserSpeechEndedAt,
|
|
427
|
+
lastError: s.error,
|
|
428
|
+
};
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
private foreignSnapshot(): VoiceSnapshot {
|
|
432
|
+
return {
|
|
433
|
+
state: 'idle', sessionId: null, muted: false, speaking: false, listening: false,
|
|
434
|
+
userSpeaking: false, lastUserSpeechEndedAt: null, lastError: null,
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
subscribe(cb: (event: VoiceEvent) => void, caller: VoiceCaller = PLATFORM_VOICE_CALLER): VoiceSubscription {
|
|
439
|
+
// Scope the stream: a non-owner's callback never fires (it has no session to
|
|
440
|
+
// observe), so a plugin cannot watch another solution's conversation by
|
|
441
|
+
// pre-registering a subscription before/while it is live. When this caller
|
|
442
|
+
// later owns a session, canAccess flips true and its callback starts firing.
|
|
443
|
+
const wrapped = (event: VoiceEvent) => { if (this.canAccess(caller)) cb(event); };
|
|
444
|
+
this.listeners.add(wrapped);
|
|
445
|
+
return { unsubscribe: () => { this.listeners.delete(wrapped); } };
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
transcript(caller: VoiceCaller = PLATFORM_VOICE_CALLER): TranscriptEntry[] {
|
|
449
|
+
return this.canAccess(caller) ? this.state.transcript.slice() : [];
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
// ── WebView plumbing ─────────────────────────────────────────
|
|
453
|
+
|
|
454
|
+
private buildUri(skillSlug?: string): string {
|
|
455
|
+
const token = getToken();
|
|
456
|
+
const tenant = getTenant();
|
|
457
|
+
const actorId = getUser()?.actorId || '';
|
|
458
|
+
const skillParam = skillSlug ? `&skillSlug=${encodeURIComponent(skillSlug)}` : '';
|
|
459
|
+
return `${getVoiceApiUrl()}/app?mode=mini&embedded=rn&authToken=${encodeURIComponent(token || '')}&tenant=${encodeURIComponent(tenant)}&actorId=${encodeURIComponent(actorId)}${skillParam}`;
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
/** Handle a postMessage from the WebView. `event` is the RN WebView event. */
|
|
463
|
+
handleMessage(event: any): void {
|
|
464
|
+
let data: any;
|
|
465
|
+
try {
|
|
466
|
+
data = JSON.parse(event.nativeEvent.data);
|
|
467
|
+
} catch {
|
|
468
|
+
return;
|
|
469
|
+
}
|
|
470
|
+
switch (data.type) {
|
|
471
|
+
case 'callState':
|
|
472
|
+
case 'call-state':
|
|
473
|
+
if (data.state) this.setState({ callState: data.state });
|
|
474
|
+
if (data.sessionId) this.setState({ sessionId: data.sessionId });
|
|
475
|
+
if (data.error) this.setState({ error: data.error });
|
|
476
|
+
if (data.state === 'connected') this.resolvePending();
|
|
477
|
+
if (data.state === 'error') this.rejectPending(new Error(data.error || 'voice: connection error'));
|
|
478
|
+
break;
|
|
479
|
+
case 'transcript':
|
|
480
|
+
this.appendTranscript({ role: data.role, text: data.text, ts: Date.now() });
|
|
481
|
+
break;
|
|
482
|
+
case 'aiPartial':
|
|
483
|
+
this.setState({ aiPartial: data.text });
|
|
484
|
+
break;
|
|
485
|
+
case 'userPartial':
|
|
486
|
+
this.setState({ userPartial: data.text });
|
|
487
|
+
break;
|
|
488
|
+
case 'clearPartials':
|
|
489
|
+
this.setState({ aiPartial: '', userPartial: '' });
|
|
490
|
+
break;
|
|
491
|
+
// User VAD from the mini-app. `at` is when it happened; on speaking:false
|
|
492
|
+
// we stamp lastUserSpeechEndedAt so a solution can gate "speak after N s of
|
|
493
|
+
// silence" on the first real stop, never before the user has spoken.
|
|
494
|
+
case 'speechActivity': {
|
|
495
|
+
const speaking = !!data.speaking;
|
|
496
|
+
const at = typeof data.at === 'number' ? data.at : Date.now();
|
|
497
|
+
this.setState(
|
|
498
|
+
speaking
|
|
499
|
+
? { userSpeaking: true }
|
|
500
|
+
: { userSpeaking: false, lastUserSpeechEndedAt: at },
|
|
501
|
+
);
|
|
502
|
+
break;
|
|
503
|
+
}
|
|
504
|
+
case 'error':
|
|
505
|
+
this.setState({ error: data.message, callState: 'error' });
|
|
506
|
+
this.rejectPending(new Error(data.message || 'voice: error'));
|
|
507
|
+
break;
|
|
508
|
+
// A proactive turn couldn't run (older voice page / hook threw). NON-FATAL:
|
|
509
|
+
// the session stays live; we only surface why the guide didn't speak.
|
|
510
|
+
case 'agentTurnUnsupported':
|
|
511
|
+
case 'agentTurnFailed':
|
|
512
|
+
case 'observeUnsupported':
|
|
513
|
+
case 'observeFailed':
|
|
514
|
+
this.setState({ error: data.message || 'voice: channel notice' });
|
|
515
|
+
break;
|
|
516
|
+
// Device tools — executed locally, origin-checked. The model's function
|
|
517
|
+
// call already arrived on this phone; only the voice app's own origin may
|
|
518
|
+
// drive the camera/GPS.
|
|
519
|
+
case 'device-tool-list': {
|
|
520
|
+
const listSender = originOf(event?.nativeEvent?.url);
|
|
521
|
+
const listExpected = originOf(getVoiceApiUrl());
|
|
522
|
+
if (!listSender || !listExpected || listSender !== listExpected) break;
|
|
523
|
+
this.sendDeviceToolList(data.correlationId);
|
|
524
|
+
break;
|
|
525
|
+
}
|
|
526
|
+
case 'device-tool-call': {
|
|
527
|
+
const sender = originOf(event?.nativeEvent?.url);
|
|
528
|
+
const expected = originOf(getVoiceApiUrl());
|
|
529
|
+
if (!sender || !expected || sender !== expected) {
|
|
530
|
+
console.warn(`[voiceSession] REFUSED device tool call from "${sender || 'unknown origin'}" (expected "${expected}")`);
|
|
531
|
+
break;
|
|
532
|
+
}
|
|
533
|
+
void this.runDeviceTool(data);
|
|
534
|
+
break;
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/** Answer "what device tools do you have?" — the client is the authority. */
|
|
540
|
+
private sendDeviceToolList(correlationId: string): void {
|
|
541
|
+
const tools = listDeviceTools().map(t => ({
|
|
542
|
+
name: t.name,
|
|
543
|
+
description: t.description,
|
|
544
|
+
input_schema: t.input_schema,
|
|
545
|
+
}));
|
|
546
|
+
this.webView?.injectJavaScript?.(
|
|
547
|
+
`window.__adasDeviceToolResult && window.__adasDeviceToolResult(${JSON.stringify(
|
|
548
|
+
JSON.stringify({ correlationId, ok: true, tools }),
|
|
549
|
+
)}); true;`,
|
|
550
|
+
);
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
/** Run a device tool and post the outcome back into the WebView. */
|
|
554
|
+
private async runDeviceTool(data: any): Promise<void> {
|
|
555
|
+
const { correlationId, name, args, timeoutMs } = data || {};
|
|
556
|
+
if (!correlationId || typeof name !== 'string') return;
|
|
557
|
+
let payload: any;
|
|
558
|
+
try {
|
|
559
|
+
const outcome = await callDeviceTool(name, args || {}, { timeoutMs });
|
|
560
|
+
payload = outcome.ok
|
|
561
|
+
? { correlationId, ok: true, result: outcome.result }
|
|
562
|
+
: { correlationId, ok: false, error: outcome.error, reason: outcome.reason };
|
|
563
|
+
} catch (err: any) {
|
|
564
|
+
payload = { correlationId, ok: false, reason: 'failed', error: err?.message || String(err) };
|
|
565
|
+
}
|
|
566
|
+
this.webView?.injectJavaScript?.(
|
|
567
|
+
`window.__adasDeviceToolResult && window.__adasDeviceToolResult(${JSON.stringify(
|
|
568
|
+
JSON.stringify(payload),
|
|
569
|
+
)}); true;`,
|
|
570
|
+
);
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
// ── internals ────────────────────────────────────────────────
|
|
574
|
+
|
|
575
|
+
private setState(partial: Partial<InternalState>): void {
|
|
576
|
+
this.state = { ...this.state, ...partial };
|
|
577
|
+
this.emit({ type: 'state' });
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
private appendTranscript(entry: TranscriptEntry): void {
|
|
581
|
+
this.state = { ...this.state, transcript: [...this.state.transcript, entry] };
|
|
582
|
+
this.emit({ type: 'transcript' });
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
private emit(event: VoiceEvent): void {
|
|
586
|
+
for (const cb of this.listeners) {
|
|
587
|
+
try { cb(event); } catch (e) { console.warn('[voiceSession] listener threw', e); }
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
private waitForConnect(): Promise<VoiceSnapshot> {
|
|
592
|
+
return new Promise<VoiceSnapshot>((resolve, reject) => {
|
|
593
|
+
const timer = setTimeout(() => {
|
|
594
|
+
this.pending = this.pending.filter(p => p.timer !== timer);
|
|
595
|
+
reject(new Error('voice: timed out waiting to connect (is VoiceSessionHost mounted?)'));
|
|
596
|
+
}, CONNECT_TIMEOUT_MS);
|
|
597
|
+
this.pending.push({ resolve, reject, timer });
|
|
598
|
+
});
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
private resolvePending(): void {
|
|
602
|
+
// Everyone still waiting on connect already passed the access check in
|
|
603
|
+
// start(), so they get the real snapshot.
|
|
604
|
+
const snap = this.realSnapshot();
|
|
605
|
+
const waiting = this.pending;
|
|
606
|
+
this.pending = [];
|
|
607
|
+
for (const p of waiting) { clearTimeout(p.timer); p.resolve(snap); }
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
private rejectPending(err: Error): void {
|
|
611
|
+
const waiting = this.pending;
|
|
612
|
+
this.pending = [];
|
|
613
|
+
for (const p of waiting) { clearTimeout(p.timer); p.reject(err); }
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
/** TEST ONLY: reset the singleton to a clean idle state (mirrors the
|
|
617
|
+
* __resetSessionSyncForTest precedent). Not part of any surface. */
|
|
618
|
+
__resetForTest(): void {
|
|
619
|
+
this.rejectPending(new Error('voice: reset'));
|
|
620
|
+
this.state = { ...IDLE };
|
|
621
|
+
this.owner = null;
|
|
622
|
+
this.listeners.clear();
|
|
623
|
+
this.webView = null;
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
/** The one live session. Handed to setVoiceProvider(voiceSession) at startup. */
|
|
628
|
+
export const voiceSession = new VoiceSessionManager();
|