@ateam-ai-mobile/ui 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +95 -7
- package/dist/index.d.ts +95 -7
- package/dist/index.js +219 -36
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +220 -36
- package/dist/index.mjs.map +1 -1
- package/package.json +3 -3
- package/src/plugin/Plugin.tsx +4 -3
- package/src/voice/voiceSession.ts +307 -42
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ateam-ai-mobile/ui",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.9",
|
|
4
4
|
"description": "Prebuilt ADAS chat experience for React Native — full-screen chat, chips, widget views. Built on @ateam-ai-mobile/core + @ateam-ai-mobile/plugin-runtime.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -9,8 +9,8 @@
|
|
|
9
9
|
"build": "tsup"
|
|
10
10
|
},
|
|
11
11
|
"dependencies": {
|
|
12
|
-
"@ateam-ai-mobile/core": "0.1.
|
|
13
|
-
"@ateam-ai-mobile/plugin-runtime": "0.1.
|
|
12
|
+
"@ateam-ai-mobile/core": "0.1.9",
|
|
13
|
+
"@ateam-ai-mobile/plugin-runtime": "0.1.9",
|
|
14
14
|
"zustand": "^5.0.0"
|
|
15
15
|
},
|
|
16
16
|
"peerDependencies": {
|
package/src/plugin/Plugin.tsx
CHANGED
|
@@ -86,9 +86,10 @@ export default function Plugin({ id, manifest, commandBufferRef, props: enginePr
|
|
|
86
86
|
const bridge = useNativeBridge(id, connectorId, commandBufferRef);
|
|
87
87
|
// ONE runtime contract — the SAME resolver every surface uses, so this drawer/
|
|
88
88
|
// fullscreen mount gates device/native + deviceState identically to the
|
|
89
|
-
// embedded mount
|
|
90
|
-
//
|
|
91
|
-
//
|
|
89
|
+
// embedded mount. Memoized so the plugin Component doesn't re-render on a fresh
|
|
90
|
+
// object every pass (useApi effects key off prop identity). The manifest is the
|
|
91
|
+
// sole authority here (it is a required prop); `plugin` is read by the resolver
|
|
92
|
+
// only when there is no manifest, so the loaded bundle never widens its grants.
|
|
92
93
|
const { device, native, deviceState, voice } = useMemo(
|
|
93
94
|
() => resolvePluginRuntimeProps({ pluginId: id, manifest: manifest as any, plugin: plugin as any }),
|
|
94
95
|
[id, manifest, plugin],
|
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
callDeviceTool,
|
|
29
29
|
ensureDeviceToolsLoaded,
|
|
30
30
|
listDeviceTools,
|
|
31
|
+
hasDeviceTool,
|
|
31
32
|
PLATFORM_VOICE_CALLER,
|
|
32
33
|
type VoiceSessionProvider,
|
|
33
34
|
type VoiceCaller,
|
|
@@ -35,8 +36,11 @@ import {
|
|
|
35
36
|
type VoiceState,
|
|
36
37
|
type VoiceTranscriptEntry,
|
|
37
38
|
type VoiceStartOptions,
|
|
39
|
+
type VoicePersona,
|
|
38
40
|
type VoiceEvent,
|
|
39
41
|
type VoiceSubscription,
|
|
42
|
+
type VoiceDeviceToolCall,
|
|
43
|
+
type VoiceDeviceToolReason,
|
|
40
44
|
} from '@ateam-ai-mobile/plugin-runtime';
|
|
41
45
|
|
|
42
46
|
// Re-export the transcript/state shapes under their historical names so existing
|
|
@@ -218,6 +222,33 @@ const buildObserveScript = (text: string) => `
|
|
|
218
222
|
})(${JSON.stringify(text)}); true;
|
|
219
223
|
`;
|
|
220
224
|
|
|
225
|
+
/** How much of a device-tool error the snapshot keeps. The model still gets the
|
|
226
|
+
* whole message; the snapshot is a diagnostic any reader of it can see, and a
|
|
227
|
+
* tool's error can echo the call's arguments (what the user said). */
|
|
228
|
+
const DEVICE_TOOL_ERROR_MAX = 200;
|
|
229
|
+
|
|
230
|
+
function clipError(message: string): string {
|
|
231
|
+
const chars = Array.from(message); // by code point, so an emoji is never split
|
|
232
|
+
return chars.length > DEVICE_TOOL_ERROR_MAX
|
|
233
|
+
? chars.slice(0, DEVICE_TOOL_ERROR_MAX - 1).join('') + '…'
|
|
234
|
+
: message;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** A device-tool call's answer — what the mini-app gets back (plus its
|
|
238
|
+
* correlationId) and what `lastDeviceTool` is built from. ONE object for both,
|
|
239
|
+
* so the model's answer and the snapshot can never disagree. */
|
|
240
|
+
type DeviceToolAnswer =
|
|
241
|
+
| { ok: true; result: any }
|
|
242
|
+
| { ok: false; reason: VoiceDeviceToolReason; error: string };
|
|
243
|
+
|
|
244
|
+
/** A finished call, tagged with the session owner's pluginId when the tool is
|
|
245
|
+
* that plugin's own; null when it is not (another solution's tool, the
|
|
246
|
+
* platform's, an unregistered name, no name) — then only the platform sees it. */
|
|
247
|
+
interface DeviceToolRecord {
|
|
248
|
+
owner: string | null;
|
|
249
|
+
call: VoiceDeviceToolCall;
|
|
250
|
+
}
|
|
251
|
+
|
|
221
252
|
/** Origin of a URL, or null if it can't be parsed. */
|
|
222
253
|
function originOf(url?: string | null): string | null {
|
|
223
254
|
if (!url) return null;
|
|
@@ -237,12 +268,20 @@ interface InternalState {
|
|
|
237
268
|
userSpeaking: boolean;
|
|
238
269
|
/** Epoch ms of the last speaking:false, or null before the first utterance. */
|
|
239
270
|
lastUserSpeechEndedAt: number | null;
|
|
271
|
+
/** This session's finished device-tool calls: the LATEST per owner tag (see
|
|
272
|
+
* DeviceToolRecord), in the order they finished. The platform reads the last
|
|
273
|
+
* one; the owning plugin reads its own (lastDeviceToolFor) — one store, both
|
|
274
|
+
* views derived from it. */
|
|
275
|
+
deviceToolCalls: DeviceToolRecord[];
|
|
240
276
|
/** The WebView should be mounted (a session is wanted / live). */
|
|
241
277
|
mounted: boolean;
|
|
242
278
|
/** The WebView source, computed ONCE at start() so state changes (transcript,
|
|
243
279
|
* partials) never remount the WebView and drop the session. */
|
|
244
280
|
webViewSource: { uri: string } | null;
|
|
245
281
|
skillSlug?: string;
|
|
282
|
+
/** Per-call persona override ({ name, style }) — sent to the voice page on the
|
|
283
|
+
* session URL and declared on POST /session; the server layers it. */
|
|
284
|
+
persona?: VoicePersona;
|
|
246
285
|
}
|
|
247
286
|
|
|
248
287
|
const IDLE: InternalState = {
|
|
@@ -255,12 +294,13 @@ const IDLE: InternalState = {
|
|
|
255
294
|
muted: false,
|
|
256
295
|
userSpeaking: false,
|
|
257
296
|
lastUserSpeechEndedAt: null,
|
|
297
|
+
deviceToolCalls: [],
|
|
258
298
|
mounted: false,
|
|
259
299
|
webViewSource: null,
|
|
260
300
|
skillSlug: undefined,
|
|
261
301
|
};
|
|
262
302
|
|
|
263
|
-
type Pending = { resolve: (s: VoiceSnapshot) => void; reject: (e: Error) => void; timer: any };
|
|
303
|
+
type Pending = { caller: VoiceCaller; resolve: (s: VoiceSnapshot) => void; reject: (e: Error) => void; timer: any };
|
|
264
304
|
|
|
265
305
|
class VoiceSessionManager implements VoiceSessionProvider {
|
|
266
306
|
private state: InternalState = { ...IDLE };
|
|
@@ -279,6 +319,50 @@ class VoiceSessionManager implements VoiceSessionProvider {
|
|
|
279
319
|
*/
|
|
280
320
|
private owner: VoiceCaller | null = null;
|
|
281
321
|
|
|
322
|
+
/** EVERY start() still in its pre-mount awaits (tool preload, token refresh).
|
|
323
|
+
* The session is not 'connecting' yet, so a stop() arriving now would otherwise
|
|
324
|
+
* be an idle no-op — and a start would then mount a call for a screen that has
|
|
325
|
+
* already gone. A set, not one slot: two concurrent starts must BOTH be
|
|
326
|
+
* cancellable. stop() cancels every pending start of its caller (the platform:
|
|
327
|
+
* all of them); another plugin's stop cancels none. */
|
|
328
|
+
private pendingStarts = new Set<{ caller: VoiceCaller; cancelled: boolean }>();
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* The device-tool names Core ALLOWED for this voice session — Core's sanitized
|
|
332
|
+
* allowset, delivered to the phone in a `deviceToolAllowset` message.
|
|
333
|
+
* Authorization gates against THIS (necessary, alongside the registry's can-do).
|
|
334
|
+
*
|
|
335
|
+
* WHY (GPT gate 1, corrected): the phone only PROPOSES its available tools
|
|
336
|
+
* (sendDeviceToolList); CORE is the session authority and decides which are
|
|
337
|
+
* accepted (session.device_tools). Freezing the phone's own proposal would let
|
|
338
|
+
* a tool Core REJECTED still run locally. So the authz set is populated from
|
|
339
|
+
* Core's accepted set, never from listDeviceTools(). Until Core's set arrives
|
|
340
|
+
* this is null and every call FAILS CLOSED. Pinned once per session; the live
|
|
341
|
+
* registry (which drifts via lazy loads / host registration / tenant changes)
|
|
342
|
+
* never feeds it.
|
|
343
|
+
*/
|
|
344
|
+
private frozenTools: Set<string> | null = null;
|
|
345
|
+
|
|
346
|
+
/** The voice `sessionId` the current allowset is BOUND to. Authorization
|
|
347
|
+
* requires it to equal the active session's id (state.sessionId), so a stale
|
|
348
|
+
* allowset from a previous/other session is refused. */
|
|
349
|
+
private frozenSessionId: string | null = null;
|
|
350
|
+
|
|
351
|
+
/** A CLEAN call (skills:'none') started by a plugin offers and runs ONLY that
|
|
352
|
+
* plugin's own device tools — not other solutions' tools that happen to be
|
|
353
|
+
* registered on the same phone. null = every registered tool (a normal call,
|
|
354
|
+
* or the platform's). Set by every start(); like `owner`, kept after stop.
|
|
355
|
+
* ONE identity: the caller is the TRUSTED manifest pluginId gateVoice stamps,
|
|
356
|
+
* and the bundleManager registers a plugin's tools under that same id (never
|
|
357
|
+
* the bundle's own short `id`), so exact equality is the whole check. */
|
|
358
|
+
private toolScope: string | null = null;
|
|
359
|
+
|
|
360
|
+
/** Bumped whenever a session begins or ends. A device-tool call remembers the
|
|
361
|
+
* epoch it arrived in, and its outcome is recorded only if that session is
|
|
362
|
+
* still the current one — a tool that finishes after hang-up (or after a new
|
|
363
|
+
* call started) must not show up as the new session's last call. */
|
|
364
|
+
private sessionEpoch = 0;
|
|
365
|
+
|
|
282
366
|
/** The platform (built-in sheet, mic indicator, session host) sees and
|
|
283
367
|
* controls any session — it is the user's own surface. A plugin caller may
|
|
284
368
|
* only touch a session it owns. */
|
|
@@ -319,12 +403,18 @@ class VoiceSessionManager implements VoiceSessionProvider {
|
|
|
319
403
|
// different skillSlug does NOT restart it.
|
|
320
404
|
if (this.state.callState === 'connected') {
|
|
321
405
|
if (!this.canAccess(caller)) throw this.foreignError();
|
|
322
|
-
return this.realSnapshot();
|
|
406
|
+
return this.realSnapshot(caller);
|
|
323
407
|
}
|
|
324
408
|
if (this.state.callState === 'connecting') {
|
|
325
409
|
if (!this.canAccess(caller)) throw this.foreignError();
|
|
326
|
-
return this.waitForConnect();
|
|
410
|
+
return this.waitForConnect(caller);
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
if (opts?.skills === 'none' && opts?.skillSlug) {
|
|
414
|
+
throw new Error("voice: skills:'none' (a clean call) cannot also hand the mic to a skill (skillSlug).");
|
|
327
415
|
}
|
|
416
|
+
const ticket = { caller, cancelled: false };
|
|
417
|
+
this.pendingStarts.add(ticket);
|
|
328
418
|
|
|
329
419
|
// Device tools only exist once their bundle is evaluated; a user who goes
|
|
330
420
|
// straight to voice can otherwise ask for a tool the phone never loaded.
|
|
@@ -344,8 +434,28 @@ class VoiceSessionManager implements VoiceSessionProvider {
|
|
|
344
434
|
// error rather than us blocking the call.
|
|
345
435
|
}
|
|
346
436
|
|
|
347
|
-
|
|
437
|
+
this.pendingStarts.delete(ticket);
|
|
438
|
+
if (ticket.cancelled) throw new Error('voice: stopped before the session started');
|
|
439
|
+
// Another start() may have mounted a session while we awaited — attach to it
|
|
440
|
+
// (or be refused) exactly as the entry checks would have, never mount twice.
|
|
441
|
+
// (Re-read: the awaits above can change it; the entry checks' narrowing can't.)
|
|
442
|
+
const after = this.getInternal().callState;
|
|
443
|
+
if (after === 'connected') {
|
|
444
|
+
if (!this.canAccess(caller)) throw this.foreignError();
|
|
445
|
+
return this.realSnapshot(caller);
|
|
446
|
+
}
|
|
447
|
+
if (after === 'connecting') {
|
|
448
|
+
if (!this.canAccess(caller)) throw this.foreignError();
|
|
449
|
+
return this.waitForConnect(caller);
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
// Idle → this caller becomes the owner of the new session. A NEW session
|
|
453
|
+
// re-freezes its tool set on the next tool-list request.
|
|
348
454
|
this.owner = caller;
|
|
455
|
+
this.toolScope = opts?.skills === 'none' && typeof caller === 'string' ? caller : null;
|
|
456
|
+
this.frozenTools = null;
|
|
457
|
+
this.frozenSessionId = null;
|
|
458
|
+
this.sessionEpoch++;
|
|
349
459
|
const skillSlug = opts?.skillSlug;
|
|
350
460
|
this.setState({
|
|
351
461
|
callState: 'connecting',
|
|
@@ -354,19 +464,34 @@ class VoiceSessionManager implements VoiceSessionProvider {
|
|
|
354
464
|
aiPartial: '',
|
|
355
465
|
userPartial: '',
|
|
356
466
|
muted: false,
|
|
467
|
+
// A session that ended from inside the voice page (not via stop()) left
|
|
468
|
+
// its calls here; the new session starts with none.
|
|
469
|
+
deviceToolCalls: [],
|
|
357
470
|
mounted: true,
|
|
358
471
|
skillSlug,
|
|
359
|
-
|
|
472
|
+
persona: opts?.persona,
|
|
473
|
+
webViewSource: { uri: this.buildUri(skillSlug, opts?.persona, opts?.skills) },
|
|
360
474
|
});
|
|
361
|
-
return this.waitForConnect();
|
|
475
|
+
return this.waitForConnect(caller);
|
|
362
476
|
}
|
|
363
477
|
|
|
364
478
|
stop(caller: VoiceCaller = PLATFORM_VOICE_CALLER): void {
|
|
365
479
|
// A plugin may not hang up another solution's live call. (Idle stop is a
|
|
366
480
|
// harmless no-op for anyone.)
|
|
367
481
|
if (this.state.callState !== 'idle' && !this.canAccess(caller)) throw this.foreignError();
|
|
482
|
+
// Starts still before their mount: cancel ALL of this caller's (the platform:
|
|
483
|
+
// everyone's) — another plugin must not be able to abort someone's call.
|
|
484
|
+
for (const t of [...this.pendingStarts]) {
|
|
485
|
+
if (caller === PLATFORM_VOICE_CALLER || caller === t.caller) {
|
|
486
|
+
t.cancelled = true;
|
|
487
|
+
this.pendingStarts.delete(t);
|
|
488
|
+
}
|
|
489
|
+
}
|
|
368
490
|
this.webView?.injectJavaScript?.('window.hangUp && window.hangUp(); true;');
|
|
369
491
|
this.rejectPending(new Error('voice: session stopped before it connected'));
|
|
492
|
+
this.frozenTools = null; // the session's authorized set ends with the session
|
|
493
|
+
this.frozenSessionId = null;
|
|
494
|
+
this.sessionEpoch++; // a tool still running for this session records nothing
|
|
370
495
|
this.setState({
|
|
371
496
|
...IDLE,
|
|
372
497
|
// keep the transcript of the call that just ended for any reader still up
|
|
@@ -393,10 +518,9 @@ class VoiceSessionManager implements VoiceSessionProvider {
|
|
|
393
518
|
|
|
394
519
|
/**
|
|
395
520
|
* Stream a silent observation into the live session — tell the model without
|
|
396
|
-
* asking it to speak. The cheap
|
|
397
|
-
*
|
|
398
|
-
*
|
|
399
|
-
* plugin API is added. Owner-scoped like every other control.
|
|
521
|
+
* asking it to speak. The cheap channel behind the gated props.voice.observe();
|
|
522
|
+
* every item stays in the model's context for the call, so a caller sends only
|
|
523
|
+
* material changes. Owner-scoped like every other control.
|
|
400
524
|
*/
|
|
401
525
|
observe(text: string, caller: VoiceCaller = PLATFORM_VOICE_CALLER): void {
|
|
402
526
|
if (this.state.callState !== 'connected') {
|
|
@@ -411,10 +535,12 @@ class VoiceSessionManager implements VoiceSessionProvider {
|
|
|
411
535
|
// nothing about the conversation another solution is holding. This is not a
|
|
412
536
|
// fake idle (invariant 6 is about an UNDECLARED plugin): a declared plugin
|
|
413
537
|
// that does not own the live session genuinely has none of its own.
|
|
414
|
-
return this.canAccess(caller) ? this.realSnapshot() : this.foreignSnapshot();
|
|
538
|
+
return this.canAccess(caller) ? this.realSnapshot(caller) : this.foreignSnapshot();
|
|
415
539
|
}
|
|
416
540
|
|
|
417
|
-
|
|
541
|
+
/** The snapshot for a caller that may access the session. Everything is the
|
|
542
|
+
* session's, except `lastDeviceTool`, which is scoped to the caller's tools. */
|
|
543
|
+
private realSnapshot(caller: VoiceCaller): VoiceSnapshot {
|
|
418
544
|
const s = this.state;
|
|
419
545
|
return {
|
|
420
546
|
state: s.callState,
|
|
@@ -424,14 +550,30 @@ class VoiceSessionManager implements VoiceSessionProvider {
|
|
|
424
550
|
listening: !!s.userPartial,
|
|
425
551
|
userSpeaking: s.userSpeaking,
|
|
426
552
|
lastUserSpeechEndedAt: s.lastUserSpeechEndedAt,
|
|
553
|
+
lastDeviceTool: this.lastDeviceToolFor(caller),
|
|
554
|
+
authorizedSessionId: this.frozenSessionId,
|
|
427
555
|
lastError: s.error,
|
|
428
556
|
};
|
|
429
557
|
}
|
|
430
558
|
|
|
559
|
+
/**
|
|
560
|
+
* The last finished device-tool call THIS caller may see. The platform (the
|
|
561
|
+
* user's own surface) sees every call. A plugin sees only calls to tools it
|
|
562
|
+
* registered — even on a session it owns, the model can call another
|
|
563
|
+
* solution's tool (a normal call offers every registered tool), and that
|
|
564
|
+
* tool's name and error are not this plugin's to read.
|
|
565
|
+
*/
|
|
566
|
+
private lastDeviceToolFor(caller: VoiceCaller): VoiceDeviceToolCall | null {
|
|
567
|
+
const calls = this.state.deviceToolCalls;
|
|
568
|
+
if (caller === PLATFORM_VOICE_CALLER) return calls.length ? calls[calls.length - 1].call : null;
|
|
569
|
+
return calls.find(r => r.owner === caller)?.call ?? null;
|
|
570
|
+
}
|
|
571
|
+
|
|
431
572
|
private foreignSnapshot(): VoiceSnapshot {
|
|
432
573
|
return {
|
|
433
574
|
state: 'idle', sessionId: null, muted: false, speaking: false, listening: false,
|
|
434
|
-
userSpeaking: false, lastUserSpeechEndedAt: null,
|
|
575
|
+
userSpeaking: false, lastUserSpeechEndedAt: null, lastDeviceTool: null,
|
|
576
|
+
authorizedSessionId: null, lastError: null,
|
|
435
577
|
};
|
|
436
578
|
}
|
|
437
579
|
|
|
@@ -451,12 +593,21 @@ class VoiceSessionManager implements VoiceSessionProvider {
|
|
|
451
593
|
|
|
452
594
|
// ── WebView plumbing ─────────────────────────────────────────
|
|
453
595
|
|
|
454
|
-
private buildUri(skillSlug?: string): string {
|
|
596
|
+
private buildUri(skillSlug?: string, persona?: VoicePersona, skills?: VoiceStartOptions['skills']): string {
|
|
455
597
|
const token = getToken();
|
|
456
598
|
const tenant = getTenant();
|
|
457
599
|
const actorId = getUser()?.actorId || '';
|
|
458
600
|
const skillParam = skillSlug ? `&skillSlug=${encodeURIComponent(skillSlug)}` : '';
|
|
459
|
-
|
|
601
|
+
// Declared start option, carried on the same URL as tenant/authToken; the
|
|
602
|
+
// voice page forwards it as POST /session `persona`, where it's sanitized.
|
|
603
|
+
const p = persona && (persona.name || persona.style)
|
|
604
|
+
? { ...(persona.name ? { name: persona.name } : {}), ...(persona.style ? { style: persona.style } : {}) }
|
|
605
|
+
: null;
|
|
606
|
+
const personaParam = p ? `&persona=${encodeURIComponent(JSON.stringify(p))}` : '';
|
|
607
|
+
// Clean call: only the one reduce-only value is ever sent; anything else is
|
|
608
|
+
// not a request at all (the server accepts only 'none' too).
|
|
609
|
+
const skillsParam = skills === 'none' ? '&skills=none' : '';
|
|
610
|
+
return `${getVoiceApiUrl()}/app?mode=mini&embedded=rn&authToken=${encodeURIComponent(token || '')}&tenant=${encodeURIComponent(tenant)}&actorId=${encodeURIComponent(actorId)}${skillParam}${personaParam}${skillsParam}`;
|
|
460
611
|
}
|
|
461
612
|
|
|
462
613
|
/** Handle a postMessage from the WebView. `event` is the RN WebView event. */
|
|
@@ -523,6 +674,31 @@ class VoiceSessionManager implements VoiceSessionProvider {
|
|
|
523
674
|
this.sendDeviceToolList(data.correlationId);
|
|
524
675
|
break;
|
|
525
676
|
}
|
|
677
|
+
// CORE's ALLOWSET for this session — what the model may ASK for. Core sends
|
|
678
|
+
// it (even empty, on purpose) at session start. It is a NECESSARY gate, not
|
|
679
|
+
// the whole story: a call must be in the allowset AND the device must
|
|
680
|
+
// actually have the tool (the registry check inside callDeviceTool) — Core
|
|
681
|
+
// says what may be asked, the device decides what it will do.
|
|
682
|
+
//
|
|
683
|
+
// Origin-checked (only the voice app may set it) and pinned ONCE per session
|
|
684
|
+
// (a later message cannot widen it). An EMPTY allowset (arrived) refuses
|
|
685
|
+
// everything but is distinct from NEVER-arrived (frozenTools stays null);
|
|
686
|
+
// both fail closed here — we never fall back to the phone's own registry,
|
|
687
|
+
// which is the seam this closes.
|
|
688
|
+
case 'deviceToolAllowset': {
|
|
689
|
+
const sender = originOf(event?.nativeEvent?.url);
|
|
690
|
+
const expected = originOf(getVoiceApiUrl());
|
|
691
|
+
if (!sender || !expected || sender !== expected) break;
|
|
692
|
+
// Bind the allowset to its sessionId. Pin (or REPLACE for a different
|
|
693
|
+
// session) but never WIDEN within the same session — a second allowset
|
|
694
|
+
// for the same sessionId is ignored. Requires a string sessionId, so an
|
|
695
|
+
// allowset without one never authorizes anything (fail closed).
|
|
696
|
+
if (Array.isArray(data.tools) && typeof data.sessionId === 'string' && data.sessionId !== this.frozenSessionId) {
|
|
697
|
+
this.frozenTools = new Set(data.tools.filter((n: unknown) => typeof n === 'string'));
|
|
698
|
+
this.frozenSessionId = data.sessionId;
|
|
699
|
+
}
|
|
700
|
+
break;
|
|
701
|
+
}
|
|
526
702
|
case 'device-tool-call': {
|
|
527
703
|
const sender = originOf(event?.nativeEvent?.url);
|
|
528
704
|
const expected = originOf(getVoiceApiUrl());
|
|
@@ -530,44 +706,128 @@ class VoiceSessionManager implements VoiceSessionProvider {
|
|
|
530
706
|
console.warn(`[voiceSession] REFUSED device tool call from "${sender || 'unknown origin'}" (expected "${expected}")`);
|
|
531
707
|
break;
|
|
532
708
|
}
|
|
533
|
-
void this.
|
|
709
|
+
void this.handleDeviceToolCall(data);
|
|
534
710
|
break;
|
|
535
711
|
}
|
|
536
712
|
}
|
|
537
713
|
}
|
|
538
714
|
|
|
539
|
-
/**
|
|
715
|
+
/**
|
|
716
|
+
* One device-tool call, arrival to outcome: validate → authorize → run. The
|
|
717
|
+
* answer goes back to the mini-app (the model reads it) AND becomes
|
|
718
|
+
* `lastDeviceTool` — the same object, recorded only once it is known, so a
|
|
719
|
+
* refused call never looks like one that ran.
|
|
720
|
+
*/
|
|
721
|
+
private async handleDeviceToolCall(data: any): Promise<void> {
|
|
722
|
+
const { correlationId, name, args, timeoutMs } = data || {};
|
|
723
|
+
const at = Date.now();
|
|
724
|
+
const epoch = this.sessionEpoch;
|
|
725
|
+
const toolName = typeof name === 'string' ? name : '';
|
|
726
|
+
// Which plugin may later read this call. The only plugin that can read this
|
|
727
|
+
// session's snapshot at all is its owner (any other gets the foreign one), so
|
|
728
|
+
// the call is tagged with the owner only if the tool is registered BY it, as
|
|
729
|
+
// of arrival — the same check a clean call's scope uses. Anything else
|
|
730
|
+
// (another solution's tool, the platform's, an unregistered name) is null:
|
|
731
|
+
// the platform's to see, no plugin's.
|
|
732
|
+
const sessionOwner = this.owner;
|
|
733
|
+
const owner = typeof sessionOwner === 'string' && toolName && hasDeviceTool(toolName, sessionOwner)
|
|
734
|
+
? sessionOwner
|
|
735
|
+
: null;
|
|
736
|
+
|
|
737
|
+
let answer: DeviceToolAnswer;
|
|
738
|
+
if (!toolName) {
|
|
739
|
+
answer = { ok: false, reason: 'invalid', error: 'The device tool call named no tool.' };
|
|
740
|
+
} else if (!correlationId) {
|
|
741
|
+
answer = {
|
|
742
|
+
ok: false, reason: 'invalid',
|
|
743
|
+
error: `The call to "${toolName}" carried no correlationId, so it could not be answered.`,
|
|
744
|
+
};
|
|
745
|
+
} else if (!this.isToolAuthorized(toolName)) {
|
|
746
|
+
// AUTHORIZATION against the FROZEN session set, not the live registry
|
|
747
|
+
// (GPT gate 1). A tool that appeared after connect — the drift case — is
|
|
748
|
+
// refused here, before execution, even if it is registered right now.
|
|
749
|
+
// Both refusals fail closed, but they are DIFFERENT answers to the model:
|
|
750
|
+
// NOT-YET-TOLD (Core's allowset hasn't arrived) is transient — retry;
|
|
751
|
+
// TOLD-AND-DENIED (allowset present, tool not in it / wrong session /
|
|
752
|
+
// unregistered) is final. Same refusal, different sentence.
|
|
753
|
+
const notReady = this.frozenTools === null;
|
|
754
|
+
console.warn(`[voiceSession] REFUSED "${toolName}" — ${notReady ? 'allowset not received yet' : 'not authorized for this session'}`);
|
|
755
|
+
answer = notReady
|
|
756
|
+
? {
|
|
757
|
+
ok: false, reason: 'not_ready',
|
|
758
|
+
error: 'The voice session is still starting up — its tool allowset has not arrived yet. Try again in a moment.',
|
|
759
|
+
}
|
|
760
|
+
: { ok: false, reason: 'not_authorized', error: `"${toolName}" is not available in this voice session.` };
|
|
761
|
+
} else {
|
|
762
|
+
answer = await this.runDeviceTool(toolName, args, timeoutMs);
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
// A call without a correlationId cannot be answered — nothing is waiting on it.
|
|
766
|
+
if (correlationId) this.postDeviceToolResult({ correlationId, ...answer });
|
|
767
|
+
|
|
768
|
+
if (epoch !== this.sessionEpoch) return; // its session ended while it ran
|
|
769
|
+
const ms = Math.max(0, Date.now() - at);
|
|
770
|
+
const call: VoiceDeviceToolCall = answer.ok
|
|
771
|
+
? { name: toolName, at, ms, ok: true, reason: null, error: null }
|
|
772
|
+
: { name: toolName, at, ms, ok: false, reason: answer.reason, error: clipError(answer.error) };
|
|
773
|
+
this.setState({
|
|
774
|
+
deviceToolCalls: [...this.state.deviceToolCalls.filter(r => r.owner !== owner), { owner, call }],
|
|
775
|
+
});
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
/**
|
|
779
|
+
* Final authorization = Core's frozen allowset ∩ the current valid local
|
|
780
|
+
* registry, bound to the ACTIVE session. All must hold:
|
|
781
|
+
* - a Core allowset arrived and is bound to THIS session (frozenSessionId ===
|
|
782
|
+
* the active state.sessionId) — a missing or stale/wrong-session allowset
|
|
783
|
+
* fails closed;
|
|
784
|
+
* - the tool is IN that allowset (Core approved it); AND
|
|
785
|
+
* - the tool is STILL registered locally right now (removed-after-approval is
|
|
786
|
+
* refused; a lazily-added tool Core never approved is refused) — and, on a
|
|
787
|
+
* clean call, registered by the call's owner.
|
|
788
|
+
* No fallback to the live registry alone, ever.
|
|
789
|
+
*/
|
|
790
|
+
private isToolAuthorized(name: unknown): boolean {
|
|
791
|
+
if (typeof name !== 'string') return false;
|
|
792
|
+
if (this.frozenTools === null || this.frozenSessionId === null) return false;
|
|
793
|
+
if (this.frozenSessionId !== this.state.sessionId) return false;
|
|
794
|
+
if (!this.frozenTools.has(name)) return false;
|
|
795
|
+
return hasDeviceTool(name, this.toolScope ?? undefined); // still registered locally right now
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
/** Answer the mini-app's pending device-tool request (a call's outcome, or the
|
|
799
|
+
* tool list) — it resolves the request with this payload by correlationId. */
|
|
800
|
+
private postDeviceToolResult(payload: { correlationId: string } & Record<string, unknown>): void {
|
|
801
|
+
this.webView?.injectJavaScript?.(
|
|
802
|
+
`window.__adasDeviceToolResult && window.__adasDeviceToolResult(${JSON.stringify(
|
|
803
|
+
JSON.stringify(payload),
|
|
804
|
+
)}); true;`,
|
|
805
|
+
);
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
/** PROPOSE the phone's available tools — this is an offer to Core, not the
|
|
809
|
+
* authorization set. Core sanitizes it and returns the accepted set in a
|
|
810
|
+
* `deviceToolAllowset` message, which is what actually gates execution. */
|
|
540
811
|
private sendDeviceToolList(correlationId: string): void {
|
|
541
|
-
const
|
|
812
|
+
const scope = this.toolScope;
|
|
813
|
+
const tools = listDeviceTools().filter(t => scope === null || t.pluginId === scope).map(t => ({
|
|
542
814
|
name: t.name,
|
|
543
815
|
description: t.description,
|
|
544
816
|
input_schema: t.input_schema,
|
|
545
817
|
}));
|
|
546
|
-
this.
|
|
547
|
-
`window.__adasDeviceToolResult && window.__adasDeviceToolResult(${JSON.stringify(
|
|
548
|
-
JSON.stringify({ correlationId, ok: true, tools }),
|
|
549
|
-
)}); true;`,
|
|
550
|
-
);
|
|
818
|
+
this.postDeviceToolResult({ correlationId, ok: true, tools });
|
|
551
819
|
}
|
|
552
820
|
|
|
553
|
-
/** Run
|
|
554
|
-
private async runDeviceTool(
|
|
555
|
-
const { correlationId, name, args, timeoutMs } = data || {};
|
|
556
|
-
if (!correlationId || typeof name !== 'string') return;
|
|
557
|
-
let payload: any;
|
|
821
|
+
/** Run an authorized device tool. Never throws: every failure is an answer. */
|
|
822
|
+
private async runDeviceTool(name: string, args: any, timeoutMs: number | undefined): Promise<DeviceToolAnswer> {
|
|
558
823
|
try {
|
|
559
824
|
const outcome = await callDeviceTool(name, args || {}, { timeoutMs });
|
|
560
|
-
|
|
561
|
-
? {
|
|
562
|
-
: {
|
|
825
|
+
return outcome.ok
|
|
826
|
+
? { ok: true, result: outcome.result }
|
|
827
|
+
: { ok: false, reason: outcome.reason, error: outcome.error };
|
|
563
828
|
} catch (err: any) {
|
|
564
|
-
|
|
829
|
+
return { ok: false, reason: 'failed', error: err?.message || String(err) };
|
|
565
830
|
}
|
|
566
|
-
this.webView?.injectJavaScript?.(
|
|
567
|
-
`window.__adasDeviceToolResult && window.__adasDeviceToolResult(${JSON.stringify(
|
|
568
|
-
JSON.stringify(payload),
|
|
569
|
-
)}); true;`,
|
|
570
|
-
);
|
|
571
831
|
}
|
|
572
832
|
|
|
573
833
|
// ── internals ────────────────────────────────────────────────
|
|
@@ -588,23 +848,23 @@ class VoiceSessionManager implements VoiceSessionProvider {
|
|
|
588
848
|
}
|
|
589
849
|
}
|
|
590
850
|
|
|
591
|
-
private waitForConnect(): Promise<VoiceSnapshot> {
|
|
851
|
+
private waitForConnect(caller: VoiceCaller): Promise<VoiceSnapshot> {
|
|
592
852
|
return new Promise<VoiceSnapshot>((resolve, reject) => {
|
|
593
853
|
const timer = setTimeout(() => {
|
|
594
854
|
this.pending = this.pending.filter(p => p.timer !== timer);
|
|
595
855
|
reject(new Error('voice: timed out waiting to connect (is VoiceSessionHost mounted?)'));
|
|
596
856
|
}, CONNECT_TIMEOUT_MS);
|
|
597
|
-
this.pending.push({ resolve, reject, timer });
|
|
857
|
+
this.pending.push({ caller, resolve, reject, timer });
|
|
598
858
|
});
|
|
599
859
|
}
|
|
600
860
|
|
|
601
861
|
private resolvePending(): void {
|
|
602
862
|
// Everyone still waiting on connect already passed the access check in
|
|
603
|
-
// start(), so
|
|
604
|
-
|
|
863
|
+
// start(), so each gets the real snapshot — as ITS caller (lastDeviceTool
|
|
864
|
+
// is scoped per caller).
|
|
605
865
|
const waiting = this.pending;
|
|
606
866
|
this.pending = [];
|
|
607
|
-
for (const p of waiting) { clearTimeout(p.timer); p.resolve(
|
|
867
|
+
for (const p of waiting) { clearTimeout(p.timer); p.resolve(this.realSnapshot(p.caller)); }
|
|
608
868
|
}
|
|
609
869
|
|
|
610
870
|
private rejectPending(err: Error): void {
|
|
@@ -619,6 +879,11 @@ class VoiceSessionManager implements VoiceSessionProvider {
|
|
|
619
879
|
this.rejectPending(new Error('voice: reset'));
|
|
620
880
|
this.state = { ...IDLE };
|
|
621
881
|
this.owner = null;
|
|
882
|
+
this.toolScope = null;
|
|
883
|
+
this.pendingStarts.clear();
|
|
884
|
+
this.frozenTools = null;
|
|
885
|
+
this.frozenSessionId = null;
|
|
886
|
+
this.sessionEpoch++;
|
|
622
887
|
this.listeners.clear();
|
|
623
888
|
this.webView = null;
|
|
624
889
|
}
|