@ateam-ai-mobile/ui 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +95 -7
- package/dist/index.d.ts +95 -7
- package/dist/index.js +219 -36
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +220 -36
- package/dist/index.mjs.map +1 -1
- package/package.json +3 -3
- package/src/plugin/Plugin.tsx +4 -3
- package/src/voice/voiceSession.ts +307 -42
package/dist/index.d.mts
CHANGED
|
@@ -3,7 +3,7 @@ import { Message, JobProgress, UIAction, AdasSession, TenantMembership, HandoffE
|
|
|
3
3
|
import * as React$1 from 'react';
|
|
4
4
|
import React__default, { ReactNode } from 'react';
|
|
5
5
|
import * as _ateam_ai_mobile_plugin_runtime from '@ateam-ai-mobile/plugin-runtime';
|
|
6
|
-
import { PluginSurfaceMeta, PlatformPluginManifest, VoiceState, VoiceTranscriptEntry, VoiceSessionProvider, VoiceStartOptions, VoiceCaller, VoiceSnapshot, VoiceEvent, VoiceSubscription } from '@ateam-ai-mobile/plugin-runtime';
|
|
6
|
+
import { PluginSurfaceMeta, PlatformPluginManifest, VoiceState, VoiceTranscriptEntry, VoiceSessionProvider, VoiceDeviceToolCall, VoicePersona, VoiceStartOptions, VoiceCaller, VoiceSnapshot, VoiceEvent, VoiceSubscription } from '@ateam-ai-mobile/plugin-runtime';
|
|
7
7
|
import * as zustand from 'zustand';
|
|
8
8
|
|
|
9
9
|
interface AdasChatScreenProps {
|
|
@@ -382,6 +382,13 @@ declare function VoiceCallSheet({ visible, onClose, skillSlug }: Props): react_j
|
|
|
382
382
|
|
|
383
383
|
type CallState = VoiceState;
|
|
384
384
|
type TranscriptEntry = VoiceTranscriptEntry;
|
|
385
|
+
/** A finished call, tagged with the session owner's pluginId when the tool is
|
|
386
|
+
* that plugin's own; null when it is not (another solution's tool, the
|
|
387
|
+
* platform's, an unregistered name, no name) — then only the platform sees it. */
|
|
388
|
+
interface DeviceToolRecord {
|
|
389
|
+
owner: string | null;
|
|
390
|
+
call: VoiceDeviceToolCall;
|
|
391
|
+
}
|
|
385
392
|
interface InternalState {
|
|
386
393
|
callState: CallState;
|
|
387
394
|
sessionId: string | null;
|
|
@@ -394,6 +401,11 @@ interface InternalState {
|
|
|
394
401
|
userSpeaking: boolean;
|
|
395
402
|
/** Epoch ms of the last speaking:false, or null before the first utterance. */
|
|
396
403
|
lastUserSpeechEndedAt: number | null;
|
|
404
|
+
/** This session's finished device-tool calls: the LATEST per owner tag (see
|
|
405
|
+
* DeviceToolRecord), in the order they finished. The platform reads the last
|
|
406
|
+
* one; the owning plugin reads its own (lastDeviceToolFor) — one store, both
|
|
407
|
+
* views derived from it. */
|
|
408
|
+
deviceToolCalls: DeviceToolRecord[];
|
|
397
409
|
/** The WebView should be mounted (a session is wanted / live). */
|
|
398
410
|
mounted: boolean;
|
|
399
411
|
/** The WebView source, computed ONCE at start() so state changes (transcript,
|
|
@@ -402,6 +414,9 @@ interface InternalState {
|
|
|
402
414
|
uri: string;
|
|
403
415
|
} | null;
|
|
404
416
|
skillSlug?: string;
|
|
417
|
+
/** Per-call persona override ({ name, style }) — sent to the voice page on the
|
|
418
|
+
* session URL and declared on POST /session; the server layers it. */
|
|
419
|
+
persona?: VoicePersona;
|
|
405
420
|
}
|
|
406
421
|
declare class VoiceSessionManager implements VoiceSessionProvider {
|
|
407
422
|
private state;
|
|
@@ -419,6 +434,45 @@ declare class VoiceSessionManager implements VoiceSessionProvider {
|
|
|
419
434
|
* conversation just because it declared voice.
|
|
420
435
|
*/
|
|
421
436
|
private owner;
|
|
437
|
+
/** EVERY start() still in its pre-mount awaits (tool preload, token refresh).
|
|
438
|
+
* The session is not 'connecting' yet, so a stop() arriving now would otherwise
|
|
439
|
+
* be an idle no-op — and a start would then mount a call for a screen that has
|
|
440
|
+
* already gone. A set, not one slot: two concurrent starts must BOTH be
|
|
441
|
+
* cancellable. stop() cancels every pending start of its caller (the platform:
|
|
442
|
+
* all of them); another plugin's stop cancels none. */
|
|
443
|
+
private pendingStarts;
|
|
444
|
+
/**
|
|
445
|
+
* The device-tool names Core ALLOWED for this voice session — Core's sanitized
|
|
446
|
+
* allowset, delivered to the phone in a `deviceToolAllowset` message.
|
|
447
|
+
* Authorization gates against THIS (necessary, alongside the registry's can-do).
|
|
448
|
+
*
|
|
449
|
+
* WHY (GPT gate 1, corrected): the phone only PROPOSES its available tools
|
|
450
|
+
* (sendDeviceToolList); CORE is the session authority and decides which are
|
|
451
|
+
* accepted (session.device_tools). Freezing the phone's own proposal would let
|
|
452
|
+
* a tool Core REJECTED still run locally. So the authz set is populated from
|
|
453
|
+
* Core's accepted set, never from listDeviceTools(). Until Core's set arrives
|
|
454
|
+
* this is null and every call FAILS CLOSED. Pinned once per session; the live
|
|
455
|
+
* registry (which drifts via lazy loads / host registration / tenant changes)
|
|
456
|
+
* never feeds it.
|
|
457
|
+
*/
|
|
458
|
+
private frozenTools;
|
|
459
|
+
/** The voice `sessionId` the current allowset is BOUND to. Authorization
|
|
460
|
+
* requires it to equal the active session's id (state.sessionId), so a stale
|
|
461
|
+
* allowset from a previous/other session is refused. */
|
|
462
|
+
private frozenSessionId;
|
|
463
|
+
/** A CLEAN call (skills:'none') started by a plugin offers and runs ONLY that
|
|
464
|
+
* plugin's own device tools — not other solutions' tools that happen to be
|
|
465
|
+
* registered on the same phone. null = every registered tool (a normal call,
|
|
466
|
+
* or the platform's). Set by every start(); like `owner`, kept after stop.
|
|
467
|
+
* ONE identity: the caller is the TRUSTED manifest pluginId gateVoice stamps,
|
|
468
|
+
* and the bundleManager registers a plugin's tools under that same id (never
|
|
469
|
+
* the bundle's own short `id`), so exact equality is the whole check. */
|
|
470
|
+
private toolScope;
|
|
471
|
+
/** Bumped whenever a session begins or ends. A device-tool call remembers the
|
|
472
|
+
* epoch it arrived in, and its outcome is recorded only if that session is
|
|
473
|
+
* still the current one — a tool that finishes after hang-up (or after a new
|
|
474
|
+
* call started) must not show up as the new session's last call. */
|
|
475
|
+
private sessionEpoch;
|
|
422
476
|
/** The platform (built-in sheet, mic indicator, session host) sees and
|
|
423
477
|
* controls any session — it is the user's own surface. A plugin caller may
|
|
424
478
|
* only touch a session it owns. */
|
|
@@ -438,23 +492,57 @@ declare class VoiceSessionManager implements VoiceSessionProvider {
|
|
|
438
492
|
prompt(input: string, caller?: VoiceCaller): void;
|
|
439
493
|
/**
|
|
440
494
|
* Stream a silent observation into the live session — tell the model without
|
|
441
|
-
* asking it to speak. The cheap
|
|
442
|
-
*
|
|
443
|
-
*
|
|
444
|
-
* plugin API is added. Owner-scoped like every other control.
|
|
495
|
+
* asking it to speak. The cheap channel behind the gated props.voice.observe();
|
|
496
|
+
* every item stays in the model's context for the call, so a caller sends only
|
|
497
|
+
* material changes. Owner-scoped like every other control.
|
|
445
498
|
*/
|
|
446
499
|
observe(text: string, caller?: VoiceCaller): void;
|
|
447
500
|
getSnapshot(caller?: VoiceCaller): VoiceSnapshot;
|
|
501
|
+
/** The snapshot for a caller that may access the session. Everything is the
|
|
502
|
+
* session's, except `lastDeviceTool`, which is scoped to the caller's tools. */
|
|
448
503
|
private realSnapshot;
|
|
504
|
+
/**
|
|
505
|
+
* The last finished device-tool call THIS caller may see. The platform (the
|
|
506
|
+
* user's own surface) sees every call. A plugin sees only calls to tools it
|
|
507
|
+
* registered — even on a session it owns, the model can call another
|
|
508
|
+
* solution's tool (a normal call offers every registered tool), and that
|
|
509
|
+
* tool's name and error are not this plugin's to read.
|
|
510
|
+
*/
|
|
511
|
+
private lastDeviceToolFor;
|
|
449
512
|
private foreignSnapshot;
|
|
450
513
|
subscribe(cb: (event: VoiceEvent) => void, caller?: VoiceCaller): VoiceSubscription;
|
|
451
514
|
transcript(caller?: VoiceCaller): TranscriptEntry[];
|
|
452
515
|
private buildUri;
|
|
453
516
|
/** Handle a postMessage from the WebView. `event` is the RN WebView event. */
|
|
454
517
|
handleMessage(event: any): void;
|
|
455
|
-
/**
|
|
518
|
+
/**
|
|
519
|
+
* One device-tool call, arrival to outcome: validate → authorize → run. The
|
|
520
|
+
* answer goes back to the mini-app (the model reads it) AND becomes
|
|
521
|
+
* `lastDeviceTool` — the same object, recorded only once it is known, so a
|
|
522
|
+
* refused call never looks like one that ran.
|
|
523
|
+
*/
|
|
524
|
+
private handleDeviceToolCall;
|
|
525
|
+
/**
|
|
526
|
+
* Final authorization = Core's frozen allowset ∩ the current valid local
|
|
527
|
+
* registry, bound to the ACTIVE session. All must hold:
|
|
528
|
+
* - a Core allowset arrived and is bound to THIS session (frozenSessionId ===
|
|
529
|
+
* the active state.sessionId) — a missing or stale/wrong-session allowset
|
|
530
|
+
* fails closed;
|
|
531
|
+
* - the tool is IN that allowset (Core approved it); AND
|
|
532
|
+
* - the tool is STILL registered locally right now (removed-after-approval is
|
|
533
|
+
* refused; a lazily-added tool Core never approved is refused) — and, on a
|
|
534
|
+
* clean call, registered by the call's owner.
|
|
535
|
+
* No fallback to the live registry alone, ever.
|
|
536
|
+
*/
|
|
537
|
+
private isToolAuthorized;
|
|
538
|
+
/** Answer the mini-app's pending device-tool request (a call's outcome, or the
|
|
539
|
+
* tool list) — it resolves the request with this payload by correlationId. */
|
|
540
|
+
private postDeviceToolResult;
|
|
541
|
+
/** PROPOSE the phone's available tools — this is an offer to Core, not the
|
|
542
|
+
* authorization set. Core sanitizes it and returns the accepted set in a
|
|
543
|
+
* `deviceToolAllowset` message, which is what actually gates execution. */
|
|
456
544
|
private sendDeviceToolList;
|
|
457
|
-
/** Run
|
|
545
|
+
/** Run an authorized device tool. Never throws: every failure is an answer. */
|
|
458
546
|
private runDeviceTool;
|
|
459
547
|
private setState;
|
|
460
548
|
private appendTranscript;
|
package/dist/index.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { Message, JobProgress, UIAction, AdasSession, TenantMembership, HandoffE
|
|
|
3
3
|
import * as React$1 from 'react';
|
|
4
4
|
import React__default, { ReactNode } from 'react';
|
|
5
5
|
import * as _ateam_ai_mobile_plugin_runtime from '@ateam-ai-mobile/plugin-runtime';
|
|
6
|
-
import { PluginSurfaceMeta, PlatformPluginManifest, VoiceState, VoiceTranscriptEntry, VoiceSessionProvider, VoiceStartOptions, VoiceCaller, VoiceSnapshot, VoiceEvent, VoiceSubscription } from '@ateam-ai-mobile/plugin-runtime';
|
|
6
|
+
import { PluginSurfaceMeta, PlatformPluginManifest, VoiceState, VoiceTranscriptEntry, VoiceSessionProvider, VoiceDeviceToolCall, VoicePersona, VoiceStartOptions, VoiceCaller, VoiceSnapshot, VoiceEvent, VoiceSubscription } from '@ateam-ai-mobile/plugin-runtime';
|
|
7
7
|
import * as zustand from 'zustand';
|
|
8
8
|
|
|
9
9
|
interface AdasChatScreenProps {
|
|
@@ -382,6 +382,13 @@ declare function VoiceCallSheet({ visible, onClose, skillSlug }: Props): react_j
|
|
|
382
382
|
|
|
383
383
|
type CallState = VoiceState;
|
|
384
384
|
type TranscriptEntry = VoiceTranscriptEntry;
|
|
385
|
+
/** A finished call, tagged with the session owner's pluginId when the tool is
|
|
386
|
+
* that plugin's own; null when it is not (another solution's tool, the
|
|
387
|
+
* platform's, an unregistered name, no name) — then only the platform sees it. */
|
|
388
|
+
interface DeviceToolRecord {
|
|
389
|
+
owner: string | null;
|
|
390
|
+
call: VoiceDeviceToolCall;
|
|
391
|
+
}
|
|
385
392
|
interface InternalState {
|
|
386
393
|
callState: CallState;
|
|
387
394
|
sessionId: string | null;
|
|
@@ -394,6 +401,11 @@ interface InternalState {
|
|
|
394
401
|
userSpeaking: boolean;
|
|
395
402
|
/** Epoch ms of the last speaking:false, or null before the first utterance. */
|
|
396
403
|
lastUserSpeechEndedAt: number | null;
|
|
404
|
+
/** This session's finished device-tool calls: the LATEST per owner tag (see
|
|
405
|
+
* DeviceToolRecord), in the order they finished. The platform reads the last
|
|
406
|
+
* one; the owning plugin reads its own (lastDeviceToolFor) — one store, both
|
|
407
|
+
* views derived from it. */
|
|
408
|
+
deviceToolCalls: DeviceToolRecord[];
|
|
397
409
|
/** The WebView should be mounted (a session is wanted / live). */
|
|
398
410
|
mounted: boolean;
|
|
399
411
|
/** The WebView source, computed ONCE at start() so state changes (transcript,
|
|
@@ -402,6 +414,9 @@ interface InternalState {
|
|
|
402
414
|
uri: string;
|
|
403
415
|
} | null;
|
|
404
416
|
skillSlug?: string;
|
|
417
|
+
/** Per-call persona override ({ name, style }) — sent to the voice page on the
|
|
418
|
+
* session URL and declared on POST /session; the server layers it. */
|
|
419
|
+
persona?: VoicePersona;
|
|
405
420
|
}
|
|
406
421
|
declare class VoiceSessionManager implements VoiceSessionProvider {
|
|
407
422
|
private state;
|
|
@@ -419,6 +434,45 @@ declare class VoiceSessionManager implements VoiceSessionProvider {
|
|
|
419
434
|
* conversation just because it declared voice.
|
|
420
435
|
*/
|
|
421
436
|
private owner;
|
|
437
|
+
/** EVERY start() still in its pre-mount awaits (tool preload, token refresh).
|
|
438
|
+
* The session is not 'connecting' yet, so a stop() arriving now would otherwise
|
|
439
|
+
* be an idle no-op — and a start would then mount a call for a screen that has
|
|
440
|
+
* already gone. A set, not one slot: two concurrent starts must BOTH be
|
|
441
|
+
* cancellable. stop() cancels every pending start of its caller (the platform:
|
|
442
|
+
* all of them); another plugin's stop cancels none. */
|
|
443
|
+
private pendingStarts;
|
|
444
|
+
/**
|
|
445
|
+
* The device-tool names Core ALLOWED for this voice session — Core's sanitized
|
|
446
|
+
* allowset, delivered to the phone in a `deviceToolAllowset` message.
|
|
447
|
+
* Authorization gates against THIS (necessary, alongside the registry's can-do).
|
|
448
|
+
*
|
|
449
|
+
* WHY (GPT gate 1, corrected): the phone only PROPOSES its available tools
|
|
450
|
+
* (sendDeviceToolList); CORE is the session authority and decides which are
|
|
451
|
+
* accepted (session.device_tools). Freezing the phone's own proposal would let
|
|
452
|
+
* a tool Core REJECTED still run locally. So the authz set is populated from
|
|
453
|
+
* Core's accepted set, never from listDeviceTools(). Until Core's set arrives
|
|
454
|
+
* this is null and every call FAILS CLOSED. Pinned once per session; the live
|
|
455
|
+
* registry (which drifts via lazy loads / host registration / tenant changes)
|
|
456
|
+
* never feeds it.
|
|
457
|
+
*/
|
|
458
|
+
private frozenTools;
|
|
459
|
+
/** The voice `sessionId` the current allowset is BOUND to. Authorization
|
|
460
|
+
* requires it to equal the active session's id (state.sessionId), so a stale
|
|
461
|
+
* allowset from a previous/other session is refused. */
|
|
462
|
+
private frozenSessionId;
|
|
463
|
+
/** A CLEAN call (skills:'none') started by a plugin offers and runs ONLY that
|
|
464
|
+
* plugin's own device tools — not other solutions' tools that happen to be
|
|
465
|
+
* registered on the same phone. null = every registered tool (a normal call,
|
|
466
|
+
* or the platform's). Set by every start(); like `owner`, kept after stop.
|
|
467
|
+
* ONE identity: the caller is the TRUSTED manifest pluginId gateVoice stamps,
|
|
468
|
+
* and the bundleManager registers a plugin's tools under that same id (never
|
|
469
|
+
* the bundle's own short `id`), so exact equality is the whole check. */
|
|
470
|
+
private toolScope;
|
|
471
|
+
/** Bumped whenever a session begins or ends. A device-tool call remembers the
|
|
472
|
+
* epoch it arrived in, and its outcome is recorded only if that session is
|
|
473
|
+
* still the current one — a tool that finishes after hang-up (or after a new
|
|
474
|
+
* call started) must not show up as the new session's last call. */
|
|
475
|
+
private sessionEpoch;
|
|
422
476
|
/** The platform (built-in sheet, mic indicator, session host) sees and
|
|
423
477
|
* controls any session — it is the user's own surface. A plugin caller may
|
|
424
478
|
* only touch a session it owns. */
|
|
@@ -438,23 +492,57 @@ declare class VoiceSessionManager implements VoiceSessionProvider {
|
|
|
438
492
|
prompt(input: string, caller?: VoiceCaller): void;
|
|
439
493
|
/**
|
|
440
494
|
* Stream a silent observation into the live session — tell the model without
|
|
441
|
-
* asking it to speak. The cheap
|
|
442
|
-
*
|
|
443
|
-
*
|
|
444
|
-
* plugin API is added. Owner-scoped like every other control.
|
|
495
|
+
* asking it to speak. The cheap channel behind the gated props.voice.observe();
|
|
496
|
+
* every item stays in the model's context for the call, so a caller sends only
|
|
497
|
+
* material changes. Owner-scoped like every other control.
|
|
445
498
|
*/
|
|
446
499
|
observe(text: string, caller?: VoiceCaller): void;
|
|
447
500
|
getSnapshot(caller?: VoiceCaller): VoiceSnapshot;
|
|
501
|
+
/** The snapshot for a caller that may access the session. Everything is the
|
|
502
|
+
* session's, except `lastDeviceTool`, which is scoped to the caller's tools. */
|
|
448
503
|
private realSnapshot;
|
|
504
|
+
/**
|
|
505
|
+
* The last finished device-tool call THIS caller may see. The platform (the
|
|
506
|
+
* user's own surface) sees every call. A plugin sees only calls to tools it
|
|
507
|
+
* registered — even on a session it owns, the model can call another
|
|
508
|
+
* solution's tool (a normal call offers every registered tool), and that
|
|
509
|
+
* tool's name and error are not this plugin's to read.
|
|
510
|
+
*/
|
|
511
|
+
private lastDeviceToolFor;
|
|
449
512
|
private foreignSnapshot;
|
|
450
513
|
subscribe(cb: (event: VoiceEvent) => void, caller?: VoiceCaller): VoiceSubscription;
|
|
451
514
|
transcript(caller?: VoiceCaller): TranscriptEntry[];
|
|
452
515
|
private buildUri;
|
|
453
516
|
/** Handle a postMessage from the WebView. `event` is the RN WebView event. */
|
|
454
517
|
handleMessage(event: any): void;
|
|
455
|
-
/**
|
|
518
|
+
/**
|
|
519
|
+
* One device-tool call, arrival to outcome: validate → authorize → run. The
|
|
520
|
+
* answer goes back to the mini-app (the model reads it) AND becomes
|
|
521
|
+
* `lastDeviceTool` — the same object, recorded only once it is known, so a
|
|
522
|
+
* refused call never looks like one that ran.
|
|
523
|
+
*/
|
|
524
|
+
private handleDeviceToolCall;
|
|
525
|
+
/**
|
|
526
|
+
* Final authorization = Core's frozen allowset ∩ the current valid local
|
|
527
|
+
* registry, bound to the ACTIVE session. All must hold:
|
|
528
|
+
* - a Core allowset arrived and is bound to THIS session (frozenSessionId ===
|
|
529
|
+
* the active state.sessionId) — a missing or stale/wrong-session allowset
|
|
530
|
+
* fails closed;
|
|
531
|
+
* - the tool is IN that allowset (Core approved it); AND
|
|
532
|
+
* - the tool is STILL registered locally right now (removed-after-approval is
|
|
533
|
+
* refused; a lazily-added tool Core never approved is refused) — and, on a
|
|
534
|
+
* clean call, registered by the call's owner.
|
|
535
|
+
* No fallback to the live registry alone, ever.
|
|
536
|
+
*/
|
|
537
|
+
private isToolAuthorized;
|
|
538
|
+
/** Answer the mini-app's pending device-tool request (a call's outcome, or the
|
|
539
|
+
* tool list) — it resolves the request with this payload by correlationId. */
|
|
540
|
+
private postDeviceToolResult;
|
|
541
|
+
/** PROPOSE the phone's available tools — this is an offer to Core, not the
|
|
542
|
+
* authorization set. Core sanitizes it and returns the accepted set in a
|
|
543
|
+
* `deviceToolAllowset` message, which is what actually gates execution. */
|
|
456
544
|
private sendDeviceToolList;
|
|
457
|
-
/** Run
|
|
545
|
+
/** Run an authorized device tool. Never throws: every failure is an answer. */
|
|
458
546
|
private runDeviceTool;
|
|
459
547
|
private setState;
|
|
460
548
|
private appendTranscript;
|