@runtypelabs/persona 4.24.0 → 4.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/animations/glyph-cycle.d.cts +1 -1
- package/dist/animations/glyph-cycle.d.ts +1 -1
- package/dist/animations/{types-Db61aI9J.d.cts → types-BwKTZhg9.d.cts} +14 -0
- package/dist/animations/{types-Db61aI9J.d.ts → types-BwKTZhg9.d.ts} +14 -0
- package/dist/animations/wipe.d.cts +1 -1
- package/dist/animations/wipe.d.ts +1 -1
- package/dist/animations-extra.d.cts +14 -0
- package/dist/animations-extra.d.ts +14 -0
- package/dist/approval-ui-QAKWNFNM.js +1 -0
- package/dist/approval-ui.cjs +2 -2
- package/dist/approval-ui.d.cts +243 -8
- package/dist/approval-ui.d.ts +243 -8
- package/dist/approval-ui.js +2 -2
- package/dist/{artifacts-ui-IGB5YKLO.js → artifacts-ui-NJGMM27I.js} +1 -1
- package/dist/artifacts-ui.cjs +4 -4
- package/dist/artifacts-ui.d.cts +243 -8
- package/dist/artifacts-ui.d.ts +243 -8
- package/dist/artifacts-ui.js +5 -5
- package/dist/chunk-2ANTRKTV.js +1 -0
- package/dist/chunk-4HPTPUC6.js +1 -0
- package/dist/chunk-6V3MNEKB.js +1 -0
- package/dist/chunk-HGUCGM3K.js +9 -0
- package/dist/chunk-IO5VVUKP.js +3 -0
- package/dist/chunk-J4J3Y2P2.js +1 -0
- package/dist/chunk-K6HKIP5K.js +1 -0
- package/dist/chunk-K7NA7G7J.js +1 -0
- package/dist/chunk-MXUTS3LB.js +1 -0
- package/dist/chunk-PDPVORUF.js +1 -0
- package/dist/chunk-QPPJUSZH.js +1 -0
- package/dist/chunk-QRKGQPU7.js +1 -0
- package/dist/chunk-S6PBWE75.js +1 -0
- package/dist/chunk-T7QM7YMF.js +1 -0
- package/dist/chunk-TR2H475E.js +1 -0
- package/dist/chunk-UPO4GUFC.js +1 -0
- package/dist/chunk-ZC66NAIR.js +1 -0
- package/dist/client-history-YPEG45Q4.js +1 -0
- package/dist/client-history.js +1 -0
- package/dist/client-stream-42FOI3EC.js +4 -0
- package/dist/client-stream.js +5 -0
- package/dist/codegen.cjs +1 -1
- package/dist/codegen.js +1 -1
- package/dist/context-mentions-E7HJYWO6.js +161 -0
- package/dist/{context-mentions-inline-I2RDEBAK.js → context-mentions-inline-QVESLW55.js} +1 -1
- package/dist/context-mentions.cjs +7 -7
- package/dist/context-mentions.d.cts +243 -8
- package/dist/context-mentions.d.ts +243 -8
- package/dist/context-mentions.js +8 -8
- package/dist/event-stream-view-FOIJHFQO.js +1 -0
- package/dist/event-stream-view.cjs +1 -1
- package/dist/event-stream-view.d.cts +246 -31
- package/dist/event-stream-view.d.ts +246 -31
- package/dist/event-stream-view.js +1 -1
- package/dist/forms-ui.d.cts +295 -667
- package/dist/forms-ui.d.ts +295 -667
- package/dist/history-shell-QDQ44PSE.js +3 -0
- package/dist/history-shell.js +3 -0
- package/dist/{history-view-BD6OXLNA.js → history-view-NCFBI5GR.js} +2 -2
- package/dist/history-view.cjs +2 -2
- package/dist/history-view.d.cts +35 -1
- package/dist/history-view.d.ts +35 -1
- package/dist/history-view.js +2 -2
- package/dist/index.cjs +80 -74
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +301 -738
- package/dist/index.d.ts +301 -738
- package/dist/index.global.js +3 -215
- package/dist/index.global.js.map +1 -1
- package/dist/index.js +81 -75
- package/dist/index.js.map +1 -1
- package/dist/install.global.js +1 -1
- package/dist/install.global.js.map +1 -1
- package/dist/launcher.global.js +3 -67
- package/dist/launcher.global.js.map +1 -1
- package/dist/markdown-parsers.js +22 -22
- package/dist/runtype-tts-entry-NFBVWR7N.js +1 -0
- package/dist/runtype-tts.js +1 -1
- package/dist/session-actions-ISSKSI66.js +1 -0
- package/dist/session-actions.js +1 -0
- package/dist/session-reconnect.d.cts +243 -8
- package/dist/session-reconnect.d.ts +243 -8
- package/dist/smart-dom-reader.cjs +13 -13
- package/dist/smart-dom-reader.d.cts +243 -8
- package/dist/smart-dom-reader.d.ts +243 -8
- package/dist/smart-dom-reader.js +11 -11
- package/dist/theme-editor-preview.cjs +116 -98
- package/dist/theme-editor-preview.d.cts +245 -8
- package/dist/theme-editor-preview.d.ts +245 -8
- package/dist/theme-editor-preview.js +49 -54
- package/dist/theme-editor.cjs +2 -2
- package/dist/theme-editor.d.cts +244 -23
- package/dist/theme-editor.d.ts +244 -23
- package/dist/theme-editor.js +2 -2
- package/dist/theme-reference.d.cts +14 -0
- package/dist/theme-reference.d.ts +14 -0
- package/dist/ui-extras-entry-WAYFKVNW.js +3 -0
- package/dist/ui-extras.js +72 -0
- package/dist/voice-runtime-UOQOFAFY.js +17 -0
- package/dist/voice-runtime.cjs +17 -1
- package/dist/voice-runtime.d.cts +11643 -192
- package/dist/voice-runtime.d.ts +11643 -192
- package/dist/voice-runtime.js +17 -1
- package/dist/voice-worklet-player.cjs +19 -4
- package/dist/voice-worklet-player.d.cts +9 -10
- package/dist/voice-worklet-player.d.ts +9 -10
- package/dist/voice-worklet-player.js +19 -4
- package/dist/webmcp-polyfill.js +1 -1
- package/dist/webmcp-runtime-V6ZWW3PI.js +6 -0
- package/dist/webmcp-runtime.cjs +2 -2
- package/dist/webmcp-runtime.d.cts +1 -33
- package/dist/webmcp-runtime.d.ts +1 -33
- package/dist/webmcp-runtime.js +2 -2
- package/dist/widget.css +1 -1
- package/package.json +11 -5
- package/src/approval-paused-tool-settle.test.ts +221 -0
- package/src/client-approve.test.ts +311 -0
- package/src/client-history-eager.ts +9 -0
- package/src/client-history-loader.ts +23 -0
- package/src/client-history.ts +579 -0
- package/src/client-resume.ts +177 -0
- package/src/client-stream-eager.ts +12 -0
- package/src/client-stream-loader.ts +25 -0
- package/src/client-stream.ts +1880 -0
- package/src/client.test.ts +31 -1
- package/src/client.ts +293 -2607
- package/src/components/approval-actions.test.ts +27 -0
- package/src/components/approval-actions.ts +27 -3
- package/src/components/message-bubble.test.ts +19 -0
- package/src/components/message-bubble.ts +2 -0
- package/src/components/registry.ts +6 -6
- package/src/components/tool-bubble.test.ts +9 -0
- package/src/components/tool-bubble.ts +1 -0
- package/src/history-shell-bundle.test.ts +54 -0
- package/src/history-shell-eager.ts +12 -0
- package/src/history-shell-loader.ts +27 -0
- package/src/history-shell.ts +2362 -0
- package/src/history-view-loader.ts +2 -0
- package/src/history-view.ts +3 -0
- package/src/index-core.ts +8 -0
- package/src/index-global.ts +90 -248
- package/src/index.ts +13 -0
- package/src/install.ts +6 -4
- package/src/internal/runtype-history-provider.test.ts +1 -0
- package/src/internal/runtype-history-provider.ts +28 -13
- package/src/plugins/registry.ts +8 -8
- package/src/session-actions-chunk.test.ts +55 -0
- package/src/session-actions-eager.ts +11 -0
- package/src/session-actions-loader.ts +37 -0
- package/src/session-actions.ts +909 -0
- package/src/session-history.test.ts +3 -1
- package/src/session.ts +875 -1530
- package/src/session.voice.test.ts +1241 -5
- package/src/styles/widget.css +58 -0
- package/src/theme-editor/state.ts +91 -91
- package/src/types.ts +269 -5
- package/src/ui-ask-user-sheet.ts +382 -0
- package/src/ui-extras-eager.ts +10 -0
- package/src/ui-extras-entry.ts +9 -0
- package/src/ui-extras-loader.ts +26 -0
- package/src/ui.artifact-rail-resize.test.ts +2 -1
- package/src/ui.ask-user-chunk.test.ts +93 -0
- package/src/ui.history-chunk.test.ts +2 -1
- package/src/ui.history-config-provider.test.ts +2 -1
- package/src/ui.history-confirm-fallback.test.ts +142 -0
- package/src/ui.history-controller.test.ts +2 -1
- package/src/ui.history-lifecycle.test.ts +2 -1
- package/src/ui.history-render-hooks.test.ts +2 -1
- package/src/ui.history-shell.test.ts +2 -1
- package/src/ui.rail-overlay.test.ts +2 -1
- package/src/ui.rail-resize.test.ts +2 -1
- package/src/ui.rail-trigger-pending.test.ts +2 -1
- package/src/ui.ts +399 -2770
- package/src/ui.voice-prewarm.test.ts +222 -0
- package/src/utils/attachment-manager.ts +67 -67
- package/src/utils/constants.ts +3 -0
- package/src/utils/context-mention-controller.ts +323 -323
- package/src/utils/context-mention-manager.ts +40 -40
- package/src/utils/context-mention-orchestrator.ts +28 -0
- package/src/utils/event-stream-buffer.ts +50 -50
- package/src/utils/event-stream-store.ts +43 -43
- package/src/utils/throughput-tracker.ts +34 -34
- package/src/utils/virtual-scroller.ts +67 -67
- package/src/utils/visitor-store.test.ts +50 -0
- package/src/utils/visitor-store.ts +23 -5
- package/src/voice/__fixtures__/voice-wire/README.md +1 -0
- package/src/voice/__fixtures__/voice-wire/client/cancel.json +3 -0
- package/src/voice/__fixtures__/voice-wire/client/context.json +4 -0
- package/src/voice/__fixtures__/voice-wire/client/delegation_result.json +6 -0
- package/src/voice/__fixtures__/voice-wire/client/delegation_update.json +6 -0
- package/src/voice/__fixtures__/voice-wire/client/ping.json +3 -0
- package/src/voice/__fixtures__/voice-wire/client/playback_progress.json +5 -0
- package/src/voice/__fixtures__/voice-wire/client/start.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/artifact.json +11 -0
- package/src/voice/__fixtures__/voice-wire/server/attached.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/audio_clear.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/audio_end.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_cancelled.json +5 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_completed.json +8 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_started.json +15 -0
- package/src/voice/__fixtures__/voice-wire/server/error.json +6 -0
- package/src/voice/__fixtures__/voice-wire/server/metrics.json +10 -0
- package/src/voice/__fixtures__/voice-wire/server/pong.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/session_config.json +16 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_final.json +7 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_interim.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_partial.json +6 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_update.json +10 -0
- package/src/voice/__fixtures__/voice-wire/server/warning.json +5 -0
- package/src/voice/audio-playback-manager.test.ts +57 -0
- package/src/voice/audio-playback-manager.ts +120 -98
- package/src/voice/browser-voice-provider.ts +37 -37
- package/src/voice/fallback-speech-engine.ts +22 -16
- package/src/voice/keyed-voice-transcript.ts +314 -0
- package/src/voice/mic-state-styles.ts +182 -0
- package/src/voice/read-aloud-controller.ts +46 -44
- package/src/voice/runtype-speech-engine.ts +35 -33
- package/src/voice/runtype-voice-provider.ts +856 -164
- package/src/voice/session-voice-wiring.ts +266 -0
- package/src/voice/voice-delegation.test.ts +71 -0
- package/src/voice/voice-delegation.ts +364 -0
- package/src/voice/voice-turn-tracker.ts +6 -6
- package/src/voice/voice.test.ts +1250 -7
- package/src/voice/worklet-playback-engine.test.ts +81 -0
- package/src/voice/worklet-playback-engine.ts +21 -3
- package/src/voice-runtime-loader.ts +4 -1
- package/src/voice-runtime.ts +7 -0
- package/src/webmcp-runtime-entry.ts +41 -39
- package/dist/approval-ui-3NMYNXBD.js +0 -1
- package/dist/chunk-5EIIHQLQ.js +0 -1
- package/dist/chunk-7CO4T5KB.js +0 -1
- package/dist/chunk-EFJBUH5F.js +0 -1
- package/dist/chunk-R2TMIMDI.js +0 -3
- package/dist/chunk-UTVLOTWB.js +0 -9
- package/dist/context-mentions-45MODWOY.js +0 -161
- package/dist/event-stream-view-GL3ADRY2.js +0 -1
- package/dist/runtype-tts-entry-UJAEF7NZ.js +0 -1
- package/dist/voice-runtime-EFR3R5ES.js +0 -1
- package/dist/webmcp-runtime-JI7T4QYF.js +0 -6
|
@@ -6577,6 +6577,22 @@ type AgentWidgetStatusIndicatorConfig = {
|
|
|
6577
6577
|
/** Status text while a reconnect attempt is in flight. */
|
|
6578
6578
|
resumingText?: string;
|
|
6579
6579
|
};
|
|
6580
|
+
/** Built-in `voiceRecognition.prewarm` triggers. See {@link AgentWidgetVoiceRecognitionConfig.prewarm}. */
|
|
6581
|
+
type AgentWidgetVoicePrewarmTrigger = 'hover';
|
|
6582
|
+
/**
|
|
6583
|
+
* Custom `voiceRecognition.prewarm` trigger. Runs while the composer has a mic
|
|
6584
|
+
* button, and again when the button is rebuilt or `update()` swaps in a
|
|
6585
|
+
* different function. Call `warm()` whenever the visitor shows intent to talk
|
|
6586
|
+
* (it is safe to call repeatedly). Return a cleanup to run on replacement.
|
|
6587
|
+
*/
|
|
6588
|
+
type AgentWidgetVoicePrewarmHook = (context: {
|
|
6589
|
+
/** Fire-and-forget voice warm-up; latched and error-swallowing. */
|
|
6590
|
+
warm: () => void;
|
|
6591
|
+
/** The composer's current mic button. */
|
|
6592
|
+
micButton: HTMLButtonElement;
|
|
6593
|
+
/** The widget's mount element. */
|
|
6594
|
+
mount: HTMLElement;
|
|
6595
|
+
}) => void | (() => void);
|
|
6580
6596
|
type AgentWidgetVoiceRecognitionConfig = {
|
|
6581
6597
|
/**
|
|
6582
6598
|
* Enable the mic button and voice input. The mic still only renders when the
|
|
@@ -6642,6 +6658,29 @@ type AgentWidgetVoiceRecognitionConfig = {
|
|
|
6642
6658
|
* Fires once per turn after the reply completes.
|
|
6643
6659
|
*/
|
|
6644
6660
|
onMetrics?: (metrics: VoiceMetrics) => void;
|
|
6661
|
+
/**
|
|
6662
|
+
* Warm the voice path when the visitor shows intent to talk, before the mic
|
|
6663
|
+
* click, so the call starts faster.
|
|
6664
|
+
*
|
|
6665
|
+
* - `'hover'`: when the pointer enters the mic button, it receives focus, or
|
|
6666
|
+
* (on touch) the first pointer-down on it.
|
|
6667
|
+
* - `false`: never.
|
|
6668
|
+
* - A function: custom trigger. Called with `{ warm, micButton, mount }`
|
|
6669
|
+
* while the mic button exists (and again when the button is rebuilt or
|
|
6670
|
+
* `update()` swaps in a different function); call `warm()` whenever you
|
|
6671
|
+
* decide, e.g. when the panel opens. May return a cleanup that runs on
|
|
6672
|
+
* destroy, rebuild, or replacement. Replaces the built-in trigger.
|
|
6673
|
+
*
|
|
6674
|
+
* What `warm()` does depends on the provider: `runtype` fires its prewarm
|
|
6675
|
+
* (see `provider.runtype.prewarmMode`), a `custom` provider's optional
|
|
6676
|
+
* {@link VoiceProvider.prewarm} runs, and the browser (Web Speech) path
|
|
6677
|
+
* starts the client-token session init (as `sessionInit` does), since speech
|
|
6678
|
+
* recognition itself cannot start before the click. Warming is
|
|
6679
|
+
* fire-and-forget, latched, and never surfaces an error or UI state.
|
|
6680
|
+
*
|
|
6681
|
+
* @default 'hover' in client token mode, otherwise `false`
|
|
6682
|
+
*/
|
|
6683
|
+
prewarm?: AgentWidgetVoicePrewarmTrigger | AgentWidgetVoicePrewarmHook | false;
|
|
6645
6684
|
provider?: {
|
|
6646
6685
|
type: 'browser' | 'runtype' | 'custom';
|
|
6647
6686
|
browser?: {
|
|
@@ -6663,6 +6702,62 @@ type AgentWidgetVoiceRecognitionConfig = {
|
|
|
6663
6702
|
* `@runtypelabs/persona/voice-worklet-player` for a jitter-buffered engine.
|
|
6664
6703
|
*/
|
|
6665
6704
|
createPlaybackEngine?: () => VoicePlaybackEngine | Promise<VoicePlaybackEngine>;
|
|
6705
|
+
/**
|
|
6706
|
+
* How `prewarm()` warms the voice path (see `voiceRecognition.prewarm`).
|
|
6707
|
+
*
|
|
6708
|
+
* - `'request'` (default): a fire-and-forget
|
|
6709
|
+
* `POST {host}/v1/client/agents/{agentId}/voice/prewarm`.
|
|
6710
|
+
* - `'attach'`: open the voice WebSocket ahead of the click without
|
|
6711
|
+
* starting a call; the click then starts the call on that socket. It can
|
|
6712
|
+
* create server work before the visitor clicks, so it is opt-in.
|
|
6713
|
+
* @default 'request'
|
|
6714
|
+
*/
|
|
6715
|
+
prewarmMode?: 'request' | 'attach';
|
|
6716
|
+
/**
|
|
6717
|
+
* `'attach'` mode: how long (ms) an attached socket waits for the click
|
|
6718
|
+
* before it closes. The server clamps it to 30000..600000.
|
|
6719
|
+
* @default 30000
|
|
6720
|
+
*/
|
|
6721
|
+
attachIdleMs?: number;
|
|
6722
|
+
/**
|
|
6723
|
+
* Full-duplex (speech-to-speech, e.g. GPT-Live) calls: run the turns the
|
|
6724
|
+
* voice model hands off to the agent through this widget's normal chat
|
|
6725
|
+
* pipeline. The spoken request becomes the user message, and the answer
|
|
6726
|
+
* renders as a regular assistant message (streaming, tool and approval
|
|
6727
|
+
* UI, WebMCP page tools, persistence) that the voice model then reads
|
|
6728
|
+
* aloud. Only takes effect when the server supports it (it confirms in
|
|
6729
|
+
* `session_config`); otherwise the agent turn runs on the server and its
|
|
6730
|
+
* spoken reply is transcribed, as before. Set `false` to always keep the
|
|
6731
|
+
* agent turn on the server.
|
|
6732
|
+
* @default true
|
|
6733
|
+
*/
|
|
6734
|
+
clientDelegation?: boolean;
|
|
6735
|
+
/**
|
|
6736
|
+
* Full-duplex calls: extra context for the voice model (for example the
|
|
6737
|
+
* page the visitor is on), sent after the chat history as of call start.
|
|
6738
|
+
* The frame goes out once, when the visitor finishes their first
|
|
6739
|
+
* utterance (or at the first agent hand-off). A function is
|
|
6740
|
+
* called at call start and may be async. This text is capped at 4000 characters and the whole frame
|
|
6741
|
+
* (history included) at 8000. Sent only to servers that accept call
|
|
6742
|
+
* context (they announce it in `session_config`).
|
|
6743
|
+
*/
|
|
6744
|
+
callContext?: string | (() => string | Promise<string>);
|
|
6745
|
+
/**
|
|
6746
|
+
* Full-duplex calls with client delegation: how long a tool approval
|
|
6747
|
+
* that a spoken request raised may wait before it is declined as
|
|
6748
|
+
* expired (and the voice model says so). Server-side approval gates
|
|
6749
|
+
* keep their own timeout too. Hanging up never declines it: the
|
|
6750
|
+
* approval card in the chat stays usable. Capped at 540000 (9 minutes),
|
|
6751
|
+
* under the server's 10-minute delegation deadline.
|
|
6752
|
+
* @default 300000
|
|
6753
|
+
*/
|
|
6754
|
+
approvalTimeoutMs?: number;
|
|
6755
|
+
/**
|
|
6756
|
+
* Speech-to-speech calls: the one-line notice shown in the composer
|
|
6757
|
+
* status area when the call starts. `false` hides it.
|
|
6758
|
+
* @default "You're talking to an AI assistant. Voice is processed by OpenAI."
|
|
6759
|
+
*/
|
|
6760
|
+
disclosureText?: string | false;
|
|
6666
6761
|
/** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
|
|
6667
6762
|
pauseDuration?: number;
|
|
6668
6763
|
/** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
|
|
@@ -6892,6 +6987,13 @@ interface VoicePlaybackEngine {
|
|
|
6892
6987
|
onFinished(callback: () => void): void;
|
|
6893
6988
|
/** Release all audio resources. */
|
|
6894
6989
|
destroy(): Promise<void> | void;
|
|
6990
|
+
/**
|
|
6991
|
+
* Optional. Speech-to-speech calls stream reply audio with no end-of-stream
|
|
6992
|
+
* marker, so an engine that holds audio below a prebuffer waterline must, while
|
|
6993
|
+
* continuous, release that held tail on its own (e.g. after a short input gap)
|
|
6994
|
+
* instead of waiting for {@link markStreamEnd}. Engines without a prebuffer can omit it.
|
|
6995
|
+
*/
|
|
6996
|
+
setContinuousMode?(enabled: boolean): void;
|
|
6895
6997
|
}
|
|
6896
6998
|
/**
|
|
6897
6999
|
* A {@link VoicePlaybackEngine} that also supports pause/resume. Returned by
|
|
@@ -6935,14 +7037,23 @@ interface VoiceProvider {
|
|
|
6935
7037
|
* Register a callback for incremental transcript updates during a voice turn.
|
|
6936
7038
|
* `isFinal=false` is a live interim update (user partials, or assistant deltas
|
|
6937
7039
|
* on providers that stream them); `isFinal=true` finalizes that role's text.
|
|
6938
|
-
* On the realtime `runtype` path, interim updates fire for the `user`
|
|
6939
|
-
* the `assistant` arrives as a single final
|
|
6940
|
-
*
|
|
6941
|
-
*
|
|
6942
|
-
|
|
6943
|
-
|
|
6944
|
-
|
|
6945
|
-
|
|
7040
|
+
* On the classic realtime `runtype` path, interim updates fire for the `user`
|
|
7041
|
+
* only and the `assistant` arrives as a single final; without a turnId the
|
|
7042
|
+
* session assumes strictly alternating turns, and providers must discard
|
|
7043
|
+
* cancelled output before emitting the next user final.
|
|
7044
|
+
*
|
|
7045
|
+
* When `metadata.turnId` is present (full-duplex / speech-to-speech, e.g.
|
|
7046
|
+
* GPT-Live), the session reconciles by `(turnId, role)`: each pair owns one
|
|
7047
|
+
* bubble that later updates replace in place, turns may overlap or arrive in
|
|
7048
|
+
* any order, and no empty assistant placeholder is injected. User and
|
|
7049
|
+
* assistant may share a turnId (a late user bubble then renders above its
|
|
7050
|
+
* reply) or use distinct ones. `metadata.startMs` (call-relative ms when the
|
|
7051
|
+
* utterance began) orders a new bubble among the call's other keyed bubbles;
|
|
7052
|
+
* without it bubbles order by first arrival. After a stop, keyed providers
|
|
7053
|
+
* must drop the cancelled reply's output until the server acknowledges the
|
|
7054
|
+
* cancel: the session renders every reply it receives.
|
|
7055
|
+
*/
|
|
7056
|
+
onTranscript?(callback: (role: 'user' | 'assistant', text: string, isFinal: boolean, metadata?: VoiceTranscriptMetadata) => void): void;
|
|
6946
7057
|
/** Register a callback for per-turn latency metrics (realtime path). */
|
|
6947
7058
|
onMetrics?(callback: (metrics: VoiceMetrics) => void): void;
|
|
6948
7059
|
/**
|
|
@@ -6963,6 +7074,116 @@ interface VoiceProvider {
|
|
|
6963
7074
|
deactivateBargeIn?(): Promise<void>;
|
|
6964
7075
|
/** Stop playback / cancel in-flight request without starting recording */
|
|
6965
7076
|
stopPlayback?(): void;
|
|
7077
|
+
/**
|
|
7078
|
+
* Warm the provider ahead of `startListening()` (the widget calls it on mic
|
|
7079
|
+
* intent, see `voiceRecognition.prewarm`). Must be fire-and-forget: never
|
|
7080
|
+
* throw, emit errors or status, or acquire the microphone. May be called
|
|
7081
|
+
* repeatedly; throttle as needed.
|
|
7082
|
+
*/
|
|
7083
|
+
prewarm?(): void;
|
|
7084
|
+
/**
|
|
7085
|
+
* Called by the session once after construction with the chat-side surface
|
|
7086
|
+
* a full-duplex provider uses to hand agent turns to the chat pipeline
|
|
7087
|
+
* ({@link VoiceSessionBridge}). Providers without client delegation omit it.
|
|
7088
|
+
*/
|
|
7089
|
+
setSessionBridge?(bridge: VoiceSessionBridge): void;
|
|
7090
|
+
/** The AI-disclosure notice for the live call, if one should show (speech-to-speech). */
|
|
7091
|
+
getDisclosure?(): string | null;
|
|
7092
|
+
}
|
|
7093
|
+
/** Metadata on a {@link VoiceProvider.onTranscript} update. */
|
|
7094
|
+
type VoiceTranscriptMetadata = {
|
|
7095
|
+
/** Groups updates that belong to one utterance bubble (full duplex). */
|
|
7096
|
+
turnId?: string;
|
|
7097
|
+
/** Call-relative ms when the utterance began; orders late bubbles. */
|
|
7098
|
+
startMs?: number;
|
|
7099
|
+
/** Call-relative ms of the latest audio this update covers. */
|
|
7100
|
+
endMs?: number;
|
|
7101
|
+
/**
|
|
7102
|
+
* The bubble only captions speech (see `AgentWidgetMessage.voiceCaption`).
|
|
7103
|
+
* The chat pipeline owns the conversation, as with client delegation.
|
|
7104
|
+
*/
|
|
7105
|
+
caption?: boolean;
|
|
7106
|
+
};
|
|
7107
|
+
/** An agent turn a full-duplex voice model handed to the widget. */
|
|
7108
|
+
type VoiceDelegationRequest = {
|
|
7109
|
+
/** The voice model's delegation id (`delegation_started.delegationId`). */
|
|
7110
|
+
delegationId: string;
|
|
7111
|
+
/** What the visitor asked, as the voice model heard it. */
|
|
7112
|
+
userText: string;
|
|
7113
|
+
/**
|
|
7114
|
+
* The user transcript utterances `userText` joins, oldest first (the last
|
|
7115
|
+
* is the one it came from). May be empty.
|
|
7116
|
+
*/
|
|
7117
|
+
userUtteranceIds: string[];
|
|
7118
|
+
/** The voice model's view of the conversation it delegated from. */
|
|
7119
|
+
messages: Array<{
|
|
7120
|
+
role: string;
|
|
7121
|
+
content: unknown;
|
|
7122
|
+
}>;
|
|
7123
|
+
};
|
|
7124
|
+
/**
|
|
7125
|
+
* How a delegated turn ended, or that it waits on an approval (contract
|
|
7126
|
+
* Amendment 5). `pending_approval` is the only non-terminal status. Open:
|
|
7127
|
+
* a receiver treats an unknown terminal status as `failed`.
|
|
7128
|
+
* - `completed`: the work ran.
|
|
7129
|
+
* - `denied`: the visitor said no (Deny tap, or a spoken decline).
|
|
7130
|
+
* - `timeout`: the approval lapsed (`approvalTimeoutMs`, or the server's own timeout).
|
|
7131
|
+
* - `cancelled`: replaced by a newer request, or abandoned by the system.
|
|
7132
|
+
* - `failed`: an error.
|
|
7133
|
+
*/
|
|
7134
|
+
type VoiceDelegationStatus = 'completed' | 'pending_approval' | 'denied' | 'timeout' | 'cancelled' | 'failed' | (string & {});
|
|
7135
|
+
/** The terminal result of a turn that parked on approvals. */
|
|
7136
|
+
type VoiceDelegationFollowUp = {
|
|
7137
|
+
status: VoiceDelegationStatus;
|
|
7138
|
+
/** What to read aloud (Markdown allowed). May be empty: nothing more to say. */
|
|
7139
|
+
text: string;
|
|
7140
|
+
};
|
|
7141
|
+
/** The delegated turn's answer, sent back for the voice model to read aloud. */
|
|
7142
|
+
type VoiceDelegationResult = {
|
|
7143
|
+
/** `pending_approval` when `followUp` is set; otherwise terminal. */
|
|
7144
|
+
status: VoiceDelegationStatus;
|
|
7145
|
+
/** The assistant text (Markdown allowed). */
|
|
7146
|
+
text: string;
|
|
7147
|
+
/**
|
|
7148
|
+
* Set when the turn parked on approvals (`text` then asks for the decision).
|
|
7149
|
+
* Resolves with the terminal result once the visitor decides (or the
|
|
7150
|
+
* approval is replaced, declined by voice, or unanswered for
|
|
7151
|
+
* `approvalTimeoutMs`) and the resumed turn finishes; `null` only when
|
|
7152
|
+
* `signal` aborts first (hang-up, or the server cancelled the delegation).
|
|
7153
|
+
*/
|
|
7154
|
+
followUp?: (options: {
|
|
7155
|
+
signal: AbortSignal;
|
|
7156
|
+
approvalTimeoutMs?: number;
|
|
7157
|
+
/** `false`: run only the approval bookkeeping (expiry, supersede); nothing will be read back. @default true */
|
|
7158
|
+
readBack?: boolean;
|
|
7159
|
+
/** The resumed turn stopped on another approval: ask for it too (`text` is its approval script). */
|
|
7160
|
+
onUpdate?: (text: string) => void;
|
|
7161
|
+
}) => Promise<VoiceDelegationFollowUp | null>;
|
|
7162
|
+
};
|
|
7163
|
+
/**
|
|
7164
|
+
* Chat-side surface the session hands a full-duplex provider: the visible
|
|
7165
|
+
* history (for the call-start context) and a way to run a delegated turn
|
|
7166
|
+
* through the normal chat pipeline.
|
|
7167
|
+
*/
|
|
7168
|
+
interface VoiceSessionBridge {
|
|
7169
|
+
/** Visible user/assistant messages, oldest first. */
|
|
7170
|
+
getHistory(): Array<{
|
|
7171
|
+
role: 'user' | 'assistant';
|
|
7172
|
+
content: string;
|
|
7173
|
+
}>;
|
|
7174
|
+
/**
|
|
7175
|
+
* Submit the turn as the visitor's message (reusing its transcript bubble)
|
|
7176
|
+
* and resolve with the assistant's final answer once the turn settles.
|
|
7177
|
+
* Waits for any chat turn already in flight first. Never rejects.
|
|
7178
|
+
*/
|
|
7179
|
+
runDelegatedTurn(request: VoiceDelegationRequest): Promise<VoiceDelegationResult>;
|
|
7180
|
+
/**
|
|
7181
|
+
* The server dropped this delegation's result (cancelled it, or refused a
|
|
7182
|
+
* late result): its chat answer is no longer spoken, so browser TTS may read it.
|
|
7183
|
+
* `expired`: the server's deadline passed, so its pending approvals are
|
|
7184
|
+
* declined too, as the approval TTL would.
|
|
7185
|
+
*/
|
|
7186
|
+
dropDelegation?(delegationId: string, expired?: boolean): void;
|
|
6966
7187
|
}
|
|
6967
7188
|
/**
|
|
6968
7189
|
* Extra context for an approval decision, surfaced to `onDecision` and to the
|
|
@@ -9559,6 +9780,11 @@ type AgentWidgetToolCall = {
|
|
|
9559
9780
|
startedAt?: number;
|
|
9560
9781
|
completedAt?: number;
|
|
9561
9782
|
durationMs?: number;
|
|
9783
|
+
/**
|
|
9784
|
+
* The call paused for an approval that was granted, and the agent re-ran it
|
|
9785
|
+
* as a new tool call: that later call carries the result, this one has none.
|
|
9786
|
+
*/
|
|
9787
|
+
superseded?: boolean;
|
|
9562
9788
|
};
|
|
9563
9789
|
/**
|
|
9564
9790
|
* Represents a tool approval request in the chat conversation.
|
|
@@ -9580,6 +9806,8 @@ type AgentWidgetApproval = {
|
|
|
9580
9806
|
*/
|
|
9581
9807
|
reason?: string;
|
|
9582
9808
|
parameters?: unknown;
|
|
9809
|
+
/** The paused tool call this approval gates (`approval_start.toolCallId`). */
|
|
9810
|
+
toolCallId?: string;
|
|
9583
9811
|
resolvedAt?: number;
|
|
9584
9812
|
};
|
|
9585
9813
|
type AgentWidgetMessageVariant = "assistant" | "reasoning" | "tool" | "approval";
|
|
@@ -9679,6 +9907,13 @@ type AgentWidgetMessage = {
|
|
|
9679
9907
|
* }
|
|
9680
9908
|
*/
|
|
9681
9909
|
voiceProcessing?: boolean;
|
|
9910
|
+
/**
|
|
9911
|
+
* Display-only caption of full-duplex voice speech, such as the voice
|
|
9912
|
+
* model's own words, or something the visitor said that wasn't submitted
|
|
9913
|
+
* as a chat turn. It renders in the transcript but is never sent to the
|
|
9914
|
+
* model as conversation.
|
|
9915
|
+
*/
|
|
9916
|
+
voiceCaption?: boolean;
|
|
9682
9917
|
/**
|
|
9683
9918
|
* Raw structured payload for this message (e.g., JSON action response).
|
|
9684
9919
|
* Populated automatically when structured parsers run.
|
|
@@ -6577,6 +6577,22 @@ type AgentWidgetStatusIndicatorConfig = {
|
|
|
6577
6577
|
/** Status text while a reconnect attempt is in flight. */
|
|
6578
6578
|
resumingText?: string;
|
|
6579
6579
|
};
|
|
6580
|
+
/** Built-in `voiceRecognition.prewarm` triggers. See {@link AgentWidgetVoiceRecognitionConfig.prewarm}. */
|
|
6581
|
+
type AgentWidgetVoicePrewarmTrigger = 'hover';
|
|
6582
|
+
/**
|
|
6583
|
+
* Custom `voiceRecognition.prewarm` trigger. Runs while the composer has a mic
|
|
6584
|
+
* button, and again when the button is rebuilt or `update()` swaps in a
|
|
6585
|
+
* different function. Call `warm()` whenever the visitor shows intent to talk
|
|
6586
|
+
* (it is safe to call repeatedly). Return a cleanup to run on replacement.
|
|
6587
|
+
*/
|
|
6588
|
+
type AgentWidgetVoicePrewarmHook = (context: {
|
|
6589
|
+
/** Fire-and-forget voice warm-up; latched and error-swallowing. */
|
|
6590
|
+
warm: () => void;
|
|
6591
|
+
/** The composer's current mic button. */
|
|
6592
|
+
micButton: HTMLButtonElement;
|
|
6593
|
+
/** The widget's mount element. */
|
|
6594
|
+
mount: HTMLElement;
|
|
6595
|
+
}) => void | (() => void);
|
|
6580
6596
|
type AgentWidgetVoiceRecognitionConfig = {
|
|
6581
6597
|
/**
|
|
6582
6598
|
* Enable the mic button and voice input. The mic still only renders when the
|
|
@@ -6642,6 +6658,29 @@ type AgentWidgetVoiceRecognitionConfig = {
|
|
|
6642
6658
|
* Fires once per turn after the reply completes.
|
|
6643
6659
|
*/
|
|
6644
6660
|
onMetrics?: (metrics: VoiceMetrics) => void;
|
|
6661
|
+
/**
|
|
6662
|
+
* Warm the voice path when the visitor shows intent to talk, before the mic
|
|
6663
|
+
* click, so the call starts faster.
|
|
6664
|
+
*
|
|
6665
|
+
* - `'hover'`: when the pointer enters the mic button, it receives focus, or
|
|
6666
|
+
* (on touch) the first pointer-down on it.
|
|
6667
|
+
* - `false`: never.
|
|
6668
|
+
* - A function: custom trigger. Called with `{ warm, micButton, mount }`
|
|
6669
|
+
* while the mic button exists (and again when the button is rebuilt or
|
|
6670
|
+
* `update()` swaps in a different function); call `warm()` whenever you
|
|
6671
|
+
* decide, e.g. when the panel opens. May return a cleanup that runs on
|
|
6672
|
+
* destroy, rebuild, or replacement. Replaces the built-in trigger.
|
|
6673
|
+
*
|
|
6674
|
+
* What `warm()` does depends on the provider: `runtype` fires its prewarm
|
|
6675
|
+
* (see `provider.runtype.prewarmMode`), a `custom` provider's optional
|
|
6676
|
+
* {@link VoiceProvider.prewarm} runs, and the browser (Web Speech) path
|
|
6677
|
+
* starts the client-token session init (as `sessionInit` does), since speech
|
|
6678
|
+
* recognition itself cannot start before the click. Warming is
|
|
6679
|
+
* fire-and-forget, latched, and never surfaces an error or UI state.
|
|
6680
|
+
*
|
|
6681
|
+
* @default 'hover' in client token mode, otherwise `false`
|
|
6682
|
+
*/
|
|
6683
|
+
prewarm?: AgentWidgetVoicePrewarmTrigger | AgentWidgetVoicePrewarmHook | false;
|
|
6645
6684
|
provider?: {
|
|
6646
6685
|
type: 'browser' | 'runtype' | 'custom';
|
|
6647
6686
|
browser?: {
|
|
@@ -6663,6 +6702,62 @@ type AgentWidgetVoiceRecognitionConfig = {
|
|
|
6663
6702
|
* `@runtypelabs/persona/voice-worklet-player` for a jitter-buffered engine.
|
|
6664
6703
|
*/
|
|
6665
6704
|
createPlaybackEngine?: () => VoicePlaybackEngine | Promise<VoicePlaybackEngine>;
|
|
6705
|
+
/**
|
|
6706
|
+
* How `prewarm()` warms the voice path (see `voiceRecognition.prewarm`).
|
|
6707
|
+
*
|
|
6708
|
+
* - `'request'` (default): a fire-and-forget
|
|
6709
|
+
* `POST {host}/v1/client/agents/{agentId}/voice/prewarm`.
|
|
6710
|
+
* - `'attach'`: open the voice WebSocket ahead of the click without
|
|
6711
|
+
* starting a call; the click then starts the call on that socket. It can
|
|
6712
|
+
* create server work before the visitor clicks, so it is opt-in.
|
|
6713
|
+
* @default 'request'
|
|
6714
|
+
*/
|
|
6715
|
+
prewarmMode?: 'request' | 'attach';
|
|
6716
|
+
/**
|
|
6717
|
+
* `'attach'` mode: how long (ms) an attached socket waits for the click
|
|
6718
|
+
* before it closes. The server clamps it to 30000..600000.
|
|
6719
|
+
* @default 30000
|
|
6720
|
+
*/
|
|
6721
|
+
attachIdleMs?: number;
|
|
6722
|
+
/**
|
|
6723
|
+
* Full-duplex (speech-to-speech, e.g. GPT-Live) calls: run the turns the
|
|
6724
|
+
* voice model hands off to the agent through this widget's normal chat
|
|
6725
|
+
* pipeline. The spoken request becomes the user message, and the answer
|
|
6726
|
+
* renders as a regular assistant message (streaming, tool and approval
|
|
6727
|
+
* UI, WebMCP page tools, persistence) that the voice model then reads
|
|
6728
|
+
* aloud. Only takes effect when the server supports it (it confirms in
|
|
6729
|
+
* `session_config`); otherwise the agent turn runs on the server and its
|
|
6730
|
+
* spoken reply is transcribed, as before. Set `false` to always keep the
|
|
6731
|
+
* agent turn on the server.
|
|
6732
|
+
* @default true
|
|
6733
|
+
*/
|
|
6734
|
+
clientDelegation?: boolean;
|
|
6735
|
+
/**
|
|
6736
|
+
* Full-duplex calls: extra context for the voice model (for example the
|
|
6737
|
+
* page the visitor is on), sent after the chat history as of call start.
|
|
6738
|
+
* The frame goes out once, when the visitor finishes their first
|
|
6739
|
+
* utterance (or at the first agent hand-off). A function is
|
|
6740
|
+
* called at call start and may be async. This text is capped at 4000 characters and the whole frame
|
|
6741
|
+
* (history included) at 8000. Sent only to servers that accept call
|
|
6742
|
+
* context (they announce it in `session_config`).
|
|
6743
|
+
*/
|
|
6744
|
+
callContext?: string | (() => string | Promise<string>);
|
|
6745
|
+
/**
|
|
6746
|
+
* Full-duplex calls with client delegation: how long a tool approval
|
|
6747
|
+
* that a spoken request raised may wait before it is declined as
|
|
6748
|
+
* expired (and the voice model says so). Server-side approval gates
|
|
6749
|
+
* keep their own timeout too. Hanging up never declines it: the
|
|
6750
|
+
* approval card in the chat stays usable. Capped at 540000 (9 minutes),
|
|
6751
|
+
* under the server's 10-minute delegation deadline.
|
|
6752
|
+
* @default 300000
|
|
6753
|
+
*/
|
|
6754
|
+
approvalTimeoutMs?: number;
|
|
6755
|
+
/**
|
|
6756
|
+
* Speech-to-speech calls: the one-line notice shown in the composer
|
|
6757
|
+
* status area when the call starts. `false` hides it.
|
|
6758
|
+
* @default "You're talking to an AI assistant. Voice is processed by OpenAI."
|
|
6759
|
+
*/
|
|
6760
|
+
disclosureText?: string | false;
|
|
6666
6761
|
/** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
|
|
6667
6762
|
pauseDuration?: number;
|
|
6668
6763
|
/** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
|
|
@@ -6892,6 +6987,13 @@ interface VoicePlaybackEngine {
|
|
|
6892
6987
|
onFinished(callback: () => void): void;
|
|
6893
6988
|
/** Release all audio resources. */
|
|
6894
6989
|
destroy(): Promise<void> | void;
|
|
6990
|
+
/**
|
|
6991
|
+
* Optional. Speech-to-speech calls stream reply audio with no end-of-stream
|
|
6992
|
+
* marker, so an engine that holds audio below a prebuffer waterline must, while
|
|
6993
|
+
* continuous, release that held tail on its own (e.g. after a short input gap)
|
|
6994
|
+
* instead of waiting for {@link markStreamEnd}. Engines without a prebuffer can omit it.
|
|
6995
|
+
*/
|
|
6996
|
+
setContinuousMode?(enabled: boolean): void;
|
|
6895
6997
|
}
|
|
6896
6998
|
/**
|
|
6897
6999
|
* A {@link VoicePlaybackEngine} that also supports pause/resume. Returned by
|
|
@@ -6935,14 +7037,23 @@ interface VoiceProvider {
|
|
|
6935
7037
|
* Register a callback for incremental transcript updates during a voice turn.
|
|
6936
7038
|
* `isFinal=false` is a live interim update (user partials, or assistant deltas
|
|
6937
7039
|
* on providers that stream them); `isFinal=true` finalizes that role's text.
|
|
6938
|
-
* On the realtime `runtype` path, interim updates fire for the `user`
|
|
6939
|
-
* the `assistant` arrives as a single final
|
|
6940
|
-
*
|
|
6941
|
-
*
|
|
6942
|
-
|
|
6943
|
-
|
|
6944
|
-
|
|
6945
|
-
|
|
7040
|
+
* On the classic realtime `runtype` path, interim updates fire for the `user`
|
|
7041
|
+
* only and the `assistant` arrives as a single final; without a turnId the
|
|
7042
|
+
* session assumes strictly alternating turns, and providers must discard
|
|
7043
|
+
* cancelled output before emitting the next user final.
|
|
7044
|
+
*
|
|
7045
|
+
* When `metadata.turnId` is present (full-duplex / speech-to-speech, e.g.
|
|
7046
|
+
* GPT-Live), the session reconciles by `(turnId, role)`: each pair owns one
|
|
7047
|
+
* bubble that later updates replace in place, turns may overlap or arrive in
|
|
7048
|
+
* any order, and no empty assistant placeholder is injected. User and
|
|
7049
|
+
* assistant may share a turnId (a late user bubble then renders above its
|
|
7050
|
+
* reply) or use distinct ones. `metadata.startMs` (call-relative ms when the
|
|
7051
|
+
* utterance began) orders a new bubble among the call's other keyed bubbles;
|
|
7052
|
+
* without it bubbles order by first arrival. After a stop, keyed providers
|
|
7053
|
+
* must drop the cancelled reply's output until the server acknowledges the
|
|
7054
|
+
* cancel: the session renders every reply it receives.
|
|
7055
|
+
*/
|
|
7056
|
+
onTranscript?(callback: (role: 'user' | 'assistant', text: string, isFinal: boolean, metadata?: VoiceTranscriptMetadata) => void): void;
|
|
6946
7057
|
/** Register a callback for per-turn latency metrics (realtime path). */
|
|
6947
7058
|
onMetrics?(callback: (metrics: VoiceMetrics) => void): void;
|
|
6948
7059
|
/**
|
|
@@ -6963,6 +7074,116 @@ interface VoiceProvider {
|
|
|
6963
7074
|
deactivateBargeIn?(): Promise<void>;
|
|
6964
7075
|
/** Stop playback / cancel in-flight request without starting recording */
|
|
6965
7076
|
stopPlayback?(): void;
|
|
7077
|
+
/**
|
|
7078
|
+
* Warm the provider ahead of `startListening()` (the widget calls it on mic
|
|
7079
|
+
* intent, see `voiceRecognition.prewarm`). Must be fire-and-forget: never
|
|
7080
|
+
* throw, emit errors or status, or acquire the microphone. May be called
|
|
7081
|
+
* repeatedly; throttle as needed.
|
|
7082
|
+
*/
|
|
7083
|
+
prewarm?(): void;
|
|
7084
|
+
/**
|
|
7085
|
+
* Called by the session once after construction with the chat-side surface
|
|
7086
|
+
* a full-duplex provider uses to hand agent turns to the chat pipeline
|
|
7087
|
+
* ({@link VoiceSessionBridge}). Providers without client delegation omit it.
|
|
7088
|
+
*/
|
|
7089
|
+
setSessionBridge?(bridge: VoiceSessionBridge): void;
|
|
7090
|
+
/** The AI-disclosure notice for the live call, if one should show (speech-to-speech). */
|
|
7091
|
+
getDisclosure?(): string | null;
|
|
7092
|
+
}
|
|
7093
|
+
/** Metadata on a {@link VoiceProvider.onTranscript} update. */
|
|
7094
|
+
type VoiceTranscriptMetadata = {
|
|
7095
|
+
/** Groups updates that belong to one utterance bubble (full duplex). */
|
|
7096
|
+
turnId?: string;
|
|
7097
|
+
/** Call-relative ms when the utterance began; orders late bubbles. */
|
|
7098
|
+
startMs?: number;
|
|
7099
|
+
/** Call-relative ms of the latest audio this update covers. */
|
|
7100
|
+
endMs?: number;
|
|
7101
|
+
/**
|
|
7102
|
+
* The bubble only captions speech (see `AgentWidgetMessage.voiceCaption`).
|
|
7103
|
+
* The chat pipeline owns the conversation, as with client delegation.
|
|
7104
|
+
*/
|
|
7105
|
+
caption?: boolean;
|
|
7106
|
+
};
|
|
7107
|
+
/** An agent turn a full-duplex voice model handed to the widget. */
|
|
7108
|
+
type VoiceDelegationRequest = {
|
|
7109
|
+
/** The voice model's delegation id (`delegation_started.delegationId`). */
|
|
7110
|
+
delegationId: string;
|
|
7111
|
+
/** What the visitor asked, as the voice model heard it. */
|
|
7112
|
+
userText: string;
|
|
7113
|
+
/**
|
|
7114
|
+
* The user transcript utterances `userText` joins, oldest first (the last
|
|
7115
|
+
* is the one it came from). May be empty.
|
|
7116
|
+
*/
|
|
7117
|
+
userUtteranceIds: string[];
|
|
7118
|
+
/** The voice model's view of the conversation it delegated from. */
|
|
7119
|
+
messages: Array<{
|
|
7120
|
+
role: string;
|
|
7121
|
+
content: unknown;
|
|
7122
|
+
}>;
|
|
7123
|
+
};
|
|
7124
|
+
/**
|
|
7125
|
+
* How a delegated turn ended, or that it waits on an approval (contract
|
|
7126
|
+
* Amendment 5). `pending_approval` is the only non-terminal status. Open:
|
|
7127
|
+
* a receiver treats an unknown terminal status as `failed`.
|
|
7128
|
+
* - `completed`: the work ran.
|
|
7129
|
+
* - `denied`: the visitor said no (Deny tap, or a spoken decline).
|
|
7130
|
+
* - `timeout`: the approval lapsed (`approvalTimeoutMs`, or the server's own timeout).
|
|
7131
|
+
* - `cancelled`: replaced by a newer request, or abandoned by the system.
|
|
7132
|
+
* - `failed`: an error.
|
|
7133
|
+
*/
|
|
7134
|
+
type VoiceDelegationStatus = 'completed' | 'pending_approval' | 'denied' | 'timeout' | 'cancelled' | 'failed' | (string & {});
|
|
7135
|
+
/** The terminal result of a turn that parked on approvals. */
|
|
7136
|
+
type VoiceDelegationFollowUp = {
|
|
7137
|
+
status: VoiceDelegationStatus;
|
|
7138
|
+
/** What to read aloud (Markdown allowed). May be empty: nothing more to say. */
|
|
7139
|
+
text: string;
|
|
7140
|
+
};
|
|
7141
|
+
/** The delegated turn's answer, sent back for the voice model to read aloud. */
|
|
7142
|
+
type VoiceDelegationResult = {
|
|
7143
|
+
/** `pending_approval` when `followUp` is set; otherwise terminal. */
|
|
7144
|
+
status: VoiceDelegationStatus;
|
|
7145
|
+
/** The assistant text (Markdown allowed). */
|
|
7146
|
+
text: string;
|
|
7147
|
+
/**
|
|
7148
|
+
* Set when the turn parked on approvals (`text` then asks for the decision).
|
|
7149
|
+
* Resolves with the terminal result once the visitor decides (or the
|
|
7150
|
+
* approval is replaced, declined by voice, or unanswered for
|
|
7151
|
+
* `approvalTimeoutMs`) and the resumed turn finishes; `null` only when
|
|
7152
|
+
* `signal` aborts first (hang-up, or the server cancelled the delegation).
|
|
7153
|
+
*/
|
|
7154
|
+
followUp?: (options: {
|
|
7155
|
+
signal: AbortSignal;
|
|
7156
|
+
approvalTimeoutMs?: number;
|
|
7157
|
+
/** `false`: run only the approval bookkeeping (expiry, supersede); nothing will be read back. @default true */
|
|
7158
|
+
readBack?: boolean;
|
|
7159
|
+
/** The resumed turn stopped on another approval: ask for it too (`text` is its approval script). */
|
|
7160
|
+
onUpdate?: (text: string) => void;
|
|
7161
|
+
}) => Promise<VoiceDelegationFollowUp | null>;
|
|
7162
|
+
};
|
|
7163
|
+
/**
|
|
7164
|
+
* Chat-side surface the session hands a full-duplex provider: the visible
|
|
7165
|
+
* history (for the call-start context) and a way to run a delegated turn
|
|
7166
|
+
* through the normal chat pipeline.
|
|
7167
|
+
*/
|
|
7168
|
+
interface VoiceSessionBridge {
|
|
7169
|
+
/** Visible user/assistant messages, oldest first. */
|
|
7170
|
+
getHistory(): Array<{
|
|
7171
|
+
role: 'user' | 'assistant';
|
|
7172
|
+
content: string;
|
|
7173
|
+
}>;
|
|
7174
|
+
/**
|
|
7175
|
+
* Submit the turn as the visitor's message (reusing its transcript bubble)
|
|
7176
|
+
* and resolve with the assistant's final answer once the turn settles.
|
|
7177
|
+
* Waits for any chat turn already in flight first. Never rejects.
|
|
7178
|
+
*/
|
|
7179
|
+
runDelegatedTurn(request: VoiceDelegationRequest): Promise<VoiceDelegationResult>;
|
|
7180
|
+
/**
|
|
7181
|
+
* The server dropped this delegation's result (cancelled it, or refused a
|
|
7182
|
+
* late result): its chat answer is no longer spoken, so browser TTS may read it.
|
|
7183
|
+
* `expired`: the server's deadline passed, so its pending approvals are
|
|
7184
|
+
* declined too, as the approval TTL would.
|
|
7185
|
+
*/
|
|
7186
|
+
dropDelegation?(delegationId: string, expired?: boolean): void;
|
|
6966
7187
|
}
|
|
6967
7188
|
/**
|
|
6968
7189
|
* Extra context for an approval decision, surfaced to `onDecision` and to the
|
|
@@ -9559,6 +9780,11 @@ type AgentWidgetToolCall = {
|
|
|
9559
9780
|
startedAt?: number;
|
|
9560
9781
|
completedAt?: number;
|
|
9561
9782
|
durationMs?: number;
|
|
9783
|
+
/**
|
|
9784
|
+
* The call paused for an approval that was granted, and the agent re-ran it
|
|
9785
|
+
* as a new tool call: that later call carries the result, this one has none.
|
|
9786
|
+
*/
|
|
9787
|
+
superseded?: boolean;
|
|
9562
9788
|
};
|
|
9563
9789
|
/**
|
|
9564
9790
|
* Represents a tool approval request in the chat conversation.
|
|
@@ -9580,6 +9806,8 @@ type AgentWidgetApproval = {
|
|
|
9580
9806
|
*/
|
|
9581
9807
|
reason?: string;
|
|
9582
9808
|
parameters?: unknown;
|
|
9809
|
+
/** The paused tool call this approval gates (`approval_start.toolCallId`). */
|
|
9810
|
+
toolCallId?: string;
|
|
9583
9811
|
resolvedAt?: number;
|
|
9584
9812
|
};
|
|
9585
9813
|
type AgentWidgetMessageVariant = "assistant" | "reasoning" | "tool" | "approval";
|
|
@@ -9679,6 +9907,13 @@ type AgentWidgetMessage = {
|
|
|
9679
9907
|
* }
|
|
9680
9908
|
*/
|
|
9681
9909
|
voiceProcessing?: boolean;
|
|
9910
|
+
/**
|
|
9911
|
+
* Display-only caption of full-duplex voice speech, such as the voice
|
|
9912
|
+
* model's own words, or something the visitor said that wasn't submitted
|
|
9913
|
+
* as a chat turn. It renders in the transcript but is never sent to the
|
|
9914
|
+
* model as conversation.
|
|
9915
|
+
*/
|
|
9916
|
+
voiceCaption?: boolean;
|
|
9682
9917
|
/**
|
|
9683
9918
|
* Raw structured payload for this message (e.g., JSON action response).
|
|
9684
9919
|
* Populated automatically when structured parsers run.
|