@runtypelabs/persona 4.24.0 → 4.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/animations/glyph-cycle.d.cts +1 -1
- package/dist/animations/glyph-cycle.d.ts +1 -1
- package/dist/animations/{types-Db61aI9J.d.cts → types-BwKTZhg9.d.cts} +14 -0
- package/dist/animations/{types-Db61aI9J.d.ts → types-BwKTZhg9.d.ts} +14 -0
- package/dist/animations/wipe.d.cts +1 -1
- package/dist/animations/wipe.d.ts +1 -1
- package/dist/animations-extra.d.cts +14 -0
- package/dist/animations-extra.d.ts +14 -0
- package/dist/approval-ui-QAKWNFNM.js +1 -0
- package/dist/approval-ui.cjs +2 -2
- package/dist/approval-ui.d.cts +243 -8
- package/dist/approval-ui.d.ts +243 -8
- package/dist/approval-ui.js +2 -2
- package/dist/{artifacts-ui-IGB5YKLO.js → artifacts-ui-NJGMM27I.js} +1 -1
- package/dist/artifacts-ui.cjs +4 -4
- package/dist/artifacts-ui.d.cts +243 -8
- package/dist/artifacts-ui.d.ts +243 -8
- package/dist/artifacts-ui.js +5 -5
- package/dist/chunk-2ANTRKTV.js +1 -0
- package/dist/chunk-4HPTPUC6.js +1 -0
- package/dist/chunk-6V3MNEKB.js +1 -0
- package/dist/chunk-HGUCGM3K.js +9 -0
- package/dist/chunk-IO5VVUKP.js +3 -0
- package/dist/chunk-J4J3Y2P2.js +1 -0
- package/dist/chunk-K6HKIP5K.js +1 -0
- package/dist/chunk-K7NA7G7J.js +1 -0
- package/dist/chunk-MXUTS3LB.js +1 -0
- package/dist/chunk-PDPVORUF.js +1 -0
- package/dist/chunk-QPPJUSZH.js +1 -0
- package/dist/chunk-QRKGQPU7.js +1 -0
- package/dist/chunk-S6PBWE75.js +1 -0
- package/dist/chunk-T7QM7YMF.js +1 -0
- package/dist/chunk-TR2H475E.js +1 -0
- package/dist/chunk-UPO4GUFC.js +1 -0
- package/dist/chunk-ZC66NAIR.js +1 -0
- package/dist/client-history-YPEG45Q4.js +1 -0
- package/dist/client-history.js +1 -0
- package/dist/client-stream-42FOI3EC.js +4 -0
- package/dist/client-stream.js +5 -0
- package/dist/codegen.cjs +1 -1
- package/dist/codegen.js +1 -1
- package/dist/context-mentions-E7HJYWO6.js +161 -0
- package/dist/{context-mentions-inline-I2RDEBAK.js → context-mentions-inline-QVESLW55.js} +1 -1
- package/dist/context-mentions.cjs +7 -7
- package/dist/context-mentions.d.cts +243 -8
- package/dist/context-mentions.d.ts +243 -8
- package/dist/context-mentions.js +8 -8
- package/dist/event-stream-view-FOIJHFQO.js +1 -0
- package/dist/event-stream-view.cjs +1 -1
- package/dist/event-stream-view.d.cts +246 -31
- package/dist/event-stream-view.d.ts +246 -31
- package/dist/event-stream-view.js +1 -1
- package/dist/forms-ui.d.cts +295 -667
- package/dist/forms-ui.d.ts +295 -667
- package/dist/history-shell-QDQ44PSE.js +3 -0
- package/dist/history-shell.js +3 -0
- package/dist/{history-view-BD6OXLNA.js → history-view-NCFBI5GR.js} +2 -2
- package/dist/history-view.cjs +2 -2
- package/dist/history-view.d.cts +35 -1
- package/dist/history-view.d.ts +35 -1
- package/dist/history-view.js +2 -2
- package/dist/index.cjs +80 -74
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +301 -738
- package/dist/index.d.ts +301 -738
- package/dist/index.global.js +3 -215
- package/dist/index.global.js.map +1 -1
- package/dist/index.js +81 -75
- package/dist/index.js.map +1 -1
- package/dist/install.global.js +1 -1
- package/dist/install.global.js.map +1 -1
- package/dist/launcher.global.js +3 -67
- package/dist/launcher.global.js.map +1 -1
- package/dist/markdown-parsers.js +22 -22
- package/dist/runtype-tts-entry-NFBVWR7N.js +1 -0
- package/dist/runtype-tts.js +1 -1
- package/dist/session-actions-ISSKSI66.js +1 -0
- package/dist/session-actions.js +1 -0
- package/dist/session-reconnect.d.cts +243 -8
- package/dist/session-reconnect.d.ts +243 -8
- package/dist/smart-dom-reader.cjs +13 -13
- package/dist/smart-dom-reader.d.cts +243 -8
- package/dist/smart-dom-reader.d.ts +243 -8
- package/dist/smart-dom-reader.js +11 -11
- package/dist/theme-editor-preview.cjs +116 -98
- package/dist/theme-editor-preview.d.cts +245 -8
- package/dist/theme-editor-preview.d.ts +245 -8
- package/dist/theme-editor-preview.js +49 -54
- package/dist/theme-editor.cjs +2 -2
- package/dist/theme-editor.d.cts +244 -23
- package/dist/theme-editor.d.ts +244 -23
- package/dist/theme-editor.js +2 -2
- package/dist/theme-reference.d.cts +14 -0
- package/dist/theme-reference.d.ts +14 -0
- package/dist/ui-extras-entry-WAYFKVNW.js +3 -0
- package/dist/ui-extras.js +72 -0
- package/dist/voice-runtime-UOQOFAFY.js +17 -0
- package/dist/voice-runtime.cjs +17 -1
- package/dist/voice-runtime.d.cts +11643 -192
- package/dist/voice-runtime.d.ts +11643 -192
- package/dist/voice-runtime.js +17 -1
- package/dist/voice-worklet-player.cjs +19 -4
- package/dist/voice-worklet-player.d.cts +9 -10
- package/dist/voice-worklet-player.d.ts +9 -10
- package/dist/voice-worklet-player.js +19 -4
- package/dist/webmcp-polyfill.js +1 -1
- package/dist/webmcp-runtime-V6ZWW3PI.js +6 -0
- package/dist/webmcp-runtime.cjs +2 -2
- package/dist/webmcp-runtime.d.cts +1 -33
- package/dist/webmcp-runtime.d.ts +1 -33
- package/dist/webmcp-runtime.js +2 -2
- package/dist/widget.css +1 -1
- package/package.json +11 -5
- package/src/approval-paused-tool-settle.test.ts +221 -0
- package/src/client-approve.test.ts +311 -0
- package/src/client-history-eager.ts +9 -0
- package/src/client-history-loader.ts +23 -0
- package/src/client-history.ts +579 -0
- package/src/client-resume.ts +177 -0
- package/src/client-stream-eager.ts +12 -0
- package/src/client-stream-loader.ts +25 -0
- package/src/client-stream.ts +1880 -0
- package/src/client.test.ts +31 -1
- package/src/client.ts +293 -2607
- package/src/components/approval-actions.test.ts +27 -0
- package/src/components/approval-actions.ts +27 -3
- package/src/components/message-bubble.test.ts +19 -0
- package/src/components/message-bubble.ts +2 -0
- package/src/components/registry.ts +6 -6
- package/src/components/tool-bubble.test.ts +9 -0
- package/src/components/tool-bubble.ts +1 -0
- package/src/history-shell-bundle.test.ts +54 -0
- package/src/history-shell-eager.ts +12 -0
- package/src/history-shell-loader.ts +27 -0
- package/src/history-shell.ts +2362 -0
- package/src/history-view-loader.ts +2 -0
- package/src/history-view.ts +3 -0
- package/src/index-core.ts +8 -0
- package/src/index-global.ts +90 -248
- package/src/index.ts +13 -0
- package/src/install.ts +6 -4
- package/src/internal/runtype-history-provider.test.ts +1 -0
- package/src/internal/runtype-history-provider.ts +28 -13
- package/src/plugins/registry.ts +8 -8
- package/src/session-actions-chunk.test.ts +55 -0
- package/src/session-actions-eager.ts +11 -0
- package/src/session-actions-loader.ts +37 -0
- package/src/session-actions.ts +909 -0
- package/src/session-history.test.ts +3 -1
- package/src/session.ts +875 -1530
- package/src/session.voice.test.ts +1241 -5
- package/src/styles/widget.css +58 -0
- package/src/theme-editor/state.ts +91 -91
- package/src/types.ts +269 -5
- package/src/ui-ask-user-sheet.ts +382 -0
- package/src/ui-extras-eager.ts +10 -0
- package/src/ui-extras-entry.ts +9 -0
- package/src/ui-extras-loader.ts +26 -0
- package/src/ui.artifact-rail-resize.test.ts +2 -1
- package/src/ui.ask-user-chunk.test.ts +93 -0
- package/src/ui.history-chunk.test.ts +2 -1
- package/src/ui.history-config-provider.test.ts +2 -1
- package/src/ui.history-confirm-fallback.test.ts +142 -0
- package/src/ui.history-controller.test.ts +2 -1
- package/src/ui.history-lifecycle.test.ts +2 -1
- package/src/ui.history-render-hooks.test.ts +2 -1
- package/src/ui.history-shell.test.ts +2 -1
- package/src/ui.rail-overlay.test.ts +2 -1
- package/src/ui.rail-resize.test.ts +2 -1
- package/src/ui.rail-trigger-pending.test.ts +2 -1
- package/src/ui.ts +399 -2770
- package/src/ui.voice-prewarm.test.ts +222 -0
- package/src/utils/attachment-manager.ts +67 -67
- package/src/utils/constants.ts +3 -0
- package/src/utils/context-mention-controller.ts +323 -323
- package/src/utils/context-mention-manager.ts +40 -40
- package/src/utils/context-mention-orchestrator.ts +28 -0
- package/src/utils/event-stream-buffer.ts +50 -50
- package/src/utils/event-stream-store.ts +43 -43
- package/src/utils/throughput-tracker.ts +34 -34
- package/src/utils/virtual-scroller.ts +67 -67
- package/src/utils/visitor-store.test.ts +50 -0
- package/src/utils/visitor-store.ts +23 -5
- package/src/voice/__fixtures__/voice-wire/README.md +1 -0
- package/src/voice/__fixtures__/voice-wire/client/cancel.json +3 -0
- package/src/voice/__fixtures__/voice-wire/client/context.json +4 -0
- package/src/voice/__fixtures__/voice-wire/client/delegation_result.json +6 -0
- package/src/voice/__fixtures__/voice-wire/client/delegation_update.json +6 -0
- package/src/voice/__fixtures__/voice-wire/client/ping.json +3 -0
- package/src/voice/__fixtures__/voice-wire/client/playback_progress.json +5 -0
- package/src/voice/__fixtures__/voice-wire/client/start.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/artifact.json +11 -0
- package/src/voice/__fixtures__/voice-wire/server/attached.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/audio_clear.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/audio_end.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_cancelled.json +5 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_completed.json +8 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_started.json +15 -0
- package/src/voice/__fixtures__/voice-wire/server/error.json +6 -0
- package/src/voice/__fixtures__/voice-wire/server/metrics.json +10 -0
- package/src/voice/__fixtures__/voice-wire/server/pong.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/session_config.json +16 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_final.json +7 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_interim.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_partial.json +6 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_update.json +10 -0
- package/src/voice/__fixtures__/voice-wire/server/warning.json +5 -0
- package/src/voice/audio-playback-manager.test.ts +57 -0
- package/src/voice/audio-playback-manager.ts +120 -98
- package/src/voice/browser-voice-provider.ts +37 -37
- package/src/voice/fallback-speech-engine.ts +22 -16
- package/src/voice/keyed-voice-transcript.ts +314 -0
- package/src/voice/mic-state-styles.ts +182 -0
- package/src/voice/read-aloud-controller.ts +46 -44
- package/src/voice/runtype-speech-engine.ts +35 -33
- package/src/voice/runtype-voice-provider.ts +856 -164
- package/src/voice/session-voice-wiring.ts +266 -0
- package/src/voice/voice-delegation.test.ts +71 -0
- package/src/voice/voice-delegation.ts +364 -0
- package/src/voice/voice-turn-tracker.ts +6 -6
- package/src/voice/voice.test.ts +1250 -7
- package/src/voice/worklet-playback-engine.test.ts +81 -0
- package/src/voice/worklet-playback-engine.ts +21 -3
- package/src/voice-runtime-loader.ts +4 -1
- package/src/voice-runtime.ts +7 -0
- package/src/webmcp-runtime-entry.ts +41 -39
- package/dist/approval-ui-3NMYNXBD.js +0 -1
- package/dist/chunk-5EIIHQLQ.js +0 -1
- package/dist/chunk-7CO4T5KB.js +0 -1
- package/dist/chunk-EFJBUH5F.js +0 -1
- package/dist/chunk-R2TMIMDI.js +0 -3
- package/dist/chunk-UTVLOTWB.js +0 -9
- package/dist/context-mentions-45MODWOY.js +0 -161
- package/dist/event-stream-view-GL3ADRY2.js +0 -1
- package/dist/runtype-tts-entry-UJAEF7NZ.js +0 -1
- package/dist/voice-runtime-EFR3R5ES.js +0 -1
- package/dist/webmcp-runtime-JI7T4QYF.js +0 -6
package/dist/artifacts-ui.d.cts
CHANGED
|
@@ -6579,6 +6579,22 @@ type AgentWidgetStatusIndicatorConfig = {
|
|
|
6579
6579
|
/** Status text while a reconnect attempt is in flight. */
|
|
6580
6580
|
resumingText?: string;
|
|
6581
6581
|
};
|
|
6582
|
+
/** Built-in `voiceRecognition.prewarm` triggers. See {@link AgentWidgetVoiceRecognitionConfig.prewarm}. */
|
|
6583
|
+
type AgentWidgetVoicePrewarmTrigger = 'hover';
|
|
6584
|
+
/**
|
|
6585
|
+
* Custom `voiceRecognition.prewarm` trigger. Runs while the composer has a mic
|
|
6586
|
+
* button, and again when the button is rebuilt or `update()` swaps in a
|
|
6587
|
+
* different function. Call `warm()` whenever the visitor shows intent to talk
|
|
6588
|
+
* (it is safe to call repeatedly). Return a cleanup to run on replacement.
|
|
6589
|
+
*/
|
|
6590
|
+
type AgentWidgetVoicePrewarmHook = (context: {
|
|
6591
|
+
/** Fire-and-forget voice warm-up; latched and error-swallowing. */
|
|
6592
|
+
warm: () => void;
|
|
6593
|
+
/** The composer's current mic button. */
|
|
6594
|
+
micButton: HTMLButtonElement;
|
|
6595
|
+
/** The widget's mount element. */
|
|
6596
|
+
mount: HTMLElement;
|
|
6597
|
+
}) => void | (() => void);
|
|
6582
6598
|
type AgentWidgetVoiceRecognitionConfig = {
|
|
6583
6599
|
/**
|
|
6584
6600
|
* Enable the mic button and voice input. The mic still only renders when the
|
|
@@ -6644,6 +6660,29 @@ type AgentWidgetVoiceRecognitionConfig = {
|
|
|
6644
6660
|
* Fires once per turn after the reply completes.
|
|
6645
6661
|
*/
|
|
6646
6662
|
onMetrics?: (metrics: VoiceMetrics) => void;
|
|
6663
|
+
/**
|
|
6664
|
+
* Warm the voice path when the visitor shows intent to talk, before the mic
|
|
6665
|
+
* click, so the call starts faster.
|
|
6666
|
+
*
|
|
6667
|
+
* - `'hover'`: when the pointer enters the mic button, it receives focus, or
|
|
6668
|
+
* (on touch) the first pointer-down on it.
|
|
6669
|
+
* - `false`: never.
|
|
6670
|
+
* - A function: custom trigger. Called with `{ warm, micButton, mount }`
|
|
6671
|
+
* while the mic button exists (and again when the button is rebuilt or
|
|
6672
|
+
* `update()` swaps in a different function); call `warm()` whenever you
|
|
6673
|
+
* decide, e.g. when the panel opens. May return a cleanup that runs on
|
|
6674
|
+
* destroy, rebuild, or replacement. Replaces the built-in trigger.
|
|
6675
|
+
*
|
|
6676
|
+
* What `warm()` does depends on the provider: `runtype` fires its prewarm
|
|
6677
|
+
* (see `provider.runtype.prewarmMode`), a `custom` provider's optional
|
|
6678
|
+
* {@link VoiceProvider.prewarm} runs, and the browser (Web Speech) path
|
|
6679
|
+
* starts the client-token session init (as `sessionInit` does), since speech
|
|
6680
|
+
* recognition itself cannot start before the click. Warming is
|
|
6681
|
+
* fire-and-forget, latched, and never surfaces an error or UI state.
|
|
6682
|
+
*
|
|
6683
|
+
* @default 'hover' in client token mode, otherwise `false`
|
|
6684
|
+
*/
|
|
6685
|
+
prewarm?: AgentWidgetVoicePrewarmTrigger | AgentWidgetVoicePrewarmHook | false;
|
|
6647
6686
|
provider?: {
|
|
6648
6687
|
type: 'browser' | 'runtype' | 'custom';
|
|
6649
6688
|
browser?: {
|
|
@@ -6665,6 +6704,62 @@ type AgentWidgetVoiceRecognitionConfig = {
|
|
|
6665
6704
|
* `@runtypelabs/persona/voice-worklet-player` for a jitter-buffered engine.
|
|
6666
6705
|
*/
|
|
6667
6706
|
createPlaybackEngine?: () => VoicePlaybackEngine | Promise<VoicePlaybackEngine>;
|
|
6707
|
+
/**
|
|
6708
|
+
* How `prewarm()` warms the voice path (see `voiceRecognition.prewarm`).
|
|
6709
|
+
*
|
|
6710
|
+
* - `'request'` (default): a fire-and-forget
|
|
6711
|
+
* `POST {host}/v1/client/agents/{agentId}/voice/prewarm`.
|
|
6712
|
+
* - `'attach'`: open the voice WebSocket ahead of the click without
|
|
6713
|
+
* starting a call; the click then starts the call on that socket. It can
|
|
6714
|
+
* create server work before the visitor clicks, so it is opt-in.
|
|
6715
|
+
* @default 'request'
|
|
6716
|
+
*/
|
|
6717
|
+
prewarmMode?: 'request' | 'attach';
|
|
6718
|
+
/**
|
|
6719
|
+
* `'attach'` mode: how long (ms) an attached socket waits for the click
|
|
6720
|
+
* before it closes. The server clamps it to 30000..600000.
|
|
6721
|
+
* @default 30000
|
|
6722
|
+
*/
|
|
6723
|
+
attachIdleMs?: number;
|
|
6724
|
+
/**
|
|
6725
|
+
* Full-duplex (speech-to-speech, e.g. GPT-Live) calls: run the turns the
|
|
6726
|
+
* voice model hands off to the agent through this widget's normal chat
|
|
6727
|
+
* pipeline. The spoken request becomes the user message, and the answer
|
|
6728
|
+
* renders as a regular assistant message (streaming, tool and approval
|
|
6729
|
+
* UI, WebMCP page tools, persistence) that the voice model then reads
|
|
6730
|
+
* aloud. Only takes effect when the server supports it (it confirms in
|
|
6731
|
+
* `session_config`); otherwise the agent turn runs on the server and its
|
|
6732
|
+
* spoken reply is transcribed, as before. Set `false` to always keep the
|
|
6733
|
+
* agent turn on the server.
|
|
6734
|
+
* @default true
|
|
6735
|
+
*/
|
|
6736
|
+
clientDelegation?: boolean;
|
|
6737
|
+
/**
|
|
6738
|
+
* Full-duplex calls: extra context for the voice model (for example the
|
|
6739
|
+
* page the visitor is on), sent after the chat history as of call start.
|
|
6740
|
+
* The frame goes out once, when the visitor finishes their first
|
|
6741
|
+
* utterance (or at the first agent hand-off). A function is
|
|
6742
|
+
* called at call start and may be async. This text is capped at 4000 characters and the whole frame
|
|
6743
|
+
* (history included) at 8000. Sent only to servers that accept call
|
|
6744
|
+
* context (they announce it in `session_config`).
|
|
6745
|
+
*/
|
|
6746
|
+
callContext?: string | (() => string | Promise<string>);
|
|
6747
|
+
/**
|
|
6748
|
+
* Full-duplex calls with client delegation: how long a tool approval
|
|
6749
|
+
* that a spoken request raised may wait before it is declined as
|
|
6750
|
+
* expired (and the voice model says so). Server-side approval gates
|
|
6751
|
+
* keep their own timeout too. Hanging up never declines it: the
|
|
6752
|
+
* approval card in the chat stays usable. Capped at 540000 (9 minutes),
|
|
6753
|
+
* under the server's 10-minute delegation deadline.
|
|
6754
|
+
* @default 300000
|
|
6755
|
+
*/
|
|
6756
|
+
approvalTimeoutMs?: number;
|
|
6757
|
+
/**
|
|
6758
|
+
* Speech-to-speech calls: the one-line notice shown in the composer
|
|
6759
|
+
* status area when the call starts. `false` hides it.
|
|
6760
|
+
* @default "You're talking to an AI assistant. Voice is processed by OpenAI."
|
|
6761
|
+
*/
|
|
6762
|
+
disclosureText?: string | false;
|
|
6668
6763
|
/** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
|
|
6669
6764
|
pauseDuration?: number;
|
|
6670
6765
|
/** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
|
|
@@ -6894,6 +6989,13 @@ interface VoicePlaybackEngine {
|
|
|
6894
6989
|
onFinished(callback: () => void): void;
|
|
6895
6990
|
/** Release all audio resources. */
|
|
6896
6991
|
destroy(): Promise<void> | void;
|
|
6992
|
+
/**
|
|
6993
|
+
* Optional. Speech-to-speech calls stream reply audio with no end-of-stream
|
|
6994
|
+
* marker, so an engine that holds audio below a prebuffer waterline must, while
|
|
6995
|
+
* continuous, release that held tail on its own (e.g. after a short input gap)
|
|
6996
|
+
* instead of waiting for {@link markStreamEnd}. Engines without a prebuffer can omit it.
|
|
6997
|
+
*/
|
|
6998
|
+
setContinuousMode?(enabled: boolean): void;
|
|
6897
6999
|
}
|
|
6898
7000
|
/**
|
|
6899
7001
|
* A {@link VoicePlaybackEngine} that also supports pause/resume. Returned by
|
|
@@ -6937,14 +7039,23 @@ interface VoiceProvider {
|
|
|
6937
7039
|
* Register a callback for incremental transcript updates during a voice turn.
|
|
6938
7040
|
* `isFinal=false` is a live interim update (user partials, or assistant deltas
|
|
6939
7041
|
* on providers that stream them); `isFinal=true` finalizes that role's text.
|
|
6940
|
-
* On the realtime `runtype` path, interim updates fire for the `user`
|
|
6941
|
-
* the `assistant` arrives as a single final
|
|
6942
|
-
*
|
|
6943
|
-
*
|
|
6944
|
-
|
|
6945
|
-
|
|
6946
|
-
|
|
6947
|
-
|
|
7042
|
+
* On the classic realtime `runtype` path, interim updates fire for the `user`
|
|
7043
|
+
* only and the `assistant` arrives as a single final; without a turnId the
|
|
7044
|
+
* session assumes strictly alternating turns, and providers must discard
|
|
7045
|
+
* cancelled output before emitting the next user final.
|
|
7046
|
+
*
|
|
7047
|
+
* When `metadata.turnId` is present (full-duplex / speech-to-speech, e.g.
|
|
7048
|
+
* GPT-Live), the session reconciles by `(turnId, role)`: each pair owns one
|
|
7049
|
+
* bubble that later updates replace in place, turns may overlap or arrive in
|
|
7050
|
+
* any order, and no empty assistant placeholder is injected. User and
|
|
7051
|
+
* assistant may share a turnId (a late user bubble then renders above its
|
|
7052
|
+
* reply) or use distinct ones. `metadata.startMs` (call-relative ms when the
|
|
7053
|
+
* utterance began) orders a new bubble among the call's other keyed bubbles;
|
|
7054
|
+
* without it bubbles order by first arrival. After a stop, keyed providers
|
|
7055
|
+
* must drop the cancelled reply's output until the server acknowledges the
|
|
7056
|
+
* cancel: the session renders every reply it receives.
|
|
7057
|
+
*/
|
|
7058
|
+
onTranscript?(callback: (role: 'user' | 'assistant', text: string, isFinal: boolean, metadata?: VoiceTranscriptMetadata) => void): void;
|
|
6948
7059
|
/** Register a callback for per-turn latency metrics (realtime path). */
|
|
6949
7060
|
onMetrics?(callback: (metrics: VoiceMetrics) => void): void;
|
|
6950
7061
|
/**
|
|
@@ -6965,6 +7076,116 @@ interface VoiceProvider {
|
|
|
6965
7076
|
deactivateBargeIn?(): Promise<void>;
|
|
6966
7077
|
/** Stop playback / cancel in-flight request without starting recording */
|
|
6967
7078
|
stopPlayback?(): void;
|
|
7079
|
+
/**
|
|
7080
|
+
* Warm the provider ahead of `startListening()` (the widget calls it on mic
|
|
7081
|
+
* intent, see `voiceRecognition.prewarm`). Must be fire-and-forget: never
|
|
7082
|
+
* throw, emit errors or status, or acquire the microphone. May be called
|
|
7083
|
+
* repeatedly; throttle as needed.
|
|
7084
|
+
*/
|
|
7085
|
+
prewarm?(): void;
|
|
7086
|
+
/**
|
|
7087
|
+
* Called by the session once after construction with the chat-side surface
|
|
7088
|
+
* a full-duplex provider uses to hand agent turns to the chat pipeline
|
|
7089
|
+
* ({@link VoiceSessionBridge}). Providers without client delegation omit it.
|
|
7090
|
+
*/
|
|
7091
|
+
setSessionBridge?(bridge: VoiceSessionBridge): void;
|
|
7092
|
+
/** The AI-disclosure notice for the live call, if one should show (speech-to-speech). */
|
|
7093
|
+
getDisclosure?(): string | null;
|
|
7094
|
+
}
|
|
7095
|
+
/** Metadata on a {@link VoiceProvider.onTranscript} update. */
|
|
7096
|
+
type VoiceTranscriptMetadata = {
|
|
7097
|
+
/** Groups updates that belong to one utterance bubble (full duplex). */
|
|
7098
|
+
turnId?: string;
|
|
7099
|
+
/** Call-relative ms when the utterance began; orders late bubbles. */
|
|
7100
|
+
startMs?: number;
|
|
7101
|
+
/** Call-relative ms of the latest audio this update covers. */
|
|
7102
|
+
endMs?: number;
|
|
7103
|
+
/**
|
|
7104
|
+
* The bubble only captions speech (see `AgentWidgetMessage.voiceCaption`).
|
|
7105
|
+
* The chat pipeline owns the conversation, as with client delegation.
|
|
7106
|
+
*/
|
|
7107
|
+
caption?: boolean;
|
|
7108
|
+
};
|
|
7109
|
+
/** An agent turn a full-duplex voice model handed to the widget. */
|
|
7110
|
+
type VoiceDelegationRequest = {
|
|
7111
|
+
/** The voice model's delegation id (`delegation_started.delegationId`). */
|
|
7112
|
+
delegationId: string;
|
|
7113
|
+
/** What the visitor asked, as the voice model heard it. */
|
|
7114
|
+
userText: string;
|
|
7115
|
+
/**
|
|
7116
|
+
* The user transcript utterances `userText` joins, oldest first (the last
|
|
7117
|
+
* is the one it came from). May be empty.
|
|
7118
|
+
*/
|
|
7119
|
+
userUtteranceIds: string[];
|
|
7120
|
+
/** The voice model's view of the conversation it delegated from. */
|
|
7121
|
+
messages: Array<{
|
|
7122
|
+
role: string;
|
|
7123
|
+
content: unknown;
|
|
7124
|
+
}>;
|
|
7125
|
+
};
|
|
7126
|
+
/**
|
|
7127
|
+
* How a delegated turn ended, or that it waits on an approval (contract
|
|
7128
|
+
* Amendment 5). `pending_approval` is the only non-terminal status. Open:
|
|
7129
|
+
* a receiver treats an unknown terminal status as `failed`.
|
|
7130
|
+
* - `completed`: the work ran.
|
|
7131
|
+
* - `denied`: the visitor said no (Deny tap, or a spoken decline).
|
|
7132
|
+
* - `timeout`: the approval lapsed (`approvalTimeoutMs`, or the server's own timeout).
|
|
7133
|
+
* - `cancelled`: replaced by a newer request, or abandoned by the system.
|
|
7134
|
+
* - `failed`: an error.
|
|
7135
|
+
*/
|
|
7136
|
+
type VoiceDelegationStatus = 'completed' | 'pending_approval' | 'denied' | 'timeout' | 'cancelled' | 'failed' | (string & {});
|
|
7137
|
+
/** The terminal result of a turn that parked on approvals. */
|
|
7138
|
+
type VoiceDelegationFollowUp = {
|
|
7139
|
+
status: VoiceDelegationStatus;
|
|
7140
|
+
/** What to read aloud (Markdown allowed). May be empty: nothing more to say. */
|
|
7141
|
+
text: string;
|
|
7142
|
+
};
|
|
7143
|
+
/** The delegated turn's answer, sent back for the voice model to read aloud. */
|
|
7144
|
+
type VoiceDelegationResult = {
|
|
7145
|
+
/** `pending_approval` when `followUp` is set; otherwise terminal. */
|
|
7146
|
+
status: VoiceDelegationStatus;
|
|
7147
|
+
/** The assistant text (Markdown allowed). */
|
|
7148
|
+
text: string;
|
|
7149
|
+
/**
|
|
7150
|
+
* Set when the turn parked on approvals (`text` then asks for the decision).
|
|
7151
|
+
* Resolves with the terminal result once the visitor decides (or the
|
|
7152
|
+
* approval is replaced, declined by voice, or unanswered for
|
|
7153
|
+
* `approvalTimeoutMs`) and the resumed turn finishes; `null` only when
|
|
7154
|
+
* `signal` aborts first (hang-up, or the server cancelled the delegation).
|
|
7155
|
+
*/
|
|
7156
|
+
followUp?: (options: {
|
|
7157
|
+
signal: AbortSignal;
|
|
7158
|
+
approvalTimeoutMs?: number;
|
|
7159
|
+
/** `false`: run only the approval bookkeeping (expiry, supersede); nothing will be read back. @default true */
|
|
7160
|
+
readBack?: boolean;
|
|
7161
|
+
/** The resumed turn stopped on another approval: ask for it too (`text` is its approval script). */
|
|
7162
|
+
onUpdate?: (text: string) => void;
|
|
7163
|
+
}) => Promise<VoiceDelegationFollowUp | null>;
|
|
7164
|
+
};
|
|
7165
|
+
/**
|
|
7166
|
+
* Chat-side surface the session hands a full-duplex provider: the visible
|
|
7167
|
+
* history (for the call-start context) and a way to run a delegated turn
|
|
7168
|
+
* through the normal chat pipeline.
|
|
7169
|
+
*/
|
|
7170
|
+
interface VoiceSessionBridge {
|
|
7171
|
+
/** Visible user/assistant messages, oldest first. */
|
|
7172
|
+
getHistory(): Array<{
|
|
7173
|
+
role: 'user' | 'assistant';
|
|
7174
|
+
content: string;
|
|
7175
|
+
}>;
|
|
7176
|
+
/**
|
|
7177
|
+
* Submit the turn as the visitor's message (reusing its transcript bubble)
|
|
7178
|
+
* and resolve with the assistant's final answer once the turn settles.
|
|
7179
|
+
* Waits for any chat turn already in flight first. Never rejects.
|
|
7180
|
+
*/
|
|
7181
|
+
runDelegatedTurn(request: VoiceDelegationRequest): Promise<VoiceDelegationResult>;
|
|
7182
|
+
/**
|
|
7183
|
+
* The server dropped this delegation's result (cancelled it, or refused a
|
|
7184
|
+
* late result): its chat answer is no longer spoken, so browser TTS may read it.
|
|
7185
|
+
* `expired`: the server's deadline passed, so its pending approvals are
|
|
7186
|
+
* declined too, as the approval TTL would.
|
|
7187
|
+
*/
|
|
7188
|
+
dropDelegation?(delegationId: string, expired?: boolean): void;
|
|
6968
7189
|
}
|
|
6969
7190
|
/**
|
|
6970
7191
|
* Extra context for an approval decision, surfaced to `onDecision` and to the
|
|
@@ -9561,6 +9782,11 @@ type AgentWidgetToolCall = {
|
|
|
9561
9782
|
startedAt?: number;
|
|
9562
9783
|
completedAt?: number;
|
|
9563
9784
|
durationMs?: number;
|
|
9785
|
+
/**
|
|
9786
|
+
* The call paused for an approval that was granted, and the agent re-ran it
|
|
9787
|
+
* as a new tool call: that later call carries the result, this one has none.
|
|
9788
|
+
*/
|
|
9789
|
+
superseded?: boolean;
|
|
9564
9790
|
};
|
|
9565
9791
|
/**
|
|
9566
9792
|
* Represents a tool approval request in the chat conversation.
|
|
@@ -9582,6 +9808,8 @@ type AgentWidgetApproval = {
|
|
|
9582
9808
|
*/
|
|
9583
9809
|
reason?: string;
|
|
9584
9810
|
parameters?: unknown;
|
|
9811
|
+
/** The paused tool call this approval gates (`approval_start.toolCallId`). */
|
|
9812
|
+
toolCallId?: string;
|
|
9585
9813
|
resolvedAt?: number;
|
|
9586
9814
|
};
|
|
9587
9815
|
type AgentWidgetMessageVariant = "assistant" | "reasoning" | "tool" | "approval";
|
|
@@ -9681,6 +9909,13 @@ type AgentWidgetMessage = {
|
|
|
9681
9909
|
* }
|
|
9682
9910
|
*/
|
|
9683
9911
|
voiceProcessing?: boolean;
|
|
9912
|
+
/**
|
|
9913
|
+
* Display-only caption of full-duplex voice speech, such as the voice
|
|
9914
|
+
* model's own words, or something the visitor said that wasn't submitted
|
|
9915
|
+
* as a chat turn. It renders in the transcript but is never sent to the
|
|
9916
|
+
* model as conversation.
|
|
9917
|
+
*/
|
|
9918
|
+
voiceCaption?: boolean;
|
|
9684
9919
|
/**
|
|
9685
9920
|
* Raw structured payload for this message (e.g., JSON action response).
|
|
9686
9921
|
* Populated automatically when structured parsers run.
|
package/dist/artifacts-ui.d.ts
CHANGED
|
@@ -6579,6 +6579,22 @@ type AgentWidgetStatusIndicatorConfig = {
|
|
|
6579
6579
|
/** Status text while a reconnect attempt is in flight. */
|
|
6580
6580
|
resumingText?: string;
|
|
6581
6581
|
};
|
|
6582
|
+
/** Built-in `voiceRecognition.prewarm` triggers. See {@link AgentWidgetVoiceRecognitionConfig.prewarm}. */
|
|
6583
|
+
type AgentWidgetVoicePrewarmTrigger = 'hover';
|
|
6584
|
+
/**
|
|
6585
|
+
* Custom `voiceRecognition.prewarm` trigger. Runs while the composer has a mic
|
|
6586
|
+
* button, and again when the button is rebuilt or `update()` swaps in a
|
|
6587
|
+
* different function. Call `warm()` whenever the visitor shows intent to talk
|
|
6588
|
+
* (it is safe to call repeatedly). Return a cleanup to run on replacement.
|
|
6589
|
+
*/
|
|
6590
|
+
type AgentWidgetVoicePrewarmHook = (context: {
|
|
6591
|
+
/** Fire-and-forget voice warm-up; latched and error-swallowing. */
|
|
6592
|
+
warm: () => void;
|
|
6593
|
+
/** The composer's current mic button. */
|
|
6594
|
+
micButton: HTMLButtonElement;
|
|
6595
|
+
/** The widget's mount element. */
|
|
6596
|
+
mount: HTMLElement;
|
|
6597
|
+
}) => void | (() => void);
|
|
6582
6598
|
type AgentWidgetVoiceRecognitionConfig = {
|
|
6583
6599
|
/**
|
|
6584
6600
|
* Enable the mic button and voice input. The mic still only renders when the
|
|
@@ -6644,6 +6660,29 @@ type AgentWidgetVoiceRecognitionConfig = {
|
|
|
6644
6660
|
* Fires once per turn after the reply completes.
|
|
6645
6661
|
*/
|
|
6646
6662
|
onMetrics?: (metrics: VoiceMetrics) => void;
|
|
6663
|
+
/**
|
|
6664
|
+
* Warm the voice path when the visitor shows intent to talk, before the mic
|
|
6665
|
+
* click, so the call starts faster.
|
|
6666
|
+
*
|
|
6667
|
+
* - `'hover'`: when the pointer enters the mic button, it receives focus, or
|
|
6668
|
+
* (on touch) the first pointer-down on it.
|
|
6669
|
+
* - `false`: never.
|
|
6670
|
+
* - A function: custom trigger. Called with `{ warm, micButton, mount }`
|
|
6671
|
+
* while the mic button exists (and again when the button is rebuilt or
|
|
6672
|
+
* `update()` swaps in a different function); call `warm()` whenever you
|
|
6673
|
+
* decide, e.g. when the panel opens. May return a cleanup that runs on
|
|
6674
|
+
* destroy, rebuild, or replacement. Replaces the built-in trigger.
|
|
6675
|
+
*
|
|
6676
|
+
* What `warm()` does depends on the provider: `runtype` fires its prewarm
|
|
6677
|
+
* (see `provider.runtype.prewarmMode`), a `custom` provider's optional
|
|
6678
|
+
* {@link VoiceProvider.prewarm} runs, and the browser (Web Speech) path
|
|
6679
|
+
* starts the client-token session init (as `sessionInit` does), since speech
|
|
6680
|
+
* recognition itself cannot start before the click. Warming is
|
|
6681
|
+
* fire-and-forget, latched, and never surfaces an error or UI state.
|
|
6682
|
+
*
|
|
6683
|
+
* @default 'hover' in client token mode, otherwise `false`
|
|
6684
|
+
*/
|
|
6685
|
+
prewarm?: AgentWidgetVoicePrewarmTrigger | AgentWidgetVoicePrewarmHook | false;
|
|
6647
6686
|
provider?: {
|
|
6648
6687
|
type: 'browser' | 'runtype' | 'custom';
|
|
6649
6688
|
browser?: {
|
|
@@ -6665,6 +6704,62 @@ type AgentWidgetVoiceRecognitionConfig = {
|
|
|
6665
6704
|
* `@runtypelabs/persona/voice-worklet-player` for a jitter-buffered engine.
|
|
6666
6705
|
*/
|
|
6667
6706
|
createPlaybackEngine?: () => VoicePlaybackEngine | Promise<VoicePlaybackEngine>;
|
|
6707
|
+
/**
|
|
6708
|
+
* How `prewarm()` warms the voice path (see `voiceRecognition.prewarm`).
|
|
6709
|
+
*
|
|
6710
|
+
* - `'request'` (default): a fire-and-forget
|
|
6711
|
+
* `POST {host}/v1/client/agents/{agentId}/voice/prewarm`.
|
|
6712
|
+
* - `'attach'`: open the voice WebSocket ahead of the click without
|
|
6713
|
+
* starting a call; the click then starts the call on that socket. It can
|
|
6714
|
+
* create server work before the visitor clicks, so it is opt-in.
|
|
6715
|
+
* @default 'request'
|
|
6716
|
+
*/
|
|
6717
|
+
prewarmMode?: 'request' | 'attach';
|
|
6718
|
+
/**
|
|
6719
|
+
* `'attach'` mode: how long (ms) an attached socket waits for the click
|
|
6720
|
+
* before it closes. The server clamps it to 30000..600000.
|
|
6721
|
+
* @default 30000
|
|
6722
|
+
*/
|
|
6723
|
+
attachIdleMs?: number;
|
|
6724
|
+
/**
|
|
6725
|
+
* Full-duplex (speech-to-speech, e.g. GPT-Live) calls: run the turns the
|
|
6726
|
+
* voice model hands off to the agent through this widget's normal chat
|
|
6727
|
+
* pipeline. The spoken request becomes the user message, and the answer
|
|
6728
|
+
* renders as a regular assistant message (streaming, tool and approval
|
|
6729
|
+
* UI, WebMCP page tools, persistence) that the voice model then reads
|
|
6730
|
+
* aloud. Only takes effect when the server supports it (it confirms in
|
|
6731
|
+
* `session_config`); otherwise the agent turn runs on the server and its
|
|
6732
|
+
* spoken reply is transcribed, as before. Set `false` to always keep the
|
|
6733
|
+
* agent turn on the server.
|
|
6734
|
+
* @default true
|
|
6735
|
+
*/
|
|
6736
|
+
clientDelegation?: boolean;
|
|
6737
|
+
/**
|
|
6738
|
+
* Full-duplex calls: extra context for the voice model (for example the
|
|
6739
|
+
* page the visitor is on), sent after the chat history as of call start.
|
|
6740
|
+
* The frame goes out once, when the visitor finishes their first
|
|
6741
|
+
* utterance (or at the first agent hand-off). A function is
|
|
6742
|
+
* called at call start and may be async. This text is capped at 4000 characters and the whole frame
|
|
6743
|
+
* (history included) at 8000. Sent only to servers that accept call
|
|
6744
|
+
* context (they announce it in `session_config`).
|
|
6745
|
+
*/
|
|
6746
|
+
callContext?: string | (() => string | Promise<string>);
|
|
6747
|
+
/**
|
|
6748
|
+
* Full-duplex calls with client delegation: how long a tool approval
|
|
6749
|
+
* that a spoken request raised may wait before it is declined as
|
|
6750
|
+
* expired (and the voice model says so). Server-side approval gates
|
|
6751
|
+
* keep their own timeout too. Hanging up never declines it: the
|
|
6752
|
+
* approval card in the chat stays usable. Capped at 540000 (9 minutes),
|
|
6753
|
+
* under the server's 10-minute delegation deadline.
|
|
6754
|
+
* @default 300000
|
|
6755
|
+
*/
|
|
6756
|
+
approvalTimeoutMs?: number;
|
|
6757
|
+
/**
|
|
6758
|
+
* Speech-to-speech calls: the one-line notice shown in the composer
|
|
6759
|
+
* status area when the call starts. `false` hides it.
|
|
6760
|
+
* @default "You're talking to an AI assistant. Voice is processed by OpenAI."
|
|
6761
|
+
*/
|
|
6762
|
+
disclosureText?: string | false;
|
|
6668
6763
|
/** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
|
|
6669
6764
|
pauseDuration?: number;
|
|
6670
6765
|
/** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
|
|
@@ -6894,6 +6989,13 @@ interface VoicePlaybackEngine {
|
|
|
6894
6989
|
onFinished(callback: () => void): void;
|
|
6895
6990
|
/** Release all audio resources. */
|
|
6896
6991
|
destroy(): Promise<void> | void;
|
|
6992
|
+
/**
|
|
6993
|
+
* Optional. Speech-to-speech calls stream reply audio with no end-of-stream
|
|
6994
|
+
* marker, so an engine that holds audio below a prebuffer waterline must, while
|
|
6995
|
+
* continuous, release that held tail on its own (e.g. after a short input gap)
|
|
6996
|
+
* instead of waiting for {@link markStreamEnd}. Engines without a prebuffer can omit it.
|
|
6997
|
+
*/
|
|
6998
|
+
setContinuousMode?(enabled: boolean): void;
|
|
6897
6999
|
}
|
|
6898
7000
|
/**
|
|
6899
7001
|
* A {@link VoicePlaybackEngine} that also supports pause/resume. Returned by
|
|
@@ -6937,14 +7039,23 @@ interface VoiceProvider {
|
|
|
6937
7039
|
* Register a callback for incremental transcript updates during a voice turn.
|
|
6938
7040
|
* `isFinal=false` is a live interim update (user partials, or assistant deltas
|
|
6939
7041
|
* on providers that stream them); `isFinal=true` finalizes that role's text.
|
|
6940
|
-
* On the realtime `runtype` path, interim updates fire for the `user`
|
|
6941
|
-
* the `assistant` arrives as a single final
|
|
6942
|
-
*
|
|
6943
|
-
*
|
|
6944
|
-
|
|
6945
|
-
|
|
6946
|
-
|
|
6947
|
-
|
|
7042
|
+
* On the classic realtime `runtype` path, interim updates fire for the `user`
|
|
7043
|
+
* only and the `assistant` arrives as a single final; without a turnId the
|
|
7044
|
+
* session assumes strictly alternating turns, and providers must discard
|
|
7045
|
+
* cancelled output before emitting the next user final.
|
|
7046
|
+
*
|
|
7047
|
+
* When `metadata.turnId` is present (full-duplex / speech-to-speech, e.g.
|
|
7048
|
+
* GPT-Live), the session reconciles by `(turnId, role)`: each pair owns one
|
|
7049
|
+
* bubble that later updates replace in place, turns may overlap or arrive in
|
|
7050
|
+
* any order, and no empty assistant placeholder is injected. User and
|
|
7051
|
+
* assistant may share a turnId (a late user bubble then renders above its
|
|
7052
|
+
* reply) or use distinct ones. `metadata.startMs` (call-relative ms when the
|
|
7053
|
+
* utterance began) orders a new bubble among the call's other keyed bubbles;
|
|
7054
|
+
* without it bubbles order by first arrival. After a stop, keyed providers
|
|
7055
|
+
* must drop the cancelled reply's output until the server acknowledges the
|
|
7056
|
+
* cancel: the session renders every reply it receives.
|
|
7057
|
+
*/
|
|
7058
|
+
onTranscript?(callback: (role: 'user' | 'assistant', text: string, isFinal: boolean, metadata?: VoiceTranscriptMetadata) => void): void;
|
|
6948
7059
|
/** Register a callback for per-turn latency metrics (realtime path). */
|
|
6949
7060
|
onMetrics?(callback: (metrics: VoiceMetrics) => void): void;
|
|
6950
7061
|
/**
|
|
@@ -6965,6 +7076,116 @@ interface VoiceProvider {
|
|
|
6965
7076
|
deactivateBargeIn?(): Promise<void>;
|
|
6966
7077
|
/** Stop playback / cancel in-flight request without starting recording */
|
|
6967
7078
|
stopPlayback?(): void;
|
|
7079
|
+
/**
|
|
7080
|
+
* Warm the provider ahead of `startListening()` (the widget calls it on mic
|
|
7081
|
+
* intent, see `voiceRecognition.prewarm`). Must be fire-and-forget: never
|
|
7082
|
+
* throw, emit errors or status, or acquire the microphone. May be called
|
|
7083
|
+
* repeatedly; throttle as needed.
|
|
7084
|
+
*/
|
|
7085
|
+
prewarm?(): void;
|
|
7086
|
+
/**
|
|
7087
|
+
* Called by the session once after construction with the chat-side surface
|
|
7088
|
+
* a full-duplex provider uses to hand agent turns to the chat pipeline
|
|
7089
|
+
* ({@link VoiceSessionBridge}). Providers without client delegation omit it.
|
|
7090
|
+
*/
|
|
7091
|
+
setSessionBridge?(bridge: VoiceSessionBridge): void;
|
|
7092
|
+
/** The AI-disclosure notice for the live call, if one should show (speech-to-speech). */
|
|
7093
|
+
getDisclosure?(): string | null;
|
|
7094
|
+
}
|
|
7095
|
+
/** Metadata on a {@link VoiceProvider.onTranscript} update. */
|
|
7096
|
+
type VoiceTranscriptMetadata = {
|
|
7097
|
+
/** Groups updates that belong to one utterance bubble (full duplex). */
|
|
7098
|
+
turnId?: string;
|
|
7099
|
+
/** Call-relative ms when the utterance began; orders late bubbles. */
|
|
7100
|
+
startMs?: number;
|
|
7101
|
+
/** Call-relative ms of the latest audio this update covers. */
|
|
7102
|
+
endMs?: number;
|
|
7103
|
+
/**
|
|
7104
|
+
* The bubble only captions speech (see `AgentWidgetMessage.voiceCaption`).
|
|
7105
|
+
* The chat pipeline owns the conversation, as with client delegation.
|
|
7106
|
+
*/
|
|
7107
|
+
caption?: boolean;
|
|
7108
|
+
};
|
|
7109
|
+
/** An agent turn a full-duplex voice model handed to the widget. */
|
|
7110
|
+
type VoiceDelegationRequest = {
|
|
7111
|
+
/** The voice model's delegation id (`delegation_started.delegationId`). */
|
|
7112
|
+
delegationId: string;
|
|
7113
|
+
/** What the visitor asked, as the voice model heard it. */
|
|
7114
|
+
userText: string;
|
|
7115
|
+
/**
|
|
7116
|
+
* The user transcript utterances `userText` joins, oldest first (the last
|
|
7117
|
+
* is the one it came from). May be empty.
|
|
7118
|
+
*/
|
|
7119
|
+
userUtteranceIds: string[];
|
|
7120
|
+
/** The voice model's view of the conversation it delegated from. */
|
|
7121
|
+
messages: Array<{
|
|
7122
|
+
role: string;
|
|
7123
|
+
content: unknown;
|
|
7124
|
+
}>;
|
|
7125
|
+
};
|
|
7126
|
+
/**
|
|
7127
|
+
* How a delegated turn ended, or that it waits on an approval (contract
|
|
7128
|
+
* Amendment 5). `pending_approval` is the only non-terminal status. Open:
|
|
7129
|
+
* a receiver treats an unknown terminal status as `failed`.
|
|
7130
|
+
* - `completed`: the work ran.
|
|
7131
|
+
* - `denied`: the visitor said no (Deny tap, or a spoken decline).
|
|
7132
|
+
* - `timeout`: the approval lapsed (`approvalTimeoutMs`, or the server's own timeout).
|
|
7133
|
+
* - `cancelled`: replaced by a newer request, or abandoned by the system.
|
|
7134
|
+
* - `failed`: an error.
|
|
7135
|
+
*/
|
|
7136
|
+
type VoiceDelegationStatus = 'completed' | 'pending_approval' | 'denied' | 'timeout' | 'cancelled' | 'failed' | (string & {});
|
|
7137
|
+
/** The terminal result of a turn that parked on approvals. */
|
|
7138
|
+
type VoiceDelegationFollowUp = {
|
|
7139
|
+
status: VoiceDelegationStatus;
|
|
7140
|
+
/** What to read aloud (Markdown allowed). May be empty: nothing more to say. */
|
|
7141
|
+
text: string;
|
|
7142
|
+
};
|
|
7143
|
+
/** The delegated turn's answer, sent back for the voice model to read aloud. */
|
|
7144
|
+
type VoiceDelegationResult = {
|
|
7145
|
+
/** `pending_approval` when `followUp` is set; otherwise terminal. */
|
|
7146
|
+
status: VoiceDelegationStatus;
|
|
7147
|
+
/** The assistant text (Markdown allowed). */
|
|
7148
|
+
text: string;
|
|
7149
|
+
/**
|
|
7150
|
+
* Set when the turn parked on approvals (`text` then asks for the decision).
|
|
7151
|
+
* Resolves with the terminal result once the visitor decides (or the
|
|
7152
|
+
* approval is replaced, declined by voice, or unanswered for
|
|
7153
|
+
* `approvalTimeoutMs`) and the resumed turn finishes; `null` only when
|
|
7154
|
+
* `signal` aborts first (hang-up, or the server cancelled the delegation).
|
|
7155
|
+
*/
|
|
7156
|
+
followUp?: (options: {
|
|
7157
|
+
signal: AbortSignal;
|
|
7158
|
+
approvalTimeoutMs?: number;
|
|
7159
|
+
/** `false`: run only the approval bookkeeping (expiry, supersede); nothing will be read back. @default true */
|
|
7160
|
+
readBack?: boolean;
|
|
7161
|
+
/** The resumed turn stopped on another approval: ask for it too (`text` is its approval script). */
|
|
7162
|
+
onUpdate?: (text: string) => void;
|
|
7163
|
+
}) => Promise<VoiceDelegationFollowUp | null>;
|
|
7164
|
+
};
|
|
7165
|
+
/**
|
|
7166
|
+
* Chat-side surface the session hands a full-duplex provider: the visible
|
|
7167
|
+
* history (for the call-start context) and a way to run a delegated turn
|
|
7168
|
+
* through the normal chat pipeline.
|
|
7169
|
+
*/
|
|
7170
|
+
interface VoiceSessionBridge {
|
|
7171
|
+
/** Visible user/assistant messages, oldest first. */
|
|
7172
|
+
getHistory(): Array<{
|
|
7173
|
+
role: 'user' | 'assistant';
|
|
7174
|
+
content: string;
|
|
7175
|
+
}>;
|
|
7176
|
+
/**
|
|
7177
|
+
* Submit the turn as the visitor's message (reusing its transcript bubble)
|
|
7178
|
+
* and resolve with the assistant's final answer once the turn settles.
|
|
7179
|
+
* Waits for any chat turn already in flight first. Never rejects.
|
|
7180
|
+
*/
|
|
7181
|
+
runDelegatedTurn(request: VoiceDelegationRequest): Promise<VoiceDelegationResult>;
|
|
7182
|
+
/**
|
|
7183
|
+
* The server dropped this delegation's result (cancelled it, or refused a
|
|
7184
|
+
* late result): its chat answer is no longer spoken, so browser TTS may read it.
|
|
7185
|
+
* `expired`: the server's deadline passed, so its pending approvals are
|
|
7186
|
+
* declined too, as the approval TTL would.
|
|
7187
|
+
*/
|
|
7188
|
+
dropDelegation?(delegationId: string, expired?: boolean): void;
|
|
6968
7189
|
}
|
|
6969
7190
|
/**
|
|
6970
7191
|
* Extra context for an approval decision, surfaced to `onDecision` and to the
|
|
@@ -9561,6 +9782,11 @@ type AgentWidgetToolCall = {
|
|
|
9561
9782
|
startedAt?: number;
|
|
9562
9783
|
completedAt?: number;
|
|
9563
9784
|
durationMs?: number;
|
|
9785
|
+
/**
|
|
9786
|
+
* The call paused for an approval that was granted, and the agent re-ran it
|
|
9787
|
+
* as a new tool call: that later call carries the result, this one has none.
|
|
9788
|
+
*/
|
|
9789
|
+
superseded?: boolean;
|
|
9564
9790
|
};
|
|
9565
9791
|
/**
|
|
9566
9792
|
* Represents a tool approval request in the chat conversation.
|
|
@@ -9582,6 +9808,8 @@ type AgentWidgetApproval = {
|
|
|
9582
9808
|
*/
|
|
9583
9809
|
reason?: string;
|
|
9584
9810
|
parameters?: unknown;
|
|
9811
|
+
/** The paused tool call this approval gates (`approval_start.toolCallId`). */
|
|
9812
|
+
toolCallId?: string;
|
|
9585
9813
|
resolvedAt?: number;
|
|
9586
9814
|
};
|
|
9587
9815
|
type AgentWidgetMessageVariant = "assistant" | "reasoning" | "tool" | "approval";
|
|
@@ -9681,6 +9909,13 @@ type AgentWidgetMessage = {
|
|
|
9681
9909
|
* }
|
|
9682
9910
|
*/
|
|
9683
9911
|
voiceProcessing?: boolean;
|
|
9912
|
+
/**
|
|
9913
|
+
* Display-only caption of full-duplex voice speech, such as the voice
|
|
9914
|
+
* model's own words, or something the visitor said that wasn't submitted
|
|
9915
|
+
* as a chat turn. It renders in the transcript but is never sent to the
|
|
9916
|
+
* model as conversation.
|
|
9917
|
+
*/
|
|
9918
|
+
voiceCaption?: boolean;
|
|
9684
9919
|
/**
|
|
9685
9920
|
* Raw structured payload for this message (e.g., JSON action response).
|
|
9686
9921
|
* Populated automatically when structured parsers run.
|