@runtypelabs/persona 4.24.0 → 4.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/animations/glyph-cycle.d.cts +1 -1
- package/dist/animations/glyph-cycle.d.ts +1 -1
- package/dist/animations/{types-Db61aI9J.d.cts → types-BwKTZhg9.d.cts} +14 -0
- package/dist/animations/{types-Db61aI9J.d.ts → types-BwKTZhg9.d.ts} +14 -0
- package/dist/animations/wipe.d.cts +1 -1
- package/dist/animations/wipe.d.ts +1 -1
- package/dist/animations-extra.d.cts +14 -0
- package/dist/animations-extra.d.ts +14 -0
- package/dist/approval-ui-QAKWNFNM.js +1 -0
- package/dist/approval-ui.cjs +2 -2
- package/dist/approval-ui.d.cts +243 -8
- package/dist/approval-ui.d.ts +243 -8
- package/dist/approval-ui.js +2 -2
- package/dist/{artifacts-ui-IGB5YKLO.js → artifacts-ui-NJGMM27I.js} +1 -1
- package/dist/artifacts-ui.cjs +4 -4
- package/dist/artifacts-ui.d.cts +243 -8
- package/dist/artifacts-ui.d.ts +243 -8
- package/dist/artifacts-ui.js +5 -5
- package/dist/chunk-2ANTRKTV.js +1 -0
- package/dist/chunk-4HPTPUC6.js +1 -0
- package/dist/chunk-6V3MNEKB.js +1 -0
- package/dist/chunk-HGUCGM3K.js +9 -0
- package/dist/chunk-IO5VVUKP.js +3 -0
- package/dist/chunk-J4J3Y2P2.js +1 -0
- package/dist/chunk-K6HKIP5K.js +1 -0
- package/dist/chunk-K7NA7G7J.js +1 -0
- package/dist/chunk-MXUTS3LB.js +1 -0
- package/dist/chunk-PDPVORUF.js +1 -0
- package/dist/chunk-QPPJUSZH.js +1 -0
- package/dist/chunk-QRKGQPU7.js +1 -0
- package/dist/chunk-S6PBWE75.js +1 -0
- package/dist/chunk-T7QM7YMF.js +1 -0
- package/dist/chunk-TR2H475E.js +1 -0
- package/dist/chunk-UPO4GUFC.js +1 -0
- package/dist/chunk-ZC66NAIR.js +1 -0
- package/dist/client-history-YPEG45Q4.js +1 -0
- package/dist/client-history.js +1 -0
- package/dist/client-stream-42FOI3EC.js +4 -0
- package/dist/client-stream.js +5 -0
- package/dist/codegen.cjs +1 -1
- package/dist/codegen.js +1 -1
- package/dist/context-mentions-E7HJYWO6.js +161 -0
- package/dist/{context-mentions-inline-I2RDEBAK.js → context-mentions-inline-QVESLW55.js} +1 -1
- package/dist/context-mentions.cjs +7 -7
- package/dist/context-mentions.d.cts +243 -8
- package/dist/context-mentions.d.ts +243 -8
- package/dist/context-mentions.js +8 -8
- package/dist/event-stream-view-FOIJHFQO.js +1 -0
- package/dist/event-stream-view.cjs +1 -1
- package/dist/event-stream-view.d.cts +246 -31
- package/dist/event-stream-view.d.ts +246 -31
- package/dist/event-stream-view.js +1 -1
- package/dist/forms-ui.d.cts +295 -667
- package/dist/forms-ui.d.ts +295 -667
- package/dist/history-shell-QDQ44PSE.js +3 -0
- package/dist/history-shell.js +3 -0
- package/dist/{history-view-BD6OXLNA.js → history-view-NCFBI5GR.js} +2 -2
- package/dist/history-view.cjs +2 -2
- package/dist/history-view.d.cts +35 -1
- package/dist/history-view.d.ts +35 -1
- package/dist/history-view.js +2 -2
- package/dist/index.cjs +80 -74
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +301 -738
- package/dist/index.d.ts +301 -738
- package/dist/index.global.js +3 -215
- package/dist/index.global.js.map +1 -1
- package/dist/index.js +81 -75
- package/dist/index.js.map +1 -1
- package/dist/install.global.js +1 -1
- package/dist/install.global.js.map +1 -1
- package/dist/launcher.global.js +3 -67
- package/dist/launcher.global.js.map +1 -1
- package/dist/markdown-parsers.js +22 -22
- package/dist/runtype-tts-entry-NFBVWR7N.js +1 -0
- package/dist/runtype-tts.js +1 -1
- package/dist/session-actions-ISSKSI66.js +1 -0
- package/dist/session-actions.js +1 -0
- package/dist/session-reconnect.d.cts +243 -8
- package/dist/session-reconnect.d.ts +243 -8
- package/dist/smart-dom-reader.cjs +13 -13
- package/dist/smart-dom-reader.d.cts +243 -8
- package/dist/smart-dom-reader.d.ts +243 -8
- package/dist/smart-dom-reader.js +11 -11
- package/dist/theme-editor-preview.cjs +116 -98
- package/dist/theme-editor-preview.d.cts +245 -8
- package/dist/theme-editor-preview.d.ts +245 -8
- package/dist/theme-editor-preview.js +49 -54
- package/dist/theme-editor.cjs +2 -2
- package/dist/theme-editor.d.cts +244 -23
- package/dist/theme-editor.d.ts +244 -23
- package/dist/theme-editor.js +2 -2
- package/dist/theme-reference.d.cts +14 -0
- package/dist/theme-reference.d.ts +14 -0
- package/dist/ui-extras-entry-WAYFKVNW.js +3 -0
- package/dist/ui-extras.js +72 -0
- package/dist/voice-runtime-UOQOFAFY.js +17 -0
- package/dist/voice-runtime.cjs +17 -1
- package/dist/voice-runtime.d.cts +11643 -192
- package/dist/voice-runtime.d.ts +11643 -192
- package/dist/voice-runtime.js +17 -1
- package/dist/voice-worklet-player.cjs +19 -4
- package/dist/voice-worklet-player.d.cts +9 -10
- package/dist/voice-worklet-player.d.ts +9 -10
- package/dist/voice-worklet-player.js +19 -4
- package/dist/webmcp-polyfill.js +1 -1
- package/dist/webmcp-runtime-V6ZWW3PI.js +6 -0
- package/dist/webmcp-runtime.cjs +2 -2
- package/dist/webmcp-runtime.d.cts +1 -33
- package/dist/webmcp-runtime.d.ts +1 -33
- package/dist/webmcp-runtime.js +2 -2
- package/dist/widget.css +1 -1
- package/package.json +11 -5
- package/src/approval-paused-tool-settle.test.ts +221 -0
- package/src/client-approve.test.ts +311 -0
- package/src/client-history-eager.ts +9 -0
- package/src/client-history-loader.ts +23 -0
- package/src/client-history.ts +579 -0
- package/src/client-resume.ts +177 -0
- package/src/client-stream-eager.ts +12 -0
- package/src/client-stream-loader.ts +25 -0
- package/src/client-stream.ts +1880 -0
- package/src/client.test.ts +31 -1
- package/src/client.ts +293 -2607
- package/src/components/approval-actions.test.ts +27 -0
- package/src/components/approval-actions.ts +27 -3
- package/src/components/message-bubble.test.ts +19 -0
- package/src/components/message-bubble.ts +2 -0
- package/src/components/registry.ts +6 -6
- package/src/components/tool-bubble.test.ts +9 -0
- package/src/components/tool-bubble.ts +1 -0
- package/src/history-shell-bundle.test.ts +54 -0
- package/src/history-shell-eager.ts +12 -0
- package/src/history-shell-loader.ts +27 -0
- package/src/history-shell.ts +2362 -0
- package/src/history-view-loader.ts +2 -0
- package/src/history-view.ts +3 -0
- package/src/index-core.ts +8 -0
- package/src/index-global.ts +90 -248
- package/src/index.ts +13 -0
- package/src/install.ts +6 -4
- package/src/internal/runtype-history-provider.test.ts +1 -0
- package/src/internal/runtype-history-provider.ts +28 -13
- package/src/plugins/registry.ts +8 -8
- package/src/session-actions-chunk.test.ts +55 -0
- package/src/session-actions-eager.ts +11 -0
- package/src/session-actions-loader.ts +37 -0
- package/src/session-actions.ts +909 -0
- package/src/session-history.test.ts +3 -1
- package/src/session.ts +875 -1530
- package/src/session.voice.test.ts +1241 -5
- package/src/styles/widget.css +58 -0
- package/src/theme-editor/state.ts +91 -91
- package/src/types.ts +269 -5
- package/src/ui-ask-user-sheet.ts +382 -0
- package/src/ui-extras-eager.ts +10 -0
- package/src/ui-extras-entry.ts +9 -0
- package/src/ui-extras-loader.ts +26 -0
- package/src/ui.artifact-rail-resize.test.ts +2 -1
- package/src/ui.ask-user-chunk.test.ts +93 -0
- package/src/ui.history-chunk.test.ts +2 -1
- package/src/ui.history-config-provider.test.ts +2 -1
- package/src/ui.history-confirm-fallback.test.ts +142 -0
- package/src/ui.history-controller.test.ts +2 -1
- package/src/ui.history-lifecycle.test.ts +2 -1
- package/src/ui.history-render-hooks.test.ts +2 -1
- package/src/ui.history-shell.test.ts +2 -1
- package/src/ui.rail-overlay.test.ts +2 -1
- package/src/ui.rail-resize.test.ts +2 -1
- package/src/ui.rail-trigger-pending.test.ts +2 -1
- package/src/ui.ts +399 -2770
- package/src/ui.voice-prewarm.test.ts +222 -0
- package/src/utils/attachment-manager.ts +67 -67
- package/src/utils/constants.ts +3 -0
- package/src/utils/context-mention-controller.ts +323 -323
- package/src/utils/context-mention-manager.ts +40 -40
- package/src/utils/context-mention-orchestrator.ts +28 -0
- package/src/utils/event-stream-buffer.ts +50 -50
- package/src/utils/event-stream-store.ts +43 -43
- package/src/utils/throughput-tracker.ts +34 -34
- package/src/utils/virtual-scroller.ts +67 -67
- package/src/utils/visitor-store.test.ts +50 -0
- package/src/utils/visitor-store.ts +23 -5
- package/src/voice/__fixtures__/voice-wire/README.md +1 -0
- package/src/voice/__fixtures__/voice-wire/client/cancel.json +3 -0
- package/src/voice/__fixtures__/voice-wire/client/context.json +4 -0
- package/src/voice/__fixtures__/voice-wire/client/delegation_result.json +6 -0
- package/src/voice/__fixtures__/voice-wire/client/delegation_update.json +6 -0
- package/src/voice/__fixtures__/voice-wire/client/ping.json +3 -0
- package/src/voice/__fixtures__/voice-wire/client/playback_progress.json +5 -0
- package/src/voice/__fixtures__/voice-wire/client/start.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/artifact.json +11 -0
- package/src/voice/__fixtures__/voice-wire/server/attached.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/audio_clear.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/audio_end.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_cancelled.json +5 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_completed.json +8 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_started.json +15 -0
- package/src/voice/__fixtures__/voice-wire/server/error.json +6 -0
- package/src/voice/__fixtures__/voice-wire/server/metrics.json +10 -0
- package/src/voice/__fixtures__/voice-wire/server/pong.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/session_config.json +16 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_final.json +7 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_interim.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_partial.json +6 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_update.json +10 -0
- package/src/voice/__fixtures__/voice-wire/server/warning.json +5 -0
- package/src/voice/audio-playback-manager.test.ts +57 -0
- package/src/voice/audio-playback-manager.ts +120 -98
- package/src/voice/browser-voice-provider.ts +37 -37
- package/src/voice/fallback-speech-engine.ts +22 -16
- package/src/voice/keyed-voice-transcript.ts +314 -0
- package/src/voice/mic-state-styles.ts +182 -0
- package/src/voice/read-aloud-controller.ts +46 -44
- package/src/voice/runtype-speech-engine.ts +35 -33
- package/src/voice/runtype-voice-provider.ts +856 -164
- package/src/voice/session-voice-wiring.ts +266 -0
- package/src/voice/voice-delegation.test.ts +71 -0
- package/src/voice/voice-delegation.ts +364 -0
- package/src/voice/voice-turn-tracker.ts +6 -6
- package/src/voice/voice.test.ts +1250 -7
- package/src/voice/worklet-playback-engine.test.ts +81 -0
- package/src/voice/worklet-playback-engine.ts +21 -3
- package/src/voice-runtime-loader.ts +4 -1
- package/src/voice-runtime.ts +7 -0
- package/src/webmcp-runtime-entry.ts +41 -39
- package/dist/approval-ui-3NMYNXBD.js +0 -1
- package/dist/chunk-5EIIHQLQ.js +0 -1
- package/dist/chunk-7CO4T5KB.js +0 -1
- package/dist/chunk-EFJBUH5F.js +0 -1
- package/dist/chunk-R2TMIMDI.js +0 -3
- package/dist/chunk-UTVLOTWB.js +0 -9
- package/dist/context-mentions-45MODWOY.js +0 -161
- package/dist/event-stream-view-GL3ADRY2.js +0 -1
- package/dist/runtype-tts-entry-UJAEF7NZ.js +0 -1
- package/dist/voice-runtime-EFR3R5ES.js +0 -1
- package/dist/webmcp-runtime-JI7T4QYF.js +0 -6
|
@@ -8,6 +8,17 @@
|
|
|
8
8
|
// JSON control frames (transcript_interim / transcript_final /
|
|
9
9
|
// audio_end / metrics).
|
|
10
10
|
//
|
|
11
|
+
// Full duplex (speech-to-speech, e.g. GPT-Live): the client always declares
|
|
12
|
+
// `voiceCapabilities=full-duplex-v1`; engines that don't speak it ignore the
|
|
13
|
+
// param. A full-duplex session announces itself with
|
|
14
|
+
// `session_config{speechMode:'speech_to_speech'}` and then sends turn-keyed
|
|
15
|
+
// `transcript_update{role,text,turnId,final}` frames (later frames replace the
|
|
16
|
+
// text for the same (turnId, role); user and assistant turns can overlap),
|
|
17
|
+
// `delegation_started/completed` around agent tool turns, and `audio_clear` on
|
|
18
|
+
// barge-in. Reply audio has no per-turn end marker in that mode, so playback is
|
|
19
|
+
// treated as continuous: the "speaking" status is released by a timer derived
|
|
20
|
+
// from the queued audio duration rather than by the engine's end-of-stream.
|
|
21
|
+
//
|
|
11
22
|
// The server's STT owns turn-taking, so the client streams continuously and
|
|
12
23
|
// has no client-side VAD, barge-in monitoring, or batch upload. Auth rides the
|
|
13
24
|
// `Sec-WebSocket-Protocol` subprotocol (`['runtype.bearer', clientToken]`),
|
|
@@ -16,20 +27,68 @@
|
|
|
16
27
|
// A continuous always-hot mic is, in UX terms, a permanent barge-in session, so
|
|
17
28
|
// `getInterruptionMode()` reports the constant `'barge-in'` and the existing
|
|
18
29
|
// mic-button wiring (ui.ts) treats a click as "hang up at any state" unchanged.
|
|
30
|
+
//
|
|
31
|
+
// Prewarm (`prewarm()`, called on mic intent): by default a fire-and-forget
|
|
32
|
+
// `POST /v1/client/agents/:agentId/voice/prewarm`. With `prewarmMode: 'attach'`
|
|
33
|
+
// the WebSocket opens early with `clientCapabilities=attach` and the extra
|
|
34
|
+
// `runtype.attach` subprotocol. If the server negotiates `runtype.attach`, the
|
|
35
|
+
// socket idles (no call) until `startListening()` sends `{"type":"start"}`; if
|
|
36
|
+
// it negotiates plain `runtype.bearer`, it ignored attach and the call is
|
|
37
|
+
// already live, so `startListening()` adopts it as is.
|
|
38
|
+
//
|
|
39
|
+
// Wire vocabulary v1 (contract Amendment 5). Every socket sends
|
|
40
|
+
// `voiceProtocol=runtype-browser-v1`, `clientVersion=persona/<version>` and the
|
|
41
|
+
// snake_case `clientCapabilities` it supports. The server answers with
|
|
42
|
+
// `session_config.capabilities` (the negotiated intersection), the only gate
|
|
43
|
+
// for the client frames `context`, `delegation_update` and client delegation:
|
|
44
|
+
// a server without `capabilities` gets none of them. Receivers ignore unknown
|
|
45
|
+
// frames, fields and enum values. Non-fatal problems arrive as `warning`
|
|
46
|
+
// frames; an `error` always ends the call.
|
|
47
|
+
//
|
|
48
|
+
// Client delegation (full duplex, when the session supplied a bridge and
|
|
49
|
+
// `clientDelegation` isn't false, and `client_delegation` was negotiated): a
|
|
50
|
+
// `delegation_started{delegationId, input}` asks the widget to run the turn
|
|
51
|
+
// through its chat pipeline (one at a time). The provider answers with exactly
|
|
52
|
+
// one terminal `delegation_result{delegationId, status, text}`. A turn that
|
|
53
|
+
// parks on a tool approval first sends the non-terminal
|
|
54
|
+
// `delegation_update{delegationId, status:'pending_approval', text}`, then the
|
|
55
|
+
// terminal result once the visitor decides (or the approval lapses or is
|
|
56
|
+
// replaced). A second gated tool in the same turn sends another update.
|
|
57
|
+
// `delegation_cancelled{delegationId}` (or a `warning` refusing a frame for
|
|
58
|
+
// it) stops all frames for that id; the chat turn still renders, and since
|
|
59
|
+
// nobody speaks it now, browser TTS may read it. The voice model's spoken read-back after each
|
|
60
|
+
// `delegation_completed` of an answered delegation (the first new assistant
|
|
61
|
+
// utterance, unless the visitor speaks first) is folded: the chat already shows
|
|
62
|
+
// it. Transcripts in such a call are display-only captions, never sent to the
|
|
63
|
+
// agent as conversation; a user bubble becomes conversation only when it is
|
|
64
|
+
// submitted. With `context` negotiated, one `context{text}` frame (chat history
|
|
65
|
+
// as of call start plus the host's `callContext`) is held until the visitor's
|
|
66
|
+
// first final transcript or first delegation.
|
|
19
67
|
|
|
20
68
|
import type {
|
|
21
69
|
VoiceProvider,
|
|
22
70
|
VoiceResult,
|
|
23
71
|
VoiceStatus,
|
|
24
72
|
VoiceConfig,
|
|
73
|
+
VoiceDelegationRequest,
|
|
74
|
+
VoiceDelegationResult,
|
|
25
75
|
VoiceMetrics,
|
|
26
76
|
VoicePlaybackEngine,
|
|
77
|
+
VoiceSessionBridge,
|
|
78
|
+
VoiceTranscriptMetadata,
|
|
27
79
|
} from "../types";
|
|
28
80
|
import { AudioPlaybackManager } from "./audio-playback-manager";
|
|
81
|
+
import { VERSION } from "../version";
|
|
29
82
|
|
|
30
83
|
const CAPTURE_SAMPLE_RATE = 16000;
|
|
31
84
|
const PLAYBACK_SAMPLE_RATE = 24000;
|
|
32
85
|
const CAPTURE_BUFFER_SIZE = 4096;
|
|
86
|
+
/**
|
|
87
|
+
* Mic audio held while the call socket handshakes (a slow upgrade can take
|
|
88
|
+
* seconds, and the visitor is already talking): 8 s of 16-bit PCM, 256 KB,
|
|
89
|
+
* well under the server's startup queue. Past it the oldest frames go.
|
|
90
|
+
*/
|
|
91
|
+
const PRE_OPEN_AUDIO_MAX_BYTES = CAPTURE_SAMPLE_RATE * 2 * 8;
|
|
33
92
|
/**
|
|
34
93
|
* RMS-to-0..1 gain for the published capture level. Conversational speech sits
|
|
35
94
|
* around 0.05 to 0.3 RMS, so the raw value would never leave the bottom of the
|
|
@@ -37,6 +96,93 @@ const CAPTURE_BUFFER_SIZE = 4096;
|
|
|
37
96
|
*/
|
|
38
97
|
const LEVEL_RMS_SCALE = 4;
|
|
39
98
|
const RIFF_MAGIC = 0x52494646; // "RIFF"
|
|
99
|
+
/** `prewarm()` warms at most once per this window per provider instance. */
|
|
100
|
+
const PREWARM_THROTTLE_MS = 30_000;
|
|
101
|
+
/** How long `startListening()` waits on an attach still handshaking. */
|
|
102
|
+
const ATTACH_HANDSHAKE_TIMEOUT_MS = 5_000;
|
|
103
|
+
/** Attached-socket idle window: server default and clamp range. */
|
|
104
|
+
const ATTACH_IDLE_DEFAULT_MS = 30_000;
|
|
105
|
+
const ATTACH_IDLE_MIN_MS = 30_000;
|
|
106
|
+
const ATTACH_IDLE_MAX_MS = 600_000;
|
|
107
|
+
|
|
108
|
+
/** A socket opened by an `'attach'` prewarm, waiting for the click. */
|
|
109
|
+
type AttachedSocket = {
|
|
110
|
+
ws: WebSocket;
|
|
111
|
+
/**
|
|
112
|
+
* `connecting` until the handshake completes; then `attached` (no call yet,
|
|
113
|
+
* needs `start`) or `live` (the server ignored attach and the call is live).
|
|
114
|
+
*/
|
|
115
|
+
state: "connecting" | "attached" | "live";
|
|
116
|
+
/** Settles when the handshake completes, fails, or the socket is released. */
|
|
117
|
+
ready: Promise<void>;
|
|
118
|
+
settle: () => void;
|
|
119
|
+
openedAt: number;
|
|
120
|
+
idleTimer: ReturnType<typeof setTimeout> | undefined;
|
|
121
|
+
};
|
|
122
|
+
/** Declared on every call; non-full-duplex engines ignore it. */
|
|
123
|
+
const FULL_DUPLEX_CAPABILITY = "full-duplex-v1";
|
|
124
|
+
/**
|
|
125
|
+
* Slack added to the computed end of queued speech-to-speech audio before the
|
|
126
|
+
* provider reports the reply as drained (covers engine prebuffer + scheduling).
|
|
127
|
+
*/
|
|
128
|
+
const CONTINUOUS_DRAIN_GRACE_MS = 300;
|
|
129
|
+
/**
|
|
130
|
+
* How long a client `cancel` waits for the server's `audio_clear` before audio
|
|
131
|
+
* is accepted again, so a lost acknowledgement can't mute the call for good.
|
|
132
|
+
*/
|
|
133
|
+
const CANCEL_ACK_TIMEOUT_MS = 2000;
|
|
134
|
+
/**
|
|
135
|
+
* How long a `delegation_result` waits for the call context still being built
|
|
136
|
+
* (a slow host `callContext`), so context always precedes the result. Past it,
|
|
137
|
+
* the result goes out and that context is dropped.
|
|
138
|
+
*/
|
|
139
|
+
const CONTEXT_BEFORE_RESULT_TIMEOUT_MS = 2000;
|
|
140
|
+
const VOICE_PROTOCOL = "runtype-browser-v1";
|
|
141
|
+
/** `approvalTimeoutMs` cap: below core's delegation deadline (600 s, restarted by each update). */
|
|
142
|
+
const APPROVAL_TIMEOUT_MAX_MS = 540_000;
|
|
143
|
+
const DISCLOSURE_TEXT = "You're talking to an AI assistant. Voice is processed by OpenAI.";
|
|
144
|
+
/** Call-start context frame: total cap, host share, history window, per-message cap. */
|
|
145
|
+
const CONTEXT_MAX_CHARS = 8000;
|
|
146
|
+
const CONTEXT_HOST_MAX_CHARS = 4000;
|
|
147
|
+
const CONTEXT_MESSAGES = 12;
|
|
148
|
+
const CONTEXT_MESSAGE_MAX_CHARS = 2000;
|
|
149
|
+
|
|
150
|
+
type TranscriptCallback = (
|
|
151
|
+
role: "user" | "assistant",
|
|
152
|
+
text: string,
|
|
153
|
+
isFinal: boolean,
|
|
154
|
+
metadata?: VoiceTranscriptMetadata,
|
|
155
|
+
) => void;
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* The call-start `context` text: "Conversation so far:" plus the last
|
|
159
|
+
* {@link CONTEXT_MESSAGES} messages (newest kept when over budget), then the
|
|
160
|
+
* host's extra context (at most {@link CONTEXT_HOST_MAX_CHARS}, so history
|
|
161
|
+
* always fits). Capped at {@link CONTEXT_MAX_CHARS}; "" when empty.
|
|
162
|
+
*/
|
|
163
|
+
export function buildCallContext(
|
|
164
|
+
history: Array<{ role: "user" | "assistant"; content: string }>,
|
|
165
|
+
extra: string,
|
|
166
|
+
): string {
|
|
167
|
+
const header = "Conversation so far:";
|
|
168
|
+
const tail = extra.slice(0, CONTEXT_HOST_MAX_CHARS).trim();
|
|
169
|
+
let budget = CONTEXT_MAX_CHARS - (tail ? tail.length + 2 : 0) - header.length;
|
|
170
|
+
const lines: string[] = [];
|
|
171
|
+
for (const message of history.slice(-CONTEXT_MESSAGES).reverse()) {
|
|
172
|
+
let text = message.content.replace(/\s+/g, " ").trim();
|
|
173
|
+
if (!text) continue;
|
|
174
|
+
if (text.length > CONTEXT_MESSAGE_MAX_CHARS) {
|
|
175
|
+
text = `${text.slice(0, CONTEXT_MESSAGE_MAX_CHARS - 1)}…`;
|
|
176
|
+
}
|
|
177
|
+
const line = `${message.role === "user" ? "User" : "Assistant"}: ${text}`;
|
|
178
|
+
if (line.length + 1 > budget) break;
|
|
179
|
+
budget -= line.length + 1;
|
|
180
|
+
lines.unshift(line);
|
|
181
|
+
}
|
|
182
|
+
const parts = lines.length > 0 ? [`${header}\n${lines.join("\n")}`] : [];
|
|
183
|
+
if (tail) parts.push(tail);
|
|
184
|
+
return parts.join("\n\n");
|
|
185
|
+
}
|
|
40
186
|
|
|
41
187
|
/**
|
|
42
188
|
* Strip the canonical 44-byte WAV header (if present) and return the raw PCM16
|
|
@@ -63,39 +209,85 @@ function toWsBase(host: string): string {
|
|
|
63
209
|
return `${secure ? "wss:" : "ws:"}//${trimmed}`;
|
|
64
210
|
}
|
|
65
211
|
|
|
212
|
+
/** Derive an http(s):// base URL from a configured host (the ws base, reversed). */
|
|
213
|
+
function toHttpBase(host: string): string {
|
|
214
|
+
return toWsBase(host).replace(/^ws/i, "http");
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function clampAttachIdleMs(ms: number | undefined): number {
|
|
218
|
+
if (typeof ms !== "number" || !Number.isFinite(ms)) return ATTACH_IDLE_DEFAULT_MS;
|
|
219
|
+
return Math.min(ATTACH_IDLE_MAX_MS, Math.max(ATTACH_IDLE_MIN_MS, Math.round(ms)));
|
|
220
|
+
}
|
|
221
|
+
|
|
66
222
|
export class RuntypeVoiceProvider implements VoiceProvider {
|
|
67
223
|
type: "runtype" = "runtype";
|
|
68
224
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
225
|
+
#ws: WebSocket | null = null;
|
|
226
|
+
#captureContext: AudioContext | null = null;
|
|
227
|
+
#levelCallbacks: ((level: number) => void)[] = [];
|
|
228
|
+
#mediaStream: MediaStream | null = null;
|
|
229
|
+
#sourceNode: MediaStreamAudioSourceNode | null = null;
|
|
230
|
+
#processor: ScriptProcessorNode | null = null;
|
|
231
|
+
// Captured before the call socket opened; flushed in order on open.
|
|
232
|
+
#preOpenAudio: ArrayBuffer[] = [];
|
|
233
|
+
#preOpenBytes = 0;
|
|
234
|
+
#playback: VoicePlaybackEngine | null = null;
|
|
76
235
|
|
|
77
236
|
// True while a call (WS session) is live: drives the idempotent start guard
|
|
78
237
|
// and `isBargeInActive()`.
|
|
79
|
-
|
|
80
|
-
|
|
238
|
+
#callLive = false;
|
|
239
|
+
#isSpeaking = false;
|
|
81
240
|
|
|
82
241
|
// Invalidates in-flight async work (playback-engine creation, late frames,
|
|
83
242
|
// status transitions) after a teardown/restart so a stale callback can't act
|
|
84
243
|
// on a newer call's resources. Bumped on every start and every cleanup.
|
|
85
|
-
|
|
244
|
+
#callGeneration = 0;
|
|
86
245
|
|
|
87
246
|
// Distinguishes a user-initiated close (code 1000) from a dropped connection.
|
|
88
|
-
|
|
247
|
+
#intentionalClose = false;
|
|
89
248
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
249
|
+
// Prewarm latch and the (at most one) socket an `'attach'` prewarm opened.
|
|
250
|
+
#lastPrewarmAt = Number.NEGATIVE_INFINITY;
|
|
251
|
+
#attached: AttachedSocket | null = null;
|
|
252
|
+
// Full-duplex (speech-to-speech) session state; reset on every cleanup.
|
|
253
|
+
#speechToSpeech = false;
|
|
254
|
+
#delegating = false;
|
|
255
|
+
// Until this time, audio and assistant text belong to a reply the client cancelled.
|
|
256
|
+
#cancelledUntil = 0;
|
|
257
|
+
#playbackEndsAt = 0;
|
|
258
|
+
#drainTimer: ReturnType<typeof setTimeout> | undefined;
|
|
259
|
+
// Engine `onFinished` callbacks are one-shot (cleared on fire and on flush),
|
|
260
|
+
// so the provider re-registers before each reply that may need it.
|
|
261
|
+
#finishedArmed = false;
|
|
262
|
+
// Client delegation (see header); per-call state reset on every cleanup.
|
|
263
|
+
private bridge: VoiceSessionBridge | null = null;
|
|
264
|
+
#clientDelegation = false;
|
|
265
|
+
// session_config.capabilities: the negotiated client frame types.
|
|
266
|
+
#capabilities = new Set<string>();
|
|
267
|
+
// Per delegation: aborts its pending approval follow-up (cancelled, or call end).
|
|
268
|
+
#followUps = new Map<string, AbortController>();
|
|
269
|
+
// Delegations the server cancelled (or refused a frame for): no more frames for them.
|
|
270
|
+
#cancelledDelegations = new Set<string>();
|
|
271
|
+
// session_config.callId: the server's id for this call, for logs.
|
|
272
|
+
#callId: string | undefined;
|
|
273
|
+
// Call-start context, built at session_config and held until the visitor's
|
|
274
|
+
// first final transcript (sent earlier, the voice model tends to answer it).
|
|
275
|
+
#contextSent = false;
|
|
276
|
+
#pendingContext: Promise<string> | null = null;
|
|
277
|
+
// The released context's send; settles once it went out (or never will).
|
|
278
|
+
#contextSend: { done: Promise<void>; drop: () => void } | null = null;
|
|
279
|
+
#delegations: Promise<void> = Promise.resolve();
|
|
280
|
+
// Delegations answered ok: their spoken read-back is folded.
|
|
281
|
+
#answered = new Set<string>();
|
|
282
|
+
#foldReadback = false;
|
|
283
|
+
#assistantTurns = new Set<string>();
|
|
284
|
+
#foldedTurns = new Set<string>();
|
|
285
|
+
|
|
286
|
+
#resultCallbacks: ((result: VoiceResult) => void)[] = [];
|
|
287
|
+
#errorCallbacks: ((error: Error) => void)[] = [];
|
|
288
|
+
#statusCallbacks: ((status: VoiceStatus) => void)[] = [];
|
|
289
|
+
#transcriptCallbacks: TranscriptCallback[] = [];
|
|
290
|
+
#metricsCallbacks: ((metrics: VoiceMetrics) => void)[] = [];
|
|
99
291
|
|
|
100
292
|
constructor(private config: VoiceConfig["runtype"]) {}
|
|
101
293
|
|
|
@@ -104,9 +296,175 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
104
296
|
/** No-op: the WS session opens lazily in `startListening` (the "call"). */
|
|
105
297
|
async connect(): Promise<void> {}
|
|
106
298
|
|
|
299
|
+
setSessionBridge(bridge: VoiceSessionBridge): void {
|
|
300
|
+
this.bridge = bridge;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/**
|
|
304
|
+
* Warm the voice path ahead of the click. Fire-and-forget and throttled to
|
|
305
|
+
* once per {@link PREWARM_THROTTLE_MS}; never emits an error or a status.
|
|
306
|
+
*/
|
|
307
|
+
prewarm(): void {
|
|
308
|
+
if (this.#callLive || this.#attached) return;
|
|
309
|
+
const agentId = this.config?.agentId;
|
|
310
|
+
const token = this.config?.clientToken;
|
|
311
|
+
const host = this.config?.host;
|
|
312
|
+
if (!agentId || !token || !host) return;
|
|
313
|
+
const now = Date.now();
|
|
314
|
+
if (now - this.#lastPrewarmAt < PREWARM_THROTTLE_MS) return;
|
|
315
|
+
this.#lastPrewarmAt = now;
|
|
316
|
+
try {
|
|
317
|
+
if (this.config?.prewarmMode === "attach") {
|
|
318
|
+
this.#attach(host, agentId, token);
|
|
319
|
+
} else {
|
|
320
|
+
this.#requestPrewarm(host, agentId, token);
|
|
321
|
+
}
|
|
322
|
+
} catch {
|
|
323
|
+
// A prewarm must never surface an error; the click connects normally.
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
#requestPrewarm(host: string, agentId: string, token: string): void {
|
|
328
|
+
if (typeof fetch !== "function") return;
|
|
329
|
+
const url = `${toHttpBase(host)}/v1/client/agents/${encodeURIComponent(agentId)}/voice/prewarm`;
|
|
330
|
+
void fetch(url, {
|
|
331
|
+
method: "POST",
|
|
332
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
333
|
+
keepalive: true,
|
|
334
|
+
}).catch(() => {});
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
#attach(host: string, agentId: string, token: string): void {
|
|
338
|
+
const params = new URLSearchParams();
|
|
339
|
+
const configuredIdleMs = this.config?.attachIdleMs;
|
|
340
|
+
const idleMs = clampAttachIdleMs(configuredIdleMs);
|
|
341
|
+
if (configuredIdleMs !== undefined) params.set("attachIdleMs", String(idleMs));
|
|
342
|
+
const ws = new WebSocket(this.#voiceSocketUrl(host, agentId, params), [
|
|
343
|
+
"runtype.bearer",
|
|
344
|
+
"runtype.attach",
|
|
345
|
+
token,
|
|
346
|
+
]);
|
|
347
|
+
ws.binaryType = "arraybuffer";
|
|
348
|
+
let settle: () => void = () => {};
|
|
349
|
+
const ready = new Promise<void>((resolve) => {
|
|
350
|
+
settle = resolve;
|
|
351
|
+
});
|
|
352
|
+
const entry: AttachedSocket = {
|
|
353
|
+
ws,
|
|
354
|
+
state: "connecting",
|
|
355
|
+
ready,
|
|
356
|
+
settle,
|
|
357
|
+
openedAt: 0,
|
|
358
|
+
idleTimer: undefined,
|
|
359
|
+
};
|
|
360
|
+
this.#attached = entry;
|
|
361
|
+
|
|
362
|
+
ws.onopen = () => {
|
|
363
|
+
if (this.#attached !== entry) return;
|
|
364
|
+
entry.state = ws.protocol === "runtype.attach" ? "attached" : "live";
|
|
365
|
+
entry.openedAt = Date.now();
|
|
366
|
+
this.#armAttachIdle(entry, idleMs);
|
|
367
|
+
if (entry.state === "attached") ws.send('{"type":"ping"}');
|
|
368
|
+
entry.settle();
|
|
369
|
+
};
|
|
370
|
+
ws.onmessage = (event) => {
|
|
371
|
+
if (this.#attached !== entry || typeof event.data !== "string") return;
|
|
372
|
+
let msg: { type?: unknown; idleMs?: unknown };
|
|
373
|
+
try {
|
|
374
|
+
msg = JSON.parse(event.data);
|
|
375
|
+
} catch {
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
// The server's idle window is authoritative; close no later than it does.
|
|
379
|
+
if (msg.type === "attached" && typeof msg.idleMs === "number") {
|
|
380
|
+
this.#armAttachIdle(entry, msg.idleMs - (Date.now() - entry.openedAt));
|
|
381
|
+
}
|
|
382
|
+
};
|
|
383
|
+
ws.onerror = () => {};
|
|
384
|
+
ws.onclose = () => {
|
|
385
|
+
if (this.#attached === entry) this.#releaseAttached();
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
#armAttachIdle(entry: AttachedSocket, ms: number): void {
|
|
390
|
+
clearTimeout(entry.idleTimer);
|
|
391
|
+
entry.idleTimer = setTimeout(() => {
|
|
392
|
+
if (this.#attached === entry) this.#releaseAttached();
|
|
393
|
+
}, Math.max(0, ms));
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/** Close and forget the prewarmed socket, if any. Silent by design. */
|
|
397
|
+
#releaseAttached(): void {
|
|
398
|
+
const entry = this.#attached;
|
|
399
|
+
if (!entry) return;
|
|
400
|
+
this.#attached = null;
|
|
401
|
+
clearTimeout(entry.idleTimer);
|
|
402
|
+
const { ws } = entry;
|
|
403
|
+
ws.onopen = null;
|
|
404
|
+
ws.onmessage = null;
|
|
405
|
+
ws.onerror = null;
|
|
406
|
+
ws.onclose = null;
|
|
407
|
+
entry.settle();
|
|
408
|
+
if (ws.readyState === WebSocket.CONNECTING || ws.readyState === WebSocket.OPEN) {
|
|
409
|
+
try {
|
|
410
|
+
ws.close(1000, "prewarm released");
|
|
411
|
+
} catch {
|
|
412
|
+
// ignore
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* Hand the prewarmed socket to the call, waiting up to
|
|
419
|
+
* {@link ATTACH_HANDSHAKE_TIMEOUT_MS} for a handshake still in flight.
|
|
420
|
+
* Resolves `null` when there is none, it stalled, or it dropped, so the
|
|
421
|
+
* caller opens a fresh socket.
|
|
422
|
+
*/
|
|
423
|
+
async #takeAttachedSocket(): Promise<{ ws: WebSocket; live: boolean } | null> {
|
|
424
|
+
const entry = this.#attached;
|
|
425
|
+
if (!entry) return null;
|
|
426
|
+
if (entry.state === "connecting") {
|
|
427
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
428
|
+
await Promise.race([
|
|
429
|
+
entry.ready,
|
|
430
|
+
new Promise<void>((resolve) => {
|
|
431
|
+
timer = setTimeout(resolve, ATTACH_HANDSHAKE_TIMEOUT_MS);
|
|
432
|
+
}),
|
|
433
|
+
]);
|
|
434
|
+
clearTimeout(timer);
|
|
435
|
+
}
|
|
436
|
+
if (this.#attached !== entry) return null;
|
|
437
|
+
if (entry.state === "connecting" || entry.ws.readyState !== WebSocket.OPEN) {
|
|
438
|
+
this.#releaseAttached();
|
|
439
|
+
return null;
|
|
440
|
+
}
|
|
441
|
+
this.#attached = null;
|
|
442
|
+
clearTimeout(entry.idleTimer);
|
|
443
|
+
return { ws: entry.ws, live: entry.state === "live" };
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
/**
|
|
447
|
+
* The call socket URL. Every socket declares the full-duplex capability;
|
|
448
|
+
* `extra` adds params (e.g. an attach prewarm's). The token never goes in
|
|
449
|
+
* the URL, and these params are not secrets.
|
|
450
|
+
*/
|
|
451
|
+
#voiceSocketUrl(host: string, agentId: string, attach?: URLSearchParams): string {
|
|
452
|
+
const capabilities = ["partial_transcript", "context"];
|
|
453
|
+
if (this.bridge && this.config?.clientDelegation !== false) capabilities.push("client_delegation", "delegation_update");
|
|
454
|
+
if (attach) capabilities.unshift("attach");
|
|
455
|
+
const params = new URLSearchParams({
|
|
456
|
+
voiceProtocol: VOICE_PROTOCOL,
|
|
457
|
+
clientVersion: `persona/${VERSION}`,
|
|
458
|
+
voiceCapabilities: FULL_DUPLEX_CAPABILITY,
|
|
459
|
+
clientCapabilities: capabilities.join(","),
|
|
460
|
+
});
|
|
461
|
+
attach?.forEach((value, key) => params.set(key, value));
|
|
462
|
+
return `${toWsBase(host)}/ws/agents/${encodeURIComponent(agentId)}/voice?${params}`;
|
|
463
|
+
}
|
|
464
|
+
|
|
107
465
|
/** Start the call: acquire mic, open the WS, stream PCM until hang-up. */
|
|
108
466
|
async startListening(): Promise<void> {
|
|
109
|
-
if (this
|
|
467
|
+
if (this.#callLive) return; // idempotent: a call is already live
|
|
110
468
|
|
|
111
469
|
const agentId = this.config?.agentId;
|
|
112
470
|
const token = this.config?.clientToken;
|
|
@@ -115,9 +473,9 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
115
473
|
if (!token) throw new Error("Runtype voice requires a clientToken");
|
|
116
474
|
if (!host) throw new Error("Runtype voice requires a host (or widget apiUrl)");
|
|
117
475
|
|
|
118
|
-
const generation = ++this
|
|
119
|
-
this
|
|
120
|
-
this
|
|
476
|
+
const generation = ++this.#callGeneration;
|
|
477
|
+
this.#intentionalClose = false;
|
|
478
|
+
this.#callLive = true;
|
|
121
479
|
|
|
122
480
|
try {
|
|
123
481
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
@@ -127,11 +485,11 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
127
485
|
echoCancellation: true,
|
|
128
486
|
},
|
|
129
487
|
});
|
|
130
|
-
if (generation !== this
|
|
488
|
+
if (generation !== this.#callGeneration) {
|
|
131
489
|
stream.getTracks().forEach((t) => t.stop());
|
|
132
490
|
return;
|
|
133
491
|
}
|
|
134
|
-
this
|
|
492
|
+
this.#mediaStream = stream;
|
|
135
493
|
|
|
136
494
|
// Create + resume both contexts inside the click gesture (iOS autoplay).
|
|
137
495
|
const AudioCtx =
|
|
@@ -142,97 +500,124 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
142
500
|
if (captureContext.state === "suspended") {
|
|
143
501
|
await captureContext.resume().catch(() => {});
|
|
144
502
|
}
|
|
145
|
-
this
|
|
503
|
+
this.#captureContext = captureContext;
|
|
146
504
|
|
|
147
505
|
const engine = this.config?.createPlaybackEngine
|
|
148
506
|
? await this.config.createPlaybackEngine()
|
|
149
507
|
: new AudioPlaybackManager(PLAYBACK_SAMPLE_RATE);
|
|
150
|
-
if (generation !== this
|
|
508
|
+
if (generation !== this.#callGeneration) {
|
|
151
509
|
// Torn down while async work was in flight: free what we acquired.
|
|
152
510
|
void engine.destroy();
|
|
153
511
|
stream.getTracks().forEach((t) => t.stop());
|
|
154
512
|
captureContext.close().catch(() => {});
|
|
155
513
|
return;
|
|
156
514
|
}
|
|
157
|
-
this
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
515
|
+
this.#playback = engine;
|
|
516
|
+
this.#armPlaybackFinished();
|
|
517
|
+
// Capture from now: frames buffer until the call socket is open.
|
|
518
|
+
this.#startCapture(captureContext, stream, generation);
|
|
519
|
+
|
|
520
|
+
const adopted = await this.#takeAttachedSocket();
|
|
521
|
+
if (generation !== this.#callGeneration) {
|
|
522
|
+
adopted?.ws.close(1000, "client ended call");
|
|
523
|
+
return;
|
|
524
|
+
}
|
|
525
|
+
if (adopted) {
|
|
526
|
+
const { ws } = adopted;
|
|
527
|
+
this.#ws = ws;
|
|
528
|
+
this.#bindCallSocket(ws, generation);
|
|
529
|
+
if (!adopted.live) ws.send('{"type":"start"}');
|
|
530
|
+
this.#flushPreOpenAudio(ws);
|
|
531
|
+
this.#emitStatus("listening");
|
|
532
|
+
return;
|
|
533
|
+
}
|
|
166
534
|
|
|
167
|
-
const wsUrl = `${toWsBase(host)}/ws/agents/${encodeURIComponent(agentId)}/voice`;
|
|
168
535
|
// Token rides the subprotocol; `runtype.bearer` is the marker the server
|
|
169
536
|
// echoes as the negotiated subprotocol (browsers fail the handshake if an
|
|
170
537
|
// offered subprotocol goes unanswered).
|
|
171
|
-
const ws = new WebSocket(
|
|
538
|
+
const ws = new WebSocket(this.#voiceSocketUrl(host, agentId), ["runtype.bearer", token]);
|
|
172
539
|
ws.binaryType = "arraybuffer";
|
|
173
|
-
this
|
|
540
|
+
this.#ws = ws;
|
|
174
541
|
|
|
175
542
|
ws.onopen = () => {
|
|
176
|
-
if (generation !== this
|
|
177
|
-
this
|
|
178
|
-
this
|
|
179
|
-
};
|
|
180
|
-
|
|
181
|
-
ws.onmessage = (event) => this.handleMessage(event, generation);
|
|
182
|
-
|
|
183
|
-
ws.onerror = () => {
|
|
184
|
-
if (generation !== this.callGeneration) return;
|
|
185
|
-
this.emitError(new Error("Voice connection failed"));
|
|
186
|
-
this.emitStatus("error");
|
|
187
|
-
this.cleanup();
|
|
188
|
-
};
|
|
189
|
-
|
|
190
|
-
ws.onclose = (evt) => {
|
|
191
|
-
if (this.intentionalClose) {
|
|
192
|
-
this.intentionalClose = false;
|
|
193
|
-
return;
|
|
194
|
-
}
|
|
195
|
-
if (generation !== this.callGeneration) return;
|
|
196
|
-
if (evt.code !== 1000) {
|
|
197
|
-
const codeMsg = evt.code ? ` (code ${evt.code})` : "";
|
|
198
|
-
this.emitError(new Error(`Voice connection closed${codeMsg}`));
|
|
199
|
-
this.emitStatus("error");
|
|
200
|
-
} else {
|
|
201
|
-
this.emitStatus("idle");
|
|
202
|
-
}
|
|
203
|
-
this.cleanup();
|
|
543
|
+
if (generation !== this.#callGeneration) return;
|
|
544
|
+
this.#flushPreOpenAudio(ws);
|
|
545
|
+
this.#emitStatus("listening");
|
|
204
546
|
};
|
|
547
|
+
this.#bindCallSocket(ws, generation);
|
|
205
548
|
} catch (error) {
|
|
206
|
-
this
|
|
207
|
-
this
|
|
208
|
-
this
|
|
549
|
+
this.#cleanup();
|
|
550
|
+
this.#emitError(error as Error);
|
|
551
|
+
this.#emitStatus("error");
|
|
209
552
|
throw error;
|
|
210
553
|
}
|
|
211
554
|
}
|
|
212
555
|
|
|
556
|
+
/** Route a call socket's frames, errors, and close into this call. */
|
|
557
|
+
#bindCallSocket(ws: WebSocket, generation: number): void {
|
|
558
|
+
ws.onmessage = (event) => this.#handleMessage(event, generation);
|
|
559
|
+
|
|
560
|
+
ws.onerror = () => {
|
|
561
|
+
if (generation !== this.#callGeneration) return;
|
|
562
|
+
this.#emitError(new Error("Voice connection failed"));
|
|
563
|
+
this.#emitStatus("error");
|
|
564
|
+
this.#cleanup();
|
|
565
|
+
};
|
|
566
|
+
|
|
567
|
+
ws.onclose = (evt) => {
|
|
568
|
+
if (this.#intentionalClose) {
|
|
569
|
+
this.#intentionalClose = false;
|
|
570
|
+
return;
|
|
571
|
+
}
|
|
572
|
+
if (generation !== this.#callGeneration) return;
|
|
573
|
+
if (evt.code !== 1000) {
|
|
574
|
+
const codeMsg = evt.code ? ` (code ${evt.code})` : "";
|
|
575
|
+
this.#emitError(new Error(`Voice connection closed${codeMsg}`));
|
|
576
|
+
this.#emitStatus("error");
|
|
577
|
+
} else {
|
|
578
|
+
this.#emitStatus("idle");
|
|
579
|
+
}
|
|
580
|
+
this.#cleanup();
|
|
581
|
+
};
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
/** The AI-disclosure notice for a live speech-to-speech call (`disclosureText`; `false` hides it). */
|
|
585
|
+
getDisclosure(): string | null {
|
|
586
|
+
const text = this.config?.disclosureText;
|
|
587
|
+
return this.#callLive && this.#speechToSpeech && text !== false ? text || DISCLOSURE_TEXT : null;
|
|
588
|
+
}
|
|
589
|
+
|
|
213
590
|
/** End the call (hang up). */
|
|
214
591
|
async stopListening(): Promise<void> {
|
|
215
|
-
this
|
|
216
|
-
this
|
|
592
|
+
this.#cleanup();
|
|
593
|
+
this.#emitStatus("idle");
|
|
217
594
|
}
|
|
218
595
|
|
|
219
596
|
/** Tear down the call and drop all callbacks (used by `cleanupVoice`). */
|
|
220
597
|
async disconnect(): Promise<void> {
|
|
221
|
-
this
|
|
222
|
-
this
|
|
223
|
-
this
|
|
224
|
-
this
|
|
225
|
-
this
|
|
226
|
-
this
|
|
227
|
-
this
|
|
598
|
+
this.#cleanup();
|
|
599
|
+
this.#emitStatus("disconnected");
|
|
600
|
+
this.#resultCallbacks = [];
|
|
601
|
+
this.#errorCallbacks = [];
|
|
602
|
+
this.#statusCallbacks = [];
|
|
603
|
+
this.#transcriptCallbacks = [];
|
|
604
|
+
this.#metricsCallbacks = [];
|
|
228
605
|
}
|
|
229
606
|
|
|
230
|
-
/**
|
|
607
|
+
/**
|
|
608
|
+
* Stop the spoken reply without ending the call. In a speech-to-speech
|
|
609
|
+
* session this also asks the server to cancel the response; audio is dropped
|
|
610
|
+
* until its `audio_clear` acknowledgement arrives.
|
|
611
|
+
*/
|
|
231
612
|
stopPlayback(): void {
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
if (
|
|
235
|
-
this
|
|
613
|
+
this.#clearLocalPlayback();
|
|
614
|
+
const ws = this.#ws;
|
|
615
|
+
if (ws && ws.readyState === WebSocket.OPEN) {
|
|
616
|
+
if (this.#speechToSpeech) {
|
|
617
|
+
this.#cancelledUntil = Date.now() + CANCEL_ACK_TIMEOUT_MS;
|
|
618
|
+
ws.send('{"type":"cancel"}');
|
|
619
|
+
}
|
|
620
|
+
this.#emitStatus("listening");
|
|
236
621
|
}
|
|
237
622
|
}
|
|
238
623
|
|
|
@@ -245,35 +630,33 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
245
630
|
|
|
246
631
|
/** True while the call (hot mic) is live. */
|
|
247
632
|
isBargeInActive(): boolean {
|
|
248
|
-
return this
|
|
633
|
+
return this.#callLive;
|
|
249
634
|
}
|
|
250
635
|
|
|
251
636
|
/** "Hang up" the always-on mic. */
|
|
252
637
|
onLevel(callback: (level: number) => void): void {
|
|
253
|
-
this
|
|
638
|
+
this.#levelCallbacks.push(callback);
|
|
254
639
|
}
|
|
255
640
|
|
|
256
641
|
async deactivateBargeIn(): Promise<void> {
|
|
257
|
-
this
|
|
258
|
-
this
|
|
642
|
+
this.#cleanup();
|
|
643
|
+
this.#emitStatus("idle");
|
|
259
644
|
}
|
|
260
645
|
|
|
261
646
|
// --- Capture ---------------------------------------------------------------
|
|
262
647
|
|
|
263
|
-
|
|
264
|
-
context: AudioContext,
|
|
265
|
-
stream: MediaStream,
|
|
266
|
-
ws: WebSocket,
|
|
267
|
-
generation: number,
|
|
268
|
-
): void {
|
|
648
|
+
#startCapture(context: AudioContext, stream: MediaStream, generation: number): void {
|
|
269
649
|
const source = context.createMediaStreamSource(stream);
|
|
270
|
-
this
|
|
650
|
+
this.#sourceNode = source;
|
|
271
651
|
const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
|
|
272
|
-
this
|
|
652
|
+
this.#processor = processor;
|
|
273
653
|
|
|
274
654
|
processor.onaudioprocess = (e) => {
|
|
275
|
-
if (generation !== this
|
|
276
|
-
|
|
655
|
+
if (generation !== this.#callGeneration) return;
|
|
656
|
+
// No socket yet (adopting a prewarm) or still handshaking: buffer.
|
|
657
|
+
const ws = this.#ws;
|
|
658
|
+
const open = ws?.readyState === WebSocket.OPEN;
|
|
659
|
+
if (ws && !open && ws.readyState !== WebSocket.CONNECTING) return;
|
|
277
660
|
const input = e.inputBuffer.getChannelData(0);
|
|
278
661
|
const pcm16 = new Int16Array(input.length);
|
|
279
662
|
let sumSquares = 0;
|
|
@@ -282,14 +665,23 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
282
665
|
sumSquares += s * s;
|
|
283
666
|
pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
|
284
667
|
}
|
|
285
|
-
ws
|
|
668
|
+
if (ws && open) {
|
|
669
|
+
this.#flushPreOpenAudio(ws);
|
|
670
|
+
ws.send(pcm16.buffer);
|
|
671
|
+
} else {
|
|
672
|
+
this.#preOpenAudio.push(pcm16.buffer);
|
|
673
|
+
this.#preOpenBytes += pcm16.byteLength;
|
|
674
|
+
while (this.#preOpenBytes > PRE_OPEN_AUDIO_MAX_BYTES) {
|
|
675
|
+
this.#preOpenBytes -= this.#preOpenAudio.shift()!.byteLength;
|
|
676
|
+
}
|
|
677
|
+
}
|
|
286
678
|
// Amplitude comes free from the buffer we already walked: no analyser
|
|
287
679
|
// node, no second capture. RMS is scaled because speech rarely exceeds
|
|
288
680
|
// ~0.3 RMS, so raw values would sit near the bottom of the 0..1 range.
|
|
289
|
-
if (this
|
|
681
|
+
if (this.#levelCallbacks.length > 0) {
|
|
290
682
|
const rms = Math.sqrt(sumSquares / input.length);
|
|
291
683
|
const level = Math.max(0, Math.min(1, rms * LEVEL_RMS_SCALE));
|
|
292
|
-
for (const cb of this
|
|
684
|
+
for (const cb of this.#levelCallbacks) cb(level);
|
|
293
685
|
}
|
|
294
686
|
};
|
|
295
687
|
|
|
@@ -299,13 +691,19 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
299
691
|
processor.connect(context.destination);
|
|
300
692
|
}
|
|
301
693
|
|
|
694
|
+
#flushPreOpenAudio(ws: WebSocket): void {
|
|
695
|
+
for (const frame of this.#preOpenAudio) ws.send(frame);
|
|
696
|
+
this.#preOpenAudio = [];
|
|
697
|
+
this.#preOpenBytes = 0;
|
|
698
|
+
}
|
|
699
|
+
|
|
302
700
|
// --- Downstream ------------------------------------------------------------
|
|
303
701
|
|
|
304
|
-
|
|
305
|
-
if (generation !== this
|
|
702
|
+
#handleMessage(event: MessageEvent, generation: number): void {
|
|
703
|
+
if (generation !== this.#callGeneration) return;
|
|
306
704
|
|
|
307
705
|
if (event.data instanceof ArrayBuffer) {
|
|
308
|
-
this
|
|
706
|
+
this.#handleAudioFrame(event.data, generation);
|
|
309
707
|
return;
|
|
310
708
|
}
|
|
311
709
|
|
|
@@ -317,30 +715,142 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
317
715
|
}
|
|
318
716
|
|
|
319
717
|
switch (msg.type) {
|
|
718
|
+
case "session_config":
|
|
719
|
+
// The follow-up session_config carries only interruptionMode: keep the mode.
|
|
720
|
+
if (msg.speechMode) {
|
|
721
|
+
this.#speechToSpeech = msg.speechMode === "speech_to_speech";
|
|
722
|
+
this.#playback?.setContinuousMode?.(this.#speechToSpeech);
|
|
723
|
+
}
|
|
724
|
+
// The negotiated client frame types. A server without `capabilities`
|
|
725
|
+
// predates them: no client delegation, context or updates.
|
|
726
|
+
if (Array.isArray(msg.capabilities)) this.#capabilities = new Set(msg.capabilities);
|
|
727
|
+
if (typeof msg.callId === "string") this.#callId = msg.callId;
|
|
728
|
+
this.#clientDelegation = this.#speechToSpeech && this.#capabilities.has("client_delegation") && !!this.bridge;
|
|
729
|
+
// Speech-to-speech is known only now: let the UI show its disclosure.
|
|
730
|
+
if (this.#speechToSpeech && !this.#isSpeaking) this.#emitStatus("listening");
|
|
731
|
+
if (this.#capabilities.has("context") && !this.#contextSent) {
|
|
732
|
+
this.#contextSent = true;
|
|
733
|
+
this.#pendingContext = this.#buildContextText();
|
|
734
|
+
}
|
|
735
|
+
break;
|
|
736
|
+
|
|
737
|
+
case "transcript_update": {
|
|
738
|
+
const role = msg.role === "assistant" ? "assistant" : "user";
|
|
739
|
+
// A reply the client cancelled keeps streaming until the server clears it.
|
|
740
|
+
const utteranceId = msg.utteranceId ?? msg.turnId;
|
|
741
|
+
if (!utteranceId || (role === "assistant" && this.#isCancelling())) break;
|
|
742
|
+
const turnId = String(utteranceId);
|
|
743
|
+
if (role === "user") {
|
|
744
|
+
// Release on a final user transcript, not a partial: a mid-utterance
|
|
745
|
+
// context append makes the voice model answer early (or not delegate).
|
|
746
|
+
if (msg.final === true) this.#flushCallContext(generation);
|
|
747
|
+
this.#foldReadback = false;
|
|
748
|
+
} else {
|
|
749
|
+
// Read-back of a delegated result: core rotates the assistant id at
|
|
750
|
+
// completion, so it is the first new id after it. The chat renders it.
|
|
751
|
+
if (this.#foldReadback && !this.#assistantTurns.has(turnId)) {
|
|
752
|
+
this.#foldedTurns.add(turnId);
|
|
753
|
+
this.#foldReadback = false;
|
|
754
|
+
}
|
|
755
|
+
this.#assistantTurns.add(turnId);
|
|
756
|
+
if (this.#foldedTurns.has(turnId)) break;
|
|
757
|
+
}
|
|
758
|
+
this.#emitTranscript(role, msg.text ?? "", msg.final === true, {
|
|
759
|
+
turnId,
|
|
760
|
+
...(typeof msg.startMs === "number" && { startMs: msg.startMs }),
|
|
761
|
+
...(typeof msg.endMs === "number" && { endMs: msg.endMs }),
|
|
762
|
+
// The chat pipeline owns the conversation: speech is only captioned.
|
|
763
|
+
...(this.#clientDelegation && { caption: true }),
|
|
764
|
+
});
|
|
765
|
+
break;
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
case "delegation_started":
|
|
769
|
+
case "delegation_completed": {
|
|
770
|
+
const delegationId = String(msg.delegationId ?? msg.turnId);
|
|
771
|
+
if (msg.type === "delegation_started") {
|
|
772
|
+
this.#flushCallContext(generation);
|
|
773
|
+
// `input`: run this turn through the chat pipeline (client delegation).
|
|
774
|
+
const input = msg.input;
|
|
775
|
+
if (this.#clientDelegation && input) {
|
|
776
|
+
this.#runDelegation(
|
|
777
|
+
delegationId,
|
|
778
|
+
{
|
|
779
|
+
delegationId,
|
|
780
|
+
userText: String(input.text ?? ""),
|
|
781
|
+
userUtteranceIds: Array.isArray(input.userUtteranceIds)
|
|
782
|
+
? input.userUtteranceIds.filter((id: unknown): id is string => typeof id === "string")
|
|
783
|
+
: [],
|
|
784
|
+
messages: Array.isArray(input.messages) ? input.messages : [],
|
|
785
|
+
},
|
|
786
|
+
generation,
|
|
787
|
+
);
|
|
788
|
+
}
|
|
789
|
+
} else if (msg.speak !== false && this.#answered.has(delegationId)) {
|
|
790
|
+
// Every spoken phase (the approval ask, the late result) of an answer
|
|
791
|
+
// the chat shows is read back. A refusal of one that never started
|
|
792
|
+
// here has no chat answer: it renders. `final` may never come.
|
|
793
|
+
this.#foldReadback = true;
|
|
794
|
+
}
|
|
795
|
+
this.#delegating = msg.type === "delegation_started";
|
|
796
|
+
if (!this.#isSpeaking && !this.#isCancelling()) {
|
|
797
|
+
this.#emitStatus(this.#delegating ? "processing" : "listening");
|
|
798
|
+
}
|
|
799
|
+
break;
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
case "delegation_cancelled":
|
|
803
|
+
// The server gave up on it: no more frames for it, and its approval
|
|
804
|
+
// timers stop. The chat turn still renders. Past its deadline (or for
|
|
805
|
+
// an unknown reason), a parked approval card is also declined, as the
|
|
806
|
+
// approval TTL would; the call ending or the voice model cancelling
|
|
807
|
+
// leaves the card usable in the chat.
|
|
808
|
+
this.#dropDelegation(
|
|
809
|
+
String(msg.delegationId),
|
|
810
|
+
msg.reason !== "session_ending" && msg.reason !== "provider_cancelled",
|
|
811
|
+
);
|
|
812
|
+
break;
|
|
813
|
+
|
|
814
|
+
case "warning":
|
|
815
|
+
// Non-fatal (an unknown or refused frame): the call goes on. A refused
|
|
816
|
+
// delegation frame means the server is done with that delegation.
|
|
817
|
+
console.warn(`[Persona voice] ${msg.code}: ${msg.message ?? ""}`, this.#callId ?? "");
|
|
818
|
+
if (msg.delegationId && (msg.code === "UNKNOWN_DELEGATION" || msg.code === "LATE_RESULT_LIMIT")) {
|
|
819
|
+
this.#dropDelegation(String(msg.delegationId));
|
|
820
|
+
}
|
|
821
|
+
break;
|
|
822
|
+
|
|
823
|
+
case "audio_clear":
|
|
824
|
+
// Barge-in (or our own cancel acknowledged): stop playback now.
|
|
825
|
+
this.#clearLocalPlayback();
|
|
826
|
+
this.#cancelledUntil = 0;
|
|
827
|
+
this.#emitStatus("listening");
|
|
828
|
+
break;
|
|
829
|
+
|
|
320
830
|
case "transcript_interim":
|
|
321
|
-
this
|
|
322
|
-
this
|
|
831
|
+
this.#emitStatus("listening");
|
|
832
|
+
this.#emitTranscript("user", msg.text ?? "", false);
|
|
323
833
|
break;
|
|
324
834
|
|
|
325
835
|
case "transcript_final": {
|
|
326
836
|
const role = msg.role === "assistant" ? "assistant" : "user";
|
|
327
837
|
// user final → agent is now thinking; assistant final → reply incoming.
|
|
328
|
-
this
|
|
329
|
-
this
|
|
838
|
+
this.#emitStatus(role === "user" ? "processing" : "speaking");
|
|
839
|
+
this.#emitTranscript(role, msg.text ?? "", true);
|
|
330
840
|
break;
|
|
331
841
|
}
|
|
332
842
|
|
|
333
843
|
case "audio_end":
|
|
334
|
-
if (this
|
|
335
|
-
this
|
|
844
|
+
if (this.#playback) {
|
|
845
|
+
this.#playback.markStreamEnd();
|
|
336
846
|
} else {
|
|
337
|
-
this
|
|
338
|
-
this
|
|
847
|
+
this.#isSpeaking = false;
|
|
848
|
+
this.#emitStatus("listening");
|
|
339
849
|
}
|
|
340
850
|
break;
|
|
341
851
|
|
|
342
852
|
case "metrics":
|
|
343
|
-
this
|
|
853
|
+
this.#emitMetrics({
|
|
344
854
|
llmMs: msg.llm_ms,
|
|
345
855
|
ttsMs: msg.tts_ms,
|
|
346
856
|
firstAudioMs: msg.first_audio_ms,
|
|
@@ -349,105 +859,287 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
349
859
|
break;
|
|
350
860
|
|
|
351
861
|
case "error":
|
|
352
|
-
this
|
|
353
|
-
this
|
|
862
|
+
this.#emitError(new Error(msg.error || "Voice error"));
|
|
863
|
+
this.#emitStatus("error");
|
|
354
864
|
break;
|
|
355
865
|
}
|
|
356
866
|
}
|
|
357
867
|
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
868
|
+
/** Send the call-start `context` frame (history + host context), if any. */
|
|
869
|
+
/**
|
|
870
|
+
* The call-start `context` text. History is read now, before any of this
|
|
871
|
+
* call's voice bubbles exist, so it never includes the utterance in progress.
|
|
872
|
+
*/
|
|
873
|
+
async #buildContextText(): Promise<string> {
|
|
874
|
+
const history = this.bridge?.getHistory() ?? [];
|
|
875
|
+
let extra = "";
|
|
876
|
+
try {
|
|
877
|
+
const source = this.config?.callContext;
|
|
878
|
+
extra = String((typeof source === "function" ? await source() : source) ?? "");
|
|
879
|
+
} catch {
|
|
880
|
+
// A failing host callback only loses its own part of the context.
|
|
881
|
+
}
|
|
882
|
+
return buildCallContext(history, extra);
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
/** Send the held call-start context, once: at the first final user transcript or delegation. */
|
|
886
|
+
#flushCallContext(generation: number): void {
|
|
887
|
+
const pending = this.#pendingContext;
|
|
888
|
+
if (!pending) return;
|
|
889
|
+
this.#pendingContext = null;
|
|
890
|
+
let dropped = false;
|
|
891
|
+
const done = pending.then((text) => {
|
|
892
|
+
const ws = this.#ws;
|
|
893
|
+
if (dropped || !text || generation !== this.#callGeneration || ws?.readyState !== WebSocket.OPEN) return;
|
|
894
|
+
ws.send(JSON.stringify({ type: "context", text }));
|
|
895
|
+
});
|
|
896
|
+
this.#contextSend = { done, drop: () => (dropped = true) };
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
/** Let a released context go out first; a hung host callback is dropped. */
|
|
900
|
+
async #awaitContextSend(): Promise<void> {
|
|
901
|
+
const send = this.#contextSend;
|
|
902
|
+
if (!send) return;
|
|
903
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
904
|
+
const timedOut = await Promise.race([
|
|
905
|
+
send.done.then(() => false),
|
|
906
|
+
new Promise<boolean>((resolve) => {
|
|
907
|
+
timer = setTimeout(() => resolve(true), CONTEXT_BEFORE_RESULT_TIMEOUT_MS);
|
|
908
|
+
}),
|
|
909
|
+
]);
|
|
910
|
+
clearTimeout(timer);
|
|
911
|
+
if (timedOut) send.drop();
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
/** No more frames for this delegation; its chat answer goes back to browser TTS. */
|
|
915
|
+
#dropDelegation(delegationId: string, expired = false): void {
|
|
916
|
+
this.#cancelledDelegations.add(delegationId);
|
|
917
|
+
this.bridge?.dropDelegation?.(delegationId, expired);
|
|
918
|
+
this.#followUps.get(delegationId)?.abort();
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
/** Run a delegated turn through the session bridge (one at a time) and answer it. */
|
|
922
|
+
#runDelegation(delegationId: string, request: VoiceDelegationRequest, generation: number): void {
|
|
923
|
+
const bridge = this.bridge!;
|
|
924
|
+
/** Send a frame for this delegation, unless the call or the delegation ended. */
|
|
925
|
+
const send = (type: "delegation_update" | "delegation_result", status: string, text: string) => {
|
|
926
|
+
const ws = this.#ws;
|
|
927
|
+
if (generation !== this.#callGeneration || ws?.readyState !== WebSocket.OPEN) return false;
|
|
928
|
+
if (this.#cancelledDelegations.has(delegationId)) return false;
|
|
929
|
+
ws.send(JSON.stringify({ type, delegationId, status, text }));
|
|
930
|
+
return true;
|
|
931
|
+
};
|
|
932
|
+
this.#delegations = this.#delegations.then(async () => {
|
|
933
|
+
// Cancelled while queued behind another turn: never start it.
|
|
934
|
+
if (generation !== this.#callGeneration || this.#cancelledDelegations.has(delegationId)) return;
|
|
935
|
+
const result: VoiceDelegationResult = await bridge
|
|
936
|
+
.runDelegatedTurn(request)
|
|
937
|
+
.catch(() => ({ status: "failed", text: "" }));
|
|
938
|
+
await this.#awaitContextSend();
|
|
939
|
+
// Parked on an approval: ask now (non-terminal), answer once the visitor
|
|
940
|
+
// decides. A server that can't take an update gets the ask as the result.
|
|
941
|
+
const parked = !!result.followUp;
|
|
942
|
+
const update = parked && this.#capabilities.has("delegation_update");
|
|
943
|
+
const sent = send(
|
|
944
|
+
update ? "delegation_update" : "delegation_result",
|
|
945
|
+
parked && !update ? "completed" : result.status,
|
|
946
|
+
result.text,
|
|
947
|
+
);
|
|
948
|
+
if (!sent) return;
|
|
949
|
+
if (result.status !== "failed") this.#answered.add(delegationId);
|
|
950
|
+
if (!result.followUp) return;
|
|
951
|
+
// The approval bookkeeping (expiry, supersede) runs either way.
|
|
952
|
+
const abort = new AbortController();
|
|
953
|
+
this.#followUps.set(delegationId, abort);
|
|
954
|
+
void result
|
|
955
|
+
.followUp({
|
|
956
|
+
signal: abort.signal,
|
|
957
|
+
// Under the server's 600 s deadline, which restarts at each update.
|
|
958
|
+
approvalTimeoutMs: this.config?.approvalTimeoutMs && Math.min(this.config.approvalTimeoutMs, APPROVAL_TIMEOUT_MAX_MS),
|
|
959
|
+
readBack: update,
|
|
960
|
+
// Another gated tool in the same turn: another (non-terminal) update.
|
|
961
|
+
onUpdate: (text) => send("delegation_update", "pending_approval", text),
|
|
962
|
+
})
|
|
963
|
+
.then((followUp) => {
|
|
964
|
+
this.#followUps.delete(delegationId);
|
|
965
|
+
// Hang-up and cancellation send nothing.
|
|
966
|
+
if (update && followUp && !abort.signal.aborted) send("delegation_result", followUp.status, followUp.text);
|
|
967
|
+
});
|
|
968
|
+
});
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
#handleAudioFrame(buf: ArrayBuffer, generation: number): void {
|
|
972
|
+
if (generation !== this.#callGeneration) return;
|
|
973
|
+
if (!this.#playback) return;
|
|
974
|
+
if (this.#isCancelling()) return;
|
|
361
975
|
const pcm = stripWavHeader(buf);
|
|
362
976
|
if (pcm.length === 0) return;
|
|
363
|
-
if (!this
|
|
364
|
-
this
|
|
365
|
-
this
|
|
977
|
+
if (!this.#isSpeaking) {
|
|
978
|
+
this.#isSpeaking = true;
|
|
979
|
+
this.#emitStatus("speaking");
|
|
980
|
+
}
|
|
981
|
+
this.#armPlaybackFinished();
|
|
982
|
+
this.#playback.enqueue(pcm);
|
|
983
|
+
if (this.#speechToSpeech) this.#scheduleContinuousDrain(pcm.length);
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
#isCancelling(): boolean {
|
|
987
|
+
return Date.now() < this.#cancelledUntil;
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
/** Register the (one-shot) engine drain callback if none is pending. */
|
|
991
|
+
#armPlaybackFinished(): void {
|
|
992
|
+
if (this.#finishedArmed || !this.#playback) return;
|
|
993
|
+
this.#finishedArmed = true;
|
|
994
|
+
const generation = this.#callGeneration;
|
|
995
|
+
this.#playback.onFinished(() => {
|
|
996
|
+
if (generation !== this.#callGeneration) return;
|
|
997
|
+
this.#finishedArmed = false;
|
|
998
|
+
this.#handlePlaybackDrained();
|
|
999
|
+
});
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
/**
|
|
1003
|
+
* Speech-to-speech replies never get an end-of-stream, so estimate when the
|
|
1004
|
+
* queued audio finishes from its duration and release "speaking" then. The
|
|
1005
|
+
* engine is never marked ended, so the next reply's audio plays through the
|
|
1006
|
+
* same stream untouched and an underrun is never mistaken for a reply end.
|
|
1007
|
+
*/
|
|
1008
|
+
#scheduleContinuousDrain(byteLength: number): void {
|
|
1009
|
+
const now = Date.now();
|
|
1010
|
+
// PCM16 @ 24 kHz: 48 bytes per millisecond.
|
|
1011
|
+
this.#playbackEndsAt = Math.max(now, this.#playbackEndsAt) + byteLength / 48;
|
|
1012
|
+
clearTimeout(this.#drainTimer);
|
|
1013
|
+
this.#drainTimer = setTimeout(
|
|
1014
|
+
() => this.#handlePlaybackDrained(),
|
|
1015
|
+
this.#playbackEndsAt - now + CONTINUOUS_DRAIN_GRACE_MS,
|
|
1016
|
+
);
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
#handlePlaybackDrained(): void {
|
|
1020
|
+
this.#isSpeaking = false;
|
|
1021
|
+
this.#playbackEndsAt = 0;
|
|
1022
|
+
// Reply drained: the call stays open, so return to listening (or to
|
|
1023
|
+
// processing while an agent delegation is still running).
|
|
1024
|
+
if (this.#ws && this.#ws.readyState === WebSocket.OPEN) {
|
|
1025
|
+
this.#emitStatus(this.#delegating ? "processing" : "listening");
|
|
366
1026
|
}
|
|
367
|
-
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
/** Drop queued audio locally and reset the speaking/drain bookkeeping. */
|
|
1030
|
+
#clearLocalPlayback(): void {
|
|
1031
|
+
this.#playback?.flush();
|
|
1032
|
+
// flush() discards registered callbacks: re-arm for the next reply.
|
|
1033
|
+
this.#finishedArmed = false;
|
|
1034
|
+
this.#armPlaybackFinished();
|
|
1035
|
+
this.#isSpeaking = false;
|
|
1036
|
+
this.#playbackEndsAt = 0;
|
|
1037
|
+
clearTimeout(this.#drainTimer);
|
|
368
1038
|
}
|
|
369
1039
|
|
|
370
1040
|
// --- Teardown --------------------------------------------------------------
|
|
371
1041
|
|
|
372
|
-
|
|
1042
|
+
#cleanup(): void {
|
|
373
1043
|
// Invalidate any in-flight async continuation / late frames first.
|
|
374
|
-
this
|
|
375
|
-
this
|
|
376
|
-
this
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
1044
|
+
this.#callGeneration += 1;
|
|
1045
|
+
this.#callLive = false;
|
|
1046
|
+
this.#isSpeaking = false;
|
|
1047
|
+
this.#releaseAttached();
|
|
1048
|
+
this.#speechToSpeech = false;
|
|
1049
|
+
this.#delegating = false;
|
|
1050
|
+
this.#clientDelegation = false;
|
|
1051
|
+
this.#capabilities = new Set();
|
|
1052
|
+
for (const abort of this.#followUps.values()) abort.abort();
|
|
1053
|
+
this.#followUps.clear();
|
|
1054
|
+
this.#cancelledDelegations.clear();
|
|
1055
|
+
this.#callId = undefined;
|
|
1056
|
+
this.#contextSent = false;
|
|
1057
|
+
this.#pendingContext = null;
|
|
1058
|
+
this.#contextSend = null;
|
|
1059
|
+
this.#delegations = Promise.resolve();
|
|
1060
|
+
this.#answered.clear();
|
|
1061
|
+
this.#foldReadback = false;
|
|
1062
|
+
this.#assistantTurns.clear();
|
|
1063
|
+
this.#foldedTurns.clear();
|
|
1064
|
+
this.#finishedArmed = false;
|
|
1065
|
+
this.#cancelledUntil = 0;
|
|
1066
|
+
this.#playbackEndsAt = 0;
|
|
1067
|
+
clearTimeout(this.#drainTimer);
|
|
1068
|
+
this.#preOpenAudio = [];
|
|
1069
|
+
this.#preOpenBytes = 0;
|
|
1070
|
+
|
|
1071
|
+
if (this.#processor) {
|
|
1072
|
+
this.#processor.onaudioprocess = null;
|
|
1073
|
+
this.#processor.disconnect();
|
|
1074
|
+
this.#processor = null;
|
|
382
1075
|
}
|
|
383
|
-
if (this
|
|
384
|
-
this
|
|
385
|
-
this
|
|
1076
|
+
if (this.#sourceNode) {
|
|
1077
|
+
this.#sourceNode.disconnect();
|
|
1078
|
+
this.#sourceNode = null;
|
|
386
1079
|
}
|
|
387
|
-
if (this
|
|
388
|
-
this
|
|
389
|
-
this
|
|
1080
|
+
if (this.#mediaStream) {
|
|
1081
|
+
this.#mediaStream.getTracks().forEach((t) => t.stop());
|
|
1082
|
+
this.#mediaStream = null;
|
|
390
1083
|
}
|
|
391
|
-
if (this
|
|
392
|
-
this
|
|
393
|
-
this
|
|
1084
|
+
if (this.#captureContext) {
|
|
1085
|
+
this.#captureContext.close().catch(() => {});
|
|
1086
|
+
this.#captureContext = null;
|
|
394
1087
|
}
|
|
395
|
-
if (this
|
|
396
|
-
void this
|
|
397
|
-
this
|
|
1088
|
+
if (this.#playback) {
|
|
1089
|
+
void this.#playback.destroy();
|
|
1090
|
+
this.#playback = null;
|
|
398
1091
|
}
|
|
399
|
-
if (this
|
|
400
|
-
this
|
|
1092
|
+
if (this.#ws) {
|
|
1093
|
+
this.#intentionalClose = true;
|
|
401
1094
|
try {
|
|
402
|
-
this
|
|
1095
|
+
this.#ws.close(1000, "client ended call");
|
|
403
1096
|
} catch {
|
|
404
1097
|
// ignore
|
|
405
1098
|
}
|
|
406
|
-
this
|
|
1099
|
+
this.#ws = null;
|
|
407
1100
|
}
|
|
408
1101
|
}
|
|
409
1102
|
|
|
410
1103
|
// --- Callback registration + emit -----------------------------------------
|
|
411
1104
|
|
|
412
1105
|
onResult(callback: (result: VoiceResult) => void): void {
|
|
413
|
-
this
|
|
1106
|
+
this.#resultCallbacks.push(callback);
|
|
414
1107
|
}
|
|
415
1108
|
|
|
416
1109
|
onError(callback: (error: Error) => void): void {
|
|
417
|
-
this
|
|
1110
|
+
this.#errorCallbacks.push(callback);
|
|
418
1111
|
}
|
|
419
1112
|
|
|
420
1113
|
onStatusChange(callback: (status: VoiceStatus) => void): void {
|
|
421
|
-
this
|
|
1114
|
+
this.#statusCallbacks.push(callback);
|
|
422
1115
|
}
|
|
423
1116
|
|
|
424
|
-
onTranscript(
|
|
425
|
-
callback
|
|
426
|
-
): void {
|
|
427
|
-
this.transcriptCallbacks.push(callback);
|
|
1117
|
+
onTranscript(callback: TranscriptCallback): void {
|
|
1118
|
+
this.#transcriptCallbacks.push(callback);
|
|
428
1119
|
}
|
|
429
1120
|
|
|
430
1121
|
onMetrics(callback: (metrics: VoiceMetrics) => void): void {
|
|
431
|
-
this
|
|
1122
|
+
this.#metricsCallbacks.push(callback);
|
|
432
1123
|
}
|
|
433
1124
|
|
|
434
|
-
|
|
435
|
-
this
|
|
1125
|
+
#emitStatus(status: VoiceStatus): void {
|
|
1126
|
+
this.#statusCallbacks.forEach((cb) => cb(status));
|
|
436
1127
|
}
|
|
437
1128
|
|
|
438
|
-
|
|
439
|
-
this
|
|
1129
|
+
#emitError(error: Error): void {
|
|
1130
|
+
this.#errorCallbacks.forEach((cb) => cb(error));
|
|
440
1131
|
}
|
|
441
1132
|
|
|
442
|
-
|
|
1133
|
+
#emitTranscript(
|
|
443
1134
|
role: "user" | "assistant",
|
|
444
1135
|
text: string,
|
|
445
1136
|
isFinal: boolean,
|
|
1137
|
+
metadata?: VoiceTranscriptMetadata,
|
|
446
1138
|
): void {
|
|
447
|
-
this
|
|
1139
|
+
this.#transcriptCallbacks.forEach((cb) => cb(role, text, isFinal, metadata));
|
|
448
1140
|
}
|
|
449
1141
|
|
|
450
|
-
|
|
451
|
-
this
|
|
1142
|
+
#emitMetrics(metrics: VoiceMetrics): void {
|
|
1143
|
+
this.#metricsCallbacks.forEach((cb) => cb(metrics));
|
|
452
1144
|
}
|
|
453
1145
|
}
|