@runtypelabs/persona 4.25.0 → 4.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/animations/glyph-cycle.d.cts +1 -1
- package/dist/animations/glyph-cycle.d.ts +1 -1
- package/dist/animations/{types-Db61aI9J.d.cts → types-BwKTZhg9.d.cts} +14 -0
- package/dist/animations/{types-Db61aI9J.d.ts → types-BwKTZhg9.d.ts} +14 -0
- package/dist/animations/wipe.d.cts +1 -1
- package/dist/animations/wipe.d.ts +1 -1
- package/dist/animations-extra.d.cts +14 -0
- package/dist/animations-extra.d.ts +14 -0
- package/dist/approval-ui-QAKWNFNM.js +1 -0
- package/dist/approval-ui.cjs +2 -2
- package/dist/approval-ui.d.cts +180 -8
- package/dist/approval-ui.d.ts +180 -8
- package/dist/approval-ui.js +2 -2
- package/dist/{artifacts-ui-IGB5YKLO.js → artifacts-ui-NJGMM27I.js} +1 -1
- package/dist/artifacts-ui.cjs +4 -4
- package/dist/artifacts-ui.d.cts +180 -8
- package/dist/artifacts-ui.d.ts +180 -8
- package/dist/artifacts-ui.js +5 -5
- package/dist/chunk-2ANTRKTV.js +1 -0
- package/dist/chunk-4HPTPUC6.js +1 -0
- package/dist/chunk-6V3MNEKB.js +1 -0
- package/dist/chunk-HGUCGM3K.js +9 -0
- package/dist/chunk-IO5VVUKP.js +3 -0
- package/dist/chunk-J4J3Y2P2.js +1 -0
- package/dist/chunk-K6HKIP5K.js +1 -0
- package/dist/chunk-K7NA7G7J.js +1 -0
- package/dist/chunk-MXUTS3LB.js +1 -0
- package/dist/chunk-PDPVORUF.js +1 -0
- package/dist/chunk-QPPJUSZH.js +1 -0
- package/dist/chunk-QRKGQPU7.js +1 -0
- package/dist/chunk-S6PBWE75.js +1 -0
- package/dist/chunk-T7QM7YMF.js +1 -0
- package/dist/chunk-TR2H475E.js +1 -0
- package/dist/chunk-UPO4GUFC.js +1 -0
- package/dist/chunk-ZC66NAIR.js +1 -0
- package/dist/client-history-YPEG45Q4.js +1 -0
- package/dist/client-history.js +1 -0
- package/dist/client-stream-42FOI3EC.js +4 -0
- package/dist/client-stream.js +5 -0
- package/dist/codegen.cjs +1 -1
- package/dist/codegen.js +1 -1
- package/dist/context-mentions-E7HJYWO6.js +161 -0
- package/dist/{context-mentions-inline-I2RDEBAK.js → context-mentions-inline-QVESLW55.js} +1 -1
- package/dist/context-mentions.cjs +7 -7
- package/dist/context-mentions.d.cts +180 -8
- package/dist/context-mentions.d.ts +180 -8
- package/dist/context-mentions.js +8 -8
- package/dist/event-stream-view-FOIJHFQO.js +1 -0
- package/dist/event-stream-view.cjs +1 -1
- package/dist/event-stream-view.d.cts +183 -31
- package/dist/event-stream-view.d.ts +183 -31
- package/dist/event-stream-view.js +1 -1
- package/dist/forms-ui.d.cts +221 -667
- package/dist/forms-ui.d.ts +221 -667
- package/dist/history-shell-QDQ44PSE.js +3 -0
- package/dist/history-shell.js +3 -0
- package/dist/{history-view-BD6OXLNA.js → history-view-NCFBI5GR.js} +2 -2
- package/dist/history-view.cjs +2 -2
- package/dist/history-view.d.cts +35 -1
- package/dist/history-view.d.ts +35 -1
- package/dist/history-view.js +2 -2
- package/dist/index.cjs +80 -74
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +227 -738
- package/dist/index.d.ts +227 -738
- package/dist/index.global.js +3 -215
- package/dist/index.global.js.map +1 -1
- package/dist/index.js +81 -75
- package/dist/index.js.map +1 -1
- package/dist/install.global.js +1 -1
- package/dist/install.global.js.map +1 -1
- package/dist/launcher.global.js +3 -67
- package/dist/launcher.global.js.map +1 -1
- package/dist/markdown-parsers.js +22 -22
- package/dist/runtype-tts-entry-NFBVWR7N.js +1 -0
- package/dist/runtype-tts.js +1 -1
- package/dist/session-actions-ISSKSI66.js +1 -0
- package/dist/session-actions.js +1 -0
- package/dist/session-reconnect.d.cts +180 -8
- package/dist/session-reconnect.d.ts +180 -8
- package/dist/smart-dom-reader.cjs +13 -13
- package/dist/smart-dom-reader.d.cts +180 -8
- package/dist/smart-dom-reader.d.ts +180 -8
- package/dist/smart-dom-reader.js +11 -11
- package/dist/theme-editor-preview.cjs +116 -98
- package/dist/theme-editor-preview.d.cts +182 -8
- package/dist/theme-editor-preview.d.ts +182 -8
- package/dist/theme-editor-preview.js +49 -54
- package/dist/theme-editor.cjs +2 -2
- package/dist/theme-editor.d.cts +181 -23
- package/dist/theme-editor.d.ts +181 -23
- package/dist/theme-editor.js +2 -2
- package/dist/theme-reference.d.cts +14 -0
- package/dist/theme-reference.d.ts +14 -0
- package/dist/ui-extras-entry-WAYFKVNW.js +3 -0
- package/dist/ui-extras.js +72 -0
- package/dist/voice-runtime-UOQOFAFY.js +17 -0
- package/dist/voice-runtime.cjs +17 -1
- package/dist/voice-runtime.d.cts +11650 -232
- package/dist/voice-runtime.d.ts +11650 -232
- package/dist/voice-runtime.js +17 -1
- package/dist/voice-worklet-player.cjs +19 -4
- package/dist/voice-worklet-player.d.cts +9 -10
- package/dist/voice-worklet-player.d.ts +9 -10
- package/dist/voice-worklet-player.js +19 -4
- package/dist/webmcp-polyfill.js +1 -1
- package/dist/webmcp-runtime-V6ZWW3PI.js +6 -0
- package/dist/webmcp-runtime.cjs +2 -2
- package/dist/webmcp-runtime.d.cts +1 -33
- package/dist/webmcp-runtime.d.ts +1 -33
- package/dist/webmcp-runtime.js +2 -2
- package/dist/widget.css +1 -1
- package/package.json +11 -5
- package/src/approval-paused-tool-settle.test.ts +221 -0
- package/src/client-approve.test.ts +311 -0
- package/src/client-history-eager.ts +9 -0
- package/src/client-history-loader.ts +23 -0
- package/src/client-history.ts +579 -0
- package/src/client-resume.ts +177 -0
- package/src/client-stream-eager.ts +12 -0
- package/src/client-stream-loader.ts +25 -0
- package/src/client-stream.ts +1880 -0
- package/src/client.test.ts +31 -1
- package/src/client.ts +293 -2607
- package/src/components/approval-actions.test.ts +27 -0
- package/src/components/approval-actions.ts +27 -3
- package/src/components/message-bubble.test.ts +19 -0
- package/src/components/message-bubble.ts +2 -0
- package/src/components/registry.ts +6 -6
- package/src/components/tool-bubble.test.ts +9 -0
- package/src/components/tool-bubble.ts +1 -0
- package/src/history-shell-bundle.test.ts +54 -0
- package/src/history-shell-eager.ts +12 -0
- package/src/history-shell-loader.ts +27 -0
- package/src/history-shell.ts +2362 -0
- package/src/history-view-loader.ts +2 -0
- package/src/history-view.ts +3 -0
- package/src/index-core.ts +6 -0
- package/src/index-global.ts +90 -248
- package/src/index.ts +13 -0
- package/src/install.ts +6 -4
- package/src/internal/runtype-history-provider.test.ts +1 -0
- package/src/internal/runtype-history-provider.ts +28 -13
- package/src/plugins/registry.ts +8 -8
- package/src/session-actions-chunk.test.ts +55 -0
- package/src/session-actions-eager.ts +11 -0
- package/src/session-actions-loader.ts +37 -0
- package/src/session-actions.ts +909 -0
- package/src/session-history.test.ts +3 -1
- package/src/session.ts +852 -1536
- package/src/session.voice.test.ts +1170 -5
- package/src/styles/widget.css +58 -0
- package/src/theme-editor/state.ts +91 -91
- package/src/types.ts +198 -5
- package/src/ui-ask-user-sheet.ts +382 -0
- package/src/ui-extras-eager.ts +10 -0
- package/src/ui-extras-entry.ts +9 -0
- package/src/ui-extras-loader.ts +26 -0
- package/src/ui.artifact-rail-resize.test.ts +2 -1
- package/src/ui.ask-user-chunk.test.ts +93 -0
- package/src/ui.history-chunk.test.ts +2 -1
- package/src/ui.history-config-provider.test.ts +2 -1
- package/src/ui.history-confirm-fallback.test.ts +142 -0
- package/src/ui.history-controller.test.ts +2 -1
- package/src/ui.history-lifecycle.test.ts +2 -1
- package/src/ui.history-render-hooks.test.ts +2 -1
- package/src/ui.history-shell.test.ts +2 -1
- package/src/ui.rail-overlay.test.ts +2 -1
- package/src/ui.rail-resize.test.ts +2 -1
- package/src/ui.rail-trigger-pending.test.ts +2 -1
- package/src/ui.ts +340 -2770
- package/src/utils/attachment-manager.ts +67 -67
- package/src/utils/constants.ts +3 -0
- package/src/utils/context-mention-controller.ts +323 -323
- package/src/utils/context-mention-manager.ts +40 -40
- package/src/utils/context-mention-orchestrator.ts +28 -0
- package/src/utils/event-stream-buffer.ts +50 -50
- package/src/utils/event-stream-store.ts +43 -43
- package/src/utils/throughput-tracker.ts +34 -34
- package/src/utils/virtual-scroller.ts +67 -67
- package/src/utils/visitor-store.test.ts +50 -0
- package/src/utils/visitor-store.ts +23 -5
- package/src/voice/__fixtures__/voice-wire/README.md +1 -0
- package/src/voice/__fixtures__/voice-wire/client/cancel.json +3 -0
- package/src/voice/__fixtures__/voice-wire/client/context.json +4 -0
- package/src/voice/__fixtures__/voice-wire/client/delegation_result.json +6 -0
- package/src/voice/__fixtures__/voice-wire/client/delegation_update.json +6 -0
- package/src/voice/__fixtures__/voice-wire/client/ping.json +3 -0
- package/src/voice/__fixtures__/voice-wire/client/playback_progress.json +5 -0
- package/src/voice/__fixtures__/voice-wire/client/start.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/artifact.json +11 -0
- package/src/voice/__fixtures__/voice-wire/server/attached.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/audio_clear.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/audio_end.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_cancelled.json +5 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_completed.json +8 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_started.json +15 -0
- package/src/voice/__fixtures__/voice-wire/server/error.json +6 -0
- package/src/voice/__fixtures__/voice-wire/server/metrics.json +10 -0
- package/src/voice/__fixtures__/voice-wire/server/pong.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/session_config.json +16 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_final.json +7 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_interim.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_partial.json +6 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_update.json +10 -0
- package/src/voice/__fixtures__/voice-wire/server/warning.json +5 -0
- package/src/voice/audio-playback-manager.test.ts +57 -0
- package/src/voice/audio-playback-manager.ts +120 -98
- package/src/voice/browser-voice-provider.ts +37 -37
- package/src/voice/fallback-speech-engine.ts +22 -16
- package/src/voice/keyed-voice-transcript.ts +314 -0
- package/src/voice/mic-state-styles.ts +182 -0
- package/src/voice/read-aloud-controller.ts +46 -44
- package/src/voice/runtype-speech-engine.ts +35 -33
- package/src/voice/runtype-voice-provider.ts +670 -189
- package/src/voice/session-voice-wiring.ts +266 -0
- package/src/voice/voice-delegation.test.ts +71 -0
- package/src/voice/voice-delegation.ts +364 -0
- package/src/voice/voice-turn-tracker.ts +6 -6
- package/src/voice/voice.test.ts +981 -8
- package/src/voice/worklet-playback-engine.test.ts +81 -0
- package/src/voice/worklet-playback-engine.ts +21 -3
- package/src/voice-runtime-loader.ts +4 -1
- package/src/voice-runtime.ts +7 -0
- package/src/webmcp-runtime-entry.ts +41 -39
- package/dist/approval-ui-3NMYNXBD.js +0 -1
- package/dist/chunk-5EIIHQLQ.js +0 -1
- package/dist/chunk-7CO4T5KB.js +0 -1
- package/dist/chunk-EFJBUH5F.js +0 -1
- package/dist/chunk-R2TMIMDI.js +0 -3
- package/dist/chunk-UTVLOTWB.js +0 -9
- package/dist/context-mentions-45MODWOY.js +0 -161
- package/dist/event-stream-view-GL3ADRY2.js +0 -1
- package/dist/runtype-tts-entry-UJAEF7NZ.js +0 -1
- package/dist/voice-runtime-7DA7MJH4.js +0 -1
- package/dist/webmcp-runtime-JI7T4QYF.js +0 -6
|
@@ -8,6 +8,17 @@
|
|
|
8
8
|
// JSON control frames (transcript_interim / transcript_final /
|
|
9
9
|
// audio_end / metrics).
|
|
10
10
|
//
|
|
11
|
+
// Full duplex (speech-to-speech, e.g. GPT-Live): the client always declares
|
|
12
|
+
// `voiceCapabilities=full-duplex-v1`; engines that don't speak it ignore the
|
|
13
|
+
// param. A full-duplex session announces itself with
|
|
14
|
+
// `session_config{speechMode:'speech_to_speech'}` and then sends turn-keyed
|
|
15
|
+
// `transcript_update{role,text,turnId,final}` frames (later frames replace the
|
|
16
|
+
// text for the same (turnId, role); user and assistant turns can overlap),
|
|
17
|
+
// `delegation_started/completed` around agent tool turns, and `audio_clear` on
|
|
18
|
+
// barge-in. Reply audio has no per-turn end marker in that mode, so playback is
|
|
19
|
+
// treated as continuous: the "speaking" status is released by a timer derived
|
|
20
|
+
// from the queued audio duration rather than by the engine's end-of-stream.
|
|
21
|
+
//
|
|
11
22
|
// The server's STT owns turn-taking, so the client streams continuously and
|
|
12
23
|
// has no client-side VAD, barge-in monitoring, or batch upload. Auth rides the
|
|
13
24
|
// `Sec-WebSocket-Protocol` subprotocol (`['runtype.bearer', clientToken]`),
|
|
@@ -24,20 +35,60 @@
|
|
|
24
35
|
// socket idles (no call) until `startListening()` sends `{"type":"start"}`; if
|
|
25
36
|
// it negotiates plain `runtype.bearer`, it ignored attach and the call is
|
|
26
37
|
// already live, so `startListening()` adopts it as is.
|
|
38
|
+
//
|
|
39
|
+
// Wire vocabulary v1 (contract Amendment 5). Every socket sends
|
|
40
|
+
// `voiceProtocol=runtype-browser-v1`, `clientVersion=persona/<version>` and the
|
|
41
|
+
// snake_case `clientCapabilities` it supports. The server answers with
|
|
42
|
+
// `session_config.capabilities` (the negotiated intersection), the only gate
|
|
43
|
+
// for the client frames `context`, `delegation_update` and client delegation:
|
|
44
|
+
// a server without `capabilities` gets none of them. Receivers ignore unknown
|
|
45
|
+
// frames, fields and enum values. Non-fatal problems arrive as `warning`
|
|
46
|
+
// frames; an `error` always ends the call.
|
|
47
|
+
//
|
|
48
|
+
// Client delegation (full duplex, when the session supplied a bridge and
|
|
49
|
+
// `clientDelegation` isn't false, and `client_delegation` was negotiated): a
|
|
50
|
+
// `delegation_started{delegationId, input}` asks the widget to run the turn
|
|
51
|
+
// through its chat pipeline (one at a time). The provider answers with exactly
|
|
52
|
+
// one terminal `delegation_result{delegationId, status, text}`. A turn that
|
|
53
|
+
// parks on a tool approval first sends the non-terminal
|
|
54
|
+
// `delegation_update{delegationId, status:'pending_approval', text}`, then the
|
|
55
|
+
// terminal result once the visitor decides (or the approval lapses or is
|
|
56
|
+
// replaced). A second gated tool in the same turn sends another update.
|
|
57
|
+
// `delegation_cancelled{delegationId}` (or a `warning` refusing a frame for
|
|
58
|
+
// it) stops all frames for that id; the chat turn still renders, and since
|
|
59
|
+
// nobody speaks it now, browser TTS may read it. The voice model's spoken read-back after each
|
|
60
|
+
// `delegation_completed` of an answered delegation (the first new assistant
|
|
61
|
+
// utterance, unless the visitor speaks first) is folded: the chat already shows
|
|
62
|
+
// it. Transcripts in such a call are display-only captions, never sent to the
|
|
63
|
+
// agent as conversation; a user bubble becomes conversation only when it is
|
|
64
|
+
// submitted. With `context` negotiated, one `context{text}` frame (chat history
|
|
65
|
+
// as of call start plus the host's `callContext`) is held until the visitor's
|
|
66
|
+
// first final transcript or first delegation.
|
|
27
67
|
|
|
28
68
|
import type {
|
|
29
69
|
VoiceProvider,
|
|
30
70
|
VoiceResult,
|
|
31
71
|
VoiceStatus,
|
|
32
72
|
VoiceConfig,
|
|
73
|
+
VoiceDelegationRequest,
|
|
74
|
+
VoiceDelegationResult,
|
|
33
75
|
VoiceMetrics,
|
|
34
76
|
VoicePlaybackEngine,
|
|
77
|
+
VoiceSessionBridge,
|
|
78
|
+
VoiceTranscriptMetadata,
|
|
35
79
|
} from "../types";
|
|
36
80
|
import { AudioPlaybackManager } from "./audio-playback-manager";
|
|
81
|
+
import { VERSION } from "../version";
|
|
37
82
|
|
|
38
83
|
const CAPTURE_SAMPLE_RATE = 16000;
|
|
39
84
|
const PLAYBACK_SAMPLE_RATE = 24000;
|
|
40
85
|
const CAPTURE_BUFFER_SIZE = 4096;
|
|
86
|
+
/**
|
|
87
|
+
* Mic audio held while the call socket handshakes (a slow upgrade can take
|
|
88
|
+
* seconds, and the visitor is already talking): 8 s of 16-bit PCM, 256 KB,
|
|
89
|
+
* well under the server's startup queue. Past it the oldest frames go.
|
|
90
|
+
*/
|
|
91
|
+
const PRE_OPEN_AUDIO_MAX_BYTES = CAPTURE_SAMPLE_RATE * 2 * 8;
|
|
41
92
|
/**
|
|
42
93
|
* RMS-to-0..1 gain for the published capture level. Conversational speech sits
|
|
43
94
|
* around 0.05 to 0.3 RMS, so the raw value would never leave the bottom of the
|
|
@@ -68,6 +119,70 @@ type AttachedSocket = {
|
|
|
68
119
|
openedAt: number;
|
|
69
120
|
idleTimer: ReturnType<typeof setTimeout> | undefined;
|
|
70
121
|
};
|
|
122
|
+
/** Declared on every call; non-full-duplex engines ignore it. */
|
|
123
|
+
const FULL_DUPLEX_CAPABILITY = "full-duplex-v1";
|
|
124
|
+
/**
|
|
125
|
+
* Slack added to the computed end of queued speech-to-speech audio before the
|
|
126
|
+
* provider reports the reply as drained (covers engine prebuffer + scheduling).
|
|
127
|
+
*/
|
|
128
|
+
const CONTINUOUS_DRAIN_GRACE_MS = 300;
|
|
129
|
+
/**
|
|
130
|
+
* How long a client `cancel` waits for the server's `audio_clear` before audio
|
|
131
|
+
* is accepted again, so a lost acknowledgement can't mute the call for good.
|
|
132
|
+
*/
|
|
133
|
+
const CANCEL_ACK_TIMEOUT_MS = 2000;
|
|
134
|
+
/**
|
|
135
|
+
* How long a `delegation_result` waits for the call context still being built
|
|
136
|
+
* (a slow host `callContext`), so context always precedes the result. Past it,
|
|
137
|
+
* the result goes out and that context is dropped.
|
|
138
|
+
*/
|
|
139
|
+
const CONTEXT_BEFORE_RESULT_TIMEOUT_MS = 2000;
|
|
140
|
+
const VOICE_PROTOCOL = "runtype-browser-v1";
|
|
141
|
+
/** `approvalTimeoutMs` cap: below core's delegation deadline (600 s, restarted by each update). */
|
|
142
|
+
const APPROVAL_TIMEOUT_MAX_MS = 540_000;
|
|
143
|
+
const DISCLOSURE_TEXT = "You're talking to an AI assistant. Voice is processed by OpenAI.";
|
|
144
|
+
/** Call-start context frame: total cap, host share, history window, per-message cap. */
|
|
145
|
+
const CONTEXT_MAX_CHARS = 8000;
|
|
146
|
+
const CONTEXT_HOST_MAX_CHARS = 4000;
|
|
147
|
+
const CONTEXT_MESSAGES = 12;
|
|
148
|
+
const CONTEXT_MESSAGE_MAX_CHARS = 2000;
|
|
149
|
+
|
|
150
|
+
type TranscriptCallback = (
|
|
151
|
+
role: "user" | "assistant",
|
|
152
|
+
text: string,
|
|
153
|
+
isFinal: boolean,
|
|
154
|
+
metadata?: VoiceTranscriptMetadata,
|
|
155
|
+
) => void;
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* The call-start `context` text: "Conversation so far:" plus the last
|
|
159
|
+
* {@link CONTEXT_MESSAGES} messages (newest kept when over budget), then the
|
|
160
|
+
* host's extra context (at most {@link CONTEXT_HOST_MAX_CHARS}, so history
|
|
161
|
+
* always fits). Capped at {@link CONTEXT_MAX_CHARS}; "" when empty.
|
|
162
|
+
*/
|
|
163
|
+
export function buildCallContext(
|
|
164
|
+
history: Array<{ role: "user" | "assistant"; content: string }>,
|
|
165
|
+
extra: string,
|
|
166
|
+
): string {
|
|
167
|
+
const header = "Conversation so far:";
|
|
168
|
+
const tail = extra.slice(0, CONTEXT_HOST_MAX_CHARS).trim();
|
|
169
|
+
let budget = CONTEXT_MAX_CHARS - (tail ? tail.length + 2 : 0) - header.length;
|
|
170
|
+
const lines: string[] = [];
|
|
171
|
+
for (const message of history.slice(-CONTEXT_MESSAGES).reverse()) {
|
|
172
|
+
let text = message.content.replace(/\s+/g, " ").trim();
|
|
173
|
+
if (!text) continue;
|
|
174
|
+
if (text.length > CONTEXT_MESSAGE_MAX_CHARS) {
|
|
175
|
+
text = `${text.slice(0, CONTEXT_MESSAGE_MAX_CHARS - 1)}…`;
|
|
176
|
+
}
|
|
177
|
+
const line = `${message.role === "user" ? "User" : "Assistant"}: ${text}`;
|
|
178
|
+
if (line.length + 1 > budget) break;
|
|
179
|
+
budget -= line.length + 1;
|
|
180
|
+
lines.unshift(line);
|
|
181
|
+
}
|
|
182
|
+
const parts = lines.length > 0 ? [`${header}\n${lines.join("\n")}`] : [];
|
|
183
|
+
if (tail) parts.push(tail);
|
|
184
|
+
return parts.join("\n\n");
|
|
185
|
+
}
|
|
71
186
|
|
|
72
187
|
/**
|
|
73
188
|
* Strip the canonical 44-byte WAV header (if present) and return the raw PCM16
|
|
@@ -107,40 +222,72 @@ function clampAttachIdleMs(ms: number | undefined): number {
|
|
|
107
222
|
export class RuntypeVoiceProvider implements VoiceProvider {
|
|
108
223
|
type: "runtype" = "runtype";
|
|
109
224
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
225
|
+
#ws: WebSocket | null = null;
|
|
226
|
+
#captureContext: AudioContext | null = null;
|
|
227
|
+
#levelCallbacks: ((level: number) => void)[] = [];
|
|
228
|
+
#mediaStream: MediaStream | null = null;
|
|
229
|
+
#sourceNode: MediaStreamAudioSourceNode | null = null;
|
|
230
|
+
#processor: ScriptProcessorNode | null = null;
|
|
231
|
+
// Captured before the call socket opened; flushed in order on open.
|
|
232
|
+
#preOpenAudio: ArrayBuffer[] = [];
|
|
233
|
+
#preOpenBytes = 0;
|
|
234
|
+
#playback: VoicePlaybackEngine | null = null;
|
|
117
235
|
|
|
118
236
|
// True while a call (WS session) is live: drives the idempotent start guard
|
|
119
237
|
// and `isBargeInActive()`.
|
|
120
|
-
|
|
121
|
-
|
|
238
|
+
#callLive = false;
|
|
239
|
+
#isSpeaking = false;
|
|
122
240
|
|
|
123
241
|
// Invalidates in-flight async work (playback-engine creation, late frames,
|
|
124
242
|
// status transitions) after a teardown/restart so a stale callback can't act
|
|
125
243
|
// on a newer call's resources. Bumped on every start and every cleanup.
|
|
126
|
-
|
|
244
|
+
#callGeneration = 0;
|
|
127
245
|
|
|
128
246
|
// Distinguishes a user-initiated close (code 1000) from a dropped connection.
|
|
129
|
-
|
|
247
|
+
#intentionalClose = false;
|
|
130
248
|
|
|
131
249
|
// Prewarm latch and the (at most one) socket an `'attach'` prewarm opened.
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
250
|
+
#lastPrewarmAt = Number.NEGATIVE_INFINITY;
|
|
251
|
+
#attached: AttachedSocket | null = null;
|
|
252
|
+
// Full-duplex (speech-to-speech) session state; reset on every cleanup.
|
|
253
|
+
#speechToSpeech = false;
|
|
254
|
+
#delegating = false;
|
|
255
|
+
// Until this time, audio and assistant text belong to a reply the client cancelled.
|
|
256
|
+
#cancelledUntil = 0;
|
|
257
|
+
#playbackEndsAt = 0;
|
|
258
|
+
#drainTimer: ReturnType<typeof setTimeout> | undefined;
|
|
259
|
+
// Engine `onFinished` callbacks are one-shot (cleared on fire and on flush),
|
|
260
|
+
// so the provider re-registers before each reply that may need it.
|
|
261
|
+
#finishedArmed = false;
|
|
262
|
+
// Client delegation (see header); per-call state reset on every cleanup.
|
|
263
|
+
private bridge: VoiceSessionBridge | null = null;
|
|
264
|
+
#clientDelegation = false;
|
|
265
|
+
// session_config.capabilities: the negotiated client frame types.
|
|
266
|
+
#capabilities = new Set<string>();
|
|
267
|
+
// Per delegation: aborts its pending approval follow-up (cancelled, or call end).
|
|
268
|
+
#followUps = new Map<string, AbortController>();
|
|
269
|
+
// Delegations the server cancelled (or refused a frame for): no more frames for them.
|
|
270
|
+
#cancelledDelegations = new Set<string>();
|
|
271
|
+
// session_config.callId: the server's id for this call, for logs.
|
|
272
|
+
#callId: string | undefined;
|
|
273
|
+
// Call-start context, built at session_config and held until the visitor's
|
|
274
|
+
// first final transcript (sent earlier, the voice model tends to answer it).
|
|
275
|
+
#contextSent = false;
|
|
276
|
+
#pendingContext: Promise<string> | null = null;
|
|
277
|
+
// The released context's send; settles once it went out (or never will).
|
|
278
|
+
#contextSend: { done: Promise<void>; drop: () => void } | null = null;
|
|
279
|
+
#delegations: Promise<void> = Promise.resolve();
|
|
280
|
+
// Delegations answered ok: their spoken read-back is folded.
|
|
281
|
+
#answered = new Set<string>();
|
|
282
|
+
#foldReadback = false;
|
|
283
|
+
#assistantTurns = new Set<string>();
|
|
284
|
+
#foldedTurns = new Set<string>();
|
|
285
|
+
|
|
286
|
+
#resultCallbacks: ((result: VoiceResult) => void)[] = [];
|
|
287
|
+
#errorCallbacks: ((error: Error) => void)[] = [];
|
|
288
|
+
#statusCallbacks: ((status: VoiceStatus) => void)[] = [];
|
|
289
|
+
#transcriptCallbacks: TranscriptCallback[] = [];
|
|
290
|
+
#metricsCallbacks: ((metrics: VoiceMetrics) => void)[] = [];
|
|
144
291
|
|
|
145
292
|
constructor(private config: VoiceConfig["runtype"]) {}
|
|
146
293
|
|
|
@@ -149,31 +296,35 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
149
296
|
/** No-op: the WS session opens lazily in `startListening` (the "call"). */
|
|
150
297
|
async connect(): Promise<void> {}
|
|
151
298
|
|
|
299
|
+
setSessionBridge(bridge: VoiceSessionBridge): void {
|
|
300
|
+
this.bridge = bridge;
|
|
301
|
+
}
|
|
302
|
+
|
|
152
303
|
/**
|
|
153
304
|
* Warm the voice path ahead of the click. Fire-and-forget and throttled to
|
|
154
305
|
* once per {@link PREWARM_THROTTLE_MS}; never emits an error or a status.
|
|
155
306
|
*/
|
|
156
307
|
prewarm(): void {
|
|
157
|
-
if (this
|
|
308
|
+
if (this.#callLive || this.#attached) return;
|
|
158
309
|
const agentId = this.config?.agentId;
|
|
159
310
|
const token = this.config?.clientToken;
|
|
160
311
|
const host = this.config?.host;
|
|
161
312
|
if (!agentId || !token || !host) return;
|
|
162
313
|
const now = Date.now();
|
|
163
|
-
if (now - this
|
|
164
|
-
this
|
|
314
|
+
if (now - this.#lastPrewarmAt < PREWARM_THROTTLE_MS) return;
|
|
315
|
+
this.#lastPrewarmAt = now;
|
|
165
316
|
try {
|
|
166
317
|
if (this.config?.prewarmMode === "attach") {
|
|
167
|
-
this
|
|
318
|
+
this.#attach(host, agentId, token);
|
|
168
319
|
} else {
|
|
169
|
-
this
|
|
320
|
+
this.#requestPrewarm(host, agentId, token);
|
|
170
321
|
}
|
|
171
322
|
} catch {
|
|
172
323
|
// A prewarm must never surface an error; the click connects normally.
|
|
173
324
|
}
|
|
174
325
|
}
|
|
175
326
|
|
|
176
|
-
|
|
327
|
+
#requestPrewarm(host: string, agentId: string, token: string): void {
|
|
177
328
|
if (typeof fetch !== "function") return;
|
|
178
329
|
const url = `${toHttpBase(host)}/v1/client/agents/${encodeURIComponent(agentId)}/voice/prewarm`;
|
|
179
330
|
void fetch(url, {
|
|
@@ -183,12 +334,12 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
183
334
|
}).catch(() => {});
|
|
184
335
|
}
|
|
185
336
|
|
|
186
|
-
|
|
187
|
-
const params = new URLSearchParams(
|
|
337
|
+
#attach(host: string, agentId: string, token: string): void {
|
|
338
|
+
const params = new URLSearchParams();
|
|
188
339
|
const configuredIdleMs = this.config?.attachIdleMs;
|
|
189
340
|
const idleMs = clampAttachIdleMs(configuredIdleMs);
|
|
190
341
|
if (configuredIdleMs !== undefined) params.set("attachIdleMs", String(idleMs));
|
|
191
|
-
const ws = new WebSocket(
|
|
342
|
+
const ws = new WebSocket(this.#voiceSocketUrl(host, agentId, params), [
|
|
192
343
|
"runtype.bearer",
|
|
193
344
|
"runtype.attach",
|
|
194
345
|
token,
|
|
@@ -206,18 +357,18 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
206
357
|
openedAt: 0,
|
|
207
358
|
idleTimer: undefined,
|
|
208
359
|
};
|
|
209
|
-
this
|
|
360
|
+
this.#attached = entry;
|
|
210
361
|
|
|
211
362
|
ws.onopen = () => {
|
|
212
|
-
if (this
|
|
363
|
+
if (this.#attached !== entry) return;
|
|
213
364
|
entry.state = ws.protocol === "runtype.attach" ? "attached" : "live";
|
|
214
365
|
entry.openedAt = Date.now();
|
|
215
|
-
this
|
|
366
|
+
this.#armAttachIdle(entry, idleMs);
|
|
216
367
|
if (entry.state === "attached") ws.send('{"type":"ping"}');
|
|
217
368
|
entry.settle();
|
|
218
369
|
};
|
|
219
370
|
ws.onmessage = (event) => {
|
|
220
|
-
if (this
|
|
371
|
+
if (this.#attached !== entry || typeof event.data !== "string") return;
|
|
221
372
|
let msg: { type?: unknown; idleMs?: unknown };
|
|
222
373
|
try {
|
|
223
374
|
msg = JSON.parse(event.data);
|
|
@@ -226,27 +377,27 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
226
377
|
}
|
|
227
378
|
// The server's idle window is authoritative; close no later than it does.
|
|
228
379
|
if (msg.type === "attached" && typeof msg.idleMs === "number") {
|
|
229
|
-
this
|
|
380
|
+
this.#armAttachIdle(entry, msg.idleMs - (Date.now() - entry.openedAt));
|
|
230
381
|
}
|
|
231
382
|
};
|
|
232
383
|
ws.onerror = () => {};
|
|
233
384
|
ws.onclose = () => {
|
|
234
|
-
if (this
|
|
385
|
+
if (this.#attached === entry) this.#releaseAttached();
|
|
235
386
|
};
|
|
236
387
|
}
|
|
237
388
|
|
|
238
|
-
|
|
389
|
+
#armAttachIdle(entry: AttachedSocket, ms: number): void {
|
|
239
390
|
clearTimeout(entry.idleTimer);
|
|
240
391
|
entry.idleTimer = setTimeout(() => {
|
|
241
|
-
if (this
|
|
392
|
+
if (this.#attached === entry) this.#releaseAttached();
|
|
242
393
|
}, Math.max(0, ms));
|
|
243
394
|
}
|
|
244
395
|
|
|
245
396
|
/** Close and forget the prewarmed socket, if any. Silent by design. */
|
|
246
|
-
|
|
247
|
-
const entry = this
|
|
397
|
+
#releaseAttached(): void {
|
|
398
|
+
const entry = this.#attached;
|
|
248
399
|
if (!entry) return;
|
|
249
|
-
this
|
|
400
|
+
this.#attached = null;
|
|
250
401
|
clearTimeout(entry.idleTimer);
|
|
251
402
|
const { ws } = entry;
|
|
252
403
|
ws.onopen = null;
|
|
@@ -269,8 +420,8 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
269
420
|
* Resolves `null` when there is none, it stalled, or it dropped, so the
|
|
270
421
|
* caller opens a fresh socket.
|
|
271
422
|
*/
|
|
272
|
-
|
|
273
|
-
const entry = this
|
|
423
|
+
async #takeAttachedSocket(): Promise<{ ws: WebSocket; live: boolean } | null> {
|
|
424
|
+
const entry = this.#attached;
|
|
274
425
|
if (!entry) return null;
|
|
275
426
|
if (entry.state === "connecting") {
|
|
276
427
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
@@ -282,23 +433,38 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
282
433
|
]);
|
|
283
434
|
clearTimeout(timer);
|
|
284
435
|
}
|
|
285
|
-
if (this
|
|
436
|
+
if (this.#attached !== entry) return null;
|
|
286
437
|
if (entry.state === "connecting" || entry.ws.readyState !== WebSocket.OPEN) {
|
|
287
|
-
this
|
|
438
|
+
this.#releaseAttached();
|
|
288
439
|
return null;
|
|
289
440
|
}
|
|
290
|
-
this
|
|
441
|
+
this.#attached = null;
|
|
291
442
|
clearTimeout(entry.idleTimer);
|
|
292
443
|
return { ws: entry.ws, live: entry.state === "live" };
|
|
293
444
|
}
|
|
294
445
|
|
|
295
|
-
|
|
296
|
-
|
|
446
|
+
/**
|
|
447
|
+
* The call socket URL. Every socket declares the full-duplex capability;
|
|
448
|
+
* `extra` adds params (e.g. an attach prewarm's). The token never goes in
|
|
449
|
+
* the URL, and these params are not secrets.
|
|
450
|
+
*/
|
|
451
|
+
#voiceSocketUrl(host: string, agentId: string, attach?: URLSearchParams): string {
|
|
452
|
+
const capabilities = ["partial_transcript", "context"];
|
|
453
|
+
if (this.bridge && this.config?.clientDelegation !== false) capabilities.push("client_delegation", "delegation_update");
|
|
454
|
+
if (attach) capabilities.unshift("attach");
|
|
455
|
+
const params = new URLSearchParams({
|
|
456
|
+
voiceProtocol: VOICE_PROTOCOL,
|
|
457
|
+
clientVersion: `persona/${VERSION}`,
|
|
458
|
+
voiceCapabilities: FULL_DUPLEX_CAPABILITY,
|
|
459
|
+
clientCapabilities: capabilities.join(","),
|
|
460
|
+
});
|
|
461
|
+
attach?.forEach((value, key) => params.set(key, value));
|
|
462
|
+
return `${toWsBase(host)}/ws/agents/${encodeURIComponent(agentId)}/voice?${params}`;
|
|
297
463
|
}
|
|
298
464
|
|
|
299
465
|
/** Start the call: acquire mic, open the WS, stream PCM until hang-up. */
|
|
300
466
|
async startListening(): Promise<void> {
|
|
301
|
-
if (this
|
|
467
|
+
if (this.#callLive) return; // idempotent: a call is already live
|
|
302
468
|
|
|
303
469
|
const agentId = this.config?.agentId;
|
|
304
470
|
const token = this.config?.clientToken;
|
|
@@ -307,9 +473,9 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
307
473
|
if (!token) throw new Error("Runtype voice requires a clientToken");
|
|
308
474
|
if (!host) throw new Error("Runtype voice requires a host (or widget apiUrl)");
|
|
309
475
|
|
|
310
|
-
const generation = ++this
|
|
311
|
-
this
|
|
312
|
-
this
|
|
476
|
+
const generation = ++this.#callGeneration;
|
|
477
|
+
this.#intentionalClose = false;
|
|
478
|
+
this.#callLive = true;
|
|
313
479
|
|
|
314
480
|
try {
|
|
315
481
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
@@ -319,11 +485,11 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
319
485
|
echoCancellation: true,
|
|
320
486
|
},
|
|
321
487
|
});
|
|
322
|
-
if (generation !== this
|
|
488
|
+
if (generation !== this.#callGeneration) {
|
|
323
489
|
stream.getTracks().forEach((t) => t.stop());
|
|
324
490
|
return;
|
|
325
491
|
}
|
|
326
|
-
this
|
|
492
|
+
this.#mediaStream = stream;
|
|
327
493
|
|
|
328
494
|
// Create + resume both contexts inside the click gesture (iOS autoplay).
|
|
329
495
|
const AudioCtx =
|
|
@@ -334,115 +500,124 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
334
500
|
if (captureContext.state === "suspended") {
|
|
335
501
|
await captureContext.resume().catch(() => {});
|
|
336
502
|
}
|
|
337
|
-
this
|
|
503
|
+
this.#captureContext = captureContext;
|
|
338
504
|
|
|
339
505
|
const engine = this.config?.createPlaybackEngine
|
|
340
506
|
? await this.config.createPlaybackEngine()
|
|
341
507
|
: new AudioPlaybackManager(PLAYBACK_SAMPLE_RATE);
|
|
342
|
-
if (generation !== this
|
|
508
|
+
if (generation !== this.#callGeneration) {
|
|
343
509
|
// Torn down while async work was in flight: free what we acquired.
|
|
344
510
|
void engine.destroy();
|
|
345
511
|
stream.getTracks().forEach((t) => t.stop());
|
|
346
512
|
captureContext.close().catch(() => {});
|
|
347
513
|
return;
|
|
348
514
|
}
|
|
349
|
-
this
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
// Reply drained: the call stays open, so return to listening.
|
|
354
|
-
if (this.ws && this.ws.readyState === WebSocket.OPEN) {
|
|
355
|
-
this.emitStatus("listening");
|
|
356
|
-
}
|
|
357
|
-
});
|
|
515
|
+
this.#playback = engine;
|
|
516
|
+
this.#armPlaybackFinished();
|
|
517
|
+
// Capture from now: frames buffer until the call socket is open.
|
|
518
|
+
this.#startCapture(captureContext, stream, generation);
|
|
358
519
|
|
|
359
|
-
const adopted = await this
|
|
360
|
-
if (generation !== this
|
|
520
|
+
const adopted = await this.#takeAttachedSocket();
|
|
521
|
+
if (generation !== this.#callGeneration) {
|
|
361
522
|
adopted?.ws.close(1000, "client ended call");
|
|
362
523
|
return;
|
|
363
524
|
}
|
|
364
525
|
if (adopted) {
|
|
365
526
|
const { ws } = adopted;
|
|
366
|
-
this
|
|
367
|
-
this
|
|
527
|
+
this.#ws = ws;
|
|
528
|
+
this.#bindCallSocket(ws, generation);
|
|
368
529
|
if (!adopted.live) ws.send('{"type":"start"}');
|
|
369
|
-
this
|
|
370
|
-
this
|
|
530
|
+
this.#flushPreOpenAudio(ws);
|
|
531
|
+
this.#emitStatus("listening");
|
|
371
532
|
return;
|
|
372
533
|
}
|
|
373
534
|
|
|
374
535
|
// Token rides the subprotocol; `runtype.bearer` is the marker the server
|
|
375
536
|
// echoes as the negotiated subprotocol (browsers fail the handshake if an
|
|
376
537
|
// offered subprotocol goes unanswered).
|
|
377
|
-
const ws = new WebSocket(this
|
|
538
|
+
const ws = new WebSocket(this.#voiceSocketUrl(host, agentId), ["runtype.bearer", token]);
|
|
378
539
|
ws.binaryType = "arraybuffer";
|
|
379
|
-
this
|
|
540
|
+
this.#ws = ws;
|
|
380
541
|
|
|
381
542
|
ws.onopen = () => {
|
|
382
|
-
if (generation !== this
|
|
383
|
-
this
|
|
384
|
-
this
|
|
543
|
+
if (generation !== this.#callGeneration) return;
|
|
544
|
+
this.#flushPreOpenAudio(ws);
|
|
545
|
+
this.#emitStatus("listening");
|
|
385
546
|
};
|
|
386
|
-
this
|
|
547
|
+
this.#bindCallSocket(ws, generation);
|
|
387
548
|
} catch (error) {
|
|
388
|
-
this
|
|
389
|
-
this
|
|
390
|
-
this
|
|
549
|
+
this.#cleanup();
|
|
550
|
+
this.#emitError(error as Error);
|
|
551
|
+
this.#emitStatus("error");
|
|
391
552
|
throw error;
|
|
392
553
|
}
|
|
393
554
|
}
|
|
394
555
|
|
|
395
556
|
/** Route a call socket's frames, errors, and close into this call. */
|
|
396
|
-
|
|
397
|
-
ws.onmessage = (event) => this
|
|
557
|
+
#bindCallSocket(ws: WebSocket, generation: number): void {
|
|
558
|
+
ws.onmessage = (event) => this.#handleMessage(event, generation);
|
|
398
559
|
|
|
399
560
|
ws.onerror = () => {
|
|
400
|
-
if (generation !== this
|
|
401
|
-
this
|
|
402
|
-
this
|
|
403
|
-
this
|
|
561
|
+
if (generation !== this.#callGeneration) return;
|
|
562
|
+
this.#emitError(new Error("Voice connection failed"));
|
|
563
|
+
this.#emitStatus("error");
|
|
564
|
+
this.#cleanup();
|
|
404
565
|
};
|
|
405
566
|
|
|
406
567
|
ws.onclose = (evt) => {
|
|
407
|
-
if (this
|
|
408
|
-
this
|
|
568
|
+
if (this.#intentionalClose) {
|
|
569
|
+
this.#intentionalClose = false;
|
|
409
570
|
return;
|
|
410
571
|
}
|
|
411
|
-
if (generation !== this
|
|
572
|
+
if (generation !== this.#callGeneration) return;
|
|
412
573
|
if (evt.code !== 1000) {
|
|
413
574
|
const codeMsg = evt.code ? ` (code ${evt.code})` : "";
|
|
414
|
-
this
|
|
415
|
-
this
|
|
575
|
+
this.#emitError(new Error(`Voice connection closed${codeMsg}`));
|
|
576
|
+
this.#emitStatus("error");
|
|
416
577
|
} else {
|
|
417
|
-
this
|
|
578
|
+
this.#emitStatus("idle");
|
|
418
579
|
}
|
|
419
|
-
this
|
|
580
|
+
this.#cleanup();
|
|
420
581
|
};
|
|
421
582
|
}
|
|
422
583
|
|
|
584
|
+
/** The AI-disclosure notice for a live speech-to-speech call (`disclosureText`; `false` hides it). */
|
|
585
|
+
getDisclosure(): string | null {
|
|
586
|
+
const text = this.config?.disclosureText;
|
|
587
|
+
return this.#callLive && this.#speechToSpeech && text !== false ? text || DISCLOSURE_TEXT : null;
|
|
588
|
+
}
|
|
589
|
+
|
|
423
590
|
/** End the call (hang up). */
|
|
424
591
|
async stopListening(): Promise<void> {
|
|
425
|
-
this
|
|
426
|
-
this
|
|
592
|
+
this.#cleanup();
|
|
593
|
+
this.#emitStatus("idle");
|
|
427
594
|
}
|
|
428
595
|
|
|
429
596
|
/** Tear down the call and drop all callbacks (used by `cleanupVoice`). */
|
|
430
597
|
async disconnect(): Promise<void> {
|
|
431
|
-
this
|
|
432
|
-
this
|
|
433
|
-
this
|
|
434
|
-
this
|
|
435
|
-
this
|
|
436
|
-
this
|
|
437
|
-
this
|
|
598
|
+
this.#cleanup();
|
|
599
|
+
this.#emitStatus("disconnected");
|
|
600
|
+
this.#resultCallbacks = [];
|
|
601
|
+
this.#errorCallbacks = [];
|
|
602
|
+
this.#statusCallbacks = [];
|
|
603
|
+
this.#transcriptCallbacks = [];
|
|
604
|
+
this.#metricsCallbacks = [];
|
|
438
605
|
}
|
|
439
606
|
|
|
440
|
-
/**
|
|
607
|
+
/**
|
|
608
|
+
* Stop the spoken reply without ending the call. In a speech-to-speech
|
|
609
|
+
* session this also asks the server to cancel the response; audio is dropped
|
|
610
|
+
* until its `audio_clear` acknowledgement arrives.
|
|
611
|
+
*/
|
|
441
612
|
stopPlayback(): void {
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
if (
|
|
445
|
-
this
|
|
613
|
+
this.#clearLocalPlayback();
|
|
614
|
+
const ws = this.#ws;
|
|
615
|
+
if (ws && ws.readyState === WebSocket.OPEN) {
|
|
616
|
+
if (this.#speechToSpeech) {
|
|
617
|
+
this.#cancelledUntil = Date.now() + CANCEL_ACK_TIMEOUT_MS;
|
|
618
|
+
ws.send('{"type":"cancel"}');
|
|
619
|
+
}
|
|
620
|
+
this.#emitStatus("listening");
|
|
446
621
|
}
|
|
447
622
|
}
|
|
448
623
|
|
|
@@ -455,35 +630,33 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
455
630
|
|
|
456
631
|
/** True while the call (hot mic) is live. */
|
|
457
632
|
isBargeInActive(): boolean {
|
|
458
|
-
return this
|
|
633
|
+
return this.#callLive;
|
|
459
634
|
}
|
|
460
635
|
|
|
461
636
|
/** "Hang up" the always-on mic. */
|
|
462
637
|
onLevel(callback: (level: number) => void): void {
|
|
463
|
-
this
|
|
638
|
+
this.#levelCallbacks.push(callback);
|
|
464
639
|
}
|
|
465
640
|
|
|
466
641
|
async deactivateBargeIn(): Promise<void> {
|
|
467
|
-
this
|
|
468
|
-
this
|
|
642
|
+
this.#cleanup();
|
|
643
|
+
this.#emitStatus("idle");
|
|
469
644
|
}
|
|
470
645
|
|
|
471
646
|
// --- Capture ---------------------------------------------------------------
|
|
472
647
|
|
|
473
|
-
|
|
474
|
-
context: AudioContext,
|
|
475
|
-
stream: MediaStream,
|
|
476
|
-
ws: WebSocket,
|
|
477
|
-
generation: number,
|
|
478
|
-
): void {
|
|
648
|
+
#startCapture(context: AudioContext, stream: MediaStream, generation: number): void {
|
|
479
649
|
const source = context.createMediaStreamSource(stream);
|
|
480
|
-
this
|
|
650
|
+
this.#sourceNode = source;
|
|
481
651
|
const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
|
|
482
|
-
this
|
|
652
|
+
this.#processor = processor;
|
|
483
653
|
|
|
484
654
|
processor.onaudioprocess = (e) => {
|
|
485
|
-
if (generation !== this
|
|
486
|
-
|
|
655
|
+
if (generation !== this.#callGeneration) return;
|
|
656
|
+
// No socket yet (adopting a prewarm) or still handshaking: buffer.
|
|
657
|
+
const ws = this.#ws;
|
|
658
|
+
const open = ws?.readyState === WebSocket.OPEN;
|
|
659
|
+
if (ws && !open && ws.readyState !== WebSocket.CONNECTING) return;
|
|
487
660
|
const input = e.inputBuffer.getChannelData(0);
|
|
488
661
|
const pcm16 = new Int16Array(input.length);
|
|
489
662
|
let sumSquares = 0;
|
|
@@ -492,14 +665,23 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
492
665
|
sumSquares += s * s;
|
|
493
666
|
pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
|
494
667
|
}
|
|
495
|
-
ws
|
|
668
|
+
if (ws && open) {
|
|
669
|
+
this.#flushPreOpenAudio(ws);
|
|
670
|
+
ws.send(pcm16.buffer);
|
|
671
|
+
} else {
|
|
672
|
+
this.#preOpenAudio.push(pcm16.buffer);
|
|
673
|
+
this.#preOpenBytes += pcm16.byteLength;
|
|
674
|
+
while (this.#preOpenBytes > PRE_OPEN_AUDIO_MAX_BYTES) {
|
|
675
|
+
this.#preOpenBytes -= this.#preOpenAudio.shift()!.byteLength;
|
|
676
|
+
}
|
|
677
|
+
}
|
|
496
678
|
// Amplitude comes free from the buffer we already walked: no analyser
|
|
497
679
|
// node, no second capture. RMS is scaled because speech rarely exceeds
|
|
498
680
|
// ~0.3 RMS, so raw values would sit near the bottom of the 0..1 range.
|
|
499
|
-
if (this
|
|
681
|
+
if (this.#levelCallbacks.length > 0) {
|
|
500
682
|
const rms = Math.sqrt(sumSquares / input.length);
|
|
501
683
|
const level = Math.max(0, Math.min(1, rms * LEVEL_RMS_SCALE));
|
|
502
|
-
for (const cb of this
|
|
684
|
+
for (const cb of this.#levelCallbacks) cb(level);
|
|
503
685
|
}
|
|
504
686
|
};
|
|
505
687
|
|
|
@@ -509,13 +691,19 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
509
691
|
processor.connect(context.destination);
|
|
510
692
|
}
|
|
511
693
|
|
|
694
|
+
#flushPreOpenAudio(ws: WebSocket): void {
|
|
695
|
+
for (const frame of this.#preOpenAudio) ws.send(frame);
|
|
696
|
+
this.#preOpenAudio = [];
|
|
697
|
+
this.#preOpenBytes = 0;
|
|
698
|
+
}
|
|
699
|
+
|
|
512
700
|
// --- Downstream ------------------------------------------------------------
|
|
513
701
|
|
|
514
|
-
|
|
515
|
-
if (generation !== this
|
|
702
|
+
#handleMessage(event: MessageEvent, generation: number): void {
|
|
703
|
+
if (generation !== this.#callGeneration) return;
|
|
516
704
|
|
|
517
705
|
if (event.data instanceof ArrayBuffer) {
|
|
518
|
-
this
|
|
706
|
+
this.#handleAudioFrame(event.data, generation);
|
|
519
707
|
return;
|
|
520
708
|
}
|
|
521
709
|
|
|
@@ -527,30 +715,142 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
527
715
|
}
|
|
528
716
|
|
|
529
717
|
switch (msg.type) {
|
|
718
|
+
case "session_config":
|
|
719
|
+
// The follow-up session_config carries only interruptionMode: keep the mode.
|
|
720
|
+
if (msg.speechMode) {
|
|
721
|
+
this.#speechToSpeech = msg.speechMode === "speech_to_speech";
|
|
722
|
+
this.#playback?.setContinuousMode?.(this.#speechToSpeech);
|
|
723
|
+
}
|
|
724
|
+
// The negotiated client frame types. A server without `capabilities`
|
|
725
|
+
// predates them: no client delegation, context or updates.
|
|
726
|
+
if (Array.isArray(msg.capabilities)) this.#capabilities = new Set(msg.capabilities);
|
|
727
|
+
if (typeof msg.callId === "string") this.#callId = msg.callId;
|
|
728
|
+
this.#clientDelegation = this.#speechToSpeech && this.#capabilities.has("client_delegation") && !!this.bridge;
|
|
729
|
+
// Speech-to-speech is known only now: let the UI show its disclosure.
|
|
730
|
+
if (this.#speechToSpeech && !this.#isSpeaking) this.#emitStatus("listening");
|
|
731
|
+
if (this.#capabilities.has("context") && !this.#contextSent) {
|
|
732
|
+
this.#contextSent = true;
|
|
733
|
+
this.#pendingContext = this.#buildContextText();
|
|
734
|
+
}
|
|
735
|
+
break;
|
|
736
|
+
|
|
737
|
+
case "transcript_update": {
|
|
738
|
+
const role = msg.role === "assistant" ? "assistant" : "user";
|
|
739
|
+
// A reply the client cancelled keeps streaming until the server clears it.
|
|
740
|
+
const utteranceId = msg.utteranceId ?? msg.turnId;
|
|
741
|
+
if (!utteranceId || (role === "assistant" && this.#isCancelling())) break;
|
|
742
|
+
const turnId = String(utteranceId);
|
|
743
|
+
if (role === "user") {
|
|
744
|
+
// Release on a final user transcript, not a partial: a mid-utterance
|
|
745
|
+
// context append makes the voice model answer early (or not delegate).
|
|
746
|
+
if (msg.final === true) this.#flushCallContext(generation);
|
|
747
|
+
this.#foldReadback = false;
|
|
748
|
+
} else {
|
|
749
|
+
// Read-back of a delegated result: core rotates the assistant id at
|
|
750
|
+
// completion, so it is the first new id after it. The chat renders it.
|
|
751
|
+
if (this.#foldReadback && !this.#assistantTurns.has(turnId)) {
|
|
752
|
+
this.#foldedTurns.add(turnId);
|
|
753
|
+
this.#foldReadback = false;
|
|
754
|
+
}
|
|
755
|
+
this.#assistantTurns.add(turnId);
|
|
756
|
+
if (this.#foldedTurns.has(turnId)) break;
|
|
757
|
+
}
|
|
758
|
+
this.#emitTranscript(role, msg.text ?? "", msg.final === true, {
|
|
759
|
+
turnId,
|
|
760
|
+
...(typeof msg.startMs === "number" && { startMs: msg.startMs }),
|
|
761
|
+
...(typeof msg.endMs === "number" && { endMs: msg.endMs }),
|
|
762
|
+
// The chat pipeline owns the conversation: speech is only captioned.
|
|
763
|
+
...(this.#clientDelegation && { caption: true }),
|
|
764
|
+
});
|
|
765
|
+
break;
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
case "delegation_started":
|
|
769
|
+
case "delegation_completed": {
|
|
770
|
+
const delegationId = String(msg.delegationId ?? msg.turnId);
|
|
771
|
+
if (msg.type === "delegation_started") {
|
|
772
|
+
this.#flushCallContext(generation);
|
|
773
|
+
// `input`: run this turn through the chat pipeline (client delegation).
|
|
774
|
+
const input = msg.input;
|
|
775
|
+
if (this.#clientDelegation && input) {
|
|
776
|
+
this.#runDelegation(
|
|
777
|
+
delegationId,
|
|
778
|
+
{
|
|
779
|
+
delegationId,
|
|
780
|
+
userText: String(input.text ?? ""),
|
|
781
|
+
userUtteranceIds: Array.isArray(input.userUtteranceIds)
|
|
782
|
+
? input.userUtteranceIds.filter((id: unknown): id is string => typeof id === "string")
|
|
783
|
+
: [],
|
|
784
|
+
messages: Array.isArray(input.messages) ? input.messages : [],
|
|
785
|
+
},
|
|
786
|
+
generation,
|
|
787
|
+
);
|
|
788
|
+
}
|
|
789
|
+
} else if (msg.speak !== false && this.#answered.has(delegationId)) {
|
|
790
|
+
// Every spoken phase (the approval ask, the late result) of an answer
|
|
791
|
+
// the chat shows is read back. A refusal of one that never started
|
|
792
|
+
// here has no chat answer: it renders. `final` may never come.
|
|
793
|
+
this.#foldReadback = true;
|
|
794
|
+
}
|
|
795
|
+
this.#delegating = msg.type === "delegation_started";
|
|
796
|
+
if (!this.#isSpeaking && !this.#isCancelling()) {
|
|
797
|
+
this.#emitStatus(this.#delegating ? "processing" : "listening");
|
|
798
|
+
}
|
|
799
|
+
break;
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
case "delegation_cancelled":
|
|
803
|
+
// The server gave up on it: no more frames for it, and its approval
|
|
804
|
+
// timers stop. The chat turn still renders. Past its deadline (or for
|
|
805
|
+
// an unknown reason), a parked approval card is also declined, as the
|
|
806
|
+
// approval TTL would; the call ending or the voice model cancelling
|
|
807
|
+
// leaves the card usable in the chat.
|
|
808
|
+
this.#dropDelegation(
|
|
809
|
+
String(msg.delegationId),
|
|
810
|
+
msg.reason !== "session_ending" && msg.reason !== "provider_cancelled",
|
|
811
|
+
);
|
|
812
|
+
break;
|
|
813
|
+
|
|
814
|
+
case "warning":
|
|
815
|
+
// Non-fatal (an unknown or refused frame): the call goes on. A refused
|
|
816
|
+
// delegation frame means the server is done with that delegation.
|
|
817
|
+
console.warn(`[Persona voice] ${msg.code}: ${msg.message ?? ""}`, this.#callId ?? "");
|
|
818
|
+
if (msg.delegationId && (msg.code === "UNKNOWN_DELEGATION" || msg.code === "LATE_RESULT_LIMIT")) {
|
|
819
|
+
this.#dropDelegation(String(msg.delegationId));
|
|
820
|
+
}
|
|
821
|
+
break;
|
|
822
|
+
|
|
823
|
+
case "audio_clear":
|
|
824
|
+
// Barge-in (or our own cancel acknowledged): stop playback now.
|
|
825
|
+
this.#clearLocalPlayback();
|
|
826
|
+
this.#cancelledUntil = 0;
|
|
827
|
+
this.#emitStatus("listening");
|
|
828
|
+
break;
|
|
829
|
+
|
|
530
830
|
case "transcript_interim":
|
|
531
|
-
this
|
|
532
|
-
this
|
|
831
|
+
this.#emitStatus("listening");
|
|
832
|
+
this.#emitTranscript("user", msg.text ?? "", false);
|
|
533
833
|
break;
|
|
534
834
|
|
|
535
835
|
case "transcript_final": {
|
|
536
836
|
const role = msg.role === "assistant" ? "assistant" : "user";
|
|
537
837
|
// user final → agent is now thinking; assistant final → reply incoming.
|
|
538
|
-
this
|
|
539
|
-
this
|
|
838
|
+
this.#emitStatus(role === "user" ? "processing" : "speaking");
|
|
839
|
+
this.#emitTranscript(role, msg.text ?? "", true);
|
|
540
840
|
break;
|
|
541
841
|
}
|
|
542
842
|
|
|
543
843
|
case "audio_end":
|
|
544
|
-
if (this
|
|
545
|
-
this
|
|
844
|
+
if (this.#playback) {
|
|
845
|
+
this.#playback.markStreamEnd();
|
|
546
846
|
} else {
|
|
547
|
-
this
|
|
548
|
-
this
|
|
847
|
+
this.#isSpeaking = false;
|
|
848
|
+
this.#emitStatus("listening");
|
|
549
849
|
}
|
|
550
850
|
break;
|
|
551
851
|
|
|
552
852
|
case "metrics":
|
|
553
|
-
this
|
|
853
|
+
this.#emitMetrics({
|
|
554
854
|
llmMs: msg.llm_ms,
|
|
555
855
|
ttsMs: msg.tts_ms,
|
|
556
856
|
firstAudioMs: msg.first_audio_ms,
|
|
@@ -559,106 +859,287 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
559
859
|
break;
|
|
560
860
|
|
|
561
861
|
case "error":
|
|
562
|
-
this
|
|
563
|
-
this
|
|
862
|
+
this.#emitError(new Error(msg.error || "Voice error"));
|
|
863
|
+
this.#emitStatus("error");
|
|
564
864
|
break;
|
|
565
865
|
}
|
|
566
866
|
}
|
|
567
867
|
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
868
|
+
/** Send the call-start `context` frame (history + host context), if any. */
|
|
869
|
+
/**
|
|
870
|
+
* The call-start `context` text. History is read now, before any of this
|
|
871
|
+
* call's voice bubbles exist, so it never includes the utterance in progress.
|
|
872
|
+
*/
|
|
873
|
+
async #buildContextText(): Promise<string> {
|
|
874
|
+
const history = this.bridge?.getHistory() ?? [];
|
|
875
|
+
let extra = "";
|
|
876
|
+
try {
|
|
877
|
+
const source = this.config?.callContext;
|
|
878
|
+
extra = String((typeof source === "function" ? await source() : source) ?? "");
|
|
879
|
+
} catch {
|
|
880
|
+
// A failing host callback only loses its own part of the context.
|
|
881
|
+
}
|
|
882
|
+
return buildCallContext(history, extra);
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
/** Send the held call-start context, once: at the first final user transcript or delegation. */
|
|
886
|
+
#flushCallContext(generation: number): void {
|
|
887
|
+
const pending = this.#pendingContext;
|
|
888
|
+
if (!pending) return;
|
|
889
|
+
this.#pendingContext = null;
|
|
890
|
+
let dropped = false;
|
|
891
|
+
const done = pending.then((text) => {
|
|
892
|
+
const ws = this.#ws;
|
|
893
|
+
if (dropped || !text || generation !== this.#callGeneration || ws?.readyState !== WebSocket.OPEN) return;
|
|
894
|
+
ws.send(JSON.stringify({ type: "context", text }));
|
|
895
|
+
});
|
|
896
|
+
this.#contextSend = { done, drop: () => (dropped = true) };
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
/** Let a released context go out first; a hung host callback is dropped. */
|
|
900
|
+
async #awaitContextSend(): Promise<void> {
|
|
901
|
+
const send = this.#contextSend;
|
|
902
|
+
if (!send) return;
|
|
903
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
904
|
+
const timedOut = await Promise.race([
|
|
905
|
+
send.done.then(() => false),
|
|
906
|
+
new Promise<boolean>((resolve) => {
|
|
907
|
+
timer = setTimeout(() => resolve(true), CONTEXT_BEFORE_RESULT_TIMEOUT_MS);
|
|
908
|
+
}),
|
|
909
|
+
]);
|
|
910
|
+
clearTimeout(timer);
|
|
911
|
+
if (timedOut) send.drop();
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
/** No more frames for this delegation; its chat answer goes back to browser TTS. */
|
|
915
|
+
#dropDelegation(delegationId: string, expired = false): void {
|
|
916
|
+
this.#cancelledDelegations.add(delegationId);
|
|
917
|
+
this.bridge?.dropDelegation?.(delegationId, expired);
|
|
918
|
+
this.#followUps.get(delegationId)?.abort();
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
/** Run a delegated turn through the session bridge (one at a time) and answer it. */
|
|
922
|
+
#runDelegation(delegationId: string, request: VoiceDelegationRequest, generation: number): void {
|
|
923
|
+
const bridge = this.bridge!;
|
|
924
|
+
/** Send a frame for this delegation, unless the call or the delegation ended. */
|
|
925
|
+
const send = (type: "delegation_update" | "delegation_result", status: string, text: string) => {
|
|
926
|
+
const ws = this.#ws;
|
|
927
|
+
if (generation !== this.#callGeneration || ws?.readyState !== WebSocket.OPEN) return false;
|
|
928
|
+
if (this.#cancelledDelegations.has(delegationId)) return false;
|
|
929
|
+
ws.send(JSON.stringify({ type, delegationId, status, text }));
|
|
930
|
+
return true;
|
|
931
|
+
};
|
|
932
|
+
this.#delegations = this.#delegations.then(async () => {
|
|
933
|
+
// Cancelled while queued behind another turn: never start it.
|
|
934
|
+
if (generation !== this.#callGeneration || this.#cancelledDelegations.has(delegationId)) return;
|
|
935
|
+
const result: VoiceDelegationResult = await bridge
|
|
936
|
+
.runDelegatedTurn(request)
|
|
937
|
+
.catch(() => ({ status: "failed", text: "" }));
|
|
938
|
+
await this.#awaitContextSend();
|
|
939
|
+
// Parked on an approval: ask now (non-terminal), answer once the visitor
|
|
940
|
+
// decides. A server that can't take an update gets the ask as the result.
|
|
941
|
+
const parked = !!result.followUp;
|
|
942
|
+
const update = parked && this.#capabilities.has("delegation_update");
|
|
943
|
+
const sent = send(
|
|
944
|
+
update ? "delegation_update" : "delegation_result",
|
|
945
|
+
parked && !update ? "completed" : result.status,
|
|
946
|
+
result.text,
|
|
947
|
+
);
|
|
948
|
+
if (!sent) return;
|
|
949
|
+
if (result.status !== "failed") this.#answered.add(delegationId);
|
|
950
|
+
if (!result.followUp) return;
|
|
951
|
+
// The approval bookkeeping (expiry, supersede) runs either way.
|
|
952
|
+
const abort = new AbortController();
|
|
953
|
+
this.#followUps.set(delegationId, abort);
|
|
954
|
+
void result
|
|
955
|
+
.followUp({
|
|
956
|
+
signal: abort.signal,
|
|
957
|
+
// Under the server's 600 s deadline, which restarts at each update.
|
|
958
|
+
approvalTimeoutMs: this.config?.approvalTimeoutMs && Math.min(this.config.approvalTimeoutMs, APPROVAL_TIMEOUT_MAX_MS),
|
|
959
|
+
readBack: update,
|
|
960
|
+
// Another gated tool in the same turn: another (non-terminal) update.
|
|
961
|
+
onUpdate: (text) => send("delegation_update", "pending_approval", text),
|
|
962
|
+
})
|
|
963
|
+
.then((followUp) => {
|
|
964
|
+
this.#followUps.delete(delegationId);
|
|
965
|
+
// Hang-up and cancellation send nothing.
|
|
966
|
+
if (update && followUp && !abort.signal.aborted) send("delegation_result", followUp.status, followUp.text);
|
|
967
|
+
});
|
|
968
|
+
});
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
#handleAudioFrame(buf: ArrayBuffer, generation: number): void {
|
|
972
|
+
if (generation !== this.#callGeneration) return;
|
|
973
|
+
if (!this.#playback) return;
|
|
974
|
+
if (this.#isCancelling()) return;
|
|
571
975
|
const pcm = stripWavHeader(buf);
|
|
572
976
|
if (pcm.length === 0) return;
|
|
573
|
-
if (!this
|
|
574
|
-
this
|
|
575
|
-
this
|
|
977
|
+
if (!this.#isSpeaking) {
|
|
978
|
+
this.#isSpeaking = true;
|
|
979
|
+
this.#emitStatus("speaking");
|
|
980
|
+
}
|
|
981
|
+
this.#armPlaybackFinished();
|
|
982
|
+
this.#playback.enqueue(pcm);
|
|
983
|
+
if (this.#speechToSpeech) this.#scheduleContinuousDrain(pcm.length);
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
#isCancelling(): boolean {
|
|
987
|
+
return Date.now() < this.#cancelledUntil;
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
/** Register the (one-shot) engine drain callback if none is pending. */
|
|
991
|
+
#armPlaybackFinished(): void {
|
|
992
|
+
if (this.#finishedArmed || !this.#playback) return;
|
|
993
|
+
this.#finishedArmed = true;
|
|
994
|
+
const generation = this.#callGeneration;
|
|
995
|
+
this.#playback.onFinished(() => {
|
|
996
|
+
if (generation !== this.#callGeneration) return;
|
|
997
|
+
this.#finishedArmed = false;
|
|
998
|
+
this.#handlePlaybackDrained();
|
|
999
|
+
});
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
/**
|
|
1003
|
+
* Speech-to-speech replies never get an end-of-stream, so estimate when the
|
|
1004
|
+
* queued audio finishes from its duration and release "speaking" then. The
|
|
1005
|
+
* engine is never marked ended, so the next reply's audio plays through the
|
|
1006
|
+
* same stream untouched and an underrun is never mistaken for a reply end.
|
|
1007
|
+
*/
|
|
1008
|
+
#scheduleContinuousDrain(byteLength: number): void {
|
|
1009
|
+
const now = Date.now();
|
|
1010
|
+
// PCM16 @ 24 kHz: 48 bytes per millisecond.
|
|
1011
|
+
this.#playbackEndsAt = Math.max(now, this.#playbackEndsAt) + byteLength / 48;
|
|
1012
|
+
clearTimeout(this.#drainTimer);
|
|
1013
|
+
this.#drainTimer = setTimeout(
|
|
1014
|
+
() => this.#handlePlaybackDrained(),
|
|
1015
|
+
this.#playbackEndsAt - now + CONTINUOUS_DRAIN_GRACE_MS,
|
|
1016
|
+
);
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
#handlePlaybackDrained(): void {
|
|
1020
|
+
this.#isSpeaking = false;
|
|
1021
|
+
this.#playbackEndsAt = 0;
|
|
1022
|
+
// Reply drained: the call stays open, so return to listening (or to
|
|
1023
|
+
// processing while an agent delegation is still running).
|
|
1024
|
+
if (this.#ws && this.#ws.readyState === WebSocket.OPEN) {
|
|
1025
|
+
this.#emitStatus(this.#delegating ? "processing" : "listening");
|
|
576
1026
|
}
|
|
577
|
-
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
/** Drop queued audio locally and reset the speaking/drain bookkeeping. */
|
|
1030
|
+
#clearLocalPlayback(): void {
|
|
1031
|
+
this.#playback?.flush();
|
|
1032
|
+
// flush() discards registered callbacks: re-arm for the next reply.
|
|
1033
|
+
this.#finishedArmed = false;
|
|
1034
|
+
this.#armPlaybackFinished();
|
|
1035
|
+
this.#isSpeaking = false;
|
|
1036
|
+
this.#playbackEndsAt = 0;
|
|
1037
|
+
clearTimeout(this.#drainTimer);
|
|
578
1038
|
}
|
|
579
1039
|
|
|
580
1040
|
// --- Teardown --------------------------------------------------------------
|
|
581
1041
|
|
|
582
|
-
|
|
1042
|
+
#cleanup(): void {
|
|
583
1043
|
// Invalidate any in-flight async continuation / late frames first.
|
|
584
|
-
this
|
|
585
|
-
this
|
|
586
|
-
this
|
|
587
|
-
this
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
1044
|
+
this.#callGeneration += 1;
|
|
1045
|
+
this.#callLive = false;
|
|
1046
|
+
this.#isSpeaking = false;
|
|
1047
|
+
this.#releaseAttached();
|
|
1048
|
+
this.#speechToSpeech = false;
|
|
1049
|
+
this.#delegating = false;
|
|
1050
|
+
this.#clientDelegation = false;
|
|
1051
|
+
this.#capabilities = new Set();
|
|
1052
|
+
for (const abort of this.#followUps.values()) abort.abort();
|
|
1053
|
+
this.#followUps.clear();
|
|
1054
|
+
this.#cancelledDelegations.clear();
|
|
1055
|
+
this.#callId = undefined;
|
|
1056
|
+
this.#contextSent = false;
|
|
1057
|
+
this.#pendingContext = null;
|
|
1058
|
+
this.#contextSend = null;
|
|
1059
|
+
this.#delegations = Promise.resolve();
|
|
1060
|
+
this.#answered.clear();
|
|
1061
|
+
this.#foldReadback = false;
|
|
1062
|
+
this.#assistantTurns.clear();
|
|
1063
|
+
this.#foldedTurns.clear();
|
|
1064
|
+
this.#finishedArmed = false;
|
|
1065
|
+
this.#cancelledUntil = 0;
|
|
1066
|
+
this.#playbackEndsAt = 0;
|
|
1067
|
+
clearTimeout(this.#drainTimer);
|
|
1068
|
+
this.#preOpenAudio = [];
|
|
1069
|
+
this.#preOpenBytes = 0;
|
|
1070
|
+
|
|
1071
|
+
if (this.#processor) {
|
|
1072
|
+
this.#processor.onaudioprocess = null;
|
|
1073
|
+
this.#processor.disconnect();
|
|
1074
|
+
this.#processor = null;
|
|
593
1075
|
}
|
|
594
|
-
if (this
|
|
595
|
-
this
|
|
596
|
-
this
|
|
1076
|
+
if (this.#sourceNode) {
|
|
1077
|
+
this.#sourceNode.disconnect();
|
|
1078
|
+
this.#sourceNode = null;
|
|
597
1079
|
}
|
|
598
|
-
if (this
|
|
599
|
-
this
|
|
600
|
-
this
|
|
1080
|
+
if (this.#mediaStream) {
|
|
1081
|
+
this.#mediaStream.getTracks().forEach((t) => t.stop());
|
|
1082
|
+
this.#mediaStream = null;
|
|
601
1083
|
}
|
|
602
|
-
if (this
|
|
603
|
-
this
|
|
604
|
-
this
|
|
1084
|
+
if (this.#captureContext) {
|
|
1085
|
+
this.#captureContext.close().catch(() => {});
|
|
1086
|
+
this.#captureContext = null;
|
|
605
1087
|
}
|
|
606
|
-
if (this
|
|
607
|
-
void this
|
|
608
|
-
this
|
|
1088
|
+
if (this.#playback) {
|
|
1089
|
+
void this.#playback.destroy();
|
|
1090
|
+
this.#playback = null;
|
|
609
1091
|
}
|
|
610
|
-
if (this
|
|
611
|
-
this
|
|
1092
|
+
if (this.#ws) {
|
|
1093
|
+
this.#intentionalClose = true;
|
|
612
1094
|
try {
|
|
613
|
-
this
|
|
1095
|
+
this.#ws.close(1000, "client ended call");
|
|
614
1096
|
} catch {
|
|
615
1097
|
// ignore
|
|
616
1098
|
}
|
|
617
|
-
this
|
|
1099
|
+
this.#ws = null;
|
|
618
1100
|
}
|
|
619
1101
|
}
|
|
620
1102
|
|
|
621
1103
|
// --- Callback registration + emit -----------------------------------------
|
|
622
1104
|
|
|
623
1105
|
onResult(callback: (result: VoiceResult) => void): void {
|
|
624
|
-
this
|
|
1106
|
+
this.#resultCallbacks.push(callback);
|
|
625
1107
|
}
|
|
626
1108
|
|
|
627
1109
|
onError(callback: (error: Error) => void): void {
|
|
628
|
-
this
|
|
1110
|
+
this.#errorCallbacks.push(callback);
|
|
629
1111
|
}
|
|
630
1112
|
|
|
631
1113
|
onStatusChange(callback: (status: VoiceStatus) => void): void {
|
|
632
|
-
this
|
|
1114
|
+
this.#statusCallbacks.push(callback);
|
|
633
1115
|
}
|
|
634
1116
|
|
|
635
|
-
onTranscript(
|
|
636
|
-
callback
|
|
637
|
-
): void {
|
|
638
|
-
this.transcriptCallbacks.push(callback);
|
|
1117
|
+
onTranscript(callback: TranscriptCallback): void {
|
|
1118
|
+
this.#transcriptCallbacks.push(callback);
|
|
639
1119
|
}
|
|
640
1120
|
|
|
641
1121
|
onMetrics(callback: (metrics: VoiceMetrics) => void): void {
|
|
642
|
-
this
|
|
1122
|
+
this.#metricsCallbacks.push(callback);
|
|
643
1123
|
}
|
|
644
1124
|
|
|
645
|
-
|
|
646
|
-
this
|
|
1125
|
+
#emitStatus(status: VoiceStatus): void {
|
|
1126
|
+
this.#statusCallbacks.forEach((cb) => cb(status));
|
|
647
1127
|
}
|
|
648
1128
|
|
|
649
|
-
|
|
650
|
-
this
|
|
1129
|
+
#emitError(error: Error): void {
|
|
1130
|
+
this.#errorCallbacks.forEach((cb) => cb(error));
|
|
651
1131
|
}
|
|
652
1132
|
|
|
653
|
-
|
|
1133
|
+
#emitTranscript(
|
|
654
1134
|
role: "user" | "assistant",
|
|
655
1135
|
text: string,
|
|
656
1136
|
isFinal: boolean,
|
|
1137
|
+
metadata?: VoiceTranscriptMetadata,
|
|
657
1138
|
): void {
|
|
658
|
-
this
|
|
1139
|
+
this.#transcriptCallbacks.forEach((cb) => cb(role, text, isFinal, metadata));
|
|
659
1140
|
}
|
|
660
1141
|
|
|
661
|
-
|
|
662
|
-
this
|
|
1142
|
+
#emitMetrics(metrics: VoiceMetrics): void {
|
|
1143
|
+
this.#metricsCallbacks.forEach((cb) => cb(metrics));
|
|
663
1144
|
}
|
|
664
1145
|
}
|