@runtypelabs/persona 4.25.0 → 4.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (236) hide show
  1. package/dist/animations/glyph-cycle.d.cts +1 -1
  2. package/dist/animations/glyph-cycle.d.ts +1 -1
  3. package/dist/animations/{types-Db61aI9J.d.cts → types-BwKTZhg9.d.cts} +14 -0
  4. package/dist/animations/{types-Db61aI9J.d.ts → types-BwKTZhg9.d.ts} +14 -0
  5. package/dist/animations/wipe.d.cts +1 -1
  6. package/dist/animations/wipe.d.ts +1 -1
  7. package/dist/animations-extra.d.cts +14 -0
  8. package/dist/animations-extra.d.ts +14 -0
  9. package/dist/approval-ui-QAKWNFNM.js +1 -0
  10. package/dist/approval-ui.cjs +2 -2
  11. package/dist/approval-ui.d.cts +180 -8
  12. package/dist/approval-ui.d.ts +180 -8
  13. package/dist/approval-ui.js +2 -2
  14. package/dist/{artifacts-ui-IGB5YKLO.js → artifacts-ui-NJGMM27I.js} +1 -1
  15. package/dist/artifacts-ui.cjs +4 -4
  16. package/dist/artifacts-ui.d.cts +180 -8
  17. package/dist/artifacts-ui.d.ts +180 -8
  18. package/dist/artifacts-ui.js +5 -5
  19. package/dist/chunk-2ANTRKTV.js +1 -0
  20. package/dist/chunk-4HPTPUC6.js +1 -0
  21. package/dist/chunk-6V3MNEKB.js +1 -0
  22. package/dist/chunk-HGUCGM3K.js +9 -0
  23. package/dist/chunk-IO5VVUKP.js +3 -0
  24. package/dist/chunk-J4J3Y2P2.js +1 -0
  25. package/dist/chunk-K6HKIP5K.js +1 -0
  26. package/dist/chunk-K7NA7G7J.js +1 -0
  27. package/dist/chunk-MXUTS3LB.js +1 -0
  28. package/dist/chunk-PDPVORUF.js +1 -0
  29. package/dist/chunk-QPPJUSZH.js +1 -0
  30. package/dist/chunk-QRKGQPU7.js +1 -0
  31. package/dist/chunk-S6PBWE75.js +1 -0
  32. package/dist/chunk-T7QM7YMF.js +1 -0
  33. package/dist/chunk-TR2H475E.js +1 -0
  34. package/dist/chunk-UPO4GUFC.js +1 -0
  35. package/dist/chunk-ZC66NAIR.js +1 -0
  36. package/dist/client-history-YPEG45Q4.js +1 -0
  37. package/dist/client-history.js +1 -0
  38. package/dist/client-stream-42FOI3EC.js +4 -0
  39. package/dist/client-stream.js +5 -0
  40. package/dist/codegen.cjs +1 -1
  41. package/dist/codegen.js +1 -1
  42. package/dist/context-mentions-E7HJYWO6.js +161 -0
  43. package/dist/{context-mentions-inline-I2RDEBAK.js → context-mentions-inline-QVESLW55.js} +1 -1
  44. package/dist/context-mentions.cjs +7 -7
  45. package/dist/context-mentions.d.cts +180 -8
  46. package/dist/context-mentions.d.ts +180 -8
  47. package/dist/context-mentions.js +8 -8
  48. package/dist/event-stream-view-FOIJHFQO.js +1 -0
  49. package/dist/event-stream-view.cjs +1 -1
  50. package/dist/event-stream-view.d.cts +183 -31
  51. package/dist/event-stream-view.d.ts +183 -31
  52. package/dist/event-stream-view.js +1 -1
  53. package/dist/forms-ui.d.cts +221 -667
  54. package/dist/forms-ui.d.ts +221 -667
  55. package/dist/history-shell-QDQ44PSE.js +3 -0
  56. package/dist/history-shell.js +3 -0
  57. package/dist/{history-view-BD6OXLNA.js → history-view-NCFBI5GR.js} +2 -2
  58. package/dist/history-view.cjs +2 -2
  59. package/dist/history-view.d.cts +35 -1
  60. package/dist/history-view.d.ts +35 -1
  61. package/dist/history-view.js +2 -2
  62. package/dist/index.cjs +80 -74
  63. package/dist/index.cjs.map +1 -1
  64. package/dist/index.d.cts +227 -738
  65. package/dist/index.d.ts +227 -738
  66. package/dist/index.global.js +3 -215
  67. package/dist/index.global.js.map +1 -1
  68. package/dist/index.js +81 -75
  69. package/dist/index.js.map +1 -1
  70. package/dist/install.global.js +1 -1
  71. package/dist/install.global.js.map +1 -1
  72. package/dist/launcher.global.js +3 -67
  73. package/dist/launcher.global.js.map +1 -1
  74. package/dist/markdown-parsers.js +22 -22
  75. package/dist/runtype-tts-entry-NFBVWR7N.js +1 -0
  76. package/dist/runtype-tts.js +1 -1
  77. package/dist/session-actions-ISSKSI66.js +1 -0
  78. package/dist/session-actions.js +1 -0
  79. package/dist/session-reconnect.d.cts +180 -8
  80. package/dist/session-reconnect.d.ts +180 -8
  81. package/dist/smart-dom-reader.cjs +13 -13
  82. package/dist/smart-dom-reader.d.cts +180 -8
  83. package/dist/smart-dom-reader.d.ts +180 -8
  84. package/dist/smart-dom-reader.js +11 -11
  85. package/dist/theme-editor-preview.cjs +116 -98
  86. package/dist/theme-editor-preview.d.cts +182 -8
  87. package/dist/theme-editor-preview.d.ts +182 -8
  88. package/dist/theme-editor-preview.js +49 -54
  89. package/dist/theme-editor.cjs +2 -2
  90. package/dist/theme-editor.d.cts +181 -23
  91. package/dist/theme-editor.d.ts +181 -23
  92. package/dist/theme-editor.js +2 -2
  93. package/dist/theme-reference.d.cts +14 -0
  94. package/dist/theme-reference.d.ts +14 -0
  95. package/dist/ui-extras-entry-WAYFKVNW.js +3 -0
  96. package/dist/ui-extras.js +72 -0
  97. package/dist/voice-runtime-UOQOFAFY.js +17 -0
  98. package/dist/voice-runtime.cjs +17 -1
  99. package/dist/voice-runtime.d.cts +11650 -232
  100. package/dist/voice-runtime.d.ts +11650 -232
  101. package/dist/voice-runtime.js +17 -1
  102. package/dist/voice-worklet-player.cjs +19 -4
  103. package/dist/voice-worklet-player.d.cts +9 -10
  104. package/dist/voice-worklet-player.d.ts +9 -10
  105. package/dist/voice-worklet-player.js +19 -4
  106. package/dist/webmcp-polyfill.js +1 -1
  107. package/dist/webmcp-runtime-V6ZWW3PI.js +6 -0
  108. package/dist/webmcp-runtime.cjs +2 -2
  109. package/dist/webmcp-runtime.d.cts +1 -33
  110. package/dist/webmcp-runtime.d.ts +1 -33
  111. package/dist/webmcp-runtime.js +2 -2
  112. package/dist/widget.css +1 -1
  113. package/package.json +11 -5
  114. package/src/approval-paused-tool-settle.test.ts +221 -0
  115. package/src/client-approve.test.ts +311 -0
  116. package/src/client-history-eager.ts +9 -0
  117. package/src/client-history-loader.ts +23 -0
  118. package/src/client-history.ts +579 -0
  119. package/src/client-resume.ts +177 -0
  120. package/src/client-stream-eager.ts +12 -0
  121. package/src/client-stream-loader.ts +25 -0
  122. package/src/client-stream.ts +1880 -0
  123. package/src/client.test.ts +31 -1
  124. package/src/client.ts +293 -2607
  125. package/src/components/approval-actions.test.ts +27 -0
  126. package/src/components/approval-actions.ts +27 -3
  127. package/src/components/message-bubble.test.ts +19 -0
  128. package/src/components/message-bubble.ts +2 -0
  129. package/src/components/registry.ts +6 -6
  130. package/src/components/tool-bubble.test.ts +9 -0
  131. package/src/components/tool-bubble.ts +1 -0
  132. package/src/history-shell-bundle.test.ts +54 -0
  133. package/src/history-shell-eager.ts +12 -0
  134. package/src/history-shell-loader.ts +27 -0
  135. package/src/history-shell.ts +2362 -0
  136. package/src/history-view-loader.ts +2 -0
  137. package/src/history-view.ts +3 -0
  138. package/src/index-core.ts +6 -0
  139. package/src/index-global.ts +90 -248
  140. package/src/index.ts +13 -0
  141. package/src/install.ts +6 -4
  142. package/src/internal/runtype-history-provider.test.ts +1 -0
  143. package/src/internal/runtype-history-provider.ts +28 -13
  144. package/src/plugins/registry.ts +8 -8
  145. package/src/session-actions-chunk.test.ts +55 -0
  146. package/src/session-actions-eager.ts +11 -0
  147. package/src/session-actions-loader.ts +37 -0
  148. package/src/session-actions.ts +909 -0
  149. package/src/session-history.test.ts +3 -1
  150. package/src/session.ts +852 -1536
  151. package/src/session.voice.test.ts +1170 -5
  152. package/src/styles/widget.css +58 -0
  153. package/src/theme-editor/state.ts +91 -91
  154. package/src/types.ts +198 -5
  155. package/src/ui-ask-user-sheet.ts +382 -0
  156. package/src/ui-extras-eager.ts +10 -0
  157. package/src/ui-extras-entry.ts +9 -0
  158. package/src/ui-extras-loader.ts +26 -0
  159. package/src/ui.artifact-rail-resize.test.ts +2 -1
  160. package/src/ui.ask-user-chunk.test.ts +93 -0
  161. package/src/ui.history-chunk.test.ts +2 -1
  162. package/src/ui.history-config-provider.test.ts +2 -1
  163. package/src/ui.history-confirm-fallback.test.ts +142 -0
  164. package/src/ui.history-controller.test.ts +2 -1
  165. package/src/ui.history-lifecycle.test.ts +2 -1
  166. package/src/ui.history-render-hooks.test.ts +2 -1
  167. package/src/ui.history-shell.test.ts +2 -1
  168. package/src/ui.rail-overlay.test.ts +2 -1
  169. package/src/ui.rail-resize.test.ts +2 -1
  170. package/src/ui.rail-trigger-pending.test.ts +2 -1
  171. package/src/ui.ts +340 -2770
  172. package/src/utils/attachment-manager.ts +67 -67
  173. package/src/utils/constants.ts +3 -0
  174. package/src/utils/context-mention-controller.ts +323 -323
  175. package/src/utils/context-mention-manager.ts +40 -40
  176. package/src/utils/context-mention-orchestrator.ts +28 -0
  177. package/src/utils/event-stream-buffer.ts +50 -50
  178. package/src/utils/event-stream-store.ts +43 -43
  179. package/src/utils/throughput-tracker.ts +34 -34
  180. package/src/utils/virtual-scroller.ts +67 -67
  181. package/src/utils/visitor-store.test.ts +50 -0
  182. package/src/utils/visitor-store.ts +23 -5
  183. package/src/voice/__fixtures__/voice-wire/README.md +1 -0
  184. package/src/voice/__fixtures__/voice-wire/client/cancel.json +3 -0
  185. package/src/voice/__fixtures__/voice-wire/client/context.json +4 -0
  186. package/src/voice/__fixtures__/voice-wire/client/delegation_result.json +6 -0
  187. package/src/voice/__fixtures__/voice-wire/client/delegation_update.json +6 -0
  188. package/src/voice/__fixtures__/voice-wire/client/ping.json +3 -0
  189. package/src/voice/__fixtures__/voice-wire/client/playback_progress.json +5 -0
  190. package/src/voice/__fixtures__/voice-wire/client/start.json +3 -0
  191. package/src/voice/__fixtures__/voice-wire/server/artifact.json +11 -0
  192. package/src/voice/__fixtures__/voice-wire/server/attached.json +4 -0
  193. package/src/voice/__fixtures__/voice-wire/server/audio_clear.json +4 -0
  194. package/src/voice/__fixtures__/voice-wire/server/audio_end.json +3 -0
  195. package/src/voice/__fixtures__/voice-wire/server/delegation_cancelled.json +5 -0
  196. package/src/voice/__fixtures__/voice-wire/server/delegation_completed.json +8 -0
  197. package/src/voice/__fixtures__/voice-wire/server/delegation_started.json +15 -0
  198. package/src/voice/__fixtures__/voice-wire/server/error.json +6 -0
  199. package/src/voice/__fixtures__/voice-wire/server/metrics.json +10 -0
  200. package/src/voice/__fixtures__/voice-wire/server/pong.json +3 -0
  201. package/src/voice/__fixtures__/voice-wire/server/session_config.json +16 -0
  202. package/src/voice/__fixtures__/voice-wire/server/transcript_final.json +7 -0
  203. package/src/voice/__fixtures__/voice-wire/server/transcript_interim.json +4 -0
  204. package/src/voice/__fixtures__/voice-wire/server/transcript_partial.json +6 -0
  205. package/src/voice/__fixtures__/voice-wire/server/transcript_update.json +10 -0
  206. package/src/voice/__fixtures__/voice-wire/server/warning.json +5 -0
  207. package/src/voice/audio-playback-manager.test.ts +57 -0
  208. package/src/voice/audio-playback-manager.ts +120 -98
  209. package/src/voice/browser-voice-provider.ts +37 -37
  210. package/src/voice/fallback-speech-engine.ts +22 -16
  211. package/src/voice/keyed-voice-transcript.ts +314 -0
  212. package/src/voice/mic-state-styles.ts +182 -0
  213. package/src/voice/read-aloud-controller.ts +46 -44
  214. package/src/voice/runtype-speech-engine.ts +35 -33
  215. package/src/voice/runtype-voice-provider.ts +670 -189
  216. package/src/voice/session-voice-wiring.ts +266 -0
  217. package/src/voice/voice-delegation.test.ts +71 -0
  218. package/src/voice/voice-delegation.ts +364 -0
  219. package/src/voice/voice-turn-tracker.ts +6 -6
  220. package/src/voice/voice.test.ts +981 -8
  221. package/src/voice/worklet-playback-engine.test.ts +81 -0
  222. package/src/voice/worklet-playback-engine.ts +21 -3
  223. package/src/voice-runtime-loader.ts +4 -1
  224. package/src/voice-runtime.ts +7 -0
  225. package/src/webmcp-runtime-entry.ts +41 -39
  226. package/dist/approval-ui-3NMYNXBD.js +0 -1
  227. package/dist/chunk-5EIIHQLQ.js +0 -1
  228. package/dist/chunk-7CO4T5KB.js +0 -1
  229. package/dist/chunk-EFJBUH5F.js +0 -1
  230. package/dist/chunk-R2TMIMDI.js +0 -3
  231. package/dist/chunk-UTVLOTWB.js +0 -9
  232. package/dist/context-mentions-45MODWOY.js +0 -161
  233. package/dist/event-stream-view-GL3ADRY2.js +0 -1
  234. package/dist/runtype-tts-entry-UJAEF7NZ.js +0 -1
  235. package/dist/voice-runtime-7DA7MJH4.js +0 -1
  236. package/dist/webmcp-runtime-JI7T4QYF.js +0 -6
@@ -8,6 +8,17 @@
8
8
  // JSON control frames (transcript_interim / transcript_final /
9
9
  // audio_end / metrics).
10
10
  //
11
+ // Full duplex (speech-to-speech, e.g. GPT-Live): the client always declares
12
+ // `voiceCapabilities=full-duplex-v1`; engines that don't speak it ignore the
13
+ // param. A full-duplex session announces itself with
14
+ // `session_config{speechMode:'speech_to_speech'}` and then sends turn-keyed
15
+ // `transcript_update{role,text,turnId,final}` frames (later frames replace the
16
+ // text for the same (turnId, role); user and assistant turns can overlap),
17
+ // `delegation_started/completed` around agent tool turns, and `audio_clear` on
18
+ // barge-in. Reply audio has no per-turn end marker in that mode, so playback is
19
+ // treated as continuous: the "speaking" status is released by a timer derived
20
+ // from the queued audio duration rather than by the engine's end-of-stream.
21
+ //
11
22
  // The server's STT owns turn-taking, so the client streams continuously and
12
23
  // has no client-side VAD, barge-in monitoring, or batch upload. Auth rides the
13
24
  // `Sec-WebSocket-Protocol` subprotocol (`['runtype.bearer', clientToken]`),
@@ -24,20 +35,60 @@
24
35
  // socket idles (no call) until `startListening()` sends `{"type":"start"}`; if
25
36
  // it negotiates plain `runtype.bearer`, it ignored attach and the call is
26
37
  // already live, so `startListening()` adopts it as is.
38
+ //
39
+ // Wire vocabulary v1 (contract Amendment 5). Every socket sends
40
+ // `voiceProtocol=runtype-browser-v1`, `clientVersion=persona/<version>` and the
41
+ // snake_case `clientCapabilities` it supports. The server answers with
42
+ // `session_config.capabilities` (the negotiated intersection), the only gate
43
+ // for the client frames `context`, `delegation_update` and client delegation:
44
+ // a server without `capabilities` gets none of them. Receivers ignore unknown
45
+ // frames, fields and enum values. Non-fatal problems arrive as `warning`
46
+ // frames; an `error` always ends the call.
47
+ //
48
+ // Client delegation (full duplex, when the session supplied a bridge and
49
+ // `clientDelegation` isn't false, and `client_delegation` was negotiated): a
50
+ // `delegation_started{delegationId, input}` asks the widget to run the turn
51
+ // through its chat pipeline (one at a time). The provider answers with exactly
52
+ // one terminal `delegation_result{delegationId, status, text}`. A turn that
53
+ // parks on a tool approval first sends the non-terminal
54
+ // `delegation_update{delegationId, status:'pending_approval', text}`, then the
55
+ // terminal result once the visitor decides (or the approval lapses or is
56
+ // replaced). A second gated tool in the same turn sends another update.
57
+ // `delegation_cancelled{delegationId}` (or a `warning` refusing a frame for
58
+ // it) stops all frames for that id; the chat turn still renders, and since
59
+ // nobody speaks it now, browser TTS may read it. The voice model's spoken read-back after each
60
+ // `delegation_completed` of an answered delegation (the first new assistant
61
+ // utterance, unless the visitor speaks first) is folded: the chat already shows
62
+ // it. Transcripts in such a call are display-only captions, never sent to the
63
+ // agent as conversation; a user bubble becomes conversation only when it is
64
+ // submitted. With `context` negotiated, one `context{text}` frame (chat history
65
+ // as of call start plus the host's `callContext`) is held until the visitor's
66
+ // first final transcript or first delegation.
27
67
 
28
68
  import type {
29
69
  VoiceProvider,
30
70
  VoiceResult,
31
71
  VoiceStatus,
32
72
  VoiceConfig,
73
+ VoiceDelegationRequest,
74
+ VoiceDelegationResult,
33
75
  VoiceMetrics,
34
76
  VoicePlaybackEngine,
77
+ VoiceSessionBridge,
78
+ VoiceTranscriptMetadata,
35
79
  } from "../types";
36
80
  import { AudioPlaybackManager } from "./audio-playback-manager";
81
+ import { VERSION } from "../version";
37
82
 
38
83
  const CAPTURE_SAMPLE_RATE = 16000;
39
84
  const PLAYBACK_SAMPLE_RATE = 24000;
40
85
  const CAPTURE_BUFFER_SIZE = 4096;
86
+ /**
87
+ * Mic audio held while the call socket handshakes (a slow upgrade can take
88
+ * seconds, and the visitor is already talking): 8 s of 16-bit PCM, 256 KB,
89
+ * well under the server's startup queue. Past it the oldest frames go.
90
+ */
91
+ const PRE_OPEN_AUDIO_MAX_BYTES = CAPTURE_SAMPLE_RATE * 2 * 8;
41
92
  /**
42
93
  * RMS-to-0..1 gain for the published capture level. Conversational speech sits
43
94
  * around 0.05 to 0.3 RMS, so the raw value would never leave the bottom of the
@@ -68,6 +119,70 @@ type AttachedSocket = {
68
119
  openedAt: number;
69
120
  idleTimer: ReturnType<typeof setTimeout> | undefined;
70
121
  };
122
+ /** Declared on every call; non-full-duplex engines ignore it. */
123
+ const FULL_DUPLEX_CAPABILITY = "full-duplex-v1";
124
+ /**
125
+ * Slack added to the computed end of queued speech-to-speech audio before the
126
+ * provider reports the reply as drained (covers engine prebuffer + scheduling).
127
+ */
128
+ const CONTINUOUS_DRAIN_GRACE_MS = 300;
129
+ /**
130
+ * How long a client `cancel` waits for the server's `audio_clear` before audio
131
+ * is accepted again, so a lost acknowledgement can't mute the call for good.
132
+ */
133
+ const CANCEL_ACK_TIMEOUT_MS = 2000;
134
+ /**
135
+ * How long a `delegation_result` waits for the call context still being built
136
+ * (a slow host `callContext`), so context always precedes the result. Past it,
137
+ * the result goes out and that context is dropped.
138
+ */
139
+ const CONTEXT_BEFORE_RESULT_TIMEOUT_MS = 2000;
140
+ const VOICE_PROTOCOL = "runtype-browser-v1";
141
+ /** `approvalTimeoutMs` cap: below core's delegation deadline (600 s, restarted by each update). */
142
+ const APPROVAL_TIMEOUT_MAX_MS = 540_000;
143
+ const DISCLOSURE_TEXT = "You're talking to an AI assistant. Voice is processed by OpenAI.";
144
+ /** Call-start context frame: total cap, host share, history window, per-message cap. */
145
+ const CONTEXT_MAX_CHARS = 8000;
146
+ const CONTEXT_HOST_MAX_CHARS = 4000;
147
+ const CONTEXT_MESSAGES = 12;
148
+ const CONTEXT_MESSAGE_MAX_CHARS = 2000;
149
+
150
+ type TranscriptCallback = (
151
+ role: "user" | "assistant",
152
+ text: string,
153
+ isFinal: boolean,
154
+ metadata?: VoiceTranscriptMetadata,
155
+ ) => void;
156
+
157
+ /**
158
+ * The call-start `context` text: "Conversation so far:" plus the last
159
+ * {@link CONTEXT_MESSAGES} messages (newest kept when over budget), then the
160
+ * host's extra context (at most {@link CONTEXT_HOST_MAX_CHARS}, so history
161
+ * always fits). Capped at {@link CONTEXT_MAX_CHARS}; "" when empty.
162
+ */
163
+ export function buildCallContext(
164
+ history: Array<{ role: "user" | "assistant"; content: string }>,
165
+ extra: string,
166
+ ): string {
167
+ const header = "Conversation so far:";
168
+ const tail = extra.slice(0, CONTEXT_HOST_MAX_CHARS).trim();
169
+ let budget = CONTEXT_MAX_CHARS - (tail ? tail.length + 2 : 0) - header.length;
170
+ const lines: string[] = [];
171
+ for (const message of history.slice(-CONTEXT_MESSAGES).reverse()) {
172
+ let text = message.content.replace(/\s+/g, " ").trim();
173
+ if (!text) continue;
174
+ if (text.length > CONTEXT_MESSAGE_MAX_CHARS) {
175
+ text = `${text.slice(0, CONTEXT_MESSAGE_MAX_CHARS - 1)}…`;
176
+ }
177
+ const line = `${message.role === "user" ? "User" : "Assistant"}: ${text}`;
178
+ if (line.length + 1 > budget) break;
179
+ budget -= line.length + 1;
180
+ lines.unshift(line);
181
+ }
182
+ const parts = lines.length > 0 ? [`${header}\n${lines.join("\n")}`] : [];
183
+ if (tail) parts.push(tail);
184
+ return parts.join("\n\n");
185
+ }
71
186
 
72
187
  /**
73
188
  * Strip the canonical 44-byte WAV header (if present) and return the raw PCM16
@@ -107,40 +222,72 @@ function clampAttachIdleMs(ms: number | undefined): number {
107
222
  export class RuntypeVoiceProvider implements VoiceProvider {
108
223
  type: "runtype" = "runtype";
109
224
 
110
- private ws: WebSocket | null = null;
111
- private captureContext: AudioContext | null = null;
112
- private levelCallbacks: ((level: number) => void)[] = [];
113
- private mediaStream: MediaStream | null = null;
114
- private sourceNode: MediaStreamAudioSourceNode | null = null;
115
- private processor: ScriptProcessorNode | null = null;
116
- private playback: VoicePlaybackEngine | null = null;
225
+ #ws: WebSocket | null = null;
226
+ #captureContext: AudioContext | null = null;
227
+ #levelCallbacks: ((level: number) => void)[] = [];
228
+ #mediaStream: MediaStream | null = null;
229
+ #sourceNode: MediaStreamAudioSourceNode | null = null;
230
+ #processor: ScriptProcessorNode | null = null;
231
+ // Captured before the call socket opened; flushed in order on open.
232
+ #preOpenAudio: ArrayBuffer[] = [];
233
+ #preOpenBytes = 0;
234
+ #playback: VoicePlaybackEngine | null = null;
117
235
 
118
236
  // True while a call (WS session) is live: drives the idempotent start guard
119
237
  // and `isBargeInActive()`.
120
- private callLive = false;
121
- private isSpeaking = false;
238
+ #callLive = false;
239
+ #isSpeaking = false;
122
240
 
123
241
  // Invalidates in-flight async work (playback-engine creation, late frames,
124
242
  // status transitions) after a teardown/restart so a stale callback can't act
125
243
  // on a newer call's resources. Bumped on every start and every cleanup.
126
- private callGeneration = 0;
244
+ #callGeneration = 0;
127
245
 
128
246
  // Distinguishes a user-initiated close (code 1000) from a dropped connection.
129
- private intentionalClose = false;
247
+ #intentionalClose = false;
130
248
 
131
249
  // Prewarm latch and the (at most one) socket an `'attach'` prewarm opened.
132
- private lastPrewarmAt = Number.NEGATIVE_INFINITY;
133
- private attached: AttachedSocket | null = null;
134
-
135
- private resultCallbacks: ((result: VoiceResult) => void)[] = [];
136
- private errorCallbacks: ((error: Error) => void)[] = [];
137
- private statusCallbacks: ((status: VoiceStatus) => void)[] = [];
138
- private transcriptCallbacks: ((
139
- role: "user" | "assistant",
140
- text: string,
141
- isFinal: boolean,
142
- ) => void)[] = [];
143
- private metricsCallbacks: ((metrics: VoiceMetrics) => void)[] = [];
250
+ #lastPrewarmAt = Number.NEGATIVE_INFINITY;
251
+ #attached: AttachedSocket | null = null;
252
+ // Full-duplex (speech-to-speech) session state; reset on every cleanup.
253
+ #speechToSpeech = false;
254
+ #delegating = false;
255
+ // Until this time, audio and assistant text belong to a reply the client cancelled.
256
+ #cancelledUntil = 0;
257
+ #playbackEndsAt = 0;
258
+ #drainTimer: ReturnType<typeof setTimeout> | undefined;
259
+ // Engine `onFinished` callbacks are one-shot (cleared on fire and on flush),
260
+ // so the provider re-registers before each reply that may need it.
261
+ #finishedArmed = false;
262
+ // Client delegation (see header); per-call state reset on every cleanup.
263
+ private bridge: VoiceSessionBridge | null = null;
264
+ #clientDelegation = false;
265
+ // session_config.capabilities: the negotiated client frame types.
266
+ #capabilities = new Set<string>();
267
+ // Per delegation: aborts its pending approval follow-up (cancelled, or call end).
268
+ #followUps = new Map<string, AbortController>();
269
+ // Delegations the server cancelled (or refused a frame for): no more frames for them.
270
+ #cancelledDelegations = new Set<string>();
271
+ // session_config.callId: the server's id for this call, for logs.
272
+ #callId: string | undefined;
273
+ // Call-start context, built at session_config and held until the visitor's
274
+ // first final transcript (sent earlier, the voice model tends to answer it).
275
+ #contextSent = false;
276
+ #pendingContext: Promise<string> | null = null;
277
+ // The released context's send; settles once it went out (or never will).
278
+ #contextSend: { done: Promise<void>; drop: () => void } | null = null;
279
+ #delegations: Promise<void> = Promise.resolve();
280
+ // Delegations answered ok: their spoken read-back is folded.
281
+ #answered = new Set<string>();
282
+ #foldReadback = false;
283
+ #assistantTurns = new Set<string>();
284
+ #foldedTurns = new Set<string>();
285
+
286
+ #resultCallbacks: ((result: VoiceResult) => void)[] = [];
287
+ #errorCallbacks: ((error: Error) => void)[] = [];
288
+ #statusCallbacks: ((status: VoiceStatus) => void)[] = [];
289
+ #transcriptCallbacks: TranscriptCallback[] = [];
290
+ #metricsCallbacks: ((metrics: VoiceMetrics) => void)[] = [];
144
291
 
145
292
  constructor(private config: VoiceConfig["runtype"]) {}
146
293
 
@@ -149,31 +296,35 @@ export class RuntypeVoiceProvider implements VoiceProvider {
149
296
  /** No-op: the WS session opens lazily in `startListening` (the "call"). */
150
297
  async connect(): Promise<void> {}
151
298
 
299
+ setSessionBridge(bridge: VoiceSessionBridge): void {
300
+ this.bridge = bridge;
301
+ }
302
+
152
303
  /**
153
304
  * Warm the voice path ahead of the click. Fire-and-forget and throttled to
154
305
  * once per {@link PREWARM_THROTTLE_MS}; never emits an error or a status.
155
306
  */
156
307
  prewarm(): void {
157
- if (this.callLive || this.attached) return;
308
+ if (this.#callLive || this.#attached) return;
158
309
  const agentId = this.config?.agentId;
159
310
  const token = this.config?.clientToken;
160
311
  const host = this.config?.host;
161
312
  if (!agentId || !token || !host) return;
162
313
  const now = Date.now();
163
- if (now - this.lastPrewarmAt < PREWARM_THROTTLE_MS) return;
164
- this.lastPrewarmAt = now;
314
+ if (now - this.#lastPrewarmAt < PREWARM_THROTTLE_MS) return;
315
+ this.#lastPrewarmAt = now;
165
316
  try {
166
317
  if (this.config?.prewarmMode === "attach") {
167
- this.attach(host, agentId, token);
318
+ this.#attach(host, agentId, token);
168
319
  } else {
169
- this.requestPrewarm(host, agentId, token);
320
+ this.#requestPrewarm(host, agentId, token);
170
321
  }
171
322
  } catch {
172
323
  // A prewarm must never surface an error; the click connects normally.
173
324
  }
174
325
  }
175
326
 
176
- private requestPrewarm(host: string, agentId: string, token: string): void {
327
+ #requestPrewarm(host: string, agentId: string, token: string): void {
177
328
  if (typeof fetch !== "function") return;
178
329
  const url = `${toHttpBase(host)}/v1/client/agents/${encodeURIComponent(agentId)}/voice/prewarm`;
179
330
  void fetch(url, {
@@ -183,12 +334,12 @@ export class RuntypeVoiceProvider implements VoiceProvider {
183
334
  }).catch(() => {});
184
335
  }
185
336
 
186
- private attach(host: string, agentId: string, token: string): void {
187
- const params = new URLSearchParams({ clientCapabilities: "attach" });
337
+ #attach(host: string, agentId: string, token: string): void {
338
+ const params = new URLSearchParams();
188
339
  const configuredIdleMs = this.config?.attachIdleMs;
189
340
  const idleMs = clampAttachIdleMs(configuredIdleMs);
190
341
  if (configuredIdleMs !== undefined) params.set("attachIdleMs", String(idleMs));
191
- const ws = new WebSocket(`${this.voiceSocketUrl(host, agentId)}?${params}`, [
342
+ const ws = new WebSocket(this.#voiceSocketUrl(host, agentId, params), [
192
343
  "runtype.bearer",
193
344
  "runtype.attach",
194
345
  token,
@@ -206,18 +357,18 @@ export class RuntypeVoiceProvider implements VoiceProvider {
206
357
  openedAt: 0,
207
358
  idleTimer: undefined,
208
359
  };
209
- this.attached = entry;
360
+ this.#attached = entry;
210
361
 
211
362
  ws.onopen = () => {
212
- if (this.attached !== entry) return;
363
+ if (this.#attached !== entry) return;
213
364
  entry.state = ws.protocol === "runtype.attach" ? "attached" : "live";
214
365
  entry.openedAt = Date.now();
215
- this.armAttachIdle(entry, idleMs);
366
+ this.#armAttachIdle(entry, idleMs);
216
367
  if (entry.state === "attached") ws.send('{"type":"ping"}');
217
368
  entry.settle();
218
369
  };
219
370
  ws.onmessage = (event) => {
220
- if (this.attached !== entry || typeof event.data !== "string") return;
371
+ if (this.#attached !== entry || typeof event.data !== "string") return;
221
372
  let msg: { type?: unknown; idleMs?: unknown };
222
373
  try {
223
374
  msg = JSON.parse(event.data);
@@ -226,27 +377,27 @@ export class RuntypeVoiceProvider implements VoiceProvider {
226
377
  }
227
378
  // The server's idle window is authoritative; close no later than it does.
228
379
  if (msg.type === "attached" && typeof msg.idleMs === "number") {
229
- this.armAttachIdle(entry, msg.idleMs - (Date.now() - entry.openedAt));
380
+ this.#armAttachIdle(entry, msg.idleMs - (Date.now() - entry.openedAt));
230
381
  }
231
382
  };
232
383
  ws.onerror = () => {};
233
384
  ws.onclose = () => {
234
- if (this.attached === entry) this.releaseAttached();
385
+ if (this.#attached === entry) this.#releaseAttached();
235
386
  };
236
387
  }
237
388
 
238
- private armAttachIdle(entry: AttachedSocket, ms: number): void {
389
+ #armAttachIdle(entry: AttachedSocket, ms: number): void {
239
390
  clearTimeout(entry.idleTimer);
240
391
  entry.idleTimer = setTimeout(() => {
241
- if (this.attached === entry) this.releaseAttached();
392
+ if (this.#attached === entry) this.#releaseAttached();
242
393
  }, Math.max(0, ms));
243
394
  }
244
395
 
245
396
  /** Close and forget the prewarmed socket, if any. Silent by design. */
246
- private releaseAttached(): void {
247
- const entry = this.attached;
397
+ #releaseAttached(): void {
398
+ const entry = this.#attached;
248
399
  if (!entry) return;
249
- this.attached = null;
400
+ this.#attached = null;
250
401
  clearTimeout(entry.idleTimer);
251
402
  const { ws } = entry;
252
403
  ws.onopen = null;
@@ -269,8 +420,8 @@ export class RuntypeVoiceProvider implements VoiceProvider {
269
420
  * Resolves `null` when there is none, it stalled, or it dropped, so the
270
421
  * caller opens a fresh socket.
271
422
  */
272
- private async takeAttachedSocket(): Promise<{ ws: WebSocket; live: boolean } | null> {
273
- const entry = this.attached;
423
+ async #takeAttachedSocket(): Promise<{ ws: WebSocket; live: boolean } | null> {
424
+ const entry = this.#attached;
274
425
  if (!entry) return null;
275
426
  if (entry.state === "connecting") {
276
427
  let timer: ReturnType<typeof setTimeout> | undefined;
@@ -282,23 +433,38 @@ export class RuntypeVoiceProvider implements VoiceProvider {
282
433
  ]);
283
434
  clearTimeout(timer);
284
435
  }
285
- if (this.attached !== entry) return null;
436
+ if (this.#attached !== entry) return null;
286
437
  if (entry.state === "connecting" || entry.ws.readyState !== WebSocket.OPEN) {
287
- this.releaseAttached();
438
+ this.#releaseAttached();
288
439
  return null;
289
440
  }
290
- this.attached = null;
441
+ this.#attached = null;
291
442
  clearTimeout(entry.idleTimer);
292
443
  return { ws: entry.ws, live: entry.state === "live" };
293
444
  }
294
445
 
295
- private voiceSocketUrl(host: string, agentId: string): string {
296
- return `${toWsBase(host)}/ws/agents/${encodeURIComponent(agentId)}/voice`;
446
+ /**
447
+ * The call socket URL. Every socket declares the full-duplex capability;
448
+ * `extra` adds params (e.g. an attach prewarm's). The token never goes in
449
+ * the URL, and these params are not secrets.
450
+ */
451
+ #voiceSocketUrl(host: string, agentId: string, attach?: URLSearchParams): string {
452
+ const capabilities = ["partial_transcript", "context"];
453
+ if (this.bridge && this.config?.clientDelegation !== false) capabilities.push("client_delegation", "delegation_update");
454
+ if (attach) capabilities.unshift("attach");
455
+ const params = new URLSearchParams({
456
+ voiceProtocol: VOICE_PROTOCOL,
457
+ clientVersion: `persona/${VERSION}`,
458
+ voiceCapabilities: FULL_DUPLEX_CAPABILITY,
459
+ clientCapabilities: capabilities.join(","),
460
+ });
461
+ attach?.forEach((value, key) => params.set(key, value));
462
+ return `${toWsBase(host)}/ws/agents/${encodeURIComponent(agentId)}/voice?${params}`;
297
463
  }
298
464
 
299
465
  /** Start the call: acquire mic, open the WS, stream PCM until hang-up. */
300
466
  async startListening(): Promise<void> {
301
- if (this.callLive) return; // idempotent: a call is already live
467
+ if (this.#callLive) return; // idempotent: a call is already live
302
468
 
303
469
  const agentId = this.config?.agentId;
304
470
  const token = this.config?.clientToken;
@@ -307,9 +473,9 @@ export class RuntypeVoiceProvider implements VoiceProvider {
307
473
  if (!token) throw new Error("Runtype voice requires a clientToken");
308
474
  if (!host) throw new Error("Runtype voice requires a host (or widget apiUrl)");
309
475
 
310
- const generation = ++this.callGeneration;
311
- this.intentionalClose = false;
312
- this.callLive = true;
476
+ const generation = ++this.#callGeneration;
477
+ this.#intentionalClose = false;
478
+ this.#callLive = true;
313
479
 
314
480
  try {
315
481
  const stream = await navigator.mediaDevices.getUserMedia({
@@ -319,11 +485,11 @@ export class RuntypeVoiceProvider implements VoiceProvider {
319
485
  echoCancellation: true,
320
486
  },
321
487
  });
322
- if (generation !== this.callGeneration) {
488
+ if (generation !== this.#callGeneration) {
323
489
  stream.getTracks().forEach((t) => t.stop());
324
490
  return;
325
491
  }
326
- this.mediaStream = stream;
492
+ this.#mediaStream = stream;
327
493
 
328
494
  // Create + resume both contexts inside the click gesture (iOS autoplay).
329
495
  const AudioCtx =
@@ -334,115 +500,124 @@ export class RuntypeVoiceProvider implements VoiceProvider {
334
500
  if (captureContext.state === "suspended") {
335
501
  await captureContext.resume().catch(() => {});
336
502
  }
337
- this.captureContext = captureContext;
503
+ this.#captureContext = captureContext;
338
504
 
339
505
  const engine = this.config?.createPlaybackEngine
340
506
  ? await this.config.createPlaybackEngine()
341
507
  : new AudioPlaybackManager(PLAYBACK_SAMPLE_RATE);
342
- if (generation !== this.callGeneration) {
508
+ if (generation !== this.#callGeneration) {
343
509
  // Torn down while async work was in flight: free what we acquired.
344
510
  void engine.destroy();
345
511
  stream.getTracks().forEach((t) => t.stop());
346
512
  captureContext.close().catch(() => {});
347
513
  return;
348
514
  }
349
- this.playback = engine;
350
- engine.onFinished(() => {
351
- if (generation !== this.callGeneration) return;
352
- this.isSpeaking = false;
353
- // Reply drained: the call stays open, so return to listening.
354
- if (this.ws && this.ws.readyState === WebSocket.OPEN) {
355
- this.emitStatus("listening");
356
- }
357
- });
515
+ this.#playback = engine;
516
+ this.#armPlaybackFinished();
517
+ // Capture from now: frames buffer until the call socket is open.
518
+ this.#startCapture(captureContext, stream, generation);
358
519
 
359
- const adopted = await this.takeAttachedSocket();
360
- if (generation !== this.callGeneration) {
520
+ const adopted = await this.#takeAttachedSocket();
521
+ if (generation !== this.#callGeneration) {
361
522
  adopted?.ws.close(1000, "client ended call");
362
523
  return;
363
524
  }
364
525
  if (adopted) {
365
526
  const { ws } = adopted;
366
- this.ws = ws;
367
- this.bindCallSocket(ws, generation);
527
+ this.#ws = ws;
528
+ this.#bindCallSocket(ws, generation);
368
529
  if (!adopted.live) ws.send('{"type":"start"}');
369
- this.emitStatus("listening");
370
- this.startCapture(captureContext, stream, ws, generation);
530
+ this.#flushPreOpenAudio(ws);
531
+ this.#emitStatus("listening");
371
532
  return;
372
533
  }
373
534
 
374
535
  // Token rides the subprotocol; `runtype.bearer` is the marker the server
375
536
  // echoes as the negotiated subprotocol (browsers fail the handshake if an
376
537
  // offered subprotocol goes unanswered).
377
- const ws = new WebSocket(this.voiceSocketUrl(host, agentId), ["runtype.bearer", token]);
538
+ const ws = new WebSocket(this.#voiceSocketUrl(host, agentId), ["runtype.bearer", token]);
378
539
  ws.binaryType = "arraybuffer";
379
- this.ws = ws;
540
+ this.#ws = ws;
380
541
 
381
542
  ws.onopen = () => {
382
- if (generation !== this.callGeneration) return;
383
- this.emitStatus("listening");
384
- this.startCapture(captureContext, stream, ws, generation);
543
+ if (generation !== this.#callGeneration) return;
544
+ this.#flushPreOpenAudio(ws);
545
+ this.#emitStatus("listening");
385
546
  };
386
- this.bindCallSocket(ws, generation);
547
+ this.#bindCallSocket(ws, generation);
387
548
  } catch (error) {
388
- this.cleanup();
389
- this.emitError(error as Error);
390
- this.emitStatus("error");
549
+ this.#cleanup();
550
+ this.#emitError(error as Error);
551
+ this.#emitStatus("error");
391
552
  throw error;
392
553
  }
393
554
  }
394
555
 
395
556
  /** Route a call socket's frames, errors, and close into this call. */
396
- private bindCallSocket(ws: WebSocket, generation: number): void {
397
- ws.onmessage = (event) => this.handleMessage(event, generation);
557
+ #bindCallSocket(ws: WebSocket, generation: number): void {
558
+ ws.onmessage = (event) => this.#handleMessage(event, generation);
398
559
 
399
560
  ws.onerror = () => {
400
- if (generation !== this.callGeneration) return;
401
- this.emitError(new Error("Voice connection failed"));
402
- this.emitStatus("error");
403
- this.cleanup();
561
+ if (generation !== this.#callGeneration) return;
562
+ this.#emitError(new Error("Voice connection failed"));
563
+ this.#emitStatus("error");
564
+ this.#cleanup();
404
565
  };
405
566
 
406
567
  ws.onclose = (evt) => {
407
- if (this.intentionalClose) {
408
- this.intentionalClose = false;
568
+ if (this.#intentionalClose) {
569
+ this.#intentionalClose = false;
409
570
  return;
410
571
  }
411
- if (generation !== this.callGeneration) return;
572
+ if (generation !== this.#callGeneration) return;
412
573
  if (evt.code !== 1000) {
413
574
  const codeMsg = evt.code ? ` (code ${evt.code})` : "";
414
- this.emitError(new Error(`Voice connection closed${codeMsg}`));
415
- this.emitStatus("error");
575
+ this.#emitError(new Error(`Voice connection closed${codeMsg}`));
576
+ this.#emitStatus("error");
416
577
  } else {
417
- this.emitStatus("idle");
578
+ this.#emitStatus("idle");
418
579
  }
419
- this.cleanup();
580
+ this.#cleanup();
420
581
  };
421
582
  }
422
583
 
584
+ /** The AI-disclosure notice for a live speech-to-speech call (`disclosureText`; `false` hides it). */
585
+ getDisclosure(): string | null {
586
+ const text = this.config?.disclosureText;
587
+ return this.#callLive && this.#speechToSpeech && text !== false ? text || DISCLOSURE_TEXT : null;
588
+ }
589
+
423
590
  /** End the call (hang up). */
424
591
  async stopListening(): Promise<void> {
425
- this.cleanup();
426
- this.emitStatus("idle");
592
+ this.#cleanup();
593
+ this.#emitStatus("idle");
427
594
  }
428
595
 
429
596
  /** Tear down the call and drop all callbacks (used by `cleanupVoice`). */
430
597
  async disconnect(): Promise<void> {
431
- this.cleanup();
432
- this.emitStatus("disconnected");
433
- this.resultCallbacks = [];
434
- this.errorCallbacks = [];
435
- this.statusCallbacks = [];
436
- this.transcriptCallbacks = [];
437
- this.metricsCallbacks = [];
598
+ this.#cleanup();
599
+ this.#emitStatus("disconnected");
600
+ this.#resultCallbacks = [];
601
+ this.#errorCallbacks = [];
602
+ this.#statusCallbacks = [];
603
+ this.#transcriptCallbacks = [];
604
+ this.#metricsCallbacks = [];
438
605
  }
439
606
 
440
- /** Stop the spoken reply without ending the call. */
607
+ /**
608
+ * Stop the spoken reply without ending the call. In a speech-to-speech
609
+ * session this also asks the server to cancel the response; audio is dropped
610
+ * until its `audio_clear` acknowledgement arrives.
611
+ */
441
612
  stopPlayback(): void {
442
- if (this.playback) this.playback.flush();
443
- this.isSpeaking = false;
444
- if (this.ws && this.ws.readyState === WebSocket.OPEN) {
445
- this.emitStatus("listening");
613
+ this.#clearLocalPlayback();
614
+ const ws = this.#ws;
615
+ if (ws && ws.readyState === WebSocket.OPEN) {
616
+ if (this.#speechToSpeech) {
617
+ this.#cancelledUntil = Date.now() + CANCEL_ACK_TIMEOUT_MS;
618
+ ws.send('{"type":"cancel"}');
619
+ }
620
+ this.#emitStatus("listening");
446
621
  }
447
622
  }
448
623
 
@@ -455,35 +630,33 @@ export class RuntypeVoiceProvider implements VoiceProvider {
455
630
 
456
631
  /** True while the call (hot mic) is live. */
457
632
  isBargeInActive(): boolean {
458
- return this.callLive;
633
+ return this.#callLive;
459
634
  }
460
635
 
461
636
  /** "Hang up" the always-on mic. */
462
637
  onLevel(callback: (level: number) => void): void {
463
- this.levelCallbacks.push(callback);
638
+ this.#levelCallbacks.push(callback);
464
639
  }
465
640
 
466
641
  async deactivateBargeIn(): Promise<void> {
467
- this.cleanup();
468
- this.emitStatus("idle");
642
+ this.#cleanup();
643
+ this.#emitStatus("idle");
469
644
  }
470
645
 
471
646
  // --- Capture ---------------------------------------------------------------
472
647
 
473
- private startCapture(
474
- context: AudioContext,
475
- stream: MediaStream,
476
- ws: WebSocket,
477
- generation: number,
478
- ): void {
648
+ #startCapture(context: AudioContext, stream: MediaStream, generation: number): void {
479
649
  const source = context.createMediaStreamSource(stream);
480
- this.sourceNode = source;
650
+ this.#sourceNode = source;
481
651
  const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
482
- this.processor = processor;
652
+ this.#processor = processor;
483
653
 
484
654
  processor.onaudioprocess = (e) => {
485
- if (generation !== this.callGeneration) return;
486
- if (ws.readyState !== WebSocket.OPEN) return;
655
+ if (generation !== this.#callGeneration) return;
656
+ // No socket yet (adopting a prewarm) or still handshaking: buffer.
657
+ const ws = this.#ws;
658
+ const open = ws?.readyState === WebSocket.OPEN;
659
+ if (ws && !open && ws.readyState !== WebSocket.CONNECTING) return;
487
660
  const input = e.inputBuffer.getChannelData(0);
488
661
  const pcm16 = new Int16Array(input.length);
489
662
  let sumSquares = 0;
@@ -492,14 +665,23 @@ export class RuntypeVoiceProvider implements VoiceProvider {
492
665
  sumSquares += s * s;
493
666
  pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
494
667
  }
495
- ws.send(pcm16.buffer);
668
+ if (ws && open) {
669
+ this.#flushPreOpenAudio(ws);
670
+ ws.send(pcm16.buffer);
671
+ } else {
672
+ this.#preOpenAudio.push(pcm16.buffer);
673
+ this.#preOpenBytes += pcm16.byteLength;
674
+ while (this.#preOpenBytes > PRE_OPEN_AUDIO_MAX_BYTES) {
675
+ this.#preOpenBytes -= this.#preOpenAudio.shift()!.byteLength;
676
+ }
677
+ }
496
678
  // Amplitude comes free from the buffer we already walked: no analyser
497
679
  // node, no second capture. RMS is scaled because speech rarely exceeds
498
680
  // ~0.3 RMS, so raw values would sit near the bottom of the 0..1 range.
499
- if (this.levelCallbacks.length > 0) {
681
+ if (this.#levelCallbacks.length > 0) {
500
682
  const rms = Math.sqrt(sumSquares / input.length);
501
683
  const level = Math.max(0, Math.min(1, rms * LEVEL_RMS_SCALE));
502
- for (const cb of this.levelCallbacks) cb(level);
684
+ for (const cb of this.#levelCallbacks) cb(level);
503
685
  }
504
686
  };
505
687
 
@@ -509,13 +691,19 @@ export class RuntypeVoiceProvider implements VoiceProvider {
509
691
  processor.connect(context.destination);
510
692
  }
511
693
 
694
+ #flushPreOpenAudio(ws: WebSocket): void {
695
+ for (const frame of this.#preOpenAudio) ws.send(frame);
696
+ this.#preOpenAudio = [];
697
+ this.#preOpenBytes = 0;
698
+ }
699
+
512
700
  // --- Downstream ------------------------------------------------------------
513
701
 
514
- private handleMessage(event: MessageEvent, generation: number): void {
515
- if (generation !== this.callGeneration) return;
702
+ #handleMessage(event: MessageEvent, generation: number): void {
703
+ if (generation !== this.#callGeneration) return;
516
704
 
517
705
  if (event.data instanceof ArrayBuffer) {
518
- this.handleAudioFrame(event.data, generation);
706
+ this.#handleAudioFrame(event.data, generation);
519
707
  return;
520
708
  }
521
709
 
@@ -527,30 +715,142 @@ export class RuntypeVoiceProvider implements VoiceProvider {
527
715
  }
528
716
 
529
717
  switch (msg.type) {
718
+ case "session_config":
719
+ // The follow-up session_config carries only interruptionMode: keep the mode.
720
+ if (msg.speechMode) {
721
+ this.#speechToSpeech = msg.speechMode === "speech_to_speech";
722
+ this.#playback?.setContinuousMode?.(this.#speechToSpeech);
723
+ }
724
+ // The negotiated client frame types. A server without `capabilities`
725
+ // predates them: no client delegation, context or updates.
726
+ if (Array.isArray(msg.capabilities)) this.#capabilities = new Set(msg.capabilities);
727
+ if (typeof msg.callId === "string") this.#callId = msg.callId;
728
+ this.#clientDelegation = this.#speechToSpeech && this.#capabilities.has("client_delegation") && !!this.bridge;
729
+ // Speech-to-speech is known only now: let the UI show its disclosure.
730
+ if (this.#speechToSpeech && !this.#isSpeaking) this.#emitStatus("listening");
731
+ if (this.#capabilities.has("context") && !this.#contextSent) {
732
+ this.#contextSent = true;
733
+ this.#pendingContext = this.#buildContextText();
734
+ }
735
+ break;
736
+
737
+ case "transcript_update": {
738
+ const role = msg.role === "assistant" ? "assistant" : "user";
739
+ // A reply the client cancelled keeps streaming until the server clears it.
740
+ const utteranceId = msg.utteranceId ?? msg.turnId;
741
+ if (!utteranceId || (role === "assistant" && this.#isCancelling())) break;
742
+ const turnId = String(utteranceId);
743
+ if (role === "user") {
744
+ // Release on a final user transcript, not a partial: a mid-utterance
745
+ // context append makes the voice model answer early (or not delegate).
746
+ if (msg.final === true) this.#flushCallContext(generation);
747
+ this.#foldReadback = false;
748
+ } else {
749
+ // Read-back of a delegated result: core rotates the assistant id at
750
+ // completion, so it is the first new id after it. The chat renders it.
751
+ if (this.#foldReadback && !this.#assistantTurns.has(turnId)) {
752
+ this.#foldedTurns.add(turnId);
753
+ this.#foldReadback = false;
754
+ }
755
+ this.#assistantTurns.add(turnId);
756
+ if (this.#foldedTurns.has(turnId)) break;
757
+ }
758
+ this.#emitTranscript(role, msg.text ?? "", msg.final === true, {
759
+ turnId,
760
+ ...(typeof msg.startMs === "number" && { startMs: msg.startMs }),
761
+ ...(typeof msg.endMs === "number" && { endMs: msg.endMs }),
762
+ // The chat pipeline owns the conversation: speech is only captioned.
763
+ ...(this.#clientDelegation && { caption: true }),
764
+ });
765
+ break;
766
+ }
767
+
768
+ case "delegation_started":
769
+ case "delegation_completed": {
770
+ const delegationId = String(msg.delegationId ?? msg.turnId);
771
+ if (msg.type === "delegation_started") {
772
+ this.#flushCallContext(generation);
773
+ // `input`: run this turn through the chat pipeline (client delegation).
774
+ const input = msg.input;
775
+ if (this.#clientDelegation && input) {
776
+ this.#runDelegation(
777
+ delegationId,
778
+ {
779
+ delegationId,
780
+ userText: String(input.text ?? ""),
781
+ userUtteranceIds: Array.isArray(input.userUtteranceIds)
782
+ ? input.userUtteranceIds.filter((id: unknown): id is string => typeof id === "string")
783
+ : [],
784
+ messages: Array.isArray(input.messages) ? input.messages : [],
785
+ },
786
+ generation,
787
+ );
788
+ }
789
+ } else if (msg.speak !== false && this.#answered.has(delegationId)) {
790
+ // Every spoken phase (the approval ask, the late result) of an answer
791
+ // the chat shows is read back. A refusal of one that never started
792
+ // here has no chat answer: it renders. `final` may never come.
793
+ this.#foldReadback = true;
794
+ }
795
+ this.#delegating = msg.type === "delegation_started";
796
+ if (!this.#isSpeaking && !this.#isCancelling()) {
797
+ this.#emitStatus(this.#delegating ? "processing" : "listening");
798
+ }
799
+ break;
800
+ }
801
+
802
+ case "delegation_cancelled":
803
+ // The server gave up on it: no more frames for it, and its approval
804
+ // timers stop. The chat turn still renders. Past its deadline (or for
805
+ // an unknown reason), a parked approval card is also declined, as the
806
+ // approval TTL would; the call ending or the voice model cancelling
807
+ // leaves the card usable in the chat.
808
+ this.#dropDelegation(
809
+ String(msg.delegationId),
810
+ msg.reason !== "session_ending" && msg.reason !== "provider_cancelled",
811
+ );
812
+ break;
813
+
814
+ case "warning":
815
+ // Non-fatal (an unknown or refused frame): the call goes on. A refused
816
+ // delegation frame means the server is done with that delegation.
817
+ console.warn(`[Persona voice] ${msg.code}: ${msg.message ?? ""}`, this.#callId ?? "");
818
+ if (msg.delegationId && (msg.code === "UNKNOWN_DELEGATION" || msg.code === "LATE_RESULT_LIMIT")) {
819
+ this.#dropDelegation(String(msg.delegationId));
820
+ }
821
+ break;
822
+
823
+ case "audio_clear":
824
+ // Barge-in (or our own cancel acknowledged): stop playback now.
825
+ this.#clearLocalPlayback();
826
+ this.#cancelledUntil = 0;
827
+ this.#emitStatus("listening");
828
+ break;
829
+
530
830
  case "transcript_interim":
531
- this.emitStatus("listening");
532
- this.emitTranscript("user", msg.text ?? "", false);
831
+ this.#emitStatus("listening");
832
+ this.#emitTranscript("user", msg.text ?? "", false);
533
833
  break;
534
834
 
535
835
  case "transcript_final": {
536
836
  const role = msg.role === "assistant" ? "assistant" : "user";
537
837
  // user final → agent is now thinking; assistant final → reply incoming.
538
- this.emitStatus(role === "user" ? "processing" : "speaking");
539
- this.emitTranscript(role, msg.text ?? "", true);
838
+ this.#emitStatus(role === "user" ? "processing" : "speaking");
839
+ this.#emitTranscript(role, msg.text ?? "", true);
540
840
  break;
541
841
  }
542
842
 
543
843
  case "audio_end":
544
- if (this.playback) {
545
- this.playback.markStreamEnd();
844
+ if (this.#playback) {
845
+ this.#playback.markStreamEnd();
546
846
  } else {
547
- this.isSpeaking = false;
548
- this.emitStatus("listening");
847
+ this.#isSpeaking = false;
848
+ this.#emitStatus("listening");
549
849
  }
550
850
  break;
551
851
 
552
852
  case "metrics":
553
- this.emitMetrics({
853
+ this.#emitMetrics({
554
854
  llmMs: msg.llm_ms,
555
855
  ttsMs: msg.tts_ms,
556
856
  firstAudioMs: msg.first_audio_ms,
@@ -559,106 +859,287 @@ export class RuntypeVoiceProvider implements VoiceProvider {
559
859
  break;
560
860
 
561
861
  case "error":
562
- this.emitError(new Error(msg.error || "Voice error"));
563
- this.emitStatus("error");
862
+ this.#emitError(new Error(msg.error || "Voice error"));
863
+ this.#emitStatus("error");
564
864
  break;
565
865
  }
566
866
  }
567
867
 
568
- private handleAudioFrame(buf: ArrayBuffer, generation: number): void {
569
- if (generation !== this.callGeneration) return;
570
- if (!this.playback) return;
868
+ /** Send the call-start `context` frame (history + host context), if any. */
869
+ /**
870
+ * The call-start `context` text. History is read now, before any of this
871
+ * call's voice bubbles exist, so it never includes the utterance in progress.
872
+ */
873
+ async #buildContextText(): Promise<string> {
874
+ const history = this.bridge?.getHistory() ?? [];
875
+ let extra = "";
876
+ try {
877
+ const source = this.config?.callContext;
878
+ extra = String((typeof source === "function" ? await source() : source) ?? "");
879
+ } catch {
880
+ // A failing host callback only loses its own part of the context.
881
+ }
882
+ return buildCallContext(history, extra);
883
+ }
884
+
885
+ /** Send the held call-start context, once: at the first final user transcript or delegation. */
886
+ #flushCallContext(generation: number): void {
887
+ const pending = this.#pendingContext;
888
+ if (!pending) return;
889
+ this.#pendingContext = null;
890
+ let dropped = false;
891
+ const done = pending.then((text) => {
892
+ const ws = this.#ws;
893
+ if (dropped || !text || generation !== this.#callGeneration || ws?.readyState !== WebSocket.OPEN) return;
894
+ ws.send(JSON.stringify({ type: "context", text }));
895
+ });
896
+ this.#contextSend = { done, drop: () => (dropped = true) };
897
+ }
898
+
899
+ /** Let a released context go out first; a hung host callback is dropped. */
900
+ async #awaitContextSend(): Promise<void> {
901
+ const send = this.#contextSend;
902
+ if (!send) return;
903
+ let timer: ReturnType<typeof setTimeout> | undefined;
904
+ const timedOut = await Promise.race([
905
+ send.done.then(() => false),
906
+ new Promise<boolean>((resolve) => {
907
+ timer = setTimeout(() => resolve(true), CONTEXT_BEFORE_RESULT_TIMEOUT_MS);
908
+ }),
909
+ ]);
910
+ clearTimeout(timer);
911
+ if (timedOut) send.drop();
912
+ }
913
+
914
+ /** No more frames for this delegation; its chat answer goes back to browser TTS. */
915
+ #dropDelegation(delegationId: string, expired = false): void {
916
+ this.#cancelledDelegations.add(delegationId);
917
+ this.bridge?.dropDelegation?.(delegationId, expired);
918
+ this.#followUps.get(delegationId)?.abort();
919
+ }
920
+
921
+ /** Run a delegated turn through the session bridge (one at a time) and answer it. */
922
+ #runDelegation(delegationId: string, request: VoiceDelegationRequest, generation: number): void {
923
+ const bridge = this.bridge!;
924
+ /** Send a frame for this delegation, unless the call or the delegation ended. */
925
+ const send = (type: "delegation_update" | "delegation_result", status: string, text: string) => {
926
+ const ws = this.#ws;
927
+ if (generation !== this.#callGeneration || ws?.readyState !== WebSocket.OPEN) return false;
928
+ if (this.#cancelledDelegations.has(delegationId)) return false;
929
+ ws.send(JSON.stringify({ type, delegationId, status, text }));
930
+ return true;
931
+ };
932
+ this.#delegations = this.#delegations.then(async () => {
933
+ // Cancelled while queued behind another turn: never start it.
934
+ if (generation !== this.#callGeneration || this.#cancelledDelegations.has(delegationId)) return;
935
+ const result: VoiceDelegationResult = await bridge
936
+ .runDelegatedTurn(request)
937
+ .catch(() => ({ status: "failed", text: "" }));
938
+ await this.#awaitContextSend();
939
+ // Parked on an approval: ask now (non-terminal), answer once the visitor
940
+ // decides. A server that can't take an update gets the ask as the result.
941
+ const parked = !!result.followUp;
942
+ const update = parked && this.#capabilities.has("delegation_update");
943
+ const sent = send(
944
+ update ? "delegation_update" : "delegation_result",
945
+ parked && !update ? "completed" : result.status,
946
+ result.text,
947
+ );
948
+ if (!sent) return;
949
+ if (result.status !== "failed") this.#answered.add(delegationId);
950
+ if (!result.followUp) return;
951
+ // The approval bookkeeping (expiry, supersede) runs either way.
952
+ const abort = new AbortController();
953
+ this.#followUps.set(delegationId, abort);
954
+ void result
955
+ .followUp({
956
+ signal: abort.signal,
957
+ // Under the server's 600 s deadline, which restarts at each update.
958
+ approvalTimeoutMs: this.config?.approvalTimeoutMs && Math.min(this.config.approvalTimeoutMs, APPROVAL_TIMEOUT_MAX_MS),
959
+ readBack: update,
960
+ // Another gated tool in the same turn: another (non-terminal) update.
961
+ onUpdate: (text) => send("delegation_update", "pending_approval", text),
962
+ })
963
+ .then((followUp) => {
964
+ this.#followUps.delete(delegationId);
965
+ // Hang-up and cancellation send nothing.
966
+ if (update && followUp && !abort.signal.aborted) send("delegation_result", followUp.status, followUp.text);
967
+ });
968
+ });
969
+ }
970
+
971
+ #handleAudioFrame(buf: ArrayBuffer, generation: number): void {
972
+ if (generation !== this.#callGeneration) return;
973
+ if (!this.#playback) return;
974
+ if (this.#isCancelling()) return;
571
975
  const pcm = stripWavHeader(buf);
572
976
  if (pcm.length === 0) return;
573
- if (!this.isSpeaking) {
574
- this.isSpeaking = true;
575
- this.emitStatus("speaking");
977
+ if (!this.#isSpeaking) {
978
+ this.#isSpeaking = true;
979
+ this.#emitStatus("speaking");
980
+ }
981
+ this.#armPlaybackFinished();
982
+ this.#playback.enqueue(pcm);
983
+ if (this.#speechToSpeech) this.#scheduleContinuousDrain(pcm.length);
984
+ }
985
+
986
+ #isCancelling(): boolean {
987
+ return Date.now() < this.#cancelledUntil;
988
+ }
989
+
990
+ /** Register the (one-shot) engine drain callback if none is pending. */
991
+ #armPlaybackFinished(): void {
992
+ if (this.#finishedArmed || !this.#playback) return;
993
+ this.#finishedArmed = true;
994
+ const generation = this.#callGeneration;
995
+ this.#playback.onFinished(() => {
996
+ if (generation !== this.#callGeneration) return;
997
+ this.#finishedArmed = false;
998
+ this.#handlePlaybackDrained();
999
+ });
1000
+ }
1001
+
1002
+ /**
1003
+ * Speech-to-speech replies never get an end-of-stream, so estimate when the
1004
+ * queued audio finishes from its duration and release "speaking" then. The
1005
+ * engine is never marked ended, so the next reply's audio plays through the
1006
+ * same stream untouched and an underrun is never mistaken for a reply end.
1007
+ */
1008
+ #scheduleContinuousDrain(byteLength: number): void {
1009
+ const now = Date.now();
1010
+ // PCM16 @ 24 kHz: 48 bytes per millisecond.
1011
+ this.#playbackEndsAt = Math.max(now, this.#playbackEndsAt) + byteLength / 48;
1012
+ clearTimeout(this.#drainTimer);
1013
+ this.#drainTimer = setTimeout(
1014
+ () => this.#handlePlaybackDrained(),
1015
+ this.#playbackEndsAt - now + CONTINUOUS_DRAIN_GRACE_MS,
1016
+ );
1017
+ }
1018
+
1019
+ #handlePlaybackDrained(): void {
1020
+ this.#isSpeaking = false;
1021
+ this.#playbackEndsAt = 0;
1022
+ // Reply drained: the call stays open, so return to listening (or to
1023
+ // processing while an agent delegation is still running).
1024
+ if (this.#ws && this.#ws.readyState === WebSocket.OPEN) {
1025
+ this.#emitStatus(this.#delegating ? "processing" : "listening");
576
1026
  }
577
- this.playback.enqueue(pcm);
1027
+ }
1028
+
1029
+ /** Drop queued audio locally and reset the speaking/drain bookkeeping. */
1030
+ #clearLocalPlayback(): void {
1031
+ this.#playback?.flush();
1032
+ // flush() discards registered callbacks: re-arm for the next reply.
1033
+ this.#finishedArmed = false;
1034
+ this.#armPlaybackFinished();
1035
+ this.#isSpeaking = false;
1036
+ this.#playbackEndsAt = 0;
1037
+ clearTimeout(this.#drainTimer);
578
1038
  }
579
1039
 
580
1040
  // --- Teardown --------------------------------------------------------------
581
1041
 
582
- private cleanup(): void {
1042
+ #cleanup(): void {
583
1043
  // Invalidate any in-flight async continuation / late frames first.
584
- this.callGeneration += 1;
585
- this.callLive = false;
586
- this.isSpeaking = false;
587
- this.releaseAttached();
588
-
589
- if (this.processor) {
590
- this.processor.onaudioprocess = null;
591
- this.processor.disconnect();
592
- this.processor = null;
1044
+ this.#callGeneration += 1;
1045
+ this.#callLive = false;
1046
+ this.#isSpeaking = false;
1047
+ this.#releaseAttached();
1048
+ this.#speechToSpeech = false;
1049
+ this.#delegating = false;
1050
+ this.#clientDelegation = false;
1051
+ this.#capabilities = new Set();
1052
+ for (const abort of this.#followUps.values()) abort.abort();
1053
+ this.#followUps.clear();
1054
+ this.#cancelledDelegations.clear();
1055
+ this.#callId = undefined;
1056
+ this.#contextSent = false;
1057
+ this.#pendingContext = null;
1058
+ this.#contextSend = null;
1059
+ this.#delegations = Promise.resolve();
1060
+ this.#answered.clear();
1061
+ this.#foldReadback = false;
1062
+ this.#assistantTurns.clear();
1063
+ this.#foldedTurns.clear();
1064
+ this.#finishedArmed = false;
1065
+ this.#cancelledUntil = 0;
1066
+ this.#playbackEndsAt = 0;
1067
+ clearTimeout(this.#drainTimer);
1068
+ this.#preOpenAudio = [];
1069
+ this.#preOpenBytes = 0;
1070
+
1071
+ if (this.#processor) {
1072
+ this.#processor.onaudioprocess = null;
1073
+ this.#processor.disconnect();
1074
+ this.#processor = null;
593
1075
  }
594
- if (this.sourceNode) {
595
- this.sourceNode.disconnect();
596
- this.sourceNode = null;
1076
+ if (this.#sourceNode) {
1077
+ this.#sourceNode.disconnect();
1078
+ this.#sourceNode = null;
597
1079
  }
598
- if (this.mediaStream) {
599
- this.mediaStream.getTracks().forEach((t) => t.stop());
600
- this.mediaStream = null;
1080
+ if (this.#mediaStream) {
1081
+ this.#mediaStream.getTracks().forEach((t) => t.stop());
1082
+ this.#mediaStream = null;
601
1083
  }
602
- if (this.captureContext) {
603
- this.captureContext.close().catch(() => {});
604
- this.captureContext = null;
1084
+ if (this.#captureContext) {
1085
+ this.#captureContext.close().catch(() => {});
1086
+ this.#captureContext = null;
605
1087
  }
606
- if (this.playback) {
607
- void this.playback.destroy();
608
- this.playback = null;
1088
+ if (this.#playback) {
1089
+ void this.#playback.destroy();
1090
+ this.#playback = null;
609
1091
  }
610
- if (this.ws) {
611
- this.intentionalClose = true;
1092
+ if (this.#ws) {
1093
+ this.#intentionalClose = true;
612
1094
  try {
613
- this.ws.close(1000, "client ended call");
1095
+ this.#ws.close(1000, "client ended call");
614
1096
  } catch {
615
1097
  // ignore
616
1098
  }
617
- this.ws = null;
1099
+ this.#ws = null;
618
1100
  }
619
1101
  }
620
1102
 
621
1103
  // --- Callback registration + emit -----------------------------------------
622
1104
 
623
1105
  onResult(callback: (result: VoiceResult) => void): void {
624
- this.resultCallbacks.push(callback);
1106
+ this.#resultCallbacks.push(callback);
625
1107
  }
626
1108
 
627
1109
  onError(callback: (error: Error) => void): void {
628
- this.errorCallbacks.push(callback);
1110
+ this.#errorCallbacks.push(callback);
629
1111
  }
630
1112
 
631
1113
  onStatusChange(callback: (status: VoiceStatus) => void): void {
632
- this.statusCallbacks.push(callback);
1114
+ this.#statusCallbacks.push(callback);
633
1115
  }
634
1116
 
635
- onTranscript(
636
- callback: (role: "user" | "assistant", text: string, isFinal: boolean) => void,
637
- ): void {
638
- this.transcriptCallbacks.push(callback);
1117
+ onTranscript(callback: TranscriptCallback): void {
1118
+ this.#transcriptCallbacks.push(callback);
639
1119
  }
640
1120
 
641
1121
  onMetrics(callback: (metrics: VoiceMetrics) => void): void {
642
- this.metricsCallbacks.push(callback);
1122
+ this.#metricsCallbacks.push(callback);
643
1123
  }
644
1124
 
645
- private emitStatus(status: VoiceStatus): void {
646
- this.statusCallbacks.forEach((cb) => cb(status));
1125
+ #emitStatus(status: VoiceStatus): void {
1126
+ this.#statusCallbacks.forEach((cb) => cb(status));
647
1127
  }
648
1128
 
649
- private emitError(error: Error): void {
650
- this.errorCallbacks.forEach((cb) => cb(error));
1129
+ #emitError(error: Error): void {
1130
+ this.#errorCallbacks.forEach((cb) => cb(error));
651
1131
  }
652
1132
 
653
- private emitTranscript(
1133
+ #emitTranscript(
654
1134
  role: "user" | "assistant",
655
1135
  text: string,
656
1136
  isFinal: boolean,
1137
+ metadata?: VoiceTranscriptMetadata,
657
1138
  ): void {
658
- this.transcriptCallbacks.forEach((cb) => cb(role, text, isFinal));
1139
+ this.#transcriptCallbacks.forEach((cb) => cb(role, text, isFinal, metadata));
659
1140
  }
660
1141
 
661
- private emitMetrics(metrics: VoiceMetrics): void {
662
- this.metricsCallbacks.forEach((cb) => cb(metrics));
1142
+ #emitMetrics(metrics: VoiceMetrics): void {
1143
+ this.#metricsCallbacks.forEach((cb) => cb(metrics));
663
1144
  }
664
1145
  }