@alexkroman1/aai-ui 0.12.3 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. package/README.md +253 -0
  2. package/dist/_colors-CpZO-88A.js +78 -0
  3. package/dist/_module-url-C13kAJ87.js +15 -0
  4. package/dist/_page-shell.d.ts +11 -0
  5. package/dist/_recover-run.d.ts +79 -0
  6. package/dist/_repeat-until.d.ts +30 -0
  7. package/dist/_run-controls.d.ts +33 -0
  8. package/dist/_submission-state.d.ts +92 -0
  9. package/dist/_tap-to-talk-state.d.ts +88 -0
  10. package/dist/_utils-Mi-SCFA1.js +47 -0
  11. package/dist/_utils.d.ts +26 -0
  12. package/dist/_web-storage.d.ts +43 -0
  13. package/dist/_workflow-api-ref.d.ts +37 -0
  14. package/dist/_workflow-files.d.ts +34 -0
  15. package/dist/_workflow-test-defs.d.ts +34 -0
  16. package/dist/aai-logo-D8pMVn9_.js +85 -0
  17. package/dist/agent-state-labels.d.ts +65 -0
  18. package/dist/agent-state.d.ts +108 -0
  19. package/dist/audio/capture.d.ts +44 -0
  20. package/dist/audio/index.d.ts +18 -0
  21. package/dist/audio/index.js +431 -0
  22. package/dist/audio/preconnect.d.ts +87 -0
  23. package/dist/audio/voice-io.d.ts +112 -0
  24. package/dist/build-default-client.d.ts +1 -0
  25. package/dist/chat-view-BCbI7gD4.js +396 -0
  26. package/dist/client-config-C7fQrioM.js +139 -0
  27. package/dist/client-config.d.ts +119 -0
  28. package/dist/client-dir.d.ts +44 -0
  29. package/dist/client-dir.js +58 -0
  30. package/dist/client-identity.d.ts +131 -0
  31. package/dist/client-tool.d.ts +51 -0
  32. package/dist/components/_colors.d.ts +73 -0
  33. package/dist/components/_form-readiness.d.ts +56 -0
  34. package/dist/components/_form-values.d.ts +19 -0
  35. package/dist/components/aai-logo.d.ts +12 -0
  36. package/dist/components/audio-result.d.ts +94 -0
  37. package/dist/components/auto-scroll.d.ts +74 -0
  38. package/dist/components/bullet-list.d.ts +74 -0
  39. package/dist/components/button.d.ts +29 -21
  40. package/dist/components/button.js +62 -23
  41. package/dist/components/chat-view.d.ts +24 -18
  42. package/dist/components/chat-view.js +2 -60
  43. package/dist/components/console-shell.d.ts +85 -0
  44. package/dist/components/controls.d.ts +21 -6
  45. package/dist/components/controls.js +31 -19
  46. package/dist/components/conversation-view.d.ts +130 -0
  47. package/dist/components/eyebrow.d.ts +12 -0
  48. package/dist/components/facts.d.ts +81 -0
  49. package/dist/components/form-fields.d.ts +142 -0
  50. package/dist/components/form-types.d.ts +73 -0
  51. package/dist/components/form.d.ts +96 -0
  52. package/dist/components/markdown.d.ts +57 -0
  53. package/dist/components/message-list.d.ts +34 -7
  54. package/dist/components/message-list.js +2 -75
  55. package/dist/components/session-controls.d.ts +125 -0
  56. package/dist/components/session-error-banner.d.ts +69 -0
  57. package/dist/components/session-state-dot.d.ts +111 -0
  58. package/dist/components/sidebar-layout.d.ts +31 -10
  59. package/dist/components/sidebar-layout.js +32 -12
  60. package/dist/components/start-screen.d.ts +17 -6
  61. package/dist/components/start-screen.js +34 -14
  62. package/dist/components/tool-call-block.d.ts +15 -14
  63. package/dist/components/tool-call-block.js +2 -117
  64. package/dist/components/tool-call-row.d.ts +75 -0
  65. package/dist/components/tool-config-context.d.ts +22 -0
  66. package/dist/components/upload-progress.d.ts +76 -0
  67. package/dist/components/url-chips.d.ts +29 -0
  68. package/dist/components/workflow-fields.d.ts +46 -0
  69. package/dist/components/workflow-pending-note.d.ts +81 -0
  70. package/dist/components/workflow-progress.d.ts +81 -0
  71. package/dist/components/workflow-run-error.d.ts +48 -0
  72. package/dist/components/workflow-run-panel.d.ts +79 -0
  73. package/dist/context.d.ts +288 -0
  74. package/dist/context.js +382 -0
  75. package/dist/conversation-log.d.ts +88 -0
  76. package/dist/default-client/assets/_module-url-BX0RuRU2.js +1 -0
  77. package/dist/default-client/assets/audio-mpgipo2Q.js +1 -0
  78. package/dist/default-client/assets/capture-processor-D0q7Qrdd.js +90 -0
  79. package/dist/default-client/assets/index-8ihb1Urm.css +2 -0
  80. package/dist/default-client/assets/index-DPfmSFmc.js +126 -0
  81. package/dist/default-client/assets/playback-processor-CjDbc9iH.js +314 -0
  82. package/dist/default-client/favicon.ico +0 -0
  83. package/dist/default-client/index.html +15 -0
  84. package/dist/default-client-entry.d.ts +6 -0
  85. package/dist/default-client.d.ts +19 -0
  86. package/dist/define-client.d.ts +169 -26
  87. package/dist/define-client.js +209 -55
  88. package/dist/eyebrow-UfmSz9yy.js +27 -0
  89. package/dist/hooks-DN4gHYsj.js +182 -0
  90. package/dist/hooks.d.ts +172 -0
  91. package/dist/hooks.js +3 -0
  92. package/dist/hooks.test-d.d.ts +14 -0
  93. package/dist/inbox-protocol.d.ts +128 -0
  94. package/dist/inbox.d.ts +132 -0
  95. package/dist/index.d.ts +126 -29
  96. package/dist/index.js +5500 -15
  97. package/dist/internal.d.ts +43 -0
  98. package/dist/internal.js +8 -0
  99. package/dist/linked-client.d.ts +70 -0
  100. package/dist/message-list-DHZcLhRA.js +682 -0
  101. package/dist/notice-player.d.ts +70 -0
  102. package/dist/page.d.ts +114 -0
  103. package/dist/phone.d.ts +50 -0
  104. package/dist/route-fetch.d.ts +79 -0
  105. package/dist/session/audio-effects.d.ts +37 -0
  106. package/dist/session/audio-setup.d.ts +57 -0
  107. package/dist/session/audio-state.d.ts +157 -0
  108. package/dist/session/browser-session.d.ts +39 -0
  109. package/dist/session/close.d.ts +26 -0
  110. package/dist/session/dial.d.ts +65 -0
  111. package/dist/session/handshake.d.ts +60 -0
  112. package/dist/session/index.d.ts +17 -0
  113. package/dist/session/index.js +2 -0
  114. package/dist/session/messages.d.ts +63 -0
  115. package/dist/session/mic.d.ts +64 -0
  116. package/dist/session/preconnect.d.ts +48 -0
  117. package/dist/session/reconnect.d.ts +25 -0
  118. package/dist/session/resume-store.d.ts +57 -0
  119. package/dist/session/state.d.ts +146 -0
  120. package/dist/session/ticket.d.ts +44 -0
  121. package/dist/session/types.d.ts +476 -0
  122. package/dist/session/url.d.ts +17 -0
  123. package/dist/session/user-turn.d.ts +55 -0
  124. package/dist/session-D9jM5-TF.js +2140 -0
  125. package/dist/stored-value.d.ts +103 -0
  126. package/dist/tool-call-block-uXtMOUWN.js +204 -0
  127. package/dist/tool-config-context-Es4YUzV2.js +19 -0
  128. package/dist/types.d.ts +272 -34
  129. package/dist/types.js +35 -4
  130. package/dist/upload/files.d.ts +83 -0
  131. package/dist/upload/index.d.ts +15 -0
  132. package/dist/upload/pause.d.ts +31 -0
  133. package/dist/upload/recall.d.ts +67 -0
  134. package/dist/upload/report.d.ts +26 -0
  135. package/dist/upload/session.d.ts +99 -0
  136. package/dist/url-chips-oyy_Eiqh.js +239 -0
  137. package/dist/use-client-runs.d.ts +87 -0
  138. package/dist/use-conversation-log.d.ts +86 -0
  139. package/dist/use-conversation.d.ts +131 -0
  140. package/dist/use-copy.d.ts +74 -0
  141. package/dist/use-download-url.d.ts +84 -0
  142. package/dist/use-flash.d.ts +61 -0
  143. package/dist/use-inbox.d.ts +101 -0
  144. package/dist/use-push-to-talk.d.ts +105 -0
  145. package/dist/use-route-mutation.d.ts +113 -0
  146. package/dist/use-route.d.ts +82 -0
  147. package/dist/use-run-key.d.ts +133 -0
  148. package/dist/use-session-controls.d.ts +69 -0
  149. package/dist/use-session-id.d.ts +57 -0
  150. package/dist/use-tap-to-talk.d.ts +138 -0
  151. package/dist/use-user-transcript-uyHhzy4d.js +73 -0
  152. package/dist/use-user-transcript.d.ts +75 -0
  153. package/dist/use-workflow-form.d.ts +285 -0
  154. package/dist/use-workflow-progress.d.ts +137 -0
  155. package/dist/use-workflow-run-DdhWmbx-.js +697 -0
  156. package/dist/use-workflow-run.d.ts +100 -0
  157. package/dist/use-workflow-runs.d.ts +71 -0
  158. package/dist/use-workflow-stream.d.ts +154 -0
  159. package/dist/use-workflows.d.ts +77 -0
  160. package/dist/workflow-client.d.ts +136 -0
  161. package/dist/workflow-def-types.d.ts +31 -0
  162. package/dist/workflow-events.d.ts +39 -0
  163. package/dist/workflow-field-kind.d.ts +95 -0
  164. package/dist/workflow-status-labels.d.ts +35 -0
  165. package/dist/worklets/_module-url.d.ts +10 -0
  166. package/dist/worklets/_playback-bench-harness.d.ts +181 -0
  167. package/dist/worklets/_playback-bench-host.d.ts +63 -0
  168. package/dist/worklets/_playback-bench-page.d.ts +65 -0
  169. package/dist/worklets/_tts-trace-harness.d.ts +142 -0
  170. package/dist/worklets/_worklet-test-utils.d.ts +27 -0
  171. package/dist/worklets/capture-processor.d.ts +4 -2
  172. package/dist/worklets/capture-processor.js +73 -33
  173. package/dist/worklets/playback-processor.d.ts +4 -2
  174. package/dist/worklets/playback-processor.js +258 -42
  175. package/package.json +59 -32
  176. package/styles.css +152 -30
  177. package/dist/_jsdom-setup.d.ts +0 -0
  178. package/dist/audio.d.ts +0 -45
  179. package/dist/audio.js +0 -121
  180. package/dist/client-context.d.ts +0 -33
  181. package/dist/client-context.js +0 -15
  182. package/dist/client-handler.d.ts +0 -43
  183. package/dist/components/app.d.ts +0 -23
  184. package/dist/components/app.js +0 -41
  185. package/dist/components/error-banner.d.ts +0 -21
  186. package/dist/components/error-banner.js +0 -27
  187. package/dist/components/message-bubble.d.ts +0 -23
  188. package/dist/components/message-bubble.js +0 -35
  189. package/dist/components/state-indicator.d.ts +0 -29
  190. package/dist/components/state-indicator.js +0 -37
  191. package/dist/components/thinking-indicator.d.ts +0 -16
  192. package/dist/components/thinking-indicator.js +0 -34
  193. package/dist/components/tool-icons.d.ts +0 -17
  194. package/dist/components/tool-icons.js +0 -128
  195. package/dist/components/transcript.d.ts +0 -25
  196. package/dist/components/transcript.js +0 -35
  197. package/dist/session-CWB7vmqz.js +0 -429
  198. package/dist/session.d.ts +0 -69
  199. package/dist/session.js +0 -2
  200. package/dist/signals.d.ts +0 -120
  201. package/dist/signals.js +0 -161
  202. package/dist/tsdown.config.d.ts +0 -2
  203. package/dist/types.test-d.d.ts +0 -7
@@ -0,0 +1,431 @@
1
+ import { CAPTURE_STOP_ACK_TIMEOUT_MS, MIC_BUFFER_SECONDS, PLAYBACK_DONE_MAX_WAIT_MS, PLAYBACK_DONE_POLL_MS, VOICE_CAPTURE_CONSTRAINTS } from "../types.js";
2
+ //#region src/audio/capture.ts
3
+ /**
4
+ * The capture half's primitives, shared by the audio path (`audio/voice-io.ts`) and
5
+ * the pre-connect capture (`audio/preconnect.ts`): the rate assertion, the
6
+ * failed-init release, and the capture worklet node's port protocol.
7
+ *
8
+ * @module
9
+ */
10
+ /**
11
+ * Throw unless the browser honored a requested context sample rate. The
12
+ * requested rates are never advisory: captured audio is tagged with the
13
+ * requested rate on the wire, so a context running at some other rate ships
14
+ * audio that only sounds like speech to the wrong decoder. Every capture
15
+ * path must call this after context creation.
16
+ */
17
+ function assertGranted(granted, requested, side) {
18
+ if (granted === requested) return;
19
+ throw new Error(`Browser refused the ${side} sample rate: asked for ${requested} Hz, got ${granted} Hz`);
20
+ }
21
+ /**
22
+ * The error a dead worklet processor reports.
23
+ *
24
+ * A processor exception permanently kills the node — no further messages or
25
+ * audio will ever arrive — so both sides log it and hand it on. One spelling of
26
+ * that, because the pair differed only in the word "capture"/"playback" and in
27
+ * what the playback side has to settle afterwards.
28
+ */
29
+ function workletCrash(side) {
30
+ const err = /* @__PURE__ */ new Error(`Audio ${side} worklet crashed`);
31
+ console.error("[aai-ui]", err.message);
32
+ return err;
33
+ }
34
+ /**
35
+ * Release a microphone that was (or later gets) granted while another init
36
+ * step failed; if `getUserMedia` itself rejected, this is a no-op. Without
37
+ * it, a mic granted after a failed init keeps the browser's recording
38
+ * indicator lit with no way to turn it off.
39
+ */
40
+ function releaseStreamOnFailure(streamPromise) {
41
+ streamPromise.then((s) => {
42
+ for (const t of s.getTracks()) t.stop();
43
+ }).catch(() => {});
44
+ }
45
+ /**
46
+ * Wire one capture worklet node: node construction, the chunk/silent/stopped
47
+ * port protocol, and the stop→ack handshake. No `processorOptions`: the
48
+ * worklet reads the context rate from its global scope (callers assert the
49
+ * granted rate first) and owns its own batching default — re-spelling
50
+ * defaults caller-side is drift.
51
+ */
52
+ function createCaptureNode(ctx, onChunk, onSilent) {
53
+ const node = new AudioWorkletNode(ctx, "capture-processor", {
54
+ channelCount: 1,
55
+ channelCountMode: "explicit"
56
+ });
57
+ let onStopped = null;
58
+ node.port.onmessage = (e) => {
59
+ const d = e.data;
60
+ if (d.event === "chunk" && d.buffer) onChunk(d.buffer);
61
+ else if (d.event === "silent") onSilent?.();
62
+ else if (d.event === "stopped") {
63
+ onStopped?.();
64
+ onStopped = null;
65
+ }
66
+ };
67
+ return {
68
+ node,
69
+ start() {
70
+ node.port.postMessage({ event: "start" });
71
+ },
72
+ stop() {
73
+ const { promise, resolve } = Promise.withResolvers();
74
+ const cap = setTimeout(resolve, CAPTURE_STOP_ACK_TIMEOUT_MS);
75
+ onStopped = () => {
76
+ clearTimeout(cap);
77
+ resolve();
78
+ };
79
+ node.port.postMessage({ event: "stop" });
80
+ return promise;
81
+ }
82
+ };
83
+ }
84
+ //#endregion
85
+ //#region src/audio/preconnect.ts
86
+ /**
87
+ * Pre-connect capture: the microphone opened before the session's audio rates
88
+ * are known, and the resample that fits its buffer to the rate the server
89
+ * then names. `createVoiceIO` (`audio/voice-io.ts`) is what takes it over.
90
+ *
91
+ * @module
92
+ */
93
+ /**
94
+ * Open a {@link PreConnectCapture}: ask for the microphone and start buffering
95
+ * at `sampleRate`, keeping at most the latest `maxSeconds` — the most RECENT
96
+ * audio rather than the first, so the buffer always runs contiguously into
97
+ * the live stream that follows it.
98
+ *
99
+ * Rejects like {@link createVoiceIO} does (a denied prompt, a refused rate);
100
+ * the caller falls back to opening the path the ordinary way.
101
+ */
102
+ async function openPreConnectCapture(opts) {
103
+ const { sampleRate, captureWorkletSrc, maxSeconds } = opts;
104
+ const ctx = new AudioContext({
105
+ sampleRate,
106
+ latencyHint: "interactive"
107
+ });
108
+ const streamPromise = navigator.mediaDevices.getUserMedia({ audio: {
109
+ deviceId: { ideal: "default" },
110
+ ...VOICE_CAPTURE_CONSTRAINTS
111
+ } });
112
+ let stream;
113
+ try {
114
+ [stream] = await Promise.all([
115
+ streamPromise,
116
+ ctx.resume(),
117
+ ctx.audioWorklet.addModule(captureWorkletSrc)
118
+ ]);
119
+ assertGranted(ctx.sampleRate, sampleRate, "capture");
120
+ } catch (err) {
121
+ releaseStreamOnFailure(streamPromise);
122
+ await ctx.close().catch(() => {});
123
+ throw err;
124
+ }
125
+ const maxBytes = Math.round(sampleRate * maxSeconds) * 2;
126
+ let chunks = [];
127
+ let bytes = 0;
128
+ let silent = false;
129
+ let sinks = null;
130
+ const mic = ctx.createMediaStreamSource(stream);
131
+ const capture = createCaptureNode(ctx, (pcm16) => {
132
+ if (sinks) {
133
+ sinks.onChunk(pcm16);
134
+ return;
135
+ }
136
+ chunks.push(pcm16);
137
+ bytes += pcm16.byteLength;
138
+ while (bytes > maxBytes && chunks.length > 1) bytes -= chunks.shift()?.byteLength ?? 0;
139
+ }, () => {
140
+ if (sinks) sinks.onSilent?.();
141
+ else silent = true;
142
+ });
143
+ mic.connect(capture.node);
144
+ capture.start();
145
+ /** Take the buffer, leaving it empty. */
146
+ const takeChunks = () => {
147
+ const taken = chunks;
148
+ chunks = [];
149
+ bytes = 0;
150
+ return taken;
151
+ };
152
+ let closed = false;
153
+ const closeCtx = async () => {
154
+ if (closed) return;
155
+ closed = true;
156
+ mic.disconnect();
157
+ capture.node.disconnect();
158
+ await ctx.close().catch((err) => {
159
+ console.warn("AudioContext close failed:", err);
160
+ });
161
+ };
162
+ return {
163
+ sampleRate,
164
+ stream,
165
+ ctx,
166
+ handOff(next) {
167
+ sinks = next;
168
+ const buffered = takeChunks();
169
+ if (buffered.length > 0) next.onBuffered(buffered);
170
+ if (silent) next.onSilent?.();
171
+ return {
172
+ mic,
173
+ capture
174
+ };
175
+ },
176
+ async detach() {
177
+ await capture.stop();
178
+ await closeCtx();
179
+ return {
180
+ chunks: takeChunks(),
181
+ sampleRate
182
+ };
183
+ },
184
+ async close() {
185
+ if (!closed) await capture.stop();
186
+ for (const t of stream.getTracks()) t.stop();
187
+ await closeCtx();
188
+ }
189
+ };
190
+ }
191
+ /** Float32 → PCM16, clamped and scaled as the capture worklet does. */
192
+ function encodePcm16(samples) {
193
+ const pcm = new Int16Array(samples.length);
194
+ for (let i = 0; i < samples.length; i++) {
195
+ const s = Math.max(-1, Math.min(1, samples[i] ?? 0));
196
+ pcm[i] = s < 0 ? s * 32768 : s * 32767;
197
+ }
198
+ return pcm.buffer;
199
+ }
200
+ /**
201
+ * Convert buffered PCM16 from one rate to another with the browser's own
202
+ * band-limited resampler (an `OfflineAudioContext` render), re-cut into
203
+ * {@link MIC_BUFFER_SECONDS} chunks so the burst matches the live cadence.
204
+ *
205
+ * Never rejects: pre-connect audio is a bonus, and a browser that cannot
206
+ * render it offline loses the bonus rather than the session.
207
+ */
208
+ async function resamplePcm16(chunks, fromRate, toRate) {
209
+ const total = chunks.reduce((n, c) => n + c.byteLength / 2, 0);
210
+ if (total === 0) return [];
211
+ if (fromRate === toRate) return chunks;
212
+ try {
213
+ const outLength = Math.round(total * toRate / fromRate);
214
+ const offline = new OfflineAudioContext(1, outLength, toRate);
215
+ const input = offline.createBuffer(1, total, fromRate);
216
+ const samples = input.getChannelData(0);
217
+ let at = 0;
218
+ for (const chunk of chunks) for (const v of new Int16Array(chunk)) samples[at++] = v / 32768;
219
+ const source = offline.createBufferSource();
220
+ source.buffer = input;
221
+ source.connect(offline.destination);
222
+ source.start();
223
+ const rendered = (await offline.startRendering()).getChannelData(0);
224
+ const size = Math.max(1, Math.round(toRate * MIC_BUFFER_SECONDS));
225
+ const out = [];
226
+ for (let start = 0; start < rendered.length; start += size) out.push(encodePcm16(rendered.subarray(start, start + size)));
227
+ return out;
228
+ } catch (err) {
229
+ console.warn("[aai-ui] pre-connect audio dropped: resample failed", err);
230
+ return [];
231
+ }
232
+ }
233
+ /**
234
+ * Wire and start a fresh capture node on `ctx`, first flushing any audio a
235
+ * detached {@link PreConnectCapture} held, resampled to `ctx`'s rate.
236
+ */
237
+ async function startCapture(ctx, stream, opts) {
238
+ const mic = ctx.createMediaStreamSource(stream);
239
+ const capture = createCaptureNode(ctx, opts.onMicData, opts.onMicSilent);
240
+ mic.connect(capture.node);
241
+ if (opts.buffered) {
242
+ const { chunks, sampleRate } = opts.buffered;
243
+ const resampled = await resamplePcm16(chunks, sampleRate, ctx.sampleRate);
244
+ if (resampled.length > 0) opts.onBuffered(resampled);
245
+ }
246
+ capture.start();
247
+ return {
248
+ mic,
249
+ capture
250
+ };
251
+ }
252
+ //#endregion
253
+ //#region src/audio/voice-io.ts
254
+ /**
255
+ * Create a {@link VoiceIO} instance that captures microphone audio and
256
+ * plays back TTS audio using the Web Audio API.
257
+ *
258
+ * Playback runs on a context at the TTS sample rate for fidelity, and capture
259
+ * on its own context at the STT rate so the *browser* performs the rate
260
+ * conversion with its band-limited resampler. The two collapse into one
261
+ * context when the rates match. A browser that declines either requested rate
262
+ * fails init rather than falling back to converting in the worklet, which
263
+ * would alias.
264
+ *
265
+ * @param opts - Voice I/O configuration options.
266
+ * @returns A promise that resolves to a {@link VoiceIO} handle.
267
+ * @throws If microphone access is denied or AudioWorklet registration fails.
268
+ */
269
+ async function createVoiceIO(opts) {
270
+ const { sttSampleRate, ttsSampleRate, captureWorkletSrc, playbackWorkletSrc, onMicData, onError, onPlaybackStats, onPlaybackProgress, onMicSilent, preConnect } = opts;
271
+ const onPreConnectAudio = opts.onPreConnectAudio ?? ((chunks) => {
272
+ for (const chunk of chunks) onMicData(chunk);
273
+ });
274
+ const adopted = preConnect?.sampleRate === sttSampleRate ? preConnect : null;
275
+ const detached = preConnect && !adopted ? await preConnect.detach() : null;
276
+ const ctx = new AudioContext({
277
+ sampleRate: ttsSampleRate,
278
+ latencyHint: "playback"
279
+ });
280
+ const sharesContext = !adopted && sttSampleRate === ttsSampleRate;
281
+ const capCtx = adopted?.ctx ?? (sharesContext ? ctx : new AudioContext({
282
+ sampleRate: sttSampleRate,
283
+ latencyHint: "interactive"
284
+ }));
285
+ async function closeContexts() {
286
+ const contexts = sharesContext ? [ctx] : [ctx, capCtx];
287
+ await Promise.all(contexts.map((c) => c.close().catch((err) => {
288
+ console.warn("AudioContext close failed:", err);
289
+ })));
290
+ }
291
+ const streamPromise = preConnect ? Promise.resolve(preConnect.stream) : navigator.mediaDevices.getUserMedia({ audio: {
292
+ deviceId: { ideal: "default" },
293
+ ...VOICE_CAPTURE_CONSTRAINTS
294
+ } });
295
+ let stream;
296
+ try {
297
+ [stream] = await Promise.all([
298
+ streamPromise,
299
+ ctx.resume(),
300
+ capCtx.resume(),
301
+ adopted ? void 0 : capCtx.audioWorklet.addModule(captureWorkletSrc),
302
+ ctx.audioWorklet.addModule(playbackWorkletSrc)
303
+ ]);
304
+ assertGranted(capCtx.sampleRate, sttSampleRate, "capture");
305
+ assertGranted(ctx.sampleRate, ttsSampleRate, "playback");
306
+ } catch (err) {
307
+ releaseStreamOnFailure(streamPromise);
308
+ await closeContexts();
309
+ throw err;
310
+ }
311
+ const { mic, capture } = adopted ? adopted.handOff({
312
+ onBuffered: onPreConnectAudio,
313
+ onChunk: onMicData,
314
+ onSilent: onMicSilent
315
+ }) : await startCapture(capCtx, stream, {
316
+ onMicData,
317
+ onMicSilent,
318
+ buffered: detached,
319
+ onBuffered: onPreConnectAudio
320
+ });
321
+ capture.node.onprocessorerror = () => onError?.(workletCrash("capture"));
322
+ let playNode = null;
323
+ /**
324
+ * Turn ids for the drain handshake. Every `done()` posts a fresh id and the
325
+ * worklet echoes it on the matching 'stop', so a stop the worklet posted for
326
+ * an EARLIER turn — already in flight when a barge-in flushed that turn —
327
+ * cannot settle the current turn's wait. Dropping only `reason: 'interrupt'`
328
+ * stops was not enough: the stale one is a legitimate drain stop, just for a
329
+ * turn the host has already moved past, and settling on it reports the live
330
+ * reply finished while it is still speaking.
331
+ */
332
+ let turnSeq = 0;
333
+ /**
334
+ * The drain wait in flight, if any.
335
+ *
336
+ * ONE field, because the resolver and the turn it belongs to are one fact:
337
+ * as two independently-nulled variables they could disagree, and did — the
338
+ * suspended-context branch of `done()` cleared the turn and left the resolver
339
+ * set. A single nullable makes that unrepresentable.
340
+ */
341
+ let pendingStop = null;
342
+ const lifecycle = new AbortController();
343
+ /** Settle whatever drain wait is pending and forget the turn it belonged to. */
344
+ function settlePendingStop() {
345
+ const pending = pendingStop;
346
+ pendingStop = null;
347
+ pending?.settle();
348
+ }
349
+ function onWorkletStop(msg) {
350
+ if (msg.stats && msg.stats.concealedSamples > 0) onPlaybackStats?.(msg.stats);
351
+ if (msg.reason === "interrupt") return;
352
+ if (pendingStop !== null && msg.turn !== pendingStop.turn) return;
353
+ settlePendingStop();
354
+ }
355
+ function ensurePlayNode() {
356
+ if (playNode) return playNode;
357
+ const node = new AudioWorkletNode(ctx, "playback-processor");
358
+ node.connect(ctx.destination);
359
+ node.port.onmessage = (e) => {
360
+ if (e.data.event === "stop") onWorkletStop(e.data);
361
+ else if (e.data.event === "progress") onPlaybackProgress?.(e.data.bufferedMs);
362
+ };
363
+ node.onprocessorerror = () => {
364
+ const err = workletCrash("playback");
365
+ settlePendingStop();
366
+ onError?.(err);
367
+ };
368
+ playNode = node;
369
+ return node;
370
+ }
371
+ const io = {
372
+ enqueue(pcm16Buffer) {
373
+ if (lifecycle.signal.aborted) return;
374
+ if (pcm16Buffer.byteLength === 0) return;
375
+ ensurePlayNode().port.postMessage({
376
+ event: "write",
377
+ buffer: new Uint8Array(pcm16Buffer)
378
+ }, [pcm16Buffer]);
379
+ },
380
+ done() {
381
+ if (!playNode) return Promise.resolve();
382
+ const turn = ++turnSeq;
383
+ playNode.port.postMessage({
384
+ event: "done",
385
+ turn
386
+ });
387
+ if (ctx.state !== "running") {
388
+ settlePendingStop();
389
+ return Promise.resolve();
390
+ }
391
+ const { promise, resolve } = Promise.withResolvers();
392
+ settlePendingStop();
393
+ const settle = () => {
394
+ clearInterval(poll);
395
+ clearTimeout(cap);
396
+ if (pendingStop?.settle === settle) pendingStop = null;
397
+ resolve();
398
+ };
399
+ const poll = setInterval(() => {
400
+ if (ctx.state !== "running") settle();
401
+ }, PLAYBACK_DONE_POLL_MS);
402
+ const cap = setTimeout(settle, PLAYBACK_DONE_MAX_WAIT_MS);
403
+ pendingStop = {
404
+ turn,
405
+ settle
406
+ };
407
+ return promise;
408
+ },
409
+ flush() {
410
+ if (!playNode) return;
411
+ settlePendingStop();
412
+ playNode.port.postMessage({ event: "interrupt" });
413
+ },
414
+ async close() {
415
+ if (lifecycle.signal.aborted) return;
416
+ lifecycle.abort();
417
+ await capture.stop();
418
+ mic.disconnect();
419
+ capture.node.disconnect();
420
+ if (playNode) playNode.disconnect();
421
+ for (const t of stream.getTracks()) t.stop();
422
+ await closeContexts();
423
+ },
424
+ async [Symbol.asyncDispose]() {
425
+ await io.close();
426
+ }
427
+ };
428
+ return io;
429
+ }
430
+ //#endregion
431
+ export { createVoiceIO, openPreConnectCapture };
@@ -0,0 +1,87 @@
1
+ /**
2
+ * Pre-connect capture: the microphone opened before the session's audio rates
3
+ * are known, and the resample that fits its buffer to the rate the server
4
+ * then names. `createVoiceIO` (`audio/voice-io.ts`) is what takes it over.
5
+ *
6
+ * @module
7
+ */
8
+ import { type CaptureNode } from "./capture.ts";
9
+ /**
10
+ * A microphone opened BEFORE the session's audio rates are known, buffering
11
+ * what the caller says until {@link createVoiceIO} takes it over.
12
+ *
13
+ * The connect-time half of pre-connect audio: the server names its STT rate
14
+ * only in its `config` frame, one handshake (and on the platform, possibly a
15
+ * sandbox boot) after the user pressed start — and an eager caller is talking
16
+ * by then. This captures at a guessed rate; {@link createVoiceIO} adopts it
17
+ * whole when the guess was right and resamples the buffer when it was not.
18
+ */
19
+ export type PreConnectCapture = {
20
+ /** The rate this capture runs at — the guess. */
21
+ readonly sampleRate: number;
22
+ /** The granted microphone, handed to the {@link VoiceIO} that takes over. */
23
+ readonly stream: MediaStream;
24
+ /** The capture context, adopted as the path's own when the rate matches. */
25
+ readonly ctx: AudioContext;
26
+ /**
27
+ * Adopt the running capture: flush the buffer to `onBuffered`, then route
28
+ * every later frame to `onChunk` — synchronously, so nothing interleaves.
29
+ */
30
+ handOff(sinks: {
31
+ onBuffered: (chunks: ArrayBuffer[]) => void;
32
+ onChunk: (pcm16: ArrayBuffer) => void;
33
+ onSilent?: (() => void) | undefined;
34
+ }): {
35
+ mic: MediaStreamAudioSourceNode;
36
+ capture: CaptureNode;
37
+ };
38
+ /**
39
+ * Stop capturing (after the worklet's tail flush) and close the context,
40
+ * keeping the stream: for a path at another rate. Never rejects.
41
+ */
42
+ detach(): Promise<{
43
+ chunks: ArrayBuffer[];
44
+ sampleRate: number;
45
+ }>;
46
+ /** Release everything, stream included. Best-effort, idempotent. */
47
+ close(): Promise<void>;
48
+ };
49
+ /**
50
+ * Open a {@link PreConnectCapture}: ask for the microphone and start buffering
51
+ * at `sampleRate`, keeping at most the latest `maxSeconds` — the most RECENT
52
+ * audio rather than the first, so the buffer always runs contiguously into
53
+ * the live stream that follows it.
54
+ *
55
+ * Rejects like {@link createVoiceIO} does (a denied prompt, a refused rate);
56
+ * the caller falls back to opening the path the ordinary way.
57
+ */
58
+ export declare function openPreConnectCapture(opts: {
59
+ sampleRate: number;
60
+ captureWorkletSrc: string;
61
+ maxSeconds: number;
62
+ }): Promise<PreConnectCapture>;
63
+ /**
64
+ * Convert buffered PCM16 from one rate to another with the browser's own
65
+ * band-limited resampler (an `OfflineAudioContext` render), re-cut into
66
+ * {@link MIC_BUFFER_SECONDS} chunks so the burst matches the live cadence.
67
+ *
68
+ * Never rejects: pre-connect audio is a bonus, and a browser that cannot
69
+ * render it offline loses the bonus rather than the session.
70
+ */
71
+ export declare function resamplePcm16(chunks: ArrayBuffer[], fromRate: number, toRate: number): Promise<ArrayBuffer[]>;
72
+ /**
73
+ * Wire and start a fresh capture node on `ctx`, first flushing any audio a
74
+ * detached {@link PreConnectCapture} held, resampled to `ctx`'s rate.
75
+ */
76
+ export declare function startCapture(ctx: AudioContext, stream: MediaStream, opts: {
77
+ onMicData: (pcm16: ArrayBuffer) => void;
78
+ onMicSilent?: (() => void) | undefined;
79
+ buffered: {
80
+ chunks: ArrayBuffer[];
81
+ sampleRate: number;
82
+ } | null;
83
+ onBuffered: (chunks: ArrayBuffer[]) => void;
84
+ }): Promise<{
85
+ mic: MediaStreamAudioSourceNode;
86
+ capture: CaptureNode;
87
+ }>;
@@ -0,0 +1,112 @@
1
+ import { type PreConnectCapture } from "./preconnect.ts";
2
+ /**
3
+ * How much of one turn's playback was covered by concealment rather than
4
+ * received audio — the playback worklet's underrun report, in the shape
5
+ * WebRTC's `inbound-rtp` audio stats use, so the numbers mean the same thing
6
+ * here as in a `getStats()` dump.
7
+ *
8
+ * A turn with `concealmentEvents: 0` never needed its jitter buffer; a turn
9
+ * with a high `silentConcealedSamples` share starved for longer than
10
+ * concealment can plausibly cover, which is a bandwidth problem rather than a
11
+ * buffer-tuning one.
12
+ *
13
+ * @public
14
+ */
15
+ export type PlaybackStats = {
16
+ /** Samples emitted to cover a gap, including the silent ones. */
17
+ concealedSamples: number;
18
+ /** The subset of {@link PlaybackStats.concealedSamples} that were silence. */
19
+ silentConcealedSamples: number;
20
+ /** Distinct underrun episodes, however many render quanta each spanned. */
21
+ concealmentEvents: number;
22
+ /** Episodes that lasted long enough to decay to silence. */
23
+ silentConcealmentEvents: number;
24
+ };
25
+ /** Configuration for creating a {@link VoiceIO} instance. */
26
+ export type VoiceIOOptions = {
27
+ /** Sample rate in Hz expected by the STT engine (e.g. 16000). */
28
+ sttSampleRate: number;
29
+ /** Sample rate in Hz used by the TTS engine (e.g. 22050). */
30
+ ttsSampleRate: number;
31
+ /** Source URL or data URI for the capture AudioWorklet processor. */
32
+ captureWorkletSrc: string;
33
+ /** Source URL or data URI for the playback AudioWorklet processor. */
34
+ playbackWorkletSrc: string;
35
+ /** Callback invoked with buffered PCM16 microphone data to send to the server. */
36
+ onMicData: (pcm16: ArrayBuffer) => void;
37
+ /**
38
+ * Called when an AudioWorklet processor throws and is killed by the browser
39
+ * (the node produces no further audio or messages), so the session can
40
+ * transition out of listening/speaking instead of looking healthy forever.
41
+ */
42
+ onError?: ((err: Error) => void) | undefined;
43
+ /**
44
+ * Called at the end of any turn whose playback had to conceal a gap. Never
45
+ * called for a clean turn, so it can be wired straight to a warning.
46
+ */
47
+ onPlaybackStats?: ((stats: PlaybackStats) => void) | undefined;
48
+ /**
49
+ * Called every {@link PLAYBACK_PROGRESS_INTERVAL_MS} while the playback
50
+ * buffer holds unplayed agent audio, with its depth in ms. Wire it to the
51
+ * session's `playback_progress` frame: it is the host's only closed-loop
52
+ * view of playback, and without it the host assumes every chunk it forwards
53
+ * starts playing on arrival at exactly 1.0x — so a client whose buffer runs
54
+ * ahead of the wall clock is told the line is silent while the caller is
55
+ * still listening. Unwired, the host degrades to that estimate.
56
+ */
57
+ onPlaybackProgress?: ((bufferedMs: number) => void) | undefined;
58
+ /**
59
+ * Called once if the microphone delivers nothing but digital silence for
60
+ * the first {@link MIC_SILENCE_PROBE_MS} of capture — a muted or wrong input
61
+ * device, which otherwise looks exactly like a user who hasn't spoken.
62
+ */
63
+ onMicSilent?: (() => void) | undefined;
64
+ /**
65
+ * A microphone already opened by {@link openPreConnectCapture}, whose buffered
66
+ * audio is the start of this path's capture. Ownership passes to the
67
+ * returned {@link VoiceIO}: its `close()` stops the stream.
68
+ */
69
+ preConnect?: PreConnectCapture | undefined;
70
+ /**
71
+ * Called once, before any live frame reaches `onMicData`, with the audio
72
+ * {@link VoiceIOOptions.preConnect} held — already at `sttSampleRate`, in
73
+ * order. Separate from `onMicData` because it is a deliberate burst the
74
+ * sender's backpressure check must not drop. Unwired, it goes to
75
+ * `onMicData` chunk by chunk.
76
+ */
77
+ onPreConnectAudio?: ((chunks: ArrayBuffer[]) => void) | undefined;
78
+ };
79
+ /**
80
+ * Audio I/O interface for voice capture and playback.
81
+ *
82
+ * Manages microphone capture via an AudioWorklet and TTS audio playback
83
+ * through a second AudioWorklet. Implements {@link AsyncDisposable} for
84
+ * resource cleanup.
85
+ */
86
+ export type VoiceIO = AsyncDisposable & {
87
+ /** Enqueue a PCM16 audio buffer for playback through the TTS pipeline. */
88
+ enqueue(pcm16Buffer: ArrayBuffer): void;
89
+ /** Signal that all TTS audio for the current turn has been enqueued.
90
+ * Resolves when the worklet has finished playing all buffered audio. */
91
+ done(): Promise<void>;
92
+ /** Immediately stop playback and discard any buffered TTS audio. */
93
+ flush(): void;
94
+ /** Release all audio resources (microphone, AudioContext, worklets). */
95
+ close(): Promise<void>;
96
+ };
97
+ /**
98
+ * Create a {@link VoiceIO} instance that captures microphone audio and
99
+ * plays back TTS audio using the Web Audio API.
100
+ *
101
+ * Playback runs on a context at the TTS sample rate for fidelity, and capture
102
+ * on its own context at the STT rate so the *browser* performs the rate
103
+ * conversion with its band-limited resampler. The two collapse into one
104
+ * context when the rates match. A browser that declines either requested rate
105
+ * fails init rather than falling back to converting in the worklet, which
106
+ * would alias.
107
+ *
108
+ * @param opts - Voice I/O configuration options.
109
+ * @returns A promise that resolves to a {@link VoiceIO} handle.
110
+ * @throws If microphone access is denied or AudioWorklet registration fails.
111
+ */
112
+ export declare function createVoiceIO(opts: VoiceIOOptions): Promise<VoiceIO>;
@@ -0,0 +1 @@
1
+ export {};