@alexkroman1/aai-ui 0.12.3 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +253 -0
- package/dist/_colors-CpZO-88A.js +78 -0
- package/dist/_module-url-C13kAJ87.js +15 -0
- package/dist/_page-shell.d.ts +11 -0
- package/dist/_recover-run.d.ts +79 -0
- package/dist/_repeat-until.d.ts +30 -0
- package/dist/_run-controls.d.ts +33 -0
- package/dist/_submission-state.d.ts +92 -0
- package/dist/_tap-to-talk-state.d.ts +88 -0
- package/dist/_utils-Mi-SCFA1.js +47 -0
- package/dist/_utils.d.ts +26 -0
- package/dist/_web-storage.d.ts +43 -0
- package/dist/_workflow-api-ref.d.ts +37 -0
- package/dist/_workflow-files.d.ts +34 -0
- package/dist/_workflow-test-defs.d.ts +34 -0
- package/dist/aai-logo-D8pMVn9_.js +85 -0
- package/dist/agent-state-labels.d.ts +65 -0
- package/dist/agent-state.d.ts +108 -0
- package/dist/audio/capture.d.ts +44 -0
- package/dist/audio/index.d.ts +18 -0
- package/dist/audio/index.js +431 -0
- package/dist/audio/preconnect.d.ts +87 -0
- package/dist/audio/voice-io.d.ts +112 -0
- package/dist/build-default-client.d.ts +1 -0
- package/dist/chat-view-BCbI7gD4.js +396 -0
- package/dist/client-config-C7fQrioM.js +139 -0
- package/dist/client-config.d.ts +119 -0
- package/dist/client-dir.d.ts +44 -0
- package/dist/client-dir.js +58 -0
- package/dist/client-identity.d.ts +131 -0
- package/dist/client-tool.d.ts +51 -0
- package/dist/components/_colors.d.ts +73 -0
- package/dist/components/_form-readiness.d.ts +56 -0
- package/dist/components/_form-values.d.ts +19 -0
- package/dist/components/aai-logo.d.ts +12 -0
- package/dist/components/audio-result.d.ts +94 -0
- package/dist/components/auto-scroll.d.ts +74 -0
- package/dist/components/bullet-list.d.ts +74 -0
- package/dist/components/button.d.ts +29 -21
- package/dist/components/button.js +62 -23
- package/dist/components/chat-view.d.ts +24 -18
- package/dist/components/chat-view.js +2 -60
- package/dist/components/console-shell.d.ts +85 -0
- package/dist/components/controls.d.ts +21 -6
- package/dist/components/controls.js +31 -19
- package/dist/components/conversation-view.d.ts +130 -0
- package/dist/components/eyebrow.d.ts +12 -0
- package/dist/components/facts.d.ts +81 -0
- package/dist/components/form-fields.d.ts +142 -0
- package/dist/components/form-types.d.ts +73 -0
- package/dist/components/form.d.ts +96 -0
- package/dist/components/markdown.d.ts +57 -0
- package/dist/components/message-list.d.ts +34 -7
- package/dist/components/message-list.js +2 -75
- package/dist/components/session-controls.d.ts +125 -0
- package/dist/components/session-error-banner.d.ts +69 -0
- package/dist/components/session-state-dot.d.ts +111 -0
- package/dist/components/sidebar-layout.d.ts +31 -10
- package/dist/components/sidebar-layout.js +32 -12
- package/dist/components/start-screen.d.ts +17 -6
- package/dist/components/start-screen.js +34 -14
- package/dist/components/tool-call-block.d.ts +15 -14
- package/dist/components/tool-call-block.js +2 -117
- package/dist/components/tool-call-row.d.ts +75 -0
- package/dist/components/tool-config-context.d.ts +22 -0
- package/dist/components/upload-progress.d.ts +76 -0
- package/dist/components/url-chips.d.ts +29 -0
- package/dist/components/workflow-fields.d.ts +46 -0
- package/dist/components/workflow-pending-note.d.ts +81 -0
- package/dist/components/workflow-progress.d.ts +81 -0
- package/dist/components/workflow-run-error.d.ts +48 -0
- package/dist/components/workflow-run-panel.d.ts +79 -0
- package/dist/context.d.ts +288 -0
- package/dist/context.js +382 -0
- package/dist/conversation-log.d.ts +88 -0
- package/dist/default-client/assets/_module-url-BX0RuRU2.js +1 -0
- package/dist/default-client/assets/audio-mpgipo2Q.js +1 -0
- package/dist/default-client/assets/capture-processor-D0q7Qrdd.js +90 -0
- package/dist/default-client/assets/index-8ihb1Urm.css +2 -0
- package/dist/default-client/assets/index-DPfmSFmc.js +126 -0
- package/dist/default-client/assets/playback-processor-CjDbc9iH.js +314 -0
- package/dist/default-client/favicon.ico +0 -0
- package/dist/default-client/index.html +15 -0
- package/dist/default-client-entry.d.ts +6 -0
- package/dist/default-client.d.ts +19 -0
- package/dist/define-client.d.ts +169 -26
- package/dist/define-client.js +209 -55
- package/dist/eyebrow-UfmSz9yy.js +27 -0
- package/dist/hooks-DN4gHYsj.js +182 -0
- package/dist/hooks.d.ts +172 -0
- package/dist/hooks.js +3 -0
- package/dist/hooks.test-d.d.ts +14 -0
- package/dist/inbox-protocol.d.ts +128 -0
- package/dist/inbox.d.ts +132 -0
- package/dist/index.d.ts +126 -29
- package/dist/index.js +5500 -15
- package/dist/internal.d.ts +43 -0
- package/dist/internal.js +8 -0
- package/dist/linked-client.d.ts +70 -0
- package/dist/message-list-DHZcLhRA.js +682 -0
- package/dist/notice-player.d.ts +70 -0
- package/dist/page.d.ts +114 -0
- package/dist/phone.d.ts +50 -0
- package/dist/route-fetch.d.ts +79 -0
- package/dist/session/audio-effects.d.ts +37 -0
- package/dist/session/audio-setup.d.ts +57 -0
- package/dist/session/audio-state.d.ts +157 -0
- package/dist/session/browser-session.d.ts +39 -0
- package/dist/session/close.d.ts +26 -0
- package/dist/session/dial.d.ts +65 -0
- package/dist/session/handshake.d.ts +60 -0
- package/dist/session/index.d.ts +17 -0
- package/dist/session/index.js +2 -0
- package/dist/session/messages.d.ts +63 -0
- package/dist/session/mic.d.ts +64 -0
- package/dist/session/preconnect.d.ts +48 -0
- package/dist/session/reconnect.d.ts +25 -0
- package/dist/session/resume-store.d.ts +57 -0
- package/dist/session/state.d.ts +146 -0
- package/dist/session/ticket.d.ts +44 -0
- package/dist/session/types.d.ts +476 -0
- package/dist/session/url.d.ts +17 -0
- package/dist/session/user-turn.d.ts +55 -0
- package/dist/session-D9jM5-TF.js +2140 -0
- package/dist/stored-value.d.ts +103 -0
- package/dist/tool-call-block-uXtMOUWN.js +204 -0
- package/dist/tool-config-context-Es4YUzV2.js +19 -0
- package/dist/types.d.ts +272 -34
- package/dist/types.js +35 -4
- package/dist/upload/files.d.ts +83 -0
- package/dist/upload/index.d.ts +15 -0
- package/dist/upload/pause.d.ts +31 -0
- package/dist/upload/recall.d.ts +67 -0
- package/dist/upload/report.d.ts +26 -0
- package/dist/upload/session.d.ts +99 -0
- package/dist/url-chips-oyy_Eiqh.js +239 -0
- package/dist/use-client-runs.d.ts +87 -0
- package/dist/use-conversation-log.d.ts +86 -0
- package/dist/use-conversation.d.ts +131 -0
- package/dist/use-copy.d.ts +74 -0
- package/dist/use-download-url.d.ts +84 -0
- package/dist/use-flash.d.ts +61 -0
- package/dist/use-inbox.d.ts +101 -0
- package/dist/use-push-to-talk.d.ts +105 -0
- package/dist/use-route-mutation.d.ts +113 -0
- package/dist/use-route.d.ts +82 -0
- package/dist/use-run-key.d.ts +133 -0
- package/dist/use-session-controls.d.ts +69 -0
- package/dist/use-session-id.d.ts +57 -0
- package/dist/use-tap-to-talk.d.ts +138 -0
- package/dist/use-user-transcript-uyHhzy4d.js +73 -0
- package/dist/use-user-transcript.d.ts +75 -0
- package/dist/use-workflow-form.d.ts +285 -0
- package/dist/use-workflow-progress.d.ts +137 -0
- package/dist/use-workflow-run-DdhWmbx-.js +697 -0
- package/dist/use-workflow-run.d.ts +100 -0
- package/dist/use-workflow-runs.d.ts +71 -0
- package/dist/use-workflow-stream.d.ts +154 -0
- package/dist/use-workflows.d.ts +77 -0
- package/dist/workflow-client.d.ts +136 -0
- package/dist/workflow-def-types.d.ts +31 -0
- package/dist/workflow-events.d.ts +39 -0
- package/dist/workflow-field-kind.d.ts +95 -0
- package/dist/workflow-status-labels.d.ts +35 -0
- package/dist/worklets/_module-url.d.ts +10 -0
- package/dist/worklets/_playback-bench-harness.d.ts +181 -0
- package/dist/worklets/_playback-bench-host.d.ts +63 -0
- package/dist/worklets/_playback-bench-page.d.ts +65 -0
- package/dist/worklets/_tts-trace-harness.d.ts +142 -0
- package/dist/worklets/_worklet-test-utils.d.ts +27 -0
- package/dist/worklets/capture-processor.d.ts +4 -2
- package/dist/worklets/capture-processor.js +73 -33
- package/dist/worklets/playback-processor.d.ts +4 -2
- package/dist/worklets/playback-processor.js +258 -42
- package/package.json +59 -32
- package/styles.css +152 -30
- package/dist/_jsdom-setup.d.ts +0 -0
- package/dist/audio.d.ts +0 -45
- package/dist/audio.js +0 -121
- package/dist/client-context.d.ts +0 -33
- package/dist/client-context.js +0 -15
- package/dist/client-handler.d.ts +0 -43
- package/dist/components/app.d.ts +0 -23
- package/dist/components/app.js +0 -41
- package/dist/components/error-banner.d.ts +0 -21
- package/dist/components/error-banner.js +0 -27
- package/dist/components/message-bubble.d.ts +0 -23
- package/dist/components/message-bubble.js +0 -35
- package/dist/components/state-indicator.d.ts +0 -29
- package/dist/components/state-indicator.js +0 -37
- package/dist/components/thinking-indicator.d.ts +0 -16
- package/dist/components/thinking-indicator.js +0 -34
- package/dist/components/tool-icons.d.ts +0 -17
- package/dist/components/tool-icons.js +0 -128
- package/dist/components/transcript.d.ts +0 -25
- package/dist/components/transcript.js +0 -35
- package/dist/session-CWB7vmqz.js +0 -429
- package/dist/session.d.ts +0 -69
- package/dist/session.js +0 -2
- package/dist/signals.d.ts +0 -120
- package/dist/signals.js +0 -161
- package/dist/tsdown.config.d.ts +0 -2
- package/dist/types.test-d.d.ts +0 -7
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
import { CAPTURE_STOP_ACK_TIMEOUT_MS, MIC_BUFFER_SECONDS, PLAYBACK_DONE_MAX_WAIT_MS, PLAYBACK_DONE_POLL_MS, VOICE_CAPTURE_CONSTRAINTS } from "../types.js";
|
|
2
|
+
//#region src/audio/capture.ts
|
|
3
|
+
/**
|
|
4
|
+
* The capture half's primitives, shared by the audio path (`audio/voice-io.ts`) and
|
|
5
|
+
* the pre-connect capture (`audio/preconnect.ts`): the rate assertion, the
|
|
6
|
+
* failed-init release, and the capture worklet node's port protocol.
|
|
7
|
+
*
|
|
8
|
+
* @module
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* Throw unless the browser honored a requested context sample rate. The
|
|
12
|
+
* requested rates are never advisory: captured audio is tagged with the
|
|
13
|
+
* requested rate on the wire, so a context running at some other rate ships
|
|
14
|
+
* audio that only sounds like speech to the wrong decoder. Every capture
|
|
15
|
+
* path must call this after context creation.
|
|
16
|
+
*/
|
|
17
|
+
function assertGranted(granted, requested, side) {
|
|
18
|
+
if (granted === requested) return;
|
|
19
|
+
throw new Error(`Browser refused the ${side} sample rate: asked for ${requested} Hz, got ${granted} Hz`);
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* The error a dead worklet processor reports.
|
|
23
|
+
*
|
|
24
|
+
* A processor exception permanently kills the node — no further messages or
|
|
25
|
+
* audio will ever arrive — so both sides log it and hand it on. One spelling of
|
|
26
|
+
* that, because the pair differed only in the word "capture"/"playback" and in
|
|
27
|
+
* what the playback side has to settle afterwards.
|
|
28
|
+
*/
|
|
29
|
+
function workletCrash(side) {
|
|
30
|
+
const err = /* @__PURE__ */ new Error(`Audio ${side} worklet crashed`);
|
|
31
|
+
console.error("[aai-ui]", err.message);
|
|
32
|
+
return err;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Release a microphone that was (or later gets) granted while another init
|
|
36
|
+
* step failed; if `getUserMedia` itself rejected, this is a no-op. Without
|
|
37
|
+
* it, a mic granted after a failed init keeps the browser's recording
|
|
38
|
+
* indicator lit with no way to turn it off.
|
|
39
|
+
*/
|
|
40
|
+
function releaseStreamOnFailure(streamPromise) {
|
|
41
|
+
streamPromise.then((s) => {
|
|
42
|
+
for (const t of s.getTracks()) t.stop();
|
|
43
|
+
}).catch(() => {});
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Wire one capture worklet node: node construction, the chunk/silent/stopped
|
|
47
|
+
* port protocol, and the stop→ack handshake. No `processorOptions`: the
|
|
48
|
+
* worklet reads the context rate from its global scope (callers assert the
|
|
49
|
+
* granted rate first) and owns its own batching default — re-spelling
|
|
50
|
+
* defaults caller-side is drift.
|
|
51
|
+
*/
|
|
52
|
+
function createCaptureNode(ctx, onChunk, onSilent) {
|
|
53
|
+
const node = new AudioWorkletNode(ctx, "capture-processor", {
|
|
54
|
+
channelCount: 1,
|
|
55
|
+
channelCountMode: "explicit"
|
|
56
|
+
});
|
|
57
|
+
let onStopped = null;
|
|
58
|
+
node.port.onmessage = (e) => {
|
|
59
|
+
const d = e.data;
|
|
60
|
+
if (d.event === "chunk" && d.buffer) onChunk(d.buffer);
|
|
61
|
+
else if (d.event === "silent") onSilent?.();
|
|
62
|
+
else if (d.event === "stopped") {
|
|
63
|
+
onStopped?.();
|
|
64
|
+
onStopped = null;
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
return {
|
|
68
|
+
node,
|
|
69
|
+
start() {
|
|
70
|
+
node.port.postMessage({ event: "start" });
|
|
71
|
+
},
|
|
72
|
+
stop() {
|
|
73
|
+
const { promise, resolve } = Promise.withResolvers();
|
|
74
|
+
const cap = setTimeout(resolve, CAPTURE_STOP_ACK_TIMEOUT_MS);
|
|
75
|
+
onStopped = () => {
|
|
76
|
+
clearTimeout(cap);
|
|
77
|
+
resolve();
|
|
78
|
+
};
|
|
79
|
+
node.port.postMessage({ event: "stop" });
|
|
80
|
+
return promise;
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
//#endregion
|
|
85
|
+
//#region src/audio/preconnect.ts
|
|
86
|
+
/**
|
|
87
|
+
* Pre-connect capture: the microphone opened before the session's audio rates
|
|
88
|
+
* are known, and the resample that fits its buffer to the rate the server
|
|
89
|
+
* then names. `createVoiceIO` (`audio/voice-io.ts`) is what takes it over.
|
|
90
|
+
*
|
|
91
|
+
* @module
|
|
92
|
+
*/
|
|
93
|
+
/**
|
|
94
|
+
* Open a {@link PreConnectCapture}: ask for the microphone and start buffering
|
|
95
|
+
* at `sampleRate`, keeping at most the latest `maxSeconds` — the most RECENT
|
|
96
|
+
* audio rather than the first, so the buffer always runs contiguously into
|
|
97
|
+
* the live stream that follows it.
|
|
98
|
+
*
|
|
99
|
+
* Rejects like {@link createVoiceIO} does (a denied prompt, a refused rate);
|
|
100
|
+
* the caller falls back to opening the path the ordinary way.
|
|
101
|
+
*/
|
|
102
|
+
async function openPreConnectCapture(opts) {
|
|
103
|
+
const { sampleRate, captureWorkletSrc, maxSeconds } = opts;
|
|
104
|
+
const ctx = new AudioContext({
|
|
105
|
+
sampleRate,
|
|
106
|
+
latencyHint: "interactive"
|
|
107
|
+
});
|
|
108
|
+
const streamPromise = navigator.mediaDevices.getUserMedia({ audio: {
|
|
109
|
+
deviceId: { ideal: "default" },
|
|
110
|
+
...VOICE_CAPTURE_CONSTRAINTS
|
|
111
|
+
} });
|
|
112
|
+
let stream;
|
|
113
|
+
try {
|
|
114
|
+
[stream] = await Promise.all([
|
|
115
|
+
streamPromise,
|
|
116
|
+
ctx.resume(),
|
|
117
|
+
ctx.audioWorklet.addModule(captureWorkletSrc)
|
|
118
|
+
]);
|
|
119
|
+
assertGranted(ctx.sampleRate, sampleRate, "capture");
|
|
120
|
+
} catch (err) {
|
|
121
|
+
releaseStreamOnFailure(streamPromise);
|
|
122
|
+
await ctx.close().catch(() => {});
|
|
123
|
+
throw err;
|
|
124
|
+
}
|
|
125
|
+
const maxBytes = Math.round(sampleRate * maxSeconds) * 2;
|
|
126
|
+
let chunks = [];
|
|
127
|
+
let bytes = 0;
|
|
128
|
+
let silent = false;
|
|
129
|
+
let sinks = null;
|
|
130
|
+
const mic = ctx.createMediaStreamSource(stream);
|
|
131
|
+
const capture = createCaptureNode(ctx, (pcm16) => {
|
|
132
|
+
if (sinks) {
|
|
133
|
+
sinks.onChunk(pcm16);
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
chunks.push(pcm16);
|
|
137
|
+
bytes += pcm16.byteLength;
|
|
138
|
+
while (bytes > maxBytes && chunks.length > 1) bytes -= chunks.shift()?.byteLength ?? 0;
|
|
139
|
+
}, () => {
|
|
140
|
+
if (sinks) sinks.onSilent?.();
|
|
141
|
+
else silent = true;
|
|
142
|
+
});
|
|
143
|
+
mic.connect(capture.node);
|
|
144
|
+
capture.start();
|
|
145
|
+
/** Take the buffer, leaving it empty. */
|
|
146
|
+
const takeChunks = () => {
|
|
147
|
+
const taken = chunks;
|
|
148
|
+
chunks = [];
|
|
149
|
+
bytes = 0;
|
|
150
|
+
return taken;
|
|
151
|
+
};
|
|
152
|
+
let closed = false;
|
|
153
|
+
const closeCtx = async () => {
|
|
154
|
+
if (closed) return;
|
|
155
|
+
closed = true;
|
|
156
|
+
mic.disconnect();
|
|
157
|
+
capture.node.disconnect();
|
|
158
|
+
await ctx.close().catch((err) => {
|
|
159
|
+
console.warn("AudioContext close failed:", err);
|
|
160
|
+
});
|
|
161
|
+
};
|
|
162
|
+
return {
|
|
163
|
+
sampleRate,
|
|
164
|
+
stream,
|
|
165
|
+
ctx,
|
|
166
|
+
handOff(next) {
|
|
167
|
+
sinks = next;
|
|
168
|
+
const buffered = takeChunks();
|
|
169
|
+
if (buffered.length > 0) next.onBuffered(buffered);
|
|
170
|
+
if (silent) next.onSilent?.();
|
|
171
|
+
return {
|
|
172
|
+
mic,
|
|
173
|
+
capture
|
|
174
|
+
};
|
|
175
|
+
},
|
|
176
|
+
async detach() {
|
|
177
|
+
await capture.stop();
|
|
178
|
+
await closeCtx();
|
|
179
|
+
return {
|
|
180
|
+
chunks: takeChunks(),
|
|
181
|
+
sampleRate
|
|
182
|
+
};
|
|
183
|
+
},
|
|
184
|
+
async close() {
|
|
185
|
+
if (!closed) await capture.stop();
|
|
186
|
+
for (const t of stream.getTracks()) t.stop();
|
|
187
|
+
await closeCtx();
|
|
188
|
+
}
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
/** Float32 → PCM16, clamped and scaled as the capture worklet does. */
|
|
192
|
+
function encodePcm16(samples) {
|
|
193
|
+
const pcm = new Int16Array(samples.length);
|
|
194
|
+
for (let i = 0; i < samples.length; i++) {
|
|
195
|
+
const s = Math.max(-1, Math.min(1, samples[i] ?? 0));
|
|
196
|
+
pcm[i] = s < 0 ? s * 32768 : s * 32767;
|
|
197
|
+
}
|
|
198
|
+
return pcm.buffer;
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Convert buffered PCM16 from one rate to another with the browser's own
|
|
202
|
+
* band-limited resampler (an `OfflineAudioContext` render), re-cut into
|
|
203
|
+
* {@link MIC_BUFFER_SECONDS} chunks so the burst matches the live cadence.
|
|
204
|
+
*
|
|
205
|
+
* Never rejects: pre-connect audio is a bonus, and a browser that cannot
|
|
206
|
+
* render it offline loses the bonus rather than the session.
|
|
207
|
+
*/
|
|
208
|
+
async function resamplePcm16(chunks, fromRate, toRate) {
|
|
209
|
+
const total = chunks.reduce((n, c) => n + c.byteLength / 2, 0);
|
|
210
|
+
if (total === 0) return [];
|
|
211
|
+
if (fromRate === toRate) return chunks;
|
|
212
|
+
try {
|
|
213
|
+
const outLength = Math.round(total * toRate / fromRate);
|
|
214
|
+
const offline = new OfflineAudioContext(1, outLength, toRate);
|
|
215
|
+
const input = offline.createBuffer(1, total, fromRate);
|
|
216
|
+
const samples = input.getChannelData(0);
|
|
217
|
+
let at = 0;
|
|
218
|
+
for (const chunk of chunks) for (const v of new Int16Array(chunk)) samples[at++] = v / 32768;
|
|
219
|
+
const source = offline.createBufferSource();
|
|
220
|
+
source.buffer = input;
|
|
221
|
+
source.connect(offline.destination);
|
|
222
|
+
source.start();
|
|
223
|
+
const rendered = (await offline.startRendering()).getChannelData(0);
|
|
224
|
+
const size = Math.max(1, Math.round(toRate * MIC_BUFFER_SECONDS));
|
|
225
|
+
const out = [];
|
|
226
|
+
for (let start = 0; start < rendered.length; start += size) out.push(encodePcm16(rendered.subarray(start, start + size)));
|
|
227
|
+
return out;
|
|
228
|
+
} catch (err) {
|
|
229
|
+
console.warn("[aai-ui] pre-connect audio dropped: resample failed", err);
|
|
230
|
+
return [];
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Wire and start a fresh capture node on `ctx`, first flushing any audio a
|
|
235
|
+
* detached {@link PreConnectCapture} held, resampled to `ctx`'s rate.
|
|
236
|
+
*/
|
|
237
|
+
async function startCapture(ctx, stream, opts) {
|
|
238
|
+
const mic = ctx.createMediaStreamSource(stream);
|
|
239
|
+
const capture = createCaptureNode(ctx, opts.onMicData, opts.onMicSilent);
|
|
240
|
+
mic.connect(capture.node);
|
|
241
|
+
if (opts.buffered) {
|
|
242
|
+
const { chunks, sampleRate } = opts.buffered;
|
|
243
|
+
const resampled = await resamplePcm16(chunks, sampleRate, ctx.sampleRate);
|
|
244
|
+
if (resampled.length > 0) opts.onBuffered(resampled);
|
|
245
|
+
}
|
|
246
|
+
capture.start();
|
|
247
|
+
return {
|
|
248
|
+
mic,
|
|
249
|
+
capture
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
//#endregion
|
|
253
|
+
//#region src/audio/voice-io.ts
|
|
254
|
+
/**
|
|
255
|
+
* Create a {@link VoiceIO} instance that captures microphone audio and
|
|
256
|
+
* plays back TTS audio using the Web Audio API.
|
|
257
|
+
*
|
|
258
|
+
* Playback runs on a context at the TTS sample rate for fidelity, and capture
|
|
259
|
+
* on its own context at the STT rate so the *browser* performs the rate
|
|
260
|
+
* conversion with its band-limited resampler. The two collapse into one
|
|
261
|
+
* context when the rates match. A browser that declines either requested rate
|
|
262
|
+
* fails init rather than falling back to converting in the worklet, which
|
|
263
|
+
* would alias.
|
|
264
|
+
*
|
|
265
|
+
* @param opts - Voice I/O configuration options.
|
|
266
|
+
* @returns A promise that resolves to a {@link VoiceIO} handle.
|
|
267
|
+
* @throws If microphone access is denied or AudioWorklet registration fails.
|
|
268
|
+
*/
|
|
269
|
+
async function createVoiceIO(opts) {
|
|
270
|
+
const { sttSampleRate, ttsSampleRate, captureWorkletSrc, playbackWorkletSrc, onMicData, onError, onPlaybackStats, onPlaybackProgress, onMicSilent, preConnect } = opts;
|
|
271
|
+
const onPreConnectAudio = opts.onPreConnectAudio ?? ((chunks) => {
|
|
272
|
+
for (const chunk of chunks) onMicData(chunk);
|
|
273
|
+
});
|
|
274
|
+
const adopted = preConnect?.sampleRate === sttSampleRate ? preConnect : null;
|
|
275
|
+
const detached = preConnect && !adopted ? await preConnect.detach() : null;
|
|
276
|
+
const ctx = new AudioContext({
|
|
277
|
+
sampleRate: ttsSampleRate,
|
|
278
|
+
latencyHint: "playback"
|
|
279
|
+
});
|
|
280
|
+
const sharesContext = !adopted && sttSampleRate === ttsSampleRate;
|
|
281
|
+
const capCtx = adopted?.ctx ?? (sharesContext ? ctx : new AudioContext({
|
|
282
|
+
sampleRate: sttSampleRate,
|
|
283
|
+
latencyHint: "interactive"
|
|
284
|
+
}));
|
|
285
|
+
async function closeContexts() {
|
|
286
|
+
const contexts = sharesContext ? [ctx] : [ctx, capCtx];
|
|
287
|
+
await Promise.all(contexts.map((c) => c.close().catch((err) => {
|
|
288
|
+
console.warn("AudioContext close failed:", err);
|
|
289
|
+
})));
|
|
290
|
+
}
|
|
291
|
+
const streamPromise = preConnect ? Promise.resolve(preConnect.stream) : navigator.mediaDevices.getUserMedia({ audio: {
|
|
292
|
+
deviceId: { ideal: "default" },
|
|
293
|
+
...VOICE_CAPTURE_CONSTRAINTS
|
|
294
|
+
} });
|
|
295
|
+
let stream;
|
|
296
|
+
try {
|
|
297
|
+
[stream] = await Promise.all([
|
|
298
|
+
streamPromise,
|
|
299
|
+
ctx.resume(),
|
|
300
|
+
capCtx.resume(),
|
|
301
|
+
adopted ? void 0 : capCtx.audioWorklet.addModule(captureWorkletSrc),
|
|
302
|
+
ctx.audioWorklet.addModule(playbackWorkletSrc)
|
|
303
|
+
]);
|
|
304
|
+
assertGranted(capCtx.sampleRate, sttSampleRate, "capture");
|
|
305
|
+
assertGranted(ctx.sampleRate, ttsSampleRate, "playback");
|
|
306
|
+
} catch (err) {
|
|
307
|
+
releaseStreamOnFailure(streamPromise);
|
|
308
|
+
await closeContexts();
|
|
309
|
+
throw err;
|
|
310
|
+
}
|
|
311
|
+
const { mic, capture } = adopted ? adopted.handOff({
|
|
312
|
+
onBuffered: onPreConnectAudio,
|
|
313
|
+
onChunk: onMicData,
|
|
314
|
+
onSilent: onMicSilent
|
|
315
|
+
}) : await startCapture(capCtx, stream, {
|
|
316
|
+
onMicData,
|
|
317
|
+
onMicSilent,
|
|
318
|
+
buffered: detached,
|
|
319
|
+
onBuffered: onPreConnectAudio
|
|
320
|
+
});
|
|
321
|
+
capture.node.onprocessorerror = () => onError?.(workletCrash("capture"));
|
|
322
|
+
let playNode = null;
|
|
323
|
+
/**
|
|
324
|
+
* Turn ids for the drain handshake. Every `done()` posts a fresh id and the
|
|
325
|
+
* worklet echoes it on the matching 'stop', so a stop the worklet posted for
|
|
326
|
+
* an EARLIER turn — already in flight when a barge-in flushed that turn —
|
|
327
|
+
* cannot settle the current turn's wait. Dropping only `reason: 'interrupt'`
|
|
328
|
+
* stops was not enough: the stale one is a legitimate drain stop, just for a
|
|
329
|
+
* turn the host has already moved past, and settling on it reports the live
|
|
330
|
+
* reply finished while it is still speaking.
|
|
331
|
+
*/
|
|
332
|
+
let turnSeq = 0;
|
|
333
|
+
/**
|
|
334
|
+
* The drain wait in flight, if any.
|
|
335
|
+
*
|
|
336
|
+
* ONE field, because the resolver and the turn it belongs to are one fact:
|
|
337
|
+
* as two independently-nulled variables they could disagree, and did — the
|
|
338
|
+
* suspended-context branch of `done()` cleared the turn and left the resolver
|
|
339
|
+
* set. A single nullable makes that unrepresentable.
|
|
340
|
+
*/
|
|
341
|
+
let pendingStop = null;
|
|
342
|
+
const lifecycle = new AbortController();
|
|
343
|
+
/** Settle whatever drain wait is pending and forget the turn it belonged to. */
|
|
344
|
+
function settlePendingStop() {
|
|
345
|
+
const pending = pendingStop;
|
|
346
|
+
pendingStop = null;
|
|
347
|
+
pending?.settle();
|
|
348
|
+
}
|
|
349
|
+
function onWorkletStop(msg) {
|
|
350
|
+
if (msg.stats && msg.stats.concealedSamples > 0) onPlaybackStats?.(msg.stats);
|
|
351
|
+
if (msg.reason === "interrupt") return;
|
|
352
|
+
if (pendingStop !== null && msg.turn !== pendingStop.turn) return;
|
|
353
|
+
settlePendingStop();
|
|
354
|
+
}
|
|
355
|
+
function ensurePlayNode() {
|
|
356
|
+
if (playNode) return playNode;
|
|
357
|
+
const node = new AudioWorkletNode(ctx, "playback-processor");
|
|
358
|
+
node.connect(ctx.destination);
|
|
359
|
+
node.port.onmessage = (e) => {
|
|
360
|
+
if (e.data.event === "stop") onWorkletStop(e.data);
|
|
361
|
+
else if (e.data.event === "progress") onPlaybackProgress?.(e.data.bufferedMs);
|
|
362
|
+
};
|
|
363
|
+
node.onprocessorerror = () => {
|
|
364
|
+
const err = workletCrash("playback");
|
|
365
|
+
settlePendingStop();
|
|
366
|
+
onError?.(err);
|
|
367
|
+
};
|
|
368
|
+
playNode = node;
|
|
369
|
+
return node;
|
|
370
|
+
}
|
|
371
|
+
const io = {
|
|
372
|
+
enqueue(pcm16Buffer) {
|
|
373
|
+
if (lifecycle.signal.aborted) return;
|
|
374
|
+
if (pcm16Buffer.byteLength === 0) return;
|
|
375
|
+
ensurePlayNode().port.postMessage({
|
|
376
|
+
event: "write",
|
|
377
|
+
buffer: new Uint8Array(pcm16Buffer)
|
|
378
|
+
}, [pcm16Buffer]);
|
|
379
|
+
},
|
|
380
|
+
done() {
|
|
381
|
+
if (!playNode) return Promise.resolve();
|
|
382
|
+
const turn = ++turnSeq;
|
|
383
|
+
playNode.port.postMessage({
|
|
384
|
+
event: "done",
|
|
385
|
+
turn
|
|
386
|
+
});
|
|
387
|
+
if (ctx.state !== "running") {
|
|
388
|
+
settlePendingStop();
|
|
389
|
+
return Promise.resolve();
|
|
390
|
+
}
|
|
391
|
+
const { promise, resolve } = Promise.withResolvers();
|
|
392
|
+
settlePendingStop();
|
|
393
|
+
const settle = () => {
|
|
394
|
+
clearInterval(poll);
|
|
395
|
+
clearTimeout(cap);
|
|
396
|
+
if (pendingStop?.settle === settle) pendingStop = null;
|
|
397
|
+
resolve();
|
|
398
|
+
};
|
|
399
|
+
const poll = setInterval(() => {
|
|
400
|
+
if (ctx.state !== "running") settle();
|
|
401
|
+
}, PLAYBACK_DONE_POLL_MS);
|
|
402
|
+
const cap = setTimeout(settle, PLAYBACK_DONE_MAX_WAIT_MS);
|
|
403
|
+
pendingStop = {
|
|
404
|
+
turn,
|
|
405
|
+
settle
|
|
406
|
+
};
|
|
407
|
+
return promise;
|
|
408
|
+
},
|
|
409
|
+
flush() {
|
|
410
|
+
if (!playNode) return;
|
|
411
|
+
settlePendingStop();
|
|
412
|
+
playNode.port.postMessage({ event: "interrupt" });
|
|
413
|
+
},
|
|
414
|
+
async close() {
|
|
415
|
+
if (lifecycle.signal.aborted) return;
|
|
416
|
+
lifecycle.abort();
|
|
417
|
+
await capture.stop();
|
|
418
|
+
mic.disconnect();
|
|
419
|
+
capture.node.disconnect();
|
|
420
|
+
if (playNode) playNode.disconnect();
|
|
421
|
+
for (const t of stream.getTracks()) t.stop();
|
|
422
|
+
await closeContexts();
|
|
423
|
+
},
|
|
424
|
+
async [Symbol.asyncDispose]() {
|
|
425
|
+
await io.close();
|
|
426
|
+
}
|
|
427
|
+
};
|
|
428
|
+
return io;
|
|
429
|
+
}
|
|
430
|
+
//#endregion
|
|
431
|
+
export { createVoiceIO, openPreConnectCapture };
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pre-connect capture: the microphone opened before the session's audio rates
|
|
3
|
+
* are known, and the resample that fits its buffer to the rate the server
|
|
4
|
+
* then names. `createVoiceIO` (`audio/voice-io.ts`) is what takes it over.
|
|
5
|
+
*
|
|
6
|
+
* @module
|
|
7
|
+
*/
|
|
8
|
+
import { type CaptureNode } from "./capture.ts";
|
|
9
|
+
/**
|
|
10
|
+
* A microphone opened BEFORE the session's audio rates are known, buffering
|
|
11
|
+
* what the caller says until {@link createVoiceIO} takes it over.
|
|
12
|
+
*
|
|
13
|
+
* The connect-time half of pre-connect audio: the server names its STT rate
|
|
14
|
+
* only in its `config` frame, one handshake (and on the platform, possibly a
|
|
15
|
+
* sandbox boot) after the user pressed start — and an eager caller is talking
|
|
16
|
+
* by then. This captures at a guessed rate; {@link createVoiceIO} adopts it
|
|
17
|
+
* whole when the guess was right and resamples the buffer when it was not.
|
|
18
|
+
*/
|
|
19
|
+
export type PreConnectCapture = {
|
|
20
|
+
/** The rate this capture runs at — the guess. */
|
|
21
|
+
readonly sampleRate: number;
|
|
22
|
+
/** The granted microphone, handed to the {@link VoiceIO} that takes over. */
|
|
23
|
+
readonly stream: MediaStream;
|
|
24
|
+
/** The capture context, adopted as the path's own when the rate matches. */
|
|
25
|
+
readonly ctx: AudioContext;
|
|
26
|
+
/**
|
|
27
|
+
* Adopt the running capture: flush the buffer to `onBuffered`, then route
|
|
28
|
+
* every later frame to `onChunk` — synchronously, so nothing interleaves.
|
|
29
|
+
*/
|
|
30
|
+
handOff(sinks: {
|
|
31
|
+
onBuffered: (chunks: ArrayBuffer[]) => void;
|
|
32
|
+
onChunk: (pcm16: ArrayBuffer) => void;
|
|
33
|
+
onSilent?: (() => void) | undefined;
|
|
34
|
+
}): {
|
|
35
|
+
mic: MediaStreamAudioSourceNode;
|
|
36
|
+
capture: CaptureNode;
|
|
37
|
+
};
|
|
38
|
+
/**
|
|
39
|
+
* Stop capturing (after the worklet's tail flush) and close the context,
|
|
40
|
+
* keeping the stream: for a path at another rate. Never rejects.
|
|
41
|
+
*/
|
|
42
|
+
detach(): Promise<{
|
|
43
|
+
chunks: ArrayBuffer[];
|
|
44
|
+
sampleRate: number;
|
|
45
|
+
}>;
|
|
46
|
+
/** Release everything, stream included. Best-effort, idempotent. */
|
|
47
|
+
close(): Promise<void>;
|
|
48
|
+
};
|
|
49
|
+
/**
|
|
50
|
+
* Open a {@link PreConnectCapture}: ask for the microphone and start buffering
|
|
51
|
+
* at `sampleRate`, keeping at most the latest `maxSeconds` — the most RECENT
|
|
52
|
+
* audio rather than the first, so the buffer always runs contiguously into
|
|
53
|
+
* the live stream that follows it.
|
|
54
|
+
*
|
|
55
|
+
* Rejects like {@link createVoiceIO} does (a denied prompt, a refused rate);
|
|
56
|
+
* the caller falls back to opening the path the ordinary way.
|
|
57
|
+
*/
|
|
58
|
+
export declare function openPreConnectCapture(opts: {
|
|
59
|
+
sampleRate: number;
|
|
60
|
+
captureWorkletSrc: string;
|
|
61
|
+
maxSeconds: number;
|
|
62
|
+
}): Promise<PreConnectCapture>;
|
|
63
|
+
/**
|
|
64
|
+
* Convert buffered PCM16 from one rate to another with the browser's own
|
|
65
|
+
* band-limited resampler (an `OfflineAudioContext` render), re-cut into
|
|
66
|
+
* {@link MIC_BUFFER_SECONDS} chunks so the burst matches the live cadence.
|
|
67
|
+
*
|
|
68
|
+
* Never rejects: pre-connect audio is a bonus, and a browser that cannot
|
|
69
|
+
* render it offline loses the bonus rather than the session.
|
|
70
|
+
*/
|
|
71
|
+
export declare function resamplePcm16(chunks: ArrayBuffer[], fromRate: number, toRate: number): Promise<ArrayBuffer[]>;
|
|
72
|
+
/**
|
|
73
|
+
* Wire and start a fresh capture node on `ctx`, first flushing any audio a
|
|
74
|
+
* detached {@link PreConnectCapture} held, resampled to `ctx`'s rate.
|
|
75
|
+
*/
|
|
76
|
+
export declare function startCapture(ctx: AudioContext, stream: MediaStream, opts: {
|
|
77
|
+
onMicData: (pcm16: ArrayBuffer) => void;
|
|
78
|
+
onMicSilent?: (() => void) | undefined;
|
|
79
|
+
buffered: {
|
|
80
|
+
chunks: ArrayBuffer[];
|
|
81
|
+
sampleRate: number;
|
|
82
|
+
} | null;
|
|
83
|
+
onBuffered: (chunks: ArrayBuffer[]) => void;
|
|
84
|
+
}): Promise<{
|
|
85
|
+
mic: MediaStreamAudioSourceNode;
|
|
86
|
+
capture: CaptureNode;
|
|
87
|
+
}>;
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { type PreConnectCapture } from "./preconnect.ts";
|
|
2
|
+
/**
|
|
3
|
+
* How much of one turn's playback was covered by concealment rather than
|
|
4
|
+
* received audio — the playback worklet's underrun report, in the shape
|
|
5
|
+
* WebRTC's `inbound-rtp` audio stats use, so the numbers mean the same thing
|
|
6
|
+
* here as in a `getStats()` dump.
|
|
7
|
+
*
|
|
8
|
+
* A turn with `concealmentEvents: 0` never needed its jitter buffer; a turn
|
|
9
|
+
* with a high `silentConcealedSamples` share starved for longer than
|
|
10
|
+
* concealment can plausibly cover, which is a bandwidth problem rather than a
|
|
11
|
+
* buffer-tuning one.
|
|
12
|
+
*
|
|
13
|
+
* @public
|
|
14
|
+
*/
|
|
15
|
+
export type PlaybackStats = {
|
|
16
|
+
/** Samples emitted to cover a gap, including the silent ones. */
|
|
17
|
+
concealedSamples: number;
|
|
18
|
+
/** The subset of {@link PlaybackStats.concealedSamples} that were silence. */
|
|
19
|
+
silentConcealedSamples: number;
|
|
20
|
+
/** Distinct underrun episodes, however many render quanta each spanned. */
|
|
21
|
+
concealmentEvents: number;
|
|
22
|
+
/** Episodes that lasted long enough to decay to silence. */
|
|
23
|
+
silentConcealmentEvents: number;
|
|
24
|
+
};
|
|
25
|
+
/** Configuration for creating a {@link VoiceIO} instance. */
|
|
26
|
+
export type VoiceIOOptions = {
|
|
27
|
+
/** Sample rate in Hz expected by the STT engine (e.g. 16000). */
|
|
28
|
+
sttSampleRate: number;
|
|
29
|
+
/** Sample rate in Hz used by the TTS engine (e.g. 22050). */
|
|
30
|
+
ttsSampleRate: number;
|
|
31
|
+
/** Source URL or data URI for the capture AudioWorklet processor. */
|
|
32
|
+
captureWorkletSrc: string;
|
|
33
|
+
/** Source URL or data URI for the playback AudioWorklet processor. */
|
|
34
|
+
playbackWorkletSrc: string;
|
|
35
|
+
/** Callback invoked with buffered PCM16 microphone data to send to the server. */
|
|
36
|
+
onMicData: (pcm16: ArrayBuffer) => void;
|
|
37
|
+
/**
|
|
38
|
+
* Called when an AudioWorklet processor throws and is killed by the browser
|
|
39
|
+
* (the node produces no further audio or messages), so the session can
|
|
40
|
+
* transition out of listening/speaking instead of looking healthy forever.
|
|
41
|
+
*/
|
|
42
|
+
onError?: ((err: Error) => void) | undefined;
|
|
43
|
+
/**
|
|
44
|
+
* Called at the end of any turn whose playback had to conceal a gap. Never
|
|
45
|
+
* called for a clean turn, so it can be wired straight to a warning.
|
|
46
|
+
*/
|
|
47
|
+
onPlaybackStats?: ((stats: PlaybackStats) => void) | undefined;
|
|
48
|
+
/**
|
|
49
|
+
* Called every {@link PLAYBACK_PROGRESS_INTERVAL_MS} while the playback
|
|
50
|
+
* buffer holds unplayed agent audio, with its depth in ms. Wire it to the
|
|
51
|
+
* session's `playback_progress` frame: it is the host's only closed-loop
|
|
52
|
+
* view of playback, and without it the host assumes every chunk it forwards
|
|
53
|
+
* starts playing on arrival at exactly 1.0x — so a client whose buffer runs
|
|
54
|
+
* ahead of the wall clock is told the line is silent while the caller is
|
|
55
|
+
* still listening. Unwired, the host degrades to that estimate.
|
|
56
|
+
*/
|
|
57
|
+
onPlaybackProgress?: ((bufferedMs: number) => void) | undefined;
|
|
58
|
+
/**
|
|
59
|
+
* Called once if the microphone delivers nothing but digital silence for
|
|
60
|
+
* the first {@link MIC_SILENCE_PROBE_MS} of capture — a muted or wrong input
|
|
61
|
+
* device, which otherwise looks exactly like a user who hasn't spoken.
|
|
62
|
+
*/
|
|
63
|
+
onMicSilent?: (() => void) | undefined;
|
|
64
|
+
/**
|
|
65
|
+
* A microphone already opened by {@link openPreConnectCapture}, whose buffered
|
|
66
|
+
* audio is the start of this path's capture. Ownership passes to the
|
|
67
|
+
* returned {@link VoiceIO}: its `close()` stops the stream.
|
|
68
|
+
*/
|
|
69
|
+
preConnect?: PreConnectCapture | undefined;
|
|
70
|
+
/**
|
|
71
|
+
* Called once, before any live frame reaches `onMicData`, with the audio
|
|
72
|
+
* {@link VoiceIOOptions.preConnect} held — already at `sttSampleRate`, in
|
|
73
|
+
* order. Separate from `onMicData` because it is a deliberate burst the
|
|
74
|
+
* sender's backpressure check must not drop. Unwired, it goes to
|
|
75
|
+
* `onMicData` chunk by chunk.
|
|
76
|
+
*/
|
|
77
|
+
onPreConnectAudio?: ((chunks: ArrayBuffer[]) => void) | undefined;
|
|
78
|
+
};
|
|
79
|
+
/**
|
|
80
|
+
* Audio I/O interface for voice capture and playback.
|
|
81
|
+
*
|
|
82
|
+
* Manages microphone capture via an AudioWorklet and TTS audio playback
|
|
83
|
+
* through a second AudioWorklet. Implements {@link AsyncDisposable} for
|
|
84
|
+
* resource cleanup.
|
|
85
|
+
*/
|
|
86
|
+
export type VoiceIO = AsyncDisposable & {
|
|
87
|
+
/** Enqueue a PCM16 audio buffer for playback through the TTS pipeline. */
|
|
88
|
+
enqueue(pcm16Buffer: ArrayBuffer): void;
|
|
89
|
+
/** Signal that all TTS audio for the current turn has been enqueued.
|
|
90
|
+
* Resolves when the worklet has finished playing all buffered audio. */
|
|
91
|
+
done(): Promise<void>;
|
|
92
|
+
/** Immediately stop playback and discard any buffered TTS audio. */
|
|
93
|
+
flush(): void;
|
|
94
|
+
/** Release all audio resources (microphone, AudioContext, worklets). */
|
|
95
|
+
close(): Promise<void>;
|
|
96
|
+
};
|
|
97
|
+
/**
|
|
98
|
+
* Create a {@link VoiceIO} instance that captures microphone audio and
|
|
99
|
+
* plays back TTS audio using the Web Audio API.
|
|
100
|
+
*
|
|
101
|
+
* Playback runs on a context at the TTS sample rate for fidelity, and capture
|
|
102
|
+
* on its own context at the STT rate so the *browser* performs the rate
|
|
103
|
+
* conversion with its band-limited resampler. The two collapse into one
|
|
104
|
+
* context when the rates match. A browser that declines either requested rate
|
|
105
|
+
* fails init rather than falling back to converting in the worklet, which
|
|
106
|
+
* would alias.
|
|
107
|
+
*
|
|
108
|
+
* @param opts - Voice I/O configuration options.
|
|
109
|
+
* @returns A promise that resolves to a {@link VoiceIO} handle.
|
|
110
|
+
* @throws If microphone access is denied or AudioWorklet registration fails.
|
|
111
|
+
*/
|
|
112
|
+
export declare function createVoiceIO(opts: VoiceIOOptions): Promise<VoiceIO>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|