@alexkroman1/aai-ui 16.1.0 → 18.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +3 -2
  2. package/dist/_page-shell.d.ts +11 -0
  3. package/dist/{chat-view-DsylXJsw.js → chat-view-P12pHKQU.js} +1 -1
  4. package/dist/{client-config-DJQHnYjm.js → client-config-CFhkMkmv.js} +15 -13
  5. package/dist/client-config.d.ts +14 -13
  6. package/dist/components/chat-view.js +1 -1
  7. package/dist/components/controls.js +1 -1
  8. package/dist/components/message-list.js +1 -1
  9. package/dist/components/tool-call-block.js +1 -1
  10. package/dist/context.d.ts +27 -5
  11. package/dist/context.js +3 -3
  12. package/dist/default-client/assets/index-5XG8PqGw.css +2 -0
  13. package/dist/default-client/assets/index-tc-nAWq3.js +121 -0
  14. package/dist/default-client/index.html +2 -2
  15. package/dist/default-client.d.ts +21 -0
  16. package/dist/define-client.js +4 -4
  17. package/dist/hooks.d.ts +19 -9
  18. package/dist/hooks.js +1 -1
  19. package/dist/index.d.ts +17 -13
  20. package/dist/index.js +734 -433
  21. package/dist/internal.js +3 -3
  22. package/dist/{message-list-BLyPRS1Q.js → message-list-CJX560b5.js} +1 -1
  23. package/dist/page.d.ts +47 -21
  24. package/dist/{session-core-gwePM95B.js → session-core-TOdm_Con.js} +424 -174
  25. package/dist/session-core-audio-effects.d.ts +26 -0
  26. package/dist/session-core-audio-setup.d.ts +38 -31
  27. package/dist/session-core-audio-state.d.ts +152 -0
  28. package/dist/session-core-messages.d.ts +9 -10
  29. package/dist/session-core-types.d.ts +60 -21
  30. package/dist/session-core-user-turn.d.ts +30 -0
  31. package/dist/session-core.js +1 -1
  32. package/dist/use-push-to-talk.d.ts +105 -0
  33. package/dist/{use-workflow-run-CNahvlgL.js → use-workflow-run-ZqJWJn2-.js} +24 -11
  34. package/dist/workflow-client.d.ts +26 -12
  35. package/package.json +2 -2
  36. package/dist/default-client/assets/index-CMh4E-Ci.css +0 -2
  37. package/dist/default-client/assets/index-eOmPq2sS.js +0 -284
  38. package/dist/{tool-call-block-C2t_5fpp.js → tool-call-block-4gXoL_eP.js} +1 -1
  39. package/dist/{url-chips-D62RTz7v.js → url-chips-0n35fPJY.js} +1 -1
@@ -1,11 +1,52 @@
1
- import { r as loadClientConfig, t as buildAgentUrl } from "./client-config-DJQHnYjm.js";
1
+ import { r as loadClientConfig, t as buildAgentUrl } from "./client-config-CFhkMkmv.js";
2
2
  import { MIC_SEND_MAX_BUFFERED_BYTES } from "./types.js";
3
- import { SessionEventSchema, lenientParse } from "@alexkroman1/aai/protocol";
4
- import { errorMessage, safeJsonParse } from "@alexkroman1/aai";
3
+ import { lenientParse } from "@alexkroman1/aai/protocol";
4
+ import { SessionEventSchema, errorMessage, safeJsonParse } from "@alexkroman1/aai";
5
5
  import { omitUndefined } from "@alexkroman1/aai/utils";
6
6
  import { DEFAULT_MAX_HISTORY, WS_OPEN, createEpoch, toArgsRecord } from "@alexkroman1/aai/internal";
7
+ import { and, assign, createActor, fromPromise, not, setup, stateIn } from "xstate";
7
8
  import ReconnectingWebSocket from "partysocket/ws";
8
- import { and, assign, createActor, not, setup, stateIn } from "xstate";
9
+ //#region src/session-core-audio-setup.ts
10
+ let audioModulesPromise = null;
11
+ /**
12
+ * Load the audio module and worklet sources, memoized so a prefetch at
13
+ * `connect()` time overlaps the chunk fetch with the WebSocket handshake
14
+ * instead of serializing it behind the server's `config` frame. A failed
15
+ * load clears the memo so the next attempt retries the import.
16
+ */
17
+ function loadAudioModules() {
18
+ audioModulesPromise ??= Promise.all([
19
+ import("./audio.js"),
20
+ import("./worklets/capture-processor.js").then((m) => m.default),
21
+ import("./worklets/playback-processor.js").then((m) => m.default)
22
+ ]).catch((err) => {
23
+ audioModulesPromise = null;
24
+ throw err;
25
+ });
26
+ return audioModulesPromise;
27
+ }
28
+ /**
29
+ * Open one audio path: load the worklets, ask for the microphone, and build
30
+ * the {@link VoiceIO} wired to `callbacks`.
31
+ *
32
+ * Rejects when any of the three refuses — a denied mic prompt being the
33
+ * ordinary case. The caller owns closing what this resolves: a path that comes
34
+ * up for a connection nobody wants any more is a live microphone, and no
35
+ * amount of cancellation upstream releases the device.
36
+ */
37
+ async function openAudioPath(config, callbacks) {
38
+ const [{ createVoiceIO }, captureWorklet, playbackWorklet] = await loadAudioModules();
39
+ return createVoiceIO({
40
+ sttSampleRate: config.sampleRate,
41
+ ttsSampleRate: config.ttsSampleRate,
42
+ captureWorkletSrc: captureWorklet,
43
+ playbackWorkletSrc: playbackWorklet,
44
+ onMicData: callbacks.onMicData,
45
+ onPlaybackProgress: callbacks.onProgress,
46
+ onError: (err) => callbacks.onFailure(err.message)
47
+ });
48
+ }
49
+ //#endregion
9
50
  //#region src/session-core-types.ts
10
51
  /**
11
52
  * The two liveness fields at rest.
@@ -39,122 +80,331 @@ const STOPPED = {
39
80
  * bump on its own now reads as a deliberate choice rather than a forgotten
40
81
  * flush.
41
82
  */
42
- function bargeIn(conn) {
83
+ function bargeIn(conn, audio) {
43
84
  conn.turn.bump();
44
- conn.voiceIO?.flush();
85
+ audio.flush();
45
86
  }
46
87
  //#endregion
47
- //#region src/session-core-audio-setup.ts
88
+ //#region src/session-core-audio-effects.ts
89
+ /** Build the effects for one session's audio path. */
90
+ function createAudioEffects(deps) {
91
+ const { conn, updateState, agentState, sendJson, sendAudio } = deps;
92
+ /**
93
+ * Wait for `io`'s playback queue to drain, then go back to listening.
94
+ *
95
+ * Captures `conn.turn` so a completion (or failure) landing after a turn
96
+ * boundary — a barge-in, a committed user turn, an audio-path teardown — is
97
+ * discarded instead of overwriting the newer turn's, or a dead session's,
98
+ * state.
99
+ */
100
+ function settleWhenDrained(io) {
101
+ const at = conn.turn.current();
102
+ io.done().then(() => {
103
+ if (!conn.turn.isCurrent(at)) return;
104
+ updateState(agentState.apply({ type: "LISTEN" }));
105
+ }).catch((err) => {
106
+ console.warn("Audio playback done failed:", err);
107
+ });
108
+ }
109
+ return {
110
+ open: openAudioPath,
111
+ sendMicAudio: (pcm16) => {
112
+ try {
113
+ sendAudio(pcm16);
114
+ } catch {
115
+ console.debug("[aai-ui] sendAudio dropped: connection closed");
116
+ }
117
+ },
118
+ reportProgress: (bufferedMs) => {
119
+ try {
120
+ sendJson({
121
+ type: "playback_progress",
122
+ bufferedMs
123
+ });
124
+ } catch {}
125
+ },
126
+ announceReady: () => sendJson({ type: "audio_ready" }),
127
+ micLive: () => updateState({ recording: true }),
128
+ listen: () => updateState(agentState.apply({ type: "LISTEN" })),
129
+ settleWhenDrained,
130
+ reportFailure: (message) => {
131
+ updateState({
132
+ ...agentState.apply({
133
+ type: "FAILED",
134
+ error: {
135
+ code: "audio",
136
+ message,
137
+ fatal: false
138
+ }
139
+ }),
140
+ ...STOPPED
141
+ });
142
+ },
143
+ endTurn: () => conn.turn.bump(),
144
+ release: (io) => {
145
+ io.close().catch(() => {});
146
+ }
147
+ };
148
+ }
149
+ //#endregion
150
+ //#region src/session-core-audio-state.ts
48
151
  /**
49
- * Audio-path initialization for the voice session core.
152
+ * The browser session's AUDIO PATH — the mic grant, the worklets, and the
153
+ * {@link VoiceIO} they produce — as a statechart.
154
+ *
155
+ * It was a latch, an epoch counter and three buffers spread across
156
+ * `ConnState`, and every one of them was a question the position already
157
+ * answers:
158
+ *
159
+ * - **`audioSetupInFlight` was a dedup, and dedup is a state.**
160
+ * `initAudioCapture` opened with `if (conn.audioSetupInFlight) return` and
161
+ * closed with "only the init that still owns the flag may clear it", which is
162
+ * an ownership problem a boolean cannot express and a position does not have.
163
+ * Here `starting` simply does not handle `START`, and there is no flag to
164
+ * hand back.
165
+ * - **`conn.generation` existed for this module and nothing else.** Four
166
+ * `bump()` sites, four `isCurrent(gen)` reads, and its own field on
167
+ * `ConnState`. All four bumps sat immediately after a `cleanupAudio()` that
168
+ * had *just* cleared the in-flight flag — the reconnect close handler said so
169
+ * outright ("cleanupAudio just cleared the in-flight flag, so a survivor
170
+ * would otherwise pass the same-generation guard and double-run"). Two
171
+ * mechanisms for one question, the second one layered on because the first
172
+ * could not stop the work already running. A stopped `invoke` can, so the
173
+ * counter is gone.
174
+ * - **A counter cannot tell two bring-ups in ONE generation apart, and they
175
+ * happen.** A repeated `config` frame on a live connection re-opened the path
176
+ * without bumping anything, so the outgoing `VoiceIO`'s `onError` still
177
+ * passed `isCurrent(gen)` and could tear down the replacement that had just
178
+ * taken its slot. `PROGRESS` and `IO_FAILED` carry the instance that fired
179
+ * them now, and identity is exact where a counter was approximate.
180
+ * - **`preInitAudio` / `preInitDone` are only meaningful while the path is
181
+ * down.** As connection fields they were readable — and writable — from
182
+ * anywhere; as context of `down`/`starting` they are cleared by entering
183
+ * `down` rather than by three call sites remembering to.
184
+ *
185
+ * ## What cancellation does NOT do, and the one guard that had to survive
50
186
  *
51
- * Split out of `session-core.ts`: this module owns the async
52
- * mic-permission → worklet-registration → `VoiceIO` bring-up (and its
53
- * staleness/failure handling), while `session-core.ts` owns the state store
54
- * and connection lifecycle.
187
+ * Stopping the `bringUp` actor hides its resolution; it does not close a
188
+ * microphone the browser has already granted. So `bringUp` checks its own
189
+ * `signal` after `openAudioPath` settles and releases what it built. That is
190
+ * the single remaining "is this still wanted?" test, and it lives in the
191
+ * producer — which owns the value nobody adopted — rather than at the four
192
+ * consumer sites that used to re-ask.
193
+ *
194
+ * ## Effects stay in the session core
195
+ *
196
+ * The machine owns WHEN. Every HOW — the socket frame, the snapshot write, the
197
+ * turn-boundary drain — arrives as an injected {@link AudioPathEffects} call,
198
+ * the same shape `s2s-lifecycle.ts` uses. So this module touches no socket and
199
+ * no snapshot.
55
200
  */
56
- let audioModulesPromise = null;
57
201
  /**
58
- * Load the audio module and worklet sources, memoized so a prefetch at
59
- * `connect()` time overlaps the chunk fetch with the WebSocket handshake
60
- * instead of serializing it behind the server's `config` frame. A failed
61
- * load clears the memo so the next attempt retries the import.
202
+ * Cap on audio chunks buffered while the path is coming up. ~100 chunks at
203
+ * typical S2S chunk sizes is well over a second of audio — far longer than a
204
+ * bring-up takes in practice, but bounded against pathological cases
205
+ * (mic-permission stalls).
62
206
  */
63
- function loadAudioModules() {
64
- audioModulesPromise ??= Promise.all([
65
- import("./audio.js"),
66
- import("./worklets/capture-processor.js").then((m) => m.default),
67
- import("./worklets/playback-processor.js").then((m) => m.default)
68
- ]).catch((err) => {
69
- audioModulesPromise = null;
70
- throw err;
71
- });
72
- return audioModulesPromise;
73
- }
207
+ const MAX_BUFFERED_CHUNKS = 100;
74
208
  /**
75
- * Initialize audio capture and playback after the server sends a ready config.
76
- *
77
- * Lifecycle: dynamically import audio modules -> request microphone access ->
78
- * register AudioWorklet processors -> create a `VoiceIO` instance -> send
79
- * `audio_ready` to the server -> transition state to `"listening"`.
209
+ * The rejection a bring-up abandons itself with.
80
210
  *
81
- * Uses the connection `generation` counter to detect if `connect()` was called
82
- * (or a reconnect happened) while awaiting async operations; if so, the stale
83
- * VoiceIO is closed immediately to prevent it from being assigned to a newer
84
- * connection.
85
- *
86
- * A failure is fatal: a voice session can't function without the mic, so it
87
- * sets the error state and ends the session.
211
+ * Never observed: the actor is already stopped when it is thrown, and XState
212
+ * drops a settlement from a stopped actor. It exists so the `async` function
213
+ * has a way to end that is not "resolve with a VoiceIO I have just closed".
88
214
  */
89
- async function initAudioCapture(conn, msg, deps) {
90
- if (conn.audioSetupInFlight) return;
91
- conn.audioSetupInFlight = true;
92
- const gen = conn.generation.current();
93
- const stale = () => !(conn.generation.isCurrent(gen) && conn.ws) || conn.ws.readyState !== WS_OPEN;
94
- const reportAudioFailure = (message) => {
95
- deps.cleanupAudio();
96
- deps.updateState({
97
- ...deps.agentState.apply({
98
- type: "FAILED",
99
- error: {
100
- code: "audio",
101
- message,
102
- fatal: false
103
- }
104
- }),
105
- ...STOPPED
106
- });
107
- };
108
- try {
109
- const [{ createVoiceIO }, captureWorklet, playbackWorklet] = await loadAudioModules();
110
- const io = await createVoiceIO({
111
- sttSampleRate: msg.sampleRate,
112
- ttsSampleRate: msg.ttsSampleRate,
113
- captureWorkletSrc: captureWorklet,
114
- playbackWorkletSrc: playbackWorklet,
115
- onMicData: (pcm16) => {
116
- try {
117
- deps.sendAudio(pcm16);
118
- } catch {
119
- console.debug("[aai-ui] sendAudio dropped: connection closed");
120
- }
121
- },
122
- onPlaybackProgress: (bufferedMs) => {
123
- if (!conn.generation.isCurrent(gen)) return;
124
- try {
125
- deps.sendJson({
126
- type: "playback_progress",
127
- bufferedMs
128
- });
129
- } catch {}
215
+ const ABANDONED = /* @__PURE__ */ new Error("audio path abandoned");
216
+ const audioPathMachine = setup({
217
+ types: {},
218
+ actors: {
219
+ /**
220
+ * One bring-up. An actor rather than a fire-and-forget `void init(...)`
221
+ * precisely so that leaving `starting` — a hang-up, a reconnect, a fatal
222
+ * frame — stops it, instead of letting it settle into a connection that is
223
+ * already over.
224
+ */
225
+ bringUp: fromPromise(async ({ input, signal }) => {
226
+ let mine = null;
227
+ const io = await input.effects.open(input.config, {
228
+ onMicData: (pcm16) => input.effects.sendMicAudio(pcm16),
229
+ onProgress: (bufferedMs) => {
230
+ if (mine) input.send({
231
+ type: "PROGRESS",
232
+ io: mine,
233
+ bufferedMs
234
+ });
130
235
  },
131
- onError: (err) => {
132
- if (!conn.generation.isCurrent(gen)) return;
133
- reportAudioFailure(err.message);
236
+ onFailure: (message) => {
237
+ if (mine) input.send({
238
+ type: "IO_FAILED",
239
+ io: mine,
240
+ message
241
+ });
134
242
  }
135
243
  });
136
- if (stale()) {
137
- io.close().catch(() => {});
138
- return;
244
+ if (signal.aborted) {
245
+ input.effects.release(io);
246
+ throw ABANDONED;
139
247
  }
140
- conn.voiceIO?.close().catch(() => {});
141
- conn.voiceIO = io;
142
- if (conn.preInitAudio.length > 0) {
143
- for (const chunk of conn.preInitAudio) io.enqueue(chunk);
144
- conn.preInitAudio = [];
248
+ mine = io;
249
+ return io;
250
+ }) },
251
+ guards: {
252
+ /**
253
+ * Did the live path report this, or one it replaced?
254
+ *
255
+ * Identity rather than `generation.isCurrent(gen)`: two bring-ups inside a
256
+ * single connection share a generation, and the counter passed for the
257
+ * instance that had just been closed.
258
+ */
259
+ fromLiveIo: ({ context, event }) => (event.type === "PROGRESS" || event.type === "IO_FAILED") && context.io === event.io },
260
+ actions: {
261
+ rememberConfig: assign({ config: ({ context, event }) => event.type === "START" ? event.config : context.config }),
262
+ /** Hold a chunk that arrived too early, up to the cap. */
263
+ buffer: assign({ pending: ({ context, event }) => event.type === "CHUNK" && context.pending.length < MAX_BUFFERED_CHUNKS ? [...context.pending, event.chunk] : context.pending }),
264
+ /** Record a done that arrived too early. */
265
+ rememberDone: assign({ pendingDone: true }),
266
+ /** Answer it optimistically: with no audio pipeline there is nothing to wait for. */
267
+ listenNow: ({ context }) => context.effects.listen(),
268
+ /** Nothing buffered belongs to the next path. */
269
+ forgetBuffered: assign({
270
+ io: null,
271
+ pending: [],
272
+ pendingDone: false
273
+ }),
274
+ /** Releasing the path ends the turn it was playing — see `endTurn`. */
275
+ endTurn: ({ context }) => context.effects.endTurn(),
276
+ releaseIo: ({ context }) => {
277
+ if (context.io) context.effects.release(context.io);
278
+ }
279
+ }
280
+ }).createMachine({
281
+ id: "audioPath",
282
+ context: ({ input }) => ({
283
+ effects: input,
284
+ io: null,
285
+ pending: [],
286
+ pendingDone: false,
287
+ config: null
288
+ }),
289
+ initial: "down",
290
+ on: {
291
+ CHUNK: { actions: "buffer" },
292
+ DONE: { actions: ["rememberDone", "listenNow"] },
293
+ TEARDOWN: ".down"
294
+ },
295
+ states: {
296
+ /**
297
+ * No audio path. Entered on every release, so its entry is the one place
298
+ * that ends the turn and drops what was buffered for a path that is gone.
299
+ */
300
+ down: {
301
+ entry: ["endTurn", "forgetBuffered"],
302
+ on: { START: {
303
+ target: "starting",
304
+ actions: "rememberConfig"
305
+ } }
306
+ },
307
+ /**
308
+ * A bring-up is in flight.
309
+ *
310
+ * `START` is absent deliberately: that is the whole of what
311
+ * `audioSetupInFlight` said, and a second `config` frame arriving mid-grant
312
+ * is now declined by the position rather than by a flag whose owner had to
313
+ * be tracked.
314
+ */
315
+ starting: { invoke: {
316
+ src: "bringUp",
317
+ input: ({ context, self }) => ({
318
+ effects: context.effects,
319
+ config: context.config ?? {
320
+ sampleRate: 0,
321
+ ttsSampleRate: 0
322
+ },
323
+ send: (event) => {
324
+ self.send(event);
325
+ }
326
+ }),
327
+ onDone: {
328
+ target: "up",
329
+ actions: [
330
+ ({ context, event }) => {
331
+ for (const chunk of context.pending) event.output.enqueue(chunk);
332
+ },
333
+ assign({
334
+ io: ({ event }) => event.output,
335
+ pending: []
336
+ }),
337
+ ({ context, event }) => {
338
+ context.effects.announceReady();
339
+ context.effects.micLive();
340
+ if (context.pendingDone) context.effects.settleWhenDrained(event.output);
341
+ else context.effects.listen();
342
+ },
343
+ assign({ pendingDone: false })
344
+ ]
345
+ },
346
+ onError: {
347
+ target: "down",
348
+ actions: ({ context, event }) => {
349
+ context.effects.reportFailure(`Microphone access failed: ${errorMessage(event.error)}`);
350
+ }
351
+ }
352
+ } },
353
+ /**
354
+ * A live path. Its exit releases it, which covers all three ways out —
355
+ * teardown, a worklet death, and the repeated `config` frame that opens a
356
+ * replacement (the "defensive" close that used to sit inside the bring-up,
357
+ * where it could only run if the bring-up got that far).
358
+ */
359
+ up: {
360
+ exit: "releaseIo",
361
+ on: {
362
+ START: {
363
+ target: "starting",
364
+ actions: "rememberConfig"
365
+ },
366
+ CHUNK: { actions: ({ context, event }) => context.io?.enqueue(event.chunk) },
367
+ DONE: { actions: ({ context }) => {
368
+ if (context.io) context.effects.settleWhenDrained(context.io);
369
+ } },
370
+ PROGRESS: {
371
+ guard: "fromLiveIo",
372
+ actions: ({ context, event }) => context.effects.reportProgress(event.bufferedMs)
373
+ },
374
+ IO_FAILED: {
375
+ guard: "fromLiveIo",
376
+ target: "down",
377
+ actions: ({ context, event }) => context.effects.reportFailure(event.message)
378
+ }
379
+ }
145
380
  }
146
- deps.sendJson({ type: "audio_ready" });
147
- deps.updateState({ recording: true });
148
- if (conn.preInitDone) {
149
- conn.preInitDone = false;
150
- deps.settleWhenAudioDrained(io);
151
- } else deps.updateState(deps.agentState.apply({ type: "LISTEN" }));
152
- } catch (err) {
153
- if (stale()) return;
154
- reportAudioFailure(`Microphone access failed: ${errorMessage(err)}`);
155
- } finally {
156
- if (conn.generation.isCurrent(gen)) conn.audioSetupInFlight = false;
157
381
  }
382
+ });
383
+ /** Create the audio path for one browser session. */
384
+ function createAudioPath(effects) {
385
+ const actor = createActor(audioPathMachine, { input: effects }).start();
386
+ return {
387
+ phase: () => actor.getSnapshot().value,
388
+ start: (config) => {
389
+ actor.send({
390
+ type: "START",
391
+ config
392
+ });
393
+ },
394
+ enqueue: (chunk) => {
395
+ actor.send({
396
+ type: "CHUNK",
397
+ chunk
398
+ });
399
+ },
400
+ done: () => {
401
+ actor.send({ type: "DONE" });
402
+ },
403
+ flush: () => actor.getSnapshot().context.io?.flush(),
404
+ teardown: () => {
405
+ actor.send({ type: "TEARDOWN" });
406
+ }
407
+ };
158
408
  }
159
409
  //#endregion
160
410
  //#region src/session-core-close.ts
@@ -544,7 +794,7 @@ function createHandshakeGuard(opts) {
544
794
  *
545
795
  * Split out of `session-core.ts`: this module owns the interpretation of
546
796
  * server→client frames (audio chunks + JSON {@link SessionEvent}s) and the
547
- * turn-boundary generation counters, while `session-core.ts` owns the state
797
+ * turn-boundary epoch, while `session-core.ts` owns the state
548
798
  * store and connection lifecycle. The handlers read and mutate session state
549
799
  * exclusively through the injected `getSnapshot`/`updateState` deps.
550
800
  */
@@ -552,10 +802,6 @@ function createHandshakeGuard(opts) {
552
802
  const MAX_CUSTOM_EVENTS = 200;
553
803
  /** Cap on `messages` retained in the session snapshot; matches the host-side history cap. */
554
804
  const MAX_MESSAGES = DEFAULT_MAX_HISTORY;
555
- /** Cap on pre-init audio chunks buffered while `voiceIO` is initializing. ~100 chunks at
556
- * typical S2S chunk sizes is well over a second of audio — far longer than init takes
557
- * in practice, but bounded against pathological cases (mic-permission stalls). */
558
- const MAX_PREINIT_AUDIO_CHUNKS = 100;
559
805
  /**
560
806
  * Snapshot fields cleared when a session's conversation state is wiped —
561
807
  * shared by the initial snapshot, `resetState()`, and `session.reset`.
@@ -587,7 +833,7 @@ function appendCapped(list, item, cap) {
587
833
  * `ConnState.turn`).
588
834
  */
589
835
  function createMessageHandlers(deps) {
590
- const { getSnapshot, updateState, conn, agentState, cleanupAudio } = deps;
836
+ const { getSnapshot, updateState, conn, agentState, audio } = deps;
591
837
  /** Monotonically increasing counter for custom events -- used by useEvent to deduplicate. */
592
838
  let customEventSeq = 0;
593
839
  /** Monotonically increasing counter for chat messages -- stable render keys
@@ -694,8 +940,7 @@ function createMessageHandlers(deps) {
694
940
  error
695
941
  }));
696
942
  else {
697
- cleanupAudio();
698
- conn.generation.bump();
943
+ audio.teardown();
699
944
  updateState({
700
945
  ...agentState.apply({
701
946
  type: "FATAL",
@@ -752,12 +997,12 @@ function createMessageHandlers(deps) {
752
997
  toListening();
753
998
  break;
754
999
  case "reply.cancelled":
755
- bargeIn(conn);
1000
+ bargeIn(conn, audio);
756
1001
  commitAgentTranscript();
757
1002
  toListening({ userTranscript: null });
758
1003
  break;
759
1004
  case "session.reset": {
760
- bargeIn(conn);
1005
+ bargeIn(conn, audio);
761
1006
  const next = agentState.apply({ type: "RESET" });
762
1007
  updateState(agentState.fatal() ? next : {
763
1008
  ...CLEARED_SESSION_STATE,
@@ -802,34 +1047,18 @@ function createMessageHandlers(deps) {
802
1047
  /** Enqueue a PCM16 audio chunk for playback. Transitions state to `"speaking"` on the first chunk. */
803
1048
  function playAudioChunk(chunk) {
804
1049
  updateState(agentState.apply({ type: "SPEAK" }));
805
- if (conn.voiceIO) conn.voiceIO.enqueue(chunk);
806
- else if (conn.preInitAudio.length < MAX_PREINIT_AUDIO_CHUNKS) conn.preInitAudio.push(chunk);
807
- }
808
- /** See {@link MessageHandlers.settleWhenAudioDrained}. Captures
809
- * `conn.turn` so a completion (or failure) that lands after a turn
810
- * boundary — including an audio-path teardown — is discarded instead of
811
- * overwriting the newer turn's (or the dead session's) state. */
812
- function settleWhenAudioDrained(io) {
813
- const gen = conn.turn.current();
814
- io.done().then(() => {
815
- if (!conn.turn.isCurrent(gen)) return;
816
- updateState(agentState.apply({ type: "LISTEN" }));
817
- }).catch((err) => {
818
- console.warn("Audio playback done failed:", err);
819
- });
1050
+ audio.enqueue(chunk);
820
1051
  }
821
1052
  /**
822
1053
  * Signal that the server has finished sending audio for this turn.
823
- * Waits for the audio queue to drain, then transitions state to `"listening"`.
824
- * Uses `conn.turn` to discard stale completions from interrupted turns.
1054
+ *
1055
+ * A path that is up waits for its queue to drain before going back to
1056
+ * listening; one still coming up records the done and answers optimistically,
1057
+ * then replays it once the buffered greeting has somewhere to go. Both arms
1058
+ * live in `session-core-audio-state.ts`.
825
1059
  */
826
1060
  function playAudioDone() {
827
- const io = conn.voiceIO;
828
- if (io) settleWhenAudioDrained(io);
829
- else {
830
- conn.preInitDone = true;
831
- updateState(agentState.apply({ type: "LISTEN" }));
832
- }
1061
+ audio.done();
833
1062
  }
834
1063
  function handleMessage(data) {
835
1064
  if (data instanceof ArrayBuffer) {
@@ -865,10 +1094,7 @@ function createMessageHandlers(deps) {
865
1094
  }
866
1095
  handleEvent(msg);
867
1096
  }
868
- return {
869
- handleMessage,
870
- settleWhenAudioDrained
871
- };
1097
+ return { handleMessage };
872
1098
  }
873
1099
  //#endregion
874
1100
  //#region src/session-core-state.ts
@@ -1053,6 +1279,31 @@ function createSessionStateMachine() {
1053
1279
  };
1054
1280
  }
1055
1281
  //#endregion
1282
+ //#region src/session-core-user-turn.ts
1283
+ /** Build the three push-to-talk edges. @internal */
1284
+ function createUserTurnActions(deps) {
1285
+ return {
1286
+ start() {
1287
+ if (!deps.connected()) return;
1288
+ const { state } = deps.snapshot();
1289
+ if (state === "speaking" || state === "thinking") {
1290
+ deps.bargeIn();
1291
+ deps.updateState(deps.agentState.apply({ type: "LISTEN" }));
1292
+ }
1293
+ deps.sendJson({ type: "user_turn_start" });
1294
+ },
1295
+ commit() {
1296
+ if (!deps.connected()) return;
1297
+ deps.sendJson({ type: "user_turn_commit" });
1298
+ },
1299
+ clear() {
1300
+ if (!deps.connected()) return;
1301
+ deps.updateState({ userTranscript: null });
1302
+ deps.sendJson({ type: "user_turn_clear" });
1303
+ }
1304
+ };
1305
+ }
1306
+ //#endregion
1056
1307
  //#region src/session-core.ts
1057
1308
  /**
1058
1309
  * Framework-agnostic voice session core.
@@ -1148,23 +1399,10 @@ function createBrowserSession(options) {
1148
1399
  const conn = {
1149
1400
  ws: null,
1150
1401
  retiredByServer: false,
1151
- voiceIO: null,
1152
- audioSetupInFlight: false,
1153
- generation: createEpoch(),
1154
- turn: createEpoch(),
1155
- preInitAudio: [],
1156
- preInitDone: false
1402
+ turn: createEpoch()
1157
1403
  };
1158
1404
  let connectionController = null;
1159
1405
  const dialer = createDialer(options);
1160
- function cleanupAudio() {
1161
- conn.audioSetupInFlight = false;
1162
- conn.turn.bump();
1163
- conn.voiceIO?.close().catch(() => {});
1164
- conn.voiceIO = null;
1165
- conn.preInitAudio = [];
1166
- conn.preInitDone = false;
1167
- }
1168
1406
  function resetState() {
1169
1407
  updateState(CLEARED_SESSION_STATE);
1170
1408
  }
@@ -1187,26 +1425,32 @@ function createBrowserSession(options) {
1187
1425
  if (ws.bufferedAmount > MIC_SEND_MAX_BUFFERED_BYTES) return;
1188
1426
  ws.send(bytes);
1189
1427
  }
1190
- const { handleMessage, settleWhenAudioDrained } = createMessageHandlers({
1191
- getSnapshot,
1192
- updateState,
1428
+ /**
1429
+ * The microphone, the worklets and the buffer in front of them, as a
1430
+ * statechart — see `session-core-audio-state.ts`, which carries what the
1431
+ * in-flight latch, the generation epoch and the two pre-init fields on
1432
+ * `ConnState` cost. `session-core-audio-effects.ts` is the other half: every
1433
+ * frame and snapshot write the machine decides on but cannot make.
1434
+ */
1435
+ const audio = createAudioPath(createAudioEffects({
1193
1436
  conn,
1437
+ updateState,
1194
1438
  agentState,
1195
- cleanupAudio
1196
- });
1197
- const audioDeps = {
1198
1439
  sendJson,
1199
- sendAudio,
1440
+ sendAudio
1441
+ }));
1442
+ const { handleMessage } = createMessageHandlers({
1443
+ getSnapshot,
1200
1444
  updateState,
1445
+ conn,
1201
1446
  agentState,
1202
- settleWhenAudioDrained,
1203
- cleanupAudio
1204
- };
1447
+ audio
1448
+ });
1205
1449
  /** Abort the in-flight connection and release audio + WebSocket resources. */
1206
1450
  function teardownConnection() {
1207
1451
  connectionController?.abort();
1208
1452
  connectionController = null;
1209
- cleanupAudio();
1453
+ audio.teardown();
1210
1454
  conn.ws?.close();
1211
1455
  conn.ws = null;
1212
1456
  }
@@ -1224,7 +1468,7 @@ function createBrowserSession(options) {
1224
1468
  function onServerConfig(config) {
1225
1469
  dialer.configured(config.sid);
1226
1470
  if (config.sid) options.onSessionId?.(config.sid);
1227
- initAudioCapture(conn, config, audioDeps);
1471
+ audio.start(config);
1228
1472
  }
1229
1473
  function connect(opts) {
1230
1474
  if (opts?.signal?.aborted) {
@@ -1234,7 +1478,6 @@ function createBrowserSession(options) {
1234
1478
  updateState(agentState.apply({ type: "CONNECT" }));
1235
1479
  loadAudioModules().catch(() => {});
1236
1480
  teardownConnection();
1237
- conn.generation.bump();
1238
1481
  conn.retiredByServer = false;
1239
1482
  const controller = new AbortController();
1240
1483
  connectionController = controller;
@@ -1248,15 +1491,14 @@ function createBrowserSession(options) {
1248
1491
  socket,
1249
1492
  signal: sig,
1250
1493
  onRetry: () => {
1251
- cleanupAudio();
1252
- conn.generation.bump();
1494
+ audio.teardown();
1253
1495
  updateState({
1254
1496
  ...agentState.apply({ type: "CONNECT" }),
1255
1497
  recording: false
1256
1498
  });
1257
1499
  },
1258
1500
  onExhausted: () => {
1259
- cleanupAudio();
1501
+ audio.teardown();
1260
1502
  controller.abort();
1261
1503
  socket.close();
1262
1504
  conn.ws = null;
@@ -1285,9 +1527,8 @@ function createBrowserSession(options) {
1285
1527
  socket.addEventListener("close", (event) => {
1286
1528
  if (sig.aborted) return;
1287
1529
  handshake.disarm();
1288
- cleanupAudio();
1530
+ audio.teardown();
1289
1531
  if (!(conn.retiredByServer || agentState.fatal()) && reconnectPending(socket)) {
1290
- conn.generation.bump();
1291
1532
  socketErrored = false;
1292
1533
  updateState({
1293
1534
  ...agentState.apply({ type: "CONNECT" }),
@@ -1314,12 +1555,20 @@ function createBrowserSession(options) {
1314
1555
  }
1315
1556
  function cancel() {
1316
1557
  if (!openSocket()) return;
1317
- bargeIn(conn);
1558
+ bargeIn(conn, audio);
1318
1559
  updateState(agentState.apply({ type: "LISTEN" }));
1319
1560
  sendJson({ type: "cancel" });
1320
1561
  }
1562
+ const userTurn = createUserTurnActions({
1563
+ snapshot: () => currentSnapshot,
1564
+ connected: () => openSocket() !== null,
1565
+ bargeIn: () => bargeIn(conn, audio),
1566
+ agentState,
1567
+ updateState,
1568
+ sendJson
1569
+ });
1321
1570
  function reset() {
1322
- bargeIn(conn);
1571
+ bargeIn(conn, audio);
1323
1572
  if (openSocket()) {
1324
1573
  sendJson({ type: "reset" });
1325
1574
  return;
@@ -1367,6 +1616,7 @@ function createBrowserSession(options) {
1367
1616
  subscribe,
1368
1617
  connect,
1369
1618
  cancel,
1619
+ userTurn: Object.freeze(userTurn),
1370
1620
  resetState,
1371
1621
  reset,
1372
1622
  disconnect,