@alexkroman1/aai-ui 16.1.0 → 18.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/_page-shell.d.ts +11 -0
- package/dist/{chat-view-DsylXJsw.js → chat-view-P12pHKQU.js} +1 -1
- package/dist/{client-config-DJQHnYjm.js → client-config-CFhkMkmv.js} +15 -13
- package/dist/client-config.d.ts +14 -13
- package/dist/components/chat-view.js +1 -1
- package/dist/components/controls.js +1 -1
- package/dist/components/message-list.js +1 -1
- package/dist/components/tool-call-block.js +1 -1
- package/dist/context.d.ts +27 -5
- package/dist/context.js +3 -3
- package/dist/default-client/assets/index-5XG8PqGw.css +2 -0
- package/dist/default-client/assets/index-tc-nAWq3.js +121 -0
- package/dist/default-client/index.html +2 -2
- package/dist/default-client.d.ts +21 -0
- package/dist/define-client.js +4 -4
- package/dist/hooks.d.ts +19 -9
- package/dist/hooks.js +1 -1
- package/dist/index.d.ts +17 -13
- package/dist/index.js +734 -433
- package/dist/internal.js +3 -3
- package/dist/{message-list-BLyPRS1Q.js → message-list-CJX560b5.js} +1 -1
- package/dist/page.d.ts +47 -21
- package/dist/{session-core-gwePM95B.js → session-core-TOdm_Con.js} +424 -174
- package/dist/session-core-audio-effects.d.ts +26 -0
- package/dist/session-core-audio-setup.d.ts +38 -31
- package/dist/session-core-audio-state.d.ts +152 -0
- package/dist/session-core-messages.d.ts +9 -10
- package/dist/session-core-types.d.ts +60 -21
- package/dist/session-core-user-turn.d.ts +30 -0
- package/dist/session-core.js +1 -1
- package/dist/use-push-to-talk.d.ts +105 -0
- package/dist/{use-workflow-run-CNahvlgL.js → use-workflow-run-ZqJWJn2-.js} +24 -11
- package/dist/workflow-client.d.ts +26 -12
- package/package.json +2 -2
- package/dist/default-client/assets/index-CMh4E-Ci.css +0 -2
- package/dist/default-client/assets/index-eOmPq2sS.js +0 -284
- package/dist/{tool-call-block-C2t_5fpp.js → tool-call-block-4gXoL_eP.js} +1 -1
- package/dist/{url-chips-D62RTz7v.js → url-chips-0n35fPJY.js} +1 -1
|
@@ -1,11 +1,52 @@
|
|
|
1
|
-
import { r as loadClientConfig, t as buildAgentUrl } from "./client-config-
|
|
1
|
+
import { r as loadClientConfig, t as buildAgentUrl } from "./client-config-CFhkMkmv.js";
|
|
2
2
|
import { MIC_SEND_MAX_BUFFERED_BYTES } from "./types.js";
|
|
3
|
-
import {
|
|
4
|
-
import { errorMessage, safeJsonParse } from "@alexkroman1/aai";
|
|
3
|
+
import { lenientParse } from "@alexkroman1/aai/protocol";
|
|
4
|
+
import { SessionEventSchema, errorMessage, safeJsonParse } from "@alexkroman1/aai";
|
|
5
5
|
import { omitUndefined } from "@alexkroman1/aai/utils";
|
|
6
6
|
import { DEFAULT_MAX_HISTORY, WS_OPEN, createEpoch, toArgsRecord } from "@alexkroman1/aai/internal";
|
|
7
|
+
import { and, assign, createActor, fromPromise, not, setup, stateIn } from "xstate";
|
|
7
8
|
import ReconnectingWebSocket from "partysocket/ws";
|
|
8
|
-
|
|
9
|
+
//#region src/session-core-audio-setup.ts
|
|
10
|
+
let audioModulesPromise = null;
|
|
11
|
+
/**
|
|
12
|
+
* Load the audio module and worklet sources, memoized so a prefetch at
|
|
13
|
+
* `connect()` time overlaps the chunk fetch with the WebSocket handshake
|
|
14
|
+
* instead of serializing it behind the server's `config` frame. A failed
|
|
15
|
+
* load clears the memo so the next attempt retries the import.
|
|
16
|
+
*/
|
|
17
|
+
function loadAudioModules() {
|
|
18
|
+
audioModulesPromise ??= Promise.all([
|
|
19
|
+
import("./audio.js"),
|
|
20
|
+
import("./worklets/capture-processor.js").then((m) => m.default),
|
|
21
|
+
import("./worklets/playback-processor.js").then((m) => m.default)
|
|
22
|
+
]).catch((err) => {
|
|
23
|
+
audioModulesPromise = null;
|
|
24
|
+
throw err;
|
|
25
|
+
});
|
|
26
|
+
return audioModulesPromise;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Open one audio path: load the worklets, ask for the microphone, and build
|
|
30
|
+
* the {@link VoiceIO} wired to `callbacks`.
|
|
31
|
+
*
|
|
32
|
+
* Rejects when any of the three refuses — a denied mic prompt being the
|
|
33
|
+
* ordinary case. The caller owns closing what this resolves: a path that comes
|
|
34
|
+
* up for a connection nobody wants any more is a live microphone, and no
|
|
35
|
+
* amount of cancellation upstream releases the device.
|
|
36
|
+
*/
|
|
37
|
+
async function openAudioPath(config, callbacks) {
|
|
38
|
+
const [{ createVoiceIO }, captureWorklet, playbackWorklet] = await loadAudioModules();
|
|
39
|
+
return createVoiceIO({
|
|
40
|
+
sttSampleRate: config.sampleRate,
|
|
41
|
+
ttsSampleRate: config.ttsSampleRate,
|
|
42
|
+
captureWorkletSrc: captureWorklet,
|
|
43
|
+
playbackWorkletSrc: playbackWorklet,
|
|
44
|
+
onMicData: callbacks.onMicData,
|
|
45
|
+
onPlaybackProgress: callbacks.onProgress,
|
|
46
|
+
onError: (err) => callbacks.onFailure(err.message)
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
//#endregion
|
|
9
50
|
//#region src/session-core-types.ts
|
|
10
51
|
/**
|
|
11
52
|
* The two liveness fields at rest.
|
|
@@ -39,122 +80,331 @@ const STOPPED = {
|
|
|
39
80
|
* bump on its own now reads as a deliberate choice rather than a forgotten
|
|
40
81
|
* flush.
|
|
41
82
|
*/
|
|
42
|
-
function bargeIn(conn) {
|
|
83
|
+
function bargeIn(conn, audio) {
|
|
43
84
|
conn.turn.bump();
|
|
44
|
-
|
|
85
|
+
audio.flush();
|
|
45
86
|
}
|
|
46
87
|
//#endregion
|
|
47
|
-
//#region src/session-core-audio-
|
|
88
|
+
//#region src/session-core-audio-effects.ts
|
|
89
|
+
/** Build the effects for one session's audio path. */
|
|
90
|
+
function createAudioEffects(deps) {
|
|
91
|
+
const { conn, updateState, agentState, sendJson, sendAudio } = deps;
|
|
92
|
+
/**
|
|
93
|
+
* Wait for `io`'s playback queue to drain, then go back to listening.
|
|
94
|
+
*
|
|
95
|
+
* Captures `conn.turn` so a completion (or failure) landing after a turn
|
|
96
|
+
* boundary — a barge-in, a committed user turn, an audio-path teardown — is
|
|
97
|
+
* discarded instead of overwriting the newer turn's, or a dead session's,
|
|
98
|
+
* state.
|
|
99
|
+
*/
|
|
100
|
+
function settleWhenDrained(io) {
|
|
101
|
+
const at = conn.turn.current();
|
|
102
|
+
io.done().then(() => {
|
|
103
|
+
if (!conn.turn.isCurrent(at)) return;
|
|
104
|
+
updateState(agentState.apply({ type: "LISTEN" }));
|
|
105
|
+
}).catch((err) => {
|
|
106
|
+
console.warn("Audio playback done failed:", err);
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
open: openAudioPath,
|
|
111
|
+
sendMicAudio: (pcm16) => {
|
|
112
|
+
try {
|
|
113
|
+
sendAudio(pcm16);
|
|
114
|
+
} catch {
|
|
115
|
+
console.debug("[aai-ui] sendAudio dropped: connection closed");
|
|
116
|
+
}
|
|
117
|
+
},
|
|
118
|
+
reportProgress: (bufferedMs) => {
|
|
119
|
+
try {
|
|
120
|
+
sendJson({
|
|
121
|
+
type: "playback_progress",
|
|
122
|
+
bufferedMs
|
|
123
|
+
});
|
|
124
|
+
} catch {}
|
|
125
|
+
},
|
|
126
|
+
announceReady: () => sendJson({ type: "audio_ready" }),
|
|
127
|
+
micLive: () => updateState({ recording: true }),
|
|
128
|
+
listen: () => updateState(agentState.apply({ type: "LISTEN" })),
|
|
129
|
+
settleWhenDrained,
|
|
130
|
+
reportFailure: (message) => {
|
|
131
|
+
updateState({
|
|
132
|
+
...agentState.apply({
|
|
133
|
+
type: "FAILED",
|
|
134
|
+
error: {
|
|
135
|
+
code: "audio",
|
|
136
|
+
message,
|
|
137
|
+
fatal: false
|
|
138
|
+
}
|
|
139
|
+
}),
|
|
140
|
+
...STOPPED
|
|
141
|
+
});
|
|
142
|
+
},
|
|
143
|
+
endTurn: () => conn.turn.bump(),
|
|
144
|
+
release: (io) => {
|
|
145
|
+
io.close().catch(() => {});
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
//#endregion
|
|
150
|
+
//#region src/session-core-audio-state.ts
|
|
48
151
|
/**
|
|
49
|
-
*
|
|
152
|
+
* The browser session's AUDIO PATH — the mic grant, the worklets, and the
|
|
153
|
+
* {@link VoiceIO} they produce — as a statechart.
|
|
154
|
+
*
|
|
155
|
+
* It was a latch, an epoch counter and three buffers spread across
|
|
156
|
+
* `ConnState`, and every one of them was a question the position already
|
|
157
|
+
* answers:
|
|
158
|
+
*
|
|
159
|
+
* - **`audioSetupInFlight` was a dedup, and dedup is a state.**
|
|
160
|
+
* `initAudioCapture` opened with `if (conn.audioSetupInFlight) return` and
|
|
161
|
+
* closed with "only the init that still owns the flag may clear it", which is
|
|
162
|
+
* an ownership problem a boolean cannot express and a position does not have.
|
|
163
|
+
* Here `starting` simply does not handle `START`, and there is no flag to
|
|
164
|
+
* hand back.
|
|
165
|
+
* - **`conn.generation` existed for this module and nothing else.** Four
|
|
166
|
+
* `bump()` sites, four `isCurrent(gen)` reads, and its own field on
|
|
167
|
+
* `ConnState`. All four bumps sat immediately after a `cleanupAudio()` that
|
|
168
|
+
* had *just* cleared the in-flight flag — the reconnect close handler said so
|
|
169
|
+
* outright ("cleanupAudio just cleared the in-flight flag, so a survivor
|
|
170
|
+
* would otherwise pass the same-generation guard and double-run"). Two
|
|
171
|
+
* mechanisms for one question, the second one layered on because the first
|
|
172
|
+
* could not stop the work already running. A stopped `invoke` can, so the
|
|
173
|
+
* counter is gone.
|
|
174
|
+
* - **A counter cannot tell two bring-ups in ONE generation apart, and they
|
|
175
|
+
* happen.** A repeated `config` frame on a live connection re-opened the path
|
|
176
|
+
* without bumping anything, so the outgoing `VoiceIO`'s `onError` still
|
|
177
|
+
* passed `isCurrent(gen)` and could tear down the replacement that had just
|
|
178
|
+
* taken its slot. `PROGRESS` and `IO_FAILED` carry the instance that fired
|
|
179
|
+
* them now, and identity is exact where a counter was approximate.
|
|
180
|
+
* - **`preInitAudio` / `preInitDone` are only meaningful while the path is
|
|
181
|
+
* down.** As connection fields they were readable — and writable — from
|
|
182
|
+
* anywhere; as context of `down`/`starting` they are cleared by entering
|
|
183
|
+
* `down` rather than by three call sites remembering to.
|
|
184
|
+
*
|
|
185
|
+
* ## What cancellation does NOT do, and the one guard that had to survive
|
|
50
186
|
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
* and
|
|
187
|
+
* Stopping the `bringUp` actor hides its resolution; it does not close a
|
|
188
|
+
* microphone the browser has already granted. So `bringUp` checks its own
|
|
189
|
+
* `signal` after `openAudioPath` settles and releases what it built. That is
|
|
190
|
+
* the single remaining "is this still wanted?" test, and it lives in the
|
|
191
|
+
* producer — which owns the value nobody adopted — rather than at the four
|
|
192
|
+
* consumer sites that used to re-ask.
|
|
193
|
+
*
|
|
194
|
+
* ## Effects stay in the session core
|
|
195
|
+
*
|
|
196
|
+
* The machine owns WHEN. Every HOW — the socket frame, the snapshot write, the
|
|
197
|
+
* turn-boundary drain — arrives as an injected {@link AudioPathEffects} call,
|
|
198
|
+
* the same shape `s2s-lifecycle.ts` uses. So this module touches no socket and
|
|
199
|
+
* no snapshot.
|
|
55
200
|
*/
|
|
56
|
-
let audioModulesPromise = null;
|
|
57
201
|
/**
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
202
|
+
* Cap on audio chunks buffered while the path is coming up. ~100 chunks at
|
|
203
|
+
* typical S2S chunk sizes is well over a second of audio — far longer than a
|
|
204
|
+
* bring-up takes in practice, but bounded against pathological cases
|
|
205
|
+
* (mic-permission stalls).
|
|
62
206
|
*/
|
|
63
|
-
|
|
64
|
-
audioModulesPromise ??= Promise.all([
|
|
65
|
-
import("./audio.js"),
|
|
66
|
-
import("./worklets/capture-processor.js").then((m) => m.default),
|
|
67
|
-
import("./worklets/playback-processor.js").then((m) => m.default)
|
|
68
|
-
]).catch((err) => {
|
|
69
|
-
audioModulesPromise = null;
|
|
70
|
-
throw err;
|
|
71
|
-
});
|
|
72
|
-
return audioModulesPromise;
|
|
73
|
-
}
|
|
207
|
+
const MAX_BUFFERED_CHUNKS = 100;
|
|
74
208
|
/**
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
* Lifecycle: dynamically import audio modules -> request microphone access ->
|
|
78
|
-
* register AudioWorklet processors -> create a `VoiceIO` instance -> send
|
|
79
|
-
* `audio_ready` to the server -> transition state to `"listening"`.
|
|
209
|
+
* The rejection a bring-up abandons itself with.
|
|
80
210
|
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
* connection.
|
|
85
|
-
*
|
|
86
|
-
* A failure is fatal: a voice session can't function without the mic, so it
|
|
87
|
-
* sets the error state and ends the session.
|
|
211
|
+
* Never observed: the actor is already stopped when it is thrown, and XState
|
|
212
|
+
* drops a settlement from a stopped actor. It exists so the `async` function
|
|
213
|
+
* has a way to end that is not "resolve with a VoiceIO I have just closed".
|
|
88
214
|
*/
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
const [{ createVoiceIO }, captureWorklet, playbackWorklet] = await loadAudioModules();
|
|
110
|
-
const io = await createVoiceIO({
|
|
111
|
-
sttSampleRate: msg.sampleRate,
|
|
112
|
-
ttsSampleRate: msg.ttsSampleRate,
|
|
113
|
-
captureWorkletSrc: captureWorklet,
|
|
114
|
-
playbackWorkletSrc: playbackWorklet,
|
|
115
|
-
onMicData: (pcm16) => {
|
|
116
|
-
try {
|
|
117
|
-
deps.sendAudio(pcm16);
|
|
118
|
-
} catch {
|
|
119
|
-
console.debug("[aai-ui] sendAudio dropped: connection closed");
|
|
120
|
-
}
|
|
121
|
-
},
|
|
122
|
-
onPlaybackProgress: (bufferedMs) => {
|
|
123
|
-
if (!conn.generation.isCurrent(gen)) return;
|
|
124
|
-
try {
|
|
125
|
-
deps.sendJson({
|
|
126
|
-
type: "playback_progress",
|
|
127
|
-
bufferedMs
|
|
128
|
-
});
|
|
129
|
-
} catch {}
|
|
215
|
+
const ABANDONED = /* @__PURE__ */ new Error("audio path abandoned");
|
|
216
|
+
const audioPathMachine = setup({
|
|
217
|
+
types: {},
|
|
218
|
+
actors: {
|
|
219
|
+
/**
|
|
220
|
+
* One bring-up. An actor rather than a fire-and-forget `void init(...)`
|
|
221
|
+
* precisely so that leaving `starting` — a hang-up, a reconnect, a fatal
|
|
222
|
+
* frame — stops it, instead of letting it settle into a connection that is
|
|
223
|
+
* already over.
|
|
224
|
+
*/
|
|
225
|
+
bringUp: fromPromise(async ({ input, signal }) => {
|
|
226
|
+
let mine = null;
|
|
227
|
+
const io = await input.effects.open(input.config, {
|
|
228
|
+
onMicData: (pcm16) => input.effects.sendMicAudio(pcm16),
|
|
229
|
+
onProgress: (bufferedMs) => {
|
|
230
|
+
if (mine) input.send({
|
|
231
|
+
type: "PROGRESS",
|
|
232
|
+
io: mine,
|
|
233
|
+
bufferedMs
|
|
234
|
+
});
|
|
130
235
|
},
|
|
131
|
-
|
|
132
|
-
if (
|
|
133
|
-
|
|
236
|
+
onFailure: (message) => {
|
|
237
|
+
if (mine) input.send({
|
|
238
|
+
type: "IO_FAILED",
|
|
239
|
+
io: mine,
|
|
240
|
+
message
|
|
241
|
+
});
|
|
134
242
|
}
|
|
135
243
|
});
|
|
136
|
-
if (
|
|
137
|
-
|
|
138
|
-
|
|
244
|
+
if (signal.aborted) {
|
|
245
|
+
input.effects.release(io);
|
|
246
|
+
throw ABANDONED;
|
|
139
247
|
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
248
|
+
mine = io;
|
|
249
|
+
return io;
|
|
250
|
+
}) },
|
|
251
|
+
guards: {
|
|
252
|
+
/**
|
|
253
|
+
* Did the live path report this, or one it replaced?
|
|
254
|
+
*
|
|
255
|
+
* Identity rather than `generation.isCurrent(gen)`: two bring-ups inside a
|
|
256
|
+
* single connection share a generation, and the counter passed for the
|
|
257
|
+
* instance that had just been closed.
|
|
258
|
+
*/
|
|
259
|
+
fromLiveIo: ({ context, event }) => (event.type === "PROGRESS" || event.type === "IO_FAILED") && context.io === event.io },
|
|
260
|
+
actions: {
|
|
261
|
+
rememberConfig: assign({ config: ({ context, event }) => event.type === "START" ? event.config : context.config }),
|
|
262
|
+
/** Hold a chunk that arrived too early, up to the cap. */
|
|
263
|
+
buffer: assign({ pending: ({ context, event }) => event.type === "CHUNK" && context.pending.length < MAX_BUFFERED_CHUNKS ? [...context.pending, event.chunk] : context.pending }),
|
|
264
|
+
/** Record a done that arrived too early. */
|
|
265
|
+
rememberDone: assign({ pendingDone: true }),
|
|
266
|
+
/** Answer it optimistically: with no audio pipeline there is nothing to wait for. */
|
|
267
|
+
listenNow: ({ context }) => context.effects.listen(),
|
|
268
|
+
/** Nothing buffered belongs to the next path. */
|
|
269
|
+
forgetBuffered: assign({
|
|
270
|
+
io: null,
|
|
271
|
+
pending: [],
|
|
272
|
+
pendingDone: false
|
|
273
|
+
}),
|
|
274
|
+
/** Releasing the path ends the turn it was playing — see `endTurn`. */
|
|
275
|
+
endTurn: ({ context }) => context.effects.endTurn(),
|
|
276
|
+
releaseIo: ({ context }) => {
|
|
277
|
+
if (context.io) context.effects.release(context.io);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
}).createMachine({
|
|
281
|
+
id: "audioPath",
|
|
282
|
+
context: ({ input }) => ({
|
|
283
|
+
effects: input,
|
|
284
|
+
io: null,
|
|
285
|
+
pending: [],
|
|
286
|
+
pendingDone: false,
|
|
287
|
+
config: null
|
|
288
|
+
}),
|
|
289
|
+
initial: "down",
|
|
290
|
+
on: {
|
|
291
|
+
CHUNK: { actions: "buffer" },
|
|
292
|
+
DONE: { actions: ["rememberDone", "listenNow"] },
|
|
293
|
+
TEARDOWN: ".down"
|
|
294
|
+
},
|
|
295
|
+
states: {
|
|
296
|
+
/**
|
|
297
|
+
* No audio path. Entered on every release, so its entry is the one place
|
|
298
|
+
* that ends the turn and drops what was buffered for a path that is gone.
|
|
299
|
+
*/
|
|
300
|
+
down: {
|
|
301
|
+
entry: ["endTurn", "forgetBuffered"],
|
|
302
|
+
on: { START: {
|
|
303
|
+
target: "starting",
|
|
304
|
+
actions: "rememberConfig"
|
|
305
|
+
} }
|
|
306
|
+
},
|
|
307
|
+
/**
|
|
308
|
+
* A bring-up is in flight.
|
|
309
|
+
*
|
|
310
|
+
* `START` is absent deliberately: that is the whole of what
|
|
311
|
+
* `audioSetupInFlight` said, and a second `config` frame arriving mid-grant
|
|
312
|
+
* is now declined by the position rather than by a flag whose owner had to
|
|
313
|
+
* be tracked.
|
|
314
|
+
*/
|
|
315
|
+
starting: { invoke: {
|
|
316
|
+
src: "bringUp",
|
|
317
|
+
input: ({ context, self }) => ({
|
|
318
|
+
effects: context.effects,
|
|
319
|
+
config: context.config ?? {
|
|
320
|
+
sampleRate: 0,
|
|
321
|
+
ttsSampleRate: 0
|
|
322
|
+
},
|
|
323
|
+
send: (event) => {
|
|
324
|
+
self.send(event);
|
|
325
|
+
}
|
|
326
|
+
}),
|
|
327
|
+
onDone: {
|
|
328
|
+
target: "up",
|
|
329
|
+
actions: [
|
|
330
|
+
({ context, event }) => {
|
|
331
|
+
for (const chunk of context.pending) event.output.enqueue(chunk);
|
|
332
|
+
},
|
|
333
|
+
assign({
|
|
334
|
+
io: ({ event }) => event.output,
|
|
335
|
+
pending: []
|
|
336
|
+
}),
|
|
337
|
+
({ context, event }) => {
|
|
338
|
+
context.effects.announceReady();
|
|
339
|
+
context.effects.micLive();
|
|
340
|
+
if (context.pendingDone) context.effects.settleWhenDrained(event.output);
|
|
341
|
+
else context.effects.listen();
|
|
342
|
+
},
|
|
343
|
+
assign({ pendingDone: false })
|
|
344
|
+
]
|
|
345
|
+
},
|
|
346
|
+
onError: {
|
|
347
|
+
target: "down",
|
|
348
|
+
actions: ({ context, event }) => {
|
|
349
|
+
context.effects.reportFailure(`Microphone access failed: ${errorMessage(event.error)}`);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
} },
|
|
353
|
+
/**
|
|
354
|
+
* A live path. Its exit releases it, which covers all three ways out —
|
|
355
|
+
* teardown, a worklet death, and the repeated `config` frame that opens a
|
|
356
|
+
* replacement (the "defensive" close that used to sit inside the bring-up,
|
|
357
|
+
* where it could only run if the bring-up got that far).
|
|
358
|
+
*/
|
|
359
|
+
up: {
|
|
360
|
+
exit: "releaseIo",
|
|
361
|
+
on: {
|
|
362
|
+
START: {
|
|
363
|
+
target: "starting",
|
|
364
|
+
actions: "rememberConfig"
|
|
365
|
+
},
|
|
366
|
+
CHUNK: { actions: ({ context, event }) => context.io?.enqueue(event.chunk) },
|
|
367
|
+
DONE: { actions: ({ context }) => {
|
|
368
|
+
if (context.io) context.effects.settleWhenDrained(context.io);
|
|
369
|
+
} },
|
|
370
|
+
PROGRESS: {
|
|
371
|
+
guard: "fromLiveIo",
|
|
372
|
+
actions: ({ context, event }) => context.effects.reportProgress(event.bufferedMs)
|
|
373
|
+
},
|
|
374
|
+
IO_FAILED: {
|
|
375
|
+
guard: "fromLiveIo",
|
|
376
|
+
target: "down",
|
|
377
|
+
actions: ({ context, event }) => context.effects.reportFailure(event.message)
|
|
378
|
+
}
|
|
379
|
+
}
|
|
145
380
|
}
|
|
146
|
-
deps.sendJson({ type: "audio_ready" });
|
|
147
|
-
deps.updateState({ recording: true });
|
|
148
|
-
if (conn.preInitDone) {
|
|
149
|
-
conn.preInitDone = false;
|
|
150
|
-
deps.settleWhenAudioDrained(io);
|
|
151
|
-
} else deps.updateState(deps.agentState.apply({ type: "LISTEN" }));
|
|
152
|
-
} catch (err) {
|
|
153
|
-
if (stale()) return;
|
|
154
|
-
reportAudioFailure(`Microphone access failed: ${errorMessage(err)}`);
|
|
155
|
-
} finally {
|
|
156
|
-
if (conn.generation.isCurrent(gen)) conn.audioSetupInFlight = false;
|
|
157
381
|
}
|
|
382
|
+
});
|
|
383
|
+
/** Create the audio path for one browser session. */
|
|
384
|
+
function createAudioPath(effects) {
|
|
385
|
+
const actor = createActor(audioPathMachine, { input: effects }).start();
|
|
386
|
+
return {
|
|
387
|
+
phase: () => actor.getSnapshot().value,
|
|
388
|
+
start: (config) => {
|
|
389
|
+
actor.send({
|
|
390
|
+
type: "START",
|
|
391
|
+
config
|
|
392
|
+
});
|
|
393
|
+
},
|
|
394
|
+
enqueue: (chunk) => {
|
|
395
|
+
actor.send({
|
|
396
|
+
type: "CHUNK",
|
|
397
|
+
chunk
|
|
398
|
+
});
|
|
399
|
+
},
|
|
400
|
+
done: () => {
|
|
401
|
+
actor.send({ type: "DONE" });
|
|
402
|
+
},
|
|
403
|
+
flush: () => actor.getSnapshot().context.io?.flush(),
|
|
404
|
+
teardown: () => {
|
|
405
|
+
actor.send({ type: "TEARDOWN" });
|
|
406
|
+
}
|
|
407
|
+
};
|
|
158
408
|
}
|
|
159
409
|
//#endregion
|
|
160
410
|
//#region src/session-core-close.ts
|
|
@@ -544,7 +794,7 @@ function createHandshakeGuard(opts) {
|
|
|
544
794
|
*
|
|
545
795
|
* Split out of `session-core.ts`: this module owns the interpretation of
|
|
546
796
|
* server→client frames (audio chunks + JSON {@link SessionEvent}s) and the
|
|
547
|
-
* turn-boundary
|
|
797
|
+
* turn-boundary epoch, while `session-core.ts` owns the state
|
|
548
798
|
* store and connection lifecycle. The handlers read and mutate session state
|
|
549
799
|
* exclusively through the injected `getSnapshot`/`updateState` deps.
|
|
550
800
|
*/
|
|
@@ -552,10 +802,6 @@ function createHandshakeGuard(opts) {
|
|
|
552
802
|
const MAX_CUSTOM_EVENTS = 200;
|
|
553
803
|
/** Cap on `messages` retained in the session snapshot; matches the host-side history cap. */
|
|
554
804
|
const MAX_MESSAGES = DEFAULT_MAX_HISTORY;
|
|
555
|
-
/** Cap on pre-init audio chunks buffered while `voiceIO` is initializing. ~100 chunks at
|
|
556
|
-
* typical S2S chunk sizes is well over a second of audio — far longer than init takes
|
|
557
|
-
* in practice, but bounded against pathological cases (mic-permission stalls). */
|
|
558
|
-
const MAX_PREINIT_AUDIO_CHUNKS = 100;
|
|
559
805
|
/**
|
|
560
806
|
* Snapshot fields cleared when a session's conversation state is wiped —
|
|
561
807
|
* shared by the initial snapshot, `resetState()`, and `session.reset`.
|
|
@@ -587,7 +833,7 @@ function appendCapped(list, item, cap) {
|
|
|
587
833
|
* `ConnState.turn`).
|
|
588
834
|
*/
|
|
589
835
|
function createMessageHandlers(deps) {
|
|
590
|
-
const { getSnapshot, updateState, conn, agentState,
|
|
836
|
+
const { getSnapshot, updateState, conn, agentState, audio } = deps;
|
|
591
837
|
/** Monotonically increasing counter for custom events -- used by useEvent to deduplicate. */
|
|
592
838
|
let customEventSeq = 0;
|
|
593
839
|
/** Monotonically increasing counter for chat messages -- stable render keys
|
|
@@ -694,8 +940,7 @@ function createMessageHandlers(deps) {
|
|
|
694
940
|
error
|
|
695
941
|
}));
|
|
696
942
|
else {
|
|
697
|
-
|
|
698
|
-
conn.generation.bump();
|
|
943
|
+
audio.teardown();
|
|
699
944
|
updateState({
|
|
700
945
|
...agentState.apply({
|
|
701
946
|
type: "FATAL",
|
|
@@ -752,12 +997,12 @@ function createMessageHandlers(deps) {
|
|
|
752
997
|
toListening();
|
|
753
998
|
break;
|
|
754
999
|
case "reply.cancelled":
|
|
755
|
-
bargeIn(conn);
|
|
1000
|
+
bargeIn(conn, audio);
|
|
756
1001
|
commitAgentTranscript();
|
|
757
1002
|
toListening({ userTranscript: null });
|
|
758
1003
|
break;
|
|
759
1004
|
case "session.reset": {
|
|
760
|
-
bargeIn(conn);
|
|
1005
|
+
bargeIn(conn, audio);
|
|
761
1006
|
const next = agentState.apply({ type: "RESET" });
|
|
762
1007
|
updateState(agentState.fatal() ? next : {
|
|
763
1008
|
...CLEARED_SESSION_STATE,
|
|
@@ -802,34 +1047,18 @@ function createMessageHandlers(deps) {
|
|
|
802
1047
|
/** Enqueue a PCM16 audio chunk for playback. Transitions state to `"speaking"` on the first chunk. */
|
|
803
1048
|
function playAudioChunk(chunk) {
|
|
804
1049
|
updateState(agentState.apply({ type: "SPEAK" }));
|
|
805
|
-
|
|
806
|
-
else if (conn.preInitAudio.length < MAX_PREINIT_AUDIO_CHUNKS) conn.preInitAudio.push(chunk);
|
|
807
|
-
}
|
|
808
|
-
/** See {@link MessageHandlers.settleWhenAudioDrained}. Captures
|
|
809
|
-
* `conn.turn` so a completion (or failure) that lands after a turn
|
|
810
|
-
* boundary — including an audio-path teardown — is discarded instead of
|
|
811
|
-
* overwriting the newer turn's (or the dead session's) state. */
|
|
812
|
-
function settleWhenAudioDrained(io) {
|
|
813
|
-
const gen = conn.turn.current();
|
|
814
|
-
io.done().then(() => {
|
|
815
|
-
if (!conn.turn.isCurrent(gen)) return;
|
|
816
|
-
updateState(agentState.apply({ type: "LISTEN" }));
|
|
817
|
-
}).catch((err) => {
|
|
818
|
-
console.warn("Audio playback done failed:", err);
|
|
819
|
-
});
|
|
1050
|
+
audio.enqueue(chunk);
|
|
820
1051
|
}
|
|
821
1052
|
/**
|
|
822
1053
|
* Signal that the server has finished sending audio for this turn.
|
|
823
|
-
*
|
|
824
|
-
*
|
|
1054
|
+
*
|
|
1055
|
+
* A path that is up waits for its queue to drain before going back to
|
|
1056
|
+
* listening; one still coming up records the done and answers optimistically,
|
|
1057
|
+
* then replays it once the buffered greeting has somewhere to go. Both arms
|
|
1058
|
+
* live in `session-core-audio-state.ts`.
|
|
825
1059
|
*/
|
|
826
1060
|
function playAudioDone() {
|
|
827
|
-
|
|
828
|
-
if (io) settleWhenAudioDrained(io);
|
|
829
|
-
else {
|
|
830
|
-
conn.preInitDone = true;
|
|
831
|
-
updateState(agentState.apply({ type: "LISTEN" }));
|
|
832
|
-
}
|
|
1061
|
+
audio.done();
|
|
833
1062
|
}
|
|
834
1063
|
function handleMessage(data) {
|
|
835
1064
|
if (data instanceof ArrayBuffer) {
|
|
@@ -865,10 +1094,7 @@ function createMessageHandlers(deps) {
|
|
|
865
1094
|
}
|
|
866
1095
|
handleEvent(msg);
|
|
867
1096
|
}
|
|
868
|
-
return {
|
|
869
|
-
handleMessage,
|
|
870
|
-
settleWhenAudioDrained
|
|
871
|
-
};
|
|
1097
|
+
return { handleMessage };
|
|
872
1098
|
}
|
|
873
1099
|
//#endregion
|
|
874
1100
|
//#region src/session-core-state.ts
|
|
@@ -1053,6 +1279,31 @@ function createSessionStateMachine() {
|
|
|
1053
1279
|
};
|
|
1054
1280
|
}
|
|
1055
1281
|
//#endregion
|
|
1282
|
+
//#region src/session-core-user-turn.ts
|
|
1283
|
+
/** Build the three push-to-talk edges. @internal */
|
|
1284
|
+
function createUserTurnActions(deps) {
|
|
1285
|
+
return {
|
|
1286
|
+
start() {
|
|
1287
|
+
if (!deps.connected()) return;
|
|
1288
|
+
const { state } = deps.snapshot();
|
|
1289
|
+
if (state === "speaking" || state === "thinking") {
|
|
1290
|
+
deps.bargeIn();
|
|
1291
|
+
deps.updateState(deps.agentState.apply({ type: "LISTEN" }));
|
|
1292
|
+
}
|
|
1293
|
+
deps.sendJson({ type: "user_turn_start" });
|
|
1294
|
+
},
|
|
1295
|
+
commit() {
|
|
1296
|
+
if (!deps.connected()) return;
|
|
1297
|
+
deps.sendJson({ type: "user_turn_commit" });
|
|
1298
|
+
},
|
|
1299
|
+
clear() {
|
|
1300
|
+
if (!deps.connected()) return;
|
|
1301
|
+
deps.updateState({ userTranscript: null });
|
|
1302
|
+
deps.sendJson({ type: "user_turn_clear" });
|
|
1303
|
+
}
|
|
1304
|
+
};
|
|
1305
|
+
}
|
|
1306
|
+
//#endregion
|
|
1056
1307
|
//#region src/session-core.ts
|
|
1057
1308
|
/**
|
|
1058
1309
|
* Framework-agnostic voice session core.
|
|
@@ -1148,23 +1399,10 @@ function createBrowserSession(options) {
|
|
|
1148
1399
|
const conn = {
|
|
1149
1400
|
ws: null,
|
|
1150
1401
|
retiredByServer: false,
|
|
1151
|
-
|
|
1152
|
-
audioSetupInFlight: false,
|
|
1153
|
-
generation: createEpoch(),
|
|
1154
|
-
turn: createEpoch(),
|
|
1155
|
-
preInitAudio: [],
|
|
1156
|
-
preInitDone: false
|
|
1402
|
+
turn: createEpoch()
|
|
1157
1403
|
};
|
|
1158
1404
|
let connectionController = null;
|
|
1159
1405
|
const dialer = createDialer(options);
|
|
1160
|
-
function cleanupAudio() {
|
|
1161
|
-
conn.audioSetupInFlight = false;
|
|
1162
|
-
conn.turn.bump();
|
|
1163
|
-
conn.voiceIO?.close().catch(() => {});
|
|
1164
|
-
conn.voiceIO = null;
|
|
1165
|
-
conn.preInitAudio = [];
|
|
1166
|
-
conn.preInitDone = false;
|
|
1167
|
-
}
|
|
1168
1406
|
function resetState() {
|
|
1169
1407
|
updateState(CLEARED_SESSION_STATE);
|
|
1170
1408
|
}
|
|
@@ -1187,26 +1425,32 @@ function createBrowserSession(options) {
|
|
|
1187
1425
|
if (ws.bufferedAmount > MIC_SEND_MAX_BUFFERED_BYTES) return;
|
|
1188
1426
|
ws.send(bytes);
|
|
1189
1427
|
}
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1428
|
+
/**
|
|
1429
|
+
* The microphone, the worklets and the buffer in front of them, as a
|
|
1430
|
+
* statechart — see `session-core-audio-state.ts`, which carries what the
|
|
1431
|
+
* in-flight latch, the generation epoch and the two pre-init fields on
|
|
1432
|
+
* `ConnState` cost. `session-core-audio-effects.ts` is the other half: every
|
|
1433
|
+
* frame and snapshot write the machine decides on but cannot make.
|
|
1434
|
+
*/
|
|
1435
|
+
const audio = createAudioPath(createAudioEffects({
|
|
1193
1436
|
conn,
|
|
1437
|
+
updateState,
|
|
1194
1438
|
agentState,
|
|
1195
|
-
cleanupAudio
|
|
1196
|
-
});
|
|
1197
|
-
const audioDeps = {
|
|
1198
1439
|
sendJson,
|
|
1199
|
-
sendAudio
|
|
1440
|
+
sendAudio
|
|
1441
|
+
}));
|
|
1442
|
+
const { handleMessage } = createMessageHandlers({
|
|
1443
|
+
getSnapshot,
|
|
1200
1444
|
updateState,
|
|
1445
|
+
conn,
|
|
1201
1446
|
agentState,
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
};
|
|
1447
|
+
audio
|
|
1448
|
+
});
|
|
1205
1449
|
/** Abort the in-flight connection and release audio + WebSocket resources. */
|
|
1206
1450
|
function teardownConnection() {
|
|
1207
1451
|
connectionController?.abort();
|
|
1208
1452
|
connectionController = null;
|
|
1209
|
-
|
|
1453
|
+
audio.teardown();
|
|
1210
1454
|
conn.ws?.close();
|
|
1211
1455
|
conn.ws = null;
|
|
1212
1456
|
}
|
|
@@ -1224,7 +1468,7 @@ function createBrowserSession(options) {
|
|
|
1224
1468
|
function onServerConfig(config) {
|
|
1225
1469
|
dialer.configured(config.sid);
|
|
1226
1470
|
if (config.sid) options.onSessionId?.(config.sid);
|
|
1227
|
-
|
|
1471
|
+
audio.start(config);
|
|
1228
1472
|
}
|
|
1229
1473
|
function connect(opts) {
|
|
1230
1474
|
if (opts?.signal?.aborted) {
|
|
@@ -1234,7 +1478,6 @@ function createBrowserSession(options) {
|
|
|
1234
1478
|
updateState(agentState.apply({ type: "CONNECT" }));
|
|
1235
1479
|
loadAudioModules().catch(() => {});
|
|
1236
1480
|
teardownConnection();
|
|
1237
|
-
conn.generation.bump();
|
|
1238
1481
|
conn.retiredByServer = false;
|
|
1239
1482
|
const controller = new AbortController();
|
|
1240
1483
|
connectionController = controller;
|
|
@@ -1248,15 +1491,14 @@ function createBrowserSession(options) {
|
|
|
1248
1491
|
socket,
|
|
1249
1492
|
signal: sig,
|
|
1250
1493
|
onRetry: () => {
|
|
1251
|
-
|
|
1252
|
-
conn.generation.bump();
|
|
1494
|
+
audio.teardown();
|
|
1253
1495
|
updateState({
|
|
1254
1496
|
...agentState.apply({ type: "CONNECT" }),
|
|
1255
1497
|
recording: false
|
|
1256
1498
|
});
|
|
1257
1499
|
},
|
|
1258
1500
|
onExhausted: () => {
|
|
1259
|
-
|
|
1501
|
+
audio.teardown();
|
|
1260
1502
|
controller.abort();
|
|
1261
1503
|
socket.close();
|
|
1262
1504
|
conn.ws = null;
|
|
@@ -1285,9 +1527,8 @@ function createBrowserSession(options) {
|
|
|
1285
1527
|
socket.addEventListener("close", (event) => {
|
|
1286
1528
|
if (sig.aborted) return;
|
|
1287
1529
|
handshake.disarm();
|
|
1288
|
-
|
|
1530
|
+
audio.teardown();
|
|
1289
1531
|
if (!(conn.retiredByServer || agentState.fatal()) && reconnectPending(socket)) {
|
|
1290
|
-
conn.generation.bump();
|
|
1291
1532
|
socketErrored = false;
|
|
1292
1533
|
updateState({
|
|
1293
1534
|
...agentState.apply({ type: "CONNECT" }),
|
|
@@ -1314,12 +1555,20 @@ function createBrowserSession(options) {
|
|
|
1314
1555
|
}
|
|
1315
1556
|
function cancel() {
|
|
1316
1557
|
if (!openSocket()) return;
|
|
1317
|
-
bargeIn(conn);
|
|
1558
|
+
bargeIn(conn, audio);
|
|
1318
1559
|
updateState(agentState.apply({ type: "LISTEN" }));
|
|
1319
1560
|
sendJson({ type: "cancel" });
|
|
1320
1561
|
}
|
|
1562
|
+
const userTurn = createUserTurnActions({
|
|
1563
|
+
snapshot: () => currentSnapshot,
|
|
1564
|
+
connected: () => openSocket() !== null,
|
|
1565
|
+
bargeIn: () => bargeIn(conn, audio),
|
|
1566
|
+
agentState,
|
|
1567
|
+
updateState,
|
|
1568
|
+
sendJson
|
|
1569
|
+
});
|
|
1321
1570
|
function reset() {
|
|
1322
|
-
bargeIn(conn);
|
|
1571
|
+
bargeIn(conn, audio);
|
|
1323
1572
|
if (openSocket()) {
|
|
1324
1573
|
sendJson({ type: "reset" });
|
|
1325
1574
|
return;
|
|
@@ -1367,6 +1616,7 @@ function createBrowserSession(options) {
|
|
|
1367
1616
|
subscribe,
|
|
1368
1617
|
connect,
|
|
1369
1618
|
cancel,
|
|
1619
|
+
userTurn: Object.freeze(userTurn),
|
|
1370
1620
|
resetState,
|
|
1371
1621
|
reset,
|
|
1372
1622
|
disconnect,
|