@jskit-ai/assistant-voice 0.0.0-stage → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/README.md +169 -2
  2. package/examples/conversation/App.vue +94 -0
  3. package/examples/conversation/Conversation.vue +28 -0
  4. package/examples/conversation/README.md +181 -0
  5. package/examples/conversation/config.js +15 -0
  6. package/examples/conversation/index.html +1 -0
  7. package/examples/conversation/integrations.example.json +16 -0
  8. package/examples/conversation/main.js +58 -0
  9. package/examples/conversation/package.json +32 -0
  10. package/examples/conversation/server.js +105 -0
  11. package/examples/conversation/src/error.js +1 -0
  12. package/examples/conversation/src/placement.js +1 -0
  13. package/examples/conversation/src/placementTopology.js +1 -0
  14. package/examples/conversation/vite.config.js +9 -0
  15. package/package.json +67 -4
  16. package/src/client/ConversationDialog.vue +67 -0
  17. package/src/client/VoiceAvatar.vue +18 -0
  18. package/src/client/VoiceConversation.vue +117 -0
  19. package/src/client/VoiceConversationHost.vue +78 -0
  20. package/src/client/audioCapture.js +198 -0
  21. package/src/client/conversationNarration.js +108 -0
  22. package/src/client/conversationVoiceState.js +18 -0
  23. package/src/client/index.js +9 -0
  24. package/src/client/voiceAvatar.js +234 -0
  25. package/src/client/voiceController.js +124 -0
  26. package/src/client/voiceConversation.js +557 -0
  27. package/src/client/voiceLauncher.js +63 -0
  28. package/src/client/voicePlayback.js +58 -0
  29. package/src/client/voiceTransport.js +890 -0
  30. package/src/server/index.js +5 -0
  31. package/src/server/voiceAccessToken.js +79 -0
  32. package/src/server/voiceDaemon.js +599 -0
  33. package/src/server/voiceProxy.js +199 -0
  34. package/src/server/voiceProxyConfig.js +70 -0
  35. package/src/server/voiceSocketClose.js +31 -0
  36. package/src/shared/protocol.js +439 -0
  37. package/src/shared/visemes.js +130 -0
@@ -0,0 +1,5 @@
1
+ export { createVoiceDaemon } from "./voiceDaemon.js";
2
+ export { registerVoiceProxyRoute, readVoiceCatalogue } from "./voiceProxy.js";
3
+ export { resolveVoiceProxyConfig, readVoiceAccessToken, voiceEndpoint } from "./voiceProxyConfig.js";
4
+ export { createVoiceAccessToken, verifyVoiceAccessToken, bearerVoiceAccessToken } from "./voiceAccessToken.js";
5
+ export { normalizeVoiceCloseMetadata } from "./voiceSocketClose.js";
@@ -0,0 +1,79 @@
1
+ import crypto from "node:crypto";
2
+
3
+ const VOICE_ACCESS_TOKEN_VERSION = "v1";
4
+ const VOICE_TENANT_PATTERN = /^[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?$/u;
5
+
6
+ function voiceTenant(value = "") {
7
+ const tenant = String(value || "").trim().toLowerCase();
8
+ if (!VOICE_TENANT_PATTERN.test(tenant)) {
9
+ const error = new Error("Voice access requires a valid tenant name.");
10
+ error.code = "voice_tenant_invalid";
11
+ throw error;
12
+ }
13
+ return tenant;
14
+ }
15
+
16
+ function voiceAccessKey(value = "") {
17
+ const key = Buffer.isBuffer(value) ? value : Buffer.from(String(value || ""), "utf8");
18
+ if (key.byteLength < 32) {
19
+ const error = new Error("Voice access signing keys must contain at least 32 bytes.");
20
+ error.code = "voice_access_key_invalid";
21
+ throw error;
22
+ }
23
+ return key;
24
+ }
25
+
26
+ function voiceAccessSignature(key, tenant) {
27
+ return crypto
28
+ .createHmac("sha256", voiceAccessKey(key))
29
+ .update(`${VOICE_ACCESS_TOKEN_VERSION}.${voiceTenant(tenant)}`, "utf8")
30
+ .digest("base64url");
31
+ }
32
+
33
+ function createVoiceAccessToken({ key, tenant } = {}) {
34
+ const normalizedTenant = voiceTenant(tenant);
35
+ return [
36
+ VOICE_ACCESS_TOKEN_VERSION,
37
+ normalizedTenant,
38
+ voiceAccessSignature(key, normalizedTenant)
39
+ ].join(".");
40
+ }
41
+
42
+ function verifyVoiceAccessToken(token = "", { key } = {}) {
43
+ const [version = "", rawTenant = "", signature = "", ...extra] = String(token || "").trim().split(".");
44
+ if (version !== VOICE_ACCESS_TOKEN_VERSION || !rawTenant || !signature || extra.length) {
45
+ return null;
46
+ }
47
+ let tenant;
48
+ let expected;
49
+ try {
50
+ tenant = voiceTenant(rawTenant);
51
+ expected = voiceAccessSignature(key, tenant);
52
+ } catch {
53
+ return null;
54
+ }
55
+ const actualBytes = Buffer.from(signature, "utf8");
56
+ const expectedBytes = Buffer.from(expected, "utf8");
57
+ if (
58
+ actualBytes.byteLength !== expectedBytes.byteLength ||
59
+ !crypto.timingSafeEqual(actualBytes, expectedBytes)
60
+ ) {
61
+ return null;
62
+ }
63
+ return Object.freeze({ tenant, version });
64
+ }
65
+
66
+ function bearerVoiceAccessToken(headers = {}) {
67
+ const value = String(headers.authorization || headers.Authorization || "").trim();
68
+ const match = /^Bearer\s+([^\s]+)$/iu.exec(value);
69
+ return match ? match[1] : "";
70
+ }
71
+
72
+ export {
73
+ VOICE_ACCESS_TOKEN_VERSION,
74
+ bearerVoiceAccessToken,
75
+ createVoiceAccessToken,
76
+ verifyVoiceAccessToken,
77
+ voiceAccessKey,
78
+ voiceTenant
79
+ };
@@ -0,0 +1,599 @@
1
+ import http from "node:http";
2
+
3
+ import { WebSocketServer } from "ws";
4
+
5
+ import {
6
+ VOICE_DAEMON_WEBSOCKET_PATH,
7
+ VOICE_INPUT_SAMPLE_RATE,
8
+ VOICE_MAX_AUDIO_FRAME_BYTES,
9
+ VOICE_MAX_CONTROL_BYTES,
10
+ VOICE_MAX_UTTERANCE_SECONDS,
11
+ VOICE_PROTOCOL_VERSION,
12
+ assertVoiceAudioFrame,
13
+ parseVoiceClientControl,
14
+ speechPauseDurationMs,
15
+ splitSpeechText,
16
+ voiceServerMessage
17
+ } from "../shared/protocol.js";
18
+ import { visemeCuesFromText } from "../shared/visemes.js";
19
+ import {
20
+ bearerVoiceAccessToken,
21
+ verifyVoiceAccessToken
22
+ } from "./voiceAccessToken.js";
23
+
24
+ const WEBSOCKET_OPEN = 1;
25
+
26
+ function sendJson(socket, type, fields = {}) {
27
+ if (socket.readyState !== WEBSOCKET_OPEN) {
28
+ return false;
29
+ }
30
+ socket.send(voiceServerMessage(type, fields));
31
+ return true;
32
+ }
33
+
34
+ function sendPcm(socket, audio = Buffer.alloc(0)) {
35
+ for (let offset = 0; offset < audio.byteLength; offset += VOICE_MAX_AUDIO_FRAME_BYTES) {
36
+ if (socket.readyState !== WEBSOCKET_OPEN) {
37
+ return false;
38
+ }
39
+ const end = Math.min(audio.byteLength, offset + VOICE_MAX_AUDIO_FRAME_BYTES);
40
+ socket.send(audio.subarray(offset, end), { binary: true });
41
+ }
42
+ return true;
43
+ }
44
+
45
+ function voiceDaemonError(error, fallback = "Voice operation failed.") {
46
+ return {
47
+ code: String(error?.code || "voice_operation_failed"),
48
+ message: String(error?.message || error || fallback)
49
+ };
50
+ }
51
+
52
+ function createBoundedSerialQueue({ maximumQueued = 4 } = {}) {
53
+ const pending = [];
54
+ let running = false;
55
+ let closed = false;
56
+
57
+ const drain = async () => {
58
+ if (running || closed) {
59
+ return;
60
+ }
61
+ const next = pending.shift();
62
+ if (!next) {
63
+ return;
64
+ }
65
+ running = true;
66
+ try {
67
+ next.resolve(await next.operation());
68
+ } catch (error) {
69
+ next.reject(error);
70
+ } finally {
71
+ running = false;
72
+ void drain();
73
+ }
74
+ };
75
+
76
+ return Object.freeze({
77
+ close() {
78
+ closed = true;
79
+ const error = new Error("Voice speech queue is shutting down.");
80
+ error.code = "voice_queue_closed";
81
+ for (const entry of pending.splice(0)) {
82
+ entry.reject(error);
83
+ }
84
+ },
85
+ get depth() {
86
+ return pending.length + (running ? 1 : 0);
87
+ },
88
+ run(operation) {
89
+ if (closed) {
90
+ const error = new Error("Voice speech queue is unavailable.");
91
+ error.code = "voice_queue_closed";
92
+ return Promise.reject(error);
93
+ }
94
+ if (typeof operation !== "function") {
95
+ return Promise.reject(new TypeError("Voice speech queue requires an operation."));
96
+ }
97
+ if (pending.length >= maximumQueued) {
98
+ const error = new Error("Voice speech capacity is currently busy. Try again shortly.");
99
+ error.code = "voice_busy";
100
+ return Promise.reject(error);
101
+ }
102
+ return new Promise((resolve, reject) => {
103
+ pending.push({ operation, reject, resolve });
104
+ void drain();
105
+ });
106
+ }
107
+ });
108
+ }
109
+
110
+ function requestPathname(request = {}) {
111
+ try {
112
+ return new URL(String(request.url || "/"), "http://voice.local").pathname;
113
+ } catch {
114
+ return "/";
115
+ }
116
+ }
117
+
118
+ function writeJsonResponse(response, statusCode, payload) {
119
+ const body = Buffer.from(JSON.stringify(payload));
120
+ response.writeHead(statusCode, {
121
+ "cache-control": "no-store",
122
+ "content-length": String(body.byteLength),
123
+ "content-type": "application/json; charset=utf-8"
124
+ });
125
+ response.end(body);
126
+ }
127
+
128
+ function createVoiceDaemon({
129
+ accessKey,
130
+ authorize = token => verifyVoiceAccessToken(token, { key: accessKey }),
131
+ handleRequest = null,
132
+ engine,
133
+ host = "127.0.0.1",
134
+ idleTimeoutMs = 90_000,
135
+ logger = console,
136
+ maximumConnections = 8,
137
+ maximumConnectionsPerTenant = 2,
138
+ maximumQueuedSpeech = 4,
139
+ port = 3092
140
+ } = {}) {
141
+ if (!engine || typeof engine.createListeningSession !== "function" || typeof engine.synthesize !== "function") {
142
+ throw new TypeError("createVoiceDaemon requires a speech engine.");
143
+ }
144
+ if (typeof authorize !== "function" || (handleRequest !== null && typeof handleRequest !== "function")) throw new TypeError("Voice host authorization and request handlers must be functions.");
145
+ const tenantConnections = new Map();
146
+ const sockets = new Set();
147
+ const principals = new WeakMap();
148
+ const speechQueue = createBoundedSerialQueue({ maximumQueued: maximumQueuedSpeech });
149
+ let ready = false;
150
+ let startedAt = "";
151
+ let stopping = false;
152
+ const voiceCatalogue = () => ({
153
+ voices: (engine.voices || []).map(({ id, label, language }) => ({ id, label, ...(language ? { language } : {}) })),
154
+ defaultVoice: engine.defaultVoice || ""
155
+ });
156
+ const server = http.createServer(async (request, response) => {
157
+ if (request.method === "GET" && requestPathname(request) === "/health") {
158
+ writeJsonResponse(response, ready && !stopping ? 200 : 503, {
159
+ connections: sockets.size,
160
+ ok: ready && !stopping,
161
+ protocolVersion: VOICE_PROTOCOL_VERSION,
162
+ queueDepth: speechQueue.depth,
163
+ startedAt
164
+ });
165
+ return;
166
+ }
167
+ try {
168
+ if (request.method === "GET" && requestPathname(request) === "/voices") {
169
+ let principal;
170
+ try { principal = await authorize(bearerVoiceAccessToken(request.headers), request); } catch { /* Deny invalid credentials. */ }
171
+ if (!principal) { writeJsonResponse(response, 401, { ok: false }); return; }
172
+ if (!ready || stopping) { writeJsonResponse(response, 503, { ok: false }); return; }
173
+ writeJsonResponse(response, 200, voiceCatalogue());
174
+ return;
175
+ }
176
+ if (await handleRequest?.(request, response)) return;
177
+ writeJsonResponse(response, 404, { ok: false });
178
+ } catch {
179
+ if (!response.headersSent) writeJsonResponse(response, 500, { ok: false, error: "Voice host request failed." });
180
+ else response.end();
181
+ }
182
+ });
183
+ const websocketServer = new WebSocketServer({
184
+ maxPayload: Math.max(VOICE_MAX_AUDIO_FRAME_BYTES, VOICE_MAX_CONTROL_BYTES),
185
+ noServer: true,
186
+ perMessageDeflate: false
187
+ });
188
+
189
+ function quotaKeys(principal) { return [...new Set([principal.tenant, principal.parentTenant].filter(Boolean))]; }
190
+
191
+ function connectionCapacityAvailable(principal) {
192
+ return sockets.size < maximumConnections && quotaKeys(principal).every(key =>
193
+ (tenantConnections.get(key) || 0) < maximumConnectionsPerTenant);
194
+ }
195
+
196
+ function claimConnection(principal, socket) {
197
+ sockets.add(socket);
198
+ principals.set(socket, { ...principal });
199
+ for (const key of quotaKeys(principal)) tenantConnections.set(key, (tenantConnections.get(key) || 0) + 1);
200
+ }
201
+
202
+ function releaseConnection(socket) {
203
+ if (!sockets.delete(socket)) return;
204
+ for (const key of quotaKeys(principals.get(socket))) {
205
+ const next = (tenantConnections.get(key) || 1) - 1;
206
+ if (next) tenantConnections.set(key, next);
207
+ else tenantConnections.delete(key);
208
+ }
209
+ principals.delete(socket);
210
+ }
211
+
212
+ function revokeTenant(tenant) {
213
+ for (const socket of sockets) {
214
+ if (quotaKeys(principals.get(socket)).includes(tenant)) {
215
+ principals.get(socket).cancel?.();
216
+ sendJson(socket, "error", { code: "voice_access_revoked", message: "This application's voice access was revoked." });
217
+ socket.close(1008, "Voice access was revoked");
218
+ }
219
+ }
220
+ }
221
+
222
+ websocketServer.on("connection", (socket, request, context = {}) => {
223
+ const tenantId = context.tenant;
224
+ claimConnection(context, socket);
225
+ let listening = null;
226
+ let speaking = null;
227
+ let invalidMessages = 0;
228
+ let lastActivity = Date.now();
229
+
230
+ const cancelListening = () => {
231
+ clearImmediate(listening?.decodeTimer);
232
+ listening?.recognizer?.cancel?.();
233
+ listening = null;
234
+ };
235
+ const cancelSpeaking = () => {
236
+ speaking?.controller?.abort();
237
+ speaking = null;
238
+ };
239
+ const resetIdle = () => {
240
+ lastActivity = Date.now();
241
+ };
242
+ const idleTimer = setInterval(() => {
243
+ if (Date.now() - lastActivity < idleTimeoutMs || socket.readyState !== WEBSOCKET_OPEN) {
244
+ return;
245
+ }
246
+ socket.close(1000, "Voice connection idle");
247
+ }, Math.min(idleTimeoutMs, 15_000));
248
+ idleTimer.unref?.();
249
+
250
+ const failMessage = (error) => {
251
+ invalidMessages += 1;
252
+ const publicError = voiceDaemonError(error);
253
+ sendJson(socket, "error", publicError);
254
+ if (invalidMessages >= 3) {
255
+ socket.close(1008, "Invalid voice protocol messages");
256
+ }
257
+ };
258
+
259
+ principals.get(socket).cancel = () => { cancelListening(); cancelSpeaking(); };
260
+
261
+ const startListening = (control) => {
262
+ cancelListening();
263
+ const turnId = control.turnId;
264
+ const recognizer = engine.createListeningSession({
265
+ continuous: control.continuous === true,
266
+ onPartial(text) {
267
+ if (listening?.turnId === turnId) {
268
+ listening.revision += 1;
269
+ listening.candidateRevision = 0;
270
+ sendJson(socket, "transcript.partial", { text, turnId, revision: listening.revision });
271
+ }
272
+ },
273
+ onEndpoint(text) {
274
+ if (listening?.turnId !== turnId || !listening.continuous) return;
275
+ listening.candidateRevision = listening.revision;
276
+ sendJson(socket, "transcript.endpoint", { text, turnId, revision: listening.revision });
277
+ }
278
+ });
279
+ listening = { recognizer, turnId, revision: 0, candidateRevision: 0, continuous: control.continuous === true,
280
+ pending: [], pendingBytes: 0, decodeTimer: null, ending: false };
281
+ sendJson(socket, "listen.started", { turnId });
282
+ };
283
+
284
+ const drainListening = (active) => {
285
+ if (active.decodeTimer || listening !== active || !active.pending.length) return;
286
+ // Native recognition is synchronous. Yield between small audio slices so
287
+ // a burst of microphone frames cannot starve PCM delivery or controls.
288
+ active.decodeTimer = setImmediate(() => {
289
+ active.decodeTimer = null;
290
+ if (listening !== active) return;
291
+ const { frame, control } = active.pending.shift();
292
+ try {
293
+ if (frame) {
294
+ active.pendingBytes -= frame.byteLength;
295
+ if (active.recognizer.acceptedSamples + frame.byteLength / 2 > VOICE_INPUT_SAMPLE_RATE * VOICE_MAX_UTTERANCE_SECONDS) {
296
+ throw Object.assign(new Error(`Voice recordings are limited to ${VOICE_MAX_UTTERANCE_SECONDS} seconds.`), {
297
+ code: "voice_utterance_too_long"
298
+ });
299
+ }
300
+ active.recognizer.acceptPcm(frame);
301
+ } else if (control.type === "listen.stop") {
302
+ listening = null;
303
+ const text = active.recognizer.finish();
304
+ sendJson(socket, "transcript.final", { text, turnId: active.turnId });
305
+ } else if (!active.continuous || active.candidateRevision !== control.revision || !active.recognizer.isEndpoint()) {
306
+ sendJson(socket, "transcript.stale", { turnId: active.turnId, revision: control.revision });
307
+ } else {
308
+ const text = active.recognizer.reset();
309
+ active.candidateRevision = 0;
310
+ active.revision += 1;
311
+ if (control.type === "listen.commit") sendJson(socket, "transcript.final", {
312
+ text, turnId: active.turnId, revision: control.revision, continuous: true
313
+ });
314
+ sendJson(socket, "transcript.reset", { turnId: active.turnId, revision: active.revision });
315
+ }
316
+ } catch (error) {
317
+ if (listening === active) cancelListening();
318
+ else active.recognizer.cancel();
319
+ failMessage(error);
320
+ }
321
+ drainListening(active);
322
+ });
323
+ };
324
+
325
+ const queueListening = (active, entry) => {
326
+ const bytes = entry.frame?.byteLength || 0;
327
+ // About two seconds of unrecognized input; a clipped instruction must
328
+ // return to review instead of silently building a delayed conversation.
329
+ if (active.pendingBytes + bytes > VOICE_MAX_AUDIO_FRAME_BYTES || active.pending.length >= 128) {
330
+ cancelListening();
331
+ throw Object.assign(new Error("Speech recognition fell behind. Your recording may be incomplete; review the recognized words."), {
332
+ code: "voice_recognition_backpressure"
333
+ });
334
+ }
335
+ active.pending.push(entry);
336
+ active.pendingBytes += bytes;
337
+ drainListening(active);
338
+ };
339
+
340
+ const drainSpeech = async (active) => {
341
+ if (active.processing || speaking !== active) return;
342
+ active.processing = true;
343
+ const { controller, turnId } = active;
344
+ try {
345
+ while (active.pending.length && !controller.signal.aborted) {
346
+ const chunk = active.pending.shift();
347
+ // Queue each phrase separately so another tenant can get a turn.
348
+ const phrases = splitSpeechText(chunk);
349
+ for (const [position, text] of phrases.entries()) {
350
+ if (active.segmentIndex > active.stopAfter || controller.signal.aborted) break;
351
+ if (!active.started && !text.trim()) continue;
352
+ await speechQueue.run(async () => {
353
+ if (controller.signal.aborted || socket.readyState !== WEBSOCKET_OPEN) return;
354
+ if (!active.started) {
355
+ active.started = true;
356
+ sendJson(socket, "speech.start", { sampleRate: Number(engine.sampleRate), turnId });
357
+ }
358
+ const segmentIndex = active.segmentIndex++;
359
+ sendJson(socket, "speech.segment.start", { segmentIndex, turnId });
360
+ let sampleCount = 0;
361
+ const result = text.trim() ? await engine.synthesize(text.trim(), {
362
+ onAudio(frame) {
363
+ if (controller.signal.aborted || socket.readyState !== WEBSOCKET_OPEN) return;
364
+ // Do not wait for the phrase's duration/visemes to start audio.
365
+ resetIdle();
366
+ sampleCount += frame.byteLength / 2;
367
+ if (socket.bufferedAmount > 512 * 1024) {
368
+ controller.abort();
369
+ sendJson(socket, "error", { turnId, code: "voice_playback_backpressure",
370
+ message: "Voice playback fell behind. Speech stopped; the answer remains in chat." });
371
+ return;
372
+ }
373
+ sendPcm(socket, Buffer.from(frame));
374
+ },
375
+ signal: controller.signal,
376
+ voiceId: active.voiceId
377
+ }) : { sampleRate: engine.sampleRate };
378
+ if (controller.signal.aborted || socket.readyState !== WEBSOCKET_OPEN) return;
379
+ const sampleRate = Number(result?.sampleRate || engine.sampleRate);
380
+ const spokenDurationMs = Math.round(sampleCount / sampleRate * 1000);
381
+ const last = active.ended && !active.pending.length && position === phrases.length - 1;
382
+ const pauseSamples = last && !text.endsWith("\n") ? 0 : Math.round(speechPauseDurationMs(text) * sampleRate / 1000);
383
+ if (pauseSamples) sendPcm(socket, Buffer.alloc(pauseSamples * 2));
384
+ sampleCount += pauseSamples;
385
+ sendJson(socket, "speech.segment", { cues: visemeCuesFromText(text, spokenDurationMs),
386
+ durationMs: Math.round(sampleCount / sampleRate * 1000), sampleCount, segmentIndex, turnId });
387
+ });
388
+ }
389
+ if (!controller.signal.aborted) sendJson(socket, "speech.chunk.end", { turnId });
390
+ }
391
+ if (active.ended && !controller.signal.aborted && socket.readyState === WEBSOCKET_OPEN) {
392
+ sendJson(socket, "speech.end", { turnId });
393
+ if (speaking === active) speaking = null;
394
+ }
395
+ } catch (error) {
396
+ if (!controller.signal.aborted) sendJson(socket, error?.code === "voice_busy" ? "busy" : "error", {
397
+ ...voiceDaemonError(error, "Speech could not be generated."), turnId
398
+ });
399
+ controller.abort();
400
+ if (speaking === active) speaking = null;
401
+ } finally { active.processing = false; }
402
+ };
403
+
404
+ const startSpeaking = (control) => {
405
+ const voiceId = control.voiceId || engine.defaultVoice;
406
+ if (voiceId && !engine.voices?.some(voice => voice.id === voiceId)) {
407
+ throw Object.assign(new Error("This voice is unavailable. Choose another voice."), { code: "voice_selection_invalid" });
408
+ }
409
+ cancelSpeaking();
410
+ speaking = { controller: new AbortController(), turnId: control.turnId,
411
+ voiceId, pending: [control.text], ended: !control.stream, processing: false, started: false, segmentIndex: 0, stopAfter: Infinity };
412
+ void drainSpeech(speaking);
413
+ };
414
+
415
+ socket.on("message", (raw, isBinary) => {
416
+ if (socket.readyState !== WEBSOCKET_OPEN) return;
417
+ resetIdle();
418
+ try {
419
+ if (isBinary) {
420
+ if (!listening || listening.ending) {
421
+ const error = new Error("Audio arrived without an active recording.");
422
+ error.code = "voice_listen_not_active";
423
+ throw error;
424
+ }
425
+ const frame = assertVoiceAudioFrame(raw);
426
+ const active = listening;
427
+ const sliceBytes = VOICE_INPUT_SAMPLE_RATE / 10 * 2;
428
+ for (let offset = 0; offset < frame.byteLength; offset += sliceBytes) {
429
+ queueListening(active, { frame: frame.subarray(offset, offset + sliceBytes) });
430
+ }
431
+ invalidMessages = 0;
432
+ return;
433
+ }
434
+ const control = parseVoiceClientControl(raw);
435
+ if (control.type === "ping") {
436
+ sendJson(socket, "pong", { turnId: control.turnId });
437
+ } else if (control.type === "listen.start") {
438
+ startListening(control);
439
+ } else if (control.type === "listen.stop") {
440
+ if (!listening || listening.ending || listening.turnId !== control.turnId) {
441
+ throw Object.assign(new Error("No matching voice recording is active."), { code: "voice_listen_not_active" });
442
+ }
443
+ listening.ending = true;
444
+ queueListening(listening, { control });
445
+ } else if (control.type === "listen.commit" || control.type === "listen.discard") {
446
+ const active = listening;
447
+ if (!active?.continuous || active.ending || active.turnId !== control.turnId) {
448
+ sendJson(socket, "transcript.stale", { turnId: control.turnId, revision: control.revision });
449
+ return;
450
+ }
451
+ queueListening(active, { control });
452
+ } else if (control.type === "speak.start") {
453
+ startSpeaking(control);
454
+ } else if (control.type === "speak.stop-after") {
455
+ const active = speaking;
456
+ if (active?.turnId === control.turnId) {
457
+ active.stopAfter = control.segmentIndex;
458
+ active.pending.length = 0;
459
+ active.ended = true;
460
+ if (active.segmentIndex - 1 > control.segmentIndex) cancelSpeaking();
461
+ else { void drainSpeech(active); return; }
462
+ }
463
+ sendJson(socket, "speech.end", { turnId: control.turnId });
464
+ } else if (control.type === "speak.append" || control.type === "speak.end") {
465
+ if (!speaking || speaking.turnId !== control.turnId || speaking.ended) {
466
+ throw Object.assign(new Error("No matching speech stream is open."), { code: "voice_speech_not_active" });
467
+ }
468
+ if (control.type === "speak.append") {
469
+ if (speaking.pending.length >= 2) throw Object.assign(new Error("Speech is arriving faster than playback."), { code: "voice_busy" });
470
+ speaking.pending.push(control.text);
471
+ } else speaking.ended = true;
472
+ void drainSpeech(speaking);
473
+ } else if (control.type === "cancel") {
474
+ if (listening?.turnId === control.turnId) {
475
+ cancelListening();
476
+ }
477
+ if (speaking?.turnId === control.turnId) {
478
+ cancelSpeaking();
479
+ }
480
+ sendJson(socket, "cancelled", { turnId: control.turnId });
481
+ }
482
+ invalidMessages = 0;
483
+ } catch (error) {
484
+ failMessage(error);
485
+ }
486
+ });
487
+
488
+ socket.once("close", () => {
489
+ clearInterval(idleTimer);
490
+ cancelListening();
491
+ cancelSpeaking();
492
+ releaseConnection(socket);
493
+ });
494
+ socket.once("error", (error) => {
495
+ logger?.warn?.({ error: String(error?.message || error), tenantId }, "Voice WebSocket failed.");
496
+ });
497
+ sendJson(socket, "voice.ready", {
498
+ inputSampleRate: VOICE_INPUT_SAMPLE_RATE,
499
+ outputSampleRate: Number(engine.sampleRate),
500
+ ...voiceCatalogue()
501
+ });
502
+ });
503
+
504
+ server.on("upgrade", (request, socket, head) => {
505
+ if (stopping || !ready || requestPathname(request) !== VOICE_DAEMON_WEBSOCKET_PATH) {
506
+ socket.write("HTTP/1.1 503 Service Unavailable\r\nConnection: close\r\n\r\n");
507
+ socket.destroy();
508
+ return;
509
+ }
510
+ let verified;
511
+ try { verified = authorize(bearerVoiceAccessToken(request.headers), request); }
512
+ catch { verified = null; }
513
+ if (!verified || typeof verified.tenant !== "string" || !verified.tenant) {
514
+ socket.write("HTTP/1.1 401 Unauthorized\r\nConnection: close\r\n\r\n");
515
+ socket.destroy();
516
+ return;
517
+ }
518
+ if (!connectionCapacityAvailable(verified)) {
519
+ socket.write("HTTP/1.1 429 Too Many Requests\r\nConnection: close\r\nRetry-After: 2\r\n\r\n");
520
+ socket.destroy();
521
+ return;
522
+ }
523
+ websocketServer.handleUpgrade(request, socket, head, (websocket) => {
524
+ websocketServer.emit("connection", websocket, request, verified);
525
+ });
526
+ });
527
+
528
+ async function start() {
529
+ if (ready) {
530
+ return address();
531
+ }
532
+ await engine.warmup?.();
533
+ await new Promise((resolve, reject) => {
534
+ const fail = (error) => {
535
+ server.off("listening", listen);
536
+ reject(error);
537
+ };
538
+ const listen = () => {
539
+ server.off("error", fail);
540
+ resolve();
541
+ };
542
+ server.once("error", fail);
543
+ server.once("listening", listen);
544
+ server.listen(port, host);
545
+ });
546
+ ready = true;
547
+ startedAt = new Date().toISOString();
548
+ return address();
549
+ }
550
+
551
+ function address() {
552
+ const value = server.address();
553
+ return value && typeof value === "object"
554
+ ? { address: value.address, family: value.family, port: value.port }
555
+ : null;
556
+ }
557
+
558
+ async function close() {
559
+ if (stopping) {
560
+ return;
561
+ }
562
+ stopping = true;
563
+ ready = false;
564
+ speechQueue.close();
565
+ for (const socket of sockets) {
566
+ socket.close(1012, "Voice service restarting");
567
+ }
568
+ await new Promise((resolve, reject) => {
569
+ websocketServer.close(() => {
570
+ server.close((error) => error ? reject(error) : resolve());
571
+ });
572
+ }).catch((error) => {
573
+ if (error?.code !== "ERR_SERVER_NOT_RUNNING") {
574
+ throw error;
575
+ }
576
+ });
577
+ }
578
+
579
+ return Object.freeze({
580
+ address,
581
+ close,
582
+ revokeTenant,
583
+ metrics() {
584
+ return Object.freeze({
585
+ connections: sockets.size,
586
+ queueDepth: speechQueue.depth,
587
+ tenants: tenantConnections.size
588
+ });
589
+ },
590
+ start
591
+ });
592
+ }
593
+
594
+ export {
595
+ createBoundedSerialQueue,
596
+ createVoiceDaemon,
597
+ sendJson,
598
+ voiceDaemonError
599
+ };