@kuralle-syrinx/server-websocket 4.2.0 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -5
- package/src/carrier-commands.ts +332 -0
- package/src/edge-telnyx.ts +389 -0
- package/src/edge.ts +23 -2
- package/src/index.ts +25 -2
- package/src/telnyx-codec.ts +163 -0
- package/src/telnyx.ts +54 -59
- package/src/turn-metrics.ts +107 -5
- package/src/twilio.ts +23 -0
- package/src/wire-carrier-control.ts +179 -0
- package/src/admission-control.test.ts +0 -270
- package/src/background-audio.test.ts +0 -224
- package/src/browser-opus.test.ts +0 -41
- package/src/browser-pacing.test.ts +0 -440
- package/src/edge-twilio.test.ts +0 -293
- package/src/edge.test.ts +0 -591
- package/src/graceful-drain.test.ts +0 -306
- package/src/inbound-audio.test.ts +0 -58
- package/src/index.test.ts +0 -2074
- package/src/outbound-playout-pipeline.test.ts +0 -314
- package/src/paced-playout.test.ts +0 -247
- package/src/playout-progress.test.ts +0 -56
- package/src/session-store.test.ts +0 -274
- package/src/smartpbx.test.ts +0 -967
- package/src/telnyx.test.ts +0 -1457
- package/src/transport-host.test.ts +0 -51
- package/src/turn-metrics.test.ts +0 -327
- package/src/twilio-auth.test.ts +0 -36
- package/src/twilio.test.ts +0 -1275
- package/src/websocket-close.test.ts +0 -63
- package/src/websocket-lifecycle.test.ts +0 -49
- package/tsconfig.json +0 -22
- package/vitest.config.ts +0 -7
package/src/index.test.ts
DELETED
|
@@ -1,2074 +0,0 @@
|
|
|
1
|
-
// SPDX-License-Identifier: MIT
|
|
2
|
-
|
|
3
|
-
import { createServer } from "node:http";
|
|
4
|
-
import { describe, expect, it, vi } from "vitest";
|
|
5
|
-
import WebSocket from "ws";
|
|
6
|
-
import {
|
|
7
|
-
Route,
|
|
8
|
-
VoiceAgentSession,
|
|
9
|
-
type ConversationMetricPacket,
|
|
10
|
-
type PipelineBus,
|
|
11
|
-
type PluginConfig,
|
|
12
|
-
type UserAudioReceivedPacket,
|
|
13
|
-
type UserTextReceivedPacket,
|
|
14
|
-
type VadAudioPacket,
|
|
15
|
-
type VoicePlugin,
|
|
16
|
-
} from "@kuralle-syrinx/core";
|
|
17
|
-
import { Decoder as OpusDecoder, Encoder as OpusEncoder } from "@evan/opus";
|
|
18
|
-
import { pcm16BytesToSamples, pcm16SamplesToBytes } from "@kuralle-syrinx/core/audio";
|
|
19
|
-
import {
|
|
20
|
-
createSmartPbxMediaStreamServer,
|
|
21
|
-
createTelnyxMediaStreamServer,
|
|
22
|
-
createTwilioMediaStreamServer,
|
|
23
|
-
createVoiceWebSocketServer,
|
|
24
|
-
} from "./index.js";
|
|
25
|
-
import { BROWSER_OPUS_FRAME_DURATION_MS } from "./browser-opus.js";
|
|
26
|
-
import {
|
|
27
|
-
openBrowserClientAndReadReady,
|
|
28
|
-
openBrowserSocketReady,
|
|
29
|
-
openSocket,
|
|
30
|
-
readJson,
|
|
31
|
-
readJsonMatching,
|
|
32
|
-
registerHttpServer,
|
|
33
|
-
registerServer,
|
|
34
|
-
registerSocket,
|
|
35
|
-
setupTransportTestCleanup,
|
|
36
|
-
waitForClose,
|
|
37
|
-
waitForCondition,
|
|
38
|
-
} from "./test-helpers.js";
|
|
39
|
-
|
|
40
|
-
setupTransportTestCleanup();
|
|
41
|
-
|
|
42
|
-
function websocketUrl(port: number): string {
|
|
43
|
-
return `ws://127.0.0.1:${port}/ws`;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
function websocketUrlWithSession(port: number, sessionId: string): string {
|
|
47
|
-
return `ws://127.0.0.1:${port}/ws?sessionId=${encodeURIComponent(sessionId)}`;
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
const BINARY_AUDIO_ENVELOPE_MAGIC = Buffer.from("SYRXA1\n", "ascii");
|
|
51
|
-
|
|
52
|
-
function encodeTestBinaryAudioEnvelope(header: Record<string, unknown>, audio: Uint8Array): Buffer {
|
|
53
|
-
const headerBytes = Buffer.from(JSON.stringify(header), "utf8");
|
|
54
|
-
const output = Buffer.alloc(BINARY_AUDIO_ENVELOPE_MAGIC.byteLength + 4 + headerBytes.byteLength + audio.byteLength);
|
|
55
|
-
BINARY_AUDIO_ENVELOPE_MAGIC.copy(output, 0);
|
|
56
|
-
output.writeUInt32LE(headerBytes.byteLength, BINARY_AUDIO_ENVELOPE_MAGIC.byteLength);
|
|
57
|
-
headerBytes.copy(output, BINARY_AUDIO_ENVELOPE_MAGIC.byteLength + 4);
|
|
58
|
-
Buffer.from(audio).copy(output, BINARY_AUDIO_ENVELOPE_MAGIC.byteLength + 4 + headerBytes.byteLength);
|
|
59
|
-
return output;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
function decodeTestBinaryAudioEnvelope(data: Buffer): { readonly header: any; readonly audio: Buffer } {
|
|
63
|
-
expect(data.subarray(0, BINARY_AUDIO_ENVELOPE_MAGIC.byteLength)).toEqual(BINARY_AUDIO_ENVELOPE_MAGIC);
|
|
64
|
-
const headerLength = data.readUInt32LE(BINARY_AUDIO_ENVELOPE_MAGIC.byteLength);
|
|
65
|
-
const headerStart = BINARY_AUDIO_ENVELOPE_MAGIC.byteLength + 4;
|
|
66
|
-
const headerEnd = headerStart + headerLength;
|
|
67
|
-
return {
|
|
68
|
-
header: JSON.parse(data.subarray(headerStart, headerEnd).toString("utf8")),
|
|
69
|
-
audio: data.subarray(headerEnd),
|
|
70
|
-
};
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
class VadAlignmentProbe implements VoicePlugin {
|
|
74
|
-
readonly observed: Array<{ contextId: string; byteOffsetParity: number; samples: number[] }> = [];
|
|
75
|
-
private dispose: (() => void) | null = null;
|
|
76
|
-
|
|
77
|
-
async initialize(bus: PipelineBus, _config: PluginConfig): Promise<void> {
|
|
78
|
-
this.dispose = bus.on("vad.audio", (pkt) => {
|
|
79
|
-
const audioPkt = pkt as VadAudioPacket;
|
|
80
|
-
this.observed.push({
|
|
81
|
-
contextId: audioPkt.contextId,
|
|
82
|
-
byteOffsetParity: audioPkt.audio.byteOffset % 2,
|
|
83
|
-
samples: Array.from(pcm16BytesToSamples(audioPkt.audio)),
|
|
84
|
-
});
|
|
85
|
-
});
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
async close(): Promise<void> {
|
|
89
|
-
this.dispose?.();
|
|
90
|
-
this.dispose = null;
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
describe("createVoiceWebSocketServer", () => {
|
|
95
|
-
it("routes multiple provider websocket paths on a shared HTTP server without handshake cross-talk", async () => {
|
|
96
|
-
const httpServer = registerHttpServer(createServer());
|
|
97
|
-
const [twilio, telnyx, smartpbx] = await Promise.all([
|
|
98
|
-
registerServer(await createTwilioMediaStreamServer({
|
|
99
|
-
server: httpServer,
|
|
100
|
-
createSession: () => new VoiceAgentSession({ plugins: {} }),
|
|
101
|
-
})),
|
|
102
|
-
registerServer(await createTelnyxMediaStreamServer({
|
|
103
|
-
server: httpServer,
|
|
104
|
-
createSession: () => new VoiceAgentSession({ plugins: {} }),
|
|
105
|
-
})),
|
|
106
|
-
registerServer(await createSmartPbxMediaStreamServer({
|
|
107
|
-
server: httpServer,
|
|
108
|
-
createSession: () => new VoiceAgentSession({ plugins: {} }),
|
|
109
|
-
})),
|
|
110
|
-
]);
|
|
111
|
-
await new Promise<void>((resolveListen, reject) => {
|
|
112
|
-
httpServer.once("error", reject);
|
|
113
|
-
httpServer.listen(0, "127.0.0.1", () => {
|
|
114
|
-
httpServer.off("error", reject);
|
|
115
|
-
resolveListen();
|
|
116
|
-
});
|
|
117
|
-
});
|
|
118
|
-
const address = httpServer.address();
|
|
119
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
120
|
-
|
|
121
|
-
const clients = await Promise.all([
|
|
122
|
-
openSocket(`ws://127.0.0.1:${String(address.port)}/twilio`, { perMessageDeflate: false }),
|
|
123
|
-
openSocket(`ws://127.0.0.1:${String(address.port)}/telnyx`, { perMessageDeflate: false }),
|
|
124
|
-
openSocket(`ws://127.0.0.1:${String(address.port)}/media-stream`, { perMessageDeflate: false }),
|
|
125
|
-
]);
|
|
126
|
-
expect(clients.map((client) => client.readyState)).toEqual([
|
|
127
|
-
WebSocket.OPEN,
|
|
128
|
-
WebSocket.OPEN,
|
|
129
|
-
WebSocket.OPEN,
|
|
130
|
-
]);
|
|
131
|
-
|
|
132
|
-
for (const client of clients) client.close();
|
|
133
|
-
await Promise.all([twilio.close(), telnyx.close(), smartpbx.close()]);
|
|
134
|
-
await new Promise<void>((resolveClose) => httpServer.close(() => resolveClose()));
|
|
135
|
-
});
|
|
136
|
-
|
|
137
|
-
it("does not negotiate websocket compression for browser media sessions", async () => {
|
|
138
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
139
|
-
port: 0,
|
|
140
|
-
createSession: () => new VoiceAgentSession({ plugins: {} }),
|
|
141
|
-
}));
|
|
142
|
-
const address = server.address();
|
|
143
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
144
|
-
|
|
145
|
-
const client = await openSocket(websocketUrl(address.port), { perMessageDeflate: true });
|
|
146
|
-
expect(client.extensions).toBe("");
|
|
147
|
-
|
|
148
|
-
client.close();
|
|
149
|
-
await server.close();
|
|
150
|
-
});
|
|
151
|
-
|
|
152
|
-
it("rejects raw binary browser audio unless explicitly enabled", async () => {
|
|
153
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
154
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
155
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
156
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
157
|
-
});
|
|
158
|
-
|
|
159
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
160
|
-
port: 0,
|
|
161
|
-
createSession: () => session,
|
|
162
|
-
contextId: () => "turn-test",
|
|
163
|
-
}));
|
|
164
|
-
const address = server.address();
|
|
165
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
166
|
-
|
|
167
|
-
const [client, ready] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
168
|
-
expect(ready).toMatchObject({ type: "ready", turnId: "turn-test", resumed: false });
|
|
169
|
-
expect(ready.audio.rawBinaryInput).toBe(false);
|
|
170
|
-
// Ready frame advertises the target output frame duration so clients can size playout. (VE-01.3)
|
|
171
|
-
expect(ready.audio.targetFrameDurationMs).toBe(20);
|
|
172
|
-
const errorMessage = new Promise<any>((resolve) => {
|
|
173
|
-
client.on("message", (data, isBinary) => {
|
|
174
|
-
if (isBinary) return;
|
|
175
|
-
const message = JSON.parse(data.toString());
|
|
176
|
-
if (message.type === "error") resolve(message);
|
|
177
|
-
});
|
|
178
|
-
});
|
|
179
|
-
|
|
180
|
-
client.send(Buffer.from([1, 2, 3, 4]));
|
|
181
|
-
|
|
182
|
-
await expect(errorMessage).resolves.toMatchObject({
|
|
183
|
-
type: "error",
|
|
184
|
-
component: "transport",
|
|
185
|
-
category: "invalid_input",
|
|
186
|
-
message: "Raw binary websocket audio is disabled; use syrinx.audio.v1 or JSON audio frames",
|
|
187
|
-
});
|
|
188
|
-
expect(received).toEqual([]);
|
|
189
|
-
|
|
190
|
-
client.close();
|
|
191
|
-
await server.close();
|
|
192
|
-
});
|
|
193
|
-
|
|
194
|
-
it("bridges raw binary browser audio only when rawBinaryInput is enabled", async () => {
|
|
195
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
196
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
197
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
198
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
199
|
-
});
|
|
200
|
-
|
|
201
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
202
|
-
port: 0,
|
|
203
|
-
rawBinaryInput: true,
|
|
204
|
-
createSession: () => session,
|
|
205
|
-
contextId: () => "turn-test",
|
|
206
|
-
}));
|
|
207
|
-
const address = server.address();
|
|
208
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
209
|
-
|
|
210
|
-
const [client, ready] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
211
|
-
expect(ready).toMatchObject({ type: "ready", turnId: "turn-test", resumed: false });
|
|
212
|
-
expect(ready.sessionId).toMatch(/^session-/);
|
|
213
|
-
expect(ready.maxSessionDurationMs).toBe(30 * 60_000);
|
|
214
|
-
expect(ready.audio).toMatchObject({
|
|
215
|
-
inputSampleRateHz: 16000,
|
|
216
|
-
outputSampleRateHz: 16000,
|
|
217
|
-
encoding: "opus",
|
|
218
|
-
supportedInputCodecs: ["pcm_s16le", "opus"],
|
|
219
|
-
channels: 1,
|
|
220
|
-
binaryEnvelope: "syrinx.audio.v1",
|
|
221
|
-
rawBinaryInput: true,
|
|
222
|
-
maxInboundMessageBytes: 2097152,
|
|
223
|
-
});
|
|
224
|
-
|
|
225
|
-
client.send(Buffer.from([1, 2, 3, 4]));
|
|
226
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
227
|
-
|
|
228
|
-
expect(received).toEqual([
|
|
229
|
-
expect.objectContaining({
|
|
230
|
-
kind: "user.audio_received",
|
|
231
|
-
contextId: "turn-test",
|
|
232
|
-
audio: new Uint8Array([1, 2, 3, 4]),
|
|
233
|
-
}),
|
|
234
|
-
]);
|
|
235
|
-
|
|
236
|
-
client.close();
|
|
237
|
-
await server.close();
|
|
238
|
-
});
|
|
239
|
-
|
|
240
|
-
it("buffers early websocket input sent before session ready", async () => {
|
|
241
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
242
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
243
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
244
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
245
|
-
});
|
|
246
|
-
|
|
247
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
248
|
-
port: 0,
|
|
249
|
-
createSession: async () => {
|
|
250
|
-
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
251
|
-
return session;
|
|
252
|
-
},
|
|
253
|
-
rawBinaryInput: true,
|
|
254
|
-
contextId: () => "turn-early",
|
|
255
|
-
}));
|
|
256
|
-
const address = server.address();
|
|
257
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
258
|
-
|
|
259
|
-
const client = registerSocket(new WebSocket(websocketUrl(address.port)));
|
|
260
|
-
const ready = readJson(client);
|
|
261
|
-
await new Promise<void>((resolveOpen, reject) => {
|
|
262
|
-
client.once("open", resolveOpen);
|
|
263
|
-
client.once("error", reject);
|
|
264
|
-
});
|
|
265
|
-
client.send(Buffer.from([1, 2, 3, 4]));
|
|
266
|
-
|
|
267
|
-
await expect(ready).resolves.toMatchObject({ type: "ready", turnId: "turn-early" });
|
|
268
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
269
|
-
|
|
270
|
-
expect(received).toEqual([
|
|
271
|
-
expect.objectContaining({
|
|
272
|
-
kind: "user.audio_received",
|
|
273
|
-
contextId: "turn-early",
|
|
274
|
-
audio: new Uint8Array([1, 2, 3, 4]),
|
|
275
|
-
}),
|
|
276
|
-
]);
|
|
277
|
-
|
|
278
|
-
client.close();
|
|
279
|
-
await server.close();
|
|
280
|
-
});
|
|
281
|
-
|
|
282
|
-
it("reports malformed early websocket input as a transport error after ready", async () => {
|
|
283
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
284
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
285
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
286
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
287
|
-
});
|
|
288
|
-
|
|
289
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
290
|
-
port: 0,
|
|
291
|
-
createSession: async () => {
|
|
292
|
-
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
293
|
-
return session;
|
|
294
|
-
},
|
|
295
|
-
rawBinaryInput: true,
|
|
296
|
-
contextId: () => "turn-early",
|
|
297
|
-
}));
|
|
298
|
-
const address = server.address();
|
|
299
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
300
|
-
|
|
301
|
-
const client = registerSocket(new WebSocket(websocketUrl(address.port)));
|
|
302
|
-
const messages: any[] = [];
|
|
303
|
-
const transportError = new Promise<any>((resolve) => {
|
|
304
|
-
client.on("message", (data, isBinary) => {
|
|
305
|
-
if (isBinary) return;
|
|
306
|
-
const message = JSON.parse(data.toString());
|
|
307
|
-
messages.push(message);
|
|
308
|
-
if (message.type === "error") resolve(message);
|
|
309
|
-
});
|
|
310
|
-
});
|
|
311
|
-
await new Promise<void>((resolveOpen, reject) => {
|
|
312
|
-
client.once("open", resolveOpen);
|
|
313
|
-
client.once("error", reject);
|
|
314
|
-
});
|
|
315
|
-
client.send(Buffer.from([1, 2, 3]));
|
|
316
|
-
|
|
317
|
-
await expect(transportError).resolves.toMatchObject({
|
|
318
|
-
type: "error",
|
|
319
|
-
component: "transport",
|
|
320
|
-
category: "invalid_input",
|
|
321
|
-
message: "PCM16 audio payload must contain an even number of bytes",
|
|
322
|
-
});
|
|
323
|
-
expect(messages.some((message) => message.type === "ready")).toBe(true);
|
|
324
|
-
expect(received).toEqual([]);
|
|
325
|
-
|
|
326
|
-
client.close();
|
|
327
|
-
await server.close();
|
|
328
|
-
});
|
|
329
|
-
|
|
330
|
-
it("closes browser websocket connections when session startup exceeds startupTimeoutMs", async () => {
|
|
331
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
332
|
-
port: 0,
|
|
333
|
-
startupTimeoutMs: 10,
|
|
334
|
-
createSession: () => new Promise<VoiceAgentSession>(() => undefined),
|
|
335
|
-
contextId: () => "turn-startup-timeout",
|
|
336
|
-
}));
|
|
337
|
-
const address = server.address();
|
|
338
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
339
|
-
|
|
340
|
-
const client = await openSocket(websocketUrl(address.port));
|
|
341
|
-
const errorMessage = readJson(client);
|
|
342
|
-
const closed = new Promise<{ code: number; reason: string }>((resolve) => {
|
|
343
|
-
client.once("close", (code, reason) => {
|
|
344
|
-
resolve({ code, reason: reason.toString() });
|
|
345
|
-
});
|
|
346
|
-
});
|
|
347
|
-
|
|
348
|
-
await expect(errorMessage).resolves.toMatchObject({
|
|
349
|
-
type: "error",
|
|
350
|
-
component: "session",
|
|
351
|
-
category: "startup_timeout",
|
|
352
|
-
});
|
|
353
|
-
await expect(closed).resolves.toEqual({
|
|
354
|
-
code: 1011,
|
|
355
|
-
reason: "session initialization failed",
|
|
356
|
-
});
|
|
357
|
-
|
|
358
|
-
await server.close();
|
|
359
|
-
});
|
|
360
|
-
|
|
361
|
-
it("resumes a browser websocket session within the retention window", async () => {
|
|
362
|
-
let created = 0;
|
|
363
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
364
|
-
const textPackets: UserTextReceivedPacket[] = [];
|
|
365
|
-
session.bus.on("user.text_received", (pkt) => {
|
|
366
|
-
textPackets.push(pkt as UserTextReceivedPacket);
|
|
367
|
-
});
|
|
368
|
-
|
|
369
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
370
|
-
port: 0,
|
|
371
|
-
resumeWindowMs: 200,
|
|
372
|
-
createSession: () => {
|
|
373
|
-
created += 1;
|
|
374
|
-
return session;
|
|
375
|
-
},
|
|
376
|
-
contextId: () => "turn-test",
|
|
377
|
-
}));
|
|
378
|
-
const address = server.address();
|
|
379
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
380
|
-
|
|
381
|
-
const [first, firstReady] = await openBrowserClientAndReadReady(websocketUrlWithSession(address.port, "resume-test"));
|
|
382
|
-
expect(firstReady).toMatchObject({
|
|
383
|
-
type: "ready",
|
|
384
|
-
sessionId: "resume-test",
|
|
385
|
-
resumed: false,
|
|
386
|
-
});
|
|
387
|
-
first.close();
|
|
388
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
389
|
-
|
|
390
|
-
const [second, secondReady] = await openBrowserClientAndReadReady(websocketUrlWithSession(address.port, "resume-test"));
|
|
391
|
-
expect(secondReady).toMatchObject({
|
|
392
|
-
type: "ready",
|
|
393
|
-
sessionId: "resume-test",
|
|
394
|
-
resumed: true,
|
|
395
|
-
});
|
|
396
|
-
second.send(JSON.stringify({ type: "text", text: "after reconnect", contextId: "turn-resumed" }));
|
|
397
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
398
|
-
|
|
399
|
-
expect(created).toBe(1);
|
|
400
|
-
expect(textPackets).toEqual([
|
|
401
|
-
expect.objectContaining({
|
|
402
|
-
kind: "user.text_received",
|
|
403
|
-
contextId: "turn-resumed",
|
|
404
|
-
text: "after reconnect",
|
|
405
|
-
}),
|
|
406
|
-
]);
|
|
407
|
-
|
|
408
|
-
second.close();
|
|
409
|
-
await server.close();
|
|
410
|
-
});
|
|
411
|
-
|
|
412
|
-
it("keeps browser websocket audio format invariants across session resume", async () => {
|
|
413
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
414
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
415
|
-
port: 0,
|
|
416
|
-
resumeWindowMs: 200,
|
|
417
|
-
createSession: () => session,
|
|
418
|
-
contextId: () => "turn-initial",
|
|
419
|
-
}));
|
|
420
|
-
const address = server.address();
|
|
421
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
422
|
-
|
|
423
|
-
const [first] = await openBrowserClientAndReadReady(websocketUrlWithSession(address.port, "resume-rate-test"));
|
|
424
|
-
first.send(JSON.stringify({
|
|
425
|
-
type: "audio",
|
|
426
|
-
audio: Buffer.from([1, 0, 2, 0]).toString("base64"),
|
|
427
|
-
contextId: "turn-resume-rate",
|
|
428
|
-
sampleRateHz: 48000,
|
|
429
|
-
sequence: 1,
|
|
430
|
-
}));
|
|
431
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
432
|
-
first.close();
|
|
433
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
434
|
-
|
|
435
|
-
const [second, secondReady] = await openBrowserClientAndReadReady(websocketUrlWithSession(address.port, "resume-rate-test"));
|
|
436
|
-
expect(secondReady).toMatchObject({
|
|
437
|
-
type: "ready",
|
|
438
|
-
sessionId: "resume-rate-test",
|
|
439
|
-
turnId: "turn-resume-rate",
|
|
440
|
-
resumed: true,
|
|
441
|
-
});
|
|
442
|
-
const errorMessage = new Promise<any>((resolve) => {
|
|
443
|
-
second.on("message", (data, isBinary) => {
|
|
444
|
-
if (isBinary) return;
|
|
445
|
-
const message = JSON.parse(data.toString());
|
|
446
|
-
if (message.type === "error") resolve(message);
|
|
447
|
-
});
|
|
448
|
-
});
|
|
449
|
-
|
|
450
|
-
second.send(JSON.stringify({
|
|
451
|
-
type: "audio",
|
|
452
|
-
audio: Buffer.from([3, 0, 4, 0]).toString("base64"),
|
|
453
|
-
contextId: "turn-resume-rate",
|
|
454
|
-
sampleRateHz: 44100,
|
|
455
|
-
sequence: 2,
|
|
456
|
-
}));
|
|
457
|
-
|
|
458
|
-
await expect(errorMessage).resolves.toMatchObject({
|
|
459
|
-
type: "error",
|
|
460
|
-
component: "transport",
|
|
461
|
-
category: "invalid_input",
|
|
462
|
-
message: "Websocket audio sampleRateHz changed within context turn-resume-rate: 48000 -> 44100",
|
|
463
|
-
});
|
|
464
|
-
|
|
465
|
-
second.close();
|
|
466
|
-
await server.close();
|
|
467
|
-
});
|
|
468
|
-
|
|
469
|
-
it("keeps browser websocket audio sequence invariants across session resume", async () => {
|
|
470
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
471
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
472
|
-
port: 0,
|
|
473
|
-
resumeWindowMs: 200,
|
|
474
|
-
createSession: () => session,
|
|
475
|
-
contextId: () => "turn-initial",
|
|
476
|
-
}));
|
|
477
|
-
const address = server.address();
|
|
478
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
479
|
-
|
|
480
|
-
const [first] = await openBrowserClientAndReadReady(websocketUrlWithSession(address.port, "resume-sequence-test"));
|
|
481
|
-
first.send(JSON.stringify({
|
|
482
|
-
type: "audio",
|
|
483
|
-
audio: Buffer.from([1, 0, 2, 0]).toString("base64"),
|
|
484
|
-
contextId: "turn-resume-sequence",
|
|
485
|
-
sampleRateHz: 16000,
|
|
486
|
-
sequence: 5,
|
|
487
|
-
}));
|
|
488
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
489
|
-
first.close();
|
|
490
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
491
|
-
|
|
492
|
-
const [second, secondReady] = await openBrowserClientAndReadReady(websocketUrlWithSession(address.port, "resume-sequence-test"));
|
|
493
|
-
expect(secondReady).toMatchObject({
|
|
494
|
-
type: "ready",
|
|
495
|
-
sessionId: "resume-sequence-test",
|
|
496
|
-
turnId: "turn-resume-sequence",
|
|
497
|
-
resumed: true,
|
|
498
|
-
});
|
|
499
|
-
const errorMessage = new Promise<any>((resolve) => {
|
|
500
|
-
second.on("message", (data, isBinary) => {
|
|
501
|
-
if (isBinary) return;
|
|
502
|
-
const message = JSON.parse(data.toString());
|
|
503
|
-
if (message.type === "error") resolve(message);
|
|
504
|
-
});
|
|
505
|
-
});
|
|
506
|
-
|
|
507
|
-
second.send(JSON.stringify({
|
|
508
|
-
type: "audio",
|
|
509
|
-
audio: Buffer.from([3, 0, 4, 0]).toString("base64"),
|
|
510
|
-
contextId: "turn-resume-sequence",
|
|
511
|
-
sampleRateHz: 16000,
|
|
512
|
-
sequence: 4,
|
|
513
|
-
}));
|
|
514
|
-
|
|
515
|
-
await expect(errorMessage).resolves.toMatchObject({
|
|
516
|
-
type: "error",
|
|
517
|
-
component: "transport",
|
|
518
|
-
category: "invalid_input",
|
|
519
|
-
message: "Websocket audio sequence must increase monotonically: 5 -> 4",
|
|
520
|
-
});
|
|
521
|
-
|
|
522
|
-
second.close();
|
|
523
|
-
await server.close();
|
|
524
|
-
});
|
|
525
|
-
|
|
526
|
-
it("closes retained browser websocket sessions after the resume window expires", async () => {
|
|
527
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
528
|
-
const closeSpy = vi.spyOn(session, "close");
|
|
529
|
-
|
|
530
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
531
|
-
port: 0,
|
|
532
|
-
resumeWindowMs: 10,
|
|
533
|
-
createSession: () => session,
|
|
534
|
-
}));
|
|
535
|
-
const address = server.address();
|
|
536
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
537
|
-
|
|
538
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrlWithSession(address.port, "expire-test"));
|
|
539
|
-
client.close();
|
|
540
|
-
await new Promise((resolve) => setTimeout(resolve, 40));
|
|
541
|
-
|
|
542
|
-
expect(closeSpy).toHaveBeenCalledTimes(1);
|
|
543
|
-
expect(session.state).toBe("closed");
|
|
544
|
-
|
|
545
|
-
await server.close();
|
|
546
|
-
});
|
|
547
|
-
|
|
548
|
-
it("bridges browser text messages and streams TTS audio back to the socket", async () => {
|
|
549
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
550
|
-
const textPackets: UserTextReceivedPacket[] = [];
|
|
551
|
-
session.bus.on("user.text_received", (pkt) => {
|
|
552
|
-
textPackets.push(pkt as UserTextReceivedPacket);
|
|
553
|
-
});
|
|
554
|
-
|
|
555
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
556
|
-
port: 0,
|
|
557
|
-
createSession: () => session,
|
|
558
|
-
contextId: () => "turn-test",
|
|
559
|
-
}));
|
|
560
|
-
const address = server.address();
|
|
561
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
562
|
-
|
|
563
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
564
|
-
|
|
565
|
-
const audioMessage = new Promise<Buffer>((resolve) => {
|
|
566
|
-
client.on("message", (data, isBinary) => {
|
|
567
|
-
if (isBinary) resolve(data as Buffer);
|
|
568
|
-
});
|
|
569
|
-
});
|
|
570
|
-
const metadataMessage = new Promise<any>((resolve) => {
|
|
571
|
-
client.on("message", (data, isBinary) => {
|
|
572
|
-
if (isBinary) return;
|
|
573
|
-
const message = JSON.parse(data.toString());
|
|
574
|
-
if (message.type === "tts_chunk") resolve(message);
|
|
575
|
-
});
|
|
576
|
-
});
|
|
577
|
-
|
|
578
|
-
client.send(JSON.stringify({ type: "text", text: "hello", contextId: "turn-2" }));
|
|
579
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
580
|
-
session.bus.push(Route.Main, {
|
|
581
|
-
kind: "tts.audio",
|
|
582
|
-
contextId: "turn-2",
|
|
583
|
-
timestampMs: Date.now(),
|
|
584
|
-
audio: new Uint8Array([8, 9, 10, 11]),
|
|
585
|
-
sampleRateHz: 16000,
|
|
586
|
-
});
|
|
587
|
-
session.bus.push(Route.Main, {
|
|
588
|
-
kind: "tts.end",
|
|
589
|
-
contextId: "turn-2",
|
|
590
|
-
timestampMs: Date.now(),
|
|
591
|
-
});
|
|
592
|
-
|
|
593
|
-
expect(textPackets).toEqual([
|
|
594
|
-
expect.objectContaining({
|
|
595
|
-
kind: "user.text_received",
|
|
596
|
-
contextId: "turn-2",
|
|
597
|
-
text: "hello",
|
|
598
|
-
}),
|
|
599
|
-
]);
|
|
600
|
-
await expect(metadataMessage).resolves.toMatchObject({
|
|
601
|
-
type: "tts_chunk",
|
|
602
|
-
turnId: "turn-2",
|
|
603
|
-
sequence: 1,
|
|
604
|
-
sampleRateHz: 48000, // opus frames are labeled at the 48 kHz codec rate
|
|
605
|
-
encoding: "opus",
|
|
606
|
-
channels: 1,
|
|
607
|
-
});
|
|
608
|
-
const envelope = decodeTestBinaryAudioEnvelope(await audioMessage);
|
|
609
|
-
expect(envelope.header).toMatchObject({
|
|
610
|
-
type: "audio",
|
|
611
|
-
contextId: "turn-2",
|
|
612
|
-
sequence: 1,
|
|
613
|
-
sampleRateHz: 48000,
|
|
614
|
-
encoding: "opus",
|
|
615
|
-
});
|
|
616
|
-
expect(envelope.audio.byteLength).toBeGreaterThan(0);
|
|
617
|
-
|
|
618
|
-
client.close();
|
|
619
|
-
await server.close();
|
|
620
|
-
});
|
|
621
|
-
|
|
622
|
-
it("forwards VAD-driven assistant interruption as audio clear events", async () => {
|
|
623
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
624
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
625
|
-
port: 0,
|
|
626
|
-
createSession: () => session,
|
|
627
|
-
contextId: () => "turn-test",
|
|
628
|
-
}));
|
|
629
|
-
const address = server.address();
|
|
630
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
631
|
-
|
|
632
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
633
|
-
const clearMessage = new Promise<any>((resolve) => {
|
|
634
|
-
client.on("message", (data, isBinary) => {
|
|
635
|
-
if (isBinary) return;
|
|
636
|
-
const message = JSON.parse(data.toString());
|
|
637
|
-
if (message.type === "audio_clear") resolve(message);
|
|
638
|
-
});
|
|
639
|
-
});
|
|
640
|
-
|
|
641
|
-
session.bus.push(Route.Critical, {
|
|
642
|
-
kind: "interrupt.tts",
|
|
643
|
-
contextId: "assistant-turn",
|
|
644
|
-
timestampMs: Date.now(),
|
|
645
|
-
});
|
|
646
|
-
|
|
647
|
-
await expect(clearMessage).resolves.toEqual({
|
|
648
|
-
type: "audio_clear",
|
|
649
|
-
turnId: "assistant-turn",
|
|
650
|
-
reason: "barge_in",
|
|
651
|
-
});
|
|
652
|
-
|
|
653
|
-
client.close();
|
|
654
|
-
await server.close();
|
|
655
|
-
});
|
|
656
|
-
|
|
657
|
-
it("does not send late browser audio chunks after a TTS interruption", async () => {
|
|
658
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
659
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
660
|
-
port: 0,
|
|
661
|
-
createSession: () => session,
|
|
662
|
-
contextId: () => "turn-test",
|
|
663
|
-
}));
|
|
664
|
-
const address = server.address();
|
|
665
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
666
|
-
|
|
667
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
668
|
-
const messages: any[] = [];
|
|
669
|
-
const binaries: Buffer[] = [];
|
|
670
|
-
client.on("message", (data, isBinary) => {
|
|
671
|
-
if (isBinary) {
|
|
672
|
-
binaries.push(Buffer.from(data as Buffer));
|
|
673
|
-
return;
|
|
674
|
-
}
|
|
675
|
-
messages.push(JSON.parse(data.toString()));
|
|
676
|
-
});
|
|
677
|
-
|
|
678
|
-
session.bus.push(Route.Critical, {
|
|
679
|
-
kind: "interrupt.tts",
|
|
680
|
-
contextId: "assistant-turn",
|
|
681
|
-
timestampMs: Date.now(),
|
|
682
|
-
});
|
|
683
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
684
|
-
session.bus.push(Route.Main, {
|
|
685
|
-
kind: "tts.audio",
|
|
686
|
-
contextId: "assistant-turn",
|
|
687
|
-
timestampMs: Date.now(),
|
|
688
|
-
audio: new Uint8Array([1, 2, 3, 4]),
|
|
689
|
-
sampleRateHz: 16000,
|
|
690
|
-
});
|
|
691
|
-
session.bus.push(Route.Main, {
|
|
692
|
-
kind: "tts.end",
|
|
693
|
-
contextId: "assistant-turn",
|
|
694
|
-
timestampMs: Date.now(),
|
|
695
|
-
});
|
|
696
|
-
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
697
|
-
|
|
698
|
-
expect(messages.some((message) => message.type === "audio_clear")).toBe(true);
|
|
699
|
-
expect(messages.some((message) => message.type === "tts_chunk" || message.type === "tts_end")).toBe(false);
|
|
700
|
-
expect(binaries).toEqual([]);
|
|
701
|
-
|
|
702
|
-
client.close();
|
|
703
|
-
await server.close();
|
|
704
|
-
});
|
|
705
|
-
|
|
706
|
-
it("routes browser client_interrupt through the assistant interruption path", async () => {
|
|
707
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
708
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
709
|
-
port: 0,
|
|
710
|
-
createSession: () => session,
|
|
711
|
-
contextId: () => "turn-test",
|
|
712
|
-
}));
|
|
713
|
-
const address = server.address();
|
|
714
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
715
|
-
|
|
716
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
717
|
-
const messages: any[] = [];
|
|
718
|
-
const binaries: Buffer[] = [];
|
|
719
|
-
const metrics: ConversationMetricPacket[] = [];
|
|
720
|
-
session.bus.on("metric.conversation", (pkt) => {
|
|
721
|
-
metrics.push(pkt as ConversationMetricPacket);
|
|
722
|
-
});
|
|
723
|
-
client.on("message", (data, isBinary) => {
|
|
724
|
-
if (isBinary) {
|
|
725
|
-
binaries.push(Buffer.from(data as Buffer));
|
|
726
|
-
return;
|
|
727
|
-
}
|
|
728
|
-
messages.push(JSON.parse(data.toString()));
|
|
729
|
-
});
|
|
730
|
-
|
|
731
|
-
session.bus.push(Route.Main, {
|
|
732
|
-
kind: "tts.audio",
|
|
733
|
-
contextId: "assistant-turn",
|
|
734
|
-
timestampMs: Date.now(),
|
|
735
|
-
audio: new Uint8Array(1600),
|
|
736
|
-
sampleRateHz: 16000,
|
|
737
|
-
});
|
|
738
|
-
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
739
|
-
client.send(JSON.stringify({
|
|
740
|
-
type: "client_interrupt",
|
|
741
|
-
assistantContextId: "assistant-turn",
|
|
742
|
-
reason: "local_vad_speech_start",
|
|
743
|
-
}));
|
|
744
|
-
await waitForCondition(() => messages.some((message) => message.type === "audio_clear"));
|
|
745
|
-
binaries.length = 0;
|
|
746
|
-
|
|
747
|
-
session.bus.push(Route.Main, {
|
|
748
|
-
kind: "tts.audio",
|
|
749
|
-
contextId: "assistant-turn",
|
|
750
|
-
timestampMs: Date.now(),
|
|
751
|
-
audio: new Uint8Array([1, 2, 3, 4]),
|
|
752
|
-
sampleRateHz: 16000,
|
|
753
|
-
});
|
|
754
|
-
session.bus.push(Route.Main, {
|
|
755
|
-
kind: "tts.end",
|
|
756
|
-
contextId: "assistant-turn",
|
|
757
|
-
timestampMs: Date.now(),
|
|
758
|
-
});
|
|
759
|
-
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
760
|
-
|
|
761
|
-
expect(messages).toEqual(expect.arrayContaining([
|
|
762
|
-
expect.objectContaining({ type: "audio_clear", turnId: "assistant-turn", reason: "barge_in" }),
|
|
763
|
-
expect.objectContaining({ type: "agent_interrupted", turnId: "assistant-turn", reason: "barge_in" }),
|
|
764
|
-
]));
|
|
765
|
-
expect(metrics).toEqual(expect.arrayContaining([
|
|
766
|
-
expect.objectContaining({ name: "interrupt.committed_after_ms", value: "0" }),
|
|
767
|
-
]));
|
|
768
|
-
expect(messages.some((message) => message.type === "tts_end")).toBe(false);
|
|
769
|
-
expect(binaries).toEqual([]);
|
|
770
|
-
|
|
771
|
-
client.close();
|
|
772
|
-
await server.close();
|
|
773
|
-
});
|
|
774
|
-
|
|
775
|
-
it("forwards VAD speech boundary events to websocket clients", async () => {
|
|
776
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
777
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
778
|
-
port: 0,
|
|
779
|
-
createSession: () => session,
|
|
780
|
-
contextId: () => "turn-test",
|
|
781
|
-
}));
|
|
782
|
-
const address = server.address();
|
|
783
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
784
|
-
|
|
785
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
786
|
-
const messages: any[] = [];
|
|
787
|
-
const speechMessages = new Promise<any[]>((resolve) => {
|
|
788
|
-
client.on("message", (data, isBinary) => {
|
|
789
|
-
if (isBinary) return;
|
|
790
|
-
const message = JSON.parse(data.toString());
|
|
791
|
-
if (message.type === "speech_started" || message.type === "speech_ended") {
|
|
792
|
-
messages.push(message);
|
|
793
|
-
}
|
|
794
|
-
if (messages.length === 2) resolve(messages);
|
|
795
|
-
});
|
|
796
|
-
});
|
|
797
|
-
|
|
798
|
-
session.bus.push(Route.Main, {
|
|
799
|
-
kind: "vad.speech_started",
|
|
800
|
-
contextId: "turn-2",
|
|
801
|
-
timestampMs: Date.now(),
|
|
802
|
-
confidence: 0.91,
|
|
803
|
-
});
|
|
804
|
-
session.bus.push(Route.Main, {
|
|
805
|
-
kind: "vad.speech_ended",
|
|
806
|
-
contextId: "turn-2",
|
|
807
|
-
timestampMs: Date.now(),
|
|
808
|
-
});
|
|
809
|
-
|
|
810
|
-
await expect(speechMessages).resolves.toEqual([
|
|
811
|
-
{ type: "speech_started", turnId: "turn-2" },
|
|
812
|
-
{ type: "speech_ended", turnId: "turn-2" },
|
|
813
|
-
]);
|
|
814
|
-
|
|
815
|
-
client.close();
|
|
816
|
-
await server.close();
|
|
817
|
-
});
|
|
818
|
-
|
|
819
|
-
it("normalizes JSON audio frames with explicit sample-rate metadata to the engine rate", async () => {
|
|
820
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
821
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
822
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
823
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
824
|
-
});
|
|
825
|
-
|
|
826
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
827
|
-
port: 0,
|
|
828
|
-
inputSampleRateHz: 16000,
|
|
829
|
-
createSession: () => session,
|
|
830
|
-
contextId: () => "turn-test",
|
|
831
|
-
}));
|
|
832
|
-
const address = server.address();
|
|
833
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
834
|
-
|
|
835
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
836
|
-
const samples48k = new Int16Array([0, 3000, 6000, 9000, 12000, 15000]);
|
|
837
|
-
|
|
838
|
-
client.send(JSON.stringify({
|
|
839
|
-
type: "audio",
|
|
840
|
-
contextId: "turn-resample",
|
|
841
|
-
sampleRateHz: 48000,
|
|
842
|
-
audio: Buffer.from(samples48k.buffer).toString("base64"),
|
|
843
|
-
}));
|
|
844
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
845
|
-
|
|
846
|
-
expect(received).toEqual([
|
|
847
|
-
expect.objectContaining({
|
|
848
|
-
kind: "user.audio_received",
|
|
849
|
-
contextId: "turn-resample",
|
|
850
|
-
}),
|
|
851
|
-
]);
|
|
852
|
-
// FIR-resampled output: 48k→16k on a 6-sample ramp gives these weighted sums.
|
|
853
|
-
expect(Buffer.from(received[0]!.audio)).toEqual(Buffer.from(new Int16Array([476, 10050]).buffer));
|
|
854
|
-
|
|
855
|
-
client.close();
|
|
856
|
-
await server.close();
|
|
857
|
-
});
|
|
858
|
-
|
|
859
|
-
it("rejects non-object websocket JSON messages before forwarding them", async () => {
|
|
860
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
861
|
-
const textPackets: UserTextReceivedPacket[] = [];
|
|
862
|
-
const audioPackets: UserAudioReceivedPacket[] = [];
|
|
863
|
-
session.bus.on("user.text_received", (pkt) => {
|
|
864
|
-
textPackets.push(pkt as UserTextReceivedPacket);
|
|
865
|
-
});
|
|
866
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
867
|
-
audioPackets.push(pkt as UserAudioReceivedPacket);
|
|
868
|
-
});
|
|
869
|
-
|
|
870
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
871
|
-
port: 0,
|
|
872
|
-
createSession: () => session,
|
|
873
|
-
contextId: () => "turn-test",
|
|
874
|
-
}));
|
|
875
|
-
const address = server.address();
|
|
876
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
877
|
-
|
|
878
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
879
|
-
const errorMessage = new Promise<any>((resolve) => {
|
|
880
|
-
client.on("message", (data, isBinary) => {
|
|
881
|
-
if (isBinary) return;
|
|
882
|
-
const message = JSON.parse(data.toString());
|
|
883
|
-
if (message.type === "error") resolve(message);
|
|
884
|
-
});
|
|
885
|
-
});
|
|
886
|
-
|
|
887
|
-
client.send("null");
|
|
888
|
-
|
|
889
|
-
await expect(errorMessage).resolves.toMatchObject({
|
|
890
|
-
type: "error",
|
|
891
|
-
component: "transport",
|
|
892
|
-
category: "invalid_input",
|
|
893
|
-
message: "Websocket JSON message must be an object",
|
|
894
|
-
});
|
|
895
|
-
expect(textPackets).toEqual([]);
|
|
896
|
-
expect(audioPackets).toEqual([]);
|
|
897
|
-
|
|
898
|
-
client.close();
|
|
899
|
-
await server.close();
|
|
900
|
-
});
|
|
901
|
-
|
|
902
|
-
it("rejects malformed websocket JSON text messages before forwarding them", async () => {
|
|
903
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
904
|
-
const textPackets: UserTextReceivedPacket[] = [];
|
|
905
|
-
session.bus.on("user.text_received", (pkt) => {
|
|
906
|
-
textPackets.push(pkt as UserTextReceivedPacket);
|
|
907
|
-
});
|
|
908
|
-
|
|
909
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
910
|
-
port: 0,
|
|
911
|
-
createSession: () => session,
|
|
912
|
-
contextId: () => "turn-test",
|
|
913
|
-
}));
|
|
914
|
-
const address = server.address();
|
|
915
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
916
|
-
|
|
917
|
-
const client = await openBrowserSocketReady(websocketUrl(address.port));
|
|
918
|
-
const errorMessage = readJsonMatching(client, (message) => (message as { type?: string }).type === "error");
|
|
919
|
-
|
|
920
|
-
client.send(JSON.stringify({ type: "text", text: 42, contextId: "turn-bad-text" }));
|
|
921
|
-
|
|
922
|
-
await expect(errorMessage).resolves.toMatchObject({
|
|
923
|
-
type: "error",
|
|
924
|
-
component: "transport",
|
|
925
|
-
category: "invalid_input",
|
|
926
|
-
message: "Websocket JSON text must be a non-empty string",
|
|
927
|
-
});
|
|
928
|
-
expect(textPackets).toEqual([]);
|
|
929
|
-
|
|
930
|
-
client.close();
|
|
931
|
-
await server.close();
|
|
932
|
-
});
|
|
933
|
-
|
|
934
|
-
it("rejects malformed websocket JSON context identifiers before forwarding them", async () => {
|
|
935
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
936
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
937
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
938
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
939
|
-
});
|
|
940
|
-
|
|
941
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
942
|
-
port: 0,
|
|
943
|
-
createSession: () => session,
|
|
944
|
-
contextId: () => "turn-test",
|
|
945
|
-
}));
|
|
946
|
-
const address = server.address();
|
|
947
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
948
|
-
|
|
949
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
950
|
-
const errorMessage = new Promise<any>((resolve) => {
|
|
951
|
-
client.on("message", (data, isBinary) => {
|
|
952
|
-
if (isBinary) return;
|
|
953
|
-
const message = JSON.parse(data.toString());
|
|
954
|
-
if (message.type === "error") resolve(message);
|
|
955
|
-
});
|
|
956
|
-
});
|
|
957
|
-
|
|
958
|
-
client.send(JSON.stringify({
|
|
959
|
-
type: "audio",
|
|
960
|
-
contextId: 123,
|
|
961
|
-
sampleRateHz: 16000,
|
|
962
|
-
audio: Buffer.from(new Int16Array([0, 1000]).buffer).toString("base64"),
|
|
963
|
-
}));
|
|
964
|
-
|
|
965
|
-
await expect(errorMessage).resolves.toMatchObject({
|
|
966
|
-
type: "error",
|
|
967
|
-
component: "transport",
|
|
968
|
-
category: "invalid_input",
|
|
969
|
-
message: "Websocket JSON contextId must be a non-empty string",
|
|
970
|
-
});
|
|
971
|
-
expect(received).toEqual([]);
|
|
972
|
-
|
|
973
|
-
client.close();
|
|
974
|
-
await server.close();
|
|
975
|
-
});
|
|
976
|
-
|
|
977
|
-
it("rejects JSON audio that changes sample rate inside the same websocket context", async () => {
|
|
978
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
979
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
980
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
981
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
982
|
-
});
|
|
983
|
-
|
|
984
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
985
|
-
port: 0,
|
|
986
|
-
inputSampleRateHz: 16000,
|
|
987
|
-
createSession: () => session,
|
|
988
|
-
contextId: () => "turn-test",
|
|
989
|
-
}));
|
|
990
|
-
const address = server.address();
|
|
991
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
992
|
-
|
|
993
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
994
|
-
const errorMessage = new Promise<any>((resolve) => {
|
|
995
|
-
client.on("message", (data, isBinary) => {
|
|
996
|
-
if (isBinary) return;
|
|
997
|
-
const message = JSON.parse(data.toString());
|
|
998
|
-
if (message.type === "error") resolve(message);
|
|
999
|
-
});
|
|
1000
|
-
});
|
|
1001
|
-
|
|
1002
|
-
client.send(JSON.stringify({
|
|
1003
|
-
type: "audio",
|
|
1004
|
-
contextId: "turn-fixed-rate",
|
|
1005
|
-
sampleRateHz: 48000,
|
|
1006
|
-
audio: Buffer.from(new Int16Array([0, 3000, 6000, 9000, 12000, 15000]).buffer).toString("base64"),
|
|
1007
|
-
}));
|
|
1008
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
1009
|
-
client.send(JSON.stringify({
|
|
1010
|
-
type: "audio",
|
|
1011
|
-
contextId: "turn-fixed-rate",
|
|
1012
|
-
sampleRateHz: 44100,
|
|
1013
|
-
audio: Buffer.from(new Int16Array([0, 1000, 2000, 3000]).buffer).toString("base64"),
|
|
1014
|
-
}));
|
|
1015
|
-
|
|
1016
|
-
await expect(errorMessage).resolves.toMatchObject({
|
|
1017
|
-
type: "error",
|
|
1018
|
-
component: "transport",
|
|
1019
|
-
category: "invalid_input",
|
|
1020
|
-
message: "Websocket audio sampleRateHz changed within context turn-fixed-rate: 48000 -> 44100",
|
|
1021
|
-
});
|
|
1022
|
-
expect(received).toHaveLength(1);
|
|
1023
|
-
|
|
1024
|
-
client.close();
|
|
1025
|
-
await server.close();
|
|
1026
|
-
});
|
|
1027
|
-
|
|
1028
|
-
it("allows different JSON audio sample rates on different websocket contexts", async () => {
|
|
1029
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1030
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
1031
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
1032
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
1033
|
-
});
|
|
1034
|
-
|
|
1035
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1036
|
-
port: 0,
|
|
1037
|
-
inputSampleRateHz: 16000,
|
|
1038
|
-
createSession: () => session,
|
|
1039
|
-
contextId: () => "turn-test",
|
|
1040
|
-
}));
|
|
1041
|
-
const address = server.address();
|
|
1042
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1043
|
-
|
|
1044
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1045
|
-
|
|
1046
|
-
client.send(JSON.stringify({
|
|
1047
|
-
type: "audio",
|
|
1048
|
-
contextId: "turn-48k",
|
|
1049
|
-
sampleRateHz: 48000,
|
|
1050
|
-
audio: Buffer.from(new Int16Array([0, 3000, 6000, 9000, 12000, 15000]).buffer).toString("base64"),
|
|
1051
|
-
}));
|
|
1052
|
-
client.send(JSON.stringify({
|
|
1053
|
-
type: "audio",
|
|
1054
|
-
contextId: "turn-44k",
|
|
1055
|
-
sampleRateHz: 44100,
|
|
1056
|
-
audio: Buffer.from(new Int16Array([0, 2000, 4000, 6000]).buffer).toString("base64"),
|
|
1057
|
-
}));
|
|
1058
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
1059
|
-
|
|
1060
|
-
expect(received.map((packet) => packet.contextId)).toEqual(["turn-48k", "turn-44k"]);
|
|
1061
|
-
|
|
1062
|
-
client.close();
|
|
1063
|
-
await server.close();
|
|
1064
|
-
});
|
|
1065
|
-
|
|
1066
|
-
it("records JSON audio sequence gaps without dropping monotonic frames", async () => {
|
|
1067
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1068
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
1069
|
-
const metrics: ConversationMetricPacket[] = [];
|
|
1070
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
1071
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
1072
|
-
});
|
|
1073
|
-
session.bus.on("metric.conversation", (pkt) => {
|
|
1074
|
-
metrics.push(pkt as ConversationMetricPacket);
|
|
1075
|
-
});
|
|
1076
|
-
|
|
1077
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1078
|
-
port: 0,
|
|
1079
|
-
inputSampleRateHz: 16000,
|
|
1080
|
-
createSession: () => session,
|
|
1081
|
-
contextId: () => "turn-test",
|
|
1082
|
-
}));
|
|
1083
|
-
const address = server.address();
|
|
1084
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1085
|
-
|
|
1086
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1087
|
-
const audio = Buffer.from(new Int16Array([0, 1000]).buffer).toString("base64");
|
|
1088
|
-
|
|
1089
|
-
client.send(JSON.stringify({
|
|
1090
|
-
type: "audio",
|
|
1091
|
-
contextId: "turn-sequence-gap",
|
|
1092
|
-
sampleRateHz: 16000,
|
|
1093
|
-
sequence: 1,
|
|
1094
|
-
audio,
|
|
1095
|
-
}));
|
|
1096
|
-
client.send(JSON.stringify({
|
|
1097
|
-
type: "audio",
|
|
1098
|
-
contextId: "turn-sequence-gap",
|
|
1099
|
-
sampleRateHz: 16000,
|
|
1100
|
-
sequence: 4,
|
|
1101
|
-
audio,
|
|
1102
|
-
}));
|
|
1103
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
1104
|
-
|
|
1105
|
-
expect(received).toHaveLength(2);
|
|
1106
|
-
expect(metrics).toContainEqual(expect.objectContaining({
|
|
1107
|
-
kind: "metric.conversation",
|
|
1108
|
-
contextId: "turn-sequence-gap",
|
|
1109
|
-
name: "websocket.audio_sequence_gap",
|
|
1110
|
-
value: JSON.stringify({ expected: 2, actual: 4, missed: 2 }),
|
|
1111
|
-
}));
|
|
1112
|
-
|
|
1113
|
-
client.close();
|
|
1114
|
-
await server.close();
|
|
1115
|
-
});
|
|
1116
|
-
|
|
1117
|
-
it("rejects duplicate JSON audio sequence numbers before forwarding the duplicate frame", async () => {
|
|
1118
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1119
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
1120
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
1121
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
1122
|
-
});
|
|
1123
|
-
|
|
1124
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1125
|
-
port: 0,
|
|
1126
|
-
inputSampleRateHz: 16000,
|
|
1127
|
-
createSession: () => session,
|
|
1128
|
-
contextId: () => "turn-test",
|
|
1129
|
-
}));
|
|
1130
|
-
const address = server.address();
|
|
1131
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1132
|
-
|
|
1133
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1134
|
-
const errorMessage = new Promise<any>((resolve) => {
|
|
1135
|
-
client.on("message", (data, isBinary) => {
|
|
1136
|
-
if (isBinary) return;
|
|
1137
|
-
const message = JSON.parse(data.toString());
|
|
1138
|
-
if (message.type === "error") resolve(message);
|
|
1139
|
-
});
|
|
1140
|
-
});
|
|
1141
|
-
const audio = Buffer.from(new Int16Array([0, 1000]).buffer).toString("base64");
|
|
1142
|
-
|
|
1143
|
-
client.send(JSON.stringify({
|
|
1144
|
-
type: "audio",
|
|
1145
|
-
contextId: "turn-sequence-duplicate",
|
|
1146
|
-
sampleRateHz: 16000,
|
|
1147
|
-
sequence: 2,
|
|
1148
|
-
audio,
|
|
1149
|
-
}));
|
|
1150
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
1151
|
-
client.send(JSON.stringify({
|
|
1152
|
-
type: "audio",
|
|
1153
|
-
contextId: "turn-sequence-duplicate",
|
|
1154
|
-
sampleRateHz: 16000,
|
|
1155
|
-
sequence: 2,
|
|
1156
|
-
audio,
|
|
1157
|
-
}));
|
|
1158
|
-
|
|
1159
|
-
await expect(errorMessage).resolves.toMatchObject({
|
|
1160
|
-
type: "error",
|
|
1161
|
-
component: "transport",
|
|
1162
|
-
category: "invalid_input",
|
|
1163
|
-
message: "Websocket audio sequence must increase monotonically: 2 -> 2",
|
|
1164
|
-
});
|
|
1165
|
-
expect(received).toHaveLength(1);
|
|
1166
|
-
|
|
1167
|
-
client.close();
|
|
1168
|
-
await server.close();
|
|
1169
|
-
});
|
|
1170
|
-
|
|
1171
|
-
it("rejects malformed odd-byte PCM16 websocket audio without forwarding it", async () => {
|
|
1172
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1173
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
1174
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
1175
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
1176
|
-
});
|
|
1177
|
-
|
|
1178
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1179
|
-
port: 0,
|
|
1180
|
-
rawBinaryInput: true,
|
|
1181
|
-
createSession: () => session,
|
|
1182
|
-
contextId: () => "turn-test",
|
|
1183
|
-
}));
|
|
1184
|
-
const address = server.address();
|
|
1185
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1186
|
-
|
|
1187
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1188
|
-
const errorMessage = new Promise<any>((resolve) => {
|
|
1189
|
-
client.on("message", (data, isBinary) => {
|
|
1190
|
-
if (isBinary) return;
|
|
1191
|
-
const message = JSON.parse(data.toString());
|
|
1192
|
-
if (message.type === "error") resolve(message);
|
|
1193
|
-
});
|
|
1194
|
-
});
|
|
1195
|
-
|
|
1196
|
-
client.send(Buffer.from([1, 2, 3]));
|
|
1197
|
-
|
|
1198
|
-
await expect(errorMessage).resolves.toMatchObject({
|
|
1199
|
-
type: "error",
|
|
1200
|
-
component: "transport",
|
|
1201
|
-
category: "invalid_input",
|
|
1202
|
-
message: "PCM16 audio payload must contain an even number of bytes",
|
|
1203
|
-
});
|
|
1204
|
-
expect(received).toEqual([]);
|
|
1205
|
-
|
|
1206
|
-
client.close();
|
|
1207
|
-
await server.close();
|
|
1208
|
-
});
|
|
1209
|
-
|
|
1210
|
-
it("rejects malformed base64 JSON audio without forwarding it", async () => {
|
|
1211
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1212
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
1213
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
1214
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
1215
|
-
});
|
|
1216
|
-
|
|
1217
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1218
|
-
port: 0,
|
|
1219
|
-
createSession: () => session,
|
|
1220
|
-
contextId: () => "turn-test",
|
|
1221
|
-
}));
|
|
1222
|
-
const address = server.address();
|
|
1223
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1224
|
-
|
|
1225
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1226
|
-
const errorMessage = new Promise<any>((resolve) => {
|
|
1227
|
-
client.on("message", (data, isBinary) => {
|
|
1228
|
-
if (isBinary) return;
|
|
1229
|
-
const message = JSON.parse(data.toString());
|
|
1230
|
-
if (message.type === "error") resolve(message);
|
|
1231
|
-
});
|
|
1232
|
-
});
|
|
1233
|
-
|
|
1234
|
-
client.send(JSON.stringify({
|
|
1235
|
-
type: "audio",
|
|
1236
|
-
contextId: "turn-bad-base64",
|
|
1237
|
-
sampleRateHz: 16000,
|
|
1238
|
-
audio: "not base64",
|
|
1239
|
-
}));
|
|
1240
|
-
|
|
1241
|
-
await expect(errorMessage).resolves.toMatchObject({
|
|
1242
|
-
type: "error",
|
|
1243
|
-
component: "transport",
|
|
1244
|
-
category: "invalid_input",
|
|
1245
|
-
message: "audio must be valid base64",
|
|
1246
|
-
});
|
|
1247
|
-
expect(received).toEqual([]);
|
|
1248
|
-
|
|
1249
|
-
client.close();
|
|
1250
|
-
await server.close();
|
|
1251
|
-
});
|
|
1252
|
-
|
|
1253
|
-
it("rejects JSON audio without sample-rate metadata before forwarding it", async () => {
|
|
1254
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1255
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
1256
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
1257
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
1258
|
-
});
|
|
1259
|
-
|
|
1260
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1261
|
-
port: 0,
|
|
1262
|
-
createSession: () => session,
|
|
1263
|
-
contextId: () => "turn-test",
|
|
1264
|
-
}));
|
|
1265
|
-
const address = server.address();
|
|
1266
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1267
|
-
|
|
1268
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1269
|
-
const errorMessage = new Promise<any>((resolve) => {
|
|
1270
|
-
client.on("message", (data, isBinary) => {
|
|
1271
|
-
if (isBinary) return;
|
|
1272
|
-
const message = JSON.parse(data.toString());
|
|
1273
|
-
if (message.type === "error") resolve(message);
|
|
1274
|
-
});
|
|
1275
|
-
});
|
|
1276
|
-
|
|
1277
|
-
client.send(JSON.stringify({
|
|
1278
|
-
type: "audio",
|
|
1279
|
-
contextId: "turn-missing-rate",
|
|
1280
|
-
audio: Buffer.from(new Int16Array([0, 1000]).buffer).toString("base64"),
|
|
1281
|
-
}));
|
|
1282
|
-
|
|
1283
|
-
await expect(errorMessage).resolves.toMatchObject({
|
|
1284
|
-
type: "error",
|
|
1285
|
-
component: "transport",
|
|
1286
|
-
category: "invalid_input",
|
|
1287
|
-
message: "JSON websocket audio sampleRateHz must be a positive integer",
|
|
1288
|
-
});
|
|
1289
|
-
expect(received).toEqual([]);
|
|
1290
|
-
|
|
1291
|
-
client.close();
|
|
1292
|
-
await server.close();
|
|
1293
|
-
});
|
|
1294
|
-
|
|
1295
|
-
it("closes oversized inbound websocket messages before decoding or forwarding", async () => {
|
|
1296
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1297
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
1298
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
1299
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
1300
|
-
});
|
|
1301
|
-
|
|
1302
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1303
|
-
port: 0,
|
|
1304
|
-
maxInboundMessageBytes: 4,
|
|
1305
|
-
createSession: () => session,
|
|
1306
|
-
contextId: () => "turn-test",
|
|
1307
|
-
}));
|
|
1308
|
-
const address = server.address();
|
|
1309
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1310
|
-
|
|
1311
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1312
|
-
const closed = new Promise<{ code: number; reason: string }>((resolve) => {
|
|
1313
|
-
client.once("close", (code, reason) => {
|
|
1314
|
-
resolve({ code, reason: reason.toString() });
|
|
1315
|
-
});
|
|
1316
|
-
});
|
|
1317
|
-
|
|
1318
|
-
client.send(Buffer.from([1, 2, 3, 4, 5, 6]));
|
|
1319
|
-
|
|
1320
|
-
await expect(closed).resolves.toEqual({
|
|
1321
|
-
code: 1009,
|
|
1322
|
-
reason: "websocket message too large",
|
|
1323
|
-
});
|
|
1324
|
-
expect(received).toEqual([]);
|
|
1325
|
-
|
|
1326
|
-
await server.close();
|
|
1327
|
-
});
|
|
1328
|
-
|
|
1329
|
-
it("accepts binary audio envelopes with turn and sample-rate metadata", async () => {
|
|
1330
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1331
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
1332
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
1333
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
1334
|
-
});
|
|
1335
|
-
|
|
1336
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1337
|
-
port: 0,
|
|
1338
|
-
inputSampleRateHz: 16000,
|
|
1339
|
-
createSession: () => session,
|
|
1340
|
-
contextId: () => "turn-test",
|
|
1341
|
-
}));
|
|
1342
|
-
const address = server.address();
|
|
1343
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1344
|
-
|
|
1345
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1346
|
-
const samples48k = new Int16Array([0, 3000, 6000, 9000, 12000, 15000]);
|
|
1347
|
-
|
|
1348
|
-
client.send(encodeTestBinaryAudioEnvelope({
|
|
1349
|
-
type: "audio",
|
|
1350
|
-
contextId: "turn-envelope",
|
|
1351
|
-
sampleRateHz: 48000,
|
|
1352
|
-
encoding: "pcm_s16le",
|
|
1353
|
-
channels: 1,
|
|
1354
|
-
byteLength: samples48k.byteLength,
|
|
1355
|
-
sequence: 7,
|
|
1356
|
-
}, new Uint8Array(samples48k.buffer)));
|
|
1357
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
1358
|
-
|
|
1359
|
-
expect(received).toEqual([
|
|
1360
|
-
expect.objectContaining({
|
|
1361
|
-
kind: "user.audio_received",
|
|
1362
|
-
contextId: "turn-envelope",
|
|
1363
|
-
}),
|
|
1364
|
-
]);
|
|
1365
|
-
// FIR-resampled output: same algorithm as JSON path gives the same weighted sums.
|
|
1366
|
-
expect(Buffer.from(received[0]!.audio)).toEqual(Buffer.from(new Int16Array([476, 10050]).buffer));
|
|
1367
|
-
|
|
1368
|
-
client.close();
|
|
1369
|
-
await server.close();
|
|
1370
|
-
});
|
|
1371
|
-
|
|
1372
|
-
it("routes odd-offset binary envelope PCM through the session VAD branch", async () => {
|
|
1373
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1374
|
-
const probe = new VadAlignmentProbe();
|
|
1375
|
-
session.registerPlugin("vad_alignment_probe", probe);
|
|
1376
|
-
const transportErrors: any[] = [];
|
|
1377
|
-
session.bus.on("pipeline.error", (pkt) => {
|
|
1378
|
-
transportErrors.push(pkt);
|
|
1379
|
-
});
|
|
1380
|
-
|
|
1381
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1382
|
-
port: 0,
|
|
1383
|
-
inputSampleRateHz: 16000,
|
|
1384
|
-
createSession: () => session,
|
|
1385
|
-
contextId: () => "turn-test",
|
|
1386
|
-
}));
|
|
1387
|
-
const address = server.address();
|
|
1388
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1389
|
-
|
|
1390
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1391
|
-
const audio = pcm16SamplesToBytes(new Int16Array([0, 32767, -32768, 16384]));
|
|
1392
|
-
const contexts = Array.from({ length: 8 }, (_, i) => `turn-envelope-vad-${"x".repeat(i)}`);
|
|
1393
|
-
for (const [index, contextId] of contexts.entries()) {
|
|
1394
|
-
client.send(encodeTestBinaryAudioEnvelope({
|
|
1395
|
-
type: "audio",
|
|
1396
|
-
contextId,
|
|
1397
|
-
sampleRateHz: 16000,
|
|
1398
|
-
encoding: "pcm_s16le",
|
|
1399
|
-
channels: 1,
|
|
1400
|
-
byteLength: audio.byteLength,
|
|
1401
|
-
sequence: index + 1,
|
|
1402
|
-
}, audio));
|
|
1403
|
-
}
|
|
1404
|
-
await waitForCondition(() => probe.observed.length === contexts.length, 500);
|
|
1405
|
-
|
|
1406
|
-
expect(transportErrors).toEqual([]);
|
|
1407
|
-
const oddOffsetPacket = probe.observed.find((entry) => entry.byteOffsetParity === 1);
|
|
1408
|
-
expect(probe.observed.map((entry) => entry.byteOffsetParity)).toContain(1);
|
|
1409
|
-
expect(oddOffsetPacket?.samples).toEqual([0, 32767, -32768, 16384]);
|
|
1410
|
-
|
|
1411
|
-
client.close();
|
|
1412
|
-
await server.close();
|
|
1413
|
-
});
|
|
1414
|
-
|
|
1415
|
-
it("rejects binary audio envelopes that change sample rate inside the same websocket context", async () => {
|
|
1416
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1417
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
1418
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
1419
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
1420
|
-
});
|
|
1421
|
-
|
|
1422
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1423
|
-
port: 0,
|
|
1424
|
-
inputSampleRateHz: 16000,
|
|
1425
|
-
createSession: () => session,
|
|
1426
|
-
contextId: () => "turn-test",
|
|
1427
|
-
}));
|
|
1428
|
-
const address = server.address();
|
|
1429
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1430
|
-
|
|
1431
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1432
|
-
const errorMessage = new Promise<any>((resolve) => {
|
|
1433
|
-
client.on("message", (data, isBinary) => {
|
|
1434
|
-
if (isBinary) return;
|
|
1435
|
-
const message = JSON.parse(data.toString());
|
|
1436
|
-
if (message.type === "error") resolve(message);
|
|
1437
|
-
});
|
|
1438
|
-
});
|
|
1439
|
-
|
|
1440
|
-
client.send(encodeTestBinaryAudioEnvelope({
|
|
1441
|
-
type: "audio",
|
|
1442
|
-
contextId: "turn-envelope-rate",
|
|
1443
|
-
sampleRateHz: 48000,
|
|
1444
|
-
encoding: "pcm_s16le",
|
|
1445
|
-
channels: 1,
|
|
1446
|
-
byteLength: 12,
|
|
1447
|
-
}, new Uint8Array(new Int16Array([0, 3000, 6000, 9000, 12000, 15000]).buffer)));
|
|
1448
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
1449
|
-
client.send(encodeTestBinaryAudioEnvelope({
|
|
1450
|
-
type: "audio",
|
|
1451
|
-
contextId: "turn-envelope-rate",
|
|
1452
|
-
sampleRateHz: 44100,
|
|
1453
|
-
encoding: "pcm_s16le",
|
|
1454
|
-
channels: 1,
|
|
1455
|
-
byteLength: 8,
|
|
1456
|
-
}, new Uint8Array(new Int16Array([0, 1000, 2000, 3000]).buffer)));
|
|
1457
|
-
|
|
1458
|
-
await expect(errorMessage).resolves.toMatchObject({
|
|
1459
|
-
type: "error",
|
|
1460
|
-
component: "transport",
|
|
1461
|
-
category: "invalid_input",
|
|
1462
|
-
message: "Websocket audio sampleRateHz changed within context turn-envelope-rate: 48000 -> 44100",
|
|
1463
|
-
});
|
|
1464
|
-
expect(received).toHaveLength(1);
|
|
1465
|
-
|
|
1466
|
-
client.close();
|
|
1467
|
-
await server.close();
|
|
1468
|
-
});
|
|
1469
|
-
|
|
1470
|
-
it("rejects regressing binary audio envelope sequence numbers", async () => {
|
|
1471
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1472
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
1473
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
1474
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
1475
|
-
});
|
|
1476
|
-
|
|
1477
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1478
|
-
port: 0,
|
|
1479
|
-
inputSampleRateHz: 16000,
|
|
1480
|
-
createSession: () => session,
|
|
1481
|
-
contextId: () => "turn-test",
|
|
1482
|
-
}));
|
|
1483
|
-
const address = server.address();
|
|
1484
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1485
|
-
|
|
1486
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1487
|
-
const errorMessage = new Promise<any>((resolve) => {
|
|
1488
|
-
client.on("message", (data, isBinary) => {
|
|
1489
|
-
if (isBinary) return;
|
|
1490
|
-
const message = JSON.parse(data.toString());
|
|
1491
|
-
if (message.type === "error") resolve(message);
|
|
1492
|
-
});
|
|
1493
|
-
});
|
|
1494
|
-
const audio = new Uint8Array(new Int16Array([0, 1000]).buffer);
|
|
1495
|
-
|
|
1496
|
-
client.send(encodeTestBinaryAudioEnvelope({
|
|
1497
|
-
type: "audio",
|
|
1498
|
-
contextId: "turn-envelope-sequence",
|
|
1499
|
-
sampleRateHz: 16000,
|
|
1500
|
-
encoding: "pcm_s16le",
|
|
1501
|
-
channels: 1,
|
|
1502
|
-
byteLength: audio.byteLength,
|
|
1503
|
-
sequence: 5,
|
|
1504
|
-
}, audio));
|
|
1505
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
1506
|
-
client.send(encodeTestBinaryAudioEnvelope({
|
|
1507
|
-
type: "audio",
|
|
1508
|
-
contextId: "turn-envelope-sequence",
|
|
1509
|
-
sampleRateHz: 16000,
|
|
1510
|
-
encoding: "pcm_s16le",
|
|
1511
|
-
channels: 1,
|
|
1512
|
-
byteLength: audio.byteLength,
|
|
1513
|
-
sequence: 4,
|
|
1514
|
-
}, audio));
|
|
1515
|
-
|
|
1516
|
-
await expect(errorMessage).resolves.toMatchObject({
|
|
1517
|
-
type: "error",
|
|
1518
|
-
component: "transport",
|
|
1519
|
-
category: "invalid_input",
|
|
1520
|
-
message: "Websocket audio sequence must increase monotonically: 5 -> 4",
|
|
1521
|
-
});
|
|
1522
|
-
expect(received).toHaveLength(1);
|
|
1523
|
-
|
|
1524
|
-
client.close();
|
|
1525
|
-
await server.close();
|
|
1526
|
-
});
|
|
1527
|
-
|
|
1528
|
-
it("rejects enveloped audio without valid sample-rate metadata", async () => {
|
|
1529
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1530
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
1531
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
1532
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
1533
|
-
});
|
|
1534
|
-
|
|
1535
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1536
|
-
port: 0,
|
|
1537
|
-
inputSampleRateHz: 16000,
|
|
1538
|
-
createSession: () => session,
|
|
1539
|
-
contextId: () => "turn-test",
|
|
1540
|
-
}));
|
|
1541
|
-
const address = server.address();
|
|
1542
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1543
|
-
|
|
1544
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1545
|
-
const errorMessage = new Promise<any>((resolve) => {
|
|
1546
|
-
client.on("message", (data, isBinary) => {
|
|
1547
|
-
if (isBinary) return;
|
|
1548
|
-
const message = JSON.parse(data.toString());
|
|
1549
|
-
if (message.type === "error") resolve(message);
|
|
1550
|
-
});
|
|
1551
|
-
});
|
|
1552
|
-
|
|
1553
|
-
client.send(encodeTestBinaryAudioEnvelope({
|
|
1554
|
-
type: "audio",
|
|
1555
|
-
contextId: "turn-envelope",
|
|
1556
|
-
encoding: "pcm_s16le",
|
|
1557
|
-
channels: 1,
|
|
1558
|
-
byteLength: 4,
|
|
1559
|
-
}, new Uint8Array([1, 2, 3, 4])));
|
|
1560
|
-
|
|
1561
|
-
await expect(errorMessage).resolves.toMatchObject({
|
|
1562
|
-
type: "error",
|
|
1563
|
-
component: "transport",
|
|
1564
|
-
category: "invalid_input",
|
|
1565
|
-
message: "Syrinx binary audio envelope sampleRateHz must be a positive integer",
|
|
1566
|
-
});
|
|
1567
|
-
expect(received).toEqual([]);
|
|
1568
|
-
|
|
1569
|
-
client.close();
|
|
1570
|
-
await server.close();
|
|
1571
|
-
});
|
|
1572
|
-
|
|
1573
|
-
it("rejects enveloped audio with inconsistent duration metadata", async () => {
|
|
1574
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1575
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
1576
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
1577
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
1578
|
-
});
|
|
1579
|
-
|
|
1580
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1581
|
-
port: 0,
|
|
1582
|
-
inputSampleRateHz: 16000,
|
|
1583
|
-
createSession: () => session,
|
|
1584
|
-
contextId: () => "turn-test",
|
|
1585
|
-
}));
|
|
1586
|
-
const address = server.address();
|
|
1587
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1588
|
-
|
|
1589
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1590
|
-
const errorMessage = new Promise<any>((resolve) => {
|
|
1591
|
-
client.on("message", (data, isBinary) => {
|
|
1592
|
-
if (isBinary) return;
|
|
1593
|
-
const message = JSON.parse(data.toString());
|
|
1594
|
-
if (message.type === "error") resolve(message);
|
|
1595
|
-
});
|
|
1596
|
-
});
|
|
1597
|
-
|
|
1598
|
-
client.send(encodeTestBinaryAudioEnvelope({
|
|
1599
|
-
type: "audio",
|
|
1600
|
-
contextId: "turn-envelope",
|
|
1601
|
-
sampleRateHz: 16000,
|
|
1602
|
-
encoding: "pcm_s16le",
|
|
1603
|
-
channels: 1,
|
|
1604
|
-
byteLength: 640,
|
|
1605
|
-
durationMs: 200,
|
|
1606
|
-
}, new Uint8Array(640)));
|
|
1607
|
-
|
|
1608
|
-
await expect(errorMessage).resolves.toMatchObject({
|
|
1609
|
-
type: "error",
|
|
1610
|
-
component: "transport",
|
|
1611
|
-
category: "invalid_input",
|
|
1612
|
-
message: "Syrinx binary audio envelope durationMs does not match payload and sampleRateHz",
|
|
1613
|
-
});
|
|
1614
|
-
expect(received).toEqual([]);
|
|
1615
|
-
|
|
1616
|
-
client.close();
|
|
1617
|
-
await server.close();
|
|
1618
|
-
});
|
|
1619
|
-
|
|
1620
|
-
it("sends PCM downlink after client reports pcm codec capability", async () => {
|
|
1621
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1622
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1623
|
-
port: 0,
|
|
1624
|
-
outputSampleRateHz: 16000,
|
|
1625
|
-
createSession: () => session,
|
|
1626
|
-
contextId: () => "turn-test",
|
|
1627
|
-
}));
|
|
1628
|
-
const address = server.address();
|
|
1629
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1630
|
-
|
|
1631
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1632
|
-
client.send(JSON.stringify({ type: "codec_capability", downlinkEncoding: "pcm_s16le" }));
|
|
1633
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
1634
|
-
|
|
1635
|
-
const binaryMessage = new Promise<Buffer>((resolve) => {
|
|
1636
|
-
client.on("message", (data, isBinary) => {
|
|
1637
|
-
if (isBinary) resolve(data as Buffer);
|
|
1638
|
-
});
|
|
1639
|
-
});
|
|
1640
|
-
|
|
1641
|
-
const pcmFrame = new Uint8Array(640);
|
|
1642
|
-
pcmFrame.set([1, 2, 3, 4]);
|
|
1643
|
-
session.bus.push(Route.Main, {
|
|
1644
|
-
kind: "tts.audio",
|
|
1645
|
-
contextId: "turn-tts",
|
|
1646
|
-
timestampMs: Date.now(),
|
|
1647
|
-
audio: pcmFrame,
|
|
1648
|
-
sampleRateHz: 16000,
|
|
1649
|
-
});
|
|
1650
|
-
|
|
1651
|
-
const envelope = decodeTestBinaryAudioEnvelope(await binaryMessage);
|
|
1652
|
-
expect(envelope.header).toMatchObject({
|
|
1653
|
-
encoding: "pcm_s16le",
|
|
1654
|
-
sampleRateHz: 16000,
|
|
1655
|
-
});
|
|
1656
|
-
expect(envelope.audio.byteLength).toBe(640);
|
|
1657
|
-
|
|
1658
|
-
// The PCM-only browser decoder rejects odd payloads and a durationMs that does not
|
|
1659
|
-
// match round(bytes/2/rate*1000); the PCM downlink must satisfy both or playback
|
|
1660
|
-
// throws "PCM16 payload must contain an even number of bytes" / "durationMs mismatch".
|
|
1661
|
-
expect(envelope.audio.byteLength % 2).toBe(0);
|
|
1662
|
-
const expectedDurationMs = Math.round((envelope.audio.byteLength / 2 / 16000) * 1000);
|
|
1663
|
-
expect(Math.abs((envelope.header.durationMs ?? -1) - expectedDurationMs)).toBeLessThanOrEqual(1);
|
|
1664
|
-
|
|
1665
|
-
client.close();
|
|
1666
|
-
await server.close();
|
|
1667
|
-
});
|
|
1668
|
-
|
|
1669
|
-
it("wraps outgoing assistant audio in the binary audio envelope by default", async () => {
|
|
1670
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1671
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1672
|
-
port: 0,
|
|
1673
|
-
outputSampleRateHz: 16000,
|
|
1674
|
-
createSession: () => session,
|
|
1675
|
-
contextId: () => "turn-test",
|
|
1676
|
-
}));
|
|
1677
|
-
const address = server.address();
|
|
1678
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1679
|
-
|
|
1680
|
-
const [client, ready] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1681
|
-
expect(ready.audio).toMatchObject({ binaryEnvelope: "syrinx.audio.v1" });
|
|
1682
|
-
|
|
1683
|
-
const binaryMessage = new Promise<Buffer>((resolve) => {
|
|
1684
|
-
client.on("message", (data, isBinary) => {
|
|
1685
|
-
if (isBinary) resolve(data as Buffer);
|
|
1686
|
-
});
|
|
1687
|
-
});
|
|
1688
|
-
|
|
1689
|
-
const pcmFrame = new Uint8Array(640);
|
|
1690
|
-
pcmFrame.set([1, 2, 3, 4]);
|
|
1691
|
-
session.bus.push(Route.Main, {
|
|
1692
|
-
kind: "tts.audio",
|
|
1693
|
-
contextId: "turn-tts",
|
|
1694
|
-
timestampMs: Date.now(),
|
|
1695
|
-
audio: pcmFrame,
|
|
1696
|
-
sampleRateHz: 16000,
|
|
1697
|
-
});
|
|
1698
|
-
|
|
1699
|
-
const envelope = decodeTestBinaryAudioEnvelope(await binaryMessage);
|
|
1700
|
-
// Opus frames are encoded at the codec rate (48 kHz) and MUST be labeled as such,
|
|
1701
|
-
// so the client decodes then downsamples 48→16 kHz. Labeling them 16 kHz made the
|
|
1702
|
-
// client skip the downsample and play assistant audio ~3× slow.
|
|
1703
|
-
expect(envelope.header).toMatchObject({
|
|
1704
|
-
type: "audio",
|
|
1705
|
-
contextId: "turn-tts",
|
|
1706
|
-
sequence: 1,
|
|
1707
|
-
sampleRateHz: 48000,
|
|
1708
|
-
encoding: "opus",
|
|
1709
|
-
channels: 1,
|
|
1710
|
-
});
|
|
1711
|
-
expect(envelope.audio.byteLength).toBeGreaterThan(0);
|
|
1712
|
-
expect(envelope.header.byteLength).toBe(envelope.audio.byteLength);
|
|
1713
|
-
|
|
1714
|
-
client.close();
|
|
1715
|
-
await server.close();
|
|
1716
|
-
});
|
|
1717
|
-
|
|
1718
|
-
it("can disable outgoing binary audio envelopes for raw PCM websocket clients", async () => {
|
|
1719
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1720
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1721
|
-
port: 0,
|
|
1722
|
-
outputSampleRateHz: 16000,
|
|
1723
|
-
binaryAudioEnvelope: false,
|
|
1724
|
-
createSession: () => session,
|
|
1725
|
-
contextId: () => "turn-test",
|
|
1726
|
-
}));
|
|
1727
|
-
const address = server.address();
|
|
1728
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1729
|
-
|
|
1730
|
-
const [client, ready] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1731
|
-
expect(ready.audio.binaryEnvelope).toBeUndefined();
|
|
1732
|
-
|
|
1733
|
-
const binaryMessage = new Promise<Buffer>((resolve) => {
|
|
1734
|
-
client.on("message", (data, isBinary) => {
|
|
1735
|
-
if (isBinary) resolve(data as Buffer);
|
|
1736
|
-
});
|
|
1737
|
-
});
|
|
1738
|
-
|
|
1739
|
-
session.bus.push(Route.Main, {
|
|
1740
|
-
kind: "tts.audio",
|
|
1741
|
-
contextId: "turn-tts",
|
|
1742
|
-
timestampMs: Date.now(),
|
|
1743
|
-
audio: new Uint8Array([1, 2, 3, 4]),
|
|
1744
|
-
sampleRateHz: 16000,
|
|
1745
|
-
});
|
|
1746
|
-
|
|
1747
|
-
expect(await binaryMessage).toEqual(Buffer.from([1, 2, 3, 4]));
|
|
1748
|
-
|
|
1749
|
-
client.close();
|
|
1750
|
-
await server.close();
|
|
1751
|
-
});
|
|
1752
|
-
|
|
1753
|
-
it("sends heartbeat pings so idle websocket peers are probed", async () => {
|
|
1754
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1755
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1756
|
-
port: 0,
|
|
1757
|
-
heartbeatIntervalMs: 10,
|
|
1758
|
-
createSession: () => session,
|
|
1759
|
-
contextId: () => "turn-test",
|
|
1760
|
-
}));
|
|
1761
|
-
const address = server.address();
|
|
1762
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1763
|
-
|
|
1764
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1765
|
-
const ping = new Promise<void>((resolve) => {
|
|
1766
|
-
client.once("ping", () => resolve());
|
|
1767
|
-
});
|
|
1768
|
-
|
|
1769
|
-
await expect(ping).resolves.toBeUndefined();
|
|
1770
|
-
|
|
1771
|
-
client.close();
|
|
1772
|
-
await server.close();
|
|
1773
|
-
});
|
|
1774
|
-
|
|
1775
|
-
it("closes browser websocket sessions that exceed maxSessionDurationMs", async () => {
|
|
1776
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1777
|
-
const closeSpy = vi.spyOn(session, "close");
|
|
1778
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1779
|
-
port: 0,
|
|
1780
|
-
maxSessionDurationMs: 10,
|
|
1781
|
-
createSession: () => session,
|
|
1782
|
-
contextId: () => "turn-test",
|
|
1783
|
-
}));
|
|
1784
|
-
const address = server.address();
|
|
1785
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1786
|
-
|
|
1787
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1788
|
-
const closed = new Promise<{ code: number; reason: string }>((resolve) => {
|
|
1789
|
-
client.once("close", (code, reason) => {
|
|
1790
|
-
resolve({ code, reason: reason.toString() });
|
|
1791
|
-
});
|
|
1792
|
-
});
|
|
1793
|
-
|
|
1794
|
-
await expect(closed).resolves.toEqual({
|
|
1795
|
-
code: 1000,
|
|
1796
|
-
reason: "websocket max session duration exceeded",
|
|
1797
|
-
});
|
|
1798
|
-
await vi.waitFor(() => {
|
|
1799
|
-
expect(closeSpy).toHaveBeenCalled();
|
|
1800
|
-
});
|
|
1801
|
-
|
|
1802
|
-
await server.close();
|
|
1803
|
-
});
|
|
1804
|
-
|
|
1805
|
-
it("closes slow websocket consumers before assistant audio buffers grow unbounded", async () => {
|
|
1806
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1807
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1808
|
-
port: 0,
|
|
1809
|
-
maxBufferedAmountBytes: 4096,
|
|
1810
|
-
createSession: () => session,
|
|
1811
|
-
contextId: () => "turn-test",
|
|
1812
|
-
}));
|
|
1813
|
-
const address = server.address();
|
|
1814
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1815
|
-
|
|
1816
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1817
|
-
const serverSocket = [...server.wsServer.clients][0];
|
|
1818
|
-
if (!serverSocket) throw new Error("Expected server websocket");
|
|
1819
|
-
Object.defineProperty(serverSocket, "bufferedAmount", { configurable: true, value: 4097 });
|
|
1820
|
-
|
|
1821
|
-
const closed = new Promise<{ code: number; reason: string }>((resolve) => {
|
|
1822
|
-
client.once("close", (code, reason) => {
|
|
1823
|
-
resolve({ code, reason: reason.toString() });
|
|
1824
|
-
});
|
|
1825
|
-
});
|
|
1826
|
-
session.bus.push(Route.Main, {
|
|
1827
|
-
kind: "tts.audio",
|
|
1828
|
-
contextId: "turn-tts",
|
|
1829
|
-
timestampMs: Date.now(),
|
|
1830
|
-
audio: pcm16SamplesToBytes(new Int16Array(640)),
|
|
1831
|
-
sampleRateHz: 16000,
|
|
1832
|
-
});
|
|
1833
|
-
session.bus.push(Route.Main, {
|
|
1834
|
-
kind: "tts.end",
|
|
1835
|
-
contextId: "turn-tts",
|
|
1836
|
-
timestampMs: Date.now(),
|
|
1837
|
-
});
|
|
1838
|
-
|
|
1839
|
-
await expect(closed).resolves.toEqual({
|
|
1840
|
-
code: 1013,
|
|
1841
|
-
reason: "websocket send buffer exceeded",
|
|
1842
|
-
});
|
|
1843
|
-
|
|
1844
|
-
await server.close();
|
|
1845
|
-
});
|
|
1846
|
-
|
|
1847
|
-
it("closes websocket consumers before sending one oversized assistant audio frame", async () => {
|
|
1848
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1849
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1850
|
-
port: 0,
|
|
1851
|
-
maxBufferedAmountBytes: 4096,
|
|
1852
|
-
outboundFrameDurationMs: 300,
|
|
1853
|
-
maxQueuedOutputAudioMs: 1000,
|
|
1854
|
-
browserOpusDownlink: false,
|
|
1855
|
-
createSession: () => session,
|
|
1856
|
-
contextId: () => "turn-test",
|
|
1857
|
-
}));
|
|
1858
|
-
const address = server.address();
|
|
1859
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1860
|
-
|
|
1861
|
-
const client = await openBrowserSocketReady(websocketUrl(address.port));
|
|
1862
|
-
let receivedBinary = false;
|
|
1863
|
-
client.on("message", (_data, isBinary) => {
|
|
1864
|
-
if (isBinary) receivedBinary = true;
|
|
1865
|
-
});
|
|
1866
|
-
let closeCode = -1;
|
|
1867
|
-
let closeReason = "";
|
|
1868
|
-
client.on("close", (code, reason) => {
|
|
1869
|
-
closeCode = code;
|
|
1870
|
-
closeReason = reason.toString();
|
|
1871
|
-
});
|
|
1872
|
-
|
|
1873
|
-
session.bus.push(Route.Main, {
|
|
1874
|
-
kind: "tts.audio",
|
|
1875
|
-
contextId: "turn-tts",
|
|
1876
|
-
timestampMs: Date.now(),
|
|
1877
|
-
audio: new Uint8Array(8192),
|
|
1878
|
-
sampleRateHz: 16000,
|
|
1879
|
-
});
|
|
1880
|
-
|
|
1881
|
-
await waitForCondition(() => client.readyState === WebSocket.CLOSED);
|
|
1882
|
-
expect({ code: closeCode, reason: closeReason }).toEqual({
|
|
1883
|
-
code: 1013,
|
|
1884
|
-
reason: "websocket send buffer exceeded",
|
|
1885
|
-
});
|
|
1886
|
-
expect(receivedBinary).toBe(false);
|
|
1887
|
-
|
|
1888
|
-
await server.close();
|
|
1889
|
-
});
|
|
1890
|
-
|
|
1891
|
-
it("records a send_after_close metric and does not throw when tts.audio arrives after socket closes", async () => {
|
|
1892
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1893
|
-
const metrics: ConversationMetricPacket[] = [];
|
|
1894
|
-
session.bus.on("metric.conversation", (pkt) => {
|
|
1895
|
-
metrics.push(pkt as ConversationMetricPacket);
|
|
1896
|
-
});
|
|
1897
|
-
|
|
1898
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1899
|
-
port: 0,
|
|
1900
|
-
createSession: () => session,
|
|
1901
|
-
contextId: () => "turn-test",
|
|
1902
|
-
}));
|
|
1903
|
-
const address = server.address();
|
|
1904
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1905
|
-
|
|
1906
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1907
|
-
const serverSocket = [...server.wsServer.clients][0]!;
|
|
1908
|
-
|
|
1909
|
-
// terminate() sets readyState to CLOSING synchronously; the 'close' event (and disposer cleanup) fires asynchronously
|
|
1910
|
-
serverSocket.terminate();
|
|
1911
|
-
|
|
1912
|
-
// Push tts.audio while the bus handler is still registered but socket is no longer OPEN — must not throw
|
|
1913
|
-
session.bus.push(Route.Main, {
|
|
1914
|
-
kind: "tts.audio",
|
|
1915
|
-
contextId: "turn-test",
|
|
1916
|
-
timestampMs: Date.now(),
|
|
1917
|
-
audio: new Uint8Array(32),
|
|
1918
|
-
sampleRateHz: 16000,
|
|
1919
|
-
});
|
|
1920
|
-
|
|
1921
|
-
// Allow the close event to propagate
|
|
1922
|
-
await new Promise<void>((resolve) => client.once("close", resolve));
|
|
1923
|
-
|
|
1924
|
-
expect(metrics.some((m) => m.name === "websocket.send_after_close")).toBe(true);
|
|
1925
|
-
|
|
1926
|
-
await server.close();
|
|
1927
|
-
});
|
|
1928
|
-
|
|
1929
|
-
it("decodes browser opus ingress into engine PCM16 and advertises supported codecs in ready", async () => {
|
|
1930
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1931
|
-
const received: UserAudioReceivedPacket[] = [];
|
|
1932
|
-
session.bus.on("user.audio_received", (pkt) => {
|
|
1933
|
-
received.push(pkt as UserAudioReceivedPacket);
|
|
1934
|
-
});
|
|
1935
|
-
|
|
1936
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1937
|
-
port: 0,
|
|
1938
|
-
createSession: () => session,
|
|
1939
|
-
contextId: () => "turn-opus",
|
|
1940
|
-
}));
|
|
1941
|
-
const address = server.address();
|
|
1942
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1943
|
-
|
|
1944
|
-
const [client, ready] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1945
|
-
expect(ready.audio).toMatchObject({
|
|
1946
|
-
encoding: "opus",
|
|
1947
|
-
supportedInputCodecs: ["pcm_s16le", "opus"],
|
|
1948
|
-
});
|
|
1949
|
-
|
|
1950
|
-
const pcm = new Int16Array(960);
|
|
1951
|
-
pcm[0] = 1000;
|
|
1952
|
-
pcm[3] = -1000;
|
|
1953
|
-
const encoder = new OpusEncoder({ channels: 1, sample_rate: 48000, application: "voip" });
|
|
1954
|
-
const opus = encoder.encode(pcm16SamplesToBytes(pcm));
|
|
1955
|
-
|
|
1956
|
-
client.send(encodeTestBinaryAudioEnvelope({
|
|
1957
|
-
type: "audio",
|
|
1958
|
-
contextId: "turn-opus",
|
|
1959
|
-
sampleRateHz: 48000,
|
|
1960
|
-
sequence: 1,
|
|
1961
|
-
encoding: "opus",
|
|
1962
|
-
channels: 1,
|
|
1963
|
-
byteLength: opus.byteLength,
|
|
1964
|
-
durationMs: BROWSER_OPUS_FRAME_DURATION_MS,
|
|
1965
|
-
}, opus));
|
|
1966
|
-
|
|
1967
|
-
await waitForCondition(() => received.length > 0, 2_000);
|
|
1968
|
-
expect(received[0]!.audio.byteLength).toBeGreaterThan(0);
|
|
1969
|
-
expect(received[0]!.audio.byteLength % 2).toBe(0);
|
|
1970
|
-
|
|
1971
|
-
client.close();
|
|
1972
|
-
await server.close();
|
|
1973
|
-
});
|
|
1974
|
-
|
|
1975
|
-
it("emits populated metrics after paced TTS playout completes", async () => {
|
|
1976
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
1977
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
1978
|
-
port: 0,
|
|
1979
|
-
outboundFrameDurationMs: 20,
|
|
1980
|
-
createSession: () => session,
|
|
1981
|
-
contextId: () => "turn-test",
|
|
1982
|
-
}));
|
|
1983
|
-
const address = server.address();
|
|
1984
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
1985
|
-
|
|
1986
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
1987
|
-
const metricsMessage = readJsonMatching(client, (message) =>
|
|
1988
|
-
(message as { type?: string }).type === "metrics",
|
|
1989
|
-
);
|
|
1990
|
-
|
|
1991
|
-
const speechEndMs = Date.now();
|
|
1992
|
-
session.bus.push(Route.Main, {
|
|
1993
|
-
kind: "vad.speech_ended",
|
|
1994
|
-
contextId: "metrics-turn",
|
|
1995
|
-
timestampMs: speechEndMs,
|
|
1996
|
-
});
|
|
1997
|
-
session.bus.push(Route.Main, {
|
|
1998
|
-
kind: "stt.result",
|
|
1999
|
-
contextId: "metrics-turn",
|
|
2000
|
-
timestampMs: speechEndMs + 200,
|
|
2001
|
-
text: "hello",
|
|
2002
|
-
confidence: 0.95,
|
|
2003
|
-
});
|
|
2004
|
-
session.bus.push(Route.Main, {
|
|
2005
|
-
kind: "llm.delta",
|
|
2006
|
-
contextId: "metrics-turn",
|
|
2007
|
-
timestampMs: speechEndMs + 500,
|
|
2008
|
-
text: "hi there",
|
|
2009
|
-
});
|
|
2010
|
-
|
|
2011
|
-
const pcmFrame = pcm16SamplesToBytes(new Int16Array(640).fill(1000));
|
|
2012
|
-
session.bus.push(Route.Main, {
|
|
2013
|
-
kind: "tts.audio",
|
|
2014
|
-
contextId: "metrics-turn",
|
|
2015
|
-
timestampMs: speechEndMs + 700,
|
|
2016
|
-
audio: pcmFrame,
|
|
2017
|
-
sampleRateHz: 16000,
|
|
2018
|
-
});
|
|
2019
|
-
session.bus.push(Route.Main, {
|
|
2020
|
-
kind: "tts.end",
|
|
2021
|
-
contextId: "metrics-turn",
|
|
2022
|
-
timestampMs: speechEndMs + 710,
|
|
2023
|
-
});
|
|
2024
|
-
|
|
2025
|
-
await expect(metricsMessage).resolves.toMatchObject({
|
|
2026
|
-
type: "metrics",
|
|
2027
|
-
turnId: "metrics-turn",
|
|
2028
|
-
correlationId: "metrics-turn",
|
|
2029
|
-
sttMs: 200,
|
|
2030
|
-
llmTTFTMs: 300,
|
|
2031
|
-
ttsTTFBMs: 200,
|
|
2032
|
-
});
|
|
2033
|
-
const metrics = await metricsMessage;
|
|
2034
|
-
expect(typeof (metrics as { e2eMs?: number }).e2eMs).toBe("number");
|
|
2035
|
-
expect((metrics as { firstAudioPlayedMs?: number }).firstAudioPlayedMs).toBeGreaterThan(0);
|
|
2036
|
-
expect((metrics as { lastAudioPlayedMs?: number }).lastAudioPlayedMs).toBeGreaterThan(0);
|
|
2037
|
-
|
|
2038
|
-
client.close();
|
|
2039
|
-
await server.close();
|
|
2040
|
-
});
|
|
2041
|
-
|
|
2042
|
-
it("notifies browser clients when the server commits the semantic turn", async () => {
|
|
2043
|
-
const session = new VoiceAgentSession({ plugins: {} });
|
|
2044
|
-
const server = registerServer(await createVoiceWebSocketServer({
|
|
2045
|
-
port: 0,
|
|
2046
|
-
createSession: () => session,
|
|
2047
|
-
contextId: () => "turn-test",
|
|
2048
|
-
}));
|
|
2049
|
-
const address = server.address();
|
|
2050
|
-
if (!address || typeof address === "string") throw new Error("Expected TCP address");
|
|
2051
|
-
|
|
2052
|
-
const [client] = await openBrowserClientAndReadReady(websocketUrl(address.port));
|
|
2053
|
-
const turnComplete = readJsonMatching(client, (message) =>
|
|
2054
|
-
(message as { type?: string }).type === "turn_complete",
|
|
2055
|
-
);
|
|
2056
|
-
|
|
2057
|
-
session.bus.push(Route.Main, {
|
|
2058
|
-
kind: "eos.turn_complete",
|
|
2059
|
-
contextId: "semantic-turn",
|
|
2060
|
-
timestampMs: Date.now(),
|
|
2061
|
-
text: "I need help choosing modules.",
|
|
2062
|
-
transcripts: [],
|
|
2063
|
-
});
|
|
2064
|
-
|
|
2065
|
-
await expect(turnComplete).resolves.toMatchObject({
|
|
2066
|
-
type: "turn_complete",
|
|
2067
|
-
turnId: "semantic-turn",
|
|
2068
|
-
transcript: "I need help choosing modules.",
|
|
2069
|
-
});
|
|
2070
|
-
|
|
2071
|
-
client.close();
|
|
2072
|
-
await server.close();
|
|
2073
|
-
});
|
|
2074
|
-
});
|