@kuralle-syrinx/realtime 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/README.md +152 -0
- package/package.json +22 -0
- package/src/base64.ts +16 -0
- package/src/edge-safety.test.ts +173 -0
- package/src/from-gemini-live.test.ts +197 -0
- package/src/from-gemini-live.ts +274 -0
- package/src/from-openai-realtime.test.ts +447 -0
- package/src/from-openai-realtime.ts +104 -0
- package/src/gemini-translate.test.ts +82 -0
- package/src/gemini-translate.ts +128 -0
- package/src/index.ts +17 -0
- package/src/openai-compatible-realtime.ts +373 -0
- package/src/realtime-adapter.ts +39 -0
- package/src/realtime-bridge.test.ts +653 -0
- package/src/realtime-bridge.ts +422 -0
- package/src/workers-seam.test.ts +143 -0
- package/tsconfig.json +21 -0
|
@@ -0,0 +1,653 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
|
|
3
|
+
import { afterEach, describe, expect, it } from "vitest";
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
PipelineBusImpl,
|
|
7
|
+
Route,
|
|
8
|
+
VoiceAgentSession,
|
|
9
|
+
type EndOfSpeechPacket,
|
|
10
|
+
type InterruptTtsPacket,
|
|
11
|
+
type LlmErrorPacket,
|
|
12
|
+
type LlmToolResultPacket,
|
|
13
|
+
type Reasoner,
|
|
14
|
+
type ReasonerMessage,
|
|
15
|
+
type RecordAssistantAudioPacket,
|
|
16
|
+
type SttResultPacket,
|
|
17
|
+
type TextToSpeechAudioPacket,
|
|
18
|
+
type TextToSpeechEndPacket,
|
|
19
|
+
type TurnChangePacket,
|
|
20
|
+
type VoicePacket,
|
|
21
|
+
} from "@kuralle-syrinx/core";
|
|
22
|
+
|
|
23
|
+
import type { RealtimeAdapter, RealtimeEvent } from "./realtime-adapter.js";
|
|
24
|
+
import { RealtimeBridge } from "./realtime-bridge.js";
|
|
25
|
+
|
|
26
|
+
class FakeRealtimeAdapter implements RealtimeAdapter {
|
|
27
|
+
readonly caps: RealtimeAdapter["caps"];
|
|
28
|
+
|
|
29
|
+
constructor(caps?: Partial<RealtimeAdapter["caps"]>) {
|
|
30
|
+
this.caps = {
|
|
31
|
+
inputSampleRateHz: 24_000,
|
|
32
|
+
outputSampleRateHz: 24_000,
|
|
33
|
+
supportsConcurrentToolAudio: true,
|
|
34
|
+
supportsTruncate: true,
|
|
35
|
+
emitsServerSpeechStarted: true,
|
|
36
|
+
...caps,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
private readonly queued: RealtimeEvent[] = [];
|
|
41
|
+
private readonly waiters: Array<(event: RealtimeEvent | null) => void> = [];
|
|
42
|
+
private closed = false;
|
|
43
|
+
readonly sentAudio: Uint8Array[] = [];
|
|
44
|
+
|
|
45
|
+
readonly events: AsyncIterable<RealtimeEvent> = {
|
|
46
|
+
[Symbol.asyncIterator]: () => ({
|
|
47
|
+
next: async (): Promise<IteratorResult<RealtimeEvent>> => {
|
|
48
|
+
if (this.queued.length > 0) {
|
|
49
|
+
return { value: this.queued.shift()!, done: false };
|
|
50
|
+
}
|
|
51
|
+
if (this.closed) return { value: undefined, done: true };
|
|
52
|
+
const event = await new Promise<RealtimeEvent | null>((resolve) => {
|
|
53
|
+
this.waiters.push(resolve);
|
|
54
|
+
});
|
|
55
|
+
if (event === null) return { value: undefined, done: true };
|
|
56
|
+
return { value: event, done: false };
|
|
57
|
+
},
|
|
58
|
+
}),
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
async open(_signal: AbortSignal): Promise<void> {}
|
|
62
|
+
|
|
63
|
+
sendAudio(pcm16: Uint8Array): void {
|
|
64
|
+
this.sentAudio.push(pcm16);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
readonly cancelCalls: number[] = [];
|
|
68
|
+
|
|
69
|
+
cancelResponse(audioEndMs: number): void {
|
|
70
|
+
this.cancelCalls.push(audioEndMs);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
readonly injectedToolResults: Array<{ toolId: string; text: string }> = [];
|
|
74
|
+
|
|
75
|
+
injectToolResult(toolId: string, text: string): void {
|
|
76
|
+
this.injectedToolResults.push({ toolId, text });
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async close(): Promise<void> {
|
|
80
|
+
this.closed = true;
|
|
81
|
+
for (const resolve of this.waiters.splice(0)) resolve(null);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
emit(event: RealtimeEvent): void {
|
|
85
|
+
const waiter = this.waiters.shift();
|
|
86
|
+
if (waiter) waiter(event);
|
|
87
|
+
else this.queued.push(event);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function pcmFromSamples(samples: readonly number[]): Uint8Array {
|
|
93
|
+
const pcm = Int16Array.from(samples);
|
|
94
|
+
return new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function frameSizedPcm24k(): Uint8Array {
|
|
98
|
+
return pcmFromSamples(Array.from({ length: 960 }, (_, i) => i));
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function frameDurationMs(frame: TextToSpeechAudioPacket): number {
|
|
102
|
+
return (frame.audio.byteLength / 2 / frame.sampleRateHz) * 1000;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async function waitForCondition(predicate: () => boolean, timeoutMs = 2000): Promise<void> {
|
|
106
|
+
const startedAt = Date.now();
|
|
107
|
+
while (Date.now() - startedAt < timeoutMs) {
|
|
108
|
+
if (predicate()) return;
|
|
109
|
+
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
110
|
+
}
|
|
111
|
+
throw new Error("Timed out waiting for condition");
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
describe("RealtimeBridge", () => {
|
|
115
|
+
const buses: PipelineBusImpl[] = [];
|
|
116
|
+
|
|
117
|
+
afterEach(() => {
|
|
118
|
+
for (const bus of buses.splice(0)) bus.stop();
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it("maps one turn to turn.change → tts.audio → eos.turn_complete → tts.end with one contextId", async () => {
|
|
122
|
+
const adapter = new FakeRealtimeAdapter();
|
|
123
|
+
const bridge = new RealtimeBridge(adapter);
|
|
124
|
+
const bus = new PipelineBusImpl();
|
|
125
|
+
buses.push(bus);
|
|
126
|
+
const packets: VoicePacket[] = [];
|
|
127
|
+
bus.on("turn.change", (pkt) => { packets.push(pkt as TurnChangePacket); });
|
|
128
|
+
bus.on("tts.audio", (pkt) => { packets.push(pkt as TextToSpeechAudioPacket); });
|
|
129
|
+
bus.on("eos.turn_complete", (pkt) => { packets.push(pkt as EndOfSpeechPacket); });
|
|
130
|
+
bus.on("tts.end", (pkt) => { packets.push(pkt as TextToSpeechEndPacket); });
|
|
131
|
+
bus.on("stt.result", (pkt) => { packets.push(pkt as SttResultPacket); });
|
|
132
|
+
|
|
133
|
+
const started = bus.start();
|
|
134
|
+
await bridge.initialize(bus, {});
|
|
135
|
+
|
|
136
|
+
bus.push(Route.Main, {
|
|
137
|
+
kind: "user.audio_received",
|
|
138
|
+
contextId: "transport-turn",
|
|
139
|
+
timestampMs: Date.now(),
|
|
140
|
+
audio: pcmFromSamples([100, 200, 300, 400]),
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
adapter.emit({ type: "response_started" });
|
|
144
|
+
adapter.emit({
|
|
145
|
+
type: "audio",
|
|
146
|
+
pcm16: pcmFromSamples(Array.from({ length: 960 }, (_, i) => i)),
|
|
147
|
+
sampleRateHz: 24_000,
|
|
148
|
+
});
|
|
149
|
+
adapter.emit({ type: "transcript", role: "user", text: "hello there", final: true });
|
|
150
|
+
adapter.emit({ type: "response_done" });
|
|
151
|
+
|
|
152
|
+
await waitForCondition(() => packets.some((p) => p.kind === "tts.end"));
|
|
153
|
+
|
|
154
|
+
const turnChanges = packets.filter((p): p is TurnChangePacket => p.kind === "turn.change");
|
|
155
|
+
const audio = packets.filter((p): p is TextToSpeechAudioPacket => p.kind === "tts.audio");
|
|
156
|
+
const turnComplete = packets.filter((p): p is EndOfSpeechPacket => p.kind === "eos.turn_complete");
|
|
157
|
+
const ends = packets.filter((p): p is TextToSpeechEndPacket => p.kind === "tts.end");
|
|
158
|
+
|
|
159
|
+
expect(turnChanges).toHaveLength(1);
|
|
160
|
+
expect(audio.length).toBeGreaterThan(0);
|
|
161
|
+
expect(turnComplete).toHaveLength(1);
|
|
162
|
+
expect(ends).toHaveLength(1);
|
|
163
|
+
|
|
164
|
+
const contextId = turnChanges[0]!.contextId;
|
|
165
|
+
expect(contextId.length).toBeGreaterThan(0);
|
|
166
|
+
expect(audio.every((frame) => frame.contextId === contextId)).toBe(true);
|
|
167
|
+
expect(turnComplete[0]!.contextId).toBe(contextId);
|
|
168
|
+
expect(ends[0]!.contextId).toBe(contextId);
|
|
169
|
+
expect(audio.every((frame) => frame.sampleRateHz === 16_000)).toBe(true);
|
|
170
|
+
expect(audio.every((frame) => frameDurationMs(frame) <= 20)).toBe(true);
|
|
171
|
+
expect(adapter.sentAudio.length).toBeGreaterThan(0);
|
|
172
|
+
|
|
173
|
+
await bridge.close();
|
|
174
|
+
bus.stop();
|
|
175
|
+
await started;
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
it("mints a fresh contextId for each response_started (R1)", async () => {
|
|
179
|
+
const adapter = new FakeRealtimeAdapter();
|
|
180
|
+
const bridge = new RealtimeBridge(adapter);
|
|
181
|
+
const bus = new PipelineBusImpl();
|
|
182
|
+
buses.push(bus);
|
|
183
|
+
const turnChanges: TurnChangePacket[] = [];
|
|
184
|
+
bus.on("turn.change", (pkt) => { turnChanges.push(pkt as TurnChangePacket); });
|
|
185
|
+
|
|
186
|
+
const started = bus.start();
|
|
187
|
+
await bridge.initialize(bus, {});
|
|
188
|
+
|
|
189
|
+
adapter.emit({ type: "response_started" });
|
|
190
|
+
adapter.emit({ type: "response_done" });
|
|
191
|
+
adapter.emit({ type: "response_started" });
|
|
192
|
+
adapter.emit({ type: "response_done" });
|
|
193
|
+
|
|
194
|
+
await waitForCondition(() => turnChanges.length >= 2);
|
|
195
|
+
|
|
196
|
+
expect(turnChanges[0]!.contextId).not.toBe(turnChanges[1]!.contextId);
|
|
197
|
+
expect(turnChanges[1]!.previousContextId).toBe(turnChanges[0]!.contextId);
|
|
198
|
+
|
|
199
|
+
await bridge.close();
|
|
200
|
+
bus.stop();
|
|
201
|
+
await started;
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
it("R-04: mints contextId via globalThis.crypto.randomUUID without node:crypto", async () => {
|
|
205
|
+
const adapter = new FakeRealtimeAdapter();
|
|
206
|
+
const bridge = new RealtimeBridge(adapter);
|
|
207
|
+
const bus = new PipelineBusImpl();
|
|
208
|
+
buses.push(bus);
|
|
209
|
+
const turnChanges: TurnChangePacket[] = [];
|
|
210
|
+
bus.on("turn.change", (pkt) => { turnChanges.push(pkt as TurnChangePacket); });
|
|
211
|
+
|
|
212
|
+
const started = bus.start();
|
|
213
|
+
await bridge.initialize(bus, {});
|
|
214
|
+
|
|
215
|
+
adapter.emit({ type: "response_started" });
|
|
216
|
+
await waitForCondition(() => turnChanges.length >= 1);
|
|
217
|
+
expect(turnChanges[0]!.contextId).toMatch(
|
|
218
|
+
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/,
|
|
219
|
+
);
|
|
220
|
+
|
|
221
|
+
await bridge.close();
|
|
222
|
+
bus.stop();
|
|
223
|
+
await started;
|
|
224
|
+
});
|
|
225
|
+
|
|
226
|
+
it("runDelegate injects finish.text when the Reasoner yields no deltas (R-08)", async () => {
|
|
227
|
+
const adapter = new FakeRealtimeAdapter();
|
|
228
|
+
const answerText = "Submit the Late Add Petition via the Student Relations portal.";
|
|
229
|
+
const reasoner: Reasoner = {
|
|
230
|
+
stream: () => (async function* () {
|
|
231
|
+
yield { type: "finish", reason: "stop", text: answerText };
|
|
232
|
+
})(),
|
|
233
|
+
};
|
|
234
|
+
const bridge = new RealtimeBridge(adapter, reasoner);
|
|
235
|
+
const bus = new PipelineBusImpl();
|
|
236
|
+
buses.push(bus);
|
|
237
|
+
const toolResults: LlmToolResultPacket[] = [];
|
|
238
|
+
bus.on("llm.tool_result", (pkt) => { toolResults.push(pkt as LlmToolResultPacket); });
|
|
239
|
+
|
|
240
|
+
const started = bus.start();
|
|
241
|
+
await bridge.initialize(bus, {});
|
|
242
|
+
|
|
243
|
+
adapter.emit({ type: "response_started" });
|
|
244
|
+
adapter.emit({
|
|
245
|
+
type: "tool_call",
|
|
246
|
+
toolId: "call_delegate_1",
|
|
247
|
+
toolName: "consult_knowledge",
|
|
248
|
+
args: { query: "Can I still add Biology 101?" },
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
await waitForCondition(() => adapter.injectedToolResults.length === 1);
|
|
252
|
+
|
|
253
|
+
expect(toolResults).toHaveLength(1);
|
|
254
|
+
expect(toolResults[0]!.result).toBe(answerText);
|
|
255
|
+
expect(adapter.injectedToolResults[0]).toEqual({ toolId: "call_delegate_1", text: answerText });
|
|
256
|
+
|
|
257
|
+
await bridge.close();
|
|
258
|
+
bus.stop();
|
|
259
|
+
await started;
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
it("R-08: finish reason length uses accumulated text", async () => {
|
|
263
|
+
const adapter = new FakeRealtimeAdapter();
|
|
264
|
+
const reasoner: Reasoner = {
|
|
265
|
+
stream: () => (async function* () {
|
|
266
|
+
yield { type: "text-delta", text: "Partial " };
|
|
267
|
+
yield { type: "finish", reason: "length", text: "Partial answer truncated" };
|
|
268
|
+
})(),
|
|
269
|
+
};
|
|
270
|
+
const bridge = new RealtimeBridge(adapter, reasoner);
|
|
271
|
+
const bus = new PipelineBusImpl();
|
|
272
|
+
buses.push(bus);
|
|
273
|
+
|
|
274
|
+
const started = bus.start();
|
|
275
|
+
await bridge.initialize(bus, {});
|
|
276
|
+
|
|
277
|
+
adapter.emit({ type: "response_started" });
|
|
278
|
+
adapter.emit({
|
|
279
|
+
type: "tool_call",
|
|
280
|
+
toolId: "call_len",
|
|
281
|
+
toolName: "consult_knowledge",
|
|
282
|
+
args: { query: "long query" },
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
await waitForCondition(() => adapter.injectedToolResults.length === 1);
|
|
286
|
+
expect(adapter.injectedToolResults[0]!.text).toBe("Partial ");
|
|
287
|
+
|
|
288
|
+
await bridge.close();
|
|
289
|
+
bus.stop();
|
|
290
|
+
await started;
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
it("R-08: recoverable error surfaces llm.error without injecting empty output", async () => {
|
|
294
|
+
const adapter = new FakeRealtimeAdapter();
|
|
295
|
+
const reasoner: Reasoner = {
|
|
296
|
+
stream: () => (async function* () {
|
|
297
|
+
yield { type: "error", cause: new Error("rate limited"), recoverable: true };
|
|
298
|
+
})(),
|
|
299
|
+
};
|
|
300
|
+
const bridge = new RealtimeBridge(adapter, reasoner);
|
|
301
|
+
const bus = new PipelineBusImpl();
|
|
302
|
+
buses.push(bus);
|
|
303
|
+
const errors: LlmErrorPacket[] = [];
|
|
304
|
+
bus.on("llm.error", (pkt) => { errors.push(pkt as LlmErrorPacket); });
|
|
305
|
+
|
|
306
|
+
const started = bus.start();
|
|
307
|
+
await bridge.initialize(bus, {});
|
|
308
|
+
|
|
309
|
+
adapter.emit({ type: "response_started" });
|
|
310
|
+
adapter.emit({
|
|
311
|
+
type: "tool_call",
|
|
312
|
+
toolId: "call_recover",
|
|
313
|
+
toolName: "consult_knowledge",
|
|
314
|
+
args: { query: "test" },
|
|
315
|
+
});
|
|
316
|
+
|
|
317
|
+
await waitForCondition(() => errors.length === 1);
|
|
318
|
+
expect(errors[0]!.isRecoverable).toBe(true);
|
|
319
|
+
expect(adapter.injectedToolResults).toHaveLength(0);
|
|
320
|
+
|
|
321
|
+
await bridge.close();
|
|
322
|
+
bus.stop();
|
|
323
|
+
await started;
|
|
324
|
+
});
|
|
325
|
+
|
|
326
|
+
it("R-08: nonrecoverable error emits llm.error without injecting output", async () => {
|
|
327
|
+
const adapter = new FakeRealtimeAdapter();
|
|
328
|
+
const reasoner: Reasoner = {
|
|
329
|
+
stream: () => (async function* () {
|
|
330
|
+
yield { type: "error", cause: new Error("auth failed"), recoverable: false };
|
|
331
|
+
})(),
|
|
332
|
+
};
|
|
333
|
+
const bridge = new RealtimeBridge(adapter, reasoner);
|
|
334
|
+
const bus = new PipelineBusImpl();
|
|
335
|
+
buses.push(bus);
|
|
336
|
+
const errors: LlmErrorPacket[] = [];
|
|
337
|
+
bus.on("llm.error", (pkt) => { errors.push(pkt as LlmErrorPacket); });
|
|
338
|
+
|
|
339
|
+
const started = bus.start();
|
|
340
|
+
await bridge.initialize(bus, {});
|
|
341
|
+
|
|
342
|
+
adapter.emit({ type: "response_started" });
|
|
343
|
+
adapter.emit({
|
|
344
|
+
type: "tool_call",
|
|
345
|
+
toolId: "call_fatal",
|
|
346
|
+
toolName: "consult_knowledge",
|
|
347
|
+
args: { query: "test" },
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
await waitForCondition(() => errors.length === 1);
|
|
351
|
+
expect(errors[0]!.isRecoverable).toBe(false);
|
|
352
|
+
expect(adapter.injectedToolResults).toHaveLength(0);
|
|
353
|
+
|
|
354
|
+
await bridge.close();
|
|
355
|
+
bus.stop();
|
|
356
|
+
await started;
|
|
357
|
+
});
|
|
358
|
+
|
|
359
|
+
it("runDelegate rejects suspended Reasoner without injecting an answer (R-08)", async () => {
|
|
360
|
+
const adapter = new FakeRealtimeAdapter();
|
|
361
|
+
const reasoner: Reasoner = {
|
|
362
|
+
stream: () => (async function* () {
|
|
363
|
+
yield { type: "suspended", runId: "run-suspended-1", payload: { step: "approval" } };
|
|
364
|
+
})(),
|
|
365
|
+
};
|
|
366
|
+
const bridge = new RealtimeBridge(adapter, reasoner);
|
|
367
|
+
const bus = new PipelineBusImpl();
|
|
368
|
+
buses.push(bus);
|
|
369
|
+
const toolResults: LlmToolResultPacket[] = [];
|
|
370
|
+
const errors: LlmErrorPacket[] = [];
|
|
371
|
+
bus.on("llm.tool_result", (pkt) => { toolResults.push(pkt as LlmToolResultPacket); });
|
|
372
|
+
bus.on("llm.error", (pkt) => { errors.push(pkt as LlmErrorPacket); });
|
|
373
|
+
|
|
374
|
+
const started = bus.start();
|
|
375
|
+
await bridge.initialize(bus, {});
|
|
376
|
+
|
|
377
|
+
adapter.emit({ type: "response_started" });
|
|
378
|
+
adapter.emit({
|
|
379
|
+
type: "tool_call",
|
|
380
|
+
toolId: "call_delegate_2",
|
|
381
|
+
toolName: "consult_knowledge",
|
|
382
|
+
args: { query: "Need advisor approval" },
|
|
383
|
+
});
|
|
384
|
+
|
|
385
|
+
await waitForCondition(() => errors.length === 1);
|
|
386
|
+
|
|
387
|
+
expect(adapter.injectedToolResults).toHaveLength(0);
|
|
388
|
+
expect(toolResults).toHaveLength(0);
|
|
389
|
+
|
|
390
|
+
await bridge.close();
|
|
391
|
+
bus.stop();
|
|
392
|
+
await started;
|
|
393
|
+
});
|
|
394
|
+
|
|
395
|
+
it("R-07: mismatched tool_call name emits recoverable llm.error instead of silent ignore", async () => {
|
|
396
|
+
const adapter = new FakeRealtimeAdapter();
|
|
397
|
+
const reasoner: Reasoner = {
|
|
398
|
+
stream: () => (async function* () {
|
|
399
|
+
yield { type: "finish", reason: "stop", text: "never called" };
|
|
400
|
+
})(),
|
|
401
|
+
};
|
|
402
|
+
const bridge = new RealtimeBridge(adapter, reasoner, "consult_knowledge");
|
|
403
|
+
const bus = new PipelineBusImpl();
|
|
404
|
+
buses.push(bus);
|
|
405
|
+
const errors: LlmErrorPacket[] = [];
|
|
406
|
+
bus.on("llm.error", (pkt) => { errors.push(pkt as LlmErrorPacket); });
|
|
407
|
+
|
|
408
|
+
const started = bus.start();
|
|
409
|
+
await bridge.initialize(bus, {});
|
|
410
|
+
|
|
411
|
+
adapter.emit({ type: "response_started" });
|
|
412
|
+
adapter.emit({
|
|
413
|
+
type: "tool_call",
|
|
414
|
+
toolId: "call_wrong",
|
|
415
|
+
toolName: "wrong_tool",
|
|
416
|
+
args: { query: "test" },
|
|
417
|
+
});
|
|
418
|
+
|
|
419
|
+
await waitForCondition(() => errors.length === 1);
|
|
420
|
+
expect(errors[0]!.isRecoverable).toBe(true);
|
|
421
|
+
expect(errors[0]!.cause.message).toContain("wrong_tool");
|
|
422
|
+
expect(adapter.injectedToolResults).toHaveLength(0);
|
|
423
|
+
|
|
424
|
+
await bridge.close();
|
|
425
|
+
bus.stop();
|
|
426
|
+
await started;
|
|
427
|
+
});
|
|
428
|
+
|
|
429
|
+
it("R-09: contextProvider messages are passed to reasoner.stream", async () => {
|
|
430
|
+
const adapter = new FakeRealtimeAdapter();
|
|
431
|
+
const prior: ReasonerMessage[] = [
|
|
432
|
+
{ role: "user", content: "prior question" },
|
|
433
|
+
{ role: "assistant", content: "prior answer" },
|
|
434
|
+
];
|
|
435
|
+
let receivedMessages: readonly ReasonerMessage[] | undefined;
|
|
436
|
+
const reasoner: Reasoner = {
|
|
437
|
+
stream: (turn) => {
|
|
438
|
+
receivedMessages = turn.messages;
|
|
439
|
+
return (async function* () {
|
|
440
|
+
yield { type: "finish", reason: "stop", text: "with context" };
|
|
441
|
+
})();
|
|
442
|
+
},
|
|
443
|
+
};
|
|
444
|
+
const bridge = new RealtimeBridge(adapter, reasoner, "consult_knowledge", {
|
|
445
|
+
contextProvider: () => prior,
|
|
446
|
+
});
|
|
447
|
+
const bus = new PipelineBusImpl();
|
|
448
|
+
buses.push(bus);
|
|
449
|
+
|
|
450
|
+
const started = bus.start();
|
|
451
|
+
await bridge.initialize(bus, {});
|
|
452
|
+
|
|
453
|
+
adapter.emit({ type: "response_started" });
|
|
454
|
+
adapter.emit({
|
|
455
|
+
type: "tool_call",
|
|
456
|
+
toolId: "call_ctx",
|
|
457
|
+
toolName: "consult_knowledge",
|
|
458
|
+
args: { query: "follow up" },
|
|
459
|
+
});
|
|
460
|
+
|
|
461
|
+
await waitForCondition(() => adapter.injectedToolResults.length === 1);
|
|
462
|
+
expect(receivedMessages).toEqual(prior);
|
|
463
|
+
|
|
464
|
+
await bridge.close();
|
|
465
|
+
bus.stop();
|
|
466
|
+
await started;
|
|
467
|
+
});
|
|
468
|
+
|
|
469
|
+
it("R-12: coalesces tiny audio deltas and never emits odd-length tts.audio", async () => {
|
|
470
|
+
const adapter = new FakeRealtimeAdapter();
|
|
471
|
+
const bridge = new RealtimeBridge(adapter);
|
|
472
|
+
const bus = new PipelineBusImpl();
|
|
473
|
+
buses.push(bus);
|
|
474
|
+
const audio: TextToSpeechAudioPacket[] = [];
|
|
475
|
+
bus.on("tts.audio", (pkt) => { audio.push(pkt as TextToSpeechAudioPacket); });
|
|
476
|
+
|
|
477
|
+
const started = bus.start();
|
|
478
|
+
await bridge.initialize(bus, {});
|
|
479
|
+
|
|
480
|
+
adapter.emit({ type: "response_started" });
|
|
481
|
+
adapter.emit({ type: "audio", pcm16: new Uint8Array([1, 2]), sampleRateHz: 24_000 });
|
|
482
|
+
adapter.emit({ type: "audio", pcm16: new Uint8Array([3, 4]), sampleRateHz: 24_000 });
|
|
483
|
+
expect(audio).toHaveLength(0);
|
|
484
|
+
|
|
485
|
+
adapter.emit({ type: "audio", pcm16: new Uint8Array([5]), sampleRateHz: 24_000 });
|
|
486
|
+
expect(audio).toHaveLength(0);
|
|
487
|
+
|
|
488
|
+
adapter.emit({ type: "response_done" });
|
|
489
|
+
await waitForCondition(() => audio.length > 0);
|
|
490
|
+
|
|
491
|
+
expect(audio.every((frame) => frame.audio.byteLength % 2 === 0)).toBe(true);
|
|
492
|
+
expect(audio.every((frame) => frame.audio.byteLength <= 640 || frameDurationMs(frame) <= 20)).toBe(true);
|
|
493
|
+
|
|
494
|
+
await bridge.close();
|
|
495
|
+
bus.stop();
|
|
496
|
+
await started;
|
|
497
|
+
});
|
|
498
|
+
|
|
499
|
+
it("R-13: adapter event handler throw emits llm.error; adapter close exits quietly", async () => {
|
|
500
|
+
const adapter = new FakeRealtimeAdapter();
|
|
501
|
+
const bridge = new RealtimeBridge(adapter);
|
|
502
|
+
const bus = new PipelineBusImpl();
|
|
503
|
+
buses.push(bus);
|
|
504
|
+
const errors: LlmErrorPacket[] = [];
|
|
505
|
+
bus.on("llm.error", (pkt) => { errors.push(pkt as LlmErrorPacket); });
|
|
506
|
+
|
|
507
|
+
const started = bus.start();
|
|
508
|
+
await bridge.initialize(bus, {});
|
|
509
|
+
|
|
510
|
+
const originalRandomUUID = globalThis.crypto.randomUUID.bind(globalThis.crypto);
|
|
511
|
+
globalThis.crypto.randomUUID = () => {
|
|
512
|
+
throw new Error("handler boom");
|
|
513
|
+
};
|
|
514
|
+
try {
|
|
515
|
+
adapter.emit({ type: "response_started" });
|
|
516
|
+
await waitForCondition(() => errors.length === 1);
|
|
517
|
+
expect(errors[0]!.isRecoverable).toBe(false);
|
|
518
|
+
expect(errors[0]!.cause.message).toBe("handler boom");
|
|
519
|
+
} finally {
|
|
520
|
+
globalThis.crypto.randomUUID = originalRandomUUID;
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
const errorsBeforeClose = errors.length;
|
|
524
|
+
await bridge.close();
|
|
525
|
+
expect(errors.length).toBe(errorsBeforeClose);
|
|
526
|
+
|
|
527
|
+
bus.stop();
|
|
528
|
+
await started;
|
|
529
|
+
});
|
|
530
|
+
|
|
531
|
+
it("does not emit interrupt.detected when emitsServerSpeechStarted is false", async () => {
|
|
532
|
+
const adapter = new FakeRealtimeAdapter({ emitsServerSpeechStarted: false });
|
|
533
|
+
const bridge = new RealtimeBridge(adapter);
|
|
534
|
+
const bus = new PipelineBusImpl();
|
|
535
|
+
buses.push(bus);
|
|
536
|
+
const interrupts: Array<{ kind: string }> = [];
|
|
537
|
+
bus.on("interrupt.detected", (pkt) => { interrupts.push(pkt as { kind: string }); });
|
|
538
|
+
|
|
539
|
+
const started = bus.start();
|
|
540
|
+
await bridge.initialize(bus, {});
|
|
541
|
+
|
|
542
|
+
adapter.emit({ type: "response_started" });
|
|
543
|
+
adapter.emit({ type: "speech_started" });
|
|
544
|
+
|
|
545
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
546
|
+
expect(interrupts).toHaveLength(0);
|
|
547
|
+
|
|
548
|
+
await bridge.close();
|
|
549
|
+
bus.stop();
|
|
550
|
+
await started;
|
|
551
|
+
});
|
|
552
|
+
|
|
553
|
+
it("double-barge-in: second turn tts.audio is not dropped after barge-in (R1, R3)", async () => {
|
|
554
|
+
const adapter = new FakeRealtimeAdapter();
|
|
555
|
+
let delegateSignal: AbortSignal | undefined;
|
|
556
|
+
const reasoner: Reasoner = {
|
|
557
|
+
stream: ({ signal }) => {
|
|
558
|
+
delegateSignal = signal;
|
|
559
|
+
return (async function* () {
|
|
560
|
+
await new Promise((resolve) => setTimeout(resolve, 60_000));
|
|
561
|
+
yield { type: "finish", reason: "stop", text: "never" };
|
|
562
|
+
})();
|
|
563
|
+
},
|
|
564
|
+
};
|
|
565
|
+
const bridge = new RealtimeBridge(adapter, reasoner);
|
|
566
|
+
const session = new VoiceAgentSession({
|
|
567
|
+
plugins: { realtime: {} },
|
|
568
|
+
endpointingOwner: "timer",
|
|
569
|
+
minInterruptionMs: 0,
|
|
570
|
+
});
|
|
571
|
+
session.registerPlugin("realtime", bridge);
|
|
572
|
+
|
|
573
|
+
const turnChanges: TurnChangePacket[] = [];
|
|
574
|
+
const recorded: RecordAssistantAudioPacket[] = [];
|
|
575
|
+
const interruptTts: InterruptTtsPacket[] = [];
|
|
576
|
+
|
|
577
|
+
session.bus.on("turn.change", (pkt) => { turnChanges.push(pkt as TurnChangePacket); });
|
|
578
|
+
session.bus.on("record.assistant_audio", (pkt) => {
|
|
579
|
+
const p = pkt as RecordAssistantAudioPacket;
|
|
580
|
+
if (!p.truncate) recorded.push(p);
|
|
581
|
+
});
|
|
582
|
+
session.bus.on("interrupt.tts", (pkt) => { interruptTts.push(pkt as InterruptTtsPacket); });
|
|
583
|
+
|
|
584
|
+
await session.start();
|
|
585
|
+
|
|
586
|
+
adapter.emit({ type: "response_started" });
|
|
587
|
+
await waitForCondition(() => turnChanges.length >= 1);
|
|
588
|
+
const contextA = turnChanges[0]!.contextId;
|
|
589
|
+
|
|
590
|
+
adapter.emit({
|
|
591
|
+
type: "audio",
|
|
592
|
+
pcm16: pcmFromSamples([100, 200, 300, 400]),
|
|
593
|
+
sampleRateHz: 24_000,
|
|
594
|
+
});
|
|
595
|
+
adapter.emit({ type: "speech_started" });
|
|
596
|
+
await waitForCondition(() => interruptTts.length >= 1);
|
|
597
|
+
expect(adapter.cancelCalls.length).toBeGreaterThanOrEqual(1);
|
|
598
|
+
expect(interruptTts[0]!.contextId).toBe(contextA);
|
|
599
|
+
|
|
600
|
+
adapter.emit({ type: "response_done" });
|
|
601
|
+
|
|
602
|
+
adapter.emit({ type: "response_started" });
|
|
603
|
+
await waitForCondition(() => turnChanges.length >= 2);
|
|
604
|
+
const contextB = turnChanges[1]!.contextId;
|
|
605
|
+
expect(contextB).not.toBe(contextA);
|
|
606
|
+
|
|
607
|
+
adapter.emit({
|
|
608
|
+
type: "audio",
|
|
609
|
+
pcm16: frameSizedPcm24k(),
|
|
610
|
+
sampleRateHz: 24_000,
|
|
611
|
+
});
|
|
612
|
+
await waitForCondition(() => recorded.some((p) => p.contextId === contextB));
|
|
613
|
+
|
|
614
|
+
adapter.emit({ type: "speech_started" });
|
|
615
|
+
await waitForCondition(() => interruptTts.length >= 2);
|
|
616
|
+
expect(adapter.cancelCalls.length).toBeGreaterThanOrEqual(2);
|
|
617
|
+
|
|
618
|
+
adapter.emit({ type: "response_done" });
|
|
619
|
+
|
|
620
|
+
adapter.emit({ type: "response_started" });
|
|
621
|
+
await waitForCondition(() => turnChanges.length >= 3);
|
|
622
|
+
const contextC = turnChanges[2]!.contextId;
|
|
623
|
+
|
|
624
|
+
adapter.emit({
|
|
625
|
+
type: "audio",
|
|
626
|
+
pcm16: frameSizedPcm24k(),
|
|
627
|
+
sampleRateHz: 24_000,
|
|
628
|
+
});
|
|
629
|
+
await waitForCondition(() => recorded.some((p) => p.contextId === contextC));
|
|
630
|
+
|
|
631
|
+
expect(recorded.filter((p) => p.contextId === contextB).length).toBeGreaterThan(0);
|
|
632
|
+
expect(recorded.filter((p) => p.contextId === contextC).length).toBeGreaterThan(0);
|
|
633
|
+
|
|
634
|
+
adapter.emit({ type: "response_started" });
|
|
635
|
+
await waitForCondition(() => turnChanges.length >= 4);
|
|
636
|
+
const contextDelegate = turnChanges[3]!.contextId;
|
|
637
|
+
adapter.emit({
|
|
638
|
+
type: "tool_call",
|
|
639
|
+
toolId: "call_barge_delegate",
|
|
640
|
+
toolName: "consult_knowledge",
|
|
641
|
+
args: { query: "late add policy" },
|
|
642
|
+
});
|
|
643
|
+
await waitForCondition(() => delegateSignal !== undefined);
|
|
644
|
+
session.bus.push(Route.Critical, {
|
|
645
|
+
kind: "interrupt.tts",
|
|
646
|
+
contextId: contextDelegate,
|
|
647
|
+
timestampMs: Date.now(),
|
|
648
|
+
});
|
|
649
|
+
await waitForCondition(() => delegateSignal!.aborted);
|
|
650
|
+
|
|
651
|
+
await session.close();
|
|
652
|
+
});
|
|
653
|
+
});
|