@kuralle-syrinx/realtime 4.0.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +24 -4
- package/src/from-openai-realtime.test.ts +1 -0
- package/src/from-openai-realtime.ts +1 -0
- package/src/openai-compatible-realtime.test.ts +188 -0
- package/src/openai-compatible-realtime.ts +29 -0
- package/src/realtime-adapter.ts +17 -0
- package/src/realtime-bridge.test.ts +211 -0
- package/src/realtime-bridge.ts +73 -3
package/package.json
CHANGED
|
@@ -1,15 +1,35 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kuralle-syrinx/realtime",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.2.0",
|
|
4
4
|
"private": false,
|
|
5
|
-
"
|
|
5
|
+
"description": "Realtime speech-to-speech bridge for Syrinx — OpenAI Realtime, Gemini Live, and openai-compatible adapters; the Responder-Thinker delegate seam",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"voice",
|
|
8
|
+
"voice-agent",
|
|
9
|
+
"speech",
|
|
10
|
+
"syrinx",
|
|
11
|
+
"realtime",
|
|
12
|
+
"speech-to-speech",
|
|
13
|
+
"openai-realtime",
|
|
14
|
+
"gemini-live"
|
|
15
|
+
],
|
|
6
16
|
"license": "MIT",
|
|
17
|
+
"homepage": "https://github.com/kuralle/syrinx#readme",
|
|
18
|
+
"repository": {
|
|
19
|
+
"type": "git",
|
|
20
|
+
"url": "git+https://github.com/kuralle/syrinx.git",
|
|
21
|
+
"directory": "packages/realtime"
|
|
22
|
+
},
|
|
23
|
+
"bugs": {
|
|
24
|
+
"url": "https://github.com/kuralle/syrinx/issues"
|
|
25
|
+
},
|
|
26
|
+
"type": "module",
|
|
7
27
|
"main": "./src/index.ts",
|
|
8
28
|
"types": "./src/index.ts",
|
|
9
29
|
"dependencies": {
|
|
10
30
|
"@google/genai": "^2.8.0",
|
|
11
|
-
"@kuralle-syrinx/core": "4.
|
|
12
|
-
"@kuralle-syrinx/ws": "4.
|
|
31
|
+
"@kuralle-syrinx/core": "4.2.0",
|
|
32
|
+
"@kuralle-syrinx/ws": "4.2.0"
|
|
13
33
|
},
|
|
14
34
|
"devDependencies": {
|
|
15
35
|
"typescript": "^5.7.0",
|
|
@@ -60,6 +60,7 @@ export function fromOpenAIRealtime(opts: OpenAIRealtimeOptions): RealtimeAdapter
|
|
|
60
60
|
supportsConcurrentToolAudio: true,
|
|
61
61
|
supportsTruncate: true,
|
|
62
62
|
emitsServerSpeechStarted: true,
|
|
63
|
+
supportsTextOnlyModality: opts.modalities?.length === 1 && opts.modalities[0] === "text",
|
|
63
64
|
},
|
|
64
65
|
buildSessionUpdate: () => {
|
|
65
66
|
const inputAudio: Record<string, unknown> = {
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
|
|
3
|
+
import { describe, expect, it } from "vitest";
|
|
4
|
+
|
|
5
|
+
import type { ManagedSocket, SocketData, SocketFactory } from "@kuralle-syrinx/ws";
|
|
6
|
+
|
|
7
|
+
import { createOpenAiCompatibleRealtimeAdapter } from "./openai-compatible-realtime.js";
|
|
8
|
+
import type { RealtimeEvent } from "./realtime-adapter.js";
|
|
9
|
+
|
|
10
|
+
interface MockSocketHarness {
|
|
11
|
+
readonly factory: SocketFactory;
|
|
12
|
+
readonly sent: string[];
|
|
13
|
+
inject(msg: Record<string, unknown>): void;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function createMockSocketHarness(): MockSocketHarness {
|
|
17
|
+
const sent: string[] = [];
|
|
18
|
+
let messageHandler: ((data: SocketData, isBinary: boolean) => void) | null = null;
|
|
19
|
+
|
|
20
|
+
const socket: ManagedSocket = {
|
|
21
|
+
get isOpen() {
|
|
22
|
+
return true;
|
|
23
|
+
},
|
|
24
|
+
send: (data: SocketData) => {
|
|
25
|
+
sent.push(typeof data === "string" ? data : "");
|
|
26
|
+
},
|
|
27
|
+
keepAlivePing: () => {},
|
|
28
|
+
verify: async () => true,
|
|
29
|
+
dispose: () => {},
|
|
30
|
+
onOpen: (handler) => {
|
|
31
|
+
queueMicrotask(() => handler());
|
|
32
|
+
},
|
|
33
|
+
onMessage: (handler) => {
|
|
34
|
+
messageHandler = handler;
|
|
35
|
+
},
|
|
36
|
+
onClose: () => {},
|
|
37
|
+
onError: () => {},
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
return {
|
|
41
|
+
factory: () => socket,
|
|
42
|
+
sent,
|
|
43
|
+
inject: (msg) => messageHandler?.(JSON.stringify(msg), false),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async function collectEvents(
|
|
48
|
+
events: AsyncIterable<RealtimeEvent>,
|
|
49
|
+
max = 8,
|
|
50
|
+
): Promise<RealtimeEvent[]> {
|
|
51
|
+
const out: RealtimeEvent[] = [];
|
|
52
|
+
for await (const event of events) {
|
|
53
|
+
out.push(event);
|
|
54
|
+
if (out.length >= max) break;
|
|
55
|
+
}
|
|
56
|
+
return out;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async function waitFor(predicate: () => boolean, timeoutMs = 2000): Promise<void> {
|
|
60
|
+
const startedAt = Date.now();
|
|
61
|
+
while (Date.now() - startedAt < timeoutMs) {
|
|
62
|
+
if (predicate()) return;
|
|
63
|
+
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
64
|
+
}
|
|
65
|
+
throw new Error("Timed out waiting for condition");
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function createAdapter(mock: MockSocketHarness) {
|
|
69
|
+
return createOpenAiCompatibleRealtimeAdapter({
|
|
70
|
+
apiKey: "test-key",
|
|
71
|
+
socketFactory: mock.factory,
|
|
72
|
+
defaultModel: "gpt-realtime-2",
|
|
73
|
+
url: () => "wss://example.test/realtime?model=gpt-realtime-2",
|
|
74
|
+
caps: {
|
|
75
|
+
inputSampleRateHz: 24_000,
|
|
76
|
+
outputSampleRateHz: 24_000,
|
|
77
|
+
supportsConcurrentToolAudio: true,
|
|
78
|
+
supportsTruncate: true,
|
|
79
|
+
emitsServerSpeechStarted: true,
|
|
80
|
+
supportsTextOnlyModality: true,
|
|
81
|
+
},
|
|
82
|
+
buildSessionUpdate: () => ({
|
|
83
|
+
type: "realtime",
|
|
84
|
+
model: "gpt-realtime-2",
|
|
85
|
+
output_modalities: ["text"],
|
|
86
|
+
}),
|
|
87
|
+
supportsTruncate: true,
|
|
88
|
+
defaultErrorMessage: "OpenAI Realtime error",
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
describe("createOpenAiCompatibleRealtimeAdapter text-only output", () => {
|
|
93
|
+
it("surfaces response.output_text.delta as assistant transcript (final=false)", async () => {
|
|
94
|
+
const mock = createMockSocketHarness();
|
|
95
|
+
const adapter = createAdapter(mock);
|
|
96
|
+
|
|
97
|
+
const eventsTask = collectEvents(adapter.events, 2);
|
|
98
|
+
const openTask = adapter.open(new AbortController().signal);
|
|
99
|
+
await waitFor(() => mock.sent.length > 0);
|
|
100
|
+
mock.inject({ type: "session.updated" });
|
|
101
|
+
await openTask;
|
|
102
|
+
|
|
103
|
+
mock.inject({ type: "response.created" });
|
|
104
|
+
mock.inject({
|
|
105
|
+
type: "response.output_text.delta",
|
|
106
|
+
delta: "Let me ",
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
const events = await eventsTask;
|
|
110
|
+
expect(events).toEqual([
|
|
111
|
+
{ type: "response_started" },
|
|
112
|
+
{ type: "transcript", role: "assistant", text: "Let me ", final: false },
|
|
113
|
+
]);
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it("surfaces response.output_text.done as final assistant transcript with accumulated text", async () => {
|
|
117
|
+
const mock = createMockSocketHarness();
|
|
118
|
+
const adapter = createAdapter(mock);
|
|
119
|
+
|
|
120
|
+
const eventsTask = collectEvents(adapter.events, 3);
|
|
121
|
+
const openTask = adapter.open(new AbortController().signal);
|
|
122
|
+
await waitFor(() => mock.sent.length > 0);
|
|
123
|
+
mock.inject({ type: "session.updated" });
|
|
124
|
+
await openTask;
|
|
125
|
+
|
|
126
|
+
mock.inject({ type: "response.created" });
|
|
127
|
+
mock.inject({
|
|
128
|
+
type: "response.output_text.delta",
|
|
129
|
+
delta: "Let me ",
|
|
130
|
+
});
|
|
131
|
+
mock.inject({
|
|
132
|
+
type: "response.output_text.done",
|
|
133
|
+
text: "Let me check that.",
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
const events = await eventsTask;
|
|
137
|
+
expect(events).toEqual([
|
|
138
|
+
{ type: "response_started" },
|
|
139
|
+
{ type: "transcript", role: "assistant", text: "Let me ", final: false },
|
|
140
|
+
{ type: "transcript", role: "assistant", text: "Let me check that.", final: true },
|
|
141
|
+
]);
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
it("uses assistantTranscript when response.output_text.done omits text", async () => {
|
|
145
|
+
const mock = createMockSocketHarness();
|
|
146
|
+
const adapter = createAdapter(mock);
|
|
147
|
+
|
|
148
|
+
const eventsTask = collectEvents(adapter.events, 3);
|
|
149
|
+
const openTask = adapter.open(new AbortController().signal);
|
|
150
|
+
await waitFor(() => mock.sent.length > 0);
|
|
151
|
+
mock.inject({ type: "session.updated" });
|
|
152
|
+
await openTask;
|
|
153
|
+
|
|
154
|
+
mock.inject({ type: "response.created" });
|
|
155
|
+
mock.inject({
|
|
156
|
+
type: "response.output_text.delta",
|
|
157
|
+
delta: "Hello",
|
|
158
|
+
});
|
|
159
|
+
mock.inject({
|
|
160
|
+
type: "response.output_text.done",
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
const events = await eventsTask;
|
|
164
|
+
expect(events).toEqual([
|
|
165
|
+
{ type: "response_started" },
|
|
166
|
+
{ type: "transcript", role: "assistant", text: "Hello", final: false },
|
|
167
|
+
{ type: "transcript", role: "assistant", text: "Hello", final: true },
|
|
168
|
+
]);
|
|
169
|
+
});
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
describe("createOpenAiCompatibleRealtimeAdapter requestResponse (Syrinx-owned turns)", () => {
|
|
173
|
+
it("sends input_audio_buffer.commit then response.create", async () => {
|
|
174
|
+
const mock = createMockSocketHarness();
|
|
175
|
+
const adapter = createAdapter(mock);
|
|
176
|
+
|
|
177
|
+
const openTask = adapter.open(new AbortController().signal);
|
|
178
|
+
await waitFor(() => mock.sent.length > 0);
|
|
179
|
+
mock.inject({ type: "session.updated" });
|
|
180
|
+
await openTask;
|
|
181
|
+
|
|
182
|
+
const baseline = mock.sent.length;
|
|
183
|
+
adapter.requestResponse!();
|
|
184
|
+
|
|
185
|
+
expect(JSON.parse(mock.sent[baseline]!)).toEqual({ type: "input_audio_buffer.commit" });
|
|
186
|
+
expect(JSON.parse(mock.sent[baseline + 1]!)).toEqual({ type: "response.create" });
|
|
187
|
+
});
|
|
188
|
+
});
|
|
@@ -171,6 +171,11 @@ class OpenAiCompatibleRealtimeAdapter implements RealtimeAdapter {
|
|
|
171
171
|
this.requestResponseCreate();
|
|
172
172
|
}
|
|
173
173
|
|
|
174
|
+
requestResponse(): void {
|
|
175
|
+
this.requireSocket().send({ type: "input_audio_buffer.commit" });
|
|
176
|
+
this.requestResponseCreate();
|
|
177
|
+
}
|
|
178
|
+
|
|
174
179
|
cancelResponse(audioEndMs: number): void {
|
|
175
180
|
if (!this.activeResponse) return;
|
|
176
181
|
const socket = this.requireSocket();
|
|
@@ -331,6 +336,30 @@ class OpenAiCompatibleRealtimeAdapter implements RealtimeAdapter {
|
|
|
331
336
|
this.assistantTranscript = "";
|
|
332
337
|
break;
|
|
333
338
|
}
|
|
339
|
+
case "response.output_text.delta": {
|
|
340
|
+
const delta = msg["delta"];
|
|
341
|
+
if (typeof delta === "string" && delta.length > 0) {
|
|
342
|
+
this.assistantTranscript += delta;
|
|
343
|
+
this.stream.push({
|
|
344
|
+
type: "transcript",
|
|
345
|
+
role: "assistant",
|
|
346
|
+
text: delta,
|
|
347
|
+
final: false,
|
|
348
|
+
});
|
|
349
|
+
}
|
|
350
|
+
break;
|
|
351
|
+
}
|
|
352
|
+
case "response.output_text.done": {
|
|
353
|
+
const t = typeof msg["text"] === "string" ? msg["text"] : this.assistantTranscript;
|
|
354
|
+
this.stream.push({
|
|
355
|
+
type: "transcript",
|
|
356
|
+
role: "assistant",
|
|
357
|
+
text: t,
|
|
358
|
+
final: true,
|
|
359
|
+
});
|
|
360
|
+
this.assistantTranscript = "";
|
|
361
|
+
break;
|
|
362
|
+
}
|
|
334
363
|
case "conversation.item.input_audio_transcription.completed": {
|
|
335
364
|
// User-side transcript (requires input transcription enabled in session config).
|
|
336
365
|
const transcript = typeof msg["transcript"] === "string" ? msg["transcript"] : "";
|
package/src/realtime-adapter.ts
CHANGED
|
@@ -14,6 +14,16 @@ export interface RealtimeAdapter {
|
|
|
14
14
|
* be replayed — that would double-apply history (RFC bimodel-delegate-seam R6).
|
|
15
15
|
*/
|
|
16
16
|
readonly supportsNativeResume?: boolean;
|
|
17
|
+
/** The front model owns full-duplex interaction decisions (turn-taking, barge-in). When true,
|
|
18
|
+
* Syrinx's InteractionPolicy runs observe-only and does not drive its own turn/interrupt decisions
|
|
19
|
+
* (RFC InteractionPolicy REQ-4). Absent/false → Syrinx drives. No current adapter sets this. */
|
|
20
|
+
readonly supportsFullDuplex?: boolean;
|
|
21
|
+
/** The front emits its own backchannels ("mhmm"). When true, Syrinx suppresses its own backchannel
|
|
22
|
+
* cues (RFC InteractionPolicy REQ-4). Absent/false → Syrinx may emit. */
|
|
23
|
+
readonly emitsBackchannel?: boolean;
|
|
24
|
+
/** Half-cascade: provider can run text-only modality (`modalities:["text"]`) so Syrinx TTS
|
|
25
|
+
* drives speech from assistant transcript events. Absent/false → no text-only half-cascade. */
|
|
26
|
+
readonly supportsTextOnlyModality?: boolean;
|
|
17
27
|
};
|
|
18
28
|
|
|
19
29
|
open(signal: AbortSignal): Promise<void>;
|
|
@@ -23,6 +33,13 @@ export interface RealtimeAdapter {
|
|
|
23
33
|
* provider cannot accept text input omit it, and the bridge silently ignores typed turns for them.
|
|
24
34
|
*/
|
|
25
35
|
sendText?(text: string): void;
|
|
36
|
+
/**
|
|
37
|
+
* Commit any buffered user input and request a response. For Syrinx-OWNED turn detection
|
|
38
|
+
* (provider server VAD disabled via turnDetection:null): the host calls this when its own
|
|
39
|
+
* endpointing (InteractionPolicy / VAD) signals end-of-turn. Optional — adapters without
|
|
40
|
+
* manual turn control omit it.
|
|
41
|
+
*/
|
|
42
|
+
requestResponse?(): void;
|
|
26
43
|
cancelResponse(audioEndMs: number): void;
|
|
27
44
|
injectToolResult(toolId: string, text: string): void;
|
|
28
45
|
/** Close the provider socket and end the event stream. */
|
|
@@ -75,6 +75,12 @@ class FakeRealtimeAdapter implements RealtimeAdapter {
|
|
|
75
75
|
this.sentText.push(text);
|
|
76
76
|
}
|
|
77
77
|
|
|
78
|
+
requestResponseCalls = 0;
|
|
79
|
+
|
|
80
|
+
requestResponse(): void {
|
|
81
|
+
this.requestResponseCalls += 1;
|
|
82
|
+
}
|
|
83
|
+
|
|
78
84
|
readonly cancelCalls: number[] = [];
|
|
79
85
|
|
|
80
86
|
cancelResponse(audioEndMs: number): void {
|
|
@@ -216,6 +222,86 @@ describe("RealtimeBridge", () => {
|
|
|
216
222
|
await started;
|
|
217
223
|
});
|
|
218
224
|
|
|
225
|
+
it("with syrinxTurns, eos.turn_complete calls adapter.requestResponse exactly once", async () => {
|
|
226
|
+
const adapter = new FakeRealtimeAdapter();
|
|
227
|
+
const bridge = new RealtimeBridge(adapter, undefined, undefined, { syrinxTurns: true });
|
|
228
|
+
const bus = new PipelineBusImpl();
|
|
229
|
+
buses.push(bus);
|
|
230
|
+
|
|
231
|
+
const started = bus.start();
|
|
232
|
+
await bridge.initialize(bus, {});
|
|
233
|
+
|
|
234
|
+
bus.push(Route.Main, {
|
|
235
|
+
kind: "eos.turn_complete",
|
|
236
|
+
contextId: "user-turn",
|
|
237
|
+
timestampMs: Date.now(),
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
await waitForCondition(() => adapter.requestResponseCalls >= 1);
|
|
241
|
+
expect(adapter.requestResponseCalls).toBe(1);
|
|
242
|
+
|
|
243
|
+
await bridge.close();
|
|
244
|
+
bus.stop();
|
|
245
|
+
await started;
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
it("without syrinxTurns, eos.turn_complete does not call adapter.requestResponse", async () => {
|
|
249
|
+
const adapter = new FakeRealtimeAdapter();
|
|
250
|
+
const bridge = new RealtimeBridge(adapter);
|
|
251
|
+
const bus = new PipelineBusImpl();
|
|
252
|
+
buses.push(bus);
|
|
253
|
+
|
|
254
|
+
const started = bus.start();
|
|
255
|
+
await bridge.initialize(bus, {});
|
|
256
|
+
|
|
257
|
+
bus.push(Route.Main, {
|
|
258
|
+
kind: "eos.turn_complete",
|
|
259
|
+
contextId: "user-turn",
|
|
260
|
+
timestampMs: Date.now(),
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
264
|
+
expect(adapter.requestResponseCalls).toBe(0);
|
|
265
|
+
|
|
266
|
+
await bridge.close();
|
|
267
|
+
bus.stop();
|
|
268
|
+
await started;
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
it("syrinxTurns+textOnly: response_done does not re-trigger requestResponse via eos.turn_complete", async () => {
|
|
272
|
+
const adapter = new FakeRealtimeAdapter();
|
|
273
|
+
const bridge = new RealtimeBridge(adapter, undefined, undefined, {
|
|
274
|
+
textOnly: true,
|
|
275
|
+
syrinxTurns: true,
|
|
276
|
+
});
|
|
277
|
+
const bus = new PipelineBusImpl();
|
|
278
|
+
buses.push(bus);
|
|
279
|
+
|
|
280
|
+
const started = bus.start();
|
|
281
|
+
await bridge.initialize(bus, {});
|
|
282
|
+
|
|
283
|
+
// Endpointing owns the user-turn signal → one requestResponse.
|
|
284
|
+
bus.push(Route.Main, {
|
|
285
|
+
kind: "eos.turn_complete",
|
|
286
|
+
contextId: "user-turn",
|
|
287
|
+
timestampMs: Date.now(),
|
|
288
|
+
});
|
|
289
|
+
await waitForCondition(() => adapter.requestResponseCalls >= 1);
|
|
290
|
+
expect(adapter.requestResponseCalls).toBe(1);
|
|
291
|
+
|
|
292
|
+
// Full provider response cycle must not re-emit eos.turn_complete → requestResponse.
|
|
293
|
+
adapter.emit({ type: "response_started" });
|
|
294
|
+
adapter.emit({ type: "transcript", role: "assistant", text: "hi", final: true });
|
|
295
|
+
adapter.emit({ type: "response_done" });
|
|
296
|
+
|
|
297
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
298
|
+
expect(adapter.requestResponseCalls).toBe(1);
|
|
299
|
+
|
|
300
|
+
await bridge.close();
|
|
301
|
+
bus.stop();
|
|
302
|
+
await started;
|
|
303
|
+
});
|
|
304
|
+
|
|
219
305
|
it("mints a fresh contextId for each response_started (R1)", async () => {
|
|
220
306
|
const adapter = new FakeRealtimeAdapter();
|
|
221
307
|
const bridge = new RealtimeBridge(adapter);
|
|
@@ -1144,4 +1230,129 @@ describe("RealtimeBridge", () => {
|
|
|
1144
1230
|
|
|
1145
1231
|
await session.close();
|
|
1146
1232
|
});
|
|
1233
|
+
|
|
1234
|
+
// ── Half-cascade C1: textOnly routing ────────────────────────────────────
|
|
1235
|
+
// In text-only mode the bridge streams assistant transcript into the cascade
|
|
1236
|
+
// llm.delta → segmenter → tts.text path. The TTS plugin owns tts.end.
|
|
1237
|
+
|
|
1238
|
+
it("textOnly: streams assistant transcript deltas as llm.delta in order as they arrive", async () => {
|
|
1239
|
+
const adapter = new FakeRealtimeAdapter();
|
|
1240
|
+
const bridge = new RealtimeBridge(adapter, undefined, "consult_knowledge", { textOnly: true });
|
|
1241
|
+
const bus = new PipelineBusImpl();
|
|
1242
|
+
buses.push(bus);
|
|
1243
|
+
const deltas: LlmDeltaPacket[] = [];
|
|
1244
|
+
bus.on("llm.delta", (pkt) => { deltas.push(pkt as LlmDeltaPacket); });
|
|
1245
|
+
|
|
1246
|
+
const started = bus.start();
|
|
1247
|
+
await bridge.initialize(bus, {});
|
|
1248
|
+
|
|
1249
|
+
adapter.emit({ type: "response_started" });
|
|
1250
|
+
adapter.emit({ type: "transcript", role: "assistant", text: "The capital", final: false });
|
|
1251
|
+
adapter.emit({ type: "transcript", role: "assistant", text: " of France", final: false });
|
|
1252
|
+
adapter.emit({ type: "transcript", role: "assistant", text: " is Paris.", final: false });
|
|
1253
|
+
|
|
1254
|
+
await waitForCondition(() => deltas.length === 3);
|
|
1255
|
+
expect(deltas.map((d) => d.text)).toEqual(["The capital", " of France", " is Paris."]);
|
|
1256
|
+
|
|
1257
|
+
await bridge.close();
|
|
1258
|
+
bus.stop();
|
|
1259
|
+
await started;
|
|
1260
|
+
});
|
|
1261
|
+
|
|
1262
|
+
it("textOnly: final assistant transcript pushes llm.done and no full-text llm.delta", async () => {
|
|
1263
|
+
const adapter = new FakeRealtimeAdapter();
|
|
1264
|
+
const bridge = new RealtimeBridge(adapter, undefined, "consult_knowledge", { textOnly: true });
|
|
1265
|
+
const bus = new PipelineBusImpl();
|
|
1266
|
+
buses.push(bus);
|
|
1267
|
+
const deltas: LlmDeltaPacket[] = [];
|
|
1268
|
+
const dones: LlmResponseDonePacket[] = [];
|
|
1269
|
+
bus.on("llm.delta", (pkt) => { deltas.push(pkt as LlmDeltaPacket); });
|
|
1270
|
+
bus.on("llm.done", (pkt) => { dones.push(pkt as LlmResponseDonePacket); });
|
|
1271
|
+
|
|
1272
|
+
const started = bus.start();
|
|
1273
|
+
await bridge.initialize(bus, {});
|
|
1274
|
+
|
|
1275
|
+
adapter.emit({ type: "response_started" });
|
|
1276
|
+
adapter.emit({ type: "transcript", role: "assistant", text: "Hello ", final: false });
|
|
1277
|
+
adapter.emit({ type: "transcript", role: "assistant", text: "world.", final: false });
|
|
1278
|
+
// C0 adapter output_text.done carries the FULL accumulated text — must not re-emit as delta.
|
|
1279
|
+
adapter.emit({ type: "transcript", role: "assistant", text: "Hello world.", final: true });
|
|
1280
|
+
|
|
1281
|
+
await waitForCondition(() => dones.length === 1);
|
|
1282
|
+
expect(deltas.map((d) => d.text)).toEqual(["Hello ", "world."]);
|
|
1283
|
+
expect(dones).toHaveLength(1);
|
|
1284
|
+
expect(dones[0]!.text).toBe("Hello world.");
|
|
1285
|
+
// No extra full-text llm.delta for the final event.
|
|
1286
|
+
expect(deltas.some((d) => d.text === "Hello world.")).toBe(false);
|
|
1287
|
+
|
|
1288
|
+
await bridge.close();
|
|
1289
|
+
bus.stop();
|
|
1290
|
+
await started;
|
|
1291
|
+
});
|
|
1292
|
+
|
|
1293
|
+
it("textOnly: provider audio events push no tts.audio", async () => {
|
|
1294
|
+
const adapter = new FakeRealtimeAdapter();
|
|
1295
|
+
const bridge = new RealtimeBridge(adapter, undefined, "consult_knowledge", { textOnly: true });
|
|
1296
|
+
const bus = new PipelineBusImpl();
|
|
1297
|
+
buses.push(bus);
|
|
1298
|
+
const audio: TextToSpeechAudioPacket[] = [];
|
|
1299
|
+
bus.on("tts.audio", (pkt) => { audio.push(pkt as TextToSpeechAudioPacket); });
|
|
1300
|
+
|
|
1301
|
+
const started = bus.start();
|
|
1302
|
+
await bridge.initialize(bus, {});
|
|
1303
|
+
|
|
1304
|
+
adapter.emit({ type: "response_started" });
|
|
1305
|
+
adapter.emit({
|
|
1306
|
+
type: "audio",
|
|
1307
|
+
pcm16: frameSizedPcm24k(),
|
|
1308
|
+
sampleRateHz: 24_000,
|
|
1309
|
+
});
|
|
1310
|
+
adapter.emit({ type: "response_done" });
|
|
1311
|
+
|
|
1312
|
+
await waitForCondition(() => true); // allow pump to drain
|
|
1313
|
+
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
1314
|
+
expect(audio).toHaveLength(0);
|
|
1315
|
+
|
|
1316
|
+
await bridge.close();
|
|
1317
|
+
bus.stop();
|
|
1318
|
+
await started;
|
|
1319
|
+
});
|
|
1320
|
+
|
|
1321
|
+
it("textOnly: onResponseDone does not re-emit assistant llm.delta and does not emit tts.end", async () => {
|
|
1322
|
+
const adapter = new FakeRealtimeAdapter();
|
|
1323
|
+
const bridge = new RealtimeBridge(adapter, undefined, "consult_knowledge", { textOnly: true });
|
|
1324
|
+
const bus = new PipelineBusImpl();
|
|
1325
|
+
buses.push(bus);
|
|
1326
|
+
const deltas: LlmDeltaPacket[] = [];
|
|
1327
|
+
const dones: LlmResponseDonePacket[] = [];
|
|
1328
|
+
const ends: TextToSpeechEndPacket[] = [];
|
|
1329
|
+
const turnCompletes: EndOfSpeechPacket[] = [];
|
|
1330
|
+
bus.on("llm.delta", (pkt) => { deltas.push(pkt as LlmDeltaPacket); });
|
|
1331
|
+
bus.on("llm.done", (pkt) => { dones.push(pkt as LlmResponseDonePacket); });
|
|
1332
|
+
bus.on("tts.end", (pkt) => { ends.push(pkt as TextToSpeechEndPacket); });
|
|
1333
|
+
bus.on("eos.turn_complete", (pkt) => { turnCompletes.push(pkt as EndOfSpeechPacket); });
|
|
1334
|
+
|
|
1335
|
+
const started = bus.start();
|
|
1336
|
+
await bridge.initialize(bus, {});
|
|
1337
|
+
|
|
1338
|
+
adapter.emit({ type: "response_started" });
|
|
1339
|
+
adapter.emit({ type: "transcript", role: "assistant", text: "Streamed.", final: false });
|
|
1340
|
+
adapter.emit({ type: "transcript", role: "assistant", text: "Streamed.", final: true });
|
|
1341
|
+
await waitForCondition(() => dones.length === 1);
|
|
1342
|
+
const deltasBeforeDone = deltas.length;
|
|
1343
|
+
|
|
1344
|
+
adapter.emit({ type: "response_done" });
|
|
1345
|
+
await waitForCondition(() => turnCompletes.length === 1);
|
|
1346
|
+
|
|
1347
|
+
// No duplicate display-only llm.delta/llm.done from onResponseDone.
|
|
1348
|
+
expect(deltas).toHaveLength(deltasBeforeDone);
|
|
1349
|
+
expect(dones).toHaveLength(1);
|
|
1350
|
+
// TTS plugin owns tts.end in text-only mode — bridge must not force it.
|
|
1351
|
+
expect(ends).toHaveLength(0);
|
|
1352
|
+
expect(turnCompletes).toHaveLength(1);
|
|
1353
|
+
|
|
1354
|
+
await bridge.close();
|
|
1355
|
+
bus.stop();
|
|
1356
|
+
await started;
|
|
1357
|
+
});
|
|
1147
1358
|
});
|
package/src/realtime-bridge.ts
CHANGED
|
@@ -89,6 +89,22 @@ export interface RealtimeBridgeOptions {
|
|
|
89
89
|
readonly toolName: string;
|
|
90
90
|
readonly args: Record<string, unknown>;
|
|
91
91
|
}) => string | undefined | Promise<string | undefined>;
|
|
92
|
+
/**
|
|
93
|
+
* Half-cascade text-only mode: stream assistant transcript into the cascade
|
|
94
|
+
* `llm.delta → segmenter → tts.text` path and ignore provider audio.
|
|
95
|
+
* REQUIRES a TTS plugin registered on the session bus (the bridge does not own the registry).
|
|
96
|
+
*/
|
|
97
|
+
readonly textOnly?: boolean;
|
|
98
|
+
/**
|
|
99
|
+
* Syrinx-owned turn detection (REQ-6): on `eos.turn_complete`, call
|
|
100
|
+
* `adapter.requestResponse` so Syrinx endpointing/VAD/InteractionPolicy drives
|
|
101
|
+
* the provider response instead of server VAD.
|
|
102
|
+
*
|
|
103
|
+
* MUST be paired with `turnDetection: null` on the adapter — otherwise the
|
|
104
|
+
* provider's server VAD and Syrinx would BOTH trigger responses (double-turn).
|
|
105
|
+
* Native mode (`syrinxTurns` absent/false) is unchanged.
|
|
106
|
+
*/
|
|
107
|
+
readonly syrinxTurns?: boolean;
|
|
92
108
|
}
|
|
93
109
|
|
|
94
110
|
export class RealtimeBridge implements VoicePlugin {
|
|
@@ -138,6 +154,14 @@ export class RealtimeBridge implements VoicePlugin {
|
|
|
138
154
|
}),
|
|
139
155
|
);
|
|
140
156
|
|
|
157
|
+
if (this.opts.syrinxTurns) {
|
|
158
|
+
this.disposers.push(
|
|
159
|
+
bus.on("eos.turn_complete", () => {
|
|
160
|
+
this.adapter.requestResponse?.();
|
|
161
|
+
}),
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
|
|
141
165
|
await this.adapter.open(this.sessionAbort.signal);
|
|
142
166
|
void this.pump();
|
|
143
167
|
}
|
|
@@ -176,11 +200,15 @@ export class RealtimeBridge implements VoicePlugin {
|
|
|
176
200
|
this.onResponseStarted(bus);
|
|
177
201
|
break;
|
|
178
202
|
case "audio":
|
|
179
|
-
|
|
203
|
+
// textOnly: provider audio must not be used (REQ-4) — TTS plugin owns playout.
|
|
204
|
+
if (!this.opts.textOnly) this.onAudio(bus, ev.pcm16, ev.sampleRateHz);
|
|
180
205
|
break;
|
|
181
206
|
case "transcript":
|
|
182
207
|
if (ev.role === "user") {
|
|
183
208
|
if (ev.final) this.onFinalTranscript(bus, ev.text);
|
|
209
|
+
} else if (this.opts.textOnly) {
|
|
210
|
+
// Drive cascade TTS path as text arrives; do not buffer for display-only re-emit.
|
|
211
|
+
this.onTextOnlyAssistantTranscript(bus, ev.text, ev.final);
|
|
184
212
|
} else if (ev.final && ev.text.trim()) {
|
|
185
213
|
this.turnAssistantText = this.turnAssistantText
|
|
186
214
|
? `${this.turnAssistantText} ${ev.text.trim()}`
|
|
@@ -455,10 +483,42 @@ export class RealtimeBridge implements VoicePlugin {
|
|
|
455
483
|
bus.push(Route.Main, packet);
|
|
456
484
|
}
|
|
457
485
|
|
|
486
|
+
/**
|
|
487
|
+
* textOnly: stream assistant transcript into the cascade LLM→segmenter→TTS path.
|
|
488
|
+
* Non-final deltas drive `llm.delta` immediately (no buffer). Final emits `llm.done` only —
|
|
489
|
+
* the C0 adapter's final carries the full accumulated text, so re-emitting as delta would
|
|
490
|
+
* duplicate every streamed fragment.
|
|
491
|
+
*/
|
|
492
|
+
private onTextOnlyAssistantTranscript(bus: PipelineBus, text: string, final: boolean): void {
|
|
493
|
+
if (!this.contextId) return;
|
|
494
|
+
const timestampMs = Date.now();
|
|
495
|
+
if (!final && text) {
|
|
496
|
+
const delta: LlmDeltaPacket = {
|
|
497
|
+
kind: "llm.delta",
|
|
498
|
+
contextId: this.contextId,
|
|
499
|
+
timestampMs,
|
|
500
|
+
text,
|
|
501
|
+
};
|
|
502
|
+
bus.push(Route.Main, delta);
|
|
503
|
+
return;
|
|
504
|
+
}
|
|
505
|
+
if (final && text.trim()) {
|
|
506
|
+
const done: LlmResponseDonePacket = {
|
|
507
|
+
kind: "llm.done",
|
|
508
|
+
contextId: this.contextId,
|
|
509
|
+
timestampMs,
|
|
510
|
+
text,
|
|
511
|
+
};
|
|
512
|
+
bus.push(Route.Main, done);
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
|
|
458
516
|
private onResponseDone(bus: PipelineBus): void {
|
|
459
517
|
if (!this.contextId) return;
|
|
460
|
-
this.
|
|
461
|
-
|
|
518
|
+
if (!this.opts.textOnly) {
|
|
519
|
+
this.emitCoalescedAudio(bus, true);
|
|
520
|
+
this.audioRemainder = new Uint8Array(0);
|
|
521
|
+
}
|
|
462
522
|
|
|
463
523
|
const timestampMs = Date.now();
|
|
464
524
|
const transcripts: SttResultPacket[] = this.turnUserText
|
|
@@ -477,6 +537,16 @@ export class RealtimeBridge implements VoicePlugin {
|
|
|
477
537
|
text: this.turnUserText,
|
|
478
538
|
transcripts,
|
|
479
539
|
};
|
|
540
|
+
|
|
541
|
+
if (this.opts.textOnly) {
|
|
542
|
+
// Already streamed llm.delta/llm.done above. TTS plugin owns tts.end (REQ-5) — forcing
|
|
543
|
+
// it here would cut playout / break barge-in heard-prefix timing.
|
|
544
|
+
// syrinxTurns: endpointing already emitted the user-turn eos.turn_complete that drove
|
|
545
|
+
// requestResponse; re-emitting here would re-subscribe-fire requestResponse → loop.
|
|
546
|
+
if (!this.opts.syrinxTurns) bus.push(Route.Main, turnComplete);
|
|
547
|
+
return;
|
|
548
|
+
}
|
|
549
|
+
|
|
480
550
|
const ttsEnd: TextToSpeechEndPacket = {
|
|
481
551
|
kind: "tts.end",
|
|
482
552
|
contextId: this.contextId,
|