@kuralle-syrinx/pipecat-smart-turn 4.2.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -3
- package/src/eos-plugin.ts +564 -0
- package/src/index.ts +9 -450
- package/src/interaction-policy.ts +18 -5
- package/src/predictor.ts +3 -6
- package/src/semantic-completeness.ts +3 -1
- package/src/smart-turn-types.ts +10 -0
- package/src/index.test.ts +0 -803
- package/src/interaction-policy.test.ts +0 -136
- package/src/semantic-completeness.test.ts +0 -164
- package/tsconfig.json +0 -21
package/src/index.test.ts
DELETED
|
@@ -1,803 +0,0 @@
|
|
|
1
|
-
// SPDX-License-Identifier: MIT
|
|
2
|
-
|
|
3
|
-
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
4
|
-
import {
|
|
5
|
-
PipelineBusImpl,
|
|
6
|
-
Route,
|
|
7
|
-
type EndOfSpeechPacket,
|
|
8
|
-
type InterimEndOfSpeechPacket,
|
|
9
|
-
} from "@kuralle-syrinx/core";
|
|
10
|
-
import { pcm16SamplesToBytes } from "@kuralle-syrinx/core/audio";
|
|
11
|
-
import { PipecatEOSPlugin, type SmartTurnPredictor } from "./index.js";
|
|
12
|
-
|
|
13
|
-
class PredictableSmartTurn implements SmartTurnPredictor {
|
|
14
|
-
readonly audioInputs: Float32Array[] = [];
|
|
15
|
-
|
|
16
|
-
constructor(private readonly predictions: number[] = [1]) {}
|
|
17
|
-
|
|
18
|
-
async initialize(): Promise<void> {
|
|
19
|
-
// no-op
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
async predict(audio: Float32Array): Promise<number> {
|
|
23
|
-
this.audioInputs.push(audio);
|
|
24
|
-
return this.predictions.shift() ?? 1;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
async close(): Promise<void> {
|
|
28
|
-
// no-op
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
function startBus(bus: PipelineBusImpl): Promise<void> {
|
|
33
|
-
return bus.start();
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
describe("PipecatEOSPlugin", () => {
|
|
37
|
-
afterEach(() => {
|
|
38
|
-
vi.useRealTimers();
|
|
39
|
-
});
|
|
40
|
-
|
|
41
|
-
it("emits interim EOS packets from interim transcripts", async () => {
|
|
42
|
-
const bus = new PipelineBusImpl();
|
|
43
|
-
const started = startBus(bus);
|
|
44
|
-
const plugin = new PipecatEOSPlugin(new PredictableSmartTurn());
|
|
45
|
-
const interims: InterimEndOfSpeechPacket[] = [];
|
|
46
|
-
bus.on("eos.interim", (pkt) => {
|
|
47
|
-
interims.push(pkt as InterimEndOfSpeechPacket);
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
await plugin.initialize(bus, {});
|
|
51
|
-
bus.push(Route.Main, {
|
|
52
|
-
kind: "stt.interim",
|
|
53
|
-
contextId: "turn-1",
|
|
54
|
-
timestampMs: Date.now(),
|
|
55
|
-
text: "hello",
|
|
56
|
-
});
|
|
57
|
-
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
58
|
-
|
|
59
|
-
expect(interims).toEqual([
|
|
60
|
-
expect.objectContaining({
|
|
61
|
-
kind: "eos.interim",
|
|
62
|
-
contextId: "turn-1",
|
|
63
|
-
text: "hello",
|
|
64
|
-
}),
|
|
65
|
-
]);
|
|
66
|
-
|
|
67
|
-
await plugin.close();
|
|
68
|
-
bus.stop();
|
|
69
|
-
await started;
|
|
70
|
-
});
|
|
71
|
-
|
|
72
|
-
it("finalizes after VAD stop and final STT result", async () => {
|
|
73
|
-
const bus = new PipelineBusImpl();
|
|
74
|
-
const started = startBus(bus);
|
|
75
|
-
const plugin = new PipecatEOSPlugin(new PredictableSmartTurn([0.9]));
|
|
76
|
-
const completions: EndOfSpeechPacket[] = [];
|
|
77
|
-
const finalizeRequests: string[] = [];
|
|
78
|
-
bus.on("eos.turn_complete", (pkt) => {
|
|
79
|
-
completions.push(pkt as EndOfSpeechPacket);
|
|
80
|
-
});
|
|
81
|
-
bus.on("stt.finalize", (pkt) => {
|
|
82
|
-
finalizeRequests.push(pkt.contextId);
|
|
83
|
-
});
|
|
84
|
-
|
|
85
|
-
await plugin.initialize(bus, { finalize_delay_ms: 5, max_delay_ms: 50 });
|
|
86
|
-
bus.push(Route.Main, {
|
|
87
|
-
kind: "vad.speech_started",
|
|
88
|
-
contextId: "turn-1",
|
|
89
|
-
timestampMs: Date.now(),
|
|
90
|
-
confidence: 0.9,
|
|
91
|
-
});
|
|
92
|
-
bus.push(Route.Main, {
|
|
93
|
-
kind: "stt.result",
|
|
94
|
-
contextId: "turn-1",
|
|
95
|
-
timestampMs: Date.now(),
|
|
96
|
-
text: "hello world",
|
|
97
|
-
confidence: 0.95,
|
|
98
|
-
language: "en-US",
|
|
99
|
-
});
|
|
100
|
-
bus.push(Route.Main, {
|
|
101
|
-
kind: "vad.speech_ended",
|
|
102
|
-
contextId: "turn-1",
|
|
103
|
-
timestampMs: Date.now(),
|
|
104
|
-
});
|
|
105
|
-
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
106
|
-
|
|
107
|
-
expect(completions).toEqual([
|
|
108
|
-
expect.objectContaining({
|
|
109
|
-
kind: "eos.turn_complete",
|
|
110
|
-
contextId: "turn-1",
|
|
111
|
-
text: "hello world",
|
|
112
|
-
transcripts: [
|
|
113
|
-
expect.objectContaining({
|
|
114
|
-
kind: "stt.result",
|
|
115
|
-
text: "hello world",
|
|
116
|
-
}),
|
|
117
|
-
],
|
|
118
|
-
}),
|
|
119
|
-
]);
|
|
120
|
-
expect(finalizeRequests).toEqual(["turn-1"]);
|
|
121
|
-
|
|
122
|
-
await plugin.close();
|
|
123
|
-
bus.stop();
|
|
124
|
-
await started;
|
|
125
|
-
});
|
|
126
|
-
|
|
127
|
-
it("finalizes on max timeout even if VAD stop never arrives", async () => {
|
|
128
|
-
const bus = new PipelineBusImpl();
|
|
129
|
-
const started = startBus(bus);
|
|
130
|
-
const plugin = new PipecatEOSPlugin(new PredictableSmartTurn());
|
|
131
|
-
const completions: EndOfSpeechPacket[] = [];
|
|
132
|
-
bus.on("eos.turn_complete", (pkt) => {
|
|
133
|
-
completions.push(pkt as EndOfSpeechPacket);
|
|
134
|
-
});
|
|
135
|
-
|
|
136
|
-
await plugin.initialize(bus, { finalize_delay_ms: 5, max_delay_ms: 10 });
|
|
137
|
-
bus.push(Route.Main, {
|
|
138
|
-
kind: "stt.result",
|
|
139
|
-
contextId: "turn-2",
|
|
140
|
-
timestampMs: Date.now(),
|
|
141
|
-
text: "timeout final",
|
|
142
|
-
confidence: 0.8,
|
|
143
|
-
});
|
|
144
|
-
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
145
|
-
|
|
146
|
-
expect(completions).toEqual([
|
|
147
|
-
expect.objectContaining({
|
|
148
|
-
kind: "eos.turn_complete",
|
|
149
|
-
contextId: "turn-2",
|
|
150
|
-
text: "timeout final",
|
|
151
|
-
}),
|
|
152
|
-
]);
|
|
153
|
-
|
|
154
|
-
await plugin.close();
|
|
155
|
-
bus.stop();
|
|
156
|
-
await started;
|
|
157
|
-
});
|
|
158
|
-
|
|
159
|
-
it("does not max-finalize an early provider final while VAD still reports active speech", async () => {
|
|
160
|
-
const bus = new PipelineBusImpl();
|
|
161
|
-
const started = startBus(bus);
|
|
162
|
-
const plugin = new PipecatEOSPlugin(new PredictableSmartTurn([0.9]));
|
|
163
|
-
const completions: EndOfSpeechPacket[] = [];
|
|
164
|
-
bus.on("eos.turn_complete", (pkt) => {
|
|
165
|
-
completions.push(pkt as EndOfSpeechPacket);
|
|
166
|
-
});
|
|
167
|
-
|
|
168
|
-
await plugin.initialize(bus, { finalize_delay_ms: 5, max_delay_ms: 10 });
|
|
169
|
-
bus.push(Route.Main, {
|
|
170
|
-
kind: "vad.speech_started",
|
|
171
|
-
contextId: "turn-early-final",
|
|
172
|
-
timestampMs: Date.now(),
|
|
173
|
-
confidence: 0.9,
|
|
174
|
-
});
|
|
175
|
-
bus.push(Route.Main, {
|
|
176
|
-
kind: "stt.result",
|
|
177
|
-
contextId: "turn-early-final",
|
|
178
|
-
timestampMs: Date.now(),
|
|
179
|
-
text: "first half",
|
|
180
|
-
confidence: 0.9,
|
|
181
|
-
});
|
|
182
|
-
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
183
|
-
expect(completions).toEqual([]);
|
|
184
|
-
|
|
185
|
-
bus.push(Route.Main, {
|
|
186
|
-
kind: "vad.speech_ended",
|
|
187
|
-
contextId: "turn-early-final",
|
|
188
|
-
timestampMs: Date.now(),
|
|
189
|
-
});
|
|
190
|
-
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
191
|
-
|
|
192
|
-
expect(completions).toEqual([
|
|
193
|
-
expect.objectContaining({
|
|
194
|
-
kind: "eos.turn_complete",
|
|
195
|
-
contextId: "turn-early-final",
|
|
196
|
-
text: "first half",
|
|
197
|
-
}),
|
|
198
|
-
]);
|
|
199
|
-
|
|
200
|
-
await plugin.close();
|
|
201
|
-
bus.stop();
|
|
202
|
-
await started;
|
|
203
|
-
});
|
|
204
|
-
|
|
205
|
-
it("does not complete a turn for an incomplete smart-turn pause", async () => {
|
|
206
|
-
const bus = new PipelineBusImpl();
|
|
207
|
-
const started = startBus(bus);
|
|
208
|
-
const plugin = new PipecatEOSPlugin(new PredictableSmartTurn([0.1, 0.9]));
|
|
209
|
-
const completions: EndOfSpeechPacket[] = [];
|
|
210
|
-
bus.on("eos.turn_complete", (pkt) => {
|
|
211
|
-
completions.push(pkt as EndOfSpeechPacket);
|
|
212
|
-
});
|
|
213
|
-
|
|
214
|
-
await plugin.initialize(bus, { finalize_delay_ms: 5, max_delay_ms: 100 });
|
|
215
|
-
bus.push(Route.Main, {
|
|
216
|
-
kind: "stt.result",
|
|
217
|
-
contextId: "turn-3",
|
|
218
|
-
timestampMs: Date.now(),
|
|
219
|
-
text: "I need to know",
|
|
220
|
-
confidence: 0.95,
|
|
221
|
-
});
|
|
222
|
-
bus.push(Route.Main, {
|
|
223
|
-
kind: "vad.speech_ended",
|
|
224
|
-
contextId: "turn-3",
|
|
225
|
-
timestampMs: Date.now(),
|
|
226
|
-
});
|
|
227
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
228
|
-
expect(completions).toEqual([]);
|
|
229
|
-
|
|
230
|
-
bus.push(Route.Main, {
|
|
231
|
-
kind: "vad.speech_started",
|
|
232
|
-
contextId: "turn-3",
|
|
233
|
-
timestampMs: Date.now(),
|
|
234
|
-
confidence: 0.9,
|
|
235
|
-
});
|
|
236
|
-
bus.push(Route.Main, {
|
|
237
|
-
kind: "stt.result",
|
|
238
|
-
contextId: "turn-3",
|
|
239
|
-
timestampMs: Date.now(),
|
|
240
|
-
text: "whether the petition is approved.",
|
|
241
|
-
confidence: 0.95,
|
|
242
|
-
});
|
|
243
|
-
bus.push(Route.Main, {
|
|
244
|
-
kind: "vad.speech_ended",
|
|
245
|
-
contextId: "turn-3",
|
|
246
|
-
timestampMs: Date.now(),
|
|
247
|
-
});
|
|
248
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
249
|
-
|
|
250
|
-
expect(completions).toEqual([
|
|
251
|
-
expect.objectContaining({
|
|
252
|
-
contextId: "turn-3",
|
|
253
|
-
text: "I need to know whether the petition is approved.",
|
|
254
|
-
}),
|
|
255
|
-
]);
|
|
256
|
-
|
|
257
|
-
await plugin.close();
|
|
258
|
-
bus.stop();
|
|
259
|
-
await started;
|
|
260
|
-
});
|
|
261
|
-
|
|
262
|
-
it("falls back after sustained silence when smart turn predicts an incomplete pause", async () => {
|
|
263
|
-
const bus = new PipelineBusImpl();
|
|
264
|
-
const started = startBus(bus);
|
|
265
|
-
const plugin = new PipecatEOSPlugin(new PredictableSmartTurn([0.1]));
|
|
266
|
-
const completions: EndOfSpeechPacket[] = [];
|
|
267
|
-
bus.on("eos.turn_complete", (pkt) => {
|
|
268
|
-
completions.push(pkt as EndOfSpeechPacket);
|
|
269
|
-
});
|
|
270
|
-
|
|
271
|
-
await plugin.initialize(bus, { finalize_delay_ms: 5, max_delay_ms: 100, incomplete_fallback_ms: 10, semantic_shortcut_delay_ms: 5 });
|
|
272
|
-
bus.push(Route.Main, {
|
|
273
|
-
kind: "stt.result",
|
|
274
|
-
contextId: "turn-4",
|
|
275
|
-
timestampMs: Date.now(),
|
|
276
|
-
text: "finished despite low confidence",
|
|
277
|
-
confidence: 0.95,
|
|
278
|
-
});
|
|
279
|
-
bus.push(Route.Main, {
|
|
280
|
-
kind: "vad.speech_ended",
|
|
281
|
-
contextId: "turn-4",
|
|
282
|
-
timestampMs: Date.now(),
|
|
283
|
-
});
|
|
284
|
-
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
285
|
-
|
|
286
|
-
expect(completions).toEqual([
|
|
287
|
-
expect.objectContaining({
|
|
288
|
-
contextId: "turn-4",
|
|
289
|
-
text: "finished despite low confidence",
|
|
290
|
-
}),
|
|
291
|
-
]);
|
|
292
|
-
|
|
293
|
-
await plugin.close();
|
|
294
|
-
bus.stop();
|
|
295
|
-
await started;
|
|
296
|
-
});
|
|
297
|
-
|
|
298
|
-
it("requests STT finalization only after smart turn approves the boundary", async () => {
|
|
299
|
-
const bus = new PipelineBusImpl();
|
|
300
|
-
const started = startBus(bus);
|
|
301
|
-
const plugin = new PipecatEOSPlugin(new PredictableSmartTurn([0.9]));
|
|
302
|
-
const requests: string[] = [];
|
|
303
|
-
bus.on("stt.finalize", (pkt) => {
|
|
304
|
-
requests.push(pkt.contextId);
|
|
305
|
-
});
|
|
306
|
-
|
|
307
|
-
await plugin.initialize(bus, { finalize_delay_ms: 5 });
|
|
308
|
-
bus.push(Route.Main, {
|
|
309
|
-
kind: "vad.speech_ended",
|
|
310
|
-
contextId: "turn-5",
|
|
311
|
-
timestampMs: Date.now(),
|
|
312
|
-
});
|
|
313
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
314
|
-
|
|
315
|
-
expect(requests).toEqual(["turn-5"]);
|
|
316
|
-
|
|
317
|
-
await plugin.close();
|
|
318
|
-
bus.stop();
|
|
319
|
-
await started;
|
|
320
|
-
});
|
|
321
|
-
|
|
322
|
-
it("accumulates odd-byteOffset VAD PCM without constructing an unaligned Int16Array view", async () => {
|
|
323
|
-
const bus = new PipelineBusImpl();
|
|
324
|
-
const started = startBus(bus);
|
|
325
|
-
const predictor = new PredictableSmartTurn([0.9]);
|
|
326
|
-
const plugin = new PipecatEOSPlugin(predictor);
|
|
327
|
-
const requests: string[] = [];
|
|
328
|
-
bus.on("stt.finalize", (pkt) => {
|
|
329
|
-
requests.push(pkt.contextId);
|
|
330
|
-
});
|
|
331
|
-
|
|
332
|
-
await plugin.initialize(bus, { finalize_delay_ms: 5 });
|
|
333
|
-
const bytes = pcm16SamplesToBytes(new Int16Array([0, 32767, -32768, 16384]));
|
|
334
|
-
const backing = new Uint8Array(bytes.byteLength + 1);
|
|
335
|
-
backing.set(bytes, 1);
|
|
336
|
-
const oddOffsetFrame = backing.subarray(1);
|
|
337
|
-
expect(oddOffsetFrame.byteOffset % 2).toBe(1);
|
|
338
|
-
|
|
339
|
-
bus.push(Route.Main, {
|
|
340
|
-
kind: "vad.audio",
|
|
341
|
-
contextId: "turn-odd-offset",
|
|
342
|
-
timestampMs: Date.now(),
|
|
343
|
-
audio: oddOffsetFrame,
|
|
344
|
-
});
|
|
345
|
-
bus.push(Route.Main, {
|
|
346
|
-
kind: "vad.speech_ended",
|
|
347
|
-
contextId: "turn-odd-offset",
|
|
348
|
-
timestampMs: Date.now(),
|
|
349
|
-
});
|
|
350
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
351
|
-
|
|
352
|
-
expect(requests).toEqual(["turn-odd-offset"]);
|
|
353
|
-
expect(predictor.audioInputs).toHaveLength(1);
|
|
354
|
-
expect(Array.from(predictor.audioInputs[0]!)).toEqual([
|
|
355
|
-
0,
|
|
356
|
-
32767 / 32768,
|
|
357
|
-
-1,
|
|
358
|
-
0.5,
|
|
359
|
-
]);
|
|
360
|
-
|
|
361
|
-
await plugin.close();
|
|
362
|
-
bus.stop();
|
|
363
|
-
await started;
|
|
364
|
-
});
|
|
365
|
-
|
|
366
|
-
it("defers semantic mid-thought pauses even when smart turn approves the boundary", async () => {
|
|
367
|
-
const bus = new PipelineBusImpl();
|
|
368
|
-
const started = startBus(bus);
|
|
369
|
-
const plugin = new PipecatEOSPlugin(new PredictableSmartTurn([0.9, 0.9]));
|
|
370
|
-
const completions: EndOfSpeechPacket[] = [];
|
|
371
|
-
const finalizeRequests: string[] = [];
|
|
372
|
-
bus.on("eos.turn_complete", (pkt) => {
|
|
373
|
-
completions.push(pkt as EndOfSpeechPacket);
|
|
374
|
-
});
|
|
375
|
-
bus.on("stt.finalize", (pkt) => {
|
|
376
|
-
finalizeRequests.push(pkt.contextId);
|
|
377
|
-
});
|
|
378
|
-
|
|
379
|
-
await plugin.initialize(bus, {
|
|
380
|
-
finalize_delay_ms: 5,
|
|
381
|
-
semantic_defer_fallback_ms: 100,
|
|
382
|
-
incomplete_fallback_ms: 100,
|
|
383
|
-
});
|
|
384
|
-
bus.push(Route.Main, {
|
|
385
|
-
kind: "stt.result",
|
|
386
|
-
contextId: "turn-6",
|
|
387
|
-
timestampMs: Date.now(),
|
|
388
|
-
text: "I need to know",
|
|
389
|
-
confidence: 0.95,
|
|
390
|
-
});
|
|
391
|
-
bus.push(Route.Main, {
|
|
392
|
-
kind: "vad.speech_ended",
|
|
393
|
-
contextId: "turn-6",
|
|
394
|
-
timestampMs: Date.now(),
|
|
395
|
-
});
|
|
396
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
397
|
-
expect(completions).toEqual([]);
|
|
398
|
-
expect(finalizeRequests).toEqual([]);
|
|
399
|
-
|
|
400
|
-
bus.push(Route.Main, {
|
|
401
|
-
kind: "vad.speech_started",
|
|
402
|
-
contextId: "turn-6",
|
|
403
|
-
timestampMs: Date.now(),
|
|
404
|
-
confidence: 0.9,
|
|
405
|
-
});
|
|
406
|
-
bus.push(Route.Main, {
|
|
407
|
-
kind: "stt.result",
|
|
408
|
-
contextId: "turn-6",
|
|
409
|
-
timestampMs: Date.now(),
|
|
410
|
-
text: "whether the petition is approved",
|
|
411
|
-
confidence: 0.95,
|
|
412
|
-
});
|
|
413
|
-
bus.push(Route.Main, {
|
|
414
|
-
kind: "vad.speech_ended",
|
|
415
|
-
contextId: "turn-6",
|
|
416
|
-
timestampMs: Date.now(),
|
|
417
|
-
});
|
|
418
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
419
|
-
|
|
420
|
-
expect(completions).toEqual([
|
|
421
|
-
expect.objectContaining({
|
|
422
|
-
contextId: "turn-6",
|
|
423
|
-
text: "I need to know whether the petition is approved",
|
|
424
|
-
}),
|
|
425
|
-
]);
|
|
426
|
-
expect(finalizeRequests).toEqual(["turn-6"]);
|
|
427
|
-
|
|
428
|
-
await plugin.close();
|
|
429
|
-
bus.stop();
|
|
430
|
-
await started;
|
|
431
|
-
});
|
|
432
|
-
|
|
433
|
-
it("shortcuts complete utterances when smart turn is uncertain", async () => {
|
|
434
|
-
const bus = new PipelineBusImpl();
|
|
435
|
-
const started = startBus(bus);
|
|
436
|
-
const plugin = new PipecatEOSPlugin(new PredictableSmartTurn([0.1]));
|
|
437
|
-
const completions: EndOfSpeechPacket[] = [];
|
|
438
|
-
bus.on("eos.turn_complete", (pkt) => {
|
|
439
|
-
completions.push(pkt as EndOfSpeechPacket);
|
|
440
|
-
});
|
|
441
|
-
|
|
442
|
-
await plugin.initialize(bus, {
|
|
443
|
-
finalize_delay_ms: 100,
|
|
444
|
-
semantic_shortcut_delay_ms: 5,
|
|
445
|
-
incomplete_fallback_ms: 100,
|
|
446
|
-
});
|
|
447
|
-
bus.push(Route.Main, {
|
|
448
|
-
kind: "stt.result",
|
|
449
|
-
contextId: "turn-7",
|
|
450
|
-
timestampMs: Date.now(),
|
|
451
|
-
text: "What are your office hours?",
|
|
452
|
-
confidence: 0.95,
|
|
453
|
-
});
|
|
454
|
-
bus.push(Route.Main, {
|
|
455
|
-
kind: "vad.speech_ended",
|
|
456
|
-
contextId: "turn-7",
|
|
457
|
-
timestampMs: Date.now(),
|
|
458
|
-
});
|
|
459
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
460
|
-
|
|
461
|
-
expect(completions).toEqual([
|
|
462
|
-
expect.objectContaining({
|
|
463
|
-
contextId: "turn-7",
|
|
464
|
-
text: "What are your office hours?",
|
|
465
|
-
}),
|
|
466
|
-
]);
|
|
467
|
-
|
|
468
|
-
await plugin.close();
|
|
469
|
-
bus.stop();
|
|
470
|
-
await started;
|
|
471
|
-
});
|
|
472
|
-
|
|
473
|
-
it("honors semantic shortcut delay when VAD ends before the STT final", async () => {
|
|
474
|
-
const bus = new PipelineBusImpl();
|
|
475
|
-
const started = startBus(bus);
|
|
476
|
-
const plugin = new PipecatEOSPlugin(new PredictableSmartTurn([0.1]));
|
|
477
|
-
const completions: EndOfSpeechPacket[] = [];
|
|
478
|
-
bus.on("eos.turn_complete", (pkt) => {
|
|
479
|
-
completions.push(pkt as EndOfSpeechPacket);
|
|
480
|
-
});
|
|
481
|
-
|
|
482
|
-
await plugin.initialize(bus, {
|
|
483
|
-
finalize_delay_ms: 250,
|
|
484
|
-
semantic_shortcut_delay_ms: 50,
|
|
485
|
-
incomplete_fallback_ms: 2000,
|
|
486
|
-
max_delay_ms: 0,
|
|
487
|
-
});
|
|
488
|
-
|
|
489
|
-
bus.push(Route.Main, {
|
|
490
|
-
kind: "vad.speech_started",
|
|
491
|
-
contextId: "turn-vad-first",
|
|
492
|
-
timestampMs: Date.now(),
|
|
493
|
-
confidence: 0.9,
|
|
494
|
-
});
|
|
495
|
-
bus.push(Route.Main, {
|
|
496
|
-
kind: "stt.interim",
|
|
497
|
-
contextId: "turn-vad-first",
|
|
498
|
-
timestampMs: Date.now(),
|
|
499
|
-
text: "Thanks, that answers everything I needed to know today.",
|
|
500
|
-
});
|
|
501
|
-
bus.push(Route.Main, {
|
|
502
|
-
kind: "vad.speech_ended",
|
|
503
|
-
contextId: "turn-vad-first",
|
|
504
|
-
timestampMs: Date.now(),
|
|
505
|
-
});
|
|
506
|
-
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
507
|
-
expect(completions).toEqual([]);
|
|
508
|
-
|
|
509
|
-
bus.push(Route.Main, {
|
|
510
|
-
kind: "stt.result",
|
|
511
|
-
contextId: "turn-vad-first",
|
|
512
|
-
timestampMs: Date.now(),
|
|
513
|
-
text: "Thanks, that answers everything I needed to know today.",
|
|
514
|
-
confidence: 0.95,
|
|
515
|
-
});
|
|
516
|
-
await new Promise((resolve) => setTimeout(resolve, 60));
|
|
517
|
-
expect(completions).toHaveLength(1);
|
|
518
|
-
|
|
519
|
-
await plugin.close();
|
|
520
|
-
bus.stop();
|
|
521
|
-
await started;
|
|
522
|
-
});
|
|
523
|
-
|
|
524
|
-
it("does not shortcut low-confidence semantic partials before fallback", async () => {
|
|
525
|
-
const bus = new PipelineBusImpl();
|
|
526
|
-
const started = startBus(bus);
|
|
527
|
-
const plugin = new PipecatEOSPlugin(new PredictableSmartTurn([0.1]));
|
|
528
|
-
const completions: EndOfSpeechPacket[] = [];
|
|
529
|
-
bus.on("eos.turn_complete", (pkt) => {
|
|
530
|
-
completions.push(pkt as EndOfSpeechPacket);
|
|
531
|
-
});
|
|
532
|
-
|
|
533
|
-
await plugin.initialize(bus, {
|
|
534
|
-
finalize_delay_ms: 5,
|
|
535
|
-
semantic_shortcut_delay_ms: 5,
|
|
536
|
-
incomplete_fallback_ms: 80,
|
|
537
|
-
max_delay_ms: 0,
|
|
538
|
-
});
|
|
539
|
-
bus.push(Route.Main, {
|
|
540
|
-
kind: "stt.result",
|
|
541
|
-
contextId: "turn-weak-semantic",
|
|
542
|
-
timestampMs: Date.now(),
|
|
543
|
-
text: "I was wondering if I can still add biology",
|
|
544
|
-
confidence: 0.95,
|
|
545
|
-
});
|
|
546
|
-
bus.push(Route.Main, {
|
|
547
|
-
kind: "vad.speech_ended",
|
|
548
|
-
contextId: "turn-weak-semantic",
|
|
549
|
-
timestampMs: Date.now(),
|
|
550
|
-
});
|
|
551
|
-
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
552
|
-
expect(completions).toEqual([]);
|
|
553
|
-
|
|
554
|
-
await new Promise((resolve) => setTimeout(resolve, 70));
|
|
555
|
-
expect(completions).toHaveLength(1);
|
|
556
|
-
|
|
557
|
-
await plugin.close();
|
|
558
|
-
bus.stop();
|
|
559
|
-
await started;
|
|
560
|
-
});
|
|
561
|
-
|
|
562
|
-
it("suppresses same-context duplicate completions while the assistant is still playing", async () => {
|
|
563
|
-
const bus = new PipelineBusImpl();
|
|
564
|
-
const started = startBus(bus);
|
|
565
|
-
const plugin = new PipecatEOSPlugin(new PredictableSmartTurn([0.9, 0.9]));
|
|
566
|
-
const completions: EndOfSpeechPacket[] = [];
|
|
567
|
-
bus.on("eos.turn_complete", (pkt) => {
|
|
568
|
-
completions.push(pkt as EndOfSpeechPacket);
|
|
569
|
-
});
|
|
570
|
-
|
|
571
|
-
await plugin.initialize(bus, { finalize_delay_ms: 5, max_delay_ms: 0 });
|
|
572
|
-
bus.push(Route.Main, {
|
|
573
|
-
kind: "vad.speech_started",
|
|
574
|
-
contextId: "review-turn",
|
|
575
|
-
timestampMs: Date.now(),
|
|
576
|
-
confidence: 0.9,
|
|
577
|
-
});
|
|
578
|
-
bus.push(Route.Main, {
|
|
579
|
-
kind: "stt.result",
|
|
580
|
-
contextId: "review-turn",
|
|
581
|
-
timestampMs: Date.now(),
|
|
582
|
-
text: "Hi. I'm Maya Chen, and I can still add biology one oh one.",
|
|
583
|
-
confidence: 0.95,
|
|
584
|
-
});
|
|
585
|
-
bus.push(Route.Main, {
|
|
586
|
-
kind: "vad.speech_ended",
|
|
587
|
-
contextId: "review-turn",
|
|
588
|
-
timestampMs: Date.now(),
|
|
589
|
-
});
|
|
590
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
591
|
-
expect(completions).toHaveLength(1);
|
|
592
|
-
|
|
593
|
-
bus.push(Route.Main, {
|
|
594
|
-
kind: "tts.audio",
|
|
595
|
-
contextId: "review-turn",
|
|
596
|
-
timestampMs: Date.now(),
|
|
597
|
-
audio: new Uint8Array([1, 2, 3, 4]),
|
|
598
|
-
sampleRateHz: 16000,
|
|
599
|
-
});
|
|
600
|
-
bus.push(Route.Main, {
|
|
601
|
-
kind: "vad.speech_started",
|
|
602
|
-
contextId: "review-turn",
|
|
603
|
-
timestampMs: Date.now(),
|
|
604
|
-
confidence: 0.9,
|
|
605
|
-
});
|
|
606
|
-
bus.push(Route.Main, {
|
|
607
|
-
kind: "stt.result",
|
|
608
|
-
contextId: "review-turn",
|
|
609
|
-
timestampMs: Date.now(),
|
|
610
|
-
text: "and what form I should submit.",
|
|
611
|
-
confidence: 0.95,
|
|
612
|
-
});
|
|
613
|
-
bus.push(Route.Main, {
|
|
614
|
-
kind: "vad.speech_ended",
|
|
615
|
-
contextId: "review-turn",
|
|
616
|
-
timestampMs: Date.now(),
|
|
617
|
-
});
|
|
618
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
619
|
-
|
|
620
|
-
expect(completions).toEqual([
|
|
621
|
-
expect.objectContaining({
|
|
622
|
-
contextId: "review-turn",
|
|
623
|
-
text: "Hi. I'm Maya Chen, and I can still add biology one oh one.",
|
|
624
|
-
}),
|
|
625
|
-
]);
|
|
626
|
-
|
|
627
|
-
await plugin.close();
|
|
628
|
-
bus.stop();
|
|
629
|
-
await started;
|
|
630
|
-
});
|
|
631
|
-
|
|
632
|
-
it("finalizes immediately when semantic defer expires still incomplete", async () => {
|
|
633
|
-
const bus = new PipelineBusImpl();
|
|
634
|
-
const started = startBus(bus);
|
|
635
|
-
const plugin = new PipecatEOSPlugin(new PredictableSmartTurn([0.9]));
|
|
636
|
-
const completions: EndOfSpeechPacket[] = [];
|
|
637
|
-
bus.on("eos.turn_complete", (pkt) => {
|
|
638
|
-
completions.push(pkt as EndOfSpeechPacket);
|
|
639
|
-
});
|
|
640
|
-
|
|
641
|
-
await plugin.initialize(bus, {
|
|
642
|
-
finalize_delay_ms: 250,
|
|
643
|
-
semantic_defer_fallback_ms: 40,
|
|
644
|
-
incomplete_fallback_ms: 2000,
|
|
645
|
-
max_delay_ms: 0,
|
|
646
|
-
});
|
|
647
|
-
|
|
648
|
-
bus.push(Route.Main, {
|
|
649
|
-
kind: "stt.result",
|
|
650
|
-
contextId: "turn-defer",
|
|
651
|
-
timestampMs: Date.now(),
|
|
652
|
-
text: "I need to know",
|
|
653
|
-
confidence: 0.95,
|
|
654
|
-
});
|
|
655
|
-
bus.push(Route.Main, {
|
|
656
|
-
kind: "vad.speech_ended",
|
|
657
|
-
contextId: "turn-defer",
|
|
658
|
-
timestampMs: Date.now(),
|
|
659
|
-
});
|
|
660
|
-
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
661
|
-
expect(completions).toEqual([]);
|
|
662
|
-
|
|
663
|
-
await new Promise((resolve) => setTimeout(resolve, 35));
|
|
664
|
-
expect(completions).toHaveLength(1);
|
|
665
|
-
|
|
666
|
-
await plugin.close();
|
|
667
|
-
bus.stop();
|
|
668
|
-
await started;
|
|
669
|
-
});
|
|
670
|
-
|
|
671
|
-
it("does not force-finalize semantic defer while speech is active again", async () => {
|
|
672
|
-
const bus = new PipelineBusImpl();
|
|
673
|
-
const started = startBus(bus);
|
|
674
|
-
const plugin = new PipecatEOSPlugin(new PredictableSmartTurn([0.9]));
|
|
675
|
-
const completions: EndOfSpeechPacket[] = [];
|
|
676
|
-
bus.on("eos.turn_complete", (pkt) => {
|
|
677
|
-
completions.push(pkt as EndOfSpeechPacket);
|
|
678
|
-
});
|
|
679
|
-
|
|
680
|
-
await plugin.initialize(bus, {
|
|
681
|
-
finalize_delay_ms: 250,
|
|
682
|
-
semantic_defer_fallback_ms: 30,
|
|
683
|
-
incomplete_fallback_ms: 2000,
|
|
684
|
-
max_delay_ms: 0,
|
|
685
|
-
});
|
|
686
|
-
|
|
687
|
-
bus.push(Route.Main, {
|
|
688
|
-
kind: "stt.result",
|
|
689
|
-
contextId: "turn-defer-active",
|
|
690
|
-
timestampMs: Date.now(),
|
|
691
|
-
text: "I need to know",
|
|
692
|
-
confidence: 0.95,
|
|
693
|
-
});
|
|
694
|
-
bus.push(Route.Main, {
|
|
695
|
-
kind: "vad.speech_ended",
|
|
696
|
-
contextId: "turn-defer-active",
|
|
697
|
-
timestampMs: Date.now(),
|
|
698
|
-
});
|
|
699
|
-
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
700
|
-
bus.push(Route.Main, {
|
|
701
|
-
kind: "vad.speech_started",
|
|
702
|
-
contextId: "turn-defer-active",
|
|
703
|
-
timestampMs: Date.now(),
|
|
704
|
-
confidence: 0.9,
|
|
705
|
-
});
|
|
706
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
707
|
-
|
|
708
|
-
expect(completions).toEqual([]);
|
|
709
|
-
|
|
710
|
-
await plugin.close();
|
|
711
|
-
bus.stop();
|
|
712
|
-
await started;
|
|
713
|
-
});
|
|
714
|
-
});
|
|
715
|
-
|
|
716
|
-
describe("PipecatEOSPlugin — STT-quiet fallback (wedged VAD)", () => {
|
|
717
|
-
afterEach(() => {
|
|
718
|
-
vi.useRealTimers();
|
|
719
|
-
});
|
|
720
|
-
|
|
721
|
-
it("completes the turn when transcripts go quiet but VAD never ends the segment", async () => {
|
|
722
|
-
const bus = new PipelineBusImpl();
|
|
723
|
-
const started = startBus(bus);
|
|
724
|
-
const plugin = new PipecatEOSPlugin(new PredictableSmartTurn([0.9]));
|
|
725
|
-
const turns: EndOfSpeechPacket[] = [];
|
|
726
|
-
bus.on("eos.turn_complete", (pkt) => {
|
|
727
|
-
turns.push(pkt as EndOfSpeechPacket);
|
|
728
|
-
});
|
|
729
|
-
|
|
730
|
-
await plugin.initialize(bus, { finalize_delay_ms: 5, stt_quiet_fallback_ms: 40, max_delay_ms: 0 });
|
|
731
|
-
|
|
732
|
-
// Speech starts and NEVER ends (saturated VAD on a long telephony segment).
|
|
733
|
-
bus.push(Route.Main, {
|
|
734
|
-
kind: "vad.speech_started",
|
|
735
|
-
contextId: "turn-quiet",
|
|
736
|
-
timestampMs: Date.now(),
|
|
737
|
-
confidence: 0.95,
|
|
738
|
-
});
|
|
739
|
-
// Provider STT delivers the user's whole question, then goes quiet.
|
|
740
|
-
bus.push(Route.Main, {
|
|
741
|
-
kind: "stt.result",
|
|
742
|
-
contextId: "turn-quiet",
|
|
743
|
-
timestampMs: Date.now(),
|
|
744
|
-
text: "what is the application deadline",
|
|
745
|
-
confidence: 0.99,
|
|
746
|
-
});
|
|
747
|
-
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
748
|
-
expect(turns).toHaveLength(0); // not yet — quiet window still open
|
|
749
|
-
|
|
750
|
-
await new Promise((resolve) => setTimeout(resolve, 120));
|
|
751
|
-
expect(turns).toEqual([
|
|
752
|
-
expect.objectContaining({
|
|
753
|
-
kind: "eos.turn_complete",
|
|
754
|
-
contextId: "turn-quiet",
|
|
755
|
-
text: "what is the application deadline",
|
|
756
|
-
}),
|
|
757
|
-
]);
|
|
758
|
-
|
|
759
|
-
await plugin.close();
|
|
760
|
-
bus.stop();
|
|
761
|
-
await started;
|
|
762
|
-
});
|
|
763
|
-
|
|
764
|
-
it("does not fire the fallback while transcripts keep arriving", async () => {
|
|
765
|
-
const bus = new PipelineBusImpl();
|
|
766
|
-
const started = startBus(bus);
|
|
767
|
-
const plugin = new PipecatEOSPlugin(new PredictableSmartTurn([0.9]));
|
|
768
|
-
const turns: EndOfSpeechPacket[] = [];
|
|
769
|
-
bus.on("eos.turn_complete", (pkt) => {
|
|
770
|
-
turns.push(pkt as EndOfSpeechPacket);
|
|
771
|
-
});
|
|
772
|
-
|
|
773
|
-
await plugin.initialize(bus, { finalize_delay_ms: 5, stt_quiet_fallback_ms: 60, max_delay_ms: 0 });
|
|
774
|
-
bus.push(Route.Main, {
|
|
775
|
-
kind: "vad.speech_started",
|
|
776
|
-
contextId: "turn-active",
|
|
777
|
-
timestampMs: Date.now(),
|
|
778
|
-
confidence: 0.95,
|
|
779
|
-
});
|
|
780
|
-
bus.push(Route.Main, {
|
|
781
|
-
kind: "stt.result",
|
|
782
|
-
contextId: "turn-active",
|
|
783
|
-
timestampMs: Date.now(),
|
|
784
|
-
text: "first part",
|
|
785
|
-
confidence: 0.99,
|
|
786
|
-
});
|
|
787
|
-
// Keep the transcript active: interims every ~30ms push the quiet deadline back.
|
|
788
|
-
for (let i = 0; i < 4; i += 1) {
|
|
789
|
-
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
790
|
-
bus.push(Route.Main, {
|
|
791
|
-
kind: "stt.interim",
|
|
792
|
-
contextId: "turn-active",
|
|
793
|
-
timestampMs: Date.now(),
|
|
794
|
-
text: `still talking ${String(i)}`,
|
|
795
|
-
});
|
|
796
|
-
}
|
|
797
|
-
expect(turns).toHaveLength(0);
|
|
798
|
-
|
|
799
|
-
await plugin.close();
|
|
800
|
-
bus.stop();
|
|
801
|
-
await started;
|
|
802
|
-
});
|
|
803
|
-
});
|