@kuralle-syrinx/pipecat-smart-turn 4.2.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -3
- package/src/eos-plugin.ts +564 -0
- package/src/index.ts +9 -450
- package/src/interaction-policy.ts +18 -5
- package/src/predictor.ts +3 -6
- package/src/semantic-completeness.ts +3 -1
- package/src/smart-turn-types.ts +10 -0
- package/src/index.test.ts +0 -803
- package/src/interaction-policy.test.ts +0 -136
- package/src/semantic-completeness.test.ts +0 -164
- package/tsconfig.json +0 -21
package/src/index.ts
CHANGED
|
@@ -5,450 +5,19 @@
|
|
|
5
5
|
// Mirrors Pipecat's LocalSmartTurnAnalyzerV3 turn-stop strategy:
|
|
6
6
|
// Silero determines candidate speech boundaries, then Smart Turn v3 decides
|
|
7
7
|
// whether a pause is an actual completed user turn.
|
|
8
|
+
//
|
|
9
|
+
// Node / local default: `new PipecatEOSPlugin()` uses LocalSmartTurnV3Predictor
|
|
10
|
+
// (onnxruntime-node). Edge hosts must import from `@kuralle-syrinx/pipecat-smart-turn/eos`
|
|
11
|
+
// and inject a non-ONNX predictor (e.g. WorkersAiSmartTurnPredictor).
|
|
8
12
|
|
|
9
|
-
import {
|
|
10
|
-
fuseEndpointDecision,
|
|
11
|
-
latestTranscript,
|
|
12
|
-
scoreSemanticCompleteness,
|
|
13
|
-
type SemanticEndpointFusionConfig,
|
|
14
|
-
} from "./semantic-completeness.js";
|
|
15
|
-
import {
|
|
16
|
-
Route,
|
|
17
|
-
type EndOfSpeechPacket,
|
|
18
|
-
type FinalizeSttPacket,
|
|
19
|
-
type InterruptionDetectedPacket,
|
|
20
|
-
type InterimEndOfSpeechPacket,
|
|
21
|
-
type PipelineBus,
|
|
22
|
-
type PluginConfig,
|
|
23
|
-
type SttInterimPacket,
|
|
24
|
-
type SttResultPacket,
|
|
25
|
-
type TextToSpeechAudioPacket,
|
|
26
|
-
type TextToSpeechEndPacket,
|
|
27
|
-
type TextToSpeechPlayoutProgressPacket,
|
|
28
|
-
type VadAudioPacket,
|
|
29
|
-
type VadSpeechEndedPacket,
|
|
30
|
-
type VadSpeechStartedPacket,
|
|
31
|
-
type VoicePlugin,
|
|
32
|
-
} from "@kuralle-syrinx/core";
|
|
33
|
-
import { pcm16BytesToSamples } from "@kuralle-syrinx/core/audio";
|
|
13
|
+
import { PipecatEOSPlugin as PipecatEOSPluginBase } from "./eos-plugin.js";
|
|
34
14
|
import { LocalSmartTurnV3Predictor, type SmartTurnPredictor } from "./predictor.js";
|
|
35
15
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
readonly contextId: string;
|
|
41
|
-
audio: number[];
|
|
42
|
-
finalPackets: SttResultPacket[];
|
|
43
|
-
finalSegments: string[];
|
|
44
|
-
latestInterim: string;
|
|
45
|
-
boundaryAnalyzed: boolean;
|
|
46
|
-
smartTurnComplete: boolean;
|
|
47
|
-
semanticComplete: boolean;
|
|
48
|
-
speechActive: boolean;
|
|
49
|
-
finalized: boolean;
|
|
50
|
-
analysisSequence: number;
|
|
51
|
-
finalizeTimer: ReturnType<typeof setTimeout> | null;
|
|
52
|
-
sttQuietTimer: ReturnType<typeof setTimeout> | null;
|
|
53
|
-
maxTimer: ReturnType<typeof setTimeout> | null;
|
|
54
|
-
deferTimer: ReturnType<typeof setTimeout> | null;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
export class PipecatEOSPlugin implements VoicePlugin {
|
|
58
|
-
readonly endpointingCapability = { owner: "smart_turn" as const };
|
|
59
|
-
|
|
60
|
-
private bus: PipelineBus | null = null;
|
|
61
|
-
private disposers: Array<() => void> = [];
|
|
62
|
-
private turns = new Map<string, TurnState>();
|
|
63
|
-
private finalizeDelayMs = 250;
|
|
64
|
-
private sttQuietFallbackMs = 2500;
|
|
65
|
-
private maxDelayMs = 2000;
|
|
66
|
-
private incompleteFallbackMs = 2000;
|
|
67
|
-
private semanticShortcutDelayMs = 50;
|
|
68
|
-
private semanticDeferFallbackMs = 4000;
|
|
69
|
-
private semanticEndpointingEnabled = true;
|
|
70
|
-
private probabilityThreshold = 0.5;
|
|
71
|
-
private readonly lockedContextIds = new Set<string>();
|
|
72
|
-
private readonly lockedContextsWithAssistantAudio = new Set<string>();
|
|
73
|
-
|
|
74
|
-
constructor(private readonly predictor: SmartTurnPredictor = new LocalSmartTurnV3Predictor()) {}
|
|
75
|
-
|
|
76
|
-
async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
|
|
77
|
-
this.bus = bus;
|
|
78
|
-
this.finalizeDelayMs = readNonNegativeNumber(config["finalize_delay_ms"], 250);
|
|
79
|
-
this.sttQuietFallbackMs = readNonNegativeNumber(config["stt_quiet_fallback_ms"], 2500);
|
|
80
|
-
this.maxDelayMs = readNonNegativeNumber(config["max_delay_ms"], 2000);
|
|
81
|
-
this.incompleteFallbackMs = readNonNegativeNumber(config["incomplete_fallback_ms"], 2000);
|
|
82
|
-
this.semanticShortcutDelayMs = readNonNegativeNumber(config["semantic_shortcut_delay_ms"], 50);
|
|
83
|
-
this.semanticDeferFallbackMs = readNonNegativeNumber(config["semantic_defer_fallback_ms"], 4000);
|
|
84
|
-
this.semanticEndpointingEnabled = readBooleanConfig(config["semantic_endpointing_enabled"], true);
|
|
85
|
-
this.probabilityThreshold = readProbability(config["probability_threshold"], 0.5);
|
|
86
|
-
await this.predictor.initialize(config);
|
|
87
|
-
|
|
88
|
-
this.disposers.push(
|
|
89
|
-
bus.on("vad.audio", (pkt) => {
|
|
90
|
-
this.handleAudio(pkt as VadAudioPacket);
|
|
91
|
-
}),
|
|
92
|
-
bus.on("stt.interim", (pkt) => {
|
|
93
|
-
this.handleInterim(pkt as SttInterimPacket);
|
|
94
|
-
}),
|
|
95
|
-
bus.on("stt.result", (pkt) => {
|
|
96
|
-
this.handleFinal(pkt as SttResultPacket);
|
|
97
|
-
}),
|
|
98
|
-
bus.on("vad.speech_started", (pkt) => {
|
|
99
|
-
this.handleSpeechStarted(pkt as VadSpeechStartedPacket);
|
|
100
|
-
}),
|
|
101
|
-
bus.on("vad.speech_ended", async (pkt) => {
|
|
102
|
-
await this.handleSpeechEnded(pkt as VadSpeechEndedPacket);
|
|
103
|
-
}),
|
|
104
|
-
bus.on("tts.audio", (pkt) => {
|
|
105
|
-
this.handleTtsAudio(pkt as TextToSpeechAudioPacket);
|
|
106
|
-
}),
|
|
107
|
-
bus.on("tts.end", (pkt) => {
|
|
108
|
-
this.handleTtsEnd(pkt as TextToSpeechEndPacket);
|
|
109
|
-
}),
|
|
110
|
-
bus.on("tts.playout_progress", (pkt) => {
|
|
111
|
-
this.handleTtsPlayoutProgress(pkt as TextToSpeechPlayoutProgressPacket);
|
|
112
|
-
}),
|
|
113
|
-
bus.on("interrupt.detected", (pkt) => {
|
|
114
|
-
this.releaseContextLock((pkt as InterruptionDetectedPacket).contextId);
|
|
115
|
-
}),
|
|
116
|
-
);
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
async close(): Promise<void> {
|
|
120
|
-
for (const dispose of this.disposers.splice(0)) {
|
|
121
|
-
dispose();
|
|
122
|
-
}
|
|
123
|
-
for (const state of this.turns.values()) {
|
|
124
|
-
clearTurnTimers(state);
|
|
125
|
-
}
|
|
126
|
-
this.turns.clear();
|
|
127
|
-
this.lockedContextIds.clear();
|
|
128
|
-
this.lockedContextsWithAssistantAudio.clear();
|
|
129
|
-
await this.predictor.close();
|
|
130
|
-
this.bus = null;
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
private handleAudio(pkt: VadAudioPacket): void {
|
|
134
|
-
if (this.lockedContextIds.has(pkt.contextId)) return;
|
|
135
|
-
if (pkt.audio.byteLength % 2 !== 0) return;
|
|
136
|
-
const state = this.stateFor(pkt.contextId);
|
|
137
|
-
const samples = pcm16BytesToSamples(pkt.audio);
|
|
138
|
-
for (const sample of samples) {
|
|
139
|
-
state.audio.push(sample / 32768);
|
|
140
|
-
}
|
|
141
|
-
if (state.audio.length > MAX_AUDIO_SAMPLES) {
|
|
142
|
-
state.audio.splice(0, state.audio.length - MAX_AUDIO_SAMPLES);
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
private handleInterim(pkt: SttInterimPacket): void {
|
|
147
|
-
if (this.lockedContextIds.has(pkt.contextId)) return;
|
|
148
|
-
if (!pkt.text.trim()) return;
|
|
149
|
-
const state = this.stateFor(pkt.contextId);
|
|
150
|
-
state.latestInterim = pkt.text.trim();
|
|
151
|
-
if (state.sttQuietTimer) this.armSttQuietFallback(state);
|
|
152
|
-
this.bus?.push(Route.Main, {
|
|
153
|
-
kind: "eos.interim",
|
|
154
|
-
contextId: pkt.contextId,
|
|
155
|
-
timestampMs: Date.now(),
|
|
156
|
-
text: pkt.text,
|
|
157
|
-
} satisfies InterimEndOfSpeechPacket);
|
|
16
|
+
/** Node-facing EOS plugin: defaults to the local ONNX Smart Turn predictor. */
|
|
17
|
+
export class PipecatEOSPlugin extends PipecatEOSPluginBase {
|
|
18
|
+
constructor(predictor: SmartTurnPredictor = new LocalSmartTurnV3Predictor()) {
|
|
19
|
+
super(predictor);
|
|
158
20
|
}
|
|
159
|
-
|
|
160
|
-
private handleFinal(pkt: SttResultPacket): void {
|
|
161
|
-
if (this.lockedContextIds.has(pkt.contextId)) return;
|
|
162
|
-
if (!pkt.text.trim()) return;
|
|
163
|
-
const state = this.stateFor(pkt.contextId);
|
|
164
|
-
appendFinalPacket(state, pkt);
|
|
165
|
-
state.latestInterim = "";
|
|
166
|
-
|
|
167
|
-
const transcript = latestTranscript(state.finalSegments, state.latestInterim);
|
|
168
|
-
const semantic = scoreSemanticCompleteness(transcript);
|
|
169
|
-
state.semanticComplete = semantic.complete;
|
|
170
|
-
|
|
171
|
-
if (state.boundaryAnalyzed && state.smartTurnComplete && state.semanticComplete) {
|
|
172
|
-
this.scheduleFinalize(state, this.finalizeDelayMs);
|
|
173
|
-
return;
|
|
174
|
-
}
|
|
175
|
-
if (state.smartTurnComplete && state.semanticComplete) {
|
|
176
|
-
this.scheduleFinalize(state, this.finalizeDelayMs);
|
|
177
|
-
return;
|
|
178
|
-
}
|
|
179
|
-
if (state.boundaryAnalyzed && state.smartTurnComplete && !state.semanticComplete) {
|
|
180
|
-
return;
|
|
181
|
-
}
|
|
182
|
-
if (state.boundaryAnalyzed && !state.smartTurnComplete && state.semanticComplete) {
|
|
183
|
-
this.scheduleSemanticShortcut(state);
|
|
184
|
-
return;
|
|
185
|
-
}
|
|
186
|
-
if (state.boundaryAnalyzed) {
|
|
187
|
-
this.scheduleIncompleteFallback(state);
|
|
188
|
-
return;
|
|
189
|
-
}
|
|
190
|
-
if (state.speechActive) {
|
|
191
|
-
this.armSttQuietFallback(state);
|
|
192
|
-
return;
|
|
193
|
-
}
|
|
194
|
-
this.scheduleMaxFinalize(state);
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
private handleSpeechStarted(pkt: VadSpeechStartedPacket): void {
|
|
198
|
-
if (this.lockedContextIds.has(pkt.contextId)) return;
|
|
199
|
-
const state = this.stateFor(pkt.contextId);
|
|
200
|
-
if (state.sttQuietTimer) {
|
|
201
|
-
clearTimeout(state.sttQuietTimer);
|
|
202
|
-
state.sttQuietTimer = null;
|
|
203
|
-
}
|
|
204
|
-
state.boundaryAnalyzed = false;
|
|
205
|
-
state.smartTurnComplete = false;
|
|
206
|
-
state.semanticComplete = false;
|
|
207
|
-
state.speechActive = true;
|
|
208
|
-
state.latestInterim = "";
|
|
209
|
-
state.analysisSequence += 1;
|
|
210
|
-
clearTurnTimers(state);
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
private async handleSpeechEnded(pkt: VadSpeechEndedPacket): Promise<void> {
|
|
214
|
-
if (this.lockedContextIds.has(pkt.contextId)) return;
|
|
215
|
-
const state = this.stateFor(pkt.contextId);
|
|
216
|
-
if (state.sttQuietTimer) {
|
|
217
|
-
clearTimeout(state.sttQuietTimer);
|
|
218
|
-
state.sttQuietTimer = null;
|
|
219
|
-
}
|
|
220
|
-
await this.analyzeBoundary(state);
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
// Shared end-of-speech boundary analysis: used by the VAD speech_ended path
|
|
224
|
-
// and by the STT-quiet fallback (when the provider transcript has gone quiet
|
|
225
|
-
// but the VAD never closed the segment — e.g. model state saturation on long
|
|
226
|
-
// telephony audio). The turn must never be held hostage by a wedged VAD.
|
|
227
|
-
private async analyzeBoundary(state: TurnState): Promise<void> {
|
|
228
|
-
state.speechActive = false;
|
|
229
|
-
const sequence = ++state.analysisSequence;
|
|
230
|
-
const probability = await this.predictor.predict(Float32Array.from(state.audio));
|
|
231
|
-
if (state.finalized || state.analysisSequence !== sequence) return;
|
|
232
|
-
|
|
233
|
-
state.boundaryAnalyzed = true;
|
|
234
|
-
state.smartTurnComplete = probability > this.probabilityThreshold;
|
|
235
|
-
|
|
236
|
-
const transcript = latestTranscript(state.finalSegments, state.latestInterim);
|
|
237
|
-
const semantic = transcript.trim()
|
|
238
|
-
? scoreSemanticCompleteness(transcript)
|
|
239
|
-
: { complete: state.smartTurnComplete, label: "complete" as const, confidence: 0 };
|
|
240
|
-
state.semanticComplete = semantic.complete;
|
|
241
|
-
|
|
242
|
-
const fusion = transcript.trim()
|
|
243
|
-
? fuseEndpointDecision(state.smartTurnComplete, semantic, this.fusionConfig())
|
|
244
|
-
: {
|
|
245
|
-
release: state.smartTurnComplete,
|
|
246
|
-
requestFinalize: state.smartTurnComplete,
|
|
247
|
-
finalizeDelayMs: this.finalizeDelayMs,
|
|
248
|
-
};
|
|
249
|
-
|
|
250
|
-
if (state.maxTimer) {
|
|
251
|
-
clearTimeout(state.maxTimer);
|
|
252
|
-
state.maxTimer = null;
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
if (fusion.deferReason) {
|
|
256
|
-
this.scheduleSemanticDefer(state);
|
|
257
|
-
return;
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
if (!fusion.release) {
|
|
261
|
-
this.scheduleIncompleteFallback(state);
|
|
262
|
-
return;
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
if (fusion.requestFinalize) {
|
|
266
|
-
this.requestSttFinalize(state.contextId);
|
|
267
|
-
}
|
|
268
|
-
if (state.finalPackets.length > 0) {
|
|
269
|
-
this.scheduleFinalize(state, fusion.finalizeDelayMs);
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
private stateFor(contextId: string): TurnState {
|
|
274
|
-
const existing = this.turns.get(contextId);
|
|
275
|
-
if (existing) return existing;
|
|
276
|
-
|
|
277
|
-
const state: TurnState = {
|
|
278
|
-
contextId,
|
|
279
|
-
audio: [],
|
|
280
|
-
finalPackets: [],
|
|
281
|
-
finalSegments: [],
|
|
282
|
-
latestInterim: "",
|
|
283
|
-
boundaryAnalyzed: false,
|
|
284
|
-
smartTurnComplete: false,
|
|
285
|
-
semanticComplete: false,
|
|
286
|
-
speechActive: false,
|
|
287
|
-
finalized: false,
|
|
288
|
-
analysisSequence: 0,
|
|
289
|
-
finalizeTimer: null,
|
|
290
|
-
sttQuietTimer: null,
|
|
291
|
-
maxTimer: null,
|
|
292
|
-
deferTimer: null,
|
|
293
|
-
};
|
|
294
|
-
this.turns.set(contextId, state);
|
|
295
|
-
return state;
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
private scheduleFinalize(state: TurnState, delayMs: number): void {
|
|
299
|
-
if (state.finalized || state.finalizeTimer) return;
|
|
300
|
-
state.finalizeTimer = setTimeout(() => {
|
|
301
|
-
state.finalizeTimer = null;
|
|
302
|
-
this.finalize(state);
|
|
303
|
-
}, delayMs);
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
private scheduleIncompleteFallback(state: TurnState): void {
|
|
307
|
-
if (state.finalized || state.finalizeTimer) return;
|
|
308
|
-
state.finalizeTimer = setTimeout(() => {
|
|
309
|
-
state.finalizeTimer = null;
|
|
310
|
-
state.smartTurnComplete = true;
|
|
311
|
-
if (state.finalPackets.length > 0) {
|
|
312
|
-
this.finalize(state);
|
|
313
|
-
return;
|
|
314
|
-
}
|
|
315
|
-
this.requestSttFinalize(state.contextId);
|
|
316
|
-
}, this.incompleteFallbackMs);
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
private scheduleSemanticShortcut(state: TurnState): void {
|
|
320
|
-
if (state.finalized || state.finalizeTimer) return;
|
|
321
|
-
state.finalizeTimer = setTimeout(() => {
|
|
322
|
-
state.finalizeTimer = null;
|
|
323
|
-
state.smartTurnComplete = true;
|
|
324
|
-
this.requestSttFinalize(state.contextId);
|
|
325
|
-
if (state.finalPackets.length > 0) {
|
|
326
|
-
this.finalize(state);
|
|
327
|
-
}
|
|
328
|
-
}, this.semanticShortcutDelayMs);
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
private scheduleSemanticDefer(state: TurnState): void {
|
|
332
|
-
if (state.finalized || state.deferTimer) return;
|
|
333
|
-
state.deferTimer = setTimeout(() => {
|
|
334
|
-
state.deferTimer = null;
|
|
335
|
-
if (state.finalized || state.speechActive) return;
|
|
336
|
-
const transcript = latestTranscript(state.finalSegments, state.latestInterim);
|
|
337
|
-
const semantic = scoreSemanticCompleteness(transcript);
|
|
338
|
-
state.smartTurnComplete = true;
|
|
339
|
-
this.requestSttFinalize(state.contextId);
|
|
340
|
-
if (state.finalPackets.length > 0) {
|
|
341
|
-
if (semantic.complete) {
|
|
342
|
-
state.semanticComplete = true;
|
|
343
|
-
this.scheduleFinalize(state, this.finalizeDelayMs);
|
|
344
|
-
} else {
|
|
345
|
-
this.finalize(state);
|
|
346
|
-
}
|
|
347
|
-
}
|
|
348
|
-
}, this.semanticDeferFallbackMs);
|
|
349
|
-
}
|
|
350
|
-
|
|
351
|
-
private fusionConfig(): SemanticEndpointFusionConfig {
|
|
352
|
-
return {
|
|
353
|
-
enabled: this.semanticEndpointingEnabled,
|
|
354
|
-
finalizeDelayMs: this.finalizeDelayMs,
|
|
355
|
-
semanticShortcutDelayMs: this.semanticShortcutDelayMs,
|
|
356
|
-
incompleteFallbackMs: this.incompleteFallbackMs,
|
|
357
|
-
};
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
private armSttQuietFallback(state: TurnState): void {
|
|
361
|
-
if (this.sttQuietFallbackMs <= 0 || state.finalized) return;
|
|
362
|
-
if (state.sttQuietTimer) clearTimeout(state.sttQuietTimer);
|
|
363
|
-
state.sttQuietTimer = setTimeout(() => {
|
|
364
|
-
state.sttQuietTimer = null;
|
|
365
|
-
if (state.finalized || !state.speechActive || state.finalPackets.length === 0) return;
|
|
366
|
-
this.bus?.push(Route.Main, {
|
|
367
|
-
kind: "metric.conversation",
|
|
368
|
-
contextId: state.contextId,
|
|
369
|
-
timestampMs: Date.now(),
|
|
370
|
-
name: "eos.stt_quiet_fallback",
|
|
371
|
-
value: String(this.sttQuietFallbackMs),
|
|
372
|
-
});
|
|
373
|
-
void this.analyzeBoundary(state);
|
|
374
|
-
}, this.sttQuietFallbackMs);
|
|
375
|
-
}
|
|
376
|
-
|
|
377
|
-
private scheduleMaxFinalize(state: TurnState): void {
|
|
378
|
-
if (state.finalized || state.maxTimer || this.maxDelayMs <= 0) return;
|
|
379
|
-
state.maxTimer = setTimeout(() => {
|
|
380
|
-
state.maxTimer = null;
|
|
381
|
-
this.finalize(state);
|
|
382
|
-
}, this.maxDelayMs);
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
private handleTtsAudio(pkt: TextToSpeechAudioPacket): void {
|
|
386
|
-
if (this.lockedContextIds.has(pkt.contextId)) {
|
|
387
|
-
this.lockedContextsWithAssistantAudio.add(pkt.contextId);
|
|
388
|
-
}
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
private handleTtsEnd(pkt: TextToSpeechEndPacket): void {
|
|
392
|
-
if (!this.lockedContextsWithAssistantAudio.has(pkt.contextId)) {
|
|
393
|
-
this.releaseContextLock(pkt.contextId);
|
|
394
|
-
}
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
private handleTtsPlayoutProgress(pkt: TextToSpeechPlayoutProgressPacket): void {
|
|
398
|
-
if (pkt.complete) this.releaseContextLock(pkt.contextId);
|
|
399
|
-
}
|
|
400
|
-
|
|
401
|
-
private releaseContextLock(contextId: string): void {
|
|
402
|
-
this.lockedContextIds.delete(contextId);
|
|
403
|
-
this.lockedContextsWithAssistantAudio.delete(contextId);
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
private finalize(state: TurnState): void {
|
|
407
|
-
const text = state.finalSegments.join(" ").replace(/\s+/g, " ").trim();
|
|
408
|
-
if (state.finalized || state.finalPackets.length === 0 || !text) return;
|
|
409
|
-
state.finalized = true;
|
|
410
|
-
this.lockedContextIds.add(state.contextId);
|
|
411
|
-
clearTurnTimers(state);
|
|
412
|
-
this.bus?.push(Route.Main, {
|
|
413
|
-
kind: "eos.turn_complete",
|
|
414
|
-
contextId: state.contextId,
|
|
415
|
-
timestampMs: Date.now(),
|
|
416
|
-
text,
|
|
417
|
-
transcripts: state.finalPackets,
|
|
418
|
-
} satisfies EndOfSpeechPacket);
|
|
419
|
-
this.turns.delete(state.contextId);
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
private requestSttFinalize(contextId: string): void {
|
|
423
|
-
this.bus?.push(Route.Critical, {
|
|
424
|
-
kind: "stt.finalize",
|
|
425
|
-
contextId,
|
|
426
|
-
timestampMs: Date.now(),
|
|
427
|
-
} satisfies FinalizeSttPacket);
|
|
428
|
-
}
|
|
429
|
-
}
|
|
430
|
-
|
|
431
|
-
function appendFinalPacket(state: TurnState, packet: SttResultPacket): void {
|
|
432
|
-
const text = packet.text.trim();
|
|
433
|
-
if (state.finalSegments.at(-1) === text) return;
|
|
434
|
-
state.finalSegments.push(text);
|
|
435
|
-
state.finalPackets.push(packet);
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
function clearTurnTimers(state: TurnState): void {
|
|
439
|
-
if (state.finalizeTimer) clearTimeout(state.finalizeTimer);
|
|
440
|
-
if (state.sttQuietTimer) clearTimeout(state.sttQuietTimer);
|
|
441
|
-
if (state.maxTimer) clearTimeout(state.maxTimer);
|
|
442
|
-
if (state.deferTimer) clearTimeout(state.deferTimer);
|
|
443
|
-
state.finalizeTimer = null;
|
|
444
|
-
state.sttQuietTimer = null;
|
|
445
|
-
state.maxTimer = null;
|
|
446
|
-
state.deferTimer = null;
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
function readBooleanConfig(value: unknown, fallback: boolean): boolean {
|
|
450
|
-
if (typeof value === "boolean") return value;
|
|
451
|
-
return fallback;
|
|
452
21
|
}
|
|
453
22
|
|
|
454
23
|
export {
|
|
@@ -466,13 +35,3 @@ export {
|
|
|
466
35
|
SmartTurnInteractionPolicy,
|
|
467
36
|
type SmartTurnInteractionPolicyConfig,
|
|
468
37
|
} from "./interaction-policy.js";
|
|
469
|
-
|
|
470
|
-
function readNonNegativeNumber(value: unknown, fallback: number): number {
|
|
471
|
-
if (typeof value !== "number" || !Number.isFinite(value)) return fallback;
|
|
472
|
-
return Math.max(0, Math.floor(value));
|
|
473
|
-
}
|
|
474
|
-
|
|
475
|
-
function readProbability(value: unknown, fallback: number): number {
|
|
476
|
-
if (typeof value !== "number" || !Number.isFinite(value)) return fallback;
|
|
477
|
-
return Math.min(1, Math.max(0, value));
|
|
478
|
-
}
|
|
@@ -22,6 +22,7 @@ const DEFAULT_MAX_AUDIO_SAMPLES = SAMPLE_RATE * 8;
|
|
|
22
22
|
interface SmartTurnState {
|
|
23
23
|
readonly contextId: string;
|
|
24
24
|
audio: number[];
|
|
25
|
+
speechMs: number;
|
|
25
26
|
finalSegments: string[];
|
|
26
27
|
latestInterim: string;
|
|
27
28
|
speechActive: boolean;
|
|
@@ -40,6 +41,7 @@ export interface SmartTurnInteractionPolicyConfig {
|
|
|
40
41
|
readonly semantic_shortcut_delay_ms?: number;
|
|
41
42
|
readonly incomplete_fallback_ms?: number;
|
|
42
43
|
readonly semantic_defer_fallback_ms?: number;
|
|
44
|
+
readonly min_speech_ms?: number;
|
|
43
45
|
readonly max_audio_samples?: number;
|
|
44
46
|
}
|
|
45
47
|
|
|
@@ -51,6 +53,7 @@ export class SmartTurnInteractionPolicy implements LifecycleInteractionPolicy {
|
|
|
51
53
|
private semanticShortcutDelayMs = 50;
|
|
52
54
|
private incompleteFallbackMs = 2000;
|
|
53
55
|
private semanticDeferFallbackMs = 4000;
|
|
56
|
+
private minSpeechMs = 0;
|
|
54
57
|
private maxAudioSamples = DEFAULT_MAX_AUDIO_SAMPLES;
|
|
55
58
|
private initialized = false;
|
|
56
59
|
|
|
@@ -63,6 +66,7 @@ export class SmartTurnInteractionPolicy implements LifecycleInteractionPolicy {
|
|
|
63
66
|
this.semanticShortcutDelayMs = readNonNegativeNumber(config["semantic_shortcut_delay_ms"], 50);
|
|
64
67
|
this.incompleteFallbackMs = readNonNegativeNumber(config["incomplete_fallback_ms"], 2000);
|
|
65
68
|
this.semanticDeferFallbackMs = readNonNegativeNumber(config["semantic_defer_fallback_ms"], 4000);
|
|
69
|
+
this.minSpeechMs = readNonNegativeNumber(config["min_speech_ms"], 0);
|
|
66
70
|
this.maxAudioSamples = readPositiveNumber(config["max_audio_samples"], DEFAULT_MAX_AUDIO_SAMPLES);
|
|
67
71
|
await this.predictor.initialize(config);
|
|
68
72
|
this.initialized = true;
|
|
@@ -72,7 +76,7 @@ export class SmartTurnInteractionPolicy implements LifecycleInteractionPolicy {
|
|
|
72
76
|
const state = this.stateFor(observation.contextId);
|
|
73
77
|
switch (observation.kind) {
|
|
74
78
|
case "audio_frame":
|
|
75
|
-
this.appendAudio(state, observation.audio);
|
|
79
|
+
this.appendAudio(state, observation.audio, observation.sampleRateHz);
|
|
76
80
|
break;
|
|
77
81
|
case "stt_partial":
|
|
78
82
|
if (observation.text.trim()) state.latestInterim = observation.text.trim();
|
|
@@ -118,6 +122,7 @@ export class SmartTurnInteractionPolicy implements LifecycleInteractionPolicy {
|
|
|
118
122
|
const state: SmartTurnState = {
|
|
119
123
|
contextId,
|
|
120
124
|
audio: [],
|
|
125
|
+
speechMs: 0,
|
|
121
126
|
finalSegments: [],
|
|
122
127
|
latestInterim: "",
|
|
123
128
|
speechActive: false,
|
|
@@ -135,6 +140,7 @@ export class SmartTurnInteractionPolicy implements LifecycleInteractionPolicy {
|
|
|
135
140
|
private beginSpeech(state: SmartTurnState): void {
|
|
136
141
|
this.clearFallback(state);
|
|
137
142
|
state.audio = [];
|
|
143
|
+
state.speechMs = 0;
|
|
138
144
|
state.finalSegments = [];
|
|
139
145
|
state.latestInterim = "";
|
|
140
146
|
state.speechActive = true;
|
|
@@ -145,9 +151,12 @@ export class SmartTurnInteractionPolicy implements LifecycleInteractionPolicy {
|
|
|
145
151
|
state.pendingDecisions = [];
|
|
146
152
|
}
|
|
147
153
|
|
|
148
|
-
private appendAudio(state: SmartTurnState, audio
|
|
154
|
+
private appendAudio(state: SmartTurnState, audio: Int16Array | undefined, sampleRateHz = SAMPLE_RATE): void {
|
|
149
155
|
if (!audio?.length) return;
|
|
150
156
|
for (const sample of audio) state.audio.push(sample / 32768);
|
|
157
|
+
if (state.speechActive && sampleRateHz > 0) {
|
|
158
|
+
state.speechMs += (audio.length / sampleRateHz) * 1000;
|
|
159
|
+
}
|
|
151
160
|
const overflow = state.audio.length - this.maxAudioSamples;
|
|
152
161
|
if (overflow > 0) state.audio.splice(0, overflow);
|
|
153
162
|
}
|
|
@@ -177,13 +186,16 @@ export class SmartTurnInteractionPolicy implements LifecycleInteractionPolicy {
|
|
|
177
186
|
const smartTurnComplete = state.probability > this.probabilityThreshold;
|
|
178
187
|
const transcript = latestTranscript(state.finalSegments, state.latestInterim);
|
|
179
188
|
if (!transcript) {
|
|
180
|
-
if (smartTurnComplete
|
|
181
|
-
|
|
189
|
+
if (smartTurnComplete && state.speechMs >= this.minSpeechMs) {
|
|
190
|
+
this.issueTakeTurn(state, state.probability);
|
|
191
|
+
} else {
|
|
192
|
+
this.scheduleFallback(state, this.incompleteFallbackMs);
|
|
193
|
+
}
|
|
182
194
|
return;
|
|
183
195
|
}
|
|
184
196
|
|
|
185
197
|
const semantic = scoreSemanticCompleteness(transcript);
|
|
186
|
-
const fusion = fuseEndpointDecision(smartTurnComplete, semantic, this.fusionConfig());
|
|
198
|
+
const fusion = fuseEndpointDecision(smartTurnComplete, semantic, state.speechMs, this.fusionConfig());
|
|
187
199
|
if (fusion.release) {
|
|
188
200
|
const confidence = smartTurnComplete ? Math.max(state.probability, semantic.confidence) : semantic.confidence;
|
|
189
201
|
this.issueTakeTurn(state, confidence);
|
|
@@ -237,6 +249,7 @@ export class SmartTurnInteractionPolicy implements LifecycleInteractionPolicy {
|
|
|
237
249
|
finalizeDelayMs: this.finalizeDelayMs,
|
|
238
250
|
semanticShortcutDelayMs: this.semanticShortcutDelayMs,
|
|
239
251
|
incompleteFallbackMs: this.incompleteFallbackMs,
|
|
252
|
+
minSpeechMs: this.minSpeechMs,
|
|
240
253
|
};
|
|
241
254
|
}
|
|
242
255
|
}
|
package/src/predictor.ts
CHANGED
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
|
|
5
5
|
import { optionalStringConfig, type PluginConfig } from "@kuralle-syrinx/core";
|
|
6
|
+
import type { SmartTurnPredictor } from "./smart-turn-types.js";
|
|
7
|
+
|
|
8
|
+
export type { SmartTurnPredictor } from "./smart-turn-types.js";
|
|
6
9
|
|
|
7
10
|
type Ort = typeof import("onnxruntime-node");
|
|
8
11
|
type InferenceSession = import("onnxruntime-node").InferenceSession;
|
|
@@ -15,12 +18,6 @@ const SAMPLE_RATE = 16000;
|
|
|
15
18
|
const MAX_AUDIO_SAMPLES = SAMPLE_RATE * 8;
|
|
16
19
|
const DEFAULT_MODEL_PATH = fileURLToPath(new URL("../models/smart-turn-v3.2-cpu.onnx", import.meta.url));
|
|
17
20
|
|
|
18
|
-
export interface SmartTurnPredictor {
|
|
19
|
-
initialize(config: PluginConfig): Promise<void>;
|
|
20
|
-
predict(audio: Float32Array): Promise<number>;
|
|
21
|
-
close(): Promise<void>;
|
|
22
|
-
}
|
|
23
|
-
|
|
24
21
|
export class LocalSmartTurnV3Predictor implements SmartTurnPredictor {
|
|
25
22
|
private ort: Ort | null = null;
|
|
26
23
|
private session: InferenceSession | null = null;
|
|
@@ -11,6 +11,7 @@ export interface SemanticEndpointFusionConfig {
|
|
|
11
11
|
readonly finalizeDelayMs: number;
|
|
12
12
|
readonly semanticShortcutDelayMs: number;
|
|
13
13
|
readonly incompleteFallbackMs: number;
|
|
14
|
+
readonly minSpeechMs?: number;
|
|
14
15
|
}
|
|
15
16
|
|
|
16
17
|
export interface EndpointFusionDecision {
|
|
@@ -111,6 +112,7 @@ export function scoreSemanticCompleteness(text: string): SemanticCompletenessSco
|
|
|
111
112
|
export function fuseEndpointDecision(
|
|
112
113
|
smartTurnComplete: boolean,
|
|
113
114
|
semantic: SemanticCompletenessScore,
|
|
115
|
+
speechMs: number,
|
|
114
116
|
config: SemanticEndpointFusionConfig,
|
|
115
117
|
): EndpointFusionDecision {
|
|
116
118
|
if (!config.enabled) {
|
|
@@ -121,7 +123,7 @@ export function fuseEndpointDecision(
|
|
|
121
123
|
};
|
|
122
124
|
}
|
|
123
125
|
|
|
124
|
-
if (smartTurnComplete && semantic.complete) {
|
|
126
|
+
if (smartTurnComplete && semantic.complete && speechMs >= (config.minSpeechMs ?? 0)) {
|
|
125
127
|
return {
|
|
126
128
|
release: true,
|
|
127
129
|
requestFinalize: true,
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
|
|
3
|
+
import type { PluginConfig } from "@kuralle-syrinx/core";
|
|
4
|
+
|
|
5
|
+
/** Host- or model-agnostic Smart Turn probability scorer. */
|
|
6
|
+
export interface SmartTurnPredictor {
|
|
7
|
+
initialize(config: PluginConfig): Promise<void>;
|
|
8
|
+
predict(audio: Float32Array): Promise<number>;
|
|
9
|
+
close(): Promise<void>;
|
|
10
|
+
}
|