@kuralle-syrinx/pipecat-smart-turn 4.1.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -0
- package/package.json +28 -4
- package/src/eos-plugin.ts +564 -0
- package/src/index.ts +15 -515
- package/src/interaction-policy.ts +278 -0
- package/src/predictor.ts +74 -0
- package/src/semantic-completeness.ts +3 -1
- package/src/smart-turn-types.ts +10 -0
- package/src/index.test.ts +0 -803
- package/src/semantic-completeness.test.ts +0 -164
- package/tsconfig.json +0 -21
package/README.md
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# @kuralle-syrinx/pipecat-smart-turn
|
|
2
|
+
|
|
3
|
+
Smart Turn v3 endpointing for Syrinx, fused with transcript-level semantic completeness.
|
|
4
|
+
|
|
5
|
+
## InteractionPolicy selection
|
|
6
|
+
|
|
7
|
+
Use `SmartTurnInteractionPolicy` when the session policy seam should own endpointing. Register Silero VAD
|
|
8
|
+
and provider STT as usual, but do not register `PipecatEOSPlugin`; the session disables provider-owned EOS
|
|
9
|
+
and routes the policy's `take_turn` through STT finalization.
|
|
10
|
+
|
|
11
|
+
```ts
|
|
12
|
+
const session = new VoiceAgentSession({
|
|
13
|
+
plugins: pluginConfig,
|
|
14
|
+
interactionPolicy: new SmartTurnInteractionPolicy(),
|
|
15
|
+
interactionPolicyConfig: pluginConfig.eos,
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
session.registerPlugin("stt", new DeepgramSTTPlugin());
|
|
19
|
+
session.registerPlugin("vad", new SileroVADPlugin());
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
The policy consumes session `audio_frame`, Silero speech-boundary, STT, and playout observations. Acoustic
|
|
23
|
+
and semantic completion confidence maps monotonically to a 150-2000 ms wait before turn commit. The model
|
|
24
|
+
inference is asynchronous; `observe()` remains synchronous and is gated at p99 <= 5 ms/frame.
|
|
25
|
+
|
|
26
|
+
`PipecatEOSPlugin` remains available for existing plugin-based sessions.
|
package/package.json
CHANGED
|
@@ -1,19 +1,43 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kuralle-syrinx/pipecat-smart-turn",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.3.0",
|
|
4
4
|
"private": false,
|
|
5
|
-
"
|
|
5
|
+
"description": "Pipecat Smart Turn v3 semantic endpointing for Syrinx — local ONNX end-of-turn analysis",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"voice",
|
|
8
|
+
"voice-agent",
|
|
9
|
+
"speech",
|
|
10
|
+
"syrinx",
|
|
11
|
+
"pipecat",
|
|
12
|
+
"turn-taking",
|
|
13
|
+
"endpointing",
|
|
14
|
+
"onnx"
|
|
15
|
+
],
|
|
6
16
|
"license": "MIT",
|
|
17
|
+
"homepage": "https://github.com/kuralle/syrinx#readme",
|
|
18
|
+
"repository": {
|
|
19
|
+
"type": "git",
|
|
20
|
+
"url": "git+https://github.com/kuralle/syrinx.git",
|
|
21
|
+
"directory": "packages/pipecat-smart-turn"
|
|
22
|
+
},
|
|
23
|
+
"bugs": {
|
|
24
|
+
"url": "https://github.com/kuralle/syrinx/issues"
|
|
25
|
+
},
|
|
26
|
+
"type": "module",
|
|
7
27
|
"main": "./src/index.ts",
|
|
8
28
|
"types": "./src/index.ts",
|
|
29
|
+
"exports": {
|
|
30
|
+
".": "./src/index.ts",
|
|
31
|
+
"./eos": "./src/eos-plugin.ts"
|
|
32
|
+
},
|
|
9
33
|
"dependencies": {
|
|
10
34
|
"@huggingface/transformers": "^4.2.0",
|
|
11
35
|
"onnxruntime-node": "1.24.3",
|
|
12
|
-
"@kuralle-syrinx/core": "4.
|
|
36
|
+
"@kuralle-syrinx/core": "4.3.0"
|
|
13
37
|
},
|
|
14
38
|
"devDependencies": {
|
|
15
39
|
"typescript": "^5.7.0",
|
|
16
|
-
"vitest": "^2.
|
|
40
|
+
"vitest": "^3.2.6"
|
|
17
41
|
},
|
|
18
42
|
"scripts": {
|
|
19
43
|
"typecheck": "tsc --noEmit",
|
|
@@ -0,0 +1,564 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
//
|
|
3
|
+
// Edge-safe Pipecat Smart Turn EOS plugin.
|
|
4
|
+
// Consumes an injected SmartTurnPredictor — does NOT import LocalSmartTurnV3
|
|
5
|
+
// or onnxruntime-node. Workers hosts import this subpath (`/eos`).
|
|
6
|
+
|
|
7
|
+
import {
|
|
8
|
+
fuseEndpointDecision,
|
|
9
|
+
latestTranscript,
|
|
10
|
+
scoreSemanticCompleteness,
|
|
11
|
+
type SemanticEndpointFusionConfig,
|
|
12
|
+
} from "./semantic-completeness.js";
|
|
13
|
+
import {
|
|
14
|
+
Route,
|
|
15
|
+
type EndOfSpeechPacket,
|
|
16
|
+
type FinalizeSttPacket,
|
|
17
|
+
type InterruptionDetectedPacket,
|
|
18
|
+
type InterimEndOfSpeechPacket,
|
|
19
|
+
type PipelineBus,
|
|
20
|
+
type PluginConfig,
|
|
21
|
+
type SttInterimPacket,
|
|
22
|
+
type SttResultPacket,
|
|
23
|
+
type TextToSpeechAudioPacket,
|
|
24
|
+
type TextToSpeechEndPacket,
|
|
25
|
+
type TextToSpeechPlayoutProgressPacket,
|
|
26
|
+
type VadAudioPacket,
|
|
27
|
+
type VadSpeechEndedPacket,
|
|
28
|
+
type VadSpeechStartedPacket,
|
|
29
|
+
type VoicePlugin,
|
|
30
|
+
} from "@kuralle-syrinx/core";
|
|
31
|
+
import { pcm16BytesToSamples } from "@kuralle-syrinx/core/audio";
|
|
32
|
+
import type { SmartTurnPredictor } from "./smart-turn-types.js";
|
|
33
|
+
|
|
34
|
+
export type { SmartTurnPredictor } from "./smart-turn-types.js";
|
|
35
|
+
|
|
36
|
+
const SAMPLE_RATE = 16000;
|
|
37
|
+
const MAX_AUDIO_SAMPLES = SAMPLE_RATE * 8;
|
|
38
|
+
/** Grace added to the estimated playout end before the fallback lock-release fires. */
|
|
39
|
+
const LOCK_RELEASE_GRACE_MS = 500;
|
|
40
|
+
|
|
41
|
+
interface TurnState {
|
|
42
|
+
readonly contextId: string;
|
|
43
|
+
audio: number[];
|
|
44
|
+
speechMs: number;
|
|
45
|
+
finalPackets: SttResultPacket[];
|
|
46
|
+
finalSegments: string[];
|
|
47
|
+
latestInterim: string;
|
|
48
|
+
boundaryAnalyzed: boolean;
|
|
49
|
+
smartTurnComplete: boolean;
|
|
50
|
+
semanticComplete: boolean;
|
|
51
|
+
speechActive: boolean;
|
|
52
|
+
finalized: boolean;
|
|
53
|
+
analysisSequence: number;
|
|
54
|
+
finalizeTimer: ReturnType<typeof setTimeout> | null;
|
|
55
|
+
sttQuietTimer: ReturnType<typeof setTimeout> | null;
|
|
56
|
+
maxTimer: ReturnType<typeof setTimeout> | null;
|
|
57
|
+
deferTimer: ReturnType<typeof setTimeout> | null;
|
|
58
|
+
/** Absolute per-turn cap; never cleared by clearTurnTimers (survives speech restarts). */
|
|
59
|
+
absoluteTimer: ReturnType<typeof setTimeout> | null;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export class PipecatEOSPlugin implements VoicePlugin {
|
|
63
|
+
readonly endpointingCapability = { owner: "smart_turn" as const };
|
|
64
|
+
|
|
65
|
+
private bus: PipelineBus | null = null;
|
|
66
|
+
private disposers: Array<() => void> = [];
|
|
67
|
+
private turns = new Map<string, TurnState>();
|
|
68
|
+
private finalizeDelayMs = 250;
|
|
69
|
+
private sttQuietFallbackMs = 2500;
|
|
70
|
+
private maxDelayMs = 2000;
|
|
71
|
+
private incompleteFallbackMs = 2000;
|
|
72
|
+
private semanticShortcutDelayMs = 50;
|
|
73
|
+
private semanticDeferFallbackMs = 4000;
|
|
74
|
+
private minSpeechMs = 0;
|
|
75
|
+
private maxTurnDurationMs = 15000;
|
|
76
|
+
private semanticEndpointingEnabled = true;
|
|
77
|
+
private probabilityThreshold = 0.5;
|
|
78
|
+
private readonly lockedContextIds = new Set<string>();
|
|
79
|
+
private readonly lockedContextsWithAssistantAudio = new Set<string>();
|
|
80
|
+
/**
|
|
81
|
+
* Playout estimate + fallback release timer for locked contexts that produced assistant
|
|
82
|
+
* audio. The lock is normally released by `tts.playout_progress {complete}` (a client
|
|
83
|
+
* signal), but some clients (e.g. the browser studio) never emit it — without this
|
|
84
|
+
* fallback the lock persists and the NEXT turn stalls forever. We estimate when the
|
|
85
|
+
* assistant audio should have finished playing (from the emitted bytes) and release then.
|
|
86
|
+
*/
|
|
87
|
+
private readonly lockedPlayout = new Map<
|
|
88
|
+
string,
|
|
89
|
+
{ audioMs: number; firstAudioAtMs: number; timer: ReturnType<typeof setTimeout> | null }
|
|
90
|
+
>();
|
|
91
|
+
|
|
92
|
+
constructor(private readonly predictor: SmartTurnPredictor) {}
|
|
93
|
+
|
|
94
|
+
async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
|
|
95
|
+
this.bus = bus;
|
|
96
|
+
this.finalizeDelayMs = readNonNegativeNumber(config["finalize_delay_ms"], 250);
|
|
97
|
+
this.sttQuietFallbackMs = readNonNegativeNumber(config["stt_quiet_fallback_ms"], 2500);
|
|
98
|
+
this.maxDelayMs = readNonNegativeNumber(config["max_delay_ms"], 2000);
|
|
99
|
+
this.incompleteFallbackMs = readNonNegativeNumber(config["incomplete_fallback_ms"], 2000);
|
|
100
|
+
this.semanticShortcutDelayMs = readNonNegativeNumber(config["semantic_shortcut_delay_ms"], 50);
|
|
101
|
+
this.semanticDeferFallbackMs = readNonNegativeNumber(config["semantic_defer_fallback_ms"], 4000);
|
|
102
|
+
this.minSpeechMs = readNonNegativeNumber(config["min_speech_ms"], 0);
|
|
103
|
+
this.maxTurnDurationMs = readNonNegativeNumber(config["max_turn_duration_ms"], 15000);
|
|
104
|
+
this.semanticEndpointingEnabled = readBooleanConfig(config["semantic_endpointing_enabled"], true);
|
|
105
|
+
this.probabilityThreshold = readProbability(config["probability_threshold"], 0.5);
|
|
106
|
+
await this.predictor.initialize(config);
|
|
107
|
+
|
|
108
|
+
this.disposers.push(
|
|
109
|
+
bus.on("vad.audio", (pkt) => {
|
|
110
|
+
this.handleAudio(pkt as VadAudioPacket);
|
|
111
|
+
}),
|
|
112
|
+
bus.on("stt.interim", (pkt) => {
|
|
113
|
+
this.handleInterim(pkt as SttInterimPacket);
|
|
114
|
+
}),
|
|
115
|
+
bus.on("stt.result", (pkt) => {
|
|
116
|
+
this.handleFinal(pkt as SttResultPacket);
|
|
117
|
+
}),
|
|
118
|
+
bus.on("vad.speech_started", (pkt) => {
|
|
119
|
+
this.handleSpeechStarted(pkt as VadSpeechStartedPacket);
|
|
120
|
+
}),
|
|
121
|
+
bus.on("vad.speech_ended", async (pkt) => {
|
|
122
|
+
await this.handleSpeechEnded(pkt as VadSpeechEndedPacket);
|
|
123
|
+
}),
|
|
124
|
+
bus.on("tts.audio", (pkt) => {
|
|
125
|
+
this.handleTtsAudio(pkt as TextToSpeechAudioPacket);
|
|
126
|
+
}),
|
|
127
|
+
bus.on("tts.end", (pkt) => {
|
|
128
|
+
this.handleTtsEnd(pkt as TextToSpeechEndPacket);
|
|
129
|
+
}),
|
|
130
|
+
bus.on("tts.playout_progress", (pkt) => {
|
|
131
|
+
this.handleTtsPlayoutProgress(pkt as TextToSpeechPlayoutProgressPacket);
|
|
132
|
+
}),
|
|
133
|
+
bus.on("interrupt.detected", (pkt) => {
|
|
134
|
+
this.releaseContextLock((pkt as InterruptionDetectedPacket).contextId);
|
|
135
|
+
}),
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async close(): Promise<void> {
|
|
140
|
+
for (const dispose of this.disposers.splice(0)) {
|
|
141
|
+
dispose();
|
|
142
|
+
}
|
|
143
|
+
for (const state of this.turns.values()) {
|
|
144
|
+
clearTurnTimers(state);
|
|
145
|
+
this.clearAbsoluteCap(state);
|
|
146
|
+
}
|
|
147
|
+
this.turns.clear();
|
|
148
|
+
for (const playout of this.lockedPlayout.values()) {
|
|
149
|
+
if (playout.timer) clearTimeout(playout.timer);
|
|
150
|
+
}
|
|
151
|
+
this.lockedPlayout.clear();
|
|
152
|
+
this.lockedContextIds.clear();
|
|
153
|
+
this.lockedContextsWithAssistantAudio.clear();
|
|
154
|
+
await this.predictor.close();
|
|
155
|
+
this.bus = null;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
private handleAudio(pkt: VadAudioPacket): void {
|
|
159
|
+
if (this.lockedContextIds.has(pkt.contextId)) return;
|
|
160
|
+
if (pkt.audio.byteLength % 2 !== 0) return;
|
|
161
|
+
const state = this.stateFor(pkt.contextId);
|
|
162
|
+
const samples = pcm16BytesToSamples(pkt.audio);
|
|
163
|
+
for (const sample of samples) {
|
|
164
|
+
state.audio.push(sample / 32768);
|
|
165
|
+
}
|
|
166
|
+
if (state.speechActive) {
|
|
167
|
+
state.speechMs += (samples.length / SAMPLE_RATE) * 1000;
|
|
168
|
+
}
|
|
169
|
+
if (state.audio.length > MAX_AUDIO_SAMPLES) {
|
|
170
|
+
state.audio.splice(0, state.audio.length - MAX_AUDIO_SAMPLES);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
private handleInterim(pkt: SttInterimPacket): void {
|
|
175
|
+
if (this.lockedContextIds.has(pkt.contextId)) return;
|
|
176
|
+
if (!pkt.text.trim()) return;
|
|
177
|
+
const state = this.stateFor(pkt.contextId);
|
|
178
|
+
state.latestInterim = pkt.text.trim();
|
|
179
|
+
if (state.sttQuietTimer) this.armSttQuietFallback(state);
|
|
180
|
+
this.bus?.push(Route.Main, {
|
|
181
|
+
kind: "eos.interim",
|
|
182
|
+
contextId: pkt.contextId,
|
|
183
|
+
timestampMs: Date.now(),
|
|
184
|
+
text: pkt.text,
|
|
185
|
+
} satisfies InterimEndOfSpeechPacket);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
private handleFinal(pkt: SttResultPacket): void {
|
|
189
|
+
if (this.lockedContextIds.has(pkt.contextId)) return;
|
|
190
|
+
if (!pkt.text.trim()) return;
|
|
191
|
+
const state = this.stateFor(pkt.contextId);
|
|
192
|
+
appendFinalPacket(state, pkt);
|
|
193
|
+
state.latestInterim = "";
|
|
194
|
+
|
|
195
|
+
const transcript = latestTranscript(state.finalSegments, state.latestInterim);
|
|
196
|
+
const semantic = scoreSemanticCompleteness(transcript);
|
|
197
|
+
state.semanticComplete = semantic.complete;
|
|
198
|
+
|
|
199
|
+
if (
|
|
200
|
+
state.boundaryAnalyzed &&
|
|
201
|
+
state.smartTurnComplete &&
|
|
202
|
+
state.semanticComplete &&
|
|
203
|
+
state.speechMs >= this.minSpeechMs
|
|
204
|
+
) {
|
|
205
|
+
this.scheduleFinalize(state, this.finalizeDelayMs);
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
if (state.smartTurnComplete && state.semanticComplete && state.speechMs >= this.minSpeechMs) {
|
|
209
|
+
this.scheduleFinalize(state, this.finalizeDelayMs);
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
if (state.boundaryAnalyzed && state.smartTurnComplete && !state.semanticComplete) {
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
if (state.boundaryAnalyzed && !state.smartTurnComplete && state.semanticComplete) {
|
|
216
|
+
this.scheduleSemanticShortcut(state);
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
if (state.boundaryAnalyzed) {
|
|
220
|
+
this.scheduleIncompleteFallback(state);
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
if (state.speechActive) {
|
|
224
|
+
this.armSttQuietFallback(state);
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
this.scheduleMaxFinalize(state);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
private handleSpeechStarted(pkt: VadSpeechStartedPacket): void {
|
|
231
|
+
if (this.lockedContextIds.has(pkt.contextId)) return;
|
|
232
|
+
const state = this.stateFor(pkt.contextId);
|
|
233
|
+
const startsTurn = !state.boundaryAnalyzed && state.finalPackets.length === 0;
|
|
234
|
+
if (state.sttQuietTimer) {
|
|
235
|
+
clearTimeout(state.sttQuietTimer);
|
|
236
|
+
state.sttQuietTimer = null;
|
|
237
|
+
}
|
|
238
|
+
if (startsTurn) state.speechMs = 0;
|
|
239
|
+
state.boundaryAnalyzed = false;
|
|
240
|
+
state.smartTurnComplete = false;
|
|
241
|
+
state.semanticComplete = false;
|
|
242
|
+
state.speechActive = true;
|
|
243
|
+
state.latestInterim = "";
|
|
244
|
+
state.analysisSequence += 1;
|
|
245
|
+
clearTurnTimers(state);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
private async handleSpeechEnded(pkt: VadSpeechEndedPacket): Promise<void> {
|
|
249
|
+
if (this.lockedContextIds.has(pkt.contextId)) return;
|
|
250
|
+
const state = this.stateFor(pkt.contextId);
|
|
251
|
+
if (state.sttQuietTimer) {
|
|
252
|
+
clearTimeout(state.sttQuietTimer);
|
|
253
|
+
state.sttQuietTimer = null;
|
|
254
|
+
}
|
|
255
|
+
await this.analyzeBoundary(state);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// Shared end-of-speech boundary analysis: used by the VAD speech_ended path
|
|
259
|
+
// and by the STT-quiet fallback (when the provider transcript has gone quiet
|
|
260
|
+
// but the VAD never closed the segment — e.g. model state saturation on long
|
|
261
|
+
// telephony audio). The turn must never be held hostage by a wedged VAD.
|
|
262
|
+
private async analyzeBoundary(state: TurnState): Promise<void> {
|
|
263
|
+
state.speechActive = false;
|
|
264
|
+
const sequence = ++state.analysisSequence;
|
|
265
|
+
const probability = await this.predictor.predict(Float32Array.from(state.audio));
|
|
266
|
+
if (state.finalized || state.analysisSequence !== sequence) return;
|
|
267
|
+
|
|
268
|
+
state.boundaryAnalyzed = true;
|
|
269
|
+
state.smartTurnComplete = probability > this.probabilityThreshold;
|
|
270
|
+
|
|
271
|
+
const transcript = latestTranscript(state.finalSegments, state.latestInterim);
|
|
272
|
+
const semantic = transcript.trim()
|
|
273
|
+
? scoreSemanticCompleteness(transcript)
|
|
274
|
+
: { complete: state.smartTurnComplete, label: "complete" as const, confidence: 0 };
|
|
275
|
+
state.semanticComplete = semantic.complete;
|
|
276
|
+
|
|
277
|
+
const fusion = transcript.trim()
|
|
278
|
+
? fuseEndpointDecision(state.smartTurnComplete, semantic, state.speechMs, this.fusionConfig())
|
|
279
|
+
: {
|
|
280
|
+
release: state.smartTurnComplete && state.speechMs >= this.minSpeechMs,
|
|
281
|
+
requestFinalize: state.smartTurnComplete && state.speechMs >= this.minSpeechMs,
|
|
282
|
+
finalizeDelayMs: this.finalizeDelayMs,
|
|
283
|
+
};
|
|
284
|
+
|
|
285
|
+
if (state.maxTimer) {
|
|
286
|
+
clearTimeout(state.maxTimer);
|
|
287
|
+
state.maxTimer = null;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
if (fusion.deferReason) {
|
|
291
|
+
this.scheduleSemanticDefer(state);
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
if (!fusion.release) {
|
|
296
|
+
this.scheduleIncompleteFallback(state);
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
if (fusion.requestFinalize) {
|
|
301
|
+
this.requestSttFinalize(state.contextId);
|
|
302
|
+
}
|
|
303
|
+
if (state.finalPackets.length > 0) {
|
|
304
|
+
this.scheduleFinalize(state, fusion.finalizeDelayMs);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
private stateFor(contextId: string): TurnState {
|
|
309
|
+
const existing = this.turns.get(contextId);
|
|
310
|
+
if (existing) return existing;
|
|
311
|
+
|
|
312
|
+
const state: TurnState = {
|
|
313
|
+
contextId,
|
|
314
|
+
audio: [],
|
|
315
|
+
speechMs: 0,
|
|
316
|
+
finalPackets: [],
|
|
317
|
+
finalSegments: [],
|
|
318
|
+
latestInterim: "",
|
|
319
|
+
boundaryAnalyzed: false,
|
|
320
|
+
smartTurnComplete: false,
|
|
321
|
+
semanticComplete: false,
|
|
322
|
+
speechActive: false,
|
|
323
|
+
finalized: false,
|
|
324
|
+
analysisSequence: 0,
|
|
325
|
+
finalizeTimer: null,
|
|
326
|
+
sttQuietTimer: null,
|
|
327
|
+
maxTimer: null,
|
|
328
|
+
deferTimer: null,
|
|
329
|
+
absoluteTimer: null,
|
|
330
|
+
};
|
|
331
|
+
this.turns.set(contextId, state);
|
|
332
|
+
this.scheduleAbsoluteCap(state);
|
|
333
|
+
return state;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// Armed once at turn onset; never rescheduled so continuous noise cannot starve the ceiling.
|
|
337
|
+
private scheduleAbsoluteCap(state: TurnState): void {
|
|
338
|
+
if (state.finalized || state.absoluteTimer || this.maxTurnDurationMs <= 0) return;
|
|
339
|
+
state.absoluteTimer = setTimeout(() => {
|
|
340
|
+
state.absoluteTimer = null;
|
|
341
|
+
this.onAbsoluteCap(state);
|
|
342
|
+
}, this.maxTurnDurationMs);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
private clearAbsoluteCap(state: TurnState): void {
|
|
346
|
+
if (state.absoluteTimer) clearTimeout(state.absoluteTimer);
|
|
347
|
+
state.absoluteTimer = null;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
private onAbsoluteCap(state: TurnState): void {
|
|
351
|
+
if (state.finalized) return;
|
|
352
|
+
const transcript = latestTranscript(state.finalSegments, state.latestInterim);
|
|
353
|
+
if (!transcript.trim() && state.finalPackets.length === 0) {
|
|
354
|
+
clearTurnTimers(state);
|
|
355
|
+
this.clearAbsoluteCap(state);
|
|
356
|
+
this.turns.delete(state.contextId);
|
|
357
|
+
return;
|
|
358
|
+
}
|
|
359
|
+
state.boundaryAnalyzed = true;
|
|
360
|
+
state.smartTurnComplete = true;
|
|
361
|
+
state.semanticComplete = true;
|
|
362
|
+
this.requestSttFinalize(state.contextId);
|
|
363
|
+
if (state.finalPackets.length > 0) {
|
|
364
|
+
this.finalize(state);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
private scheduleFinalize(state: TurnState, delayMs: number): void {
|
|
369
|
+
if (state.finalized || state.finalizeTimer) return;
|
|
370
|
+
state.finalizeTimer = setTimeout(() => {
|
|
371
|
+
state.finalizeTimer = null;
|
|
372
|
+
this.finalize(state);
|
|
373
|
+
}, delayMs);
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
private scheduleIncompleteFallback(state: TurnState): void {
|
|
377
|
+
if (state.finalized || state.finalizeTimer) return;
|
|
378
|
+
state.finalizeTimer = setTimeout(() => {
|
|
379
|
+
state.finalizeTimer = null;
|
|
380
|
+
state.smartTurnComplete = true;
|
|
381
|
+
if (state.finalPackets.length > 0) {
|
|
382
|
+
this.finalize(state);
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
this.requestSttFinalize(state.contextId);
|
|
386
|
+
}, this.incompleteFallbackMs);
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
private scheduleSemanticShortcut(state: TurnState): void {
|
|
390
|
+
if (state.finalized || state.finalizeTimer) return;
|
|
391
|
+
state.finalizeTimer = setTimeout(() => {
|
|
392
|
+
state.finalizeTimer = null;
|
|
393
|
+
state.smartTurnComplete = true;
|
|
394
|
+
this.requestSttFinalize(state.contextId);
|
|
395
|
+
if (state.finalPackets.length > 0) {
|
|
396
|
+
this.finalize(state);
|
|
397
|
+
}
|
|
398
|
+
}, this.semanticShortcutDelayMs);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
private scheduleSemanticDefer(state: TurnState): void {
|
|
402
|
+
if (state.finalized || state.deferTimer) return;
|
|
403
|
+
state.deferTimer = setTimeout(() => {
|
|
404
|
+
state.deferTimer = null;
|
|
405
|
+
if (state.finalized || state.speechActive) return;
|
|
406
|
+
const transcript = latestTranscript(state.finalSegments, state.latestInterim);
|
|
407
|
+
const semantic = scoreSemanticCompleteness(transcript);
|
|
408
|
+
state.smartTurnComplete = true;
|
|
409
|
+
this.requestSttFinalize(state.contextId);
|
|
410
|
+
if (state.finalPackets.length > 0) {
|
|
411
|
+
if (semantic.complete) {
|
|
412
|
+
state.semanticComplete = true;
|
|
413
|
+
this.scheduleFinalize(state, this.finalizeDelayMs);
|
|
414
|
+
} else {
|
|
415
|
+
this.finalize(state);
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
}, this.semanticDeferFallbackMs);
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
private fusionConfig(): SemanticEndpointFusionConfig {
|
|
422
|
+
return {
|
|
423
|
+
enabled: this.semanticEndpointingEnabled,
|
|
424
|
+
finalizeDelayMs: this.finalizeDelayMs,
|
|
425
|
+
semanticShortcutDelayMs: this.semanticShortcutDelayMs,
|
|
426
|
+
incompleteFallbackMs: this.incompleteFallbackMs,
|
|
427
|
+
minSpeechMs: this.minSpeechMs,
|
|
428
|
+
};
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
private armSttQuietFallback(state: TurnState): void {
|
|
432
|
+
if (this.sttQuietFallbackMs <= 0 || state.finalized) return;
|
|
433
|
+
if (state.sttQuietTimer) clearTimeout(state.sttQuietTimer);
|
|
434
|
+
state.sttQuietTimer = setTimeout(() => {
|
|
435
|
+
state.sttQuietTimer = null;
|
|
436
|
+
if (state.finalized || !state.speechActive || state.finalPackets.length === 0) return;
|
|
437
|
+
this.bus?.push(Route.Main, {
|
|
438
|
+
kind: "metric.conversation",
|
|
439
|
+
contextId: state.contextId,
|
|
440
|
+
timestampMs: Date.now(),
|
|
441
|
+
name: "eos.stt_quiet_fallback",
|
|
442
|
+
value: String(this.sttQuietFallbackMs),
|
|
443
|
+
});
|
|
444
|
+
void this.analyzeBoundary(state);
|
|
445
|
+
}, this.sttQuietFallbackMs);
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
private scheduleMaxFinalize(state: TurnState): void {
|
|
449
|
+
if (state.finalized || state.maxTimer || this.maxDelayMs <= 0) return;
|
|
450
|
+
state.maxTimer = setTimeout(() => {
|
|
451
|
+
state.maxTimer = null;
|
|
452
|
+
this.finalize(state);
|
|
453
|
+
}, this.maxDelayMs);
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
private handleTtsAudio(pkt: TextToSpeechAudioPacket): void {
|
|
457
|
+
if (!this.lockedContextIds.has(pkt.contextId)) return;
|
|
458
|
+
this.lockedContextsWithAssistantAudio.add(pkt.contextId);
|
|
459
|
+
const sampleRate = pkt.sampleRateHz > 0 ? pkt.sampleRateHz : SAMPLE_RATE;
|
|
460
|
+
const chunkMs = (pkt.audio.byteLength / 2 / sampleRate) * 1000;
|
|
461
|
+
const existing = this.lockedPlayout.get(pkt.contextId);
|
|
462
|
+
if (existing) {
|
|
463
|
+
existing.audioMs += chunkMs;
|
|
464
|
+
} else {
|
|
465
|
+
this.lockedPlayout.set(pkt.contextId, {
|
|
466
|
+
audioMs: chunkMs,
|
|
467
|
+
firstAudioAtMs: Date.now(),
|
|
468
|
+
timer: null,
|
|
469
|
+
});
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
private handleTtsEnd(pkt: TextToSpeechEndPacket): void {
|
|
474
|
+
if (!this.lockedContextsWithAssistantAudio.has(pkt.contextId)) {
|
|
475
|
+
this.releaseContextLock(pkt.contextId);
|
|
476
|
+
return;
|
|
477
|
+
}
|
|
478
|
+
// The lock is released by `tts.playout_progress {complete}` when the client sends it.
|
|
479
|
+
// Fallback for clients that never do: release once the assistant audio should have
|
|
480
|
+
// finished playing (estimated from the emitted bytes), so the next turn isn't stalled.
|
|
481
|
+
const contextId = pkt.contextId;
|
|
482
|
+
const playout = this.lockedPlayout.get(contextId);
|
|
483
|
+
const remainingMs = playout
|
|
484
|
+
? Math.max(0, playout.firstAudioAtMs + playout.audioMs - Date.now())
|
|
485
|
+
: 0;
|
|
486
|
+
const timer = setTimeout(() => this.releaseContextLock(contextId), remainingMs + LOCK_RELEASE_GRACE_MS);
|
|
487
|
+
if (playout) {
|
|
488
|
+
if (playout.timer) clearTimeout(playout.timer);
|
|
489
|
+
playout.timer = timer;
|
|
490
|
+
} else {
|
|
491
|
+
this.lockedPlayout.set(contextId, { audioMs: 0, firstAudioAtMs: Date.now(), timer });
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
private handleTtsPlayoutProgress(pkt: TextToSpeechPlayoutProgressPacket): void {
|
|
496
|
+
if (pkt.complete) this.releaseContextLock(pkt.contextId);
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
private releaseContextLock(contextId: string): void {
|
|
500
|
+
this.lockedContextIds.delete(contextId);
|
|
501
|
+
this.lockedContextsWithAssistantAudio.delete(contextId);
|
|
502
|
+
const playout = this.lockedPlayout.get(contextId);
|
|
503
|
+
if (playout?.timer) clearTimeout(playout.timer);
|
|
504
|
+
this.lockedPlayout.delete(contextId);
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
private finalize(state: TurnState): void {
|
|
508
|
+
const text = state.finalSegments.join(" ").replace(/\s+/g, " ").trim();
|
|
509
|
+
if (state.finalized || state.finalPackets.length === 0 || !text) return;
|
|
510
|
+
state.finalized = true;
|
|
511
|
+
this.lockedContextIds.add(state.contextId);
|
|
512
|
+
clearTurnTimers(state);
|
|
513
|
+
this.clearAbsoluteCap(state);
|
|
514
|
+
this.bus?.push(Route.Main, {
|
|
515
|
+
kind: "eos.turn_complete",
|
|
516
|
+
contextId: state.contextId,
|
|
517
|
+
timestampMs: Date.now(),
|
|
518
|
+
text,
|
|
519
|
+
transcripts: state.finalPackets,
|
|
520
|
+
} satisfies EndOfSpeechPacket);
|
|
521
|
+
this.turns.delete(state.contextId);
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
private requestSttFinalize(contextId: string): void {
|
|
525
|
+
this.bus?.push(Route.Critical, {
|
|
526
|
+
kind: "stt.finalize",
|
|
527
|
+
contextId,
|
|
528
|
+
timestampMs: Date.now(),
|
|
529
|
+
} satisfies FinalizeSttPacket);
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
function appendFinalPacket(state: TurnState, packet: SttResultPacket): void {
|
|
534
|
+
const text = packet.text.trim();
|
|
535
|
+
if (state.finalSegments.at(-1) === text) return;
|
|
536
|
+
state.finalSegments.push(text);
|
|
537
|
+
state.finalPackets.push(packet);
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
function clearTurnTimers(state: TurnState): void {
|
|
541
|
+
if (state.finalizeTimer) clearTimeout(state.finalizeTimer);
|
|
542
|
+
if (state.sttQuietTimer) clearTimeout(state.sttQuietTimer);
|
|
543
|
+
if (state.maxTimer) clearTimeout(state.maxTimer);
|
|
544
|
+
if (state.deferTimer) clearTimeout(state.deferTimer);
|
|
545
|
+
state.finalizeTimer = null;
|
|
546
|
+
state.sttQuietTimer = null;
|
|
547
|
+
state.maxTimer = null;
|
|
548
|
+
state.deferTimer = null;
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
function readBooleanConfig(value: unknown, fallback: boolean): boolean {
|
|
552
|
+
if (typeof value === "boolean") return value;
|
|
553
|
+
return fallback;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
function readNonNegativeNumber(value: unknown, fallback: number): number {
|
|
557
|
+
if (typeof value !== "number" || !Number.isFinite(value)) return fallback;
|
|
558
|
+
return Math.max(0, Math.floor(value));
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
function readProbability(value: unknown, fallback: number): number {
|
|
562
|
+
if (typeof value !== "number" || !Number.isFinite(value)) return fallback;
|
|
563
|
+
return Math.min(1, Math.max(0, value));
|
|
564
|
+
}
|