@kuralle-syrinx/silero-vad 2.1.0 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/index.ts +30 -195
- package/src/vad-state-machine.ts +234 -0
- package/src/workers.test.ts +124 -0
- package/src/workers.ts +31 -150
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kuralle-syrinx/silero-vad",
|
|
3
|
-
"version": "2.1.
|
|
3
|
+
"version": "2.1.1",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
"dependencies": {
|
|
14
14
|
"onnxruntime-node": "1.24.3",
|
|
15
15
|
"onnxruntime-web": "1.24.3",
|
|
16
|
-
"@kuralle-syrinx/core": "2.1.
|
|
16
|
+
"@kuralle-syrinx/core": "2.1.1"
|
|
17
17
|
},
|
|
18
18
|
"devDependencies": {
|
|
19
19
|
"typescript": "^5.7.0",
|
package/src/index.ts
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
// SPDX-License-Identifier: MIT
|
|
2
2
|
//
|
|
3
|
-
// Syrinx Kernel v2 — Silero VAD Plugin
|
|
3
|
+
// Syrinx Kernel v2 — Silero VAD Plugin (Node, onnxruntime-node)
|
|
4
4
|
//
|
|
5
5
|
// Runs the Silero ONNX model locally and emits VAD packets through PipelineBus.
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
6
|
+
// All turn/state decisions live in the shared SileroVadStateMachine
|
|
7
|
+
// (vad-state-machine.ts) — this file only owns the Node-specific ONNX runtime
|
|
8
|
+
// and on-disk model loading. The Workers variant (workers.ts) shares the same
|
|
9
|
+
// machine with onnxruntime-web, so the two can never drift again.
|
|
9
10
|
|
|
10
11
|
import { fileURLToPath } from "node:url";
|
|
11
12
|
|
|
@@ -14,25 +15,23 @@ import {
|
|
|
14
15
|
ErrorCategory,
|
|
15
16
|
Route,
|
|
16
17
|
type PluginConfig,
|
|
17
|
-
type VadSpeechActivityPacket,
|
|
18
|
-
type VadSpeechEndedPacket,
|
|
19
|
-
type VadSpeechStartedPacket,
|
|
20
18
|
type VoiceErrorPacket,
|
|
21
19
|
type VoicePlugin,
|
|
22
20
|
isRecoverable,
|
|
23
21
|
optionalStringConfig,
|
|
24
22
|
} from "@kuralle-syrinx/core";
|
|
25
|
-
import {
|
|
23
|
+
import {
|
|
24
|
+
CONTEXT_SAMPLES_16K,
|
|
25
|
+
Pcm16WindowBuffer,
|
|
26
|
+
SileroVadStateMachine,
|
|
27
|
+
readVadTuning,
|
|
28
|
+
} from "./vad-state-machine.js";
|
|
26
29
|
|
|
27
30
|
type Ort = typeof import("onnxruntime-node");
|
|
28
31
|
type InferenceSession = import("onnxruntime-node").InferenceSession;
|
|
29
32
|
|
|
30
33
|
const DEFAULT_MODEL_PATH = fileURLToPath(new URL("../models/silero_vad.onnx", import.meta.url));
|
|
31
34
|
const DEFAULT_SAMPLE_RATE = 16000;
|
|
32
|
-
const WINDOW_SAMPLES_16K = 512;
|
|
33
|
-
const CONTEXT_SAMPLES_16K = 64;
|
|
34
|
-
|
|
35
|
-
type VadState = "quiet" | "starting" | "speaking" | "stopping";
|
|
36
35
|
|
|
37
36
|
export class SileroVADPlugin implements VoicePlugin {
|
|
38
37
|
private bus: PipelineBus | null = null;
|
|
@@ -40,31 +39,15 @@ export class SileroVADPlugin implements VoicePlugin {
|
|
|
40
39
|
private ort: Ort | null = null;
|
|
41
40
|
private state = new Float32Array(2 * 1 * 128);
|
|
42
41
|
private context = new Float32Array(CONTEXT_SAMPLES_16K);
|
|
43
|
-
private
|
|
44
|
-
private
|
|
45
|
-
private speechFrames = 0;
|
|
46
|
-
private confidenceThreshold = 0.5;
|
|
47
|
-
private minSilenceDurationMs = 200;
|
|
48
|
-
private speakingStateResetIntervalMs = 12_000;
|
|
49
|
-
private speechPadMs = 80;
|
|
50
|
-
private speechStartDurationMs = 32;
|
|
42
|
+
private readonly windows = new Pcm16WindowBuffer();
|
|
43
|
+
private machine: SileroVadStateMachine | null = null;
|
|
51
44
|
private sampleRate = DEFAULT_SAMPLE_RATE;
|
|
52
|
-
private silenceFrames = 0;
|
|
53
|
-
private stoppingSpikeFrames = 0;
|
|
54
|
-
private lastResetMs = 0;
|
|
55
45
|
private disposers: Array<() => void> = [];
|
|
56
46
|
|
|
57
47
|
async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
|
|
58
48
|
this.bus = bus;
|
|
59
|
-
this.
|
|
60
|
-
this.
|
|
61
|
-
this.speakingStateResetIntervalMs = readNumber(config, "speaking_state_reset_interval_ms", 12_000);
|
|
62
|
-
this.speechPadMs = readNumber(config, "speech_pad_ms", 80);
|
|
63
|
-
this.speechStartDurationMs = readNumber(config, "speech_start_duration_ms", 32);
|
|
64
|
-
this.sampleRate = readNumber(config, "sample_rate", DEFAULT_SAMPLE_RATE);
|
|
65
|
-
if (this.sampleRate !== 16000) {
|
|
66
|
-
throw new Error(`SileroVADPlugin requires 16 kHz PCM input, got ${String(this.sampleRate)} Hz`);
|
|
67
|
-
}
|
|
49
|
+
this.sampleRate = readSampleRate(config);
|
|
50
|
+
this.machine = new SileroVadStateMachine(bus, readVadTuning(config), () => this.resetModelState());
|
|
68
51
|
|
|
69
52
|
const modelPath = optionalStringConfig(config, "model_path") ?? DEFAULT_MODEL_PATH;
|
|
70
53
|
this.ort = await import("onnxruntime-node");
|
|
@@ -74,6 +57,7 @@ export class SileroVADPlugin implements VoicePlugin {
|
|
|
74
57
|
intraOpNumThreads: 1,
|
|
75
58
|
});
|
|
76
59
|
this.resetModelState();
|
|
60
|
+
this.machine.noteModelReset();
|
|
77
61
|
|
|
78
62
|
this.disposers.push(
|
|
79
63
|
bus.on("vad.audio", async (pkt: unknown) => {
|
|
@@ -84,28 +68,18 @@ export class SileroVADPlugin implements VoicePlugin {
|
|
|
84
68
|
}
|
|
85
69
|
|
|
86
70
|
async processAudio(audio: Uint8Array, contextId: string): Promise<void> {
|
|
87
|
-
if (!this.bus || !this.session || !this.ort) return;
|
|
71
|
+
if (!this.bus || !this.session || !this.ort || !this.machine) return;
|
|
88
72
|
if (audio.byteLength % 2 !== 0) {
|
|
89
73
|
this.emitError(contextId, new Error("VAD audio must be 16-bit PCM with even byte length"));
|
|
90
74
|
return;
|
|
91
75
|
}
|
|
92
76
|
|
|
93
77
|
// Offset-safe: inbound PCM is often a Uint8Array view into a pooled Node
|
|
94
|
-
// Buffer at an ODD byteOffset
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
const samples = pcm16BytesToSamples(audio);
|
|
98
|
-
for (let i = 0; i < samples.length; i += 1) {
|
|
99
|
-
this.pendingSamples.push(samples[i]! / 32768);
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
while (this.pendingSamples.length >= WINDOW_SAMPLES_16K) {
|
|
103
|
-
const window = new Float32Array(WINDOW_SAMPLES_16K);
|
|
104
|
-
for (let i = 0; i < WINDOW_SAMPLES_16K; i += 1) {
|
|
105
|
-
window[i] = this.pendingSamples.shift()!;
|
|
106
|
-
}
|
|
78
|
+
// Buffer at an ODD byteOffset — the shared buffer reads via DataView.
|
|
79
|
+
this.windows.push(audio);
|
|
80
|
+
for (let window = this.windows.next(); window; window = this.windows.next()) {
|
|
107
81
|
const confidence = await this.runModel(window, contextId);
|
|
108
|
-
this.
|
|
82
|
+
this.machine.observe(confidence, contextId);
|
|
109
83
|
}
|
|
110
84
|
}
|
|
111
85
|
|
|
@@ -114,14 +88,14 @@ export class SileroVADPlugin implements VoicePlugin {
|
|
|
114
88
|
this.bus = null;
|
|
115
89
|
this.session = null;
|
|
116
90
|
this.ort = null;
|
|
117
|
-
this.
|
|
91
|
+
this.windows.clear();
|
|
118
92
|
this.resetModelState();
|
|
119
93
|
}
|
|
120
94
|
|
|
121
95
|
private async runModel(window: Float32Array, contextId: string): Promise<number> {
|
|
122
96
|
if (!this.session || !this.ort) return 0;
|
|
123
97
|
|
|
124
|
-
const input = new Float32Array(CONTEXT_SAMPLES_16K +
|
|
98
|
+
const input = new Float32Array(CONTEXT_SAMPLES_16K + window.length);
|
|
125
99
|
input.set(this.context, 0);
|
|
126
100
|
input.set(window, CONTEXT_SAMPLES_16K);
|
|
127
101
|
|
|
@@ -139,11 +113,6 @@ export class SileroVADPlugin implements VoicePlugin {
|
|
|
139
113
|
}
|
|
140
114
|
this.context = input.slice(-CONTEXT_SAMPLES_16K);
|
|
141
115
|
|
|
142
|
-
const now = Date.now();
|
|
143
|
-
if (this.vadState === "quiet" && now - this.lastResetMs >= 5000) {
|
|
144
|
-
this.resetModelState();
|
|
145
|
-
}
|
|
146
|
-
|
|
147
116
|
return typeof probability === "number" ? probability : 0;
|
|
148
117
|
} catch (err) {
|
|
149
118
|
this.emitError(contextId, err instanceof Error ? err : new Error(String(err)));
|
|
@@ -151,147 +120,9 @@ export class SileroVADPlugin implements VoicePlugin {
|
|
|
151
120
|
}
|
|
152
121
|
}
|
|
153
122
|
|
|
154
|
-
private emitVadState(confidence: number, contextId: string): void {
|
|
155
|
-
if (!this.bus) return;
|
|
156
|
-
|
|
157
|
-
const now = Date.now();
|
|
158
|
-
const isSpeech = confidence >= this.confidenceThreshold;
|
|
159
|
-
const silenceFrameTarget = Math.max(1, Math.ceil(this.minSilenceDurationMs / 32));
|
|
160
|
-
|
|
161
|
-
switch (this.vadState) {
|
|
162
|
-
case "quiet":
|
|
163
|
-
if (isSpeech) {
|
|
164
|
-
this.vadState = "starting";
|
|
165
|
-
this.speechFrames = 1;
|
|
166
|
-
this.promoteStartingToSpeakingIfReady(confidence, contextId, now);
|
|
167
|
-
}
|
|
168
|
-
break;
|
|
169
|
-
|
|
170
|
-
case "starting":
|
|
171
|
-
if (isSpeech) {
|
|
172
|
-
if (!this.promoteStartingToSpeakingIfReady(confidence, contextId, now)) {
|
|
173
|
-
this.speechFrames += 1;
|
|
174
|
-
this.promoteStartingToSpeakingIfReady(confidence, contextId, now);
|
|
175
|
-
}
|
|
176
|
-
} else {
|
|
177
|
-
this.vadState = "quiet";
|
|
178
|
-
this.speechFrames = 0;
|
|
179
|
-
}
|
|
180
|
-
break;
|
|
181
|
-
|
|
182
|
-
case "speaking":
|
|
183
|
-
if (isSpeech) {
|
|
184
|
-
// Silero v5 LSTM state saturates on long continuous segments
|
|
185
|
-
// (telephony monologues): confidence then flaps high through genuine
|
|
186
|
-
// silence and the end never fires. Periodically reset model state
|
|
187
|
-
// mid-speech; the spike debounce in "stopping" makes the brief
|
|
188
|
-
// post-reset confidence dip harmless for real speech.
|
|
189
|
-
if (now - this.lastResetMs >= this.speakingStateResetIntervalMs) {
|
|
190
|
-
this.resetModelState();
|
|
191
|
-
this.bus.push(Route.Main, {
|
|
192
|
-
kind: "metric.conversation",
|
|
193
|
-
contextId,
|
|
194
|
-
timestampMs: now,
|
|
195
|
-
name: "vad.state_reset_in_speech",
|
|
196
|
-
value: "1",
|
|
197
|
-
});
|
|
198
|
-
}
|
|
199
|
-
this.emitSpeechActivity(contextId, now);
|
|
200
|
-
} else {
|
|
201
|
-
this.vadState = "stopping";
|
|
202
|
-
this.silenceFrames = 1;
|
|
203
|
-
this.stoppingSpikeFrames = 0;
|
|
204
|
-
}
|
|
205
|
-
break;
|
|
206
|
-
|
|
207
|
-
case "stopping":
|
|
208
|
-
if (isSpeech) {
|
|
209
|
-
// Single-frame confidence spikes are a known Silero failure mode on
|
|
210
|
-
// long telephony segments (state saturation flaps high through real
|
|
211
|
-
// silence). Require sustained speech to leave "stopping" so one
|
|
212
|
-
// spike cannot reset the silence countdown forever.
|
|
213
|
-
this.stoppingSpikeFrames += 1;
|
|
214
|
-
if (this.stoppingSpikeFrames >= 2) {
|
|
215
|
-
this.vadState = "speaking";
|
|
216
|
-
this.silenceFrames = 0;
|
|
217
|
-
this.stoppingSpikeFrames = 0;
|
|
218
|
-
this.emitSpeechActivity(contextId, now);
|
|
219
|
-
}
|
|
220
|
-
break;
|
|
221
|
-
}
|
|
222
|
-
this.stoppingSpikeFrames = 0;
|
|
223
|
-
this.silenceFrames += 1;
|
|
224
|
-
if (this.silenceFrames * 32 < this.minSilenceDurationMs + this.speechPadMs) break;
|
|
225
|
-
if (this.silenceFrames < silenceFrameTarget) break;
|
|
226
|
-
|
|
227
|
-
{
|
|
228
|
-
const hangoverMs = this.silenceFrames * 32;
|
|
229
|
-
this.vadState = "quiet";
|
|
230
|
-
this.silenceFrames = 0;
|
|
231
|
-
this.speechFrames = 0;
|
|
232
|
-
this.stoppingSpikeFrames = 0;
|
|
233
|
-
const ended: VadSpeechEndedPacket = {
|
|
234
|
-
kind: "vad.speech_ended",
|
|
235
|
-
contextId,
|
|
236
|
-
timestampMs: now,
|
|
237
|
-
};
|
|
238
|
-
this.bus.push(Route.Main, ended);
|
|
239
|
-
this.bus.push(Route.Main, {
|
|
240
|
-
kind: "metric.conversation",
|
|
241
|
-
contextId,
|
|
242
|
-
timestampMs: now,
|
|
243
|
-
name: "vad.stop_hangover_ms",
|
|
244
|
-
value: String(hangoverMs),
|
|
245
|
-
});
|
|
246
|
-
}
|
|
247
|
-
break;
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
private promoteStartingToSpeakingIfReady(
|
|
252
|
-
confidence: number,
|
|
253
|
-
contextId: string,
|
|
254
|
-
now: number,
|
|
255
|
-
): boolean {
|
|
256
|
-
if (!this.bus || this.vadState !== "starting") return false;
|
|
257
|
-
if (this.speechFrames * 32 < this.speechStartDurationMs) return false;
|
|
258
|
-
|
|
259
|
-
const startDelayMs = this.speechFrames * 32;
|
|
260
|
-
this.vadState = "speaking";
|
|
261
|
-
this.speechFrames = 0;
|
|
262
|
-
const started: VadSpeechStartedPacket = {
|
|
263
|
-
kind: "vad.speech_started",
|
|
264
|
-
contextId,
|
|
265
|
-
timestampMs: now,
|
|
266
|
-
confidence,
|
|
267
|
-
};
|
|
268
|
-
this.bus.push(Route.Main, started);
|
|
269
|
-
this.bus.push(Route.Main, {
|
|
270
|
-
kind: "metric.conversation",
|
|
271
|
-
contextId,
|
|
272
|
-
timestampMs: now,
|
|
273
|
-
name: "vad.start_delay_ms",
|
|
274
|
-
value: String(startDelayMs),
|
|
275
|
-
});
|
|
276
|
-
this.emitSpeechActivity(contextId, now);
|
|
277
|
-
return true;
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
private emitSpeechActivity(contextId: string, now: number): void {
|
|
281
|
-
if (!this.bus) return;
|
|
282
|
-
const activity: VadSpeechActivityPacket = {
|
|
283
|
-
kind: "vad.speech_activity",
|
|
284
|
-
contextId,
|
|
285
|
-
timestampMs: now,
|
|
286
|
-
isAsync: true,
|
|
287
|
-
};
|
|
288
|
-
this.bus.push(Route.Main, activity);
|
|
289
|
-
}
|
|
290
|
-
|
|
291
123
|
private resetModelState(): void {
|
|
292
124
|
this.state = new Float32Array(2 * 1 * 128);
|
|
293
125
|
this.context = new Float32Array(CONTEXT_SAMPLES_16K);
|
|
294
|
-
this.lastResetMs = Date.now();
|
|
295
126
|
}
|
|
296
127
|
|
|
297
128
|
private emitError(contextId: string, err: Error): void {
|
|
@@ -308,7 +139,11 @@ export class SileroVADPlugin implements VoicePlugin {
|
|
|
308
139
|
}
|
|
309
140
|
}
|
|
310
141
|
|
|
311
|
-
function
|
|
312
|
-
const value = config[
|
|
313
|
-
|
|
142
|
+
function readSampleRate(config: PluginConfig): number {
|
|
143
|
+
const value = config["sample_rate"];
|
|
144
|
+
const sampleRate = typeof value === "number" && Number.isFinite(value) ? value : DEFAULT_SAMPLE_RATE;
|
|
145
|
+
if (sampleRate !== 16000) {
|
|
146
|
+
throw new Error(`SileroVADPlugin requires 16 kHz PCM input, got ${String(sampleRate)} Hz`);
|
|
147
|
+
}
|
|
148
|
+
return sampleRate;
|
|
314
149
|
}
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
//
|
|
3
|
+
// Shared Silero VAD turn state machine + PCM windowing — the single source of
|
|
4
|
+
// truth for both the Node (onnxruntime-node) and Workers (onnxruntime-web)
|
|
5
|
+
// plugins. The two runtimes differ only in how the ONNX session is created and
|
|
6
|
+
// where the model bytes come from; everything that decides "is the user
|
|
7
|
+
// speaking" lives here so the variants can never drift again (v2.1.0 shipped
|
|
8
|
+
// the telephony saturation hardening in the Node copy only).
|
|
9
|
+
|
|
10
|
+
import {
|
|
11
|
+
Route,
|
|
12
|
+
type PipelineBus,
|
|
13
|
+
type PluginConfig,
|
|
14
|
+
type VadSpeechActivityPacket,
|
|
15
|
+
type VadSpeechEndedPacket,
|
|
16
|
+
type VadSpeechStartedPacket,
|
|
17
|
+
} from "@kuralle-syrinx/core";
|
|
18
|
+
import { pcm16BytesToSamples } from "@kuralle-syrinx/core/audio";
|
|
19
|
+
|
|
20
|
+
export const WINDOW_SAMPLES_16K = 512;
|
|
21
|
+
export const CONTEXT_SAMPLES_16K = 64;
|
|
22
|
+
|
|
23
|
+
const QUIET_MODEL_RESET_INTERVAL_MS = 5000;
|
|
24
|
+
|
|
25
|
+
type VadState = "quiet" | "starting" | "speaking" | "stopping";
|
|
26
|
+
|
|
27
|
+
export interface VadTuning {
|
|
28
|
+
readonly confidenceThreshold: number;
|
|
29
|
+
readonly minSilenceDurationMs: number;
|
|
30
|
+
readonly speechPadMs: number;
|
|
31
|
+
readonly speechStartDurationMs: number;
|
|
32
|
+
readonly speakingStateResetIntervalMs: number;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function readVadTuning(config: PluginConfig): VadTuning {
|
|
36
|
+
return {
|
|
37
|
+
confidenceThreshold: readNumber(config, "threshold", 0.5),
|
|
38
|
+
minSilenceDurationMs: readNumber(config, "min_silence_duration_ms", 200),
|
|
39
|
+
speechPadMs: readNumber(config, "speech_pad_ms", 80),
|
|
40
|
+
speechStartDurationMs: readNumber(config, "speech_start_duration_ms", 32),
|
|
41
|
+
speakingStateResetIntervalMs: readNumber(config, "speaking_state_reset_interval_ms", 12_000),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Accumulates inbound PCM16 bytes and yields normalized 512-sample model windows. */
|
|
46
|
+
export class Pcm16WindowBuffer {
|
|
47
|
+
private pendingSamples: number[] = [];
|
|
48
|
+
|
|
49
|
+
push(audio: Uint8Array): void {
|
|
50
|
+
const samples = pcm16BytesToSamples(audio);
|
|
51
|
+
for (let i = 0; i < samples.length; i += 1) {
|
|
52
|
+
this.pendingSamples.push(samples[i]! / 32768);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
next(): Float32Array | null {
|
|
57
|
+
if (this.pendingSamples.length < WINDOW_SAMPLES_16K) return null;
|
|
58
|
+
const window = new Float32Array(WINDOW_SAMPLES_16K);
|
|
59
|
+
for (let i = 0; i < WINDOW_SAMPLES_16K; i += 1) {
|
|
60
|
+
window[i] = this.pendingSamples.shift()!;
|
|
61
|
+
}
|
|
62
|
+
return window;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
clear(): void {
|
|
66
|
+
this.pendingSamples = [];
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export class SileroVadStateMachine {
|
|
71
|
+
private vadState: VadState = "quiet";
|
|
72
|
+
private speechFrames = 0;
|
|
73
|
+
private silenceFrames = 0;
|
|
74
|
+
private stoppingSpikeFrames = 0;
|
|
75
|
+
private lastResetMs = Date.now();
|
|
76
|
+
|
|
77
|
+
constructor(
|
|
78
|
+
private readonly bus: PipelineBus,
|
|
79
|
+
private readonly tuning: VadTuning,
|
|
80
|
+
/** Zero the runtime's model state/context tensors; the machine owns when. */
|
|
81
|
+
private readonly onModelReset: () => void,
|
|
82
|
+
) {}
|
|
83
|
+
|
|
84
|
+
noteModelReset(): void {
|
|
85
|
+
this.lastResetMs = Date.now();
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
observe(confidence: number, contextId: string): void {
|
|
89
|
+
const now = Date.now();
|
|
90
|
+
|
|
91
|
+
// Long-quiet model reset: keep the LSTM fresh between utterances.
|
|
92
|
+
if (this.vadState === "quiet" && now - this.lastResetMs >= QUIET_MODEL_RESET_INTERVAL_MS) {
|
|
93
|
+
this.onModelReset();
|
|
94
|
+
this.lastResetMs = now;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const isSpeech = confidence >= this.tuning.confidenceThreshold;
|
|
98
|
+
const silenceFrameTarget = Math.max(1, Math.ceil(this.tuning.minSilenceDurationMs / 32));
|
|
99
|
+
|
|
100
|
+
switch (this.vadState) {
|
|
101
|
+
case "quiet":
|
|
102
|
+
if (isSpeech) {
|
|
103
|
+
this.vadState = "starting";
|
|
104
|
+
this.speechFrames = 1;
|
|
105
|
+
this.promoteStartingToSpeakingIfReady(confidence, contextId, now);
|
|
106
|
+
}
|
|
107
|
+
break;
|
|
108
|
+
|
|
109
|
+
case "starting":
|
|
110
|
+
if (isSpeech) {
|
|
111
|
+
if (!this.promoteStartingToSpeakingIfReady(confidence, contextId, now)) {
|
|
112
|
+
this.speechFrames += 1;
|
|
113
|
+
this.promoteStartingToSpeakingIfReady(confidence, contextId, now);
|
|
114
|
+
}
|
|
115
|
+
} else {
|
|
116
|
+
this.vadState = "quiet";
|
|
117
|
+
this.speechFrames = 0;
|
|
118
|
+
}
|
|
119
|
+
break;
|
|
120
|
+
|
|
121
|
+
case "speaking":
|
|
122
|
+
if (isSpeech) {
|
|
123
|
+
// Silero v5 LSTM state saturates on long continuous segments
|
|
124
|
+
// (telephony monologues): confidence then flaps high through genuine
|
|
125
|
+
// silence and the end never fires. Periodically reset model state
|
|
126
|
+
// mid-speech; the spike debounce in "stopping" makes the brief
|
|
127
|
+
// post-reset confidence dip harmless for real speech.
|
|
128
|
+
if (now - this.lastResetMs >= this.tuning.speakingStateResetIntervalMs) {
|
|
129
|
+
this.onModelReset();
|
|
130
|
+
this.lastResetMs = now;
|
|
131
|
+
this.bus.push(Route.Main, {
|
|
132
|
+
kind: "metric.conversation",
|
|
133
|
+
contextId,
|
|
134
|
+
timestampMs: now,
|
|
135
|
+
name: "vad.state_reset_in_speech",
|
|
136
|
+
value: "1",
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
this.emitSpeechActivity(contextId, now);
|
|
140
|
+
} else {
|
|
141
|
+
this.vadState = "stopping";
|
|
142
|
+
this.silenceFrames = 1;
|
|
143
|
+
this.stoppingSpikeFrames = 0;
|
|
144
|
+
}
|
|
145
|
+
break;
|
|
146
|
+
|
|
147
|
+
case "stopping":
|
|
148
|
+
if (isSpeech) {
|
|
149
|
+
// Single-frame confidence spikes are a known Silero failure mode on
|
|
150
|
+
// long telephony segments (state saturation flaps high through real
|
|
151
|
+
// silence). Require sustained speech to leave "stopping" so one
|
|
152
|
+
// spike cannot reset the silence countdown forever.
|
|
153
|
+
this.stoppingSpikeFrames += 1;
|
|
154
|
+
if (this.stoppingSpikeFrames >= 2) {
|
|
155
|
+
this.vadState = "speaking";
|
|
156
|
+
this.silenceFrames = 0;
|
|
157
|
+
this.stoppingSpikeFrames = 0;
|
|
158
|
+
this.emitSpeechActivity(contextId, now);
|
|
159
|
+
}
|
|
160
|
+
break;
|
|
161
|
+
}
|
|
162
|
+
this.stoppingSpikeFrames = 0;
|
|
163
|
+
this.silenceFrames += 1;
|
|
164
|
+
if (this.silenceFrames * 32 < this.tuning.minSilenceDurationMs + this.tuning.speechPadMs) break;
|
|
165
|
+
if (this.silenceFrames < silenceFrameTarget) break;
|
|
166
|
+
|
|
167
|
+
{
|
|
168
|
+
const hangoverMs = this.silenceFrames * 32;
|
|
169
|
+
this.vadState = "quiet";
|
|
170
|
+
this.silenceFrames = 0;
|
|
171
|
+
this.speechFrames = 0;
|
|
172
|
+
this.stoppingSpikeFrames = 0;
|
|
173
|
+
const ended: VadSpeechEndedPacket = {
|
|
174
|
+
kind: "vad.speech_ended",
|
|
175
|
+
contextId,
|
|
176
|
+
timestampMs: now,
|
|
177
|
+
};
|
|
178
|
+
this.bus.push(Route.Main, ended);
|
|
179
|
+
this.bus.push(Route.Main, {
|
|
180
|
+
kind: "metric.conversation",
|
|
181
|
+
contextId,
|
|
182
|
+
timestampMs: now,
|
|
183
|
+
name: "vad.stop_hangover_ms",
|
|
184
|
+
value: String(hangoverMs),
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
break;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
private promoteStartingToSpeakingIfReady(
|
|
192
|
+
confidence: number,
|
|
193
|
+
contextId: string,
|
|
194
|
+
now: number,
|
|
195
|
+
): boolean {
|
|
196
|
+
if (this.vadState !== "starting") return false;
|
|
197
|
+
if (this.speechFrames * 32 < this.tuning.speechStartDurationMs) return false;
|
|
198
|
+
|
|
199
|
+
const startDelayMs = this.speechFrames * 32;
|
|
200
|
+
this.vadState = "speaking";
|
|
201
|
+
this.speechFrames = 0;
|
|
202
|
+
const started: VadSpeechStartedPacket = {
|
|
203
|
+
kind: "vad.speech_started",
|
|
204
|
+
contextId,
|
|
205
|
+
timestampMs: now,
|
|
206
|
+
confidence,
|
|
207
|
+
};
|
|
208
|
+
this.bus.push(Route.Main, started);
|
|
209
|
+
this.bus.push(Route.Main, {
|
|
210
|
+
kind: "metric.conversation",
|
|
211
|
+
contextId,
|
|
212
|
+
timestampMs: now,
|
|
213
|
+
name: "vad.start_delay_ms",
|
|
214
|
+
value: String(startDelayMs),
|
|
215
|
+
});
|
|
216
|
+
this.emitSpeechActivity(contextId, now);
|
|
217
|
+
return true;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
private emitSpeechActivity(contextId: string, now: number): void {
|
|
221
|
+
const activity: VadSpeechActivityPacket = {
|
|
222
|
+
kind: "vad.speech_activity",
|
|
223
|
+
contextId,
|
|
224
|
+
timestampMs: now,
|
|
225
|
+
isAsync: true,
|
|
226
|
+
};
|
|
227
|
+
this.bus.push(Route.Main, activity);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function readNumber(config: PluginConfig, key: string, fallback: number): number {
|
|
232
|
+
const value = config[key];
|
|
233
|
+
return typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
|
234
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
|
|
3
|
+
import { beforeEach, afterEach, describe, expect, it, vi } from "vitest";
|
|
4
|
+
import type { PipelineBus } from "@kuralle-syrinx/core";
|
|
5
|
+
|
|
6
|
+
import { SileroVADPlugin } from "./workers.js";
|
|
7
|
+
|
|
8
|
+
const mockControl = vi.hoisted(() => ({
|
|
9
|
+
confidence: 0.9,
|
|
10
|
+
stateCallArgs: [] as Float32Array[],
|
|
11
|
+
}));
|
|
12
|
+
|
|
13
|
+
vi.mock("onnxruntime-web", () => {
|
|
14
|
+
function MockTensor(this: { data: unknown }, _type: string, data: unknown, _shape: unknown) {
|
|
15
|
+
this.data = data;
|
|
16
|
+
}
|
|
17
|
+
const run = vi.fn(async (inputs: Record<string, { data: unknown }>) => {
|
|
18
|
+
const state = inputs["state"]?.data;
|
|
19
|
+
if (state instanceof Float32Array) {
|
|
20
|
+
mockControl.stateCallArgs.push(new Float32Array(state));
|
|
21
|
+
}
|
|
22
|
+
return {
|
|
23
|
+
output: { data: [mockControl.confidence] },
|
|
24
|
+
stateN: { data: new Float32Array(2 * 1 * 128).fill(0.5) },
|
|
25
|
+
};
|
|
26
|
+
});
|
|
27
|
+
return {
|
|
28
|
+
InferenceSession: { create: vi.fn(async () => ({ run })) },
|
|
29
|
+
Tensor: MockTensor,
|
|
30
|
+
};
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
function makeBus(): { bus: PipelineBus; emitted: Array<{ kind: string }> } {
|
|
34
|
+
const emitted: Array<{ kind: string }> = [];
|
|
35
|
+
const bus = {
|
|
36
|
+
push(_route: unknown, pkt: unknown) {
|
|
37
|
+
emitted.push(pkt as { kind: string });
|
|
38
|
+
},
|
|
39
|
+
on(_event: string, _handler: unknown) {
|
|
40
|
+
return () => {};
|
|
41
|
+
},
|
|
42
|
+
} as unknown as PipelineBus;
|
|
43
|
+
return { bus, emitted };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function makePcmFrame(): Uint8Array {
|
|
47
|
+
return new Uint8Array(512 * 2);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async function initPlugin(bus: PipelineBus, config: Record<string, unknown> = {}): Promise<SileroVADPlugin> {
|
|
51
|
+
const plugin = new SileroVADPlugin();
|
|
52
|
+
await plugin.initialize(bus, { model_bytes: new Uint8Array(8), ...config });
|
|
53
|
+
return plugin;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function countKind(emitted: Array<{ kind: string }>, kind: string): number {
|
|
57
|
+
return emitted.filter((p) => p.kind === kind).length;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function metricValue(
|
|
61
|
+
emitted: Array<{ kind: string; name?: string; value?: string }>,
|
|
62
|
+
name: string,
|
|
63
|
+
): string | undefined {
|
|
64
|
+
return emitted.find((p) => p.kind === "metric.conversation" && p.name === name)?.value;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
describe("SileroVADPlugin (Workers) — shared state machine parity", () => {
|
|
68
|
+
beforeEach(() => {
|
|
69
|
+
vi.useFakeTimers();
|
|
70
|
+
vi.setSystemTime(0);
|
|
71
|
+
mockControl.confidence = 0.9;
|
|
72
|
+
mockControl.stateCallArgs.length = 0;
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
afterEach(() => {
|
|
76
|
+
vi.useRealTimers();
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
async function frame(plugin: SileroVADPlugin, confidence: number): Promise<void> {
|
|
80
|
+
mockControl.confidence = confidence;
|
|
81
|
+
await plugin.processAudio(makePcmFrame(), "ctx-w");
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
it("absorbs single-frame confidence spikes during stopping so speech still ends", async () => {
|
|
85
|
+
const { bus, emitted } = makeBus();
|
|
86
|
+
const plugin = await initPlugin(bus);
|
|
87
|
+
|
|
88
|
+
await frame(plugin, 0.9);
|
|
89
|
+
expect(countKind(emitted, "vad.speech_started")).toBe(1);
|
|
90
|
+
|
|
91
|
+
for (const confidence of [0.2, 0.2, 0.2, 0.2, 0.9, 0.2, 0.2, 0.2, 0.2, 0.2]) {
|
|
92
|
+
await frame(plugin, confidence);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
expect(countKind(emitted, "vad.speech_ended")).toBe(1);
|
|
96
|
+
expect(metricValue(emitted, "vad.stop_hangover_ms")).toBeDefined();
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
it("resets model state during prolonged continuous speech (saturation guard)", async () => {
|
|
100
|
+
const { bus, emitted } = makeBus();
|
|
101
|
+
const plugin = await initPlugin(bus);
|
|
102
|
+
|
|
103
|
+
await frame(plugin, 0.9);
|
|
104
|
+
vi.setSystemTime(13_000);
|
|
105
|
+
await frame(plugin, 0.9);
|
|
106
|
+
await frame(plugin, 0.9);
|
|
107
|
+
|
|
108
|
+
expect(metricValue(emitted, "vad.state_reset_in_speech")).toBe("1");
|
|
109
|
+
expect(mockControl.stateCallArgs[2]!.every((value) => value === 0)).toBe(true);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
it("two consecutive speech frames during stopping resume speaking", async () => {
|
|
113
|
+
const { bus, emitted } = makeBus();
|
|
114
|
+
const plugin = await initPlugin(bus);
|
|
115
|
+
|
|
116
|
+
await frame(plugin, 0.9);
|
|
117
|
+
for (const confidence of [0.2, 0.2, 0.2]) await frame(plugin, confidence);
|
|
118
|
+
for (const confidence of [0.9, 0.9]) await frame(plugin, confidence);
|
|
119
|
+
expect(countKind(emitted, "vad.speech_ended")).toBe(0);
|
|
120
|
+
|
|
121
|
+
for (let i = 0; i < 9; i += 1) await frame(plugin, 0.2);
|
|
122
|
+
expect(countKind(emitted, "vad.speech_ended")).toBe(1);
|
|
123
|
+
});
|
|
124
|
+
});
|
package/src/workers.ts
CHANGED
|
@@ -1,28 +1,33 @@
|
|
|
1
1
|
// SPDX-License-Identifier: MIT
|
|
2
|
+
//
|
|
3
|
+
// Syrinx Kernel v2 — Silero VAD Plugin (Cloudflare Workers, onnxruntime-web)
|
|
4
|
+
//
|
|
5
|
+
// Same plugin as ./index.ts but for workerd: WASM execution provider and model
|
|
6
|
+
// bytes supplied via `model_bytes` or fetched from `model_url` (no filesystem).
|
|
7
|
+
// All turn/state decisions live in the shared SileroVadStateMachine
|
|
8
|
+
// (vad-state-machine.ts), so the Node and Workers variants can never drift.
|
|
2
9
|
|
|
3
10
|
import type { PipelineBus } from "@kuralle-syrinx/core";
|
|
4
11
|
import {
|
|
5
12
|
ErrorCategory,
|
|
6
13
|
Route,
|
|
7
14
|
type PluginConfig,
|
|
8
|
-
type VadSpeechActivityPacket,
|
|
9
|
-
type VadSpeechEndedPacket,
|
|
10
|
-
type VadSpeechStartedPacket,
|
|
11
15
|
type VoiceErrorPacket,
|
|
12
16
|
type VoicePlugin,
|
|
13
17
|
isRecoverable,
|
|
14
18
|
optionalStringConfig,
|
|
15
19
|
} from "@kuralle-syrinx/core";
|
|
16
|
-
import {
|
|
20
|
+
import {
|
|
21
|
+
CONTEXT_SAMPLES_16K,
|
|
22
|
+
Pcm16WindowBuffer,
|
|
23
|
+
SileroVadStateMachine,
|
|
24
|
+
readVadTuning,
|
|
25
|
+
} from "./vad-state-machine.js";
|
|
17
26
|
|
|
18
27
|
type Ort = typeof import("onnxruntime-web");
|
|
19
28
|
type InferenceSession = import("onnxruntime-web").InferenceSession;
|
|
20
29
|
|
|
21
30
|
const DEFAULT_SAMPLE_RATE = 16000;
|
|
22
|
-
const WINDOW_SAMPLES_16K = 512;
|
|
23
|
-
const CONTEXT_SAMPLES_16K = 64;
|
|
24
|
-
|
|
25
|
-
type VadState = "quiet" | "starting" | "speaking" | "stopping";
|
|
26
31
|
|
|
27
32
|
export class SileroVADPlugin implements VoicePlugin {
|
|
28
33
|
private bus: PipelineBus | null = null;
|
|
@@ -30,34 +35,22 @@ export class SileroVADPlugin implements VoicePlugin {
|
|
|
30
35
|
private ort: Ort | null = null;
|
|
31
36
|
private state = new Float32Array(2 * 1 * 128);
|
|
32
37
|
private context = new Float32Array(CONTEXT_SAMPLES_16K);
|
|
33
|
-
private
|
|
34
|
-
private
|
|
35
|
-
private speechFrames = 0;
|
|
36
|
-
private confidenceThreshold = 0.5;
|
|
37
|
-
private minSilenceDurationMs = 200;
|
|
38
|
-
private speechPadMs = 80;
|
|
39
|
-
private speechStartDurationMs = 32;
|
|
38
|
+
private readonly windows = new Pcm16WindowBuffer();
|
|
39
|
+
private machine: SileroVadStateMachine | null = null;
|
|
40
40
|
private sampleRate = DEFAULT_SAMPLE_RATE;
|
|
41
|
-
private silenceFrames = 0;
|
|
42
|
-
private lastResetMs = 0;
|
|
43
41
|
private disposers: Array<() => void> = [];
|
|
44
42
|
|
|
45
43
|
async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
|
|
46
44
|
this.bus = bus;
|
|
47
|
-
this.
|
|
48
|
-
this.
|
|
49
|
-
this.speechPadMs = readNumber(config, "speech_pad_ms", 80);
|
|
50
|
-
this.speechStartDurationMs = readNumber(config, "speech_start_duration_ms", 32);
|
|
51
|
-
this.sampleRate = readNumber(config, "sample_rate", DEFAULT_SAMPLE_RATE);
|
|
52
|
-
if (this.sampleRate !== 16000) {
|
|
53
|
-
throw new Error(`SileroVADPlugin requires 16 kHz PCM input, got ${String(this.sampleRate)} Hz`);
|
|
54
|
-
}
|
|
45
|
+
this.sampleRate = readSampleRate(config);
|
|
46
|
+
this.machine = new SileroVadStateMachine(bus, readVadTuning(config), () => this.resetModelState());
|
|
55
47
|
|
|
56
48
|
this.ort = await import("onnxruntime-web");
|
|
57
49
|
this.session = await this.ort.InferenceSession.create(await readModelBytes(config), {
|
|
58
50
|
executionProviders: ["wasm"],
|
|
59
51
|
});
|
|
60
52
|
this.resetModelState();
|
|
53
|
+
this.machine.noteModelReset();
|
|
61
54
|
|
|
62
55
|
this.disposers.push(
|
|
63
56
|
bus.on("vad.audio", async (pkt: unknown) => {
|
|
@@ -68,24 +61,16 @@ export class SileroVADPlugin implements VoicePlugin {
|
|
|
68
61
|
}
|
|
69
62
|
|
|
70
63
|
async processAudio(audio: Uint8Array, contextId: string): Promise<void> {
|
|
71
|
-
if (!this.bus || !this.session || !this.ort) return;
|
|
64
|
+
if (!this.bus || !this.session || !this.ort || !this.machine) return;
|
|
72
65
|
if (audio.byteLength % 2 !== 0) {
|
|
73
66
|
this.emitError(contextId, new Error("VAD audio must be 16-bit PCM with even byte length"));
|
|
74
67
|
return;
|
|
75
68
|
}
|
|
76
69
|
|
|
77
|
-
|
|
78
|
-
for (let
|
|
79
|
-
this.pendingSamples.push(samples[i]! / 32768);
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
while (this.pendingSamples.length >= WINDOW_SAMPLES_16K) {
|
|
83
|
-
const window = new Float32Array(WINDOW_SAMPLES_16K);
|
|
84
|
-
for (let i = 0; i < WINDOW_SAMPLES_16K; i += 1) {
|
|
85
|
-
window[i] = this.pendingSamples.shift()!;
|
|
86
|
-
}
|
|
70
|
+
this.windows.push(audio);
|
|
71
|
+
for (let window = this.windows.next(); window; window = this.windows.next()) {
|
|
87
72
|
const confidence = await this.runModel(window, contextId);
|
|
88
|
-
this.
|
|
73
|
+
this.machine.observe(confidence, contextId);
|
|
89
74
|
}
|
|
90
75
|
}
|
|
91
76
|
|
|
@@ -94,14 +79,14 @@ export class SileroVADPlugin implements VoicePlugin {
|
|
|
94
79
|
this.bus = null;
|
|
95
80
|
this.session = null;
|
|
96
81
|
this.ort = null;
|
|
97
|
-
this.
|
|
82
|
+
this.windows.clear();
|
|
98
83
|
this.resetModelState();
|
|
99
84
|
}
|
|
100
85
|
|
|
101
86
|
private async runModel(window: Float32Array, contextId: string): Promise<number> {
|
|
102
87
|
if (!this.session || !this.ort) return 0;
|
|
103
88
|
|
|
104
|
-
const input = new Float32Array(CONTEXT_SAMPLES_16K +
|
|
89
|
+
const input = new Float32Array(CONTEXT_SAMPLES_16K + window.length);
|
|
105
90
|
input.set(this.context, 0);
|
|
106
91
|
input.set(window, CONTEXT_SAMPLES_16K);
|
|
107
92
|
|
|
@@ -119,11 +104,6 @@ export class SileroVADPlugin implements VoicePlugin {
|
|
|
119
104
|
}
|
|
120
105
|
this.context = input.slice(-CONTEXT_SAMPLES_16K);
|
|
121
106
|
|
|
122
|
-
const now = Date.now();
|
|
123
|
-
if (this.vadState === "quiet" && now - this.lastResetMs >= 5000) {
|
|
124
|
-
this.resetModelState();
|
|
125
|
-
}
|
|
126
|
-
|
|
127
107
|
return typeof probability === "number" ? probability : 0;
|
|
128
108
|
} catch (err) {
|
|
129
109
|
this.emitError(contextId, err instanceof Error ? err : new Error(String(err)));
|
|
@@ -131,112 +111,9 @@ export class SileroVADPlugin implements VoicePlugin {
|
|
|
131
111
|
}
|
|
132
112
|
}
|
|
133
113
|
|
|
134
|
-
private emitVadState(confidence: number, contextId: string): void {
|
|
135
|
-
if (!this.bus) return;
|
|
136
|
-
|
|
137
|
-
const now = Date.now();
|
|
138
|
-
const isSpeech = confidence >= this.confidenceThreshold;
|
|
139
|
-
const silenceFrameTarget = Math.max(1, Math.ceil(this.minSilenceDurationMs / 32));
|
|
140
|
-
|
|
141
|
-
switch (this.vadState) {
|
|
142
|
-
case "quiet":
|
|
143
|
-
if (isSpeech) {
|
|
144
|
-
this.vadState = "starting";
|
|
145
|
-
this.speechFrames = 1;
|
|
146
|
-
this.promoteStartingToSpeakingIfReady(confidence, contextId, now);
|
|
147
|
-
}
|
|
148
|
-
break;
|
|
149
|
-
case "starting":
|
|
150
|
-
if (isSpeech) {
|
|
151
|
-
if (!this.promoteStartingToSpeakingIfReady(confidence, contextId, now)) {
|
|
152
|
-
this.speechFrames += 1;
|
|
153
|
-
this.promoteStartingToSpeakingIfReady(confidence, contextId, now);
|
|
154
|
-
}
|
|
155
|
-
} else {
|
|
156
|
-
this.vadState = "quiet";
|
|
157
|
-
this.speechFrames = 0;
|
|
158
|
-
}
|
|
159
|
-
break;
|
|
160
|
-
case "speaking":
|
|
161
|
-
if (isSpeech) {
|
|
162
|
-
this.emitSpeechActivity(contextId, now);
|
|
163
|
-
} else {
|
|
164
|
-
this.vadState = "stopping";
|
|
165
|
-
this.silenceFrames = 1;
|
|
166
|
-
}
|
|
167
|
-
break;
|
|
168
|
-
case "stopping":
|
|
169
|
-
if (isSpeech) {
|
|
170
|
-
this.vadState = "speaking";
|
|
171
|
-
this.silenceFrames = 0;
|
|
172
|
-
this.emitSpeechActivity(contextId, now);
|
|
173
|
-
} else {
|
|
174
|
-
this.silenceFrames += 1;
|
|
175
|
-
if (this.silenceFrames * 32 < this.minSilenceDurationMs + this.speechPadMs) break;
|
|
176
|
-
if (this.silenceFrames < silenceFrameTarget) break;
|
|
177
|
-
|
|
178
|
-
const hangoverMs = this.silenceFrames * 32;
|
|
179
|
-
this.vadState = "quiet";
|
|
180
|
-
this.silenceFrames = 0;
|
|
181
|
-
this.speechFrames = 0;
|
|
182
|
-
const ended: VadSpeechEndedPacket = {
|
|
183
|
-
kind: "vad.speech_ended",
|
|
184
|
-
contextId,
|
|
185
|
-
timestampMs: now,
|
|
186
|
-
};
|
|
187
|
-
this.bus.push(Route.Main, ended);
|
|
188
|
-
this.bus.push(Route.Main, {
|
|
189
|
-
kind: "metric.conversation",
|
|
190
|
-
contextId,
|
|
191
|
-
timestampMs: now,
|
|
192
|
-
name: "vad.stop_hangover_ms",
|
|
193
|
-
value: String(hangoverMs),
|
|
194
|
-
});
|
|
195
|
-
}
|
|
196
|
-
break;
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
private promoteStartingToSpeakingIfReady(confidence: number, contextId: string, now: number): boolean {
|
|
201
|
-
if (!this.bus || this.vadState !== "starting") return false;
|
|
202
|
-
if (this.speechFrames * 32 < this.speechStartDurationMs) return false;
|
|
203
|
-
|
|
204
|
-
const startDelayMs = this.speechFrames * 32;
|
|
205
|
-
this.vadState = "speaking";
|
|
206
|
-
this.speechFrames = 0;
|
|
207
|
-
const started: VadSpeechStartedPacket = {
|
|
208
|
-
kind: "vad.speech_started",
|
|
209
|
-
contextId,
|
|
210
|
-
timestampMs: now,
|
|
211
|
-
confidence,
|
|
212
|
-
};
|
|
213
|
-
this.bus.push(Route.Main, started);
|
|
214
|
-
this.bus.push(Route.Main, {
|
|
215
|
-
kind: "metric.conversation",
|
|
216
|
-
contextId,
|
|
217
|
-
timestampMs: now,
|
|
218
|
-
name: "vad.start_delay_ms",
|
|
219
|
-
value: String(startDelayMs),
|
|
220
|
-
});
|
|
221
|
-
this.emitSpeechActivity(contextId, now);
|
|
222
|
-
return true;
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
private emitSpeechActivity(contextId: string, now: number): void {
|
|
226
|
-
if (!this.bus) return;
|
|
227
|
-
const activity: VadSpeechActivityPacket = {
|
|
228
|
-
kind: "vad.speech_activity",
|
|
229
|
-
contextId,
|
|
230
|
-
timestampMs: now,
|
|
231
|
-
isAsync: true,
|
|
232
|
-
};
|
|
233
|
-
this.bus.push(Route.Main, activity);
|
|
234
|
-
}
|
|
235
|
-
|
|
236
114
|
private resetModelState(): void {
|
|
237
115
|
this.state = new Float32Array(2 * 1 * 128);
|
|
238
116
|
this.context = new Float32Array(CONTEXT_SAMPLES_16K);
|
|
239
|
-
this.lastResetMs = Date.now();
|
|
240
117
|
}
|
|
241
118
|
|
|
242
119
|
private emitError(contextId: string, err: Error): void {
|
|
@@ -266,7 +143,11 @@ async function readModelBytes(config: PluginConfig): Promise<Uint8Array> {
|
|
|
266
143
|
return new Uint8Array(await response.arrayBuffer());
|
|
267
144
|
}
|
|
268
145
|
|
|
269
|
-
function
|
|
270
|
-
const value = config[
|
|
271
|
-
|
|
146
|
+
function readSampleRate(config: PluginConfig): number {
|
|
147
|
+
const value = config["sample_rate"];
|
|
148
|
+
const sampleRate = typeof value === "number" && Number.isFinite(value) ? value : DEFAULT_SAMPLE_RATE;
|
|
149
|
+
if (sampleRate !== 16000) {
|
|
150
|
+
throw new Error(`SileroVADPlugin requires 16 kHz PCM input, got ${String(sampleRate)} Hz`);
|
|
151
|
+
}
|
|
152
|
+
return sampleRate;
|
|
272
153
|
}
|