@juspay/neurolink 13.6.5 → 13.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -2
- package/dist/auth/sessionManager.js +8 -3
- package/dist/browser/neurolink.min.js +480 -460
- package/dist/cli/commands/serve.js +11 -1
- package/dist/cli/commands/transcribe.js +39 -3
- package/dist/constants/langfuse.d.ts +2 -0
- package/dist/constants/langfuse.js +2 -0
- package/dist/core/redisConversationMemoryManager.js +6 -16
- package/dist/factories/sttDescriptors.js +7 -0
- package/dist/index.d.ts +3 -2
- package/dist/index.js +4 -2
- package/dist/neurolink.js +33 -1
- package/dist/observability/exporters/langfuseExporter.d.ts +4 -0
- package/dist/observability/exporters/langfuseExporter.js +29 -8
- package/dist/observability/otlpEncoding.js +2 -2
- package/dist/server/abstract/baseServerAdapter.js +2 -0
- package/dist/server/adapters/expressAdapter.js +1 -1
- package/dist/server/adapters/fastifyAdapter.js +4 -1
- package/dist/server/adapters/honoAdapter.js +1 -1
- package/dist/server/adapters/koaAdapter.js +1 -1
- package/dist/server/routes/transcribeRoutes.d.ts +0 -8
- package/dist/server/routes/transcribeRoutes.js +130 -3
- package/dist/server/utils/validation.d.ts +409 -15
- package/dist/server/utils/validation.js +106 -4
- package/dist/server/websocket/frameQueue.d.ts +16 -0
- package/dist/server/websocket/frameQueue.js +55 -0
- package/dist/server/websocket/realtimeTranscription.d.ts +48 -0
- package/dist/server/websocket/realtimeTranscription.js +603 -0
- package/dist/server/websocket/transcribeWebSocket.d.ts +17 -1
- package/dist/server/websocket/transcribeWebSocket.js +263 -64
- package/dist/services/server/ai/observability/langfuseSetup.js +6 -1
- package/dist/telemetry/attributes.d.ts +1 -1
- package/dist/telemetry/attributes.js +84 -8
- package/dist/types/cli.d.ts +6 -0
- package/dist/types/exporter.d.ts +3 -1
- package/dist/types/server.d.ts +304 -2
- package/dist/types/stt.d.ts +38 -3
- package/dist/types/transcribe.d.ts +401 -5
- package/dist/utils/redis.d.ts +17 -0
- package/dist/utils/redis.js +31 -0
- package/dist/voice/audioLevel.d.ts +14 -0
- package/dist/voice/audioLevel.js +60 -0
- package/dist/voice/codeMix.d.ts +40 -0
- package/dist/voice/codeMix.js +282 -0
- package/dist/voice/correction/align.d.ts +12 -0
- package/dist/voice/correction/align.js +48 -0
- package/dist/voice/correction/guards.d.ts +39 -0
- package/dist/voice/correction/guards.js +122 -0
- package/dist/voice/correction/index.d.ts +25 -3
- package/dist/voice/correction/index.js +132 -4
- package/dist/voice/correction/rewrite.d.ts +64 -3
- package/dist/voice/correction/rewrite.js +295 -25
- package/dist/voice/eval/sttEvalLangfuse.d.ts +14 -0
- package/dist/voice/eval/sttEvalLangfuse.js +182 -0
- package/dist/voice/eval/sttEvalStore.d.ts +27 -0
- package/dist/voice/eval/sttEvalStore.js +397 -0
- package/dist/voice/index.js +3 -0
- package/dist/voice/languages.d.ts +34 -0
- package/dist/voice/languages.js +290 -0
- package/dist/voice/modelAssets.d.ts +28 -0
- package/dist/voice/modelAssets.js +113 -0
- package/dist/voice/output/render.d.ts +32 -0
- package/dist/voice/output/render.js +368 -0
- package/dist/voice/providers/OpenAISTT.d.ts +11 -1
- package/dist/voice/providers/OpenAISTT.js +57 -6
- package/dist/voice/session/transcriptionSession.d.ts +55 -0
- package/dist/voice/session/transcriptionSession.js +225 -0
- package/dist/voice/streaming/chunkedStreamAdapter.d.ts +7 -0
- package/dist/voice/streaming/chunkedStreamAdapter.js +711 -28
- package/dist/voice/streaming/endpointer.d.ts +50 -17
- package/dist/voice/streaming/endpointer.js +115 -19
- package/dist/voice/transcribe.d.ts +18 -1
- package/dist/voice/transcribe.js +459 -18
- package/dist/voice/transcribeStream.d.ts +18 -1
- package/dist/voice/transcribeStream.js +413 -16
- package/dist/voice/vad/assets.d.ts +36 -0
- package/dist/voice/vad/assets.js +142 -0
- package/dist/voice/vad/frameScorer.d.ts +28 -0
- package/dist/voice/vad/frameScorer.js +90 -0
- package/dist/voice/vad/sileroVad.d.ts +43 -0
- package/dist/voice/vad/sileroVad.js +178 -0
- package/dist/voice/whistle/assets.d.ts +3 -4
- package/dist/voice/whistle/assets.js +11 -95
- package/docs-site/static/search-index.json +21 -6
- package/models/silero-vad/README.md +19 -0
- package/models/silero-vad/silero_vad.onnx +0 -0
- package/package.json +4 -1
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Silero VAD model file: the pinned revision, where it lives and the
|
|
3
|
+
* first-use download. The same rules as Whistle's assets, for one file
|
|
4
|
+
* instead of three: an explicit directory, then `NEUROLINK_VAD_DIR`, then
|
|
5
|
+
* `NEUROLINK_MODEL_DIR/silero-vad`, then the copy shipped with the package
|
|
6
|
+
* (`models/silero-vad`), then `~/.neurolink/models/silero-vad`, the only
|
|
7
|
+
* directory a download ever writes to.
|
|
8
|
+
*
|
|
9
|
+
* Upstream: `snakers4/silero-vad`, MIT.
|
|
10
|
+
*
|
|
11
|
+
* @module voice/vad/assets
|
|
12
|
+
*/
|
|
13
|
+
import { existsSync } from "node:fs";
|
|
14
|
+
import { mkdir } from "node:fs/promises";
|
|
15
|
+
import { homedir } from "node:os";
|
|
16
|
+
import { join, resolve } from "node:path";
|
|
17
|
+
import { ErrorCategory, ErrorSeverity } from "../../constants/enums.js";
|
|
18
|
+
import { STT_ERROR_CODES } from "../../types/index.js";
|
|
19
|
+
import { logger } from "../../utils/logger.js";
|
|
20
|
+
import { STTError } from "../errors.js";
|
|
21
|
+
import { checkModelFile, downloadModelAsset, packageModelsDir, } from "../modelAssets.js";
|
|
22
|
+
const SILERO_VAD_REVISION = "bfdc0193023f121ea5b3cc7b176dbed570a68a59";
|
|
23
|
+
/** The pinned model: Silero VAD v6.2, 2.3 MB, 8 and 16 kHz. */
|
|
24
|
+
export const SILERO_VAD_ASSET = {
|
|
25
|
+
name: "silero_vad.onnx",
|
|
26
|
+
url: `https://raw.githubusercontent.com/snakers4/silero-vad/${SILERO_VAD_REVISION}/src/silero_vad/data/silero_vad.onnx`,
|
|
27
|
+
sha256: "1a153a22f4509e292a94e67d6f9b85e8deb25b4988682b7e174c65279d8788e3",
|
|
28
|
+
bytes: 2_327_524,
|
|
29
|
+
};
|
|
30
|
+
/** One in-flight preparation per directory, shared by concurrent callers. */
|
|
31
|
+
const inFlight = new Map();
|
|
32
|
+
/** Directories already verified in this process. */
|
|
33
|
+
const verified = new Map();
|
|
34
|
+
/**
|
|
35
|
+
* Where the model lives: `options.modelDir` → `NEUROLINK_VAD_DIR` →
|
|
36
|
+
* `NEUROLINK_MODEL_DIR/silero-vad` → the copy shipped with the package →
|
|
37
|
+
* `~/.neurolink/models/silero-vad`.
|
|
38
|
+
*/
|
|
39
|
+
export function resolveVadDir(options) {
|
|
40
|
+
const explicit = options?.modelDir?.trim() || process.env.NEUROLINK_VAD_DIR?.trim();
|
|
41
|
+
if (explicit) {
|
|
42
|
+
return resolve(explicit);
|
|
43
|
+
}
|
|
44
|
+
const modelRoot = process.env.NEUROLINK_MODEL_DIR?.trim();
|
|
45
|
+
if (modelRoot) {
|
|
46
|
+
return resolve(modelRoot, "silero-vad");
|
|
47
|
+
}
|
|
48
|
+
const root = packageModelsDir();
|
|
49
|
+
if (root) {
|
|
50
|
+
const bundled = join(root, "silero-vad");
|
|
51
|
+
if (vadModelPresent(bundled)) {
|
|
52
|
+
return bundled;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return join(homedir(), ".neurolink", "models", "silero-vad");
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Whether a missing model may be fetched: on unless `options.autoDownload`
|
|
59
|
+
* is `false` or `NEUROLINK_VAD_AUTO_DOWNLOAD` is `0` / `false` / `off` / `no`.
|
|
60
|
+
*/
|
|
61
|
+
export function vadAutoDownloadEnabled(options) {
|
|
62
|
+
if (options?.autoDownload !== undefined) {
|
|
63
|
+
return options.autoDownload;
|
|
64
|
+
}
|
|
65
|
+
const env = process.env.NEUROLINK_VAD_AUTO_DOWNLOAD?.trim().toLowerCase();
|
|
66
|
+
return !(env === "0" || env === "false" || env === "off" || env === "no");
|
|
67
|
+
}
|
|
68
|
+
/** Cheap presence check (no hashing). */
|
|
69
|
+
export function vadModelPresent(dir) {
|
|
70
|
+
return existsSync(join(dir, SILERO_VAD_ASSET.name));
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Makes sure the model is in `dir` and matches its pinned hash, downloading
|
|
74
|
+
* it when missing and allowed. Resolves to the model's path. Concurrent
|
|
75
|
+
* calls for the same directory share one preparation.
|
|
76
|
+
*/
|
|
77
|
+
export function ensureVadModel(dir, autoDownload) {
|
|
78
|
+
const done = verified.get(dir);
|
|
79
|
+
if (done) {
|
|
80
|
+
return Promise.resolve(done);
|
|
81
|
+
}
|
|
82
|
+
const pending = inFlight.get(dir);
|
|
83
|
+
if (pending) {
|
|
84
|
+
return pending;
|
|
85
|
+
}
|
|
86
|
+
const job = prepare(dir, autoDownload)
|
|
87
|
+
.then((path) => {
|
|
88
|
+
verified.set(dir, path);
|
|
89
|
+
return path;
|
|
90
|
+
})
|
|
91
|
+
.finally(() => {
|
|
92
|
+
inFlight.delete(dir);
|
|
93
|
+
});
|
|
94
|
+
inFlight.set(dir, job);
|
|
95
|
+
return job;
|
|
96
|
+
}
|
|
97
|
+
async function prepare(dir, autoDownload) {
|
|
98
|
+
const path = join(dir, SILERO_VAD_ASSET.name);
|
|
99
|
+
const state = await checkModelFile(path, SILERO_VAD_ASSET);
|
|
100
|
+
if (state !== "ok") {
|
|
101
|
+
if (!autoDownload) {
|
|
102
|
+
throw modelUnavailable(dir, state === "corrupt"
|
|
103
|
+
? `${SILERO_VAD_ASSET.name} in ${dir} does not match its pinned SHA-256, and automatic download is off`
|
|
104
|
+
: "automatic download is off (NEUROLINK_VAD_AUTO_DOWNLOAD=0 or autoDownload: false)");
|
|
105
|
+
}
|
|
106
|
+
logger.info(`[silero-vad] downloading the Silero VAD model (${Math.max(1, Math.round(SILERO_VAD_ASSET.bytes / 1_000_000))} MB) to ${dir} — first use only`);
|
|
107
|
+
try {
|
|
108
|
+
await mkdir(dir, { recursive: true });
|
|
109
|
+
await downloadModelAsset(SILERO_VAD_ASSET, dir, "silero-vad");
|
|
110
|
+
}
|
|
111
|
+
catch (error) {
|
|
112
|
+
if (error instanceof STTError) {
|
|
113
|
+
throw error;
|
|
114
|
+
}
|
|
115
|
+
throw modelUnavailable(dir, `the download failed (${error instanceof Error ? error.message : String(error)})`, error instanceof Error ? error : undefined);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return path;
|
|
119
|
+
}
|
|
120
|
+
/** The "here is exactly what to put where" body of every "unusable" message. */
|
|
121
|
+
export function describeVadModel(dir, reason) {
|
|
122
|
+
return (`The Silero speech detector needs its model file, and ${reason}.\n` +
|
|
123
|
+
`Place it here:\n ${join(dir, SILERO_VAD_ASSET.name)} ← ${SILERO_VAD_ASSET.url}\n` +
|
|
124
|
+
"Or point NEUROLINK_VAD_DIR (or streaming.vad.modelDir) at a directory that has it " +
|
|
125
|
+
"(a full install of @juspay/neurolink ships a copy in its models/silero-vad directory), " +
|
|
126
|
+
'or set streaming.vad.engine (NEUROLINK_STT_VAD) to "energy" to stream without it.');
|
|
127
|
+
}
|
|
128
|
+
function modelUnavailable(dir, reason, originalError) {
|
|
129
|
+
return new STTError({
|
|
130
|
+
code: STT_ERROR_CODES.PROVIDER_NOT_CONFIGURED,
|
|
131
|
+
message: describeVadModel(dir, reason),
|
|
132
|
+
category: ErrorCategory.CONFIGURATION,
|
|
133
|
+
severity: ErrorSeverity.HIGH,
|
|
134
|
+
retriable: false,
|
|
135
|
+
context: {
|
|
136
|
+
provider: "silero-vad",
|
|
137
|
+
modelDir: dir,
|
|
138
|
+
files: [{ name: SILERO_VAD_ASSET.name, url: SILERO_VAD_ASSET.url }],
|
|
139
|
+
},
|
|
140
|
+
originalError,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runs a speech detector over a stream's samples as they arrive and keeps
|
|
3
|
+
* its verdicts on the stream clock, so the endpointer (which works on 20 ms
|
|
4
|
+
* blocks at the input rate) can ask "was this moment speech?" without
|
|
5
|
+
* knowing the detector's frame size or sample rate. Audio at another rate is
|
|
6
|
+
* resampled to 16 kHz for scoring only; the engine path is untouched.
|
|
7
|
+
*
|
|
8
|
+
* @module voice/vad/frameScorer
|
|
9
|
+
*/
|
|
10
|
+
import type { STTSpeechDetector } from "../../types/index.js";
|
|
11
|
+
export declare class SpeechFrameScorer {
|
|
12
|
+
readonly detector: STTSpeechDetector;
|
|
13
|
+
private readonly probs;
|
|
14
|
+
/** Frames scored so far: the detector's clock. */
|
|
15
|
+
private frames;
|
|
16
|
+
private readonly frame;
|
|
17
|
+
private frameFill;
|
|
18
|
+
private readonly resampler;
|
|
19
|
+
constructor(detector: STTSpeechDetector, inputRate: number);
|
|
20
|
+
/** Scores every full frame the samples complete; a partial frame waits for the next push. */
|
|
21
|
+
push(samples: Float32Array): Promise<void>;
|
|
22
|
+
/**
|
|
23
|
+
* The probability of the frame covering `seconds` on the stream clock; the
|
|
24
|
+
* newest frame when that one is not complete yet (a lag of at most one
|
|
25
|
+
* frame), `undefined` before anything was scored.
|
|
26
|
+
*/
|
|
27
|
+
probabilityAt(seconds: number): number | undefined;
|
|
28
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runs a speech detector over a stream's samples as they arrive and keeps
|
|
3
|
+
* its verdicts on the stream clock, so the endpointer (which works on 20 ms
|
|
4
|
+
* blocks at the input rate) can ask "was this moment speech?" without
|
|
5
|
+
* knowing the detector's frame size or sample rate. Audio at another rate is
|
|
6
|
+
* resampled to 16 kHz for scoring only; the engine path is untouched.
|
|
7
|
+
*
|
|
8
|
+
* @module voice/vad/frameScorer
|
|
9
|
+
*/
|
|
10
|
+
/** Verdicts kept: 8192 frames of 32 ms is about 4.4 minutes, far past what any block asks about. */
|
|
11
|
+
const RING = 8192;
|
|
12
|
+
const RATE = 16000;
|
|
13
|
+
/** Linear-interpolation resampler that carries its position across pushes. */
|
|
14
|
+
class StreamingResampler {
|
|
15
|
+
ratio;
|
|
16
|
+
carry = new Float32Array(0);
|
|
17
|
+
/** Position of the next output sample, in input samples, relative to `carry[0]`. */
|
|
18
|
+
phase = 0;
|
|
19
|
+
constructor(ratio) {
|
|
20
|
+
this.ratio = ratio;
|
|
21
|
+
}
|
|
22
|
+
push(input) {
|
|
23
|
+
const buf = new Float32Array(this.carry.length + input.length);
|
|
24
|
+
buf.set(this.carry, 0);
|
|
25
|
+
buf.set(input, this.carry.length);
|
|
26
|
+
const n = buf.length;
|
|
27
|
+
const out = new Float32Array(Math.ceil(n / this.ratio) + 1);
|
|
28
|
+
let count = 0;
|
|
29
|
+
let pos = this.phase;
|
|
30
|
+
while (Math.floor(pos) + 1 < n) {
|
|
31
|
+
const i0 = Math.floor(pos);
|
|
32
|
+
const f = pos - i0;
|
|
33
|
+
out[count++] = buf[i0] + (buf[i0 + 1] - buf[i0]) * f;
|
|
34
|
+
pos += this.ratio;
|
|
35
|
+
}
|
|
36
|
+
const keepFrom = Math.min(Math.floor(pos), n);
|
|
37
|
+
this.carry = buf.slice(keepFrom);
|
|
38
|
+
this.phase = pos - keepFrom;
|
|
39
|
+
return out.subarray(0, count);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
export class SpeechFrameScorer {
|
|
43
|
+
detector;
|
|
44
|
+
probs = new Float32Array(RING);
|
|
45
|
+
/** Frames scored so far: the detector's clock. */
|
|
46
|
+
frames = 0;
|
|
47
|
+
frame;
|
|
48
|
+
frameFill = 0;
|
|
49
|
+
resampler;
|
|
50
|
+
constructor(detector, inputRate) {
|
|
51
|
+
this.detector = detector;
|
|
52
|
+
this.frame = new Float32Array(detector.frameSamples);
|
|
53
|
+
this.resampler =
|
|
54
|
+
inputRate === RATE ? null : new StreamingResampler(inputRate / RATE);
|
|
55
|
+
detector.reset();
|
|
56
|
+
}
|
|
57
|
+
/** Scores every full frame the samples complete; a partial frame waits for the next push. */
|
|
58
|
+
async push(samples) {
|
|
59
|
+
const pcm = this.resampler ? this.resampler.push(samples) : samples;
|
|
60
|
+
let i = 0;
|
|
61
|
+
while (i < pcm.length) {
|
|
62
|
+
const n = Math.min(pcm.length - i, this.frame.length - this.frameFill);
|
|
63
|
+
this.frame.set(pcm.subarray(i, i + n), this.frameFill);
|
|
64
|
+
this.frameFill += n;
|
|
65
|
+
i += n;
|
|
66
|
+
if (this.frameFill === this.frame.length) {
|
|
67
|
+
const p = await this.detector.score(this.frame);
|
|
68
|
+
this.probs[this.frames % RING] = p;
|
|
69
|
+
this.frames++;
|
|
70
|
+
this.frameFill = 0;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* The probability of the frame covering `seconds` on the stream clock; the
|
|
76
|
+
* newest frame when that one is not complete yet (a lag of at most one
|
|
77
|
+
* frame), `undefined` before anything was scored.
|
|
78
|
+
*/
|
|
79
|
+
probabilityAt(seconds) {
|
|
80
|
+
if (this.frames === 0) {
|
|
81
|
+
return undefined;
|
|
82
|
+
}
|
|
83
|
+
let k = Math.floor((seconds * RATE) / this.frame.length);
|
|
84
|
+
k = Math.min(Math.max(0, k), this.frames - 1);
|
|
85
|
+
if (k < this.frames - RING) {
|
|
86
|
+
return undefined;
|
|
87
|
+
}
|
|
88
|
+
return this.probs[k % RING];
|
|
89
|
+
}
|
|
90
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Silero VAD as the chunked stream's speech detector.
|
|
3
|
+
*
|
|
4
|
+
* A 2.3 MB ONNX model (MIT, trained on 6000+ languages) scores 32 ms frames
|
|
5
|
+
* of 16 kHz audio: the probability that the frame is a human voice, which an
|
|
6
|
+
* energy gate cannot tell from a chair, a keyboard or a far-away TV. It runs
|
|
7
|
+
* through `onnxruntime-node`, an optional dependency with prebuilt CPU
|
|
8
|
+
* binaries for Linux, macOS and Windows on x64 and arm64; measured at about
|
|
9
|
+
* 0.1 ms per frame on one thread, so a stream costs well under one percent
|
|
10
|
+
* of a core. One inference session per model file is shared by every stream
|
|
11
|
+
* in the process; each detector keeps only its own LSTM state and the 64
|
|
12
|
+
* samples of context the model wants in front of every frame.
|
|
13
|
+
*
|
|
14
|
+
* Nothing here is imported statically by the rest of the package: the
|
|
15
|
+
* adapter loads this module only when a stream needs it, and the runtime
|
|
16
|
+
* itself is imported through a variable specifier so a typecheck or a bundle
|
|
17
|
+
* does not need the native module present.
|
|
18
|
+
*
|
|
19
|
+
* @module voice/vad/sileroVad
|
|
20
|
+
*/
|
|
21
|
+
import type { STTSpeechDetector, STTVadEngine, STTVadOptions } from "../../types/index.js";
|
|
22
|
+
/** Samples per frame at 16 kHz (32 ms). */
|
|
23
|
+
export declare const SILERO_FRAME_SAMPLES = 512;
|
|
24
|
+
/** Thresholds of `STTVadOptions` when the caller sets none. */
|
|
25
|
+
export declare const VAD_DEFAULTS: {
|
|
26
|
+
readonly onset: 0.6;
|
|
27
|
+
readonly sustain: 0.35;
|
|
28
|
+
readonly startMs: 120;
|
|
29
|
+
};
|
|
30
|
+
/** `options.engine`, else `NEUROLINK_STT_VAD`, else `"auto"`. */
|
|
31
|
+
export declare function resolveVadEngine(options?: STTVadOptions): STTVadEngine;
|
|
32
|
+
/** The three thresholds with the defaults applied and nonsense clamped. */
|
|
33
|
+
export declare function resolveVadThresholds(options?: STTVadOptions): {
|
|
34
|
+
onset: number;
|
|
35
|
+
sustain: number;
|
|
36
|
+
startMs: number;
|
|
37
|
+
};
|
|
38
|
+
/**
|
|
39
|
+
* A detector for one stream: the model file located (and fetched when
|
|
40
|
+
* allowed), the runtime loaded, the shared session created on first use.
|
|
41
|
+
* Throws an `STTError` saying exactly what is missing when it cannot.
|
|
42
|
+
*/
|
|
43
|
+
export declare function loadSileroDetector(options?: STTVadOptions): Promise<STTSpeechDetector>;
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Silero VAD as the chunked stream's speech detector.
|
|
3
|
+
*
|
|
4
|
+
* A 2.3 MB ONNX model (MIT, trained on 6000+ languages) scores 32 ms frames
|
|
5
|
+
* of 16 kHz audio: the probability that the frame is a human voice, which an
|
|
6
|
+
* energy gate cannot tell from a chair, a keyboard or a far-away TV. It runs
|
|
7
|
+
* through `onnxruntime-node`, an optional dependency with prebuilt CPU
|
|
8
|
+
* binaries for Linux, macOS and Windows on x64 and arm64; measured at about
|
|
9
|
+
* 0.1 ms per frame on one thread, so a stream costs well under one percent
|
|
10
|
+
* of a core. One inference session per model file is shared by every stream
|
|
11
|
+
* in the process; each detector keeps only its own LSTM state and the 64
|
|
12
|
+
* samples of context the model wants in front of every frame.
|
|
13
|
+
*
|
|
14
|
+
* Nothing here is imported statically by the rest of the package: the
|
|
15
|
+
* adapter loads this module only when a stream needs it, and the runtime
|
|
16
|
+
* itself is imported through a variable specifier so a typecheck or a bundle
|
|
17
|
+
* does not need the native module present.
|
|
18
|
+
*
|
|
19
|
+
* @module voice/vad/sileroVad
|
|
20
|
+
*/
|
|
21
|
+
import { ErrorCategory, ErrorSeverity } from "../../constants/enums.js";
|
|
22
|
+
import { STT_ERROR_CODES } from "../../types/index.js";
|
|
23
|
+
import { STTError } from "../errors.js";
|
|
24
|
+
import { ensureVadModel, resolveVadDir, vadAutoDownloadEnabled, } from "./assets.js";
|
|
25
|
+
/** Samples per frame at 16 kHz (32 ms). */
|
|
26
|
+
export const SILERO_FRAME_SAMPLES = 512;
|
|
27
|
+
/** Samples of the previous frame the model wants in front of the new one. */
|
|
28
|
+
const CONTEXT_SAMPLES = 64;
|
|
29
|
+
const STATE_DIMS = [2, 1, 128];
|
|
30
|
+
const STATE_SIZE = 2 * 128;
|
|
31
|
+
/** Thresholds of `STTVadOptions` when the caller sets none. */
|
|
32
|
+
export const VAD_DEFAULTS = {
|
|
33
|
+
onset: 0.6,
|
|
34
|
+
sustain: 0.35,
|
|
35
|
+
startMs: 120,
|
|
36
|
+
};
|
|
37
|
+
/** `options.engine`, else `NEUROLINK_STT_VAD`, else `"auto"`. */
|
|
38
|
+
export function resolveVadEngine(options) {
|
|
39
|
+
const raw = (options?.engine ?? process.env.NEUROLINK_STT_VAD ?? "")
|
|
40
|
+
.trim()
|
|
41
|
+
.toLowerCase();
|
|
42
|
+
return raw === "silero" || raw === "energy" ? raw : "auto";
|
|
43
|
+
}
|
|
44
|
+
/** The three thresholds with the defaults applied and nonsense clamped. */
|
|
45
|
+
export function resolveVadThresholds(options) {
|
|
46
|
+
const unit = (value, fallback) => typeof value === "number" && Number.isFinite(value)
|
|
47
|
+
? Math.min(1, Math.max(0, value))
|
|
48
|
+
: fallback;
|
|
49
|
+
const onset = unit(options?.onset, VAD_DEFAULTS.onset);
|
|
50
|
+
const sustain = Math.min(unit(options?.sustain, VAD_DEFAULTS.sustain), onset);
|
|
51
|
+
const startMs = typeof options?.startMs === "number" &&
|
|
52
|
+
Number.isFinite(options.startMs) &&
|
|
53
|
+
options.startMs >= 0
|
|
54
|
+
? options.startMs
|
|
55
|
+
: VAD_DEFAULTS.startMs;
|
|
56
|
+
return { onset, sustain, startMs };
|
|
57
|
+
}
|
|
58
|
+
let runtime;
|
|
59
|
+
const sessions = new Map();
|
|
60
|
+
function isRuntime(value) {
|
|
61
|
+
if (!value || typeof value !== "object") {
|
|
62
|
+
return false;
|
|
63
|
+
}
|
|
64
|
+
const o = value;
|
|
65
|
+
const session = o.InferenceSession;
|
|
66
|
+
const create = typeof session === "function" ? Reflect.get(session, "create") : undefined;
|
|
67
|
+
return typeof create === "function" && typeof o.Tensor === "function";
|
|
68
|
+
}
|
|
69
|
+
async function loadRuntime() {
|
|
70
|
+
// A variable specifier: bundlers leave the native module alone, and a
|
|
71
|
+
// typecheck does not need the optional dependency installed.
|
|
72
|
+
const specifier = "onnxruntime-node";
|
|
73
|
+
let mod;
|
|
74
|
+
try {
|
|
75
|
+
mod = await import(specifier);
|
|
76
|
+
}
|
|
77
|
+
catch (error) {
|
|
78
|
+
throw new STTError({
|
|
79
|
+
code: STT_ERROR_CODES.PROVIDER_NOT_CONFIGURED,
|
|
80
|
+
message: "The Silero speech detector needs the optional dependency onnxruntime-node, which did not load " +
|
|
81
|
+
`(${error instanceof Error ? error.message : String(error)}). Install it (pnpm add onnxruntime-node), ` +
|
|
82
|
+
'or set streaming.vad.engine (NEUROLINK_STT_VAD) to "energy" to stream without it.',
|
|
83
|
+
category: ErrorCategory.CONFIGURATION,
|
|
84
|
+
severity: ErrorSeverity.HIGH,
|
|
85
|
+
retriable: false,
|
|
86
|
+
context: { provider: "silero-vad" },
|
|
87
|
+
originalError: error instanceof Error ? error : undefined,
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
const candidates = [mod, mod.default];
|
|
91
|
+
const found = candidates.find(isRuntime);
|
|
92
|
+
if (!found) {
|
|
93
|
+
throw new STTError({
|
|
94
|
+
code: STT_ERROR_CODES.PROVIDER_NOT_CONFIGURED,
|
|
95
|
+
message: "onnxruntime-node loaded but did not export InferenceSession and Tensor; the Silero speech detector cannot run.",
|
|
96
|
+
category: ErrorCategory.CONFIGURATION,
|
|
97
|
+
severity: ErrorSeverity.HIGH,
|
|
98
|
+
retriable: false,
|
|
99
|
+
context: { provider: "silero-vad" },
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
return found;
|
|
103
|
+
}
|
|
104
|
+
function sessionFor(rt, modelPath) {
|
|
105
|
+
let session = sessions.get(modelPath);
|
|
106
|
+
if (!session) {
|
|
107
|
+
session = rt.InferenceSession.create(modelPath, {
|
|
108
|
+
executionProviders: ["cpu"],
|
|
109
|
+
intraOpNumThreads: 1,
|
|
110
|
+
interOpNumThreads: 1,
|
|
111
|
+
logSeverityLevel: 3,
|
|
112
|
+
});
|
|
113
|
+
sessions.set(modelPath, session);
|
|
114
|
+
session.catch(() => sessions.delete(modelPath));
|
|
115
|
+
}
|
|
116
|
+
return session;
|
|
117
|
+
}
|
|
118
|
+
class SileroSpeechDetector {
|
|
119
|
+
rt;
|
|
120
|
+
session;
|
|
121
|
+
name = "silero";
|
|
122
|
+
frameSamples = SILERO_FRAME_SAMPLES;
|
|
123
|
+
state;
|
|
124
|
+
context = new Float32Array(CONTEXT_SAMPLES);
|
|
125
|
+
sr;
|
|
126
|
+
constructor(rt, session) {
|
|
127
|
+
this.rt = rt;
|
|
128
|
+
this.session = session;
|
|
129
|
+
this.sr = new rt.Tensor("int64", BigInt64Array.from([16000n]), []);
|
|
130
|
+
this.state = this.freshState();
|
|
131
|
+
}
|
|
132
|
+
reset() {
|
|
133
|
+
this.state = this.freshState();
|
|
134
|
+
this.context = new Float32Array(CONTEXT_SAMPLES);
|
|
135
|
+
}
|
|
136
|
+
async score(frame) {
|
|
137
|
+
if (frame.length !== SILERO_FRAME_SAMPLES) {
|
|
138
|
+
throw new Error(`Silero VAD scores ${SILERO_FRAME_SAMPLES}-sample frames, got ${frame.length}`);
|
|
139
|
+
}
|
|
140
|
+
const input = new Float32Array(CONTEXT_SAMPLES + SILERO_FRAME_SAMPLES);
|
|
141
|
+
input.set(this.context, 0);
|
|
142
|
+
input.set(frame, CONTEXT_SAMPLES);
|
|
143
|
+
const out = await this.session.run({
|
|
144
|
+
input: new this.rt.Tensor("float32", input, [1, input.length]),
|
|
145
|
+
state: this.state,
|
|
146
|
+
sr: this.sr,
|
|
147
|
+
});
|
|
148
|
+
const prob = out.output?.data[0];
|
|
149
|
+
const next = out.stateN;
|
|
150
|
+
if (typeof prob !== "number" || !next) {
|
|
151
|
+
throw new Error("Silero VAD returned no probability");
|
|
152
|
+
}
|
|
153
|
+
this.state = next;
|
|
154
|
+
this.context = frame.slice(SILERO_FRAME_SAMPLES - CONTEXT_SAMPLES);
|
|
155
|
+
return Math.min(1, Math.max(0, prob));
|
|
156
|
+
}
|
|
157
|
+
freshState() {
|
|
158
|
+
return new this.rt.Tensor("float32", new Float32Array(STATE_SIZE), STATE_DIMS);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* A detector for one stream: the model file located (and fetched when
|
|
163
|
+
* allowed), the runtime loaded, the shared session created on first use.
|
|
164
|
+
* Throws an `STTError` saying exactly what is missing when it cannot.
|
|
165
|
+
*/
|
|
166
|
+
export async function loadSileroDetector(options) {
|
|
167
|
+
// The runtime first: without onnxruntime-node there is nothing to run the
|
|
168
|
+
// model on, so it must not be prepared (or downloaded) for nothing.
|
|
169
|
+
if (!runtime) {
|
|
170
|
+
runtime = loadRuntime().catch((error) => {
|
|
171
|
+
runtime = undefined;
|
|
172
|
+
throw error;
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
const rt = await runtime;
|
|
176
|
+
const modelPath = await ensureVadModel(resolveVadDir(options), vadAutoDownloadEnabled(options));
|
|
177
|
+
return new SileroSpeechDetector(rt, await sessionFor(rt, modelPath));
|
|
178
|
+
}
|
|
@@ -23,10 +23,9 @@ export declare const WHISTLE_ASSETS: readonly WhistleAsset[];
|
|
|
23
23
|
*/
|
|
24
24
|
export declare function resolveWhistleDir(slice?: STTCredentials["whistle"]): string;
|
|
25
25
|
/**
|
|
26
|
-
* The package's own `models/whistle`,
|
|
27
|
-
* to the
|
|
28
|
-
*
|
|
29
|
-
* below the package root; `null` when nothing above looks like the package.
|
|
26
|
+
* The package's own `models/whistle` (see `packageModelsDir`, which walks up
|
|
27
|
+
* from this module to the package root); `null` when nothing above looks
|
|
28
|
+
* like the package.
|
|
30
29
|
*/
|
|
31
30
|
export declare function bundledWhistleDir(): string | null;
|
|
32
31
|
/**
|
|
@@ -12,16 +12,15 @@
|
|
|
12
12
|
*
|
|
13
13
|
* @module voice/whistle/assets
|
|
14
14
|
*/
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
import { mkdir, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
15
|
+
import { existsSync } from "node:fs";
|
|
16
|
+
import { mkdir } from "node:fs/promises";
|
|
18
17
|
import { homedir } from "node:os";
|
|
19
|
-
import {
|
|
20
|
-
import { fileURLToPath } from "node:url";
|
|
18
|
+
import { join, resolve } from "node:path";
|
|
21
19
|
import { ErrorCategory, ErrorSeverity } from "../../constants/enums.js";
|
|
22
20
|
import { STT_ERROR_CODES } from "../../types/index.js";
|
|
23
21
|
import { logger } from "../../utils/logger.js";
|
|
24
22
|
import { STTError } from "../errors.js";
|
|
23
|
+
import { checkModelFile, downloadModelAsset, packageModelsDir, } from "../modelAssets.js";
|
|
25
24
|
const NEEDLE_REVISION = "2ae11323dc000f5e70c49f7403efa6af12ba9e67";
|
|
26
25
|
const WHISTLE_REVISION = "b358ddadd89b7a713b5aa131f23032d3cca1b251";
|
|
27
26
|
/** Pinned files, in the order they are fetched. */
|
|
@@ -45,8 +44,6 @@ export const WHISTLE_ASSETS = [
|
|
|
45
44
|
bytes: 16_919_407,
|
|
46
45
|
},
|
|
47
46
|
];
|
|
48
|
-
/** Per-file download deadline; generous because the model is 17 MB. */
|
|
49
|
-
const DOWNLOAD_TIMEOUT_MS = 10 * 60_000;
|
|
50
47
|
/** One in-flight preparation per directory, shared by concurrent callers. */
|
|
51
48
|
const inFlight = new Map();
|
|
52
49
|
/** Directories already verified in this process. */
|
|
@@ -72,46 +69,14 @@ export function resolveWhistleDir(slice) {
|
|
|
72
69
|
}
|
|
73
70
|
return join(homedir(), ".neurolink", "models", "whistle");
|
|
74
71
|
}
|
|
75
|
-
let bundledDir;
|
|
76
72
|
/**
|
|
77
|
-
* The package's own `models/whistle`,
|
|
78
|
-
* to the
|
|
79
|
-
*
|
|
80
|
-
* below the package root; `null` when nothing above looks like the package.
|
|
73
|
+
* The package's own `models/whistle` (see `packageModelsDir`, which walks up
|
|
74
|
+
* from this module to the package root); `null` when nothing above looks
|
|
75
|
+
* like the package.
|
|
81
76
|
*/
|
|
82
77
|
export function bundledWhistleDir() {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
}
|
|
86
|
-
bundledDir = null;
|
|
87
|
-
let dir;
|
|
88
|
-
try {
|
|
89
|
-
dir = dirname(fileURLToPath(import.meta.url));
|
|
90
|
-
}
|
|
91
|
-
catch {
|
|
92
|
-
return bundledDir;
|
|
93
|
-
}
|
|
94
|
-
for (let depth = 0; depth < 8; depth += 1) {
|
|
95
|
-
const manifest = join(dir, "package.json");
|
|
96
|
-
if (existsSync(manifest)) {
|
|
97
|
-
try {
|
|
98
|
-
const parsed = JSON.parse(readFileSync(manifest, "utf8"));
|
|
99
|
-
if (parsed.name === "@juspay/neurolink") {
|
|
100
|
-
bundledDir = join(dir, "models", "whistle");
|
|
101
|
-
return bundledDir;
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
catch {
|
|
105
|
-
// Not a manifest we can read; keep walking up.
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
const parent = dirname(dir);
|
|
109
|
-
if (parent === dir) {
|
|
110
|
-
break;
|
|
111
|
-
}
|
|
112
|
-
dir = parent;
|
|
113
|
-
}
|
|
114
|
-
return bundledDir;
|
|
78
|
+
const root = packageModelsDir();
|
|
79
|
+
return root ? join(root, "whistle") : null;
|
|
115
80
|
}
|
|
116
81
|
/**
|
|
117
82
|
* Whether missing files may be fetched: on unless `slice.autoDownload` is
|
|
@@ -157,7 +122,7 @@ async function prepare(dir, autoDownload) {
|
|
|
157
122
|
const missing = [];
|
|
158
123
|
const corrupt = [];
|
|
159
124
|
for (const asset of WHISTLE_ASSETS) {
|
|
160
|
-
const state = await
|
|
125
|
+
const state = await checkModelFile(join(dir, asset.name), asset);
|
|
161
126
|
if (state === "missing") {
|
|
162
127
|
missing.push(asset);
|
|
163
128
|
}
|
|
@@ -177,7 +142,7 @@ async function prepare(dir, autoDownload) {
|
|
|
177
142
|
try {
|
|
178
143
|
await mkdir(dir, { recursive: true });
|
|
179
144
|
for (const asset of needed) {
|
|
180
|
-
await
|
|
145
|
+
await downloadModelAsset(asset, dir, "whistle");
|
|
181
146
|
}
|
|
182
147
|
}
|
|
183
148
|
catch (error) {
|
|
@@ -193,55 +158,6 @@ async function prepare(dir, autoDownload) {
|
|
|
193
158
|
"whistle.cact": join(dir, "whistle.cact"),
|
|
194
159
|
};
|
|
195
160
|
}
|
|
196
|
-
async function checkFile(path, asset) {
|
|
197
|
-
try {
|
|
198
|
-
const info = await stat(path);
|
|
199
|
-
if (!info.isFile()) {
|
|
200
|
-
return "missing";
|
|
201
|
-
}
|
|
202
|
-
if (info.size !== asset.bytes) {
|
|
203
|
-
return "corrupt";
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
catch {
|
|
207
|
-
return "missing";
|
|
208
|
-
}
|
|
209
|
-
return (await hashFile(path)) === asset.sha256 ? "ok" : "corrupt";
|
|
210
|
-
}
|
|
211
|
-
function hashFile(path) {
|
|
212
|
-
return new Promise((resolvePromise, reject) => {
|
|
213
|
-
const hash = createHash("sha256");
|
|
214
|
-
createReadStream(path)
|
|
215
|
-
.on("data", (chunk) => hash.update(chunk))
|
|
216
|
-
.on("error", reject)
|
|
217
|
-
.on("end", () => resolvePromise(hash.digest("hex")));
|
|
218
|
-
});
|
|
219
|
-
}
|
|
220
|
-
async function download(asset, dir) {
|
|
221
|
-
const response = await fetch(asset.url, {
|
|
222
|
-
signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS),
|
|
223
|
-
redirect: "follow",
|
|
224
|
-
});
|
|
225
|
-
if (!response.ok) {
|
|
226
|
-
throw new Error(`${asset.name}: HTTP ${response.status}`);
|
|
227
|
-
}
|
|
228
|
-
const body = Buffer.from(await response.arrayBuffer());
|
|
229
|
-
const digest = createHash("sha256").update(body).digest("hex");
|
|
230
|
-
if (digest !== asset.sha256) {
|
|
231
|
-
throw new Error(`${asset.name}: SHA-256 mismatch (expected ${asset.sha256}, got ${digest})`);
|
|
232
|
-
}
|
|
233
|
-
const target = join(dir, asset.name);
|
|
234
|
-
const temp = join(dir, `.${asset.name}.${process.pid}.${randomBytes(4).toString("hex")}.part`);
|
|
235
|
-
try {
|
|
236
|
-
await writeFile(temp, body);
|
|
237
|
-
await rename(temp, target);
|
|
238
|
-
}
|
|
239
|
-
catch (error) {
|
|
240
|
-
await rm(temp, { force: true }).catch(() => undefined);
|
|
241
|
-
throw error;
|
|
242
|
-
}
|
|
243
|
-
logger.debug(`[whistle] ${asset.name} verified and saved to ${target}`);
|
|
244
|
-
}
|
|
245
161
|
/** The friendly "here is exactly what to put where" error. */
|
|
246
162
|
/** The three files, where they go and where they come from — the body of every "unusable" message. */
|
|
247
163
|
export function describeWhistleAssets(dir, reason) {
|