@juspay/neurolink 12.48.2 → 12.49.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/CHANGELOG.md +3 -3
  2. package/dist/browser/neurolink.min.js +616 -595
  3. package/dist/cli/commands/serve.d.ts +0 -3
  4. package/dist/cli/commands/serve.js +85 -3
  5. package/dist/cli/commands/transcribe.d.ts +24 -0
  6. package/dist/cli/commands/transcribe.js +718 -0
  7. package/dist/cli/factories/commandFactory.d.ts +10 -0
  8. package/dist/cli/factories/commandFactory.js +13 -0
  9. package/dist/cli/parser.js +3 -0
  10. package/dist/factories/mediaHandlerCatalog.js +11 -9
  11. package/dist/factories/sttDescriptors.d.ts +63 -0
  12. package/dist/factories/sttDescriptors.js +286 -0
  13. package/dist/index.d.ts +4 -1
  14. package/dist/index.js +7 -1
  15. package/dist/neurolink.d.ts +68 -1
  16. package/dist/neurolink.js +202 -4
  17. package/dist/processors/media/AudioProcessor.d.ts +15 -56
  18. package/dist/processors/media/AudioProcessor.js +105 -276
  19. package/dist/processors/media/VideoProcessor.d.ts +3 -5
  20. package/dist/processors/media/VideoProcessor.js +52 -51
  21. package/dist/server/abstract/baseServerAdapter.d.ts +7 -0
  22. package/dist/server/abstract/baseServerAdapter.js +13 -1
  23. package/dist/server/adapters/expressAdapter.d.ts +1 -0
  24. package/dist/server/adapters/expressAdapter.js +48 -4
  25. package/dist/server/adapters/fastifyAdapter.d.ts +1 -0
  26. package/dist/server/adapters/fastifyAdapter.js +38 -1
  27. package/dist/server/adapters/honoAdapter.d.ts +1 -4
  28. package/dist/server/adapters/honoAdapter.js +83 -1
  29. package/dist/server/adapters/koaAdapter.d.ts +1 -0
  30. package/dist/server/adapters/koaAdapter.js +43 -0
  31. package/dist/server/index.d.ts +3 -2
  32. package/dist/server/index.js +3 -2
  33. package/dist/server/routes/index.d.ts +1 -0
  34. package/dist/server/routes/index.js +3 -0
  35. package/dist/server/routes/transcribeRoutes.d.ts +23 -0
  36. package/dist/server/routes/transcribeRoutes.js +506 -0
  37. package/dist/server/utils/rawBody.d.ts +32 -0
  38. package/dist/server/utils/rawBody.js +76 -0
  39. package/dist/server/utils/validation.d.ts +192 -0
  40. package/dist/server/utils/validation.js +100 -0
  41. package/dist/server/websocket/transcribeWebSocket.d.ts +42 -0
  42. package/dist/server/websocket/transcribeWebSocket.js +264 -0
  43. package/dist/types/cli.d.ts +46 -0
  44. package/dist/types/index.d.ts +1 -0
  45. package/dist/types/index.js +1 -0
  46. package/dist/types/providers.d.ts +7 -1
  47. package/dist/types/server.d.ts +117 -1
  48. package/dist/types/stt.d.ts +160 -7
  49. package/dist/types/transcribe.d.ts +506 -0
  50. package/dist/types/transcribe.js +29 -0
  51. package/dist/utils/speakerTurns.d.ts +28 -0
  52. package/dist/utils/speakerTurns.js +93 -0
  53. package/dist/utils/sttProcessor.d.ts +20 -2
  54. package/dist/utils/sttProcessor.js +73 -5
  55. package/dist/voice/correction/dictionary.d.ts +49 -0
  56. package/dist/voice/correction/dictionary.js +134 -0
  57. package/dist/voice/correction/guard.d.ts +35 -0
  58. package/dist/voice/correction/guard.js +161 -0
  59. package/dist/voice/correction/index.d.ts +47 -0
  60. package/dist/voice/correction/index.js +211 -0
  61. package/dist/voice/correction/rewrite.d.ts +92 -0
  62. package/dist/voice/correction/rewrite.js +322 -0
  63. package/dist/voice/errors.d.ts +15 -0
  64. package/dist/voice/errors.js +45 -0
  65. package/dist/voice/index.d.ts +20 -2
  66. package/dist/voice/index.js +75 -9
  67. package/dist/voice/providers/AzureSTT.d.ts +10 -7
  68. package/dist/voice/providers/AzureSTT.js +23 -75
  69. package/dist/voice/providers/DeepgramSTT.d.ts +17 -2
  70. package/dist/voice/providers/DeepgramSTT.js +109 -13
  71. package/dist/voice/providers/ElevenLabsSTT.d.ts +12 -2
  72. package/dist/voice/providers/ElevenLabsSTT.js +34 -8
  73. package/dist/voice/providers/GoogleSTT.d.ts +10 -8
  74. package/dist/voice/providers/GoogleSTT.js +21 -75
  75. package/dist/voice/providers/OpenAISTT.d.ts +30 -9
  76. package/dist/voice/providers/OpenAISTT.js +196 -21
  77. package/dist/voice/providers/WhistleSTT.d.ts +87 -0
  78. package/dist/voice/providers/WhistleSTT.js +313 -0
  79. package/dist/voice/streaming/chunkedStreamAdapter.d.ts +19 -0
  80. package/dist/voice/streaming/chunkedStreamAdapter.js +519 -0
  81. package/dist/voice/streaming/endpointer.d.ts +81 -0
  82. package/dist/voice/streaming/endpointer.js +247 -0
  83. package/dist/voice/streaming/localAgreement.d.ts +30 -0
  84. package/dist/voice/streaming/localAgreement.js +44 -0
  85. package/dist/voice/sttEnvCredentials.d.ts +12 -0
  86. package/dist/voice/sttEnvCredentials.js +12 -0
  87. package/dist/voice/transcribe.d.ts +47 -0
  88. package/dist/voice/transcribe.js +519 -0
  89. package/dist/voice/transcribeStream.d.ts +4 -0
  90. package/dist/voice/transcribeStream.js +584 -0
  91. package/dist/voice/whistle/assets.d.ts +47 -0
  92. package/dist/voice/whistle/assets.js +270 -0
  93. package/dist/voice/whistle/audio.d.ts +32 -0
  94. package/dist/voice/whistle/audio.js +267 -0
  95. package/dist/voice/whistle/engine.d.ts +32 -0
  96. package/dist/voice/whistle/engine.js +193 -0
  97. package/dist/voice/whistle/worker.d.ts +18 -0
  98. package/dist/voice/whistle/worker.js +294 -0
  99. package/docs-site/static/search-index.json +66 -8
  100. package/models/whistle/README.md +18 -0
  101. package/models/whistle/needle.js +2 -0
  102. package/models/whistle/needle.wasm +0 -0
  103. package/models/whistle/whistle.cact +0 -0
  104. package/package.json +7 -3
@@ -0,0 +1,270 @@
1
+ /**
2
+ * Whistle assets: the pinned manifest, where the files live, and the
3
+ * first-use download.
4
+ *
5
+ * Three files make up the engine — the Cactus "needle" runtime (an Emscripten
6
+ * build: `needle.js` + `needle.wasm`) and the Whistle speech model
7
+ * (`whistle.cact`). They are fetched from pinned Hugging Face revisions on
8
+ * first use, verified against a SHA-256, and written atomically (temp file +
9
+ * rename), so an interrupted download never leaves a file that looks complete.
10
+ *
11
+ * Both upstream repositories are Apache-2.0.
12
+ *
13
+ * @module voice/whistle/assets
14
+ */
15
+ import { createHash, randomBytes } from "node:crypto";
16
+ import { createReadStream, existsSync, readFileSync } from "node:fs";
17
+ import { mkdir, rename, rm, stat, writeFile } from "node:fs/promises";
18
+ import { homedir } from "node:os";
19
+ import { dirname, join, resolve } from "node:path";
20
+ import { fileURLToPath } from "node:url";
21
+ import { ErrorCategory, ErrorSeverity } from "../../constants/enums.js";
22
+ import { STT_ERROR_CODES } from "../../types/index.js";
23
+ import { logger } from "../../utils/logger.js";
24
+ import { STTError } from "../errors.js";
25
+ const NEEDLE_REVISION = "2ae11323dc000f5e70c49f7403efa6af12ba9e67";
26
+ const WHISTLE_REVISION = "b358ddadd89b7a713b5aa131f23032d3cca1b251";
27
+ /** Pinned files, in the order they are fetched. */
28
+ export const WHISTLE_ASSETS = [
29
+ {
30
+ name: "needle.js",
31
+ url: `https://huggingface.co/Cactus-Compute/needle3/resolve/${NEEDLE_REVISION}/wasm/needle.js`,
32
+ sha256: "964681b2a5ec3c4db2f06e45a5b60c8981a7d4ab4cac16ef6bf5b1988657a5f1",
33
+ bytes: 63_072,
34
+ },
35
+ {
36
+ name: "needle.wasm",
37
+ url: `https://huggingface.co/Cactus-Compute/needle3/resolve/${NEEDLE_REVISION}/wasm/needle.wasm`,
38
+ sha256: "c43f48e11f302087250d1e406024956781cd2f02595a5e343b3d7ddd5ef707fa",
39
+ bytes: 923_348,
40
+ },
41
+ {
42
+ name: "whistle.cact",
43
+ url: `https://huggingface.co/Cactus-Compute/whistle/resolve/${WHISTLE_REVISION}/whistle.cact`,
44
+ sha256: "b6e02f048568ac5d01a2042556c658061e699acbc0aa2a1439f52f3d461dffeb",
45
+ bytes: 16_919_407,
46
+ },
47
+ ];
48
+ /** Per-file download deadline; generous because the model is 17 MB. */
49
+ const DOWNLOAD_TIMEOUT_MS = 10 * 60_000;
50
+ /** One in-flight preparation per directory, shared by concurrent callers. */
51
+ const inFlight = new Map();
52
+ /** Directories already verified in this process. */
53
+ const verified = new Map();
54
+ /**
55
+ * Where Whistle keeps its files: `slice.modelDir` → `NEUROLINK_WHISTLE_DIR` →
56
+ * `NEUROLINK_MODEL_DIR/whistle` → the copy shipped with the package
57
+ * (`models/whistle`, when all three files are there) →
58
+ * `~/.neurolink/models/whistle`, the only one a download ever writes to.
59
+ */
60
+ export function resolveWhistleDir(slice) {
61
+ const explicit = slice?.modelDir?.trim() || process.env.NEUROLINK_WHISTLE_DIR?.trim();
62
+ if (explicit) {
63
+ return resolve(explicit);
64
+ }
65
+ const modelRoot = process.env.NEUROLINK_MODEL_DIR?.trim();
66
+ if (modelRoot) {
67
+ return resolve(modelRoot, "whistle");
68
+ }
69
+ const bundled = bundledWhistleDir();
70
+ if (bundled && whistleAssetsPresent(bundled)) {
71
+ return bundled;
72
+ }
73
+ return join(homedir(), ".neurolink", "models", "whistle");
74
+ }
75
+ let bundledDir;
76
+ /**
77
+ * The package's own `models/whistle`, found by walking up from this module
78
+ * to the directory holding NeuroLink's `package.json`. Works from the source
79
+ * tree, from `dist/` and from the single-file bundles, since all of them sit
80
+ * below the package root; `null` when nothing above looks like the package.
81
+ */
82
+ export function bundledWhistleDir() {
83
+ if (bundledDir !== undefined) {
84
+ return bundledDir;
85
+ }
86
+ bundledDir = null;
87
+ let dir;
88
+ try {
89
+ dir = dirname(fileURLToPath(import.meta.url));
90
+ }
91
+ catch {
92
+ return bundledDir;
93
+ }
94
+ for (let depth = 0; depth < 8; depth += 1) {
95
+ const manifest = join(dir, "package.json");
96
+ if (existsSync(manifest)) {
97
+ try {
98
+ const parsed = JSON.parse(readFileSync(manifest, "utf8"));
99
+ if (parsed.name === "@juspay/neurolink") {
100
+ bundledDir = join(dir, "models", "whistle");
101
+ return bundledDir;
102
+ }
103
+ }
104
+ catch {
105
+ // Not a manifest we can read; keep walking up.
106
+ }
107
+ }
108
+ const parent = dirname(dir);
109
+ if (parent === dir) {
110
+ break;
111
+ }
112
+ dir = parent;
113
+ }
114
+ return bundledDir;
115
+ }
116
+ /**
117
+ * Whether missing files may be fetched: on unless `slice.autoDownload` is
118
+ * `false` or `NEUROLINK_WHISTLE_AUTO_DOWNLOAD` is `0` / `false` / `off` / `no`.
119
+ */
120
+ export function whistleAutoDownloadEnabled(slice) {
121
+ if (slice?.autoDownload !== undefined) {
122
+ return slice.autoDownload;
123
+ }
124
+ const env = process.env.NEUROLINK_WHISTLE_AUTO_DOWNLOAD?.trim().toLowerCase();
125
+ return !(env === "0" || env === "false" || env === "off" || env === "no");
126
+ }
127
+ /** Cheap presence check (no hashing): all three files exist in `dir`. */
128
+ export function whistleAssetsPresent(dir) {
129
+ return WHISTLE_ASSETS.every((asset) => existsSync(join(dir, asset.name)));
130
+ }
131
+ /**
132
+ * Make sure all three files are in `dir` and match their pinned hashes,
133
+ * downloading what is missing when allowed. Concurrent calls for the same
134
+ * directory share one preparation.
135
+ */
136
+ export function ensureWhistleAssets(dir, autoDownload) {
137
+ const done = verified.get(dir);
138
+ if (done) {
139
+ return Promise.resolve(done);
140
+ }
141
+ const pending = inFlight.get(dir);
142
+ if (pending) {
143
+ return pending;
144
+ }
145
+ const job = prepare(dir, autoDownload)
146
+ .then((paths) => {
147
+ verified.set(dir, paths);
148
+ return paths;
149
+ })
150
+ .finally(() => {
151
+ inFlight.delete(dir);
152
+ });
153
+ inFlight.set(dir, job);
154
+ return job;
155
+ }
156
+ async function prepare(dir, autoDownload) {
157
+ const missing = [];
158
+ const corrupt = [];
159
+ for (const asset of WHISTLE_ASSETS) {
160
+ const state = await checkFile(join(dir, asset.name), asset);
161
+ if (state === "missing") {
162
+ missing.push(asset);
163
+ }
164
+ else if (state === "corrupt") {
165
+ corrupt.push(asset);
166
+ }
167
+ }
168
+ const needed = [...missing, ...corrupt];
169
+ if (needed.length > 0) {
170
+ if (!autoDownload) {
171
+ throw assetsUnavailable(dir, corrupt.length > 0
172
+ ? `${corrupt.map((a) => a.name).join(", ")} in ${dir} does not match its pinned SHA-256, and automatic download is off`
173
+ : "automatic download is off (NEUROLINK_WHISTLE_AUTO_DOWNLOAD=0 or autoDownload: false)");
174
+ }
175
+ const total = needed.reduce((sum, a) => sum + a.bytes, 0);
176
+ logger.info(`[whistle] downloading Whistle (${Math.max(1, Math.round(total / 1_000_000))} MB) to ${dir} — first use only`);
177
+ try {
178
+ await mkdir(dir, { recursive: true });
179
+ for (const asset of needed) {
180
+ await download(asset, dir);
181
+ }
182
+ }
183
+ catch (error) {
184
+ if (error instanceof STTError) {
185
+ throw error;
186
+ }
187
+ throw assetsUnavailable(dir, `the download failed (${error instanceof Error ? error.message : String(error)})`, error instanceof Error ? error : undefined);
188
+ }
189
+ }
190
+ return {
191
+ "needle.js": join(dir, "needle.js"),
192
+ "needle.wasm": join(dir, "needle.wasm"),
193
+ "whistle.cact": join(dir, "whistle.cact"),
194
+ };
195
+ }
196
+ async function checkFile(path, asset) {
197
+ try {
198
+ const info = await stat(path);
199
+ if (!info.isFile()) {
200
+ return "missing";
201
+ }
202
+ if (info.size !== asset.bytes) {
203
+ return "corrupt";
204
+ }
205
+ }
206
+ catch {
207
+ return "missing";
208
+ }
209
+ return (await hashFile(path)) === asset.sha256 ? "ok" : "corrupt";
210
+ }
211
+ function hashFile(path) {
212
+ return new Promise((resolvePromise, reject) => {
213
+ const hash = createHash("sha256");
214
+ createReadStream(path)
215
+ .on("data", (chunk) => hash.update(chunk))
216
+ .on("error", reject)
217
+ .on("end", () => resolvePromise(hash.digest("hex")));
218
+ });
219
+ }
220
+ async function download(asset, dir) {
221
+ const response = await fetch(asset.url, {
222
+ signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS),
223
+ redirect: "follow",
224
+ });
225
+ if (!response.ok) {
226
+ throw new Error(`${asset.name}: HTTP ${response.status}`);
227
+ }
228
+ const body = Buffer.from(await response.arrayBuffer());
229
+ const digest = createHash("sha256").update(body).digest("hex");
230
+ if (digest !== asset.sha256) {
231
+ throw new Error(`${asset.name}: SHA-256 mismatch (expected ${asset.sha256}, got ${digest})`);
232
+ }
233
+ const target = join(dir, asset.name);
234
+ const temp = join(dir, `.${asset.name}.${process.pid}.${randomBytes(4).toString("hex")}.part`);
235
+ try {
236
+ await writeFile(temp, body);
237
+ await rename(temp, target);
238
+ }
239
+ catch (error) {
240
+ await rm(temp, { force: true }).catch(() => undefined);
241
+ throw error;
242
+ }
243
+ logger.debug(`[whistle] ${asset.name} verified and saved to ${target}`);
244
+ }
245
+ /** The friendly "here is exactly what to put where" error. */
246
+ /** The three files, where they go and where they come from — the body of every "unusable" message. */
247
+ export function describeWhistleAssets(dir, reason) {
248
+ const width = Math.max(...WHISTLE_ASSETS.map((a) => a.name.length));
249
+ const lines = WHISTLE_ASSETS.map((a) => ` ${join(dir, a.name).padEnd(dir.length + 1 + width)} ← ${a.url}`);
250
+ return (`Whistle (the built-in local speech-to-text engine) needs three files, and ${reason}.\n` +
251
+ `Place them here:\n${lines.join("\n")}\n` +
252
+ "Or point NEUROLINK_WHISTLE_DIR (or credentials.stt.whistle.modelDir) at a directory that has them " +
253
+ "(a full install of @juspay/neurolink ships a copy in its models/whistle directory), " +
254
+ "or configure a cloud STT provider.");
255
+ }
256
+ function assetsUnavailable(dir, reason, originalError) {
257
+ return new STTError({
258
+ code: STT_ERROR_CODES.PROVIDER_NOT_CONFIGURED,
259
+ message: describeWhistleAssets(dir, reason),
260
+ category: ErrorCategory.CONFIGURATION,
261
+ severity: ErrorSeverity.HIGH,
262
+ retriable: false,
263
+ context: {
264
+ provider: "whistle",
265
+ modelDir: dir,
266
+ files: WHISTLE_ASSETS.map((a) => ({ name: a.name, url: a.url })),
267
+ },
268
+ originalError,
269
+ });
270
+ }
@@ -0,0 +1,32 @@
1
+ /**
2
+ * Audio decoding for the Whistle engine: anything in, 16 kHz mono Float32 out.
3
+ *
4
+ * WAV (PCM 8/16/24/32-bit integer or 32/64-bit float, any sample rate and
5
+ * channel count, including WAVE_FORMAT_EXTENSIBLE) and raw `pcm16` (PCM16LE
6
+ * mono at `options.sampleRate`, default 16 kHz) are decoded here with no
7
+ * dependencies. Compressed formats (mp3, m4a, ogg/opus, webm, flac, mp4 …) go
8
+ * through ffmpeg — `FFMPEG_PATH`, then the optional `ffmpeg-static` package,
9
+ * then `ffmpeg` on `PATH` — and are refused with a clear error when none of
10
+ * those exists.
11
+ *
12
+ * @module voice/whistle/audio
13
+ */
14
+ import type { TTSAudioFormat } from "../../types/index.js";
15
+ /** Formats Whistle decodes without any external tool. */
16
+ export declare const WHISTLE_NATIVE_FORMATS: readonly TTSAudioFormat[];
17
+ /** Formats Whistle decodes through ffmpeg when it is available. */
18
+ export declare const WHISTLE_FFMPEG_FORMATS: readonly TTSAudioFormat[];
19
+ /** The engine's fixed input rate. */
20
+ export declare const WHISTLE_SAMPLE_RATE = 16000;
21
+ /**
22
+ * Decode `audio` to 16 kHz mono Float32 samples in [-1, 1]. `format` is a
23
+ * hint; a RIFF/WAVE header is recognised whatever it says.
24
+ */
25
+ export declare function decodeToWhistlePcm(audio: Buffer, format: TTSAudioFormat | undefined, sampleRate: number | undefined): Promise<Float32Array>;
26
+ /** PCM16LE frames (as streamed by microphones and the chunked adapter) → 16 kHz Float32. */
27
+ export declare function decodePcm16(buf: Buffer, sampleRate: number): Float32Array;
28
+ /**
29
+ * Resample to 16 kHz: a box filter over each output sample's footprint when
30
+ * downsampling (a cheap anti-alias), linear interpolation when upsampling.
31
+ */
32
+ export declare function resampleTo16k(input: Float32Array, rate: number): Float32Array;
@@ -0,0 +1,267 @@
1
+ /**
2
+ * Audio decoding for the Whistle engine: anything in, 16 kHz mono Float32 out.
3
+ *
4
+ * WAV (PCM 8/16/24/32-bit integer or 32/64-bit float, any sample rate and
5
+ * channel count, including WAVE_FORMAT_EXTENSIBLE) and raw `pcm16` (PCM16LE
6
+ * mono at `options.sampleRate`, default 16 kHz) are decoded here with no
7
+ * dependencies. Compressed formats (mp3, m4a, ogg/opus, webm, flac, mp4 …) go
8
+ * through ffmpeg — `FFMPEG_PATH`, then the optional `ffmpeg-static` package,
9
+ * then `ffmpeg` on `PATH` — and are refused with a clear error when none of
10
+ * those exists.
11
+ *
12
+ * @module voice/whistle/audio
13
+ */
14
+ import { spawn } from "node:child_process";
15
+ import { randomUUID } from "node:crypto";
16
+ import { rm, writeFile } from "node:fs/promises";
17
+ import { tmpdir } from "node:os";
18
+ import { join } from "node:path";
19
+ import { STT_ERROR_CODES } from "../../types/index.js";
20
+ import { tryImport } from "../../utils/tryImport.js";
21
+ import { STTError } from "../errors.js";
22
+ /** Formats Whistle decodes without any external tool. */
23
+ export const WHISTLE_NATIVE_FORMATS = [
24
+ "wav",
25
+ "pcm16",
26
+ ];
27
+ /** Formats Whistle decodes through ffmpeg when it is available. */
28
+ export const WHISTLE_FFMPEG_FORMATS = [
29
+ "mp3",
30
+ "m4a",
31
+ "mp4",
32
+ "ogg",
33
+ "opus",
34
+ "webm",
35
+ "flac",
36
+ "mpeg",
37
+ "mpga",
38
+ ];
39
+ /** The engine's fixed input rate. */
40
+ export const WHISTLE_SAMPLE_RATE = 16_000;
41
+ const FFMPEG_TIMEOUT_MS = 5 * 60_000;
42
+ /**
43
+ * Decode `audio` to 16 kHz mono Float32 samples in [-1, 1]. `format` is a
44
+ * hint; a RIFF/WAVE header is recognised whatever it says.
45
+ */
46
+ export async function decodeToWhistlePcm(audio, format, sampleRate) {
47
+ if (isWav(audio)) {
48
+ return decodeWav(audio);
49
+ }
50
+ if (format === "pcm16") {
51
+ return decodePcm16(audio, sampleRate ?? WHISTLE_SAMPLE_RATE);
52
+ }
53
+ if (format === "wav") {
54
+ throw STTError.invalidFormat("wav (no RIFF/WAVE header found)", [...WHISTLE_NATIVE_FORMATS, ...WHISTLE_FFMPEG_FORMATS], "whistle");
55
+ }
56
+ return decodeWithFfmpeg(audio, format);
57
+ }
58
+ function isWav(buf) {
59
+ return (buf.length >= 12 &&
60
+ buf.toString("ascii", 0, 4) === "RIFF" &&
61
+ buf.toString("ascii", 8, 12) === "WAVE");
62
+ }
63
+ /** PCM16LE frames (as streamed by microphones and the chunked adapter) → 16 kHz Float32. */
64
+ export function decodePcm16(buf, sampleRate) {
65
+ const n = buf.length >> 1;
66
+ const out = new Float32Array(n);
67
+ for (let i = 0; i < n; i++) {
68
+ out[i] = buf.readInt16LE(i * 2) / 32768;
69
+ }
70
+ return resampleTo16k(out, sampleRate);
71
+ }
72
+ function decodeWav(buf) {
73
+ let offset = 12;
74
+ let fmt;
75
+ let data;
76
+ while (offset + 8 <= buf.length) {
77
+ const id = buf.toString("ascii", offset, offset + 4);
78
+ const size = buf.readUInt32LE(offset + 4);
79
+ const body = offset + 8;
80
+ if (id === "fmt " && body + 16 <= buf.length) {
81
+ let format = buf.readUInt16LE(body);
82
+ // WAVE_FORMAT_EXTENSIBLE: the real format is the first two bytes of the sub-format GUID.
83
+ if (format === 0xfffe && size >= 26 && body + 26 <= buf.length) {
84
+ format = buf.readUInt16LE(body + 24);
85
+ }
86
+ fmt = {
87
+ format,
88
+ channels: buf.readUInt16LE(body + 2),
89
+ rate: buf.readUInt32LE(body + 4),
90
+ bits: buf.readUInt16LE(body + 14),
91
+ };
92
+ }
93
+ else if (id === "data") {
94
+ // Streamed WAVs leave the size at 0 or 0xFFFFFFFF; read to the end then.
95
+ const available = buf.length - body;
96
+ data = {
97
+ start: body,
98
+ size: size === 0 || size > available ? available : size,
99
+ };
100
+ break;
101
+ }
102
+ offset = body + size + (size & 1);
103
+ }
104
+ if (!fmt || !data) {
105
+ throw STTError.invalidFormat("wav (missing fmt or data chunk)", "whistle");
106
+ }
107
+ const { format, channels, rate, bits } = fmt;
108
+ const isFloat = format === 3;
109
+ if (channels < 1 ||
110
+ rate < 1 ||
111
+ !((format === 1 && [8, 16, 24, 32].includes(bits)) ||
112
+ (isFloat && (bits === 32 || bits === 64)))) {
113
+ throw STTError.invalidFormat(`wav (format ${format}, ${bits}-bit) — Whistle reads PCM 8/16/24/32-bit and 32/64-bit float WAV`, "whistle");
114
+ }
115
+ const bytes = bits / 8;
116
+ const frameBytes = bytes * channels;
117
+ const frames = Math.floor(data.size / frameBytes);
118
+ const mono = new Float32Array(frames);
119
+ const read = (pos) => {
120
+ if (isFloat) {
121
+ return bits === 32 ? buf.readFloatLE(pos) : buf.readDoubleLE(pos);
122
+ }
123
+ switch (bits) {
124
+ case 8:
125
+ return (buf[pos] - 128) / 128;
126
+ case 16:
127
+ return buf.readInt16LE(pos) / 32768;
128
+ case 24:
129
+ return buf.readIntLE(pos, 3) / 8388608;
130
+ default:
131
+ return buf.readInt32LE(pos) / 2147483648;
132
+ }
133
+ };
134
+ for (let f = 0; f < frames; f++) {
135
+ const base = data.start + f * frameBytes;
136
+ let sum = 0;
137
+ for (let c = 0; c < channels; c++) {
138
+ sum += read(base + c * bytes);
139
+ }
140
+ mono[f] = sum / channels;
141
+ }
142
+ return resampleTo16k(mono, rate);
143
+ }
144
+ /**
145
+ * Resample to 16 kHz: a box filter over each output sample's footprint when
146
+ * downsampling (a cheap anti-alias), linear interpolation when upsampling.
147
+ */
148
+ export function resampleTo16k(input, rate) {
149
+ if (rate === WHISTLE_SAMPLE_RATE || input.length === 0) {
150
+ return input;
151
+ }
152
+ const ratio = rate / WHISTLE_SAMPLE_RATE;
153
+ const outLength = Math.floor(input.length / ratio);
154
+ const out = new Float32Array(outLength);
155
+ if (ratio > 1) {
156
+ for (let i = 0; i < outLength; i++) {
157
+ const from = Math.floor(i * ratio);
158
+ const to = Math.min(input.length, Math.max(from + 1, Math.floor((i + 1) * ratio)));
159
+ let sum = 0;
160
+ for (let j = from; j < to; j++) {
161
+ sum += input[j];
162
+ }
163
+ out[i] = sum / (to - from);
164
+ }
165
+ }
166
+ else {
167
+ for (let i = 0; i < outLength; i++) {
168
+ const pos = i * ratio;
169
+ const j = Math.floor(pos);
170
+ const frac = pos - j;
171
+ const a = input[j];
172
+ const b = j + 1 < input.length ? input[j + 1] : a;
173
+ out[i] = a + (b - a) * frac;
174
+ }
175
+ }
176
+ return out;
177
+ }
178
+ async function resolveFfmpeg() {
179
+ const fromEnv = process.env.FFMPEG_PATH?.trim();
180
+ if (fromEnv) {
181
+ return fromEnv;
182
+ }
183
+ try {
184
+ const mod = await tryImport("ffmpeg-static", "Whistle compressed-audio decoding");
185
+ if (typeof mod.default === "string" && mod.default.length > 0) {
186
+ return mod.default;
187
+ }
188
+ }
189
+ catch {
190
+ // Optional: fall back to a system ffmpeg.
191
+ }
192
+ return "ffmpeg";
193
+ }
194
+ function ffmpegUnavailable(format, cause) {
195
+ return new STTError({
196
+ code: STT_ERROR_CODES.INVALID_AUDIO_FORMAT,
197
+ message: `Whistle cannot decode ${format ?? "this"} audio without ffmpeg. It reads WAV ` +
198
+ "(PCM or float, any rate and channel count) and raw pcm16 by itself; for " +
199
+ `${WHISTLE_FFMPEG_FORMATS.join(", ")} install ffmpeg (pnpm add ffmpeg-static, ` +
200
+ "or put ffmpeg on PATH, or set FFMPEG_PATH), or convert the audio to WAV first.",
201
+ context: {
202
+ provider: "whistle",
203
+ format,
204
+ supportedFormats: [...WHISTLE_NATIVE_FORMATS, ...WHISTLE_FFMPEG_FORMATS],
205
+ },
206
+ originalError: cause,
207
+ });
208
+ }
209
+ async function decodeWithFfmpeg(audio, format) {
210
+ const binary = await resolveFfmpeg();
211
+ // A file, not a pipe: MP4/M4A often keep their index at the end, which a pipe cannot seek to.
212
+ const input = join(tmpdir(), `neurolink-whistle-${randomUUID()}.${format ?? "bin"}`);
213
+ await writeFile(input, audio);
214
+ try {
215
+ const raw = await new Promise((resolvePromise, reject) => {
216
+ const proc = spawn(binary, [
217
+ "-nostdin",
218
+ "-hide_banner",
219
+ "-loglevel",
220
+ "error",
221
+ "-i",
222
+ input,
223
+ "-f",
224
+ "f32le",
225
+ "-ac",
226
+ "1",
227
+ "-ar",
228
+ String(WHISTLE_SAMPLE_RATE),
229
+ "pipe:1",
230
+ ], { stdio: ["ignore", "pipe", "pipe"] });
231
+ const out = [];
232
+ let err = "";
233
+ const timer = setTimeout(() => proc.kill("SIGKILL"), FFMPEG_TIMEOUT_MS);
234
+ proc.stdout.on("data", (c) => out.push(c));
235
+ proc.stderr.on("data", (c) => {
236
+ err = (err + c.toString("utf8")).slice(-2000);
237
+ });
238
+ proc.on("error", (e) => {
239
+ clearTimeout(timer);
240
+ reject(e);
241
+ });
242
+ proc.on("close", (code) => {
243
+ clearTimeout(timer);
244
+ if (code === 0) {
245
+ resolvePromise(Buffer.concat(out));
246
+ }
247
+ else {
248
+ reject(new Error(`ffmpeg exited with code ${code}: ${err.trim()}`));
249
+ }
250
+ });
251
+ });
252
+ const samples = new Float32Array(raw.length >> 2);
253
+ for (let i = 0; i < samples.length; i++) {
254
+ samples[i] = raw.readFloatLE(i * 4);
255
+ }
256
+ return samples;
257
+ }
258
+ catch (error) {
259
+ if (error.code === "ENOENT") {
260
+ throw ffmpegUnavailable(format, error instanceof Error ? error : undefined);
261
+ }
262
+ throw STTError.invalidFormat(`${format ?? "unknown"} (ffmpeg could not decode it: ${error instanceof Error ? error.message : String(error)})`, [...WHISTLE_NATIVE_FORMATS, ...WHISTLE_FFMPEG_FORMATS], "whistle");
263
+ }
264
+ finally {
265
+ await rm(input, { force: true }).catch(() => undefined);
266
+ }
267
+ }
@@ -0,0 +1,32 @@
1
+ /**
2
+ * Main-thread side of the Whistle engine: one warm `worker_threads` Worker
3
+ * that holds the WASM engine and the model, requests answered in order, and a
4
+ * fresh worker after a crash.
5
+ *
6
+ * The worker is unreferenced while idle, so a warm engine never keeps the
7
+ * process alive on its own; it is referenced again while a request is open.
8
+ *
9
+ * @module voice/whistle/engine
10
+ */
11
+ import type { WhistleAssetPaths, WhistleEngineOutput, WhistleStreamStep } from "../../types/index.js";
12
+ export declare class WhistleEngine {
13
+ private readonly paths;
14
+ private worker;
15
+ private ready;
16
+ private nextId;
17
+ private readonly pending;
18
+ constructor(paths: WhistleAssetPaths);
19
+ /** Batch transcription of 16 kHz mono samples; long audio is chunked in the worker. */
20
+ transcribe(pcm: Float32Array, language: string | null, keywords: string | null, timestamps: boolean): Promise<WhistleEngineOutput>;
21
+ /** Feed about a second of samples to the engine's native stream. */
22
+ streamProcess(pcm: Float32Array, language: string | null, keywords: string | null): Promise<WhistleStreamStep>;
23
+ /** Flush the stream's tail and reset its state. */
24
+ streamStop(): Promise<WhistleStreamStep>;
25
+ /** Terminate the worker; the next call starts a new one. */
26
+ dispose(): Promise<void>;
27
+ private request;
28
+ private start;
29
+ /** Drop a dead worker: reject what it held, and let the next call start a new one. */
30
+ private crash;
31
+ private failAll;
32
+ }