@moxxy/plugin-stt-local 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/audio.d.ts +55 -0
- package/dist/audio.d.ts.map +1 -0
- package/dist/audio.js +197 -0
- package/dist/audio.js.map +1 -0
- package/dist/decode.d.ts +25 -0
- package/dist/decode.d.ts.map +1 -0
- package/dist/decode.js +50 -0
- package/dist/decode.js.map +1 -0
- package/dist/ffmpeg.d.ts +26 -0
- package/dist/ffmpeg.d.ts.map +1 -0
- package/dist/ffmpeg.js +207 -0
- package/dist/ffmpeg.js.map +1 -0
- package/dist/host-client.d.ts +70 -0
- package/dist/host-client.d.ts.map +1 -0
- package/dist/host-client.js +156 -0
- package/dist/host-client.js.map +1 -0
- package/dist/host-protocol.d.ts +96 -0
- package/dist/host-protocol.d.ts.map +1 -0
- package/dist/host-protocol.js +70 -0
- package/dist/host-protocol.js.map +1 -0
- package/dist/index.d.ts +32 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +61 -0
- package/dist/index.js.map +1 -0
- package/dist/local-stt.d.ts +86 -0
- package/dist/local-stt.d.ts.map +1 -0
- package/dist/local-stt.js +201 -0
- package/dist/local-stt.js.map +1 -0
- package/dist/models.d.ts +47 -0
- package/dist/models.d.ts.map +1 -0
- package/dist/models.js +61 -0
- package/dist/models.js.map +1 -0
- package/dist/platform.d.ts +37 -0
- package/dist/platform.d.ts.map +1 -0
- package/dist/platform.js +96 -0
- package/dist/platform.js.map +1 -0
- package/dist/sidecar.d.ts +18 -0
- package/dist/sidecar.d.ts.map +1 -0
- package/dist/sidecar.js +86 -0
- package/dist/sidecar.js.map +1 -0
- package/package.json +67 -0
- package/src/audio.test.ts +213 -0
- package/src/audio.ts +220 -0
- package/src/decode.test.ts +126 -0
- package/src/decode.ts +74 -0
- package/src/ffmpeg.test.ts +142 -0
- package/src/ffmpeg.ts +215 -0
- package/src/host-client.test.ts +208 -0
- package/src/host-client.ts +200 -0
- package/src/host-protocol.test.ts +171 -0
- package/src/host-protocol.ts +152 -0
- package/src/index.ts +108 -0
- package/src/live.test.ts +80 -0
- package/src/local-stt.test.ts +224 -0
- package/src/local-stt.ts +273 -0
- package/src/models.test.ts +58 -0
- package/src/models.ts +111 -0
- package/src/platform.test.ts +77 -0
- package/src/platform.ts +109 -0
- package/src/sidecar.ts +97 -0
package/package.json
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@moxxy/plugin-stt-local",
|
|
3
|
+
"version": "0.28.0",
|
|
4
|
+
"description": "Fully local, on-device speech-to-text Transcriber for moxxy. Runs multilingual Whisper (English + Polish) via sherpa-onnx in a forked sidecar process; models download once on first use from sherpa-onnx's pinned releases (sha256-verified). No API key, no network at transcription time. Plugs into the session TranscriberRegistry — the TUI and channel voice notes consume it transparently.",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"moxxy",
|
|
7
|
+
"agent",
|
|
8
|
+
"stt",
|
|
9
|
+
"speech-to-text",
|
|
10
|
+
"whisper",
|
|
11
|
+
"offline",
|
|
12
|
+
"on-device",
|
|
13
|
+
"sherpa-onnx",
|
|
14
|
+
"transcriber"
|
|
15
|
+
],
|
|
16
|
+
"homepage": "https://moxxy.ai",
|
|
17
|
+
"bugs": {
|
|
18
|
+
"url": "https://github.com/moxxy-ai/moxxy/issues"
|
|
19
|
+
},
|
|
20
|
+
"repository": {
|
|
21
|
+
"type": "git",
|
|
22
|
+
"url": "git+https://github.com/moxxy-ai/moxxy.git",
|
|
23
|
+
"directory": "packages/plugin-stt-local"
|
|
24
|
+
},
|
|
25
|
+
"author": "Michal Makowski <michal.makowski97@gmail.com>",
|
|
26
|
+
"license": "MIT",
|
|
27
|
+
"publishConfig": {
|
|
28
|
+
"access": "public"
|
|
29
|
+
},
|
|
30
|
+
"type": "module",
|
|
31
|
+
"main": "./dist/index.js",
|
|
32
|
+
"types": "./dist/index.d.ts",
|
|
33
|
+
"exports": {
|
|
34
|
+
".": {
|
|
35
|
+
"types": "./dist/index.d.ts",
|
|
36
|
+
"import": "./dist/index.js"
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
"files": [
|
|
40
|
+
"dist",
|
|
41
|
+
"src"
|
|
42
|
+
],
|
|
43
|
+
"moxxy": {
|
|
44
|
+
"plugin": {
|
|
45
|
+
"entry": "./dist/index.js",
|
|
46
|
+
"kind": "transcriber"
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
"dependencies": {
|
|
50
|
+
"sherpa-onnx-node": "^1.13.3",
|
|
51
|
+
"@moxxy/model-fetch": "0.1.0",
|
|
52
|
+
"@moxxy/sdk": "0.28.0"
|
|
53
|
+
},
|
|
54
|
+
"devDependencies": {
|
|
55
|
+
"@types/node": "^22.10.0",
|
|
56
|
+
"typescript": "^5.7.3",
|
|
57
|
+
"vitest": "^2.1.8",
|
|
58
|
+
"@moxxy/tsconfig": "0.0.0",
|
|
59
|
+
"@moxxy/vitest-preset": "0.0.0"
|
|
60
|
+
},
|
|
61
|
+
"scripts": {
|
|
62
|
+
"build": "tsc -p tsconfig.json",
|
|
63
|
+
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
64
|
+
"test": "vitest run",
|
|
65
|
+
"clean": "rm -rf dist .turbo"
|
|
66
|
+
}
|
|
67
|
+
}
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
downmixToMono,
|
|
5
|
+
isRiffWave,
|
|
6
|
+
parseWav,
|
|
7
|
+
pcm16ToFloat32,
|
|
8
|
+
resampleLinear,
|
|
9
|
+
} from './audio.js';
|
|
10
|
+
|
|
11
|
+
/** Build a canonical PCM16 WAV buffer from interleaved int16 samples. */
|
|
12
|
+
function buildWav(
|
|
13
|
+
int16: number[],
|
|
14
|
+
opts: { channels?: number; sampleRate?: number; audioFormat?: number; bitsPerSample?: number } = {},
|
|
15
|
+
): Uint8Array {
|
|
16
|
+
const channels = opts.channels ?? 1;
|
|
17
|
+
const sampleRate = opts.sampleRate ?? 16_000;
|
|
18
|
+
const audioFormat = opts.audioFormat ?? 1;
|
|
19
|
+
const bitsPerSample = opts.bitsPerSample ?? 16;
|
|
20
|
+
const bytesPerSample = bitsPerSample / 8;
|
|
21
|
+
const dataSize = int16.length * bytesPerSample;
|
|
22
|
+
const buf = new Uint8Array(44 + dataSize);
|
|
23
|
+
const view = new DataView(buf.buffer);
|
|
24
|
+
const ascii = (o: number, s: string): void => {
|
|
25
|
+
for (let i = 0; i < s.length; i += 1) buf[o + i] = s.charCodeAt(i);
|
|
26
|
+
};
|
|
27
|
+
ascii(0, 'RIFF');
|
|
28
|
+
view.setUint32(4, 36 + dataSize, true);
|
|
29
|
+
ascii(8, 'WAVE');
|
|
30
|
+
ascii(12, 'fmt ');
|
|
31
|
+
view.setUint32(16, 16, true);
|
|
32
|
+
view.setUint16(20, audioFormat, true);
|
|
33
|
+
view.setUint16(22, channels, true);
|
|
34
|
+
view.setUint32(24, sampleRate, true);
|
|
35
|
+
view.setUint32(28, sampleRate * channels * bytesPerSample, true);
|
|
36
|
+
view.setUint16(32, channels * bytesPerSample, true);
|
|
37
|
+
view.setUint16(34, bitsPerSample, true);
|
|
38
|
+
ascii(36, 'data');
|
|
39
|
+
view.setUint32(40, dataSize, true);
|
|
40
|
+
let off = 44;
|
|
41
|
+
for (const s of int16) {
|
|
42
|
+
if (bitsPerSample === 16) view.setInt16(off, s, true);
|
|
43
|
+
else view.setUint8(off, s & 0xff);
|
|
44
|
+
off += bytesPerSample;
|
|
45
|
+
}
|
|
46
|
+
return buf;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
describe('pcm16ToFloat32', () => {
|
|
50
|
+
it('maps int16 range edges to ~[-1, 1) and 0 to 0', () => {
|
|
51
|
+
const bytes = new Uint8Array(6);
|
|
52
|
+
const v = new DataView(bytes.buffer);
|
|
53
|
+
v.setInt16(0, 0, true);
|
|
54
|
+
v.setInt16(2, 32767, true);
|
|
55
|
+
v.setInt16(4, -32768, true);
|
|
56
|
+
const f = pcm16ToFloat32(bytes);
|
|
57
|
+
expect(f[0]).toBe(0);
|
|
58
|
+
expect(f[1]).toBeCloseTo(0.99997, 4);
|
|
59
|
+
expect(f[2]).toBe(-1); // -32768 / 32768
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it('drops a trailing odd byte instead of misreading it', () => {
|
|
63
|
+
const bytes = new Uint8Array([0, 0, 0, 0, 7]); // 2 samples + 1 stray byte
|
|
64
|
+
expect(pcm16ToFloat32(bytes)).toHaveLength(2);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it('works on a non-2-aligned subarray view', () => {
|
|
68
|
+
const backing = new Uint8Array(5);
|
|
69
|
+
const v = new DataView(backing.buffer);
|
|
70
|
+
v.setInt16(1, 16384, true); // write at offset 1 (odd)
|
|
71
|
+
const f = pcm16ToFloat32(backing.subarray(1, 3));
|
|
72
|
+
expect(f[0]).toBeCloseTo(0.5, 5);
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
describe('downmixToMono', () => {
|
|
77
|
+
it('averages stereo frames', () => {
|
|
78
|
+
// interleaved L,R,L,R → [(1+3)/2, (0.5-0.5)/2]
|
|
79
|
+
const stereo = Float32Array.from([1, 3, 0.5, -0.5]);
|
|
80
|
+
const mono = downmixToMono(stereo, 2);
|
|
81
|
+
expect(Array.from(mono)).toEqual([2, 0]);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it('returns the input unchanged for mono', () => {
|
|
85
|
+
const mono = Float32Array.from([0.1, 0.2]);
|
|
86
|
+
expect(downmixToMono(mono, 1)).toBe(mono);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it('ignores a trailing partial frame', () => {
|
|
90
|
+
const stereo = Float32Array.from([1, 1, 2]); // 1.5 frames
|
|
91
|
+
expect(downmixToMono(stereo, 2)).toHaveLength(1);
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
describe('resampleLinear', () => {
|
|
96
|
+
it('returns the input unchanged when rates match', () => {
|
|
97
|
+
const x = Float32Array.from([0.1, 0.2, 0.3]);
|
|
98
|
+
expect(resampleLinear(x, 16_000, 16_000)).toBe(x);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it('preserves a DC (constant) signal', () => {
|
|
102
|
+
const x = new Float32Array(240).fill(0.42);
|
|
103
|
+
const y = resampleLinear(x, 24_000, 16_000);
|
|
104
|
+
expect(y.length).toBe(160);
|
|
105
|
+
for (const s of y) expect(s).toBeCloseTo(0.42, 6);
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
it('changes length by the rate ratio (24k → 16k ≈ 2/3)', () => {
|
|
109
|
+
const y = resampleLinear(new Float32Array(300), 24_000, 16_000);
|
|
110
|
+
expect(y.length).toBe(200);
|
|
111
|
+
const up = resampleLinear(new Float32Array(160), 16_000, 24_000);
|
|
112
|
+
expect(up.length).toBe(240);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it('tracks a sine wave in continuous time after resampling 24k → 16k', () => {
|
|
116
|
+
const fromRate = 24_000;
|
|
117
|
+
const toRate = 16_000;
|
|
118
|
+
const freq = 220; // Hz — well below Nyquist at both rates
|
|
119
|
+
const n = 2400; // 0.1 s
|
|
120
|
+
const src = new Float32Array(n);
|
|
121
|
+
for (let i = 0; i < n; i += 1) src[i] = Math.sin((2 * Math.PI * freq * i) / fromRate);
|
|
122
|
+
const out = resampleLinear(src, fromRate, toRate);
|
|
123
|
+
// Each output sample should approximate the same continuous sine sampled at
|
|
124
|
+
// the target rate (linear interpolation error is tiny for 220 Hz).
|
|
125
|
+
for (let i = 0; i < out.length - 1; i += 1) {
|
|
126
|
+
const expected = Math.sin((2 * Math.PI * freq * i) / toRate);
|
|
127
|
+
expect(out[i]).toBeCloseTo(expected, 2);
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
describe('isRiffWave', () => {
|
|
133
|
+
it('detects RIFF/WAVE magic', () => {
|
|
134
|
+
expect(isRiffWave(buildWav([0, 0]))).toBe(true);
|
|
135
|
+
expect(isRiffWave(new Uint8Array([1, 2, 3, 4]))).toBe(false);
|
|
136
|
+
expect(isRiffWave(new Uint8Array(0))).toBe(false);
|
|
137
|
+
});
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
describe('parseWav', () => {
|
|
141
|
+
it('parses a mono PCM16 file', () => {
|
|
142
|
+
const wav = buildWav([0, 16384, -16384], { channels: 1, sampleRate: 16_000 });
|
|
143
|
+
const parsed = parseWav(wav);
|
|
144
|
+
expect(parsed.channels).toBe(1);
|
|
145
|
+
expect(parsed.sampleRate).toBe(16_000);
|
|
146
|
+
expect(parsed.samples[1]).toBeCloseTo(0.5, 4);
|
|
147
|
+
expect(parsed.samples[2]).toBeCloseTo(-0.5, 4);
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
it('parses a stereo PCM16 file (interleaved, still needs downmix)', () => {
|
|
151
|
+
const wav = buildWav([1000, 2000, 3000, 4000], { channels: 2, sampleRate: 8_000 });
|
|
152
|
+
const parsed = parseWav(wav);
|
|
153
|
+
expect(parsed.channels).toBe(2);
|
|
154
|
+
expect(parsed.samples).toHaveLength(4);
|
|
155
|
+
expect(Array.from(downmixToMono(parsed.samples, parsed.channels))).toHaveLength(2);
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
it('rejects IEEE-float WAV politely', () => {
|
|
159
|
+
const wav = buildWav([0, 0], { audioFormat: 3, bitsPerSample: 32 });
|
|
160
|
+
expect(() => parseWav(wav)).toThrow(/IEEE float/);
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
it('rejects 8-bit PCM WAV', () => {
|
|
164
|
+
const wav = buildWav([0, 0], { audioFormat: 1, bitsPerSample: 8 });
|
|
165
|
+
expect(() => parseWav(wav)).toThrow(/8-bit PCM/);
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it('throws CONFIG_INVALID for a non-RIFF buffer', () => {
|
|
169
|
+
expect(() => parseWav(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]))).toThrow(
|
|
170
|
+
/not a RIFF/,
|
|
171
|
+
);
|
|
172
|
+
try {
|
|
173
|
+
parseWav(new Uint8Array(12));
|
|
174
|
+
} catch (err) {
|
|
175
|
+
expect((err as { code?: string }).code).toBe('CONFIG_INVALID');
|
|
176
|
+
}
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
it('tolerates an extra chunk before data and a padded odd-size chunk', () => {
|
|
180
|
+
// Hand-build: RIFF WAVE | fmt(16) | LIST(3, padded to 4) | data(4)
|
|
181
|
+
const parts: number[] = [];
|
|
182
|
+
const push32 = (n: number): void => {
|
|
183
|
+
parts.push(n & 0xff, (n >> 8) & 0xff, (n >> 16) & 0xff, (n >> 24) & 0xff);
|
|
184
|
+
};
|
|
185
|
+
const pushStr = (s: string): void => {
|
|
186
|
+
for (const c of s) parts.push(c.charCodeAt(0));
|
|
187
|
+
};
|
|
188
|
+
// placeholder RIFF size filled later
|
|
189
|
+
pushStr('RIFF');
|
|
190
|
+
push32(0);
|
|
191
|
+
pushStr('WAVE');
|
|
192
|
+
pushStr('fmt ');
|
|
193
|
+
push32(16);
|
|
194
|
+
parts.push(1, 0); // PCM
|
|
195
|
+
parts.push(1, 0); // mono
|
|
196
|
+
push32(16_000);
|
|
197
|
+
push32(16_000 * 2);
|
|
198
|
+
parts.push(2, 0); // block align
|
|
199
|
+
parts.push(16, 0); // bits
|
|
200
|
+
pushStr('LIST');
|
|
201
|
+
push32(3); // odd size
|
|
202
|
+
parts.push(9, 9, 9, 0); // 3 bytes + 1 pad
|
|
203
|
+
pushStr('data');
|
|
204
|
+
push32(4);
|
|
205
|
+
parts.push(0, 0, 0, 64); // two int16 samples: 0, 16384
|
|
206
|
+
const bytes = new Uint8Array(parts);
|
|
207
|
+
new DataView(bytes.buffer).setUint32(4, bytes.length - 8, true);
|
|
208
|
+
const parsed = parseWav(bytes);
|
|
209
|
+
expect(parsed.sampleRate).toBe(16_000);
|
|
210
|
+
expect(parsed.samples).toHaveLength(2);
|
|
211
|
+
expect(parsed.samples[1]).toBeCloseTo(0.5, 4);
|
|
212
|
+
});
|
|
213
|
+
});
|
package/src/audio.ts
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure, dependency-free audio DSP for the local Whisper transcriber. sherpa's
|
|
3
|
+
* OfflineRecognizer wants Float32 mono PCM in [-1, 1] at 16 kHz; inbound audio
|
|
4
|
+
* arrives in several shapes, so these helpers convert each to that canonical
|
|
5
|
+
* form IN-PROCESS (no ffmpeg for raw PCM / WAV):
|
|
6
|
+
*
|
|
7
|
+
* - `pcm16ToFloat32` int16 little-endian bytes → Float32 in [-1, 1]
|
|
8
|
+
* - `downmixToMono` interleaved N-channel Float32 → averaged mono
|
|
9
|
+
* - `resampleLinear` linear-interpolation resample between two rates
|
|
10
|
+
* - `parseWav` RIFF/WAVE header parse → PCM16 samples (float) + rate +
|
|
11
|
+
* channels; rejects non-PCM16 (float/compressed) WAV
|
|
12
|
+
*
|
|
13
|
+
* The `decodeToMono16k` orchestrator (in ./decode.ts) drives these plus the
|
|
14
|
+
* ffmpeg path. Everything here is synchronous and unit-tested against synthetic
|
|
15
|
+
* fixtures — the header offsets and the resampler math are load-bearing.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { MoxxyError } from '@moxxy/sdk';
|
|
19
|
+
|
|
20
|
+
/** The sample rate sherpa's Whisper feature extractor expects. */
|
|
21
|
+
export const TARGET_SAMPLE_RATE = 16_000;
|
|
22
|
+
|
|
23
|
+
/** Full-scale divisor: int16 spans [-32768, 32767]; dividing by 32768 keeps
|
|
24
|
+
* the result in [-1, 1) without clipping the negative rail. */
|
|
25
|
+
const INT16_SCALE = 32_768;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Convert raw 16-bit little-endian PCM bytes to Float32 in [-1, 1]. A trailing
|
|
29
|
+
* odd byte (half a sample) is dropped. Channel interleaving is preserved — a
|
|
30
|
+
* stereo buffer stays interleaved, so callers downmix afterwards.
|
|
31
|
+
*/
|
|
32
|
+
export function pcm16ToFloat32(pcm: Uint8Array | ArrayBuffer): Float32Array {
|
|
33
|
+
const bytes = pcm instanceof Uint8Array ? pcm : new Uint8Array(pcm);
|
|
34
|
+
const usable = bytes.byteLength & ~1; // drop a trailing odd byte
|
|
35
|
+
const count = usable / 2;
|
|
36
|
+
const out = new Float32Array(count);
|
|
37
|
+
// Read via DataView so we don't depend on the platform being little-endian
|
|
38
|
+
// and don't require the byte offset to be 2-aligned (a subarray view can
|
|
39
|
+
// start on an odd offset).
|
|
40
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, usable);
|
|
41
|
+
for (let i = 0; i < count; i += 1) {
|
|
42
|
+
out[i] = view.getInt16(i * 2, true) / INT16_SCALE;
|
|
43
|
+
}
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Average `channels` interleaved Float32 channels down to mono. `channels` must
|
|
49
|
+
* be a positive integer; a value of 1 returns the input unchanged. Trailing
|
|
50
|
+
* samples that don't complete a frame are ignored.
|
|
51
|
+
*/
|
|
52
|
+
export function downmixToMono(interleaved: Float32Array, channels: number): Float32Array {
|
|
53
|
+
if (!Number.isInteger(channels) || channels < 1) {
|
|
54
|
+
throw new MoxxyError({
|
|
55
|
+
code: 'INTERNAL',
|
|
56
|
+
message: `downmixToMono: invalid channel count ${channels}.`,
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
if (channels === 1) return interleaved;
|
|
60
|
+
const frames = Math.floor(interleaved.length / channels);
|
|
61
|
+
const out = new Float32Array(frames);
|
|
62
|
+
for (let f = 0; f < frames; f += 1) {
|
|
63
|
+
let sum = 0;
|
|
64
|
+
const base = f * channels;
|
|
65
|
+
for (let c = 0; c < channels; c += 1) sum += interleaved[base + c]!;
|
|
66
|
+
out[f] = sum / channels;
|
|
67
|
+
}
|
|
68
|
+
return out;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Resample mono Float32 audio from `fromRate` to `toRate` by linear
|
|
73
|
+
* interpolation. Cheap and dependency-free — good enough for speech recognition
|
|
74
|
+
* (Whisper is robust to the mild low-pass a linear kernel imposes). Returns the
|
|
75
|
+
* input unchanged when the rates match, and an empty array for empty input.
|
|
76
|
+
*/
|
|
77
|
+
export function resampleLinear(
|
|
78
|
+
input: Float32Array,
|
|
79
|
+
fromRate: number,
|
|
80
|
+
toRate: number,
|
|
81
|
+
): Float32Array {
|
|
82
|
+
if (!Number.isFinite(fromRate) || !Number.isFinite(toRate) || fromRate <= 0 || toRate <= 0) {
|
|
83
|
+
throw new MoxxyError({
|
|
84
|
+
code: 'INTERNAL',
|
|
85
|
+
message: `resampleLinear: invalid rate(s) from=${fromRate} to=${toRate}.`,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
if (fromRate === toRate) return input;
|
|
89
|
+
if (input.length === 0) return new Float32Array(0);
|
|
90
|
+
if (input.length === 1) return Float32Array.of(input[0]!);
|
|
91
|
+
|
|
92
|
+
const ratio = fromRate / toRate;
|
|
93
|
+
const outLen = Math.max(1, Math.round(input.length / ratio));
|
|
94
|
+
const out = new Float32Array(outLen);
|
|
95
|
+
const lastIdx = input.length - 1;
|
|
96
|
+
for (let i = 0; i < outLen; i += 1) {
|
|
97
|
+
const srcPos = i * ratio;
|
|
98
|
+
const i0 = Math.floor(srcPos);
|
|
99
|
+
if (i0 >= lastIdx) {
|
|
100
|
+
out[i] = input[lastIdx]!;
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
const frac = srcPos - i0;
|
|
104
|
+
out[i] = input[i0]! * (1 - frac) + input[i0 + 1]! * frac;
|
|
105
|
+
}
|
|
106
|
+
return out;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** True when `bytes` begins with the `RIFF….WAVE` magic of a WAV container. */
|
|
110
|
+
export function isRiffWave(bytes: Uint8Array): boolean {
|
|
111
|
+
return (
|
|
112
|
+
bytes.length >= 12 &&
|
|
113
|
+
bytes[0] === 0x52 && // R
|
|
114
|
+
bytes[1] === 0x49 && // I
|
|
115
|
+
bytes[2] === 0x46 && // F
|
|
116
|
+
bytes[3] === 0x46 && // F
|
|
117
|
+
bytes[8] === 0x57 && // W
|
|
118
|
+
bytes[9] === 0x41 && // A
|
|
119
|
+
bytes[10] === 0x56 && // V
|
|
120
|
+
bytes[11] === 0x45 // E
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export interface ParsedWav {
|
|
125
|
+
/** PCM samples as Float32 in [-1, 1], still interleaved if multi-channel. */
|
|
126
|
+
readonly samples: Float32Array;
|
|
127
|
+
readonly sampleRate: number;
|
|
128
|
+
readonly channels: number;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// WAV `fmt ` audio-format codes we care about.
|
|
132
|
+
const WAVE_FORMAT_PCM = 1;
|
|
133
|
+
const WAVE_FORMAT_IEEE_FLOAT = 3;
|
|
134
|
+
const WAVE_FORMAT_EXTENSIBLE = 0xfffe;
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Parse a canonical RIFF/WAVE buffer into PCM16 samples. Only linear 16-bit PCM
|
|
138
|
+
* is accepted — IEEE-float, μ-law, and compressed WAV are rejected with a clear
|
|
139
|
+
* `CONFIG_INVALID` MoxxyError (the caller falls back to ffmpeg for real
|
|
140
|
+
* compressed formats, but a mislabelled/exotic WAV should fail loudly, not
|
|
141
|
+
* silently mis-decode). Walks the chunk list rather than assuming a fixed
|
|
142
|
+
* 44-byte header, so files with extra chunks (`LIST`, `fact`, …) still parse.
|
|
143
|
+
*/
|
|
144
|
+
export function parseWav(bytes: Uint8Array): ParsedWav {
|
|
145
|
+
if (!isRiffWave(bytes)) {
|
|
146
|
+
throw wavError('not a RIFF/WAVE file (bad magic).');
|
|
147
|
+
}
|
|
148
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
149
|
+
|
|
150
|
+
let fmt: { format: number; channels: number; sampleRate: number; bitsPerSample: number } | null =
|
|
151
|
+
null;
|
|
152
|
+
let dataOffset = -1;
|
|
153
|
+
let dataLen = 0;
|
|
154
|
+
|
|
155
|
+
// Chunks begin right after `RIFF<size>WAVE` (offset 12). Each chunk is a
|
|
156
|
+
// 4-byte id + uint32 little-endian size + payload, word-aligned (odd payloads
|
|
157
|
+
// carry a pad byte).
|
|
158
|
+
let off = 12;
|
|
159
|
+
while (off + 8 <= bytes.byteLength) {
|
|
160
|
+
const id = ascii4(bytes, off);
|
|
161
|
+
const size = view.getUint32(off + 4, true);
|
|
162
|
+
const body = off + 8;
|
|
163
|
+
if (id === 'fmt ') {
|
|
164
|
+
if (body + 16 > bytes.byteLength) throw wavError('truncated fmt chunk.');
|
|
165
|
+
fmt = {
|
|
166
|
+
format: view.getUint16(body, true),
|
|
167
|
+
channels: view.getUint16(body + 2, true),
|
|
168
|
+
sampleRate: view.getUint32(body + 4, true),
|
|
169
|
+
bitsPerSample: view.getUint16(body + 14, true),
|
|
170
|
+
};
|
|
171
|
+
} else if (id === 'data') {
|
|
172
|
+
dataOffset = body;
|
|
173
|
+
// Clamp a declared size that overruns the buffer (some encoders write 0
|
|
174
|
+
// or 0xffffffff for streamed data) to what's actually present.
|
|
175
|
+
dataLen = Math.min(size, bytes.byteLength - body);
|
|
176
|
+
}
|
|
177
|
+
// Advance past the payload + pad byte. Guard against a zero/garbage size
|
|
178
|
+
// that would loop forever.
|
|
179
|
+
const advance = 8 + size + (size & 1);
|
|
180
|
+
if (advance <= 8) break;
|
|
181
|
+
off += advance;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
if (!fmt) throw wavError('missing fmt chunk.');
|
|
185
|
+
if (dataOffset < 0) throw wavError('missing data chunk.');
|
|
186
|
+
|
|
187
|
+
const { format, channels, sampleRate, bitsPerSample } = fmt;
|
|
188
|
+
if (!Number.isInteger(channels) || channels < 1) {
|
|
189
|
+
throw wavError(`unsupported channel count ${channels}.`);
|
|
190
|
+
}
|
|
191
|
+
if (!Number.isInteger(sampleRate) || sampleRate <= 0) {
|
|
192
|
+
throw wavError(`unsupported sample rate ${sampleRate}.`);
|
|
193
|
+
}
|
|
194
|
+
// Accept plain PCM; also accept EXTENSIBLE only when it's declared 16-bit PCM
|
|
195
|
+
// (a common wrapper the codecs emit for the same data). Reject float/exotic.
|
|
196
|
+
const isPcm16 =
|
|
197
|
+
(format === WAVE_FORMAT_PCM || format === WAVE_FORMAT_EXTENSIBLE) && bitsPerSample === 16;
|
|
198
|
+
if (!isPcm16) {
|
|
199
|
+
const kind =
|
|
200
|
+
format === WAVE_FORMAT_IEEE_FLOAT
|
|
201
|
+
? 'IEEE float'
|
|
202
|
+
: format === WAVE_FORMAT_PCM || format === WAVE_FORMAT_EXTENSIBLE
|
|
203
|
+
? `${bitsPerSample}-bit PCM`
|
|
204
|
+
: `format 0x${format.toString(16)}`;
|
|
205
|
+
throw wavError(
|
|
206
|
+
`unsupported WAV encoding (${kind}). Only 16-bit PCM WAV is decoded in-process; re-export as PCM16 WAV, or install ffmpeg to decode other formats.`,
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const pcm = bytes.subarray(dataOffset, dataOffset + dataLen);
|
|
211
|
+
return { samples: pcm16ToFloat32(pcm), sampleRate, channels };
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function ascii4(bytes: Uint8Array, off: number): string {
|
|
215
|
+
return String.fromCharCode(bytes[off]!, bytes[off + 1]!, bytes[off + 2]!, bytes[off + 3]!);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function wavError(detail: string): MoxxyError {
|
|
219
|
+
return new MoxxyError({ code: 'CONFIG_INVALID', message: `Invalid WAV audio: ${detail}` });
|
|
220
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
import { EventEmitter } from 'node:events';
|
|
3
|
+
import type { spawn } from 'node:child_process';
|
|
4
|
+
import { describe, expect, it } from 'vitest';
|
|
5
|
+
|
|
6
|
+
import { MOXXY_PCM16_24KHZ_MIME } from '@moxxy/sdk';
|
|
7
|
+
|
|
8
|
+
import { decodeToMono16k } from './decode.js';
|
|
9
|
+
|
|
10
|
+
/** Minimal fake `spawn`: probe → exit 0, decode → emit the given f32le bytes. */
|
|
11
|
+
function fakeSpawn(samples: Float32Array): typeof spawn {
|
|
12
|
+
return ((_cmd: string, args: readonly string[]) => {
|
|
13
|
+
const child = new EventEmitter() as EventEmitter & {
|
|
14
|
+
stdout: EventEmitter;
|
|
15
|
+
stderr: EventEmitter;
|
|
16
|
+
stdin: { writable: boolean; on: () => void; end: () => void };
|
|
17
|
+
kill: () => boolean;
|
|
18
|
+
killed: boolean;
|
|
19
|
+
};
|
|
20
|
+
child.stdout = new EventEmitter();
|
|
21
|
+
child.stderr = new EventEmitter();
|
|
22
|
+
child.killed = false;
|
|
23
|
+
child.kill = () => ((child.killed = true), true);
|
|
24
|
+
child.stdin = { writable: true, on: () => {}, end: () => {} };
|
|
25
|
+
const isProbe = args.includes('-version');
|
|
26
|
+
setImmediate(() => {
|
|
27
|
+
if (isProbe) {
|
|
28
|
+
child.emit('close', 0);
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
if (samples.length > 0) child.stdout.emit('data', Buffer.from(samples.buffer.slice(0)));
|
|
32
|
+
child.emit('close', 0);
|
|
33
|
+
});
|
|
34
|
+
return child;
|
|
35
|
+
}) as unknown as typeof spawn;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Build raw PCM16 mono little-endian bytes for the given float samples. */
|
|
39
|
+
function rawPcm16(samples: number[]): Uint8Array {
|
|
40
|
+
const bytes = new Uint8Array(samples.length * 2);
|
|
41
|
+
const view = new DataView(bytes.buffer);
|
|
42
|
+
samples.forEach((s, i) => view.setInt16(i * 2, Math.round(s * 32767), true));
|
|
43
|
+
return bytes;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Build a mono PCM16 WAV at the given sample rate. */
|
|
47
|
+
function monoWav(samples: number[], sampleRate: number): Uint8Array {
|
|
48
|
+
const dataSize = samples.length * 2;
|
|
49
|
+
const buf = new Uint8Array(44 + dataSize);
|
|
50
|
+
const view = new DataView(buf.buffer);
|
|
51
|
+
const ascii = (o: number, s: string): void => {
|
|
52
|
+
for (let i = 0; i < s.length; i += 1) buf[o + i] = s.charCodeAt(i);
|
|
53
|
+
};
|
|
54
|
+
ascii(0, 'RIFF');
|
|
55
|
+
view.setUint32(4, 36 + dataSize, true);
|
|
56
|
+
ascii(8, 'WAVE');
|
|
57
|
+
ascii(12, 'fmt ');
|
|
58
|
+
view.setUint32(16, 16, true);
|
|
59
|
+
view.setUint16(20, 1, true);
|
|
60
|
+
view.setUint16(22, 1, true);
|
|
61
|
+
view.setUint32(24, sampleRate, true);
|
|
62
|
+
view.setUint32(28, sampleRate * 2, true);
|
|
63
|
+
view.setUint16(32, 2, true);
|
|
64
|
+
view.setUint16(34, 16, true);
|
|
65
|
+
ascii(36, 'data');
|
|
66
|
+
view.setUint32(40, dataSize, true);
|
|
67
|
+
samples.forEach((s, i) => view.setInt16(44 + i * 2, Math.round(s * 32767), true));
|
|
68
|
+
return buf;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
describe('decodeToMono16k', () => {
|
|
72
|
+
it('converts raw PCM16 mono @24k and resamples to 16k (no ffmpeg)', async () => {
|
|
73
|
+
const dc = new Array<number>(240).fill(0.5); // 240 samples @24k → 160 @16k
|
|
74
|
+
const out = await decodeToMono16k(rawPcm16(dc), MOXXY_PCM16_24KHZ_MIME);
|
|
75
|
+
expect(out.length).toBe(160);
|
|
76
|
+
for (const s of out) expect(s).toBeCloseTo(0.5, 2);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it('parses a 16k mono WAV unchanged (no resample, no ffmpeg)', async () => {
|
|
80
|
+
const wav = monoWav([0, 0.5, -0.5], 16_000);
|
|
81
|
+
const out = await decodeToMono16k(wav, 'audio/wav');
|
|
82
|
+
expect(out.length).toBe(3);
|
|
83
|
+
expect(out[1]).toBeCloseTo(0.5, 3);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it('resamples a 8k WAV up to 16k', async () => {
|
|
87
|
+
const wav = monoWav(new Array<number>(80).fill(0.25), 8_000);
|
|
88
|
+
const out = await decodeToMono16k(wav, 'audio/x-wav');
|
|
89
|
+
expect(out.length).toBe(160);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it('sniffs RIFF/WAVE magic when the MIME is missing', async () => {
|
|
93
|
+
const wav = monoWav([0.1, 0.2], 16_000);
|
|
94
|
+
const out = await decodeToMono16k(wav, undefined);
|
|
95
|
+
expect(out.length).toBe(2);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it('routes compressed audio (audio/ogg) through ffmpeg', async () => {
|
|
99
|
+
const decoded = Float32Array.from([0.1, -0.2, 0.3]);
|
|
100
|
+
const out = await decodeToMono16k(new Uint8Array([1, 2, 3]), 'audio/ogg', {
|
|
101
|
+
spawnImpl: fakeSpawn(decoded),
|
|
102
|
+
});
|
|
103
|
+
expect(Array.from(out)).toEqual([
|
|
104
|
+
expect.closeTo(0.1, 5),
|
|
105
|
+
expect.closeTo(-0.2, 5),
|
|
106
|
+
expect.closeTo(0.3, 5),
|
|
107
|
+
]);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
it('handles a MIME with a codecs= parameter (case-insensitive)', async () => {
|
|
111
|
+
const decoded = Float32Array.from([0.42]);
|
|
112
|
+
const out = await decodeToMono16k(new Uint8Array([1]), 'AUDIO/OGG; codecs=opus', {
|
|
113
|
+
spawnImpl: fakeSpawn(decoded),
|
|
114
|
+
});
|
|
115
|
+
expect(out[0]).toBeCloseTo(0.42, 5);
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
it('rejects an IEEE-float WAV politely instead of mis-decoding', async () => {
|
|
119
|
+
// Build a float WAV header (format 3) — parseWav must reject it.
|
|
120
|
+
const buf = monoWav([0], 16_000);
|
|
121
|
+
new DataView(buf.buffer).setUint16(20, 3, true); // audioFormat = IEEE float
|
|
122
|
+
await expect(decodeToMono16k(buf, 'audio/wav')).rejects.toMatchObject({
|
|
123
|
+
code: 'CONFIG_INVALID',
|
|
124
|
+
});
|
|
125
|
+
});
|
|
126
|
+
});
|