@happyvertical/speech 0.97.0 → 0.99.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT.md +31 -3
- package/README.md +310 -2
- package/dist/__tests__/fake-websocket.d.ts +38 -0
- package/dist/__tests__/fake-websocket.d.ts.map +1 -0
- package/dist/adapters/local/abort.d.ts +6 -0
- package/dist/adapters/local/abort.d.ts.map +1 -0
- package/dist/adapters/local/audio.d.ts +45 -0
- package/dist/adapters/local/audio.d.ts.map +1 -0
- package/dist/adapters/local/env.d.ts +15 -0
- package/dist/adapters/local/env.d.ts.map +1 -0
- package/dist/adapters/local/runtime.d.ts +58 -0
- package/dist/adapters/local/runtime.d.ts.map +1 -0
- package/dist/adapters/local/transcriber.d.ts +40 -0
- package/dist/adapters/local/transcriber.d.ts.map +1 -0
- package/dist/adapters/local/transformers-shim.d.ts +8 -0
- package/dist/adapters/local/transformers-shim.d.ts.map +1 -0
- package/dist/adapters/local/types.d.ts +92 -0
- package/dist/adapters/local/types.d.ts.map +1 -0
- package/dist/adapters/local/worker.d.ts +118 -0
- package/dist/adapters/local/worker.d.ts.map +1 -0
- package/dist/adapters/openai-compatible.d.ts +2 -1
- package/dist/adapters/openai-compatible.d.ts.map +1 -1
- package/dist/adapters/openai-realtime.d.ts +48 -0
- package/dist/adapters/openai-realtime.d.ts.map +1 -0
- package/dist/adapters/voxtral-realtime.d.ts +44 -0
- package/dist/adapters/voxtral-realtime.d.ts.map +1 -0
- package/dist/chunks/errors-CQBX3OPS.js +41 -0
- package/dist/chunks/errors-CQBX3OPS.js.map +1 -0
- package/dist/chunks/registry-C_5BakmM.js +306 -0
- package/dist/chunks/registry-C_5BakmM.js.map +1 -0
- package/dist/chunks/websocket-CR9uXAuS.js +44 -0
- package/dist/chunks/websocket-CR9uXAuS.js.map +1 -0
- package/dist/conversation/openai-webrtc.d.ts +4 -0
- package/dist/conversation/openai-webrtc.d.ts.map +1 -0
- package/dist/conversation/types.d.ts +60 -0
- package/dist/conversation/types.d.ts.map +1 -0
- package/dist/conversation-server.d.ts +40 -0
- package/dist/conversation-server.d.ts.map +1 -0
- package/dist/conversation-server.js +174 -0
- package/dist/conversation-server.js.map +1 -0
- package/dist/conversation.d.ts +4 -0
- package/dist/conversation.d.ts.map +1 -0
- package/dist/conversation.js +346 -0
- package/dist/conversation.js.map +1 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1824 -326
- package/dist/index.js.map +1 -1
- package/dist/local.d.ts +21 -0
- package/dist/local.d.ts.map +1 -0
- package/dist/local.js +964 -0
- package/dist/local.js.map +1 -0
- package/dist/shared/env.d.ts +5 -0
- package/dist/shared/env.d.ts.map +1 -1
- package/dist/shared/factory.d.ts.map +1 -1
- package/dist/shared/pcm.d.ts +21 -0
- package/dist/shared/pcm.d.ts.map +1 -0
- package/dist/shared/realtime-auth.d.ts +34 -0
- package/dist/shared/realtime-auth.d.ts.map +1 -0
- package/dist/shared/realtime-session.d.ts +240 -0
- package/dist/shared/realtime-session.d.ts.map +1 -0
- package/dist/shared/registry.d.ts +9 -0
- package/dist/shared/registry.d.ts.map +1 -0
- package/dist/shared/streaming-client-secret.d.ts +90 -0
- package/dist/shared/streaming-client-secret.d.ts.map +1 -0
- package/dist/shared/streaming-factory.d.ts +65 -0
- package/dist/shared/streaming-factory.d.ts.map +1 -0
- package/dist/shared/streaming-types.d.ts +303 -0
- package/dist/shared/streaming-types.d.ts.map +1 -0
- package/dist/shared/turn-limit.d.ts +84 -0
- package/dist/shared/turn-limit.d.ts.map +1 -0
- package/dist/shared/types.d.ts +24 -6
- package/dist/shared/types.d.ts.map +1 -1
- package/dist/shared/websocket.d.ts +56 -0
- package/dist/shared/websocket.d.ts.map +1 -0
- package/metadata.json +9 -2
- package/package.json +27 -3
package/dist/local.js
ADDED
|
@@ -0,0 +1,964 @@
|
|
|
1
|
+
import { i as SpeechProviderError, n as SpeechConfigurationError, r as SpeechError } from "./chunks/errors-CQBX3OPS.js";
|
|
2
|
+
import { f as readEnv, h as reportSpeechUsage, o as defaultEnv, p as resolveTranscriberConfig, r as registerOptionalTranscriber, v as normalizeAudioInput, y as normalizeMimeType } from "./chunks/registry-C_5BakmM.js";
|
|
3
|
+
//#region src/adapters/local/env.ts
|
|
4
|
+
/**
|
|
5
|
+
* Environment configuration for the `local` transcriber. Explicit options win,
|
|
6
|
+
* then the factory context, then `HAVE_SPEECH_TRANSCRIBER_*` variables (Node).
|
|
7
|
+
*/
|
|
8
|
+
/** Local-only environment variables, in priority order. */
|
|
9
|
+
var LOCAL_TRANSCRIBER_ENV_KEYS = {
|
|
10
|
+
device: ["HAVE_SPEECH_TRANSCRIBER_DEVICE"],
|
|
11
|
+
dtype: ["HAVE_SPEECH_TRANSCRIBER_DTYPE"],
|
|
12
|
+
cacheDir: ["HAVE_SPEECH_TRANSCRIBER_CACHE_DIR"],
|
|
13
|
+
modelHost: ["HAVE_SPEECH_TRANSCRIBER_MODEL_HOST"]
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Merges explicit options with the environment. `model` and `maxBytes` reuse
|
|
17
|
+
* the shared `HAVE_SPEECH_TRANSCRIBER_MODEL`/`_MAX_BYTES` keys.
|
|
18
|
+
*/
|
|
19
|
+
function resolveLocalTranscriberOptions(options = {}, context = {}) {
|
|
20
|
+
const env = context.env ?? defaultEnv();
|
|
21
|
+
const shared = resolveTranscriberConfig({
|
|
22
|
+
model: options.model,
|
|
23
|
+
maxBytes: options.maxBytes
|
|
24
|
+
}, { env });
|
|
25
|
+
return {
|
|
26
|
+
...options,
|
|
27
|
+
type: "local",
|
|
28
|
+
model: shared.model,
|
|
29
|
+
maxBytes: shared.maxBytes,
|
|
30
|
+
device: options.device ?? readEnv(env, ...LOCAL_TRANSCRIBER_ENV_KEYS.device),
|
|
31
|
+
dtype: options.dtype ?? readEnv(env, ...LOCAL_TRANSCRIBER_ENV_KEYS.dtype),
|
|
32
|
+
cacheDir: options.cacheDir ?? readEnv(env, ...LOCAL_TRANSCRIBER_ENV_KEYS.cacheDir),
|
|
33
|
+
modelHost: options.modelHost ?? readEnv(env, ...LOCAL_TRANSCRIBER_ENV_KEYS.modelHost)
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
//#endregion
|
|
37
|
+
//#region src/adapters/local/runtime.ts
|
|
38
|
+
/**
|
|
39
|
+
* Loads the optional `@huggingface/transformers` peer dependency and builds
|
|
40
|
+
* `automatic-speech-recognition` pipelines from it.
|
|
41
|
+
*
|
|
42
|
+
* This is the only module that names the peer. It is reachable only from the
|
|
43
|
+
* `@happyvertical/speech/local` entry, never from the core entry.
|
|
44
|
+
*/
|
|
45
|
+
var TRANSFORMERS_PACKAGE = "@huggingface/transformers";
|
|
46
|
+
var TRANSFORMERS_INSTALL_HINT = `Local transcription needs the optional peer dependency ${TRANSFORMERS_PACKAGE}. Install it with \`pnpm add ${TRANSFORMERS_PACKAGE}\` (or npm/yarn equivalent).`;
|
|
47
|
+
/**
|
|
48
|
+
* Resolves the transformers.js module: an injected module or loader, else a
|
|
49
|
+
* dynamic import of the peer. A missing peer becomes a
|
|
50
|
+
* `SpeechConfigurationError` with an install hint.
|
|
51
|
+
*/
|
|
52
|
+
async function loadTransformers(runtime) {
|
|
53
|
+
let loaded;
|
|
54
|
+
try {
|
|
55
|
+
loaded = typeof runtime === "function" ? await runtime() : runtime ?? await importTransformers();
|
|
56
|
+
} catch (error) {
|
|
57
|
+
if (error instanceof SpeechConfigurationError) throw error;
|
|
58
|
+
const configurationError = new SpeechConfigurationError(`${TRANSFORMERS_INSTALL_HINT} (${errorMessage$2(error)})`, "local");
|
|
59
|
+
configurationError.cause = error;
|
|
60
|
+
throw configurationError;
|
|
61
|
+
}
|
|
62
|
+
const candidate = loaded;
|
|
63
|
+
const module = typeof candidate?.pipeline === "function" ? candidate : candidate?.default;
|
|
64
|
+
if (typeof module?.pipeline !== "function") throw new SpeechConfigurationError(`${TRANSFORMERS_PACKAGE} did not export pipeline(); expected v4 or later`, "local");
|
|
65
|
+
return module;
|
|
66
|
+
}
|
|
67
|
+
/** Isolated so tests can observe whether the peer was ever imported. */
|
|
68
|
+
function importTransformers() {
|
|
69
|
+
return import("@huggingface/transformers");
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Creates the ASR pipeline. `device: 'auto'` resolves to WebGPU when a
|
|
73
|
+
* browser adapter is available (retrying on WASM if WebGPU fails to
|
|
74
|
+
* initialise) and to CPU elsewhere.
|
|
75
|
+
*/
|
|
76
|
+
async function createAsrPipeline(transformers, options) {
|
|
77
|
+
const device = options.device === "auto" ? await detectAutoDevice() : options.device;
|
|
78
|
+
const build = (selected) => transformers.pipeline("automatic-speech-recognition", options.model, {
|
|
79
|
+
device: selected,
|
|
80
|
+
...options.dtype ? { dtype: options.dtype } : {},
|
|
81
|
+
...options.revision ? { revision: options.revision } : {},
|
|
82
|
+
...options.cacheDir ? { cache_dir: options.cacheDir } : {},
|
|
83
|
+
...options.onProgress ? { progress_callback: options.onProgress } : {}
|
|
84
|
+
});
|
|
85
|
+
try {
|
|
86
|
+
return {
|
|
87
|
+
pipeline: await build(device),
|
|
88
|
+
device
|
|
89
|
+
};
|
|
90
|
+
} catch (error) {
|
|
91
|
+
if (options.device === "auto" && device === "webgpu") return {
|
|
92
|
+
pipeline: await build("wasm"),
|
|
93
|
+
device: "wasm"
|
|
94
|
+
};
|
|
95
|
+
throw error;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/** `webgpu` when a browser exposes a usable adapter, `wasm` in other browsers, `cpu` in Node. */
|
|
99
|
+
async function detectAutoDevice() {
|
|
100
|
+
if (isNodeRuntime()) return "cpu";
|
|
101
|
+
const gpu = globalThis.navigator?.gpu;
|
|
102
|
+
if (gpu) try {
|
|
103
|
+
if (await gpu.requestAdapter()) return "webgpu";
|
|
104
|
+
} catch {}
|
|
105
|
+
return "wasm";
|
|
106
|
+
}
|
|
107
|
+
function isNodeRuntime() {
|
|
108
|
+
return Boolean(globalThis.process?.versions?.node);
|
|
109
|
+
}
|
|
110
|
+
function errorMessage$2(error) {
|
|
111
|
+
return error instanceof Error ? error.message : String(error);
|
|
112
|
+
}
|
|
113
|
+
//#endregion
|
|
114
|
+
//#region src/adapters/local/abort.ts
|
|
115
|
+
/**
|
|
116
|
+
* Rejects with the signal's reason as soon as it aborts, without waiting for
|
|
117
|
+
* `promise` (which keeps running; its eventual result is ignored).
|
|
118
|
+
*/
|
|
119
|
+
function raceAbort(promise, signal) {
|
|
120
|
+
if (!signal) return promise;
|
|
121
|
+
if (signal.aborted) {
|
|
122
|
+
promise.catch(() => void 0);
|
|
123
|
+
return Promise.reject(signal.reason);
|
|
124
|
+
}
|
|
125
|
+
return new Promise((resolve, reject) => {
|
|
126
|
+
const onAbort = () => reject(signal.reason);
|
|
127
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
128
|
+
promise.then((value) => {
|
|
129
|
+
signal.removeEventListener("abort", onAbort);
|
|
130
|
+
resolve(value);
|
|
131
|
+
}, (error) => {
|
|
132
|
+
signal.removeEventListener("abort", onAbort);
|
|
133
|
+
reject(error);
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
//#endregion
|
|
138
|
+
//#region src/adapters/local/audio.ts
|
|
139
|
+
/**
|
|
140
|
+
* Audio decoding for on-device inference: turns encoded bytes into 16 kHz
|
|
141
|
+
* mono Float32 PCM, the input Whisper and Moonshine expect.
|
|
142
|
+
*
|
|
143
|
+
* - WAV (PCM 8/16/24/32-bit, IEEE float 32/64, WAVE_FORMAT_EXTENSIBLE) and raw
|
|
144
|
+
* `audio/pcm` are decoded in-process in every runtime, including workers.
|
|
145
|
+
* - Anything else goes to the caller's `decodeAudio` hook, or, in browsers,
|
|
146
|
+
* to `OfflineAudioContext.decodeAudioData` (MediaRecorder WebM/Opus, Safari
|
|
147
|
+
* `audio/mp4`, MP3, ...).
|
|
148
|
+
*/
|
|
149
|
+
/** Sample rate expected by Whisper and Moonshine feature extractors. */
|
|
150
|
+
var LOCAL_SAMPLE_RATE = 16e3;
|
|
151
|
+
var ADAPTER = "local";
|
|
152
|
+
var WAV_MIME_TYPES = /* @__PURE__ */ new Set([
|
|
153
|
+
"audio/wav",
|
|
154
|
+
"audio/wave",
|
|
155
|
+
"audio/x-wav",
|
|
156
|
+
"audio/vnd.wave"
|
|
157
|
+
]);
|
|
158
|
+
var PCM_MIME_TYPES = /* @__PURE__ */ new Set([
|
|
159
|
+
"audio/pcm",
|
|
160
|
+
"audio/x-pcm",
|
|
161
|
+
"audio/raw"
|
|
162
|
+
]);
|
|
163
|
+
/**
|
|
164
|
+
* Decodes `bytes` and returns 16 kHz mono Float32 samples.
|
|
165
|
+
*
|
|
166
|
+
* Raw PCM (`audio/pcm`) is little-endian signed 16-bit by default; pass
|
|
167
|
+
* `encoding=f32le` for Float32. The rate comes from the `rate` MIME parameter
|
|
168
|
+
* or `sampleRate`, the channel count from `channels` (default 1).
|
|
169
|
+
*/
|
|
170
|
+
async function decodeToPcm16k(bytes, options) {
|
|
171
|
+
return toPcm16k(await raceAbort(decodeAudioBytes(bytes, options), options.signal));
|
|
172
|
+
}
|
|
173
|
+
/** Downmixes decoded audio to mono and resamples it to 16 kHz. */
|
|
174
|
+
function toPcm16k(decoded) {
|
|
175
|
+
return resample(downmix(decoded.samples), decoded.sampleRate, LOCAL_SAMPLE_RATE);
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* True for WAV and raw PCM, which this module decodes itself in every
|
|
179
|
+
* runtime, including Web Workers.
|
|
180
|
+
*/
|
|
181
|
+
function isInProcessDecodable(bytes, mimeType) {
|
|
182
|
+
const essence = normalizeMimeType(mimeType);
|
|
183
|
+
return Boolean(essence && (WAV_MIME_TYPES.has(essence) || PCM_MIME_TYPES.has(essence)) || isRiffWave(bytes));
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Decodes a format without an in-process decoder through the caller's
|
|
187
|
+
* `decodeAudio` hook or, in browsers, `OfflineAudioContext`. Resolves
|
|
188
|
+
* `undefined` when neither is available.
|
|
189
|
+
*/
|
|
190
|
+
async function decodeExternally(bytes, options) {
|
|
191
|
+
if (options.decodeAudio) {
|
|
192
|
+
const decoded = await options.decodeAudio({
|
|
193
|
+
bytes,
|
|
194
|
+
mimeType: options.mimeType,
|
|
195
|
+
signal: options.signal
|
|
196
|
+
});
|
|
197
|
+
assertDecodedAudio(decoded);
|
|
198
|
+
return decoded;
|
|
199
|
+
}
|
|
200
|
+
if (hasOfflineAudioContext()) return decodeWithAudioContext(bytes);
|
|
201
|
+
}
|
|
202
|
+
async function decodeAudioBytes(bytes, options) {
|
|
203
|
+
const essence = normalizeMimeType(options.mimeType);
|
|
204
|
+
if (essence && WAV_MIME_TYPES.has(essence) || isRiffWave(bytes)) return decodeWav(bytes);
|
|
205
|
+
if (essence && PCM_MIME_TYPES.has(essence)) return decodeRawPcm(bytes, options);
|
|
206
|
+
const decoded = await decodeExternally(bytes, options);
|
|
207
|
+
if (decoded) return decoded;
|
|
208
|
+
throw new SpeechConfigurationError(`Cannot decode '${options.mimeType}' audio on this runtime: pass WAV or audio/pcm, or supply a decodeAudio hook (e.g. ffmpeg) for compressed formats`, ADAPTER);
|
|
209
|
+
}
|
|
210
|
+
function isRiffWave(bytes) {
|
|
211
|
+
return bytes.byteLength >= 12 && ascii(bytes, 0, 4) === "RIFF" && ascii(bytes, 8, 4) === "WAVE";
|
|
212
|
+
}
|
|
213
|
+
/** Parses a RIFF/WAVE file into per-channel Float32 samples. */
|
|
214
|
+
function decodeWav(bytes) {
|
|
215
|
+
if (!isRiffWave(bytes)) throw new SpeechConfigurationError("Invalid WAV data: missing RIFF/WAVE header", ADAPTER);
|
|
216
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
217
|
+
let format;
|
|
218
|
+
let offset = 12;
|
|
219
|
+
while (offset + 8 <= bytes.byteLength) {
|
|
220
|
+
const id = ascii(bytes, offset, 4);
|
|
221
|
+
const size = view.getUint32(offset + 4, true);
|
|
222
|
+
const body = offset + 8;
|
|
223
|
+
if (id === "fmt ") format = readWavFormat(view, body, size);
|
|
224
|
+
else if (id === "data") {
|
|
225
|
+
if (!format) throw new SpeechConfigurationError("Invalid WAV data: data chunk before fmt chunk", ADAPTER);
|
|
226
|
+
const available = bytes.byteLength - body;
|
|
227
|
+
const length = size === 0 || size > available ? available : size;
|
|
228
|
+
return {
|
|
229
|
+
samples: deinterleave(bytes.subarray(body, body + length), format.channels, format.encoding),
|
|
230
|
+
sampleRate: format.sampleRate
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
offset = body + size + size % 2;
|
|
234
|
+
}
|
|
235
|
+
throw new SpeechConfigurationError("Invalid WAV data: no data chunk", ADAPTER);
|
|
236
|
+
}
|
|
237
|
+
var WAVE_FORMAT_PCM = 1;
|
|
238
|
+
var WAVE_FORMAT_IEEE_FLOAT = 3;
|
|
239
|
+
var WAVE_FORMAT_EXTENSIBLE = 65534;
|
|
240
|
+
function readWavFormat(view, body, size) {
|
|
241
|
+
if (size < 16) throw new SpeechConfigurationError("Invalid WAV data: fmt chunk too short", ADAPTER);
|
|
242
|
+
let tag = view.getUint16(body, true);
|
|
243
|
+
const channels = view.getUint16(body + 2, true);
|
|
244
|
+
const sampleRate = view.getUint32(body + 4, true);
|
|
245
|
+
const bitsPerSample = view.getUint16(body + 14, true);
|
|
246
|
+
if (tag === WAVE_FORMAT_EXTENSIBLE && size >= 26) tag = view.getUint16(body + 24, true);
|
|
247
|
+
const encoding = tag === WAVE_FORMAT_PCM ? {
|
|
248
|
+
8: "u8",
|
|
249
|
+
16: "s16le",
|
|
250
|
+
24: "s24le",
|
|
251
|
+
32: "s32le"
|
|
252
|
+
}[bitsPerSample] : tag === WAVE_FORMAT_IEEE_FLOAT ? {
|
|
253
|
+
32: "f32le",
|
|
254
|
+
64: "f64le"
|
|
255
|
+
}[bitsPerSample] : void 0;
|
|
256
|
+
if (!encoding || channels < 1 || sampleRate < 1) throw new SpeechConfigurationError(`Unsupported WAV encoding (format ${tag}, ${bitsPerSample}-bit, ${channels} channels); use PCM or IEEE float, or supply a decodeAudio hook`, ADAPTER);
|
|
257
|
+
return {
|
|
258
|
+
channels,
|
|
259
|
+
sampleRate,
|
|
260
|
+
encoding
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
function decodeRawPcm(bytes, options) {
|
|
264
|
+
const params = mimeParameters(options.mimeType);
|
|
265
|
+
const sampleRate = positiveInteger(params.rate) ?? explicitPositiveInteger(options.sampleRate, "sampleRate");
|
|
266
|
+
const channels = positiveInteger(params.channels) ?? explicitPositiveInteger(options.channels, "channels") ?? 1;
|
|
267
|
+
const encoding = (params.encoding ?? "s16le").toLowerCase();
|
|
268
|
+
if (!sampleRate) throw new SpeechConfigurationError("Raw PCM audio needs a sample rate: use 'audio/pcm;rate=16000' or AudioInput.sampleRate", ADAPTER);
|
|
269
|
+
if (encoding !== "s16le" && encoding !== "f32le") throw new SpeechConfigurationError(`Unsupported raw PCM encoding '${encoding}' (expected s16le or f32le)`, ADAPTER);
|
|
270
|
+
return {
|
|
271
|
+
samples: deinterleave(bytes, channels, encoding),
|
|
272
|
+
sampleRate
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
var BYTES_PER_SAMPLE = {
|
|
276
|
+
u8: 1,
|
|
277
|
+
s16le: 2,
|
|
278
|
+
s24le: 3,
|
|
279
|
+
s32le: 4,
|
|
280
|
+
f32le: 4,
|
|
281
|
+
f64le: 8
|
|
282
|
+
};
|
|
283
|
+
function deinterleave(bytes, channels, encoding) {
|
|
284
|
+
const width = BYTES_PER_SAMPLE[encoding];
|
|
285
|
+
const frames = Math.floor(bytes.byteLength / (width * channels));
|
|
286
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
287
|
+
const output = Array.from({ length: channels }, () => new Float32Array(frames));
|
|
288
|
+
for (let frame = 0; frame < frames; frame++) for (let channel = 0; channel < channels; channel++) {
|
|
289
|
+
const at = (frame * channels + channel) * width;
|
|
290
|
+
output[channel][frame] = readSample(view, at, encoding);
|
|
291
|
+
}
|
|
292
|
+
return output;
|
|
293
|
+
}
|
|
294
|
+
function readSample(view, at, encoding) {
|
|
295
|
+
switch (encoding) {
|
|
296
|
+
case "u8": return (view.getUint8(at) - 128) / 128;
|
|
297
|
+
case "s16le": return view.getInt16(at, true) / 32768;
|
|
298
|
+
case "s24le": return (view.getUint8(at) | view.getUint8(at + 1) << 8 | view.getInt8(at + 2) << 16) / 8388608;
|
|
299
|
+
case "s32le": return view.getInt32(at, true) / 2147483648;
|
|
300
|
+
case "f32le": return view.getFloat32(at, true);
|
|
301
|
+
case "f64le": return view.getFloat64(at, true);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
/** Averages channels into one mono track. */
|
|
305
|
+
function downmix(samples) {
|
|
306
|
+
if (samples instanceof Float32Array) return samples;
|
|
307
|
+
if (samples.length === 1) return samples[0];
|
|
308
|
+
const length = Math.min(...samples.map((channel) => channel.length));
|
|
309
|
+
const mono = new Float32Array(length);
|
|
310
|
+
for (const channel of samples) for (let index = 0; index < length; index++) mono[index] = mono[index] + channel[index] / samples.length;
|
|
311
|
+
return mono;
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Resamples mono audio with linear interpolation. When downsampling, a box
|
|
315
|
+
* filter one source period wide is applied first to limit aliasing; this is
|
|
316
|
+
* adequate for speech recognition, not for music playback.
|
|
317
|
+
*/
|
|
318
|
+
function resample(samples, fromRate, toRate) {
|
|
319
|
+
if (fromRate === toRate || samples.length === 0) return samples;
|
|
320
|
+
const ratio = fromRate / toRate;
|
|
321
|
+
const source = ratio > 1 ? boxFilter(samples, Math.round(ratio)) : samples;
|
|
322
|
+
const length = Math.max(1, Math.round(samples.length / ratio));
|
|
323
|
+
const output = new Float32Array(length);
|
|
324
|
+
for (let index = 0; index < length; index++) {
|
|
325
|
+
const position = index * ratio;
|
|
326
|
+
const left = Math.floor(position);
|
|
327
|
+
const right = Math.min(left + 1, source.length - 1);
|
|
328
|
+
const fraction = position - left;
|
|
329
|
+
output[index] = source[left] * (1 - fraction) + source[right] * fraction;
|
|
330
|
+
}
|
|
331
|
+
return output;
|
|
332
|
+
}
|
|
333
|
+
function boxFilter(samples, width) {
|
|
334
|
+
if (width <= 1) return samples;
|
|
335
|
+
const output = new Float32Array(samples.length);
|
|
336
|
+
const half = Math.floor(width / 2);
|
|
337
|
+
let sum = 0;
|
|
338
|
+
let count = 0;
|
|
339
|
+
let start = 0;
|
|
340
|
+
let end = 0;
|
|
341
|
+
for (let index = 0; index < samples.length; index++) {
|
|
342
|
+
const windowStart = Math.max(0, index - half);
|
|
343
|
+
const windowEnd = Math.min(samples.length, index - half + width);
|
|
344
|
+
while (end < windowEnd) {
|
|
345
|
+
sum += samples[end];
|
|
346
|
+
count++;
|
|
347
|
+
end++;
|
|
348
|
+
}
|
|
349
|
+
while (start < windowStart) {
|
|
350
|
+
sum -= samples[start];
|
|
351
|
+
count--;
|
|
352
|
+
start++;
|
|
353
|
+
}
|
|
354
|
+
output[index] = sum / count;
|
|
355
|
+
}
|
|
356
|
+
return output;
|
|
357
|
+
}
|
|
358
|
+
function hasOfflineAudioContext() {
|
|
359
|
+
return typeof globalThis.OfflineAudioContext === "function";
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Browser decode path. Decoding through a 16 kHz `OfflineAudioContext` makes
|
|
363
|
+
* the browser resample with its own high-quality resampler. Not available in
|
|
364
|
+
* Web Workers: decode on the main thread (see the worker client).
|
|
365
|
+
*/
|
|
366
|
+
async function decodeWithAudioContext(bytes) {
|
|
367
|
+
const context = new OfflineAudioContext(1, 1, LOCAL_SAMPLE_RATE);
|
|
368
|
+
const copy = new Uint8Array(bytes.byteLength);
|
|
369
|
+
copy.set(bytes);
|
|
370
|
+
let buffer;
|
|
371
|
+
try {
|
|
372
|
+
buffer = await context.decodeAudioData(copy.buffer);
|
|
373
|
+
} catch (error) {
|
|
374
|
+
throw new SpeechConfigurationError(`The browser could not decode the audio: ${errorMessage$1(error)}`, ADAPTER);
|
|
375
|
+
}
|
|
376
|
+
return {
|
|
377
|
+
samples: Array.from({ length: buffer.numberOfChannels }, (_, index) => buffer.getChannelData(index)),
|
|
378
|
+
sampleRate: buffer.sampleRate
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
function assertDecodedAudio(decoded) {
|
|
382
|
+
const channels = decoded?.samples instanceof Float32Array ? [decoded.samples] : decoded?.samples;
|
|
383
|
+
if (!Array.isArray(channels) || channels.length === 0 || !channels.every((channel) => channel instanceof Float32Array) || !(decoded.sampleRate > 0)) throw new SpeechConfigurationError("decodeAudio must return { samples: Float32Array | Float32Array[], sampleRate }", ADAPTER);
|
|
384
|
+
}
|
|
385
|
+
function mimeParameters(mimeType) {
|
|
386
|
+
const params = {};
|
|
387
|
+
for (const part of mimeType.split(";").slice(1)) {
|
|
388
|
+
const [key, value] = part.split("=");
|
|
389
|
+
if (key && value) params[key.trim().toLowerCase()] = value.trim().replace(/^"|"$/g, "");
|
|
390
|
+
}
|
|
391
|
+
return params;
|
|
392
|
+
}
|
|
393
|
+
function positiveInteger(value) {
|
|
394
|
+
const parsed = value ? Number.parseInt(value, 10) : NaN;
|
|
395
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : void 0;
|
|
396
|
+
}
|
|
397
|
+
/** Validates a caller-supplied `AudioInput.sampleRate`/`channels`. */
|
|
398
|
+
function explicitPositiveInteger(value, name) {
|
|
399
|
+
if (value === void 0 || value === null) return;
|
|
400
|
+
if (!Number.isInteger(value) || value < 1) throw new SpeechConfigurationError(`AudioInput.${name} must be a positive integer, got ${String(value)}`, ADAPTER);
|
|
401
|
+
return value;
|
|
402
|
+
}
|
|
403
|
+
function ascii(bytes, offset, length) {
|
|
404
|
+
return String.fromCharCode(...bytes.subarray(offset, offset + length));
|
|
405
|
+
}
|
|
406
|
+
function errorMessage$1(error) {
|
|
407
|
+
return error instanceof Error ? error.message : String(error);
|
|
408
|
+
}
|
|
409
|
+
//#endregion
|
|
410
|
+
//#region src/adapters/local/transcriber.ts
|
|
411
|
+
/** Small multilingual Whisper (~80 MB at the default q8 quantisation). */
|
|
412
|
+
var DEFAULT_LOCAL_MODEL = "onnx-community/whisper-base";
|
|
413
|
+
var DEFAULT_CHUNK_LENGTH_SECONDS = 30;
|
|
414
|
+
var WHISPER_WINDOW_SECONDS = 30;
|
|
415
|
+
var WHISPER_MODEL_TYPES = /* @__PURE__ */ new Set(["whisper", "lite-whisper"]);
|
|
416
|
+
/**
|
|
417
|
+
* On-device speech-to-text with transformers.js (Whisper or Moonshine ONNX).
|
|
418
|
+
* Runs on WebGPU/WASM in browsers and onnxruntime-node in Node; no audio
|
|
419
|
+
* leaves the device and no API key is involved.
|
|
420
|
+
*/
|
|
421
|
+
var LocalTranscriber = class {
|
|
422
|
+
type = "local";
|
|
423
|
+
options;
|
|
424
|
+
model;
|
|
425
|
+
maxBytes;
|
|
426
|
+
onUsage;
|
|
427
|
+
runtime;
|
|
428
|
+
pipelines = /* @__PURE__ */ new Map();
|
|
429
|
+
/** Serialises inference: ONNX sessions (especially WebGPU) run one call at a time. */
|
|
430
|
+
queue = Promise.resolve();
|
|
431
|
+
constructor(options = {}) {
|
|
432
|
+
this.options = options;
|
|
433
|
+
this.model = options.model?.trim() || "onnx-community/whisper-base";
|
|
434
|
+
this.maxBytes = options.maxBytes ?? 26214400;
|
|
435
|
+
this.onUsage = options.onUsage;
|
|
436
|
+
}
|
|
437
|
+
/**
|
|
438
|
+
* Downloads (first time) and initialises the model so the first
|
|
439
|
+
* `transcribe()` is fast. Progress goes to `onProgress`.
|
|
440
|
+
*/
|
|
441
|
+
async preload(model, signal) {
|
|
442
|
+
await raceAbort(this.loadPipeline(model?.trim() || this.model), signal);
|
|
443
|
+
}
|
|
444
|
+
async transcribe(request) {
|
|
445
|
+
const { signal } = request;
|
|
446
|
+
const audio = await normalizeAudioInput(request.audio, {
|
|
447
|
+
mimeType: request.mimeType,
|
|
448
|
+
maxBytes: request.maxBytes ?? this.maxBytes,
|
|
449
|
+
adapter: this.type,
|
|
450
|
+
signal
|
|
451
|
+
});
|
|
452
|
+
const format = pcmFormat(request.audio);
|
|
453
|
+
const samples = await decodeToPcm16k(new Uint8Array(await audio.blob.arrayBuffer()), {
|
|
454
|
+
mimeType: audio.mimeType,
|
|
455
|
+
sampleRate: format.sampleRate,
|
|
456
|
+
channels: format.channels,
|
|
457
|
+
decodeAudio: this.options.decodeAudio,
|
|
458
|
+
signal
|
|
459
|
+
});
|
|
460
|
+
signal?.throwIfAborted();
|
|
461
|
+
const model = request.model?.trim() || this.model;
|
|
462
|
+
const loaded = await raceAbort(this.loadPipeline(model), signal);
|
|
463
|
+
const audioSeconds = samples.length / LOCAL_SAMPLE_RATE;
|
|
464
|
+
const output = await raceAbort(this.enqueue(() => this.run(loaded.pipeline, samples, audioSeconds, request)), signal);
|
|
465
|
+
const usage = {
|
|
466
|
+
operation: "transcription",
|
|
467
|
+
provider: this.type,
|
|
468
|
+
model,
|
|
469
|
+
audioSeconds,
|
|
470
|
+
bytes: audio.bytes
|
|
471
|
+
};
|
|
472
|
+
const result = {
|
|
473
|
+
...mapAsrOutput(output, request.timestampGranularities?.includes("word") ?? false),
|
|
474
|
+
language: request.language,
|
|
475
|
+
durationSeconds: audioSeconds,
|
|
476
|
+
provider: this.type,
|
|
477
|
+
model,
|
|
478
|
+
raw: output,
|
|
479
|
+
usage
|
|
480
|
+
};
|
|
481
|
+
await reportSpeechUsage(usage, this.onUsage, request.onUsage);
|
|
482
|
+
return result;
|
|
483
|
+
}
|
|
484
|
+
/** Waits for in-flight inference, then releases every loaded model session. */
|
|
485
|
+
async dispose() {
|
|
486
|
+
await this.queue;
|
|
487
|
+
const loaded = [...this.pipelines.values()];
|
|
488
|
+
this.pipelines.clear();
|
|
489
|
+
await Promise.all(loaded.map(async (entry) => {
|
|
490
|
+
const { pipeline } = await entry.catch(() => ({ pipeline: void 0 }));
|
|
491
|
+
await pipeline?.dispose?.();
|
|
492
|
+
}));
|
|
493
|
+
}
|
|
494
|
+
async run(pipeline, samples, audioSeconds, request) {
|
|
495
|
+
const { signal } = request;
|
|
496
|
+
signal?.throwIfAborted();
|
|
497
|
+
const transformers = await this.loadRuntime();
|
|
498
|
+
const options = {};
|
|
499
|
+
if (WHISPER_MODEL_TYPES.has(pipeline.model?.config?.model_type ?? "whisper")) {
|
|
500
|
+
const granularities = request.timestampGranularities ?? [];
|
|
501
|
+
if (granularities.includes("word")) options.return_timestamps = "word";
|
|
502
|
+
else if (granularities.includes("segment")) options.return_timestamps = true;
|
|
503
|
+
const chunkLength = this.options.chunkLengthSeconds ?? DEFAULT_CHUNK_LENGTH_SECONDS;
|
|
504
|
+
if (chunkLength > 0 && audioSeconds > WHISPER_WINDOW_SECONDS) options.chunk_length_s = chunkLength;
|
|
505
|
+
if (request.language) {
|
|
506
|
+
options.language = request.language;
|
|
507
|
+
options.task = "transcribe";
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
let stopping;
|
|
511
|
+
if (signal && transformers.InterruptableStoppingCriteria) {
|
|
512
|
+
stopping = new transformers.InterruptableStoppingCriteria();
|
|
513
|
+
options.stopping_criteria = stopping;
|
|
514
|
+
}
|
|
515
|
+
const onAbort = () => stopping?.interrupt();
|
|
516
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
517
|
+
try {
|
|
518
|
+
const output = await pipeline(samples, options);
|
|
519
|
+
return Array.isArray(output) ? output[0] ?? { text: "" } : output;
|
|
520
|
+
} catch (error) {
|
|
521
|
+
if (signal?.aborted) throw signal.reason;
|
|
522
|
+
const hint = options.return_timestamps === "word" && /cross attentions/i.test(errorMessage(error)) ? " (word timestamps need a model exported with cross attentions, e.g. onnx-community/whisper-base_timestamped)" : "";
|
|
523
|
+
throw new SpeechProviderError(this.type, `Local transcription failed: ${errorMessage(error)}${hint}`, { cause: error });
|
|
524
|
+
} finally {
|
|
525
|
+
signal?.removeEventListener("abort", onAbort);
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
enqueue(task) {
|
|
529
|
+
const next = this.queue.then(task, task);
|
|
530
|
+
this.queue = next.catch(() => void 0);
|
|
531
|
+
return next;
|
|
532
|
+
}
|
|
533
|
+
loadRuntime() {
|
|
534
|
+
if (!this.runtime) {
|
|
535
|
+
const { transformers, configureEnv, modelHost } = this.options;
|
|
536
|
+
this.runtime = loadTransformers(transformers).then((module) => {
|
|
537
|
+
if (module.env) {
|
|
538
|
+
if (modelHost) module.env.remoteHost = modelHost;
|
|
539
|
+
configureEnv?.(module.env);
|
|
540
|
+
}
|
|
541
|
+
return module;
|
|
542
|
+
});
|
|
543
|
+
this.runtime.catch(() => {
|
|
544
|
+
this.runtime = void 0;
|
|
545
|
+
});
|
|
546
|
+
}
|
|
547
|
+
return this.runtime;
|
|
548
|
+
}
|
|
549
|
+
loadPipeline(model) {
|
|
550
|
+
let loaded = this.pipelines.get(model);
|
|
551
|
+
if (!loaded) {
|
|
552
|
+
loaded = this.loadRuntime().then((transformers) => createAsrPipeline(transformers, {
|
|
553
|
+
model,
|
|
554
|
+
device: this.options.device ?? "auto",
|
|
555
|
+
dtype: this.options.dtype,
|
|
556
|
+
revision: this.options.revision,
|
|
557
|
+
cacheDir: this.options.cacheDir,
|
|
558
|
+
onProgress: this.options.onProgress
|
|
559
|
+
}).catch((error) => {
|
|
560
|
+
if (error instanceof SpeechConfigurationError) throw error;
|
|
561
|
+
throw new SpeechProviderError(this.type, `Failed to load local model '${model}': ${errorMessage(error)}`, { cause: error });
|
|
562
|
+
}));
|
|
563
|
+
this.pipelines.set(model, loaded);
|
|
564
|
+
loaded.catch(() => {
|
|
565
|
+
if (this.pipelines.get(model) === loaded) this.pipelines.delete(model);
|
|
566
|
+
});
|
|
567
|
+
}
|
|
568
|
+
return loaded;
|
|
569
|
+
}
|
|
570
|
+
};
|
|
571
|
+
/**
|
|
572
|
+
* Maps transformers.js chunks onto `words` (when word timestamps were
|
|
573
|
+
* requested) or `segments`.
|
|
574
|
+
*/
|
|
575
|
+
function mapAsrOutput(output, wordLevel) {
|
|
576
|
+
const result = { text: (output.text ?? "").trim() };
|
|
577
|
+
const chunks = output.chunks;
|
|
578
|
+
if (!chunks?.length) return result;
|
|
579
|
+
if (wordLevel) result.words = chunks.filter((chunk) => chunk.text.trim()).map((chunk) => ({
|
|
580
|
+
word: chunk.text.trim(),
|
|
581
|
+
startSeconds: chunk.timestamp[0],
|
|
582
|
+
endSeconds: chunk.timestamp[1] ?? chunk.timestamp[0]
|
|
583
|
+
}));
|
|
584
|
+
else result.segments = chunks.map((chunk) => ({
|
|
585
|
+
text: chunk.text.trim(),
|
|
586
|
+
startSeconds: chunk.timestamp[0],
|
|
587
|
+
endSeconds: chunk.timestamp[1] ?? void 0
|
|
588
|
+
}));
|
|
589
|
+
return result;
|
|
590
|
+
}
|
|
591
|
+
function pcmFormat(audio) {
|
|
592
|
+
if (audio && typeof audio === "object" && "data" in audio && !(audio instanceof Uint8Array)) return {
|
|
593
|
+
sampleRate: audio.sampleRate,
|
|
594
|
+
channels: audio.channels
|
|
595
|
+
};
|
|
596
|
+
return {};
|
|
597
|
+
}
|
|
598
|
+
function errorMessage(error) {
|
|
599
|
+
return error instanceof Error ? error.message : String(error);
|
|
600
|
+
}
|
|
601
|
+
//#endregion
|
|
602
|
+
//#region src/adapters/local/worker.ts
|
|
603
|
+
/**
|
|
604
|
+
* Web Worker support for the `local` transcriber.
|
|
605
|
+
*
|
|
606
|
+
* Inference runs in a worker (`serveLocalTranscriber`) so long recordings never
|
|
607
|
+
* block the UI thread. The main-thread client (`LocalTranscriberWorkerClient`)
|
|
608
|
+
* transfers WAV and raw PCM bytes to the worker, which parses, downmixes, and
|
|
609
|
+
* resamples them there. Only formats that need `AudioContext` (which workers
|
|
610
|
+
* lack) or the client's `decodeAudio` hook are decoded on the main thread;
|
|
611
|
+
* their Float32 channels are then transferred and resampled in the worker.
|
|
612
|
+
*/
|
|
613
|
+
var PCM_MIME_TYPE = `audio/pcm;rate=${LOCAL_SAMPLE_RATE};encoding=f32le`;
|
|
614
|
+
/**
|
|
615
|
+
* Runs a {@link LocalTranscriber} inside a worker and answers client
|
|
616
|
+
* requests. Call it from your worker module:
|
|
617
|
+
*
|
|
618
|
+
* ```ts
|
|
619
|
+
* // transcriber.worker.ts
|
|
620
|
+
* import { serveLocalTranscriber } from '@happyvertical/speech/local';
|
|
621
|
+
* serveLocalTranscriber({ model: 'onnx-community/whisper-base' });
|
|
622
|
+
* ```
|
|
623
|
+
*
|
|
624
|
+
* Returns a function that stops listening and disposes the model.
|
|
625
|
+
*/
|
|
626
|
+
function serveLocalTranscriber(options = {}, endpoint = globalThis) {
|
|
627
|
+
const userProgress = options.onProgress;
|
|
628
|
+
const transcriber = new LocalTranscriber({
|
|
629
|
+
...resolveLocalTranscriberOptions(options),
|
|
630
|
+
maxBytes: Number.POSITIVE_INFINITY,
|
|
631
|
+
onProgress: (progress) => {
|
|
632
|
+
userProgress?.(progress);
|
|
633
|
+
endpoint.postMessage({
|
|
634
|
+
kind: "progress",
|
|
635
|
+
progress: toCloneable(progress)
|
|
636
|
+
});
|
|
637
|
+
}
|
|
638
|
+
});
|
|
639
|
+
const pending = /* @__PURE__ */ new Map();
|
|
640
|
+
const reply = (message) => endpoint.postMessage(message);
|
|
641
|
+
const listener = (event) => {
|
|
642
|
+
const message = event.data;
|
|
643
|
+
if (!message || typeof message !== "object") return;
|
|
644
|
+
if (message.kind === "abort") {
|
|
645
|
+
pending.get(message.id)?.abort();
|
|
646
|
+
return;
|
|
647
|
+
}
|
|
648
|
+
const controller = new AbortController();
|
|
649
|
+
pending.set(message.id, controller);
|
|
650
|
+
(message.kind === "preload" ? transcriber.preload(message.model, controller.signal).then(() => {
|
|
651
|
+
reply({
|
|
652
|
+
kind: "result",
|
|
653
|
+
id: message.id
|
|
654
|
+
});
|
|
655
|
+
}) : Promise.resolve().then(() => transcriber.transcribe({
|
|
656
|
+
audio: toAudioInput(message.audio),
|
|
657
|
+
model: message.model,
|
|
658
|
+
language: message.language,
|
|
659
|
+
timestampGranularities: message.timestampGranularities,
|
|
660
|
+
signal: controller.signal
|
|
661
|
+
})).then((result) => {
|
|
662
|
+
reply({
|
|
663
|
+
kind: "result",
|
|
664
|
+
id: message.id,
|
|
665
|
+
result
|
|
666
|
+
});
|
|
667
|
+
})).catch((error) => {
|
|
668
|
+
reply({
|
|
669
|
+
kind: "error",
|
|
670
|
+
id: message.id,
|
|
671
|
+
error: serializeError(error)
|
|
672
|
+
});
|
|
673
|
+
}).finally(() => pending.delete(message.id));
|
|
674
|
+
};
|
|
675
|
+
endpoint.addEventListener("message", listener);
|
|
676
|
+
return async () => {
|
|
677
|
+
endpoint.removeEventListener("message", listener);
|
|
678
|
+
for (const controller of pending.values()) controller.abort();
|
|
679
|
+
await transcriber.dispose();
|
|
680
|
+
};
|
|
681
|
+
}
|
|
682
|
+
/**
|
|
683
|
+
* Main-thread {@link Transcriber} that delegates inference to a worker running
|
|
684
|
+
* {@link serveLocalTranscriber}.
|
|
685
|
+
*
|
|
686
|
+
* ```ts
|
|
687
|
+
* const worker = new Worker(new URL('./transcriber.worker.ts', import.meta.url), { type: 'module' });
|
|
688
|
+
* const transcriber = new LocalTranscriberWorkerClient(worker, { onProgress });
|
|
689
|
+
* const { text } = await transcriber.transcribe({ audio: recordingBlob });
|
|
690
|
+
* ```
|
|
691
|
+
*/
|
|
692
|
+
var LocalTranscriberWorkerClient = class {
|
|
693
|
+
type = "local";
|
|
694
|
+
endpoint;
|
|
695
|
+
options;
|
|
696
|
+
calls = /* @__PURE__ */ new Map();
|
|
697
|
+
nextId = 1;
|
|
698
|
+
closed = false;
|
|
699
|
+
/** Aborted by `close()` to reject calls still buffering or decoding. */
|
|
700
|
+
closing = new AbortController();
|
|
701
|
+
listener = (event) => this.onMessage(event.data);
|
|
702
|
+
constructor(endpoint, options = {}) {
|
|
703
|
+
this.endpoint = endpoint;
|
|
704
|
+
this.options = options;
|
|
705
|
+
endpoint.addEventListener("message", this.listener);
|
|
706
|
+
}
|
|
707
|
+
/** Loads the model in the worker ahead of the first transcription. */
|
|
708
|
+
async preload(model, signal) {
|
|
709
|
+
await this.call({
|
|
710
|
+
kind: "preload",
|
|
711
|
+
model
|
|
712
|
+
}, signal);
|
|
713
|
+
}
|
|
714
|
+
async transcribe(request) {
|
|
715
|
+
this.assertOpen();
|
|
716
|
+
const { signal } = request;
|
|
717
|
+
const { audio, bytes } = await raceAbort(this.prepare(request), this.closing.signal);
|
|
718
|
+
const result = await this.call({
|
|
719
|
+
kind: "transcribe",
|
|
720
|
+
audio,
|
|
721
|
+
model: request.model,
|
|
722
|
+
language: request.language,
|
|
723
|
+
timestampGranularities: request.timestampGranularities
|
|
724
|
+
}, signal, transferList(audio));
|
|
725
|
+
if (!result) throw new SpeechProviderError(this.type, "Worker returned no result");
|
|
726
|
+
const usage = {
|
|
727
|
+
...result.usage ?? {
|
|
728
|
+
operation: "transcription",
|
|
729
|
+
provider: this.type
|
|
730
|
+
},
|
|
731
|
+
bytes
|
|
732
|
+
};
|
|
733
|
+
await reportSpeechUsage(usage, this.options.onUsage, request.onUsage);
|
|
734
|
+
return {
|
|
735
|
+
...result,
|
|
736
|
+
usage
|
|
737
|
+
};
|
|
738
|
+
}
|
|
739
|
+
/**
|
|
740
|
+
* Stops listening, rejects in-flight and later calls, and asks the worker to
|
|
741
|
+
* abort posted work. Does not terminate the worker.
|
|
742
|
+
*/
|
|
743
|
+
close() {
|
|
744
|
+
if (this.closed) return;
|
|
745
|
+
this.closed = true;
|
|
746
|
+
this.endpoint.removeEventListener("message", this.listener);
|
|
747
|
+
this.closing.abort(closedError());
|
|
748
|
+
for (const [id, call] of this.calls) {
|
|
749
|
+
this.endpoint.postMessage({
|
|
750
|
+
kind: "abort",
|
|
751
|
+
id
|
|
752
|
+
});
|
|
753
|
+
call.reject(closedError());
|
|
754
|
+
}
|
|
755
|
+
this.calls.clear();
|
|
756
|
+
}
|
|
757
|
+
/** Buffers the input and, only when the worker cannot, decodes it here. */
|
|
758
|
+
async prepare(request) {
|
|
759
|
+
const { signal } = request;
|
|
760
|
+
const normalized = await normalizeAudioInput(request.audio, {
|
|
761
|
+
mimeType: request.mimeType,
|
|
762
|
+
maxBytes: request.maxBytes ?? this.options.maxBytes ?? 26214400,
|
|
763
|
+
adapter: this.type,
|
|
764
|
+
signal
|
|
765
|
+
});
|
|
766
|
+
const bytes = new Uint8Array(await normalized.blob.arrayBuffer());
|
|
767
|
+
if (!isInProcessDecodable(bytes, normalized.mimeType)) {
|
|
768
|
+
const decoded = await raceAbort(decodeExternally(bytes, {
|
|
769
|
+
mimeType: normalized.mimeType,
|
|
770
|
+
decodeAudio: this.options.decodeAudio,
|
|
771
|
+
signal
|
|
772
|
+
}), signal);
|
|
773
|
+
if (decoded) return {
|
|
774
|
+
audio: {
|
|
775
|
+
kind: "decoded",
|
|
776
|
+
samples: (decoded.samples instanceof Float32Array ? [decoded.samples] : decoded.samples).map(ownedCopy),
|
|
777
|
+
sampleRate: decoded.sampleRate
|
|
778
|
+
},
|
|
779
|
+
bytes: normalized.bytes
|
|
780
|
+
};
|
|
781
|
+
}
|
|
782
|
+
const wrapped = typeof request.audio === "object" && "data" in request.audio && !(request.audio instanceof Uint8Array) ? request.audio : void 0;
|
|
783
|
+
return {
|
|
784
|
+
audio: {
|
|
785
|
+
kind: "encoded",
|
|
786
|
+
bytes: ownedCopy(bytes),
|
|
787
|
+
mimeType: normalized.mimeType,
|
|
788
|
+
sampleRate: wrapped?.sampleRate,
|
|
789
|
+
channels: wrapped?.channels
|
|
790
|
+
},
|
|
791
|
+
bytes: normalized.bytes
|
|
792
|
+
};
|
|
793
|
+
}
|
|
794
|
+
assertOpen() {
|
|
795
|
+
if (this.closed) throw closedError();
|
|
796
|
+
}
|
|
797
|
+
call(message, signal, transfer = []) {
|
|
798
|
+
this.assertOpen();
|
|
799
|
+
signal?.throwIfAborted();
|
|
800
|
+
const id = this.nextId++;
|
|
801
|
+
return new Promise((resolve, reject) => {
|
|
802
|
+
const onAbort = () => {
|
|
803
|
+
this.calls.delete(id);
|
|
804
|
+
this.endpoint.postMessage({
|
|
805
|
+
kind: "abort",
|
|
806
|
+
id
|
|
807
|
+
});
|
|
808
|
+
reject(signal?.reason);
|
|
809
|
+
};
|
|
810
|
+
const settle = () => signal?.removeEventListener("abort", onAbort);
|
|
811
|
+
this.calls.set(id, {
|
|
812
|
+
resolve: (result) => {
|
|
813
|
+
settle();
|
|
814
|
+
resolve(result);
|
|
815
|
+
},
|
|
816
|
+
reject: (error) => {
|
|
817
|
+
settle();
|
|
818
|
+
reject(error);
|
|
819
|
+
}
|
|
820
|
+
});
|
|
821
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
822
|
+
this.endpoint.postMessage({
|
|
823
|
+
...message,
|
|
824
|
+
id
|
|
825
|
+
}, transfer);
|
|
826
|
+
});
|
|
827
|
+
}
|
|
828
|
+
onMessage(message) {
|
|
829
|
+
if (!message || typeof message !== "object") return;
|
|
830
|
+
if (message.kind === "progress") {
|
|
831
|
+
this.options.onProgress?.(message.progress);
|
|
832
|
+
return;
|
|
833
|
+
}
|
|
834
|
+
const call = this.calls.get(message.id);
|
|
835
|
+
if (!call) return;
|
|
836
|
+
this.calls.delete(message.id);
|
|
837
|
+
if (message.kind === "result") call.resolve(message.result);
|
|
838
|
+
else call.reject(deserializeError(message.error));
|
|
839
|
+
}
|
|
840
|
+
};
|
|
841
|
+
function closedError() {
|
|
842
|
+
return new SpeechConfigurationError("Worker client closed", "local");
|
|
843
|
+
}
|
|
844
|
+
/** Rebuilds the transcriber input from a worker message, resampling decoded channels here. */
|
|
845
|
+
function toAudioInput(audio) {
|
|
846
|
+
if (audio.kind === "decoded") {
|
|
847
|
+
const pcm = toPcm16k({
|
|
848
|
+
samples: audio.samples,
|
|
849
|
+
sampleRate: audio.sampleRate
|
|
850
|
+
});
|
|
851
|
+
return {
|
|
852
|
+
data: new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength),
|
|
853
|
+
mimeType: PCM_MIME_TYPE
|
|
854
|
+
};
|
|
855
|
+
}
|
|
856
|
+
return {
|
|
857
|
+
data: audio.bytes,
|
|
858
|
+
mimeType: audio.mimeType,
|
|
859
|
+
sampleRate: audio.sampleRate,
|
|
860
|
+
channels: audio.channels
|
|
861
|
+
};
|
|
862
|
+
}
|
|
863
|
+
/**
|
|
864
|
+
* Returns `view` when it spans a whole ordinary `ArrayBuffer` (safe to
|
|
865
|
+
* transfer), otherwise a copy. Views of a larger buffer would transfer
|
|
866
|
+
* unrelated bytes, and a `SharedArrayBuffer` cannot be transferred at all.
|
|
867
|
+
*/
|
|
868
|
+
function ownedCopy(view) {
|
|
869
|
+
const { buffer } = view;
|
|
870
|
+
return isPlainArrayBuffer(buffer) && view.byteOffset === 0 && view.byteLength === buffer.byteLength ? view : view.slice();
|
|
871
|
+
}
|
|
872
|
+
function isPlainArrayBuffer(buffer) {
|
|
873
|
+
return Object.prototype.toString.call(buffer) === "[object ArrayBuffer]";
|
|
874
|
+
}
|
|
875
|
+
/** Distinct buffers to transfer (two channels may share one). */
|
|
876
|
+
function transferList(audio) {
|
|
877
|
+
const views = audio.kind === "decoded" ? audio.samples : [audio.bytes];
|
|
878
|
+
return [...new Set(views.map((view) => view.buffer))];
|
|
879
|
+
}
|
|
880
|
+
function serializeError(error) {
|
|
881
|
+
if (error instanceof SpeechError) return {
|
|
882
|
+
name: error.name,
|
|
883
|
+
message: error.message,
|
|
884
|
+
code: error.code
|
|
885
|
+
};
|
|
886
|
+
if (error instanceof Error) return {
|
|
887
|
+
name: error.name,
|
|
888
|
+
message: error.message
|
|
889
|
+
};
|
|
890
|
+
return {
|
|
891
|
+
name: "Error",
|
|
892
|
+
message: String(error)
|
|
893
|
+
};
|
|
894
|
+
}
|
|
895
|
+
function deserializeError(error) {
|
|
896
|
+
switch (error.name) {
|
|
897
|
+
case "SpeechConfigurationError": return new SpeechConfigurationError(error.message, "local");
|
|
898
|
+
case "SpeechProviderError": return new SpeechProviderError("local", error.message);
|
|
899
|
+
case "AbortError": return new DOMException(error.message, "AbortError");
|
|
900
|
+
default: {
|
|
901
|
+
const restored = new Error(error.message);
|
|
902
|
+
restored.name = error.name;
|
|
903
|
+
return restored;
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
/** Progress objects cross `postMessage`, so drop anything not cloneable. */
|
|
908
|
+
function toCloneable(progress) {
|
|
909
|
+
const clone = { status: progress.status };
|
|
910
|
+
for (const [key, value] of Object.entries(progress)) if (value === null || [
|
|
911
|
+
"string",
|
|
912
|
+
"number",
|
|
913
|
+
"boolean"
|
|
914
|
+
].includes(typeof value)) clone[key] = value;
|
|
915
|
+
return clone;
|
|
916
|
+
}
|
|
917
|
+
//#endregion
|
|
918
|
+
//#region src/local.ts
|
|
919
|
+
/**
|
|
920
|
+
* @happyvertical/speech/local
|
|
921
|
+
*
|
|
922
|
+
* On-device speech-to-text with transformers.js (`@huggingface/transformers`,
|
|
923
|
+
* an optional peer dependency loaded on first use). Importing this entry
|
|
924
|
+
* registers `type: 'local'` with `getTranscriber()` and `getSpeech()`.
|
|
925
|
+
*
|
|
926
|
+
* @example
|
|
927
|
+
* ```typescript
|
|
928
|
+
* import { getTranscriber } from '@happyvertical/speech';
|
|
929
|
+
* import '@happyvertical/speech/local';
|
|
930
|
+
*
|
|
931
|
+
* const transcriber = await getTranscriber({ type: 'local' });
|
|
932
|
+
* const { text } = await transcriber.transcribe({ audio: wavBytes, mimeType: 'audio/wav' });
|
|
933
|
+
* ```
|
|
934
|
+
*
|
|
935
|
+
* @packageDocumentation
|
|
936
|
+
*/
|
|
937
|
+
/**
|
|
938
|
+
* Creates an on-device transcriber. Explicit options win over
|
|
939
|
+
* `HAVE_SPEECH_TRANSCRIBER_*` environment variables (see
|
|
940
|
+
* `LOCAL_TRANSCRIBER_ENV_KEYS`). The model loads on first use.
|
|
941
|
+
*/
|
|
942
|
+
function createLocalTranscriber(options = {}, context = {}) {
|
|
943
|
+
return new LocalTranscriber(resolveLocalTranscriberOptions(options, context));
|
|
944
|
+
}
|
|
945
|
+
/**
|
|
946
|
+
* Resolves `true` when `@huggingface/transformers` can be imported in this
|
|
947
|
+
* runtime. This imports the runtime, so call it only when you intend to use it.
|
|
948
|
+
*/
|
|
949
|
+
async function isLocalTranscriberAvailable() {
|
|
950
|
+
try {
|
|
951
|
+
await loadTransformers(void 0);
|
|
952
|
+
return true;
|
|
953
|
+
} catch {
|
|
954
|
+
return false;
|
|
955
|
+
}
|
|
956
|
+
}
|
|
957
|
+
registerOptionalTranscriber("local", (options, context) => createLocalTranscriber({
|
|
958
|
+
...options,
|
|
959
|
+
type: "local"
|
|
960
|
+
}, context));
|
|
961
|
+
//#endregion
|
|
962
|
+
export { DEFAULT_LOCAL_MODEL, LOCAL_SAMPLE_RATE, LOCAL_TRANSCRIBER_ENV_KEYS, LocalTranscriber, LocalTranscriberWorkerClient, TRANSFORMERS_INSTALL_HINT, TRANSFORMERS_PACKAGE, createLocalTranscriber, decodeToPcm16k, isLocalTranscriberAvailable, serveLocalTranscriber };
|
|
963
|
+
|
|
964
|
+
//# sourceMappingURL=local.js.map
|