@mcut/transcription-local 0.1.0-alpha.0

package/dist/index.js

@@ -0,0 +1,357 @@
+//#region src/wav.ts
+/**
+* Minimal WAV reader → 16kHz mono Float32Array (Whisper's input format).
+* The editor extracts clip audio as 16kHz WAV already (`extractAudioToWav`),
+* so this covers the hot path without any AudioContext — it runs in workers
+* and tests alike. Non-WAV input falls back to decodeAudioData upstream.
+*/
+const WHISPER_SAMPLE_RATE = 16e3;
+/** Parse a PCM/float WAV file. Returns null when it isn't one. */
+function parseWav(buffer) {
+	const view = new DataView(buffer);
+	if (buffer.byteLength < 44) return null;
+	if (view.getUint32(0) !== 1380533830) return null;
+	if (view.getUint32(8) !== 1463899717) return null;
+	let offset = 12;
+	let format = null;
+	while (offset + 8 <= buffer.byteLength) {
+		const id = view.getUint32(offset);
+		const size = view.getUint32(offset + 4, true);
+		const body = offset + 8;
+		if (id === 1718449184) format = {
+			audioFormat: view.getUint16(body, true),
+			channels: view.getUint16(body + 2, true),
+			sampleRate: view.getUint32(body + 4, true),
+			bitsPerSample: view.getUint16(body + 14, true)
+		};
+		else if (id === 1684108385 && format) return decodeData(view, body, Math.min(buffer.byteLength, body + size), format);
+		offset = body + size + size % 2;
+	}
+	return null;
+}
+function decodeData(view, start, end, format) {
+	const { audioFormat, channels, sampleRate, bitsPerSample } = format;
+	if (channels < 1 || sampleRate <= 0) return null;
+	const bytesPerSample = bitsPerSample / 8;
+	const frameBytes = bytesPerSample * channels;
+	const frames = Math.floor((end - start) / frameBytes);
+	const samples = new Float32Array(frames);
+	const read = (at) => {
+		if (audioFormat === 3 && bitsPerSample === 32) return view.getFloat32(at, true);
+		if (audioFormat === 1 && bitsPerSample === 16) return view.getInt16(at, true) / 32768;
+		if (audioFormat === 1 && bitsPerSample === 32) return view.getInt32(at, true) / 2147483648;
+		if (audioFormat === 1 && bitsPerSample === 8) return (view.getUint8(at) - 128) / 128;
+		if (audioFormat === 1 && bitsPerSample === 24) return (view.getUint8(at) | view.getUint8(at + 1) << 8 | view.getInt8(at + 2) << 16) / 8388608;
+		return NaN;
+	};
+	if (frames > 0 && Number.isNaN(read(start))) return null;
+	for (let frame = 0; frame < frames; frame++) {
+		const at = start + frame * frameBytes;
+		let sum = 0;
+		for (let c = 0; c < channels; c++) sum += read(at + c * bytesPerSample);
+		samples[frame] = sum / channels;
+	}
+	return {
+		samples,
+		sampleRate
+	};
+}
+/** Linear-interpolation resample (fine for speech models). */
+function resampleTo(audio, targetRate) {
+	if (audio.sampleRate === targetRate) return audio.samples;
+	const ratio = audio.sampleRate / targetRate;
+	const length = Math.max(1, Math.round(audio.samples.length / ratio));
+	const out = new Float32Array(length);
+	for (let i = 0; i < length; i++) {
+		const position = i * ratio;
+		const index = Math.floor(position);
+		const fraction = position - index;
+		const a = audio.samples[Math.min(index, audio.samples.length - 1)];
+		out[i] = a + (audio.samples[Math.min(index + 1, audio.samples.length - 1)] - a) * fraction;
+	}
+	return out;
+}
+//#endregion
+//#region src/chunking.ts
+const CHUNK_WINDOW_S = 30;
+const CHUNK_OVERLAP_S = 5;
+/** Split a duration into overlapping windows (last window may be shorter). */
+function planChunks(durationS, windowS = 30, overlapS = 5) {
+	if (durationS <= 0) return [];
+	if (durationS <= windowS) return [{
+		startS: 0,
+		endS: durationS
+	}];
+	const step = windowS - overlapS;
+	const chunks = [];
+	for (let start = 0; start < durationS - overlapS; start += step) chunks.push({
+		startS: start,
+		endS: Math.min(durationS, start + windowS)
+	});
+	return chunks;
+}
+/**
+* Merge consecutive chunk transcripts on word timestamps rather than
+* concatenating: inside each overlap the cut lands on the largest silence
+* between the incoming chunk's words (falling back to the overlap midpoint),
+* the outgoing chunk keeps words before the cut, the incoming one after.
+* This absorbs the timestamp drift Whisper accumulates near window edges.
+*/
+function mergeChunkWords(results) {
+	const present = results.filter((r) => r.words.length > 0);
+	if (present.length === 0) return [];
+	let merged = [...present[0].words];
+	for (let i = 1; i < present.length; i++) {
+		const next = present[i];
+		const overlapStartMs = next.chunk.startS * 1e3;
+		const prevEndMs = present[i - 1].chunk.endS * 1e3;
+		const overlapEndMs = Math.min(prevEndMs, next.chunk.endS * 1e3);
+		if (overlapEndMs <= overlapStartMs) {
+			merged = [...merged, ...next.words];
+			continue;
+		}
+		const cutMs = pickCut(next.words, overlapStartMs, overlapEndMs);
+		merged = [...merged.filter((w) => w.startMs < cutMs), ...next.words.filter((w) => w.startMs >= cutMs)];
+	}
+	return merged.sort((a, b) => a.startMs - b.startMs);
+}
+/** Same overlap-cut strategy as {@link mergeChunkWords}, for segment-timed models. */
+function mergeChunkSegments(results) {
+	const present = results.filter((r) => r.segments.length > 0);
+	if (present.length === 0) return [];
+	let merged = [...present[0].segments];
+	for (let i = 1; i < present.length; i++) {
+		const next = present[i];
+		const overlapStartMs = next.chunk.startS * 1e3;
+		const prevEndMs = present[i - 1].chunk.endS * 1e3;
+		const overlapEndMs = Math.min(prevEndMs, next.chunk.endS * 1e3);
+		if (overlapEndMs <= overlapStartMs) {
+			merged = [...merged, ...next.segments];
+			continue;
+		}
+		const cutMs = pickCut(next.segments, overlapStartMs, overlapEndMs);
+		merged = [...merged.filter((s) => s.startMs < cutMs), ...next.segments.filter((s) => s.startMs >= cutMs)];
+	}
+	return merged.sort((a, b) => a.startMs - b.startMs);
+}
+/**
+* The middle of the largest inter-word gap inside the overlap, else the
+* overlap midpoint. Cutting mid-gap (not at a word edge) keeps the two
+* chunks' slightly-drifted copies of the same word from both surviving.
+*/
+function pickCut(words, overlapStartMs, overlapEndMs) {
+	let bestGap = 0;
+	let bestCut = (overlapStartMs + overlapEndMs) / 2;
+	const inWindow = words.filter((w) => w.endMs > overlapStartMs && w.startMs < overlapEndMs);
+	for (let i = 1; i < inWindow.length; i++) {
+		const gap = inWindow[i].startMs - inWindow[i - 1].endMs;
+		if (gap > bestGap) {
+			bestGap = gap;
+			bestCut = (inWindow[i - 1].endMs + inWindow[i].startMs) / 2;
+		}
+	}
+	return bestGap >= 120 ? bestCut : (overlapStartMs + overlapEndMs) / 2;
+}
+//#endregion
+//#region src/repetition.ts
+/** True when the token stream ends up looping the same n-gram. */
+function hasRepetitionLoop(tokens, options = {}) {
+	const maxNgram = options.maxNgram ?? 4;
+	const normalized = tokens.map((t) => t.toLowerCase().replace(/[^\p{L}\p{N}']+/gu, "")).filter(Boolean);
+	for (let n = 1; n <= maxNgram; n++) {
+		const needed = options.minRepeats ?? (n === 1 ? 6 : n === 2 ? 4 : 3);
+		if (normalized.length < n * needed) continue;
+		let repeats = 1;
+		for (let i = n; i + n <= normalized.length; i += n) {
+			let same = true;
+			for (let j = 0; j < n; j++) if (normalized[i + j] !== normalized[i + j - n]) {
+				same = false;
+				break;
+			}
+			repeats = same ? repeats + 1 : 1;
+			if (repeats >= needed) return true;
+		}
+	}
+	return false;
+}
+/** Convenience for plain text output. */
+function textHasRepetitionLoop(text, options) {
+	return hasRepetitionLoop(text.split(/\s+/), options);
+}
+//#endregion
+//#region src/vad.ts
+/**
+* Lightweight energy-based voice activity pre-pass: windows that are
+* essentially silent never reach the model, which is where Whisper
+* hallucinates text. Deliberately conservative — it only skips clear
+* silence, anything ambiguous (music, low speech) still gets transcribed.
+* The seam is shaped so a silero-vad WASM pass can replace it later.
+*/
+const FRAME_S = .03;
+function measureActivity(samples, sampleRate) {
+	const frameLength = Math.max(1, Math.round(sampleRate * FRAME_S));
+	let active = 0;
+	let frames = 0;
+	let peak = 0;
+	for (let start = 0; start < samples.length; start += frameLength) {
+		const end = Math.min(samples.length, start + frameLength);
+		let sum = 0;
+		for (let i = start; i < end; i++) sum += samples[i] * samples[i];
+		const rms = Math.sqrt(sum / Math.max(1, end - start));
+		peak = Math.max(peak, rms);
+		if (rms > .004) active++;
+		frames++;
+	}
+	return {
+		activeFraction: frames > 0 ? active / frames : 0,
+		peakRms: peak
+	};
+}
+/** Whether a window plausibly contains any speech at all. */
+function hasSpeech(samples, sampleRate) {
+	const { activeFraction, peakRms } = measureActivity(samples, sampleRate);
+	return peakRms >= .006 && activeFraction >= .01;
+}
+//#endregion
+//#region src/index.ts
+/**
+* On-device Whisper provider (Transformers.js in a dedicated worker).
+* Reliability over flash: capability-gated hard (WebGPU + enough memory),
+* chunked with per-window progress, VAD + repetition guards in the worker.
+* On-device is OFFERED, never forced — keep a server provider as the
+* default and let users opt in (the model is a 40–150MB one-time download,
+* cached by the browser after that).
+*/
+/** Built-in model choices (ONNX community builds of OpenAI Whisper). */
+const WHISPER_MODELS = {
+	/** Multilingual, ~145MB at q8 — the WebGPU default. */
+	base: "onnx-community/whisper-base",
+	/** English-only, ~40MB — the low-memory default. */
+	"tiny.en": "onnx-community/whisper-tiny.en"
+};
+function capabilities() {
+	return typeof navigator === "undefined" ? {} : navigator;
+}
+/**
+* Hard capability gate: WebGPU plus ≥4GB device memory. Browsers that don't
+* report `deviceMemory` (Safari/Firefox) pass on WebGPU alone — the spec
+* caps reported values at 8 anyway.
+*/
+function isLocalTranscriptionSupported() {
+	if (typeof Worker === "undefined") return false;
+	const { gpu, deviceMemory } = capabilities();
+	if (!gpu) return false;
+	return deviceMemory === void 0 || deviceMemory >= 4;
+}
+/** whisper-base on roomy machines, whisper-tiny.en when memory is tight. */
+function pickDefaultModel() {
+	const { deviceMemory } = capabilities();
+	return deviceMemory !== void 0 && deviceMemory < 8 ? WHISPER_MODELS["tiny.en"] : WHISPER_MODELS.base;
+}
+function createLocalWhisperProvider(options = {}) {
+	const model = options.model && options.model in WHISPER_MODELS ? WHISPER_MODELS[options.model] : options.model ?? pickDefaultModel();
+	const device = options.device ?? "webgpu";
+	const dtype = options.dtype ?? "q8";
+	let worker = null;
+	let requestId = 0;
+	const ensureWorker = () => {
+		worker ??= options.createWorker ? options.createWorker() : new Worker(new URL("./whisper-worker.js", import.meta.url), { type: "module" });
+		return worker;
+	};
+	return {
+		id: options.id ?? "whisper-local",
+		async transcribe(input, transcribeOptions) {
+			const signal = transcribeOptions?.signal;
+			signal?.throwIfAborted();
+			const audio = await decodeToWhisperInput(input);
+			signal?.throwIfAborted();
+			const target = ensureWorker();
+			const id = requestId++;
+			return new Promise((resolve, reject) => {
+				const cleanup = () => {
+					target.removeEventListener("message", onMessage);
+					target.removeEventListener("error", onError);
+					signal?.removeEventListener("abort", onAbort);
+				};
+				const onAbort = () => {
+					cleanup();
+					target.terminate();
+					worker = null;
+					reject(signal?.reason ?? new DOMException("Transcription aborted", "AbortError"));
+				};
+				const onError = (event) => {
+					cleanup();
+					worker = null;
+					reject(event.error instanceof Error ? event.error : new Error(event.message || "Whisper worker crashed"));
+				};
+				const onMessage = (event) => {
+					const message = event.data;
+					if (message.type === "progress" && message.id === id) options.onProgress?.({
+						phase: message.phase,
+						progress: message.progress
+					});
+					else if (message.type === "result" && message.id === id) {
+						cleanup();
+						resolve(message.result);
+					} else if (message.type === "error" && message.id === id) {
+						cleanup();
+						reject(new Error(message.message));
+					}
+				};
+				signal?.addEventListener("abort", onAbort, { once: true });
+				target.addEventListener("message", onMessage);
+				target.addEventListener("error", onError);
+				const request = {
+					type: "transcribe",
+					id,
+					config: {
+						model,
+						device,
+						dtype
+					},
+					audio,
+					...transcribeOptions?.language ? { language: transcribeOptions.language } : {}
+				};
+				target.postMessage(request, [audio.buffer]);
+			});
+		}
+	};
+}
+/** Normalize any {@link TranscribeInput} into 16kHz mono PCM. */
+async function decodeToWhisperInput(input) {
+	const buffer = await toArrayBuffer(input.audio);
+	const wav = parseWav(buffer);
+	if (wav) return resampleTo(wav, WHISPER_SAMPLE_RATE);
+	if (typeof AudioContext !== "undefined") {
+		const context = new AudioContext({ sampleRate: WHISPER_SAMPLE_RATE });
+		try {
+			const decoded = await context.decodeAudioData(buffer.slice(0));
+			const mono = new Float32Array(decoded.length);
+			for (let c = 0; c < decoded.numberOfChannels; c++) {
+				const channel = decoded.getChannelData(c);
+				for (let i = 0; i < channel.length; i++) mono[i] += channel[i] / decoded.numberOfChannels;
+			}
+			return resampleTo({
+				samples: mono,
+				sampleRate: decoded.sampleRate
+			}, WHISPER_SAMPLE_RATE);
+		} finally {
+			context.close();
+		}
+	}
+	throw new Error("Unsupported audio: expected WAV (use extractAudioToWav) or a browser context");
+}
+async function toArrayBuffer(audio) {
+	if (typeof audio === "string") {
+		const response = await fetch(audio);
+		if (!response.ok) throw new Error(`Could not fetch audio (${response.status})`);
+		return response.arrayBuffer();
+	}
+	if (audio instanceof Blob) return audio.arrayBuffer();
+	if (audio instanceof ArrayBuffer) return audio;
+	const copy = new Uint8Array(audio.byteLength);
+	copy.set(audio);
+	return copy.buffer;
+}
+//#endregion
+export { CHUNK_OVERLAP_S, CHUNK_WINDOW_S, WHISPER_MODELS, WHISPER_SAMPLE_RATE, createLocalWhisperProvider, hasRepetitionLoop, hasSpeech, isLocalTranscriptionSupported, measureActivity, mergeChunkSegments, mergeChunkWords, parseWav, pickDefaultModel, planChunks, resampleTo, textHasRepetitionLoop };