@glissade/narrate 0.4.4 → 0.5.0-pre.0

package/dist/index.d.ts

@@ -21,6 +21,13 @@ interface NarrationScript {
   gap?: number;
   /** silence before the first segment (s); default 0 */
   leadIn?: number;
+  /**
+   * Word-timing aligner for providers that don't emit word timestamps
+   * (espeak / openai / piper). 'heuristic' (default) estimates from text;
+   * 'vosk' derives real timings from the audio (offline ASR); 'none' leaves
+   * segments word-less. Providers that supply their own words ignore this.
+   */
+  align?: string;
   segments: NarrationSegment[];
 }
 interface TimedWord {

package/dist/providers.d.ts

@@ -36,10 +36,112 @@ declare function espeakProvider(): TtsProvider;
 declare function openaiProvider(opts?: {
   model?: string;
 }): TtsProvider;
+/**
+ * VITS-based local TTS: far more natural than espeak, runs on CPU, fully
+ * offline. Needs a voice MODEL (`.onnx` + sibling `.onnx.json`) — pass its
+ * path as `model`, or per-segment as `voice`. Emits no word timestamps; the
+ * alignment step (below) fills them in.
+ */
+declare function piperProvider(opts?: {
+  model?: string;
+}): TtsProvider;
 declare function providerById(id: string): TtsProvider;
+interface AlignRequest {
+  /** the synthesized RIFF/WAV bytes */
+  wav: Buffer;
+  /** the spoken text (the segment text) */
+  text: string;
+}
+/**
+ * Turns (audio, known text) into per-word timings — the provider-independent
+ * way to get word timestamps. Runs ONLY in the prepare step (heavy work is
+ * fine; it runs once and the result is cached). Same three-member shape as
+ * TtsProvider: `version()` participates in the cache so swapping aligners
+ * re-aligns the CACHED wav without re-synthesizing.
+ */
+interface Aligner {
+  readonly id: string;
+  version(): Promise<string>;
+  align(req: AlignRequest): Promise<{
+    word: string;
+    start: number;
+    end: number;
+  }[]>;
+}
+/**
+ * Distribute words across the clip by estimated spoken length (syllables, not
+ * characters — closer to real timing). Pure, deterministic, zero-dependency:
+ * the always-available floor. Good enough for captions; karaoke on a very
+ * slow/fast word wants a real aligner.
+ */
+declare function heuristicWords(text: string, duration: number): {
+  word: string;
+  start: number;
+  end: number;
+}[];
+declare function heuristicAligner(): Aligner;
+/**
+ * Fill words whose start/end are NaN by linear interpolation between their
+ * known neighbours (edges clamp). Keeps the result monotonic. Used after
+ * mapping when some script words got no timing.
+ */
+declare function interpolateMissing(words: {
+  word: string;
+  start: number;
+  end: number;
+}[]): {
+  word: string;
+  start: number;
+  end: number;
+}[];
+/**
+ * Transfer timed words (from an aligner) onto the script's own word tokens.
+ * Forced aligners return near-identical words; ASR (whisper) can differ
+ * (numbers spelled out, punctuation), so we LCS-align the normalized
+ * sequences and interpolate script words the aligner didn't time. Output
+ * length === script word count, in script order — what `wordBoxes()` indexes
+ * against. If nothing matched, distribute by syllable over the timed span.
+ */
+declare function mapAsrToScript(timed: {
+  word: string;
+  start: number;
+  end: number;
+}[], scriptText: string): {
+  word: string;
+  start: number;
+  end: number;
+}[];
+interface WavMono {
+  /** mono samples in [-1, 1] */
+  samples: Float32Array;
+  sampleRate: number;
+}
+/** Decode a 16-bit PCM RIFF/WAV to mono float samples (channels averaged). */
+declare function decodeWavMono(wav: Buffer): WavMono;
+/** Linear-resample mono float to a 16 kHz int16 LE PCM buffer (Vosk's input). */
+declare function resampleTo16kPcm(input: WavMono): Buffer;
+/**
+ * Word timings via Vosk (alphacephei) — offline, Apache-2.0, ~50 MB model, a
+ * real Node binding (no Python, no Docker, no multi-GB download). `vosk` is an
+ * OPTIONAL peer: install it (`npm i vosk`) and point at a model
+ * (`opts.model` / `VOSK_MODEL`) only if you use this aligner. ASR words are
+ * mapped onto the script tokens by `mapAsrToScript`.
+ */
+declare function voskAligner(opts?: {
+  model?: string;
+}): Aligner;
+/** Resolve an aligner id; 'none' disables alignment (word-less segments). */
+declare function alignerById(id: string): Aligner | null;
 interface SynthesizeOptions {
-  /** override the script's provider */
+  /** override the script's provider (by id) */
   provider?: string;
+  /** override the script's aligner ('heuristic' | 'vosk' | 'none') */
+  aligner?: string;
+  /** a provider INSTANCE — wins over `provider`; the bring-your-own seam
+   *  (e.g. a custom ElevenLabs/Azure TtsProvider) */
+  providerImpl?: TtsProvider;
+  /** an aligner INSTANCE (or null to disable) — wins over `aligner` */
+  alignerImpl?: Aligner | null;
   /** ignore the cache and re-synthesize everything */
   force?: boolean;
 }
@@ -49,6 +151,10 @@ interface SynthesizeResult {
   cacheDir: string;
   synthesized: string[];
   reused: string[];
+  /** segment ids whose words came from the aligner (not the provider) */
+  aligned: string[];
+  /** the aligner id used, or null when alignment was disabled */
+  aligner: string | null;
 }
 declare function cacheKey(seg: {
   text: string;
@@ -64,4 +170,4 @@ declare function synthesizeScript(scriptPath: string, opts?: SynthesizeOptions):
 /** Resolve `<scene>.narration.json` for a scene-module path (or accept the script itself). */
 declare function scriptPathFor(input: string): string;
 //#endregion
-export { SynthesizeOptions, SynthesizeResult, TtsProvider, TtsRequest, TtsResult, cacheKey, espeakProvider, fakeProvider, openaiProvider, providerById, scriptPathFor, synthesizeScript, wavDuration };
+export { AlignRequest, Aligner, SynthesizeOptions, SynthesizeResult, TtsProvider, TtsRequest, TtsResult, alignerById, cacheKey, decodeWavMono, espeakProvider, fakeProvider, heuristicAligner, heuristicWords, interpolateMissing, mapAsrToScript, openaiProvider, piperProvider, providerById, resampleTo16kPcm, scriptPathFor, synthesizeScript, voskAligner, wavDuration };

package/dist/providers.js

@@ -1,7 +1,8 @@
 import { NarrationError } from "./index.js";
 import { createHash } from "node:crypto";
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
 import { basename, dirname, join } from "node:path";
+import { tmpdir } from "node:os";
 import { spawnSync } from "node:child_process";
 //#region src/providers.ts
 /**
@@ -132,12 +133,277 @@ function openaiProvider(opts = {}) {
 		}
 	};
 }
+/**
+* VITS-based local TTS: far more natural than espeak, runs on CPU, fully
+* offline. Needs a voice MODEL (`.onnx` + sibling `.onnx.json`) — pass its
+* path as `model`, or per-segment as `voice`. Emits no word timestamps; the
+* alignment step (below) fills them in.
+*/
+function piperProvider(opts = {}) {
+	return {
+		id: "piper",
+		version: () => {
+			const r = spawnSync("piper", ["--version"], { encoding: "utf8" });
+			if (r.status !== 0) throw new NarrationError("piper not found on PATH — install rhasspy/piper, or use --provider fake/espeak/openai");
+			const v = (r.stdout.trim() || r.stderr.trim() || "piper").split("\n")[0];
+			return Promise.resolve(opts.model ? `${v} ${basename(opts.model)}` : v);
+		},
+		synthesize: (req) => {
+			const model = req.voice ?? opts.model;
+			if (!model) throw new NarrationError("piper needs a voice model (.onnx) — pass { model }, or set the segment voice to its path");
+			const tag = createHash("sha256").update(req.text).digest("hex").slice(0, 8);
+			const out = join(tmpdir(), `glissade-piper-${process.pid}-${tag}.wav`);
+			const args = [
+				"--model",
+				model,
+				"--output_file",
+				out
+			];
+			if (req.rate !== void 0 && req.rate > 0) args.push("--length_scale", String(1 / req.rate));
+			const r = spawnSync("piper", args, {
+				input: req.text,
+				maxBuffer: 64 * 1024 * 1024
+			});
+			try {
+				if (r.status !== 0 || !existsSync(out)) throw new NarrationError(`piper failed: ${r.stderr?.toString().slice(0, 300) ?? "no output"}`);
+				const wav = readFileSync(out);
+				return Promise.resolve({
+					wav,
+					duration: wavDuration(wav)
+				});
+			} finally {
+				if (existsSync(out)) unlinkSync(out);
+			}
+		}
+	};
+}
 function providerById(id) {
 	switch (id) {
 		case "fake": return fakeProvider();
 		case "espeak": return espeakProvider();
+		case "piper": return piperProvider();
 		case "openai": return openaiProvider();
-		default: throw new NarrationError(`unknown TTS provider '${id}' (have: fake, espeak, openai)`);
+		default: throw new NarrationError(`unknown TTS provider '${id}' (have: fake, espeak, piper, openai)`);
+	}
+}
+/** ≈ syllable count: vowel groups, floored at 1 — a cheap spoken-length proxy. */
+function syllableWeight(word) {
+	const groups = word.toLowerCase().match(/[aeiouy]+/g);
+	return Math.max(1, groups ? groups.length : 1);
+}
+/**
+* Distribute words across the clip by estimated spoken length (syllables, not
+* characters — closer to real timing). Pure, deterministic, zero-dependency:
+* the always-available floor. Good enough for captions; karaoke on a very
+* slow/fast word wants a real aligner.
+*/
+function heuristicWords(text, duration) {
+	const words = text.trim().split(/\s+/).filter(Boolean);
+	if (words.length === 0) return [];
+	const weights = words.map(syllableWeight);
+	const total = weights.reduce((a, b) => a + b, 0);
+	const out = [];
+	let cursor = 0;
+	for (let i = 0; i < words.length; i++) {
+		const span = weights[i] / total * duration;
+		out.push({
+			word: words[i],
+			start: cursor,
+			end: cursor + span
+		});
+		cursor += span;
+	}
+	return out;
+}
+function heuristicAligner() {
+	return {
+		id: "heuristic",
+		version: () => Promise.resolve("heuristic-1"),
+		align: (req) => Promise.resolve(heuristicWords(req.text, wavDuration(req.wav)))
+	};
+}
+const normalizeWord = (w) => w.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, "");
+/**
+* Fill words whose start/end are NaN by linear interpolation between their
+* known neighbours (edges clamp). Keeps the result monotonic. Used after
+* mapping when some script words got no timing.
+*/
+function interpolateMissing(words) {
+	const out = words.map((w) => ({ ...w }));
+	const n = out.length;
+	let k = 0;
+	while (k < n) {
+		if (!Number.isNaN(out[k].start)) {
+			k++;
+			continue;
+		}
+		let j = k;
+		while (j < n && Number.isNaN(out[j].start)) j++;
+		const lo = k > 0 ? out[k - 1].end : j < n ? out[j].start : 0;
+		const hi = j < n ? out[j].start : lo;
+		const count = j - k;
+		const span = Math.max(0, hi - lo);
+		for (let t = 0; t < count; t++) {
+			out[k + t].start = lo + span * t / count;
+			out[k + t].end = lo + span * (t + 1) / count;
+		}
+		k = j;
+	}
+	return out;
+}
+/**
+* Transfer timed words (from an aligner) onto the script's own word tokens.
+* Forced aligners return near-identical words; ASR (whisper) can differ
+* (numbers spelled out, punctuation), so we LCS-align the normalized
+* sequences and interpolate script words the aligner didn't time. Output
+* length === script word count, in script order — what `wordBoxes()` indexes
+* against. If nothing matched, distribute by syllable over the timed span.
+*/
+function mapAsrToScript(timed, scriptText) {
+	const script = scriptText.trim().split(/\s+/).filter(Boolean);
+	if (script.length === 0 || timed.length === 0) return [];
+	const s = script.map(normalizeWord);
+	const a = timed.map((w) => normalizeWord(w.word));
+	const n = s.length;
+	const m = a.length;
+	const dp = Array.from({ length: n + 1 }, () => new Array(m + 1).fill(0));
+	for (let i = n - 1; i >= 0; i--) for (let j = m - 1; j >= 0; j--) dp[i][j] = s[i] !== "" && s[i] === a[j] ? dp[i + 1][j + 1] + 1 : Math.max(dp[i + 1][j], dp[i][j + 1]);
+	const matched = new Array(n).fill(null);
+	let i = 0;
+	let j = 0;
+	while (i < n && j < m) if (s[i] !== "" && s[i] === a[j]) {
+		matched[i] = j;
+		i++;
+		j++;
+	} else if (dp[i + 1][j] >= dp[i][j + 1]) i++;
+	else j++;
+	if (matched.every((x) => x === null)) {
+		const lo = Math.min(...timed.map((w) => w.start));
+		const hi = Math.max(...timed.map((w) => w.end));
+		return heuristicWords(scriptText, Math.max(0, hi - lo)).map((w) => ({
+			...w,
+			start: w.start + lo,
+			end: w.end + lo
+		}));
+	}
+	return interpolateMissing(script.map((word, k) => {
+		const mi = matched[k];
+		return mi != null ? {
+			word,
+			start: timed[mi].start,
+			end: timed[mi].end
+		} : {
+			word,
+			start: NaN,
+			end: NaN
+		};
+	}));
+}
+/** Decode a 16-bit PCM RIFF/WAV to mono float samples (channels averaged). */
+function decodeWavMono(wav) {
+	if (wav.length < 44 || wav.toString("ascii", 0, 4) !== "RIFF" || wav.toString("ascii", 8, 12) !== "WAVE") throw new NarrationError("not a RIFF/WAVE file");
+	let channels = 1;
+	let sampleRate = 16e3;
+	let bits = 16;
+	let dataOffset = -1;
+	let dataSize = 0;
+	let offset = 12;
+	while (offset + 8 <= wav.length) {
+		const id = wav.toString("ascii", offset, offset + 4);
+		const size = wav.readUInt32LE(offset + 4);
+		if (id === "fmt ") {
+			channels = wav.readUInt16LE(offset + 10);
+			sampleRate = wav.readUInt32LE(offset + 12);
+			bits = wav.readUInt16LE(offset + 22);
+		} else if (id === "data") {
+			dataOffset = offset + 8;
+			dataSize = size;
+		}
+		offset += 8 + size + size % 2;
+	}
+	if (bits !== 16) throw new NarrationError(`only 16-bit PCM WAV is supported (got ${bits}-bit)`);
+	if (dataOffset < 0) throw new NarrationError("WAV has no data chunk");
+	const frames = Math.floor(dataSize / 2 / Math.max(1, channels));
+	const samples = new Float32Array(frames);
+	for (let f = 0; f < frames; f++) {
+		let acc = 0;
+		for (let c = 0; c < channels; c++) acc += wav.readInt16LE(dataOffset + (f * channels + c) * 2);
+		samples[f] = acc / channels / 32768;
+	}
+	return {
+		samples,
+		sampleRate
+	};
+}
+/** Linear-resample mono float to a 16 kHz int16 LE PCM buffer (Vosk's input). */
+function resampleTo16kPcm(input) {
+	const ratio = input.sampleRate / 16e3;
+	const outLen = Math.max(1, Math.round(input.samples.length / ratio));
+	const out = Buffer.alloc(outLen * 2);
+	for (let i = 0; i < outLen; i++) {
+		const src = i * ratio;
+		const j = Math.floor(src);
+		const frac = src - j;
+		const a = input.samples[j] ?? 0;
+		const b = input.samples[j + 1] ?? a;
+		const v = Math.max(-1, Math.min(1, a + (b - a) * frac));
+		out.writeInt16LE(Math.round(v * 32767), i * 2);
+	}
+	return out;
+}
+/**
+* Word timings via Vosk (alphacephei) — offline, Apache-2.0, ~50 MB model, a
+* real Node binding (no Python, no Docker, no multi-GB download). `vosk` is an
+* OPTIONAL peer: install it (`npm i vosk`) and point at a model
+* (`opts.model` / `VOSK_MODEL`) only if you use this aligner. ASR words are
+* mapped onto the script tokens by `mapAsrToScript`.
+*/
+function voskAligner(opts = {}) {
+	const modelPath = opts.model ?? process.env["VOSK_MODEL"];
+	let vosk = null;
+	const load = async () => {
+		if (vosk) return vosk;
+		try {
+			vosk = await import("vosk");
+		} catch {
+			throw new NarrationError("vosk is not installed — `npm i vosk` and download a model, or use --align heuristic");
+		}
+		vosk.setLogLevel(-1);
+		return vosk;
+	};
+	return {
+		id: "vosk",
+		version: async () => {
+			if (!modelPath) throw new NarrationError("vosk needs a model — set VOSK_MODEL or pass { model } (alphacephei.com/vosk/models)");
+			if (!existsSync(modelPath)) throw new NarrationError(`vosk model not found at ${modelPath}`);
+			await load();
+			return `vosk:${basename(modelPath)}`;
+		},
+		align: async (req) => {
+			const v = await load();
+			const model = new v.Model(modelPath);
+			const rec = new v.Recognizer({
+				model,
+				sampleRate: 16e3
+			});
+			try {
+				rec.setWords(true);
+				rec.acceptWaveform(resampleTo16kPcm(decodeWavMono(req.wav)));
+				return mapAsrToScript(rec.finalResult().result ?? [], req.text);
+			} finally {
+				rec.free();
+				model.free();
+			}
+		}
+	};
+}
+/** Resolve an aligner id; 'none' disables alignment (word-less segments). */
+function alignerById(id) {
+	switch (id) {
+		case "none": return null;
+		case "heuristic": return heuristicAligner();
+		case "vosk": return voskAligner();
+		default: throw new NarrationError(`unknown aligner '${id}' (have: heuristic, vosk, none)`);
 	}
 }
 function cacheKey(seg, provider, providerVersion) {
@@ -162,8 +428,14 @@ async function synthesizeScript(scriptPath, opts = {}) {
 		if (ids.has(s.id)) throw new NarrationError(`duplicate segment id '${s.id}'`);
 		ids.add(s.id);
 	}
-	const provider = providerById(opts.provider ?? raw.provider ?? "espeak");
+	const provider = opts.providerImpl ?? providerById(opts.provider ?? raw.provider ?? "espeak");
 	const providerVersion = await provider.version();
+	const aligner = opts.alignerImpl !== void 0 ? opts.alignerImpl : alignerById(opts.aligner ?? raw.align ?? "heuristic");
+	let alignerTag = null;
+	const alignerTagFor = async () => {
+		if (alignerTag === null) alignerTag = `${aligner.id}@${await aligner.version()}`;
+		return alignerTag;
+	};
 	const base = scriptPath.replace(/\.narration\.json$/, "");
 	if (base === scriptPath) throw new NarrationError(`script path must end with .narration.json: ${scriptPath}`);
 	const cacheDir = `${base}.narration-cache`;
@@ -175,6 +447,7 @@ async function synthesizeScript(scriptPath, opts = {}) {
 	};
 	const synthesized = [];
 	const reused = [];
+	const aligned = [];
 	const segments = [];
 	let cursor = raw.leadIn ?? 0;
 	for (const seg of raw.segments) {
@@ -186,24 +459,42 @@ async function synthesizeScript(scriptPath, opts = {}) {
 		const hash = cacheKey(req, provider.id, providerVersion);
 		let entry = cache.entries[hash];
 		let duration;
+		let wavBuf;
 		let words;
 		if (entry !== void 0 && !opts.force && existsSync(join(cacheDir, entry.file))) {
-			duration = wavDuration(readFileSync(join(cacheDir, entry.file)));
-			words = entry.words;
+			wavBuf = readFileSync(join(cacheDir, entry.file));
+			duration = wavDuration(wavBuf);
 			reused.push(seg.id);
 		} else {
 			const result = await provider.synthesize(req);
 			const file = `${seg.id}-${hash.slice(0, 8)}.wav`;
 			writeFileSync(join(cacheDir, file), result.wav);
+			wavBuf = result.wav;
 			duration = wavDuration(result.wav);
-			words = result.words;
 			entry = {
 				file,
-				...words !== void 0 ? { words } : {}
+				...result.words !== void 0 ? {
+					words: result.words,
+					wordsFrom: "provider"
+				} : {}
 			};
 			cache.entries[hash] = entry;
 			synthesized.push(seg.id);
 		}
+		if (entry.wordsFrom === "provider") words = entry.words;
+		else if (aligner !== null) {
+			const tag = await alignerTagFor();
+			if (entry.wordsFrom === tag && entry.words !== void 0) words = entry.words;
+			else {
+				words = await aligner.align({
+					wav: wavBuf,
+					text: seg.text
+				});
+				entry.words = words;
+				entry.wordsFrom = tag;
+				aligned.push(seg.id);
+			}
+		}
 		const timed = {
 			id: seg.id,
 			text: seg.text,
@@ -235,7 +526,9 @@ async function synthesizeScript(scriptPath, opts = {}) {
 		timingPath,
 		cacheDir,
 		synthesized,
-		reused
+		reused,
+		aligned,
+		aligner: aligner?.id ?? null
 	};
 }
 /** Resolve `<scene>.narration.json` for a scene-module path (or accept the script itself). */
@@ -246,4 +539,4 @@ function scriptPathFor(input) {
 	return candidate;
 }
 //#endregion
-export { cacheKey, espeakProvider, fakeProvider, openaiProvider, providerById, scriptPathFor, synthesizeScript, wavDuration };
+export { alignerById, cacheKey, decodeWavMono, espeakProvider, fakeProvider, heuristicAligner, heuristicWords, interpolateMissing, mapAsrToScript, openaiProvider, piperProvider, providerById, resampleTo16kPcm, scriptPathFor, synthesizeScript, voskAligner, wavDuration };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@glissade/narrate",
-  "version": "0.4.4",
+  "version": "0.5.0-pre.0",
   "description": "glissade narration + captions: TTS at prepare time (gs narrate), deterministic caching, narration-anchored timeline beats, and captions as plain tracks. Render stays offline.",
   "license": "Apache-2.0",
   "type": "module",
@@ -19,8 +19,8 @@
     "dist"
   ],
   "dependencies": {
-    "@glissade/core": "0.4.4",
-    "@glissade/scene": "0.4.4"
+    "@glissade/core": "0.5.0-pre.0",
+    "@glissade/scene": "0.5.0-pre.0"
   },
   "repository": {
     "type": "git",