@glissade/narrate 0.4.5 → 0.5.0-pre.1

package/dist/index.d.ts

@@ -21,6 +21,13 @@ interface NarrationScript {
   gap?: number;
   /** silence before the first segment (s); default 0 */
   leadIn?: number;
+  /**
+   * Word-timing aligner for providers that don't emit word timestamps
+   * (espeak / openai / piper). 'heuristic' (default) estimates from text;
+   * 'vosk' derives real timings from the audio (offline ASR); 'none' leaves
+   * segments word-less. Providers that supply their own words ignore this.
+   */
+  align?: string;
   segments: NarrationSegment[];
 }
 interface TimedWord {

package/dist/providers.d.ts

@@ -36,10 +36,118 @@ declare function espeakProvider(): TtsProvider;
 declare function openaiProvider(opts?: {
   model?: string;
 }): TtsProvider;
+/**
+ * VITS-based local TTS: far more natural than espeak, runs on CPU, fully
+ * offline. Needs a voice MODEL (`.onnx` + sibling `.onnx.json`) — pass its
+ * path as `model`, or per-segment as `voice`. Emits no word timestamps; the
+ * alignment step (below) fills them in.
+ */
+declare function piperProvider(opts?: {
+  model?: string;
+}): TtsProvider;
 declare function providerById(id: string): TtsProvider;
+interface AlignRequest {
+  /** the synthesized RIFF/WAV bytes */
+  wav: Buffer;
+  /** the spoken text (the segment text) */
+  text: string;
+}
+/**
+ * Turns (audio, known text) into per-word timings — the provider-independent
+ * way to get word timestamps. Runs ONLY in the prepare step (heavy work is
+ * fine; it runs once and the result is cached). Same three-member shape as
+ * TtsProvider: `version()` participates in the cache so swapping aligners
+ * re-aligns the CACHED wav without re-synthesizing.
+ */
+interface Aligner {
+  readonly id: string;
+  version(): Promise<string>;
+  align(req: AlignRequest): Promise<{
+    word: string;
+    start: number;
+    end: number;
+  }[]>;
+}
+/**
+ * Distribute words across the clip by estimated spoken length (syllables, not
+ * characters — closer to real timing). Pure, deterministic, zero-dependency:
+ * the always-available floor. Good enough for captions; karaoke on a very
+ * slow/fast word wants a real aligner.
+ */
+declare function heuristicWords(text: string, duration: number): {
+  word: string;
+  start: number;
+  end: number;
+}[];
+declare function heuristicAligner(): Aligner;
+/**
+ * Fill words whose start/end are NaN by linear interpolation between their
+ * known neighbours (edges clamp). Keeps the result monotonic. Used after
+ * mapping when some script words got no timing.
+ */
+declare function interpolateMissing(words: {
+  word: string;
+  start: number;
+  end: number;
+}[]): {
+  word: string;
+  start: number;
+  end: number;
+}[];
+/**
+ * Transfer timed words (from an aligner) onto the script's own word tokens.
+ * Forced aligners return near-identical words; ASR (whisper) can differ
+ * (numbers spelled out, punctuation), so we LCS-align the normalized
+ * sequences and interpolate script words the aligner didn't time. Output
+ * length === script word count, in script order — what `wordBoxes()` indexes
+ * against. If nothing matched, distribute by syllable over the timed span.
+ */
+declare function mapAsrToScript(timed: {
+  word: string;
+  start: number;
+  end: number;
+}[], scriptText: string): {
+  word: string;
+  start: number;
+  end: number;
+}[];
+/** one word from vosk-align's JSON output */
+interface VoskAlignWord {
+  word: string;
+  start: number;
+  end: number;
+  conf?: number;
+}
+/**
+ * Word timings via Vosk (alphacephei) — offline ASR, Apache-2.0. Shells out to
+ * a `vosk-align` command (the Python `vosk` binding + ffmpeg — deliberately NOT
+ * the npm `vosk` package, whose `ffi-napi` native build is broken on modern
+ * Node). The command reads any audio and writes
+ *   { "words": [ { "word", "start", "end", "conf"? }, … ] }
+ * to stdout; its recognized words are LCS-mapped onto the script tokens by
+ * `mapAsrToScript`, so mis-recognitions (e.g. an unknown proper noun) just
+ * interpolate cleanly between the words around them.
+ *
+ * Provide the command via `opts.command` / `VOSK_ALIGN` (default `vosk-align`);
+ * the model is the command's own concern (its default, or `--model`/VOSK_MODEL),
+ * passed through with `opts.model`.
+ */
+declare function voskAligner(opts?: {
+  command?: string;
+  model?: string;
+}): Aligner;
+/** Resolve an aligner id; 'none' disables alignment (word-less segments). */
+declare function alignerById(id: string): Aligner | null;
 interface SynthesizeOptions {
-  /** override the script's provider */
+  /** override the script's provider (by id) */
   provider?: string;
+  /** override the script's aligner ('heuristic' | 'vosk' | 'none') */
+  aligner?: string;
+  /** a provider INSTANCE — wins over `provider`; the bring-your-own seam
+   *  (e.g. a custom ElevenLabs/Azure TtsProvider) */
+  providerImpl?: TtsProvider;
+  /** an aligner INSTANCE (or null to disable) — wins over `aligner` */
+  alignerImpl?: Aligner | null;
   /** ignore the cache and re-synthesize everything */
   force?: boolean;
 }
@@ -49,6 +157,10 @@ interface SynthesizeResult {
   cacheDir: string;
   synthesized: string[];
   reused: string[];
+  /** segment ids whose words came from the aligner (not the provider) */
+  aligned: string[];
+  /** the aligner id used, or null when alignment was disabled */
+  aligner: string | null;
 }
 declare function cacheKey(seg: {
   text: string;
@@ -64,4 +176,4 @@ declare function synthesizeScript(scriptPath: string, opts?: SynthesizeOptions):
 /** Resolve `<scene>.narration.json` for a scene-module path (or accept the script itself). */
 declare function scriptPathFor(input: string): string;
 //#endregion
-export { SynthesizeOptions, SynthesizeResult, TtsProvider, TtsRequest, TtsResult, cacheKey, espeakProvider, fakeProvider, openaiProvider, providerById, scriptPathFor, synthesizeScript, wavDuration };
+export { AlignRequest, Aligner, SynthesizeOptions, SynthesizeResult, TtsProvider, TtsRequest, TtsResult, VoskAlignWord, alignerById, cacheKey, espeakProvider, fakeProvider, heuristicAligner, heuristicWords, interpolateMissing, mapAsrToScript, openaiProvider, piperProvider, providerById, scriptPathFor, synthesizeScript, voskAligner, wavDuration };

package/dist/providers.js

@@ -1,7 +1,8 @@
 import { NarrationError } from "./index.js";
 import { createHash } from "node:crypto";
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
 import { basename, dirname, join } from "node:path";
+import { tmpdir } from "node:os";
 import { spawnSync } from "node:child_process";
 //#region src/providers.ts
 /**
@@ -132,12 +133,232 @@ function openaiProvider(opts = {}) {
 		}
 	};
 }
+/**
+* VITS-based local TTS: far more natural than espeak, runs on CPU, fully
+* offline. Needs a voice MODEL (`.onnx` + sibling `.onnx.json`) — pass its
+* path as `model`, or per-segment as `voice`. Emits no word timestamps; the
+* alignment step (below) fills them in.
+*/
+function piperProvider(opts = {}) {
+	return {
+		id: "piper",
+		version: () => {
+			const r = spawnSync("piper", ["--version"], { encoding: "utf8" });
+			if (r.error) {
+				if (r.error.code === "ENOENT") throw new NarrationError("piper not found on PATH — `pip install piper-tts` (or the standalone rhasspy/piper), or use --provider fake/espeak/openai");
+				throw new NarrationError(`could not run piper: ${r.error.message}`);
+			}
+			const m = /\b\d+\.\d+\.\d+\b/.exec(r.stdout ?? "");
+			const v = m ? `piper ${m[0]}` : "piper (version unknown)";
+			return Promise.resolve(opts.model ? `${v} ${basename(opts.model)}` : v);
+		},
+		synthesize: (req) => {
+			const model = req.voice ?? opts.model;
+			if (!model) throw new NarrationError("piper needs a voice model (.onnx) — pass { model }, or set the segment voice to its path");
+			const tag = createHash("sha256").update(req.text).digest("hex").slice(0, 8);
+			const out = join(tmpdir(), `glissade-piper-${process.pid}-${tag}.wav`);
+			const args = [
+				"--model",
+				model,
+				"--output_file",
+				out
+			];
+			if (req.rate !== void 0 && req.rate > 0) args.push("--length_scale", String(1 / req.rate));
+			const r = spawnSync("piper", args, {
+				input: req.text,
+				maxBuffer: 64 * 1024 * 1024
+			});
+			try {
+				if (r.status !== 0 || !existsSync(out)) throw new NarrationError(`piper failed: ${r.stderr?.toString().slice(0, 300) ?? "no output"}`);
+				const wav = readFileSync(out);
+				return Promise.resolve({
+					wav,
+					duration: wavDuration(wav)
+				});
+			} finally {
+				if (existsSync(out)) unlinkSync(out);
+			}
+		}
+	};
+}
 function providerById(id) {
 	switch (id) {
 		case "fake": return fakeProvider();
 		case "espeak": return espeakProvider();
+		case "piper": return piperProvider();
 		case "openai": return openaiProvider();
-		default: throw new NarrationError(`unknown TTS provider '${id}' (have: fake, espeak, openai)`);
+		default: throw new NarrationError(`unknown TTS provider '${id}' (have: fake, espeak, piper, openai)`);
+	}
+}
+/** ≈ syllable count: vowel groups, floored at 1 — a cheap spoken-length proxy. */
+function syllableWeight(word) {
+	const groups = word.toLowerCase().match(/[aeiouy]+/g);
+	return Math.max(1, groups ? groups.length : 1);
+}
+/**
+* Distribute words across the clip by estimated spoken length (syllables, not
+* characters — closer to real timing). Pure, deterministic, zero-dependency:
+* the always-available floor. Good enough for captions; karaoke on a very
+* slow/fast word wants a real aligner.
+*/
+function heuristicWords(text, duration) {
+	const words = text.trim().split(/\s+/).filter(Boolean);
+	if (words.length === 0) return [];
+	const weights = words.map(syllableWeight);
+	const total = weights.reduce((a, b) => a + b, 0);
+	const out = [];
+	let cursor = 0;
+	for (let i = 0; i < words.length; i++) {
+		const span = weights[i] / total * duration;
+		out.push({
+			word: words[i],
+			start: cursor,
+			end: cursor + span
+		});
+		cursor += span;
+	}
+	return out;
+}
+function heuristicAligner() {
+	return {
+		id: "heuristic",
+		version: () => Promise.resolve("heuristic-1"),
+		align: (req) => Promise.resolve(heuristicWords(req.text, wavDuration(req.wav)))
+	};
+}
+const normalizeWord = (w) => w.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, "");
+/**
+* Fill words whose start/end are NaN by linear interpolation between their
+* known neighbours (edges clamp). Keeps the result monotonic. Used after
+* mapping when some script words got no timing.
+*/
+function interpolateMissing(words) {
+	const out = words.map((w) => ({ ...w }));
+	const n = out.length;
+	let k = 0;
+	while (k < n) {
+		if (!Number.isNaN(out[k].start)) {
+			k++;
+			continue;
+		}
+		let j = k;
+		while (j < n && Number.isNaN(out[j].start)) j++;
+		const lo = k > 0 ? out[k - 1].end : j < n ? out[j].start : 0;
+		const hi = j < n ? out[j].start : lo;
+		const count = j - k;
+		const span = Math.max(0, hi - lo);
+		for (let t = 0; t < count; t++) {
+			out[k + t].start = lo + span * t / count;
+			out[k + t].end = lo + span * (t + 1) / count;
+		}
+		k = j;
+	}
+	return out;
+}
+/**
+* Transfer timed words (from an aligner) onto the script's own word tokens.
+* Forced aligners return near-identical words; ASR (whisper) can differ
+* (numbers spelled out, punctuation), so we LCS-align the normalized
+* sequences and interpolate script words the aligner didn't time. Output
+* length === script word count, in script order — what `wordBoxes()` indexes
+* against. If nothing matched, distribute by syllable over the timed span.
+*/
+function mapAsrToScript(timed, scriptText) {
+	const script = scriptText.trim().split(/\s+/).filter(Boolean);
+	if (script.length === 0 || timed.length === 0) return [];
+	const s = script.map(normalizeWord);
+	const a = timed.map((w) => normalizeWord(w.word));
+	const n = s.length;
+	const m = a.length;
+	const dp = Array.from({ length: n + 1 }, () => new Array(m + 1).fill(0));
+	for (let i = n - 1; i >= 0; i--) for (let j = m - 1; j >= 0; j--) dp[i][j] = s[i] !== "" && s[i] === a[j] ? dp[i + 1][j + 1] + 1 : Math.max(dp[i + 1][j], dp[i][j + 1]);
+	const matched = new Array(n).fill(null);
+	let i = 0;
+	let j = 0;
+	while (i < n && j < m) if (s[i] !== "" && s[i] === a[j]) {
+		matched[i] = j;
+		i++;
+		j++;
+	} else if (dp[i + 1][j] >= dp[i][j + 1]) i++;
+	else j++;
+	if (matched.every((x) => x === null)) {
+		const lo = Math.min(...timed.map((w) => w.start));
+		const hi = Math.max(...timed.map((w) => w.end));
+		return heuristicWords(scriptText, Math.max(0, hi - lo)).map((w) => ({
+			...w,
+			start: w.start + lo,
+			end: w.end + lo
+		}));
+	}
+	return interpolateMissing(script.map((word, k) => {
+		const mi = matched[k];
+		return mi != null ? {
+			word,
+			start: timed[mi].start,
+			end: timed[mi].end
+		} : {
+			word,
+			start: NaN,
+			end: NaN
+		};
+	}));
+}
+/**
+* Word timings via Vosk (alphacephei) — offline ASR, Apache-2.0. Shells out to
+* a `vosk-align` command (the Python `vosk` binding + ffmpeg — deliberately NOT
+* the npm `vosk` package, whose `ffi-napi` native build is broken on modern
+* Node). The command reads any audio and writes
+*   { "words": [ { "word", "start", "end", "conf"? }, … ] }
+* to stdout; its recognized words are LCS-mapped onto the script tokens by
+* `mapAsrToScript`, so mis-recognitions (e.g. an unknown proper noun) just
+* interpolate cleanly between the words around them.
+*
+* Provide the command via `opts.command` / `VOSK_ALIGN` (default `vosk-align`);
+* the model is the command's own concern (its default, or `--model`/VOSK_MODEL),
+* passed through with `opts.model`.
+*/
+function voskAligner(opts = {}) {
+	const command = opts.command ?? process.env["VOSK_ALIGN"] ?? "vosk-align";
+	return {
+		id: "vosk",
+		version: () => {
+			const r = spawnSync(command, ["--help"], { encoding: "utf8" });
+			if (r.error) {
+				if (r.error.code === "ENOENT") throw new NarrationError(`'${command}' not found — provide a vosk-align command (Apache-2.0 Vosk + ffmpeg, JSON {words:[{word,start,end}]} on stdout), or use --align heuristic`);
+				throw new NarrationError(`could not run ${command}: ${r.error.message}`);
+			}
+			return Promise.resolve(opts.model ? `vosk ${basename(opts.model)}` : "vosk");
+		},
+		align: (req) => {
+			const tag = createHash("sha256").update(req.text).digest("hex").slice(0, 8);
+			const wavPath = join(tmpdir(), `glissade-vosk-${process.pid}-${tag}.wav`);
+			try {
+				writeFileSync(wavPath, req.wav);
+				const r = spawnSync(command, [wavPath, ...opts.model ? ["--model", opts.model] : []], {
+					encoding: "utf8",
+					maxBuffer: 64 * 1024 * 1024
+				});
+				if (r.error) throw new NarrationError(`${command} failed to run: ${r.error.message}`);
+				if (r.status !== 0) throw new NarrationError(`${command} failed: ${(r.stderr || "").slice(0, 300)}`);
+				const timed = (JSON.parse(r.stdout).words ?? []).filter((w) => typeof w.start === "number" && typeof w.end === "number").map((w) => ({
+					word: w.word,
+					start: w.start,
+					end: w.end
+				}));
+				return Promise.resolve(mapAsrToScript(timed, req.text));
+			} finally {
+				if (existsSync(wavPath)) unlinkSync(wavPath);
+			}
+		}
+	};
+}
+/** Resolve an aligner id; 'none' disables alignment (word-less segments). */
+function alignerById(id) {
+	switch (id) {
+		case "none": return null;
+		case "heuristic": return heuristicAligner();
+		case "vosk": return voskAligner();
+		default: throw new NarrationError(`unknown aligner '${id}' (have: heuristic, vosk, none)`);
 	}
 }
 function cacheKey(seg, provider, providerVersion) {
@@ -162,8 +383,14 @@ async function synthesizeScript(scriptPath, opts = {}) {
 		if (ids.has(s.id)) throw new NarrationError(`duplicate segment id '${s.id}'`);
 		ids.add(s.id);
 	}
-	const provider = providerById(opts.provider ?? raw.provider ?? "espeak");
+	const provider = opts.providerImpl ?? providerById(opts.provider ?? raw.provider ?? "espeak");
 	const providerVersion = await provider.version();
+	const aligner = opts.alignerImpl !== void 0 ? opts.alignerImpl : alignerById(opts.aligner ?? raw.align ?? "heuristic");
+	let alignerTag = null;
+	const alignerTagFor = async () => {
+		if (alignerTag === null) alignerTag = `${aligner.id}@${await aligner.version()}`;
+		return alignerTag;
+	};
 	const base = scriptPath.replace(/\.narration\.json$/, "");
 	if (base === scriptPath) throw new NarrationError(`script path must end with .narration.json: ${scriptPath}`);
 	const cacheDir = `${base}.narration-cache`;
@@ -175,6 +402,7 @@ async function synthesizeScript(scriptPath, opts = {}) {
 	};
 	const synthesized = [];
 	const reused = [];
+	const aligned = [];
 	const segments = [];
 	let cursor = raw.leadIn ?? 0;
 	for (const seg of raw.segments) {
@@ -186,24 +414,42 @@ async function synthesizeScript(scriptPath, opts = {}) {
 		const hash = cacheKey(req, provider.id, providerVersion);
 		let entry = cache.entries[hash];
 		let duration;
+		let wavBuf;
 		let words;
 		if (entry !== void 0 && !opts.force && existsSync(join(cacheDir, entry.file))) {
-			duration = wavDuration(readFileSync(join(cacheDir, entry.file)));
-			words = entry.words;
+			wavBuf = readFileSync(join(cacheDir, entry.file));
+			duration = wavDuration(wavBuf);
 			reused.push(seg.id);
 		} else {
 			const result = await provider.synthesize(req);
 			const file = `${seg.id}-${hash.slice(0, 8)}.wav`;
 			writeFileSync(join(cacheDir, file), result.wav);
+			wavBuf = result.wav;
 			duration = wavDuration(result.wav);
-			words = result.words;
 			entry = {
 				file,
-				...words !== void 0 ? { words } : {}
+				...result.words !== void 0 ? {
+					words: result.words,
+					wordsFrom: "provider"
+				} : {}
 			};
 			cache.entries[hash] = entry;
 			synthesized.push(seg.id);
 		}
+		if (entry.wordsFrom === "provider") words = entry.words;
+		else if (aligner !== null) {
+			const tag = await alignerTagFor();
+			if (entry.wordsFrom === tag && entry.words !== void 0) words = entry.words;
+			else {
+				words = await aligner.align({
+					wav: wavBuf,
+					text: seg.text
+				});
+				entry.words = words;
+				entry.wordsFrom = tag;
+				aligned.push(seg.id);
+			}
+		}
 		const timed = {
 			id: seg.id,
 			text: seg.text,
@@ -235,7 +481,9 @@ async function synthesizeScript(scriptPath, opts = {}) {
 		timingPath,
 		cacheDir,
 		synthesized,
-		reused
+		reused,
+		aligned,
+		aligner: aligner?.id ?? null
 	};
 }
 /** Resolve `<scene>.narration.json` for a scene-module path (or accept the script itself). */
@@ -246,4 +494,4 @@ function scriptPathFor(input) {
 	return candidate;
 }
 //#endregion
-export { cacheKey, espeakProvider, fakeProvider, openaiProvider, providerById, scriptPathFor, synthesizeScript, wavDuration };
+export { alignerById, cacheKey, espeakProvider, fakeProvider, heuristicAligner, heuristicWords, interpolateMissing, mapAsrToScript, openaiProvider, piperProvider, providerById, scriptPathFor, synthesizeScript, voskAligner, wavDuration };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@glissade/narrate",
-  "version": "0.4.5",
+  "version": "0.5.0-pre.1",
   "description": "glissade narration + captions: TTS at prepare time (gs narrate), deterministic caching, narration-anchored timeline beats, and captions as plain tracks. Render stays offline.",
   "license": "Apache-2.0",
   "type": "module",
@@ -19,8 +19,8 @@
     "dist"
   ],
   "dependencies": {
-    "@glissade/core": "0.4.5",
-    "@glissade/scene": "0.4.5"
+    "@glissade/core": "0.5.0-pre.1",
+    "@glissade/scene": "0.5.0-pre.1"
   },
   "repository": {
     "type": "git",