@bendyline/squisq 1.5.1 → 2.0.0

package/src/narration/features.ts

@@ -0,0 +1,231 @@
+/**
+ * Frame feature extraction: full-band RMS, speech-band energy, and
+ * zero-crossing rate over hopped (possibly overlapping) frames.
+ *
+ * Band energy uses two cascaded RBJ biquads — a high-pass at
+ * `bandLowHz` and a low-pass at `bandHighHz` (both Q = 1/√2,
+ * Audio-EQ-Cookbook closed forms) — rather than an FFT or Goertzel
+ * bins: a Butterworth-ish cascade gives a true maximally-flat band in
+ * one O(N) pass, and the filter state persists across hops, which is
+ * exactly right for streaming (no window-boundary energy jitter).
+ * Goertzel measures single frequencies (DTMF-style) — wrong tool for
+ * broadband speech energy.
+ */
+
+import type { FeatureConfig, FrameFeatures } from './types.js';
+import { DEFAULT_FEATURE_CONFIG } from './types.js';
+
+interface BiquadCoeffs {
+  b0: number;
+  b1: number;
+  b2: number;
+  a1: number;
+  a2: number;
+}
+
+interface BiquadState {
+  x1: number;
+  x2: number;
+  y1: number;
+  y2: number;
+}
+
+export interface BandpassState {
+  coeffs: { hp: BiquadCoeffs; lp: BiquadCoeffs };
+  hp: BiquadState;
+  lp: BiquadState;
+}
+
+function highpassCoeffs(sampleRate: number, freqHz: number): BiquadCoeffs {
+  const w0 = (2 * Math.PI * freqHz) / sampleRate;
+  const cosW0 = Math.cos(w0);
+  // alpha = sin(w0) / (2Q) with Butterworth Q = 1/√2.
+  const alpha = Math.sin(w0) / (2 * Math.SQRT1_2);
+  const a0 = 1 + alpha;
+  return {
+    b0: (1 + cosW0) / 2 / a0,
+    b1: -(1 + cosW0) / a0,
+    b2: (1 + cosW0) / 2 / a0,
+    a1: (-2 * cosW0) / a0,
+    a2: (1 - alpha) / a0,
+  };
+}
+
+function lowpassCoeffs(sampleRate: number, freqHz: number): BiquadCoeffs {
+  const w0 = (2 * Math.PI * freqHz) / sampleRate;
+  const cosW0 = Math.cos(w0);
+  const alpha = Math.sin(w0) / (2 * Math.SQRT1_2);
+  const a0 = 1 + alpha;
+  return {
+    b0: (1 - cosW0) / 2 / a0,
+    b1: (1 - cosW0) / a0,
+    b2: (1 - cosW0) / 2 / a0,
+    a1: (-2 * cosW0) / a0,
+    a2: (1 - alpha) / a0,
+  };
+}
+
+/** Create bandpass state for a sample rate (corners derived at init). */
+export function createBandpass(
+  sampleRate: number,
+  lowHz = DEFAULT_FEATURE_CONFIG.bandLowHz,
+  highHz = DEFAULT_FEATURE_CONFIG.bandHighHz,
+): BandpassState {
+  return {
+    coeffs: { hp: highpassCoeffs(sampleRate, lowHz), lp: lowpassCoeffs(sampleRate, highHz) },
+    hp: { x1: 0, x2: 0, y1: 0, y2: 0 },
+    lp: { x1: 0, x2: 0, y1: 0, y2: 0 },
+  };
+}
+
+function biquadRun(
+  coeffs: BiquadCoeffs,
+  state: BiquadState,
+  input: Float32Array,
+  output: Float32Array,
+): BiquadState {
+  let { x1, x2, y1, y2 } = state;
+  const { b0, b1, b2, a1, a2 } = coeffs;
+  for (let i = 0; i < input.length; i++) {
+    const x0 = input[i];
+    const y0 = b0 * x0 + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2;
+    output[i] = y0;
+    x2 = x1;
+    x1 = x0;
+    y2 = y1;
+    y1 = y0;
+  }
+  return { x1, x2, y1, y2 };
+}
+
+/**
+ * Run the bandpass over a chunk. Pure in the state: returns a NEW state,
+ * writes filtered samples into `output` (may alias `input`).
+ */
+export function bandpassRun(
+  state: BandpassState,
+  input: Float32Array,
+  output: Float32Array,
+): BandpassState {
+  const hp = biquadRun(state.coeffs.hp, state.hp, input, output);
+  const lp = biquadRun(state.coeffs.lp, state.lp, output, output);
+  return { coeffs: state.coeffs, hp, lp };
+}
+
+export interface FeatureState {
+  sampleRate: number;
+  config: FeatureConfig;
+  /** Hop between frame starts, in samples. */
+  hopSamples: number;
+  bandpass: BandpassState;
+  /** Unconsumed raw samples (the next frame's prefix). */
+  rawCarry: Float32Array;
+  /** Bandpassed twin of `rawCarry`. */
+  bandCarry: Float32Array;
+  /** Absolute sample index of `rawCarry[0]` from the start of the signal. */
+  carryStartSample: number;
+}
+
+export function createFeatureState(
+  sampleRate: number,
+  config?: Partial<FeatureConfig>,
+): FeatureState {
+  const merged: FeatureConfig = { ...DEFAULT_FEATURE_CONFIG, ...config };
+  return {
+    sampleRate,
+    config: merged,
+    hopSamples: Math.max(1, Math.round(merged.hopSec * sampleRate)),
+    bandpass: createBandpass(sampleRate, merged.bandLowHz, merged.bandHighHz),
+    rawCarry: new Float32Array(0),
+    bandCarry: new Float32Array(0),
+    carryStartSample: 0,
+  };
+}
+
+function concatF32(a: Float32Array, b: Float32Array): Float32Array {
+  if (a.length === 0) return b;
+  const out = new Float32Array(a.length + b.length);
+  out.set(a, 0);
+  out.set(b, a.length);
+  return out;
+}
+
+function frameOf(
+  raw: Float32Array,
+  band: Float32Array,
+  start: number,
+  size: number,
+  startSample: number,
+  sampleRate: number,
+): FrameFeatures {
+  let sumSq = 0;
+  let bandSumSq = 0;
+  let crossings = 0;
+  let prev = raw[start];
+  for (let i = 0; i < size; i++) {
+    const r = raw[start + i];
+    const b = band[start + i];
+    sumSq += r * r;
+    bandSumSq += b * b;
+    if (i > 0 && ((r >= 0 && prev < 0) || (r < 0 && prev >= 0))) crossings++;
+    prev = r;
+  }
+  return {
+    tSec: (startSample + size / 2) / sampleRate,
+    rms: Math.sqrt(sumSq / size),
+    bandEnergy: bandSumSq / size,
+    zcr: size > 1 ? crossings / (size - 1) : 0,
+  };
+}
+
+/**
+ * Feed a hop of PCM. Returns a new state plus every complete frame the
+ * buffered signal now covers (usually 0–1 for worklet-sized hops, more
+ * for large chunks).
+ */
+export function featureStep(
+  state: FeatureState,
+  hop: Float32Array,
+): { state: FeatureState; frames: FrameFeatures[] } {
+  const banded = new Float32Array(hop.length);
+  const bandpass = bandpassRun(state.bandpass, hop, banded);
+
+  const raw = concatF32(state.rawCarry, hop);
+  const band = concatF32(state.bandCarry, banded);
+  const { frameSize } = state.config;
+  const frames: FrameFeatures[] = [];
+
+  let cursor = 0;
+  while (raw.length - cursor >= frameSize) {
+    frames.push(
+      frameOf(raw, band, cursor, frameSize, state.carryStartSample + cursor, state.sampleRate),
+    );
+    cursor += state.hopSamples;
+  }
+
+  return {
+    state: {
+      ...state,
+      bandpass,
+      // slice(), not subarray(): the carry must OWN its memory. `raw` can
+      // alias the caller's hop buffer, and callers may recycle those
+      // (e.g. Chrome invalidates buffers transferred out of an
+      // AudioWorklet after the message handler returns) — a retained
+      // view would silently read zeros on the next step.
+      rawCarry: raw.slice(cursor),
+      bandCarry: band.slice(cursor),
+      carryStartSample: state.carryStartSample + cursor,
+    },
+    frames,
+  };
+}
+
+/** Batch feature extraction over a whole take (the aligner's entry point). */
+export function extractFrameFeatures(
+  pcm: Float32Array,
+  sampleRate: number,
+  config?: Partial<FeatureConfig>,
+): FrameFeatures[] {
+  const state = createFeatureState(sampleRate, config);
+  return featureStep(state, pcm).frames;
+}

package/src/narration/index.ts

@@ -0,0 +1,86 @@
+/**
+ * @bendyline/squisq/narration — the narration/teleprompter engine.
+ *
+ * Pure TypeScript, zero dependencies, no DOM/Node APIs: a script model
+ * built from a Doc, streaming DSP (band features → adaptive VAD →
+ * syllable nuclei), a deterministic voice-adaptive pacing controller,
+ * and an offline DTW aligner that turns a recorded take into word/block
+ * timestamps. Determinism is a hard rule here — no Date.now, no
+ * Math.random, no timers — so every stage is unit-testable in Node
+ * with synthetic PCM.
+ */
+
+export type {
+  ScriptToken,
+  ScriptBlockRange,
+  NarrationScript,
+  FrameFeatures,
+  FeatureConfig,
+  VadConfig,
+  NucleiConfig,
+  PacingConfig,
+  AlignConfig,
+  WordTiming,
+  NarrationBlockRange,
+  NarrationAlignment,
+} from './types.js';
+export {
+  DEFAULT_FEATURE_CONFIG,
+  DEFAULT_VAD_CONFIG,
+  DEFAULT_NUCLEI_CONFIG,
+  DEFAULT_PACING_CONFIG,
+  DEFAULT_ALIGN_CONFIG,
+} from './types.js';
+
+export { estimateSyllables } from './syllables.js';
+
+export {
+  buildNarrationScript,
+  expectedSyllablesAt,
+  wordPosAtExpectedSyllables,
+  wordIndexAtChar,
+  wordIndexAtTime,
+  type BuildScriptOptions,
+} from './script.js';
+
+export {
+  createBandpass,
+  bandpassRun,
+  createFeatureState,
+  featureStep,
+  extractFrameFeatures,
+  type BandpassState,
+  type FeatureState,
+} from './features.js';
+
+export { createVadState, vadStep, type VadState } from './vad.js';
+
+export { createNucleiState, nucleiStep, detectSyllableOnsets, type NucleiState } from './nuclei.js';
+
+export {
+  createPacingState,
+  pacingStep,
+  reanchorPacing,
+  type PacingState,
+  type PacingTick,
+} from './pacing.js';
+
+export {
+  createNarrationSession,
+  narrationSessionStep,
+  reanchorSession,
+  type NarrationSessionConfig,
+  type NarrationSessionState,
+} from './session.js';
+
+export { traceWordPosAt, downsampleTrace, type NarrationTrace, type TraceSample } from './trace.js';
+
+export { alignNarration, type AlignInput } from './align.js';
+
+export {
+  buildNarrationTimingJson,
+  parseNarrationTimingJson,
+  type NarrationTimingJsonV3,
+  type NarrationTimingBlock,
+  type BuildNarrationTimingOptions,
+} from './sidecar.js';

package/src/narration/nuclei.ts

@@ -0,0 +1,118 @@
+/**
+ * Syllable-nuclei detection: vowel nuclei show up as peaks in a
+ * low-passed amplitude envelope of the speech-band signal.
+ *
+ * Envelope = one-pole low-pass (≈10 Hz) over √bandEnergy (amplitude
+ * domain, so peaks are nuclei rather than energy spikes). A peak is
+ * emitted one frame late (local-max needs one frame of lookahead —
+ * ~20 ms live latency) and accepted only when: VAD says speaking, it
+ * clears the minimum inter-onset spacing, it is prominent relative to
+ * the valley since the last accepted peak, and it clears a decaying
+ * loudness reference (guards against ripples inside quiet speech).
+ */
+
+import type { FrameFeatures, NucleiConfig } from './types.js';
+import { DEFAULT_NUCLEI_CONFIG } from './types.js';
+
+export interface NucleiState {
+  /** One-pole envelope of √bandEnergy. */
+  env: number;
+  prevEnv: number;
+  prevPrevEnv: number;
+  /** tSec of the previous frame (the peak-candidate time). */
+  prevTSec: number;
+  /** Envelope minimum since the last accepted peak. */
+  valley: number;
+  lastOnsetTSec: number;
+  /** Decaying loudness reference. */
+  peakRef: number;
+  lastTSec: number;
+  framesSeen: number;
+}
+
+export function createNucleiState(_config?: Partial<NucleiConfig>): NucleiState {
+  return {
+    env: 0,
+    prevEnv: 0,
+    prevPrevEnv: 0,
+    prevTSec: 0,
+    valley: Number.POSITIVE_INFINITY,
+    lastOnsetTSec: Number.NEGATIVE_INFINITY,
+    peakRef: 0,
+    lastTSec: 0,
+    framesSeen: 0,
+  };
+}
+
+/**
+ * Pure per-frame step. `onset` is the tSec of a newly accepted syllable
+ * nucleus (emitted once, at its peak frame), or null.
+ */
+export function nucleiStep(
+  state: NucleiState,
+  frame: FrameFeatures,
+  speaking: boolean,
+  config?: Partial<NucleiConfig>,
+): { state: NucleiState; onset: number | null } {
+  const c = { ...DEFAULT_NUCLEI_CONFIG, ...config };
+  const amp = Math.sqrt(Math.max(0, frame.bandEnergy));
+  const dt = state.framesSeen > 0 ? Math.max(0, frame.tSec - state.lastTSec) : 0;
+
+  const env =
+    state.framesSeen > 0
+      ? state.env + (amp - state.env) * (1 - Math.exp(-2 * Math.PI * c.envelopeHz * dt))
+      : amp;
+  const peakRef = Math.max(env, state.peakRef * Math.exp(-(dt || 0) / c.peakRefTauSec));
+
+  let onset: number | null = null;
+  let valley = state.framesSeen > 0 ? state.valley : env;
+  let lastOnsetTSec = state.lastOnsetTSec;
+
+  // Candidate peak at the PREVIOUS frame: env[t-1] ≥ env[t-2] && env[t-1] > env[t].
+  if (
+    state.framesSeen >= 2 &&
+    state.prevEnv >= state.prevPrevEnv &&
+    state.prevEnv > env &&
+    speaking &&
+    state.prevTSec - lastOnsetTSec >= c.minInterOnsetMs / 1000 &&
+    state.prevEnv >= valley * c.prominenceRatio &&
+    state.prevEnv >= peakRef * c.minRelPeak
+  ) {
+    onset = state.prevTSec;
+    lastOnsetTSec = state.prevTSec;
+    // Re-seed the valley at the accepted peak; it decays via min() below.
+    valley = state.prevEnv;
+  }
+  valley = Math.min(valley, env);
+
+  return {
+    state: {
+      env,
+      prevEnv: env,
+      prevPrevEnv: state.framesSeen > 0 ? state.prevEnv : env,
+      prevTSec: frame.tSec,
+      valley,
+      lastOnsetTSec,
+      peakRef,
+      lastTSec: frame.tSec,
+      framesSeen: state.framesSeen + 1,
+    },
+    onset,
+  };
+}
+
+/** Batch wrapper for the offline aligner. `vadFlags[i]` pairs with `frames[i]`. */
+export function detectSyllableOnsets(
+  frames: FrameFeatures[],
+  vadFlags: boolean[],
+  config?: Partial<NucleiConfig>,
+): number[] {
+  let state = createNucleiState(config);
+  const onsets: number[] = [];
+  for (let i = 0; i < frames.length; i++) {
+    const step = nucleiStep(state, frames[i], vadFlags[i] ?? false, config);
+    state = step.state;
+    if (step.onset !== null) onsets.push(step.onset);
+  }
+  return onsets;
+}

package/src/narration/pacing.ts

@@ -0,0 +1,211 @@
+/**
+ * The teleprompter pacing controller.
+ *
+ * A pure, deterministic step function: given the current state, one
+ * analysis tick (VAD flag + syllable onsets), and the script, produce
+ * the next state. The prompter position (`wordPos`) advances at a
+ * velocity that (a) halts within ~250 ms of silence, (b) tracks the
+ * reader's detected syllable rate while speaking, and (c) is corrected
+ * by a PI loop on cumulative detected-vs-expected syllables so the
+ * highlight can't drift arbitrarily far from the voice. Silence near an
+ * expected paragraph/block break is legitimate and accrues no error.
+ */
+
+import { expectedSyllablesAt, wordPosAtExpectedSyllables } from './script.js';
+import type { NarrationScript, PacingConfig } from './types.js';
+import { DEFAULT_PACING_CONFIG } from './types.js';
+
+export interface PacingTick {
+  tSec: number;
+  speaking: boolean;
+  /** Syllable onsets detected in this tick (0 or 1 for frame-sized ticks). */
+  onsets: number;
+}
+
+export interface PacingState {
+  /** Fractional word position — the active token is `floor(wordPos)`. */
+  wordPos: number;
+  /** Current scroll velocity in words per second. */
+  velocityWps: number;
+  /** EMA of the detected syllable rate (syl/s); 0 until two onsets arrive. */
+  emaSylPerSec: number;
+  lastOnsetTSec: number | null;
+  /** Syllables detected since the last anchor. */
+  cumDetectedSyl: number;
+  /** Expected cumulative syllables at the last anchor position. */
+  anchorExpectedSyl: number;
+  /** PI integral of the cumulative-syllable error (syl·s). */
+  errIntegral: number;
+  /** Continuous silence observed so far (s). */
+  silenceSec: number;
+  lastTSec: number | null;
+  /** True when silent and fully stopped. */
+  halted: boolean;
+}
+
+export function createPacingState(startWordPos = 0): PacingState {
+  return {
+    wordPos: startWordPos,
+    velocityWps: 0,
+    emaSylPerSec: 0,
+    lastOnsetTSec: null,
+    cumDetectedSyl: 0,
+    anchorExpectedSyl: 0,
+    errIntegral: 0,
+    silenceSec: 0,
+    lastTSec: null,
+    halted: true,
+  };
+}
+
+function clamp(value: number, min: number, max: number): number {
+  return value < min ? min : value > max ? max : value;
+}
+
+/**
+ * Mean syllables-per-SPOKEN-word over a short window ahead of the prompter
+ * position. Standalone punctuation (0 syllables, `spoken: false`) is
+ * excluded so it doesn't deflate the average.
+ */
+function localSyllablesPerWord(
+  script: NarrationScript,
+  wordPos: number,
+  windowWords: number,
+): number {
+  const n = script.tokens.length;
+  if (n === 0) return 1.4;
+  const start = clamp(Math.floor(wordPos), 0, n - 1);
+  let sum = 0;
+  let count = 0;
+  for (let i = start; i < n && count < windowWords; i++) {
+    if (!script.tokens[i].spoken) continue;
+    sum += script.tokens[i].syllables;
+    count += 1;
+  }
+  // Default to the English average (~1.4) when the window has no words yet.
+  return count > 0 ? Math.max(1, sum / count) : 1.4;
+}
+
+/** Pure controller step — same inputs always yield the same output state. */
+export function pacingStep(
+  state: PacingState,
+  tick: PacingTick,
+  script: NarrationScript,
+  config?: Partial<PacingConfig>,
+): PacingState {
+  const c = { ...DEFAULT_PACING_CONFIG, ...config };
+  const baseWps = c.baseWpm / 60;
+  const dt = clamp(tick.tSec - (state.lastTSec ?? tick.tSec), 0, 0.25);
+
+  // 1. Onset bookkeeping: cumulative count + inter-onset-interval rate EMA.
+  let emaSylPerSec = state.emaSylPerSec;
+  let lastOnsetTSec = state.lastOnsetTSec;
+  let cumDetectedSyl = state.cumDetectedSyl;
+  for (let k = 0; k < tick.onsets; k++) {
+    if (lastOnsetTSec !== null) {
+      const ioi = tick.tSec - lastOnsetTSec;
+      if (ioi > 0) {
+        const instRate = clamp(1 / ioi, 0.5, 12);
+        const alpha = 1 - Math.exp(-ioi / c.rateEmaTauSec);
+        emaSylPerSec =
+          emaSylPerSec === 0 ? instRate : emaSylPerSec + (instRate - emaSylPerSec) * alpha;
+      }
+    }
+    lastOnsetTSec = tick.tSec;
+    cumDetectedSyl += 1;
+  }
+
+  // 2–3. Cruise velocity: BLEND the user's set pace with the voice-derived
+  // rate. The old design used `baseWps · clamp(voiceWps / baseWps)`, in
+  // which baseWps algebraically cancels in the unclamped range — so the WPM
+  // slider did nothing and a chronically under-counting detector dragged
+  // the prompter slow with no way to compensate. Blending keeps the set
+  // pace as a real feedforward term (WPM always felt), while the voice
+  // steers within a band. Then clamp to a band around the set pace so
+  // neither term alone can stall or run away.
+  const sylPerWordLocal = localSyllablesPerWord(script, state.wordPos, c.sylWindowWords);
+  const voiceWps = emaSylPerSec > 0 ? emaSylPerSec / sylPerWordLocal : baseWps;
+  const cruise = clamp(
+    c.voiceBlend * voiceWps + (1 - c.voiceBlend) * baseWps,
+    baseWps * c.minRateMult,
+    baseWps * c.maxRateMult,
+  );
+
+  // 4–5. FORWARD-ONLY position correction. `err` compares where the
+  // prompter is (expected syllables at wordPos) against how many syllables
+  // the voice has actually delivered. err < 0 → the reader is AHEAD of the
+  // prompter → speed up to catch them. err > 0 → the prompter looks ahead
+  // of the reader, but this is exactly the signal a chronically
+  // under-counting syllable detector produces even when the prompter is
+  // really BEHIND — so we deliberately do NOT slow or snap backward on it.
+  // (The old symmetric PI pinned the prompter to the DETECTED count, so a
+  // detector catching half the syllables parked the prompter at half the
+  // reader's true position — the lag the user saw.) Slowing down is instead
+  // handled honestly by the cruise blend (a slow voice lowers voiceWps) and
+  // by halt-on-silence.
+  const err =
+    expectedSyllablesAt(script, state.wordPos) - (state.anchorExpectedSyl + cumDetectedSyl);
+  let errIntegral = clamp(Math.min(0, state.errIntegral + err * dt), -c.intClamp, 0);
+  const boost = err < 0 ? clamp(c.kP * -err + c.kI * -errIntegral, 0, c.maxCorrection) : 0;
+
+  // 6. Velocity: track while speaking, decay to a halt in silence.
+  let velocityWps = state.velocityWps;
+  let silenceSec = state.silenceSec;
+  if (tick.speaking) {
+    silenceSec = 0;
+    const target = Math.max(0, cruise * (1 + boost));
+    velocityWps += (target - velocityWps) * (1 - Math.exp(-dt / c.velSlewTauSec));
+  } else {
+    silenceSec += dt;
+    velocityWps *= Math.exp(-dt / c.haltTauSec);
+    if (velocityWps < 0.02) velocityWps = 0;
+  }
+
+  // 7. Hard resync forward only: when the reader has clearly outrun the
+  // prompter by more than a paragraph, jump ahead to catch up. Never jump
+  // backward (a false "prompter ahead" from under-counting must not yank
+  // the highlight back).
+  let wordPos = state.wordPos;
+  if (err < -c.resyncSyllables) {
+    wordPos = Math.max(
+      wordPos,
+      wordPosAtExpectedSyllables(script, state.anchorExpectedSyl + cumDetectedSyl),
+    );
+    errIntegral = 0;
+  }
+
+  // 8. Integrate.
+  wordPos = clamp(wordPos + velocityWps * dt, 0, script.tokens.length);
+
+  return {
+    wordPos,
+    velocityWps,
+    emaSylPerSec,
+    lastOnsetTSec,
+    cumDetectedSyl,
+    anchorExpectedSyl: state.anchorExpectedSyl,
+    errIntegral,
+    silenceSec,
+    lastTSec: tick.tSec,
+    halted: !tick.speaking && velocityWps === 0,
+  };
+}
+
+/**
+ * Re-anchor after a manual nudge/scroll: the new position becomes the
+ * zero-error reference. The learned rate EMA survives.
+ */
+export function reanchorPacing(
+  state: PacingState,
+  wordPos: number,
+  script: NarrationScript,
+): PacingState {
+  const pos = clamp(wordPos, 0, script.tokens.length);
+  return {
+    ...state,
+    wordPos: pos,
+    anchorExpectedSyl: expectedSyllablesAt(script, pos),
+    cumDetectedSyl: 0,
+    errIntegral: 0,
+  };
+}