@bendyline/squisq 2.0.1 → 2.2.0

package/src/narration/features.ts

@@ -1,231 +0,0 @@
-/**
- * Frame feature extraction: full-band RMS, speech-band energy, and
- * zero-crossing rate over hopped (possibly overlapping) frames.
- *
- * Band energy uses two cascaded RBJ biquads — a high-pass at
- * `bandLowHz` and a low-pass at `bandHighHz` (both Q = 1/√2,
- * Audio-EQ-Cookbook closed forms) — rather than an FFT or Goertzel
- * bins: a Butterworth-ish cascade gives a true maximally-flat band in
- * one O(N) pass, and the filter state persists across hops, which is
- * exactly right for streaming (no window-boundary energy jitter).
- * Goertzel measures single frequencies (DTMF-style) — wrong tool for
- * broadband speech energy.
- */
-
-import type { FeatureConfig, FrameFeatures } from './types.js';
-import { DEFAULT_FEATURE_CONFIG } from './types.js';
-
-interface BiquadCoeffs {
-  b0: number;
-  b1: number;
-  b2: number;
-  a1: number;
-  a2: number;
-}
-
-interface BiquadState {
-  x1: number;
-  x2: number;
-  y1: number;
-  y2: number;
-}
-
-export interface BandpassState {
-  coeffs: { hp: BiquadCoeffs; lp: BiquadCoeffs };
-  hp: BiquadState;
-  lp: BiquadState;
-}
-
-function highpassCoeffs(sampleRate: number, freqHz: number): BiquadCoeffs {
-  const w0 = (2 * Math.PI * freqHz) / sampleRate;
-  const cosW0 = Math.cos(w0);
-  // alpha = sin(w0) / (2Q) with Butterworth Q = 1/√2.
-  const alpha = Math.sin(w0) / (2 * Math.SQRT1_2);
-  const a0 = 1 + alpha;
-  return {
-    b0: (1 + cosW0) / 2 / a0,
-    b1: -(1 + cosW0) / a0,
-    b2: (1 + cosW0) / 2 / a0,
-    a1: (-2 * cosW0) / a0,
-    a2: (1 - alpha) / a0,
-  };
-}
-
-function lowpassCoeffs(sampleRate: number, freqHz: number): BiquadCoeffs {
-  const w0 = (2 * Math.PI * freqHz) / sampleRate;
-  const cosW0 = Math.cos(w0);
-  const alpha = Math.sin(w0) / (2 * Math.SQRT1_2);
-  const a0 = 1 + alpha;
-  return {
-    b0: (1 - cosW0) / 2 / a0,
-    b1: (1 - cosW0) / a0,
-    b2: (1 - cosW0) / 2 / a0,
-    a1: (-2 * cosW0) / a0,
-    a2: (1 - alpha) / a0,
-  };
-}
-
-/** Create bandpass state for a sample rate (corners derived at init). */
-export function createBandpass(
-  sampleRate: number,
-  lowHz = DEFAULT_FEATURE_CONFIG.bandLowHz,
-  highHz = DEFAULT_FEATURE_CONFIG.bandHighHz,
-): BandpassState {
-  return {
-    coeffs: { hp: highpassCoeffs(sampleRate, lowHz), lp: lowpassCoeffs(sampleRate, highHz) },
-    hp: { x1: 0, x2: 0, y1: 0, y2: 0 },
-    lp: { x1: 0, x2: 0, y1: 0, y2: 0 },
-  };
-}
-
-function biquadRun(
-  coeffs: BiquadCoeffs,
-  state: BiquadState,
-  input: Float32Array,
-  output: Float32Array,
-): BiquadState {
-  let { x1, x2, y1, y2 } = state;
-  const { b0, b1, b2, a1, a2 } = coeffs;
-  for (let i = 0; i < input.length; i++) {
-    const x0 = input[i];
-    const y0 = b0 * x0 + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2;
-    output[i] = y0;
-    x2 = x1;
-    x1 = x0;
-    y2 = y1;
-    y1 = y0;
-  }
-  return { x1, x2, y1, y2 };
-}
-
-/**
- * Run the bandpass over a chunk. Pure in the state: returns a NEW state,
- * writes filtered samples into `output` (may alias `input`).
- */
-export function bandpassRun(
-  state: BandpassState,
-  input: Float32Array,
-  output: Float32Array,
-): BandpassState {
-  const hp = biquadRun(state.coeffs.hp, state.hp, input, output);
-  const lp = biquadRun(state.coeffs.lp, state.lp, output, output);
-  return { coeffs: state.coeffs, hp, lp };
-}
-
-export interface FeatureState {
-  sampleRate: number;
-  config: FeatureConfig;
-  /** Hop between frame starts, in samples. */
-  hopSamples: number;
-  bandpass: BandpassState;
-  /** Unconsumed raw samples (the next frame's prefix). */
-  rawCarry: Float32Array;
-  /** Bandpassed twin of `rawCarry`. */
-  bandCarry: Float32Array;
-  /** Absolute sample index of `rawCarry[0]` from the start of the signal. */
-  carryStartSample: number;
-}
-
-export function createFeatureState(
-  sampleRate: number,
-  config?: Partial<FeatureConfig>,
-): FeatureState {
-  const merged: FeatureConfig = { ...DEFAULT_FEATURE_CONFIG, ...config };
-  return {
-    sampleRate,
-    config: merged,
-    hopSamples: Math.max(1, Math.round(merged.hopSec * sampleRate)),
-    bandpass: createBandpass(sampleRate, merged.bandLowHz, merged.bandHighHz),
-    rawCarry: new Float32Array(0),
-    bandCarry: new Float32Array(0),
-    carryStartSample: 0,
-  };
-}
-
-function concatF32(a: Float32Array, b: Float32Array): Float32Array {
-  if (a.length === 0) return b;
-  const out = new Float32Array(a.length + b.length);
-  out.set(a, 0);
-  out.set(b, a.length);
-  return out;
-}
-
-function frameOf(
-  raw: Float32Array,
-  band: Float32Array,
-  start: number,
-  size: number,
-  startSample: number,
-  sampleRate: number,
-): FrameFeatures {
-  let sumSq = 0;
-  let bandSumSq = 0;
-  let crossings = 0;
-  let prev = raw[start];
-  for (let i = 0; i < size; i++) {
-    const r = raw[start + i];
-    const b = band[start + i];
-    sumSq += r * r;
-    bandSumSq += b * b;
-    if (i > 0 && ((r >= 0 && prev < 0) || (r < 0 && prev >= 0))) crossings++;
-    prev = r;
-  }
-  return {
-    tSec: (startSample + size / 2) / sampleRate,
-    rms: Math.sqrt(sumSq / size),
-    bandEnergy: bandSumSq / size,
-    zcr: size > 1 ? crossings / (size - 1) : 0,
-  };
-}
-
-/**
- * Feed a hop of PCM. Returns a new state plus every complete frame the
- * buffered signal now covers (usually 0–1 for worklet-sized hops, more
- * for large chunks).
- */
-export function featureStep(
-  state: FeatureState,
-  hop: Float32Array,
-): { state: FeatureState; frames: FrameFeatures[] } {
-  const banded = new Float32Array(hop.length);
-  const bandpass = bandpassRun(state.bandpass, hop, banded);
-
-  const raw = concatF32(state.rawCarry, hop);
-  const band = concatF32(state.bandCarry, banded);
-  const { frameSize } = state.config;
-  const frames: FrameFeatures[] = [];
-
-  let cursor = 0;
-  while (raw.length - cursor >= frameSize) {
-    frames.push(
-      frameOf(raw, band, cursor, frameSize, state.carryStartSample + cursor, state.sampleRate),
-    );
-    cursor += state.hopSamples;
-  }
-
-  return {
-    state: {
-      ...state,
-      bandpass,
-      // slice(), not subarray(): the carry must OWN its memory. `raw` can
-      // alias the caller's hop buffer, and callers may recycle those
-      // (e.g. Chrome invalidates buffers transferred out of an
-      // AudioWorklet after the message handler returns) — a retained
-      // view would silently read zeros on the next step.
-      rawCarry: raw.slice(cursor),
-      bandCarry: band.slice(cursor),
-      carryStartSample: state.carryStartSample + cursor,
-    },
-    frames,
-  };
-}
-
-/** Batch feature extraction over a whole take (the aligner's entry point). */
-export function extractFrameFeatures(
-  pcm: Float32Array,
-  sampleRate: number,
-  config?: Partial<FeatureConfig>,
-): FrameFeatures[] {
-  const state = createFeatureState(sampleRate, config);
-  return featureStep(state, pcm).frames;
-}

package/src/narration/index.ts

@@ -1,86 +0,0 @@
-/**
- * @bendyline/squisq/narration — the narration/teleprompter engine.
- *
- * Pure TypeScript, zero dependencies, no DOM/Node APIs: a script model
- * built from a Doc, streaming DSP (band features → adaptive VAD →
- * syllable nuclei), a deterministic voice-adaptive pacing controller,
- * and an offline DTW aligner that turns a recorded take into word/block
- * timestamps. Determinism is a hard rule here — no Date.now, no
- * Math.random, no timers — so every stage is unit-testable in Node
- * with synthetic PCM.
- */
-
-export type {
-  ScriptToken,
-  ScriptBlockRange,
-  NarrationScript,
-  FrameFeatures,
-  FeatureConfig,
-  VadConfig,
-  NucleiConfig,
-  PacingConfig,
-  AlignConfig,
-  WordTiming,
-  NarrationBlockRange,
-  NarrationAlignment,
-} from './types.js';
-export {
-  DEFAULT_FEATURE_CONFIG,
-  DEFAULT_VAD_CONFIG,
-  DEFAULT_NUCLEI_CONFIG,
-  DEFAULT_PACING_CONFIG,
-  DEFAULT_ALIGN_CONFIG,
-} from './types.js';
-
-export { estimateSyllables } from './syllables.js';
-
-export {
-  buildNarrationScript,
-  expectedSyllablesAt,
-  wordPosAtExpectedSyllables,
-  wordIndexAtChar,
-  wordIndexAtTime,
-  type BuildScriptOptions,
-} from './script.js';
-
-export {
-  createBandpass,
-  bandpassRun,
-  createFeatureState,
-  featureStep,
-  extractFrameFeatures,
-  type BandpassState,
-  type FeatureState,
-} from './features.js';
-
-export { createVadState, vadStep, type VadState } from './vad.js';
-
-export { createNucleiState, nucleiStep, detectSyllableOnsets, type NucleiState } from './nuclei.js';
-
-export {
-  createPacingState,
-  pacingStep,
-  reanchorPacing,
-  type PacingState,
-  type PacingTick,
-} from './pacing.js';
-
-export {
-  createNarrationSession,
-  narrationSessionStep,
-  reanchorSession,
-  type NarrationSessionConfig,
-  type NarrationSessionState,
-} from './session.js';
-
-export { traceWordPosAt, downsampleTrace, type NarrationTrace, type TraceSample } from './trace.js';
-
-export { alignNarration, type AlignInput } from './align.js';
-
-export {
-  buildNarrationTimingJson,
-  parseNarrationTimingJson,
-  type NarrationTimingJsonV3,
-  type NarrationTimingBlock,
-  type BuildNarrationTimingOptions,
-} from './sidecar.js';

package/src/narration/nuclei.ts

@@ -1,118 +0,0 @@
-/**
- * Syllable-nuclei detection: vowel nuclei show up as peaks in a
- * low-passed amplitude envelope of the speech-band signal.
- *
- * Envelope = one-pole low-pass (≈10 Hz) over √bandEnergy (amplitude
- * domain, so peaks are nuclei rather than energy spikes). A peak is
- * emitted one frame late (local-max needs one frame of lookahead —
- * ~20 ms live latency) and accepted only when: VAD says speaking, it
- * clears the minimum inter-onset spacing, it is prominent relative to
- * the valley since the last accepted peak, and it clears a decaying
- * loudness reference (guards against ripples inside quiet speech).
- */
-
-import type { FrameFeatures, NucleiConfig } from './types.js';
-import { DEFAULT_NUCLEI_CONFIG } from './types.js';
-
-export interface NucleiState {
-  /** One-pole envelope of √bandEnergy. */
-  env: number;
-  prevEnv: number;
-  prevPrevEnv: number;
-  /** tSec of the previous frame (the peak-candidate time). */
-  prevTSec: number;
-  /** Envelope minimum since the last accepted peak. */
-  valley: number;
-  lastOnsetTSec: number;
-  /** Decaying loudness reference. */
-  peakRef: number;
-  lastTSec: number;
-  framesSeen: number;
-}
-
-export function createNucleiState(_config?: Partial<NucleiConfig>): NucleiState {
-  return {
-    env: 0,
-    prevEnv: 0,
-    prevPrevEnv: 0,
-    prevTSec: 0,
-    valley: Number.POSITIVE_INFINITY,
-    lastOnsetTSec: Number.NEGATIVE_INFINITY,
-    peakRef: 0,
-    lastTSec: 0,
-    framesSeen: 0,
-  };
-}
-
-/**
- * Pure per-frame step. `onset` is the tSec of a newly accepted syllable
- * nucleus (emitted once, at its peak frame), or null.
- */
-export function nucleiStep(
-  state: NucleiState,
-  frame: FrameFeatures,
-  speaking: boolean,
-  config?: Partial<NucleiConfig>,
-): { state: NucleiState; onset: number | null } {
-  const c = { ...DEFAULT_NUCLEI_CONFIG, ...config };
-  const amp = Math.sqrt(Math.max(0, frame.bandEnergy));
-  const dt = state.framesSeen > 0 ? Math.max(0, frame.tSec - state.lastTSec) : 0;
-
-  const env =
-    state.framesSeen > 0
-      ? state.env + (amp - state.env) * (1 - Math.exp(-2 * Math.PI * c.envelopeHz * dt))
-      : amp;
-  const peakRef = Math.max(env, state.peakRef * Math.exp(-(dt || 0) / c.peakRefTauSec));
-
-  let onset: number | null = null;
-  let valley = state.framesSeen > 0 ? state.valley : env;
-  let lastOnsetTSec = state.lastOnsetTSec;
-
-  // Candidate peak at the PREVIOUS frame: env[t-1] ≥ env[t-2] && env[t-1] > env[t].
-  if (
-    state.framesSeen >= 2 &&
-    state.prevEnv >= state.prevPrevEnv &&
-    state.prevEnv > env &&
-    speaking &&
-    state.prevTSec - lastOnsetTSec >= c.minInterOnsetMs / 1000 &&
-    state.prevEnv >= valley * c.prominenceRatio &&
-    state.prevEnv >= peakRef * c.minRelPeak
-  ) {
-    onset = state.prevTSec;
-    lastOnsetTSec = state.prevTSec;
-    // Re-seed the valley at the accepted peak; it decays via min() below.
-    valley = state.prevEnv;
-  }
-  valley = Math.min(valley, env);
-
-  return {
-    state: {
-      env,
-      prevEnv: env,
-      prevPrevEnv: state.framesSeen > 0 ? state.prevEnv : env,
-      prevTSec: frame.tSec,
-      valley,
-      lastOnsetTSec,
-      peakRef,
-      lastTSec: frame.tSec,
-      framesSeen: state.framesSeen + 1,
-    },
-    onset,
-  };
-}
-
-/** Batch wrapper for the offline aligner. `vadFlags[i]` pairs with `frames[i]`. */
-export function detectSyllableOnsets(
-  frames: FrameFeatures[],
-  vadFlags: boolean[],
-  config?: Partial<NucleiConfig>,
-): number[] {
-  let state = createNucleiState(config);
-  const onsets: number[] = [];
-  for (let i = 0; i < frames.length; i++) {
-    const step = nucleiStep(state, frames[i], vadFlags[i] ?? false, config);
-    state = step.state;
-    if (step.onset !== null) onsets.push(step.onset);
-  }
-  return onsets;
-}

package/src/narration/pacing.ts

@@ -1,211 +0,0 @@
-/**
- * The teleprompter pacing controller.
- *
- * A pure, deterministic step function: given the current state, one
- * analysis tick (VAD flag + syllable onsets), and the script, produce
- * the next state. The prompter position (`wordPos`) advances at a
- * velocity that (a) halts within ~250 ms of silence, (b) tracks the
- * reader's detected syllable rate while speaking, and (c) is corrected
- * by a PI loop on cumulative detected-vs-expected syllables so the
- * highlight can't drift arbitrarily far from the voice. Silence near an
- * expected paragraph/block break is legitimate and accrues no error.
- */
-
-import { expectedSyllablesAt, wordPosAtExpectedSyllables } from './script.js';
-import type { NarrationScript, PacingConfig } from './types.js';
-import { DEFAULT_PACING_CONFIG } from './types.js';
-
-export interface PacingTick {
-  tSec: number;
-  speaking: boolean;
-  /** Syllable onsets detected in this tick (0 or 1 for frame-sized ticks). */
-  onsets: number;
-}
-
-export interface PacingState {
-  /** Fractional word position — the active token is `floor(wordPos)`. */
-  wordPos: number;
-  /** Current scroll velocity in words per second. */
-  velocityWps: number;
-  /** EMA of the detected syllable rate (syl/s); 0 until two onsets arrive. */
-  emaSylPerSec: number;
-  lastOnsetTSec: number | null;
-  /** Syllables detected since the last anchor. */
-  cumDetectedSyl: number;
-  /** Expected cumulative syllables at the last anchor position. */
-  anchorExpectedSyl: number;
-  /** PI integral of the cumulative-syllable error (syl·s). */
-  errIntegral: number;
-  /** Continuous silence observed so far (s). */
-  silenceSec: number;
-  lastTSec: number | null;
-  /** True when silent and fully stopped. */
-  halted: boolean;
-}
-
-export function createPacingState(startWordPos = 0): PacingState {
-  return {
-    wordPos: startWordPos,
-    velocityWps: 0,
-    emaSylPerSec: 0,
-    lastOnsetTSec: null,
-    cumDetectedSyl: 0,
-    anchorExpectedSyl: 0,
-    errIntegral: 0,
-    silenceSec: 0,
-    lastTSec: null,
-    halted: true,
-  };
-}
-
-function clamp(value: number, min: number, max: number): number {
-  return value < min ? min : value > max ? max : value;
-}
-
-/**
- * Mean syllables-per-SPOKEN-word over a short window ahead of the prompter
- * position. Standalone punctuation (0 syllables, `spoken: false`) is
- * excluded so it doesn't deflate the average.
- */
-function localSyllablesPerWord(
-  script: NarrationScript,
-  wordPos: number,
-  windowWords: number,
-): number {
-  const n = script.tokens.length;
-  if (n === 0) return 1.4;
-  const start = clamp(Math.floor(wordPos), 0, n - 1);
-  let sum = 0;
-  let count = 0;
-  for (let i = start; i < n && count < windowWords; i++) {
-    if (!script.tokens[i].spoken) continue;
-    sum += script.tokens[i].syllables;
-    count += 1;
-  }
-  // Default to the English average (~1.4) when the window has no words yet.
-  return count > 0 ? Math.max(1, sum / count) : 1.4;
-}
-
-/** Pure controller step — same inputs always yield the same output state. */
-export function pacingStep(
-  state: PacingState,
-  tick: PacingTick,
-  script: NarrationScript,
-  config?: Partial<PacingConfig>,
-): PacingState {
-  const c = { ...DEFAULT_PACING_CONFIG, ...config };
-  const baseWps = c.baseWpm / 60;
-  const dt = clamp(tick.tSec - (state.lastTSec ?? tick.tSec), 0, 0.25);
-
-  // 1. Onset bookkeeping: cumulative count + inter-onset-interval rate EMA.
-  let emaSylPerSec = state.emaSylPerSec;
-  let lastOnsetTSec = state.lastOnsetTSec;
-  let cumDetectedSyl = state.cumDetectedSyl;
-  for (let k = 0; k < tick.onsets; k++) {
-    if (lastOnsetTSec !== null) {
-      const ioi = tick.tSec - lastOnsetTSec;
-      if (ioi > 0) {
-        const instRate = clamp(1 / ioi, 0.5, 12);
-        const alpha = 1 - Math.exp(-ioi / c.rateEmaTauSec);
-        emaSylPerSec =
-          emaSylPerSec === 0 ? instRate : emaSylPerSec + (instRate - emaSylPerSec) * alpha;
-      }
-    }
-    lastOnsetTSec = tick.tSec;
-    cumDetectedSyl += 1;
-  }
-
-  // 2–3. Cruise velocity: BLEND the user's set pace with the voice-derived
-  // rate. The old design used `baseWps · clamp(voiceWps / baseWps)`, in
-  // which baseWps algebraically cancels in the unclamped range — so the WPM
-  // slider did nothing and a chronically under-counting detector dragged
-  // the prompter slow with no way to compensate. Blending keeps the set
-  // pace as a real feedforward term (WPM always felt), while the voice
-  // steers within a band. Then clamp to a band around the set pace so
-  // neither term alone can stall or run away.
-  const sylPerWordLocal = localSyllablesPerWord(script, state.wordPos, c.sylWindowWords);
-  const voiceWps = emaSylPerSec > 0 ? emaSylPerSec / sylPerWordLocal : baseWps;
-  const cruise = clamp(
-    c.voiceBlend * voiceWps + (1 - c.voiceBlend) * baseWps,
-    baseWps * c.minRateMult,
-    baseWps * c.maxRateMult,
-  );
-
-  // 4–5. FORWARD-ONLY position correction. `err` compares where the
-  // prompter is (expected syllables at wordPos) against how many syllables
-  // the voice has actually delivered. err < 0 → the reader is AHEAD of the
-  // prompter → speed up to catch them. err > 0 → the prompter looks ahead
-  // of the reader, but this is exactly the signal a chronically
-  // under-counting syllable detector produces even when the prompter is
-  // really BEHIND — so we deliberately do NOT slow or snap backward on it.
-  // (The old symmetric PI pinned the prompter to the DETECTED count, so a
-  // detector catching half the syllables parked the prompter at half the
-  // reader's true position — the lag the user saw.) Slowing down is instead
-  // handled honestly by the cruise blend (a slow voice lowers voiceWps) and
-  // by halt-on-silence.
-  const err =
-    expectedSyllablesAt(script, state.wordPos) - (state.anchorExpectedSyl + cumDetectedSyl);
-  let errIntegral = clamp(Math.min(0, state.errIntegral + err * dt), -c.intClamp, 0);
-  const boost = err < 0 ? clamp(c.kP * -err + c.kI * -errIntegral, 0, c.maxCorrection) : 0;
-
-  // 6. Velocity: track while speaking, decay to a halt in silence.
-  let velocityWps = state.velocityWps;
-  let silenceSec = state.silenceSec;
-  if (tick.speaking) {
-    silenceSec = 0;
-    const target = Math.max(0, cruise * (1 + boost));
-    velocityWps += (target - velocityWps) * (1 - Math.exp(-dt / c.velSlewTauSec));
-  } else {
-    silenceSec += dt;
-    velocityWps *= Math.exp(-dt / c.haltTauSec);
-    if (velocityWps < 0.02) velocityWps = 0;
-  }
-
-  // 7. Hard resync forward only: when the reader has clearly outrun the
-  // prompter by more than a paragraph, jump ahead to catch up. Never jump
-  // backward (a false "prompter ahead" from under-counting must not yank
-  // the highlight back).
-  let wordPos = state.wordPos;
-  if (err < -c.resyncSyllables) {
-    wordPos = Math.max(
-      wordPos,
-      wordPosAtExpectedSyllables(script, state.anchorExpectedSyl + cumDetectedSyl),
-    );
-    errIntegral = 0;
-  }
-
-  // 8. Integrate.
-  wordPos = clamp(wordPos + velocityWps * dt, 0, script.tokens.length);
-
-  return {
-    wordPos,
-    velocityWps,
-    emaSylPerSec,
-    lastOnsetTSec,
-    cumDetectedSyl,
-    anchorExpectedSyl: state.anchorExpectedSyl,
-    errIntegral,
-    silenceSec,
-    lastTSec: tick.tSec,
-    halted: !tick.speaking && velocityWps === 0,
-  };
-}
-
-/**
- * Re-anchor after a manual nudge/scroll: the new position becomes the
- * zero-error reference. The learned rate EMA survives.
- */
-export function reanchorPacing(
-  state: PacingState,
-  wordPos: number,
-  script: NarrationScript,
-): PacingState {
-  const pos = clamp(wordPos, 0, script.tokens.length);
-  return {
-    ...state,
-    wordPos: pos,
-    anchorExpectedSyl: expectedSyllablesAt(script, pos),
-    cumDetectedSyl: 0,
-    errIntegral: 0,
-  };
-}