@maravilla-labs/frames 0.5.1 → 0.8.0

package/src/audio.ts

@@ -0,0 +1,349 @@
+/**
+ * Browser-side audio preview for a frames timeline.
+ *
+ * The render worker mixes audio with ffmpeg (the browser can't expose
+ * decoded audio to the capture pipeline), so this engine exists purely so
+ * the **editor / preview** plays the same mix the final video will have:
+ * multiple time-bound tracks (voiceover, background music, sfx), per-track
+ * gain + fades, and static-envelope ducking of music under voiceover (see
+ * [`ducking.ts`] — the exact curve the worker bakes in).
+ *
+ * Scrubbing (`applyState(t)`) stays silent by design — like every real NLE,
+ * audio is heard only during continuous playback. The preview host calls
+ * [`AudioEngine.playFrom`] on play and [`AudioEngine.stop`] on pause.
+ */
+
+import {
+  DEFAULT_DUCK,
+  duckAutomation,
+  mergeSegments,
+  type DuckParams,
+  type Segment,
+} from "./ducking.js";
+import { fadeAutomation, type FadeCurve } from "./fades.js";
+
+/** A timeline audio track, resolved for browser playback. Times in ms. */
+export type PreviewAudioTrack = {
+  /** Playable URL fetched + decoded for preview. */
+  src: string;
+  /** Timeline start, ms. */
+  at: number;
+  /** Play length on the timeline, ms. `null` = natural clip length. */
+  duration: number | null;
+  /** Start offset within the source, ms. */
+  seek: number;
+  /** Linear base gain (1 = unity). */
+  gain: number;
+  /** Fade-in / fade-out, ms. */
+  fadeIn: number;
+  fadeOut: number;
+  /** Shape of each fade (see `fades.ts`). Default `linear`. */
+  fadeInCurve?: FadeCurve;
+  fadeOutCurve?: FadeCurve;
+  /** "voiceover" drives ducking; "music" gets ducked when `duck`. */
+  role: "voiceover" | "music" | "sfx";
+  /** Whether this (music) track dips under voiceover. */
+  duck: boolean;
+  /** Optional per-track effect (structurally matches `AudioEffect`). */
+  effect?: { kind: "radio"; drive?: number; tone?: number; depth?: number };
+};
+
+type WindowAudioContext = typeof AudioContext;
+
+function getAudioContextCtor(): WindowAudioContext | null {
+  if (typeof window === "undefined") return null;
+  return (
+    window.AudioContext ||
+    (window as unknown as { webkitAudioContext?: WindowAudioContext })
+      .webkitAudioContext ||
+    null
+  );
+}
+
+export class AudioEngine {
+  private tracks: PreviewAudioTrack[];
+  private duck: DuckParams;
+  private ctx: AudioContext | null = null;
+  private master: GainNode | null = null;
+  private buffers = new Map<string, AudioBuffer>();
+  private active: AudioBufferSourceNode[] = [];
+  private muted = false;
+
+  constructor(tracks: PreviewAudioTrack[], duck: DuckParams = DEFAULT_DUCK) {
+    this.tracks = tracks;
+    this.duck = duck;
+  }
+
+  /** Merged voiceover regions (seconds) — what music ducks under. */
+  private voiceoverSegments(): Segment[] {
+    const segs = this.tracks
+      .filter((t) => t.role === "voiceover")
+      .map((t) => ({
+        start: t.at / 1000,
+        end: (t.at + (t.duration ?? 0)) / 1000,
+      }))
+      .filter((s) => s.end > s.start);
+    return mergeSegments(segs);
+  }
+
+  /** Fetch + decode every track's source (idempotent, cached by URL). */
+  private async ensureBuffers(): Promise<void> {
+    const Ctor = getAudioContextCtor();
+    if (!Ctor) return;
+    if (!this.ctx) {
+      this.ctx = new Ctor();
+      this.master = this.ctx.createGain();
+      this.master.connect(this.ctx.destination);
+    }
+    const ctx = this.ctx;
+    await Promise.all(
+      this.tracks.map(async (t) => {
+        if (this.buffers.has(t.src)) return;
+        try {
+          const resp = await fetch(t.src);
+          const raw = await resp.arrayBuffer();
+          const buf = await ctx.decodeAudioData(raw);
+          this.buffers.set(t.src, buf);
+        } catch (err) {
+          // A bad/missing audio URL must never wedge the preview — skip it.
+          // eslint-disable-next-line no-console
+          console.warn(`[@maravilla-labs/frames] audio decode failed for ${t.src}:`, err);
+        }
+      }),
+    );
+  }
+
+  /** Start (or restart) playback from timeline position `fromMs`. */
+  async playFrom(fromMs: number): Promise<void> {
+    await this.ensureBuffers();
+    if (!this.ctx || !this.master) return;
+    this.stop();
+    // Resume may reject if there's no user activation (autoplay policy). Don't
+    // let that abort scheduling — the caller invokes this from a click handler,
+    // so activation is usually present; if not, the context stays suspended and
+    // simply produces no sound rather than throwing.
+    if (this.ctx.state === "suspended") {
+      try {
+        await this.ctx.resume();
+      } catch {
+        /* no user activation yet — stay silent, don't throw */
+      }
+    }
+
+    const ctx = this.ctx;
+    const t0 = fromMs / 1000;
+    const startCtx = ctx.currentTime + 0.05; // small lead so scheduling lands
+    const segments = this.voiceoverSegments();
+    this.master.gain.value = this.muted ? 0 : 1;
+
+    for (const track of this.tracks) {
+      const buf = this.buffers.get(track.src);
+      if (!buf) continue;
+
+      const naturalMs = buf.duration * 1000 - track.seek;
+      const durMs = track.duration ?? Math.max(0, naturalMs);
+      const trackEnd = track.at + durMs;
+      if (t0 * 1000 >= trackEnd) continue; // already past this clip
+
+      // When (timeline ms) the clip actually begins sounding from the playhead.
+      const playStart = Math.max(t0 * 1000, track.at);
+      const delaySec = (playStart - t0 * 1000) / 1000;
+      const offsetInClip = (track.seek + (playStart - track.at)) / 1000;
+      const remainingSec = (trackEnd - playStart) / 1000;
+      if (remainingSec <= 0) continue;
+
+      // Map a timeline time (seconds) to this AudioContext's clock.
+      const toCtx = (timelineSec: number) =>
+        Math.max(startCtx, startCtx + (timelineSec - t0));
+
+      // Two gains, in series: the fade-in and the fade-out are separate
+      // automations that MULTIPLY where they overlap — the same as ffmpeg's two
+      // chained `afade` filters, and it keeps each curve's events on its own
+      // AudioParam so they can never collide.
+      const fadeGain = ctx.createGain();
+      const fadeOutGain = ctx.createGain();
+      fadeGain.connect(fadeOutGain);
+      this.applyFades(fadeGain, fadeOutGain, track, t0, toCtx);
+
+      let tail: AudioNode = fadeOutGain;
+      if (track.role === "music" && track.duck && segments.length) {
+        const duckGain = ctx.createGain();
+        this.applyDuck(duckGain, track.gain, segments, toCtx);
+        fadeOutGain.connect(duckGain);
+        tail = duckGain;
+      }
+      tail.connect(this.master);
+
+      // Optional effect (e.g. radio voice) sits between the source and the
+      // fade/duck gains, so it shapes the dry signal exactly like the ffmpeg
+      // render does.
+      const node = ctx.createBufferSource();
+      node.buffer = buf;
+      const radio = track.effect?.kind === "radio" ? buildRadioChain(ctx, track.effect) : null;
+      if (radio) {
+        node.connect(radio.input);
+        radio.output.connect(fadeGain);
+      } else {
+        node.connect(fadeGain);
+      }
+      node.start(startCtx + delaySec, offsetInClip, remainingSec);
+      this.active.push(node);
+    }
+  }
+
+  /**
+   * Per-track base gain + fade-in on `inGain`, fade-out on `outGain`, each
+   * shaped by its curve. `t0` is the playhead (seconds): a fade the playhead is
+   * already inside continues from where it is rather than restarting.
+   */
+  private applyFades(
+    inGain: GainNode,
+    outGain: GainNode,
+    track: PreviewAudioTrack,
+    t0: number,
+    toCtx: (timelineSec: number) => number,
+  ): void {
+    const base = track.role === "music" && track.duck ? 1 : track.gain;
+    const startSec = track.at / 1000;
+    const durMs = track.duration ?? 0;
+    const endSec = (track.at + durMs) / 1000;
+    const fadeInSec = Math.max(0, track.fadeIn) / 1000;
+    const fadeOutSec = Math.max(0, track.fadeOut) / 1000;
+    const inCurve = track.fadeInCurve ?? "linear";
+    const outCurve = track.fadeOutCurve ?? "linear";
+
+    // Fade-in (carries the base gain).
+    if (fadeInSec > 0) {
+      const inEnd = startSec + fadeInSec;
+      if (t0 >= inEnd) {
+        inGain.gain.setValueAtTime(base, toCtx(t0));
+      } else {
+        const from = Math.max(0, (t0 - startSec) / fadeInSec);
+        const at = Math.max(startSec, t0);
+        // A value event AT the curve's start time counts as an overlap and
+        // throws — hold silence only when the clip starts later than the playhead.
+        if (at > t0) inGain.gain.setValueAtTime(0, toCtx(t0));
+        inGain.gain.setValueCurveAtTime(fadeAutomation(inCurve, "in", base, from), toCtx(at), inEnd - at);
+      }
+    } else {
+      inGain.gain.setValueAtTime(base, toCtx(t0));
+    }
+
+    // Fade-out (unity outside the fade).
+    const outStart = Math.max(startSec, endSec - fadeOutSec);
+    const hasOut = fadeOutSec > 0 && durMs > 0 && endSec > outStart && t0 < endSec;
+    const outAt = Math.max(outStart, t0);
+    if (!hasOut || outAt > t0) outGain.gain.setValueAtTime(1, toCtx(t0));
+    if (hasOut) {
+      const from = Math.max(0, (t0 - outStart) / (endSec - outStart));
+      outGain.gain.setValueCurveAtTime(fadeAutomation(outCurve, "out", 1, from), toCtx(outAt), endSec - outAt);
+    }
+  }
+
+  /** Ducking automation (scaled by the music track's base gain) on `g`. */
+  private applyDuck(
+    g: GainNode,
+    base: number,
+    segments: Segment[],
+    toCtx: (timelineSec: number) => number,
+  ): void {
+    const points = duckAutomation(segments, base, this.duck);
+    let first = true;
+    for (const [timeSec, value] of points) {
+      if (first) {
+        g.gain.setValueAtTime(value, toCtx(timeSec));
+        first = false;
+      } else {
+        g.gain.linearRampToValueAtTime(value, toCtx(timeSec));
+      }
+    }
+  }
+
+  /** Stop all currently-playing sources (idempotent). */
+  stop(): void {
+    for (const node of this.active) {
+      try {
+        node.stop();
+      } catch {
+        /* already stopped */
+      }
+    }
+    this.active = [];
+  }
+
+  setMuted(muted: boolean): void {
+    this.muted = muted;
+    if (this.master) this.master.gain.value = muted ? 0 : 1;
+  }
+}
+
+function clamp01(v: number): number {
+  return v < 0 ? 0 : v > 1 ? 1 : v;
+}
+
+/**
+ * Resolved "radio voice" parameters. The exact same formulas are mirrored by
+ * the ffmpeg render (worker `audio_mix.rs`) so preview == export:
+ * a band-pass (high-pass + low-pass) for the radio band, a low-shelf for
+ * "depth", and a tanh waveshaper (pushed by `pre`, tamed by `post`) for drive.
+ */
+export function radioParams(effect: { drive?: number; tone?: number; depth?: number }): {
+  hp: number;
+  lp: number;
+  shelfGainDb: number;
+  pre: number;
+  post: number;
+  k: number;
+} {
+  const drive = clamp01(effect.drive ?? 0.4);
+  const tone = clamp01(effect.tone ?? 0.4);
+  const depth = clamp01(effect.depth ?? 0.5);
+  return {
+    hp: 500 - 350 * tone, // 500Hz (telephone) → 150Hz (open)
+    lp: 2400 + 2800 * tone, // 2400Hz (narrow) → 5200Hz (bright)
+    shelfGainDb: depth * 12, // up to +12 dB low-shelf
+    pre: 1 + drive * 3, // drive into the saturator
+    post: 1 / (1 + drive * 1.6), // make-up / tame
+    k: 1 + drive * 5, // tanh steepness
+  };
+}
+
+function tanhCurve(k: number) {
+  const n = 1024;
+  const curve = new Float32Array(n);
+  for (let i = 0; i < n; i++) {
+    const x = (i / (n - 1)) * 2 - 1;
+    curve[i] = Math.tanh(k * x);
+  }
+  return curve;
+}
+
+function buildRadioChain(
+  ctx: AudioContext,
+  effect: { drive?: number; tone?: number; depth?: number },
+): { input: AudioNode; output: AudioNode } {
+  const p = radioParams(effect);
+  const hp = ctx.createBiquadFilter();
+  hp.type = "highpass";
+  hp.frequency.value = p.hp;
+  const lp = ctx.createBiquadFilter();
+  lp.type = "lowpass";
+  lp.frequency.value = p.lp;
+  const shelf = ctx.createBiquadFilter();
+  shelf.type = "lowshelf";
+  shelf.frequency.value = 180;
+  shelf.gain.value = p.shelfGainDb;
+  const pre = ctx.createGain();
+  pre.gain.value = p.pre;
+  const shaper = ctx.createWaveShaper();
+  shaper.curve = tanhCurve(p.k);
+  shaper.oversample = "2x";
+  const post = ctx.createGain();
+  post.gain.value = p.post;
+  hp.connect(lp);
+  lp.connect(shelf);
+  shelf.connect(pre);
+  pre.connect(shaper);
+  shaper.connect(post);
+  return { input: hp, output: post };
+}

package/src/ducking.ts

@@ -0,0 +1,103 @@
+/**
+ * Static-envelope ducking — the single source of truth for "music dips
+ * under voiceover" shared by the browser preview ([`audio.ts`]) and the
+ * Rust render worker (which mirrors this math when it builds the ffmpeg
+ * `volume` expression).
+ *
+ * The model is deliberately *static*: the dip is computed purely from the
+ * voiceover clips' timeline positions (`at` / `duration`), NOT from the
+ * voiceover's actual loudness. That makes it deterministic — the same
+ * timeline produces the same envelope every render — and, crucially, lets
+ * the browser preview reproduce the exact gain curve the final mp4 will
+ * have (a sidechain compressor could not be mirrored frame-for-frame in
+ * Web Audio).
+ *
+ * All times here are in **seconds** (Web Audio + ffmpeg both want seconds);
+ * the DSL stores milliseconds, so callers convert at the boundary.
+ */
+
+/** Tunable ducking parameters. Defaults match the shorts editor. */
+export type DuckParams = {
+  /** Music multiplier while voiceover is active. `0.25` ≈ −12 dB. */
+  level: number;
+  /** Linear ramp into / out of the dip, seconds. */
+  ramp: number;
+};
+
+export const DEFAULT_DUCK: DuckParams = { level: 0.25, ramp: 0.25 };
+
+/** A half-open `[start, end)` interval on the timeline, seconds. */
+export type Segment = { start: number; end: number };
+
+/**
+ * Merge overlapping / touching voiceover segments into a disjoint, sorted
+ * set. Merging is what makes the preview automation and the ffmpeg
+ * expression agree even when voiceover clips overlap: a union of regions
+ * is unambiguous, whereas per-clip ramps could otherwise fight each other.
+ */
+export function mergeSegments(segments: Segment[]): Segment[] {
+  const sorted = segments
+    .filter((s) => s.end > s.start)
+    .sort((a, b) => a.start - b.start);
+  const out: Segment[] = [];
+  for (const s of sorted) {
+    const last = out[out.length - 1];
+    if (last && s.start <= last.end) {
+      last.end = Math.max(last.end, s.end);
+    } else {
+      out.push({ ...s });
+    }
+  }
+  return out;
+}
+
+/**
+ * The ducking multiplier at time `t` (seconds) for a music track, given the
+ * merged voiceover `segments`. `1` = full volume, `level` = fully ducked,
+ * with linear ramps of width `ramp` on each edge. Used by the preview to
+ * sample the curve and by tests to assert the Rust expression matches.
+ */
+export function duckMultiplier(
+  t: number,
+  segments: Segment[],
+  duck: DuckParams = DEFAULT_DUCK,
+): number {
+  let m = 1;
+  for (const { start, end } of segments) {
+    const seg = segMultiplier(t, start, end, duck);
+    if (seg < m) m = seg;
+  }
+  return m;
+}
+
+function segMultiplier(t: number, s: number, e: number, duck: DuckParams): number {
+  const { level: d, ramp: r } = duck;
+  if (t <= s - r || t >= e + r) return 1;
+  if (t < s) return 1 - (1 - d) * (t - (s - r)) / r; // ramp down
+  if (t <= e) return d; // hold
+  return d + (1 - d) * (t - e) / r; // ramp up
+}
+
+/**
+ * Gain-automation breakpoints `[time, value]` for a music track's
+ * `GainNode`, scaled by `base` gain. The preview replays these with
+ * `setValueAtTime` / `linearRampToValueAtTime` so the dip in the player
+ * matches the dip baked into the render. Times are absolute timeline
+ * seconds.
+ */
+export function duckAutomation(
+  segments: Segment[],
+  base: number,
+  duck: DuckParams = DEFAULT_DUCK,
+): Array<[number, number]> {
+  const merged = mergeSegments(segments);
+  const points: Array<[number, number]> = [[0, base]];
+  const { level: d, ramp: r } = duck;
+  for (const { start, end } of merged) {
+    points.push([Math.max(0, start - r), base]); // begin ramp down
+    points.push([start, base * d]); // fully ducked
+    points.push([end, base * d]); // hold to clip end
+    points.push([end + r, base]); // ramp back up
+  }
+  return points;
+}

package/src/fades.ts

@@ -0,0 +1,81 @@
+/**
+ * Fade curves for audio tracks — the shape of a fade-in / fade-out, not just
+ * its length.
+ *
+ * The render worker mixes with ffmpeg's `afade`, so these are ffmpeg's OWN
+ * curve formulas (libavfilter `af_afade.c`, `fade_gain`), evaluated here for the
+ * browser preview. The worker maps each name to the matching `afade` curve
+ * (`audio_mix.rs`, `ffmpeg_fade_curve`), which is what keeps preview == export.
+ *
+ * | name          | afade | shape                                              |
+ * |---------------|-------|----------------------------------------------------|
+ * | `linear`      | tri   | straight gain ramp — a dip in the middle           |
+ * | `equal-power` | qsin  | quarter sine — constant perceived loudness         |
+ * | `s-curve`     | hsin  | half sine — gentle at both ends                     |
+ * | `exponential` | exp   | slow start, fast finish (~-100 dB floor)           |
+ * | `logarithmic` | log   | fast start, slow finish                            |
+ *
+ * `linear` is the default because it is what `afade` does when no curve is
+ * named, so a track written before curves existed renders unchanged.
+ */
+
+export type FadeCurve = "linear" | "equal-power" | "s-curve" | "exponential" | "logarithmic";
+
+/** Every curve, in the order a picker should offer them. */
+export const FADE_CURVES: readonly FadeCurve[] = [
+  "linear",
+  "equal-power",
+  "s-curve",
+  "exponential",
+  "logarithmic",
+];
+
+/** Resolve an untrusted value to a known curve (unknown ⇒ `linear`). */
+export function resolveFadeCurve(value: unknown): FadeCurve {
+  return typeof value === "string" && (FADE_CURVES as readonly string[]).includes(value)
+    ? (value as FadeCurve)
+    : "linear";
+}
+
+/**
+ * Gain (0..1) of a FADE-IN at progress `x` (0 = start of the fade, 1 = end).
+ * A fade-out at progress `p` is `fadeGain(curve, 1 - p)` — exactly how `afade`
+ * runs its curve backwards for `t=out`.
+ */
+export function fadeGain(curve: FadeCurve, x: number): number {
+  const g = x <= 0 ? 0 : x >= 1 ? 1 : x;
+  switch (curve) {
+    case "equal-power":
+      return Math.sin((g * Math.PI) / 2);
+    case "s-curve":
+      return (1 - Math.cos(g * Math.PI)) / 2;
+    case "exponential":
+      return g >= 1 ? 1 : Math.exp(-11.512925464970227 * (1 - g));
+    case "logarithmic":
+      return g <= 0 ? 0 : Math.min(1, Math.max(0, 1 + 0.2 * Math.log10(g)));
+    case "linear":
+    default:
+      return g;
+  }
+}
+
+/**
+ * Sampled gain automation for `AudioParam.setValueCurveAtTime`, scaled by
+ * `base`. `from` lets a curve start part-way through (playback that begins in
+ * the middle of a fade), so the preview never restarts a fade from silence.
+ */
+export function fadeAutomation(
+  curve: FadeCurve,
+  direction: "in" | "out",
+  base: number,
+  from = 0,
+  samples = 128,
+): Float32Array {
+  const out = new Float32Array(Math.max(2, samples));
+  const start = from <= 0 ? 0 : from >= 1 ? 1 : from;
+  for (let i = 0; i < out.length; i++) {
+    const p = start + ((1 - start) * i) / (out.length - 1);
+    out[i] = base * fadeGain(curve, direction === "in" ? p : 1 - p);
+  }
+  return out;
+}