@maravilla-labs/frames 0.5.0 → 0.7.0

package/src/ducking.ts

@@ -0,0 +1,103 @@
+/**
+ * Static-envelope ducking — the single source of truth for "music dips
+ * under voiceover" shared by the browser preview ([`audio.ts`]) and the
+ * Rust render worker (which mirrors this math when it builds the ffmpeg
+ * `volume` expression).
+ *
+ * The model is deliberately *static*: the dip is computed purely from the
+ * voiceover clips' timeline positions (`at` / `duration`), NOT from the
+ * voiceover's actual loudness. That makes it deterministic — the same
+ * timeline produces the same envelope every render — and, crucially, lets
+ * the browser preview reproduce the exact gain curve the final mp4 will
+ * have (a sidechain compressor could not be mirrored frame-for-frame in
+ * Web Audio).
+ *
+ * All times here are in **seconds** (Web Audio + ffmpeg both want seconds);
+ * the DSL stores milliseconds, so callers convert at the boundary.
+ */
+
+/** Tunable ducking parameters. Defaults match the shorts editor. */
+export type DuckParams = {
+  /** Music multiplier while voiceover is active. `0.25` ≈ −12 dB. */
+  level: number;
+  /** Linear ramp into / out of the dip, seconds. */
+  ramp: number;
+};
+
+export const DEFAULT_DUCK: DuckParams = { level: 0.25, ramp: 0.25 };
+
+/** A half-open `[start, end)` interval on the timeline, seconds. */
+export type Segment = { start: number; end: number };
+
+/**
+ * Merge overlapping / touching voiceover segments into a disjoint, sorted
+ * set. Merging is what makes the preview automation and the ffmpeg
+ * expression agree even when voiceover clips overlap: a union of regions
+ * is unambiguous, whereas per-clip ramps could otherwise fight each other.
+ */
+export function mergeSegments(segments: Segment[]): Segment[] {
+  const sorted = segments
+    .filter((s) => s.end > s.start)
+    .sort((a, b) => a.start - b.start);
+  const out: Segment[] = [];
+  for (const s of sorted) {
+    const last = out[out.length - 1];
+    if (last && s.start <= last.end) {
+      last.end = Math.max(last.end, s.end);
+    } else {
+      out.push({ ...s });
+    }
+  }
+  return out;
+}
+
+/**
+ * The ducking multiplier at time `t` (seconds) for a music track, given the
+ * merged voiceover `segments`. `1` = full volume, `level` = fully ducked,
+ * with linear ramps of width `ramp` on each edge. Used by the preview to
+ * sample the curve and by tests to assert the Rust expression matches.
+ */
+export function duckMultiplier(
+  t: number,
+  segments: Segment[],
+  duck: DuckParams = DEFAULT_DUCK,
+): number {
+  let m = 1;
+  for (const { start, end } of segments) {
+    const seg = segMultiplier(t, start, end, duck);
+    if (seg < m) m = seg;
+  }
+  return m;
+}
+
+function segMultiplier(t: number, s: number, e: number, duck: DuckParams): number {
+  const { level: d, ramp: r } = duck;
+  if (t <= s - r || t >= e + r) return 1;
+  if (t < s) return 1 - (1 - d) * (t - (s - r)) / r; // ramp down
+  if (t <= e) return d; // hold
+  return d + (1 - d) * (t - e) / r; // ramp up
+}
+
+/**
+ * Gain-automation breakpoints `[time, value]` for a music track's
+ * `GainNode`, scaled by `base` gain. The preview replays these with
+ * `setValueAtTime` / `linearRampToValueAtTime` so the dip in the player
+ * matches the dip baked into the render. Times are absolute timeline
+ * seconds.
+ */
+export function duckAutomation(
+  segments: Segment[],
+  base: number,
+  duck: DuckParams = DEFAULT_DUCK,
+): Array<[number, number]> {
+  const merged = mergeSegments(segments);
+  const points: Array<[number, number]> = [[0, base]];
+  const { level: d, ramp: r } = duck;
+  for (const { start, end } of merged) {
+    points.push([Math.max(0, start - r), base]); // begin ramp down
+    points.push([start, base * d]); // fully ducked
+    points.push([end, base * d]); // hold to clip end
+    points.push([end + r, base]); // ramp back up
+  }
+  return points;
+}

package/src/index.ts

@@ -8,6 +8,19 @@
  *   frame to produce deterministic video output.
  */
 
+import { AudioEngine, type PreviewAudioTrack } from "./audio.js";
+
+export { AudioEngine } from "./audio.js";
+export type { PreviewAudioTrack } from "./audio.js";
+export {
+  DEFAULT_DUCK,
+  duckAutomation,
+  duckMultiplier,
+  mergeSegments,
+  type DuckParams,
+  type Segment,
+} from "./ducking.js";
+
 export type AnimationInstr = {
   /** Optional discriminator. Defaults to `"animation"`. */
   kind?: "animation";
@@ -67,9 +80,87 @@ export type CssClassInstr = {
   classes: string[];
 };
 
-export type Instr = AnimationInstr | VideoInstr | CssClassInstr;
+/** Role of an audio track. `"voiceover"` is what music ducks *under*;
+ * `"music"` is the background bed that gets ducked; `"sfx"` is a one-shot
+ * that is neither ducked nor a ducking trigger. */
+export type AudioRole = "voiceover" | "music" | "sfx";
+
+/**
+ * A per-track audio effect. `"radio"` is a band-limited, saturated "deep
+ * radio/telephone voice" — applied identically in the browser preview (Web
+ * Audio filter chain, [`audio.ts`]) and in the render (ffmpeg, worker-side), so
+ * preview matches export. All knobs are 0..1.
+ */
+export type AudioEffect = {
+  kind: "radio";
+  /** Overdrive / saturation amount (0 = clean, 1 = crunchy). Default 0.4. */
+  drive?: number;
+  /** Band brightness (0 = narrow telephone band, 1 = wide/open). Default 0.4. */
+  tone?: number;
+  /** Low-end "depth" boost for a deeper voice. Default 0.5. */
+  depth?: number;
+};
+
+/**
+ * An audio track on the timeline. Unlike the visual instructions it owns no
+ * DOM element — it's mixed into the final video by the render worker (ffmpeg)
+ * and played in the browser preview by the Web Audio engine ([`audio.ts`]).
+ * Multiple tracks coexist (e.g. an intro bed + a voiceover); music tracks
+ * dip under voiceover via static-envelope ducking (see [`ducking.ts`]).
+ */
+export type AudioInstr = {
+  /** Discriminator — required. */
+  kind: "audio";
+  /** When on the timeline the track starts, in milliseconds. */
+  at: number;
+  /** How long it plays from `at`, in ms. Defaults to the clip's natural length. */
+  duration?: number;
+  /** Start offset WITHIN the source file, in ms. Defaults to 0. */
+  seek?: number;
+  /**
+   * Playable URL for the browser preview (fetched + decoded via Web Audio).
+   * For a stored asset, a URL that resolves to it (e.g. `/_assets/<key>`).
+   */
+  src: string;
+  /**
+   * Storage key the render worker fetches + stages to mix into the final
+   * video. Without it the track is preview-only (it can't be rendered).
+   */
+  key?: string;
+  /** Track role. Defaults to `"music"`. */
+  role?: AudioRole;
+  /** Linear gain (1 = unity). Defaults to 1. */
+  gain?: number;
+  /** Fade-in / fade-out length from the clip edges, ms. Default 0. */
+  fadeIn?: number;
+  fadeOut?: number;
+  /** Whether a music track ducks under voiceover. Defaults to `role === "music"`. */
+  duck?: boolean;
+  /** Optional per-track effect (e.g. a "radio voice"). */
+  effect?: AudioEffect;
+};
+
+export type Instr = AnimationInstr | VideoInstr | CssClassInstr | AudioInstr;
 export type TimelineSchema = { duration: number; instructions: Instr[] };
 
+/** An audio track as exposed to the render worker via `window.__mvFrames.audio`.
+ * The worker stages `key` and builds the ffmpeg mix; `duration === null`
+ * means "use the source's natural length". Mirrors [`AudioInstr`] with
+ * defaults resolved. */
+export type AudioSlotInfo = {
+  /** Storage key the worker stages (absent ⇒ preview-only, not rendered). */
+  key?: string;
+  at: number;
+  duration: number | null;
+  seek: number;
+  gain: number;
+  role: AudioRole;
+  duck: boolean;
+  fadeIn: number;
+  fadeOut: number;
+  effect?: AudioEffect;
+};
+
 /** A `<video>` slot as exposed to the renderer engine (worker). The engine
  * pre-extracts each clip to frames with ffmpeg and swaps the live `<video>`
  * for the matching frame image per captured frame — so the live video is never
@@ -96,6 +187,17 @@ declare global {
       /** Video slots in this timeline — read by the renderer engine to
        * swap video frames out-of-band. Empty when no video. */
       videos: VideoSlotInfo[];
+      /** Audio tracks in this timeline — read by the render worker to build
+       * the ffmpeg mix. Empty when no audio. */
+      audio: AudioSlotInfo[];
+      /** Preview transport (browser only; no-ops under the renderer). Start
+       * the audio mix from timeline position `fromMs`. The preview host calls
+       * this on play; scrubbing via `applyState` stays silent. */
+      playAudio(fromMs: number): void;
+      /** Stop preview audio playback. */
+      stopAudio(): void;
+      /** Mute / unmute preview audio without tearing down playback. */
+      setAudioMuted(muted: boolean): void;
     };
     /** Set true by the headless renderer when it captures (`__mvFramesRendererPresent`). */
     __mvFramesRendererPresent?: boolean;
@@ -189,11 +291,73 @@ export function stylesheetsReady(): Promise<void> {
     .then(() => (document.fonts ? document.fonts.ready.then(() => undefined) : undefined));
 }
 
+/**
+ * Seek `el` to `targetSec` and resolve only once the seeked frame has
+ * actually decoded — so a subsequent capture grabs the right frame instead
+ * of racing an in-flight seek.
+ *
+ * Resolution waits for the `seeked` event, then for one decoded video frame
+ * via `requestVideoFrameCallback` where available (fallback: a microtask, so
+ * we still yield to the decode). A timeout guards against `seeked` never
+ * firing (e.g. seeking to the exact current position, or an unsupported
+ * codec) so `applyState` can't hang the whole render. Already-satisfied
+ * seeks (`readyState` high enough and time already at target) resolve fast.
+ */
+function seekVideo(el: HTMLVideoElement, targetSec: number): Promise<void> {
+  const HAVE_CURRENT_DATA = 2;
+  // If we're already at (or within a frame of) the target with data ready,
+  // there's nothing to wait for — assigning currentTime won't fire `seeked`.
+  const alreadyThere =
+    Math.abs(el.currentTime - targetSec) < 1e-3 && el.readyState >= HAVE_CURRENT_DATA;
+  el.currentTime = targetSec;
+  if (alreadyThere) return Promise.resolve();
+
+  return new Promise<void>((resolve) => {
+    let done = false;
+    const finish = () => {
+      if (done) return;
+      done = true;
+      el.removeEventListener("seeked", onSeeked);
+      clearTimeout(timer);
+      resolve();
+    };
+    const afterSeeked = () => {
+      // Wait for one decoded frame to be presented when the API exists.
+      const rvfc = (
+        el as unknown as {
+          requestVideoFrameCallback?: (cb: () => void) => number;
+        }
+      ).requestVideoFrameCallback;
+      if (typeof rvfc === "function") {
+        rvfc.call(el, () => finish());
+      } else {
+        // No rVFC: yield a microtask so the decode can settle, then finish.
+        Promise.resolve().then(finish);
+      }
+    };
+    const onSeeked = () => afterSeeked();
+    el.addEventListener("seeked", onSeeked, { once: true });
+    // Guard: never let a missing `seeked` stall the render.
+    const timer = setTimeout(finish, 2000);
+  });
+}
+
 function installTimeline(schema: TimelineSchema): void {
   const rendererPresent =
     typeof window !== "undefined" && window.__mvFramesRendererPresent === true;
 
-  const slots: Slot[] = schema.instructions.flatMap<Slot>((i) => {
+  // Audio tracks own no DOM element — split them out before the visual
+  // instructions hit `querySelector`. They're mixed by the worker (which
+  // reads `__mvFrames.audio`) and previewed by the Web Audio engine below.
+  const audioInstrs = schema.instructions.filter(
+    (i): i is AudioInstr => (i as { kind?: string }).kind === "audio",
+  );
+  const visualInstrs = schema.instructions.filter(
+    (i): i is Exclude<Instr, AudioInstr> =>
+      (i as { kind?: string }).kind !== "audio",
+  );
+
+  const slots: Slot[] = visualInstrs.flatMap<Slot>((i) => {
     const el = document.querySelector(i.selector);
     if (!el) throw new Error(`defineTimeline: no element matches ${i.selector}`);
 
@@ -296,9 +460,70 @@ function installTimeline(schema: TimelineSchema): void {
       frameset: s.frameset,
     }));
 
+  // Resolve audio instruction defaults once — shared by the worker-facing
+  // slot list and the browser preview engine so both see the same mix.
+  const audioResolved = audioInstrs.map((a) => {
+    const role: AudioRole = a.role ?? "music";
+    return {
+      src: a.src,
+      key: a.key,
+      at: a.at,
+      duration: a.duration ?? null,
+      seek: a.seek ?? 0,
+      gain: a.gain ?? 1,
+      role,
+      duck: a.duck ?? role === "music",
+      fadeIn: a.fadeIn ?? 0,
+      fadeOut: a.fadeOut ?? 0,
+      effect: a.effect,
+    };
+  });
+
+  // Worker-facing slots (read via CDP to build the ffmpeg mix). Drop tracks
+  // with no storage key — those are preview-only and can't be rendered.
+  const audio: AudioSlotInfo[] = audioResolved
+    .filter((a) => typeof a.key === "string" && a.key.length > 0)
+    .map(({ src: _src, ...slot }) => slot);
+
+  // Browser preview engine. Constructed whenever there are tracks AND we're in
+  // a real browser — INCLUDING under `__mvFramesRendererPresent` (the Studio
+  // preview iframe sets that flag so it can drive `applyState`, yet still wants
+  // audible playback). It's lazy — no AudioContext, no fetch until `playFrom`
+  // is called — so in the *headless* renderer (which only ever calls
+  // `applyState`, never `playAudio`) it stays completely inert and the real mix
+  // is done by ffmpeg. Construction is just field storage.
+  const audioEngine =
+    typeof window !== "undefined" && typeof AudioContext !== "undefined" && audioResolved.length > 0
+      ? new AudioEngine(
+          audioResolved.map<PreviewAudioTrack>((a) => ({
+            src: a.src,
+            at: a.at,
+            duration: a.duration,
+            seek: a.seek,
+            gain: a.gain,
+            fadeIn: a.fadeIn,
+            fadeOut: a.fadeOut,
+            role: a.role,
+            duck: a.duck,
+            effect: a.effect,
+          })),
+        )
+      : null;
+
   window.__mvFrames = {
     videos,
+    audio,
+    playAudio(fromMs: number) {
+      void audioEngine?.playFrom(fromMs);
+    },
+    stopAudio() {
+      audioEngine?.stop();
+    },
+    setAudioMuted(muted: boolean) {
+      audioEngine?.setMuted(muted);
+    },
     async applyState(t: number) {
+      const videoWaits: Promise<void>[] = [];
       for (const s of slots) {
         if (s.kind === "css-pending") continue; // only relevant in browser preview
         if (s.kind === "video") {
@@ -306,18 +531,24 @@ function installTimeline(schema: TimelineSchema): void {
           // Seek the live <video> EXCEPT on the renderer's begin-frame-control
           // capture path (where seeking crashes the headless tab — the engine
           // swaps in pre-extracted frame images instead, flagged via
-          // `__mvFramesNoVideoSeek`). So the editor, live preview, AND the
-          // renderer's screenshot path all seek normally — frame-accurate
-          // "time travel" (StagePreview drives this via applyState).
+          // `__mvFramesNoVideoSeek`). On every live-seek path (editor, live
+          // preview, AND the renderer's screenshot capture) we must AWAIT the
+          // seeked frame's decode before resolving — otherwise a capture races
+          // an in-flight seek and grabs a stale/blank frame (and the per-frame
+          // loop spins, hitting the render wall-clock budget). `seekVideo`
+          // guards with a timeout so a missing `seeked` can't hang the render.
           if (!window.__mvFramesNoVideoSeek) {
-            s.el.currentTime = (s.seek + local) / 1000;
+            videoWaits.push(seekVideo(s.el, (s.seek + local) / 1000));
           }
           continue;
         }
         const local = clamp(t - s.at, 0, s.duration);
         s.anim.currentTime = local;
       }
-      await document.fonts.ready;
+      await Promise.all([
+        document.fonts ? document.fonts.ready.then(() => undefined) : Promise.resolve(),
+        ...videoWaits,
+      ]);
     },
   };