@maravilla-labs/frames 0.5.1 → 0.8.0

package/src/index.ts

@@ -8,6 +8,22 @@
  *   frame to produce deterministic video output.
  */
 
+import { AudioEngine, type PreviewAudioTrack } from "./audio.js";
+import { resolveFadeCurve, type FadeCurve } from "./fades.js";
+
+export { AudioEngine } from "./audio.js";
+export type { PreviewAudioTrack } from "./audio.js";
+export { FADE_CURVES, fadeAutomation, fadeGain, resolveFadeCurve } from "./fades.js";
+export type { FadeCurve } from "./fades.js";
+export {
+  DEFAULT_DUCK,
+  duckAutomation,
+  duckMultiplier,
+  mergeSegments,
+  type DuckParams,
+  type Segment,
+} from "./ducking.js";
+
 export type AnimationInstr = {
   /** Optional discriminator. Defaults to `"animation"`. */
   kind?: "animation";
@@ -67,9 +83,95 @@ export type CssClassInstr = {
   classes: string[];
 };
 
-export type Instr = AnimationInstr | VideoInstr | CssClassInstr;
+/** Role of an audio track. `"voiceover"` is what music ducks *under*;
+ * `"music"` is the background bed that gets ducked; `"sfx"` is a one-shot
+ * that is neither ducked nor a ducking trigger. */
+export type AudioRole = "voiceover" | "music" | "sfx";
+
+/**
+ * A per-track audio effect. `"radio"` is a band-limited, saturated "deep
+ * radio/telephone voice" — applied identically in the browser preview (Web
+ * Audio filter chain, [`audio.ts`]) and in the render (ffmpeg, worker-side), so
+ * preview matches export. All knobs are 0..1.
+ */
+export type AudioEffect = {
+  kind: "radio";
+  /** Overdrive / saturation amount (0 = clean, 1 = crunchy). Default 0.4. */
+  drive?: number;
+  /** Band brightness (0 = narrow telephone band, 1 = wide/open). Default 0.4. */
+  tone?: number;
+  /** Low-end "depth" boost for a deeper voice. Default 0.5. */
+  depth?: number;
+};
+
+/**
+ * An audio track on the timeline. Unlike the visual instructions it owns no
+ * DOM element — it's mixed into the final video by the render worker (ffmpeg)
+ * and played in the browser preview by the Web Audio engine ([`audio.ts`]).
+ * Multiple tracks coexist (e.g. an intro bed + a voiceover); music tracks
+ * dip under voiceover via static-envelope ducking (see [`ducking.ts`]).
+ */
+export type AudioInstr = {
+  /** Discriminator — required. */
+  kind: "audio";
+  /** When on the timeline the track starts, in milliseconds. */
+  at: number;
+  /** How long it plays from `at`, in ms. Defaults to the clip's natural length. */
+  duration?: number;
+  /** Start offset WITHIN the source file, in ms. Defaults to 0. */
+  seek?: number;
+  /**
+   * Playable URL for the browser preview (fetched + decoded via Web Audio).
+   * For a stored asset, a URL that resolves to it (e.g. `/_assets/<key>`).
+   */
+  src: string;
+  /**
+   * Storage key the render worker fetches + stages to mix into the final
+   * video. Without it the track is preview-only (it can't be rendered).
+   */
+  key?: string;
+  /** Track role. Defaults to `"music"`. */
+  role?: AudioRole;
+  /** Linear gain (1 = unity). Defaults to 1. */
+  gain?: number;
+  /** Fade-in / fade-out length from the clip edges, ms. Default 0. */
+  fadeIn?: number;
+  fadeOut?: number;
+  /**
+   * Shape of each fade — `linear` | `equal-power` | `s-curve` | `exponential`
+   * | `logarithmic` (ffmpeg `afade` curves; see `fades.ts`). Default `linear`.
+   */
+  fadeInCurve?: FadeCurve;
+  fadeOutCurve?: FadeCurve;
+  /** Whether a music track ducks under voiceover. Defaults to `role === "music"`. */
+  duck?: boolean;
+  /** Optional per-track effect (e.g. a "radio voice"). */
+  effect?: AudioEffect;
+};
+
+export type Instr = AnimationInstr | VideoInstr | CssClassInstr | AudioInstr;
 export type TimelineSchema = { duration: number; instructions: Instr[] };
 
+/** An audio track as exposed to the render worker via `window.__mvFrames.audio`.
+ * The worker stages `key` and builds the ffmpeg mix; `duration === null`
+ * means "use the source's natural length". Mirrors [`AudioInstr`] with
+ * defaults resolved. */
+export type AudioSlotInfo = {
+  /** Storage key the worker stages (absent ⇒ preview-only, not rendered). */
+  key?: string;
+  at: number;
+  duration: number | null;
+  seek: number;
+  gain: number;
+  role: AudioRole;
+  duck: boolean;
+  fadeIn: number;
+  fadeOut: number;
+  fadeInCurve: FadeCurve;
+  fadeOutCurve: FadeCurve;
+  effect?: AudioEffect;
+};
+
 /** A `<video>` slot as exposed to the renderer engine (worker). The engine
  * pre-extracts each clip to frames with ffmpeg and swaps the live `<video>`
  * for the matching frame image per captured frame — so the live video is never
@@ -96,6 +198,17 @@ declare global {
       /** Video slots in this timeline — read by the renderer engine to
        * swap video frames out-of-band. Empty when no video. */
       videos: VideoSlotInfo[];
+      /** Audio tracks in this timeline — read by the render worker to build
+       * the ffmpeg mix. Empty when no audio. */
+      audio: AudioSlotInfo[];
+      /** Preview transport (browser only; no-ops under the renderer). Start
+       * the audio mix from timeline position `fromMs`. The preview host calls
+       * this on play; scrubbing via `applyState` stays silent. */
+      playAudio(fromMs: number): void;
+      /** Stop preview audio playback. */
+      stopAudio(): void;
+      /** Mute / unmute preview audio without tearing down playback. */
+      setAudioMuted(muted: boolean): void;
     };
     /** Set true by the headless renderer when it captures (`__mvFramesRendererPresent`). */
     __mvFramesRendererPresent?: boolean;
@@ -244,7 +357,18 @@ function installTimeline(schema: TimelineSchema): void {
   const rendererPresent =
     typeof window !== "undefined" && window.__mvFramesRendererPresent === true;
 
-  const slots: Slot[] = schema.instructions.flatMap<Slot>((i) => {
+  // Audio tracks own no DOM element — split them out before the visual
+  // instructions hit `querySelector`. They're mixed by the worker (which
+  // reads `__mvFrames.audio`) and previewed by the Web Audio engine below.
+  const audioInstrs = schema.instructions.filter(
+    (i): i is AudioInstr => (i as { kind?: string }).kind === "audio",
+  );
+  const visualInstrs = schema.instructions.filter(
+    (i): i is Exclude<Instr, AudioInstr> =>
+      (i as { kind?: string }).kind !== "audio",
+  );
+
+  const slots: Slot[] = visualInstrs.flatMap<Slot>((i) => {
     const el = document.querySelector(i.selector);
     if (!el) throw new Error(`defineTimeline: no element matches ${i.selector}`);
 
@@ -347,8 +471,72 @@ function installTimeline(schema: TimelineSchema): void {
       frameset: s.frameset,
     }));
 
+  // Resolve audio instruction defaults once — shared by the worker-facing
+  // slot list and the browser preview engine so both see the same mix.
+  const audioResolved = audioInstrs.map((a) => {
+    const role: AudioRole = a.role ?? "music";
+    return {
+      src: a.src,
+      key: a.key,
+      at: a.at,
+      duration: a.duration ?? null,
+      seek: a.seek ?? 0,
+      gain: a.gain ?? 1,
+      role,
+      duck: a.duck ?? role === "music",
+      fadeIn: a.fadeIn ?? 0,
+      fadeOut: a.fadeOut ?? 0,
+      fadeInCurve: resolveFadeCurve(a.fadeInCurve),
+      fadeOutCurve: resolveFadeCurve(a.fadeOutCurve),
+      effect: a.effect,
+    };
+  });
+
+  // Worker-facing slots (read via CDP to build the ffmpeg mix). Drop tracks
+  // with no storage key — those are preview-only and can't be rendered.
+  const audio: AudioSlotInfo[] = audioResolved
+    .filter((a) => typeof a.key === "string" && a.key.length > 0)
+    .map(({ src: _src, ...slot }) => slot);
+
+  // Browser preview engine. Constructed whenever there are tracks AND we're in
+  // a real browser — INCLUDING under `__mvFramesRendererPresent` (the Studio
+  // preview iframe sets that flag so it can drive `applyState`, yet still wants
+  // audible playback). It's lazy — no AudioContext, no fetch until `playFrom`
+  // is called — so in the *headless* renderer (which only ever calls
+  // `applyState`, never `playAudio`) it stays completely inert and the real mix
+  // is done by ffmpeg. Construction is just field storage.
+  const audioEngine =
+    typeof window !== "undefined" && typeof AudioContext !== "undefined" && audioResolved.length > 0
+      ? new AudioEngine(
+          audioResolved.map<PreviewAudioTrack>((a) => ({
+            src: a.src,
+            at: a.at,
+            duration: a.duration,
+            seek: a.seek,
+            gain: a.gain,
+            fadeIn: a.fadeIn,
+            fadeOut: a.fadeOut,
+            fadeInCurve: a.fadeInCurve,
+            fadeOutCurve: a.fadeOutCurve,
+            role: a.role,
+            duck: a.duck,
+            effect: a.effect,
+          })),
+        )
+      : null;
+
   window.__mvFrames = {
     videos,
+    audio,
+    playAudio(fromMs: number) {
+      void audioEngine?.playFrom(fromMs);
+    },
+    stopAudio() {
+      audioEngine?.stop();
+    },
+    setAudioMuted(muted: boolean) {
+      audioEngine?.setMuted(muted);
+    },
     async applyState(t: number) {
       const videoWaits: Promise<void>[] = [];
       for (const s of slots) {