@dailies-studio/shot-editor 0.1.0

package/src/audio/preview.ts

@@ -0,0 +1,364 @@
+// Clock-synced audio preview for the shot lane. A port of the motion editor's
+// `editor/src/audio/preview.ts`, adapted to `Doc3D` and to this lane's monitor
+// controls; the differences from that file are all noted where they occur.
+//
+// ── THE ENVELOPE IS THE ENGINE'S, NOT A SECOND ONE ──────────────────────
+//
+// `AudioClipRef` is deliberately engine's `AudioClip` field-for-field except
+// for `frame` vs `at` (types.ts says so), so a clip is WIDENED onto that shape
+// and handed to `audioClipWindow` / `audioSourceTimeAt` / `audioEnvelopePoints`
+// unchanged. Those three are the ONE definition of what a clip sounds like at a
+// frame, shared with both export tiers' offline mixes. A shot-flavoured copy of
+// the fade maths is exactly how preview stops matching export, and the F3-A
+// review caught that divergence in the wild once already.
+//
+// There are no gain TRACKS here. Doc3D has blocks, not tracks, and no block
+// writes a clip's gain — so `sampledGain` is never supplied and the envelope
+// falls back to the clip's static gain, which is what the shot exporter mixes.
+//
+// ── TIME MODEL ──────────────────────────────────────────────────────────
+//
+// The chrome's rAF transport is the authority; this anchors frame→audio-time as
+// t(f) = anchorTime + framesToSeconds(f - anchor, fps) per (re)schedule. Both
+// clocks are wall-clock and drift between them is NOT corrected: a reschedule
+// (scrub, doc edit, unmute) re-anchors. Only edges, discontinuities and doc
+// identity changes do work; the per-frame path just records the snapshot.
+//
+// ── AND IT MOUNTS FROM THE CHROME ───────────────────────────────────────
+//
+// Never from the stage. The stage is a GL document behind an iframe that must
+// never remount, and audio is not part of the document: it is neither drawn nor
+// sampled by `ShotRuntime`. Putting a live AudioContext in there would tie the
+// score to the lifetime of a WebGPU adapter.
+//
+// No DOM lib types: every Web Audio surface is a minimal structural *Like type
+// and every effect is injected, so the jsdom tests drive a hand-rolled fake.
+import { audioClipWindow, audioEnvelopePoints, audioSourceTimeAt, framesToSeconds } from "@dailies-studio/engine";
+import type { AudioClip, Rational } from "@dailies-studio/engine";
+import { docDurationFrames } from "@dailies-studio/shot/doc";
+import type { AudioClipRef, Doc3D } from "@dailies-studio/shot/doc";
+
+// --- minimal structural Web Audio surface (fakes stay trivial) ---------------
+
+export interface AudioParamLike {
+  value: number;
+  setValueAtTime(value: number, time: number): unknown;
+  linearRampToValueAtTime(value: number, time: number): unknown;
+}
+
+export interface GainNodeLike {
+  readonly gain: AudioParamLike;
+  connect(destination: unknown): unknown;
+  disconnect(): void;
+}
+
+export interface AudioBufferLike {
+  readonly duration: number;
+}
+
+export interface AudioBufferSourceNodeLike {
+  buffer: AudioBufferLike | null;
+  connect(destination: unknown): unknown;
+  start(when?: number, offset?: number, duration?: number): void;
+  stop(when?: number): void;
+  disconnect(): void;
+}
+
+export interface AudioContextLike {
+  readonly currentTime: number;
+  readonly destination: unknown;
+  createGain(): GainNodeLike;
+  createBufferSource(): AudioBufferSourceNodeLike;
+  resume?(): unknown;
+  close?(): unknown;
+}
+
+export interface AudioPreviewDeps {
+  /** Called lazily on the first audible play edge — a user gesture, so the
+   *  real context is born unblocked by autoplay policy. */
+  createContext(): AudioContextLike;
+  fetchBytes(url: string): Promise<ArrayBuffer>;
+  decode(ctx: AudioContextLike, bytes: ArrayBuffer): Promise<AudioBufferLike>;
+  /** Audio-clock override for tests; defaults to ctx.currentTime. */
+  now?(): number;
+}
+
+/** The transport truth the host reads off the store on every change. */
+export interface TransportSnapshot {
+  playing: boolean;
+  frame: number;
+  doc: Doc3D | null;
+}
+
+export interface AudioPreview {
+  /** Push the current transport snapshot; cheap unless an edge/discontinuity. */
+  update(snap: TransportSnapshot): void;
+  /** Master preview mute: mute tears the session down, unmute reschedules. */
+  setMasterMute(muted: boolean): void;
+  /**
+   * Monitoring level, 0..1, applied on a MASTER gain node.
+   *
+   * A live parameter write and NOT a reschedule, which is the whole reason the
+   * master node exists: dragging the volume slider during playback must not
+   * stop and restart every voice, and re-anchoring the clock on every pixel of
+   * a slider drag would make the audio stutter against the picture.
+   */
+  setMasterVolume(volume: number): void;
+  dispose(): void;
+}
+
+/** A frame jump beyond this (or any decrease) while playing is a scrub. */
+export const SCRUB_JUMP_FRAMES = 10;
+
+type CacheEntry =
+  | { status: "pending" }
+  | { status: "ready"; buffer: AudioBufferLike }
+  | { status: "failed" };
+
+interface Voice {
+  source: AudioBufferSourceNodeLike;
+  gain: GainNodeLike;
+}
+
+/** `AudioClipRef` as engine's `AudioClip`: the same fields under the one name
+ *  that differs, with this lane's absent-means-default filled in. */
+function asEngineClip(clip: AudioClipRef): AudioClip {
+  return {
+    id: clip.id,
+    name: clip.id,
+    asset: clip.asset,
+    at: clip.frame,
+    offsetFrames: clip.offsetFrames ?? 0,
+    durationFrames: clip.durationFrames,
+    gain: clip.gain ?? 1,
+    fadeInFrames: clip.fadeInFrames ?? 0,
+    fadeOutFrames: clip.fadeOutFrames ?? 0,
+    ...(clip.muted === undefined ? {} : { muted: clip.muted }),
+  };
+}
+
+export function createAudioPreview(deps: AudioPreviewDeps): AudioPreview {
+  let ctx: AudioContextLike | null = null;
+  let ctxFailed = false;
+  let master: GainNodeLike | null = null;
+  let masterMuted = false;
+  let masterVolume = 1;
+  let disposed = false;
+  let last: TransportSnapshot = { playing: false, frame: 0, doc: null };
+
+  /**
+   * AudioBuffer cache by asset PATH. "failed" is terminal: one error, no retries.
+   *
+   * NOT by asset id. `ShotAsset.id` is a slug of the filename uniquified only
+   * against the ids already in ITS OWN document (`freshId(usedIdsOf(doc), …)`),
+   * so every shot that imports a `track.wav` mints `as_track`. This map outlives
+   * a document switch — `openShot` swaps `state.doc` in place and the bridge
+   * that owns this preview never remounts — so an id key serves one shot's
+   * audio to another shot with the same filename. `asset.path` is the
+   * content-addressed `<hash8>-<name>` the asset store writes, which is unique
+   * across every document and identical for identical bytes: keying on it also
+   * keeps the cache hit for a genuine re-import of the same file.
+   */
+  const buffers = new Map<string, CacheEntry>();
+
+  /** clipId -> live voice. Non-null exactly while an audible session runs —
+   *  late decodes join it; mute/pause null it. */
+  let session: Map<string, Voice> | null = null;
+
+  function now(): number {
+    return deps.now ? deps.now() : ctx!.currentTime;
+  }
+
+  function ensureContext(): boolean {
+    if (ctxFailed) return false;
+    if (!ctx) {
+      try {
+        ctx = deps.createContext();
+      } catch (e) {
+        ctxFailed = true; // e.g. no Web Audio in this environment — stay silent
+        console.error("[audio] preview disabled: AudioContext unavailable", e);
+        return false;
+      }
+      // ONE master node for the session's whole life, so the slider writes a
+      // parameter rather than rebuilding the graph.
+      master = ctx.createGain();
+      master.gain.value = masterVolume;
+      master.connect(ctx.destination);
+    }
+    try {
+      void ctx.resume?.(); // autoplay policy: nudge on every play gesture
+    } catch {
+      // resume is best-effort; a suspended context just stays silent
+    }
+    return true;
+  }
+
+  /** Schedule one clip's voice, anchored at (anchorFrame ↔ anchorTime). */
+  function scheduleClip(
+    ref: AudioClipRef,
+    buffer: AudioBufferLike,
+    endFrameExclusive: number,
+    fps: Rational,
+    anchorFrame: number,
+    anchorTime: number,
+  ): void {
+    const clip = asEngineClip(ref);
+    const { start, end } = audioClipWindow(clip);
+    const schedEnd = Math.min(end, endFrameExclusive);
+    const first = Math.max(start, anchorFrame); // mid-window start advances the offset
+    if (first >= schedEnd) return;
+    const offset = audioSourceTimeAt(clip, first, fps);
+    if (offset >= buffer.duration) return; // trimmed past the source: silence is legal
+    const t = (f: number): number => anchorTime + framesToSeconds(f - anchorFrame, fps);
+    const c = ctx!;
+    const source = c.createBufferSource();
+    source.buffer = buffer;
+    const gain = c.createGain();
+    source.connect(gain);
+    gain.connect(master!);
+    // The preview===export envelope: the engine's ONE point list, rendered as
+    // set-then-ramps; the source stops at the last point's frame.
+    const points = audioEnvelopePoints(clip, first, endFrameExclusive);
+    if (points.length === 0) return;
+    gain.gain.setValueAtTime(points[0]!.gain, t(points[0]!.frame));
+    for (const p of points.slice(1)) {
+      gain.gain.linearRampToValueAtTime(p.gain, t(p.frame));
+    }
+    source.start(t(first), offset, framesToSeconds(points.at(-1)!.frame - first, fps));
+    session!.set(clip.id, { source, gain });
+  }
+
+  /** Begin an audible session from `fromFrame`: schedule every decoded clip
+   *  whose window intersects [fromFrame, docDuration); kick decodes for the
+   *  rest (their resolve handler joins the then-current session). */
+  function startSession(fromFrame: number): void {
+    const doc = last.doc;
+    if (doc === null || masterMuted) return;
+    if (!ensureContext()) return;
+    session = new Map();
+    const anchorTime = now();
+    const endFrame = docDurationFrames(doc);
+    for (const clip of Object.values(doc.audio)) {
+      if (clip.muted === true) continue; // gain would be 0 — skip the node entirely
+      const window = audioClipWindow(asEngineClip(clip));
+      if (Math.min(window.end, endFrame) <= Math.max(window.start, fromFrame)) continue;
+      const asset = doc.assets[clip.asset];
+      if (asset === undefined) continue; // validation requires it; a hole stays silent
+      const entry = buffers.get(asset.path);
+      if (entry?.status === "ready") {
+        scheduleClip(clip, entry.buffer, endFrame, doc.fps, fromFrame, anchorTime);
+      } else if (entry === undefined) {
+        requestDecode(asset.path);
+      }
+      // pending: scheduled on resolve if still playing; failed: already errored
+    }
+  }
+
+  function requestDecode(path: string): void {
+    const c = ctx!; // non-null: only called under an ensured context; captured
+    // so a dispose() while the fetch is in flight can't null it away.
+    buffers.set(path, { status: "pending" });
+    void deps
+      .fetchBytes(`/studio/assets/${encodeURIComponent(path)}`)
+      .then((bytes) => deps.decode(c, bytes))
+      .then(
+        (buffer) => {
+          buffers.set(path, { status: "ready", buffer });
+          onBufferReady(path, buffer);
+        },
+        (e: unknown) => {
+          buffers.set(path, { status: "failed" }); // terminal — never retried
+          console.error(`[audio] asset ${path} failed to load; its clips stay silent`, e);
+        },
+      );
+  }
+
+  /** A late decode joins the CURRENT session from the then-current store frame
+   *  — never the (stale) frame play started at. No session = arrived after
+   *  pause/mute: schedule nothing.
+   *
+   *  Matched through the then-current document's own asset record, so a decode
+   *  started in one shot cannot claim the same-named asset of the shot that
+   *  replaced it while the fetch was in flight. */
+  function onBufferReady(path: string, buffer: AudioBufferLike): void {
+    if (disposed || session === null || !last.playing) return;
+    const doc = last.doc;
+    if (doc === null) return;
+    const anchorTime = now();
+    const endFrame = docDurationFrames(doc);
+    for (const clip of Object.values(doc.audio)) {
+      if (doc.assets[clip.asset]?.path !== path) continue;
+      if (clip.muted === true || session.has(clip.id)) continue;
+      scheduleClip(clip, buffer, endFrame, doc.fps, last.frame, anchorTime);
+    }
+  }
+
+  function teardown(): void {
+    if (session === null) return;
+    for (const { source, gain } of session.values()) {
+      try {
+        source.stop();
+      } catch {
+        // already ended — stopping is best-effort
+      }
+      source.disconnect();
+      gain.disconnect();
+    }
+    session = null;
+  }
+
+  return {
+    update(snap) {
+      if (disposed) return;
+      const prev = last;
+      last = snap;
+      if (!snap.playing) {
+        if (prev.playing) teardown();
+        return;
+      }
+      if (!prev.playing) {
+        // Play edge — the user-gesture moment; the context is created here.
+        startSession(snap.frame);
+        return;
+      }
+      // While playing: reschedule on document identity change (a gain or fade
+      // edit audibly applies) or a scrub-jump; the monotone +small frame path
+      // (the rAF clock ticking) does nothing but the snapshot record above.
+      const scrubbed = snap.frame < prev.frame || snap.frame - prev.frame > SCRUB_JUMP_FRAMES;
+      if (snap.doc !== prev.doc || scrubbed) {
+        teardown();
+        startSession(snap.frame);
+      }
+    },
+
+    setMasterMute(muted) {
+      if (disposed || muted === masterMuted) return;
+      masterMuted = muted;
+      if (muted) teardown();
+      else if (last.playing) startSession(last.frame);
+    },
+
+    setMasterVolume(volume) {
+      const next = Math.max(0, Math.min(1, volume));
+      if (disposed || next === masterVolume) return;
+      masterVolume = next;
+      // No reschedule: see the interface. A context that does not exist yet
+      // picks the level up when `ensureContext` builds the master node.
+      if (master !== null) master.gain.value = next;
+    },
+
+    dispose() {
+      if (disposed) return;
+      disposed = true;
+      teardown();
+      master?.disconnect();
+      master = null;
+      try {
+        void ctx?.close?.();
+      } catch {
+        // closing is best-effort
+      }
+      ctx = null;
+    },
+  };
+}

package/src/bridge.ts

@@ -0,0 +1,45 @@
+// Chrome side of the stage bridge: the iframe loads /shot-stage.html (served
+// through the PROJECT's Vite so the three stack resolves to the studio's
+// pinned copies), and its `boot()` assigns window.__shotStudio BEFORE first
+// paint. Module boot is async after the iframe's load event, so connecting
+// POLLS for the global — the canvas lane's posture, for the same reason.
+//
+// connectStage also REGISTERS the handlers, because the stage defers its first
+// render until a chrome is connected: there is nothing worth painting before
+// one exists, and a stage that rendered first would show an empty frame that
+// then jumps.
+import { SHOT_BRIDGE_VERSION } from "./bridgeTypes.js";
+import type { ShotBridgeHandlers, ShotStageBridge } from "./bridgeTypes.js";
+
+const POLL_MS = 50;
+/** Generous, because the stage's first paint waits on an async WebGPU adapter
+ *  request that can take a while on a cold profile — and failing early here
+ *  would look to the author like the editor is broken rather than starting. */
+const DEFAULT_TIMEOUT_MS = 20_000;
+
+export async function connectStage(
+  iframe: HTMLIFrameElement,
+  handlers: ShotBridgeHandlers,
+  opts: { timeoutMs?: number; signal?: AbortSignal } = {},
+): Promise<ShotStageBridge> {
+  const deadline = performance.now() + (opts.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+  for (;;) {
+    if (opts.signal?.aborted) throw new Error("stage connect aborted");
+    const bridge = (iframe.contentWindow as (Window & { __shotStudio?: ShotStageBridge }) | null)
+      ?.__shotStudio;
+    if (bridge) {
+      if (bridge.version !== SHOT_BRIDGE_VERSION) {
+        throw new Error(
+          `stage bridge version ${bridge.version}, chrome needs ${SHOT_BRIDGE_VERSION} — ` +
+            `rebuild @dailies-studio/shot-editor`,
+        );
+      }
+      bridge.connect(handlers);
+      return bridge;
+    }
+    if (performance.now() > deadline) {
+      throw new Error("stage bridge did not appear (is /shot-stage.html loading?)");
+    }
+    await new Promise((r) => setTimeout(r, POLL_MS));
+  }
+}