@dailies-studio/shot-editor 0.1.0 → 0.2.0

package/src/audio/offlineMix.ts

@@ -0,0 +1,289 @@
+// The shot lane's OFFLINE audio mix, for Tier-1 export: one deterministic
+// OfflineAudioContext render of the document's clips, handed to the encode
+// worker as planar f32. The browser-side sibling of the server render's mix
+// (cli's mixCompositionAudio via shotRender's withShotAudio), and the same
+// three engine functions the live preview schedules from — so what the author
+// heard is what the file carries, on every path.
+//
+// ── TWO HOMES, ONE FLATTEN ──────────────────────────────────────────────
+//
+// Clips live in two places since schema v13: `doc.audio` (the master-timed bed)
+// and `Scene3D.audio` (what a shot owns, on scene-local frames). Everything
+// below reads `masterAudioClips`, never either record — see the note at the
+// call. The rest of this file never learns that a second home exists.
+//
+// ── SCENE SCOPE IS THE SLICE'S PROBLEM, NOT OURS ────────────────────────
+//
+// A per-scene export mixes the audio of `sliceShotToScene(doc, sceneId)`.
+// The slice has already done the master→scene retiming — dropping what falls
+// outside the window, trimming a clip that starts before it and advancing its
+// `offsetFrames` by the amount removed. That is the same call the server
+// render's withShotAudio leans on, for the same reason: re-deriving the
+// arithmetic here would be a second implementation of the one thing in this
+// lane most likely to go subtly out of sync with the picture.
+//
+// ── ENVELOPE RULE (preview === export, non-negotiable) ──────────────────
+//
+// The point list comes verbatim from the engine's audioEnvelopePoints —
+// setValueAtTime at the first point, linearRamp for the rest, source stopped
+// at the last point's frame. Never hand-roll the fade maths here. Schema v13's
+// `gainTrack` supplies the sampler argument, so a keyframed duck rides the same
+// one list the live preview schedules from and the server render materialises
+// as PCM; a clip with no automation supplies no sampler and the envelope falls
+// back to the static gain, exactly as before.
+//
+// Everything Web-Audio-shaped is injectable so jsdom tests drive fakes; the
+// real defaults are OfflineAudioContext + decodeAudioData + fetch.
+import {
+  MIX_SAMPLE_RATE,
+  audioClipWindow,
+  audioEnvelopePoints,
+  audioSampleCount,
+  audioSourceTimeAt,
+  framesToSeconds,
+  measurePeak,
+  wirePan,
+} from "@dailies-studio/engine";
+import type { AudioClip, AudioPeak, Rational, ScheduledClip } from "@dailies-studio/engine";
+import {
+  audioAnalysisFileName,
+  audioGainSampler,
+  audioPlayPath,
+  derivedGainTracks,
+  docDurationFrames,
+  duckSourceAssets,
+  masterAudioClips,
+  sliceShotToScene,
+} from "@dailies-studio/shot/doc";
+import type { AudioAnalysis, Doc3D } from "@dailies-studio/shot/doc";
+import { asEngineClip } from "./preview.js";
+import type { AudioBufferLike, AudioBufferSourceNodeLike, GainNodeLike, PanContextLike } from "./preview.js";
+
+/** 48 kHz stereo, the studio's one mix layout — the ENGINE's constant, which
+ *  is also the rate the server lane's graph and the import transcode run at.
+ *  This used to be a local 48_000 with a comment pointing at cli's copy,
+ *  because a package boundary stood between them; the clock moved into engine
+ *  so the two mixes could stop describing the same number twice. */
+export { MIX_SAMPLE_RATE };
+
+export interface OfflineAudioContextLike extends PanContextLike {
+  destination: unknown;
+  createBufferSource(): AudioBufferSourceNodeLike;
+  createGain(): GainNodeLike;
+  startRendering(): Promise<{ getChannelData(channel: number): Float32Array }>;
+}
+
+export interface OfflineMixDeps {
+  fetchBytes(url: string): Promise<ArrayBuffer>;
+  decode(ctx: OfflineAudioContextLike, bytes: ArrayBuffer): Promise<AudioBufferLike>;
+  createContext(channels: number, length: number, sampleRate: number): OfflineAudioContextLike;
+}
+
+function defaultDeps(): OfflineMixDeps {
+  if (typeof OfflineAudioContext === "undefined") {
+    throw new Error("audio mixing needs OfflineAudioContext (a real browser)");
+  }
+  return {
+    fetchBytes: async (url) => {
+      const res = await fetch(url);
+      if (!res.ok) throw new Error(`HTTP ${res.status}`);
+      return res.arrayBuffer();
+    },
+    decode: (ctx, bytes) =>
+      (ctx as unknown as OfflineAudioContext).decodeAudioData(bytes) as Promise<AudioBufferLike>,
+    createContext: (channels, length, sampleRate) =>
+      new OfflineAudioContext(channels, length, sampleRate) as unknown as OfflineAudioContextLike,
+  };
+}
+
+const why = (e: unknown): string => (e instanceof Error ? e.message : String(e));
+
+/** cli's audioMix.scheduleClip minus the gain-track sampler a Doc3D cannot
+ *  have: absolute time axis, engine envelope, stop at the last point. */
+function scheduleClip(
+  ctx: OfflineAudioContextLike,
+  clip: AudioClip,
+  buffer: AudioBufferLike,
+  endFrames: number,
+  fps: Rational,
+  /** The clip's gain automation, or undefined when it has none — handed
+   *  straight to the engine, which falls back to the static gain. */
+  sampledGain?: (frame: number) => number | undefined,
+): { start: number; end: number } | null {
+  const pan = clip.pan ?? 0;
+  const window = audioClipWindow(clip);
+  // Trim starts past the decoded source: the whole clip is silence — legal
+  // (validation cannot know asset lengths), so no node at all.
+  const offset = audioSourceTimeAt(clip, window.start, fps);
+  if (offset >= buffer.duration) return null;
+  const points = audioEnvelopePoints(clip, window.start, endFrames, fps, sampledGain);
+  if (points.length === 0) return null;
+  const gain = ctx.createGain();
+  gain.gain.setValueAtTime(points[0]!.gain, framesToSeconds(points[0]!.frame, fps));
+  for (const p of points.slice(1)) {
+    gain.gain.linearRampToValueAtTime(p.gain, framesToSeconds(p.frame, fps));
+  }
+  const source = ctx.createBufferSource();
+  source.buffer = buffer;
+  source.connect(gain);
+  // A centred clip keeps the graph it always had. A panned one goes through
+  // the same splitter stage the preview uses, on the same engine law; a
+  // context without the splitter pair cannot honour it, and mixing it centred
+  // would be a file that disagrees with what the author heard.
+  if (pan === 0) gain.connect(ctx.destination);
+  else if (wirePan(ctx, gain, ctx.destination, pan) === null) {
+    throw new Error(`audio clip ${clip.name} is panned and this context has no ChannelSplitter`);
+  }
+  source.start(
+    framesToSeconds(window.start, fps),
+    offset,
+    framesToSeconds(points.at(-1)!.frame - window.start, fps),
+  );
+  // The span this clip ACTUALLY contributed, for the peak's attribution — the
+  // same shape cli's mixer reports, so a clip the scheduler skipped can never
+  // be blamed for a sample it did not make.
+  return { start: window.start, end: points.at(-1)!.frame };
+}
+
+/**
+ * Render the shot's audio clips into a 48 kHz stereo mix for the export
+ * scope. Null when the scope has no unmuted clips (a silent export carries
+ * no audio track). Missing and non-audio assets THROW — validateShotDoc
+ * refuses both at the gate, so reaching them here means the document
+ * bypassed validation, and a silent skip would be an export that looks
+ * successful and has no sound.
+ *
+ * The returned `peak` is measured HERE, on the float mix, because this is the
+ * last moment it still has its true amplitude: Web Audio sums gain nodes
+ * UNCLAMPED, so two clips that both peak at full scale reach +6.02 dBFS, and
+ * everything past ±1 is flattened on the way into the encoder. The dialog
+ * renders the engine's advisory from it, which is the same sentence the server
+ * render prints. A non-finite sample THROWS from the scan rather than becoming
+ * silence in the file.
+ */
+export async function mixShotAudio(
+  doc: Doc3D,
+  /** A scene id, a master `{ start, frames }` window of the whole timeline
+   *  (the work area), or undefined for the whole timeline. A window is the
+   *  whole film's mix cut to its samples, so it sounds as the transport
+   *  played it: a fade the window starts inside starts mid-fade. */
+  scope: string | { start: number; frames: number } | undefined,
+  deps: OfflineMixDeps = defaultDeps(),
+): Promise<{ sampleRate: number; channels: [Float32Array, Float32Array]; peak: AudioPeak } | null> {
+  const window = typeof scope === "object" ? scope : undefined;
+  const source = typeof scope === "string" ? (sliceShotToScene(doc, scope) ?? doc) : doc;
+  // THROUGH THE FLATTEN. `source.audio` has been the BED only since schema v13,
+  // so mixing from it exports a file missing every shot-owned clip — the
+  // preview would have played them and the MP4 would not, which breaks this
+  // lane's one rule. A slice re-zeroes on its scene, so the flatten is the
+  // identity over the clips that scene owns.
+  const flat = masterAudioClips(source);
+  const panOf = new Map(flat.map((m) => [m.clip.id, m.pan]));
+  const clips = flat
+    .map((m) => m.clip)
+    .filter(
+      (clip) =>
+        clip.muted !== true &&
+        (window === undefined ||
+          (clip.frame < window.start + window.frames && clip.frame + clip.durationFrames > window.start)),
+    );
+  if (clips.length === 0) return null;
+
+  // THE SIDECARS A DUCK NEEDS, fetched before a single node is scheduled.
+  //
+  // A THROW, not a fallback. The preview settles on the ducked envelope once
+  // the sidecar lands, so exporting the unducked one would ship a file quieter
+  // in the wrong places than the thing the author approved, with nothing on
+  // screen to say so. Refusal over corruption: the author re-runs the export
+  // after the analysis exists.
+  const analyses = new Map<string, AudioAnalysis>();
+  for (const asset of duckSourceAssets(source)) {
+    try {
+      const bytes = await deps.fetchBytes(`/studio/assets/${encodeURIComponent(audioAnalysisFileName(asset.path))}`);
+      analyses.set(asset.path, JSON.parse(new TextDecoder().decode(bytes)) as AudioAnalysis);
+    } catch (e) {
+      throw new Error(
+        `audio analysis for ${asset.path}: ${why(e)}. A ducked bed cannot be mixed until its voice is measured.`,
+      );
+    }
+  }
+  const derived = derivedGainTracks(source, (asset) => analyses.get(asset.path));
+
+  const endFrames = window === undefined ? docDurationFrames(source) : window.start + window.frames;
+  // The ENGINE's integer clock, not `Math.ceil(seconds * rate)`. That spelling
+  // is off by a sample at 30000/1001 (audioTiming.ts measures it: 248249
+  // against a true 248248 over five seconds), which is a mix one sample longer
+  // than the picture and a muxer left to resolve the difference however its
+  // container prefers.
+  const length = audioSampleCount(endFrames, source.fps);
+  if (length <= 0) return null;
+
+  const ctx = deps.createContext(2, length, MIX_SAMPLE_RATE);
+  // N clips of one asset decode once — buffers are shared read-only.
+  const decoded = new Map<string, AudioBufferLike>();
+  const scheduled: ScheduledClip[] = [];
+  for (const clip of clips) {
+    const asset = source.assets[clip.asset];
+    if (asset === undefined) {
+      throw new Error(`audio clip ${clip.id} references unknown asset ${clip.asset}`);
+    }
+    if (asset.kind !== "audio") {
+      throw new Error(`audio clip ${clip.id} references asset ${clip.asset}, which is a ${asset.kind}`);
+    }
+    // The file the clip PLAYS: its processed file when it has effects (schema
+    // v49), which the host makes on this request if it has not yet.
+    const file = audioPlayPath(clip, asset);
+    let buffer = decoded.get(file);
+    if (!buffer) {
+      let bytes: ArrayBuffer;
+      try {
+        bytes = await deps.fetchBytes(`/studio/assets/${encodeURIComponent(file)}`);
+      } catch (e) {
+        throw new Error(
+          file === asset.path ? `audio asset ${asset.path}: ${why(e)}` : `audio clip ${clip.id}: its effects could not be rendered (${why(e)})`,
+        );
+      }
+      try {
+        buffer = await deps.decode(ctx, bytes);
+      } catch (e) {
+        throw new Error(`audio asset ${file}: could not decode (${why(e)})`);
+      }
+      decoded.set(file, buffer);
+    }
+    // `clip.frame` is the MASTER start the flatten put it at, and the track's
+    // frames are local to exactly that — including after `clipToWindow` shifted
+    // them by whatever a window ate off the head.
+    const span = scheduleClip(
+      ctx,
+      asEngineClip(clip, panOf.get(clip.id)),
+      buffer,
+      endFrames,
+      source.fps,
+      // The derived duck outranks the authored track because it INCLUDES it:
+      // `duckedGainTrack` multiplies whatever gain/gainTrack resolve to.
+      audioGainSampler(derived.get(clip.id) ?? clip.gainTrack, clip.frame, source.fps),
+    );
+    // `name` is the clip id: `AudioClipRef` has no name of its own (asEngineClip
+    // fills the engine's field with the id), and the id is what the lane shows.
+    if (span) scheduled.push({ name: clip.id, ...span });
+  }
+
+  const rendered = await ctx.startRendering();
+  // A window keeps its own samples, on the same integer clock as the picture,
+  // and its peak is the file's: frames and spans counted from its start.
+  const from = window === undefined ? 0 : audioSampleCount(window.start, source.fps);
+  const channels: [Float32Array, Float32Array] = [
+    rendered.getChannelData(0).subarray(from),
+    rendered.getChannelData(1).subarray(from),
+  ];
+  const shift = window?.start ?? 0;
+  return {
+    sampleRate: MIX_SAMPLE_RATE,
+    channels,
+    peak: measurePeak(
+      channels,
+      scheduled.map((c) => ({ ...c, start: Math.max(0, c.start - shift), end: c.end - shift })),
+      source.fps,
+    ),
+  };
+}