@pictor-sdk/video-android 0.1.0

package/src/previewAudioSession.ts

@@ -0,0 +1,168 @@
+import { compileAudioMixProgram, type VideoProject } from '@pictor-sdk/video-core'
+
+/**
+ * Deciding what a preview should be doing about audio.
+ *
+ * Pure, and separate from the hook that acts on it, because every rule worth
+ * getting right here is a rule about state rather than about React: whether a
+ * project needs a mix at all, whether the mix on hand is the one the document
+ * describes, whether a result that just arrived is still wanted, and whether
+ * two players have drifted far enough to be worth correcting.
+ *
+ * ## Two sources of sound, never both
+ *
+ * Without music, the video player's own audio is the preview and nothing here
+ * applies. With music, the mix covers the *whole* timeline — including the
+ * audio of the clips themselves — so playing the clips' audio as well would
+ * play everything twice. There is no state in which both are audible, and the
+ * transition between them is the thing this file exists to sequence.
+ *
+ * @experimental
+ */
+
+/** What a preview's audio is doing. @experimental */
+export type PreviewAudioStatus =
+  /** No music in the project. The video player's own audio is the preview. */
+  | 'source'
+  /** Music exists and the mix for *this* document is not ready. */
+  | 'preparing'
+  /** The mix on hand matches the document. Source audio is muted. */
+  | 'ready'
+  /** The mix could not be prepared. Nothing is played rather than the wrong thing. */
+  | 'failed'
+
+/** @experimental */
+export interface PreviewAudioDecision {
+  status: PreviewAudioStatus
+  /** The key the document wants. Empty when there is no music. */
+  wantedKey: string
+  /** Whether a job should be started for `wantedKey`. */
+  shouldPrepare: boolean
+  /** Whether the video player's own audio must be silenced. */
+  muteSourceAudio: boolean
+  /** Whether playback may run right now. */
+  canPlay: boolean
+}
+
+/** @experimental */
+export interface PreviewAudioInputs {
+  /** The key the current document compiles to, or '' when it has no music. */
+  wantedKey: string
+  /** The key of the mix already prepared and loaded, if any. */
+  readyKey: string | null
+  /** The key a job is currently preparing, if any. */
+  preparingKey: string | null
+  /** The key whose preparation failed, if any. */
+  failedKey: string | null
+}
+
+/**
+ * Whether a project needs a mixed preview at all.
+ *
+ * True when anything in the compiled program is audible *and comes from an
+ * audio item* — a project whose only sound is clip audio does not need a mix,
+ * because the video player already plays exactly that and does it in realtime.
+ *
+ * @experimental
+ */
+export function previewAudioNeeded(project: VideoProject): boolean {
+  return project.audio.length > 0
+}
+
+/**
+ * What to do, given what is wanted and what is on hand.
+ *
+ * @experimental
+ */
+export function decidePreviewAudio(inputs: PreviewAudioInputs): PreviewAudioDecision {
+  const { wantedKey, readyKey, preparingKey, failedKey } = inputs
+
+  if (wantedKey === '') {
+    // No music. The video player's audio is the preview, exactly as it was
+    // before any of this existed.
+    return {
+      status: 'source',
+      wantedKey,
+      shouldPrepare: false,
+      muteSourceAudio: false,
+      canPlay: true,
+    }
+  }
+
+  if (readyKey === wantedKey) {
+    return {
+      status: 'ready',
+      wantedKey,
+      shouldPrepare: false,
+      // The mix contains the clips' audio too. Leaving the source audible
+      // would play the whole timeline twice.
+      muteSourceAudio: true,
+      canPlay: true,
+    }
+  }
+
+  if (failedKey === wantedKey) {
+    /*
+      Refusing rather than falling back.
+
+      The tempting fallback is "play the clip audio instead", and it is wrong:
+      the person would hear a preview missing its music, with no reliable way to
+      tell that from a project whose music is silent. A named failure is worse
+      to look at and better to act on.
+    */
+    return {
+      status: 'failed',
+      wantedKey,
+      shouldPrepare: false,
+      muteSourceAudio: true,
+      canPlay: false,
+    }
+  }
+
+  /*
+    Preparing. A mix for an older document may still be loaded, and it must not
+    play: it is the previous edit's sound, which is the most convincing kind of
+    wrong. Source audio stays muted throughout, so the transition is silence
+    rather than a burst of un-mixed clip audio.
+  */
+  return {
+    status: 'preparing',
+    wantedKey,
+    shouldPrepare: preparingKey !== wantedKey,
+    muteSourceAudio: true,
+    canPlay: false,
+  }
+}
+
+/** The key a project's audio compiles to, given resolved sources. @experimental */
+export function previewProgramJson(project: VideoProject): string {
+  return JSON.stringify(compileAudioMixProgram(project))
+}
+
+/*
+  The drift comparison itself lives in native, not here.
+
+  It used to be a function in this file, and it was never wired to a caller —
+  measured, a clip seam left the mix 240 ms ahead of the picture and it stayed
+  there. Reimplementing it here now would be worse than that: both anchors are
+  published on one monotonic clock in native, and a JavaScript twin would have
+  no way to read them at the same instant, so the two would disagree about what
+  "at the same time" meant. See `PreviewTrace.driftCheck`.
+*/
+
+/**
+ * The default tolerance.
+ *
+ * The drift this used to chase is prevented rather than corrected: `PreviewSync`
+ * holds both transports through a stall and releases them aligned, which is
+ * where all of it came from. A correcting seek during playback costs 200-340 ms
+ * on a Pixel 7 — the same size as the error it was chasing, so it oscillated
+ * between +264 ms and -198 ms — and realignment now happens while everything is
+ * held, where a seek has nothing to chase.
+ *
+ * So this is back to the tolerance the gates were written against. It is a
+ * backstop for something nobody has seen, not the mechanism.
+ *
+ * @experimental
+ */
+export const PREVIEW_DRIFT_TOLERANCE_US = 60_000