@pictor-sdk/video-android 0.1.0

package/src/useMedia3Playback.ts

@@ -0,0 +1,492 @@
+import {
+  playIntent,
+  playableDuration,
+  playlistOf,
+  projectDuration,
+  projectTimeToWindow,
+  windowToProjectTime,
+  type PlayIntent,
+  type PlaylistItem,
+  type UnsupportedSource,
+  type VideoProject,
+} from '@pictor-sdk/video-core'
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
+import { AppState } from 'react-native'
+import { Media3, type Media3State } from './media3'
+import { createPlaybackLifecycle, type PlaybackLifecycle } from './playbackLifecycle'
+import {
+  classifyStall,
+  shouldRecover,
+  type RecoverySample,
+  type StallKind,
+} from './surfaceRecovery'
+import {
+  NO_SURFACE,
+  afterPrepare,
+  idlePlaylistSession,
+  shouldPrepare,
+} from './playlistSession'
+
+/**
+ * Keep a Media3 playlist in step with a project, and the playhead with playback.
+ *
+ * Two directions, deliberately separated:
+ *
+ * - **Project → player.** When the clips, their order or their trims change,
+ *   the playlist is replaced. The current *project time* is preserved across
+ *   that, not the window index — deleting the second of four clips must not
+ *   leave the playhead in whatever the second window now happens to be.
+ * - **Player → playhead.** While playing, the player's position is polled at a
+ *   readable rate and mapped back to project time. This is a **throttled
+ *   semantic update**, not a per-frame one: the picture is composed on the UI
+ *   runtime from the external texture and never waits for React.
+ *
+ * The playlist is only replaced when it genuinely differs. A trim drag
+ * dispatches a command per finger sample, and rebuilding the playlist on each
+ * would restart decoding many times a second.
+ *
+ * @experimental
+ */
+
+/** @experimental */
+export interface UseMedia3PlaybackOptions {
+  project: VideoProject
+  /** Timeline time the host wants shown. */
+  playheadUs: number
+  /** Called with the project time as playback advances. Throttled. */
+  onPositionUs?: (timeUs: number) => void
+  /** How often to poll the player, in milliseconds. */
+  pollMs?: number
+  /**
+   * The live presentation surface generation, from `useVideoSurface`.
+   *
+   * Not a boolean. A playlist prepared against a retired surface renders into
+   * something nobody is watching — the player decodes, releases frames and
+   * reports no error — so the question is *which* surface, not whether one
+   * exists. Every change is a reason to reconcile, including the change to 0.
+   */
+  surfaceGeneration?: number
+  enabled?: boolean
+}
+
+/** @experimental */
+export interface Media3Playback {
+  playing: boolean
+  /**
+   * Start or resume playback, replaying from zero when there is nothing left.
+   *
+   * Returns what it decided, so a host can show it or count it. See
+   * `playIntent` in `@pictor-sdk/video-core` for why the end is a special case.
+   */
+  play(): PlayIntent
+  pause(): void
+  setRate(rate: number): void
+  /** Seek in project time. Mapped to the player's own coordinates. */
+  seek(timeUs: number): void
+  state: Media3State | null
+  /** The windows currently loaded, for diagnostics and gates. */
+  items: readonly PlaylistItem[]
+  /**
+   * Clips the player cannot open. Non-empty means playback is **not** loaded.
+   *
+   * Surfaced rather than swallowed: the alternative is an editor that plays a
+   * shorter timeline than it draws, which reads as a synchronisation bug.
+   */
+  unsupported: readonly UnsupportedSource[]
+  /**
+   * Which half of the pipeline stalled after a surface restoration, or `null`.
+   *
+   * Non-null means a black picture was detected and the handshake re-run — a
+   * host should surface it rather than show a silent audio-only result.
+   */
+  stall: StallKind | null
+  /**
+   * This screen's ownership of the shared player, for diagnostics and gates.
+   *
+   * Exposed so a host can log which path stopped playback and assert that a
+   * screen released exactly once.
+   */
+  lifecycle: PlaybackLifecycle
+  /**
+   * How far the timeline and the player disagree, in microseconds. Zero when
+   * everything is playable, and asserted by the device gates.
+   */
+  divergenceUs: number
+  /**
+   * The player's position in project time, read now, without rendering.
+   *
+   * For a host driving a UI-thread playhead: it samples this a few times a
+   * second and advances between samples itself. Returns `null` when the player
+   * has no position to give — before a playlist is loaded, or after release.
+   */
+  positionUs(): number | null
+  /**
+   * The rate actually in force: clip speed and preview rate, composed and
+   * clamped by the adapter. A host showing a rate should show this one.
+   */
+  effectiveRate: number
+  /**
+   * The last seek this hook actually issued.
+   *
+   * Global time in, and the `{windowIndex, localUs}` it was converted to. A
+   * player reporting an unexpected position is either being sent the wrong
+   * thing or mishandling the right thing, and those need different fixes — this
+   * is what tells them apart instead of leaving it to arithmetic on a
+   * screenshot.
+   */
+  lastSeek: { globalUs: number; windowIndex: number; localUs: number } | null
+}
+
+const DEFAULT_POLL_MS = 100
+
+/** How often a refused preparation is retried, in milliseconds. */
+const RETRY_INTERVAL_MS = 250
+
+/** @experimental */
+export function useMedia3Playback(options: UseMedia3PlaybackOptions): Media3Playback {
+  const { project, playheadUs, enabled = true } = options
+  const surfaceGeneration = options.surfaceGeneration ?? NO_SURFACE
+  const pollMs = options.pollMs ?? DEFAULT_POLL_MS
+  const [state, setState] = useState<Media3State | null>(null)
+  const [playing, setPlaying] = useState(false)
+
+  const playlist = useMemo(() => playlistOf(project), [project])
+  const items = playlist.items
+  const divergenceUs = useMemo(
+    () => projectDuration(project) - playableDuration(playlist),
+    [project, playlist],
+  )
+
+  /**
+   * A stable description of the playlist.
+   *
+   * Compared as a string rather than by reference: every edit produces a new
+   * project object, but most edits — a selection, a playhead move, an opacity
+   * change — do not change what the player should be playing.
+   */
+  const signature = useMemo(
+    () =>
+      items
+        .map((i) => `${JSON.stringify(i.source)}|${i.startUs}|${i.endUs}|${i.speed}|${i.muted}`)
+        .join(';') +
+      // Unsupported clips belong in the signature even though they are not
+      // windows. Adding a photo between two videos leaves the video windows
+      // identical, so a signature built from items alone would not change and
+      // the player would keep playing a timeline that had grown underneath it —
+      // the exact divergence this is all meant to prevent.
+      `!${playlist.unsupported.map((u) => `${u.clipId}:${u.reason}`).join(',')}`,
+    [items, playlist.unsupported],
+  )
+
+  const playheadRef = useRef(playheadUs)
+  playheadRef.current = playheadUs
+  const itemsRef = useRef(items)
+  itemsRef.current = items
+
+  const session = useRef(idlePlaylistSession())
+  /** Bumped by the retry timer, purely to run the reconciliation again. */
+  const [retry, setRetry] = useState(0)
+
+  /**
+   * Reconcile what Media3 holds with what the document and the surface imply.
+   *
+   * Level-triggered. The old version fired on the playlist *changing*, which is
+   * the one moment it must not rely on: clips are added while the system picker
+   * is closing and there is no surface to prepare against, so the change has
+   * already happened by the time preparation becomes possible. See
+   * `playlistSession.ts`.
+   */
+  useEffect(() => {
+    if (!enabled) return
+    const intent = { signature, generation: surfaceGeneration }
+    if (!shouldPrepare(session.current, intent)) return
+
+    let ok: boolean
+    try {
+      if (playlist.unsupported.length > 0 || items.length === 0) {
+        // Deliberately not loaded. The host is told why through `unsupported`,
+        // and the timeline keeps drawing — what it must not do is play
+        // something shorter than what it shows and call that working.
+        ok = Media3.setPlaylist({ items: [], unsupported: [] }) !== false
+      } else {
+        // Preserve project time, not window index. The window that held the
+        // playhead may not be the same window — or may not exist — after the
+        // edit.
+        const start = projectTimeToWindow(items, playheadRef.current) ?? {
+          windowIndex: 0,
+          positionUs: 0,
+        }
+        ok = Media3.setPlaylist(playlist, start) !== false
+      }
+    } catch {
+      // A refusal, not a crash. Recording it as applied is what left an editor
+      // waiting for an edit the user had already made.
+      ok = false
+    }
+    session.current = afterPrepare(session.current, intent, ok)
+  }, [signature, items, playlist, enabled, surfaceGeneration, retry])
+
+  /**
+   * Retry a refused preparation without waiting for anything else to move.
+   *
+   * A surface can arrive without its generation changing again, and a user who
+   * has added their clips will not add them a second time to prod it.
+   */
+  useEffect(() => {
+    if (!enabled) return
+    if (!session.current.pending) return
+    const id = setInterval(() => setRetry((n) => n + 1), RETRY_INTERVAL_MS)
+    return () => clearInterval(id)
+  }, [enabled, retry, surfaceGeneration, signature])
+
+  /*
+    This screen's ownership of the process-wide player.
+
+    Media3's player outlives any screen, so leaving the editor while playing
+    left the audio running and backgrounding the app did the same — the picture
+    stopped because its surface was retired, and nothing had told the player.
+    One lifecycle per hook instance: a screen mounted later starts from nothing
+    rather than inheriting what the last one was doing.
+  */
+  const stallRef = useRef<StallKind | null>(null)
+  const surfaceGenerationRef = useRef(surfaceGeneration ?? 0)
+  surfaceGenerationRef.current = surfaceGeneration ?? 0
+  const lifecycleRef = useRef<PlaybackLifecycle | null>(null)
+  if (lifecycleRef.current === null) {
+    lifecycleRef.current = createPlaybackLifecycle({
+      pause: () => Media3.pause(),
+      release: () => Media3.release(),
+    })
+  }
+  const lifecycle = lifecycleRef.current
+  /*
+    The epoch, made visible to React.
+
+    The polls capture an epoch when they start and compare on every tick, so
+    resuming has to restart them under the new one — otherwise a screen that
+    came back from the background would poll forever under an epoch that no
+    longer matches and never report a position again.
+  */
+  const [lifecycleEpoch, setLifecycleEpoch] = useState(lifecycle.epoch)
+
+  /*
+    Backgrounding pauses; returning re-arms.
+
+    Suspend rather than teardown: the picker comes up over the editor as a
+    background transition, and rebuilding a player the screen never lost would
+    be a stall nobody asked for. The document is not touched either way —
+    playback stopping says nothing about where the playhead should be.
+  */
+  useEffect(() => {
+    const subscription = AppState.addEventListener('change', (next) => {
+      if (next === 'active') {
+        lifecycle.resume()
+        setLifecycleEpoch(lifecycle.epoch)
+        return
+      }
+      lifecycle.suspend('background')
+      setLifecycleEpoch(lifecycle.epoch)
+      setPlaying(false)
+    })
+    return () => subscription.remove()
+  }, [lifecycle])
+
+  /*
+    The screen going is the end of its playback, exactly once.
+
+    Last of the effects declared here, so it is the first to be cleaned up: a
+    teardown that ran after the polls were cleared would release a player the
+    polls were still reading.
+  */
+  useEffect(() => {
+    return () => {
+      lifecycle.teardown('unmount')
+    }
+  }, [lifecycle])
+
+  /*
+    The watchdog for a surface that came back half-attached.
+
+    The failure it exists for reports nothing: after a background or a picker
+    the clock advances, audio plays, no error is raised and the picture is
+    black. A sample is kept from the last time the pipeline looked healthy, and
+    when the clock has moved for long enough without a frame being latched the
+    handshake is run again. `classifyStall` names which half is broken, because
+    re-preparing fixes a stale generation and does nothing for a consumer that
+    has stopped pulling.
+  */
+  const healthy = useRef<RecoverySample | null>(null)
+  const [stall, setStall] = useState<StallKind | null>(null)
+
+  const lastMeaning = useRef('')
+
+  useEffect(() => {
+    if (!enabled) return
+    // Captured now, checked on every tick: an interval is not cleared the
+    // instant the screen goes, and a poll that lands afterwards must not write
+    // a position into a screen that is no longer there.
+    const epoch = lifecycle.epoch
+    const id = setInterval(() => {
+      if (!lifecycle.accepts(epoch)) return
+      const snapshot = Media3.state()
+      // Position is not part of this. Replacing the state object on every tick
+      // re-renders the whole editor ten times a second to move a line, which is
+      // both wasteful and — because it lands at ten hertz — visibly steppy. The
+      // meaning of the state changes far less often than its numbers do, so
+      // only a change in meaning is worth a render, and a host that wants the
+      // position drives it from `positionUs()` on the UI thread.
+      const meaning = [
+        snapshot.playing,
+        snapshot.ready,
+        snapshot.buffering,
+        snapshot.ended,
+        snapshot.released,
+        snapshot.width,
+        snapshot.height,
+        snapshot.windowIndex,
+        snapshot.windowCount,
+        snapshot.errorCount,
+        snapshot.errorMessage,
+        snapshot.rateConstrained,
+      ].join('|')
+      if (meaning !== lastMeaning.current) {
+        lastMeaning.current = meaning
+        setState(snapshot)
+        setPlaying(snapshot.playing)
+      }
+      const sample: RecoverySample = {
+        atMs: Date.now(),
+        positionUs: snapshot.windowPositionUs,
+        framesOffered: snapshot.framesOffered,
+        framesAvailable: snapshot.framesAvailable,
+        // Latched is not reported directly: what arrived, less what is still
+        // waiting, is the count the consumer has actually taken.
+        framesLatched: snapshot.framesAvailable - snapshot.pendingDepth,
+        framesRejected: snapshot.framesRejected,
+        pendingGeneration: snapshot.pendingGeneration,
+        consumerGeneration: surfaceGenerationRef.current,
+      }
+      const since = healthy.current
+      if (since === null) {
+        healthy.current = sample
+      } else if (classifyStall(since, sample) === null) {
+        // Healthy, or simply not running: either way this is the new baseline.
+        healthy.current = sample
+        if (stallRef.current !== null) setStall(null)
+      } else if (shouldRecover(since, sample)) {
+        const kind = classifyStall(since, sample)
+        healthy.current = sample
+        stallRef.current = kind
+        setStall(kind)
+        // Re-run the surface/playlist handshake. Never silently: a plausible
+        // audio-only result is the thing this refuses to leave alone.
+        session.current = { ...session.current, pending: true }
+        setRetry((n) => n + 1)
+      }
+
+      if (!snapshot.playing) return
+      const projectUs = windowToProjectTime(
+        itemsRef.current,
+        snapshot.windowIndex,
+        snapshot.windowPositionUs,
+      )
+      if (projectUs !== null) options.onPositionUs?.(projectUs)
+    }, pollMs)
+    return () => clearInterval(id)
+    // `onPositionUs` is read through the closure on each tick; depending on it
+    // would restart the interval whenever the host re-created the callback.
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [enabled, pollMs, lifecycle, lifecycleEpoch])
+
+  const lastSeekRef = useRef<Media3Playback['lastSeek']>(null)
+
+  /**
+   * Seek in **global** timeline time.
+   *
+   * The store's playhead is global and stays global; the conversion to a window
+   * and a position inside it happens here, once, through `projectTimeToWindow` —
+   * the same mapping `play` and the playlist load use. Handing a player a global
+   * time as though it were window-local is silent: it clamps to the end of
+   * whichever clip it is in and reports a perfectly plausible position.
+   */
+  const seek = useCallback((timeUs: number) => {
+    const target = projectTimeToWindow(itemsRef.current, timeUs)
+    if (!target) return
+    lastSeekRef.current = {
+      globalUs: Math.round(timeUs),
+      windowIndex: target.windowIndex,
+      localUs: target.positionUs,
+    }
+    Media3.seekToWindow(target.windowIndex, target.positionUs)
+  }, [])
+
+  const positionUs = useCallback((): number | null => {
+    const snapshot = Media3.state()
+    if (snapshot.released || itemsRef.current.length === 0) return null
+    return windowToProjectTime(
+      itemsRef.current,
+      snapshot.windowIndex,
+      snapshot.windowPositionUs,
+    )
+  }, [])
+
+  const durationUsRef = useRef(0)
+  durationUsRef.current = projectDuration(project)
+
+  /**
+   * Play, replaying from the start if there is nothing left to play.
+   *
+   * The decision is `playIntent` in `@pictor-sdk/video-core`, not an `if` here: it
+   * is the same question a host's own transport has to answer, and two answers
+   * would eventually disagree. Read fresh from the player rather than from
+   * `state`, which only re-renders on a change of meaning and can be a poll
+   * behind at the moment of the press.
+   */
+  const play = useCallback(() => {
+    const snapshot = Media3.state()
+    const intent = playIntent({
+      positionUs:
+        windowToProjectTime(
+          itemsRef.current,
+          snapshot.windowIndex,
+          snapshot.windowPositionUs,
+        ) ?? 0,
+      durationUs: durationUsRef.current,
+      playbackState: snapshot.playbackState,
+      ended: snapshot.ended,
+    })
+    if (intent.seekToUs !== null) {
+      const target = projectTimeToWindow(itemsRef.current, intent.seekToUs)
+      if (target) Media3.seekToWindow(target.windowIndex, target.positionUs)
+    }
+    if (!intent.play) return intent
+    // Refused once the screen has given the player back: a late press from a
+    // control that has not unmounted yet must not start a released player.
+    if (lifecycle.released) return intent
+    Media3.play()
+    setPlaying(true)
+    return intent
+  }, [lifecycle])
+
+  return {
+    playing,
+    play,
+    get lastSeek() {
+      return lastSeekRef.current
+    },
+    positionUs,
+    effectiveRate: state?.effectiveRate ?? 1,
+    pause: useCallback(() => {
+      Media3.pause()
+      setPlaying(false)
+    }, []),
+    setRate: useCallback((rate: number) => Media3.setRate(rate), []),
+    seek,
+    state,
+    items,
+    unsupported: playlist.unsupported,
+    divergenceUs,
+    lifecycle,
+    stall,
+  }
+}