@pictor-sdk/video-android 0.1.0

package/src/usePreviewAudio.ts

@@ -0,0 +1,305 @@
+import {
+  compileAudioMixProgram,
+  type ExportSourceContext,
+  type MediaSource,
+  type VideoProject,
+} from '@pictor-sdk/video-core'
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
+
+import { publishState } from './previewAudioDiagnostics'
+import { resolveMixSources } from './mixSources'
+import { requireNative } from './player'
+import {
+  cancelPreviewMix,
+  preparePreviewMix,
+  previewMixKey,
+  sweepPreviewMixes,
+} from './previewMix'
+import {
+  decidePreviewAudio,
+  type PreviewAudioStatus,
+} from './previewAudioSession'
+
+/**
+ * Preview audio, bound to a screen's lifetime.
+ *
+ * The decisions live in `previewAudioSession.ts` and the mixing lives in
+ * native; this is the part that acts — starting jobs, loading the player,
+ * muting the other one, and refusing to publish an answer that has been
+ * superseded.
+ *
+ * ## Epochs
+ *
+ * Every load, play and seek carries the epoch of the mix it belongs to. A
+ * callback or a correction arriving with an older epoch is from a mix that has
+ * since been replaced, and acting on it would move the picture to match audio
+ * that is no longer playing. The epoch is bumped on every publish, so a late
+ * result cannot masquerade as the current one.
+ *
+ * @experimental
+ */
+
+/** @experimental */
+export interface UsePreviewAudioOptions {
+  project: VideoProject
+  /** The same resolver the exporter is given. */
+  resolveSource: (source: MediaSource, context: ExportSourceContext) => string | null | undefined
+  /** Off entirely — for a screen that never plays. */
+  enabled?: boolean
+  /** How far apart the players may drift before audio is moved. */
+  toleranceUs?: number
+}
+
+/** @experimental */
+export interface PreviewAudioBinding {
+  status: PreviewAudioStatus
+  /** A sentence for a person when preparation failed. */
+  error: string | null
+  /** Whether playback may run. False while preparing or after a failure. */
+  canPlay: boolean
+  /** True once a mix matching the document is loaded. */
+  ready: boolean
+  /** How long the last preparation took, in ms. Null when it was cached. */
+  lastPrepareMs: number | null
+  /** Where that time went. Null for a cached mix. */
+  lastProfile: unknown
+  /*
+    `play`, `pause`, `seek`, `setRate` and `checkDrift` were here.
+
+    They were the second half of a two-player preview: this hook played the
+    prepared mix on its own player while the video player played the picture,
+    and `checkDrift` compared them and corrected past a tolerance. The preview
+    now plays both from one composition on one clock, so there is no second
+    position to compare and nothing to correct — the correction was a seek, and
+    a seek at a clip seam cost the decode it was trying to cover.
+
+    What is left is preparation: compile the program, produce the file, and say
+    whether it is ready. The composition opens it. @see useCompositionPlayback
+  */
+  /**
+   * The prepared mix, for a composition that carries the sound itself.
+   *
+   * Null until one matching the document exists. The preview plays audio and
+   * video from one composition on one clock, so what this hook contributes is
+   * the prepared program — the file and how long it runs — rather than a second
+   * player to keep in step with the first. `play`, `pause`, `seek`, `setRate`
+   * and `checkDrift` are what is left of that second player and are no longer
+   * part of the preview path.
+   */
+  mix: { uri: string; durationUs: number } | null
+  /** Cancel, release the player and drop the mix. */
+  release(): void
+}
+
+/** @experimental */
+export function usePreviewAudio(options: UsePreviewAudioOptions): PreviewAudioBinding {
+  const { project, enabled = true } = options
+  const resolveSource = useRef(options.resolveSource)
+  resolveSource.current = options.resolveSource
+
+  const [status, setStatusState] = useState<PreviewAudioStatus>('source')
+  const [error, setErrorState] = useState<string | null>(null)
+  const [readyKey, setReadyKeyState] = useState<string | null>(null)
+  const [lastPrepareMs, setLastPrepareMsState] = useState<number | null>(null)
+  const [lastProfile, setLastProfileState] = useState<unknown>(null)
+  const [mix, setMix] = useState<{ uri: string; durationUs: number } | null>(null)
+
+  /*
+    Every publication goes through one seam. Five setters scattered across
+    three effects is exactly the shape that makes "does rendering cause this"
+    unanswerable, and routing them here costs a call and makes it a switch.
+    See `previewAudioDiagnostics`.
+  */
+  // Memoized so each publisher has one identity for the hook's whole life:
+  // React's setters are stable, so these never rebuild, and the effects below
+  // can depend on them without that dependency meaning anything at runtime.
+  const setStatus = useMemo(() => publishState('status', setStatusState), [setStatusState])
+  const setError = useMemo(() => publishState('error', setErrorState), [setErrorState])
+  const setReadyKey = useMemo(
+    () => publishState('readyKey', setReadyKeyState),
+    [setReadyKeyState],
+  )
+  const setLastPrepareMs = useMemo(
+    () => publishState('lastPrepareMs', setLastPrepareMsState),
+    [setLastPrepareMsState],
+  )
+  const setLastProfile = useMemo(
+    () => publishState('lastProfile', setLastProfileState),
+    [setLastProfileState],
+  )
+
+  const preparingKey = useRef<string | null>(null)
+  const failedKey = useRef<string | null>(null)
+  const epoch = useRef(0)
+  const mounted = useRef(true)
+
+  /**
+   * What this document wants, and what it needs to decode.
+   *
+   * Recomputed whenever the project changes and keyed by the compiled program,
+   * so an edit that cannot change the sound produces the same key and starts
+   * nothing.
+   */
+  const wanted = useMemo(() => {
+    if (!enabled || project.audio.length === 0) {
+      return {
+        key: '',
+        programJson: '',
+        urisJson: '',
+        durationUs: 0,
+        unresolved: [] as string[],
+      }
+    }
+    const program = compileAudioMixProgram(project)
+    const resolved = resolveMixSources(project, program, (s, c) => resolveSource.current(s, c))
+    const programJson = JSON.stringify(program)
+    const urisJson = JSON.stringify(resolved.uris)
+    return {
+      key: resolved.unresolved.length > 0 ? '' : previewMixKey(programJson, urisJson),
+      programJson,
+      urisJson,
+      durationUs: Number(program.durationUs),
+      unresolved: resolved.unresolved,
+    }
+  }, [project, enabled])
+
+  const decision = decidePreviewAudio({
+    wantedKey: wanted.key,
+    readyKey,
+    preparingKey: preparingKey.current,
+    failedKey: failedKey.current,
+  })
+
+  /* A source that cannot be resolved is a failure with a name, not a wait. */
+  const unresolved = wanted.unresolved.length > 0 && enabled && project.audio.length > 0
+
+  useEffect(() => {
+    mounted.current = true
+    return () => {
+      mounted.current = false
+    }
+  }, [])
+
+  /** Silence the other player exactly when the mix owns the sound. */
+  useEffect(() => {
+    if (!enabled) return
+    try {
+      requireNative().previewMuteSourceAudio(decision.muteSourceAudio || unresolved)
+    } catch {
+      // No adapter. The decision still stands for everything else.
+    }
+  }, [decision.muteSourceAudio, unresolved, enabled])
+
+  useEffect(() => {
+    if (unresolved) {
+      setStatus('failed')
+      setError(wanted.unresolved[0] ?? 'a sound could not be read')
+      return
+    }
+    setStatus(decision.status)
+    if (decision.status !== 'failed') setError(null)
+  }, [decision.status, unresolved, wanted.unresolved, setError, setStatus])
+
+  /** Start a job when the document wants a mix nobody is preparing. */
+  useEffect(() => {
+    if (!decision.shouldPrepare || unresolved) return
+    const key = wanted.key
+    preparingKey.current = key
+    const jobId = `preview-${key.slice(0, 8)}-${epoch.current + 1}`
+    const startedAt = Date.now()
+
+    void preparePreviewMix(jobId, wanted.programJson, wanted.urisJson).then((result) => {
+      if (!mounted.current) return
+      /*
+        Two independent staleness checks, and both are needed. The job may have
+        been superseded inside native — it says so — and the document may have
+        moved on while the promise was in flight, which native cannot know.
+      */
+      if (result.ok === false && result.superseded) return
+      if (key !== preparingKey.current) return
+      preparingKey.current = null
+
+      if (result.ok === false) {
+        failedKey.current = key
+        setError(result.reason)
+        setStatus('failed')
+        return
+      }
+
+      failedKey.current = null
+      epoch.current += 1
+      setLastPrepareMs(result.cached ? null : Date.now() - startedAt)
+      setLastProfile(result.cached ? null : (result.profile ?? null))
+      try {
+        requireNative().previewAudioLoad(result.uri, epoch.current)
+      } catch {
+        failedKey.current = key
+        setError('the prepared mix could not be opened')
+        setStatus('failed')
+        return
+      }
+      /*
+        Published for the composition, which is what plays it.
+
+        The duration comes from the compiled program rather than from the file:
+        it is the length the mix was trimmed to, and taking it from the compiler
+        keeps it on the same side of the boundary as every other timing value.
+      */
+      setMix({ uri: result.uri, durationUs: Number(wanted.durationUs) })
+      setReadyKey(result.key)
+      // Bounded: keep the one being played and let the rest go.
+      sweepPreviewMixes([result.key])
+    })
+  }, [
+    decision.shouldPrepare,
+    wanted.key,
+    wanted.programJson,
+    wanted.urisJson,
+    unresolved,
+    setError,
+    setLastPrepareMs,
+    setLastProfile,
+    setReadyKey,
+    setStatus,
+  ])
+
+  /*
+    The second player's transport lived here: `play`, `pause`, `seek`,
+    `setRate` and `checkDrift`.
+
+    Deleted with the architecture that needed them. The preview plays the mix
+    from the composition, on the same clock as the picture, so there is no
+    second position to seek, rate or compare. `previewSyncStop` survives in
+    `release` because a preparation in flight still has to be abandoned.
+  */
+
+  const release = useCallback(() => {
+    cancelPreviewMix()
+    try {
+      requireNative().previewSyncStop('release')
+      requireNative().previewAudioRelease()
+      requireNative().previewMuteSourceAudio(false)
+    } catch {
+      // Teardown paths overlap on purpose; each must be safe after the others.
+    }
+    preparingKey.current = null
+    setReadyKey(null)
+    setMix(null)
+  }, [setReadyKey])
+
+  /* Backgrounding and unmount both stop work and give the player back. */
+  useEffect(() => release, [release])
+
+  return {
+    status: unresolved ? 'failed' : status,
+    error,
+    canPlay: !unresolved && decision.canPlay,
+    ready: decision.status === 'ready',
+    lastPrepareMs,
+    lastProfile,
+    // Null until a mix matching the document is ready, so a composition built
+    // from it is never built from a stale one.
+    mix: decision.status === 'ready' ? mix : null,
+    release,
+  }
+}

package/src/useVideoSurface.ts

@@ -0,0 +1,282 @@
+import { useCallback, useEffect, useRef, useState } from 'react'
+import { AppState, type AppStateStatus } from 'react-native'
+import { runOnJS, runOnUI, useSharedValue } from 'react-native-reanimated'
+import { NO_SURFACE } from './playlistSession'
+import { AndroidVideoPlayer } from './player'
+import { nextOwnerToken, surfaceOwner, type SurfaceCreateResult } from './surfaceOwner'
+
+/**
+ * The presentation surface, and whether it is usable *right now*.
+ *
+ * This was host code, and every host would have got it wrong in the same way.
+ * `create()` must run on the UI runtime, where Skia's GL context lives, and
+ * `runOnUI` **schedules** that rather than performing it — so a host that sets
+ * its state beside the call declares the surface ready a frame or two before the
+ * texture exists. A playlist built in that window is refused outright, and the
+ * editor shows clips on the timeline with a player holding nothing.
+ *
+ * Readiness is therefore reported from *inside* the worklet, after the texture
+ * and the native `Surface` both exist, and it carries **the generation native
+ * assigned**. This hook keeps no counter of its own: a second counter is a
+ * second answer to "which surface is live", and the fault it replaced was
+ * exactly two answers disagreeing.
+ *
+ * ## Level-triggered, not one-shot
+ *
+ * The surface is re-examined whenever the app returns to the foreground, and
+ * marked unavailable when it leaves. Backgrounding is the ordinary case, not an
+ * edge case: choosing media means going to another activity and coming back, and
+ * the clips arrive in the document *while there is no surface to prepare
+ * against*. Anything that only reacts to the document changing has already
+ * missed its chance by the time it returns.
+ *
+ * ## Serialisation is process-wide
+ *
+ * Creation is arbitrated by {@link surfaceOwner}, a module-level singleton,
+ * because the thing it guards — the one external texture — is one per process.
+ * A lock held per hook instance is no lock at all: a remount brings a fresh one,
+ * two creates overlap, and the second overwrites the native handles the first is
+ * still holding. Nor does unmounting release it: the earlier attempt is still
+ * out on the UI runtime and will still come back.
+ *
+ * A surface built by an attempt whose owner has since gone is **released**, not
+ * ignored. It exists, it is registered, and Media3 could attach to it.
+ *
+ * @experimental
+ */
+
+/** How often to retry creating a surface while there is none, in milliseconds. */
+const CREATE_RETRY_MS = 250
+
+/**
+ * The one stable slot, installed on first ask.
+ *
+ * Synchronous, and callable during a render, because the consumer has to be able
+ * to capture it *before* any surface exists. An effect is too late: the frame
+ * callback is built on the first render, and anything it captures then is what
+ * it keeps.
+ *
+ * The object never changes. `create` fills its target in place and retirement
+ * clears it, so every call through it resolves to whatever the slot points at
+ * now. There is deliberately nothing replaceable to hold.
+ *
+ * @experimental
+ */
+export function videoSurfaceSlot(): VideoSurfaceHandle | undefined {
+  const published = (globalThis as unknown as { __pictorVideoSurface?: VideoSurfaceHandle })
+    .__pictorVideoSurface
+  if (published) return published
+  if (!AndroidVideoPlayer.installVideoSurface()) return undefined
+  return (globalThis as unknown as { __pictorVideoSurface?: VideoSurfaceHandle })
+    .__pictorVideoSurface
+}
+
+/** @experimental */
+export interface VideoSurfaceOptions {
+  /** Backing size of the texture. Frames are scaled into it. */
+  width: number
+  height: number
+  enabled?: boolean
+}
+
+/**
+ * The surface as the UI runtime sees it.
+ *
+ * Structural rather than an imported Android type: nothing platform-specific
+ * reaches Pictor's public API.
+ *
+ * @experimental
+ */
+export interface VideoSurfaceHandle {
+  create(
+    width: number,
+    height: number,
+  ): { textureId: number; ok: boolean; generation: number; error?: string }
+  generation(): number
+  alive(): boolean
+  /** Release a named generation. Does nothing if it is not the one held. */
+  releaseGeneration(generation: number): boolean
+}
+
+/** @experimental */
+export interface VideoSurface {
+  /**
+   * The stable slot.
+   *
+   * Available from the first render and never replaced, so a consumer captures
+   * it once and reads through it. It answers `generation() === 0` while empty,
+   * which is how a renderer knows there is nothing to draw — a null would have
+   * to be captured, and a captured null is what it stays.
+   */
+  surface: unknown | undefined
+  /**
+   * The live generation, exactly as native reported it. 0 when there is none.
+   *
+   * Feed to `useMedia3Playback`. Every change is a reason to reconcile the
+   * playlist, including the change back to 0.
+   */
+  generation: number
+  /** Failure to install or create, for a host's error slot. Empty when healthy. */
+  error: string
+}
+
+/** @experimental */
+export function useVideoSurface(options: VideoSurfaceOptions): VideoSurface {
+  const { width, height, enabled = true } = options
+  const [readiness, setReadiness] = useState({ generation: NO_SURFACE, error: '' })
+  // Resolved during render, not in an effect: a consumer building a frame
+  // callback on its first render must have the slot to capture by then.
+  const handleRef = useRef<VideoSurfaceHandle | undefined>(undefined)
+  if (!handleRef.current) handleRef.current = videoSurfaceSlot()
+  const textureId = useSharedValue(0)
+
+  /** This hook instance's identity, so the owner can tell it from a remount. */
+  const tokenRef = useRef(0)
+  if (tokenRef.current === 0) tokenRef.current = nextOwnerToken()
+  const token = tokenRef.current
+
+  const [attempt, setAttempt] = useState(0)
+
+  const ownerRef = useRef<ReturnType<typeof surfaceOwner> | undefined>(undefined)
+
+  /**
+   * Resolve an attempt on the process-wide owner.
+   *
+   * Reached through `runOnJS` from inside the worklet, which is the only place
+   * that can honestly say the texture and the `Surface` exist.
+   */
+  const complete = useCallback((result: SurfaceCreateResult) => {
+    ownerRef.current?.complete(result)
+  }, [])
+
+  const owner = surfaceOwner({
+    create() {
+      const published = handleRef.current
+      if (!published) {
+        complete({ ok: false, generation: 0, textureId: 0, error: '__pictorVideoSurface missing' })
+        return
+      }
+      runOnUI(() => {
+        'worklet'
+        const created = published.create(width, height)
+        textureId.value = created.textureId
+        if (created.ok && !(created.generation > 0)) {
+          runOnJS(complete)({
+            ok: false,
+            generation: 0,
+            textureId: created.textureId,
+            error: 'adapter too old: create() reports no generation',
+          })
+          return
+        }
+        runOnJS(complete)({
+          ok: created.ok && created.textureId > 0,
+          generation: created.generation,
+          textureId: created.textureId,
+          // The native reason, not a summary of it. "No EGL context current on
+          // this thread" and "no JNIEnv" send a reader to different places.
+          error: created.ok ? '' : created.error || 'surface create failed',
+        })
+      })()
+    },
+    retire(generation: number) {
+      const published = handleRef.current
+      if (!published || typeof published.releaseGeneration !== 'function') return
+      // On the UI runtime, because deleting a GL texture needs the context that
+      // owns it to be current — which is true on exactly one thread.
+      runOnUI(() => {
+        'worklet'
+        published.releaseGeneration(generation)
+      })()
+    },
+  })
+  ownerRef.current = owner
+
+  useEffect(() => {
+    const unsubscribe = owner.subscribe(() => setReadiness(owner.readiness))
+    setReadiness(owner.readiness)
+    return unsubscribe
+  }, [owner])
+
+  const ensure = useCallback(() => {
+    if (!enabled) return
+    // Through the same accessor the first render used. Calling
+    // `installVideoSurface` again here is what produced two host objects: the
+    // consumer captured the one installed during render, `create` ran on the one
+    // this effect installed over it, and the two reported different generations
+    // forever with neither being wrong about itself.
+    const published = videoSurfaceSlot()
+    if (!published) {
+      setReadiness((prev) => ({ ...prev, error: '__pictorVideoSurface missing' }))
+      return
+    }
+    handleRef.current = published
+
+    // Already live: adopt it rather than build a second. Guarded because an
+    // adapter built before these accessors answers `undefined`, and calling that
+    // throws inside an effect — which presents as a blank screen rather than as
+    // the version mismatch it is.
+    if (
+      typeof published.alive === 'function' &&
+      typeof published.generation === 'function' &&
+      published.alive() &&
+      published.generation() !== NO_SURFACE
+    ) {
+      owner.adopt(published.generation(), token)
+      return
+    }
+
+    owner.request(token)
+  }, [enabled, owner, token])
+
+  useEffect(() => {
+    ensure()
+  }, [ensure, attempt])
+
+  /**
+   * Keep asking until there is a surface.
+   *
+   * `create` needs Skia's GL context current on the UI runtime, and that is only
+   * true once a canvas has drawn — which has not happened when this hook's first
+   * effect runs. Measured on a Pixel 7: the first attempt returns "no EGL
+   * context current on this thread", and a later one succeeds.
+   *
+   * A retry rather than a wait on some readiness signal, because there is no
+   * such signal: the context becoming current is a side effect of rendering, not
+   * an event anything publishes. The owner refuses to overlap attempts, so
+   * asking often is safe.
+   */
+  useEffect(() => {
+    if (!enabled) return
+    if (readiness.generation !== NO_SURFACE) return
+    const id = setInterval(() => setAttempt((n) => n + 1), CREATE_RETRY_MS)
+    return () => clearInterval(id)
+  }, [enabled, readiness.generation])
+
+  useEffect(() => {
+    if (!enabled) return
+    const onChange = (status: AppStateStatus) => {
+      if (status === 'active') {
+        // Level-triggered: ask again rather than assume the answer survived.
+        ensure()
+        return
+      }
+      // This owner is gone for now. The lock is deliberately not released — an
+      // attempt on the UI runtime is still out there — and a surface it produces
+      // will be retired rather than published to nobody.
+      owner.retireOwner(token)
+    }
+    const subscription = AppState.addEventListener('change', onChange)
+    return () => subscription.remove()
+  }, [enabled, ensure, owner, token])
+
+  useEffect(() => () => owner.retireOwner(token), [owner, token])
+
+  return {
+    // Always the slot, never undefined once installed. Gating this on readiness
+    // is what put a null into the consumer's closure and kept it there.
+    surface: handleRef.current,
+    generation: readiness.generation,
+    error: readiness.error,
+  }
+}