@pictor-sdk/video-android 0.1.0

package/src/mixSources.ts

@@ -0,0 +1,82 @@
+import type {
+  AudioMixProgram,
+  ExportSourceContext,
+  MediaSource,
+  VideoProject,
+} from '@pictor-sdk/video-core'
+
+/**
+ * Turning a compiled mix program's spans into files to decode.
+ *
+ * Shared by the exporter and by preview preparation, deliberately. The two must
+ * agree about which file a span means, or a preview would be mixed from one set
+ * of sources and the export from another — and the difference would only be
+ * audible in the finished video.
+ *
+ * @experimental
+ */
+
+/**
+ * What a span is, so a host can resolve it as what it actually is.
+ *
+ * A span carved from a clip's own audio resolves as that clip — the same
+ * context the video pass used, so the same media resolves the same way. A span
+ * from a music item resolves as that item. The id alone cannot say which, and a
+ * host that guessed would resolve the first thing that matched.
+ */
+export function mixSpanContext(
+  project: VideoProject,
+  spanId: string,
+): ExportSourceContext | null {
+  const isClip = project.clips.some((clip) => clip.id === spanId)
+  const isAudio = project.audio.some((item) => item.id === spanId)
+  // Ambiguous or absent are the same answer: there is no one owner to resolve
+  // against, and picking either would be a guess.
+  if (isClip === isAudio) return null
+  return isClip ? { kind: 'clip', clipId: spanId } : { kind: 'audio', audioId: spanId }
+}
+
+/** @experimental */
+export interface ResolvedMixSources {
+  /** Span id to readable URI, for every span that needs decoding. */
+  uris: Record<string, string>
+  /** One sentence per span that could not be resolved. Empty means usable. */
+  unresolved: string[]
+}
+
+/**
+ * Resolve every span that needs decoding.
+ *
+ * Silent spans are skipped: they occupy their time and contribute no samples,
+ * so asking a host to resolve one would be asking for a file that is not
+ * needed — and a host that returned nothing would look like a failure.
+ *
+ * @experimental
+ */
+export function resolveMixSources(
+  project: VideoProject,
+  program: AudioMixProgram,
+  resolveSource: (source: MediaSource, context: ExportSourceContext) => string | null | undefined,
+): ResolvedMixSources {
+  const uris: Record<string, string> = {}
+  const unresolved: string[] = []
+
+  for (const span of program.spans) {
+    if (span.silent || span.source === null) continue
+    const context = mixSpanContext(project, span.id)
+    if (context === null) {
+      unresolved.push(
+        `audio span '${span.id}' matches no clip or audio item, so it has no owner to resolve against`,
+      )
+      continue
+    }
+    const uri = resolveSource(span.source, context)
+    if (typeof uri !== 'string' || uri.length === 0) {
+      unresolved.push(`audio span '${span.id}' could not be resolved to a readable source`)
+      continue
+    }
+    uris[span.id] = uri
+  }
+
+  return { uris, unresolved }
+}

package/src/overlayStaging.ts

@@ -0,0 +1,216 @@
+import { requireNative } from './player'
+
+/**
+ * Where an export's overlay rasters are written.
+ *
+ * ## Why the adapter owns this
+ *
+ * `OverlayResources.kt`, in this package, is what reads these files back and
+ * checks their length and CRC. The writer belongs beside the reader: a host
+ * that produced the bytes *and* chose where they lived would need a filesystem
+ * dependency and a path convention, and the two ends of one format would be
+ * maintained apart.
+ *
+ * It also keeps the promise the rest of the library makes. `@pictor-sdk/video-core`,
+ * `@pictor-sdk/video-text` and the example app acquire no filesystem package for
+ * this; they receive a structural sink and never learn a path.
+ *
+ * ## Why it is synchronous
+ *
+ * `OverlayRasterSink.stage` must return a descriptor before the renderer can
+ * continue building its preparation, and React Native has no synchronous
+ * filesystem. The bytes are written by a JSI host function, which runs on the
+ * calling thread and returns a value like any other function — no bridge round
+ * trip, no second event loop, no waiting on a promise that cannot be awaited.
+ *
+ * That is a real cost: the JS thread is blocked for the write. Measured on a
+ * Pixel 7, release build, at roughly 5 ms per megabyte:
+ *
+ * | raster        | bytes  | stall    |
+ * |---------------|--------|----------|
+ * | 671 x 114     | 299 KB | 2 ms     |
+ * | 1080 x 400    | 1.7 MB | 7 ms     |
+ * | 1080 x 1920   | 8.1 MB | 36–48 ms |
+ * | 3240 x 5760   | 71 MB  | 357 ms   |
+ *
+ * The first three are the operating point: a raster covers the overlay's own
+ * bounds at export scale, not the project, and the two overlays in the Text
+ * acceptance document staged 426 KB and 306 KB. A full-bleed overlay in a
+ * 1080x1920 export is the realistic worst case at ~40 ms. The last row is the
+ * pessimistic bound — a project-sized raster at 3x — and it is what the note
+ * below is about.
+ *
+ * All of it is paid once per overlay during preparation, before the export
+ * begins and before there is a progress bar to stutter. It is not a per-frame
+ * cost. So the synchronous contract stands at these sizes; see
+ * `measureStagingStall` for when it should not.
+ *
+ * @experimental
+ */
+
+/** What native returns for one staged raster. */
+interface StagedDescriptor {
+  /** Opaque ownership handle. The only thing that may be released. */
+  token: string
+  /** Where it went. Read by the export reader; not shown to anyone. */
+  file: string
+  byteLength: number
+  checksum: number
+}
+
+interface StagingHost {
+  stage(bytes: ArrayBuffer, width: number, height: number): StagedDescriptor
+  release(token: string): boolean
+  releaseAll(): number
+  sweep(): number
+  ledger(): { created: number; released: number; live: number }
+}
+
+declare const global: { __pictorOverlayStaging?: StagingHost }
+
+let installed = false
+
+/**
+ * Install the staging primitive, once.
+ *
+ * Idempotent, and safe to call before an export rather than at module load: a
+ * host that never exports an overlay never pays for it.
+ */
+export function installOverlayStaging(): boolean {
+  if (installed && global.__pictorOverlayStaging !== undefined) return true
+  installed = requireNative().installOverlayStaging()
+  return installed && global.__pictorOverlayStaging !== undefined
+}
+
+function host(): StagingHost {
+  const h = global.__pictorOverlayStaging
+  if (h === undefined) {
+    throw new Error('@pictor-sdk/video-android: overlay staging is not installed')
+  }
+  return h
+}
+
+/** What a sink hands back, plus the token that owns it. @experimental */
+export interface StagedRaster {
+  file: string
+  byteLength: number
+  checksum: number
+}
+
+/**
+ * A sink, and the tokens it is holding.
+ *
+ * The token list is the whole point of the wrapper: the structural
+ * `OverlayRasterSink` contract identifies a staged raster by its file, and a
+ * caller that released by path would be naming something it was never given
+ * authority over. Every release here goes through the token native issued.
+ *
+ * @experimental
+ */
+export interface OverlayStagingSink {
+  stage(
+    bytes: Uint8Array,
+    meta: { overlayId: string; resourceId: string; rasterWidth: number; rasterHeight: number },
+  ): StagedRaster
+  /** Discard one staged raster, by the file the descriptor carried. */
+  remove(file: string): void
+  /** Discard everything this sink still holds. Idempotent. */
+  releaseAll(): number
+  /** For assertions: created, released and still live. */
+  ledger(): { created: number; released: number; live: number }
+}
+
+/**
+ * Remove sidecars a previous process left behind.
+ *
+ * Scoped by native to this module's own directory and file suffix, so it cannot
+ * take a cache file belonging to anything else. Returns how many it removed.
+ *
+ * @experimental
+ */
+export function sweepOverlayStaging(): number {
+  if (!installOverlayStaging()) return 0
+  return host().sweep()
+}
+
+/** @experimental */
+export function createOverlayStagingSink(): OverlayStagingSink {
+  if (!installOverlayStaging()) {
+    throw new Error('@pictor-sdk/video-android: overlay staging could not be installed')
+  }
+  /** file → token, so a structural `remove(file)` can release exactly. */
+  const tokens = new Map<string, string>()
+
+  return {
+    stage(bytes, meta) {
+      /*
+        A view over a larger buffer is copied rather than passed whole.
+
+        `Uint8Array` may be a window onto a bigger `ArrayBuffer`, and handing
+        native the underlying buffer would stage the whole thing — the right
+        length by accident when the view happens to cover it, and silently wrong
+        when it does not.
+      */
+      const exact =
+        bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength
+          ? (bytes.buffer as ArrayBuffer)
+          : (bytes.slice().buffer as ArrayBuffer)
+
+      const staged = host().stage(exact, meta.rasterWidth, meta.rasterHeight)
+      tokens.set(staged.file, staged.token)
+      return {
+        file: staged.file,
+        byteLength: staged.byteLength,
+        checksum: staged.checksum,
+      }
+    },
+
+    remove(file) {
+      const token = tokens.get(file)
+      // Releasing something never staged, or already released, is not an error:
+      // the caller's cleanup paths overlap on purpose, and each must be safe to
+      // run after the others.
+      if (token === undefined) return
+      tokens.delete(file)
+      host().release(token)
+    },
+
+    releaseAll() {
+      tokens.clear()
+      return host().releaseAll()
+    },
+
+    ledger() {
+      return host().ledger()
+    },
+  }
+}
+
+/**
+ * How long a synchronous stage blocks the JS thread, for a raster this size.
+ *
+ * Exists because the synchronous contract is only defensible if the number is
+ * small. If a maximum-size raster stalls long enough to drop frames in the
+ * export sheet, the answer is to make `OverlayRasterSink.stage` asynchronous —
+ * not to keep a blocking write behind a synchronous signature because it is
+ * convenient.
+ *
+ * @experimental
+ */
+export function measureStagingStall(
+  sink: OverlayStagingSink,
+  width: number,
+  height: number,
+): { ms: number; bytes: number } {
+  const bytes = new Uint8Array(width * height * 4)
+  const at = Date.now()
+  const staged = sink.stage(bytes, {
+    overlayId: 'measure',
+    resourceId: 'measure',
+    rasterWidth: width,
+    rasterHeight: height,
+  })
+  const ms = Date.now() - at
+  sink.remove(staged.file)
+  return { ms, bytes: bytes.byteLength }
+}

package/src/playbackLifecycle.ts

@@ -0,0 +1,137 @@
+/**
+ * Who owns playback, and until when.
+ *
+ * Media3's player is a **process-wide singleton**. Nothing about it ends when
+ * the screen that started it goes away, so on a Pixel 7 leaving the editor while
+ * playing left the audio running, and backgrounding the app did the same: the
+ * picture stopped because its surface was retired, and the sound carried on
+ * because nobody had told the player anything.
+ *
+ * A singleton with no owner is the whole problem, so this gives it one. A screen
+ * takes ownership when it mounts and gives it back exactly once — and a screen
+ * that mounts afterwards starts from nothing rather than inheriting whatever the
+ * last one was doing.
+ *
+ * ## Two kinds of stopping, deliberately different
+ *
+ * **Suspend** is temporary: the app went to the background, or the picker came
+ * up over the editor. Playback pauses and the player is kept, because the screen
+ * is coming back and rebuilding a player it never lost would be a stall the user
+ * did nothing to earn.
+ *
+ * **Teardown** is final: the screen unmounted, or navigation left it. The player
+ * is paused *and then* released, in that order, so nothing is audible in the
+ * gap. After a teardown this object does nothing at all — a late suspend from a
+ * listener that has not been removed yet must not resurrect anything.
+ *
+ * Neither is a reason to touch the document. Playback stopping says nothing
+ * about what the project is or where its playhead should be, and clearing either
+ * would turn leaving a screen into an edit.
+ *
+ * ## Epochs
+ *
+ * Every stop bumps an epoch, and callbacks carry the epoch they were scheduled
+ * under. A poll that fires after the screen backgrounded still runs — clearing
+ * an interval is not instantaneous — and `accepts` is how it finds out that the
+ * answer it is about to deliver is no longer wanted.
+ *
+ * @experimental
+ */
+
+/** Why playback stopped. Recorded so a log can say which path ran. @experimental */
+export type PlaybackStopReason = 'unmount' | 'background' | 'blur'
+
+/** @experimental */
+export interface PlaybackLifecycleEffects {
+  /** Stop the sound. Must be safe to call when already paused. */
+  pause(): void
+  /** Give up the native player, and with it audio focus. */
+  release(): void
+}
+
+/** @experimental */
+export interface PlaybackLifecycle {
+  /** Whether this screen currently owns running playback. */
+  readonly active: boolean
+  /** Whether the player has been given back. Final. */
+  readonly released: boolean
+  /** Why it last stopped, or `null`. */
+  readonly reason: PlaybackStopReason | null
+  /** Bumped by every stop; callbacks carry the value they were scheduled under. */
+  readonly epoch: number
+  /** Whether work scheduled under `epoch` may still act. */
+  accepts(epoch: number): boolean
+  /** Pause and keep the player. Idempotent within one suspension. */
+  suspend(reason: PlaybackStopReason): void
+  /** Become active again. Idempotent; ignored after teardown. */
+  resume(): void
+  /** Pause, then release, exactly once, forever. */
+  teardown(reason: PlaybackStopReason): void
+}
+
+/** @experimental */
+export function createPlaybackLifecycle(
+  effects: PlaybackLifecycleEffects,
+): PlaybackLifecycle {
+  let active = true
+  let released = false
+  let reason: PlaybackStopReason | null = null
+  let epoch = 1
+
+  return {
+    get active() {
+      return active
+    },
+    get released() {
+      return released
+    },
+    get reason() {
+      return reason
+    },
+    get epoch() {
+      return epoch
+    },
+
+    accepts(at) {
+      // Never after teardown, whatever epoch is offered: the player is gone and
+      // there is nothing a late callback could correctly do with it.
+      if (released) return false
+      return active && at === epoch
+    },
+
+    suspend(why) {
+      if (released || !active) return
+      active = false
+      reason = why
+      epoch += 1
+      effects.pause()
+    },
+
+    resume() {
+      if (released || active) return
+      active = true
+      reason = null
+      epoch += 1
+    },
+
+    teardown(why) {
+      if (released) return
+      const wasActive = active
+      released = true
+      active = false
+      reason = why
+      epoch += 1
+      /*
+        Pause first, then release — and only if it was still running.
+
+        Releasing a playing player is the one ordering that can be heard: the
+        release is not instantaneous, and the gap between "still playing" and
+        "gone" is exactly the audio that outlived the screen. A screen torn down
+        while already suspended was paused on the way in, so pausing it again
+        would be a native call that says nothing.
+      */
+      if (wasActive) effects.pause()
+      effects.release()
+    },
+  }
+}

package/src/player.ts

@@ -0,0 +1,39 @@
+/**
+ * Reaching the native adapter.
+ *
+ * This file used to hold `AndroidVideoPlayer`: a full player over `MediaExtractor`,
+ * `MediaCodec`, `AudioTrack` and an `ImageReader` pool of `AHardwareBuffer`s.
+ * That path reached one-frame synchronisation and is gone. Media3's ExoPlayer
+ * rendering into the persistent external-OES `SurfaceTexture` replaced it, for
+ * the reasons in `docs/research/media3-mapping.md` — codec pacing, buffer
+ * ownership, async-flush races and per-device workarounds are ExoPlayer's
+ * problem to have already solved, not Pictor's to solve again.
+ *
+ * What remains is the module accessor and the one call that is not playback:
+ * publishing the presentation surface.
+ *
+ * @experimental
+ */
+import Native from './NativePictorVideo'
+
+/** @experimental */
+export const AndroidVideoPlayer = {
+  /**
+   * Publish the presentation surface host object on the JS runtime.
+   *
+   * It is captured into a worklet closure from there; the UI runtime has its
+   * own global and does not see this one.
+   */
+  installVideoSurface(): boolean {
+    return requireNative().installVideoSurface()
+  },
+}
+
+export function requireNative() {
+  if (!Native) {
+    throw new Error(
+      '@pictor-sdk/video-android: the native module is not installed. It is Android-only and requires minSdkVersion 29.',
+    )
+  }
+  return Native
+}

package/src/playlistSession.ts

@@ -0,0 +1,125 @@
+/**
+ * Which playlist Media3 is holding, and against which surface.
+ *
+ * The bug this exists to prevent, in order:
+ *
+ * 1. The editor opens and builds surface A.
+ * 2. The system picker opens. The app backgrounds; A is no longer usable.
+ * 3. Two clips are added to the document while the picker is closing.
+ * 4. The editor returns and builds surface B.
+ * 5. Nothing prepares, because the only thing that ever triggered preparation
+ *    was the *playlist changing*, and it changed in step 3 — when there was no
+ *    surface to prepare against. `setPlaylist` was refused, the signature was
+ *    recorded as applied anyway, and the editor sat there with two clips on the
+ *    timeline and a player holding nothing.
+ *
+ * Two corrections follow from that. Readiness is **level-triggered**: the
+ * question is "does the playlist Media3 holds match the document, against the
+ * surface that is live *now*", asked whenever either side moves — not "did the
+ * playlist just change". And a refusal is **not** recorded as applied, so it is
+ * retried rather than remembered as done.
+ *
+ * A generation is what makes "the surface that is live now" expressible. Without
+ * it a caller can only know that *a* surface exists, which cannot distinguish
+ * the one it prepared against from the one that replaced it.
+ *
+ * Pure, so the sequence above is a test rather than a device run.
+ *
+ * @experimental
+ */
+
+/** No surface. Distinct from every real generation, which start at 1. */
+export const NO_SURFACE = 0
+
+/** @experimental */
+export interface PlaylistSession {
+  /** The playlist Media3 is holding, or null if it holds none. */
+  appliedSignature: string | null
+  /** The surface generation it was prepared against. */
+  appliedGeneration: number
+  /**
+   * A preparation was attempted and refused.
+   *
+   * Kept so the retry is driven by this rather than by hoping the document
+   * changes again. A user who has added their clips will not add them twice.
+   */
+  pending: boolean
+}
+
+/** @experimental */
+export function idlePlaylistSession(): PlaylistSession {
+  return { appliedSignature: null, appliedGeneration: NO_SURFACE, pending: false }
+}
+
+/** What the document and the device currently want. @experimental */
+export interface PlaylistIntent {
+  /** A stable description of the playlist the document implies. */
+  signature: string
+  /** The live surface generation, or {@link NO_SURFACE}. */
+  generation: number
+}
+
+/**
+ * Whether to call `setPlaylist` now.
+ *
+ * @experimental
+ */
+export function shouldPrepare(session: PlaylistSession, intent: PlaylistIntent): boolean {
+  // Nothing to prepare against. Attempting it is the refusal that started all
+  // of this, and it would poison the session by recording a signature.
+  if (intent.generation === NO_SURFACE) return false
+  if (session.pending) return true
+  if (session.appliedSignature !== intent.signature) return true
+  // The playlist is unchanged but the surface underneath it is not. A player
+  // attached to a retired surface decodes, releases frames and reports no
+  // error while the live texture receives nothing.
+  return session.appliedGeneration !== intent.generation
+}
+
+/**
+ * Record the outcome.
+ *
+ * A refusal leaves the session **unchanged except for `pending`**. Recording the
+ * signature on failure is precisely the mistake that made the editor wait for an
+ * edit that was never going to come.
+ *
+ * @experimental
+ */
+export function afterPrepare(
+  session: PlaylistSession,
+  intent: PlaylistIntent,
+  ok: boolean,
+): PlaylistSession {
+  if (!ok) return { ...session, pending: true }
+  return {
+    appliedSignature: intent.signature,
+    appliedGeneration: intent.generation,
+    pending: false,
+  }
+}
+
+/**
+ * The surface went away — backgrounded, replaced, released.
+ *
+ * The signature is kept: it still describes what Media3 was asked to hold, and
+ * the comparison that matters next is against the *generation*, which is now
+ * guaranteed to differ from any live one.
+ *
+ * @experimental
+ */
+export function retireSurface(session: PlaylistSession): PlaylistSession {
+  return { ...session, appliedGeneration: NO_SURFACE, pending: true }
+}
+
+/**
+ * Whether a callback carrying `generation` is still worth acting on.
+ *
+ * A screen's teardown routinely runs after the next screen's setup, and an
+ * asynchronous surface creation can land after the surface it created has been
+ * replaced. Both arrive looking exactly like a valid result.
+ *
+ * @experimental
+ */
+export function isCurrentGeneration(generation: number, live: number): boolean {
+  return generation !== NO_SURFACE && generation === live
+}

package/src/previewAudioDiagnostics.ts

@@ -0,0 +1,67 @@
+/**
+ * One isolation control for the preview-audio hook, and nothing else.
+ *
+ * ## What it is for
+ *
+ * The picture stops advancing when a mix is loaded, and the minimal failing
+ * configuration loads a *cached* mix — no decode, no encode, no codecs at all.
+ * What is left in common is this hook being active, and being active means two
+ * separable things: the native calls it makes, and the React state it
+ * publishes. Five `useState` values re-render the editor while a surface, a
+ * texture consumer and a composition callback are all being re-established.
+ *
+ * Suppressing the publications while leaving every native call in place
+ * separates those two. If the freeze survives suppression it belongs to the
+ * native side; if it disappears, the trigger is a render-driven ownership
+ * transition and the fix belongs in whatever that render disturbs.
+ *
+ * ## What it is not
+ *
+ * Not a fix, and not reachable from a production build. Suppressed, the hook
+ * cannot report readiness, so audio never plays — the control is deliberately
+ * useless for anything except attribution. Anyone tempted to leave it on to
+ * "reduce renders" is turning a diagnostic into a silent feature regression.
+ *
+ * @experimental
+ */
+
+/** @experimental */
+export interface PreviewAudioDiagnostics {
+  /** Drop the hook's React state publications, keeping every native call. */
+  suppressPublications: boolean
+  /** Log each publication with its value. Off by default; noisy under a mix. */
+  logPublications: boolean
+  /** Publications made, and publications suppressed. */
+  published: number
+  suppressed: number
+}
+
+/** @experimental */
+export const previewAudioDiagnostics: PreviewAudioDiagnostics = {
+  suppressPublications: false,
+  logPublications: false,
+  published: 0,
+  suppressed: 0,
+}
+
+/**
+ * Wrap a setter so it can be observed and, under the control, withheld.
+ *
+ * Takes the name rather than deriving one, because a minified build would
+ * otherwise label every publication `t`.
+ *
+ * @experimental
+ */
+export function publishState<T>(name: string, set: (value: T) => void): (value: T) => void {
+  return (value: T) => {
+    if (previewAudioDiagnostics.suppressPublications) {
+      previewAudioDiagnostics.suppressed += 1
+      return
+    }
+    previewAudioDiagnostics.published += 1
+    if (previewAudioDiagnostics.logPublications) {
+      console.log(`AUDIOPUB ${name} ${JSON.stringify(value ?? null)}`)
+    }
+    set(value)
+  }
+}