@pictor-sdk/video-android 0.1.0

package/src/exporter.ts

@@ -0,0 +1,655 @@
+import { createExportJobs } from './exportJob'
+import { resolveMixSources } from './mixSources'
+import { describeTarget, resolveDestination } from './exportDestination'
+import { planProgressWeights } from './exportProgress'
+import { createProgressPump } from './exportProgressPump'
+import {
+  VideoExportCancelled,
+  VideoExportFailed,
+  VideoExportStalled,
+  compileAudioMixProgram,
+  compileVideoSceneProgram,
+  isPlayableWindow,
+  layout,
+  planClipShards,
+  projectDuration,
+  validateAudioMixProgram,
+  validateProject,
+  VIDEO_SCENE_PROGRAM_VERSION,
+  type ExportSourceContext,
+  type MediaSource,
+  type ResampleRequest,
+  type VideoExportProgress,
+  type VideoExportSpec,
+  type VideoExporter,
+  type VideoProject,
+} from '@pictor-sdk/video-core'
+import NativePictorVideoModule from './NativePictorVideo'
+
+/** The turbo module, or a failure that says so rather than a null dereference. */
+function nativeModule(): NonNullable<typeof NativePictorVideoModule> {
+  if (!NativePictorVideoModule) {
+    throw new VideoExportRejected(['the Pictor video native module is not installed'])
+  }
+  return NativePictorVideoModule
+}
+
+/**
+ * The Media3 exporter.
+ *
+ * Honours `VideoExporter` in full: asynchronous, cancellable while it runs,
+ * monotonic progress, temporary and file destinations, one export at a time,
+ * typed refusal, cancellation, stall and failure, and deterministic cleanup on
+ * every one of those paths.
+ *
+ * Nothing constructs it for you. It is not on `VideoRuntime`, not behind the
+ * editor's Next button, and not reachable from `@pictor-sdk/video-ui` — a host
+ * builds it explicitly, which is what keeps the surface reviewable. Wiring it
+ * to a button is Slice 5's job, not this file's.
+ *
+ * ## The gate boundary
+ *
+ * Nothing here reaches a `exportGate*` method. Provenance hashing, fault
+ * injection, artificial holds and test beats live behind
+ * `exportGateEnableDiagnostics`, which this never calls, and which leaves them
+ * inert even when a payload asks for them.
+ *
+ * ## The whole project is interpreted here, in TypeScript
+ *
+ * `planClipShards` and `compileVideoSceneProgram` run once, sources are resolved
+ * once, overlays are rasterised once, and what crosses to native is a list of
+ * shards, a compiled program and a resource manifest — all JSON and file paths.
+ * Native never sees a `VideoProject`, a `VideoClip`, a trim, a speed or a
+ * transform field. It looks up segments by timestamp and draws what they say.
+ *
+ * ## It refuses rather than approximating
+ *
+ * Every clip kind, spec field and overlay state this slice does not implement is
+ * rejected by name before anything is encoded. A photo clip silently skipped, or
+ * a bitrate silently ignored, produces a file that is plausible and wrong — and
+ * a plausible wrong file is worse than no file, because nothing downstream can
+ * tell it apart from a correct one.
+ *
+ * @experimental
+ */
+
+/**
+ * An overlay raster, described structurally.
+ *
+ * Deliberately not imported from `@pictor-sdk/video-skia`. The adapter has no
+ * opinion about how an overlay is rendered — that is the whole point of the
+ * boundary — and importing the renderer would drag Skia into a package whose
+ * job is to talk to Media3. What it needs is the shape of a descriptor, which
+ * is JSON.
+ *
+ * @experimental
+ */
+export interface OverlayResourceDescriptor {
+  overlayId: string
+  resourceId: string
+  size: { width: number; height: number }
+  rasterBounds: { x: number; y: number; width: number; height: number }
+  rasterScale: number
+  rasterWidth: number
+  rasterHeight: number
+  format: string
+  premultiplied: boolean
+  colorSpace: string
+  file: string
+  byteLength: number
+  checksum: number
+}
+
+/** What preparing a program's overlays produced. @experimental */
+export interface OverlayPreparationResult {
+  resources: readonly OverlayResourceDescriptor[]
+  /** Still resolving. An export must not proceed and omit them. */
+  pending: readonly string[]
+  /** Cannot be exported at all. */
+  invalid: readonly { overlayId: string; kind: string; reason: string }[]
+}
+
+/** Everything the host must supply. @experimental */
+export interface Media3ExporterOptions {
+  /**
+   * Turn an opaque `MediaSource` into something Android can open.
+   *
+   * The exporter never inspects a descriptor: `MediaSource` is `JsonValue` and
+   * only the host knows whether it is an asset id, a content URI or a row in a
+   * database. Returning `null` fails the export naming the clip or the span.
+   *
+   * `context` says which part of the project is being resolved. It used to be
+   * the clip, which left the audio path with nothing honest to pass: every
+   * span, project music included, was resolved with the *first video clip* as
+   * context. A resolver that keys on the clip returned that clip's media for
+   * the whole timeline's audio — a plausible file with the wrong sound in it.
+   */
+  resolveSource(source: MediaSource, context: ExportSourceContext): string | null
+  /**
+   * Rasterise every overlay the compiled program names.
+   *
+   * The host wires its registry, its prepared resources and its raster sink
+   * behind this — typically `prepareProgramOverlayExports` from
+   * `@pictor-sdk/video-skia`. Injecting the whole step rather than its three inputs
+   * is what keeps this package free of a renderer dependency, and it is the same
+   * reason `resolveSource` is injected rather than a media library being.
+   *
+   * Omitted means "no overlays are exportable": a project that has any then
+   * fails naming each one, rather than exporting without them.
+   */
+  prepareOverlays?(
+    program: { segments: readonly { commands: readonly unknown[] }[] },
+    projectSize: { width: number; height: number },
+    rasterScale: number,
+  ): OverlayPreparationResult
+  /**
+   * Called after an export finishes, successfully or not.
+   *
+   * The sidecars belong to the host's raster sink, so the host removes them.
+   * Passing the list rather than the sink keeps this from having to know how
+   * they were stored.
+   */
+  releaseResources?(files: readonly string[]): void
+  /**
+   * How long without any heartbeat counts as wedged.
+   *
+   * Defaults to {@link STALL_GRACE_MS}, which is **Pixel 7 evidence**: the worst
+   * legitimate silence measured across seven production-path cases on that
+   * device, with margin. It is not a claim about Android. A slower encoder, a
+   * thermally throttled phone or a codec that batches differently can produce a
+   * longer honest gap, and the number to change is this one — raising it costs
+   * only how late a genuine wedge is noticed, while leaving it too low kills
+   * healthy exports.
+   */
+  stallGraceMs?: number
+  /**
+   * How often native counters are read, and progress delivered.
+   *
+   * Defaults to 125 ms — 8 Hz, inside the 5–10 Hz a person can actually read.
+   * Native counting is unthrottled regardless; this is only the bridge rate.
+   */
+  progressPollMs?: number
+}
+
+/** Why an export will not start. @experimental */
+export class VideoExportRejected extends Error {
+  readonly reasons: readonly string[]
+  constructor(reasons: readonly string[]) {
+    super(`export rejected: ${reasons.join('; ')}`)
+    this.name = 'VideoExportRejected'
+    this.reasons = reasons
+  }
+}
+
+/** The frame rate an export runs at when the spec does not say. */
+const DEFAULT_FPS = 30
+
+
+/**
+ * What this slice cannot encode, stated rather than skipped.
+ *
+ * Each of these has a milestone. Until then the export fails naming the clip or
+ * the field, because the alternative is a file that looks finished.
+ */
+function describeUnsupported(project: VideoProject, spec: VideoExportSpec): string[] {
+  const reasons: string[] = []
+
+  for (const clip of project.clips) {
+    if (clip.kind === 'photo') {
+      reasons.push(`clip '${clip.id}' is a photo, which this exporter does not encode yet`)
+      continue
+    }
+  }
+
+  if (spec.format !== undefined && spec.format !== 'mp4') {
+    reasons.push(`format '${spec.format}' is not supported; this exporter writes mp4`)
+  }
+  if (spec.scale !== undefined && spec.scale !== 1) {
+    // Honoured or rejected, never quietly ignored.
+    reasons.push(`scale ${spec.scale} is not supported yet; only 1 is`)
+  }
+  if (spec.fps !== undefined && spec.fps !== DEFAULT_FPS) {
+    reasons.push(`fps ${spec.fps} is not supported yet; only ${DEFAULT_FPS} is`)
+  }
+  if (spec.bitrate !== undefined) {
+    reasons.push('bitrate is not supported yet; the encoder default is used')
+  }
+  return reasons
+}
+
+/**
+ * Build an exporter.
+ *
+ * @experimental
+ */
+export function createMedia3VideoExporter(options: Media3ExporterOptions): VideoExporter {
+  /*
+    One job at a time, cancellable at every stage.
+
+    The registry owns identity, exclusivity and settling exactly once; the body
+    below only has to call `job.checkpoint` where stopping is safe. See
+    `exportJob.ts` for why cancellation cannot simply reject the promise.
+  */
+  const jobs = createExportJobs({
+    cancelNative: (jobId) => {
+      nativeModule().exportCancelJob(jobId)
+    },
+  })
+
+  return {
+    async export(
+      project: VideoProject,
+      spec: VideoExportSpec,
+      onProgress?: (progress: VideoExportProgress) => void,
+    ) {
+      return jobs.run(async (job) => {
+      // First thing, so a cancel pressed before any work is observed before
+      // the compile, the overlay pass and the source resolution rather than
+      // after them. Measured: without it a pre-native cancel took 4.7 s to
+      // settle, because the first checkpoint was past all of that.
+      job.checkpoint('resolving')
+      const issues = validateProject(project)
+      if (issues.length > 0) {
+        throw new VideoExportRejected(issues.map((issue) => issue.message))
+      }
+      const unsupported = describeUnsupported(project, spec)
+      if (unsupported.length > 0) throw new VideoExportRejected(unsupported)
+
+      /*
+        Refused, never redirected.
+
+        Falling back to a temporary would produce an export that reports success
+        while the file is nowhere the caller asked for — and nothing downstream
+        can tell that apart from having worked.
+      */
+      const decision = resolveDestination(spec.destination)
+      if (decision.destination === null) throw new VideoExportRejected(decision.reasons)
+      const destination = decision.destination
+
+      const playable = project.clips.filter((clip) => isPlayableWindow(clip))
+      if (playable.length === 0) throw new VideoExportRejected(['the project has no playable clips'])
+      if (!(projectDuration(project) > 0)) {
+        throw new VideoExportRejected(['the project has no duration'])
+      }
+
+      // Resolved without inspecting the descriptor: only the host knows what a
+      // `MediaSource` is, and this must not learn.
+      const uris: string[] = []
+      const unresolved: string[] = []
+      for (const clip of playable) {
+        const uri = options.resolveSource(clip.source, { kind: 'clip', clipId: clip.id })
+        if (typeof uri !== 'string' || uri.length === 0) {
+          unresolved.push(`clip '${clip.id}' could not be resolved to a readable source`)
+          continue
+        }
+        uris.push(uri)
+      }
+      if (unresolved.length > 0) throw new VideoExportRejected(unresolved)
+
+      const fps = DEFAULT_FPS
+      const placements = layout(project)
+      const requests: ResampleRequest[] = playable.map((clip) => ({
+        sourceStartUs: clip.sourceIn,
+        sourceEndUs: clip.sourceOut,
+        speed: clip.speed,
+        sourceFps: clip.metadata.fps ?? fps,
+        targetFps: fps,
+      }))
+      const plan = planClipShards(requests, fps)
+      const program = compileVideoSceneProgram(project)
+      if (program.segments.length === 0) {
+        throw new VideoExportRejected(['the compiled program has no segments'])
+      }
+      void placements
+
+      /*
+        Overlays are rasterised and checked **before** any native work.
+
+        A resource discovered missing once encoding has started leaves a partial
+        file, and the frames already written are silently missing something the
+        timeline said was on screen.
+      */
+      const named = new Set<string>()
+      for (const segment of program.segments) {
+        for (const command of segment.commands) {
+          if (command.op === 'overlay') named.add(command.overlayId)
+        }
+      }
+      const overlayPlan: OverlayPreparationResult = options.prepareOverlays
+        ? options.prepareOverlays(program, project.size, 1)
+        : {
+            resources: [],
+            pending: [],
+            invalid: [...named].map((overlayId) => ({
+              overlayId,
+              kind: 'unknown',
+              reason: 'no prepareOverlays was supplied, so this overlay has no portable form',
+            })),
+          }
+      // Before the plan is judged: a cancel raised from inside the host's
+      // preparation must be reported as `overlays`, not as whatever the
+      // validation below happens to say first.
+      job.checkpoint('overlays')
+
+      /*
+        Anything already staged is released on every failure path below.
+
+        A rejection after rasterising leaves sidecars nothing else will come back
+        for, and the run that failed is exactly the one nobody cleans up after.
+      */
+      const stagedNow = overlayPlan.resources.map((resource) => resource.file)
+      if (overlayPlan.pending.length > 0) {
+        options.releaseResources?.(stagedNow)
+        throw new VideoExportRejected(
+          overlayPlan.pending.map(
+            (id: string) => `overlay '${id}' is still preparing; export would omit it`,
+          ),
+        )
+      }
+      if (overlayPlan.invalid.length > 0) {
+        options.releaseResources?.(stagedNow)
+        throw new VideoExportRejected(
+          overlayPlan.invalid.map(
+            (bad: { overlayId: string; kind: string; reason: string }) =>
+              `overlay '${bad.overlayId}' (${bad.kind}): ${bad.reason}`,
+          ),
+        )
+      }
+      const resources = overlayPlan.resources
+
+      const shards = plan.map((shard, index) => {
+        const clip = playable[index]!
+        const cadence = cadenceFor(shard.plan.frames)
+        return {
+          uri: uris[index]!,
+          startMs: Math.round(clip.sourceIn / 1000),
+          endMs: Math.round(clip.sourceOut / 1000),
+          speed: clip.speed,
+          timelineOffsetUs: shard.timelineOffsetUs,
+          /*
+            The shard's own audio is dropped and replaced with generated silence.
+
+            Not because anything wants that silence — the final mux takes only
+            the video track from this pipeline — but because Media3 drives its
+            speed path through audio, and a video-only item at any rate other
+            than 1 stalls with "no output sample written". So the track exists to
+            satisfy the re-timing and is discarded.
+
+            What matters is that the *source's* audio never enters this pipeline:
+            a per-shard audio track has to be re-encoded to concatenate, which is
+            what put a second generation on everything the user hears.
+          */
+          forceAudioTrack: true,
+          removeAudio: true,
+          emitsPerInput: cadence.emitsPerInput,
+          outputPtsUs: cadence.outputPtsUs,
+          spanUs: Math.round((shard.plan.frameCount * 1_000_000) / fps),
+        }
+      })
+
+      /*
+        The whole timeline's audio, compiled once and resolved once.
+
+        `compileAudioMixProgram` settles placements, trims, gains and fades; the
+        backend mixes and encodes exactly what it says. Sources are resolved by
+        span id through the same host resolver the video pipeline uses, and a
+        silent span needs none.
+      */
+      const mixProgram = compileAudioMixProgram(project)
+      const mixProblems = validateAudioMixProgram(mixProgram)
+      if (mixProblems.length > 0) throw new VideoExportRejected(mixProblems)
+      /*
+        Resolved as what each span actually is, not as whichever clip came
+        first — and by the same function preview preparation calls, so the two
+        cannot disagree about which file a span means. A disagreement there
+        would be audible only in the finished video.
+      */
+      const resolved = resolveMixSources(project, mixProgram, options.resolveSource)
+      const mixUris = resolved.uris
+      if (resolved.unresolved.length > 0) throw new VideoExportRejected(resolved.unresolved)
+      job.checkpoint('resolving')
+
+      job.checkpoint('overlays')
+      const staged = resources.map((resource) => resource.file)
+
+      /*
+        Progress starts here, after everything that can refuse.
+
+        A bar that appears and then vanishes because a photo clip was rejected
+        is worse than one that never appeared: the export never began, and
+        showing 0% claims otherwise. Everything above this line either throws or
+        establishes the plan the weights are computed from.
+      */
+      const durationUs = projectDuration(project)
+      const progressPlan = planProgressWeights({
+        shards: plan.map((shard) => ({ frames: shard.plan.frameCount })),
+        audioSamples: mixProgram.spans.some((span) => !span.silent)
+          ? Math.max(0, mixProgram.durationUs)
+          : 0,
+        durationUs,
+      })
+      // A box rather than a `let`: the assignment happens in a callback, and
+      // TypeScript narrows a closed-over `let` to `never` at the read.
+      const stalled: { at: { stage: string; idleMs: number } | null } = { at: null }
+      const pump = createProgressPump({
+        plan: progressPlan,
+        durationUs,
+        read: () => JSON.parse(nativeModule().exportProgressSnapshot()) as never,
+        // Delivered through the job, so nothing arrives after settlement: a
+        // host that has already shown a result must not be told 90%.
+        onProgress: onProgress
+          ? (value) => { job.progress(value.progress, () => onProgress(value)) }
+          : undefined,
+        onStall: (verdict) => {
+          stalled.at = { stage: String(verdict.stage), idleMs: verdict.idleMs }
+          /*
+            Torn down through the path cancellation already proved.
+
+            A wedged export holds a Transformer, a muxer and its codecs, and the
+            only mechanism in this process that reliably lets go of all three is
+            the native cancel. What differs is what the caller is told: nobody
+            asked to stop, so this settles as `VideoExportStalled`.
+          */
+          nativeModule().exportCancelJob(job.id)
+        },
+        thresholdMs: options.stallGraceMs,
+        pollMs: options.progressPollMs,
+      })
+      pump.begin()
+      const ticker = setInterval(() => { pump.tick() }, pump.pollMs)
+
+      let outcome: { ok: boolean; uri: string; error: string }
+      try {
+        outcome = JSON.parse(
+          await nativeModule().exportRunProjectAsync(
+            job.id,
+            JSON.stringify(shards),
+            JSON.stringify({
+              version: VIDEO_SCENE_PROGRAM_VERSION,
+              projectSize: { width: project.size.width, height: project.size.height },
+              durationUs: program.durationUs,
+              segments: program.segments,
+            }),
+            JSON.stringify(resources),
+            JSON.stringify(mixProgram),
+            JSON.stringify(mixUris),
+            /* timeoutSeconds= */ 600,
+            destination.kind === 'file'
+              ? JSON.stringify({
+                  kind: 'file',
+                  path: destination.path,
+                  overwrite: destination.overwrite,
+                })
+              : '',
+          ),
+        ) as { ok: boolean; uri: string; error: string }
+      } finally {
+        clearInterval(ticker)
+        // Every path, including the one that threw: the sidecars are this
+        // export's and nothing else will come back for them.
+        options.releaseResources?.(staged)
+      }
+
+      /*
+        A stall outranks whatever native said on the way out.
+
+        Tearing down a wedged Transformer reports itself as a cancellation,
+        because a cancellation is how it was torn down. Reporting that would
+        tell a host the user pressed stop when nothing of the sort happened.
+      */
+      const wedged = stalled.at
+      if (wedged) {
+        pump.stop()
+        throw new VideoExportStalled(job.id, wedged.stage, wedged.idleMs)
+      }
+
+      /*
+        A native cancellation is a cancellation, with the stage native recorded.
+
+        Without this the job registry falls back to its own default and every
+        cancellation reports the same stage, which is worse than no stage at
+        all: it looks like information.
+      */
+      const cancelledDuring = /^cancelled during (\w+)/.exec(outcome.error ?? '')
+      if (!outcome.ok && cancelledDuring) {
+        pump.stop()
+        throw new VideoExportCancelled(job.id, cancelledDuring[1]!)
+      }
+
+      if (!outcome.ok) {
+        /*
+          No uri either way.
+
+          A partial file is worse than none — it plays, it is the wrong length,
+          and a caller that took the uri would have no way to know. Native
+          deletes what it created on every one of its own failure paths; nothing
+          is swept here, because a sweep would also take the temporary a
+          *previous* export handed the caller.
+        */
+        pump.stop()
+        const error = outcome.error || 'the export failed'
+        if (isRefusal(error)) throw new VideoExportRejected([error])
+        throw new VideoExportFailed(job.id, 'encoding', error)
+      }
+      job.checkpoint('publishing')
+
+      /*
+        One is the destination being readable, not the encoder stopping.
+
+        The container is closed after the last sample is written, and a file
+        that is closed is not necessarily one that parses. Probing it costs a
+        `MediaExtractor` open and turns "the encode finished" into "there is a
+        file you can play", which is the only claim a bar reaching 1 should be
+        making.
+      */
+      const probe = JSON.parse(nativeModule().exportProbeOutput(outcome.uri)) as {
+        readable: boolean
+        error?: string
+        tracks?: number
+      }
+      if (!probe.readable) {
+        pump.stop()
+        // Written and unopenable: the work happened and the platform produced
+        // something unusable, which is a failure rather than a refusal.
+        throw new VideoExportFailed(
+          job.id,
+          'publishing',
+          `${describeTarget(outcome.uri)} could not be read back: ${probe.error ?? 'no video track'}`,
+        )
+      }
+      pump.complete()
+      pump.stop()
+
+      /*
+        Ownership, stated in the type.
+
+        A temporary is the adapter's until the caller says otherwise, so it
+        carries the one call that disposes of it. A file destination is the
+        caller's — they chose the path — so that branch has no `release` at all,
+        rather than one that quietly does nothing. An optional cleanup method on
+        a single shape would put "delete the user's video" one typo away from
+        every success path.
+      */
+      if (destination.kind === 'file') return { kind: 'file', uri: outcome.uri }
+      const temporaryUri = outcome.uri
+      return {
+        kind: 'temporary',
+        uri: temporaryUri,
+        release: async () => {
+          // Idempotent, and scoped to this file: native refuses any path it did
+          // not create, so a second call — or one after the cache was cleared —
+          // is not an error.
+          await Promise.resolve()
+          nativeModule().exportReleaseTemporaryOutput(temporaryUri)
+        },
+      }
+      })
+    },
+
+    /**
+     * Ask the running export to stop.
+     *
+     * Idempotent, and a no-op when nothing is running. Reaches the native job
+     * before the promise settles: a Transformer nobody is listening to still
+     * encodes, still writes and still holds its codecs, so rejecting in
+     * JavaScript alone would show a cancelled state over a running export.
+     */
+    cancel(): void {
+      jobs.cancel()
+    },
+  }
+}
+
+/**
+ * Refusal or failure — the distinction a caller acts on.
+ *
+ * A refusal means the project, the destination or a resource was not something
+ * this adapter could encode, decided before or instead of doing work: change
+ * the input. A failure means the work started and the platform gave up: retry,
+ * check storage, or report a bug. Reporting one as the other sends a caller to
+ * the wrong place, which is worse than an unhelpful message.
+ *
+ * Matched on the prefixes native produces rather than on a code, because the
+ * native side reports strings — and the list is exhaustive over the reasons
+ * `runProject` can refuse, so anything unrecognised is a failure. That default
+ * is the safe one: a new encoder error misreported as "unsupported" tells a user
+ * to edit a project that was fine.
+ */
+const REFUSAL_PATTERNS = [
+  /^program rejected/,
+  /^overlay resources? /,
+  /^overlay resource /,
+  /^no shards/,
+  /already exists; pass overwrite/,
+  /^the directory for /,
+  /^the destination /,
+  /cannot be written without overwrite/,
+  /^the staged file could not be read back/,
+]
+
+function isRefusal(error: string): boolean {
+  return REFUSAL_PATTERNS.some((pattern) => pattern.test(error))
+}
+
+
+/**
+ * The cadence transport: copies owed per arriving input, and their timestamps.
+ *
+ * Straight from `resampleClip`'s plan. Native walks these arrays and computes no
+ * grid arithmetic of its own.
+ */
+function cadenceFor(
+  frames: readonly { outputPtsUs: number; sourceFrameIndex: number }[],
+): { emitsPerInput: number[]; outputPtsUs: number[] } {
+  const first = frames[0]!.sourceFrameIndex
+  const last = frames[frames.length - 1]!.sourceFrameIndex
+  const emitsPerInput: number[] = []
+  for (let i = 0; i <= last - first; i += 1) emitsPerInput.push(0)
+  const outputPtsUs: number[] = []
+  for (const frame of frames) {
+    const slot = frame.sourceFrameIndex - first
+    emitsPerInput[slot] = (emitsPerInput[slot] ?? 0) + 1
+    outputPtsUs.push(frame.outputPtsUs)
+  }
+  return { emitsPerInput, outputPtsUs }
+}