@pictor-sdk/video-android 0.1.0

package/src/surfaceOwner.ts

@@ -0,0 +1,264 @@
+import { NO_SURFACE } from './playlistSession'
+
+/**
+ * Who is allowed to create a presentation surface, and when.
+ *
+ * There is one external texture in the process, so there is one of these. Not
+ * one per hook instance: a hook is remounted by a navigation, a fast refresh or
+ * a rotation, and each remount would bring its own lock and its own idea of
+ * what is in flight. Two creates then overlap, each registers a generation, and
+ * the second overwrites the native handles the first is still holding — the GL
+ * texture and `SurfaceTexture` from the first are then unreachable and never
+ * released. The symptom is a hook reporting one generation while the adapter
+ * has moved on to another, with an abandoned surface in between.
+ *
+ * ## The generation is native's, and only native's
+ *
+ * `create()` returns the generation `VideoSurface.register` assigned. This
+ * module publishes that number unchanged and keeps no counter of its own — a
+ * second counter is a second answer to "which surface is live", and the whole
+ * class of bug here is two answers to one question.
+ *
+ * ## A superseded attempt is retired, not ignored
+ *
+ * An attempt whose owner went away while it was on the UI runtime still
+ * *succeeds*: the texture exists, the `Surface` is registered, Media3 could
+ * attach to it. Ignoring its report leaves exactly that — a live, registered
+ * surface nobody will ever draw or release. So it is released, on the runtime
+ * that owns the GL context, before anything else is started.
+ *
+ * ## Serialised, and retirement does not unlock
+ *
+ * A request that arrives while an attempt is unresolved is queued, and
+ * backgrounding or unmounting does **not** release the lock: the earlier
+ * attempt is still out there on the UI runtime and will still come back. The
+ * queue is one deep, because every waiting request wants the same thing.
+ *
+ * Effects are injected so the interleavings are deterministic in a test rather
+ * than a matter of timing on a device.
+ *
+ * @experimental
+ */
+
+/** What native `create()` reports. @experimental */
+export interface SurfaceCreateResult {
+  ok: boolean
+  /** The generation `register` assigned, or 0 on failure. */
+  generation: number
+  textureId: number
+  error: string
+}
+
+/** @experimental */
+export interface SurfaceOwnerEffects {
+  /**
+   * Schedule a create on the UI runtime.
+   *
+   * Must eventually call {@link SurfaceOwner.complete}, whatever happens.
+   */
+  create(): void
+  /**
+   * Release a specific generation, on the runtime that owns the GL context.
+   *
+   * Takes a generation rather than "the current surface" so a late attempt
+   * cannot release the surface that replaced it.
+   */
+  retire(generation: number): void
+}
+
+/** @experimental */
+export interface SurfaceReadiness {
+  /** The live generation, exactly as native reported it. 0 when there is none. */
+  generation: number
+  /** The last failure, or empty. */
+  error: string
+}
+
+/** @experimental */
+export interface SurfaceOwner {
+  readonly readiness: SurfaceReadiness
+  /** Whether an attempt is on the UI runtime right now. */
+  readonly busy: boolean
+  /** Generations created by a superseded attempt and released. Diagnostics. */
+  readonly retired: number
+  /**
+   * Ask for a surface on behalf of `token`.
+   *
+   * Returns false when the request was queued or refused rather than started.
+   */
+  request(token: number): boolean
+  /** That owner is gone. Does not unlock: an unresolved attempt is still out there. */
+  retireOwner(token: number): void
+  /** An attempt finished. */
+  complete(result: SurfaceCreateResult): void
+  /** Adopt a surface that already exists, without creating one. */
+  adopt(generation: number, token: number): void
+  subscribe(listener: () => void): () => void
+  /** Point the owner at the newest caller's effects. */
+  useEffects(next: SurfaceOwnerEffects): void
+}
+
+/** @experimental */
+export function createSurfaceOwner(initial: SurfaceOwnerEffects): SurfaceOwner {
+  // Mutable, because the singleton outlives every hook that uses it and each
+  // one brings its own closure over the published handle and the texture size.
+  // The newest is the right one: they all drive the same native object.
+  let effects = initial
+  let readiness: SurfaceReadiness = { generation: NO_SURFACE, error: '' }
+  let busy = false
+  let retired = 0
+  /** The owner the in-flight attempt was started for. */
+  let activeToken: number | null = null
+  /**
+   * Whether that owner went away while its attempt was on the UI runtime.
+   *
+   * A flag rather than a comparison against whoever is current now: a *second*
+   * hook asking for a surface does not invalidate the one being built, because
+   * they share it. Only the asking owner disappearing does. Comparing against
+   * the current token retired a perfectly good surface every time a remount
+   * overlapped a create, which is a leak dressed as caution.
+   */
+  let activeRetired = false
+  /** The owner currently entitled to a surface. */
+  let currentToken: number | null = null
+  /** One waiting request. Every waiter wants the same thing. */
+  let queuedToken: number | null = null
+  const listeners = new Set<() => void>()
+
+  const emit = () => {
+    for (const listener of listeners) listener()
+  }
+
+  const publish = (next: SurfaceReadiness) => {
+    if (next.generation === readiness.generation && next.error === readiness.error) return
+    readiness = next
+    emit()
+  }
+
+  const start = (token: number) => {
+    busy = true
+    activeToken = token
+    activeRetired = false
+    effects.create()
+  }
+
+  const owner: SurfaceOwner = {
+    get readiness() {
+      return readiness
+    },
+    get busy() {
+      return busy
+    },
+    get retired() {
+      return retired
+    },
+
+    request(token) {
+      currentToken = token
+      if (readiness.generation !== NO_SURFACE) return false
+      if (busy) {
+        queuedToken = token
+        return false
+      }
+      start(token)
+      return true
+    },
+
+    retireOwner(token) {
+      if (activeToken === token) activeRetired = true
+      if (currentToken === token) currentToken = null
+      if (queuedToken === token) queuedToken = null
+      // Deliberately not `busy = false`. The attempt is on the UI runtime and
+      // will come back; unlocking here is what lets a second create overlap it.
+    },
+
+    adopt(generation, token) {
+      currentToken = token
+      if (generation === NO_SURFACE) return
+      publish({ generation, error: '' })
+    },
+
+    complete(result) {
+      const wasRetired = activeRetired
+      const hasOwner = currentToken !== null
+      busy = false
+      activeToken = null
+      activeRetired = false
+
+      if (result.ok && result.generation > NO_SURFACE) {
+        // Superseded means nobody is left who asked for this: the owner that
+        // started it retired, or every owner has gone. Not "somebody else asked
+        // in the meantime" — they want the same surface.
+        const superseded = wasRetired || !hasOwner
+        const stale = result.generation < readiness.generation
+        if (superseded || stale) {
+          // It exists and is registered, so it has to be released rather than
+          // forgotten. Releasing by generation means a late attempt cannot take
+          // down the surface that replaced it.
+          retired += 1
+          effects.retire(result.generation)
+        } else {
+          publish({ generation: result.generation, error: '' })
+        }
+      } else {
+        publish({
+          generation: readiness.generation,
+          error: result.error || 'surface create failed',
+        })
+      }
+
+      const next = queuedToken
+      queuedToken = null
+      // Only if there is still no surface: a queued request that has since been
+      // satisfied by the attempt that just finished wants nothing more.
+      if (next !== null && readiness.generation === NO_SURFACE && next === currentToken) {
+        start(next)
+      }
+    },
+
+    useEffects(next) {
+      effects = next
+    },
+
+    subscribe(listener) {
+      listeners.add(listener)
+      return () => {
+        listeners.delete(listener)
+      }
+    },
+  }
+
+  return owner
+}
+
+/**
+ * The process-wide owner, created on first use.
+ *
+ * A module-level singleton because the thing it guards — the one external
+ * texture — is one per process. `resetSurfaceOwnerForTests` exists so a test can
+ * start from a known state; nothing in production calls it.
+ *
+ * @experimental
+ */
+let singleton: SurfaceOwner | null = null
+
+/** @experimental */
+export function surfaceOwner(effects: SurfaceOwnerEffects): SurfaceOwner {
+  if (!singleton) singleton = createSurfaceOwner(effects)
+  else singleton.useEffects(effects)
+  return singleton
+}
+
+/** @experimental */
+export function resetSurfaceOwnerForTests(): void {
+  singleton = null
+}
+
+/** Fresh token for a hook instance. Only ever increases. @experimental */
+let nextToken = 0
+
+/** @experimental */
+export function nextOwnerToken(): number {
+  nextToken += 1
+  return nextToken
+}

package/src/surfaceRecovery.ts

@@ -0,0 +1,121 @@
+/**
+ * Deciding that a restored surface is not actually showing anything.
+ *
+ * After a background, a picker, or any other trip away from the editor, the
+ * player and the surface have to find each other again. When that handshake
+ * half-succeeds the result is the worst kind of failure: audio plays, the clock
+ * advances, no error is reported, and the picture is black. Nothing in the
+ * player's own state says anything is wrong, because from its point of view
+ * nothing is.
+ *
+ * So it is decided from the counters instead, and the counters are already
+ * separated into the four things that can go wrong — see `Media3State`, where
+ * `framesOffered` is the decoder saying a frame is coming and `framesAvailable`
+ * is the surface saying one arrived.
+ *
+ * The point of naming the branch is that the repairs differ. Re-running the
+ * handshake fixes a stale generation and does nothing at all for a consumer
+ * that has stopped pulling, and "retry everything" would have made both look
+ * like the same intermittent bug.
+ *
+ * @experimental
+ */
+
+/** How long a stall must persist before it counts. @experimental */
+export const RECOVERY_GRACE_MS = 1500
+
+/** What a sample of the pipeline looked like at one instant. @experimental */
+export interface RecoverySample {
+  atMs: number
+  /** The player's own clock. Advancing means audio is being produced. */
+  positionUs: number
+  /** Media3 announcing frames. */
+  framesOffered: number
+  /** The SurfaceTexture receiving them. */
+  framesAvailable: number
+  /** Frames the consumer actually took. */
+  framesLatched: number
+  /** Announcements for a generation already retired. */
+  framesRejected: number
+  /** The generation the producer is filling. */
+  pendingGeneration: number
+  /** The generation the consumer is watching. */
+  consumerGeneration: number
+}
+
+/**
+ * Which half of the pipeline is broken, or `null` when it is healthy.
+ *
+ * @experimental
+ */
+export type StallKind =
+  /** Frames arrive and nobody takes them: the consumer or its frame loop. */
+  | 'consumer'
+  /** Producer and consumer disagree about which surface: a stale handshake. */
+  | 'handshake'
+  /** The player is running but nothing reaches the surface: attachment. */
+  | 'attachment'
+  /** Frames are latched and the picture is still wrong: scene or image. */
+  | 'resource'
+
+/**
+ * Name the branch from two samples.
+ *
+ * `null` means nothing is wrong *that this can see* — including the ordinary
+ * case of a player that is simply not advancing, which is what a paused editor
+ * looks like and is not a fault.
+ *
+ * @experimental
+ */
+export function classifyStall(since: RecoverySample, now: RecoverySample): StallKind | null {
+  // A clock that is not moving is not a stall. Nothing is expected of a player
+  // that is paused, ended or buffering, and calling that a fault would make
+  // every pause look like a broken surface.
+  if (now.positionUs <= since.positionUs) return null
+
+  // Checked before anything else: when the two sides disagree about the
+  // generation, every other counter is describing a surface nobody is watching.
+  if (
+    now.pendingGeneration !== now.consumerGeneration ||
+    now.framesRejected > since.framesRejected
+  ) {
+    return 'handshake'
+  }
+
+  const latched = now.framesLatched > since.framesLatched
+  if (latched) {
+    // The pipeline is delivering. Whatever is wrong is further along — the
+    // scene, the image, the matrix — and re-running the handshake would only
+    // interrupt something that is working.
+    return 'resource'
+  }
+
+  const arrived = now.framesAvailable > since.framesAvailable
+  if (arrived) return 'consumer'
+
+  // The decoder may or may not be announcing; either way nothing reached the
+  // surface, so the player is not attached to the one being watched.
+  return 'attachment'
+}
+
+/**
+ * Whether the stall has lasted long enough to act on.
+ *
+ * A grace period because none of this is instantaneous: a surface is recreated,
+ * a playlist is reconciled and the first frame decoded, and a check that fired
+ * inside that window would recover a pipeline that was about to work.
+ *
+ * @experimental
+ */
+export function shouldRecover(
+  since: RecoverySample,
+  now: RecoverySample,
+  graceMs: number = RECOVERY_GRACE_MS,
+): boolean {
+  if (now.atMs - since.atMs < graceMs) return false
+  const kind = classifyStall(since, now)
+  // `resource` is deliberately excluded: frames are being latched, so the
+  // handshake this recovers is not what is wrong, and retrying it would blank a
+  // working picture on a timer.
+  return kind === 'consumer' || kind === 'handshake' || kind === 'attachment'
+}

package/src/syncTrace.ts

@@ -0,0 +1,189 @@
+/**
+ * Per-tick synchronisation forensics.
+ *
+ * A single offset number cannot say *why* video is late, and optimising against
+ * it invites exactly the wrong fixes — presenting future frames, nudging the
+ * clock, delaying audio. Each of those improves the number and worsens the
+ * playback.
+ *
+ * So every tick is decomposed into causes that can be acted on separately:
+ *
+ * - **Frame quantisation** — video only changes every frame period, so the
+ *   picture is always somewhat behind the audio instant. Bounded by one frame
+ *   by definition, and not a defect.
+ * - **Decoder lateness** — the frame that *should* be showing had not been
+ *   decoded yet. Fix by adjusting the bounded lead or queue depth.
+ * - **Scheduler lateness** — the right frame *was* available and was not
+ *   selected. Fix the missed tick or the contention causing it.
+ * - **Clock correction** — how far each raw `AudioTrack` report moves the
+ *   extrapolated position. Large jumps mean stale timestamps or a bad baseline.
+ *
+ * Pure, so the decomposition is tested rather than trusted.
+ *
+ * @experimental
+ */
+
+export interface TickSample {
+  /** Authoritative extrapolated audio time. */
+  audioUs: number
+  /** Age of the newest raw `getTimestamp()` report at this tick. */
+  rawAgeUs: number
+  /** How far the newest raw report moved the extrapolated position. */
+  correctionUs: number
+  /** Newest decoded frame at or before `audioUs`, or null if none is eligible. */
+  newestEligiblePtsUs: number | null
+  /** What was actually presented, or null. */
+  presentedPtsUs: number | null
+  /** Interval since the previous tick. */
+  tickIntervalUs: number
+  queueDepth: number
+  decoderLeadUs: number
+}
+
+export interface SyncTrace {
+  readonly frameUs: number
+  quantisation: number[]
+  decoderLate: number[]
+  schedulerLate: number[]
+  correction: number[]
+  presentedOffset: number[]
+  tickInterval: number[]
+  ticks: number
+  decoderLateTicks: number
+  schedulerLateTicks: number
+}
+
+const MAX_SAMPLES = 20_000
+
+export function createSyncTrace(fps: number): SyncTrace {
+  const rate = Number.isFinite(fps) && fps > 0 ? fps : 30
+  return {
+    frameUs: Math.round(1_000_000 / rate),
+    quantisation: [],
+    decoderLate: [],
+    schedulerLate: [],
+    correction: [],
+    presentedOffset: [],
+    tickInterval: [],
+    ticks: 0,
+    decoderLateTicks: 0,
+    schedulerLateTicks: 0,
+  }
+}
+
+/**
+ * The frame that ought to be on screen at `audioUs`.
+ *
+ * Assumes a regular frame grid anchored at zero, which the controlled fixture
+ * has. On a variable-rate source this is an approximation and the quantisation
+ * figure should be read as indicative — which is why it is stated here rather
+ * than left for a reader to infer from a clean-looking number.
+ */
+export function expectedPtsAt(audioUs: number, frameUs: number): number {
+  if (audioUs <= 0 || frameUs <= 0) return 0
+  return Math.floor(audioUs / frameUs) * frameUs
+}
+
+export function record(trace: SyncTrace, sample: TickSample): SyncTrace {
+  if (trace.ticks >= MAX_SAMPLES) return { ...trace, ticks: trace.ticks + 1 }
+
+  const expected = expectedPtsAt(sample.audioUs, trace.frameUs)
+  const next: SyncTrace = {
+    ...trace,
+    ticks: trace.ticks + 1,
+    quantisation: [...trace.quantisation, sample.audioUs - expected],
+    correction: [...trace.correction, Math.abs(sample.correctionUs)],
+    tickInterval: [...trace.tickInterval, sample.tickIntervalUs],
+    decoderLate: trace.decoderLate,
+    schedulerLate: trace.schedulerLate,
+    presentedOffset: trace.presentedOffset,
+    decoderLateTicks: trace.decoderLateTicks,
+    schedulerLateTicks: trace.schedulerLateTicks,
+  }
+
+  // Decoder lateness: the frame that should be showing was not available.
+  // Measured as how far short of `expected` the newest eligible frame fell.
+  const eligible = sample.newestEligiblePtsUs
+  if (eligible === null || eligible < expected) {
+    const shortfall = eligible === null ? sample.audioUs : expected - eligible
+    next.decoderLate = [...trace.decoderLate, shortfall]
+    next.decoderLateTicks = trace.decoderLateTicks + 1
+  }
+
+  // Scheduler lateness: an eligible frame existed and something older — or
+  // nothing — was shown instead. This is the category that means a tick was
+  // missed rather than a frame being absent.
+  if (eligible !== null && (sample.presentedPtsUs === null || sample.presentedPtsUs < eligible)) {
+    const behind = eligible - (sample.presentedPtsUs ?? 0)
+    next.schedulerLate = [...trace.schedulerLate, behind]
+    next.schedulerLateTicks = trace.schedulerLateTicks + 1
+  }
+
+  if (sample.presentedPtsUs !== null) {
+    next.presentedOffset = [...trace.presentedOffset, sample.presentedPtsUs - sample.audioUs]
+  }
+  return next
+}
+
+export interface Distribution {
+  n: number
+  p50: number
+  p95: number
+  p99: number
+  max: number
+}
+
+export function distribution(values: readonly number[]): Distribution {
+  if (values.length === 0) return { n: 0, p50: 0, p95: 0, p99: 0, max: 0 }
+  const abs = values.map(Math.abs).sort((a, b) => a - b)
+  const at = (q: number) => abs[Math.min(abs.length - 1, Math.floor(q * abs.length))]!
+  return { n: abs.length, p50: at(0.5), p95: at(0.95), p99: at(0.99), max: abs[abs.length - 1]! }
+}
+
+export interface SyncReport {
+  quantisation: Distribution
+  decoderLate: Distribution
+  schedulerLate: Distribution
+  correction: Distribution
+  presentedOffset: Distribution
+  tickInterval: Distribution
+  ticks: number
+  decoderLateTicks: number
+  schedulerLateTicks: number
+  frameUs: number
+  /** Which cause dominates, so the fix is chosen by evidence. */
+  dominant: 'none' | 'decoder' | 'scheduler' | 'quantisation'
+}
+
+export function syncReport(trace: SyncTrace): SyncReport {
+  const decoderLate = distribution(trace.decoderLate)
+  const schedulerLate = distribution(trace.schedulerLate)
+  const quantisation = distribution(trace.quantisation)
+  const presentedOffset = distribution(trace.presentedOffset)
+
+  // Attribute by how often each fires and how badly, not by which p95 is
+  // numerically larger: a rare huge decoder stall and a constant one-frame
+  // scheduler lag need different fixes.
+  const decoderShare = trace.ticks === 0 ? 0 : trace.decoderLateTicks / trace.ticks
+  const schedulerShare = trace.ticks === 0 ? 0 : trace.schedulerLateTicks / trace.ticks
+  let dominant: SyncReport['dominant'] = 'none'
+  if (presentedOffset.p95 > trace.frameUs) {
+    if (decoderShare >= schedulerShare && decoderShare > 0.01) dominant = 'decoder'
+    else if (schedulerShare > 0.01) dominant = 'scheduler'
+    else dominant = 'quantisation'
+  }
+
+  return {
+    quantisation,
+    decoderLate,
+    schedulerLate,
+    correction: distribution(trace.correction),
+    presentedOffset,
+    tickInterval: distribution(trace.tickInterval),
+    ticks: trace.ticks,
+    decoderLateTicks: trace.decoderLateTicks,
+    schedulerLateTicks: trace.schedulerLateTicks,
+    frameUs: trace.frameUs,
+    dominant,
+  }
+}