@pictor-sdk/video-android 0.1.0

package/src/spikeExcursions.ts

@@ -0,0 +1,405 @@
+import type { SpikeSample } from './compositionSpike'
+
+/**
+ * Where a shared-clock run left the tolerance, and why.
+ *
+ * ## What is being compared
+ *
+ * The audio clock's position and the presented frame's timestamp, brought to
+ * one instant. The frame was latched at `frameConsumedNs`; the position was
+ * read at `atNs`. Rolling the position back to the latch is what puts them on
+ * the same monotonic instant — the composition runs at project rate 1x, so the
+ * roll-back is the elapsed time itself.
+ *
+ * ## Which boundary this observes
+ *
+ * The **latch**, not the display. `frameConsumedNs` is when the consumer called
+ * `updateTexImage`; `frameReleaseNs` is the time Media3 *asked* the surface to
+ * present at, which is an intention and not an acknowledgement. Actual scanout
+ * happens one compositor pass later and is not observed anywhere in this path.
+ * So an offset here is "audio against latched picture", and the display adds a
+ * further unmeasured delay common to every sample.
+ *
+ * ## Uncertainty is stated, never subtracted
+ *
+ * Comparing a 30 fps frame's timestamp against a continuous clock carries an
+ * irreducible ±16.7 ms, and the paired read adds a few more. That is reported
+ * beside the verdict. It is not deducted from an excursion to bring it under
+ * the line: a 95 ms excursion is not a 78 ms excursion because the instrument
+ * is imperfect.
+ *
+ * @experimental
+ */
+
+/** @experimental */
+export const SPIKE_TOLERANCE_US = 60_000
+
+/**
+ * Why one sample left the tolerance.
+ *
+ * Ordered by precedence, because more than one can be true and the first is the
+ * one that explains the sample: a frame from before a seek is cross-epoch
+ * whatever else is also true of it.
+ *
+ * @experimental
+ */
+export type ExcursionCause =
+  /**
+   * The latched frame predates the moment playback was released.
+   *
+   * Not a filter and not a drift. The readiness gate releases on a frame whose
+   * *position* matches the epoch, and that frame may have been latched long
+   * before — measured at 13.9 s, the whole interval between attaching the
+   * composition and the person pressing play. The picture was right; the
+   * frame's wall-clock age simply includes a pause, and charging that to
+   * synchronisation measures the pause.
+   *
+   * Reported rather than excluded, because the size of it is the size of the
+   * interval in which the first sample after a release is not yet a
+   * synchronisation reading.
+   */
+  | 'pre-release-frame'
+  /** The frame belongs to a different transition epoch or frame generation. */
+  | 'cross-epoch'
+  /** The frame was latched long enough ago that it is not a current reading. */
+  | 'stale-pairing'
+  /** The first frames after a seek or after readiness, before the picture caught up. */
+  | 'first-frame-after-seek'
+  /** The frame was latched well after the time it was asked to be presented at. */
+  | 'delayed-presentation'
+  /** The audio clock moved discontinuously between adjacent samples. */
+  | 'audio-discontinuity'
+  /** The mapped project time disagrees with the frame's own window and PTS. */
+  | 'mapping'
+  /** None of the above fitted. Deliberately not folded into a neighbour. */
+  | 'unexplained'
+
+/** @experimental */
+export interface Excursion {
+  seq: number
+  /** Against the latch: observed, and earlier than display. */
+  offsetUs: number
+  /** Against the time the frame was asked to appear. An intention. */
+  offsetAtIntendedUs: number
+  cause: ExcursionCause
+  epoch: number
+  sinceEpochMs: number
+  frameWindow: number
+  framePtsUs: number
+  /** Latched minus asked-for. Large means the picture arrived late. */
+  latchLagUs: number
+  /** How old the latched frame was when the position was read. */
+  frameAgeUs: number
+  /** The audio clock's step from the previous sample, in microseconds. */
+  positionStepUs: number
+  /** The wall-clock step from the previous sample. */
+  wallStepUs: number
+}
+
+/** @experimental */
+export interface ExcursionReport {
+  toleranceUs: number
+  /** Samples where playback was declared active. Only these are judged. */
+  activeSamples: number
+  /** Samples before readiness, exposed rather than dropped. */
+  preparingSamples: number
+  excursions: Excursion[]
+  byCause: Record<string, number>
+  /**
+   * Samples still showing a frame latched before the release.
+   *
+   * Reported apart, and *retained* in the acceptance set: they are real
+   * playback, and if the picture never catches up they are the only thing that
+   * says so.
+   */
+  preReleaseSamples: number
+  preReleaseWorstUs: number
+  worstAbsUs: number
+  /** The same run judged at the intended-presentation boundary. */
+  worstAtIntendedUs: number
+  overAtIntended: number
+  passAtIntended: boolean
+  /** Mean latch-minus-intended over the excursions. Negative means early. */
+  meanLatchLagUs: number
+  pass: boolean
+  uncertainty: {
+    /** Half a frame interval at the measured rate. Irreducible. */
+    frameQuantumUs: number
+    /** The widest paired read in the run. */
+    readSpanUs: number
+    positionQuantumUs: number
+  }
+}
+
+/**
+ * The offset at each observable boundary.
+ *
+ * They are not the same instant and the difference is not noise. Measured on a
+ * Pixel 7, the consumer latches a buffer about 34 ms *before* the time Media3
+ * asked the surface to present it — `releaseOutputBuffer(index, releaseTimeNs)`
+ * queues immediately with a desired present time in the future, and the
+ * SurfaceTexture consumer takes it as soon as it is queued.
+ *
+ * So `atLatch` compares the audio clock against a frame that is in the texture
+ * but has not been displayed, and is systematically early by that gap;
+ * `atIntended` compares it against when that frame was *meant* to appear, which
+ * is the best available estimate of what a viewer experiences and is still an
+ * intention rather than an acknowledgement. The true display instant lies at or
+ * after `atIntended`, and nothing in this process observes it.
+ *
+ * Both are returned because choosing one silently is how a boundary decision
+ * becomes a verdict.
+ *
+ * @experimental
+ */
+export function offsetAt(
+  sample: SpikeSample,
+  /**
+   * When the current playing interval was released, if known.
+   *
+   * A frame latched while paused carries the pause in its age. Charging that to
+   * synchronisation measured a 13.9 second "error" for a picture that was
+   * showing exactly the right position. Clamping the reference to the release
+   * instant charges only the time playback has actually been running — so at
+   * release the offset is (audio position - displayed PTS), and it grows from
+   * there if the audio advances while the picture stays put.
+   *
+   * That growth is the point: this corrects the baseline without removing
+   * coverage. A frame that is never replaced still accrues the offset it
+   * deserves.
+   */
+  releaseAtNs = 0,
+): { atLatch: number; atIntended: number } {
+  /*
+    A frame already on screen when playback began has no age to subtract.
+
+    For a frame presented during playback, the sound being ahead of it by the
+    time since it appeared is not error — the frame is a sample of the recent
+    past. Subtracting that age is what makes the measure a synchronisation
+    error rather than a restatement of the frame rate.
+
+    A pre-release frame is different in kind. It was displayed before the clock
+    started, so there is no elapsed-since-display to discount: at the moment of
+    release the comparison is simply "what is on screen" against "where the
+    sound is", which is near zero when the readiness gate did its job. And if
+    the picture then stays put while the sound runs on, the gap is exactly the
+    staleness a viewer sees — so it is counted, not cancelled. Subtracting the
+    age here would have the two grow together and report zero forever.
+  */
+  if (releaseAtNs > 0 && sample.frameConsumedNs < releaseAtNs) {
+    const raw = sample.positionUs - sample.framePtsUs
+    return { atLatch: raw, atIntended: raw }
+  }
+  return {
+    atLatch: sample.positionUs - (sample.atNs - sample.frameConsumedNs) / 1000 - sample.framePtsUs,
+    atIntended:
+      sample.positionUs - (sample.atNs - sample.frameReleaseNs) / 1000 - sample.framePtsUs,
+  }
+}
+
+/**
+ * Classify a run's over-tolerance samples.
+ *
+ * `frameIntervalUs` is the source's nominal frame interval; half of it is the
+ * irreducible quantisation of comparing a discrete frame against a continuous
+ * clock.
+ *
+ * @experimental
+ */
+export function classifyExcursions(
+  samples: readonly SpikeSample[],
+  options: { toleranceUs?: number; frameIntervalUs?: number } = {},
+): ExcursionReport {
+  const toleranceUs = options.toleranceUs ?? SPIKE_TOLERANCE_US
+  const frameIntervalUs = options.frameIntervalUs ?? 33_333
+
+  /*
+    Judged only once playback is active, and the rest is exposed rather than
+    dropped. A position that has not started moving is not synchronised or
+    unsynchronised; counting it either way is a statement about nothing.
+  */
+  /*
+    A frame from before this epoch is not a reading about this epoch.
+
+    After a start or a seek the clock moves immediately while the first frame
+    for the new position is still being decoded, so the newest latched frame is
+    the one from *before* the transition. Pairing it against the moving clock
+    produces an excursion the size of the seek — measured here at up to 13.8 s
+    — and no classification of that number is about synchronisation.
+
+    So the interval is exposed as preparing rather than judged, which is also
+    what makes "include every sample once playback is active" mean something:
+    active begins when the picture for this epoch has actually arrived.
+  */
+  /*
+    And the clock has to be moving.
+
+    `isPlaying` and `STATE_READY` both go true before the composition's clock
+    starts: measured, the player reported both for fourteen seconds with its
+    position at zero and exactly one frame ever latched. Judging that interval
+    produced a 13.9 second "excursion" that is not a synchronisation reading at
+    all — it is the preparation the brief asks to be exposed rather than scored.
+
+    Observed rather than assumed: the position must have changed somewhere in
+    the preceding window, so the test is "this clock is running" and not "this
+    player says it is".
+  */
+  const ADVANCE_WINDOW_NS = 200_000_000
+  const advancing = samples.map((s, i) => {
+    for (let j = i - 1; j >= 0; j -= 1) {
+      if (s.atNs - samples[j]!.atNs > ADVANCE_WINDOW_NS) break
+      if (samples[j]!.positionUs !== s.positionUs) return true
+    }
+    return false
+  })
+
+  /*
+    When the current playing interval began.
+
+    A frame latched before it carries the pause in its age, so the first
+    samples after a release are not yet synchronisation readings. Tracked so
+    they can be *named* rather than dropped.
+  */
+  const releaseAtNs: number[] = []
+  let intervalStart = 0
+  for (let i = 0; i < samples.length; i += 1) {
+    if (!advancing[i]) intervalStart = 0
+    else if (intervalStart === 0) intervalStart = samples[i]!.atNs
+    releaseAtNs.push(intervalStart)
+  }
+
+  const isActive = (s: SpikeSample, i: number) =>
+    s.playing &&
+    s.ready &&
+    s.framePtsUs >= 0 &&
+    s.frameConsumedNs > 0 &&
+    // The picture for this epoch has arrived.
+    s.frameConsumedNs >= s.atNs - s.sinceEpochNs &&
+    advancing[i] === true
+  const active = samples.filter(isActive)
+  const preparing = samples.length - active.length
+
+  const excursions: Excursion[] = []
+  const byCause: Record<string, number> = {}
+  let worstAbsUs = 0
+  let worstAtIntendedUs = 0
+  let overAtIntended = 0
+  let readSpanUs = 0
+  let latchLagSum = 0
+  let preReleaseSamples = 0
+  let preReleaseWorstUs = 0
+
+  // Indexed by the sample's own position in the full run, because the release
+  // boundary was computed there and `active` has holes.
+  const releaseFor = new Map<number, number>()
+  samples.forEach((s, i) => releaseFor.set(s.seq, releaseAtNs[i]!))
+
+  for (let i = 0; i < active.length; i += 1) {
+    const s = active[i]!
+    const previous = i > 0 ? active[i - 1]! : null
+    readSpanUs = Math.max(readSpanUs, s.readSpanNs / 1000)
+    const releasedAt = releaseFor.get(s.seq) ?? 0
+    const both = offsetAt(s, releasedAt)
+    const offsetUs = both.atLatch
+    const abs = Math.abs(offsetUs)
+    const absIntended = Math.abs(both.atIntended)
+    if (releasedAt > 0 && s.frameConsumedNs < releasedAt) {
+      preReleaseSamples += 1
+      if (abs > preReleaseWorstUs) preReleaseWorstUs = abs
+    }
+    if (abs > worstAbsUs) worstAbsUs = abs
+    if (absIntended > worstAtIntendedUs) worstAtIntendedUs = absIntended
+    if (absIntended > toleranceUs) overAtIntended += 1
+    if (abs <= toleranceUs) continue
+
+    const frameAgeUs = (s.atNs - s.frameConsumedNs) / 1000
+    const latchLagUs = (s.frameConsumedNs - s.frameReleaseNs) / 1000
+    latchLagSum += latchLagUs
+    const positionStepUs = previous ? s.positionUs - previous.positionUs : 0
+    const wallStepUs = previous ? (s.atNs - previous.atNs) / 1000 : 0
+
+    /*
+      First match wins, and the order is the point.
+
+      A frame carried across a seek is cross-epoch whether or not it is also
+      old and also late; calling it "delayed presentation" would send the fix
+      to the renderer instead of to the epoch check.
+    */
+    let cause: ExcursionCause
+    if (releasedAt > 0 && s.frameConsumedNs < releasedAt) {
+      cause = 'pre-release-frame'
+    } else if (s.frameConsumedNs > 0 && s.frameConsumedNs < s.atNs - s.sinceEpochNs) {
+      cause = 'cross-epoch'
+    } else if (previous && s.frameGeneration !== previous.frameGeneration) {
+      cause = 'cross-epoch'
+    } else if (frameAgeUs > 2 * frameIntervalUs) {
+      cause = 'stale-pairing'
+    } else if (s.sinceEpochNs / 1000 < 500_000) {
+      cause = 'first-frame-after-seek'
+    } else if (latchLagUs > frameIntervalUs) {
+      cause = 'delayed-presentation'
+    } else if (previous && Math.abs(positionStepUs - wallStepUs) > toleranceUs) {
+      cause = 'audio-discontinuity'
+    } else if (previous && s.frameWindow !== previous.frameWindow) {
+      cause = 'mapping'
+    } else {
+      cause = 'unexplained'
+    }
+
+    byCause[cause] = (byCause[cause] ?? 0) + 1
+    excursions.push({
+      seq: s.seq,
+      offsetUs: Math.round(offsetUs),
+      offsetAtIntendedUs: Math.round(both.atIntended),
+      cause,
+      epoch: s.epoch,
+      sinceEpochMs: Math.round(s.sinceEpochNs / 1_000_000),
+      frameWindow: s.frameWindow,
+      framePtsUs: s.framePtsUs,
+      latchLagUs: Math.round(latchLagUs),
+      frameAgeUs: Math.round(frameAgeUs),
+      positionStepUs: Math.round(positionStepUs),
+      wallStepUs: Math.round(wallStepUs),
+    })
+  }
+
+  return {
+    toleranceUs,
+    activeSamples: active.length,
+    preparingSamples: preparing,
+    excursions,
+    byCause,
+    preReleaseSamples,
+    preReleaseWorstUs: Math.round(preReleaseWorstUs),
+    worstAbsUs: Math.round(worstAbsUs),
+    worstAtIntendedUs: Math.round(worstAtIntendedUs),
+    overAtIntended,
+    passAtIntended: active.length > 0 && overAtIntended === 0,
+    meanLatchLagUs: excursions.length
+      ? Math.round(latchLagSum / excursions.length)
+      : 0,
+    // No samples is not a pass. A run that never played has not been judged.
+    pass: active.length > 0 && excursions.length === 0,
+    uncertainty: {
+      frameQuantumUs: Math.round(frameIntervalUs / 2),
+      readSpanUs: Math.round(readSpanUs),
+      positionQuantumUs: 1000,
+    },
+  }
+}
+
+/**
+ * The samples around one excursion, for looking at rather than inferring from.
+ *
+ * @experimental
+ */
+export function windowAround(
+  samples: readonly SpikeSample[],
+  seq: number,
+  spanUs = 250_000,
+): SpikeSample[] {
+  const centre = samples.find((s) => s.seq === seq)
+  if (!centre) return []
+  const spanNs = spanUs * 1000
+  return samples.filter((s) => Math.abs(s.atNs - centre.atNs) <= spanNs)
+}