@pictor-sdk/video-android 0.1.0

package/src/submissionScoring.ts

@@ -0,0 +1,680 @@
+import type { SpikeSample } from './compositionSpike'
+
+/**
+ * Score the picture where it is handed to composition.
+ *
+ * ## The boundary, stated
+ *
+ * **Submission**, not scanout. A frame is scored at the instant composition
+ * begins consuming it; the compositor presents it some time later and nothing
+ * in this process observes that. The delay is common to every frame, so it
+ * shifts the whole series rather than spreading it — but it is not measured
+ * here and is not subtracted from anything.
+ *
+ * ## Why an interval rather than a point
+ *
+ * A frame is not a sample of one instant; it is the correct picture for a span
+ * of project time. Comparing a continuous audio clock against a single
+ * timestamp charges the frame's own duration to synchronisation, which is why
+ * the 0.5x clip — whose frames are correct for 66.7 ms each — looked worse than
+ * the 1x clip at identical quality.
+ *
+ * So a submitted frame carries `[ptsUs, ptsUs + intervalUs)`, and the error is
+ * how far outside that span the audio clock sits. Zero inside, and the distance
+ * to the nearer edge outside. The interval comes from the measured cadence,
+ * which for a clip of speed `s` and source interval `i` is `i / s` — 33.3 ms at
+ * 1x, 66.7 ms at 0.5x.
+ *
+ * @experimental
+ */
+
+/** One submission, as the consumer recorded it. @experimental */
+export interface SubmissionRecord {
+  frameId: number
+  epoch: number
+  ptsUs: number
+  intervalUs: number
+  submittedNs: number
+  /** What the texture reported holding at acquisition. */
+  texturePtsUs: number
+  /** Which ring slot carried it. */
+  slot: number
+  occupancy: number
+  /**
+   * The frame index read out of the slot's pixels when it was acquired.
+   *
+   * -1 when the probe was off or the band could not be read; `markerFailure`
+   * says which.
+   */
+  acquiredMarker: number
+  /** The frame index read out of the picture actually handed to composition. */
+  submittedMarker: number
+  /** Empty when the band was read; otherwise why it was not. */
+  markerFailure: string
+  /**
+   * The picture's whole life, so one row can be attributed.
+   *
+   * When it was acquired, when it was due for presentation, when it was
+   * promoted. Without these, a late submission cannot be separated into a late
+   * arrival, a long wait for its due time, and a slow promotion.
+   */
+  acquiredNs: number
+  dueNs: number
+  promotedNs: number
+  /**
+   * Rendering error against the frame the submitted band names: background
+   * level in 0..255, bar position in 0..1 of height. -1 when not measured.
+   */
+  levelDelta: number
+  barDelta: number
+}
+
+/** @experimental */
+export interface ScoredSubmission {
+  frameId: number
+  ptsUs: number
+  intervalUs: number
+  submittedNs: number
+  /** The audio clock at the submission instant, on the same monotonic timeline. */
+  audioUs: number
+  /** 0 inside the interval; otherwise the distance to the nearer edge. */
+  errorUs: number
+  /** Positive when the sound is past the frame's span — the picture is behind. */
+  signedUs: number
+  occupancy: number
+}
+
+/** @experimental */
+export interface SubmissionReport {
+  toleranceUs: number
+  scored: number
+  /** Submissions with no audio sample close enough to pair with. */
+  unpaired: number
+  excursions: ScoredSubmission[]
+  worstErrorUs: number
+  pass: boolean
+  /**
+   * What this run established, as three outcomes.
+   *
+   * A score computed from nothing is not a pass. Zero scored submissions means
+   * the run did not observe playback, and that has to be distinguishable from
+   * a run that observed it and found it clean.
+   */
+  verdict: 'pass' | 'fail' | 'invalid'
+  verdictReason: string
+  /** Gaps between consecutive submitted timestamps: the delivered cadence. */
+  cadenceUs: { median: number; min: number; max: number; samples: number }
+  /**
+   * Wall-clock gaps between consecutive submissions.
+   *
+   * The tail is the point: a median at the output interval says nothing about
+   * the frame that took four times as long, and that frame is the one somebody
+   * sees. Taken from `submittedNs`, which is the instant composition began
+   * consuming each picture.
+   */
+  frameTimeUs: { p50: number; p95: number; p99: number; max: number; samples: number }
+  peakOccupancy: number
+}
+
+/**
+ * The audio clock at an arbitrary instant, from the paired samples around it.
+ *
+ * The composition runs at project rate 1x, so the clock is linear between
+ * samples and interpolation is exact rather than an approximation. Returns null
+ * when there is no sample near enough to interpolate from, which is reported
+ * rather than guessed.
+ */
+function audioAt(
+  samples: readonly SpikeSample[],
+  atNs: number,
+  windowNs: number,
+): number | null {
+  let before: SpikeSample | null = null
+  let after: SpikeSample | null = null
+  for (const s of samples) {
+    if (!s.playing) continue
+    if (s.atNs <= atNs && (before === null || s.atNs > before.atNs)) before = s
+    if (s.atNs >= atNs && (after === null || s.atNs < after.atNs)) after = s
+  }
+  if (before !== null && after !== null && after.atNs > before.atNs) {
+    const spanUs = (after.atNs - before.atNs) / 1000
+    const movedUs = after.positionUs - before.positionUs
+    /*
+      Never interpolate across a seek.
+
+      Between two samples the clock advances with the wall clock, because the
+      composition runs at project rate 1x. When it does not, it was *moved* —
+      and interpolating across that produced a 2.7 second "error" for a frame
+      that was correct when it was handed over: the picture was at 23.9 s and
+      the position it was paired with had already jumped back to 21.2 s.
+
+      Detected from the clock itself rather than from an epoch number, because
+      the consumer's frame generation and the player's transition counter are
+      different counters and equal values between them mean nothing.
+    */
+    if (Math.abs(movedUs - spanUs) <= DISCONTINUITY_US) {
+      const t = (atNs - before.atNs) / (after.atNs - before.atNs)
+      return before.positionUs + t * movedUs
+    }
+    // A jump: use the sample on the submission's own side of it.
+    return before.positionUs + (atNs - before.atNs) / 1000
+  }
+  const nearest = before ?? after
+  if (nearest === null) return null
+  if (Math.abs(nearest.atNs - atNs) > windowNs) return null
+  // Rate 1x, so extrapolating the short distance to the edge sample is exact.
+  return nearest.positionUs + (atNs - nearest.atNs) / 1000
+}
+
+/**
+ * How far the clock may diverge from the wall clock and still be running.
+ *
+ * Beyond this it was moved rather than advanced, which is a seek.
+ */
+const DISCONTINUITY_US = 100_000
+
+/** @experimental */
+export function scoreSubmissions(
+  submissions: readonly SubmissionRecord[],
+  samples: readonly SpikeSample[],
+  options: { toleranceUs?: number; pairWindowNs?: number } = {},
+): SubmissionReport {
+  const toleranceUs = options.toleranceUs ?? 60_000
+  const pairWindowNs = options.pairWindowNs ?? 50_000_000
+
+  const scored: ScoredSubmission[] = []
+  const excursions: ScoredSubmission[] = []
+  let unpaired = 0
+  let worstErrorUs = 0
+  let peakOccupancy = 0
+
+  for (const record of submissions) {
+    if (record.ptsUs < 0 || record.submittedNs <= 0) continue
+    if (record.occupancy > peakOccupancy) peakOccupancy = record.occupancy
+    const audioUs = audioAt(samples, record.submittedNs, pairWindowNs)
+    if (audioUs === null) {
+      unpaired += 1
+      continue
+    }
+    /*
+      Inside the span is not an error of any size.
+
+      The frame is the correct picture for the whole interval, so a clock
+      anywhere within it is synchronised. Scoring the distance to `ptsUs`
+      instead would report the frame's own duration as drift.
+    */
+    const start = record.ptsUs
+    const end = record.ptsUs + Math.max(record.intervalUs, 0)
+    const signedUs = audioUs < start ? audioUs - start : audioUs > end ? audioUs - end : 0
+    const errorUs = Math.abs(signedUs)
+    const entry: ScoredSubmission = {
+      frameId: record.frameId,
+      ptsUs: record.ptsUs,
+      intervalUs: record.intervalUs,
+      submittedNs: record.submittedNs,
+      audioUs: Math.round(audioUs),
+      errorUs: Math.round(errorUs),
+      signedUs: Math.round(signedUs),
+      occupancy: record.occupancy,
+    }
+    scored.push(entry)
+    if (errorUs > worstErrorUs) worstErrorUs = errorUs
+    if (errorUs > toleranceUs) excursions.push(entry)
+  }
+
+  const gaps: number[] = []
+  const frameTimes: number[] = []
+  for (let i = 1; i < scored.length; i += 1) {
+    const gap = scored[i]!.ptsUs - scored[i - 1]!.ptsUs
+    if (gap > 0) gaps.push(gap)
+    const wall = (scored[i]!.submittedNs - scored[i - 1]!.submittedNs) / 1000
+    if (wall > 0) frameTimes.push(wall)
+  }
+  gaps.sort((a, b) => a - b)
+  frameTimes.sort((a, b) => a - b)
+  const at = (fraction: number) =>
+    frameTimes.length === 0
+      ? 0
+      : Math.round(frameTimes[Math.min(frameTimes.length - 1, Math.floor(frameTimes.length * fraction))]!)
+
+  return {
+    toleranceUs,
+    scored: scored.length,
+    unpaired,
+    excursions,
+    worstErrorUs: Math.round(worstErrorUs),
+    // No submissions is not a pass: a run that showed nothing was not judged.
+    pass: scored.length > 0 && excursions.length === 0,
+    verdict: scored.length === 0 ? 'invalid' : excursions.length === 0 ? 'pass' : 'fail',
+    verdictReason:
+      scored.length === 0
+        ? 'no submissions scored'
+        : excursions.length === 0
+          ? ''
+          : `${excursions.length} excursions, worst ${Math.round(worstErrorUs / 1000)} ms`,
+    cadenceUs: {
+      median: gaps.length > 0 ? gaps[Math.floor(gaps.length / 2)]! : 0,
+      min: gaps.length > 0 ? gaps[0]! : 0,
+      max: gaps.length > 0 ? gaps[gaps.length - 1]! : 0,
+      samples: gaps.length,
+    },
+    frameTimeUs: {
+      p50: at(0.5),
+      p95: at(0.95),
+      p99: at(0.99),
+      max: frameTimes.length > 0 ? Math.round(frameTimes[frameTimes.length - 1]!) : 0,
+      samples: frameTimes.length,
+    },
+    peakOccupancy,
+  }
+}
+
+/** One submission, with its whole life and the clock it was scored against. @experimental */
+export interface TraceRow {
+  frameId: number
+  /** Offsets in ms from the window's first row, so the numbers are readable. */
+  acquiredMs: number
+  dueMs: number
+  promotedMs: number
+  submittedMs: number
+  /**
+   * Acquisition to due, due to promotion, promotion to submission, in us.
+   *
+   * A negative `waitUs` means the presentation time had already passed when the
+   * frame was acquired — it was late before the ring saw it.
+   */
+  waitUs: number
+  promoteUs: number
+  submitUs: number
+  ptsUs: number
+  audioUs: number
+  errorUs: number
+  slot: number
+  occupancy: number
+  marker: number
+}
+
+/** @experimental */
+export interface SubmissionTrace {
+  /** The frame the window is centred on. */
+  frameId: number
+  rows: TraceRow[]
+  /**
+   * Where the centre frame's budget went, as the largest single interval.
+   *
+   * `arrival` when it was acquired late, `due` when it sat waiting for its
+   * presentation time, `promotion` when it was due and not chosen, `submission`
+   * when promotion and submission diverged. Empty when the row is incomplete.
+   */
+  spentOn: 'arrival' | 'due' | 'promotion' | 'submission' | 'unknown'
+  /** The dominant leg's size, so a label cannot imply a magnitude it lacks. */
+  spentUs: number
+  /** True when the window could not be built from complete records. */
+  invalid: boolean
+  reason: string
+}
+
+/**
+ * The life of one submission and its neighbours.
+ *
+ * Built for attribution, not for scoring: a submission excursion says a picture
+ * was wrong at an instant, and this says where its time went. Acquisition, due
+ * time, promotion and submission are four stamps on one frame, and the largest
+ * gap between consecutive ones is the interval to investigate.
+ *
+ * Incomplete rows are reported rather than dropped. A ring buffer that wrapped,
+ * a record written before the ring carried these stamps, or a frame that never
+ * reached submission all produce a window that cannot be attributed, and a
+ * silently shortened window would read as a clean one.
+ *
+ * @experimental
+ */
+export function traceSubmission(
+  records: readonly SubmissionRecord[],
+  scored: readonly ScoredSubmission[],
+  frameId: number,
+  span = 8,
+): SubmissionTrace {
+  const ordered = records.filter((r) => r.submittedNs > 0).sort((a, b) => a.submittedNs - b.submittedNs)
+  const centre = ordered.findIndex((r) => r.frameId === frameId)
+  if (centre < 0) {
+    return {
+      frameId,
+      rows: [],
+      spentOn: 'unknown',
+      spentUs: 0,
+      invalid: true,
+      reason: 'frame not retained',
+    }
+  }
+  const from = Math.max(0, centre - span)
+  const window = ordered.slice(from, centre + span + 1)
+  const base = window[0]!.submittedNs
+  const ms = (ns: number) => (ns > 0 ? Math.round((ns - base) / 1e6) : -1)
+
+  const rows: TraceRow[] = window.map((r) => {
+    const match = scored.find((s) => s.frameId === r.frameId)
+    return {
+      frameId: r.frameId,
+      acquiredMs: ms(r.acquiredNs),
+      dueMs: ms(r.dueNs),
+      promotedMs: ms(r.promotedNs),
+      submittedMs: ms(r.submittedNs),
+      waitUs: r.acquiredNs > 0 && r.dueNs > 0 ? Math.round((r.dueNs - r.acquiredNs) / 1000) : -1,
+      promoteUs: r.dueNs > 0 && r.promotedNs > 0 ? Math.round((r.promotedNs - r.dueNs) / 1000) : -1,
+      submitUs:
+        r.promotedNs > 0 && r.submittedNs > 0 ? Math.round((r.submittedNs - r.promotedNs) / 1000) : -1,
+      ptsUs: r.ptsUs,
+      audioUs: match === undefined ? -1 : match.audioUs,
+      errorUs: match === undefined ? -1 : match.errorUs,
+      slot: r.slot,
+      occupancy: r.occupancy,
+      marker: r.submittedMarker,
+    }
+  })
+
+  const target = rows.find((r) => r.frameId === frameId)!
+  /*
+    Presence is checked on the raw stamps, not on the offsets.
+
+    An offset is relative to the window's first submission, so a frame acquired
+    before that submission has a negative one — which is ordinary, and which an
+    earlier version of this check read as "not recorded" and refused to
+    attribute. Zero nanoseconds means never stamped; a negative offset means
+    earlier than the window.
+  */
+  const raw = window.find((r) => r.frameId === frameId)!
+  if (raw.acquiredNs <= 0 || raw.dueNs <= 0 || raw.promotedNs <= 0) {
+    return {
+      frameId,
+      rows,
+      spentOn: 'unknown',
+      spentUs: 0,
+      invalid: true,
+      reason: 'incomplete stamps',
+    }
+  }
+  /*
+    A frame acquired after its own due time was late before the ring saw it.
+
+    `waitUs` is due minus acquired, so a negative value means the presentation
+    time had already passed on arrival. Measuring promotion from that stale due
+    time then charges the ring for the wait as well, and the first version of
+    this did exactly that: it reported `promotion` for frames whose promote leg
+    was 96 ms and whose wait was minus 52 ms — the same 52 ms counted twice, in
+    the wrong place. Promotion is measured from whichever came later.
+  */
+  const index = rows.indexOf(target)
+  const previous = index > 0 ? rows[index - 1]! : null
+  /*
+    Arrival is the gap since the previous picture went out, always.
+
+    An earlier version used the overshoot past the due time instead whenever the
+    frame arrived late, and that hid the thing actually worth seeing. Measured
+    at the clip seam: the frame missed its due time by 2 ms and had been
+    preceded by 133 ms in which nothing was acquired at all. Charging it 2 ms
+    named the wrong interval by two orders of magnitude.
+  */
+  const arrivalUs =
+    previous === null ? 0 : Math.round((target.acquiredMs - previous.submittedMs) * 1000)
+  const promoteUs = target.waitUs < 0 ? target.promoteUs + target.waitUs : target.promoteUs
+  const legs: [SubmissionTrace['spentOn'], number][] = [
+    ['arrival', arrivalUs],
+    ['due', Math.max(0, target.waitUs)],
+    ['promotion', promoteUs],
+    ['submission', target.submitUs],
+  ]
+  let spentOn: SubmissionTrace['spentOn'] = 'unknown'
+  let worst = -1
+  for (const [name, value] of legs) {
+    if (value > worst) {
+      worst = value
+      spentOn = name
+    }
+  }
+  return { frameId, rows, spentOn, spentUs: Math.max(0, worst), invalid: false, reason: '' }
+}
+
+/** @experimental */
+export interface SeamStage {
+  framePtsUs: number
+  /** Offsets in ms from the window's first sample. */
+  metadataMs: number
+  availableMs: number
+  consumedMs: number
+  /** Renderer to queue, queue to latch, in us. -1 when a stamp is missing. */
+  graphUs: number
+  latchUs: number
+}
+
+/** @experimental */
+export interface SeamTrace {
+  /** The boundary this window is centred on, in project time. */
+  atUs: number
+  /** The largest gap between consecutive frames reaching the renderer, in us. */
+  outputGapUs: number
+  /** The same gap measured at acquisition. */
+  acquireGapUs: number
+  /**
+   * Who owns the gap.
+   *
+   * `graph` when the renderer produced and the queue did not follow.
+   * `consumer` when it was queued and not latched. `production` when nothing
+   * reached the renderer's output callback at all.
+   *
+   * That last one is deliberately broader than "the decoder was unprepared":
+   * this stamp fires when the renderer is about to hand a frame to the surface,
+   * so its absence covers a decoder that produced nothing *and* a renderer
+   * blocked waiting for the sink to accept a new input stream. Separating those
+   * two needs a stamp inside Media3 that is not exposed here. What it does
+   * establish is that the graph and this consumer are not responsible, which is
+   * what the acquisition gap alone could not say.
+   */
+  owner: 'production' | 'graph' | 'consumer' | 'unknown'
+  stages: SeamStage[]
+  invalid: boolean
+  reason: string
+}
+
+/**
+ * Where output stops at a clip boundary.
+ *
+ * The acquisition gap says delivery stopped; it cannot say whether the decoder
+ * produced nothing, the video graph held what it produced, or this consumer
+ * failed to latch what was queued. Those are three owners and three different
+ * fixes, and they are separated by one stamp: when the renderer said a frame
+ * was about to go to the surface.
+ *
+ * Samples carrying no frame are skipped rather than treated as a gap — the
+ * poller runs faster than the frame rate, so most samples repeat the last
+ * frame, and counting those would report a stall on every tick.
+ *
+ * @experimental
+ */
+export function traceSeam(
+  samples: readonly SpikeSample[],
+  atUs: number,
+  spanUs = 600_000,
+): SeamTrace {
+  const near = samples
+    .filter(
+      (s) =>
+        s.framePtsUs >= 0 &&
+        s.framePtsUs >= atUs - spanUs &&
+        s.framePtsUs <= atUs + spanUs &&
+        s.frameConsumedNs > 0,
+    )
+    .sort((a, b) => a.frameConsumedNs - b.frameConsumedNs)
+
+  // One row per distinct frame: the poller ticks faster than frames arrive.
+  const unique: SpikeSample[] = []
+  for (const s of near) {
+    if (unique.length === 0 || unique[unique.length - 1]!.frameId !== s.frameId) unique.push(s)
+  }
+  if (unique.length < 3) {
+    return {
+      atUs,
+      outputGapUs: -1,
+      acquireGapUs: -1,
+      owner: 'unknown',
+      stages: [],
+      invalid: true,
+      reason: 'too few frames around the boundary',
+    }
+  }
+
+  const base = unique[0]!.frameConsumedNs
+  const ms = (ns: number) => (ns > 0 ? Math.round((ns - base) / 1e6) : -1)
+  const stages: SeamStage[] = unique.map((s) => ({
+    framePtsUs: s.framePtsUs,
+    metadataMs: ms(s.frameMetadataNs),
+    availableMs: ms(s.frameAvailableNs),
+    consumedMs: ms(s.frameConsumedNs),
+    graphUs:
+      s.frameMetadataNs > 0 && s.frameAvailableNs > 0
+        ? Math.round((s.frameAvailableNs - s.frameMetadataNs) / 1000)
+        : -1,
+    latchUs:
+      s.frameAvailableNs > 0 && s.frameConsumedNs > 0
+        ? Math.round((s.frameConsumedNs - s.frameAvailableNs) / 1000)
+        : -1,
+  }))
+
+  let outputGapUs = 0
+  let acquireGapUs = 0
+  let at = -1
+  for (let i = 1; i < unique.length; i += 1) {
+    const acquire = Math.round((unique[i]!.frameConsumedNs - unique[i - 1]!.frameConsumedNs) / 1000)
+    if (acquire > acquireGapUs) {
+      acquireGapUs = acquire
+      at = i
+    }
+    const a = unique[i]!.frameMetadataNs
+    const b = unique[i - 1]!.frameMetadataNs
+    if (a > 0 && b > 0) outputGapUs = Math.max(outputGapUs, Math.round((a - b) / 1000))
+  }
+
+  /*
+    The owner is read off the widest frame's own stages, not off the gap.
+
+    A gap is a distance between two frames; the question is where the later one
+    was held, and that is inside its own row.
+  */
+  let owner: SeamTrace['owner'] = 'unknown'
+  if (at > 0) {
+    const s = stages[at]!
+    if (s.graphUs < 0 || s.latchUs < 0) owner = 'unknown'
+    else if (s.graphUs >= s.latchUs && s.graphUs > 16_000) owner = 'graph'
+    else if (s.latchUs > 16_000) owner = 'consumer'
+    else owner = 'production'
+  }
+  return { atUs, outputGapUs, acquireGapUs, owner, stages, invalid: false, reason: '' }
+}
+
+/** One frame, seen from both sides of the pairing. @experimental */
+export interface StartupRow {
+  frameId: number
+  /** What Media3 said this frame's presentation time was. */
+  renderedPtsUs: number
+  /** What the submission record was written under. The same number, or a bug. */
+  recordedPtsUs: number
+  /** What the pixels say the frame is. Independent of both. */
+  marker: number
+  /** The source frame the marker's own index implies, at this clip's speed. */
+  markerPtsUs: number
+  metadataMs: number
+  consumedMs: number
+  submittedMs: number
+}
+
+/** @experimental */
+export interface StartupTrace {
+  rows: StartupRow[]
+  /**
+   * Frames the renderer never emitted, by source index.
+   *
+   * The discriminator. If the renderer emitted a presentation time and the
+   * pixels under it belong to a different frame, the *association* is wrong. If
+   * it never emitted that time at all, the frame was dropped or never produced.
+   * Those are different defects and the pixels alone cannot tell them apart.
+   */
+  missingSourceFrames: number[]
+  /** Frames whose pixels disagree with the time they were rendered under. */
+  misassociated: number[]
+  invalid: boolean
+  reason: string
+}
+
+/**
+ * The first frames of a clip, from both sides of the pairing at once.
+ *
+ * Media3's presentation time and the marker in the pixels are independent
+ * claims about the same frame. Joined by frame id, they separate "frames 2 to 6
+ * were never produced" from "frame 7's pixels arrived labelled as frame 2" —
+ * which look identical in a cadence result and need opposite fixes.
+ *
+ * @experimental
+ */
+export function traceStartup(
+  samples: readonly SpikeSample[],
+  records: readonly SubmissionRecord[],
+  options: { sourceFps: number; speed: number; count?: number },
+): StartupTrace {
+  const count = options.count ?? 10
+  if (options.sourceFps <= 0 || options.speed <= 0) {
+    return { rows: [], missingSourceFrames: [], misassociated: [], invalid: true, reason: 'no clip rate' }
+  }
+  // One source frame in project time: the unit both sides are compared in.
+  const stepUs = 1_000_000 / options.sourceFps / options.speed
+
+  const byFrame = new Map<number, SpikeSample>()
+  for (const s of samples) {
+    if (s.frameId < 0 || s.frameConsumedNs <= 0) continue
+    const seen = byFrame.get(s.frameId)
+    if (seen === undefined || s.frameConsumedNs < seen.frameConsumedNs) byFrame.set(s.frameId, s)
+  }
+  const ordered = records
+    .filter((r) => r.submittedNs > 0)
+    .sort((a, b) => a.submittedNs - b.submittedNs)
+    .slice(0, count)
+  if (ordered.length === 0) {
+    return { rows: [], missingSourceFrames: [], misassociated: [], invalid: true, reason: 'no submissions' }
+  }
+
+  const base = ordered[0]!.submittedNs
+  const ms = (ns: number) => (ns > 0 ? Math.round((ns - base) / 1e6) : -1)
+  const rows: StartupRow[] = ordered.map((r) => {
+    const s = byFrame.get(r.frameId)
+    return {
+      frameId: r.frameId,
+      renderedPtsUs: s === undefined ? -1 : s.framePtsUs,
+      recordedPtsUs: r.ptsUs,
+      marker: r.submittedMarker,
+      markerPtsUs: r.submittedMarker >= 0 ? Math.round(r.submittedMarker * stepUs) : -1,
+      metadataMs: s === undefined ? -1 : ms(s.frameMetadataNs),
+      consumedMs: s === undefined ? -1 : ms(s.frameConsumedNs),
+      submittedMs: ms(r.submittedNs),
+    }
+  })
+
+  /*
+    Misassociated when the pixels and the rendered time name different frames;
+    missing when the rule's frame never appears as a rendered time at all.
+  */
+  const misassociated: number[] = []
+  const rendered = new Set<number>()
+  for (const row of rows) {
+    if (row.renderedPtsUs >= 0) rendered.add(Math.round(row.renderedPtsUs / stepUs))
+    if (row.marker < 0 || row.renderedPtsUs < 0) continue
+    if (Math.abs(row.markerPtsUs - row.renderedPtsUs) > stepUs / 2) misassociated.push(row.frameId)
+  }
+  const last = rows[rows.length - 1]!
+  const highest = Math.max(
+    last.marker >= 0 ? last.marker : 0,
+    last.renderedPtsUs >= 0 ? Math.round(last.renderedPtsUs / stepUs) : 0,
+  )
+  const missingSourceFrames: number[] = []
+  for (let i = 0; i <= highest; i += 1) if (!rendered.has(i)) missingSourceFrames.push(i)
+
+  return { rows, missingSourceFrames, misassociated, invalid: false, reason: '' }
+}