deepline 0.3.258 → 0.3.259

package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts

@@ -7,6 +7,7 @@ import {
   normalizePlayRunLedgerTerminalSource,
 } from './run-terminal-source';
 import { MAX_LEDGER_LOG_LINES_PER_EVENT } from './ledger-safe-payload';
+import type { RunStopCause } from './run-execution-state';
 import {
   normalizePlayDocflowNodeIoState,
   type PlayDocflowNodeIoState,
@@ -94,6 +95,14 @@ export type PlayRunLedgerStepSnapshot = {
   kind?: string;
   status: PlayRunLedgerStepStatus;
   error?: string | null;
+  /**
+   * Set when the run's own terminal event settled this step while it was in
+   * flight: the step did not fail, the run stopped under it. Its status reads
+   * `failed` for compatibility; readers derive `stopped` through
+   * `deriveNodeExecutionState` (run-execution-state.ts). Absent on snapshots
+   * written before the field and on every step that settled itself.
+   */
+  stoppedBy?: RunStopCause;
   artifactTableNamespace?: string | null;
   startedAt?: number | null;
   completedAt?: number | null;
@@ -863,6 +872,11 @@ export function normalizePlayRunLedgerSnapshot(
       kind: optionalString(rawStep.kind),
       status,
       error: optionalNullableString(rawStep.error),
+      ...(status === 'failed' &&
+      (rawStep.stoppedBy === 'run_failed' ||
+        rawStep.stoppedBy === 'run_cancelled')
+        ? { stoppedBy: rawStep.stoppedBy }
+        : {}),
       artifactTableNamespace: optionalNullableString(
         rawStep.artifactTableNamespace,
       ),
@@ -1034,6 +1048,24 @@ function maxProgressNumber(
   return Math.max(current, next);
 }
 
+/**
+ * A row is counted once: it either succeeded or failed. `failed` is merged by
+ * max, so a row that failed on one attempt and then succeeded on a later one
+ * (a resumed run re-executes it) would otherwise stay counted in both, and a
+ * 2-row map settled "2 completed, 2 failed of 2". Once `completed` and
+ * `total` are known, failed rows are at most the rows that did not succeed.
+ */
+function settledFailedRows(input: {
+  failed: number;
+  completed: number | undefined;
+  total: number | undefined;
+}): number {
+  if (input.completed === undefined || input.total === undefined) {
+    return input.failed;
+  }
+  return Math.min(input.failed, Math.max(0, input.total - input.completed));
+}
+
 function mergeMonotonicStepProgress(input: {
   current?: PlayRunLedgerStepProgress | null;
   next: PlayRunLedgerStepProgress;
@@ -1056,7 +1088,13 @@ function mergeMonotonicStepProgress(input: {
       ? { total: maxProgressNumber(current.total, next.total) }
       : {}),
     ...(maxProgressNumber(current.failed, next.failed) !== undefined
-      ? { failed: maxProgressNumber(current.failed, next.failed) }
+      ? {
+          failed: settledFailedRows({
+            failed: maxProgressNumber(current.failed, next.failed)!,
+            completed: maxProgressNumber(current.completed, next.completed),
+            total: maxProgressNumber(current.total, next.total),
+          }),
+        }
       : {}),
     ...(maxProgressNumber(current.startedRows, next.startedRows) !== undefined
       ? {
@@ -1089,15 +1127,57 @@ function clearInFlightProgressOnTerminal(
   progress: PlayRunLedgerStepProgress | null | undefined,
 ): PlayRunLedgerStepProgress | null | undefined {
   if (!progress) return progress;
-  if (progress.activeRows === undefined && progress.waitingRows === undefined) {
+  const message =
+    typeof progress.message === 'string'
+      ? settledProgressMessage(progress.message, progress)
+      : progress.message;
+  if (
+    progress.activeRows === undefined &&
+    progress.waitingRows === undefined &&
+    message === progress.message
+  ) {
     return progress;
   }
   const cleared = { ...progress };
   delete cleared.activeRows;
   delete cleared.waitingRows;
+  if (message === undefined) delete cleared.message;
+  else cleared.message = message;
   return cleared;
 }
 
+/**
+ * A live progress line's in-flight clause ("20 calls in flight · ") is a
+ * live-only counter in words, like `activeRows`: a settled step has nothing in
+ * flight. A run that stopped kept "20 calls in flight · 1,241 / 50,000 rows
+ * processed" as its final word. Drop the clause and keep the settled count; a
+ * line that was only the clause says nothing once it is gone.
+ */
+const IN_FLIGHT_MESSAGE_CLAUSE = /^\s*[\d,]+ calls? in flight(?:\s*·\s*)?/i;
+/** The producer's plain count line, "1,241 / 50,000 rows processed". */
+const ROWS_PROCESSED_MESSAGE = /^[\d,]+ \/ [\d,]+ rows processed$/;
+
+function settledProgressMessage(
+  message: string,
+  progress: PlayRunLedgerStepProgress,
+): string | undefined {
+  const settled = message.replace(IN_FLIGHT_MESSAGE_CLAUSE, '').trim();
+  if (settled.length === 0) return undefined;
+  // The line is the last live tick's words; the settled counts can be newer
+  // (a failed run's terminal row settles them from the runtime sheet). A plain
+  // count line restates the counts it now sits beside, never older ones.
+  const completed = progress.completedRows ?? progress.completed;
+  if (
+    ROWS_PROCESSED_MESSAGE.test(settled) &&
+    typeof completed === 'number' &&
+    typeof progress.total === 'number' &&
+    progress.total > 0
+  ) {
+    return `${completed.toLocaleString()} / ${progress.total.toLocaleString()} rows processed`;
+  }
+  return settled === message ? message : settled;
+}
+
 function isTerminalStepStatus(status: PlayRunLedgerStepStatus): boolean {
   return status === 'completed' || status === 'failed' || status === 'skipped';
 }
@@ -1402,9 +1482,12 @@ function retryablePlatformDeployFailureSnapshot(
 // counts, just mark it settled so the UI stops rendering a live ticker.
 function settleRunningStepsOnTerminal(
   snapshot: PlayRunLedgerSnapshot,
-  status: Extract<PlayRunLedgerStepStatus, 'completed' | 'failed'>,
+  settle:
+    | { status: 'completed' }
+    | { status: 'failed'; stoppedBy: RunStopCause },
   occurredAt: number,
 ): PlayRunLedgerSnapshot {
+  const { status } = settle;
   let changed = false;
   const stepsById = Object.fromEntries(
     Object.entries(snapshot.stepsById).map(([stepId, step]) => {
@@ -1417,6 +1500,9 @@ function settleRunningStepsOnTerminal(
         {
           ...step,
           status,
+          ...(settle.status === 'failed'
+            ? { stoppedBy: settle.stoppedBy }
+            : {}),
           completedAt: step.completedAt ?? occurredAt,
           updatedAt: occurredAt,
           progress: step.progress
@@ -1621,7 +1707,11 @@ export function reducePlayRunLedgerEvent(
               terminalWorkCounterSegment !== undefined),
         }) ??
         withTiming({
-          ...settleRunningStepsOnTerminal(base, 'completed', occurredAt),
+          ...settleRunningStepsOnTerminal(
+            base,
+            { status: 'completed' },
+            occurredAt,
+          ),
           status: 'completed',
           waitKind: null,
           waitUntil: null,
@@ -1664,7 +1754,11 @@ export function reducePlayRunLedgerEvent(
               terminalWorkCounterSegment !== undefined),
         }) ??
         withTiming({
-          ...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
+          ...settleRunningStepsOnTerminal(
+            base,
+            { status: 'failed', stoppedBy: 'run_failed' },
+            occurredAt,
+          ),
           status: 'failed',
           waitKind: null,
           waitUntil: null,
@@ -1697,7 +1791,11 @@ export function reducePlayRunLedgerEvent(
             canPromoteTerminalResultReference(base.result, event.result),
         }) ??
         withTiming({
-          ...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
+          ...settleRunningStepsOnTerminal(
+            base,
+            { status: 'failed', stoppedBy: 'run_cancelled' },
+            occurredAt,
+          ),
           status: 'cancelled',
           waitKind: null,
           waitUntil: null,
@@ -1915,6 +2013,8 @@ export function reducePlayRunLedgerEvent(
         // Terminal step: clear live in-flight/waiting accounting.
         progress: clearInFlightProgressOnTerminal(current?.progress) ?? null,
       };
+      // The step's own settle event replaces a stop the run recorded.
+      if (!preserveFailedStatus) delete nextStep.stoppedBy;
       const runningStepIds = base.orderedStepIds.filter((stepId) => {
         if (stepId === event.stepId) return false;
         return base.stepsById[stepId]?.status === 'running';

package/dist/bundling-sources/shared_libs/play-runtime/run-observation.ts

@@ -0,0 +1,440 @@
+import {
+  buildPlayRunStatusSnapshot,
+  type LedgerBackedRunLike,
+  type PlayRunStreamNodeState,
+} from './run-snapshot-stream';
+import {
+  decodePlayRunPublicStatus,
+  isActivePlayRunLifecycleStatus,
+  isTerminalPlayRunLifecycleStatus,
+  type PlayRunPublicStatus,
+} from './run-lifecycle-policy';
+import type { RunStopCause } from './run-execution-state';
+
+/**
+ * Run Observation (ADR-0022, docs/plans/2026-09-30-live-run-observation.md §3.2).
+ *
+ * The one bounded document every dashboard surface reads a Play Run's live
+ * state from. It is a pure projection of the Convex Run Snapshot (ADR-0001);
+ * Runtime Postgres stays the lifecycle authority and is never read here.
+ *
+ * Bounded BY TYPE (AGENTS.md "Status and metadata documents"): every field is
+ * a scalar, a fixed-shape summary, or `nodes`, whose length is the static
+ * graph's step count (an artifact property, not a function of rows, calls,
+ * facts or logs). Row payloads, log lines, calls and billing facts live behind
+ * their own paginated endpoints. `run-observation-size-contract.test.ts` pins
+ * the byte bound for a 20,000-row / 200-node / 50,000-log run.
+ */
+
+/**
+ * How the run started, as far as its admission recorded it: the automated
+ * trigger that fired it, a child or rerun of another run, or the client
+ * surface a person used (`ui`, `cli`, `api`). `unknown` when nothing says —
+ * never a guess. The run table reads the same derivation (`runLaunchTrigger`),
+ * so the canvas Start pill and the runs grid name the same provenance.
+ */
+export type RunObservationTrigger =
+  | 'ui'
+  | 'cli'
+  | 'api'
+  | 'rerun'
+  | 'schedule'
+  | 'webhook'
+  | 'sql'
+  | 'child'
+  | 'unknown';
+
+/** Public lifecycle status, or `unknown` when the stored value is unreadable. */
+export type RunObservationStatus = PlayRunPublicStatus | 'unknown';
+
+export type RunObservationNodeStatus = PlayRunStreamNodeState['status'];
+
+export type RunObservationNode = {
+  nodeId: string;
+  status: RunObservationNodeStatus;
+  startedAt: number | null;
+  completedAt: number | null;
+  /** The step's declared row total, when the runtime reported one. */
+  total: number | null;
+  /** Rows the step has finished (the progress `completed` counter). */
+  settled: number;
+  failed: number;
+  /**
+   * The run stopped under this step (it did not fail itself): the projected
+   * Run Snapshot fact `deriveNodeExecutionState` reads. Absent otherwise.
+   */
+  stoppedBy?: RunStopCause;
+};
+
+export type RunObservation = {
+  runId: string;
+  playName: string;
+  trigger: RunObservationTrigger;
+  status: RunObservationStatus;
+  /**
+   * The seq of the last applied Run Event (Convex assigns it in the append),
+   * or `null` for a row that predates the ledger cursor and for an admission
+   * bridge answer. Freshness falls back to `updatedAt`; see
+   * {@link compareRunObservationFreshness}.
+   */
+  ledgerSeq: number | null;
+  updatedAt: number;
+  /** `admission` = the pre-projection bridge: scalar, no node claims. */
+  source: 'snapshot' | 'admission';
+  createdAt: number;
+  startedAt: number | null;
+  finishedAt: number | null;
+  /** Present only while waiting. The UI derives "overdue" against serverTime. */
+  wait: { kind: string; deadlineAt: number | null } | null;
+  activity: { target: string; state: string; label: string } | null;
+  error: { code: string; message: string } | null;
+  /** Terminal row outcomes; `null` while the run is active (counts churn). */
+  rows: { total: number; ok: number; failed: number; pending: number } | null;
+  nodes: RunObservationNode[];
+  /** Cumulative ingested log lines. Lines stay on `/runs/:id/logs`. */
+  logs: { total: number };
+};
+
+/** One row of the observe head: enough for the grid, rail and header count. */
+export type RunObservationHeadEntry = {
+  runId: string;
+  status: RunObservationStatus;
+  trigger: RunObservationTrigger;
+  createdAt: number;
+  updatedAt: number;
+  finishedAt: number | null;
+};
+
+/** The error message bound. Full text stays on `GET /runs/:id`. */
+export const RUN_OBSERVATION_ERROR_MESSAGE_MAX_CHARS = 500;
+/** The head is the newest runs of one Play. */
+export const RUN_OBSERVATION_HEAD_LIMIT = 20;
+/** Per-run observations one request may ask for. */
+export const RUN_OBSERVATION_WATCH_LIMIT = 10;
+
+/** The stored `playRuns` fields the projection reads. */
+export type RunObservationSourceRun = LedgerBackedRunLike &
+  RunLaunchProvenance & {
+    name: string;
+    createdAt: number;
+    updatedAt: number;
+    ledgerNextSeq?: number | null;
+    error?: string | null;
+  };
+
+/** The stored fields a run's launch provenance is read from. */
+export type RunLaunchProvenance = {
+  triggerSource?: string | null;
+  launchSurface?: string | null;
+  replayedFromRunId?: string | null;
+  parentRunId?: string | null;
+};
+
+/**
+ * The ONE derivation of how a run started. An automated trigger names
+ * itself; a child or a rerun says so; otherwise the client surface recorded
+ * at admission. `triggerSource: 'api'` alone is the default for every
+ * person-started run, so without a recorded surface it names nothing.
+ */
+export function runLaunchTrigger(
+  run: RunLaunchProvenance,
+): RunObservationTrigger {
+  switch (run.triggerSource) {
+    case 'cron':
+      return 'schedule';
+    case 'webhook':
+      return 'webhook';
+    case 'sql_listener':
+      return 'sql';
+    case 'async_child':
+      return 'child';
+  }
+  if (run.parentRunId) return 'child';
+  if (run.replayedFromRunId) return 'rerun';
+  switch (run.launchSurface) {
+    case 'ui':
+    case 'cli':
+    case 'api':
+      return run.launchSurface;
+    default:
+      return 'unknown';
+  }
+}
+
+export function runObservationStatus(value: unknown): RunObservationStatus {
+  return decodePlayRunPublicStatus(value) ?? 'unknown';
+}
+
+function finite(value: unknown): number | null {
+  return typeof value === 'number' && Number.isFinite(value) ? value : null;
+}
+
+function nonNegative(value: unknown): number {
+  const number = finite(value);
+  return number !== null && number > 0 ? number : 0;
+}
+
+/** `ledgerNextSeq` is the NEXT seq; the last applied one is one less. */
+export function ledgerSeqFromNextSeq(nextSeq: unknown): number | null {
+  const next = finite(nextSeq);
+  return next !== null && Number.isInteger(next) && next >= 2 ? next - 1 : null;
+}
+
+export function boundRunObservationError(
+  error: string | null | undefined,
+): RunObservation['error'] {
+  const text = typeof error === 'string' ? error.trim() : '';
+  if (!text) return null;
+  const code = /^([A-Za-z][A-Za-z0-9_]{0,79}):/.exec(text)?.[1] ?? 'run_error';
+  return {
+    code,
+    message:
+      text.length > RUN_OBSERVATION_ERROR_MESSAGE_MAX_CHARS
+        ? text.slice(0, RUN_OBSERVATION_ERROR_MESSAGE_MAX_CHARS)
+        : text,
+  };
+}
+
+function projectNode(node: PlayRunStreamNodeState): RunObservationNode {
+  return {
+    nodeId: node.nodeId,
+    status: node.status,
+    startedAt: finite(node.startedAt),
+    completedAt: finite(node.completedAt),
+    total: finite(node.progress?.total),
+    settled: nonNegative(node.progress?.completed),
+    failed: nonNegative(node.progress?.failed),
+    ...(node.stoppedBy ? { stoppedBy: node.stoppedBy } : {}),
+  };
+}
+
+/**
+ * Project a stored Convex Run Snapshot into its Run Observation. Pure: the
+ * same stored document always yields the same observation.
+ */
+export function projectRunObservation(
+  run: RunObservationSourceRun,
+): RunObservation {
+  const snapshot = buildPlayRunStatusSnapshot({ run });
+  const status = runObservationStatus(
+    snapshot.status === 'unknown' ? run.status : snapshot.status,
+  );
+  const waitKind = run.waitKind ?? run.wait?.kind ?? null;
+  const rowOutcomes = snapshot.rowOutcomes ?? null;
+  const activity = snapshot.activity ?? null;
+  const terminal = isTerminalPlayRunLifecycleStatus(status);
+  return {
+    runId: run.workflowId,
+    playName: run.name,
+    trigger: runLaunchTrigger(run),
+    status,
+    ledgerSeq: ledgerSeqFromNextSeq(run.ledgerNextSeq),
+    updatedAt: Math.max(run.updatedAt, finite(snapshot.updatedAt) ?? 0),
+    source: 'snapshot',
+    createdAt: run.createdAt,
+    startedAt: finite(snapshot.startedAt) ?? finite(run.startedAt),
+    finishedAt: finite(snapshot.finishedAt) ?? finite(run.finishedAt),
+    wait:
+      status === 'waiting'
+        ? {
+            kind: waitKind ?? 'unknown',
+            deadlineAt: finite(run.waitUntil),
+          }
+        : null,
+    activity: activity
+      ? {
+          target: activity.observation.target.kind,
+          state: activity.observation.state.kind,
+          label: activity.summary,
+        }
+      : null,
+    error:
+      status === 'failed' || status === 'cancelled' || status === 'unknown'
+        ? boundRunObservationError(run.error)
+        : null,
+    rows:
+      terminal && rowOutcomes
+        ? {
+            total: rowOutcomes.totalRows,
+            ok: rowOutcomes.completedRows,
+            failed: rowOutcomes.failedRows,
+            pending: Math.max(
+              0,
+              rowOutcomes.totalRows -
+                rowOutcomes.completedRows -
+                rowOutcomes.failedRows -
+                (rowOutcomes.supersededRows ?? 0),
+            ),
+          }
+        : null,
+    nodes: snapshot.nodeStates.map(projectNode),
+    logs: { total: nonNegative(snapshot.totalLogCount) },
+  };
+}
+
+/**
+ * The admission bridge's answer for a run Runtime Postgres admitted but whose
+ * `run.created` fact Convex has not projected yet. Scalar only: it makes no
+ * node, row or log claims, and any snapshot observation supersedes it.
+ */
+export function projectAdmissionRunObservation(run: {
+  workflowId: string;
+  name: string;
+  status: string;
+  createdAt: number | null;
+  startedAt: number | null;
+  updatedAt?: number | null;
+  waitKind?: string | null;
+  waitUntil?: number | null;
+}): RunObservation {
+  // The bridge proves ADMISSION, not execution: the scheduler row is stamped
+  // `running` the moment the run is admitted (the start route's 202 says the
+  // same), before any work has been claimed. Until the snapshot projects the
+  // run's own facts, an active run reads `queued`; the snapshot then moves it
+  // forward, and the monotonic merge never lets it step back.
+  const reported = runObservationStatus(run.status);
+  const status: RunObservationStatus =
+    reported === 'running' ? 'queued' : reported;
+  const createdAt = finite(run.createdAt) ?? 0;
+  return {
+    runId: run.workflowId,
+    playName: run.name,
+    trigger: 'unknown',
+    status,
+    ledgerSeq: null,
+    updatedAt: finite(run.updatedAt) ?? createdAt,
+    source: 'admission',
+    createdAt,
+    startedAt: finite(run.startedAt),
+    finishedAt: null,
+    wait:
+      status === 'waiting'
+        ? { kind: run.waitKind ?? 'unknown', deadlineAt: finite(run.waitUntil) }
+        : null,
+    activity: null,
+    error: null,
+    rows: null,
+    nodes: [],
+    logs: { total: 0 },
+  };
+}
+
+export function isRunObservationActive(status: RunObservationStatus): boolean {
+  return isActivePlayRunLifecycleStatus(status);
+}
+
+export function isRunObservationTerminal(
+  status: RunObservationStatus,
+): boolean {
+  return isTerminalPlayRunLifecycleStatus(status);
+}
+
+/**
+ * Positive when `a` is fresher than `b`. Ordered by the ledger seq (a row
+ * without one sorts before any row with one), then by `updatedAt`, which
+ * orders same-seq snapshot rewrites and every pre-cursor row.
+ */
+export function compareRunObservationFreshness(
+  a: Pick<RunObservation, 'ledgerSeq' | 'updatedAt'>,
+  b: Pick<RunObservation, 'ledgerSeq' | 'updatedAt'>,
+): number {
+  const seqA = a.ledgerSeq ?? -1;
+  const seqB = b.ledgerSeq ?? -1;
+  if (seqA !== seqB) return seqA - seqB;
+  return a.updatedAt - b.updatedAt;
+}
+
+/**
+ * The monotonic merge every reader applies (§3.4). Returns the observation to
+ * hold. Rules, in order:
+ *  1. A terminal status never regresses to a non-terminal one, and a
+ *     terminal answer always replaces a non-terminal one.
+ *  2. A snapshot always replaces an admission answer, and an admission answer
+ *     never replaces a snapshot.
+ *  3. Otherwise the fresher observation wins; a tie keeps the held one, so a
+ *     repeated answer is referentially stable.
+ *  4. The status never steps back a lifecycle rank (queued → running/waiting
+ *     → terminal): the winner's data with the held status, see
+ *     `runObservationStatusRank`.
+ */
+export function mergeRunObservation(
+  held: RunObservation | null | undefined,
+  incoming: RunObservation,
+): RunObservation {
+  if (!held) return incoming;
+  const heldTerminal = isRunObservationTerminal(held.status);
+  const incomingTerminal = isRunObservationTerminal(incoming.status);
+  if (heldTerminal && !incomingTerminal) return held;
+  // A terminal answer always ends a run the store still holds as active,
+  // whatever its seq, clock or source: an answer that ran AHEAD of it (a
+  // stale per-instance read, an admission row with a later clock) must not
+  // leave a finished run Queued/Running with Stop until a reload.
+  if (incomingTerminal && !heldTerminal) return incoming;
+  const winner =
+    held.source !== incoming.source
+      ? incoming.source === 'snapshot'
+        ? incoming
+        : held
+      : compareRunObservationFreshness(incoming, held) > 0
+        ? incoming
+        : held;
+  return winner === held ? held : liftRunObservationStatus(held, winner);
+}
+
+/**
+ * Where a status sits in a run's lifecycle: `queued` → `running`/`waiting` →
+ * terminal. A run only ever moves forward through these ranks; `running` and
+ * `waiting` share one (a run may wait and resume any number of times).
+ * `unknown` ranks below everything, so it never displaces a known status.
+ */
+export function runObservationStatusRank(status: RunObservationStatus): number {
+  if (isRunObservationTerminal(status)) return 3;
+  if (status === 'running' || status === 'waiting') return 2;
+  if (status === 'queued') return 1;
+  return 0;
+}
+
+/**
+ * Rule 4 of the merge: a run's lifecycle never steps back. The fresher
+ * observation's data wins, but a status from an EARLIER rank than the one
+ * already held is a lagging source, not news — the Convex projection still
+ * saying `queued` after the runtime reported the run started, say. The held
+ * status stands until a later rank replaces it, so a surface never shows
+ * Running → Queued → Completed.
+ */
+function liftRunObservationStatus(
+  held: RunObservation,
+  winner: RunObservation,
+): RunObservation {
+  if (
+    runObservationStatusRank(winner.status) >=
+    runObservationStatusRank(held.status)
+  ) {
+    return winner;
+  }
+  return {
+    ...winner,
+    status: held.status,
+    startedAt: winner.startedAt ?? held.startedAt,
+    wait: held.status === 'waiting' ? (winner.wait ?? held.wait) : null,
+  };
+}
+
+/** The same rules for a head row (no seq: `updatedAt` orders it). */
+export function mergeRunObservationHeadEntry(
+  held: RunObservationHeadEntry | null | undefined,
+  incoming: RunObservationHeadEntry,
+): RunObservationHeadEntry {
+  if (!held) return incoming;
+  const heldTerminal = isRunObservationTerminal(held.status);
+  const incomingTerminal = isRunObservationTerminal(incoming.status);
+  if (heldTerminal && !incomingTerminal) return held;
+  // Terminal always wins over an active row (see `mergeRunObservation`).
+  if (incomingTerminal && !heldTerminal) return incoming;
+  if (incoming.updatedAt <= held.updatedAt) return held;
+  // Never a step back in the lifecycle (see `liftRunObservationStatus`).
+  return runObservationStatusRank(incoming.status) <
+    runObservationStatusRank(held.status)
+    ? { ...incoming, status: held.status }
+    : incoming;
+}