deepline 0.3.258 → 0.3.260

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts

@@ -412,6 +412,8 @@ export function createPostgresSchedulerProgressReporter(input: {
 
   let liveNodeProgress: PlayVisualNodeProgressMap = {};
   let liveNodeProgressDirty = false;
+  /** Set once this reporter finalizes a FAILED result; see `finalize`. */
+  let nodeProgressSealed = false;
   let liveLogs: string[] = [];
   let liveLogContexts: Array<PlayLogContext | null> = [];
   let liveLogsTotalCount = input.initialLiveLogTotalCount ?? 0;
@@ -525,7 +527,7 @@ export function createPostgresSchedulerProgressReporter(input: {
               await publishStatusUpdate({
                 playId: launch.runId,
                 status: request.status,
-                liveNodeProgress: nextProgress,
+                liveNodeProgress: nodeProgressSealed ? {} : nextProgress,
                 liveLogs: nextLogs,
                 liveLogContexts: nextLogContexts,
                 liveLogTotalCount: nextLogTotalCount,
@@ -1453,19 +1455,28 @@ export function createPostgresSchedulerProgressReporter(input: {
       };
       workCountersDirty = true;
     }
-    // Mark started steps terminal with real timestamps. Static branches that
-    // never started remain non-executed; suspended runs stay in-flight.
+    // A failed result's terminal is already decided: every caller that
+    // finalizes one does so after (or while) the scheduler commits
+    // `run.failed`, and the ledger freezes a terminal run's steps. Its step
+    // facts from here are projections of this reporter's own partial view — a
+    // parked worker's fold saw page-sized map totals and never saw the
+    // runner's failures — so they used to land after `run.failed` claiming
+    // docflow nodes were "running" and a 50,000-row map had "completed" 2,000.
+    // The settled counts that ARE true travel in the terminal outbox row
+    // (`buildFailedTerminalSettledProgressEvents`). Logs, work counters,
+    // dataset and activity facts still flush.
     if (result?.status === 'failed') {
-      stepLifecycle.markStartedFailed();
+      nodeProgressSealed = true;
     } else if (result?.status === 'completed') {
+      // Mark started steps terminal with real timestamps. Static branches
+      // that never started remain non-executed; suspended runs stay
+      // in-flight.
       stepLifecycle.markAllTerminal();
-    }
-    // Terminal runs derive their per-map step progress from the durable data
-    // plane, overriding any stale/dropped event-fed counters. Checkpoint
-    // frames first (cheap, in-result), then sheet-table counts as the
-    // reconciliation of record. Suspended runs stay mid-flight, so leave their
-    // live counters untouched.
-    if (result?.status === 'completed' || result?.status === 'failed') {
+      // Completed runs derive their per-map step progress from the durable
+      // data plane, overriding any stale/dropped event-fed counters.
+      // Checkpoint frames first (cheap, in-result), then sheet-table counts as
+      // the reconciliation of record. Suspended runs stay mid-flight, so
+      // leave their live counters untouched.
       reconcileTerminalMapProgressFromCheckpoint(result);
       reconcileTerminalMapProgressFromSheetOutcomes();
     }
@@ -1502,6 +1513,62 @@ export function createPostgresSchedulerProgressReporter(input: {
   };
 }
 
+/**
+ * The settled row counts of a FAILED run's maps, as ledger facts that travel
+ * in the `run.failed` outbox row ahead of the terminal fact.
+ *
+ * A failed run's final counts used to arrive only through the worker's
+ * post-terminal progress finalize, after `run.failed` had made the ledger's
+ * steps immutable, so they were dropped: the ledger kept the last live tick
+ * (1,241 completed) while the runtime sheet held 1,260 completed and 38 failed
+ * rows. These facts carry counts only — the runtime sheet's settled split
+ * (ADR 0016 rule 3) — and nothing else a stopped run cannot vouch for:
+ *
+ *   - no total: the sheet holds the rows materialized so far (a map admits its
+ *     input in pages), not the map's input count, so it would understate it;
+ *   - no completion stamp or status change: the map did not complete; the
+ *     terminal fact that follows settles it as failed;
+ *   - no progress line: the ledger rewrites the settled line from its counts.
+ */
+export function buildFailedTerminalSettledProgressEvents(input: {
+  launch: Pick<PlaySchedulerSubmitInput, 'runId' | 'contractSnapshot'>;
+  outcomes: ReadonlyArray<{
+    tableNamespace: string;
+    completed: number;
+    failed: number;
+  }>;
+  occurredAt: number;
+}): PlayRunLedgerEvent[] {
+  const lookup = buildMapNodeLookup(input.launch.contractSnapshot);
+  const events: PlayRunLedgerEvent[] = [];
+  for (const outcome of input.outcomes) {
+    const nodeRef = resolveDatasetNodeRefFromTableNamespace(
+      lookup,
+      outcome.tableNamespace,
+    );
+    if (!nodeRef) continue;
+    const completed = Math.max(0, Math.floor(outcome.completed) || 0);
+    const failed = Math.max(0, Math.floor(outcome.failed) || 0);
+    if (completed <= 0 && failed <= 0) continue;
+    events.push({
+      type: 'step.progress',
+      runId: input.launch.runId,
+      source: 'worker',
+      occurredAt: input.occurredAt,
+      stepId: nodeRef.nodeId,
+      status: 'running',
+      progress: {
+        completed,
+        completedRows: completed,
+        failed,
+        artifactTableNamespace: nodeRef.artifactTableNamespace,
+        updatedAt: input.occurredAt,
+      },
+    });
+  }
+  return events;
+}
+
 export const createDirectWorkerProgressReporter = (input: {
   launch: PlaySchedulerSubmitInput;
   producerAttempt?: number;