deepline 0.3.89 → 0.3.90

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts

@@ -0,0 +1,1255 @@
+/**
+ * Live step/progress ledger reporting for Node runtime workers.
+ *
+ * Projection of the runner liveNodeProgress event stream: the runner's event stream is
+ * folded into a PlayVisualNodeProgressMap and posted as `update_run_status`
+ * patches through the app runtime API. The server-side run ledger
+ * (buildPlayRunLedgerEventsFromStatusPatch in run-ledger.ts) converts those
+ * patches into step.started/step.progress/step.completed events — nothing is
+ * reimplemented here beyond producing the same progress patches.
+ *
+ * PERF: the runner event path is hot. Nothing here awaits in that path:
+ * observations merge into the latest state while at most one upload is active.
+ * Runtime Sheet row durability belongs to the context-level row writer.
+ */
+import { PlayStepLifecycleTracker } from '../step-lifecycle-tracker';
+import {
+  updateRunStatusViaAppRuntime,
+  type RuntimeStatusUpdate,
+} from '../app-runtime-api';
+import type { PlayVisualNodeProgressMap } from '../worker-api-types';
+import type { PlayExecutionEvent } from '../ctx-types';
+import type {
+  PlayRunnerLogEvent,
+  PlayRunnerResult,
+} from '../protocol';
+import type { PlaySchedulerSubmitInput } from '../scheduler-backend';
+import type { DocflowObservationRecord } from '../docflow-observation';
+import {
+  getTopLevelPipelineSubsteps,
+  type PlayStaticPipeline,
+  type PlayStaticSubstep,
+} from '../../plays/static-pipeline';
+import { getStaticSubstepNodeId } from '../step-progress';
+import { runtimeApiBaseUrlFromLaunch } from './runtime-api-origin';
+
+// Patterns mirrored from runner events — they parse
+// the same Node play-runner log stream.
+const MAP_START_LOG_PATTERN =
+  /Starting map over (\d+) items .* \((?:table namespace|key): ([^;)]+)(?:; (\d+) (?:already satisfied|duplicate keys skipped); (\d+) pending)?\)/i;
+const PERF_MAP_SETUP_LOG_PATTERN =
+  /^\[perf\] runtime map setup table=([^\s]+) rows=(\d+)\b/i;
+const PLAY_TOOL_BATCH_LOG_PATTERN =
+  /(?:^|\]\s*)Executing tool batch\s+([A-Za-z0-9_:-]+):\s+(\d+)\s+calls\b/i;
+const CTX_LOG_TIMESTAMP_PREFIX_RE = /^\[\d{4}-\d{2}-\d{2}T[\d:.]+Z\]\s+/;
+const LIVE_LOG_TAIL_LIMIT = 1_000;
+
+function docflowInvocationSequence(invocationId?: string): number {
+  if (!invocationId) return -1;
+  const sequence = Number(invocationId.split(':').at(-1));
+  return Number.isSafeInteger(sequence) && sequence >= 0 ? sequence : -1;
+}
+
+// PERF: progress flushes
+// here throttle purely on the adaptive interval. The bypass would chain ~800
+// HTTP posts on a 200k-row run and finalize() awaits that chain. Worst case
+// the live ledger lags by one interval; the forced finalize flush lands final
+// truth.
+function getAdaptiveRunEventIntervalMs(totalRows: number): number {
+  if (totalRows >= 1_000_000) return 30_000;
+  if (totalRows >= 100_000) return 15_000;
+  if (totalRows >= 10_000) return 8_000;
+  return 1_500;
+}
+
+function staticPipelineFromContractSnapshot(
+  value: unknown,
+): PlayStaticPipeline | null {
+  if (!value || typeof value !== 'object') return null;
+  const staticPipeline = (value as { staticPipeline?: unknown }).staticPipeline;
+  if (!staticPipeline || typeof staticPipeline !== 'object') return null;
+  return staticPipeline as PlayStaticPipeline;
+}
+
+/**
+ * Build ordered list of all static pipeline node IDs with their types.
+ * Used to track step-level timing for the Gantt waterfall.
+ */
+export function buildOrderedNodeList(
+  contractSnapshot: unknown,
+): Array<{ nodeId: string; type: PlayStaticSubstep['type'] }> {
+  const substeps = getTopLevelPipelineSubsteps(
+    staticPipelineFromContractSnapshot(contractSnapshot),
+  );
+  return substeps.map((substep, index) => ({
+    nodeId: getStaticSubstepNodeId(substep, index),
+    type: substep.type,
+  }));
+}
+
+/** Map logical dataset namespaces to their static-pipeline node IDs. */
+type StaticDatasetNodeRef = {
+  nodeId: string;
+  logicalNamespace: string;
+};
+
+type RuntimeDatasetNodeRef = StaticDatasetNodeRef & {
+  artifactTableNamespace: string;
+};
+
+function buildMapNodeLookup(
+  contractSnapshot: unknown,
+): Map<string, StaticDatasetNodeRef> {
+  const substeps = getTopLevelPipelineSubsteps(
+    staticPipelineFromContractSnapshot(contractSnapshot),
+  );
+  const lookup = new Map<string, StaticDatasetNodeRef>();
+  for (let index = 0; index < substeps.length; index += 1) {
+    const substep = substeps[index]!;
+    if (substep.type !== 'dataset') continue;
+    const logicalNamespace =
+      substep.tableNamespace?.trim() || substep.field.trim();
+    if (!logicalNamespace) continue;
+    lookup.set(logicalNamespace, {
+      nodeId: getStaticSubstepNodeId(substep, index),
+      logicalNamespace,
+    });
+  }
+  return lookup;
+}
+
+export function buildMapNodeIdLookup(
+  contractSnapshot: unknown,
+): Map<string, string> {
+  return new Map(
+    [...buildMapNodeLookup(contractSnapshot).entries()].map(
+      ([namespace, ref]) => [namespace, ref.nodeId],
+    ),
+  );
+}
+
+function resolveDatasetNodeRefFromTableNamespace(
+  lookup: ReadonlyMap<string, StaticDatasetNodeRef>,
+  physicalNamespace: string,
+): RuntimeDatasetNodeRef | null {
+  const trimmed = physicalNamespace.trim();
+  if (!trimmed) {
+    return null;
+  }
+  for (const [logicalNamespace, ref] of lookup.entries()) {
+    if (
+      trimmed === logicalNamespace ||
+      trimmed.startsWith(`${logicalNamespace}_`)
+    ) {
+      return {
+        ...ref,
+        artifactTableNamespace: trimmed,
+      };
+    }
+  }
+  return null;
+}
+
+function rowsProcessedMessage(
+  completed: number,
+  total: number | undefined,
+): string | undefined {
+  return total !== undefined && total > 0
+    ? `${Math.max(0, completed).toLocaleString()} / ${total.toLocaleString()} rows processed`
+    : undefined;
+}
+
+function callsInFlightMessage(input: {
+  activeRows: number;
+  completedRows: number;
+  totalRows: number;
+}): string {
+  const active = input.activeRows.toLocaleString();
+  return input.totalRows > 0
+    ? `${active} calls in flight · ${input.completedRows.toLocaleString()} / ${input.totalRows.toLocaleString()} rows processed`
+    : `${active} calls in flight`;
+}
+
+export type PostgresSchedulerProgressReporter = {
+  /**
+   * Consume one raw runner event. Synchronous and cheap on purpose: this is
+   * called for EVERY runner event before any stdout sampling.
+   */
+  onRunnerEvent(event: unknown): void;
+  /**
+   * Fold durable docflow observations (read from the scheduler observations
+   * table at worker finalize, ADR 0016 rule 3) into the live node progress so a
+   * completed absurd run persists docflow node I/O even when the live
+   * `docflow.node.*` events were dropped mid-run. Terminal reconciliation only:
+   * a durable record supersedes the event-fed nodeIo for its node when it is at
+   * least as recent (attempt, then invocation sequence).
+   */
+  applyDurableObservations(observations: DocflowObservationRecord[]): void;
+  /**
+   * Record the data-plane row-outcome counts read from the runtime sheet
+   * tables at worker finalize (ADR 0016 rule 3). Sheets exist regardless of
+   * runner mode (a detached Daytona push run streams no events and can omit
+   * its terminal checkpoint entirely), so these counts are the reconciliation
+   * of record: finalize applies them AFTER the checkpoint mapFrames, and they
+   * win whenever the two disagree or frames are missing. Works from an empty
+   * liveNodeProgress — a zero-event run still persists real map step progress.
+   */
+  applyTerminalSheetOutcomes(
+    outcomes: Array<{
+      tableNamespace: string;
+      completed: number;
+      failed: number;
+      total: number;
+    }>,
+  ): void;
+  /**
+   * Apply terminal step transitions from the runner result and await the
+   * final ledger flush. The inline executor returns the result instead of
+   * emitting a `result` event, so it is passed in here.
+   */
+  finalize(
+    result?: PlayRunnerResult | null,
+    options?: {
+      status?: 'running' | 'completed' | 'failed';
+      /** Propagate a running-state transport failure to the caller. */
+      required?: boolean;
+      replayAllTerminalLogs?: boolean;
+      terminalLogProducerAttempt?: number;
+      skipTerminalLogReplay?: boolean;
+    },
+  ): Promise<void>;
+};
+
+export type PostgresSchedulerDurableLogMode =
+  | 'progress_reporter'
+  | 'direct_worker';
+
+function shouldPublishDurableLogEvent(
+  mode: PostgresSchedulerDurableLogMode,
+  event:
+    | Pick<PlayRunnerLogEvent, 'source' | 'line'>
+    | { source?: unknown; line?: unknown },
+): boolean {
+  // Daytona host lifecycle logs use source "worker" (sandbox
+  // create/upload/execute). Keep those in worker logs while still storing
+  // play-authored ctx.log events in the durable Run Log Stream. The one
+  // exception is the structured performance marker: it is bounded (one
+  // terminal phase observation per runner phase), run-scoped, and is the
+  // canonical black-box regression signal for dispatcher throughput. Do not
+  // broaden this to arbitrary worker output.
+  return (
+    mode !== 'direct_worker' ||
+    event.source !== 'worker' ||
+    (typeof event.line === 'string' && event.line.startsWith('[perf-trace] '))
+  );
+}
+
+function shouldPublishTerminalResultLog(
+  mode: PostgresSchedulerDurableLogMode,
+  line: string,
+): boolean {
+  return mode !== 'direct_worker' || CTX_LOG_TIMESTAMP_PREFIX_RE.test(line);
+}
+
+type TerminalLogReplayPayload = NonNullable<
+  RuntimeStatusUpdate['terminalLogReplay']
+>;
+
+function terminalReplayLinesForMode(input: {
+  resultLogs: readonly string[];
+  durableLogMode: PostgresSchedulerDurableLogMode;
+}): string[] {
+  return input.resultLogs
+    .map((line) => line.trim())
+    .filter(
+      (line) =>
+        line.length > 0 &&
+        shouldPublishTerminalResultLog(input.durableLogMode, line),
+    );
+}
+
+function terminalReplayPayloadForMode(input: {
+  resultLogs: readonly string[];
+  durableLogMode: PostgresSchedulerDurableLogMode;
+  liveLogs: readonly string[];
+  liveLogsTotalCount: number;
+  replayAllTerminalLogs: boolean;
+  producerAttempt?: number;
+}): TerminalLogReplayPayload | null {
+  let lines = terminalReplayLinesForMode(input);
+  // Detached recovery is a complete attempt-local buffer. Its total count
+  // gives the replay an offset of zero on that attempt's positional channel;
+  // producerAttempt fences stale deliveries while the ledger absorbs only an
+  // exact retransmission from that same attempt.
+  let totalCount = lines.length;
+  if (
+    input.durableLogMode === 'direct_worker' &&
+    !input.replayAllTerminalLogs
+  ) {
+    const observedCounts = new Map<string, number>();
+    for (const line of input.liveLogs) {
+      const key = line.trim().replace(CTX_LOG_TIMESTAMP_PREFIX_RE, '');
+      if (!key) continue;
+      observedCounts.set(key, (observedCounts.get(key) ?? 0) + 1);
+    }
+    const boundedLines: string[] = [];
+    for (const line of lines) {
+      const key = line.trim().replace(CTX_LOG_TIMESTAMP_PREFIX_RE, '');
+      const remaining = observedCounts.get(key) ?? 0;
+      if (remaining > 0) {
+        observedCounts.set(key, remaining - 1);
+        continue;
+      }
+      boundedLines.push(line);
+    }
+    lines = boundedLines;
+    totalCount = input.liveLogsTotalCount + lines.length;
+  }
+  if (lines.length === 0) {
+    return null;
+  }
+  return {
+    lines,
+    ...(input.replayAllTerminalLogs ? { dedupeMode: 'exact' as const } : {}),
+    ...(typeof input.producerAttempt === 'number'
+      ? { producerAttempt: input.producerAttempt }
+      : {}),
+    totalCount,
+  };
+}
+
+export function createPostgresSchedulerProgressReporter(input: {
+  launch: PlaySchedulerSubmitInput;
+  durableLogMode?: PostgresSchedulerDurableLogMode;
+  producerAttempt?: number;
+  isCurrentAttempt?: () => boolean | Promise<boolean>;
+  /** Durable log position before this reporter's first event. Push runners own
+   * this cursor so successive batches may land on different gateway Machines. */
+  initialLiveLogTotalCount?: number;
+}): PostgresSchedulerProgressReporter {
+  const launch = input.launch;
+  const durableLogMode = input.durableLogMode ?? 'progress_reporter';
+  const producerAttempt = input.producerAttempt;
+  const isCurrentAttempt = input.isCurrentAttempt ?? (() => true);
+  const orderedNodes = buildOrderedNodeList(launch.contractSnapshot);
+  const mapNodeLookup = buildMapNodeLookup(launch.contractSnapshot);
+  const runtimeApi = {
+    baseUrl: runtimeApiBaseUrlFromLaunch(launch),
+    executorToken: launch.executorToken,
+    vercelProtectionBypassToken: launch.vercelProtectionBypassToken ?? null,
+  };
+
+  let liveNodeProgress: PlayVisualNodeProgressMap = {};
+  let liveNodeProgressDirty = false;
+  let liveLogs: string[] = [];
+  let liveLogsTotalCount = input.initialLiveLogTotalCount ?? 0;
+  let liveLogsDirty = false;
+  let lastFlushAt = 0;
+  let totalRows = 0;
+  let activeMapNodeId: string | null = null;
+  let pendingDatasetLifecycleEvents: Array<
+    Extract<PlayExecutionEvent, { type: 'dataset.lifecycle' }>
+  > = [];
+  const pendingActivityObservations = new Map<
+    string,
+    Extract<PlayExecutionEvent, { type: 'activity.observed' }>['observation']
+  >();
+
+  type ProgressFlushRequest = {
+    status: 'running' | 'completed' | 'failed';
+    required: boolean;
+    terminalLogReplay: TerminalLogReplayPayload | null;
+    waiters: Array<{
+      resolve: () => void;
+      reject: (error: unknown) => void;
+    }>;
+  };
+
+  let activeProgressUpload: Promise<void> | null = null;
+  let pendingProgressUpload: ProgressFlushRequest | null = null;
+
+  const mergeProgressFlushRequest = (
+    current: ProgressFlushRequest | null,
+    next: Omit<ProgressFlushRequest, 'waiters'>,
+    waiter: ProgressFlushRequest['waiters'][number],
+  ): ProgressFlushRequest => ({
+    status:
+      next.status !== 'running'
+        ? next.status
+        : (current?.status ?? next.status),
+    required: next.required || current?.required === true,
+    terminalLogReplay:
+      next.terminalLogReplay ?? current?.terminalLogReplay ?? null,
+    waiters: [...(current?.waiters ?? []), waiter],
+  });
+
+  const requeueProgressEvents = (input: {
+    datasetLifecycleEvents: typeof pendingDatasetLifecycleEvents;
+    activityObservations: Array<
+      Extract<PlayExecutionEvent, { type: 'activity.observed' }>['observation']
+    >;
+  }): void => {
+    pendingDatasetLifecycleEvents = [
+      ...input.datasetLifecycleEvents,
+      ...pendingDatasetLifecycleEvents,
+    ];
+    for (const observation of input.activityObservations) {
+      const newer = pendingActivityObservations.get(observation.activityId);
+      if (!newer || newer.observedAt < observation.observedAt) {
+        pendingActivityObservations.set(observation.activityId, observation);
+      }
+    }
+  };
+
+  const startProgressUploadPump = (): void => {
+    if (activeProgressUpload || !pendingProgressUpload) return;
+
+    // Start on the next microtask so an attempt-fence change made by the
+    // scheduler in the same turn can invalidate queued observations before
+    // any transport side effect begins.
+    activeProgressUpload = Promise.resolve()
+      .then(async () => {
+        while (pendingProgressUpload) {
+          const request = pendingProgressUpload;
+          pendingProgressUpload = null;
+
+          const nextProgress = { ...liveNodeProgress };
+          const nextLogs = liveLogs.slice();
+          const nextLogTotalCount = liveLogsTotalCount;
+          const nextDatasetLifecycleEvents = pendingDatasetLifecycleEvents;
+          pendingDatasetLifecycleEvents = [];
+          const nextActivityObservations = [
+            ...pendingActivityObservations.values(),
+          ];
+          pendingActivityObservations.clear();
+          liveNodeProgressDirty = false;
+          liveLogsDirty = false;
+
+          try {
+            if (await isCurrentAttempt()) {
+              await updateRunStatusViaAppRuntime(runtimeApi, {
+                playId: launch.runId,
+                status: request.status,
+                liveNodeProgress: nextProgress,
+                liveLogs: nextLogs,
+                liveLogTotalCount: nextLogTotalCount,
+                liveLogProducerAttempt: producerAttempt,
+                datasetLifecycleEvents: nextDatasetLifecycleEvents,
+                activityObservations: nextActivityObservations,
+                terminalLogReplay: request.terminalLogReplay,
+                lastCheckpointAt: Date.now(),
+              });
+            }
+            request.waiters.forEach(({ resolve }) => resolve());
+          } catch (error) {
+            requeueProgressEvents({
+              datasetLifecycleEvents: nextDatasetLifecycleEvents,
+              activityObservations: nextActivityObservations,
+            });
+            console.warn('[postgres-scheduler.progress_flush_failed]', {
+              runId: launch.runId,
+              error: error instanceof Error ? error.message : String(error),
+            });
+            if (request.required) {
+              request.waiters.forEach(({ reject }) => reject(error));
+            } else {
+              request.waiters.forEach(({ resolve }) => resolve());
+            }
+          }
+        }
+      })
+      .finally(() => {
+        activeProgressUpload = null;
+        if (pendingProgressUpload) startProgressUploadPump();
+      });
+  };
+
+  const flush = (input?: {
+    force?: boolean;
+    status?: 'running' | 'completed' | 'failed';
+    required?: boolean;
+    terminalLogReplay?: TerminalLogReplayPayload | null;
+  }): Promise<void> => {
+    const force = input?.force === true;
+    const status = input?.status ?? 'running';
+    const required = input?.required === true;
+    if (
+      !force &&
+      !liveNodeProgressDirty &&
+      !liveLogsDirty &&
+      pendingDatasetLifecycleEvents.length === 0 &&
+      pendingActivityObservations.size === 0
+    ) {
+      return activeProgressUpload ?? Promise.resolve();
+    }
+    const now = Date.now();
+    if (
+      !force &&
+      now - lastFlushAt < getAdaptiveRunEventIntervalMs(totalRows)
+    ) {
+      return activeProgressUpload ?? Promise.resolve();
+    }
+    lastFlushAt = now;
+    const requested = new Promise<void>((resolve, reject) => {
+      pendingProgressUpload = mergeProgressFlushRequest(
+        pendingProgressUpload,
+        {
+          status,
+          required,
+          terminalLogReplay: input?.terminalLogReplay ?? null,
+        },
+        { resolve, reject },
+      );
+    });
+    startProgressUploadPump();
+    return requested;
+  };
+
+  const scheduleFlush = (force = false) => {
+    // A best-effort event flush can coalesce with a required terminal flush.
+    // The upload pump logs failure and rejects the terminal caller; consume
+    // only this detached observer's rejection to avoid an unhandled promise.
+    void flush({ force }).catch(() => undefined);
+  };
+
+  const appendLiveLog = (line: string) => {
+    const trimmed = line.trim();
+    if (!trimmed) {
+      return;
+    }
+    liveLogsTotalCount += 1;
+    liveLogs = [...liveLogs, trimmed];
+    if (liveLogs.length > LIVE_LOG_TAIL_LIMIT) {
+      liveLogs = liveLogs.slice(liveLogs.length - LIVE_LOG_TAIL_LIMIT);
+    }
+    liveLogsDirty = true;
+  };
+
+  const setNodeProgress = (
+    nodeId: string,
+    patch: Partial<PlayVisualNodeProgressMap[string]>,
+  ) => {
+    const current = liveNodeProgress[nodeId] ?? {};
+    const nextCompleted = Number(patch.completed ?? current.completed ?? 0);
+    const nextTotal = Number(patch.total ?? current.total ?? 0);
+    const clearCompletedAt =
+      patch.completedAt === undefined &&
+      Number.isFinite(nextCompleted) &&
+      Number.isFinite(nextTotal) &&
+      nextTotal > 0 &&
+      nextCompleted < nextTotal;
+    liveNodeProgress = {
+      ...liveNodeProgress,
+      [nodeId]: {
+        ...(current.completed !== undefined
+          ? { completed: current.completed }
+          : {}),
+        ...(current.total !== undefined ? { total: current.total } : {}),
+        ...(current.failed !== undefined ? { failed: current.failed } : {}),
+        ...(current.startedRows !== undefined
+          ? { startedRows: current.startedRows }
+          : {}),
+        ...(current.activeRows !== undefined
+          ? { activeRows: current.activeRows }
+          : {}),
+        ...(current.waitingRows !== undefined
+          ? { waitingRows: current.waitingRows }
+          : {}),
+        ...(current.completedRows !== undefined
+          ? { completedRows: current.completedRows }
+          : {}),
+        ...(current.message !== undefined ? { message: current.message } : {}),
+        ...(current.artifactTableNamespace !== undefined
+          ? { artifactTableNamespace: current.artifactTableNamespace }
+          : {}),
+        ...(current.startedAt !== undefined
+          ? { startedAt: current.startedAt }
+          : {}),
+        ...(current.completedAt !== undefined && !clearCompletedAt
+          ? { completedAt: current.completedAt }
+          : {}),
+        ...(current.nodeIo !== undefined ? { nodeIo: current.nodeIo } : {}),
+        ...(patch.completed !== undefined
+          ? { completed: patch.completed }
+          : {}),
+        ...(patch.total !== undefined ? { total: patch.total } : {}),
+        ...(patch.failed !== undefined ? { failed: patch.failed } : {}),
+        ...(patch.startedRows !== undefined
+          ? { startedRows: patch.startedRows }
+          : {}),
+        ...(patch.activeRows !== undefined
+          ? { activeRows: patch.activeRows }
+          : {}),
+        ...(patch.waitingRows !== undefined
+          ? { waitingRows: patch.waitingRows }
+          : {}),
+        ...(patch.completedRows !== undefined
+          ? { completedRows: patch.completedRows }
+          : {}),
+        ...(patch.message !== undefined ? { message: patch.message } : {}),
+        ...(patch.artifactTableNamespace !== undefined
+          ? { artifactTableNamespace: patch.artifactTableNamespace }
+          : {}),
+        ...(patch.startedAt !== undefined
+          ? { startedAt: patch.startedAt }
+          : {}),
+        ...(patch.completedAt !== undefined
+          ? { completedAt: patch.completedAt }
+          : {}),
+        ...(patch.nodeIo !== undefined ? { nodeIo: patch.nodeIo } : {}),
+        updatedAt: Date.now(),
+      },
+    };
+    liveNodeProgressDirty = true;
+  };
+
+  // Every lifecycle transition flows through setNodeProgress so the
+  // liveNodeProgress patch stream is canonical.
+  const stepLifecycle = new PlayStepLifecycleTracker(
+    orderedNodes,
+    () => liveNodeProgress,
+    (event) => {
+      const patch: Partial<PlayVisualNodeProgressMap[string]> =
+        event.transition === 'started'
+          ? { startedAt: event.at }
+          : { completedAt: event.at };
+      setNodeProgress(event.nodeId, patch);
+      scheduleFlush();
+    },
+  );
+  stepLifecycle.markPreDatasetStepsStarted();
+
+  const markMapStartedFromLog = (line: string) => {
+    const match = line.match(MAP_START_LOG_PATTERN);
+    if (!match) {
+      return;
+    }
+    const total = Number(match[1] ?? 0);
+    const physicalNamespace = match[2]?.trim() ?? '';
+    const alreadySatisfied = Number(match[3] ?? 0);
+    const nodeRef = resolveDatasetNodeRefFromTableNamespace(
+      mapNodeLookup,
+      physicalNamespace,
+    );
+    if (!nodeRef) {
+      return;
+    }
+    activeMapNodeId = nodeRef.nodeId;
+    if (Number.isFinite(total) && total > totalRows) {
+      totalRows = total;
+    }
+    stepLifecycle.onMapStarted(nodeRef.nodeId);
+    // Streamed maps emit this diagnostic once per source page. The value in
+    // the log is page-local duplicate reuse, not cumulative durable progress.
+    // Preserve the typed commit-event counters already folded for prior pages;
+    // otherwise every new page transiently resets completed to zero and makes
+    // waitingRows contradict the canonical total.
+    const current = liveNodeProgress[nodeRef.nodeId] ?? {};
+    const effectiveTotal = Math.max(
+      Number.isFinite(total) && total > 0 ? total : 0,
+      Number(current.total ?? 0),
+    );
+    const completed = Math.max(
+      0,
+      Number(current.completed ?? 0),
+      Number.isFinite(alreadySatisfied) ? alreadySatisfied : 0,
+    );
+    const failed = Math.max(0, Number(current.failed ?? 0));
+    const settled = completed + failed;
+    setNodeProgress(nodeRef.nodeId, {
+      completed,
+      failed,
+      total: effectiveTotal > 0 ? effectiveTotal : undefined,
+      startedRows: Math.max(Number(current.startedRows ?? 0), settled),
+      activeRows: 0,
+      waitingRows:
+        effectiveTotal > 0 ? Math.max(0, effectiveTotal - settled) : undefined,
+      completedRows: completed,
+      artifactTableNamespace: nodeRef.artifactTableNamespace,
+      startedAt: current.startedAt ?? Date.now(),
+      message: rowsProcessedMessage(completed, effectiveTotal),
+    });
+  };
+
+  const markMapStartedFromPerfLog = (line: string) => {
+    const normalizedLine = line.replace(CTX_LOG_TIMESTAMP_PREFIX_RE, '');
+    const match = normalizedLine.match(PERF_MAP_SETUP_LOG_PATTERN);
+    if (!match) {
+      return;
+    }
+    const nodeRef = resolveDatasetNodeRefFromTableNamespace(
+      mapNodeLookup,
+      match[1] ?? '',
+    );
+    if (!nodeRef) {
+      return;
+    }
+    const total = Number(match[2] ?? 0);
+    activeMapNodeId = nodeRef.nodeId;
+    if (Number.isFinite(total) && total > totalRows) {
+      totalRows = total;
+    }
+    stepLifecycle.onMapStarted(nodeRef.nodeId);
+    const current = liveNodeProgress[nodeRef.nodeId] ?? {};
+    const effectiveTotal = Math.max(
+      Number.isFinite(total) && total > 0 ? total : 0,
+      Number(current.total ?? 0),
+    );
+    const completed = Math.max(0, Number(current.completed ?? 0));
+    const failed = Math.max(0, Number(current.failed ?? 0));
+    const settled = completed + failed;
+    setNodeProgress(nodeRef.nodeId, {
+      completed,
+      failed,
+      total: effectiveTotal > 0 ? effectiveTotal : undefined,
+      startedRows: Math.max(Number(current.startedRows ?? 0), settled),
+      activeRows: 0,
+      waitingRows:
+        effectiveTotal > 0 ? Math.max(0, effectiveTotal - settled) : undefined,
+      completedRows: completed,
+      artifactTableNamespace: nodeRef.artifactTableNamespace,
+      startedAt: current.startedAt ?? Date.now(),
+      message: rowsProcessedMessage(completed, effectiveTotal),
+    });
+  };
+
+  const markToolBatchInFlightFromLog = (line: string) => {
+    if (!activeMapNodeId) {
+      return;
+    }
+    const match = line.match(PLAY_TOOL_BATCH_LOG_PATTERN);
+    if (!match) {
+      return;
+    }
+    const callCount = Number(match[2] ?? 0);
+    if (!Number.isFinite(callCount) || callCount <= 0) {
+      return;
+    }
+    const current = liveNodeProgress[activeMapNodeId] ?? {};
+    const completed = Number(current.completed ?? 0);
+    const failed = Number(current.failed ?? 0);
+    const total = Number(current.total ?? 0);
+    const settled = completed + failed;
+    const activeRows =
+      total > 0 ? Math.min(callCount, Math.max(0, total - settled)) : callCount;
+    const startedRows = Math.max(
+      Number(current.startedRows ?? 0),
+      settled + activeRows,
+    );
+    setNodeProgress(activeMapNodeId, {
+      startedRows: total > 0 ? Math.min(total, startedRows) : startedRows,
+      activeRows,
+      waitingRows:
+        total > 0 ? Math.max(0, total - settled - activeRows) : undefined,
+      completedRows: completed,
+      message: callsInFlightMessage({
+        activeRows,
+        completedRows: completed,
+        totalRows: total,
+      }),
+    });
+  };
+
+  const applyExecutionEvent = (event: PlayExecutionEvent) => {
+    if (event.type === 'docflow.node.hit') {
+      setNodeProgress(event.nodeId, {
+        startedAt: event.at,
+        completedAt: event.at,
+      });
+      scheduleFlush(true);
+      return;
+    }
+    if (
+      event.type === 'docflow.node.started' ||
+      event.type === 'docflow.node.completed' ||
+      event.type === 'docflow.node.failed'
+    ) {
+      const current = liveNodeProgress[event.nodeId];
+      const currentAttempt = current?.nodeIo?.attempt ?? -1;
+      if (event.attempt < currentAttempt) return;
+      if (
+        event.attempt === currentAttempt &&
+        docflowInvocationSequence(event.invocationId) <
+          docflowInvocationSequence(current?.nodeIo?.invocationId)
+      ) {
+        return;
+      }
+      if (event.type === 'docflow.node.started') {
+        liveNodeProgress = {
+          ...liveNodeProgress,
+          [event.nodeId]: {
+            startedAt: event.at,
+            updatedAt: event.at,
+            nodeIo: {
+              attempt: event.attempt,
+              invocationId: event.invocationId,
+              inputs: event.inputs,
+              ...(event.inputsTruncated ? { inputsTruncated: true } : {}),
+            },
+          },
+        };
+        liveNodeProgressDirty = true;
+        scheduleFlush(true);
+        return;
+      }
+      if (event.type === 'docflow.node.completed') {
+        setNodeProgress(event.nodeId, {
+          startedAt: current?.startedAt ?? event.at,
+          completedAt: event.at,
+          nodeIo: {
+            attempt: event.attempt,
+            invocationId: event.invocationId,
+            inputs: event.inputs,
+            ...(event.inputsTruncated ? { inputsTruncated: true } : {}),
+            outputs: event.outputs,
+            ...(event.outputsTruncated ? { outputsTruncated: true } : {}),
+            error: null,
+          },
+        });
+        scheduleFlush(true);
+        return;
+      }
+      if (event.type === 'docflow.node.failed') {
+        setNodeProgress(event.nodeId, {
+          startedAt: current?.startedAt ?? event.at,
+          completedAt: event.at,
+          nodeIo: {
+            attempt: event.attempt,
+            invocationId: event.invocationId,
+            inputs: event.inputs,
+            ...(event.inputsTruncated ? { inputsTruncated: true } : {}),
+            error: event.error,
+          },
+        });
+        scheduleFlush(true);
+        return;
+      }
+    }
+    if (event.type === 'activity.observed') {
+      pendingActivityObservations.set(
+        event.observation.activityId,
+        event.observation,
+      );
+      scheduleFlush(true);
+      return;
+    }
+    if (event.type === 'dataset.lifecycle') {
+      pendingDatasetLifecycleEvents.push(event);
+      scheduleFlush(true);
+      return;
+    }
+    // ADR 0018: tool-node lifecycle is an explicit typed runtime event. It
+    // replaces the `Calling tool: <id>` / `Tool <id> failed` stdout regexes
+    // this reporter used to run over the customer log channel.
+    if (event.type === 'tool.call.started') {
+      stepLifecycle.onToolCalled(event.toolId);
+      scheduleFlush();
+      return;
+    }
+    if (event.type === 'tool.call.settled') {
+      if (event.outcome === 'failed') {
+        stepLifecycle.onToolFailed(event.toolId);
+        scheduleFlush(true);
+      }
+      return;
+    }
+    if (!event.type.startsWith('map.')) {
+      return;
+    }
+    // The inline Node runner emits map events with mapNodeId null (unlike the
+    // workers runtime), so fall back to resolving the static-pipeline node
+    // from the event's table namespace — same lookup the log parser uses.
+    const nodeId =
+      event.mapNodeId?.trim() ||
+      resolveDatasetNodeRefFromTableNamespace(
+        mapNodeLookup,
+        event.artifactTableNamespace || event.logicalNamespace,
+      )?.nodeId;
+    if (!nodeId) {
+      return;
+    }
+    activeMapNodeId = nodeId;
+    const eventTotalRows = 'totalRows' in event ? event.totalRows : undefined;
+    if (
+      typeof eventTotalRows === 'number' &&
+      Number.isFinite(eventTotalRows) &&
+      eventTotalRows > totalRows
+    ) {
+      totalRows = eventTotalRows;
+    }
+    if (
+      event.type === 'map.preparing' ||
+      event.type === 'map.started' ||
+      event.type === 'map.resumed'
+    ) {
+      stepLifecycle.onMapStarted(nodeId);
+      setNodeProgress(nodeId, {
+        completed: event.completedRows,
+        failed: 0,
+        total: eventTotalRows,
+        startedRows: event.completedRows,
+        activeRows: 0,
+        waitingRows:
+          typeof eventTotalRows === 'number'
+            ? Math.max(0, eventTotalRows - event.completedRows)
+            : undefined,
+        completedRows: event.completedRows,
+        artifactTableNamespace: event.artifactTableNamespace,
+        startedAt: Date.now(),
+        message:
+          event.type === 'map.preparing'
+            ? 'Preparing durable table...'
+            : rowsProcessedMessage(event.completedRows, eventTotalRows ?? 0),
+      });
+      return;
+    }
+    if (event.type === 'map.progress') {
+      const current = liveNodeProgress[nodeId] ?? {};
+      const settled = Math.max(0, event.completedRows + event.failedRows);
+      const progressTotalRows = Math.max(
+        typeof event.totalRows === 'number' && Number.isFinite(event.totalRows)
+          ? event.totalRows
+          : 0,
+        Number(current.total ?? 0),
+      );
+      const startedRows = Math.max(settled, Number(current.startedRows ?? 0));
+      const activeRows = Math.max(0, startedRows - settled);
+      setNodeProgress(nodeId, {
+        completed: event.completedRows,
+        failed: event.failedRows,
+        total: progressTotalRows > 0 ? progressTotalRows : undefined,
+        startedRows,
+        activeRows,
+        waitingRows:
+          progressTotalRows > 0
+            ? Math.max(0, progressTotalRows - startedRows)
+            : undefined,
+        completedRows: event.completedRows,
+        artifactTableNamespace: event.artifactTableNamespace,
+        message: rowsProcessedMessage(event.completedRows, progressTotalRows),
+      });
+      return;
+    }
+    if (event.type === 'map.completed' || event.type === 'map.failed') {
+      if (event.type === 'map.failed') {
+        stepLifecycle.onMapFailed(nodeId);
+      } else {
+        stepLifecycle.onMapCompleted(nodeId);
+      }
+      const processedRows = Math.max(0, event.completedRows + event.failedRows);
+      const terminalTotal =
+        typeof event.totalRows === 'number' && Number.isFinite(event.totalRows)
+          ? event.totalRows
+          : 0;
+      setNodeProgress(nodeId, {
+        completed: event.completedRows,
+        failed: event.failedRows,
+        total: event.totalRows,
+        startedRows:
+          terminalTotal > 0
+            ? Math.min(terminalTotal, processedRows)
+            : processedRows,
+        activeRows: 0,
+        waitingRows: 0,
+        completedRows: event.completedRows,
+        artifactTableNamespace: event.artifactTableNamespace,
+        completedAt: Date.now(),
+        message: rowsProcessedMessage(event.completedRows, event.totalRows),
+      });
+      activeMapNodeId = null;
+      return;
+    }
+    if (event.type === 'map.suspended') {
+      setNodeProgress(nodeId, {
+        completed: event.completedRows,
+        failed: event.failedRows,
+        total: event.totalRows,
+        completedRows: event.completedRows,
+        artifactTableNamespace: event.artifactTableNamespace,
+        message: rowsProcessedMessage(event.completedRows, event.totalRows),
+      });
+    }
+  };
+
+  const onRunnerEvent = (event: unknown): void => {
+    if (!event || typeof event !== 'object') {
+      return;
+    }
+    const type = (event as { type?: unknown }).type;
+    if (type === 'log') {
+      const logEvent = event as { line?: unknown; source?: unknown };
+      const rawLine = logEvent.line;
+      if (typeof rawLine !== 'string' || !rawLine) {
+        return;
+      }
+      if (shouldPublishDurableLogEvent(durableLogMode, logEvent)) {
+        appendLiveLog(rawLine);
+      }
+      markMapStartedFromLog(rawLine);
+      markMapStartedFromPerfLog(rawLine);
+      markToolBatchInFlightFromLog(rawLine);
+      scheduleFlush();
+      return;
+    }
+    if (type === 'row_update') {
+      // Runtime Sheet rows are owned by the context-level row writer. Progress
+      // consumes execution and dataset lifecycle facts only; it never patches
+      // the same durable rows through a second write path.
+      return;
+    }
+    if (type === 'execution_event') {
+      const executionEvent = (event as { event?: PlayExecutionEvent }).event;
+      if (executionEvent && typeof executionEvent.type === 'string') {
+        applyExecutionEvent(executionEvent);
+        scheduleFlush();
+      }
+      return;
+    }
+  };
+
+  // ADR 0016 rule 3: fold durable docflow observations into liveNodeProgress at
+  // terminal so a completed absurd run carries docflow node I/O even when the
+  // live docflow.node.* events were dropped mid-run. A record only supersedes
+  // the current node state when it is at least as recent — later attempt, or
+  // same attempt with a >= invocation sequence — matching the event-path guard
+  // in applyExecutionEvent so live and durable never regress each other.
+  const applyDurableObservations = (
+    observations: DocflowObservationRecord[],
+  ): void => {
+    for (const observation of observations) {
+      const nodeId = observation.nodeId.trim();
+      if (!nodeId) continue;
+      const current = liveNodeProgress[nodeId];
+      const currentAttempt = current?.nodeIo?.attempt ?? -1;
+      if (observation.attempt < currentAttempt) continue;
+      if (
+        observation.attempt === currentAttempt &&
+        observation.invocationSeq <
+          docflowInvocationSequence(current?.nodeIo?.invocationId)
+      ) {
+        continue;
+      }
+      const at = observation.settledAt ?? observation.startedAt ?? Date.now();
+      setNodeProgress(nodeId, {
+        startedAt: current?.startedAt ?? observation.startedAt ?? at,
+        ...(observation.status === 'started' ? {} : { completedAt: at }),
+        nodeIo: {
+          attempt: observation.attempt,
+          ...(observation.invocationId
+            ? { invocationId: observation.invocationId }
+            : {}),
+          ...(observation.inputs ? { inputs: observation.inputs } : {}),
+          ...(observation.inputsTruncated ? { inputsTruncated: true } : {}),
+          ...(observation.status === 'completed' && observation.outputs
+            ? { outputs: observation.outputs }
+            : {}),
+          ...(observation.status === 'completed' && observation.outputsTruncated
+            ? { outputsTruncated: true }
+            : {}),
+          ...(observation.status === 'failed'
+            ? { error: observation.error ?? 'Docflow node failed.' }
+            : observation.status === 'completed'
+              ? { error: null }
+              : {}),
+        },
+      });
+    }
+    liveNodeProgressDirty = true;
+  };
+
+  // Terminal truth for map/dataset nodes is the durable per-map checkpoint
+  // frame carried on the runner result — NOT the event-fed liveNodeProgress
+  // counters, which can be stale or dropped (a missed map.completed frame, a
+  // node-id resolution miss on a docflow-gated run) and then become the
+  // terminal claim. ADR 0016 rule 3: terminal step progress is derived from the
+  // data plane. `checkpoint.mapFrames` is the map executor's authoritative
+  // settled account (`completedRowsCount` successes-only, `failedRowsCount`
+  // settled-but-failed, `totalRows` the settled denominator), written after the
+  // incremental-persistence flush barrier, so it agrees with the runtime
+  // sheets. Reconcile each map node's terminal progress from it before the
+  // final flush so the persisted ledger — and `summarizeRunRowOutcomes` — carry
+  // the truth.
+  const reconcileTerminalMapProgressFromCheckpoint = (
+    result: PlayRunnerResult | null | undefined,
+  ): void => {
+    const mapFrames = result?.checkpoint?.mapFrames;
+    if (!mapFrames || typeof mapFrames !== 'object') {
+      return;
+    }
+    for (const frame of Object.values(mapFrames)) {
+      if (!frame || typeof frame !== 'object') continue;
+      if (frame.status !== 'completed' && frame.status !== 'failed') continue;
+      const nodeId =
+        frame.mapNodeId?.trim() ||
+        resolveDatasetNodeRefFromTableNamespace(
+          mapNodeLookup,
+          frame.artifactTableNamespace || frame.logicalNamespace,
+        )?.nodeId;
+      if (!nodeId) continue;
+      // `completedRowKeys`/`pendingRowKeys` are emptied at map completion, so
+      // trust the *Count fields; fall back to the key-array length only while a
+      // count is absent (never at a settled frame the executor fully accounts).
+      const failed = Math.max(0, Number(frame.failedRowsCount ?? 0) || 0);
+      const completed = Math.max(
+        0,
+        Number(
+          frame.completedRowsCount ?? frame.completedRowKeys?.length ?? 0,
+        ) || 0,
+      );
+      const total =
+        Number.isFinite(frame.totalRows) && frame.totalRows > 0
+          ? frame.totalRows
+          : completed + failed;
+      const existing = liveNodeProgress[nodeId] ?? {};
+      // Only correct upward-or-equal on the settled denominator and never lose
+      // a larger already-observed success count from a concurrent flush; the
+      // checkpoint frame is authoritative for the settled split.
+      setNodeProgress(nodeId, {
+        completed,
+        failed,
+        total: total > 0 ? total : undefined,
+        startedRows: total > 0 ? total : completed + failed,
+        activeRows: 0,
+        waitingRows: 0,
+        completedRows: completed,
+        artifactTableNamespace:
+          frame.artifactTableNamespace ||
+          existing.artifactTableNamespace ||
+          undefined,
+        completedAt: existing.completedAt ?? Date.now(),
+        message: rowsProcessedMessage(completed, total > 0 ? total : undefined),
+      });
+    }
+    liveNodeProgressDirty = true;
+  };
+
+  // Sheet-table row-outcome counts recorded by the worker before finalize.
+  // These are the reconciliation of record (ADR 0016 rule 3): the runtime
+  // sheet rows exist regardless of runner mode, while checkpoint frames can be
+  // omitted on the detached terminal wire (OMITTED_TERMINAL_REPLAY) and events
+  // never stream while the worker is parked.
+  let terminalSheetOutcomes: Array<{
+    tableNamespace: string;
+    completed: number;
+    failed: number;
+    total: number;
+  }> = [];
+
+  const applyTerminalSheetOutcomes = (
+    outcomes: Array<{
+      tableNamespace: string;
+      completed: number;
+      failed: number;
+      total: number;
+    }>,
+  ): void => {
+    terminalSheetOutcomes = outcomes;
+  };
+
+  // Applied AFTER the checkpoint-frame reconcile so the data plane wins when
+  // the two disagree; also the only terminal source when frames are missing
+  // (detached push runs) and liveNodeProgress is empty (zero-event runs).
+  const reconcileTerminalMapProgressFromSheetOutcomes = (): void => {
+    for (const outcome of terminalSheetOutcomes) {
+      const tableNamespace = outcome.tableNamespace.trim();
+      if (!tableNamespace) continue;
+      const nodeRef = resolveDatasetNodeRefFromTableNamespace(
+        mapNodeLookup,
+        tableNamespace,
+      );
+      if (!nodeRef) continue;
+      const completed = Math.max(0, Math.floor(outcome.completed) || 0);
+      const failed = Math.max(0, Math.floor(outcome.failed) || 0);
+      const total =
+        Number.isFinite(outcome.total) && outcome.total > 0
+          ? Math.floor(outcome.total)
+          : completed + failed;
+      if (total <= 0 && completed <= 0 && failed <= 0) continue;
+      const existing = liveNodeProgress[nodeRef.nodeId] ?? {};
+      const now = Date.now();
+      setNodeProgress(nodeRef.nodeId, {
+        completed,
+        failed,
+        total: total > 0 ? total : undefined,
+        startedRows: total > 0 ? total : completed + failed,
+        activeRows: 0,
+        waitingRows: 0,
+        completedRows: completed,
+        artifactTableNamespace:
+          existing.artifactTableNamespace ?? nodeRef.artifactTableNamespace,
+        startedAt: existing.startedAt ?? now,
+        completedAt: existing.completedAt ?? now,
+        message: rowsProcessedMessage(completed, total > 0 ? total : undefined),
+      });
+    }
+    liveNodeProgressDirty = true;
+  };
+
+  const finalize = async (
+    result?: PlayRunnerResult | null,
+    options: {
+      status?: 'running' | 'completed' | 'failed';
+      required?: boolean;
+      replayAllTerminalLogs?: boolean;
+      terminalLogProducerAttempt?: number;
+      skipTerminalLogReplay?: boolean;
+    } = {},
+  ): Promise<void> => {
+    // Mark started steps terminal with real timestamps. Static branches that
+    // never started remain non-executed; suspended runs stay in-flight.
+    if (result?.status === 'failed') {
+      stepLifecycle.markStartedFailed();
+    } else if (result?.status === 'completed') {
+      stepLifecycle.markAllTerminal();
+    }
+    // Terminal runs derive their per-map step progress from the durable data
+    // plane, overriding any stale/dropped event-fed counters. Checkpoint
+    // frames first (cheap, in-result), then sheet-table counts as the
+    // reconciliation of record. Suspended runs stay mid-flight, so leave their
+    // live counters untouched.
+    if (result?.status === 'completed' || result?.status === 'failed') {
+      reconcileTerminalMapProgressFromCheckpoint(result);
+      reconcileTerminalMapProgressFromSheetOutcomes();
+    }
+    const status = options.status ?? 'running';
+    await flush({
+      force: true,
+      status,
+      required:
+        options.required === true ||
+        status === 'completed' ||
+        status === 'failed',
+      terminalLogReplay:
+        options.skipTerminalLogReplay || !Array.isArray(result?.logs)
+          ? null
+          : terminalReplayPayloadForMode({
+              resultLogs: result.logs,
+              durableLogMode,
+              liveLogs,
+              liveLogsTotalCount,
+              replayAllTerminalLogs: options.replayAllTerminalLogs === true,
+              producerAttempt:
+                options.terminalLogProducerAttempt ?? producerAttempt,
+            }),
+    });
+  };
+
+  return {
+    onRunnerEvent,
+    applyDurableObservations,
+    applyTerminalSheetOutcomes,
+    finalize,
+  };
+}
+
+export const createDirectWorkerProgressReporter = (input: {
+  launch: PlaySchedulerSubmitInput;
+  producerAttempt?: number;
+  isCurrentAttempt?: () => boolean | Promise<boolean>;
+  initialLiveLogTotalCount?: number;
+}) =>
+  createPostgresSchedulerProgressReporter({
+    ...input,
+    durableLogMode: 'direct_worker',
+  });
+export type DirectWorkerProgressReporter = PostgresSchedulerProgressReporter;