deepline 0.3.391 → 0.3.393

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -46,6 +46,11 @@ import {
 } from '../play-data-plane/sheet-column-storage';
 import {
   RUNTIME_SHEET_CELL_META_COLUMN,
+  RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE,
+  RUNTIME_SHEET_RUN_MEMBERSHIP_WRITES_ENABLED,
+  appendRuntimeSheetRunRowMembershipBatch,
+  isRuntimeSheetRunMembershipStorageAvailable,
+  type RuntimeSheetRunRowMembership,
   runtimeSheetResultFreshnessSql,
   runtimeSheetResultAdmissionSql,
 } from '../play-data-plane/runtime-sheet-storage-contract';
@@ -66,6 +71,7 @@ import type {
 } from './db-session';
 import {
   derivePlayRowIdentity,
+  stableStringify,
   normalizePlayNameForSheet,
   normalizeTableNamespace,
 } from '../plays/row-identity';
@@ -503,6 +509,8 @@ export type PrepareRuntimeSheetResult = {
   attemptExpiresAt?: string;
   attemptSeq?: number;
   writeVersion?: number;
+  /** Optional reader-first capture capability; absent retains legacy behavior. */
+  membershipCapture?: boolean;
 };
 
 export type RuntimeSheetTiming = {
@@ -645,6 +653,10 @@ type RuntimeApiActionRequest =
       action: keyof typeof RUNTIME_SHEET_ADMISSION_CAPABILITIES;
       input: Parameters<typeof startRuntimeSheetDataset>[1];
     }
+  | {
+      action: 'runtime_sheet_finish_membership';
+      input: Parameters<typeof finishRuntimeSheetRunMembership>[1];
+    }
   | {
       action: 'runtime_sheet_complete_map_rows';
       input: Parameters<typeof completeRuntimeMapRows>[1];
@@ -2872,6 +2884,151 @@ async function registerRuntimeDataset(
   );
 }
 
+async function runtimeSheetMembershipCaptureAvailable(
+  client: RuntimeQueryClient,
+  session: RuntimePostgresSession,
+): Promise<boolean> {
+  if (!RUNTIME_SHEET_RUN_MEMBERSHIP_WRITES_ENABLED) return false;
+  return isRuntimeSheetRunMembershipStorageAvailable(
+    async <Row>(sql: string, params?: readonly unknown[]) => {
+      const result = await client.query(sql, params ? [...params] : undefined);
+      return { rows: result.rows as Row[] };
+    },
+    session.postgres.schema,
+    true,
+  );
+}
+
+async function persistRuntimeSheetMembershipBatch(
+  client: RuntimeQueryClient,
+  session: RuntimePostgresSession,
+  rows: Array<Omit<RuntimeSheetRunRowMembership, 'recordedAt'>>,
+) {
+  await appendRuntimeSheetRunRowMembershipBatch(
+    async <Row>(sql: string, params?: readonly unknown[]) => {
+      const result = await client.query(sql, params ? [...params] : undefined);
+      return { rows: result.rows as Row[] };
+    },
+    rows,
+    session.postgres.schema,
+  );
+}
+
+async function captureRuntimeSheetTerminalMembership(
+  client: RuntimeQueryClient,
+  session: RuntimePostgresSession,
+  input: {
+    runId: string;
+    tableNamespace: string;
+    sheetContract: PlaySheetContract;
+    keys: string[];
+  },
+): Promise<void> {
+  if (input.keys.length === 0) return;
+  const datasetId = createRuntimeDatasetId(
+    session.playName,
+    input.tableNamespace,
+  );
+  // Read canonical values AFTER the fenced mutation, inside its transaction.
+  // A stale submitted body must never become a fabricated historical result.
+  const result = await client.query<Record<string, unknown>>(
+    `SELECT target.*, admission.input_digest AS membership_input_digest, admission.source_ref AS membership_source_ref
+    FROM ${sheetTable(session)} target
+    JOIN ${fqRuntimeTable(session, RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE)} admission
+      ON admission.run_id = $1 AND admission.dataset_id = $2 AND admission.row_key = target._key AND admission.phase = 'admitted'
+    WHERE target._key = ANY($3::text[]) AND target._run_id = $1 AND target._status IN ('enriched', 'failed')`,
+    [input.runId, datasetId, input.keys],
+  );
+  const snapshots = result.rows.map((raw) => {
+    const attemptSeq =
+      raw._attempt_seq == null ? null : Number(raw._attempt_seq);
+    const rowVersion = Number(raw._version);
+    const data = mapRuntimePostgresRow({
+      raw,
+      sheetContract: input.sheetContract,
+    }).data;
+    delete data[DEEPLINE_CELL_META_FIELD];
+    return {
+      runId: input.runId,
+      datasetId,
+      rowKey: String(raw._key),
+      phase: 'terminal' as const,
+      generation: `attempt:${attemptSeq ?? 'legacy'}:version:${rowVersion}`,
+      attemptSeq,
+      rowVersion,
+      inputIndex: raw._input_index == null ? null : Number(raw._input_index),
+      inputDigest: String(raw.membership_input_digest),
+      status: raw._status as 'enriched' | 'failed',
+      data,
+      source: {
+        ...(raw.membership_source_ref as Record<string, unknown>),
+        ...(raw._error == null ? {} : { rowError: String(raw._error) }),
+      },
+    };
+  });
+  await persistRuntimeSheetMembershipBatch(client, session, snapshots);
+}
+
+/** Named completion is called only after admission advertised capture. */
+export async function finishRuntimeSheetRunMembership(
+  context: RuntimeApiContext & { playName: string },
+  input: {
+    runId: string;
+    tableNamespace: string;
+    sheetContract: PlaySheetContract;
+    expectedRows: number;
+  },
+): Promise<{ captured: boolean; complete: boolean }> {
+  if (!Number.isSafeInteger(input.expectedRows) || input.expectedRows < 0)
+    throw new Error('Membership expectedRows must be a non-negative integer.');
+  if (context.dbSessionStrategy === 'gateway_only')
+    return postRuntimeApi(context, {
+      action: 'runtime_sheet_finish_membership',
+      input,
+    });
+  const session = requireRuntimePostgresSession(
+    await getRuntimeDbSession(context, {
+      tableNamespace: input.tableNamespace,
+      logicalTable: 'sheet_rows',
+      operations: ['rows.read', 'rows.upsert'],
+      limits: {},
+    }),
+  );
+  return withRuntimeSheetQueryClient(
+    context,
+    session,
+    {
+      playName: context.playName,
+      tableNamespace: input.tableNamespace,
+      sheetContract: input.sheetContract,
+      transactional: true,
+    },
+    async (client) => {
+      if (!(await runtimeSheetMembershipCaptureAvailable(client, session)))
+        return { captured: false, complete: false };
+      const datasetId = createRuntimeDatasetId(
+        session.playName,
+        input.tableNamespace,
+      );
+      const result = await client.query<{ complete: boolean }>(
+        `UPDATE ${runDatasetCatalogTable(session)} catalog
+      SET membership_expected_rows = $3,
+          membership_completed_at = CASE WHEN
+            (SELECT count(*) FROM ${fqRuntimeTable(session, RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE)} WHERE run_id = $1 AND dataset_id = $2 AND phase = 'admitted') = $3
+            AND (SELECT count(DISTINCT row_key) FROM ${fqRuntimeTable(session, RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE)} WHERE run_id = $1 AND dataset_id = $2 AND phase = 'terminal') = $3
+          THEN coalesce(catalog.membership_completed_at, now()) ELSE NULL END
+      WHERE run_id = $1 AND dataset_id = $2 AND membership_started_at IS NOT NULL
+      RETURNING membership_completed_at IS NOT NULL AS complete`,
+        [input.runId, datasetId, input.expectedRows],
+      );
+      return {
+        captured: result.rows.length === 1,
+        complete: result.rows[0]?.complete === true,
+      };
+    },
+  );
+}
+
 function workReceiptTable(session: RuntimePostgresSession): string {
   return fqRuntimeTable(
     session,
@@ -6877,7 +7034,11 @@ export async function startRuntimeSheetDataset(
   const normalizeStartedAt = Date.now();
   const uniqueRows = new Map<
     string,
-    { row: Record<string, unknown>; inputIndex: number | null }
+    {
+      row: Record<string, unknown>;
+      inputIndex: number | null;
+      source: Record<string, unknown>;
+    }
   >();
   for (const row of input.rows) {
     // Materializes projected CSV aliases as visible cells and drops internal
@@ -6890,6 +7051,13 @@ export async function startRuntimeSheetDataset(
     if (key && !uniqueRows.has(key)) {
       uniqueRows.set(key, {
         row: sanitizePostgresJsonValue(cleanedRow),
+        source:
+          row.__deeplineSourceRow && typeof row.__deeplineSourceRow === 'object'
+            ? (sanitizePostgresJsonValue(row.__deeplineSourceRow) as Record<
+                string,
+                unknown
+              >)
+            : {},
         inputIndex: normalizeRuntimeMapInputIndex(
           row[MAP_ROW_OUTCOME_RUNTIME_FIELDS.inputIndex],
         ),
@@ -6901,6 +7069,7 @@ export async function startRuntimeSheetDataset(
     ([key, entry], inputIndex) => ({
       key,
       row: entry.row,
+      source: entry.source,
       inputIndex: entry.inputIndex ?? inputOffset + inputIndex,
     }),
   );
@@ -7135,6 +7304,60 @@ export async function startRuntimeSheetDataset(
         force: input.force === true,
         ...prepared,
       });
+      const membershipCapture = await runtimeSheetMembershipCaptureAvailable(
+        client,
+        session,
+      );
+      if (membershipCapture) {
+        const dispositions = new Set(
+          prepared.rowDispositions
+            .filter(
+              (row) =>
+                row.disposition === 'pending' ||
+                row.disposition === 'completed',
+            )
+            .map((row) => row.key),
+        );
+        const admitted = rowEntries.filter((entry) =>
+          dispositions.has(entry.key),
+        );
+        await client.query(
+          `UPDATE ${runDatasetCatalogTable(session)} SET membership_started_at = coalesce(membership_started_at, now()) WHERE run_id = $1 AND dataset_id = $2`,
+          [
+            input.runId,
+            createRuntimeDatasetId(session.playName, input.tableNamespace),
+          ],
+        );
+        await persistRuntimeSheetMembershipBatch(
+          client,
+          session,
+          admitted.map((entry) => ({
+            runId: input.runId,
+            datasetId: createRuntimeDatasetId(
+              session.playName,
+              input.tableNamespace,
+            ),
+            rowKey: entry.key,
+            phase: 'admitted',
+            generation: 'admitted',
+            attemptSeq: null,
+            rowVersion: null,
+            inputIndex: entry.inputIndex,
+            inputDigest: createHash('sha256')
+              .update(stableStringify(entry.row))
+              .digest('hex'),
+            status: 'pending',
+            data: entry.row,
+            source: entry.source,
+          })),
+        );
+        await captureRuntimeSheetTerminalMembership(client, session, {
+          runId: input.runId,
+          tableNamespace: input.tableNamespace,
+          sheetContract: input.sheetContract,
+          keys: admitted.map((entry) => entry.key),
+        });
+      }
       timings.push({
         phase: 'build_result',
         ms: Date.now() - buildStartedAt,
@@ -7146,6 +7369,7 @@ export async function startRuntimeSheetDataset(
       });
       return {
         ...built,
+        membershipCapture,
         attemptExpiresAt,
         attemptSeq,
         writeVersion,
@@ -8982,11 +9206,20 @@ export async function completeRuntimeMapRows(
                 }),
             )
           : { updated: 0, fencedRows: [] };
-      return classifyRuntimeMapRowsWrite({
+      const classified = classifyRuntimeMapRowsWrite({
         submittedKeys: uniqueRows.keys(),
         updated: completed.updated + failed.updated,
         fencedRows: [...completed.fencedRows, ...failed.fencedRows],
       });
+      if (await runtimeSheetMembershipCaptureAvailable(client, session)) {
+        await captureRuntimeSheetTerminalMembership(client, session, {
+          runId: input.runId,
+          tableNamespace: input.tableNamespace,
+          sheetContract,
+          keys: classified.committedKeys,
+        });
+      }
+      return classified;
     },
   )
     .then((result) => {

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -178,6 +178,9 @@ import {
   readPostgresSchedulerRunnerLiveness,
   readPostgresSchedulerRuntimeResourcesForRun,
   isPostgresSchedulerRunnerProgressAttemptCurrent,
+  appendPostgresRunLedgerEvents,
+  readPostgresSchedulerRunnerTerminalReceivedAtMs,
+  claimPostgresSchedulerWaitingSuspensionAttemptForFailure,
   requestComputeBillingDispatchToken,
   blockPostgresSchedulerCronSchedule,
   PostgresSchedulerCronScheduleConfigurationError,
@@ -233,6 +236,7 @@ import {
 import { runWithAbsurdSchedulerQuerySignal } from './absurd-pool';
 import {
   buildFailedTerminalSettledProgressEvents,
+  collectGatewayRunnerTerminalProgressLedgerEvents,
   createPostgresSchedulerProgressReporter,
   type PostgresSchedulerProgressReporter,
 } from './postgres-progress';
@@ -3261,11 +3265,57 @@ async function executeAssignedAbsurdAttempt(input: {
             terminalLogProducerAttempt: input.terminalLogProducerAttempt,
           }),
       });
-    // A real suspension closes an invocation's counter segment before its
-    // wait or handoff. Detached terminal recovery can replay the retained
-    // result and its idempotent final segment after a crash. A detached_runner
-    // placeholder still owns a live runner; preserve its park-first recovery.
-    if (suspension.kind !== 'detached_runner') {
+    const retainedDetachedSuspension =
+      input.fromDetachedRunner && suspension.kind !== 'detached_runner';
+    let finalProgressRetained = false;
+    if (retainedDetachedSuspension) {
+      await runWithAbsurdExecutionLeaseHeartbeat({
+        ctx,
+        runId,
+        operation: () =>
+          deps.runWithClient(async (client) => {
+            const runnerAttempt =
+              input.terminalLogProducerAttempt ?? generation;
+            const acceptedAt =
+              await readPostgresSchedulerRunnerTerminalReceivedAtMs(
+                client,
+                { runId, attempt: runnerAttempt },
+                options,
+              );
+            if (acceptedAt !== null) {
+              const events =
+                await collectGatewayRunnerTerminalProgressLedgerEvents({
+                  launch,
+                  attempt: runnerAttempt,
+                  result: input.runnerResult,
+                  events: [],
+                  liveLogOffset: 0,
+                  suspensionAcceptedAtMs: acceptedAt,
+                });
+              if (events.length > 0) {
+                await appendPostgresRunLedgerEvents(
+                  client,
+                  { orgId: launch.orgId, runId, events },
+                  options,
+                );
+              }
+              finalProgressRetained = true;
+            }
+            // The pushed result proves this exact attempt's runner exited.
+            // Persist its provider-scoped obligation before publishing a wait
+            // that can outlive this worker. Only the cleanup lease deletes it.
+            await enqueuePostgresSchedulerSandboxCleanupForAttempt(
+              client,
+              { runId, attempt: runnerAttempt },
+              options,
+            );
+          }),
+      });
+    }
+    // Local and historical results still require final publication before
+    // advancing their checkpoint. Otherwise all final facts and cleanup are
+    // durable, so progress transport cannot hide the typed waiting state.
+    if (suspension.kind !== 'detached_runner' && !finalProgressRetained) {
       await finalizeSuspensionProgress();
     }
     const outcome = await createPlayRunAttemptSuspensionJob({
@@ -3284,6 +3334,9 @@ async function executeAssignedAbsurdAttempt(input: {
             options,
             transition: 'waiting',
           });
+          if (finalProgressRetained) {
+            await finalizeSuspensionProgress();
+          }
         },
         resumed: async (lifecycle) =>
           await projectLifecycleOutbox({
@@ -3695,9 +3748,6 @@ async function executeAssignedAbsurdAttempt(input: {
                   ),
                 ),
               inspectDetachedRunner: deps.inspectDetachedRunner,
-              deleteRuntimeSandbox: deps.deleteRuntimeSandbox,
-              deleteRuntimeSandboxRef: deps.deleteRuntimeSandboxRef,
-              sandboxGcTimeoutMs: deps.sandboxGcTimeoutMs,
             }),
         });
       } catch (error) {
@@ -3800,9 +3850,6 @@ async function executeAssignedAbsurdAttempt(input: {
           inspectDetachedRunner: deps.inspectDetachedRunner,
           readDetachedRunnerRuntimeCompletion:
             deps.readDetachedRunnerRuntimeCompletion,
-          deleteRuntimeSandbox: deps.deleteRuntimeSandbox,
-          deleteRuntimeSandboxRef: deps.deleteRuntimeSandboxRef,
-          sandboxGcTimeoutMs: deps.sandboxGcTimeoutMs,
         });
       } else {
         if (backendExecutions > 0) {
@@ -4824,10 +4871,9 @@ export async function enforceRuntimeTrafficPolicyBeforeSandboxStart(input: {
  *    terminal transaction;
  *  - thrown outcomes   -> `releaseAbsurdAttemptRuntimeState` in the caller's
  *    catch (B4/FIX 3);
- *  - SUSPENDED results -> HERE, immediately: the runner already exited, and
- *    the follow-up play park (sleep/HITL) can last hours — waiting for
- *    the eventual terminal sweep would leak a dead sandbox for that whole
- *    window.
+ *  - suspended results -> the existing durable cleanup controller, after
+ *    `resolveRunnerSuspension` persists its obligation before typed waiting.
+ *    Provider deletion never delays publishing the user's wait.
  */
 async function resolveDetachedRunnerOutcome(input: {
   runId: string;
@@ -4845,9 +4891,6 @@ async function resolveDetachedRunnerOutcome(input: {
   } | null>;
   inspectDetachedRunner?: AbsurdRuntimeWorkerDeps['inspectDetachedRunner'];
   readDetachedRunnerRuntimeCompletion?: AbsurdRuntimeWorkerDeps['readDetachedRunnerRuntimeCompletion'];
-  deleteRuntimeSandbox: AbsurdRuntimeWorkerDeps['deleteRuntimeSandbox'];
-  deleteRuntimeSandboxRef?: AbsurdRuntimeWorkerDeps['deleteRuntimeSandboxRef'];
-  sandboxGcTimeoutMs?: number;
 }): Promise<PlayRunnerResult> {
   const { suspension } = input;
   const runtimeLimitSeconds =
@@ -4873,40 +4916,10 @@ async function resolveDetachedRunnerOutcome(input: {
       }),
     );
   };
-  const gcSandboxIfSuspended = async (result: PlayRunnerResult) => {
-    if (result.status !== 'suspended') return;
-    try {
-      const deleteResult = await deleteRuntimeSandboxBounded<
-        boolean | RuntimeSandboxDeleteOutcome
-      >({
-        deleteRuntimeSandbox: input.deleteRuntimeSandboxRef
-          ? () =>
-              input.deleteRuntimeSandboxRef!(
-                runtimeSandboxRefFromSuspension(suspension),
-              )
-          : input.deleteRuntimeSandbox,
-        sandboxId: suspension.sandboxId,
-        timeoutMs: input.sandboxGcTimeoutMs ?? ABSURD_SANDBOX_GC_TIMEOUT_MS,
-      });
-      requireRuntimeSandboxDeleted(
-        runtimeSandboxRefFromSuspension(suspension),
-        deleteResult,
-      );
-    } catch (error) {
-      console.warn('[absurd-runtime.worker] detached_sandbox_gc_failed', {
-        runId: input.runId,
-        runnerAttempt: suspension.runnerAttempt,
-        sandboxId: suspension.sandboxId,
-        stage: 'suspended_result',
-        error: error instanceof Error ? error.message : String(error),
-      });
-    }
-  };
   const settle = async (
     result: PlayRunnerResult,
   ): Promise<PlayRunnerResult> => {
     await recordWorkCompletion(result);
-    await gcSandboxIfSuspended(result);
     return result;
   };
   const settleFirstWriteWinner = async (
@@ -5548,6 +5561,7 @@ async function salvageAbsurdFinalAttemptTerminal(input: {
     ) {
       try {
         result = await input.deps.runWithClient(async (client) => {
+          let protectWaitingSuspensionFailure = false;
           // A detached-runner wake keeps its attempt `waiting` until setup
           // succeeds. Take that same fence to `running` (the overdue-ceiling
           // claim's transition) so the failure terminal can be written. When
@@ -5555,22 +5569,42 @@ async function salvageAbsurdFinalAttemptTerminal(input: {
           // fence or pushed terminal owns it — the ordinary fenced path below
           // decides.
           if (input.detachedRecovery) {
-            await claimPostgresSchedulerWaitingDetachedAttemptForTerminal(
-              client,
-              {
-                runId: input.runId,
-                attempt: input.generation,
-                leaseToken: input.leaseToken,
-              },
-              input.options,
-            );
+            const detachedClaimed =
+              await claimPostgresSchedulerWaitingDetachedAttemptForTerminal(
+                client,
+                {
+                  runId: input.runId,
+                  attempt: input.generation,
+                  leaseToken: input.leaseToken,
+                },
+                input.options,
+              );
+            if (!detachedClaimed) {
+              const preparation =
+                await claimPostgresSchedulerWaitingSuspensionAttemptForFailure(
+                  client,
+                  {
+                    runId: input.runId,
+                    attempt: input.generation,
+                    leaseToken: input.leaseToken,
+                  },
+                  input.options,
+                );
+              if (preparation === 'fence_lost')
+                return { status: 'idle' as const };
+              protectWaitingSuspensionFailure = preparation === 'prepared';
+            }
           }
           return await executePostgresSchedulerClaim(
             client,
             claim,
             async () => failedResult,
             input.detachedRecovery
-              ? { ...input.options, preserveRunnerTerminal: true }
+              ? {
+                  ...input.options,
+                  preserveRunnerTerminal: true,
+                  protectWaitingSuspensionFailure,
+                }
               : input.options,
           );
         });

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts

@@ -1086,7 +1086,13 @@ export async function withRuntimeSchedulerClient<T>(
               try {
                 const queryResult = await poolClient.query(sql, params);
                 throwIfSchedulerAborted(signal);
-                return { rows: queryResult.rows };
+                // Simple-protocol transactions return one result per command.
+                // Keep the same rows contract for callers of either protocol.
+                return {
+                  rows: Array.isArray(queryResult)
+                    ? queryResult.flatMap((result) => result.rows)
+                    : queryResult.rows,
+                };
               } catch (error) {
                 // node-postgres' query timeout and an AbortSignal reject the
                 // promise, but the server may still be processing the SQL.

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts

@@ -39,6 +39,7 @@ import { runtimeWorkCountersFromStats } from '../runtime-work-counters';
 import {
   buildPlayRunLedgerEventsFromStatusPatch,
   createEmptyPlayRunLedgerSnapshot,
+  buildTerminalLogReplayEvents,
   type PlayMapRowActivitySource,
   type PlayRunLedgerEvent,
 } from '../run-ledger';
@@ -1902,8 +1903,9 @@ export type DirectWorkerProgressReporter = PostgresSchedulerProgressReporter;
  * network boundary. The returned events are stored in the same Postgres
  * terminal outbox row as `run.completed`, so the projector can append the
  * final step/log/activity facts and the terminal fact in one ordered Convex
- * mutation. This is the completed-run fast path; failed/suspended runs retain
- * the compatibility projection until their parked worker resumes.
+ * mutation. A worker can also retain a suspended runner's complete final
+ * progress before advancing its checkpoint, using the immutable acceptance
+ * timestamp for the same outbox identity across crash or response-loss replay.
  */
 export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
   launch: PlaySchedulerSubmitInput;
@@ -1911,14 +1913,17 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
   result: PlayRunnerResult;
   events: unknown[];
   liveLogOffset: number;
+  suspensionAcceptedAtMs?: number;
 }): Promise<PlayRunLedgerEvent[]> {
   const captured: { update?: RuntimeStatusUpdate } = {};
-  const reporter = createDirectWorkerProgressReporter({
+  const retainedSuspension = input.suspensionAcceptedAtMs !== undefined;
+  const reporter = createPostgresSchedulerProgressReporter({
     launch: input.launch,
     producerAttempt: input.attempt,
     initialLiveLogTotalCount: input.liveLogOffset,
     isCurrentAttempt: () => true,
     attemptAlreadyVerified: true,
+    durableLogMode: retainedSuspension ? 'progress_reporter' : 'direct_worker',
     publishStatusUpdate: async (update) => {
       captured.update = update;
     },
@@ -1929,6 +1934,7 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
     input.launch.contractSnapshot,
   );
   if (
+    !retainedSuspension &&
     pipeline?.docflow?.nodes.some(
       (node) => node.kind === 'decision' && !node.sketch,
     )
@@ -1977,11 +1983,14 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
   await reporter.finalize(input.result, {
     status: 'running',
     required: true,
-    skipTerminalLogReplay: true,
+    skipTerminalLogReplay: !retainedSuspension,
+    replayAllTerminalLogs: retainedSuspension,
     terminalLogProducerAttempt: input.attempt,
   });
   const finalUpdate = captured.update;
   if (!finalUpdate) return [];
+  const checkpointAt =
+    input.suspensionAcceptedAtMs ?? finalUpdate.lastCheckpointAt;
 
   const liveLogs = finalUpdate.liveLogs ?? [];
   const liveLogTotalCount =
@@ -1990,7 +1999,7 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
     patch: {
       runId: input.launch.runId,
       status: 'running',
-      lastCheckpointAt: finalUpdate.lastCheckpointAt,
+      lastCheckpointAt: checkpointAt,
       liveNodeProgress: finalUpdate.liveNodeProgress,
       liveLogs,
       liveLogContexts: finalUpdate.liveLogContexts,
@@ -2011,7 +2020,7 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
       type: RUN_EVENT_KIND.workCounters,
       runId: input.launch.runId,
       source: 'worker',
-      occurredAt: finalUpdate.lastCheckpointAt ?? Date.now(),
+      occurredAt: checkpointAt ?? Date.now(),
       ...finalUpdate.workCounters,
     });
   }
@@ -2045,5 +2054,19 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
     occurredAt: observation.observedAt,
     observation,
   }));
-  return [...progressEvents, ...datasetEvents, ...activityEvents];
+  const replay = finalUpdate.terminalLogReplay;
+  const terminalLogEvents = replay
+    ? buildTerminalLogReplayEvents({
+        runId: input.launch.runId,
+        ...replay,
+        occurredAt: checkpointAt ?? undefined,
+        liveLogTotalCount: replay.totalCount,
+      })
+    : [];
+  return [
+    ...progressEvents,
+    ...datasetEvents,
+    ...activityEvents,
+    ...terminalLogEvents,
+  ];
 }