deepline 0.3.392 → 0.3.394

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -178,6 +178,9 @@ import {
   readPostgresSchedulerRunnerLiveness,
   readPostgresSchedulerRuntimeResourcesForRun,
   isPostgresSchedulerRunnerProgressAttemptCurrent,
+  appendPostgresRunLedgerEvents,
+  readPostgresSchedulerRunnerTerminalReceivedAtMs,
+  claimPostgresSchedulerWaitingSuspensionAttemptForFailure,
   requestComputeBillingDispatchToken,
   blockPostgresSchedulerCronSchedule,
   PostgresSchedulerCronScheduleConfigurationError,
@@ -233,6 +236,7 @@ import {
 import { runWithAbsurdSchedulerQuerySignal } from './absurd-pool';
 import {
   buildFailedTerminalSettledProgressEvents,
+  collectGatewayRunnerTerminalProgressLedgerEvents,
   createPostgresSchedulerProgressReporter,
   type PostgresSchedulerProgressReporter,
 } from './postgres-progress';
@@ -3261,11 +3265,57 @@ async function executeAssignedAbsurdAttempt(input: {
             terminalLogProducerAttempt: input.terminalLogProducerAttempt,
           }),
       });
-    // A real suspension closes an invocation's counter segment before its
-    // wait or handoff. Detached terminal recovery can replay the retained
-    // result and its idempotent final segment after a crash. A detached_runner
-    // placeholder still owns a live runner; preserve its park-first recovery.
-    if (suspension.kind !== 'detached_runner') {
+    const retainedDetachedSuspension =
+      input.fromDetachedRunner && suspension.kind !== 'detached_runner';
+    let finalProgressRetained = false;
+    if (retainedDetachedSuspension) {
+      await runWithAbsurdExecutionLeaseHeartbeat({
+        ctx,
+        runId,
+        operation: () =>
+          deps.runWithClient(async (client) => {
+            const runnerAttempt =
+              input.terminalLogProducerAttempt ?? generation;
+            const acceptedAt =
+              await readPostgresSchedulerRunnerTerminalReceivedAtMs(
+                client,
+                { runId, attempt: runnerAttempt },
+                options,
+              );
+            if (acceptedAt !== null) {
+              const events =
+                await collectGatewayRunnerTerminalProgressLedgerEvents({
+                  launch,
+                  attempt: runnerAttempt,
+                  result: input.runnerResult,
+                  events: [],
+                  liveLogOffset: 0,
+                  suspensionAcceptedAtMs: acceptedAt,
+                });
+              if (events.length > 0) {
+                await appendPostgresRunLedgerEvents(
+                  client,
+                  { orgId: launch.orgId, runId, events },
+                  options,
+                );
+              }
+              finalProgressRetained = true;
+            }
+            // The pushed result proves this exact attempt's runner exited.
+            // Persist its provider-scoped obligation before publishing a wait
+            // that can outlive this worker. Only the cleanup lease deletes it.
+            await enqueuePostgresSchedulerSandboxCleanupForAttempt(
+              client,
+              { runId, attempt: runnerAttempt },
+              options,
+            );
+          }),
+      });
+    }
+    // Local and historical results still require final publication before
+    // advancing their checkpoint. Otherwise all final facts and cleanup are
+    // durable, so progress transport cannot hide the typed waiting state.
+    if (suspension.kind !== 'detached_runner' && !finalProgressRetained) {
       await finalizeSuspensionProgress();
     }
     const outcome = await createPlayRunAttemptSuspensionJob({
@@ -3284,6 +3334,9 @@ async function executeAssignedAbsurdAttempt(input: {
             options,
             transition: 'waiting',
           });
+          if (finalProgressRetained) {
+            await finalizeSuspensionProgress();
+          }
         },
         resumed: async (lifecycle) =>
           await projectLifecycleOutbox({
@@ -3695,9 +3748,6 @@ async function executeAssignedAbsurdAttempt(input: {
                   ),
                 ),
               inspectDetachedRunner: deps.inspectDetachedRunner,
-              deleteRuntimeSandbox: deps.deleteRuntimeSandbox,
-              deleteRuntimeSandboxRef: deps.deleteRuntimeSandboxRef,
-              sandboxGcTimeoutMs: deps.sandboxGcTimeoutMs,
             }),
         });
       } catch (error) {
@@ -3800,9 +3850,6 @@ async function executeAssignedAbsurdAttempt(input: {
           inspectDetachedRunner: deps.inspectDetachedRunner,
           readDetachedRunnerRuntimeCompletion:
             deps.readDetachedRunnerRuntimeCompletion,
-          deleteRuntimeSandbox: deps.deleteRuntimeSandbox,
-          deleteRuntimeSandboxRef: deps.deleteRuntimeSandboxRef,
-          sandboxGcTimeoutMs: deps.sandboxGcTimeoutMs,
         });
       } else {
         if (backendExecutions > 0) {
@@ -4824,10 +4871,9 @@ export async function enforceRuntimeTrafficPolicyBeforeSandboxStart(input: {
  *    terminal transaction;
  *  - thrown outcomes   -> `releaseAbsurdAttemptRuntimeState` in the caller's
  *    catch (B4/FIX 3);
- *  - SUSPENDED results -> HERE, immediately: the runner already exited, and
- *    the follow-up play park (sleep/HITL) can last hours — waiting for
- *    the eventual terminal sweep would leak a dead sandbox for that whole
- *    window.
+ *  - suspended results -> the existing durable cleanup controller, after
+ *    `resolveRunnerSuspension` persists its obligation before typed waiting.
+ *    Provider deletion never delays publishing the user's wait.
  */
 async function resolveDetachedRunnerOutcome(input: {
   runId: string;
@@ -4845,9 +4891,6 @@ async function resolveDetachedRunnerOutcome(input: {
   } | null>;
   inspectDetachedRunner?: AbsurdRuntimeWorkerDeps['inspectDetachedRunner'];
   readDetachedRunnerRuntimeCompletion?: AbsurdRuntimeWorkerDeps['readDetachedRunnerRuntimeCompletion'];
-  deleteRuntimeSandbox: AbsurdRuntimeWorkerDeps['deleteRuntimeSandbox'];
-  deleteRuntimeSandboxRef?: AbsurdRuntimeWorkerDeps['deleteRuntimeSandboxRef'];
-  sandboxGcTimeoutMs?: number;
 }): Promise<PlayRunnerResult> {
   const { suspension } = input;
   const runtimeLimitSeconds =
@@ -4873,40 +4916,10 @@ async function resolveDetachedRunnerOutcome(input: {
       }),
     );
   };
-  const gcSandboxIfSuspended = async (result: PlayRunnerResult) => {
-    if (result.status !== 'suspended') return;
-    try {
-      const deleteResult = await deleteRuntimeSandboxBounded<
-        boolean | RuntimeSandboxDeleteOutcome
-      >({
-        deleteRuntimeSandbox: input.deleteRuntimeSandboxRef
-          ? () =>
-              input.deleteRuntimeSandboxRef!(
-                runtimeSandboxRefFromSuspension(suspension),
-              )
-          : input.deleteRuntimeSandbox,
-        sandboxId: suspension.sandboxId,
-        timeoutMs: input.sandboxGcTimeoutMs ?? ABSURD_SANDBOX_GC_TIMEOUT_MS,
-      });
-      requireRuntimeSandboxDeleted(
-        runtimeSandboxRefFromSuspension(suspension),
-        deleteResult,
-      );
-    } catch (error) {
-      console.warn('[absurd-runtime.worker] detached_sandbox_gc_failed', {
-        runId: input.runId,
-        runnerAttempt: suspension.runnerAttempt,
-        sandboxId: suspension.sandboxId,
-        stage: 'suspended_result',
-        error: error instanceof Error ? error.message : String(error),
-      });
-    }
-  };
   const settle = async (
     result: PlayRunnerResult,
   ): Promise<PlayRunnerResult> => {
     await recordWorkCompletion(result);
-    await gcSandboxIfSuspended(result);
     return result;
   };
   const settleFirstWriteWinner = async (
@@ -5548,6 +5561,7 @@ async function salvageAbsurdFinalAttemptTerminal(input: {
     ) {
       try {
         result = await input.deps.runWithClient(async (client) => {
+          let protectWaitingSuspensionFailure = false;
           // A detached-runner wake keeps its attempt `waiting` until setup
           // succeeds. Take that same fence to `running` (the overdue-ceiling
           // claim's transition) so the failure terminal can be written. When
@@ -5555,22 +5569,42 @@ async function salvageAbsurdFinalAttemptTerminal(input: {
           // fence or pushed terminal owns it — the ordinary fenced path below
           // decides.
           if (input.detachedRecovery) {
-            await claimPostgresSchedulerWaitingDetachedAttemptForTerminal(
-              client,
-              {
-                runId: input.runId,
-                attempt: input.generation,
-                leaseToken: input.leaseToken,
-              },
-              input.options,
-            );
+            const detachedClaimed =
+              await claimPostgresSchedulerWaitingDetachedAttemptForTerminal(
+                client,
+                {
+                  runId: input.runId,
+                  attempt: input.generation,
+                  leaseToken: input.leaseToken,
+                },
+                input.options,
+              );
+            if (!detachedClaimed) {
+              const preparation =
+                await claimPostgresSchedulerWaitingSuspensionAttemptForFailure(
+                  client,
+                  {
+                    runId: input.runId,
+                    attempt: input.generation,
+                    leaseToken: input.leaseToken,
+                  },
+                  input.options,
+                );
+              if (preparation === 'fence_lost')
+                return { status: 'idle' as const };
+              protectWaitingSuspensionFailure = preparation === 'prepared';
+            }
           }
           return await executePostgresSchedulerClaim(
             client,
             claim,
             async () => failedResult,
             input.detachedRecovery
-              ? { ...input.options, preserveRunnerTerminal: true }
+              ? {
+                  ...input.options,
+                  preserveRunnerTerminal: true,
+                  protectWaitingSuspensionFailure,
+                }
               : input.options,
           );
         });

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts

@@ -1086,7 +1086,13 @@ export async function withRuntimeSchedulerClient<T>(
               try {
                 const queryResult = await poolClient.query(sql, params);
                 throwIfSchedulerAborted(signal);
-                return { rows: queryResult.rows };
+                // Simple-protocol transactions return one result per command.
+                // Keep the same rows contract for callers of either protocol.
+                return {
+                  rows: Array.isArray(queryResult)
+                    ? queryResult.flatMap((result) => result.rows)
+                    : queryResult.rows,
+                };
               } catch (error) {
                 // node-postgres' query timeout and an AbortSignal reject the
                 // promise, but the server may still be processing the SQL.

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts

@@ -39,6 +39,7 @@ import { runtimeWorkCountersFromStats } from '../runtime-work-counters';
 import {
   buildPlayRunLedgerEventsFromStatusPatch,
   createEmptyPlayRunLedgerSnapshot,
+  buildTerminalLogReplayEvents,
   type PlayMapRowActivitySource,
   type PlayRunLedgerEvent,
 } from '../run-ledger';
@@ -1902,8 +1903,9 @@ export type DirectWorkerProgressReporter = PostgresSchedulerProgressReporter;
  * network boundary. The returned events are stored in the same Postgres
  * terminal outbox row as `run.completed`, so the projector can append the
  * final step/log/activity facts and the terminal fact in one ordered Convex
- * mutation. This is the completed-run fast path; failed/suspended runs retain
- * the compatibility projection until their parked worker resumes.
+ * mutation. A worker can also retain a suspended runner's complete final
+ * progress before advancing its checkpoint, using the immutable acceptance
+ * timestamp for the same outbox identity across crash or response-loss replay.
  */
 export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
   launch: PlaySchedulerSubmitInput;
@@ -1911,14 +1913,17 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
   result: PlayRunnerResult;
   events: unknown[];
   liveLogOffset: number;
+  suspensionAcceptedAtMs?: number;
 }): Promise<PlayRunLedgerEvent[]> {
   const captured: { update?: RuntimeStatusUpdate } = {};
-  const reporter = createDirectWorkerProgressReporter({
+  const retainedSuspension = input.suspensionAcceptedAtMs !== undefined;
+  const reporter = createPostgresSchedulerProgressReporter({
     launch: input.launch,
     producerAttempt: input.attempt,
     initialLiveLogTotalCount: input.liveLogOffset,
     isCurrentAttempt: () => true,
     attemptAlreadyVerified: true,
+    durableLogMode: retainedSuspension ? 'progress_reporter' : 'direct_worker',
     publishStatusUpdate: async (update) => {
       captured.update = update;
     },
@@ -1929,6 +1934,7 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
     input.launch.contractSnapshot,
   );
   if (
+    !retainedSuspension &&
     pipeline?.docflow?.nodes.some(
       (node) => node.kind === 'decision' && !node.sketch,
     )
@@ -1977,11 +1983,14 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
   await reporter.finalize(input.result, {
     status: 'running',
     required: true,
-    skipTerminalLogReplay: true,
+    skipTerminalLogReplay: !retainedSuspension,
+    replayAllTerminalLogs: retainedSuspension,
     terminalLogProducerAttempt: input.attempt,
   });
   const finalUpdate = captured.update;
   if (!finalUpdate) return [];
+  const checkpointAt =
+    input.suspensionAcceptedAtMs ?? finalUpdate.lastCheckpointAt;
 
   const liveLogs = finalUpdate.liveLogs ?? [];
   const liveLogTotalCount =
@@ -1990,7 +1999,7 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
     patch: {
       runId: input.launch.runId,
       status: 'running',
-      lastCheckpointAt: finalUpdate.lastCheckpointAt,
+      lastCheckpointAt: checkpointAt,
       liveNodeProgress: finalUpdate.liveNodeProgress,
       liveLogs,
       liveLogContexts: finalUpdate.liveLogContexts,
@@ -2011,7 +2020,7 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
       type: RUN_EVENT_KIND.workCounters,
       runId: input.launch.runId,
       source: 'worker',
-      occurredAt: finalUpdate.lastCheckpointAt ?? Date.now(),
+      occurredAt: checkpointAt ?? Date.now(),
       ...finalUpdate.workCounters,
     });
   }
@@ -2045,5 +2054,19 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
     occurredAt: observation.observedAt,
     observation,
   }));
-  return [...progressEvents, ...datasetEvents, ...activityEvents];
+  const replay = finalUpdate.terminalLogReplay;
+  const terminalLogEvents = replay
+    ? buildTerminalLogReplayEvents({
+        runId: input.launch.runId,
+        ...replay,
+        occurredAt: checkpointAt ?? undefined,
+        liveLogTotalCount: replay.totalCount,
+      })
+    : [];
+  return [
+    ...progressEvents,
+    ...datasetEvents,
+    ...activityEvents,
+    ...terminalLogEvents,
+  ];
 }