deepline 0.3.283 → 0.3.285

package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts

@@ -44,6 +44,13 @@ export type PlayRunLedgerStepStatus =
 
 export type PlayRunLedgerEventSource = RunEventSource;
 
+/** Authority of the row sample, independent of gateway process lifetime. */
+export type PlayMapRowActivitySource =
+  | 'resolver'
+  | 'map_start'
+  | 'map_event'
+  | 'log';
+
 /**
  * Live progress for a single step/node.
  *
@@ -76,6 +83,13 @@ export type PlayRunLedgerStepProgress = {
   startedRows?: number;
   activeRows?: number;
   waitingRows?: number;
+  /** Internal provenance for merging stateless progress batches. */
+  rowActivitySource?: PlayMapRowActivitySource;
+  /** Existing logical map identity, absent from legacy progress writers. */
+  mapInvocationId?: string;
+  /** Existing runner event times for pool boundaries and row observations. */
+  rowActivityStartedAt?: number;
+  rowActivityObservedAt?: number;
   completedRows?: number;
   supersededRows?: number;
   message?: string;
@@ -1073,8 +1087,112 @@ function mergeMonotonicStepProgress(input: {
   current?: PlayRunLedgerStepProgress | null;
   next: PlayRunLedgerStepProgress;
 }): PlayRunLedgerStepProgress {
-  const current = input.current ?? {};
-  const next = input.next;
+  const previous = input.current ?? {};
+  const hasMapInvocationIdentities =
+    previous.mapInvocationId !== undefined &&
+    input.next.mapInvocationId !== undefined;
+  const differentMapInvocation =
+    hasMapInvocationIdentities &&
+    previous.mapInvocationId !== input.next.mapInvocationId;
+  const sameMapInvocation =
+    previous.mapInvocationId !== undefined &&
+    previous.mapInvocationId === input.next.mapInvocationId;
+  const previousPoolStartedAt =
+    previous.rowActivityStartedAt ?? previous.rowActivityObservedAt;
+  const observesEarlierSourceTime =
+    typeof previous.rowActivityObservedAt === 'number' &&
+    typeof input.next.rowActivityObservedAt === 'number' &&
+    input.next.rowActivityObservedAt < previous.rowActivityObservedAt;
+  const replaysOlderMapBoundary =
+    hasMapInvocationIdentities &&
+    input.next.rowActivitySource === 'map_start' &&
+    (observesEarlierSourceTime ||
+      (sameMapInvocation &&
+        typeof previousPoolStartedAt === 'number' &&
+        typeof input.next.rowActivityStartedAt === 'number' &&
+        input.next.rowActivityStartedAt <= previousPoolStartedAt));
+  const replaysOlderResolverSample =
+    hasMapInvocationIdentities &&
+    input.next.rowActivitySource === 'resolver' &&
+    observesEarlierSourceTime;
+  const replaysOlderCountObservation =
+    hasMapInvocationIdentities &&
+    input.next.rowActivitySource === 'map_event' &&
+    input.next.completedAt === undefined &&
+    observesEarlierSourceTime;
+  const replaysOlderMapObservation =
+    replaysOlderMapBoundary ||
+    replaysOlderResolverSample ||
+    replaysOlderCountObservation;
+  const changesMapInvocation =
+    differentMapInvocation &&
+    !replaysOlderMapObservation &&
+    input.next.rowActivitySource !== undefined &&
+    input.next.rowActivitySource !== 'log';
+  const startsExecutionPool =
+    sameMapInvocation &&
+    !replaysOlderMapObservation &&
+    (input.next.rowActivitySource === 'map_start' ||
+      ((input.next.rowActivitySource === 'resolver' ||
+        input.next.rowActivitySource === 'map_event') &&
+        typeof previousPoolStartedAt === 'number' &&
+        typeof input.next.rowActivityStartedAt === 'number' &&
+        input.next.rowActivityStartedAt > previousPoolStartedAt));
+  const replacesInferredActivity =
+    sameMapInvocation &&
+    !replaysOlderMapObservation &&
+    input.next.rowActivitySource === 'resolver' &&
+    previous.rowActivitySource !== undefined &&
+    previous.rowActivitySource !== 'resolver';
+  // Admission belongs to the current execution pool. The logical map identity
+  // survives replay/pages, so only a witnessed pool boundary or newer typed
+  // invocation resets activity. Accepted row outcomes remain monotonic.
+  const current =
+    changesMapInvocation || startsExecutionPool || replacesInferredActivity
+      ? {
+          ...previous,
+          startedRows: undefined,
+          activeRows: undefined,
+          waitingRows: undefined,
+          rowActivitySource: undefined,
+          ...(changesMapInvocation
+            ? {
+                rowActivityStartedAt: undefined,
+                rowActivityObservedAt: undefined,
+              }
+            : {}),
+        }
+      : previous;
+  // A gateway reporter is recreated for every batch. Only the durable fold
+  // can keep a row-resolver sample authoritative when a later batch contains
+  // tool-call or page-setup diagnostics without a repeated activity sample.
+  // Lifecycle facts explicitly reset this authority; legacy absence retains
+  // the historical merge behavior instead of guessing from numeric fields.
+  const next =
+    (current.rowActivitySource === 'resolver' &&
+      input.next.rowActivitySource === 'log') ||
+    (differentMapInvocation &&
+      input.next.rowActivitySource === 'log' &&
+      (current.rowActivitySource === 'map_start' ||
+        current.rowActivitySource === 'map_event')) ||
+    replaysOlderMapObservation
+      ? {
+          ...input.next,
+          completed: current.completed,
+          total: current.total,
+          failed: current.failed,
+          startedRows: current.startedRows,
+          activeRows: current.activeRows,
+          waitingRows: current.waitingRows,
+          completedRows: current.completedRows,
+          supersededRows: current.supersededRows,
+          message: current.message,
+          rowActivitySource: current.rowActivitySource,
+          mapInvocationId: current.mapInvocationId,
+          rowActivityStartedAt: current.rowActivityStartedAt,
+          rowActivityObservedAt: current.rowActivityObservedAt,
+        }
+      : input.next;
   const supersededRows = maxProgressNumber(
     current.supersededRows,
     next.supersededRows === undefined
@@ -1084,6 +1202,7 @@ function mergeMonotonicStepProgress(input: {
   return {
     ...current,
     ...next,
+    rowActivitySource: next.rowActivitySource,
     ...(maxProgressNumber(current.completed, next.completed) !== undefined
       ? { completed: maxProgressNumber(current.completed, next.completed) }
       : {}),
@@ -1118,13 +1237,14 @@ function mergeMonotonicStepProgress(input: {
 }
 
 /**
- * Zero the in-flight row accounting on a settled step. `activeRows` (calls in
- * flight) and `waitingRows` (rows queued behind rate/concurrency limits) are
- * live-only counters: once a step is terminal nothing is in flight or waiting,
- * so leaving the last observed values renders a contradiction like
+ * Clear stale in-flight row accounting on a settled step. Once a step is
+ * terminal nothing is in flight or waiting, so leaving the last observed
+ * values renders a contradiction like
  * "status: completed · 1 in flight · waiting 29". This preserves the settled
- * counts (including supersededRows) and only clears the two live accumulators.
- * Idempotent: dropping already-absent fields is a no-op.
+ * counts (including supersededRows). A terminal map event or reconciliation
+ * explicitly witnesses zero active and waiting rows; retain those zeros for
+ * readers that follow the same counters through completion. Older writers
+ * and nonterminal activity samples retain their existing omission behavior.
  */
 function clearInFlightProgressOnTerminal(
   progress: PlayRunLedgerStepProgress | null | undefined,
@@ -1134,16 +1254,23 @@ function clearInFlightProgressOnTerminal(
     typeof progress.message === 'string'
       ? settledProgressMessage(progress.message, progress)
       : progress.message;
+  const witnessedTerminalZeroActivity =
+    progress.rowActivitySource === 'map_event' &&
+    progress.activeRows === 0 &&
+    progress.waitingRows === 0;
   if (
-    progress.activeRows === undefined &&
-    progress.waitingRows === undefined &&
+    (witnessedTerminalZeroActivity ||
+      (progress.activeRows === undefined &&
+        progress.waitingRows === undefined)) &&
     message === progress.message
   ) {
     return progress;
   }
   const cleared = { ...progress };
-  delete cleared.activeRows;
-  delete cleared.waitingRows;
+  if (!witnessedTerminalZeroActivity) {
+    delete cleared.activeRows;
+    delete cleared.waitingRows;
+  }
   if (message === undefined) delete cleared.message;
   else cleared.message = message;
   return cleared;
@@ -1222,6 +1349,29 @@ function normalizeStepProgress(
     ...(optionalFiniteNumber(value.waitingRows) !== undefined
       ? { waitingRows: optionalFiniteNumber(value.waitingRows) }
       : {}),
+    ...(value.rowActivitySource === 'resolver' ||
+    value.rowActivitySource === 'map_start' ||
+    value.rowActivitySource === 'map_event' ||
+    value.rowActivitySource === 'log'
+      ? { rowActivitySource: value.rowActivitySource }
+      : {}),
+    ...(optionalString(value.mapInvocationId)
+      ? { mapInvocationId: optionalString(value.mapInvocationId) }
+      : {}),
+    ...(optionalFiniteNumber(value.rowActivityStartedAt) !== undefined
+      ? {
+          rowActivityStartedAt: optionalFiniteNumber(
+            value.rowActivityStartedAt,
+          ),
+        }
+      : {}),
+    ...(optionalFiniteNumber(value.rowActivityObservedAt) !== undefined
+      ? {
+          rowActivityObservedAt: optionalFiniteNumber(
+            value.rowActivityObservedAt,
+          ),
+        }
+      : {}),
     ...(optionalFiniteNumber(value.completedRows) !== undefined
       ? { completedRows: optionalFiniteNumber(value.completedRows) }
       : {}),
@@ -1936,8 +2086,8 @@ export function reducePlayRunLedgerEvent(
         (status === 'completed' || status === 'failed'
           ? (completedAt ?? event.progress.updatedAt ?? occurredAt)
           : null);
-      // Once the step is settled, drop the live in-flight/waiting accumulators
-      // so a terminal node never reports active or waiting rows.
+      // Once the step is settled, clear stale live counters while retaining
+      // explicit terminal zero activity from the owning map event.
       const progress = isTerminalStepStatus(status)
         ? clearInFlightProgressOnTerminal(mergedProgress)
         : mergedProgress;
@@ -2139,6 +2289,10 @@ function progressSignature(
     startedRows: progress?.startedRows ?? null,
     activeRows: progress?.activeRows ?? null,
     waitingRows: progress?.waitingRows ?? null,
+    rowActivitySource: progress?.rowActivitySource ?? null,
+    mapInvocationId: progress?.mapInvocationId ?? null,
+    rowActivityStartedAt: progress?.rowActivityStartedAt ?? null,
+    rowActivityObservedAt: progress?.rowActivityObservedAt ?? null,
     completedRows: progress?.completedRows ?? null,
     supersededRows: progress?.supersededRows ?? null,
     message: progress?.message ?? null,
@@ -2438,6 +2592,18 @@ export function buildPlayRunLedgerEventsFromStatusPatch(input: {
       ...(progress.waitingRows !== undefined
         ? { waitingRows: progress.waitingRows }
         : {}),
+      ...(progress.rowActivitySource !== undefined
+        ? { rowActivitySource: progress.rowActivitySource }
+        : {}),
+      ...(progress.mapInvocationId !== undefined
+        ? { mapInvocationId: progress.mapInvocationId }
+        : {}),
+      ...(progress.rowActivityStartedAt !== undefined
+        ? { rowActivityStartedAt: progress.rowActivityStartedAt }
+        : {}),
+      ...(progress.rowActivityObservedAt !== undefined
+        ? { rowActivityObservedAt: progress.rowActivityObservedAt }
+        : {}),
       ...(progress.completedRows !== undefined
         ? { completedRows: progress.completedRows }
         : {}),

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts

@@ -2577,6 +2577,7 @@ async function run(
                 columnTypes: mapContext.columnTypes,
               }),
               rows: items,
+              pendingRerunAdmission: true,
               inputOffset: mapContext.inputOffset,
               mode: mapContext.mode,
               runId,

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -1839,29 +1839,10 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
         }),
       );
     } catch (error) {
-      // Resource persistence belongs to the scheduler control plane. Preserve
-      // its typed capacity/fence errors so Absurd can defer or fence the
-      // attempt; converting them into a runner failure would terminally fail a
-      // play that never began executing customer code. The sandbox exists
-      // before that durable callback can succeed, so synchronously delete it
-      // before returning the control-plane error. The old fire-and-forget
-      // cleanup path was skipped by this rethrow and leaked the acquisition.
+      // Publication can fail after it commits, or after a successor adopts
+      // this sandbox. Preserve the scheduler error and leave deletion to its
+      // fenced cleanup owner; this backend no longer has deletion authority.
       if (runtimeResourceReportErrors.has(error)) {
-        const sandbox = sandboxCleanup.currentSandbox();
-        if (sandbox) {
-          try {
-            await preparedExecution.daytona.delete(sandbox, 30);
-            console.info(
-              '[play-runner.daytona.resource_report_failure_cleanup_done]',
-              { sandboxId: sandbox.id },
-            );
-          } catch (cleanupError) {
-            throw new AggregateError(
-              [error, cleanupError],
-              `Failed to persist or delete acquired Daytona sandbox ${sandbox.id}.`,
-            );
-          }
-        }
         throw error;
       }
       if (isRuntimeSandboxCapacityLimitError(error)) {

package/dist/bundling-sources/shared_libs/play-runtime/runtime-incident-drills.ts

@@ -13,6 +13,7 @@ export const RUNTIME_TEST_FAULT_IDS = [
   'receipt_fail_write_fail',
   'worker_receipt_complete_write_fail',
   'invocation_response_delivery_abort',
+  'execution_upstream_response_loss',
   'receipt_gateway_hold_ms',
   'receipt_claim_query_hold_once_ms',
   'receipt_claim_response_timeout',
@@ -226,6 +227,31 @@ export const RUNTIME_INCIDENT_DRILL_CATALOG: readonly RuntimeIncidentDrill[] = [
       target: 'invocation-delivery-replay-e2e',
     },
   },
+  {
+    id: 'execution_upstream_response_loss',
+    description:
+      'Lose the first response of each keyed execution after the upstream app durably stores it.',
+    topologies: ['preview_fly'],
+    control: {
+      kind: 'runtime_test_fault',
+      defaultValue: 1,
+      valueMeaning: 'occurrence',
+    },
+    expected: {
+      outcome:
+        'The same invocation recovers the stored response without repeating provider work or customer charges, and every row retains its usable result.',
+      safetyFacts: [
+        'same_logical_run',
+        'bounded_provider_execution',
+        'all_rows_settle_once',
+        'no_false_success',
+      ],
+    },
+    verification: {
+      kind: 'preview_black_box',
+      target: '101-agent-heavy-gateway-load-e2e',
+    },
+  },
   {
     id: 'receipt_gateway_hold_ms',
     description: 'Hold one checked-out receipt-gateway scheduler client.',

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -175,6 +175,7 @@ import {
   readPostgresSchedulerRunnerTerminal,
   readPostgresSchedulerRunnerLiveness,
   readPostgresSchedulerRuntimeResourcesForRun,
+  isPostgresSchedulerRunnerProgressAttemptCurrent,
   requestComputeBillingDispatchToken,
   blockPostgresSchedulerCronSchedule,
   PostgresSchedulerCronScheduleConfigurationError,
@@ -2640,7 +2641,26 @@ async function executeAbsurdAttemptAdapter(
       runtimeApi,
     });
   } catch (error) {
-    if (error instanceof SuspendTask || error instanceof DeferTask) throw error;
+    if (error instanceof SuspendTask || error instanceof DeferTask) {
+      throw error;
+    }
+    if (error instanceof RuntimeResourceFenceLostError) {
+      // A rejected resource promotion can retain the current attempt fence.
+      // Its final failure must settle visibly; a successor's fence denies
+      // this terminal write before any resource settlement or cleanup.
+      await salvageAbsurdFinalAttemptTerminal({
+        ctx,
+        deps,
+        options,
+        runId,
+        launch: executionLaunch,
+        generation,
+        leaseToken,
+        error,
+        detachedRecovery: isDetachedRecovery,
+      });
+      throw error;
+    }
     let detached: Awaited<
       ReturnType<typeof resumePostgresSchedulerDetachedRunnerAttempt>
     >;
@@ -3397,6 +3417,20 @@ async function executeAssignedAbsurdAttempt(input: {
       )
     )
       return;
+    const stillOwnsAttempt = await deps.runWithClient((client) =>
+      isPostgresSchedulerRunnerProgressAttemptCurrent(
+        client,
+        { runId, attempt: generation },
+        options,
+      ),
+    );
+    if (!stillOwnsAttempt) {
+      console.info('[absurd-runtime.worker] stale_attempt_cleanup_skipped', {
+        runId,
+        attempt: generation,
+      });
+      return;
+    }
     const cleanupTerminalReason =
       error instanceof CancelledTask ? 'lease_expired' : 'runner_failed';
     await deps
@@ -3407,6 +3441,7 @@ async function executeAssignedAbsurdAttempt(input: {
             runId,
             billingEndedAt: Date.now(),
             terminalReason: cleanupTerminalReason,
+            throughAttempt: generation,
           },
           options,
         ),
@@ -3422,10 +3457,8 @@ async function executeAssignedAbsurdAttempt(input: {
               : String(settlementError),
         });
       });
-    // Sheet/receipt lease release is attempt-scoped (this generation's
-    // fences), but sandbox GC must be GENERATION-AGNOSTIC: a reclaimed prior
-    // attempt's sandbox is recorded under the OLD generation and would leak
-    // if the cleanup read only this attempt's resources.
+    // Include resources from retired attempts through this generation, while
+    // excluding a successor if one is assigned during cleanup.
     let attemptResourceReadFailed = false;
     const attemptResources = await deps
       .runWithClient((client) =>
@@ -3451,7 +3484,7 @@ async function executeAssignedAbsurdAttempt(input: {
       .runWithClient((client) =>
         readPostgresSchedulerRuntimeResourcesForRun(
           client,
-          { runId, attempt: null },
+          { runId, attempt: null, throughAttempt: generation },
           options,
         ),
       )
@@ -3462,7 +3495,7 @@ async function executeAssignedAbsurdAttempt(input: {
       await deps.runWithClient((client) =>
         enqueuePostgresSchedulerSandboxCleanupForRun(
           client,
-          { runId },
+          { runId, throughAttempt: generation },
           options,
         ),
       );
@@ -3490,6 +3523,7 @@ async function executeAssignedAbsurdAttempt(input: {
           readSandboxCrashDiagnostic: deps.readSandboxCrashDiagnostic,
           leaseOwner: `attempt-error:${runId}:${generation}`,
           runId,
+          throughAttempt: generation,
           environment,
           schedulerOptions: options,
           timeoutMs: deps.sandboxGcTimeoutMs,
@@ -3905,7 +3939,11 @@ async function executeAssignedAbsurdAttempt(input: {
                             reservationInput,
                             options,
                           );
-                  const reserve = () => deps.runWithClient(reserveOperation);
+                  const reserve = () =>
+                    deps.runWithClient(reserveOperation, {
+                      runId,
+                      phase: 'sandbox_capacity_admission',
+                    });
                   const deferForSchedulerFailure = (error: unknown): never => {
                     console.warn(
                       '[absurd-runtime.worker] sandbox_capacity_admission_deferred',
@@ -6974,6 +7012,7 @@ async function claimPostgresSchedulerSandboxCleanupBatch(input: {
   runWithClient: AbsurdRuntimeWorkerDeps['runWithClient'];
   leaseOwner: string;
   runId?: string;
+  throughAttempt?: number;
   environment: 'preview' | 'production';
   schedulerOptions: Pick<PostgresSchedulerOptions, 'schema'>;
   limit: number;
@@ -6985,6 +7024,7 @@ async function claimPostgresSchedulerSandboxCleanupBatch(input: {
       {
         leaseOwner: input.leaseOwner,
         runId: input.runId,
+        throughAttempt: input.throughAttempt,
         environment: input.environment,
         limit: input.limit,
         providers: input.providers,
@@ -7300,6 +7340,7 @@ export async function drainPostgresSchedulerSandboxCleanupOnce(input: {
   deleteRuntimeSandboxWithOutcome?: AbsurdRuntimeWorkerDeps['deleteRuntimeSandboxWithOutcome'];
   leaseOwner: string;
   runId?: string;
+  throughAttempt?: number;
   environment: 'preview' | 'production';
   schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
   limit?: number;
@@ -7322,6 +7363,7 @@ export async function drainPostgresSchedulerSandboxCleanupOnce(input: {
     runWithClient: input.runWithClient,
     leaseOwner: input.leaseOwner,
     runId: input.runId,
+    throughAttempt: input.throughAttempt,
     environment: input.environment,
     schedulerOptions: options,
     limit,