deepline 0.1.178 → 0.1.180

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -155,10 +155,10 @@ import {
 import type { RuntimeStepReceipt } from '../../../shared_libs/play-runtime/ctx-types';
 import {
   canReclaimFailedWorkerToolReceipt,
-  canReclaimTimedOutWorkerToolReceipt,
   markWorkerToolReceiptResultCached,
   markWorkerToolReceiptResultExecution,
   planWorkerToolReceiptGroups,
+  reclaimRunningReceiptGroupAfterWait,
   resolveWorkerToolReceiptGroupWaitMaxAttempts,
   resolveWorkerToolRuntimeTimeoutMs,
 } from './runtime/tool-receipts';
@@ -181,6 +181,10 @@ import {
   setHarnessBinding,
 } from '../../../sdk/src/plays/harness-stub';
 import { createHarnessWorkerReceiptStore } from './runtime/harness-receipt-store';
+import {
+  createCoordinatorProgressPublisher,
+  isCoordinatorProgressAuthorizationError,
+} from './runtime/coordinator-progress';
 import {
   hydrateSerializedResultDatasets,
   projectTerminalResultDatasets,
@@ -212,6 +216,22 @@ import {
 } from '../../../shared_libs/play-runtime/csv-rename';
 import { coordinatorRequestHeaders } from '../../../shared_libs/play-runtime/coordinator-headers';
 import { normalizePlayRunFailure } from '../../../shared_libs/play-runtime/run-failure';
+import {
+  createRuntimePersistenceLatch,
+  isRuntimePersistenceCircuitOpenError,
+  partitionDispatchWaves,
+  PROVIDER_DISPATCH_WAVE_SIZE,
+  RuntimePersistenceCircuitOpenError,
+  shouldShortenCompleteReceiptLadder,
+  tripRuntimePersistenceLatch,
+  type RuntimePersistenceLatch,
+} from '../../../shared_libs/play-runtime/persistence-latch';
+import {
+  createReceiptSalvageBuffer,
+  receiptSalvageFailureSuffix,
+  type ReceiptSalvage,
+  type ReceiptSalvageBuffer,
+} from '../../../shared_libs/play-runtime/receipt-salvage';
 import { createSecretRedactionContext } from '../../../shared_libs/play-runtime/secret-redaction';
 import {
   assertNoSecretTaint,
@@ -1735,10 +1755,23 @@ function isRuntimeReceiptPersistenceError(error: unknown): boolean {
 
 function isRunFatalWorkerRowError(error: unknown): boolean {
   return (
-    isRowIsolationExemptError(error) || isRuntimeReceiptPersistenceError(error)
+    isCoordinatorProgressAuthorizationError(error) ||
+    isRowIsolationExemptError(error) ||
+    isRuntimeReceiptPersistenceError(error) ||
+    isRuntimePersistenceCircuitOpenError(error)
   );
 }
 
+/**
+ * completeReceipt salvage retry ladder. On a persistence failure after a billed
+ * call we retry the receipt completion past the server-side 5s connect window:
+ * 1 initial attempt + 3 retries at 1s / 3s / 9s (single isolate — no jitter).
+ * Bounded by the run deadline (`runtimeDeadlineMs`); we stop early unless there
+ * is at least one backoff's worth plus headroom of budget left.
+ */
+const COMPLETE_RECEIPT_RETRY_BACKOFFS_MS = [1_000, 3_000, 9_000] as const;
+const COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS = 10_000;
+
 // Fallback batch-chunk parallelism when a tool declares no provider rate hints.
 // Matches the prior hardcoded `Math.min(4, ...)` so undeclared providers keep
 // their previous batching behavior; declared providers tighten via the
@@ -1814,8 +1847,96 @@ class WorkerToolBatchScheduler {
     private readonly receiptStore?: WorkerRuntimeReceiptStore,
     private readonly allowLocalRetryReceipts = false,
     private readonly runtimeDeadlineMs?: number,
+    // Map-wide (or ctx-wide for the root scheduler) persistence-failure circuit
+    // breaker. Shared by every row worker so the first persistence failure
+    // halts NEW provider dispatch across the whole map.
+    private readonly latch: RuntimePersistenceLatch = createRuntimePersistenceLatch(),
+    // Run-scoped buffer for orphaned billed results whose receipt could not be
+    // completed even after the retry ladder.
+    private readonly salvage?: ReceiptSalvageBuffer,
   ) {}
 
+  /**
+   * Run `completeReceipt`/`completeReceipts` with the salvage retry ladder.
+   * Retries a receipt-completion failure past the server-side 5s connect window
+   * so a transient blip does not orphan a billed result. Bounded by the run
+   * deadline; skips retries when the latch has already burned a ladder in this
+   * map so many in-flight orphans do not each serialize a long loop.
+   */
+  private async completeReceiptWithRetryLadder<T>(
+    run: () => Promise<T>,
+    onGiveUp?: (attempts: number) => void,
+  ): Promise<T> {
+    let attempt = 0;
+    // A ladder that has already started retrying keeps its full budget even if
+    // a peer trips the latch mid-flight — only FRESH entrants collapse once the
+    // failure is confirmed. This is what guarantees the first ladder rides out
+    // a genuine transient blip.
+    let committedToFullLadder = false;
+    const giveUp = (error: unknown): never => {
+      this.latch.latchRetryBudgetExhausted = true;
+      onGiveUp?.(attempt);
+      throw error;
+    };
+    while (true) {
+      try {
+        return await run();
+      } catch (error) {
+        attempt += 1;
+        if (
+          shouldShortenCompleteReceiptLadder(this.latch, committedToFullLadder) ||
+          attempt > COMPLETE_RECEIPT_RETRY_BACKOFFS_MS.length ||
+          this.abortSignal?.aborted
+        ) {
+          return giveUp(error);
+        }
+        const backoffMs = COMPLETE_RECEIPT_RETRY_BACKOFFS_MS[attempt - 1]!;
+        // resolveRuntimeDeadlineRemainingMs THROWS WorkflowAbortError once the
+        // run deadline has passed — the correct loud outcome (there is no
+        // separate per-chunk step timeout). Route that throw through giveUp so
+        // the billed result is still salvaged and the ladder budget flag is
+        // set: a billed output must never lose its durable trace because the
+        // clock ran out mid-ladder.
+        let remainingMs: number;
+        try {
+          remainingMs = resolveRuntimeDeadlineRemainingMs(
+            this.runtimeDeadlineMs,
+          );
+        } catch (deadlineError) {
+          return giveUp(deadlineError);
+        }
+        if (remainingMs <= backoffMs + COMPLETE_RECEIPT_RETRY_MIN_HEADROOM_MS) {
+          return giveUp(error);
+        }
+        // Past every give-up gate: this ladder is now committed to the full
+        // budget and must not be collapsed by a peer tripping the latch mid-sleep.
+        committedToFullLadder = true;
+        await sleepWorkerMs(backoffMs);
+      }
+    }
+  }
+
+  /**
+   * Record a billed-but-unpersisted tool output into the run-scoped salvage
+   * buffer. Called only after the completeReceipt ladder is exhausted, with the
+   * exact serialized payload a receipt would have stored.
+   */
+  private recordReceiptSalvage(input: {
+    claimed: ClaimedWorkerToolBatchRequest;
+    output: unknown;
+    attempts: number;
+  }): void {
+    if (!this.salvage || !input.claimed.receiptKey) return;
+    this.salvage.push({
+      receiptKey: input.claimed.receiptKey,
+      toolId: input.claimed.request.toolId,
+      cacheKey: input.claimed.request.cacheKey,
+      output: input.output,
+      completeAttempts: input.attempts,
+      firstErrorAt: nowMs(),
+    });
+  }
+
   /**
    * Report a provider 429 / Retry-After back into the Governor's shared pacer
    * so future acquires for this (org, provider) bucket back off across all
@@ -1858,9 +1979,7 @@ class WorkerToolBatchScheduler {
         id,
         cacheKey: receiptKey,
         receiptKey,
-        force:
-          options?.force === true ||
-          !!this.req.force,
+        force: options?.force === true || !!this.req.force,
         receiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
           typeof runtimeOptions?.receiptWaitMs === 'number'
             ? { max_wait_ms: runtimeOptions.receiptWaitMs }
@@ -2050,37 +2169,73 @@ class WorkerToolBatchScheduler {
       );
       return [];
     }
-    let waitError: unknown = null;
-    try {
-      const receipt = await this.waitForDurableToolReceipt({
-        receiptKey: input.receiptKey,
-        maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
-          input.group,
-          (groupRequest) => groupRequest.receiptWaitMaxAttempts,
-        ),
-      });
-      await this.resolveCompletedDurableToolReceiptGroup({
-        group: input.group,
-        receiptKey: input.receiptKey,
-        receipt,
-        source: 'in_flight',
-      });
-      return [];
-    } catch (error) {
-      if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
-        this.rejectRawRequests(input.group, error);
-        return [];
-      }
-      waitError = error;
-    }
-    if (
-      !canReclaimTimedOutWorkerToolReceipt({
-        ownerRunId: input.runningReceipt.runId,
-        currentRunId: this.req.runId,
-        updatedAt: input.runningReceipt.updatedAt,
-      })
-    ) {
-      this.rejectRawRequests(input.group, waitError);
+    // A `running` receipt left by a DIFFERENT run whose last write is stale will
+    // never be completed by a live worker. Polling it burns one service-binding
+    // subrequest per tick, up to the receipt's wait budget (~1320) — many such
+    // orphans on a resume exhaust Cloudflare's per-invocation subrequest budget
+    // and kill the run. The orchestrator reclaims those immediately (skipping the
+    // poll) and re-executes, matching the "on resume it reclaims and re-executes"
+    // contract; live/fresh/same-run owners still poll before reclaiming. Kept as
+    // an injected helper so the skip-poll decision is unit-testable without
+    // loading this cloudflare:workers Worker entrypoint.
+    return await reclaimRunningReceiptGroupAfterWait<ClaimedWorkerToolBatchRequest>(
+      {
+        ownership: {
+          ownerRunId: input.runningReceipt.runId,
+          currentRunId: this.req.runId,
+          updatedAt: input.runningReceipt.updatedAt,
+        },
+        timeoutError: new RuntimeReceiptWaitTimeoutError(input.receiptKey),
+        poll: async () => {
+          try {
+            const receipt = await this.waitForDurableToolReceipt({
+              receiptKey: input.receiptKey,
+              maxAttempts: resolveWorkerToolReceiptGroupWaitMaxAttempts(
+                input.group,
+                (groupRequest) => groupRequest.receiptWaitMaxAttempts,
+              ),
+            });
+            await this.resolveCompletedDurableToolReceiptGroup({
+              group: input.group,
+              receiptKey: input.receiptKey,
+              receipt,
+              source: 'in_flight',
+            });
+            return { kind: 'completed' };
+          } catch (error) {
+            if (error instanceof RuntimeReceiptWaitTimeoutError) {
+              return { kind: 'timed_out', error };
+            }
+            return { kind: 'errored', error };
+          }
+        },
+        reclaim: (rejectionOnFallthrough) =>
+          this.forceReclaimRunningDurableToolReceiptGroup({
+            group: input.group,
+            request,
+            followers,
+            receiptKey: input.receiptKey,
+            rejectionOnFallthrough,
+          }),
+        reject: (error) => this.rejectRawRequests(input.group, error),
+      },
+    );
+  }
+
+  /**
+   * Steal a `running` receipt (reclaimRunning) and route by claim disposition.
+   * Shared by the post-poll-timeout reclaim and the dead-owner fast path, so
+   * both dispose of a reclaimed/reused/failed receipt identically.
+   */
+  private async forceReclaimRunningDurableToolReceiptGroup(input: {
+    group: WorkerToolBatchRequest[];
+    request: WorkerToolBatchRequest;
+    followers: WorkerToolBatchRequest[];
+    receiptKey: string;
+    rejectionOnFallthrough: unknown;
+  }): Promise<ClaimedWorkerToolBatchRequest[]> {
+    if (!this.receiptStore) {
+      this.rejectRawRequests(input.group, input.rejectionOnFallthrough);
       return [];
     }
     let claim: WorkerRuntimeReceiptClaim;
@@ -2096,7 +2251,13 @@ class WorkerToolBatchScheduler {
       return [];
     }
     if (claim.disposition === 'claimed') {
-      return [{ request, receiptKey: input.receiptKey, followers }];
+      return [
+        {
+          request: input.request,
+          receiptKey: input.receiptKey,
+          followers: input.followers,
+        },
+      ];
     }
     if (claim.disposition === 'reused') {
       await this.resolveCompletedDurableToolReceiptGroup({
@@ -2114,7 +2275,7 @@ class WorkerToolBatchScheduler {
         failedReceipt: claim.receipt,
       });
     }
-    this.rejectRawRequests(input.group, waitError);
+    this.rejectRawRequests(input.group, input.rejectionOnFallthrough);
     return [];
   }
 
@@ -2199,6 +2360,7 @@ class WorkerToolBatchScheduler {
       await this.failDurableToolRequest(claimed, error);
       return error;
     } catch (receiptError) {
+      tripRuntimePersistenceLatch(this.latch, receiptError);
       return new RuntimeReceiptPersistenceError(
         'Tool call failed and durable receipt could not be marked failed',
         [error, receiptError],
@@ -2350,6 +2512,9 @@ class WorkerToolBatchScheduler {
         claimedRequests.push({ request, receiptKey, followers });
       }
     } catch (error) {
+      // A receipt claim failure means the tenant Postgres is already
+      // unwritable: trip the breaker so remaining groups/rows stop dispatching.
+      tripRuntimePersistenceLatch(this.latch, error);
       for (const group of receiptGroupPlan.durableGroups) {
         this.rejectRawRequests(group.requests, error);
       }
@@ -2368,12 +2533,25 @@ class WorkerToolBatchScheduler {
       kind: 'live',
       receiptKey: claimed.receiptKey,
     });
-    const completed = await this.receiptStore.completeReceipt({
-      playName: this.req.playName,
-      runId: this.req.runId,
-      key: claimed.receiptKey,
-      output: serializeDurableStepValue(ownerResult),
-    });
+    const serializedOutput = serializeDurableStepValue(ownerResult);
+    const completed = await this.completeReceiptWithRetryLadder(
+      () =>
+        this.receiptStore!.completeReceipt({
+          playName: this.req.playName,
+          runId: this.req.runId,
+          key: claimed.receiptKey!,
+          output: serializedOutput,
+        }),
+      // Ladder exhausted: the call billed but the receipt could not land.
+      // Salvage the exact serialized output onto the run record before the
+      // caller turns this into a RuntimeReceiptPersistenceError.
+      (attempts) =>
+        this.recordReceiptSalvage({
+          claimed,
+          output: serializedOutput,
+          attempts,
+        }),
+    );
     if (
       completed &&
       (completed.status === 'completed' || completed.status === 'skipped') &&
@@ -2449,14 +2627,32 @@ class WorkerToolBatchScheduler {
       );
       return results;
     }
-    const completed = await this.receiptStore.completeReceipts({
-      playName: this.req.playName,
-      receipts: durableEntries.map((entry) => ({
-        runId: this.req.runId,
-        key: entry.claimed.receiptKey!,
-        output: serializeDurableStepValue(entry.ownerResult),
-      })),
-    });
+    const serializedByEntry = durableEntries.map((entry) =>
+      serializeDurableStepValue(entry.ownerResult),
+    );
+    const completed = await this.completeReceiptWithRetryLadder(
+      () =>
+        this.receiptStore!.completeReceipts!({
+          playName: this.req.playName,
+          receipts: durableEntries.map((entry, entryIndex) => ({
+            runId: this.req.runId,
+            key: entry.claimed.receiptKey!,
+            output: serializedByEntry[entryIndex],
+          })),
+        }),
+      // Ladder exhausted: every entry in this batch billed but could not be
+      // marked completed. Salvage each serialized output before the caller
+      // turns this into a RuntimeReceiptPersistenceError.
+      (attempts) => {
+        durableEntries.forEach((entry, entryIndex) => {
+          this.recordReceiptSalvage({
+            claimed: entry.claimed,
+            output: serializedByEntry[entryIndex],
+            attempts,
+          });
+        });
+      },
+    );
     durableEntries.forEach((entry, resultIndex) => {
       results[entry.index] = this.completedDurableToolRequestResult({
         claimed: entry.claimed,
@@ -2510,6 +2706,17 @@ class WorkerToolBatchScheduler {
     toolId: string,
     requests: WorkerToolBatchRequest[],
   ): Promise<void> {
+    // Circuit breaker, gated BEFORE the receipt claim: never-dispatched rows
+    // get NO receipt (not even `running`), so a re-run claims them fresh and
+    // executes them exactly once.
+    if (this.latch.tripped) {
+      this.latch.preventedCallCount += requests.length;
+      this.rejectRawRequests(
+        requests,
+        new RuntimePersistenceCircuitOpenError(this.latch),
+      );
+      return;
+    }
     const { claimedRequests, deferredClaimedRequests } =
       await this.prepareDurableToolRequests(requests);
     await this.executeClaimedToolRequests(
@@ -2545,9 +2752,23 @@ class WorkerToolBatchScheduler {
       claimedRequests.length < 2
     ) {
       const groupStartedAt = nowMs();
-      await Promise.all(
-        claimedRequests.map(async (claimed) => {
+      const dispatchOne = async (
+        claimed: ClaimedWorkerToolBatchRequest,
+      ): Promise<void> => {
           const { request } = claimed;
+          // Circuit breaker: the latch may have tripped after this group was
+          // claimed but before this particular call started. Do NOT call the
+          // provider. This row has a `running` receipt (claimed, never
+          // executed); on resume it reclaims and re-executes — safe, it never
+          // billed.
+          if (this.latch.tripped) {
+            this.latch.preventedCallCount += 1 + claimed.followers.length;
+            this.rejectRequests(
+              claimed,
+              new RuntimePersistenceCircuitOpenError(this.latch),
+            );
+            return;
+          }
           const toolContract = await this.resolvePacing(toolId).catch(
             () => null,
           );
@@ -2586,6 +2807,7 @@ class WorkerToolBatchScheduler {
               await this.completeDurableToolRequest(claimed, result),
             );
           } catch (receiptError) {
+            tripRuntimePersistenceLatch(this.latch, receiptError);
             this.rejectRequests(
               claimed,
               new RuntimeReceiptPersistenceError(
@@ -2596,8 +2818,32 @@ class WorkerToolBatchScheduler {
           } finally {
             slot.release();
           }
-        }),
-      );
+      };
+      // Bounded dispatch waves: an unbatched tool group can carry a whole chunk's
+      // rows (enrich compiles to one ~245-row chunk). Dispatching them all at
+      // once billed every call before the first receipt-completion failure could
+      // trip the latch (the 250/250 incident). Instead, dispatch one wave, AWAIT
+      // it so any completion failure trips the latch, then read the latch before
+      // the next wave — remaining waves are prevented (counted + rejected) exactly
+      // like the mid-wave gate above. The wave calls stay fully concurrent; the
+      // only added cost on the healthy path is one latch read per wave.
+      for (const wave of partitionDispatchWaves(
+        claimedRequests,
+        () => 1,
+        PROVIDER_DISPATCH_WAVE_SIZE,
+      )) {
+        if (this.latch.tripped) {
+          for (const claimed of wave) {
+            this.latch.preventedCallCount += 1 + claimed.followers.length;
+            this.rejectRequests(
+              claimed,
+              new RuntimePersistenceCircuitOpenError(this.latch),
+            );
+          }
+          continue;
+        }
+        await Promise.all(wave.map(dispatchOne));
+      }
       recordRunnerPerfTrace({
         req: this.req,
         phase: 'runner.tool.group',
@@ -2635,6 +2881,7 @@ class WorkerToolBatchScheduler {
         await this.failDurableToolRequests(claimedRequests, error),
       settleRequest: (claimed, result) => this.settleRequests(claimed, result),
       rejectRequest: (claimed, error) => this.rejectRequests(claimed, error),
+      latch: this.latch,
     });
     recordRunnerPerfTrace({
       req: this.req,
@@ -2692,6 +2939,7 @@ async function executeBatchedWorkerToolGroup(input: {
     request: ClaimedWorkerToolBatchRequest,
     error: unknown,
   ) => void;
+  latch: RuntimePersistenceLatch;
 }): Promise<void> {
   const compiledBatches = compileRequestsWithStrategy({
     requests: input.requests,
@@ -2719,7 +2967,20 @@ async function executeBatchedWorkerToolGroup(input: {
       1,
       Math.min(input.suggestedParallelism, compiledBatches.length || 1),
     ),
+    // Bounded dispatch wave: cap the provider calls (counted by batch member
+    // size) a single chunk dispatches before the latch is read again, so a
+    // persistence failure prevents the rest of a large batch group instead of
+    // billing it all. A single over-budget batch still forms its own wave.
+    maxChunkWeight: PROVIDER_DISPATCH_WAVE_SIZE,
+    weightOf: (batch) => batch.memberRequests.length,
     execute: async (batch) => {
+      // Circuit breaker: once a persistence failure has occurred in this map,
+      // do NOT dispatch this batch's provider call. The chunk plumbing rejects
+      // its member requests with the latch error.
+      if (input.latch.tripped) {
+        input.latch.preventedCallCount += batch.memberRequests.length;
+        throw new RuntimePersistenceCircuitOpenError(input.latch);
+      }
       const toolContract = await input
         .resolveToolContract(batch.batchOperation)
         .catch(() => null);
@@ -2768,6 +3029,7 @@ async function executeBatchedWorkerToolGroup(input: {
           try {
             await input.failRequests(entry.request.memberRequests, entry.error);
           } catch (receiptError) {
+            tripRuntimePersistenceLatch(input.latch, receiptError);
             rejection = new RuntimeReceiptPersistenceError(
               'Tool call failed and durable receipts could not be marked failed',
               [entry.error, receiptError],
@@ -2798,6 +3060,7 @@ async function executeBatchedWorkerToolGroup(input: {
             })),
           );
         } catch (receiptError) {
+          tripRuntimePersistenceLatch(input.latch, receiptError);
           const rejection = new RuntimeReceiptPersistenceError(
             'Tool call succeeded but durable receipts could not be marked completed',
             [receiptError],
@@ -2819,6 +3082,7 @@ async function executeBatchedWorkerToolGroup(input: {
     try {
       await input.failRequests(input.requests, error);
     } catch (receiptError) {
+      tripRuntimePersistenceLatch(input.latch, receiptError);
       rejection = new RuntimeReceiptPersistenceError(
         'Tool call failed and durable receipts could not be marked failed',
         [error, receiptError],
@@ -4147,10 +4411,15 @@ async function prepareMapRows(input: {
 
 function assertNotAborted(signal: AbortSignal | undefined): void {
   if (signal?.aborted) {
+    if (isCoordinatorProgressAuthorizationError(signal.reason)) {
+      throw signal.reason;
+    }
     throw new WorkflowAbortError(
-      typeof signal.reason === 'string' && signal.reason
-        ? signal.reason
-        : 'Play run cancelled.',
+      signal.reason instanceof Error && signal.reason.message
+        ? signal.reason.message
+        : typeof signal.reason === 'string' && signal.reason
+          ? signal.reason
+          : 'Play run cancelled.',
     );
   }
 }
@@ -4478,6 +4747,7 @@ function createMinimalWorkerCtx(
   abortSignal?: AbortSignal,
   callbacks?: WorkerCtxCallbacks,
   runtimeDeadlineMs?: number,
+  receiptSalvage?: ReceiptSalvageBuffer,
 ): unknown {
   const { governor, resolvePacing: resolveToolPacing } =
     createGovernorForRun(req);
@@ -4591,6 +4861,10 @@ function createMinimalWorkerCtx(
       staleAfterSeconds,
     });
   };
+  // Ctx-wide breaker for top-level ctx.tool calls (outside any map). Each map
+  // creates its own map-scoped latch (see runMap) so per-map dispatch stats and
+  // the runMap failure message stay map-local.
+  const rootPersistenceLatch = createRuntimePersistenceLatch();
   const rootToolBatchScheduler = new WorkerToolBatchScheduler(
     req,
     governor,
@@ -4602,6 +4876,8 @@ function createMinimalWorkerCtx(
     receiptStore,
     true,
     runtimeDeadlineMs,
+    rootPersistenceLatch,
+    receiptSalvage,
   );
   // Local ancestry chain that always ENDS with the currently-executing play
   // (req.playName). The /api/v2/plays/run lineage validator requires the
@@ -4637,6 +4913,12 @@ function createMinimalWorkerCtx(
   ): Promise<unknown> => {
     const mapStartedAt = nowMs();
     const mapNodeId = `map:${name}`;
+    // Map-wide persistence-failure circuit breaker. Shared across every chunk's
+    // scheduler (the scheduler is rebuilt per chunk) and the map's sheet-persist
+    // flush chain, so the first persistence failure halts NEW provider dispatch
+    // for the rest of the map and its prevented-call count surfaces in the
+    // runMap failure message below.
+    const persistenceLatch = createRuntimePersistenceLatch();
     const inputRows = rows;
     const rowCountHint = datasetRowCountHint(inputRows);
     const baseOffset = 0;
@@ -5080,6 +5362,8 @@ function createMinimalWorkerCtx(
         receiptStore,
         false,
         runtimeDeadlineMs,
+        persistenceLatch,
+        receiptSalvage,
       );
       let stepCellsCompleted = 0;
       let stepCellsSkipped = 0;
@@ -5222,6 +5506,9 @@ function createMinimalWorkerCtx(
         });
         persistFlushChain = task.catch((error) => {
           persistFailure ??= error;
+          // Output-sheet flush failed: trip the breaker so the scheduler stops
+          // dispatching new provider calls for the rest of this map.
+          tripRuntimePersistenceLatch(persistenceLatch, error);
         });
         return task;
       };
@@ -5370,11 +5657,18 @@ function createMinimalWorkerCtx(
                         request.input,
                         workflowStep,
                         {
-                          force:
-                            request.force === true ||
-                            !!req.force,
+                          force: request.force === true || !!req.force,
                           staleAfterSeconds: request.staleAfterSeconds,
                         },
+                        // Parity with the play-level ctx.tools.execute (and the
+                        // cjs runner's map path): the public ToolExecutionRequest
+                        // runtime options must not be silently dropped for map
+                        // rows — receiptWaitMs bounds how long a resumed run
+                        // waits on a stuck `running` receipt before reclaiming.
+                        {
+                          timeoutMs: request.timeoutMs,
+                          receiptWaitMs: request.receiptWaitMs,
+                        },
                       );
                     },
                   },
@@ -5668,7 +5962,13 @@ function createMinimalWorkerCtx(
           result.status === 'rejected',
       );
       if (rejectedWorker) {
-        if (isRuntimeReceiptPersistenceError(rejectedWorker.reason)) {
+        if (
+          isRuntimeReceiptPersistenceError(rejectedWorker.reason) ||
+          isRuntimePersistenceCircuitOpenError(rejectedWorker.reason)
+        ) {
+          // Persistence-failure / circuit-breaker rejections are returned as a
+          // fatal summary (NOT rethrown into the durable step) so the chunk is
+          // not retried and provider calls are not re-billed.
           return await fatalMapChunkSummary(rejectedWorker.reason);
         }
         throw rejectedWorker.reason;
@@ -5872,7 +6172,7 @@ function createMinimalWorkerCtx(
           ? ` cells=${totalStepCells}/${totalStepCellsCompleted}/${totalStepCellsSkipped}`
           : '';
       callbacks?.onMapCompleted?.(mapNodeId, completedAt);
-      void updateMapProgress({
+      await updateMapProgress({
         completed: totalRowsWritten,
         total: totalRowsWritten + totalRowsFailed,
         failed: totalRowsFailed,
@@ -5975,10 +6275,16 @@ function createMinimalWorkerCtx(
       }
       const chunkResult = await runChunkStep(chunkRows, chunkStart, chunkIndex);
       if (chunkResult.fatalError) {
+        const circuitBreakerSentence =
+          persistenceLatch.tripped && persistenceLatch.preventedCallCount > 0
+            ? `Circuit breaker prevented ${persistenceLatch.preventedCallCount} queued provider call(s) ` +
+              `from dispatching after the first persistence failure. `
+            : '';
         throw new Error(
           `ctx.dataset("${name}") stopped after a runtime persistence failure ` +
             `outside the retryable chunk step. Provider calls already executed for ` +
             `${chunkResult.rowsExecuted} row(s), so the chunk was not retried. ` +
+            circuitBreakerSentence +
             `Fix the runtime persistence cause and re-run to resume. ` +
             `First error: ${chunkResult.fatalError}`,
         );
@@ -6360,9 +6666,7 @@ function createMinimalWorkerCtx(
           request.input,
           workflowStep,
           {
-            force:
-              request.force === true ||
-              !!req.force,
+            force: request.force === true || !!req.force,
             staleAfterSeconds: request.staleAfterSeconds,
           },
           {
@@ -7196,9 +7500,6 @@ async function executeRunRequest(
   let lastLedgerFlushAt = startedAt;
   let ledgerFlushInFlight: Promise<void> = Promise.resolve();
   let ledgerFlushQueueDepth = 0;
-  let lastCoordinatorProgressPublishAt = 0;
-  let coordinatorProgressPublishInFlight: Promise<void> = Promise.resolve();
-  let coordinatorProgressPublishQueueDepth = 0;
 
   const appendRunLogLine = (line: string) => {
     const trimmed = redactSecretsFromLogString(line.trim());
@@ -7225,130 +7526,24 @@ async function executeRunRequest(
   };
 
   const stepProgressSnapshot = () => ({ ...stepProgressByNodeId });
-
-  const publishCoordinatorProgressEvent = async (
-    occurredAt: number,
-  ): Promise<void> => {
-    const coordinatorUrl = req.coordinatorUrl?.trim();
-    if (!coordinatorUrl) {
-      recordRunnerPerfTrace({
-        req,
-        phase: 'runner.coordinator_progress_publish',
-        ms: 0,
-        extra: { status: 'skipped_no_url' },
-      });
-      return;
-    }
-    const publishStartedAt = nowMs();
-    const liveNodeProgress = stepProgressSnapshot();
-    const activeEntry =
-      Object.entries(liveNodeProgress).find(
-        ([, progress]) => typeof progress.completedAt !== 'number',
-      ) ?? Object.entries(liveNodeProgress).at(-1);
-    const activeNodeId = activeEntry?.[0] ?? null;
-    const activeProgress = activeEntry?.[1] ?? null;
-    const activeArtifactTableNamespace =
-      typeof activeProgress?.artifactTableNamespace === 'string'
-        ? activeProgress.artifactTableNamespace
-        : null;
-    const activeCompleted =
-      typeof activeProgress?.completed === 'number'
-        ? activeProgress.completed
-        : null;
-    const activeTotal =
-      typeof activeProgress?.total === 'number' ? activeProgress.total : null;
-    const activeMessage =
-      typeof activeProgress?.message === 'string'
-        ? activeProgress.message
-        : null;
-    const response = await fetch(
-      `${coordinatorUrl.replace(/\/$/, '')}/dedup/${encodeURIComponent(
-        req.runId,
-      )}/event-add`,
-      {
-        method: 'POST',
-        headers: {
-          'x-deepline-request-id': makeRequestId(),
-          ...coordinatorRequestHeaders({
-            runId: req.runId,
-            contentType: 'application/json',
-            internalToken: req.coordinatorInternalToken,
-          }),
-        },
-        body: JSON.stringify({
-          runId: req.runId,
-          type: 'progress',
-          status: 'running',
-          ts: occurredAt,
-          logs: runLogBuffer,
-          activeNodeId,
-          activeArtifactTableNamespace,
-          updatedAt: occurredAt,
-          liveNodeProgress,
-        }),
-      },
-    );
-    if (!response.ok) {
+  const coordinatorProgressPublisher = createCoordinatorProgressPublisher({
+    runId: req.runId,
+    coordinatorUrl: req.coordinatorUrl,
+    coordinatorInternalToken: req.coordinatorInternalToken,
+    abortController,
+    getRunLogBuffer: () => runLogBuffer,
+    getLiveNodeProgress: stepProgressSnapshot,
+    makeRequestId,
+    nowMs,
+    heartbeatIntervalMs: MAP_EXECUTION_HEARTBEAT_INTERVAL_MS,
+    recordTrace: (trace) =>
       recordRunnerPerfTrace({
         req,
-        phase: 'runner.coordinator_progress_publish',
-        ms: nowMs() - publishStartedAt,
-        extra: {
-          status: 'failed',
-          httpStatus: response.status,
-          activeNodeId,
-          activeArtifactTableNamespace,
-          activeCompleted,
-          activeTotal,
-          activeMessage,
-        },
-      });
-      throw new Error(`coordinator progress event failed ${response.status}`);
-    }
-    recordRunnerPerfTrace({
-      req,
-      phase: 'runner.coordinator_progress_publish',
-      ms: nowMs() - publishStartedAt,
-      extra: {
-        status: 'ok',
-        activeNodeId,
-        activeArtifactTableNamespace,
-        activeCompleted,
-        activeTotal,
-        activeMessage,
-      },
-    });
-  };
-
-  const flushCoordinatorProgressEvent = (force: boolean): Promise<void> => {
-    const now = nowMs();
-    if (
-      !force &&
-      now - lastCoordinatorProgressPublishAt <
-        MAP_EXECUTION_HEARTBEAT_INTERVAL_MS
-    ) {
-      return Promise.resolve();
-    }
-    if (!force && coordinatorProgressPublishQueueDepth > 0) {
-      return Promise.resolve();
-    }
-    lastCoordinatorProgressPublishAt = now;
-    coordinatorProgressPublishQueueDepth += 1;
-    coordinatorProgressPublishInFlight = coordinatorProgressPublishInFlight
-      .catch(() => undefined)
-      .then(async () => {
-        try {
-          await publishCoordinatorProgressEvent(now);
-        } finally {
-          coordinatorProgressPublishQueueDepth = Math.max(
-            0,
-            coordinatorProgressPublishQueueDepth - 1,
-          );
-        }
-      })
-      .catch(() => undefined);
-    return force ? coordinatorProgressPublishInFlight : Promise.resolve();
-  };
+        phase: trace.phase,
+        ms: trace.ms,
+        extra: trace.extra,
+      }),
+  });
 
   const appendStepLifecycleEvent = (event: PlayStepLifecycleEvent) => {
     updateStepProgress({
@@ -7537,7 +7732,7 @@ async function executeRunRequest(
       updateStepProgress(input);
       const force = Boolean(input.forceFlush);
       const ledgerFlush = flushLedgerEvents(force);
-      const progressFlush = flushCoordinatorProgressEvent(force);
+      const progressFlush = coordinatorProgressPublisher.flush(force);
       return force
         ? Promise.all([ledgerFlush, progressFlush]).then(() => undefined)
         : Promise.resolve();
@@ -7576,6 +7771,11 @@ async function executeRunRequest(
   stepLifecycle?.markPreDatasetStepsStarted(startedAt);
   flushLedgerEvents(false);
   const runtimeDeadlineMs = nowMs() + STANDARD_PLAY_RUNTIME_LIMIT_MS;
+  // Run-scoped buffer for provider results that billed but whose durable
+  // receipt could not be marked completed even after the retry ladder. On
+  // terminal failure the buffer is built into a size-capped salvage record and
+  // attached to the run.failed ledger event so billed data is never memory-only.
+  const receiptSalvage = createReceiptSalvageBuffer();
   const ctx = createMinimalWorkerCtx(
     req,
     wrappedEmit,
@@ -7584,6 +7784,7 @@ async function executeRunRequest(
     abortSignal,
     workerCallbacks,
     runtimeDeadlineMs,
+    receiptSalvage,
   );
   // Hard wall-clock cap on active user-code runtime. CF Workflows does not
   // impose a play-level execution ceiling on this substrate, so without this a
@@ -7778,8 +7979,16 @@ async function executeRunRequest(
     stepLifecycle?.markStartedFailed(nowMs());
     // A runtime-limit abort is a timeout failure, not a user cancellation, so
     // it should be reported as run.failed with the limit message rather than
-    // run.cancelled.
-    const aborted = isAbortLikeError(error) && !runtimeLimitExceeded;
+    // run.cancelled. The deadline can also surface as a WorkflowAbortError
+    // thrown synchronously from the completeReceipt retry ladder BEFORE the
+    // runtimeDeadlineTimer macrotask flips the flag — match the limit message
+    // directly so that race can never misclassify a timeout as a cancel (which
+    // would silently drop the salvage buffer of billed-but-unpersisted rows).
+    const runtimeLimitError =
+      runtimeLimitExceeded ||
+      (error instanceof Error &&
+        error.message === STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
+    const aborted = isAbortLikeError(error) && !runtimeLimitError;
     if (aborted) {
       // Flip the controller so any concurrent user code observes the abort
       // through ctx.signal. We mark the run cancelled instead of failed.
@@ -7788,7 +7997,15 @@ async function executeRunRequest(
       );
     }
     const failure = normalizePlayRunFailure(error);
-    const message = failure.message;
+    // Billed-but-unpersisted salvage rides on the run.failed event's `result`.
+    // The run STILL FAILS — salvage is a loud marker, never a success fallback.
+    const salvage: ReceiptSalvage | null = aborted
+      ? null
+      : receiptSalvage.build();
+    const message =
+      salvage && salvage.totalEntries > 0
+        ? `${failure.message} ${receiptSalvageFailureSuffix(salvage)}`
+        : failure.message;
     const errorBilling = extractErrorBilling(error);
     if (options?.persistResultDatasets) {
       appendRunLogLine(
@@ -7817,6 +8034,7 @@ async function executeRunRequest(
                   ...(failure.cause ? { cause: failure.cause } : {}),
                 },
               ],
+              ...(salvage && salvage.totalEntries > 0 ? { salvage } : {}),
             },
       });
       recordRunnerPerfTrace({