deepline 0.3.183 → 0.3.185

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -104,12 +104,12 @@ import {
   type PacingResolver,
   type PlayExecutionGovernor,
 } from './governor/governor';
-import { resolveExecutionPolicy } from './governor/policy';
 import {
   createRuntimeResourceGovernor,
   type RuntimeResourceGovernor,
   type RuntimeResourceObservation,
   type RuntimeResourceSnapshot,
+  type RuntimeProviderPermit,
 } from './resource-governor';
 import { CTX_FETCH_EGRESS_TOOL_ID } from './builtin-pacing';
 import {
@@ -192,6 +192,8 @@ import {
   TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS,
   TOOL_EXECUTE_RUNNER_ACCEPTS_HEADER,
   TOOL_EXECUTE_RUNNER_ACCEPTS_VALUE,
+  TOOL_EXECUTE_PROVIDER_RATE_SCOPE_HEADER,
+  TOOL_EXECUTE_PROVIDER_RATE_RETRY_SNAPSHOT_HEADER,
   classifyToolExecuteHttpFailure,
   createToolExecuteHttpFailureAttemptTracker,
   parseToolExecuteRetryAfterMs,
@@ -358,6 +360,7 @@ import { DISALLOWED_RUN_JAVASCRIPT_TOOL_MESSAGE } from '../plays/runtime-constra
 import {
   PlayExecutionSuspendedError,
   PlayRowExecutionSuspendedError,
+  PlayRowProviderBackpressureError,
   isPlayExecutionSuspendedError,
   isPlayRowExecutionSuspendedError,
   type PlayExecutionSuspension,
@@ -1374,10 +1377,11 @@ export async function reconcileNodeRuntimeMapResultsWithPersistedSheet(input: {
   expectedRows: number;
   currentRows: Record<string, unknown>[];
   failedRowCount: number;
+  rowMode?: 'all' | 'output';
   readPersistedRows: (input: {
     limit: number;
     offset: number;
-    rowMode: 'all';
+    rowMode: 'all' | 'output';
   }) => Promise<Record<string, unknown>[]>;
   log?: (line: string) => void;
 }): Promise<Record<string, unknown>[]> {
@@ -1395,7 +1399,7 @@ export async function reconcileNodeRuntimeMapResultsWithPersistedSheet(input: {
     const page = await input.readPersistedRows({
       limit: Math.min(10_000, input.expectedRows - persistedRows.length),
       offset,
-      rowMode: 'all',
+      rowMode: input.rowMode ?? 'all',
     });
     if (page.length === 0) break;
     persistedRows.push(...page);
@@ -1924,6 +1928,8 @@ type ToolExecutionApiOptions = {
    * inside a row's async scope, where the scope is read from there instead.
    */
   memoryLedgerRowScopes?: readonly (MemoryLedgerRowScope | null | undefined)[];
+  /** Runner-local map ownership; this must never be serialized to the gateway. */
+  providerBackpressureMapRows?: boolean;
   /**
    * Present only on a native batch's one physical call. The members share a
    * single body whose per-member sizes the splitter has not produced yet, so
@@ -2512,6 +2518,7 @@ class RuntimeDatasetBuilder<T extends Record<string, unknown>> {
 }
 
 const WAITING_ROW = Symbol('deepline.waiting_row');
+
 const FAILED_ROW = Symbol('deepline.failed_row');
 const COMPLETED_ROW = Symbol('deepline.completed_row');
 type MapRowExecutionResult =
@@ -2604,6 +2611,9 @@ function cloneMapFrame(frame: MapExecutionFrame): MapExecutionFrame {
           suspendedInputRows: frame.suspendedInputRows.map((row) => ({
             inputIndex: row.inputIndex,
             data: cloneCsvAliasedRow(row.data),
+            ...(row.authoredInputDigest
+              ? { authoredInputDigest: row.authoredInputDigest }
+              : {}),
           })),
         }
       : {}),
@@ -2613,7 +2623,8 @@ function cloneMapFrame(frame: MapExecutionFrame): MapExecutionFrame {
 /**
  * Adapt this runtime's per-tool queue hints into the Play Execution Governor's
  * pacing resolver. The Governor owns per-(org, provider) pacing; this only maps
- * the existing queue-hint metadata into the Governor's `PacingRule` shape.
+ * declared hints into `PacingRule`s. An empty rule list deliberately leaves
+ * the substrate-specific default to the Governor.
  */
 function createPacingResolver(
   getToolQueueHints: ContextOptions['getToolQueueHints'],
@@ -2627,10 +2638,7 @@ function createPacingResolver(
     if (declared) return declared;
     const provider = (await getToolProvider?.(toolId))?.trim();
     if (!provider) return null;
-    return {
-      ...defaultPacingForTool(toolId, resolveExecutionPolicy('cjs_node20')),
-      provider,
-    };
+    return { provider, rules: [] };
   };
 }
 
@@ -2817,6 +2825,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     boundaryId: string;
     eventKey: string;
     timeoutMs: number;
+    eventExpiresAtMs?: number;
     messageRef?: { channel: string; ts: string };
   }> = [];
   private processedRowCount = 0;
@@ -6726,6 +6735,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 fieldName: input.request.fieldName,
               },
             ],
+            providerBackpressureMapRows: Boolean(
+              input.request.tableNamespace && input.request.rowKey,
+            ),
             // Same reason, for the delivery clock: this request really did
             // make a provider call, so its row's delivery site has a body
             // completion to measure from. A retry re-stamps; the last body is
@@ -8578,10 +8590,36 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     // before a stateful source query can decide that a sibling action is no
     // longer eligible because an earlier action already completed.
     const replayMapInvocationId = `${normalizedMapNamespace}:${this.mapInvocationIndex}`;
+    const replayMapFrame = this.checkpoint.mapFrames?.[replayMapInvocationId];
+    const canReadPersistedMapResults = Boolean(
+      this.#options.onMapRowsCompleted &&
+      this.#options.dbSessionStrategy &&
+      this.#options.baseUrl &&
+      this.#options.executorToken &&
+      this.#options.playName &&
+      this.#options.runId,
+    );
+    // A suspended map's admitted pending inputs remain authoritative even
+    // when a stateful authored source returns fewer rows after a wake.
     const suspendedInputRows = shouldStreamRuntimeBackedDataset
       ? null
-      : (this.checkpoint.mapFrames?.[replayMapInvocationId]
-          ?.suspendedInputRows ?? null);
+      : (replayMapFrame?.suspendedInputRows ?? null);
+    // Completed siblings require either the Runtime Sheet or a complete
+    // authored replay. Never return only the pending subset as a full result.
+    const replayWithoutSheet = Boolean(
+      suspendedInputRows?.length && !canReadPersistedMapResults,
+    );
+    const resumedPendingRowsOnly = Boolean(
+      suspendedInputRows?.length &&
+      (canReadPersistedMapResults ||
+        (replayMapFrame?.completedRowKeys.length ?? 0) === 0),
+    );
+    const previouslyCompletedRowKeys = resumedPendingRowsOnly
+      ? (replayMapFrame?.completedRowKeys ?? [])
+      : [];
+    const previouslyFailedRowsCount = resumedPendingRowsOnly
+      ? (replayMapFrame?.failedRowsCount ?? 0)
+      : 0;
     if (shouldStreamRuntimeBackedDataset) {
       // Net-new admission happens one source page at a time. Its progress and
       // returned dataset describe admitted rows, not every provider/CSV row
@@ -9392,18 +9430,19 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         },
       });
     }
-    const rawMaterializedEntries = suspendedInputRows?.length
-      ? suspendedInputRows.map((row) => ({
-          item: cloneCsvAliasedRow(row.data) as T,
-          inputIndex: row.inputIndex,
-        }))
-      : (
-          await materializePlayDatasetInput(items, {
-            onRow: (row) => {
-              materializationMemoryTracker.track(row);
-            },
-          })
-        ).map((item, inputIndex) => ({ item, inputIndex }));
+    const rawMaterializedEntries =
+      resumedPendingRowsOnly && suspendedInputRows
+        ? suspendedInputRows.map((row) => ({
+            item: cloneCsvAliasedRow(row.data) as T,
+            inputIndex: row.inputIndex,
+          }))
+        : (
+            await materializePlayDatasetInput(items, {
+              onRow: (row) => {
+                materializationMemoryTracker.track(row);
+              },
+            })
+          ).map((item, inputIndex) => ({ item, inputIndex }));
     const rawMaterializedItems = rawMaterializedEntries.map(
       (entry) => entry.item,
     );
@@ -9434,7 +9473,105 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       );
     }
     const materializedItems = dedupedMaterialized.rows;
-    totalInputCount = materializedItems.length;
+    const priorAuthoredDigestsByIndex = new Map(
+      (suspendedInputRows ?? []).flatMap((row) =>
+        row.authoredInputDigest
+          ? [[row.inputIndex, row.authoredInputDigest] as const]
+          : [],
+      ),
+    );
+    const authoredInputDigestsByIndex = new Map<number, string>();
+    if (!canReadPersistedMapResults) {
+      for (const [index, item] of materializedItems.entries()) {
+        const inputIndex = originalInputIndexByItem.get(item) ?? index;
+        const priorDigest = priorAuthoredDigestsByIndex.get(inputIndex);
+        // A legacy pending-only checkpoint contains post-merge row data, not
+        // the original authored input. Do not manufacture a digest from it.
+        if (resumedPendingRowsOnly) {
+          if (priorDigest)
+            authoredInputDigestsByIndex.set(inputIndex, priorDigest);
+          continue;
+        }
+        const authored = Object.fromEntries(
+          Object.entries(
+            toSerializableCsvAliasedRow(
+              this.toOutputRow(item as Record<string, unknown>),
+            ),
+          ).filter(([field]) => !field.startsWith('__deepline')),
+        );
+        authoredInputDigestsByIndex.set(
+          inputIndex,
+          stableDigest(stableStringify(authored)),
+        );
+      }
+    }
+    if (replayWithoutSheet && !resumedPendingRowsOnly) {
+      // A no-sheet replay can only rebuild completed output by executing the
+      // same admitted input again. The previous failed row's identity/output
+      // is not in the checkpoint, so re-executing it would change its terminal
+      // outcome; require a readable sheet instead of silently reclassifying it.
+      if ((replayMapFrame?.failedRowsCount ?? 0) > 0) {
+        throw new Error(
+          `Cannot reconstruct the complete result for ctx.dataset("${normalizedMapNamespace}") after suspension: ` +
+            'a prior failed row has no readable Runtime Sheet.',
+        );
+      }
+      const authoredRowsByKey = new Map(
+        materializedItems.map((item, index) => {
+          const inputIndex = originalInputIndexByItem.get(item) ?? index;
+          const row = this.toOutputRow(item as Record<string, unknown>);
+          return [
+            rowIdentity(row, inputIndex),
+            this.toPersistedOutputRow(row),
+          ] as const;
+        }),
+      );
+      const missingCompleted = replayMapFrame!.completedRowKeys.some(
+        (key) => !authoredRowsByKey.has(key),
+      );
+      const changedPending = suspendedInputRows!.some(
+        ({ data, inputIndex, authoredInputDigest }) => {
+          const authored = authoredRowsByKey.get(rowIdentity(data, inputIndex));
+          if (!authored) return true;
+          if (authoredInputDigest) {
+            return (
+              authoredInputDigestsByIndex.get(inputIndex) !==
+              authoredInputDigest
+            );
+          }
+          // A saved waiting row may include output columns restored by
+          // onMapStart. Legacy checkpoints lack the original input digest,
+          // so compare their stable input projection and supplied fields.
+          if (
+            stableStringify(stableInputRow(authored)) !==
+            stableStringify(stableInputRow(data))
+          ) {
+            return true;
+          }
+          return Object.entries(authored).some(
+            ([field, value]) =>
+              !field.startsWith('__deepline') &&
+              (!(field in data) ||
+                stableStringify(value) !== stableStringify(data[field])),
+          );
+        },
+      );
+      if (
+        missingCompleted ||
+        changedPending ||
+        materializedItems.length !== replayMapFrame!.totalRows
+      ) {
+        throw new Error(
+          `Cannot reconstruct the complete result for ctx.dataset("${normalizedMapNamespace}") after suspension: ` +
+            'the authored input changed and this run has no readable Runtime Sheet.',
+        );
+      }
+    }
+    // A suspended checkpoint retains only waiting inputs, but its original
+    // total and completed keys still describe the full result of this run.
+    totalInputCount = resumedPendingRowsOnly
+      ? Math.max(replayMapFrame?.totalRows ?? 0, materializedItems.length)
+      : materializedItems.length;
     rawItems = materializedItems.map((item) => this.toOutputRow(item));
     itemsToProcess = materializedItems.map((item) =>
       this.toOutputRow(item as Record<string, unknown>),
@@ -9546,7 +9683,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       rawItems = admittedItems.map((item) => item.row);
       itemsToProcess = admittedItems.map((item) => item.row);
       itemOriginalIndexes = admittedItems.map((item) => item.index);
-      totalInputCount = admittedItems.length;
+      totalInputCount = resumedPendingRowsOnly
+        ? Math.max(replayMapFrame?.totalRows ?? 0, admittedItems.length)
+        : admittedItems.length;
     }
 
     const mapScope = this.createMapExecutionScope({
@@ -9554,7 +9693,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       artifactTableNamespace: resolvedTableNamespace,
       explicitKey: explicitKeyResolver,
     });
-    const completedRowKeys: string[] = [];
+    const completedRowKeys: string[] = [...previouslyCompletedRowKeys];
     const pendingRowKeys = itemsToProcess.map((item, index) =>
       rowIdentity(this.toOutputRow(item), itemOriginalIndexes[index] ?? index),
     );
@@ -9567,8 +9706,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       totalRows: totalInputCount,
       completedRowKeys,
       pendingRowKeys,
-      failedRowsCount: 0,
-      startedAt: Date.now(),
+      failedRowsCount: previouslyFailedRowsCount,
+      startedAt: resumedPendingRowsOnly
+        ? (replayMapFrame?.startedAt ?? Date.now())
+        : Date.now(),
       updatedAt: Date.now(),
     });
     this.emitExecutionEvent({
@@ -9690,7 +9831,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         options?.description,
         {
           totalRows: totalInputCount,
-          completedRows: duplicateReuseCount,
+          completedRows:
+            duplicateReuseCount + previouslyCompletedRowKeys.length,
         },
         {
           emitTerminalEvent: false,
@@ -9699,8 +9841,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           executionRowIndexes: rowsToExecuteEntries.map(
             (entry) => entry.originalIndex,
           ),
+          ...(!canReadPersistedMapResults
+            ? {
+                authoredInputDigests: rowsToExecuteEntries.map((entry) =>
+                  authoredInputDigestsByIndex.get(entry.originalIndex),
+                ),
+              }
+            : {}),
           retainedRowsMemoryBudgetBytes:
             mapMemoryLimits.materializedBudgetBytes,
+          progressFailedOffset: previouslyFailedRowsCount,
           incrementalPersistence,
         },
       );
@@ -9751,7 +9901,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               this.toPublicOutputRow(resultsByKey.get(rowKey) ?? rawItem),
             ];
           });
-    const executedCount = rowsToExecute.length;
+    const executedCount =
+      rowsToExecute.length +
+      previouslyCompletedRowKeys.length +
+      previouslyFailedRowsCount;
     const reusedCount = duplicateReuseCount;
 
     // Persist executed rows to the tenant runtime sheet — the sheet is the
@@ -9794,9 +9947,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
 
     const durableCompletedRows = Math.max(
       0,
-      reusedCount + visibleCompletedRows.length,
+      previouslyCompletedRowKeys.length +
+        reusedCount +
+        visibleCompletedRows.length,
     );
-    const durableFailedRows = visibleFailedRows.length;
+    const durableFailedRows =
+      previouslyFailedRowsCount + visibleFailedRows.length;
     const terminalFrame = this.checkpoint.mapFrames?.[mapScope.mapInvocationId];
     if (terminalFrame) {
       this.setMapFrame({
@@ -9829,6 +9985,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     });
 
     if (this.#options.onMapRowsCompleted) {
+      // Output mode selects enriched rows when this replay's earlier
+      // invocation also persisted isolated failures.
+      const finalizationRowMode = resumedPendingRowsOnly ? 'output' : 'all';
       const finalizationPageReader = createRuntimeSheetDatasetPageReader({
         context: {
           baseUrl: this.#options.baseUrl!,
@@ -9840,15 +9999,21 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         },
         tableNamespace: resolvedTableNamespace,
         runId: this.#options.runId!,
-        rowMode: 'all',
+        rowMode: finalizationRowMode,
       });
       results = await reconcileNodeRuntimeMapResultsWithPersistedSheet({
         mapName: normalizedMapNamespace,
         tableNamespace: resolvedTableNamespace,
         runId: this.#options.runId,
-        expectedRows: Math.max(0, totalInputCount - staleCompletionKeys.size),
+        expectedRows: Math.max(
+          0,
+          totalInputCount -
+            staleCompletionKeys.size -
+            (resumedPendingRowsOnly ? durableFailedRows : 0),
+        ),
         currentRows: results,
-        failedRowCount: visibleFailedRows.length,
+        failedRowCount: resumedPendingRowsOnly ? 0 : visibleFailedRows.length,
+        rowMode: finalizationRowMode,
         log: (line) => this.runtimeLog(line),
         readPersistedRows: async (readInput) => {
           if (
@@ -9968,7 +10133,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         reused: reusedCount,
         skipped: reusedCount,
         pending: 0,
-        failed: visibleFailedRows.length,
+        failed: durableFailedRows,
         ...(duplicateReuseCount > 0
           ? { duplicates: { exact: duplicateReuseCount } }
           : {}),
@@ -10008,6 +10173,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       onRowError?: 'isolate' | 'fail';
       executionRowKeys?: string[];
       executionRowIndexes?: number[];
+      authoredInputDigests?: Array<string | undefined>;
       /**
        * Opt-in cap on how many row resolvers run their body concurrently. When
        * unset, the Governor's rowDefault is used. Excess rows stay pending as
@@ -10552,6 +10718,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     );
     let nextRowIndex = 0;
     let firstRowError: unknown = null;
+    const providerRetryDeadlines: Array<{
+      provider: string;
+      retryAtMs: number;
+    }> = [];
+    let providerBackpressureRowCount = 0;
+    const waitingBaseRows = new Map<number, Record<string, unknown>>();
 
     const runRow = async (
       idx: number,
@@ -10644,7 +10816,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           } catch (error) {
             if (
               isPlayRowExecutionSuspendedError(error) ||
-              error instanceof PlayExecutionSuspendedError
+              error instanceof PlayExecutionSuspendedError ||
+              error instanceof PlayRowProviderBackpressureError
             ) {
               throw error;
             }
@@ -10831,6 +11004,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         return COMPLETED_ROW;
       } catch (error) {
         if (isPlayRowExecutionSuspendedError(error)) {
+          waitingBaseRows.set(idx, baseRow);
           this.pendingRowEventBoundaries.push(error.boundary);
           updateMapFrameProgress({
             pendingRowKey: rowKey,
@@ -10852,6 +11026,47 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           });
           return WAITING_ROW;
         }
+        if (error instanceof PlayRowProviderBackpressureError) {
+          waitingBaseRows.set(idx, baseRow);
+          // Each row retains its own deadline. Provider names are not rate
+          // scopes: two credentials for one provider can recover separately.
+          providerRetryDeadlines.push({
+            provider: error.provider,
+            retryAtMs: error.retryAtMs,
+          });
+          providerBackpressureRowCount += 1;
+          updateMapFrameProgress({ pendingRowKey: rowKey });
+          const fieldName = activeFieldName;
+          if (!fieldName) {
+            throw new Error(
+              'Provider backpressure escaped a mapped row without an active field.',
+              { cause: error },
+            );
+          }
+          this.emitScopedFieldMetaUpdate({
+            rowId: idx,
+            key: rowKey,
+            tableNamespace: normalizedTableNamespace,
+            fieldName,
+            status: 'running',
+            rowStatus: 'running',
+            stage: 'waiting_for_provider',
+            provider: error.provider,
+            error: null,
+            dataPatch: {},
+          });
+          this.runtimeLog(
+            `[runtime.provider.row_wait] ${JSON.stringify({
+              provider: error.provider,
+              retry_at: new Date(error.retryAtMs).toISOString(),
+              wait_ms: Math.max(0, error.retryAtMs - Date.now()),
+              row_key_digest: stableDigest(rowKey),
+              field: fieldName,
+            })}`,
+            { level: 'warn' },
+          );
+          return WAITING_ROW;
+        }
         if (isPlayExecutionSuspendedError(error)) {
           const completedKeys = new Set(
             completedRowsToPersist.map((row) => row.key),
@@ -10866,6 +11081,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 data: this.toPersistedOutputRow(
                   this.toOutputRow(item as Record<string, unknown>),
                 ),
+                ...(runtimeOptions?.authoredInputDigests?.[index]
+                  ? {
+                      authoredInputDigest:
+                        runtimeOptions.authoredInputDigests[index],
+                    }
+                  : {}),
               },
             ];
           });
@@ -11089,8 +11310,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         settled = await Promise.allSettled(workers);
       }
     } finally {
-      datasetStep.items = items.length;
-      this.processedRowCount = items.length;
+      datasetStep.items = currentTotalRows();
+      this.processedRowCount = currentTotalRows();
       mapStallObserver?.stopHeartbeat();
       unsubscribePersistenceLatch();
       this.#options.rowAdmissionSignal?.removeEventListener(
@@ -11166,36 +11387,44 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         accountedRowIndexes.add(row.inputIndex);
       }
     }
-    if (this.pendingRowEventBoundaries.length > 0) {
-      const completedRowKeys = new Set<string>();
+    if (
+      this.pendingRowEventBoundaries.length > 0 ||
+      providerRetryDeadlines.length > 0
+    ) {
+      const waitingRowKeys = new Set<string>();
+      const previousSuspendedInputs =
+        this.checkpoint.mapFrames?.[mapScope.mapInvocationId]
+          ?.suspendedInputRows;
+      const suspendedInputRows =
+        rowFeed ||
+        (providerRetryDeadlines.length === 0 && !previousSuspendedInputs)
+          ? undefined
+          : rowResults.flatMap((result, index) => {
+              if (result !== WAITING_ROW) return [];
+              const baseRow = waitingBaseRows.get(index);
+              if (!baseRow) {
+                throw new Error(
+                  `Waiting row ${index} lost its resumable input before map suspension.`,
+                );
+              }
+              return [
+                {
+                  inputIndex: executionRowIndex(index),
+                  data: this.toPersistedOutputRow(baseRow),
+                  ...(runtimeOptions?.authoredInputDigests?.[index]
+                    ? {
+                        authoredInputDigest:
+                          runtimeOptions.authoredInputDigests[index],
+                      }
+                    : {}),
+                },
+              ];
+            });
       for (let index = 0; index < rowResults.length; index += 1) {
-        const result = rowResults[index];
-        const key = keyForIndex(index);
-        if (result === WAITING_ROW) {
-          if (key) {
-            completedRowKeys.add(key);
-          }
-          accountedRowIndexes.add(executionRowIndex(index));
-          continue;
+        if (rowResults[index] === WAITING_ROW) {
+          waitingRowKeys.add(keyForIndex(index));
         }
-        if (key) completedRowKeys.add(key);
       }
-      this.setMapFrame({
-        ...(this.checkpoint.mapFrames?.[mapScope.mapInvocationId] ?? {
-          mapInvocationId: mapScope.mapInvocationId,
-          logicalNamespace: mapScope.logicalNamespace,
-          artifactTableNamespace: mapScope.artifactTableNamespace,
-          status: 'suspended' as const,
-          totalRows: currentTotalRows(),
-          completedRowKeys: [...completedRowKeys],
-          pendingRowKeys: items.map((_item, index) => keyForIndex(index)),
-          startedAt: Date.now(),
-          updatedAt: Date.now(),
-        }),
-        status: 'suspended',
-        completedRowKeys: [...completedRowKeys],
-        updatedAt: Date.now(),
-      });
       const uniqueBoundaries = [
         ...new Map(
           this.pendingRowEventBoundaries.map((boundary) => [
@@ -11204,19 +11433,69 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           ]),
         ).values(),
       ];
+      const earliestHold = providerRetryDeadlines.reduce<
+        (typeof providerRetryDeadlines)[number] | null
+      >(
+        (earliest, next) =>
+          !earliest || next.retryAtMs < earliest.retryAtMs ? next : earliest,
+        null,
+      );
+      const providerWakeAtMs = earliestHold
+        ? Math.max(Date.now() + 1_000, earliestHold.retryAtMs)
+        : null;
       const checkpointUpdates = this.activeMapCheckpointUpdatesForTable(
         normalizedTableNamespace,
       );
-      // A suspension is not observable until both terminal rows and partial
-      // completed-cell patches cross the same writer's durability barrier.
+      // Both terminal siblings and partial waiting-cell patches must cross
+      // the writer's durability barrier before this map becomes parked.
       await incrementalPersistence?.checkpoint(checkpointUpdates);
       await incrementalPersistence?.flush();
       await this.runtimeStepReceiptCompletionWriter.flush();
-      this.pendingRowEventBoundaries = [];
-      this.suspendDurably({
-        kind: 'integration_event_batch',
-        boundaries: uniqueBoundaries,
+      updateMapFrameProgress({
+        status: 'suspended',
+        activeBoundaryId: null,
+        emitEventType: 'map.suspended',
       });
+      const frame = this.checkpoint.mapFrames?.[mapScope.mapInvocationId];
+      if (!frame) {
+        throw new Error('Suspended map lost its durable execution frame.');
+      }
+      this.setMapFrame({
+        ...frame,
+        completedRowKeys: frame.completedRowKeys.filter(
+          (key) => !waitingRowKeys.has(key),
+        ),
+        pendingRowKeys: rowFeed ? [] : [...waitingRowKeys],
+        suspendedInputRows,
+        activeBoundaryId: null,
+        updatedAt: Date.now(),
+      });
+      if (earliestHold && providerWakeAtMs !== null) {
+        this.runtimeLog(
+          `[runtime.provider.map_wait] ${JSON.stringify({
+            provider: earliestHold.provider,
+            retry_at: new Date(providerWakeAtMs).toISOString(),
+            wait_ms: Math.max(0, providerWakeAtMs - Date.now()),
+            waiting_rows: providerBackpressureRowCount,
+            table_namespace: normalizedTableNamespace,
+          })}`,
+          { level: 'warn' },
+        );
+      }
+      this.pendingRowEventBoundaries = [];
+      if (uniqueBoundaries.length > 0) {
+        this.suspendDurably({
+          kind: 'integration_event_batch',
+          boundaries: uniqueBoundaries,
+          ...(providerWakeAtMs !== null ? { providerWakeAtMs } : {}),
+        });
+      }
+      if (earliestHold && providerWakeAtMs !== null) {
+        await this.waitForProviderBackpressure(
+          earliestHold.provider,
+          providerWakeAtMs,
+        );
+      }
     }
     for (let index = 0; index < rowResults.length; index += 1) {
       if (rowResults[index] !== WAITING_ROW) {
@@ -13491,6 +13770,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           throw error;
         }
         if (suspendAfterTimeoutMs !== undefined) {
+          const finalizedAtMs = existing.completedAt ?? Date.now();
+          if (existing.completedAt === undefined) {
+            this.checkpoint.resolvedBoundaries = {
+              ...(this.checkpoint.resolvedBoundaries ?? {}),
+              [preparedBoundary.boundaryId]: {
+                ...existing,
+                completedAt: finalizedAtMs,
+              },
+            };
+          }
           this.runtimeLog(
             `Integration event ${preparedBoundary.boundaryId}: timeout finalized externally; keeping Play parked`,
           );
@@ -13499,6 +13788,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             boundaryId: preparedBoundary.boundaryId,
             eventKey: preparedBoundary.eventKey,
             timeoutMs: suspendAfterTimeoutMs,
+            eventExpiresAtMs: finalizedAtMs + suspendAfterTimeoutMs,
+            postTimeoutPark: true,
             ...(existing.messageRef ? { messageRef: existing.messageRef } : {}),
           });
         }
@@ -13530,6 +13821,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         boundaryId: preparedBoundary.boundaryId,
         eventKey: existing.eventKey,
         timeoutMs: existing.timeoutMs,
+        ...(existing.eventExpiresAtMs !== undefined
+          ? { eventExpiresAtMs: existing.eventExpiresAtMs }
+          : {}),
         ...(existing.messageRef ? { messageRef: existing.messageRef } : {}),
       };
       if (rowScope) {
@@ -13544,6 +13838,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       boundary: preparedBoundary,
       ...(force ? { force: true } : {}),
     });
+    // An unanswered checkpoint suspends above without re-arming the provider.
+    // New boundaries receive one absolute expiry, persisted for later replays.
+    const eventExpiresAtMs = Date.now() + boundary.timeoutMs;
 
     if (boundary.resolvedOutput) {
       this.checkpoint.resolvedBoundaries = {
@@ -13552,6 +13849,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           kind: 'integration_event',
           eventKey: boundary.eventKey,
           timeoutMs: boundary.timeoutMs,
+          eventExpiresAtMs,
           provider: boundary.provider,
           toolId: boundary.toolId,
           output: boundary.resolvedOutput,
@@ -13597,6 +13895,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         kind: 'integration_event',
         eventKey: boundary.eventKey,
         timeoutMs: boundary.timeoutMs,
+        eventExpiresAtMs,
         provider: boundary.provider,
         toolId: boundary.toolId,
         ...(boundary.requestKey ? { requestKey: boundary.requestKey } : {}),
@@ -13626,6 +13925,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         boundaryId: boundary.boundaryId,
         eventKey: boundary.eventKey,
         timeoutMs: boundary.timeoutMs,
+        eventExpiresAtMs,
         ...(boundary.messageRef ? { messageRef: boundary.messageRef } : {}),
       });
     }
@@ -13635,6 +13935,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       boundaryId: boundary.boundaryId,
       eventKey: boundary.eventKey,
       timeoutMs: boundary.timeoutMs,
+      eventExpiresAtMs,
       ...(boundary.messageRef ? { messageRef: boundary.messageRef } : {}),
     });
   }
@@ -14176,6 +14477,60 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     );
   }
 
+  private async waitForProviderBackpressure(
+    provider: string,
+    retryAtMs: number,
+  ): Promise<void> {
+    this.assertNoPendingDurableSuspension();
+    const prefix = this.durableBoundaryId(
+      `provider-backpressure-${stableDigest(provider)}-`,
+    );
+    let completedWaits = 0;
+    for (const [boundaryId, value] of Object.entries(
+      this.checkpoint.resolvedBoundaries ?? {},
+    )) {
+      if (!boundaryId.startsWith(prefix)) continue;
+      const boundary = assertKnownResolvedBoundary(boundaryId, value);
+      if (boundary?.kind !== 'sleep') {
+        throw new Error(
+          `Provider backpressure boundary ${boundaryId} is not a sleep.`,
+        );
+      }
+      if (boundary.completedAt !== undefined) completedWaits += 1;
+    }
+    // Count completed waits, not this process's call order. A parked Play is
+    // re-executed by a new sandbox with fresh in-memory counters, and another
+    // provider quota window must not reuse the previous completed sleep.
+    const boundaryId = `${prefix}${completedWaits}`;
+    const existing = assertKnownResolvedBoundary(
+      boundaryId,
+      this.checkpoint.resolvedBoundaries?.[boundaryId],
+    );
+    if (existing?.kind === 'sleep' && existing.completedAt !== undefined) {
+      throw new Error(
+        `Provider backpressure boundary ${boundaryId} was already completed.`,
+      );
+    }
+    const delayMs =
+      existing?.kind === 'sleep'
+        ? existing.delayMs
+        : Math.max(1_000, Math.ceil(retryAtMs - Date.now()));
+    this.checkpoint.resolvedBoundaries = {
+      ...(this.checkpoint.resolvedBoundaries ?? {}),
+      [boundaryId]: { kind: 'sleep', delayMs },
+    };
+    this.runtimeLog(
+      `[runtime.provider.wait] ${JSON.stringify({
+        provider,
+        retry_at: new Date(Date.now() + delayMs).toISOString(),
+        wait_ms: delayMs,
+        boundary_id: boundaryId,
+      })}`,
+      { level: 'warn' },
+    );
+    this.suspendDurably({ kind: 'sleep', boundaryId, delayMs });
+  }
+
   async sleep(ms: number): Promise<void> {
     if (!rowContext.getStore()) this.assertRowAdmissionOpen();
     this.assertInlineChildContract('suspending_child');
@@ -15834,6 +16189,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                         rowKey: member.rowKey,
                         fieldName: member.fieldName,
                       })),
+                      providerBackpressureMapRows: detail.members.some(
+                        (member) =>
+                          Boolean(member.tableNamespace && member.rowKey),
+                      ),
                       nativeBatchBodyAttribution: bodyAttribution,
                       playNodeScope: playNodeScopeForToolCallRequest(
                         detail.members[0]!,
@@ -16109,7 +16468,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             // provider. Keep infrastructure and contract failures loud.
             if (
               isProviderExhaustedFailure(entry.error) ||
-              isProviderUnavailable(entry.error)
+              isProviderUnavailable(entry.error) ||
+              entry.error instanceof PlayRowProviderBackpressureError
             ) {
               continue;
             }
@@ -16295,6 +16655,50 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         ): Promise<void> => {
           const followers = liveFollowersByOwnerCallId.get(owner.callId) ?? [];
           liveFollowersByOwnerCallId.delete(owner.callId);
+          if (error instanceof PlayRowProviderBackpressureError) {
+            let releaseFailure: unknown = null;
+            const receiptKey = owner.receiptKey?.trim() || null;
+            try {
+              if (receiptKey && this.durableMappedToolResultsBackedByReceipts) {
+                const leaseId = owner.receiptLeaseId?.trim();
+                if (!leaseId) {
+                  throw new Error(
+                    `Provider wait cannot release receipt ${receiptKey} without its owned lease.`,
+                  );
+                }
+                const released = await this.releaseRuntimeStepReceipt(
+                  receiptKey,
+                  this.currentReceiptOwnerRunId,
+                  leaseId,
+                );
+                if (
+                  !released ||
+                  (released.status !== 'queued' &&
+                    released.status !== 'pending')
+                ) {
+                  throw new Error(
+                    `Provider wait could not release owned receipt ${receiptKey}.`,
+                  );
+                }
+              }
+            } catch (failure) {
+              releaseFailure = failure;
+            }
+            // A provider deadline is a nonterminal control signal. The row
+            // catch records waiting metadata after the resolver unwinds; no
+            // follower may see or persist a failed receipt in between. If the
+            // lease could not be released, wake every follower with that loud
+            // infrastructure failure rather than stranding their promises.
+            for (const request of [owner, ...followers]) {
+              const resolver = this.toolCallResolvers.get(request.callId);
+              if (resolver) {
+                resolver.reject(releaseFailure ?? error);
+                this.toolCallResolvers.delete(request.callId);
+              }
+            }
+            if (releaseFailure) throw releaseFailure;
+            return;
+          }
           const shouldPersistReceiptFailure =
             !(error instanceof ToolExecuteAuthScopeChangedError) &&
             !this.persistenceLatch.tripped;
@@ -17363,9 +17767,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 onChunkComplete: async (chunkResults) => {
                   for (const entry of chunkResults) {
                     if (entry.error !== undefined) {
-                      for (const request of entry.request.memberRequests) {
-                        await rejectWithLiveFollowers(request, entry.error);
-                      }
+                      const settlements = await Promise.allSettled(
+                        entry.request.memberRequests.map((request) =>
+                          rejectWithLiveFollowers(request, entry.error),
+                        ),
+                      );
+                      const rejected = settlements.find(
+                        (settlement): settlement is PromiseRejectedResult =>
+                          settlement.status === 'rejected',
+                      );
+                      if (rejected) throw rejected.reason;
                       continue;
                     }
                     const batchExecution = entry.result;
@@ -17543,11 +17954,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               const failedEntries = dispatchResults.filter(
                 (entry) => entry.error !== undefined,
               );
-              await Promise.allSettled(
+              const failedSettlements = await Promise.allSettled(
                 failedEntries.map((entry) =>
                   rejectWithLiveFollowers(entry.job, entry.error),
                 ),
               );
+              const failedRejection = failedSettlements.find(
+                (settlement): settlement is PromiseRejectedResult =>
+                  settlement.status === 'rejected',
+              );
+              if (failedRejection) throw failedRejection.reason;
               recordToolStep(pendingRequests);
               this.#options.onBatchComplete?.(this.checkpoint);
             }
@@ -18500,6 +18916,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
 
           let providerConcurrencyGrant: ProviderConcurrencyGrant | null = null;
           let providerPacingGrant: ProviderPacingGrant | null = null;
+          let providerGatewayDispatchProof: NonNullable<
+            RuntimeProviderPermit['gatewayDispatchProof']
+          > | null = null;
           // The permit returns the canonical pacing bucket. Keep it alongside
           // the tool-keyed dispatch identity so terminal attribution can join
           // provider admission with the response residence it admitted.
@@ -18509,9 +18928,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             const permit = await this.resourceGovernor.acquireProviderPermit({
               toolId,
               signal,
+              yieldProviderHolds: this.#options.durableBoundaries === true,
             });
             providerConcurrencyGrant = permit.concurrencyGrant ?? null;
             providerPacingGrant = permit.pacingGrant ?? null;
+            providerGatewayDispatchProof = permit.gatewayDispatchProof ?? null;
             providerTelemetryKey = permit.provider;
             return permit;
           };
@@ -18529,6 +18950,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             providerConcurrencyGrant = null;
             providerPacingGrant = null;
             let response: Response | null = null;
+            let providerRetryAtMs: number | undefined;
             let responseData: Record<string, unknown> | null = null;
             let responseErrorText: string | null = null;
             let providerCallStartedAt: number | null = null;
@@ -18647,6 +19069,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                     if (!permit) return null;
                     providerConcurrencyGrant = permit.concurrencyGrant ?? null;
                     providerPacingGrant = permit.pacingGrant ?? null;
+                    providerGatewayDispatchProof =
+                      permit.gatewayDispatchProof ?? null;
                     providerTelemetryKey = permit.provider;
                     return permit;
                   },
@@ -18674,6 +19098,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                     providerResourceKey: `tool:${toolId}`,
                     signal: abortController?.signal,
                     requiresTransportPermit: !fixtureExecution,
+                    deferProviderPermitValidation: true,
                     onTransportAdmitted: (waitMs) => {
                       transportSlotWaitMs = waitMs;
                     },
@@ -18702,117 +19127,151 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                     };
                     const releaseProviderExecution = () => {
                       const execution = providerExecution.current;
+                      providerExecution.current = null;
                       if (execution) execution.release();
                     };
-                    // Provider admission is our queue, not provider execution.
-                    // Start the tool deadline only after admission so a busy
-                    // runtime cannot consume the remote-call budget while this
-                    // attempt is still waiting for permission to leave.
-                    if (
-                      timeoutMs &&
-                      abortController &&
-                      !abortController.signal.aborted
-                    ) {
-                      timeoutHandle = setTimeout(() => {
-                        abortController.abort(
-                          new Error(
-                            `Tool ${toolId} runtime API call timed out after ${timeoutMs}ms.`,
-                          ),
-                        );
-                      }, timeoutMs);
-                    }
-                    const fixtureBehavior = this.#options.fixtureBehavior;
-                    if (
-                      fixtureExecution &&
-                      fixtureBehavior &&
-                      shouldRouteFixtureToolId(toolId)
-                    ) {
-                      const canonicalProvider =
-                        (
-                          await this.#options.getToolProvider?.(toolId)
-                        )?.trim() || provider;
-                      if (shouldRouteFixtureProvider(canonicalProvider)) {
-                        const canonicalOperation =
-                          (
-                            await this.#options.getToolOperation?.(toolId)
-                          )?.trim() || toolId;
-                        // A fixture schedule needs a stable logical-call
-                        // identity, not a cache receipt. Ordinary /execute
-                        // calls deliberately do not create durable receipts;
-                        // deeplineRequestId remains stable across physical
-                        // transport retries.
-                        const fixtureRequestIdentity =
-                          durableCallReceiptKey ?? deeplineRequestId;
-                        const stableRequestKey = `${canonicalProvider}:${canonicalOperation}:${fixtureRequestIdentity}`;
-                        const behaviorDigest = stableDigest(
-                          JSON.stringify(fixtureBehavior),
-                        );
-                        const requestKeyDigest = stableDigest(stableRequestKey);
-                        let selected: {
-                          delayMs: number;
-                          sampleIndex: number;
-                        } | null = null;
-                        beginProviderExecution();
-                        const delayStartedAt = Date.now();
-                        providerCallStartedAt = delayStartedAt;
-                        try {
-                          selected = await waitForFixtureResponseDelay({
-                            behavior: fixtureBehavior,
-                            stableRequestKey,
-                            signal: abortController?.signal,
-                            onSelected: (value) => {
-                              selected = value;
+                    try {
+                      return await request(
+                        async (beforeProviderDispatch) => {
+                          // Fixture delay models provider residence. The shared
+                          // permit may still need re-admission afterwards, so
+                          // pause this budget around that queueing boundary.
+                          let fixtureTimeoutUsedMs = 0;
+                          const armProviderTimeout = () => {
+                            const controller = abortController;
+                            if (
+                              !timeoutMs ||
+                              !controller ||
+                              controller.signal.aborted
+                            )
+                              return;
+                            timeoutHandle = setTimeout(
+                              () => {
+                                controller.abort(
+                                  new Error(
+                                    `Tool ${toolId} runtime API call timed out after ${timeoutMs}ms.`,
+                                  ),
+                                );
+                              },
+                              Math.max(1, timeoutMs - fixtureTimeoutUsedMs),
+                            );
+                          };
+                          const fixtureBehavior = this.#options.fixtureBehavior;
+                          if (
+                            fixtureExecution &&
+                            fixtureBehavior &&
+                            shouldRouteFixtureToolId(toolId)
+                          ) {
+                            const canonicalProvider =
+                              (
+                                await this.#options.getToolProvider?.(toolId)
+                              )?.trim() || provider;
+                            if (shouldRouteFixtureProvider(canonicalProvider)) {
+                              const canonicalOperation =
+                                (
+                                  await this.#options.getToolOperation?.(toolId)
+                                )?.trim() || toolId;
+                              // A fixture schedule needs a stable logical-call
+                              // identity, not a cache receipt. Ordinary /execute
+                              // calls deliberately do not create durable receipts;
+                              // deeplineRequestId remains stable across physical
+                              // transport retries.
+                              const fixtureRequestIdentity =
+                                durableCallReceiptKey ?? deeplineRequestId;
+                              const stableRequestKey = `${canonicalProvider}:${canonicalOperation}:${fixtureRequestIdentity}`;
+                              const behaviorDigest = stableDigest(
+                                JSON.stringify(fixtureBehavior),
+                              );
+                              const requestKeyDigest =
+                                stableDigest(stableRequestKey);
+                              let selected: {
+                                delayMs: number;
+                                sampleIndex: number;
+                              } | null = null;
+                              beginProviderExecution();
+                              const delayStartedAt = Date.now();
+                              providerCallStartedAt = delayStartedAt;
+                              armProviderTimeout();
+                              try {
+                                selected = await waitForFixtureResponseDelay({
+                                  behavior: fixtureBehavior,
+                                  stableRequestKey,
+                                  signal: abortController?.signal,
+                                  onSelected: (value) => {
+                                    selected = value;
+                                    this.runtimeLog(
+                                      `[fixture.response_delay] ${JSON.stringify(
+                                        {
+                                          outcome: 'scheduled',
+                                          provider: canonicalProvider,
+                                          operation: canonicalOperation,
+                                          behavior_digest: `sha256_${behaviorDigest}`,
+                                          request_key_digest: `sha256_${requestKeyDigest}`,
+                                          delay_ms: value.delayMs,
+                                          sample_index: value.sampleIndex,
+                                          physical_attempt: physicalAttempt,
+                                        },
+                                      )}`,
+                                    );
+                                  },
+                                });
+                              } catch (error) {
+                                this.runtimeLog(
+                                  `[fixture.response_delay] ${JSON.stringify({
+                                    outcome: 'aborted',
+                                    provider: canonicalProvider,
+                                    operation: canonicalOperation,
+                                    behavior_digest: `sha256_${behaviorDigest}`,
+                                    request_key_digest: `sha256_${requestKeyDigest}`,
+                                    delay_ms: selected?.delayMs ?? null,
+                                    sample_index: selected?.sampleIndex ?? null,
+                                    physical_attempt: physicalAttempt,
+                                    elapsed_ms: Date.now() - delayStartedAt,
+                                  })}`,
+                                );
+                                releaseProviderExecution();
+                                throw error;
+                              }
+                              fixtureDelayMs = Date.now() - delayStartedAt;
+                              fixtureTimeoutUsedMs = fixtureDelayMs;
                               this.runtimeLog(
                                 `[fixture.response_delay] ${JSON.stringify({
-                                  outcome: 'scheduled',
+                                  outcome: 'completed',
                                   provider: canonicalProvider,
                                   operation: canonicalOperation,
                                   behavior_digest: `sha256_${behaviorDigest}`,
                                   request_key_digest: `sha256_${requestKeyDigest}`,
-                                  delay_ms: value.delayMs,
-                                  sample_index: value.sampleIndex,
+                                  delay_ms: selected.delayMs,
+                                  sample_index: selected.sampleIndex,
                                   physical_attempt: physicalAttempt,
+                                  elapsed_ms: Date.now() - delayStartedAt,
                                 })}`,
                               );
-                            },
-                          });
-                        } catch (error) {
-                          this.runtimeLog(
-                            `[fixture.response_delay] ${JSON.stringify({
-                              outcome: 'aborted',
-                              provider: canonicalProvider,
-                              operation: canonicalOperation,
-                              behavior_digest: `sha256_${behaviorDigest}`,
-                              request_key_digest: `sha256_${requestKeyDigest}`,
-                              delay_ms: selected?.delayMs ?? null,
-                              sample_index: selected?.sampleIndex ?? null,
-                              physical_attempt: physicalAttempt,
-                              elapsed_ms: Date.now() - delayStartedAt,
-                            })}`,
+                            }
+                          }
+                          if (timeoutHandle) {
+                            clearTimeout(timeoutHandle);
+                            timeoutHandle = null;
+                          }
+                          const dispatchBoundary = beforeProviderDispatch(
+                            releaseProviderExecution,
                           );
-                          releaseProviderExecution();
-                          throw error;
-                        }
-                        fixtureDelayMs = Date.now() - delayStartedAt;
-                        this.runtimeLog(
-                          `[fixture.response_delay] ${JSON.stringify({
-                            outcome: 'completed',
-                            provider: canonicalProvider,
-                            operation: canonicalOperation,
-                            behavior_digest: `sha256_${behaviorDigest}`,
-                            request_key_digest: `sha256_${requestKeyDigest}`,
-                            delay_ms: selected.delayMs,
-                            sample_index: selected.sampleIndex,
-                            physical_attempt: physicalAttempt,
-                            elapsed_ms: Date.now() - delayStartedAt,
-                          })}`,
-                        );
-                      }
-                    }
-                    beginProviderExecution();
-                    try {
-                      return await request(
-                        async () => {
+                          const permitReacquired =
+                            typeof dispatchBoundary === 'boolean'
+                              ? dispatchBoundary
+                              : await dispatchBoundary;
+                          if (permitReacquired) {
+                            // A shared Retry-After may have arrived while
+                            // fixture/request preparation was suspended. The
+                            // old residence was not provider work; restart the
+                            // latency clock with the fresh permit.
+                            providerCallStartedAt = Date.now();
+                          }
+                          // A stale permit can take arbitrarily long to
+                          // replace. Only fixture residence and the physical
+                          // HTTP exchange spend the tool execution deadline.
+                          armProviderTimeout();
+                          beginProviderExecution();
                           const fetchedAt = Date.now();
                           const dispatchedProviderCallStartedAt =
                             providerCallStartedAt ?? fetchedAt;
@@ -18842,6 +19301,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                               // settle the row as failed instead of pacing.
                               [TOOL_EXECUTE_RUNNER_ACCEPTS_HEADER]:
                                 TOOL_EXECUTE_RUNNER_ACCEPTS_VALUE,
+                              ...(providerGatewayDispatchProof
+                                ? {
+                                    [TOOL_EXECUTE_PROVIDER_RATE_SCOPE_HEADER]:
+                                      providerGatewayDispatchProof.rateScopeToken,
+                                    [TOOL_EXECUTE_PROVIDER_RATE_RETRY_SNAPSHOT_HEADER]:
+                                      String(
+                                        providerGatewayDispatchProof.retryAtMsAtGrant,
+                                      ),
+                                  }
+                                : {}),
                               'x-deepline-request-id': deeplineRequestId,
                               ...(providerExecutionFenceKey
                                 ? {
@@ -18868,6 +19337,19 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                           if (dispatchedResponse.status === 429) {
                             providerRequestsInFlightAt429 =
                               activeProviderRequest?.inFlightCount() ?? null;
+                            const observedAt = Date.now();
+                            const retryAfter =
+                              dispatchedResponse.headers.get('retry-after');
+                            // Without a provider deadline the classifier owns
+                            // the conservative bare-429 cooldown. Do not replace
+                            // it with the generic one-second HTTP retry fallback.
+                            providerRetryAtMs = retryAfter?.trim()
+                              ? observedAt +
+                                parseToolExecuteRetryAfterMs(
+                                  retryAfter,
+                                  observedAt,
+                                )
+                              : undefined;
                           }
                           const dispatchedFetchToHeadersMs =
                             Date.now() - fetchedAt;
@@ -19056,6 +19538,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                                 ? null
                                 : Date.now() - providerCallStartedAt,
                               providerConcurrencyGrant,
+                              providerRetryAtMs,
                             );
                             providerBackpressureReported = true;
                           } else if (
@@ -19076,6 +19559,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                                 ? null
                                 : Date.now() - providerCallStartedAt,
                               providerConcurrencyGrant,
+                              providerRetryAtMs,
                             );
                             providerBackpressureReported = true;
                           }
@@ -19144,6 +19628,36 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               }
               if (error instanceof ProviderExhaustedError) {
                 const retryAtMs = Date.parse(error.retryAt);
+                if (!this.#options.durableBoundaries) {
+                  await this.sleepWithCheckpointHeartbeat(
+                    Math.max(
+                      1_000,
+                      Number.isFinite(retryAtMs)
+                        ? Math.ceil(retryAtMs - Date.now())
+                        : 1_000,
+                    ),
+                  );
+                  continue;
+                }
+                if (this.#options.durableBoundaries) {
+                  const rowScope = rowContext.getStore();
+                  if (options?.providerBackpressureMapRows || rowScope) {
+                    throw new PlayRowProviderBackpressureError(
+                      error.provider,
+                      Number.isFinite(retryAtMs) ? retryAtMs : Date.now(),
+                    );
+                  }
+                  // This is a provider admission hold, not a row outcome.
+                  // Release the sandbox and resume the same Play after the
+                  // shared pacer's deadline. The receipt executor releases
+                  // this call's claim on suspension, so completed work remains
+                  // reusable and this tool can re-admit safely.
+                  await this.waitForProviderBackpressure(
+                    error.provider,
+                    Number.isFinite(retryAtMs) ? retryAtMs : Date.now(),
+                  );
+                  continue;
+                }
                 const providerFailure =
                   error.cause instanceof ToolExecutionError
                     ? error.cause
@@ -19411,22 +19925,41 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                   failure.backpressureDelayMs,
                   providerCallElapsedMs,
                   providerConcurrencyGrant,
+                  providerRetryAtMs,
                 );
                 providerBackpressureReported = true;
-                // A Retry-After beyond the durable pacer's maximum wait has
-                // no useful in-process retry. It is already an exhausted
-                // provider outcome: surface the same typed error the next
-                // provider admission would produce, without sleeping before a
-                // waterfall can choose an independent fallback.
+              }
+              // The normal 429 path reports its hold before releasing the
+              // provider lease. A mapped row must now yield even for a short
+              // hold: sleeping here would occupy its row slot and prevent
+              // unrelated rows from making progress.
+              const durableProviderWaitMs =
+                failure.providerPacingWaitMs ?? failure.backpressureDelayMs;
+              if (
+                this.#options.durableBoundaries &&
+                durableProviderWaitMs !== null &&
+                durableProviderWaitMs > 0
+              ) {
+                const retryAtMs =
+                  (failure.providerPacingWaitMs !== null
+                    ? Date.now() + failure.providerPacingWaitMs
+                    : providerRetryAtMs) ?? Date.now() + durableProviderWaitMs;
+                const rowScope = rowContext.getStore();
                 if (
-                  failure.backpressureDelayMs > PROVIDER_EXHAUSTED_MAX_WAIT_MS
+                  this.#options.durableBoundaries &&
+                  (options?.providerBackpressureMapRows || rowScope)
                 ) {
-                  throw new ProviderExhaustedError({
+                  throw new PlayRowProviderBackpressureError(
                     provider,
-                    retryAtMs: Date.now() + failure.backpressureDelayMs,
-                    cause: surfacedError,
-                  });
+                    retryAtMs,
+                  );
                 }
+                await this.waitForProviderBackpressure(provider, retryAtMs);
+                throw new ProviderExhaustedError({
+                  provider,
+                  retryAtMs,
+                  cause: surfacedError,
+                });
               }
               // A Deepline concurrency shed is issued by our own gateway/relay
               // before it opens any provider connection and before any durable
@@ -19438,18 +19971,25 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               // 5xx retries safe — otherwise a protective limit on the relay
               // turns straight into customer row failures.
               const shedKnownNotDispatched =
-                failure.reason === 'concurrency_backpressure';
+                failure.reason === 'concurrency_backpressure' ||
+                failure.reason === 'provider_pacing_wait';
               if (
                 failure.shouldRetry &&
                 (failure.knownNotDispatched ||
                   failure.reason === 'gateway_invocation_in_progress' ||
                   ambiguousTransportReplayAllowed ||
                   retrySafeTransientHttp ||
-                  fixtureReplaySafe)
+                  fixtureReplaySafe ||
+                  // An explicit provider-owned 429 is a rejected operation,
+                  // independently of whether generic 5xx replay is safe.
+                  // The gateway attests the prior 429 before accepting a
+                  // fresh fenced invocation attempt; never extend this to
+                  // platform/caller 429s or ambiguous transport failures.
+                  failure.backpressureDelayMs !== null)
               ) {
                 // A gateway follower is retrying its owner's durable
-                // invocation, while a concurrency shed wrote no invocation at
-                // all. Both must reuse this logical attempt: advancing it
+                // invocation, while a concurrency or provider-pacing refusal
+                // wrote no invocation at all. Both reuse this logical attempt: advancing it
                 // either bypasses the live-owner replay fence or makes the
                 // prior-attempt sweep see an intentionally absent receipt.
                 if (
@@ -19466,9 +20006,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 if (failure.chargeRetryBudget) {
                   await this.governor.chargeBudget('retry');
                 }
-                const retryAttributePrefix = failure.isRateLimit
-                  ? 'rate_limit'
-                  : 'transient_http';
+                const retryAttributePrefix =
+                  failure.reason === 'provider_pacing_wait'
+                    ? 'provider_pacing_wait'
+                    : failure.isRateLimit
+                      ? 'rate_limit'
+                      : 'transient_http';
                 span.setAttribute(
                   `plays.${retryAttributePrefix}_retry_after_ms`,
                   failure.retryDelayMs,
@@ -19499,7 +20042,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                       reason: failure.isRateLimit
                         ? 'rate_limit'
                         : 'provider_error',
-                      retryAt: Date.now() + failure.retryDelayMs,
+                      retryAt:
+                        failure.providerPacingWaitMs !== null
+                          ? Date.now() + failure.providerPacingWaitMs
+                          : Date.now() + failure.retryDelayMs,
                       attempt: httpFailureAttempt,
                     },
                     observedAt: Date.now(),
@@ -19614,6 +20160,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     retryAfterMs: number,
     latencyMs: number | null = null,
     concurrencyGrant: ProviderConcurrencyGrant | null = null,
+    retryAtMs = Date.now() + Math.max(0, retryAfterMs),
   ): Promise<void> {
     const hints: readonly PlayQueueHint[] = this.#options.getToolQueueHints
       ? await this.#options.getToolQueueHints(toolId)
@@ -19623,9 +20170,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       provider,
       toolId,
       retryAfterMs,
+      retryAtMs,
       latencyMs,
       concurrencyGrant,
     });
+    this.runtimeLog(
+      `[runtime.provider.backpressure_report] ${JSON.stringify({
+        toolId,
+        provider,
+        retryAfterMs,
+        retryAtMs,
+        outcome: 'recorded',
+      })}`,
+      { level: 'warn' },
+    );
   }
 
   private async sleepWithCheckpointHeartbeat(ms: number): Promise<void> {