deepline 0.3.322 → 0.3.324

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -211,6 +211,11 @@ import {
   TOOL_EXECUTE_RUNNER_ACCEPTS_VALUE,
   TOOL_EXECUTE_PROVIDER_RATE_SCOPE_HEADER,
   TOOL_EXECUTE_PROVIDER_RATE_RETRY_SNAPSHOT_HEADER,
+  TOOL_EXECUTE_PROVIDER_PERMIT_PATH_SUFFIX,
+  TOOL_EXECUTE_PROVIDER_PERMIT_RECEIVER_HEADER,
+  isProviderPermitExecuteReceiverUnavailable,
+  TOOL_EXECUTE_APP_PACING_REFUSAL_HEADER,
+  TOOL_EXECUTE_APP_PACING_REFUSAL_VALUE,
   isAttestedProvider429Response,
   classifyToolExecuteHttpFailure,
   createToolExecuteHttpFailureAttemptTracker,
@@ -362,6 +367,8 @@ import {
   type RuntimeMapHeapAdmissionSnapshot,
 } from './map-memory-limits';
 import { RuntimeMapRowFeed } from './runtime-map-row-feed';
+import { ProviderRowRetryQueue } from './provider-row-retry-queue';
+import type { ProviderRowRetry } from './provider-row-retry-queue';
 import type { RuntimeMapRowFeedSnapshot } from './runtime-map-row-feed';
 import { createRuntimeDispatchFlow, createRuntimeFlow } from './runtime-flow';
 import {
@@ -523,8 +530,62 @@ const rowContext = new AsyncLocalStorage<{
   resolverGap?: RowResolverGapTracker;
   /** Shared through nested fields, reset for each row execution attempt. */
   providerBackpressure?: { pending: PlayRowProviderBackpressureError | null };
+  replayCallIndexes?: RowReplayCallIndexes;
+  replayStepCaptures?: Map<string, number>[];
 }>();
 
+/** Re-enter a held row with the call positions from its original attempt. */
+class RowReplayCallIndexes {
+  private readonly recorded = new Map<string, number[]>();
+  private readonly positions = new Map<string, number>();
+  private readonly completedSteps = new Map<string, Map<string, number>>();
+
+  resetAttempt(): void {
+    this.positions.clear();
+  }
+
+  next(key: string, freshIndex: number): number {
+    const position = this.positions.get(key) ?? 0;
+    this.positions.set(key, position + 1);
+    let recorded = this.recorded.get(key);
+    if (!recorded) {
+      recorded = [];
+      this.recorded.set(key, recorded);
+    }
+    if (recorded[position] === undefined) recorded[position] = freshIndex;
+    return recorded[position]!;
+  }
+
+  completeStep(boundaryId: string, advances: Map<string, number>): void {
+    this.completedSteps.set(boundaryId, advances);
+  }
+
+  recoverStep(boundaryId: string): boolean {
+    // A cached step does not run its body. Consume its recorded nested calls
+    // so the next sibling cannot inherit one of their invocation positions.
+    const advances = this.completedSteps.get(boundaryId);
+    if (!advances) return false;
+    for (const [key, advance] of advances) {
+      this.positions.set(key, (this.positions.get(key) ?? 0) + advance);
+      recordReplayCallAdvance(key, advance);
+    }
+    return true;
+  }
+}
+
+function recordReplayCallAdvance(key: string, advance: number): void {
+  for (const capture of rowContext.getStore()?.replayStepCaptures ?? []) {
+    capture.set(key, (capture.get(key) ?? 0) + advance);
+  }
+}
+
+function nextReplayableCallIndex(key: string, freshIndex: number): number {
+  const indexes = rowContext.getStore()?.replayCallIndexes;
+  if (!indexes) return freshIndex;
+  recordReplayCallAdvance(key, 1);
+  return indexes.next(key, freshIndex);
+}
+
 /**
  * The open end of one `rowResolverGap` segment.
  *
@@ -2714,6 +2775,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
   private toolDispatcherFailure: unknown | null = null;
   /** Active provider HTTP attempts in this Play execution, grouped by provider. */
   private readonly inFlightProviderRequests = new Map<string, number>();
+  /** A row replay must not restart this call's spent live-sandbox wait budget. */
+  private readonly exhaustedProviderResidentWaitCalls = new Set<string>();
   /**
    * Deepline's own zero-credit denial is run-fatal. Keep it independently of
    * customer control flow so a broad `try/catch` cannot turn it into a
@@ -9260,12 +9323,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       // aliases ride on non-enumerable props after stripCsvProjectionMetadata,
       // so a spread would silently drop them before the sheet write and the
       // aliases would never persist as visible input cells.
-      const mapStartRows = rawItems.map((row, index) => ({
-        ...toSerializableCsvAliasedRow(row),
-        ...mapRowOutcomeRuntimeFields({
-          key: seedRowIdentity(row, index),
-        }),
-      }));
+      // A resumed checkpoint contains only pending rows. Preserve their
+      // original indexes through admission instead of renumbering the subset.
+      const mapStartRows = rawItems.map((row, index) => {
+        const inputIndex = itemOriginalIndexes[index] ?? index;
+        return {
+          ...toSerializableCsvAliasedRow(row),
+          ...mapRowOutcomeRuntimeFields({
+            key: seedRowIdentity(row, inputIndex),
+            inputIndex,
+          }),
+        };
+      });
       const mapStartResult = await this.#options.onMapStart(
         mapStartRows,
         resolvedTableNamespace,
@@ -9303,7 +9372,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       const seededItems = materializedItems
         .map((item, index) => ({
           row: this.toOutputRow(item as Record<string, unknown>),
-          index,
+          index: originalInputIndexByItem.get(item) ?? index,
         }))
         .map(({ row, index }) => {
           // Look up by the SAME seed identity the stamp above used — the
@@ -9332,7 +9401,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 )
               : row,
             index,
-            originalIndex: itemOriginalIndexes[index] ?? index,
+            // `index` already names the authored input, not a position in
+            // the pending subset. Looking it up again aliases resumed rows.
+            originalIndex: index,
           };
         });
       const admittedItems =
@@ -10268,16 +10339,31 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     );
     let nextRowIndex = 0;
     let firstRowError: unknown = null;
-    const providerRetryDeadlines: Array<{
-      provider: string;
-      retryAtMs: number;
-    }> = [];
-    let providerBackpressureRowCount = 0;
-    const waitingBaseRows = new Map<number, Record<string, unknown>>();
+    const providerRowRetries = new ProviderRowRetryQueue();
+    const parkedProviderRows = new ProviderRowRetryQueue();
+    const nextProviderRetry = (includeParked: boolean) => {
+      const resident = providerRowRetries.peek();
+      const parked = includeParked ? parkedProviderRows.peek() : undefined;
+      return !resident || (parked && parked.readyAtMs < resident.readyAtMs)
+        ? parked
+        : resident;
+    };
+    const retryWakeFeed = rowFeed ?? new RuntimeMapRowFeed();
+    const waitingBaseRows = new Map<
+      number,
+      {
+        baseRow: Record<string, unknown>;
+        callIndexes: RowReplayCallIndexes;
+      }
+    >();
 
     const runRow = async (
       idx: number,
       baseRow: Record<string, unknown>,
+      retryAfterRelease: {
+        pending: ProviderRowRetry | null;
+        callIndexes: RowReplayCallIndexes;
+      },
     ): Promise<MapRowExecutionResult> => {
       const item = items[idx];
       const rowKey = executionRowKey(baseRow, idx);
@@ -10350,6 +10436,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 mapStallResolverToken: resolverObservation?.token,
                 resolverGap,
                 providerBackpressure,
+                replayCallIndexes: retryAfterRelease.callIndexes,
               },
               async () => {
                 try {
@@ -10559,7 +10646,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         return COMPLETED_ROW;
       } catch (error) {
         if (isPlayRowExecutionSuspendedError(error)) {
-          waitingBaseRows.set(idx, baseRow);
+          waitingBaseRows.set(idx, {
+            baseRow,
+            callIndexes: retryAfterRelease.callIndexes,
+          });
           this.pendingRowEventBoundaries.push(error.boundary);
           mapProgress.update({
             pendingRowKey: rowKey,
@@ -10582,14 +10672,21 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           return WAITING_ROW;
         }
         if (error instanceof PlayRowProviderBackpressureError) {
-          waitingBaseRows.set(idx, baseRow);
+          waitingBaseRows.set(idx, {
+            baseRow,
+            callIndexes: retryAfterRelease.callIndexes,
+          });
           // Each row retains its own deadline. Provider names are not rate
           // scopes: two credentials for one provider can recover separately.
-          providerRetryDeadlines.push({
+          retryAfterRelease.pending = {
+            rowIndex: idx,
             provider: error.provider,
             retryAtMs: error.retryAtMs,
-          });
-          providerBackpressureRowCount += 1;
+            readyAtMs: Math.max(Date.now() + 1_000, error.retryAtMs),
+            ...(error.residentWaitBudgetExhausted
+              ? { residentWaitBudgetExhausted: true }
+              : {}),
+          };
           mapProgress.update({ pendingRowKey: rowKey });
           const fieldName = activeFieldName;
           if (!fieldName) {
@@ -10703,7 +10800,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         this.#options.rowAdmissionSignal?.reason ??
         new Error('Row admission stopped.');
       heapAdmission.close(firstRowError);
-      rowFeed?.fail(firstRowError);
+      retryWakeFeed.fail(firstRowError);
     };
     this.#options.rowAdmissionSignal?.addEventListener('abort', stopAdmission, {
       once: true,
@@ -10718,7 +10815,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       if (firstRowError == null) firstRowError = cause;
       this.wakeToolDispatcher();
       heapAdmission.close(firstRowError);
-      rowFeed?.fail(firstRowError);
+      retryWakeFeed.fail(firstRowError);
     };
     const unsubscribePersistenceLatch = onRuntimePersistenceLatchTripped(
       this.persistenceLatch,
@@ -10736,29 +10833,109 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       PROGRESS_HEARTBEAT_INTERVAL_MS,
     );
     mapActivityTimer.unref();
+    // Short holds should reuse the live sandbox after every row returns its
+    // leases. Bound total idle residency from the first all-waiting transition,
+    // rather than resetting it for each successive provider response.
+    let providerIdleWaitStartedAt: number | null = null;
+    const canWaitForProviderRetry = (retry: ProviderRowRetry) => {
+      const elapsedMs =
+        providerIdleWaitStartedAt === null
+          ? 0
+          : performance.now() - providerIdleWaitStartedAt;
+      return (
+        Math.max(1, retry.readyAtMs - Date.now()) <=
+        Math.max(0, TOOL_PROVIDER_RESIDENT_WAIT_BUDGET_MS - elapsedMs)
+      );
+    };
     const workers = Array.from({ length: workerCount }, async () => {
       while (firstRowError == null) {
-        if (nextRowIndex >= items.length) {
+        // A short provider hold must not leave this row behind the entire
+        // fresh backlog. Its previous flow released both admission leases;
+        // take a new one after its own deadline while other rows make progress.
+        const nextReadyRetry = nextProviderRetry(true);
+        const providerRetry = nextReadyRetry?.residentWaitBudgetExhausted
+          ? parkedProviderRows.takeReady(Date.now())
+          : providerRowRetries.takeReady(Date.now());
+        if (!providerRetry && nextRowIndex >= items.length) {
+          // Spent calls may join productive work, but only an unspent call can
+          // keep an otherwise idle sandbox waiting for its next deadline.
+          const nextRetry = nextProviderRetry(mapProgress.activeResolvers > 0);
+          if (
+            nextRetry &&
+            (mapProgress.activeResolvers > 0 ||
+              canWaitForProviderRetry(nextRetry)) &&
+            (!rowFeed || rowFeed.closed)
+          ) {
+            if (
+              mapProgress.activeResolvers === 0 &&
+              providerIdleWaitStartedAt === null
+            )
+              providerIdleWaitStartedAt = performance.now();
+            await retryWakeFeed.waitForWorkerWake(nextRetry.readyAtMs);
+            continue;
+          }
           if (!rowFeed) return;
+          if (nextRetry) rowFeed.wakeWorkersAt(nextRetry.readyAtMs);
           const more = await rowFeed.waitForRows(nextRowIndex);
-          if (!more) return;
+          const pendingRetry = nextProviderRetry(
+            mapProgress.activeResolvers > 0,
+          );
+          if (
+            !more &&
+            !(
+              pendingRetry &&
+              (mapProgress.activeResolvers > 0 ||
+                canWaitForProviderRetry(pendingRetry))
+            )
+          )
+            return;
           continue;
         }
-        const idx = nextRowIndex;
-        nextRowIndex += 1;
-        rowFeed?.markDispatched();
+        const idx = providerRetry?.rowIndex ?? nextRowIndex++;
+        if (!providerRetry) rowFeed?.markDispatched();
         try {
-          const baseRow = this.toOutputRow(items[idx]);
-          releaseDispatchedInputRow?.(idx);
+          const waitingRow = providerRetry
+            ? waitingBaseRows.get(idx)
+            : undefined;
+          const baseRow = providerRetry
+            ? waitingRow?.baseRow
+            : this.toOutputRow(items[idx]);
+          if (!baseRow) {
+            throw new Error(
+              `Provider retry row ${idx} lost its resumable input.`,
+            );
+          }
+          if (providerRetry) waitingBaseRows.delete(idx);
+          else releaseDispatchedInputRow?.(idx);
+          const callIndexes =
+            waitingRow?.callIndexes ?? new RowReplayCallIndexes();
+          callIndexes.resetAttempt();
+          const retryAfterRelease = {
+            pending: null as ProviderRowRetry | null,
+            callIndexes,
+          };
           settledResults[idx] = await mapRowFlow.run({ baseRow }, async () => {
             this.assertRowAdmissionOpen();
-            mapProgress.resolverStarted();
+            mapProgress.resolverStarted({ retry: Boolean(providerRetry) });
             try {
-              return await runRow(idx, baseRow);
+              return await runRow(idx, baseRow, retryAfterRelease);
             } finally {
               mapProgress.resolverFinished();
             }
           });
+          // Publish only after both flow leases have returned. Another worker
+          // must never re-admit this index while the old attempt is unwinding.
+          if (retryAfterRelease.pending) {
+            if (retryAfterRelease.pending.residentWaitBudgetExhausted) {
+              parkedProviderRows.add(retryAfterRelease.pending);
+            } else {
+              providerRowRetries.add(retryAfterRelease.pending);
+            }
+            rowFeed?.wakeWorkersAt(nextProviderRetry(true)!.readyAtMs);
+          } else if (settledResults[idx] !== WAITING_ROW) {
+            providerIdleWaitStartedAt = null;
+          }
+          if (mapProgress.activeResolvers === 0) retryWakeFeed.wakeWorkers();
         } catch (error) {
           // RuntimeFlow has already returned the row slot and memory lease.
           firstRowError = this.pendingDurableSuspension
@@ -10766,7 +10943,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             : (firstRowError ?? error);
           this.wakeToolDispatcher();
           heapAdmission.close(firstRowError);
-          rowFeed?.fail(firstRowError);
+          retryWakeFeed.fail(firstRowError);
           throw error;
         }
       }
@@ -10852,6 +11029,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       }
     } finally {
       clearInterval(mapActivityTimer);
+      retryWakeFeed.wakeWorkers();
       publishMapActivity();
       datasetStep.items = currentTotalRows();
       this.processedRowCount = currentTotalRows();
@@ -10932,19 +11110,21 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     }
     if (
       this.pendingRowEventBoundaries.length > 0 ||
-      providerRetryDeadlines.length > 0
+      providerRowRetries.size > 0 ||
+      parkedProviderRows.size > 0
     ) {
+      const waitingProviderRows =
+        providerRowRetries.size + parkedProviderRows.size;
       const waitingRowKeys = new Set<string>();
       const previousSuspendedInputs =
         this.checkpoint.mapFrames?.[mapScope.mapInvocationId]
           ?.suspendedInputRows;
       const suspendedInputRows =
-        rowFeed ||
-        (providerRetryDeadlines.length === 0 && !previousSuspendedInputs)
+        rowFeed || (waitingProviderRows === 0 && !previousSuspendedInputs)
           ? undefined
           : rowResults.flatMap((result, index) => {
               if (result !== WAITING_ROW) return [];
-              const baseRow = waitingBaseRows.get(index);
+              const baseRow = waitingBaseRows.get(index)?.baseRow;
               if (!baseRow) {
                 throw new Error(
                   `Waiting row ${index} lost its resumable input before map suspension.`,
@@ -10976,13 +11156,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           ]),
         ).values(),
       ];
-      const earliestHold = providerRetryDeadlines.reduce<
-        (typeof providerRetryDeadlines)[number] | null
-      >(
-        (earliest, next) =>
-          !earliest || next.retryAtMs < earliest.retryAtMs ? next : earliest,
-        null,
-      );
+      const residentHold = providerRowRetries.peek();
+      const parkedHold = parkedProviderRows.peek();
+      const earliestHold =
+        !residentHold ||
+        (parkedHold && parkedHold.retryAtMs < residentHold.retryAtMs)
+          ? parkedHold
+          : residentHold;
       const providerWakeAtMs = earliestHold
         ? Math.max(Date.now() + 1_000, earliestHold.retryAtMs)
         : null;
@@ -11005,7 +11185,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             provider: earliestHold.provider,
             retry_at: new Date(providerWakeAtMs).toISOString(),
             wait_ms: Math.max(0, providerWakeAtMs - Date.now()),
-            waiting_rows: providerBackpressureRowCount,
+            waiting_rows: waitingProviderRows,
             table_namespace: normalizedTableNamespace,
           })}`,
           { level: 'warn' },
@@ -12534,8 +12714,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         ].join(':')
       : 'workflow';
     const callIndexKey = `${this.currentExecutionScope.receipt.namespace}:${logicalCallScope}:${normalizedKey}:${toolRequestIdentity}`;
-    const callIndex = this.toolCallIndexByKey.get(callIndexKey) ?? 0;
-    this.toolCallIndexByKey.set(callIndexKey, callIndex + 1);
+    const freshCallIndex = this.toolCallIndexByKey.get(callIndexKey) ?? 0;
+    const callIndex = nextReplayableCallIndex(
+      `tool:${callIndexKey}`,
+      freshCallIndex,
+    );
+    this.toolCallIndexByKey.set(
+      callIndexKey,
+      Math.max(freshCallIndex, callIndex + 1),
+    );
     const logicalCallId = stableDigest(`${callIndexKey}:${callIndex}`);
     this.resourceGovernor.recordLogicalToolCall();
     let executionAuthScopeDigest =
@@ -14399,8 +14586,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         : Math.max(0, Math.round(ms));
     if (this.#options.durableBoundaries) {
       const rowStore = rowContext.getStore();
-      const legacyIndex = this.sleepBoundaryIndex;
-      this.sleepBoundaryIndex += 1;
+      const legacyIndex = nextReplayableCallIndex(
+        'sleep:legacy',
+        this.sleepBoundaryIndex,
+      );
+      this.sleepBoundaryIndex = Math.max(
+        this.sleepBoundaryIndex,
+        legacyIndex + 1,
+      );
       const legacyBoundaryId = this.durableBoundaryId(
         `sleep-${legacyIndex}-${delayMs}`,
       );
@@ -14414,9 +14607,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           ? `${rowStore.mapScope.mapInvocationId}:${rowStore.rowKey ?? rowStore.rowId}`
           : `row:${rowStore.rowKey ?? rowStore.rowId}`;
         const scopedIndexKey = `${this.currentExecutionScope.receipt.namespace}:${rowScope}`;
-        const scopedIndex =
+        const freshScopedIndex =
           this.sleepBoundaryIndexByScope.get(scopedIndexKey) ?? 0;
-        this.sleepBoundaryIndexByScope.set(scopedIndexKey, scopedIndex + 1);
+        const scopedIndex = nextReplayableCallIndex(
+          `sleep:${scopedIndexKey}`,
+          freshScopedIndex,
+        );
+        this.sleepBoundaryIndexByScope.set(
+          scopedIndexKey,
+          Math.max(freshScopedIndex, scopedIndex + 1),
+        );
         const scopedBoundaryId = this.durableBoundaryId(
           `sleep-${stableDigest(`${rowScope}:${scopedIndex}`)}-${delayMs}`,
         );
@@ -14771,8 +14971,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     const rowStore = rowContext.getStore();
     const scope = rowStore ? `row-${rowStore.rowId}` : 'workflow';
     const callIndexKey = `${this.currentExecutionScope.receipt.namespace}:${scope}:${normalizedKey}`;
-    const callIndex = this.stepCallIndexByKey.get(callIndexKey) ?? 0;
-    this.stepCallIndexByKey.set(callIndexKey, callIndex + 1);
+    const freshCallIndex = this.stepCallIndexByKey.get(callIndexKey) ?? 0;
+    const callIndex = nextReplayableCallIndex(
+      `step:${callIndexKey}`,
+      freshCallIndex,
+    );
+    this.stepCallIndexByKey.set(
+      callIndexKey,
+      Math.max(freshCallIndex, callIndex + 1),
+    );
     const boundarySuffix = callIndex === 0 ? '' : `:${callIndex}`;
     const boundaryId = this.durableBoundaryId(
       `step-${scope}:${normalizedKey}${boundarySuffix}`,
@@ -14783,6 +14990,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         this.checkpoint.resolvedBoundaries?.[boundaryId],
       );
       if (existing?.kind === 'step' && 'output' in existing) {
+        const replayedCalls =
+          rowStore?.replayCallIndexes?.recoverStep(boundaryId) ?? false;
         this.runtimeLog(
           `ctx.step(${normalizedKey}): recovered result from checkpoint`,
         );
@@ -14792,6 +15001,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         // the next sibling operation. This is deliberately run-local state;
         // it is not a cross-run sleep cache or an input-derived identity.
         if (
+          !replayedCalls &&
           Number.isSafeInteger(existing.sleepCount) &&
           existing.sleepCount !== undefined &&
           existing.sleepCount > 0
@@ -14802,7 +15012,19 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       }
 
       const sleepBoundaryIndexBefore = this.sleepBoundaryIndex;
-      const output = await run();
+      const callAdvances = new Map<string, number>();
+      const output = rowStore?.replayCallIndexes
+        ? await rowContext.run(
+            {
+              ...rowStore,
+              replayStepCaptures: [
+                ...(rowStore.replayStepCaptures ?? []),
+                callAdvances,
+              ],
+            },
+            run,
+          )
+        : await run();
       assertJsonSerializableStepOutput(normalizedKey, output);
       // A caught durable suspension is still the invocation's outcome. Never
       // publish an enclosing step fallback that would make a later resume skip
@@ -14814,11 +15036,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           kind: 'step',
           stepId: normalizedKey,
           output,
-          sleepCount: this.sleepBoundaryIndex - sleepBoundaryIndexBefore,
+          sleepCount: rowStore?.replayCallIndexes
+            ? (callAdvances.get('sleep:legacy') ?? 0)
+            : this.sleepBoundaryIndex - sleepBoundaryIndexBefore,
           completedAt: Date.now(),
         },
       };
       this.#options.onBatchComplete?.(this.checkpoint);
+      rowStore?.replayCallIndexes?.completeStep(boundaryId, callAdvances);
       return output;
     };
 
@@ -17607,9 +17832,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               : options?.providerOperationForceRefreshIdentities;
           const executionRequestNonce = crypto.randomUUID();
           // Transport ambiguity must retain the billing identity. An ordinary
-          // (unfenced) call advances it only after a definite provider-owned
-          // 429 released that attempt's reservation. Fenced calls retain it:
-          // the gateway-qualified retry owns their distinct charge lifecycle.
+          // (unfenced) call advances it after a definite rejection released
+          // that attempt's reservation, including app pacing refusals. Fenced
+          // calls retain it: gateway-qualified retries own their distinct charge lifecycle.
           let deeplineRequestId = `ctx-tool-${stableDigest(executionRequestNonce).slice(0, 32)}`;
           // The provider operation has a semantic identity independent of
           // the response receipt. Its key is supplied by the caller that owns
@@ -17769,10 +17994,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 ? { integration_mode: this.#options.integrationMode }
                 : {}),
             });
+          // Legacy retry migration retains this transport identity even when
+          // the current checkpoint cursor index belongs to a newer batch.
+          const providerResidentWaitKey = durableCallReceiptKey;
           let transportAttempt = 0;
           let invocationAttempt =
             savedProviderRetryCursor?.invocationAttempt ?? 0;
           clearProviderRetryCursor = () => {
+            if (providerResidentWaitKey)
+              this.exhaustedProviderResidentWaitCalls.delete(
+                providerResidentWaitKey,
+              );
             if (
               !providerRetryCursorKey ||
               !this.checkpoint.providerRetryCursors
@@ -17838,6 +18070,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           // work. Repeated zero or expired holds must also yield durably.
           let residentProviderWaitStartedAt: number | null = null;
           const canWaitForProviderInSandbox = (waitMs: number) => {
+            if (
+              providerResidentWaitKey &&
+              this.exhaustedProviderResidentWaitCalls.has(
+                providerResidentWaitKey,
+              )
+            )
+              return false;
             const elapsedMs =
               residentProviderWaitStartedAt === null
                 ? 0
@@ -17847,6 +18086,21 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               Math.max(0, TOOL_PROVIDER_RESIDENT_WAIT_BUDGET_MS - elapsedMs)
             );
           };
+          const markProviderResidentWaitBudgetExhausted = () => {
+            const exhausted =
+              residentProviderWaitStartedAt !== null ||
+              Boolean(
+                providerResidentWaitKey &&
+                this.exhaustedProviderResidentWaitCalls.has(
+                  providerResidentWaitKey,
+                ),
+              );
+            if (exhausted && providerResidentWaitKey)
+              this.exhaustedProviderResidentWaitCalls.add(
+                providerResidentWaitKey,
+              );
+            return exhausted;
+          };
           const waitForToolRetry = async (
             waitMs: number,
             providerHold: boolean,
@@ -17947,6 +18201,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           // the tool-keyed dispatch identity so terminal attribution can join
           // provider admission with the response residence it admitted.
           let providerTelemetryKey: string | null = null;
+          let providerPermitReceiverUnavailable = false;
           let providerRunInFlightAtDispatch: number | null = null;
           const acquireProviderPermit = async (signal?: AbortSignal) => {
             options?.onProviderAdmissionPending?.();
@@ -17955,6 +18210,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               toolId,
               signal,
               yieldProviderHolds: this.#options.durableBoundaries === true,
+              ...(invocationAttempt > 0 ||
+              residentProviderWaitStartedAt !== null
+                ? { retryPriority: true }
+                : {}),
             });
             providerConcurrencyGrant = permit.concurrencyGrant ?? null;
             providerPacingGrant = permit.pacingGrant ?? null;
@@ -18000,6 +18259,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               release: () => void;
             } | null = null;
             let fetchDispatched = false;
+            let dispatchedWithProviderPermit = false;
+            let providerPermitReceiverRejected = false;
+            let providerDispatchCounted = false;
+            const recordProviderDispatch = () => {
+              if (providerDispatchCounted) return;
+              providerDispatchCounted = true;
+              this.resourceGovernor.recordProviderDispatch();
+            };
             // A provider 429 must be reported while its server-minted
             // admission lease is still live. Keep its parsed outcome here so
             // the ordinary retry boundary can reuse it after Flow releases the
@@ -18142,6 +18409,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                     signal: abortController?.signal,
                     requiresTransportPermit: !fixtureExecution,
                     deferProviderPermitValidation: true,
+                    providerDispatchAtGateway:
+                      !providerPermitReceiverUnavailable,
                     onTransportAdmitted: (waitMs) => {
                       transportSlotWaitMs = waitMs;
                     },
@@ -18327,63 +18596,90 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                           logProvider429RetryPhase('http_dispatch_started');
                           activeProviderRequest =
                             this.beginProviderRequest(provider);
-                          // Count at dispatch, including ambiguous transport
-                          // failures that never produce a residence sample.
-                          this.resourceGovernor.recordProviderDispatch();
-                          const dispatchedResponse = await fetch(url, {
-                            method: 'POST',
-                            signal: abortController?.signal,
-                            headers: {
-                              'Content-Type': 'application/json',
-                              Authorization: `Bearer ${this.#options.executorToken}`,
-                              [EXECUTE_RESPONSE_CONTRACT_HEADER]:
-                                RAW_V2_TOOL_RESPONSE_CONTRACT,
-                              [EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
-                              [EXECUTE_TOOL_METADATA_HEADER]: 'true',
-                              [TOOL_EXECUTION_ERROR_SCHEMA_HEADER]: String(
-                                toolErrorSchemaVersion,
-                              ),
-                              // Tell the gateway which protective outcomes this
-                              // runner can absorb. The relay may only shed a
-                              // provider execute with the Deepline concurrency
-                              // shape when this token is present; without it the
-                              // relay must queue, because an older artifact would
-                              // settle the row as failed instead of pacing.
-                              [TOOL_EXECUTE_RUNNER_ACCEPTS_HEADER]:
-                                TOOL_EXECUTE_RUNNER_ACCEPTS_VALUE,
-                              ...(providerGatewayDispatchProof
-                                ? {
-                                    [TOOL_EXECUTE_PROVIDER_RATE_SCOPE_HEADER]:
-                                      providerGatewayDispatchProof.rateScopeToken,
-                                    [TOOL_EXECUTE_PROVIDER_RATE_RETRY_SNAPSHOT_HEADER]:
-                                      String(
-                                        providerGatewayDispatchProof.retryAtMsAtGrant,
-                                      ),
-                                  }
-                                : {}),
-                              'x-deepline-request-id': deeplineRequestId,
-                              ...(providerExecutionFenceKey
-                                ? {
-                                    'x-deepline-idempotency-key':
-                                      providerExecutionFenceKey,
-                                  }
-                                : {}),
-                              ...protectionHeaders,
-                              ...(this.#options.runtimeTestFaultHeader
-                                ? {
-                                    'x-deepline-test-fault':
-                                      this.#options.runtimeTestFaultHeader,
-                                  }
-                                : {}),
+
+                          // The new action binds delegated validation to the
+                          // actual receiver. Older gateways refuse this path
+                          // before admission; direct app routes do not serve it.
+                          dispatchedWithProviderPermit =
+                            !providerPermitReceiverUnavailable &&
+                            providerGatewayDispatchProof?.dispatchValidationSupported ===
+                              true;
+                          if (!dispatchedWithProviderPermit)
+                            recordProviderDispatch();
+                          const dispatchedResponse = await fetch(
+                            dispatchedWithProviderPermit
+                              ? `${url}${TOOL_EXECUTE_PROVIDER_PERMIT_PATH_SUFFIX}`
+                              : url,
+                            {
+                              method: 'POST',
+                              signal: abortController?.signal,
+                              headers: {
+                                'Content-Type': 'application/json',
+                                Authorization: `Bearer ${this.#options.executorToken}`,
+                                [EXECUTE_RESPONSE_CONTRACT_HEADER]:
+                                  RAW_V2_TOOL_RESPONSE_CONTRACT,
+                                [EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
+                                [EXECUTE_TOOL_METADATA_HEADER]: 'true',
+                                [TOOL_EXECUTION_ERROR_SCHEMA_HEADER]: String(
+                                  toolErrorSchemaVersion,
+                                ),
+                                // Tell the gateway which protective outcomes this
+                                // runner can absorb. The relay may only shed a
+                                // provider execute with the Deepline concurrency
+                                // shape when this token is present; without it the
+                                // relay must queue, because an older artifact would
+                                // settle the row as failed instead of pacing.
+                                [TOOL_EXECUTE_RUNNER_ACCEPTS_HEADER]:
+                                  TOOL_EXECUTE_RUNNER_ACCEPTS_VALUE,
+                                ...(!usesDurableInvocationFence &&
+                                !providerExecutionFenceKey
+                                  ? {
+                                      [TOOL_EXECUTE_APP_PACING_REFUSAL_HEADER]:
+                                        TOOL_EXECUTE_APP_PACING_REFUSAL_VALUE,
+                                    }
+                                  : {}),
+                                ...(providerGatewayDispatchProof
+                                  ? {
+                                      [TOOL_EXECUTE_PROVIDER_RATE_SCOPE_HEADER]:
+                                        providerGatewayDispatchProof.rateScopeToken,
+                                      [TOOL_EXECUTE_PROVIDER_RATE_RETRY_SNAPSHOT_HEADER]:
+                                        String(
+                                          providerGatewayDispatchProof.retryAtMsAtGrant,
+                                        ),
+                                    }
+                                  : {}),
+                                'x-deepline-request-id': deeplineRequestId,
+                                ...(providerExecutionFenceKey
+                                  ? {
+                                      'x-deepline-idempotency-key':
+                                        providerExecutionFenceKey,
+                                    }
+                                  : {}),
+                                ...protectionHeaders,
+                                ...(this.#options.runtimeTestFaultHeader
+                                  ? {
+                                      'x-deepline-test-fault':
+                                        this.#options.runtimeTestFaultHeader,
+                                    }
+                                  : {}),
+                              },
+                              body: serializedRequestBody(invocationAttempt),
                             },
-                            body: serializedRequestBody(invocationAttempt),
-                          });
+                          );
                           // Error classification needs the status and headers even
                           // when the body is truncated. Keep the response visible
                           // to the enclosing retry/billing boundary as soon as
                           // headers arrive; decoded body data stays scoped until
                           // it has been read and verified below.
                           response = dispatchedResponse;
+                          if (
+                            dispatchedResponse.headers.has(
+                              TOOL_EXECUTE_PROVIDER_PERMIT_RECEIVER_HEADER,
+                            ) ||
+                            (dispatchedResponse.status !== 403 &&
+                              dispatchedResponse.status !== 404)
+                          )
+                            recordProviderDispatch();
                           if (dispatchedResponse.status === 429) {
                             const observedAt = Date.now();
                             const retryAfter =
@@ -18539,6 +18835,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                               );
                             providerBodyAdmissionWaitMs += bodyRead.waitMs;
                           }
+                          providerPermitReceiverRejected =
+                            dispatchedWithProviderPermit &&
+                            isProviderPermitExecuteReceiverUnavailable(
+                              dispatchedResponse,
+                              dispatchedResponseErrorText ?? '',
+                            );
+                          if (!providerPermitReceiverRejected)
+                            recordProviderDispatch();
                           if (dispatchedResponse.status === 429) {
                             const bodyText = dispatchedResponseErrorText ?? '';
                             let attempt = httpFailureAttempts.next({
@@ -18616,6 +18920,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                           // diagnostic counter now; the flow still holds the
                           // real provider lease until this callback returns.
                           releaseProviderExecution();
+                          if (providerPermitReceiverRejected)
+                            return 'not_dispatched' as const;
                           const failure =
                             preclassifiedHttpFailure.current?.failure;
                           if (
@@ -18658,6 +18964,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                         },
                       );
                     } catch (error) {
+                      // A failed exchange whose dispatch is unknown remains a
+                      // physical attempt, including failure before headers.
+                      if (fetchDispatched && !providerPermitReceiverRejected)
+                        recordProviderDispatch();
                       // `request()` can fail while waiting for its transport
                       // lease, before its beforeRelease callback runs.
                       activeProviderRequest?.release();
@@ -18693,24 +19003,25 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 providerCallElapsedMs =
                   providerBodyCompletedAtMs - providerCallStartedAt;
                 options?.onProviderBodyCompleted?.(providerBodyCompletedAtMs);
-                this.resourceGovernor.observe({
-                  toolId,
-                  providerResourceKey: `tool:${toolId}`,
-                  providerTelemetryKey,
-                  providerConcurrencyGrant,
-                  providerPacingGrant,
-                  providerRunInFlightAtDispatch,
-                  providerLatencyMs: providerCallElapsedMs,
-                  fixtureDelayMs,
-                  transportSlotWaitMs,
-                  fetchToHeadersMs,
-                  executeServerTiming,
-                  bodyReadMs,
-                  providerBodyAdmissionWaitMs:
-                    dispatched.providerBodyAdmissionWaitMs,
-                  providerSuccess: response.ok,
-                  provider429: response.status === 429,
-                });
+                if (!providerPermitReceiverRejected)
+                  this.resourceGovernor.observe({
+                    toolId,
+                    providerResourceKey: `tool:${toolId}`,
+                    providerTelemetryKey,
+                    providerConcurrencyGrant,
+                    providerPacingGrant,
+                    providerRunInFlightAtDispatch,
+                    providerLatencyMs: providerCallElapsedMs,
+                    fixtureDelayMs,
+                    transportSlotWaitMs,
+                    fetchToHeadersMs,
+                    executeServerTiming,
+                    bodyReadMs,
+                    providerBodyAdmissionWaitMs:
+                      dispatched.providerBodyAdmissionWaitMs,
+                    providerSuccess: response.ok,
+                    provider429: response.status === 429,
+                  });
               } finally {
                 receiptHeartbeatSupervisor?.stop();
               }
@@ -18747,6 +19058,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                     throw new PlayRowProviderBackpressureError(
                       error.provider,
                       Number.isFinite(retryAtMs) ? retryAtMs : Date.now(),
+                      markProviderResidentWaitBudgetExhausted(),
                     );
                   }
                   // This is a provider admission hold, not a row outcome.
@@ -18904,6 +19216,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             }
             span.setAttribute('plays.http_status_code', response.status);
 
+            if (providerPermitReceiverRejected) {
+              // This named action was rejected before provider work. Retry the
+              // same invocation once through the legacy worker validation path.
+              providerPermitReceiverUnavailable = true;
+              continue;
+            }
+
             if (!response.ok) {
               const text = responseErrorText ?? '';
               const developerTunnelOrigin502 =
@@ -19075,6 +19394,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                   throw new PlayRowProviderBackpressureError(
                     provider,
                     retryAtMs,
+                    markProviderResidentWaitBudgetExhausted(),
                   );
                 }
                 await this.waitForProviderBackpressure(provider, retryAtMs);
@@ -19185,14 +19505,19 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 if (
                   !usesDurableInvocationFence &&
                   (failure.backpressureDelayMs !== null ||
+                    failure.reason === 'provider_pacing_wait' ||
                     (failure.reason === 'rate_limit' &&
                       failure.knownNotDispatched))
                 ) {
-                  // A classified provider rejection has no completed value to
-                  // charge. A credential guard refusal also releases its charge
-                  // before replying. The next attempt needs a new reservation;
-                  // durable result and provider-operation keys stay unchanged.
-                  deeplineRequestId = `ctx-tool-${stableDigest(`${executionRequestNonce}:provider-rate-limit-retry:${invocationAttempt}`).slice(0, 32)}`;
+                  // These refusals release the prior reservation. App pacing
+                  // does not advance invocationAttempt, so use its own HTTP
+                  // failure count to avoid reusing a released retry reservation.
+                  // Durable result and provider-operation keys stay unchanged.
+                  const billingRetryIdentity =
+                    failure.reason === 'provider_pacing_wait'
+                      ? `app-pacing-refusal:${httpFailureAttempt}`
+                      : `provider-rate-limit-retry:${invocationAttempt}`;
+                  deeplineRequestId = `ctx-tool-${stableDigest(`${executionRequestNonce}:${billingRetryIdentity}`).slice(0, 32)}`;
                 }
                 this.emitExecutionEvent({
                   type: RUN_EVENT_KIND.activityObserved,