deepline 0.3.201 → 0.3.203

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -125,6 +125,11 @@ import {
   ProviderExhaustedError,
 } from './run-failure';
 import { PROVIDER_EXHAUSTED_MAX_WAIT_MS } from './governor/app-runtime-rate-state-backend';
+import {
+  observeProviderNoProgress,
+  providerNoProgressExpired,
+  PROVIDER_NO_PROGRESS_MAX_MS,
+} from './provider-no-progress';
 import {
   buildPlayContractCompatibility,
   normalizePlayContractCompatibility,
@@ -174,6 +179,7 @@ import {
   createToolCallReceiptCohortJob,
   createRuntimeStepToolResultReceipts,
   createToolCallJob,
+  resolveToolCallResultReceiptPolicy,
   ToolCallReceiptLeaseLostError,
   type ToolCallReceiptCohortStore,
   type ToolCallReceiptCohortTimings,
@@ -197,6 +203,7 @@ import {
   classifyToolExecuteHttpFailure,
   createToolExecuteHttpFailureAttemptTracker,
   parseToolExecuteRetryAfterMs,
+  parseToolExecuteRetryAtMs,
   parseToolExecuteAuthScopeChangedError,
   ToolExecuteAuthScopeChangedError,
   type ToolExecuteHttpFailureOutcome,
@@ -494,6 +501,8 @@ const rowContext = new AsyncLocalStorage<{
    * null between segments. Measurement only; see `RowResolverGapTracker`.
    */
   resolverGap?: RowResolverGapTracker;
+  /** Shared through nested fields, reset for each row execution attempt. */
+  providerBackpressure?: { pending: PlayRowProviderBackpressureError | null };
 }>();
 
 /**
@@ -562,6 +571,7 @@ const TOOL_RETRY_AFTER_FALLBACK_MS = 1_000;
 // broken gateway-to-runner connection, not slow provider work.
 export const TOOL_EXECUTE_RESPONSE_BODY_TIMEOUT_MS = 30_000;
 const TOOL_RETRY_HEARTBEAT_INTERVAL_MS = 30_000;
+const TOOL_PROVIDER_RESIDENT_WAIT_BUDGET_MS = 5_000;
 const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000;
 // Every runtime-API tool fetch needs a client-side deadline. Without one, a
 // stalled execute request (e.g. a credential-less call whose gateway connection
@@ -3104,15 +3114,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
   /**
    * Called at ctx effect boundaries and by the runner before it can settle a
    * normal result. This makes suspension sticky even when authored code uses a
-   * broad catch around `await ctx.sleep(...)`.
+   * broad catch around a sleep or provider hold. A row hold stays row-local
+   * until its siblings drain and the map commits the scheduler handoff.
    */
   assertNoPendingDurableSuspension(): void {
+    this.#options.executionSignal?.throwIfAborted();
     this.assertNoSlackHitlTimeoutFailure();
+    this.assertNoRowProviderBackpressure();
     if (this.pendingDurableSuspension) {
       throw new PlayExecutionSuspendedError(this.pendingDurableSuspension);
     }
   }
 
+  private assertNoRowProviderBackpressure(): void {
+    const pending = rowContext.getStore()?.providerBackpressure?.pending;
+    if (pending) throw pending;
+  }
+
   /**
    * The single seam that hands this invocation back to the scheduler. The
    * first suspension wins: a later boundary reached by code that caught the
@@ -6795,7 +6813,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       execution: {
         mode: 'map',
         ownerRunId: this.currentReceiptOwnerRunId,
-        force: input.request.force === true,
+        force: input.resultReceipts
+          ? this.mappedToolResultReceiptPolicy(input.request).force
+          : input.request.force === true,
       },
     };
     if (alwaysFresh) {
@@ -6829,8 +6849,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
   }
 
   /**
-   * A forced call bypasses the immutable Tool Result Receipt, so its physical
-   * gateway invocation must bypass that receipt's replay identity too. Keep
+   * A forced call bypasses an earlier run's shared result, so its physical
+   * gateway invocation must bypass that result's replay identity too. Keep
    * the transport key deterministic for one logical call: a retry of that
    * call still converges, while a later explicit force gets a fresh identity.
    */
@@ -6847,6 +6867,22 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     })}${stableDigest(`forced-provider-invocation:${input.providerOperationKey}`)}`;
   }
 
+  private mappedToolResultReceiptPolicy(request: ToolCallRequest) {
+    if (!request.receiptKey || !request.logicalCallId) {
+      throw new Error(
+        'Mapped Tool Call needs stable result and call identities.',
+      );
+    }
+    return resolveToolCallResultReceiptPolicy({
+      orgId: this.#options.orgId,
+      toolId: request.toolId,
+      resultReceiptKey: request.receiptKey,
+      runId: this.currentRunId,
+      logicalCallId: request.logicalCallId,
+      force: request.force === true,
+    });
+  }
+
   /**
    * Ordinary operations do not carry provider-idempotency metadata, but the
    * execution relay still needs a semantic signal that an explicit force is
@@ -10733,6 +10769,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       const rowKey = executionRowKey(baseRow, idx);
       const rowIndex = executionRowIndex(idx);
       const computedFields: Record<string, unknown> = {};
+      // A provider hold is scheduler control, not a catchable row outcome.
+      // Keep its intent outside authored code so catch/fallback logic cannot
+      // publish a successful row that silently omitted an unfinished call.
+      const providerBackpressure = {
+        pending: null as PlayRowProviderBackpressureError | null,
+      };
       let activeFieldName: string | null = null;
       // ADR 0019: the row's read order, computed once. Every cell reads a
       // prefix of it — the inputs, then each column this run wrote — so this
@@ -10793,6 +10835,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 mapStallObserver: mapStallObserver ?? undefined,
                 mapStallResolverToken: resolverObservation?.token,
                 resolverGap,
+                providerBackpressure,
               },
               async () => {
                 try {
@@ -12675,7 +12718,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       return resolver;
     }
 
-    const value = await resolver(row, this, fields, index, previousCell);
+    let value: unknown;
+    try {
+      value = await resolver(row, this, fields, index, previousCell);
+    } catch (error) {
+      // Cancellation and persistence/ownership failures stay run-fatal. A
+      // provider wait must never convert them into another scheduled retry.
+      if (isRowIsolationExemptError(error)) throw error;
+      // Also retain the wait when authored fallback code throws a different
+      // error after catching it. This check is local to the held row.
+      this.assertNoRowProviderBackpressure();
+      throw error;
+    }
     // This is the common user-code return seam for every mapped field path
     // (incremental, pure, and step-program adapters). Check before callers
     // serialize a cell, emit a completion event, or advance a map frame.
@@ -12772,6 +12826,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     options?: ToolCallOptions,
   ): Promise<unknown> {
     if (!rowContext.getStore()) this.assertRowAdmissionOpen();
+    this.assertNoPendingDurableSuspension();
     if (options?.description !== undefined) {
       validatePlayAuthoringField(
         'ctx.tools.execute.description',
@@ -13184,6 +13239,21 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 toolId,
               })}${stableDigest(`${this.currentRunId}:${logicalCallId}:always-fresh`)}`;
           const resultReceiptKey = directResultReceiptKey;
+          const resultReceiptPolicy = resolveToolCallResultReceiptPolicy({
+            orgId: this.#options.orgId,
+            toolId,
+            resultReceiptKey,
+            runId: this.currentRunId,
+            logicalCallId,
+            force: toolCachePolicy.force,
+            canStoreResult: Boolean(
+              this.#options.claimRuntimeStepReceipt &&
+              (this.#options.getRuntimeStepReceipt ||
+                this.#options.getRuntimeStepReceipts) &&
+              (this.#options.completeRuntimeStepReceipt ||
+                this.#options.completeRuntimeStepReceipts),
+            ),
+          });
           const toolRetryPolicy = await this.#options
             .getToolRetryPolicy?.(toolId, input)
             .catch(() => null);
@@ -13337,7 +13407,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                         },
                       }
                     : {}),
-                  recover: ({ result, resultReceiptKey, source }) =>
+                  recover: ({ result, source }) =>
                     cloneToolExecuteResultWithExecution(
                       result,
                       toolExecutionMetadataForOutcome(
@@ -13372,14 +13442,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             execution: {
               mode: 'direct',
               ownerRunId: this.currentReceiptOwnerRunId,
-              force: toolCachePolicy.force,
+              force: resultReceiptPolicy.force,
             },
           };
           const wrapped = toolCall
             ? await toolCall.call({
                 ...call,
                 compatibility: {
-                  resultReceiptKey,
+                  resultReceiptKey: resultReceiptPolicy.resultReceiptKey,
                   providerOperationKey,
                 },
               })
@@ -13593,7 +13663,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     };
 
     if (store || !cacheableToolResult) {
-      return await executeTool();
+      try {
+        return await executeTool();
+      } catch (error) {
+        if (
+          error instanceof PlayRowProviderBackpressureError &&
+          store?.providerBackpressure
+        ) {
+          // The batch dispatcher has no reliable single-row async scope.
+          // Capture the signal at delivery to its actual requesting row,
+          // before authored code can catch the rejected tool promise.
+          store.providerBackpressure.pending ??= error;
+        }
+        throw error;
+      }
     }
 
     for (
@@ -14554,6 +14637,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
 
   async sleep(ms: number): Promise<void> {
     if (!rowContext.getStore()) this.assertRowAdmissionOpen();
+    this.assertNoPendingDurableSuspension();
     this.assertInlineChildContract('suspending_child');
     const delayMs =
       this.currentRuntimeSemantics.validation === 'strict'
@@ -15555,7 +15639,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             const request = input.requestForReceipt.get(receiptKey);
             const candidates = [
               receiptKey,
-              ...(request?.legacyReceiptKeys ?? []),
+              ...(request?.force === true
+                ? []
+                : (request?.legacyReceiptKeys ?? [])),
             ];
             for (const candidate of candidates) {
               const record = await asRecord(
@@ -15577,8 +15663,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             ...new Set(
               resultReceiptKeys.flatMap((receiptKey) => [
                 receiptKey,
-                ...(input.requestForReceipt.get(receiptKey)
-                  ?.legacyReceiptKeys ?? []),
+                ...(input.requestForReceipt.get(receiptKey)?.force === true
+                  ? []
+                  : (input.requestForReceipt.get(receiptKey)
+                      ?.legacyReceiptKeys ?? [])),
               ]),
             ),
           ];
@@ -15777,12 +15865,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     const requestsForReceipt = new Map<string, ToolCallRequest[]>();
     const requestByCallId = new Map<string, ToolCallRequest>();
     for (const request of requests) {
-      const receiptKey = request.receiptKey?.trim();
-      if (!receiptKey) {
-        throw new Error(
-          'Cacheable non-native map Tool Call cohort requires receipt keys.',
-        );
-      }
+      const receiptKey =
+        this.mappedToolResultReceiptPolicy(request).resultReceiptKey;
       requestForReceipt.set(receiptKey, request);
       const shared = requestsForReceipt.get(receiptKey);
       if (shared) shared.push(request);
@@ -15806,8 +15890,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       },
       members: requests.map((request) => ({
         memberId: request.callId,
-        resultReceiptKey: request.receiptKey!,
-        force: request.force === true,
+        ...this.mappedToolResultReceiptPolicy(request),
         forceFailedRefresh: request.forceFailedRefresh === true,
         waitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
           typeof request.receiptWaitMs === 'number'
@@ -15990,7 +16073,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               request,
               dispatchToolId: toolId,
               dispatchPayload: request.input,
-              resultReceiptKey: request.receiptKey!,
+              resultReceiptKey:
+                this.mappedToolResultReceiptPolicy(request).resultReceiptKey,
               providerOperationKey,
               resultReceipts: receipts,
               apiOptions: {
@@ -16052,12 +16136,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     const requestsForReceipt = new Map<string, ToolCallRequest[]>();
     const requestByCallId = new Map<string, ToolCallRequest>();
     for (const request of requests) {
-      const receiptKey = request.receiptKey?.trim();
-      if (!receiptKey) {
-        throw new Error(
-          'Cacheable native map Tool Call cohort requires receipt keys.',
-        );
-      }
+      const receiptKey =
+        this.mappedToolResultReceiptPolicy(request).resultReceiptKey;
       requestForReceipt.set(receiptKey, request);
       const shared = requestsForReceipt.get(receiptKey);
       if (shared) shared.push(request);
@@ -16101,12 +16181,22 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     const successful: ToolCallRequest[] = [];
 
     const deliver = (request: ToolCallRequest, result: ToolExecuteResult) => {
+      const presented =
+        request.force === true
+          ? cloneToolExecuteResultWithExecution(result, {
+              ...result._metadata.execution,
+              receiptKey: request.receiptKey!,
+              ...(result._metadata.execution.attachedToReceiptKey
+                ? { attachedToReceiptKey: request.receiptKey! }
+                : {}),
+            })
+          : result;
       if (request.cacheable !== false) {
-        this.cacheToolResult(toolId, request.cacheKey, result);
+        this.cacheToolResult(toolId, request.cacheKey, presented);
       }
       const resolver = this.toolCallResolvers.get(request.callId);
       if (resolver) {
-        resolver.resolve(result);
+        resolver.resolve(presented);
         this.toolCallResolvers.delete(request.callId);
       }
       this.emitScopedFieldMetaUpdate({
@@ -16115,12 +16205,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         tableNamespace: request.tableNamespace ?? null,
         fieldName: request.fieldName,
         status:
-          result._metadata.execution.cached === true ? 'cached' : 'running',
+          presented._metadata.execution.cached === true ? 'cached' : 'running',
         rowStatus: 'running',
         stage: toolId,
         provider: null,
         error: null,
-        ...(result._metadata.execution.cached === true ? { reused: true } : {}),
+        ...(presented._metadata.execution.cached === true
+          ? { reused: true }
+          : {}),
         dataPatch: {},
       });
       successful.push(request);
@@ -16309,10 +16401,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 execution: {
                   mode: 'map',
                   ownerRunId: this.currentReceiptOwnerRunId,
-                  force: request.force === true,
+                  force: this.mappedToolResultReceiptPolicy(request).force,
                 },
                 compatibility: {
-                  resultReceiptKey: request.receiptKey!,
+                  resultReceiptKey:
+                    this.mappedToolResultReceiptPolicy(request)
+                      .resultReceiptKey,
                   providerOperationKey: request.receiptKey!,
                 },
               },
@@ -16328,7 +16422,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                     compiled.batchOperation,
                   )
                 )?.trim() ?? '';
-              const receiptKeys = owned.map((request) => request.receiptKey!);
+              const receiptKeys = owned.map(
+                (request) =>
+                  this.mappedToolResultReceiptPolicy(request).resultReceiptKey,
+              );
               const aggregateReceiptKey = buildDurableToolAggregateReceiptKey({
                 receiptKeys,
                 prefix: 'batch',
@@ -16501,14 +16598,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             // before they are delivered to rows.
             const durable = await store.read({
               resultReceiptKeys: entry.request.memberRequests.map(
-                (request) => request.receiptKey!,
+                (request) =>
+                  this.mappedToolResultReceiptPolicy(request).resultReceiptKey,
               ),
             });
             const byReceipt = new Map(
               durable.map((record) => [record.resultReceiptKey, record]),
             );
             for (const request of entry.request.memberRequests) {
-              const record = byReceipt.get(request.receiptKey!);
+              const record = byReceipt.get(
+                this.mappedToolResultReceiptPolicy(request).resultReceiptKey,
+              );
               if (
                 record &&
                 (record.status === 'completed' ||
@@ -18557,6 +18657,52 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     const executionBaseUrl =
       this.#options.executionGatewayBaseUrl?.trim() || this.#options.baseUrl;
     const toolErrorSchemaVersion = this.currentToolErrorSchemaVersion;
+    const providerNoProgressScope = stableDigest(
+      `${provider}\u0000${executionAuthScopeDigest ?? ''}`,
+    );
+    const providerNoProgressError = (retryAtMs: number) =>
+      createToolHttpError(
+        toolErrorSchemaVersion,
+        `Provider ${provider} made no progress within ${PROVIDER_NO_PROGRESS_MAX_MS / 60_000} minutes. Completed work was saved; retry the failed work after the provider recovers.`,
+        null,
+        429,
+        'repairable',
+        {
+          toolId,
+          provider,
+          operation: toolId,
+          code: 'PROVIDER_EXHAUSTED',
+          origin: 'provider',
+          category: 'rate_limit',
+          retryable: true,
+          statusCode: 429,
+          requestId: null,
+          retryAfterMs: Math.max(0, retryAtMs - Date.now()),
+          networkKind: null,
+          networkScope: null,
+        },
+      );
+    const observeProviderWait = (retryAtMs: number) => {
+      const startedAtMsByScope =
+        (this.checkpoint.providerNoProgressStartedAtMsByScope ??= {});
+      const outcome = observeProviderNoProgress({
+        startedAtMsByScope,
+        scope: providerNoProgressScope,
+        retryAtMs,
+        nowMs: Date.now(),
+      });
+      if (!outcome.exhausted) return;
+      this.runtimeLog(
+        `[runtime.provider.no_progress] ${JSON.stringify({
+          provider,
+          started_at: new Date(outcome.startedAtMs).toISOString(),
+          deadline_at: new Date(outcome.deadlineMs).toISOString(),
+          retry_at: new Date(retryAtMs).toISOString(),
+        })}`,
+        { level: 'warn' },
+      );
+      throw providerNoProgressError(retryAtMs);
+    };
     const timeoutMs = resolveToolRuntimeTimeoutMs(
       toolId,
       options?.timeoutMs,
@@ -18594,6 +18740,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     const toolInputSnapshotMemory: {
       current: RuntimeMemoryLedgerHandle | null;
     } = { current: null };
+    let preserveProviderRetryCursorForSuspension = false;
+    let clearProviderRetryCursor = () => {};
     try {
       return await withActiveSpan(
         'plays.tool.execute',
@@ -18703,11 +18851,22 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                     `transport:${this.currentRunId}:${toolId}:${crypto.randomUUID()}`,
                   )}`
                 : null);
+          // A durable row replay reconstructs this function from scratch. Keep
+          // the gateway's invocation sequence in the checkpoint, keyed by the
+          // stable result receipt identity, until this logical call completes.
+          const providerRetryCursorKey =
+            this.#options.durableBoundaries && durableCallReceiptKey
+              ? stableDigest(`provider-retry:${durableCallReceiptKey}`)
+              : null;
+          const savedProviderRetryCursor = providerRetryCursorKey
+            ? this.checkpoint.providerRetryCursors?.[providerRetryCursorKey]
+            : undefined;
           const executionRequestNonce = crypto.randomUUID();
-          // This identifies one logical tool call to billing. Physical retry
-          // count remains in request metadata, so a transport retry cannot
-          // reserve or charge the same call twice.
-          const deeplineRequestId = `ctx-tool-${stableDigest(executionRequestNonce).slice(0, 32)}`;
+          // Transport ambiguity must retain the billing identity. An ordinary
+          // (unfenced) call advances it only after a definite provider-owned
+          // 429 released that attempt's reservation. Fenced calls retain it:
+          // the gateway-qualified retry owns their distinct charge lifecycle.
+          let deeplineRequestId = `ctx-tool-${stableDigest(executionRequestNonce).slice(0, 32)}`;
           // The provider operation has a semantic identity independent of
           // the response receipt. Its key is supplied by the caller that owns
           // that compatibility contract; never substitute request/billing
@@ -18758,7 +18917,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           // *durable* provider attempt. The execution gateway validates and
           // attests it before a retried call can use its fresh charge
           // lifecycle key; it is not a general retry counter.
-          let providerRateLimitRetryFromInvocationAttempt: number | null = null;
+          let providerRateLimitRetryFromInvocationAttempt: number | null =
+            savedProviderRetryCursor?.retryFromInvocationAttempt ?? null;
           const serializedRequestBody = (invocationAttempt: number) =>
             JSON.stringify({
               payload: shouldResolveAsyncOperationInRuntime
@@ -18859,8 +19019,89 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 : {}),
             });
           let transportAttempt = 0;
-          let invocationAttempt = 0;
+          let invocationAttempt =
+            savedProviderRetryCursor?.invocationAttempt ?? 0;
+          clearProviderRetryCursor = () => {
+            if (
+              !providerRetryCursorKey ||
+              !this.checkpoint.providerRetryCursors
+            ) {
+              return;
+            }
+            delete this.checkpoint.providerRetryCursors[providerRetryCursorKey];
+            if (
+              Object.keys(this.checkpoint.providerRetryCursors).length === 0
+            ) {
+              delete this.checkpoint.providerRetryCursors;
+            }
+          };
+          const persistProviderRetryCursor = (retryFrom: number) => {
+            if (!providerRetryCursorKey) return;
+            const cursor = {
+              invocationAttempt: retryFrom + 1,
+              retryFromInvocationAttempt: retryFrom,
+            };
+            this.checkpoint.providerRetryCursors ??= {};
+            this.checkpoint.providerRetryCursors[providerRetryCursorKey] =
+              cursor;
+          };
           let physicalAttempt = 0;
+          // Once this logical call receives an HTTP 429, keep a small
+          // phase trace for its retry so a stalled retry can be located without
+          // logging request, receipt, organization, or provider identifiers.
+          let provider429RetryDiagnosticNextInvocationAttempt: number | null =
+            null;
+          let provider429RetryDiagnosticStartedAt: number | null = null;
+          const logProvider429RetryPhase = (phase: string) => {
+            if (
+              provider429RetryDiagnosticNextInvocationAttempt === null ||
+              provider429RetryDiagnosticStartedAt === null
+            )
+              return;
+            this.runtimeLog(
+              `[runtime.provider_429_retry_phase] ${JSON.stringify({
+                next_invocation_attempt:
+                  provider429RetryDiagnosticNextInvocationAttempt,
+                phase,
+                elapsed_ms: Math.round(
+                  performance.now() - provider429RetryDiagnosticStartedAt,
+                ),
+              })}`,
+            );
+          };
+          const executionSignal = this.#options.executionSignal;
+          const toolAbortSignal =
+            executionSignal && options?.abortSignal
+              ? AbortSignal.any([executionSignal, options.abortSignal])
+              : (executionSignal ?? options?.abortSignal);
+          // A short provider hold costs less than checkpointing, parking, and
+          // starting another sandbox. Once we wait locally, bound all elapsed
+          // residency for this logical call, including subsequent HTTP/pacer
+          // work. Repeated zero or expired holds must also yield durably.
+          let residentProviderWaitStartedAt: number | null = null;
+          const canWaitForProviderInSandbox = (waitMs: number) => {
+            const elapsedMs =
+              residentProviderWaitStartedAt === null
+                ? 0
+                : performance.now() - residentProviderWaitStartedAt;
+            return (
+              Math.max(1, waitMs) <=
+              Math.max(0, TOOL_PROVIDER_RESIDENT_WAIT_BUDGET_MS - elapsedMs)
+            );
+          };
+          const waitForToolRetry = async (
+            waitMs: number,
+            providerHold: boolean,
+          ) => {
+            await options?.parkProviderCall?.();
+            if (providerHold && residentProviderWaitStartedAt === null) {
+              residentProviderWaitStartedAt = performance.now();
+            }
+            await this.sleepWithCheckpointHeartbeat(
+              waitMs,
+              options?.abortSignal,
+            );
+          };
           const retryToolTransportFailure = async (input: {
             error: unknown;
             elapsedMs: number;
@@ -18946,6 +19187,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           let providerTelemetryKey: string | null = null;
           let providerRunInFlightAtDispatch: number | null = null;
           const acquireProviderPermit = async (signal?: AbortSignal) => {
+            logProvider429RetryPhase('provider_permit_wait_started');
             const permit = await this.resourceGovernor.acquireProviderPermit({
               toolId,
               signal,
@@ -18955,6 +19197,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             providerPacingGrant = permit.pacingGrant ?? null;
             providerGatewayDispatchProof = permit.gatewayDispatchProof ?? null;
             providerTelemetryKey = permit.provider;
+            logProvider429RetryPhase('provider_permit_acquired');
             return permit;
           };
 
@@ -18967,6 +19210,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           }
 
           while (true) {
+            toolAbortSignal?.throwIfAborted();
+            if (
+              providerNoProgressExpired({
+                startedAtMsByScope:
+                  this.checkpoint.providerNoProgressStartedAtMsByScope,
+                scope: providerNoProgressScope,
+                nowMs: Date.now(),
+              })
+            ) {
+              throw providerNoProgressError(Date.now());
+            }
             physicalAttempt += 1;
             providerConcurrencyGrant = null;
             providerPacingGrant = null;
@@ -19012,28 +19266,28 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             // not share the coordinator's TTL environment.
             const hasReceiptHeartbeat = Boolean(options?.heartbeatReceipt);
             const abortController =
-              timeoutMs || hasReceiptHeartbeat || options?.abortSignal
+              timeoutMs || hasReceiptHeartbeat || toolAbortSignal
                 ? new AbortController()
                 : null;
             const forwardAbortSignal = () => {
-              if (!options?.abortSignal?.aborted) return;
-              abortController?.abort(options.abortSignal.reason);
+              if (!toolAbortSignal?.aborted) return;
+              abortController?.abort(toolAbortSignal.reason);
             };
             const receiptHeartbeat = options?.heartbeatReceipt;
             let heartbeatFailure: unknown = null;
             let timeoutHandle: ReturnType<typeof setTimeout> | null = null;
             try {
-              if (options?.abortSignal?.aborted) {
+              if (toolAbortSignal?.aborted) {
                 forwardAbortSignal();
               } else {
-                options?.abortSignal?.addEventListener(
-                  'abort',
-                  forwardAbortSignal,
-                  { once: true },
-                );
+                toolAbortSignal?.addEventListener('abort', forwardAbortSignal, {
+                  once: true,
+                });
               }
               const ownershipStartedAt = Date.now();
+              logProvider429RetryPhase('receipt_ownership_check_started');
               await options?.beforeProviderCall?.();
+              logProvider429RetryPhase('receipt_ownership_check_completed');
               if (runtimeReceiptReadTraceEnabled) {
                 this.runtimeLog(
                   `[perf] tool call id=${toolId} phase=before_provider_ownership elapsed_ms=${Date.now() - ownershipStartedAt}`,
@@ -19083,6 +19337,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                   acquireProviderPermit: async ({ signal }) =>
                     await acquireProviderPermit(signal),
                   tryAcquireProviderPermit: () => {
+                    logProvider429RetryPhase(
+                      'provider_permit_acquire_attempted',
+                    );
                     const permit =
                       this.resourceGovernor.tryAcquireProviderPermit({
                         toolId,
@@ -19093,6 +19350,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                     providerGatewayDispatchProof =
                       permit.gatewayDispatchProof ?? null;
                     providerTelemetryKey = permit.provider;
+                    logProvider429RetryPhase('provider_permit_acquired');
                     return permit;
                   },
                   tryAcquireExecutionSlot: ({ providerResourceKey }) =>
@@ -19299,6 +19557,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                           providerCallStartedAt =
                             dispatchedProviderCallStartedAt;
                           fetchDispatched = true;
+                          logProvider429RetryPhase('http_dispatch_started');
                           activeProviderRequest =
                             this.beginProviderRequest(provider);
                           const dispatchedResponse = await fetch(url, {
@@ -19356,6 +19615,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                           // it has been read and verified below.
                           response = dispatchedResponse;
                           if (dispatchedResponse.status === 429) {
+                            provider429RetryDiagnosticStartedAt =
+                              performance.now();
+                            provider429RetryDiagnosticNextInvocationAttempt =
+                              invocationAttempt + 1;
+                            logProvider429RetryPhase(
+                              'http_429_response_received',
+                            );
                             providerRequestsInFlightAt429 =
                               activeProviderRequest?.inFlightCount() ?? null;
                             const observedAt = Date.now();
@@ -19365,8 +19631,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                             // the conservative bare-429 cooldown. Do not replace
                             // it with the generic one-second HTTP retry fallback.
                             providerRetryAtMs = retryAfter?.trim()
-                              ? observedAt +
-                                parseToolExecuteRetryAfterMs(
+                              ? parseToolExecuteRetryAtMs(
                                   retryAfter,
                                   observedAt,
                                 )
@@ -19495,16 +19760,34 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                           }
                           if (dispatchedResponse.status === 429) {
                             const bodyText = dispatchedResponseErrorText ?? '';
-                            const attempt = httpFailureAttempts.next({
+                            let attempt = httpFailureAttempts.next({
                               toolId,
                               status: dispatchedResponse.status,
                               bodyText,
                               transientHttpRetrySafe:
                                 retrySafeTransientHttp || fixtureReplaySafe,
                             });
-                            preclassifiedHttpFailure.current = {
+                            let failure = classifyToolExecuteHttpFailure({
+                              toolId,
+                              status: dispatchedResponse.status,
                               attempt,
-                              failure: classifyToolExecuteHttpFailure({
+                              bodyText,
+                              schemaVersion: toolErrorSchemaVersion,
+                              retryAfterHeader:
+                                dispatchedResponse.headers.get('retry-after'),
+                              transientHttpRetrySafe:
+                                retrySafeTransientHttp || fixtureReplaySafe,
+                              providerLatencyMs:
+                                Date.now() - dispatchedProviderCallStartedAt,
+                            });
+                            if (failure.backpressureDelayMs !== null) {
+                              // The durable invocation attempt survives a map
+                              // suspension; use its absolute 1-based count for
+                              // the provider retry budget instead of the
+                              // process-local HTTP tracker, which resets when
+                              // a new runner resumes this row.
+                              attempt = invocationAttempt + 1;
+                              failure = classifyToolExecuteHttpFailure({
                                 toolId,
                                 status: dispatchedResponse.status,
                                 attempt,
@@ -19516,7 +19799,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                                   retrySafeTransientHttp || fixtureReplaySafe,
                                 providerLatencyMs:
                                   Date.now() - dispatchedProviderCallStartedAt,
-                              }),
+                              });
+                            }
+                            preclassifiedHttpFailure.current = {
+                              attempt,
+                              failure,
                             };
                           }
                           activeProviderRequest?.release();
@@ -19649,14 +19936,21 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               }
               if (error instanceof ProviderExhaustedError) {
                 const retryAtMs = Date.parse(error.retryAt);
-                if (!this.#options.durableBoundaries) {
-                  await this.sleepWithCheckpointHeartbeat(
-                    Math.max(
-                      1_000,
-                      Number.isFinite(retryAtMs)
-                        ? Math.ceil(retryAtMs - Date.now())
-                        : 1_000,
-                    ),
+                observeProviderWait(
+                  Number.isFinite(retryAtMs) ? retryAtMs : Date.now() + 1_000,
+                );
+                const waitMs = Number.isFinite(retryAtMs)
+                  ? Math.max(0, Math.ceil(retryAtMs - Date.now()))
+                  : 1_000;
+                if (
+                  !this.#options.durableBoundaries ||
+                  canWaitForProviderInSandbox(waitMs)
+                ) {
+                  await waitForToolRetry(
+                    this.#options.durableBoundaries
+                      ? waitMs
+                      : Math.max(1_000, waitMs),
+                    true,
                   );
                   continue;
                 }
@@ -19812,10 +20106,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               if (timeoutHandle) {
                 clearTimeout(timeoutHandle);
               }
-              options?.abortSignal?.removeEventListener(
-                'abort',
-                forwardAbortSignal,
-              );
+              toolAbortSignal?.removeEventListener('abort', forwardAbortSignal);
             }
 
             if (!response) {
@@ -19950,26 +20241,47 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 );
                 providerBackpressureReported = true;
               }
-              // The normal 429 path reports its hold before releasing the
-              // provider lease. A mapped row must now yield even for a short
-              // hold: sleeping here would occupy its row slot and prevent
-              // unrelated rows from making progress.
+              // Provider permits are already released. Brief holds can use
+              // the existing retry loop; longer or accumulated holds yield
+              // the row slot and resume from durable completed receipts.
               const durableProviderWaitMs =
                 failure.providerPacingWaitMs ?? failure.backpressureDelayMs;
+              const providerHoldRetryAtMs =
+                durableProviderWaitMs === null
+                  ? null
+                  : ((failure.providerPacingWaitMs !== null
+                      ? Date.now() + failure.providerPacingWaitMs
+                      : providerRetryAtMs) ??
+                    Date.now() + durableProviderWaitMs);
+              const remainingProviderWaitMs =
+                providerHoldRetryAtMs === null
+                  ? null
+                  : Math.max(0, Math.ceil(providerHoldRetryAtMs - Date.now()));
               if (
+                failure.backpressureDelayMs !== null ||
+                failure.providerPacingWaitMs !== null
+              ) {
+                observeProviderWait(
+                  providerHoldRetryAtMs ?? Date.now() + 1_000,
+                );
+              }
+              if (
+                failure.shouldRetry &&
                 this.#options.durableBoundaries &&
-                durableProviderWaitMs !== null &&
-                durableProviderWaitMs > 0
+                remainingProviderWaitMs !== null &&
+                !canWaitForProviderInSandbox(remainingProviderWaitMs)
               ) {
-                const retryAtMs =
-                  (failure.providerPacingWaitMs !== null
-                    ? Date.now() + failure.providerPacingWaitMs
-                    : providerRetryAtMs) ?? Date.now() + durableProviderWaitMs;
+                if (failure.backpressureDelayMs !== null) {
+                  persistProviderRetryCursor(invocationAttempt);
+                }
+                const retryAtMs = providerHoldRetryAtMs!;
                 const rowScope = rowContext.getStore();
                 if (
                   this.#options.durableBoundaries &&
                   (options?.providerBackpressureMapRows || rowScope)
                 ) {
+                  preserveProviderRetryCursorForSuspension =
+                    failure.backpressureDelayMs !== null;
                   throw new PlayRowProviderBackpressureError(
                     provider,
                     retryAtMs,
@@ -20018,6 +20330,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                   !shedKnownNotDispatched
                 ) {
                   const previousInvocationAttempt = invocationAttempt;
+                  if (failure.backpressureDelayMs !== null) {
+                    persistProviderRetryCursor(previousInvocationAttempt);
+                  }
                   invocationAttempt += 1;
                   providerRateLimitRetryFromInvocationAttempt =
                     failure.backpressureDelayMs !== null
@@ -20027,6 +20342,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 if (failure.chargeRetryBudget) {
                   await this.governor.chargeBudget('retry');
                 }
+                const retryDelayMs =
+                  remainingProviderWaitMs ?? failure.retryDelayMs;
                 const retryAttributePrefix =
                   failure.reason === 'provider_pacing_wait'
                     ? 'provider_pacing_wait'
@@ -20035,14 +20352,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                       : 'transient_http';
                 span.setAttribute(
                   `plays.${retryAttributePrefix}_retry_after_ms`,
-                  failure.retryDelayMs,
+                  retryDelayMs,
                 );
                 span.setAttribute(
                   `plays.${retryAttributePrefix}_attempt`,
                   httpFailureAttempt,
                 );
                 this.runtimeLog(
-                  `Tool ${toolId} returned ${response.status} (${failure.reason}); retrying after ${failure.retryDelayMs}ms ` +
+                  `Tool ${toolId} returned ${response.status} (${failure.reason}); retrying after ${retryDelayMs}ms ` +
                     `(request_id=${deeplineRequestId ?? 'none'} gateway=${transportGatewayOriginForDiagnostic(url)})`,
                   { level: 'warn' },
                 );
@@ -20064,16 +20381,26 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                         ? 'rate_limit'
                         : 'provider_error',
                       retryAt:
-                        failure.providerPacingWaitMs !== null
-                          ? Date.now() + failure.providerPacingWaitMs
-                          : Date.now() + failure.retryDelayMs,
+                        providerHoldRetryAtMs ?? Date.now() + retryDelayMs,
                       attempt: httpFailureAttempt,
                     },
                     observedAt: Date.now(),
                   },
                 });
-                await options?.parkProviderCall?.();
-                await this.sleepWithCheckpointHeartbeat(failure.retryDelayMs);
+                await waitForToolRetry(
+                  retryDelayMs,
+                  remainingProviderWaitMs !== null,
+                );
+                logProvider429RetryPhase('retry_sleep_completed');
+                if (
+                  !usesDurableInvocationFence &&
+                  failure.backpressureDelayMs !== null
+                ) {
+                  // A classified provider rejection has no completed value to
+                  // charge. The next physical attempt needs a new reservation;
+                  // durable result and provider-operation keys stay unchanged.
+                  deeplineRequestId = `ctx-tool-${stableDigest(`${executionRequestNonce}:provider-rate-limit-retry:${invocationAttempt}`).slice(0, 32)}`;
+                }
                 this.emitExecutionEvent({
                   type: 'activity.observed',
                   observation: {
@@ -20141,13 +20468,30 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                     : typeof resolved.result,
             });
             toolCallSucceeded = true;
+            if (this.checkpoint.providerNoProgressStartedAtMsByScope) {
+              delete this.checkpoint.providerNoProgressStartedAtMsByScope[
+                providerNoProgressScope
+              ];
+            }
             return receiptOutputRef
               ? { ...resolved, receiptOutputRef }
               : resolved;
           }
         },
       );
+    } catch (error) {
+      // The checkpoint becomes durable with the scheduler's suspension handoff.
+      // Keep cursors only on that path; terminal failures and cancellation must
+      // not leave retry state behind for an unrelated future execution.
+      preserveProviderRetryCursorForSuspension =
+        preserveProviderRetryCursorForSuspension ||
+        error instanceof PlayRowProviderBackpressureError ||
+        isPlayExecutionSuspendedError(error);
+      throw error;
     } finally {
+      if (!preserveProviderRetryCursorForSuspension) {
+        clearProviderRetryCursor();
+      }
       toolInputSnapshotMemory.current?.release();
       if (retryActivityEmitted) {
         this.emitExecutionEvent({
@@ -20207,21 +20551,41 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     );
   }
 
-  private async sleepWithCheckpointHeartbeat(ms: number): Promise<void> {
+  private async sleepWithCheckpointHeartbeat(
+    ms: number,
+    callerSignal?: AbortSignal,
+  ): Promise<void> {
     const waitMs = Math.max(1, Math.ceil(ms));
-    const startedAt = Date.now();
-
-    while (Date.now() - startedAt < waitMs) {
+    const startedAt = performance.now();
+    const executionSignal = this.#options.executionSignal;
+    const signal =
+      executionSignal && callerSignal
+        ? AbortSignal.any([executionSignal, callerSignal])
+        : (executionSignal ?? callerSignal);
+
+    signal?.throwIfAborted();
+    while (performance.now() - startedAt < waitMs) {
       this.#options.onBatchComplete?.(this.checkpoint);
-      const remainingMs = waitMs - (Date.now() - startedAt);
-      await new Promise((resolve) =>
-        setTimeout(
-          resolve,
+      signal?.throwIfAborted();
+      const remainingMs = waitMs - (performance.now() - startedAt);
+      await new Promise<void>((resolve, reject) => {
+        const onAbort = () => {
+          clearTimeout(timer);
+          signal?.removeEventListener('abort', onAbort);
+          reject(signal?.reason ?? new DOMException('Aborted', 'AbortError'));
+        };
+        const timer = setTimeout(
+          () => {
+            signal?.removeEventListener('abort', onAbort);
+            resolve();
+          },
           Math.min(TOOL_RETRY_HEARTBEAT_INTERVAL_MS, Math.max(1, remainingMs)),
-        ),
-      );
+        );
+        signal?.addEventListener('abort', onAbort, { once: true });
+      });
     }
 
+    signal?.throwIfAborted();
     this.#options.onBatchComplete?.(this.checkpoint);
   }
 }