deepline 0.3.87 → 0.3.89

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -109,7 +109,11 @@ import {
   type ToolResponseContract,
   type ToolResponseView,
 } from '../plays/tool-response-contract';
-import { ProviderExhaustedError } from './run-failure';
+import {
+  isProviderExhaustedFailure,
+  ProviderExhaustedError,
+} from './run-failure';
+import { PROVIDER_EXHAUSTED_MAX_WAIT_MS } from './governor/app-runtime-rate-state-backend';
 import {
   buildPlayContractCompatibility,
   normalizePlayContractCompatibility,
@@ -353,7 +357,11 @@ import type {
   PlayRowReadOrder,
   PlayRowMeta,
   PlaySheetCellProducer,
+  PlayAsyncCallOptions,
 } from './ctx-types';
+import { assertKnownResolvedBoundary } from './checkpoint-boundaries';
+import type { AsyncPlayRunRef } from '../plays/authoring-contract';
+import { admitAsyncPlayLineage } from './async-play-invocation';
 import {
   MAX_CELL_READ_REFS,
   MAX_ROW_READ_CELLS,
@@ -1982,6 +1990,19 @@ function normalizeStepDescription(
   return trimmed ? trimmed : undefined;
 }
 
+function resolvePlayReferenceName(playRef: unknown): string {
+  if (typeof playRef === 'string') return playRef;
+  if (
+    playRef === null ||
+    (typeof playRef !== 'object' && typeof playRef !== 'function')
+  ) {
+    return '';
+  }
+  const reference = playRef as { playName?: unknown; name?: unknown };
+  if (typeof reference.playName === 'string') return reference.playName;
+  return typeof reference.name === 'string' ? reference.name : '';
+}
+
 function emptyCheckpoint(): PlayCheckpoint {
   return {
     completedBatches: {},
@@ -2103,6 +2124,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
   >();
   private readonly ctxFetchHttpFailureDiagnosticIdentities = new Set<string>();
   private checkpoint: PlayCheckpoint;
+  /**
+   * A durable boundary is a control-flow handoff, not a customer-catchable
+   * business failure. Keep it on the context as the source of truth so a
+   * broad catch cannot turn the invocation into a normal completed result.
+   */
+  private pendingDurableSuspension: PlayExecutionSuspension | null = null;
   private readonly durableCallCacheEpochMs: number;
   /**
    * Durable tool receipts are the replay/cache authority for the execution
@@ -2215,6 +2242,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       timeoutMs?: number;
       receiptWaitMs?: number;
     }): Promise<TOutput> => {
+      this.assertNoPendingDurableSuspension();
       if (!request || typeof request !== 'object' || Array.isArray(request)) {
         throw new Error(
           'ctx.tools.execute requires a request object: ctx.tools.execute({ id, tool, input, description }).',
@@ -2303,6 +2331,28 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     }
   }
 
+  /**
+   * Called at ctx effect boundaries and by the runner before it can settle a
+   * normal result. This makes suspension sticky even when authored code uses a
+   * broad catch around `await ctx.sleep(...)`.
+   */
+  assertNoPendingDurableSuspension(): void {
+    if (this.pendingDurableSuspension) {
+      throw new PlayExecutionSuspendedError(this.pendingDurableSuspension);
+    }
+  }
+
+  /**
+   * The single seam that hands this invocation back to the scheduler. The
+   * first suspension wins: a later boundary reached by code that caught the
+   * first one cannot replace the handoff the scheduler already owes a resume.
+   */
+  private suspendDurably(suspension: PlayExecutionSuspension): never {
+    this.pendingDurableSuspension ??= suspension;
+    this.#options.onBatchComplete?.(this.checkpoint);
+    throw new PlayExecutionSuspendedError(this.pendingDurableSuspension);
+  }
+
   async tool<TOutput = unknown>(
     key: string,
     toolId: string,
@@ -2475,11 +2525,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       options.playId ||
       options.workflowId ||
       'anonymous-play';
-    const rootRunId = options.runId ?? options.workflowId ?? 'anonymous-run';
+    const currentRunId = options.runId ?? options.workflowId ?? 'anonymous-run';
+    const rootRunId = options.rootRunId ?? currentRunId;
     this.executionScope =
       options.executionScope ??
       createRootRunExecutionScope({
-        runId: rootRunId,
+        runId: currentRunId,
         playId: rootPlayId,
         attempt:
           typeof options.runAttempt === 'number' &&
@@ -2489,7 +2540,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         receiptNamespace: options.runtimeReceiptScope ?? rootPlayId,
         authority: {
           orgId: options.orgId ?? 'org',
-          workflowId: options.workflowId ?? rootRunId,
+          workflowId: options.workflowId ?? currentRunId,
           integrationMode: options.integrationMode ?? null,
           capabilities: DEFAULT_RUNTIME_EXECUTION_CAPABILITIES,
         },
@@ -2505,7 +2556,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         adapter: 'cjs_node20',
         scope: {
           orgId: options.orgId ?? 'org',
-          rootRunId: options.runId ?? options.workflowId ?? 'run',
+          rootRunId,
         },
         rateState: options.rateState ?? new InMemoryRateStateBackend(),
         budgetState: options.budgetState,
@@ -2710,6 +2761,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
   private assertInlineChildContract(
     reason: 'dataset_child' | 'suspending_child',
   ): void {
+    this.assertNoPendingDurableSuspension();
     const composition = this.activeInlineComposition;
     if (!composition) return;
     throw new Error(ctxRunPlayInlineOnlyMessage(composition.playName, reason));
@@ -4429,6 +4481,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       }) => Promise<T>;
     },
   ): Promise<T> {
+    this.assertNoPendingDurableSuspension();
     let releaseExternalCallSlot: (() => void) | null = null;
     const execute = async (leaseId: string | null): Promise<T> =>
       await opts.execute({
@@ -4471,7 +4524,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         cacheEpochMs: this.durableCallCacheEpochMs,
       });
     try {
-      return await executeWithDurableRuntimeReceipt<T>({
+      const output = await executeWithDurableRuntimeReceipt<T>({
         operation,
         id,
         runId: this.currentReceiptOwnerRunId,
@@ -4499,6 +4552,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         log: (message) => this.runtimeLog(message),
         execute: ({ leaseId }) => execute(leaseId),
       });
+      // Every receipt publication and cache return crosses this one success
+      // seam. A customer catch cannot turn a pending durable handoff into a
+      // reusable completed value.
+      this.assertNoPendingDurableSuspension();
+      return output;
     } finally {
       const release = releaseExternalCallSlot as (() => void) | null;
       release?.();
@@ -4705,6 +4763,62 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     });
   }
 
+  /**
+   * A forced call bypasses the immutable Tool Result Receipt, so its physical
+   * gateway invocation must bypass that receipt's replay identity too. Keep
+   * the transport key deterministic for one logical call: a retry of that
+   * call still converges, while a later explicit force gets a fresh identity.
+   */
+  private toolTransportReceiptKey(input: {
+    toolId: string;
+    resultReceiptKey: string;
+    providerOperationKey: string;
+    force?: boolean;
+  }): string {
+    if (input.force !== true) return input.resultReceiptKey;
+    return `${buildDurableToolReceiptPrefix({
+      orgId: this.#options.orgId,
+      toolId: input.toolId,
+    })}${stableDigest(`forced-provider-invocation:${input.providerOperationKey}`)}`;
+  }
+
+  /**
+   * Ordinary operations do not carry provider-idempotency metadata, but the
+   * execution relay still needs a semantic signal that an explicit force is
+   * a new provider operation. Unsafe-write operations already carry their
+   * forced provider key directly, so only ordinary calls need this attested
+   * canonical-operation salt.
+   */
+  private providerForceRefreshIdentitiesForToolCall(input: {
+    operationKey: string;
+    force?: boolean;
+    requiresExecutionFence?: boolean;
+    logicalCallId?: string;
+  }): string[] {
+    if (input.force !== true || input.requiresExecutionFence === true) {
+      return [];
+    }
+    return [
+      this.providerIdempotencyKeyForToolCall({
+        operationKey: input.operationKey,
+        force: true,
+        logicalCallId: input.logicalCallId,
+      }),
+    ];
+  }
+
+  private providerForceRefreshOptionsForToolCall(input: {
+    operationKey: string;
+    force?: boolean;
+    requiresExecutionFence?: boolean;
+    logicalCallId?: string;
+  }): Pick<ToolExecutionApiOptions, 'providerOperationForceRefreshIdentities'> {
+    const identities = this.providerForceRefreshIdentitiesForToolCall(input);
+    return identities.length > 0
+      ? { providerOperationForceRefreshIdentities: identities }
+      : {};
+  }
+
   /**
    * Bounded per-cell producer trace (ADR 0018).
    *
@@ -6125,6 +6239,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     input: TInput,
     options?: { description?: string },
   ): Promise<TOutput> {
+    this.assertNoPendingDurableSuspension();
     if (!isRuntimeStepProgram(program)) {
       throw new Error('ctx.runSteps(program, input) requires steps().');
     }
@@ -6149,6 +6264,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     input: MapFieldDefinition<T, TColumns>,
     options?: RuntimeDatasetOptions<T>,
   ): Promise<PlayDataset<Record<string, unknown>>> {
+    this.assertNoPendingDurableSuspension();
     if (rowContext.getStore()) {
       throw new Error(
         'Nested ctx.dataset() is not supported. Flatten your columns into one dataset, or keep custom per-row logic inside a single column.',
@@ -8356,8 +8472,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       await incrementalPersistence?.checkpoint(checkpointUpdates);
       await incrementalPersistence?.flush();
       this.pendingRowEventBoundaries = [];
-      this.#options.onBatchComplete?.(this.checkpoint);
-      throw new PlayExecutionSuspendedError({
+      this.suspendDurably({
         kind: 'integration_event_batch',
         boundaries: uniqueBoundaries,
       });
@@ -8452,7 +8567,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           if (
             (!continueOnProviderUnavailable &&
               step.continueOnProviderUnavailable !== true) ||
-            (!(error instanceof ProviderExhaustedError) &&
+            (!isProviderExhaustedFailure(error) &&
               !isProviderUnavailable(error))
           ) {
             throw error;
@@ -8537,6 +8652,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       let value: unknown;
       try {
         value = await runStep();
+        // A step-program resolver has the same publication boundary as a
+        // normal mapped field. Its customer callback may catch `ctx.sleep()`;
+        // do not turn that parked invocation into a waterfall output/cell.
+        this.assertNoPendingDurableSuspension();
       } catch (error) {
         // A provider admission, transient upstream failure, or exhausted
         // managed provider account is a visible unavailable leg, not a row
@@ -8545,7 +8664,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         // author hand-writing error handling.
         if (
           !continueOnProviderUnavailable ||
-          (!(error instanceof ProviderExhaustedError) &&
+          (!isProviderExhaustedFailure(error) &&
             !isProviderUnavailable(error))
         ) {
           throw error;
@@ -8755,7 +8874,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
 
       const source = Function.prototype.toString.call(resolver);
       return (
-        !source.includes('.tools.execute(') && !source.includes('.runPlay(')
+        !source.includes('.tools.execute(') &&
+        !source.includes('.runPlay(') &&
+        !source.includes('.runPlayAsync(')
       );
     });
   }
@@ -9237,7 +9358,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       return resolver;
     }
 
-    return await resolver(row, this, fields, index, previousCell);
+    const value = await resolver(row, this, fields, index, previousCell);
+    // This is the common user-code return seam for every mapped field path
+    // (incremental, pure, and step-program adapters). Check before callers
+    // serialize a cell, emit a completion event, or advance a map frame.
+    this.assertNoPendingDurableSuspension();
+    return value;
   }
 
   private toOutputRow<T>(item: T): Record<string, unknown> {
@@ -9592,14 +9718,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           }
           const useDirectResultReceipts =
             cacheableToolResult && !bypassDirectResultCache;
+          const providerOperationKey = this.providerIdempotencyKeyForToolCall({
+            operationKey: providerIdempotencyKeyBase,
+            force: cacheableToolResult && toolCachePolicy.force,
+            logicalCallId,
+          });
           const physicalDirectKey = useDirectResultReceipts
-            ? directCacheKey
+            ? this.toolTransportReceiptKey({
+                toolId,
+                resultReceiptKey: directCacheKey,
+                providerOperationKey,
+                force: toolCachePolicy.force,
+              })
             : `${buildDurableToolReceiptPrefix({
                 orgId: this.#options.orgId ?? 'unknown-org',
                 toolId,
               })}${stableDigest(`${this.currentRunId}:${logicalCallId}:always-fresh`)}`;
           const resultReceiptKey = directResultReceiptKey;
-          const providerOperationKey = providerIdempotencyKeyBase;
           const toolRetryPolicy = await this.#options
             .getToolRetryPolicy?.(toolId, input)
             .catch(() => null);
@@ -9665,16 +9800,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                             ? resultReceiptKey
                             : physicalDirectKey,
                         providerIdempotencyKey:
-                          this.providerIdempotencyKeyForToolCall({
-                            operationKey:
-                              useDirectResultReceipts && operationKey
-                                ? providerIdempotencyKeyBase
-                                : physicalDirectKey,
-                            force: cacheableToolResult && toolCachePolicy.force,
-                            logicalCallId,
-                          }),
+                          useDirectResultReceipts && operationKey
+                            ? providerOperationKey
+                            : physicalDirectKey,
                       }
                     : {}),
+                  ...(useDirectResultReceipts && operationKey
+                    ? this.providerForceRefreshOptionsForToolCall({
+                        operationKey: providerIdempotencyKeyBase,
+                        force: toolCachePolicy.force,
+                        requiresExecutionFence:
+                          toolRetryPolicy?.requiresExecutionFence,
+                        logicalCallId,
+                      })
+                    : {}),
                   playNodeScope: buildPlayNodeScope({
                     toolId,
                     callKey: normalizedKey,
@@ -10019,6 +10158,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       workflowId: this.#options.workflowId,
       orgId: this.#options.orgId,
       executorToken: this.#options.executorToken,
+      slackHitlDefaultTimeoutMs:
+        this.currentRuntimeSemantics.slackHitl.defaultTimeoutMs,
     };
     const basePreparedBoundary = await handler.prepare({
       payload: input,
@@ -10039,8 +10180,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         }
       : basePreparedBoundary;
 
-    const existing =
-      this.checkpoint.resolvedBoundaries?.[preparedBoundary.boundaryId];
+    const existing = assertKnownResolvedBoundary(
+      preparedBoundary.boundaryId,
+      this.checkpoint.resolvedBoundaries?.[preparedBoundary.boundaryId],
+    );
     if (existing?.kind === 'integration_event' && 'output' in existing) {
       const output = recordOrNull(existing.output);
       if (output?.timed_out === true) {
@@ -10056,6 +10199,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             ...(existing.messageRef ? { messageRef: existing.messageRef } : {}),
           },
         });
+        if (handler.suspendAfterTimeoutMs !== undefined) {
+          this.runtimeLog(
+            `Integration event ${preparedBoundary.boundaryId}: timeout finalized externally; keeping Play parked`,
+          );
+          this.suspendDurably({
+            kind: 'integration_event',
+            boundaryId: preparedBoundary.boundaryId,
+            eventKey: preparedBoundary.eventKey,
+            timeoutMs: handler.suspendAfterTimeoutMs,
+            ...(existing.messageRef ? { messageRef: existing.messageRef } : {}),
+          });
+        }
       }
       this.runtimeLog(
         `Integration event ${preparedBoundary.boundaryId}: recovered response from checkpoint`,
@@ -10105,9 +10260,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         ...(boundary.messageRef ? { messageRef: boundary.messageRef } : {}),
       },
     };
-    this.#options.onBatchComplete?.(this.checkpoint);
-
     if (rowScope) {
+      this.#options.onBatchComplete?.(this.checkpoint);
       throw new PlayRowExecutionSuspendedError({
         boundaryId: boundary.boundaryId,
         eventKey: boundary.eventKey,
@@ -10116,7 +10270,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       });
     }
 
-    throw new PlayExecutionSuspendedError({
+    this.suspendDurably({
       kind: 'integration_event',
       boundaryId: boundary.boundaryId,
       eventKey: boundary.eventKey,
@@ -10131,6 +10285,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     input: Record<string, unknown>,
     options?: PlayCallOptions,
   ): Promise<TOutput> {
+    this.assertNoPendingDurableSuspension();
     if (
       arguments.length === 3 &&
       typeof playRef === 'object' &&
@@ -10140,19 +10295,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         'ctx.runPlay(...) signature is ctx.runPlay(key, playRef, input, { description }). Add a stable call key as the first argument.',
       );
     }
-    const resolvedName =
-      typeof playRef === 'string'
-        ? playRef
-        : playRef && typeof playRef.playName === 'string'
-          ? playRef.playName
-          : playRef && typeof playRef.name === 'string'
-            ? playRef.name
-            : '';
+    const resolvedName = resolvePlayReferenceName(playRef);
 
     if (!resolvedName.trim()) {
       throw new Error('ctx.runPlay(...) requires a resolvable play name.');
     }
-    validatePlayAuthoringField('ctx.runPlay.playRef', playRef);
+    // Imported Play dependencies are callable proxy functions at runtime. The
+    // authoring contract describes their durable identity, so validate the
+    // normalized name rather than requiring the ephemeral handle itself to be
+    // JSON-shaped.
+    validatePlayAuthoringField('ctx.runPlay.playRef', resolvedName);
     validatePlayAuthoringField('ctx.runPlay.input', input);
     assertNoSecretTaint(input, 'ctx.runPlay input');
     validatePlayAuthoringField('ctx.runPlay.key', key);
@@ -10264,6 +10416,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           },
           () => this.executeResolvedPlay(resolvedPlay, this, input),
         );
+        // Inline composition shares this Play Invocation. A child may catch a
+        // parent suspension, but it cannot publish child success while that
+        // handoff is still pending.
+        this.assertNoPendingDurableSuspension();
         this.inlineChildAggregates.ok += 1;
         if (rowStore) {
           this.emitScopedFieldMetaUpdate({
@@ -10313,6 +10469,81 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     }
   }
 
+  async runPlayAsync(
+    key: string,
+    playRef: string | { playName?: string; name?: string },
+    input: Record<string, unknown>,
+    options: PlayAsyncCallOptions,
+  ): Promise<AsyncPlayRunRef> {
+    const resolvedName = resolvePlayReferenceName(playRef);
+    if (!resolvedName.trim()) {
+      throw new Error(
+        'ctx.runPlayAsync(...) requires a resolvable Play name or definition handle.',
+      );
+    }
+    validatePlayAuthoringField('ctx.runPlayAsync.playRef', resolvedName);
+    validatePlayAuthoringField('ctx.runPlayAsync.input', input);
+    assertNoSecretTaint(input, 'ctx.runPlayAsync input');
+    validatePlayAuthoringField('ctx.runPlayAsync.key', key);
+    validatePlayAuthoringField(
+      'ctx.runPlayAsync.options.description',
+      options?.description,
+    );
+    const normalizedKey = this.normalizeContextKey(key, 'runPlayAsync');
+    if (!this.#options.launchPlayAsync) {
+      throw new Error(
+        'ctx.runPlayAsync(...) is unavailable because no async Play admission Adapter was configured.',
+      );
+    }
+    const ancestryPlayIds =
+      this.currentExecutionGovernor.snapshot().ancestryPlayIds;
+    admitAsyncPlayLineage({
+      ancestryPlayIds,
+      childPlayName: resolvedName,
+    });
+
+    const rowScope = rowContext.getStore();
+    const invocationKey = sha256Hex(
+      stableStringify({
+        parentRunId: this.currentExecutionScope.logical.runId,
+        callerScope: this.currentExecutionScope.receipt.namespace,
+        key: normalizedKey,
+        row: rowScope
+          ? {
+              tableNamespace: rowScope.tableNamespace ?? null,
+              rowKey: rowScope.rowKey ?? null,
+              rowId: rowScope.rowKey ? null : rowScope.rowId,
+              fieldName: rowScope.fieldName ?? null,
+            }
+          : null,
+      }),
+    );
+    const launched = await this.#options.launchPlayAsync({
+      key: normalizedKey,
+      invocationKey,
+      playRef: resolvedName,
+      input,
+      description: options.description,
+      schedulerSchema: this.#options.runtimeSchedulerSchema ?? null,
+      ancestryPlayIds,
+    });
+    const step = {
+      type: 'play_launch' as const,
+      playId: resolvedName,
+      runId: launched.runId,
+      description: normalizeStepDescription(options?.description),
+    };
+    if (this.activeDatasetStep) {
+      this.activeDatasetStep.substeps.push(step);
+    } else {
+      this.steps.push(step);
+    }
+    this.log(
+      `ctx.runPlayAsync(${normalizedKey}): admitted ${resolvedName} as ${launched.runId}`,
+    );
+    return launched;
+  }
+
   /**
    * Replay-stable receipt namespace for primitives executed by one inline
    * `ctx.runPlay` invocation. Format
@@ -10577,7 +10808,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         `sleep-${this.sleepBoundaryIndex}-${delayMs}`,
       );
       this.sleepBoundaryIndex += 1;
-      const existing = this.checkpoint.resolvedBoundaries?.[boundaryId];
+      const existing = assertKnownResolvedBoundary(
+        boundaryId,
+        this.checkpoint.resolvedBoundaries?.[boundaryId],
+      );
       if (existing?.kind === 'sleep' && existing.completedAt !== undefined) {
         return;
       }
@@ -10588,8 +10822,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           delayMs,
         },
       };
-      this.#options.onBatchComplete?.(this.checkpoint);
-      throw new PlayExecutionSuspendedError({
+      this.suspendDurably({
         kind: 'sleep',
         boundaryId,
         delayMs,
@@ -10759,7 +10992,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             )}`,
           );
 
-          const existing = this.checkpoint.resolvedBoundaries?.[boundaryId];
+          const existing = assertKnownResolvedBoundary(
+            boundaryId,
+            this.checkpoint.resolvedBoundaries?.[boundaryId],
+          );
           // A transient request deliberately has no durable recovery path, so
           // its response stays in the running Play rather than a checkpoint.
           if (
@@ -10951,6 +11187,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               options?.transient !== true &&
               !this.durableDirectToolResultsBackedByReceipts
             ) {
+              // Publishing a fetch boundary makes the result replay-visible.
+              // A caught durable suspension is still the invocation's outcome,
+              // so never checkpoint over a pending handoff.
+              this.assertNoPendingDurableSuspension();
               this.checkpoint.resolvedBoundaries = {
                 ...(this.checkpoint.resolvedBoundaries ?? {}),
                 [boundaryId]: {
@@ -10998,7 +11238,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       `step-${scope}:${normalizedKey}${boundarySuffix}`,
     );
     const executeStep = async (): Promise<T> => {
-      const existing = this.checkpoint.resolvedBoundaries?.[boundaryId];
+      const existing = assertKnownResolvedBoundary(
+        boundaryId,
+        this.checkpoint.resolvedBoundaries?.[boundaryId],
+      );
       if (existing?.kind === 'step' && 'output' in existing) {
         this.runtimeLog(
           `ctx.step(${normalizedKey}): recovered result from checkpoint`,
@@ -11008,6 +11251,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
 
       const output = await run();
       assertJsonSerializableStepOutput(normalizedKey, output);
+      // A caught durable suspension is still the invocation's outcome. Never
+      // publish an enclosing step fallback that would make a later resume skip
+      // the suspended work.
+      this.assertNoPendingDurableSuspension();
       this.checkpoint.resolvedBoundaries = {
         ...(this.checkpoint.resolvedBoundaries ?? {}),
         [boundaryId]: {
@@ -11566,19 +11813,38 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           }
           let releaseToolSlot: () => void = () => undefined;
           try {
+            const providerOperationKey = this.providerIdempotencyKeyForToolCall(
+              {
+                operationKey:
+                  request.providerIdempotencyKeyBase ?? request.receiptKey!,
+                force: request.force === true,
+                logicalCallId: request.logicalCallId,
+              },
+            );
             const result = await this.executeMappedToolCallThroughModule({
               request,
               dispatchToolId: toolId,
               dispatchPayload: request.input,
               resultReceiptKey: request.receiptKey!,
-              providerOperationKey:
-                request.providerIdempotencyKeyBase ?? request.receiptKey!,
+              providerOperationKey,
               resultReceipts: receipts,
               apiOptions: {
                 playNodeScope: playNodeScopeForToolCallRequest(request),
-                durableCallReceiptKey: request.receiptKey,
+                durableCallReceiptKey: this.toolTransportReceiptKey({
+                  toolId,
+                  resultReceiptKey: request.receiptKey!,
+                  providerOperationKey,
+                  force: request.force,
+                }),
                 completedResultCache: true,
                 executionAuthScopeDigest: request.executionAuthScopeDigest,
+                ...this.providerForceRefreshOptionsForToolCall({
+                  operationKey:
+                    request.providerIdempotencyKeyBase ?? request.receiptKey!,
+                  force: request.force,
+                  requiresExecutionFence: request.requiresExecutionFence,
+                  logicalCallId: request.logicalCallId,
+                }),
                 timeoutMs:
                   typeof request.timeoutMs === 'number' && request.timeoutMs > 0
                     ? request.timeoutMs
@@ -11705,6 +11971,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             batch.memberRequests.length;
           throw new RuntimePersistenceCircuitOpenError(this.persistenceLatch);
         }
+        const batchProviderActionVersion =
+          (
+            await this.#options.getToolActionCacheVersion?.(
+              batch.batchOperation,
+            )
+          )?.trim() ?? '';
         const batchRequestByCallId = new Map(
           batch.memberRequests.map((request) => [request.callId, request]),
         );
@@ -11714,6 +11986,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             members: ToolCallRequest[];
             compiled: CompiledRequestBatch<ToolCallRequest>;
             providerOperationKey: string;
+            forceRefreshIdentities: string[];
             receiptLeaseExpiresAt: string | null;
             timeoutMs: number | undefined;
           }
@@ -11752,7 +12025,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                     call.tool.id,
                     call.payload,
                     {
-                      durableCallReceiptKey: compatibility.resultReceiptKey,
+                      durableCallReceiptKey: this.toolTransportReceiptKey({
+                        toolId: call.tool.id,
+                        resultReceiptKey: compatibility.resultReceiptKey,
+                        providerOperationKey: detail.providerOperationKey,
+                        force: detail.members.some(
+                          (request) => request.force === true,
+                        ),
+                      }),
                       completedResultCache: true,
                       playNodeScope: playNodeScopeForToolCallRequest(
                         detail.members[0]!,
@@ -11760,6 +12040,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                       ),
                       executionAuthScopeDigest:
                         detail.members[0]?.executionAuthScopeDigest ?? null,
+                      ...(detail.forceRefreshIdentities.length > 0
+                        ? {
+                            providerOperationForceRefreshIdentities:
+                              detail.forceRefreshIdentities,
+                          }
+                        : {}),
                       receiptLeaseExpiresAt: detail.receiptLeaseExpiresAt,
                       timeoutMs: detail.timeoutMs,
                     },
@@ -11850,11 +12136,30 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 'Native batch owned members changed provider operation during compilation.',
               );
             }
-            const providerOperationKey = aggregateReceiptKey;
+            const forceRefreshIdentities = owned.flatMap((request) =>
+              this.providerForceRefreshIdentitiesForToolCall({
+                operationKey:
+                  this.providerOperationKeyForQueuedToolCall(request),
+                force: request.force,
+                requiresExecutionFence: request.requiresExecutionFence,
+                logicalCallId: request.logicalCallId,
+              }),
+            );
+            const providerOperationKey =
+              buildDurableToolPayloadProviderIdempotencyKey({
+                orgId: this.#options.orgId,
+                toolId: batch.batchOperation,
+                executionAuthScopeDigest:
+                  owned[0]?.executionAuthScopeDigest ?? null,
+                providerActionVersion: batchProviderActionVersion,
+                payload: compiled.batchPayload,
+                forceRefreshIdentities,
+              });
             dispatchDetails.set(aggregateReceiptKey, {
               members: owned,
               compiled,
               providerOperationKey,
+              forceRefreshIdentities,
               receiptLeaseExpiresAt: owned.reduce<string | null>(
                 (earliest, request) => {
                   const expiresAt = request.receiptLeaseExpiresAt ?? null;
@@ -11877,7 +12182,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             });
             return {
               ...members[0]!.call,
-              tool: { id: batch.batchOperation, providerActionVersion: '' },
+              tool: {
+                id: batch.batchOperation,
+                providerActionVersion: batchProviderActionVersion,
+              },
               payload: compiled.batchPayload,
               compatibility: {
                 resultReceiptKey: aggregateReceiptKey,
@@ -11964,6 +12272,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             for (const request of entry.request.memberRequests) {
               await reject(request, entry.error);
             }
+            // A provider-unavailable physical batch is a logical outcome for
+            // each member, not a map-fatal scheduler failure. Each rejected
+            // row resolver now receives the typed error above, where a
+            // waterfall may persist the failed leg and advance to its next
+            // provider. Keep infrastructure and contract failures loud.
+            if (
+              isProviderExhaustedFailure(entry.error) ||
+              isProviderUnavailable(entry.error)
+            ) {
+              continue;
+            }
             throw entry.error;
           }
           if (!entry.result) {
@@ -12618,6 +12937,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                             force: owner.force === true,
                             logicalCallId: owner.logicalCallId,
                           }),
+                        ...this.providerForceRefreshOptionsForToolCall({
+                          operationKey:
+                            this.providerOperationKeyForQueuedToolCall(owner),
+                          force: owner.force,
+                          requiresExecutionFence: owner.requiresExecutionFence,
+                          logicalCallId: owner.logicalCallId,
+                        }),
                         timeoutMs: resolveRuntimeTimeoutMsForClaimedOwners([
                           owner,
                         ]),
@@ -13232,6 +13558,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                                     force: request.force === true,
                                     logicalCallId: request.logicalCallId,
                                   }),
+                                ...this.providerForceRefreshOptionsForToolCall({
+                                  operationKey:
+                                    this.providerOperationKeyForQueuedToolCall(
+                                      request,
+                                    ),
+                                  force: request.force,
+                                  requiresExecutionFence:
+                                    request.requiresExecutionFence,
+                                  logicalCallId: request.logicalCallId,
+                                }),
                                 receiptLeaseExpiresAt:
                                   request.receiptLeaseExpiresAt,
                                 heartbeatReceipt: () =>
@@ -13635,13 +13971,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               `Tool ${toolId} requires a provider execution fence but no provider operation identity was supplied.`,
             );
           }
-          const canonicalOperationActionVersion = providerExecutionFenceKey
-            ? (
-                await this.#options.getToolActionCacheVersion?.(toolId)
-              )?.trim() || null
-            : null;
+          const canonicalOperationActionVersion =
+            providerExecutionFenceKey ||
+            options?.providerOperationForceRefreshIdentities?.length
+              ? (
+                  await this.#options.getToolActionCacheVersion?.(toolId)
+                )?.trim() || null
+              : null;
           const canonicalOperation =
-            providerExecutionFenceKey && canonicalOperationActionVersion
+            (providerExecutionFenceKey ||
+              options?.providerOperationForceRefreshIdentities?.length) &&
+            canonicalOperationActionVersion
               ? {
                   version: 1,
                   tool_id: toolId,
@@ -14242,6 +14582,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                   toolId,
                   failure.backpressureDelayMs,
                 );
+                // A Retry-After beyond the durable pacer's maximum wait has
+                // no useful in-process retry. It is already an exhausted
+                // provider outcome: surface the same typed error the next
+                // provider admission would produce, without sleeping before a
+                // waterfall can choose an independent fallback.
+                if (
+                  failure.backpressureDelayMs > PROVIDER_EXHAUSTED_MAX_WAIT_MS
+                ) {
+                  throw new ProviderExhaustedError({
+                    provider,
+                    retryAtMs: Date.now() + failure.backpressureDelayMs,
+                    cause: failure.error,
+                  });
+                }
               }
               if (
                 failure.shouldRetry &&