deepline 0.3.392 → 0.3.393

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -25,6 +25,10 @@ import { AsyncLocalStorage } from 'async_hooks';
 import { isDatasetPayloadField } from '../play-data-plane/sheet-contract';
 import {
   createDeferredPlayDataset,
+  copyPlayDatasetSourceRowIdentity,
+  getPlayDatasetSourceRowIdentity,
+  getPlayDatasetSourceRowIdentityAt,
+  setPlayDatasetSourceRowIdentity,
   deserializeLegacyPlayDataset,
   deserializePlayDatasetCell,
   isPlayDataset,
@@ -32,6 +36,7 @@ import {
   isSerializedPlayDatasetCell,
   iteratePlayDatasetInputPages,
   materializePlayDatasetInput,
+  residentPlayDatasetRows,
   resolveMaterializeLimitCap,
   serializePlayDatasetCell,
 } from '../plays/dataset';
@@ -69,7 +74,10 @@ import type {
   ProviderPacingGrant,
 } from './governor/rate-state-backend';
 import type { MapRowOutcome } from './durability-store';
-import { stringifyPostgresJson } from './postgres-json';
+import {
+  sanitizePostgresJsonValue,
+  stringifyPostgresJson,
+} from './postgres-json';
 import { providerCapacityLabel } from './activity-observation';
 import { RUNTIME_ROW_OUTPUT_NOT_JSON_SERIALIZABLE } from './runtime-sheet-row-transition';
 import {
@@ -91,7 +99,10 @@ import {
 import {
   DURABLE_INVOCATION_FENCE_VERSION,
   durableInvocationFenceExecutePath,
+  buildRuntimeExecutionRecoveryKey,
+  RUNTIME_EXECUTION_RECOVERY_SUPPORTED_HEADER,
 } from './durable-invocation-fence';
+import { isImmutableRuntimeExecutionRecoveryOrigin } from './scheduler-backends/runtime-api-origin';
 import {
   isRetryableSecretResolutionStatus,
   secretResolutionRetryDecision,
@@ -304,9 +315,11 @@ import {
   classifyAsyncOperationResult,
   executeAsyncOperationLifecycle,
   extractAsyncOperationJobId,
+  formatAsyncOperationTimeout,
   readAsyncOperationPath,
   asyncOperationShouldWait,
   hasExplicitAsyncOperationTerminalMatch,
+  resolveAsyncOperationPolling,
   type AsyncOperationContractWire,
 } from './async-operation';
 import {
@@ -8206,6 +8219,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         options.onRowError,
       );
       validateOptionalPlayAuthoringField('ctx.dataset.run.mode', options.mode);
+      validateOptionalPlayAuthoringField(
+        'ctx.dataset.run.rowIdentity',
+        options.rowIdentity,
+      );
       validateOptionalPlayAuthoringField(
         'ctx.dataset.run.undrawnColumns',
         options.undrawnColumns,
@@ -8260,11 +8277,53 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         ),
       );
 
+    const residentSourceRows = isPlayDataset(items)
+      ? residentPlayDatasetRows(items)
+      : Array.isArray(items)
+        ? items
+        : null;
+    const csvSource =
+      (isPlayDataset(items) && items.datasetKind === 'csv') ||
+      residentSourceRows?.some(
+        (_row, index) =>
+          getPlayDatasetSourceRowIdentityAt(residentSourceRows, index) !==
+          undefined,
+      ) === true;
+    const csvOccurrenceIdentityRequired =
+      csvSource &&
+      this.currentRuntimeSemantics.datasets.csvRowIdentity === 'source_row';
+    const rowIdentityMode = csvOccurrenceIdentityRequired
+      ? 'source_row'
+      : (options?.rowIdentity ??
+        (options?.key !== undefined
+          ? 'key'
+          : csvSource
+            ? this.currentRuntimeSemantics.datasets.csvRowIdentity
+            : 'key'));
+    const useSourceRowIdentity = rowIdentityMode === 'source_row';
     const userKeyOption = options?.key;
     let explicitKeyResolver:
       | ((row: Record<string, unknown>, index: number) => string)
       | null = null;
-    if (userKeyOption !== undefined) {
+    if (useSourceRowIdentity) {
+      explicitKeyResolver = (row, index) => {
+        const source = getPlayDatasetSourceRowIdentity(row);
+        const inputRow = Object.fromEntries(
+          Object.entries(toSerializableCsvAliasedRow(row)).filter(
+            ([field]) => !field.startsWith('__deepline'),
+          ),
+        );
+        // Position distinguishes equal records; the input digest prevents a
+        // changed file at the same path from replaying an earlier row's output.
+        return JSON.stringify([
+          'source-row-v1',
+          source?.datasetId ??
+            (isPlayDataset(items) ? items.datasetId : normalizedMapNamespace),
+          source?.rowIndex ?? index,
+          stableDigest(stableStringify(inputRow)),
+        ]);
+      };
+    } else if (userKeyOption !== undefined) {
       explicitKeyResolver = (row, index) => {
         const inputRow = stripFieldOutputs(row);
         const raw =
@@ -8469,6 +8528,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       let failedCount = 0;
       let duplicateReuseCount = 0;
       let executedCount = 0;
+      let sourceRowsCount = 0;
       const failureSamples: Array<{ rowLabel: string; error: string }> = [];
       const failureSampleErrors = new Set<string>();
       const previewRows: Record<string, unknown>[] = [];
@@ -8572,9 +8632,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         offset: number;
       }): Promise<PreparedPage> => {
         this.assertRowAdmissionOpen();
-        const pageRawItems = page.rows.map((item) =>
-          this.toOutputRow(item as Record<string, unknown>),
-        );
+        sourceRowsCount += page.rows.length;
+        const pageRawItems = page.rows.map((item, index) => {
+          const row = this.toOutputRow(item as Record<string, unknown>);
+          const sourceIdentity = getPlayDatasetSourceRowIdentityAt(
+            page.rows,
+            index,
+          );
+          return sourceIdentity
+            ? setPlayDatasetSourceRowIdentity(row, sourceIdentity)
+            : row;
+        });
         const pageMemoryEstimate = estimateRuntimeMapRowsMemory(pageRawItems);
         if (pageMemoryEstimate.estimatedResidentBytes > pageBudgetBytes) {
           throw new RuntimeMapMemoryLimitError({
@@ -8588,6 +8656,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           const originalIndex = page.offset + index;
           return {
             ...toSerializableCsvAliasedRow(row),
+            ...(getPlayDatasetSourceRowIdentity(row)
+              ? { __deeplineSourceRow: getPlayDatasetSourceRowIdentity(row) }
+              : {}),
             ...mapRowOutcomeRuntimeFields({
               key: seedRowIdentity(row, originalIndex),
               inputIndex: originalIndex,
@@ -8630,9 +8701,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           completedRowsByKey.set(rowKey, row);
         }
 
-        const seededItems = page.rows.map((item, index) => {
+        const seededItems = pageRawItems.map((row, index) => {
           const originalIndex = page.offset + index;
-          const row = this.toOutputRow(item as Record<string, unknown>);
           const rowKey = seedRowIdentity(row, originalIndex);
           const persisted =
             pendingRowsByKey.get(rowKey) ?? completedRowsByKey.get(rowKey);
@@ -8661,7 +8731,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           { row: Record<string, unknown>; originalIndex: number }
         >();
         for (const { row, index } of admittedSeededItems) {
-          if (explicitKeyResolver) {
+          if (explicitKeyResolver && !useSourceRowIdentity) {
             const explicitKey = explicitKeyResolver(row, index);
             if (seenExplicitKeys.has(explicitKey)) {
               duplicateReuseCount += 1;
@@ -8990,12 +9060,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       );
       successfulCount = visibleResults.completedRows;
       failedCount = visibleResults.failedRows;
-      this.#options.onMapRowsSettled?.({
+      await this.#options.onMapRowsSettled?.({
         playName: this.#options.playName,
         runId: this.#options.runId,
         tableNamespace: resolvedTableNamespace,
         succeededRows: successfulCount,
         failedRows: failedCount,
+        expectedRows: totalInputCount - duplicateReuseCount,
+        staticPipeline: this.currentStaticPipeline,
       });
 
       // Isolated failures are a partial result only when at least one admitted
@@ -9181,6 +9253,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           skipped: duplicateReuseCount,
           pending: 0,
           failed: failedCount,
+          cardinality: {
+            sourceRows: sourceRowsCount,
+            uniqueInputRows: processedCount - duplicateReuseCount,
+            duplicateOccurrences: duplicateReuseCount,
+            outputRows: successfulCount,
+            rowIdentity: rowIdentityMode,
+          },
           ...(duplicateReuseCount > 0
             ? { duplicates: { exact: duplicateReuseCount } }
             : {}),
@@ -9228,24 +9307,66 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         },
       });
     }
-    const rawMaterializedEntries =
+    const rawMaterializedEntries: {
+      item: T;
+      inputIndex: number;
+      sourceIdentity?: ReturnType<typeof getPlayDatasetSourceRowIdentityAt>;
+    }[] =
       resumedPendingRowsOnly && suspendedInputRows
-        ? suspendedInputRows.map((row) => ({
-            item: cloneCsvAliasedRow(row.data) as T,
-            inputIndex: row.inputIndex,
-          }))
-        : (
-            await materializePlayDatasetInput(items, {
+        ? suspendedInputRows.map((row) => {
+            const retainedKey = resolveMapRowOutcomeKey(row.data);
+            return {
+              // Older checkpoints contain only execution data. New checkpoints
+              // also retain the original admission input before sheet outputs
+              // were restored; keep the already-admitted key across either path.
+              item: cloneCsvAliasedRow(
+                row.authoredInput?.data ?? row.data,
+                retainedKey
+                  ? mapRowOutcomeRuntimeFields({ key: retainedKey })
+                  : {},
+              ) as T,
+              inputIndex: row.inputIndex,
+              sourceIdentity: row.authoredInput?.source,
+            };
+          })
+        : await (async () => {
+            const materialized = await materializePlayDatasetInput(items, {
               onRow: (row) => {
                 materializationMemoryTracker.track(row);
               },
-            })
-          ).map((item, inputIndex) => ({ item, inputIndex }));
+            });
+            return materialized.map((item, inputIndex) => ({
+              item,
+              inputIndex,
+              sourceIdentity: getPlayDatasetSourceRowIdentityAt(
+                materialized,
+                inputIndex,
+              ),
+            }));
+          })();
     const rawMaterializedItems = rawMaterializedEntries.map(
-      (entry) => entry.item,
+      // Occurrence identity also applies to an explicitly selected array whose
+      // author reused the same object reference. Give each occurrence its own
+      // execution object so the original-index lookup cannot alias the last row.
+      (entry) => {
+        const sourceIdentity =
+          'sourceIdentity' in entry
+            ? entry.sourceIdentity
+            : getPlayDatasetSourceRowIdentity(entry.item);
+        if (!useSourceRowIdentity && !sourceIdentity) return entry.item;
+        const row = this.toOutputRow(entry.item);
+        return (
+          sourceIdentity
+            ? setPlayDatasetSourceRowIdentity(row, sourceIdentity)
+            : row
+        ) as T;
+      },
     );
     const originalInputIndexByItem = new Map<T, number>(
-      rawMaterializedEntries.map(({ item, inputIndex }) => [item, inputIndex]),
+      rawMaterializedItems.map((item, index) => [
+        item,
+        rawMaterializedEntries[index]!.inputIndex,
+      ]),
     );
     // Silent dedupe of duplicate explicit map keys: keep the first row per
     // canonical key and drop subsequent duplicates. Deduping the materialized
@@ -9253,24 +9374,75 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     // counts, sheet writes, resumable keys) consistent — the resolver derives
     // keys from row content, so dropped rows are excluded everywhere with no
     // index mismatch. Replaces the prior throw-on-duplicate behavior.
-    const dedupedMaterialized = explicitKeyResolver
-      ? dedupeExplicitMapKeyRows({
-          rows: rawMaterializedItems,
-          resolver: (item, index) =>
-            explicitKeyResolver(
-              this.toOutputRow(item as Record<string, unknown>),
-              originalInputIndexByItem.get(item) ?? index,
-            ),
-        })
-      : { rows: rawMaterializedItems, droppedCount: 0, duplicateKeys: [] };
+    const dedupedMaterialized =
+      explicitKeyResolver && !useSourceRowIdentity
+        ? dedupeExplicitMapKeyRows({
+            rows: rawMaterializedItems,
+            resolver: (item, index) =>
+              explicitKeyResolver(
+                this.toOutputRow(item as Record<string, unknown>),
+                originalInputIndexByItem.get(item) ?? index,
+              ),
+          })
+        : { rows: rawMaterializedItems, droppedCount: 0, duplicateKeys: [] };
     if (dedupedMaterialized.droppedCount > 0) {
-      const keySample = dedupedMaterialized.duplicateKeys.join(', ');
       this.runtimeLog(
-        `deduped ${dedupedMaterialized.droppedCount} duplicate dataset key(s) for ctx.dataset("${normalizedMapNamespace}"); keeping first occurrence` +
-          (keySample ? ` (e.g. ${keySample})` : ''),
+        `[runtime.dataset_input_cardinality] ${JSON.stringify({
+          dataset: normalizedMapNamespace,
+          rowIdentity: rowIdentityMode,
+          sourceRows: rawMaterializedEntries.length,
+          uniqueInputRows: dedupedMaterialized.rows.length,
+          duplicateOccurrences: dedupedMaterialized.droppedCount,
+          duplicatePolicy: 'first_wins',
+        })}`,
       );
     }
     const materializedItems = dedupedMaterialized.rows;
+    // A legacy checkpoint does not contain its pre-dedupe source cardinality.
+    // Leave the optional summary absent rather than report its pending subset
+    // as the original source. New checkpoints retain only fixed-size counts.
+    const inputCardinality = resumedPendingRowsOnly
+      ? replayMapFrame?.inputCardinality
+      : {
+          sourceRows: rawMaterializedEntries.length,
+          uniqueInputRows: materializedItems.length,
+          duplicateOccurrences: dedupedMaterialized.droppedCount,
+          rowIdentity: rowIdentityMode,
+        };
+    // Admission provenance is captured before onMapStart restores output
+    // cells. The execution row stays intact; only pending inputs enter a
+    // checkpoint, and their sheet outputs are restored by the existing owner
+    // after the next admission. Never infer original input from enriched data.
+    const retainedAuthoredInputsByIndex = new Map(
+      (suspendedInputRows ?? []).flatMap((row) =>
+        row.authoredInput ? [[row.inputIndex, row.authoredInput] as const] : [],
+      ),
+    );
+    const authoredInputsByIndex = new Map(
+      canReadPersistedMapResults
+        ? materializedItems.flatMap((item, index) => {
+            const inputIndex = originalInputIndexByItem.get(item) ?? index;
+            if (resumedPendingRowsOnly) {
+              const retained = retainedAuthoredInputsByIndex.get(inputIndex);
+              // An older pending checkpoint has execution data only. Keep its
+              // optional provenance absent rather than relabel enriched cells.
+              return retained ? [[inputIndex, retained] as const] : [];
+            }
+            const row = this.toOutputRow(item as Record<string, unknown>);
+            const source = getPlayDatasetSourceRowIdentity(row);
+            const data = sanitizePostgresJsonValue(
+              toSerializableCsvAliasedRow(row),
+            );
+            materializationMemoryTracker.track(data);
+            return [
+              [
+                inputIndex,
+                { data, ...(source ? { source: { ...source } } : {}) },
+              ] as const,
+            ];
+          })
+        : [],
+    );
     const priorAuthoredDigestsByIndex = new Map(
       (suspendedInputRows ?? []).flatMap((row) =>
         row.authoredInputDigest
@@ -9405,6 +9577,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         const inputIndex = itemOriginalIndexes[index] ?? index;
         return {
           ...toSerializableCsvAliasedRow(row),
+          ...(getPlayDatasetSourceRowIdentity(row)
+            ? { __deeplineSourceRow: getPlayDatasetSourceRowIdentity(row) }
+            : {}),
           ...mapRowOutcomeRuntimeFields({
             key: seedRowIdentity(row, inputIndex),
             inputIndex,
@@ -9513,6 +9688,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       artifactTableNamespace: mapScope.artifactTableNamespace,
       status: 'running',
       totalRows: totalInputCount,
+      inputCardinality,
       completedRowKeys,
       pendingRowKeys,
       completedRowsCount: previouslyCompletedRowsCount,
@@ -9663,13 +9839,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           executionRowIndexes: rowsToExecuteEntries.map(
             (entry) => entry.originalIndex,
           ),
-          ...(!canReadPersistedMapResults
+          ...(canReadPersistedMapResults
             ? {
+                authoredInputs: rowsToExecuteEntries.map((entry) =>
+                  authoredInputsByIndex.get(entry.originalIndex),
+                ),
+              }
+            : {
                 authoredInputDigests: rowsToExecuteEntries.map((entry) =>
                   authoredInputDigestsByIndex.get(entry.originalIndex),
                 ),
-              }
-            : {}),
+              }),
           retainedRowsMemoryBudgetBytes:
             mapMemoryLimits.materializedBudgetBytes,
           progressCompletedOffset:
@@ -9854,13 +10034,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       });
     }
 
-    this.#options.onMapRowsSettled?.({
+    await this.#options.onMapRowsSettled?.({
       playName: this.#options.playName,
       runId: this.#options.runId,
       tableNamespace: resolvedTableNamespace,
       succeededRows:
         previouslyCompletedRowKeys.length + visibleCompletedRows.length,
       failedRows: durableFailedRows,
+      expectedRows: totalInputCount - duplicateReuseCount,
+      staticPipeline: this.currentStaticPipeline,
     });
 
     // This map's terminal rows have already crossed the persistence barrier
@@ -9950,8 +10132,24 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         skipped: reusedCount,
         pending: 0,
         failed: durableFailedRows,
-        ...(duplicateReuseCount > 0
-          ? { duplicates: { exact: duplicateReuseCount } }
+        ...(inputCardinality
+          ? {
+              cardinality: {
+                ...inputCardinality,
+                uniqueInputRows:
+                  inputCardinality.uniqueInputRows - duplicateReuseCount,
+                duplicateOccurrences:
+                  inputCardinality.duplicateOccurrences + duplicateReuseCount,
+                outputRows: results.length,
+              },
+            }
+          : {}),
+        ...(dedupedMaterialized.droppedCount + duplicateReuseCount > 0
+          ? {
+              duplicates: {
+                exact: dedupedMaterialized.droppedCount + duplicateReuseCount,
+              },
+            }
           : {}),
       },
       resolvers: {
@@ -9989,6 +10187,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       onRowError?: 'isolate' | 'fail';
       executionRowKeys?: string[];
       executionRowIndexes?: number[];
+      authoredInputs?: Array<
+        NonNullable<
+          MapExecutionFrame['suspendedInputRows']
+        >[number]['authoredInput']
+      >;
       authoredInputDigests?: Array<string | undefined>;
       /**
        * Opt-in cap on how many row resolvers run their body concurrently. When
@@ -10796,6 +10999,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                   ),
                   ...mapRowOutcomeRuntimeFields({ key: rowKey }),
                 },
+                ...(runtimeOptions?.authoredInputs?.[index]
+                  ? { authoredInput: runtimeOptions.authoredInputs[index] }
+                  : {}),
                 ...(runtimeOptions?.authoredInputDigests?.[index]
                   ? {
                       authoredInputDigest:
@@ -11204,6 +11410,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                       key: keyForIndex(index),
                     }),
                   },
+                  ...(runtimeOptions?.authoredInputs?.[index]
+                    ? { authoredInput: runtimeOptions.authoredInputs[index] }
+                    : {}),
                   ...(runtimeOptions?.authoredInputDigests?.[index]
                     ? {
                         authoredInputDigest:
@@ -12530,7 +12739,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
 
   private toOutputRow<T>(item: T): Record<string, unknown> {
     if (item != null && typeof item === 'object' && !Array.isArray(item)) {
-      return stripCsvProjectionMetadata(item as Record<string, unknown>);
+      return copyPlayDatasetSourceRowIdentity(
+        item,
+        stripCsvProjectionMetadata(item as Record<string, unknown>),
+      );
     }
 
     return { value: item };
@@ -17746,6 +17958,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     );
     const shouldWaitAtExecutionBoundary =
       shouldWaitForAsyncOperation && !shouldResolveAsyncOperationInRuntime;
+    const asyncExecutionRecovery = shouldWaitAtExecutionBoundary
+      ? await this.#options.getToolAsyncExecutionRecovery?.(toolId)
+      : null;
+    if (
+      asyncExecutionRecovery &&
+      !isImmutableRuntimeExecutionRecoveryOrigin(asyncExecutionRecovery.origin)
+    ) {
+      throw new Error(
+        'Async execution recovery requires the serving app immutable origin. No provider request was sent.',
+      );
+    }
     // An explicitly opted-in caller-managed launch gives the Play author the
     // provider handle and leaves polling to the authored graph. Do not let a
     // provider's legacy synchronous helper turn a wide start wave into
@@ -18106,6 +18329,70 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           // map cells, batched owners) supply the fuller scope.
           const playNodeScope =
             options?.playNodeScope ?? buildPlayNodeScope({ toolId });
+          const asyncRecoveryInvocationKey = asyncExecutionRecovery
+            ? stableDigest(
+                stableStringify({
+                  toolId,
+                  // A queued call/chunk supplies its exact durable transport
+                  // identity. A direct invocation owns a fresh call identity.
+                  callIdentity:
+                    options?.durableCallReceiptKey ??
+                    (options?.providerOperationForceRefreshIdentities?.length
+                      ? options.providerOperationForceRefreshIdentities
+                      : (options?.providerIdempotencyKey ??
+                        executionRequestNonce)),
+                  payload: toolInputSnapshot,
+                  authScope: executionAuthScopeDigest,
+                  playNodeScope,
+                  forceRefreshIdentities:
+                    options?.providerOperationForceRefreshIdentities ?? [],
+                }),
+              )
+            : null;
+          const asyncRecoveryPolling = asyncExecutionRecovery
+            ? resolveAsyncOperationPolling({
+                contract: asyncContract!,
+                startInput: toolInputSnapshot,
+                playRuntimeWait: true,
+              })
+            : null;
+          const asyncRecoveryDeadlineAtMs = asyncRecoveryPolling
+            ? Date.now() + asyncRecoveryPolling.timeoutMs
+            : null;
+          let confirmedAsyncRecoveryPending = false;
+          let asyncRecoveryWaitTimeout: ReturnType<
+            typeof createToolHttpError
+          > | null = null;
+          const asyncRecoveryWaitTimeoutError = () =>
+            (asyncRecoveryWaitTimeout ??= createToolHttpError(
+              toolErrorSchemaVersion,
+              formatAsyncOperationTimeout({
+                startAction: asyncContract!.lifecycle.startAction,
+                jobId: null,
+                timeoutDescription: `${asyncRecoveryPolling!.timeoutMs}ms`,
+              }) +
+                ' The execution remains recoverable under its original idempotency key; it will not be relaunched.',
+              null,
+              504,
+              'repairable',
+              {
+                toolId,
+                provider,
+                operation: toolId,
+                code: 'ASYNC_OPERATION_TIMEOUT',
+                origin: 'provider',
+                category: 'upstream',
+                retryable: false,
+                statusCode: 504,
+              },
+            ));
+          const asyncRecoveryKey = (attempt: number) =>
+            asyncRecoveryInvocationKey
+              ? buildRuntimeExecutionRecoveryKey({
+                  parentRunId: this.currentRunId,
+                  invocationKey: `${asyncRecoveryInvocationKey}:attempt:${attempt}`,
+                })
+              : null;
           // This is deliberately a claim about only the immediately preceding
           // *durable* provider attempt. The execution gateway validates and
           // attests it before a retried call can use its fresh charge
@@ -18361,6 +18648,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS &&
               (input.knownNotDispatched === true ||
                 usesDurableInvocationFence ||
+                asyncExecutionRecovery ||
                 ambiguousTransportReplayAllowed ||
                 fixtureReplaySafe)
             ) {
@@ -18368,7 +18656,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               // attempt and any prior provider-429 proof belong to the logical
               // invocation; only a definite retryable HTTP outcome advances it.
               // Ordinary transport retries retain their physical attempt count.
-              if (!usesDurableInvocationFence) {
+              if (!usesDurableInvocationFence && !asyncExecutionRecovery) {
                 providerRateLimitRetryFromInvocationAttempt = null;
                 invocationAttempt += 1;
               }
@@ -18454,6 +18742,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
 
           while (true) {
             toolAbortSignal?.throwIfAborted();
+            if (
+              confirmedAsyncRecoveryPending &&
+              Date.now() >= asyncRecoveryDeadlineAtMs!
+            ) {
+              throw asyncRecoveryWaitTimeoutError();
+            }
             if (
               providerNoProgressExpired({
                 startedAtMsByScope:
@@ -18806,6 +19100,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                           // A stale permit can take arbitrarily long to
                           // replace. Only fixture residence and the physical
                           // HTTP exchange spend the tool execution deadline.
+                          if (
+                            confirmedAsyncRecoveryPending &&
+                            Date.now() >= asyncRecoveryDeadlineAtMs!
+                          ) {
+                            throw asyncRecoveryWaitTimeoutError();
+                          }
                           armProviderTimeout();
                           beginProviderExecution();
                           const fetchedAt = Date.now();
@@ -18865,6 +19165,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                                     }
                                   : {}),
                                 'x-deepline-request-id': deeplineRequestId,
+                                ...(asyncRecoveryKey(invocationAttempt)
+                                  ? {
+                                      'idempotency-key':
+                                        asyncRecoveryKey(invocationAttempt)!,
+                                    }
+                                  : {}),
                                 ...(providerExecutionFenceKey
                                   ? {
                                       'x-deepline-idempotency-key':
@@ -19247,6 +19553,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 receiptHeartbeatSupervisor?.stop();
               }
             } catch (error) {
+              if (
+                asyncRecoveryWaitTimeout !== null &&
+                error === asyncRecoveryWaitTimeout
+              ) {
+                throw error;
+              }
               if (heartbeatFailure) {
                 throw heartbeatFailure;
               }
@@ -19366,6 +19678,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               const transportReplaySafe =
                 !ambiguousDispatchedFailure ||
                 usesDurableInvocationFence ||
+                Boolean(asyncExecutionRecovery) ||
                 ambiguousTransportReplayAllowed ||
                 fixtureReplaySafe;
               if (!transportReplaySafe) {
@@ -19444,8 +19757,62 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               continue;
             }
 
+            if (
+              asyncExecutionRecovery &&
+              response.ok &&
+              response.headers.get(
+                RUNTIME_EXECUTION_RECOVERY_SUPPORTED_HEADER,
+              ) !== 'true'
+            ) {
+              throw createToolHttpError(
+                toolErrorSchemaVersion,
+                `Tool ${toolId} did not confirm the declared async recovery reader. The execution will not be relaunched.`,
+                null,
+                response.status,
+                'repairable',
+              );
+            }
+
             if (!response.ok) {
               const text = responseErrorText ?? '';
+              const recoveryKey = asyncRecoveryKey(invocationAttempt);
+              if (
+                recoveryKey &&
+                response.status === 409 &&
+                response.headers.get(
+                  RUNTIME_EXECUTION_RECOVERY_SUPPORTED_HEADER,
+                ) === 'true'
+              ) {
+                const pending = (() => {
+                  try {
+                    return JSON.parse(text) as Record<string, unknown>;
+                  } catch {
+                    return null;
+                  }
+                })();
+                const recovery = pending?.executionRecovery as
+                  | Record<string, unknown>
+                  | undefined;
+                if (
+                  pending?.code === 'EXECUTION_IN_PROGRESS' &&
+                  pending.retryable === true &&
+                  recovery?.idempotencyKey === recoveryKey &&
+                  (recovery.state === 'running' ||
+                    recovery.state === 'completed')
+                ) {
+                  confirmedAsyncRecoveryPending = true;
+                  if (Date.now() >= asyncRecoveryDeadlineAtMs!) {
+                    throw asyncRecoveryWaitTimeoutError();
+                  }
+                  // This is continuation of already accepted work. Keep the
+                  // original body/key and let the route attach its saved job.
+                  await waitForToolRetry(
+                    Math.min(1_000, asyncRecoveryDeadlineAtMs! - Date.now()),
+                    false,
+                  );
+                  continue;
+                }
+              }
               const developerTunnelOrigin502 =
                 isDeeplineDeveloperTunnelOrigin502({
                   url,
@@ -19487,7 +19854,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                   status: response.status,
                   bodyText: text,
                   transientHttpRetrySafe:
-                    retrySafeTransientHttp || fixtureReplaySafe,
+                    retrySafeTransientHttp ||
+                    fixtureReplaySafe ||
+                    Boolean(asyncExecutionRecovery),
                   executionFenced: usesDurableInvocationFence,
                 });
               const failure =
@@ -19501,7 +19870,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                   retryAfterHeader: response.headers.get('retry-after'),
                   retryAtMs: parseToolExecuteExactProviderRetryAtMs(response),
                   transientHttpRetrySafe:
-                    retrySafeTransientHttp || fixtureReplaySafe,
+                    retrySafeTransientHttp ||
+                    fixtureReplaySafe ||
+                    Boolean(asyncExecutionRecovery),
                   executionFenced: usesDurableInvocationFence,
                   ...(providerCallElapsedMs !== null
                     ? { providerLatencyMs: providerCallElapsedMs }
@@ -19643,6 +20014,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 (failure.knownNotDispatched ||
                   failure.reason === 'gateway_invocation_in_progress' ||
                   ambiguousTransportReplayAllowed ||
+                  Boolean(asyncExecutionRecovery) ||
                   retrySafeTransientHttp ||
                   fixtureReplaySafe ||
                   // An explicit provider-owned 429 is a rejected operation,
@@ -19659,7 +20031,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 // prior-attempt sweep see an intentionally absent receipt.
                 if (
                   failure.reason !== 'gateway_invocation_in_progress' &&
-                  !shedKnownNotDispatched
+                  !shedKnownNotDispatched &&
+                  (!asyncExecutionRecovery ||
+                    failure.backpressureDelayMs !== null)
                 ) {
                   const previousInvocationAttempt = invocationAttempt;
                   if (failure.backpressureDelayMs !== null) {