deepline 0.1.166 → 0.1.168

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -148,10 +148,13 @@ import {
 } from './runtime/receipts';
 import {
   RuntimeReceiptWaitTimeoutError,
+  resolveRuntimeToolReceiptWaitMaxAttempts,
+  resolveRuntimeToolReceiptWaitTimeoutMs,
   waitForCompletedRuntimeReceipt,
 } from '../../../shared_libs/play-runtime/durable-receipt-execution';
 import type { RuntimeStepReceipt } from '../../../shared_libs/play-runtime/ctx-types';
 import {
+  canReclaimFailedWorkerToolReceipt,
   canReclaimTimedOutWorkerToolReceipt,
   markWorkerToolReceiptResultCached,
   markWorkerToolReceiptResultExecution,
@@ -531,20 +534,74 @@ const RUNTIME_API_INTEGRATION_EXECUTE_TIMEOUT_MS = 180_000;
 const RUNTIME_API_RETRY_DELAYS_MS = [
   250, 750, 1500, 3000, 5000, 10000,
 ] as const;
+const STANDARD_PLAY_RUNTIME_LIMIT_MS =
+  STANDARD_PLAY_RUNTIME_LIMIT_SECONDS * 1000;
+const STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE = `Based on this plan, max runtime is ${STANDARD_PLAY_RUNTIME_LIMIT_LABEL}. Use smaller batches; ask for runtime.`;
+
+function resolveRuntimeDeadlineRemainingMs(runtimeDeadlineMs?: number): number {
+  if (
+    typeof runtimeDeadlineMs !== 'number' ||
+    !Number.isFinite(runtimeDeadlineMs)
+  ) {
+    return Number.POSITIVE_INFINITY;
+  }
+  const remainingMs = Math.floor(runtimeDeadlineMs - nowMs());
+  if (remainingMs <= 0) {
+    throw new WorkflowAbortError(STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
+  }
+  return remainingMs;
+}
+
+function resolveToolRuntimeApiTimeout(input: {
+  requestInput: Record<string, unknown>;
+  runtimeDeadlineMs?: number;
+}): { timeoutMs: number; timeoutErrorMessage?: string } {
+  const toolTimeoutMs = resolveRuntimeToolReceiptWaitTimeoutMs(
+    input.requestInput,
+  );
+  const remainingMs = resolveRuntimeDeadlineRemainingMs(
+    input.runtimeDeadlineMs,
+  );
+  if (remainingMs < toolTimeoutMs) {
+    return {
+      timeoutMs: Math.max(1, remainingMs),
+      timeoutErrorMessage: STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE,
+    };
+  }
+  return { timeoutMs: toolTimeoutMs };
+}
 
 async function fetchRuntimeApi(
   baseUrl: string,
   path: string,
   init: RequestInit,
+  options?: {
+    timeoutMsOverride?: number;
+    timeoutErrorMessage?: string;
+    abortSignal?: AbortSignal;
+  },
 ): Promise<Response> {
   const timeoutMs =
-    path === '/api/v2/plays/run'
-      ? RUNTIME_API_PLAY_RUN_TIMEOUT_MS
-      : path === '/api/v2/plays/internal/egress-fetch'
-        ? RUNTIME_API_EGRESS_FETCH_TIMEOUT_MS
-        : /^\/api\/v2\/integrations\/[^/]+\/execute$/.test(path)
-          ? RUNTIME_API_INTEGRATION_EXECUTE_TIMEOUT_MS
-          : RUNTIME_API_TIMEOUT_MS;
+    typeof options?.timeoutMsOverride === 'number' &&
+    Number.isFinite(options.timeoutMsOverride)
+      ? Math.max(1, Math.ceil(options.timeoutMsOverride))
+      : path === '/api/v2/plays/run'
+        ? RUNTIME_API_PLAY_RUN_TIMEOUT_MS
+        : path === '/api/v2/plays/internal/egress-fetch'
+          ? RUNTIME_API_EGRESS_FETCH_TIMEOUT_MS
+          : /^\/api\/v2\/integrations\/[^/]+\/execute$/.test(path)
+            ? RUNTIME_API_INTEGRATION_EXECUTE_TIMEOUT_MS
+            : RUNTIME_API_TIMEOUT_MS;
+  const abortSignal = options?.abortSignal;
+  const abortError = () =>
+    abortSignal?.reason instanceof Error
+      ? abortSignal.reason
+      : new Error(
+          `[play-harness] runtime API call aborted. path=${path} baseUrl=${baseUrl}`,
+        );
+  if (abortSignal?.aborted) {
+    throw abortError();
+  }
   const controller = new AbortController();
   let timeout: ReturnType<typeof setTimeout> | null = null;
   const timeoutPromise = new Promise<never>((_, reject) => {
@@ -552,7 +609,8 @@ async function fetchRuntimeApi(
       controller.abort();
       reject(
         new Error(
-          `[play-harness] runtime API call timed out after ${timeoutMs}ms. path=${path} baseUrl=${baseUrl}`,
+          options?.timeoutErrorMessage ??
+            `[play-harness] runtime API call timed out after ${timeoutMs}ms. path=${path} baseUrl=${baseUrl}`,
         ),
       );
     }, timeoutMs);
@@ -566,6 +624,9 @@ async function fetchRuntimeApi(
     if (!cachedRuntimeApiBinding) {
       throw new Error('[play-harness] RUNTIME_API service binding is required');
     }
+    if (abortSignal?.aborted) {
+      throw abortError();
+    }
     const responsePromise = callRuntimeApiRpcBinding(
       cachedRuntimeApiBinding,
       mergedInit,
@@ -575,6 +636,10 @@ async function fetchRuntimeApi(
         timeoutMs,
       },
     );
+    // RUNTIME_API service bindings do not consume RequestInit.signal once the RPC
+    // is in flight. After dispatch, wait for the owner call to settle so the
+    // durable receipt records the real provider result instead of failing early
+    // and inviting duplicate provider work on retry.
     const response = await Promise.race([responsePromise, timeoutPromise]);
     if (await isRuntimeApiBindingNotFoundResponse(response)) {
       throw new Error(
@@ -585,7 +650,8 @@ async function fetchRuntimeApi(
   } catch (err) {
     if (err instanceof Error && err.name === 'AbortError') {
       throw new Error(
-        `[play-harness] runtime API call timed out after ${timeoutMs}ms. path=${path} baseUrl=${baseUrl}`,
+        options?.timeoutErrorMessage ??
+          `[play-harness] runtime API call timed out after ${timeoutMs}ms. path=${path} baseUrl=${baseUrl}`,
       );
     }
     throw err;
@@ -1188,6 +1254,8 @@ async function executeTool(
   onProviderBackpressure?: (retryAfterMs: number) => void,
   onRetryAttempt?: () => void,
   transientHttpRetrySafe = false,
+  abortSignal?: AbortSignal,
+  runtimeDeadlineMs?: number,
 ): Promise<ToolExecuteResult> {
   if (args.toolId === 'test_wait_for_event' && workflowStep) {
     const result = await waitForSyntheticIntegrationEvent(
@@ -1208,6 +1276,8 @@ async function executeTool(
     onProviderBackpressure,
     onRetryAttempt,
     transientHttpRetrySafe,
+    abortSignal,
+    runtimeDeadlineMs,
   );
 }
 
@@ -1219,6 +1289,8 @@ async function executeToolWithLifecycle(
   onProviderBackpressure?: (retryAfterMs: number) => void,
   onRetryAttempt?: () => void,
   transientHttpRetrySafe = false,
+  abortSignal?: AbortSignal,
+  runtimeDeadlineMs?: number,
 ): Promise<ToolExecuteResult> {
   callbacks?.onToolCalled?.(args.toolId, nowMs());
   try {
@@ -1229,6 +1301,8 @@ async function executeToolWithLifecycle(
       onProviderBackpressure,
       onRetryAttempt,
       transientHttpRetrySafe,
+      abortSignal,
+      runtimeDeadlineMs,
     );
   } catch (error) {
     callbacks?.onToolFailed?.(args.toolId, nowMs());
@@ -1365,6 +1439,8 @@ async function callToolDirect(
   // policy.budgets.maxRetryCount effectively unenforced.
   onRetryAttempt?: () => void,
   transientHttpRetrySafe = false,
+  abortSignal?: AbortSignal,
+  runtimeDeadlineMs?: number,
 ): Promise<ToolExecuteResult> {
   const { id, toolId, input } = args;
   const path = `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
@@ -1377,20 +1453,33 @@ async function callToolDirect(
     requestAttempt += 1;
     let res: Response;
     try {
-      res = await fetchRuntimeApi(req.baseUrl, path, {
-        method: 'POST',
-        headers: {
-          'content-type': 'application/json',
-          authorization: `Bearer ${req.executorToken}`,
-          'x-deepline-request-id': `${req.runId}:${toolId}:${id}:attempt:${requestAttempt}`,
-          [EXECUTE_RESPONSE_CONTRACT_HEADER]: V2_EXECUTE_RESPONSE_CONTRACT,
-          [EXECUTE_TOOL_METADATA_HEADER]: 'true',
-        },
-        body: JSON.stringify({
-          payload: input,
-          metadata: { parent_run_id: req.runId },
-        }),
+      const runtimeApiTimeout = resolveToolRuntimeApiTimeout({
+        requestInput: input,
+        runtimeDeadlineMs,
       });
+      res = await fetchRuntimeApi(
+        req.baseUrl,
+        path,
+        {
+          method: 'POST',
+          headers: {
+            'content-type': 'application/json',
+            authorization: `Bearer ${req.executorToken}`,
+            'x-deepline-request-id': `${req.runId}:${toolId}:${id}:attempt:${requestAttempt}`,
+            [EXECUTE_RESPONSE_CONTRACT_HEADER]: V2_EXECUTE_RESPONSE_CONTRACT,
+            [EXECUTE_TOOL_METADATA_HEADER]: 'true',
+          },
+          body: JSON.stringify({
+            payload: input,
+            metadata: { parent_run_id: req.runId },
+          }),
+        },
+        {
+          timeoutMsOverride: runtimeApiTimeout.timeoutMs,
+          timeoutErrorMessage: runtimeApiTimeout.timeoutErrorMessage,
+          abortSignal,
+        },
+      );
     } catch (error) {
       transportAttempt += 1;
       const message = error instanceof Error ? error.message : String(error);
@@ -1538,6 +1627,7 @@ type WorkerToolBatchRequest = {
   cacheKey: string;
   receiptKey: string | null;
   force: boolean;
+  receiptWaitMaxAttempts: number;
   toolId: string;
   input: Record<string, unknown>;
   workflowStep?: WorkflowStep;
@@ -1641,6 +1731,7 @@ class WorkerToolBatchScheduler {
     private readonly callbacks?: WorkerCtxCallbacks,
     private readonly receiptStore?: WorkerRuntimeReceiptStore,
     private readonly allowLocalRetryReceipts = false,
+    private readonly runtimeDeadlineMs?: number,
   ) {}
 
   /**
@@ -1685,6 +1776,7 @@ class WorkerToolBatchScheduler {
         cacheKey: receiptKey,
         receiptKey,
         force: options?.force === true,
+        receiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(input),
         toolId,
         input,
         workflowStep,
@@ -1747,19 +1839,19 @@ class WorkerToolBatchScheduler {
     }
   }
 
-  private async waitForDurableToolReceipt(
-    receiptKey: string,
-  ): Promise<unknown> {
-    if (!this.receiptStore) {
-      throw new Error('Worker durable tool receipt store is not configured.');
-    }
-    if (!this.receiptStore.getReceipt) {
+  private async waitForDurableToolReceipt(input: {
+    receiptKey: string;
+    maxAttempts: number;
+  }): Promise<WorkerRuntimeReceipt> {
+    if (!this.receiptStore?.getReceipt) {
       throw new Error(
-        'Worker durable tool receipt wait requires read-only receipt lookup.',
+        'Worker durable tool receipt wait requires receipt lookup.',
       );
     }
     const receipt = await waitForCompletedRuntimeReceipt({
-      receiptKey,
+      receiptKey: input.receiptKey,
+      maxAttempts: input.maxAttempts,
+      abortSignal: this.abortSignal,
       store: {
         getMany: async (receiptKeys) => {
           const receipts = new Map<string, RuntimeStepReceipt>();
@@ -1783,11 +1875,7 @@ class WorkerToolBatchScheduler {
         },
       },
     });
-    return markWorkerToolReceiptResultCached(
-      deserializeDurableStepValue(receipt.output),
-      receiptKey,
-      receiptKey,
-    );
+    return receipt;
   }
 
   private settleRequests(
@@ -1830,24 +1918,73 @@ class WorkerToolBatchScheduler {
     this.onRequestsSettled?.(requests.length);
   }
 
-  private async reclaimTimedOutDurableToolReceiptGroup(input: {
+  private async resolveCompletedDurableToolReceiptGroup(input: {
+    group: WorkerToolBatchRequest[];
+    receiptKey: string;
+    receipt: WorkerRuntimeReceipt;
+    source: 'cache' | 'in_flight';
+  }): Promise<void> {
+    const [first] = input.group;
+    if (!first) return;
+    const value = deserializeDurableStepValue(input.receipt.output);
+    const result =
+      input.source === 'cache'
+        ? markWorkerToolReceiptResultCached(
+            value,
+            first.cacheKey,
+            input.receiptKey,
+          )
+        : markWorkerToolReceiptResultExecution(value, {
+            kind: 'in_flight',
+            receiptKey: input.receiptKey,
+            attachedToReceiptKey: input.receiptKey,
+          });
+    for (const request of input.group) {
+      request.resolve(result);
+    }
+    this.onRequestsSettled?.(input.group.length);
+  }
+
+  private async reclaimRunningDurableToolReceiptGroupAfterWait(input: {
     group: WorkerToolBatchRequest[];
     receiptKey: string;
     runningReceipt: WorkerRuntimeReceipt;
-    waitError: unknown;
   }): Promise<ClaimedWorkerToolBatchRequest[]> {
     const [request, ...followers] = input.group;
     if (!request || !this.receiptStore) {
-      this.rejectRawRequests(input.group, input.waitError);
+      this.rejectRawRequests(
+        input.group,
+        new RuntimeReceiptWaitTimeoutError(input.receiptKey),
+      );
       return [];
     }
+    let waitError: unknown = null;
+    try {
+      const receipt = await this.waitForDurableToolReceipt({
+        receiptKey: input.receiptKey,
+        maxAttempts: request.receiptWaitMaxAttempts,
+      });
+      await this.resolveCompletedDurableToolReceiptGroup({
+        group: input.group,
+        receiptKey: input.receiptKey,
+        receipt,
+        source: 'in_flight',
+      });
+      return [];
+    } catch (error) {
+      if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
+        this.rejectRawRequests(input.group, error);
+        return [];
+      }
+      waitError = error;
+    }
     if (
       !canReclaimTimedOutWorkerToolReceipt({
         ownerRunId: input.runningReceipt.runId,
         currentRunId: this.req.runId,
       })
     ) {
-      this.rejectRawRequests(input.group, input.waitError);
+      this.rejectRawRequests(input.group, waitError);
       return [];
     }
     let claim: WorkerRuntimeReceiptClaim;
@@ -1866,27 +2003,92 @@ class WorkerToolBatchScheduler {
       return [{ request, receiptKey: input.receiptKey, followers }];
     }
     if (claim.disposition === 'reused') {
-      const result = markWorkerToolReceiptResultCached(
-        deserializeDurableStepValue(claim.receipt.output),
-        request.cacheKey,
-        input.receiptKey,
-      );
-      for (const pending of input.group) {
-        pending.resolve(result);
-      }
-      this.onRequestsSettled?.(input.group.length);
+      await this.resolveCompletedDurableToolReceiptGroup({
+        group: input.group,
+        receiptKey: input.receiptKey,
+        receipt: claim.receipt,
+        source: 'cache',
+      });
       return [];
     }
     if (claim.disposition === 'failed') {
-      this.rejectRawRequests(
-        input.group,
-        new Error(
-          `Durable tool call ${input.receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
-        ),
-      );
+      return await this.reclaimFailedDurableToolReceiptGroup({
+        group: input.group,
+        receiptKey: input.receiptKey,
+        failedReceipt: claim.receipt,
+      });
+    }
+    this.rejectRawRequests(input.group, waitError);
+    return [];
+  }
+
+  private async reclaimFailedDurableToolReceiptGroup(input: {
+    group: WorkerToolBatchRequest[];
+    receiptKey: string;
+    failedReceipt: WorkerRuntimeReceipt;
+  }): Promise<ClaimedWorkerToolBatchRequest[]> {
+    const failedError = new Error(
+      `Durable tool call ${input.receiptKey} failed: ${input.failedReceipt.error ?? 'unknown error'}`,
+    );
+    const [request, ...followers] = input.group;
+    if (!request || !this.receiptStore) {
+      this.rejectRawRequests(input.group, failedError);
+      return [];
+    }
+    if (
+      !canReclaimFailedWorkerToolReceipt({
+        error: input.failedReceipt.error,
+      })
+    ) {
+      this.rejectRawRequests(input.group, failedError);
+      return [];
+    }
+    let claim: WorkerRuntimeReceiptClaim;
+    try {
+      claim = await this.receiptStore.claimReceipt({
+        playName: this.req.playName,
+        runId: this.req.runId,
+        key: input.receiptKey,
+      });
+    } catch (error) {
+      this.rejectRawRequests(input.group, error);
+      return [];
+    }
+    if (claim.disposition === 'claimed') {
+      return [{ request, receiptKey: input.receiptKey, followers }];
+    }
+    if (claim.disposition === 'reused') {
+      await this.resolveCompletedDurableToolReceiptGroup({
+        group: input.group,
+        receiptKey: input.receiptKey,
+        receipt: claim.receipt,
+        source: 'cache',
+      });
+      return [];
+    }
+    if (claim.disposition === 'running') {
+      try {
+        const receipt = await this.waitForDurableToolReceipt({
+          receiptKey: input.receiptKey,
+          maxAttempts: request.receiptWaitMaxAttempts,
+        });
+        await this.resolveCompletedDurableToolReceiptGroup({
+          group: input.group,
+          receiptKey: input.receiptKey,
+          receipt,
+          source: 'in_flight',
+        });
+      } catch (error) {
+        this.rejectRawRequests(input.group, error);
+      }
       return [];
     }
-    this.rejectRawRequests(input.group, input.waitError);
+    this.rejectRawRequests(
+      input.group,
+      new Error(
+        `Durable tool call ${input.receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
+      ),
+    );
     return [];
   }
 
@@ -2025,45 +2227,22 @@ class WorkerToolBatchScheduler {
           continue;
         }
         if (claim.disposition === 'failed') {
-          const error = new Error(
-            `Durable tool call ${receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
+          claimedRequests.push(
+            ...(await this.reclaimFailedDurableToolReceiptGroup({
+              group,
+              receiptKey,
+              failedReceipt: claim.receipt,
+            })),
           );
-          this.rejectRawRequests(group, error);
           continue;
         }
         if (claim.disposition === 'running') {
           deferredClaimedRequests.push(
-            (async (): Promise<ClaimedWorkerToolBatchRequest[]> => {
-              let waitError: unknown = new RuntimeReceiptWaitTimeoutError(
-                receiptKey,
-              );
-              try {
-                const result = await this.waitForDurableToolReceipt(receiptKey);
-                for (const request of group) {
-                  request.resolve(
-                    markWorkerToolReceiptResultExecution(result, {
-                      kind: 'in_flight',
-                      receiptKey,
-                      attachedToReceiptKey: receiptKey,
-                    }),
-                  );
-                }
-                this.onRequestsSettled?.(group.length);
-                return [];
-              } catch (error) {
-                waitError = error;
-                if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
-                  this.rejectRawRequests(group, error);
-                  return [];
-                }
-              }
-              return await this.reclaimTimedOutDurableToolReceiptGroup({
-                group,
-                receiptKey,
-                runningReceipt: claim.receipt,
-                waitError,
-              });
-            })(),
+            this.reclaimRunningDurableToolReceiptGroupAfterWait({
+              group,
+              receiptKey,
+              runningReceipt: claim.receipt,
+            }),
           );
           continue;
         }
@@ -2288,6 +2467,8 @@ class WorkerToolBatchScheduler {
               (retryAfterMs) => this.reportBackpressure(toolId, retryAfterMs),
               () => this.governor.chargeBudget('retry'),
               toolContract?.retrySafeTransientHttp === true,
+              this.abortSignal,
+              this.runtimeDeadlineMs,
             );
             this.settleRequests(
               claimed,
@@ -2328,6 +2509,7 @@ class WorkerToolBatchScheduler {
         WORKER_TOOL_BATCH_DEFAULT_PARALLELISM,
       ),
       abortSignal: this.abortSignal,
+      runtimeDeadlineMs: this.runtimeDeadlineMs,
       reportBackpressure: (retryAfterMs) =>
         this.reportBackpressure(toolId, retryAfterMs),
       resolveToolContract: this.resolvePacing,
@@ -2376,6 +2558,7 @@ async function executeBatchedWorkerToolGroup(input: {
   governor: PlayExecutionGovernor;
   suggestedParallelism: number;
   abortSignal?: AbortSignal;
+  runtimeDeadlineMs?: number;
   reportBackpressure: (retryAfterMs: number) => void;
   resolveToolContract: WorkerPacingResolver;
   onRequestsSettled?: (count: number) => void;
@@ -2444,6 +2627,8 @@ async function executeBatchedWorkerToolGroup(input: {
           input.reportBackpressure,
           () => input.governor.chargeBudget('retry'),
           toolContract?.retrySafeTransientHttp === true,
+          input.abortSignal,
+          input.runtimeDeadlineMs,
         );
       } catch (error) {
         input.callbacks?.onToolFailed?.(batch.batchOperation, nowMs());
@@ -3684,21 +3869,13 @@ async function persistCompletedMapRows(input: {
       const retry = await harnessPersistCompletedSheetRows(persistRequest);
       retryWritten = retry.rowsWritten;
       retryVisible = await readVisibleRowCount();
-      if (retryVisible >= rows.length) {
-        return {
-          rows: rows.length,
-          written: result.rowsWritten,
-          visible: visibleRows,
-          retryWritten,
-          retryVisible,
-        };
-      }
-      throw new Error(
-        `Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}; retry wrote ${retryWritten}/${rows.length}; retry visible ${retryVisible}; run ${req.runId}.`,
-      );
+      if (retryVisible < rows.length)
+        throw new Error(
+          `Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}; retry wrote ${retryWritten}/${rows.length}; retry visible ${retryVisible}; run ${req.runId}.`,
+        );
     }
   }
-  if (result.rowsWritten !== rows.length) {
+  if (result.rowsWritten !== rows.length && visibleRows < rows.length) {
     throw new Error(
       `Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; run ${req.runId}.`,
     );
@@ -4137,6 +4314,7 @@ function createMinimalWorkerCtx(
   workflowStep?: WorkflowStep,
   abortSignal?: AbortSignal,
   callbacks?: WorkerCtxCallbacks,
+  runtimeDeadlineMs?: number,
 ): unknown {
   const { governor, resolvePacing: resolveToolPacing } =
     createGovernorForRun(req);
@@ -4258,6 +4436,7 @@ function createMinimalWorkerCtx(
     callbacks,
     receiptStore,
     true,
+    runtimeDeadlineMs,
   );
   // Local ancestry chain that always ENDS with the currently-executing play
   // (req.playName). The /api/v2/plays/run lineage validator requires the
@@ -4397,9 +4576,7 @@ function createMinimalWorkerCtx(
       row: Record<string, unknown>,
     ): number | null => {
       const value = row[MAP_ROW_OUTCOME_RUNTIME_FIELDS.inputIndex];
-      return typeof value === 'number' && Number.isFinite(value)
-        ? value
-        : null;
+      return typeof value === 'number' && Number.isFinite(value) ? value : null;
     };
     const deriveDefaultMapRowIdentity = (
       row: Record<string, unknown>,
@@ -4461,12 +4638,7 @@ function createMinimalWorkerCtx(
     ): string => {
       const explicitKeyValue = resolveExplicitKeyValue(row, index);
       return explicitKeyValue == null
-        ? deriveDefaultMapRowIdentity(
-            row,
-            index,
-            chunkIndex,
-            chunkLocalIndex,
-          )
+        ? deriveDefaultMapRowIdentity(row, index, chunkIndex, chunkLocalIndex)
         : derivePlayRowIdentityFromKey(explicitKeyValue, name);
     };
     // Cross-chunk dedupe accumulators for the single end-of-map log line.
@@ -4743,6 +4915,7 @@ function createMinimalWorkerCtx(
         callbacks,
         receiptStore,
         false,
+        runtimeDeadlineMs,
       );
       let stepCellsCompleted = 0;
       let stepCellsSkipped = 0;
@@ -7165,6 +7338,7 @@ async function executeRunRequest(
 
   stepLifecycle?.markPreDatasetStepsStarted(startedAt);
   flushLedgerEvents(false);
+  const runtimeDeadlineMs = nowMs() + STANDARD_PLAY_RUNTIME_LIMIT_MS;
   const ctx = createMinimalWorkerCtx(
     req,
     wrappedEmit,
@@ -7172,6 +7346,7 @@ async function executeRunRequest(
     workflowStep,
     abortSignal,
     workerCallbacks,
+    runtimeDeadlineMs,
   );
   // Hard wall-clock cap on active user-code runtime. CF Workflows does not
   // impose a play-level execution ceiling on this substrate, so without this a
@@ -7179,14 +7354,15 @@ async function executeRunRequest(
   // token expires. Aborting the controller surfaces cooperatively through the
   // same assertNotAborted checks used for harness cancellation.
   let runtimeLimitExceeded = false;
-  const runtimeDeadlineTimer = setTimeout(() => {
-    runtimeLimitExceeded = true;
-    if (!abortSignal.aborted) {
-      abortController.abort(
-        `Based on this plan, max runtime is ${STANDARD_PLAY_RUNTIME_LIMIT_LABEL}. Use smaller batches; ask for runtime.`,
-      );
-    }
-  }, STANDARD_PLAY_RUNTIME_LIMIT_SECONDS * 1000);
+  const runtimeDeadlineTimer = setTimeout(
+    () => {
+      runtimeLimitExceeded = true;
+      if (!abortSignal.aborted) {
+        abortController.abort(STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
+      }
+    },
+    Math.max(1, runtimeDeadlineMs - nowMs()),
+  );
   try {
     const playStartedAt = nowMs();
     const result = await (

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts

@@ -23,9 +23,19 @@ export function canReclaimTimedOutWorkerToolReceipt(input: {
   ownerRunId?: string | null;
   currentRunId: string;
 }): boolean {
-  const ownerRunId = input.ownerRunId?.trim();
   const currentRunId = input.currentRunId.trim();
-  return Boolean(ownerRunId && currentRunId && ownerRunId === currentRunId);
+  return Boolean(currentRunId);
+}
+
+export function canReclaimFailedWorkerToolReceipt(input: {
+  error?: string | null;
+}): boolean {
+  const error = input.error?.trim();
+  return Boolean(
+    error &&
+    (/\b(cancell?ed|aborted|terminate[d]?)\b/i.test(error) ||
+      /\bmax runtime\b/i.test(error)),
+  );
 }
 
 export function selectWorkerToolReceipt(input: {

package/dist/bundling-sources/sdk/src/release.ts

@@ -104,10 +104,10 @@ export const SDK_RELEASE = {
   // 0.1.111 ships dataset-native tool list getters and result row datasets.
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
-  version: '0.1.166',
+  version: '0.1.168',
   apiContract: '2026-06-dataset-handle-results-hard-cutover',
   supportPolicy: {
-    latest: '0.1.166',
+    latest: '0.1.168',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.53',
     commandMinimumSupported: [

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -4335,7 +4335,7 @@ export class PlayContextImpl {
           }
         };
 
-        const durableWaiters: Promise<void>[] = [];
+        const durableExistingRunningHandlers: Promise<void>[] = [];
         if (
           pendingRequests.length > 0 &&
           (this.#options.claimRuntimeStepReceipts ||
@@ -4448,7 +4448,7 @@ export class PlayContextImpl {
               liveFollowersByOwnerCallId.set(owner.callId, followers);
             }
           };
-          const waitForReceiptOrTimeout = async (
+          const reclaimExistingRunningReceiptAfterWait = async (
             receiptKey: string,
             requestsForKey: ToolCallRequest[],
           ): Promise<void> => {
@@ -4469,6 +4469,65 @@ export class PlayContextImpl {
                 return;
               }
             }
+            const reclaimed = (
+              await this.claimRuntimeStepReceipts(
+                [receiptKey],
+                this.currentRunId,
+                true,
+              )
+            ).get(receiptKey);
+            if (
+              reclaimed?.status === 'completed' ||
+              reclaimed?.status === 'skipped'
+            ) {
+              await resolveRequestsFromReceipt(
+                requestsForKey,
+                reclaimed,
+                'cache',
+              );
+              durableRecoveredRequests.push(...requestsForKey);
+              return;
+            }
+            if (reclaimed?.status === 'failed') {
+              for (const request of requestsForKey) {
+                await this.rejectToolCall(
+                  toolId,
+                  request,
+                  new Error(reclaimed.error ?? 'Durable tool call failed.'),
+                );
+              }
+              return;
+            }
+            if (
+              reclaimed?.status === 'running' &&
+              reclaimed.claimState !== 'existing'
+            ) {
+              const [owner, ...waiters] = requestsForKey;
+              if (!owner) return;
+              if (waiters.length > 0) {
+                liveFollowersByOwnerCallId.set(owner.callId, waiters);
+              }
+              try {
+                const execution = await this.callToolExecutionAPI(
+                  toolId,
+                  owner.input,
+                );
+                const result = await this.resolveToolCall(
+                  toolId,
+                  owner,
+                  execution?.result ?? null,
+                  execution?.metadata ?? null,
+                  execution?.jobId,
+                  execution?.meta,
+                );
+                resolveLiveFollowers(owner, result);
+                recordToolStep([owner]);
+                this.#options.onBatchComplete?.(this.checkpoint);
+              } catch (error) {
+                await rejectWithLiveFollowers(owner, error);
+              }
+              return;
+            }
             for (const request of requestsForKey) {
               await this.rejectToolCall(
                 toolId,
@@ -4510,8 +4569,23 @@ export class PlayContextImpl {
               claimOwnerWithLiveFollowers(owner, waiters);
               continue;
             }
-            durableWaiters.push(
-              waitForReceiptOrTimeout(receiptKey, requestsForKey),
+            if (
+              claim?.status === 'running' &&
+              claim.claimState === 'existing'
+            ) {
+              durableExistingRunningHandlers.push(
+                reclaimExistingRunningReceiptAfterWait(
+                  receiptKey,
+                  requestsForKey,
+                ),
+              );
+              continue;
+            }
+            durableExistingRunningHandlers.push(
+              reclaimExistingRunningReceiptAfterWait(
+                receiptKey,
+                requestsForKey,
+              ),
             );
           }
 
@@ -4621,8 +4695,8 @@ export class PlayContextImpl {
             }
           }
         }
-        if (durableWaiters.length > 0) {
-          await Promise.allSettled(durableWaiters);
+        if (durableExistingRunningHandlers.length > 0) {
+          await Promise.allSettled(durableExistingRunningHandlers);
         }
       }),
     );

package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts

@@ -10,6 +10,9 @@ import type { DurableReceiptRecoverySource } from './tool-execution-outcome';
 
 const DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS = 240;
 const DURABLE_RECEIPT_WAIT_DELAY_MS = 250;
+const TOOL_RECEIPT_DEFAULT_WAIT_MS = 300_000;
+const TOOL_RECEIPT_COMPLETION_BUFFER_MS = 30_000;
+const TOOL_RECEIPT_MAX_WAIT_MS = 30 * 60_000;
 
 export class RuntimeReceiptWaitTimeoutError extends Error {
   constructor(key: string) {
@@ -18,6 +21,76 @@ export class RuntimeReceiptWaitTimeoutError extends Error {
   }
 }
 
+export function resolveRuntimeToolReceiptWaitTimeoutMs(
+  requestInput: Record<string, unknown>,
+): number {
+  const explicitTimeoutCandidate =
+    requestInput.timeoutMs ??
+    requestInput.timeout_ms ??
+    requestInput.max_wait_ms;
+  const explicitTimeoutMs =
+    typeof explicitTimeoutCandidate === 'number' &&
+    isFinite(explicitTimeoutCandidate) &&
+    explicitTimeoutCandidate > 0
+      ? explicitTimeoutCandidate
+      : undefined;
+  const toolTimeoutMs = explicitTimeoutMs ?? TOOL_RECEIPT_DEFAULT_WAIT_MS;
+  return Math.min(
+    TOOL_RECEIPT_MAX_WAIT_MS,
+    Math.max(60_000, toolTimeoutMs + TOOL_RECEIPT_COMPLETION_BUFFER_MS),
+  );
+}
+
+export function resolveRuntimeToolReceiptWaitMaxAttempts(
+  requestInput: Record<string, unknown>,
+): number {
+  return Math.ceil(
+    resolveRuntimeToolReceiptWaitTimeoutMs(requestInput) /
+      DURABLE_RECEIPT_WAIT_DELAY_MS,
+  );
+}
+
+function throwIfReceiptWaitAborted(signal?: AbortSignal): void {
+  if (!signal?.aborted) return;
+  throw signal.reason instanceof Error
+    ? signal.reason
+    : new Error(
+        typeof signal.reason === 'string' && signal.reason.trim()
+          ? signal.reason
+          : 'Durable receipt wait aborted.',
+      );
+}
+
+async function sleepReceiptWait(
+  delayMs: number,
+  signal?: AbortSignal,
+): Promise<void> {
+  throwIfReceiptWaitAborted(signal);
+  if (delayMs <= 0) return;
+  await new Promise<void>((resolve, reject) => {
+    const timeout = setTimeout(finish, delayMs);
+    const abort = () => {
+      clearTimeout(timeout);
+      reject(
+        signal?.reason instanceof Error
+          ? signal.reason
+          : new Error(
+              typeof signal?.reason === 'string' && signal.reason.trim()
+                ? signal.reason
+                : 'Durable receipt wait aborted.',
+            ),
+      );
+    };
+    function finish() {
+      signal?.removeEventListener('abort', abort);
+      resolve();
+    }
+    signal?.addEventListener('abort', abort, { once: true });
+    if (signal?.aborted) abort();
+  });
+  throwIfReceiptWaitAborted(signal);
+}
+
 export type DurableReceiptOperation = 'step' | 'tool' | 'fetch' | 'runPlay';
 
 export type DurableReceiptExecutionStore = {
@@ -55,12 +128,14 @@ export async function waitForCompletedRuntimeReceipt(input: {
   store: Pick<DurableReceiptExecutionStore, 'getMany'>;
   maxAttempts?: number;
   delayMs?: number;
+  abortSignal?: AbortSignal;
 }): Promise<RuntimeStepReceipt> {
   const maxAttempts = input.maxAttempts ?? DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS;
   const delayMs = input.delayMs ?? DURABLE_RECEIPT_WAIT_DELAY_MS;
   for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
+    throwIfReceiptWaitAborted(input.abortSignal);
     if (attempt > 0) {
-      await new Promise((resolve) => setTimeout(resolve, delayMs));
+      await sleepReceiptWait(delayMs, input.abortSignal);
     }
     const receipt = (await input.store.getMany([input.receiptKey])).get(
       input.receiptKey,
@@ -189,13 +264,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
     try {
       return await waitForRunningReceiptOrTimeout();
     } catch (error) {
-      if (
-        input.repairRunningReceiptForSameRunAfterWaitTimeout === true &&
-        error instanceof RuntimeReceiptWaitTimeoutError &&
-        receipt.status === 'running' &&
-        typeof receipt.runId === 'string' &&
-        receipt.runId.trim() === input.runId
-      ) {
+      if (error instanceof RuntimeReceiptWaitTimeoutError) {
         const recovered = await reclaimRunningReceipt();
         if (recovered.kind === 'recovered') return recovered;
         return { kind: 'claimed' };

package/dist/cli/index.js

@@ -622,10 +622,10 @@ var SDK_RELEASE = {
   // 0.1.111 ships dataset-native tool list getters and result row datasets.
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
-  version: "0.1.166",
+  version: "0.1.168",
   apiContract: "2026-06-dataset-handle-results-hard-cutover",
   supportPolicy: {
-    latest: "0.1.166",
+    latest: "0.1.168",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.53",
     commandMinimumSupported: [

package/dist/cli/index.mjs

@@ -607,10 +607,10 @@ var SDK_RELEASE = {
   // 0.1.111 ships dataset-native tool list getters and result row datasets.
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
-  version: "0.1.166",
+  version: "0.1.168",
   apiContract: "2026-06-dataset-handle-results-hard-cutover",
   supportPolicy: {
-    latest: "0.1.166",
+    latest: "0.1.168",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.53",
     commandMinimumSupported: [

package/dist/index.js

@@ -421,10 +421,10 @@ var SDK_RELEASE = {
   // 0.1.111 ships dataset-native tool list getters and result row datasets.
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
-  version: "0.1.166",
+  version: "0.1.168",
   apiContract: "2026-06-dataset-handle-results-hard-cutover",
   supportPolicy: {
-    latest: "0.1.166",
+    latest: "0.1.168",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.53",
     commandMinimumSupported: [

package/dist/index.mjs

@@ -351,10 +351,10 @@ var SDK_RELEASE = {
   // 0.1.111 ships dataset-native tool list getters and result row datasets.
   // 0.1.154 removes the short-lived generated enrich StepOptions recompute
   // fields shipped in 0.1.153.
-  version: "0.1.166",
+  version: "0.1.168",
   apiContract: "2026-06-dataset-handle-results-hard-cutover",
   supportPolicy: {
-    latest: "0.1.166",
+    latest: "0.1.168",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.53",
     commandMinimumSupported: [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.166",
+  "version": "0.1.168",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {