deepline 0.1.165 → 0.1.167

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -148,10 +148,13 @@ import {
 } from './runtime/receipts';
 import {
   RuntimeReceiptWaitTimeoutError,
+  resolveRuntimeToolReceiptWaitMaxAttempts,
+  resolveRuntimeToolReceiptWaitTimeoutMs,
   waitForCompletedRuntimeReceipt,
 } from '../../../shared_libs/play-runtime/durable-receipt-execution';
 import type { RuntimeStepReceipt } from '../../../shared_libs/play-runtime/ctx-types';
 import {
+  canReclaimFailedWorkerToolReceipt,
   canReclaimTimedOutWorkerToolReceipt,
   markWorkerToolReceiptResultCached,
   markWorkerToolReceiptResultExecution,
@@ -531,20 +534,74 @@ const RUNTIME_API_INTEGRATION_EXECUTE_TIMEOUT_MS = 180_000;
 const RUNTIME_API_RETRY_DELAYS_MS = [
   250, 750, 1500, 3000, 5000, 10000,
 ] as const;
+const STANDARD_PLAY_RUNTIME_LIMIT_MS =
+  STANDARD_PLAY_RUNTIME_LIMIT_SECONDS * 1000;
+const STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE = `Based on this plan, max runtime is ${STANDARD_PLAY_RUNTIME_LIMIT_LABEL}. Use smaller batches; ask for runtime.`;
+
+function resolveRuntimeDeadlineRemainingMs(runtimeDeadlineMs?: number): number {
+  if (
+    typeof runtimeDeadlineMs !== 'number' ||
+    !Number.isFinite(runtimeDeadlineMs)
+  ) {
+    return Number.POSITIVE_INFINITY;
+  }
+  const remainingMs = Math.floor(runtimeDeadlineMs - nowMs());
+  if (remainingMs <= 0) {
+    throw new WorkflowAbortError(STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
+  }
+  return remainingMs;
+}
+
+function resolveToolRuntimeApiTimeout(input: {
+  requestInput: Record<string, unknown>;
+  runtimeDeadlineMs?: number;
+}): { timeoutMs: number; timeoutErrorMessage?: string } {
+  const toolTimeoutMs = resolveRuntimeToolReceiptWaitTimeoutMs(
+    input.requestInput,
+  );
+  const remainingMs = resolveRuntimeDeadlineRemainingMs(
+    input.runtimeDeadlineMs,
+  );
+  if (remainingMs < toolTimeoutMs) {
+    return {
+      timeoutMs: Math.max(1, remainingMs),
+      timeoutErrorMessage: STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE,
+    };
+  }
+  return { timeoutMs: toolTimeoutMs };
+}
 
 async function fetchRuntimeApi(
   baseUrl: string,
   path: string,
   init: RequestInit,
+  options?: {
+    timeoutMsOverride?: number;
+    timeoutErrorMessage?: string;
+    abortSignal?: AbortSignal;
+  },
 ): Promise<Response> {
   const timeoutMs =
-    path === '/api/v2/plays/run'
-      ? RUNTIME_API_PLAY_RUN_TIMEOUT_MS
-      : path === '/api/v2/plays/internal/egress-fetch'
-        ? RUNTIME_API_EGRESS_FETCH_TIMEOUT_MS
-        : /^\/api\/v2\/integrations\/[^/]+\/execute$/.test(path)
-          ? RUNTIME_API_INTEGRATION_EXECUTE_TIMEOUT_MS
-          : RUNTIME_API_TIMEOUT_MS;
+    typeof options?.timeoutMsOverride === 'number' &&
+    Number.isFinite(options.timeoutMsOverride)
+      ? Math.max(1, Math.ceil(options.timeoutMsOverride))
+      : path === '/api/v2/plays/run'
+        ? RUNTIME_API_PLAY_RUN_TIMEOUT_MS
+        : path === '/api/v2/plays/internal/egress-fetch'
+          ? RUNTIME_API_EGRESS_FETCH_TIMEOUT_MS
+          : /^\/api\/v2\/integrations\/[^/]+\/execute$/.test(path)
+            ? RUNTIME_API_INTEGRATION_EXECUTE_TIMEOUT_MS
+            : RUNTIME_API_TIMEOUT_MS;
+  const abortSignal = options?.abortSignal;
+  const abortError = () =>
+    abortSignal?.reason instanceof Error
+      ? abortSignal.reason
+      : new Error(
+          `[play-harness] runtime API call aborted. path=${path} baseUrl=${baseUrl}`,
+        );
+  if (abortSignal?.aborted) {
+    throw abortError();
+  }
   const controller = new AbortController();
   let timeout: ReturnType<typeof setTimeout> | null = null;
   const timeoutPromise = new Promise<never>((_, reject) => {
@@ -552,7 +609,8 @@ async function fetchRuntimeApi(
       controller.abort();
       reject(
         new Error(
-          `[play-harness] runtime API call timed out after ${timeoutMs}ms. path=${path} baseUrl=${baseUrl}`,
+          options?.timeoutErrorMessage ??
+            `[play-harness] runtime API call timed out after ${timeoutMs}ms. path=${path} baseUrl=${baseUrl}`,
         ),
       );
     }, timeoutMs);
@@ -566,6 +624,9 @@ async function fetchRuntimeApi(
     if (!cachedRuntimeApiBinding) {
       throw new Error('[play-harness] RUNTIME_API service binding is required');
     }
+    if (abortSignal?.aborted) {
+      throw abortError();
+    }
     const responsePromise = callRuntimeApiRpcBinding(
       cachedRuntimeApiBinding,
       mergedInit,
@@ -575,6 +636,10 @@ async function fetchRuntimeApi(
         timeoutMs,
       },
     );
+    // RUNTIME_API service bindings do not consume RequestInit.signal once the RPC
+    // is in flight. After dispatch, wait for the owner call to settle so the
+    // durable receipt records the real provider result instead of failing early
+    // and inviting duplicate provider work on retry.
     const response = await Promise.race([responsePromise, timeoutPromise]);
     if (await isRuntimeApiBindingNotFoundResponse(response)) {
       throw new Error(
@@ -585,7 +650,8 @@ async function fetchRuntimeApi(
   } catch (err) {
     if (err instanceof Error && err.name === 'AbortError') {
       throw new Error(
-        `[play-harness] runtime API call timed out after ${timeoutMs}ms. path=${path} baseUrl=${baseUrl}`,
+        options?.timeoutErrorMessage ??
+          `[play-harness] runtime API call timed out after ${timeoutMs}ms. path=${path} baseUrl=${baseUrl}`,
       );
     }
     throw err;
@@ -1188,6 +1254,8 @@ async function executeTool(
   onProviderBackpressure?: (retryAfterMs: number) => void,
   onRetryAttempt?: () => void,
   transientHttpRetrySafe = false,
+  abortSignal?: AbortSignal,
+  runtimeDeadlineMs?: number,
 ): Promise<ToolExecuteResult> {
   if (args.toolId === 'test_wait_for_event' && workflowStep) {
     const result = await waitForSyntheticIntegrationEvent(
@@ -1208,6 +1276,8 @@ async function executeTool(
     onProviderBackpressure,
     onRetryAttempt,
     transientHttpRetrySafe,
+    abortSignal,
+    runtimeDeadlineMs,
   );
 }
 
@@ -1219,6 +1289,8 @@ async function executeToolWithLifecycle(
   onProviderBackpressure?: (retryAfterMs: number) => void,
   onRetryAttempt?: () => void,
   transientHttpRetrySafe = false,
+  abortSignal?: AbortSignal,
+  runtimeDeadlineMs?: number,
 ): Promise<ToolExecuteResult> {
   callbacks?.onToolCalled?.(args.toolId, nowMs());
   try {
@@ -1229,6 +1301,8 @@ async function executeToolWithLifecycle(
       onProviderBackpressure,
       onRetryAttempt,
       transientHttpRetrySafe,
+      abortSignal,
+      runtimeDeadlineMs,
     );
   } catch (error) {
     callbacks?.onToolFailed?.(args.toolId, nowMs());
@@ -1365,6 +1439,8 @@ async function callToolDirect(
   // policy.budgets.maxRetryCount effectively unenforced.
   onRetryAttempt?: () => void,
   transientHttpRetrySafe = false,
+  abortSignal?: AbortSignal,
+  runtimeDeadlineMs?: number,
 ): Promise<ToolExecuteResult> {
   const { id, toolId, input } = args;
   const path = `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
@@ -1377,20 +1453,33 @@ async function callToolDirect(
     requestAttempt += 1;
     let res: Response;
     try {
-      res = await fetchRuntimeApi(req.baseUrl, path, {
-        method: 'POST',
-        headers: {
-          'content-type': 'application/json',
-          authorization: `Bearer ${req.executorToken}`,
-          'x-deepline-request-id': `${req.runId}:${toolId}:${id}:attempt:${requestAttempt}`,
-          [EXECUTE_RESPONSE_CONTRACT_HEADER]: V2_EXECUTE_RESPONSE_CONTRACT,
-          [EXECUTE_TOOL_METADATA_HEADER]: 'true',
-        },
-        body: JSON.stringify({
-          payload: input,
-          metadata: { parent_run_id: req.runId },
-        }),
+      const runtimeApiTimeout = resolveToolRuntimeApiTimeout({
+        requestInput: input,
+        runtimeDeadlineMs,
       });
+      res = await fetchRuntimeApi(
+        req.baseUrl,
+        path,
+        {
+          method: 'POST',
+          headers: {
+            'content-type': 'application/json',
+            authorization: `Bearer ${req.executorToken}`,
+            'x-deepline-request-id': `${req.runId}:${toolId}:${id}:attempt:${requestAttempt}`,
+            [EXECUTE_RESPONSE_CONTRACT_HEADER]: V2_EXECUTE_RESPONSE_CONTRACT,
+            [EXECUTE_TOOL_METADATA_HEADER]: 'true',
+          },
+          body: JSON.stringify({
+            payload: input,
+            metadata: { parent_run_id: req.runId },
+          }),
+        },
+        {
+          timeoutMsOverride: runtimeApiTimeout.timeoutMs,
+          timeoutErrorMessage: runtimeApiTimeout.timeoutErrorMessage,
+          abortSignal,
+        },
+      );
     } catch (error) {
       transportAttempt += 1;
       const message = error instanceof Error ? error.message : String(error);
@@ -1538,6 +1627,7 @@ type WorkerToolBatchRequest = {
   cacheKey: string;
   receiptKey: string | null;
   force: boolean;
+  receiptWaitMaxAttempts: number;
   toolId: string;
   input: Record<string, unknown>;
   workflowStep?: WorkflowStep;
@@ -1641,6 +1731,7 @@ class WorkerToolBatchScheduler {
     private readonly callbacks?: WorkerCtxCallbacks,
     private readonly receiptStore?: WorkerRuntimeReceiptStore,
     private readonly allowLocalRetryReceipts = false,
+    private readonly runtimeDeadlineMs?: number,
   ) {}
 
   /**
@@ -1685,6 +1776,7 @@ class WorkerToolBatchScheduler {
         cacheKey: receiptKey,
         receiptKey,
         force: options?.force === true,
+        receiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(input),
         toolId,
         input,
         workflowStep,
@@ -1747,19 +1839,19 @@ class WorkerToolBatchScheduler {
     }
   }
 
-  private async waitForDurableToolReceipt(
-    receiptKey: string,
-  ): Promise<unknown> {
-    if (!this.receiptStore) {
-      throw new Error('Worker durable tool receipt store is not configured.');
-    }
-    if (!this.receiptStore.getReceipt) {
+  private async waitForDurableToolReceipt(input: {
+    receiptKey: string;
+    maxAttempts: number;
+  }): Promise<WorkerRuntimeReceipt> {
+    if (!this.receiptStore?.getReceipt) {
       throw new Error(
-        'Worker durable tool receipt wait requires read-only receipt lookup.',
+        'Worker durable tool receipt wait requires receipt lookup.',
       );
     }
     const receipt = await waitForCompletedRuntimeReceipt({
-      receiptKey,
+      receiptKey: input.receiptKey,
+      maxAttempts: input.maxAttempts,
+      abortSignal: this.abortSignal,
       store: {
         getMany: async (receiptKeys) => {
           const receipts = new Map<string, RuntimeStepReceipt>();
@@ -1783,11 +1875,7 @@ class WorkerToolBatchScheduler {
         },
       },
     });
-    return markWorkerToolReceiptResultCached(
-      deserializeDurableStepValue(receipt.output),
-      receiptKey,
-      receiptKey,
-    );
+    return receipt;
   }
 
   private settleRequests(
@@ -1830,16 +1918,65 @@ class WorkerToolBatchScheduler {
     this.onRequestsSettled?.(requests.length);
   }
 
-  private async reclaimTimedOutDurableToolReceiptGroup(input: {
+  private async resolveCompletedDurableToolReceiptGroup(input: {
+    group: WorkerToolBatchRequest[];
+    receiptKey: string;
+    receipt: WorkerRuntimeReceipt;
+    source: 'cache' | 'in_flight';
+  }): Promise<void> {
+    const [first] = input.group;
+    if (!first) return;
+    const value = deserializeDurableStepValue(input.receipt.output);
+    const result =
+      input.source === 'cache'
+        ? markWorkerToolReceiptResultCached(
+            value,
+            first.cacheKey,
+            input.receiptKey,
+          )
+        : markWorkerToolReceiptResultExecution(value, {
+            kind: 'in_flight',
+            receiptKey: input.receiptKey,
+            attachedToReceiptKey: input.receiptKey,
+          });
+    for (const request of input.group) {
+      request.resolve(result);
+    }
+    this.onRequestsSettled?.(input.group.length);
+  }
+
+  private async reclaimRunningDurableToolReceiptGroupAfterWait(input: {
     group: WorkerToolBatchRequest[];
     receiptKey: string;
     runningReceipt: WorkerRuntimeReceipt;
-    waitError: unknown;
   }): Promise<ClaimedWorkerToolBatchRequest[]> {
     const [request, ...followers] = input.group;
     if (!request || !this.receiptStore) {
-      this.rejectRawRequests(input.group, input.waitError);
+      this.rejectRawRequests(
+        input.group,
+        new RuntimeReceiptWaitTimeoutError(input.receiptKey),
+      );
+      return [];
+    }
+    let waitError: unknown = null;
+    try {
+      const receipt = await this.waitForDurableToolReceipt({
+        receiptKey: input.receiptKey,
+        maxAttempts: request.receiptWaitMaxAttempts,
+      });
+      await this.resolveCompletedDurableToolReceiptGroup({
+        group: input.group,
+        receiptKey: input.receiptKey,
+        receipt,
+        source: 'in_flight',
+      });
       return [];
+    } catch (error) {
+      if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
+        this.rejectRawRequests(input.group, error);
+        return [];
+      }
+      waitError = error;
     }
     if (
       !canReclaimTimedOutWorkerToolReceipt({
@@ -1847,7 +1984,7 @@ class WorkerToolBatchScheduler {
         currentRunId: this.req.runId,
       })
     ) {
-      this.rejectRawRequests(input.group, input.waitError);
+      this.rejectRawRequests(input.group, waitError);
       return [];
     }
     let claim: WorkerRuntimeReceiptClaim;
@@ -1866,27 +2003,92 @@ class WorkerToolBatchScheduler {
       return [{ request, receiptKey: input.receiptKey, followers }];
     }
     if (claim.disposition === 'reused') {
-      const result = markWorkerToolReceiptResultCached(
-        deserializeDurableStepValue(claim.receipt.output),
-        request.cacheKey,
-        input.receiptKey,
-      );
-      for (const pending of input.group) {
-        pending.resolve(result);
-      }
-      this.onRequestsSettled?.(input.group.length);
+      await this.resolveCompletedDurableToolReceiptGroup({
+        group: input.group,
+        receiptKey: input.receiptKey,
+        receipt: claim.receipt,
+        source: 'cache',
+      });
       return [];
     }
     if (claim.disposition === 'failed') {
-      this.rejectRawRequests(
-        input.group,
-        new Error(
-          `Durable tool call ${input.receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
-        ),
-      );
+      return await this.reclaimFailedDurableToolReceiptGroup({
+        group: input.group,
+        receiptKey: input.receiptKey,
+        failedReceipt: claim.receipt,
+      });
+    }
+    this.rejectRawRequests(input.group, waitError);
+    return [];
+  }
+
+  private async reclaimFailedDurableToolReceiptGroup(input: {
+    group: WorkerToolBatchRequest[];
+    receiptKey: string;
+    failedReceipt: WorkerRuntimeReceipt;
+  }): Promise<ClaimedWorkerToolBatchRequest[]> {
+    const failedError = new Error(
+      `Durable tool call ${input.receiptKey} failed: ${input.failedReceipt.error ?? 'unknown error'}`,
+    );
+    const [request, ...followers] = input.group;
+    if (!request || !this.receiptStore) {
+      this.rejectRawRequests(input.group, failedError);
       return [];
     }
-    this.rejectRawRequests(input.group, input.waitError);
+    if (
+      !canReclaimFailedWorkerToolReceipt({
+        error: input.failedReceipt.error,
+      })
+    ) {
+      this.rejectRawRequests(input.group, failedError);
+      return [];
+    }
+    let claim: WorkerRuntimeReceiptClaim;
+    try {
+      claim = await this.receiptStore.claimReceipt({
+        playName: this.req.playName,
+        runId: this.req.runId,
+        key: input.receiptKey,
+      });
+    } catch (error) {
+      this.rejectRawRequests(input.group, error);
+      return [];
+    }
+    if (claim.disposition === 'claimed') {
+      return [{ request, receiptKey: input.receiptKey, followers }];
+    }
+    if (claim.disposition === 'reused') {
+      await this.resolveCompletedDurableToolReceiptGroup({
+        group: input.group,
+        receiptKey: input.receiptKey,
+        receipt: claim.receipt,
+        source: 'cache',
+      });
+      return [];
+    }
+    if (claim.disposition === 'running') {
+      try {
+        const receipt = await this.waitForDurableToolReceipt({
+          receiptKey: input.receiptKey,
+          maxAttempts: request.receiptWaitMaxAttempts,
+        });
+        await this.resolveCompletedDurableToolReceiptGroup({
+          group: input.group,
+          receiptKey: input.receiptKey,
+          receipt,
+          source: 'in_flight',
+        });
+      } catch (error) {
+        this.rejectRawRequests(input.group, error);
+      }
+      return [];
+    }
+    this.rejectRawRequests(
+      input.group,
+      new Error(
+        `Durable tool call ${input.receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
+      ),
+    );
     return [];
   }
 
@@ -2025,45 +2227,22 @@ class WorkerToolBatchScheduler {
           continue;
         }
         if (claim.disposition === 'failed') {
-          const error = new Error(
-            `Durable tool call ${receiptKey} failed: ${claim.receipt.error ?? 'unknown error'}`,
+          claimedRequests.push(
+            ...(await this.reclaimFailedDurableToolReceiptGroup({
+              group,
+              receiptKey,
+              failedReceipt: claim.receipt,
+            })),
           );
-          this.rejectRawRequests(group, error);
           continue;
         }
         if (claim.disposition === 'running') {
           deferredClaimedRequests.push(
-            (async (): Promise<ClaimedWorkerToolBatchRequest[]> => {
-              let waitError: unknown = new RuntimeReceiptWaitTimeoutError(
-                receiptKey,
-              );
-              try {
-                const result = await this.waitForDurableToolReceipt(receiptKey);
-                for (const request of group) {
-                  request.resolve(
-                    markWorkerToolReceiptResultExecution(result, {
-                      kind: 'in_flight',
-                      receiptKey,
-                      attachedToReceiptKey: receiptKey,
-                    }),
-                  );
-                }
-                this.onRequestsSettled?.(group.length);
-                return [];
-              } catch (error) {
-                waitError = error;
-                if (!(error instanceof RuntimeReceiptWaitTimeoutError)) {
-                  this.rejectRawRequests(group, error);
-                  return [];
-                }
-              }
-              return await this.reclaimTimedOutDurableToolReceiptGroup({
-                group,
-                receiptKey,
-                runningReceipt: claim.receipt,
-                waitError,
-              });
-            })(),
+            this.reclaimRunningDurableToolReceiptGroupAfterWait({
+              group,
+              receiptKey,
+              runningReceipt: claim.receipt,
+            }),
           );
           continue;
         }
@@ -2288,6 +2467,8 @@ class WorkerToolBatchScheduler {
               (retryAfterMs) => this.reportBackpressure(toolId, retryAfterMs),
               () => this.governor.chargeBudget('retry'),
               toolContract?.retrySafeTransientHttp === true,
+              this.abortSignal,
+              this.runtimeDeadlineMs,
             );
             this.settleRequests(
               claimed,
@@ -2328,6 +2509,7 @@ class WorkerToolBatchScheduler {
         WORKER_TOOL_BATCH_DEFAULT_PARALLELISM,
       ),
       abortSignal: this.abortSignal,
+      runtimeDeadlineMs: this.runtimeDeadlineMs,
       reportBackpressure: (retryAfterMs) =>
         this.reportBackpressure(toolId, retryAfterMs),
       resolveToolContract: this.resolvePacing,
@@ -2376,6 +2558,7 @@ async function executeBatchedWorkerToolGroup(input: {
   governor: PlayExecutionGovernor;
   suggestedParallelism: number;
   abortSignal?: AbortSignal;
+  runtimeDeadlineMs?: number;
   reportBackpressure: (retryAfterMs: number) => void;
   resolveToolContract: WorkerPacingResolver;
   onRequestsSettled?: (count: number) => void;
@@ -2444,6 +2627,8 @@ async function executeBatchedWorkerToolGroup(input: {
           input.reportBackpressure,
           () => input.governor.chargeBudget('retry'),
           toolContract?.retrySafeTransientHttp === true,
+          input.abortSignal,
+          input.runtimeDeadlineMs,
         );
       } catch (error) {
         input.callbacks?.onToolFailed?.(batch.batchOperation, nowMs());
@@ -4137,6 +4322,7 @@ function createMinimalWorkerCtx(
   workflowStep?: WorkflowStep,
   abortSignal?: AbortSignal,
   callbacks?: WorkerCtxCallbacks,
+  runtimeDeadlineMs?: number,
 ): unknown {
   const { governor, resolvePacing: resolveToolPacing } =
     createGovernorForRun(req);
@@ -4258,6 +4444,7 @@ function createMinimalWorkerCtx(
     callbacks,
     receiptStore,
     true,
+    runtimeDeadlineMs,
   );
   // Local ancestry chain that always ENDS with the currently-executing play
   // (req.playName). The /api/v2/plays/run lineage validator requires the
@@ -4397,9 +4584,7 @@ function createMinimalWorkerCtx(
       row: Record<string, unknown>,
     ): number | null => {
       const value = row[MAP_ROW_OUTCOME_RUNTIME_FIELDS.inputIndex];
-      return typeof value === 'number' && Number.isFinite(value)
-        ? value
-        : null;
+      return typeof value === 'number' && Number.isFinite(value) ? value : null;
     };
     const deriveDefaultMapRowIdentity = (
       row: Record<string, unknown>,
@@ -4461,12 +4646,7 @@ function createMinimalWorkerCtx(
     ): string => {
       const explicitKeyValue = resolveExplicitKeyValue(row, index);
       return explicitKeyValue == null
-        ? deriveDefaultMapRowIdentity(
-            row,
-            index,
-            chunkIndex,
-            chunkLocalIndex,
-          )
+        ? deriveDefaultMapRowIdentity(row, index, chunkIndex, chunkLocalIndex)
         : derivePlayRowIdentityFromKey(explicitKeyValue, name);
     };
     // Cross-chunk dedupe accumulators for the single end-of-map log line.
@@ -4743,6 +4923,7 @@ function createMinimalWorkerCtx(
         callbacks,
         receiptStore,
         false,
+        runtimeDeadlineMs,
       );
       let stepCellsCompleted = 0;
       let stepCellsSkipped = 0;
@@ -7165,6 +7346,7 @@ async function executeRunRequest(
 
   stepLifecycle?.markPreDatasetStepsStarted(startedAt);
   flushLedgerEvents(false);
+  const runtimeDeadlineMs = nowMs() + STANDARD_PLAY_RUNTIME_LIMIT_MS;
   const ctx = createMinimalWorkerCtx(
     req,
     wrappedEmit,
@@ -7172,6 +7354,7 @@ async function executeRunRequest(
     workflowStep,
     abortSignal,
     workerCallbacks,
+    runtimeDeadlineMs,
   );
   // Hard wall-clock cap on active user-code runtime. CF Workflows does not
   // impose a play-level execution ceiling on this substrate, so without this a
@@ -7179,14 +7362,15 @@ async function executeRunRequest(
   // token expires. Aborting the controller surfaces cooperatively through the
   // same assertNotAborted checks used for harness cancellation.
   let runtimeLimitExceeded = false;
-  const runtimeDeadlineTimer = setTimeout(() => {
-    runtimeLimitExceeded = true;
-    if (!abortSignal.aborted) {
-      abortController.abort(
-        `Based on this plan, max runtime is ${STANDARD_PLAY_RUNTIME_LIMIT_LABEL}. Use smaller batches; ask for runtime.`,
-      );
-    }
-  }, STANDARD_PLAY_RUNTIME_LIMIT_SECONDS * 1000);
+  const runtimeDeadlineTimer = setTimeout(
+    () => {
+      runtimeLimitExceeded = true;
+      if (!abortSignal.aborted) {
+        abortController.abort(STANDARD_PLAY_RUNTIME_LIMIT_MESSAGE);
+      }
+    },
+    Math.max(1, runtimeDeadlineMs - nowMs()),
+  );
   try {
     const playStartedAt = nowMs();
     const result = await (

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts

@@ -23,9 +23,19 @@ export function canReclaimTimedOutWorkerToolReceipt(input: {
   ownerRunId?: string | null;
   currentRunId: string;
 }): boolean {
-  const ownerRunId = input.ownerRunId?.trim();
   const currentRunId = input.currentRunId.trim();
-  return Boolean(ownerRunId && currentRunId && ownerRunId === currentRunId);
+  return Boolean(currentRunId);
+}
+
+export function canReclaimFailedWorkerToolReceipt(input: {
+  error?: string | null;
+}): boolean {
+  const error = input.error?.trim();
+  return Boolean(
+    error &&
+    (/\b(cancell?ed|aborted|terminate[d]?)\b/i.test(error) ||
+      /\bmax runtime\b/i.test(error)),
+  );
 }
 
 export function selectWorkerToolReceipt(input: {

package/dist/bundling-sources/sdk/src/agent-runtime.ts

@@ -35,9 +35,9 @@ export function isCoworkLikeSandbox(): boolean {
   const projectDir = Boolean(process.env.CLAUDE_PROJECT_DIR?.trim());
   const pluginRoot = Boolean(process.env.DEEPLINE_PLUGIN_ROOT?.trim());
   const home = process.env.HOME?.trim() || homedir();
-  const sessionHome = home.startsWith('/sessions/');
+  const sessionHome = home === '/sessions' || home.startsWith('/sessions/');
   return (
-    (pluginMode || pluginRoot) && (claudeRemote || projectDir || sessionHome)
+    claudeRemote || sessionHome || ((pluginMode || pluginRoot) && projectDir)
   );
 }