deepline 0.1.198 → 0.1.200

package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts

@@ -5745,7 +5745,11 @@ async function handleCoordinatorWarmup(
       `https://deepline.coordinator.internal/workflow/${encodeURIComponent(runId)}/submit`,
       {
         method: 'POST',
-        headers: { 'content-type': 'application/json' },
+        headers: {
+          'content-type': 'application/json',
+          [COORDINATOR_INTERNAL_TOKEN_HEADER]:
+            env.DEEPLINE_INTERNAL_TOKEN?.trim() ?? '',
+        },
         body: JSON.stringify(params),
       },
     ),

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -39,7 +39,7 @@ import {
 import {
   STANDARD_PLAY_RUNTIME_LIMIT_LABEL,
   STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
-} from '../../../shared_libs/temporal/constants';
+} from '../../../shared_libs/play-runtime/runtime-constants';
 import {
   createPlayExecutionGovernor,
   type GovernanceSnapshot,
@@ -194,6 +194,11 @@ import {
   runtimeLeaseHeartbeatIntervalMs,
 } from '../../../shared_libs/play-runtime/lease-policy';
 import { RuntimeSheetRowsBlockedError } from '../../../shared_libs/play-runtime/runtime-sheet-errors';
+import {
+  resolveRuntimeMapRowAdmission,
+  runtimeMapJsonByteLength,
+} from '../../../shared_libs/play-runtime/map-memory-limits';
+import { createSerialTaskQueue } from '../../../shared_libs/play-runtime/serial-task-queue';
 // The harness stub forwards leaf calls (validation, runtime-api HTTP) into
 // the long-lived Play Harness Worker via env.HARNESS. We import the
 // `setHarnessBinding` setter eagerly so it's available the moment
@@ -4440,10 +4445,14 @@ function createMinimalWorkerCtx(
     );
   const executeWithRuntimeReceipt = async <T>(
     key: string,
-    execute: (context: WorkerRuntimeReceiptExecutionContext) => Promise<T> | T,
+    execute: (
+      context: WorkerRuntimeReceiptExecutionContext,
+      wasFailed: boolean,
+    ) => Promise<T> | T,
     options: {
       repairRunningReceiptForSameRun?: boolean;
       reclaimRunning?: boolean;
+      forceFailedRefresh?: boolean;
       repairRunningReceiptForSameRunAfterWaitTimeout?: boolean;
       runningReceiptWaitMaxAttempts?: number;
     } = {},
@@ -4454,11 +4463,12 @@ function createMinimalWorkerCtx(
       runId: req.runId,
       key,
       receiptStore,
-      execute: async (context) =>
-        serializeDurableStepValue(await execute(context)),
+      execute: async (context, wasFailed) =>
+        serializeDurableStepValue(await execute(context, wasFailed)),
       repairRunningReceiptForSameRun:
         options.repairRunningReceiptForSameRun ?? false,
       reclaimRunning: options.reclaimRunning ?? false,
+      forceFailedRefresh: options.forceFailedRefresh ?? false,
       repairRunningReceiptForSameRunAfterWaitTimeout:
         options.repairRunningReceiptForSameRunAfterWaitTimeout ?? false,
       runningReceiptWaitMaxAttempts: options.runningReceiptWaitMaxAttempts,
@@ -5131,7 +5141,32 @@ function createMinimalWorkerCtx(
       // body additionally acquires a global row slot (the Governor's rowMax
       // semaphore) so total in-flight rows across all maps in this isolate stay
       // bounded even when several maps run at once.
-      const concurrency = governor.resolveRowConcurrency();
+      const requestedRowConcurrency = governor.resolveRowConcurrency();
+      const largestActiveRowBytes = uniqueRowsToExecuteEntries.reduce(
+        (largest, entry) =>
+          Math.max(
+            largest,
+            runtimeMapJsonByteLength(
+              runtimeCsvExecutionRow(
+                entry.row,
+                pendingRowsByKey.get(entry.rowKey),
+              ),
+            ),
+          ),
+        0,
+      );
+      const rowAdmission = resolveRuntimeMapRowAdmission({
+        rowCount: rowsToExecute.length,
+        requestedConcurrency: requestedRowConcurrency,
+        largestRowBytes: largestActiveRowBytes,
+        activeRowsBudgetBytes: WORKERS_EDGE_MAP_RESIDENT_ROW_BYTES,
+      });
+      const concurrency = rowAdmission.concurrency;
+      if (rowsToExecute.length > 0 && concurrency === 0) {
+        throw new Error(
+          `RUNTIME_MAP_MEMORY_LIMIT_EXCEEDED:${name} row needs ${rowAdmission.estimatedBytesPerActiveRow} bytes; limit ${WORKERS_EDGE_MAP_RESIDENT_ROW_BYTES}. Reduce or stage blobs.`,
+        );
+      }
       const executedRows: Array<T & Record<string, unknown>> = new Array(
         rowsToExecute.length,
       );
@@ -5191,11 +5226,10 @@ function createMinimalWorkerCtx(
       let pendingPersistRows = 0;
       let pendingPersistBytes = 0;
       let scheduledPersistTimer: ReturnType<typeof setTimeout> | null = null;
-      let persistFlushChain: Promise<void> = Promise.resolve();
       let persistFailure: unknown = null;
       let liveUpdateTimer: ReturnType<typeof setTimeout> | null = null;
-      let liveFlushChain: Promise<void> = Promise.resolve();
       let liveUpdateFailures = 0;
+      const sheetWriteQueue = createSerialTaskQueue();
 
       const clearScheduledPersistTimer = () => {
         if (scheduledPersistTimer) {
@@ -5326,12 +5360,12 @@ function createMinimalWorkerCtx(
         clearScheduledPersistTimer();
         pendingPersistRows = 0;
         pendingPersistBytes = 0;
-        const task = persistFlushChain.then(async () => {
+        const task = sheetWriteQueue.enqueue(async () => {
           const circuitError = runtimePersistenceCircuitError();
           if (circuitError) throw circuitError;
           await persistExecutedRows();
         });
-        persistFlushChain = task.catch((error) => {
+        void task.catch((error) => {
           persistFailure ??= error;
           tripRuntimePersistenceLatch(persistenceLatch, error);
         });
@@ -5363,11 +5397,11 @@ function createMinimalWorkerCtx(
       const flushLiveRowUpdates = (): Promise<void> => {
         clearLiveUpdateTimer();
         if (pendingLiveUpdates.length === 0) {
-          return liveFlushChain;
+          return sheetWriteQueue.idle();
         }
         const updates = coalesceLiveUpdates(pendingLiveUpdates.splice(0));
         const extraOutputFields = Array.from(generatedOutputFields);
-        const task = liveFlushChain.then(async () => {
+        return sheetWriteQueue.enqueue(async () => {
           try {
             await applyLiveMapRowUpdates({
               req,
@@ -5390,8 +5424,6 @@ function createMinimalWorkerCtx(
             }
           }
         });
-        liveFlushChain = task.catch(() => undefined);
-        return task;
       };
 
       const scheduleLiveRowUpdates = () => {
@@ -5849,9 +5881,8 @@ function createMinimalWorkerCtx(
       });
       try {
         await flushLiveRowUpdates();
-        await liveFlushChain;
         await enqueuePersistExecutedRows();
-        await persistFlushChain;
+        await sheetWriteQueue.idle();
         if (persistFailure) throw persistFailure;
         recordRunnerPerfTrace({
           req,
@@ -6897,7 +6928,7 @@ function createMinimalWorkerCtx(
       });
       return await executeWithRuntimeReceipt(
         receiptKey,
-        async () => {
+        async (receiptContext, wasFailed) => {
           // The Governor owns the play-call lineage: forkChild does the cycle
           // guard, depth/per-parent/playCall/descendant budget charges, and
           // returns the snapshot to thread into the child so budgets accumulate
@@ -6980,7 +7011,9 @@ function createMinimalWorkerCtx(
                   options?.timeoutMs == null && !childNeedsWorkflowScheduler,
                 body: {
                   name: resolvedName,
-                  childIdempotencyKey: receiptKey,
+                  childIdempotencyKey: wasFailed && receiptContext.leaseId
+                    ? `${receiptKey}:claim:${receiptContext.leaseId}`
+                    : receiptKey,
                   input: isRecord(input) ? input : {},
                   orgId: req.orgId,
                   callbackUrl: req.callbackUrl,
@@ -7259,6 +7292,7 @@ function createMinimalWorkerCtx(
           }
         },
         {
+          forceFailedRefresh: true,
           repairRunningReceiptForSameRunAfterWaitTimeout: true,
           runningReceiptWaitMaxAttempts:
             resolveRuntimeToolReceiptWaitMaxAttempts(input),

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/README.md

@@ -7,7 +7,7 @@ These modules are bundled into every `esm_workers` play artifact. Keep them:
 - specific to the Cloudflare Worker execution path
 
 Do not import broad `shared_libs/play-runtime/*` modules here. Shared runtime
-modules are optimized for the Node/Temporal runner and often bring along code
+modules are optimized for the Node runner and often bring along code
 that is correct but expensive in a per-graphHash Worker isolate.
 
 If a helper needs zod, Neon, runtime API transport, validation registries, or

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts

@@ -7,6 +7,7 @@ import {
   harnessFailRuntimeReceipts,
   harnessGetRuntimeReceipt,
   harnessHeartbeatRuntimeReceipts,
+  harnessMarkRuntimeReceiptsRunning,
   harnessReleaseRuntimeReceipt,
 } from '../../../../sdk/src/plays/harness-stub';
 import type { PreloadedRuntimeDbSessionInput } from '../../../play-harness-worker/src/rpc-types';
@@ -124,6 +125,21 @@ export function createHarnessWorkerReceiptStore(input: {
         ...command,
       });
     },
+    markReceiptsRunning(command) {
+      return harnessMarkRuntimeReceiptsRunning({
+        executorToken: input.executorToken,
+        orgId: input.orgId,
+        preloadedDbSessions: input.preloadedDbSessions ?? null,
+        userEmail: input.userEmail ?? null,
+        runtimeTestFaultHeader: input.runtimeTestFaultHeader ?? null,
+        ...leaseTtlField,
+        ...command,
+        receipts: command.receipts.map((receipt) => ({
+          ...runAttemptField,
+          ...receipt,
+        })),
+      });
+    },
     completeReceipt(command) {
       return harnessCompleteRuntimeReceipt({
         executorToken: input.executorToken,

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts

@@ -1,6 +1,7 @@
 import {
   buildScopedWorkReceiptKey,
   type WorkReceipt,
+  type WorkReceiptBatchMarkRunningCommand,
   type WorkReceiptClaim,
   type WorkReceiptCommand,
   type WorkReceiptFailureKind,
@@ -41,7 +42,14 @@ export type WorkerRuntimeReceiptStore = Pick<
       | 'releaseReceipt'
       | 'heartbeatReceipts'
     >
-  >;
+  > & {
+    markReceiptsRunning?: (
+      input: WorkReceiptBatchMarkRunningCommand,
+    ) => Promise<Array<WorkReceipt | null>>;
+  };
+
+export type WorkerRuntimeReceiptMarkRunningCommand =
+  WorkReceiptBatchMarkRunningCommand;
 
 type RuntimeReceiptContext = {
   orgId?: string | null;
@@ -57,6 +65,11 @@ export type WorkerRuntimeReceiptExecutionContext = {
   leaseId: string | null;
 };
 
+type WorkerRuntimeReceiptExecute<T> = (
+  context: WorkerRuntimeReceiptExecutionContext,
+  wasFailed: boolean,
+) => Promise<T> | T;
+
 const WORKER_RECEIPT_DEFAULT_WAIT_MS = 300_000;
 const WORKER_RECEIPT_WAIT_DELAY_MS = 250;
 const WORKER_RECEIPT_WAIT_MAX_ATTEMPTS = Math.ceil(
@@ -194,6 +207,21 @@ function buildLeaseLostError(input: {
   return new RuntimeLeaseLostError(input);
 }
 
+function receiptHeartbeatStillOwnsOrSettled(input: {
+  receipt: WorkerRuntimeReceipt | null | undefined;
+  leaseId: string;
+}): boolean {
+  const { receipt } = input;
+  if (!receipt) return false;
+  if (receipt.status === 'completed' || receipt.status === 'skipped') {
+    return true;
+  }
+  if (receipt.status !== 'running' && receipt.status !== 'queued') {
+    return false;
+  }
+  return receipt.leaseId === input.leaseId;
+}
+
 export async function executeWithWorkerReceiptHeartbeats<T>(input: {
   playName: string;
   runId: string;
@@ -244,9 +272,10 @@ export async function executeWithWorkerReceiptHeartbeats<T>(input: {
       for (let index = 0; index < group.keys.length; index += 1) {
         const receipt = receipts[index];
         if (
-          !receipt ||
-          receipt.status !== 'running' ||
-          receipt.leaseId !== group.leaseId
+          !receiptHeartbeatStillOwnsOrSettled({
+            receipt,
+            leaseId: group.leaseId,
+          })
         ) {
           throw buildLeaseLostError({
             keys: group.keys,
@@ -316,7 +345,8 @@ async function executeAndPersistReceipt<T>(input: {
   runId: string;
   runAttempt?: number | null;
   leaseId?: string | null;
-  execute: (context: WorkerRuntimeReceiptExecutionContext) => Promise<T> | T;
+  wasFailed?: boolean;
+  execute: WorkerRuntimeReceiptExecute<T>;
   receiptStore: WorkerRuntimeReceiptStore;
   ownership: 'claimed' | 'reconciled';
   budgetMeter?: WorkerWorkBudgetMeter;
@@ -334,9 +364,7 @@ async function executeAndPersistReceipt<T>(input: {
       heartbeatIntervalMs: input.heartbeatIntervalMs,
       targets: [{ key: input.key, leaseId }],
       execute: () =>
-        input.execute({
-          leaseId,
-        }),
+        input.execute({ leaseId }, input.wasFailed === true),
     });
   } catch (error) {
     if (error instanceof RuntimeLeaseLostError) {
@@ -406,7 +434,7 @@ async function executeAndPersistReceipt<T>(input: {
 
 export async function runWorkerRuntimeReceiptBoundary<T>(
   input: RuntimeReceiptContext & {
-    execute: (context: WorkerRuntimeReceiptExecutionContext) => Promise<T> | T;
+    execute: WorkerRuntimeReceiptExecute<T>;
     repairRunningReceiptForSameRun?: boolean;
     repairRunningReceiptForSameRunAfterWaitTimeout?: boolean;
     runningReceiptWaitMaxAttempts?: number;
@@ -414,6 +442,7 @@ export async function runWorkerRuntimeReceiptBoundary<T>(
     runtimeReceiptHeartbeatIntervalMs?: number;
     runtimeReceiptLeaseTtlMs?: number;
     reclaimRunning?: boolean;
+    forceFailedRefresh?: boolean;
   },
 ): Promise<T> {
   const key = scopedReceiptKey(input);
@@ -426,6 +455,7 @@ export async function runWorkerRuntimeReceiptBoundary<T>(
     leaseAware: true,
     leaseTtlMs: input.runtimeReceiptLeaseTtlMs,
     ...(input.reclaimRunning === true ? { reclaimRunning: true } : {}),
+    ...(input.forceFailedRefresh === true ? { forceFailedRefresh: true } : {}),
   });
   if (claimed.disposition === 'reused') {
     return receiptOutput<T>(claimed.receipt);
@@ -540,6 +570,7 @@ export async function runWorkerRuntimeReceiptBoundary<T>(
     runId: input.runId,
     ...(input.runAttempt != null ? { runAttempt: input.runAttempt } : {}),
     leaseId: claimed.receipt.leaseId ?? null,
+    wasFailed: claimed.wasFailed === true,
     execute: input.execute,
     receiptStore,
     ownership: 'claimed',

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts

@@ -15,10 +15,10 @@ import {
 import {
   RuntimeReceiptWaitTimeoutError,
   resolveRuntimeToolReceiptWaitMaxAttempts,
+  runtimeReceiptFailureKindForError,
   waitForCompletedRuntimeReceipt,
 } from '../../../../shared_libs/play-runtime/durable-receipt-execution';
 import type { PlayExecutionGovernor } from '../../../../shared_libs/play-runtime/governor/governor';
-import { getToolHttpErrorReceiptFailureKind } from '../../../../shared_libs/play-runtime/tool-http-errors';
 import { ToolExecuteAuthScopeChangedError } from '../../../../shared_libs/play-runtime/tool-execute-retry-policy';
 import { getPlayRuntimeBatchStrategy } from '../../../../shared_libs/play-runtime/play-runtime-batching-registry';
 import {
@@ -46,10 +46,7 @@ import {
   type ToolResultMetadataInput,
 } from '../../../../shared_libs/play-runtime/tool-result';
 import { toolExecutionMetadataForOutcome } from '../../../../shared_libs/play-runtime/tool-execution-outcome';
-import {
-  buildScopedWorkReceiptKey,
-  type WorkReceiptFailureKind,
-} from '../../../../shared_libs/play-runtime/work-receipts';
+import { buildScopedWorkReceiptKey } from '../../../../shared_libs/play-runtime/work-receipts';
 import {
   canReclaimFailedWorkerToolReceipt,
   canReclaimTimedOutWorkerToolReceipt,
@@ -213,6 +210,7 @@ type ClaimedWorkerToolBatchRequest = {
   request: WorkerToolBatchRequest;
   receiptKey: string | null;
   receiptLeaseId: string | null;
+  receiptStatus: WorkerRuntimeReceipt['status'] | null;
   providerForce?: boolean;
   followers: WorkerToolBatchRequest[];
 };
@@ -281,18 +279,6 @@ async function sleepWorkerMs(ms: number): Promise<void> {
   await new Promise((resolve) => setTimeout(resolve, ms));
 }
 
-function workReceiptFailureKindForError(
-  error: unknown,
-): WorkReceiptFailureKind {
-  if (
-    isPlayExecutionSuspendedError(error) ||
-    isPlayRowExecutionSuspendedError(error)
-  ) {
-    return 'repairable';
-  }
-  return getToolHttpErrorReceiptFailureKind(error) ?? 'terminal';
-}
-
 export class RuntimeReceiptPersistenceError extends Error {
   readonly receiptKeys: string[];
   readonly errors: unknown[];
@@ -312,7 +298,7 @@ export class RuntimeReceiptPersistenceError extends Error {
 }
 
 function receiptPersistenceError(input: {
-  operation: 'complete' | 'fail';
+  operation: 'mark-running' | 'complete' | 'fail';
   receiptKeys: string[];
   cause: unknown;
 }): RuntimeReceiptPersistenceError {
@@ -434,7 +420,18 @@ function toolMetadataFallback(toolId: string): ToolResultMetadataInput {
   return { toolId };
 }
 
-function workerDurableToolCallCacheKey(input: {
+function durableReceiptInputForTool(
+  toolId: string,
+  input: Record<string, unknown>,
+): Record<string, unknown> {
+  if (toolId !== 'test_rate_limit') {
+    return input;
+  }
+  const { simulated_delay_ms: _simulatedDelayMs, ...semanticInput } = input;
+  return semanticInput;
+}
+
+export function workerDurableToolCallCacheKey(input: {
   req: WorkerToolDispatchRunRequest;
   toolId: string;
   requestInput: Record<string, unknown>;
@@ -446,7 +443,7 @@ function workerDurableToolCallCacheKey(input: {
     orgId: input.req.orgId,
     playLocalScope: input.req.playName,
     toolId: input.toolId,
-    requestInput: input.requestInput,
+    requestInput: durableReceiptInputForTool(input.toolId, input.requestInput),
     authScopeDigest: buildDurableToolCallAuthScopeDigest({
       orgId: input.req.orgId,
       toolId: input.toolId,
@@ -928,6 +925,7 @@ export class WorkerToolBatchScheduler {
           request,
           receiptKey: input.receiptKey,
           receiptLeaseId: claim.receipt.leaseId ?? null,
+          receiptStatus: claim.receipt.status,
           providerForce: true,
           followers,
         },
@@ -995,6 +993,7 @@ export class WorkerToolBatchScheduler {
           request,
           receiptKey: input.receiptKey,
           receiptLeaseId: claim.receipt.leaseId ?? null,
+          receiptStatus: claim.receipt.status,
           providerForce: true,
           followers,
         },
@@ -1136,6 +1135,7 @@ export class WorkerToolBatchScheduler {
           request,
           receiptKey: null,
           receiptLeaseId: null,
+          receiptStatus: null,
           followers: [],
         })),
         deferredClaimedRequests: [],
@@ -1159,6 +1159,7 @@ export class WorkerToolBatchScheduler {
         request,
         receiptKey: null,
         receiptLeaseId: null,
+        receiptStatus: null,
         followers: [],
       });
     }
@@ -1277,6 +1278,7 @@ export class WorkerToolBatchScheduler {
           request,
           receiptKey,
           receiptLeaseId: claim.receipt.leaseId ?? null,
+          receiptStatus: claim.receipt.status,
           providerForce: groupState.forceDurableRefresh,
           followers,
         });
@@ -1340,6 +1342,77 @@ export class WorkerToolBatchScheduler {
     throw completedResult.error;
   }
 
+  private async markAdmittedToolRequestsRunning(
+    claimedRequests: ClaimedWorkerToolBatchRequest[],
+  ): Promise<void> {
+    const queued = claimedRequests.filter(
+      (claimed) =>
+        claimed.receiptKey !== null && claimed.receiptStatus !== 'running',
+    );
+    if (queued.length === 0) return;
+
+    const markReceiptsRunning = this.options.receiptStore?.markReceiptsRunning;
+    const receiptKeys = queued.map((claimed) => claimed.receiptKey!);
+    if (!markReceiptsRunning) {
+      throw receiptPersistenceError({
+        operation: 'mark-running',
+        receiptKeys,
+        cause: new Error(
+          'Queued runtime receipts require a fenced mark-running store operation.',
+        ),
+      });
+    }
+
+    const runId = this.options.req.runId;
+    const runAttempt = this.options.req.runAttempt ?? 0;
+    this.options.budgetMeter?.count('receipt');
+    let marked: Array<WorkerRuntimeReceipt | null>;
+    try {
+      marked = await markReceiptsRunning({
+        playName: this.options.req.playName,
+        leaseTtlMs: this.options.runtimeReceiptLeaseTtlMs,
+        receipts: queued.map((claimed) => ({
+          runId,
+          runAttempt,
+          key: claimed.receiptKey!,
+          leaseId: claimed.receiptLeaseId,
+        })),
+      });
+    } catch (error) {
+      throw receiptPersistenceError({
+        operation: 'mark-running',
+        receiptKeys,
+        cause: error,
+      });
+    }
+
+    for (let index = 0; index < queued.length; index += 1) {
+      const claimed = queued[index]!;
+      const receipt = marked[index];
+      const leaseId = claimed.receiptLeaseId?.trim() || null;
+      const ownerRunId = receipt?.leaseOwnerRunId ?? receipt?.runId ?? null;
+      if (
+        !receipt ||
+        receipt.key !== claimed.receiptKey ||
+        receipt.status !== 'running' ||
+        !leaseId ||
+        receipt.leaseId !== leaseId ||
+        ownerRunId !== runId ||
+        receipt.leaseOwnerAttempt !== runAttempt
+      ) {
+        throw new RuntimeLeaseLostError({
+          keys: [claimed.receiptKey!],
+          runId,
+          leaseId: leaseId ?? 'missing',
+        });
+      }
+    }
+
+    for (const claimed of queued) {
+      claimed.receiptStatus = 'running';
+    }
+  }
+
   private completedDurableToolRequestResult(input: {
     claimed: ClaimedWorkerToolBatchRequest;
     ownerResult: unknown;
@@ -1380,14 +1453,8 @@ export class WorkerToolBatchScheduler {
       };
     }
     return {
-      ok: false,
-      error: receiptPersistenceError({
-        operation: 'complete',
-        receiptKeys: [claimed.receiptKey ?? claimed.request.cacheKey],
-        cause: new Error(
-          `Durable tool call ${claimed.receiptKey ?? claimed.request.cacheKey} lost receipt ownership before completion.`,
-        ),
-      }),
+      ok: true,
+      value: ownerResult,
     };
   }
 
@@ -1803,7 +1870,7 @@ export class WorkerToolBatchScheduler {
         key: claimed.receiptKey,
         leaseId: claimed.receiptLeaseId,
         error: error instanceof Error ? error.message : String(error),
-        failureKind: workReceiptFailureKindForError(error),
+        failureKind: runtimeReceiptFailureKindForError(error),
       });
     } catch (receiptError) {
       if (this.options.persistenceLatch) {
@@ -1900,7 +1967,7 @@ export class WorkerToolBatchScheduler {
           key: claimed.receiptKey!,
           leaseId: claimed.receiptLeaseId,
           error: error instanceof Error ? error.message : String(error),
-          failureKind: workReceiptFailureKindForError(error),
+          failureKind: runtimeReceiptFailureKindForError(error),
         })),
       });
     } catch (receiptError) {
@@ -1991,17 +2058,18 @@ export class WorkerToolBatchScheduler {
             );
             return;
           }
-          this.options.budgetMeter?.count('provider');
-          this.options.budgetMeter?.count(
-            'egress',
-            Math.max(
-              0,
-              WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL - 1,
-            ),
-          );
           try {
             let result: unknown;
             try {
+              await this.markAdmittedToolRequestsRunning([claimed]);
+              this.options.budgetMeter?.count('provider');
+              this.options.budgetMeter?.count(
+                'egress',
+                Math.max(
+                  0,
+                  WORKER_PLATFORM_SUBREQUESTS_PER_UNBATCHED_TOOL_CALL - 1,
+                ),
+              );
               const durableReceiptKey = claimed.receiptKey
                 ? durableCallReceiptKeyForClaimed({
                     req: this.options.req,
@@ -2176,8 +2244,9 @@ export class WorkerToolBatchScheduler {
             signal: this.options.abortSignal,
           },
         );
-        this.options.budgetMeter?.count('provider');
         try {
+          await this.markAdmittedToolRequestsRunning(batch.memberRequests);
+          this.options.budgetMeter?.count('provider');
           const receiptKeys = batch.memberRequests.map((member) =>
             durableCallReceiptKeyForClaimed({
               req: this.options.req,

package/dist/bundling-sources/sdk/src/http.ts

@@ -9,7 +9,7 @@
  * - **Max attempts**: `1 + maxRetries` (default: 4 total attempts)
  * - **Backoff**: exponential — 1s, 2s, 4s, 8s, ... capped at 30s
  * - **Retryable**: network errors, timeouts, HTTP 429 (rate limit)
- * - **Not retryable**: HTTP 401/403 (auth errors), other 4xx/5xx API errors
+ * - **Not retryable**: HTTP 401/403 (auth errors), other 4xx API errors
  *
  * ## Localhost failover
  *
@@ -504,7 +504,10 @@ export class HttpClient {
    * @param path - API path (e.g. `"/api/v2/tools"`)
    */
   async get<T = unknown>(path: string): Promise<T> {
-    return this.request<T>(path, { method: 'GET' });
+    return this.request<T>(path, {
+      method: 'GET',
+      retryApiErrors: true,
+    });
   }
 
   async *streamSse<TEvent extends LiveEventEnvelope = LiveEventEnvelope>(

package/dist/bundling-sources/sdk/src/play.ts

@@ -936,7 +936,7 @@ export interface DeeplinePlayRuntimeContext {
  *
  * // Poll status
  * const status = await job.status();
- * console.log(status.temporalStatus); // 'RUNNING'
+ * console.log(status.status); // 'running'
  *
  * // Stream logs until completion
  * const finalStatus = await job.tail({

package/dist/bundling-sources/sdk/src/plays/harness-stub.ts

@@ -37,6 +37,7 @@ import type {
   FailRuntimeReceiptInput,
   FailRuntimeReceiptsInput,
   HeartbeatRuntimeReceiptsInput,
+  MarkRuntimeReceiptsRunningInput,
   ReleaseRuntimeReceiptInput,
   RuntimeApiCallInput,
   RuntimeApiCallResult,
@@ -157,6 +158,12 @@ export async function harnessGetRuntimeReceipt(
   return requireBinding().getRuntimeReceipt(input);
 }
 
+export async function harnessMarkRuntimeReceiptsRunning(
+  input: MarkRuntimeReceiptsRunningInput,
+): Promise<Array<WorkReceipt | null>> {
+  return requireBinding().markRuntimeReceiptsRunning(input);
+}
+
 export async function harnessCompleteRuntimeReceipt(
   input: CompleteRuntimeReceiptInput,
 ): Promise<WorkReceipt | null> {