deepline 0.2.8 → 0.2.10

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -530,6 +530,8 @@ export interface ContextOptions {
   vercelProtectionBypassToken?: string | null;
   /** Optional per-run integration execution mode for provider calls. */
   integrationMode?: 'live' | 'eval_stub' | 'fixture';
+  /** Preview/dev test seam that applies provider pacing to fixture responses. */
+  enforceFixtureProviderPacing?: boolean;
   orgId?: string;
   userEmail?: string;
   playName?: string;

package/dist/bundling-sources/shared_libs/play-runtime/db-session.ts

@@ -2,6 +2,7 @@ import type {
   EncryptedPostgresUrl,
   PostgresUrlEncryptionRequest,
 } from './db-session-crypto';
+import { WORKFLOW_EXECUTOR_TOKEN_TTL_SECONDS } from './runtime-constants';
 
 // Workflow DB sessions must not expire before the workflow run/retry state they
 // pair with. Run/retry state lives 60 min (WORKFLOW_RUN_STATE_TTL_MS in the
@@ -11,6 +12,13 @@ import type {
 export const DB_SESSION_DEFAULT_TTL_SECONDS = 60 * 60;
 export const DB_SESSION_MAX_TTL_SECONDS = 60 * 60;
 
+// A sandbox cannot mint new database authority after launch. Its preloaded
+// sessions therefore have to remain valid for the same bounded activity as
+// the executor token that unwraps them. Keep the shorter default/max above for
+// renewable control-plane sessions; they do not govern launch-time preloads.
+export const PRELOADED_RUNTIME_DB_SESSION_TTL_SECONDS =
+  WORKFLOW_EXECUTOR_TOKEN_TTL_SECONDS;
+
 export const DB_SESSION_OPERATIONS = [
   'rows.read',
   'rows.append',

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -122,6 +122,8 @@ export interface PlayRunnerContextConfig {
   runtimeTestFaultHeader?: string | null;
   vercelProtectionBypassToken?: string | null;
   integrationMode?: 'live' | 'eval_stub' | 'fixture';
+  /** Preview/dev test seam that applies provider pacing to fixture responses. */
+  enforceFixtureProviderPacing?: boolean;
   /** Immutable tool-error payload schema copied from the run contract. */
   toolErrorSchemaVersion?: ToolExecutionErrorSchemaVersion;
   orgId?: string;

package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts

@@ -178,6 +178,40 @@ export type PlayRunFailureDetails = {
   causes?: string[];
 };
 
+function formatRuntimeLimitDuration(timeoutSeconds: number): string {
+  if (timeoutSeconds % 3_600 === 0) {
+    const hours = timeoutSeconds / 3_600;
+    return `${hours} ${hours === 1 ? 'hour' : 'hours'}`;
+  }
+  if (timeoutSeconds % 60 === 0) {
+    const minutes = timeoutSeconds / 60;
+    return `${minutes} ${minutes === 1 ? 'minute' : 'minutes'}`;
+  }
+  return `${timeoutSeconds} ${timeoutSeconds === 1 ? 'second' : 'seconds'}`;
+}
+
+export function runtimeLimitExceededCause(timeoutSeconds: number): string {
+  return `RUNTIME_LIMIT_EXCEEDED: Play exceeded its configured ${timeoutSeconds} second runtime limit and was stopped.`;
+}
+
+export function runtimeLimitExceededMessage(timeoutSeconds: number): string {
+  if (timeoutSeconds === 30 * 60) return RUNTIME_LIMIT_EXCEEDED_MESSAGE;
+  return `The play reached its configured runtime limit of ${formatRuntimeLimitDuration(timeoutSeconds)} and was stopped. Completed row state was preserved; run a smaller batch or continue from the persisted rows.`;
+}
+
+export function runtimeLimitExceededFailure(
+  timeoutSeconds: number,
+  cause = runtimeLimitExceededCause(timeoutSeconds),
+): PlayRunFailureDetails {
+  return {
+    code: 'RUNTIME_LIMIT_EXCEEDED',
+    phase: 'runtime',
+    message: runtimeLimitExceededMessage(timeoutSeconds),
+    retryable: false,
+    cause: boundedFailureText(cause),
+  };
+}
+
 const PUBLIC_FAILURE_STACK_LINE_LIMIT = 12;
 const PUBLIC_FAILURE_TEXT_BYTE_LIMIT = 64 * 1024;
 const PUBLIC_FAILURE_CAUSE_LIMIT = 8;
@@ -261,13 +295,12 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
     };
   }
   if (RUNTIME_LIMIT_EXCEEDED_RE.test(rawCause)) {
-    return {
-      code: 'RUNTIME_LIMIT_EXCEEDED',
-      phase: 'runtime',
-      message: RUNTIME_LIMIT_EXCEEDED_MESSAGE,
-      retryable: false,
-      cause,
-    };
+    const configuredSeconds = Number(
+      /configured\s+(\d+)\s+second runtime limit/i.exec(rawCause)?.[1],
+    );
+    return Number.isSafeInteger(configuredSeconds) && configuredSeconds > 0
+      ? runtimeLimitExceededFailure(configuredSeconds, cause)
+      : runtimeLimitExceededFailure(30 * 60, cause);
   }
   if (RUNTIME_RUNNER_LOST_RE.test(rawCause)) {
     const stack = error instanceof Error ? boundedFailureStack(error) : null;

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts

@@ -14,6 +14,10 @@ import {
   PLAY_RUNNER_STARTUP_GRACE_SECONDS,
   STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
 } from '@shared_libs/play-runtime/runtime-constants';
+import {
+  STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
+  validatePlaySandboxRuntimeLimits,
+} from '@shared_libs/play-runtime/sandbox-runtime-limits';
 import {
   RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES,
   RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES,
@@ -508,9 +512,11 @@ export async function stageRunnerPayload(input: {
     materializedFiles: remoteMaterializedFiles,
     context: remoteRuntimeContextForDaytona(input.config.context),
   };
-  const runtimeLimitSeconds =
-    input.config.context.sandboxRuntimeLimits?.timeoutSeconds ??
-    STANDARD_PLAY_RUNTIME_LIMIT_SECONDS;
+  const runtimeLimitSeconds = validatePlaySandboxRuntimeLimits(
+    input.config.context.sandboxRuntimeLimits ?? {
+      ...STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
+    },
+  ).timeoutSeconds;
 
   const envelopeUpload = input.bundlePromise.then((bundle) =>
     timedRunnerPayloadUpload({

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts

@@ -18,6 +18,7 @@ const limitMs = Math.max(1, Number.parseInt(process.argv[5] || '', 10) || 1);
 const startupLimitMs = Math.max(1, Number.parseInt(process.argv[6] || '', 10) || 1);
 const terminalGraceMs = Math.max(1, Number.parseInt(process.argv[7] || '', 10) || 1);
 const runtimeLimitMarkerPath = process.argv[8] || '';
+const runtimeLimitSeconds = Math.max(1, Math.ceil(limitMs / 1_000));
 const child = spawn(process.execPath, [runnerPath, configPath], {
   env: process.env,
   stdio: 'inherit',
@@ -54,7 +55,11 @@ function armRuntimeDeadline() {
     }
     timedOut = true;
     markRuntimeLimit();
-    process.stderr.write('RUNTIME_LIMIT_EXCEEDED: external runtime watchdog stopped the play.\\n');
+    process.stderr.write(
+      'RUNTIME_LIMIT_EXCEEDED: Play exceeded its configured ' +
+        runtimeLimitSeconds +
+        ' second runtime limit and was stopped by the external watchdog.\\n',
+    );
     stopChild();
   }, limitMs);
 }
@@ -113,5 +118,7 @@ function finish(code) {
 
 child.once('error', () => finish(1));
 child.once('exit', (code) => finish(code));
+process.once('SIGTERM', stopChild);
+process.once('SIGINT', stopChild);
 `;
 }

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -15,7 +15,6 @@ import type {
 } from '@shared_libs/play-runtime/protocol';
 import {
   PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
-  STANDARD_PLAY_RUNTIME_LIMIT_LABEL,
   STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
 } from '@shared_libs/play-runtime/runtime-constants';
 import {
@@ -39,10 +38,6 @@ import {
   prepareDetachedDaytonaRunner,
 } from './daytona-session-execution';
 
-const DAYTONA_EXECUTE_TIMEOUT_SECONDS = STANDARD_PLAY_RUNTIME_LIMIT_SECONDS;
-const DAYTONA_DETACHED_CEILING_SECONDS =
-  STANDARD_PLAY_RUNTIME_LIMIT_SECONDS + PLAY_RUNNER_TERMINAL_GRACE_SECONDS;
-const STANDARD_WORKFLOW_RUNTIME_LIMIT_ERROR = `Based on this plan, max runtime is ${STANDARD_PLAY_RUNTIME_LIMIT_LABEL}. Use smaller batches; ask for runtime.`;
 const DAYTONA_COMMAND_RECOVERY_TIMEOUT_MS = 60_000;
 const DAYTONA_COMMAND_RECOVERY_POLL_MS = 2_000;
 const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
@@ -459,13 +454,6 @@ export async function deleteDaytonaSandboxById(input: {
   return outcome.kind === 'deleted' || outcome.kind === 'already_absent';
 }
 
-function formatDaytonaExecutionError(error: unknown): string {
-  const message = formatDaytonaError(error);
-  return isConfiguredDaytonaRuntimeLimit(error)
-    ? STANDARD_WORKFLOW_RUNTIME_LIMIT_ERROR
-    : message;
-}
-
 function formatDaytonaError(error: unknown): string {
   if (!(error instanceof Error)) {
     return String(error);
@@ -483,14 +471,6 @@ function formatDaytonaError(error: unknown): string {
   return `${details || 'Error'}${code}${cause}`;
 }
 
-function isConfiguredDaytonaRuntimeLimit(error: unknown): boolean {
-  const message = error instanceof Error ? error.message : String(error);
-  return new RegExp(
-    `(?:execution|command).{0,80}(?:timed out|timeout).{0,80}${DAYTONA_EXECUTE_TIMEOUT_SECONDS}|${DAYTONA_EXECUTE_TIMEOUT_SECONDS}.{0,80}(?:timed out|timeout)`,
-    'i',
-  ).test(message);
-}
-
 function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -1070,6 +1050,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
               runtimeCompletedPath: stagedPayload.runtimeCompletedPath,
               startedAtMs: Date.now(),
               heartbeatTimeoutMs: push.leaseSeconds * 1_000,
+              runtimeLimitSeconds: executionTimeoutSeconds,
               ceilingMs:
                 (executionTimeoutSeconds + PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
                 1_000,
@@ -1164,7 +1145,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
       return withCleanup(
         createDaytonaFailedResult({
           config,
-          error: formatDaytonaExecutionError(error),
+          error: formatDaytonaError(error),
           runtimeTiming,
         }),
       );

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts

@@ -1,6 +1,6 @@
 import { randomUUID } from 'node:crypto';
 import { spawn } from 'node:child_process';
-import { cp, mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises';
+import { cp, mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import type { SpawnOptionsWithoutStdio } from 'node:child_process';
 import {
@@ -20,7 +20,22 @@ import {
   recordComputeBillingItemViaAppRuntime,
   type WorkerRuntimeApiContext,
 } from '@shared_libs/play-runtime/app-runtime-api';
-import { STANDARD_PLAY_RUNTIME_LIMIT_SECONDS } from '@shared_libs/play-runtime/runtime-constants';
+import {
+  PLAY_RUNNER_STARTUP_GRACE_SECONDS,
+  PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
+} from '@shared_libs/play-runtime/runtime-constants';
+import {
+  STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
+  validatePlaySandboxRuntimeLimits,
+} from '@shared_libs/play-runtime/sandbox-runtime-limits';
+import {
+  runtimeLimitExceededCause,
+  runtimeLimitExceededFailure,
+} from '@shared_libs/play-runtime/run-failure';
+import {
+  buildDaytonaRuntimeWatchdogSource,
+  DAYTONA_RUNTIME_WATCHDOG_EXIT_CODE,
+} from './daytona-runtime-watchdog';
 
 interface CommandExecutionResult {
   exitCode: number;
@@ -41,9 +56,6 @@ const LOCAL_PROCESS_COMPUTE_MEMORY_GIB = 1;
 const LOCAL_PROCESS_COMPUTE_DISK_GIB = 3;
 export const LOCAL_PROCESS_CANCEL_GRACE_MS = 2_000;
 const LOCAL_PROCESS_CANCELLED_ERROR = 'Local play runner cancelled';
-function localProcessRuntimeLimitError(runtimeLimitSeconds: number): string {
-  return `Configured max runtime is ${runtimeLimitSeconds} seconds. Use smaller batches or increase runtime.timeout.`;
-}
 
 export const LOCAL_PROCESS_COMPUTE_PROFILE = {
   source: LOCAL_PROCESS_COMPUTE_SOURCE,
@@ -194,55 +206,11 @@ async function runCommand(
 export function resolveLocalProcessRuntimeLimitSeconds(
   context: PlayRunnerExecutionConfig['context'],
 ): number {
-  return (
-    context.sandboxRuntimeLimits?.timeoutSeconds ??
-    STANDARD_PLAY_RUNTIME_LIMIT_SECONDS
-  );
-}
-
-async function runCommandWithRuntimeLimit(
-  command: string,
-  args: string[],
-  runtimeLimitSeconds: number,
-  options?: SpawnOptionsWithoutStdio,
-  callbacks?: StreamedCommandCallbacks,
-): Promise<CommandExecutionResult> {
-  const runtimeLimitController = new AbortController();
-  const runtimeDeadlineTimer = setTimeout(() => {
-    runtimeLimitController.abort(
-      localProcessRuntimeLimitError(runtimeLimitSeconds),
-    );
-  }, runtimeLimitSeconds * 1000);
-  runtimeDeadlineTimer.unref?.();
-
-  const onExternalCancel = () => {
-    if (!runtimeLimitController.signal.aborted) {
-      runtimeLimitController.abort(
-        callbacks?.cancellationSignal?.reason ?? LOCAL_PROCESS_CANCELLED_ERROR,
-      );
-    }
-  };
-
-  if (callbacks?.cancellationSignal?.aborted) {
-    onExternalCancel();
-  } else {
-    callbacks?.cancellationSignal?.addEventListener('abort', onExternalCancel, {
-      once: true,
-    });
-  }
-
-  try {
-    return await runCommand(command, args, options, {
-      ...callbacks,
-      cancellationSignal: runtimeLimitController.signal,
-    });
-  } finally {
-    clearTimeout(runtimeDeadlineTimer);
-    callbacks?.cancellationSignal?.removeEventListener(
-      'abort',
-      onExternalCancel,
-    );
-  }
+  return validatePlaySandboxRuntimeLimits(
+    context.sandboxRuntimeLimits ?? {
+      ...STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
+    },
+  ).timeoutSeconds;
 }
 
 function createFailedResult(config: PlayRunnerExecutionConfig, error: string) {
@@ -424,10 +392,34 @@ export const localProcessPlayRunnerBackend: PlayRunnerBackend = {
               },
             },
             async (span) => {
-              const result = await runCommandWithRuntimeLimit(
+              const runtimeLimitSeconds =
+                resolveLocalProcessRuntimeLimitSeconds(config.context);
+              const runtimeStartedPath = join(
+                workspaceRoot,
+                'runtime-started.json',
+              );
+              const runtimeCompletedPath = join(
+                workspaceRoot,
+                'runtime-completed.json',
+              );
+              const runtimeLimitMarkerPath = join(
+                workspaceRoot,
+                'runtime-limit.marker',
+              );
+              const result = await runCommand(
                 'node',
-                [runnerPath, configPath],
-                resolveLocalProcessRuntimeLimitSeconds(config.context),
+                [
+                  '-e',
+                  buildDaytonaRuntimeWatchdogSource(),
+                  runnerPath,
+                  configPath,
+                  runtimeStartedPath,
+                  runtimeCompletedPath,
+                  String(runtimeLimitSeconds * 1_000),
+                  String(PLAY_RUNNER_STARTUP_GRACE_SECONDS * 1_000),
+                  String(PLAY_RUNNER_TERMINAL_GRACE_SECONDS * 1_000),
+                  runtimeLimitMarkerPath,
+                ],
                 {
                   cwd: workspaceRoot,
                   env: {
@@ -436,6 +428,10 @@ export const localProcessPlayRunnerBackend: PlayRunnerBackend = {
                     TMPDIR: workspaceRoot,
                     TEMP: workspaceRoot,
                     TMP: workspaceRoot,
+                    DEEPLINE_PLAY_RUNNER_RUNTIME_STARTED_PATH:
+                      runtimeStartedPath,
+                    DEEPLINE_PLAY_RUNNER_RUNTIME_COMPLETED_PATH:
+                      runtimeCompletedPath,
                   },
                 },
                 {
@@ -463,12 +459,18 @@ export const localProcessPlayRunnerBackend: PlayRunnerBackend = {
                   },
                 },
               );
+              const runtimeLimitReached =
+                result.exitCode === DAYTONA_RUNTIME_WATCHDOG_EXIT_CODE &&
+                (await readFile(runtimeLimitMarkerPath, 'utf-8')
+                  .then((value) => value.trim() === 'runtime_limit')
+                  .catch(() => false));
               setSpanAttributes(span, {
                 'plays.child_exit_code': result.exitCode,
                 'plays.stdout_bytes': result.stdout.length,
                 'plays.stderr_bytes': result.stderr.length,
+                'plays.runtime_limit_reached': runtimeLimitReached,
               });
-              return result;
+              return { ...result, runtimeLimitReached, runtimeLimitSeconds };
             },
           );
           logLocalRunnerPerf(
@@ -514,6 +516,21 @@ export const localProcessPlayRunnerBackend: PlayRunnerBackend = {
             return result;
           }
 
+          if (execution.runtimeLimitReached) {
+            const error = runtimeLimitExceededCause(
+              execution.runtimeLimitSeconds,
+            );
+            return {
+              ...createFailedResult(config, error),
+              errors: [
+                runtimeLimitExceededFailure(
+                  execution.runtimeLimitSeconds,
+                  error,
+                ),
+              ],
+            };
+          }
+
           return createFailedResult(
             config,
             execution.exitCode !== 0

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts

@@ -17,6 +17,7 @@ import {
   loadModalClientConfig,
   loadModalRequiredConfig,
 } from '@shared_libs/play-runtime/modal-runtime-config';
+import type { PlaySandboxRuntimeLimits } from '@shared_libs/play-runtime/sandbox-runtime-limits';
 import { validateDaytonaExecutionContext } from './daytona-lifecycle';
 import { stageRunnerPayload } from './daytona-payload-transport';
 import { captureDetachedDaytonaRunnerReadinessBaseline } from './daytona-session-execution';
@@ -24,6 +25,23 @@ import { captureDetachedDaytonaRunnerReadinessBaseline } from './daytona-session
 const MODAL_RUNNER_READY_TIMEOUT_MS = 30_000;
 type ModalSandbox = Awaited<ReturnType<ModalClient['sandboxes']['create']>>;
 
+export function modalSandboxLifetimeMs(
+  limits: PlaySandboxRuntimeLimits,
+): number {
+  return (
+    (limits.timeoutSeconds +
+      PLAY_RUNNER_STARTUP_GRACE_SECONDS +
+      PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
+    1_000
+  );
+}
+
+export function modalDetachedRunnerCeilingMs(
+  limits: PlaySandboxRuntimeLimits,
+): number {
+  return (limits.timeoutSeconds + PLAY_RUNNER_TERMINAL_GRACE_SECONDS) * 1_000;
+}
+
 function failed(
   config: PlayRunnerExecutionConfig,
   error: unknown,
@@ -126,16 +144,8 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
         cpuLimit: MODAL_SANDBOX_CPU_CORES,
         memoryMiB: MODAL_SANDBOX_MEMORY_MIB,
         memoryLimitMiB: MODAL_SANDBOX_MEMORY_MIB,
-        timeoutMs:
-          (modalConfig.limits.timeoutSeconds +
-            PLAY_RUNNER_STARTUP_GRACE_SECONDS +
-            PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
-          1_000,
-        idleTimeoutMs:
-          (modalConfig.limits.timeoutSeconds +
-            PLAY_RUNNER_STARTUP_GRACE_SECONDS +
-            PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
-          1_000,
+        timeoutMs: modalSandboxLifetimeMs(modalConfig.limits),
+        idleTimeoutMs: modalSandboxLifetimeMs(modalConfig.limits),
         workdir: modalConfig.workdir,
         tags: {
           source: 'deepline-play-runner',
@@ -265,10 +275,8 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
             runtimeCompletedPath: payload.runtimeCompletedPath,
             startedAtMs: Date.now(),
             heartbeatTimeoutMs: push.leaseSeconds * 1_000,
-            ceilingMs:
-              (modalConfig.limits.timeoutSeconds +
-                PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
-              1_000,
+            runtimeLimitSeconds: modalConfig.limits.timeoutSeconds,
+            ceilingMs: modalDetachedRunnerCeilingMs(modalConfig.limits),
           },
           logs: [],
           stats: {},

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -4396,10 +4396,26 @@ export async function claimRuntimeWorkReceipts(
     leaseTtlMs?: number | null;
   },
 ): Promise<WorkReceiptClaim[]> {
-  const keys = [
-    ...new Set(input.keys.map((key) => key.trim()).filter(Boolean)),
-  ];
-  if (keys.length === 0) return [];
+  if (
+    input.leaseIds !== undefined &&
+    input.leaseIds.length !== input.keys.length
+  ) {
+    throw new Error(
+      `Runtime receipt bulk claim requires one lease ID per key. Received ${input.leaseIds.length} lease IDs for ${input.keys.length} keys.`,
+    );
+  }
+  const positionalEntries = input.keys
+    .map((key, originalIndex) => ({ key: key.trim(), originalIndex }))
+    .filter((entry) => Boolean(entry.key));
+  const positionalKeys = positionalEntries.map((entry) => entry.key);
+  if (positionalKeys.length === 0) return [];
+  const firstPositionByKey = new Map<string, number>();
+  positionalEntries.forEach(({ key, originalIndex }) => {
+    if (!firstPositionByKey.has(key)) {
+      firstPositionByKey.set(key, originalIndex);
+    }
+  });
+  const keys = [...firstPositionByKey.keys()];
   const session = await getRuntimeWorkReceiptSessionForKeys(context, {
     playName: input.playName,
     keys,
@@ -4409,16 +4425,9 @@ export async function claimRuntimeWorkReceipts(
     session,
     async (client) => {
       const keyHexes = keys.map(workReceiptKeyHex);
-      if (
-        input.leaseIds !== undefined &&
-        input.leaseIds.length !== keys.length
-      ) {
-        throw new Error(
-          `Runtime receipt bulk claim requires one lease ID per key. Received ${input.leaseIds.length} lease IDs for ${keys.length} keys.`,
-        );
-      }
-      const leaseIds = keys.map((_, index) => {
-        const providedLeaseId = input.leaseIds?.[index]?.trim();
+      const leaseIds = keys.map((key) => {
+        const position = firstPositionByKey.get(key)!;
+        const providedLeaseId = input.leaseIds?.[position]?.trim();
         return (
           providedLeaseId ||
           (input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null)
@@ -4604,19 +4613,40 @@ export async function claimRuntimeWorkReceipts(
           input.reclaimRunning === true,
         ],
       );
-      return rows.map((row) => {
+      const claimsByKey = new Map<string, WorkReceiptClaim>();
+      for (const row of rows) {
         const receipt = mapRuntimeWorkReceiptRow(row);
         if (row.claimed === true) {
-          return { disposition: 'claimed', receipt } satisfies WorkReceiptClaim;
+          claimsByKey.set(receipt.key, {
+            disposition: 'claimed',
+            receipt,
+          });
+          continue;
         }
         if (input.forceRefresh !== true && isReusableWorkReceipt(receipt)) {
-          return { disposition: 'reused', receipt } satisfies WorkReceiptClaim;
+          claimsByKey.set(receipt.key, {
+            disposition: 'reused',
+            receipt,
+          });
+          continue;
         }
-        return runtimeWorkReceiptClaimDispositionForBlockedReceipt({
-          receipt,
-          claimantRunId: input.runId,
-          claimantRunAttempt: runAttempt,
-        });
+        claimsByKey.set(
+          receipt.key,
+          runtimeWorkReceiptClaimDispositionForBlockedReceipt({
+            receipt,
+            claimantRunId: input.runId,
+            claimantRunAttempt: runAttempt,
+          }),
+        );
+      }
+      return positionalKeys.map((key) => {
+        const claim = claimsByKey.get(key);
+        if (!claim) {
+          throw new Error(
+            `Runtime receipt ${key} bulk claim did not return a positional result.`,
+          );
+        }
+        return claim;
       });
     },
   );

package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts

@@ -69,6 +69,21 @@ export class RuntimeReceiptWriterClosedError extends Error {
   }
 }
 
+export class RuntimeReceiptWriterRetryDeadlineError extends Error {
+  constructor(
+    readonly attempts: number,
+    readonly elapsedMs: number,
+    readonly maxRetryElapsedMs: number,
+    options?: { cause?: unknown },
+  ) {
+    super(
+      `Runtime receipt persistence did not recover within ${maxRetryElapsedMs}ms after ${attempts} attempts.`,
+      options,
+    );
+    this.name = 'RuntimeReceiptWriterRetryDeadlineError';
+  }
+}
+
 /**
  * Private batching implementation used by a Play Durability Store Adapter.
  *
@@ -90,6 +105,7 @@ export class RuntimeReceiptWriter<Input, Output> {
   readonly #maxBatchBytes: number;
   readonly #maxBufferedBytes: number;
   readonly #maxFlushMs: number;
+  readonly #maxRetryElapsedMs: number;
   readonly #onRetryEvent:
     | ((event: RuntimeReceiptWriterRetryEvent<Input>) => void)
     | null;
@@ -121,6 +137,7 @@ export class RuntimeReceiptWriter<Input, Output> {
     maxBatchBytes?: number;
     maxBufferedBytes?: number;
     maxFlushMs?: number;
+    maxRetryElapsedMs?: number;
     onRetryEvent?: (event: RuntimeReceiptWriterRetryEvent<Input>) => void;
   }) {
     this.#batchKey = options.batchKey;
@@ -138,6 +155,10 @@ export class RuntimeReceiptWriter<Input, Output> {
       this.#maxBufferedBytes,
     );
     this.#maxFlushMs = Math.max(0, Math.floor(options.maxFlushMs ?? 5));
+    this.#maxRetryElapsedMs = normalizePositiveInteger(
+      options.maxRetryElapsedMs,
+      2 * 60_000,
+    );
     this.#onRetryEvent = options.onRetryEvent ?? null;
   }
 
@@ -357,7 +378,24 @@ export class RuntimeReceiptWriter<Input, Output> {
       } catch (error) {
         const retry = this.#classifyRetryableError(error);
         if (!retry) throw error;
+        const elapsedMs = Date.now() - startedAt;
+        if (elapsedMs >= this.#maxRetryElapsedMs) {
+          throw new RuntimeReceiptWriterRetryDeadlineError(
+            attempt,
+            elapsedMs,
+            this.#maxRetryElapsedMs,
+            { cause: error },
+          );
+        }
         const retryAfterMs = jitteredRetryDelayMs(retry.retryAfterMs);
+        if (elapsedMs + retryAfterMs > this.#maxRetryElapsedMs) {
+          throw new RuntimeReceiptWriterRetryDeadlineError(
+            attempt,
+            elapsedMs,
+            this.#maxRetryElapsedMs,
+            { cause: error },
+          );
+        }
         this.#emitRetryEvent({
           phase: 'retry',
           attempt,