deepline 0.1.199 → 0.1.201

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -66,10 +66,10 @@ import {
 import { RECEIPT_STATUS_CODE, receiptStatusFromCode } from './receipt-status';
 import {
   workReceiptClaimableStatusCodes,
+  workReceiptActiveOwnerPredicateSql,
   workReceiptClaimConflictPredicateSql,
   workReceiptHeartbeatPredicateSql,
   workReceiptReleaseOwnerPredicateSql,
-  workReceiptRunningOwnerPredicateSql,
 } from './receipt-sql';
 import {
   activeRuntimeSheetAttemptFenceSql,
@@ -105,6 +105,7 @@ import {
   PLAY_RUNTIME_TEST_FAULT_HEADER,
   type RuntimeTestFaultName,
 } from './test-runtime-seams';
+import { vercelProtectionBypassHeader } from './vercel-protection';
 
 type RuntimeApiContext = {
   baseUrl?: string | null;
@@ -302,10 +303,11 @@ const RUNTIME_DB_SESSION_RENEWAL_WINDOW_MS = 60_000;
 const RUNTIME_API_RETRY_DELAYS_MS = [
   250, 500, 1_000, 2_000, 4_000, 8_000, 8_000,
 ] as const;
-const RUNTIME_API_MAX_ATTEMPTS = RUNTIME_API_RETRY_DELAYS_MS.length + 1;
+const RUNTIME_API_RESOLVE_PLAY_RETRY_DELAYS_MS = [
+  250, 500, 1_000, 2_000, 4_000, 8_000, 8_000, 15_000, 15_000, 30_000,
+] as const;
 const RUNTIME_API_DEFAULT_RETRY_AFTER_MS = 2_000;
 const RUNTIME_API_REQUEST_TIMEOUT_MS = 30_000;
-const vercelProtectionCookieCache = new Map<string, Promise<string | null>>();
 const RUNTIME_POSTGRES_PREWARM_MAX_ATTEMPTS = 4;
 const RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS = [250, 750, 1_500] as const;
 const RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS = 4;
@@ -315,11 +317,12 @@ const RUNTIME_WORK_RECEIPT_QUERY_RETRY_DELAYS_MS = [250, 750] as const;
 // Runtime DB sessions are minted against the pooled tenant endpoint. A healthy
 // connect is sub-second; spending minutes on one sandbox dial only hides a
 // broken route and stalls the whole play. Keep retries bounded and loud.
-const RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS = 5_000;
+const RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS = 10_000;
 // Daytona executes one play per runner process. This is the bounded data-plane
 // budget for that one run's sheet reads/writes, receipts, and terminal/export
 // reads over the PgBouncer URL; it is not a scheduler/control-plane fan-out.
 const RUNTIME_POSTGRES_POOL_MAX_CONNECTIONS = 8;
+const RECEIPT_STATUS_QUEUED = RECEIPT_STATUS_CODE.queued;
 const RECEIPT_STATUS_PENDING = RECEIPT_STATUS_CODE.pending;
 const RECEIPT_STATUS_RUNNING = RECEIPT_STATUS_CODE.running;
 const RECEIPT_STATUS_COMPLETED = RECEIPT_STATUS_CODE.completed;
@@ -456,20 +459,12 @@ function resolveRuntimeApiUrl(context: RuntimeApiContext): string {
   if (!baseUrl) {
     throw new Error('Runner runtime API requires a baseUrl.');
   }
-  const url = new URL(
-    `${baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_COMPAT_PATH}`,
-  );
-  const bypassToken = context.vercelProtectionBypassToken?.trim();
-  if (bypassToken) {
-    url.searchParams.set('x-vercel-set-bypass-cookie', 'true');
-    url.searchParams.set('x-vercel-protection-bypass', bypassToken);
-  }
-  return url.toString();
+  return `${baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_COMPAT_PATH}`;
 }
 
 function resolveRuntimeApiHeaders(
   context: RuntimeApiContext,
-  vercelProtectionCookie?: string | null,
+  vercelHeaders: Record<string, string>,
 ): Record<string, string> {
   const token = context.executorToken?.trim();
   if (!token) {
@@ -479,61 +474,16 @@ function resolveRuntimeApiHeaders(
     'content-type': 'application/json',
     authorization: `Bearer ${token}`,
     [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
-    ...(context.vercelProtectionBypassToken
-      ? { 'x-vercel-protection-bypass': context.vercelProtectionBypassToken }
-      : {}),
+    ...vercelHeaders,
     ...(context.runtimeTestFaultHeader?.trim()
       ? {
           [PLAY_RUNTIME_TEST_FAULT_HEADER]:
             context.runtimeTestFaultHeader.trim(),
         }
       : {}),
-    ...(vercelProtectionCookie ? { cookie: vercelProtectionCookie } : {}),
   };
 }
 
-function setCookieHeaders(headers: Headers): string[] {
-  const getter = (headers as Headers & { getSetCookie?: () => string[] })
-    .getSetCookie;
-  if (typeof getter === 'function') {
-    return getter.call(headers);
-  }
-  const single = headers.get('set-cookie');
-  return single ? [single] : [];
-}
-
-function cookieHeaderFromSetCookie(headers: Headers): string | null {
-  const cookies = setCookieHeaders(headers)
-    .flatMap((header) => header.split(/,(?=\s*[^;,=]+=[^;,]+)/g))
-    .map((header) => header.split(';', 1)[0]?.trim() ?? '')
-    .filter(Boolean);
-  return cookies.length > 0 ? cookies.join('; ') : null;
-}
-
-function resolveVercelProtectionCookie(
-  context: RuntimeApiContext,
-): Promise<string | null> {
-  const bypassToken = context.vercelProtectionBypassToken?.trim();
-  const baseUrl = context.baseUrl?.trim().replace(/\/$/, '');
-  if (!bypassToken || !baseUrl) return Promise.resolve(null);
-
-  const cacheKey = `${baseUrl}\n${bypassToken}`;
-  const cached = vercelProtectionCookieCache.get(cacheKey);
-  if (cached) return cached;
-
-  const promise = (async () => {
-    const url = new URL(`${baseUrl}/api/v2/health`);
-    url.searchParams.set('x-vercel-set-bypass-cookie', 'true');
-    url.searchParams.set('x-vercel-protection-bypass', bypassToken);
-    const response = await fetch(url.toString(), {
-      headers: { 'x-vercel-protection-bypass': bypassToken },
-    }).catch(() => null);
-    return response ? cookieHeaderFromSetCookie(response.headers) : null;
-  })();
-  vercelProtectionCookieCache.set(cacheKey, promise);
-  return promise;
-}
-
 function normalizeRuntimeUserEmail(
   value: string | null | undefined,
 ): string | null {
@@ -546,8 +496,11 @@ async function postRuntimeApi<TResponse>(
   body: RuntimeApiActionRequest,
 ): Promise<TResponse> {
   const url = resolveRuntimeApiUrl(context);
-  const vercelProtectionCookie = await resolveVercelProtectionCookie(context);
-  for (let attempt = 1; attempt <= RUNTIME_API_MAX_ATTEMPTS; attempt += 1) {
+  const vercelHeaders = vercelProtectionBypassHeader(
+    context.vercelProtectionBypassToken,
+  );
+  const maxAttempts = runtimeApiMaxAttempts(body.action);
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
     let response: Response;
     const abortController = new AbortController();
     const timeout = setTimeout(
@@ -557,13 +510,13 @@ async function postRuntimeApi<TResponse>(
     try {
       response = await fetch(url, {
         method: 'POST',
-        headers: resolveRuntimeApiHeaders(context, vercelProtectionCookie),
+        headers: resolveRuntimeApiHeaders(context, vercelHeaders),
         body: JSON.stringify(body),
         signal: abortController.signal,
       });
     } catch (error) {
-      if (attempt < RUNTIME_API_MAX_ATTEMPTS) {
-        await sleepRuntimeApiRetry(attempt);
+      if (attempt < maxAttempts) {
+        await sleepRuntimeApiRetry(body.action, attempt);
         continue;
       }
       const message = error instanceof Error ? error.message : String(error);
@@ -605,8 +558,8 @@ async function postRuntimeApi<TResponse>(
         isTransientRuntimeApiErrorMessage(
           `${typeof parsed?.error === 'string' ? parsed.error : ''}\n${details ?? ''}`,
         ));
-    if (shouldRetryRuntimeResponse && attempt < RUNTIME_API_MAX_ATTEMPTS) {
-      await sleepRuntimeApiRetry(attempt, retryAfterMs);
+    if (shouldRetryRuntimeResponse && attempt < maxAttempts) {
+      await sleepRuntimeApiRetry(body.action, attempt, retryAfterMs);
       continue;
     }
 
@@ -625,11 +578,12 @@ async function postRuntimeApi<TResponse>(
 }
 
 function runtimeApiRetryDelayMs(
+  action: RuntimeApiActionRequest['action'],
   attempt: number,
   retryAfterMs?: number,
 ): number {
   const configuredDelay =
-    RUNTIME_API_RETRY_DELAYS_MS[attempt - 1] ??
+    runtimeApiRetryDelaysForAction(action)[attempt - 1] ??
     RUNTIME_API_DEFAULT_RETRY_AFTER_MS;
   return retryAfterMs === undefined
     ? configuredDelay
@@ -637,14 +591,29 @@ function runtimeApiRetryDelayMs(
 }
 
 async function sleepRuntimeApiRetry(
+  action: RuntimeApiActionRequest['action'],
   attempt: number,
   retryAfterMs?: number,
 ): Promise<void> {
   await new Promise((resolve) =>
-    setTimeout(resolve, runtimeApiRetryDelayMs(attempt, retryAfterMs)),
+    setTimeout(resolve, runtimeApiRetryDelayMs(action, attempt, retryAfterMs)),
   );
 }
 
+function runtimeApiRetryDelaysForAction(
+  action: RuntimeApiActionRequest['action'],
+): readonly number[] {
+  return action === 'resolve_play'
+    ? RUNTIME_API_RESOLVE_PLAY_RETRY_DELAYS_MS
+    : RUNTIME_API_RETRY_DELAYS_MS;
+}
+
+function runtimeApiMaxAttempts(
+  action: RuntimeApiActionRequest['action'],
+): number {
+  return runtimeApiRetryDelaysForAction(action).length + 1;
+}
+
 function isTransientRuntimeApiErrorMessage(message: string): boolean {
   return /timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|requested endpoint could not be found, or you don't have access/i.test(
     message,
@@ -757,6 +726,75 @@ async function isRuntimeSheetSchemaReady(
   }
 }
 
+/**
+ * Report exactly what the scoped runtime role can see for the sheet table, from
+ * the same connection the readiness probe uses. When ensure_sheet reports the
+ * table as provisioned but the runner probe still returns not-ready, the cause
+ * is almost always that the runtime role lacks a privilege on the freshly
+ * created table (so information_schema.columns hides the attempt columns) or a
+ * schema/table-name mismatch. This turns the previously blind "still not ready"
+ * error into an actionable one that names the concrete missing piece.
+ */
+async function readRuntimeSheetSchemaDiagnostic(
+  session: RuntimePostgresSession,
+): Promise<Record<string, unknown>> {
+  const query = async (
+    client: RuntimeQueryClient,
+  ): Promise<Record<string, unknown>> => {
+    const visibleColumns = await client.query<{ column_name: string }>(
+      `
+        SELECT column_name
+        FROM information_schema.columns
+        WHERE table_schema = $1
+          AND table_name = $2
+        ORDER BY ordinal_position
+      `,
+      [session.postgres.schema, session.postgres.sheetTable],
+    );
+    const privileges = await client.query<Record<string, unknown>>(
+      `
+        SELECT
+          current_user AS probe_role,
+          to_regclass($3) IS NOT NULL AS regclass_visible,
+          has_table_privilege(current_user, $3, 'SELECT') AS can_select,
+          has_table_privilege(current_user, $3, 'INSERT') AS can_insert,
+          has_table_privilege(current_user, $3, 'UPDATE') AS can_update
+      `,
+      [
+        session.postgres.schema,
+        session.postgres.sheetTable,
+        `${quoteIdentifier(session.postgres.schema)}.${quoteIdentifier(session.postgres.sheetTable)}`,
+      ],
+    );
+    const visible = new Set(visibleColumns.rows.map((row) => row.column_name));
+    return {
+      schema: session.postgres.schema,
+      table: session.postgres.sheetTable,
+      probeRole: privileges.rows[0]?.probe_role ?? null,
+      regclassVisible: privileges.rows[0]?.regclass_visible ?? null,
+      canSelect: privileges.rows[0]?.can_select ?? null,
+      canInsert: privileges.rows[0]?.can_insert ?? null,
+      canUpdate: privileges.rows[0]?.can_update ?? null,
+      visibleColumnCount: visible.size,
+      missingAttemptColumns: RUNTIME_SHEET_ATTEMPT_COLUMNS.filter(
+        (column) => !visible.has(column),
+      ),
+    };
+  };
+  try {
+    if (isRuntimeOneShotQueryFactoryRegistered()) {
+      return await withRuntimeOneShotPostgres(session, query);
+    }
+    return await withRuntimePostgres(session, query);
+  } catch (error) {
+    return {
+      schema: session.postgres.schema,
+      table: session.postgres.sheetTable,
+      diagnosticError: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
 async function waitForRuntimeSheetSchemaReadyAfterEnsure(
   session: RuntimePostgresSession,
   input: {
@@ -870,7 +908,7 @@ async function ensureRuntimeSheetForPreloadedSession(
       phase: 'ensure_sheet_for_preloaded_session',
       ms: Date.now() - ensureStartedAt,
     });
-    const readyAfterEnsure = await waitForRuntimeSheetSchemaReadyAfterEnsure(
+    let readyAfterEnsure = await waitForRuntimeSheetSchemaReadyAfterEnsure(
       input.session,
       {
         sheetContract: input.sheetContract,
@@ -878,8 +916,39 @@ async function ensureRuntimeSheetForPreloadedSession(
       },
     );
     if (!readyAfterEnsure) {
+      // ensure_sheet provisions the sheet as the tenant admin role, but the
+      // preloaded session probes as the scoped runtime role. When the tenant
+      // role contract was only partially applied (e.g. role-membership grants
+      // skipped on a data plane where the owner cannot administer roles), the
+      // runtime role never inherits access to the freshly created table, so its
+      // information_schema view hides the attempt columns and the probe stays
+      // not-ready even though the table exists. Re-apply the storage grants
+      // through the same self-heal the pre-ensure probe uses, then re-check
+      // once. This is a grant repair, not a blind retry: if the runtime role
+      // still cannot see the sheet, we fail loudly below with the concrete
+      // runtime-role view.
+      const repairStartedAt = Date.now();
+      await repairRuntimeStorageGrants(context, {
+        playName: context.playName,
+      });
+      input.timings?.push({
+        phase: 'repair_storage_grants_after_preloaded_session_ensure',
+        ms: Date.now() - repairStartedAt,
+        retried: true,
+      });
+      readyAfterEnsure = await waitForRuntimeSheetSchemaReadyAfterEnsure(
+        input.session,
+        {
+          sheetContract: input.sheetContract,
+          timings: input.timings,
+        },
+      );
+    }
+    if (!readyAfterEnsure) {
+      const diagnostic = await readRuntimeSheetSchemaDiagnostic(input.session);
       throw new Error(
-        `Runtime sheet schema for ctx.dataset("${input.tableNamespace}") is still not ready after ensure_sheet.`,
+        `Runtime sheet schema for ctx.dataset("${input.tableNamespace}") is still not ready after ensure_sheet. ` +
+          `Runtime-role view: ${JSON.stringify(diagnostic)}`,
       );
     }
   })();
@@ -1303,9 +1372,13 @@ async function prewarmRuntimePostgresSession(
     attempt += 1
   ) {
     try {
-      await withRuntimePostgres(session, async (client) => {
-        await client.query('SELECT 1');
-      });
+      await withRuntimePostgres(
+        session,
+        async (client) => {
+          await client.query('SELECT 1');
+        },
+        { maxConnectAttempts: 1 },
+      );
       return;
     } catch (error) {
       if (
@@ -1454,17 +1527,17 @@ async function resetRuntimePostgresPool(postgresUrl: string): Promise<void> {
 async function withRuntimePostgres<T>(
   session: RuntimePostgresSession,
   fn: (client: RuntimePoolClient) => Promise<T>,
-  options: { cachePool?: boolean } = {},
+  options: { cachePool?: boolean; maxConnectAttempts?: number } = {},
 ): Promise<T> {
   let client: RuntimePoolClient | null = null;
   let requestLocalPool: RuntimePool | null = null;
   const cachePool =
     (options.cachePool ?? true) && canReuseRuntimePostgresPoolsAcrossRequests();
-  for (
-    let attempt = 1;
-    attempt <= RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS;
-    attempt += 1
-  ) {
+  const maxConnectAttempts =
+    options.maxConnectAttempts === undefined
+      ? RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS
+      : Math.max(1, Math.floor(options.maxConnectAttempts));
+  for (let attempt = 1; attempt <= maxConnectAttempts; attempt += 1) {
     try {
       const pool = getPostgresPool(session.postgresUrl, cachePool);
       if (!cachePool) {
@@ -1484,7 +1557,7 @@ async function withRuntimePostgres<T>(
         requestLocalPool = null;
       }
       if (
-        attempt >= RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS ||
+        attempt >= maxConnectAttempts ||
         !isTransientRuntimePostgresConnectionError(error)
       ) {
         throw error;
@@ -2658,6 +2731,8 @@ async function prepareRuntimeSheetDatasetRows(
       '$9::timestamptz',
       '$10::integer',
     );
+    const targetOlderForeignAttemptOwnerSql =
+      'coalesce(target._attempt_owner_run_id, target._run_id) IS DISTINCT FROM $8::text AND COALESCE(target._attempt_seq, 0) < $10::integer';
     const existingAttemptFenceSql = activeRuntimeSheetAttemptFenceSql(
       'existing',
       '$7::text',
@@ -2665,6 +2740,8 @@ async function prepareRuntimeSheetDatasetRows(
       '$9::timestamptz',
       '$10::integer',
     );
+    const existingForeignAttemptOwnerSql =
+      'coalesce(existing._attempt_owner_run_id, existing._run_id) IS DISTINCT FROM $8::text';
     const targetNewerTerminalRowSql = newerTerminalRuntimeSheetRowSql(
       'target',
       '$9::timestamptz',
@@ -2776,10 +2853,14 @@ async function prepareRuntimeSheetDatasetRows(
           WHERE target._key = input_rows._key
             AND target._status IN ('pending', 'running', 'failed')
             AND NOT (${targetNewerTerminalRowSql})
-	            AND (
-	              target._status = 'failed'
-	              OR ${targetAttemptFenceSql}
-	            )
+            AND (
+              ${targetAttemptFenceSql}
+              OR ${targetOlderForeignAttemptOwnerSql}
+              OR (
+                $11::boolean
+                AND target._status = 'failed'
+              )
+            )
             AND (
               target._run_id IS DISTINCT FROM $4::text
               OR target._input_index IS DISTINCT FROM input_rows._input_index
@@ -2808,10 +2889,17 @@ async function prepareRuntimeSheetDatasetRows(
           WHERE (
                existing._status IN ('pending', 'running', 'failed')
                AND NOT (${existingNewerTerminalRowSql})
-	               AND (
-	                 existing._status = 'failed'
-	                 OR ${existingAttemptFenceSql}
-	               )
+               AND (
+                 ${existingAttemptFenceSql}
+                 OR (
+                   ${existingForeignAttemptOwnerSql}
+                   AND COALESCE(existing._attempt_seq, 0) < $10::integer
+                 )
+                 OR (
+                   $11::boolean
+                   AND existing._status = 'failed'
+                 )
+               )
              )
              OR (
                existing._status = 'enriched'
@@ -3110,6 +3198,8 @@ async function markRuntimeRowsPendingForRecompute(
     '$7::timestamptz',
     '$8::integer',
   );
+  const targetOlderForeignAttemptOwnerSql =
+    'coalesce(target._attempt_owner_run_id, target._run_id) IS DISTINCT FROM $6::text AND COALESCE(target._attempt_seq, 0) < $8::integer';
   const targetWritableEnrichedAttemptSql =
     writableEnrichedRuntimeSheetAttemptSql(
       'target',
@@ -3138,7 +3228,18 @@ async function markRuntimeRowsPendingForRecompute(
         FROM target_rows
       WHERE target._key = target_rows._key
          AND (
-           ${targetAttemptFenceSql}
+           (
+             target._status IN ('pending', 'running', 'failed')
+             AND (
+               ${targetAttemptFenceSql}
+               OR ${targetOlderForeignAttemptOwnerSql}
+               OR (
+                 $10::boolean
+                 AND target._status = 'failed'
+               )
+             )
+           )
+           OR ${targetAttemptFenceSql}
            OR (
              $10::boolean
              AND target._status = 'enriched'
@@ -3450,6 +3551,10 @@ export async function claimRuntimeWorkReceipt(
         input.leaseTtlMs,
         PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
       );
+      const claimStatus =
+        input.leaseAware === true
+          ? RECEIPT_STATUS_QUEUED
+          : RECEIPT_STATUS_RUNNING;
       const claimableStatuses = workReceiptClaimableStatusCodes({
         forceRefresh: input.forceRefresh,
         forceFailedRefresh: input.forceFailedRefresh,
@@ -3457,7 +3562,13 @@ export async function claimRuntimeWorkReceipt(
       });
       const { rows } = await client.query<Record<string, unknown>>(
         `
-        WITH claimed AS (
+        WITH existing AS MATERIALIZED (
+          SELECT status
+          FROM ${workReceiptTable(session)}
+          WHERE k = decode($1, 'hex')
+          FOR UPDATE
+        ),
+        claimed AS (
           INSERT INTO ${workReceiptTable(session)} (
             k,
             status,
@@ -3500,6 +3611,7 @@ export async function claimRuntimeWorkReceipt(
             claimableStatusesSql: '$5',
             claimantRunIdSql: '$3',
             claimantRunAttemptSql: '$9',
+            reclaimRunningSql: '$11',
             forceRefreshSql: '$8',
             forceFailedRefreshSql: '$10',
           })}
@@ -3525,25 +3637,39 @@ export async function claimRuntimeWorkReceipt(
                lease_owner_run_id,
                lease_owner_attempt,
                lease_expires_at,
-               updated_at
+               updated_at,
+               EXISTS (
+                 SELECT 1
+                 FROM existing
+                 WHERE status = $12::smallint
+               ) AS refreshed_failed_receipt
         FROM claimed
       `,
         [
           workReceiptKeyHex(input.key),
-          RECEIPT_STATUS_RUNNING,
+          claimStatus,
           input.runId,
-          RECEIPT_STATUS_RUNNING,
+          claimStatus,
           claimableStatuses,
           leaseId,
           leaseTtlMs,
           input.forceRefresh === true,
           runAttempt,
           input.forceFailedRefresh === true,
+          input.reclaimRunning === true,
+          RECEIPT_STATUS_CODE.failed,
         ],
       );
-      const claimed = rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
+      const claimedRow = rows[0] ?? null;
+      const claimed = claimedRow ? mapRuntimeWorkReceiptRow(claimedRow) : null;
       if (claimed) {
-        return { disposition: 'claimed', receipt: claimed };
+        return {
+          disposition: 'claimed',
+          receipt: claimed,
+          wasFailed:
+            claimedRow.refreshed_failed_receipt === true ||
+            claimedRow.refreshed_failed_receipt === 'true',
+        };
       }
 
       const latest = await readRuntimeWorkReceipt(client, session, input.key);
@@ -3577,7 +3703,11 @@ function runtimeWorkReceiptClaimDispositionForBlockedReceipt(input: {
   if (receipt.status === 'failed') {
     return { disposition: 'failed', receipt };
   }
-  if (receipt.status === 'pending' || receipt.status === 'running') {
+  if (
+    receipt.status === 'queued' ||
+    receipt.status === 'pending' ||
+    receipt.status === 'running'
+  ) {
     const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
     const leaseExpiresAtMs =
       typeof receipt.leaseExpiresAt === 'string'
@@ -3628,6 +3758,10 @@ export async function claimRuntimeWorkReceipts(
       const leaseIds = keys.map(() =>
         input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null,
       );
+      const claimStatus =
+        input.leaseAware === true
+          ? RECEIPT_STATUS_QUEUED
+          : RECEIPT_STATUS_RUNNING;
       const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
       const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
         input.leaseTtlMs,
@@ -3697,6 +3831,7 @@ export async function claimRuntimeWorkReceipts(
           claimableStatusesSql: '$6',
           claimantRunIdSql: '$4::text',
           claimantRunAttemptSql: '$9',
+          reclaimRunningSql: '$11',
           forceRefreshSql: '$8',
           forceFailedRefreshSql: '$10',
         })}
@@ -3783,14 +3918,15 @@ export async function claimRuntimeWorkReceipts(
         [
           keyHexes,
           leaseIds,
-          RECEIPT_STATUS_RUNNING,
+          claimStatus,
           input.runId,
-          RECEIPT_STATUS_RUNNING,
+          claimStatus,
           reclaimStatuses,
           leaseTtlMs,
           input.forceRefresh === true,
           runAttempt,
           input.forceFailedRefresh === true,
+          input.reclaimRunning === true,
         ],
       );
       return rows.map((row) => {
@@ -3811,6 +3947,317 @@ export async function claimRuntimeWorkReceipts(
   );
 }
 
+export async function markRuntimeWorkReceiptRunning(
+  context: RuntimeApiContext,
+  input: {
+    playName: string;
+    runId: string;
+    key: string;
+    runAttempt?: number | null;
+    leaseId?: string | null;
+    leaseTtlMs?: number | null;
+  },
+): Promise<WorkReceipt | null> {
+  const session = await getRuntimeWorkReceiptSession(context, {
+    playName: input.playName,
+    key: input.key,
+  });
+  return await withRuntimeWorkReceiptClient(
+    context,
+    session,
+    async (client) => {
+      const leaseId = input.leaseId?.trim() || null;
+      const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
+      const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
+        input.leaseTtlMs,
+        PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
+      );
+      const { rows } = await client.query<Record<string, unknown>>(
+        `
+        WITH running AS (
+          UPDATE ${workReceiptTable(session)}
+             SET status = $2::smallint,
+                 lease_expires_at = CASE
+                   WHEN lease_id IS NULL THEN NULL
+                   ELSE now() + ($6::double precision * interval '1 millisecond')
+                 END,
+                 updated_at = now()
+           WHERE k = decode($1, 'hex')
+             AND ${workReceiptActiveOwnerPredicateSql({
+               receiptTable: workReceiptTable(session),
+               ownerRunIdSql: '$3',
+               ownerRunAttemptSql: '$5',
+               leaseIdSql: '$4::text',
+             })}
+          RETURNING convert_from(k, 'UTF8') AS k,
+                    status,
+                    output,
+                    error,
+                    failure_kind,
+                    run_id,
+                    lease_id,
+                    lease_owner_run_id,
+                    lease_owner_attempt,
+                    lease_expires_at,
+                    updated_at
+        )
+        SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at
+          FROM running
+      `,
+        [
+          workReceiptKeyHex(input.key),
+          RECEIPT_STATUS_RUNNING,
+          input.runId,
+          leaseId,
+          runAttempt,
+          leaseTtlMs,
+        ],
+      );
+      return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
+    },
+  );
+}
+
+export async function markRuntimeWorkReceiptsRunning(
+  context: RuntimeApiContext,
+  input: {
+    playName: string;
+    receipts: Array<{
+      runId: string;
+      key: string;
+      leaseId?: string | null;
+      runAttempt?: number | null;
+    }>;
+    leaseTtlMs?: number | null;
+  },
+): Promise<Array<WorkReceipt | null>> {
+  const receipts = input.receipts.filter((receipt) => receipt.key.trim());
+  if (receipts.length === 0) return [];
+  const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
+    input.leaseTtlMs,
+    PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
+  );
+  const session = await getRuntimeWorkReceiptSessionForKeys(context, {
+    playName: input.playName,
+    keys: receipts.map((receipt) => receipt.key),
+  });
+  return await withRuntimeWorkReceiptClient(
+    context,
+    session,
+    async (client) => {
+      const { rows } = await client.query<Record<string, unknown>>(
+        `
+      WITH inputs AS (
+        SELECT key_values.key_hex,
+               run_values.run_id,
+               lease_values.lease_id,
+               attempt_values.run_attempt,
+               key_values.ord
+          FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
+          JOIN unnest($2::text[]) WITH ORDINALITY AS run_values(run_id, ord)
+            ON run_values.ord = key_values.ord
+          JOIN unnest($3::text[]) WITH ORDINALITY AS lease_values(lease_id, ord)
+            ON lease_values.ord = key_values.ord
+          JOIN unnest($4::integer[]) WITH ORDINALITY AS attempt_values(run_attempt, ord)
+            ON attempt_values.ord = key_values.ord
+      ),
+      running AS (
+        UPDATE ${workReceiptTable(session)} AS target
+           SET status = $5::smallint,
+               lease_expires_at = CASE
+                 WHEN target.lease_id IS NULL THEN NULL
+                 ELSE now() + ($6::double precision * interval '1 millisecond')
+               END,
+               updated_at = now()
+          FROM inputs
+         WHERE target.k = decode(inputs.key_hex, 'hex')
+           AND ${workReceiptActiveOwnerPredicateSql({
+             receiptTable: 'target',
+             ownerRunIdSql: 'inputs.run_id',
+             ownerRunAttemptSql: 'inputs.run_attempt',
+             leaseIdSql: 'inputs.lease_id',
+           })}
+        RETURNING target.k,
+                  target.status,
+                  target.output,
+                  target.error,
+                  target.failure_kind,
+                  target.run_id,
+                  target.lease_id,
+                  target.lease_owner_run_id,
+                  target.lease_owner_attempt,
+                  target.lease_expires_at,
+                  target.updated_at,
+                  inputs.ord
+      ),
+      returned AS (
+        SELECT running.k,
+               running.status,
+               running.output,
+               running.error,
+               running.failure_kind,
+               running.run_id,
+               running.lease_id,
+               running.lease_owner_run_id,
+               running.lease_owner_attempt,
+               running.lease_expires_at,
+               running.updated_at,
+               inputs.ord
+          FROM inputs
+          LEFT JOIN running ON running.ord = inputs.ord
+      )
+      SELECT convert_from(returned.k, 'UTF8') AS k,
+             returned.status,
+             returned.output,
+             returned.error,
+             returned.failure_kind,
+             returned.run_id,
+             returned.lease_id,
+             returned.lease_owner_run_id,
+             returned.lease_owner_attempt,
+             returned.lease_expires_at,
+             returned.updated_at
+        FROM returned
+       ORDER BY returned.ord
+    `,
+        [
+          receipts.map((receipt) => workReceiptKeyHex(receipt.key)),
+          receipts.map((receipt) => receipt.runId),
+          receipts.map((receipt) => receipt.leaseId?.trim() || null),
+          receipts.map((receipt) =>
+            normalizeRuntimeRunAttempt(receipt.runAttempt),
+          ),
+          RECEIPT_STATUS_RUNNING,
+          leaseTtlMs,
+        ],
+      );
+      return rows.map((row) =>
+        row.k == null ? null : mapRuntimeWorkReceiptRow(row),
+      );
+    },
+  );
+}
+
+export async function markRuntimeWorkReceiptsQueued(
+  context: RuntimeApiContext,
+  input: {
+    playName: string;
+    receipts: Array<{
+      runId: string;
+      key: string;
+      leaseId?: string | null;
+      runAttempt?: number | null;
+    }>;
+    leaseTtlMs?: number | null;
+  },
+): Promise<Array<WorkReceipt | null>> {
+  const receipts = input.receipts.filter((receipt) => receipt.key.trim());
+  if (receipts.length === 0) return [];
+  const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
+    input.leaseTtlMs,
+    PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
+  );
+  const session = await getRuntimeWorkReceiptSessionForKeys(context, {
+    playName: input.playName,
+    keys: receipts.map((receipt) => receipt.key),
+  });
+  return await withRuntimeWorkReceiptClient(
+    context,
+    session,
+    async (client) => {
+      const { rows } = await client.query<Record<string, unknown>>(
+        `
+      WITH inputs AS (
+        SELECT key_values.key_hex,
+               run_values.run_id,
+               lease_values.lease_id,
+               attempt_values.run_attempt,
+               key_values.ord
+          FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
+          JOIN unnest($2::text[]) WITH ORDINALITY AS run_values(run_id, ord)
+            ON run_values.ord = key_values.ord
+          JOIN unnest($3::text[]) WITH ORDINALITY AS lease_values(lease_id, ord)
+            ON lease_values.ord = key_values.ord
+          JOIN unnest($4::integer[]) WITH ORDINALITY AS attempt_values(run_attempt, ord)
+            ON attempt_values.ord = key_values.ord
+      ),
+      queued AS (
+        UPDATE ${workReceiptTable(session)} AS target
+           SET status = $5::smallint,
+               lease_expires_at = CASE
+                 WHEN target.lease_id IS NULL THEN NULL
+                 ELSE now() + ($6::double precision * interval '1 millisecond')
+               END,
+               updated_at = now()
+          FROM inputs
+         WHERE target.k = decode(inputs.key_hex, 'hex')
+           AND ${workReceiptActiveOwnerPredicateSql({
+             receiptTable: 'target',
+             ownerRunIdSql: 'inputs.run_id',
+             ownerRunAttemptSql: 'inputs.run_attempt',
+             leaseIdSql: 'inputs.lease_id',
+           })}
+        RETURNING target.k,
+                  target.status,
+                  target.output,
+                  target.error,
+                  target.failure_kind,
+                  target.run_id,
+                  target.lease_id,
+                  target.lease_owner_run_id,
+                  target.lease_owner_attempt,
+                  target.lease_expires_at,
+                  target.updated_at,
+                  inputs.ord
+      ),
+      returned AS (
+        SELECT queued.k,
+               queued.status,
+               queued.output,
+               queued.error,
+               queued.failure_kind,
+               queued.run_id,
+               queued.lease_id,
+               queued.lease_owner_run_id,
+               queued.lease_owner_attempt,
+               queued.lease_expires_at,
+               queued.updated_at,
+               inputs.ord
+          FROM inputs
+          LEFT JOIN queued ON queued.ord = inputs.ord
+      )
+      SELECT convert_from(returned.k, 'UTF8') AS k,
+             returned.status,
+             returned.output,
+             returned.error,
+             returned.failure_kind,
+             returned.run_id,
+             returned.lease_id,
+             returned.lease_owner_run_id,
+             returned.lease_owner_attempt,
+             returned.lease_expires_at,
+             returned.updated_at
+        FROM returned
+       ORDER BY returned.ord
+    `,
+        [
+          receipts.map((receipt) => workReceiptKeyHex(receipt.key)),
+          receipts.map((receipt) => receipt.runId),
+          receipts.map((receipt) => receipt.leaseId?.trim() || null),
+          receipts.map((receipt) =>
+            normalizeRuntimeRunAttempt(receipt.runAttempt),
+          ),
+          RECEIPT_STATUS_QUEUED,
+          leaseTtlMs,
+        ],
+      );
+      return rows.map((row) =>
+        row.k == null ? null : mapRuntimeWorkReceiptRow(row),
+      );
+    },
+  );
+}
+
 export async function completeRuntimeWorkReceipt(
   context: RuntimeApiContext,
   input: {
@@ -3876,7 +4323,7 @@ export async function completeRuntimeWorkReceipt(
               lease_expires_at = NULL,
               updated_at = now()
           WHERE k = decode($1, 'hex')
-            AND ${workReceiptRunningOwnerPredicateSql({
+            AND ${workReceiptActiveOwnerPredicateSql({
               receiptTable: workReceiptTable(session),
               ownerRunIdSql: '$4',
               ownerRunAttemptSql: '$6',
@@ -4012,7 +4459,7 @@ export async function completeRuntimeWorkReceipts(
                updated_at = now()
           FROM inputs
          WHERE target.k = decode(inputs.key_hex, 'hex')
-           AND ${workReceiptRunningOwnerPredicateSql({
+           AND ${workReceiptActiveOwnerPredicateSql({
              receiptTable: 'target',
              ownerRunIdSql: 'inputs.run_id',
              ownerRunAttemptSql: 'inputs.run_attempt',
@@ -4120,7 +4567,7 @@ export async function failRuntimeWorkReceipt(
               lease_expires_at = NULL,
               updated_at = now()
           WHERE k = decode($1, 'hex')
-            AND ${workReceiptRunningOwnerPredicateSql({
+            AND ${workReceiptActiveOwnerPredicateSql({
               receiptTable: workReceiptTable(session),
               ownerRunIdSql: '$4',
               ownerRunAttemptSql: '$7',
@@ -4190,7 +4637,7 @@ export async function releaseRuntimeWorkReceipt(
                  lease_expires_at = NULL,
                  updated_at = now()
            WHERE k = decode($1, 'hex')
-             AND ${workReceiptRunningOwnerPredicateSql({
+             AND ${workReceiptActiveOwnerPredicateSql({
                receiptTable: workReceiptTable(session),
                ownerRunIdSql: '$3',
                ownerRunAttemptSql: '$5',
@@ -4283,7 +4730,7 @@ export async function failRuntimeWorkReceipts(
                updated_at = now()
           FROM inputs
          WHERE target.k = decode(inputs.key_hex, 'hex')
-	           AND ${workReceiptRunningOwnerPredicateSql({
+	           AND ${workReceiptActiveOwnerPredicateSql({
                receiptTable: 'target',
                ownerRunIdSql: 'inputs.run_id',
                ownerRunAttemptSql: 'inputs.run_attempt',
@@ -4493,7 +4940,7 @@ export async function skipRuntimeWorkReceipt(
               lease_expires_at = NULL,
               updated_at = now()
           WHERE k = decode($1, 'hex')
-            AND ${workReceiptRunningOwnerPredicateSql({
+            AND ${workReceiptActiveOwnerPredicateSql({
               receiptTable: workReceiptTable(session),
               ownerRunIdSql: '$4',
               ownerRunAttemptSql: '$6',
@@ -5122,6 +5569,13 @@ async function completeRuntimeMapRowChunks(
       '$11::timestamptz',
       '$12',
     );
+    const writableNonTerminalAttemptSql = `(
+      target._status <> 'enriched'
+      AND (
+        ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')}
+        OR coalesce(target._attempt_owner_run_id, target._run_id) IS DISTINCT FROM $10::text
+      )
+    )`;
     const sameOwnerTerminalEpochSql = sameOwnerTerminalAttemptEpochSql(
       'target',
       '$10::text',
@@ -5174,6 +5628,7 @@ async function completeRuntimeMapRowChunks(
            AND NOT (${sameOwnerTerminalEpochSql})
            AND (
              ($9::text IS NULL AND target._run_id = $5::text)
+             OR ($9::text IS NOT NULL AND ${writableNonTerminalAttemptSql})
              OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
 		             OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
            )
@@ -5208,6 +5663,7 @@ async function completeRuntimeMapRowChunks(
 	              (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
               OR NOT (
                 ($9::text IS NULL AND target._run_id = $5::text)
+                OR ($9::text IS NOT NULL AND ${writableNonTerminalAttemptSql})
                 OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
                 OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
               )