deepline 0.2.55 → 0.2.57

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -275,10 +275,25 @@ type RuntimeSheetPreparedRowDisposition = {
   disposition: RuntimeSheetPrepareDisposition;
 };
 
+// A failed connect retires one cached generation. It cannot end that pool
+// until every request that already borrowed the generation has returned.
+type CachedRuntimePostgresPool = {
+  pool: RuntimePool;
+  borrowers: number;
+  retired: boolean;
+  endPromise: Promise<void> | null;
+};
+
+type RuntimePostgresPoolLease = {
+  pool: RuntimePool;
+  cached: CachedRuntimePostgresPool | null;
+  release(): Promise<void>;
+};
+
 type RuntimeSheetDatasetMode = 'upsert' | 'net_new';
 const dbSessionCache = new Map<string, DbSessionCacheEntry>();
 const dbSessionInFlight = new Map<string, Promise<CreateDbSessionResponse>>();
-const postgresPools = new Map<string, RuntimePool>();
+const postgresPools = new Map<string, CachedRuntimePostgresPool>();
 const runtimePostgresAdmissions = new Map<string, RuntimePostgresAdmission>();
 const runtimeWorkReceiptEnsureCache = new Map<string, Promise<void>>();
 const runtimeSheetEnsureCache = new Map<
@@ -331,6 +346,14 @@ const RUNTIME_API_REQUEST_TIMEOUT_MS = 30_000;
 // writes. Keep the caller alive long enough for the FIFO admission queue to
 // make progress instead of aborting a healthy queued start after 30 seconds.
 const RUNTIME_SHEET_ADMISSION_REQUEST_TIMEOUT_MS = 10 * 60_000;
+const RUNTIME_SHEET_COMPLETION_SERVER_OPERATION_TIMEOUT_MS = 75_000;
+// Server admission is bounded to 30s and the single connection attempt to
+// 10s. Keep the transport outside that envelope so it never retries while the
+// first server-side write can still be active.
+const RUNTIME_SHEET_COMPLETION_REQUEST_TIMEOUT_MS = 150_000;
+const RUNTIME_SHEET_COMPLETION_TOTAL_TIMEOUT_MS = 180_000;
+const RUNTIME_SHEET_COMPLETION_STATEMENT_TIMEOUT_MS = 60_000;
+const RUNTIME_POSTGRES_SHEET_ADMISSION_TIMEOUT_MS = 30_000;
 const RUNTIME_POSTGRES_PREWARM_MAX_ATTEMPTS = 4;
 const RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS = [250, 750, 1_500] as const;
 const RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS = 4;
@@ -624,13 +647,21 @@ async function postRuntimeApi<TResponse>(
     context.vercelProtectionBypassToken,
   );
   const maxAttempts = runtimeApiMaxAttempts(body.action);
+  const operationStartedAt = Date.now();
   for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
     let response: Response | null = null;
     let parsed: Record<string, unknown> | null = null;
     const abortController = new AbortController();
+    const operationRemainingMs = runtimeApiOperationRemainingMs(
+      body.action,
+      operationStartedAt,
+    );
+    if (operationRemainingMs <= 0) {
+      throw runtimeApiTotalDeadlineError(body.action);
+    }
     const timeout = setTimeout(
       () => abortController.abort(),
-      runtimeApiRequestTimeoutMs(body.action),
+      Math.min(runtimeApiRequestTimeoutMs(body.action), operationRemainingMs),
     );
     try {
       response = await fetch(url, {
@@ -661,8 +692,18 @@ async function postRuntimeApi<TResponse>(
       if (context.abortSignal?.aborted) {
         throw context.abortSignal.reason ?? error;
       }
+      if (
+        runtimeApiOperationRemainingMs(body.action, operationStartedAt) <= 0
+      ) {
+        throw runtimeApiTotalDeadlineError(body.action);
+      }
       if (attempt < maxAttempts) {
-        await sleepRuntimeApiRetry(body.action, attempt);
+        await sleepRuntimeApiRetry({
+          action: body.action,
+          attempt,
+          operationStartedAt,
+          abortSignal: context.abortSignal ?? undefined,
+        });
         continue;
       }
       const message = error instanceof Error ? error.message : String(error);
@@ -686,7 +727,12 @@ async function postRuntimeApi<TResponse>(
       // every runtime action returns a JSON object and runtime writes are
       // already retry-safe across the equivalent fetch-failed boundary.
       if (attempt < maxAttempts) {
-        await sleepRuntimeApiRetry(body.action, attempt);
+        await sleepRuntimeApiRetry({
+          action: body.action,
+          attempt,
+          operationStartedAt,
+          abortSignal: context.abortSignal ?? undefined,
+        });
         continue;
       }
       const requestId = response.headers.get('x-deepline-request-id');
@@ -740,7 +786,13 @@ async function postRuntimeApi<TResponse>(
           `${typeof parsed?.error === 'string' ? parsed.error : ''}\n${details ?? ''}`,
         ));
     if (shouldRetryRuntimeResponse && attempt < maxAttempts) {
-      await sleepRuntimeApiRetry(body.action, attempt, retryAfterMs);
+      await sleepRuntimeApiRetry({
+        action: body.action,
+        attempt,
+        retryAfterMs,
+        operationStartedAt,
+        abortSignal: context.abortSignal ?? undefined,
+      });
       continue;
     }
 
@@ -786,22 +838,75 @@ function runtimeApiRetryDelayMs(
 function runtimeApiRequestTimeoutMs(
   action: RuntimeApiActionRequest['action'],
 ): number {
-  return action === 'runtime_sheet_start' ||
-    action === 'runtime_sheet_complete_map_rows'
+  if (action === 'runtime_sheet_complete_map_rows') {
+    return RUNTIME_SHEET_COMPLETION_REQUEST_TIMEOUT_MS;
+  }
+  return action === 'runtime_sheet_start'
     ? RUNTIME_SHEET_ADMISSION_REQUEST_TIMEOUT_MS
     : RUNTIME_API_REQUEST_TIMEOUT_MS;
 }
 
-async function sleepRuntimeApiRetry(
+function runtimeApiOperationRemainingMs(
   action: RuntimeApiActionRequest['action'],
-  attempt: number,
-  retryAfterMs?: number,
-): Promise<void> {
-  await new Promise((resolve) =>
-    setTimeout(resolve, runtimeApiRetryDelayMs(action, attempt, retryAfterMs)),
+  startedAt: number,
+): number {
+  return action === 'runtime_sheet_complete_map_rows'
+    ? Math.max(
+        0,
+        RUNTIME_SHEET_COMPLETION_TOTAL_TIMEOUT_MS - (Date.now() - startedAt),
+      )
+    : Number.POSITIVE_INFINITY;
+}
+
+function runtimeApiTotalDeadlineError(
+  action: RuntimeApiActionRequest['action'],
+): Error {
+  return new Error(
+    `Runtime API ${action} exceeded its ` +
+      `${RUNTIME_SHEET_COMPLETION_TOTAL_TIMEOUT_MS}ms total deadline.`,
   );
 }
 
+async function sleepRuntimeApiRetry(input: {
+  action: RuntimeApiActionRequest['action'];
+  attempt: number;
+  retryAfterMs?: number;
+  operationStartedAt: number;
+  abortSignal?: AbortSignal;
+}): Promise<void> {
+  const requestedDelayMs = runtimeApiRetryDelayMs(
+    input.action,
+    input.attempt,
+    input.retryAfterMs,
+  );
+  const operationRemainingMs = runtimeApiOperationRemainingMs(
+    input.action,
+    input.operationStartedAt,
+  );
+  if (operationRemainingMs <= 0) {
+    throw runtimeApiTotalDeadlineError(input.action);
+  }
+  const delayMs = Math.min(requestedDelayMs, operationRemainingMs);
+  await new Promise<void>((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      input.abortSignal?.removeEventListener('abort', onAbort);
+      resolve();
+    }, delayMs);
+    const onAbort = () => {
+      clearTimeout(timeout);
+      reject(input.abortSignal?.reason ?? new Error('Runtime API aborted.'));
+    };
+    if (input.abortSignal?.aborted) {
+      onAbort();
+      return;
+    }
+    input.abortSignal?.addEventListener('abort', onAbort, { once: true });
+  });
+  if (requestedDelayMs >= operationRemainingMs) {
+    throw runtimeApiTotalDeadlineError(input.action);
+  }
+}
+
 function runtimeApiRetryDelaysForAction(
   action: RuntimeApiActionRequest['action'],
 ): readonly number[] {
@@ -1708,10 +1813,17 @@ async function connectRuntimePostgresPool(
     let released = false;
     return {
       query: (text, params) => client.query(text, params),
-      release: () => {
+      release: (destroy = false) => {
         if (released) return;
         released = true;
-        client.release();
+        client.release(destroy);
+        releaseAdmission();
+      },
+      destroy: (error) => {
+        if (released) return;
+        released = true;
+        if (client.destroy) client.destroy(error);
+        else client.release(true);
         releaseAdmission();
       },
     };
@@ -1753,15 +1865,13 @@ function getRuntimePostgresAdmission(
       receipts: RUNTIME_POSTGRES_RECEIPT_ADMISSION_MAX_QUEUED,
       sheets: RUNTIME_POSTGRES_SHEET_ADMISSION_MAX_QUEUED,
     },
-    // Sheet capacity is backpressure, not a sheet-persistence failure. The
-    // surrounding runtime request has its own bounded deadline and the queue
-    // stays FIFO; no healthy request is discarded merely because two writes
-    // are active for ten seconds. Receipt admission retains its short timeout:
-    // a receipt can follow the ordinary retry/recovery path without holding a
-    // row's data-plane transaction open.
+    // Capacity is backpressure, not a sheet-persistence failure. Bound both
+    // queues so the gateway can return typed capacity before an HTTP caller
+    // times out. The runner then parks the same frozen write and retries it;
+    // it never leaves an invisible request queued behind a dead client.
     acquireTimeoutMs: {
       receipts: RUNTIME_POSTGRES_RECEIPT_ADMISSION_TIMEOUT_MS,
-      sheets: null,
+      sheets: RUNTIME_POSTGRES_SHEET_ADMISSION_TIMEOUT_MS,
     },
   });
   runtimePostgresAdmissions.set(postgresUrl, admission);
@@ -1774,46 +1884,149 @@ export function runtimePostgresAdmissionTelemetry(
   return runtimePostgresAdmissions.get(postgresUrl)?.snapshot() ?? null;
 }
 
-function getPostgresPool(
+function endRetiredRuntimePostgresPool(
+  cached: CachedRuntimePostgresPool,
+): Promise<void> {
+  if (!cached.retired || cached.borrowers > 0) {
+    return Promise.resolve();
+  }
+  cached.endPromise ??= Promise.resolve()
+    .then(() => cached.pool.end())
+    .catch(() => {});
+  return cached.endPromise;
+}
+
+async function retireRuntimePostgresPool(
+  poolKey: string,
+  cached: CachedRuntimePostgresPool,
+): Promise<void> {
+  if (postgresPools.get(poolKey) === cached) {
+    postgresPools.delete(poolKey);
+  }
+  cached.retired = true;
+  await endRetiredRuntimePostgresPool(cached);
+}
+
+function getPostgresPoolLease(
   postgresUrl: string,
   lane: RuntimePostgresLane,
   cachePool = true,
-): RuntimePool {
+): RuntimePostgresPoolLease {
   if (!cachePool) {
-    return createRuntimePool({
-      connectionString: postgresUrl,
-      maxConnections: RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE,
-      idleTimeoutMs: 0,
-      connectTimeoutMs: RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS,
-    });
+    return {
+      pool: createRuntimePool({
+        connectionString: postgresUrl,
+        maxConnections: RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE,
+        idleTimeoutMs: 0,
+        connectTimeoutMs: RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS,
+      }),
+      cached: null,
+      release: async () => {},
+    };
   }
   const poolKey = runtimePostgresPoolKey(postgresUrl, lane);
-  const existing = postgresPools.get(poolKey);
-  if (existing) {
-    return existing;
-  }
-  const pool = createRuntimePool({
-    connectionString: postgresUrl,
-    maxConnections: RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE,
-    idleTimeoutMs: 15_000,
-    connectTimeoutMs: RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS,
-  });
-  postgresPools.set(poolKey, pool);
-  return pool;
+  let cached = postgresPools.get(poolKey);
+  if (!cached) {
+    cached = {
+      pool: createRuntimePool({
+        connectionString: postgresUrl,
+        maxConnections: RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE,
+        idleTimeoutMs: 15_000,
+        connectTimeoutMs: RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS,
+      }),
+      borrowers: 0,
+      retired: false,
+      endPromise: null,
+    };
+    postgresPools.set(poolKey, cached);
+  }
+  cached.borrowers += 1;
+  let released = false;
+  return {
+    pool: cached.pool,
+    cached,
+    release: async () => {
+      if (released) return;
+      released = true;
+      cached.borrowers -= 1;
+      await endRetiredRuntimePostgresPool(cached);
+    },
+  };
 }
 
 async function resetRuntimePostgresPool(postgresUrl: string): Promise<void> {
-  const pools = (['receipts', 'sheets'] as const)
+  const retirements = (['receipts', 'sheets'] as const)
     .map((lane) => runtimePostgresPoolKey(postgresUrl, lane))
     .map((key) => {
-      const pool = postgresPools.get(key);
-      postgresPools.delete(key);
-      return pool;
+      const cached = postgresPools.get(key);
+      if (!cached) return null;
+      return retireRuntimePostgresPool(key, cached);
     })
-    .filter((pool): pool is RuntimePool => pool !== undefined);
-  await Promise.all(
-    pools.map((pool) => Promise.resolve(pool.end()).catch(() => {})),
-  );
+    .filter((retirement): retirement is Promise<void> => retirement !== null);
+  await Promise.all(retirements);
+}
+
+async function awaitRuntimePostgresOperation<T>(input: {
+  client: RuntimePoolClient;
+  deadlineAt: number | null;
+  operation: Promise<T>;
+  signal?: AbortSignal | null;
+}): Promise<T> {
+  if (input.deadlineAt === null && !input.signal) return await input.operation;
+  const remainingMs =
+    input.deadlineAt === null
+      ? null
+      : Math.max(0, input.deadlineAt - Date.now());
+  if (remainingMs === 0 || input.signal?.aborted) {
+    const error =
+      input.signal?.reason ??
+      new Error('Runtime Postgres operation exceeded its total deadline.');
+    if (input.client.destroy) input.client.destroy(error);
+    else input.client.release(true);
+    throw error;
+  }
+  return await new Promise<T>((resolve, reject) => {
+    let timeout: ReturnType<typeof setTimeout> | null = null;
+    const cleanup = () => {
+      if (timeout) clearTimeout(timeout);
+      input.signal?.removeEventListener('abort', onAbort);
+    };
+    const rejectAndDestroy = (error: unknown) => {
+      const normalizedError =
+        error instanceof Error ? error : new Error(String(error));
+      if (input.client.destroy) input.client.destroy(normalizedError);
+      else input.client.release(true);
+      cleanup();
+      reject(error);
+    };
+    const onAbort = () =>
+      rejectAndDestroy(
+        input.signal?.reason ??
+          new Error('Runtime Postgres operation aborted.'),
+      );
+    if (remainingMs !== null) {
+      timeout = setTimeout(
+        () =>
+          rejectAndDestroy(
+            new Error(
+              `Runtime Postgres operation exceeded its ${remainingMs}ms remaining deadline.`,
+            ),
+          ),
+        remainingMs,
+      );
+    }
+    input.signal?.addEventListener('abort', onAbort, { once: true });
+    input.operation.then(
+      (value) => {
+        cleanup();
+        resolve(value);
+      },
+      (error) => {
+        cleanup();
+        reject(error);
+      },
+    );
+  });
 }
 
 async function withRuntimePostgres<T>(
@@ -1825,12 +2038,15 @@ async function withRuntimePostgres<T>(
   options: {
     cachePool?: boolean;
     maxConnectAttempts?: number;
+    operationTimeoutMs?: number;
     signal?: AbortSignal | null;
   } = {},
 ): Promise<T> {
   let client: RuntimePoolClient | null = null;
   let requestLocalPool: RuntimePool | null = null;
+  let poolLease: RuntimePostgresPoolLease | null = null;
   let roleTransactionStarted = false;
+  let operationDeadlineAt: number | null = null;
   const cachePool =
     (options.cachePool ?? true) && canReuseRuntimePostgresPoolsAcrossRequests();
   const maxConnectAttempts =
@@ -1841,7 +2057,8 @@ async function withRuntimePostgres<T>(
   const admission = getRuntimePostgresAdmission(session.postgresUrl);
   for (let attempt = 1; attempt <= maxConnectAttempts; attempt += 1) {
     try {
-      const pool = getPostgresPool(session.postgresUrl, lane, cachePool);
+      poolLease = getPostgresPoolLease(session.postgresUrl, lane, cachePool);
+      const pool = poolLease.pool;
       if (!cachePool) {
         requestLocalPool = pool;
       }
@@ -1852,18 +2069,37 @@ async function withRuntimePostgres<T>(
         session.postgresUrl,
         options.signal,
       );
+      operationDeadlineAt =
+        options.operationTimeoutMs === undefined
+          ? null
+          : Date.now() + Math.max(1, Math.floor(options.operationTimeoutMs));
       if (session.executionRole) {
-        await client.query('BEGIN');
+        await awaitRuntimePostgresOperation({
+          client,
+          deadlineAt: operationDeadlineAt,
+          operation: client.query('BEGIN'),
+          signal: options.signal,
+        });
         roleTransactionStarted = true;
-        await client.query(
-          `SET LOCAL ROLE ${quoteIdentifier(session.executionRole)}`,
-        );
+        await awaitRuntimePostgresOperation({
+          client,
+          deadlineAt: operationDeadlineAt,
+          operation: client.query(
+            `SET LOCAL ROLE ${quoteIdentifier(session.executionRole)}`,
+          ),
+          signal: options.signal,
+        });
       }
       break;
     } catch (error) {
       if (client) {
         if (roleTransactionStarted) {
-          await client.query('ROLLBACK').catch(() => {});
+          await awaitRuntimePostgresOperation({
+            client,
+            deadlineAt: operationDeadlineAt,
+            operation: client.query('ROLLBACK'),
+            signal: options.signal,
+          }).catch(() => {});
           roleTransactionStarted = false;
         }
         client.release();
@@ -1875,17 +2111,15 @@ async function withRuntimePostgres<T>(
       // the backpressure in the first place.
       const preserveSharedPool =
         cachePool && isRuntimePostgresAdmissionError(error);
-      if (cachePool && !preserveSharedPool) {
+      if (cachePool && !preserveSharedPool && poolLease?.cached) {
         const poolKey = runtimePostgresPoolKey(session.postgresUrl, lane);
-        const pool = postgresPools.get(poolKey);
-        postgresPools.delete(poolKey);
-        if (pool) {
-          await Promise.resolve(pool.end()).catch(() => {});
-        }
+        await retireRuntimePostgresPool(poolKey, poolLease.cached);
       } else if (requestLocalPool) {
         await Promise.resolve(requestLocalPool.end()).catch(() => {});
         requestLocalPool = null;
       }
+      await poolLease?.release();
+      poolLease = null;
       if (
         attempt >= maxConnectAttempts ||
         !isTransientRuntimePostgresConnectionError(error)
@@ -1904,20 +2138,36 @@ async function withRuntimePostgres<T>(
     throw new Error('Runtime Postgres connection was not acquired.');
   }
   try {
-    const result = await fn(client, { active: roleTransactionStarted });
+    const result = await awaitRuntimePostgresOperation({
+      client,
+      deadlineAt: operationDeadlineAt,
+      operation: fn(client, { active: roleTransactionStarted }),
+      signal: options.signal,
+    });
     if (roleTransactionStarted) {
-      await client.query('COMMIT');
+      await awaitRuntimePostgresOperation({
+        client,
+        deadlineAt: operationDeadlineAt,
+        operation: client.query('COMMIT'),
+        signal: options.signal,
+      });
       roleTransactionStarted = false;
     }
     return result;
   } catch (error) {
     if (roleTransactionStarted) {
-      await client.query('ROLLBACK').catch(() => {});
+      await awaitRuntimePostgresOperation({
+        client,
+        deadlineAt: operationDeadlineAt,
+        operation: client.query('ROLLBACK'),
+        signal: options.signal,
+      }).catch(() => {});
       roleTransactionStarted = false;
     }
     throw error;
   } finally {
     client.release();
+    await poolLease?.release();
     if (requestLocalPool) {
       await Promise.resolve(requestLocalPool.end()).catch(() => {});
     }
@@ -2083,6 +2333,7 @@ async function withRuntimeSheetQueryClient<T>(
     tableNamespace: string;
     sheetContract: PlaySheetContract;
     transactional: boolean;
+    statementTimeoutMs?: number;
     timings?: RuntimeSheetTiming[];
   },
   operation: (client: RuntimeQueryClient) => Promise<T>,
@@ -2092,7 +2343,11 @@ async function withRuntimeSheetQueryClient<T>(
     context,
     input,
     async () => {
-      if (!input.transactional && isRuntimeOneShotQueryFactoryRegistered()) {
+      if (
+        !input.transactional &&
+        input.statementTimeoutMs === undefined &&
+        isRuntimeOneShotQueryFactoryRegistered()
+      ) {
         const operationStartedAt = Date.now();
         return await withRuntimeOneShotPostgres(session, operation).finally(
           () => {
@@ -2107,9 +2362,16 @@ async function withRuntimeSheetQueryClient<T>(
       return await withRuntimePostgres(
         session,
         async (client, transaction) => {
-          const managesTransaction = input.transactional && !transaction.active;
+          const managesTransaction =
+            (input.transactional || input.statementTimeoutMs !== undefined) &&
+            !transaction.active;
           if (managesTransaction) await client.query('BEGIN');
           try {
+            if (input.statementTimeoutMs !== undefined) {
+              await client.query(
+                `SET LOCAL statement_timeout TO ${Math.max(1, Math.floor(input.statementTimeoutMs))}`,
+              );
+            }
             const result = await operation(client);
             if (managesTransaction) await client.query('COMMIT');
             return result;
@@ -2122,6 +2384,12 @@ async function withRuntimeSheetQueryClient<T>(
         },
         {
           cachePool: !context.disablePostgresPoolCache,
+          maxConnectAttempts:
+            input.statementTimeoutMs === undefined ? undefined : 1,
+          operationTimeoutMs:
+            input.statementTimeoutMs === undefined
+              ? undefined
+              : RUNTIME_SHEET_COMPLETION_SERVER_OPERATION_TIMEOUT_MS,
           signal: context.abortSignal,
         },
       );
@@ -6695,6 +6963,8 @@ export type RuntimeMapRowsWriteResult = {
   fencedKeys: string[];
   staleDropped?: number;
   staleDroppedKeys?: string[];
+  /** Synthetic Preview-only proof that the gateway applied the page-tail hold. */
+  runtimeTestPageTailHoldMs?: number;
 };
 
 type RuntimeMapRowsMutationResult = {
@@ -7802,6 +8072,9 @@ export async function completeRuntimeMapRows(
       }),
       // Preserve the gateway's compatibility classification during rollout.
       fencedKeys: result.fencedKeys,
+      ...(typeof result.runtimeTestPageTailHoldMs === 'number'
+        ? { runtimeTestPageTailHoldMs: result.runtimeTestPageTailHoldMs }
+        : {}),
     };
   }
   const sheetContract = augmentSheetContractWithDatasetFields({
@@ -7897,6 +8170,7 @@ export async function completeRuntimeMapRows(
       tableNamespace: input.tableNamespace,
       sheetContract,
       transactional: needsTransaction,
+      statementTimeoutMs: RUNTIME_SHEET_COMPLETION_STATEMENT_TIMEOUT_MS,
     },
     async (client) => {
       const completed =

package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts

@@ -23,6 +23,8 @@ export const RUNTIME_CAPACITY_POLICY = {
   },
   receiptGateway: {
     admissionTimeoutMs: 10_000,
+    /** End identical receipt-row work before the caller can overlap a retry. */
+    claimOperationTimeoutMs: 20_000,
     requestTimeoutMs: 30_000,
   },
 } as const;
@@ -103,6 +105,14 @@ export function assertRuntimeCapacityPolicy(): void {
       'Receipt gateway request timeout must exceed its admission timeout.',
     );
   }
+  if (
+    RUNTIME_CAPACITY_POLICY.receiptGateway.claimOperationTimeoutMs + 5_000 >
+    RUNTIME_CAPACITY_POLICY.receiptGateway.requestTimeoutMs
+  ) {
+    throw new Error(
+      'Receipt claim operations must leave at least five seconds for cancellation and response delivery.',
+    );
+  }
   if (
     RUNTIME_CAPACITY_POLICY.absurd.perOrgConcurrency >
     ABSURD_GLOBAL_RUN_CONCURRENCY

package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts

@@ -1,4 +1,4 @@
-export const PLAY_RUNTIME_CONTRACT = 4;
+export const PLAY_RUNTIME_CONTRACT = 5;
 export const PLAY_RUNTIME_CONTRACT_MIN_SUPPORTED = 1;
 export const PLAY_RUNTIME_CONTRACT_HEADER = 'x-deepline-runtime-contract';
 /** Shared runner/gateway envelope limit for unacknowledged progress recovery. */
@@ -27,6 +27,13 @@ export function resolvePlayRuntimeTransportAttemptId(
 }
 
 export const RUNTIME_CONTRACT_CHANGELOG = [
+  {
+    contract: 5,
+    changed:
+      'Runtime Sheet completion and lazy launch lookup use bounded, cancellation-aware gateway operations.',
+    compatOwner: 'runtime',
+    compatRemoval: 'contract 4 runtime lane drain',
+  },
   {
     contract: 4,
     changed: