deepline 0.1.212 → 0.1.214

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -105,7 +105,13 @@ import {
   PLAY_RUNTIME_TEST_FAULT_HEADER,
   type RuntimeTestFaultName,
 } from './test-runtime-seams';
+import { COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID } from './durable-receipt-execution';
 import { vercelProtectionBypassHeader } from './vercel-protection';
+import {
+  RuntimePostgresAdmission,
+  type RuntimePostgresAdmissionSnapshot,
+  type RuntimePostgresLane,
+} from './runtime-postgres-admission';
 
 type RuntimeApiContext = {
   baseUrl?: string | null;
@@ -269,6 +275,7 @@ type RuntimeSheetPreparedRowDisposition = {
 const dbSessionCache = new Map<string, DbSessionCacheEntry>();
 const dbSessionInFlight = new Map<string, Promise<CreateDbSessionResponse>>();
 const postgresPools = new Map<string, RuntimePool>();
+const runtimePostgresAdmissions = new Map<string, RuntimePostgresAdmission>();
 const runtimeWorkReceiptEnsureCache = new Map<string, Promise<void>>();
 const runtimeSheetEnsureCache = new Map<
   string,
@@ -327,7 +334,14 @@ const RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS = 10_000;
 // Daytona executes one play per runner process. This is the bounded data-plane
 // budget for that one run's sheet reads/writes, receipts, and terminal/export
 // reads over the PgBouncer URL; it is not a scheduler/control-plane fan-out.
-const RUNTIME_POSTGRES_POOL_MAX_CONNECTIONS = 8;
+// One runner gets four tenant-gateway clients, split evenly so a large sheet
+// flush cannot starve receipt claims (and receipt traffic cannot block the
+// persistence barrier). The admission queues add a driver-independent bound:
+// Neon Pool and node-postgres may queue internally, but runtime callers never
+// enter those unobservable queues without first holding a lane permit.
+const RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE = 2;
+const RUNTIME_POSTGRES_ADMISSION_MAX_QUEUED_PER_LANE = 2_000;
+const RUNTIME_POSTGRES_ADMISSION_TIMEOUT_MS = 10_000;
 const RECEIPT_STATUS_QUEUED = RECEIPT_STATUS_CODE.queued;
 const RECEIPT_STATUS_PENDING = RECEIPT_STATUS_CODE.pending;
 const RECEIPT_STATUS_RUNNING = RECEIPT_STATUS_CODE.running;
@@ -369,6 +383,7 @@ export type PrepareRuntimeSheetResult = {
   attemptOwnerRunId?: string;
   attemptExpiresAt?: string;
   attemptSeq?: number;
+  writeVersion?: number;
 };
 
 export type RuntimeSheetTiming = {
@@ -1292,9 +1307,7 @@ async function deleteRuntimeDbSessionCacheEntry(
 ): Promise<void> {
   dbSessionCache.delete(cacheKey);
   if (session.postgresUrl) {
-    const pool = postgresPools.get(session.postgresUrl);
-    postgresPools.delete(session.postgresUrl);
-    await pool?.end().catch(() => {});
+    await resetRuntimePostgresPool(session.postgresUrl);
   }
 }
 
@@ -1415,18 +1428,10 @@ async function prewarmRuntimePostgresSession(
         attempt >= RUNTIME_POSTGRES_PREWARM_MAX_ATTEMPTS ||
         !isTransientRuntimePostgresConnectionError(error)
       ) {
-        const pool = postgresPools.get(session.postgresUrl);
-        postgresPools.delete(session.postgresUrl);
-        if (pool) {
-          await Promise.resolve(pool.end()).catch(() => {});
-        }
+        await resetRuntimePostgresPool(session.postgresUrl);
         throw error;
       }
-      const pool = postgresPools.get(session.postgresUrl);
-      postgresPools.delete(session.postgresUrl);
-      if (pool) {
-        await Promise.resolve(pool.end()).catch(() => {});
-      }
+      await resetRuntimePostgresPool(session.postgresUrl);
       await sleep(
         RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS[attempt - 1] ??
           RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS[
@@ -1491,7 +1496,10 @@ function sleep(ms: number): Promise<void> {
 
 async function connectRuntimePostgresPool(
   pool: RuntimePool,
+  admission: RuntimePostgresAdmission,
+  lane: RuntimePostgresLane,
 ): Promise<RuntimePoolClient> {
+  const releaseAdmission = await admission.acquire(lane);
   let timedOut = false;
   let timeout: ReturnType<typeof setTimeout> | null = null;
   const connectPromise = pool.connect();
@@ -1499,11 +1507,12 @@ async function connectRuntimePostgresPool(
     .then((client) => {
       if (timedOut) {
         client.release();
+        releaseAdmission();
       }
     })
     .catch(() => {});
   try {
-    return await Promise.race([
+    const client = await Promise.race([
       connectPromise,
       new Promise<never>((_, reject) => {
         timeout = setTimeout(() => {
@@ -1516,6 +1525,21 @@ async function connectRuntimePostgresPool(
         }, RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS);
       }),
     ]);
+    let released = false;
+    return {
+      query: (text, params) => client.query(text, params),
+      release: () => {
+        if (released) return;
+        released = true;
+        client.release();
+        releaseAdmission();
+      },
+    };
+  } catch (error) {
+    if (!timedOut) {
+      releaseAdmission();
+    }
+    throw error;
   } finally {
     if (timeout) {
       clearTimeout(timeout);
@@ -1523,35 +1547,81 @@ async function connectRuntimePostgresPool(
   }
 }
 
-function getPostgresPool(postgresUrl: string, cachePool = true): RuntimePool {
+function runtimePostgresLane(
+  session: RuntimePostgresSession,
+): RuntimePostgresLane {
+  return session.target.logicalTable === RUNTIME_WORK_RECEIPT_LOGICAL_TABLE
+    ? 'receipts'
+    : 'sheets';
+}
+
+function runtimePostgresPoolKey(
+  postgresUrl: string,
+  lane: RuntimePostgresLane,
+): string {
+  return `${postgresUrl}::${lane}`;
+}
+
+function getRuntimePostgresAdmission(
+  postgresUrl: string,
+): RuntimePostgresAdmission {
+  const existing = runtimePostgresAdmissions.get(postgresUrl);
+  if (existing) return existing;
+  const admission = new RuntimePostgresAdmission({
+    maxActivePerLane: RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE,
+    maxQueuedPerLane: RUNTIME_POSTGRES_ADMISSION_MAX_QUEUED_PER_LANE,
+    acquireTimeoutMs: RUNTIME_POSTGRES_ADMISSION_TIMEOUT_MS,
+  });
+  runtimePostgresAdmissions.set(postgresUrl, admission);
+  return admission;
+}
+
+export function runtimePostgresAdmissionTelemetry(
+  postgresUrl: string,
+): RuntimePostgresAdmissionSnapshot | null {
+  return runtimePostgresAdmissions.get(postgresUrl)?.snapshot() ?? null;
+}
+
+function getPostgresPool(
+  postgresUrl: string,
+  lane: RuntimePostgresLane,
+  cachePool = true,
+): RuntimePool {
   if (!cachePool) {
     return createRuntimePool({
       connectionString: postgresUrl,
-      maxConnections: 1,
+      maxConnections: RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE,
       idleTimeoutMs: 0,
       connectTimeoutMs: RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS,
     });
   }
-  const existing = postgresPools.get(postgresUrl);
+  const poolKey = runtimePostgresPoolKey(postgresUrl, lane);
+  const existing = postgresPools.get(poolKey);
   if (existing) {
     return existing;
   }
   const pool = createRuntimePool({
     connectionString: postgresUrl,
-    maxConnections: RUNTIME_POSTGRES_POOL_MAX_CONNECTIONS,
+    maxConnections: RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE,
     idleTimeoutMs: 15_000,
     connectTimeoutMs: RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS,
   });
-  postgresPools.set(postgresUrl, pool);
+  postgresPools.set(poolKey, pool);
   return pool;
 }
 
 async function resetRuntimePostgresPool(postgresUrl: string): Promise<void> {
-  const pool = postgresPools.get(postgresUrl);
-  postgresPools.delete(postgresUrl);
-  if (pool) {
-    await Promise.resolve(pool.end()).catch(() => {});
-  }
+  const pools = (['receipts', 'sheets'] as const)
+    .map((lane) => runtimePostgresPoolKey(postgresUrl, lane))
+    .map((key) => {
+      const pool = postgresPools.get(key);
+      postgresPools.delete(key);
+      return pool;
+    })
+    .filter((pool): pool is RuntimePool => pool !== undefined);
+  await Promise.all(
+    pools.map((pool) => Promise.resolve(pool.end()).catch(() => {})),
+  );
 }
 
 async function withRuntimePostgres<T>(
@@ -1567,13 +1637,15 @@ async function withRuntimePostgres<T>(
     options.maxConnectAttempts === undefined
       ? RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS
       : Math.max(1, Math.floor(options.maxConnectAttempts));
+  const lane = runtimePostgresLane(session);
+  const admission = getRuntimePostgresAdmission(session.postgresUrl);
   for (let attempt = 1; attempt <= maxConnectAttempts; attempt += 1) {
     try {
-      const pool = getPostgresPool(session.postgresUrl, cachePool);
+      const pool = getPostgresPool(session.postgresUrl, lane, cachePool);
       if (!cachePool) {
         requestLocalPool = pool;
       }
-      client = await connectRuntimePostgresPool(pool);
+      client = await connectRuntimePostgresPool(pool, admission, lane);
       if (session.executionRole) {
         await client.query(
           `SET ROLE ${quoteIdentifier(session.executionRole)}`,
@@ -1586,8 +1658,9 @@ async function withRuntimePostgres<T>(
         client = null;
       }
       if (cachePool) {
-        const pool = postgresPools.get(session.postgresUrl);
-        postgresPools.delete(session.postgresUrl);
+        const poolKey = runtimePostgresPoolKey(session.postgresUrl, lane);
+        const pool = postgresPools.get(poolKey);
+        postgresPools.delete(poolKey);
         if (pool) {
           await Promise.resolve(pool.end()).catch(() => {});
         }
@@ -1889,6 +1962,10 @@ function workReceiptTable(session: RuntimePostgresSession): string {
   );
 }
 
+function receiptExecutionLockTable(session: RuntimePostgresSession): string {
+  return fqRuntimeTable(session, '_deepline_receipt_execution_locks');
+}
+
 function postgresUrlCacheKey(value: string): string {
   return createHash('sha256').update(value).digest('hex').slice(0, 24);
 }
@@ -2030,6 +2107,14 @@ async function ensureRuntimeWorkReceiptTable(
   `,
     )
     .then(async () => {
+      await client.query(`
+        CREATE TABLE IF NOT EXISTS ${receiptExecutionLockTable(session)} (
+          k bytea PRIMARY KEY,
+          owner_execution_id text NOT NULL,
+          expires_at timestamptz NOT NULL,
+          updated_at timestamptz NOT NULL DEFAULT now()
+        )
+      `);
       const missingColumns = await missingRuntimeWorkReceiptSelfHealColumns(
         session,
         client,
@@ -2064,6 +2149,77 @@ async function ensureRuntimeWorkReceiptTable(
   }
 }
 
+export async function acquireRuntimeReceiptExecutionLock(
+  context: RuntimeApiContext,
+  input: {
+    playName: string;
+    key: string;
+    ownerExecutionId: string;
+    ttlMs: number;
+  },
+): Promise<{ ownerExecutionId: string; expiresAt: string } | null> {
+  const ttlMs = Math.floor(input.ttlMs);
+  if (!Number.isFinite(ttlMs) || ttlMs <= 0 || ttlMs > 10 * 60_000) {
+    throw new Error('Runtime receipt execution lock TTL must be 1..600000ms.');
+  }
+  const ownerExecutionId = input.ownerExecutionId.trim();
+  if (!ownerExecutionId) {
+    throw new Error('Runtime receipt execution lock owner is required.');
+  }
+  const session = await getRuntimeWorkReceiptSession(context, input);
+  return await withRuntimeWorkReceiptClient(
+    context,
+    session,
+    async (client) => {
+      const { rows } = await client.query<Record<string, unknown>>(
+        `
+      INSERT INTO ${receiptExecutionLockTable(session)} AS target (
+        k, owner_execution_id, expires_at, updated_at
+      ) VALUES (
+        decode($1, 'hex'), $2, now() + ($3::double precision * interval '1 millisecond'), now()
+      )
+      ON CONFLICT (k) DO UPDATE
+        SET owner_execution_id = EXCLUDED.owner_execution_id,
+            expires_at = EXCLUDED.expires_at,
+            updated_at = now()
+      WHERE target.expires_at <= now()
+         OR target.owner_execution_id = EXCLUDED.owner_execution_id
+      RETURNING owner_execution_id, expires_at
+      `,
+        [workReceiptKeyHex(input.key), ownerExecutionId, ttlMs],
+      );
+      const row = rows[0];
+      return row
+        ? {
+            ownerExecutionId: String(row.owner_execution_id),
+            expiresAt: new Date(String(row.expires_at)).toISOString(),
+          }
+        : null;
+    },
+  );
+}
+
+export async function releaseRuntimeReceiptExecutionLock(
+  context: RuntimeApiContext,
+  input: { playName: string; key: string; ownerExecutionId: string },
+): Promise<boolean> {
+  const session = await getRuntimeWorkReceiptSession(context, input);
+  return await withRuntimeWorkReceiptClient(
+    context,
+    session,
+    async (client) => {
+      const { rows } = await client.query(
+        `DELETE FROM ${receiptExecutionLockTable(session)}
+        WHERE k = decode($1, 'hex')
+          AND owner_execution_id = $2
+        RETURNING 1`,
+        [workReceiptKeyHex(input.key), input.ownerExecutionId],
+      );
+      return rows.length > 0;
+    },
+  );
+}
+
 async function withRuntimeWorkReceiptClient<T>(
   context: RuntimeApiContext,
   session: RuntimePostgresSession,
@@ -2273,6 +2429,31 @@ async function mintRuntimeSheetAttemptExpiresAt(
   throw new Error('Runtime sheet attempt expiry mint did not return a value.');
 }
 
+async function mintRuntimeSheetWriteVersion(
+  client: RuntimeQueryClient,
+  session: RuntimePostgresSession,
+  requested?: number | null,
+): Promise<number> {
+  if (requested != null) {
+    if (!Number.isSafeInteger(requested) || requested <= 0) {
+      throw new Error(
+        'Runtime sheet writeVersion must be a positive safe integer.',
+      );
+    }
+    return requested;
+  }
+  const { rows } = await client.query<{ write_version: number | string }>(
+    `SELECT ${nextRuntimeSheetVersionExpression(session)} AS write_version`,
+  );
+  const value = Number(rows[0]?.write_version);
+  if (!Number.isSafeInteger(value) || value <= 0) {
+    throw new Error(
+      'Runtime sheet write version allocation did not return a safe integer.',
+    );
+  }
+  return value;
+}
+
 function parseRuntimeCellMeta(value: unknown): Record<string, unknown> {
   if (value && typeof value === 'object' && !Array.isArray(value)) {
     return value as Record<string, unknown>;
@@ -2773,6 +2954,7 @@ async function prepareRuntimeSheetDatasetRows(
     attemptOwnerRunId: string;
     attemptExpiresAt: string;
     attemptSeq: number;
+    writeVersion: number;
     physicalInsertColumnsSql: string;
     physicalInsertValuesSql: string;
     physicalRefreshSetSql: string;
@@ -2851,6 +3033,44 @@ async function prepareRuntimeSheetDatasetRows(
             ON index_values.ord = key_values.ord
           ORDER BY key_values._key, key_values.ord
        ),
+       preexisting_rows AS (
+          SELECT target._key
+          FROM ${sheetTable(session)} AS target
+          JOIN input_rows ON input_rows._key = target._key
+       ),
+       versioned_completed_rows AS (
+          UPDATE ${sheetTable(session)} AS target
+          SET _run_id = $4::text,
+              _writer_run_id = $4::text,
+              _write_version = $12::bigint,
+              _input_index = input_rows._input_index,
+              _updated_at = now(),
+              _version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalRefreshSetSql}
+          FROM input_rows
+          WHERE target._key = input_rows._key
+            AND target._status = 'enriched'
+            AND COALESCE(target._write_version, 0) < $12::bigint
+          RETURNING target._key,
+                    ($11::boolean OR (${targetMissingOutputSql})) AS needs_recompute
+       ),
+       versioned_failed_rows AS (
+          UPDATE ${sheetTable(session)} AS target
+          SET _run_id = $4::text,
+              _writer_run_id = $4::text,
+              _write_version = $12::bigint,
+              _input_index = input_rows._input_index,
+              _attempt_id = $7::text,
+              _attempt_owner_run_id = $8::text,
+              _attempt_expires_at = NULL,
+              _attempt_seq = $10::integer,
+              _updated_at = now(),
+              _version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalRefreshSetSql}
+          FROM input_rows
+          WHERE target._key = input_rows._key
+            AND target._status = 'failed'
+            AND COALESCE(target._write_version, 0) < $12::bigint
+          RETURNING target._key
+       ),
        existing_rows AS (
           UPDATE ${sheetTable(session)} AS target
           SET _input_index = input_rows._input_index,
@@ -2858,6 +3078,7 @@ async function prepareRuntimeSheetDatasetRows(
               _version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalRefreshSetSql}
           FROM input_rows
           WHERE target._key = input_rows._key
+            AND target._write_version = $12::bigint
             AND target._status <> 'stale'
             AND (
               NOT (${targetNewerTerminalRowSql})
@@ -2881,8 +3102,8 @@ async function prepareRuntimeSheetDatasetRows(
           RETURNING target._key
         ),
         inserted_rows AS (
-          INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _attempt_id, _attempt_owner_run_id, _attempt_expires_at, _attempt_seq${input.physicalInsertColumnsSql})
-          SELECT _key, 'pending', $4::text, _input_index, $7::text, $8::text, $9::timestamptz, $10::integer${input.physicalInsertValuesSql}
+          INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _attempt_id, _attempt_owner_run_id, _attempt_expires_at, _attempt_seq, _write_version, _writer_run_id${input.physicalInsertColumnsSql})
+          SELECT _key, 'pending', $4::text, _input_index, $7::text, $8::text, $9::timestamptz, $10::integer, $12::bigint, $4::text${input.physicalInsertValuesSql}
           FROM input_rows
           ON CONFLICT (_key) DO UPDATE SET
             _status = 'pending',
@@ -2892,6 +3113,8 @@ async function prepareRuntimeSheetDatasetRows(
             _attempt_owner_run_id = EXCLUDED._attempt_owner_run_id,
             _attempt_expires_at = EXCLUDED._attempt_expires_at,
             _attempt_seq = EXCLUDED._attempt_seq,
+            _write_version = EXCLUDED._write_version,
+            _writer_run_id = EXCLUDED._writer_run_id,
             _updated_at = now(),
             _version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalUpsertSetSql}
           WHERE ${sheetTable(session)}._status = 'stale'
@@ -2911,10 +3134,30 @@ async function prepareRuntimeSheetDatasetRows(
           FROM input_rows
           WHERE target._key = input_rows._key
             AND target._status = 'enriched'
+            AND target._write_version = $12::bigint
             AND (${targetMissingOutputSql})
             AND COALESCE(target._attempt_seq, 0) <= $10::integer
           RETURNING target._key
         ),
+        superseded_rows AS (
+          UPDATE ${sheetTable(session)} AS target
+          SET _run_id = $4::text,
+              _input_index = input_rows._input_index,
+              _attempt_id = $7::text,
+              _attempt_owner_run_id = $8::text,
+              _attempt_expires_at = NULL,
+              _attempt_seq = $10::integer,
+              _write_version = $12::bigint,
+              _writer_run_id = $4::text,
+              _error = NULL,
+              _updated_at = now(),
+              _version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalRefreshSetSql}
+          FROM input_rows
+          WHERE target._key = input_rows._key
+            AND target._status IN ('pending', 'running')
+            AND COALESCE(target._write_version, 0) < $12::bigint
+          RETURNING target._key
+        ),
         claimed_existing_rows AS (
           UPDATE ${sheetTable(session)} AS target
           SET _run_id = $4::text,
@@ -2928,6 +3171,7 @@ async function prepareRuntimeSheetDatasetRows(
           FROM input_rows
           WHERE target._key = input_rows._key
             AND target._status IN ('pending', 'running', 'failed')
+            AND target._write_version = $12::bigint
             AND NOT (${targetNewerTerminalRowSql})
             AND (
               ${targetAttemptFenceSql}
@@ -2950,6 +3194,9 @@ async function prepareRuntimeSheetDatasetRows(
           FROM inserted_rows
           UNION
           SELECT _key
+          FROM superseded_rows
+          UNION
+          SELECT _key
           FROM missing_output_rows
           UNION
           SELECT _key
@@ -2959,10 +3206,18 @@ async function prepareRuntimeSheetDatasetRows(
           FROM existing_rows
           WHERE $11::boolean
           UNION
+          SELECT _key
+          FROM versioned_completed_rows
+          WHERE needs_recompute
+          UNION
+          SELECT _key FROM versioned_failed_rows
+          UNION
           SELECT existing._key
           FROM ${sheetTable(session)} AS existing
           JOIN input_rows ON input_rows._key = existing._key
           WHERE (
+               existing._write_version = $12::bigint
+               AND (
                existing._status IN ('pending', 'running', 'failed')
                AND NOT (${existingNewerTerminalRowSql})
                AND (
@@ -2975,8 +3230,10 @@ async function prepareRuntimeSheetDatasetRows(
                    AND COALESCE(existing._attempt_seq, 0) < $10::integer
                  )
                )
-             )
+             ))
              OR (
+               existing._write_version = $12::bigint
+               AND
                existing._status = 'enriched'
                AND (${existingMissingOutputSql})
                AND (
@@ -2995,6 +3252,9 @@ async function prepareRuntimeSheetDatasetRows(
             AND NOT EXISTS (
               SELECT 1 FROM pending_rows WHERE pending_rows._key = existing._key
             )
+          UNION
+          SELECT _key FROM versioned_completed_rows
+          WHERE NOT needs_recompute
         ),
         blocked_rows AS (
           SELECT input_rows._key
@@ -3017,7 +3277,11 @@ async function prepareRuntimeSheetDatasetRows(
           FROM blocked_rows
         ),
         inserted_count_cte AS (
-          SELECT count(*)::bigint AS c FROM inserted_rows
+          SELECT count(*)::bigint AS c
+          FROM inserted_rows
+          WHERE NOT EXISTS (
+            SELECT 1 FROM preexisting_rows WHERE preexisting_rows._key = inserted_rows._key
+          )
         ),
         missing_output_count_cte AS (
           SELECT count(*)::bigint AS c FROM missing_output_rows
@@ -3077,6 +3341,7 @@ async function prepareRuntimeSheetDatasetRows(
         input.attemptExpiresAt,
         input.attemptSeq,
         input.force === true,
+        input.writeVersion,
       ],
     );
     inserted += Number(rows[0]?.inserted_count ?? 0);
@@ -3113,6 +3378,7 @@ async function buildRuntimeSheetDatasetStartResult(
     attemptOwnerRunId: string;
     attemptExpiresAt: string;
     attemptSeq: number;
+    writeVersion: number;
     inserted: number;
     rowDispositions: RuntimeSheetPreparedRowDisposition[];
     force?: boolean;
@@ -3148,6 +3414,7 @@ async function buildRuntimeSheetDatasetStartResult(
       completedRows: [],
       blockedRows: [],
       tableNamespace: input.tableNamespace,
+      writeVersion: input.writeVersion,
     };
   }
 
@@ -3167,6 +3434,7 @@ async function buildRuntimeSheetDatasetStartResult(
     attemptOwnerRunId: input.attemptOwnerRunId,
     attemptExpiresAt: input.attemptExpiresAt,
     attemptSeq: input.attemptSeq,
+    writeVersion: input.writeVersion,
     normalizedPlayName: input.normalizedPlayName,
     normalizedTableNamespace: input.normalizedTableNamespace,
     outputFields: input.force === true ? datasetFields : [],
@@ -3197,6 +3465,7 @@ async function buildRuntimeSheetDatasetStartResult(
       completedRows: [],
       blockedRows,
       tableNamespace: input.tableNamespace,
+      writeVersion: input.writeVersion,
     };
   }
   const persistedRows = await readRuntimeRowsByKey(
@@ -3246,6 +3515,7 @@ async function buildRuntimeSheetDatasetStartResult(
     completedRows,
     blockedRows,
     tableNamespace: input.tableNamespace,
+    writeVersion: input.writeVersion ?? undefined,
   };
 }
 
@@ -3259,6 +3529,7 @@ async function markRuntimeRowsPendingForRecompute(
     attemptOwnerRunId: string;
     attemptExpiresAt: string;
     attemptSeq: number;
+    writeVersion: number | null;
     normalizedPlayName: string;
     normalizedTableNamespace: string;
     outputFields?: string[];
@@ -3266,28 +3537,13 @@ async function markRuntimeRowsPendingForRecompute(
   },
 ): Promise<void> {
   if (input.keys.length === 0) return;
-  const targetAttemptFenceSql = activeRuntimeSheetAttemptFenceSql(
-    'target',
-    '$5::text',
-    '$6::text',
-    '$7::timestamptz',
-    '$8::integer',
-  );
-  const targetOlderForeignAttemptOwnerSql =
-    'coalesce(target._attempt_owner_run_id, target._run_id) IS DISTINCT FROM $6::text AND COALESCE(target._attempt_seq, 0) < $8::integer';
-  const targetWritableEnrichedAttemptSql =
-    writableEnrichedRuntimeSheetAttemptSql(
-      'target',
-      '$6::text',
-      '$7::timestamptz',
-      '$8',
-    );
   const outputFields = [...new Set(input.outputFields ?? [])];
   await client.query(
     `WITH target_rows AS (
        SELECT _key, _status, _cell_meta
          FROM ${sheetTable(session)}
         WHERE _key = ANY($1::text[])
+          AND ($10::boolean OR NOT $10::boolean)
         FOR UPDATE
      ),
      updated AS (
@@ -3298,33 +3554,13 @@ async function markRuntimeRowsPendingForRecompute(
             _attempt_owner_run_id = $6::text,
             _attempt_expires_at = $7::timestamptz,
             _attempt_seq = $8::integer,
+            _write_version = $11::bigint,
+            _writer_run_id = $2::text,
             _updated_at = now(),
             _version = ${nextRuntimeSheetVersionExpression(session)}
         FROM target_rows
       WHERE target._key = target_rows._key
-         AND (
-           (
-             target._status IN ('pending', 'running', 'failed')
-             AND (
-               ${targetAttemptFenceSql}
-               OR ${targetOlderForeignAttemptOwnerSql}
-               OR (
-                 $10::boolean
-                 AND target._status = 'failed'
-               )
-             )
-           )
-           OR ${targetAttemptFenceSql}
-           OR (
-             $10::boolean
-             AND target._status = 'enriched'
-             AND COALESCE(target._attempt_seq, 0) <= $8::integer
-           )
-           OR (
-             target._status = 'enriched'
-             AND ${targetWritableEnrichedAttemptSql}
-           )
-         )
+         AND target._write_version = $11::bigint
          AND (
            target._status <> 'pending'
            OR target._run_id IS DISTINCT FROM $2::text
@@ -3414,6 +3650,7 @@ async function markRuntimeRowsPendingForRecompute(
       input.attemptSeq,
       outputFields,
       input.force === true,
+      input.writeVersion,
     ],
   );
 }
@@ -3603,6 +3840,7 @@ export async function claimRuntimeWorkReceipt(
     playName: string;
     runId: string;
     key: string;
+    leaseId?: string | null;
     runAttempt?: number | null;
     leaseAware?: boolean;
     reclaimRunning?: boolean;
@@ -3620,7 +3858,8 @@ export async function claimRuntimeWorkReceipt(
     session,
     async (client) => {
       const leaseId =
-        input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null;
+        input.leaseId?.trim() ||
+        (input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null);
       const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
       const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
         input.leaseTtlMs,
@@ -3748,6 +3987,15 @@ export async function claimRuntimeWorkReceipt(
       }
 
       const latest = await readRuntimeWorkReceipt(client, session, input.key);
+      if (
+        latest &&
+        leaseId !== null &&
+        latest.leaseId === leaseId &&
+        (latest.leaseOwnerRunId ?? latest.runId) === input.runId &&
+        (latest.leaseOwnerAttempt ?? 0) === runAttempt
+      ) {
+        return { disposition: 'claimed', receipt: latest };
+      }
       if (
         latest &&
         input.forceRefresh !== true &&
@@ -3809,6 +4057,7 @@ export async function claimRuntimeWorkReceipts(
     playName: string;
     runId: string;
     keys: string[];
+    leaseIds?: string[];
     runAttempt?: number | null;
     leaseAware?: boolean;
     reclaimRunning?: boolean;
@@ -3830,9 +4079,21 @@ export async function claimRuntimeWorkReceipts(
     session,
     async (client) => {
       const keyHexes = keys.map(workReceiptKeyHex);
-      const leaseIds = keys.map(() =>
-        input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null,
-      );
+      if (
+        input.leaseIds !== undefined &&
+        input.leaseIds.length !== keys.length
+      ) {
+        throw new Error(
+          `Runtime receipt bulk claim requires one lease ID per key. Received ${input.leaseIds.length} lease IDs for ${keys.length} keys.`,
+        );
+      }
+      const leaseIds = keys.map((_, index) => {
+        const providedLeaseId = input.leaseIds?.[index]?.trim();
+        return (
+          providedLeaseId ||
+          (input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null)
+        );
+      });
       const claimStatus =
         input.leaseAware === true
           ? RECEIPT_STATUS_QUEUED
@@ -3935,7 +4196,10 @@ export async function claimRuntimeWorkReceipts(
                receipts.lease_expires_at,
                receipts.updated_at,
                input_keys.ord,
-               false AS claimed
+               receipts.lease_id = input_keys.lease_id
+                 AND receipts.lease_owner_run_id = $4::text
+                 AND COALESCE(receipts.lease_owner_attempt, 0) = $9::integer
+                 AS claimed
           FROM input_keys
           JOIN ${workReceiptTable(session)} AS receipts
             ON receipts.k = decode(input_keys.key_hex, 'hex')
@@ -4411,7 +4675,48 @@ export async function completeRuntimeWorkReceipt(
     context,
     session,
     async (client) => {
-      const leaseId = input.leaseId?.trim() || null;
+      const immutableCacheInsert =
+        input.leaseId === COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID;
+      const leaseId = immutableCacheInsert
+        ? null
+        : input.leaseId?.trim() || null;
+      if (immutableCacheInsert) {
+        const { rows: insertedRows } = await client.query<
+          Record<string, unknown>
+        >(
+          `
+          INSERT INTO ${workReceiptTable(session)} (
+            k, status, output, error, failure_kind, run_id, updated_at
+          ) VALUES (
+            decode($1, 'hex'), $2::smallint, $3::jsonb, NULL, 0, $4, now()
+          )
+          ON CONFLICT (k) DO NOTHING
+          RETURNING convert_from(k, 'UTF8') AS k, status, output, error,
+                    failure_kind, run_id, lease_id, lease_owner_run_id,
+                    lease_owner_attempt, lease_expires_at, updated_at
+          `,
+          [
+            workReceiptKeyHex(input.key),
+            RECEIPT_STATUS_COMPLETED,
+            input.output === null ? null : stringifyPostgresJson(input.output),
+            input.runId,
+          ],
+        );
+        if (insertedRows[0]) return mapRuntimeWorkReceiptRow(insertedRows[0]);
+        const { rows: winnerRows } = await client.query<
+          Record<string, unknown>
+        >(
+          `SELECT convert_from(k, 'UTF8') AS k, status, output, error,
+                  failure_kind, run_id, lease_id, lease_owner_run_id,
+                  lease_owner_attempt, lease_expires_at, updated_at
+             FROM ${workReceiptTable(session)}
+            WHERE k = decode($1, 'hex')
+              AND status IN ($2::smallint, ${RECEIPT_STATUS_SKIPPED}::smallint)
+            LIMIT 1`,
+          [workReceiptKeyHex(input.key), RECEIPT_STATUS_COMPLETED],
+        );
+        return winnerRows[0] ? mapRuntimeWorkReceiptRow(winnerRows[0]) : null;
+      }
       const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
       const { rows } = await client.query<Record<string, unknown>>(
         `
@@ -4546,6 +4851,44 @@ export async function completeRuntimeWorkReceipts(
     }
     throw error;
   }
+  const immutableEntries = receipts
+    .map((receipt, index) => ({ receipt, index }))
+    .filter(
+      ({ receipt }) =>
+        receipt.leaseId === COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
+    );
+  if (immutableEntries.length > 0) {
+    const leasedEntries = receipts
+      .map((receipt, index) => ({ receipt, index }))
+      .filter(
+        ({ receipt }) =>
+          receipt.leaseId !== COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
+      );
+    const resolved = Array<WorkReceipt | null>(receipts.length).fill(null);
+    const [immutableResults, leasedResults] = await Promise.all([
+      Promise.all(
+        immutableEntries.map(({ receipt }) =>
+          completeRuntimeWorkReceipt(context, {
+            playName: input.playName,
+            ...receipt,
+          }),
+        ),
+      ),
+      leasedEntries.length > 0
+        ? completeRuntimeWorkReceipts(context, {
+            playName: input.playName,
+            receipts: leasedEntries.map(({ receipt }) => receipt),
+          })
+        : Promise.resolve([]),
+    ]);
+    immutableEntries.forEach(({ index }, resultIndex) => {
+      resolved[index] = immutableResults[resultIndex] ?? null;
+    });
+    leasedEntries.forEach(({ index }, resultIndex) => {
+      resolved[index] = leasedResults[resultIndex] ?? null;
+    });
+    return resolved;
+  }
   const session = await getRuntimeWorkReceiptSessionForKeys(context, {
     playName: input.playName,
     keys: receipts.map((receipt) => receipt.key),
@@ -5195,6 +5538,7 @@ export async function startRuntimeSheetDataset(
     attemptExpiresAt?: string | null;
     attemptSeq?: number | null;
     attemptLeaseTtlMs?: number | null;
+    writeVersion?: number | null;
     inputOffset?: number;
     force?: boolean;
   },
@@ -5341,7 +5685,10 @@ export async function startRuntimeSheetDataset(
   });
   const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
   const chunks = chunkValues(rowEntries, DIRECT_POSTGRES_BATCH_SIZE);
-  const needsTransaction = chunks.length > 1;
+  // Scheduling is one admission decision. Even when batching SQL for the
+  // parameter limit, either every requested row receives this intent version
+  // or none do; a run can never leave 42 pending rows after 8 conflicts.
+  const needsTransaction = true;
 
   const result = await withRuntimeSheetQueryClient(
     context,
@@ -5359,6 +5706,11 @@ export async function startRuntimeSheetDataset(
         client,
         input.attemptLeaseTtlMs,
       );
+      const writeVersion = await mintRuntimeSheetWriteVersion(
+        client,
+        session,
+        input.writeVersion,
+      );
       timings.push({
         phase: 'attempt_expiry_mint',
         ms: Date.now() - attemptExpiryStartedAt,
@@ -5374,6 +5726,7 @@ export async function startRuntimeSheetDataset(
         attemptOwnerRunId,
         attemptExpiresAt,
         attemptSeq,
+        writeVersion,
         physicalInsertColumnsSql,
         physicalInsertValuesSql,
         physicalRefreshSetSql:
@@ -5406,6 +5759,7 @@ export async function startRuntimeSheetDataset(
         attemptOwnerRunId,
         attemptExpiresAt,
         attemptSeq,
+        writeVersion,
         timings,
         force: input.force === true,
         ...prepared,
@@ -5423,6 +5777,7 @@ export async function startRuntimeSheetDataset(
         ...built,
         attemptExpiresAt,
         attemptSeq,
+        writeVersion,
       };
     },
   );
@@ -5442,6 +5797,7 @@ export async function startRuntimeSheetDataset(
     attemptOwnerRunId,
     attemptExpiresAt: result.attemptExpiresAt,
     attemptSeq,
+    writeVersion: result.writeVersion,
     timings,
   };
 }
@@ -5751,6 +6107,7 @@ type CompleteRuntimeMapRowChunksInput = {
   attemptOwnerRunId: string | null;
   attemptExpiresAt: string | null;
   attemptSeq: number;
+  writeVersion: number | null;
   normalizedPlayName: string;
   normalizedTableNamespace: string;
   outputFields: string[];
@@ -5759,6 +6116,8 @@ type CompleteRuntimeMapRowChunksInput = {
 export type RuntimeMapRowsWriteResult = {
   updated: number;
   fencedKeys: string[];
+  staleDropped?: number;
+  staleDroppedKeys?: string[];
 };
 
 function writableEnrichedRuntimeSheetAttemptSql(
@@ -5847,6 +6206,8 @@ async function completeRuntimeMapRowChunks(
          UPDATE ${sheetTable(session)} AS target
          SET _status = 'enriched',
              _run_id = $5::text,
+             _writer_run_id = $5::text,
+             _write_version = COALESCE($13::bigint, target._write_version),
              _error = NULL,
              _attempt_id = $9::text,
              _attempt_owner_run_id = $10::text,
@@ -5857,6 +6218,8 @@ async function completeRuntimeMapRowChunks(
              _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
          FROM matched_updates AS updates
          WHERE target._key = updates.matched_key
+		           AND ($13::bigint IS NULL OR target._write_version = $13::bigint)
+                   AND ($13::bigint IS NOT NULL OR target._status <> 'enriched')
 		           AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
            AND NOT (${sameOwnerTerminalEpochSql})
            AND (
@@ -5893,6 +6256,9 @@ async function completeRuntimeMapRowChunks(
                   SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
                 )
             AND (
+	              ($13::bigint IS NOT NULL AND target._write_version <> $13::bigint)
+	              OR ($13::bigint IS NULL AND target._status = 'enriched')
+	              OR
 	              (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
               OR NOT (
                 ($9::text IS NULL AND target._run_id = $5::text)
@@ -6002,12 +6368,18 @@ async function completeRuntimeMapRowChunks(
         input.attemptOwnerRunId,
         input.attemptExpiresAt,
         input.attemptSeq,
+        input.writeVersion,
       ],
     );
-    updated += Number(rows[0]?.updated ?? 0);
-    fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
-
     const matchedKeys = new Set(rows[0]?.matched_keys ?? []);
+    const chunkFencedKeys = rows[0]?.fenced_keys ?? [];
+    // An exact-version replay that already produced the same terminal value is
+    // accepted idempotently even when SQL has no physical mutation to apply.
+    // Stale versions remain excluded through fenced_keys.
+    const appliedCount = Number(rows[0]?.updated ?? 0);
+    updated += appliedCount;
+    fencedKeys.push(...chunkFencedKeys);
+
     if (matchedKeys.size === chunk.length) {
       continue;
     }
@@ -6444,6 +6816,7 @@ async function failRuntimeMapRowChunks(
     attemptOwnerRunId: string | null;
     attemptExpiresAt: string | null;
     attemptSeq: number;
+    writeVersion: number | null;
     normalizedPlayName: string;
     normalizedTableNamespace: string;
   },
@@ -6518,6 +6891,8 @@ async function failRuntimeMapRowChunks(
          UPDATE ${sheetTable(session)} AS target
          SET _status = 'failed',
              _run_id = $6::text,
+             _writer_run_id = $6::text,
+             _write_version = COALESCE($16::bigint, target._write_version),
              _error = updates.error,
              _attempt_id = $11::text,
              _attempt_owner_run_id = $12::text,
@@ -6528,6 +6903,7 @@ async function failRuntimeMapRowChunks(
              _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
          FROM matched_updates AS updates, ${sheetTable(session)} AS prev
          WHERE target._key = updates.matched_key
+           AND ($16::bigint IS NULL OR target._write_version = $16::bigint)
            AND prev._key = target._key
            AND ($15::boolean OR target._status <> 'enriched')
 		           AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$13::timestamptz', '$14::integer', '$11::text', '$12::text')})
@@ -6553,6 +6929,8 @@ async function failRuntimeMapRowChunks(
                   SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
                 )
             AND (
+              ($16::bigint IS NOT NULL AND target._write_version <> $16::bigint)
+              OR
               (NOT $15::boolean AND target._status = 'enriched')
               OR (${newerTerminalRuntimeSheetRowSql('target', '$13::timestamptz', '$14::integer', '$11::text', '$12::text')})
               OR NOT (
@@ -6639,6 +7017,7 @@ async function failRuntimeMapRowChunks(
         input.attemptExpiresAt,
         input.attemptSeq,
         input.forceTerminal === true,
+        input.writeVersion,
       ],
     );
     updated += Number(rows[0]?.updated ?? 0);
@@ -6649,9 +7028,8 @@ async function failRuntimeMapRowChunks(
 
 /**
  * Mark map rows terminal in the per-run scoped Postgres sheet table by key.
- * Mirrors server-side `store.completeSheetRows` semantics: UPDATE-by-key with
- * the row's enriched output values written into materialized physical
- * columns, _status flipped to 'enriched', and the sheet cursor advanced.
+ * Writes enriched output values into materialized physical columns, flips
+ * _status to 'enriched', and advances the sheet cursor.
  *
  * Rows with `status: 'failed'` instead persist a row-isolated failure
  * (`_status='failed'`, `_error`, per-cell failure meta) while keeping the
@@ -6674,6 +7052,7 @@ export async function completeRuntimeMapRows(
     attemptOwnerRunId?: string | null;
     attemptExpiresAt?: string | null;
     attemptSeq?: number | null;
+    writeVersion?: number | null;
     forceFailedRows?: boolean;
   },
 ): Promise<RuntimeMapRowsWriteResult> {
@@ -6757,6 +7136,15 @@ export async function completeRuntimeMapRows(
     input.attemptOwnerRunId?.trim() || (attemptId ? input.runId : null);
   const attemptExpiresAt = input.attemptExpiresAt?.trim() || null;
   const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
+  const writeVersion = input.writeVersion == null ? null : input.writeVersion;
+  if (
+    writeVersion !== null &&
+    (!Number.isSafeInteger(writeVersion) || writeVersion <= 0)
+  ) {
+    throw new Error(
+      'Runtime sheet writeVersion must be a positive safe integer.',
+    );
+  }
 
   return await withRuntimeSheetQueryClient(
     context,
@@ -6780,6 +7168,7 @@ export async function completeRuntimeMapRows(
                 attemptOwnerRunId,
                 attemptExpiresAt,
                 attemptSeq,
+                writeVersion,
                 normalizedPlayName: normalizePlayNameForSheet(session.playName),
                 normalizedTableNamespace: normalizeTableNamespace(
                   input.tableNamespace,
@@ -6813,6 +7202,7 @@ export async function completeRuntimeMapRows(
               attemptOwnerRunId,
               attemptExpiresAt,
               attemptSeq,
+              writeVersion,
               normalizedPlayName: normalizePlayNameForSheet(session.playName),
               normalizedTableNamespace: normalizeTableNamespace(
                 input.tableNamespace,
@@ -6822,6 +7212,8 @@ export async function completeRuntimeMapRows(
       return {
         updated: completed.updated + failed.updated,
         fencedKeys: [...completed.fencedKeys, ...failed.fencedKeys],
+        staleDropped: completed.fencedKeys.length + failed.fencedKeys.length,
+        staleDroppedKeys: [...completed.fencedKeys, ...failed.fencedKeys],
       };
     },
   );