deepline 0.1.211 → 0.1.213

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +317 -340
  2. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/customer-console.ts +23 -0
  3. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +1 -1
  4. package/dist/bundling-sources/sdk/src/client.ts +118 -3
  5. package/dist/bundling-sources/sdk/src/index.ts +2 -0
  6. package/dist/bundling-sources/sdk/src/play.ts +8 -1
  7. package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +1 -1
  8. package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +11 -1
  9. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  10. package/dist/bundling-sources/sdk/src/types.ts +47 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +302 -30
  12. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-placement.ts +14 -0
  13. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +57 -23
  14. package/dist/bundling-sources/shared_libs/play-runtime/completed-receipt-cache.ts +166 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +164 -121
  16. package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +2 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +42 -1
  18. package/dist/bundling-sources/shared_libs/play-runtime/dataset-id.ts +10 -4
  19. package/dist/bundling-sources/shared_libs/play-runtime/db-session.ts +9 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +163 -7
  21. package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts +93 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +14 -2
  23. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +4 -2
  24. package/dist/bundling-sources/shared_libs/play-runtime/play-call-execution.ts +1 -0
  25. package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +16 -1
  26. package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +21 -1
  27. package/dist/bundling-sources/shared_libs/play-runtime/receipt-state-sink.ts +10 -0
  28. package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +80 -3
  29. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +128 -6
  30. package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +7 -0
  31. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +94 -0
  32. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +91 -6
  33. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +19 -10
  34. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +540 -92
  35. package/dist/bundling-sources/shared_libs/play-runtime/runtime-postgres-admission.ts +162 -0
  36. package/dist/bundling-sources/shared_libs/play-runtime/secret-capability.ts +18 -4
  37. package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +16 -17
  38. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +29 -9
  39. package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +22 -1
  40. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +1 -0
  41. package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +79 -1
  42. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +2 -4
  43. package/dist/cli/index.js +637 -229
  44. package/dist/cli/index.mjs +637 -229
  45. package/dist/{compiler-manifest-j6z8qzpd.d.mts → compiler-manifest-BhgZ23A4.d.mts} +1 -0
  46. package/dist/{compiler-manifest-j6z8qzpd.d.ts → compiler-manifest-BhgZ23A4.d.ts} +1 -0
  47. package/dist/index.d.mts +84 -11
  48. package/dist/index.d.ts +84 -11
  49. package/dist/index.js +225 -43
  50. package/dist/index.mjs +225 -43
  51. package/dist/plays/bundle-play-file.d.mts +2 -2
  52. package/dist/plays/bundle-play-file.d.ts +2 -2
  53. package/dist/plays/bundle-play-file.mjs +65 -4
  54. package/package.json +1 -1
@@ -105,7 +105,13 @@ import {
105
105
  PLAY_RUNTIME_TEST_FAULT_HEADER,
106
106
  type RuntimeTestFaultName,
107
107
  } from './test-runtime-seams';
108
+ import { COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID } from './durable-receipt-execution';
108
109
  import { vercelProtectionBypassHeader } from './vercel-protection';
110
+ import {
111
+ RuntimePostgresAdmission,
112
+ type RuntimePostgresAdmissionSnapshot,
113
+ type RuntimePostgresLane,
114
+ } from './runtime-postgres-admission';
109
115
 
110
116
  type RuntimeApiContext = {
111
117
  baseUrl?: string | null;
@@ -127,6 +133,8 @@ type RuntimeApiContext = {
127
133
  postgresSessionUnwrapKey?: string | null;
128
134
  };
129
135
 
136
+ const RUNTIME_RUN_DATASET_CATALOG_TABLE = '_deepline_run_datasets';
137
+
130
138
  type DirectRuntimeTestFaultState = {
131
139
  headerValue: string;
132
140
  counts: Partial<Record<RuntimeTestFaultName, number>>;
@@ -267,6 +275,7 @@ type RuntimeSheetPreparedRowDisposition = {
267
275
  const dbSessionCache = new Map<string, DbSessionCacheEntry>();
268
276
  const dbSessionInFlight = new Map<string, Promise<CreateDbSessionResponse>>();
269
277
  const postgresPools = new Map<string, RuntimePool>();
278
+ const runtimePostgresAdmissions = new Map<string, RuntimePostgresAdmission>();
270
279
  const runtimeWorkReceiptEnsureCache = new Map<string, Promise<void>>();
271
280
  const runtimeSheetEnsureCache = new Map<
272
281
  string,
@@ -325,7 +334,14 @@ const RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS = 10_000;
325
334
  // Daytona executes one play per runner process. This is the bounded data-plane
326
335
  // budget for that one run's sheet reads/writes, receipts, and terminal/export
327
336
  // reads over the PgBouncer URL; it is not a scheduler/control-plane fan-out.
328
- const RUNTIME_POSTGRES_POOL_MAX_CONNECTIONS = 8;
337
+ // One runner gets four tenant-gateway clients, split evenly so a large sheet
338
+ // flush cannot starve receipt claims (and receipt traffic cannot block the
339
+ // persistence barrier). The admission queues add a driver-independent bound:
340
+ // Neon Pool and node-postgres may queue internally, but runtime callers never
341
+ // enter those unobservable queues without first holding a lane permit.
342
+ const RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE = 2;
343
+ const RUNTIME_POSTGRES_ADMISSION_MAX_QUEUED_PER_LANE = 2_000;
344
+ const RUNTIME_POSTGRES_ADMISSION_TIMEOUT_MS = 10_000;
329
345
  const RECEIPT_STATUS_QUEUED = RECEIPT_STATUS_CODE.queued;
330
346
  const RECEIPT_STATUS_PENDING = RECEIPT_STATUS_CODE.pending;
331
347
  const RECEIPT_STATUS_RUNNING = RECEIPT_STATUS_CODE.running;
@@ -367,6 +383,7 @@ export type PrepareRuntimeSheetResult = {
367
383
  attemptOwnerRunId?: string;
368
384
  attemptExpiresAt?: string;
369
385
  attemptSeq?: number;
386
+ writeVersion?: number;
370
387
  };
371
388
 
372
389
  export type RuntimeSheetTiming = {
@@ -1290,9 +1307,7 @@ async function deleteRuntimeDbSessionCacheEntry(
1290
1307
  ): Promise<void> {
1291
1308
  dbSessionCache.delete(cacheKey);
1292
1309
  if (session.postgresUrl) {
1293
- const pool = postgresPools.get(session.postgresUrl);
1294
- postgresPools.delete(session.postgresUrl);
1295
- await pool?.end().catch(() => {});
1310
+ await resetRuntimePostgresPool(session.postgresUrl);
1296
1311
  }
1297
1312
  }
1298
1313
 
@@ -1413,18 +1428,10 @@ async function prewarmRuntimePostgresSession(
1413
1428
  attempt >= RUNTIME_POSTGRES_PREWARM_MAX_ATTEMPTS ||
1414
1429
  !isTransientRuntimePostgresConnectionError(error)
1415
1430
  ) {
1416
- const pool = postgresPools.get(session.postgresUrl);
1417
- postgresPools.delete(session.postgresUrl);
1418
- if (pool) {
1419
- await Promise.resolve(pool.end()).catch(() => {});
1420
- }
1431
+ await resetRuntimePostgresPool(session.postgresUrl);
1421
1432
  throw error;
1422
1433
  }
1423
- const pool = postgresPools.get(session.postgresUrl);
1424
- postgresPools.delete(session.postgresUrl);
1425
- if (pool) {
1426
- await Promise.resolve(pool.end()).catch(() => {});
1427
- }
1434
+ await resetRuntimePostgresPool(session.postgresUrl);
1428
1435
  await sleep(
1429
1436
  RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS[attempt - 1] ??
1430
1437
  RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS[
@@ -1489,7 +1496,10 @@ function sleep(ms: number): Promise<void> {
1489
1496
 
1490
1497
  async function connectRuntimePostgresPool(
1491
1498
  pool: RuntimePool,
1499
+ admission: RuntimePostgresAdmission,
1500
+ lane: RuntimePostgresLane,
1492
1501
  ): Promise<RuntimePoolClient> {
1502
+ const releaseAdmission = await admission.acquire(lane);
1493
1503
  let timedOut = false;
1494
1504
  let timeout: ReturnType<typeof setTimeout> | null = null;
1495
1505
  const connectPromise = pool.connect();
@@ -1497,11 +1507,12 @@ async function connectRuntimePostgresPool(
1497
1507
  .then((client) => {
1498
1508
  if (timedOut) {
1499
1509
  client.release();
1510
+ releaseAdmission();
1500
1511
  }
1501
1512
  })
1502
1513
  .catch(() => {});
1503
1514
  try {
1504
- return await Promise.race([
1515
+ const client = await Promise.race([
1505
1516
  connectPromise,
1506
1517
  new Promise<never>((_, reject) => {
1507
1518
  timeout = setTimeout(() => {
@@ -1514,6 +1525,21 @@ async function connectRuntimePostgresPool(
1514
1525
  }, RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS);
1515
1526
  }),
1516
1527
  ]);
1528
+ let released = false;
1529
+ return {
1530
+ query: (text, params) => client.query(text, params),
1531
+ release: () => {
1532
+ if (released) return;
1533
+ released = true;
1534
+ client.release();
1535
+ releaseAdmission();
1536
+ },
1537
+ };
1538
+ } catch (error) {
1539
+ if (!timedOut) {
1540
+ releaseAdmission();
1541
+ }
1542
+ throw error;
1517
1543
  } finally {
1518
1544
  if (timeout) {
1519
1545
  clearTimeout(timeout);
@@ -1521,35 +1547,81 @@ async function connectRuntimePostgresPool(
1521
1547
  }
1522
1548
  }
1523
1549
 
1524
- function getPostgresPool(postgresUrl: string, cachePool = true): RuntimePool {
1550
+ function runtimePostgresLane(
1551
+ session: RuntimePostgresSession,
1552
+ ): RuntimePostgresLane {
1553
+ return session.target.logicalTable === RUNTIME_WORK_RECEIPT_LOGICAL_TABLE
1554
+ ? 'receipts'
1555
+ : 'sheets';
1556
+ }
1557
+
1558
+ function runtimePostgresPoolKey(
1559
+ postgresUrl: string,
1560
+ lane: RuntimePostgresLane,
1561
+ ): string {
1562
+ return `${postgresUrl}::${lane}`;
1563
+ }
1564
+
1565
+ function getRuntimePostgresAdmission(
1566
+ postgresUrl: string,
1567
+ ): RuntimePostgresAdmission {
1568
+ const existing = runtimePostgresAdmissions.get(postgresUrl);
1569
+ if (existing) return existing;
1570
+ const admission = new RuntimePostgresAdmission({
1571
+ maxActivePerLane: RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE,
1572
+ maxQueuedPerLane: RUNTIME_POSTGRES_ADMISSION_MAX_QUEUED_PER_LANE,
1573
+ acquireTimeoutMs: RUNTIME_POSTGRES_ADMISSION_TIMEOUT_MS,
1574
+ });
1575
+ runtimePostgresAdmissions.set(postgresUrl, admission);
1576
+ return admission;
1577
+ }
1578
+
1579
+ export function runtimePostgresAdmissionTelemetry(
1580
+ postgresUrl: string,
1581
+ ): RuntimePostgresAdmissionSnapshot | null {
1582
+ return runtimePostgresAdmissions.get(postgresUrl)?.snapshot() ?? null;
1583
+ }
1584
+
1585
+ function getPostgresPool(
1586
+ postgresUrl: string,
1587
+ lane: RuntimePostgresLane,
1588
+ cachePool = true,
1589
+ ): RuntimePool {
1525
1590
  if (!cachePool) {
1526
1591
  return createRuntimePool({
1527
1592
  connectionString: postgresUrl,
1528
- maxConnections: 1,
1593
+ maxConnections: RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE,
1529
1594
  idleTimeoutMs: 0,
1530
1595
  connectTimeoutMs: RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS,
1531
1596
  });
1532
1597
  }
1533
- const existing = postgresPools.get(postgresUrl);
1598
+ const poolKey = runtimePostgresPoolKey(postgresUrl, lane);
1599
+ const existing = postgresPools.get(poolKey);
1534
1600
  if (existing) {
1535
1601
  return existing;
1536
1602
  }
1537
1603
  const pool = createRuntimePool({
1538
1604
  connectionString: postgresUrl,
1539
- maxConnections: RUNTIME_POSTGRES_POOL_MAX_CONNECTIONS,
1605
+ maxConnections: RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE,
1540
1606
  idleTimeoutMs: 15_000,
1541
1607
  connectTimeoutMs: RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS,
1542
1608
  });
1543
- postgresPools.set(postgresUrl, pool);
1609
+ postgresPools.set(poolKey, pool);
1544
1610
  return pool;
1545
1611
  }
1546
1612
 
1547
1613
  async function resetRuntimePostgresPool(postgresUrl: string): Promise<void> {
1548
- const pool = postgresPools.get(postgresUrl);
1549
- postgresPools.delete(postgresUrl);
1550
- if (pool) {
1551
- await Promise.resolve(pool.end()).catch(() => {});
1552
- }
1614
+ const pools = (['receipts', 'sheets'] as const)
1615
+ .map((lane) => runtimePostgresPoolKey(postgresUrl, lane))
1616
+ .map((key) => {
1617
+ const pool = postgresPools.get(key);
1618
+ postgresPools.delete(key);
1619
+ return pool;
1620
+ })
1621
+ .filter((pool): pool is RuntimePool => pool !== undefined);
1622
+ await Promise.all(
1623
+ pools.map((pool) => Promise.resolve(pool.end()).catch(() => {})),
1624
+ );
1553
1625
  }
1554
1626
 
1555
1627
  async function withRuntimePostgres<T>(
@@ -1565,18 +1637,30 @@ async function withRuntimePostgres<T>(
1565
1637
  options.maxConnectAttempts === undefined
1566
1638
  ? RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS
1567
1639
  : Math.max(1, Math.floor(options.maxConnectAttempts));
1640
+ const lane = runtimePostgresLane(session);
1641
+ const admission = getRuntimePostgresAdmission(session.postgresUrl);
1568
1642
  for (let attempt = 1; attempt <= maxConnectAttempts; attempt += 1) {
1569
1643
  try {
1570
- const pool = getPostgresPool(session.postgresUrl, cachePool);
1644
+ const pool = getPostgresPool(session.postgresUrl, lane, cachePool);
1571
1645
  if (!cachePool) {
1572
1646
  requestLocalPool = pool;
1573
1647
  }
1574
- client = await connectRuntimePostgresPool(pool);
1648
+ client = await connectRuntimePostgresPool(pool, admission, lane);
1649
+ if (session.executionRole) {
1650
+ await client.query(
1651
+ `SET ROLE ${quoteIdentifier(session.executionRole)}`,
1652
+ );
1653
+ }
1575
1654
  break;
1576
1655
  } catch (error) {
1656
+ if (client) {
1657
+ client.release();
1658
+ client = null;
1659
+ }
1577
1660
  if (cachePool) {
1578
- const pool = postgresPools.get(session.postgresUrl);
1579
- postgresPools.delete(session.postgresUrl);
1661
+ const poolKey = runtimePostgresPoolKey(session.postgresUrl, lane);
1662
+ const pool = postgresPools.get(poolKey);
1663
+ postgresPools.delete(poolKey);
1580
1664
  if (pool) {
1581
1665
  await Promise.resolve(pool.end()).catch(() => {});
1582
1666
  }
@@ -1604,7 +1688,13 @@ async function withRuntimePostgres<T>(
1604
1688
  try {
1605
1689
  return await fn(client);
1606
1690
  } finally {
1607
- client.release();
1691
+ try {
1692
+ if (session.executionRole) {
1693
+ await client.query('RESET ROLE');
1694
+ }
1695
+ } finally {
1696
+ client.release();
1697
+ }
1608
1698
  if (requestLocalPool) {
1609
1699
  await Promise.resolve(requestLocalPool.end()).catch(() => {});
1610
1700
  }
@@ -1833,6 +1923,38 @@ function columnSummaryTable(session: RuntimePostgresSession): string {
1833
1923
  return fqRuntimeTable(session, session.postgres.columnSummaryTable);
1834
1924
  }
1835
1925
 
1926
+ function runDatasetCatalogTable(session: RuntimePostgresSession): string {
1927
+ return fqRuntimeTable(session, RUNTIME_RUN_DATASET_CATALOG_TABLE);
1928
+ }
1929
+
1930
+ async function registerRuntimeDataset(
1931
+ session: RuntimePostgresSession,
1932
+ input: { playName: string; tableNamespace: string; runId: string },
1933
+ ): Promise<void> {
1934
+ const tableNamespace = normalizeTableNamespace(input.tableNamespace);
1935
+ const playName = normalizePlayNameForSheet(input.playName);
1936
+ const datasetId = createRuntimeDatasetId(playName, tableNamespace);
1937
+ await withRuntimePostgres(session, async (client) => {
1938
+ await client.query(
1939
+ `INSERT INTO ${runDatasetCatalogTable(session)} (
1940
+ run_id, dataset_id, play_name, table_namespace, public_path
1941
+ ) VALUES ($1, $2, $3, $4, $5)
1942
+ ON CONFLICT (run_id, dataset_id) DO UPDATE SET
1943
+ play_name = EXCLUDED.play_name,
1944
+ table_namespace = EXCLUDED.table_namespace,
1945
+ public_path = EXCLUDED.public_path,
1946
+ updated_at = now()`,
1947
+ [
1948
+ input.runId,
1949
+ datasetId,
1950
+ playName,
1951
+ tableNamespace,
1952
+ `datasets.${tableNamespace}`,
1953
+ ],
1954
+ );
1955
+ });
1956
+ }
1957
+
1836
1958
  function workReceiptTable(session: RuntimePostgresSession): string {
1837
1959
  return fqRuntimeTable(
1838
1960
  session,
@@ -1840,6 +1962,10 @@ function workReceiptTable(session: RuntimePostgresSession): string {
1840
1962
  );
1841
1963
  }
1842
1964
 
1965
+ function receiptExecutionLockTable(session: RuntimePostgresSession): string {
1966
+ return fqRuntimeTable(session, '_deepline_receipt_execution_locks');
1967
+ }
1968
+
1843
1969
  function postgresUrlCacheKey(value: string): string {
1844
1970
  return createHash('sha256').update(value).digest('hex').slice(0, 24);
1845
1971
  }
@@ -1981,6 +2107,14 @@ async function ensureRuntimeWorkReceiptTable(
1981
2107
  `,
1982
2108
  )
1983
2109
  .then(async () => {
2110
+ await client.query(`
2111
+ CREATE TABLE IF NOT EXISTS ${receiptExecutionLockTable(session)} (
2112
+ k bytea PRIMARY KEY,
2113
+ owner_execution_id text NOT NULL,
2114
+ expires_at timestamptz NOT NULL,
2115
+ updated_at timestamptz NOT NULL DEFAULT now()
2116
+ )
2117
+ `);
1984
2118
  const missingColumns = await missingRuntimeWorkReceiptSelfHealColumns(
1985
2119
  session,
1986
2120
  client,
@@ -2015,6 +2149,77 @@ async function ensureRuntimeWorkReceiptTable(
2015
2149
  }
2016
2150
  }
2017
2151
 
2152
+ export async function acquireRuntimeReceiptExecutionLock(
2153
+ context: RuntimeApiContext,
2154
+ input: {
2155
+ playName: string;
2156
+ key: string;
2157
+ ownerExecutionId: string;
2158
+ ttlMs: number;
2159
+ },
2160
+ ): Promise<{ ownerExecutionId: string; expiresAt: string } | null> {
2161
+ const ttlMs = Math.floor(input.ttlMs);
2162
+ if (!Number.isFinite(ttlMs) || ttlMs <= 0 || ttlMs > 10 * 60_000) {
2163
+ throw new Error('Runtime receipt execution lock TTL must be 1..600000ms.');
2164
+ }
2165
+ const ownerExecutionId = input.ownerExecutionId.trim();
2166
+ if (!ownerExecutionId) {
2167
+ throw new Error('Runtime receipt execution lock owner is required.');
2168
+ }
2169
+ const session = await getRuntimeWorkReceiptSession(context, input);
2170
+ return await withRuntimeWorkReceiptClient(
2171
+ context,
2172
+ session,
2173
+ async (client) => {
2174
+ const { rows } = await client.query<Record<string, unknown>>(
2175
+ `
2176
+ INSERT INTO ${receiptExecutionLockTable(session)} AS target (
2177
+ k, owner_execution_id, expires_at, updated_at
2178
+ ) VALUES (
2179
+ decode($1, 'hex'), $2, now() + ($3::double precision * interval '1 millisecond'), now()
2180
+ )
2181
+ ON CONFLICT (k) DO UPDATE
2182
+ SET owner_execution_id = EXCLUDED.owner_execution_id,
2183
+ expires_at = EXCLUDED.expires_at,
2184
+ updated_at = now()
2185
+ WHERE target.expires_at <= now()
2186
+ OR target.owner_execution_id = EXCLUDED.owner_execution_id
2187
+ RETURNING owner_execution_id, expires_at
2188
+ `,
2189
+ [workReceiptKeyHex(input.key), ownerExecutionId, ttlMs],
2190
+ );
2191
+ const row = rows[0];
2192
+ return row
2193
+ ? {
2194
+ ownerExecutionId: String(row.owner_execution_id),
2195
+ expiresAt: new Date(String(row.expires_at)).toISOString(),
2196
+ }
2197
+ : null;
2198
+ },
2199
+ );
2200
+ }
2201
+
2202
+ export async function releaseRuntimeReceiptExecutionLock(
2203
+ context: RuntimeApiContext,
2204
+ input: { playName: string; key: string; ownerExecutionId: string },
2205
+ ): Promise<boolean> {
2206
+ const session = await getRuntimeWorkReceiptSession(context, input);
2207
+ return await withRuntimeWorkReceiptClient(
2208
+ context,
2209
+ session,
2210
+ async (client) => {
2211
+ const { rows } = await client.query(
2212
+ `DELETE FROM ${receiptExecutionLockTable(session)}
2213
+ WHERE k = decode($1, 'hex')
2214
+ AND owner_execution_id = $2
2215
+ RETURNING 1`,
2216
+ [workReceiptKeyHex(input.key), input.ownerExecutionId],
2217
+ );
2218
+ return rows.length > 0;
2219
+ },
2220
+ );
2221
+ }
2222
+
2018
2223
  async function withRuntimeWorkReceiptClient<T>(
2019
2224
  context: RuntimeApiContext,
2020
2225
  session: RuntimePostgresSession,
@@ -2022,8 +2227,11 @@ async function withRuntimeWorkReceiptClient<T>(
2022
2227
  ): Promise<T> {
2023
2228
  // Receipt tables are part of customer Postgres bootstrap, so the hot path
2024
2229
  // should not pay DDL on every fresh runtime isolate. Try the receipt query
2025
- // first, then self-heal once on a missing relation/column. A genuinely
2026
- // missing schema, permission error, or second failure still rethrows loudly.
2230
+ // first, then self-heal once on a missing relation/column. Permission loss
2231
+ // during an operational receipt query must fail loud: silently repairing it
2232
+ // can let provider work continue after durable receipt persistence is gone.
2233
+ // Only the schema-migration branch below may repair ownership before its
2234
+ // retry. A genuinely missing schema or second failure also rethrows loudly.
2027
2235
  const runWithSelfHeal = async (client: RuntimeQueryClient): Promise<T> => {
2028
2236
  try {
2029
2237
  return await operation(client);
@@ -2048,10 +2256,6 @@ async function withRuntimeWorkReceiptClient<T>(
2048
2256
  playName: context.playName?.trim() || session.playName,
2049
2257
  });
2050
2258
  }
2051
- } else if (isPostgresPermissionDeniedError(error)) {
2052
- await repairRuntimeStorageGrants(context, {
2053
- playName: context.playName?.trim() || session.playName,
2054
- });
2055
2259
  } else {
2056
2260
  throw error;
2057
2261
  }
@@ -2225,6 +2429,31 @@ async function mintRuntimeSheetAttemptExpiresAt(
2225
2429
  throw new Error('Runtime sheet attempt expiry mint did not return a value.');
2226
2430
  }
2227
2431
 
2432
+ async function mintRuntimeSheetWriteVersion(
2433
+ client: RuntimeQueryClient,
2434
+ session: RuntimePostgresSession,
2435
+ requested?: number | null,
2436
+ ): Promise<number> {
2437
+ if (requested != null) {
2438
+ if (!Number.isSafeInteger(requested) || requested <= 0) {
2439
+ throw new Error(
2440
+ 'Runtime sheet writeVersion must be a positive safe integer.',
2441
+ );
2442
+ }
2443
+ return requested;
2444
+ }
2445
+ const { rows } = await client.query<{ write_version: number | string }>(
2446
+ `SELECT ${nextRuntimeSheetVersionExpression(session)} AS write_version`,
2447
+ );
2448
+ const value = Number(rows[0]?.write_version);
2449
+ if (!Number.isSafeInteger(value) || value <= 0) {
2450
+ throw new Error(
2451
+ 'Runtime sheet write version allocation did not return a safe integer.',
2452
+ );
2453
+ }
2454
+ return value;
2455
+ }
2456
+
2228
2457
  function parseRuntimeCellMeta(value: unknown): Record<string, unknown> {
2229
2458
  if (value && typeof value === 'object' && !Array.isArray(value)) {
2230
2459
  return value as Record<string, unknown>;
@@ -2725,6 +2954,7 @@ async function prepareRuntimeSheetDatasetRows(
2725
2954
  attemptOwnerRunId: string;
2726
2955
  attemptExpiresAt: string;
2727
2956
  attemptSeq: number;
2957
+ writeVersion: number;
2728
2958
  physicalInsertColumnsSql: string;
2729
2959
  physicalInsertValuesSql: string;
2730
2960
  physicalRefreshSetSql: string;
@@ -2803,6 +3033,44 @@ async function prepareRuntimeSheetDatasetRows(
2803
3033
  ON index_values.ord = key_values.ord
2804
3034
  ORDER BY key_values._key, key_values.ord
2805
3035
  ),
3036
+ preexisting_rows AS (
3037
+ SELECT target._key
3038
+ FROM ${sheetTable(session)} AS target
3039
+ JOIN input_rows ON input_rows._key = target._key
3040
+ ),
3041
+ versioned_completed_rows AS (
3042
+ UPDATE ${sheetTable(session)} AS target
3043
+ SET _run_id = $4::text,
3044
+ _writer_run_id = $4::text,
3045
+ _write_version = $12::bigint,
3046
+ _input_index = input_rows._input_index,
3047
+ _updated_at = now(),
3048
+ _version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalRefreshSetSql}
3049
+ FROM input_rows
3050
+ WHERE target._key = input_rows._key
3051
+ AND target._status = 'enriched'
3052
+ AND COALESCE(target._write_version, 0) < $12::bigint
3053
+ RETURNING target._key,
3054
+ ($11::boolean OR (${targetMissingOutputSql})) AS needs_recompute
3055
+ ),
3056
+ versioned_failed_rows AS (
3057
+ UPDATE ${sheetTable(session)} AS target
3058
+ SET _run_id = $4::text,
3059
+ _writer_run_id = $4::text,
3060
+ _write_version = $12::bigint,
3061
+ _input_index = input_rows._input_index,
3062
+ _attempt_id = $7::text,
3063
+ _attempt_owner_run_id = $8::text,
3064
+ _attempt_expires_at = NULL,
3065
+ _attempt_seq = $10::integer,
3066
+ _updated_at = now(),
3067
+ _version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalRefreshSetSql}
3068
+ FROM input_rows
3069
+ WHERE target._key = input_rows._key
3070
+ AND target._status = 'failed'
3071
+ AND COALESCE(target._write_version, 0) < $12::bigint
3072
+ RETURNING target._key
3073
+ ),
2806
3074
  existing_rows AS (
2807
3075
  UPDATE ${sheetTable(session)} AS target
2808
3076
  SET _input_index = input_rows._input_index,
@@ -2810,6 +3078,7 @@ async function prepareRuntimeSheetDatasetRows(
2810
3078
  _version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalRefreshSetSql}
2811
3079
  FROM input_rows
2812
3080
  WHERE target._key = input_rows._key
3081
+ AND target._write_version = $12::bigint
2813
3082
  AND target._status <> 'stale'
2814
3083
  AND (
2815
3084
  NOT (${targetNewerTerminalRowSql})
@@ -2833,8 +3102,8 @@ async function prepareRuntimeSheetDatasetRows(
2833
3102
  RETURNING target._key
2834
3103
  ),
2835
3104
  inserted_rows AS (
2836
- INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _attempt_id, _attempt_owner_run_id, _attempt_expires_at, _attempt_seq${input.physicalInsertColumnsSql})
2837
- SELECT _key, 'pending', $4::text, _input_index, $7::text, $8::text, $9::timestamptz, $10::integer${input.physicalInsertValuesSql}
3105
+ INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _attempt_id, _attempt_owner_run_id, _attempt_expires_at, _attempt_seq, _write_version, _writer_run_id${input.physicalInsertColumnsSql})
3106
+ SELECT _key, 'pending', $4::text, _input_index, $7::text, $8::text, $9::timestamptz, $10::integer, $12::bigint, $4::text${input.physicalInsertValuesSql}
2838
3107
  FROM input_rows
2839
3108
  ON CONFLICT (_key) DO UPDATE SET
2840
3109
  _status = 'pending',
@@ -2844,6 +3113,8 @@ async function prepareRuntimeSheetDatasetRows(
2844
3113
  _attempt_owner_run_id = EXCLUDED._attempt_owner_run_id,
2845
3114
  _attempt_expires_at = EXCLUDED._attempt_expires_at,
2846
3115
  _attempt_seq = EXCLUDED._attempt_seq,
3116
+ _write_version = EXCLUDED._write_version,
3117
+ _writer_run_id = EXCLUDED._writer_run_id,
2847
3118
  _updated_at = now(),
2848
3119
  _version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalUpsertSetSql}
2849
3120
  WHERE ${sheetTable(session)}._status = 'stale'
@@ -2863,10 +3134,30 @@ async function prepareRuntimeSheetDatasetRows(
2863
3134
  FROM input_rows
2864
3135
  WHERE target._key = input_rows._key
2865
3136
  AND target._status = 'enriched'
3137
+ AND target._write_version = $12::bigint
2866
3138
  AND (${targetMissingOutputSql})
2867
3139
  AND COALESCE(target._attempt_seq, 0) <= $10::integer
2868
3140
  RETURNING target._key
2869
3141
  ),
3142
+ superseded_rows AS (
3143
+ UPDATE ${sheetTable(session)} AS target
3144
+ SET _run_id = $4::text,
3145
+ _input_index = input_rows._input_index,
3146
+ _attempt_id = $7::text,
3147
+ _attempt_owner_run_id = $8::text,
3148
+ _attempt_expires_at = NULL,
3149
+ _attempt_seq = $10::integer,
3150
+ _write_version = $12::bigint,
3151
+ _writer_run_id = $4::text,
3152
+ _error = NULL,
3153
+ _updated_at = now(),
3154
+ _version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalRefreshSetSql}
3155
+ FROM input_rows
3156
+ WHERE target._key = input_rows._key
3157
+ AND target._status IN ('pending', 'running')
3158
+ AND COALESCE(target._write_version, 0) < $12::bigint
3159
+ RETURNING target._key
3160
+ ),
2870
3161
  claimed_existing_rows AS (
2871
3162
  UPDATE ${sheetTable(session)} AS target
2872
3163
  SET _run_id = $4::text,
@@ -2880,6 +3171,7 @@ async function prepareRuntimeSheetDatasetRows(
2880
3171
  FROM input_rows
2881
3172
  WHERE target._key = input_rows._key
2882
3173
  AND target._status IN ('pending', 'running', 'failed')
3174
+ AND target._write_version = $12::bigint
2883
3175
  AND NOT (${targetNewerTerminalRowSql})
2884
3176
  AND (
2885
3177
  ${targetAttemptFenceSql}
@@ -2902,6 +3194,9 @@ async function prepareRuntimeSheetDatasetRows(
2902
3194
  FROM inserted_rows
2903
3195
  UNION
2904
3196
  SELECT _key
3197
+ FROM superseded_rows
3198
+ UNION
3199
+ SELECT _key
2905
3200
  FROM missing_output_rows
2906
3201
  UNION
2907
3202
  SELECT _key
@@ -2911,10 +3206,18 @@ async function prepareRuntimeSheetDatasetRows(
2911
3206
  FROM existing_rows
2912
3207
  WHERE $11::boolean
2913
3208
  UNION
3209
+ SELECT _key
3210
+ FROM versioned_completed_rows
3211
+ WHERE needs_recompute
3212
+ UNION
3213
+ SELECT _key FROM versioned_failed_rows
3214
+ UNION
2914
3215
  SELECT existing._key
2915
3216
  FROM ${sheetTable(session)} AS existing
2916
3217
  JOIN input_rows ON input_rows._key = existing._key
2917
3218
  WHERE (
3219
+ existing._write_version = $12::bigint
3220
+ AND (
2918
3221
  existing._status IN ('pending', 'running', 'failed')
2919
3222
  AND NOT (${existingNewerTerminalRowSql})
2920
3223
  AND (
@@ -2927,8 +3230,10 @@ async function prepareRuntimeSheetDatasetRows(
2927
3230
  AND COALESCE(existing._attempt_seq, 0) < $10::integer
2928
3231
  )
2929
3232
  )
2930
- )
3233
+ ))
2931
3234
  OR (
3235
+ existing._write_version = $12::bigint
3236
+ AND
2932
3237
  existing._status = 'enriched'
2933
3238
  AND (${existingMissingOutputSql})
2934
3239
  AND (
@@ -2947,6 +3252,9 @@ async function prepareRuntimeSheetDatasetRows(
2947
3252
  AND NOT EXISTS (
2948
3253
  SELECT 1 FROM pending_rows WHERE pending_rows._key = existing._key
2949
3254
  )
3255
+ UNION
3256
+ SELECT _key FROM versioned_completed_rows
3257
+ WHERE NOT needs_recompute
2950
3258
  ),
2951
3259
  blocked_rows AS (
2952
3260
  SELECT input_rows._key
@@ -2969,7 +3277,11 @@ async function prepareRuntimeSheetDatasetRows(
2969
3277
  FROM blocked_rows
2970
3278
  ),
2971
3279
  inserted_count_cte AS (
2972
- SELECT count(*)::bigint AS c FROM inserted_rows
3280
+ SELECT count(*)::bigint AS c
3281
+ FROM inserted_rows
3282
+ WHERE NOT EXISTS (
3283
+ SELECT 1 FROM preexisting_rows WHERE preexisting_rows._key = inserted_rows._key
3284
+ )
2973
3285
  ),
2974
3286
  missing_output_count_cte AS (
2975
3287
  SELECT count(*)::bigint AS c FROM missing_output_rows
@@ -3029,6 +3341,7 @@ async function prepareRuntimeSheetDatasetRows(
3029
3341
  input.attemptExpiresAt,
3030
3342
  input.attemptSeq,
3031
3343
  input.force === true,
3344
+ input.writeVersion,
3032
3345
  ],
3033
3346
  );
3034
3347
  inserted += Number(rows[0]?.inserted_count ?? 0);
@@ -3065,6 +3378,7 @@ async function buildRuntimeSheetDatasetStartResult(
3065
3378
  attemptOwnerRunId: string;
3066
3379
  attemptExpiresAt: string;
3067
3380
  attemptSeq: number;
3381
+ writeVersion: number;
3068
3382
  inserted: number;
3069
3383
  rowDispositions: RuntimeSheetPreparedRowDisposition[];
3070
3384
  force?: boolean;
@@ -3100,6 +3414,7 @@ async function buildRuntimeSheetDatasetStartResult(
3100
3414
  completedRows: [],
3101
3415
  blockedRows: [],
3102
3416
  tableNamespace: input.tableNamespace,
3417
+ writeVersion: input.writeVersion,
3103
3418
  };
3104
3419
  }
3105
3420
 
@@ -3119,6 +3434,7 @@ async function buildRuntimeSheetDatasetStartResult(
3119
3434
  attemptOwnerRunId: input.attemptOwnerRunId,
3120
3435
  attemptExpiresAt: input.attemptExpiresAt,
3121
3436
  attemptSeq: input.attemptSeq,
3437
+ writeVersion: input.writeVersion,
3122
3438
  normalizedPlayName: input.normalizedPlayName,
3123
3439
  normalizedTableNamespace: input.normalizedTableNamespace,
3124
3440
  outputFields: input.force === true ? datasetFields : [],
@@ -3149,6 +3465,7 @@ async function buildRuntimeSheetDatasetStartResult(
3149
3465
  completedRows: [],
3150
3466
  blockedRows,
3151
3467
  tableNamespace: input.tableNamespace,
3468
+ writeVersion: input.writeVersion,
3152
3469
  };
3153
3470
  }
3154
3471
  const persistedRows = await readRuntimeRowsByKey(
@@ -3198,6 +3515,7 @@ async function buildRuntimeSheetDatasetStartResult(
3198
3515
  completedRows,
3199
3516
  blockedRows,
3200
3517
  tableNamespace: input.tableNamespace,
3518
+ writeVersion: input.writeVersion ?? undefined,
3201
3519
  };
3202
3520
  }
3203
3521
 
@@ -3211,6 +3529,7 @@ async function markRuntimeRowsPendingForRecompute(
3211
3529
  attemptOwnerRunId: string;
3212
3530
  attemptExpiresAt: string;
3213
3531
  attemptSeq: number;
3532
+ writeVersion: number | null;
3214
3533
  normalizedPlayName: string;
3215
3534
  normalizedTableNamespace: string;
3216
3535
  outputFields?: string[];
@@ -3218,28 +3537,13 @@ async function markRuntimeRowsPendingForRecompute(
3218
3537
  },
3219
3538
  ): Promise<void> {
3220
3539
  if (input.keys.length === 0) return;
3221
- const targetAttemptFenceSql = activeRuntimeSheetAttemptFenceSql(
3222
- 'target',
3223
- '$5::text',
3224
- '$6::text',
3225
- '$7::timestamptz',
3226
- '$8::integer',
3227
- );
3228
- const targetOlderForeignAttemptOwnerSql =
3229
- 'coalesce(target._attempt_owner_run_id, target._run_id) IS DISTINCT FROM $6::text AND COALESCE(target._attempt_seq, 0) < $8::integer';
3230
- const targetWritableEnrichedAttemptSql =
3231
- writableEnrichedRuntimeSheetAttemptSql(
3232
- 'target',
3233
- '$6::text',
3234
- '$7::timestamptz',
3235
- '$8',
3236
- );
3237
3540
  const outputFields = [...new Set(input.outputFields ?? [])];
3238
3541
  await client.query(
3239
3542
  `WITH target_rows AS (
3240
3543
  SELECT _key, _status, _cell_meta
3241
3544
  FROM ${sheetTable(session)}
3242
3545
  WHERE _key = ANY($1::text[])
3546
+ AND ($10::boolean OR NOT $10::boolean)
3243
3547
  FOR UPDATE
3244
3548
  ),
3245
3549
  updated AS (
@@ -3250,33 +3554,13 @@ async function markRuntimeRowsPendingForRecompute(
3250
3554
  _attempt_owner_run_id = $6::text,
3251
3555
  _attempt_expires_at = $7::timestamptz,
3252
3556
  _attempt_seq = $8::integer,
3557
+ _write_version = $11::bigint,
3558
+ _writer_run_id = $2::text,
3253
3559
  _updated_at = now(),
3254
3560
  _version = ${nextRuntimeSheetVersionExpression(session)}
3255
3561
  FROM target_rows
3256
3562
  WHERE target._key = target_rows._key
3257
- AND (
3258
- (
3259
- target._status IN ('pending', 'running', 'failed')
3260
- AND (
3261
- ${targetAttemptFenceSql}
3262
- OR ${targetOlderForeignAttemptOwnerSql}
3263
- OR (
3264
- $10::boolean
3265
- AND target._status = 'failed'
3266
- )
3267
- )
3268
- )
3269
- OR ${targetAttemptFenceSql}
3270
- OR (
3271
- $10::boolean
3272
- AND target._status = 'enriched'
3273
- AND COALESCE(target._attempt_seq, 0) <= $8::integer
3274
- )
3275
- OR (
3276
- target._status = 'enriched'
3277
- AND ${targetWritableEnrichedAttemptSql}
3278
- )
3279
- )
3563
+ AND target._write_version = $11::bigint
3280
3564
  AND (
3281
3565
  target._status <> 'pending'
3282
3566
  OR target._run_id IS DISTINCT FROM $2::text
@@ -3366,6 +3650,7 @@ async function markRuntimeRowsPendingForRecompute(
3366
3650
  input.attemptSeq,
3367
3651
  outputFields,
3368
3652
  input.force === true,
3653
+ input.writeVersion,
3369
3654
  ],
3370
3655
  );
3371
3656
  }
@@ -3555,6 +3840,7 @@ export async function claimRuntimeWorkReceipt(
3555
3840
  playName: string;
3556
3841
  runId: string;
3557
3842
  key: string;
3843
+ leaseId?: string | null;
3558
3844
  runAttempt?: number | null;
3559
3845
  leaseAware?: boolean;
3560
3846
  reclaimRunning?: boolean;
@@ -3572,7 +3858,8 @@ export async function claimRuntimeWorkReceipt(
3572
3858
  session,
3573
3859
  async (client) => {
3574
3860
  const leaseId =
3575
- input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null;
3861
+ input.leaseId?.trim() ||
3862
+ (input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null);
3576
3863
  const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
3577
3864
  const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
3578
3865
  input.leaseTtlMs,
@@ -3700,6 +3987,15 @@ export async function claimRuntimeWorkReceipt(
3700
3987
  }
3701
3988
 
3702
3989
  const latest = await readRuntimeWorkReceipt(client, session, input.key);
3990
+ if (
3991
+ latest &&
3992
+ leaseId !== null &&
3993
+ latest.leaseId === leaseId &&
3994
+ (latest.leaseOwnerRunId ?? latest.runId) === input.runId &&
3995
+ (latest.leaseOwnerAttempt ?? 0) === runAttempt
3996
+ ) {
3997
+ return { disposition: 'claimed', receipt: latest };
3998
+ }
3703
3999
  if (
3704
4000
  latest &&
3705
4001
  input.forceRefresh !== true &&
@@ -3761,6 +4057,7 @@ export async function claimRuntimeWorkReceipts(
3761
4057
  playName: string;
3762
4058
  runId: string;
3763
4059
  keys: string[];
4060
+ leaseIds?: string[];
3764
4061
  runAttempt?: number | null;
3765
4062
  leaseAware?: boolean;
3766
4063
  reclaimRunning?: boolean;
@@ -3782,9 +4079,21 @@ export async function claimRuntimeWorkReceipts(
3782
4079
  session,
3783
4080
  async (client) => {
3784
4081
  const keyHexes = keys.map(workReceiptKeyHex);
3785
- const leaseIds = keys.map(() =>
3786
- input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null,
3787
- );
4082
+ if (
4083
+ input.leaseIds !== undefined &&
4084
+ input.leaseIds.length !== keys.length
4085
+ ) {
4086
+ throw new Error(
4087
+ `Runtime receipt bulk claim requires one lease ID per key. Received ${input.leaseIds.length} lease IDs for ${keys.length} keys.`,
4088
+ );
4089
+ }
4090
+ const leaseIds = keys.map((_, index) => {
4091
+ const providedLeaseId = input.leaseIds?.[index]?.trim();
4092
+ return (
4093
+ providedLeaseId ||
4094
+ (input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null)
4095
+ );
4096
+ });
3788
4097
  const claimStatus =
3789
4098
  input.leaseAware === true
3790
4099
  ? RECEIPT_STATUS_QUEUED
@@ -3887,7 +4196,10 @@ export async function claimRuntimeWorkReceipts(
3887
4196
  receipts.lease_expires_at,
3888
4197
  receipts.updated_at,
3889
4198
  input_keys.ord,
3890
- false AS claimed
4199
+ receipts.lease_id = input_keys.lease_id
4200
+ AND receipts.lease_owner_run_id = $4::text
4201
+ AND COALESCE(receipts.lease_owner_attempt, 0) = $9::integer
4202
+ AS claimed
3891
4203
  FROM input_keys
3892
4204
  JOIN ${workReceiptTable(session)} AS receipts
3893
4205
  ON receipts.k = decode(input_keys.key_hex, 'hex')
@@ -4363,7 +4675,48 @@ export async function completeRuntimeWorkReceipt(
4363
4675
  context,
4364
4676
  session,
4365
4677
  async (client) => {
4366
- const leaseId = input.leaseId?.trim() || null;
4678
+ const immutableCacheInsert =
4679
+ input.leaseId === COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID;
4680
+ const leaseId = immutableCacheInsert
4681
+ ? null
4682
+ : input.leaseId?.trim() || null;
4683
+ if (immutableCacheInsert) {
4684
+ const { rows: insertedRows } = await client.query<
4685
+ Record<string, unknown>
4686
+ >(
4687
+ `
4688
+ INSERT INTO ${workReceiptTable(session)} (
4689
+ k, status, output, error, failure_kind, run_id, updated_at
4690
+ ) VALUES (
4691
+ decode($1, 'hex'), $2::smallint, $3::jsonb, NULL, 0, $4, now()
4692
+ )
4693
+ ON CONFLICT (k) DO NOTHING
4694
+ RETURNING convert_from(k, 'UTF8') AS k, status, output, error,
4695
+ failure_kind, run_id, lease_id, lease_owner_run_id,
4696
+ lease_owner_attempt, lease_expires_at, updated_at
4697
+ `,
4698
+ [
4699
+ workReceiptKeyHex(input.key),
4700
+ RECEIPT_STATUS_COMPLETED,
4701
+ input.output === null ? null : stringifyPostgresJson(input.output),
4702
+ input.runId,
4703
+ ],
4704
+ );
4705
+ if (insertedRows[0]) return mapRuntimeWorkReceiptRow(insertedRows[0]);
4706
+ const { rows: winnerRows } = await client.query<
4707
+ Record<string, unknown>
4708
+ >(
4709
+ `SELECT convert_from(k, 'UTF8') AS k, status, output, error,
4710
+ failure_kind, run_id, lease_id, lease_owner_run_id,
4711
+ lease_owner_attempt, lease_expires_at, updated_at
4712
+ FROM ${workReceiptTable(session)}
4713
+ WHERE k = decode($1, 'hex')
4714
+ AND status IN ($2::smallint, ${RECEIPT_STATUS_SKIPPED}::smallint)
4715
+ LIMIT 1`,
4716
+ [workReceiptKeyHex(input.key), RECEIPT_STATUS_COMPLETED],
4717
+ );
4718
+ return winnerRows[0] ? mapRuntimeWorkReceiptRow(winnerRows[0]) : null;
4719
+ }
4367
4720
  const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
4368
4721
  const { rows } = await client.query<Record<string, unknown>>(
4369
4722
  `
@@ -4498,6 +4851,44 @@ export async function completeRuntimeWorkReceipts(
4498
4851
  }
4499
4852
  throw error;
4500
4853
  }
4854
+ const immutableEntries = receipts
4855
+ .map((receipt, index) => ({ receipt, index }))
4856
+ .filter(
4857
+ ({ receipt }) =>
4858
+ receipt.leaseId === COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
4859
+ );
4860
+ if (immutableEntries.length > 0) {
4861
+ const leasedEntries = receipts
4862
+ .map((receipt, index) => ({ receipt, index }))
4863
+ .filter(
4864
+ ({ receipt }) =>
4865
+ receipt.leaseId !== COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
4866
+ );
4867
+ const resolved = Array<WorkReceipt | null>(receipts.length).fill(null);
4868
+ const [immutableResults, leasedResults] = await Promise.all([
4869
+ Promise.all(
4870
+ immutableEntries.map(({ receipt }) =>
4871
+ completeRuntimeWorkReceipt(context, {
4872
+ playName: input.playName,
4873
+ ...receipt,
4874
+ }),
4875
+ ),
4876
+ ),
4877
+ leasedEntries.length > 0
4878
+ ? completeRuntimeWorkReceipts(context, {
4879
+ playName: input.playName,
4880
+ receipts: leasedEntries.map(({ receipt }) => receipt),
4881
+ })
4882
+ : Promise.resolve([]),
4883
+ ]);
4884
+ immutableEntries.forEach(({ index }, resultIndex) => {
4885
+ resolved[index] = immutableResults[resultIndex] ?? null;
4886
+ });
4887
+ leasedEntries.forEach(({ index }, resultIndex) => {
4888
+ resolved[index] = leasedResults[resultIndex] ?? null;
4889
+ });
4890
+ return resolved;
4891
+ }
4501
4892
  const session = await getRuntimeWorkReceiptSessionForKeys(context, {
4502
4893
  playName: input.playName,
4503
4894
  keys: receipts.map((receipt) => receipt.key),
@@ -5147,6 +5538,7 @@ export async function startRuntimeSheetDataset(
5147
5538
  attemptExpiresAt?: string | null;
5148
5539
  attemptSeq?: number | null;
5149
5540
  attemptLeaseTtlMs?: number | null;
5541
+ writeVersion?: number | null;
5150
5542
  inputOffset?: number;
5151
5543
  force?: boolean;
5152
5544
  },
@@ -5188,6 +5580,14 @@ export async function startRuntimeSheetDataset(
5188
5580
  ms: Date.now() - sessionStartedAt,
5189
5581
  rows: input.rows.length,
5190
5582
  });
5583
+ // Register the exact run-scoped Dataset Handle before any row can become
5584
+ // durable. A crash after a later sheet write can therefore never leave
5585
+ // customer data discoverable only through static pipeline inference.
5586
+ await registerRuntimeDataset(session, {
5587
+ playName,
5588
+ tableNamespace: input.tableNamespace,
5589
+ runId: input.runId,
5590
+ });
5191
5591
  const normalizeStartedAt = Date.now();
5192
5592
  const uniqueRows = new Map<
5193
5593
  string,
@@ -5285,7 +5685,10 @@ export async function startRuntimeSheetDataset(
5285
5685
  });
5286
5686
  const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
5287
5687
  const chunks = chunkValues(rowEntries, DIRECT_POSTGRES_BATCH_SIZE);
5288
- const needsTransaction = chunks.length > 1;
5688
+ // Scheduling is one admission decision. Even when batching SQL for the
5689
+ // parameter limit, either every requested row receives this intent version
5690
+ // or none do; a run can never leave 42 pending rows after 8 conflicts.
5691
+ const needsTransaction = true;
5289
5692
 
5290
5693
  const result = await withRuntimeSheetQueryClient(
5291
5694
  context,
@@ -5303,6 +5706,11 @@ export async function startRuntimeSheetDataset(
5303
5706
  client,
5304
5707
  input.attemptLeaseTtlMs,
5305
5708
  );
5709
+ const writeVersion = await mintRuntimeSheetWriteVersion(
5710
+ client,
5711
+ session,
5712
+ input.writeVersion,
5713
+ );
5306
5714
  timings.push({
5307
5715
  phase: 'attempt_expiry_mint',
5308
5716
  ms: Date.now() - attemptExpiryStartedAt,
@@ -5318,6 +5726,7 @@ export async function startRuntimeSheetDataset(
5318
5726
  attemptOwnerRunId,
5319
5727
  attemptExpiresAt,
5320
5728
  attemptSeq,
5729
+ writeVersion,
5321
5730
  physicalInsertColumnsSql,
5322
5731
  physicalInsertValuesSql,
5323
5732
  physicalRefreshSetSql:
@@ -5350,6 +5759,7 @@ export async function startRuntimeSheetDataset(
5350
5759
  attemptOwnerRunId,
5351
5760
  attemptExpiresAt,
5352
5761
  attemptSeq,
5762
+ writeVersion,
5353
5763
  timings,
5354
5764
  force: input.force === true,
5355
5765
  ...prepared,
@@ -5367,6 +5777,7 @@ export async function startRuntimeSheetDataset(
5367
5777
  ...built,
5368
5778
  attemptExpiresAt,
5369
5779
  attemptSeq,
5780
+ writeVersion,
5370
5781
  };
5371
5782
  },
5372
5783
  );
@@ -5386,6 +5797,7 @@ export async function startRuntimeSheetDataset(
5386
5797
  attemptOwnerRunId,
5387
5798
  attemptExpiresAt: result.attemptExpiresAt,
5388
5799
  attemptSeq,
5800
+ writeVersion: result.writeVersion,
5389
5801
  timings,
5390
5802
  };
5391
5803
  }
@@ -5695,6 +6107,7 @@ type CompleteRuntimeMapRowChunksInput = {
5695
6107
  attemptOwnerRunId: string | null;
5696
6108
  attemptExpiresAt: string | null;
5697
6109
  attemptSeq: number;
6110
+ writeVersion: number | null;
5698
6111
  normalizedPlayName: string;
5699
6112
  normalizedTableNamespace: string;
5700
6113
  outputFields: string[];
@@ -5703,6 +6116,8 @@ type CompleteRuntimeMapRowChunksInput = {
5703
6116
  export type RuntimeMapRowsWriteResult = {
5704
6117
  updated: number;
5705
6118
  fencedKeys: string[];
6119
+ staleDropped?: number;
6120
+ staleDroppedKeys?: string[];
5706
6121
  };
5707
6122
 
5708
6123
  function writableEnrichedRuntimeSheetAttemptSql(
@@ -5791,6 +6206,8 @@ async function completeRuntimeMapRowChunks(
5791
6206
  UPDATE ${sheetTable(session)} AS target
5792
6207
  SET _status = 'enriched',
5793
6208
  _run_id = $5::text,
6209
+ _writer_run_id = $5::text,
6210
+ _write_version = COALESCE($13::bigint, target._write_version),
5794
6211
  _error = NULL,
5795
6212
  _attempt_id = $9::text,
5796
6213
  _attempt_owner_run_id = $10::text,
@@ -5801,6 +6218,8 @@ async function completeRuntimeMapRowChunks(
5801
6218
  _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
5802
6219
  FROM matched_updates AS updates
5803
6220
  WHERE target._key = updates.matched_key
6221
+ AND ($13::bigint IS NULL OR target._write_version = $13::bigint)
6222
+ AND ($13::bigint IS NOT NULL OR target._status <> 'enriched')
5804
6223
  AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
5805
6224
  AND NOT (${sameOwnerTerminalEpochSql})
5806
6225
  AND (
@@ -5837,6 +6256,9 @@ async function completeRuntimeMapRowChunks(
5837
6256
  SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
5838
6257
  )
5839
6258
  AND (
6259
+ ($13::bigint IS NOT NULL AND target._write_version <> $13::bigint)
6260
+ OR ($13::bigint IS NULL AND target._status = 'enriched')
6261
+ OR
5840
6262
  (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
5841
6263
  OR NOT (
5842
6264
  ($9::text IS NULL AND target._run_id = $5::text)
@@ -5946,12 +6368,18 @@ async function completeRuntimeMapRowChunks(
5946
6368
  input.attemptOwnerRunId,
5947
6369
  input.attemptExpiresAt,
5948
6370
  input.attemptSeq,
6371
+ input.writeVersion,
5949
6372
  ],
5950
6373
  );
5951
- updated += Number(rows[0]?.updated ?? 0);
5952
- fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
5953
-
5954
6374
  const matchedKeys = new Set(rows[0]?.matched_keys ?? []);
6375
+ const chunkFencedKeys = rows[0]?.fenced_keys ?? [];
6376
+ // An exact-version replay that already produced the same terminal value is
6377
+ // accepted idempotently even when SQL has no physical mutation to apply.
6378
+ // Stale versions remain excluded through fenced_keys.
6379
+ const appliedCount = Number(rows[0]?.updated ?? 0);
6380
+ updated += appliedCount;
6381
+ fencedKeys.push(...chunkFencedKeys);
6382
+
5955
6383
  if (matchedKeys.size === chunk.length) {
5956
6384
  continue;
5957
6385
  }
@@ -6388,6 +6816,7 @@ async function failRuntimeMapRowChunks(
6388
6816
  attemptOwnerRunId: string | null;
6389
6817
  attemptExpiresAt: string | null;
6390
6818
  attemptSeq: number;
6819
+ writeVersion: number | null;
6391
6820
  normalizedPlayName: string;
6392
6821
  normalizedTableNamespace: string;
6393
6822
  },
@@ -6462,6 +6891,8 @@ async function failRuntimeMapRowChunks(
6462
6891
  UPDATE ${sheetTable(session)} AS target
6463
6892
  SET _status = 'failed',
6464
6893
  _run_id = $6::text,
6894
+ _writer_run_id = $6::text,
6895
+ _write_version = COALESCE($16::bigint, target._write_version),
6465
6896
  _error = updates.error,
6466
6897
  _attempt_id = $11::text,
6467
6898
  _attempt_owner_run_id = $12::text,
@@ -6472,6 +6903,7 @@ async function failRuntimeMapRowChunks(
6472
6903
  _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
6473
6904
  FROM matched_updates AS updates, ${sheetTable(session)} AS prev
6474
6905
  WHERE target._key = updates.matched_key
6906
+ AND ($16::bigint IS NULL OR target._write_version = $16::bigint)
6475
6907
  AND prev._key = target._key
6476
6908
  AND ($15::boolean OR target._status <> 'enriched')
6477
6909
  AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$13::timestamptz', '$14::integer', '$11::text', '$12::text')})
@@ -6497,6 +6929,8 @@ async function failRuntimeMapRowChunks(
6497
6929
  SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
6498
6930
  )
6499
6931
  AND (
6932
+ ($16::bigint IS NOT NULL AND target._write_version <> $16::bigint)
6933
+ OR
6500
6934
  (NOT $15::boolean AND target._status = 'enriched')
6501
6935
  OR (${newerTerminalRuntimeSheetRowSql('target', '$13::timestamptz', '$14::integer', '$11::text', '$12::text')})
6502
6936
  OR NOT (
@@ -6583,6 +7017,7 @@ async function failRuntimeMapRowChunks(
6583
7017
  input.attemptExpiresAt,
6584
7018
  input.attemptSeq,
6585
7019
  input.forceTerminal === true,
7020
+ input.writeVersion,
6586
7021
  ],
6587
7022
  );
6588
7023
  updated += Number(rows[0]?.updated ?? 0);
@@ -6593,9 +7028,8 @@ async function failRuntimeMapRowChunks(
6593
7028
 
6594
7029
  /**
6595
7030
  * Mark map rows terminal in the per-run scoped Postgres sheet table by key.
6596
- * Mirrors server-side `store.completeSheetRows` semantics: UPDATE-by-key with
6597
- * the row's enriched output values written into materialized physical
6598
- * columns, _status flipped to 'enriched', and the sheet cursor advanced.
7031
+ * Writes enriched output values into materialized physical columns, flips
7032
+ * _status to 'enriched', and advances the sheet cursor.
6599
7033
  *
6600
7034
  * Rows with `status: 'failed'` instead persist a row-isolated failure
6601
7035
  * (`_status='failed'`, `_error`, per-cell failure meta) while keeping the
@@ -6618,6 +7052,7 @@ export async function completeRuntimeMapRows(
6618
7052
  attemptOwnerRunId?: string | null;
6619
7053
  attemptExpiresAt?: string | null;
6620
7054
  attemptSeq?: number | null;
7055
+ writeVersion?: number | null;
6621
7056
  forceFailedRows?: boolean;
6622
7057
  },
6623
7058
  ): Promise<RuntimeMapRowsWriteResult> {
@@ -6701,6 +7136,15 @@ export async function completeRuntimeMapRows(
6701
7136
  input.attemptOwnerRunId?.trim() || (attemptId ? input.runId : null);
6702
7137
  const attemptExpiresAt = input.attemptExpiresAt?.trim() || null;
6703
7138
  const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
7139
+ const writeVersion = input.writeVersion == null ? null : input.writeVersion;
7140
+ if (
7141
+ writeVersion !== null &&
7142
+ (!Number.isSafeInteger(writeVersion) || writeVersion <= 0)
7143
+ ) {
7144
+ throw new Error(
7145
+ 'Runtime sheet writeVersion must be a positive safe integer.',
7146
+ );
7147
+ }
6704
7148
 
6705
7149
  return await withRuntimeSheetQueryClient(
6706
7150
  context,
@@ -6724,6 +7168,7 @@ export async function completeRuntimeMapRows(
6724
7168
  attemptOwnerRunId,
6725
7169
  attemptExpiresAt,
6726
7170
  attemptSeq,
7171
+ writeVersion,
6727
7172
  normalizedPlayName: normalizePlayNameForSheet(session.playName),
6728
7173
  normalizedTableNamespace: normalizeTableNamespace(
6729
7174
  input.tableNamespace,
@@ -6757,6 +7202,7 @@ export async function completeRuntimeMapRows(
6757
7202
  attemptOwnerRunId,
6758
7203
  attemptExpiresAt,
6759
7204
  attemptSeq,
7205
+ writeVersion,
6760
7206
  normalizedPlayName: normalizePlayNameForSheet(session.playName),
6761
7207
  normalizedTableNamespace: normalizeTableNamespace(
6762
7208
  input.tableNamespace,
@@ -6766,6 +7212,8 @@ export async function completeRuntimeMapRows(
6766
7212
  return {
6767
7213
  updated: completed.updated + failed.updated,
6768
7214
  fencedKeys: [...completed.fencedKeys, ...failed.fencedKeys],
7215
+ staleDropped: completed.fencedKeys.length + failed.fencedKeys.length,
7216
+ staleDroppedKeys: [...completed.fencedKeys, ...failed.fencedKeys],
6769
7217
  };
6770
7218
  },
6771
7219
  );