deepline 0.1.211 → 0.1.213
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +317 -340
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/customer-console.ts +23 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +1 -1
- package/dist/bundling-sources/sdk/src/client.ts +118 -3
- package/dist/bundling-sources/sdk/src/index.ts +2 -0
- package/dist/bundling-sources/sdk/src/play.ts +8 -1
- package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +1 -1
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +11 -1
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +47 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +302 -30
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-placement.ts +14 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +57 -23
- package/dist/bundling-sources/shared_libs/play-runtime/completed-receipt-cache.ts +166 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +164 -121
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +42 -1
- package/dist/bundling-sources/shared_libs/play-runtime/dataset-id.ts +10 -4
- package/dist/bundling-sources/shared_libs/play-runtime/db-session.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +163 -7
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts +93 -0
- package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +14 -2
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +4 -2
- package/dist/bundling-sources/shared_libs/play-runtime/play-call-execution.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +16 -1
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +21 -1
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-state-sink.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +80 -3
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +128 -6
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +94 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +91 -6
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +19 -10
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +540 -92
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-postgres-admission.ts +162 -0
- package/dist/bundling-sources/shared_libs/play-runtime/secret-capability.ts +18 -4
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +16 -17
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +29 -9
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +22 -1
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +1 -0
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +79 -1
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +2 -4
- package/dist/cli/index.js +637 -229
- package/dist/cli/index.mjs +637 -229
- package/dist/{compiler-manifest-j6z8qzpd.d.mts → compiler-manifest-BhgZ23A4.d.mts} +1 -0
- package/dist/{compiler-manifest-j6z8qzpd.d.ts → compiler-manifest-BhgZ23A4.d.ts} +1 -0
- package/dist/index.d.mts +84 -11
- package/dist/index.d.ts +84 -11
- package/dist/index.js +225 -43
- package/dist/index.mjs +225 -43
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +65 -4
- package/package.json +1 -1
|
@@ -105,7 +105,13 @@ import {
|
|
|
105
105
|
PLAY_RUNTIME_TEST_FAULT_HEADER,
|
|
106
106
|
type RuntimeTestFaultName,
|
|
107
107
|
} from './test-runtime-seams';
|
|
108
|
+
import { COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID } from './durable-receipt-execution';
|
|
108
109
|
import { vercelProtectionBypassHeader } from './vercel-protection';
|
|
110
|
+
import {
|
|
111
|
+
RuntimePostgresAdmission,
|
|
112
|
+
type RuntimePostgresAdmissionSnapshot,
|
|
113
|
+
type RuntimePostgresLane,
|
|
114
|
+
} from './runtime-postgres-admission';
|
|
109
115
|
|
|
110
116
|
type RuntimeApiContext = {
|
|
111
117
|
baseUrl?: string | null;
|
|
@@ -127,6 +133,8 @@ type RuntimeApiContext = {
|
|
|
127
133
|
postgresSessionUnwrapKey?: string | null;
|
|
128
134
|
};
|
|
129
135
|
|
|
136
|
+
const RUNTIME_RUN_DATASET_CATALOG_TABLE = '_deepline_run_datasets';
|
|
137
|
+
|
|
130
138
|
type DirectRuntimeTestFaultState = {
|
|
131
139
|
headerValue: string;
|
|
132
140
|
counts: Partial<Record<RuntimeTestFaultName, number>>;
|
|
@@ -267,6 +275,7 @@ type RuntimeSheetPreparedRowDisposition = {
|
|
|
267
275
|
const dbSessionCache = new Map<string, DbSessionCacheEntry>();
|
|
268
276
|
const dbSessionInFlight = new Map<string, Promise<CreateDbSessionResponse>>();
|
|
269
277
|
const postgresPools = new Map<string, RuntimePool>();
|
|
278
|
+
const runtimePostgresAdmissions = new Map<string, RuntimePostgresAdmission>();
|
|
270
279
|
const runtimeWorkReceiptEnsureCache = new Map<string, Promise<void>>();
|
|
271
280
|
const runtimeSheetEnsureCache = new Map<
|
|
272
281
|
string,
|
|
@@ -325,7 +334,14 @@ const RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS = 10_000;
|
|
|
325
334
|
// Daytona executes one play per runner process. This is the bounded data-plane
|
|
326
335
|
// budget for that one run's sheet reads/writes, receipts, and terminal/export
|
|
327
336
|
// reads over the PgBouncer URL; it is not a scheduler/control-plane fan-out.
|
|
328
|
-
|
|
337
|
+
// One runner gets four tenant-gateway clients, split evenly so a large sheet
|
|
338
|
+
// flush cannot starve receipt claims (and receipt traffic cannot block the
|
|
339
|
+
// persistence barrier). The admission queues add a driver-independent bound:
|
|
340
|
+
// Neon Pool and node-postgres may queue internally, but runtime callers never
|
|
341
|
+
// enter those unobservable queues without first holding a lane permit.
|
|
342
|
+
const RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE = 2;
|
|
343
|
+
const RUNTIME_POSTGRES_ADMISSION_MAX_QUEUED_PER_LANE = 2_000;
|
|
344
|
+
const RUNTIME_POSTGRES_ADMISSION_TIMEOUT_MS = 10_000;
|
|
329
345
|
const RECEIPT_STATUS_QUEUED = RECEIPT_STATUS_CODE.queued;
|
|
330
346
|
const RECEIPT_STATUS_PENDING = RECEIPT_STATUS_CODE.pending;
|
|
331
347
|
const RECEIPT_STATUS_RUNNING = RECEIPT_STATUS_CODE.running;
|
|
@@ -367,6 +383,7 @@ export type PrepareRuntimeSheetResult = {
|
|
|
367
383
|
attemptOwnerRunId?: string;
|
|
368
384
|
attemptExpiresAt?: string;
|
|
369
385
|
attemptSeq?: number;
|
|
386
|
+
writeVersion?: number;
|
|
370
387
|
};
|
|
371
388
|
|
|
372
389
|
export type RuntimeSheetTiming = {
|
|
@@ -1290,9 +1307,7 @@ async function deleteRuntimeDbSessionCacheEntry(
|
|
|
1290
1307
|
): Promise<void> {
|
|
1291
1308
|
dbSessionCache.delete(cacheKey);
|
|
1292
1309
|
if (session.postgresUrl) {
|
|
1293
|
-
|
|
1294
|
-
postgresPools.delete(session.postgresUrl);
|
|
1295
|
-
await pool?.end().catch(() => {});
|
|
1310
|
+
await resetRuntimePostgresPool(session.postgresUrl);
|
|
1296
1311
|
}
|
|
1297
1312
|
}
|
|
1298
1313
|
|
|
@@ -1413,18 +1428,10 @@ async function prewarmRuntimePostgresSession(
|
|
|
1413
1428
|
attempt >= RUNTIME_POSTGRES_PREWARM_MAX_ATTEMPTS ||
|
|
1414
1429
|
!isTransientRuntimePostgresConnectionError(error)
|
|
1415
1430
|
) {
|
|
1416
|
-
|
|
1417
|
-
postgresPools.delete(session.postgresUrl);
|
|
1418
|
-
if (pool) {
|
|
1419
|
-
await Promise.resolve(pool.end()).catch(() => {});
|
|
1420
|
-
}
|
|
1431
|
+
await resetRuntimePostgresPool(session.postgresUrl);
|
|
1421
1432
|
throw error;
|
|
1422
1433
|
}
|
|
1423
|
-
|
|
1424
|
-
postgresPools.delete(session.postgresUrl);
|
|
1425
|
-
if (pool) {
|
|
1426
|
-
await Promise.resolve(pool.end()).catch(() => {});
|
|
1427
|
-
}
|
|
1434
|
+
await resetRuntimePostgresPool(session.postgresUrl);
|
|
1428
1435
|
await sleep(
|
|
1429
1436
|
RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS[attempt - 1] ??
|
|
1430
1437
|
RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS[
|
|
@@ -1489,7 +1496,10 @@ function sleep(ms: number): Promise<void> {
|
|
|
1489
1496
|
|
|
1490
1497
|
async function connectRuntimePostgresPool(
|
|
1491
1498
|
pool: RuntimePool,
|
|
1499
|
+
admission: RuntimePostgresAdmission,
|
|
1500
|
+
lane: RuntimePostgresLane,
|
|
1492
1501
|
): Promise<RuntimePoolClient> {
|
|
1502
|
+
const releaseAdmission = await admission.acquire(lane);
|
|
1493
1503
|
let timedOut = false;
|
|
1494
1504
|
let timeout: ReturnType<typeof setTimeout> | null = null;
|
|
1495
1505
|
const connectPromise = pool.connect();
|
|
@@ -1497,11 +1507,12 @@ async function connectRuntimePostgresPool(
|
|
|
1497
1507
|
.then((client) => {
|
|
1498
1508
|
if (timedOut) {
|
|
1499
1509
|
client.release();
|
|
1510
|
+
releaseAdmission();
|
|
1500
1511
|
}
|
|
1501
1512
|
})
|
|
1502
1513
|
.catch(() => {});
|
|
1503
1514
|
try {
|
|
1504
|
-
|
|
1515
|
+
const client = await Promise.race([
|
|
1505
1516
|
connectPromise,
|
|
1506
1517
|
new Promise<never>((_, reject) => {
|
|
1507
1518
|
timeout = setTimeout(() => {
|
|
@@ -1514,6 +1525,21 @@ async function connectRuntimePostgresPool(
|
|
|
1514
1525
|
}, RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS);
|
|
1515
1526
|
}),
|
|
1516
1527
|
]);
|
|
1528
|
+
let released = false;
|
|
1529
|
+
return {
|
|
1530
|
+
query: (text, params) => client.query(text, params),
|
|
1531
|
+
release: () => {
|
|
1532
|
+
if (released) return;
|
|
1533
|
+
released = true;
|
|
1534
|
+
client.release();
|
|
1535
|
+
releaseAdmission();
|
|
1536
|
+
},
|
|
1537
|
+
};
|
|
1538
|
+
} catch (error) {
|
|
1539
|
+
if (!timedOut) {
|
|
1540
|
+
releaseAdmission();
|
|
1541
|
+
}
|
|
1542
|
+
throw error;
|
|
1517
1543
|
} finally {
|
|
1518
1544
|
if (timeout) {
|
|
1519
1545
|
clearTimeout(timeout);
|
|
@@ -1521,35 +1547,81 @@ async function connectRuntimePostgresPool(
|
|
|
1521
1547
|
}
|
|
1522
1548
|
}
|
|
1523
1549
|
|
|
1524
|
-
function
|
|
1550
|
+
function runtimePostgresLane(
|
|
1551
|
+
session: RuntimePostgresSession,
|
|
1552
|
+
): RuntimePostgresLane {
|
|
1553
|
+
return session.target.logicalTable === RUNTIME_WORK_RECEIPT_LOGICAL_TABLE
|
|
1554
|
+
? 'receipts'
|
|
1555
|
+
: 'sheets';
|
|
1556
|
+
}
|
|
1557
|
+
|
|
1558
|
+
function runtimePostgresPoolKey(
|
|
1559
|
+
postgresUrl: string,
|
|
1560
|
+
lane: RuntimePostgresLane,
|
|
1561
|
+
): string {
|
|
1562
|
+
return `${postgresUrl}::${lane}`;
|
|
1563
|
+
}
|
|
1564
|
+
|
|
1565
|
+
function getRuntimePostgresAdmission(
|
|
1566
|
+
postgresUrl: string,
|
|
1567
|
+
): RuntimePostgresAdmission {
|
|
1568
|
+
const existing = runtimePostgresAdmissions.get(postgresUrl);
|
|
1569
|
+
if (existing) return existing;
|
|
1570
|
+
const admission = new RuntimePostgresAdmission({
|
|
1571
|
+
maxActivePerLane: RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE,
|
|
1572
|
+
maxQueuedPerLane: RUNTIME_POSTGRES_ADMISSION_MAX_QUEUED_PER_LANE,
|
|
1573
|
+
acquireTimeoutMs: RUNTIME_POSTGRES_ADMISSION_TIMEOUT_MS,
|
|
1574
|
+
});
|
|
1575
|
+
runtimePostgresAdmissions.set(postgresUrl, admission);
|
|
1576
|
+
return admission;
|
|
1577
|
+
}
|
|
1578
|
+
|
|
1579
|
+
export function runtimePostgresAdmissionTelemetry(
|
|
1580
|
+
postgresUrl: string,
|
|
1581
|
+
): RuntimePostgresAdmissionSnapshot | null {
|
|
1582
|
+
return runtimePostgresAdmissions.get(postgresUrl)?.snapshot() ?? null;
|
|
1583
|
+
}
|
|
1584
|
+
|
|
1585
|
+
function getPostgresPool(
|
|
1586
|
+
postgresUrl: string,
|
|
1587
|
+
lane: RuntimePostgresLane,
|
|
1588
|
+
cachePool = true,
|
|
1589
|
+
): RuntimePool {
|
|
1525
1590
|
if (!cachePool) {
|
|
1526
1591
|
return createRuntimePool({
|
|
1527
1592
|
connectionString: postgresUrl,
|
|
1528
|
-
maxConnections:
|
|
1593
|
+
maxConnections: RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE,
|
|
1529
1594
|
idleTimeoutMs: 0,
|
|
1530
1595
|
connectTimeoutMs: RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS,
|
|
1531
1596
|
});
|
|
1532
1597
|
}
|
|
1533
|
-
const
|
|
1598
|
+
const poolKey = runtimePostgresPoolKey(postgresUrl, lane);
|
|
1599
|
+
const existing = postgresPools.get(poolKey);
|
|
1534
1600
|
if (existing) {
|
|
1535
1601
|
return existing;
|
|
1536
1602
|
}
|
|
1537
1603
|
const pool = createRuntimePool({
|
|
1538
1604
|
connectionString: postgresUrl,
|
|
1539
|
-
maxConnections:
|
|
1605
|
+
maxConnections: RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE,
|
|
1540
1606
|
idleTimeoutMs: 15_000,
|
|
1541
1607
|
connectTimeoutMs: RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS,
|
|
1542
1608
|
});
|
|
1543
|
-
postgresPools.set(
|
|
1609
|
+
postgresPools.set(poolKey, pool);
|
|
1544
1610
|
return pool;
|
|
1545
1611
|
}
|
|
1546
1612
|
|
|
1547
1613
|
async function resetRuntimePostgresPool(postgresUrl: string): Promise<void> {
|
|
1548
|
-
const
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1614
|
+
const pools = (['receipts', 'sheets'] as const)
|
|
1615
|
+
.map((lane) => runtimePostgresPoolKey(postgresUrl, lane))
|
|
1616
|
+
.map((key) => {
|
|
1617
|
+
const pool = postgresPools.get(key);
|
|
1618
|
+
postgresPools.delete(key);
|
|
1619
|
+
return pool;
|
|
1620
|
+
})
|
|
1621
|
+
.filter((pool): pool is RuntimePool => pool !== undefined);
|
|
1622
|
+
await Promise.all(
|
|
1623
|
+
pools.map((pool) => Promise.resolve(pool.end()).catch(() => {})),
|
|
1624
|
+
);
|
|
1553
1625
|
}
|
|
1554
1626
|
|
|
1555
1627
|
async function withRuntimePostgres<T>(
|
|
@@ -1565,18 +1637,30 @@ async function withRuntimePostgres<T>(
|
|
|
1565
1637
|
options.maxConnectAttempts === undefined
|
|
1566
1638
|
? RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS
|
|
1567
1639
|
: Math.max(1, Math.floor(options.maxConnectAttempts));
|
|
1640
|
+
const lane = runtimePostgresLane(session);
|
|
1641
|
+
const admission = getRuntimePostgresAdmission(session.postgresUrl);
|
|
1568
1642
|
for (let attempt = 1; attempt <= maxConnectAttempts; attempt += 1) {
|
|
1569
1643
|
try {
|
|
1570
|
-
const pool = getPostgresPool(session.postgresUrl, cachePool);
|
|
1644
|
+
const pool = getPostgresPool(session.postgresUrl, lane, cachePool);
|
|
1571
1645
|
if (!cachePool) {
|
|
1572
1646
|
requestLocalPool = pool;
|
|
1573
1647
|
}
|
|
1574
|
-
client = await connectRuntimePostgresPool(pool);
|
|
1648
|
+
client = await connectRuntimePostgresPool(pool, admission, lane);
|
|
1649
|
+
if (session.executionRole) {
|
|
1650
|
+
await client.query(
|
|
1651
|
+
`SET ROLE ${quoteIdentifier(session.executionRole)}`,
|
|
1652
|
+
);
|
|
1653
|
+
}
|
|
1575
1654
|
break;
|
|
1576
1655
|
} catch (error) {
|
|
1656
|
+
if (client) {
|
|
1657
|
+
client.release();
|
|
1658
|
+
client = null;
|
|
1659
|
+
}
|
|
1577
1660
|
if (cachePool) {
|
|
1578
|
-
const
|
|
1579
|
-
postgresPools.
|
|
1661
|
+
const poolKey = runtimePostgresPoolKey(session.postgresUrl, lane);
|
|
1662
|
+
const pool = postgresPools.get(poolKey);
|
|
1663
|
+
postgresPools.delete(poolKey);
|
|
1580
1664
|
if (pool) {
|
|
1581
1665
|
await Promise.resolve(pool.end()).catch(() => {});
|
|
1582
1666
|
}
|
|
@@ -1604,7 +1688,13 @@ async function withRuntimePostgres<T>(
|
|
|
1604
1688
|
try {
|
|
1605
1689
|
return await fn(client);
|
|
1606
1690
|
} finally {
|
|
1607
|
-
|
|
1691
|
+
try {
|
|
1692
|
+
if (session.executionRole) {
|
|
1693
|
+
await client.query('RESET ROLE');
|
|
1694
|
+
}
|
|
1695
|
+
} finally {
|
|
1696
|
+
client.release();
|
|
1697
|
+
}
|
|
1608
1698
|
if (requestLocalPool) {
|
|
1609
1699
|
await Promise.resolve(requestLocalPool.end()).catch(() => {});
|
|
1610
1700
|
}
|
|
@@ -1833,6 +1923,38 @@ function columnSummaryTable(session: RuntimePostgresSession): string {
|
|
|
1833
1923
|
return fqRuntimeTable(session, session.postgres.columnSummaryTable);
|
|
1834
1924
|
}
|
|
1835
1925
|
|
|
1926
|
+
function runDatasetCatalogTable(session: RuntimePostgresSession): string {
|
|
1927
|
+
return fqRuntimeTable(session, RUNTIME_RUN_DATASET_CATALOG_TABLE);
|
|
1928
|
+
}
|
|
1929
|
+
|
|
1930
|
+
async function registerRuntimeDataset(
|
|
1931
|
+
session: RuntimePostgresSession,
|
|
1932
|
+
input: { playName: string; tableNamespace: string; runId: string },
|
|
1933
|
+
): Promise<void> {
|
|
1934
|
+
const tableNamespace = normalizeTableNamespace(input.tableNamespace);
|
|
1935
|
+
const playName = normalizePlayNameForSheet(input.playName);
|
|
1936
|
+
const datasetId = createRuntimeDatasetId(playName, tableNamespace);
|
|
1937
|
+
await withRuntimePostgres(session, async (client) => {
|
|
1938
|
+
await client.query(
|
|
1939
|
+
`INSERT INTO ${runDatasetCatalogTable(session)} (
|
|
1940
|
+
run_id, dataset_id, play_name, table_namespace, public_path
|
|
1941
|
+
) VALUES ($1, $2, $3, $4, $5)
|
|
1942
|
+
ON CONFLICT (run_id, dataset_id) DO UPDATE SET
|
|
1943
|
+
play_name = EXCLUDED.play_name,
|
|
1944
|
+
table_namespace = EXCLUDED.table_namespace,
|
|
1945
|
+
public_path = EXCLUDED.public_path,
|
|
1946
|
+
updated_at = now()`,
|
|
1947
|
+
[
|
|
1948
|
+
input.runId,
|
|
1949
|
+
datasetId,
|
|
1950
|
+
playName,
|
|
1951
|
+
tableNamespace,
|
|
1952
|
+
`datasets.${tableNamespace}`,
|
|
1953
|
+
],
|
|
1954
|
+
);
|
|
1955
|
+
});
|
|
1956
|
+
}
|
|
1957
|
+
|
|
1836
1958
|
function workReceiptTable(session: RuntimePostgresSession): string {
|
|
1837
1959
|
return fqRuntimeTable(
|
|
1838
1960
|
session,
|
|
@@ -1840,6 +1962,10 @@ function workReceiptTable(session: RuntimePostgresSession): string {
|
|
|
1840
1962
|
);
|
|
1841
1963
|
}
|
|
1842
1964
|
|
|
1965
|
+
function receiptExecutionLockTable(session: RuntimePostgresSession): string {
|
|
1966
|
+
return fqRuntimeTable(session, '_deepline_receipt_execution_locks');
|
|
1967
|
+
}
|
|
1968
|
+
|
|
1843
1969
|
function postgresUrlCacheKey(value: string): string {
|
|
1844
1970
|
return createHash('sha256').update(value).digest('hex').slice(0, 24);
|
|
1845
1971
|
}
|
|
@@ -1981,6 +2107,14 @@ async function ensureRuntimeWorkReceiptTable(
|
|
|
1981
2107
|
`,
|
|
1982
2108
|
)
|
|
1983
2109
|
.then(async () => {
|
|
2110
|
+
await client.query(`
|
|
2111
|
+
CREATE TABLE IF NOT EXISTS ${receiptExecutionLockTable(session)} (
|
|
2112
|
+
k bytea PRIMARY KEY,
|
|
2113
|
+
owner_execution_id text NOT NULL,
|
|
2114
|
+
expires_at timestamptz NOT NULL,
|
|
2115
|
+
updated_at timestamptz NOT NULL DEFAULT now()
|
|
2116
|
+
)
|
|
2117
|
+
`);
|
|
1984
2118
|
const missingColumns = await missingRuntimeWorkReceiptSelfHealColumns(
|
|
1985
2119
|
session,
|
|
1986
2120
|
client,
|
|
@@ -2015,6 +2149,77 @@ async function ensureRuntimeWorkReceiptTable(
|
|
|
2015
2149
|
}
|
|
2016
2150
|
}
|
|
2017
2151
|
|
|
2152
|
+
export async function acquireRuntimeReceiptExecutionLock(
|
|
2153
|
+
context: RuntimeApiContext,
|
|
2154
|
+
input: {
|
|
2155
|
+
playName: string;
|
|
2156
|
+
key: string;
|
|
2157
|
+
ownerExecutionId: string;
|
|
2158
|
+
ttlMs: number;
|
|
2159
|
+
},
|
|
2160
|
+
): Promise<{ ownerExecutionId: string; expiresAt: string } | null> {
|
|
2161
|
+
const ttlMs = Math.floor(input.ttlMs);
|
|
2162
|
+
if (!Number.isFinite(ttlMs) || ttlMs <= 0 || ttlMs > 10 * 60_000) {
|
|
2163
|
+
throw new Error('Runtime receipt execution lock TTL must be 1..600000ms.');
|
|
2164
|
+
}
|
|
2165
|
+
const ownerExecutionId = input.ownerExecutionId.trim();
|
|
2166
|
+
if (!ownerExecutionId) {
|
|
2167
|
+
throw new Error('Runtime receipt execution lock owner is required.');
|
|
2168
|
+
}
|
|
2169
|
+
const session = await getRuntimeWorkReceiptSession(context, input);
|
|
2170
|
+
return await withRuntimeWorkReceiptClient(
|
|
2171
|
+
context,
|
|
2172
|
+
session,
|
|
2173
|
+
async (client) => {
|
|
2174
|
+
const { rows } = await client.query<Record<string, unknown>>(
|
|
2175
|
+
`
|
|
2176
|
+
INSERT INTO ${receiptExecutionLockTable(session)} AS target (
|
|
2177
|
+
k, owner_execution_id, expires_at, updated_at
|
|
2178
|
+
) VALUES (
|
|
2179
|
+
decode($1, 'hex'), $2, now() + ($3::double precision * interval '1 millisecond'), now()
|
|
2180
|
+
)
|
|
2181
|
+
ON CONFLICT (k) DO UPDATE
|
|
2182
|
+
SET owner_execution_id = EXCLUDED.owner_execution_id,
|
|
2183
|
+
expires_at = EXCLUDED.expires_at,
|
|
2184
|
+
updated_at = now()
|
|
2185
|
+
WHERE target.expires_at <= now()
|
|
2186
|
+
OR target.owner_execution_id = EXCLUDED.owner_execution_id
|
|
2187
|
+
RETURNING owner_execution_id, expires_at
|
|
2188
|
+
`,
|
|
2189
|
+
[workReceiptKeyHex(input.key), ownerExecutionId, ttlMs],
|
|
2190
|
+
);
|
|
2191
|
+
const row = rows[0];
|
|
2192
|
+
return row
|
|
2193
|
+
? {
|
|
2194
|
+
ownerExecutionId: String(row.owner_execution_id),
|
|
2195
|
+
expiresAt: new Date(String(row.expires_at)).toISOString(),
|
|
2196
|
+
}
|
|
2197
|
+
: null;
|
|
2198
|
+
},
|
|
2199
|
+
);
|
|
2200
|
+
}
|
|
2201
|
+
|
|
2202
|
+
export async function releaseRuntimeReceiptExecutionLock(
|
|
2203
|
+
context: RuntimeApiContext,
|
|
2204
|
+
input: { playName: string; key: string; ownerExecutionId: string },
|
|
2205
|
+
): Promise<boolean> {
|
|
2206
|
+
const session = await getRuntimeWorkReceiptSession(context, input);
|
|
2207
|
+
return await withRuntimeWorkReceiptClient(
|
|
2208
|
+
context,
|
|
2209
|
+
session,
|
|
2210
|
+
async (client) => {
|
|
2211
|
+
const { rows } = await client.query(
|
|
2212
|
+
`DELETE FROM ${receiptExecutionLockTable(session)}
|
|
2213
|
+
WHERE k = decode($1, 'hex')
|
|
2214
|
+
AND owner_execution_id = $2
|
|
2215
|
+
RETURNING 1`,
|
|
2216
|
+
[workReceiptKeyHex(input.key), input.ownerExecutionId],
|
|
2217
|
+
);
|
|
2218
|
+
return rows.length > 0;
|
|
2219
|
+
},
|
|
2220
|
+
);
|
|
2221
|
+
}
|
|
2222
|
+
|
|
2018
2223
|
async function withRuntimeWorkReceiptClient<T>(
|
|
2019
2224
|
context: RuntimeApiContext,
|
|
2020
2225
|
session: RuntimePostgresSession,
|
|
@@ -2022,8 +2227,11 @@ async function withRuntimeWorkReceiptClient<T>(
|
|
|
2022
2227
|
): Promise<T> {
|
|
2023
2228
|
// Receipt tables are part of customer Postgres bootstrap, so the hot path
|
|
2024
2229
|
// should not pay DDL on every fresh runtime isolate. Try the receipt query
|
|
2025
|
-
// first, then self-heal once on a missing relation/column.
|
|
2026
|
-
//
|
|
2230
|
+
// first, then self-heal once on a missing relation/column. Permission loss
|
|
2231
|
+
// during an operational receipt query must fail loud: silently repairing it
|
|
2232
|
+
// can let provider work continue after durable receipt persistence is gone.
|
|
2233
|
+
// Only the schema-migration branch below may repair ownership before its
|
|
2234
|
+
// retry. A genuinely missing schema or second failure also rethrows loudly.
|
|
2027
2235
|
const runWithSelfHeal = async (client: RuntimeQueryClient): Promise<T> => {
|
|
2028
2236
|
try {
|
|
2029
2237
|
return await operation(client);
|
|
@@ -2048,10 +2256,6 @@ async function withRuntimeWorkReceiptClient<T>(
|
|
|
2048
2256
|
playName: context.playName?.trim() || session.playName,
|
|
2049
2257
|
});
|
|
2050
2258
|
}
|
|
2051
|
-
} else if (isPostgresPermissionDeniedError(error)) {
|
|
2052
|
-
await repairRuntimeStorageGrants(context, {
|
|
2053
|
-
playName: context.playName?.trim() || session.playName,
|
|
2054
|
-
});
|
|
2055
2259
|
} else {
|
|
2056
2260
|
throw error;
|
|
2057
2261
|
}
|
|
@@ -2225,6 +2429,31 @@ async function mintRuntimeSheetAttemptExpiresAt(
|
|
|
2225
2429
|
throw new Error('Runtime sheet attempt expiry mint did not return a value.');
|
|
2226
2430
|
}
|
|
2227
2431
|
|
|
2432
|
+
async function mintRuntimeSheetWriteVersion(
|
|
2433
|
+
client: RuntimeQueryClient,
|
|
2434
|
+
session: RuntimePostgresSession,
|
|
2435
|
+
requested?: number | null,
|
|
2436
|
+
): Promise<number> {
|
|
2437
|
+
if (requested != null) {
|
|
2438
|
+
if (!Number.isSafeInteger(requested) || requested <= 0) {
|
|
2439
|
+
throw new Error(
|
|
2440
|
+
'Runtime sheet writeVersion must be a positive safe integer.',
|
|
2441
|
+
);
|
|
2442
|
+
}
|
|
2443
|
+
return requested;
|
|
2444
|
+
}
|
|
2445
|
+
const { rows } = await client.query<{ write_version: number | string }>(
|
|
2446
|
+
`SELECT ${nextRuntimeSheetVersionExpression(session)} AS write_version`,
|
|
2447
|
+
);
|
|
2448
|
+
const value = Number(rows[0]?.write_version);
|
|
2449
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
2450
|
+
throw new Error(
|
|
2451
|
+
'Runtime sheet write version allocation did not return a safe integer.',
|
|
2452
|
+
);
|
|
2453
|
+
}
|
|
2454
|
+
return value;
|
|
2455
|
+
}
|
|
2456
|
+
|
|
2228
2457
|
function parseRuntimeCellMeta(value: unknown): Record<string, unknown> {
|
|
2229
2458
|
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
|
2230
2459
|
return value as Record<string, unknown>;
|
|
@@ -2725,6 +2954,7 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2725
2954
|
attemptOwnerRunId: string;
|
|
2726
2955
|
attemptExpiresAt: string;
|
|
2727
2956
|
attemptSeq: number;
|
|
2957
|
+
writeVersion: number;
|
|
2728
2958
|
physicalInsertColumnsSql: string;
|
|
2729
2959
|
physicalInsertValuesSql: string;
|
|
2730
2960
|
physicalRefreshSetSql: string;
|
|
@@ -2803,6 +3033,44 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2803
3033
|
ON index_values.ord = key_values.ord
|
|
2804
3034
|
ORDER BY key_values._key, key_values.ord
|
|
2805
3035
|
),
|
|
3036
|
+
preexisting_rows AS (
|
|
3037
|
+
SELECT target._key
|
|
3038
|
+
FROM ${sheetTable(session)} AS target
|
|
3039
|
+
JOIN input_rows ON input_rows._key = target._key
|
|
3040
|
+
),
|
|
3041
|
+
versioned_completed_rows AS (
|
|
3042
|
+
UPDATE ${sheetTable(session)} AS target
|
|
3043
|
+
SET _run_id = $4::text,
|
|
3044
|
+
_writer_run_id = $4::text,
|
|
3045
|
+
_write_version = $12::bigint,
|
|
3046
|
+
_input_index = input_rows._input_index,
|
|
3047
|
+
_updated_at = now(),
|
|
3048
|
+
_version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalRefreshSetSql}
|
|
3049
|
+
FROM input_rows
|
|
3050
|
+
WHERE target._key = input_rows._key
|
|
3051
|
+
AND target._status = 'enriched'
|
|
3052
|
+
AND COALESCE(target._write_version, 0) < $12::bigint
|
|
3053
|
+
RETURNING target._key,
|
|
3054
|
+
($11::boolean OR (${targetMissingOutputSql})) AS needs_recompute
|
|
3055
|
+
),
|
|
3056
|
+
versioned_failed_rows AS (
|
|
3057
|
+
UPDATE ${sheetTable(session)} AS target
|
|
3058
|
+
SET _run_id = $4::text,
|
|
3059
|
+
_writer_run_id = $4::text,
|
|
3060
|
+
_write_version = $12::bigint,
|
|
3061
|
+
_input_index = input_rows._input_index,
|
|
3062
|
+
_attempt_id = $7::text,
|
|
3063
|
+
_attempt_owner_run_id = $8::text,
|
|
3064
|
+
_attempt_expires_at = NULL,
|
|
3065
|
+
_attempt_seq = $10::integer,
|
|
3066
|
+
_updated_at = now(),
|
|
3067
|
+
_version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalRefreshSetSql}
|
|
3068
|
+
FROM input_rows
|
|
3069
|
+
WHERE target._key = input_rows._key
|
|
3070
|
+
AND target._status = 'failed'
|
|
3071
|
+
AND COALESCE(target._write_version, 0) < $12::bigint
|
|
3072
|
+
RETURNING target._key
|
|
3073
|
+
),
|
|
2806
3074
|
existing_rows AS (
|
|
2807
3075
|
UPDATE ${sheetTable(session)} AS target
|
|
2808
3076
|
SET _input_index = input_rows._input_index,
|
|
@@ -2810,6 +3078,7 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2810
3078
|
_version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalRefreshSetSql}
|
|
2811
3079
|
FROM input_rows
|
|
2812
3080
|
WHERE target._key = input_rows._key
|
|
3081
|
+
AND target._write_version = $12::bigint
|
|
2813
3082
|
AND target._status <> 'stale'
|
|
2814
3083
|
AND (
|
|
2815
3084
|
NOT (${targetNewerTerminalRowSql})
|
|
@@ -2833,8 +3102,8 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2833
3102
|
RETURNING target._key
|
|
2834
3103
|
),
|
|
2835
3104
|
inserted_rows AS (
|
|
2836
|
-
INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _attempt_id, _attempt_owner_run_id, _attempt_expires_at, _attempt_seq${input.physicalInsertColumnsSql})
|
|
2837
|
-
SELECT _key, 'pending', $4::text, _input_index, $7::text, $8::text, $9::timestamptz, $10::integer${input.physicalInsertValuesSql}
|
|
3105
|
+
INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _attempt_id, _attempt_owner_run_id, _attempt_expires_at, _attempt_seq, _write_version, _writer_run_id${input.physicalInsertColumnsSql})
|
|
3106
|
+
SELECT _key, 'pending', $4::text, _input_index, $7::text, $8::text, $9::timestamptz, $10::integer, $12::bigint, $4::text${input.physicalInsertValuesSql}
|
|
2838
3107
|
FROM input_rows
|
|
2839
3108
|
ON CONFLICT (_key) DO UPDATE SET
|
|
2840
3109
|
_status = 'pending',
|
|
@@ -2844,6 +3113,8 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2844
3113
|
_attempt_owner_run_id = EXCLUDED._attempt_owner_run_id,
|
|
2845
3114
|
_attempt_expires_at = EXCLUDED._attempt_expires_at,
|
|
2846
3115
|
_attempt_seq = EXCLUDED._attempt_seq,
|
|
3116
|
+
_write_version = EXCLUDED._write_version,
|
|
3117
|
+
_writer_run_id = EXCLUDED._writer_run_id,
|
|
2847
3118
|
_updated_at = now(),
|
|
2848
3119
|
_version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalUpsertSetSql}
|
|
2849
3120
|
WHERE ${sheetTable(session)}._status = 'stale'
|
|
@@ -2863,10 +3134,30 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2863
3134
|
FROM input_rows
|
|
2864
3135
|
WHERE target._key = input_rows._key
|
|
2865
3136
|
AND target._status = 'enriched'
|
|
3137
|
+
AND target._write_version = $12::bigint
|
|
2866
3138
|
AND (${targetMissingOutputSql})
|
|
2867
3139
|
AND COALESCE(target._attempt_seq, 0) <= $10::integer
|
|
2868
3140
|
RETURNING target._key
|
|
2869
3141
|
),
|
|
3142
|
+
superseded_rows AS (
|
|
3143
|
+
UPDATE ${sheetTable(session)} AS target
|
|
3144
|
+
SET _run_id = $4::text,
|
|
3145
|
+
_input_index = input_rows._input_index,
|
|
3146
|
+
_attempt_id = $7::text,
|
|
3147
|
+
_attempt_owner_run_id = $8::text,
|
|
3148
|
+
_attempt_expires_at = NULL,
|
|
3149
|
+
_attempt_seq = $10::integer,
|
|
3150
|
+
_write_version = $12::bigint,
|
|
3151
|
+
_writer_run_id = $4::text,
|
|
3152
|
+
_error = NULL,
|
|
3153
|
+
_updated_at = now(),
|
|
3154
|
+
_version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalRefreshSetSql}
|
|
3155
|
+
FROM input_rows
|
|
3156
|
+
WHERE target._key = input_rows._key
|
|
3157
|
+
AND target._status IN ('pending', 'running')
|
|
3158
|
+
AND COALESCE(target._write_version, 0) < $12::bigint
|
|
3159
|
+
RETURNING target._key
|
|
3160
|
+
),
|
|
2870
3161
|
claimed_existing_rows AS (
|
|
2871
3162
|
UPDATE ${sheetTable(session)} AS target
|
|
2872
3163
|
SET _run_id = $4::text,
|
|
@@ -2880,6 +3171,7 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2880
3171
|
FROM input_rows
|
|
2881
3172
|
WHERE target._key = input_rows._key
|
|
2882
3173
|
AND target._status IN ('pending', 'running', 'failed')
|
|
3174
|
+
AND target._write_version = $12::bigint
|
|
2883
3175
|
AND NOT (${targetNewerTerminalRowSql})
|
|
2884
3176
|
AND (
|
|
2885
3177
|
${targetAttemptFenceSql}
|
|
@@ -2902,6 +3194,9 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2902
3194
|
FROM inserted_rows
|
|
2903
3195
|
UNION
|
|
2904
3196
|
SELECT _key
|
|
3197
|
+
FROM superseded_rows
|
|
3198
|
+
UNION
|
|
3199
|
+
SELECT _key
|
|
2905
3200
|
FROM missing_output_rows
|
|
2906
3201
|
UNION
|
|
2907
3202
|
SELECT _key
|
|
@@ -2911,10 +3206,18 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2911
3206
|
FROM existing_rows
|
|
2912
3207
|
WHERE $11::boolean
|
|
2913
3208
|
UNION
|
|
3209
|
+
SELECT _key
|
|
3210
|
+
FROM versioned_completed_rows
|
|
3211
|
+
WHERE needs_recompute
|
|
3212
|
+
UNION
|
|
3213
|
+
SELECT _key FROM versioned_failed_rows
|
|
3214
|
+
UNION
|
|
2914
3215
|
SELECT existing._key
|
|
2915
3216
|
FROM ${sheetTable(session)} AS existing
|
|
2916
3217
|
JOIN input_rows ON input_rows._key = existing._key
|
|
2917
3218
|
WHERE (
|
|
3219
|
+
existing._write_version = $12::bigint
|
|
3220
|
+
AND (
|
|
2918
3221
|
existing._status IN ('pending', 'running', 'failed')
|
|
2919
3222
|
AND NOT (${existingNewerTerminalRowSql})
|
|
2920
3223
|
AND (
|
|
@@ -2927,8 +3230,10 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2927
3230
|
AND COALESCE(existing._attempt_seq, 0) < $10::integer
|
|
2928
3231
|
)
|
|
2929
3232
|
)
|
|
2930
|
-
)
|
|
3233
|
+
))
|
|
2931
3234
|
OR (
|
|
3235
|
+
existing._write_version = $12::bigint
|
|
3236
|
+
AND
|
|
2932
3237
|
existing._status = 'enriched'
|
|
2933
3238
|
AND (${existingMissingOutputSql})
|
|
2934
3239
|
AND (
|
|
@@ -2947,6 +3252,9 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2947
3252
|
AND NOT EXISTS (
|
|
2948
3253
|
SELECT 1 FROM pending_rows WHERE pending_rows._key = existing._key
|
|
2949
3254
|
)
|
|
3255
|
+
UNION
|
|
3256
|
+
SELECT _key FROM versioned_completed_rows
|
|
3257
|
+
WHERE NOT needs_recompute
|
|
2950
3258
|
),
|
|
2951
3259
|
blocked_rows AS (
|
|
2952
3260
|
SELECT input_rows._key
|
|
@@ -2969,7 +3277,11 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2969
3277
|
FROM blocked_rows
|
|
2970
3278
|
),
|
|
2971
3279
|
inserted_count_cte AS (
|
|
2972
|
-
SELECT count(*)::bigint AS c
|
|
3280
|
+
SELECT count(*)::bigint AS c
|
|
3281
|
+
FROM inserted_rows
|
|
3282
|
+
WHERE NOT EXISTS (
|
|
3283
|
+
SELECT 1 FROM preexisting_rows WHERE preexisting_rows._key = inserted_rows._key
|
|
3284
|
+
)
|
|
2973
3285
|
),
|
|
2974
3286
|
missing_output_count_cte AS (
|
|
2975
3287
|
SELECT count(*)::bigint AS c FROM missing_output_rows
|
|
@@ -3029,6 +3341,7 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
3029
3341
|
input.attemptExpiresAt,
|
|
3030
3342
|
input.attemptSeq,
|
|
3031
3343
|
input.force === true,
|
|
3344
|
+
input.writeVersion,
|
|
3032
3345
|
],
|
|
3033
3346
|
);
|
|
3034
3347
|
inserted += Number(rows[0]?.inserted_count ?? 0);
|
|
@@ -3065,6 +3378,7 @@ async function buildRuntimeSheetDatasetStartResult(
|
|
|
3065
3378
|
attemptOwnerRunId: string;
|
|
3066
3379
|
attemptExpiresAt: string;
|
|
3067
3380
|
attemptSeq: number;
|
|
3381
|
+
writeVersion: number;
|
|
3068
3382
|
inserted: number;
|
|
3069
3383
|
rowDispositions: RuntimeSheetPreparedRowDisposition[];
|
|
3070
3384
|
force?: boolean;
|
|
@@ -3100,6 +3414,7 @@ async function buildRuntimeSheetDatasetStartResult(
|
|
|
3100
3414
|
completedRows: [],
|
|
3101
3415
|
blockedRows: [],
|
|
3102
3416
|
tableNamespace: input.tableNamespace,
|
|
3417
|
+
writeVersion: input.writeVersion,
|
|
3103
3418
|
};
|
|
3104
3419
|
}
|
|
3105
3420
|
|
|
@@ -3119,6 +3434,7 @@ async function buildRuntimeSheetDatasetStartResult(
|
|
|
3119
3434
|
attemptOwnerRunId: input.attemptOwnerRunId,
|
|
3120
3435
|
attemptExpiresAt: input.attemptExpiresAt,
|
|
3121
3436
|
attemptSeq: input.attemptSeq,
|
|
3437
|
+
writeVersion: input.writeVersion,
|
|
3122
3438
|
normalizedPlayName: input.normalizedPlayName,
|
|
3123
3439
|
normalizedTableNamespace: input.normalizedTableNamespace,
|
|
3124
3440
|
outputFields: input.force === true ? datasetFields : [],
|
|
@@ -3149,6 +3465,7 @@ async function buildRuntimeSheetDatasetStartResult(
|
|
|
3149
3465
|
completedRows: [],
|
|
3150
3466
|
blockedRows,
|
|
3151
3467
|
tableNamespace: input.tableNamespace,
|
|
3468
|
+
writeVersion: input.writeVersion,
|
|
3152
3469
|
};
|
|
3153
3470
|
}
|
|
3154
3471
|
const persistedRows = await readRuntimeRowsByKey(
|
|
@@ -3198,6 +3515,7 @@ async function buildRuntimeSheetDatasetStartResult(
|
|
|
3198
3515
|
completedRows,
|
|
3199
3516
|
blockedRows,
|
|
3200
3517
|
tableNamespace: input.tableNamespace,
|
|
3518
|
+
writeVersion: input.writeVersion ?? undefined,
|
|
3201
3519
|
};
|
|
3202
3520
|
}
|
|
3203
3521
|
|
|
@@ -3211,6 +3529,7 @@ async function markRuntimeRowsPendingForRecompute(
|
|
|
3211
3529
|
attemptOwnerRunId: string;
|
|
3212
3530
|
attemptExpiresAt: string;
|
|
3213
3531
|
attemptSeq: number;
|
|
3532
|
+
writeVersion: number | null;
|
|
3214
3533
|
normalizedPlayName: string;
|
|
3215
3534
|
normalizedTableNamespace: string;
|
|
3216
3535
|
outputFields?: string[];
|
|
@@ -3218,28 +3537,13 @@ async function markRuntimeRowsPendingForRecompute(
|
|
|
3218
3537
|
},
|
|
3219
3538
|
): Promise<void> {
|
|
3220
3539
|
if (input.keys.length === 0) return;
|
|
3221
|
-
const targetAttemptFenceSql = activeRuntimeSheetAttemptFenceSql(
|
|
3222
|
-
'target',
|
|
3223
|
-
'$5::text',
|
|
3224
|
-
'$6::text',
|
|
3225
|
-
'$7::timestamptz',
|
|
3226
|
-
'$8::integer',
|
|
3227
|
-
);
|
|
3228
|
-
const targetOlderForeignAttemptOwnerSql =
|
|
3229
|
-
'coalesce(target._attempt_owner_run_id, target._run_id) IS DISTINCT FROM $6::text AND COALESCE(target._attempt_seq, 0) < $8::integer';
|
|
3230
|
-
const targetWritableEnrichedAttemptSql =
|
|
3231
|
-
writableEnrichedRuntimeSheetAttemptSql(
|
|
3232
|
-
'target',
|
|
3233
|
-
'$6::text',
|
|
3234
|
-
'$7::timestamptz',
|
|
3235
|
-
'$8',
|
|
3236
|
-
);
|
|
3237
3540
|
const outputFields = [...new Set(input.outputFields ?? [])];
|
|
3238
3541
|
await client.query(
|
|
3239
3542
|
`WITH target_rows AS (
|
|
3240
3543
|
SELECT _key, _status, _cell_meta
|
|
3241
3544
|
FROM ${sheetTable(session)}
|
|
3242
3545
|
WHERE _key = ANY($1::text[])
|
|
3546
|
+
AND ($10::boolean OR NOT $10::boolean)
|
|
3243
3547
|
FOR UPDATE
|
|
3244
3548
|
),
|
|
3245
3549
|
updated AS (
|
|
@@ -3250,33 +3554,13 @@ async function markRuntimeRowsPendingForRecompute(
|
|
|
3250
3554
|
_attempt_owner_run_id = $6::text,
|
|
3251
3555
|
_attempt_expires_at = $7::timestamptz,
|
|
3252
3556
|
_attempt_seq = $8::integer,
|
|
3557
|
+
_write_version = $11::bigint,
|
|
3558
|
+
_writer_run_id = $2::text,
|
|
3253
3559
|
_updated_at = now(),
|
|
3254
3560
|
_version = ${nextRuntimeSheetVersionExpression(session)}
|
|
3255
3561
|
FROM target_rows
|
|
3256
3562
|
WHERE target._key = target_rows._key
|
|
3257
|
-
AND
|
|
3258
|
-
(
|
|
3259
|
-
target._status IN ('pending', 'running', 'failed')
|
|
3260
|
-
AND (
|
|
3261
|
-
${targetAttemptFenceSql}
|
|
3262
|
-
OR ${targetOlderForeignAttemptOwnerSql}
|
|
3263
|
-
OR (
|
|
3264
|
-
$10::boolean
|
|
3265
|
-
AND target._status = 'failed'
|
|
3266
|
-
)
|
|
3267
|
-
)
|
|
3268
|
-
)
|
|
3269
|
-
OR ${targetAttemptFenceSql}
|
|
3270
|
-
OR (
|
|
3271
|
-
$10::boolean
|
|
3272
|
-
AND target._status = 'enriched'
|
|
3273
|
-
AND COALESCE(target._attempt_seq, 0) <= $8::integer
|
|
3274
|
-
)
|
|
3275
|
-
OR (
|
|
3276
|
-
target._status = 'enriched'
|
|
3277
|
-
AND ${targetWritableEnrichedAttemptSql}
|
|
3278
|
-
)
|
|
3279
|
-
)
|
|
3563
|
+
AND target._write_version = $11::bigint
|
|
3280
3564
|
AND (
|
|
3281
3565
|
target._status <> 'pending'
|
|
3282
3566
|
OR target._run_id IS DISTINCT FROM $2::text
|
|
@@ -3366,6 +3650,7 @@ async function markRuntimeRowsPendingForRecompute(
|
|
|
3366
3650
|
input.attemptSeq,
|
|
3367
3651
|
outputFields,
|
|
3368
3652
|
input.force === true,
|
|
3653
|
+
input.writeVersion,
|
|
3369
3654
|
],
|
|
3370
3655
|
);
|
|
3371
3656
|
}
|
|
@@ -3555,6 +3840,7 @@ export async function claimRuntimeWorkReceipt(
|
|
|
3555
3840
|
playName: string;
|
|
3556
3841
|
runId: string;
|
|
3557
3842
|
key: string;
|
|
3843
|
+
leaseId?: string | null;
|
|
3558
3844
|
runAttempt?: number | null;
|
|
3559
3845
|
leaseAware?: boolean;
|
|
3560
3846
|
reclaimRunning?: boolean;
|
|
@@ -3572,7 +3858,8 @@ export async function claimRuntimeWorkReceipt(
|
|
|
3572
3858
|
session,
|
|
3573
3859
|
async (client) => {
|
|
3574
3860
|
const leaseId =
|
|
3575
|
-
input.
|
|
3861
|
+
input.leaseId?.trim() ||
|
|
3862
|
+
(input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null);
|
|
3576
3863
|
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
3577
3864
|
const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
|
|
3578
3865
|
input.leaseTtlMs,
|
|
@@ -3700,6 +3987,15 @@ export async function claimRuntimeWorkReceipt(
|
|
|
3700
3987
|
}
|
|
3701
3988
|
|
|
3702
3989
|
const latest = await readRuntimeWorkReceipt(client, session, input.key);
|
|
3990
|
+
if (
|
|
3991
|
+
latest &&
|
|
3992
|
+
leaseId !== null &&
|
|
3993
|
+
latest.leaseId === leaseId &&
|
|
3994
|
+
(latest.leaseOwnerRunId ?? latest.runId) === input.runId &&
|
|
3995
|
+
(latest.leaseOwnerAttempt ?? 0) === runAttempt
|
|
3996
|
+
) {
|
|
3997
|
+
return { disposition: 'claimed', receipt: latest };
|
|
3998
|
+
}
|
|
3703
3999
|
if (
|
|
3704
4000
|
latest &&
|
|
3705
4001
|
input.forceRefresh !== true &&
|
|
@@ -3761,6 +4057,7 @@ export async function claimRuntimeWorkReceipts(
|
|
|
3761
4057
|
playName: string;
|
|
3762
4058
|
runId: string;
|
|
3763
4059
|
keys: string[];
|
|
4060
|
+
leaseIds?: string[];
|
|
3764
4061
|
runAttempt?: number | null;
|
|
3765
4062
|
leaseAware?: boolean;
|
|
3766
4063
|
reclaimRunning?: boolean;
|
|
@@ -3782,9 +4079,21 @@ export async function claimRuntimeWorkReceipts(
|
|
|
3782
4079
|
session,
|
|
3783
4080
|
async (client) => {
|
|
3784
4081
|
const keyHexes = keys.map(workReceiptKeyHex);
|
|
3785
|
-
|
|
3786
|
-
input.
|
|
3787
|
-
|
|
4082
|
+
if (
|
|
4083
|
+
input.leaseIds !== undefined &&
|
|
4084
|
+
input.leaseIds.length !== keys.length
|
|
4085
|
+
) {
|
|
4086
|
+
throw new Error(
|
|
4087
|
+
`Runtime receipt bulk claim requires one lease ID per key. Received ${input.leaseIds.length} lease IDs for ${keys.length} keys.`,
|
|
4088
|
+
);
|
|
4089
|
+
}
|
|
4090
|
+
const leaseIds = keys.map((_, index) => {
|
|
4091
|
+
const providedLeaseId = input.leaseIds?.[index]?.trim();
|
|
4092
|
+
return (
|
|
4093
|
+
providedLeaseId ||
|
|
4094
|
+
(input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null)
|
|
4095
|
+
);
|
|
4096
|
+
});
|
|
3788
4097
|
const claimStatus =
|
|
3789
4098
|
input.leaseAware === true
|
|
3790
4099
|
? RECEIPT_STATUS_QUEUED
|
|
@@ -3887,7 +4196,10 @@ export async function claimRuntimeWorkReceipts(
|
|
|
3887
4196
|
receipts.lease_expires_at,
|
|
3888
4197
|
receipts.updated_at,
|
|
3889
4198
|
input_keys.ord,
|
|
3890
|
-
|
|
4199
|
+
receipts.lease_id = input_keys.lease_id
|
|
4200
|
+
AND receipts.lease_owner_run_id = $4::text
|
|
4201
|
+
AND COALESCE(receipts.lease_owner_attempt, 0) = $9::integer
|
|
4202
|
+
AS claimed
|
|
3891
4203
|
FROM input_keys
|
|
3892
4204
|
JOIN ${workReceiptTable(session)} AS receipts
|
|
3893
4205
|
ON receipts.k = decode(input_keys.key_hex, 'hex')
|
|
@@ -4363,7 +4675,48 @@ export async function completeRuntimeWorkReceipt(
|
|
|
4363
4675
|
context,
|
|
4364
4676
|
session,
|
|
4365
4677
|
async (client) => {
|
|
4366
|
-
const
|
|
4678
|
+
const immutableCacheInsert =
|
|
4679
|
+
input.leaseId === COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID;
|
|
4680
|
+
const leaseId = immutableCacheInsert
|
|
4681
|
+
? null
|
|
4682
|
+
: input.leaseId?.trim() || null;
|
|
4683
|
+
if (immutableCacheInsert) {
|
|
4684
|
+
const { rows: insertedRows } = await client.query<
|
|
4685
|
+
Record<string, unknown>
|
|
4686
|
+
>(
|
|
4687
|
+
`
|
|
4688
|
+
INSERT INTO ${workReceiptTable(session)} (
|
|
4689
|
+
k, status, output, error, failure_kind, run_id, updated_at
|
|
4690
|
+
) VALUES (
|
|
4691
|
+
decode($1, 'hex'), $2::smallint, $3::jsonb, NULL, 0, $4, now()
|
|
4692
|
+
)
|
|
4693
|
+
ON CONFLICT (k) DO NOTHING
|
|
4694
|
+
RETURNING convert_from(k, 'UTF8') AS k, status, output, error,
|
|
4695
|
+
failure_kind, run_id, lease_id, lease_owner_run_id,
|
|
4696
|
+
lease_owner_attempt, lease_expires_at, updated_at
|
|
4697
|
+
`,
|
|
4698
|
+
[
|
|
4699
|
+
workReceiptKeyHex(input.key),
|
|
4700
|
+
RECEIPT_STATUS_COMPLETED,
|
|
4701
|
+
input.output === null ? null : stringifyPostgresJson(input.output),
|
|
4702
|
+
input.runId,
|
|
4703
|
+
],
|
|
4704
|
+
);
|
|
4705
|
+
if (insertedRows[0]) return mapRuntimeWorkReceiptRow(insertedRows[0]);
|
|
4706
|
+
const { rows: winnerRows } = await client.query<
|
|
4707
|
+
Record<string, unknown>
|
|
4708
|
+
>(
|
|
4709
|
+
`SELECT convert_from(k, 'UTF8') AS k, status, output, error,
|
|
4710
|
+
failure_kind, run_id, lease_id, lease_owner_run_id,
|
|
4711
|
+
lease_owner_attempt, lease_expires_at, updated_at
|
|
4712
|
+
FROM ${workReceiptTable(session)}
|
|
4713
|
+
WHERE k = decode($1, 'hex')
|
|
4714
|
+
AND status IN ($2::smallint, ${RECEIPT_STATUS_SKIPPED}::smallint)
|
|
4715
|
+
LIMIT 1`,
|
|
4716
|
+
[workReceiptKeyHex(input.key), RECEIPT_STATUS_COMPLETED],
|
|
4717
|
+
);
|
|
4718
|
+
return winnerRows[0] ? mapRuntimeWorkReceiptRow(winnerRows[0]) : null;
|
|
4719
|
+
}
|
|
4367
4720
|
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
4368
4721
|
const { rows } = await client.query<Record<string, unknown>>(
|
|
4369
4722
|
`
|
|
@@ -4498,6 +4851,44 @@ export async function completeRuntimeWorkReceipts(
|
|
|
4498
4851
|
}
|
|
4499
4852
|
throw error;
|
|
4500
4853
|
}
|
|
4854
|
+
const immutableEntries = receipts
|
|
4855
|
+
.map((receipt, index) => ({ receipt, index }))
|
|
4856
|
+
.filter(
|
|
4857
|
+
({ receipt }) =>
|
|
4858
|
+
receipt.leaseId === COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
|
|
4859
|
+
);
|
|
4860
|
+
if (immutableEntries.length > 0) {
|
|
4861
|
+
const leasedEntries = receipts
|
|
4862
|
+
.map((receipt, index) => ({ receipt, index }))
|
|
4863
|
+
.filter(
|
|
4864
|
+
({ receipt }) =>
|
|
4865
|
+
receipt.leaseId !== COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
|
|
4866
|
+
);
|
|
4867
|
+
const resolved = Array<WorkReceipt | null>(receipts.length).fill(null);
|
|
4868
|
+
const [immutableResults, leasedResults] = await Promise.all([
|
|
4869
|
+
Promise.all(
|
|
4870
|
+
immutableEntries.map(({ receipt }) =>
|
|
4871
|
+
completeRuntimeWorkReceipt(context, {
|
|
4872
|
+
playName: input.playName,
|
|
4873
|
+
...receipt,
|
|
4874
|
+
}),
|
|
4875
|
+
),
|
|
4876
|
+
),
|
|
4877
|
+
leasedEntries.length > 0
|
|
4878
|
+
? completeRuntimeWorkReceipts(context, {
|
|
4879
|
+
playName: input.playName,
|
|
4880
|
+
receipts: leasedEntries.map(({ receipt }) => receipt),
|
|
4881
|
+
})
|
|
4882
|
+
: Promise.resolve([]),
|
|
4883
|
+
]);
|
|
4884
|
+
immutableEntries.forEach(({ index }, resultIndex) => {
|
|
4885
|
+
resolved[index] = immutableResults[resultIndex] ?? null;
|
|
4886
|
+
});
|
|
4887
|
+
leasedEntries.forEach(({ index }, resultIndex) => {
|
|
4888
|
+
resolved[index] = leasedResults[resultIndex] ?? null;
|
|
4889
|
+
});
|
|
4890
|
+
return resolved;
|
|
4891
|
+
}
|
|
4501
4892
|
const session = await getRuntimeWorkReceiptSessionForKeys(context, {
|
|
4502
4893
|
playName: input.playName,
|
|
4503
4894
|
keys: receipts.map((receipt) => receipt.key),
|
|
@@ -5147,6 +5538,7 @@ export async function startRuntimeSheetDataset(
|
|
|
5147
5538
|
attemptExpiresAt?: string | null;
|
|
5148
5539
|
attemptSeq?: number | null;
|
|
5149
5540
|
attemptLeaseTtlMs?: number | null;
|
|
5541
|
+
writeVersion?: number | null;
|
|
5150
5542
|
inputOffset?: number;
|
|
5151
5543
|
force?: boolean;
|
|
5152
5544
|
},
|
|
@@ -5188,6 +5580,14 @@ export async function startRuntimeSheetDataset(
|
|
|
5188
5580
|
ms: Date.now() - sessionStartedAt,
|
|
5189
5581
|
rows: input.rows.length,
|
|
5190
5582
|
});
|
|
5583
|
+
// Register the exact run-scoped Dataset Handle before any row can become
|
|
5584
|
+
// durable. A crash after a later sheet write can therefore never leave
|
|
5585
|
+
// customer data discoverable only through static pipeline inference.
|
|
5586
|
+
await registerRuntimeDataset(session, {
|
|
5587
|
+
playName,
|
|
5588
|
+
tableNamespace: input.tableNamespace,
|
|
5589
|
+
runId: input.runId,
|
|
5590
|
+
});
|
|
5191
5591
|
const normalizeStartedAt = Date.now();
|
|
5192
5592
|
const uniqueRows = new Map<
|
|
5193
5593
|
string,
|
|
@@ -5285,7 +5685,10 @@ export async function startRuntimeSheetDataset(
|
|
|
5285
5685
|
});
|
|
5286
5686
|
const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
|
|
5287
5687
|
const chunks = chunkValues(rowEntries, DIRECT_POSTGRES_BATCH_SIZE);
|
|
5288
|
-
|
|
5688
|
+
// Scheduling is one admission decision. Even when batching SQL for the
|
|
5689
|
+
// parameter limit, either every requested row receives this intent version
|
|
5690
|
+
// or none do; a run can never leave 42 pending rows after 8 conflicts.
|
|
5691
|
+
const needsTransaction = true;
|
|
5289
5692
|
|
|
5290
5693
|
const result = await withRuntimeSheetQueryClient(
|
|
5291
5694
|
context,
|
|
@@ -5303,6 +5706,11 @@ export async function startRuntimeSheetDataset(
|
|
|
5303
5706
|
client,
|
|
5304
5707
|
input.attemptLeaseTtlMs,
|
|
5305
5708
|
);
|
|
5709
|
+
const writeVersion = await mintRuntimeSheetWriteVersion(
|
|
5710
|
+
client,
|
|
5711
|
+
session,
|
|
5712
|
+
input.writeVersion,
|
|
5713
|
+
);
|
|
5306
5714
|
timings.push({
|
|
5307
5715
|
phase: 'attempt_expiry_mint',
|
|
5308
5716
|
ms: Date.now() - attemptExpiryStartedAt,
|
|
@@ -5318,6 +5726,7 @@ export async function startRuntimeSheetDataset(
|
|
|
5318
5726
|
attemptOwnerRunId,
|
|
5319
5727
|
attemptExpiresAt,
|
|
5320
5728
|
attemptSeq,
|
|
5729
|
+
writeVersion,
|
|
5321
5730
|
physicalInsertColumnsSql,
|
|
5322
5731
|
physicalInsertValuesSql,
|
|
5323
5732
|
physicalRefreshSetSql:
|
|
@@ -5350,6 +5759,7 @@ export async function startRuntimeSheetDataset(
|
|
|
5350
5759
|
attemptOwnerRunId,
|
|
5351
5760
|
attemptExpiresAt,
|
|
5352
5761
|
attemptSeq,
|
|
5762
|
+
writeVersion,
|
|
5353
5763
|
timings,
|
|
5354
5764
|
force: input.force === true,
|
|
5355
5765
|
...prepared,
|
|
@@ -5367,6 +5777,7 @@ export async function startRuntimeSheetDataset(
|
|
|
5367
5777
|
...built,
|
|
5368
5778
|
attemptExpiresAt,
|
|
5369
5779
|
attemptSeq,
|
|
5780
|
+
writeVersion,
|
|
5370
5781
|
};
|
|
5371
5782
|
},
|
|
5372
5783
|
);
|
|
@@ -5386,6 +5797,7 @@ export async function startRuntimeSheetDataset(
|
|
|
5386
5797
|
attemptOwnerRunId,
|
|
5387
5798
|
attemptExpiresAt: result.attemptExpiresAt,
|
|
5388
5799
|
attemptSeq,
|
|
5800
|
+
writeVersion: result.writeVersion,
|
|
5389
5801
|
timings,
|
|
5390
5802
|
};
|
|
5391
5803
|
}
|
|
@@ -5695,6 +6107,7 @@ type CompleteRuntimeMapRowChunksInput = {
|
|
|
5695
6107
|
attemptOwnerRunId: string | null;
|
|
5696
6108
|
attemptExpiresAt: string | null;
|
|
5697
6109
|
attemptSeq: number;
|
|
6110
|
+
writeVersion: number | null;
|
|
5698
6111
|
normalizedPlayName: string;
|
|
5699
6112
|
normalizedTableNamespace: string;
|
|
5700
6113
|
outputFields: string[];
|
|
@@ -5703,6 +6116,8 @@ type CompleteRuntimeMapRowChunksInput = {
|
|
|
5703
6116
|
export type RuntimeMapRowsWriteResult = {
|
|
5704
6117
|
updated: number;
|
|
5705
6118
|
fencedKeys: string[];
|
|
6119
|
+
staleDropped?: number;
|
|
6120
|
+
staleDroppedKeys?: string[];
|
|
5706
6121
|
};
|
|
5707
6122
|
|
|
5708
6123
|
function writableEnrichedRuntimeSheetAttemptSql(
|
|
@@ -5791,6 +6206,8 @@ async function completeRuntimeMapRowChunks(
|
|
|
5791
6206
|
UPDATE ${sheetTable(session)} AS target
|
|
5792
6207
|
SET _status = 'enriched',
|
|
5793
6208
|
_run_id = $5::text,
|
|
6209
|
+
_writer_run_id = $5::text,
|
|
6210
|
+
_write_version = COALESCE($13::bigint, target._write_version),
|
|
5794
6211
|
_error = NULL,
|
|
5795
6212
|
_attempt_id = $9::text,
|
|
5796
6213
|
_attempt_owner_run_id = $10::text,
|
|
@@ -5801,6 +6218,8 @@ async function completeRuntimeMapRowChunks(
|
|
|
5801
6218
|
_cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
|
|
5802
6219
|
FROM matched_updates AS updates
|
|
5803
6220
|
WHERE target._key = updates.matched_key
|
|
6221
|
+
AND ($13::bigint IS NULL OR target._write_version = $13::bigint)
|
|
6222
|
+
AND ($13::bigint IS NOT NULL OR target._status <> 'enriched')
|
|
5804
6223
|
AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
|
|
5805
6224
|
AND NOT (${sameOwnerTerminalEpochSql})
|
|
5806
6225
|
AND (
|
|
@@ -5837,6 +6256,9 @@ async function completeRuntimeMapRowChunks(
|
|
|
5837
6256
|
SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
|
|
5838
6257
|
)
|
|
5839
6258
|
AND (
|
|
6259
|
+
($13::bigint IS NOT NULL AND target._write_version <> $13::bigint)
|
|
6260
|
+
OR ($13::bigint IS NULL AND target._status = 'enriched')
|
|
6261
|
+
OR
|
|
5840
6262
|
(${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
|
|
5841
6263
|
OR NOT (
|
|
5842
6264
|
($9::text IS NULL AND target._run_id = $5::text)
|
|
@@ -5946,12 +6368,18 @@ async function completeRuntimeMapRowChunks(
|
|
|
5946
6368
|
input.attemptOwnerRunId,
|
|
5947
6369
|
input.attemptExpiresAt,
|
|
5948
6370
|
input.attemptSeq,
|
|
6371
|
+
input.writeVersion,
|
|
5949
6372
|
],
|
|
5950
6373
|
);
|
|
5951
|
-
updated += Number(rows[0]?.updated ?? 0);
|
|
5952
|
-
fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
|
|
5953
|
-
|
|
5954
6374
|
const matchedKeys = new Set(rows[0]?.matched_keys ?? []);
|
|
6375
|
+
const chunkFencedKeys = rows[0]?.fenced_keys ?? [];
|
|
6376
|
+
// An exact-version replay that already produced the same terminal value is
|
|
6377
|
+
// accepted idempotently even when SQL has no physical mutation to apply.
|
|
6378
|
+
// Stale versions remain excluded through fenced_keys.
|
|
6379
|
+
const appliedCount = Number(rows[0]?.updated ?? 0);
|
|
6380
|
+
updated += appliedCount;
|
|
6381
|
+
fencedKeys.push(...chunkFencedKeys);
|
|
6382
|
+
|
|
5955
6383
|
if (matchedKeys.size === chunk.length) {
|
|
5956
6384
|
continue;
|
|
5957
6385
|
}
|
|
@@ -6388,6 +6816,7 @@ async function failRuntimeMapRowChunks(
|
|
|
6388
6816
|
attemptOwnerRunId: string | null;
|
|
6389
6817
|
attemptExpiresAt: string | null;
|
|
6390
6818
|
attemptSeq: number;
|
|
6819
|
+
writeVersion: number | null;
|
|
6391
6820
|
normalizedPlayName: string;
|
|
6392
6821
|
normalizedTableNamespace: string;
|
|
6393
6822
|
},
|
|
@@ -6462,6 +6891,8 @@ async function failRuntimeMapRowChunks(
|
|
|
6462
6891
|
UPDATE ${sheetTable(session)} AS target
|
|
6463
6892
|
SET _status = 'failed',
|
|
6464
6893
|
_run_id = $6::text,
|
|
6894
|
+
_writer_run_id = $6::text,
|
|
6895
|
+
_write_version = COALESCE($16::bigint, target._write_version),
|
|
6465
6896
|
_error = updates.error,
|
|
6466
6897
|
_attempt_id = $11::text,
|
|
6467
6898
|
_attempt_owner_run_id = $12::text,
|
|
@@ -6472,6 +6903,7 @@ async function failRuntimeMapRowChunks(
|
|
|
6472
6903
|
_cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
|
|
6473
6904
|
FROM matched_updates AS updates, ${sheetTable(session)} AS prev
|
|
6474
6905
|
WHERE target._key = updates.matched_key
|
|
6906
|
+
AND ($16::bigint IS NULL OR target._write_version = $16::bigint)
|
|
6475
6907
|
AND prev._key = target._key
|
|
6476
6908
|
AND ($15::boolean OR target._status <> 'enriched')
|
|
6477
6909
|
AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$13::timestamptz', '$14::integer', '$11::text', '$12::text')})
|
|
@@ -6497,6 +6929,8 @@ async function failRuntimeMapRowChunks(
|
|
|
6497
6929
|
SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
|
|
6498
6930
|
)
|
|
6499
6931
|
AND (
|
|
6932
|
+
($16::bigint IS NOT NULL AND target._write_version <> $16::bigint)
|
|
6933
|
+
OR
|
|
6500
6934
|
(NOT $15::boolean AND target._status = 'enriched')
|
|
6501
6935
|
OR (${newerTerminalRuntimeSheetRowSql('target', '$13::timestamptz', '$14::integer', '$11::text', '$12::text')})
|
|
6502
6936
|
OR NOT (
|
|
@@ -6583,6 +7017,7 @@ async function failRuntimeMapRowChunks(
|
|
|
6583
7017
|
input.attemptExpiresAt,
|
|
6584
7018
|
input.attemptSeq,
|
|
6585
7019
|
input.forceTerminal === true,
|
|
7020
|
+
input.writeVersion,
|
|
6586
7021
|
],
|
|
6587
7022
|
);
|
|
6588
7023
|
updated += Number(rows[0]?.updated ?? 0);
|
|
@@ -6593,9 +7028,8 @@ async function failRuntimeMapRowChunks(
|
|
|
6593
7028
|
|
|
6594
7029
|
/**
|
|
6595
7030
|
* Mark map rows terminal in the per-run scoped Postgres sheet table by key.
|
|
6596
|
-
*
|
|
6597
|
-
*
|
|
6598
|
-
* columns, _status flipped to 'enriched', and the sheet cursor advanced.
|
|
7031
|
+
* Writes enriched output values into materialized physical columns, flips
|
|
7032
|
+
* _status to 'enriched', and advances the sheet cursor.
|
|
6599
7033
|
*
|
|
6600
7034
|
* Rows with `status: 'failed'` instead persist a row-isolated failure
|
|
6601
7035
|
* (`_status='failed'`, `_error`, per-cell failure meta) while keeping the
|
|
@@ -6618,6 +7052,7 @@ export async function completeRuntimeMapRows(
|
|
|
6618
7052
|
attemptOwnerRunId?: string | null;
|
|
6619
7053
|
attemptExpiresAt?: string | null;
|
|
6620
7054
|
attemptSeq?: number | null;
|
|
7055
|
+
writeVersion?: number | null;
|
|
6621
7056
|
forceFailedRows?: boolean;
|
|
6622
7057
|
},
|
|
6623
7058
|
): Promise<RuntimeMapRowsWriteResult> {
|
|
@@ -6701,6 +7136,15 @@ export async function completeRuntimeMapRows(
|
|
|
6701
7136
|
input.attemptOwnerRunId?.trim() || (attemptId ? input.runId : null);
|
|
6702
7137
|
const attemptExpiresAt = input.attemptExpiresAt?.trim() || null;
|
|
6703
7138
|
const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
|
|
7139
|
+
const writeVersion = input.writeVersion == null ? null : input.writeVersion;
|
|
7140
|
+
if (
|
|
7141
|
+
writeVersion !== null &&
|
|
7142
|
+
(!Number.isSafeInteger(writeVersion) || writeVersion <= 0)
|
|
7143
|
+
) {
|
|
7144
|
+
throw new Error(
|
|
7145
|
+
'Runtime sheet writeVersion must be a positive safe integer.',
|
|
7146
|
+
);
|
|
7147
|
+
}
|
|
6704
7148
|
|
|
6705
7149
|
return await withRuntimeSheetQueryClient(
|
|
6706
7150
|
context,
|
|
@@ -6724,6 +7168,7 @@ export async function completeRuntimeMapRows(
|
|
|
6724
7168
|
attemptOwnerRunId,
|
|
6725
7169
|
attemptExpiresAt,
|
|
6726
7170
|
attemptSeq,
|
|
7171
|
+
writeVersion,
|
|
6727
7172
|
normalizedPlayName: normalizePlayNameForSheet(session.playName),
|
|
6728
7173
|
normalizedTableNamespace: normalizeTableNamespace(
|
|
6729
7174
|
input.tableNamespace,
|
|
@@ -6757,6 +7202,7 @@ export async function completeRuntimeMapRows(
|
|
|
6757
7202
|
attemptOwnerRunId,
|
|
6758
7203
|
attemptExpiresAt,
|
|
6759
7204
|
attemptSeq,
|
|
7205
|
+
writeVersion,
|
|
6760
7206
|
normalizedPlayName: normalizePlayNameForSheet(session.playName),
|
|
6761
7207
|
normalizedTableNamespace: normalizeTableNamespace(
|
|
6762
7208
|
input.tableNamespace,
|
|
@@ -6766,6 +7212,8 @@ export async function completeRuntimeMapRows(
|
|
|
6766
7212
|
return {
|
|
6767
7213
|
updated: completed.updated + failed.updated,
|
|
6768
7214
|
fencedKeys: [...completed.fencedKeys, ...failed.fencedKeys],
|
|
7215
|
+
staleDropped: completed.fencedKeys.length + failed.fencedKeys.length,
|
|
7216
|
+
staleDroppedKeys: [...completed.fencedKeys, ...failed.fencedKeys],
|
|
6769
7217
|
};
|
|
6770
7218
|
},
|
|
6771
7219
|
);
|