deepline 0.1.259 → 0.1.260
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/config.ts +110 -8
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +124 -34
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-postgres-admission.ts +130 -21
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +21 -0
- package/dist/cli/index.js +1642 -496
- package/dist/cli/index.mjs +1783 -629
- package/dist/index.js +1 -1
- package/dist/index.mjs +2 -2
- package/package.json +1 -1
|
@@ -41,7 +41,7 @@ import {
|
|
|
41
41
|
writeFileSync,
|
|
42
42
|
} from 'node:fs';
|
|
43
43
|
import { homedir } from 'node:os';
|
|
44
|
-
import { dirname, join, resolve } from 'node:path';
|
|
44
|
+
import { dirname, isAbsolute, join, resolve } from 'node:path';
|
|
45
45
|
import type { DeeplineClientOptions, ResolvedConfig } from './types.js';
|
|
46
46
|
import { ConfigError } from './errors.js';
|
|
47
47
|
|
|
@@ -429,6 +429,26 @@ export function resolveApiKeyForBaseUrl(
|
|
|
429
429
|
);
|
|
430
430
|
}
|
|
431
431
|
|
|
432
|
+
export function resolveProjectApiKeyForBaseUrl(
|
|
433
|
+
baseUrl: string,
|
|
434
|
+
startDir: string = process.cwd(),
|
|
435
|
+
): string {
|
|
436
|
+
const normalizedBaseUrl = normalizeBaseUrl(baseUrl);
|
|
437
|
+
return firstNonEmpty(
|
|
438
|
+
...loadProjectEnvCandidates(startDir).map(({ env }) => {
|
|
439
|
+
const projectBaseUrl = normalizeBaseUrl(env[HOST_URL_ENV] ?? '');
|
|
440
|
+
return projectBaseUrl === normalizedBaseUrl ? env[API_KEY_ENV] : '';
|
|
441
|
+
}),
|
|
442
|
+
);
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
export function resolveGlobalApiKeyForBaseUrl(baseUrl: string): string {
|
|
446
|
+
return firstNonEmpty(
|
|
447
|
+
process.env[API_KEY_ENV],
|
|
448
|
+
loadCliEnv(normalizeBaseUrl(baseUrl) || baseUrl)[API_KEY_ENV],
|
|
449
|
+
);
|
|
450
|
+
}
|
|
451
|
+
|
|
432
452
|
function getResolvedProjectAuthSource(
|
|
433
453
|
baseUrl: string,
|
|
434
454
|
apiKey: string,
|
|
@@ -491,18 +511,100 @@ function mergeProjectEnvFile(filePath: string, values: EnvValues): void {
|
|
|
491
511
|
}
|
|
492
512
|
|
|
493
513
|
function ensureProjectEnvIsIgnored(dir: string): void {
|
|
514
|
+
ensureProjectPrivatePathsIgnored(dir, [
|
|
515
|
+
PROJECT_DEEPLINE_ENV_FILE,
|
|
516
|
+
'.deepline/',
|
|
517
|
+
]);
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
export function ensureProjectPrivatePathsIgnored(
|
|
521
|
+
dir: string,
|
|
522
|
+
entries: readonly string[],
|
|
523
|
+
): void {
|
|
524
|
+
const gitDir = findNearestGitCommonDir(dir);
|
|
525
|
+
if (gitDir) {
|
|
526
|
+
const excludePath = join(gitDir, 'info', 'exclude');
|
|
527
|
+
const existing = existsSync(excludePath)
|
|
528
|
+
? readFileSync(excludePath, 'utf-8')
|
|
529
|
+
: '';
|
|
530
|
+
const existingEntries = new Set(
|
|
531
|
+
existing
|
|
532
|
+
.split(/\r?\n/)
|
|
533
|
+
.map((line) => line.trim())
|
|
534
|
+
.filter(Boolean),
|
|
535
|
+
);
|
|
536
|
+
const missing = entries.filter(
|
|
537
|
+
(entry) =>
|
|
538
|
+
!existingEntries.has(entry) && !existingEntries.has(`/${entry}`),
|
|
539
|
+
);
|
|
540
|
+
if (missing.length === 0) return;
|
|
541
|
+
mkdirSync(dirname(excludePath), { recursive: true });
|
|
542
|
+
const prefix = existing && !existing.endsWith('\n') ? '\n' : '';
|
|
543
|
+
writeFileSync(
|
|
544
|
+
excludePath,
|
|
545
|
+
`${existing}${prefix}${missing.join('\n')}\n`,
|
|
546
|
+
'utf-8',
|
|
547
|
+
);
|
|
548
|
+
return;
|
|
549
|
+
}
|
|
550
|
+
|
|
494
551
|
const gitignorePath = join(dir, '.gitignore');
|
|
495
|
-
const entry = PROJECT_DEEPLINE_ENV_FILE;
|
|
496
552
|
const existing = existsSync(gitignorePath)
|
|
497
553
|
? readFileSync(gitignorePath, 'utf-8')
|
|
498
554
|
: '';
|
|
499
|
-
const
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
555
|
+
const existingEntries = new Set(
|
|
556
|
+
existing
|
|
557
|
+
.split(/\r?\n/)
|
|
558
|
+
.map((line) => line.trim())
|
|
559
|
+
.filter(Boolean),
|
|
560
|
+
);
|
|
561
|
+
const missing = entries.filter(
|
|
562
|
+
(entry) => !existingEntries.has(entry) && !existingEntries.has(`/${entry}`),
|
|
563
|
+
);
|
|
564
|
+
if (missing.length === 0) return;
|
|
504
565
|
const prefix = existing && !existing.endsWith('\n') ? '\n' : '';
|
|
505
|
-
writeFileSync(
|
|
566
|
+
writeFileSync(
|
|
567
|
+
gitignorePath,
|
|
568
|
+
`${existing}${prefix}${missing.join('\n')}\n`,
|
|
569
|
+
'utf-8',
|
|
570
|
+
);
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
function findNearestGitCommonDir(startDir: string): string | null {
|
|
574
|
+
let current = resolve(startDir);
|
|
575
|
+
while (true) {
|
|
576
|
+
const candidate = join(current, '.git');
|
|
577
|
+
if (existsSync(candidate)) {
|
|
578
|
+
try {
|
|
579
|
+
const stat = statSync(candidate);
|
|
580
|
+
if (stat.isDirectory()) return candidate;
|
|
581
|
+
if (stat.isFile()) {
|
|
582
|
+
const match = readFileSync(candidate, 'utf8').match(
|
|
583
|
+
/^gitdir:\s*(.+)$/m,
|
|
584
|
+
);
|
|
585
|
+
const rawGitDir = match?.[1]?.trim();
|
|
586
|
+
if (rawGitDir) {
|
|
587
|
+
const gitDir = isAbsolute(rawGitDir)
|
|
588
|
+
? rawGitDir
|
|
589
|
+
: resolve(dirname(candidate), rawGitDir);
|
|
590
|
+
const commonDirPath = join(gitDir, 'commondir');
|
|
591
|
+
if (!existsSync(commonDirPath)) return gitDir;
|
|
592
|
+
const rawCommonDir = readFileSync(commonDirPath, 'utf8').trim();
|
|
593
|
+
return rawCommonDir
|
|
594
|
+
? isAbsolute(rawCommonDir)
|
|
595
|
+
? rawCommonDir
|
|
596
|
+
: resolve(gitDir, rawCommonDir)
|
|
597
|
+
: gitDir;
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
} catch {
|
|
601
|
+
return null;
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
const parent = dirname(current);
|
|
605
|
+
if (parent === current) return null;
|
|
606
|
+
current = parent;
|
|
607
|
+
}
|
|
506
608
|
}
|
|
507
609
|
|
|
508
610
|
export function saveProjectDeeplineEnvValues(
|
|
@@ -123,7 +123,7 @@ export const SDK_RELEASE = {
|
|
|
123
123
|
// Deepline-native radars. Older clients must update before discovering,
|
|
124
124
|
// checking, or deploying an unlaunched monitor integration.
|
|
125
125
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
126
|
-
version: '0.1.
|
|
126
|
+
version: '0.1.260',
|
|
127
127
|
apiContract: '2026-07-native-monitor-launch-hard-cutover',
|
|
128
128
|
supportPolicy: {
|
|
129
129
|
minimumSupported: '0.1.53',
|
|
@@ -108,6 +108,7 @@ import {
|
|
|
108
108
|
import { COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID } from './durable-receipt-execution';
|
|
109
109
|
import { vercelProtectionBypassHeader } from './vercel-protection';
|
|
110
110
|
import {
|
|
111
|
+
isRuntimePostgresAdmissionError,
|
|
111
112
|
RuntimePostgresAdmission,
|
|
112
113
|
type RuntimePostgresAdmissionSnapshot,
|
|
113
114
|
type RuntimePostgresLane,
|
|
@@ -127,6 +128,7 @@ type RuntimeApiContext = {
|
|
|
127
128
|
runtimeTestFaultHeader?: string | null;
|
|
128
129
|
disablePostgresPoolCache?: boolean | null;
|
|
129
130
|
postgresSessionUnwrapKey?: string | null;
|
|
131
|
+
abortSignal?: AbortSignal | null;
|
|
130
132
|
};
|
|
131
133
|
|
|
132
134
|
const RUNTIME_RUN_DATASET_CATALOG_TABLE = '_deepline_run_datasets';
|
|
@@ -319,6 +321,10 @@ const RUNTIME_API_RESOLVE_PLAY_RETRY_DELAYS_MS = [
|
|
|
319
321
|
] as const;
|
|
320
322
|
const RUNTIME_API_DEFAULT_RETRY_AFTER_MS = 2_000;
|
|
321
323
|
const RUNTIME_API_REQUEST_TIMEOUT_MS = 30_000;
|
|
324
|
+
// A sheet admission can legitimately wait behind two large transactional
|
|
325
|
+
// writes. Keep the caller alive long enough for the FIFO admission queue to
|
|
326
|
+
// make progress instead of aborting a healthy queued start after 30 seconds.
|
|
327
|
+
const RUNTIME_SHEET_ADMISSION_REQUEST_TIMEOUT_MS = 10 * 60_000;
|
|
322
328
|
const RUNTIME_POSTGRES_PREWARM_MAX_ATTEMPTS = 4;
|
|
323
329
|
const RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS = [250, 750, 1_500] as const;
|
|
324
330
|
const RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS = 4;
|
|
@@ -338,8 +344,12 @@ const RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS = 10_000;
|
|
|
338
344
|
// Neon Pool and node-postgres may queue internally, but runtime callers never
|
|
339
345
|
// enter those unobservable queues without first holding a lane permit.
|
|
340
346
|
const RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE = 2;
|
|
341
|
-
const
|
|
342
|
-
|
|
347
|
+
const RUNTIME_POSTGRES_RECEIPT_ADMISSION_MAX_QUEUED = 2_000;
|
|
348
|
+
// Sheet starts retain their parsed request body while waiting. Bound that
|
|
349
|
+
// memory independently from lightweight receipt requests; overflow is a
|
|
350
|
+
// retryable 503 and waits outside this long-lived gateway process.
|
|
351
|
+
const RUNTIME_POSTGRES_SHEET_ADMISSION_MAX_QUEUED = 4;
|
|
352
|
+
const RUNTIME_POSTGRES_RECEIPT_ADMISSION_TIMEOUT_MS = 10_000;
|
|
343
353
|
const RECEIPT_STATUS_QUEUED = RECEIPT_STATUS_CODE.queued;
|
|
344
354
|
const RECEIPT_STATUS_PENDING = RECEIPT_STATUS_CODE.pending;
|
|
345
355
|
const RECEIPT_STATUS_RUNNING = RECEIPT_STATUS_CODE.running;
|
|
@@ -563,7 +573,7 @@ async function postRuntimeApi<TResponse>(
|
|
|
563
573
|
const abortController = new AbortController();
|
|
564
574
|
const timeout = setTimeout(
|
|
565
575
|
() => abortController.abort(),
|
|
566
|
-
|
|
576
|
+
runtimeApiRequestTimeoutMs(body.action),
|
|
567
577
|
);
|
|
568
578
|
try {
|
|
569
579
|
response = await fetch(url, {
|
|
@@ -662,6 +672,14 @@ function runtimeApiRetryDelayMs(
|
|
|
662
672
|
: Math.max(retryAfterMs, configuredDelay);
|
|
663
673
|
}
|
|
664
674
|
|
|
675
|
+
function runtimeApiRequestTimeoutMs(
|
|
676
|
+
action: RuntimeApiActionRequest['action'],
|
|
677
|
+
): number {
|
|
678
|
+
return action === 'runtime_sheet_start'
|
|
679
|
+
? RUNTIME_SHEET_ADMISSION_REQUEST_TIMEOUT_MS
|
|
680
|
+
: RUNTIME_API_REQUEST_TIMEOUT_MS;
|
|
681
|
+
}
|
|
682
|
+
|
|
665
683
|
async function sleepRuntimeApiRetry(
|
|
666
684
|
action: RuntimeApiActionRequest['action'],
|
|
667
685
|
attempt: number,
|
|
@@ -1525,8 +1543,29 @@ async function connectRuntimePostgresPool(
|
|
|
1525
1543
|
pool: RuntimePool,
|
|
1526
1544
|
admission: RuntimePostgresAdmission,
|
|
1527
1545
|
lane: RuntimePostgresLane,
|
|
1546
|
+
postgresUrl: string,
|
|
1547
|
+
signal?: AbortSignal | null,
|
|
1528
1548
|
): Promise<RuntimePoolClient> {
|
|
1529
|
-
const
|
|
1549
|
+
const admissionStartedAt = Date.now();
|
|
1550
|
+
const releaseAdmission = await admission.acquire(lane, {
|
|
1551
|
+
signal: signal ?? undefined,
|
|
1552
|
+
});
|
|
1553
|
+
const admissionWaitMs = Math.max(0, Date.now() - admissionStartedAt);
|
|
1554
|
+
if (admissionWaitMs > 0) {
|
|
1555
|
+
const telemetry = admission.snapshot()[lane];
|
|
1556
|
+
console.info('[runtime-postgres-admission]', {
|
|
1557
|
+
event: 'acquired_after_wait',
|
|
1558
|
+
lane,
|
|
1559
|
+
waitMs: admissionWaitMs,
|
|
1560
|
+
active: telemetry.active,
|
|
1561
|
+
queued: telemetry.queued,
|
|
1562
|
+
poolKeyHash: createHash('sha256')
|
|
1563
|
+
.update(postgresUrl)
|
|
1564
|
+
.digest('hex')
|
|
1565
|
+
.slice(0, 12),
|
|
1566
|
+
phase: 'pool_connect',
|
|
1567
|
+
});
|
|
1568
|
+
}
|
|
1530
1569
|
let timedOut = false;
|
|
1531
1570
|
let timeout: ReturnType<typeof setTimeout> | null = null;
|
|
1532
1571
|
const connectPromise = pool.connect();
|
|
@@ -1534,7 +1573,6 @@ async function connectRuntimePostgresPool(
|
|
|
1534
1573
|
.then((client) => {
|
|
1535
1574
|
if (timedOut) {
|
|
1536
1575
|
client.release();
|
|
1537
|
-
releaseAdmission();
|
|
1538
1576
|
}
|
|
1539
1577
|
})
|
|
1540
1578
|
.catch(() => {});
|
|
@@ -1544,6 +1582,9 @@ async function connectRuntimePostgresPool(
|
|
|
1544
1582
|
new Promise<never>((_, reject) => {
|
|
1545
1583
|
timeout = setTimeout(() => {
|
|
1546
1584
|
timedOut = true;
|
|
1585
|
+
// Do not make a timed-out driver connect pin an admission permit.
|
|
1586
|
+
// A late connect still releases its client in the handler above.
|
|
1587
|
+
releaseAdmission();
|
|
1547
1588
|
reject(
|
|
1548
1589
|
new Error(
|
|
1549
1590
|
`Runtime Postgres connection timed out after ${RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS}ms.`,
|
|
@@ -1596,8 +1637,20 @@ function getRuntimePostgresAdmission(
|
|
|
1596
1637
|
if (existing) return existing;
|
|
1597
1638
|
const admission = new RuntimePostgresAdmission({
|
|
1598
1639
|
maxActivePerLane: RUNTIME_POSTGRES_POOL_CONNECTIONS_PER_LANE,
|
|
1599
|
-
maxQueuedPerLane:
|
|
1600
|
-
|
|
1640
|
+
maxQueuedPerLane: {
|
|
1641
|
+
receipts: RUNTIME_POSTGRES_RECEIPT_ADMISSION_MAX_QUEUED,
|
|
1642
|
+
sheets: RUNTIME_POSTGRES_SHEET_ADMISSION_MAX_QUEUED,
|
|
1643
|
+
},
|
|
1644
|
+
// Sheet capacity is backpressure, not a sheet-persistence failure. The
|
|
1645
|
+
// surrounding runtime request has its own bounded deadline and the queue
|
|
1646
|
+
// stays FIFO; no healthy request is discarded merely because two writes
|
|
1647
|
+
// are active for ten seconds. Receipt admission retains its short timeout:
|
|
1648
|
+
// a receipt can follow the ordinary retry/recovery path without holding a
|
|
1649
|
+
// row's data-plane transaction open.
|
|
1650
|
+
acquireTimeoutMs: {
|
|
1651
|
+
receipts: RUNTIME_POSTGRES_RECEIPT_ADMISSION_TIMEOUT_MS,
|
|
1652
|
+
sheets: null,
|
|
1653
|
+
},
|
|
1601
1654
|
});
|
|
1602
1655
|
runtimePostgresAdmissions.set(postgresUrl, admission);
|
|
1603
1656
|
return admission;
|
|
@@ -1654,7 +1707,11 @@ async function resetRuntimePostgresPool(postgresUrl: string): Promise<void> {
|
|
|
1654
1707
|
async function withRuntimePostgres<T>(
|
|
1655
1708
|
session: RuntimePostgresSession,
|
|
1656
1709
|
fn: (client: RuntimePoolClient) => Promise<T>,
|
|
1657
|
-
options: {
|
|
1710
|
+
options: {
|
|
1711
|
+
cachePool?: boolean;
|
|
1712
|
+
maxConnectAttempts?: number;
|
|
1713
|
+
signal?: AbortSignal | null;
|
|
1714
|
+
} = {},
|
|
1658
1715
|
): Promise<T> {
|
|
1659
1716
|
let client: RuntimePoolClient | null = null;
|
|
1660
1717
|
let requestLocalPool: RuntimePool | null = null;
|
|
@@ -1672,7 +1729,13 @@ async function withRuntimePostgres<T>(
|
|
|
1672
1729
|
if (!cachePool) {
|
|
1673
1730
|
requestLocalPool = pool;
|
|
1674
1731
|
}
|
|
1675
|
-
client = await connectRuntimePostgresPool(
|
|
1732
|
+
client = await connectRuntimePostgresPool(
|
|
1733
|
+
pool,
|
|
1734
|
+
admission,
|
|
1735
|
+
lane,
|
|
1736
|
+
session.postgresUrl,
|
|
1737
|
+
options.signal,
|
|
1738
|
+
);
|
|
1676
1739
|
if (session.executionRole) {
|
|
1677
1740
|
await client.query(
|
|
1678
1741
|
`SET ROLE ${quoteIdentifier(session.executionRole)}`,
|
|
@@ -1684,7 +1747,13 @@ async function withRuntimePostgres<T>(
|
|
|
1684
1747
|
client.release();
|
|
1685
1748
|
client = null;
|
|
1686
1749
|
}
|
|
1687
|
-
|
|
1750
|
+
// Queue overflow and caller cancellation happen before pool.connect().
|
|
1751
|
+
// They describe admission state, not a broken shared pool. Ending that
|
|
1752
|
+
// pool would invalidate the active clients and FIFO waiters that caused
|
|
1753
|
+
// the backpressure in the first place.
|
|
1754
|
+
const preserveSharedPool =
|
|
1755
|
+
cachePool && isRuntimePostgresAdmissionError(error);
|
|
1756
|
+
if (cachePool && !preserveSharedPool) {
|
|
1688
1757
|
const poolKey = runtimePostgresPoolKey(session.postgresUrl, lane);
|
|
1689
1758
|
const pool = postgresPools.get(poolKey);
|
|
1690
1759
|
postgresPools.delete(poolKey);
|
|
@@ -1923,7 +1992,10 @@ async function withRuntimeSheetQueryClient<T>(
|
|
|
1923
1992
|
throw error;
|
|
1924
1993
|
}
|
|
1925
1994
|
},
|
|
1926
|
-
{
|
|
1995
|
+
{
|
|
1996
|
+
cachePool: !context.disablePostgresPoolCache,
|
|
1997
|
+
signal: context.abortSignal,
|
|
1998
|
+
},
|
|
1927
1999
|
);
|
|
1928
2000
|
},
|
|
1929
2001
|
);
|
|
@@ -1968,13 +2040,16 @@ function runDatasetCatalogTable(session: RuntimePostgresSession): string {
|
|
|
1968
2040
|
async function registerRuntimeDataset(
|
|
1969
2041
|
session: RuntimePostgresSession,
|
|
1970
2042
|
input: { playName: string; tableNamespace: string; runId: string },
|
|
2043
|
+
signal?: AbortSignal | null,
|
|
1971
2044
|
): Promise<void> {
|
|
1972
2045
|
const tableNamespace = normalizeTableNamespace(input.tableNamespace);
|
|
1973
2046
|
const playName = normalizePlayNameForSheet(input.playName);
|
|
1974
2047
|
const datasetId = createRuntimeDatasetId(playName, tableNamespace);
|
|
1975
|
-
await withRuntimePostgres(
|
|
1976
|
-
|
|
1977
|
-
|
|
2048
|
+
await withRuntimePostgres(
|
|
2049
|
+
session,
|
|
2050
|
+
async (client) => {
|
|
2051
|
+
await client.query(
|
|
2052
|
+
`INSERT INTO ${runDatasetCatalogTable(session)} (
|
|
1978
2053
|
run_id, dataset_id, play_name, table_namespace, public_path
|
|
1979
2054
|
) VALUES ($1, $2, $3, $4, $5)
|
|
1980
2055
|
ON CONFLICT (run_id, dataset_id) DO UPDATE SET
|
|
@@ -1982,15 +2057,17 @@ async function registerRuntimeDataset(
|
|
|
1982
2057
|
table_namespace = EXCLUDED.table_namespace,
|
|
1983
2058
|
public_path = EXCLUDED.public_path,
|
|
1984
2059
|
updated_at = now()`,
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
2060
|
+
[
|
|
2061
|
+
input.runId,
|
|
2062
|
+
datasetId,
|
|
2063
|
+
playName,
|
|
2064
|
+
tableNamespace,
|
|
2065
|
+
`datasets.${tableNamespace}`,
|
|
2066
|
+
],
|
|
2067
|
+
);
|
|
2068
|
+
},
|
|
2069
|
+
{ signal },
|
|
2070
|
+
);
|
|
1994
2071
|
}
|
|
1995
2072
|
|
|
1996
2073
|
function workReceiptTable(session: RuntimePostgresSession): string {
|
|
@@ -2312,7 +2389,10 @@ async function withRuntimeWorkReceiptClient<T>(
|
|
|
2312
2389
|
: await withRuntimePostgres(
|
|
2313
2390
|
session,
|
|
2314
2391
|
(client) => runWithSelfHeal(client),
|
|
2315
|
-
{
|
|
2392
|
+
{
|
|
2393
|
+
cachePool: !context.disablePostgresPoolCache,
|
|
2394
|
+
signal: context.abortSignal,
|
|
2395
|
+
},
|
|
2316
2396
|
);
|
|
2317
2397
|
} catch (error) {
|
|
2318
2398
|
if (
|
|
@@ -5770,11 +5850,15 @@ export async function startRuntimeSheetDataset(
|
|
|
5770
5850
|
// Register the exact run-scoped Dataset Handle before any row can become
|
|
5771
5851
|
// durable. A crash after a later sheet write can therefore never leave
|
|
5772
5852
|
// customer data discoverable only through static pipeline inference.
|
|
5773
|
-
await registerRuntimeDataset(
|
|
5774
|
-
|
|
5775
|
-
|
|
5776
|
-
|
|
5777
|
-
|
|
5853
|
+
await registerRuntimeDataset(
|
|
5854
|
+
session,
|
|
5855
|
+
{
|
|
5856
|
+
playName,
|
|
5857
|
+
tableNamespace: input.tableNamespace,
|
|
5858
|
+
runId: input.runId,
|
|
5859
|
+
},
|
|
5860
|
+
context.abortSignal,
|
|
5861
|
+
);
|
|
5778
5862
|
const normalizeStartedAt = Date.now();
|
|
5779
5863
|
const uniqueRows = new Map<
|
|
5780
5864
|
string,
|
|
@@ -5840,9 +5924,18 @@ export async function startRuntimeSheetDataset(
|
|
|
5840
5924
|
.map((column) => `payload -> ${quoteLiteral(column)}`)
|
|
5841
5925
|
.join(', ')}`
|
|
5842
5926
|
: '';
|
|
5927
|
+
const outputPhysicalColumns = outputPhysicalSheetColumnProjections(
|
|
5928
|
+
input.sheetContract,
|
|
5929
|
+
);
|
|
5930
|
+
const outputPhysicalColumnNames = new Set(
|
|
5931
|
+
outputPhysicalColumns.map((column) => column.sqlName),
|
|
5932
|
+
);
|
|
5933
|
+
const physicalRefreshColumns = physicalColumns.filter(
|
|
5934
|
+
(column) => !outputPhysicalColumnNames.has(column),
|
|
5935
|
+
);
|
|
5843
5936
|
const physicalRefreshSetSql =
|
|
5844
|
-
|
|
5845
|
-
? `, ${
|
|
5937
|
+
physicalRefreshColumns.length > 0
|
|
5938
|
+
? `, ${physicalRefreshColumns
|
|
5846
5939
|
.map(
|
|
5847
5940
|
(column) =>
|
|
5848
5941
|
`${quoteIdentifier(column)} = input_rows.payload -> ${quoteLiteral(column)}`,
|
|
@@ -5858,9 +5951,6 @@ export async function startRuntimeSheetDataset(
|
|
|
5858
5951
|
)
|
|
5859
5952
|
.join(', ')}`
|
|
5860
5953
|
: '';
|
|
5861
|
-
const outputPhysicalColumns = outputPhysicalSheetColumnProjections(
|
|
5862
|
-
input.sheetContract,
|
|
5863
|
-
);
|
|
5864
5954
|
const normalizedPlayName = normalizePlayNameForSheet(playName);
|
|
5865
5955
|
const normalizedTableNamespace = normalizeTableNamespace(
|
|
5866
5956
|
input.tableNamespace,
|
|
@@ -7,6 +7,7 @@ export type RuntimePostgresLaneTelemetry = {
|
|
|
7
7
|
waited: number;
|
|
8
8
|
timedOut: number;
|
|
9
9
|
rejected: number;
|
|
10
|
+
cancelled: number;
|
|
10
11
|
totalWaitMs: number;
|
|
11
12
|
maxWaitMs: number;
|
|
12
13
|
};
|
|
@@ -21,9 +22,13 @@ export class RuntimePostgresAdmissionTimeoutError extends Error {
|
|
|
21
22
|
readonly lane: RuntimePostgresLane,
|
|
22
23
|
readonly queueDepth: number,
|
|
23
24
|
readonly timeoutMs: number,
|
|
25
|
+
readonly active: number,
|
|
26
|
+
readonly queuedBeforeTimeout: number,
|
|
27
|
+
readonly waitedMs: number,
|
|
24
28
|
) {
|
|
25
29
|
super(
|
|
26
|
-
`Runtime Postgres ${lane} admission timed out after ${timeoutMs}ms
|
|
30
|
+
`Runtime Postgres ${lane} admission timed out after ${timeoutMs}ms ` +
|
|
31
|
+
`(${active} active, ${queuedBeforeTimeout} queued before timeout; ${queueDepth} queued after removal).`,
|
|
27
32
|
);
|
|
28
33
|
this.name = 'RuntimePostgresAdmissionTimeoutError';
|
|
29
34
|
}
|
|
@@ -41,11 +46,44 @@ export class RuntimePostgresAdmissionQueueFullError extends Error {
|
|
|
41
46
|
}
|
|
42
47
|
}
|
|
43
48
|
|
|
49
|
+
export class RuntimePostgresAdmissionCancelledError extends Error {
|
|
50
|
+
constructor(readonly lane: RuntimePostgresLane) {
|
|
51
|
+
super(`Runtime Postgres ${lane} admission was cancelled.`);
|
|
52
|
+
this.name = 'RuntimePostgresAdmissionCancelledError';
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function isRuntimePostgresAdmissionCapacityError(
|
|
57
|
+
error: unknown,
|
|
58
|
+
): error is
|
|
59
|
+
| RuntimePostgresAdmissionTimeoutError
|
|
60
|
+
| RuntimePostgresAdmissionQueueFullError {
|
|
61
|
+
return (
|
|
62
|
+
error instanceof RuntimePostgresAdmissionTimeoutError ||
|
|
63
|
+
error instanceof RuntimePostgresAdmissionQueueFullError
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function isRuntimePostgresAdmissionError(
|
|
68
|
+
error: unknown,
|
|
69
|
+
): error is
|
|
70
|
+
| RuntimePostgresAdmissionTimeoutError
|
|
71
|
+
| RuntimePostgresAdmissionQueueFullError
|
|
72
|
+
| RuntimePostgresAdmissionCancelledError {
|
|
73
|
+
return (
|
|
74
|
+
isRuntimePostgresAdmissionCapacityError(error) ||
|
|
75
|
+
error instanceof RuntimePostgresAdmissionCancelledError
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
|
|
44
79
|
type Waiter = {
|
|
45
80
|
enqueuedAt: number;
|
|
46
|
-
timeout: ReturnType<typeof setTimeout
|
|
81
|
+
timeout: ReturnType<typeof setTimeout> | null;
|
|
47
82
|
resolve: (release: () => void) => void;
|
|
48
83
|
reject: (error: Error) => void;
|
|
84
|
+
signal: AbortSignal | null;
|
|
85
|
+
onAbort: (() => void) | null;
|
|
86
|
+
settled: boolean;
|
|
49
87
|
};
|
|
50
88
|
|
|
51
89
|
type LaneState = RuntimePostgresLaneTelemetry & { waiters: Waiter[] };
|
|
@@ -58,6 +96,7 @@ function createLaneState(): LaneState {
|
|
|
58
96
|
waited: 0,
|
|
59
97
|
timedOut: 0,
|
|
60
98
|
rejected: 0,
|
|
99
|
+
cancelled: 0,
|
|
61
100
|
totalWaitMs: 0,
|
|
62
101
|
maxWaitMs: 0,
|
|
63
102
|
waiters: [],
|
|
@@ -66,8 +105,8 @@ function createLaneState(): LaneState {
|
|
|
66
105
|
|
|
67
106
|
export class RuntimePostgresAdmission {
|
|
68
107
|
readonly #maxActivePerLane: number;
|
|
69
|
-
readonly #maxQueuedPerLane: number
|
|
70
|
-
readonly #acquireTimeoutMs: number
|
|
108
|
+
readonly #maxQueuedPerLane: Record<RuntimePostgresLane, number>;
|
|
109
|
+
readonly #acquireTimeoutMs: Record<RuntimePostgresLane, number | null>;
|
|
71
110
|
readonly #lanes: Record<RuntimePostgresLane, LaneState> = {
|
|
72
111
|
receipts: createLaneState(),
|
|
73
112
|
sheets: createLaneState(),
|
|
@@ -75,22 +114,59 @@ export class RuntimePostgresAdmission {
|
|
|
75
114
|
|
|
76
115
|
constructor(input: {
|
|
77
116
|
maxActivePerLane: number;
|
|
78
|
-
maxQueuedPerLane: number
|
|
79
|
-
acquireTimeoutMs:
|
|
117
|
+
maxQueuedPerLane: number | Record<RuntimePostgresLane, number>;
|
|
118
|
+
acquireTimeoutMs:
|
|
119
|
+
| number
|
|
120
|
+
| null
|
|
121
|
+
| Record<RuntimePostgresLane, number | null>;
|
|
80
122
|
}) {
|
|
81
123
|
this.#maxActivePerLane = Math.max(1, Math.floor(input.maxActivePerLane));
|
|
82
|
-
|
|
83
|
-
|
|
124
|
+
const normalizeQueueLimit = (value: number): number =>
|
|
125
|
+
Math.max(0, Math.floor(value));
|
|
126
|
+
this.#maxQueuedPerLane =
|
|
127
|
+
typeof input.maxQueuedPerLane === 'object'
|
|
128
|
+
? {
|
|
129
|
+
receipts: normalizeQueueLimit(input.maxQueuedPerLane.receipts),
|
|
130
|
+
sheets: normalizeQueueLimit(input.maxQueuedPerLane.sheets),
|
|
131
|
+
}
|
|
132
|
+
: {
|
|
133
|
+
receipts: normalizeQueueLimit(input.maxQueuedPerLane),
|
|
134
|
+
sheets: normalizeQueueLimit(input.maxQueuedPerLane),
|
|
135
|
+
};
|
|
136
|
+
const normalizeTimeout = (value: number | null): number | null =>
|
|
137
|
+
value === null ? null : Math.max(1, Math.floor(value));
|
|
138
|
+
const acquireTimeoutMs = input.acquireTimeoutMs;
|
|
139
|
+
this.#acquireTimeoutMs =
|
|
140
|
+
acquireTimeoutMs !== null && typeof acquireTimeoutMs === 'object'
|
|
141
|
+
? {
|
|
142
|
+
receipts: normalizeTimeout(acquireTimeoutMs.receipts),
|
|
143
|
+
sheets: normalizeTimeout(acquireTimeoutMs.sheets),
|
|
144
|
+
}
|
|
145
|
+
: {
|
|
146
|
+
receipts: normalizeTimeout(
|
|
147
|
+
typeof acquireTimeoutMs === 'number' ? acquireTimeoutMs : null,
|
|
148
|
+
),
|
|
149
|
+
sheets: normalizeTimeout(
|
|
150
|
+
typeof acquireTimeoutMs === 'number' ? acquireTimeoutMs : null,
|
|
151
|
+
),
|
|
152
|
+
};
|
|
84
153
|
}
|
|
85
154
|
|
|
86
|
-
async acquire(
|
|
155
|
+
async acquire(
|
|
156
|
+
lane: RuntimePostgresLane,
|
|
157
|
+
options: { signal?: AbortSignal } = {},
|
|
158
|
+
): Promise<() => void> {
|
|
87
159
|
const state = this.#lanes[lane];
|
|
160
|
+
if (options.signal?.aborted) {
|
|
161
|
+
state.cancelled += 1;
|
|
162
|
+
throw new RuntimePostgresAdmissionCancelledError(lane);
|
|
163
|
+
}
|
|
88
164
|
if (state.active < this.#maxActivePerLane && state.waiters.length === 0) {
|
|
89
165
|
state.active += 1;
|
|
90
166
|
state.acquired += 1;
|
|
91
167
|
return this.#releaseFor(lane);
|
|
92
168
|
}
|
|
93
|
-
if (state.waiters.length >= this.#maxQueuedPerLane) {
|
|
169
|
+
if (state.waiters.length >= this.#maxQueuedPerLane[lane]) {
|
|
94
170
|
state.rejected += 1;
|
|
95
171
|
throw new RuntimePostgresAdmissionQueueFullError(
|
|
96
172
|
lane,
|
|
@@ -98,28 +174,56 @@ export class RuntimePostgresAdmission {
|
|
|
98
174
|
);
|
|
99
175
|
}
|
|
100
176
|
state.waited += 1;
|
|
177
|
+
const acquireTimeoutMs = this.#acquireTimeoutMs[lane];
|
|
101
178
|
return await new Promise<() => void>((resolve, reject) => {
|
|
102
179
|
const waiter: Waiter = {
|
|
103
180
|
enqueuedAt: Date.now(),
|
|
104
|
-
timeout:
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
181
|
+
timeout: null,
|
|
182
|
+
resolve,
|
|
183
|
+
reject,
|
|
184
|
+
signal: options.signal ?? null,
|
|
185
|
+
onAbort: null,
|
|
186
|
+
settled: false,
|
|
187
|
+
};
|
|
188
|
+
const removeWaiter = (): boolean => {
|
|
189
|
+
if (waiter.settled) return false;
|
|
190
|
+
const index = state.waiters.indexOf(waiter);
|
|
191
|
+
if (index < 0) return false;
|
|
192
|
+
waiter.settled = true;
|
|
193
|
+
state.waiters.splice(index, 1);
|
|
194
|
+
state.queued = state.waiters.length;
|
|
195
|
+
if (waiter.timeout) clearTimeout(waiter.timeout);
|
|
196
|
+
if (waiter.signal && waiter.onAbort) {
|
|
197
|
+
waiter.signal.removeEventListener('abort', waiter.onAbort);
|
|
198
|
+
}
|
|
199
|
+
return true;
|
|
200
|
+
};
|
|
201
|
+
waiter.onAbort = () => {
|
|
202
|
+
if (!removeWaiter()) return;
|
|
203
|
+
state.cancelled += 1;
|
|
204
|
+
reject(new RuntimePostgresAdmissionCancelledError(lane));
|
|
205
|
+
};
|
|
206
|
+
waiter.signal?.addEventListener('abort', waiter.onAbort, { once: true });
|
|
207
|
+
if (acquireTimeoutMs !== null) {
|
|
208
|
+
waiter.timeout = setTimeout(() => {
|
|
209
|
+
const queuedBeforeTimeout = state.waiters.length;
|
|
210
|
+
if (!removeWaiter()) return;
|
|
109
211
|
state.timedOut += 1;
|
|
110
212
|
reject(
|
|
111
213
|
new RuntimePostgresAdmissionTimeoutError(
|
|
112
214
|
lane,
|
|
113
215
|
state.waiters.length,
|
|
114
|
-
|
|
216
|
+
acquireTimeoutMs,
|
|
217
|
+
state.active,
|
|
218
|
+
queuedBeforeTimeout,
|
|
219
|
+
Math.max(0, Date.now() - waiter.enqueuedAt),
|
|
115
220
|
),
|
|
116
221
|
);
|
|
117
|
-
},
|
|
118
|
-
|
|
119
|
-
reject,
|
|
120
|
-
};
|
|
222
|
+
}, acquireTimeoutMs);
|
|
223
|
+
}
|
|
121
224
|
state.waiters.push(waiter);
|
|
122
225
|
state.queued = state.waiters.length;
|
|
226
|
+
if (waiter.signal?.aborted) waiter.onAbort();
|
|
123
227
|
});
|
|
124
228
|
}
|
|
125
229
|
|
|
@@ -131,6 +235,7 @@ export class RuntimePostgresAdmission {
|
|
|
131
235
|
waited: state.waited,
|
|
132
236
|
timedOut: state.timedOut,
|
|
133
237
|
rejected: state.rejected,
|
|
238
|
+
cancelled: state.cancelled,
|
|
134
239
|
totalWaitMs: state.totalWaitMs,
|
|
135
240
|
maxWaitMs: state.maxWaitMs,
|
|
136
241
|
});
|
|
@@ -150,7 +255,11 @@ export class RuntimePostgresAdmission {
|
|
|
150
255
|
const waiter = state.waiters.shift();
|
|
151
256
|
state.queued = state.waiters.length;
|
|
152
257
|
if (!waiter) return;
|
|
153
|
-
|
|
258
|
+
waiter.settled = true;
|
|
259
|
+
if (waiter.timeout) clearTimeout(waiter.timeout);
|
|
260
|
+
if (waiter.signal && waiter.onAbort) {
|
|
261
|
+
waiter.signal.removeEventListener('abort', waiter.onAbort);
|
|
262
|
+
}
|
|
154
263
|
const waitMs = Math.max(0, Date.now() - waiter.enqueuedAt);
|
|
155
264
|
state.totalWaitMs += waitMs;
|
|
156
265
|
state.maxWaitMs = Math.max(state.maxWaitMs, waitMs);
|