deepline 0.3.288 → 0.3.290
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +30 -10
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +108 -12
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/launch-identity.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/log-provenance.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/worker-terminal-projection.ts +6 -5
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-event-catalog.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +31 -3
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +23 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +30 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +38 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +2 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +22 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool-options.ts +145 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +1 -5
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -139
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +19 -5
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +37 -15
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +7 -5
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +961 -281
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +55 -25
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-dispatch-authority.ts +97 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +33 -14
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +452 -43
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +10 -1
- package/dist/bundling-sources/shared_libs/play-runtime/test-admission-batching.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/zerobounce-batching.ts +75 -28
- package/dist/cli/index.js +24 -11
- package/dist/cli/index.mjs +24 -11
- package/dist/index.js +3 -1
- package/dist/index.mjs +3 -1
- package/dist/install-integrity.json +3 -0
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
import {
|
|
2
|
+
parseRunLedgerDispatchLaunch,
|
|
3
|
+
runLedgerDispatchLaunchSql,
|
|
4
|
+
type RunLedgerDispatchLaunch,
|
|
5
|
+
} from './run-ledger-dispatch-authority';
|
|
6
|
+
import { normalizeRunLogIds } from '../run-log-buffer';
|
|
1
7
|
import {
|
|
2
8
|
retainedPlayPipelineFields,
|
|
3
9
|
hydrateRetainedPlayPipelines,
|
|
@@ -28,6 +34,8 @@ import type {
|
|
|
28
34
|
import { PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA } from '../runtime-scheduler-topology';
|
|
29
35
|
import {
|
|
30
36
|
RUNTIME_EVENT_CANONICAL_RETENTION_INDEX_READY_MARKER,
|
|
37
|
+
RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD,
|
|
38
|
+
RUNTIME_EVENT_SEARCH_ONLY_FIELD,
|
|
31
39
|
RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER,
|
|
32
40
|
} from './runtime-event-search';
|
|
33
41
|
import type { PlayRunnerRuntimeResource } from '../runner-backends/types';
|
|
@@ -105,7 +113,6 @@ import {
|
|
|
105
113
|
RUN_SCHEDULER_OUTBOX_START_EVENT_TYPES,
|
|
106
114
|
RUN_SCHEDULER_OUTBOX_EVENT_TYPE,
|
|
107
115
|
RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL,
|
|
108
|
-
RUN_SCHEDULER_OUTBOX_PREFIX_STOP_EVENT_TYPES,
|
|
109
116
|
RUN_SCHEDULER_OUTBOX_TERMINAL_EVENT_TYPES,
|
|
110
117
|
} from '../run-event-catalog';
|
|
111
118
|
import { stableStringify } from '../../plays/row-identity';
|
|
@@ -336,7 +343,10 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
|
|
|
336
343
|
// recovery, which scans only active runs for each release lane.
|
|
337
344
|
// v57 refreshes log-position lookups after the operator concurrently prepares
|
|
338
345
|
// channel-specific indexes; outgoing readers retain the existing GIN index.
|
|
339
|
-
|
|
346
|
+
// v58 installs deferred outbox readers while retaining synchronous writers
|
|
347
|
+
// until the separately operated, reader-first cutover.
|
|
348
|
+
// v59 adds origin identity bulk projection and independent durable search progress.
|
|
349
|
+
export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 59;
|
|
340
350
|
const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
|
|
341
351
|
'scheduler_schema_migrations';
|
|
342
352
|
export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
|
|
@@ -1928,6 +1938,24 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
|
1928
1938
|
AND index_row.indrelid = to_regclass($3)
|
|
1929
1939
|
AND index_row.indisready AND index_row.indisvalid
|
|
1930
1940
|
)
|
|
1941
|
+
AND EXISTS (
|
|
1942
|
+
SELECT 1 FROM pg_index AS index_row
|
|
1943
|
+
JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
|
|
1944
|
+
JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
|
|
1945
|
+
WHERE index_namespace.nspname = $1
|
|
1946
|
+
AND index_class.relname = 'runtime_scheduler_outbox_lifecycle_pending_order_idx'
|
|
1947
|
+
AND index_row.indrelid = to_regclass($3)
|
|
1948
|
+
AND index_row.indisready AND index_row.indisvalid
|
|
1949
|
+
)
|
|
1950
|
+
AND EXISTS (
|
|
1951
|
+
SELECT 1 FROM pg_index AS index_row
|
|
1952
|
+
JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
|
|
1953
|
+
JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
|
|
1954
|
+
WHERE index_namespace.nspname = $1
|
|
1955
|
+
AND index_class.relname = 'runtime_scheduler_outbox_search_pending_order_idx'
|
|
1956
|
+
AND index_row.indrelid = to_regclass($3)
|
|
1957
|
+
AND index_row.indisready AND index_row.indisvalid
|
|
1958
|
+
)
|
|
1931
1959
|
AND EXISTS (
|
|
1932
1960
|
SELECT 1 FROM pg_index AS index_row
|
|
1933
1961
|
JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
|
|
@@ -2461,6 +2489,7 @@ export async function ensurePostgresSchedulerLaunchAdmissionFunctions(
|
|
|
2461
2489
|
jsonb_set(
|
|
2462
2490
|
launch_json - ARRAY[
|
|
2463
2491
|
'launchIdentityDigest',
|
|
2492
|
+
'runtimeLogIdentity',
|
|
2464
2493
|
'executorToken',
|
|
2465
2494
|
'inputFileId',
|
|
2466
2495
|
'preloadedDbSessionUnwrapKey',
|
|
@@ -3944,6 +3973,26 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
3944
3973
|
)
|
|
3945
3974
|
WHERE delivered_at IS NULL
|
|
3946
3975
|
`);
|
|
3976
|
+
await client.query(`
|
|
3977
|
+
CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_lifecycle_pending_order_idx
|
|
3978
|
+
ON ${tableName('outbox', options)} (
|
|
3979
|
+
run_id,
|
|
3980
|
+
(CASE event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END),
|
|
3981
|
+
created_at,
|
|
3982
|
+
event_id
|
|
3983
|
+
)
|
|
3984
|
+
WHERE delivered_at IS NULL AND ${searchLaneSql('outbox')} = false
|
|
3985
|
+
`);
|
|
3986
|
+
await client.query(`
|
|
3987
|
+
CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_search_pending_order_idx
|
|
3988
|
+
ON ${tableName('outbox', options)} (
|
|
3989
|
+
run_id,
|
|
3990
|
+
(CASE event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END),
|
|
3991
|
+
created_at,
|
|
3992
|
+
event_id
|
|
3993
|
+
)
|
|
3994
|
+
WHERE delivered_at IS NULL AND ${searchLaneSql('outbox')} = true
|
|
3995
|
+
`);
|
|
3947
3996
|
await client.query(`
|
|
3948
3997
|
CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_lease_health_idx
|
|
3949
3998
|
ON ${tableName('outbox', options)}
|
|
@@ -4094,6 +4143,22 @@ const POSTGRES_SCHEDULER_RUNTIME_ADMIN_INDEXES = [
|
|
|
4094
4143
|
created_at, event_id)
|
|
4095
4144
|
WHERE delivered_at IS NULL`,
|
|
4096
4145
|
},
|
|
4146
|
+
{
|
|
4147
|
+
name: 'runtime_scheduler_outbox_lifecycle_pending_order_idx',
|
|
4148
|
+
table: 'outbox',
|
|
4149
|
+
definition: `(run_id,
|
|
4150
|
+
(CASE event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END),
|
|
4151
|
+
created_at, event_id)
|
|
4152
|
+
WHERE delivered_at IS NULL AND ${searchLaneSql('outbox')} = false`,
|
|
4153
|
+
},
|
|
4154
|
+
{
|
|
4155
|
+
name: 'runtime_scheduler_outbox_search_pending_order_idx',
|
|
4156
|
+
table: 'outbox',
|
|
4157
|
+
definition: `(run_id,
|
|
4158
|
+
(CASE event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END),
|
|
4159
|
+
created_at, event_id)
|
|
4160
|
+
WHERE delivered_at IS NULL AND ${searchLaneSql('outbox')} = true`,
|
|
4161
|
+
},
|
|
4097
4162
|
{
|
|
4098
4163
|
name: 'runtime_scheduler_outbox_lease_health_idx',
|
|
4099
4164
|
table: 'outbox',
|
|
@@ -10318,8 +10383,8 @@ export async function recordPostgresSchedulerRunnerTerminal(
|
|
|
10318
10383
|
const recorded = await client.query<{ run_id: string }>(
|
|
10319
10384
|
`
|
|
10320
10385
|
WITH accepted AS MATERIALIZED (
|
|
10321
|
-
--
|
|
10322
|
-
--
|
|
10386
|
+
-- Record the authoritative terminal boundary before accepting the
|
|
10387
|
+
-- attempt payload; deferred log indexing owns no Run row lock.
|
|
10323
10388
|
SELECT clock_timestamp() AS accepted_at
|
|
10324
10389
|
FROM ${tableName('runs', options)}
|
|
10325
10390
|
WHERE run_id = $1
|
|
@@ -10758,6 +10823,7 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
|
|
|
10758
10823
|
'producerAttempt', $2::integer,
|
|
10759
10824
|
'terminalLogLines', $7::jsonb,
|
|
10760
10825
|
'terminalLogContexts', $17::jsonb,
|
|
10826
|
+
'terminalLogIds', $19::jsonb->'terminalLogIds',
|
|
10761
10827
|
'terminalProgressEvents', coalesce((
|
|
10762
10828
|
SELECT accepted.runner_log_payload_json->'terminalProgressEvents'
|
|
10763
10829
|
FROM completed_attempt AS accepted
|
|
@@ -13134,6 +13200,7 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13134
13200
|
runtimeTiming?: PlayRunnerRuntimeTiming | null;
|
|
13135
13201
|
terminalLogLines?: readonly string[] | null;
|
|
13136
13202
|
terminalLogContexts?: readonly (PlayLogContext | null)[] | null;
|
|
13203
|
+
terminalLogIds?: readonly (string | null)[] | null;
|
|
13137
13204
|
terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
|
|
13138
13205
|
},
|
|
13139
13206
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
@@ -13151,10 +13218,15 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13151
13218
|
'Terminal log contexts must align with terminal log lines.',
|
|
13152
13219
|
);
|
|
13153
13220
|
}
|
|
13221
|
+
const terminalLogIds = normalizeRunLogIds(
|
|
13222
|
+
input.terminalLogIds,
|
|
13223
|
+
input.terminalLogLines ?? [],
|
|
13224
|
+
);
|
|
13154
13225
|
const terminalLogEntries = (input.terminalLogLines ?? [])
|
|
13155
13226
|
.map((line, index) => ({
|
|
13156
13227
|
line: line.trim(),
|
|
13157
13228
|
context: input.terminalLogContexts?.[index] ?? null,
|
|
13229
|
+
entryId: terminalLogIds?.[index] ?? null,
|
|
13158
13230
|
}))
|
|
13159
13231
|
.filter((entry) => entry.line.length > 0)
|
|
13160
13232
|
.slice(-LOG_TAIL_LIMIT);
|
|
@@ -13340,6 +13412,7 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13340
13412
|
'producerAttempt', $2::integer,
|
|
13341
13413
|
'terminalLogLines', $8::jsonb,
|
|
13342
13414
|
'terminalLogContexts', $11::jsonb,
|
|
13415
|
+
'terminalLogIds', $13::jsonb,
|
|
13343
13416
|
'resultSummary', $10::jsonb,
|
|
13344
13417
|
'terminalWorkCounterSegment', $12::jsonb
|
|
13345
13418
|
),
|
|
@@ -13365,6 +13438,11 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13365
13438
|
stringifyPostgresJson(resultSummary),
|
|
13366
13439
|
JSON.stringify(terminalLogContexts),
|
|
13367
13440
|
stringifyPostgresJson(input.terminalWorkCounterSegment ?? null),
|
|
13441
|
+
stringifyPostgresJson(
|
|
13442
|
+
terminalLogIds
|
|
13443
|
+
? terminalLogEntries.map((entry) => entry.entryId)
|
|
13444
|
+
: null,
|
|
13445
|
+
),
|
|
13368
13446
|
],
|
|
13369
13447
|
);
|
|
13370
13448
|
if (completed.rows.length === 0) {
|
|
@@ -13383,6 +13461,7 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13383
13461
|
result?: unknown;
|
|
13384
13462
|
terminalLogLines?: readonly string[] | null;
|
|
13385
13463
|
terminalLogContexts?: readonly (PlayLogContext | null)[] | null;
|
|
13464
|
+
terminalLogIds?: readonly (string | null)[] | null;
|
|
13386
13465
|
/** Refuse to overwrite an authoritative detached runner terminal that won
|
|
13387
13466
|
* after claim. Resumable `suspended` receipts may be superseded by the
|
|
13388
13467
|
* hard ceiling in the same fenced statement. */
|
|
@@ -13429,10 +13508,15 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13429
13508
|
'Terminal log contexts must align with terminal log lines.',
|
|
13430
13509
|
);
|
|
13431
13510
|
}
|
|
13511
|
+
const terminalLogIds = normalizeRunLogIds(
|
|
13512
|
+
input.terminalLogIds,
|
|
13513
|
+
input.terminalLogLines ?? [],
|
|
13514
|
+
);
|
|
13432
13515
|
const terminalLogEntries = (input.terminalLogLines ?? [])
|
|
13433
13516
|
.map((line, index) => ({
|
|
13434
13517
|
line: line.trim(),
|
|
13435
13518
|
context: input.terminalLogContexts?.[index] ?? null,
|
|
13519
|
+
entryId: terminalLogIds?.[index] ?? null,
|
|
13436
13520
|
}))
|
|
13437
13521
|
.filter((entry) => entry.line.length > 0)
|
|
13438
13522
|
.slice(-LOG_TAIL_LIMIT);
|
|
@@ -13678,6 +13762,7 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13678
13762
|
'producerAttempt', $2::integer,
|
|
13679
13763
|
'terminalLogLines', $12::jsonb,
|
|
13680
13764
|
'terminalLogContexts', $14::jsonb,
|
|
13765
|
+
'terminalLogIds', $17::jsonb,
|
|
13681
13766
|
'resultSummary', $13::jsonb,
|
|
13682
13767
|
'terminalWorkCounterSegment', $15::jsonb
|
|
13683
13768
|
) || CASE
|
|
@@ -13711,6 +13796,11 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13711
13796
|
JSON.stringify(terminalLogContexts),
|
|
13712
13797
|
stringifyPostgresJson(terminalWorkCounterSegment ?? null),
|
|
13713
13798
|
stringifyPostgresJson(input.terminalProgressEvents ?? []),
|
|
13799
|
+
stringifyPostgresJson(
|
|
13800
|
+
terminalLogIds
|
|
13801
|
+
? terminalLogEntries.map((entry) => entry.entryId)
|
|
13802
|
+
: null,
|
|
13803
|
+
),
|
|
13714
13804
|
],
|
|
13715
13805
|
);
|
|
13716
13806
|
return failed.rows.length > 0;
|
|
@@ -14413,6 +14503,7 @@ function runnerTerminalLogPayload(input: {
|
|
|
14413
14503
|
producerAttempt: input.attempt,
|
|
14414
14504
|
terminalLogLines: projection.lines,
|
|
14415
14505
|
terminalLogContexts: projection.lineContexts,
|
|
14506
|
+
...(projection.lineIds ? { terminalLogIds: projection.lineIds } : {}),
|
|
14416
14507
|
terminalProgressEvents: input.terminalProgressEvents ?? [],
|
|
14417
14508
|
// Older runner envelopes conveyed these diagnostics only in tagged log
|
|
14418
14509
|
// text. Preserve their small structured facts before removing that text.
|
|
@@ -14433,7 +14524,11 @@ function runnerTerminalLogPayload(input: {
|
|
|
14433
14524
|
function terminalLogProjectionForOutbox(
|
|
14434
14525
|
runnerResult: unknown,
|
|
14435
14526
|
mode: PostgresSchedulerTerminalLogOutbox | undefined,
|
|
14436
|
-
): {
|
|
14527
|
+
): {
|
|
14528
|
+
lines: string[];
|
|
14529
|
+
lineContexts: Array<PlayLogContext | null>;
|
|
14530
|
+
lineIds?: Array<string | null>;
|
|
14531
|
+
} {
|
|
14437
14532
|
if (
|
|
14438
14533
|
mode !== 'ctx_log_tail' ||
|
|
14439
14534
|
!runnerResult ||
|
|
@@ -14444,6 +14539,7 @@ function terminalLogProjectionForOutbox(
|
|
|
14444
14539
|
const result = runnerResult as {
|
|
14445
14540
|
logs?: unknown;
|
|
14446
14541
|
logContexts?: unknown;
|
|
14542
|
+
logIds?: unknown;
|
|
14447
14543
|
};
|
|
14448
14544
|
const logs = result.logs;
|
|
14449
14545
|
if (!Array.isArray(logs)) {
|
|
@@ -14459,6 +14555,7 @@ function terminalLogProjectionForOutbox(
|
|
|
14459
14555
|
const logContexts = Array.isArray(result.logContexts)
|
|
14460
14556
|
? result.logContexts
|
|
14461
14557
|
: undefined;
|
|
14558
|
+
const logIds = normalizeRunLogIds(result.logIds, logs);
|
|
14462
14559
|
const entries = logs
|
|
14463
14560
|
.map((line, index) => {
|
|
14464
14561
|
if (typeof line !== 'string') return null;
|
|
@@ -14475,16 +14572,23 @@ function terminalLogProjectionForOutbox(
|
|
|
14475
14572
|
return {
|
|
14476
14573
|
line: line.trim(),
|
|
14477
14574
|
context: normalizePlayLogContext(logContexts?.[index]),
|
|
14575
|
+
entryId: logIds?.[index] ?? null,
|
|
14478
14576
|
};
|
|
14479
14577
|
})
|
|
14480
14578
|
.filter(
|
|
14481
|
-
(
|
|
14482
|
-
entry
|
|
14579
|
+
(
|
|
14580
|
+
entry,
|
|
14581
|
+
): entry is {
|
|
14582
|
+
line: string;
|
|
14583
|
+
context: PlayLogContext | null;
|
|
14584
|
+
entryId: string | null;
|
|
14585
|
+
} => entry !== null,
|
|
14483
14586
|
)
|
|
14484
14587
|
.slice(-LOG_TAIL_LIMIT);
|
|
14485
14588
|
return {
|
|
14486
14589
|
lines: entries.map((entry) => entry.line),
|
|
14487
14590
|
lineContexts: entries.map((entry) => entry.context),
|
|
14591
|
+
...(logIds ? { lineIds: entries.map((entry) => entry.entryId) } : {}),
|
|
14488
14592
|
};
|
|
14489
14593
|
}
|
|
14490
14594
|
|
|
@@ -14733,6 +14837,7 @@ export async function executePostgresSchedulerClaim(
|
|
|
14733
14837
|
runtimeTiming: extractRunnerRuntimeTiming(output),
|
|
14734
14838
|
terminalLogLines: terminalLogProjection.lines,
|
|
14735
14839
|
terminalLogContexts: terminalLogProjection.lineContexts,
|
|
14840
|
+
terminalLogIds: terminalLogProjection.lineIds,
|
|
14736
14841
|
terminalWorkCounterSegment: terminalWorkCounterSegmentFromRunnerResult(
|
|
14737
14842
|
output,
|
|
14738
14843
|
claim.attempt,
|
|
@@ -14794,6 +14899,7 @@ export async function executePostgresSchedulerClaim(
|
|
|
14794
14899
|
result: terminalFailureResult,
|
|
14795
14900
|
terminalLogLines: terminalLogProjection.lines,
|
|
14796
14901
|
terminalLogContexts: terminalLogProjection.lineContexts,
|
|
14902
|
+
terminalLogIds: terminalLogProjection.lineIds,
|
|
14797
14903
|
sandboxCrashDiagnostic: options.sandboxCrashDiagnostic,
|
|
14798
14904
|
...(options.preserveRunnerTerminal
|
|
14799
14905
|
? { preserveRunnerTerminal: true }
|
|
@@ -14987,6 +15093,9 @@ export function buildPostgresSchedulerRunnerContext(
|
|
|
14987
15093
|
const rootRunId = launch.rootRunId?.trim() || launch.runId;
|
|
14988
15094
|
return {
|
|
14989
15095
|
executorToken: launch.executorToken,
|
|
15096
|
+
...(launch.runtimeLogIdentity === 'origin'
|
|
15097
|
+
? { runtimeLogIdentity: 'origin' as const }
|
|
15098
|
+
: {}),
|
|
14990
15099
|
baseUrl: runtimeApiBaseUrlFromLaunch(launch),
|
|
14991
15100
|
executionGatewayBaseUrl,
|
|
14992
15101
|
receiptGatewayBaseUrl,
|
|
@@ -17388,6 +17497,12 @@ async function appendRunLedgerOutboxFact(
|
|
|
17388
17497
|
WITH admitted AS MATERIALIZED (
|
|
17389
17498
|
SELECT run_id FROM ${tableName('runs', options)}
|
|
17390
17499
|
WHERE run_id = $1 AND org_id = $2
|
|
17500
|
+
-- Evaluate ingress time only after the projector/lifecycle fence. An
|
|
17501
|
+
-- insert waiting behind a frozen receipt must become its successor,
|
|
17502
|
+
-- never a late predecessor that changes the attempted receipt.
|
|
17503
|
+
-- NO KEY UPDATE still permits legacy inserts' foreign-key KEY SHARE
|
|
17504
|
+
-- while they finish under the event-search advisory mutex.
|
|
17505
|
+
FOR NO KEY UPDATE
|
|
17391
17506
|
), recorded AS (
|
|
17392
17507
|
INSERT INTO ${tableName('outbox', options)}
|
|
17393
17508
|
(event_id, run_id, event_type, payload_json, created_at)
|
|
@@ -17413,23 +17528,43 @@ export { RunLedgerProjectionLeaseLostError } from '../run-ledger-projection-cont
|
|
|
17413
17528
|
/** Extend only our still-live lease before another bounded transport piece. */
|
|
17414
17529
|
export async function renewPostgresRunLedgerDelivery(
|
|
17415
17530
|
client: PostgresSchedulerQueryClient,
|
|
17416
|
-
input: {
|
|
17531
|
+
input: {
|
|
17532
|
+
eventIds: string[];
|
|
17533
|
+
leaseOwner: string;
|
|
17534
|
+
leaseSeconds: number;
|
|
17535
|
+
/** Freeze an ingress receipt before any HTTP attempt, including singletons. */
|
|
17536
|
+
freezeReceipt?: boolean;
|
|
17537
|
+
},
|
|
17417
17538
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
17418
17539
|
): Promise<void> {
|
|
17419
|
-
const
|
|
17420
|
-
|
|
17540
|
+
const renew = (freezeReceipt: boolean) =>
|
|
17541
|
+
client.query<{ event_id: string }>(
|
|
17542
|
+
`
|
|
17421
17543
|
UPDATE ${tableName('outbox', options)}
|
|
17422
17544
|
SET lease_expires_at = now() + make_interval(secs => $3::integer)
|
|
17545
|
+
${freezeReceipt ? ', delivery_batch_started_at = coalesce(delivery_batch_started_at, statement_timestamp())' : ''}
|
|
17423
17546
|
WHERE event_id = ANY($1::text[]) AND lease_owner = $2
|
|
17424
17547
|
AND delivered_at IS NULL AND failed_at IS NULL AND lease_expires_at > now()
|
|
17425
17548
|
RETURNING event_id
|
|
17426
17549
|
`,
|
|
17427
|
-
|
|
17428
|
-
|
|
17429
|
-
|
|
17430
|
-
|
|
17431
|
-
|
|
17432
|
-
|
|
17550
|
+
[
|
|
17551
|
+
input.eventIds,
|
|
17552
|
+
input.leaseOwner,
|
|
17553
|
+
Math.max(5, Math.min(300, input.leaseSeconds)),
|
|
17554
|
+
],
|
|
17555
|
+
);
|
|
17556
|
+
let result;
|
|
17557
|
+
try {
|
|
17558
|
+
result = await renew(
|
|
17559
|
+
input.freezeReceipt === true &&
|
|
17560
|
+
supportsOutboxDeliveryBatchMarker(options),
|
|
17561
|
+
);
|
|
17562
|
+
} catch (error) {
|
|
17563
|
+
if (!input.freezeReceipt || !isMissingOutboxDeliveryBatchMarker(error))
|
|
17564
|
+
throw error;
|
|
17565
|
+
markOutboxDeliveryBatchMarkerUnavailable(options);
|
|
17566
|
+
result = await renew(false);
|
|
17567
|
+
}
|
|
17433
17568
|
if (result.rows.length !== input.eventIds.length)
|
|
17434
17569
|
throw new RunLedgerProjectionLeaseLostError('delivery_lease_not_current');
|
|
17435
17570
|
}
|
|
@@ -17457,6 +17592,7 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
17457
17592
|
const result = await client.query<{
|
|
17458
17593
|
delivery_key: string;
|
|
17459
17594
|
event_type: string;
|
|
17595
|
+
delivery_batch_started_at: string | null;
|
|
17460
17596
|
lease_owner: string | null;
|
|
17461
17597
|
lease_live: boolean;
|
|
17462
17598
|
due: boolean;
|
|
@@ -17467,6 +17603,8 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
17467
17603
|
`
|
|
17468
17604
|
SELECT outbox.delivery_key,
|
|
17469
17605
|
outbox.event_type,
|
|
17606
|
+
-- Row JSON keeps this optional on historical scheduler schemas.
|
|
17607
|
+
to_jsonb(outbox)->>'delivery_batch_started_at' AS delivery_batch_started_at,
|
|
17470
17608
|
outbox.lease_owner,
|
|
17471
17609
|
outbox.lease_expires_at > now() AS lease_live,
|
|
17472
17610
|
coalesce(outbox.next_attempt_at, outbox.created_at) <= now() AS due,
|
|
@@ -17476,6 +17614,7 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
17476
17614
|
FROM ${tableName('outbox', options)} AS outbox
|
|
17477
17615
|
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = outbox.run_id
|
|
17478
17616
|
WHERE outbox.run_id = $1 AND runs.org_id = $2
|
|
17617
|
+
AND NOT ${searchLaneSql('outbox')}
|
|
17479
17618
|
AND outbox.delivered_at IS NULL
|
|
17480
17619
|
ORDER BY ${runLedgerOrderSql('outbox')}
|
|
17481
17620
|
-- Later producer facts cannot belong to this receipt. Do not hydrate their
|
|
@@ -17497,9 +17636,12 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
17497
17636
|
// Cancellation may own several pending facts at once. Validate the exact
|
|
17498
17637
|
// contiguous, due prefix named by the receipt instead of all rows sharing
|
|
17499
17638
|
// that cancellation lease.
|
|
17639
|
+
const ingressReceiptStarted =
|
|
17640
|
+
rows[0]?.event_type === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents &&
|
|
17641
|
+
rows[0]?.delivery_batch_started_at != null;
|
|
17500
17642
|
const prefix: typeof rows = [];
|
|
17501
17643
|
let prefixStopReason: RunLedgerProjectionLeaseLostReason | null = null;
|
|
17502
|
-
for (const row of rows) {
|
|
17644
|
+
for (const [index, row] of rows.entries()) {
|
|
17503
17645
|
if (row.lease_owner !== leaseOwner) {
|
|
17504
17646
|
prefixStopReason = 'prefix_lease_owner_mismatch';
|
|
17505
17647
|
break;
|
|
@@ -17516,11 +17658,6 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
17516
17658
|
prefixStopReason = 'predecessor_failed';
|
|
17517
17659
|
break;
|
|
17518
17660
|
}
|
|
17519
|
-
if (
|
|
17520
|
-
prefix.length > 0 &&
|
|
17521
|
-
row.event_type === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents
|
|
17522
|
-
)
|
|
17523
|
-
break;
|
|
17524
17661
|
prefix.push(row);
|
|
17525
17662
|
|
|
17526
17663
|
const first = prefix[0]!;
|
|
@@ -17567,10 +17704,17 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
17567
17704
|
runId: input.runId,
|
|
17568
17705
|
deliveryKey: first.delivery_key,
|
|
17569
17706
|
});
|
|
17707
|
+
// A receipt already attempted as a compound prefix cannot be split by
|
|
17708
|
+
// an older competing projector after lease expiry. Reject with the normal
|
|
17709
|
+
// retryable lease-lost response until a capable writer reclaims it whole.
|
|
17710
|
+
const atStartedPrefixEnd =
|
|
17711
|
+
!ingressReceiptStarted ||
|
|
17712
|
+
rows[index + 1]?.delivery_batch_started_at == null;
|
|
17570
17713
|
if (
|
|
17571
|
-
|
|
17572
|
-
input.idempotencyKey ===
|
|
17573
|
-
|
|
17714
|
+
atStartedPrefixEnd &&
|
|
17715
|
+
(input.idempotencyKey === expectedKey ||
|
|
17716
|
+
input.idempotencyKey === batchKey ||
|
|
17717
|
+
legacySingletonKey)
|
|
17574
17718
|
) {
|
|
17575
17719
|
if (input.projectedEvents === undefined) return;
|
|
17576
17720
|
const prefixEvents: PlayRunLedgerIngressEvent[] = [];
|
|
@@ -17612,11 +17756,18 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
17612
17756
|
|
|
17613
17757
|
if (
|
|
17614
17758
|
prefix.length === 1 &&
|
|
17615
|
-
|
|
17759
|
+
RUN_SCHEDULER_OUTBOX_START_EVENT_TYPES.includes(first.event_type)
|
|
17616
17760
|
) {
|
|
17617
17761
|
break;
|
|
17618
17762
|
}
|
|
17619
|
-
|
|
17763
|
+
// A frozen ingress receipt can include late runner logs after terminal.
|
|
17764
|
+
// Prove its complete immutable boundary, including transport chunks after
|
|
17765
|
+
// terminal, rather than splitting a receipt that has already been attempted.
|
|
17766
|
+
if (
|
|
17767
|
+
(terminal && !ingressReceiptStarted) ||
|
|
17768
|
+
(ingressReceiptStarted && atStartedPrefixEnd) ||
|
|
17769
|
+
prefix.length >= RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS
|
|
17770
|
+
) {
|
|
17620
17771
|
break;
|
|
17621
17772
|
}
|
|
17622
17773
|
}
|
|
@@ -17631,7 +17782,11 @@ function runLedgerPrecedesSql(
|
|
|
17631
17782
|
successor: string,
|
|
17632
17783
|
inclusive = false,
|
|
17633
17784
|
): string {
|
|
17634
|
-
return
|
|
17785
|
+
return `(${searchLaneSql(predecessor)} = ${searchLaneSql(successor)} AND ${runLedgerPositionSql(predecessor)} ${inclusive ? '<=' : '<'} ${runLedgerPositionSql(successor)})`;
|
|
17786
|
+
}
|
|
17787
|
+
|
|
17788
|
+
function searchLaneSql(alias: string): string {
|
|
17789
|
+
return `coalesce(${alias}.payload_json->'${RUNTIME_EVENT_SEARCH_ONLY_FIELD}' = 'true'::jsonb, false)`;
|
|
17635
17790
|
}
|
|
17636
17791
|
|
|
17637
17792
|
function runLedgerPositionSql(alias: string): string {
|
|
@@ -17658,6 +17813,32 @@ function runLedgerTerminalPendingSql(
|
|
|
17658
17813
|
)`;
|
|
17659
17814
|
}
|
|
17660
17815
|
|
|
17816
|
+
function runLedgerWaitingRunsSql(
|
|
17817
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
17818
|
+
runIdParameter?: string,
|
|
17819
|
+
): string {
|
|
17820
|
+
return `SELECT DISTINCT waiting_event.run_id
|
|
17821
|
+
FROM ${tableName('outbox', options)} AS waiting_event
|
|
17822
|
+
WHERE waiting_event.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runWaiting}
|
|
17823
|
+
-- Detached runner parks are transport-only and project no public wait.
|
|
17824
|
+
AND waiting_event.payload_json->>'kind' IS DISTINCT FROM 'detached_runner'
|
|
17825
|
+
AND waiting_event.delivered_at IS NULL
|
|
17826
|
+
AND waiting_event.failed_at IS NULL
|
|
17827
|
+
AND coalesce(waiting_event.next_attempt_at, waiting_event.created_at) <= now()
|
|
17828
|
+
${runIdParameter ? `AND (${runIdParameter}::text IS NULL OR waiting_event.run_id = ${runIdParameter}::text)` : ''}`;
|
|
17829
|
+
}
|
|
17830
|
+
|
|
17831
|
+
function runLedgerWaitingPendingSql(alias: string): string {
|
|
17832
|
+
return `(${alias}.run_id IN (SELECT run_id FROM waiting_runs))`;
|
|
17833
|
+
}
|
|
17834
|
+
|
|
17835
|
+
function runLedgerUrgentLifecyclePendingSql(
|
|
17836
|
+
alias: string,
|
|
17837
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
17838
|
+
): string {
|
|
17839
|
+
return `(${runLedgerTerminalPendingSql(alias, options)} OR ${runLedgerWaitingPendingSql(alias)})`;
|
|
17840
|
+
}
|
|
17841
|
+
|
|
17661
17842
|
/**
|
|
17662
17843
|
* Claim a causal prefix, preserving the v33 adapter while the additive batch
|
|
17663
17844
|
* marker is absent. An enclosing transaction uses a savepoint so the missing
|
|
@@ -17670,6 +17851,12 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17670
17851
|
runId?: string | null;
|
|
17671
17852
|
leaseOwner?: string | null;
|
|
17672
17853
|
leaseSeconds?: number;
|
|
17854
|
+
/** Independent raw search work uses the same outbox leases and controls. */
|
|
17855
|
+
searchOnly?: boolean;
|
|
17856
|
+
/** Advertised by the launch-pinned receipt verifier, never inferred locally. */
|
|
17857
|
+
allowIngressPrefix?: boolean;
|
|
17858
|
+
/** Extend this still-owned head atomically under the run cancellation fence. */
|
|
17859
|
+
extendOwnedHead?: string;
|
|
17673
17860
|
/** The dispatcher owns a short, exclusive claim transaction. */
|
|
17674
17861
|
inTransaction?: boolean;
|
|
17675
17862
|
} = {},
|
|
@@ -17682,8 +17869,16 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17682
17869
|
const runId = input.runId?.trim() || null;
|
|
17683
17870
|
const leaseOwner = input.leaseOwner?.trim() || null;
|
|
17684
17871
|
const leaseSeconds = Math.max(5, Math.min(300, input.leaseSeconds ?? 30));
|
|
17685
|
-
const claim = async (supportsTerminalBatchMarker: boolean) =>
|
|
17686
|
-
|
|
17872
|
+
const claim = async (supportsTerminalBatchMarker: boolean) => {
|
|
17873
|
+
const urgentLifecyclePendingSql = (alias: string) =>
|
|
17874
|
+
input.searchOnly === true
|
|
17875
|
+
? runLedgerTerminalPendingSql(alias, options)
|
|
17876
|
+
: runLedgerUrgentLifecyclePendingSql(alias, options);
|
|
17877
|
+
const ingressPrefixSql =
|
|
17878
|
+
supportsTerminalBatchMarker && input.allowIngressPrefix === true
|
|
17879
|
+
? `(head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} AND ${urgentLifecyclePendingSql('head')})`
|
|
17880
|
+
: 'false';
|
|
17881
|
+
return await client.query<{
|
|
17687
17882
|
event_id: string;
|
|
17688
17883
|
run_id: string;
|
|
17689
17884
|
event_type: string;
|
|
@@ -17692,10 +17887,33 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17692
17887
|
delivery_key: string | null;
|
|
17693
17888
|
}>(
|
|
17694
17889
|
`
|
|
17695
|
-
WITH
|
|
17696
|
-
|
|
17890
|
+
WITH ${
|
|
17891
|
+
input.searchOnly === true
|
|
17892
|
+
? ''
|
|
17893
|
+
: `waiting_runs AS MATERIALIZED (
|
|
17894
|
+
${runLedgerWaitingRunsSql(options, '$2')}
|
|
17895
|
+
), `
|
|
17896
|
+
}${
|
|
17897
|
+
input.extendOwnedHead
|
|
17898
|
+
? `owned_run AS MATERIALIZED (
|
|
17899
|
+
SELECT run_id FROM ${tableName('runs', options)}
|
|
17900
|
+
WHERE run_id = $2
|
|
17901
|
+
FOR UPDATE SKIP LOCKED
|
|
17902
|
+
),`
|
|
17903
|
+
: ''
|
|
17904
|
+
} head AS MATERIALIZED (
|
|
17905
|
+
SELECT candidate.event_id, candidate.run_id, candidate.event_type,
|
|
17906
|
+
${supportsTerminalBatchMarker ? 'candidate.delivery_batch_started_at' : 'NULL::timestamptz'} AS delivery_batch_started_at
|
|
17697
17907
|
FROM ${tableName('outbox', options)} AS candidate
|
|
17908
|
+
${input.extendOwnedHead ? 'JOIN owned_run ON owned_run.run_id = candidate.run_id' : ''}
|
|
17698
17909
|
WHERE candidate.delivered_at IS NULL
|
|
17910
|
+
AND ${searchLaneSql('candidate')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
17911
|
+
${
|
|
17912
|
+
input.extendOwnedHead
|
|
17913
|
+
? `AND candidate.event_id = $5
|
|
17914
|
+
AND candidate.lease_owner = $3 AND candidate.lease_expires_at > now()`
|
|
17915
|
+
: ''
|
|
17916
|
+
}
|
|
17699
17917
|
AND candidate.failed_at IS NULL
|
|
17700
17918
|
AND coalesce(candidate.next_attempt_at, candidate.created_at) <= now()
|
|
17701
17919
|
AND ($2::text IS NULL OR candidate.run_id = $2)
|
|
@@ -17711,6 +17929,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17711
17929
|
AND NOT EXISTS (
|
|
17712
17930
|
SELECT 1 FROM ${tableName('outbox', options)} AS active
|
|
17713
17931
|
WHERE active.run_id = candidate.run_id
|
|
17932
|
+
AND ${searchLaneSql('active')} = ${searchLaneSql('candidate')}
|
|
17714
17933
|
AND active.delivered_at IS NULL
|
|
17715
17934
|
AND active.lease_owner IS NOT NULL
|
|
17716
17935
|
AND active.lease_expires_at > now()
|
|
@@ -17738,7 +17957,8 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17738
17957
|
CASE
|
|
17739
17958
|
WHEN candidate.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0
|
|
17740
17959
|
WHEN ${runLedgerTerminalPendingSql('candidate', options)} THEN 1
|
|
17741
|
-
|
|
17960
|
+
${input.searchOnly === true ? '' : `WHEN ${runLedgerWaitingPendingSql('candidate')} THEN 2`}
|
|
17961
|
+
ELSE 3
|
|
17742
17962
|
END,
|
|
17743
17963
|
coalesce(candidate.next_attempt_at, candidate.created_at),
|
|
17744
17964
|
candidate.created_at,
|
|
@@ -17750,16 +17970,25 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17750
17970
|
FROM ${tableName('outbox', options)} AS outbox
|
|
17751
17971
|
JOIN head ON head.run_id = outbox.run_id
|
|
17752
17972
|
WHERE outbox.delivered_at IS NULL
|
|
17973
|
+
AND ${searchLaneSql('outbox')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
17753
17974
|
AND outbox.failed_at IS NULL
|
|
17754
17975
|
-- Admission is causally first and must be projected alone. A later
|
|
17755
17976
|
-- start/terminal append may not leapfrog a delayed queued row.
|
|
17756
17977
|
AND (
|
|
17757
17978
|
head.event_type NOT IN (${RUN_EVENT_KIND_SQL_LITERAL.runCreated}, ${RUN_EVENT_KIND_SQL_LITERAL.runStarted}, ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents})
|
|
17758
17979
|
OR outbox.event_id = head.event_id
|
|
17980
|
+
OR (head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
|
|
17981
|
+
AND ${ingressPrefixSql})
|
|
17759
17982
|
)
|
|
17760
|
-
--
|
|
17761
|
-
--
|
|
17762
|
-
|
|
17983
|
+
-- Only a receiver with the exact-prefix verifier may combine ingress
|
|
17984
|
+
-- facts while draining a pending terminal. Old launch origins and
|
|
17985
|
+
-- schemas retain each producer's existing singleton receipt.
|
|
17986
|
+
AND (outbox.event_type <> ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
|
|
17987
|
+
OR outbox.event_id = head.event_id
|
|
17988
|
+
OR ${ingressPrefixSql})
|
|
17989
|
+
-- Once any ingress prefix was attempted, its receipt is immutable,
|
|
17990
|
+
-- including singleton appends whose HTTP/ACK outcome was ambiguous.
|
|
17991
|
+
AND (${supportsTerminalBatchMarker && input.allowIngressPrefix === true ? 'head.delivery_batch_started_at IS NULL OR outbox.delivery_batch_started_at IS NOT NULL' : 'true'})
|
|
17763
17992
|
AND coalesce(outbox.next_attempt_at, outbox.created_at) <= now()
|
|
17764
17993
|
AND (
|
|
17765
17994
|
$3::text IS NULL
|
|
@@ -17802,7 +18031,9 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17802
18031
|
AND predecessor.delivered_at IS NULL
|
|
17803
18032
|
AND ${runLedgerPrecedesSql('predecessor', 'outbox')}
|
|
17804
18033
|
AND (
|
|
17805
|
-
predecessor.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
|
|
18034
|
+
(predecessor.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
|
|
18035
|
+
AND NOT (${ingressPrefixSql}))
|
|
18036
|
+
OR (${supportsTerminalBatchMarker && input.allowIngressPrefix === true ? 'head.delivery_batch_started_at IS NOT NULL AND predecessor.delivery_batch_started_at IS NULL' : 'false'})
|
|
17806
18037
|
OR predecessor.failed_at IS NOT NULL
|
|
17807
18038
|
OR coalesce(predecessor.next_attempt_at, predecessor.created_at) > now()
|
|
17808
18039
|
OR (
|
|
@@ -17833,8 +18064,8 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17833
18064
|
${
|
|
17834
18065
|
supportsTerminalBatchMarker
|
|
17835
18066
|
? `, delivery_batch_started_at = CASE
|
|
17836
|
-
WHEN terminal_batch.has_terminal
|
|
17837
|
-
THEN COALESCE(outbox.delivery_batch_started_at,
|
|
18067
|
+
WHEN terminal_batch.has_terminal ${input.allowIngressPrefix === true ? `OR ${urgentLifecyclePendingSql('outbox')}` : ''}
|
|
18068
|
+
THEN COALESCE(outbox.delivery_batch_started_at, statement_timestamp())
|
|
17838
18069
|
ELSE outbox.delivery_batch_started_at
|
|
17839
18070
|
END`
|
|
17840
18071
|
: ''
|
|
@@ -17851,8 +18082,15 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17851
18082
|
FROM claimed
|
|
17852
18083
|
ORDER BY ${runLedgerOrderSql('claimed')}
|
|
17853
18084
|
`,
|
|
17854
|
-
[
|
|
18085
|
+
[
|
|
18086
|
+
limit,
|
|
18087
|
+
runId,
|
|
18088
|
+
leaseOwner,
|
|
18089
|
+
leaseSeconds,
|
|
18090
|
+
...(input.extendOwnedHead ? [input.extendOwnedHead] : []),
|
|
18091
|
+
],
|
|
17855
18092
|
);
|
|
18093
|
+
};
|
|
17856
18094
|
let result;
|
|
17857
18095
|
if (input.inTransaction) await client.query('SAVEPOINT run_ledger_claim');
|
|
17858
18096
|
if (supportsOutboxDeliveryBatchMarker(options)) {
|
|
@@ -17910,6 +18148,7 @@ export async function readPostgresSchedulerOutboxNextDueAt(
|
|
|
17910
18148
|
(SELECT max(active.lease_expires_at)
|
|
17911
18149
|
FROM ${tableName('outbox', options)} AS active
|
|
17912
18150
|
WHERE active.run_id = candidate.run_id
|
|
18151
|
+
AND ${searchLaneSql('active')} = ${searchLaneSql('candidate')}
|
|
17913
18152
|
AND active.delivered_at IS NULL AND active.lease_owner IS NOT NULL
|
|
17914
18153
|
AND active.lease_expires_at > now())
|
|
17915
18154
|
)) AS due_at
|
|
@@ -18149,7 +18388,7 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
|
|
|
18149
18388
|
'{events}',
|
|
18150
18389
|
(SELECT jsonb_agg(
|
|
18151
18390
|
CASE WHEN item.value->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
|
|
18152
|
-
THEN item.value - 'lines' - 'lineContexts'
|
|
18391
|
+
THEN item.value - 'lines' - 'lineContexts' - 'lineIds'
|
|
18153
18392
|
ELSE item.value
|
|
18154
18393
|
END ORDER BY item.ordinality
|
|
18155
18394
|
)
|
|
@@ -18162,11 +18401,11 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
|
|
|
18162
18401
|
)
|
|
18163
18402
|
WHEN jsonb_typeof(outbox.payload_json->'terminalProgressEvents') = 'array'
|
|
18164
18403
|
THEN jsonb_set(
|
|
18165
|
-
outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts',
|
|
18404
|
+
outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogIds',
|
|
18166
18405
|
'{terminalProgressEvents}',
|
|
18167
18406
|
coalesce((SELECT jsonb_agg(
|
|
18168
18407
|
CASE WHEN item.value->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
|
|
18169
|
-
THEN item.value - 'lines' - 'lineContexts'
|
|
18408
|
+
THEN item.value - 'lines' - 'lineContexts' - 'lineIds'
|
|
18170
18409
|
ELSE item.value
|
|
18171
18410
|
END ORDER BY item.ordinality
|
|
18172
18411
|
)
|
|
@@ -18174,7 +18413,7 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
|
|
|
18174
18413
|
WITH ORDINALITY AS item(value, ordinality)), '[]'::jsonb),
|
|
18175
18414
|
false
|
|
18176
18415
|
)
|
|
18177
|
-
ELSE outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts'
|
|
18416
|
+
ELSE outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogIds'
|
|
18178
18417
|
END
|
|
18179
18418
|
FROM candidates
|
|
18180
18419
|
WHERE outbox.event_id = candidates.event_id`,
|
|
@@ -18538,6 +18777,22 @@ function normalizeOutboxIngressPreview(
|
|
|
18538
18777
|
export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
18539
18778
|
event: PostgresSchedulerOutboxEvent,
|
|
18540
18779
|
): PlayRunLedgerIngressEvent[] {
|
|
18780
|
+
if (
|
|
18781
|
+
payloadField(event.payload, RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD) ===
|
|
18782
|
+
true
|
|
18783
|
+
) {
|
|
18784
|
+
throw new Error(
|
|
18785
|
+
'Deferred outbox input must be projected before ledger delivery.',
|
|
18786
|
+
);
|
|
18787
|
+
}
|
|
18788
|
+
if (
|
|
18789
|
+
payloadField(event.payload, RUNTIME_EVENT_SEARCH_ONLY_FIELD) === true ||
|
|
18790
|
+
event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.receiptEvent ||
|
|
18791
|
+
event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runnerLogs
|
|
18792
|
+
) {
|
|
18793
|
+
// Search-only carriers are already durable in the canonical journal.
|
|
18794
|
+
return [];
|
|
18795
|
+
}
|
|
18541
18796
|
const occurredAt = event.createdAt.getTime();
|
|
18542
18797
|
if (event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents) {
|
|
18543
18798
|
const events = payloadField(event.payload, 'events');
|
|
@@ -18774,6 +19029,8 @@ export type PostgresSchedulerOutboxMaterializationPlan = {
|
|
|
18774
19029
|
};
|
|
18775
19030
|
|
|
18776
19031
|
const RUN_LEDGER_KNOWN_OUTBOX_EVENT_TYPES: ReadonlySet<string> = new Set([
|
|
19032
|
+
RUN_SCHEDULER_OUTBOX_EVENT_TYPE.receiptEvent,
|
|
19033
|
+
RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runnerLogs,
|
|
18777
19034
|
RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents,
|
|
18778
19035
|
RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runStarted,
|
|
18779
19036
|
RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runCompleted,
|
|
@@ -18912,6 +19169,21 @@ export type RunLedgerOutboxTokenAuthority = 'local_mint' | 'gateway';
|
|
|
18912
19169
|
/** Cross-run service policy; causal order within a run is always unchanged. */
|
|
18913
19170
|
export type RunLedgerOutboxHeadPriority = 'admission_first' | 'oldest_ready';
|
|
18914
19171
|
|
|
19172
|
+
class DeferredRuntimeProjectionInputError extends AppRuntimeApiResponseError {
|
|
19173
|
+
constructor(
|
|
19174
|
+
readonly eventId: string,
|
|
19175
|
+
detail: string,
|
|
19176
|
+
) {
|
|
19177
|
+
super({
|
|
19178
|
+
action: 'project_run_events',
|
|
19179
|
+
status: 422,
|
|
19180
|
+
code: 'runtime_event_projection_invalid',
|
|
19181
|
+
retryable: false,
|
|
19182
|
+
detail,
|
|
19183
|
+
});
|
|
19184
|
+
}
|
|
19185
|
+
}
|
|
19186
|
+
|
|
18915
19187
|
const POSTGRES_SCHEDULER_CANCELLATION_OUTBOX_LEASE_SECONDS =
|
|
18916
19188
|
// The direct projection can run for the full app-runtime request deadline.
|
|
18917
19189
|
// Keep another 30 seconds for dispatch setup and acknowledgement so a
|
|
@@ -18942,6 +19214,10 @@ export function postgresSchedulerCancellationOutboxLease(runId: string): {
|
|
|
18942
19214
|
}
|
|
18943
19215
|
|
|
18944
19216
|
export type RunLedgerOutboxDispatchInput = {
|
|
19217
|
+
/** Only the existing background projector claims the independent search prefix. */
|
|
19218
|
+
searchOnly?: boolean;
|
|
19219
|
+
/** Only the existing background projector indexes deferred raw inputs. */
|
|
19220
|
+
materializeObservability?: boolean;
|
|
18945
19221
|
/**
|
|
18946
19222
|
* Optional runtime context. `gateway` authority always replaces its
|
|
18947
19223
|
* credentials with a fresh lease-scoped token and keeps only supported
|
|
@@ -18983,8 +19259,9 @@ export type RunLedgerOutboxDispatchInput = {
|
|
|
18983
19259
|
* untargeted fast path) set this: after acknowledging a batch, also carry
|
|
18984
19260
|
* the run's terminal fact through when it is now pending (see
|
|
18985
19261
|
* `continueRunLedgerOutboxThroughTerminal`). Callers that drive their own
|
|
18986
|
-
* bounded batches
|
|
18987
|
-
*
|
|
19262
|
+
* bounded batches (cancellation, inline `run.created`, the targeted terminal
|
|
19263
|
+
* drain) leave it unset. Targeted drains retain successors through their
|
|
19264
|
+
* explicit run/owner/target scope instead.
|
|
18988
19265
|
*/
|
|
18989
19266
|
continueThroughTerminal?: boolean;
|
|
18990
19267
|
};
|
|
@@ -19066,7 +19343,10 @@ type RunLedgerOutboxAuthorityScope = {
|
|
|
19066
19343
|
authority: {
|
|
19067
19344
|
runId: string;
|
|
19068
19345
|
leaseOwner: string;
|
|
19069
|
-
runtimeApi: WorkerRuntimeApiContext
|
|
19346
|
+
runtimeApi: WorkerRuntimeApiContext & {
|
|
19347
|
+
refreshExecutorToken?: () => Promise<string>;
|
|
19348
|
+
invalidateExecutorToken?: () => void;
|
|
19349
|
+
};
|
|
19070
19350
|
expiresAtMs: number | null;
|
|
19071
19351
|
} | null;
|
|
19072
19352
|
};
|
|
@@ -19101,6 +19381,9 @@ async function dispatchRunLedgerOutboxThroughEventWithScope(
|
|
|
19101
19381
|
input.leaseOwner?.trim() ||
|
|
19102
19382
|
scope?.leaseOwner ||
|
|
19103
19383
|
`run-ledger-outbox-dispatch:${randomUUID()}`;
|
|
19384
|
+
// The same owner retains each successor in the ACK transaction, including
|
|
19385
|
+
// for a fresh worker drain that did not arrive with a preclaimed prefix.
|
|
19386
|
+
scope ??= { runId, leaseOwner, targetEventId, nextClaimed: [] };
|
|
19104
19387
|
const scopedInput = { ...input, leaseOwner };
|
|
19105
19388
|
const batchSize = RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS;
|
|
19106
19389
|
const eventIds: string[] = [];
|
|
@@ -19111,75 +19394,87 @@ async function dispatchRunLedgerOutboxThroughEventWithScope(
|
|
|
19111
19394
|
let failed = 0;
|
|
19112
19395
|
let retrying = 0;
|
|
19113
19396
|
|
|
19114
|
-
|
|
19115
|
-
|
|
19116
|
-
|
|
19117
|
-
|
|
19118
|
-
|
|
19119
|
-
|
|
19120
|
-
|
|
19121
|
-
|
|
19122
|
-
|
|
19123
|
-
|
|
19124
|
-
|
|
19125
|
-
|
|
19126
|
-
|
|
19397
|
+
try {
|
|
19398
|
+
let remaining = maxEvents;
|
|
19399
|
+
while (remaining === null || remaining > 0) {
|
|
19400
|
+
const result = await dispatchRunLedgerOutboxBatchWithScope(
|
|
19401
|
+
withClient,
|
|
19402
|
+
{
|
|
19403
|
+
...scopedInput,
|
|
19404
|
+
waitForRunLock: true,
|
|
19405
|
+
continueThroughTerminal: false,
|
|
19406
|
+
limit:
|
|
19407
|
+
remaining === null ? batchSize : Math.min(batchSize, remaining),
|
|
19408
|
+
},
|
|
19409
|
+
options,
|
|
19410
|
+
scope,
|
|
19411
|
+
authorityScope,
|
|
19412
|
+
remaining,
|
|
19413
|
+
);
|
|
19414
|
+
attempted += result.attempted;
|
|
19415
|
+
projected += result.projected;
|
|
19416
|
+
failed += result.failed;
|
|
19417
|
+
retrying += result.retrying;
|
|
19418
|
+
eventIds.push(...result.eventIds);
|
|
19419
|
+
failedEventIds.push(...result.failedEventIds);
|
|
19420
|
+
retryingEventIds.push(...result.retryingEventIds);
|
|
19421
|
+
if (remaining !== null) remaining -= result.attempted;
|
|
19422
|
+
if (result.eventIds.includes(targetEventId)) {
|
|
19423
|
+
return {
|
|
19424
|
+
attempted,
|
|
19425
|
+
projected,
|
|
19426
|
+
eventIds,
|
|
19427
|
+
failed,
|
|
19428
|
+
failedEventIds,
|
|
19429
|
+
retrying,
|
|
19430
|
+
retryingEventIds,
|
|
19431
|
+
targetEventId,
|
|
19432
|
+
targetDelivered: true,
|
|
19433
|
+
targetBlocked: false,
|
|
19434
|
+
};
|
|
19435
|
+
}
|
|
19436
|
+
// A projection failure is durably scheduled for retry or moved to terminal
|
|
19437
|
+
// DLQ. Its policy owns the next attempt; spinning here would only reclaim
|
|
19438
|
+
// the same failed predecessor and delay the park indefinitely.
|
|
19439
|
+
// An absent target cannot retain a successor in ACK. Keep the existing
|
|
19440
|
+
// fresh-claim traversal in that case until ready work is exhausted.
|
|
19441
|
+
if (result.projected === 0 || result.attempted === 0) break;
|
|
19442
|
+
}
|
|
19443
|
+
|
|
19444
|
+
// A concurrent dispatcher can claim the target between batches. An empty
|
|
19445
|
+
// drain therefore does not prove delivery; read the durable acknowledgement
|
|
19446
|
+
// before reporting the target as pending.
|
|
19447
|
+
const target = await withClient((client) =>
|
|
19448
|
+
readPostgresSchedulerOutboxDeliveryStatus(
|
|
19449
|
+
client,
|
|
19450
|
+
{ runId, eventId: targetEventId },
|
|
19451
|
+
options,
|
|
19452
|
+
),
|
|
19127
19453
|
);
|
|
19128
|
-
|
|
19129
|
-
|
|
19130
|
-
|
|
19131
|
-
|
|
19132
|
-
|
|
19133
|
-
|
|
19134
|
-
|
|
19135
|
-
|
|
19136
|
-
|
|
19137
|
-
|
|
19138
|
-
|
|
19139
|
-
|
|
19140
|
-
|
|
19141
|
-
|
|
19142
|
-
|
|
19143
|
-
|
|
19144
|
-
|
|
19145
|
-
|
|
19146
|
-
|
|
19147
|
-
|
|
19148
|
-
|
|
19454
|
+
return {
|
|
19455
|
+
attempted,
|
|
19456
|
+
projected,
|
|
19457
|
+
eventIds,
|
|
19458
|
+
failed,
|
|
19459
|
+
failedEventIds,
|
|
19460
|
+
retrying,
|
|
19461
|
+
retryingEventIds,
|
|
19462
|
+
targetEventId,
|
|
19463
|
+
targetDelivered: target.delivered,
|
|
19464
|
+
targetBlocked: target.blocked,
|
|
19465
|
+
};
|
|
19466
|
+
} finally {
|
|
19467
|
+
if (scope.nextClaimed.length > 0) {
|
|
19468
|
+
await withClient((client) =>
|
|
19469
|
+
releasePostgresSchedulerOutboxLease(
|
|
19470
|
+
client,
|
|
19471
|
+
scope.nextClaimed.map((event) => event.eventId),
|
|
19472
|
+
{ ...options, leaseOwner },
|
|
19473
|
+
),
|
|
19474
|
+
);
|
|
19475
|
+
scope.nextClaimed.length = 0;
|
|
19149
19476
|
}
|
|
19150
|
-
// A projection failure is durably scheduled for retry or moved to terminal
|
|
19151
|
-
// DLQ. Its policy owns the next attempt; spinning here would only reclaim
|
|
19152
|
-
// the same failed predecessor and delay the park indefinitely.
|
|
19153
|
-
if (
|
|
19154
|
-
result.projected === 0 ||
|
|
19155
|
-
result.attempted === 0 ||
|
|
19156
|
-
(scope && scope.nextClaimed.length === 0)
|
|
19157
|
-
)
|
|
19158
|
-
break;
|
|
19159
19477
|
}
|
|
19160
|
-
|
|
19161
|
-
// A concurrent dispatcher can claim the target between batches. An empty
|
|
19162
|
-
// drain therefore does not prove delivery; read the durable acknowledgement
|
|
19163
|
-
// before reporting the target as pending.
|
|
19164
|
-
const target = await withClient((client) =>
|
|
19165
|
-
readPostgresSchedulerOutboxDeliveryStatus(
|
|
19166
|
-
client,
|
|
19167
|
-
{ runId, eventId: targetEventId },
|
|
19168
|
-
options,
|
|
19169
|
-
),
|
|
19170
|
-
);
|
|
19171
|
-
return {
|
|
19172
|
-
attempted,
|
|
19173
|
-
projected,
|
|
19174
|
-
eventIds,
|
|
19175
|
-
failed,
|
|
19176
|
-
failedEventIds,
|
|
19177
|
-
retrying,
|
|
19178
|
-
retryingEventIds,
|
|
19179
|
-
targetEventId,
|
|
19180
|
-
targetDelivered: target.delivered,
|
|
19181
|
-
targetBlocked: target.blocked,
|
|
19182
|
-
};
|
|
19183
19478
|
}
|
|
19184
19479
|
|
|
19185
19480
|
/**
|
|
@@ -19219,8 +19514,15 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19219
19514
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
19220
19515
|
scope?: RunLedgerOutboxDrainScope,
|
|
19221
19516
|
authorityScope: RunLedgerOutboxAuthorityScope = { authority: null },
|
|
19517
|
+
remainingEvents: number | null = null,
|
|
19222
19518
|
): Promise<PostgresSchedulerOutboxProjectionResult> {
|
|
19223
19519
|
const dispatchStartedAtMs = Date.now();
|
|
19520
|
+
const tokenAuthority = input.tokenAuthority ?? 'local_mint';
|
|
19521
|
+
const dispatchLaunchSql =
|
|
19522
|
+
!input.searchOnly && (tokenAuthority === 'gateway' || !input.runtimeApi)
|
|
19523
|
+
? runLedgerDispatchLaunchSql('runs')
|
|
19524
|
+
: 'NULL::jsonb';
|
|
19525
|
+
let claimedLaunchJson: unknown;
|
|
19224
19526
|
const leaseOwner =
|
|
19225
19527
|
input.leaseOwner?.trim() || `run-ledger-outbox-dispatch:${randomUUID()}`;
|
|
19226
19528
|
const continuation: { current: RunLedgerOutboxDrainScope | null } = {
|
|
@@ -19272,7 +19574,10 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19272
19574
|
// for each lifecycle transition. Only the two dedicated projectors may
|
|
19273
19575
|
// perform the global distinct-head scan below.
|
|
19274
19576
|
const selected = targetRunId
|
|
19275
|
-
? await client.query<{
|
|
19577
|
+
? await client.query<{
|
|
19578
|
+
run_id: string;
|
|
19579
|
+
dispatch_launch_json?: unknown;
|
|
19580
|
+
}>(
|
|
19276
19581
|
`
|
|
19277
19582
|
WITH head AS MATERIALIZED (
|
|
19278
19583
|
SELECT pending.run_id,
|
|
@@ -19284,10 +19589,11 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19284
19589
|
FROM ${tableName('outbox', options)} AS pending
|
|
19285
19590
|
WHERE pending.run_id = $1
|
|
19286
19591
|
AND pending.delivered_at IS NULL
|
|
19592
|
+
AND ${searchLaneSql('pending')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
19287
19593
|
ORDER BY ${runLedgerOrderSql('pending')}
|
|
19288
19594
|
LIMIT 1
|
|
19289
19595
|
)
|
|
19290
|
-
SELECT runs.run_id
|
|
19596
|
+
SELECT runs.run_id, ${dispatchLaunchSql} AS dispatch_launch_json
|
|
19291
19597
|
FROM head
|
|
19292
19598
|
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
|
|
19293
19599
|
WHERE head.failed_at IS NULL
|
|
@@ -19296,6 +19602,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19296
19602
|
AND NOT EXISTS (
|
|
19297
19603
|
SELECT 1 FROM ${tableName('outbox', options)} AS active
|
|
19298
19604
|
WHERE active.run_id = runs.run_id
|
|
19605
|
+
AND ${searchLaneSql('active')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
19299
19606
|
AND active.delivered_at IS NULL
|
|
19300
19607
|
AND active.lease_owner IS NOT NULL
|
|
19301
19608
|
AND active.lease_expires_at > now()
|
|
@@ -19305,8 +19612,17 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19305
19612
|
`,
|
|
19306
19613
|
[targetRunId, leaseOwner, input.headEventId?.trim() || null],
|
|
19307
19614
|
)
|
|
19308
|
-
: await client.query<{
|
|
19309
|
-
|
|
19615
|
+
: await client.query<{
|
|
19616
|
+
run_id: string;
|
|
19617
|
+
dispatch_launch_json?: unknown;
|
|
19618
|
+
}>(`
|
|
19619
|
+
WITH ${
|
|
19620
|
+
input.searchOnly === true
|
|
19621
|
+
? ''
|
|
19622
|
+
: `waiting_runs AS MATERIALIZED (
|
|
19623
|
+
${runLedgerWaitingRunsSql(options)}
|
|
19624
|
+
), `
|
|
19625
|
+
}heads AS MATERIALIZED (
|
|
19310
19626
|
SELECT DISTINCT ON (pending.run_id)
|
|
19311
19627
|
pending.run_id,
|
|
19312
19628
|
pending.event_id,
|
|
@@ -19316,9 +19632,10 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19316
19632
|
pending.failed_at
|
|
19317
19633
|
FROM ${tableName('outbox', options)} AS pending
|
|
19318
19634
|
WHERE pending.delivered_at IS NULL
|
|
19635
|
+
AND ${searchLaneSql('pending')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
19319
19636
|
ORDER BY pending.run_id, ${runLedgerOrderSql('pending')}
|
|
19320
19637
|
)
|
|
19321
|
-
SELECT runs.run_id
|
|
19638
|
+
SELECT runs.run_id, ${dispatchLaunchSql} AS dispatch_launch_json
|
|
19322
19639
|
FROM heads AS head
|
|
19323
19640
|
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
|
|
19324
19641
|
WHERE head.failed_at IS NULL
|
|
@@ -19326,12 +19643,13 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19326
19643
|
AND NOT EXISTS (
|
|
19327
19644
|
SELECT 1 FROM ${tableName('outbox', options)} AS active
|
|
19328
19645
|
WHERE active.run_id = runs.run_id
|
|
19646
|
+
AND ${searchLaneSql('active')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
19329
19647
|
AND active.delivered_at IS NULL
|
|
19330
19648
|
AND active.lease_owner IS NOT NULL
|
|
19331
19649
|
AND active.lease_expires_at > now()
|
|
19332
19650
|
)
|
|
19333
19651
|
-- admission_first: admission, then the causal head of a run with
|
|
19334
|
-
-- a pending terminal, then FIFO progress. oldest_ready keeps its
|
|
19652
|
+
-- a pending terminal or public wait, then FIFO progress. oldest_ready keeps its
|
|
19335
19653
|
-- pure age order so continuous admissions/terminals cannot starve
|
|
19336
19654
|
-- ordinary progress.
|
|
19337
19655
|
ORDER BY ${
|
|
@@ -19340,7 +19658,8 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19340
19658
|
: `CASE
|
|
19341
19659
|
WHEN head.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0
|
|
19342
19660
|
WHEN ${runLedgerTerminalPendingSql('head', options)} THEN 1
|
|
19343
|
-
|
|
19661
|
+
${input.searchOnly === true ? '' : `WHEN ${runLedgerWaitingPendingSql('head')} THEN 2`}
|
|
19662
|
+
ELSE 3
|
|
19344
19663
|
END,`
|
|
19345
19664
|
}
|
|
19346
19665
|
head.created_at, runs.run_id
|
|
@@ -19356,10 +19675,19 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19356
19675
|
leaseOwner,
|
|
19357
19676
|
leaseSeconds: input.leaseSeconds ?? 120,
|
|
19358
19677
|
inTransaction: true,
|
|
19678
|
+
searchOnly: input.searchOnly,
|
|
19359
19679
|
},
|
|
19360
19680
|
options,
|
|
19361
19681
|
)
|
|
19362
19682
|
: [];
|
|
19683
|
+
// Authority/routing is immutable and already lives on the locked Run.
|
|
19684
|
+
// Capture only its existing narrow projection with the successful lease
|
|
19685
|
+
// claim, then release the checkout before requesting gateway authority.
|
|
19686
|
+
// The gateway still independently proves that this lease is current.
|
|
19687
|
+
const launchJson = selected.rows[0]?.dispatch_launch_json;
|
|
19688
|
+
if (claimed.length > 0 && launchJson != null) {
|
|
19689
|
+
claimedLaunchJson = launchJson;
|
|
19690
|
+
}
|
|
19363
19691
|
await client.query('COMMIT');
|
|
19364
19692
|
return claimed;
|
|
19365
19693
|
} catch (error) {
|
|
@@ -19367,7 +19695,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19367
19695
|
throw error;
|
|
19368
19696
|
}
|
|
19369
19697
|
};
|
|
19370
|
-
|
|
19698
|
+
let events = scope?.nextClaimed.length
|
|
19371
19699
|
? scope.nextClaimed.splice(0)
|
|
19372
19700
|
: await withClient(claimEvents);
|
|
19373
19701
|
if (events.length === 0) {
|
|
@@ -19386,9 +19714,148 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19386
19714
|
let authorityReadyAtMs: number | null = null;
|
|
19387
19715
|
let appendFinishedAtMs: number | null = null;
|
|
19388
19716
|
let acknowledgedAtMs: number | null = null;
|
|
19717
|
+
const materializeDeferredObservability = async () => {
|
|
19718
|
+
if (
|
|
19719
|
+
events.some(
|
|
19720
|
+
(event) =>
|
|
19721
|
+
payloadField(
|
|
19722
|
+
event.payload,
|
|
19723
|
+
RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD,
|
|
19724
|
+
) === true,
|
|
19725
|
+
)
|
|
19726
|
+
) {
|
|
19727
|
+
if (!input.materializeObservability) {
|
|
19728
|
+
// Inline admission/lifecycle/read-heal paths must never index logs.
|
|
19729
|
+
// Release and wake the existing projector without charging a retry.
|
|
19730
|
+
const released = await withClient((client) =>
|
|
19731
|
+
releasePostgresSchedulerOutboxLease(
|
|
19732
|
+
client,
|
|
19733
|
+
events.map((event) => event.eventId),
|
|
19734
|
+
{ ...options, leaseOwner },
|
|
19735
|
+
),
|
|
19736
|
+
);
|
|
19737
|
+
return {
|
|
19738
|
+
attempted: events.length,
|
|
19739
|
+
projected: 0,
|
|
19740
|
+
eventIds: [],
|
|
19741
|
+
failed: 0,
|
|
19742
|
+
failedEventIds: [],
|
|
19743
|
+
retrying: released.length,
|
|
19744
|
+
retryingEventIds: released,
|
|
19745
|
+
};
|
|
19746
|
+
}
|
|
19747
|
+
for (const event of events) {
|
|
19748
|
+
if (
|
|
19749
|
+
payloadField(
|
|
19750
|
+
event.payload,
|
|
19751
|
+
RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD,
|
|
19752
|
+
) !== true
|
|
19753
|
+
)
|
|
19754
|
+
continue;
|
|
19755
|
+
await withClient(async (client) => {
|
|
19756
|
+
await renewPostgresRunLedgerDelivery(
|
|
19757
|
+
client,
|
|
19758
|
+
{
|
|
19759
|
+
eventIds: events.map((row) => row.eventId),
|
|
19760
|
+
leaseOwner,
|
|
19761
|
+
leaseSeconds: 120,
|
|
19762
|
+
},
|
|
19763
|
+
options,
|
|
19764
|
+
);
|
|
19765
|
+
// Keep projection below its renewed lease. The final payload update
|
|
19766
|
+
// fences ownership; indexing locks neither outbox nor Run rows.
|
|
19767
|
+
await client.query('BEGIN');
|
|
19768
|
+
try {
|
|
19769
|
+
// Let PostgreSQL cancel before the pool's client-side timeout,
|
|
19770
|
+
// so rollback completes while the checkout is still usable.
|
|
19771
|
+
await client.query(
|
|
19772
|
+
"SELECT set_config('statement_timeout', $1, true)",
|
|
19773
|
+
[
|
|
19774
|
+
`${Math.min(60_000, RUNTIME_RELIABILITY_POLICY.schedulerDatabase.queryTimeoutMs - 1_000)}ms`,
|
|
19775
|
+
],
|
|
19776
|
+
);
|
|
19777
|
+
const result = await client.query<{ payload: unknown }>(
|
|
19778
|
+
`SELECT ${quoteIdent(schemaName(options))}.materialize_runtime_outbox_event($1, $2) AS payload`,
|
|
19779
|
+
[event.eventId, leaseOwner],
|
|
19780
|
+
);
|
|
19781
|
+
await client.query('COMMIT');
|
|
19782
|
+
event.payload = result.rows[0]!.payload;
|
|
19783
|
+
} catch (error) {
|
|
19784
|
+
await client.query('ROLLBACK');
|
|
19785
|
+
if (
|
|
19786
|
+
error &&
|
|
19787
|
+
typeof error === 'object' &&
|
|
19788
|
+
'code' in error &&
|
|
19789
|
+
error.code === 'P0001'
|
|
19790
|
+
) {
|
|
19791
|
+
// The installed projector's explicit RAISEs reject malformed
|
|
19792
|
+
// or conflicting immutable facts. Retain them in the existing
|
|
19793
|
+
// quarantine instead of retrying the same invalid input forever.
|
|
19794
|
+
throw new DeferredRuntimeProjectionInputError(
|
|
19795
|
+
event.eventId,
|
|
19796
|
+
error instanceof Error
|
|
19797
|
+
? error.message
|
|
19798
|
+
: 'Invalid runtime event projection input.',
|
|
19799
|
+
);
|
|
19800
|
+
}
|
|
19801
|
+
throw error;
|
|
19802
|
+
}
|
|
19803
|
+
});
|
|
19804
|
+
}
|
|
19805
|
+
}
|
|
19806
|
+
return null;
|
|
19807
|
+
};
|
|
19389
19808
|
try {
|
|
19809
|
+
if (!input.materializeObservability) {
|
|
19810
|
+
const handedOff = await materializeDeferredObservability();
|
|
19811
|
+
if (handedOff) return handedOff;
|
|
19812
|
+
}
|
|
19813
|
+
if (input.searchOnly) {
|
|
19814
|
+
const handedOff = await materializeDeferredObservability();
|
|
19815
|
+
if (handedOff) return handedOff;
|
|
19816
|
+
if (
|
|
19817
|
+
events.some(
|
|
19818
|
+
(event) =>
|
|
19819
|
+
payloadField(event.payload, RUNTIME_EVENT_SEARCH_ONLY_FIELD) !==
|
|
19820
|
+
true,
|
|
19821
|
+
)
|
|
19822
|
+
)
|
|
19823
|
+
throw new Error('Search lease includes a lifecycle fact.');
|
|
19824
|
+
const delivered = await withClient((client) =>
|
|
19825
|
+
markPostgresSchedulerOutboxDeliveredRows(
|
|
19826
|
+
client,
|
|
19827
|
+
events.map((event) => event.eventId),
|
|
19828
|
+
{ ...options, leaseOwner },
|
|
19829
|
+
),
|
|
19830
|
+
);
|
|
19831
|
+
if (delivered.length !== events.length)
|
|
19832
|
+
throw new RunLedgerProjectionLeaseLostError(
|
|
19833
|
+
'delivery_lease_not_current',
|
|
19834
|
+
);
|
|
19835
|
+
if (input.onDelivered) {
|
|
19836
|
+
try {
|
|
19837
|
+
input.onDelivered(delivered);
|
|
19838
|
+
} catch (error) {
|
|
19839
|
+
console.warn('[run-ledger-outbox.delivery_observer_failed]', {
|
|
19840
|
+
error: error instanceof Error ? error.message : String(error),
|
|
19841
|
+
});
|
|
19842
|
+
}
|
|
19843
|
+
}
|
|
19844
|
+
return {
|
|
19845
|
+
attempted: events.length,
|
|
19846
|
+
projected: delivered.length,
|
|
19847
|
+
eventIds: events.map((event) => event.eventId),
|
|
19848
|
+
failed: 0,
|
|
19849
|
+
failedEventIds: [],
|
|
19850
|
+
retrying: 0,
|
|
19851
|
+
retryingEventIds: [],
|
|
19852
|
+
};
|
|
19853
|
+
}
|
|
19390
19854
|
const first = events[0]!;
|
|
19391
|
-
const
|
|
19855
|
+
const claimedLaunch =
|
|
19856
|
+
claimedLaunchJson == null
|
|
19857
|
+
? undefined
|
|
19858
|
+
: parseLaunchJson(claimedLaunchJson);
|
|
19392
19859
|
if (
|
|
19393
19860
|
scope &&
|
|
19394
19861
|
(scope.runId !== first.runId || scope.leaseOwner !== leaseOwner)
|
|
@@ -19422,6 +19889,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19422
19889
|
tokenAuthority,
|
|
19423
19890
|
options,
|
|
19424
19891
|
input.runtimeApi?.signal,
|
|
19892
|
+
claimedLaunch,
|
|
19425
19893
|
)
|
|
19426
19894
|
: null;
|
|
19427
19895
|
if (
|
|
@@ -19445,6 +19913,8 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19445
19913
|
: null,
|
|
19446
19914
|
tokenAuthority,
|
|
19447
19915
|
options,
|
|
19916
|
+
undefined,
|
|
19917
|
+
claimedLaunch,
|
|
19448
19918
|
));
|
|
19449
19919
|
const runtimeApi = {
|
|
19450
19920
|
...resolvedRuntimeApi,
|
|
@@ -19486,6 +19956,49 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19486
19956
|
};
|
|
19487
19957
|
}
|
|
19488
19958
|
authorityReadyAtMs = Date.now();
|
|
19959
|
+
// The initial claim deliberately uses the old contract until its immutable
|
|
19960
|
+
// launch receiver advertises support. Widen under the same run/owner fence
|
|
19961
|
+
// before the first HTTP append, so an already-finished run need not pay one
|
|
19962
|
+
// authority/append/ACK round trip for every ingress predecessor.
|
|
19963
|
+
if (
|
|
19964
|
+
runtimeApi.runLedgerIngressPrefixBatching === true &&
|
|
19965
|
+
events.length === 1 &&
|
|
19966
|
+
first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents
|
|
19967
|
+
) {
|
|
19968
|
+
const widened = await withClient((client) =>
|
|
19969
|
+
readPendingPostgresSchedulerOutboxEvents(
|
|
19970
|
+
client,
|
|
19971
|
+
{
|
|
19972
|
+
runId: first.runId,
|
|
19973
|
+
leaseOwner,
|
|
19974
|
+
leaseSeconds: input.leaseSeconds ?? 120,
|
|
19975
|
+
limit: Math.min(
|
|
19976
|
+
RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
|
|
19977
|
+
remainingEvents ?? RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
|
|
19978
|
+
),
|
|
19979
|
+
allowIngressPrefix: true,
|
|
19980
|
+
extendOwnedHead: first.eventId,
|
|
19981
|
+
},
|
|
19982
|
+
options,
|
|
19983
|
+
),
|
|
19984
|
+
);
|
|
19985
|
+
// Expansion uses SKIP LOCKED on the run fence. A concurrent lifecycle
|
|
19986
|
+
// writer can make it empty while our original head lease remains live.
|
|
19987
|
+
// Keep that singleton; mandatory renewal below still rejects expired
|
|
19988
|
+
// or replaced ownership and freezes its receipt before any HTTP append.
|
|
19989
|
+
if (widened.length > 0) {
|
|
19990
|
+
if (widened[0]!.eventId !== first.eventId) {
|
|
19991
|
+
throw new RunLedgerProjectionLeaseLostError(
|
|
19992
|
+
'head_lease_owner_mismatch',
|
|
19993
|
+
);
|
|
19994
|
+
}
|
|
19995
|
+
events = widened;
|
|
19996
|
+
}
|
|
19997
|
+
}
|
|
19998
|
+
// Widening can add deferred facts to the lease. Project the complete
|
|
19999
|
+
// frozen prefix before any HTTP append, including its newly claimed rows.
|
|
20000
|
+
const handedOff = await materializeDeferredObservability();
|
|
20001
|
+
if (handedOff) return handedOff;
|
|
19489
20002
|
const renewDelivery = async () => {
|
|
19490
20003
|
const currentAuthority = authorityScope.authority;
|
|
19491
20004
|
if (
|
|
@@ -19508,14 +20021,16 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19508
20021
|
'Run Ledger dispatch origin changed within an immutable launch.',
|
|
19509
20022
|
);
|
|
19510
20023
|
}
|
|
19511
|
-
runtimeApi.executorToken = refreshed.
|
|
20024
|
+
runtimeApi.executorToken = refreshed.refreshExecutorToken
|
|
20025
|
+
? await refreshed.refreshExecutorToken()
|
|
20026
|
+
: refreshed.executorToken;
|
|
19512
20027
|
runtimeApi.vercelProtectionBypassToken =
|
|
19513
20028
|
refreshed.vercelProtectionBypassToken;
|
|
19514
20029
|
authorityScope.authority = {
|
|
19515
20030
|
runId: first.runId,
|
|
19516
20031
|
leaseOwner,
|
|
19517
20032
|
runtimeApi,
|
|
19518
|
-
expiresAtMs: runtimeApiTokenExpiresAtMs(
|
|
20033
|
+
expiresAtMs: runtimeApiTokenExpiresAtMs(runtimeApi.executorToken),
|
|
19519
20034
|
};
|
|
19520
20035
|
}
|
|
19521
20036
|
await withClient((client) =>
|
|
@@ -19525,70 +20040,108 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19525
20040
|
eventIds: events.map((event) => event.eventId),
|
|
19526
20041
|
leaseOwner,
|
|
19527
20042
|
leaseSeconds: input.leaseSeconds ?? 120,
|
|
20043
|
+
freezeReceipt:
|
|
20044
|
+
runtimeApi.runLedgerIngressPrefixBatching === true &&
|
|
20045
|
+
events.some(
|
|
20046
|
+
(event) =>
|
|
20047
|
+
event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents,
|
|
20048
|
+
),
|
|
19528
20049
|
},
|
|
19529
20050
|
options,
|
|
19530
20051
|
),
|
|
19531
20052
|
);
|
|
19532
20053
|
};
|
|
19533
|
-
|
|
19534
|
-
|
|
19535
|
-
|
|
19536
|
-
|
|
19537
|
-
|
|
19538
|
-
await projectPostgresSchedulerCreatedEvent(runtimeApi, first);
|
|
19539
|
-
} else if (first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runStarted) {
|
|
19540
|
-
await renewDelivery();
|
|
19541
|
-
const leaseOwner = first.leaseOwner;
|
|
19542
|
-
const launch = leaseOwner
|
|
19543
|
-
? await withClient((client) =>
|
|
19544
|
-
readRunLedgerDispatchLaunchForLease(
|
|
19545
|
-
client,
|
|
19546
|
-
{
|
|
19547
|
-
runId: first.runId,
|
|
19548
|
-
eventId: first.eventId,
|
|
19549
|
-
leaseOwner,
|
|
19550
|
-
},
|
|
19551
|
-
options,
|
|
19552
|
-
),
|
|
19553
|
-
)
|
|
19554
|
-
: null;
|
|
19555
|
-
await projectPostgresSchedulerStartEvent(runtimeApi, first, launch);
|
|
19556
|
-
} else {
|
|
19557
|
-
const ledgerEvents = events.flatMap(
|
|
19558
|
-
buildRunLedgerEventsFromPostgresSchedulerOutbox,
|
|
19559
|
-
);
|
|
19560
|
-
if (ledgerEvents.length > 0) {
|
|
19561
|
-
const terminal = [...events]
|
|
19562
|
-
.reverse()
|
|
19563
|
-
.find((event) =>
|
|
19564
|
-
RUN_SCHEDULER_OUTBOX_TERMINAL_EVENT_TYPES.includes(event.eventType),
|
|
19565
|
-
);
|
|
19566
|
-
const batchDeliveryKey = createHash('sha256')
|
|
19567
|
-
.update(events.map((event) => event.deliveryKey).join('\n'))
|
|
19568
|
-
.digest('hex');
|
|
19569
|
-
await projectRunEventsViaAppRuntime(runtimeApi, {
|
|
19570
|
-
playId: first.runId,
|
|
19571
|
-
events: ledgerEvents,
|
|
19572
|
-
deliveryLeaseOwner: leaseOwner,
|
|
19573
|
-
beforeBatch: renewDelivery,
|
|
19574
|
-
idempotencyKey: terminal
|
|
19575
|
-
? buildRunLedgerTerminalIdempotencyKey({
|
|
19576
|
-
runId: first.runId,
|
|
19577
|
-
deliveryKey: terminal.deliveryKey,
|
|
19578
|
-
})
|
|
19579
|
-
: buildRunLedgerEventsIdempotencyKey({
|
|
19580
|
-
runId: first.runId,
|
|
19581
|
-
deliveryDigest: batchDeliveryKey,
|
|
19582
|
-
}),
|
|
19583
|
-
});
|
|
19584
|
-
} else {
|
|
20054
|
+
const append = async () => {
|
|
20055
|
+
// A pending start is always claimed alone because later events require its
|
|
20056
|
+
// snapshot. It cannot share the append endpoint, so retain that mutation
|
|
20057
|
+
// as its own exact delivery.
|
|
20058
|
+
if (first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runCreated) {
|
|
19585
20059
|
await renewDelivery();
|
|
20060
|
+
await projectPostgresSchedulerCreatedEvent(runtimeApi, first);
|
|
20061
|
+
} else if (
|
|
20062
|
+
first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runStarted
|
|
20063
|
+
) {
|
|
20064
|
+
await renewDelivery();
|
|
20065
|
+
const leaseOwner = first.leaseOwner;
|
|
20066
|
+
const launch = leaseOwner
|
|
20067
|
+
? await withClient((client) =>
|
|
20068
|
+
readRunLedgerDispatchLaunchForLease(
|
|
20069
|
+
client,
|
|
20070
|
+
{
|
|
20071
|
+
runId: first.runId,
|
|
20072
|
+
eventId: first.eventId,
|
|
20073
|
+
leaseOwner,
|
|
20074
|
+
},
|
|
20075
|
+
options,
|
|
20076
|
+
),
|
|
20077
|
+
)
|
|
20078
|
+
: null;
|
|
20079
|
+
await projectPostgresSchedulerStartEvent(runtimeApi, first, launch);
|
|
20080
|
+
} else {
|
|
20081
|
+
const ledgerEvents = events.flatMap(
|
|
20082
|
+
buildRunLedgerEventsFromPostgresSchedulerOutbox,
|
|
20083
|
+
);
|
|
20084
|
+
if (ledgerEvents.length > 0) {
|
|
20085
|
+
const terminal = [...events]
|
|
20086
|
+
.reverse()
|
|
20087
|
+
.find((event) =>
|
|
20088
|
+
RUN_SCHEDULER_OUTBOX_TERMINAL_EVENT_TYPES.includes(
|
|
20089
|
+
event.eventType,
|
|
20090
|
+
),
|
|
20091
|
+
);
|
|
20092
|
+
const batchDeliveryKey = createHash('sha256')
|
|
20093
|
+
.update(events.map((event) => event.deliveryKey).join('\n'))
|
|
20094
|
+
.digest('hex');
|
|
20095
|
+
await projectRunEventsViaAppRuntime(runtimeApi, {
|
|
20096
|
+
playId: first.runId,
|
|
20097
|
+
events: ledgerEvents,
|
|
20098
|
+
deliveryLeaseOwner: leaseOwner,
|
|
20099
|
+
beforeBatch: renewDelivery,
|
|
20100
|
+
idempotencyKey: terminal
|
|
20101
|
+
? buildRunLedgerTerminalIdempotencyKey({
|
|
20102
|
+
runId: first.runId,
|
|
20103
|
+
deliveryKey: terminal.deliveryKey,
|
|
20104
|
+
})
|
|
20105
|
+
: buildRunLedgerEventsIdempotencyKey({
|
|
20106
|
+
runId: first.runId,
|
|
20107
|
+
deliveryDigest: batchDeliveryKey,
|
|
20108
|
+
}),
|
|
20109
|
+
});
|
|
20110
|
+
} else {
|
|
20111
|
+
await renewDelivery();
|
|
20112
|
+
}
|
|
19586
20113
|
}
|
|
20114
|
+
};
|
|
20115
|
+
try {
|
|
20116
|
+
await append();
|
|
20117
|
+
} catch (error) {
|
|
20118
|
+
// A cached authority can be invalidated by signer rotation. Refresh it
|
|
20119
|
+
// once through this delivery's active lease; the append remains fenced
|
|
20120
|
+
// and carries the same idempotency key on both attempts.
|
|
20121
|
+
if (
|
|
20122
|
+
!gatewayRuntimeApi?.refreshExecutorToken ||
|
|
20123
|
+
!(error instanceof AppRuntimeApiResponseError) ||
|
|
20124
|
+
error.status !== 401
|
|
20125
|
+
)
|
|
20126
|
+
throw error;
|
|
20127
|
+
runtimeApi.executorToken = await gatewayRuntimeApi.refreshExecutorToken();
|
|
20128
|
+
await append();
|
|
19587
20129
|
}
|
|
19588
20130
|
appendFinishedAtMs = Date.now();
|
|
19589
20131
|
const acknowledgement = await acknowledgeRunLedgerPrefixWithContinuation(
|
|
19590
20132
|
withClient,
|
|
19591
|
-
{
|
|
20133
|
+
{
|
|
20134
|
+
input,
|
|
20135
|
+
events,
|
|
20136
|
+
leaseOwner,
|
|
20137
|
+
runtimeApi,
|
|
20138
|
+
scope,
|
|
20139
|
+
continuation,
|
|
20140
|
+
continuationAllowance:
|
|
20141
|
+
remainingEvents === null
|
|
20142
|
+
? null
|
|
20143
|
+
: Math.max(0, remainingEvents - events.length),
|
|
20144
|
+
},
|
|
19592
20145
|
options,
|
|
19593
20146
|
);
|
|
19594
20147
|
const { deliveredRows } = acknowledgement;
|
|
@@ -19630,6 +20183,10 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19630
20183
|
authorityScope,
|
|
19631
20184
|
);
|
|
19632
20185
|
} catch (error) {
|
|
20186
|
+
// A failed append must reacquire receiver truth on its next attempt,
|
|
20187
|
+
// including an older receiver that rejected a frozen compound receipt.
|
|
20188
|
+
authorityScope.authority?.runtimeApi.invalidateExecutorToken?.();
|
|
20189
|
+
authorityScope.authority = null;
|
|
19633
20190
|
await releasePendingContinuation();
|
|
19634
20191
|
if (input.signal?.aborted) {
|
|
19635
20192
|
// The caller's latency budget expired; this is not a delivery failure.
|
|
@@ -19654,20 +20211,87 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19654
20211
|
retryingEventIds: released,
|
|
19655
20212
|
};
|
|
19656
20213
|
}
|
|
19657
|
-
|
|
19658
|
-
const
|
|
19659
|
-
|
|
19660
|
-
|
|
19661
|
-
|
|
20214
|
+
if (error instanceof DeferredRuntimeProjectionInputError) {
|
|
20215
|
+
const outcome = await withClient((client) =>
|
|
20216
|
+
settlePostgresSchedulerOutboxFailure(client, error.eventId, error, {
|
|
20217
|
+
...options,
|
|
20218
|
+
leaseOwner,
|
|
20219
|
+
}),
|
|
20220
|
+
);
|
|
20221
|
+
// Earlier facts may already be indexed. Only the invalid immutable row
|
|
20222
|
+
// is quarantined; valid predecessors remain deliverable in causal order.
|
|
20223
|
+
const released = await withClient((client) =>
|
|
20224
|
+
releasePostgresSchedulerOutboxLease(
|
|
19662
20225
|
client,
|
|
19663
|
-
|
|
19664
|
-
|
|
20226
|
+
events
|
|
20227
|
+
.filter((event) => event.eventId !== error.eventId)
|
|
20228
|
+
.map((event) => event.eventId),
|
|
19665
20229
|
{ ...options, leaseOwner },
|
|
19666
|
-
)
|
|
19667
|
-
|
|
19668
|
-
|
|
20230
|
+
),
|
|
20231
|
+
);
|
|
20232
|
+
const failed = outcome === 'blocked' ? [error.eventId] : [];
|
|
20233
|
+
const retrying =
|
|
20234
|
+
outcome === 'blocked' ? released : [...released, error.eventId];
|
|
20235
|
+
console.error('[run-ledger-outbox.invalid_projection_input]', {
|
|
20236
|
+
eventId: error.eventId,
|
|
20237
|
+
code: error.code,
|
|
20238
|
+
detail: error.message,
|
|
20239
|
+
});
|
|
20240
|
+
return {
|
|
20241
|
+
attempted: events.length,
|
|
20242
|
+
projected: 0,
|
|
20243
|
+
eventIds: [],
|
|
20244
|
+
failed: failed.length,
|
|
20245
|
+
failedEventIds: failed,
|
|
20246
|
+
retrying: retrying.length,
|
|
20247
|
+
retryingEventIds: retrying,
|
|
20248
|
+
};
|
|
20249
|
+
}
|
|
20250
|
+
const settlement = await withClient(async (client) => {
|
|
20251
|
+
await client.query('BEGIN');
|
|
20252
|
+
try {
|
|
20253
|
+
const blocked: string[] = [];
|
|
20254
|
+
const retrying: string[] = [];
|
|
20255
|
+
const ownedRetrying: string[] = [];
|
|
20256
|
+
for (const event of events) {
|
|
20257
|
+
const outcome = await settlePostgresSchedulerOutboxFailure(
|
|
20258
|
+
client,
|
|
20259
|
+
event.eventId,
|
|
20260
|
+
error,
|
|
20261
|
+
{ ...options, leaseOwner },
|
|
20262
|
+
);
|
|
20263
|
+
if (outcome === 'blocked') blocked.push(event.eventId);
|
|
20264
|
+
else retrying.push(event.eventId);
|
|
20265
|
+
if (outcome === 'retrying') ownedRetrying.push(event.eventId);
|
|
20266
|
+
}
|
|
20267
|
+
// All members of this failed append must become due together. Keep
|
|
20268
|
+
// the existing per-row retry policy, taking its latest deadline only
|
|
20269
|
+
// for rows this owner actually settled, before releasing their locks.
|
|
20270
|
+
if (ownedRetrying.length > 1) {
|
|
20271
|
+
await client.query(
|
|
20272
|
+
`
|
|
20273
|
+
WITH retry AS (
|
|
20274
|
+
SELECT max(next_attempt_at) AS retry_at
|
|
20275
|
+
FROM ${tableName('outbox', options)}
|
|
20276
|
+
WHERE event_id = ANY($1::text[])
|
|
20277
|
+
AND delivered_at IS NULL AND failed_at IS NULL
|
|
20278
|
+
)
|
|
20279
|
+
UPDATE ${tableName('outbox', options)} AS outbox
|
|
20280
|
+
SET next_attempt_at = retry.retry_at
|
|
20281
|
+
FROM retry
|
|
20282
|
+
WHERE outbox.event_id = ANY($1::text[])
|
|
20283
|
+
AND outbox.delivered_at IS NULL AND outbox.failed_at IS NULL
|
|
20284
|
+
AND retry.retry_at IS NOT NULL
|
|
20285
|
+
`,
|
|
20286
|
+
[ownedRetrying],
|
|
20287
|
+
);
|
|
20288
|
+
}
|
|
20289
|
+
await client.query('COMMIT');
|
|
20290
|
+
return { blocked, retrying };
|
|
20291
|
+
} catch (settlementError) {
|
|
20292
|
+
await client.query('ROLLBACK');
|
|
20293
|
+
throw settlementError;
|
|
19669
20294
|
}
|
|
19670
|
-
return { blocked, retrying };
|
|
19671
20295
|
});
|
|
19672
20296
|
console.warn('[run-ledger-outbox.batch_dispatch_failed]', {
|
|
19673
20297
|
runId: input.runId ?? null,
|
|
@@ -19784,6 +20408,7 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
|
|
|
19784
20408
|
runtimeApi: WorkerRuntimeApiContext;
|
|
19785
20409
|
scope?: RunLedgerOutboxDrainScope;
|
|
19786
20410
|
continuation: { current: RunLedgerOutboxDrainScope | null };
|
|
20411
|
+
continuationAllowance: number | null;
|
|
19787
20412
|
},
|
|
19788
20413
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
19789
20414
|
): Promise<{
|
|
@@ -19824,6 +20449,7 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
|
|
|
19824
20449
|
let continuation: RunLedgerOutboxDrainScope | null = null;
|
|
19825
20450
|
if (
|
|
19826
20451
|
retainContinuation &&
|
|
20452
|
+
input.continuationAllowance !== 0 &&
|
|
19827
20453
|
deliveredRows.length === input.events.length &&
|
|
19828
20454
|
!input.events.some((event) => event.eventId === targetEventId)
|
|
19829
20455
|
) {
|
|
@@ -19841,8 +20467,14 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
|
|
|
19841
20467
|
leaseSeconds: input.input.leaseSeconds ?? 120,
|
|
19842
20468
|
limit: Math.min(
|
|
19843
20469
|
RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
|
|
19844
|
-
input.
|
|
20470
|
+
input.runtimeApi.runLedgerIngressPrefixBatching === true
|
|
20471
|
+
? RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS
|
|
20472
|
+
: (input.input.limit ?? 1),
|
|
20473
|
+
input.continuationAllowance ??
|
|
20474
|
+
RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
|
|
19845
20475
|
),
|
|
20476
|
+
allowIngressPrefix:
|
|
20477
|
+
input.runtimeApi.runLedgerIngressPrefixBatching === true,
|
|
19846
20478
|
inTransaction: true,
|
|
19847
20479
|
},
|
|
19848
20480
|
options,
|
|
@@ -20019,38 +20651,7 @@ async function projectPostgresSchedulerStartEvent(
|
|
|
20019
20651
|
});
|
|
20020
20652
|
}
|
|
20021
20653
|
|
|
20022
|
-
export type RunLedgerDispatchLaunch
|
|
20023
|
-
PlaySchedulerSubmitInput,
|
|
20024
|
-
| 'runId'
|
|
20025
|
-
| 'playId'
|
|
20026
|
-
| 'playName'
|
|
20027
|
-
| 'executorToken'
|
|
20028
|
-
| 'baseUrl'
|
|
20029
|
-
| 'runtimeApiBaseUrl'
|
|
20030
|
-
| 'runtimeTokenOrigin'
|
|
20031
|
-
| 'orgId'
|
|
20032
|
-
| 'runtimeAuthority'
|
|
20033
|
-
| 'vercelProtectionBypassToken'
|
|
20034
|
-
>;
|
|
20035
|
-
|
|
20036
|
-
/** Authority reads never transfer customer input, artifacts or execution graphs. */
|
|
20037
|
-
function runLedgerDispatchLaunchSql(alias: string): string {
|
|
20038
|
-
const fields: (keyof RunLedgerDispatchLaunch)[] = [
|
|
20039
|
-
'runId',
|
|
20040
|
-
'playId',
|
|
20041
|
-
'playName',
|
|
20042
|
-
'executorToken',
|
|
20043
|
-
'baseUrl',
|
|
20044
|
-
'runtimeApiBaseUrl',
|
|
20045
|
-
'runtimeTokenOrigin',
|
|
20046
|
-
'orgId',
|
|
20047
|
-
'runtimeAuthority',
|
|
20048
|
-
'vercelProtectionBypassToken',
|
|
20049
|
-
];
|
|
20050
|
-
return `jsonb_build_object(${fields
|
|
20051
|
-
.map((field) => `'${field}', ${alias}.launch_json->'${field}'`)
|
|
20052
|
-
.join(', ')})`;
|
|
20053
|
-
}
|
|
20654
|
+
export type { RunLedgerDispatchLaunch } from './run-ledger-dispatch-authority';
|
|
20054
20655
|
|
|
20055
20656
|
async function loadPostgresSchedulerRuntimeApiForRun(
|
|
20056
20657
|
withClient: <T>(
|
|
@@ -20061,28 +20662,37 @@ async function loadPostgresSchedulerRuntimeApiForRun(
|
|
|
20061
20662
|
tokenAuthority: RunLedgerOutboxTokenAuthority,
|
|
20062
20663
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
20063
20664
|
signal?: AbortSignal,
|
|
20064
|
-
|
|
20665
|
+
claimedLaunch?: RunLedgerDispatchLaunch,
|
|
20666
|
+
): Promise<
|
|
20667
|
+
WorkerRuntimeApiContext & {
|
|
20668
|
+
refreshExecutorToken?: () => Promise<string>;
|
|
20669
|
+
invalidateExecutorToken?: () => void;
|
|
20670
|
+
}
|
|
20671
|
+
> {
|
|
20065
20672
|
// Release the scarce scheduler checkout before the token authority hop.
|
|
20066
20673
|
// A slow gateway must not retain either projector pool client, and its
|
|
20067
20674
|
// launch context must not hydrate customer execution graphs.
|
|
20068
|
-
const
|
|
20069
|
-
|
|
20070
|
-
|
|
20071
|
-
|
|
20072
|
-
|
|
20073
|
-
|
|
20074
|
-
|
|
20075
|
-
|
|
20076
|
-
|
|
20077
|
-
|
|
20078
|
-
|
|
20079
|
-
|
|
20675
|
+
const launch: RunLedgerDispatchLaunch =
|
|
20676
|
+
claimedLaunch ??
|
|
20677
|
+
(await withClient(async (client) => {
|
|
20678
|
+
const result = await client.query<{ launch_json: unknown }>(
|
|
20679
|
+
`SELECT ${runLedgerDispatchLaunchSql('runs')} AS launch_json
|
|
20680
|
+
FROM ${tableName('runs', options)} AS runs
|
|
20681
|
+
WHERE run_id = $1
|
|
20682
|
+
LIMIT 1`,
|
|
20683
|
+
[runId],
|
|
20684
|
+
);
|
|
20685
|
+
return parseLaunchJson(result.rows[0]?.launch_json);
|
|
20686
|
+
}));
|
|
20080
20687
|
if (!launch?.baseUrl) {
|
|
20081
20688
|
throw new Error(
|
|
20082
20689
|
`Run-ledger outbox dispatcher cannot resolve runtime API context for ${runId}.`,
|
|
20083
20690
|
);
|
|
20084
20691
|
}
|
|
20085
20692
|
let executorToken: string;
|
|
20693
|
+
let runLedgerIngressPrefixBatching: true | undefined;
|
|
20694
|
+
let refreshExecutorToken: (() => Promise<string>) | undefined;
|
|
20695
|
+
let invalidateExecutorToken: (() => void) | undefined;
|
|
20086
20696
|
if (tokenAuthority === 'local_mint') {
|
|
20087
20697
|
// App-side authoritative projector: sign in-process. NEVER reached on the
|
|
20088
20698
|
// Fly worker (it passes `gateway`), so this is the only place `BETTER_AUTH_
|
|
@@ -20101,34 +20711,89 @@ async function loadPostgresSchedulerRuntimeApiForRun(
|
|
|
20101
20711
|
`Run-ledger outbox gateway dispatch for ${runId} requires an active outbox lease.`,
|
|
20102
20712
|
);
|
|
20103
20713
|
}
|
|
20104
|
-
|
|
20714
|
+
const dispatchRequest = {
|
|
20105
20715
|
launch,
|
|
20106
20716
|
eventId: dispatchLease.eventId,
|
|
20107
20717
|
leaseOwner: dispatchLease.leaseOwner,
|
|
20108
20718
|
schedulerSchema: options?.schema ?? null,
|
|
20109
20719
|
signal,
|
|
20110
|
-
}
|
|
20720
|
+
};
|
|
20721
|
+
const dispatchAuthority =
|
|
20722
|
+
await requestRunLedgerDispatchToken(dispatchRequest);
|
|
20723
|
+
invalidateExecutorToken = () =>
|
|
20724
|
+
runLedgerDispatchTokens.delete(dispatchAuthority.cacheKey);
|
|
20725
|
+
if (dispatchAuthority.reused) {
|
|
20726
|
+
refreshExecutorToken = async () =>
|
|
20727
|
+
(await requestRunLedgerDispatchToken(dispatchRequest, true))
|
|
20728
|
+
.executorToken;
|
|
20729
|
+
}
|
|
20730
|
+
executorToken = dispatchAuthority.executorToken;
|
|
20731
|
+
runLedgerIngressPrefixBatching =
|
|
20732
|
+
dispatchAuthority.runLedgerIngressPrefixBatching;
|
|
20111
20733
|
}
|
|
20112
20734
|
return {
|
|
20113
20735
|
baseUrl: runtimeApiBaseUrlFromLaunch(launch),
|
|
20114
20736
|
executorToken,
|
|
20737
|
+
...(runLedgerIngressPrefixBatching
|
|
20738
|
+
? { runLedgerIngressPrefixBatching }
|
|
20739
|
+
: {}),
|
|
20115
20740
|
vercelProtectionBypassToken: launch.vercelProtectionBypassToken ?? null,
|
|
20741
|
+
...(refreshExecutorToken ? { refreshExecutorToken } : {}),
|
|
20742
|
+
...(invalidateExecutorToken ? { invalidateExecutorToken } : {}),
|
|
20116
20743
|
};
|
|
20117
20744
|
}
|
|
20118
20745
|
|
|
20119
|
-
|
|
20120
|
-
|
|
20121
|
-
|
|
20122
|
-
|
|
20123
|
-
|
|
20124
|
-
|
|
20125
|
-
|
|
20746
|
+
// Only freshly gateway-issued legacy run.progress authority is reused.
|
|
20747
|
+
// Prefix advertisements are reacquired for each drain because they select its
|
|
20748
|
+
// receipt shape, including recovery across receiver capability changes. This is not
|
|
20749
|
+
// a lease cache: every delivery still claims, renews, and proves its own lease
|
|
20750
|
+
// at append. Include the complete pinned authority/routing and internal-token
|
|
20751
|
+
// fingerprint so a different run, schema, origin, or authority never shares it.
|
|
20752
|
+
const RUN_LEDGER_DISPATCH_TOKEN_CACHE_MS = 30_000;
|
|
20753
|
+
const RUN_LEDGER_DISPATCH_TOKEN_CACHE_ENTRIES = 256;
|
|
20754
|
+
const runLedgerDispatchTokens = new Map<
|
|
20755
|
+
string,
|
|
20756
|
+
{ token: string; expiresAt: number; runLedgerIngressPrefixBatching?: true }
|
|
20757
|
+
>();
|
|
20758
|
+
|
|
20759
|
+
async function requestRunLedgerDispatchToken(
|
|
20760
|
+
input: {
|
|
20761
|
+
launch: RunLedgerDispatchLaunch;
|
|
20762
|
+
eventId: string;
|
|
20763
|
+
leaseOwner: string;
|
|
20764
|
+
schedulerSchema: string | null;
|
|
20765
|
+
signal?: AbortSignal;
|
|
20766
|
+
},
|
|
20767
|
+
forceFresh = false,
|
|
20768
|
+
): Promise<{
|
|
20769
|
+
executorToken: string;
|
|
20770
|
+
reused: boolean;
|
|
20771
|
+
cacheKey: string;
|
|
20772
|
+
runLedgerIngressPrefixBatching?: true;
|
|
20773
|
+
}> {
|
|
20126
20774
|
const internalToken = process.env.DEEPLINE_INTERNAL_TOKEN?.trim();
|
|
20127
20775
|
if (!internalToken || !input.eventId || !input.leaseOwner) {
|
|
20128
20776
|
throw new Error(
|
|
20129
20777
|
'Run-ledger outbox recovery requires a valid dispatch lease and DEEPLINE_INTERNAL_TOKEN.',
|
|
20130
20778
|
);
|
|
20131
20779
|
}
|
|
20780
|
+
input.signal?.throwIfAborted();
|
|
20781
|
+
const { executorToken: _storedToken, ...authority } = input.launch;
|
|
20782
|
+
const cacheKey = createHash('sha256')
|
|
20783
|
+
.update(JSON.stringify([input.schedulerSchema, authority, internalToken]))
|
|
20784
|
+
.digest('hex');
|
|
20785
|
+
const now = Date.now();
|
|
20786
|
+
for (const [key, entry] of runLedgerDispatchTokens) {
|
|
20787
|
+
if (entry.expiresAt <= now) runLedgerDispatchTokens.delete(key);
|
|
20788
|
+
}
|
|
20789
|
+
const cached = runLedgerDispatchTokens.get(cacheKey);
|
|
20790
|
+
if (!forceFresh && cached)
|
|
20791
|
+
return {
|
|
20792
|
+
executorToken: cached.token,
|
|
20793
|
+
reused: true,
|
|
20794
|
+
cacheKey,
|
|
20795
|
+
};
|
|
20796
|
+
runLedgerDispatchTokens.delete(cacheKey);
|
|
20132
20797
|
const url = `${runtimeApiBaseUrlFromLaunch(input.launch).replace(/\/$/, '')}/api/v2/internal/play-runtime/run-ledger-dispatch-token`;
|
|
20133
20798
|
const vercelProtectionHeaders = await vercelProtectionBypassHeaders({
|
|
20134
20799
|
baseUrl: runtimeApiBaseUrlFromLaunch(input.launch),
|
|
@@ -20164,6 +20829,7 @@ async function requestRunLedgerDispatchToken(input: {
|
|
|
20164
20829
|
try {
|
|
20165
20830
|
return JSON.parse(responseText) as {
|
|
20166
20831
|
executorToken?: unknown;
|
|
20832
|
+
runLedgerIngressPrefixBatching?: unknown;
|
|
20167
20833
|
error?: unknown;
|
|
20168
20834
|
code?: unknown;
|
|
20169
20835
|
retryable?: unknown;
|
|
@@ -20196,7 +20862,53 @@ async function requestRunLedgerDispatchToken(input: {
|
|
|
20196
20862
|
: 'invalid response',
|
|
20197
20863
|
});
|
|
20198
20864
|
}
|
|
20199
|
-
|
|
20865
|
+
// Decode only the lifetime, never an authentication decision: the app
|
|
20866
|
+
// validates the signature, run scope, and live delivery lease on append.
|
|
20867
|
+
// Opaque/older token formats continue to work without this optimization.
|
|
20868
|
+
try {
|
|
20869
|
+
const claims = JSON.parse(
|
|
20870
|
+
Buffer.from(
|
|
20871
|
+
body.executorToken.split('.')[1] ?? '',
|
|
20872
|
+
'base64url',
|
|
20873
|
+
).toString(),
|
|
20874
|
+
) as { exp?: unknown };
|
|
20875
|
+
if (
|
|
20876
|
+
body.runLedgerIngressPrefixBatching === undefined &&
|
|
20877
|
+
typeof claims.exp === 'number' &&
|
|
20878
|
+
Number.isFinite(claims.exp)
|
|
20879
|
+
) {
|
|
20880
|
+
const expiresAt = Math.min(
|
|
20881
|
+
Date.now() + RUN_LEDGER_DISPATCH_TOKEN_CACHE_MS,
|
|
20882
|
+
claims.exp * 1_000 - 5_000,
|
|
20883
|
+
);
|
|
20884
|
+
if (expiresAt > Date.now()) {
|
|
20885
|
+
runLedgerDispatchTokens.set(cacheKey, {
|
|
20886
|
+
token: body.executorToken,
|
|
20887
|
+
expiresAt,
|
|
20888
|
+
});
|
|
20889
|
+
while (
|
|
20890
|
+
runLedgerDispatchTokens.size > RUN_LEDGER_DISPATCH_TOKEN_CACHE_ENTRIES
|
|
20891
|
+
) {
|
|
20892
|
+
runLedgerDispatchTokens.delete(
|
|
20893
|
+
runLedgerDispatchTokens.keys().next().value!,
|
|
20894
|
+
);
|
|
20895
|
+
}
|
|
20896
|
+
}
|
|
20897
|
+
}
|
|
20898
|
+
} catch {
|
|
20899
|
+
// A cache miss never changes delivery or hides a gateway rejection.
|
|
20900
|
+
}
|
|
20901
|
+
if (body.runLedgerIngressPrefixBatching !== undefined) {
|
|
20902
|
+
runLedgerDispatchTokens.delete(cacheKey);
|
|
20903
|
+
}
|
|
20904
|
+
return {
|
|
20905
|
+
executorToken: body.executorToken,
|
|
20906
|
+
reused: false,
|
|
20907
|
+
cacheKey,
|
|
20908
|
+
...(body.runLedgerIngressPrefixBatching === true
|
|
20909
|
+
? { runLedgerIngressPrefixBatching: true as const }
|
|
20910
|
+
: {}),
|
|
20911
|
+
};
|
|
20200
20912
|
}
|
|
20201
20913
|
|
|
20202
20914
|
export async function requestComputeBillingDispatchToken(input: {
|
|
@@ -20272,27 +20984,7 @@ export async function requestComputeBillingDispatchToken(input: {
|
|
|
20272
20984
|
return body.executorToken;
|
|
20273
20985
|
}
|
|
20274
20986
|
|
|
20275
|
-
|
|
20276
|
-
export async function readRunLedgerDispatchAuthorityForLease(
|
|
20277
|
-
client: PostgresSchedulerQueryClient,
|
|
20278
|
-
input: { runId: string; eventId: string; leaseOwner: string },
|
|
20279
|
-
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
20280
|
-
): Promise<RunLedgerDispatchLaunch | null> {
|
|
20281
|
-
const result = await client.query<{ launch_json: unknown }>(
|
|
20282
|
-
`SELECT ${runLedgerDispatchLaunchSql('runs')} AS launch_json
|
|
20283
|
-
FROM ${tableName('outbox', options)} outbox
|
|
20284
|
-
JOIN ${tableName('runs', options)} runs ON runs.run_id = outbox.run_id
|
|
20285
|
-
WHERE outbox.event_id = $1
|
|
20286
|
-
AND outbox.run_id = $2
|
|
20287
|
-
AND outbox.lease_owner = $3
|
|
20288
|
-
AND outbox.lease_expires_at > now()
|
|
20289
|
-
AND outbox.delivered_at IS NULL
|
|
20290
|
-
AND outbox.failed_at IS NULL
|
|
20291
|
-
LIMIT 1`,
|
|
20292
|
-
[input.eventId, input.runId, input.leaseOwner],
|
|
20293
|
-
);
|
|
20294
|
-
return result.rows[0] ? parseLaunchJson(result.rows[0].launch_json) : null;
|
|
20295
|
-
}
|
|
20987
|
+
export { readRunLedgerDispatchAuthorityForLease } from './run-ledger-dispatch-authority';
|
|
20296
20988
|
|
|
20297
20989
|
export async function readRunLedgerDispatchLaunchForLease(
|
|
20298
20990
|
client: PostgresSchedulerQueryClient,
|
|
@@ -20415,22 +21107,10 @@ function sourcePayloadField(
|
|
|
20415
21107
|
}
|
|
20416
21108
|
|
|
20417
21109
|
function parseLaunchJson(value: unknown): PlaySchedulerSubmitInput {
|
|
20418
|
-
|
|
20419
|
-
|
|
20420
|
-
|
|
20421
|
-
|
|
20422
|
-
if (
|
|
20423
|
-
typeof launch.runId !== 'string' ||
|
|
20424
|
-
typeof launch.playId !== 'string' ||
|
|
20425
|
-
typeof launch.playName !== 'string' ||
|
|
20426
|
-
typeof launch.executorToken !== 'string' ||
|
|
20427
|
-
typeof launch.baseUrl !== 'string'
|
|
20428
|
-
) {
|
|
20429
|
-
throw new Error(
|
|
20430
|
-
'Postgres scheduler launch_json is missing required fields.',
|
|
20431
|
-
);
|
|
20432
|
-
}
|
|
20433
|
-
return hydrateRetainedPlayPipelines(launch as PlaySchedulerSubmitInput);
|
|
21110
|
+
const launch = parseRunLedgerDispatchLaunch(
|
|
21111
|
+
value,
|
|
21112
|
+
) as PlaySchedulerSubmitInput;
|
|
21113
|
+
return hydrateRetainedPlayPipelines(launch);
|
|
20434
21114
|
}
|
|
20435
21115
|
|
|
20436
21116
|
function parsePostgresSchedulerCronSourcePin(
|