deepline 0.3.289 → 0.3.291
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +8 -6
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/runs/overview.ts +16 -6
- package/dist/bundling-sources/sdk/src/types.ts +21 -2
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +41 -11
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +317 -63
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +10 -2
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/launch-identity.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/log-context.ts +36 -0
- package/dist/bundling-sources/shared_libs/play-runtime/log-provenance.ts +19 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/worker-terminal-projection.ts +6 -5
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +12 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-event-catalog.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +74 -33
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +73 -3
- package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +39 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +72 -9
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +2 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +77 -10
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool-options.ts +145 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +17 -9
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -139
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +19 -5
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +176 -18
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +7 -5
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +1404 -478
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +55 -25
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-dispatch-authority.ts +97 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +34 -15
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +558 -57
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +10 -1
- package/dist/bundling-sources/shared_libs/play-runtime/step-progress.ts +6 -1
- package/dist/bundling-sources/shared_libs/play-runtime/test-admission-batching.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +22 -5
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +35 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +68 -8
- package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/zerobounce-batching.ts +75 -28
- package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +2 -0
- package/dist/cli/index.js +195 -582
- package/dist/cli/index.mjs +195 -582
- package/dist/{client-FR_u0dxT.d.ts → client-Ct4jmvAg.d.ts} +29 -8
- package/dist/{client-phK1Ixcl.d.mts → client-Ct9GKrwu.d.mts} +29 -8
- package/dist/{compiler-manifest-9SQtUuKb.d.mts → compiler-manifest-qtTNYvBl.d.mts} +2 -0
- package/dist/{compiler-manifest-9SQtUuKb.d.ts → compiler-manifest-qtTNYvBl.d.ts} +2 -0
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +14 -5
- package/dist/index.mjs +14 -5
- package/dist/install-integrity.json +7 -4
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/run-request.d.mts +2 -2
- package/dist/plays/run-request.d.ts +2 -2
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
import {
|
|
2
|
+
parseRunLedgerDispatchLaunch,
|
|
3
|
+
runLedgerDispatchLaunchSql,
|
|
4
|
+
type RunLedgerDispatchLaunch,
|
|
5
|
+
} from './run-ledger-dispatch-authority';
|
|
6
|
+
import { normalizeRunLogIds } from '../run-log-buffer';
|
|
1
7
|
import {
|
|
2
8
|
retainedPlayPipelineFields,
|
|
3
9
|
hydrateRetainedPlayPipelines,
|
|
@@ -28,6 +34,8 @@ import type {
|
|
|
28
34
|
import { PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA } from '../runtime-scheduler-topology';
|
|
29
35
|
import {
|
|
30
36
|
RUNTIME_EVENT_CANONICAL_RETENTION_INDEX_READY_MARKER,
|
|
37
|
+
RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD,
|
|
38
|
+
RUNTIME_EVENT_SEARCH_ONLY_FIELD,
|
|
31
39
|
RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER,
|
|
32
40
|
} from './runtime-event-search';
|
|
33
41
|
import type { PlayRunnerRuntimeResource } from '../runner-backends/types';
|
|
@@ -83,6 +91,8 @@ import {
|
|
|
83
91
|
import { normalizePlayDocflowNodeIoState } from '../docflow-node-io';
|
|
84
92
|
import {
|
|
85
93
|
normalizePlayLogContext,
|
|
94
|
+
normalizePlayLogMetadata,
|
|
95
|
+
type PlayLogMetadata,
|
|
86
96
|
type PlayLogContext,
|
|
87
97
|
} from '../log-context';
|
|
88
98
|
import {
|
|
@@ -105,7 +115,6 @@ import {
|
|
|
105
115
|
RUN_SCHEDULER_OUTBOX_START_EVENT_TYPES,
|
|
106
116
|
RUN_SCHEDULER_OUTBOX_EVENT_TYPE,
|
|
107
117
|
RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL,
|
|
108
|
-
RUN_SCHEDULER_OUTBOX_PREFIX_STOP_EVENT_TYPES,
|
|
109
118
|
RUN_SCHEDULER_OUTBOX_TERMINAL_EVENT_TYPES,
|
|
110
119
|
} from '../run-event-catalog';
|
|
111
120
|
import { stableStringify } from '../../plays/row-identity';
|
|
@@ -336,7 +345,10 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
|
|
|
336
345
|
// recovery, which scans only active runs for each release lane.
|
|
337
346
|
// v57 refreshes log-position lookups after the operator concurrently prepares
|
|
338
347
|
// channel-specific indexes; outgoing readers retain the existing GIN index.
|
|
339
|
-
|
|
348
|
+
// v58 installs deferred outbox readers while retaining synchronous writers
|
|
349
|
+
// until the separately operated, reader-first cutover.
|
|
350
|
+
// v59 adds origin identity bulk projection and independent durable search progress.
|
|
351
|
+
export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 59;
|
|
340
352
|
const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
|
|
341
353
|
'scheduler_schema_migrations';
|
|
342
354
|
export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
|
|
@@ -471,11 +483,9 @@ type PostgresSchedulerClaimExecutionOptions = Pick<
|
|
|
471
483
|
/** Internal provider-read coordinates which must survive terminal-worker
|
|
472
484
|
* loss. They are persisted only on the matching durable cleanup obligation. */
|
|
473
485
|
sandboxCrashDiagnostic?: PostgresSchedulerSandboxCrashDiagnosticRequest;
|
|
474
|
-
/**
|
|
475
|
-
*
|
|
476
|
-
|
|
477
|
-
* settles them before the run's terminal makes its steps immutable. */
|
|
478
|
-
failedTerminalProgressEvents?: readonly unknown[];
|
|
486
|
+
/** Final step facts folded into the terminal outbox instead of a separate
|
|
487
|
+
* progress write immediately ahead of the terminal. */
|
|
488
|
+
terminalProgressEvents?: readonly unknown[];
|
|
479
489
|
/** Forwarded to `failPostgresSchedulerAttempt`: a failure terminal written
|
|
480
490
|
* for a detached-runner attempt must not overwrite an authoritative pushed
|
|
481
491
|
* runner terminal that won after the attempt was claimed. */
|
|
@@ -1928,6 +1938,24 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
|
1928
1938
|
AND index_row.indrelid = to_regclass($3)
|
|
1929
1939
|
AND index_row.indisready AND index_row.indisvalid
|
|
1930
1940
|
)
|
|
1941
|
+
AND EXISTS (
|
|
1942
|
+
SELECT 1 FROM pg_index AS index_row
|
|
1943
|
+
JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
|
|
1944
|
+
JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
|
|
1945
|
+
WHERE index_namespace.nspname = $1
|
|
1946
|
+
AND index_class.relname = 'runtime_scheduler_outbox_lifecycle_pending_order_idx'
|
|
1947
|
+
AND index_row.indrelid = to_regclass($3)
|
|
1948
|
+
AND index_row.indisready AND index_row.indisvalid
|
|
1949
|
+
)
|
|
1950
|
+
AND EXISTS (
|
|
1951
|
+
SELECT 1 FROM pg_index AS index_row
|
|
1952
|
+
JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
|
|
1953
|
+
JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
|
|
1954
|
+
WHERE index_namespace.nspname = $1
|
|
1955
|
+
AND index_class.relname = 'runtime_scheduler_outbox_search_pending_order_idx'
|
|
1956
|
+
AND index_row.indrelid = to_regclass($3)
|
|
1957
|
+
AND index_row.indisready AND index_row.indisvalid
|
|
1958
|
+
)
|
|
1931
1959
|
AND EXISTS (
|
|
1932
1960
|
SELECT 1 FROM pg_index AS index_row
|
|
1933
1961
|
JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
|
|
@@ -2461,6 +2489,7 @@ export async function ensurePostgresSchedulerLaunchAdmissionFunctions(
|
|
|
2461
2489
|
jsonb_set(
|
|
2462
2490
|
launch_json - ARRAY[
|
|
2463
2491
|
'launchIdentityDigest',
|
|
2492
|
+
'runtimeLogIdentity',
|
|
2464
2493
|
'executorToken',
|
|
2465
2494
|
'inputFileId',
|
|
2466
2495
|
'preloadedDbSessionUnwrapKey',
|
|
@@ -3944,6 +3973,26 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
3944
3973
|
)
|
|
3945
3974
|
WHERE delivered_at IS NULL
|
|
3946
3975
|
`);
|
|
3976
|
+
await client.query(`
|
|
3977
|
+
CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_lifecycle_pending_order_idx
|
|
3978
|
+
ON ${tableName('outbox', options)} (
|
|
3979
|
+
run_id,
|
|
3980
|
+
(CASE event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END),
|
|
3981
|
+
created_at,
|
|
3982
|
+
event_id
|
|
3983
|
+
)
|
|
3984
|
+
WHERE delivered_at IS NULL AND ${searchLaneSql('outbox')} = false
|
|
3985
|
+
`);
|
|
3986
|
+
await client.query(`
|
|
3987
|
+
CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_search_pending_order_idx
|
|
3988
|
+
ON ${tableName('outbox', options)} (
|
|
3989
|
+
run_id,
|
|
3990
|
+
(CASE event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END),
|
|
3991
|
+
created_at,
|
|
3992
|
+
event_id
|
|
3993
|
+
)
|
|
3994
|
+
WHERE delivered_at IS NULL AND ${searchLaneSql('outbox')} = true
|
|
3995
|
+
`);
|
|
3947
3996
|
await client.query(`
|
|
3948
3997
|
CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_lease_health_idx
|
|
3949
3998
|
ON ${tableName('outbox', options)}
|
|
@@ -4094,6 +4143,22 @@ const POSTGRES_SCHEDULER_RUNTIME_ADMIN_INDEXES = [
|
|
|
4094
4143
|
created_at, event_id)
|
|
4095
4144
|
WHERE delivered_at IS NULL`,
|
|
4096
4145
|
},
|
|
4146
|
+
{
|
|
4147
|
+
name: 'runtime_scheduler_outbox_lifecycle_pending_order_idx',
|
|
4148
|
+
table: 'outbox',
|
|
4149
|
+
definition: `(run_id,
|
|
4150
|
+
(CASE event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END),
|
|
4151
|
+
created_at, event_id)
|
|
4152
|
+
WHERE delivered_at IS NULL AND ${searchLaneSql('outbox')} = false`,
|
|
4153
|
+
},
|
|
4154
|
+
{
|
|
4155
|
+
name: 'runtime_scheduler_outbox_search_pending_order_idx',
|
|
4156
|
+
table: 'outbox',
|
|
4157
|
+
definition: `(run_id,
|
|
4158
|
+
(CASE event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END),
|
|
4159
|
+
created_at, event_id)
|
|
4160
|
+
WHERE delivered_at IS NULL AND ${searchLaneSql('outbox')} = true`,
|
|
4161
|
+
},
|
|
4097
4162
|
{
|
|
4098
4163
|
name: 'runtime_scheduler_outbox_lease_health_idx',
|
|
4099
4164
|
table: 'outbox',
|
|
@@ -10318,8 +10383,8 @@ export async function recordPostgresSchedulerRunnerTerminal(
|
|
|
10318
10383
|
const recorded = await client.query<{ run_id: string }>(
|
|
10319
10384
|
`
|
|
10320
10385
|
WITH accepted AS MATERIALIZED (
|
|
10321
|
-
--
|
|
10322
|
-
--
|
|
10386
|
+
-- Record the authoritative terminal boundary before accepting the
|
|
10387
|
+
-- attempt payload; deferred log indexing owns no Run row lock.
|
|
10323
10388
|
SELECT clock_timestamp() AS accepted_at
|
|
10324
10389
|
FROM ${tableName('runs', options)}
|
|
10325
10390
|
WHERE run_id = $1
|
|
@@ -10758,6 +10823,8 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
|
|
|
10758
10823
|
'producerAttempt', $2::integer,
|
|
10759
10824
|
'terminalLogLines', $7::jsonb,
|
|
10760
10825
|
'terminalLogContexts', $17::jsonb,
|
|
10826
|
+
'terminalLogMetadata', $20::jsonb,
|
|
10827
|
+
'terminalLogIds', $19::jsonb->'terminalLogIds',
|
|
10761
10828
|
'terminalProgressEvents', coalesce((
|
|
10762
10829
|
SELECT accepted.runner_log_payload_json->'terminalProgressEvents'
|
|
10763
10830
|
FROM completed_attempt AS accepted
|
|
@@ -10802,6 +10869,7 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
|
|
|
10802
10869
|
JSON.stringify(terminalLogProjection.lineContexts),
|
|
10803
10870
|
stringifyPostgresJson(terminalWorkCounterSegment ?? null),
|
|
10804
10871
|
stringifyPostgresJson(runnerTerminalLogPayload(input)),
|
|
10872
|
+
JSON.stringify(terminalLogProjection.lineMetadata),
|
|
10805
10873
|
],
|
|
10806
10874
|
);
|
|
10807
10875
|
if (completed.rows.length > 0) return 'completed';
|
|
@@ -13134,7 +13202,10 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13134
13202
|
runtimeTiming?: PlayRunnerRuntimeTiming | null;
|
|
13135
13203
|
terminalLogLines?: readonly string[] | null;
|
|
13136
13204
|
terminalLogContexts?: readonly (PlayLogContext | null)[] | null;
|
|
13205
|
+
terminalLogMetadata?: readonly (PlayLogMetadata | null)[] | null;
|
|
13206
|
+
terminalLogIds?: readonly (string | null)[] | null;
|
|
13137
13207
|
terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
|
|
13208
|
+
terminalProgressEvents?: readonly unknown[];
|
|
13138
13209
|
},
|
|
13139
13210
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
13140
13211
|
): Promise<boolean> {
|
|
@@ -13151,10 +13222,24 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13151
13222
|
'Terminal log contexts must align with terminal log lines.',
|
|
13152
13223
|
);
|
|
13153
13224
|
}
|
|
13225
|
+
if (
|
|
13226
|
+
input.terminalLogMetadata != null &&
|
|
13227
|
+
input.terminalLogMetadata.length !== (input.terminalLogLines ?? []).length
|
|
13228
|
+
) {
|
|
13229
|
+
throw new TypeError(
|
|
13230
|
+
'Terminal log metadata must align with terminal log lines.',
|
|
13231
|
+
);
|
|
13232
|
+
}
|
|
13233
|
+
const terminalLogIds = normalizeRunLogIds(
|
|
13234
|
+
input.terminalLogIds,
|
|
13235
|
+
input.terminalLogLines ?? [],
|
|
13236
|
+
);
|
|
13154
13237
|
const terminalLogEntries = (input.terminalLogLines ?? [])
|
|
13155
13238
|
.map((line, index) => ({
|
|
13156
13239
|
line: line.trim(),
|
|
13157
13240
|
context: input.terminalLogContexts?.[index] ?? null,
|
|
13241
|
+
metadata: input.terminalLogMetadata?.[index] ?? null,
|
|
13242
|
+
entryId: terminalLogIds?.[index] ?? null,
|
|
13158
13243
|
}))
|
|
13159
13244
|
.filter((entry) => entry.line.length > 0)
|
|
13160
13245
|
.slice(-LOG_TAIL_LIMIT);
|
|
@@ -13162,6 +13247,9 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13162
13247
|
const terminalLogContexts = terminalLogEntries.map((entry) =>
|
|
13163
13248
|
normalizePlayLogContext(entry.context),
|
|
13164
13249
|
);
|
|
13250
|
+
const terminalLogMetadata = terminalLogEntries.map((entry) =>
|
|
13251
|
+
normalizePlayLogMetadata(entry.metadata),
|
|
13252
|
+
);
|
|
13165
13253
|
const completed = await client.query<{ run_id: string }>(
|
|
13166
13254
|
`
|
|
13167
13255
|
WITH ${runtimeSandboxCustodyLocksSql('run.run_id = $1', options)}, completed AS (
|
|
@@ -13340,9 +13428,15 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13340
13428
|
'producerAttempt', $2::integer,
|
|
13341
13429
|
'terminalLogLines', $8::jsonb,
|
|
13342
13430
|
'terminalLogContexts', $11::jsonb,
|
|
13431
|
+
'terminalLogMetadata', $13::jsonb,
|
|
13432
|
+
'terminalLogIds', $15::jsonb,
|
|
13343
13433
|
'resultSummary', $10::jsonb,
|
|
13344
13434
|
'terminalWorkCounterSegment', $12::jsonb
|
|
13345
|
-
)
|
|
13435
|
+
) || CASE
|
|
13436
|
+
WHEN jsonb_array_length($14::jsonb) > 0
|
|
13437
|
+
THEN jsonb_build_object('terminalProgressEvents', $14::jsonb)
|
|
13438
|
+
ELSE '{}'::jsonb
|
|
13439
|
+
END,
|
|
13346
13440
|
${causalOutboxCreatedAt('run_terminal.run_id', options)}
|
|
13347
13441
|
FROM run_terminal
|
|
13348
13442
|
ON CONFLICT (event_id) DO NOTHING
|
|
@@ -13365,6 +13459,13 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13365
13459
|
stringifyPostgresJson(resultSummary),
|
|
13366
13460
|
JSON.stringify(terminalLogContexts),
|
|
13367
13461
|
stringifyPostgresJson(input.terminalWorkCounterSegment ?? null),
|
|
13462
|
+
JSON.stringify(terminalLogMetadata),
|
|
13463
|
+
JSON.stringify(input.terminalProgressEvents ?? []),
|
|
13464
|
+
stringifyPostgresJson(
|
|
13465
|
+
terminalLogIds
|
|
13466
|
+
? terminalLogEntries.map((entry) => entry.entryId)
|
|
13467
|
+
: null,
|
|
13468
|
+
),
|
|
13368
13469
|
],
|
|
13369
13470
|
);
|
|
13370
13471
|
if (completed.rows.length === 0) {
|
|
@@ -13383,13 +13484,15 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13383
13484
|
result?: unknown;
|
|
13384
13485
|
terminalLogLines?: readonly string[] | null;
|
|
13385
13486
|
terminalLogContexts?: readonly (PlayLogContext | null)[] | null;
|
|
13487
|
+
terminalLogMetadata?: readonly (PlayLogMetadata | null)[] | null;
|
|
13488
|
+
terminalLogIds?: readonly (string | null)[] | null;
|
|
13386
13489
|
/** Refuse to overwrite an authoritative detached runner terminal that won
|
|
13387
13490
|
* after claim. Resumable `suspended` receipts may be superseded by the
|
|
13388
13491
|
* hard ceiling in the same fenced statement. */
|
|
13389
13492
|
preserveRunnerTerminal?: boolean;
|
|
13390
13493
|
sandboxCrashDiagnostic?: PostgresSchedulerSandboxCrashDiagnosticRequest;
|
|
13391
|
-
/** Progress facts the ledger must apply before
|
|
13392
|
-
*
|
|
13494
|
+
/** Progress facts the ledger must apply before the terminal fact. Omitted
|
|
13495
|
+
* from the payload when empty. */
|
|
13393
13496
|
terminalProgressEvents?: readonly unknown[];
|
|
13394
13497
|
},
|
|
13395
13498
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
@@ -13429,10 +13532,24 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13429
13532
|
'Terminal log contexts must align with terminal log lines.',
|
|
13430
13533
|
);
|
|
13431
13534
|
}
|
|
13535
|
+
if (
|
|
13536
|
+
input.terminalLogMetadata != null &&
|
|
13537
|
+
input.terminalLogMetadata.length !== (input.terminalLogLines ?? []).length
|
|
13538
|
+
) {
|
|
13539
|
+
throw new TypeError(
|
|
13540
|
+
'Terminal log metadata must align with terminal log lines.',
|
|
13541
|
+
);
|
|
13542
|
+
}
|
|
13543
|
+
const terminalLogIds = normalizeRunLogIds(
|
|
13544
|
+
input.terminalLogIds,
|
|
13545
|
+
input.terminalLogLines ?? [],
|
|
13546
|
+
);
|
|
13432
13547
|
const terminalLogEntries = (input.terminalLogLines ?? [])
|
|
13433
13548
|
.map((line, index) => ({
|
|
13434
13549
|
line: line.trim(),
|
|
13435
13550
|
context: input.terminalLogContexts?.[index] ?? null,
|
|
13551
|
+
metadata: input.terminalLogMetadata?.[index] ?? null,
|
|
13552
|
+
entryId: terminalLogIds?.[index] ?? null,
|
|
13436
13553
|
}))
|
|
13437
13554
|
.filter((entry) => entry.line.length > 0)
|
|
13438
13555
|
.slice(-LOG_TAIL_LIMIT);
|
|
@@ -13440,6 +13557,9 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13440
13557
|
const terminalLogContexts = terminalLogEntries.map((entry) =>
|
|
13441
13558
|
normalizePlayLogContext(entry.context),
|
|
13442
13559
|
);
|
|
13560
|
+
const terminalLogMetadata = terminalLogEntries.map((entry) =>
|
|
13561
|
+
normalizePlayLogMetadata(entry.metadata),
|
|
13562
|
+
);
|
|
13443
13563
|
const failed = await client.query<{ run_id: string }>(
|
|
13444
13564
|
`
|
|
13445
13565
|
WITH ${runtimeSandboxCustodyLocksSql('run.run_id = $1', options)}, failed AS (
|
|
@@ -13678,6 +13798,8 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13678
13798
|
'producerAttempt', $2::integer,
|
|
13679
13799
|
'terminalLogLines', $12::jsonb,
|
|
13680
13800
|
'terminalLogContexts', $14::jsonb,
|
|
13801
|
+
'terminalLogMetadata', $17::jsonb,
|
|
13802
|
+
'terminalLogIds', $18::jsonb,
|
|
13681
13803
|
'resultSummary', $13::jsonb,
|
|
13682
13804
|
'terminalWorkCounterSegment', $15::jsonb
|
|
13683
13805
|
) || CASE
|
|
@@ -13711,6 +13833,12 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13711
13833
|
JSON.stringify(terminalLogContexts),
|
|
13712
13834
|
stringifyPostgresJson(terminalWorkCounterSegment ?? null),
|
|
13713
13835
|
stringifyPostgresJson(input.terminalProgressEvents ?? []),
|
|
13836
|
+
JSON.stringify(terminalLogMetadata),
|
|
13837
|
+
stringifyPostgresJson(
|
|
13838
|
+
terminalLogIds
|
|
13839
|
+
? terminalLogEntries.map((entry) => entry.entryId)
|
|
13840
|
+
: null,
|
|
13841
|
+
),
|
|
13714
13842
|
],
|
|
13715
13843
|
);
|
|
13716
13844
|
return failed.rows.length > 0;
|
|
@@ -14413,6 +14541,8 @@ function runnerTerminalLogPayload(input: {
|
|
|
14413
14541
|
producerAttempt: input.attempt,
|
|
14414
14542
|
terminalLogLines: projection.lines,
|
|
14415
14543
|
terminalLogContexts: projection.lineContexts,
|
|
14544
|
+
terminalLogMetadata: projection.lineMetadata,
|
|
14545
|
+
...(projection.lineIds ? { terminalLogIds: projection.lineIds } : {}),
|
|
14416
14546
|
terminalProgressEvents: input.terminalProgressEvents ?? [],
|
|
14417
14547
|
// Older runner envelopes conveyed these diagnostics only in tagged log
|
|
14418
14548
|
// text. Preserve their small structured facts before removing that text.
|
|
@@ -14433,21 +14563,28 @@ function runnerTerminalLogPayload(input: {
|
|
|
14433
14563
|
function terminalLogProjectionForOutbox(
|
|
14434
14564
|
runnerResult: unknown,
|
|
14435
14565
|
mode: PostgresSchedulerTerminalLogOutbox | undefined,
|
|
14436
|
-
): {
|
|
14566
|
+
): {
|
|
14567
|
+
lines: string[];
|
|
14568
|
+
lineContexts: Array<PlayLogContext | null>;
|
|
14569
|
+
lineMetadata: Array<PlayLogMetadata | null>;
|
|
14570
|
+
lineIds?: Array<string | null>;
|
|
14571
|
+
} {
|
|
14437
14572
|
if (
|
|
14438
14573
|
mode !== 'ctx_log_tail' ||
|
|
14439
14574
|
!runnerResult ||
|
|
14440
14575
|
typeof runnerResult !== 'object'
|
|
14441
14576
|
) {
|
|
14442
|
-
return { lines: [], lineContexts: [] };
|
|
14577
|
+
return { lines: [], lineContexts: [], lineMetadata: [] };
|
|
14443
14578
|
}
|
|
14444
14579
|
const result = runnerResult as {
|
|
14445
14580
|
logs?: unknown;
|
|
14446
14581
|
logContexts?: unknown;
|
|
14582
|
+
logMetadata?: unknown;
|
|
14583
|
+
logIds?: unknown;
|
|
14447
14584
|
};
|
|
14448
14585
|
const logs = result.logs;
|
|
14449
14586
|
if (!Array.isArray(logs)) {
|
|
14450
|
-
return { lines: [], lineContexts: [] };
|
|
14587
|
+
return { lines: [], lineContexts: [], lineMetadata: [] };
|
|
14451
14588
|
}
|
|
14452
14589
|
if (
|
|
14453
14590
|
result.logContexts !== undefined &&
|
|
@@ -14456,9 +14593,19 @@ function terminalLogProjectionForOutbox(
|
|
|
14456
14593
|
) {
|
|
14457
14594
|
throw new TypeError('Terminal log contexts must align with runner logs.');
|
|
14458
14595
|
}
|
|
14596
|
+
if (
|
|
14597
|
+
result.logMetadata !== undefined &&
|
|
14598
|
+
(!Array.isArray(result.logMetadata) ||
|
|
14599
|
+
result.logMetadata.length !== logs.length)
|
|
14600
|
+
)
|
|
14601
|
+
throw new TypeError('Terminal log metadata must align with logs.');
|
|
14602
|
+
const logMetadata = Array.isArray(result.logMetadata)
|
|
14603
|
+
? result.logMetadata
|
|
14604
|
+
: undefined;
|
|
14459
14605
|
const logContexts = Array.isArray(result.logContexts)
|
|
14460
14606
|
? result.logContexts
|
|
14461
14607
|
: undefined;
|
|
14608
|
+
const logIds = normalizeRunLogIds(result.logIds, logs);
|
|
14462
14609
|
const entries = logs
|
|
14463
14610
|
.map((line, index) => {
|
|
14464
14611
|
if (typeof line !== 'string') return null;
|
|
@@ -14475,16 +14622,26 @@ function terminalLogProjectionForOutbox(
|
|
|
14475
14622
|
return {
|
|
14476
14623
|
line: line.trim(),
|
|
14477
14624
|
context: normalizePlayLogContext(logContexts?.[index]),
|
|
14625
|
+
metadata: normalizePlayLogMetadata(logMetadata?.[index]),
|
|
14626
|
+
entryId: logIds?.[index] ?? null,
|
|
14478
14627
|
};
|
|
14479
14628
|
})
|
|
14480
14629
|
.filter(
|
|
14481
|
-
(
|
|
14482
|
-
entry
|
|
14630
|
+
(
|
|
14631
|
+
entry,
|
|
14632
|
+
): entry is {
|
|
14633
|
+
line: string;
|
|
14634
|
+
context: PlayLogContext | null;
|
|
14635
|
+
metadata: PlayLogMetadata | null;
|
|
14636
|
+
entryId: string | null;
|
|
14637
|
+
} => entry !== null,
|
|
14483
14638
|
)
|
|
14484
14639
|
.slice(-LOG_TAIL_LIMIT);
|
|
14485
14640
|
return {
|
|
14486
14641
|
lines: entries.map((entry) => entry.line),
|
|
14487
14642
|
lineContexts: entries.map((entry) => entry.context),
|
|
14643
|
+
lineMetadata: entries.map((entry) => entry.metadata),
|
|
14644
|
+
...(logIds ? { lineIds: entries.map((entry) => entry.entryId) } : {}),
|
|
14488
14645
|
};
|
|
14489
14646
|
}
|
|
14490
14647
|
|
|
@@ -14733,6 +14890,9 @@ export async function executePostgresSchedulerClaim(
|
|
|
14733
14890
|
runtimeTiming: extractRunnerRuntimeTiming(output),
|
|
14734
14891
|
terminalLogLines: terminalLogProjection.lines,
|
|
14735
14892
|
terminalLogContexts: terminalLogProjection.lineContexts,
|
|
14893
|
+
terminalLogMetadata: terminalLogProjection.lineMetadata,
|
|
14894
|
+
terminalProgressEvents: options.terminalProgressEvents,
|
|
14895
|
+
terminalLogIds: terminalLogProjection.lineIds,
|
|
14736
14896
|
terminalWorkCounterSegment: terminalWorkCounterSegmentFromRunnerResult(
|
|
14737
14897
|
output,
|
|
14738
14898
|
claim.attempt,
|
|
@@ -14794,12 +14954,14 @@ export async function executePostgresSchedulerClaim(
|
|
|
14794
14954
|
result: terminalFailureResult,
|
|
14795
14955
|
terminalLogLines: terminalLogProjection.lines,
|
|
14796
14956
|
terminalLogContexts: terminalLogProjection.lineContexts,
|
|
14957
|
+
terminalLogMetadata: terminalLogProjection.lineMetadata,
|
|
14958
|
+
terminalLogIds: terminalLogProjection.lineIds,
|
|
14797
14959
|
sandboxCrashDiagnostic: options.sandboxCrashDiagnostic,
|
|
14798
14960
|
...(options.preserveRunnerTerminal
|
|
14799
14961
|
? { preserveRunnerTerminal: true }
|
|
14800
14962
|
: {}),
|
|
14801
|
-
...(
|
|
14802
|
-
? { terminalProgressEvents: options.
|
|
14963
|
+
...(options.terminalProgressEvents?.length
|
|
14964
|
+
? { terminalProgressEvents: options.terminalProgressEvents }
|
|
14803
14965
|
: {}),
|
|
14804
14966
|
},
|
|
14805
14967
|
options,
|
|
@@ -14987,6 +15149,9 @@ export function buildPostgresSchedulerRunnerContext(
|
|
|
14987
15149
|
const rootRunId = launch.rootRunId?.trim() || launch.runId;
|
|
14988
15150
|
return {
|
|
14989
15151
|
executorToken: launch.executorToken,
|
|
15152
|
+
...(launch.runtimeLogIdentity === 'origin'
|
|
15153
|
+
? { runtimeLogIdentity: 'origin' as const }
|
|
15154
|
+
: {}),
|
|
14990
15155
|
baseUrl: runtimeApiBaseUrlFromLaunch(launch),
|
|
14991
15156
|
executionGatewayBaseUrl,
|
|
14992
15157
|
receiptGatewayBaseUrl,
|
|
@@ -17388,6 +17553,12 @@ async function appendRunLedgerOutboxFact(
|
|
|
17388
17553
|
WITH admitted AS MATERIALIZED (
|
|
17389
17554
|
SELECT run_id FROM ${tableName('runs', options)}
|
|
17390
17555
|
WHERE run_id = $1 AND org_id = $2
|
|
17556
|
+
-- Evaluate ingress time only after the projector/lifecycle fence. An
|
|
17557
|
+
-- insert waiting behind a frozen receipt must become its successor,
|
|
17558
|
+
-- never a late predecessor that changes the attempted receipt.
|
|
17559
|
+
-- NO KEY UPDATE still permits legacy inserts' foreign-key KEY SHARE
|
|
17560
|
+
-- while they finish under the event-search advisory mutex.
|
|
17561
|
+
FOR NO KEY UPDATE
|
|
17391
17562
|
), recorded AS (
|
|
17392
17563
|
INSERT INTO ${tableName('outbox', options)}
|
|
17393
17564
|
(event_id, run_id, event_type, payload_json, created_at)
|
|
@@ -17413,23 +17584,43 @@ export { RunLedgerProjectionLeaseLostError } from '../run-ledger-projection-cont
|
|
|
17413
17584
|
/** Extend only our still-live lease before another bounded transport piece. */
|
|
17414
17585
|
export async function renewPostgresRunLedgerDelivery(
|
|
17415
17586
|
client: PostgresSchedulerQueryClient,
|
|
17416
|
-
input: {
|
|
17587
|
+
input: {
|
|
17588
|
+
eventIds: string[];
|
|
17589
|
+
leaseOwner: string;
|
|
17590
|
+
leaseSeconds: number;
|
|
17591
|
+
/** Freeze an ingress receipt before any HTTP attempt, including singletons. */
|
|
17592
|
+
freezeReceipt?: boolean;
|
|
17593
|
+
},
|
|
17417
17594
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
17418
17595
|
): Promise<void> {
|
|
17419
|
-
const
|
|
17420
|
-
|
|
17596
|
+
const renew = (freezeReceipt: boolean) =>
|
|
17597
|
+
client.query<{ event_id: string }>(
|
|
17598
|
+
`
|
|
17421
17599
|
UPDATE ${tableName('outbox', options)}
|
|
17422
17600
|
SET lease_expires_at = now() + make_interval(secs => $3::integer)
|
|
17601
|
+
${freezeReceipt ? ', delivery_batch_started_at = coalesce(delivery_batch_started_at, statement_timestamp())' : ''}
|
|
17423
17602
|
WHERE event_id = ANY($1::text[]) AND lease_owner = $2
|
|
17424
17603
|
AND delivered_at IS NULL AND failed_at IS NULL AND lease_expires_at > now()
|
|
17425
17604
|
RETURNING event_id
|
|
17426
17605
|
`,
|
|
17427
|
-
|
|
17428
|
-
|
|
17429
|
-
|
|
17430
|
-
|
|
17431
|
-
|
|
17432
|
-
|
|
17606
|
+
[
|
|
17607
|
+
input.eventIds,
|
|
17608
|
+
input.leaseOwner,
|
|
17609
|
+
Math.max(5, Math.min(300, input.leaseSeconds)),
|
|
17610
|
+
],
|
|
17611
|
+
);
|
|
17612
|
+
let result;
|
|
17613
|
+
try {
|
|
17614
|
+
result = await renew(
|
|
17615
|
+
input.freezeReceipt === true &&
|
|
17616
|
+
supportsOutboxDeliveryBatchMarker(options),
|
|
17617
|
+
);
|
|
17618
|
+
} catch (error) {
|
|
17619
|
+
if (!input.freezeReceipt || !isMissingOutboxDeliveryBatchMarker(error))
|
|
17620
|
+
throw error;
|
|
17621
|
+
markOutboxDeliveryBatchMarkerUnavailable(options);
|
|
17622
|
+
result = await renew(false);
|
|
17623
|
+
}
|
|
17433
17624
|
if (result.rows.length !== input.eventIds.length)
|
|
17434
17625
|
throw new RunLedgerProjectionLeaseLostError('delivery_lease_not_current');
|
|
17435
17626
|
}
|
|
@@ -17457,6 +17648,7 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
17457
17648
|
const result = await client.query<{
|
|
17458
17649
|
delivery_key: string;
|
|
17459
17650
|
event_type: string;
|
|
17651
|
+
delivery_batch_started_at: string | null;
|
|
17460
17652
|
lease_owner: string | null;
|
|
17461
17653
|
lease_live: boolean;
|
|
17462
17654
|
due: boolean;
|
|
@@ -17467,6 +17659,8 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
17467
17659
|
`
|
|
17468
17660
|
SELECT outbox.delivery_key,
|
|
17469
17661
|
outbox.event_type,
|
|
17662
|
+
-- Row JSON keeps this optional on historical scheduler schemas.
|
|
17663
|
+
to_jsonb(outbox)->>'delivery_batch_started_at' AS delivery_batch_started_at,
|
|
17470
17664
|
outbox.lease_owner,
|
|
17471
17665
|
outbox.lease_expires_at > now() AS lease_live,
|
|
17472
17666
|
coalesce(outbox.next_attempt_at, outbox.created_at) <= now() AS due,
|
|
@@ -17476,6 +17670,7 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
17476
17670
|
FROM ${tableName('outbox', options)} AS outbox
|
|
17477
17671
|
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = outbox.run_id
|
|
17478
17672
|
WHERE outbox.run_id = $1 AND runs.org_id = $2
|
|
17673
|
+
AND NOT ${searchLaneSql('outbox')}
|
|
17479
17674
|
AND outbox.delivered_at IS NULL
|
|
17480
17675
|
ORDER BY ${runLedgerOrderSql('outbox')}
|
|
17481
17676
|
-- Later producer facts cannot belong to this receipt. Do not hydrate their
|
|
@@ -17497,9 +17692,12 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
17497
17692
|
// Cancellation may own several pending facts at once. Validate the exact
|
|
17498
17693
|
// contiguous, due prefix named by the receipt instead of all rows sharing
|
|
17499
17694
|
// that cancellation lease.
|
|
17695
|
+
const ingressReceiptStarted =
|
|
17696
|
+
rows[0]?.event_type === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents &&
|
|
17697
|
+
rows[0]?.delivery_batch_started_at != null;
|
|
17500
17698
|
const prefix: typeof rows = [];
|
|
17501
17699
|
let prefixStopReason: RunLedgerProjectionLeaseLostReason | null = null;
|
|
17502
|
-
for (const row of rows) {
|
|
17700
|
+
for (const [index, row] of rows.entries()) {
|
|
17503
17701
|
if (row.lease_owner !== leaseOwner) {
|
|
17504
17702
|
prefixStopReason = 'prefix_lease_owner_mismatch';
|
|
17505
17703
|
break;
|
|
@@ -17516,11 +17714,6 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
17516
17714
|
prefixStopReason = 'predecessor_failed';
|
|
17517
17715
|
break;
|
|
17518
17716
|
}
|
|
17519
|
-
if (
|
|
17520
|
-
prefix.length > 0 &&
|
|
17521
|
-
row.event_type === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents
|
|
17522
|
-
)
|
|
17523
|
-
break;
|
|
17524
17717
|
prefix.push(row);
|
|
17525
17718
|
|
|
17526
17719
|
const first = prefix[0]!;
|
|
@@ -17567,10 +17760,17 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
17567
17760
|
runId: input.runId,
|
|
17568
17761
|
deliveryKey: first.delivery_key,
|
|
17569
17762
|
});
|
|
17763
|
+
// A receipt already attempted as a compound prefix cannot be split by
|
|
17764
|
+
// an older competing projector after lease expiry. Reject with the normal
|
|
17765
|
+
// retryable lease-lost response until a capable writer reclaims it whole.
|
|
17766
|
+
const atStartedPrefixEnd =
|
|
17767
|
+
!ingressReceiptStarted ||
|
|
17768
|
+
rows[index + 1]?.delivery_batch_started_at == null;
|
|
17570
17769
|
if (
|
|
17571
|
-
|
|
17572
|
-
input.idempotencyKey ===
|
|
17573
|
-
|
|
17770
|
+
atStartedPrefixEnd &&
|
|
17771
|
+
(input.idempotencyKey === expectedKey ||
|
|
17772
|
+
input.idempotencyKey === batchKey ||
|
|
17773
|
+
legacySingletonKey)
|
|
17574
17774
|
) {
|
|
17575
17775
|
if (input.projectedEvents === undefined) return;
|
|
17576
17776
|
const prefixEvents: PlayRunLedgerIngressEvent[] = [];
|
|
@@ -17612,11 +17812,18 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
17612
17812
|
|
|
17613
17813
|
if (
|
|
17614
17814
|
prefix.length === 1 &&
|
|
17615
|
-
|
|
17815
|
+
RUN_SCHEDULER_OUTBOX_START_EVENT_TYPES.includes(first.event_type)
|
|
17616
17816
|
) {
|
|
17617
17817
|
break;
|
|
17618
17818
|
}
|
|
17619
|
-
|
|
17819
|
+
// A frozen ingress receipt can include late runner logs after terminal.
|
|
17820
|
+
// Prove its complete immutable boundary, including transport chunks after
|
|
17821
|
+
// terminal, rather than splitting a receipt that has already been attempted.
|
|
17822
|
+
if (
|
|
17823
|
+
(terminal && !ingressReceiptStarted) ||
|
|
17824
|
+
(ingressReceiptStarted && atStartedPrefixEnd) ||
|
|
17825
|
+
prefix.length >= RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS
|
|
17826
|
+
) {
|
|
17620
17827
|
break;
|
|
17621
17828
|
}
|
|
17622
17829
|
}
|
|
@@ -17631,7 +17838,11 @@ function runLedgerPrecedesSql(
|
|
|
17631
17838
|
successor: string,
|
|
17632
17839
|
inclusive = false,
|
|
17633
17840
|
): string {
|
|
17634
|
-
return
|
|
17841
|
+
return `(${searchLaneSql(predecessor)} = ${searchLaneSql(successor)} AND ${runLedgerPositionSql(predecessor)} ${inclusive ? '<=' : '<'} ${runLedgerPositionSql(successor)})`;
|
|
17842
|
+
}
|
|
17843
|
+
|
|
17844
|
+
function searchLaneSql(alias: string): string {
|
|
17845
|
+
return `coalesce(${alias}.payload_json->'${RUNTIME_EVENT_SEARCH_ONLY_FIELD}' = 'true'::jsonb, false)`;
|
|
17635
17846
|
}
|
|
17636
17847
|
|
|
17637
17848
|
function runLedgerPositionSql(alias: string): string {
|
|
@@ -17658,6 +17869,32 @@ function runLedgerTerminalPendingSql(
|
|
|
17658
17869
|
)`;
|
|
17659
17870
|
}
|
|
17660
17871
|
|
|
17872
|
+
function runLedgerWaitingRunsSql(
|
|
17873
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
17874
|
+
runIdParameter?: string,
|
|
17875
|
+
): string {
|
|
17876
|
+
return `SELECT DISTINCT waiting_event.run_id
|
|
17877
|
+
FROM ${tableName('outbox', options)} AS waiting_event
|
|
17878
|
+
WHERE waiting_event.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runWaiting}
|
|
17879
|
+
-- Detached runner parks are transport-only and project no public wait.
|
|
17880
|
+
AND waiting_event.payload_json->>'kind' IS DISTINCT FROM 'detached_runner'
|
|
17881
|
+
AND waiting_event.delivered_at IS NULL
|
|
17882
|
+
AND waiting_event.failed_at IS NULL
|
|
17883
|
+
AND coalesce(waiting_event.next_attempt_at, waiting_event.created_at) <= now()
|
|
17884
|
+
${runIdParameter ? `AND (${runIdParameter}::text IS NULL OR waiting_event.run_id = ${runIdParameter}::text)` : ''}`;
|
|
17885
|
+
}
|
|
17886
|
+
|
|
17887
|
+
function runLedgerWaitingPendingSql(alias: string): string {
|
|
17888
|
+
return `(${alias}.run_id IN (SELECT run_id FROM waiting_runs))`;
|
|
17889
|
+
}
|
|
17890
|
+
|
|
17891
|
+
function runLedgerUrgentLifecyclePendingSql(
|
|
17892
|
+
alias: string,
|
|
17893
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
17894
|
+
): string {
|
|
17895
|
+
return `(${runLedgerTerminalPendingSql(alias, options)} OR ${runLedgerWaitingPendingSql(alias)})`;
|
|
17896
|
+
}
|
|
17897
|
+
|
|
17661
17898
|
/**
|
|
17662
17899
|
* Claim a causal prefix, preserving the v33 adapter while the additive batch
|
|
17663
17900
|
* marker is absent. An enclosing transaction uses a savepoint so the missing
|
|
@@ -17670,11 +17907,43 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17670
17907
|
runId?: string | null;
|
|
17671
17908
|
leaseOwner?: string | null;
|
|
17672
17909
|
leaseSeconds?: number;
|
|
17910
|
+
/** Independent raw search work uses the same outbox leases and controls. */
|
|
17911
|
+
searchOnly?: boolean;
|
|
17912
|
+
/** Advertised by the launch-pinned receipt verifier, never inferred locally. */
|
|
17913
|
+
allowIngressPrefix?: boolean;
|
|
17914
|
+
/** Extend this still-owned head atomically under the run cancellation fence. */
|
|
17915
|
+
extendOwnedHead?: string;
|
|
17673
17916
|
/** The dispatcher owns a short, exclusive claim transaction. */
|
|
17674
17917
|
inTransaction?: boolean;
|
|
17675
17918
|
} = {},
|
|
17676
17919
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
17677
17920
|
): Promise<PostgresSchedulerOutboxEvent[]> {
|
|
17921
|
+
const { events } = await claimPostgresSchedulerOutboxPrefix(
|
|
17922
|
+
client,
|
|
17923
|
+
input,
|
|
17924
|
+
options,
|
|
17925
|
+
);
|
|
17926
|
+
return events;
|
|
17927
|
+
}
|
|
17928
|
+
|
|
17929
|
+
/**
|
|
17930
|
+
* ACK and next-prefix selection share one statement after the run lock. Data
|
|
17931
|
+
* CTEs use the statement's original snapshot, so every pending-row predicate
|
|
17932
|
+
* explicitly excludes the acknowledged IDs. A partial ACK cannot claim a
|
|
17933
|
+
* successor. The receiver's ingress capability and frozen receipts still
|
|
17934
|
+
* determine which causal facts may share a prefix.
|
|
17935
|
+
*/
|
|
17936
|
+
async function claimPostgresSchedulerOutboxPrefix(
|
|
17937
|
+
client: PostgresSchedulerQueryClient,
|
|
17938
|
+
input: NonNullable<
|
|
17939
|
+
Parameters<typeof readPendingPostgresSchedulerOutboxEvents>[1]
|
|
17940
|
+
>,
|
|
17941
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
17942
|
+
acknowledgement?: { eventIds: string[]; targetEventId: string },
|
|
17943
|
+
): Promise<{
|
|
17944
|
+
events: PostgresSchedulerOutboxEvent[];
|
|
17945
|
+
deliveredRows: RunLedgerOutboxDeliveredRow[];
|
|
17946
|
+
}> {
|
|
17678
17947
|
const requestedLimit = Math.floor(input.limit ?? 1);
|
|
17679
17948
|
const limit = Number.isFinite(requestedLimit)
|
|
17680
17949
|
? Math.max(1, requestedLimit)
|
|
@@ -17682,20 +17951,76 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17682
17951
|
const runId = input.runId?.trim() || null;
|
|
17683
17952
|
const leaseOwner = input.leaseOwner?.trim() || null;
|
|
17684
17953
|
const leaseSeconds = Math.max(5, Math.min(300, input.leaseSeconds ?? 30));
|
|
17685
|
-
const
|
|
17686
|
-
|
|
17954
|
+
const pending = (alias: string) =>
|
|
17955
|
+
`${alias}.delivered_at IS NULL${
|
|
17956
|
+
acknowledgement ? ` AND NOT (${alias}.event_id = ANY($5::text[]))` : ''
|
|
17957
|
+
}`;
|
|
17958
|
+
const claim = async (supportsTerminalBatchMarker: boolean) => {
|
|
17959
|
+
const urgentLifecyclePendingSql = (alias: string) =>
|
|
17960
|
+
input.searchOnly === true
|
|
17961
|
+
? runLedgerTerminalPendingSql(alias, options)
|
|
17962
|
+
: runLedgerUrgentLifecyclePendingSql(alias, options);
|
|
17963
|
+
const ingressPrefixSql =
|
|
17964
|
+
supportsTerminalBatchMarker && input.allowIngressPrefix === true
|
|
17965
|
+
? `(head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} AND ${urgentLifecyclePendingSql('head')})`
|
|
17966
|
+
: 'false';
|
|
17967
|
+
return await client.query<{
|
|
17968
|
+
row_kind: 'claimed' | 'delivered';
|
|
17687
17969
|
event_id: string;
|
|
17688
17970
|
run_id: string;
|
|
17689
17971
|
event_type: string;
|
|
17690
17972
|
payload_json: unknown;
|
|
17691
17973
|
created_at: Date;
|
|
17692
17974
|
delivery_key: string | null;
|
|
17975
|
+
lag_ms: number | string | null;
|
|
17693
17976
|
}>(
|
|
17694
17977
|
`
|
|
17695
|
-
WITH
|
|
17696
|
-
|
|
17978
|
+
WITH ${
|
|
17979
|
+
acknowledgement
|
|
17980
|
+
? `delivered AS (
|
|
17981
|
+
${runLedgerAcknowledgeSql(options, '$5', '$3')}
|
|
17982
|
+
),`
|
|
17983
|
+
: ''
|
|
17984
|
+
} ${
|
|
17985
|
+
input.searchOnly === true
|
|
17986
|
+
? ''
|
|
17987
|
+
: `waiting_runs AS MATERIALIZED (
|
|
17988
|
+
${runLedgerWaitingRunsSql(options, '$2')}
|
|
17989
|
+
), `
|
|
17990
|
+
}${
|
|
17991
|
+
input.extendOwnedHead
|
|
17992
|
+
? `owned_run AS MATERIALIZED (
|
|
17993
|
+
SELECT run_id FROM ${tableName('runs', options)}
|
|
17994
|
+
WHERE run_id = $2
|
|
17995
|
+
-- This dispatcher already owns the head. A producer holding the run
|
|
17996
|
+
-- lock is not a lost delivery lease: wait for its short transaction,
|
|
17997
|
+
-- then revalidate the head below. New-run selection still skips locks.
|
|
17998
|
+
FOR UPDATE
|
|
17999
|
+
),`
|
|
18000
|
+
: ''
|
|
18001
|
+
} head AS MATERIALIZED (
|
|
18002
|
+
SELECT candidate.event_id, candidate.run_id, candidate.event_type,
|
|
18003
|
+
${supportsTerminalBatchMarker ? 'candidate.delivery_batch_started_at' : 'NULL::timestamptz'} AS delivery_batch_started_at
|
|
17697
18004
|
FROM ${tableName('outbox', options)} AS candidate
|
|
17698
|
-
|
|
18005
|
+
${input.extendOwnedHead ? 'JOIN owned_run ON owned_run.run_id = candidate.run_id' : ''}
|
|
18006
|
+
WHERE ${pending('candidate')}
|
|
18007
|
+
AND ${searchLaneSql('candidate')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
18008
|
+
${
|
|
18009
|
+
acknowledgement
|
|
18010
|
+
? `AND (SELECT count(*) FROM delivered) = cardinality($5::text[])
|
|
18011
|
+
AND EXISTS (
|
|
18012
|
+
SELECT 1 FROM ${tableName('outbox', options)} AS target
|
|
18013
|
+
WHERE target.run_id = $2 AND target.event_id = $6
|
|
18014
|
+
AND ${pending('target')} AND target.failed_at IS NULL
|
|
18015
|
+
)`
|
|
18016
|
+
: ''
|
|
18017
|
+
}
|
|
18018
|
+
${
|
|
18019
|
+
input.extendOwnedHead
|
|
18020
|
+
? `AND candidate.event_id = $5
|
|
18021
|
+
AND candidate.lease_owner = $3 AND candidate.lease_expires_at > now()`
|
|
18022
|
+
: ''
|
|
18023
|
+
}
|
|
17699
18024
|
AND candidate.failed_at IS NULL
|
|
17700
18025
|
AND coalesce(candidate.next_attempt_at, candidate.created_at) <= now()
|
|
17701
18026
|
AND ($2::text IS NULL OR candidate.run_id = $2)
|
|
@@ -17711,7 +18036,8 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17711
18036
|
AND NOT EXISTS (
|
|
17712
18037
|
SELECT 1 FROM ${tableName('outbox', options)} AS active
|
|
17713
18038
|
WHERE active.run_id = candidate.run_id
|
|
17714
|
-
AND active
|
|
18039
|
+
AND ${pending('active')}
|
|
18040
|
+
AND ${searchLaneSql('active')} = ${searchLaneSql('candidate')}
|
|
17715
18041
|
AND active.lease_owner IS NOT NULL
|
|
17716
18042
|
AND active.lease_expires_at > now()
|
|
17717
18043
|
AND active.lease_owner IS DISTINCT FROM $3::text
|
|
@@ -17723,7 +18049,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17723
18049
|
SELECT 1
|
|
17724
18050
|
FROM ${tableName('outbox', options)} AS predecessor
|
|
17725
18051
|
WHERE predecessor.run_id = candidate.run_id
|
|
17726
|
-
AND predecessor
|
|
18052
|
+
AND ${pending('predecessor')}
|
|
17727
18053
|
AND ${runLedgerPrecedesSql('predecessor', 'candidate')}
|
|
17728
18054
|
)
|
|
17729
18055
|
-- A new durable admission has no causal dependency on unrelated
|
|
@@ -17738,7 +18064,8 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17738
18064
|
CASE
|
|
17739
18065
|
WHEN candidate.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0
|
|
17740
18066
|
WHEN ${runLedgerTerminalPendingSql('candidate', options)} THEN 1
|
|
17741
|
-
|
|
18067
|
+
${input.searchOnly === true ? '' : `WHEN ${runLedgerWaitingPendingSql('candidate')} THEN 2`}
|
|
18068
|
+
ELSE 3
|
|
17742
18069
|
END,
|
|
17743
18070
|
coalesce(candidate.next_attempt_at, candidate.created_at),
|
|
17744
18071
|
candidate.created_at,
|
|
@@ -17749,17 +18076,26 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17749
18076
|
SELECT outbox.event_id
|
|
17750
18077
|
FROM ${tableName('outbox', options)} AS outbox
|
|
17751
18078
|
JOIN head ON head.run_id = outbox.run_id
|
|
17752
|
-
WHERE outbox
|
|
18079
|
+
WHERE ${pending('outbox')}
|
|
18080
|
+
AND ${searchLaneSql('outbox')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
17753
18081
|
AND outbox.failed_at IS NULL
|
|
17754
18082
|
-- Admission is causally first and must be projected alone. A later
|
|
17755
18083
|
-- start/terminal append may not leapfrog a delayed queued row.
|
|
17756
18084
|
AND (
|
|
17757
18085
|
head.event_type NOT IN (${RUN_EVENT_KIND_SQL_LITERAL.runCreated}, ${RUN_EVENT_KIND_SQL_LITERAL.runStarted}, ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents})
|
|
17758
18086
|
OR outbox.event_id = head.event_id
|
|
18087
|
+
OR (head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
|
|
18088
|
+
AND ${ingressPrefixSql})
|
|
17759
18089
|
)
|
|
17760
|
-
--
|
|
17761
|
-
--
|
|
17762
|
-
|
|
18090
|
+
-- Only a receiver with the exact-prefix verifier may combine ingress
|
|
18091
|
+
-- facts while draining a pending terminal. Old launch origins and
|
|
18092
|
+
-- schemas retain each producer's existing singleton receipt.
|
|
18093
|
+
AND (outbox.event_type <> ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
|
|
18094
|
+
OR outbox.event_id = head.event_id
|
|
18095
|
+
OR ${ingressPrefixSql})
|
|
18096
|
+
-- Once any ingress prefix was attempted, its receipt is immutable,
|
|
18097
|
+
-- including singleton appends whose HTTP/ACK outcome was ambiguous.
|
|
18098
|
+
AND (${supportsTerminalBatchMarker && input.allowIngressPrefix === true ? 'head.delivery_batch_started_at IS NULL OR outbox.delivery_batch_started_at IS NOT NULL' : 'true'})
|
|
17763
18099
|
AND coalesce(outbox.next_attempt_at, outbox.created_at) <= now()
|
|
17764
18100
|
AND (
|
|
17765
18101
|
$3::text IS NULL
|
|
@@ -17777,7 +18113,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17777
18113
|
SELECT 1
|
|
17778
18114
|
FROM ${tableName('outbox', options)} AS terminal_predecessor
|
|
17779
18115
|
WHERE terminal_predecessor.run_id = outbox.run_id
|
|
17780
|
-
AND terminal_predecessor
|
|
18116
|
+
AND ${pending('terminal_predecessor')}
|
|
17781
18117
|
AND ${
|
|
17782
18118
|
supportsTerminalBatchMarker
|
|
17783
18119
|
? 'terminal_predecessor.delivery_batch_started_at IS NOT NULL'
|
|
@@ -17799,10 +18135,12 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17799
18135
|
SELECT 1
|
|
17800
18136
|
FROM ${tableName('outbox', options)} AS predecessor
|
|
17801
18137
|
WHERE predecessor.run_id = outbox.run_id
|
|
17802
|
-
AND predecessor
|
|
18138
|
+
AND ${pending('predecessor')}
|
|
17803
18139
|
AND ${runLedgerPrecedesSql('predecessor', 'outbox')}
|
|
17804
18140
|
AND (
|
|
17805
|
-
predecessor.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
|
|
18141
|
+
(predecessor.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
|
|
18142
|
+
AND NOT (${ingressPrefixSql}))
|
|
18143
|
+
OR (${supportsTerminalBatchMarker && input.allowIngressPrefix === true ? 'head.delivery_batch_started_at IS NOT NULL AND predecessor.delivery_batch_started_at IS NULL' : 'false'})
|
|
17806
18144
|
OR predecessor.failed_at IS NOT NULL
|
|
17807
18145
|
OR coalesce(predecessor.next_attempt_at, predecessor.created_at) > now()
|
|
17808
18146
|
OR (
|
|
@@ -17833,8 +18171,8 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17833
18171
|
${
|
|
17834
18172
|
supportsTerminalBatchMarker
|
|
17835
18173
|
? `, delivery_batch_started_at = CASE
|
|
17836
|
-
WHEN terminal_batch.has_terminal
|
|
17837
|
-
THEN COALESCE(outbox.delivery_batch_started_at,
|
|
18174
|
+
WHEN terminal_batch.has_terminal ${input.allowIngressPrefix === true ? `OR ${urgentLifecyclePendingSql('outbox')}` : ''}
|
|
18175
|
+
THEN COALESCE(outbox.delivery_batch_started_at, statement_timestamp())
|
|
17838
18176
|
ELSE outbox.delivery_batch_started_at
|
|
17839
18177
|
END`
|
|
17840
18178
|
: ''
|
|
@@ -17846,13 +18184,53 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17846
18184
|
outbox.payload_json, outbox.created_at, outbox.lease_owner,
|
|
17847
18185
|
outbox.delivery_key
|
|
17848
18186
|
)
|
|
17849
|
-
|
|
17850
|
-
|
|
17851
|
-
|
|
17852
|
-
|
|
18187
|
+
${
|
|
18188
|
+
acknowledgement
|
|
18189
|
+
? `, notified AS MATERIALIZED (
|
|
18190
|
+
SELECT delivered.*,
|
|
18191
|
+
pg_notify(
|
|
18192
|
+
$7,
|
|
18193
|
+
json_build_object('eventId', event_id, 'kind', 'outbox_delivered')::text
|
|
18194
|
+
) AS notification
|
|
18195
|
+
FROM delivered
|
|
18196
|
+
)`
|
|
18197
|
+
: ''
|
|
18198
|
+
}
|
|
18199
|
+
SELECT * FROM (
|
|
18200
|
+
SELECT 'claimed'::text AS row_kind,
|
|
18201
|
+
event_id, run_id, event_type, payload_json, created_at, lease_owner,
|
|
18202
|
+
delivery_key, NULL::float8 AS lag_ms
|
|
18203
|
+
FROM claimed
|
|
18204
|
+
${
|
|
18205
|
+
acknowledgement
|
|
18206
|
+
? `UNION ALL
|
|
18207
|
+
SELECT 'delivered'::text AS row_kind,
|
|
18208
|
+
event_id, NULL::text AS run_id, event_type, NULL::jsonb AS payload_json,
|
|
18209
|
+
NULL::timestamptz AS created_at, NULL::text AS lease_owner,
|
|
18210
|
+
NULL::text AS delivery_key, lag_ms
|
|
18211
|
+
FROM notified`
|
|
18212
|
+
: ''
|
|
18213
|
+
}
|
|
18214
|
+
) AS result
|
|
18215
|
+
ORDER BY row_kind, ${runLedgerOrderSql('result')}
|
|
17853
18216
|
`,
|
|
17854
|
-
[
|
|
18217
|
+
[
|
|
18218
|
+
limit,
|
|
18219
|
+
runId,
|
|
18220
|
+
leaseOwner,
|
|
18221
|
+
leaseSeconds,
|
|
18222
|
+
...(acknowledgement
|
|
18223
|
+
? [
|
|
18224
|
+
acknowledgement.eventIds,
|
|
18225
|
+
acknowledgement.targetEventId,
|
|
18226
|
+
postgresSchedulerNotifyChannel('terminal', options),
|
|
18227
|
+
]
|
|
18228
|
+
: input.extendOwnedHead
|
|
18229
|
+
? [input.extendOwnedHead]
|
|
18230
|
+
: []),
|
|
18231
|
+
],
|
|
17855
18232
|
);
|
|
18233
|
+
};
|
|
17856
18234
|
let result;
|
|
17857
18235
|
if (input.inTransaction) await client.query('SAVEPOINT run_ledger_claim');
|
|
17858
18236
|
if (supportsOutboxDeliveryBatchMarker(options)) {
|
|
@@ -17860,9 +18238,13 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17860
18238
|
result = await claim(true);
|
|
17861
18239
|
} catch (error) {
|
|
17862
18240
|
if (!isMissingOutboxDeliveryBatchMarker(error)) throw error;
|
|
18241
|
+
markOutboxDeliveryBatchMarkerUnavailable(options);
|
|
18242
|
+
// The ACK caller owns the whole short transaction. Roll it back and
|
|
18243
|
+
// repeat only this missing-column adapter, avoiding two SAVEPOINT round
|
|
18244
|
+
// trips on every successful acknowledgement.
|
|
18245
|
+
if (acknowledgement) throw error;
|
|
17863
18246
|
if (input.inTransaction)
|
|
17864
18247
|
await client.query('ROLLBACK TO SAVEPOINT run_ledger_claim');
|
|
17865
|
-
markOutboxDeliveryBatchMarkerUnavailable(options);
|
|
17866
18248
|
result = await claim(false);
|
|
17867
18249
|
}
|
|
17868
18250
|
} else {
|
|
@@ -17870,22 +18252,34 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17870
18252
|
}
|
|
17871
18253
|
if (input.inTransaction)
|
|
17872
18254
|
await client.query('RELEASE SAVEPOINT run_ledger_claim');
|
|
17873
|
-
|
|
17874
|
-
|
|
17875
|
-
|
|
17876
|
-
|
|
17877
|
-
|
|
17878
|
-
|
|
17879
|
-
|
|
17880
|
-
|
|
17881
|
-
|
|
17882
|
-
|
|
17883
|
-
|
|
17884
|
-
|
|
17885
|
-
|
|
17886
|
-
|
|
17887
|
-
|
|
17888
|
-
|
|
18255
|
+
const events = result.rows
|
|
18256
|
+
.filter((row) => row.row_kind === 'claimed')
|
|
18257
|
+
.map((row) => {
|
|
18258
|
+
if (!row.delivery_key) {
|
|
18259
|
+
throw new Error(
|
|
18260
|
+
`Run-ledger outbox event ${row.event_id} has no delivery key.`,
|
|
18261
|
+
);
|
|
18262
|
+
}
|
|
18263
|
+
return {
|
|
18264
|
+
eventId: row.event_id,
|
|
18265
|
+
runId: row.run_id,
|
|
18266
|
+
eventType: row.event_type,
|
|
18267
|
+
payload: row.payload_json,
|
|
18268
|
+
createdAt: row.created_at,
|
|
18269
|
+
leaseOwner: leaseOwner,
|
|
18270
|
+
deliveryKey: row.delivery_key,
|
|
18271
|
+
};
|
|
18272
|
+
});
|
|
18273
|
+
const deliveredRows = result.rows
|
|
18274
|
+
.filter((row) => row.row_kind === 'delivered')
|
|
18275
|
+
.map((row) => {
|
|
18276
|
+
const lagMs = Number(row.lag_ms);
|
|
18277
|
+
return {
|
|
18278
|
+
eventType: row.event_type,
|
|
18279
|
+
lagMs: Number.isFinite(lagMs) ? Math.max(0, lagMs) : 0,
|
|
18280
|
+
};
|
|
18281
|
+
});
|
|
18282
|
+
return { events, deliveredRows };
|
|
17889
18283
|
}
|
|
17890
18284
|
|
|
17891
18285
|
/**
|
|
@@ -17910,6 +18304,7 @@ export async function readPostgresSchedulerOutboxNextDueAt(
|
|
|
17910
18304
|
(SELECT max(active.lease_expires_at)
|
|
17911
18305
|
FROM ${tableName('outbox', options)} AS active
|
|
17912
18306
|
WHERE active.run_id = candidate.run_id
|
|
18307
|
+
AND ${searchLaneSql('active')} = ${searchLaneSql('candidate')}
|
|
17913
18308
|
AND active.delivered_at IS NULL AND active.lease_owner IS NOT NULL
|
|
17914
18309
|
AND active.lease_expires_at > now())
|
|
17915
18310
|
)) AS due_at
|
|
@@ -18149,7 +18544,7 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
|
|
|
18149
18544
|
'{events}',
|
|
18150
18545
|
(SELECT jsonb_agg(
|
|
18151
18546
|
CASE WHEN item.value->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
|
|
18152
|
-
THEN item.value - 'lines' - 'lineContexts'
|
|
18547
|
+
THEN item.value - 'lines' - 'lineContexts' - 'lineMetadata' - 'lineIds'
|
|
18153
18548
|
ELSE item.value
|
|
18154
18549
|
END ORDER BY item.ordinality
|
|
18155
18550
|
)
|
|
@@ -18162,11 +18557,11 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
|
|
|
18162
18557
|
)
|
|
18163
18558
|
WHEN jsonb_typeof(outbox.payload_json->'terminalProgressEvents') = 'array'
|
|
18164
18559
|
THEN jsonb_set(
|
|
18165
|
-
outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts',
|
|
18560
|
+
outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogMetadata' - 'terminalLogIds',
|
|
18166
18561
|
'{terminalProgressEvents}',
|
|
18167
18562
|
coalesce((SELECT jsonb_agg(
|
|
18168
18563
|
CASE WHEN item.value->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
|
|
18169
|
-
THEN item.value - 'lines' - 'lineContexts'
|
|
18564
|
+
THEN item.value - 'lines' - 'lineContexts' - 'lineMetadata' - 'lineIds'
|
|
18170
18565
|
ELSE item.value
|
|
18171
18566
|
END ORDER BY item.ordinality
|
|
18172
18567
|
)
|
|
@@ -18174,7 +18569,7 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
|
|
|
18174
18569
|
WITH ORDINALITY AS item(value, ordinality)), '[]'::jsonb),
|
|
18175
18570
|
false
|
|
18176
18571
|
)
|
|
18177
|
-
ELSE outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts'
|
|
18572
|
+
ELSE outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogMetadata' - 'terminalLogIds'
|
|
18178
18573
|
END
|
|
18179
18574
|
FROM candidates
|
|
18180
18575
|
WHERE outbox.event_id = candidates.event_id`,
|
|
@@ -18236,6 +18631,17 @@ function runLedgerDeliveryFailure(input: { error: unknown }): {
|
|
|
18236
18631
|
projectionMismatch: RunLedgerProjectionMismatchDiagnostic | null;
|
|
18237
18632
|
retryAfterMs: number | null;
|
|
18238
18633
|
} {
|
|
18634
|
+
if (input.error instanceof RunLedgerProjectionLeaseLostError) {
|
|
18635
|
+
return {
|
|
18636
|
+
permanent: false,
|
|
18637
|
+
code: 'RUN_LEDGER_DELIVERY_LEASE_LOST',
|
|
18638
|
+
status: null,
|
|
18639
|
+
retryable: true,
|
|
18640
|
+
failureReason: input.error.reason,
|
|
18641
|
+
projectionMismatch: input.error.projectionMismatch ?? null,
|
|
18642
|
+
retryAfterMs: null,
|
|
18643
|
+
};
|
|
18644
|
+
}
|
|
18239
18645
|
if (input.error instanceof AppRuntimeApiResponseError) {
|
|
18240
18646
|
// Projection is idempotent. A dependency can incorrectly send
|
|
18241
18647
|
// `retryable:false` with a transient overload response, but that metadata
|
|
@@ -18416,6 +18822,32 @@ export async function markPostgresSchedulerOutboxDelivered(
|
|
|
18416
18822
|
).length;
|
|
18417
18823
|
}
|
|
18418
18824
|
|
|
18825
|
+
/** Shared lease-fenced ACK, including the canonical retained-payload cleanup. */
|
|
18826
|
+
function runLedgerAcknowledgeSql(
|
|
18827
|
+
options: Pick<PostgresSchedulerOptions, 'schema'> | undefined,
|
|
18828
|
+
eventIdsParameter: string,
|
|
18829
|
+
leaseOwnerParameter: string,
|
|
18830
|
+
): string {
|
|
18831
|
+
return `UPDATE ${tableName('outbox', options)} AS outbox
|
|
18832
|
+
SET delivered_at = now(),
|
|
18833
|
+
next_attempt_at = NULL,
|
|
18834
|
+
last_error_json = NULL,
|
|
18835
|
+
lease_owner = NULL,
|
|
18836
|
+
lease_expires_at = NULL,
|
|
18837
|
+
payload_json = CASE WHEN EXISTS (
|
|
18838
|
+
SELECT 1 FROM ${tableName('runs', options)} AS run
|
|
18839
|
+
WHERE run.run_id = outbox.run_id
|
|
18840
|
+
AND run.run_event_store = 'planet_scale'
|
|
18841
|
+
) THEN '{}'::jsonb ELSE outbox.payload_json END
|
|
18842
|
+
WHERE outbox.event_id = ANY(${eventIdsParameter}::text[])
|
|
18843
|
+
AND outbox.delivered_at IS NULL
|
|
18844
|
+
AND (${leaseOwnerParameter}::text IS NULL OR outbox.lease_owner = ${leaseOwnerParameter})
|
|
18845
|
+
RETURNING outbox.event_id,
|
|
18846
|
+
outbox.event_type,
|
|
18847
|
+
(extract(epoch FROM (outbox.delivered_at - outbox.created_at)) * 1000)::float8
|
|
18848
|
+
AS lag_ms`;
|
|
18849
|
+
}
|
|
18850
|
+
|
|
18419
18851
|
/**
|
|
18420
18852
|
* Lease-fenced acknowledgement that also reports each row's event type and
|
|
18421
18853
|
* commit-to-acknowledgement lag on the database clock, so lag metrics never
|
|
@@ -18436,24 +18868,7 @@ export async function markPostgresSchedulerOutboxDeliveredRows(
|
|
|
18436
18868
|
}>(
|
|
18437
18869
|
`
|
|
18438
18870
|
WITH delivered AS (
|
|
18439
|
-
|
|
18440
|
-
SET delivered_at = now(),
|
|
18441
|
-
next_attempt_at = NULL,
|
|
18442
|
-
last_error_json = NULL,
|
|
18443
|
-
lease_owner = NULL,
|
|
18444
|
-
lease_expires_at = NULL,
|
|
18445
|
-
payload_json = CASE WHEN EXISTS (
|
|
18446
|
-
SELECT 1 FROM ${tableName('runs', options)} AS run
|
|
18447
|
-
WHERE run.run_id = outbox.run_id
|
|
18448
|
-
AND run.run_event_store = 'planet_scale'
|
|
18449
|
-
) THEN '{}'::jsonb ELSE outbox.payload_json END
|
|
18450
|
-
WHERE outbox.event_id = ANY($1::text[])
|
|
18451
|
-
AND outbox.delivered_at IS NULL
|
|
18452
|
-
AND ($2::text IS NULL OR outbox.lease_owner = $2)
|
|
18453
|
-
RETURNING outbox.event_id,
|
|
18454
|
-
outbox.event_type,
|
|
18455
|
-
(extract(epoch FROM (outbox.delivered_at - outbox.created_at)) * 1000)::float8
|
|
18456
|
-
AS lag_ms
|
|
18871
|
+
${runLedgerAcknowledgeSql(options, '$1', '$2')}
|
|
18457
18872
|
)
|
|
18458
18873
|
SELECT event_id,
|
|
18459
18874
|
event_type,
|
|
@@ -18538,6 +18953,22 @@ function normalizeOutboxIngressPreview(
|
|
|
18538
18953
|
export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
18539
18954
|
event: PostgresSchedulerOutboxEvent,
|
|
18540
18955
|
): PlayRunLedgerIngressEvent[] {
|
|
18956
|
+
if (
|
|
18957
|
+
payloadField(event.payload, RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD) ===
|
|
18958
|
+
true
|
|
18959
|
+
) {
|
|
18960
|
+
throw new Error(
|
|
18961
|
+
'Deferred outbox input must be projected before ledger delivery.',
|
|
18962
|
+
);
|
|
18963
|
+
}
|
|
18964
|
+
if (
|
|
18965
|
+
payloadField(event.payload, RUNTIME_EVENT_SEARCH_ONLY_FIELD) === true ||
|
|
18966
|
+
event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.receiptEvent ||
|
|
18967
|
+
event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runnerLogs
|
|
18968
|
+
) {
|
|
18969
|
+
// Search-only carriers are already durable in the canonical journal.
|
|
18970
|
+
return [];
|
|
18971
|
+
}
|
|
18541
18972
|
const occurredAt = event.createdAt.getTime();
|
|
18542
18973
|
if (event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents) {
|
|
18543
18974
|
const events = payloadField(event.payload, 'events');
|
|
@@ -18591,6 +19022,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
18591
19022
|
);
|
|
18592
19023
|
const lines = stringArrayPayloadField(event.payload, 'terminalLogLines');
|
|
18593
19024
|
const lineContexts = terminalLogContextsPayloadField(event.payload, lines);
|
|
19025
|
+
const lineMetadata = terminalLogMetadataPayloadField(event.payload, lines);
|
|
18594
19026
|
const resultSummary = payloadField(event.payload, 'resultSummary');
|
|
18595
19027
|
return [
|
|
18596
19028
|
...progressEvents.map(normalizeOutboxIngressPreview),
|
|
@@ -18604,6 +19036,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
18604
19036
|
source: 'worker',
|
|
18605
19037
|
lines,
|
|
18606
19038
|
...(lineContexts ? { lineContexts } : {}),
|
|
19039
|
+
...(lineMetadata ? { lineMetadata } : {}),
|
|
18607
19040
|
} as PlayRunLedgerEvent,
|
|
18608
19041
|
]),
|
|
18609
19042
|
{
|
|
@@ -18633,6 +19066,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
18633
19066
|
);
|
|
18634
19067
|
const lines = stringArrayPayloadField(event.payload, 'terminalLogLines');
|
|
18635
19068
|
const lineContexts = terminalLogContextsPayloadField(event.payload, lines);
|
|
19069
|
+
const lineMetadata = terminalLogMetadataPayloadField(event.payload, lines);
|
|
18636
19070
|
const result = terminalRunResultForLedger(
|
|
18637
19071
|
payloadField(event.payload, 'result'),
|
|
18638
19072
|
);
|
|
@@ -18649,6 +19083,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
18649
19083
|
source: 'worker',
|
|
18650
19084
|
lines,
|
|
18651
19085
|
...(lineContexts ? { lineContexts } : {}),
|
|
19086
|
+
...(lineMetadata ? { lineMetadata } : {}),
|
|
18652
19087
|
} as PlayRunLedgerEvent,
|
|
18653
19088
|
]),
|
|
18654
19089
|
{
|
|
@@ -18774,6 +19209,8 @@ export type PostgresSchedulerOutboxMaterializationPlan = {
|
|
|
18774
19209
|
};
|
|
18775
19210
|
|
|
18776
19211
|
const RUN_LEDGER_KNOWN_OUTBOX_EVENT_TYPES: ReadonlySet<string> = new Set([
|
|
19212
|
+
RUN_SCHEDULER_OUTBOX_EVENT_TYPE.receiptEvent,
|
|
19213
|
+
RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runnerLogs,
|
|
18777
19214
|
RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents,
|
|
18778
19215
|
RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runStarted,
|
|
18779
19216
|
RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runCompleted,
|
|
@@ -18891,7 +19328,7 @@ export async function listPostgresSchedulerOutboxMaterializationPlans(
|
|
|
18891
19328
|
* - `local_mint`: sign the token in-process with `BETTER_AUTH_SECRET`. ONLY the
|
|
18892
19329
|
* app (Vercel) legitimately holds the signing key, so this is app-side only
|
|
18893
19330
|
* (read-model heal path). It is the AUTHORITATIVE projector.
|
|
18894
|
-
* - `gateway`:
|
|
19331
|
+
* - `gateway`: for every app-runtime append, prove the active outbox lease to the launch-pinned
|
|
18895
19332
|
* deployment and receive a short-lived executor JWT
|
|
18896
19333
|
* (`requestRunLedgerDispatchToken`), even if the caller supplied a
|
|
18897
19334
|
* `runtimeApi`. The Fly worker has no signing key and MUST use this. A
|
|
@@ -18912,6 +19349,21 @@ export type RunLedgerOutboxTokenAuthority = 'local_mint' | 'gateway';
|
|
|
18912
19349
|
/** Cross-run service policy; causal order within a run is always unchanged. */
|
|
18913
19350
|
export type RunLedgerOutboxHeadPriority = 'admission_first' | 'oldest_ready';
|
|
18914
19351
|
|
|
19352
|
+
class DeferredRuntimeProjectionInputError extends AppRuntimeApiResponseError {
|
|
19353
|
+
constructor(
|
|
19354
|
+
readonly eventId: string,
|
|
19355
|
+
detail: string,
|
|
19356
|
+
) {
|
|
19357
|
+
super({
|
|
19358
|
+
action: 'project_run_events',
|
|
19359
|
+
status: 422,
|
|
19360
|
+
code: 'runtime_event_projection_invalid',
|
|
19361
|
+
retryable: false,
|
|
19362
|
+
detail,
|
|
19363
|
+
});
|
|
19364
|
+
}
|
|
19365
|
+
}
|
|
19366
|
+
|
|
18915
19367
|
const POSTGRES_SCHEDULER_CANCELLATION_OUTBOX_LEASE_SECONDS =
|
|
18916
19368
|
// The direct projection can run for the full app-runtime request deadline.
|
|
18917
19369
|
// Keep another 30 seconds for dispatch setup and acknowledgement so a
|
|
@@ -18942,8 +19394,12 @@ export function postgresSchedulerCancellationOutboxLease(runId: string): {
|
|
|
18942
19394
|
}
|
|
18943
19395
|
|
|
18944
19396
|
export type RunLedgerOutboxDispatchInput = {
|
|
19397
|
+
/** Only the existing background projector claims the independent search prefix. */
|
|
19398
|
+
searchOnly?: boolean;
|
|
19399
|
+
/** Only the existing background projector indexes deferred raw inputs. */
|
|
19400
|
+
materializeObservability?: boolean;
|
|
18945
19401
|
/**
|
|
18946
|
-
* Optional runtime context. `gateway`
|
|
19402
|
+
* Optional runtime context. For an app-runtime append, `gateway` replaces its
|
|
18947
19403
|
* credentials with a fresh lease-scoped token and keeps only supported
|
|
18948
19404
|
* transport options; `local_mint` and the default authority preserve the
|
|
18949
19405
|
* explicit context as before.
|
|
@@ -18983,8 +19439,9 @@ export type RunLedgerOutboxDispatchInput = {
|
|
|
18983
19439
|
* untargeted fast path) set this: after acknowledging a batch, also carry
|
|
18984
19440
|
* the run's terminal fact through when it is now pending (see
|
|
18985
19441
|
* `continueRunLedgerOutboxThroughTerminal`). Callers that drive their own
|
|
18986
|
-
* bounded batches
|
|
18987
|
-
*
|
|
19442
|
+
* bounded batches (cancellation, inline `run.created`, the targeted terminal
|
|
19443
|
+
* drain) leave it unset. Targeted drains retain successors through their
|
|
19444
|
+
* explicit run/owner/target scope instead.
|
|
18988
19445
|
*/
|
|
18989
19446
|
continueThroughTerminal?: boolean;
|
|
18990
19447
|
};
|
|
@@ -19066,7 +19523,10 @@ type RunLedgerOutboxAuthorityScope = {
|
|
|
19066
19523
|
authority: {
|
|
19067
19524
|
runId: string;
|
|
19068
19525
|
leaseOwner: string;
|
|
19069
|
-
runtimeApi: WorkerRuntimeApiContext
|
|
19526
|
+
runtimeApi: WorkerRuntimeApiContext & {
|
|
19527
|
+
refreshExecutorToken?: () => Promise<string>;
|
|
19528
|
+
invalidateExecutorToken?: () => void;
|
|
19529
|
+
};
|
|
19070
19530
|
expiresAtMs: number | null;
|
|
19071
19531
|
} | null;
|
|
19072
19532
|
};
|
|
@@ -19101,6 +19561,9 @@ async function dispatchRunLedgerOutboxThroughEventWithScope(
|
|
|
19101
19561
|
input.leaseOwner?.trim() ||
|
|
19102
19562
|
scope?.leaseOwner ||
|
|
19103
19563
|
`run-ledger-outbox-dispatch:${randomUUID()}`;
|
|
19564
|
+
// The same owner retains each successor in the ACK transaction, including
|
|
19565
|
+
// for a fresh worker drain that did not arrive with a preclaimed prefix.
|
|
19566
|
+
scope ??= { runId, leaseOwner, targetEventId, nextClaimed: [] };
|
|
19104
19567
|
const scopedInput = { ...input, leaseOwner };
|
|
19105
19568
|
const batchSize = RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS;
|
|
19106
19569
|
const eventIds: string[] = [];
|
|
@@ -19111,80 +19574,92 @@ async function dispatchRunLedgerOutboxThroughEventWithScope(
|
|
|
19111
19574
|
let failed = 0;
|
|
19112
19575
|
let retrying = 0;
|
|
19113
19576
|
|
|
19114
|
-
|
|
19115
|
-
|
|
19116
|
-
|
|
19117
|
-
|
|
19118
|
-
|
|
19119
|
-
|
|
19120
|
-
|
|
19121
|
-
|
|
19122
|
-
|
|
19123
|
-
|
|
19124
|
-
|
|
19125
|
-
|
|
19126
|
-
|
|
19127
|
-
|
|
19128
|
-
|
|
19129
|
-
|
|
19130
|
-
|
|
19131
|
-
|
|
19132
|
-
|
|
19133
|
-
|
|
19134
|
-
|
|
19135
|
-
|
|
19136
|
-
|
|
19137
|
-
|
|
19138
|
-
|
|
19139
|
-
|
|
19140
|
-
|
|
19141
|
-
|
|
19142
|
-
|
|
19143
|
-
|
|
19144
|
-
|
|
19145
|
-
|
|
19146
|
-
|
|
19147
|
-
|
|
19148
|
-
|
|
19577
|
+
try {
|
|
19578
|
+
let remaining = maxEvents;
|
|
19579
|
+
while (remaining === null || remaining > 0) {
|
|
19580
|
+
const result = await dispatchRunLedgerOutboxBatchWithScope(
|
|
19581
|
+
withClient,
|
|
19582
|
+
{
|
|
19583
|
+
...scopedInput,
|
|
19584
|
+
waitForRunLock: true,
|
|
19585
|
+
continueThroughTerminal: false,
|
|
19586
|
+
limit:
|
|
19587
|
+
remaining === null ? batchSize : Math.min(batchSize, remaining),
|
|
19588
|
+
},
|
|
19589
|
+
options,
|
|
19590
|
+
scope,
|
|
19591
|
+
authorityScope,
|
|
19592
|
+
remaining,
|
|
19593
|
+
);
|
|
19594
|
+
attempted += result.attempted;
|
|
19595
|
+
projected += result.projected;
|
|
19596
|
+
failed += result.failed;
|
|
19597
|
+
retrying += result.retrying;
|
|
19598
|
+
eventIds.push(...result.eventIds);
|
|
19599
|
+
failedEventIds.push(...result.failedEventIds);
|
|
19600
|
+
retryingEventIds.push(...result.retryingEventIds);
|
|
19601
|
+
if (remaining !== null) remaining -= result.attempted;
|
|
19602
|
+
if (result.eventIds.includes(targetEventId)) {
|
|
19603
|
+
return {
|
|
19604
|
+
attempted,
|
|
19605
|
+
projected,
|
|
19606
|
+
eventIds,
|
|
19607
|
+
failed,
|
|
19608
|
+
failedEventIds,
|
|
19609
|
+
retrying,
|
|
19610
|
+
retryingEventIds,
|
|
19611
|
+
targetEventId,
|
|
19612
|
+
targetDelivered: true,
|
|
19613
|
+
targetBlocked: false,
|
|
19614
|
+
};
|
|
19615
|
+
}
|
|
19616
|
+
// A projection failure is durably scheduled for retry or moved to terminal
|
|
19617
|
+
// DLQ. Its policy owns the next attempt; spinning here would only reclaim
|
|
19618
|
+
// the same failed predecessor and delay the park indefinitely.
|
|
19619
|
+
// An absent target cannot retain a successor in ACK. Keep the existing
|
|
19620
|
+
// fresh-claim traversal in that case until ready work is exhausted.
|
|
19621
|
+
if (result.projected === 0 || result.attempted === 0) break;
|
|
19149
19622
|
}
|
|
19150
|
-
// A projection failure is durably scheduled for retry or moved to terminal
|
|
19151
|
-
// DLQ. Its policy owns the next attempt; spinning here would only reclaim
|
|
19152
|
-
// the same failed predecessor and delay the park indefinitely.
|
|
19153
|
-
if (
|
|
19154
|
-
result.projected === 0 ||
|
|
19155
|
-
result.attempted === 0 ||
|
|
19156
|
-
(scope && scope.nextClaimed.length === 0)
|
|
19157
|
-
)
|
|
19158
|
-
break;
|
|
19159
|
-
}
|
|
19160
|
-
|
|
19161
|
-
// A concurrent dispatcher can claim the target between batches. An empty
|
|
19162
|
-
// drain therefore does not prove delivery; read the durable acknowledgement
|
|
19163
|
-
// before reporting the target as pending.
|
|
19164
|
-
const target = await withClient((client) =>
|
|
19165
|
-
readPostgresSchedulerOutboxDeliveryStatus(
|
|
19166
|
-
client,
|
|
19167
|
-
{ runId, eventId: targetEventId },
|
|
19168
|
-
options,
|
|
19169
|
-
),
|
|
19170
|
-
);
|
|
19171
|
-
return {
|
|
19172
|
-
attempted,
|
|
19173
|
-
projected,
|
|
19174
|
-
eventIds,
|
|
19175
|
-
failed,
|
|
19176
|
-
failedEventIds,
|
|
19177
|
-
retrying,
|
|
19178
|
-
retryingEventIds,
|
|
19179
|
-
targetEventId,
|
|
19180
|
-
targetDelivered: target.delivered,
|
|
19181
|
-
targetBlocked: target.blocked,
|
|
19182
|
-
};
|
|
19183
|
-
}
|
|
19184
19623
|
|
|
19185
|
-
|
|
19186
|
-
|
|
19187
|
-
|
|
19624
|
+
// A concurrent dispatcher can claim the target between batches. An empty
|
|
19625
|
+
// drain therefore does not prove delivery; read the durable acknowledgement
|
|
19626
|
+
// before reporting the target as pending.
|
|
19627
|
+
const target = await withClient((client) =>
|
|
19628
|
+
readPostgresSchedulerOutboxDeliveryStatus(
|
|
19629
|
+
client,
|
|
19630
|
+
{ runId, eventId: targetEventId },
|
|
19631
|
+
options,
|
|
19632
|
+
),
|
|
19633
|
+
);
|
|
19634
|
+
return {
|
|
19635
|
+
attempted,
|
|
19636
|
+
projected,
|
|
19637
|
+
eventIds,
|
|
19638
|
+
failed,
|
|
19639
|
+
failedEventIds,
|
|
19640
|
+
retrying,
|
|
19641
|
+
retryingEventIds,
|
|
19642
|
+
targetEventId,
|
|
19643
|
+
targetDelivered: target.delivered,
|
|
19644
|
+
targetBlocked: target.blocked,
|
|
19645
|
+
};
|
|
19646
|
+
} finally {
|
|
19647
|
+
if (scope.nextClaimed.length > 0) {
|
|
19648
|
+
await withClient((client) =>
|
|
19649
|
+
releasePostgresSchedulerOutboxLease(
|
|
19650
|
+
client,
|
|
19651
|
+
scope.nextClaimed.map((event) => event.eventId),
|
|
19652
|
+
{ ...options, leaseOwner },
|
|
19653
|
+
),
|
|
19654
|
+
);
|
|
19655
|
+
scope.nextClaimed.length = 0;
|
|
19656
|
+
}
|
|
19657
|
+
}
|
|
19658
|
+
}
|
|
19659
|
+
|
|
19660
|
+
/**
|
|
19661
|
+
* Project one run's currently due non-start lifecycle events in one app-runtime
|
|
19662
|
+
* append. The outbox remains the authority: all rows are leased first and are
|
|
19188
19663
|
* marked delivered only after the atomic append succeeds. This preserves event
|
|
19189
19664
|
* order while avoiding one callback round trip for `waiting -> resumed ->
|
|
19190
19665
|
* terminal`, the normal detached-run completion shape.
|
|
@@ -19219,10 +19694,20 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19219
19694
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
19220
19695
|
scope?: RunLedgerOutboxDrainScope,
|
|
19221
19696
|
authorityScope: RunLedgerOutboxAuthorityScope = { authority: null },
|
|
19697
|
+
remainingEvents: number | null = null,
|
|
19222
19698
|
): Promise<PostgresSchedulerOutboxProjectionResult> {
|
|
19223
19699
|
const dispatchStartedAtMs = Date.now();
|
|
19700
|
+
const tokenAuthority = input.tokenAuthority ?? 'local_mint';
|
|
19224
19701
|
const leaseOwner =
|
|
19225
19702
|
input.leaseOwner?.trim() || `run-ledger-outbox-dispatch:${randomUUID()}`;
|
|
19703
|
+
const needsLaunch =
|
|
19704
|
+
!input.searchOnly && (tokenAuthority === 'gateway' || !input.runtimeApi);
|
|
19705
|
+
const launchSelection = needsLaunch
|
|
19706
|
+
? `CASE
|
|
19707
|
+
WHEN ${runLedgerSearchOnlyOutboxFactSql('head')} THEN NULL::jsonb
|
|
19708
|
+
ELSE ${runLedgerDispatchLaunchSql('runs')}
|
|
19709
|
+
END`
|
|
19710
|
+
: 'NULL::jsonb';
|
|
19226
19711
|
const continuation: { current: RunLedgerOutboxDrainScope | null } = {
|
|
19227
19712
|
current: scope ?? null,
|
|
19228
19713
|
};
|
|
@@ -19272,22 +19757,27 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19272
19757
|
// for each lifecycle transition. Only the two dedicated projectors may
|
|
19273
19758
|
// perform the global distinct-head scan below.
|
|
19274
19759
|
const selected = targetRunId
|
|
19275
|
-
? await client.query<{
|
|
19760
|
+
? await client.query<{
|
|
19761
|
+
run_id: string;
|
|
19762
|
+
dispatch_launch_json: unknown;
|
|
19763
|
+
}>(
|
|
19276
19764
|
`
|
|
19277
19765
|
WITH head AS MATERIALIZED (
|
|
19278
19766
|
SELECT pending.run_id,
|
|
19279
19767
|
pending.event_id,
|
|
19280
19768
|
pending.event_type,
|
|
19769
|
+
pending.payload_json,
|
|
19281
19770
|
pending.created_at,
|
|
19282
19771
|
pending.next_attempt_at,
|
|
19283
19772
|
pending.failed_at
|
|
19284
19773
|
FROM ${tableName('outbox', options)} AS pending
|
|
19285
19774
|
WHERE pending.run_id = $1
|
|
19286
19775
|
AND pending.delivered_at IS NULL
|
|
19776
|
+
AND ${searchLaneSql('pending')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
19287
19777
|
ORDER BY ${runLedgerOrderSql('pending')}
|
|
19288
19778
|
LIMIT 1
|
|
19289
19779
|
)
|
|
19290
|
-
SELECT runs.run_id
|
|
19780
|
+
SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json
|
|
19291
19781
|
FROM head
|
|
19292
19782
|
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
|
|
19293
19783
|
WHERE head.failed_at IS NULL
|
|
@@ -19296,6 +19786,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19296
19786
|
AND NOT EXISTS (
|
|
19297
19787
|
SELECT 1 FROM ${tableName('outbox', options)} AS active
|
|
19298
19788
|
WHERE active.run_id = runs.run_id
|
|
19789
|
+
AND ${searchLaneSql('active')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
19299
19790
|
AND active.delivered_at IS NULL
|
|
19300
19791
|
AND active.lease_owner IS NOT NULL
|
|
19301
19792
|
AND active.lease_expires_at > now()
|
|
@@ -19305,20 +19796,31 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19305
19796
|
`,
|
|
19306
19797
|
[targetRunId, leaseOwner, input.headEventId?.trim() || null],
|
|
19307
19798
|
)
|
|
19308
|
-
: await client.query<{
|
|
19309
|
-
|
|
19799
|
+
: await client.query<{
|
|
19800
|
+
run_id: string;
|
|
19801
|
+
dispatch_launch_json: unknown;
|
|
19802
|
+
}>(`
|
|
19803
|
+
WITH ${
|
|
19804
|
+
input.searchOnly === true
|
|
19805
|
+
? ''
|
|
19806
|
+
: `waiting_runs AS MATERIALIZED (
|
|
19807
|
+
${runLedgerWaitingRunsSql(options)}
|
|
19808
|
+
), `
|
|
19809
|
+
}heads AS MATERIALIZED (
|
|
19310
19810
|
SELECT DISTINCT ON (pending.run_id)
|
|
19311
19811
|
pending.run_id,
|
|
19312
19812
|
pending.event_id,
|
|
19313
19813
|
pending.event_type,
|
|
19814
|
+
pending.payload_json,
|
|
19314
19815
|
pending.created_at,
|
|
19315
19816
|
pending.next_attempt_at,
|
|
19316
19817
|
pending.failed_at
|
|
19317
19818
|
FROM ${tableName('outbox', options)} AS pending
|
|
19318
19819
|
WHERE pending.delivered_at IS NULL
|
|
19820
|
+
AND ${searchLaneSql('pending')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
19319
19821
|
ORDER BY pending.run_id, ${runLedgerOrderSql('pending')}
|
|
19320
19822
|
)
|
|
19321
|
-
SELECT runs.run_id
|
|
19823
|
+
SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json
|
|
19322
19824
|
FROM heads AS head
|
|
19323
19825
|
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
|
|
19324
19826
|
WHERE head.failed_at IS NULL
|
|
@@ -19326,12 +19828,13 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19326
19828
|
AND NOT EXISTS (
|
|
19327
19829
|
SELECT 1 FROM ${tableName('outbox', options)} AS active
|
|
19328
19830
|
WHERE active.run_id = runs.run_id
|
|
19831
|
+
AND ${searchLaneSql('active')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
19329
19832
|
AND active.delivered_at IS NULL
|
|
19330
19833
|
AND active.lease_owner IS NOT NULL
|
|
19331
19834
|
AND active.lease_expires_at > now()
|
|
19332
19835
|
)
|
|
19333
19836
|
-- admission_first: admission, then the causal head of a run with
|
|
19334
|
-
-- a pending terminal, then FIFO progress. oldest_ready keeps its
|
|
19837
|
+
-- a pending terminal or public wait, then FIFO progress. oldest_ready keeps its
|
|
19335
19838
|
-- pure age order so continuous admissions/terminals cannot starve
|
|
19336
19839
|
-- ordinary progress.
|
|
19337
19840
|
ORDER BY ${
|
|
@@ -19340,7 +19843,8 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19340
19843
|
: `CASE
|
|
19341
19844
|
WHEN head.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0
|
|
19342
19845
|
WHEN ${runLedgerTerminalPendingSql('head', options)} THEN 1
|
|
19343
|
-
|
|
19846
|
+
${input.searchOnly === true ? '' : `WHEN ${runLedgerWaitingPendingSql('head')} THEN 2`}
|
|
19847
|
+
ELSE 3
|
|
19344
19848
|
END,`
|
|
19345
19849
|
}
|
|
19346
19850
|
head.created_at, runs.run_id
|
|
@@ -19356,20 +19860,33 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19356
19860
|
leaseOwner,
|
|
19357
19861
|
leaseSeconds: input.leaseSeconds ?? 120,
|
|
19358
19862
|
inTransaction: true,
|
|
19863
|
+
searchOnly: input.searchOnly,
|
|
19359
19864
|
},
|
|
19360
19865
|
options,
|
|
19361
19866
|
)
|
|
19362
19867
|
: [];
|
|
19363
19868
|
await client.query('COMMIT');
|
|
19364
|
-
|
|
19869
|
+
// The launch is immutable. Carry only its authority/routing fields from
|
|
19870
|
+
// the run we already locked, rather than queuing another scarce checkout
|
|
19871
|
+
// between the claim and gateway request. The gateway still proves the
|
|
19872
|
+
// active outbox lease; no database checkout spans either HTTP hop.
|
|
19873
|
+
return {
|
|
19874
|
+
events: claimed,
|
|
19875
|
+
launchJson: selected.rows[0]?.dispatch_launch_json,
|
|
19876
|
+
};
|
|
19365
19877
|
} catch (error) {
|
|
19366
19878
|
await client.query('ROLLBACK');
|
|
19367
19879
|
throw error;
|
|
19368
19880
|
}
|
|
19369
19881
|
};
|
|
19370
|
-
const
|
|
19882
|
+
const preclaimed = scope?.nextClaimed.length
|
|
19371
19883
|
? scope.nextClaimed.splice(0)
|
|
19884
|
+
: null;
|
|
19885
|
+
const claimed = preclaimed
|
|
19886
|
+
? { events: preclaimed, launchJson: null }
|
|
19372
19887
|
: await withClient(claimEvents);
|
|
19888
|
+
const { launchJson } = claimed;
|
|
19889
|
+
let events = claimed.events;
|
|
19373
19890
|
if (events.length === 0) {
|
|
19374
19891
|
return {
|
|
19375
19892
|
attempted: 0,
|
|
@@ -19386,17 +19903,160 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19386
19903
|
let authorityReadyAtMs: number | null = null;
|
|
19387
19904
|
let appendFinishedAtMs: number | null = null;
|
|
19388
19905
|
let acknowledgedAtMs: number | null = null;
|
|
19906
|
+
const materializeDeferredObservability = async () => {
|
|
19907
|
+
if (
|
|
19908
|
+
events.some(
|
|
19909
|
+
(event) =>
|
|
19910
|
+
payloadField(
|
|
19911
|
+
event.payload,
|
|
19912
|
+
RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD,
|
|
19913
|
+
) === true,
|
|
19914
|
+
)
|
|
19915
|
+
) {
|
|
19916
|
+
if (!input.materializeObservability) {
|
|
19917
|
+
// Inline admission/lifecycle/read-heal paths must never index logs.
|
|
19918
|
+
// Release and wake the existing projector without charging a retry.
|
|
19919
|
+
const released = await withClient((client) =>
|
|
19920
|
+
releasePostgresSchedulerOutboxLease(
|
|
19921
|
+
client,
|
|
19922
|
+
events.map((event) => event.eventId),
|
|
19923
|
+
{ ...options, leaseOwner },
|
|
19924
|
+
),
|
|
19925
|
+
);
|
|
19926
|
+
return {
|
|
19927
|
+
attempted: events.length,
|
|
19928
|
+
projected: 0,
|
|
19929
|
+
eventIds: [],
|
|
19930
|
+
failed: 0,
|
|
19931
|
+
failedEventIds: [],
|
|
19932
|
+
retrying: released.length,
|
|
19933
|
+
retryingEventIds: released,
|
|
19934
|
+
};
|
|
19935
|
+
}
|
|
19936
|
+
for (const event of events) {
|
|
19937
|
+
if (
|
|
19938
|
+
payloadField(
|
|
19939
|
+
event.payload,
|
|
19940
|
+
RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD,
|
|
19941
|
+
) !== true
|
|
19942
|
+
)
|
|
19943
|
+
continue;
|
|
19944
|
+
await withClient(async (client) => {
|
|
19945
|
+
await renewPostgresRunLedgerDelivery(
|
|
19946
|
+
client,
|
|
19947
|
+
{
|
|
19948
|
+
eventIds: events.map((row) => row.eventId),
|
|
19949
|
+
leaseOwner,
|
|
19950
|
+
leaseSeconds: 120,
|
|
19951
|
+
},
|
|
19952
|
+
options,
|
|
19953
|
+
);
|
|
19954
|
+
// Keep projection below its renewed lease. The final payload update
|
|
19955
|
+
// fences ownership; indexing locks neither outbox nor Run rows.
|
|
19956
|
+
await client.query('BEGIN');
|
|
19957
|
+
try {
|
|
19958
|
+
// Let PostgreSQL cancel before the pool's client-side timeout,
|
|
19959
|
+
// so rollback completes while the checkout is still usable.
|
|
19960
|
+
await client.query(
|
|
19961
|
+
"SELECT set_config('statement_timeout', $1, true)",
|
|
19962
|
+
[
|
|
19963
|
+
`${Math.min(60_000, RUNTIME_RELIABILITY_POLICY.schedulerDatabase.queryTimeoutMs - 1_000)}ms`,
|
|
19964
|
+
],
|
|
19965
|
+
);
|
|
19966
|
+
const result = await client.query<{ payload: unknown }>(
|
|
19967
|
+
`SELECT ${quoteIdent(schemaName(options))}.materialize_runtime_outbox_event($1, $2) AS payload`,
|
|
19968
|
+
[event.eventId, leaseOwner],
|
|
19969
|
+
);
|
|
19970
|
+
await client.query('COMMIT');
|
|
19971
|
+
event.payload = result.rows[0]!.payload;
|
|
19972
|
+
} catch (error) {
|
|
19973
|
+
await client.query('ROLLBACK');
|
|
19974
|
+
if (
|
|
19975
|
+
error &&
|
|
19976
|
+
typeof error === 'object' &&
|
|
19977
|
+
'code' in error &&
|
|
19978
|
+
error.code === 'P0001'
|
|
19979
|
+
) {
|
|
19980
|
+
// The installed projector's explicit RAISEs reject malformed
|
|
19981
|
+
// or conflicting immutable facts. Retain them in the existing
|
|
19982
|
+
// quarantine instead of retrying the same invalid input forever.
|
|
19983
|
+
throw new DeferredRuntimeProjectionInputError(
|
|
19984
|
+
event.eventId,
|
|
19985
|
+
error instanceof Error
|
|
19986
|
+
? error.message
|
|
19987
|
+
: 'Invalid runtime event projection input.',
|
|
19988
|
+
);
|
|
19989
|
+
}
|
|
19990
|
+
throw error;
|
|
19991
|
+
}
|
|
19992
|
+
});
|
|
19993
|
+
}
|
|
19994
|
+
}
|
|
19995
|
+
return null;
|
|
19996
|
+
};
|
|
19389
19997
|
try {
|
|
19998
|
+
// The ledger converter rejects raw deferred facts. Inline callers hand
|
|
19999
|
+
// them off; the background owner projects them before inspecting the
|
|
20000
|
+
// compact lifecycle payload or deciding that it needs no HTTP append.
|
|
20001
|
+
const initialHandedOff = await materializeDeferredObservability();
|
|
20002
|
+
if (initialHandedOff) return initialHandedOff;
|
|
20003
|
+
if (input.searchOnly) {
|
|
20004
|
+
if (
|
|
20005
|
+
events.some(
|
|
20006
|
+
(event) =>
|
|
20007
|
+
payloadField(event.payload, RUNTIME_EVENT_SEARCH_ONLY_FIELD) !==
|
|
20008
|
+
true,
|
|
20009
|
+
)
|
|
20010
|
+
)
|
|
20011
|
+
throw new Error('Search lease includes a lifecycle fact.');
|
|
20012
|
+
const delivered = await withClient((client) =>
|
|
20013
|
+
markPostgresSchedulerOutboxDeliveredRows(
|
|
20014
|
+
client,
|
|
20015
|
+
events.map((event) => event.eventId),
|
|
20016
|
+
{ ...options, leaseOwner },
|
|
20017
|
+
),
|
|
20018
|
+
);
|
|
20019
|
+
if (delivered.length !== events.length)
|
|
20020
|
+
throw new RunLedgerProjectionLeaseLostError(
|
|
20021
|
+
'delivery_lease_not_current',
|
|
20022
|
+
);
|
|
20023
|
+
if (input.onDelivered) {
|
|
20024
|
+
try {
|
|
20025
|
+
input.onDelivered(delivered);
|
|
20026
|
+
} catch (error) {
|
|
20027
|
+
console.warn('[run-ledger-outbox.delivery_observer_failed]', {
|
|
20028
|
+
error: error instanceof Error ? error.message : String(error),
|
|
20029
|
+
});
|
|
20030
|
+
}
|
|
20031
|
+
}
|
|
20032
|
+
return {
|
|
20033
|
+
attempted: events.length,
|
|
20034
|
+
projected: delivered.length,
|
|
20035
|
+
eventIds: events.map((event) => event.eventId),
|
|
20036
|
+
failed: 0,
|
|
20037
|
+
failedEventIds: [],
|
|
20038
|
+
retrying: 0,
|
|
20039
|
+
retryingEventIds: [],
|
|
20040
|
+
};
|
|
20041
|
+
}
|
|
19390
20042
|
const first = events[0]!;
|
|
19391
|
-
|
|
20043
|
+
let ledgerEvents = RUN_SCHEDULER_OUTBOX_START_EVENT_TYPES.includes(
|
|
20044
|
+
first.eventType,
|
|
20045
|
+
)
|
|
20046
|
+
? null
|
|
20047
|
+
: events.flatMap(buildRunLedgerEventsFromPostgresSchedulerOutbox);
|
|
20048
|
+
// Search-only facts are already durable and have no ledger projection.
|
|
20049
|
+
// Validate and acknowledge them without acquiring runtime authority or
|
|
20050
|
+
// renewing a network-delivery lease. A following terminal still uses the
|
|
20051
|
+
// same fenced continuation and loads authority only when it needs a write.
|
|
19392
20052
|
if (
|
|
19393
20053
|
scope &&
|
|
19394
20054
|
(scope.runId !== first.runId || scope.leaseOwner !== leaseOwner)
|
|
19395
20055
|
) {
|
|
19396
20056
|
throw new Error('Run-ledger continuation is outside its dispatch scope.');
|
|
19397
20057
|
}
|
|
19398
|
-
|
|
19399
|
-
|
|
20058
|
+
const needsRuntimeApi = ledgerEvents === null || ledgerEvents.length > 0;
|
|
20059
|
+
let runtimeApi: WorkerRuntimeApiContext | null = null;
|
|
19400
20060
|
const reuseMarginMs =
|
|
19401
20061
|
Math.max(
|
|
19402
20062
|
1,
|
|
@@ -19406,23 +20066,36 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19406
20066
|
),
|
|
19407
20067
|
) + RUNTIME_RELIABILITY_POLICY.schedulerDatabase.queryTimeoutMs;
|
|
19408
20068
|
const cachedAuthority = authorityScope.authority;
|
|
20069
|
+
const resolveRunRuntimeApi = () =>
|
|
20070
|
+
launchJson !== null
|
|
20071
|
+
? resolvePostgresSchedulerRuntimeApiForRun(
|
|
20072
|
+
launchJson,
|
|
20073
|
+
first.runId,
|
|
20074
|
+
first.leaseOwner
|
|
20075
|
+
? { eventId: first.eventId, leaseOwner: first.leaseOwner }
|
|
20076
|
+
: null,
|
|
20077
|
+
tokenAuthority,
|
|
20078
|
+
options,
|
|
20079
|
+
input.runtimeApi?.signal,
|
|
20080
|
+
)
|
|
20081
|
+
: loadPostgresSchedulerRuntimeApiForRun(
|
|
20082
|
+
withClient,
|
|
20083
|
+
first.runId,
|
|
20084
|
+
first.leaseOwner
|
|
20085
|
+
? { eventId: first.eventId, leaseOwner: first.leaseOwner }
|
|
20086
|
+
: null,
|
|
20087
|
+
tokenAuthority,
|
|
20088
|
+
options,
|
|
20089
|
+
input.runtimeApi?.signal,
|
|
20090
|
+
);
|
|
19409
20091
|
const gatewayRuntimeApi =
|
|
19410
|
-
tokenAuthority === 'gateway'
|
|
20092
|
+
needsRuntimeApi && tokenAuthority === 'gateway'
|
|
19411
20093
|
? cachedAuthority?.runId === first.runId &&
|
|
19412
20094
|
cachedAuthority.leaseOwner === leaseOwner &&
|
|
19413
20095
|
cachedAuthority.expiresAtMs !== null &&
|
|
19414
20096
|
cachedAuthority.expiresAtMs > Date.now() + reuseMarginMs
|
|
19415
20097
|
? cachedAuthority.runtimeApi
|
|
19416
|
-
: await
|
|
19417
|
-
withClient,
|
|
19418
|
-
first.runId,
|
|
19419
|
-
first.leaseOwner
|
|
19420
|
-
? { eventId: first.eventId, leaseOwner: first.leaseOwner }
|
|
19421
|
-
: null,
|
|
19422
|
-
tokenAuthority,
|
|
19423
|
-
options,
|
|
19424
|
-
input.runtimeApi?.signal,
|
|
19425
|
-
)
|
|
20098
|
+
: await resolveRunRuntimeApi()
|
|
19426
20099
|
: null;
|
|
19427
20100
|
if (
|
|
19428
20101
|
gatewayRuntimeApi &&
|
|
@@ -19434,67 +20107,101 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19434
20107
|
'Run Ledger dispatch origin changed within an immutable launch.',
|
|
19435
20108
|
);
|
|
19436
20109
|
}
|
|
19437
|
-
|
|
19438
|
-
|
|
19439
|
-
|
|
19440
|
-
|
|
19441
|
-
|
|
19442
|
-
|
|
19443
|
-
|
|
19444
|
-
|
|
19445
|
-
|
|
19446
|
-
|
|
19447
|
-
|
|
19448
|
-
|
|
19449
|
-
|
|
19450
|
-
|
|
19451
|
-
|
|
19452
|
-
|
|
19453
|
-
|
|
19454
|
-
|
|
19455
|
-
|
|
19456
|
-
|
|
19457
|
-
|
|
19458
|
-
|
|
19459
|
-
|
|
19460
|
-
|
|
19461
|
-
|
|
19462
|
-
|
|
19463
|
-
|
|
19464
|
-
|
|
19465
|
-
|
|
19466
|
-
|
|
19467
|
-
: {}),
|
|
19468
|
-
...(input.runtimeApi.signal
|
|
19469
|
-
? { signal: input.runtimeApi.signal }
|
|
19470
|
-
: {}),
|
|
19471
|
-
}
|
|
19472
|
-
: {}),
|
|
19473
|
-
retryPolicy: 'none' as const,
|
|
19474
|
-
// The caller-owned admission deadline (see `signal` above) wins.
|
|
19475
|
-
...(input.signal ? { signal: input.signal } : {}),
|
|
19476
|
-
};
|
|
19477
|
-
const tokenExpiresAtMs = runtimeApiTokenExpiresAtMs(
|
|
19478
|
-
runtimeApi.executorToken,
|
|
19479
|
-
);
|
|
19480
|
-
if (tokenAuthority === 'gateway' && gatewayRuntimeApi) {
|
|
19481
|
-
authorityScope.authority = {
|
|
19482
|
-
runId: first.runId,
|
|
19483
|
-
leaseOwner,
|
|
19484
|
-
runtimeApi,
|
|
19485
|
-
expiresAtMs: tokenExpiresAtMs,
|
|
20110
|
+
if (needsRuntimeApi) {
|
|
20111
|
+
const resolvedRuntimeApi =
|
|
20112
|
+
gatewayRuntimeApi ?? input.runtimeApi ?? (await resolveRunRuntimeApi());
|
|
20113
|
+
runtimeApi = {
|
|
20114
|
+
...resolvedRuntimeApi,
|
|
20115
|
+
...(gatewayRuntimeApi && input.runtimeApi
|
|
20116
|
+
? {
|
|
20117
|
+
...(input.runtimeApi.fetch
|
|
20118
|
+
? { fetch: input.runtimeApi.fetch }
|
|
20119
|
+
: {}),
|
|
20120
|
+
...(input.runtimeApi.requestTimeoutMs !== undefined
|
|
20121
|
+
? { requestTimeoutMs: input.runtimeApi.requestTimeoutMs }
|
|
20122
|
+
: {}),
|
|
20123
|
+
...(input.runtimeApi.onResponseMeta
|
|
20124
|
+
? { onResponseMeta: input.runtimeApi.onResponseMeta }
|
|
20125
|
+
: {}),
|
|
20126
|
+
...(input.runtimeApi.runtimeTestFaultHeader !== undefined
|
|
20127
|
+
? {
|
|
20128
|
+
runtimeTestFaultHeader:
|
|
20129
|
+
input.runtimeApi.runtimeTestFaultHeader,
|
|
20130
|
+
}
|
|
20131
|
+
: {}),
|
|
20132
|
+
...(input.runtimeApi.signal
|
|
20133
|
+
? { signal: input.runtimeApi.signal }
|
|
20134
|
+
: {}),
|
|
20135
|
+
}
|
|
20136
|
+
: {}),
|
|
20137
|
+
retryPolicy: 'none' as const,
|
|
20138
|
+
// The caller-owned admission deadline (see `signal` above) wins.
|
|
20139
|
+
...(input.signal ? { signal: input.signal } : {}),
|
|
19486
20140
|
};
|
|
20141
|
+
if (tokenAuthority === 'gateway' && gatewayRuntimeApi) {
|
|
20142
|
+
authorityScope.authority = {
|
|
20143
|
+
runId: first.runId,
|
|
20144
|
+
leaseOwner,
|
|
20145
|
+
runtimeApi,
|
|
20146
|
+
expiresAtMs: runtimeApiTokenExpiresAtMs(runtimeApi.executorToken),
|
|
20147
|
+
};
|
|
20148
|
+
}
|
|
20149
|
+
authorityReadyAtMs = Date.now();
|
|
20150
|
+
}
|
|
20151
|
+
// The initial claim deliberately uses the old contract until its immutable
|
|
20152
|
+
// launch receiver advertises support. Widen under the same run/owner fence
|
|
20153
|
+
// before the first HTTP append, so an already-finished run need not pay one
|
|
20154
|
+
// authority/append/ACK round trip for every ingress predecessor.
|
|
20155
|
+
if (
|
|
20156
|
+
runtimeApi?.runLedgerIngressPrefixBatching === true &&
|
|
20157
|
+
events.length === 1 &&
|
|
20158
|
+
first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents
|
|
20159
|
+
) {
|
|
20160
|
+
const widened = await withClient((client) =>
|
|
20161
|
+
readPendingPostgresSchedulerOutboxEvents(
|
|
20162
|
+
client,
|
|
20163
|
+
{
|
|
20164
|
+
runId: first.runId,
|
|
20165
|
+
leaseOwner,
|
|
20166
|
+
leaseSeconds: input.leaseSeconds ?? 120,
|
|
20167
|
+
limit: Math.min(
|
|
20168
|
+
RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
|
|
20169
|
+
remainingEvents ?? RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
|
|
20170
|
+
),
|
|
20171
|
+
allowIngressPrefix: true,
|
|
20172
|
+
extendOwnedHead: first.eventId,
|
|
20173
|
+
},
|
|
20174
|
+
options,
|
|
20175
|
+
),
|
|
20176
|
+
);
|
|
20177
|
+
if (widened[0]?.eventId !== first.eventId) {
|
|
20178
|
+
throw new RunLedgerProjectionLeaseLostError(
|
|
20179
|
+
'head_lease_owner_mismatch',
|
|
20180
|
+
);
|
|
20181
|
+
}
|
|
20182
|
+
events = widened;
|
|
20183
|
+
}
|
|
20184
|
+
// Widening can add deferred facts to the lease. Project the complete
|
|
20185
|
+
// frozen prefix before any HTTP append, including its newly claimed rows.
|
|
20186
|
+
const handedOff = await materializeDeferredObservability();
|
|
20187
|
+
if (handedOff) return handedOff;
|
|
20188
|
+
if (ledgerEvents !== null) {
|
|
20189
|
+
ledgerEvents = events.flatMap(
|
|
20190
|
+
buildRunLedgerEventsFromPostgresSchedulerOutbox,
|
|
20191
|
+
);
|
|
19487
20192
|
}
|
|
19488
|
-
authorityReadyAtMs = Date.now();
|
|
19489
20193
|
const renewDelivery = async () => {
|
|
19490
20194
|
const currentAuthority = authorityScope.authority;
|
|
19491
20195
|
if (
|
|
20196
|
+
runtimeApi &&
|
|
19492
20197
|
tokenAuthority === 'gateway' &&
|
|
19493
20198
|
currentAuthority?.runId === first.runId &&
|
|
19494
20199
|
currentAuthority.leaseOwner === leaseOwner &&
|
|
19495
20200
|
currentAuthority.expiresAtMs !== null &&
|
|
19496
20201
|
currentAuthority.expiresAtMs <= Date.now() + reuseMarginMs
|
|
19497
20202
|
) {
|
|
20203
|
+
// Initial delivery reuses the claim's routing fields. Renewal must
|
|
20204
|
+
// reread authority so an origin change cannot pass under that cache.
|
|
19498
20205
|
const refreshed = await loadPostgresSchedulerRuntimeApiForRun(
|
|
19499
20206
|
withClient,
|
|
19500
20207
|
first.runId,
|
|
@@ -19508,14 +20215,16 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19508
20215
|
'Run Ledger dispatch origin changed within an immutable launch.',
|
|
19509
20216
|
);
|
|
19510
20217
|
}
|
|
19511
|
-
runtimeApi.executorToken = refreshed.
|
|
20218
|
+
runtimeApi.executorToken = refreshed.refreshExecutorToken
|
|
20219
|
+
? await refreshed.refreshExecutorToken()
|
|
20220
|
+
: refreshed.executorToken;
|
|
19512
20221
|
runtimeApi.vercelProtectionBypassToken =
|
|
19513
20222
|
refreshed.vercelProtectionBypassToken;
|
|
19514
20223
|
authorityScope.authority = {
|
|
19515
20224
|
runId: first.runId,
|
|
19516
20225
|
leaseOwner,
|
|
19517
20226
|
runtimeApi,
|
|
19518
|
-
expiresAtMs: runtimeApiTokenExpiresAtMs(
|
|
20227
|
+
expiresAtMs: runtimeApiTokenExpiresAtMs(runtimeApi.executorToken),
|
|
19519
20228
|
};
|
|
19520
20229
|
}
|
|
19521
20230
|
await withClient((client) =>
|
|
@@ -19525,70 +20234,107 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19525
20234
|
eventIds: events.map((event) => event.eventId),
|
|
19526
20235
|
leaseOwner,
|
|
19527
20236
|
leaseSeconds: input.leaseSeconds ?? 120,
|
|
20237
|
+
freezeReceipt:
|
|
20238
|
+
runtimeApi?.runLedgerIngressPrefixBatching === true &&
|
|
20239
|
+
events.some(
|
|
20240
|
+
(event) =>
|
|
20241
|
+
event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents,
|
|
20242
|
+
),
|
|
19528
20243
|
},
|
|
19529
20244
|
options,
|
|
19530
20245
|
),
|
|
19531
20246
|
);
|
|
19532
20247
|
};
|
|
19533
|
-
|
|
19534
|
-
|
|
19535
|
-
|
|
19536
|
-
|
|
19537
|
-
|
|
19538
|
-
|
|
19539
|
-
|
|
19540
|
-
|
|
19541
|
-
|
|
19542
|
-
|
|
19543
|
-
|
|
19544
|
-
|
|
19545
|
-
|
|
19546
|
-
|
|
19547
|
-
|
|
19548
|
-
|
|
19549
|
-
|
|
19550
|
-
|
|
19551
|
-
|
|
19552
|
-
|
|
19553
|
-
|
|
19554
|
-
|
|
19555
|
-
|
|
19556
|
-
|
|
19557
|
-
|
|
19558
|
-
|
|
19559
|
-
|
|
19560
|
-
|
|
19561
|
-
|
|
19562
|
-
|
|
19563
|
-
|
|
19564
|
-
|
|
19565
|
-
|
|
19566
|
-
|
|
19567
|
-
|
|
19568
|
-
|
|
19569
|
-
|
|
19570
|
-
|
|
19571
|
-
|
|
19572
|
-
|
|
19573
|
-
|
|
19574
|
-
|
|
19575
|
-
|
|
19576
|
-
|
|
19577
|
-
|
|
19578
|
-
|
|
19579
|
-
|
|
19580
|
-
|
|
19581
|
-
|
|
19582
|
-
|
|
19583
|
-
|
|
19584
|
-
|
|
19585
|
-
|
|
20248
|
+
if (needsRuntimeApi && runtimeApi) {
|
|
20249
|
+
const append = async (runtimeApi: WorkerRuntimeApiContext) => {
|
|
20250
|
+
// A pending start is always claimed alone because later events require
|
|
20251
|
+
// its snapshot. Retain that mutation as its own exact delivery.
|
|
20252
|
+
if (first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runCreated) {
|
|
20253
|
+
await renewDelivery();
|
|
20254
|
+
await projectPostgresSchedulerCreatedEvent(runtimeApi, first);
|
|
20255
|
+
} else if (
|
|
20256
|
+
first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runStarted
|
|
20257
|
+
) {
|
|
20258
|
+
await renewDelivery();
|
|
20259
|
+
const startLeaseOwner = first.leaseOwner;
|
|
20260
|
+
const launch = startLeaseOwner
|
|
20261
|
+
? await withClient((client) =>
|
|
20262
|
+
readRunLedgerDispatchLaunchForLease(
|
|
20263
|
+
client,
|
|
20264
|
+
{
|
|
20265
|
+
runId: first.runId,
|
|
20266
|
+
eventId: first.eventId,
|
|
20267
|
+
leaseOwner: startLeaseOwner,
|
|
20268
|
+
},
|
|
20269
|
+
options,
|
|
20270
|
+
),
|
|
20271
|
+
)
|
|
20272
|
+
: null;
|
|
20273
|
+
await projectPostgresSchedulerStartEvent(runtimeApi, first, launch);
|
|
20274
|
+
} else if (ledgerEvents && ledgerEvents.length > 0) {
|
|
20275
|
+
const terminal = [...events]
|
|
20276
|
+
.reverse()
|
|
20277
|
+
.find((event) =>
|
|
20278
|
+
RUN_SCHEDULER_OUTBOX_TERMINAL_EVENT_TYPES.includes(
|
|
20279
|
+
event.eventType,
|
|
20280
|
+
),
|
|
20281
|
+
);
|
|
20282
|
+
const batchDeliveryKey = createHash('sha256')
|
|
20283
|
+
.update(events.map((event) => event.deliveryKey).join('\n'))
|
|
20284
|
+
.digest('hex');
|
|
20285
|
+
// The transport renews before every HTTP chunk; another renewal here
|
|
20286
|
+
// would add a checkout without protecting an intervening side effect.
|
|
20287
|
+
await projectRunEventsViaAppRuntime(runtimeApi, {
|
|
20288
|
+
playId: first.runId,
|
|
20289
|
+
events: ledgerEvents,
|
|
20290
|
+
deliveryLeaseOwner: leaseOwner,
|
|
20291
|
+
beforeBatch: renewDelivery,
|
|
20292
|
+
idempotencyKey: terminal
|
|
20293
|
+
? buildRunLedgerTerminalIdempotencyKey({
|
|
20294
|
+
runId: first.runId,
|
|
20295
|
+
deliveryKey: terminal.deliveryKey,
|
|
20296
|
+
})
|
|
20297
|
+
: buildRunLedgerEventsIdempotencyKey({
|
|
20298
|
+
runId: first.runId,
|
|
20299
|
+
deliveryDigest: batchDeliveryKey,
|
|
20300
|
+
}),
|
|
20301
|
+
});
|
|
20302
|
+
}
|
|
20303
|
+
};
|
|
20304
|
+
try {
|
|
20305
|
+
await append(runtimeApi);
|
|
20306
|
+
} catch (error) {
|
|
20307
|
+
// Refresh a reused authority once through this delivery's active lease.
|
|
20308
|
+
// Both attempts retain the same fenced append and idempotency key.
|
|
20309
|
+
if (
|
|
20310
|
+
!gatewayRuntimeApi?.refreshExecutorToken ||
|
|
20311
|
+
!(error instanceof AppRuntimeApiResponseError) ||
|
|
20312
|
+
error.status !== 401
|
|
20313
|
+
)
|
|
20314
|
+
throw error;
|
|
20315
|
+
runtimeApi.executorToken = await gatewayRuntimeApi.refreshExecutorToken();
|
|
20316
|
+
await append(runtimeApi);
|
|
19586
20317
|
}
|
|
20318
|
+
} else if (tokenAuthority !== 'gateway') {
|
|
20319
|
+
// App-side acknowledgement retains lease validation. The gateway can
|
|
20320
|
+
// acknowledge a durable search-only fact without a network lease hop.
|
|
20321
|
+
await renewDelivery();
|
|
19587
20322
|
}
|
|
19588
20323
|
appendFinishedAtMs = Date.now();
|
|
19589
20324
|
const acknowledgement = await acknowledgeRunLedgerPrefixWithContinuation(
|
|
19590
20325
|
withClient,
|
|
19591
|
-
{
|
|
20326
|
+
{
|
|
20327
|
+
input,
|
|
20328
|
+
events,
|
|
20329
|
+
leaseOwner,
|
|
20330
|
+
runtimeApi,
|
|
20331
|
+
scope,
|
|
20332
|
+
continuation,
|
|
20333
|
+
continuationAllowance:
|
|
20334
|
+
remainingEvents === null
|
|
20335
|
+
? null
|
|
20336
|
+
: Math.max(0, remainingEvents - events.length),
|
|
20337
|
+
},
|
|
19592
20338
|
options,
|
|
19593
20339
|
);
|
|
19594
20340
|
const { deliveredRows } = acknowledgement;
|
|
@@ -19630,6 +20376,10 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19630
20376
|
authorityScope,
|
|
19631
20377
|
);
|
|
19632
20378
|
} catch (error) {
|
|
20379
|
+
// A failed append must reacquire receiver truth on its next attempt,
|
|
20380
|
+
// including an older receiver that rejected a frozen compound receipt.
|
|
20381
|
+
authorityScope.authority?.runtimeApi.invalidateExecutorToken?.();
|
|
20382
|
+
authorityScope.authority = null;
|
|
19633
20383
|
await releasePendingContinuation();
|
|
19634
20384
|
if (input.signal?.aborted) {
|
|
19635
20385
|
// The caller's latency budget expired; this is not a delivery failure.
|
|
@@ -19654,25 +20404,98 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19654
20404
|
retryingEventIds: released,
|
|
19655
20405
|
};
|
|
19656
20406
|
}
|
|
19657
|
-
|
|
19658
|
-
const
|
|
19659
|
-
|
|
19660
|
-
|
|
19661
|
-
|
|
20407
|
+
if (error instanceof DeferredRuntimeProjectionInputError) {
|
|
20408
|
+
const outcome = await withClient((client) =>
|
|
20409
|
+
settlePostgresSchedulerOutboxFailure(client, error.eventId, error, {
|
|
20410
|
+
...options,
|
|
20411
|
+
leaseOwner,
|
|
20412
|
+
}),
|
|
20413
|
+
);
|
|
20414
|
+
// Earlier facts may already be indexed. Only the invalid immutable row
|
|
20415
|
+
// is quarantined; valid predecessors remain deliverable in causal order.
|
|
20416
|
+
const released = await withClient((client) =>
|
|
20417
|
+
releasePostgresSchedulerOutboxLease(
|
|
19662
20418
|
client,
|
|
19663
|
-
|
|
19664
|
-
|
|
20419
|
+
events
|
|
20420
|
+
.filter((event) => event.eventId !== error.eventId)
|
|
20421
|
+
.map((event) => event.eventId),
|
|
19665
20422
|
{ ...options, leaseOwner },
|
|
19666
|
-
)
|
|
19667
|
-
|
|
19668
|
-
|
|
20423
|
+
),
|
|
20424
|
+
);
|
|
20425
|
+
const failed = outcome === 'blocked' ? [error.eventId] : [];
|
|
20426
|
+
const retrying =
|
|
20427
|
+
outcome === 'blocked' ? released : [...released, error.eventId];
|
|
20428
|
+
console.error('[run-ledger-outbox.invalid_projection_input]', {
|
|
20429
|
+
eventId: error.eventId,
|
|
20430
|
+
code: error.code,
|
|
20431
|
+
detail: error.message,
|
|
20432
|
+
});
|
|
20433
|
+
return {
|
|
20434
|
+
attempted: events.length,
|
|
20435
|
+
projected: 0,
|
|
20436
|
+
eventIds: [],
|
|
20437
|
+
failed: failed.length,
|
|
20438
|
+
failedEventIds: failed,
|
|
20439
|
+
retrying: retrying.length,
|
|
20440
|
+
retryingEventIds: retrying,
|
|
20441
|
+
};
|
|
20442
|
+
}
|
|
20443
|
+
const settlement = await withClient(async (client) => {
|
|
20444
|
+
await client.query('BEGIN');
|
|
20445
|
+
try {
|
|
20446
|
+
const blocked: string[] = [];
|
|
20447
|
+
const retrying: string[] = [];
|
|
20448
|
+
const ownedRetrying: string[] = [];
|
|
20449
|
+
for (const event of events) {
|
|
20450
|
+
const outcome = await settlePostgresSchedulerOutboxFailure(
|
|
20451
|
+
client,
|
|
20452
|
+
event.eventId,
|
|
20453
|
+
error,
|
|
20454
|
+
{ ...options, leaseOwner },
|
|
20455
|
+
);
|
|
20456
|
+
if (outcome === 'blocked') blocked.push(event.eventId);
|
|
20457
|
+
else retrying.push(event.eventId);
|
|
20458
|
+
if (outcome === 'retrying') ownedRetrying.push(event.eventId);
|
|
20459
|
+
}
|
|
20460
|
+
// All members of this failed append must become due together. Keep
|
|
20461
|
+
// the existing per-row retry policy, taking its latest deadline only
|
|
20462
|
+
// for rows this owner actually settled, before releasing their locks.
|
|
20463
|
+
if (ownedRetrying.length > 1) {
|
|
20464
|
+
await client.query(
|
|
20465
|
+
`
|
|
20466
|
+
WITH retry AS (
|
|
20467
|
+
SELECT max(next_attempt_at) AS retry_at
|
|
20468
|
+
FROM ${tableName('outbox', options)}
|
|
20469
|
+
WHERE event_id = ANY($1::text[])
|
|
20470
|
+
AND delivered_at IS NULL AND failed_at IS NULL
|
|
20471
|
+
)
|
|
20472
|
+
UPDATE ${tableName('outbox', options)} AS outbox
|
|
20473
|
+
SET next_attempt_at = retry.retry_at
|
|
20474
|
+
FROM retry
|
|
20475
|
+
WHERE outbox.event_id = ANY($1::text[])
|
|
20476
|
+
AND outbox.delivered_at IS NULL AND outbox.failed_at IS NULL
|
|
20477
|
+
AND retry.retry_at IS NOT NULL
|
|
20478
|
+
`,
|
|
20479
|
+
[ownedRetrying],
|
|
20480
|
+
);
|
|
20481
|
+
}
|
|
20482
|
+
await client.query('COMMIT');
|
|
20483
|
+
return { blocked, retrying };
|
|
20484
|
+
} catch (settlementError) {
|
|
20485
|
+
await client.query('ROLLBACK');
|
|
20486
|
+
throw settlementError;
|
|
19669
20487
|
}
|
|
19670
|
-
return { blocked, retrying };
|
|
19671
20488
|
});
|
|
20489
|
+
const failure = runLedgerDeliveryFailure({ error });
|
|
19672
20490
|
console.warn('[run-ledger-outbox.batch_dispatch_failed]', {
|
|
19673
20491
|
runId: input.runId ?? null,
|
|
19674
20492
|
eventIds: events.map((event) => event.eventId),
|
|
19675
20493
|
error: error instanceof Error ? error.message : String(error),
|
|
20494
|
+
code: failure.code,
|
|
20495
|
+
failureReason: failure.failureReason,
|
|
20496
|
+
...(failure.projectionMismatch
|
|
20497
|
+
? { projectionMismatch: failure.projectionMismatch }
|
|
20498
|
+
: {}),
|
|
19676
20499
|
});
|
|
19677
20500
|
return {
|
|
19678
20501
|
attempted: events.length,
|
|
@@ -19781,9 +20604,10 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
|
|
|
19781
20604
|
input: RunLedgerOutboxDispatchInput;
|
|
19782
20605
|
events: PostgresSchedulerOutboxEvent[];
|
|
19783
20606
|
leaseOwner: string;
|
|
19784
|
-
runtimeApi: WorkerRuntimeApiContext;
|
|
20607
|
+
runtimeApi: WorkerRuntimeApiContext | null;
|
|
19785
20608
|
scope?: RunLedgerOutboxDrainScope;
|
|
19786
20609
|
continuation: { current: RunLedgerOutboxDrainScope | null };
|
|
20610
|
+
continuationAllowance: number | null;
|
|
19787
20611
|
},
|
|
19788
20612
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
19789
20613
|
): Promise<{
|
|
@@ -19796,74 +20620,86 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
|
|
|
19796
20620
|
!!input.scope ||
|
|
19797
20621
|
(input.input.continueThroughTerminal === true && !input.input.headEventId);
|
|
19798
20622
|
return withClient(async (client) => {
|
|
19799
|
-
|
|
19800
|
-
await client.query('BEGIN');
|
|
19801
|
-
}
|
|
19802
|
-
try {
|
|
20623
|
+
const acknowledge = async () => {
|
|
19803
20624
|
if (retainContinuation) {
|
|
19804
|
-
await client.query(
|
|
19805
|
-
`SELECT set_config('lock_timeout', $1, true), set_config('statement_timeout', $2, true)`,
|
|
19806
|
-
[
|
|
19807
|
-
`${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
|
|
19808
|
-
`${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
|
|
19809
|
-
],
|
|
19810
|
-
);
|
|
19811
|
-
// Keep the acknowledged prefix and its successor under one run lock.
|
|
19812
|
-
// No competing projector can take the next head while this ACK's
|
|
19813
|
-
// response is travelling back to the lease holder.
|
|
19814
|
-
await client.query(
|
|
19815
|
-
`SELECT run_id FROM ${tableName('runs', options)} WHERE run_id = $1 FOR UPDATE`,
|
|
19816
|
-
[runId],
|
|
19817
|
-
);
|
|
20625
|
+
await client.query('BEGIN');
|
|
19818
20626
|
}
|
|
19819
|
-
|
|
19820
|
-
|
|
19821
|
-
|
|
19822
|
-
|
|
19823
|
-
|
|
19824
|
-
|
|
19825
|
-
|
|
19826
|
-
|
|
19827
|
-
|
|
19828
|
-
|
|
19829
|
-
|
|
19830
|
-
|
|
19831
|
-
client
|
|
19832
|
-
|
|
19833
|
-
|
|
19834
|
-
);
|
|
19835
|
-
if (target.exists && !target.delivered && !target.blocked) {
|
|
19836
|
-
const nextClaimed = await readPendingPostgresSchedulerOutboxEvents(
|
|
19837
|
-
client,
|
|
19838
|
-
{
|
|
19839
|
-
runId,
|
|
19840
|
-
leaseOwner: input.leaseOwner,
|
|
19841
|
-
leaseSeconds: input.input.leaseSeconds ?? 120,
|
|
19842
|
-
limit: Math.min(
|
|
19843
|
-
RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
|
|
19844
|
-
input.input.limit ?? 1,
|
|
19845
|
-
),
|
|
19846
|
-
inTransaction: true,
|
|
19847
|
-
},
|
|
19848
|
-
options,
|
|
20627
|
+
try {
|
|
20628
|
+
if (retainContinuation) {
|
|
20629
|
+
await client.query(
|
|
20630
|
+
`SELECT set_config('lock_timeout', $1, true), set_config('statement_timeout', $2, true)`,
|
|
20631
|
+
[
|
|
20632
|
+
`${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
|
|
20633
|
+
`${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
|
|
20634
|
+
],
|
|
20635
|
+
);
|
|
20636
|
+
// Keep the acknowledged prefix and its successor under one run lock.
|
|
20637
|
+
// No competing projector can take the next head while this ACK's
|
|
20638
|
+
// response is travelling back to the lease holder.
|
|
20639
|
+
await client.query(
|
|
20640
|
+
`SELECT run_id FROM ${tableName('runs', options)} WHERE run_id = $1 FOR UPDATE`,
|
|
20641
|
+
[runId],
|
|
19849
20642
|
);
|
|
19850
|
-
if (nextClaimed.length > 0) {
|
|
19851
|
-
continuation = input.scope ?? {
|
|
19852
|
-
runId,
|
|
19853
|
-
leaseOwner: input.leaseOwner,
|
|
19854
|
-
targetEventId,
|
|
19855
|
-
nextClaimed: [],
|
|
19856
|
-
};
|
|
19857
|
-
continuation.nextClaimed = nextClaimed;
|
|
19858
|
-
input.continuation.current = continuation;
|
|
19859
|
-
}
|
|
19860
20643
|
}
|
|
20644
|
+
const eventIds = input.events.map((event) => event.eventId);
|
|
20645
|
+
const { deliveredRows, events: nextClaimed } =
|
|
20646
|
+
retainContinuation &&
|
|
20647
|
+
input.continuationAllowance !== 0 &&
|
|
20648
|
+
!eventIds.includes(targetEventId)
|
|
20649
|
+
? await claimPostgresSchedulerOutboxPrefix(
|
|
20650
|
+
client,
|
|
20651
|
+
{
|
|
20652
|
+
runId,
|
|
20653
|
+
leaseOwner: input.leaseOwner,
|
|
20654
|
+
leaseSeconds: input.input.leaseSeconds ?? 120,
|
|
20655
|
+
limit: Math.min(
|
|
20656
|
+
RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
|
|
20657
|
+
input.runtimeApi?.runLedgerIngressPrefixBatching === true
|
|
20658
|
+
? RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS
|
|
20659
|
+
: (input.input.limit ?? 1),
|
|
20660
|
+
input.continuationAllowance ??
|
|
20661
|
+
RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
|
|
20662
|
+
),
|
|
20663
|
+
allowIngressPrefix:
|
|
20664
|
+
input.runtimeApi?.runLedgerIngressPrefixBatching === true,
|
|
20665
|
+
},
|
|
20666
|
+
options,
|
|
20667
|
+
{ eventIds, targetEventId },
|
|
20668
|
+
)
|
|
20669
|
+
: {
|
|
20670
|
+
deliveredRows: await markPostgresSchedulerOutboxDeliveredRows(
|
|
20671
|
+
client,
|
|
20672
|
+
eventIds,
|
|
20673
|
+
{ ...options, leaseOwner: input.leaseOwner },
|
|
20674
|
+
),
|
|
20675
|
+
events: [],
|
|
20676
|
+
};
|
|
20677
|
+
let continuation: RunLedgerOutboxDrainScope | null = null;
|
|
20678
|
+
if (nextClaimed.length > 0) {
|
|
20679
|
+
continuation = input.scope ?? {
|
|
20680
|
+
runId,
|
|
20681
|
+
leaseOwner: input.leaseOwner,
|
|
20682
|
+
targetEventId,
|
|
20683
|
+
nextClaimed: [],
|
|
20684
|
+
};
|
|
20685
|
+
continuation.nextClaimed = nextClaimed;
|
|
20686
|
+
input.continuation.current = continuation;
|
|
20687
|
+
}
|
|
20688
|
+
if (retainContinuation) await client.query('COMMIT');
|
|
20689
|
+
return { deliveredRows, continuation };
|
|
20690
|
+
} catch (error) {
|
|
20691
|
+
if (retainContinuation) await client.query('ROLLBACK');
|
|
20692
|
+
throw error;
|
|
19861
20693
|
}
|
|
19862
|
-
|
|
19863
|
-
|
|
20694
|
+
};
|
|
20695
|
+
try {
|
|
20696
|
+
return await acknowledge();
|
|
19864
20697
|
} catch (error) {
|
|
19865
|
-
if (
|
|
19866
|
-
|
|
20698
|
+
if (!isMissingOutboxDeliveryBatchMarker(error)) throw error;
|
|
20699
|
+
// The first statement recorded this operation-local missing column and
|
|
20700
|
+
// rolled back its ACK. Reacquire the run lock and take a fresh snapshot
|
|
20701
|
+
// for the existing compatibility claim, without another ledger append.
|
|
20702
|
+
return await acknowledge();
|
|
19867
20703
|
}
|
|
19868
20704
|
});
|
|
19869
20705
|
}
|
|
@@ -20019,70 +20855,45 @@ async function projectPostgresSchedulerStartEvent(
|
|
|
20019
20855
|
});
|
|
20020
20856
|
}
|
|
20021
20857
|
|
|
20022
|
-
export type RunLedgerDispatchLaunch
|
|
20023
|
-
PlaySchedulerSubmitInput,
|
|
20024
|
-
| 'runId'
|
|
20025
|
-
| 'playId'
|
|
20026
|
-
| 'playName'
|
|
20027
|
-
| 'executorToken'
|
|
20028
|
-
| 'baseUrl'
|
|
20029
|
-
| 'runtimeApiBaseUrl'
|
|
20030
|
-
| 'runtimeTokenOrigin'
|
|
20031
|
-
| 'orgId'
|
|
20032
|
-
| 'runtimeAuthority'
|
|
20033
|
-
| 'vercelProtectionBypassToken'
|
|
20034
|
-
>;
|
|
20858
|
+
export type { RunLedgerDispatchLaunch } from './run-ledger-dispatch-authority';
|
|
20035
20859
|
|
|
20036
|
-
/**
|
|
20037
|
-
function
|
|
20038
|
-
const
|
|
20039
|
-
'
|
|
20040
|
-
|
|
20041
|
-
'
|
|
20042
|
-
|
|
20043
|
-
|
|
20044
|
-
'
|
|
20045
|
-
'
|
|
20046
|
-
|
|
20047
|
-
|
|
20048
|
-
'vercelProtectionBypassToken',
|
|
20049
|
-
];
|
|
20050
|
-
return `jsonb_build_object(${fields
|
|
20051
|
-
.map((field) => `'${field}', ${alias}.launch_json->'${field}'`)
|
|
20052
|
-
.join(', ')})`;
|
|
20860
|
+
/** Search-only `run.events` facts settle without consulting launch credentials. */
|
|
20861
|
+
function runLedgerSearchOnlyOutboxFactSql(alias: string): string {
|
|
20862
|
+
const events = `CASE
|
|
20863
|
+
WHEN jsonb_typeof(${alias}.payload_json->'events') = 'array'
|
|
20864
|
+
THEN ${alias}.payload_json->'events'
|
|
20865
|
+
ELSE '[]'::jsonb
|
|
20866
|
+
END`;
|
|
20867
|
+
return `(
|
|
20868
|
+
${alias}.event_type = '${RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents}'
|
|
20869
|
+
AND ${alias}.payload_json->>'logBodyStore' = 'planet_scale'
|
|
20870
|
+
AND jsonb_array_length(${events}) = 0
|
|
20871
|
+
)`;
|
|
20053
20872
|
}
|
|
20054
20873
|
|
|
20055
|
-
async function
|
|
20056
|
-
|
|
20057
|
-
operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
|
|
20058
|
-
) => Promise<T>,
|
|
20874
|
+
async function resolvePostgresSchedulerRuntimeApiForRun(
|
|
20875
|
+
launchJson: unknown,
|
|
20059
20876
|
runId: string,
|
|
20060
20877
|
dispatchLease: { eventId: string; leaseOwner: string } | null,
|
|
20061
20878
|
tokenAuthority: RunLedgerOutboxTokenAuthority,
|
|
20062
20879
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
20063
20880
|
signal?: AbortSignal,
|
|
20064
|
-
): Promise<
|
|
20065
|
-
|
|
20066
|
-
|
|
20067
|
-
|
|
20068
|
-
|
|
20069
|
-
|
|
20070
|
-
|
|
20071
|
-
FROM ${tableName('runs', options)} AS runs
|
|
20072
|
-
WHERE run_id = $1
|
|
20073
|
-
LIMIT 1`,
|
|
20074
|
-
[runId],
|
|
20075
|
-
),
|
|
20076
|
-
);
|
|
20077
|
-
const launch: RunLedgerDispatchLaunch = parseLaunchJson(
|
|
20078
|
-
result.rows[0]?.launch_json,
|
|
20079
|
-
);
|
|
20881
|
+
): Promise<
|
|
20882
|
+
WorkerRuntimeApiContext & {
|
|
20883
|
+
refreshExecutorToken?: () => Promise<string>;
|
|
20884
|
+
invalidateExecutorToken?: () => void;
|
|
20885
|
+
}
|
|
20886
|
+
> {
|
|
20887
|
+
const launch = parseRunLedgerDispatchLaunch(launchJson);
|
|
20080
20888
|
if (!launch?.baseUrl) {
|
|
20081
20889
|
throw new Error(
|
|
20082
20890
|
`Run-ledger outbox dispatcher cannot resolve runtime API context for ${runId}.`,
|
|
20083
20891
|
);
|
|
20084
20892
|
}
|
|
20085
20893
|
let executorToken: string;
|
|
20894
|
+
let runLedgerIngressPrefixBatching: true | undefined;
|
|
20895
|
+
let refreshExecutorToken: (() => Promise<string>) | undefined;
|
|
20896
|
+
let invalidateExecutorToken: (() => void) | undefined;
|
|
20086
20897
|
if (tokenAuthority === 'local_mint') {
|
|
20087
20898
|
// App-side authoritative projector: sign in-process. NEVER reached on the
|
|
20088
20899
|
// Fly worker (it passes `gateway`), so this is the only place `BETTER_AUTH_
|
|
@@ -20101,34 +20912,120 @@ async function loadPostgresSchedulerRuntimeApiForRun(
|
|
|
20101
20912
|
`Run-ledger outbox gateway dispatch for ${runId} requires an active outbox lease.`,
|
|
20102
20913
|
);
|
|
20103
20914
|
}
|
|
20104
|
-
|
|
20915
|
+
const dispatchRequest = {
|
|
20105
20916
|
launch,
|
|
20106
20917
|
eventId: dispatchLease.eventId,
|
|
20107
20918
|
leaseOwner: dispatchLease.leaseOwner,
|
|
20108
20919
|
schedulerSchema: options?.schema ?? null,
|
|
20109
20920
|
signal,
|
|
20110
|
-
}
|
|
20921
|
+
};
|
|
20922
|
+
const dispatchAuthority =
|
|
20923
|
+
await requestRunLedgerDispatchToken(dispatchRequest);
|
|
20924
|
+
invalidateExecutorToken = () =>
|
|
20925
|
+
runLedgerDispatchTokens.delete(dispatchAuthority.cacheKey);
|
|
20926
|
+
if (dispatchAuthority.reused) {
|
|
20927
|
+
refreshExecutorToken = async () =>
|
|
20928
|
+
(await requestRunLedgerDispatchToken(dispatchRequest, true))
|
|
20929
|
+
.executorToken;
|
|
20930
|
+
}
|
|
20931
|
+
executorToken = dispatchAuthority.executorToken;
|
|
20932
|
+
runLedgerIngressPrefixBatching =
|
|
20933
|
+
dispatchAuthority.runLedgerIngressPrefixBatching;
|
|
20111
20934
|
}
|
|
20112
20935
|
return {
|
|
20113
20936
|
baseUrl: runtimeApiBaseUrlFromLaunch(launch),
|
|
20114
20937
|
executorToken,
|
|
20938
|
+
...(runLedgerIngressPrefixBatching
|
|
20939
|
+
? { runLedgerIngressPrefixBatching }
|
|
20940
|
+
: {}),
|
|
20115
20941
|
vercelProtectionBypassToken: launch.vercelProtectionBypassToken ?? null,
|
|
20942
|
+
...(refreshExecutorToken ? { refreshExecutorToken } : {}),
|
|
20943
|
+
...(invalidateExecutorToken ? { invalidateExecutorToken } : {}),
|
|
20116
20944
|
};
|
|
20117
20945
|
}
|
|
20118
20946
|
|
|
20119
|
-
async function
|
|
20120
|
-
|
|
20121
|
-
|
|
20122
|
-
|
|
20123
|
-
|
|
20124
|
-
|
|
20125
|
-
|
|
20947
|
+
async function loadPostgresSchedulerRuntimeApiForRun(
|
|
20948
|
+
withClient: <T>(
|
|
20949
|
+
operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
|
|
20950
|
+
) => Promise<T>,
|
|
20951
|
+
runId: string,
|
|
20952
|
+
dispatchLease: { eventId: string; leaseOwner: string } | null,
|
|
20953
|
+
tokenAuthority: RunLedgerOutboxTokenAuthority,
|
|
20954
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
20955
|
+
signal?: AbortSignal,
|
|
20956
|
+
): ReturnType<typeof resolvePostgresSchedulerRuntimeApiForRun> {
|
|
20957
|
+
// A continuation already owns its next event lease, so its launch context
|
|
20958
|
+
// is not part of the preclaim result. Load only the authority fields here.
|
|
20959
|
+
const result = await withClient((client) =>
|
|
20960
|
+
client.query<{ launch_json: unknown }>(
|
|
20961
|
+
`SELECT ${runLedgerDispatchLaunchSql('runs')} AS launch_json
|
|
20962
|
+
FROM ${tableName('runs', options)} AS runs
|
|
20963
|
+
WHERE run_id = $1
|
|
20964
|
+
LIMIT 1`,
|
|
20965
|
+
[runId],
|
|
20966
|
+
),
|
|
20967
|
+
);
|
|
20968
|
+
return resolvePostgresSchedulerRuntimeApiForRun(
|
|
20969
|
+
result.rows[0]?.launch_json,
|
|
20970
|
+
runId,
|
|
20971
|
+
dispatchLease,
|
|
20972
|
+
tokenAuthority,
|
|
20973
|
+
options,
|
|
20974
|
+
signal,
|
|
20975
|
+
);
|
|
20976
|
+
}
|
|
20977
|
+
|
|
20978
|
+
// Only freshly gateway-issued legacy run.progress authority is reused.
|
|
20979
|
+
// Prefix advertisements are reacquired for each drain because they select its
|
|
20980
|
+
// receipt shape, including recovery across receiver capability changes. This is not
|
|
20981
|
+
// a lease cache: every delivery still claims, renews, and proves its own lease
|
|
20982
|
+
// at append. Include the complete pinned authority/routing and internal-token
|
|
20983
|
+
// fingerprint so a different run, schema, origin, or authority never shares it.
|
|
20984
|
+
const RUN_LEDGER_DISPATCH_TOKEN_CACHE_MS = 30_000;
|
|
20985
|
+
const RUN_LEDGER_DISPATCH_TOKEN_CACHE_ENTRIES = 256;
|
|
20986
|
+
const runLedgerDispatchTokens = new Map<
|
|
20987
|
+
string,
|
|
20988
|
+
{ token: string; expiresAt: number; runLedgerIngressPrefixBatching?: true }
|
|
20989
|
+
>();
|
|
20990
|
+
|
|
20991
|
+
async function requestRunLedgerDispatchToken(
|
|
20992
|
+
input: {
|
|
20993
|
+
launch: RunLedgerDispatchLaunch;
|
|
20994
|
+
eventId: string;
|
|
20995
|
+
leaseOwner: string;
|
|
20996
|
+
schedulerSchema: string | null;
|
|
20997
|
+
signal?: AbortSignal;
|
|
20998
|
+
},
|
|
20999
|
+
forceFresh = false,
|
|
21000
|
+
): Promise<{
|
|
21001
|
+
executorToken: string;
|
|
21002
|
+
reused: boolean;
|
|
21003
|
+
cacheKey: string;
|
|
21004
|
+
runLedgerIngressPrefixBatching?: true;
|
|
21005
|
+
}> {
|
|
20126
21006
|
const internalToken = process.env.DEEPLINE_INTERNAL_TOKEN?.trim();
|
|
20127
21007
|
if (!internalToken || !input.eventId || !input.leaseOwner) {
|
|
20128
21008
|
throw new Error(
|
|
20129
21009
|
'Run-ledger outbox recovery requires a valid dispatch lease and DEEPLINE_INTERNAL_TOKEN.',
|
|
20130
21010
|
);
|
|
20131
21011
|
}
|
|
21012
|
+
input.signal?.throwIfAborted();
|
|
21013
|
+
const { executorToken: _storedToken, ...authority } = input.launch;
|
|
21014
|
+
const cacheKey = createHash('sha256')
|
|
21015
|
+
.update(JSON.stringify([input.schedulerSchema, authority, internalToken]))
|
|
21016
|
+
.digest('hex');
|
|
21017
|
+
const now = Date.now();
|
|
21018
|
+
for (const [key, entry] of runLedgerDispatchTokens) {
|
|
21019
|
+
if (entry.expiresAt <= now) runLedgerDispatchTokens.delete(key);
|
|
21020
|
+
}
|
|
21021
|
+
const cached = runLedgerDispatchTokens.get(cacheKey);
|
|
21022
|
+
if (!forceFresh && cached)
|
|
21023
|
+
return {
|
|
21024
|
+
executorToken: cached.token,
|
|
21025
|
+
reused: true,
|
|
21026
|
+
cacheKey,
|
|
21027
|
+
};
|
|
21028
|
+
runLedgerDispatchTokens.delete(cacheKey);
|
|
20132
21029
|
const url = `${runtimeApiBaseUrlFromLaunch(input.launch).replace(/\/$/, '')}/api/v2/internal/play-runtime/run-ledger-dispatch-token`;
|
|
20133
21030
|
const vercelProtectionHeaders = await vercelProtectionBypassHeaders({
|
|
20134
21031
|
baseUrl: runtimeApiBaseUrlFromLaunch(input.launch),
|
|
@@ -20164,6 +21061,7 @@ async function requestRunLedgerDispatchToken(input: {
|
|
|
20164
21061
|
try {
|
|
20165
21062
|
return JSON.parse(responseText) as {
|
|
20166
21063
|
executorToken?: unknown;
|
|
21064
|
+
runLedgerIngressPrefixBatching?: unknown;
|
|
20167
21065
|
error?: unknown;
|
|
20168
21066
|
code?: unknown;
|
|
20169
21067
|
retryable?: unknown;
|
|
@@ -20196,7 +21094,53 @@ async function requestRunLedgerDispatchToken(input: {
|
|
|
20196
21094
|
: 'invalid response',
|
|
20197
21095
|
});
|
|
20198
21096
|
}
|
|
20199
|
-
|
|
21097
|
+
// Decode only the lifetime, never an authentication decision: the app
|
|
21098
|
+
// validates the signature, run scope, and live delivery lease on append.
|
|
21099
|
+
// Opaque/older token formats continue to work without this optimization.
|
|
21100
|
+
try {
|
|
21101
|
+
const claims = JSON.parse(
|
|
21102
|
+
Buffer.from(
|
|
21103
|
+
body.executorToken.split('.')[1] ?? '',
|
|
21104
|
+
'base64url',
|
|
21105
|
+
).toString(),
|
|
21106
|
+
) as { exp?: unknown };
|
|
21107
|
+
if (
|
|
21108
|
+
body.runLedgerIngressPrefixBatching === undefined &&
|
|
21109
|
+
typeof claims.exp === 'number' &&
|
|
21110
|
+
Number.isFinite(claims.exp)
|
|
21111
|
+
) {
|
|
21112
|
+
const expiresAt = Math.min(
|
|
21113
|
+
Date.now() + RUN_LEDGER_DISPATCH_TOKEN_CACHE_MS,
|
|
21114
|
+
claims.exp * 1_000 - 5_000,
|
|
21115
|
+
);
|
|
21116
|
+
if (expiresAt > Date.now()) {
|
|
21117
|
+
runLedgerDispatchTokens.set(cacheKey, {
|
|
21118
|
+
token: body.executorToken,
|
|
21119
|
+
expiresAt,
|
|
21120
|
+
});
|
|
21121
|
+
while (
|
|
21122
|
+
runLedgerDispatchTokens.size > RUN_LEDGER_DISPATCH_TOKEN_CACHE_ENTRIES
|
|
21123
|
+
) {
|
|
21124
|
+
runLedgerDispatchTokens.delete(
|
|
21125
|
+
runLedgerDispatchTokens.keys().next().value!,
|
|
21126
|
+
);
|
|
21127
|
+
}
|
|
21128
|
+
}
|
|
21129
|
+
}
|
|
21130
|
+
} catch {
|
|
21131
|
+
// A cache miss never changes delivery or hides a gateway rejection.
|
|
21132
|
+
}
|
|
21133
|
+
if (body.runLedgerIngressPrefixBatching !== undefined) {
|
|
21134
|
+
runLedgerDispatchTokens.delete(cacheKey);
|
|
21135
|
+
}
|
|
21136
|
+
return {
|
|
21137
|
+
executorToken: body.executorToken,
|
|
21138
|
+
reused: false,
|
|
21139
|
+
cacheKey,
|
|
21140
|
+
...(body.runLedgerIngressPrefixBatching === true
|
|
21141
|
+
? { runLedgerIngressPrefixBatching: true as const }
|
|
21142
|
+
: {}),
|
|
21143
|
+
};
|
|
20200
21144
|
}
|
|
20201
21145
|
|
|
20202
21146
|
export async function requestComputeBillingDispatchToken(input: {
|
|
@@ -20272,27 +21216,7 @@ export async function requestComputeBillingDispatchToken(input: {
|
|
|
20272
21216
|
return body.executorToken;
|
|
20273
21217
|
}
|
|
20274
21218
|
|
|
20275
|
-
|
|
20276
|
-
export async function readRunLedgerDispatchAuthorityForLease(
|
|
20277
|
-
client: PostgresSchedulerQueryClient,
|
|
20278
|
-
input: { runId: string; eventId: string; leaseOwner: string },
|
|
20279
|
-
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
20280
|
-
): Promise<RunLedgerDispatchLaunch | null> {
|
|
20281
|
-
const result = await client.query<{ launch_json: unknown }>(
|
|
20282
|
-
`SELECT ${runLedgerDispatchLaunchSql('runs')} AS launch_json
|
|
20283
|
-
FROM ${tableName('outbox', options)} outbox
|
|
20284
|
-
JOIN ${tableName('runs', options)} runs ON runs.run_id = outbox.run_id
|
|
20285
|
-
WHERE outbox.event_id = $1
|
|
20286
|
-
AND outbox.run_id = $2
|
|
20287
|
-
AND outbox.lease_owner = $3
|
|
20288
|
-
AND outbox.lease_expires_at > now()
|
|
20289
|
-
AND outbox.delivered_at IS NULL
|
|
20290
|
-
AND outbox.failed_at IS NULL
|
|
20291
|
-
LIMIT 1`,
|
|
20292
|
-
[input.eventId, input.runId, input.leaseOwner],
|
|
20293
|
-
);
|
|
20294
|
-
return result.rows[0] ? parseLaunchJson(result.rows[0].launch_json) : null;
|
|
20295
|
-
}
|
|
21219
|
+
export { readRunLedgerDispatchAuthorityForLease } from './run-ledger-dispatch-authority';
|
|
20296
21220
|
|
|
20297
21221
|
export async function readRunLedgerDispatchLaunchForLease(
|
|
20298
21222
|
client: PostgresSchedulerQueryClient,
|
|
@@ -20379,6 +21303,20 @@ function terminalLogContextsPayloadField(
|
|
|
20379
21303
|
return value.map(normalizePlayLogContext);
|
|
20380
21304
|
}
|
|
20381
21305
|
|
|
21306
|
+
function terminalLogMetadataPayloadField(
|
|
21307
|
+
payload: unknown,
|
|
21308
|
+
lines: readonly string[],
|
|
21309
|
+
): Array<PlayLogMetadata | null> | null {
|
|
21310
|
+
const value = payloadField(payload, 'terminalLogMetadata');
|
|
21311
|
+
if (value === undefined) return null;
|
|
21312
|
+
if (!Array.isArray(value) || value.length !== lines.length) {
|
|
21313
|
+
throw new TypeError(
|
|
21314
|
+
'Terminal log metadata must align with terminal log lines.',
|
|
21315
|
+
);
|
|
21316
|
+
}
|
|
21317
|
+
return value.map(normalizePlayLogMetadata);
|
|
21318
|
+
}
|
|
21319
|
+
|
|
20382
21320
|
function ledgerEventsPayloadField(
|
|
20383
21321
|
payload: unknown,
|
|
20384
21322
|
key: string,
|
|
@@ -20415,22 +21353,10 @@ function sourcePayloadField(
|
|
|
20415
21353
|
}
|
|
20416
21354
|
|
|
20417
21355
|
function parseLaunchJson(value: unknown): PlaySchedulerSubmitInput {
|
|
20418
|
-
|
|
20419
|
-
|
|
20420
|
-
|
|
20421
|
-
|
|
20422
|
-
if (
|
|
20423
|
-
typeof launch.runId !== 'string' ||
|
|
20424
|
-
typeof launch.playId !== 'string' ||
|
|
20425
|
-
typeof launch.playName !== 'string' ||
|
|
20426
|
-
typeof launch.executorToken !== 'string' ||
|
|
20427
|
-
typeof launch.baseUrl !== 'string'
|
|
20428
|
-
) {
|
|
20429
|
-
throw new Error(
|
|
20430
|
-
'Postgres scheduler launch_json is missing required fields.',
|
|
20431
|
-
);
|
|
20432
|
-
}
|
|
20433
|
-
return hydrateRetainedPlayPipelines(launch as PlaySchedulerSubmitInput);
|
|
21356
|
+
const launch = parseRunLedgerDispatchLaunch(
|
|
21357
|
+
value,
|
|
21358
|
+
) as PlaySchedulerSubmitInput;
|
|
21359
|
+
return hydrateRetainedPlayPipelines(launch);
|
|
20434
21360
|
}
|
|
20435
21361
|
|
|
20436
21362
|
function parsePostgresSchedulerCronSourcePin(
|