deepline 0.3.290 → 0.3.292
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +8 -6
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/runs/overview.ts +16 -6
- package/dist/bundling-sources/sdk/src/types.ts +21 -2
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +11 -1
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +209 -51
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +10 -2
- package/dist/bundling-sources/shared_libs/play-runtime/log-context.ts +36 -0
- package/dist/bundling-sources/shared_libs/play-runtime/log-provenance.ts +2 -1
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +5 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +54 -41
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +50 -3
- package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +9 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +35 -7
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +55 -9
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +16 -4
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +140 -4
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +559 -313
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +116 -24
- package/dist/bundling-sources/shared_libs/play-runtime/step-progress.ts +6 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +22 -5
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +35 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +68 -8
- package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +2 -0
- package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +2 -0
- package/dist/cli/index.js +209 -588
- package/dist/cli/index.mjs +209 -588
- package/dist/{client-FR_u0dxT.d.ts → client-Ct4jmvAg.d.ts} +29 -8
- package/dist/{client-phK1Ixcl.d.mts → client-Ct9GKrwu.d.mts} +29 -8
- package/dist/{compiler-manifest-9SQtUuKb.d.mts → compiler-manifest-qtTNYvBl.d.mts} +2 -0
- package/dist/{compiler-manifest-9SQtUuKb.d.ts → compiler-manifest-qtTNYvBl.d.ts} +2 -0
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +12 -5
- package/dist/index.mjs +12 -5
- package/dist/install-integrity.json +4 -4
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/run-request.d.mts +2 -2
- package/dist/plays/run-request.d.ts +2 -2
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -91,6 +91,8 @@ import {
|
|
|
91
91
|
import { normalizePlayDocflowNodeIoState } from '../docflow-node-io';
|
|
92
92
|
import {
|
|
93
93
|
normalizePlayLogContext,
|
|
94
|
+
normalizePlayLogMetadata,
|
|
95
|
+
type PlayLogMetadata,
|
|
94
96
|
type PlayLogContext,
|
|
95
97
|
} from '../log-context';
|
|
96
98
|
import {
|
|
@@ -481,11 +483,9 @@ type PostgresSchedulerClaimExecutionOptions = Pick<
|
|
|
481
483
|
/** Internal provider-read coordinates which must survive terminal-worker
|
|
482
484
|
* loss. They are persisted only on the matching durable cleanup obligation. */
|
|
483
485
|
sandboxCrashDiagnostic?: PostgresSchedulerSandboxCrashDiagnosticRequest;
|
|
484
|
-
/**
|
|
485
|
-
*
|
|
486
|
-
|
|
487
|
-
* settles them before the run's terminal makes its steps immutable. */
|
|
488
|
-
failedTerminalProgressEvents?: readonly unknown[];
|
|
486
|
+
/** Final step facts folded into the terminal outbox instead of a separate
|
|
487
|
+
* progress write immediately ahead of the terminal. */
|
|
488
|
+
terminalProgressEvents?: readonly unknown[];
|
|
489
489
|
/** Forwarded to `failPostgresSchedulerAttempt`: a failure terminal written
|
|
490
490
|
* for a detached-runner attempt must not overwrite an authoritative pushed
|
|
491
491
|
* runner terminal that won after the attempt was claimed. */
|
|
@@ -10823,6 +10823,7 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
|
|
|
10823
10823
|
'producerAttempt', $2::integer,
|
|
10824
10824
|
'terminalLogLines', $7::jsonb,
|
|
10825
10825
|
'terminalLogContexts', $17::jsonb,
|
|
10826
|
+
'terminalLogMetadata', $20::jsonb,
|
|
10826
10827
|
'terminalLogIds', $19::jsonb->'terminalLogIds',
|
|
10827
10828
|
'terminalProgressEvents', coalesce((
|
|
10828
10829
|
SELECT accepted.runner_log_payload_json->'terminalProgressEvents'
|
|
@@ -10868,6 +10869,7 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
|
|
|
10868
10869
|
JSON.stringify(terminalLogProjection.lineContexts),
|
|
10869
10870
|
stringifyPostgresJson(terminalWorkCounterSegment ?? null),
|
|
10870
10871
|
stringifyPostgresJson(runnerTerminalLogPayload(input)),
|
|
10872
|
+
JSON.stringify(terminalLogProjection.lineMetadata),
|
|
10871
10873
|
],
|
|
10872
10874
|
);
|
|
10873
10875
|
if (completed.rows.length > 0) return 'completed';
|
|
@@ -13200,8 +13202,10 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13200
13202
|
runtimeTiming?: PlayRunnerRuntimeTiming | null;
|
|
13201
13203
|
terminalLogLines?: readonly string[] | null;
|
|
13202
13204
|
terminalLogContexts?: readonly (PlayLogContext | null)[] | null;
|
|
13205
|
+
terminalLogMetadata?: readonly (PlayLogMetadata | null)[] | null;
|
|
13203
13206
|
terminalLogIds?: readonly (string | null)[] | null;
|
|
13204
13207
|
terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
|
|
13208
|
+
terminalProgressEvents?: readonly unknown[];
|
|
13205
13209
|
},
|
|
13206
13210
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
13207
13211
|
): Promise<boolean> {
|
|
@@ -13218,6 +13222,14 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13218
13222
|
'Terminal log contexts must align with terminal log lines.',
|
|
13219
13223
|
);
|
|
13220
13224
|
}
|
|
13225
|
+
if (
|
|
13226
|
+
input.terminalLogMetadata != null &&
|
|
13227
|
+
input.terminalLogMetadata.length !== (input.terminalLogLines ?? []).length
|
|
13228
|
+
) {
|
|
13229
|
+
throw new TypeError(
|
|
13230
|
+
'Terminal log metadata must align with terminal log lines.',
|
|
13231
|
+
);
|
|
13232
|
+
}
|
|
13221
13233
|
const terminalLogIds = normalizeRunLogIds(
|
|
13222
13234
|
input.terminalLogIds,
|
|
13223
13235
|
input.terminalLogLines ?? [],
|
|
@@ -13226,6 +13238,7 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13226
13238
|
.map((line, index) => ({
|
|
13227
13239
|
line: line.trim(),
|
|
13228
13240
|
context: input.terminalLogContexts?.[index] ?? null,
|
|
13241
|
+
metadata: input.terminalLogMetadata?.[index] ?? null,
|
|
13229
13242
|
entryId: terminalLogIds?.[index] ?? null,
|
|
13230
13243
|
}))
|
|
13231
13244
|
.filter((entry) => entry.line.length > 0)
|
|
@@ -13234,6 +13247,9 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13234
13247
|
const terminalLogContexts = terminalLogEntries.map((entry) =>
|
|
13235
13248
|
normalizePlayLogContext(entry.context),
|
|
13236
13249
|
);
|
|
13250
|
+
const terminalLogMetadata = terminalLogEntries.map((entry) =>
|
|
13251
|
+
normalizePlayLogMetadata(entry.metadata),
|
|
13252
|
+
);
|
|
13237
13253
|
const completed = await client.query<{ run_id: string }>(
|
|
13238
13254
|
`
|
|
13239
13255
|
WITH ${runtimeSandboxCustodyLocksSql('run.run_id = $1', options)}, completed AS (
|
|
@@ -13412,10 +13428,15 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13412
13428
|
'producerAttempt', $2::integer,
|
|
13413
13429
|
'terminalLogLines', $8::jsonb,
|
|
13414
13430
|
'terminalLogContexts', $11::jsonb,
|
|
13415
|
-
'
|
|
13431
|
+
'terminalLogMetadata', $13::jsonb,
|
|
13432
|
+
'terminalLogIds', $15::jsonb,
|
|
13416
13433
|
'resultSummary', $10::jsonb,
|
|
13417
13434
|
'terminalWorkCounterSegment', $12::jsonb
|
|
13418
|
-
)
|
|
13435
|
+
) || CASE
|
|
13436
|
+
WHEN jsonb_array_length($14::jsonb) > 0
|
|
13437
|
+
THEN jsonb_build_object('terminalProgressEvents', $14::jsonb)
|
|
13438
|
+
ELSE '{}'::jsonb
|
|
13439
|
+
END,
|
|
13419
13440
|
${causalOutboxCreatedAt('run_terminal.run_id', options)}
|
|
13420
13441
|
FROM run_terminal
|
|
13421
13442
|
ON CONFLICT (event_id) DO NOTHING
|
|
@@ -13438,6 +13459,8 @@ export async function completePostgresSchedulerAttempt(
|
|
|
13438
13459
|
stringifyPostgresJson(resultSummary),
|
|
13439
13460
|
JSON.stringify(terminalLogContexts),
|
|
13440
13461
|
stringifyPostgresJson(input.terminalWorkCounterSegment ?? null),
|
|
13462
|
+
JSON.stringify(terminalLogMetadata),
|
|
13463
|
+
JSON.stringify(input.terminalProgressEvents ?? []),
|
|
13441
13464
|
stringifyPostgresJson(
|
|
13442
13465
|
terminalLogIds
|
|
13443
13466
|
? terminalLogEntries.map((entry) => entry.entryId)
|
|
@@ -13461,14 +13484,15 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13461
13484
|
result?: unknown;
|
|
13462
13485
|
terminalLogLines?: readonly string[] | null;
|
|
13463
13486
|
terminalLogContexts?: readonly (PlayLogContext | null)[] | null;
|
|
13487
|
+
terminalLogMetadata?: readonly (PlayLogMetadata | null)[] | null;
|
|
13464
13488
|
terminalLogIds?: readonly (string | null)[] | null;
|
|
13465
13489
|
/** Refuse to overwrite an authoritative detached runner terminal that won
|
|
13466
13490
|
* after claim. Resumable `suspended` receipts may be superseded by the
|
|
13467
13491
|
* hard ceiling in the same fenced statement. */
|
|
13468
13492
|
preserveRunnerTerminal?: boolean;
|
|
13469
13493
|
sandboxCrashDiagnostic?: PostgresSchedulerSandboxCrashDiagnosticRequest;
|
|
13470
|
-
/** Progress facts the ledger must apply before
|
|
13471
|
-
*
|
|
13494
|
+
/** Progress facts the ledger must apply before the terminal fact. Omitted
|
|
13495
|
+
* from the payload when empty. */
|
|
13472
13496
|
terminalProgressEvents?: readonly unknown[];
|
|
13473
13497
|
},
|
|
13474
13498
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
@@ -13508,6 +13532,14 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13508
13532
|
'Terminal log contexts must align with terminal log lines.',
|
|
13509
13533
|
);
|
|
13510
13534
|
}
|
|
13535
|
+
if (
|
|
13536
|
+
input.terminalLogMetadata != null &&
|
|
13537
|
+
input.terminalLogMetadata.length !== (input.terminalLogLines ?? []).length
|
|
13538
|
+
) {
|
|
13539
|
+
throw new TypeError(
|
|
13540
|
+
'Terminal log metadata must align with terminal log lines.',
|
|
13541
|
+
);
|
|
13542
|
+
}
|
|
13511
13543
|
const terminalLogIds = normalizeRunLogIds(
|
|
13512
13544
|
input.terminalLogIds,
|
|
13513
13545
|
input.terminalLogLines ?? [],
|
|
@@ -13516,6 +13548,7 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13516
13548
|
.map((line, index) => ({
|
|
13517
13549
|
line: line.trim(),
|
|
13518
13550
|
context: input.terminalLogContexts?.[index] ?? null,
|
|
13551
|
+
metadata: input.terminalLogMetadata?.[index] ?? null,
|
|
13519
13552
|
entryId: terminalLogIds?.[index] ?? null,
|
|
13520
13553
|
}))
|
|
13521
13554
|
.filter((entry) => entry.line.length > 0)
|
|
@@ -13524,6 +13557,9 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13524
13557
|
const terminalLogContexts = terminalLogEntries.map((entry) =>
|
|
13525
13558
|
normalizePlayLogContext(entry.context),
|
|
13526
13559
|
);
|
|
13560
|
+
const terminalLogMetadata = terminalLogEntries.map((entry) =>
|
|
13561
|
+
normalizePlayLogMetadata(entry.metadata),
|
|
13562
|
+
);
|
|
13527
13563
|
const failed = await client.query<{ run_id: string }>(
|
|
13528
13564
|
`
|
|
13529
13565
|
WITH ${runtimeSandboxCustodyLocksSql('run.run_id = $1', options)}, failed AS (
|
|
@@ -13762,7 +13798,8 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13762
13798
|
'producerAttempt', $2::integer,
|
|
13763
13799
|
'terminalLogLines', $12::jsonb,
|
|
13764
13800
|
'terminalLogContexts', $14::jsonb,
|
|
13765
|
-
'
|
|
13801
|
+
'terminalLogMetadata', $17::jsonb,
|
|
13802
|
+
'terminalLogIds', $18::jsonb,
|
|
13766
13803
|
'resultSummary', $13::jsonb,
|
|
13767
13804
|
'terminalWorkCounterSegment', $15::jsonb
|
|
13768
13805
|
) || CASE
|
|
@@ -13796,6 +13833,7 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13796
13833
|
JSON.stringify(terminalLogContexts),
|
|
13797
13834
|
stringifyPostgresJson(terminalWorkCounterSegment ?? null),
|
|
13798
13835
|
stringifyPostgresJson(input.terminalProgressEvents ?? []),
|
|
13836
|
+
JSON.stringify(terminalLogMetadata),
|
|
13799
13837
|
stringifyPostgresJson(
|
|
13800
13838
|
terminalLogIds
|
|
13801
13839
|
? terminalLogEntries.map((entry) => entry.entryId)
|
|
@@ -14503,6 +14541,7 @@ function runnerTerminalLogPayload(input: {
|
|
|
14503
14541
|
producerAttempt: input.attempt,
|
|
14504
14542
|
terminalLogLines: projection.lines,
|
|
14505
14543
|
terminalLogContexts: projection.lineContexts,
|
|
14544
|
+
terminalLogMetadata: projection.lineMetadata,
|
|
14506
14545
|
...(projection.lineIds ? { terminalLogIds: projection.lineIds } : {}),
|
|
14507
14546
|
terminalProgressEvents: input.terminalProgressEvents ?? [],
|
|
14508
14547
|
// Older runner envelopes conveyed these diagnostics only in tagged log
|
|
@@ -14527,6 +14566,7 @@ function terminalLogProjectionForOutbox(
|
|
|
14527
14566
|
): {
|
|
14528
14567
|
lines: string[];
|
|
14529
14568
|
lineContexts: Array<PlayLogContext | null>;
|
|
14569
|
+
lineMetadata: Array<PlayLogMetadata | null>;
|
|
14530
14570
|
lineIds?: Array<string | null>;
|
|
14531
14571
|
} {
|
|
14532
14572
|
if (
|
|
@@ -14534,16 +14574,17 @@ function terminalLogProjectionForOutbox(
|
|
|
14534
14574
|
!runnerResult ||
|
|
14535
14575
|
typeof runnerResult !== 'object'
|
|
14536
14576
|
) {
|
|
14537
|
-
return { lines: [], lineContexts: [] };
|
|
14577
|
+
return { lines: [], lineContexts: [], lineMetadata: [] };
|
|
14538
14578
|
}
|
|
14539
14579
|
const result = runnerResult as {
|
|
14540
14580
|
logs?: unknown;
|
|
14541
14581
|
logContexts?: unknown;
|
|
14582
|
+
logMetadata?: unknown;
|
|
14542
14583
|
logIds?: unknown;
|
|
14543
14584
|
};
|
|
14544
14585
|
const logs = result.logs;
|
|
14545
14586
|
if (!Array.isArray(logs)) {
|
|
14546
|
-
return { lines: [], lineContexts: [] };
|
|
14587
|
+
return { lines: [], lineContexts: [], lineMetadata: [] };
|
|
14547
14588
|
}
|
|
14548
14589
|
if (
|
|
14549
14590
|
result.logContexts !== undefined &&
|
|
@@ -14552,6 +14593,15 @@ function terminalLogProjectionForOutbox(
|
|
|
14552
14593
|
) {
|
|
14553
14594
|
throw new TypeError('Terminal log contexts must align with runner logs.');
|
|
14554
14595
|
}
|
|
14596
|
+
if (
|
|
14597
|
+
result.logMetadata !== undefined &&
|
|
14598
|
+
(!Array.isArray(result.logMetadata) ||
|
|
14599
|
+
result.logMetadata.length !== logs.length)
|
|
14600
|
+
)
|
|
14601
|
+
throw new TypeError('Terminal log metadata must align with logs.');
|
|
14602
|
+
const logMetadata = Array.isArray(result.logMetadata)
|
|
14603
|
+
? result.logMetadata
|
|
14604
|
+
: undefined;
|
|
14555
14605
|
const logContexts = Array.isArray(result.logContexts)
|
|
14556
14606
|
? result.logContexts
|
|
14557
14607
|
: undefined;
|
|
@@ -14572,6 +14622,7 @@ function terminalLogProjectionForOutbox(
|
|
|
14572
14622
|
return {
|
|
14573
14623
|
line: line.trim(),
|
|
14574
14624
|
context: normalizePlayLogContext(logContexts?.[index]),
|
|
14625
|
+
metadata: normalizePlayLogMetadata(logMetadata?.[index]),
|
|
14575
14626
|
entryId: logIds?.[index] ?? null,
|
|
14576
14627
|
};
|
|
14577
14628
|
})
|
|
@@ -14581,6 +14632,7 @@ function terminalLogProjectionForOutbox(
|
|
|
14581
14632
|
): entry is {
|
|
14582
14633
|
line: string;
|
|
14583
14634
|
context: PlayLogContext | null;
|
|
14635
|
+
metadata: PlayLogMetadata | null;
|
|
14584
14636
|
entryId: string | null;
|
|
14585
14637
|
} => entry !== null,
|
|
14586
14638
|
)
|
|
@@ -14588,6 +14640,7 @@ function terminalLogProjectionForOutbox(
|
|
|
14588
14640
|
return {
|
|
14589
14641
|
lines: entries.map((entry) => entry.line),
|
|
14590
14642
|
lineContexts: entries.map((entry) => entry.context),
|
|
14643
|
+
lineMetadata: entries.map((entry) => entry.metadata),
|
|
14591
14644
|
...(logIds ? { lineIds: entries.map((entry) => entry.entryId) } : {}),
|
|
14592
14645
|
};
|
|
14593
14646
|
}
|
|
@@ -14837,6 +14890,8 @@ export async function executePostgresSchedulerClaim(
|
|
|
14837
14890
|
runtimeTiming: extractRunnerRuntimeTiming(output),
|
|
14838
14891
|
terminalLogLines: terminalLogProjection.lines,
|
|
14839
14892
|
terminalLogContexts: terminalLogProjection.lineContexts,
|
|
14893
|
+
terminalLogMetadata: terminalLogProjection.lineMetadata,
|
|
14894
|
+
terminalProgressEvents: options.terminalProgressEvents,
|
|
14840
14895
|
terminalLogIds: terminalLogProjection.lineIds,
|
|
14841
14896
|
terminalWorkCounterSegment: terminalWorkCounterSegmentFromRunnerResult(
|
|
14842
14897
|
output,
|
|
@@ -14899,13 +14954,14 @@ export async function executePostgresSchedulerClaim(
|
|
|
14899
14954
|
result: terminalFailureResult,
|
|
14900
14955
|
terminalLogLines: terminalLogProjection.lines,
|
|
14901
14956
|
terminalLogContexts: terminalLogProjection.lineContexts,
|
|
14957
|
+
terminalLogMetadata: terminalLogProjection.lineMetadata,
|
|
14902
14958
|
terminalLogIds: terminalLogProjection.lineIds,
|
|
14903
14959
|
sandboxCrashDiagnostic: options.sandboxCrashDiagnostic,
|
|
14904
14960
|
...(options.preserveRunnerTerminal
|
|
14905
14961
|
? { preserveRunnerTerminal: true }
|
|
14906
14962
|
: {}),
|
|
14907
|
-
...(
|
|
14908
|
-
? { terminalProgressEvents: options.
|
|
14963
|
+
...(options.terminalProgressEvents?.length
|
|
14964
|
+
? { terminalProgressEvents: options.terminalProgressEvents }
|
|
14909
14965
|
: {}),
|
|
14910
14966
|
},
|
|
14911
14967
|
options,
|
|
@@ -17862,6 +17918,32 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17862
17918
|
} = {},
|
|
17863
17919
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
17864
17920
|
): Promise<PostgresSchedulerOutboxEvent[]> {
|
|
17921
|
+
const { events } = await claimPostgresSchedulerOutboxPrefix(
|
|
17922
|
+
client,
|
|
17923
|
+
input,
|
|
17924
|
+
options,
|
|
17925
|
+
);
|
|
17926
|
+
return events;
|
|
17927
|
+
}
|
|
17928
|
+
|
|
17929
|
+
/**
|
|
17930
|
+
* ACK and next-prefix selection share one statement after the run lock. Data
|
|
17931
|
+
* CTEs use the statement's original snapshot, so every pending-row predicate
|
|
17932
|
+
* explicitly excludes the acknowledged IDs. A partial ACK cannot claim a
|
|
17933
|
+
* successor. The receiver's ingress capability and frozen receipts still
|
|
17934
|
+
* determine which causal facts may share a prefix.
|
|
17935
|
+
*/
|
|
17936
|
+
async function claimPostgresSchedulerOutboxPrefix(
|
|
17937
|
+
client: PostgresSchedulerQueryClient,
|
|
17938
|
+
input: NonNullable<
|
|
17939
|
+
Parameters<typeof readPendingPostgresSchedulerOutboxEvents>[1]
|
|
17940
|
+
>,
|
|
17941
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
17942
|
+
acknowledgement?: { eventIds: string[]; targetEventId: string },
|
|
17943
|
+
): Promise<{
|
|
17944
|
+
events: PostgresSchedulerOutboxEvent[];
|
|
17945
|
+
deliveredRows: RunLedgerOutboxDeliveredRow[];
|
|
17946
|
+
}> {
|
|
17865
17947
|
const requestedLimit = Math.floor(input.limit ?? 1);
|
|
17866
17948
|
const limit = Number.isFinite(requestedLimit)
|
|
17867
17949
|
? Math.max(1, requestedLimit)
|
|
@@ -17869,6 +17951,10 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17869
17951
|
const runId = input.runId?.trim() || null;
|
|
17870
17952
|
const leaseOwner = input.leaseOwner?.trim() || null;
|
|
17871
17953
|
const leaseSeconds = Math.max(5, Math.min(300, input.leaseSeconds ?? 30));
|
|
17954
|
+
const pending = (alias: string) =>
|
|
17955
|
+
`${alias}.delivered_at IS NULL${
|
|
17956
|
+
acknowledgement ? ` AND NOT (${alias}.event_id = ANY($5::text[]))` : ''
|
|
17957
|
+
}`;
|
|
17872
17958
|
const claim = async (supportsTerminalBatchMarker: boolean) => {
|
|
17873
17959
|
const urgentLifecyclePendingSql = (alias: string) =>
|
|
17874
17960
|
input.searchOnly === true
|
|
@@ -17879,15 +17965,23 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17879
17965
|
? `(head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} AND ${urgentLifecyclePendingSql('head')})`
|
|
17880
17966
|
: 'false';
|
|
17881
17967
|
return await client.query<{
|
|
17968
|
+
row_kind: 'claimed' | 'delivered';
|
|
17882
17969
|
event_id: string;
|
|
17883
17970
|
run_id: string;
|
|
17884
17971
|
event_type: string;
|
|
17885
17972
|
payload_json: unknown;
|
|
17886
17973
|
created_at: Date;
|
|
17887
17974
|
delivery_key: string | null;
|
|
17975
|
+
lag_ms: number | string | null;
|
|
17888
17976
|
}>(
|
|
17889
17977
|
`
|
|
17890
17978
|
WITH ${
|
|
17979
|
+
acknowledgement
|
|
17980
|
+
? `delivered AS (
|
|
17981
|
+
${runLedgerAcknowledgeSql(options, '$5', '$3')}
|
|
17982
|
+
),`
|
|
17983
|
+
: ''
|
|
17984
|
+
} ${
|
|
17891
17985
|
input.searchOnly === true
|
|
17892
17986
|
? ''
|
|
17893
17987
|
: `waiting_runs AS MATERIALIZED (
|
|
@@ -17898,7 +17992,10 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17898
17992
|
? `owned_run AS MATERIALIZED (
|
|
17899
17993
|
SELECT run_id FROM ${tableName('runs', options)}
|
|
17900
17994
|
WHERE run_id = $2
|
|
17901
|
-
|
|
17995
|
+
-- This dispatcher already owns the head. A producer holding the run
|
|
17996
|
+
-- lock is not a lost delivery lease: wait for its short transaction,
|
|
17997
|
+
-- then revalidate the head below. New-run selection still skips locks.
|
|
17998
|
+
FOR UPDATE
|
|
17902
17999
|
),`
|
|
17903
18000
|
: ''
|
|
17904
18001
|
} head AS MATERIALIZED (
|
|
@@ -17906,8 +18003,18 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17906
18003
|
${supportsTerminalBatchMarker ? 'candidate.delivery_batch_started_at' : 'NULL::timestamptz'} AS delivery_batch_started_at
|
|
17907
18004
|
FROM ${tableName('outbox', options)} AS candidate
|
|
17908
18005
|
${input.extendOwnedHead ? 'JOIN owned_run ON owned_run.run_id = candidate.run_id' : ''}
|
|
17909
|
-
WHERE candidate
|
|
18006
|
+
WHERE ${pending('candidate')}
|
|
17910
18007
|
AND ${searchLaneSql('candidate')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
18008
|
+
${
|
|
18009
|
+
acknowledgement
|
|
18010
|
+
? `AND (SELECT count(*) FROM delivered) = cardinality($5::text[])
|
|
18011
|
+
AND EXISTS (
|
|
18012
|
+
SELECT 1 FROM ${tableName('outbox', options)} AS target
|
|
18013
|
+
WHERE target.run_id = $2 AND target.event_id = $6
|
|
18014
|
+
AND ${pending('target')} AND target.failed_at IS NULL
|
|
18015
|
+
)`
|
|
18016
|
+
: ''
|
|
18017
|
+
}
|
|
17911
18018
|
${
|
|
17912
18019
|
input.extendOwnedHead
|
|
17913
18020
|
? `AND candidate.event_id = $5
|
|
@@ -17929,8 +18036,8 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17929
18036
|
AND NOT EXISTS (
|
|
17930
18037
|
SELECT 1 FROM ${tableName('outbox', options)} AS active
|
|
17931
18038
|
WHERE active.run_id = candidate.run_id
|
|
18039
|
+
AND ${pending('active')}
|
|
17932
18040
|
AND ${searchLaneSql('active')} = ${searchLaneSql('candidate')}
|
|
17933
|
-
AND active.delivered_at IS NULL
|
|
17934
18041
|
AND active.lease_owner IS NOT NULL
|
|
17935
18042
|
AND active.lease_expires_at > now()
|
|
17936
18043
|
AND active.lease_owner IS DISTINCT FROM $3::text
|
|
@@ -17942,7 +18049,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17942
18049
|
SELECT 1
|
|
17943
18050
|
FROM ${tableName('outbox', options)} AS predecessor
|
|
17944
18051
|
WHERE predecessor.run_id = candidate.run_id
|
|
17945
|
-
AND predecessor
|
|
18052
|
+
AND ${pending('predecessor')}
|
|
17946
18053
|
AND ${runLedgerPrecedesSql('predecessor', 'candidate')}
|
|
17947
18054
|
)
|
|
17948
18055
|
-- A new durable admission has no causal dependency on unrelated
|
|
@@ -17969,7 +18076,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
17969
18076
|
SELECT outbox.event_id
|
|
17970
18077
|
FROM ${tableName('outbox', options)} AS outbox
|
|
17971
18078
|
JOIN head ON head.run_id = outbox.run_id
|
|
17972
|
-
WHERE outbox
|
|
18079
|
+
WHERE ${pending('outbox')}
|
|
17973
18080
|
AND ${searchLaneSql('outbox')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
17974
18081
|
AND outbox.failed_at IS NULL
|
|
17975
18082
|
-- Admission is causally first and must be projected alone. A later
|
|
@@ -18006,7 +18113,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
18006
18113
|
SELECT 1
|
|
18007
18114
|
FROM ${tableName('outbox', options)} AS terminal_predecessor
|
|
18008
18115
|
WHERE terminal_predecessor.run_id = outbox.run_id
|
|
18009
|
-
AND terminal_predecessor
|
|
18116
|
+
AND ${pending('terminal_predecessor')}
|
|
18010
18117
|
AND ${
|
|
18011
18118
|
supportsTerminalBatchMarker
|
|
18012
18119
|
? 'terminal_predecessor.delivery_batch_started_at IS NOT NULL'
|
|
@@ -18028,7 +18135,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
18028
18135
|
SELECT 1
|
|
18029
18136
|
FROM ${tableName('outbox', options)} AS predecessor
|
|
18030
18137
|
WHERE predecessor.run_id = outbox.run_id
|
|
18031
|
-
AND predecessor
|
|
18138
|
+
AND ${pending('predecessor')}
|
|
18032
18139
|
AND ${runLedgerPrecedesSql('predecessor', 'outbox')}
|
|
18033
18140
|
AND (
|
|
18034
18141
|
(predecessor.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
|
|
@@ -18077,17 +18184,50 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
18077
18184
|
outbox.payload_json, outbox.created_at, outbox.lease_owner,
|
|
18078
18185
|
outbox.delivery_key
|
|
18079
18186
|
)
|
|
18080
|
-
|
|
18081
|
-
|
|
18082
|
-
|
|
18083
|
-
|
|
18187
|
+
${
|
|
18188
|
+
acknowledgement
|
|
18189
|
+
? `, notified AS MATERIALIZED (
|
|
18190
|
+
SELECT delivered.*,
|
|
18191
|
+
pg_notify(
|
|
18192
|
+
$7,
|
|
18193
|
+
json_build_object('eventId', event_id, 'kind', 'outbox_delivered')::text
|
|
18194
|
+
) AS notification
|
|
18195
|
+
FROM delivered
|
|
18196
|
+
)`
|
|
18197
|
+
: ''
|
|
18198
|
+
}
|
|
18199
|
+
SELECT * FROM (
|
|
18200
|
+
SELECT 'claimed'::text AS row_kind,
|
|
18201
|
+
event_id, run_id, event_type, payload_json, created_at, lease_owner,
|
|
18202
|
+
delivery_key, NULL::float8 AS lag_ms
|
|
18203
|
+
FROM claimed
|
|
18204
|
+
${
|
|
18205
|
+
acknowledgement
|
|
18206
|
+
? `UNION ALL
|
|
18207
|
+
SELECT 'delivered'::text AS row_kind,
|
|
18208
|
+
event_id, NULL::text AS run_id, event_type, NULL::jsonb AS payload_json,
|
|
18209
|
+
NULL::timestamptz AS created_at, NULL::text AS lease_owner,
|
|
18210
|
+
NULL::text AS delivery_key, lag_ms
|
|
18211
|
+
FROM notified`
|
|
18212
|
+
: ''
|
|
18213
|
+
}
|
|
18214
|
+
) AS result
|
|
18215
|
+
ORDER BY row_kind, ${runLedgerOrderSql('result')}
|
|
18084
18216
|
`,
|
|
18085
18217
|
[
|
|
18086
18218
|
limit,
|
|
18087
18219
|
runId,
|
|
18088
18220
|
leaseOwner,
|
|
18089
18221
|
leaseSeconds,
|
|
18090
|
-
...(
|
|
18222
|
+
...(acknowledgement
|
|
18223
|
+
? [
|
|
18224
|
+
acknowledgement.eventIds,
|
|
18225
|
+
acknowledgement.targetEventId,
|
|
18226
|
+
postgresSchedulerNotifyChannel('terminal', options),
|
|
18227
|
+
]
|
|
18228
|
+
: input.extendOwnedHead
|
|
18229
|
+
? [input.extendOwnedHead]
|
|
18230
|
+
: []),
|
|
18091
18231
|
],
|
|
18092
18232
|
);
|
|
18093
18233
|
};
|
|
@@ -18098,9 +18238,13 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
18098
18238
|
result = await claim(true);
|
|
18099
18239
|
} catch (error) {
|
|
18100
18240
|
if (!isMissingOutboxDeliveryBatchMarker(error)) throw error;
|
|
18241
|
+
markOutboxDeliveryBatchMarkerUnavailable(options);
|
|
18242
|
+
// The ACK caller owns the whole short transaction. Roll it back and
|
|
18243
|
+
// repeat only this missing-column adapter, avoiding two SAVEPOINT round
|
|
18244
|
+
// trips on every successful acknowledgement.
|
|
18245
|
+
if (acknowledgement) throw error;
|
|
18101
18246
|
if (input.inTransaction)
|
|
18102
18247
|
await client.query('ROLLBACK TO SAVEPOINT run_ledger_claim');
|
|
18103
|
-
markOutboxDeliveryBatchMarkerUnavailable(options);
|
|
18104
18248
|
result = await claim(false);
|
|
18105
18249
|
}
|
|
18106
18250
|
} else {
|
|
@@ -18108,22 +18252,34 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
18108
18252
|
}
|
|
18109
18253
|
if (input.inTransaction)
|
|
18110
18254
|
await client.query('RELEASE SAVEPOINT run_ledger_claim');
|
|
18111
|
-
|
|
18112
|
-
|
|
18113
|
-
|
|
18114
|
-
|
|
18115
|
-
|
|
18116
|
-
|
|
18117
|
-
|
|
18118
|
-
|
|
18119
|
-
|
|
18120
|
-
|
|
18121
|
-
|
|
18122
|
-
|
|
18123
|
-
|
|
18124
|
-
|
|
18125
|
-
|
|
18126
|
-
|
|
18255
|
+
const events = result.rows
|
|
18256
|
+
.filter((row) => row.row_kind === 'claimed')
|
|
18257
|
+
.map((row) => {
|
|
18258
|
+
if (!row.delivery_key) {
|
|
18259
|
+
throw new Error(
|
|
18260
|
+
`Run-ledger outbox event ${row.event_id} has no delivery key.`,
|
|
18261
|
+
);
|
|
18262
|
+
}
|
|
18263
|
+
return {
|
|
18264
|
+
eventId: row.event_id,
|
|
18265
|
+
runId: row.run_id,
|
|
18266
|
+
eventType: row.event_type,
|
|
18267
|
+
payload: row.payload_json,
|
|
18268
|
+
createdAt: row.created_at,
|
|
18269
|
+
leaseOwner: leaseOwner,
|
|
18270
|
+
deliveryKey: row.delivery_key,
|
|
18271
|
+
};
|
|
18272
|
+
});
|
|
18273
|
+
const deliveredRows = result.rows
|
|
18274
|
+
.filter((row) => row.row_kind === 'delivered')
|
|
18275
|
+
.map((row) => {
|
|
18276
|
+
const lagMs = Number(row.lag_ms);
|
|
18277
|
+
return {
|
|
18278
|
+
eventType: row.event_type,
|
|
18279
|
+
lagMs: Number.isFinite(lagMs) ? Math.max(0, lagMs) : 0,
|
|
18280
|
+
};
|
|
18281
|
+
});
|
|
18282
|
+
return { events, deliveredRows };
|
|
18127
18283
|
}
|
|
18128
18284
|
|
|
18129
18285
|
/**
|
|
@@ -18388,7 +18544,7 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
|
|
|
18388
18544
|
'{events}',
|
|
18389
18545
|
(SELECT jsonb_agg(
|
|
18390
18546
|
CASE WHEN item.value->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
|
|
18391
|
-
THEN item.value - 'lines' - 'lineContexts' - 'lineIds'
|
|
18547
|
+
THEN item.value - 'lines' - 'lineContexts' - 'lineMetadata' - 'lineIds'
|
|
18392
18548
|
ELSE item.value
|
|
18393
18549
|
END ORDER BY item.ordinality
|
|
18394
18550
|
)
|
|
@@ -18401,11 +18557,11 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
|
|
|
18401
18557
|
)
|
|
18402
18558
|
WHEN jsonb_typeof(outbox.payload_json->'terminalProgressEvents') = 'array'
|
|
18403
18559
|
THEN jsonb_set(
|
|
18404
|
-
outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogIds',
|
|
18560
|
+
outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogMetadata' - 'terminalLogIds',
|
|
18405
18561
|
'{terminalProgressEvents}',
|
|
18406
18562
|
coalesce((SELECT jsonb_agg(
|
|
18407
18563
|
CASE WHEN item.value->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
|
|
18408
|
-
THEN item.value - 'lines' - 'lineContexts' - 'lineIds'
|
|
18564
|
+
THEN item.value - 'lines' - 'lineContexts' - 'lineMetadata' - 'lineIds'
|
|
18409
18565
|
ELSE item.value
|
|
18410
18566
|
END ORDER BY item.ordinality
|
|
18411
18567
|
)
|
|
@@ -18413,7 +18569,7 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
|
|
|
18413
18569
|
WITH ORDINALITY AS item(value, ordinality)), '[]'::jsonb),
|
|
18414
18570
|
false
|
|
18415
18571
|
)
|
|
18416
|
-
ELSE outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogIds'
|
|
18572
|
+
ELSE outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogMetadata' - 'terminalLogIds'
|
|
18417
18573
|
END
|
|
18418
18574
|
FROM candidates
|
|
18419
18575
|
WHERE outbox.event_id = candidates.event_id`,
|
|
@@ -18475,6 +18631,17 @@ function runLedgerDeliveryFailure(input: { error: unknown }): {
|
|
|
18475
18631
|
projectionMismatch: RunLedgerProjectionMismatchDiagnostic | null;
|
|
18476
18632
|
retryAfterMs: number | null;
|
|
18477
18633
|
} {
|
|
18634
|
+
if (input.error instanceof RunLedgerProjectionLeaseLostError) {
|
|
18635
|
+
return {
|
|
18636
|
+
permanent: false,
|
|
18637
|
+
code: 'RUN_LEDGER_DELIVERY_LEASE_LOST',
|
|
18638
|
+
status: null,
|
|
18639
|
+
retryable: true,
|
|
18640
|
+
failureReason: input.error.reason,
|
|
18641
|
+
projectionMismatch: input.error.projectionMismatch ?? null,
|
|
18642
|
+
retryAfterMs: null,
|
|
18643
|
+
};
|
|
18644
|
+
}
|
|
18478
18645
|
if (input.error instanceof AppRuntimeApiResponseError) {
|
|
18479
18646
|
// Projection is idempotent. A dependency can incorrectly send
|
|
18480
18647
|
// `retryable:false` with a transient overload response, but that metadata
|
|
@@ -18655,6 +18822,32 @@ export async function markPostgresSchedulerOutboxDelivered(
|
|
|
18655
18822
|
).length;
|
|
18656
18823
|
}
|
|
18657
18824
|
|
|
18825
|
+
/** Shared lease-fenced ACK, including the canonical retained-payload cleanup. */
|
|
18826
|
+
function runLedgerAcknowledgeSql(
|
|
18827
|
+
options: Pick<PostgresSchedulerOptions, 'schema'> | undefined,
|
|
18828
|
+
eventIdsParameter: string,
|
|
18829
|
+
leaseOwnerParameter: string,
|
|
18830
|
+
): string {
|
|
18831
|
+
return `UPDATE ${tableName('outbox', options)} AS outbox
|
|
18832
|
+
SET delivered_at = now(),
|
|
18833
|
+
next_attempt_at = NULL,
|
|
18834
|
+
last_error_json = NULL,
|
|
18835
|
+
lease_owner = NULL,
|
|
18836
|
+
lease_expires_at = NULL,
|
|
18837
|
+
payload_json = CASE WHEN EXISTS (
|
|
18838
|
+
SELECT 1 FROM ${tableName('runs', options)} AS run
|
|
18839
|
+
WHERE run.run_id = outbox.run_id
|
|
18840
|
+
AND run.run_event_store = 'planet_scale'
|
|
18841
|
+
) THEN '{}'::jsonb ELSE outbox.payload_json END
|
|
18842
|
+
WHERE outbox.event_id = ANY(${eventIdsParameter}::text[])
|
|
18843
|
+
AND outbox.delivered_at IS NULL
|
|
18844
|
+
AND (${leaseOwnerParameter}::text IS NULL OR outbox.lease_owner = ${leaseOwnerParameter})
|
|
18845
|
+
RETURNING outbox.event_id,
|
|
18846
|
+
outbox.event_type,
|
|
18847
|
+
(extract(epoch FROM (outbox.delivered_at - outbox.created_at)) * 1000)::float8
|
|
18848
|
+
AS lag_ms`;
|
|
18849
|
+
}
|
|
18850
|
+
|
|
18658
18851
|
/**
|
|
18659
18852
|
* Lease-fenced acknowledgement that also reports each row's event type and
|
|
18660
18853
|
* commit-to-acknowledgement lag on the database clock, so lag metrics never
|
|
@@ -18675,24 +18868,7 @@ export async function markPostgresSchedulerOutboxDeliveredRows(
|
|
|
18675
18868
|
}>(
|
|
18676
18869
|
`
|
|
18677
18870
|
WITH delivered AS (
|
|
18678
|
-
|
|
18679
|
-
SET delivered_at = now(),
|
|
18680
|
-
next_attempt_at = NULL,
|
|
18681
|
-
last_error_json = NULL,
|
|
18682
|
-
lease_owner = NULL,
|
|
18683
|
-
lease_expires_at = NULL,
|
|
18684
|
-
payload_json = CASE WHEN EXISTS (
|
|
18685
|
-
SELECT 1 FROM ${tableName('runs', options)} AS run
|
|
18686
|
-
WHERE run.run_id = outbox.run_id
|
|
18687
|
-
AND run.run_event_store = 'planet_scale'
|
|
18688
|
-
) THEN '{}'::jsonb ELSE outbox.payload_json END
|
|
18689
|
-
WHERE outbox.event_id = ANY($1::text[])
|
|
18690
|
-
AND outbox.delivered_at IS NULL
|
|
18691
|
-
AND ($2::text IS NULL OR outbox.lease_owner = $2)
|
|
18692
|
-
RETURNING outbox.event_id,
|
|
18693
|
-
outbox.event_type,
|
|
18694
|
-
(extract(epoch FROM (outbox.delivered_at - outbox.created_at)) * 1000)::float8
|
|
18695
|
-
AS lag_ms
|
|
18871
|
+
${runLedgerAcknowledgeSql(options, '$1', '$2')}
|
|
18696
18872
|
)
|
|
18697
18873
|
SELECT event_id,
|
|
18698
18874
|
event_type,
|
|
@@ -18846,6 +19022,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
18846
19022
|
);
|
|
18847
19023
|
const lines = stringArrayPayloadField(event.payload, 'terminalLogLines');
|
|
18848
19024
|
const lineContexts = terminalLogContextsPayloadField(event.payload, lines);
|
|
19025
|
+
const lineMetadata = terminalLogMetadataPayloadField(event.payload, lines);
|
|
18849
19026
|
const resultSummary = payloadField(event.payload, 'resultSummary');
|
|
18850
19027
|
return [
|
|
18851
19028
|
...progressEvents.map(normalizeOutboxIngressPreview),
|
|
@@ -18859,6 +19036,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
18859
19036
|
source: 'worker',
|
|
18860
19037
|
lines,
|
|
18861
19038
|
...(lineContexts ? { lineContexts } : {}),
|
|
19039
|
+
...(lineMetadata ? { lineMetadata } : {}),
|
|
18862
19040
|
} as PlayRunLedgerEvent,
|
|
18863
19041
|
]),
|
|
18864
19042
|
{
|
|
@@ -18888,6 +19066,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
18888
19066
|
);
|
|
18889
19067
|
const lines = stringArrayPayloadField(event.payload, 'terminalLogLines');
|
|
18890
19068
|
const lineContexts = terminalLogContextsPayloadField(event.payload, lines);
|
|
19069
|
+
const lineMetadata = terminalLogMetadataPayloadField(event.payload, lines);
|
|
18891
19070
|
const result = terminalRunResultForLedger(
|
|
18892
19071
|
payloadField(event.payload, 'result'),
|
|
18893
19072
|
);
|
|
@@ -18904,6 +19083,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
18904
19083
|
source: 'worker',
|
|
18905
19084
|
lines,
|
|
18906
19085
|
...(lineContexts ? { lineContexts } : {}),
|
|
19086
|
+
...(lineMetadata ? { lineMetadata } : {}),
|
|
18907
19087
|
} as PlayRunLedgerEvent,
|
|
18908
19088
|
]),
|
|
18909
19089
|
{
|
|
@@ -19148,7 +19328,7 @@ export async function listPostgresSchedulerOutboxMaterializationPlans(
|
|
|
19148
19328
|
* - `local_mint`: sign the token in-process with `BETTER_AUTH_SECRET`. ONLY the
|
|
19149
19329
|
* app (Vercel) legitimately holds the signing key, so this is app-side only
|
|
19150
19330
|
* (read-model heal path). It is the AUTHORITATIVE projector.
|
|
19151
|
-
* - `gateway`:
|
|
19331
|
+
* - `gateway`: for every app-runtime append, prove the active outbox lease to the launch-pinned
|
|
19152
19332
|
* deployment and receive a short-lived executor JWT
|
|
19153
19333
|
* (`requestRunLedgerDispatchToken`), even if the caller supplied a
|
|
19154
19334
|
* `runtimeApi`. The Fly worker has no signing key and MUST use this. A
|
|
@@ -19219,7 +19399,7 @@ export type RunLedgerOutboxDispatchInput = {
|
|
|
19219
19399
|
/** Only the existing background projector indexes deferred raw inputs. */
|
|
19220
19400
|
materializeObservability?: boolean;
|
|
19221
19401
|
/**
|
|
19222
|
-
* Optional runtime context. `gateway`
|
|
19402
|
+
* Optional runtime context. For an app-runtime append, `gateway` replaces its
|
|
19223
19403
|
* credentials with a fresh lease-scoped token and keeps only supported
|
|
19224
19404
|
* transport options; `local_mint` and the default authority preserve the
|
|
19225
19405
|
* explicit context as before.
|
|
@@ -19518,13 +19698,16 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19518
19698
|
): Promise<PostgresSchedulerOutboxProjectionResult> {
|
|
19519
19699
|
const dispatchStartedAtMs = Date.now();
|
|
19520
19700
|
const tokenAuthority = input.tokenAuthority ?? 'local_mint';
|
|
19521
|
-
const dispatchLaunchSql =
|
|
19522
|
-
!input.searchOnly && (tokenAuthority === 'gateway' || !input.runtimeApi)
|
|
19523
|
-
? runLedgerDispatchLaunchSql('runs')
|
|
19524
|
-
: 'NULL::jsonb';
|
|
19525
|
-
let claimedLaunchJson: unknown;
|
|
19526
19701
|
const leaseOwner =
|
|
19527
19702
|
input.leaseOwner?.trim() || `run-ledger-outbox-dispatch:${randomUUID()}`;
|
|
19703
|
+
const needsLaunch =
|
|
19704
|
+
!input.searchOnly && (tokenAuthority === 'gateway' || !input.runtimeApi);
|
|
19705
|
+
const launchSelection = needsLaunch
|
|
19706
|
+
? `CASE
|
|
19707
|
+
WHEN ${runLedgerSearchOnlyOutboxFactSql('head')} THEN NULL::jsonb
|
|
19708
|
+
ELSE ${runLedgerDispatchLaunchSql('runs')}
|
|
19709
|
+
END`
|
|
19710
|
+
: 'NULL::jsonb';
|
|
19528
19711
|
const continuation: { current: RunLedgerOutboxDrainScope | null } = {
|
|
19529
19712
|
current: scope ?? null,
|
|
19530
19713
|
};
|
|
@@ -19576,13 +19759,14 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19576
19759
|
const selected = targetRunId
|
|
19577
19760
|
? await client.query<{
|
|
19578
19761
|
run_id: string;
|
|
19579
|
-
dispatch_launch_json
|
|
19762
|
+
dispatch_launch_json: unknown;
|
|
19580
19763
|
}>(
|
|
19581
19764
|
`
|
|
19582
19765
|
WITH head AS MATERIALIZED (
|
|
19583
19766
|
SELECT pending.run_id,
|
|
19584
19767
|
pending.event_id,
|
|
19585
19768
|
pending.event_type,
|
|
19769
|
+
pending.payload_json,
|
|
19586
19770
|
pending.created_at,
|
|
19587
19771
|
pending.next_attempt_at,
|
|
19588
19772
|
pending.failed_at
|
|
@@ -19593,7 +19777,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19593
19777
|
ORDER BY ${runLedgerOrderSql('pending')}
|
|
19594
19778
|
LIMIT 1
|
|
19595
19779
|
)
|
|
19596
|
-
SELECT runs.run_id, ${
|
|
19780
|
+
SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json
|
|
19597
19781
|
FROM head
|
|
19598
19782
|
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
|
|
19599
19783
|
WHERE head.failed_at IS NULL
|
|
@@ -19614,7 +19798,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19614
19798
|
)
|
|
19615
19799
|
: await client.query<{
|
|
19616
19800
|
run_id: string;
|
|
19617
|
-
dispatch_launch_json
|
|
19801
|
+
dispatch_launch_json: unknown;
|
|
19618
19802
|
}>(`
|
|
19619
19803
|
WITH ${
|
|
19620
19804
|
input.searchOnly === true
|
|
@@ -19627,6 +19811,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19627
19811
|
pending.run_id,
|
|
19628
19812
|
pending.event_id,
|
|
19629
19813
|
pending.event_type,
|
|
19814
|
+
pending.payload_json,
|
|
19630
19815
|
pending.created_at,
|
|
19631
19816
|
pending.next_attempt_at,
|
|
19632
19817
|
pending.failed_at
|
|
@@ -19635,7 +19820,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19635
19820
|
AND ${searchLaneSql('pending')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
19636
19821
|
ORDER BY pending.run_id, ${runLedgerOrderSql('pending')}
|
|
19637
19822
|
)
|
|
19638
|
-
SELECT runs.run_id, ${
|
|
19823
|
+
SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json
|
|
19639
19824
|
FROM heads AS head
|
|
19640
19825
|
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
|
|
19641
19826
|
WHERE head.failed_at IS NULL
|
|
@@ -19680,24 +19865,28 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19680
19865
|
options,
|
|
19681
19866
|
)
|
|
19682
19867
|
: [];
|
|
19683
|
-
// Authority/routing is immutable and already lives on the locked Run.
|
|
19684
|
-
// Capture only its existing narrow projection with the successful lease
|
|
19685
|
-
// claim, then release the checkout before requesting gateway authority.
|
|
19686
|
-
// The gateway still independently proves that this lease is current.
|
|
19687
|
-
const launchJson = selected.rows[0]?.dispatch_launch_json;
|
|
19688
|
-
if (claimed.length > 0 && launchJson != null) {
|
|
19689
|
-
claimedLaunchJson = launchJson;
|
|
19690
|
-
}
|
|
19691
19868
|
await client.query('COMMIT');
|
|
19692
|
-
|
|
19869
|
+
// The launch is immutable. Carry only its authority/routing fields from
|
|
19870
|
+
// the run we already locked, rather than queuing another scarce checkout
|
|
19871
|
+
// between the claim and gateway request. The gateway still proves the
|
|
19872
|
+
// active outbox lease; no database checkout spans either HTTP hop.
|
|
19873
|
+
return {
|
|
19874
|
+
events: claimed,
|
|
19875
|
+
launchJson: selected.rows[0]?.dispatch_launch_json,
|
|
19876
|
+
};
|
|
19693
19877
|
} catch (error) {
|
|
19694
19878
|
await client.query('ROLLBACK');
|
|
19695
19879
|
throw error;
|
|
19696
19880
|
}
|
|
19697
19881
|
};
|
|
19698
|
-
|
|
19882
|
+
const preclaimed = scope?.nextClaimed.length
|
|
19699
19883
|
? scope.nextClaimed.splice(0)
|
|
19884
|
+
: null;
|
|
19885
|
+
const claimed = preclaimed
|
|
19886
|
+
? { events: preclaimed, launchJson: null }
|
|
19700
19887
|
: await withClient(claimEvents);
|
|
19888
|
+
const { launchJson } = claimed;
|
|
19889
|
+
let events = claimed.events;
|
|
19701
19890
|
if (events.length === 0) {
|
|
19702
19891
|
return {
|
|
19703
19892
|
attempted: 0,
|
|
@@ -19806,13 +19995,12 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19806
19995
|
return null;
|
|
19807
19996
|
};
|
|
19808
19997
|
try {
|
|
19809
|
-
|
|
19810
|
-
|
|
19811
|
-
|
|
19812
|
-
|
|
19998
|
+
// The ledger converter rejects raw deferred facts. Inline callers hand
|
|
19999
|
+
// them off; the background owner projects them before inspecting the
|
|
20000
|
+
// compact lifecycle payload or deciding that it needs no HTTP append.
|
|
20001
|
+
const initialHandedOff = await materializeDeferredObservability();
|
|
20002
|
+
if (initialHandedOff) return initialHandedOff;
|
|
19813
20003
|
if (input.searchOnly) {
|
|
19814
|
-
const handedOff = await materializeDeferredObservability();
|
|
19815
|
-
if (handedOff) return handedOff;
|
|
19816
20004
|
if (
|
|
19817
20005
|
events.some(
|
|
19818
20006
|
(event) =>
|
|
@@ -19852,18 +20040,23 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19852
20040
|
};
|
|
19853
20041
|
}
|
|
19854
20042
|
const first = events[0]!;
|
|
19855
|
-
|
|
19856
|
-
|
|
19857
|
-
|
|
19858
|
-
|
|
20043
|
+
let ledgerEvents = RUN_SCHEDULER_OUTBOX_START_EVENT_TYPES.includes(
|
|
20044
|
+
first.eventType,
|
|
20045
|
+
)
|
|
20046
|
+
? null
|
|
20047
|
+
: events.flatMap(buildRunLedgerEventsFromPostgresSchedulerOutbox);
|
|
20048
|
+
// Search-only facts are already durable and have no ledger projection.
|
|
20049
|
+
// Validate and acknowledge them without acquiring runtime authority or
|
|
20050
|
+
// renewing a network-delivery lease. A following terminal still uses the
|
|
20051
|
+
// same fenced continuation and loads authority only when it needs a write.
|
|
19859
20052
|
if (
|
|
19860
20053
|
scope &&
|
|
19861
20054
|
(scope.runId !== first.runId || scope.leaseOwner !== leaseOwner)
|
|
19862
20055
|
) {
|
|
19863
20056
|
throw new Error('Run-ledger continuation is outside its dispatch scope.');
|
|
19864
20057
|
}
|
|
19865
|
-
|
|
19866
|
-
|
|
20058
|
+
const needsRuntimeApi = ledgerEvents === null || ledgerEvents.length > 0;
|
|
20059
|
+
let runtimeApi: WorkerRuntimeApiContext | null = null;
|
|
19867
20060
|
const reuseMarginMs =
|
|
19868
20061
|
Math.max(
|
|
19869
20062
|
1,
|
|
@@ -19873,24 +20066,36 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19873
20066
|
),
|
|
19874
20067
|
) + RUNTIME_RELIABILITY_POLICY.schedulerDatabase.queryTimeoutMs;
|
|
19875
20068
|
const cachedAuthority = authorityScope.authority;
|
|
20069
|
+
const resolveRunRuntimeApi = () =>
|
|
20070
|
+
launchJson !== null
|
|
20071
|
+
? resolvePostgresSchedulerRuntimeApiForRun(
|
|
20072
|
+
launchJson,
|
|
20073
|
+
first.runId,
|
|
20074
|
+
first.leaseOwner
|
|
20075
|
+
? { eventId: first.eventId, leaseOwner: first.leaseOwner }
|
|
20076
|
+
: null,
|
|
20077
|
+
tokenAuthority,
|
|
20078
|
+
options,
|
|
20079
|
+
input.runtimeApi?.signal,
|
|
20080
|
+
)
|
|
20081
|
+
: loadPostgresSchedulerRuntimeApiForRun(
|
|
20082
|
+
withClient,
|
|
20083
|
+
first.runId,
|
|
20084
|
+
first.leaseOwner
|
|
20085
|
+
? { eventId: first.eventId, leaseOwner: first.leaseOwner }
|
|
20086
|
+
: null,
|
|
20087
|
+
tokenAuthority,
|
|
20088
|
+
options,
|
|
20089
|
+
input.runtimeApi?.signal,
|
|
20090
|
+
);
|
|
19876
20091
|
const gatewayRuntimeApi =
|
|
19877
|
-
tokenAuthority === 'gateway'
|
|
20092
|
+
needsRuntimeApi && tokenAuthority === 'gateway'
|
|
19878
20093
|
? cachedAuthority?.runId === first.runId &&
|
|
19879
20094
|
cachedAuthority.leaseOwner === leaseOwner &&
|
|
19880
20095
|
cachedAuthority.expiresAtMs !== null &&
|
|
19881
20096
|
cachedAuthority.expiresAtMs > Date.now() + reuseMarginMs
|
|
19882
20097
|
? cachedAuthority.runtimeApi
|
|
19883
|
-
: await
|
|
19884
|
-
withClient,
|
|
19885
|
-
first.runId,
|
|
19886
|
-
first.leaseOwner
|
|
19887
|
-
? { eventId: first.eventId, leaseOwner: first.leaseOwner }
|
|
19888
|
-
: null,
|
|
19889
|
-
tokenAuthority,
|
|
19890
|
-
options,
|
|
19891
|
-
input.runtimeApi?.signal,
|
|
19892
|
-
claimedLaunch,
|
|
19893
|
-
)
|
|
20098
|
+
: await resolveRunRuntimeApi()
|
|
19894
20099
|
: null;
|
|
19895
20100
|
if (
|
|
19896
20101
|
gatewayRuntimeApi &&
|
|
@@ -19902,66 +20107,53 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19902
20107
|
'Run Ledger dispatch origin changed within an immutable launch.',
|
|
19903
20108
|
);
|
|
19904
20109
|
}
|
|
19905
|
-
|
|
19906
|
-
|
|
19907
|
-
|
|
19908
|
-
|
|
19909
|
-
|
|
19910
|
-
|
|
19911
|
-
|
|
19912
|
-
|
|
19913
|
-
|
|
19914
|
-
|
|
19915
|
-
|
|
19916
|
-
|
|
19917
|
-
|
|
19918
|
-
|
|
19919
|
-
|
|
19920
|
-
|
|
19921
|
-
|
|
19922
|
-
|
|
19923
|
-
|
|
19924
|
-
|
|
19925
|
-
|
|
19926
|
-
|
|
19927
|
-
|
|
19928
|
-
|
|
19929
|
-
|
|
19930
|
-
|
|
19931
|
-
|
|
19932
|
-
|
|
19933
|
-
|
|
19934
|
-
|
|
19935
|
-
input.runtimeApi.runtimeTestFaultHeader,
|
|
19936
|
-
}
|
|
19937
|
-
: {}),
|
|
19938
|
-
...(input.runtimeApi.signal
|
|
19939
|
-
? { signal: input.runtimeApi.signal }
|
|
19940
|
-
: {}),
|
|
19941
|
-
}
|
|
19942
|
-
: {}),
|
|
19943
|
-
retryPolicy: 'none' as const,
|
|
19944
|
-
// The caller-owned admission deadline (see `signal` above) wins.
|
|
19945
|
-
...(input.signal ? { signal: input.signal } : {}),
|
|
19946
|
-
};
|
|
19947
|
-
const tokenExpiresAtMs = runtimeApiTokenExpiresAtMs(
|
|
19948
|
-
runtimeApi.executorToken,
|
|
19949
|
-
);
|
|
19950
|
-
if (tokenAuthority === 'gateway' && gatewayRuntimeApi) {
|
|
19951
|
-
authorityScope.authority = {
|
|
19952
|
-
runId: first.runId,
|
|
19953
|
-
leaseOwner,
|
|
19954
|
-
runtimeApi,
|
|
19955
|
-
expiresAtMs: tokenExpiresAtMs,
|
|
20110
|
+
if (needsRuntimeApi) {
|
|
20111
|
+
const resolvedRuntimeApi =
|
|
20112
|
+
gatewayRuntimeApi ?? input.runtimeApi ?? (await resolveRunRuntimeApi());
|
|
20113
|
+
runtimeApi = {
|
|
20114
|
+
...resolvedRuntimeApi,
|
|
20115
|
+
...(gatewayRuntimeApi && input.runtimeApi
|
|
20116
|
+
? {
|
|
20117
|
+
...(input.runtimeApi.fetch
|
|
20118
|
+
? { fetch: input.runtimeApi.fetch }
|
|
20119
|
+
: {}),
|
|
20120
|
+
...(input.runtimeApi.requestTimeoutMs !== undefined
|
|
20121
|
+
? { requestTimeoutMs: input.runtimeApi.requestTimeoutMs }
|
|
20122
|
+
: {}),
|
|
20123
|
+
...(input.runtimeApi.onResponseMeta
|
|
20124
|
+
? { onResponseMeta: input.runtimeApi.onResponseMeta }
|
|
20125
|
+
: {}),
|
|
20126
|
+
...(input.runtimeApi.runtimeTestFaultHeader !== undefined
|
|
20127
|
+
? {
|
|
20128
|
+
runtimeTestFaultHeader:
|
|
20129
|
+
input.runtimeApi.runtimeTestFaultHeader,
|
|
20130
|
+
}
|
|
20131
|
+
: {}),
|
|
20132
|
+
...(input.runtimeApi.signal
|
|
20133
|
+
? { signal: input.runtimeApi.signal }
|
|
20134
|
+
: {}),
|
|
20135
|
+
}
|
|
20136
|
+
: {}),
|
|
20137
|
+
retryPolicy: 'none' as const,
|
|
20138
|
+
// The caller-owned admission deadline (see `signal` above) wins.
|
|
20139
|
+
...(input.signal ? { signal: input.signal } : {}),
|
|
19956
20140
|
};
|
|
20141
|
+
if (tokenAuthority === 'gateway' && gatewayRuntimeApi) {
|
|
20142
|
+
authorityScope.authority = {
|
|
20143
|
+
runId: first.runId,
|
|
20144
|
+
leaseOwner,
|
|
20145
|
+
runtimeApi,
|
|
20146
|
+
expiresAtMs: runtimeApiTokenExpiresAtMs(runtimeApi.executorToken),
|
|
20147
|
+
};
|
|
20148
|
+
}
|
|
20149
|
+
authorityReadyAtMs = Date.now();
|
|
19957
20150
|
}
|
|
19958
|
-
authorityReadyAtMs = Date.now();
|
|
19959
20151
|
// The initial claim deliberately uses the old contract until its immutable
|
|
19960
20152
|
// launch receiver advertises support. Widen under the same run/owner fence
|
|
19961
20153
|
// before the first HTTP append, so an already-finished run need not pay one
|
|
19962
20154
|
// authority/append/ACK round trip for every ingress predecessor.
|
|
19963
20155
|
if (
|
|
19964
|
-
runtimeApi
|
|
20156
|
+
runtimeApi?.runLedgerIngressPrefixBatching === true &&
|
|
19965
20157
|
events.length === 1 &&
|
|
19966
20158
|
first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents
|
|
19967
20159
|
) {
|
|
@@ -19982,32 +20174,34 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
19982
20174
|
options,
|
|
19983
20175
|
),
|
|
19984
20176
|
);
|
|
19985
|
-
|
|
19986
|
-
|
|
19987
|
-
|
|
19988
|
-
|
|
19989
|
-
if (widened.length > 0) {
|
|
19990
|
-
if (widened[0]!.eventId !== first.eventId) {
|
|
19991
|
-
throw new RunLedgerProjectionLeaseLostError(
|
|
19992
|
-
'head_lease_owner_mismatch',
|
|
19993
|
-
);
|
|
19994
|
-
}
|
|
19995
|
-
events = widened;
|
|
20177
|
+
if (widened[0]?.eventId !== first.eventId) {
|
|
20178
|
+
throw new RunLedgerProjectionLeaseLostError(
|
|
20179
|
+
'head_lease_owner_mismatch',
|
|
20180
|
+
);
|
|
19996
20181
|
}
|
|
20182
|
+
events = widened;
|
|
19997
20183
|
}
|
|
19998
20184
|
// Widening can add deferred facts to the lease. Project the complete
|
|
19999
20185
|
// frozen prefix before any HTTP append, including its newly claimed rows.
|
|
20000
20186
|
const handedOff = await materializeDeferredObservability();
|
|
20001
20187
|
if (handedOff) return handedOff;
|
|
20188
|
+
if (ledgerEvents !== null) {
|
|
20189
|
+
ledgerEvents = events.flatMap(
|
|
20190
|
+
buildRunLedgerEventsFromPostgresSchedulerOutbox,
|
|
20191
|
+
);
|
|
20192
|
+
}
|
|
20002
20193
|
const renewDelivery = async () => {
|
|
20003
20194
|
const currentAuthority = authorityScope.authority;
|
|
20004
20195
|
if (
|
|
20196
|
+
runtimeApi &&
|
|
20005
20197
|
tokenAuthority === 'gateway' &&
|
|
20006
20198
|
currentAuthority?.runId === first.runId &&
|
|
20007
20199
|
currentAuthority.leaseOwner === leaseOwner &&
|
|
20008
20200
|
currentAuthority.expiresAtMs !== null &&
|
|
20009
20201
|
currentAuthority.expiresAtMs <= Date.now() + reuseMarginMs
|
|
20010
20202
|
) {
|
|
20203
|
+
// Initial delivery reuses the claim's routing fields. Renewal must
|
|
20204
|
+
// reread authority so an origin change cannot pass under that cache.
|
|
20011
20205
|
const refreshed = await loadPostgresSchedulerRuntimeApiForRun(
|
|
20012
20206
|
withClient,
|
|
20013
20207
|
first.runId,
|
|
@@ -20041,7 +20235,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20041
20235
|
leaseOwner,
|
|
20042
20236
|
leaseSeconds: input.leaseSeconds ?? 120,
|
|
20043
20237
|
freezeReceipt:
|
|
20044
|
-
runtimeApi
|
|
20238
|
+
runtimeApi?.runLedgerIngressPrefixBatching === true &&
|
|
20045
20239
|
events.some(
|
|
20046
20240
|
(event) =>
|
|
20047
20241
|
event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents,
|
|
@@ -20051,37 +20245,33 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20051
20245
|
),
|
|
20052
20246
|
);
|
|
20053
20247
|
};
|
|
20054
|
-
|
|
20055
|
-
|
|
20056
|
-
|
|
20057
|
-
|
|
20058
|
-
|
|
20059
|
-
|
|
20060
|
-
|
|
20061
|
-
|
|
20062
|
-
|
|
20063
|
-
|
|
20064
|
-
|
|
20065
|
-
|
|
20066
|
-
|
|
20067
|
-
|
|
20068
|
-
|
|
20069
|
-
|
|
20070
|
-
|
|
20071
|
-
|
|
20072
|
-
|
|
20073
|
-
|
|
20074
|
-
|
|
20075
|
-
|
|
20076
|
-
|
|
20077
|
-
|
|
20078
|
-
|
|
20079
|
-
|
|
20080
|
-
|
|
20081
|
-
const ledgerEvents = events.flatMap(
|
|
20082
|
-
buildRunLedgerEventsFromPostgresSchedulerOutbox,
|
|
20083
|
-
);
|
|
20084
|
-
if (ledgerEvents.length > 0) {
|
|
20248
|
+
if (needsRuntimeApi && runtimeApi) {
|
|
20249
|
+
const append = async (runtimeApi: WorkerRuntimeApiContext) => {
|
|
20250
|
+
// A pending start is always claimed alone because later events require
|
|
20251
|
+
// its snapshot. Retain that mutation as its own exact delivery.
|
|
20252
|
+
if (first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runCreated) {
|
|
20253
|
+
await renewDelivery();
|
|
20254
|
+
await projectPostgresSchedulerCreatedEvent(runtimeApi, first);
|
|
20255
|
+
} else if (
|
|
20256
|
+
first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runStarted
|
|
20257
|
+
) {
|
|
20258
|
+
await renewDelivery();
|
|
20259
|
+
const startLeaseOwner = first.leaseOwner;
|
|
20260
|
+
const launch = startLeaseOwner
|
|
20261
|
+
? await withClient((client) =>
|
|
20262
|
+
readRunLedgerDispatchLaunchForLease(
|
|
20263
|
+
client,
|
|
20264
|
+
{
|
|
20265
|
+
runId: first.runId,
|
|
20266
|
+
eventId: first.eventId,
|
|
20267
|
+
leaseOwner: startLeaseOwner,
|
|
20268
|
+
},
|
|
20269
|
+
options,
|
|
20270
|
+
),
|
|
20271
|
+
)
|
|
20272
|
+
: null;
|
|
20273
|
+
await projectPostgresSchedulerStartEvent(runtimeApi, first, launch);
|
|
20274
|
+
} else if (ledgerEvents && ledgerEvents.length > 0) {
|
|
20085
20275
|
const terminal = [...events]
|
|
20086
20276
|
.reverse()
|
|
20087
20277
|
.find((event) =>
|
|
@@ -20092,6 +20282,8 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20092
20282
|
const batchDeliveryKey = createHash('sha256')
|
|
20093
20283
|
.update(events.map((event) => event.deliveryKey).join('\n'))
|
|
20094
20284
|
.digest('hex');
|
|
20285
|
+
// The transport renews before every HTTP chunk; another renewal here
|
|
20286
|
+
// would add a checkout without protecting an intervening side effect.
|
|
20095
20287
|
await projectRunEventsViaAppRuntime(runtimeApi, {
|
|
20096
20288
|
playId: first.runId,
|
|
20097
20289
|
events: ledgerEvents,
|
|
@@ -20107,25 +20299,26 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20107
20299
|
deliveryDigest: batchDeliveryKey,
|
|
20108
20300
|
}),
|
|
20109
20301
|
});
|
|
20110
|
-
} else {
|
|
20111
|
-
await renewDelivery();
|
|
20112
20302
|
}
|
|
20303
|
+
};
|
|
20304
|
+
try {
|
|
20305
|
+
await append(runtimeApi);
|
|
20306
|
+
} catch (error) {
|
|
20307
|
+
// Refresh a reused authority once through this delivery's active lease.
|
|
20308
|
+
// Both attempts retain the same fenced append and idempotency key.
|
|
20309
|
+
if (
|
|
20310
|
+
!gatewayRuntimeApi?.refreshExecutorToken ||
|
|
20311
|
+
!(error instanceof AppRuntimeApiResponseError) ||
|
|
20312
|
+
error.status !== 401
|
|
20313
|
+
)
|
|
20314
|
+
throw error;
|
|
20315
|
+
runtimeApi.executorToken = await gatewayRuntimeApi.refreshExecutorToken();
|
|
20316
|
+
await append(runtimeApi);
|
|
20113
20317
|
}
|
|
20114
|
-
}
|
|
20115
|
-
|
|
20116
|
-
|
|
20117
|
-
|
|
20118
|
-
// A cached authority can be invalidated by signer rotation. Refresh it
|
|
20119
|
-
// once through this delivery's active lease; the append remains fenced
|
|
20120
|
-
// and carries the same idempotency key on both attempts.
|
|
20121
|
-
if (
|
|
20122
|
-
!gatewayRuntimeApi?.refreshExecutorToken ||
|
|
20123
|
-
!(error instanceof AppRuntimeApiResponseError) ||
|
|
20124
|
-
error.status !== 401
|
|
20125
|
-
)
|
|
20126
|
-
throw error;
|
|
20127
|
-
runtimeApi.executorToken = await gatewayRuntimeApi.refreshExecutorToken();
|
|
20128
|
-
await append();
|
|
20318
|
+
} else if (tokenAuthority !== 'gateway') {
|
|
20319
|
+
// App-side acknowledgement retains lease validation. The gateway can
|
|
20320
|
+
// acknowledge a durable search-only fact without a network lease hop.
|
|
20321
|
+
await renewDelivery();
|
|
20129
20322
|
}
|
|
20130
20323
|
appendFinishedAtMs = Date.now();
|
|
20131
20324
|
const acknowledgement = await acknowledgeRunLedgerPrefixWithContinuation(
|
|
@@ -20293,10 +20486,16 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20293
20486
|
throw settlementError;
|
|
20294
20487
|
}
|
|
20295
20488
|
});
|
|
20489
|
+
const failure = runLedgerDeliveryFailure({ error });
|
|
20296
20490
|
console.warn('[run-ledger-outbox.batch_dispatch_failed]', {
|
|
20297
20491
|
runId: input.runId ?? null,
|
|
20298
20492
|
eventIds: events.map((event) => event.eventId),
|
|
20299
20493
|
error: error instanceof Error ? error.message : String(error),
|
|
20494
|
+
code: failure.code,
|
|
20495
|
+
failureReason: failure.failureReason,
|
|
20496
|
+
...(failure.projectionMismatch
|
|
20497
|
+
? { projectionMismatch: failure.projectionMismatch }
|
|
20498
|
+
: {}),
|
|
20300
20499
|
});
|
|
20301
20500
|
return {
|
|
20302
20501
|
attempted: events.length,
|
|
@@ -20405,7 +20604,7 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
|
|
|
20405
20604
|
input: RunLedgerOutboxDispatchInput;
|
|
20406
20605
|
events: PostgresSchedulerOutboxEvent[];
|
|
20407
20606
|
leaseOwner: string;
|
|
20408
|
-
runtimeApi: WorkerRuntimeApiContext;
|
|
20607
|
+
runtimeApi: WorkerRuntimeApiContext | null;
|
|
20409
20608
|
scope?: RunLedgerOutboxDrainScope;
|
|
20410
20609
|
continuation: { current: RunLedgerOutboxDrainScope | null };
|
|
20411
20610
|
continuationAllowance: number | null;
|
|
@@ -20421,81 +20620,86 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
|
|
|
20421
20620
|
!!input.scope ||
|
|
20422
20621
|
(input.input.continueThroughTerminal === true && !input.input.headEventId);
|
|
20423
20622
|
return withClient(async (client) => {
|
|
20424
|
-
|
|
20425
|
-
await client.query('BEGIN');
|
|
20426
|
-
}
|
|
20427
|
-
try {
|
|
20623
|
+
const acknowledge = async () => {
|
|
20428
20624
|
if (retainContinuation) {
|
|
20429
|
-
await client.query(
|
|
20430
|
-
`SELECT set_config('lock_timeout', $1, true), set_config('statement_timeout', $2, true)`,
|
|
20431
|
-
[
|
|
20432
|
-
`${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
|
|
20433
|
-
`${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
|
|
20434
|
-
],
|
|
20435
|
-
);
|
|
20436
|
-
// Keep the acknowledged prefix and its successor under one run lock.
|
|
20437
|
-
// No competing projector can take the next head while this ACK's
|
|
20438
|
-
// response is travelling back to the lease holder.
|
|
20439
|
-
await client.query(
|
|
20440
|
-
`SELECT run_id FROM ${tableName('runs', options)} WHERE run_id = $1 FOR UPDATE`,
|
|
20441
|
-
[runId],
|
|
20442
|
-
);
|
|
20625
|
+
await client.query('BEGIN');
|
|
20443
20626
|
}
|
|
20444
|
-
|
|
20445
|
-
|
|
20446
|
-
|
|
20447
|
-
|
|
20448
|
-
|
|
20449
|
-
|
|
20450
|
-
|
|
20451
|
-
|
|
20452
|
-
|
|
20453
|
-
|
|
20454
|
-
|
|
20455
|
-
|
|
20456
|
-
|
|
20457
|
-
|
|
20458
|
-
|
|
20459
|
-
options,
|
|
20460
|
-
);
|
|
20461
|
-
if (target.exists && !target.delivered && !target.blocked) {
|
|
20462
|
-
const nextClaimed = await readPendingPostgresSchedulerOutboxEvents(
|
|
20463
|
-
client,
|
|
20464
|
-
{
|
|
20465
|
-
runId,
|
|
20466
|
-
leaseOwner: input.leaseOwner,
|
|
20467
|
-
leaseSeconds: input.input.leaseSeconds ?? 120,
|
|
20468
|
-
limit: Math.min(
|
|
20469
|
-
RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
|
|
20470
|
-
input.runtimeApi.runLedgerIngressPrefixBatching === true
|
|
20471
|
-
? RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS
|
|
20472
|
-
: (input.input.limit ?? 1),
|
|
20473
|
-
input.continuationAllowance ??
|
|
20474
|
-
RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
|
|
20475
|
-
),
|
|
20476
|
-
allowIngressPrefix:
|
|
20477
|
-
input.runtimeApi.runLedgerIngressPrefixBatching === true,
|
|
20478
|
-
inTransaction: true,
|
|
20479
|
-
},
|
|
20480
|
-
options,
|
|
20627
|
+
try {
|
|
20628
|
+
if (retainContinuation) {
|
|
20629
|
+
await client.query(
|
|
20630
|
+
`SELECT set_config('lock_timeout', $1, true), set_config('statement_timeout', $2, true)`,
|
|
20631
|
+
[
|
|
20632
|
+
`${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
|
|
20633
|
+
`${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
|
|
20634
|
+
],
|
|
20635
|
+
);
|
|
20636
|
+
// Keep the acknowledged prefix and its successor under one run lock.
|
|
20637
|
+
// No competing projector can take the next head while this ACK's
|
|
20638
|
+
// response is travelling back to the lease holder.
|
|
20639
|
+
await client.query(
|
|
20640
|
+
`SELECT run_id FROM ${tableName('runs', options)} WHERE run_id = $1 FOR UPDATE`,
|
|
20641
|
+
[runId],
|
|
20481
20642
|
);
|
|
20482
|
-
if (nextClaimed.length > 0) {
|
|
20483
|
-
continuation = input.scope ?? {
|
|
20484
|
-
runId,
|
|
20485
|
-
leaseOwner: input.leaseOwner,
|
|
20486
|
-
targetEventId,
|
|
20487
|
-
nextClaimed: [],
|
|
20488
|
-
};
|
|
20489
|
-
continuation.nextClaimed = nextClaimed;
|
|
20490
|
-
input.continuation.current = continuation;
|
|
20491
|
-
}
|
|
20492
20643
|
}
|
|
20644
|
+
const eventIds = input.events.map((event) => event.eventId);
|
|
20645
|
+
const { deliveredRows, events: nextClaimed } =
|
|
20646
|
+
retainContinuation &&
|
|
20647
|
+
input.continuationAllowance !== 0 &&
|
|
20648
|
+
!eventIds.includes(targetEventId)
|
|
20649
|
+
? await claimPostgresSchedulerOutboxPrefix(
|
|
20650
|
+
client,
|
|
20651
|
+
{
|
|
20652
|
+
runId,
|
|
20653
|
+
leaseOwner: input.leaseOwner,
|
|
20654
|
+
leaseSeconds: input.input.leaseSeconds ?? 120,
|
|
20655
|
+
limit: Math.min(
|
|
20656
|
+
RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
|
|
20657
|
+
input.runtimeApi?.runLedgerIngressPrefixBatching === true
|
|
20658
|
+
? RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS
|
|
20659
|
+
: (input.input.limit ?? 1),
|
|
20660
|
+
input.continuationAllowance ??
|
|
20661
|
+
RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
|
|
20662
|
+
),
|
|
20663
|
+
allowIngressPrefix:
|
|
20664
|
+
input.runtimeApi?.runLedgerIngressPrefixBatching === true,
|
|
20665
|
+
},
|
|
20666
|
+
options,
|
|
20667
|
+
{ eventIds, targetEventId },
|
|
20668
|
+
)
|
|
20669
|
+
: {
|
|
20670
|
+
deliveredRows: await markPostgresSchedulerOutboxDeliveredRows(
|
|
20671
|
+
client,
|
|
20672
|
+
eventIds,
|
|
20673
|
+
{ ...options, leaseOwner: input.leaseOwner },
|
|
20674
|
+
),
|
|
20675
|
+
events: [],
|
|
20676
|
+
};
|
|
20677
|
+
let continuation: RunLedgerOutboxDrainScope | null = null;
|
|
20678
|
+
if (nextClaimed.length > 0) {
|
|
20679
|
+
continuation = input.scope ?? {
|
|
20680
|
+
runId,
|
|
20681
|
+
leaseOwner: input.leaseOwner,
|
|
20682
|
+
targetEventId,
|
|
20683
|
+
nextClaimed: [],
|
|
20684
|
+
};
|
|
20685
|
+
continuation.nextClaimed = nextClaimed;
|
|
20686
|
+
input.continuation.current = continuation;
|
|
20687
|
+
}
|
|
20688
|
+
if (retainContinuation) await client.query('COMMIT');
|
|
20689
|
+
return { deliveredRows, continuation };
|
|
20690
|
+
} catch (error) {
|
|
20691
|
+
if (retainContinuation) await client.query('ROLLBACK');
|
|
20692
|
+
throw error;
|
|
20493
20693
|
}
|
|
20494
|
-
|
|
20495
|
-
|
|
20694
|
+
};
|
|
20695
|
+
try {
|
|
20696
|
+
return await acknowledge();
|
|
20496
20697
|
} catch (error) {
|
|
20497
|
-
if (
|
|
20498
|
-
|
|
20698
|
+
if (!isMissingOutboxDeliveryBatchMarker(error)) throw error;
|
|
20699
|
+
// The first statement recorded this operation-local missing column and
|
|
20700
|
+
// rolled back its ACK. Reacquire the run lock and take a fresh snapshot
|
|
20701
|
+
// for the existing compatibility claim, without another ledger append.
|
|
20702
|
+
return await acknowledge();
|
|
20499
20703
|
}
|
|
20500
20704
|
});
|
|
20501
20705
|
}
|
|
@@ -20653,37 +20857,34 @@ async function projectPostgresSchedulerStartEvent(
|
|
|
20653
20857
|
|
|
20654
20858
|
export type { RunLedgerDispatchLaunch } from './run-ledger-dispatch-authority';
|
|
20655
20859
|
|
|
20656
|
-
|
|
20657
|
-
|
|
20658
|
-
|
|
20659
|
-
|
|
20860
|
+
/** Search-only `run.events` facts settle without consulting launch credentials. */
|
|
20861
|
+
function runLedgerSearchOnlyOutboxFactSql(alias: string): string {
|
|
20862
|
+
const events = `CASE
|
|
20863
|
+
WHEN jsonb_typeof(${alias}.payload_json->'events') = 'array'
|
|
20864
|
+
THEN ${alias}.payload_json->'events'
|
|
20865
|
+
ELSE '[]'::jsonb
|
|
20866
|
+
END`;
|
|
20867
|
+
return `(
|
|
20868
|
+
${alias}.event_type = '${RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents}'
|
|
20869
|
+
AND ${alias}.payload_json->>'logBodyStore' = 'planet_scale'
|
|
20870
|
+
AND jsonb_array_length(${events}) = 0
|
|
20871
|
+
)`;
|
|
20872
|
+
}
|
|
20873
|
+
|
|
20874
|
+
async function resolvePostgresSchedulerRuntimeApiForRun(
|
|
20875
|
+
launchJson: unknown,
|
|
20660
20876
|
runId: string,
|
|
20661
20877
|
dispatchLease: { eventId: string; leaseOwner: string } | null,
|
|
20662
20878
|
tokenAuthority: RunLedgerOutboxTokenAuthority,
|
|
20663
20879
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
20664
20880
|
signal?: AbortSignal,
|
|
20665
|
-
claimedLaunch?: RunLedgerDispatchLaunch,
|
|
20666
20881
|
): Promise<
|
|
20667
20882
|
WorkerRuntimeApiContext & {
|
|
20668
20883
|
refreshExecutorToken?: () => Promise<string>;
|
|
20669
20884
|
invalidateExecutorToken?: () => void;
|
|
20670
20885
|
}
|
|
20671
20886
|
> {
|
|
20672
|
-
|
|
20673
|
-
// A slow gateway must not retain either projector pool client, and its
|
|
20674
|
-
// launch context must not hydrate customer execution graphs.
|
|
20675
|
-
const launch: RunLedgerDispatchLaunch =
|
|
20676
|
-
claimedLaunch ??
|
|
20677
|
-
(await withClient(async (client) => {
|
|
20678
|
-
const result = await client.query<{ launch_json: unknown }>(
|
|
20679
|
-
`SELECT ${runLedgerDispatchLaunchSql('runs')} AS launch_json
|
|
20680
|
-
FROM ${tableName('runs', options)} AS runs
|
|
20681
|
-
WHERE run_id = $1
|
|
20682
|
-
LIMIT 1`,
|
|
20683
|
-
[runId],
|
|
20684
|
-
);
|
|
20685
|
-
return parseLaunchJson(result.rows[0]?.launch_json);
|
|
20686
|
-
}));
|
|
20887
|
+
const launch = parseRunLedgerDispatchLaunch(launchJson);
|
|
20687
20888
|
if (!launch?.baseUrl) {
|
|
20688
20889
|
throw new Error(
|
|
20689
20890
|
`Run-ledger outbox dispatcher cannot resolve runtime API context for ${runId}.`,
|
|
@@ -20743,6 +20944,37 @@ async function loadPostgresSchedulerRuntimeApiForRun(
|
|
|
20743
20944
|
};
|
|
20744
20945
|
}
|
|
20745
20946
|
|
|
20947
|
+
async function loadPostgresSchedulerRuntimeApiForRun(
|
|
20948
|
+
withClient: <T>(
|
|
20949
|
+
operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
|
|
20950
|
+
) => Promise<T>,
|
|
20951
|
+
runId: string,
|
|
20952
|
+
dispatchLease: { eventId: string; leaseOwner: string } | null,
|
|
20953
|
+
tokenAuthority: RunLedgerOutboxTokenAuthority,
|
|
20954
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
20955
|
+
signal?: AbortSignal,
|
|
20956
|
+
): ReturnType<typeof resolvePostgresSchedulerRuntimeApiForRun> {
|
|
20957
|
+
// A continuation already owns its next event lease, so its launch context
|
|
20958
|
+
// is not part of the preclaim result. Load only the authority fields here.
|
|
20959
|
+
const result = await withClient((client) =>
|
|
20960
|
+
client.query<{ launch_json: unknown }>(
|
|
20961
|
+
`SELECT ${runLedgerDispatchLaunchSql('runs')} AS launch_json
|
|
20962
|
+
FROM ${tableName('runs', options)} AS runs
|
|
20963
|
+
WHERE run_id = $1
|
|
20964
|
+
LIMIT 1`,
|
|
20965
|
+
[runId],
|
|
20966
|
+
),
|
|
20967
|
+
);
|
|
20968
|
+
return resolvePostgresSchedulerRuntimeApiForRun(
|
|
20969
|
+
result.rows[0]?.launch_json,
|
|
20970
|
+
runId,
|
|
20971
|
+
dispatchLease,
|
|
20972
|
+
tokenAuthority,
|
|
20973
|
+
options,
|
|
20974
|
+
signal,
|
|
20975
|
+
);
|
|
20976
|
+
}
|
|
20977
|
+
|
|
20746
20978
|
// Only freshly gateway-issued legacy run.progress authority is reused.
|
|
20747
20979
|
// Prefix advertisements are reacquired for each drain because they select its
|
|
20748
20980
|
// receipt shape, including recovery across receiver capability changes. This is not
|
|
@@ -21071,6 +21303,20 @@ function terminalLogContextsPayloadField(
|
|
|
21071
21303
|
return value.map(normalizePlayLogContext);
|
|
21072
21304
|
}
|
|
21073
21305
|
|
|
21306
|
+
function terminalLogMetadataPayloadField(
|
|
21307
|
+
payload: unknown,
|
|
21308
|
+
lines: readonly string[],
|
|
21309
|
+
): Array<PlayLogMetadata | null> | null {
|
|
21310
|
+
const value = payloadField(payload, 'terminalLogMetadata');
|
|
21311
|
+
if (value === undefined) return null;
|
|
21312
|
+
if (!Array.isArray(value) || value.length !== lines.length) {
|
|
21313
|
+
throw new TypeError(
|
|
21314
|
+
'Terminal log metadata must align with terminal log lines.',
|
|
21315
|
+
);
|
|
21316
|
+
}
|
|
21317
|
+
return value.map(normalizePlayLogMetadata);
|
|
21318
|
+
}
|
|
21319
|
+
|
|
21074
21320
|
function ledgerEventsPayloadField(
|
|
21075
21321
|
payload: unknown,
|
|
21076
21322
|
key: string,
|