deepline 0.3.392 → 0.3.394
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +156 -24
- package/dist/bundling-sources/sdk/src/compat.ts +9 -0
- package/dist/bundling-sources/sdk/src/http.ts +5 -1
- package/dist/bundling-sources/sdk/src/index.ts +5 -0
- package/dist/bundling-sources/sdk/src/plays/run-request.ts +182 -6
- package/dist/bundling-sources/sdk/src/release.ts +4 -1
- package/dist/bundling-sources/sdk/src/types.ts +13 -0
- package/dist/bundling-sources/shared_libs/integration-contracts/request-warnings.ts +64 -0
- package/dist/bundling-sources/shared_libs/play-data-plane/runtime-sheet-storage-contract.ts +315 -0
- package/dist/bundling-sources/shared_libs/play-runtime/async-operation.ts +41 -8
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +895 -54
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +41 -4
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-policy.ts +28 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-invocation-fence.ts +36 -0
- package/dist/bundling-sources/shared_libs/play-runtime/freshness-receipt-execution.ts +251 -0
- package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +11 -3
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +30 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-identity.ts +26 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +168 -10
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +252 -10
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +92 -58
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +7 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +30 -7
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +1161 -316
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +28 -15
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +24 -3
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +32 -0
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +103 -23
- package/dist/bundling-sources/shared_libs/plays/core.ts +1 -1
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +172 -79
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +173 -1
- package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +18 -0
- package/dist/cli/auth-IHKISLJQ.js +39 -0
- package/dist/cli/{auth-6JSA2ZCW.mjs → auth-OSXDAVCF.mjs} +4 -4
- package/dist/cli/{billing-Q5BLQHIU.js → billing-EKLYOXUY.js} +101 -74
- package/dist/cli/{billing-MRRO643H.mjs → billing-ZNNKX3UR.mjs} +32 -5
- package/dist/cli/{chunk-4SFPVW7A.js → chunk-2MDDA7A4.js} +26 -26
- package/dist/cli/{chunk-JOFPQGXW.js → chunk-354JCXA2.js} +50 -50
- package/dist/cli/{chunk-7E66A24V.mjs → chunk-3HU47EGZ.mjs} +71 -9
- package/dist/cli/{chunk-DSWSEHRX.mjs → chunk-3HWALOL2.mjs} +1 -1
- package/dist/cli/{chunk-T7OKJ2T6.js → chunk-4P27AXEA.js} +73 -63
- package/dist/cli/{chunk-24VEN5PQ.mjs → chunk-5BB2RGAX.mjs} +190 -18
- package/dist/cli/{chunk-JSLPHFP7.js → chunk-6YAUMG27.js} +3 -2
- package/dist/cli/{chunk-MU6M3URX.mjs → chunk-BP33SIZF.mjs} +1 -1
- package/dist/cli/{chunk-WMS5U6YB.js → chunk-CS6RAIVY.js} +20 -19
- package/dist/cli/{chunk-CVLM3WMJ.js → chunk-E23BQ5SC.js} +242 -183
- package/dist/cli/{chunk-2GAHHSLB.mjs → chunk-FJGW5GXU.mjs} +3 -2
- package/dist/cli/{chunk-YN6YA7ZZ.mjs → chunk-JHIJIHFB.mjs} +79 -20
- package/dist/cli/{chunk-Q6EOHNRH.mjs → chunk-JI2B4RZP.mjs} +119 -6
- package/dist/cli/{chunk-5O4B6AUN.js → chunk-KB4SDX5S.js} +71 -9
- package/dist/cli/{chunk-RV6ALZIX.js → chunk-KGNTKMRG.js} +32 -7
- package/dist/cli/{chunk-6DBOU7UH.js → chunk-KI635JN4.js} +176 -87
- package/dist/cli/{chunk-2FQG2HU4.js → chunk-KIUMEOM5.js} +8 -8
- package/dist/cli/{chunk-Y6VOGHF2.mjs → chunk-KZZC33PY.mjs} +0 -181
- package/dist/cli/{chunk-2Q3GEFL4.mjs → chunk-M57J2SK4.mjs} +2 -2
- package/dist/cli/{chunk-R3O2CZ4H.mjs → chunk-MFB2VKBO.mjs} +2 -2
- package/dist/cli/{chunk-QGKPTBRR.js → chunk-MH73HMPV.js} +3 -3
- package/dist/cli/{chunk-BB3FNYFY.js → chunk-MLPU5SMZ.js} +2 -2
- package/dist/cli/{chunk-2G7DXFHI.mjs → chunk-NR3DFKNF.mjs} +27 -2
- package/dist/cli/{chunk-EY7RFC6A.js → chunk-QIQBYKLB.js} +15 -15
- package/dist/cli/{chunk-RIV2GS5E.js → chunk-RBJYWRCF.js} +638 -466
- package/dist/cli/{chunk-3VN4FTE3.js → chunk-RDXFGPJU.js} +1 -182
- package/dist/cli/{chunk-YAZCDBXX.mjs → chunk-TJI2EN6K.mjs} +3 -2
- package/dist/cli/chunk-UBKA33KY.mjs +310 -0
- package/dist/cli/{chunk-PAIMOGCM.js → chunk-UHNDVD3P.js} +125 -12
- package/dist/cli/{chunk-4TCT77UF.js → chunk-URWF7NGQ.js} +6 -1
- package/dist/cli/{chunk-2QLML6G4.mjs → chunk-UXFSWG3Q.mjs} +4 -4
- package/dist/cli/{chunk-UCKJQG25.js → chunk-VK6FYZLP.js} +8 -8
- package/dist/cli/{chunk-ZTACINO5.mjs → chunk-X63FDZJZ.mjs} +13 -3
- package/dist/cli/chunk-XDVT3GOM.js +310 -0
- package/dist/cli/{chunk-A3APGZFP.mjs → chunk-XRQ2AQL7.mjs} +2 -2
- package/dist/cli/{chunk-3FV4GBRK.mjs → chunk-XXUDVEXK.mjs} +97 -8
- package/dist/cli/{chunk-PQSA443V.mjs → chunk-ZI2ZNDJK.mjs} +6 -1
- package/dist/cli/{chunk-HGYBEAQP.mjs → chunk-ZYBGA4AR.mjs} +3 -3
- package/dist/cli/command-docs.js +2 -2
- package/dist/cli/command-docs.mjs +1 -1
- package/dist/cli/{csv-E4QGUIGI.mjs → csv-5AXRB35H.mjs} +3 -3
- package/dist/cli/{csv-26U5T24Z.js → csv-LU7O75WE.js} +12 -12
- package/dist/cli/{db-JTRJIAQW.mjs → db-7XP5UWEW.mjs} +6 -5
- package/dist/cli/{db-QHBFLMVL.js → db-F7YTFKVM.js} +31 -30
- package/dist/cli/{describe-FDV2V74F.js → describe-FRH32RDV.js} +24 -23
- package/dist/cli/{describe-E3DE3ZK3.mjs → describe-K4ES5LIL.mjs} +17 -16
- package/dist/cli/{executions-B3GTNPRE.mjs → executions-IEHEHJUO.mjs} +6 -5
- package/dist/cli/{executions-VZYOGVQY.js → executions-RZV5XPT2.js} +8 -7
- package/dist/cli/{feedback-CR55FJA2.js → feedback-3P65QTX3.js} +7 -7
- package/dist/cli/{feedback-4CEBR3OV.mjs → feedback-FPUVZBAN.mjs} +3 -3
- package/dist/cli/{git-NM2PHHZ4.mjs → git-GLVVP7TY.mjs} +4 -4
- package/dist/cli/git-MFFAWVVX.js +23 -0
- package/dist/cli/index.js +100 -96
- package/dist/cli/index.mjs +47 -43
- package/dist/cli/{legacy-noops-A7MSW5ND.mjs → legacy-noops-3MOI6NBM.mjs} +4 -4
- package/dist/cli/{legacy-noops-ZUA4YFIH.js → legacy-noops-XKSGZR7Q.js} +7 -7
- package/dist/cli/{monitors-NVPRQYP3.mjs → monitors-FTBGNEFG.mjs} +6 -5
- package/dist/cli/{monitors-ZE3TLZ4H.js → monitors-NZ2DYNST.js} +87 -86
- package/dist/cli/{org-U7A6DV5D.js → org-MCKDQXIV.js} +66 -65
- package/dist/cli/{org-KUBILNQV.mjs → org-V2J5RBCN.mjs} +6 -5
- package/dist/cli/{play-HRDXQ4QE.mjs → play-AJFGME3B.mjs} +11 -10
- package/dist/cli/play-WLSN2EO2.js +158 -0
- package/dist/cli/{providers-VD6YTVQZ.js → providers-2JEZ2264.js} +9 -8
- package/dist/cli/{providers-OHIWI453.mjs → providers-I6RQJZFZ.mjs} +6 -5
- package/dist/cli/{quickstart-UBV2FONF.mjs → quickstart-BIDQDMO4.mjs} +3 -3
- package/dist/cli/{quickstart-ETTCWPA4.js → quickstart-GJ2RPDU3.js} +5 -5
- package/dist/cli/run-identity-G257GQED.js +25 -0
- package/dist/cli/run-identity-MRNCEGV2.mjs +25 -0
- package/dist/cli/search-5VN4YD4I.js +27 -0
- package/dist/cli/{search-F44IZRT7.mjs → search-ELRXFWG5.mjs} +7 -6
- package/dist/cli/{secrets-CAEMRORF.js → secrets-53IT6VXD.js} +15 -14
- package/dist/cli/{secrets-R6XOJZKU.mjs → secrets-SQXGZXIC.mjs} +6 -5
- package/dist/cli/{sessions-RLN6UQS3.mjs → sessions-2X5QF7SL.mjs} +4 -4
- package/dist/cli/{sessions-6GC7OFTT.js → sessions-CAV5M77Y.js} +5 -5
- package/dist/cli/{settings-DPNVUAV2.js → settings-532WNAZO.js} +76 -75
- package/dist/cli/{settings-KIPAMRPP.mjs → settings-7P67VGCJ.mjs} +9 -8
- package/dist/cli/{setup-DTL3L5SG.js → setup-6L2VSUWJ.js} +41 -41
- package/dist/cli/{setup-TU2F7PU5.mjs → setup-QGGBV2LE.mjs} +6 -6
- package/dist/cli/{skills-UV5MYJ46.mjs → skills-3FE62WBG.mjs} +4 -4
- package/dist/cli/skills-S6NPZAN7.js +28 -0
- package/dist/cli/tools-647RLXQS.js +41 -0
- package/dist/cli/{tools-QRRNDCQ7.mjs → tools-DMR25DEF.mjs} +10 -9
- package/dist/cli/{update-4426TF35.mjs → update-6RZMHJZP.mjs} +6 -6
- package/dist/cli/update-WVTSC3EM.js +28 -0
- package/dist/cli/{workflow-N4U5VLZ3.mjs → workflow-L2DQRE6R.mjs} +7 -6
- package/dist/cli/{workflow-HNLOVISB.js → workflow-TX4F2PIX.js} +17 -16
- package/dist/{client-e6Qh5ZIg.d.mts → client-BktIPdIj.d.mts} +153 -4
- package/dist/{client-Uxuy-xQ2.d.ts → client-C3O1Ipf3.d.ts} +153 -4
- package/dist/{compiler-manifest-CY2v8ab0.d.ts → compiler-manifest-DLxUY5kI.d.mts} +124 -10
- package/dist/{compiler-manifest-CY2v8ab0.d.mts → compiler-manifest-DLxUY5kI.d.ts} +124 -10
- package/dist/{index-BZ6uzZJl.d.ts → index-B_GM9E3q.d.ts} +1 -1
- package/dist/{index-B9fBZH1g.d.mts → index-SCb0NvDb.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +120 -29
- package/dist/index.mjs +111 -20
- package/dist/install-integrity.json +94 -88
- package/dist/plays/bundle-play-file.d.mts +3 -3
- package/dist/plays/bundle-play-file.d.ts +3 -3
- package/dist/plays/bundle-play-file.mjs +2 -2
- package/dist/plays/run-request.d.mts +2 -2
- package/dist/plays/run-request.d.ts +2 -2
- package/dist/plays/run-request.js +5 -2
- package/dist/plays/run-request.mjs +4 -1
- package/dist/release.d.mts +4 -1
- package/dist/release.d.ts +4 -1
- package/dist/release.js +2 -2
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
- package/dist/cli/auth-67TAELQL.js +0 -39
- package/dist/cli/git-I5JTK6YU.js +0 -23
- package/dist/cli/play-A2V7YDU7.js +0 -157
- package/dist/cli/search-WWXMBVPH.js +0 -26
- package/dist/cli/skills-SWOSEZLB.js +0 -28
- package/dist/cli/tools-IL2YCDQ6.js +0 -40
- package/dist/cli/update-BERDPEYR.js +0 -28
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { escapeLiteral } from 'pg';
|
|
1
2
|
import {
|
|
2
3
|
readRowFailureCounts,
|
|
3
4
|
type RowFailureCounts,
|
|
@@ -514,6 +515,8 @@ type PostgresSchedulerClaimExecutionOptions = Pick<
|
|
|
514
515
|
* for a detached-runner attempt must not overwrite an authoritative pushed
|
|
515
516
|
* runner terminal that won after the attempt was claimed. */
|
|
516
517
|
preserveRunnerTerminal?: boolean;
|
|
518
|
+
/** Recheck typed-wait custody and cancellation at the actual failure write. */
|
|
519
|
+
protectWaitingSuspensionFailure?: boolean;
|
|
517
520
|
};
|
|
518
521
|
|
|
519
522
|
export type PostgresSchedulerWorkerOnceResult =
|
|
@@ -8575,6 +8578,72 @@ export async function claimPostgresSchedulerWaitingDetachedAttemptForTerminal(
|
|
|
8575
8578
|
return claimed.rows.length > 0;
|
|
8576
8579
|
}
|
|
8577
8580
|
|
|
8581
|
+
/**
|
|
8582
|
+
* Prepare only an owned, retained suspension for a failure terminal. Unlike an
|
|
8583
|
+
* engine wake, this never clears the user's wait or emits `run.resumed`.
|
|
8584
|
+
*/
|
|
8585
|
+
export async function claimPostgresSchedulerWaitingSuspensionAttemptForFailure(
|
|
8586
|
+
client: PostgresSchedulerQueryClient,
|
|
8587
|
+
input: { runId: string; attempt: number; leaseToken: string },
|
|
8588
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
8589
|
+
): Promise<'prepared' | 'fence_lost' | 'not_applicable'> {
|
|
8590
|
+
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
8591
|
+
try {
|
|
8592
|
+
// A cancellation marker can commit without the custody lock. Acquire the
|
|
8593
|
+
// locks first, then evaluate it in a fresh statement snapshot; a guard in
|
|
8594
|
+
// the lock-taking statement could predate that cancellation.
|
|
8595
|
+
await client.query(
|
|
8596
|
+
`WITH ${runtimeSandboxCustodyLocksSql('run.run_id = $1', options)}
|
|
8597
|
+
SELECT count(*) FROM custody_attempts`,
|
|
8598
|
+
[input.runId],
|
|
8599
|
+
);
|
|
8600
|
+
const claimed = await client.query<{
|
|
8601
|
+
applicable: boolean;
|
|
8602
|
+
prepared: boolean;
|
|
8603
|
+
}>(
|
|
8604
|
+
`
|
|
8605
|
+
WITH waiting_suspension AS MATERIALIZED (
|
|
8606
|
+
SELECT run_id FROM ${tableName('runs', options)}
|
|
8607
|
+
WHERE run_id = $1 AND status = 'waiting'
|
|
8608
|
+
AND suspension_json ->> 'kind' <> 'detached_runner'
|
|
8609
|
+
), prepared AS (
|
|
8610
|
+
UPDATE ${tableName('attempts', options)} AS target
|
|
8611
|
+
SET status = 'running', heartbeat_at = now(), updated_at = now()
|
|
8612
|
+
FROM ${tableName('runs', options)} AS run
|
|
8613
|
+
WHERE target.run_id = $1 AND target.attempt = $2
|
|
8614
|
+
AND target.lease_token = $3 AND target.status IN ('waiting', 'running')
|
|
8615
|
+
AND run.run_id = target.run_id AND run.status = 'waiting'
|
|
8616
|
+
AND run.suspension_json ->> 'kind' <> 'detached_runner'
|
|
8617
|
+
AND target.runner_result_json ->> 'status' = 'suspended'
|
|
8618
|
+
AND target.runner_result_json -> 'suspension' = run.suspension_json
|
|
8619
|
+
AND NOT EXISTS (
|
|
8620
|
+
SELECT 1 FROM ${tableName('attempts', options)} AS newer
|
|
8621
|
+
WHERE newer.run_id = target.run_id AND newer.attempt > target.attempt
|
|
8622
|
+
AND newer.status IN ('claimed', 'running', 'waiting', 'completed', 'failed')
|
|
8623
|
+
)
|
|
8624
|
+
AND NOT EXISTS (
|
|
8625
|
+
SELECT 1 FROM ${tableName('control', options)} AS cancel
|
|
8626
|
+
WHERE cancel.key = 'runtime_cancel:' || target.run_id
|
|
8627
|
+
)
|
|
8628
|
+
RETURNING target.run_id
|
|
8629
|
+
)
|
|
8630
|
+
SELECT EXISTS (SELECT 1 FROM waiting_suspension) AS applicable,
|
|
8631
|
+
EXISTS (SELECT 1 FROM prepared) AS prepared
|
|
8632
|
+
`,
|
|
8633
|
+
[input.runId, input.attempt, input.leaseToken],
|
|
8634
|
+
);
|
|
8635
|
+
await client.query('COMMIT');
|
|
8636
|
+
return claimed.rows[0]?.prepared
|
|
8637
|
+
? 'prepared'
|
|
8638
|
+
: claimed.rows[0]?.applicable
|
|
8639
|
+
? 'fence_lost'
|
|
8640
|
+
: 'not_applicable';
|
|
8641
|
+
} catch (error) {
|
|
8642
|
+
await client.query('ROLLBACK');
|
|
8643
|
+
throw error;
|
|
8644
|
+
}
|
|
8645
|
+
}
|
|
8646
|
+
|
|
8578
8647
|
export type PostgresSchedulerRecoveryCircuit = {
|
|
8579
8648
|
dependency: string;
|
|
8580
8649
|
state: 'closed' | 'open' | 'half_open';
|
|
@@ -12960,7 +13029,7 @@ export async function readPostgresSchedulerRunnerTerminal(
|
|
|
12960
13029
|
return result.rows[0]?.runner_result_json ?? null;
|
|
12961
13030
|
}
|
|
12962
13031
|
|
|
12963
|
-
async function readPostgresSchedulerRunnerTerminalReceivedAtMs(
|
|
13032
|
+
export async function readPostgresSchedulerRunnerTerminalReceivedAtMs(
|
|
12964
13033
|
client: PostgresSchedulerQueryClient,
|
|
12965
13034
|
input: { runId: string; attempt: number },
|
|
12966
13035
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
@@ -13630,6 +13699,8 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13630
13699
|
* after claim. Resumable `suspended` receipts may be superseded by the
|
|
13631
13700
|
* hard ceiling in the same fenced statement. */
|
|
13632
13701
|
preserveRunnerTerminal?: boolean;
|
|
13702
|
+
/** Internal typed-wait failure path; absence retains ordinary terminal semantics. */
|
|
13703
|
+
protectWaitingSuspensionFailure?: boolean;
|
|
13633
13704
|
sandboxCrashDiagnostic?: PostgresSchedulerSandboxCrashDiagnosticRequest;
|
|
13634
13705
|
/** Progress facts the ledger must apply before the terminal fact. Omitted
|
|
13635
13706
|
* from the payload when empty. */
|
|
@@ -13700,8 +13771,9 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13700
13771
|
const terminalLogMetadata = terminalLogEntries.map((entry) =>
|
|
13701
13772
|
normalizePlayLogMetadata(entry.metadata),
|
|
13702
13773
|
);
|
|
13703
|
-
const
|
|
13704
|
-
|
|
13774
|
+
const writeFailure = () =>
|
|
13775
|
+
client.query<{ run_id: string }>(
|
|
13776
|
+
`
|
|
13705
13777
|
WITH ${runtimeSandboxCustodyLocksSql('run.run_id = $1', options)}, failed AS (
|
|
13706
13778
|
UPDATE ${tableName('attempts', options)}
|
|
13707
13779
|
SET status = 'failed',
|
|
@@ -13759,6 +13831,27 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13759
13831
|
OR runner_result_json IS NULL
|
|
13760
13832
|
OR runner_result_json ->> 'status' = 'suspended'
|
|
13761
13833
|
)
|
|
13834
|
+
${
|
|
13835
|
+
input.protectWaitingSuspensionFailure
|
|
13836
|
+
? `
|
|
13837
|
+
AND EXISTS (
|
|
13838
|
+
SELECT 1 FROM custody_runs AS run
|
|
13839
|
+
WHERE run.run_id = $1 AND run.status = 'waiting'
|
|
13840
|
+
AND run.suspension_json ->> 'kind' <> 'detached_runner'
|
|
13841
|
+
AND attempts.runner_result_json ->> 'status' = 'suspended'
|
|
13842
|
+
AND attempts.runner_result_json -> 'suspension' = run.suspension_json
|
|
13843
|
+
)
|
|
13844
|
+
AND NOT EXISTS (
|
|
13845
|
+
SELECT 1 FROM ${tableName('control', options)} AS cancel
|
|
13846
|
+
WHERE cancel.key = 'runtime_cancel:' || $1
|
|
13847
|
+
)
|
|
13848
|
+
AND NOT EXISTS (
|
|
13849
|
+
SELECT 1 FROM ${tableName('attempts', options)} AS newer
|
|
13850
|
+
WHERE newer.run_id = $1 AND newer.attempt > $2
|
|
13851
|
+
AND newer.status IN ('claimed', 'running', 'waiting', 'completed', 'failed')
|
|
13852
|
+
)`
|
|
13853
|
+
: ''
|
|
13854
|
+
}
|
|
13762
13855
|
RETURNING
|
|
13763
13856
|
run_id,
|
|
13764
13857
|
attempt,
|
|
@@ -13956,36 +14049,75 @@ export async function failPostgresSchedulerAttempt(
|
|
|
13956
14049
|
)
|
|
13957
14050
|
SELECT run_id FROM notify_terminal
|
|
13958
14051
|
`,
|
|
13959
|
-
|
|
13960
|
-
|
|
13961
|
-
|
|
13962
|
-
|
|
13963
|
-
|
|
13964
|
-
|
|
13965
|
-
|
|
13966
|
-
|
|
13967
|
-
|
|
13968
|
-
|
|
13969
|
-
|
|
13970
|
-
|
|
13971
|
-
|
|
13972
|
-
|
|
13973
|
-
|
|
13974
|
-
|
|
13975
|
-
|
|
13976
|
-
|
|
13977
|
-
|
|
13978
|
-
|
|
13979
|
-
|
|
13980
|
-
|
|
13981
|
-
|
|
13982
|
-
|
|
13983
|
-
|
|
13984
|
-
|
|
13985
|
-
|
|
14052
|
+
[
|
|
14053
|
+
input.runId,
|
|
14054
|
+
input.attempt,
|
|
14055
|
+
input.leaseToken,
|
|
14056
|
+
stringifyPostgresJson({ message: input.error }),
|
|
14057
|
+
stringifyPostgresJson(terminalResult ?? null),
|
|
14058
|
+
postgresSchedulerNotifyChannel('terminal', options),
|
|
14059
|
+
JSON.stringify({ runId: input.runId, status: 'failed' }),
|
|
14060
|
+
stringifyPostgresJson(ledgerTerminalResult ?? null),
|
|
14061
|
+
runtimeTerminalReasonFromError(input.error),
|
|
14062
|
+
stringifyPostgresJson(input.sandboxCrashDiagnostic ?? null),
|
|
14063
|
+
input.preserveRunnerTerminal === true,
|
|
14064
|
+
JSON.stringify(terminalLogLines),
|
|
14065
|
+
stringifyPostgresJson(resultSummary),
|
|
14066
|
+
JSON.stringify(terminalLogContexts),
|
|
14067
|
+
stringifyPostgresJson(terminalWorkCounterSegment ?? null),
|
|
14068
|
+
stringifyPostgresJson(
|
|
14069
|
+
sanitizeTerminalProgressEvents(input.terminalProgressEvents),
|
|
14070
|
+
),
|
|
14071
|
+
JSON.stringify(terminalLogMetadata),
|
|
14072
|
+
stringifyPostgresJson(
|
|
14073
|
+
terminalLogIds
|
|
14074
|
+
? terminalLogEntries.map((entry) => entry.entryId)
|
|
14075
|
+
: null,
|
|
14076
|
+
),
|
|
14077
|
+
],
|
|
14078
|
+
);
|
|
14079
|
+
const failed = input.protectWaitingSuspensionFailure
|
|
14080
|
+
? await withWaitingSuspensionFailureCustody(
|
|
14081
|
+
client,
|
|
14082
|
+
input.runId,
|
|
14083
|
+
writeFailure,
|
|
14084
|
+
options,
|
|
14085
|
+
)
|
|
14086
|
+
: await writeFailure();
|
|
13986
14087
|
return failed.rows.length > 0;
|
|
13987
14088
|
}
|
|
13988
14089
|
|
|
14090
|
+
/** The terminal statement gets a fresh snapshot after every custody lock. */
|
|
14091
|
+
async function withWaitingSuspensionFailureCustody<T>(
|
|
14092
|
+
client: PostgresSchedulerQueryClient,
|
|
14093
|
+
runId: string,
|
|
14094
|
+
write: () => Promise<T>,
|
|
14095
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
14096
|
+
): Promise<T> {
|
|
14097
|
+
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
14098
|
+
try {
|
|
14099
|
+
await client.query(
|
|
14100
|
+
`WITH ${runtimeSandboxCustodyLocksSql('run.run_id = $1', options)}
|
|
14101
|
+
SELECT count(*) FROM custody_operations`,
|
|
14102
|
+
[runId],
|
|
14103
|
+
);
|
|
14104
|
+
const result = await write();
|
|
14105
|
+
await client.query('COMMIT');
|
|
14106
|
+
return result;
|
|
14107
|
+
} catch (error) {
|
|
14108
|
+
try {
|
|
14109
|
+
await client.query('ROLLBACK');
|
|
14110
|
+
} catch (rollbackError) {
|
|
14111
|
+
throw new AggregateError(
|
|
14112
|
+
[error, rollbackError],
|
|
14113
|
+
'Typed suspension failure rollback failed.',
|
|
14114
|
+
{ cause: error },
|
|
14115
|
+
);
|
|
14116
|
+
}
|
|
14117
|
+
throw error;
|
|
14118
|
+
}
|
|
14119
|
+
}
|
|
14120
|
+
|
|
13989
14121
|
/**
|
|
13990
14122
|
* Cancel a run: terminal `cancelled` status, all live attempts cancelled, and
|
|
13991
14123
|
* terminal notify so observers settle immediately. An in-flight executor's
|
|
@@ -15171,6 +15303,9 @@ export async function executePostgresSchedulerClaim(
|
|
|
15171
15303
|
...(options.preserveRunnerTerminal
|
|
15172
15304
|
? { preserveRunnerTerminal: true }
|
|
15173
15305
|
: {}),
|
|
15306
|
+
...(options.protectWaitingSuspensionFailure
|
|
15307
|
+
? { protectWaitingSuspensionFailure: true }
|
|
15308
|
+
: {}),
|
|
15174
15309
|
terminalProgressEvents: [
|
|
15175
15310
|
...terminalWorkCounterProgressEventsFromRunnerResult(
|
|
15176
15311
|
terminalFailureResult,
|
|
@@ -18371,24 +18506,98 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
18371
18506
|
* successor. The receiver's ingress capability and frozen receipts still
|
|
18372
18507
|
* determine which causal facts may share a prefix.
|
|
18373
18508
|
*/
|
|
18374
|
-
|
|
18375
|
-
|
|
18509
|
+
type RunLedgerTransportCandidate = {
|
|
18510
|
+
eventId: string;
|
|
18511
|
+
eventType: string;
|
|
18512
|
+
deliveryKey: string;
|
|
18513
|
+
};
|
|
18514
|
+
|
|
18515
|
+
function runLedgerZeroEffectTransportSql(alias: string): string {
|
|
18516
|
+
return `${alias}.event_type IN (${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runWaiting}, ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runResumed})
|
|
18517
|
+
AND jsonb_typeof(${alias}.payload_json) = 'object'
|
|
18518
|
+
AND ${alias}.payload_json->'kind' = '"detached_runner"'::jsonb
|
|
18519
|
+
AND ${alias}.delivery_key IS NOT NULL AND ${alias}.delivery_key <> ''
|
|
18520
|
+
AND ${alias}.payload_json->'${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}' IS DISTINCT FROM 'true'::jsonb
|
|
18521
|
+
AND ${alias}.payload_json->'${RUNTIME_EVENT_SEARCH_ONLY_FIELD}' IS DISTINCT FROM 'true'::jsonb
|
|
18522
|
+
AND to_jsonb(${alias})->>'delivery_batch_started_at' IS NULL
|
|
18523
|
+
AND ${alias}.failed_at IS NULL
|
|
18524
|
+
AND coalesce(${alias}.next_attempt_at, ${alias}.created_at) <= now()
|
|
18525
|
+
AND ${alias}.lease_owner IS NULL AND ${alias}.lease_expires_at IS NULL`;
|
|
18526
|
+
}
|
|
18527
|
+
|
|
18528
|
+
type RunLedgerTransportDelivery = {
|
|
18529
|
+
eventIds: string[];
|
|
18530
|
+
deliveries: RunLedgerOutboxDeliveredRow[];
|
|
18531
|
+
};
|
|
18532
|
+
|
|
18533
|
+
type RunLedgerOutboxPrefixClaimRow = {
|
|
18534
|
+
row_kind:
|
|
18535
|
+
| 'claimed'
|
|
18536
|
+
| 'delivered'
|
|
18537
|
+
| 'transport_delivered'
|
|
18538
|
+
| 'transport_unconfirmed';
|
|
18539
|
+
event_id: string;
|
|
18540
|
+
run_id: string;
|
|
18541
|
+
event_type: string;
|
|
18542
|
+
payload_json: unknown;
|
|
18543
|
+
created_at: Date;
|
|
18544
|
+
delivery_key: string | null;
|
|
18545
|
+
lag_ms: number | string | null;
|
|
18546
|
+
lease_owner?: string | null;
|
|
18547
|
+
delivery_frozen?: boolean;
|
|
18548
|
+
lease_remaining_ms?: number | string | null;
|
|
18549
|
+
};
|
|
18550
|
+
|
|
18551
|
+
type RunLedgerFreshClaimLeaseFacts = {
|
|
18552
|
+
leaseOwner: string;
|
|
18553
|
+
prefix: readonly { eventId: string; deliveryKey: string }[];
|
|
18554
|
+
remainingLeaseMs: number;
|
|
18555
|
+
};
|
|
18556
|
+
|
|
18557
|
+
type RunLedgerFreshClaimFirstHttpProof = RunLedgerFreshClaimLeaseFacts & {
|
|
18558
|
+
receivedAtMonotonicMs: number;
|
|
18559
|
+
};
|
|
18560
|
+
|
|
18561
|
+
function freshRunLedgerClaimCoversFirstHttp(
|
|
18562
|
+
proof: RunLedgerFreshClaimFirstHttpProof | null,
|
|
18563
|
+
events: readonly PostgresSchedulerOutboxEvent[],
|
|
18564
|
+
leaseOwner: string,
|
|
18565
|
+
requiredRemainingMs: number,
|
|
18566
|
+
): boolean {
|
|
18567
|
+
if (
|
|
18568
|
+
!proof ||
|
|
18569
|
+
proof.leaseOwner !== leaseOwner ||
|
|
18570
|
+
proof.prefix.length !== events.length
|
|
18571
|
+
)
|
|
18572
|
+
return false;
|
|
18573
|
+
const elapsedMs = performance.now() - proof.receivedAtMonotonicMs;
|
|
18574
|
+
return (
|
|
18575
|
+
Number.isFinite(elapsedMs) &&
|
|
18576
|
+
elapsedMs >= 0 &&
|
|
18577
|
+
proof.remainingLeaseMs - elapsedMs > requiredRemainingMs &&
|
|
18578
|
+
proof.prefix.every(
|
|
18579
|
+
(fact, index) =>
|
|
18580
|
+
fact.eventId === events[index]?.eventId &&
|
|
18581
|
+
fact.deliveryKey === events[index]?.deliveryKey &&
|
|
18582
|
+
events[index]?.leaseOwner === leaseOwner,
|
|
18583
|
+
)
|
|
18584
|
+
);
|
|
18585
|
+
}
|
|
18586
|
+
|
|
18587
|
+
/** One SQL owner for the legacy client transaction and the optional server operation. */
|
|
18588
|
+
function runLedgerOutboxPrefixClaimSql(
|
|
18376
18589
|
input: NonNullable<
|
|
18377
18590
|
Parameters<typeof readPendingPostgresSchedulerOutboxEvents>[1]
|
|
18378
18591
|
>,
|
|
18379
|
-
options
|
|
18380
|
-
acknowledgement
|
|
18381
|
-
|
|
18382
|
-
|
|
18383
|
-
|
|
18384
|
-
}> {
|
|
18385
|
-
const requestedLimit = Math.floor(input.limit ?? 1);
|
|
18386
|
-
const limit = Number.isFinite(requestedLimit)
|
|
18387
|
-
? Math.max(1, requestedLimit)
|
|
18388
|
-
: 1;
|
|
18592
|
+
options: Pick<PostgresSchedulerOptions, 'schema'> | undefined,
|
|
18593
|
+
acknowledgement: { eventIds: string[]; targetEventId: string } | undefined,
|
|
18594
|
+
supportsTerminalBatchMarker: boolean,
|
|
18595
|
+
fence?: { runId: string; expectedHeadEventId?: string },
|
|
18596
|
+
): string {
|
|
18389
18597
|
const runId = input.runId?.trim() || null;
|
|
18390
|
-
const
|
|
18391
|
-
|
|
18598
|
+
const lockFence = fence
|
|
18599
|
+
? `current_setting('deepline.run_ledger_outbox_lock', true) = ${escapeLiteral(fence.runId)}`
|
|
18600
|
+
: undefined;
|
|
18392
18601
|
const pending = (alias: string) =>
|
|
18393
18602
|
`${alias}.delivered_at IS NULL${
|
|
18394
18603
|
acknowledgement ? ` AND NOT (${alias}.event_id = ANY($5::text[]))` : ''
|
|
@@ -18398,34 +18607,33 @@ async function claimPostgresSchedulerOutboxPrefix(
|
|
|
18398
18607
|
// fact makes a single run's burst quadratic. Failed/delayed heads must
|
|
18399
18608
|
// remain in this seek so they still block their causal successors.
|
|
18400
18609
|
const seekRunHead = runId !== null && !input.extendOwnedHead;
|
|
18401
|
-
const
|
|
18402
|
-
|
|
18403
|
-
|
|
18404
|
-
|
|
18405
|
-
|
|
18406
|
-
|
|
18407
|
-
|
|
18408
|
-
|
|
18409
|
-
|
|
18410
|
-
|
|
18411
|
-
|
|
18412
|
-
|
|
18413
|
-
|
|
18414
|
-
|
|
18415
|
-
|
|
18416
|
-
|
|
18417
|
-
|
|
18418
|
-
|
|
18419
|
-
|
|
18420
|
-
|
|
18421
|
-
|
|
18422
|
-
|
|
18423
|
-
|
|
18424
|
-
|
|
18425
|
-
|
|
18426
|
-
|
|
18427
|
-
|
|
18428
|
-
const blockingPredecessorSql = (alias: string) => `(
|
|
18610
|
+
const urgentLifecyclePendingSql = (alias: string) =>
|
|
18611
|
+
input.searchOnly === true
|
|
18612
|
+
? runLedgerTerminalPendingSql(alias, options)
|
|
18613
|
+
: runLedgerUrgentLifecyclePendingSql(
|
|
18614
|
+
alias,
|
|
18615
|
+
options,
|
|
18616
|
+
// Every claimed row belongs to the selected run. An independent
|
|
18617
|
+
// wait probe evaluates once, instead of scanning its backlog for
|
|
18618
|
+
// each row whose delivery batch is being frozen.
|
|
18619
|
+
runId !== null ? '$2::text' : undefined,
|
|
18620
|
+
);
|
|
18621
|
+
// Search is locally indexed and acknowledged; it has no remote ingress
|
|
18622
|
+
// receipt to freeze. Preserve causal ordering/due/lease checks, but admit
|
|
18623
|
+
// its full prefix. Lifecycle ingress prefixes require receiver support.
|
|
18624
|
+
const ingressPrefixSql =
|
|
18625
|
+
input.searchOnly === true
|
|
18626
|
+
? 'true'
|
|
18627
|
+
: supportsTerminalBatchMarker && input.allowIngressPrefix === true
|
|
18628
|
+
? `(head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents})`
|
|
18629
|
+
: 'false';
|
|
18630
|
+
const findSearchBlockersOnce = seekRunHead && input.searchOnly === true;
|
|
18631
|
+
const searchFactColumnsSql = (alias: string) =>
|
|
18632
|
+
`${alias}.event_id, ${alias}.run_id, ${alias}.event_type,
|
|
18633
|
+
${alias}.created_at, ${alias}.failed_at, ${alias}.next_attempt_at,
|
|
18634
|
+
${alias}.lease_owner, ${alias}.lease_expires_at,
|
|
18635
|
+
${supportsTerminalBatchMarker ? `${alias}.delivery_batch_started_at` : 'NULL::timestamptz AS delivery_batch_started_at'}`;
|
|
18636
|
+
const blockingPredecessorSql = (alias: string) => `(
|
|
18429
18637
|
(${alias}.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
|
|
18430
18638
|
AND NOT (${ingressPrefixSql}))
|
|
18431
18639
|
OR (${supportsTerminalBatchMarker && input.allowIngressPrefix === true ? `head.delivery_batch_started_at IS NOT NULL AND ${alias}.delivery_batch_started_at IS NULL` : 'false'})
|
|
@@ -18438,21 +18646,11 @@ async function claimPostgresSchedulerOutboxPrefix(
|
|
|
18438
18646
|
AND ${alias}.lease_owner IS DISTINCT FROM $3::text
|
|
18439
18647
|
)
|
|
18440
18648
|
)`;
|
|
18441
|
-
|
|
18442
|
-
row_kind: 'claimed' | 'delivered';
|
|
18443
|
-
event_id: string;
|
|
18444
|
-
run_id: string;
|
|
18445
|
-
event_type: string;
|
|
18446
|
-
payload_json: unknown;
|
|
18447
|
-
created_at: Date;
|
|
18448
|
-
delivery_key: string | null;
|
|
18449
|
-
lag_ms: number | string | null;
|
|
18450
|
-
}>(
|
|
18451
|
-
`
|
|
18649
|
+
return `
|
|
18452
18650
|
WITH ${
|
|
18453
18651
|
acknowledgement
|
|
18454
18652
|
? `delivered AS (
|
|
18455
|
-
${runLedgerAcknowledgeSql(options, '$5', '$3')}
|
|
18653
|
+
${runLedgerAcknowledgeSql(options, '$5', '$3', lockFence, runId === null ? undefined : '$2')}
|
|
18456
18654
|
),`
|
|
18457
18655
|
: ''
|
|
18458
18656
|
} ${
|
|
@@ -18486,6 +18684,8 @@ async function claimPostgresSchedulerOutboxPrefix(
|
|
|
18486
18684
|
${seekRunHead ? 'JOIN pending_head ON pending_head.event_id = candidate.event_id' : ''}
|
|
18487
18685
|
${input.extendOwnedHead ? 'JOIN owned_run ON owned_run.run_id = candidate.run_id' : ''}
|
|
18488
18686
|
WHERE ${pending('candidate')}
|
|
18687
|
+
${lockFence ? `AND (${lockFence})` : ''}
|
|
18688
|
+
${fence?.expectedHeadEventId ? `AND candidate.event_id = ${escapeLiteral(fence.expectedHeadEventId)}` : ''}
|
|
18489
18689
|
AND ${searchLaneSql('candidate')} = ${input.searchOnly === true ? 'true' : 'false'}
|
|
18490
18690
|
${
|
|
18491
18691
|
acknowledgement
|
|
@@ -18742,7 +18942,9 @@ async function claimPostgresSchedulerOutboxPrefix(
|
|
|
18742
18942
|
WHERE outbox.event_id = candidates.event_id
|
|
18743
18943
|
RETURNING outbox.event_id, outbox.run_id, outbox.event_type,
|
|
18744
18944
|
outbox.payload_json, outbox.created_at, outbox.lease_owner,
|
|
18745
|
-
outbox.delivery_key
|
|
18945
|
+
outbox.delivery_key,
|
|
18946
|
+
${supportsTerminalBatchMarker ? 'outbox.delivery_batch_started_at IS NOT NULL' : 'false'} AS delivery_frozen,
|
|
18947
|
+
(extract(epoch FROM (outbox.lease_expires_at - statement_timestamp())) * 1000)::float8 AS lease_remaining_ms
|
|
18746
18948
|
)
|
|
18747
18949
|
${
|
|
18748
18950
|
acknowledgement
|
|
@@ -18759,7 +18961,7 @@ async function claimPostgresSchedulerOutboxPrefix(
|
|
|
18759
18961
|
SELECT * FROM (
|
|
18760
18962
|
SELECT 'claimed'::text AS row_kind,
|
|
18761
18963
|
event_id, run_id, event_type, payload_json, created_at, lease_owner,
|
|
18762
|
-
delivery_key, NULL::float8 AS lag_ms
|
|
18964
|
+
delivery_key, NULL::float8 AS lag_ms, delivery_frozen, lease_remaining_ms
|
|
18763
18965
|
FROM claimed
|
|
18764
18966
|
${
|
|
18765
18967
|
acknowledgement
|
|
@@ -18767,109 +18969,483 @@ async function claimPostgresSchedulerOutboxPrefix(
|
|
|
18767
18969
|
SELECT 'delivered'::text AS row_kind,
|
|
18768
18970
|
event_id, NULL::text AS run_id, event_type, NULL::jsonb AS payload_json,
|
|
18769
18971
|
NULL::timestamptz AS created_at, NULL::text AS lease_owner,
|
|
18770
|
-
NULL::text AS delivery_key, lag_ms
|
|
18972
|
+
NULL::text AS delivery_key, lag_ms, false AS delivery_frozen, NULL::float8 AS lease_remaining_ms
|
|
18771
18973
|
FROM notified`
|
|
18772
18974
|
: ''
|
|
18773
18975
|
}
|
|
18774
18976
|
) AS result
|
|
18775
18977
|
ORDER BY row_kind, ${runLedgerOrderSql('result')}
|
|
18776
|
-
|
|
18777
|
-
|
|
18778
|
-
|
|
18779
|
-
|
|
18780
|
-
|
|
18781
|
-
|
|
18782
|
-
|
|
18783
|
-
|
|
18784
|
-
|
|
18785
|
-
|
|
18786
|
-
|
|
18787
|
-
|
|
18788
|
-
|
|
18789
|
-
|
|
18790
|
-
|
|
18791
|
-
|
|
18792
|
-
|
|
18793
|
-
|
|
18794
|
-
|
|
18795
|
-
|
|
18796
|
-
|
|
18797
|
-
|
|
18798
|
-
|
|
18799
|
-
|
|
18800
|
-
|
|
18801
|
-
|
|
18802
|
-
|
|
18803
|
-
|
|
18804
|
-
|
|
18805
|
-
|
|
18806
|
-
|
|
18807
|
-
|
|
18808
|
-
throw error;
|
|
18809
|
-
if (useSavepoint)
|
|
18810
|
-
await client.query('ROLLBACK TO SAVEPOINT run_ledger_claim');
|
|
18811
|
-
result = await claim(false);
|
|
18978
|
+
`;
|
|
18979
|
+
}
|
|
18980
|
+
|
|
18981
|
+
const runLedgerOutboxBatchUnsupportedClients = new WeakSet<object>();
|
|
18982
|
+
|
|
18983
|
+
function isUnsupportedRunLedgerOutboxBatch(error: unknown): boolean {
|
|
18984
|
+
return (
|
|
18985
|
+
!!error &&
|
|
18986
|
+
typeof error === 'object' &&
|
|
18987
|
+
'code' in error &&
|
|
18988
|
+
error.code === '42601' &&
|
|
18989
|
+
'message' in error &&
|
|
18990
|
+
error.message ===
|
|
18991
|
+
'cannot insert multiple commands into a prepared statement'
|
|
18992
|
+
);
|
|
18993
|
+
}
|
|
18994
|
+
|
|
18995
|
+
/** Render only the canonical owner's typed arguments using node-postgres quoting. */
|
|
18996
|
+
function runLedgerOutboxPrefixClaimBatchSql(input: {
|
|
18997
|
+
sql: string;
|
|
18998
|
+
schema?: string;
|
|
18999
|
+
params: unknown[];
|
|
19000
|
+
runId: string;
|
|
19001
|
+
waitForRunLock: boolean;
|
|
19002
|
+
lockStrength?: 'no_key_update';
|
|
19003
|
+
transportHeadSql?: string;
|
|
19004
|
+
}): string {
|
|
19005
|
+
const encoded = escapeLiteral(JSON.stringify(input.params));
|
|
19006
|
+
const parameter = (index: number) => {
|
|
19007
|
+
const value = input.params[index - 1];
|
|
19008
|
+
if (Array.isArray(value)) {
|
|
19009
|
+
return `(SELECT ARRAY(SELECT jsonb_array_elements_text(values_json->${index - 1})) FROM run_ledger_parameters)`;
|
|
18812
19010
|
}
|
|
18813
|
-
|
|
18814
|
-
|
|
18815
|
-
|
|
18816
|
-
|
|
18817
|
-
|
|
18818
|
-
|
|
18819
|
-
|
|
18820
|
-
|
|
18821
|
-
|
|
18822
|
-
|
|
18823
|
-
|
|
18824
|
-
|
|
18825
|
-
|
|
18826
|
-
|
|
18827
|
-
|
|
18828
|
-
|
|
18829
|
-
|
|
18830
|
-
|
|
18831
|
-
|
|
18832
|
-
|
|
18833
|
-
|
|
18834
|
-
|
|
18835
|
-
|
|
18836
|
-
|
|
18837
|
-
|
|
18838
|
-
|
|
18839
|
-
|
|
18840
|
-
|
|
18841
|
-
lagMs: Number.isFinite(lagMs) ? Math.max(0, lagMs) : 0,
|
|
18842
|
-
};
|
|
18843
|
-
});
|
|
18844
|
-
return { events, deliveredRows };
|
|
19011
|
+
const type = typeof value === 'number' ? 'integer' : 'text';
|
|
19012
|
+
const encodedParameter = `(SELECT (values_json->>${index - 1})::${type} FROM run_ledger_parameters)`;
|
|
19013
|
+
return index === 1 && input.transportHeadSql
|
|
19014
|
+
? `(${encodedParameter} - current_setting('deepline.run_ledger_transport_ack_count')::integer)`
|
|
19015
|
+
: encodedParameter;
|
|
19016
|
+
};
|
|
19017
|
+
// Skip quoted identifiers/strings and comments. A run or event ID may
|
|
19018
|
+
// legitimately contain "$1"; only actual bind tokens may be rendered.
|
|
19019
|
+
const claimSql = input.sql
|
|
19020
|
+
.replace(
|
|
19021
|
+
/'(?:[^']|'')*'|"(?:[^"]|"")*"|--[^\n]*|\/\*[\s\S]*?\*\/|\$(\d+)\b/g,
|
|
19022
|
+
(match, index: string | undefined) =>
|
|
19023
|
+
index ? parameter(Number(index)) : match,
|
|
19024
|
+
)
|
|
19025
|
+
.replace(
|
|
19026
|
+
/^\s*WITH\s/,
|
|
19027
|
+
`WITH run_ledger_parameters AS MATERIALIZED (SELECT ${encoded}::jsonb AS values_json), `,
|
|
19028
|
+
);
|
|
19029
|
+
return `BEGIN;
|
|
19030
|
+
SELECT set_config('lock_timeout', '${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms', true),
|
|
19031
|
+
set_config('statement_timeout', '${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms', true);
|
|
19032
|
+
SELECT set_config('deepline.run_ledger_outbox_lock', coalesce((
|
|
19033
|
+
SELECT run_id FROM ${tableName('runs', { schema: input.schema })}
|
|
19034
|
+
WHERE run_id = ${escapeLiteral(input.runId)} FOR ${input.lockStrength === 'no_key_update' ? 'NO KEY UPDATE' : 'UPDATE'}${input.waitForRunLock ? '' : ' SKIP LOCKED'}
|
|
19035
|
+
), ''), true);
|
|
19036
|
+
${input.transportHeadSql ? `SELECT set_config('deepline.run_ledger_transport_ack_count', '0', true);\n ${input.transportHeadSql};` : ''}
|
|
19037
|
+
${claimSql};
|
|
19038
|
+
COMMIT;`;
|
|
18845
19039
|
}
|
|
18846
19040
|
|
|
18847
19041
|
/**
|
|
18848
|
-
*
|
|
18849
|
-
*
|
|
18850
|
-
*
|
|
18851
|
-
* its retry time. The recovery controller uses this to sleep until real work
|
|
18852
|
-
* exists instead of having every runtime worker poll the whole outbox.
|
|
19042
|
+
* One simple-protocol request, still separate READ COMMITTED statements. The run
|
|
19043
|
+
* lock and the subsequent claim therefore keep a fresh snapshot after a writer
|
|
19044
|
+
* commits. No transaction or lease checkout is retained over an HTTP append.
|
|
18853
19045
|
*/
|
|
18854
|
-
|
|
19046
|
+
async function claimRunLedgerOutboxPrefixInBatch(
|
|
18855
19047
|
client: PostgresSchedulerQueryClient,
|
|
18856
|
-
|
|
18857
|
-
|
|
18858
|
-
|
|
18859
|
-
runId
|
|
19048
|
+
input: NonNullable<
|
|
19049
|
+
Parameters<typeof readPendingPostgresSchedulerOutboxEvents>[1]
|
|
19050
|
+
> & {
|
|
19051
|
+
runId: string;
|
|
19052
|
+
waitForRunLock: boolean;
|
|
19053
|
+
expectedHeadEventId?: string;
|
|
19054
|
+
lockStrength?: 'no_key_update';
|
|
19055
|
+
transportCandidate?: RunLedgerTransportCandidate;
|
|
19056
|
+
recoverTransportDelivery?: boolean;
|
|
18860
19057
|
},
|
|
18861
|
-
|
|
18862
|
-
|
|
18863
|
-
|
|
18864
|
-
|
|
18865
|
-
|
|
18866
|
-
|
|
18867
|
-
|
|
18868
|
-
|
|
18869
|
-
|
|
18870
|
-
|
|
18871
|
-
|
|
18872
|
-
|
|
19058
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
19059
|
+
acknowledgement?: { eventIds: string[]; targetEventId: string },
|
|
19060
|
+
): Promise<{
|
|
19061
|
+
events: PostgresSchedulerOutboxEvent[];
|
|
19062
|
+
deliveredRows: RunLedgerOutboxDeliveredRow[];
|
|
19063
|
+
freshClaimLease?: RunLedgerFreshClaimLeaseFacts;
|
|
19064
|
+
transportDelivered?: RunLedgerTransportDelivery;
|
|
19065
|
+
transportOwnershipUnconfirmed?: boolean;
|
|
19066
|
+
} | null> {
|
|
19067
|
+
if (
|
|
19068
|
+
runLedgerOutboxBatchUnsupportedClients.has(client) ||
|
|
19069
|
+
!supportsOutboxDeliveryBatchMarker(options)
|
|
19070
|
+
)
|
|
19071
|
+
return null;
|
|
19072
|
+
const leaseOwner = input.leaseOwner?.trim() || null;
|
|
19073
|
+
const requestedLimit = Math.floor(input.limit ?? 1);
|
|
19074
|
+
const limit = Number.isFinite(requestedLimit)
|
|
19075
|
+
? Math.max(acknowledgement ? 0 : 1, requestedLimit)
|
|
19076
|
+
: 1;
|
|
19077
|
+
const params = [
|
|
19078
|
+
limit,
|
|
19079
|
+
input.runId,
|
|
19080
|
+
leaseOwner,
|
|
19081
|
+
Math.max(5, Math.min(300, input.leaseSeconds ?? 30)),
|
|
19082
|
+
...(acknowledgement
|
|
19083
|
+
? [
|
|
19084
|
+
acknowledgement.eventIds,
|
|
19085
|
+
acknowledgement.targetEventId,
|
|
19086
|
+
postgresSchedulerNotifyChannel('terminal', options),
|
|
19087
|
+
]
|
|
19088
|
+
: input.extendOwnedHead
|
|
19089
|
+
? [input.extendOwnedHead]
|
|
19090
|
+
: []),
|
|
19091
|
+
];
|
|
19092
|
+
// Detached parks/resumes have no public ledger effects. Under the existing
|
|
19093
|
+
// run/head locks a never-leased, never-frozen carrier may be acknowledged
|
|
19094
|
+
// locally before the next READ COMMITTED statement claims its successor.
|
|
19095
|
+
// Keep the successor's existing run.events-first receipt shape unchanged.
|
|
19096
|
+
const transportHeadSql =
|
|
19097
|
+
input.transportCandidate !== undefined &&
|
|
19098
|
+
input.allowIngressPrefix === true &&
|
|
19099
|
+
!input.searchOnly &&
|
|
19100
|
+
!input.extendOwnedHead &&
|
|
19101
|
+
!input.expectedHeadEventId &&
|
|
19102
|
+
!acknowledgement &&
|
|
19103
|
+
limit >= 2
|
|
19104
|
+
? `WITH transport_head AS MATERIALIZED (
|
|
19105
|
+
SELECT pending.* FROM ${tableName('outbox', options)} AS pending
|
|
19106
|
+
WHERE pending.run_id = ${escapeLiteral(input.runId)}
|
|
19107
|
+
AND current_setting('deepline.run_ledger_outbox_lock', true) = ${escapeLiteral(input.runId)}
|
|
19108
|
+
AND pending.delivered_at IS NULL
|
|
19109
|
+
AND NOT ${searchLaneSql('pending')}
|
|
19110
|
+
ORDER BY ${runLedgerOrderSql('pending')}
|
|
19111
|
+
LIMIT 1 FOR UPDATE
|
|
19112
|
+
), eligible_transport_head AS MATERIALIZED (
|
|
19113
|
+
SELECT head.* FROM transport_head AS head
|
|
19114
|
+
WHERE current_setting('deepline.run_ledger_outbox_lock', true) = ${escapeLiteral(input.runId)}
|
|
19115
|
+
AND head.event_id = ${escapeLiteral(input.transportCandidate!.eventId)}
|
|
19116
|
+
AND head.delivery_key = ${escapeLiteral(input.transportCandidate!.deliveryKey)}
|
|
19117
|
+
AND ${runLedgerZeroEffectTransportSql('head')}
|
|
19118
|
+
AND ${runLedgerTerminalPendingSql('head', options)}
|
|
19119
|
+
AND EXISTS (
|
|
19120
|
+
SELECT 1 FROM (
|
|
19121
|
+
SELECT successor.* FROM ${tableName('outbox', options)} AS successor
|
|
19122
|
+
WHERE successor.run_id = head.run_id AND successor.delivered_at IS NULL
|
|
19123
|
+
AND NOT ${searchLaneSql('successor')} AND successor.event_id <> head.event_id
|
|
19124
|
+
ORDER BY ${runLedgerOrderSql('successor')} LIMIT 1
|
|
19125
|
+
) AS successor
|
|
19126
|
+
WHERE successor.failed_at IS NULL
|
|
19127
|
+
AND coalesce(successor.next_attempt_at, successor.created_at) <= now()
|
|
19128
|
+
AND successor.lease_owner IS NULL AND successor.lease_expires_at IS NULL
|
|
19129
|
+
)
|
|
19130
|
+
AND NOT EXISTS (
|
|
19131
|
+
SELECT 1 FROM ${tableName('outbox', options)} AS active
|
|
19132
|
+
WHERE active.run_id = head.run_id AND active.delivered_at IS NULL
|
|
19133
|
+
AND NOT ${searchLaneSql('active')}
|
|
19134
|
+
AND active.lease_owner IS NOT NULL AND active.lease_expires_at > now()
|
|
19135
|
+
)
|
|
19136
|
+
), delivered AS (
|
|
19137
|
+
${runLedgerAcknowledgeSql(
|
|
19138
|
+
options,
|
|
19139
|
+
'ARRAY(SELECT event_id FROM eligible_transport_head)',
|
|
19140
|
+
'NULL',
|
|
19141
|
+
`current_setting('deepline.run_ledger_outbox_lock', true) = ${escapeLiteral(input.runId)}`,
|
|
19142
|
+
escapeLiteral(input.runId),
|
|
19143
|
+
)}
|
|
19144
|
+
)${
|
|
19145
|
+
input.recoverTransportDelivery === true
|
|
19146
|
+
? `, prior_transport AS MATERIALIZED (
|
|
19147
|
+
SELECT prior.event_id, prior.run_id, prior.event_type, prior.delivery_key,
|
|
19148
|
+
(extract(epoch FROM (prior.delivered_at-prior.created_at))*1000)::float8 AS lag_ms,
|
|
19149
|
+
to_jsonb(prior)->>'delivery_batch_started_at' IS NULL AND EXISTS (
|
|
19150
|
+
SELECT 1 FROM ${tableName('outbox', options)} AS successor
|
|
19151
|
+
WHERE successor.run_id = prior.run_id AND successor.delivered_at IS NULL
|
|
19152
|
+
AND successor.lease_owner = ${escapeLiteral(leaseOwner!)}
|
|
19153
|
+
AND successor.lease_expires_at > clock_timestamp()
|
|
19154
|
+
AND successor.xmin = prior.xmin
|
|
19155
|
+
AND NOT ${searchLaneSql('successor')}
|
|
19156
|
+
) AS owned
|
|
19157
|
+
FROM ${tableName('outbox', options)} AS prior
|
|
19158
|
+
WHERE current_setting('deepline.run_ledger_outbox_lock', true) = ${escapeLiteral(input.runId)}
|
|
19159
|
+
AND prior.run_id = ${escapeLiteral(input.runId)}
|
|
19160
|
+
AND prior.event_id = ${escapeLiteral(input.transportCandidate!.eventId)}
|
|
19161
|
+
AND prior.delivery_key = ${escapeLiteral(input.transportCandidate!.deliveryKey)}
|
|
19162
|
+
AND prior.event_type = ${escapeLiteral(input.transportCandidate!.eventType)}
|
|
19163
|
+
AND prior.delivered_at IS NOT NULL
|
|
19164
|
+
AND NOT EXISTS (SELECT 1 FROM delivered)
|
|
19165
|
+
)`
|
|
19166
|
+
: ''
|
|
19167
|
+
}, transport_deliveries AS MATERIALIZED (
|
|
19168
|
+
SELECT delivered.event_id, head.run_id, delivered.event_type, head.delivery_key, delivered.lag_ms
|
|
19169
|
+
FROM delivered JOIN eligible_transport_head AS head ON head.event_id = delivered.event_id
|
|
19170
|
+
${
|
|
19171
|
+
input.recoverTransportDelivery === true
|
|
19172
|
+
? `UNION ALL
|
|
19173
|
+
SELECT event_id,run_id,event_type,delivery_key,lag_ms FROM prior_transport WHERE owned`
|
|
19174
|
+
: ''
|
|
19175
|
+
}
|
|
19176
|
+
)
|
|
19177
|
+
SELECT 'transport_delivered'::text AS row_kind,
|
|
19178
|
+
delivered.event_id, delivered.run_id, delivered.event_type,
|
|
19179
|
+
NULL::jsonb AS payload_json, NULL::timestamptz AS created_at,
|
|
19180
|
+
NULL::text AS lease_owner, delivered.delivery_key, delivered.lag_ms,
|
|
19181
|
+
false AS delivery_frozen, NULL::float8 AS lease_remaining_ms,
|
|
19182
|
+
set_config('deepline.run_ledger_transport_ack_count',
|
|
19183
|
+
(SELECT count(*) FROM transport_deliveries)::text, true),
|
|
19184
|
+
pg_notify(${escapeLiteral(postgresSchedulerNotifyChannel('terminal', options))},
|
|
19185
|
+
json_build_object('eventId', delivered.event_id, 'kind', 'outbox_delivered')::text)::text
|
|
19186
|
+
FROM transport_deliveries AS delivered
|
|
19187
|
+
${
|
|
19188
|
+
input.recoverTransportDelivery === true
|
|
19189
|
+
? `UNION ALL
|
|
19190
|
+
SELECT 'transport_unconfirmed'::text,event_id,run_id,event_type,NULL::jsonb,NULL::timestamptz,
|
|
19191
|
+
NULL::text,delivery_key,lag_ms,false,NULL::float8,NULL::text,NULL::text
|
|
19192
|
+
FROM prior_transport WHERE NOT owned`
|
|
19193
|
+
: ''
|
|
19194
|
+
}`
|
|
19195
|
+
: undefined;
|
|
19196
|
+
let result: QueryResult<RunLedgerOutboxPrefixClaimRow>;
|
|
19197
|
+
try {
|
|
19198
|
+
// No parameters: node-postgres sends the whole transaction through the
|
|
19199
|
+
// simple protocol, with all values escaped by its own literal encoder.
|
|
19200
|
+
result = await client.query<RunLedgerOutboxPrefixClaimRow>(
|
|
19201
|
+
runLedgerOutboxPrefixClaimBatchSql({
|
|
19202
|
+
sql: runLedgerOutboxPrefixClaimSql(
|
|
19203
|
+
input,
|
|
19204
|
+
options,
|
|
19205
|
+
acknowledgement,
|
|
19206
|
+
true,
|
|
19207
|
+
{
|
|
19208
|
+
runId: input.runId,
|
|
19209
|
+
expectedHeadEventId: input.expectedHeadEventId,
|
|
19210
|
+
},
|
|
19211
|
+
),
|
|
19212
|
+
params,
|
|
19213
|
+
runId: input.runId,
|
|
19214
|
+
waitForRunLock: input.waitForRunLock,
|
|
19215
|
+
lockStrength: input.lockStrength,
|
|
19216
|
+
schema: options?.schema,
|
|
19217
|
+
transportHeadSql,
|
|
19218
|
+
}),
|
|
19219
|
+
);
|
|
19220
|
+
} catch (error) {
|
|
19221
|
+
if (isUnsupportedRunLedgerOutboxBatch(error)) {
|
|
19222
|
+
// This exact parse rejection precedes execution of BEGIN. An extended-
|
|
19223
|
+
// protocol test driver keeps the existing adapter for this checkout.
|
|
19224
|
+
runLedgerOutboxBatchUnsupportedClients.add(client);
|
|
19225
|
+
return null;
|
|
19226
|
+
}
|
|
19227
|
+
try {
|
|
19228
|
+
await client.query('ROLLBACK');
|
|
19229
|
+
} catch (rollbackError) {
|
|
19230
|
+
console.error('[run-ledger-outbox.batch_rollback_failed]', {
|
|
19231
|
+
error: errorMessage(error),
|
|
19232
|
+
rollbackError: errorMessage(rollbackError),
|
|
19233
|
+
});
|
|
19234
|
+
throw error;
|
|
19235
|
+
}
|
|
19236
|
+
if (!isMissingOutboxDeliveryBatchMarker(error)) throw error;
|
|
19237
|
+
markOutboxDeliveryBatchMarkerUnavailable(options);
|
|
19238
|
+
return null;
|
|
19239
|
+
}
|
|
19240
|
+
// pg returns every command result; adapters may already flatten their rows.
|
|
19241
|
+
// SET/lock/COMMIT rows never enter the claim projection.
|
|
19242
|
+
const results: QueryResult<RunLedgerOutboxPrefixClaimRow>[] = Array.isArray(
|
|
19243
|
+
result,
|
|
19244
|
+
)
|
|
19245
|
+
? result
|
|
19246
|
+
: [result];
|
|
19247
|
+
return projectRunLedgerOutboxPrefixRows(
|
|
19248
|
+
results
|
|
19249
|
+
.flatMap((entry) => entry.rows)
|
|
19250
|
+
.filter(
|
|
19251
|
+
(row) =>
|
|
19252
|
+
row.row_kind === 'claimed' ||
|
|
19253
|
+
row.row_kind === 'delivered' ||
|
|
19254
|
+
row.row_kind === 'transport_delivered' ||
|
|
19255
|
+
row.row_kind === 'transport_unconfirmed',
|
|
19256
|
+
),
|
|
19257
|
+
leaseOwner,
|
|
19258
|
+
);
|
|
19259
|
+
}
|
|
19260
|
+
|
|
19261
|
+
async function claimPostgresSchedulerOutboxPrefix(
|
|
19262
|
+
client: PostgresSchedulerQueryClient,
|
|
19263
|
+
input: NonNullable<
|
|
19264
|
+
Parameters<typeof readPendingPostgresSchedulerOutboxEvents>[1]
|
|
19265
|
+
>,
|
|
19266
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
19267
|
+
acknowledgement?: { eventIds: string[]; targetEventId: string },
|
|
19268
|
+
): Promise<{
|
|
19269
|
+
events: PostgresSchedulerOutboxEvent[];
|
|
19270
|
+
deliveredRows: RunLedgerOutboxDeliveredRow[];
|
|
19271
|
+
freshClaimLease?: RunLedgerFreshClaimLeaseFacts;
|
|
19272
|
+
}> {
|
|
19273
|
+
const requestedLimit = Math.floor(input.limit ?? 1);
|
|
19274
|
+
const limit = Number.isFinite(requestedLimit)
|
|
19275
|
+
? Math.max(1, requestedLimit)
|
|
19276
|
+
: 1;
|
|
19277
|
+
const runId = input.runId?.trim() || null;
|
|
19278
|
+
const leaseOwner = input.leaseOwner?.trim() || null;
|
|
19279
|
+
const leaseSeconds = Math.max(5, Math.min(300, input.leaseSeconds ?? 30));
|
|
19280
|
+
const claim = async (supportsTerminalBatchMarker: boolean) => {
|
|
19281
|
+
return await client.query<RunLedgerOutboxPrefixClaimRow>(
|
|
19282
|
+
runLedgerOutboxPrefixClaimSql(
|
|
19283
|
+
input,
|
|
19284
|
+
options,
|
|
19285
|
+
acknowledgement,
|
|
19286
|
+
supportsTerminalBatchMarker,
|
|
19287
|
+
),
|
|
19288
|
+
[
|
|
19289
|
+
limit,
|
|
19290
|
+
runId,
|
|
19291
|
+
leaseOwner,
|
|
19292
|
+
leaseSeconds,
|
|
19293
|
+
...(acknowledgement
|
|
19294
|
+
? [
|
|
19295
|
+
acknowledgement.eventIds,
|
|
19296
|
+
acknowledgement.targetEventId,
|
|
19297
|
+
postgresSchedulerNotifyChannel('terminal', options),
|
|
19298
|
+
]
|
|
19299
|
+
: input.extendOwnedHead
|
|
19300
|
+
? [input.extendOwnedHead]
|
|
19301
|
+
: []),
|
|
19302
|
+
],
|
|
19303
|
+
);
|
|
19304
|
+
};
|
|
19305
|
+
let result;
|
|
19306
|
+
const useSavepoint =
|
|
19307
|
+
input.inTransaction && !input.restartTransactionOnMissingMarker;
|
|
19308
|
+
if (useSavepoint) await client.query('SAVEPOINT run_ledger_claim');
|
|
19309
|
+
if (supportsOutboxDeliveryBatchMarker(options)) {
|
|
19310
|
+
try {
|
|
19311
|
+
result = await claim(true);
|
|
19312
|
+
} catch (error) {
|
|
19313
|
+
if (!isMissingOutboxDeliveryBatchMarker(error)) throw error;
|
|
19314
|
+
markOutboxDeliveryBatchMarkerUnavailable(options);
|
|
19315
|
+
// The ACK caller owns the whole short transaction. Roll it back and
|
|
19316
|
+
// repeat only this missing-column adapter, avoiding two SAVEPOINT round
|
|
19317
|
+
// trips on every successful acknowledgement.
|
|
19318
|
+
if (acknowledgement || input.restartTransactionOnMissingMarker)
|
|
19319
|
+
throw error;
|
|
19320
|
+
if (useSavepoint)
|
|
19321
|
+
await client.query('ROLLBACK TO SAVEPOINT run_ledger_claim');
|
|
19322
|
+
result = await claim(false);
|
|
19323
|
+
}
|
|
19324
|
+
} else {
|
|
19325
|
+
result = await claim(false);
|
|
19326
|
+
}
|
|
19327
|
+
if (useSavepoint) await client.query('RELEASE SAVEPOINT run_ledger_claim');
|
|
19328
|
+
return projectRunLedgerOutboxPrefixRows(result.rows, leaseOwner);
|
|
19329
|
+
}
|
|
19330
|
+
|
|
19331
|
+
function projectRunLedgerOutboxPrefixRows(
|
|
19332
|
+
rows: RunLedgerOutboxPrefixClaimRow[],
|
|
19333
|
+
leaseOwner: string | null,
|
|
19334
|
+
): {
|
|
19335
|
+
events: PostgresSchedulerOutboxEvent[];
|
|
19336
|
+
deliveredRows: RunLedgerOutboxDeliveredRow[];
|
|
19337
|
+
freshClaimLease?: RunLedgerFreshClaimLeaseFacts;
|
|
19338
|
+
transportDelivered?: RunLedgerTransportDelivery;
|
|
19339
|
+
transportOwnershipUnconfirmed?: boolean;
|
|
19340
|
+
} {
|
|
19341
|
+
const events = rows
|
|
19342
|
+
.filter((row) => row.row_kind === 'claimed')
|
|
19343
|
+
.map((row) => {
|
|
19344
|
+
if (!row.delivery_key) {
|
|
19345
|
+
throw new Error(
|
|
19346
|
+
`Run-ledger outbox event ${row.event_id} has no delivery key.`,
|
|
19347
|
+
);
|
|
19348
|
+
}
|
|
19349
|
+
return {
|
|
19350
|
+
eventId: row.event_id,
|
|
19351
|
+
runId: row.run_id,
|
|
19352
|
+
eventType: row.event_type,
|
|
19353
|
+
payload: row.payload_json,
|
|
19354
|
+
createdAt: row.created_at,
|
|
19355
|
+
leaseOwner: leaseOwner,
|
|
19356
|
+
deliveryKey: row.delivery_key,
|
|
19357
|
+
};
|
|
19358
|
+
});
|
|
19359
|
+
const deliveredRows = rows
|
|
19360
|
+
.filter((row) => row.row_kind === 'delivered')
|
|
19361
|
+
.map((row) => {
|
|
19362
|
+
const lagMs = Number(row.lag_ms);
|
|
19363
|
+
return {
|
|
19364
|
+
eventType: row.event_type,
|
|
19365
|
+
lagMs: Number.isFinite(lagMs) ? Math.max(0, lagMs) : 0,
|
|
19366
|
+
};
|
|
19367
|
+
});
|
|
19368
|
+
const claimedRows = rows.filter((row) => row.row_kind === 'claimed');
|
|
19369
|
+
const remainingLeases = claimedRows.map((row) =>
|
|
19370
|
+
typeof row.lease_remaining_ms === 'number' ||
|
|
19371
|
+
(typeof row.lease_remaining_ms === 'string' &&
|
|
19372
|
+
row.lease_remaining_ms.trim())
|
|
19373
|
+
? Number(row.lease_remaining_ms)
|
|
19374
|
+
: NaN,
|
|
19375
|
+
);
|
|
19376
|
+
const freshClaimLease =
|
|
19377
|
+
leaseOwner &&
|
|
19378
|
+
claimedRows.length > 0 &&
|
|
19379
|
+
claimedRows.every(
|
|
19380
|
+
(row) => row.lease_owner === leaseOwner && row.delivery_frozen === true,
|
|
19381
|
+
) &&
|
|
19382
|
+
remainingLeases.every(
|
|
19383
|
+
(remaining) => Number.isFinite(remaining) && remaining > 0,
|
|
19384
|
+
)
|
|
19385
|
+
? {
|
|
19386
|
+
leaseOwner,
|
|
19387
|
+
prefix: events.map(({ eventId, deliveryKey }) => ({
|
|
19388
|
+
eventId,
|
|
19389
|
+
deliveryKey,
|
|
19390
|
+
})),
|
|
19391
|
+
remainingLeaseMs: Math.floor(
|
|
19392
|
+
remainingLeases.reduce(
|
|
19393
|
+
(minimum, remaining) => Math.min(minimum, remaining),
|
|
19394
|
+
Infinity,
|
|
19395
|
+
),
|
|
19396
|
+
),
|
|
19397
|
+
}
|
|
19398
|
+
: undefined;
|
|
19399
|
+
const transportRows = rows.filter(
|
|
19400
|
+
(row) => row.row_kind === 'transport_delivered',
|
|
19401
|
+
);
|
|
19402
|
+
return {
|
|
19403
|
+
events,
|
|
19404
|
+
deliveredRows,
|
|
19405
|
+
...(freshClaimLease ? { freshClaimLease } : {}),
|
|
19406
|
+
...(rows.some((row) => row.row_kind === 'transport_unconfirmed')
|
|
19407
|
+
? { transportOwnershipUnconfirmed: true }
|
|
19408
|
+
: {}),
|
|
19409
|
+
...(transportRows.length > 0
|
|
19410
|
+
? {
|
|
19411
|
+
transportDelivered: {
|
|
19412
|
+
eventIds: transportRows.map((row) => row.event_id),
|
|
19413
|
+
deliveries: transportRows.map((row) => ({
|
|
19414
|
+
eventType: row.event_type,
|
|
19415
|
+
lagMs: Math.max(0, Number(row.lag_ms) || 0),
|
|
19416
|
+
})),
|
|
19417
|
+
},
|
|
19418
|
+
}
|
|
19419
|
+
: {}),
|
|
19420
|
+
};
|
|
19421
|
+
}
|
|
19422
|
+
|
|
19423
|
+
/**
|
|
19424
|
+
* Read the exact next moment at which durable outbox work can become claimable.
|
|
19425
|
+
*
|
|
19426
|
+
* A live lease is not eligible until it expires; an unleased row is eligible at
|
|
19427
|
+
* its retry time. The recovery controller uses this to sleep until real work
|
|
19428
|
+
* exists instead of having every runtime worker poll the whole outbox.
|
|
19429
|
+
*/
|
|
19430
|
+
export async function readPostgresSchedulerOutboxNextDueAt(
|
|
19431
|
+
client: PostgresSchedulerQueryClient,
|
|
19432
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'> & {
|
|
19433
|
+
lane?: 'lifecycle' | 'observability';
|
|
19434
|
+
/** Read one notified run's clock instead of enumerating every pending run. */
|
|
19435
|
+
runId?: string;
|
|
19436
|
+
},
|
|
19437
|
+
): Promise<Date | null> {
|
|
19438
|
+
const lanes =
|
|
19439
|
+
options?.lane === 'lifecycle'
|
|
19440
|
+
? [false]
|
|
19441
|
+
: options?.lane === 'observability'
|
|
19442
|
+
? [true]
|
|
19443
|
+
: [false, true];
|
|
19444
|
+
const runId = options?.runId?.trim() || null;
|
|
19445
|
+
const result = await client.query<{ due_at: Date | string | null }>(
|
|
19446
|
+
`
|
|
19447
|
+
-- Seek distinct run identities without reading their pending fact tails.
|
|
19448
|
+
WITH RECURSIVE ${
|
|
18873
19449
|
runId
|
|
18874
19450
|
? 'pending_runs AS (SELECT $1::text AS run_id)'
|
|
18875
19451
|
: runLedgerPendingRunsSql(
|
|
@@ -19414,11 +19990,17 @@ export async function markPostgresSchedulerOutboxDelivered(
|
|
|
19414
19990
|
).length;
|
|
19415
19991
|
}
|
|
19416
19992
|
|
|
19417
|
-
/**
|
|
19993
|
+
/**
|
|
19994
|
+
* Shared lease-fenced ACK, including the canonical retained-payload cleanup.
|
|
19995
|
+
* Known-run callers keep that scope visible to the planner: a materialized ID
|
|
19996
|
+
* array alone can make stale pending-index statistics favor a cross-run scan.
|
|
19997
|
+
*/
|
|
19418
19998
|
function runLedgerAcknowledgeSql(
|
|
19419
19999
|
options: Pick<PostgresSchedulerOptions, 'schema'> | undefined,
|
|
19420
20000
|
eventIdsParameter: string,
|
|
19421
20001
|
leaseOwnerParameter: string,
|
|
20002
|
+
lockFence?: string,
|
|
20003
|
+
runIdParameter?: string,
|
|
19422
20004
|
): string {
|
|
19423
20005
|
return `UPDATE ${tableName('outbox', options)} AS outbox
|
|
19424
20006
|
SET delivered_at = now(),
|
|
@@ -19432,6 +20014,8 @@ function runLedgerAcknowledgeSql(
|
|
|
19432
20014
|
AND run.run_event_store = 'planet_scale'
|
|
19433
20015
|
) THEN '{}'::jsonb ELSE outbox.payload_json END
|
|
19434
20016
|
WHERE outbox.event_id = ANY(${eventIdsParameter}::text[])
|
|
20017
|
+
${runIdParameter ? `AND outbox.run_id = ${runIdParameter}::text` : ''}
|
|
20018
|
+
${lockFence ? `AND (${lockFence})` : ''}
|
|
19435
20019
|
AND outbox.delivered_at IS NULL
|
|
19436
20020
|
AND (${leaseOwnerParameter}::text IS NULL OR outbox.lease_owner = ${leaseOwnerParameter})
|
|
19437
20021
|
RETURNING outbox.event_id,
|
|
@@ -20373,6 +20957,18 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20373
20957
|
ELSE ${runLedgerDispatchLaunchSql('runs')}
|
|
20374
20958
|
END`
|
|
20375
20959
|
: 'NULL::jsonb';
|
|
20960
|
+
const transportHeadSelection =
|
|
20961
|
+
tokenAuthority === 'gateway' &&
|
|
20962
|
+
!input.searchOnly &&
|
|
20963
|
+
!input.headEventId &&
|
|
20964
|
+
(input.limit ?? 1) >= 2
|
|
20965
|
+
? `(SELECT CASE WHEN ${runLedgerZeroEffectTransportSql('transport')} THEN
|
|
20966
|
+
jsonb_build_object('eventId', transport.event_id, 'eventType', transport.event_type, 'deliveryKey', transport.delivery_key)
|
|
20967
|
+
ELSE NULL::jsonb END
|
|
20968
|
+
FROM ${tableName('outbox', options)} AS transport
|
|
20969
|
+
WHERE transport.run_id = runs.run_id AND transport.delivered_at IS NULL AND NOT ${searchLaneSql('transport')}
|
|
20970
|
+
ORDER BY ${runLedgerOrderSql('transport')} LIMIT 1)`
|
|
20971
|
+
: 'NULL::jsonb';
|
|
20376
20972
|
const continuation: { current: RunLedgerOutboxDrainScope | null } = {
|
|
20377
20973
|
current: scope ?? null,
|
|
20378
20974
|
};
|
|
@@ -20412,88 +21008,25 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20412
21008
|
// An abandoned attempt that already handed its lease back on a checkout it
|
|
20413
21009
|
// held does not spend another checkout on the same release.
|
|
20414
21010
|
const abandonedLease = { released: false };
|
|
20415
|
-
const
|
|
20416
|
-
|
|
20417
|
-
|
|
20418
|
-
|
|
20419
|
-
|
|
20420
|
-
|
|
20421
|
-
|
|
20422
|
-
|
|
20423
|
-
|
|
20424
|
-
|
|
20425
|
-
|
|
20426
|
-
|
|
20427
|
-
|
|
20428
|
-
|
|
20429
|
-
|
|
20430
|
-
|
|
20431
|
-
|
|
20432
|
-
|
|
20433
|
-
|
|
20434
|
-
client,
|
|
20435
|
-
[abandonedHeadEventId],
|
|
20436
|
-
{ ...options, leaseOwner },
|
|
20437
|
-
);
|
|
20438
|
-
}
|
|
20439
|
-
const unclaimed: {
|
|
20440
|
-
events: PostgresSchedulerOutboxEvent[];
|
|
20441
|
-
launchJson: unknown;
|
|
20442
|
-
ingressPrefixClaimed: boolean;
|
|
20443
|
-
} = { events: [], launchJson: undefined, ingressPrefixClaimed: false };
|
|
20444
|
-
return unclaimed;
|
|
20445
|
-
}
|
|
20446
|
-
// A re-invocation with no named head claims again under the same owner,
|
|
20447
|
-
// as it always has; the abort check after the claim releases by id.
|
|
20448
|
-
// The checkout is exclusive. Hold the existing scheduler run's row lock
|
|
20449
|
-
// only while choosing/claiming its outbox prefix, never over HTTP. A
|
|
20450
|
-
// second statement takes a fresh READ COMMITTED snapshot after the lock:
|
|
20451
|
-
// late producer commits cannot split one run between different heads.
|
|
20452
|
-
await client.query('BEGIN');
|
|
20453
|
-
try {
|
|
20454
|
-
const targetRunId = input.runId?.trim() || null;
|
|
20455
|
-
let lockedRun: {
|
|
20456
|
-
rows: { run_id: string; dispatch_launch_json: unknown }[];
|
|
20457
|
-
} | null = null;
|
|
20458
|
-
if (input.waitForRunLock && targetRunId) {
|
|
20459
|
-
// A synchronous drain must not mistake a busy run for an empty
|
|
20460
|
-
// outbox. In particular, cancellation owns its pending prefix lease,
|
|
20461
|
-
// so returning early also prevents background recovery from helping.
|
|
20462
|
-
// Wait only for this run, under the existing cancellation SQL budget,
|
|
20463
|
-
// then select the prefix using a fresh READ COMMITTED snapshot. The
|
|
20464
|
-
// transaction still ends before HTTP; background claims keep SKIP LOCKED.
|
|
20465
|
-
await client.query(
|
|
20466
|
-
`SELECT set_config('lock_timeout', $1, true),
|
|
20467
|
-
set_config('statement_timeout', $2, true)`,
|
|
20468
|
-
[
|
|
20469
|
-
`${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
|
|
20470
|
-
`${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
|
|
20471
|
-
],
|
|
20472
|
-
);
|
|
20473
|
-
lockedRun = await client.query<{
|
|
20474
|
-
run_id: string;
|
|
20475
|
-
dispatch_launch_json: unknown;
|
|
20476
|
-
}>(
|
|
20477
|
-
`SELECT runs.run_id,
|
|
20478
|
-
${needsLaunch ? runLedgerDispatchLaunchSql('runs') : 'NULL::jsonb'} AS dispatch_launch_json
|
|
20479
|
-
FROM ${tableName('runs', options)} AS runs
|
|
20480
|
-
WHERE runs.run_id = $1 FOR UPDATE`,
|
|
20481
|
-
[targetRunId],
|
|
20482
|
-
);
|
|
20483
|
-
}
|
|
20484
|
-
// A worker fast path always names one run. Keep that lookup on the
|
|
20485
|
-
// run-scoped outbox prefix: combining it with global recovery through a
|
|
20486
|
-
// nullable predicate made PostgreSQL materialize every pending run head
|
|
20487
|
-
// for each lifecycle transition. Only the two dedicated projectors may
|
|
20488
|
-
// perform the global indexed head discovery below.
|
|
20489
|
-
const selected =
|
|
20490
|
-
(input.headEventId ? null : lockedRun) ??
|
|
20491
|
-
(targetRunId
|
|
20492
|
-
? await client.query<{
|
|
20493
|
-
run_id: string;
|
|
20494
|
-
dispatch_launch_json: unknown;
|
|
20495
|
-
}>(
|
|
20496
|
-
`
|
|
21011
|
+
const selectRunForClaim = async (
|
|
21012
|
+
client: PostgresSchedulerQueryClient,
|
|
21013
|
+
targetRunId: string | null,
|
|
21014
|
+
lockedRun: {
|
|
21015
|
+
rows: {
|
|
21016
|
+
run_id: string;
|
|
21017
|
+
dispatch_launch_json: unknown;
|
|
21018
|
+
transport_head?: RunLedgerTransportCandidate | null;
|
|
21019
|
+
}[];
|
|
21020
|
+
} | null = null,
|
|
21021
|
+
) => {
|
|
21022
|
+
return await ((input.headEventId ? null : lockedRun) ??
|
|
21023
|
+
(targetRunId
|
|
21024
|
+
? await client.query<{
|
|
21025
|
+
run_id: string;
|
|
21026
|
+
dispatch_launch_json: unknown;
|
|
21027
|
+
transport_head?: RunLedgerTransportCandidate | null;
|
|
21028
|
+
}>(
|
|
21029
|
+
`
|
|
20497
21030
|
WITH head AS MATERIALIZED (
|
|
20498
21031
|
SELECT pending.run_id,
|
|
20499
21032
|
pending.event_id,
|
|
@@ -20509,7 +21042,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20509
21042
|
ORDER BY ${runLedgerOrderSql('pending')}
|
|
20510
21043
|
LIMIT 1
|
|
20511
21044
|
)
|
|
20512
|
-
SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json
|
|
21045
|
+
SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json, ${transportHeadSelection} AS transport_head
|
|
20513
21046
|
FROM head
|
|
20514
21047
|
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
|
|
20515
21048
|
WHERE head.failed_at IS NULL
|
|
@@ -20526,12 +21059,13 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20526
21059
|
)
|
|
20527
21060
|
FOR UPDATE OF runs SKIP LOCKED
|
|
20528
21061
|
`,
|
|
20529
|
-
|
|
20530
|
-
|
|
20531
|
-
|
|
20532
|
-
|
|
20533
|
-
|
|
20534
|
-
|
|
21062
|
+
[targetRunId, leaseOwner, input.headEventId?.trim() || null],
|
|
21063
|
+
)
|
|
21064
|
+
: await client.query<{
|
|
21065
|
+
run_id: string;
|
|
21066
|
+
dispatch_launch_json: unknown;
|
|
21067
|
+
transport_head?: RunLedgerTransportCandidate | null;
|
|
21068
|
+
}>(`
|
|
20535
21069
|
WITH RECURSIVE ${runLedgerPendingRunsSql(options, input.searchOnly === true)}, heads AS MATERIALIZED (
|
|
20536
21070
|
SELECT pending_run.run_id, head.*
|
|
20537
21071
|
FROM pending_runs AS pending_run
|
|
@@ -20550,7 +21084,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20550
21084
|
LIMIT 1
|
|
20551
21085
|
) AS head
|
|
20552
21086
|
)
|
|
20553
|
-
SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json
|
|
21087
|
+
SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json, ${transportHeadSelection} AS transport_head
|
|
20554
21088
|
FROM heads AS head
|
|
20555
21089
|
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
|
|
20556
21090
|
WHERE head.failed_at IS NULL
|
|
@@ -20579,29 +21113,192 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20579
21113
|
}
|
|
20580
21114
|
head.created_at, runs.run_id
|
|
20581
21115
|
FOR UPDATE OF runs SKIP LOCKED LIMIT 1
|
|
20582
|
-
`));
|
|
20583
|
-
|
|
20584
|
-
|
|
20585
|
-
|
|
20586
|
-
|
|
20587
|
-
|
|
20588
|
-
|
|
20589
|
-
|
|
20590
|
-
|
|
20591
|
-
|
|
20592
|
-
|
|
20593
|
-
|
|
20594
|
-
|
|
20595
|
-
|
|
20596
|
-
|
|
20597
|
-
|
|
20598
|
-
|
|
20599
|
-
|
|
20600
|
-
|
|
20601
|
-
|
|
21116
|
+
`)));
|
|
21117
|
+
};
|
|
21118
|
+
const ingressPrefixSupported = (selected: {
|
|
21119
|
+
rows: {
|
|
21120
|
+
run_id: string;
|
|
21121
|
+
dispatch_launch_json: unknown;
|
|
21122
|
+
transport_head?: RunLedgerTransportCandidate | null;
|
|
21123
|
+
}[];
|
|
21124
|
+
}) => {
|
|
21125
|
+
const runId = selected.rows[0]?.run_id;
|
|
21126
|
+
if (
|
|
21127
|
+
runId &&
|
|
21128
|
+
tokenAuthority === 'gateway' &&
|
|
21129
|
+
!input.searchOnly &&
|
|
21130
|
+
(input.limit ?? 1) === ingressPrefixLimit
|
|
21131
|
+
) {
|
|
21132
|
+
try {
|
|
21133
|
+
const launch = parseRunLedgerDispatchLaunch(
|
|
21134
|
+
selected.rows[0]?.dispatch_launch_json,
|
|
21135
|
+
);
|
|
21136
|
+
return (
|
|
21137
|
+
launch.runId === runId &&
|
|
21138
|
+
usableRunLedgerExecutorToken(launch, reuseMarginMs, options)
|
|
21139
|
+
?.runLedgerIngressPrefixBatching === true
|
|
21140
|
+
);
|
|
21141
|
+
} catch {
|
|
21142
|
+
// Malformed launches retain the existing post-claim failure policy.
|
|
20602
21143
|
}
|
|
21144
|
+
}
|
|
21145
|
+
return false;
|
|
21146
|
+
};
|
|
21147
|
+
// Retain a chosen run across a checkout's connection retry: an ambiguous
|
|
21148
|
+
// committed claim must be reread under the same owner rather than abandoned
|
|
21149
|
+
// by a global scan that now sees its own active lease.
|
|
21150
|
+
let selectedFastRun: {
|
|
21151
|
+
rows: {
|
|
21152
|
+
run_id: string;
|
|
21153
|
+
dispatch_launch_json: unknown;
|
|
21154
|
+
transport_head?: RunLedgerTransportCandidate | null;
|
|
21155
|
+
}[];
|
|
21156
|
+
} | null = null;
|
|
21157
|
+
const claimFastEvents = async (client: PostgresSchedulerQueryClient) => {
|
|
21158
|
+
if (
|
|
21159
|
+
runLedgerOutboxBatchUnsupportedClients.has(client) ||
|
|
21160
|
+
!supportsOutboxDeliveryBatchMarker(options) ||
|
|
21161
|
+
input.signal?.aborted
|
|
21162
|
+
)
|
|
21163
|
+
return null;
|
|
21164
|
+
const targetRunId = input.runId?.trim() || null;
|
|
21165
|
+
const selected =
|
|
21166
|
+
selectedFastRun ??
|
|
21167
|
+
(input.waitForRunLock && targetRunId
|
|
21168
|
+
? await client.query<{
|
|
21169
|
+
run_id: string;
|
|
21170
|
+
dispatch_launch_json: unknown;
|
|
21171
|
+
transport_head?: RunLedgerTransportCandidate | null;
|
|
21172
|
+
}>(
|
|
21173
|
+
`SELECT runs.run_id, ${needsLaunch ? runLedgerDispatchLaunchSql('runs') : 'NULL::jsonb'} AS dispatch_launch_json, ${transportHeadSelection} AS transport_head
|
|
21174
|
+
FROM ${tableName('runs', options)} AS runs WHERE runs.run_id = $1`,
|
|
21175
|
+
[targetRunId],
|
|
21176
|
+
)
|
|
21177
|
+
: await selectRunForClaim(client, targetRunId));
|
|
21178
|
+
const runId = selected.rows[0]?.run_id;
|
|
21179
|
+
if (!runId)
|
|
21180
|
+
return {
|
|
21181
|
+
events: [],
|
|
21182
|
+
launchJson: undefined,
|
|
21183
|
+
ingressPrefixClaimed: false,
|
|
21184
|
+
freshClaimLease: undefined,
|
|
21185
|
+
transportDelivered: undefined,
|
|
21186
|
+
transportOwnershipUnconfirmed: undefined,
|
|
21187
|
+
};
|
|
21188
|
+
selectedFastRun = selected;
|
|
21189
|
+
const ingressPrefixClaimed = ingressPrefixSupported(selected);
|
|
21190
|
+
claimInvocations += 1;
|
|
21191
|
+
const result = await claimRunLedgerOutboxPrefixInBatch(
|
|
21192
|
+
client,
|
|
21193
|
+
{
|
|
21194
|
+
runId,
|
|
21195
|
+
leaseOwner,
|
|
21196
|
+
leaseSeconds: input.leaseSeconds ?? 120,
|
|
21197
|
+
limit: input.limit ?? 1,
|
|
21198
|
+
searchOnly: input.searchOnly,
|
|
21199
|
+
allowIngressPrefix: ingressPrefixClaimed,
|
|
21200
|
+
waitForRunLock: input.waitForRunLock === true,
|
|
21201
|
+
expectedHeadEventId: input.headEventId?.trim() || undefined,
|
|
21202
|
+
transportCandidate:
|
|
21203
|
+
!input.headEventId && ingressPrefixClaimed
|
|
21204
|
+
? (selected.rows[0]?.transport_head ?? undefined)
|
|
21205
|
+
: undefined,
|
|
21206
|
+
recoverTransportDelivery: claimInvocations > 1,
|
|
21207
|
+
},
|
|
21208
|
+
options,
|
|
21209
|
+
);
|
|
21210
|
+
if (result === null) return null;
|
|
21211
|
+
return {
|
|
21212
|
+
events: result.events,
|
|
21213
|
+
launchJson: selected.rows[0]?.dispatch_launch_json,
|
|
21214
|
+
ingressPrefixClaimed,
|
|
21215
|
+
freshClaimLease: result.freshClaimLease,
|
|
21216
|
+
transportDelivered: result.transportDelivered,
|
|
21217
|
+
transportOwnershipUnconfirmed: result.transportOwnershipUnconfirmed,
|
|
21218
|
+
};
|
|
21219
|
+
};
|
|
21220
|
+
const claimEvents = async (client: PostgresSchedulerQueryClient) => {
|
|
21221
|
+
claimInvocations += 1;
|
|
21222
|
+
const abandonedHeadEventId = input.headEventId?.trim();
|
|
21223
|
+
if (
|
|
21224
|
+
input.signal?.aborted &&
|
|
21225
|
+
(claimInvocations === 1 || abandonedHeadEventId)
|
|
21226
|
+
) {
|
|
21227
|
+
// The caller's deadline elapsed while this checkout was queued. Its
|
|
21228
|
+
// attempt is already abandoned, so taking a lease now could only keep
|
|
21229
|
+
// the durable projector off the row until that lease expires. Claim
|
|
21230
|
+
// nothing: the row stays pending and unleased for its next owner.
|
|
21231
|
+
if (claimInvocations > 1 && abandonedHeadEventId) {
|
|
21232
|
+
// An earlier invocation's COMMIT may have reached the server before
|
|
21233
|
+
// its connection failed, so this owner can already hold the named
|
|
21234
|
+
// head. Hand it back here, fenced to this owner; returning empty
|
|
21235
|
+
// would leave that lease with nobody to release it. After a rolled
|
|
21236
|
+
// back invocation it matches no row. A failure propagates, so the
|
|
21237
|
+
// attempt is never reported as settled.
|
|
21238
|
+
await releasePostgresSchedulerOutboxLease(
|
|
21239
|
+
client,
|
|
21240
|
+
[abandonedHeadEventId],
|
|
21241
|
+
{ ...options, leaseOwner },
|
|
21242
|
+
);
|
|
21243
|
+
}
|
|
21244
|
+
const unclaimed: {
|
|
21245
|
+
events: PostgresSchedulerOutboxEvent[];
|
|
21246
|
+
launchJson: unknown;
|
|
21247
|
+
ingressPrefixClaimed: boolean;
|
|
21248
|
+
freshClaimLease?: RunLedgerFreshClaimLeaseFacts;
|
|
21249
|
+
transportDelivered?: RunLedgerTransportDelivery;
|
|
21250
|
+
transportOwnershipUnconfirmed?: boolean;
|
|
21251
|
+
} = { events: [], launchJson: undefined, ingressPrefixClaimed: false };
|
|
21252
|
+
return unclaimed;
|
|
21253
|
+
}
|
|
21254
|
+
// A re-invocation with no named head claims again under the same owner,
|
|
21255
|
+
// as it always has; the abort check after the claim releases by id.
|
|
21256
|
+
// The checkout is exclusive. Hold the existing scheduler run's row lock
|
|
21257
|
+
// only while choosing/claiming its outbox prefix, never over HTTP. A
|
|
21258
|
+
// second statement takes a fresh READ COMMITTED snapshot after the lock:
|
|
21259
|
+
// late producer commits cannot split one run between different heads.
|
|
21260
|
+
await client.query('BEGIN');
|
|
21261
|
+
try {
|
|
21262
|
+
const targetRunId = input.runId?.trim() || null;
|
|
21263
|
+
let lockedRun: {
|
|
21264
|
+
rows: { run_id: string; dispatch_launch_json: unknown }[];
|
|
21265
|
+
} | null = null;
|
|
21266
|
+
if (input.waitForRunLock && targetRunId) {
|
|
21267
|
+
// A synchronous drain must not mistake a busy run for an empty
|
|
21268
|
+
// outbox. In particular, cancellation owns its pending prefix lease,
|
|
21269
|
+
// so returning early also prevents background recovery from helping.
|
|
21270
|
+
// Wait only for this run, under the existing cancellation SQL budget,
|
|
21271
|
+
// then select the prefix using a fresh READ COMMITTED snapshot. The
|
|
21272
|
+
// transaction still ends before HTTP; background claims keep SKIP LOCKED.
|
|
21273
|
+
await client.query(
|
|
21274
|
+
`SELECT set_config('lock_timeout', $1, true),
|
|
21275
|
+
set_config('statement_timeout', $2, true)`,
|
|
21276
|
+
[
|
|
21277
|
+
`${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
|
|
21278
|
+
`${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
|
|
21279
|
+
],
|
|
21280
|
+
);
|
|
21281
|
+
lockedRun = await client.query<{
|
|
21282
|
+
run_id: string;
|
|
21283
|
+
dispatch_launch_json: unknown;
|
|
21284
|
+
}>(
|
|
21285
|
+
`SELECT runs.run_id,
|
|
21286
|
+
${needsLaunch ? runLedgerDispatchLaunchSql('runs') : 'NULL::jsonb'} AS dispatch_launch_json
|
|
21287
|
+
FROM ${tableName('runs', options)} AS runs
|
|
21288
|
+
WHERE runs.run_id = $1 FOR UPDATE`,
|
|
21289
|
+
[targetRunId],
|
|
21290
|
+
);
|
|
21291
|
+
}
|
|
21292
|
+
// A worker fast path always names one run. Keep that lookup on the
|
|
21293
|
+
// run-scoped outbox prefix: combining it with global recovery through a
|
|
21294
|
+
// nullable predicate made PostgreSQL materialize every pending run head
|
|
21295
|
+
// for each lifecycle transition. Only the two dedicated projectors may
|
|
21296
|
+
// perform the global indexed head discovery below.
|
|
21297
|
+
const selected = await selectRunForClaim(client, targetRunId, lockedRun);
|
|
21298
|
+
const runId = selected.rows[0]?.run_id;
|
|
21299
|
+
const ingressPrefixClaimed = ingressPrefixSupported(selected);
|
|
20603
21300
|
const claimed = runId
|
|
20604
|
-
? await
|
|
21301
|
+
? await claimPostgresSchedulerOutboxPrefix(
|
|
20605
21302
|
client,
|
|
20606
21303
|
{
|
|
20607
21304
|
limit: input.limit ?? 1,
|
|
@@ -20615,16 +21312,19 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20615
21312
|
},
|
|
20616
21313
|
options,
|
|
20617
21314
|
)
|
|
20618
|
-
: [];
|
|
21315
|
+
: { events: [], freshClaimLease: undefined };
|
|
20619
21316
|
await client.query('COMMIT');
|
|
20620
21317
|
// The launch is immutable. Carry only its authority/routing fields from
|
|
20621
21318
|
// the run we already locked, rather than queuing another scarce checkout
|
|
20622
21319
|
// between the claim and gateway request. The gateway still proves the
|
|
20623
21320
|
// active outbox lease; no database checkout spans either HTTP hop.
|
|
20624
21321
|
return {
|
|
20625
|
-
events: claimed,
|
|
21322
|
+
events: claimed.events,
|
|
20626
21323
|
launchJson: selected.rows[0]?.dispatch_launch_json,
|
|
20627
21324
|
ingressPrefixClaimed,
|
|
21325
|
+
freshClaimLease: claimed.freshClaimLease,
|
|
21326
|
+
transportDelivered: undefined,
|
|
21327
|
+
transportOwnershipUnconfirmed: undefined,
|
|
20628
21328
|
};
|
|
20629
21329
|
} catch (error) {
|
|
20630
21330
|
await client.query('ROLLBACK');
|
|
@@ -20634,9 +21334,19 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20634
21334
|
const preclaimed = scope?.nextClaimed.length
|
|
20635
21335
|
? scope.nextClaimed.splice(0)
|
|
20636
21336
|
: null;
|
|
21337
|
+
const claimStartedAtMonotonicMs = performance.now();
|
|
20637
21338
|
const claimed = preclaimed
|
|
20638
|
-
? {
|
|
21339
|
+
? {
|
|
21340
|
+
events: preclaimed,
|
|
21341
|
+
launchJson: null,
|
|
21342
|
+
ingressPrefixClaimed: false,
|
|
21343
|
+
freshClaimLease: undefined,
|
|
21344
|
+
transportDelivered: undefined,
|
|
21345
|
+
transportOwnershipUnconfirmed: undefined,
|
|
21346
|
+
}
|
|
20639
21347
|
: await withClient(async (client) => {
|
|
21348
|
+
const fast = await claimFastEvents(client);
|
|
21349
|
+
if (fast !== null) return fast;
|
|
20640
21350
|
try {
|
|
20641
21351
|
return await claimEvents(client);
|
|
20642
21352
|
} catch (error) {
|
|
@@ -20646,10 +21356,58 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20646
21356
|
return claimEvents(client);
|
|
20647
21357
|
}
|
|
20648
21358
|
});
|
|
21359
|
+
const claimReceivedAtMonotonicMs = performance.now();
|
|
21360
|
+
// Debit the complete admission/claim/COMMIT round trip, even though SQL's
|
|
21361
|
+
// lease observation occurred later. This is a conservative local proof.
|
|
21362
|
+
const claimElapsedMs = claimReceivedAtMonotonicMs - claimStartedAtMonotonicMs;
|
|
21363
|
+
let freshClaimFirstHttpProof: RunLedgerFreshClaimFirstHttpProof | null =
|
|
21364
|
+
claimed.freshClaimLease &&
|
|
21365
|
+
Number.isFinite(claimElapsedMs) &&
|
|
21366
|
+
claimElapsedMs >= 0
|
|
21367
|
+
? {
|
|
21368
|
+
...claimed.freshClaimLease,
|
|
21369
|
+
remainingLeaseMs: Math.floor(
|
|
21370
|
+
claimed.freshClaimLease.remainingLeaseMs - claimElapsedMs,
|
|
21371
|
+
),
|
|
21372
|
+
receivedAtMonotonicMs: claimReceivedAtMonotonicMs,
|
|
21373
|
+
}
|
|
21374
|
+
: null;
|
|
20649
21375
|
const { launchJson } = claimed;
|
|
21376
|
+
const transportDelivered = claimed.transportDelivered;
|
|
21377
|
+
const withTransportDelivered = (
|
|
21378
|
+
result: PostgresSchedulerOutboxProjectionResult,
|
|
21379
|
+
): PostgresSchedulerOutboxProjectionResult =>
|
|
21380
|
+
transportDelivered
|
|
21381
|
+
? {
|
|
21382
|
+
...result,
|
|
21383
|
+
attempted: transportDelivered.eventIds.length + result.attempted,
|
|
21384
|
+
projected: transportDelivered.eventIds.length + result.projected,
|
|
21385
|
+
eventIds: [...transportDelivered.eventIds, ...result.eventIds],
|
|
21386
|
+
}
|
|
21387
|
+
: result;
|
|
21388
|
+
if (transportDelivered && input.onDelivered) {
|
|
21389
|
+
try {
|
|
21390
|
+
input.onDelivered(transportDelivered.deliveries);
|
|
21391
|
+
} catch (error) {
|
|
21392
|
+
console.warn('[run-ledger-outbox.delivery_observer_failed]', {
|
|
21393
|
+
error: error instanceof Error ? error.message : String(error),
|
|
21394
|
+
});
|
|
21395
|
+
}
|
|
21396
|
+
}
|
|
20650
21397
|
let events = claimed.events;
|
|
21398
|
+
const unconfirmedTransportError = claimed.transportOwnershipUnconfirmed
|
|
21399
|
+
? new RunLedgerProjectionLeaseLostError('delivery_lease_not_current')
|
|
21400
|
+
: null;
|
|
21401
|
+
if (unconfirmedTransportError)
|
|
21402
|
+
console.error('[run-ledger-outbox.batch_dispatch_failed]', {
|
|
21403
|
+
runId: input.runId ?? events[0]?.runId ?? null,
|
|
21404
|
+
code: runLedgerDeliveryFailure({ error: unconfirmedTransportError }).code,
|
|
21405
|
+
failureReason: unconfirmedTransportError.reason,
|
|
21406
|
+
error: 'Local transport ACK transaction ownership is unconfirmed.',
|
|
21407
|
+
eventIds: events.map((event) => event.eventId),
|
|
21408
|
+
});
|
|
20651
21409
|
if (events.length === 0) {
|
|
20652
|
-
return {
|
|
21410
|
+
return withTransportDelivered({
|
|
20653
21411
|
attempted: 0,
|
|
20654
21412
|
projected: 0,
|
|
20655
21413
|
eventIds: [],
|
|
@@ -20657,7 +21415,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20657
21415
|
failedEventIds: [],
|
|
20658
21416
|
retrying: 0,
|
|
20659
21417
|
retryingEventIds: [],
|
|
20660
|
-
};
|
|
21418
|
+
});
|
|
20661
21419
|
}
|
|
20662
21420
|
|
|
20663
21421
|
const claimedAtMs = Date.now();
|
|
@@ -20694,6 +21452,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20694
21452
|
) === true,
|
|
20695
21453
|
)
|
|
20696
21454
|
) {
|
|
21455
|
+
freshClaimFirstHttpProof = null;
|
|
20697
21456
|
if (!input.materializeObservability) {
|
|
20698
21457
|
// Inline admission/lifecycle/read-heal paths must never index logs.
|
|
20699
21458
|
// Release and wake the existing projector without charging a retry.
|
|
@@ -20817,11 +21576,12 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
20817
21576
|
// any authority or renewal work. The abort branch below hands the lease
|
|
20818
21577
|
// straight back without charging a projection attempt.
|
|
20819
21578
|
input.signal?.throwIfAborted();
|
|
21579
|
+
if (unconfirmedTransportError) throw unconfirmedTransportError;
|
|
20820
21580
|
// The ledger converter rejects raw deferred facts. Inline callers hand
|
|
20821
21581
|
// them off; the background owner projects them before inspecting the
|
|
20822
21582
|
// compact lifecycle payload or deciding that it needs no HTTP append.
|
|
20823
21583
|
const initialHandedOff = await materializeDeferredObservability();
|
|
20824
|
-
if (initialHandedOff) return initialHandedOff;
|
|
21584
|
+
if (initialHandedOff) return withTransportDelivered(initialHandedOff);
|
|
20825
21585
|
if (input.searchOnly) {
|
|
20826
21586
|
if (
|
|
20827
21587
|
events.some(
|
|
@@ -21005,6 +21765,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21005
21765
|
events.length === 1 &&
|
|
21006
21766
|
first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents
|
|
21007
21767
|
) {
|
|
21768
|
+
freshClaimFirstHttpProof = null;
|
|
21008
21769
|
const widen = async (client: PostgresSchedulerQueryClient) => {
|
|
21009
21770
|
await client.query('BEGIN');
|
|
21010
21771
|
try {
|
|
@@ -21045,6 +21806,21 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21045
21806
|
}
|
|
21046
21807
|
};
|
|
21047
21808
|
const widened = await withClient(async (client) => {
|
|
21809
|
+
const fast = await claimRunLedgerOutboxPrefixInBatch(
|
|
21810
|
+
client,
|
|
21811
|
+
{
|
|
21812
|
+
runId: first.runId,
|
|
21813
|
+
leaseOwner,
|
|
21814
|
+
leaseSeconds: input.leaseSeconds ?? 120,
|
|
21815
|
+
limit: ingressPrefixLimit,
|
|
21816
|
+
allowIngressPrefix: true,
|
|
21817
|
+
extendOwnedHead: first.eventId,
|
|
21818
|
+
waitForRunLock: true,
|
|
21819
|
+
lockStrength: 'no_key_update',
|
|
21820
|
+
},
|
|
21821
|
+
options,
|
|
21822
|
+
);
|
|
21823
|
+
if (fast !== null) return fast.events;
|
|
21048
21824
|
try {
|
|
21049
21825
|
return await widen(client);
|
|
21050
21826
|
} catch (error) {
|
|
@@ -21062,13 +21838,17 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21062
21838
|
// Widening can add deferred facts to the lease. Project the complete
|
|
21063
21839
|
// frozen prefix before any HTTP append, including its newly claimed rows.
|
|
21064
21840
|
const handedOff = await materializeDeferredObservability();
|
|
21065
|
-
if (handedOff) return handedOff;
|
|
21841
|
+
if (handedOff) return withTransportDelivered(handedOff);
|
|
21066
21842
|
if (ledgerEvents !== null) {
|
|
21067
21843
|
ledgerEvents = events.flatMap(
|
|
21068
21844
|
buildRunLedgerEventsFromPostgresSchedulerOutbox,
|
|
21069
21845
|
);
|
|
21070
21846
|
}
|
|
21847
|
+
let firstHttpRenewal = true;
|
|
21071
21848
|
const renewDelivery = async () => {
|
|
21849
|
+
const firstRenewal = firstHttpRenewal;
|
|
21850
|
+
firstHttpRenewal = false;
|
|
21851
|
+
let authorityRefreshed = false;
|
|
21072
21852
|
const currentAuthority = authorityScope.authority;
|
|
21073
21853
|
if (
|
|
21074
21854
|
runtimeApi &&
|
|
@@ -21080,6 +21860,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21080
21860
|
) {
|
|
21081
21861
|
// Initial delivery reuses the claim's routing fields. Renewal must
|
|
21082
21862
|
// reread authority so an origin change cannot pass under that cache.
|
|
21863
|
+
authorityRefreshed = true;
|
|
21083
21864
|
const refreshed = await loadPostgresSchedulerRuntimeApiForRun(
|
|
21084
21865
|
withClient,
|
|
21085
21866
|
first.runId,
|
|
@@ -21106,6 +21887,26 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21106
21887
|
expiresAtMs: runtimeApiTokenExpiresAtMs(runtimeApi.executorToken),
|
|
21107
21888
|
};
|
|
21108
21889
|
}
|
|
21890
|
+
if (
|
|
21891
|
+
firstRenewal &&
|
|
21892
|
+
terminalRunId === first.runId &&
|
|
21893
|
+
tokenAuthority === 'gateway' &&
|
|
21894
|
+
runtimeApi?.runLedgerIngressPrefixBatching === true &&
|
|
21895
|
+
ledgerEvents !== null &&
|
|
21896
|
+
!authorityRefreshed &&
|
|
21897
|
+
!input.signal?.aborted &&
|
|
21898
|
+
!runtimeApi.signal?.aborted &&
|
|
21899
|
+
freshRunLedgerClaimCoversFirstHttp(
|
|
21900
|
+
freshClaimFirstHttpProof,
|
|
21901
|
+
events,
|
|
21902
|
+
leaseOwner,
|
|
21903
|
+
reuseMarginMs,
|
|
21904
|
+
)
|
|
21905
|
+
) {
|
|
21906
|
+
// The claim already froze this exact receipt and supplied enough live
|
|
21907
|
+
// lease for HTTP plus SQL. The receiver still validates ownership.
|
|
21908
|
+
return;
|
|
21909
|
+
}
|
|
21109
21910
|
const renewed = await withClient(async (client) => {
|
|
21110
21911
|
const eventIds = events.map((event) => event.eventId);
|
|
21111
21912
|
if (input.signal?.aborted) {
|
|
@@ -21251,18 +22052,19 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21251
22052
|
});
|
|
21252
22053
|
}
|
|
21253
22054
|
}
|
|
21254
|
-
const delivered: PostgresSchedulerOutboxProjectionResult =
|
|
21255
|
-
|
|
21256
|
-
|
|
21257
|
-
|
|
21258
|
-
|
|
21259
|
-
|
|
21260
|
-
|
|
21261
|
-
|
|
21262
|
-
|
|
21263
|
-
|
|
21264
|
-
|
|
21265
|
-
|
|
22055
|
+
const delivered: PostgresSchedulerOutboxProjectionResult =
|
|
22056
|
+
withTransportDelivered({
|
|
22057
|
+
attempted: events.length,
|
|
22058
|
+
projected: markedDelivered,
|
|
22059
|
+
eventIds:
|
|
22060
|
+
markedDelivered === events.length
|
|
22061
|
+
? events.map((event) => event.eventId)
|
|
22062
|
+
: [],
|
|
22063
|
+
failed: 0,
|
|
22064
|
+
failedEventIds: [],
|
|
22065
|
+
retrying: 0,
|
|
22066
|
+
retryingEventIds: [],
|
|
22067
|
+
});
|
|
21266
22068
|
if (!acknowledgement.continuation || scope) {
|
|
21267
22069
|
return delivered;
|
|
21268
22070
|
}
|
|
@@ -21280,6 +22082,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21280
22082
|
await releasePendingContinuation();
|
|
21281
22083
|
if (
|
|
21282
22084
|
input.signal?.aborted ||
|
|
22085
|
+
claimed.transportOwnershipUnconfirmed ||
|
|
21283
22086
|
(error instanceof RunLedgerProjectionLeaseLostError &&
|
|
21284
22087
|
error.reason === 'head_lease_owner_mismatch')
|
|
21285
22088
|
) {
|
|
@@ -21335,7 +22138,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21335
22138
|
error.reason === 'head_lease_owner_mismatch'
|
|
21336
22139
|
? eventIds
|
|
21337
22140
|
: released;
|
|
21338
|
-
return {
|
|
22141
|
+
return withTransportDelivered({
|
|
21339
22142
|
attempted: events.length,
|
|
21340
22143
|
projected: 0,
|
|
21341
22144
|
eventIds: [],
|
|
@@ -21343,7 +22146,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21343
22146
|
failedEventIds: [],
|
|
21344
22147
|
retrying: handedOff.length,
|
|
21345
22148
|
retryingEventIds: handedOff,
|
|
21346
|
-
};
|
|
22149
|
+
});
|
|
21347
22150
|
}
|
|
21348
22151
|
if (error instanceof DeferredRuntimeProjectionInputError) {
|
|
21349
22152
|
const outcome = await withClient(
|
|
@@ -21383,7 +22186,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21383
22186
|
code: error.code,
|
|
21384
22187
|
detail: error.message,
|
|
21385
22188
|
});
|
|
21386
|
-
return {
|
|
22189
|
+
return withTransportDelivered({
|
|
21387
22190
|
attempted: events.length,
|
|
21388
22191
|
projected: 0,
|
|
21389
22192
|
eventIds: [],
|
|
@@ -21391,7 +22194,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21391
22194
|
failedEventIds: failed,
|
|
21392
22195
|
retrying: retrying.length,
|
|
21393
22196
|
retryingEventIds: retrying,
|
|
21394
|
-
};
|
|
22197
|
+
});
|
|
21395
22198
|
}
|
|
21396
22199
|
let settlement: { blocked: string[]; retrying: string[] };
|
|
21397
22200
|
try {
|
|
@@ -21456,7 +22259,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21456
22259
|
settlementError,
|
|
21457
22260
|
deliveryError: error,
|
|
21458
22261
|
});
|
|
21459
|
-
return {
|
|
22262
|
+
return withTransportDelivered({
|
|
21460
22263
|
attempted: events.length,
|
|
21461
22264
|
projected: 0,
|
|
21462
22265
|
eventIds: [],
|
|
@@ -21464,7 +22267,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21464
22267
|
failedEventIds: [],
|
|
21465
22268
|
retrying: events.length,
|
|
21466
22269
|
retryingEventIds: events.map((event) => event.eventId),
|
|
21467
|
-
};
|
|
22270
|
+
});
|
|
21468
22271
|
}
|
|
21469
22272
|
const failure = runLedgerDeliveryFailure({ error });
|
|
21470
22273
|
console.warn('[run-ledger-outbox.batch_dispatch_failed]', {
|
|
@@ -21478,7 +22281,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21478
22281
|
? { projectionMismatch: failure.projectionMismatch }
|
|
21479
22282
|
: {}),
|
|
21480
22283
|
});
|
|
21481
|
-
return {
|
|
22284
|
+
return withTransportDelivered({
|
|
21482
22285
|
attempted: events.length,
|
|
21483
22286
|
projected: 0,
|
|
21484
22287
|
eventIds: [],
|
|
@@ -21486,7 +22289,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
|
|
|
21486
22289
|
failedEventIds: settlement.blocked,
|
|
21487
22290
|
retrying: settlement.retrying.length,
|
|
21488
22291
|
retryingEventIds: settlement.retrying,
|
|
21489
|
-
};
|
|
22292
|
+
});
|
|
21490
22293
|
} finally {
|
|
21491
22294
|
const settledAtMs = Date.now();
|
|
21492
22295
|
const headAgeAtClaimMs = Math.max(
|
|
@@ -21614,6 +22417,48 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
|
|
|
21614
22417
|
!!input.scope ||
|
|
21615
22418
|
(input.input.continueThroughTerminal === true && !input.input.headEventId);
|
|
21616
22419
|
return withClient(async (client) => {
|
|
22420
|
+
if (retainContinuation) {
|
|
22421
|
+
const eventIds = input.events.map((event) => event.eventId);
|
|
22422
|
+
const claimSuccessor =
|
|
22423
|
+
input.continuationAllowance !== 0 && !eventIds.includes(targetEventId);
|
|
22424
|
+
const fast = await claimRunLedgerOutboxPrefixInBatch(
|
|
22425
|
+
client,
|
|
22426
|
+
{
|
|
22427
|
+
runId,
|
|
22428
|
+
leaseOwner: input.leaseOwner,
|
|
22429
|
+
leaseSeconds: input.input.leaseSeconds ?? 120,
|
|
22430
|
+
limit: claimSuccessor
|
|
22431
|
+
? Math.min(
|
|
22432
|
+
RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
|
|
22433
|
+
input.runtimeApi?.runLedgerIngressPrefixBatching === true
|
|
22434
|
+
? RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS
|
|
22435
|
+
: (input.input.limit ?? 1),
|
|
22436
|
+
input.continuationAllowance ??
|
|
22437
|
+
RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
|
|
22438
|
+
)
|
|
22439
|
+
: 0,
|
|
22440
|
+
allowIngressPrefix:
|
|
22441
|
+
input.runtimeApi?.runLedgerIngressPrefixBatching === true,
|
|
22442
|
+
waitForRunLock: true,
|
|
22443
|
+
},
|
|
22444
|
+
options,
|
|
22445
|
+
{ eventIds, targetEventId },
|
|
22446
|
+
);
|
|
22447
|
+
if (fast !== null) {
|
|
22448
|
+
let continuation: RunLedgerOutboxDrainScope | null = null;
|
|
22449
|
+
if (fast.events.length > 0) {
|
|
22450
|
+
continuation = input.scope ?? {
|
|
22451
|
+
runId,
|
|
22452
|
+
leaseOwner: input.leaseOwner,
|
|
22453
|
+
targetEventId,
|
|
22454
|
+
nextClaimed: [],
|
|
22455
|
+
};
|
|
22456
|
+
continuation.nextClaimed = fast.events;
|
|
22457
|
+
input.continuation.current = continuation;
|
|
22458
|
+
}
|
|
22459
|
+
return { deliveredRows: fast.deliveredRows, continuation };
|
|
22460
|
+
}
|
|
22461
|
+
}
|
|
21617
22462
|
const acknowledge = async () => {
|
|
21618
22463
|
if (retainContinuation) {
|
|
21619
22464
|
await client.query('BEGIN');
|