deepline 0.3.392 → 0.3.393

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +156 -24
  2. package/dist/bundling-sources/sdk/src/compat.ts +9 -0
  3. package/dist/bundling-sources/sdk/src/http.ts +5 -1
  4. package/dist/bundling-sources/sdk/src/index.ts +5 -0
  5. package/dist/bundling-sources/sdk/src/plays/run-request.ts +182 -6
  6. package/dist/bundling-sources/sdk/src/release.ts +4 -1
  7. package/dist/bundling-sources/sdk/src/types.ts +13 -0
  8. package/dist/bundling-sources/shared_libs/integration-contracts/request-warnings.ts +64 -0
  9. package/dist/bundling-sources/shared_libs/play-data-plane/runtime-sheet-storage-contract.ts +315 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/async-operation.ts +41 -8
  11. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +418 -44
  12. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +36 -3
  13. package/dist/bundling-sources/shared_libs/play-runtime/durable-invocation-fence.ts +36 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +11 -3
  15. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts +1 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +19 -1
  17. package/dist/bundling-sources/shared_libs/play-runtime/run-identity.ts +26 -0
  18. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +158 -10
  19. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +235 -2
  20. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +92 -58
  21. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +7 -1
  22. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +30 -7
  23. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +1161 -316
  24. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +28 -15
  25. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +24 -3
  26. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +31 -0
  27. package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +37 -4
  28. package/dist/bundling-sources/shared_libs/plays/dataset.ts +172 -79
  29. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +173 -1
  30. package/dist/cli/{auth-6JSA2ZCW.mjs → auth-5PXHA232.mjs} +4 -4
  31. package/dist/cli/auth-P3MVDTXM.js +39 -0
  32. package/dist/cli/{billing-Q5BLQHIU.js → billing-JFB2XLSO.js} +101 -74
  33. package/dist/cli/{billing-MRRO643H.mjs → billing-Z2N7ZAGM.mjs} +32 -5
  34. package/dist/cli/{chunk-CVLM3WMJ.js → chunk-2HAD745V.js} +242 -183
  35. package/dist/cli/{chunk-HGYBEAQP.mjs → chunk-2LGC4EFT.mjs} +3 -3
  36. package/dist/cli/{chunk-A3APGZFP.mjs → chunk-4PT2VA3A.mjs} +2 -2
  37. package/dist/cli/{chunk-6DBOU7UH.js → chunk-4XX6443T.js} +176 -87
  38. package/dist/cli/{chunk-2QLML6G4.mjs → chunk-5KWKZIQR.mjs} +4 -4
  39. package/dist/cli/{chunk-2G7DXFHI.mjs → chunk-5PRVMXKB.mjs} +27 -2
  40. package/dist/cli/{chunk-DSWSEHRX.mjs → chunk-6I73X254.mjs} +1 -1
  41. package/dist/cli/{chunk-ZTACINO5.mjs → chunk-7HR7BGZL.mjs} +13 -3
  42. package/dist/cli/{chunk-BB3FNYFY.js → chunk-AARX5L4B.js} +2 -2
  43. package/dist/cli/{chunk-WMS5U6YB.js → chunk-CDFH2DYL.js} +20 -19
  44. package/dist/cli/{chunk-R3O2CZ4H.mjs → chunk-FQ6VHPW4.mjs} +2 -2
  45. package/dist/cli/{chunk-MU6M3URX.mjs → chunk-HLEESTCH.mjs} +1 -1
  46. package/dist/cli/{chunk-Q6EOHNRH.mjs → chunk-JI2B4RZP.mjs} +119 -6
  47. package/dist/cli/{chunk-2Q3GEFL4.mjs → chunk-KIYU2TGL.mjs} +2 -2
  48. package/dist/cli/{chunk-Y6VOGHF2.mjs → chunk-KZZC33PY.mjs} +0 -181
  49. package/dist/cli/{chunk-UCKJQG25.js → chunk-L2SYLGRP.js} +8 -8
  50. package/dist/cli/{chunk-2GAHHSLB.mjs → chunk-LMMEAMA7.mjs} +3 -2
  51. package/dist/cli/{chunk-YAZCDBXX.mjs → chunk-LZPCDB7U.mjs} +3 -2
  52. package/dist/cli/{chunk-RV6ALZIX.js → chunk-M7Y7JYZX.js} +32 -7
  53. package/dist/cli/{chunk-QGKPTBRR.js → chunk-MS6GUPBB.js} +3 -3
  54. package/dist/cli/{chunk-4SFPVW7A.js → chunk-OXNOAD6Y.js} +26 -26
  55. package/dist/cli/{chunk-JOFPQGXW.js → chunk-PTDECJGP.js} +50 -50
  56. package/dist/cli/{chunk-5O4B6AUN.js → chunk-PWSWAN7Q.js} +25 -2
  57. package/dist/cli/{chunk-YN6YA7ZZ.mjs → chunk-Q26LUSQ2.mjs} +79 -20
  58. package/dist/cli/{chunk-3VN4FTE3.js → chunk-RDXFGPJU.js} +1 -182
  59. package/dist/cli/{chunk-7E66A24V.mjs → chunk-RKMB3N7K.mjs} +25 -2
  60. package/dist/cli/{chunk-JSLPHFP7.js → chunk-RWHWF5ZJ.js} +3 -2
  61. package/dist/cli/{chunk-3FV4GBRK.mjs → chunk-STNYIMIV.mjs} +97 -8
  62. package/dist/cli/chunk-UBKA33KY.mjs +310 -0
  63. package/dist/cli/{chunk-PAIMOGCM.js → chunk-UHNDVD3P.js} +125 -12
  64. package/dist/cli/{chunk-24VEN5PQ.mjs → chunk-UOGZUA24.mjs} +179 -17
  65. package/dist/cli/{chunk-T7OKJ2T6.js → chunk-VIQJIX2P.js} +73 -63
  66. package/dist/cli/{chunk-RIV2GS5E.js → chunk-VIWE6WRK.js} +627 -465
  67. package/dist/cli/{chunk-EY7RFC6A.js → chunk-W4WOMXR3.js} +15 -15
  68. package/dist/cli/chunk-XDVT3GOM.js +310 -0
  69. package/dist/cli/{chunk-2FQG2HU4.js → chunk-Z3OOZM5L.js} +8 -8
  70. package/dist/cli/{csv-E4QGUIGI.mjs → csv-KJWZHKCH.mjs} +3 -3
  71. package/dist/cli/{csv-26U5T24Z.js → csv-P3Z3YE7H.js} +12 -12
  72. package/dist/cli/{db-JTRJIAQW.mjs → db-NCM4CQIS.mjs} +6 -5
  73. package/dist/cli/{db-QHBFLMVL.js → db-NUZGJNQC.js} +31 -30
  74. package/dist/cli/{describe-E3DE3ZK3.mjs → describe-WT6MM2GO.mjs} +17 -16
  75. package/dist/cli/{describe-FDV2V74F.js → describe-YGSEN5WC.js} +24 -23
  76. package/dist/cli/{executions-VZYOGVQY.js → executions-PCOLHN7Q.js} +8 -7
  77. package/dist/cli/{executions-B3GTNPRE.mjs → executions-XS2IPJ4W.mjs} +6 -5
  78. package/dist/cli/{feedback-CR55FJA2.js → feedback-4MSH7TFU.js} +7 -7
  79. package/dist/cli/{feedback-4CEBR3OV.mjs → feedback-CCLTS5SV.mjs} +3 -3
  80. package/dist/cli/{git-NM2PHHZ4.mjs → git-K2DHPMPC.mjs} +4 -4
  81. package/dist/cli/git-RNM45FX4.js +23 -0
  82. package/dist/cli/index.js +96 -92
  83. package/dist/cli/index.mjs +46 -42
  84. package/dist/cli/{legacy-noops-A7MSW5ND.mjs → legacy-noops-B5S4AR3P.mjs} +4 -4
  85. package/dist/cli/{legacy-noops-ZUA4YFIH.js → legacy-noops-GPNOPFXV.js} +7 -7
  86. package/dist/cli/{monitors-NVPRQYP3.mjs → monitors-JLMAIPRG.mjs} +6 -5
  87. package/dist/cli/{monitors-ZE3TLZ4H.js → monitors-ZAWD7NSN.js} +87 -86
  88. package/dist/cli/{org-U7A6DV5D.js → org-P2KN4CES.js} +66 -65
  89. package/dist/cli/{org-KUBILNQV.mjs → org-T3JTHP4G.mjs} +6 -5
  90. package/dist/cli/{play-HRDXQ4QE.mjs → play-NK6BLRGL.mjs} +11 -10
  91. package/dist/cli/play-PDXR6JFB.js +158 -0
  92. package/dist/cli/{providers-OHIWI453.mjs → providers-3NQRA7IT.mjs} +6 -5
  93. package/dist/cli/{providers-VD6YTVQZ.js → providers-5QRZN3DT.js} +9 -8
  94. package/dist/cli/{quickstart-UBV2FONF.mjs → quickstart-PIA2JGPV.mjs} +3 -3
  95. package/dist/cli/{quickstart-ETTCWPA4.js → quickstart-R7NUZGNW.js} +5 -5
  96. package/dist/cli/run-identity-G257GQED.js +25 -0
  97. package/dist/cli/run-identity-MRNCEGV2.mjs +25 -0
  98. package/dist/cli/{search-F44IZRT7.mjs → search-AIDIIGLY.mjs} +7 -6
  99. package/dist/cli/search-QRWL7MFM.js +27 -0
  100. package/dist/cli/{secrets-CAEMRORF.js → secrets-BT3PGMWN.js} +15 -14
  101. package/dist/cli/{secrets-R6XOJZKU.mjs → secrets-OQNHL45O.mjs} +6 -5
  102. package/dist/cli/{sessions-6GC7OFTT.js → sessions-EYXWOSWK.js} +5 -5
  103. package/dist/cli/{sessions-RLN6UQS3.mjs → sessions-SF7WJPJP.mjs} +4 -4
  104. package/dist/cli/{settings-DPNVUAV2.js → settings-BA3GHBOD.js} +76 -75
  105. package/dist/cli/{settings-KIPAMRPP.mjs → settings-Y57JTFDD.mjs} +9 -8
  106. package/dist/cli/{setup-TU2F7PU5.mjs → setup-7GR5GBHA.mjs} +6 -6
  107. package/dist/cli/{setup-DTL3L5SG.js → setup-RERXYC66.js} +41 -41
  108. package/dist/cli/{skills-UV5MYJ46.mjs → skills-SJWIQAV5.mjs} +4 -4
  109. package/dist/cli/skills-V7EY75HB.js +28 -0
  110. package/dist/cli/tools-H5HLBCHH.js +41 -0
  111. package/dist/cli/{tools-QRRNDCQ7.mjs → tools-ZF3JXPNL.mjs} +10 -9
  112. package/dist/cli/update-472HA3LT.js +28 -0
  113. package/dist/cli/{update-4426TF35.mjs → update-FFFYQAOJ.mjs} +6 -6
  114. package/dist/cli/{workflow-HNLOVISB.js → workflow-SZJVQH4T.js} +17 -16
  115. package/dist/cli/{workflow-N4U5VLZ3.mjs → workflow-XVFEEGAS.mjs} +7 -6
  116. package/dist/{client-e6Qh5ZIg.d.mts → client-DU5dtHDy.d.mts} +153 -4
  117. package/dist/{client-Uxuy-xQ2.d.ts → client-mCE88hC4.d.ts} +153 -4
  118. package/dist/{compiler-manifest-CY2v8ab0.d.ts → compiler-manifest-DE4RdAiY.d.mts} +61 -1
  119. package/dist/{compiler-manifest-CY2v8ab0.d.mts → compiler-manifest-DE4RdAiY.d.ts} +61 -1
  120. package/dist/{index-B9fBZH1g.d.mts → index-BI6nWgaV.d.mts} +1 -1
  121. package/dist/{index-BZ6uzZJl.d.ts → index-Bi-f7HFT.d.ts} +1 -1
  122. package/dist/index.d.mts +5 -5
  123. package/dist/index.d.ts +5 -5
  124. package/dist/index.js +120 -29
  125. package/dist/index.mjs +111 -20
  126. package/dist/install-integrity.json +91 -86
  127. package/dist/plays/bundle-play-file.d.mts +3 -3
  128. package/dist/plays/bundle-play-file.d.ts +3 -3
  129. package/dist/plays/bundle-play-file.mjs +2 -2
  130. package/dist/plays/run-request.d.mts +2 -2
  131. package/dist/plays/run-request.d.ts +2 -2
  132. package/dist/plays/run-request.js +5 -2
  133. package/dist/plays/run-request.mjs +4 -1
  134. package/dist/release.d.mts +4 -1
  135. package/dist/release.d.ts +4 -1
  136. package/dist/release.js +2 -2
  137. package/dist/release.mjs +1 -1
  138. package/package.json +1 -1
  139. package/dist/cli/auth-67TAELQL.js +0 -39
  140. package/dist/cli/git-I5JTK6YU.js +0 -23
  141. package/dist/cli/play-A2V7YDU7.js +0 -157
  142. package/dist/cli/search-WWXMBVPH.js +0 -26
  143. package/dist/cli/skills-SWOSEZLB.js +0 -28
  144. package/dist/cli/tools-IL2YCDQ6.js +0 -40
  145. package/dist/cli/update-BERDPEYR.js +0 -28
@@ -1,3 +1,4 @@
1
+ import { escapeLiteral } from 'pg';
1
2
  import {
2
3
  readRowFailureCounts,
3
4
  type RowFailureCounts,
@@ -514,6 +515,8 @@ type PostgresSchedulerClaimExecutionOptions = Pick<
514
515
  * for a detached-runner attempt must not overwrite an authoritative pushed
515
516
  * runner terminal that won after the attempt was claimed. */
516
517
  preserveRunnerTerminal?: boolean;
518
+ /** Recheck typed-wait custody and cancellation at the actual failure write. */
519
+ protectWaitingSuspensionFailure?: boolean;
517
520
  };
518
521
 
519
522
  export type PostgresSchedulerWorkerOnceResult =
@@ -8575,6 +8578,72 @@ export async function claimPostgresSchedulerWaitingDetachedAttemptForTerminal(
8575
8578
  return claimed.rows.length > 0;
8576
8579
  }
8577
8580
 
8581
+ /**
8582
+ * Prepare only an owned, retained suspension for a failure terminal. Unlike an
8583
+ * engine wake, this never clears the user's wait or emits `run.resumed`.
8584
+ */
8585
+ export async function claimPostgresSchedulerWaitingSuspensionAttemptForFailure(
8586
+ client: PostgresSchedulerQueryClient,
8587
+ input: { runId: string; attempt: number; leaseToken: string },
8588
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
8589
+ ): Promise<'prepared' | 'fence_lost' | 'not_applicable'> {
8590
+ await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
8591
+ try {
8592
+ // A cancellation marker can commit without the custody lock. Acquire the
8593
+ // locks first, then evaluate it in a fresh statement snapshot; a guard in
8594
+ // the lock-taking statement could predate that cancellation.
8595
+ await client.query(
8596
+ `WITH ${runtimeSandboxCustodyLocksSql('run.run_id = $1', options)}
8597
+ SELECT count(*) FROM custody_attempts`,
8598
+ [input.runId],
8599
+ );
8600
+ const claimed = await client.query<{
8601
+ applicable: boolean;
8602
+ prepared: boolean;
8603
+ }>(
8604
+ `
8605
+ WITH waiting_suspension AS MATERIALIZED (
8606
+ SELECT run_id FROM ${tableName('runs', options)}
8607
+ WHERE run_id = $1 AND status = 'waiting'
8608
+ AND suspension_json ->> 'kind' <> 'detached_runner'
8609
+ ), prepared AS (
8610
+ UPDATE ${tableName('attempts', options)} AS target
8611
+ SET status = 'running', heartbeat_at = now(), updated_at = now()
8612
+ FROM ${tableName('runs', options)} AS run
8613
+ WHERE target.run_id = $1 AND target.attempt = $2
8614
+ AND target.lease_token = $3 AND target.status IN ('waiting', 'running')
8615
+ AND run.run_id = target.run_id AND run.status = 'waiting'
8616
+ AND run.suspension_json ->> 'kind' <> 'detached_runner'
8617
+ AND target.runner_result_json ->> 'status' = 'suspended'
8618
+ AND target.runner_result_json -> 'suspension' = run.suspension_json
8619
+ AND NOT EXISTS (
8620
+ SELECT 1 FROM ${tableName('attempts', options)} AS newer
8621
+ WHERE newer.run_id = target.run_id AND newer.attempt > target.attempt
8622
+ AND newer.status IN ('claimed', 'running', 'waiting', 'completed', 'failed')
8623
+ )
8624
+ AND NOT EXISTS (
8625
+ SELECT 1 FROM ${tableName('control', options)} AS cancel
8626
+ WHERE cancel.key = 'runtime_cancel:' || target.run_id
8627
+ )
8628
+ RETURNING target.run_id
8629
+ )
8630
+ SELECT EXISTS (SELECT 1 FROM waiting_suspension) AS applicable,
8631
+ EXISTS (SELECT 1 FROM prepared) AS prepared
8632
+ `,
8633
+ [input.runId, input.attempt, input.leaseToken],
8634
+ );
8635
+ await client.query('COMMIT');
8636
+ return claimed.rows[0]?.prepared
8637
+ ? 'prepared'
8638
+ : claimed.rows[0]?.applicable
8639
+ ? 'fence_lost'
8640
+ : 'not_applicable';
8641
+ } catch (error) {
8642
+ await client.query('ROLLBACK');
8643
+ throw error;
8644
+ }
8645
+ }
8646
+
8578
8647
  export type PostgresSchedulerRecoveryCircuit = {
8579
8648
  dependency: string;
8580
8649
  state: 'closed' | 'open' | 'half_open';
@@ -12960,7 +13029,7 @@ export async function readPostgresSchedulerRunnerTerminal(
12960
13029
  return result.rows[0]?.runner_result_json ?? null;
12961
13030
  }
12962
13031
 
12963
- async function readPostgresSchedulerRunnerTerminalReceivedAtMs(
13032
+ export async function readPostgresSchedulerRunnerTerminalReceivedAtMs(
12964
13033
  client: PostgresSchedulerQueryClient,
12965
13034
  input: { runId: string; attempt: number },
12966
13035
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
@@ -13630,6 +13699,8 @@ export async function failPostgresSchedulerAttempt(
13630
13699
  * after claim. Resumable `suspended` receipts may be superseded by the
13631
13700
  * hard ceiling in the same fenced statement. */
13632
13701
  preserveRunnerTerminal?: boolean;
13702
+ /** Internal typed-wait failure path; absence retains ordinary terminal semantics. */
13703
+ protectWaitingSuspensionFailure?: boolean;
13633
13704
  sandboxCrashDiagnostic?: PostgresSchedulerSandboxCrashDiagnosticRequest;
13634
13705
  /** Progress facts the ledger must apply before the terminal fact. Omitted
13635
13706
  * from the payload when empty. */
@@ -13700,8 +13771,9 @@ export async function failPostgresSchedulerAttempt(
13700
13771
  const terminalLogMetadata = terminalLogEntries.map((entry) =>
13701
13772
  normalizePlayLogMetadata(entry.metadata),
13702
13773
  );
13703
- const failed = await client.query<{ run_id: string }>(
13704
- `
13774
+ const writeFailure = () =>
13775
+ client.query<{ run_id: string }>(
13776
+ `
13705
13777
  WITH ${runtimeSandboxCustodyLocksSql('run.run_id = $1', options)}, failed AS (
13706
13778
  UPDATE ${tableName('attempts', options)}
13707
13779
  SET status = 'failed',
@@ -13759,6 +13831,27 @@ export async function failPostgresSchedulerAttempt(
13759
13831
  OR runner_result_json IS NULL
13760
13832
  OR runner_result_json ->> 'status' = 'suspended'
13761
13833
  )
13834
+ ${
13835
+ input.protectWaitingSuspensionFailure
13836
+ ? `
13837
+ AND EXISTS (
13838
+ SELECT 1 FROM custody_runs AS run
13839
+ WHERE run.run_id = $1 AND run.status = 'waiting'
13840
+ AND run.suspension_json ->> 'kind' <> 'detached_runner'
13841
+ AND attempts.runner_result_json ->> 'status' = 'suspended'
13842
+ AND attempts.runner_result_json -> 'suspension' = run.suspension_json
13843
+ )
13844
+ AND NOT EXISTS (
13845
+ SELECT 1 FROM ${tableName('control', options)} AS cancel
13846
+ WHERE cancel.key = 'runtime_cancel:' || $1
13847
+ )
13848
+ AND NOT EXISTS (
13849
+ SELECT 1 FROM ${tableName('attempts', options)} AS newer
13850
+ WHERE newer.run_id = $1 AND newer.attempt > $2
13851
+ AND newer.status IN ('claimed', 'running', 'waiting', 'completed', 'failed')
13852
+ )`
13853
+ : ''
13854
+ }
13762
13855
  RETURNING
13763
13856
  run_id,
13764
13857
  attempt,
@@ -13956,36 +14049,75 @@ export async function failPostgresSchedulerAttempt(
13956
14049
  )
13957
14050
  SELECT run_id FROM notify_terminal
13958
14051
  `,
13959
- [
13960
- input.runId,
13961
- input.attempt,
13962
- input.leaseToken,
13963
- stringifyPostgresJson({ message: input.error }),
13964
- stringifyPostgresJson(terminalResult ?? null),
13965
- postgresSchedulerNotifyChannel('terminal', options),
13966
- JSON.stringify({ runId: input.runId, status: 'failed' }),
13967
- stringifyPostgresJson(ledgerTerminalResult ?? null),
13968
- runtimeTerminalReasonFromError(input.error),
13969
- stringifyPostgresJson(input.sandboxCrashDiagnostic ?? null),
13970
- input.preserveRunnerTerminal === true,
13971
- JSON.stringify(terminalLogLines),
13972
- stringifyPostgresJson(resultSummary),
13973
- JSON.stringify(terminalLogContexts),
13974
- stringifyPostgresJson(terminalWorkCounterSegment ?? null),
13975
- stringifyPostgresJson(
13976
- sanitizeTerminalProgressEvents(input.terminalProgressEvents),
13977
- ),
13978
- JSON.stringify(terminalLogMetadata),
13979
- stringifyPostgresJson(
13980
- terminalLogIds
13981
- ? terminalLogEntries.map((entry) => entry.entryId)
13982
- : null,
13983
- ),
13984
- ],
13985
- );
14052
+ [
14053
+ input.runId,
14054
+ input.attempt,
14055
+ input.leaseToken,
14056
+ stringifyPostgresJson({ message: input.error }),
14057
+ stringifyPostgresJson(terminalResult ?? null),
14058
+ postgresSchedulerNotifyChannel('terminal', options),
14059
+ JSON.stringify({ runId: input.runId, status: 'failed' }),
14060
+ stringifyPostgresJson(ledgerTerminalResult ?? null),
14061
+ runtimeTerminalReasonFromError(input.error),
14062
+ stringifyPostgresJson(input.sandboxCrashDiagnostic ?? null),
14063
+ input.preserveRunnerTerminal === true,
14064
+ JSON.stringify(terminalLogLines),
14065
+ stringifyPostgresJson(resultSummary),
14066
+ JSON.stringify(terminalLogContexts),
14067
+ stringifyPostgresJson(terminalWorkCounterSegment ?? null),
14068
+ stringifyPostgresJson(
14069
+ sanitizeTerminalProgressEvents(input.terminalProgressEvents),
14070
+ ),
14071
+ JSON.stringify(terminalLogMetadata),
14072
+ stringifyPostgresJson(
14073
+ terminalLogIds
14074
+ ? terminalLogEntries.map((entry) => entry.entryId)
14075
+ : null,
14076
+ ),
14077
+ ],
14078
+ );
14079
+ const failed = input.protectWaitingSuspensionFailure
14080
+ ? await withWaitingSuspensionFailureCustody(
14081
+ client,
14082
+ input.runId,
14083
+ writeFailure,
14084
+ options,
14085
+ )
14086
+ : await writeFailure();
13986
14087
  return failed.rows.length > 0;
13987
14088
  }
13988
14089
 
14090
+ /** The terminal statement gets a fresh snapshot after every custody lock. */
14091
+ async function withWaitingSuspensionFailureCustody<T>(
14092
+ client: PostgresSchedulerQueryClient,
14093
+ runId: string,
14094
+ write: () => Promise<T>,
14095
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
14096
+ ): Promise<T> {
14097
+ await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
14098
+ try {
14099
+ await client.query(
14100
+ `WITH ${runtimeSandboxCustodyLocksSql('run.run_id = $1', options)}
14101
+ SELECT count(*) FROM custody_operations`,
14102
+ [runId],
14103
+ );
14104
+ const result = await write();
14105
+ await client.query('COMMIT');
14106
+ return result;
14107
+ } catch (error) {
14108
+ try {
14109
+ await client.query('ROLLBACK');
14110
+ } catch (rollbackError) {
14111
+ throw new AggregateError(
14112
+ [error, rollbackError],
14113
+ 'Typed suspension failure rollback failed.',
14114
+ { cause: error },
14115
+ );
14116
+ }
14117
+ throw error;
14118
+ }
14119
+ }
14120
+
13989
14121
  /**
13990
14122
  * Cancel a run: terminal `cancelled` status, all live attempts cancelled, and
13991
14123
  * terminal notify so observers settle immediately. An in-flight executor's
@@ -15171,6 +15303,9 @@ export async function executePostgresSchedulerClaim(
15171
15303
  ...(options.preserveRunnerTerminal
15172
15304
  ? { preserveRunnerTerminal: true }
15173
15305
  : {}),
15306
+ ...(options.protectWaitingSuspensionFailure
15307
+ ? { protectWaitingSuspensionFailure: true }
15308
+ : {}),
15174
15309
  terminalProgressEvents: [
15175
15310
  ...terminalWorkCounterProgressEventsFromRunnerResult(
15176
15311
  terminalFailureResult,
@@ -18371,24 +18506,98 @@ export async function readPendingPostgresSchedulerOutboxEvents(
18371
18506
  * successor. The receiver's ingress capability and frozen receipts still
18372
18507
  * determine which causal facts may share a prefix.
18373
18508
  */
18374
- async function claimPostgresSchedulerOutboxPrefix(
18375
- client: PostgresSchedulerQueryClient,
18509
+ type RunLedgerTransportCandidate = {
18510
+ eventId: string;
18511
+ eventType: string;
18512
+ deliveryKey: string;
18513
+ };
18514
+
18515
+ function runLedgerZeroEffectTransportSql(alias: string): string {
18516
+ return `${alias}.event_type IN (${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runWaiting}, ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runResumed})
18517
+ AND jsonb_typeof(${alias}.payload_json) = 'object'
18518
+ AND ${alias}.payload_json->'kind' = '"detached_runner"'::jsonb
18519
+ AND ${alias}.delivery_key IS NOT NULL AND ${alias}.delivery_key <> ''
18520
+ AND ${alias}.payload_json->'${RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD}' IS DISTINCT FROM 'true'::jsonb
18521
+ AND ${alias}.payload_json->'${RUNTIME_EVENT_SEARCH_ONLY_FIELD}' IS DISTINCT FROM 'true'::jsonb
18522
+ AND to_jsonb(${alias})->>'delivery_batch_started_at' IS NULL
18523
+ AND ${alias}.failed_at IS NULL
18524
+ AND coalesce(${alias}.next_attempt_at, ${alias}.created_at) <= now()
18525
+ AND ${alias}.lease_owner IS NULL AND ${alias}.lease_expires_at IS NULL`;
18526
+ }
18527
+
18528
+ type RunLedgerTransportDelivery = {
18529
+ eventIds: string[];
18530
+ deliveries: RunLedgerOutboxDeliveredRow[];
18531
+ };
18532
+
18533
+ type RunLedgerOutboxPrefixClaimRow = {
18534
+ row_kind:
18535
+ | 'claimed'
18536
+ | 'delivered'
18537
+ | 'transport_delivered'
18538
+ | 'transport_unconfirmed';
18539
+ event_id: string;
18540
+ run_id: string;
18541
+ event_type: string;
18542
+ payload_json: unknown;
18543
+ created_at: Date;
18544
+ delivery_key: string | null;
18545
+ lag_ms: number | string | null;
18546
+ lease_owner?: string | null;
18547
+ delivery_frozen?: boolean;
18548
+ lease_remaining_ms?: number | string | null;
18549
+ };
18550
+
18551
+ type RunLedgerFreshClaimLeaseFacts = {
18552
+ leaseOwner: string;
18553
+ prefix: readonly { eventId: string; deliveryKey: string }[];
18554
+ remainingLeaseMs: number;
18555
+ };
18556
+
18557
+ type RunLedgerFreshClaimFirstHttpProof = RunLedgerFreshClaimLeaseFacts & {
18558
+ receivedAtMonotonicMs: number;
18559
+ };
18560
+
18561
+ function freshRunLedgerClaimCoversFirstHttp(
18562
+ proof: RunLedgerFreshClaimFirstHttpProof | null,
18563
+ events: readonly PostgresSchedulerOutboxEvent[],
18564
+ leaseOwner: string,
18565
+ requiredRemainingMs: number,
18566
+ ): boolean {
18567
+ if (
18568
+ !proof ||
18569
+ proof.leaseOwner !== leaseOwner ||
18570
+ proof.prefix.length !== events.length
18571
+ )
18572
+ return false;
18573
+ const elapsedMs = performance.now() - proof.receivedAtMonotonicMs;
18574
+ return (
18575
+ Number.isFinite(elapsedMs) &&
18576
+ elapsedMs >= 0 &&
18577
+ proof.remainingLeaseMs - elapsedMs > requiredRemainingMs &&
18578
+ proof.prefix.every(
18579
+ (fact, index) =>
18580
+ fact.eventId === events[index]?.eventId &&
18581
+ fact.deliveryKey === events[index]?.deliveryKey &&
18582
+ events[index]?.leaseOwner === leaseOwner,
18583
+ )
18584
+ );
18585
+ }
18586
+
18587
+ /** One SQL owner for the legacy client transaction and the optional server operation. */
18588
+ function runLedgerOutboxPrefixClaimSql(
18376
18589
  input: NonNullable<
18377
18590
  Parameters<typeof readPendingPostgresSchedulerOutboxEvents>[1]
18378
18591
  >,
18379
- options?: Pick<PostgresSchedulerOptions, 'schema'>,
18380
- acknowledgement?: { eventIds: string[]; targetEventId: string },
18381
- ): Promise<{
18382
- events: PostgresSchedulerOutboxEvent[];
18383
- deliveredRows: RunLedgerOutboxDeliveredRow[];
18384
- }> {
18385
- const requestedLimit = Math.floor(input.limit ?? 1);
18386
- const limit = Number.isFinite(requestedLimit)
18387
- ? Math.max(1, requestedLimit)
18388
- : 1;
18592
+ options: Pick<PostgresSchedulerOptions, 'schema'> | undefined,
18593
+ acknowledgement: { eventIds: string[]; targetEventId: string } | undefined,
18594
+ supportsTerminalBatchMarker: boolean,
18595
+ fence?: { runId: string; expectedHeadEventId?: string },
18596
+ ): string {
18389
18597
  const runId = input.runId?.trim() || null;
18390
- const leaseOwner = input.leaseOwner?.trim() || null;
18391
- const leaseSeconds = Math.max(5, Math.min(300, input.leaseSeconds ?? 30));
18598
+ const lockFence = fence
18599
+ ? `current_setting('deepline.run_ledger_outbox_lock', true) = ${escapeLiteral(fence.runId)}`
18600
+ : undefined;
18392
18601
  const pending = (alias: string) =>
18393
18602
  `${alias}.delivered_at IS NULL${
18394
18603
  acknowledgement ? ` AND NOT (${alias}.event_id = ANY($5::text[]))` : ''
@@ -18398,34 +18607,33 @@ async function claimPostgresSchedulerOutboxPrefix(
18398
18607
  // fact makes a single run's burst quadratic. Failed/delayed heads must
18399
18608
  // remain in this seek so they still block their causal successors.
18400
18609
  const seekRunHead = runId !== null && !input.extendOwnedHead;
18401
- const claim = async (supportsTerminalBatchMarker: boolean) => {
18402
- const urgentLifecyclePendingSql = (alias: string) =>
18403
- input.searchOnly === true
18404
- ? runLedgerTerminalPendingSql(alias, options)
18405
- : runLedgerUrgentLifecyclePendingSql(
18406
- alias,
18407
- options,
18408
- // Every claimed row belongs to the selected run. An independent
18409
- // wait probe evaluates once, instead of scanning its backlog for
18410
- // each row whose delivery batch is being frozen.
18411
- runId !== null ? '$2::text' : undefined,
18412
- );
18413
- // Search is locally indexed and acknowledged; it has no remote ingress
18414
- // receipt to freeze. Preserve causal ordering/due/lease checks, but admit
18415
- // its full prefix. Lifecycle ingress prefixes require receiver support.
18416
- const ingressPrefixSql =
18417
- input.searchOnly === true
18418
- ? 'true'
18419
- : supportsTerminalBatchMarker && input.allowIngressPrefix === true
18420
- ? `(head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents})`
18421
- : 'false';
18422
- const findSearchBlockersOnce = seekRunHead && input.searchOnly === true;
18423
- const searchFactColumnsSql = (alias: string) =>
18424
- `${alias}.event_id, ${alias}.run_id, ${alias}.event_type,
18425
- ${alias}.created_at, ${alias}.failed_at, ${alias}.next_attempt_at,
18426
- ${alias}.lease_owner, ${alias}.lease_expires_at,
18427
- ${supportsTerminalBatchMarker ? `${alias}.delivery_batch_started_at` : 'NULL::timestamptz AS delivery_batch_started_at'}`;
18428
- const blockingPredecessorSql = (alias: string) => `(
18610
+ const urgentLifecyclePendingSql = (alias: string) =>
18611
+ input.searchOnly === true
18612
+ ? runLedgerTerminalPendingSql(alias, options)
18613
+ : runLedgerUrgentLifecyclePendingSql(
18614
+ alias,
18615
+ options,
18616
+ // Every claimed row belongs to the selected run. An independent
18617
+ // wait probe evaluates once, instead of scanning its backlog for
18618
+ // each row whose delivery batch is being frozen.
18619
+ runId !== null ? '$2::text' : undefined,
18620
+ );
18621
+ // Search is locally indexed and acknowledged; it has no remote ingress
18622
+ // receipt to freeze. Preserve causal ordering/due/lease checks, but admit
18623
+ // its full prefix. Lifecycle ingress prefixes require receiver support.
18624
+ const ingressPrefixSql =
18625
+ input.searchOnly === true
18626
+ ? 'true'
18627
+ : supportsTerminalBatchMarker && input.allowIngressPrefix === true
18628
+ ? `(head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents})`
18629
+ : 'false';
18630
+ const findSearchBlockersOnce = seekRunHead && input.searchOnly === true;
18631
+ const searchFactColumnsSql = (alias: string) =>
18632
+ `${alias}.event_id, ${alias}.run_id, ${alias}.event_type,
18633
+ ${alias}.created_at, ${alias}.failed_at, ${alias}.next_attempt_at,
18634
+ ${alias}.lease_owner, ${alias}.lease_expires_at,
18635
+ ${supportsTerminalBatchMarker ? `${alias}.delivery_batch_started_at` : 'NULL::timestamptz AS delivery_batch_started_at'}`;
18636
+ const blockingPredecessorSql = (alias: string) => `(
18429
18637
  (${alias}.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
18430
18638
  AND NOT (${ingressPrefixSql}))
18431
18639
  OR (${supportsTerminalBatchMarker && input.allowIngressPrefix === true ? `head.delivery_batch_started_at IS NOT NULL AND ${alias}.delivery_batch_started_at IS NULL` : 'false'})
@@ -18438,21 +18646,11 @@ async function claimPostgresSchedulerOutboxPrefix(
18438
18646
  AND ${alias}.lease_owner IS DISTINCT FROM $3::text
18439
18647
  )
18440
18648
  )`;
18441
- return await client.query<{
18442
- row_kind: 'claimed' | 'delivered';
18443
- event_id: string;
18444
- run_id: string;
18445
- event_type: string;
18446
- payload_json: unknown;
18447
- created_at: Date;
18448
- delivery_key: string | null;
18449
- lag_ms: number | string | null;
18450
- }>(
18451
- `
18649
+ return `
18452
18650
  WITH ${
18453
18651
  acknowledgement
18454
18652
  ? `delivered AS (
18455
- ${runLedgerAcknowledgeSql(options, '$5', '$3')}
18653
+ ${runLedgerAcknowledgeSql(options, '$5', '$3', lockFence, runId === null ? undefined : '$2')}
18456
18654
  ),`
18457
18655
  : ''
18458
18656
  } ${
@@ -18486,6 +18684,8 @@ async function claimPostgresSchedulerOutboxPrefix(
18486
18684
  ${seekRunHead ? 'JOIN pending_head ON pending_head.event_id = candidate.event_id' : ''}
18487
18685
  ${input.extendOwnedHead ? 'JOIN owned_run ON owned_run.run_id = candidate.run_id' : ''}
18488
18686
  WHERE ${pending('candidate')}
18687
+ ${lockFence ? `AND (${lockFence})` : ''}
18688
+ ${fence?.expectedHeadEventId ? `AND candidate.event_id = ${escapeLiteral(fence.expectedHeadEventId)}` : ''}
18489
18689
  AND ${searchLaneSql('candidate')} = ${input.searchOnly === true ? 'true' : 'false'}
18490
18690
  ${
18491
18691
  acknowledgement
@@ -18742,7 +18942,9 @@ async function claimPostgresSchedulerOutboxPrefix(
18742
18942
  WHERE outbox.event_id = candidates.event_id
18743
18943
  RETURNING outbox.event_id, outbox.run_id, outbox.event_type,
18744
18944
  outbox.payload_json, outbox.created_at, outbox.lease_owner,
18745
- outbox.delivery_key
18945
+ outbox.delivery_key,
18946
+ ${supportsTerminalBatchMarker ? 'outbox.delivery_batch_started_at IS NOT NULL' : 'false'} AS delivery_frozen,
18947
+ (extract(epoch FROM (outbox.lease_expires_at - statement_timestamp())) * 1000)::float8 AS lease_remaining_ms
18746
18948
  )
18747
18949
  ${
18748
18950
  acknowledgement
@@ -18759,7 +18961,7 @@ async function claimPostgresSchedulerOutboxPrefix(
18759
18961
  SELECT * FROM (
18760
18962
  SELECT 'claimed'::text AS row_kind,
18761
18963
  event_id, run_id, event_type, payload_json, created_at, lease_owner,
18762
- delivery_key, NULL::float8 AS lag_ms
18964
+ delivery_key, NULL::float8 AS lag_ms, delivery_frozen, lease_remaining_ms
18763
18965
  FROM claimed
18764
18966
  ${
18765
18967
  acknowledgement
@@ -18767,109 +18969,483 @@ async function claimPostgresSchedulerOutboxPrefix(
18767
18969
  SELECT 'delivered'::text AS row_kind,
18768
18970
  event_id, NULL::text AS run_id, event_type, NULL::jsonb AS payload_json,
18769
18971
  NULL::timestamptz AS created_at, NULL::text AS lease_owner,
18770
- NULL::text AS delivery_key, lag_ms
18972
+ NULL::text AS delivery_key, lag_ms, false AS delivery_frozen, NULL::float8 AS lease_remaining_ms
18771
18973
  FROM notified`
18772
18974
  : ''
18773
18975
  }
18774
18976
  ) AS result
18775
18977
  ORDER BY row_kind, ${runLedgerOrderSql('result')}
18776
- `,
18777
- [
18778
- limit,
18779
- runId,
18780
- leaseOwner,
18781
- leaseSeconds,
18782
- ...(acknowledgement
18783
- ? [
18784
- acknowledgement.eventIds,
18785
- acknowledgement.targetEventId,
18786
- postgresSchedulerNotifyChannel('terminal', options),
18787
- ]
18788
- : input.extendOwnedHead
18789
- ? [input.extendOwnedHead]
18790
- : []),
18791
- ],
18792
- );
18793
- };
18794
- let result;
18795
- const useSavepoint =
18796
- input.inTransaction && !input.restartTransactionOnMissingMarker;
18797
- if (useSavepoint) await client.query('SAVEPOINT run_ledger_claim');
18798
- if (supportsOutboxDeliveryBatchMarker(options)) {
18799
- try {
18800
- result = await claim(true);
18801
- } catch (error) {
18802
- if (!isMissingOutboxDeliveryBatchMarker(error)) throw error;
18803
- markOutboxDeliveryBatchMarkerUnavailable(options);
18804
- // The ACK caller owns the whole short transaction. Roll it back and
18805
- // repeat only this missing-column adapter, avoiding two SAVEPOINT round
18806
- // trips on every successful acknowledgement.
18807
- if (acknowledgement || input.restartTransactionOnMissingMarker)
18808
- throw error;
18809
- if (useSavepoint)
18810
- await client.query('ROLLBACK TO SAVEPOINT run_ledger_claim');
18811
- result = await claim(false);
18978
+ `;
18979
+ }
18980
+
18981
+ const runLedgerOutboxBatchUnsupportedClients = new WeakSet<object>();
18982
+
18983
+ function isUnsupportedRunLedgerOutboxBatch(error: unknown): boolean {
18984
+ return (
18985
+ !!error &&
18986
+ typeof error === 'object' &&
18987
+ 'code' in error &&
18988
+ error.code === '42601' &&
18989
+ 'message' in error &&
18990
+ error.message ===
18991
+ 'cannot insert multiple commands into a prepared statement'
18992
+ );
18993
+ }
18994
+
18995
+ /** Render only the canonical owner's typed arguments using node-postgres quoting. */
18996
+ function runLedgerOutboxPrefixClaimBatchSql(input: {
18997
+ sql: string;
18998
+ schema?: string;
18999
+ params: unknown[];
19000
+ runId: string;
19001
+ waitForRunLock: boolean;
19002
+ lockStrength?: 'no_key_update';
19003
+ transportHeadSql?: string;
19004
+ }): string {
19005
+ const encoded = escapeLiteral(JSON.stringify(input.params));
19006
+ const parameter = (index: number) => {
19007
+ const value = input.params[index - 1];
19008
+ if (Array.isArray(value)) {
19009
+ return `(SELECT ARRAY(SELECT jsonb_array_elements_text(values_json->${index - 1})) FROM run_ledger_parameters)`;
18812
19010
  }
18813
- } else {
18814
- result = await claim(false);
18815
- }
18816
- if (useSavepoint) await client.query('RELEASE SAVEPOINT run_ledger_claim');
18817
- const events = result.rows
18818
- .filter((row) => row.row_kind === 'claimed')
18819
- .map((row) => {
18820
- if (!row.delivery_key) {
18821
- throw new Error(
18822
- `Run-ledger outbox event ${row.event_id} has no delivery key.`,
18823
- );
18824
- }
18825
- return {
18826
- eventId: row.event_id,
18827
- runId: row.run_id,
18828
- eventType: row.event_type,
18829
- payload: row.payload_json,
18830
- createdAt: row.created_at,
18831
- leaseOwner: leaseOwner,
18832
- deliveryKey: row.delivery_key,
18833
- };
18834
- });
18835
- const deliveredRows = result.rows
18836
- .filter((row) => row.row_kind === 'delivered')
18837
- .map((row) => {
18838
- const lagMs = Number(row.lag_ms);
18839
- return {
18840
- eventType: row.event_type,
18841
- lagMs: Number.isFinite(lagMs) ? Math.max(0, lagMs) : 0,
18842
- };
18843
- });
18844
- return { events, deliveredRows };
19011
+ const type = typeof value === 'number' ? 'integer' : 'text';
19012
+ const encodedParameter = `(SELECT (values_json->>${index - 1})::${type} FROM run_ledger_parameters)`;
19013
+ return index === 1 && input.transportHeadSql
19014
+ ? `(${encodedParameter} - current_setting('deepline.run_ledger_transport_ack_count')::integer)`
19015
+ : encodedParameter;
19016
+ };
19017
+ // Skip quoted identifiers/strings and comments. A run or event ID may
19018
+ // legitimately contain "$1"; only actual bind tokens may be rendered.
19019
+ const claimSql = input.sql
19020
+ .replace(
19021
+ /'(?:[^']|'')*'|"(?:[^"]|"")*"|--[^\n]*|\/\*[\s\S]*?\*\/|\$(\d+)\b/g,
19022
+ (match, index: string | undefined) =>
19023
+ index ? parameter(Number(index)) : match,
19024
+ )
19025
+ .replace(
19026
+ /^\s*WITH\s/,
19027
+ `WITH run_ledger_parameters AS MATERIALIZED (SELECT ${encoded}::jsonb AS values_json), `,
19028
+ );
19029
+ return `BEGIN;
19030
+ SELECT set_config('lock_timeout', '${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms', true),
19031
+ set_config('statement_timeout', '${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms', true);
19032
+ SELECT set_config('deepline.run_ledger_outbox_lock', coalesce((
19033
+ SELECT run_id FROM ${tableName('runs', { schema: input.schema })}
19034
+ WHERE run_id = ${escapeLiteral(input.runId)} FOR ${input.lockStrength === 'no_key_update' ? 'NO KEY UPDATE' : 'UPDATE'}${input.waitForRunLock ? '' : ' SKIP LOCKED'}
19035
+ ), ''), true);
19036
+ ${input.transportHeadSql ? `SELECT set_config('deepline.run_ledger_transport_ack_count', '0', true);\n ${input.transportHeadSql};` : ''}
19037
+ ${claimSql};
19038
+ COMMIT;`;
18845
19039
  }
18846
19040
 
18847
19041
  /**
18848
- * Read the exact next moment at which durable outbox work can become claimable.
18849
- *
18850
- * A live lease is not eligible until it expires; an unleased row is eligible at
18851
- * its retry time. The recovery controller uses this to sleep until real work
18852
- * exists instead of having every runtime worker poll the whole outbox.
19042
+ * One simple-protocol request, still separate READ COMMITTED statements. The run
19043
+ * lock and the subsequent claim therefore keep a fresh snapshot after a writer
19044
+ * commits. No transaction or lease checkout is retained over an HTTP append.
18853
19045
  */
18854
- export async function readPostgresSchedulerOutboxNextDueAt(
19046
+ async function claimRunLedgerOutboxPrefixInBatch(
18855
19047
  client: PostgresSchedulerQueryClient,
18856
- options?: Pick<PostgresSchedulerOptions, 'schema'> & {
18857
- lane?: 'lifecycle' | 'observability';
18858
- /** Read one notified run's clock instead of enumerating every pending run. */
18859
- runId?: string;
19048
+ input: NonNullable<
19049
+ Parameters<typeof readPendingPostgresSchedulerOutboxEvents>[1]
19050
+ > & {
19051
+ runId: string;
19052
+ waitForRunLock: boolean;
19053
+ expectedHeadEventId?: string;
19054
+ lockStrength?: 'no_key_update';
19055
+ transportCandidate?: RunLedgerTransportCandidate;
19056
+ recoverTransportDelivery?: boolean;
18860
19057
  },
18861
- ): Promise<Date | null> {
18862
- const lanes =
18863
- options?.lane === 'lifecycle'
18864
- ? [false]
18865
- : options?.lane === 'observability'
18866
- ? [true]
18867
- : [false, true];
18868
- const runId = options?.runId?.trim() || null;
18869
- const result = await client.query<{ due_at: Date | string | null }>(
18870
- `
18871
- -- Seek distinct run identities without reading their pending fact tails.
18872
- WITH RECURSIVE ${
19058
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
19059
+ acknowledgement?: { eventIds: string[]; targetEventId: string },
19060
+ ): Promise<{
19061
+ events: PostgresSchedulerOutboxEvent[];
19062
+ deliveredRows: RunLedgerOutboxDeliveredRow[];
19063
+ freshClaimLease?: RunLedgerFreshClaimLeaseFacts;
19064
+ transportDelivered?: RunLedgerTransportDelivery;
19065
+ transportOwnershipUnconfirmed?: boolean;
19066
+ } | null> {
19067
+ if (
19068
+ runLedgerOutboxBatchUnsupportedClients.has(client) ||
19069
+ !supportsOutboxDeliveryBatchMarker(options)
19070
+ )
19071
+ return null;
19072
+ const leaseOwner = input.leaseOwner?.trim() || null;
19073
+ const requestedLimit = Math.floor(input.limit ?? 1);
19074
+ const limit = Number.isFinite(requestedLimit)
19075
+ ? Math.max(acknowledgement ? 0 : 1, requestedLimit)
19076
+ : 1;
19077
+ const params = [
19078
+ limit,
19079
+ input.runId,
19080
+ leaseOwner,
19081
+ Math.max(5, Math.min(300, input.leaseSeconds ?? 30)),
19082
+ ...(acknowledgement
19083
+ ? [
19084
+ acknowledgement.eventIds,
19085
+ acknowledgement.targetEventId,
19086
+ postgresSchedulerNotifyChannel('terminal', options),
19087
+ ]
19088
+ : input.extendOwnedHead
19089
+ ? [input.extendOwnedHead]
19090
+ : []),
19091
+ ];
19092
+ // Detached parks/resumes have no public ledger effects. Under the existing
19093
+ // run/head locks a never-leased, never-frozen carrier may be acknowledged
19094
+ // locally before the next READ COMMITTED statement claims its successor.
19095
+ // Keep the successor's existing run.events-first receipt shape unchanged.
19096
+ const transportHeadSql =
19097
+ input.transportCandidate !== undefined &&
19098
+ input.allowIngressPrefix === true &&
19099
+ !input.searchOnly &&
19100
+ !input.extendOwnedHead &&
19101
+ !input.expectedHeadEventId &&
19102
+ !acknowledgement &&
19103
+ limit >= 2
19104
+ ? `WITH transport_head AS MATERIALIZED (
19105
+ SELECT pending.* FROM ${tableName('outbox', options)} AS pending
19106
+ WHERE pending.run_id = ${escapeLiteral(input.runId)}
19107
+ AND current_setting('deepline.run_ledger_outbox_lock', true) = ${escapeLiteral(input.runId)}
19108
+ AND pending.delivered_at IS NULL
19109
+ AND NOT ${searchLaneSql('pending')}
19110
+ ORDER BY ${runLedgerOrderSql('pending')}
19111
+ LIMIT 1 FOR UPDATE
19112
+ ), eligible_transport_head AS MATERIALIZED (
19113
+ SELECT head.* FROM transport_head AS head
19114
+ WHERE current_setting('deepline.run_ledger_outbox_lock', true) = ${escapeLiteral(input.runId)}
19115
+ AND head.event_id = ${escapeLiteral(input.transportCandidate!.eventId)}
19116
+ AND head.delivery_key = ${escapeLiteral(input.transportCandidate!.deliveryKey)}
19117
+ AND ${runLedgerZeroEffectTransportSql('head')}
19118
+ AND ${runLedgerTerminalPendingSql('head', options)}
19119
+ AND EXISTS (
19120
+ SELECT 1 FROM (
19121
+ SELECT successor.* FROM ${tableName('outbox', options)} AS successor
19122
+ WHERE successor.run_id = head.run_id AND successor.delivered_at IS NULL
19123
+ AND NOT ${searchLaneSql('successor')} AND successor.event_id <> head.event_id
19124
+ ORDER BY ${runLedgerOrderSql('successor')} LIMIT 1
19125
+ ) AS successor
19126
+ WHERE successor.failed_at IS NULL
19127
+ AND coalesce(successor.next_attempt_at, successor.created_at) <= now()
19128
+ AND successor.lease_owner IS NULL AND successor.lease_expires_at IS NULL
19129
+ )
19130
+ AND NOT EXISTS (
19131
+ SELECT 1 FROM ${tableName('outbox', options)} AS active
19132
+ WHERE active.run_id = head.run_id AND active.delivered_at IS NULL
19133
+ AND NOT ${searchLaneSql('active')}
19134
+ AND active.lease_owner IS NOT NULL AND active.lease_expires_at > now()
19135
+ )
19136
+ ), delivered AS (
19137
+ ${runLedgerAcknowledgeSql(
19138
+ options,
19139
+ 'ARRAY(SELECT event_id FROM eligible_transport_head)',
19140
+ 'NULL',
19141
+ `current_setting('deepline.run_ledger_outbox_lock', true) = ${escapeLiteral(input.runId)}`,
19142
+ escapeLiteral(input.runId),
19143
+ )}
19144
+ )${
19145
+ input.recoverTransportDelivery === true
19146
+ ? `, prior_transport AS MATERIALIZED (
19147
+ SELECT prior.event_id, prior.run_id, prior.event_type, prior.delivery_key,
19148
+ (extract(epoch FROM (prior.delivered_at-prior.created_at))*1000)::float8 AS lag_ms,
19149
+ to_jsonb(prior)->>'delivery_batch_started_at' IS NULL AND EXISTS (
19150
+ SELECT 1 FROM ${tableName('outbox', options)} AS successor
19151
+ WHERE successor.run_id = prior.run_id AND successor.delivered_at IS NULL
19152
+ AND successor.lease_owner = ${escapeLiteral(leaseOwner!)}
19153
+ AND successor.lease_expires_at > clock_timestamp()
19154
+ AND successor.xmin = prior.xmin
19155
+ AND NOT ${searchLaneSql('successor')}
19156
+ ) AS owned
19157
+ FROM ${tableName('outbox', options)} AS prior
19158
+ WHERE current_setting('deepline.run_ledger_outbox_lock', true) = ${escapeLiteral(input.runId)}
19159
+ AND prior.run_id = ${escapeLiteral(input.runId)}
19160
+ AND prior.event_id = ${escapeLiteral(input.transportCandidate!.eventId)}
19161
+ AND prior.delivery_key = ${escapeLiteral(input.transportCandidate!.deliveryKey)}
19162
+ AND prior.event_type = ${escapeLiteral(input.transportCandidate!.eventType)}
19163
+ AND prior.delivered_at IS NOT NULL
19164
+ AND NOT EXISTS (SELECT 1 FROM delivered)
19165
+ )`
19166
+ : ''
19167
+ }, transport_deliveries AS MATERIALIZED (
19168
+ SELECT delivered.event_id, head.run_id, delivered.event_type, head.delivery_key, delivered.lag_ms
19169
+ FROM delivered JOIN eligible_transport_head AS head ON head.event_id = delivered.event_id
19170
+ ${
19171
+ input.recoverTransportDelivery === true
19172
+ ? `UNION ALL
19173
+ SELECT event_id,run_id,event_type,delivery_key,lag_ms FROM prior_transport WHERE owned`
19174
+ : ''
19175
+ }
19176
+ )
19177
+ SELECT 'transport_delivered'::text AS row_kind,
19178
+ delivered.event_id, delivered.run_id, delivered.event_type,
19179
+ NULL::jsonb AS payload_json, NULL::timestamptz AS created_at,
19180
+ NULL::text AS lease_owner, delivered.delivery_key, delivered.lag_ms,
19181
+ false AS delivery_frozen, NULL::float8 AS lease_remaining_ms,
19182
+ set_config('deepline.run_ledger_transport_ack_count',
19183
+ (SELECT count(*) FROM transport_deliveries)::text, true),
19184
+ pg_notify(${escapeLiteral(postgresSchedulerNotifyChannel('terminal', options))},
19185
+ json_build_object('eventId', delivered.event_id, 'kind', 'outbox_delivered')::text)::text
19186
+ FROM transport_deliveries AS delivered
19187
+ ${
19188
+ input.recoverTransportDelivery === true
19189
+ ? `UNION ALL
19190
+ SELECT 'transport_unconfirmed'::text,event_id,run_id,event_type,NULL::jsonb,NULL::timestamptz,
19191
+ NULL::text,delivery_key,lag_ms,false,NULL::float8,NULL::text,NULL::text
19192
+ FROM prior_transport WHERE NOT owned`
19193
+ : ''
19194
+ }`
19195
+ : undefined;
19196
+ let result: QueryResult<RunLedgerOutboxPrefixClaimRow>;
19197
+ try {
19198
+ // No parameters: node-postgres sends the whole transaction through the
19199
+ // simple protocol, with all values escaped by its own literal encoder.
19200
+ result = await client.query<RunLedgerOutboxPrefixClaimRow>(
19201
+ runLedgerOutboxPrefixClaimBatchSql({
19202
+ sql: runLedgerOutboxPrefixClaimSql(
19203
+ input,
19204
+ options,
19205
+ acknowledgement,
19206
+ true,
19207
+ {
19208
+ runId: input.runId,
19209
+ expectedHeadEventId: input.expectedHeadEventId,
19210
+ },
19211
+ ),
19212
+ params,
19213
+ runId: input.runId,
19214
+ waitForRunLock: input.waitForRunLock,
19215
+ lockStrength: input.lockStrength,
19216
+ schema: options?.schema,
19217
+ transportHeadSql,
19218
+ }),
19219
+ );
19220
+ } catch (error) {
19221
+ if (isUnsupportedRunLedgerOutboxBatch(error)) {
19222
+ // This exact parse rejection precedes execution of BEGIN. An extended-
19223
+ // protocol test driver keeps the existing adapter for this checkout.
19224
+ runLedgerOutboxBatchUnsupportedClients.add(client);
19225
+ return null;
19226
+ }
19227
+ try {
19228
+ await client.query('ROLLBACK');
19229
+ } catch (rollbackError) {
19230
+ console.error('[run-ledger-outbox.batch_rollback_failed]', {
19231
+ error: errorMessage(error),
19232
+ rollbackError: errorMessage(rollbackError),
19233
+ });
19234
+ throw error;
19235
+ }
19236
+ if (!isMissingOutboxDeliveryBatchMarker(error)) throw error;
19237
+ markOutboxDeliveryBatchMarkerUnavailable(options);
19238
+ return null;
19239
+ }
19240
+ // pg returns every command result; adapters may already flatten their rows.
19241
+ // SET/lock/COMMIT rows never enter the claim projection.
19242
+ const results: QueryResult<RunLedgerOutboxPrefixClaimRow>[] = Array.isArray(
19243
+ result,
19244
+ )
19245
+ ? result
19246
+ : [result];
19247
+ return projectRunLedgerOutboxPrefixRows(
19248
+ results
19249
+ .flatMap((entry) => entry.rows)
19250
+ .filter(
19251
+ (row) =>
19252
+ row.row_kind === 'claimed' ||
19253
+ row.row_kind === 'delivered' ||
19254
+ row.row_kind === 'transport_delivered' ||
19255
+ row.row_kind === 'transport_unconfirmed',
19256
+ ),
19257
+ leaseOwner,
19258
+ );
19259
+ }
19260
+
19261
+ async function claimPostgresSchedulerOutboxPrefix(
19262
+ client: PostgresSchedulerQueryClient,
19263
+ input: NonNullable<
19264
+ Parameters<typeof readPendingPostgresSchedulerOutboxEvents>[1]
19265
+ >,
19266
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
19267
+ acknowledgement?: { eventIds: string[]; targetEventId: string },
19268
+ ): Promise<{
19269
+ events: PostgresSchedulerOutboxEvent[];
19270
+ deliveredRows: RunLedgerOutboxDeliveredRow[];
19271
+ freshClaimLease?: RunLedgerFreshClaimLeaseFacts;
19272
+ }> {
19273
+ const requestedLimit = Math.floor(input.limit ?? 1);
19274
+ const limit = Number.isFinite(requestedLimit)
19275
+ ? Math.max(1, requestedLimit)
19276
+ : 1;
19277
+ const runId = input.runId?.trim() || null;
19278
+ const leaseOwner = input.leaseOwner?.trim() || null;
19279
+ const leaseSeconds = Math.max(5, Math.min(300, input.leaseSeconds ?? 30));
19280
+ const claim = async (supportsTerminalBatchMarker: boolean) => {
19281
+ return await client.query<RunLedgerOutboxPrefixClaimRow>(
19282
+ runLedgerOutboxPrefixClaimSql(
19283
+ input,
19284
+ options,
19285
+ acknowledgement,
19286
+ supportsTerminalBatchMarker,
19287
+ ),
19288
+ [
19289
+ limit,
19290
+ runId,
19291
+ leaseOwner,
19292
+ leaseSeconds,
19293
+ ...(acknowledgement
19294
+ ? [
19295
+ acknowledgement.eventIds,
19296
+ acknowledgement.targetEventId,
19297
+ postgresSchedulerNotifyChannel('terminal', options),
19298
+ ]
19299
+ : input.extendOwnedHead
19300
+ ? [input.extendOwnedHead]
19301
+ : []),
19302
+ ],
19303
+ );
19304
+ };
19305
+ let result;
19306
+ const useSavepoint =
19307
+ input.inTransaction && !input.restartTransactionOnMissingMarker;
19308
+ if (useSavepoint) await client.query('SAVEPOINT run_ledger_claim');
19309
+ if (supportsOutboxDeliveryBatchMarker(options)) {
19310
+ try {
19311
+ result = await claim(true);
19312
+ } catch (error) {
19313
+ if (!isMissingOutboxDeliveryBatchMarker(error)) throw error;
19314
+ markOutboxDeliveryBatchMarkerUnavailable(options);
19315
+ // The ACK caller owns the whole short transaction. Roll it back and
19316
+ // repeat only this missing-column adapter, avoiding two SAVEPOINT round
19317
+ // trips on every successful acknowledgement.
19318
+ if (acknowledgement || input.restartTransactionOnMissingMarker)
19319
+ throw error;
19320
+ if (useSavepoint)
19321
+ await client.query('ROLLBACK TO SAVEPOINT run_ledger_claim');
19322
+ result = await claim(false);
19323
+ }
19324
+ } else {
19325
+ result = await claim(false);
19326
+ }
19327
+ if (useSavepoint) await client.query('RELEASE SAVEPOINT run_ledger_claim');
19328
+ return projectRunLedgerOutboxPrefixRows(result.rows, leaseOwner);
19329
+ }
19330
+
19331
+ function projectRunLedgerOutboxPrefixRows(
19332
+ rows: RunLedgerOutboxPrefixClaimRow[],
19333
+ leaseOwner: string | null,
19334
+ ): {
19335
+ events: PostgresSchedulerOutboxEvent[];
19336
+ deliveredRows: RunLedgerOutboxDeliveredRow[];
19337
+ freshClaimLease?: RunLedgerFreshClaimLeaseFacts;
19338
+ transportDelivered?: RunLedgerTransportDelivery;
19339
+ transportOwnershipUnconfirmed?: boolean;
19340
+ } {
19341
+ const events = rows
19342
+ .filter((row) => row.row_kind === 'claimed')
19343
+ .map((row) => {
19344
+ if (!row.delivery_key) {
19345
+ throw new Error(
19346
+ `Run-ledger outbox event ${row.event_id} has no delivery key.`,
19347
+ );
19348
+ }
19349
+ return {
19350
+ eventId: row.event_id,
19351
+ runId: row.run_id,
19352
+ eventType: row.event_type,
19353
+ payload: row.payload_json,
19354
+ createdAt: row.created_at,
19355
+ leaseOwner: leaseOwner,
19356
+ deliveryKey: row.delivery_key,
19357
+ };
19358
+ });
19359
+ const deliveredRows = rows
19360
+ .filter((row) => row.row_kind === 'delivered')
19361
+ .map((row) => {
19362
+ const lagMs = Number(row.lag_ms);
19363
+ return {
19364
+ eventType: row.event_type,
19365
+ lagMs: Number.isFinite(lagMs) ? Math.max(0, lagMs) : 0,
19366
+ };
19367
+ });
19368
+ const claimedRows = rows.filter((row) => row.row_kind === 'claimed');
19369
+ const remainingLeases = claimedRows.map((row) =>
19370
+ typeof row.lease_remaining_ms === 'number' ||
19371
+ (typeof row.lease_remaining_ms === 'string' &&
19372
+ row.lease_remaining_ms.trim())
19373
+ ? Number(row.lease_remaining_ms)
19374
+ : NaN,
19375
+ );
19376
+ const freshClaimLease =
19377
+ leaseOwner &&
19378
+ claimedRows.length > 0 &&
19379
+ claimedRows.every(
19380
+ (row) => row.lease_owner === leaseOwner && row.delivery_frozen === true,
19381
+ ) &&
19382
+ remainingLeases.every(
19383
+ (remaining) => Number.isFinite(remaining) && remaining > 0,
19384
+ )
19385
+ ? {
19386
+ leaseOwner,
19387
+ prefix: events.map(({ eventId, deliveryKey }) => ({
19388
+ eventId,
19389
+ deliveryKey,
19390
+ })),
19391
+ remainingLeaseMs: Math.floor(
19392
+ remainingLeases.reduce(
19393
+ (minimum, remaining) => Math.min(minimum, remaining),
19394
+ Infinity,
19395
+ ),
19396
+ ),
19397
+ }
19398
+ : undefined;
19399
+ const transportRows = rows.filter(
19400
+ (row) => row.row_kind === 'transport_delivered',
19401
+ );
19402
+ return {
19403
+ events,
19404
+ deliveredRows,
19405
+ ...(freshClaimLease ? { freshClaimLease } : {}),
19406
+ ...(rows.some((row) => row.row_kind === 'transport_unconfirmed')
19407
+ ? { transportOwnershipUnconfirmed: true }
19408
+ : {}),
19409
+ ...(transportRows.length > 0
19410
+ ? {
19411
+ transportDelivered: {
19412
+ eventIds: transportRows.map((row) => row.event_id),
19413
+ deliveries: transportRows.map((row) => ({
19414
+ eventType: row.event_type,
19415
+ lagMs: Math.max(0, Number(row.lag_ms) || 0),
19416
+ })),
19417
+ },
19418
+ }
19419
+ : {}),
19420
+ };
19421
+ }
19422
+
19423
+ /**
19424
+ * Read the exact next moment at which durable outbox work can become claimable.
19425
+ *
19426
+ * A live lease is not eligible until it expires; an unleased row is eligible at
19427
+ * its retry time. The recovery controller uses this to sleep until real work
19428
+ * exists instead of having every runtime worker poll the whole outbox.
19429
+ */
19430
+ export async function readPostgresSchedulerOutboxNextDueAt(
19431
+ client: PostgresSchedulerQueryClient,
19432
+ options?: Pick<PostgresSchedulerOptions, 'schema'> & {
19433
+ lane?: 'lifecycle' | 'observability';
19434
+ /** Read one notified run's clock instead of enumerating every pending run. */
19435
+ runId?: string;
19436
+ },
19437
+ ): Promise<Date | null> {
19438
+ const lanes =
19439
+ options?.lane === 'lifecycle'
19440
+ ? [false]
19441
+ : options?.lane === 'observability'
19442
+ ? [true]
19443
+ : [false, true];
19444
+ const runId = options?.runId?.trim() || null;
19445
+ const result = await client.query<{ due_at: Date | string | null }>(
19446
+ `
19447
+ -- Seek distinct run identities without reading their pending fact tails.
19448
+ WITH RECURSIVE ${
18873
19449
  runId
18874
19450
  ? 'pending_runs AS (SELECT $1::text AS run_id)'
18875
19451
  : runLedgerPendingRunsSql(
@@ -19414,11 +19990,17 @@ export async function markPostgresSchedulerOutboxDelivered(
19414
19990
  ).length;
19415
19991
  }
19416
19992
 
19417
- /** Shared lease-fenced ACK, including the canonical retained-payload cleanup. */
19993
+ /**
19994
+ * Shared lease-fenced ACK, including the canonical retained-payload cleanup.
19995
+ * Known-run callers keep that scope visible to the planner: a materialized ID
19996
+ * array alone can make stale pending-index statistics favor a cross-run scan.
19997
+ */
19418
19998
  function runLedgerAcknowledgeSql(
19419
19999
  options: Pick<PostgresSchedulerOptions, 'schema'> | undefined,
19420
20000
  eventIdsParameter: string,
19421
20001
  leaseOwnerParameter: string,
20002
+ lockFence?: string,
20003
+ runIdParameter?: string,
19422
20004
  ): string {
19423
20005
  return `UPDATE ${tableName('outbox', options)} AS outbox
19424
20006
  SET delivered_at = now(),
@@ -19432,6 +20014,8 @@ function runLedgerAcknowledgeSql(
19432
20014
  AND run.run_event_store = 'planet_scale'
19433
20015
  ) THEN '{}'::jsonb ELSE outbox.payload_json END
19434
20016
  WHERE outbox.event_id = ANY(${eventIdsParameter}::text[])
20017
+ ${runIdParameter ? `AND outbox.run_id = ${runIdParameter}::text` : ''}
20018
+ ${lockFence ? `AND (${lockFence})` : ''}
19435
20019
  AND outbox.delivered_at IS NULL
19436
20020
  AND (${leaseOwnerParameter}::text IS NULL OR outbox.lease_owner = ${leaseOwnerParameter})
19437
20021
  RETURNING outbox.event_id,
@@ -20373,6 +20957,18 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20373
20957
  ELSE ${runLedgerDispatchLaunchSql('runs')}
20374
20958
  END`
20375
20959
  : 'NULL::jsonb';
20960
+ const transportHeadSelection =
20961
+ tokenAuthority === 'gateway' &&
20962
+ !input.searchOnly &&
20963
+ !input.headEventId &&
20964
+ (input.limit ?? 1) >= 2
20965
+ ? `(SELECT CASE WHEN ${runLedgerZeroEffectTransportSql('transport')} THEN
20966
+ jsonb_build_object('eventId', transport.event_id, 'eventType', transport.event_type, 'deliveryKey', transport.delivery_key)
20967
+ ELSE NULL::jsonb END
20968
+ FROM ${tableName('outbox', options)} AS transport
20969
+ WHERE transport.run_id = runs.run_id AND transport.delivered_at IS NULL AND NOT ${searchLaneSql('transport')}
20970
+ ORDER BY ${runLedgerOrderSql('transport')} LIMIT 1)`
20971
+ : 'NULL::jsonb';
20376
20972
  const continuation: { current: RunLedgerOutboxDrainScope | null } = {
20377
20973
  current: scope ?? null,
20378
20974
  };
@@ -20412,88 +21008,25 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20412
21008
  // An abandoned attempt that already handed its lease back on a checkout it
20413
21009
  // held does not spend another checkout on the same release.
20414
21010
  const abandonedLease = { released: false };
20415
- const claimEvents = async (client: PostgresSchedulerQueryClient) => {
20416
- claimInvocations += 1;
20417
- const abandonedHeadEventId = input.headEventId?.trim();
20418
- if (
20419
- input.signal?.aborted &&
20420
- (claimInvocations === 1 || abandonedHeadEventId)
20421
- ) {
20422
- // The caller's deadline elapsed while this checkout was queued. Its
20423
- // attempt is already abandoned, so taking a lease now could only keep
20424
- // the durable projector off the row until that lease expires. Claim
20425
- // nothing: the row stays pending and unleased for its next owner.
20426
- if (claimInvocations > 1 && abandonedHeadEventId) {
20427
- // An earlier invocation's COMMIT may have reached the server before
20428
- // its connection failed, so this owner can already hold the named
20429
- // head. Hand it back here, fenced to this owner; returning empty
20430
- // would leave that lease with nobody to release it. After a rolled
20431
- // back invocation it matches no row. A failure propagates, so the
20432
- // attempt is never reported as settled.
20433
- await releasePostgresSchedulerOutboxLease(
20434
- client,
20435
- [abandonedHeadEventId],
20436
- { ...options, leaseOwner },
20437
- );
20438
- }
20439
- const unclaimed: {
20440
- events: PostgresSchedulerOutboxEvent[];
20441
- launchJson: unknown;
20442
- ingressPrefixClaimed: boolean;
20443
- } = { events: [], launchJson: undefined, ingressPrefixClaimed: false };
20444
- return unclaimed;
20445
- }
20446
- // A re-invocation with no named head claims again under the same owner,
20447
- // as it always has; the abort check after the claim releases by id.
20448
- // The checkout is exclusive. Hold the existing scheduler run's row lock
20449
- // only while choosing/claiming its outbox prefix, never over HTTP. A
20450
- // second statement takes a fresh READ COMMITTED snapshot after the lock:
20451
- // late producer commits cannot split one run between different heads.
20452
- await client.query('BEGIN');
20453
- try {
20454
- const targetRunId = input.runId?.trim() || null;
20455
- let lockedRun: {
20456
- rows: { run_id: string; dispatch_launch_json: unknown }[];
20457
- } | null = null;
20458
- if (input.waitForRunLock && targetRunId) {
20459
- // A synchronous drain must not mistake a busy run for an empty
20460
- // outbox. In particular, cancellation owns its pending prefix lease,
20461
- // so returning early also prevents background recovery from helping.
20462
- // Wait only for this run, under the existing cancellation SQL budget,
20463
- // then select the prefix using a fresh READ COMMITTED snapshot. The
20464
- // transaction still ends before HTTP; background claims keep SKIP LOCKED.
20465
- await client.query(
20466
- `SELECT set_config('lock_timeout', $1, true),
20467
- set_config('statement_timeout', $2, true)`,
20468
- [
20469
- `${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
20470
- `${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
20471
- ],
20472
- );
20473
- lockedRun = await client.query<{
20474
- run_id: string;
20475
- dispatch_launch_json: unknown;
20476
- }>(
20477
- `SELECT runs.run_id,
20478
- ${needsLaunch ? runLedgerDispatchLaunchSql('runs') : 'NULL::jsonb'} AS dispatch_launch_json
20479
- FROM ${tableName('runs', options)} AS runs
20480
- WHERE runs.run_id = $1 FOR UPDATE`,
20481
- [targetRunId],
20482
- );
20483
- }
20484
- // A worker fast path always names one run. Keep that lookup on the
20485
- // run-scoped outbox prefix: combining it with global recovery through a
20486
- // nullable predicate made PostgreSQL materialize every pending run head
20487
- // for each lifecycle transition. Only the two dedicated projectors may
20488
- // perform the global indexed head discovery below.
20489
- const selected =
20490
- (input.headEventId ? null : lockedRun) ??
20491
- (targetRunId
20492
- ? await client.query<{
20493
- run_id: string;
20494
- dispatch_launch_json: unknown;
20495
- }>(
20496
- `
21011
+ const selectRunForClaim = async (
21012
+ client: PostgresSchedulerQueryClient,
21013
+ targetRunId: string | null,
21014
+ lockedRun: {
21015
+ rows: {
21016
+ run_id: string;
21017
+ dispatch_launch_json: unknown;
21018
+ transport_head?: RunLedgerTransportCandidate | null;
21019
+ }[];
21020
+ } | null = null,
21021
+ ) => {
21022
+ return await ((input.headEventId ? null : lockedRun) ??
21023
+ (targetRunId
21024
+ ? await client.query<{
21025
+ run_id: string;
21026
+ dispatch_launch_json: unknown;
21027
+ transport_head?: RunLedgerTransportCandidate | null;
21028
+ }>(
21029
+ `
20497
21030
  WITH head AS MATERIALIZED (
20498
21031
  SELECT pending.run_id,
20499
21032
  pending.event_id,
@@ -20509,7 +21042,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20509
21042
  ORDER BY ${runLedgerOrderSql('pending')}
20510
21043
  LIMIT 1
20511
21044
  )
20512
- SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json
21045
+ SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json, ${transportHeadSelection} AS transport_head
20513
21046
  FROM head
20514
21047
  JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
20515
21048
  WHERE head.failed_at IS NULL
@@ -20526,12 +21059,13 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20526
21059
  )
20527
21060
  FOR UPDATE OF runs SKIP LOCKED
20528
21061
  `,
20529
- [targetRunId, leaseOwner, input.headEventId?.trim() || null],
20530
- )
20531
- : await client.query<{
20532
- run_id: string;
20533
- dispatch_launch_json: unknown;
20534
- }>(`
21062
+ [targetRunId, leaseOwner, input.headEventId?.trim() || null],
21063
+ )
21064
+ : await client.query<{
21065
+ run_id: string;
21066
+ dispatch_launch_json: unknown;
21067
+ transport_head?: RunLedgerTransportCandidate | null;
21068
+ }>(`
20535
21069
  WITH RECURSIVE ${runLedgerPendingRunsSql(options, input.searchOnly === true)}, heads AS MATERIALIZED (
20536
21070
  SELECT pending_run.run_id, head.*
20537
21071
  FROM pending_runs AS pending_run
@@ -20550,7 +21084,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20550
21084
  LIMIT 1
20551
21085
  ) AS head
20552
21086
  )
20553
- SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json
21087
+ SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json, ${transportHeadSelection} AS transport_head
20554
21088
  FROM heads AS head
20555
21089
  JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
20556
21090
  WHERE head.failed_at IS NULL
@@ -20579,29 +21113,192 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20579
21113
  }
20580
21114
  head.created_at, runs.run_id
20581
21115
  FOR UPDATE OF runs SKIP LOCKED LIMIT 1
20582
- `));
20583
- const runId = selected.rows[0]?.run_id;
20584
- let ingressPrefixClaimed = false;
20585
- if (
20586
- runId &&
20587
- tokenAuthority === 'gateway' &&
20588
- !input.searchOnly &&
20589
- (input.limit ?? 1) === ingressPrefixLimit
20590
- ) {
20591
- try {
20592
- const launch = parseRunLedgerDispatchLaunch(
20593
- selected.rows[0]?.dispatch_launch_json,
20594
- );
20595
- ingressPrefixClaimed =
20596
- launch.runId === runId &&
20597
- usableRunLedgerExecutorToken(launch, reuseMarginMs, options)
20598
- ?.runLedgerIngressPrefixBatching === true;
20599
- } catch {
20600
- // Malformed launches retain the existing post-claim failure policy.
20601
- }
21116
+ `)));
21117
+ };
21118
+ const ingressPrefixSupported = (selected: {
21119
+ rows: {
21120
+ run_id: string;
21121
+ dispatch_launch_json: unknown;
21122
+ transport_head?: RunLedgerTransportCandidate | null;
21123
+ }[];
21124
+ }) => {
21125
+ const runId = selected.rows[0]?.run_id;
21126
+ if (
21127
+ runId &&
21128
+ tokenAuthority === 'gateway' &&
21129
+ !input.searchOnly &&
21130
+ (input.limit ?? 1) === ingressPrefixLimit
21131
+ ) {
21132
+ try {
21133
+ const launch = parseRunLedgerDispatchLaunch(
21134
+ selected.rows[0]?.dispatch_launch_json,
21135
+ );
21136
+ return (
21137
+ launch.runId === runId &&
21138
+ usableRunLedgerExecutorToken(launch, reuseMarginMs, options)
21139
+ ?.runLedgerIngressPrefixBatching === true
21140
+ );
21141
+ } catch {
21142
+ // Malformed launches retain the existing post-claim failure policy.
20602
21143
  }
21144
+ }
21145
+ return false;
21146
+ };
21147
+ // Retain a chosen run across a checkout's connection retry: an ambiguous
21148
+ // committed claim must be reread under the same owner rather than abandoned
21149
+ // by a global scan that now sees its own active lease.
21150
+ let selectedFastRun: {
21151
+ rows: {
21152
+ run_id: string;
21153
+ dispatch_launch_json: unknown;
21154
+ transport_head?: RunLedgerTransportCandidate | null;
21155
+ }[];
21156
+ } | null = null;
21157
+ const claimFastEvents = async (client: PostgresSchedulerQueryClient) => {
21158
+ if (
21159
+ runLedgerOutboxBatchUnsupportedClients.has(client) ||
21160
+ !supportsOutboxDeliveryBatchMarker(options) ||
21161
+ input.signal?.aborted
21162
+ )
21163
+ return null;
21164
+ const targetRunId = input.runId?.trim() || null;
21165
+ const selected =
21166
+ selectedFastRun ??
21167
+ (input.waitForRunLock && targetRunId
21168
+ ? await client.query<{
21169
+ run_id: string;
21170
+ dispatch_launch_json: unknown;
21171
+ transport_head?: RunLedgerTransportCandidate | null;
21172
+ }>(
21173
+ `SELECT runs.run_id, ${needsLaunch ? runLedgerDispatchLaunchSql('runs') : 'NULL::jsonb'} AS dispatch_launch_json, ${transportHeadSelection} AS transport_head
21174
+ FROM ${tableName('runs', options)} AS runs WHERE runs.run_id = $1`,
21175
+ [targetRunId],
21176
+ )
21177
+ : await selectRunForClaim(client, targetRunId));
21178
+ const runId = selected.rows[0]?.run_id;
21179
+ if (!runId)
21180
+ return {
21181
+ events: [],
21182
+ launchJson: undefined,
21183
+ ingressPrefixClaimed: false,
21184
+ freshClaimLease: undefined,
21185
+ transportDelivered: undefined,
21186
+ transportOwnershipUnconfirmed: undefined,
21187
+ };
21188
+ selectedFastRun = selected;
21189
+ const ingressPrefixClaimed = ingressPrefixSupported(selected);
21190
+ claimInvocations += 1;
21191
+ const result = await claimRunLedgerOutboxPrefixInBatch(
21192
+ client,
21193
+ {
21194
+ runId,
21195
+ leaseOwner,
21196
+ leaseSeconds: input.leaseSeconds ?? 120,
21197
+ limit: input.limit ?? 1,
21198
+ searchOnly: input.searchOnly,
21199
+ allowIngressPrefix: ingressPrefixClaimed,
21200
+ waitForRunLock: input.waitForRunLock === true,
21201
+ expectedHeadEventId: input.headEventId?.trim() || undefined,
21202
+ transportCandidate:
21203
+ !input.headEventId && ingressPrefixClaimed
21204
+ ? (selected.rows[0]?.transport_head ?? undefined)
21205
+ : undefined,
21206
+ recoverTransportDelivery: claimInvocations > 1,
21207
+ },
21208
+ options,
21209
+ );
21210
+ if (result === null) return null;
21211
+ return {
21212
+ events: result.events,
21213
+ launchJson: selected.rows[0]?.dispatch_launch_json,
21214
+ ingressPrefixClaimed,
21215
+ freshClaimLease: result.freshClaimLease,
21216
+ transportDelivered: result.transportDelivered,
21217
+ transportOwnershipUnconfirmed: result.transportOwnershipUnconfirmed,
21218
+ };
21219
+ };
21220
+ const claimEvents = async (client: PostgresSchedulerQueryClient) => {
21221
+ claimInvocations += 1;
21222
+ const abandonedHeadEventId = input.headEventId?.trim();
21223
+ if (
21224
+ input.signal?.aborted &&
21225
+ (claimInvocations === 1 || abandonedHeadEventId)
21226
+ ) {
21227
+ // The caller's deadline elapsed while this checkout was queued. Its
21228
+ // attempt is already abandoned, so taking a lease now could only keep
21229
+ // the durable projector off the row until that lease expires. Claim
21230
+ // nothing: the row stays pending and unleased for its next owner.
21231
+ if (claimInvocations > 1 && abandonedHeadEventId) {
21232
+ // An earlier invocation's COMMIT may have reached the server before
21233
+ // its connection failed, so this owner can already hold the named
21234
+ // head. Hand it back here, fenced to this owner; returning empty
21235
+ // would leave that lease with nobody to release it. After a rolled
21236
+ // back invocation it matches no row. A failure propagates, so the
21237
+ // attempt is never reported as settled.
21238
+ await releasePostgresSchedulerOutboxLease(
21239
+ client,
21240
+ [abandonedHeadEventId],
21241
+ { ...options, leaseOwner },
21242
+ );
21243
+ }
21244
+ const unclaimed: {
21245
+ events: PostgresSchedulerOutboxEvent[];
21246
+ launchJson: unknown;
21247
+ ingressPrefixClaimed: boolean;
21248
+ freshClaimLease?: RunLedgerFreshClaimLeaseFacts;
21249
+ transportDelivered?: RunLedgerTransportDelivery;
21250
+ transportOwnershipUnconfirmed?: boolean;
21251
+ } = { events: [], launchJson: undefined, ingressPrefixClaimed: false };
21252
+ return unclaimed;
21253
+ }
21254
+ // A re-invocation with no named head claims again under the same owner,
21255
+ // as it always has; the abort check after the claim releases by id.
21256
+ // The checkout is exclusive. Hold the existing scheduler run's row lock
21257
+ // only while choosing/claiming its outbox prefix, never over HTTP. A
21258
+ // second statement takes a fresh READ COMMITTED snapshot after the lock:
21259
+ // late producer commits cannot split one run between different heads.
21260
+ await client.query('BEGIN');
21261
+ try {
21262
+ const targetRunId = input.runId?.trim() || null;
21263
+ let lockedRun: {
21264
+ rows: { run_id: string; dispatch_launch_json: unknown }[];
21265
+ } | null = null;
21266
+ if (input.waitForRunLock && targetRunId) {
21267
+ // A synchronous drain must not mistake a busy run for an empty
21268
+ // outbox. In particular, cancellation owns its pending prefix lease,
21269
+ // so returning early also prevents background recovery from helping.
21270
+ // Wait only for this run, under the existing cancellation SQL budget,
21271
+ // then select the prefix using a fresh READ COMMITTED snapshot. The
21272
+ // transaction still ends before HTTP; background claims keep SKIP LOCKED.
21273
+ await client.query(
21274
+ `SELECT set_config('lock_timeout', $1, true),
21275
+ set_config('statement_timeout', $2, true)`,
21276
+ [
21277
+ `${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
21278
+ `${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
21279
+ ],
21280
+ );
21281
+ lockedRun = await client.query<{
21282
+ run_id: string;
21283
+ dispatch_launch_json: unknown;
21284
+ }>(
21285
+ `SELECT runs.run_id,
21286
+ ${needsLaunch ? runLedgerDispatchLaunchSql('runs') : 'NULL::jsonb'} AS dispatch_launch_json
21287
+ FROM ${tableName('runs', options)} AS runs
21288
+ WHERE runs.run_id = $1 FOR UPDATE`,
21289
+ [targetRunId],
21290
+ );
21291
+ }
21292
+ // A worker fast path always names one run. Keep that lookup on the
21293
+ // run-scoped outbox prefix: combining it with global recovery through a
21294
+ // nullable predicate made PostgreSQL materialize every pending run head
21295
+ // for each lifecycle transition. Only the two dedicated projectors may
21296
+ // perform the global indexed head discovery below.
21297
+ const selected = await selectRunForClaim(client, targetRunId, lockedRun);
21298
+ const runId = selected.rows[0]?.run_id;
21299
+ const ingressPrefixClaimed = ingressPrefixSupported(selected);
20603
21300
  const claimed = runId
20604
- ? await readPendingPostgresSchedulerOutboxEvents(
21301
+ ? await claimPostgresSchedulerOutboxPrefix(
20605
21302
  client,
20606
21303
  {
20607
21304
  limit: input.limit ?? 1,
@@ -20615,16 +21312,19 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20615
21312
  },
20616
21313
  options,
20617
21314
  )
20618
- : [];
21315
+ : { events: [], freshClaimLease: undefined };
20619
21316
  await client.query('COMMIT');
20620
21317
  // The launch is immutable. Carry only its authority/routing fields from
20621
21318
  // the run we already locked, rather than queuing another scarce checkout
20622
21319
  // between the claim and gateway request. The gateway still proves the
20623
21320
  // active outbox lease; no database checkout spans either HTTP hop.
20624
21321
  return {
20625
- events: claimed,
21322
+ events: claimed.events,
20626
21323
  launchJson: selected.rows[0]?.dispatch_launch_json,
20627
21324
  ingressPrefixClaimed,
21325
+ freshClaimLease: claimed.freshClaimLease,
21326
+ transportDelivered: undefined,
21327
+ transportOwnershipUnconfirmed: undefined,
20628
21328
  };
20629
21329
  } catch (error) {
20630
21330
  await client.query('ROLLBACK');
@@ -20634,9 +21334,19 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20634
21334
  const preclaimed = scope?.nextClaimed.length
20635
21335
  ? scope.nextClaimed.splice(0)
20636
21336
  : null;
21337
+ const claimStartedAtMonotonicMs = performance.now();
20637
21338
  const claimed = preclaimed
20638
- ? { events: preclaimed, launchJson: null, ingressPrefixClaimed: false }
21339
+ ? {
21340
+ events: preclaimed,
21341
+ launchJson: null,
21342
+ ingressPrefixClaimed: false,
21343
+ freshClaimLease: undefined,
21344
+ transportDelivered: undefined,
21345
+ transportOwnershipUnconfirmed: undefined,
21346
+ }
20639
21347
  : await withClient(async (client) => {
21348
+ const fast = await claimFastEvents(client);
21349
+ if (fast !== null) return fast;
20640
21350
  try {
20641
21351
  return await claimEvents(client);
20642
21352
  } catch (error) {
@@ -20646,10 +21356,58 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20646
21356
  return claimEvents(client);
20647
21357
  }
20648
21358
  });
21359
+ const claimReceivedAtMonotonicMs = performance.now();
21360
+ // Debit the complete admission/claim/COMMIT round trip, even though SQL's
21361
+ // lease observation occurred later. This is a conservative local proof.
21362
+ const claimElapsedMs = claimReceivedAtMonotonicMs - claimStartedAtMonotonicMs;
21363
+ let freshClaimFirstHttpProof: RunLedgerFreshClaimFirstHttpProof | null =
21364
+ claimed.freshClaimLease &&
21365
+ Number.isFinite(claimElapsedMs) &&
21366
+ claimElapsedMs >= 0
21367
+ ? {
21368
+ ...claimed.freshClaimLease,
21369
+ remainingLeaseMs: Math.floor(
21370
+ claimed.freshClaimLease.remainingLeaseMs - claimElapsedMs,
21371
+ ),
21372
+ receivedAtMonotonicMs: claimReceivedAtMonotonicMs,
21373
+ }
21374
+ : null;
20649
21375
  const { launchJson } = claimed;
21376
+ const transportDelivered = claimed.transportDelivered;
21377
+ const withTransportDelivered = (
21378
+ result: PostgresSchedulerOutboxProjectionResult,
21379
+ ): PostgresSchedulerOutboxProjectionResult =>
21380
+ transportDelivered
21381
+ ? {
21382
+ ...result,
21383
+ attempted: transportDelivered.eventIds.length + result.attempted,
21384
+ projected: transportDelivered.eventIds.length + result.projected,
21385
+ eventIds: [...transportDelivered.eventIds, ...result.eventIds],
21386
+ }
21387
+ : result;
21388
+ if (transportDelivered && input.onDelivered) {
21389
+ try {
21390
+ input.onDelivered(transportDelivered.deliveries);
21391
+ } catch (error) {
21392
+ console.warn('[run-ledger-outbox.delivery_observer_failed]', {
21393
+ error: error instanceof Error ? error.message : String(error),
21394
+ });
21395
+ }
21396
+ }
20650
21397
  let events = claimed.events;
21398
+ const unconfirmedTransportError = claimed.transportOwnershipUnconfirmed
21399
+ ? new RunLedgerProjectionLeaseLostError('delivery_lease_not_current')
21400
+ : null;
21401
+ if (unconfirmedTransportError)
21402
+ console.error('[run-ledger-outbox.batch_dispatch_failed]', {
21403
+ runId: input.runId ?? events[0]?.runId ?? null,
21404
+ code: runLedgerDeliveryFailure({ error: unconfirmedTransportError }).code,
21405
+ failureReason: unconfirmedTransportError.reason,
21406
+ error: 'Local transport ACK transaction ownership is unconfirmed.',
21407
+ eventIds: events.map((event) => event.eventId),
21408
+ });
20651
21409
  if (events.length === 0) {
20652
- return {
21410
+ return withTransportDelivered({
20653
21411
  attempted: 0,
20654
21412
  projected: 0,
20655
21413
  eventIds: [],
@@ -20657,7 +21415,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20657
21415
  failedEventIds: [],
20658
21416
  retrying: 0,
20659
21417
  retryingEventIds: [],
20660
- };
21418
+ });
20661
21419
  }
20662
21420
 
20663
21421
  const claimedAtMs = Date.now();
@@ -20694,6 +21452,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20694
21452
  ) === true,
20695
21453
  )
20696
21454
  ) {
21455
+ freshClaimFirstHttpProof = null;
20697
21456
  if (!input.materializeObservability) {
20698
21457
  // Inline admission/lifecycle/read-heal paths must never index logs.
20699
21458
  // Release and wake the existing projector without charging a retry.
@@ -20817,11 +21576,12 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20817
21576
  // any authority or renewal work. The abort branch below hands the lease
20818
21577
  // straight back without charging a projection attempt.
20819
21578
  input.signal?.throwIfAborted();
21579
+ if (unconfirmedTransportError) throw unconfirmedTransportError;
20820
21580
  // The ledger converter rejects raw deferred facts. Inline callers hand
20821
21581
  // them off; the background owner projects them before inspecting the
20822
21582
  // compact lifecycle payload or deciding that it needs no HTTP append.
20823
21583
  const initialHandedOff = await materializeDeferredObservability();
20824
- if (initialHandedOff) return initialHandedOff;
21584
+ if (initialHandedOff) return withTransportDelivered(initialHandedOff);
20825
21585
  if (input.searchOnly) {
20826
21586
  if (
20827
21587
  events.some(
@@ -21005,6 +21765,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
21005
21765
  events.length === 1 &&
21006
21766
  first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents
21007
21767
  ) {
21768
+ freshClaimFirstHttpProof = null;
21008
21769
  const widen = async (client: PostgresSchedulerQueryClient) => {
21009
21770
  await client.query('BEGIN');
21010
21771
  try {
@@ -21045,6 +21806,21 @@ async function dispatchRunLedgerOutboxBatchWithScope(
21045
21806
  }
21046
21807
  };
21047
21808
  const widened = await withClient(async (client) => {
21809
+ const fast = await claimRunLedgerOutboxPrefixInBatch(
21810
+ client,
21811
+ {
21812
+ runId: first.runId,
21813
+ leaseOwner,
21814
+ leaseSeconds: input.leaseSeconds ?? 120,
21815
+ limit: ingressPrefixLimit,
21816
+ allowIngressPrefix: true,
21817
+ extendOwnedHead: first.eventId,
21818
+ waitForRunLock: true,
21819
+ lockStrength: 'no_key_update',
21820
+ },
21821
+ options,
21822
+ );
21823
+ if (fast !== null) return fast.events;
21048
21824
  try {
21049
21825
  return await widen(client);
21050
21826
  } catch (error) {
@@ -21062,13 +21838,17 @@ async function dispatchRunLedgerOutboxBatchWithScope(
21062
21838
  // Widening can add deferred facts to the lease. Project the complete
21063
21839
  // frozen prefix before any HTTP append, including its newly claimed rows.
21064
21840
  const handedOff = await materializeDeferredObservability();
21065
- if (handedOff) return handedOff;
21841
+ if (handedOff) return withTransportDelivered(handedOff);
21066
21842
  if (ledgerEvents !== null) {
21067
21843
  ledgerEvents = events.flatMap(
21068
21844
  buildRunLedgerEventsFromPostgresSchedulerOutbox,
21069
21845
  );
21070
21846
  }
21847
+ let firstHttpRenewal = true;
21071
21848
  const renewDelivery = async () => {
21849
+ const firstRenewal = firstHttpRenewal;
21850
+ firstHttpRenewal = false;
21851
+ let authorityRefreshed = false;
21072
21852
  const currentAuthority = authorityScope.authority;
21073
21853
  if (
21074
21854
  runtimeApi &&
@@ -21080,6 +21860,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
21080
21860
  ) {
21081
21861
  // Initial delivery reuses the claim's routing fields. Renewal must
21082
21862
  // reread authority so an origin change cannot pass under that cache.
21863
+ authorityRefreshed = true;
21083
21864
  const refreshed = await loadPostgresSchedulerRuntimeApiForRun(
21084
21865
  withClient,
21085
21866
  first.runId,
@@ -21106,6 +21887,26 @@ async function dispatchRunLedgerOutboxBatchWithScope(
21106
21887
  expiresAtMs: runtimeApiTokenExpiresAtMs(runtimeApi.executorToken),
21107
21888
  };
21108
21889
  }
21890
+ if (
21891
+ firstRenewal &&
21892
+ terminalRunId === first.runId &&
21893
+ tokenAuthority === 'gateway' &&
21894
+ runtimeApi?.runLedgerIngressPrefixBatching === true &&
21895
+ ledgerEvents !== null &&
21896
+ !authorityRefreshed &&
21897
+ !input.signal?.aborted &&
21898
+ !runtimeApi.signal?.aborted &&
21899
+ freshRunLedgerClaimCoversFirstHttp(
21900
+ freshClaimFirstHttpProof,
21901
+ events,
21902
+ leaseOwner,
21903
+ reuseMarginMs,
21904
+ )
21905
+ ) {
21906
+ // The claim already froze this exact receipt and supplied enough live
21907
+ // lease for HTTP plus SQL. The receiver still validates ownership.
21908
+ return;
21909
+ }
21109
21910
  const renewed = await withClient(async (client) => {
21110
21911
  const eventIds = events.map((event) => event.eventId);
21111
21912
  if (input.signal?.aborted) {
@@ -21251,18 +22052,19 @@ async function dispatchRunLedgerOutboxBatchWithScope(
21251
22052
  });
21252
22053
  }
21253
22054
  }
21254
- const delivered: PostgresSchedulerOutboxProjectionResult = {
21255
- attempted: events.length,
21256
- projected: markedDelivered,
21257
- eventIds:
21258
- markedDelivered === events.length
21259
- ? events.map((event) => event.eventId)
21260
- : [],
21261
- failed: 0,
21262
- failedEventIds: [],
21263
- retrying: 0,
21264
- retryingEventIds: [],
21265
- };
22055
+ const delivered: PostgresSchedulerOutboxProjectionResult =
22056
+ withTransportDelivered({
22057
+ attempted: events.length,
22058
+ projected: markedDelivered,
22059
+ eventIds:
22060
+ markedDelivered === events.length
22061
+ ? events.map((event) => event.eventId)
22062
+ : [],
22063
+ failed: 0,
22064
+ failedEventIds: [],
22065
+ retrying: 0,
22066
+ retryingEventIds: [],
22067
+ });
21266
22068
  if (!acknowledgement.continuation || scope) {
21267
22069
  return delivered;
21268
22070
  }
@@ -21280,6 +22082,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
21280
22082
  await releasePendingContinuation();
21281
22083
  if (
21282
22084
  input.signal?.aborted ||
22085
+ claimed.transportOwnershipUnconfirmed ||
21283
22086
  (error instanceof RunLedgerProjectionLeaseLostError &&
21284
22087
  error.reason === 'head_lease_owner_mismatch')
21285
22088
  ) {
@@ -21335,7 +22138,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
21335
22138
  error.reason === 'head_lease_owner_mismatch'
21336
22139
  ? eventIds
21337
22140
  : released;
21338
- return {
22141
+ return withTransportDelivered({
21339
22142
  attempted: events.length,
21340
22143
  projected: 0,
21341
22144
  eventIds: [],
@@ -21343,7 +22146,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
21343
22146
  failedEventIds: [],
21344
22147
  retrying: handedOff.length,
21345
22148
  retryingEventIds: handedOff,
21346
- };
22149
+ });
21347
22150
  }
21348
22151
  if (error instanceof DeferredRuntimeProjectionInputError) {
21349
22152
  const outcome = await withClient(
@@ -21383,7 +22186,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
21383
22186
  code: error.code,
21384
22187
  detail: error.message,
21385
22188
  });
21386
- return {
22189
+ return withTransportDelivered({
21387
22190
  attempted: events.length,
21388
22191
  projected: 0,
21389
22192
  eventIds: [],
@@ -21391,7 +22194,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
21391
22194
  failedEventIds: failed,
21392
22195
  retrying: retrying.length,
21393
22196
  retryingEventIds: retrying,
21394
- };
22197
+ });
21395
22198
  }
21396
22199
  let settlement: { blocked: string[]; retrying: string[] };
21397
22200
  try {
@@ -21456,7 +22259,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
21456
22259
  settlementError,
21457
22260
  deliveryError: error,
21458
22261
  });
21459
- return {
22262
+ return withTransportDelivered({
21460
22263
  attempted: events.length,
21461
22264
  projected: 0,
21462
22265
  eventIds: [],
@@ -21464,7 +22267,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
21464
22267
  failedEventIds: [],
21465
22268
  retrying: events.length,
21466
22269
  retryingEventIds: events.map((event) => event.eventId),
21467
- };
22270
+ });
21468
22271
  }
21469
22272
  const failure = runLedgerDeliveryFailure({ error });
21470
22273
  console.warn('[run-ledger-outbox.batch_dispatch_failed]', {
@@ -21478,7 +22281,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
21478
22281
  ? { projectionMismatch: failure.projectionMismatch }
21479
22282
  : {}),
21480
22283
  });
21481
- return {
22284
+ return withTransportDelivered({
21482
22285
  attempted: events.length,
21483
22286
  projected: 0,
21484
22287
  eventIds: [],
@@ -21486,7 +22289,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
21486
22289
  failedEventIds: settlement.blocked,
21487
22290
  retrying: settlement.retrying.length,
21488
22291
  retryingEventIds: settlement.retrying,
21489
- };
22292
+ });
21490
22293
  } finally {
21491
22294
  const settledAtMs = Date.now();
21492
22295
  const headAgeAtClaimMs = Math.max(
@@ -21614,6 +22417,48 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
21614
22417
  !!input.scope ||
21615
22418
  (input.input.continueThroughTerminal === true && !input.input.headEventId);
21616
22419
  return withClient(async (client) => {
22420
+ if (retainContinuation) {
22421
+ const eventIds = input.events.map((event) => event.eventId);
22422
+ const claimSuccessor =
22423
+ input.continuationAllowance !== 0 && !eventIds.includes(targetEventId);
22424
+ const fast = await claimRunLedgerOutboxPrefixInBatch(
22425
+ client,
22426
+ {
22427
+ runId,
22428
+ leaseOwner: input.leaseOwner,
22429
+ leaseSeconds: input.input.leaseSeconds ?? 120,
22430
+ limit: claimSuccessor
22431
+ ? Math.min(
22432
+ RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
22433
+ input.runtimeApi?.runLedgerIngressPrefixBatching === true
22434
+ ? RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS
22435
+ : (input.input.limit ?? 1),
22436
+ input.continuationAllowance ??
22437
+ RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
22438
+ )
22439
+ : 0,
22440
+ allowIngressPrefix:
22441
+ input.runtimeApi?.runLedgerIngressPrefixBatching === true,
22442
+ waitForRunLock: true,
22443
+ },
22444
+ options,
22445
+ { eventIds, targetEventId },
22446
+ );
22447
+ if (fast !== null) {
22448
+ let continuation: RunLedgerOutboxDrainScope | null = null;
22449
+ if (fast.events.length > 0) {
22450
+ continuation = input.scope ?? {
22451
+ runId,
22452
+ leaseOwner: input.leaseOwner,
22453
+ targetEventId,
22454
+ nextClaimed: [],
22455
+ };
22456
+ continuation.nextClaimed = fast.events;
22457
+ input.continuation.current = continuation;
22458
+ }
22459
+ return { deliveredRows: fast.deliveredRows, continuation };
22460
+ }
22461
+ }
21617
22462
  const acknowledge = async () => {
21618
22463
  if (retainContinuation) {
21619
22464
  await client.query('BEGIN');