deepline 0.3.290 → 0.3.292

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +8 -6
  2. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  3. package/dist/bundling-sources/sdk/src/runs/overview.ts +16 -6
  4. package/dist/bundling-sources/sdk/src/types.ts +21 -2
  5. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +11 -1
  6. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +209 -51
  7. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +10 -2
  8. package/dist/bundling-sources/shared_libs/play-runtime/log-context.ts +36 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/log-provenance.ts +2 -1
  10. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +5 -1
  11. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +54 -41
  12. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +50 -3
  13. package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +9 -2
  14. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +35 -7
  15. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +55 -9
  16. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +16 -4
  17. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +140 -4
  18. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +559 -313
  19. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +1 -1
  20. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +116 -24
  21. package/dist/bundling-sources/shared_libs/play-runtime/step-progress.ts +6 -1
  22. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +22 -5
  23. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +35 -0
  24. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +68 -8
  25. package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +2 -0
  26. package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +2 -0
  27. package/dist/cli/index.js +209 -588
  28. package/dist/cli/index.mjs +209 -588
  29. package/dist/{client-FR_u0dxT.d.ts → client-Ct4jmvAg.d.ts} +29 -8
  30. package/dist/{client-phK1Ixcl.d.mts → client-Ct9GKrwu.d.mts} +29 -8
  31. package/dist/{compiler-manifest-9SQtUuKb.d.mts → compiler-manifest-qtTNYvBl.d.mts} +2 -0
  32. package/dist/{compiler-manifest-9SQtUuKb.d.ts → compiler-manifest-qtTNYvBl.d.ts} +2 -0
  33. package/dist/index.d.mts +4 -4
  34. package/dist/index.d.ts +4 -4
  35. package/dist/index.js +12 -5
  36. package/dist/index.mjs +12 -5
  37. package/dist/install-integrity.json +4 -4
  38. package/dist/plays/bundle-play-file.d.mts +2 -2
  39. package/dist/plays/bundle-play-file.d.ts +2 -2
  40. package/dist/plays/run-request.d.mts +2 -2
  41. package/dist/plays/run-request.d.ts +2 -2
  42. package/dist/release.d.mts +1 -1
  43. package/dist/release.d.ts +1 -1
  44. package/dist/release.js +1 -1
  45. package/dist/release.mjs +1 -1
  46. package/package.json +1 -1
@@ -91,6 +91,8 @@ import {
91
91
  import { normalizePlayDocflowNodeIoState } from '../docflow-node-io';
92
92
  import {
93
93
  normalizePlayLogContext,
94
+ normalizePlayLogMetadata,
95
+ type PlayLogMetadata,
94
96
  type PlayLogContext,
95
97
  } from '../log-context';
96
98
  import {
@@ -481,11 +483,9 @@ type PostgresSchedulerClaimExecutionOptions = Pick<
481
483
  /** Internal provider-read coordinates which must survive terminal-worker
482
484
  * loss. They are persisted only on the matching durable cleanup obligation. */
483
485
  sandboxCrashDiagnostic?: PostgresSchedulerSandboxCrashDiagnosticRequest;
484
- /** Settled progress facts for a failed runner result, folded before the
485
- * terminal write. They travel in the `run.failed` outbox row ahead of the
486
- * terminal fact, the same carrier a direct completion uses, so the ledger
487
- * settles them before the run's terminal makes its steps immutable. */
488
- failedTerminalProgressEvents?: readonly unknown[];
486
+ /** Final step facts folded into the terminal outbox instead of a separate
487
+ * progress write immediately ahead of the terminal. */
488
+ terminalProgressEvents?: readonly unknown[];
489
489
  /** Forwarded to `failPostgresSchedulerAttempt`: a failure terminal written
490
490
  * for a detached-runner attempt must not overwrite an authoritative pushed
491
491
  * runner terminal that won after the attempt was claimed. */
@@ -10823,6 +10823,7 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
10823
10823
  'producerAttempt', $2::integer,
10824
10824
  'terminalLogLines', $7::jsonb,
10825
10825
  'terminalLogContexts', $17::jsonb,
10826
+ 'terminalLogMetadata', $20::jsonb,
10826
10827
  'terminalLogIds', $19::jsonb->'terminalLogIds',
10827
10828
  'terminalProgressEvents', coalesce((
10828
10829
  SELECT accepted.runner_log_payload_json->'terminalProgressEvents'
@@ -10868,6 +10869,7 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
10868
10869
  JSON.stringify(terminalLogProjection.lineContexts),
10869
10870
  stringifyPostgresJson(terminalWorkCounterSegment ?? null),
10870
10871
  stringifyPostgresJson(runnerTerminalLogPayload(input)),
10872
+ JSON.stringify(terminalLogProjection.lineMetadata),
10871
10873
  ],
10872
10874
  );
10873
10875
  if (completed.rows.length > 0) return 'completed';
@@ -13200,8 +13202,10 @@ export async function completePostgresSchedulerAttempt(
13200
13202
  runtimeTiming?: PlayRunnerRuntimeTiming | null;
13201
13203
  terminalLogLines?: readonly string[] | null;
13202
13204
  terminalLogContexts?: readonly (PlayLogContext | null)[] | null;
13205
+ terminalLogMetadata?: readonly (PlayLogMetadata | null)[] | null;
13203
13206
  terminalLogIds?: readonly (string | null)[] | null;
13204
13207
  terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
13208
+ terminalProgressEvents?: readonly unknown[];
13205
13209
  },
13206
13210
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
13207
13211
  ): Promise<boolean> {
@@ -13218,6 +13222,14 @@ export async function completePostgresSchedulerAttempt(
13218
13222
  'Terminal log contexts must align with terminal log lines.',
13219
13223
  );
13220
13224
  }
13225
+ if (
13226
+ input.terminalLogMetadata != null &&
13227
+ input.terminalLogMetadata.length !== (input.terminalLogLines ?? []).length
13228
+ ) {
13229
+ throw new TypeError(
13230
+ 'Terminal log metadata must align with terminal log lines.',
13231
+ );
13232
+ }
13221
13233
  const terminalLogIds = normalizeRunLogIds(
13222
13234
  input.terminalLogIds,
13223
13235
  input.terminalLogLines ?? [],
@@ -13226,6 +13238,7 @@ export async function completePostgresSchedulerAttempt(
13226
13238
  .map((line, index) => ({
13227
13239
  line: line.trim(),
13228
13240
  context: input.terminalLogContexts?.[index] ?? null,
13241
+ metadata: input.terminalLogMetadata?.[index] ?? null,
13229
13242
  entryId: terminalLogIds?.[index] ?? null,
13230
13243
  }))
13231
13244
  .filter((entry) => entry.line.length > 0)
@@ -13234,6 +13247,9 @@ export async function completePostgresSchedulerAttempt(
13234
13247
  const terminalLogContexts = terminalLogEntries.map((entry) =>
13235
13248
  normalizePlayLogContext(entry.context),
13236
13249
  );
13250
+ const terminalLogMetadata = terminalLogEntries.map((entry) =>
13251
+ normalizePlayLogMetadata(entry.metadata),
13252
+ );
13237
13253
  const completed = await client.query<{ run_id: string }>(
13238
13254
  `
13239
13255
  WITH ${runtimeSandboxCustodyLocksSql('run.run_id = $1', options)}, completed AS (
@@ -13412,10 +13428,15 @@ export async function completePostgresSchedulerAttempt(
13412
13428
  'producerAttempt', $2::integer,
13413
13429
  'terminalLogLines', $8::jsonb,
13414
13430
  'terminalLogContexts', $11::jsonb,
13415
- 'terminalLogIds', $13::jsonb,
13431
+ 'terminalLogMetadata', $13::jsonb,
13432
+ 'terminalLogIds', $15::jsonb,
13416
13433
  'resultSummary', $10::jsonb,
13417
13434
  'terminalWorkCounterSegment', $12::jsonb
13418
- ),
13435
+ ) || CASE
13436
+ WHEN jsonb_array_length($14::jsonb) > 0
13437
+ THEN jsonb_build_object('terminalProgressEvents', $14::jsonb)
13438
+ ELSE '{}'::jsonb
13439
+ END,
13419
13440
  ${causalOutboxCreatedAt('run_terminal.run_id', options)}
13420
13441
  FROM run_terminal
13421
13442
  ON CONFLICT (event_id) DO NOTHING
@@ -13438,6 +13459,8 @@ export async function completePostgresSchedulerAttempt(
13438
13459
  stringifyPostgresJson(resultSummary),
13439
13460
  JSON.stringify(terminalLogContexts),
13440
13461
  stringifyPostgresJson(input.terminalWorkCounterSegment ?? null),
13462
+ JSON.stringify(terminalLogMetadata),
13463
+ JSON.stringify(input.terminalProgressEvents ?? []),
13441
13464
  stringifyPostgresJson(
13442
13465
  terminalLogIds
13443
13466
  ? terminalLogEntries.map((entry) => entry.entryId)
@@ -13461,14 +13484,15 @@ export async function failPostgresSchedulerAttempt(
13461
13484
  result?: unknown;
13462
13485
  terminalLogLines?: readonly string[] | null;
13463
13486
  terminalLogContexts?: readonly (PlayLogContext | null)[] | null;
13487
+ terminalLogMetadata?: readonly (PlayLogMetadata | null)[] | null;
13464
13488
  terminalLogIds?: readonly (string | null)[] | null;
13465
13489
  /** Refuse to overwrite an authoritative detached runner terminal that won
13466
13490
  * after claim. Resumable `suspended` receipts may be superseded by the
13467
13491
  * hard ceiling in the same fenced statement. */
13468
13492
  preserveRunnerTerminal?: boolean;
13469
13493
  sandboxCrashDiagnostic?: PostgresSchedulerSandboxCrashDiagnosticRequest;
13470
- /** Progress facts the ledger must apply before `run.failed` (see
13471
- * `failedTerminalProgressEvents`). Omitted from the payload when empty. */
13494
+ /** Progress facts the ledger must apply before the terminal fact. Omitted
13495
+ * from the payload when empty. */
13472
13496
  terminalProgressEvents?: readonly unknown[];
13473
13497
  },
13474
13498
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
@@ -13508,6 +13532,14 @@ export async function failPostgresSchedulerAttempt(
13508
13532
  'Terminal log contexts must align with terminal log lines.',
13509
13533
  );
13510
13534
  }
13535
+ if (
13536
+ input.terminalLogMetadata != null &&
13537
+ input.terminalLogMetadata.length !== (input.terminalLogLines ?? []).length
13538
+ ) {
13539
+ throw new TypeError(
13540
+ 'Terminal log metadata must align with terminal log lines.',
13541
+ );
13542
+ }
13511
13543
  const terminalLogIds = normalizeRunLogIds(
13512
13544
  input.terminalLogIds,
13513
13545
  input.terminalLogLines ?? [],
@@ -13516,6 +13548,7 @@ export async function failPostgresSchedulerAttempt(
13516
13548
  .map((line, index) => ({
13517
13549
  line: line.trim(),
13518
13550
  context: input.terminalLogContexts?.[index] ?? null,
13551
+ metadata: input.terminalLogMetadata?.[index] ?? null,
13519
13552
  entryId: terminalLogIds?.[index] ?? null,
13520
13553
  }))
13521
13554
  .filter((entry) => entry.line.length > 0)
@@ -13524,6 +13557,9 @@ export async function failPostgresSchedulerAttempt(
13524
13557
  const terminalLogContexts = terminalLogEntries.map((entry) =>
13525
13558
  normalizePlayLogContext(entry.context),
13526
13559
  );
13560
+ const terminalLogMetadata = terminalLogEntries.map((entry) =>
13561
+ normalizePlayLogMetadata(entry.metadata),
13562
+ );
13527
13563
  const failed = await client.query<{ run_id: string }>(
13528
13564
  `
13529
13565
  WITH ${runtimeSandboxCustodyLocksSql('run.run_id = $1', options)}, failed AS (
@@ -13762,7 +13798,8 @@ export async function failPostgresSchedulerAttempt(
13762
13798
  'producerAttempt', $2::integer,
13763
13799
  'terminalLogLines', $12::jsonb,
13764
13800
  'terminalLogContexts', $14::jsonb,
13765
- 'terminalLogIds', $17::jsonb,
13801
+ 'terminalLogMetadata', $17::jsonb,
13802
+ 'terminalLogIds', $18::jsonb,
13766
13803
  'resultSummary', $13::jsonb,
13767
13804
  'terminalWorkCounterSegment', $15::jsonb
13768
13805
  ) || CASE
@@ -13796,6 +13833,7 @@ export async function failPostgresSchedulerAttempt(
13796
13833
  JSON.stringify(terminalLogContexts),
13797
13834
  stringifyPostgresJson(terminalWorkCounterSegment ?? null),
13798
13835
  stringifyPostgresJson(input.terminalProgressEvents ?? []),
13836
+ JSON.stringify(terminalLogMetadata),
13799
13837
  stringifyPostgresJson(
13800
13838
  terminalLogIds
13801
13839
  ? terminalLogEntries.map((entry) => entry.entryId)
@@ -14503,6 +14541,7 @@ function runnerTerminalLogPayload(input: {
14503
14541
  producerAttempt: input.attempt,
14504
14542
  terminalLogLines: projection.lines,
14505
14543
  terminalLogContexts: projection.lineContexts,
14544
+ terminalLogMetadata: projection.lineMetadata,
14506
14545
  ...(projection.lineIds ? { terminalLogIds: projection.lineIds } : {}),
14507
14546
  terminalProgressEvents: input.terminalProgressEvents ?? [],
14508
14547
  // Older runner envelopes conveyed these diagnostics only in tagged log
@@ -14527,6 +14566,7 @@ function terminalLogProjectionForOutbox(
14527
14566
  ): {
14528
14567
  lines: string[];
14529
14568
  lineContexts: Array<PlayLogContext | null>;
14569
+ lineMetadata: Array<PlayLogMetadata | null>;
14530
14570
  lineIds?: Array<string | null>;
14531
14571
  } {
14532
14572
  if (
@@ -14534,16 +14574,17 @@ function terminalLogProjectionForOutbox(
14534
14574
  !runnerResult ||
14535
14575
  typeof runnerResult !== 'object'
14536
14576
  ) {
14537
- return { lines: [], lineContexts: [] };
14577
+ return { lines: [], lineContexts: [], lineMetadata: [] };
14538
14578
  }
14539
14579
  const result = runnerResult as {
14540
14580
  logs?: unknown;
14541
14581
  logContexts?: unknown;
14582
+ logMetadata?: unknown;
14542
14583
  logIds?: unknown;
14543
14584
  };
14544
14585
  const logs = result.logs;
14545
14586
  if (!Array.isArray(logs)) {
14546
- return { lines: [], lineContexts: [] };
14587
+ return { lines: [], lineContexts: [], lineMetadata: [] };
14547
14588
  }
14548
14589
  if (
14549
14590
  result.logContexts !== undefined &&
@@ -14552,6 +14593,15 @@ function terminalLogProjectionForOutbox(
14552
14593
  ) {
14553
14594
  throw new TypeError('Terminal log contexts must align with runner logs.');
14554
14595
  }
14596
+ if (
14597
+ result.logMetadata !== undefined &&
14598
+ (!Array.isArray(result.logMetadata) ||
14599
+ result.logMetadata.length !== logs.length)
14600
+ )
14601
+ throw new TypeError('Terminal log metadata must align with logs.');
14602
+ const logMetadata = Array.isArray(result.logMetadata)
14603
+ ? result.logMetadata
14604
+ : undefined;
14555
14605
  const logContexts = Array.isArray(result.logContexts)
14556
14606
  ? result.logContexts
14557
14607
  : undefined;
@@ -14572,6 +14622,7 @@ function terminalLogProjectionForOutbox(
14572
14622
  return {
14573
14623
  line: line.trim(),
14574
14624
  context: normalizePlayLogContext(logContexts?.[index]),
14625
+ metadata: normalizePlayLogMetadata(logMetadata?.[index]),
14575
14626
  entryId: logIds?.[index] ?? null,
14576
14627
  };
14577
14628
  })
@@ -14581,6 +14632,7 @@ function terminalLogProjectionForOutbox(
14581
14632
  ): entry is {
14582
14633
  line: string;
14583
14634
  context: PlayLogContext | null;
14635
+ metadata: PlayLogMetadata | null;
14584
14636
  entryId: string | null;
14585
14637
  } => entry !== null,
14586
14638
  )
@@ -14588,6 +14640,7 @@ function terminalLogProjectionForOutbox(
14588
14640
  return {
14589
14641
  lines: entries.map((entry) => entry.line),
14590
14642
  lineContexts: entries.map((entry) => entry.context),
14643
+ lineMetadata: entries.map((entry) => entry.metadata),
14591
14644
  ...(logIds ? { lineIds: entries.map((entry) => entry.entryId) } : {}),
14592
14645
  };
14593
14646
  }
@@ -14837,6 +14890,8 @@ export async function executePostgresSchedulerClaim(
14837
14890
  runtimeTiming: extractRunnerRuntimeTiming(output),
14838
14891
  terminalLogLines: terminalLogProjection.lines,
14839
14892
  terminalLogContexts: terminalLogProjection.lineContexts,
14893
+ terminalLogMetadata: terminalLogProjection.lineMetadata,
14894
+ terminalProgressEvents: options.terminalProgressEvents,
14840
14895
  terminalLogIds: terminalLogProjection.lineIds,
14841
14896
  terminalWorkCounterSegment: terminalWorkCounterSegmentFromRunnerResult(
14842
14897
  output,
@@ -14899,13 +14954,14 @@ export async function executePostgresSchedulerClaim(
14899
14954
  result: terminalFailureResult,
14900
14955
  terminalLogLines: terminalLogProjection.lines,
14901
14956
  terminalLogContexts: terminalLogProjection.lineContexts,
14957
+ terminalLogMetadata: terminalLogProjection.lineMetadata,
14902
14958
  terminalLogIds: terminalLogProjection.lineIds,
14903
14959
  sandboxCrashDiagnostic: options.sandboxCrashDiagnostic,
14904
14960
  ...(options.preserveRunnerTerminal
14905
14961
  ? { preserveRunnerTerminal: true }
14906
14962
  : {}),
14907
- ...(runnerFailureResult && options.failedTerminalProgressEvents?.length
14908
- ? { terminalProgressEvents: options.failedTerminalProgressEvents }
14963
+ ...(options.terminalProgressEvents?.length
14964
+ ? { terminalProgressEvents: options.terminalProgressEvents }
14909
14965
  : {}),
14910
14966
  },
14911
14967
  options,
@@ -17862,6 +17918,32 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17862
17918
  } = {},
17863
17919
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
17864
17920
  ): Promise<PostgresSchedulerOutboxEvent[]> {
17921
+ const { events } = await claimPostgresSchedulerOutboxPrefix(
17922
+ client,
17923
+ input,
17924
+ options,
17925
+ );
17926
+ return events;
17927
+ }
17928
+
17929
+ /**
17930
+ * ACK and next-prefix selection share one statement after the run lock. Data
17931
+ * CTEs use the statement's original snapshot, so every pending-row predicate
17932
+ * explicitly excludes the acknowledged IDs. A partial ACK cannot claim a
17933
+ * successor. The receiver's ingress capability and frozen receipts still
17934
+ * determine which causal facts may share a prefix.
17935
+ */
17936
+ async function claimPostgresSchedulerOutboxPrefix(
17937
+ client: PostgresSchedulerQueryClient,
17938
+ input: NonNullable<
17939
+ Parameters<typeof readPendingPostgresSchedulerOutboxEvents>[1]
17940
+ >,
17941
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
17942
+ acknowledgement?: { eventIds: string[]; targetEventId: string },
17943
+ ): Promise<{
17944
+ events: PostgresSchedulerOutboxEvent[];
17945
+ deliveredRows: RunLedgerOutboxDeliveredRow[];
17946
+ }> {
17865
17947
  const requestedLimit = Math.floor(input.limit ?? 1);
17866
17948
  const limit = Number.isFinite(requestedLimit)
17867
17949
  ? Math.max(1, requestedLimit)
@@ -17869,6 +17951,10 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17869
17951
  const runId = input.runId?.trim() || null;
17870
17952
  const leaseOwner = input.leaseOwner?.trim() || null;
17871
17953
  const leaseSeconds = Math.max(5, Math.min(300, input.leaseSeconds ?? 30));
17954
+ const pending = (alias: string) =>
17955
+ `${alias}.delivered_at IS NULL${
17956
+ acknowledgement ? ` AND NOT (${alias}.event_id = ANY($5::text[]))` : ''
17957
+ }`;
17872
17958
  const claim = async (supportsTerminalBatchMarker: boolean) => {
17873
17959
  const urgentLifecyclePendingSql = (alias: string) =>
17874
17960
  input.searchOnly === true
@@ -17879,15 +17965,23 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17879
17965
  ? `(head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} AND ${urgentLifecyclePendingSql('head')})`
17880
17966
  : 'false';
17881
17967
  return await client.query<{
17968
+ row_kind: 'claimed' | 'delivered';
17882
17969
  event_id: string;
17883
17970
  run_id: string;
17884
17971
  event_type: string;
17885
17972
  payload_json: unknown;
17886
17973
  created_at: Date;
17887
17974
  delivery_key: string | null;
17975
+ lag_ms: number | string | null;
17888
17976
  }>(
17889
17977
  `
17890
17978
  WITH ${
17979
+ acknowledgement
17980
+ ? `delivered AS (
17981
+ ${runLedgerAcknowledgeSql(options, '$5', '$3')}
17982
+ ),`
17983
+ : ''
17984
+ } ${
17891
17985
  input.searchOnly === true
17892
17986
  ? ''
17893
17987
  : `waiting_runs AS MATERIALIZED (
@@ -17898,7 +17992,10 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17898
17992
  ? `owned_run AS MATERIALIZED (
17899
17993
  SELECT run_id FROM ${tableName('runs', options)}
17900
17994
  WHERE run_id = $2
17901
- FOR UPDATE SKIP LOCKED
17995
+ -- This dispatcher already owns the head. A producer holding the run
17996
+ -- lock is not a lost delivery lease: wait for its short transaction,
17997
+ -- then revalidate the head below. New-run selection still skips locks.
17998
+ FOR UPDATE
17902
17999
  ),`
17903
18000
  : ''
17904
18001
  } head AS MATERIALIZED (
@@ -17906,8 +18003,18 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17906
18003
  ${supportsTerminalBatchMarker ? 'candidate.delivery_batch_started_at' : 'NULL::timestamptz'} AS delivery_batch_started_at
17907
18004
  FROM ${tableName('outbox', options)} AS candidate
17908
18005
  ${input.extendOwnedHead ? 'JOIN owned_run ON owned_run.run_id = candidate.run_id' : ''}
17909
- WHERE candidate.delivered_at IS NULL
18006
+ WHERE ${pending('candidate')}
17910
18007
  AND ${searchLaneSql('candidate')} = ${input.searchOnly === true ? 'true' : 'false'}
18008
+ ${
18009
+ acknowledgement
18010
+ ? `AND (SELECT count(*) FROM delivered) = cardinality($5::text[])
18011
+ AND EXISTS (
18012
+ SELECT 1 FROM ${tableName('outbox', options)} AS target
18013
+ WHERE target.run_id = $2 AND target.event_id = $6
18014
+ AND ${pending('target')} AND target.failed_at IS NULL
18015
+ )`
18016
+ : ''
18017
+ }
17911
18018
  ${
17912
18019
  input.extendOwnedHead
17913
18020
  ? `AND candidate.event_id = $5
@@ -17929,8 +18036,8 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17929
18036
  AND NOT EXISTS (
17930
18037
  SELECT 1 FROM ${tableName('outbox', options)} AS active
17931
18038
  WHERE active.run_id = candidate.run_id
18039
+ AND ${pending('active')}
17932
18040
  AND ${searchLaneSql('active')} = ${searchLaneSql('candidate')}
17933
- AND active.delivered_at IS NULL
17934
18041
  AND active.lease_owner IS NOT NULL
17935
18042
  AND active.lease_expires_at > now()
17936
18043
  AND active.lease_owner IS DISTINCT FROM $3::text
@@ -17942,7 +18049,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17942
18049
  SELECT 1
17943
18050
  FROM ${tableName('outbox', options)} AS predecessor
17944
18051
  WHERE predecessor.run_id = candidate.run_id
17945
- AND predecessor.delivered_at IS NULL
18052
+ AND ${pending('predecessor')}
17946
18053
  AND ${runLedgerPrecedesSql('predecessor', 'candidate')}
17947
18054
  )
17948
18055
  -- A new durable admission has no causal dependency on unrelated
@@ -17969,7 +18076,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17969
18076
  SELECT outbox.event_id
17970
18077
  FROM ${tableName('outbox', options)} AS outbox
17971
18078
  JOIN head ON head.run_id = outbox.run_id
17972
- WHERE outbox.delivered_at IS NULL
18079
+ WHERE ${pending('outbox')}
17973
18080
  AND ${searchLaneSql('outbox')} = ${input.searchOnly === true ? 'true' : 'false'}
17974
18081
  AND outbox.failed_at IS NULL
17975
18082
  -- Admission is causally first and must be projected alone. A later
@@ -18006,7 +18113,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
18006
18113
  SELECT 1
18007
18114
  FROM ${tableName('outbox', options)} AS terminal_predecessor
18008
18115
  WHERE terminal_predecessor.run_id = outbox.run_id
18009
- AND terminal_predecessor.delivered_at IS NULL
18116
+ AND ${pending('terminal_predecessor')}
18010
18117
  AND ${
18011
18118
  supportsTerminalBatchMarker
18012
18119
  ? 'terminal_predecessor.delivery_batch_started_at IS NOT NULL'
@@ -18028,7 +18135,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
18028
18135
  SELECT 1
18029
18136
  FROM ${tableName('outbox', options)} AS predecessor
18030
18137
  WHERE predecessor.run_id = outbox.run_id
18031
- AND predecessor.delivered_at IS NULL
18138
+ AND ${pending('predecessor')}
18032
18139
  AND ${runLedgerPrecedesSql('predecessor', 'outbox')}
18033
18140
  AND (
18034
18141
  (predecessor.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
@@ -18077,17 +18184,50 @@ export async function readPendingPostgresSchedulerOutboxEvents(
18077
18184
  outbox.payload_json, outbox.created_at, outbox.lease_owner,
18078
18185
  outbox.delivery_key
18079
18186
  )
18080
- SELECT event_id, run_id, event_type, payload_json, created_at, lease_owner,
18081
- delivery_key
18082
- FROM claimed
18083
- ORDER BY ${runLedgerOrderSql('claimed')}
18187
+ ${
18188
+ acknowledgement
18189
+ ? `, notified AS MATERIALIZED (
18190
+ SELECT delivered.*,
18191
+ pg_notify(
18192
+ $7,
18193
+ json_build_object('eventId', event_id, 'kind', 'outbox_delivered')::text
18194
+ ) AS notification
18195
+ FROM delivered
18196
+ )`
18197
+ : ''
18198
+ }
18199
+ SELECT * FROM (
18200
+ SELECT 'claimed'::text AS row_kind,
18201
+ event_id, run_id, event_type, payload_json, created_at, lease_owner,
18202
+ delivery_key, NULL::float8 AS lag_ms
18203
+ FROM claimed
18204
+ ${
18205
+ acknowledgement
18206
+ ? `UNION ALL
18207
+ SELECT 'delivered'::text AS row_kind,
18208
+ event_id, NULL::text AS run_id, event_type, NULL::jsonb AS payload_json,
18209
+ NULL::timestamptz AS created_at, NULL::text AS lease_owner,
18210
+ NULL::text AS delivery_key, lag_ms
18211
+ FROM notified`
18212
+ : ''
18213
+ }
18214
+ ) AS result
18215
+ ORDER BY row_kind, ${runLedgerOrderSql('result')}
18084
18216
  `,
18085
18217
  [
18086
18218
  limit,
18087
18219
  runId,
18088
18220
  leaseOwner,
18089
18221
  leaseSeconds,
18090
- ...(input.extendOwnedHead ? [input.extendOwnedHead] : []),
18222
+ ...(acknowledgement
18223
+ ? [
18224
+ acknowledgement.eventIds,
18225
+ acknowledgement.targetEventId,
18226
+ postgresSchedulerNotifyChannel('terminal', options),
18227
+ ]
18228
+ : input.extendOwnedHead
18229
+ ? [input.extendOwnedHead]
18230
+ : []),
18091
18231
  ],
18092
18232
  );
18093
18233
  };
@@ -18098,9 +18238,13 @@ export async function readPendingPostgresSchedulerOutboxEvents(
18098
18238
  result = await claim(true);
18099
18239
  } catch (error) {
18100
18240
  if (!isMissingOutboxDeliveryBatchMarker(error)) throw error;
18241
+ markOutboxDeliveryBatchMarkerUnavailable(options);
18242
+ // The ACK caller owns the whole short transaction. Roll it back and
18243
+ // repeat only this missing-column adapter, avoiding two SAVEPOINT round
18244
+ // trips on every successful acknowledgement.
18245
+ if (acknowledgement) throw error;
18101
18246
  if (input.inTransaction)
18102
18247
  await client.query('ROLLBACK TO SAVEPOINT run_ledger_claim');
18103
- markOutboxDeliveryBatchMarkerUnavailable(options);
18104
18248
  result = await claim(false);
18105
18249
  }
18106
18250
  } else {
@@ -18108,22 +18252,34 @@ export async function readPendingPostgresSchedulerOutboxEvents(
18108
18252
  }
18109
18253
  if (input.inTransaction)
18110
18254
  await client.query('RELEASE SAVEPOINT run_ledger_claim');
18111
- return result.rows.map((row) => {
18112
- if (!row.delivery_key) {
18113
- throw new Error(
18114
- `Run-ledger outbox event ${row.event_id} has no delivery key.`,
18115
- );
18116
- }
18117
- return {
18118
- eventId: row.event_id,
18119
- runId: row.run_id,
18120
- eventType: row.event_type,
18121
- payload: row.payload_json,
18122
- createdAt: row.created_at,
18123
- leaseOwner: leaseOwner,
18124
- deliveryKey: row.delivery_key,
18125
- };
18126
- });
18255
+ const events = result.rows
18256
+ .filter((row) => row.row_kind === 'claimed')
18257
+ .map((row) => {
18258
+ if (!row.delivery_key) {
18259
+ throw new Error(
18260
+ `Run-ledger outbox event ${row.event_id} has no delivery key.`,
18261
+ );
18262
+ }
18263
+ return {
18264
+ eventId: row.event_id,
18265
+ runId: row.run_id,
18266
+ eventType: row.event_type,
18267
+ payload: row.payload_json,
18268
+ createdAt: row.created_at,
18269
+ leaseOwner: leaseOwner,
18270
+ deliveryKey: row.delivery_key,
18271
+ };
18272
+ });
18273
+ const deliveredRows = result.rows
18274
+ .filter((row) => row.row_kind === 'delivered')
18275
+ .map((row) => {
18276
+ const lagMs = Number(row.lag_ms);
18277
+ return {
18278
+ eventType: row.event_type,
18279
+ lagMs: Number.isFinite(lagMs) ? Math.max(0, lagMs) : 0,
18280
+ };
18281
+ });
18282
+ return { events, deliveredRows };
18127
18283
  }
18128
18284
 
18129
18285
  /**
@@ -18388,7 +18544,7 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
18388
18544
  '{events}',
18389
18545
  (SELECT jsonb_agg(
18390
18546
  CASE WHEN item.value->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
18391
- THEN item.value - 'lines' - 'lineContexts' - 'lineIds'
18547
+ THEN item.value - 'lines' - 'lineContexts' - 'lineMetadata' - 'lineIds'
18392
18548
  ELSE item.value
18393
18549
  END ORDER BY item.ordinality
18394
18550
  )
@@ -18401,11 +18557,11 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
18401
18557
  )
18402
18558
  WHEN jsonb_typeof(outbox.payload_json->'terminalProgressEvents') = 'array'
18403
18559
  THEN jsonb_set(
18404
- outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogIds',
18560
+ outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogMetadata' - 'terminalLogIds',
18405
18561
  '{terminalProgressEvents}',
18406
18562
  coalesce((SELECT jsonb_agg(
18407
18563
  CASE WHEN item.value->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
18408
- THEN item.value - 'lines' - 'lineContexts' - 'lineIds'
18564
+ THEN item.value - 'lines' - 'lineContexts' - 'lineMetadata' - 'lineIds'
18409
18565
  ELSE item.value
18410
18566
  END ORDER BY item.ordinality
18411
18567
  )
@@ -18413,7 +18569,7 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
18413
18569
  WITH ORDINALITY AS item(value, ordinality)), '[]'::jsonb),
18414
18570
  false
18415
18571
  )
18416
- ELSE outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogIds'
18572
+ ELSE outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogMetadata' - 'terminalLogIds'
18417
18573
  END
18418
18574
  FROM candidates
18419
18575
  WHERE outbox.event_id = candidates.event_id`,
@@ -18475,6 +18631,17 @@ function runLedgerDeliveryFailure(input: { error: unknown }): {
18475
18631
  projectionMismatch: RunLedgerProjectionMismatchDiagnostic | null;
18476
18632
  retryAfterMs: number | null;
18477
18633
  } {
18634
+ if (input.error instanceof RunLedgerProjectionLeaseLostError) {
18635
+ return {
18636
+ permanent: false,
18637
+ code: 'RUN_LEDGER_DELIVERY_LEASE_LOST',
18638
+ status: null,
18639
+ retryable: true,
18640
+ failureReason: input.error.reason,
18641
+ projectionMismatch: input.error.projectionMismatch ?? null,
18642
+ retryAfterMs: null,
18643
+ };
18644
+ }
18478
18645
  if (input.error instanceof AppRuntimeApiResponseError) {
18479
18646
  // Projection is idempotent. A dependency can incorrectly send
18480
18647
  // `retryable:false` with a transient overload response, but that metadata
@@ -18655,6 +18822,32 @@ export async function markPostgresSchedulerOutboxDelivered(
18655
18822
  ).length;
18656
18823
  }
18657
18824
 
18825
+ /** Shared lease-fenced ACK, including the canonical retained-payload cleanup. */
18826
+ function runLedgerAcknowledgeSql(
18827
+ options: Pick<PostgresSchedulerOptions, 'schema'> | undefined,
18828
+ eventIdsParameter: string,
18829
+ leaseOwnerParameter: string,
18830
+ ): string {
18831
+ return `UPDATE ${tableName('outbox', options)} AS outbox
18832
+ SET delivered_at = now(),
18833
+ next_attempt_at = NULL,
18834
+ last_error_json = NULL,
18835
+ lease_owner = NULL,
18836
+ lease_expires_at = NULL,
18837
+ payload_json = CASE WHEN EXISTS (
18838
+ SELECT 1 FROM ${tableName('runs', options)} AS run
18839
+ WHERE run.run_id = outbox.run_id
18840
+ AND run.run_event_store = 'planet_scale'
18841
+ ) THEN '{}'::jsonb ELSE outbox.payload_json END
18842
+ WHERE outbox.event_id = ANY(${eventIdsParameter}::text[])
18843
+ AND outbox.delivered_at IS NULL
18844
+ AND (${leaseOwnerParameter}::text IS NULL OR outbox.lease_owner = ${leaseOwnerParameter})
18845
+ RETURNING outbox.event_id,
18846
+ outbox.event_type,
18847
+ (extract(epoch FROM (outbox.delivered_at - outbox.created_at)) * 1000)::float8
18848
+ AS lag_ms`;
18849
+ }
18850
+
18658
18851
  /**
18659
18852
  * Lease-fenced acknowledgement that also reports each row's event type and
18660
18853
  * commit-to-acknowledgement lag on the database clock, so lag metrics never
@@ -18675,24 +18868,7 @@ export async function markPostgresSchedulerOutboxDeliveredRows(
18675
18868
  }>(
18676
18869
  `
18677
18870
  WITH delivered AS (
18678
- UPDATE ${tableName('outbox', options)} AS outbox
18679
- SET delivered_at = now(),
18680
- next_attempt_at = NULL,
18681
- last_error_json = NULL,
18682
- lease_owner = NULL,
18683
- lease_expires_at = NULL,
18684
- payload_json = CASE WHEN EXISTS (
18685
- SELECT 1 FROM ${tableName('runs', options)} AS run
18686
- WHERE run.run_id = outbox.run_id
18687
- AND run.run_event_store = 'planet_scale'
18688
- ) THEN '{}'::jsonb ELSE outbox.payload_json END
18689
- WHERE outbox.event_id = ANY($1::text[])
18690
- AND outbox.delivered_at IS NULL
18691
- AND ($2::text IS NULL OR outbox.lease_owner = $2)
18692
- RETURNING outbox.event_id,
18693
- outbox.event_type,
18694
- (extract(epoch FROM (outbox.delivered_at - outbox.created_at)) * 1000)::float8
18695
- AS lag_ms
18871
+ ${runLedgerAcknowledgeSql(options, '$1', '$2')}
18696
18872
  )
18697
18873
  SELECT event_id,
18698
18874
  event_type,
@@ -18846,6 +19022,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
18846
19022
  );
18847
19023
  const lines = stringArrayPayloadField(event.payload, 'terminalLogLines');
18848
19024
  const lineContexts = terminalLogContextsPayloadField(event.payload, lines);
19025
+ const lineMetadata = terminalLogMetadataPayloadField(event.payload, lines);
18849
19026
  const resultSummary = payloadField(event.payload, 'resultSummary');
18850
19027
  return [
18851
19028
  ...progressEvents.map(normalizeOutboxIngressPreview),
@@ -18859,6 +19036,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
18859
19036
  source: 'worker',
18860
19037
  lines,
18861
19038
  ...(lineContexts ? { lineContexts } : {}),
19039
+ ...(lineMetadata ? { lineMetadata } : {}),
18862
19040
  } as PlayRunLedgerEvent,
18863
19041
  ]),
18864
19042
  {
@@ -18888,6 +19066,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
18888
19066
  );
18889
19067
  const lines = stringArrayPayloadField(event.payload, 'terminalLogLines');
18890
19068
  const lineContexts = terminalLogContextsPayloadField(event.payload, lines);
19069
+ const lineMetadata = terminalLogMetadataPayloadField(event.payload, lines);
18891
19070
  const result = terminalRunResultForLedger(
18892
19071
  payloadField(event.payload, 'result'),
18893
19072
  );
@@ -18904,6 +19083,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
18904
19083
  source: 'worker',
18905
19084
  lines,
18906
19085
  ...(lineContexts ? { lineContexts } : {}),
19086
+ ...(lineMetadata ? { lineMetadata } : {}),
18907
19087
  } as PlayRunLedgerEvent,
18908
19088
  ]),
18909
19089
  {
@@ -19148,7 +19328,7 @@ export async function listPostgresSchedulerOutboxMaterializationPlans(
19148
19328
  * - `local_mint`: sign the token in-process with `BETTER_AUTH_SECRET`. ONLY the
19149
19329
  * app (Vercel) legitimately holds the signing key, so this is app-side only
19150
19330
  * (read-model heal path). It is the AUTHORITATIVE projector.
19151
- * - `gateway`: always prove the active outbox lease to the launch-pinned
19331
+ * - `gateway`: for every app-runtime append, prove the active outbox lease to the launch-pinned
19152
19332
  * deployment and receive a short-lived executor JWT
19153
19333
  * (`requestRunLedgerDispatchToken`), even if the caller supplied a
19154
19334
  * `runtimeApi`. The Fly worker has no signing key and MUST use this. A
@@ -19219,7 +19399,7 @@ export type RunLedgerOutboxDispatchInput = {
19219
19399
  /** Only the existing background projector indexes deferred raw inputs. */
19220
19400
  materializeObservability?: boolean;
19221
19401
  /**
19222
- * Optional runtime context. `gateway` authority always replaces its
19402
+ * Optional runtime context. For an app-runtime append, `gateway` replaces its
19223
19403
  * credentials with a fresh lease-scoped token and keeps only supported
19224
19404
  * transport options; `local_mint` and the default authority preserve the
19225
19405
  * explicit context as before.
@@ -19518,13 +19698,16 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19518
19698
  ): Promise<PostgresSchedulerOutboxProjectionResult> {
19519
19699
  const dispatchStartedAtMs = Date.now();
19520
19700
  const tokenAuthority = input.tokenAuthority ?? 'local_mint';
19521
- const dispatchLaunchSql =
19522
- !input.searchOnly && (tokenAuthority === 'gateway' || !input.runtimeApi)
19523
- ? runLedgerDispatchLaunchSql('runs')
19524
- : 'NULL::jsonb';
19525
- let claimedLaunchJson: unknown;
19526
19701
  const leaseOwner =
19527
19702
  input.leaseOwner?.trim() || `run-ledger-outbox-dispatch:${randomUUID()}`;
19703
+ const needsLaunch =
19704
+ !input.searchOnly && (tokenAuthority === 'gateway' || !input.runtimeApi);
19705
+ const launchSelection = needsLaunch
19706
+ ? `CASE
19707
+ WHEN ${runLedgerSearchOnlyOutboxFactSql('head')} THEN NULL::jsonb
19708
+ ELSE ${runLedgerDispatchLaunchSql('runs')}
19709
+ END`
19710
+ : 'NULL::jsonb';
19528
19711
  const continuation: { current: RunLedgerOutboxDrainScope | null } = {
19529
19712
  current: scope ?? null,
19530
19713
  };
@@ -19576,13 +19759,14 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19576
19759
  const selected = targetRunId
19577
19760
  ? await client.query<{
19578
19761
  run_id: string;
19579
- dispatch_launch_json?: unknown;
19762
+ dispatch_launch_json: unknown;
19580
19763
  }>(
19581
19764
  `
19582
19765
  WITH head AS MATERIALIZED (
19583
19766
  SELECT pending.run_id,
19584
19767
  pending.event_id,
19585
19768
  pending.event_type,
19769
+ pending.payload_json,
19586
19770
  pending.created_at,
19587
19771
  pending.next_attempt_at,
19588
19772
  pending.failed_at
@@ -19593,7 +19777,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19593
19777
  ORDER BY ${runLedgerOrderSql('pending')}
19594
19778
  LIMIT 1
19595
19779
  )
19596
- SELECT runs.run_id, ${dispatchLaunchSql} AS dispatch_launch_json
19780
+ SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json
19597
19781
  FROM head
19598
19782
  JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
19599
19783
  WHERE head.failed_at IS NULL
@@ -19614,7 +19798,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19614
19798
  )
19615
19799
  : await client.query<{
19616
19800
  run_id: string;
19617
- dispatch_launch_json?: unknown;
19801
+ dispatch_launch_json: unknown;
19618
19802
  }>(`
19619
19803
  WITH ${
19620
19804
  input.searchOnly === true
@@ -19627,6 +19811,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19627
19811
  pending.run_id,
19628
19812
  pending.event_id,
19629
19813
  pending.event_type,
19814
+ pending.payload_json,
19630
19815
  pending.created_at,
19631
19816
  pending.next_attempt_at,
19632
19817
  pending.failed_at
@@ -19635,7 +19820,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19635
19820
  AND ${searchLaneSql('pending')} = ${input.searchOnly === true ? 'true' : 'false'}
19636
19821
  ORDER BY pending.run_id, ${runLedgerOrderSql('pending')}
19637
19822
  )
19638
- SELECT runs.run_id, ${dispatchLaunchSql} AS dispatch_launch_json
19823
+ SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json
19639
19824
  FROM heads AS head
19640
19825
  JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
19641
19826
  WHERE head.failed_at IS NULL
@@ -19680,24 +19865,28 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19680
19865
  options,
19681
19866
  )
19682
19867
  : [];
19683
- // Authority/routing is immutable and already lives on the locked Run.
19684
- // Capture only its existing narrow projection with the successful lease
19685
- // claim, then release the checkout before requesting gateway authority.
19686
- // The gateway still independently proves that this lease is current.
19687
- const launchJson = selected.rows[0]?.dispatch_launch_json;
19688
- if (claimed.length > 0 && launchJson != null) {
19689
- claimedLaunchJson = launchJson;
19690
- }
19691
19868
  await client.query('COMMIT');
19692
- return claimed;
19869
+ // The launch is immutable. Carry only its authority/routing fields from
19870
+ // the run we already locked, rather than queuing another scarce checkout
19871
+ // between the claim and gateway request. The gateway still proves the
19872
+ // active outbox lease; no database checkout spans either HTTP hop.
19873
+ return {
19874
+ events: claimed,
19875
+ launchJson: selected.rows[0]?.dispatch_launch_json,
19876
+ };
19693
19877
  } catch (error) {
19694
19878
  await client.query('ROLLBACK');
19695
19879
  throw error;
19696
19880
  }
19697
19881
  };
19698
- let events = scope?.nextClaimed.length
19882
+ const preclaimed = scope?.nextClaimed.length
19699
19883
  ? scope.nextClaimed.splice(0)
19884
+ : null;
19885
+ const claimed = preclaimed
19886
+ ? { events: preclaimed, launchJson: null }
19700
19887
  : await withClient(claimEvents);
19888
+ const { launchJson } = claimed;
19889
+ let events = claimed.events;
19701
19890
  if (events.length === 0) {
19702
19891
  return {
19703
19892
  attempted: 0,
@@ -19806,13 +19995,12 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19806
19995
  return null;
19807
19996
  };
19808
19997
  try {
19809
- if (!input.materializeObservability) {
19810
- const handedOff = await materializeDeferredObservability();
19811
- if (handedOff) return handedOff;
19812
- }
19998
+ // The ledger converter rejects raw deferred facts. Inline callers hand
19999
+ // them off; the background owner projects them before inspecting the
20000
+ // compact lifecycle payload or deciding that it needs no HTTP append.
20001
+ const initialHandedOff = await materializeDeferredObservability();
20002
+ if (initialHandedOff) return initialHandedOff;
19813
20003
  if (input.searchOnly) {
19814
- const handedOff = await materializeDeferredObservability();
19815
- if (handedOff) return handedOff;
19816
20004
  if (
19817
20005
  events.some(
19818
20006
  (event) =>
@@ -19852,18 +20040,23 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19852
20040
  };
19853
20041
  }
19854
20042
  const first = events[0]!;
19855
- const claimedLaunch =
19856
- claimedLaunchJson == null
19857
- ? undefined
19858
- : parseLaunchJson(claimedLaunchJson);
20043
+ let ledgerEvents = RUN_SCHEDULER_OUTBOX_START_EVENT_TYPES.includes(
20044
+ first.eventType,
20045
+ )
20046
+ ? null
20047
+ : events.flatMap(buildRunLedgerEventsFromPostgresSchedulerOutbox);
20048
+ // Search-only facts are already durable and have no ledger projection.
20049
+ // Validate and acknowledge them without acquiring runtime authority or
20050
+ // renewing a network-delivery lease. A following terminal still uses the
20051
+ // same fenced continuation and loads authority only when it needs a write.
19859
20052
  if (
19860
20053
  scope &&
19861
20054
  (scope.runId !== first.runId || scope.leaseOwner !== leaseOwner)
19862
20055
  ) {
19863
20056
  throw new Error('Run-ledger continuation is outside its dispatch scope.');
19864
20057
  }
19865
- // Minted executor authority is reused only inside this finite run/owner
19866
- // drain and only while it can cover the HTTP request plus its SQL renewal.
20058
+ const needsRuntimeApi = ledgerEvents === null || ledgerEvents.length > 0;
20059
+ let runtimeApi: WorkerRuntimeApiContext | null = null;
19867
20060
  const reuseMarginMs =
19868
20061
  Math.max(
19869
20062
  1,
@@ -19873,24 +20066,36 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19873
20066
  ),
19874
20067
  ) + RUNTIME_RELIABILITY_POLICY.schedulerDatabase.queryTimeoutMs;
19875
20068
  const cachedAuthority = authorityScope.authority;
20069
+ const resolveRunRuntimeApi = () =>
20070
+ launchJson !== null
20071
+ ? resolvePostgresSchedulerRuntimeApiForRun(
20072
+ launchJson,
20073
+ first.runId,
20074
+ first.leaseOwner
20075
+ ? { eventId: first.eventId, leaseOwner: first.leaseOwner }
20076
+ : null,
20077
+ tokenAuthority,
20078
+ options,
20079
+ input.runtimeApi?.signal,
20080
+ )
20081
+ : loadPostgresSchedulerRuntimeApiForRun(
20082
+ withClient,
20083
+ first.runId,
20084
+ first.leaseOwner
20085
+ ? { eventId: first.eventId, leaseOwner: first.leaseOwner }
20086
+ : null,
20087
+ tokenAuthority,
20088
+ options,
20089
+ input.runtimeApi?.signal,
20090
+ );
19876
20091
  const gatewayRuntimeApi =
19877
- tokenAuthority === 'gateway'
20092
+ needsRuntimeApi && tokenAuthority === 'gateway'
19878
20093
  ? cachedAuthority?.runId === first.runId &&
19879
20094
  cachedAuthority.leaseOwner === leaseOwner &&
19880
20095
  cachedAuthority.expiresAtMs !== null &&
19881
20096
  cachedAuthority.expiresAtMs > Date.now() + reuseMarginMs
19882
20097
  ? cachedAuthority.runtimeApi
19883
- : await loadPostgresSchedulerRuntimeApiForRun(
19884
- withClient,
19885
- first.runId,
19886
- first.leaseOwner
19887
- ? { eventId: first.eventId, leaseOwner: first.leaseOwner }
19888
- : null,
19889
- tokenAuthority,
19890
- options,
19891
- input.runtimeApi?.signal,
19892
- claimedLaunch,
19893
- )
20098
+ : await resolveRunRuntimeApi()
19894
20099
  : null;
19895
20100
  if (
19896
20101
  gatewayRuntimeApi &&
@@ -19902,66 +20107,53 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19902
20107
  'Run Ledger dispatch origin changed within an immutable launch.',
19903
20108
  );
19904
20109
  }
19905
- const resolvedRuntimeApi =
19906
- gatewayRuntimeApi ??
19907
- input.runtimeApi ??
19908
- (await loadPostgresSchedulerRuntimeApiForRun(
19909
- withClient,
19910
- first.runId,
19911
- first.leaseOwner
19912
- ? { eventId: first.eventId, leaseOwner: first.leaseOwner }
19913
- : null,
19914
- tokenAuthority,
19915
- options,
19916
- undefined,
19917
- claimedLaunch,
19918
- ));
19919
- const runtimeApi = {
19920
- ...resolvedRuntimeApi,
19921
- ...(gatewayRuntimeApi && input.runtimeApi
19922
- ? {
19923
- ...(input.runtimeApi.fetch
19924
- ? { fetch: input.runtimeApi.fetch }
19925
- : {}),
19926
- ...(input.runtimeApi.requestTimeoutMs !== undefined
19927
- ? { requestTimeoutMs: input.runtimeApi.requestTimeoutMs }
19928
- : {}),
19929
- ...(input.runtimeApi.onResponseMeta
19930
- ? { onResponseMeta: input.runtimeApi.onResponseMeta }
19931
- : {}),
19932
- ...(input.runtimeApi.runtimeTestFaultHeader !== undefined
19933
- ? {
19934
- runtimeTestFaultHeader:
19935
- input.runtimeApi.runtimeTestFaultHeader,
19936
- }
19937
- : {}),
19938
- ...(input.runtimeApi.signal
19939
- ? { signal: input.runtimeApi.signal }
19940
- : {}),
19941
- }
19942
- : {}),
19943
- retryPolicy: 'none' as const,
19944
- // The caller-owned admission deadline (see `signal` above) wins.
19945
- ...(input.signal ? { signal: input.signal } : {}),
19946
- };
19947
- const tokenExpiresAtMs = runtimeApiTokenExpiresAtMs(
19948
- runtimeApi.executorToken,
19949
- );
19950
- if (tokenAuthority === 'gateway' && gatewayRuntimeApi) {
19951
- authorityScope.authority = {
19952
- runId: first.runId,
19953
- leaseOwner,
19954
- runtimeApi,
19955
- expiresAtMs: tokenExpiresAtMs,
20110
+ if (needsRuntimeApi) {
20111
+ const resolvedRuntimeApi =
20112
+ gatewayRuntimeApi ?? input.runtimeApi ?? (await resolveRunRuntimeApi());
20113
+ runtimeApi = {
20114
+ ...resolvedRuntimeApi,
20115
+ ...(gatewayRuntimeApi && input.runtimeApi
20116
+ ? {
20117
+ ...(input.runtimeApi.fetch
20118
+ ? { fetch: input.runtimeApi.fetch }
20119
+ : {}),
20120
+ ...(input.runtimeApi.requestTimeoutMs !== undefined
20121
+ ? { requestTimeoutMs: input.runtimeApi.requestTimeoutMs }
20122
+ : {}),
20123
+ ...(input.runtimeApi.onResponseMeta
20124
+ ? { onResponseMeta: input.runtimeApi.onResponseMeta }
20125
+ : {}),
20126
+ ...(input.runtimeApi.runtimeTestFaultHeader !== undefined
20127
+ ? {
20128
+ runtimeTestFaultHeader:
20129
+ input.runtimeApi.runtimeTestFaultHeader,
20130
+ }
20131
+ : {}),
20132
+ ...(input.runtimeApi.signal
20133
+ ? { signal: input.runtimeApi.signal }
20134
+ : {}),
20135
+ }
20136
+ : {}),
20137
+ retryPolicy: 'none' as const,
20138
+ // The caller-owned admission deadline (see `signal` above) wins.
20139
+ ...(input.signal ? { signal: input.signal } : {}),
19956
20140
  };
20141
+ if (tokenAuthority === 'gateway' && gatewayRuntimeApi) {
20142
+ authorityScope.authority = {
20143
+ runId: first.runId,
20144
+ leaseOwner,
20145
+ runtimeApi,
20146
+ expiresAtMs: runtimeApiTokenExpiresAtMs(runtimeApi.executorToken),
20147
+ };
20148
+ }
20149
+ authorityReadyAtMs = Date.now();
19957
20150
  }
19958
- authorityReadyAtMs = Date.now();
19959
20151
  // The initial claim deliberately uses the old contract until its immutable
19960
20152
  // launch receiver advertises support. Widen under the same run/owner fence
19961
20153
  // before the first HTTP append, so an already-finished run need not pay one
19962
20154
  // authority/append/ACK round trip for every ingress predecessor.
19963
20155
  if (
19964
- runtimeApi.runLedgerIngressPrefixBatching === true &&
20156
+ runtimeApi?.runLedgerIngressPrefixBatching === true &&
19965
20157
  events.length === 1 &&
19966
20158
  first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents
19967
20159
  ) {
@@ -19982,32 +20174,34 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19982
20174
  options,
19983
20175
  ),
19984
20176
  );
19985
- // Expansion uses SKIP LOCKED on the run fence. A concurrent lifecycle
19986
- // writer can make it empty while our original head lease remains live.
19987
- // Keep that singleton; mandatory renewal below still rejects expired
19988
- // or replaced ownership and freezes its receipt before any HTTP append.
19989
- if (widened.length > 0) {
19990
- if (widened[0]!.eventId !== first.eventId) {
19991
- throw new RunLedgerProjectionLeaseLostError(
19992
- 'head_lease_owner_mismatch',
19993
- );
19994
- }
19995
- events = widened;
20177
+ if (widened[0]?.eventId !== first.eventId) {
20178
+ throw new RunLedgerProjectionLeaseLostError(
20179
+ 'head_lease_owner_mismatch',
20180
+ );
19996
20181
  }
20182
+ events = widened;
19997
20183
  }
19998
20184
  // Widening can add deferred facts to the lease. Project the complete
19999
20185
  // frozen prefix before any HTTP append, including its newly claimed rows.
20000
20186
  const handedOff = await materializeDeferredObservability();
20001
20187
  if (handedOff) return handedOff;
20188
+ if (ledgerEvents !== null) {
20189
+ ledgerEvents = events.flatMap(
20190
+ buildRunLedgerEventsFromPostgresSchedulerOutbox,
20191
+ );
20192
+ }
20002
20193
  const renewDelivery = async () => {
20003
20194
  const currentAuthority = authorityScope.authority;
20004
20195
  if (
20196
+ runtimeApi &&
20005
20197
  tokenAuthority === 'gateway' &&
20006
20198
  currentAuthority?.runId === first.runId &&
20007
20199
  currentAuthority.leaseOwner === leaseOwner &&
20008
20200
  currentAuthority.expiresAtMs !== null &&
20009
20201
  currentAuthority.expiresAtMs <= Date.now() + reuseMarginMs
20010
20202
  ) {
20203
+ // Initial delivery reuses the claim's routing fields. Renewal must
20204
+ // reread authority so an origin change cannot pass under that cache.
20011
20205
  const refreshed = await loadPostgresSchedulerRuntimeApiForRun(
20012
20206
  withClient,
20013
20207
  first.runId,
@@ -20041,7 +20235,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20041
20235
  leaseOwner,
20042
20236
  leaseSeconds: input.leaseSeconds ?? 120,
20043
20237
  freezeReceipt:
20044
- runtimeApi.runLedgerIngressPrefixBatching === true &&
20238
+ runtimeApi?.runLedgerIngressPrefixBatching === true &&
20045
20239
  events.some(
20046
20240
  (event) =>
20047
20241
  event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents,
@@ -20051,37 +20245,33 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20051
20245
  ),
20052
20246
  );
20053
20247
  };
20054
- const append = async () => {
20055
- // A pending start is always claimed alone because later events require its
20056
- // snapshot. It cannot share the append endpoint, so retain that mutation
20057
- // as its own exact delivery.
20058
- if (first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runCreated) {
20059
- await renewDelivery();
20060
- await projectPostgresSchedulerCreatedEvent(runtimeApi, first);
20061
- } else if (
20062
- first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runStarted
20063
- ) {
20064
- await renewDelivery();
20065
- const leaseOwner = first.leaseOwner;
20066
- const launch = leaseOwner
20067
- ? await withClient((client) =>
20068
- readRunLedgerDispatchLaunchForLease(
20069
- client,
20070
- {
20071
- runId: first.runId,
20072
- eventId: first.eventId,
20073
- leaseOwner,
20074
- },
20075
- options,
20076
- ),
20077
- )
20078
- : null;
20079
- await projectPostgresSchedulerStartEvent(runtimeApi, first, launch);
20080
- } else {
20081
- const ledgerEvents = events.flatMap(
20082
- buildRunLedgerEventsFromPostgresSchedulerOutbox,
20083
- );
20084
- if (ledgerEvents.length > 0) {
20248
+ if (needsRuntimeApi && runtimeApi) {
20249
+ const append = async (runtimeApi: WorkerRuntimeApiContext) => {
20250
+ // A pending start is always claimed alone because later events require
20251
+ // its snapshot. Retain that mutation as its own exact delivery.
20252
+ if (first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runCreated) {
20253
+ await renewDelivery();
20254
+ await projectPostgresSchedulerCreatedEvent(runtimeApi, first);
20255
+ } else if (
20256
+ first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runStarted
20257
+ ) {
20258
+ await renewDelivery();
20259
+ const startLeaseOwner = first.leaseOwner;
20260
+ const launch = startLeaseOwner
20261
+ ? await withClient((client) =>
20262
+ readRunLedgerDispatchLaunchForLease(
20263
+ client,
20264
+ {
20265
+ runId: first.runId,
20266
+ eventId: first.eventId,
20267
+ leaseOwner: startLeaseOwner,
20268
+ },
20269
+ options,
20270
+ ),
20271
+ )
20272
+ : null;
20273
+ await projectPostgresSchedulerStartEvent(runtimeApi, first, launch);
20274
+ } else if (ledgerEvents && ledgerEvents.length > 0) {
20085
20275
  const terminal = [...events]
20086
20276
  .reverse()
20087
20277
  .find((event) =>
@@ -20092,6 +20282,8 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20092
20282
  const batchDeliveryKey = createHash('sha256')
20093
20283
  .update(events.map((event) => event.deliveryKey).join('\n'))
20094
20284
  .digest('hex');
20285
+ // The transport renews before every HTTP chunk; another renewal here
20286
+ // would add a checkout without protecting an intervening side effect.
20095
20287
  await projectRunEventsViaAppRuntime(runtimeApi, {
20096
20288
  playId: first.runId,
20097
20289
  events: ledgerEvents,
@@ -20107,25 +20299,26 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20107
20299
  deliveryDigest: batchDeliveryKey,
20108
20300
  }),
20109
20301
  });
20110
- } else {
20111
- await renewDelivery();
20112
20302
  }
20303
+ };
20304
+ try {
20305
+ await append(runtimeApi);
20306
+ } catch (error) {
20307
+ // Refresh a reused authority once through this delivery's active lease.
20308
+ // Both attempts retain the same fenced append and idempotency key.
20309
+ if (
20310
+ !gatewayRuntimeApi?.refreshExecutorToken ||
20311
+ !(error instanceof AppRuntimeApiResponseError) ||
20312
+ error.status !== 401
20313
+ )
20314
+ throw error;
20315
+ runtimeApi.executorToken = await gatewayRuntimeApi.refreshExecutorToken();
20316
+ await append(runtimeApi);
20113
20317
  }
20114
- };
20115
- try {
20116
- await append();
20117
- } catch (error) {
20118
- // A cached authority can be invalidated by signer rotation. Refresh it
20119
- // once through this delivery's active lease; the append remains fenced
20120
- // and carries the same idempotency key on both attempts.
20121
- if (
20122
- !gatewayRuntimeApi?.refreshExecutorToken ||
20123
- !(error instanceof AppRuntimeApiResponseError) ||
20124
- error.status !== 401
20125
- )
20126
- throw error;
20127
- runtimeApi.executorToken = await gatewayRuntimeApi.refreshExecutorToken();
20128
- await append();
20318
+ } else if (tokenAuthority !== 'gateway') {
20319
+ // App-side acknowledgement retains lease validation. The gateway can
20320
+ // acknowledge a durable search-only fact without a network lease hop.
20321
+ await renewDelivery();
20129
20322
  }
20130
20323
  appendFinishedAtMs = Date.now();
20131
20324
  const acknowledgement = await acknowledgeRunLedgerPrefixWithContinuation(
@@ -20293,10 +20486,16 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20293
20486
  throw settlementError;
20294
20487
  }
20295
20488
  });
20489
+ const failure = runLedgerDeliveryFailure({ error });
20296
20490
  console.warn('[run-ledger-outbox.batch_dispatch_failed]', {
20297
20491
  runId: input.runId ?? null,
20298
20492
  eventIds: events.map((event) => event.eventId),
20299
20493
  error: error instanceof Error ? error.message : String(error),
20494
+ code: failure.code,
20495
+ failureReason: failure.failureReason,
20496
+ ...(failure.projectionMismatch
20497
+ ? { projectionMismatch: failure.projectionMismatch }
20498
+ : {}),
20300
20499
  });
20301
20500
  return {
20302
20501
  attempted: events.length,
@@ -20405,7 +20604,7 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
20405
20604
  input: RunLedgerOutboxDispatchInput;
20406
20605
  events: PostgresSchedulerOutboxEvent[];
20407
20606
  leaseOwner: string;
20408
- runtimeApi: WorkerRuntimeApiContext;
20607
+ runtimeApi: WorkerRuntimeApiContext | null;
20409
20608
  scope?: RunLedgerOutboxDrainScope;
20410
20609
  continuation: { current: RunLedgerOutboxDrainScope | null };
20411
20610
  continuationAllowance: number | null;
@@ -20421,81 +20620,86 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
20421
20620
  !!input.scope ||
20422
20621
  (input.input.continueThroughTerminal === true && !input.input.headEventId);
20423
20622
  return withClient(async (client) => {
20424
- if (retainContinuation) {
20425
- await client.query('BEGIN');
20426
- }
20427
- try {
20623
+ const acknowledge = async () => {
20428
20624
  if (retainContinuation) {
20429
- await client.query(
20430
- `SELECT set_config('lock_timeout', $1, true), set_config('statement_timeout', $2, true)`,
20431
- [
20432
- `${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
20433
- `${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
20434
- ],
20435
- );
20436
- // Keep the acknowledged prefix and its successor under one run lock.
20437
- // No competing projector can take the next head while this ACK's
20438
- // response is travelling back to the lease holder.
20439
- await client.query(
20440
- `SELECT run_id FROM ${tableName('runs', options)} WHERE run_id = $1 FOR UPDATE`,
20441
- [runId],
20442
- );
20625
+ await client.query('BEGIN');
20443
20626
  }
20444
- const deliveredRows = await markPostgresSchedulerOutboxDeliveredRows(
20445
- client,
20446
- input.events.map((event) => event.eventId),
20447
- { ...options, leaseOwner: input.leaseOwner },
20448
- );
20449
- let continuation: RunLedgerOutboxDrainScope | null = null;
20450
- if (
20451
- retainContinuation &&
20452
- input.continuationAllowance !== 0 &&
20453
- deliveredRows.length === input.events.length &&
20454
- !input.events.some((event) => event.eventId === targetEventId)
20455
- ) {
20456
- const target = await readPostgresSchedulerOutboxDeliveryStatus(
20457
- client,
20458
- { runId, eventId: targetEventId },
20459
- options,
20460
- );
20461
- if (target.exists && !target.delivered && !target.blocked) {
20462
- const nextClaimed = await readPendingPostgresSchedulerOutboxEvents(
20463
- client,
20464
- {
20465
- runId,
20466
- leaseOwner: input.leaseOwner,
20467
- leaseSeconds: input.input.leaseSeconds ?? 120,
20468
- limit: Math.min(
20469
- RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
20470
- input.runtimeApi.runLedgerIngressPrefixBatching === true
20471
- ? RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS
20472
- : (input.input.limit ?? 1),
20473
- input.continuationAllowance ??
20474
- RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
20475
- ),
20476
- allowIngressPrefix:
20477
- input.runtimeApi.runLedgerIngressPrefixBatching === true,
20478
- inTransaction: true,
20479
- },
20480
- options,
20627
+ try {
20628
+ if (retainContinuation) {
20629
+ await client.query(
20630
+ `SELECT set_config('lock_timeout', $1, true), set_config('statement_timeout', $2, true)`,
20631
+ [
20632
+ `${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
20633
+ `${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
20634
+ ],
20635
+ );
20636
+ // Keep the acknowledged prefix and its successor under one run lock.
20637
+ // No competing projector can take the next head while this ACK's
20638
+ // response is travelling back to the lease holder.
20639
+ await client.query(
20640
+ `SELECT run_id FROM ${tableName('runs', options)} WHERE run_id = $1 FOR UPDATE`,
20641
+ [runId],
20481
20642
  );
20482
- if (nextClaimed.length > 0) {
20483
- continuation = input.scope ?? {
20484
- runId,
20485
- leaseOwner: input.leaseOwner,
20486
- targetEventId,
20487
- nextClaimed: [],
20488
- };
20489
- continuation.nextClaimed = nextClaimed;
20490
- input.continuation.current = continuation;
20491
- }
20492
20643
  }
20644
+ const eventIds = input.events.map((event) => event.eventId);
20645
+ const { deliveredRows, events: nextClaimed } =
20646
+ retainContinuation &&
20647
+ input.continuationAllowance !== 0 &&
20648
+ !eventIds.includes(targetEventId)
20649
+ ? await claimPostgresSchedulerOutboxPrefix(
20650
+ client,
20651
+ {
20652
+ runId,
20653
+ leaseOwner: input.leaseOwner,
20654
+ leaseSeconds: input.input.leaseSeconds ?? 120,
20655
+ limit: Math.min(
20656
+ RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
20657
+ input.runtimeApi?.runLedgerIngressPrefixBatching === true
20658
+ ? RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS
20659
+ : (input.input.limit ?? 1),
20660
+ input.continuationAllowance ??
20661
+ RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
20662
+ ),
20663
+ allowIngressPrefix:
20664
+ input.runtimeApi?.runLedgerIngressPrefixBatching === true,
20665
+ },
20666
+ options,
20667
+ { eventIds, targetEventId },
20668
+ )
20669
+ : {
20670
+ deliveredRows: await markPostgresSchedulerOutboxDeliveredRows(
20671
+ client,
20672
+ eventIds,
20673
+ { ...options, leaseOwner: input.leaseOwner },
20674
+ ),
20675
+ events: [],
20676
+ };
20677
+ let continuation: RunLedgerOutboxDrainScope | null = null;
20678
+ if (nextClaimed.length > 0) {
20679
+ continuation = input.scope ?? {
20680
+ runId,
20681
+ leaseOwner: input.leaseOwner,
20682
+ targetEventId,
20683
+ nextClaimed: [],
20684
+ };
20685
+ continuation.nextClaimed = nextClaimed;
20686
+ input.continuation.current = continuation;
20687
+ }
20688
+ if (retainContinuation) await client.query('COMMIT');
20689
+ return { deliveredRows, continuation };
20690
+ } catch (error) {
20691
+ if (retainContinuation) await client.query('ROLLBACK');
20692
+ throw error;
20493
20693
  }
20494
- if (retainContinuation) await client.query('COMMIT');
20495
- return { deliveredRows, continuation };
20694
+ };
20695
+ try {
20696
+ return await acknowledge();
20496
20697
  } catch (error) {
20497
- if (retainContinuation) await client.query('ROLLBACK');
20498
- throw error;
20698
+ if (!isMissingOutboxDeliveryBatchMarker(error)) throw error;
20699
+ // The first statement recorded this operation-local missing column and
20700
+ // rolled back its ACK. Reacquire the run lock and take a fresh snapshot
20701
+ // for the existing compatibility claim, without another ledger append.
20702
+ return await acknowledge();
20499
20703
  }
20500
20704
  });
20501
20705
  }
@@ -20653,37 +20857,34 @@ async function projectPostgresSchedulerStartEvent(
20653
20857
 
20654
20858
  export type { RunLedgerDispatchLaunch } from './run-ledger-dispatch-authority';
20655
20859
 
20656
- async function loadPostgresSchedulerRuntimeApiForRun(
20657
- withClient: <T>(
20658
- operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
20659
- ) => Promise<T>,
20860
+ /** Search-only `run.events` facts settle without consulting launch credentials. */
20861
+ function runLedgerSearchOnlyOutboxFactSql(alias: string): string {
20862
+ const events = `CASE
20863
+ WHEN jsonb_typeof(${alias}.payload_json->'events') = 'array'
20864
+ THEN ${alias}.payload_json->'events'
20865
+ ELSE '[]'::jsonb
20866
+ END`;
20867
+ return `(
20868
+ ${alias}.event_type = '${RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents}'
20869
+ AND ${alias}.payload_json->>'logBodyStore' = 'planet_scale'
20870
+ AND jsonb_array_length(${events}) = 0
20871
+ )`;
20872
+ }
20873
+
20874
+ async function resolvePostgresSchedulerRuntimeApiForRun(
20875
+ launchJson: unknown,
20660
20876
  runId: string,
20661
20877
  dispatchLease: { eventId: string; leaseOwner: string } | null,
20662
20878
  tokenAuthority: RunLedgerOutboxTokenAuthority,
20663
20879
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
20664
20880
  signal?: AbortSignal,
20665
- claimedLaunch?: RunLedgerDispatchLaunch,
20666
20881
  ): Promise<
20667
20882
  WorkerRuntimeApiContext & {
20668
20883
  refreshExecutorToken?: () => Promise<string>;
20669
20884
  invalidateExecutorToken?: () => void;
20670
20885
  }
20671
20886
  > {
20672
- // Release the scarce scheduler checkout before the token authority hop.
20673
- // A slow gateway must not retain either projector pool client, and its
20674
- // launch context must not hydrate customer execution graphs.
20675
- const launch: RunLedgerDispatchLaunch =
20676
- claimedLaunch ??
20677
- (await withClient(async (client) => {
20678
- const result = await client.query<{ launch_json: unknown }>(
20679
- `SELECT ${runLedgerDispatchLaunchSql('runs')} AS launch_json
20680
- FROM ${tableName('runs', options)} AS runs
20681
- WHERE run_id = $1
20682
- LIMIT 1`,
20683
- [runId],
20684
- );
20685
- return parseLaunchJson(result.rows[0]?.launch_json);
20686
- }));
20887
+ const launch = parseRunLedgerDispatchLaunch(launchJson);
20687
20888
  if (!launch?.baseUrl) {
20688
20889
  throw new Error(
20689
20890
  `Run-ledger outbox dispatcher cannot resolve runtime API context for ${runId}.`,
@@ -20743,6 +20944,37 @@ async function loadPostgresSchedulerRuntimeApiForRun(
20743
20944
  };
20744
20945
  }
20745
20946
 
20947
+ async function loadPostgresSchedulerRuntimeApiForRun(
20948
+ withClient: <T>(
20949
+ operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
20950
+ ) => Promise<T>,
20951
+ runId: string,
20952
+ dispatchLease: { eventId: string; leaseOwner: string } | null,
20953
+ tokenAuthority: RunLedgerOutboxTokenAuthority,
20954
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
20955
+ signal?: AbortSignal,
20956
+ ): ReturnType<typeof resolvePostgresSchedulerRuntimeApiForRun> {
20957
+ // A continuation already owns its next event lease, so its launch context
20958
+ // is not part of the preclaim result. Load only the authority fields here.
20959
+ const result = await withClient((client) =>
20960
+ client.query<{ launch_json: unknown }>(
20961
+ `SELECT ${runLedgerDispatchLaunchSql('runs')} AS launch_json
20962
+ FROM ${tableName('runs', options)} AS runs
20963
+ WHERE run_id = $1
20964
+ LIMIT 1`,
20965
+ [runId],
20966
+ ),
20967
+ );
20968
+ return resolvePostgresSchedulerRuntimeApiForRun(
20969
+ result.rows[0]?.launch_json,
20970
+ runId,
20971
+ dispatchLease,
20972
+ tokenAuthority,
20973
+ options,
20974
+ signal,
20975
+ );
20976
+ }
20977
+
20746
20978
  // Only freshly gateway-issued legacy run.progress authority is reused.
20747
20979
  // Prefix advertisements are reacquired for each drain because they select its
20748
20980
  // receipt shape, including recovery across receiver capability changes. This is not
@@ -21071,6 +21303,20 @@ function terminalLogContextsPayloadField(
21071
21303
  return value.map(normalizePlayLogContext);
21072
21304
  }
21073
21305
 
21306
+ function terminalLogMetadataPayloadField(
21307
+ payload: unknown,
21308
+ lines: readonly string[],
21309
+ ): Array<PlayLogMetadata | null> | null {
21310
+ const value = payloadField(payload, 'terminalLogMetadata');
21311
+ if (value === undefined) return null;
21312
+ if (!Array.isArray(value) || value.length !== lines.length) {
21313
+ throw new TypeError(
21314
+ 'Terminal log metadata must align with terminal log lines.',
21315
+ );
21316
+ }
21317
+ return value.map(normalizePlayLogMetadata);
21318
+ }
21319
+
21074
21320
  function ledgerEventsPayloadField(
21075
21321
  payload: unknown,
21076
21322
  key: string,