deepline 0.3.289 → 0.3.291

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +8 -6
  2. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  3. package/dist/bundling-sources/sdk/src/runs/overview.ts +16 -6
  4. package/dist/bundling-sources/sdk/src/types.ts +21 -2
  5. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +41 -11
  6. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +317 -63
  7. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +10 -2
  8. package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +3 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/launch-identity.ts +1 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/log-context.ts +36 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/log-provenance.ts +19 -1
  13. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/worker-terminal-projection.ts +6 -5
  14. package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +2 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +12 -1
  16. package/dist/bundling-sources/shared_libs/play-runtime/run-event-catalog.ts +2 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +74 -33
  18. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +73 -3
  19. package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +39 -2
  20. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +72 -9
  21. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +2 -3
  22. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +77 -10
  23. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +1 -1
  24. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -0
  25. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool-options.ts +145 -0
  26. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +17 -9
  27. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -139
  28. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +19 -5
  29. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +176 -18
  30. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +7 -5
  31. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +1404 -478
  32. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +55 -25
  33. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-dispatch-authority.ts +97 -0
  34. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +34 -15
  35. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +558 -57
  36. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +10 -1
  37. package/dist/bundling-sources/shared_libs/play-runtime/step-progress.ts +6 -1
  38. package/dist/bundling-sources/shared_libs/play-runtime/test-admission-batching.ts +103 -0
  39. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +22 -5
  40. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +35 -0
  41. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +68 -8
  42. package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +2 -0
  43. package/dist/bundling-sources/shared_libs/play-runtime/zerobounce-batching.ts +75 -28
  44. package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +2 -0
  45. package/dist/cli/index.js +195 -582
  46. package/dist/cli/index.mjs +195 -582
  47. package/dist/{client-FR_u0dxT.d.ts → client-Ct4jmvAg.d.ts} +29 -8
  48. package/dist/{client-phK1Ixcl.d.mts → client-Ct9GKrwu.d.mts} +29 -8
  49. package/dist/{compiler-manifest-9SQtUuKb.d.mts → compiler-manifest-qtTNYvBl.d.mts} +2 -0
  50. package/dist/{compiler-manifest-9SQtUuKb.d.ts → compiler-manifest-qtTNYvBl.d.ts} +2 -0
  51. package/dist/index.d.mts +4 -4
  52. package/dist/index.d.ts +4 -4
  53. package/dist/index.js +14 -5
  54. package/dist/index.mjs +14 -5
  55. package/dist/install-integrity.json +7 -4
  56. package/dist/plays/bundle-play-file.d.mts +2 -2
  57. package/dist/plays/bundle-play-file.d.ts +2 -2
  58. package/dist/plays/run-request.d.mts +2 -2
  59. package/dist/plays/run-request.d.ts +2 -2
  60. package/dist/release.d.mts +1 -1
  61. package/dist/release.d.ts +1 -1
  62. package/dist/release.js +1 -1
  63. package/dist/release.mjs +1 -1
  64. package/package.json +1 -1
@@ -1,3 +1,9 @@
1
+ import {
2
+ parseRunLedgerDispatchLaunch,
3
+ runLedgerDispatchLaunchSql,
4
+ type RunLedgerDispatchLaunch,
5
+ } from './run-ledger-dispatch-authority';
6
+ import { normalizeRunLogIds } from '../run-log-buffer';
1
7
  import {
2
8
  retainedPlayPipelineFields,
3
9
  hydrateRetainedPlayPipelines,
@@ -28,6 +34,8 @@ import type {
28
34
  import { PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA } from '../runtime-scheduler-topology';
29
35
  import {
30
36
  RUNTIME_EVENT_CANONICAL_RETENTION_INDEX_READY_MARKER,
37
+ RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD,
38
+ RUNTIME_EVENT_SEARCH_ONLY_FIELD,
31
39
  RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER,
32
40
  } from './runtime-event-search';
33
41
  import type { PlayRunnerRuntimeResource } from '../runner-backends/types';
@@ -83,6 +91,8 @@ import {
83
91
  import { normalizePlayDocflowNodeIoState } from '../docflow-node-io';
84
92
  import {
85
93
  normalizePlayLogContext,
94
+ normalizePlayLogMetadata,
95
+ type PlayLogMetadata,
86
96
  type PlayLogContext,
87
97
  } from '../log-context';
88
98
  import {
@@ -105,7 +115,6 @@ import {
105
115
  RUN_SCHEDULER_OUTBOX_START_EVENT_TYPES,
106
116
  RUN_SCHEDULER_OUTBOX_EVENT_TYPE,
107
117
  RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL,
108
- RUN_SCHEDULER_OUTBOX_PREFIX_STOP_EVENT_TYPES,
109
118
  RUN_SCHEDULER_OUTBOX_TERMINAL_EVENT_TYPES,
110
119
  } from '../run-event-catalog';
111
120
  import { stableStringify } from '../../plays/row-identity';
@@ -336,7 +345,10 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
336
345
  // recovery, which scans only active runs for each release lane.
337
346
  // v57 refreshes log-position lookups after the operator concurrently prepares
338
347
  // channel-specific indexes; outgoing readers retain the existing GIN index.
339
- export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 57;
348
+ // v58 installs deferred outbox readers while retaining synchronous writers
349
+ // until the separately operated, reader-first cutover.
350
+ // v59 adds origin identity bulk projection and independent durable search progress.
351
+ export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 59;
340
352
  const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
341
353
  'scheduler_schema_migrations';
342
354
  export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
@@ -471,11 +483,9 @@ type PostgresSchedulerClaimExecutionOptions = Pick<
471
483
  /** Internal provider-read coordinates which must survive terminal-worker
472
484
  * loss. They are persisted only on the matching durable cleanup obligation. */
473
485
  sandboxCrashDiagnostic?: PostgresSchedulerSandboxCrashDiagnosticRequest;
474
- /** Settled progress facts for a failed runner result, folded before the
475
- * terminal write. They travel in the `run.failed` outbox row ahead of the
476
- * terminal fact, the same carrier a direct completion uses, so the ledger
477
- * settles them before the run's terminal makes its steps immutable. */
478
- failedTerminalProgressEvents?: readonly unknown[];
486
+ /** Final step facts folded into the terminal outbox instead of a separate
487
+ * progress write immediately ahead of the terminal. */
488
+ terminalProgressEvents?: readonly unknown[];
479
489
  /** Forwarded to `failPostgresSchedulerAttempt`: a failure terminal written
480
490
  * for a detached-runner attempt must not overwrite an authoritative pushed
481
491
  * runner terminal that won after the attempt was claimed. */
@@ -1928,6 +1938,24 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
1928
1938
  AND index_row.indrelid = to_regclass($3)
1929
1939
  AND index_row.indisready AND index_row.indisvalid
1930
1940
  )
1941
+ AND EXISTS (
1942
+ SELECT 1 FROM pg_index AS index_row
1943
+ JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
1944
+ JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
1945
+ WHERE index_namespace.nspname = $1
1946
+ AND index_class.relname = 'runtime_scheduler_outbox_lifecycle_pending_order_idx'
1947
+ AND index_row.indrelid = to_regclass($3)
1948
+ AND index_row.indisready AND index_row.indisvalid
1949
+ )
1950
+ AND EXISTS (
1951
+ SELECT 1 FROM pg_index AS index_row
1952
+ JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
1953
+ JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
1954
+ WHERE index_namespace.nspname = $1
1955
+ AND index_class.relname = 'runtime_scheduler_outbox_search_pending_order_idx'
1956
+ AND index_row.indrelid = to_regclass($3)
1957
+ AND index_row.indisready AND index_row.indisvalid
1958
+ )
1931
1959
  AND EXISTS (
1932
1960
  SELECT 1 FROM pg_index AS index_row
1933
1961
  JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
@@ -2461,6 +2489,7 @@ export async function ensurePostgresSchedulerLaunchAdmissionFunctions(
2461
2489
  jsonb_set(
2462
2490
  launch_json - ARRAY[
2463
2491
  'launchIdentityDigest',
2492
+ 'runtimeLogIdentity',
2464
2493
  'executorToken',
2465
2494
  'inputFileId',
2466
2495
  'preloadedDbSessionUnwrapKey',
@@ -3944,6 +3973,26 @@ export async function ensurePostgresSchedulerSchema(
3944
3973
  )
3945
3974
  WHERE delivered_at IS NULL
3946
3975
  `);
3976
+ await client.query(`
3977
+ CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_lifecycle_pending_order_idx
3978
+ ON ${tableName('outbox', options)} (
3979
+ run_id,
3980
+ (CASE event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END),
3981
+ created_at,
3982
+ event_id
3983
+ )
3984
+ WHERE delivered_at IS NULL AND ${searchLaneSql('outbox')} = false
3985
+ `);
3986
+ await client.query(`
3987
+ CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_search_pending_order_idx
3988
+ ON ${tableName('outbox', options)} (
3989
+ run_id,
3990
+ (CASE event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END),
3991
+ created_at,
3992
+ event_id
3993
+ )
3994
+ WHERE delivered_at IS NULL AND ${searchLaneSql('outbox')} = true
3995
+ `);
3947
3996
  await client.query(`
3948
3997
  CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_lease_health_idx
3949
3998
  ON ${tableName('outbox', options)}
@@ -4094,6 +4143,22 @@ const POSTGRES_SCHEDULER_RUNTIME_ADMIN_INDEXES = [
4094
4143
  created_at, event_id)
4095
4144
  WHERE delivered_at IS NULL`,
4096
4145
  },
4146
+ {
4147
+ name: 'runtime_scheduler_outbox_lifecycle_pending_order_idx',
4148
+ table: 'outbox',
4149
+ definition: `(run_id,
4150
+ (CASE event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END),
4151
+ created_at, event_id)
4152
+ WHERE delivered_at IS NULL AND ${searchLaneSql('outbox')} = false`,
4153
+ },
4154
+ {
4155
+ name: 'runtime_scheduler_outbox_search_pending_order_idx',
4156
+ table: 'outbox',
4157
+ definition: `(run_id,
4158
+ (CASE event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END),
4159
+ created_at, event_id)
4160
+ WHERE delivered_at IS NULL AND ${searchLaneSql('outbox')} = true`,
4161
+ },
4097
4162
  {
4098
4163
  name: 'runtime_scheduler_outbox_lease_health_idx',
4099
4164
  table: 'outbox',
@@ -10318,8 +10383,8 @@ export async function recordPostgresSchedulerRunnerTerminal(
10318
10383
  const recorded = await client.query<{ run_id: string }>(
10319
10384
  `
10320
10385
  WITH accepted AS MATERIALIZED (
10321
- -- Match direct completion's Run -> attempt lock order. The log
10322
- -- acceptance trigger also locks the Run for sequence assignment.
10386
+ -- Record the authoritative terminal boundary before accepting the
10387
+ -- attempt payload; deferred log indexing owns no Run row lock.
10323
10388
  SELECT clock_timestamp() AS accepted_at
10324
10389
  FROM ${tableName('runs', options)}
10325
10390
  WHERE run_id = $1
@@ -10758,6 +10823,8 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
10758
10823
  'producerAttempt', $2::integer,
10759
10824
  'terminalLogLines', $7::jsonb,
10760
10825
  'terminalLogContexts', $17::jsonb,
10826
+ 'terminalLogMetadata', $20::jsonb,
10827
+ 'terminalLogIds', $19::jsonb->'terminalLogIds',
10761
10828
  'terminalProgressEvents', coalesce((
10762
10829
  SELECT accepted.runner_log_payload_json->'terminalProgressEvents'
10763
10830
  FROM completed_attempt AS accepted
@@ -10802,6 +10869,7 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
10802
10869
  JSON.stringify(terminalLogProjection.lineContexts),
10803
10870
  stringifyPostgresJson(terminalWorkCounterSegment ?? null),
10804
10871
  stringifyPostgresJson(runnerTerminalLogPayload(input)),
10872
+ JSON.stringify(terminalLogProjection.lineMetadata),
10805
10873
  ],
10806
10874
  );
10807
10875
  if (completed.rows.length > 0) return 'completed';
@@ -13134,7 +13202,10 @@ export async function completePostgresSchedulerAttempt(
13134
13202
  runtimeTiming?: PlayRunnerRuntimeTiming | null;
13135
13203
  terminalLogLines?: readonly string[] | null;
13136
13204
  terminalLogContexts?: readonly (PlayLogContext | null)[] | null;
13205
+ terminalLogMetadata?: readonly (PlayLogMetadata | null)[] | null;
13206
+ terminalLogIds?: readonly (string | null)[] | null;
13137
13207
  terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
13208
+ terminalProgressEvents?: readonly unknown[];
13138
13209
  },
13139
13210
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
13140
13211
  ): Promise<boolean> {
@@ -13151,10 +13222,24 @@ export async function completePostgresSchedulerAttempt(
13151
13222
  'Terminal log contexts must align with terminal log lines.',
13152
13223
  );
13153
13224
  }
13225
+ if (
13226
+ input.terminalLogMetadata != null &&
13227
+ input.terminalLogMetadata.length !== (input.terminalLogLines ?? []).length
13228
+ ) {
13229
+ throw new TypeError(
13230
+ 'Terminal log metadata must align with terminal log lines.',
13231
+ );
13232
+ }
13233
+ const terminalLogIds = normalizeRunLogIds(
13234
+ input.terminalLogIds,
13235
+ input.terminalLogLines ?? [],
13236
+ );
13154
13237
  const terminalLogEntries = (input.terminalLogLines ?? [])
13155
13238
  .map((line, index) => ({
13156
13239
  line: line.trim(),
13157
13240
  context: input.terminalLogContexts?.[index] ?? null,
13241
+ metadata: input.terminalLogMetadata?.[index] ?? null,
13242
+ entryId: terminalLogIds?.[index] ?? null,
13158
13243
  }))
13159
13244
  .filter((entry) => entry.line.length > 0)
13160
13245
  .slice(-LOG_TAIL_LIMIT);
@@ -13162,6 +13247,9 @@ export async function completePostgresSchedulerAttempt(
13162
13247
  const terminalLogContexts = terminalLogEntries.map((entry) =>
13163
13248
  normalizePlayLogContext(entry.context),
13164
13249
  );
13250
+ const terminalLogMetadata = terminalLogEntries.map((entry) =>
13251
+ normalizePlayLogMetadata(entry.metadata),
13252
+ );
13165
13253
  const completed = await client.query<{ run_id: string }>(
13166
13254
  `
13167
13255
  WITH ${runtimeSandboxCustodyLocksSql('run.run_id = $1', options)}, completed AS (
@@ -13340,9 +13428,15 @@ export async function completePostgresSchedulerAttempt(
13340
13428
  'producerAttempt', $2::integer,
13341
13429
  'terminalLogLines', $8::jsonb,
13342
13430
  'terminalLogContexts', $11::jsonb,
13431
+ 'terminalLogMetadata', $13::jsonb,
13432
+ 'terminalLogIds', $15::jsonb,
13343
13433
  'resultSummary', $10::jsonb,
13344
13434
  'terminalWorkCounterSegment', $12::jsonb
13345
- ),
13435
+ ) || CASE
13436
+ WHEN jsonb_array_length($14::jsonb) > 0
13437
+ THEN jsonb_build_object('terminalProgressEvents', $14::jsonb)
13438
+ ELSE '{}'::jsonb
13439
+ END,
13346
13440
  ${causalOutboxCreatedAt('run_terminal.run_id', options)}
13347
13441
  FROM run_terminal
13348
13442
  ON CONFLICT (event_id) DO NOTHING
@@ -13365,6 +13459,13 @@ export async function completePostgresSchedulerAttempt(
13365
13459
  stringifyPostgresJson(resultSummary),
13366
13460
  JSON.stringify(terminalLogContexts),
13367
13461
  stringifyPostgresJson(input.terminalWorkCounterSegment ?? null),
13462
+ JSON.stringify(terminalLogMetadata),
13463
+ JSON.stringify(input.terminalProgressEvents ?? []),
13464
+ stringifyPostgresJson(
13465
+ terminalLogIds
13466
+ ? terminalLogEntries.map((entry) => entry.entryId)
13467
+ : null,
13468
+ ),
13368
13469
  ],
13369
13470
  );
13370
13471
  if (completed.rows.length === 0) {
@@ -13383,13 +13484,15 @@ export async function failPostgresSchedulerAttempt(
13383
13484
  result?: unknown;
13384
13485
  terminalLogLines?: readonly string[] | null;
13385
13486
  terminalLogContexts?: readonly (PlayLogContext | null)[] | null;
13487
+ terminalLogMetadata?: readonly (PlayLogMetadata | null)[] | null;
13488
+ terminalLogIds?: readonly (string | null)[] | null;
13386
13489
  /** Refuse to overwrite an authoritative detached runner terminal that won
13387
13490
  * after claim. Resumable `suspended` receipts may be superseded by the
13388
13491
  * hard ceiling in the same fenced statement. */
13389
13492
  preserveRunnerTerminal?: boolean;
13390
13493
  sandboxCrashDiagnostic?: PostgresSchedulerSandboxCrashDiagnosticRequest;
13391
- /** Progress facts the ledger must apply before `run.failed` (see
13392
- * `failedTerminalProgressEvents`). Omitted from the payload when empty. */
13494
+ /** Progress facts the ledger must apply before the terminal fact. Omitted
13495
+ * from the payload when empty. */
13393
13496
  terminalProgressEvents?: readonly unknown[];
13394
13497
  },
13395
13498
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
@@ -13429,10 +13532,24 @@ export async function failPostgresSchedulerAttempt(
13429
13532
  'Terminal log contexts must align with terminal log lines.',
13430
13533
  );
13431
13534
  }
13535
+ if (
13536
+ input.terminalLogMetadata != null &&
13537
+ input.terminalLogMetadata.length !== (input.terminalLogLines ?? []).length
13538
+ ) {
13539
+ throw new TypeError(
13540
+ 'Terminal log metadata must align with terminal log lines.',
13541
+ );
13542
+ }
13543
+ const terminalLogIds = normalizeRunLogIds(
13544
+ input.terminalLogIds,
13545
+ input.terminalLogLines ?? [],
13546
+ );
13432
13547
  const terminalLogEntries = (input.terminalLogLines ?? [])
13433
13548
  .map((line, index) => ({
13434
13549
  line: line.trim(),
13435
13550
  context: input.terminalLogContexts?.[index] ?? null,
13551
+ metadata: input.terminalLogMetadata?.[index] ?? null,
13552
+ entryId: terminalLogIds?.[index] ?? null,
13436
13553
  }))
13437
13554
  .filter((entry) => entry.line.length > 0)
13438
13555
  .slice(-LOG_TAIL_LIMIT);
@@ -13440,6 +13557,9 @@ export async function failPostgresSchedulerAttempt(
13440
13557
  const terminalLogContexts = terminalLogEntries.map((entry) =>
13441
13558
  normalizePlayLogContext(entry.context),
13442
13559
  );
13560
+ const terminalLogMetadata = terminalLogEntries.map((entry) =>
13561
+ normalizePlayLogMetadata(entry.metadata),
13562
+ );
13443
13563
  const failed = await client.query<{ run_id: string }>(
13444
13564
  `
13445
13565
  WITH ${runtimeSandboxCustodyLocksSql('run.run_id = $1', options)}, failed AS (
@@ -13678,6 +13798,8 @@ export async function failPostgresSchedulerAttempt(
13678
13798
  'producerAttempt', $2::integer,
13679
13799
  'terminalLogLines', $12::jsonb,
13680
13800
  'terminalLogContexts', $14::jsonb,
13801
+ 'terminalLogMetadata', $17::jsonb,
13802
+ 'terminalLogIds', $18::jsonb,
13681
13803
  'resultSummary', $13::jsonb,
13682
13804
  'terminalWorkCounterSegment', $15::jsonb
13683
13805
  ) || CASE
@@ -13711,6 +13833,12 @@ export async function failPostgresSchedulerAttempt(
13711
13833
  JSON.stringify(terminalLogContexts),
13712
13834
  stringifyPostgresJson(terminalWorkCounterSegment ?? null),
13713
13835
  stringifyPostgresJson(input.terminalProgressEvents ?? []),
13836
+ JSON.stringify(terminalLogMetadata),
13837
+ stringifyPostgresJson(
13838
+ terminalLogIds
13839
+ ? terminalLogEntries.map((entry) => entry.entryId)
13840
+ : null,
13841
+ ),
13714
13842
  ],
13715
13843
  );
13716
13844
  return failed.rows.length > 0;
@@ -14413,6 +14541,8 @@ function runnerTerminalLogPayload(input: {
14413
14541
  producerAttempt: input.attempt,
14414
14542
  terminalLogLines: projection.lines,
14415
14543
  terminalLogContexts: projection.lineContexts,
14544
+ terminalLogMetadata: projection.lineMetadata,
14545
+ ...(projection.lineIds ? { terminalLogIds: projection.lineIds } : {}),
14416
14546
  terminalProgressEvents: input.terminalProgressEvents ?? [],
14417
14547
  // Older runner envelopes conveyed these diagnostics only in tagged log
14418
14548
  // text. Preserve their small structured facts before removing that text.
@@ -14433,21 +14563,28 @@ function runnerTerminalLogPayload(input: {
14433
14563
  function terminalLogProjectionForOutbox(
14434
14564
  runnerResult: unknown,
14435
14565
  mode: PostgresSchedulerTerminalLogOutbox | undefined,
14436
- ): { lines: string[]; lineContexts: Array<PlayLogContext | null> } {
14566
+ ): {
14567
+ lines: string[];
14568
+ lineContexts: Array<PlayLogContext | null>;
14569
+ lineMetadata: Array<PlayLogMetadata | null>;
14570
+ lineIds?: Array<string | null>;
14571
+ } {
14437
14572
  if (
14438
14573
  mode !== 'ctx_log_tail' ||
14439
14574
  !runnerResult ||
14440
14575
  typeof runnerResult !== 'object'
14441
14576
  ) {
14442
- return { lines: [], lineContexts: [] };
14577
+ return { lines: [], lineContexts: [], lineMetadata: [] };
14443
14578
  }
14444
14579
  const result = runnerResult as {
14445
14580
  logs?: unknown;
14446
14581
  logContexts?: unknown;
14582
+ logMetadata?: unknown;
14583
+ logIds?: unknown;
14447
14584
  };
14448
14585
  const logs = result.logs;
14449
14586
  if (!Array.isArray(logs)) {
14450
- return { lines: [], lineContexts: [] };
14587
+ return { lines: [], lineContexts: [], lineMetadata: [] };
14451
14588
  }
14452
14589
  if (
14453
14590
  result.logContexts !== undefined &&
@@ -14456,9 +14593,19 @@ function terminalLogProjectionForOutbox(
14456
14593
  ) {
14457
14594
  throw new TypeError('Terminal log contexts must align with runner logs.');
14458
14595
  }
14596
+ if (
14597
+ result.logMetadata !== undefined &&
14598
+ (!Array.isArray(result.logMetadata) ||
14599
+ result.logMetadata.length !== logs.length)
14600
+ )
14601
+ throw new TypeError('Terminal log metadata must align with logs.');
14602
+ const logMetadata = Array.isArray(result.logMetadata)
14603
+ ? result.logMetadata
14604
+ : undefined;
14459
14605
  const logContexts = Array.isArray(result.logContexts)
14460
14606
  ? result.logContexts
14461
14607
  : undefined;
14608
+ const logIds = normalizeRunLogIds(result.logIds, logs);
14462
14609
  const entries = logs
14463
14610
  .map((line, index) => {
14464
14611
  if (typeof line !== 'string') return null;
@@ -14475,16 +14622,26 @@ function terminalLogProjectionForOutbox(
14475
14622
  return {
14476
14623
  line: line.trim(),
14477
14624
  context: normalizePlayLogContext(logContexts?.[index]),
14625
+ metadata: normalizePlayLogMetadata(logMetadata?.[index]),
14626
+ entryId: logIds?.[index] ?? null,
14478
14627
  };
14479
14628
  })
14480
14629
  .filter(
14481
- (entry): entry is { line: string; context: PlayLogContext | null } =>
14482
- entry !== null,
14630
+ (
14631
+ entry,
14632
+ ): entry is {
14633
+ line: string;
14634
+ context: PlayLogContext | null;
14635
+ metadata: PlayLogMetadata | null;
14636
+ entryId: string | null;
14637
+ } => entry !== null,
14483
14638
  )
14484
14639
  .slice(-LOG_TAIL_LIMIT);
14485
14640
  return {
14486
14641
  lines: entries.map((entry) => entry.line),
14487
14642
  lineContexts: entries.map((entry) => entry.context),
14643
+ lineMetadata: entries.map((entry) => entry.metadata),
14644
+ ...(logIds ? { lineIds: entries.map((entry) => entry.entryId) } : {}),
14488
14645
  };
14489
14646
  }
14490
14647
 
@@ -14733,6 +14890,9 @@ export async function executePostgresSchedulerClaim(
14733
14890
  runtimeTiming: extractRunnerRuntimeTiming(output),
14734
14891
  terminalLogLines: terminalLogProjection.lines,
14735
14892
  terminalLogContexts: terminalLogProjection.lineContexts,
14893
+ terminalLogMetadata: terminalLogProjection.lineMetadata,
14894
+ terminalProgressEvents: options.terminalProgressEvents,
14895
+ terminalLogIds: terminalLogProjection.lineIds,
14736
14896
  terminalWorkCounterSegment: terminalWorkCounterSegmentFromRunnerResult(
14737
14897
  output,
14738
14898
  claim.attempt,
@@ -14794,12 +14954,14 @@ export async function executePostgresSchedulerClaim(
14794
14954
  result: terminalFailureResult,
14795
14955
  terminalLogLines: terminalLogProjection.lines,
14796
14956
  terminalLogContexts: terminalLogProjection.lineContexts,
14957
+ terminalLogMetadata: terminalLogProjection.lineMetadata,
14958
+ terminalLogIds: terminalLogProjection.lineIds,
14797
14959
  sandboxCrashDiagnostic: options.sandboxCrashDiagnostic,
14798
14960
  ...(options.preserveRunnerTerminal
14799
14961
  ? { preserveRunnerTerminal: true }
14800
14962
  : {}),
14801
- ...(runnerFailureResult && options.failedTerminalProgressEvents?.length
14802
- ? { terminalProgressEvents: options.failedTerminalProgressEvents }
14963
+ ...(options.terminalProgressEvents?.length
14964
+ ? { terminalProgressEvents: options.terminalProgressEvents }
14803
14965
  : {}),
14804
14966
  },
14805
14967
  options,
@@ -14987,6 +15149,9 @@ export function buildPostgresSchedulerRunnerContext(
14987
15149
  const rootRunId = launch.rootRunId?.trim() || launch.runId;
14988
15150
  return {
14989
15151
  executorToken: launch.executorToken,
15152
+ ...(launch.runtimeLogIdentity === 'origin'
15153
+ ? { runtimeLogIdentity: 'origin' as const }
15154
+ : {}),
14990
15155
  baseUrl: runtimeApiBaseUrlFromLaunch(launch),
14991
15156
  executionGatewayBaseUrl,
14992
15157
  receiptGatewayBaseUrl,
@@ -17388,6 +17553,12 @@ async function appendRunLedgerOutboxFact(
17388
17553
  WITH admitted AS MATERIALIZED (
17389
17554
  SELECT run_id FROM ${tableName('runs', options)}
17390
17555
  WHERE run_id = $1 AND org_id = $2
17556
+ -- Evaluate ingress time only after the projector/lifecycle fence. An
17557
+ -- insert waiting behind a frozen receipt must become its successor,
17558
+ -- never a late predecessor that changes the attempted receipt.
17559
+ -- NO KEY UPDATE still permits legacy inserts' foreign-key KEY SHARE
17560
+ -- while they finish under the event-search advisory mutex.
17561
+ FOR NO KEY UPDATE
17391
17562
  ), recorded AS (
17392
17563
  INSERT INTO ${tableName('outbox', options)}
17393
17564
  (event_id, run_id, event_type, payload_json, created_at)
@@ -17413,23 +17584,43 @@ export { RunLedgerProjectionLeaseLostError } from '../run-ledger-projection-cont
17413
17584
  /** Extend only our still-live lease before another bounded transport piece. */
17414
17585
  export async function renewPostgresRunLedgerDelivery(
17415
17586
  client: PostgresSchedulerQueryClient,
17416
- input: { eventIds: string[]; leaseOwner: string; leaseSeconds: number },
17587
+ input: {
17588
+ eventIds: string[];
17589
+ leaseOwner: string;
17590
+ leaseSeconds: number;
17591
+ /** Freeze an ingress receipt before any HTTP attempt, including singletons. */
17592
+ freezeReceipt?: boolean;
17593
+ },
17417
17594
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
17418
17595
  ): Promise<void> {
17419
- const result = await client.query<{ event_id: string }>(
17420
- `
17596
+ const renew = (freezeReceipt: boolean) =>
17597
+ client.query<{ event_id: string }>(
17598
+ `
17421
17599
  UPDATE ${tableName('outbox', options)}
17422
17600
  SET lease_expires_at = now() + make_interval(secs => $3::integer)
17601
+ ${freezeReceipt ? ', delivery_batch_started_at = coalesce(delivery_batch_started_at, statement_timestamp())' : ''}
17423
17602
  WHERE event_id = ANY($1::text[]) AND lease_owner = $2
17424
17603
  AND delivered_at IS NULL AND failed_at IS NULL AND lease_expires_at > now()
17425
17604
  RETURNING event_id
17426
17605
  `,
17427
- [
17428
- input.eventIds,
17429
- input.leaseOwner,
17430
- Math.max(5, Math.min(300, input.leaseSeconds)),
17431
- ],
17432
- );
17606
+ [
17607
+ input.eventIds,
17608
+ input.leaseOwner,
17609
+ Math.max(5, Math.min(300, input.leaseSeconds)),
17610
+ ],
17611
+ );
17612
+ let result;
17613
+ try {
17614
+ result = await renew(
17615
+ input.freezeReceipt === true &&
17616
+ supportsOutboxDeliveryBatchMarker(options),
17617
+ );
17618
+ } catch (error) {
17619
+ if (!input.freezeReceipt || !isMissingOutboxDeliveryBatchMarker(error))
17620
+ throw error;
17621
+ markOutboxDeliveryBatchMarkerUnavailable(options);
17622
+ result = await renew(false);
17623
+ }
17433
17624
  if (result.rows.length !== input.eventIds.length)
17434
17625
  throw new RunLedgerProjectionLeaseLostError('delivery_lease_not_current');
17435
17626
  }
@@ -17457,6 +17648,7 @@ export async function assertPostgresRunLedgerDelivery(
17457
17648
  const result = await client.query<{
17458
17649
  delivery_key: string;
17459
17650
  event_type: string;
17651
+ delivery_batch_started_at: string | null;
17460
17652
  lease_owner: string | null;
17461
17653
  lease_live: boolean;
17462
17654
  due: boolean;
@@ -17467,6 +17659,8 @@ export async function assertPostgresRunLedgerDelivery(
17467
17659
  `
17468
17660
  SELECT outbox.delivery_key,
17469
17661
  outbox.event_type,
17662
+ -- Row JSON keeps this optional on historical scheduler schemas.
17663
+ to_jsonb(outbox)->>'delivery_batch_started_at' AS delivery_batch_started_at,
17470
17664
  outbox.lease_owner,
17471
17665
  outbox.lease_expires_at > now() AS lease_live,
17472
17666
  coalesce(outbox.next_attempt_at, outbox.created_at) <= now() AS due,
@@ -17476,6 +17670,7 @@ export async function assertPostgresRunLedgerDelivery(
17476
17670
  FROM ${tableName('outbox', options)} AS outbox
17477
17671
  JOIN ${tableName('runs', options)} AS runs ON runs.run_id = outbox.run_id
17478
17672
  WHERE outbox.run_id = $1 AND runs.org_id = $2
17673
+ AND NOT ${searchLaneSql('outbox')}
17479
17674
  AND outbox.delivered_at IS NULL
17480
17675
  ORDER BY ${runLedgerOrderSql('outbox')}
17481
17676
  -- Later producer facts cannot belong to this receipt. Do not hydrate their
@@ -17497,9 +17692,12 @@ export async function assertPostgresRunLedgerDelivery(
17497
17692
  // Cancellation may own several pending facts at once. Validate the exact
17498
17693
  // contiguous, due prefix named by the receipt instead of all rows sharing
17499
17694
  // that cancellation lease.
17695
+ const ingressReceiptStarted =
17696
+ rows[0]?.event_type === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents &&
17697
+ rows[0]?.delivery_batch_started_at != null;
17500
17698
  const prefix: typeof rows = [];
17501
17699
  let prefixStopReason: RunLedgerProjectionLeaseLostReason | null = null;
17502
- for (const row of rows) {
17700
+ for (const [index, row] of rows.entries()) {
17503
17701
  if (row.lease_owner !== leaseOwner) {
17504
17702
  prefixStopReason = 'prefix_lease_owner_mismatch';
17505
17703
  break;
@@ -17516,11 +17714,6 @@ export async function assertPostgresRunLedgerDelivery(
17516
17714
  prefixStopReason = 'predecessor_failed';
17517
17715
  break;
17518
17716
  }
17519
- if (
17520
- prefix.length > 0 &&
17521
- row.event_type === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents
17522
- )
17523
- break;
17524
17717
  prefix.push(row);
17525
17718
 
17526
17719
  const first = prefix[0]!;
@@ -17567,10 +17760,17 @@ export async function assertPostgresRunLedgerDelivery(
17567
17760
  runId: input.runId,
17568
17761
  deliveryKey: first.delivery_key,
17569
17762
  });
17763
+ // A receipt already attempted as a compound prefix cannot be split by
17764
+ // an older competing projector after lease expiry. Reject with the normal
17765
+ // retryable lease-lost response until a capable writer reclaims it whole.
17766
+ const atStartedPrefixEnd =
17767
+ !ingressReceiptStarted ||
17768
+ rows[index + 1]?.delivery_batch_started_at == null;
17570
17769
  if (
17571
- input.idempotencyKey === expectedKey ||
17572
- input.idempotencyKey === batchKey ||
17573
- legacySingletonKey
17770
+ atStartedPrefixEnd &&
17771
+ (input.idempotencyKey === expectedKey ||
17772
+ input.idempotencyKey === batchKey ||
17773
+ legacySingletonKey)
17574
17774
  ) {
17575
17775
  if (input.projectedEvents === undefined) return;
17576
17776
  const prefixEvents: PlayRunLedgerIngressEvent[] = [];
@@ -17612,11 +17812,18 @@ export async function assertPostgresRunLedgerDelivery(
17612
17812
 
17613
17813
  if (
17614
17814
  prefix.length === 1 &&
17615
- RUN_SCHEDULER_OUTBOX_PREFIX_STOP_EVENT_TYPES.includes(first.event_type)
17815
+ RUN_SCHEDULER_OUTBOX_START_EVENT_TYPES.includes(first.event_type)
17616
17816
  ) {
17617
17817
  break;
17618
17818
  }
17619
- if (terminal || prefix.length >= RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS) {
17819
+ // A frozen ingress receipt can include late runner logs after terminal.
17820
+ // Prove its complete immutable boundary, including transport chunks after
17821
+ // terminal, rather than splitting a receipt that has already been attempted.
17822
+ if (
17823
+ (terminal && !ingressReceiptStarted) ||
17824
+ (ingressReceiptStarted && atStartedPrefixEnd) ||
17825
+ prefix.length >= RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS
17826
+ ) {
17620
17827
  break;
17621
17828
  }
17622
17829
  }
@@ -17631,7 +17838,11 @@ function runLedgerPrecedesSql(
17631
17838
  successor: string,
17632
17839
  inclusive = false,
17633
17840
  ): string {
17634
- return `${runLedgerPositionSql(predecessor)} ${inclusive ? '<=' : '<'} ${runLedgerPositionSql(successor)}`;
17841
+ return `(${searchLaneSql(predecessor)} = ${searchLaneSql(successor)} AND ${runLedgerPositionSql(predecessor)} ${inclusive ? '<=' : '<'} ${runLedgerPositionSql(successor)})`;
17842
+ }
17843
+
17844
+ function searchLaneSql(alias: string): string {
17845
+ return `coalesce(${alias}.payload_json->'${RUNTIME_EVENT_SEARCH_ONLY_FIELD}' = 'true'::jsonb, false)`;
17635
17846
  }
17636
17847
 
17637
17848
  function runLedgerPositionSql(alias: string): string {
@@ -17658,6 +17869,32 @@ function runLedgerTerminalPendingSql(
17658
17869
  )`;
17659
17870
  }
17660
17871
 
17872
+ function runLedgerWaitingRunsSql(
17873
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
17874
+ runIdParameter?: string,
17875
+ ): string {
17876
+ return `SELECT DISTINCT waiting_event.run_id
17877
+ FROM ${tableName('outbox', options)} AS waiting_event
17878
+ WHERE waiting_event.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runWaiting}
17879
+ -- Detached runner parks are transport-only and project no public wait.
17880
+ AND waiting_event.payload_json->>'kind' IS DISTINCT FROM 'detached_runner'
17881
+ AND waiting_event.delivered_at IS NULL
17882
+ AND waiting_event.failed_at IS NULL
17883
+ AND coalesce(waiting_event.next_attempt_at, waiting_event.created_at) <= now()
17884
+ ${runIdParameter ? `AND (${runIdParameter}::text IS NULL OR waiting_event.run_id = ${runIdParameter}::text)` : ''}`;
17885
+ }
17886
+
17887
+ function runLedgerWaitingPendingSql(alias: string): string {
17888
+ return `(${alias}.run_id IN (SELECT run_id FROM waiting_runs))`;
17889
+ }
17890
+
17891
+ function runLedgerUrgentLifecyclePendingSql(
17892
+ alias: string,
17893
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
17894
+ ): string {
17895
+ return `(${runLedgerTerminalPendingSql(alias, options)} OR ${runLedgerWaitingPendingSql(alias)})`;
17896
+ }
17897
+
17661
17898
  /**
17662
17899
  * Claim a causal prefix, preserving the v33 adapter while the additive batch
17663
17900
  * marker is absent. An enclosing transaction uses a savepoint so the missing
@@ -17670,11 +17907,43 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17670
17907
  runId?: string | null;
17671
17908
  leaseOwner?: string | null;
17672
17909
  leaseSeconds?: number;
17910
+ /** Independent raw search work uses the same outbox leases and controls. */
17911
+ searchOnly?: boolean;
17912
+ /** Advertised by the launch-pinned receipt verifier, never inferred locally. */
17913
+ allowIngressPrefix?: boolean;
17914
+ /** Extend this still-owned head atomically under the run cancellation fence. */
17915
+ extendOwnedHead?: string;
17673
17916
  /** The dispatcher owns a short, exclusive claim transaction. */
17674
17917
  inTransaction?: boolean;
17675
17918
  } = {},
17676
17919
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
17677
17920
  ): Promise<PostgresSchedulerOutboxEvent[]> {
17921
+ const { events } = await claimPostgresSchedulerOutboxPrefix(
17922
+ client,
17923
+ input,
17924
+ options,
17925
+ );
17926
+ return events;
17927
+ }
17928
+
17929
+ /**
17930
+ * ACK and next-prefix selection share one statement after the run lock. Data
17931
+ * CTEs use the statement's original snapshot, so every pending-row predicate
17932
+ * explicitly excludes the acknowledged IDs. A partial ACK cannot claim a
17933
+ * successor. The receiver's ingress capability and frozen receipts still
17934
+ * determine which causal facts may share a prefix.
17935
+ */
17936
+ async function claimPostgresSchedulerOutboxPrefix(
17937
+ client: PostgresSchedulerQueryClient,
17938
+ input: NonNullable<
17939
+ Parameters<typeof readPendingPostgresSchedulerOutboxEvents>[1]
17940
+ >,
17941
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
17942
+ acknowledgement?: { eventIds: string[]; targetEventId: string },
17943
+ ): Promise<{
17944
+ events: PostgresSchedulerOutboxEvent[];
17945
+ deliveredRows: RunLedgerOutboxDeliveredRow[];
17946
+ }> {
17678
17947
  const requestedLimit = Math.floor(input.limit ?? 1);
17679
17948
  const limit = Number.isFinite(requestedLimit)
17680
17949
  ? Math.max(1, requestedLimit)
@@ -17682,20 +17951,76 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17682
17951
  const runId = input.runId?.trim() || null;
17683
17952
  const leaseOwner = input.leaseOwner?.trim() || null;
17684
17953
  const leaseSeconds = Math.max(5, Math.min(300, input.leaseSeconds ?? 30));
17685
- const claim = async (supportsTerminalBatchMarker: boolean) =>
17686
- await client.query<{
17954
+ const pending = (alias: string) =>
17955
+ `${alias}.delivered_at IS NULL${
17956
+ acknowledgement ? ` AND NOT (${alias}.event_id = ANY($5::text[]))` : ''
17957
+ }`;
17958
+ const claim = async (supportsTerminalBatchMarker: boolean) => {
17959
+ const urgentLifecyclePendingSql = (alias: string) =>
17960
+ input.searchOnly === true
17961
+ ? runLedgerTerminalPendingSql(alias, options)
17962
+ : runLedgerUrgentLifecyclePendingSql(alias, options);
17963
+ const ingressPrefixSql =
17964
+ supportsTerminalBatchMarker && input.allowIngressPrefix === true
17965
+ ? `(head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} AND ${urgentLifecyclePendingSql('head')})`
17966
+ : 'false';
17967
+ return await client.query<{
17968
+ row_kind: 'claimed' | 'delivered';
17687
17969
  event_id: string;
17688
17970
  run_id: string;
17689
17971
  event_type: string;
17690
17972
  payload_json: unknown;
17691
17973
  created_at: Date;
17692
17974
  delivery_key: string | null;
17975
+ lag_ms: number | string | null;
17693
17976
  }>(
17694
17977
  `
17695
- WITH head AS MATERIALIZED (
17696
- SELECT candidate.event_id, candidate.run_id, candidate.event_type
17978
+ WITH ${
17979
+ acknowledgement
17980
+ ? `delivered AS (
17981
+ ${runLedgerAcknowledgeSql(options, '$5', '$3')}
17982
+ ),`
17983
+ : ''
17984
+ } ${
17985
+ input.searchOnly === true
17986
+ ? ''
17987
+ : `waiting_runs AS MATERIALIZED (
17988
+ ${runLedgerWaitingRunsSql(options, '$2')}
17989
+ ), `
17990
+ }${
17991
+ input.extendOwnedHead
17992
+ ? `owned_run AS MATERIALIZED (
17993
+ SELECT run_id FROM ${tableName('runs', options)}
17994
+ WHERE run_id = $2
17995
+ -- This dispatcher already owns the head. A producer holding the run
17996
+ -- lock is not a lost delivery lease: wait for its short transaction,
17997
+ -- then revalidate the head below. New-run selection still skips locks.
17998
+ FOR UPDATE
17999
+ ),`
18000
+ : ''
18001
+ } head AS MATERIALIZED (
18002
+ SELECT candidate.event_id, candidate.run_id, candidate.event_type,
18003
+ ${supportsTerminalBatchMarker ? 'candidate.delivery_batch_started_at' : 'NULL::timestamptz'} AS delivery_batch_started_at
17697
18004
  FROM ${tableName('outbox', options)} AS candidate
17698
- WHERE candidate.delivered_at IS NULL
18005
+ ${input.extendOwnedHead ? 'JOIN owned_run ON owned_run.run_id = candidate.run_id' : ''}
18006
+ WHERE ${pending('candidate')}
18007
+ AND ${searchLaneSql('candidate')} = ${input.searchOnly === true ? 'true' : 'false'}
18008
+ ${
18009
+ acknowledgement
18010
+ ? `AND (SELECT count(*) FROM delivered) = cardinality($5::text[])
18011
+ AND EXISTS (
18012
+ SELECT 1 FROM ${tableName('outbox', options)} AS target
18013
+ WHERE target.run_id = $2 AND target.event_id = $6
18014
+ AND ${pending('target')} AND target.failed_at IS NULL
18015
+ )`
18016
+ : ''
18017
+ }
18018
+ ${
18019
+ input.extendOwnedHead
18020
+ ? `AND candidate.event_id = $5
18021
+ AND candidate.lease_owner = $3 AND candidate.lease_expires_at > now()`
18022
+ : ''
18023
+ }
17699
18024
  AND candidate.failed_at IS NULL
17700
18025
  AND coalesce(candidate.next_attempt_at, candidate.created_at) <= now()
17701
18026
  AND ($2::text IS NULL OR candidate.run_id = $2)
@@ -17711,7 +18036,8 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17711
18036
  AND NOT EXISTS (
17712
18037
  SELECT 1 FROM ${tableName('outbox', options)} AS active
17713
18038
  WHERE active.run_id = candidate.run_id
17714
- AND active.delivered_at IS NULL
18039
+ AND ${pending('active')}
18040
+ AND ${searchLaneSql('active')} = ${searchLaneSql('candidate')}
17715
18041
  AND active.lease_owner IS NOT NULL
17716
18042
  AND active.lease_expires_at > now()
17717
18043
  AND active.lease_owner IS DISTINCT FROM $3::text
@@ -17723,7 +18049,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17723
18049
  SELECT 1
17724
18050
  FROM ${tableName('outbox', options)} AS predecessor
17725
18051
  WHERE predecessor.run_id = candidate.run_id
17726
- AND predecessor.delivered_at IS NULL
18052
+ AND ${pending('predecessor')}
17727
18053
  AND ${runLedgerPrecedesSql('predecessor', 'candidate')}
17728
18054
  )
17729
18055
  -- A new durable admission has no causal dependency on unrelated
@@ -17738,7 +18064,8 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17738
18064
  CASE
17739
18065
  WHEN candidate.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0
17740
18066
  WHEN ${runLedgerTerminalPendingSql('candidate', options)} THEN 1
17741
- ELSE 2
18067
+ ${input.searchOnly === true ? '' : `WHEN ${runLedgerWaitingPendingSql('candidate')} THEN 2`}
18068
+ ELSE 3
17742
18069
  END,
17743
18070
  coalesce(candidate.next_attempt_at, candidate.created_at),
17744
18071
  candidate.created_at,
@@ -17749,17 +18076,26 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17749
18076
  SELECT outbox.event_id
17750
18077
  FROM ${tableName('outbox', options)} AS outbox
17751
18078
  JOIN head ON head.run_id = outbox.run_id
17752
- WHERE outbox.delivered_at IS NULL
18079
+ WHERE ${pending('outbox')}
18080
+ AND ${searchLaneSql('outbox')} = ${input.searchOnly === true ? 'true' : 'false'}
17753
18081
  AND outbox.failed_at IS NULL
17754
18082
  -- Admission is causally first and must be projected alone. A later
17755
18083
  -- start/terminal append may not leapfrog a delayed queued row.
17756
18084
  AND (
17757
18085
  head.event_type NOT IN (${RUN_EVENT_KIND_SQL_LITERAL.runCreated}, ${RUN_EVENT_KIND_SQL_LITERAL.runStarted}, ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents})
17758
18086
  OR outbox.event_id = head.event_id
18087
+ OR (head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
18088
+ AND ${ingressPrefixSql})
17759
18089
  )
17760
- -- An ingress row already contains a producer's bounded batch. Keep
17761
- -- its receipt stable independently of the recovery caller's limit.
17762
- AND (outbox.event_type <> ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} OR outbox.event_id = head.event_id)
18090
+ -- Only a receiver with the exact-prefix verifier may combine ingress
18091
+ -- facts while draining a pending terminal. Old launch origins and
18092
+ -- schemas retain each producer's existing singleton receipt.
18093
+ AND (outbox.event_type <> ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
18094
+ OR outbox.event_id = head.event_id
18095
+ OR ${ingressPrefixSql})
18096
+ -- Once any ingress prefix was attempted, its receipt is immutable,
18097
+ -- including singleton appends whose HTTP/ACK outcome was ambiguous.
18098
+ AND (${supportsTerminalBatchMarker && input.allowIngressPrefix === true ? 'head.delivery_batch_started_at IS NULL OR outbox.delivery_batch_started_at IS NOT NULL' : 'true'})
17763
18099
  AND coalesce(outbox.next_attempt_at, outbox.created_at) <= now()
17764
18100
  AND (
17765
18101
  $3::text IS NULL
@@ -17777,7 +18113,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17777
18113
  SELECT 1
17778
18114
  FROM ${tableName('outbox', options)} AS terminal_predecessor
17779
18115
  WHERE terminal_predecessor.run_id = outbox.run_id
17780
- AND terminal_predecessor.delivered_at IS NULL
18116
+ AND ${pending('terminal_predecessor')}
17781
18117
  AND ${
17782
18118
  supportsTerminalBatchMarker
17783
18119
  ? 'terminal_predecessor.delivery_batch_started_at IS NOT NULL'
@@ -17799,10 +18135,12 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17799
18135
  SELECT 1
17800
18136
  FROM ${tableName('outbox', options)} AS predecessor
17801
18137
  WHERE predecessor.run_id = outbox.run_id
17802
- AND predecessor.delivered_at IS NULL
18138
+ AND ${pending('predecessor')}
17803
18139
  AND ${runLedgerPrecedesSql('predecessor', 'outbox')}
17804
18140
  AND (
17805
- predecessor.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
18141
+ (predecessor.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
18142
+ AND NOT (${ingressPrefixSql}))
18143
+ OR (${supportsTerminalBatchMarker && input.allowIngressPrefix === true ? 'head.delivery_batch_started_at IS NOT NULL AND predecessor.delivery_batch_started_at IS NULL' : 'false'})
17806
18144
  OR predecessor.failed_at IS NOT NULL
17807
18145
  OR coalesce(predecessor.next_attempt_at, predecessor.created_at) > now()
17808
18146
  OR (
@@ -17833,8 +18171,8 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17833
18171
  ${
17834
18172
  supportsTerminalBatchMarker
17835
18173
  ? `, delivery_batch_started_at = CASE
17836
- WHEN terminal_batch.has_terminal
17837
- THEN COALESCE(outbox.delivery_batch_started_at, clock_timestamp())
18174
+ WHEN terminal_batch.has_terminal ${input.allowIngressPrefix === true ? `OR ${urgentLifecyclePendingSql('outbox')}` : ''}
18175
+ THEN COALESCE(outbox.delivery_batch_started_at, statement_timestamp())
17838
18176
  ELSE outbox.delivery_batch_started_at
17839
18177
  END`
17840
18178
  : ''
@@ -17846,13 +18184,53 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17846
18184
  outbox.payload_json, outbox.created_at, outbox.lease_owner,
17847
18185
  outbox.delivery_key
17848
18186
  )
17849
- SELECT event_id, run_id, event_type, payload_json, created_at, lease_owner,
17850
- delivery_key
17851
- FROM claimed
17852
- ORDER BY ${runLedgerOrderSql('claimed')}
18187
+ ${
18188
+ acknowledgement
18189
+ ? `, notified AS MATERIALIZED (
18190
+ SELECT delivered.*,
18191
+ pg_notify(
18192
+ $7,
18193
+ json_build_object('eventId', event_id, 'kind', 'outbox_delivered')::text
18194
+ ) AS notification
18195
+ FROM delivered
18196
+ )`
18197
+ : ''
18198
+ }
18199
+ SELECT * FROM (
18200
+ SELECT 'claimed'::text AS row_kind,
18201
+ event_id, run_id, event_type, payload_json, created_at, lease_owner,
18202
+ delivery_key, NULL::float8 AS lag_ms
18203
+ FROM claimed
18204
+ ${
18205
+ acknowledgement
18206
+ ? `UNION ALL
18207
+ SELECT 'delivered'::text AS row_kind,
18208
+ event_id, NULL::text AS run_id, event_type, NULL::jsonb AS payload_json,
18209
+ NULL::timestamptz AS created_at, NULL::text AS lease_owner,
18210
+ NULL::text AS delivery_key, lag_ms
18211
+ FROM notified`
18212
+ : ''
18213
+ }
18214
+ ) AS result
18215
+ ORDER BY row_kind, ${runLedgerOrderSql('result')}
17853
18216
  `,
17854
- [limit, runId, leaseOwner, leaseSeconds],
18217
+ [
18218
+ limit,
18219
+ runId,
18220
+ leaseOwner,
18221
+ leaseSeconds,
18222
+ ...(acknowledgement
18223
+ ? [
18224
+ acknowledgement.eventIds,
18225
+ acknowledgement.targetEventId,
18226
+ postgresSchedulerNotifyChannel('terminal', options),
18227
+ ]
18228
+ : input.extendOwnedHead
18229
+ ? [input.extendOwnedHead]
18230
+ : []),
18231
+ ],
17855
18232
  );
18233
+ };
17856
18234
  let result;
17857
18235
  if (input.inTransaction) await client.query('SAVEPOINT run_ledger_claim');
17858
18236
  if (supportsOutboxDeliveryBatchMarker(options)) {
@@ -17860,9 +18238,13 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17860
18238
  result = await claim(true);
17861
18239
  } catch (error) {
17862
18240
  if (!isMissingOutboxDeliveryBatchMarker(error)) throw error;
18241
+ markOutboxDeliveryBatchMarkerUnavailable(options);
18242
+ // The ACK caller owns the whole short transaction. Roll it back and
18243
+ // repeat only this missing-column adapter, avoiding two SAVEPOINT round
18244
+ // trips on every successful acknowledgement.
18245
+ if (acknowledgement) throw error;
17863
18246
  if (input.inTransaction)
17864
18247
  await client.query('ROLLBACK TO SAVEPOINT run_ledger_claim');
17865
- markOutboxDeliveryBatchMarkerUnavailable(options);
17866
18248
  result = await claim(false);
17867
18249
  }
17868
18250
  } else {
@@ -17870,22 +18252,34 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17870
18252
  }
17871
18253
  if (input.inTransaction)
17872
18254
  await client.query('RELEASE SAVEPOINT run_ledger_claim');
17873
- return result.rows.map((row) => {
17874
- if (!row.delivery_key) {
17875
- throw new Error(
17876
- `Run-ledger outbox event ${row.event_id} has no delivery key.`,
17877
- );
17878
- }
17879
- return {
17880
- eventId: row.event_id,
17881
- runId: row.run_id,
17882
- eventType: row.event_type,
17883
- payload: row.payload_json,
17884
- createdAt: row.created_at,
17885
- leaseOwner: leaseOwner,
17886
- deliveryKey: row.delivery_key,
17887
- };
17888
- });
18255
+ const events = result.rows
18256
+ .filter((row) => row.row_kind === 'claimed')
18257
+ .map((row) => {
18258
+ if (!row.delivery_key) {
18259
+ throw new Error(
18260
+ `Run-ledger outbox event ${row.event_id} has no delivery key.`,
18261
+ );
18262
+ }
18263
+ return {
18264
+ eventId: row.event_id,
18265
+ runId: row.run_id,
18266
+ eventType: row.event_type,
18267
+ payload: row.payload_json,
18268
+ createdAt: row.created_at,
18269
+ leaseOwner: leaseOwner,
18270
+ deliveryKey: row.delivery_key,
18271
+ };
18272
+ });
18273
+ const deliveredRows = result.rows
18274
+ .filter((row) => row.row_kind === 'delivered')
18275
+ .map((row) => {
18276
+ const lagMs = Number(row.lag_ms);
18277
+ return {
18278
+ eventType: row.event_type,
18279
+ lagMs: Number.isFinite(lagMs) ? Math.max(0, lagMs) : 0,
18280
+ };
18281
+ });
18282
+ return { events, deliveredRows };
17889
18283
  }
17890
18284
 
17891
18285
  /**
@@ -17910,6 +18304,7 @@ export async function readPostgresSchedulerOutboxNextDueAt(
17910
18304
  (SELECT max(active.lease_expires_at)
17911
18305
  FROM ${tableName('outbox', options)} AS active
17912
18306
  WHERE active.run_id = candidate.run_id
18307
+ AND ${searchLaneSql('active')} = ${searchLaneSql('candidate')}
17913
18308
  AND active.delivered_at IS NULL AND active.lease_owner IS NOT NULL
17914
18309
  AND active.lease_expires_at > now())
17915
18310
  )) AS due_at
@@ -18149,7 +18544,7 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
18149
18544
  '{events}',
18150
18545
  (SELECT jsonb_agg(
18151
18546
  CASE WHEN item.value->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
18152
- THEN item.value - 'lines' - 'lineContexts'
18547
+ THEN item.value - 'lines' - 'lineContexts' - 'lineMetadata' - 'lineIds'
18153
18548
  ELSE item.value
18154
18549
  END ORDER BY item.ordinality
18155
18550
  )
@@ -18162,11 +18557,11 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
18162
18557
  )
18163
18558
  WHEN jsonb_typeof(outbox.payload_json->'terminalProgressEvents') = 'array'
18164
18559
  THEN jsonb_set(
18165
- outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts',
18560
+ outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogMetadata' - 'terminalLogIds',
18166
18561
  '{terminalProgressEvents}',
18167
18562
  coalesce((SELECT jsonb_agg(
18168
18563
  CASE WHEN item.value->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
18169
- THEN item.value - 'lines' - 'lineContexts'
18564
+ THEN item.value - 'lines' - 'lineContexts' - 'lineMetadata' - 'lineIds'
18170
18565
  ELSE item.value
18171
18566
  END ORDER BY item.ordinality
18172
18567
  )
@@ -18174,7 +18569,7 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
18174
18569
  WITH ORDINALITY AS item(value, ordinality)), '[]'::jsonb),
18175
18570
  false
18176
18571
  )
18177
- ELSE outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts'
18572
+ ELSE outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogMetadata' - 'terminalLogIds'
18178
18573
  END
18179
18574
  FROM candidates
18180
18575
  WHERE outbox.event_id = candidates.event_id`,
@@ -18236,6 +18631,17 @@ function runLedgerDeliveryFailure(input: { error: unknown }): {
18236
18631
  projectionMismatch: RunLedgerProjectionMismatchDiagnostic | null;
18237
18632
  retryAfterMs: number | null;
18238
18633
  } {
18634
+ if (input.error instanceof RunLedgerProjectionLeaseLostError) {
18635
+ return {
18636
+ permanent: false,
18637
+ code: 'RUN_LEDGER_DELIVERY_LEASE_LOST',
18638
+ status: null,
18639
+ retryable: true,
18640
+ failureReason: input.error.reason,
18641
+ projectionMismatch: input.error.projectionMismatch ?? null,
18642
+ retryAfterMs: null,
18643
+ };
18644
+ }
18239
18645
  if (input.error instanceof AppRuntimeApiResponseError) {
18240
18646
  // Projection is idempotent. A dependency can incorrectly send
18241
18647
  // `retryable:false` with a transient overload response, but that metadata
@@ -18416,6 +18822,32 @@ export async function markPostgresSchedulerOutboxDelivered(
18416
18822
  ).length;
18417
18823
  }
18418
18824
 
18825
+ /** Shared lease-fenced ACK, including the canonical retained-payload cleanup. */
18826
+ function runLedgerAcknowledgeSql(
18827
+ options: Pick<PostgresSchedulerOptions, 'schema'> | undefined,
18828
+ eventIdsParameter: string,
18829
+ leaseOwnerParameter: string,
18830
+ ): string {
18831
+ return `UPDATE ${tableName('outbox', options)} AS outbox
18832
+ SET delivered_at = now(),
18833
+ next_attempt_at = NULL,
18834
+ last_error_json = NULL,
18835
+ lease_owner = NULL,
18836
+ lease_expires_at = NULL,
18837
+ payload_json = CASE WHEN EXISTS (
18838
+ SELECT 1 FROM ${tableName('runs', options)} AS run
18839
+ WHERE run.run_id = outbox.run_id
18840
+ AND run.run_event_store = 'planet_scale'
18841
+ ) THEN '{}'::jsonb ELSE outbox.payload_json END
18842
+ WHERE outbox.event_id = ANY(${eventIdsParameter}::text[])
18843
+ AND outbox.delivered_at IS NULL
18844
+ AND (${leaseOwnerParameter}::text IS NULL OR outbox.lease_owner = ${leaseOwnerParameter})
18845
+ RETURNING outbox.event_id,
18846
+ outbox.event_type,
18847
+ (extract(epoch FROM (outbox.delivered_at - outbox.created_at)) * 1000)::float8
18848
+ AS lag_ms`;
18849
+ }
18850
+
18419
18851
  /**
18420
18852
  * Lease-fenced acknowledgement that also reports each row's event type and
18421
18853
  * commit-to-acknowledgement lag on the database clock, so lag metrics never
@@ -18436,24 +18868,7 @@ export async function markPostgresSchedulerOutboxDeliveredRows(
18436
18868
  }>(
18437
18869
  `
18438
18870
  WITH delivered AS (
18439
- UPDATE ${tableName('outbox', options)} AS outbox
18440
- SET delivered_at = now(),
18441
- next_attempt_at = NULL,
18442
- last_error_json = NULL,
18443
- lease_owner = NULL,
18444
- lease_expires_at = NULL,
18445
- payload_json = CASE WHEN EXISTS (
18446
- SELECT 1 FROM ${tableName('runs', options)} AS run
18447
- WHERE run.run_id = outbox.run_id
18448
- AND run.run_event_store = 'planet_scale'
18449
- ) THEN '{}'::jsonb ELSE outbox.payload_json END
18450
- WHERE outbox.event_id = ANY($1::text[])
18451
- AND outbox.delivered_at IS NULL
18452
- AND ($2::text IS NULL OR outbox.lease_owner = $2)
18453
- RETURNING outbox.event_id,
18454
- outbox.event_type,
18455
- (extract(epoch FROM (outbox.delivered_at - outbox.created_at)) * 1000)::float8
18456
- AS lag_ms
18871
+ ${runLedgerAcknowledgeSql(options, '$1', '$2')}
18457
18872
  )
18458
18873
  SELECT event_id,
18459
18874
  event_type,
@@ -18538,6 +18953,22 @@ function normalizeOutboxIngressPreview(
18538
18953
  export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
18539
18954
  event: PostgresSchedulerOutboxEvent,
18540
18955
  ): PlayRunLedgerIngressEvent[] {
18956
+ if (
18957
+ payloadField(event.payload, RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD) ===
18958
+ true
18959
+ ) {
18960
+ throw new Error(
18961
+ 'Deferred outbox input must be projected before ledger delivery.',
18962
+ );
18963
+ }
18964
+ if (
18965
+ payloadField(event.payload, RUNTIME_EVENT_SEARCH_ONLY_FIELD) === true ||
18966
+ event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.receiptEvent ||
18967
+ event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runnerLogs
18968
+ ) {
18969
+ // Search-only carriers are already durable in the canonical journal.
18970
+ return [];
18971
+ }
18541
18972
  const occurredAt = event.createdAt.getTime();
18542
18973
  if (event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents) {
18543
18974
  const events = payloadField(event.payload, 'events');
@@ -18591,6 +19022,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
18591
19022
  );
18592
19023
  const lines = stringArrayPayloadField(event.payload, 'terminalLogLines');
18593
19024
  const lineContexts = terminalLogContextsPayloadField(event.payload, lines);
19025
+ const lineMetadata = terminalLogMetadataPayloadField(event.payload, lines);
18594
19026
  const resultSummary = payloadField(event.payload, 'resultSummary');
18595
19027
  return [
18596
19028
  ...progressEvents.map(normalizeOutboxIngressPreview),
@@ -18604,6 +19036,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
18604
19036
  source: 'worker',
18605
19037
  lines,
18606
19038
  ...(lineContexts ? { lineContexts } : {}),
19039
+ ...(lineMetadata ? { lineMetadata } : {}),
18607
19040
  } as PlayRunLedgerEvent,
18608
19041
  ]),
18609
19042
  {
@@ -18633,6 +19066,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
18633
19066
  );
18634
19067
  const lines = stringArrayPayloadField(event.payload, 'terminalLogLines');
18635
19068
  const lineContexts = terminalLogContextsPayloadField(event.payload, lines);
19069
+ const lineMetadata = terminalLogMetadataPayloadField(event.payload, lines);
18636
19070
  const result = terminalRunResultForLedger(
18637
19071
  payloadField(event.payload, 'result'),
18638
19072
  );
@@ -18649,6 +19083,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
18649
19083
  source: 'worker',
18650
19084
  lines,
18651
19085
  ...(lineContexts ? { lineContexts } : {}),
19086
+ ...(lineMetadata ? { lineMetadata } : {}),
18652
19087
  } as PlayRunLedgerEvent,
18653
19088
  ]),
18654
19089
  {
@@ -18774,6 +19209,8 @@ export type PostgresSchedulerOutboxMaterializationPlan = {
18774
19209
  };
18775
19210
 
18776
19211
  const RUN_LEDGER_KNOWN_OUTBOX_EVENT_TYPES: ReadonlySet<string> = new Set([
19212
+ RUN_SCHEDULER_OUTBOX_EVENT_TYPE.receiptEvent,
19213
+ RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runnerLogs,
18777
19214
  RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents,
18778
19215
  RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runStarted,
18779
19216
  RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runCompleted,
@@ -18891,7 +19328,7 @@ export async function listPostgresSchedulerOutboxMaterializationPlans(
18891
19328
  * - `local_mint`: sign the token in-process with `BETTER_AUTH_SECRET`. ONLY the
18892
19329
  * app (Vercel) legitimately holds the signing key, so this is app-side only
18893
19330
  * (read-model heal path). It is the AUTHORITATIVE projector.
18894
- * - `gateway`: always prove the active outbox lease to the launch-pinned
19331
+ * - `gateway`: for every app-runtime append, prove the active outbox lease to the launch-pinned
18895
19332
  * deployment and receive a short-lived executor JWT
18896
19333
  * (`requestRunLedgerDispatchToken`), even if the caller supplied a
18897
19334
  * `runtimeApi`. The Fly worker has no signing key and MUST use this. A
@@ -18912,6 +19349,21 @@ export type RunLedgerOutboxTokenAuthority = 'local_mint' | 'gateway';
18912
19349
  /** Cross-run service policy; causal order within a run is always unchanged. */
18913
19350
  export type RunLedgerOutboxHeadPriority = 'admission_first' | 'oldest_ready';
18914
19351
 
19352
+ class DeferredRuntimeProjectionInputError extends AppRuntimeApiResponseError {
19353
+ constructor(
19354
+ readonly eventId: string,
19355
+ detail: string,
19356
+ ) {
19357
+ super({
19358
+ action: 'project_run_events',
19359
+ status: 422,
19360
+ code: 'runtime_event_projection_invalid',
19361
+ retryable: false,
19362
+ detail,
19363
+ });
19364
+ }
19365
+ }
19366
+
18915
19367
  const POSTGRES_SCHEDULER_CANCELLATION_OUTBOX_LEASE_SECONDS =
18916
19368
  // The direct projection can run for the full app-runtime request deadline.
18917
19369
  // Keep another 30 seconds for dispatch setup and acknowledgement so a
@@ -18942,8 +19394,12 @@ export function postgresSchedulerCancellationOutboxLease(runId: string): {
18942
19394
  }
18943
19395
 
18944
19396
  export type RunLedgerOutboxDispatchInput = {
19397
+ /** Only the existing background projector claims the independent search prefix. */
19398
+ searchOnly?: boolean;
19399
+ /** Only the existing background projector indexes deferred raw inputs. */
19400
+ materializeObservability?: boolean;
18945
19401
  /**
18946
- * Optional runtime context. `gateway` authority always replaces its
19402
+ * Optional runtime context. For an app-runtime append, `gateway` replaces its
18947
19403
  * credentials with a fresh lease-scoped token and keeps only supported
18948
19404
  * transport options; `local_mint` and the default authority preserve the
18949
19405
  * explicit context as before.
@@ -18983,8 +19439,9 @@ export type RunLedgerOutboxDispatchInput = {
18983
19439
  * untargeted fast path) set this: after acknowledging a batch, also carry
18984
19440
  * the run's terminal fact through when it is now pending (see
18985
19441
  * `continueRunLedgerOutboxThroughTerminal`). Callers that drive their own
18986
- * bounded batches over a pre-leased prefix (cancellation, inline
18987
- * `run.created`, the targeted terminal drain) leave it unset.
19442
+ * bounded batches (cancellation, inline `run.created`, the targeted terminal
19443
+ * drain) leave it unset. Targeted drains retain successors through their
19444
+ * explicit run/owner/target scope instead.
18988
19445
  */
18989
19446
  continueThroughTerminal?: boolean;
18990
19447
  };
@@ -19066,7 +19523,10 @@ type RunLedgerOutboxAuthorityScope = {
19066
19523
  authority: {
19067
19524
  runId: string;
19068
19525
  leaseOwner: string;
19069
- runtimeApi: WorkerRuntimeApiContext;
19526
+ runtimeApi: WorkerRuntimeApiContext & {
19527
+ refreshExecutorToken?: () => Promise<string>;
19528
+ invalidateExecutorToken?: () => void;
19529
+ };
19070
19530
  expiresAtMs: number | null;
19071
19531
  } | null;
19072
19532
  };
@@ -19101,6 +19561,9 @@ async function dispatchRunLedgerOutboxThroughEventWithScope(
19101
19561
  input.leaseOwner?.trim() ||
19102
19562
  scope?.leaseOwner ||
19103
19563
  `run-ledger-outbox-dispatch:${randomUUID()}`;
19564
+ // The same owner retains each successor in the ACK transaction, including
19565
+ // for a fresh worker drain that did not arrive with a preclaimed prefix.
19566
+ scope ??= { runId, leaseOwner, targetEventId, nextClaimed: [] };
19104
19567
  const scopedInput = { ...input, leaseOwner };
19105
19568
  const batchSize = RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS;
19106
19569
  const eventIds: string[] = [];
@@ -19111,80 +19574,92 @@ async function dispatchRunLedgerOutboxThroughEventWithScope(
19111
19574
  let failed = 0;
19112
19575
  let retrying = 0;
19113
19576
 
19114
- let remaining = maxEvents;
19115
- while (remaining === null || remaining > 0) {
19116
- const result = await dispatchRunLedgerOutboxBatchWithScope(
19117
- withClient,
19118
- {
19119
- ...scopedInput,
19120
- waitForRunLock: true,
19121
- continueThroughTerminal: false,
19122
- limit: remaining === null ? batchSize : Math.min(batchSize, remaining),
19123
- },
19124
- options,
19125
- scope,
19126
- authorityScope,
19127
- );
19128
- attempted += result.attempted;
19129
- projected += result.projected;
19130
- failed += result.failed;
19131
- retrying += result.retrying;
19132
- eventIds.push(...result.eventIds);
19133
- failedEventIds.push(...result.failedEventIds);
19134
- retryingEventIds.push(...result.retryingEventIds);
19135
- if (remaining !== null) remaining -= result.attempted;
19136
- if (result.eventIds.includes(targetEventId)) {
19137
- return {
19138
- attempted,
19139
- projected,
19140
- eventIds,
19141
- failed,
19142
- failedEventIds,
19143
- retrying,
19144
- retryingEventIds,
19145
- targetEventId,
19146
- targetDelivered: true,
19147
- targetBlocked: false,
19148
- };
19577
+ try {
19578
+ let remaining = maxEvents;
19579
+ while (remaining === null || remaining > 0) {
19580
+ const result = await dispatchRunLedgerOutboxBatchWithScope(
19581
+ withClient,
19582
+ {
19583
+ ...scopedInput,
19584
+ waitForRunLock: true,
19585
+ continueThroughTerminal: false,
19586
+ limit:
19587
+ remaining === null ? batchSize : Math.min(batchSize, remaining),
19588
+ },
19589
+ options,
19590
+ scope,
19591
+ authorityScope,
19592
+ remaining,
19593
+ );
19594
+ attempted += result.attempted;
19595
+ projected += result.projected;
19596
+ failed += result.failed;
19597
+ retrying += result.retrying;
19598
+ eventIds.push(...result.eventIds);
19599
+ failedEventIds.push(...result.failedEventIds);
19600
+ retryingEventIds.push(...result.retryingEventIds);
19601
+ if (remaining !== null) remaining -= result.attempted;
19602
+ if (result.eventIds.includes(targetEventId)) {
19603
+ return {
19604
+ attempted,
19605
+ projected,
19606
+ eventIds,
19607
+ failed,
19608
+ failedEventIds,
19609
+ retrying,
19610
+ retryingEventIds,
19611
+ targetEventId,
19612
+ targetDelivered: true,
19613
+ targetBlocked: false,
19614
+ };
19615
+ }
19616
+ // A projection failure is durably scheduled for retry or moved to terminal
19617
+ // DLQ. Its policy owns the next attempt; spinning here would only reclaim
19618
+ // the same failed predecessor and delay the park indefinitely.
19619
+ // An absent target cannot retain a successor in ACK. Keep the existing
19620
+ // fresh-claim traversal in that case until ready work is exhausted.
19621
+ if (result.projected === 0 || result.attempted === 0) break;
19149
19622
  }
19150
- // A projection failure is durably scheduled for retry or moved to terminal
19151
- // DLQ. Its policy owns the next attempt; spinning here would only reclaim
19152
- // the same failed predecessor and delay the park indefinitely.
19153
- if (
19154
- result.projected === 0 ||
19155
- result.attempted === 0 ||
19156
- (scope && scope.nextClaimed.length === 0)
19157
- )
19158
- break;
19159
- }
19160
-
19161
- // A concurrent dispatcher can claim the target between batches. An empty
19162
- // drain therefore does not prove delivery; read the durable acknowledgement
19163
- // before reporting the target as pending.
19164
- const target = await withClient((client) =>
19165
- readPostgresSchedulerOutboxDeliveryStatus(
19166
- client,
19167
- { runId, eventId: targetEventId },
19168
- options,
19169
- ),
19170
- );
19171
- return {
19172
- attempted,
19173
- projected,
19174
- eventIds,
19175
- failed,
19176
- failedEventIds,
19177
- retrying,
19178
- retryingEventIds,
19179
- targetEventId,
19180
- targetDelivered: target.delivered,
19181
- targetBlocked: target.blocked,
19182
- };
19183
- }
19184
19623
 
19185
- /**
19186
- * Project one run's currently due non-start lifecycle events in one app-runtime
19187
- * append. The outbox remains the authority: all rows are leased first and are
19624
+ // A concurrent dispatcher can claim the target between batches. An empty
19625
+ // drain therefore does not prove delivery; read the durable acknowledgement
19626
+ // before reporting the target as pending.
19627
+ const target = await withClient((client) =>
19628
+ readPostgresSchedulerOutboxDeliveryStatus(
19629
+ client,
19630
+ { runId, eventId: targetEventId },
19631
+ options,
19632
+ ),
19633
+ );
19634
+ return {
19635
+ attempted,
19636
+ projected,
19637
+ eventIds,
19638
+ failed,
19639
+ failedEventIds,
19640
+ retrying,
19641
+ retryingEventIds,
19642
+ targetEventId,
19643
+ targetDelivered: target.delivered,
19644
+ targetBlocked: target.blocked,
19645
+ };
19646
+ } finally {
19647
+ if (scope.nextClaimed.length > 0) {
19648
+ await withClient((client) =>
19649
+ releasePostgresSchedulerOutboxLease(
19650
+ client,
19651
+ scope.nextClaimed.map((event) => event.eventId),
19652
+ { ...options, leaseOwner },
19653
+ ),
19654
+ );
19655
+ scope.nextClaimed.length = 0;
19656
+ }
19657
+ }
19658
+ }
19659
+
19660
+ /**
19661
+ * Project one run's currently due non-start lifecycle events in one app-runtime
19662
+ * append. The outbox remains the authority: all rows are leased first and are
19188
19663
  * marked delivered only after the atomic append succeeds. This preserves event
19189
19664
  * order while avoiding one callback round trip for `waiting -> resumed ->
19190
19665
  * terminal`, the normal detached-run completion shape.
@@ -19219,10 +19694,20 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19219
19694
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
19220
19695
  scope?: RunLedgerOutboxDrainScope,
19221
19696
  authorityScope: RunLedgerOutboxAuthorityScope = { authority: null },
19697
+ remainingEvents: number | null = null,
19222
19698
  ): Promise<PostgresSchedulerOutboxProjectionResult> {
19223
19699
  const dispatchStartedAtMs = Date.now();
19700
+ const tokenAuthority = input.tokenAuthority ?? 'local_mint';
19224
19701
  const leaseOwner =
19225
19702
  input.leaseOwner?.trim() || `run-ledger-outbox-dispatch:${randomUUID()}`;
19703
+ const needsLaunch =
19704
+ !input.searchOnly && (tokenAuthority === 'gateway' || !input.runtimeApi);
19705
+ const launchSelection = needsLaunch
19706
+ ? `CASE
19707
+ WHEN ${runLedgerSearchOnlyOutboxFactSql('head')} THEN NULL::jsonb
19708
+ ELSE ${runLedgerDispatchLaunchSql('runs')}
19709
+ END`
19710
+ : 'NULL::jsonb';
19226
19711
  const continuation: { current: RunLedgerOutboxDrainScope | null } = {
19227
19712
  current: scope ?? null,
19228
19713
  };
@@ -19272,22 +19757,27 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19272
19757
  // for each lifecycle transition. Only the two dedicated projectors may
19273
19758
  // perform the global distinct-head scan below.
19274
19759
  const selected = targetRunId
19275
- ? await client.query<{ run_id: string }>(
19760
+ ? await client.query<{
19761
+ run_id: string;
19762
+ dispatch_launch_json: unknown;
19763
+ }>(
19276
19764
  `
19277
19765
  WITH head AS MATERIALIZED (
19278
19766
  SELECT pending.run_id,
19279
19767
  pending.event_id,
19280
19768
  pending.event_type,
19769
+ pending.payload_json,
19281
19770
  pending.created_at,
19282
19771
  pending.next_attempt_at,
19283
19772
  pending.failed_at
19284
19773
  FROM ${tableName('outbox', options)} AS pending
19285
19774
  WHERE pending.run_id = $1
19286
19775
  AND pending.delivered_at IS NULL
19776
+ AND ${searchLaneSql('pending')} = ${input.searchOnly === true ? 'true' : 'false'}
19287
19777
  ORDER BY ${runLedgerOrderSql('pending')}
19288
19778
  LIMIT 1
19289
19779
  )
19290
- SELECT runs.run_id
19780
+ SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json
19291
19781
  FROM head
19292
19782
  JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
19293
19783
  WHERE head.failed_at IS NULL
@@ -19296,6 +19786,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19296
19786
  AND NOT EXISTS (
19297
19787
  SELECT 1 FROM ${tableName('outbox', options)} AS active
19298
19788
  WHERE active.run_id = runs.run_id
19789
+ AND ${searchLaneSql('active')} = ${input.searchOnly === true ? 'true' : 'false'}
19299
19790
  AND active.delivered_at IS NULL
19300
19791
  AND active.lease_owner IS NOT NULL
19301
19792
  AND active.lease_expires_at > now()
@@ -19305,20 +19796,31 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19305
19796
  `,
19306
19797
  [targetRunId, leaseOwner, input.headEventId?.trim() || null],
19307
19798
  )
19308
- : await client.query<{ run_id: string }>(`
19309
- WITH heads AS MATERIALIZED (
19799
+ : await client.query<{
19800
+ run_id: string;
19801
+ dispatch_launch_json: unknown;
19802
+ }>(`
19803
+ WITH ${
19804
+ input.searchOnly === true
19805
+ ? ''
19806
+ : `waiting_runs AS MATERIALIZED (
19807
+ ${runLedgerWaitingRunsSql(options)}
19808
+ ), `
19809
+ }heads AS MATERIALIZED (
19310
19810
  SELECT DISTINCT ON (pending.run_id)
19311
19811
  pending.run_id,
19312
19812
  pending.event_id,
19313
19813
  pending.event_type,
19814
+ pending.payload_json,
19314
19815
  pending.created_at,
19315
19816
  pending.next_attempt_at,
19316
19817
  pending.failed_at
19317
19818
  FROM ${tableName('outbox', options)} AS pending
19318
19819
  WHERE pending.delivered_at IS NULL
19820
+ AND ${searchLaneSql('pending')} = ${input.searchOnly === true ? 'true' : 'false'}
19319
19821
  ORDER BY pending.run_id, ${runLedgerOrderSql('pending')}
19320
19822
  )
19321
- SELECT runs.run_id
19823
+ SELECT runs.run_id, ${launchSelection} AS dispatch_launch_json
19322
19824
  FROM heads AS head
19323
19825
  JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
19324
19826
  WHERE head.failed_at IS NULL
@@ -19326,12 +19828,13 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19326
19828
  AND NOT EXISTS (
19327
19829
  SELECT 1 FROM ${tableName('outbox', options)} AS active
19328
19830
  WHERE active.run_id = runs.run_id
19831
+ AND ${searchLaneSql('active')} = ${input.searchOnly === true ? 'true' : 'false'}
19329
19832
  AND active.delivered_at IS NULL
19330
19833
  AND active.lease_owner IS NOT NULL
19331
19834
  AND active.lease_expires_at > now()
19332
19835
  )
19333
19836
  -- admission_first: admission, then the causal head of a run with
19334
- -- a pending terminal, then FIFO progress. oldest_ready keeps its
19837
+ -- a pending terminal or public wait, then FIFO progress. oldest_ready keeps its
19335
19838
  -- pure age order so continuous admissions/terminals cannot starve
19336
19839
  -- ordinary progress.
19337
19840
  ORDER BY ${
@@ -19340,7 +19843,8 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19340
19843
  : `CASE
19341
19844
  WHEN head.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0
19342
19845
  WHEN ${runLedgerTerminalPendingSql('head', options)} THEN 1
19343
- ELSE 2
19846
+ ${input.searchOnly === true ? '' : `WHEN ${runLedgerWaitingPendingSql('head')} THEN 2`}
19847
+ ELSE 3
19344
19848
  END,`
19345
19849
  }
19346
19850
  head.created_at, runs.run_id
@@ -19356,20 +19860,33 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19356
19860
  leaseOwner,
19357
19861
  leaseSeconds: input.leaseSeconds ?? 120,
19358
19862
  inTransaction: true,
19863
+ searchOnly: input.searchOnly,
19359
19864
  },
19360
19865
  options,
19361
19866
  )
19362
19867
  : [];
19363
19868
  await client.query('COMMIT');
19364
- return claimed;
19869
+ // The launch is immutable. Carry only its authority/routing fields from
19870
+ // the run we already locked, rather than queuing another scarce checkout
19871
+ // between the claim and gateway request. The gateway still proves the
19872
+ // active outbox lease; no database checkout spans either HTTP hop.
19873
+ return {
19874
+ events: claimed,
19875
+ launchJson: selected.rows[0]?.dispatch_launch_json,
19876
+ };
19365
19877
  } catch (error) {
19366
19878
  await client.query('ROLLBACK');
19367
19879
  throw error;
19368
19880
  }
19369
19881
  };
19370
- const events = scope?.nextClaimed.length
19882
+ const preclaimed = scope?.nextClaimed.length
19371
19883
  ? scope.nextClaimed.splice(0)
19884
+ : null;
19885
+ const claimed = preclaimed
19886
+ ? { events: preclaimed, launchJson: null }
19372
19887
  : await withClient(claimEvents);
19888
+ const { launchJson } = claimed;
19889
+ let events = claimed.events;
19373
19890
  if (events.length === 0) {
19374
19891
  return {
19375
19892
  attempted: 0,
@@ -19386,17 +19903,160 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19386
19903
  let authorityReadyAtMs: number | null = null;
19387
19904
  let appendFinishedAtMs: number | null = null;
19388
19905
  let acknowledgedAtMs: number | null = null;
19906
+ const materializeDeferredObservability = async () => {
19907
+ if (
19908
+ events.some(
19909
+ (event) =>
19910
+ payloadField(
19911
+ event.payload,
19912
+ RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD,
19913
+ ) === true,
19914
+ )
19915
+ ) {
19916
+ if (!input.materializeObservability) {
19917
+ // Inline admission/lifecycle/read-heal paths must never index logs.
19918
+ // Release and wake the existing projector without charging a retry.
19919
+ const released = await withClient((client) =>
19920
+ releasePostgresSchedulerOutboxLease(
19921
+ client,
19922
+ events.map((event) => event.eventId),
19923
+ { ...options, leaseOwner },
19924
+ ),
19925
+ );
19926
+ return {
19927
+ attempted: events.length,
19928
+ projected: 0,
19929
+ eventIds: [],
19930
+ failed: 0,
19931
+ failedEventIds: [],
19932
+ retrying: released.length,
19933
+ retryingEventIds: released,
19934
+ };
19935
+ }
19936
+ for (const event of events) {
19937
+ if (
19938
+ payloadField(
19939
+ event.payload,
19940
+ RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD,
19941
+ ) !== true
19942
+ )
19943
+ continue;
19944
+ await withClient(async (client) => {
19945
+ await renewPostgresRunLedgerDelivery(
19946
+ client,
19947
+ {
19948
+ eventIds: events.map((row) => row.eventId),
19949
+ leaseOwner,
19950
+ leaseSeconds: 120,
19951
+ },
19952
+ options,
19953
+ );
19954
+ // Keep projection below its renewed lease. The final payload update
19955
+ // fences ownership; indexing locks neither outbox nor Run rows.
19956
+ await client.query('BEGIN');
19957
+ try {
19958
+ // Let PostgreSQL cancel before the pool's client-side timeout,
19959
+ // so rollback completes while the checkout is still usable.
19960
+ await client.query(
19961
+ "SELECT set_config('statement_timeout', $1, true)",
19962
+ [
19963
+ `${Math.min(60_000, RUNTIME_RELIABILITY_POLICY.schedulerDatabase.queryTimeoutMs - 1_000)}ms`,
19964
+ ],
19965
+ );
19966
+ const result = await client.query<{ payload: unknown }>(
19967
+ `SELECT ${quoteIdent(schemaName(options))}.materialize_runtime_outbox_event($1, $2) AS payload`,
19968
+ [event.eventId, leaseOwner],
19969
+ );
19970
+ await client.query('COMMIT');
19971
+ event.payload = result.rows[0]!.payload;
19972
+ } catch (error) {
19973
+ await client.query('ROLLBACK');
19974
+ if (
19975
+ error &&
19976
+ typeof error === 'object' &&
19977
+ 'code' in error &&
19978
+ error.code === 'P0001'
19979
+ ) {
19980
+ // The installed projector's explicit RAISEs reject malformed
19981
+ // or conflicting immutable facts. Retain them in the existing
19982
+ // quarantine instead of retrying the same invalid input forever.
19983
+ throw new DeferredRuntimeProjectionInputError(
19984
+ event.eventId,
19985
+ error instanceof Error
19986
+ ? error.message
19987
+ : 'Invalid runtime event projection input.',
19988
+ );
19989
+ }
19990
+ throw error;
19991
+ }
19992
+ });
19993
+ }
19994
+ }
19995
+ return null;
19996
+ };
19389
19997
  try {
19998
+ // The ledger converter rejects raw deferred facts. Inline callers hand
19999
+ // them off; the background owner projects them before inspecting the
20000
+ // compact lifecycle payload or deciding that it needs no HTTP append.
20001
+ const initialHandedOff = await materializeDeferredObservability();
20002
+ if (initialHandedOff) return initialHandedOff;
20003
+ if (input.searchOnly) {
20004
+ if (
20005
+ events.some(
20006
+ (event) =>
20007
+ payloadField(event.payload, RUNTIME_EVENT_SEARCH_ONLY_FIELD) !==
20008
+ true,
20009
+ )
20010
+ )
20011
+ throw new Error('Search lease includes a lifecycle fact.');
20012
+ const delivered = await withClient((client) =>
20013
+ markPostgresSchedulerOutboxDeliveredRows(
20014
+ client,
20015
+ events.map((event) => event.eventId),
20016
+ { ...options, leaseOwner },
20017
+ ),
20018
+ );
20019
+ if (delivered.length !== events.length)
20020
+ throw new RunLedgerProjectionLeaseLostError(
20021
+ 'delivery_lease_not_current',
20022
+ );
20023
+ if (input.onDelivered) {
20024
+ try {
20025
+ input.onDelivered(delivered);
20026
+ } catch (error) {
20027
+ console.warn('[run-ledger-outbox.delivery_observer_failed]', {
20028
+ error: error instanceof Error ? error.message : String(error),
20029
+ });
20030
+ }
20031
+ }
20032
+ return {
20033
+ attempted: events.length,
20034
+ projected: delivered.length,
20035
+ eventIds: events.map((event) => event.eventId),
20036
+ failed: 0,
20037
+ failedEventIds: [],
20038
+ retrying: 0,
20039
+ retryingEventIds: [],
20040
+ };
20041
+ }
19390
20042
  const first = events[0]!;
19391
- const tokenAuthority = input.tokenAuthority ?? 'local_mint';
20043
+ let ledgerEvents = RUN_SCHEDULER_OUTBOX_START_EVENT_TYPES.includes(
20044
+ first.eventType,
20045
+ )
20046
+ ? null
20047
+ : events.flatMap(buildRunLedgerEventsFromPostgresSchedulerOutbox);
20048
+ // Search-only facts are already durable and have no ledger projection.
20049
+ // Validate and acknowledge them without acquiring runtime authority or
20050
+ // renewing a network-delivery lease. A following terminal still uses the
20051
+ // same fenced continuation and loads authority only when it needs a write.
19392
20052
  if (
19393
20053
  scope &&
19394
20054
  (scope.runId !== first.runId || scope.leaseOwner !== leaseOwner)
19395
20055
  ) {
19396
20056
  throw new Error('Run-ledger continuation is outside its dispatch scope.');
19397
20057
  }
19398
- // Minted executor authority is reused only inside this finite run/owner
19399
- // drain and only while it can cover the HTTP request plus its SQL renewal.
20058
+ const needsRuntimeApi = ledgerEvents === null || ledgerEvents.length > 0;
20059
+ let runtimeApi: WorkerRuntimeApiContext | null = null;
19400
20060
  const reuseMarginMs =
19401
20061
  Math.max(
19402
20062
  1,
@@ -19406,23 +20066,36 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19406
20066
  ),
19407
20067
  ) + RUNTIME_RELIABILITY_POLICY.schedulerDatabase.queryTimeoutMs;
19408
20068
  const cachedAuthority = authorityScope.authority;
20069
+ const resolveRunRuntimeApi = () =>
20070
+ launchJson !== null
20071
+ ? resolvePostgresSchedulerRuntimeApiForRun(
20072
+ launchJson,
20073
+ first.runId,
20074
+ first.leaseOwner
20075
+ ? { eventId: first.eventId, leaseOwner: first.leaseOwner }
20076
+ : null,
20077
+ tokenAuthority,
20078
+ options,
20079
+ input.runtimeApi?.signal,
20080
+ )
20081
+ : loadPostgresSchedulerRuntimeApiForRun(
20082
+ withClient,
20083
+ first.runId,
20084
+ first.leaseOwner
20085
+ ? { eventId: first.eventId, leaseOwner: first.leaseOwner }
20086
+ : null,
20087
+ tokenAuthority,
20088
+ options,
20089
+ input.runtimeApi?.signal,
20090
+ );
19409
20091
  const gatewayRuntimeApi =
19410
- tokenAuthority === 'gateway'
20092
+ needsRuntimeApi && tokenAuthority === 'gateway'
19411
20093
  ? cachedAuthority?.runId === first.runId &&
19412
20094
  cachedAuthority.leaseOwner === leaseOwner &&
19413
20095
  cachedAuthority.expiresAtMs !== null &&
19414
20096
  cachedAuthority.expiresAtMs > Date.now() + reuseMarginMs
19415
20097
  ? cachedAuthority.runtimeApi
19416
- : await loadPostgresSchedulerRuntimeApiForRun(
19417
- withClient,
19418
- first.runId,
19419
- first.leaseOwner
19420
- ? { eventId: first.eventId, leaseOwner: first.leaseOwner }
19421
- : null,
19422
- tokenAuthority,
19423
- options,
19424
- input.runtimeApi?.signal,
19425
- )
20098
+ : await resolveRunRuntimeApi()
19426
20099
  : null;
19427
20100
  if (
19428
20101
  gatewayRuntimeApi &&
@@ -19434,67 +20107,101 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19434
20107
  'Run Ledger dispatch origin changed within an immutable launch.',
19435
20108
  );
19436
20109
  }
19437
- const resolvedRuntimeApi =
19438
- gatewayRuntimeApi ??
19439
- input.runtimeApi ??
19440
- (await loadPostgresSchedulerRuntimeApiForRun(
19441
- withClient,
19442
- first.runId,
19443
- first.leaseOwner
19444
- ? { eventId: first.eventId, leaseOwner: first.leaseOwner }
19445
- : null,
19446
- tokenAuthority,
19447
- options,
19448
- ));
19449
- const runtimeApi = {
19450
- ...resolvedRuntimeApi,
19451
- ...(gatewayRuntimeApi && input.runtimeApi
19452
- ? {
19453
- ...(input.runtimeApi.fetch
19454
- ? { fetch: input.runtimeApi.fetch }
19455
- : {}),
19456
- ...(input.runtimeApi.requestTimeoutMs !== undefined
19457
- ? { requestTimeoutMs: input.runtimeApi.requestTimeoutMs }
19458
- : {}),
19459
- ...(input.runtimeApi.onResponseMeta
19460
- ? { onResponseMeta: input.runtimeApi.onResponseMeta }
19461
- : {}),
19462
- ...(input.runtimeApi.runtimeTestFaultHeader !== undefined
19463
- ? {
19464
- runtimeTestFaultHeader:
19465
- input.runtimeApi.runtimeTestFaultHeader,
19466
- }
19467
- : {}),
19468
- ...(input.runtimeApi.signal
19469
- ? { signal: input.runtimeApi.signal }
19470
- : {}),
19471
- }
19472
- : {}),
19473
- retryPolicy: 'none' as const,
19474
- // The caller-owned admission deadline (see `signal` above) wins.
19475
- ...(input.signal ? { signal: input.signal } : {}),
19476
- };
19477
- const tokenExpiresAtMs = runtimeApiTokenExpiresAtMs(
19478
- runtimeApi.executorToken,
19479
- );
19480
- if (tokenAuthority === 'gateway' && gatewayRuntimeApi) {
19481
- authorityScope.authority = {
19482
- runId: first.runId,
19483
- leaseOwner,
19484
- runtimeApi,
19485
- expiresAtMs: tokenExpiresAtMs,
20110
+ if (needsRuntimeApi) {
20111
+ const resolvedRuntimeApi =
20112
+ gatewayRuntimeApi ?? input.runtimeApi ?? (await resolveRunRuntimeApi());
20113
+ runtimeApi = {
20114
+ ...resolvedRuntimeApi,
20115
+ ...(gatewayRuntimeApi && input.runtimeApi
20116
+ ? {
20117
+ ...(input.runtimeApi.fetch
20118
+ ? { fetch: input.runtimeApi.fetch }
20119
+ : {}),
20120
+ ...(input.runtimeApi.requestTimeoutMs !== undefined
20121
+ ? { requestTimeoutMs: input.runtimeApi.requestTimeoutMs }
20122
+ : {}),
20123
+ ...(input.runtimeApi.onResponseMeta
20124
+ ? { onResponseMeta: input.runtimeApi.onResponseMeta }
20125
+ : {}),
20126
+ ...(input.runtimeApi.runtimeTestFaultHeader !== undefined
20127
+ ? {
20128
+ runtimeTestFaultHeader:
20129
+ input.runtimeApi.runtimeTestFaultHeader,
20130
+ }
20131
+ : {}),
20132
+ ...(input.runtimeApi.signal
20133
+ ? { signal: input.runtimeApi.signal }
20134
+ : {}),
20135
+ }
20136
+ : {}),
20137
+ retryPolicy: 'none' as const,
20138
+ // The caller-owned admission deadline (see `signal` above) wins.
20139
+ ...(input.signal ? { signal: input.signal } : {}),
19486
20140
  };
20141
+ if (tokenAuthority === 'gateway' && gatewayRuntimeApi) {
20142
+ authorityScope.authority = {
20143
+ runId: first.runId,
20144
+ leaseOwner,
20145
+ runtimeApi,
20146
+ expiresAtMs: runtimeApiTokenExpiresAtMs(runtimeApi.executorToken),
20147
+ };
20148
+ }
20149
+ authorityReadyAtMs = Date.now();
20150
+ }
20151
+ // The initial claim deliberately uses the old contract until its immutable
20152
+ // launch receiver advertises support. Widen under the same run/owner fence
20153
+ // before the first HTTP append, so an already-finished run need not pay one
20154
+ // authority/append/ACK round trip for every ingress predecessor.
20155
+ if (
20156
+ runtimeApi?.runLedgerIngressPrefixBatching === true &&
20157
+ events.length === 1 &&
20158
+ first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents
20159
+ ) {
20160
+ const widened = await withClient((client) =>
20161
+ readPendingPostgresSchedulerOutboxEvents(
20162
+ client,
20163
+ {
20164
+ runId: first.runId,
20165
+ leaseOwner,
20166
+ leaseSeconds: input.leaseSeconds ?? 120,
20167
+ limit: Math.min(
20168
+ RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
20169
+ remainingEvents ?? RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
20170
+ ),
20171
+ allowIngressPrefix: true,
20172
+ extendOwnedHead: first.eventId,
20173
+ },
20174
+ options,
20175
+ ),
20176
+ );
20177
+ if (widened[0]?.eventId !== first.eventId) {
20178
+ throw new RunLedgerProjectionLeaseLostError(
20179
+ 'head_lease_owner_mismatch',
20180
+ );
20181
+ }
20182
+ events = widened;
20183
+ }
20184
+ // Widening can add deferred facts to the lease. Project the complete
20185
+ // frozen prefix before any HTTP append, including its newly claimed rows.
20186
+ const handedOff = await materializeDeferredObservability();
20187
+ if (handedOff) return handedOff;
20188
+ if (ledgerEvents !== null) {
20189
+ ledgerEvents = events.flatMap(
20190
+ buildRunLedgerEventsFromPostgresSchedulerOutbox,
20191
+ );
19487
20192
  }
19488
- authorityReadyAtMs = Date.now();
19489
20193
  const renewDelivery = async () => {
19490
20194
  const currentAuthority = authorityScope.authority;
19491
20195
  if (
20196
+ runtimeApi &&
19492
20197
  tokenAuthority === 'gateway' &&
19493
20198
  currentAuthority?.runId === first.runId &&
19494
20199
  currentAuthority.leaseOwner === leaseOwner &&
19495
20200
  currentAuthority.expiresAtMs !== null &&
19496
20201
  currentAuthority.expiresAtMs <= Date.now() + reuseMarginMs
19497
20202
  ) {
20203
+ // Initial delivery reuses the claim's routing fields. Renewal must
20204
+ // reread authority so an origin change cannot pass under that cache.
19498
20205
  const refreshed = await loadPostgresSchedulerRuntimeApiForRun(
19499
20206
  withClient,
19500
20207
  first.runId,
@@ -19508,14 +20215,16 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19508
20215
  'Run Ledger dispatch origin changed within an immutable launch.',
19509
20216
  );
19510
20217
  }
19511
- runtimeApi.executorToken = refreshed.executorToken;
20218
+ runtimeApi.executorToken = refreshed.refreshExecutorToken
20219
+ ? await refreshed.refreshExecutorToken()
20220
+ : refreshed.executorToken;
19512
20221
  runtimeApi.vercelProtectionBypassToken =
19513
20222
  refreshed.vercelProtectionBypassToken;
19514
20223
  authorityScope.authority = {
19515
20224
  runId: first.runId,
19516
20225
  leaseOwner,
19517
20226
  runtimeApi,
19518
- expiresAtMs: runtimeApiTokenExpiresAtMs(refreshed.executorToken),
20227
+ expiresAtMs: runtimeApiTokenExpiresAtMs(runtimeApi.executorToken),
19519
20228
  };
19520
20229
  }
19521
20230
  await withClient((client) =>
@@ -19525,70 +20234,107 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19525
20234
  eventIds: events.map((event) => event.eventId),
19526
20235
  leaseOwner,
19527
20236
  leaseSeconds: input.leaseSeconds ?? 120,
20237
+ freezeReceipt:
20238
+ runtimeApi?.runLedgerIngressPrefixBatching === true &&
20239
+ events.some(
20240
+ (event) =>
20241
+ event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents,
20242
+ ),
19528
20243
  },
19529
20244
  options,
19530
20245
  ),
19531
20246
  );
19532
20247
  };
19533
- // A pending start is always claimed alone because later events require its
19534
- // snapshot. It cannot share the append endpoint, so retain that mutation
19535
- // as its own exact delivery.
19536
- if (first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runCreated) {
19537
- await renewDelivery();
19538
- await projectPostgresSchedulerCreatedEvent(runtimeApi, first);
19539
- } else if (first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runStarted) {
19540
- await renewDelivery();
19541
- const leaseOwner = first.leaseOwner;
19542
- const launch = leaseOwner
19543
- ? await withClient((client) =>
19544
- readRunLedgerDispatchLaunchForLease(
19545
- client,
19546
- {
19547
- runId: first.runId,
19548
- eventId: first.eventId,
19549
- leaseOwner,
19550
- },
19551
- options,
19552
- ),
19553
- )
19554
- : null;
19555
- await projectPostgresSchedulerStartEvent(runtimeApi, first, launch);
19556
- } else {
19557
- const ledgerEvents = events.flatMap(
19558
- buildRunLedgerEventsFromPostgresSchedulerOutbox,
19559
- );
19560
- if (ledgerEvents.length > 0) {
19561
- const terminal = [...events]
19562
- .reverse()
19563
- .find((event) =>
19564
- RUN_SCHEDULER_OUTBOX_TERMINAL_EVENT_TYPES.includes(event.eventType),
19565
- );
19566
- const batchDeliveryKey = createHash('sha256')
19567
- .update(events.map((event) => event.deliveryKey).join('\n'))
19568
- .digest('hex');
19569
- await projectRunEventsViaAppRuntime(runtimeApi, {
19570
- playId: first.runId,
19571
- events: ledgerEvents,
19572
- deliveryLeaseOwner: leaseOwner,
19573
- beforeBatch: renewDelivery,
19574
- idempotencyKey: terminal
19575
- ? buildRunLedgerTerminalIdempotencyKey({
19576
- runId: first.runId,
19577
- deliveryKey: terminal.deliveryKey,
19578
- })
19579
- : buildRunLedgerEventsIdempotencyKey({
19580
- runId: first.runId,
19581
- deliveryDigest: batchDeliveryKey,
19582
- }),
19583
- });
19584
- } else {
19585
- await renewDelivery();
20248
+ if (needsRuntimeApi && runtimeApi) {
20249
+ const append = async (runtimeApi: WorkerRuntimeApiContext) => {
20250
+ // A pending start is always claimed alone because later events require
20251
+ // its snapshot. Retain that mutation as its own exact delivery.
20252
+ if (first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runCreated) {
20253
+ await renewDelivery();
20254
+ await projectPostgresSchedulerCreatedEvent(runtimeApi, first);
20255
+ } else if (
20256
+ first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runStarted
20257
+ ) {
20258
+ await renewDelivery();
20259
+ const startLeaseOwner = first.leaseOwner;
20260
+ const launch = startLeaseOwner
20261
+ ? await withClient((client) =>
20262
+ readRunLedgerDispatchLaunchForLease(
20263
+ client,
20264
+ {
20265
+ runId: first.runId,
20266
+ eventId: first.eventId,
20267
+ leaseOwner: startLeaseOwner,
20268
+ },
20269
+ options,
20270
+ ),
20271
+ )
20272
+ : null;
20273
+ await projectPostgresSchedulerStartEvent(runtimeApi, first, launch);
20274
+ } else if (ledgerEvents && ledgerEvents.length > 0) {
20275
+ const terminal = [...events]
20276
+ .reverse()
20277
+ .find((event) =>
20278
+ RUN_SCHEDULER_OUTBOX_TERMINAL_EVENT_TYPES.includes(
20279
+ event.eventType,
20280
+ ),
20281
+ );
20282
+ const batchDeliveryKey = createHash('sha256')
20283
+ .update(events.map((event) => event.deliveryKey).join('\n'))
20284
+ .digest('hex');
20285
+ // The transport renews before every HTTP chunk; another renewal here
20286
+ // would add a checkout without protecting an intervening side effect.
20287
+ await projectRunEventsViaAppRuntime(runtimeApi, {
20288
+ playId: first.runId,
20289
+ events: ledgerEvents,
20290
+ deliveryLeaseOwner: leaseOwner,
20291
+ beforeBatch: renewDelivery,
20292
+ idempotencyKey: terminal
20293
+ ? buildRunLedgerTerminalIdempotencyKey({
20294
+ runId: first.runId,
20295
+ deliveryKey: terminal.deliveryKey,
20296
+ })
20297
+ : buildRunLedgerEventsIdempotencyKey({
20298
+ runId: first.runId,
20299
+ deliveryDigest: batchDeliveryKey,
20300
+ }),
20301
+ });
20302
+ }
20303
+ };
20304
+ try {
20305
+ await append(runtimeApi);
20306
+ } catch (error) {
20307
+ // Refresh a reused authority once through this delivery's active lease.
20308
+ // Both attempts retain the same fenced append and idempotency key.
20309
+ if (
20310
+ !gatewayRuntimeApi?.refreshExecutorToken ||
20311
+ !(error instanceof AppRuntimeApiResponseError) ||
20312
+ error.status !== 401
20313
+ )
20314
+ throw error;
20315
+ runtimeApi.executorToken = await gatewayRuntimeApi.refreshExecutorToken();
20316
+ await append(runtimeApi);
19586
20317
  }
20318
+ } else if (tokenAuthority !== 'gateway') {
20319
+ // App-side acknowledgement retains lease validation. The gateway can
20320
+ // acknowledge a durable search-only fact without a network lease hop.
20321
+ await renewDelivery();
19587
20322
  }
19588
20323
  appendFinishedAtMs = Date.now();
19589
20324
  const acknowledgement = await acknowledgeRunLedgerPrefixWithContinuation(
19590
20325
  withClient,
19591
- { input, events, leaseOwner, runtimeApi, scope, continuation },
20326
+ {
20327
+ input,
20328
+ events,
20329
+ leaseOwner,
20330
+ runtimeApi,
20331
+ scope,
20332
+ continuation,
20333
+ continuationAllowance:
20334
+ remainingEvents === null
20335
+ ? null
20336
+ : Math.max(0, remainingEvents - events.length),
20337
+ },
19592
20338
  options,
19593
20339
  );
19594
20340
  const { deliveredRows } = acknowledgement;
@@ -19630,6 +20376,10 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19630
20376
  authorityScope,
19631
20377
  );
19632
20378
  } catch (error) {
20379
+ // A failed append must reacquire receiver truth on its next attempt,
20380
+ // including an older receiver that rejected a frozen compound receipt.
20381
+ authorityScope.authority?.runtimeApi.invalidateExecutorToken?.();
20382
+ authorityScope.authority = null;
19633
20383
  await releasePendingContinuation();
19634
20384
  if (input.signal?.aborted) {
19635
20385
  // The caller's latency budget expired; this is not a delivery failure.
@@ -19654,25 +20404,98 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19654
20404
  retryingEventIds: released,
19655
20405
  };
19656
20406
  }
19657
- const settlement = await withClient(async (client) => {
19658
- const blocked: string[] = [];
19659
- const retrying: string[] = [];
19660
- for (const event of events) {
19661
- const outcome = await settlePostgresSchedulerOutboxFailure(
20407
+ if (error instanceof DeferredRuntimeProjectionInputError) {
20408
+ const outcome = await withClient((client) =>
20409
+ settlePostgresSchedulerOutboxFailure(client, error.eventId, error, {
20410
+ ...options,
20411
+ leaseOwner,
20412
+ }),
20413
+ );
20414
+ // Earlier facts may already be indexed. Only the invalid immutable row
20415
+ // is quarantined; valid predecessors remain deliverable in causal order.
20416
+ const released = await withClient((client) =>
20417
+ releasePostgresSchedulerOutboxLease(
19662
20418
  client,
19663
- event.eventId,
19664
- error,
20419
+ events
20420
+ .filter((event) => event.eventId !== error.eventId)
20421
+ .map((event) => event.eventId),
19665
20422
  { ...options, leaseOwner },
19666
- );
19667
- if (outcome === 'blocked') blocked.push(event.eventId);
19668
- else retrying.push(event.eventId);
20423
+ ),
20424
+ );
20425
+ const failed = outcome === 'blocked' ? [error.eventId] : [];
20426
+ const retrying =
20427
+ outcome === 'blocked' ? released : [...released, error.eventId];
20428
+ console.error('[run-ledger-outbox.invalid_projection_input]', {
20429
+ eventId: error.eventId,
20430
+ code: error.code,
20431
+ detail: error.message,
20432
+ });
20433
+ return {
20434
+ attempted: events.length,
20435
+ projected: 0,
20436
+ eventIds: [],
20437
+ failed: failed.length,
20438
+ failedEventIds: failed,
20439
+ retrying: retrying.length,
20440
+ retryingEventIds: retrying,
20441
+ };
20442
+ }
20443
+ const settlement = await withClient(async (client) => {
20444
+ await client.query('BEGIN');
20445
+ try {
20446
+ const blocked: string[] = [];
20447
+ const retrying: string[] = [];
20448
+ const ownedRetrying: string[] = [];
20449
+ for (const event of events) {
20450
+ const outcome = await settlePostgresSchedulerOutboxFailure(
20451
+ client,
20452
+ event.eventId,
20453
+ error,
20454
+ { ...options, leaseOwner },
20455
+ );
20456
+ if (outcome === 'blocked') blocked.push(event.eventId);
20457
+ else retrying.push(event.eventId);
20458
+ if (outcome === 'retrying') ownedRetrying.push(event.eventId);
20459
+ }
20460
+ // All members of this failed append must become due together. Keep
20461
+ // the existing per-row retry policy, taking its latest deadline only
20462
+ // for rows this owner actually settled, before releasing their locks.
20463
+ if (ownedRetrying.length > 1) {
20464
+ await client.query(
20465
+ `
20466
+ WITH retry AS (
20467
+ SELECT max(next_attempt_at) AS retry_at
20468
+ FROM ${tableName('outbox', options)}
20469
+ WHERE event_id = ANY($1::text[])
20470
+ AND delivered_at IS NULL AND failed_at IS NULL
20471
+ )
20472
+ UPDATE ${tableName('outbox', options)} AS outbox
20473
+ SET next_attempt_at = retry.retry_at
20474
+ FROM retry
20475
+ WHERE outbox.event_id = ANY($1::text[])
20476
+ AND outbox.delivered_at IS NULL AND outbox.failed_at IS NULL
20477
+ AND retry.retry_at IS NOT NULL
20478
+ `,
20479
+ [ownedRetrying],
20480
+ );
20481
+ }
20482
+ await client.query('COMMIT');
20483
+ return { blocked, retrying };
20484
+ } catch (settlementError) {
20485
+ await client.query('ROLLBACK');
20486
+ throw settlementError;
19669
20487
  }
19670
- return { blocked, retrying };
19671
20488
  });
20489
+ const failure = runLedgerDeliveryFailure({ error });
19672
20490
  console.warn('[run-ledger-outbox.batch_dispatch_failed]', {
19673
20491
  runId: input.runId ?? null,
19674
20492
  eventIds: events.map((event) => event.eventId),
19675
20493
  error: error instanceof Error ? error.message : String(error),
20494
+ code: failure.code,
20495
+ failureReason: failure.failureReason,
20496
+ ...(failure.projectionMismatch
20497
+ ? { projectionMismatch: failure.projectionMismatch }
20498
+ : {}),
19676
20499
  });
19677
20500
  return {
19678
20501
  attempted: events.length,
@@ -19781,9 +20604,10 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
19781
20604
  input: RunLedgerOutboxDispatchInput;
19782
20605
  events: PostgresSchedulerOutboxEvent[];
19783
20606
  leaseOwner: string;
19784
- runtimeApi: WorkerRuntimeApiContext;
20607
+ runtimeApi: WorkerRuntimeApiContext | null;
19785
20608
  scope?: RunLedgerOutboxDrainScope;
19786
20609
  continuation: { current: RunLedgerOutboxDrainScope | null };
20610
+ continuationAllowance: number | null;
19787
20611
  },
19788
20612
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
19789
20613
  ): Promise<{
@@ -19796,74 +20620,86 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
19796
20620
  !!input.scope ||
19797
20621
  (input.input.continueThroughTerminal === true && !input.input.headEventId);
19798
20622
  return withClient(async (client) => {
19799
- if (retainContinuation) {
19800
- await client.query('BEGIN');
19801
- }
19802
- try {
20623
+ const acknowledge = async () => {
19803
20624
  if (retainContinuation) {
19804
- await client.query(
19805
- `SELECT set_config('lock_timeout', $1, true), set_config('statement_timeout', $2, true)`,
19806
- [
19807
- `${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
19808
- `${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
19809
- ],
19810
- );
19811
- // Keep the acknowledged prefix and its successor under one run lock.
19812
- // No competing projector can take the next head while this ACK's
19813
- // response is travelling back to the lease holder.
19814
- await client.query(
19815
- `SELECT run_id FROM ${tableName('runs', options)} WHERE run_id = $1 FOR UPDATE`,
19816
- [runId],
19817
- );
20625
+ await client.query('BEGIN');
19818
20626
  }
19819
- const deliveredRows = await markPostgresSchedulerOutboxDeliveredRows(
19820
- client,
19821
- input.events.map((event) => event.eventId),
19822
- { ...options, leaseOwner: input.leaseOwner },
19823
- );
19824
- let continuation: RunLedgerOutboxDrainScope | null = null;
19825
- if (
19826
- retainContinuation &&
19827
- deliveredRows.length === input.events.length &&
19828
- !input.events.some((event) => event.eventId === targetEventId)
19829
- ) {
19830
- const target = await readPostgresSchedulerOutboxDeliveryStatus(
19831
- client,
19832
- { runId, eventId: targetEventId },
19833
- options,
19834
- );
19835
- if (target.exists && !target.delivered && !target.blocked) {
19836
- const nextClaimed = await readPendingPostgresSchedulerOutboxEvents(
19837
- client,
19838
- {
19839
- runId,
19840
- leaseOwner: input.leaseOwner,
19841
- leaseSeconds: input.input.leaseSeconds ?? 120,
19842
- limit: Math.min(
19843
- RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
19844
- input.input.limit ?? 1,
19845
- ),
19846
- inTransaction: true,
19847
- },
19848
- options,
20627
+ try {
20628
+ if (retainContinuation) {
20629
+ await client.query(
20630
+ `SELECT set_config('lock_timeout', $1, true), set_config('statement_timeout', $2, true)`,
20631
+ [
20632
+ `${RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS}ms`,
20633
+ `${RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS}ms`,
20634
+ ],
20635
+ );
20636
+ // Keep the acknowledged prefix and its successor under one run lock.
20637
+ // No competing projector can take the next head while this ACK's
20638
+ // response is travelling back to the lease holder.
20639
+ await client.query(
20640
+ `SELECT run_id FROM ${tableName('runs', options)} WHERE run_id = $1 FOR UPDATE`,
20641
+ [runId],
19849
20642
  );
19850
- if (nextClaimed.length > 0) {
19851
- continuation = input.scope ?? {
19852
- runId,
19853
- leaseOwner: input.leaseOwner,
19854
- targetEventId,
19855
- nextClaimed: [],
19856
- };
19857
- continuation.nextClaimed = nextClaimed;
19858
- input.continuation.current = continuation;
19859
- }
19860
20643
  }
20644
+ const eventIds = input.events.map((event) => event.eventId);
20645
+ const { deliveredRows, events: nextClaimed } =
20646
+ retainContinuation &&
20647
+ input.continuationAllowance !== 0 &&
20648
+ !eventIds.includes(targetEventId)
20649
+ ? await claimPostgresSchedulerOutboxPrefix(
20650
+ client,
20651
+ {
20652
+ runId,
20653
+ leaseOwner: input.leaseOwner,
20654
+ leaseSeconds: input.input.leaseSeconds ?? 120,
20655
+ limit: Math.min(
20656
+ RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
20657
+ input.runtimeApi?.runLedgerIngressPrefixBatching === true
20658
+ ? RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS
20659
+ : (input.input.limit ?? 1),
20660
+ input.continuationAllowance ??
20661
+ RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
20662
+ ),
20663
+ allowIngressPrefix:
20664
+ input.runtimeApi?.runLedgerIngressPrefixBatching === true,
20665
+ },
20666
+ options,
20667
+ { eventIds, targetEventId },
20668
+ )
20669
+ : {
20670
+ deliveredRows: await markPostgresSchedulerOutboxDeliveredRows(
20671
+ client,
20672
+ eventIds,
20673
+ { ...options, leaseOwner: input.leaseOwner },
20674
+ ),
20675
+ events: [],
20676
+ };
20677
+ let continuation: RunLedgerOutboxDrainScope | null = null;
20678
+ if (nextClaimed.length > 0) {
20679
+ continuation = input.scope ?? {
20680
+ runId,
20681
+ leaseOwner: input.leaseOwner,
20682
+ targetEventId,
20683
+ nextClaimed: [],
20684
+ };
20685
+ continuation.nextClaimed = nextClaimed;
20686
+ input.continuation.current = continuation;
20687
+ }
20688
+ if (retainContinuation) await client.query('COMMIT');
20689
+ return { deliveredRows, continuation };
20690
+ } catch (error) {
20691
+ if (retainContinuation) await client.query('ROLLBACK');
20692
+ throw error;
19861
20693
  }
19862
- if (retainContinuation) await client.query('COMMIT');
19863
- return { deliveredRows, continuation };
20694
+ };
20695
+ try {
20696
+ return await acknowledge();
19864
20697
  } catch (error) {
19865
- if (retainContinuation) await client.query('ROLLBACK');
19866
- throw error;
20698
+ if (!isMissingOutboxDeliveryBatchMarker(error)) throw error;
20699
+ // The first statement recorded this operation-local missing column and
20700
+ // rolled back its ACK. Reacquire the run lock and take a fresh snapshot
20701
+ // for the existing compatibility claim, without another ledger append.
20702
+ return await acknowledge();
19867
20703
  }
19868
20704
  });
19869
20705
  }
@@ -20019,70 +20855,45 @@ async function projectPostgresSchedulerStartEvent(
20019
20855
  });
20020
20856
  }
20021
20857
 
20022
- export type RunLedgerDispatchLaunch = Pick<
20023
- PlaySchedulerSubmitInput,
20024
- | 'runId'
20025
- | 'playId'
20026
- | 'playName'
20027
- | 'executorToken'
20028
- | 'baseUrl'
20029
- | 'runtimeApiBaseUrl'
20030
- | 'runtimeTokenOrigin'
20031
- | 'orgId'
20032
- | 'runtimeAuthority'
20033
- | 'vercelProtectionBypassToken'
20034
- >;
20858
+ export type { RunLedgerDispatchLaunch } from './run-ledger-dispatch-authority';
20035
20859
 
20036
- /** Authority reads never transfer customer input, artifacts or execution graphs. */
20037
- function runLedgerDispatchLaunchSql(alias: string): string {
20038
- const fields: (keyof RunLedgerDispatchLaunch)[] = [
20039
- 'runId',
20040
- 'playId',
20041
- 'playName',
20042
- 'executorToken',
20043
- 'baseUrl',
20044
- 'runtimeApiBaseUrl',
20045
- 'runtimeTokenOrigin',
20046
- 'orgId',
20047
- 'runtimeAuthority',
20048
- 'vercelProtectionBypassToken',
20049
- ];
20050
- return `jsonb_build_object(${fields
20051
- .map((field) => `'${field}', ${alias}.launch_json->'${field}'`)
20052
- .join(', ')})`;
20860
+ /** Search-only `run.events` facts settle without consulting launch credentials. */
20861
+ function runLedgerSearchOnlyOutboxFactSql(alias: string): string {
20862
+ const events = `CASE
20863
+ WHEN jsonb_typeof(${alias}.payload_json->'events') = 'array'
20864
+ THEN ${alias}.payload_json->'events'
20865
+ ELSE '[]'::jsonb
20866
+ END`;
20867
+ return `(
20868
+ ${alias}.event_type = '${RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents}'
20869
+ AND ${alias}.payload_json->>'logBodyStore' = 'planet_scale'
20870
+ AND jsonb_array_length(${events}) = 0
20871
+ )`;
20053
20872
  }
20054
20873
 
20055
- async function loadPostgresSchedulerRuntimeApiForRun(
20056
- withClient: <T>(
20057
- operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
20058
- ) => Promise<T>,
20874
+ async function resolvePostgresSchedulerRuntimeApiForRun(
20875
+ launchJson: unknown,
20059
20876
  runId: string,
20060
20877
  dispatchLease: { eventId: string; leaseOwner: string } | null,
20061
20878
  tokenAuthority: RunLedgerOutboxTokenAuthority,
20062
20879
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
20063
20880
  signal?: AbortSignal,
20064
- ): Promise<WorkerRuntimeApiContext> {
20065
- // Release the scarce scheduler checkout before the token authority hop.
20066
- // A slow gateway must not retain either projector pool client, and its
20067
- // launch context must not hydrate customer execution graphs.
20068
- const result = await withClient((client) =>
20069
- client.query<{ launch_json: unknown }>(
20070
- `SELECT ${runLedgerDispatchLaunchSql('runs')} AS launch_json
20071
- FROM ${tableName('runs', options)} AS runs
20072
- WHERE run_id = $1
20073
- LIMIT 1`,
20074
- [runId],
20075
- ),
20076
- );
20077
- const launch: RunLedgerDispatchLaunch = parseLaunchJson(
20078
- result.rows[0]?.launch_json,
20079
- );
20881
+ ): Promise<
20882
+ WorkerRuntimeApiContext & {
20883
+ refreshExecutorToken?: () => Promise<string>;
20884
+ invalidateExecutorToken?: () => void;
20885
+ }
20886
+ > {
20887
+ const launch = parseRunLedgerDispatchLaunch(launchJson);
20080
20888
  if (!launch?.baseUrl) {
20081
20889
  throw new Error(
20082
20890
  `Run-ledger outbox dispatcher cannot resolve runtime API context for ${runId}.`,
20083
20891
  );
20084
20892
  }
20085
20893
  let executorToken: string;
20894
+ let runLedgerIngressPrefixBatching: true | undefined;
20895
+ let refreshExecutorToken: (() => Promise<string>) | undefined;
20896
+ let invalidateExecutorToken: (() => void) | undefined;
20086
20897
  if (tokenAuthority === 'local_mint') {
20087
20898
  // App-side authoritative projector: sign in-process. NEVER reached on the
20088
20899
  // Fly worker (it passes `gateway`), so this is the only place `BETTER_AUTH_
@@ -20101,34 +20912,120 @@ async function loadPostgresSchedulerRuntimeApiForRun(
20101
20912
  `Run-ledger outbox gateway dispatch for ${runId} requires an active outbox lease.`,
20102
20913
  );
20103
20914
  }
20104
- executorToken = await requestRunLedgerDispatchToken({
20915
+ const dispatchRequest = {
20105
20916
  launch,
20106
20917
  eventId: dispatchLease.eventId,
20107
20918
  leaseOwner: dispatchLease.leaseOwner,
20108
20919
  schedulerSchema: options?.schema ?? null,
20109
20920
  signal,
20110
- });
20921
+ };
20922
+ const dispatchAuthority =
20923
+ await requestRunLedgerDispatchToken(dispatchRequest);
20924
+ invalidateExecutorToken = () =>
20925
+ runLedgerDispatchTokens.delete(dispatchAuthority.cacheKey);
20926
+ if (dispatchAuthority.reused) {
20927
+ refreshExecutorToken = async () =>
20928
+ (await requestRunLedgerDispatchToken(dispatchRequest, true))
20929
+ .executorToken;
20930
+ }
20931
+ executorToken = dispatchAuthority.executorToken;
20932
+ runLedgerIngressPrefixBatching =
20933
+ dispatchAuthority.runLedgerIngressPrefixBatching;
20111
20934
  }
20112
20935
  return {
20113
20936
  baseUrl: runtimeApiBaseUrlFromLaunch(launch),
20114
20937
  executorToken,
20938
+ ...(runLedgerIngressPrefixBatching
20939
+ ? { runLedgerIngressPrefixBatching }
20940
+ : {}),
20115
20941
  vercelProtectionBypassToken: launch.vercelProtectionBypassToken ?? null,
20942
+ ...(refreshExecutorToken ? { refreshExecutorToken } : {}),
20943
+ ...(invalidateExecutorToken ? { invalidateExecutorToken } : {}),
20116
20944
  };
20117
20945
  }
20118
20946
 
20119
- async function requestRunLedgerDispatchToken(input: {
20120
- launch: RunLedgerDispatchLaunch;
20121
- eventId: string;
20122
- leaseOwner: string;
20123
- schedulerSchema: string | null;
20124
- signal?: AbortSignal;
20125
- }): Promise<string> {
20947
+ async function loadPostgresSchedulerRuntimeApiForRun(
20948
+ withClient: <T>(
20949
+ operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
20950
+ ) => Promise<T>,
20951
+ runId: string,
20952
+ dispatchLease: { eventId: string; leaseOwner: string } | null,
20953
+ tokenAuthority: RunLedgerOutboxTokenAuthority,
20954
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
20955
+ signal?: AbortSignal,
20956
+ ): ReturnType<typeof resolvePostgresSchedulerRuntimeApiForRun> {
20957
+ // A continuation already owns its next event lease, so its launch context
20958
+ // is not part of the preclaim result. Load only the authority fields here.
20959
+ const result = await withClient((client) =>
20960
+ client.query<{ launch_json: unknown }>(
20961
+ `SELECT ${runLedgerDispatchLaunchSql('runs')} AS launch_json
20962
+ FROM ${tableName('runs', options)} AS runs
20963
+ WHERE run_id = $1
20964
+ LIMIT 1`,
20965
+ [runId],
20966
+ ),
20967
+ );
20968
+ return resolvePostgresSchedulerRuntimeApiForRun(
20969
+ result.rows[0]?.launch_json,
20970
+ runId,
20971
+ dispatchLease,
20972
+ tokenAuthority,
20973
+ options,
20974
+ signal,
20975
+ );
20976
+ }
20977
+
20978
+ // Only freshly gateway-issued legacy run.progress authority is reused.
20979
+ // Prefix advertisements are reacquired for each drain because they select its
20980
+ // receipt shape, including recovery across receiver capability changes. This is not
20981
+ // a lease cache: every delivery still claims, renews, and proves its own lease
20982
+ // at append. Include the complete pinned authority/routing and internal-token
20983
+ // fingerprint so a different run, schema, origin, or authority never shares it.
20984
+ const RUN_LEDGER_DISPATCH_TOKEN_CACHE_MS = 30_000;
20985
+ const RUN_LEDGER_DISPATCH_TOKEN_CACHE_ENTRIES = 256;
20986
+ const runLedgerDispatchTokens = new Map<
20987
+ string,
20988
+ { token: string; expiresAt: number; runLedgerIngressPrefixBatching?: true }
20989
+ >();
20990
+
20991
+ async function requestRunLedgerDispatchToken(
20992
+ input: {
20993
+ launch: RunLedgerDispatchLaunch;
20994
+ eventId: string;
20995
+ leaseOwner: string;
20996
+ schedulerSchema: string | null;
20997
+ signal?: AbortSignal;
20998
+ },
20999
+ forceFresh = false,
21000
+ ): Promise<{
21001
+ executorToken: string;
21002
+ reused: boolean;
21003
+ cacheKey: string;
21004
+ runLedgerIngressPrefixBatching?: true;
21005
+ }> {
20126
21006
  const internalToken = process.env.DEEPLINE_INTERNAL_TOKEN?.trim();
20127
21007
  if (!internalToken || !input.eventId || !input.leaseOwner) {
20128
21008
  throw new Error(
20129
21009
  'Run-ledger outbox recovery requires a valid dispatch lease and DEEPLINE_INTERNAL_TOKEN.',
20130
21010
  );
20131
21011
  }
21012
+ input.signal?.throwIfAborted();
21013
+ const { executorToken: _storedToken, ...authority } = input.launch;
21014
+ const cacheKey = createHash('sha256')
21015
+ .update(JSON.stringify([input.schedulerSchema, authority, internalToken]))
21016
+ .digest('hex');
21017
+ const now = Date.now();
21018
+ for (const [key, entry] of runLedgerDispatchTokens) {
21019
+ if (entry.expiresAt <= now) runLedgerDispatchTokens.delete(key);
21020
+ }
21021
+ const cached = runLedgerDispatchTokens.get(cacheKey);
21022
+ if (!forceFresh && cached)
21023
+ return {
21024
+ executorToken: cached.token,
21025
+ reused: true,
21026
+ cacheKey,
21027
+ };
21028
+ runLedgerDispatchTokens.delete(cacheKey);
20132
21029
  const url = `${runtimeApiBaseUrlFromLaunch(input.launch).replace(/\/$/, '')}/api/v2/internal/play-runtime/run-ledger-dispatch-token`;
20133
21030
  const vercelProtectionHeaders = await vercelProtectionBypassHeaders({
20134
21031
  baseUrl: runtimeApiBaseUrlFromLaunch(input.launch),
@@ -20164,6 +21061,7 @@ async function requestRunLedgerDispatchToken(input: {
20164
21061
  try {
20165
21062
  return JSON.parse(responseText) as {
20166
21063
  executorToken?: unknown;
21064
+ runLedgerIngressPrefixBatching?: unknown;
20167
21065
  error?: unknown;
20168
21066
  code?: unknown;
20169
21067
  retryable?: unknown;
@@ -20196,7 +21094,53 @@ async function requestRunLedgerDispatchToken(input: {
20196
21094
  : 'invalid response',
20197
21095
  });
20198
21096
  }
20199
- return body.executorToken;
21097
+ // Decode only the lifetime, never an authentication decision: the app
21098
+ // validates the signature, run scope, and live delivery lease on append.
21099
+ // Opaque/older token formats continue to work without this optimization.
21100
+ try {
21101
+ const claims = JSON.parse(
21102
+ Buffer.from(
21103
+ body.executorToken.split('.')[1] ?? '',
21104
+ 'base64url',
21105
+ ).toString(),
21106
+ ) as { exp?: unknown };
21107
+ if (
21108
+ body.runLedgerIngressPrefixBatching === undefined &&
21109
+ typeof claims.exp === 'number' &&
21110
+ Number.isFinite(claims.exp)
21111
+ ) {
21112
+ const expiresAt = Math.min(
21113
+ Date.now() + RUN_LEDGER_DISPATCH_TOKEN_CACHE_MS,
21114
+ claims.exp * 1_000 - 5_000,
21115
+ );
21116
+ if (expiresAt > Date.now()) {
21117
+ runLedgerDispatchTokens.set(cacheKey, {
21118
+ token: body.executorToken,
21119
+ expiresAt,
21120
+ });
21121
+ while (
21122
+ runLedgerDispatchTokens.size > RUN_LEDGER_DISPATCH_TOKEN_CACHE_ENTRIES
21123
+ ) {
21124
+ runLedgerDispatchTokens.delete(
21125
+ runLedgerDispatchTokens.keys().next().value!,
21126
+ );
21127
+ }
21128
+ }
21129
+ }
21130
+ } catch {
21131
+ // A cache miss never changes delivery or hides a gateway rejection.
21132
+ }
21133
+ if (body.runLedgerIngressPrefixBatching !== undefined) {
21134
+ runLedgerDispatchTokens.delete(cacheKey);
21135
+ }
21136
+ return {
21137
+ executorToken: body.executorToken,
21138
+ reused: false,
21139
+ cacheKey,
21140
+ ...(body.runLedgerIngressPrefixBatching === true
21141
+ ? { runLedgerIngressPrefixBatching: true as const }
21142
+ : {}),
21143
+ };
20200
21144
  }
20201
21145
 
20202
21146
  export async function requestComputeBillingDispatchToken(input: {
@@ -20272,27 +21216,7 @@ export async function requestComputeBillingDispatchToken(input: {
20272
21216
  return body.executorToken;
20273
21217
  }
20274
21218
 
20275
- /** Read only the immutable authority/routing fields under the same lease fence. */
20276
- export async function readRunLedgerDispatchAuthorityForLease(
20277
- client: PostgresSchedulerQueryClient,
20278
- input: { runId: string; eventId: string; leaseOwner: string },
20279
- options?: Pick<PostgresSchedulerOptions, 'schema'>,
20280
- ): Promise<RunLedgerDispatchLaunch | null> {
20281
- const result = await client.query<{ launch_json: unknown }>(
20282
- `SELECT ${runLedgerDispatchLaunchSql('runs')} AS launch_json
20283
- FROM ${tableName('outbox', options)} outbox
20284
- JOIN ${tableName('runs', options)} runs ON runs.run_id = outbox.run_id
20285
- WHERE outbox.event_id = $1
20286
- AND outbox.run_id = $2
20287
- AND outbox.lease_owner = $3
20288
- AND outbox.lease_expires_at > now()
20289
- AND outbox.delivered_at IS NULL
20290
- AND outbox.failed_at IS NULL
20291
- LIMIT 1`,
20292
- [input.eventId, input.runId, input.leaseOwner],
20293
- );
20294
- return result.rows[0] ? parseLaunchJson(result.rows[0].launch_json) : null;
20295
- }
21219
+ export { readRunLedgerDispatchAuthorityForLease } from './run-ledger-dispatch-authority';
20296
21220
 
20297
21221
  export async function readRunLedgerDispatchLaunchForLease(
20298
21222
  client: PostgresSchedulerQueryClient,
@@ -20379,6 +21303,20 @@ function terminalLogContextsPayloadField(
20379
21303
  return value.map(normalizePlayLogContext);
20380
21304
  }
20381
21305
 
21306
+ function terminalLogMetadataPayloadField(
21307
+ payload: unknown,
21308
+ lines: readonly string[],
21309
+ ): Array<PlayLogMetadata | null> | null {
21310
+ const value = payloadField(payload, 'terminalLogMetadata');
21311
+ if (value === undefined) return null;
21312
+ if (!Array.isArray(value) || value.length !== lines.length) {
21313
+ throw new TypeError(
21314
+ 'Terminal log metadata must align with terminal log lines.',
21315
+ );
21316
+ }
21317
+ return value.map(normalizePlayLogMetadata);
21318
+ }
21319
+
20382
21320
  function ledgerEventsPayloadField(
20383
21321
  payload: unknown,
20384
21322
  key: string,
@@ -20415,22 +21353,10 @@ function sourcePayloadField(
20415
21353
  }
20416
21354
 
20417
21355
  function parseLaunchJson(value: unknown): PlaySchedulerSubmitInput {
20418
- if (!value || typeof value !== 'object' || Array.isArray(value)) {
20419
- throw new Error('Postgres scheduler launch_json is not an object.');
20420
- }
20421
- const launch = value as Partial<PlaySchedulerSubmitInput>;
20422
- if (
20423
- typeof launch.runId !== 'string' ||
20424
- typeof launch.playId !== 'string' ||
20425
- typeof launch.playName !== 'string' ||
20426
- typeof launch.executorToken !== 'string' ||
20427
- typeof launch.baseUrl !== 'string'
20428
- ) {
20429
- throw new Error(
20430
- 'Postgres scheduler launch_json is missing required fields.',
20431
- );
20432
- }
20433
- return hydrateRetainedPlayPipelines(launch as PlaySchedulerSubmitInput);
21356
+ const launch = parseRunLedgerDispatchLaunch(
21357
+ value,
21358
+ ) as PlaySchedulerSubmitInput;
21359
+ return hydrateRetainedPlayPipelines(launch);
20434
21360
  }
20435
21361
 
20436
21362
  function parsePostgresSchedulerCronSourcePin(