deepline 0.3.288 → 0.3.290

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  2. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +30 -10
  3. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +108 -12
  4. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +1 -1
  5. package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
  6. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +3 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/launch-identity.ts +1 -0
  8. package/dist/bundling-sources/shared_libs/play-runtime/log-provenance.ts +17 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/worker-terminal-projection.ts +6 -5
  10. package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +2 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +7 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/run-event-catalog.ts +2 -0
  13. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +31 -3
  14. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +23 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +30 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +38 -3
  17. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +2 -3
  18. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +22 -1
  19. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +1 -1
  20. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool-options.ts +145 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +1 -5
  23. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -139
  24. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +19 -5
  25. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +37 -15
  26. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +7 -5
  27. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +961 -281
  28. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +55 -25
  29. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-dispatch-authority.ts +97 -0
  30. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +33 -14
  31. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +452 -43
  32. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +10 -1
  33. package/dist/bundling-sources/shared_libs/play-runtime/test-admission-batching.ts +103 -0
  34. package/dist/bundling-sources/shared_libs/play-runtime/zerobounce-batching.ts +75 -28
  35. package/dist/cli/index.js +24 -11
  36. package/dist/cli/index.mjs +24 -11
  37. package/dist/index.js +3 -1
  38. package/dist/index.mjs +3 -1
  39. package/dist/install-integrity.json +3 -0
  40. package/dist/release.d.mts +1 -1
  41. package/dist/release.d.ts +1 -1
  42. package/dist/release.js +1 -1
  43. package/dist/release.mjs +1 -1
  44. package/package.json +1 -1
@@ -1,3 +1,9 @@
1
+ import {
2
+ parseRunLedgerDispatchLaunch,
3
+ runLedgerDispatchLaunchSql,
4
+ type RunLedgerDispatchLaunch,
5
+ } from './run-ledger-dispatch-authority';
6
+ import { normalizeRunLogIds } from '../run-log-buffer';
1
7
  import {
2
8
  retainedPlayPipelineFields,
3
9
  hydrateRetainedPlayPipelines,
@@ -28,6 +34,8 @@ import type {
28
34
  import { PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA } from '../runtime-scheduler-topology';
29
35
  import {
30
36
  RUNTIME_EVENT_CANONICAL_RETENTION_INDEX_READY_MARKER,
37
+ RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD,
38
+ RUNTIME_EVENT_SEARCH_ONLY_FIELD,
31
39
  RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER,
32
40
  } from './runtime-event-search';
33
41
  import type { PlayRunnerRuntimeResource } from '../runner-backends/types';
@@ -105,7 +113,6 @@ import {
105
113
  RUN_SCHEDULER_OUTBOX_START_EVENT_TYPES,
106
114
  RUN_SCHEDULER_OUTBOX_EVENT_TYPE,
107
115
  RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL,
108
- RUN_SCHEDULER_OUTBOX_PREFIX_STOP_EVENT_TYPES,
109
116
  RUN_SCHEDULER_OUTBOX_TERMINAL_EVENT_TYPES,
110
117
  } from '../run-event-catalog';
111
118
  import { stableStringify } from '../../plays/row-identity';
@@ -336,7 +343,10 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
336
343
  // recovery, which scans only active runs for each release lane.
337
344
  // v57 refreshes log-position lookups after the operator concurrently prepares
338
345
  // channel-specific indexes; outgoing readers retain the existing GIN index.
339
- export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 57;
346
+ // v58 installs deferred outbox readers while retaining synchronous writers
347
+ // until the separately operated, reader-first cutover.
348
+ // v59 adds origin identity bulk projection and independent durable search progress.
349
+ export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 59;
340
350
  const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
341
351
  'scheduler_schema_migrations';
342
352
  export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
@@ -1928,6 +1938,24 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
1928
1938
  AND index_row.indrelid = to_regclass($3)
1929
1939
  AND index_row.indisready AND index_row.indisvalid
1930
1940
  )
1941
+ AND EXISTS (
1942
+ SELECT 1 FROM pg_index AS index_row
1943
+ JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
1944
+ JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
1945
+ WHERE index_namespace.nspname = $1
1946
+ AND index_class.relname = 'runtime_scheduler_outbox_lifecycle_pending_order_idx'
1947
+ AND index_row.indrelid = to_regclass($3)
1948
+ AND index_row.indisready AND index_row.indisvalid
1949
+ )
1950
+ AND EXISTS (
1951
+ SELECT 1 FROM pg_index AS index_row
1952
+ JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
1953
+ JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
1954
+ WHERE index_namespace.nspname = $1
1955
+ AND index_class.relname = 'runtime_scheduler_outbox_search_pending_order_idx'
1956
+ AND index_row.indrelid = to_regclass($3)
1957
+ AND index_row.indisready AND index_row.indisvalid
1958
+ )
1931
1959
  AND EXISTS (
1932
1960
  SELECT 1 FROM pg_index AS index_row
1933
1961
  JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
@@ -2461,6 +2489,7 @@ export async function ensurePostgresSchedulerLaunchAdmissionFunctions(
2461
2489
  jsonb_set(
2462
2490
  launch_json - ARRAY[
2463
2491
  'launchIdentityDigest',
2492
+ 'runtimeLogIdentity',
2464
2493
  'executorToken',
2465
2494
  'inputFileId',
2466
2495
  'preloadedDbSessionUnwrapKey',
@@ -3944,6 +3973,26 @@ export async function ensurePostgresSchedulerSchema(
3944
3973
  )
3945
3974
  WHERE delivered_at IS NULL
3946
3975
  `);
3976
+ await client.query(`
3977
+ CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_lifecycle_pending_order_idx
3978
+ ON ${tableName('outbox', options)} (
3979
+ run_id,
3980
+ (CASE event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END),
3981
+ created_at,
3982
+ event_id
3983
+ )
3984
+ WHERE delivered_at IS NULL AND ${searchLaneSql('outbox')} = false
3985
+ `);
3986
+ await client.query(`
3987
+ CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_search_pending_order_idx
3988
+ ON ${tableName('outbox', options)} (
3989
+ run_id,
3990
+ (CASE event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END),
3991
+ created_at,
3992
+ event_id
3993
+ )
3994
+ WHERE delivered_at IS NULL AND ${searchLaneSql('outbox')} = true
3995
+ `);
3947
3996
  await client.query(`
3948
3997
  CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_lease_health_idx
3949
3998
  ON ${tableName('outbox', options)}
@@ -4094,6 +4143,22 @@ const POSTGRES_SCHEDULER_RUNTIME_ADMIN_INDEXES = [
4094
4143
  created_at, event_id)
4095
4144
  WHERE delivered_at IS NULL`,
4096
4145
  },
4146
+ {
4147
+ name: 'runtime_scheduler_outbox_lifecycle_pending_order_idx',
4148
+ table: 'outbox',
4149
+ definition: `(run_id,
4150
+ (CASE event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END),
4151
+ created_at, event_id)
4152
+ WHERE delivered_at IS NULL AND ${searchLaneSql('outbox')} = false`,
4153
+ },
4154
+ {
4155
+ name: 'runtime_scheduler_outbox_search_pending_order_idx',
4156
+ table: 'outbox',
4157
+ definition: `(run_id,
4158
+ (CASE event_type WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0 WHEN ${RUN_EVENT_KIND_SQL_LITERAL.runStarted} THEN 1 ELSE 2 END),
4159
+ created_at, event_id)
4160
+ WHERE delivered_at IS NULL AND ${searchLaneSql('outbox')} = true`,
4161
+ },
4097
4162
  {
4098
4163
  name: 'runtime_scheduler_outbox_lease_health_idx',
4099
4164
  table: 'outbox',
@@ -10318,8 +10383,8 @@ export async function recordPostgresSchedulerRunnerTerminal(
10318
10383
  const recorded = await client.query<{ run_id: string }>(
10319
10384
  `
10320
10385
  WITH accepted AS MATERIALIZED (
10321
- -- Match direct completion's Run -> attempt lock order. The log
10322
- -- acceptance trigger also locks the Run for sequence assignment.
10386
+ -- Record the authoritative terminal boundary before accepting the
10387
+ -- attempt payload; deferred log indexing owns no Run row lock.
10323
10388
  SELECT clock_timestamp() AS accepted_at
10324
10389
  FROM ${tableName('runs', options)}
10325
10390
  WHERE run_id = $1
@@ -10758,6 +10823,7 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
10758
10823
  'producerAttempt', $2::integer,
10759
10824
  'terminalLogLines', $7::jsonb,
10760
10825
  'terminalLogContexts', $17::jsonb,
10826
+ 'terminalLogIds', $19::jsonb->'terminalLogIds',
10761
10827
  'terminalProgressEvents', coalesce((
10762
10828
  SELECT accepted.runner_log_payload_json->'terminalProgressEvents'
10763
10829
  FROM completed_attempt AS accepted
@@ -13134,6 +13200,7 @@ export async function completePostgresSchedulerAttempt(
13134
13200
  runtimeTiming?: PlayRunnerRuntimeTiming | null;
13135
13201
  terminalLogLines?: readonly string[] | null;
13136
13202
  terminalLogContexts?: readonly (PlayLogContext | null)[] | null;
13203
+ terminalLogIds?: readonly (string | null)[] | null;
13137
13204
  terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
13138
13205
  },
13139
13206
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
@@ -13151,10 +13218,15 @@ export async function completePostgresSchedulerAttempt(
13151
13218
  'Terminal log contexts must align with terminal log lines.',
13152
13219
  );
13153
13220
  }
13221
+ const terminalLogIds = normalizeRunLogIds(
13222
+ input.terminalLogIds,
13223
+ input.terminalLogLines ?? [],
13224
+ );
13154
13225
  const terminalLogEntries = (input.terminalLogLines ?? [])
13155
13226
  .map((line, index) => ({
13156
13227
  line: line.trim(),
13157
13228
  context: input.terminalLogContexts?.[index] ?? null,
13229
+ entryId: terminalLogIds?.[index] ?? null,
13158
13230
  }))
13159
13231
  .filter((entry) => entry.line.length > 0)
13160
13232
  .slice(-LOG_TAIL_LIMIT);
@@ -13340,6 +13412,7 @@ export async function completePostgresSchedulerAttempt(
13340
13412
  'producerAttempt', $2::integer,
13341
13413
  'terminalLogLines', $8::jsonb,
13342
13414
  'terminalLogContexts', $11::jsonb,
13415
+ 'terminalLogIds', $13::jsonb,
13343
13416
  'resultSummary', $10::jsonb,
13344
13417
  'terminalWorkCounterSegment', $12::jsonb
13345
13418
  ),
@@ -13365,6 +13438,11 @@ export async function completePostgresSchedulerAttempt(
13365
13438
  stringifyPostgresJson(resultSummary),
13366
13439
  JSON.stringify(terminalLogContexts),
13367
13440
  stringifyPostgresJson(input.terminalWorkCounterSegment ?? null),
13441
+ stringifyPostgresJson(
13442
+ terminalLogIds
13443
+ ? terminalLogEntries.map((entry) => entry.entryId)
13444
+ : null,
13445
+ ),
13368
13446
  ],
13369
13447
  );
13370
13448
  if (completed.rows.length === 0) {
@@ -13383,6 +13461,7 @@ export async function failPostgresSchedulerAttempt(
13383
13461
  result?: unknown;
13384
13462
  terminalLogLines?: readonly string[] | null;
13385
13463
  terminalLogContexts?: readonly (PlayLogContext | null)[] | null;
13464
+ terminalLogIds?: readonly (string | null)[] | null;
13386
13465
  /** Refuse to overwrite an authoritative detached runner terminal that won
13387
13466
  * after claim. Resumable `suspended` receipts may be superseded by the
13388
13467
  * hard ceiling in the same fenced statement. */
@@ -13429,10 +13508,15 @@ export async function failPostgresSchedulerAttempt(
13429
13508
  'Terminal log contexts must align with terminal log lines.',
13430
13509
  );
13431
13510
  }
13511
+ const terminalLogIds = normalizeRunLogIds(
13512
+ input.terminalLogIds,
13513
+ input.terminalLogLines ?? [],
13514
+ );
13432
13515
  const terminalLogEntries = (input.terminalLogLines ?? [])
13433
13516
  .map((line, index) => ({
13434
13517
  line: line.trim(),
13435
13518
  context: input.terminalLogContexts?.[index] ?? null,
13519
+ entryId: terminalLogIds?.[index] ?? null,
13436
13520
  }))
13437
13521
  .filter((entry) => entry.line.length > 0)
13438
13522
  .slice(-LOG_TAIL_LIMIT);
@@ -13678,6 +13762,7 @@ export async function failPostgresSchedulerAttempt(
13678
13762
  'producerAttempt', $2::integer,
13679
13763
  'terminalLogLines', $12::jsonb,
13680
13764
  'terminalLogContexts', $14::jsonb,
13765
+ 'terminalLogIds', $17::jsonb,
13681
13766
  'resultSummary', $13::jsonb,
13682
13767
  'terminalWorkCounterSegment', $15::jsonb
13683
13768
  ) || CASE
@@ -13711,6 +13796,11 @@ export async function failPostgresSchedulerAttempt(
13711
13796
  JSON.stringify(terminalLogContexts),
13712
13797
  stringifyPostgresJson(terminalWorkCounterSegment ?? null),
13713
13798
  stringifyPostgresJson(input.terminalProgressEvents ?? []),
13799
+ stringifyPostgresJson(
13800
+ terminalLogIds
13801
+ ? terminalLogEntries.map((entry) => entry.entryId)
13802
+ : null,
13803
+ ),
13714
13804
  ],
13715
13805
  );
13716
13806
  return failed.rows.length > 0;
@@ -14413,6 +14503,7 @@ function runnerTerminalLogPayload(input: {
14413
14503
  producerAttempt: input.attempt,
14414
14504
  terminalLogLines: projection.lines,
14415
14505
  terminalLogContexts: projection.lineContexts,
14506
+ ...(projection.lineIds ? { terminalLogIds: projection.lineIds } : {}),
14416
14507
  terminalProgressEvents: input.terminalProgressEvents ?? [],
14417
14508
  // Older runner envelopes conveyed these diagnostics only in tagged log
14418
14509
  // text. Preserve their small structured facts before removing that text.
@@ -14433,7 +14524,11 @@ function runnerTerminalLogPayload(input: {
14433
14524
  function terminalLogProjectionForOutbox(
14434
14525
  runnerResult: unknown,
14435
14526
  mode: PostgresSchedulerTerminalLogOutbox | undefined,
14436
- ): { lines: string[]; lineContexts: Array<PlayLogContext | null> } {
14527
+ ): {
14528
+ lines: string[];
14529
+ lineContexts: Array<PlayLogContext | null>;
14530
+ lineIds?: Array<string | null>;
14531
+ } {
14437
14532
  if (
14438
14533
  mode !== 'ctx_log_tail' ||
14439
14534
  !runnerResult ||
@@ -14444,6 +14539,7 @@ function terminalLogProjectionForOutbox(
14444
14539
  const result = runnerResult as {
14445
14540
  logs?: unknown;
14446
14541
  logContexts?: unknown;
14542
+ logIds?: unknown;
14447
14543
  };
14448
14544
  const logs = result.logs;
14449
14545
  if (!Array.isArray(logs)) {
@@ -14459,6 +14555,7 @@ function terminalLogProjectionForOutbox(
14459
14555
  const logContexts = Array.isArray(result.logContexts)
14460
14556
  ? result.logContexts
14461
14557
  : undefined;
14558
+ const logIds = normalizeRunLogIds(result.logIds, logs);
14462
14559
  const entries = logs
14463
14560
  .map((line, index) => {
14464
14561
  if (typeof line !== 'string') return null;
@@ -14475,16 +14572,23 @@ function terminalLogProjectionForOutbox(
14475
14572
  return {
14476
14573
  line: line.trim(),
14477
14574
  context: normalizePlayLogContext(logContexts?.[index]),
14575
+ entryId: logIds?.[index] ?? null,
14478
14576
  };
14479
14577
  })
14480
14578
  .filter(
14481
- (entry): entry is { line: string; context: PlayLogContext | null } =>
14482
- entry !== null,
14579
+ (
14580
+ entry,
14581
+ ): entry is {
14582
+ line: string;
14583
+ context: PlayLogContext | null;
14584
+ entryId: string | null;
14585
+ } => entry !== null,
14483
14586
  )
14484
14587
  .slice(-LOG_TAIL_LIMIT);
14485
14588
  return {
14486
14589
  lines: entries.map((entry) => entry.line),
14487
14590
  lineContexts: entries.map((entry) => entry.context),
14591
+ ...(logIds ? { lineIds: entries.map((entry) => entry.entryId) } : {}),
14488
14592
  };
14489
14593
  }
14490
14594
 
@@ -14733,6 +14837,7 @@ export async function executePostgresSchedulerClaim(
14733
14837
  runtimeTiming: extractRunnerRuntimeTiming(output),
14734
14838
  terminalLogLines: terminalLogProjection.lines,
14735
14839
  terminalLogContexts: terminalLogProjection.lineContexts,
14840
+ terminalLogIds: terminalLogProjection.lineIds,
14736
14841
  terminalWorkCounterSegment: terminalWorkCounterSegmentFromRunnerResult(
14737
14842
  output,
14738
14843
  claim.attempt,
@@ -14794,6 +14899,7 @@ export async function executePostgresSchedulerClaim(
14794
14899
  result: terminalFailureResult,
14795
14900
  terminalLogLines: terminalLogProjection.lines,
14796
14901
  terminalLogContexts: terminalLogProjection.lineContexts,
14902
+ terminalLogIds: terminalLogProjection.lineIds,
14797
14903
  sandboxCrashDiagnostic: options.sandboxCrashDiagnostic,
14798
14904
  ...(options.preserveRunnerTerminal
14799
14905
  ? { preserveRunnerTerminal: true }
@@ -14987,6 +15093,9 @@ export function buildPostgresSchedulerRunnerContext(
14987
15093
  const rootRunId = launch.rootRunId?.trim() || launch.runId;
14988
15094
  return {
14989
15095
  executorToken: launch.executorToken,
15096
+ ...(launch.runtimeLogIdentity === 'origin'
15097
+ ? { runtimeLogIdentity: 'origin' as const }
15098
+ : {}),
14990
15099
  baseUrl: runtimeApiBaseUrlFromLaunch(launch),
14991
15100
  executionGatewayBaseUrl,
14992
15101
  receiptGatewayBaseUrl,
@@ -17388,6 +17497,12 @@ async function appendRunLedgerOutboxFact(
17388
17497
  WITH admitted AS MATERIALIZED (
17389
17498
  SELECT run_id FROM ${tableName('runs', options)}
17390
17499
  WHERE run_id = $1 AND org_id = $2
17500
+ -- Evaluate ingress time only after the projector/lifecycle fence. An
17501
+ -- insert waiting behind a frozen receipt must become its successor,
17502
+ -- never a late predecessor that changes the attempted receipt.
17503
+ -- NO KEY UPDATE still permits legacy inserts' foreign-key KEY SHARE
17504
+ -- while they finish under the event-search advisory mutex.
17505
+ FOR NO KEY UPDATE
17391
17506
  ), recorded AS (
17392
17507
  INSERT INTO ${tableName('outbox', options)}
17393
17508
  (event_id, run_id, event_type, payload_json, created_at)
@@ -17413,23 +17528,43 @@ export { RunLedgerProjectionLeaseLostError } from '../run-ledger-projection-cont
17413
17528
  /** Extend only our still-live lease before another bounded transport piece. */
17414
17529
  export async function renewPostgresRunLedgerDelivery(
17415
17530
  client: PostgresSchedulerQueryClient,
17416
- input: { eventIds: string[]; leaseOwner: string; leaseSeconds: number },
17531
+ input: {
17532
+ eventIds: string[];
17533
+ leaseOwner: string;
17534
+ leaseSeconds: number;
17535
+ /** Freeze an ingress receipt before any HTTP attempt, including singletons. */
17536
+ freezeReceipt?: boolean;
17537
+ },
17417
17538
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
17418
17539
  ): Promise<void> {
17419
- const result = await client.query<{ event_id: string }>(
17420
- `
17540
+ const renew = (freezeReceipt: boolean) =>
17541
+ client.query<{ event_id: string }>(
17542
+ `
17421
17543
  UPDATE ${tableName('outbox', options)}
17422
17544
  SET lease_expires_at = now() + make_interval(secs => $3::integer)
17545
+ ${freezeReceipt ? ', delivery_batch_started_at = coalesce(delivery_batch_started_at, statement_timestamp())' : ''}
17423
17546
  WHERE event_id = ANY($1::text[]) AND lease_owner = $2
17424
17547
  AND delivered_at IS NULL AND failed_at IS NULL AND lease_expires_at > now()
17425
17548
  RETURNING event_id
17426
17549
  `,
17427
- [
17428
- input.eventIds,
17429
- input.leaseOwner,
17430
- Math.max(5, Math.min(300, input.leaseSeconds)),
17431
- ],
17432
- );
17550
+ [
17551
+ input.eventIds,
17552
+ input.leaseOwner,
17553
+ Math.max(5, Math.min(300, input.leaseSeconds)),
17554
+ ],
17555
+ );
17556
+ let result;
17557
+ try {
17558
+ result = await renew(
17559
+ input.freezeReceipt === true &&
17560
+ supportsOutboxDeliveryBatchMarker(options),
17561
+ );
17562
+ } catch (error) {
17563
+ if (!input.freezeReceipt || !isMissingOutboxDeliveryBatchMarker(error))
17564
+ throw error;
17565
+ markOutboxDeliveryBatchMarkerUnavailable(options);
17566
+ result = await renew(false);
17567
+ }
17433
17568
  if (result.rows.length !== input.eventIds.length)
17434
17569
  throw new RunLedgerProjectionLeaseLostError('delivery_lease_not_current');
17435
17570
  }
@@ -17457,6 +17592,7 @@ export async function assertPostgresRunLedgerDelivery(
17457
17592
  const result = await client.query<{
17458
17593
  delivery_key: string;
17459
17594
  event_type: string;
17595
+ delivery_batch_started_at: string | null;
17460
17596
  lease_owner: string | null;
17461
17597
  lease_live: boolean;
17462
17598
  due: boolean;
@@ -17467,6 +17603,8 @@ export async function assertPostgresRunLedgerDelivery(
17467
17603
  `
17468
17604
  SELECT outbox.delivery_key,
17469
17605
  outbox.event_type,
17606
+ -- Row JSON keeps this optional on historical scheduler schemas.
17607
+ to_jsonb(outbox)->>'delivery_batch_started_at' AS delivery_batch_started_at,
17470
17608
  outbox.lease_owner,
17471
17609
  outbox.lease_expires_at > now() AS lease_live,
17472
17610
  coalesce(outbox.next_attempt_at, outbox.created_at) <= now() AS due,
@@ -17476,6 +17614,7 @@ export async function assertPostgresRunLedgerDelivery(
17476
17614
  FROM ${tableName('outbox', options)} AS outbox
17477
17615
  JOIN ${tableName('runs', options)} AS runs ON runs.run_id = outbox.run_id
17478
17616
  WHERE outbox.run_id = $1 AND runs.org_id = $2
17617
+ AND NOT ${searchLaneSql('outbox')}
17479
17618
  AND outbox.delivered_at IS NULL
17480
17619
  ORDER BY ${runLedgerOrderSql('outbox')}
17481
17620
  -- Later producer facts cannot belong to this receipt. Do not hydrate their
@@ -17497,9 +17636,12 @@ export async function assertPostgresRunLedgerDelivery(
17497
17636
  // Cancellation may own several pending facts at once. Validate the exact
17498
17637
  // contiguous, due prefix named by the receipt instead of all rows sharing
17499
17638
  // that cancellation lease.
17639
+ const ingressReceiptStarted =
17640
+ rows[0]?.event_type === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents &&
17641
+ rows[0]?.delivery_batch_started_at != null;
17500
17642
  const prefix: typeof rows = [];
17501
17643
  let prefixStopReason: RunLedgerProjectionLeaseLostReason | null = null;
17502
- for (const row of rows) {
17644
+ for (const [index, row] of rows.entries()) {
17503
17645
  if (row.lease_owner !== leaseOwner) {
17504
17646
  prefixStopReason = 'prefix_lease_owner_mismatch';
17505
17647
  break;
@@ -17516,11 +17658,6 @@ export async function assertPostgresRunLedgerDelivery(
17516
17658
  prefixStopReason = 'predecessor_failed';
17517
17659
  break;
17518
17660
  }
17519
- if (
17520
- prefix.length > 0 &&
17521
- row.event_type === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents
17522
- )
17523
- break;
17524
17661
  prefix.push(row);
17525
17662
 
17526
17663
  const first = prefix[0]!;
@@ -17567,10 +17704,17 @@ export async function assertPostgresRunLedgerDelivery(
17567
17704
  runId: input.runId,
17568
17705
  deliveryKey: first.delivery_key,
17569
17706
  });
17707
+ // A receipt already attempted as a compound prefix cannot be split by
17708
+ // an older competing projector after lease expiry. Reject with the normal
17709
+ // retryable lease-lost response until a capable writer reclaims it whole.
17710
+ const atStartedPrefixEnd =
17711
+ !ingressReceiptStarted ||
17712
+ rows[index + 1]?.delivery_batch_started_at == null;
17570
17713
  if (
17571
- input.idempotencyKey === expectedKey ||
17572
- input.idempotencyKey === batchKey ||
17573
- legacySingletonKey
17714
+ atStartedPrefixEnd &&
17715
+ (input.idempotencyKey === expectedKey ||
17716
+ input.idempotencyKey === batchKey ||
17717
+ legacySingletonKey)
17574
17718
  ) {
17575
17719
  if (input.projectedEvents === undefined) return;
17576
17720
  const prefixEvents: PlayRunLedgerIngressEvent[] = [];
@@ -17612,11 +17756,18 @@ export async function assertPostgresRunLedgerDelivery(
17612
17756
 
17613
17757
  if (
17614
17758
  prefix.length === 1 &&
17615
- RUN_SCHEDULER_OUTBOX_PREFIX_STOP_EVENT_TYPES.includes(first.event_type)
17759
+ RUN_SCHEDULER_OUTBOX_START_EVENT_TYPES.includes(first.event_type)
17616
17760
  ) {
17617
17761
  break;
17618
17762
  }
17619
- if (terminal || prefix.length >= RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS) {
17763
+ // A frozen ingress receipt can include late runner logs after terminal.
17764
+ // Prove its complete immutable boundary, including transport chunks after
17765
+ // terminal, rather than splitting a receipt that has already been attempted.
17766
+ if (
17767
+ (terminal && !ingressReceiptStarted) ||
17768
+ (ingressReceiptStarted && atStartedPrefixEnd) ||
17769
+ prefix.length >= RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS
17770
+ ) {
17620
17771
  break;
17621
17772
  }
17622
17773
  }
@@ -17631,7 +17782,11 @@ function runLedgerPrecedesSql(
17631
17782
  successor: string,
17632
17783
  inclusive = false,
17633
17784
  ): string {
17634
- return `${runLedgerPositionSql(predecessor)} ${inclusive ? '<=' : '<'} ${runLedgerPositionSql(successor)}`;
17785
+ return `(${searchLaneSql(predecessor)} = ${searchLaneSql(successor)} AND ${runLedgerPositionSql(predecessor)} ${inclusive ? '<=' : '<'} ${runLedgerPositionSql(successor)})`;
17786
+ }
17787
+
17788
+ function searchLaneSql(alias: string): string {
17789
+ return `coalesce(${alias}.payload_json->'${RUNTIME_EVENT_SEARCH_ONLY_FIELD}' = 'true'::jsonb, false)`;
17635
17790
  }
17636
17791
 
17637
17792
  function runLedgerPositionSql(alias: string): string {
@@ -17658,6 +17813,32 @@ function runLedgerTerminalPendingSql(
17658
17813
  )`;
17659
17814
  }
17660
17815
 
17816
+ function runLedgerWaitingRunsSql(
17817
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
17818
+ runIdParameter?: string,
17819
+ ): string {
17820
+ return `SELECT DISTINCT waiting_event.run_id
17821
+ FROM ${tableName('outbox', options)} AS waiting_event
17822
+ WHERE waiting_event.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runWaiting}
17823
+ -- Detached runner parks are transport-only and project no public wait.
17824
+ AND waiting_event.payload_json->>'kind' IS DISTINCT FROM 'detached_runner'
17825
+ AND waiting_event.delivered_at IS NULL
17826
+ AND waiting_event.failed_at IS NULL
17827
+ AND coalesce(waiting_event.next_attempt_at, waiting_event.created_at) <= now()
17828
+ ${runIdParameter ? `AND (${runIdParameter}::text IS NULL OR waiting_event.run_id = ${runIdParameter}::text)` : ''}`;
17829
+ }
17830
+
17831
+ function runLedgerWaitingPendingSql(alias: string): string {
17832
+ return `(${alias}.run_id IN (SELECT run_id FROM waiting_runs))`;
17833
+ }
17834
+
17835
+ function runLedgerUrgentLifecyclePendingSql(
17836
+ alias: string,
17837
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
17838
+ ): string {
17839
+ return `(${runLedgerTerminalPendingSql(alias, options)} OR ${runLedgerWaitingPendingSql(alias)})`;
17840
+ }
17841
+
17661
17842
  /**
17662
17843
  * Claim a causal prefix, preserving the v33 adapter while the additive batch
17663
17844
  * marker is absent. An enclosing transaction uses a savepoint so the missing
@@ -17670,6 +17851,12 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17670
17851
  runId?: string | null;
17671
17852
  leaseOwner?: string | null;
17672
17853
  leaseSeconds?: number;
17854
+ /** Independent raw search work uses the same outbox leases and controls. */
17855
+ searchOnly?: boolean;
17856
+ /** Advertised by the launch-pinned receipt verifier, never inferred locally. */
17857
+ allowIngressPrefix?: boolean;
17858
+ /** Extend this still-owned head atomically under the run cancellation fence. */
17859
+ extendOwnedHead?: string;
17673
17860
  /** The dispatcher owns a short, exclusive claim transaction. */
17674
17861
  inTransaction?: boolean;
17675
17862
  } = {},
@@ -17682,8 +17869,16 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17682
17869
  const runId = input.runId?.trim() || null;
17683
17870
  const leaseOwner = input.leaseOwner?.trim() || null;
17684
17871
  const leaseSeconds = Math.max(5, Math.min(300, input.leaseSeconds ?? 30));
17685
- const claim = async (supportsTerminalBatchMarker: boolean) =>
17686
- await client.query<{
17872
+ const claim = async (supportsTerminalBatchMarker: boolean) => {
17873
+ const urgentLifecyclePendingSql = (alias: string) =>
17874
+ input.searchOnly === true
17875
+ ? runLedgerTerminalPendingSql(alias, options)
17876
+ : runLedgerUrgentLifecyclePendingSql(alias, options);
17877
+ const ingressPrefixSql =
17878
+ supportsTerminalBatchMarker && input.allowIngressPrefix === true
17879
+ ? `(head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} AND ${urgentLifecyclePendingSql('head')})`
17880
+ : 'false';
17881
+ return await client.query<{
17687
17882
  event_id: string;
17688
17883
  run_id: string;
17689
17884
  event_type: string;
@@ -17692,10 +17887,33 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17692
17887
  delivery_key: string | null;
17693
17888
  }>(
17694
17889
  `
17695
- WITH head AS MATERIALIZED (
17696
- SELECT candidate.event_id, candidate.run_id, candidate.event_type
17890
+ WITH ${
17891
+ input.searchOnly === true
17892
+ ? ''
17893
+ : `waiting_runs AS MATERIALIZED (
17894
+ ${runLedgerWaitingRunsSql(options, '$2')}
17895
+ ), `
17896
+ }${
17897
+ input.extendOwnedHead
17898
+ ? `owned_run AS MATERIALIZED (
17899
+ SELECT run_id FROM ${tableName('runs', options)}
17900
+ WHERE run_id = $2
17901
+ FOR UPDATE SKIP LOCKED
17902
+ ),`
17903
+ : ''
17904
+ } head AS MATERIALIZED (
17905
+ SELECT candidate.event_id, candidate.run_id, candidate.event_type,
17906
+ ${supportsTerminalBatchMarker ? 'candidate.delivery_batch_started_at' : 'NULL::timestamptz'} AS delivery_batch_started_at
17697
17907
  FROM ${tableName('outbox', options)} AS candidate
17908
+ ${input.extendOwnedHead ? 'JOIN owned_run ON owned_run.run_id = candidate.run_id' : ''}
17698
17909
  WHERE candidate.delivered_at IS NULL
17910
+ AND ${searchLaneSql('candidate')} = ${input.searchOnly === true ? 'true' : 'false'}
17911
+ ${
17912
+ input.extendOwnedHead
17913
+ ? `AND candidate.event_id = $5
17914
+ AND candidate.lease_owner = $3 AND candidate.lease_expires_at > now()`
17915
+ : ''
17916
+ }
17699
17917
  AND candidate.failed_at IS NULL
17700
17918
  AND coalesce(candidate.next_attempt_at, candidate.created_at) <= now()
17701
17919
  AND ($2::text IS NULL OR candidate.run_id = $2)
@@ -17711,6 +17929,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17711
17929
  AND NOT EXISTS (
17712
17930
  SELECT 1 FROM ${tableName('outbox', options)} AS active
17713
17931
  WHERE active.run_id = candidate.run_id
17932
+ AND ${searchLaneSql('active')} = ${searchLaneSql('candidate')}
17714
17933
  AND active.delivered_at IS NULL
17715
17934
  AND active.lease_owner IS NOT NULL
17716
17935
  AND active.lease_expires_at > now()
@@ -17738,7 +17957,8 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17738
17957
  CASE
17739
17958
  WHEN candidate.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0
17740
17959
  WHEN ${runLedgerTerminalPendingSql('candidate', options)} THEN 1
17741
- ELSE 2
17960
+ ${input.searchOnly === true ? '' : `WHEN ${runLedgerWaitingPendingSql('candidate')} THEN 2`}
17961
+ ELSE 3
17742
17962
  END,
17743
17963
  coalesce(candidate.next_attempt_at, candidate.created_at),
17744
17964
  candidate.created_at,
@@ -17750,16 +17970,25 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17750
17970
  FROM ${tableName('outbox', options)} AS outbox
17751
17971
  JOIN head ON head.run_id = outbox.run_id
17752
17972
  WHERE outbox.delivered_at IS NULL
17973
+ AND ${searchLaneSql('outbox')} = ${input.searchOnly === true ? 'true' : 'false'}
17753
17974
  AND outbox.failed_at IS NULL
17754
17975
  -- Admission is causally first and must be projected alone. A later
17755
17976
  -- start/terminal append may not leapfrog a delayed queued row.
17756
17977
  AND (
17757
17978
  head.event_type NOT IN (${RUN_EVENT_KIND_SQL_LITERAL.runCreated}, ${RUN_EVENT_KIND_SQL_LITERAL.runStarted}, ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents})
17758
17979
  OR outbox.event_id = head.event_id
17980
+ OR (head.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
17981
+ AND ${ingressPrefixSql})
17759
17982
  )
17760
- -- An ingress row already contains a producer's bounded batch. Keep
17761
- -- its receipt stable independently of the recovery caller's limit.
17762
- AND (outbox.event_type <> ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents} OR outbox.event_id = head.event_id)
17983
+ -- Only a receiver with the exact-prefix verifier may combine ingress
17984
+ -- facts while draining a pending terminal. Old launch origins and
17985
+ -- schemas retain each producer's existing singleton receipt.
17986
+ AND (outbox.event_type <> ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
17987
+ OR outbox.event_id = head.event_id
17988
+ OR ${ingressPrefixSql})
17989
+ -- Once any ingress prefix was attempted, its receipt is immutable,
17990
+ -- including singleton appends whose HTTP/ACK outcome was ambiguous.
17991
+ AND (${supportsTerminalBatchMarker && input.allowIngressPrefix === true ? 'head.delivery_batch_started_at IS NULL OR outbox.delivery_batch_started_at IS NOT NULL' : 'true'})
17763
17992
  AND coalesce(outbox.next_attempt_at, outbox.created_at) <= now()
17764
17993
  AND (
17765
17994
  $3::text IS NULL
@@ -17802,7 +18031,9 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17802
18031
  AND predecessor.delivered_at IS NULL
17803
18032
  AND ${runLedgerPrecedesSql('predecessor', 'outbox')}
17804
18033
  AND (
17805
- predecessor.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
18034
+ (predecessor.event_type = ${RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL.runEvents}
18035
+ AND NOT (${ingressPrefixSql}))
18036
+ OR (${supportsTerminalBatchMarker && input.allowIngressPrefix === true ? 'head.delivery_batch_started_at IS NOT NULL AND predecessor.delivery_batch_started_at IS NULL' : 'false'})
17806
18037
  OR predecessor.failed_at IS NOT NULL
17807
18038
  OR coalesce(predecessor.next_attempt_at, predecessor.created_at) > now()
17808
18039
  OR (
@@ -17833,8 +18064,8 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17833
18064
  ${
17834
18065
  supportsTerminalBatchMarker
17835
18066
  ? `, delivery_batch_started_at = CASE
17836
- WHEN terminal_batch.has_terminal
17837
- THEN COALESCE(outbox.delivery_batch_started_at, clock_timestamp())
18067
+ WHEN terminal_batch.has_terminal ${input.allowIngressPrefix === true ? `OR ${urgentLifecyclePendingSql('outbox')}` : ''}
18068
+ THEN COALESCE(outbox.delivery_batch_started_at, statement_timestamp())
17838
18069
  ELSE outbox.delivery_batch_started_at
17839
18070
  END`
17840
18071
  : ''
@@ -17851,8 +18082,15 @@ export async function readPendingPostgresSchedulerOutboxEvents(
17851
18082
  FROM claimed
17852
18083
  ORDER BY ${runLedgerOrderSql('claimed')}
17853
18084
  `,
17854
- [limit, runId, leaseOwner, leaseSeconds],
18085
+ [
18086
+ limit,
18087
+ runId,
18088
+ leaseOwner,
18089
+ leaseSeconds,
18090
+ ...(input.extendOwnedHead ? [input.extendOwnedHead] : []),
18091
+ ],
17855
18092
  );
18093
+ };
17856
18094
  let result;
17857
18095
  if (input.inTransaction) await client.query('SAVEPOINT run_ledger_claim');
17858
18096
  if (supportsOutboxDeliveryBatchMarker(options)) {
@@ -17910,6 +18148,7 @@ export async function readPostgresSchedulerOutboxNextDueAt(
17910
18148
  (SELECT max(active.lease_expires_at)
17911
18149
  FROM ${tableName('outbox', options)} AS active
17912
18150
  WHERE active.run_id = candidate.run_id
18151
+ AND ${searchLaneSql('active')} = ${searchLaneSql('candidate')}
17913
18152
  AND active.delivered_at IS NULL AND active.lease_owner IS NOT NULL
17914
18153
  AND active.lease_expires_at > now())
17915
18154
  )) AS due_at
@@ -18149,7 +18388,7 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
18149
18388
  '{events}',
18150
18389
  (SELECT jsonb_agg(
18151
18390
  CASE WHEN item.value->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
18152
- THEN item.value - 'lines' - 'lineContexts'
18391
+ THEN item.value - 'lines' - 'lineContexts' - 'lineIds'
18153
18392
  ELSE item.value
18154
18393
  END ORDER BY item.ordinality
18155
18394
  )
@@ -18162,11 +18401,11 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
18162
18401
  )
18163
18402
  WHEN jsonb_typeof(outbox.payload_json->'terminalProgressEvents') = 'array'
18164
18403
  THEN jsonb_set(
18165
- outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts',
18404
+ outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogIds',
18166
18405
  '{terminalProgressEvents}',
18167
18406
  coalesce((SELECT jsonb_agg(
18168
18407
  CASE WHEN item.value->>'type' = ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
18169
- THEN item.value - 'lines' - 'lineContexts'
18408
+ THEN item.value - 'lines' - 'lineContexts' - 'lineIds'
18170
18409
  ELSE item.value
18171
18410
  END ORDER BY item.ordinality
18172
18411
  )
@@ -18174,7 +18413,7 @@ export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
18174
18413
  WITH ORDINALITY AS item(value, ordinality)), '[]'::jsonb),
18175
18414
  false
18176
18415
  )
18177
- ELSE outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts'
18416
+ ELSE outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts' - 'terminalLogIds'
18178
18417
  END
18179
18418
  FROM candidates
18180
18419
  WHERE outbox.event_id = candidates.event_id`,
@@ -18538,6 +18777,22 @@ function normalizeOutboxIngressPreview(
18538
18777
  export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
18539
18778
  event: PostgresSchedulerOutboxEvent,
18540
18779
  ): PlayRunLedgerIngressEvent[] {
18780
+ if (
18781
+ payloadField(event.payload, RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD) ===
18782
+ true
18783
+ ) {
18784
+ throw new Error(
18785
+ 'Deferred outbox input must be projected before ledger delivery.',
18786
+ );
18787
+ }
18788
+ if (
18789
+ payloadField(event.payload, RUNTIME_EVENT_SEARCH_ONLY_FIELD) === true ||
18790
+ event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.receiptEvent ||
18791
+ event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runnerLogs
18792
+ ) {
18793
+ // Search-only carriers are already durable in the canonical journal.
18794
+ return [];
18795
+ }
18541
18796
  const occurredAt = event.createdAt.getTime();
18542
18797
  if (event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents) {
18543
18798
  const events = payloadField(event.payload, 'events');
@@ -18774,6 +19029,8 @@ export type PostgresSchedulerOutboxMaterializationPlan = {
18774
19029
  };
18775
19030
 
18776
19031
  const RUN_LEDGER_KNOWN_OUTBOX_EVENT_TYPES: ReadonlySet<string> = new Set([
19032
+ RUN_SCHEDULER_OUTBOX_EVENT_TYPE.receiptEvent,
19033
+ RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runnerLogs,
18777
19034
  RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents,
18778
19035
  RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runStarted,
18779
19036
  RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runCompleted,
@@ -18912,6 +19169,21 @@ export type RunLedgerOutboxTokenAuthority = 'local_mint' | 'gateway';
18912
19169
  /** Cross-run service policy; causal order within a run is always unchanged. */
18913
19170
  export type RunLedgerOutboxHeadPriority = 'admission_first' | 'oldest_ready';
18914
19171
 
19172
+ class DeferredRuntimeProjectionInputError extends AppRuntimeApiResponseError {
19173
+ constructor(
19174
+ readonly eventId: string,
19175
+ detail: string,
19176
+ ) {
19177
+ super({
19178
+ action: 'project_run_events',
19179
+ status: 422,
19180
+ code: 'runtime_event_projection_invalid',
19181
+ retryable: false,
19182
+ detail,
19183
+ });
19184
+ }
19185
+ }
19186
+
18915
19187
  const POSTGRES_SCHEDULER_CANCELLATION_OUTBOX_LEASE_SECONDS =
18916
19188
  // The direct projection can run for the full app-runtime request deadline.
18917
19189
  // Keep another 30 seconds for dispatch setup and acknowledgement so a
@@ -18942,6 +19214,10 @@ export function postgresSchedulerCancellationOutboxLease(runId: string): {
18942
19214
  }
18943
19215
 
18944
19216
  export type RunLedgerOutboxDispatchInput = {
19217
+ /** Only the existing background projector claims the independent search prefix. */
19218
+ searchOnly?: boolean;
19219
+ /** Only the existing background projector indexes deferred raw inputs. */
19220
+ materializeObservability?: boolean;
18945
19221
  /**
18946
19222
  * Optional runtime context. `gateway` authority always replaces its
18947
19223
  * credentials with a fresh lease-scoped token and keeps only supported
@@ -18983,8 +19259,9 @@ export type RunLedgerOutboxDispatchInput = {
18983
19259
  * untargeted fast path) set this: after acknowledging a batch, also carry
18984
19260
  * the run's terminal fact through when it is now pending (see
18985
19261
  * `continueRunLedgerOutboxThroughTerminal`). Callers that drive their own
18986
- * bounded batches over a pre-leased prefix (cancellation, inline
18987
- * `run.created`, the targeted terminal drain) leave it unset.
19262
+ * bounded batches (cancellation, inline `run.created`, the targeted terminal
19263
+ * drain) leave it unset. Targeted drains retain successors through their
19264
+ * explicit run/owner/target scope instead.
18988
19265
  */
18989
19266
  continueThroughTerminal?: boolean;
18990
19267
  };
@@ -19066,7 +19343,10 @@ type RunLedgerOutboxAuthorityScope = {
19066
19343
  authority: {
19067
19344
  runId: string;
19068
19345
  leaseOwner: string;
19069
- runtimeApi: WorkerRuntimeApiContext;
19346
+ runtimeApi: WorkerRuntimeApiContext & {
19347
+ refreshExecutorToken?: () => Promise<string>;
19348
+ invalidateExecutorToken?: () => void;
19349
+ };
19070
19350
  expiresAtMs: number | null;
19071
19351
  } | null;
19072
19352
  };
@@ -19101,6 +19381,9 @@ async function dispatchRunLedgerOutboxThroughEventWithScope(
19101
19381
  input.leaseOwner?.trim() ||
19102
19382
  scope?.leaseOwner ||
19103
19383
  `run-ledger-outbox-dispatch:${randomUUID()}`;
19384
+ // The same owner retains each successor in the ACK transaction, including
19385
+ // for a fresh worker drain that did not arrive with a preclaimed prefix.
19386
+ scope ??= { runId, leaseOwner, targetEventId, nextClaimed: [] };
19104
19387
  const scopedInput = { ...input, leaseOwner };
19105
19388
  const batchSize = RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS;
19106
19389
  const eventIds: string[] = [];
@@ -19111,75 +19394,87 @@ async function dispatchRunLedgerOutboxThroughEventWithScope(
19111
19394
  let failed = 0;
19112
19395
  let retrying = 0;
19113
19396
 
19114
- let remaining = maxEvents;
19115
- while (remaining === null || remaining > 0) {
19116
- const result = await dispatchRunLedgerOutboxBatchWithScope(
19117
- withClient,
19118
- {
19119
- ...scopedInput,
19120
- waitForRunLock: true,
19121
- continueThroughTerminal: false,
19122
- limit: remaining === null ? batchSize : Math.min(batchSize, remaining),
19123
- },
19124
- options,
19125
- scope,
19126
- authorityScope,
19397
+ try {
19398
+ let remaining = maxEvents;
19399
+ while (remaining === null || remaining > 0) {
19400
+ const result = await dispatchRunLedgerOutboxBatchWithScope(
19401
+ withClient,
19402
+ {
19403
+ ...scopedInput,
19404
+ waitForRunLock: true,
19405
+ continueThroughTerminal: false,
19406
+ limit:
19407
+ remaining === null ? batchSize : Math.min(batchSize, remaining),
19408
+ },
19409
+ options,
19410
+ scope,
19411
+ authorityScope,
19412
+ remaining,
19413
+ );
19414
+ attempted += result.attempted;
19415
+ projected += result.projected;
19416
+ failed += result.failed;
19417
+ retrying += result.retrying;
19418
+ eventIds.push(...result.eventIds);
19419
+ failedEventIds.push(...result.failedEventIds);
19420
+ retryingEventIds.push(...result.retryingEventIds);
19421
+ if (remaining !== null) remaining -= result.attempted;
19422
+ if (result.eventIds.includes(targetEventId)) {
19423
+ return {
19424
+ attempted,
19425
+ projected,
19426
+ eventIds,
19427
+ failed,
19428
+ failedEventIds,
19429
+ retrying,
19430
+ retryingEventIds,
19431
+ targetEventId,
19432
+ targetDelivered: true,
19433
+ targetBlocked: false,
19434
+ };
19435
+ }
19436
+ // A projection failure is durably scheduled for retry or moved to terminal
19437
+ // DLQ. Its policy owns the next attempt; spinning here would only reclaim
19438
+ // the same failed predecessor and delay the park indefinitely.
19439
+ // An absent target cannot retain a successor in ACK. Keep the existing
19440
+ // fresh-claim traversal in that case until ready work is exhausted.
19441
+ if (result.projected === 0 || result.attempted === 0) break;
19442
+ }
19443
+
19444
+ // A concurrent dispatcher can claim the target between batches. An empty
19445
+ // drain therefore does not prove delivery; read the durable acknowledgement
19446
+ // before reporting the target as pending.
19447
+ const target = await withClient((client) =>
19448
+ readPostgresSchedulerOutboxDeliveryStatus(
19449
+ client,
19450
+ { runId, eventId: targetEventId },
19451
+ options,
19452
+ ),
19127
19453
  );
19128
- attempted += result.attempted;
19129
- projected += result.projected;
19130
- failed += result.failed;
19131
- retrying += result.retrying;
19132
- eventIds.push(...result.eventIds);
19133
- failedEventIds.push(...result.failedEventIds);
19134
- retryingEventIds.push(...result.retryingEventIds);
19135
- if (remaining !== null) remaining -= result.attempted;
19136
- if (result.eventIds.includes(targetEventId)) {
19137
- return {
19138
- attempted,
19139
- projected,
19140
- eventIds,
19141
- failed,
19142
- failedEventIds,
19143
- retrying,
19144
- retryingEventIds,
19145
- targetEventId,
19146
- targetDelivered: true,
19147
- targetBlocked: false,
19148
- };
19454
+ return {
19455
+ attempted,
19456
+ projected,
19457
+ eventIds,
19458
+ failed,
19459
+ failedEventIds,
19460
+ retrying,
19461
+ retryingEventIds,
19462
+ targetEventId,
19463
+ targetDelivered: target.delivered,
19464
+ targetBlocked: target.blocked,
19465
+ };
19466
+ } finally {
19467
+ if (scope.nextClaimed.length > 0) {
19468
+ await withClient((client) =>
19469
+ releasePostgresSchedulerOutboxLease(
19470
+ client,
19471
+ scope.nextClaimed.map((event) => event.eventId),
19472
+ { ...options, leaseOwner },
19473
+ ),
19474
+ );
19475
+ scope.nextClaimed.length = 0;
19149
19476
  }
19150
- // A projection failure is durably scheduled for retry or moved to terminal
19151
- // DLQ. Its policy owns the next attempt; spinning here would only reclaim
19152
- // the same failed predecessor and delay the park indefinitely.
19153
- if (
19154
- result.projected === 0 ||
19155
- result.attempted === 0 ||
19156
- (scope && scope.nextClaimed.length === 0)
19157
- )
19158
- break;
19159
19477
  }
19160
-
19161
- // A concurrent dispatcher can claim the target between batches. An empty
19162
- // drain therefore does not prove delivery; read the durable acknowledgement
19163
- // before reporting the target as pending.
19164
- const target = await withClient((client) =>
19165
- readPostgresSchedulerOutboxDeliveryStatus(
19166
- client,
19167
- { runId, eventId: targetEventId },
19168
- options,
19169
- ),
19170
- );
19171
- return {
19172
- attempted,
19173
- projected,
19174
- eventIds,
19175
- failed,
19176
- failedEventIds,
19177
- retrying,
19178
- retryingEventIds,
19179
- targetEventId,
19180
- targetDelivered: target.delivered,
19181
- targetBlocked: target.blocked,
19182
- };
19183
19478
  }
19184
19479
 
19185
19480
  /**
@@ -19219,8 +19514,15 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19219
19514
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
19220
19515
  scope?: RunLedgerOutboxDrainScope,
19221
19516
  authorityScope: RunLedgerOutboxAuthorityScope = { authority: null },
19517
+ remainingEvents: number | null = null,
19222
19518
  ): Promise<PostgresSchedulerOutboxProjectionResult> {
19223
19519
  const dispatchStartedAtMs = Date.now();
19520
+ const tokenAuthority = input.tokenAuthority ?? 'local_mint';
19521
+ const dispatchLaunchSql =
19522
+ !input.searchOnly && (tokenAuthority === 'gateway' || !input.runtimeApi)
19523
+ ? runLedgerDispatchLaunchSql('runs')
19524
+ : 'NULL::jsonb';
19525
+ let claimedLaunchJson: unknown;
19224
19526
  const leaseOwner =
19225
19527
  input.leaseOwner?.trim() || `run-ledger-outbox-dispatch:${randomUUID()}`;
19226
19528
  const continuation: { current: RunLedgerOutboxDrainScope | null } = {
@@ -19272,7 +19574,10 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19272
19574
  // for each lifecycle transition. Only the two dedicated projectors may
19273
19575
  // perform the global distinct-head scan below.
19274
19576
  const selected = targetRunId
19275
- ? await client.query<{ run_id: string }>(
19577
+ ? await client.query<{
19578
+ run_id: string;
19579
+ dispatch_launch_json?: unknown;
19580
+ }>(
19276
19581
  `
19277
19582
  WITH head AS MATERIALIZED (
19278
19583
  SELECT pending.run_id,
@@ -19284,10 +19589,11 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19284
19589
  FROM ${tableName('outbox', options)} AS pending
19285
19590
  WHERE pending.run_id = $1
19286
19591
  AND pending.delivered_at IS NULL
19592
+ AND ${searchLaneSql('pending')} = ${input.searchOnly === true ? 'true' : 'false'}
19287
19593
  ORDER BY ${runLedgerOrderSql('pending')}
19288
19594
  LIMIT 1
19289
19595
  )
19290
- SELECT runs.run_id
19596
+ SELECT runs.run_id, ${dispatchLaunchSql} AS dispatch_launch_json
19291
19597
  FROM head
19292
19598
  JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
19293
19599
  WHERE head.failed_at IS NULL
@@ -19296,6 +19602,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19296
19602
  AND NOT EXISTS (
19297
19603
  SELECT 1 FROM ${tableName('outbox', options)} AS active
19298
19604
  WHERE active.run_id = runs.run_id
19605
+ AND ${searchLaneSql('active')} = ${input.searchOnly === true ? 'true' : 'false'}
19299
19606
  AND active.delivered_at IS NULL
19300
19607
  AND active.lease_owner IS NOT NULL
19301
19608
  AND active.lease_expires_at > now()
@@ -19305,8 +19612,17 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19305
19612
  `,
19306
19613
  [targetRunId, leaseOwner, input.headEventId?.trim() || null],
19307
19614
  )
19308
- : await client.query<{ run_id: string }>(`
19309
- WITH heads AS MATERIALIZED (
19615
+ : await client.query<{
19616
+ run_id: string;
19617
+ dispatch_launch_json?: unknown;
19618
+ }>(`
19619
+ WITH ${
19620
+ input.searchOnly === true
19621
+ ? ''
19622
+ : `waiting_runs AS MATERIALIZED (
19623
+ ${runLedgerWaitingRunsSql(options)}
19624
+ ), `
19625
+ }heads AS MATERIALIZED (
19310
19626
  SELECT DISTINCT ON (pending.run_id)
19311
19627
  pending.run_id,
19312
19628
  pending.event_id,
@@ -19316,9 +19632,10 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19316
19632
  pending.failed_at
19317
19633
  FROM ${tableName('outbox', options)} AS pending
19318
19634
  WHERE pending.delivered_at IS NULL
19635
+ AND ${searchLaneSql('pending')} = ${input.searchOnly === true ? 'true' : 'false'}
19319
19636
  ORDER BY pending.run_id, ${runLedgerOrderSql('pending')}
19320
19637
  )
19321
- SELECT runs.run_id
19638
+ SELECT runs.run_id, ${dispatchLaunchSql} AS dispatch_launch_json
19322
19639
  FROM heads AS head
19323
19640
  JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
19324
19641
  WHERE head.failed_at IS NULL
@@ -19326,12 +19643,13 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19326
19643
  AND NOT EXISTS (
19327
19644
  SELECT 1 FROM ${tableName('outbox', options)} AS active
19328
19645
  WHERE active.run_id = runs.run_id
19646
+ AND ${searchLaneSql('active')} = ${input.searchOnly === true ? 'true' : 'false'}
19329
19647
  AND active.delivered_at IS NULL
19330
19648
  AND active.lease_owner IS NOT NULL
19331
19649
  AND active.lease_expires_at > now()
19332
19650
  )
19333
19651
  -- admission_first: admission, then the causal head of a run with
19334
- -- a pending terminal, then FIFO progress. oldest_ready keeps its
19652
+ -- a pending terminal or public wait, then FIFO progress. oldest_ready keeps its
19335
19653
  -- pure age order so continuous admissions/terminals cannot starve
19336
19654
  -- ordinary progress.
19337
19655
  ORDER BY ${
@@ -19340,7 +19658,8 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19340
19658
  : `CASE
19341
19659
  WHEN head.event_type = ${RUN_EVENT_KIND_SQL_LITERAL.runCreated} THEN 0
19342
19660
  WHEN ${runLedgerTerminalPendingSql('head', options)} THEN 1
19343
- ELSE 2
19661
+ ${input.searchOnly === true ? '' : `WHEN ${runLedgerWaitingPendingSql('head')} THEN 2`}
19662
+ ELSE 3
19344
19663
  END,`
19345
19664
  }
19346
19665
  head.created_at, runs.run_id
@@ -19356,10 +19675,19 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19356
19675
  leaseOwner,
19357
19676
  leaseSeconds: input.leaseSeconds ?? 120,
19358
19677
  inTransaction: true,
19678
+ searchOnly: input.searchOnly,
19359
19679
  },
19360
19680
  options,
19361
19681
  )
19362
19682
  : [];
19683
+ // Authority/routing is immutable and already lives on the locked Run.
19684
+ // Capture only its existing narrow projection with the successful lease
19685
+ // claim, then release the checkout before requesting gateway authority.
19686
+ // The gateway still independently proves that this lease is current.
19687
+ const launchJson = selected.rows[0]?.dispatch_launch_json;
19688
+ if (claimed.length > 0 && launchJson != null) {
19689
+ claimedLaunchJson = launchJson;
19690
+ }
19363
19691
  await client.query('COMMIT');
19364
19692
  return claimed;
19365
19693
  } catch (error) {
@@ -19367,7 +19695,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19367
19695
  throw error;
19368
19696
  }
19369
19697
  };
19370
- const events = scope?.nextClaimed.length
19698
+ let events = scope?.nextClaimed.length
19371
19699
  ? scope.nextClaimed.splice(0)
19372
19700
  : await withClient(claimEvents);
19373
19701
  if (events.length === 0) {
@@ -19386,9 +19714,148 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19386
19714
  let authorityReadyAtMs: number | null = null;
19387
19715
  let appendFinishedAtMs: number | null = null;
19388
19716
  let acknowledgedAtMs: number | null = null;
19717
+ const materializeDeferredObservability = async () => {
19718
+ if (
19719
+ events.some(
19720
+ (event) =>
19721
+ payloadField(
19722
+ event.payload,
19723
+ RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD,
19724
+ ) === true,
19725
+ )
19726
+ ) {
19727
+ if (!input.materializeObservability) {
19728
+ // Inline admission/lifecycle/read-heal paths must never index logs.
19729
+ // Release and wake the existing projector without charging a retry.
19730
+ const released = await withClient((client) =>
19731
+ releasePostgresSchedulerOutboxLease(
19732
+ client,
19733
+ events.map((event) => event.eventId),
19734
+ { ...options, leaseOwner },
19735
+ ),
19736
+ );
19737
+ return {
19738
+ attempted: events.length,
19739
+ projected: 0,
19740
+ eventIds: [],
19741
+ failed: 0,
19742
+ failedEventIds: [],
19743
+ retrying: released.length,
19744
+ retryingEventIds: released,
19745
+ };
19746
+ }
19747
+ for (const event of events) {
19748
+ if (
19749
+ payloadField(
19750
+ event.payload,
19751
+ RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD,
19752
+ ) !== true
19753
+ )
19754
+ continue;
19755
+ await withClient(async (client) => {
19756
+ await renewPostgresRunLedgerDelivery(
19757
+ client,
19758
+ {
19759
+ eventIds: events.map((row) => row.eventId),
19760
+ leaseOwner,
19761
+ leaseSeconds: 120,
19762
+ },
19763
+ options,
19764
+ );
19765
+ // Keep projection below its renewed lease. The final payload update
19766
+ // fences ownership; indexing locks neither outbox nor Run rows.
19767
+ await client.query('BEGIN');
19768
+ try {
19769
+ // Let PostgreSQL cancel before the pool's client-side timeout,
19770
+ // so rollback completes while the checkout is still usable.
19771
+ await client.query(
19772
+ "SELECT set_config('statement_timeout', $1, true)",
19773
+ [
19774
+ `${Math.min(60_000, RUNTIME_RELIABILITY_POLICY.schedulerDatabase.queryTimeoutMs - 1_000)}ms`,
19775
+ ],
19776
+ );
19777
+ const result = await client.query<{ payload: unknown }>(
19778
+ `SELECT ${quoteIdent(schemaName(options))}.materialize_runtime_outbox_event($1, $2) AS payload`,
19779
+ [event.eventId, leaseOwner],
19780
+ );
19781
+ await client.query('COMMIT');
19782
+ event.payload = result.rows[0]!.payload;
19783
+ } catch (error) {
19784
+ await client.query('ROLLBACK');
19785
+ if (
19786
+ error &&
19787
+ typeof error === 'object' &&
19788
+ 'code' in error &&
19789
+ error.code === 'P0001'
19790
+ ) {
19791
+ // The installed projector's explicit RAISEs reject malformed
19792
+ // or conflicting immutable facts. Retain them in the existing
19793
+ // quarantine instead of retrying the same invalid input forever.
19794
+ throw new DeferredRuntimeProjectionInputError(
19795
+ event.eventId,
19796
+ error instanceof Error
19797
+ ? error.message
19798
+ : 'Invalid runtime event projection input.',
19799
+ );
19800
+ }
19801
+ throw error;
19802
+ }
19803
+ });
19804
+ }
19805
+ }
19806
+ return null;
19807
+ };
19389
19808
  try {
19809
+ if (!input.materializeObservability) {
19810
+ const handedOff = await materializeDeferredObservability();
19811
+ if (handedOff) return handedOff;
19812
+ }
19813
+ if (input.searchOnly) {
19814
+ const handedOff = await materializeDeferredObservability();
19815
+ if (handedOff) return handedOff;
19816
+ if (
19817
+ events.some(
19818
+ (event) =>
19819
+ payloadField(event.payload, RUNTIME_EVENT_SEARCH_ONLY_FIELD) !==
19820
+ true,
19821
+ )
19822
+ )
19823
+ throw new Error('Search lease includes a lifecycle fact.');
19824
+ const delivered = await withClient((client) =>
19825
+ markPostgresSchedulerOutboxDeliveredRows(
19826
+ client,
19827
+ events.map((event) => event.eventId),
19828
+ { ...options, leaseOwner },
19829
+ ),
19830
+ );
19831
+ if (delivered.length !== events.length)
19832
+ throw new RunLedgerProjectionLeaseLostError(
19833
+ 'delivery_lease_not_current',
19834
+ );
19835
+ if (input.onDelivered) {
19836
+ try {
19837
+ input.onDelivered(delivered);
19838
+ } catch (error) {
19839
+ console.warn('[run-ledger-outbox.delivery_observer_failed]', {
19840
+ error: error instanceof Error ? error.message : String(error),
19841
+ });
19842
+ }
19843
+ }
19844
+ return {
19845
+ attempted: events.length,
19846
+ projected: delivered.length,
19847
+ eventIds: events.map((event) => event.eventId),
19848
+ failed: 0,
19849
+ failedEventIds: [],
19850
+ retrying: 0,
19851
+ retryingEventIds: [],
19852
+ };
19853
+ }
19390
19854
  const first = events[0]!;
19391
- const tokenAuthority = input.tokenAuthority ?? 'local_mint';
19855
+ const claimedLaunch =
19856
+ claimedLaunchJson == null
19857
+ ? undefined
19858
+ : parseLaunchJson(claimedLaunchJson);
19392
19859
  if (
19393
19860
  scope &&
19394
19861
  (scope.runId !== first.runId || scope.leaseOwner !== leaseOwner)
@@ -19422,6 +19889,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19422
19889
  tokenAuthority,
19423
19890
  options,
19424
19891
  input.runtimeApi?.signal,
19892
+ claimedLaunch,
19425
19893
  )
19426
19894
  : null;
19427
19895
  if (
@@ -19445,6 +19913,8 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19445
19913
  : null,
19446
19914
  tokenAuthority,
19447
19915
  options,
19916
+ undefined,
19917
+ claimedLaunch,
19448
19918
  ));
19449
19919
  const runtimeApi = {
19450
19920
  ...resolvedRuntimeApi,
@@ -19486,6 +19956,49 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19486
19956
  };
19487
19957
  }
19488
19958
  authorityReadyAtMs = Date.now();
19959
+ // The initial claim deliberately uses the old contract until its immutable
19960
+ // launch receiver advertises support. Widen under the same run/owner fence
19961
+ // before the first HTTP append, so an already-finished run need not pay one
19962
+ // authority/append/ACK round trip for every ingress predecessor.
19963
+ if (
19964
+ runtimeApi.runLedgerIngressPrefixBatching === true &&
19965
+ events.length === 1 &&
19966
+ first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents
19967
+ ) {
19968
+ const widened = await withClient((client) =>
19969
+ readPendingPostgresSchedulerOutboxEvents(
19970
+ client,
19971
+ {
19972
+ runId: first.runId,
19973
+ leaseOwner,
19974
+ leaseSeconds: input.leaseSeconds ?? 120,
19975
+ limit: Math.min(
19976
+ RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
19977
+ remainingEvents ?? RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
19978
+ ),
19979
+ allowIngressPrefix: true,
19980
+ extendOwnedHead: first.eventId,
19981
+ },
19982
+ options,
19983
+ ),
19984
+ );
19985
+ // Expansion uses SKIP LOCKED on the run fence. A concurrent lifecycle
19986
+ // writer can make it empty while our original head lease remains live.
19987
+ // Keep that singleton; mandatory renewal below still rejects expired
19988
+ // or replaced ownership and freezes its receipt before any HTTP append.
19989
+ if (widened.length > 0) {
19990
+ if (widened[0]!.eventId !== first.eventId) {
19991
+ throw new RunLedgerProjectionLeaseLostError(
19992
+ 'head_lease_owner_mismatch',
19993
+ );
19994
+ }
19995
+ events = widened;
19996
+ }
19997
+ }
19998
+ // Widening can add deferred facts to the lease. Project the complete
19999
+ // frozen prefix before any HTTP append, including its newly claimed rows.
20000
+ const handedOff = await materializeDeferredObservability();
20001
+ if (handedOff) return handedOff;
19489
20002
  const renewDelivery = async () => {
19490
20003
  const currentAuthority = authorityScope.authority;
19491
20004
  if (
@@ -19508,14 +20021,16 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19508
20021
  'Run Ledger dispatch origin changed within an immutable launch.',
19509
20022
  );
19510
20023
  }
19511
- runtimeApi.executorToken = refreshed.executorToken;
20024
+ runtimeApi.executorToken = refreshed.refreshExecutorToken
20025
+ ? await refreshed.refreshExecutorToken()
20026
+ : refreshed.executorToken;
19512
20027
  runtimeApi.vercelProtectionBypassToken =
19513
20028
  refreshed.vercelProtectionBypassToken;
19514
20029
  authorityScope.authority = {
19515
20030
  runId: first.runId,
19516
20031
  leaseOwner,
19517
20032
  runtimeApi,
19518
- expiresAtMs: runtimeApiTokenExpiresAtMs(refreshed.executorToken),
20033
+ expiresAtMs: runtimeApiTokenExpiresAtMs(runtimeApi.executorToken),
19519
20034
  };
19520
20035
  }
19521
20036
  await withClient((client) =>
@@ -19525,70 +20040,108 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19525
20040
  eventIds: events.map((event) => event.eventId),
19526
20041
  leaseOwner,
19527
20042
  leaseSeconds: input.leaseSeconds ?? 120,
20043
+ freezeReceipt:
20044
+ runtimeApi.runLedgerIngressPrefixBatching === true &&
20045
+ events.some(
20046
+ (event) =>
20047
+ event.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents,
20048
+ ),
19528
20049
  },
19529
20050
  options,
19530
20051
  ),
19531
20052
  );
19532
20053
  };
19533
- // A pending start is always claimed alone because later events require its
19534
- // snapshot. It cannot share the append endpoint, so retain that mutation
19535
- // as its own exact delivery.
19536
- if (first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runCreated) {
19537
- await renewDelivery();
19538
- await projectPostgresSchedulerCreatedEvent(runtimeApi, first);
19539
- } else if (first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runStarted) {
19540
- await renewDelivery();
19541
- const leaseOwner = first.leaseOwner;
19542
- const launch = leaseOwner
19543
- ? await withClient((client) =>
19544
- readRunLedgerDispatchLaunchForLease(
19545
- client,
19546
- {
19547
- runId: first.runId,
19548
- eventId: first.eventId,
19549
- leaseOwner,
19550
- },
19551
- options,
19552
- ),
19553
- )
19554
- : null;
19555
- await projectPostgresSchedulerStartEvent(runtimeApi, first, launch);
19556
- } else {
19557
- const ledgerEvents = events.flatMap(
19558
- buildRunLedgerEventsFromPostgresSchedulerOutbox,
19559
- );
19560
- if (ledgerEvents.length > 0) {
19561
- const terminal = [...events]
19562
- .reverse()
19563
- .find((event) =>
19564
- RUN_SCHEDULER_OUTBOX_TERMINAL_EVENT_TYPES.includes(event.eventType),
19565
- );
19566
- const batchDeliveryKey = createHash('sha256')
19567
- .update(events.map((event) => event.deliveryKey).join('\n'))
19568
- .digest('hex');
19569
- await projectRunEventsViaAppRuntime(runtimeApi, {
19570
- playId: first.runId,
19571
- events: ledgerEvents,
19572
- deliveryLeaseOwner: leaseOwner,
19573
- beforeBatch: renewDelivery,
19574
- idempotencyKey: terminal
19575
- ? buildRunLedgerTerminalIdempotencyKey({
19576
- runId: first.runId,
19577
- deliveryKey: terminal.deliveryKey,
19578
- })
19579
- : buildRunLedgerEventsIdempotencyKey({
19580
- runId: first.runId,
19581
- deliveryDigest: batchDeliveryKey,
19582
- }),
19583
- });
19584
- } else {
20054
+ const append = async () => {
20055
+ // A pending start is always claimed alone because later events require its
20056
+ // snapshot. It cannot share the append endpoint, so retain that mutation
20057
+ // as its own exact delivery.
20058
+ if (first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runCreated) {
19585
20059
  await renewDelivery();
20060
+ await projectPostgresSchedulerCreatedEvent(runtimeApi, first);
20061
+ } else if (
20062
+ first.eventType === RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runStarted
20063
+ ) {
20064
+ await renewDelivery();
20065
+ const leaseOwner = first.leaseOwner;
20066
+ const launch = leaseOwner
20067
+ ? await withClient((client) =>
20068
+ readRunLedgerDispatchLaunchForLease(
20069
+ client,
20070
+ {
20071
+ runId: first.runId,
20072
+ eventId: first.eventId,
20073
+ leaseOwner,
20074
+ },
20075
+ options,
20076
+ ),
20077
+ )
20078
+ : null;
20079
+ await projectPostgresSchedulerStartEvent(runtimeApi, first, launch);
20080
+ } else {
20081
+ const ledgerEvents = events.flatMap(
20082
+ buildRunLedgerEventsFromPostgresSchedulerOutbox,
20083
+ );
20084
+ if (ledgerEvents.length > 0) {
20085
+ const terminal = [...events]
20086
+ .reverse()
20087
+ .find((event) =>
20088
+ RUN_SCHEDULER_OUTBOX_TERMINAL_EVENT_TYPES.includes(
20089
+ event.eventType,
20090
+ ),
20091
+ );
20092
+ const batchDeliveryKey = createHash('sha256')
20093
+ .update(events.map((event) => event.deliveryKey).join('\n'))
20094
+ .digest('hex');
20095
+ await projectRunEventsViaAppRuntime(runtimeApi, {
20096
+ playId: first.runId,
20097
+ events: ledgerEvents,
20098
+ deliveryLeaseOwner: leaseOwner,
20099
+ beforeBatch: renewDelivery,
20100
+ idempotencyKey: terminal
20101
+ ? buildRunLedgerTerminalIdempotencyKey({
20102
+ runId: first.runId,
20103
+ deliveryKey: terminal.deliveryKey,
20104
+ })
20105
+ : buildRunLedgerEventsIdempotencyKey({
20106
+ runId: first.runId,
20107
+ deliveryDigest: batchDeliveryKey,
20108
+ }),
20109
+ });
20110
+ } else {
20111
+ await renewDelivery();
20112
+ }
19586
20113
  }
20114
+ };
20115
+ try {
20116
+ await append();
20117
+ } catch (error) {
20118
+ // A cached authority can be invalidated by signer rotation. Refresh it
20119
+ // once through this delivery's active lease; the append remains fenced
20120
+ // and carries the same idempotency key on both attempts.
20121
+ if (
20122
+ !gatewayRuntimeApi?.refreshExecutorToken ||
20123
+ !(error instanceof AppRuntimeApiResponseError) ||
20124
+ error.status !== 401
20125
+ )
20126
+ throw error;
20127
+ runtimeApi.executorToken = await gatewayRuntimeApi.refreshExecutorToken();
20128
+ await append();
19587
20129
  }
19588
20130
  appendFinishedAtMs = Date.now();
19589
20131
  const acknowledgement = await acknowledgeRunLedgerPrefixWithContinuation(
19590
20132
  withClient,
19591
- { input, events, leaseOwner, runtimeApi, scope, continuation },
20133
+ {
20134
+ input,
20135
+ events,
20136
+ leaseOwner,
20137
+ runtimeApi,
20138
+ scope,
20139
+ continuation,
20140
+ continuationAllowance:
20141
+ remainingEvents === null
20142
+ ? null
20143
+ : Math.max(0, remainingEvents - events.length),
20144
+ },
19592
20145
  options,
19593
20146
  );
19594
20147
  const { deliveredRows } = acknowledgement;
@@ -19630,6 +20183,10 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19630
20183
  authorityScope,
19631
20184
  );
19632
20185
  } catch (error) {
20186
+ // A failed append must reacquire receiver truth on its next attempt,
20187
+ // including an older receiver that rejected a frozen compound receipt.
20188
+ authorityScope.authority?.runtimeApi.invalidateExecutorToken?.();
20189
+ authorityScope.authority = null;
19633
20190
  await releasePendingContinuation();
19634
20191
  if (input.signal?.aborted) {
19635
20192
  // The caller's latency budget expired; this is not a delivery failure.
@@ -19654,20 +20211,87 @@ async function dispatchRunLedgerOutboxBatchWithScope(
19654
20211
  retryingEventIds: released,
19655
20212
  };
19656
20213
  }
19657
- const settlement = await withClient(async (client) => {
19658
- const blocked: string[] = [];
19659
- const retrying: string[] = [];
19660
- for (const event of events) {
19661
- const outcome = await settlePostgresSchedulerOutboxFailure(
20214
+ if (error instanceof DeferredRuntimeProjectionInputError) {
20215
+ const outcome = await withClient((client) =>
20216
+ settlePostgresSchedulerOutboxFailure(client, error.eventId, error, {
20217
+ ...options,
20218
+ leaseOwner,
20219
+ }),
20220
+ );
20221
+ // Earlier facts may already be indexed. Only the invalid immutable row
20222
+ // is quarantined; valid predecessors remain deliverable in causal order.
20223
+ const released = await withClient((client) =>
20224
+ releasePostgresSchedulerOutboxLease(
19662
20225
  client,
19663
- event.eventId,
19664
- error,
20226
+ events
20227
+ .filter((event) => event.eventId !== error.eventId)
20228
+ .map((event) => event.eventId),
19665
20229
  { ...options, leaseOwner },
19666
- );
19667
- if (outcome === 'blocked') blocked.push(event.eventId);
19668
- else retrying.push(event.eventId);
20230
+ ),
20231
+ );
20232
+ const failed = outcome === 'blocked' ? [error.eventId] : [];
20233
+ const retrying =
20234
+ outcome === 'blocked' ? released : [...released, error.eventId];
20235
+ console.error('[run-ledger-outbox.invalid_projection_input]', {
20236
+ eventId: error.eventId,
20237
+ code: error.code,
20238
+ detail: error.message,
20239
+ });
20240
+ return {
20241
+ attempted: events.length,
20242
+ projected: 0,
20243
+ eventIds: [],
20244
+ failed: failed.length,
20245
+ failedEventIds: failed,
20246
+ retrying: retrying.length,
20247
+ retryingEventIds: retrying,
20248
+ };
20249
+ }
20250
+ const settlement = await withClient(async (client) => {
20251
+ await client.query('BEGIN');
20252
+ try {
20253
+ const blocked: string[] = [];
20254
+ const retrying: string[] = [];
20255
+ const ownedRetrying: string[] = [];
20256
+ for (const event of events) {
20257
+ const outcome = await settlePostgresSchedulerOutboxFailure(
20258
+ client,
20259
+ event.eventId,
20260
+ error,
20261
+ { ...options, leaseOwner },
20262
+ );
20263
+ if (outcome === 'blocked') blocked.push(event.eventId);
20264
+ else retrying.push(event.eventId);
20265
+ if (outcome === 'retrying') ownedRetrying.push(event.eventId);
20266
+ }
20267
+ // All members of this failed append must become due together. Keep
20268
+ // the existing per-row retry policy, taking its latest deadline only
20269
+ // for rows this owner actually settled, before releasing their locks.
20270
+ if (ownedRetrying.length > 1) {
20271
+ await client.query(
20272
+ `
20273
+ WITH retry AS (
20274
+ SELECT max(next_attempt_at) AS retry_at
20275
+ FROM ${tableName('outbox', options)}
20276
+ WHERE event_id = ANY($1::text[])
20277
+ AND delivered_at IS NULL AND failed_at IS NULL
20278
+ )
20279
+ UPDATE ${tableName('outbox', options)} AS outbox
20280
+ SET next_attempt_at = retry.retry_at
20281
+ FROM retry
20282
+ WHERE outbox.event_id = ANY($1::text[])
20283
+ AND outbox.delivered_at IS NULL AND outbox.failed_at IS NULL
20284
+ AND retry.retry_at IS NOT NULL
20285
+ `,
20286
+ [ownedRetrying],
20287
+ );
20288
+ }
20289
+ await client.query('COMMIT');
20290
+ return { blocked, retrying };
20291
+ } catch (settlementError) {
20292
+ await client.query('ROLLBACK');
20293
+ throw settlementError;
19669
20294
  }
19670
- return { blocked, retrying };
19671
20295
  });
19672
20296
  console.warn('[run-ledger-outbox.batch_dispatch_failed]', {
19673
20297
  runId: input.runId ?? null,
@@ -19784,6 +20408,7 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
19784
20408
  runtimeApi: WorkerRuntimeApiContext;
19785
20409
  scope?: RunLedgerOutboxDrainScope;
19786
20410
  continuation: { current: RunLedgerOutboxDrainScope | null };
20411
+ continuationAllowance: number | null;
19787
20412
  },
19788
20413
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
19789
20414
  ): Promise<{
@@ -19824,6 +20449,7 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
19824
20449
  let continuation: RunLedgerOutboxDrainScope | null = null;
19825
20450
  if (
19826
20451
  retainContinuation &&
20452
+ input.continuationAllowance !== 0 &&
19827
20453
  deliveredRows.length === input.events.length &&
19828
20454
  !input.events.some((event) => event.eventId === targetEventId)
19829
20455
  ) {
@@ -19841,8 +20467,14 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
19841
20467
  leaseSeconds: input.input.leaseSeconds ?? 120,
19842
20468
  limit: Math.min(
19843
20469
  RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
19844
- input.input.limit ?? 1,
20470
+ input.runtimeApi.runLedgerIngressPrefixBatching === true
20471
+ ? RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS
20472
+ : (input.input.limit ?? 1),
20473
+ input.continuationAllowance ??
20474
+ RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS,
19845
20475
  ),
20476
+ allowIngressPrefix:
20477
+ input.runtimeApi.runLedgerIngressPrefixBatching === true,
19846
20478
  inTransaction: true,
19847
20479
  },
19848
20480
  options,
@@ -20019,38 +20651,7 @@ async function projectPostgresSchedulerStartEvent(
20019
20651
  });
20020
20652
  }
20021
20653
 
20022
- export type RunLedgerDispatchLaunch = Pick<
20023
- PlaySchedulerSubmitInput,
20024
- | 'runId'
20025
- | 'playId'
20026
- | 'playName'
20027
- | 'executorToken'
20028
- | 'baseUrl'
20029
- | 'runtimeApiBaseUrl'
20030
- | 'runtimeTokenOrigin'
20031
- | 'orgId'
20032
- | 'runtimeAuthority'
20033
- | 'vercelProtectionBypassToken'
20034
- >;
20035
-
20036
- /** Authority reads never transfer customer input, artifacts or execution graphs. */
20037
- function runLedgerDispatchLaunchSql(alias: string): string {
20038
- const fields: (keyof RunLedgerDispatchLaunch)[] = [
20039
- 'runId',
20040
- 'playId',
20041
- 'playName',
20042
- 'executorToken',
20043
- 'baseUrl',
20044
- 'runtimeApiBaseUrl',
20045
- 'runtimeTokenOrigin',
20046
- 'orgId',
20047
- 'runtimeAuthority',
20048
- 'vercelProtectionBypassToken',
20049
- ];
20050
- return `jsonb_build_object(${fields
20051
- .map((field) => `'${field}', ${alias}.launch_json->'${field}'`)
20052
- .join(', ')})`;
20053
- }
20654
+ export type { RunLedgerDispatchLaunch } from './run-ledger-dispatch-authority';
20054
20655
 
20055
20656
  async function loadPostgresSchedulerRuntimeApiForRun(
20056
20657
  withClient: <T>(
@@ -20061,28 +20662,37 @@ async function loadPostgresSchedulerRuntimeApiForRun(
20061
20662
  tokenAuthority: RunLedgerOutboxTokenAuthority,
20062
20663
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
20063
20664
  signal?: AbortSignal,
20064
- ): Promise<WorkerRuntimeApiContext> {
20665
+ claimedLaunch?: RunLedgerDispatchLaunch,
20666
+ ): Promise<
20667
+ WorkerRuntimeApiContext & {
20668
+ refreshExecutorToken?: () => Promise<string>;
20669
+ invalidateExecutorToken?: () => void;
20670
+ }
20671
+ > {
20065
20672
  // Release the scarce scheduler checkout before the token authority hop.
20066
20673
  // A slow gateway must not retain either projector pool client, and its
20067
20674
  // launch context must not hydrate customer execution graphs.
20068
- const result = await withClient((client) =>
20069
- client.query<{ launch_json: unknown }>(
20070
- `SELECT ${runLedgerDispatchLaunchSql('runs')} AS launch_json
20071
- FROM ${tableName('runs', options)} AS runs
20072
- WHERE run_id = $1
20073
- LIMIT 1`,
20074
- [runId],
20075
- ),
20076
- );
20077
- const launch: RunLedgerDispatchLaunch = parseLaunchJson(
20078
- result.rows[0]?.launch_json,
20079
- );
20675
+ const launch: RunLedgerDispatchLaunch =
20676
+ claimedLaunch ??
20677
+ (await withClient(async (client) => {
20678
+ const result = await client.query<{ launch_json: unknown }>(
20679
+ `SELECT ${runLedgerDispatchLaunchSql('runs')} AS launch_json
20680
+ FROM ${tableName('runs', options)} AS runs
20681
+ WHERE run_id = $1
20682
+ LIMIT 1`,
20683
+ [runId],
20684
+ );
20685
+ return parseLaunchJson(result.rows[0]?.launch_json);
20686
+ }));
20080
20687
  if (!launch?.baseUrl) {
20081
20688
  throw new Error(
20082
20689
  `Run-ledger outbox dispatcher cannot resolve runtime API context for ${runId}.`,
20083
20690
  );
20084
20691
  }
20085
20692
  let executorToken: string;
20693
+ let runLedgerIngressPrefixBatching: true | undefined;
20694
+ let refreshExecutorToken: (() => Promise<string>) | undefined;
20695
+ let invalidateExecutorToken: (() => void) | undefined;
20086
20696
  if (tokenAuthority === 'local_mint') {
20087
20697
  // App-side authoritative projector: sign in-process. NEVER reached on the
20088
20698
  // Fly worker (it passes `gateway`), so this is the only place `BETTER_AUTH_
@@ -20101,34 +20711,89 @@ async function loadPostgresSchedulerRuntimeApiForRun(
20101
20711
  `Run-ledger outbox gateway dispatch for ${runId} requires an active outbox lease.`,
20102
20712
  );
20103
20713
  }
20104
- executorToken = await requestRunLedgerDispatchToken({
20714
+ const dispatchRequest = {
20105
20715
  launch,
20106
20716
  eventId: dispatchLease.eventId,
20107
20717
  leaseOwner: dispatchLease.leaseOwner,
20108
20718
  schedulerSchema: options?.schema ?? null,
20109
20719
  signal,
20110
- });
20720
+ };
20721
+ const dispatchAuthority =
20722
+ await requestRunLedgerDispatchToken(dispatchRequest);
20723
+ invalidateExecutorToken = () =>
20724
+ runLedgerDispatchTokens.delete(dispatchAuthority.cacheKey);
20725
+ if (dispatchAuthority.reused) {
20726
+ refreshExecutorToken = async () =>
20727
+ (await requestRunLedgerDispatchToken(dispatchRequest, true))
20728
+ .executorToken;
20729
+ }
20730
+ executorToken = dispatchAuthority.executorToken;
20731
+ runLedgerIngressPrefixBatching =
20732
+ dispatchAuthority.runLedgerIngressPrefixBatching;
20111
20733
  }
20112
20734
  return {
20113
20735
  baseUrl: runtimeApiBaseUrlFromLaunch(launch),
20114
20736
  executorToken,
20737
+ ...(runLedgerIngressPrefixBatching
20738
+ ? { runLedgerIngressPrefixBatching }
20739
+ : {}),
20115
20740
  vercelProtectionBypassToken: launch.vercelProtectionBypassToken ?? null,
20741
+ ...(refreshExecutorToken ? { refreshExecutorToken } : {}),
20742
+ ...(invalidateExecutorToken ? { invalidateExecutorToken } : {}),
20116
20743
  };
20117
20744
  }
20118
20745
 
20119
- async function requestRunLedgerDispatchToken(input: {
20120
- launch: RunLedgerDispatchLaunch;
20121
- eventId: string;
20122
- leaseOwner: string;
20123
- schedulerSchema: string | null;
20124
- signal?: AbortSignal;
20125
- }): Promise<string> {
20746
+ // Only freshly gateway-issued legacy run.progress authority is reused.
20747
+ // Prefix advertisements are reacquired for each drain because they select its
20748
+ // receipt shape, including recovery across receiver capability changes. This is not
20749
+ // a lease cache: every delivery still claims, renews, and proves its own lease
20750
+ // at append. Include the complete pinned authority/routing and internal-token
20751
+ // fingerprint so a different run, schema, origin, or authority never shares it.
20752
+ const RUN_LEDGER_DISPATCH_TOKEN_CACHE_MS = 30_000;
20753
+ const RUN_LEDGER_DISPATCH_TOKEN_CACHE_ENTRIES = 256;
20754
+ const runLedgerDispatchTokens = new Map<
20755
+ string,
20756
+ { token: string; expiresAt: number; runLedgerIngressPrefixBatching?: true }
20757
+ >();
20758
+
20759
+ async function requestRunLedgerDispatchToken(
20760
+ input: {
20761
+ launch: RunLedgerDispatchLaunch;
20762
+ eventId: string;
20763
+ leaseOwner: string;
20764
+ schedulerSchema: string | null;
20765
+ signal?: AbortSignal;
20766
+ },
20767
+ forceFresh = false,
20768
+ ): Promise<{
20769
+ executorToken: string;
20770
+ reused: boolean;
20771
+ cacheKey: string;
20772
+ runLedgerIngressPrefixBatching?: true;
20773
+ }> {
20126
20774
  const internalToken = process.env.DEEPLINE_INTERNAL_TOKEN?.trim();
20127
20775
  if (!internalToken || !input.eventId || !input.leaseOwner) {
20128
20776
  throw new Error(
20129
20777
  'Run-ledger outbox recovery requires a valid dispatch lease and DEEPLINE_INTERNAL_TOKEN.',
20130
20778
  );
20131
20779
  }
20780
+ input.signal?.throwIfAborted();
20781
+ const { executorToken: _storedToken, ...authority } = input.launch;
20782
+ const cacheKey = createHash('sha256')
20783
+ .update(JSON.stringify([input.schedulerSchema, authority, internalToken]))
20784
+ .digest('hex');
20785
+ const now = Date.now();
20786
+ for (const [key, entry] of runLedgerDispatchTokens) {
20787
+ if (entry.expiresAt <= now) runLedgerDispatchTokens.delete(key);
20788
+ }
20789
+ const cached = runLedgerDispatchTokens.get(cacheKey);
20790
+ if (!forceFresh && cached)
20791
+ return {
20792
+ executorToken: cached.token,
20793
+ reused: true,
20794
+ cacheKey,
20795
+ };
20796
+ runLedgerDispatchTokens.delete(cacheKey);
20132
20797
  const url = `${runtimeApiBaseUrlFromLaunch(input.launch).replace(/\/$/, '')}/api/v2/internal/play-runtime/run-ledger-dispatch-token`;
20133
20798
  const vercelProtectionHeaders = await vercelProtectionBypassHeaders({
20134
20799
  baseUrl: runtimeApiBaseUrlFromLaunch(input.launch),
@@ -20164,6 +20829,7 @@ async function requestRunLedgerDispatchToken(input: {
20164
20829
  try {
20165
20830
  return JSON.parse(responseText) as {
20166
20831
  executorToken?: unknown;
20832
+ runLedgerIngressPrefixBatching?: unknown;
20167
20833
  error?: unknown;
20168
20834
  code?: unknown;
20169
20835
  retryable?: unknown;
@@ -20196,7 +20862,53 @@ async function requestRunLedgerDispatchToken(input: {
20196
20862
  : 'invalid response',
20197
20863
  });
20198
20864
  }
20199
- return body.executorToken;
20865
+ // Decode only the lifetime, never an authentication decision: the app
20866
+ // validates the signature, run scope, and live delivery lease on append.
20867
+ // Opaque/older token formats continue to work without this optimization.
20868
+ try {
20869
+ const claims = JSON.parse(
20870
+ Buffer.from(
20871
+ body.executorToken.split('.')[1] ?? '',
20872
+ 'base64url',
20873
+ ).toString(),
20874
+ ) as { exp?: unknown };
20875
+ if (
20876
+ body.runLedgerIngressPrefixBatching === undefined &&
20877
+ typeof claims.exp === 'number' &&
20878
+ Number.isFinite(claims.exp)
20879
+ ) {
20880
+ const expiresAt = Math.min(
20881
+ Date.now() + RUN_LEDGER_DISPATCH_TOKEN_CACHE_MS,
20882
+ claims.exp * 1_000 - 5_000,
20883
+ );
20884
+ if (expiresAt > Date.now()) {
20885
+ runLedgerDispatchTokens.set(cacheKey, {
20886
+ token: body.executorToken,
20887
+ expiresAt,
20888
+ });
20889
+ while (
20890
+ runLedgerDispatchTokens.size > RUN_LEDGER_DISPATCH_TOKEN_CACHE_ENTRIES
20891
+ ) {
20892
+ runLedgerDispatchTokens.delete(
20893
+ runLedgerDispatchTokens.keys().next().value!,
20894
+ );
20895
+ }
20896
+ }
20897
+ }
20898
+ } catch {
20899
+ // A cache miss never changes delivery or hides a gateway rejection.
20900
+ }
20901
+ if (body.runLedgerIngressPrefixBatching !== undefined) {
20902
+ runLedgerDispatchTokens.delete(cacheKey);
20903
+ }
20904
+ return {
20905
+ executorToken: body.executorToken,
20906
+ reused: false,
20907
+ cacheKey,
20908
+ ...(body.runLedgerIngressPrefixBatching === true
20909
+ ? { runLedgerIngressPrefixBatching: true as const }
20910
+ : {}),
20911
+ };
20200
20912
  }
20201
20913
 
20202
20914
  export async function requestComputeBillingDispatchToken(input: {
@@ -20272,27 +20984,7 @@ export async function requestComputeBillingDispatchToken(input: {
20272
20984
  return body.executorToken;
20273
20985
  }
20274
20986
 
20275
- /** Read only the immutable authority/routing fields under the same lease fence. */
20276
- export async function readRunLedgerDispatchAuthorityForLease(
20277
- client: PostgresSchedulerQueryClient,
20278
- input: { runId: string; eventId: string; leaseOwner: string },
20279
- options?: Pick<PostgresSchedulerOptions, 'schema'>,
20280
- ): Promise<RunLedgerDispatchLaunch | null> {
20281
- const result = await client.query<{ launch_json: unknown }>(
20282
- `SELECT ${runLedgerDispatchLaunchSql('runs')} AS launch_json
20283
- FROM ${tableName('outbox', options)} outbox
20284
- JOIN ${tableName('runs', options)} runs ON runs.run_id = outbox.run_id
20285
- WHERE outbox.event_id = $1
20286
- AND outbox.run_id = $2
20287
- AND outbox.lease_owner = $3
20288
- AND outbox.lease_expires_at > now()
20289
- AND outbox.delivered_at IS NULL
20290
- AND outbox.failed_at IS NULL
20291
- LIMIT 1`,
20292
- [input.eventId, input.runId, input.leaseOwner],
20293
- );
20294
- return result.rows[0] ? parseLaunchJson(result.rows[0].launch_json) : null;
20295
- }
20987
+ export { readRunLedgerDispatchAuthorityForLease } from './run-ledger-dispatch-authority';
20296
20988
 
20297
20989
  export async function readRunLedgerDispatchLaunchForLease(
20298
20990
  client: PostgresSchedulerQueryClient,
@@ -20415,22 +21107,10 @@ function sourcePayloadField(
20415
21107
  }
20416
21108
 
20417
21109
  function parseLaunchJson(value: unknown): PlaySchedulerSubmitInput {
20418
- if (!value || typeof value !== 'object' || Array.isArray(value)) {
20419
- throw new Error('Postgres scheduler launch_json is not an object.');
20420
- }
20421
- const launch = value as Partial<PlaySchedulerSubmitInput>;
20422
- if (
20423
- typeof launch.runId !== 'string' ||
20424
- typeof launch.playId !== 'string' ||
20425
- typeof launch.playName !== 'string' ||
20426
- typeof launch.executorToken !== 'string' ||
20427
- typeof launch.baseUrl !== 'string'
20428
- ) {
20429
- throw new Error(
20430
- 'Postgres scheduler launch_json is missing required fields.',
20431
- );
20432
- }
20433
- return hydrateRetainedPlayPipelines(launch as PlaySchedulerSubmitInput);
21110
+ const launch = parseRunLedgerDispatchLaunch(
21111
+ value,
21112
+ ) as PlaySchedulerSubmitInput;
21113
+ return hydrateRetainedPlayPipelines(launch);
20434
21114
  }
20435
21115
 
20436
21116
  function parsePostgresSchedulerCronSourcePin(