deepline 0.3.315 → 0.3.317

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  2. package/dist/bundling-sources/sdk/src/runs/overview.ts +17 -10
  3. package/dist/bundling-sources/sdk/src/types.ts +6 -5
  4. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +64 -0
  5. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +217 -488
  6. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +15 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +4 -1
  8. package/dist/bundling-sources/shared_libs/play-runtime/internal-bootstrap-token.ts +13 -4
  9. package/dist/bundling-sources/shared_libs/play-runtime/launch-identity.ts +1 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/live-events.ts +31 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +444 -51
  12. package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +22 -43
  13. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +4 -3
  14. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +136 -87
  15. package/dist/bundling-sources/shared_libs/play-runtime/runtime-authority.ts +24 -6
  16. package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +56 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts +94 -0
  18. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +5 -2
  20. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts +20 -11
  21. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +18 -12
  22. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +396 -35
  23. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-dispatch-authority.ts +2 -0
  24. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +123 -19
  25. package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-storage-contract.ts +13 -0
  26. package/dist/bundling-sources/shared_libs/play-runtime/workflow-executor-auth.ts +2 -0
  27. package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +3 -2
  28. package/dist/cli/auth-XPLGQ2KO.js +31 -0
  29. package/dist/cli/{auth-OVWOO5NP.mjs → auth-YAHBE5WU.mjs} +4 -4
  30. package/dist/cli/{billing-W3OXKWNE.mjs → billing-AY4OS7MR.mjs} +4 -4
  31. package/dist/cli/{billing-SRZL7UOH.js → billing-ORZDEMH2.js} +70 -70
  32. package/dist/cli/{chunk-OJP4FCCX.js → chunk-2FKXFTOF.js} +48 -48
  33. package/dist/cli/{chunk-OJ243ZA5.js → chunk-2HKPAZVJ.js} +1 -1
  34. package/dist/cli/{chunk-LOWURQMF.js → chunk-342BTES3.js} +50 -50
  35. package/dist/cli/{chunk-FGAK6FET.js → chunk-3NRNDPAE.js} +26 -26
  36. package/dist/cli/{chunk-GK67FYMS.mjs → chunk-5BLF3HDM.mjs} +5 -5
  37. package/dist/cli/{chunk-TZ5KUDPP.mjs → chunk-5OFRM64Y.mjs} +1 -1
  38. package/dist/cli/{chunk-US4S3E4Y.mjs → chunk-7PZFGB7L.mjs} +1 -1
  39. package/dist/cli/{chunk-YL42TF53.js → chunk-AMJYFPFK.js} +6 -6
  40. package/dist/cli/{chunk-PPONZ6IE.js → chunk-AW355CLN.js} +15 -15
  41. package/dist/cli/{chunk-TQK662WG.js → chunk-BGBYK2T3.js} +60 -60
  42. package/dist/cli/{chunk-TQ3F77AC.mjs → chunk-CGGM4CE6.mjs} +3 -3
  43. package/dist/cli/{chunk-FTJQO757.mjs → chunk-D5MHSO5T.mjs} +1 -1
  44. package/dist/cli/{chunk-Y42UERUY.js → chunk-DSTUP75H.js} +6 -6
  45. package/dist/cli/{chunk-HNKHMHJ7.mjs → chunk-FRK2AGSG.mjs} +1 -1
  46. package/dist/cli/{chunk-OUIIYFG3.mjs → chunk-GX3WO4M4.mjs} +2 -2
  47. package/dist/cli/{chunk-OYJCNCUW.js → chunk-GYOLWON6.js} +2 -2
  48. package/dist/cli/{chunk-N5V2SBB2.mjs → chunk-KP7W5NO6.mjs} +1 -1
  49. package/dist/cli/{chunk-L5TSFCJ6.mjs → chunk-KXOXXVYJ.mjs} +2 -2
  50. package/dist/cli/{chunk-B473QUQU.mjs → chunk-L6ERGROE.mjs} +4 -4
  51. package/dist/cli/{chunk-R3IFZDJL.mjs → chunk-LI6YPPBR.mjs} +2 -2
  52. package/dist/cli/{chunk-IGE7EJJY.mjs → chunk-NDFBAMVI.mjs} +19 -15
  53. package/dist/cli/{chunk-JF4O5OMH.mjs → chunk-OTI562P3.mjs} +1 -1
  54. package/dist/cli/{chunk-AO4IWPP7.mjs → chunk-PSHIZP3K.mjs} +17 -15
  55. package/dist/cli/{chunk-VYK23G2C.js → chunk-QFKVAUIQ.js} +239 -235
  56. package/dist/cli/{chunk-AZWQJAQF.mjs → chunk-QPKHOZI3.mjs} +1 -1
  57. package/dist/cli/{chunk-URVZGYGG.js → chunk-SUPQQYGY.js} +4 -4
  58. package/dist/cli/{chunk-C5WSTXUB.js → chunk-TJ2V2JA3.js} +537 -535
  59. package/dist/cli/{chunk-6BJACIVK.js → chunk-UGQHYOT4.js} +1 -1
  60. package/dist/cli/{chunk-IALPYID6.js → chunk-XSEMQ4II.js} +3 -3
  61. package/dist/cli/{chunk-SNFSO2AN.js → chunk-Z5NRL7A7.js} +6 -6
  62. package/dist/cli/{client-WQ3J5BSH.mjs → client-QNLVLW54.mjs} +3 -3
  63. package/dist/cli/client-ZW5VAHWD.js +20 -0
  64. package/dist/cli/{csv-36ARCBNQ.mjs → csv-AKZASZP6.mjs} +3 -3
  65. package/dist/cli/{csv-CENMEWDH.js → csv-PW6S3SMJ.js} +12 -12
  66. package/dist/cli/{db-JU7XLKQM.mjs → db-DNF53LDS.mjs} +4 -4
  67. package/dist/cli/{db-6DQZG6NB.js → db-FLDYET2D.js} +20 -20
  68. package/dist/cli/{describe-VUD4AFBY.js → describe-JDAEOCYQ.js} +17 -17
  69. package/dist/cli/{describe-34ALEEFO.mjs → describe-U6RGIFMA.mjs} +10 -10
  70. package/dist/cli/{executions-H2UXS6FP.js → executions-DASCB6LS.js} +6 -6
  71. package/dist/cli/{executions-H66BMN6N.mjs → executions-W6BBU2HN.mjs} +4 -4
  72. package/dist/cli/{feedback-Q7LXXLSH.mjs → feedback-BVWPDWZG.mjs} +3 -3
  73. package/dist/cli/{feedback-YSYFQSAB.js → feedback-N3MBCT4A.js} +6 -6
  74. package/dist/cli/{git-XV43NSWN.mjs → git-4GJOKOLU.mjs} +4 -4
  75. package/dist/cli/git-WVHH3QXQ.js +23 -0
  76. package/dist/cli/index.js +171 -98
  77. package/dist/cli/index.mjs +114 -41
  78. package/dist/cli/{legacy-noops-Z2DTZQN3.mjs → legacy-noops-5XMSV7WL.mjs} +4 -4
  79. package/dist/cli/{legacy-noops-RGDW6SCA.js → legacy-noops-D7C7ZVKK.js} +7 -7
  80. package/dist/cli/{monitors-I75KZ2BP.js → monitors-NYP4LG3K.js} +85 -85
  81. package/dist/cli/{monitors-OFQ56O4S.mjs → monitors-Y4RKWAS5.mjs} +4 -4
  82. package/dist/cli/{org-75AH6RIB.js → org-PYKQMP6F.js} +60 -60
  83. package/dist/cli/{org-ZYMCU6XO.mjs → org-XZ26CUCV.mjs} +4 -4
  84. package/dist/cli/play-BYZCR4PP.js +152 -0
  85. package/dist/cli/{play-BKXM76DV.mjs → play-FYHIYZ3W.mjs} +9 -9
  86. package/dist/cli/{providers-GWG2WFIY.mjs → providers-3UG7NK55.mjs} +4 -4
  87. package/dist/cli/{providers-Z77MM7ID.js → providers-QS5X236G.js} +7 -7
  88. package/dist/cli/{quickstart-I4RECIQC.js → quickstart-GMASUT7T.js} +5 -5
  89. package/dist/cli/{quickstart-BGVIV374.mjs → quickstart-PHHHHHWO.mjs} +3 -3
  90. package/dist/cli/search-52CC3OVI.js +26 -0
  91. package/dist/cli/{search-TIKBP3RE.mjs → search-AVGITJP4.mjs} +5 -5
  92. package/dist/cli/{secrets-SNGRI4KS.js → secrets-HY5E6F6G.js} +13 -13
  93. package/dist/cli/{secrets-YCHKLWKC.mjs → secrets-RBEZRQ6P.mjs} +4 -4
  94. package/dist/cli/{sessions-WXLADGDX.js → sessions-7CDLWLVV.js} +5 -5
  95. package/dist/cli/{sessions-IRDG6ZRX.mjs → sessions-ME5R5FTG.mjs} +4 -4
  96. package/dist/cli/{settings-KZALJDXW.mjs → settings-X4JQLBB6.mjs} +7 -7
  97. package/dist/cli/{settings-72J4BVF6.js → settings-ZCVJPOFP.js} +74 -74
  98. package/dist/cli/{setup-FFB3KLFD.js → setup-DROGU6MC.js} +41 -41
  99. package/dist/cli/{setup-4KCA7ESV.mjs → setup-NEILS2UF.mjs} +6 -6
  100. package/dist/cli/skills-KUV3WWW5.js +28 -0
  101. package/dist/cli/{skills-MQZPYUQ3.mjs → skills-PC57GDXZ.mjs} +4 -4
  102. package/dist/cli/{tools-M3WKRZ5T.mjs → tools-IFJUWQC5.mjs} +6 -6
  103. package/dist/cli/tools-NHDAOQPB.js +38 -0
  104. package/dist/cli/update-DRWQCVXJ.js +28 -0
  105. package/dist/cli/{update-QJMNYGNF.mjs → update-N4HSH2TJ.mjs} +6 -6
  106. package/dist/cli/{workflow-6T2EUFCN.mjs → workflow-2OZBTUU2.mjs} +5 -5
  107. package/dist/cli/{workflow-35UUOZ5E.js → workflow-LP4Z6RF5.js} +15 -15
  108. package/dist/{client-Dh_4Ph1R.d.ts → client-BV68GMPo.d.ts} +7 -6
  109. package/dist/{client-CvpZsBqt.d.mts → client-D9ACjiRC.d.mts} +7 -6
  110. package/dist/{compiler-manifest-wOKNiXpa.d.mts → compiler-manifest-BXcZ4P4W.d.mts} +2 -2
  111. package/dist/{compiler-manifest-wOKNiXpa.d.ts → compiler-manifest-BXcZ4P4W.d.ts} +2 -2
  112. package/dist/index.d.mts +4 -4
  113. package/dist/index.d.ts +4 -4
  114. package/dist/index.js +7 -7
  115. package/dist/index.mjs +4 -4
  116. package/dist/install-integrity.json +82 -82
  117. package/dist/plays/bundle-play-file.d.mts +2 -2
  118. package/dist/plays/bundle-play-file.d.ts +2 -2
  119. package/dist/plays/bundle-play-file.mjs +2 -2
  120. package/dist/plays/run-request.d.mts +2 -2
  121. package/dist/plays/run-request.d.ts +2 -2
  122. package/dist/release.d.mts +1 -1
  123. package/dist/release.d.ts +1 -1
  124. package/dist/release.js +2 -2
  125. package/dist/release.mjs +1 -1
  126. package/package.json +1 -1
  127. package/dist/cli/auth-P2TTZU36.js +0 -31
  128. package/dist/cli/client-CXFPFCIG.js +0 -20
  129. package/dist/cli/git-SYFCYIFP.js +0 -23
  130. package/dist/cli/play-OKXUWPMO.js +0 -152
  131. package/dist/cli/search-3KISUDDT.js +0 -26
  132. package/dist/cli/skills-XGOK4GRW.js +0 -28
  133. package/dist/cli/tools-LEAJ7EPW.js +0 -38
  134. package/dist/cli/update-ZN6LDUHS.js +0 -28
@@ -1,4 +1,5 @@
1
1
  import { RUN_EVENT_KIND } from '../run-event-catalog';
2
+ import { normalizeTableNamespace } from '../../plays/row-identity';
2
3
  /**
3
4
  * Live step/progress ledger reporting for Node runtime workers.
4
5
  *
@@ -109,7 +110,7 @@ export function buildOrderedNodeList(
109
110
  }));
110
111
  }
111
112
 
112
- /** Map logical dataset namespaces to their static-pipeline node IDs. */
113
+ /** Keep authored node identity separate from the normalized Runtime Sheet namespace. */
113
114
  type StaticDatasetNodeRef = {
114
115
  nodeId: string;
115
116
  logicalNamespace: string;
@@ -129,10 +130,16 @@ function buildMapNodeLookup(
129
130
  for (let index = 0; index < substeps.length; index += 1) {
130
131
  const substep = substeps[index]!;
131
132
  if (substep.type !== 'dataset') continue;
132
- const logicalNamespace =
133
- substep.tableNamespace?.trim() || substep.field.trim();
133
+ const staticNamespace = substep.tableNamespace?.trim();
134
+ const logicalNamespace = staticNamespace || substep.field.trim();
134
135
  if (!logicalNamespace) continue;
135
- lookup.set(logicalNamespace, {
136
+ // Literal dataset keys share the Runtime Sheet namespace contract. A legacy
137
+ // field-only label can be source-expression text, so retain its existing
138
+ // lookup behavior rather than validating or guessing a physical identity.
139
+ const lookupNamespace = staticNamespace
140
+ ? normalizeTableNamespace(staticNamespace)
141
+ : logicalNamespace;
142
+ lookup.set(lookupNamespace, {
136
143
  nodeId: getStaticSubstepNodeId(substep, index),
137
144
  logicalNamespace,
138
145
  });
@@ -140,6 +147,7 @@ function buildMapNodeLookup(
140
147
  return lookup;
141
148
  }
142
149
 
150
+ /** Resolve normalized literal dataset namespaces to unchanged authored node IDs. */
143
151
  export function buildMapNodeIdLookup(
144
152
  contractSnapshot: unknown,
145
153
  ): Map<string, string> {
@@ -1219,23 +1227,21 @@ export function createPostgresSchedulerProgressReporter(input: {
1219
1227
  ) {
1220
1228
  mapsWithResolverActivity.delete(nodeId);
1221
1229
  stepLifecycle.onMapStarted(nodeId);
1230
+ const failedRows = 'failedRows' in event ? (event.failedRows ?? 0) : 0;
1231
+ const startedRows =
1232
+ event.completedRows + failedRows + (event.supersededRows ?? 0);
1222
1233
  setNodeProgress(nodeId, {
1223
1234
  completed: event.completedRows,
1224
- failed: 0,
1235
+ failed: failedRows,
1225
1236
  total: eventTotalRows,
1226
1237
  rowActivitySource: 'map_start',
1227
1238
  rowActivityStartedAt: event.at,
1228
1239
  rowActivityObservedAt: event.at,
1229
- startedRows: event.completedRows + (event.supersededRows ?? 0),
1240
+ startedRows,
1230
1241
  activeRows: 0,
1231
1242
  waitingRows:
1232
1243
  typeof eventTotalRows === 'number'
1233
- ? Math.max(
1234
- 0,
1235
- eventTotalRows -
1236
- event.completedRows -
1237
- (event.supersededRows ?? 0),
1238
- )
1244
+ ? Math.max(0, eventTotalRows - startedRows)
1239
1245
  : undefined,
1240
1246
  completedRows: event.completedRows,
1241
1247
  ...(event.supersededRows !== undefined
@@ -32,7 +32,10 @@ import type {
32
32
  PlayRunnerResult,
33
33
  PlayRunnerRuntimeTiming,
34
34
  } from '../protocol';
35
- import { PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA } from '../runtime-scheduler-topology';
35
+ import {
36
+ normalizeRuntimeSchedulerSchema,
37
+ PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA,
38
+ } from '../runtime-scheduler-topology';
36
39
  import {
37
40
  RUNTIME_EVENT_CANONICAL_RETENTION_INDEX_READY_MARKER,
38
41
  RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD,
@@ -158,7 +161,10 @@ import {
158
161
  loadArtifactFromAppRuntime,
159
162
  loadStagedFileFromAppRuntime,
160
163
  projectRunCreatedViaAppRuntime,
164
+ readRuntimeApiDiagnosticHeader,
161
165
  startRunViaAppRuntime,
166
+ type RuntimeApiRequestAttempt,
167
+ type RuntimeApiResponseMeta,
162
168
  type StoredPlayArtifactPayload,
163
169
  type WorkerRuntimeApiContext,
164
170
  } from '../app-runtime-api';
@@ -19720,6 +19726,14 @@ export type RunLedgerOutboxDispatchInput = {
19720
19726
  * can take the same row immediately under the same idempotency key.
19721
19727
  */
19722
19728
  signal?: AbortSignal;
19729
+ /**
19730
+ * Told once when an attempt abandoned by `signal` has handed back the lease
19731
+ * it took: `true` when the owner-fenced release statement completed (a
19732
+ * release that matched no row counts, since nothing is then held), `false`
19733
+ * when it could not run and the lease is left to expire. Not called when
19734
+ * the attempt never took a lease. Never affects delivery.
19735
+ */
19736
+ onAbortedLeaseRelease?: (released: boolean) => void;
19723
19737
  /**
19724
19738
  * Observation hook for acknowledged rows (event type and commit-to-ack lag
19725
19739
  * measured on the database clock). Never affects delivery.
@@ -20076,7 +20090,45 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20076
20090
  });
20077
20091
  }
20078
20092
  };
20093
+ // The claim can be invoked more than once for one dispatch: the pool
20094
+ // re-runs a checkout's operation after a connection failure, and the claim
20095
+ // itself is retried once on the missing-marker compatibility path.
20096
+ let claimInvocations = 0;
20097
+ // An abandoned attempt that already handed its lease back on a checkout it
20098
+ // held does not spend another checkout on the same release.
20099
+ const abandonedLease = { released: false };
20079
20100
  const claimEvents = async (client: PostgresSchedulerQueryClient) => {
20101
+ claimInvocations += 1;
20102
+ const abandonedHeadEventId = input.headEventId?.trim();
20103
+ if (
20104
+ input.signal?.aborted &&
20105
+ (claimInvocations === 1 || abandonedHeadEventId)
20106
+ ) {
20107
+ // The caller's deadline elapsed while this checkout was queued. Its
20108
+ // attempt is already abandoned, so taking a lease now could only keep
20109
+ // the durable projector off the row until that lease expires. Claim
20110
+ // nothing: the row stays pending and unleased for its next owner.
20111
+ if (claimInvocations > 1 && abandonedHeadEventId) {
20112
+ // An earlier invocation's COMMIT may have reached the server before
20113
+ // its connection failed, so this owner can already hold the named
20114
+ // head. Hand it back here, fenced to this owner; returning empty
20115
+ // would leave that lease with nobody to release it. After a rolled
20116
+ // back invocation it matches no row. A failure propagates, so the
20117
+ // attempt is never reported as settled.
20118
+ await releasePostgresSchedulerOutboxLease(
20119
+ client,
20120
+ [abandonedHeadEventId],
20121
+ { ...options, leaseOwner },
20122
+ );
20123
+ }
20124
+ const unclaimed: {
20125
+ events: PostgresSchedulerOutboxEvent[];
20126
+ launchJson: unknown;
20127
+ } = { events: [], launchJson: undefined };
20128
+ return unclaimed;
20129
+ }
20130
+ // A re-invocation with no named head claims again under the same owner,
20131
+ // as it always has; the abort check after the claim releases by id.
20080
20132
  // The checkout is exclusive. Hold the existing scheduler run's row lock
20081
20133
  // only while choosing/claiming its outbox prefix, never over HTTP. A
20082
20134
  // second statement takes a fresh READ COMMITTED snapshot after the lock:
@@ -20271,6 +20323,26 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20271
20323
  let authorityReadyAtMs: number | null = null;
20272
20324
  let appendFinishedAtMs: number | null = null;
20273
20325
  let acknowledgedAtMs: number | null = null;
20326
+ // How this delivery obtained its app authority, for the slow-delivery
20327
+ // record only. `token` is set only when the dispatch-token endpoint was
20328
+ // actually consulted; a token pre-minted at admission and an authority
20329
+ // reused within the drain make no such request.
20330
+ const authorityObserved: {
20331
+ token: RunLedgerDispatchTokenObservation | null;
20332
+ reusedWithinDrain: boolean;
20333
+ } = { token: null, reusedWithinDrain: false };
20334
+ // Platform identity of the slowest append response, for the same record.
20335
+ // Null when no append was answered or its receiver sent no such header.
20336
+ let appendResponse:
20337
+ | (RunLedgerDispatchResponseTrace & { roundTripMs: number | null })
20338
+ | null = null;
20339
+ const callerOnResponseMeta = input.runtimeApi?.onResponseMeta;
20340
+ // The last append attempt sent, named even when it never gets a response
20341
+ // (an append aborted at the caller's deadline has no response to read).
20342
+ const appendAttempt: { transportAttemptId: string | null } = {
20343
+ transportAttemptId: null,
20344
+ };
20345
+ const callerOnRequestAttempt = input.runtimeApi?.onRequestAttempt;
20274
20346
  const materializeDeferredObservability = async () => {
20275
20347
  if (
20276
20348
  events.some(
@@ -20398,6 +20470,10 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20398
20470
  return null;
20399
20471
  };
20400
20472
  try {
20473
+ // A deadline that elapsed during the claim abandons the attempt before
20474
+ // any authority or renewal work. The abort branch below hands the lease
20475
+ // straight back without charging a projection attempt.
20476
+ input.signal?.throwIfAborted();
20401
20477
  // The ledger converter rejects raw deferred facts. Inline callers hand
20402
20478
  // them off; the background owner projects them before inspecting the
20403
20479
  // compact lifecycle payload or deciding that it needs no HTTP append.
@@ -20469,6 +20545,11 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20469
20545
  ),
20470
20546
  ) + RUNTIME_RELIABILITY_POLICY.schedulerDatabase.queryTimeoutMs;
20471
20547
  const cachedAuthority = authorityScope.authority;
20548
+ const observeDispatchTokenResponse = (
20549
+ observation: RunLedgerDispatchTokenObservation,
20550
+ ) => {
20551
+ authorityObserved.token = observation;
20552
+ };
20472
20553
  const resolveRunRuntimeApi = () =>
20473
20554
  launchJson !== null
20474
20555
  ? resolvePostgresSchedulerRuntimeApiForRun(
@@ -20480,6 +20561,8 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20480
20561
  tokenAuthority,
20481
20562
  options,
20482
20563
  input.runtimeApi?.signal,
20564
+ reuseMarginMs,
20565
+ observeDispatchTokenResponse,
20483
20566
  )
20484
20567
  : loadPostgresSchedulerRuntimeApiForRun(
20485
20568
  withClient,
@@ -20490,6 +20573,8 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20490
20573
  tokenAuthority,
20491
20574
  options,
20492
20575
  input.runtimeApi?.signal,
20576
+ reuseMarginMs,
20577
+ observeDispatchTokenResponse,
20493
20578
  );
20494
20579
  const gatewayRuntimeApi =
20495
20580
  needsRuntimeApi && tokenAuthority === 'gateway'
@@ -20500,6 +20585,11 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20500
20585
  ? cachedAuthority.runtimeApi
20501
20586
  : await resolveRunRuntimeApi()
20502
20587
  : null;
20588
+ // Read-only: an authority carried over from an earlier prefix of this
20589
+ // drain made no request for this delivery.
20590
+ authorityObserved.reusedWithinDrain =
20591
+ gatewayRuntimeApi !== null &&
20592
+ gatewayRuntimeApi === cachedAuthority?.runtimeApi;
20503
20593
  if (
20504
20594
  gatewayRuntimeApi &&
20505
20595
  cachedAuthority?.runId === first.runId &&
@@ -20537,6 +20627,27 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20537
20627
  : {}),
20538
20628
  }
20539
20629
  : {}),
20630
+ // Keep the slowest append response's platform identity for the
20631
+ // slow-delivery record, then hand the caller's own observer the same
20632
+ // meta it always received. Observation never affects delivery.
20633
+ onResponseMeta: (meta: RuntimeApiResponseMeta) => {
20634
+ const roundTripMs = meta.roundTripMs ?? null;
20635
+ if (
20636
+ appendResponse === null ||
20637
+ (roundTripMs ?? 0) >= (appendResponse.roundTripMs ?? 0)
20638
+ ) {
20639
+ appendResponse = {
20640
+ platformRequestId: meta.platformRequestId ?? null,
20641
+ serverTimingHeader: meta.serverTimingHeader ?? null,
20642
+ roundTripMs,
20643
+ };
20644
+ }
20645
+ callerOnResponseMeta?.(meta);
20646
+ },
20647
+ onRequestAttempt: (attempt: RuntimeApiRequestAttempt) => {
20648
+ appendAttempt.transportAttemptId = attempt.transportAttemptId;
20649
+ callerOnRequestAttempt?.(attempt);
20650
+ },
20540
20651
  retryPolicy: 'none' as const,
20541
20652
  // The caller-owned admission deadline (see `signal` above) wins.
20542
20653
  ...(input.signal ? { signal: input.signal } : {}),
@@ -20645,6 +20756,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20645
20756
  tokenAuthority,
20646
20757
  options,
20647
20758
  runtimeApi.signal,
20759
+ reuseMarginMs,
20648
20760
  );
20649
20761
  if (refreshed.baseUrl !== runtimeApi.baseUrl) {
20650
20762
  throw new Error(
@@ -20663,11 +20775,22 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20663
20775
  expiresAtMs: runtimeApiTokenExpiresAtMs(runtimeApi.executorToken),
20664
20776
  };
20665
20777
  }
20666
- await withClient((client) =>
20667
- renewPostgresRunLedgerDelivery(
20778
+ const renewed = await withClient(async (client) => {
20779
+ const eventIds = events.map((event) => event.eventId);
20780
+ if (input.signal?.aborted) {
20781
+ // The caller's deadline elapsed while this checkout was queued.
20782
+ // Renewing now would extend an abandoned attempt's lease by a full
20783
+ // term. Hand it back on this same checkout instead.
20784
+ await releasePostgresSchedulerOutboxLease(client, eventIds, {
20785
+ ...options,
20786
+ leaseOwner,
20787
+ });
20788
+ return false;
20789
+ }
20790
+ await renewPostgresRunLedgerDelivery(
20668
20791
  client,
20669
20792
  {
20670
- eventIds: events.map((event) => event.eventId),
20793
+ eventIds,
20671
20794
  leaseOwner,
20672
20795
  leaseSeconds: input.leaseSeconds ?? 120,
20673
20796
  freezeReceipt:
@@ -20678,8 +20801,14 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20678
20801
  ),
20679
20802
  },
20680
20803
  options,
20681
- ),
20682
- );
20804
+ );
20805
+ return true;
20806
+ });
20807
+ if (!renewed) {
20808
+ abandonedLease.released = true;
20809
+ // Stop before the append; the abort branch below reports the outcome.
20810
+ input.signal?.throwIfAborted();
20811
+ }
20683
20812
  };
20684
20813
  if (needsRuntimeApi && runtimeApi) {
20685
20814
  const append = async (runtimeApi: WorkerRuntimeApiContext) => {
@@ -20826,28 +20955,41 @@ async function dispatchRunLedgerOutboxBatchWithScope(
20826
20955
  // replay carries the same idempotency key and is a no-op there.
20827
20956
  const eventIds = events.map((event) => event.eventId);
20828
20957
  let released: string[];
20829
- try {
20830
- released = await withClient(
20831
- (client) =>
20832
- releasePostgresSchedulerOutboxLease(client, eventIds, {
20833
- ...options,
20834
- leaseOwner,
20835
- }),
20836
- {
20837
- runId: events[0]!.runId,
20838
- phase: 'run_ledger_outbox_lease_release',
20839
- priority: 'critical',
20840
- },
20841
- );
20842
- } catch (settlementError) {
20843
- logRunLedgerOutboxSettlementFailure({
20844
- runId: events[0]!.runId,
20845
- eventIds,
20846
- stage: 'aborted_release',
20847
- settlementError,
20848
- deliveryError: error,
20849
- });
20958
+ let releaseLanded = abandonedLease.released;
20959
+ if (releaseLanded) {
20960
+ // The renewal checkout already handed the lease back.
20850
20961
  released = eventIds;
20962
+ } else {
20963
+ try {
20964
+ released = await withClient(
20965
+ (client) =>
20966
+ releasePostgresSchedulerOutboxLease(client, eventIds, {
20967
+ ...options,
20968
+ leaseOwner,
20969
+ }),
20970
+ {
20971
+ runId: events[0]!.runId,
20972
+ phase: 'run_ledger_outbox_lease_release',
20973
+ priority: 'critical',
20974
+ },
20975
+ );
20976
+ releaseLanded = true;
20977
+ } catch (settlementError) {
20978
+ logRunLedgerOutboxSettlementFailure({
20979
+ runId: events[0]!.runId,
20980
+ eventIds,
20981
+ stage: 'aborted_release',
20982
+ settlementError,
20983
+ deliveryError: error,
20984
+ });
20985
+ released = eventIds;
20986
+ }
20987
+ }
20988
+ try {
20989
+ // The caller must not read a failed release as a settled lease.
20990
+ input.onAbortedLeaseRelease?.(releaseLanded);
20991
+ } catch {
20992
+ // Observation must never affect durable delivery.
20851
20993
  }
20852
20994
  return {
20853
20995
  attempted: events.length,
@@ -21030,6 +21172,19 @@ async function dispatchRunLedgerOutboxBatchWithScope(
21030
21172
  continuationMs:
21031
21173
  acknowledgedAtMs === null ? null : settledAtMs - acknowledgedAtMs,
21032
21174
  durationMs: settledAtMs - dispatchStartedAtMs,
21175
+ // Which platform requests served the two app hops, so a slow
21176
+ // authority or append stage can be found in the app's request log.
21177
+ ...runLedgerSlowDeliveryResponseFields(
21178
+ runLedgerDeliveryAuthoritySource({
21179
+ authorityReady: authorityReadyAtMs !== null,
21180
+ tokenAuthority,
21181
+ reusedWithinDrain: authorityObserved.reusedWithinDrain,
21182
+ token: authorityObserved.token,
21183
+ }),
21184
+ authorityObserved.token?.trace ?? null,
21185
+ appendResponse,
21186
+ appendAttempt.transportAttemptId,
21187
+ ),
21033
21188
  });
21034
21189
  }
21035
21190
  }
@@ -21373,6 +21528,10 @@ async function resolvePostgresSchedulerRuntimeApiForRun(
21373
21528
  tokenAuthority: RunLedgerOutboxTokenAuthority,
21374
21529
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
21375
21530
  signal?: AbortSignal,
21531
+ tokenReuseMarginMs = 0,
21532
+ onDispatchTokenResponse?: (
21533
+ observation: RunLedgerDispatchTokenObservation,
21534
+ ) => void,
21376
21535
  ): Promise<
21377
21536
  WorkerRuntimeApiContext & {
21378
21537
  refreshExecutorToken?: () => Promise<string>;
@@ -21414,18 +21573,45 @@ async function resolvePostgresSchedulerRuntimeApiForRun(
21414
21573
  schedulerSchema: options?.schema ?? null,
21415
21574
  signal,
21416
21575
  };
21417
- const dispatchAuthority =
21418
- await requestRunLedgerDispatchToken(dispatchRequest);
21419
- invalidateExecutorToken = () =>
21420
- runLedgerDispatchTokens.delete(dispatchAuthority.cacheKey);
21421
- if (dispatchAuthority.reused) {
21576
+ const preMintedToken = usableRunLedgerExecutorToken(
21577
+ launch,
21578
+ tokenReuseMarginMs,
21579
+ options,
21580
+ );
21581
+ if (preMintedToken) {
21582
+ // Admission minted this token with only `run.ledger.project`.
21583
+ // Each projection request still proves its exact outbox lease at the
21584
+ // receiver, and the scheduler ACK remains fenced to that same lease.
21585
+ executorToken = preMintedToken.executorToken;
21586
+ // The issuer advertises its own immutable receiver in the signed token.
21587
+ // Merely targeting an immutable URL never proves receipt support.
21588
+ runLedgerIngressPrefixBatching =
21589
+ preMintedToken.runLedgerIngressPrefixBatching;
21422
21590
  refreshExecutorToken = async () =>
21423
21591
  (await requestRunLedgerDispatchToken(dispatchRequest, true))
21424
21592
  .executorToken;
21593
+ } else {
21594
+ const dispatchAuthority =
21595
+ await requestRunLedgerDispatchToken(dispatchRequest);
21596
+ try {
21597
+ onDispatchTokenResponse?.({
21598
+ reused: dispatchAuthority.reused,
21599
+ trace: dispatchAuthority.responseTrace ?? null,
21600
+ });
21601
+ } catch {
21602
+ // Observation must never affect durable delivery.
21603
+ }
21604
+ invalidateExecutorToken = () =>
21605
+ runLedgerDispatchTokens.delete(dispatchAuthority.cacheKey);
21606
+ if (dispatchAuthority.reused) {
21607
+ refreshExecutorToken = async () =>
21608
+ (await requestRunLedgerDispatchToken(dispatchRequest, true))
21609
+ .executorToken;
21610
+ }
21611
+ executorToken = dispatchAuthority.executorToken;
21612
+ runLedgerIngressPrefixBatching =
21613
+ dispatchAuthority.runLedgerIngressPrefixBatching;
21425
21614
  }
21426
- executorToken = dispatchAuthority.executorToken;
21427
- runLedgerIngressPrefixBatching =
21428
- dispatchAuthority.runLedgerIngressPrefixBatching;
21429
21615
  }
21430
21616
  return {
21431
21617
  baseUrl: runtimeApiBaseUrlFromLaunch(launch),
@@ -21448,6 +21634,10 @@ async function loadPostgresSchedulerRuntimeApiForRun(
21448
21634
  tokenAuthority: RunLedgerOutboxTokenAuthority,
21449
21635
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
21450
21636
  signal?: AbortSignal,
21637
+ tokenReuseMarginMs = 0,
21638
+ onDispatchTokenResponse?: (
21639
+ observation: RunLedgerDispatchTokenObservation,
21640
+ ) => void,
21451
21641
  ): ReturnType<typeof resolvePostgresSchedulerRuntimeApiForRun> {
21452
21642
  // A continuation already owns its next event lease, so its launch context
21453
21643
  // is not part of the preclaim result. Load only the authority fields here.
@@ -21467,9 +21657,168 @@ async function loadPostgresSchedulerRuntimeApiForRun(
21467
21657
  tokenAuthority,
21468
21658
  options,
21469
21659
  signal,
21660
+ tokenReuseMarginMs,
21661
+ onDispatchTokenResponse,
21470
21662
  );
21471
21663
  }
21472
21664
 
21665
+ /**
21666
+ * Platform identity of one app response on the Run Ledger delivery path
21667
+ * (`x-vercel-id` and the raw `Server-Timing` header, both bounded). It exists
21668
+ * only to label the `[run-ledger-outbox.slow_delivery]` log record. It carries
21669
+ * no credential and no run, org, or customer identifier, and it must never be
21670
+ * written to a run, status, or receipt document.
21671
+ */
21672
+ type RunLedgerDispatchResponseTrace = {
21673
+ platformRequestId: string | null;
21674
+ serverTimingHeader: string | null;
21675
+ };
21676
+
21677
+ /**
21678
+ * What one consultation of the dispatch-token endpoint returned: a token
21679
+ * reused from this process's cache (`reused`, no request made, so no trace)
21680
+ * or a fresh response. Reported only where that endpoint is consulted, never
21681
+ * for a token pre-minted at admission.
21682
+ */
21683
+ type RunLedgerDispatchTokenObservation = {
21684
+ reused: boolean;
21685
+ trace: RunLedgerDispatchResponseTrace | null;
21686
+ };
21687
+
21688
+ /**
21689
+ * Where a delivery's app authority came from, for the slow-delivery record:
21690
+ * - `requested`: a dispatch-token request was made for this delivery, so
21691
+ * `authorityRequestId` names it when the receiver sent a platform id.
21692
+ * - `cached`: a dispatch token reused from this process or this drain.
21693
+ * - `pre_minted`: the run's admission-minted projection token.
21694
+ * - `local`: signed in-process or supplied by the caller.
21695
+ * Only `requested` makes an authority request; for every other source a null
21696
+ * `authorityRequestId` means no request happened, not a missing header.
21697
+ * `null` means the delivery ended before it had an authority.
21698
+ */
21699
+ type RunLedgerDeliveryAuthoritySource =
21700
+ | 'requested'
21701
+ | 'cached'
21702
+ | 'pre_minted'
21703
+ | 'local';
21704
+
21705
+ function runLedgerDeliveryAuthoritySource(input: {
21706
+ authorityReady: boolean;
21707
+ tokenAuthority: RunLedgerOutboxTokenAuthority;
21708
+ reusedWithinDrain: boolean;
21709
+ token: RunLedgerDispatchTokenObservation | null;
21710
+ }): RunLedgerDeliveryAuthoritySource | null {
21711
+ if (!input.authorityReady) return null;
21712
+ if (input.tokenAuthority !== 'gateway') return 'local';
21713
+ if (input.token) return input.token.reused ? 'cached' : 'requested';
21714
+ return input.reusedWithinDrain ? 'cached' : 'pre_minted';
21715
+ }
21716
+
21717
+ function runLedgerSlowDeliveryResponseFields(
21718
+ authoritySource: RunLedgerDeliveryAuthoritySource | null,
21719
+ authority: RunLedgerDispatchResponseTrace | null,
21720
+ append:
21721
+ | (RunLedgerDispatchResponseTrace & { roundTripMs: number | null })
21722
+ | null,
21723
+ appendTransportAttemptId: string | null,
21724
+ ): {
21725
+ authoritySource: RunLedgerDeliveryAuthoritySource | null;
21726
+ authorityRequestId: string | null;
21727
+ authorityServerTiming: string | null;
21728
+ appendRequestId: string | null;
21729
+ appendServerTiming: string | null;
21730
+ appendRoundTripMs: number | null;
21731
+ appendTransportAttemptId: string | null;
21732
+ } {
21733
+ return {
21734
+ authoritySource,
21735
+ authorityRequestId: authority?.platformRequestId ?? null,
21736
+ authorityServerTiming: authority?.serverTimingHeader ?? null,
21737
+ appendRequestId: append?.platformRequestId ?? null,
21738
+ appendServerTiming: append?.serverTimingHeader ?? null,
21739
+ appendRoundTripMs: append?.roundTripMs ?? null,
21740
+ // The id this delivery sent in its last append request's
21741
+ // transport-attempt header. Unlike the response fields above it is set
21742
+ // even when that request was aborted or never answered.
21743
+ appendTransportAttemptId,
21744
+ };
21745
+ }
21746
+
21747
+ /**
21748
+ * Admission persists an optional, run-scoped `run.ledger.project` token for the
21749
+ * worker's terminal fast path. Treat it as usable only when its decoded claims
21750
+ * match the exact durable launch contract and it has enough time left for a
21751
+ * complete projection. The app runtime still verifies the signature, action
21752
+ * capability, and live outbox lease before writing; a mismatch falls back to
21753
+ * the lease-validated mint endpoint. Prefix grouping additionally requires the
21754
+ * issuer's explicit advertisement to match the launch's immutable receiver;
21755
+ * its absence preserves legacy grouping without discarding the usable token.
21756
+ */
21757
+ function usableRunLedgerExecutorToken(
21758
+ launch: RunLedgerDispatchLaunch,
21759
+ reuseMarginMs: number,
21760
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
21761
+ ): {
21762
+ executorToken: string;
21763
+ runLedgerIngressPrefixBatching?: true;
21764
+ } | null {
21765
+ const token = launch.runLedgerExecutorToken;
21766
+ if (!token || typeof token !== 'string') return null;
21767
+
21768
+ try {
21769
+ const claims = decodeJwt(token) as Record<string, unknown>;
21770
+ const authority = launch.runtimeAuthority;
21771
+ if (!authority || typeof authority !== 'object') return null;
21772
+
21773
+ const expiresAtMs =
21774
+ typeof claims.exp === 'number' &&
21775
+ Number.isSafeInteger(claims.exp) &&
21776
+ Number.isSafeInteger(claims.exp * 1_000)
21777
+ ? claims.exp * 1_000
21778
+ : null;
21779
+ const expectedOrigin = new URL(runtimeTokenOriginFromLaunch(launch)).origin;
21780
+ const expectedSchedulerSchema = normalizeRuntimeSchedulerSchema(
21781
+ authority.schedulerSchema,
21782
+ );
21783
+ const requestedSchedulerSchema = normalizeRuntimeSchedulerSchema(
21784
+ options?.schema,
21785
+ );
21786
+
21787
+ if (
21788
+ expiresAtMs === null ||
21789
+ expiresAtMs <= Date.now() + reuseMarginMs ||
21790
+ claims.iss !== 'deepline.internal' ||
21791
+ claims.aud !== 'deepline.workflow.execution' ||
21792
+ claims.purpose !== 'workflow_run_execution' ||
21793
+ claims.org_id !== launch.orgId ||
21794
+ claims.org_id !== authority.orgId ||
21795
+ claims.run_id !== launch.runId ||
21796
+ claims.run_id !== authority.runId ||
21797
+ claims.workflow_id !== authority.workflowId ||
21798
+ claims.runtime_origin !== expectedOrigin ||
21799
+ normalizeRuntimeSchedulerSchema(claims.scheduler_schema) !==
21800
+ expectedSchedulerSchema ||
21801
+ (requestedSchedulerSchema !== null &&
21802
+ requestedSchedulerSchema !== expectedSchedulerSchema) ||
21803
+ !Array.isArray(claims.capabilities) ||
21804
+ claims.capabilities.length !== 1 ||
21805
+ claims.capabilities[0] !== 'run.ledger.project'
21806
+ ) {
21807
+ return null;
21808
+ }
21809
+ const receiverOrigin = new URL(runtimeApiBaseUrlFromLaunch(launch)).origin;
21810
+ return {
21811
+ executorToken: token,
21812
+ ...(claims.run_ledger_ingress_prefix_origin === receiverOrigin &&
21813
+ isImmutableRuntimeExecutionRecoveryOrigin(receiverOrigin)
21814
+ ? { runLedgerIngressPrefixBatching: true as const }
21815
+ : {}),
21816
+ };
21817
+ } catch {
21818
+ return null;
21819
+ }
21820
+ }
21821
+
21473
21822
  // Reuse only gateway-issued run.progress authority. A prefix advertisement
21474
21823
  // remains reusable only for the same immutable app build; mutable receivers
21475
21824
  // must reacquire their current receipt shape on every drain. Every delivery
@@ -21497,6 +21846,8 @@ async function requestRunLedgerDispatchToken(
21497
21846
  reused: boolean;
21498
21847
  cacheKey: string;
21499
21848
  runLedgerIngressPrefixBatching?: true;
21849
+ /** Present only when this call made the request (never for a reused token). */
21850
+ responseTrace?: RunLedgerDispatchResponseTrace;
21500
21851
  }> {
21501
21852
  const internalToken = process.env.DEEPLINE_INTERNAL_TOKEN?.trim();
21502
21853
  if (!internalToken || !input.eventId || !input.leaseOwner) {
@@ -21647,6 +21998,16 @@ async function requestRunLedgerDispatchToken(
21647
21998
  ...(body.runLedgerIngressPrefixBatching === true
21648
21999
  ? { runLedgerIngressPrefixBatching: true as const }
21649
22000
  : {}),
22001
+ responseTrace: {
22002
+ platformRequestId: readRuntimeApiDiagnosticHeader(
22003
+ response.headers,
22004
+ 'x-vercel-id',
22005
+ ),
22006
+ serverTimingHeader: readRuntimeApiDiagnosticHeader(
22007
+ response.headers,
22008
+ 'server-timing',
22009
+ ),
22010
+ },
21650
22011
  };
21651
22012
  }
21652
22013
 
@@ -12,6 +12,7 @@ export type RunLedgerDispatchLaunch = Pick<
12
12
  | 'playId'
13
13
  | 'playName'
14
14
  | 'executorToken'
15
+ | 'runLedgerExecutorToken'
15
16
  | 'baseUrl'
16
17
  | 'runtimeApiBaseUrl'
17
18
  | 'runtimeTokenOrigin'
@@ -27,6 +28,7 @@ export function runLedgerDispatchLaunchSql(alias: string): string {
27
28
  'playId',
28
29
  'playName',
29
30
  'executorToken',
31
+ 'runLedgerExecutorToken',
30
32
  'baseUrl',
31
33
  'runtimeApiBaseUrl',
32
34
  'runtimeTokenOrigin',