@adonis-agora/durable 0.58.3 → 0.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (219) hide show
  1. package/CHANGELOG.md +68 -0
  2. package/dist/assets/spa/assets/index-70hGzOnT.css +1 -0
  3. package/dist/assets/spa/assets/index-BI1EANJO.js +16 -0
  4. package/dist/assets/spa/assets/index-BI1EANJO.js.map +1 -0
  5. package/dist/assets/spa/index.html +2 -2
  6. package/dist/commands/lifecycle_republish.d.ts +23 -0
  7. package/dist/commands/lifecycle_republish.d.ts.map +1 -0
  8. package/dist/commands/lifecycle_republish.js +64 -0
  9. package/dist/commands/lifecycle_republish.js.map +1 -0
  10. package/dist/commands/main.d.ts +4 -3
  11. package/dist/commands/main.d.ts.map +1 -1
  12. package/dist/commands/main.js +3 -1
  13. package/dist/commands/main.js.map +1 -1
  14. package/dist/providers/dashboard_provider.d.ts +2 -0
  15. package/dist/providers/dashboard_provider.d.ts.map +1 -1
  16. package/dist/providers/dashboard_provider.js +9 -1
  17. package/dist/providers/dashboard_provider.js.map +1 -1
  18. package/dist/providers/durable_provider.d.ts.map +1 -1
  19. package/dist/providers/durable_provider.js +3 -0
  20. package/dist/providers/durable_provider.js.map +1 -1
  21. package/dist/src/codec-state-store.d.ts +10 -1
  22. package/dist/src/codec-state-store.d.ts.map +1 -1
  23. package/dist/src/codec-state-store.js +25 -0
  24. package/dist/src/codec-state-store.js.map +1 -1
  25. package/dist/src/commands/index.d.ts +1 -1
  26. package/dist/src/commands/index.d.ts.map +1 -1
  27. package/dist/src/commands/index.js +1 -1
  28. package/dist/src/commands/index.js.map +1 -1
  29. package/dist/src/commands/runs.d.ts +24 -1
  30. package/dist/src/commands/runs.d.ts.map +1 -1
  31. package/dist/src/commands/runs.js +50 -0
  32. package/dist/src/commands/runs.js.map +1 -1
  33. package/dist/src/commands/worker.d.ts.map +1 -1
  34. package/dist/src/commands/worker.js +3 -0
  35. package/dist/src/commands/worker.js.map +1 -1
  36. package/dist/src/dashboard/gateway-adapter.d.ts +5 -0
  37. package/dist/src/dashboard/gateway-adapter.d.ts.map +1 -1
  38. package/dist/src/dashboard/gateway-adapter.js +20 -0
  39. package/dist/src/dashboard/gateway-adapter.js.map +1 -1
  40. package/dist/src/dashboard/handlers.d.ts +29 -1
  41. package/dist/src/dashboard/handlers.d.ts.map +1 -1
  42. package/dist/src/dashboard/handlers.js +103 -0
  43. package/dist/src/dashboard/handlers.js.map +1 -1
  44. package/dist/src/dashboard/index.d.ts +2 -2
  45. package/dist/src/dashboard/index.d.ts.map +1 -1
  46. package/dist/src/dashboard/index.js +2 -2
  47. package/dist/src/dashboard/index.js.map +1 -1
  48. package/dist/src/dashboard/openapi.d.ts.map +1 -1
  49. package/dist/src/dashboard/openapi.js +39 -0
  50. package/dist/src/dashboard/openapi.js.map +1 -1
  51. package/dist/src/define_config.d.ts +13 -0
  52. package/dist/src/define_config.d.ts.map +1 -1
  53. package/dist/src/define_config.js.map +1 -1
  54. package/dist/src/diagnostics-bridge.d.ts +5 -1
  55. package/dist/src/diagnostics-bridge.d.ts.map +1 -1
  56. package/dist/src/diagnostics-bridge.js +16 -2
  57. package/dist/src/diagnostics-bridge.js.map +1 -1
  58. package/dist/src/engine.d.ts +426 -6
  59. package/dist/src/engine.d.ts.map +1 -1
  60. package/dist/src/engine.js +1951 -101
  61. package/dist/src/engine.js.map +1 -1
  62. package/dist/src/errors.d.ts.map +1 -1
  63. package/dist/src/errors.js +4 -0
  64. package/dist/src/errors.js.map +1 -1
  65. package/dist/src/event-accumulators.d.ts +28 -5
  66. package/dist/src/event-accumulators.d.ts.map +1 -1
  67. package/dist/src/event-accumulators.js +102 -21
  68. package/dist/src/event-accumulators.js.map +1 -1
  69. package/dist/src/events.d.ts +25 -0
  70. package/dist/src/events.d.ts.map +1 -1
  71. package/dist/src/events.js +24 -0
  72. package/dist/src/events.js.map +1 -1
  73. package/dist/src/handshake/announced.d.ts +6 -0
  74. package/dist/src/handshake/announced.d.ts.map +1 -1
  75. package/dist/src/handshake/announced.js +20 -0
  76. package/dist/src/handshake/announced.js.map +1 -1
  77. package/dist/src/handshake/descriptor.d.ts +6 -0
  78. package/dist/src/handshake/descriptor.d.ts.map +1 -1
  79. package/dist/src/handshake/descriptor.js +1 -0
  80. package/dist/src/handshake/descriptor.js.map +1 -1
  81. package/dist/src/index.d.ts +3 -2
  82. package/dist/src/index.d.ts.map +1 -1
  83. package/dist/src/index.js +3 -2
  84. package/dist/src/index.js.map +1 -1
  85. package/dist/src/interfaces.d.ts +139 -3
  86. package/dist/src/interfaces.d.ts.map +1 -1
  87. package/dist/src/interfaces.js +11 -0
  88. package/dist/src/interfaces.js.map +1 -1
  89. package/dist/src/lifecycle-events.d.ts +372 -0
  90. package/dist/src/lifecycle-events.d.ts.map +1 -0
  91. package/dist/src/lifecycle-events.js +383 -0
  92. package/dist/src/lifecycle-events.js.map +1 -0
  93. package/dist/src/metrics.d.ts +15 -0
  94. package/dist/src/metrics.d.ts.map +1 -1
  95. package/dist/src/metrics.js +36 -1
  96. package/dist/src/metrics.js.map +1 -1
  97. package/dist/src/otel/durable-metrics.d.ts +9 -0
  98. package/dist/src/otel/durable-metrics.d.ts.map +1 -1
  99. package/dist/src/otel/durable-metrics.js +17 -0
  100. package/dist/src/otel/durable-metrics.js.map +1 -1
  101. package/dist/src/otel/durable-otel.d.ts +7 -0
  102. package/dist/src/otel/durable-otel.d.ts.map +1 -1
  103. package/dist/src/otel/durable-otel.js +69 -3
  104. package/dist/src/otel/durable-otel.js.map +1 -1
  105. package/dist/src/run-detail-summary.d.ts +48 -1
  106. package/dist/src/run-detail-summary.d.ts.map +1 -1
  107. package/dist/src/run-detail-summary.js +30 -2
  108. package/dist/src/run-detail-summary.js.map +1 -1
  109. package/dist/src/run-gateway/interface.d.ts +16 -0
  110. package/dist/src/run-gateway/interface.d.ts.map +1 -1
  111. package/dist/src/run-gateway/proxy-run-gateway.d.ts +31 -1
  112. package/dist/src/run-gateway/proxy-run-gateway.d.ts.map +1 -1
  113. package/dist/src/run-gateway/proxy-run-gateway.js +59 -7
  114. package/dist/src/run-gateway/proxy-run-gateway.js.map +1 -1
  115. package/dist/src/run-gateway/run-request-responder.d.ts +4 -29
  116. package/dist/src/run-gateway/run-request-responder.d.ts.map +1 -1
  117. package/dist/src/run-gateway/run-request-responder.js +183 -6
  118. package/dist/src/run-gateway/run-request-responder.js.map +1 -1
  119. package/dist/src/run-gateway/store-run-gateway.d.ts +9 -0
  120. package/dist/src/run-gateway/store-run-gateway.d.ts.map +1 -1
  121. package/dist/src/run-gateway/store-run-gateway.js +8 -0
  122. package/dist/src/run-gateway/store-run-gateway.js.map +1 -1
  123. package/dist/src/run-waiting.d.ts +1 -0
  124. package/dist/src/run-waiting.d.ts.map +1 -1
  125. package/dist/src/run-waiting.js +6 -1
  126. package/dist/src/run-waiting.js.map +1 -1
  127. package/dist/src/sticky-continuation.d.ts +141 -0
  128. package/dist/src/sticky-continuation.d.ts.map +1 -0
  129. package/dist/src/sticky-continuation.js +231 -0
  130. package/dist/src/sticky-continuation.js.map +1 -0
  131. package/dist/src/stores/factory.d.ts +6 -0
  132. package/dist/src/stores/factory.d.ts.map +1 -1
  133. package/dist/src/stores/factory.js +3 -0
  134. package/dist/src/stores/factory.js.map +1 -1
  135. package/dist/src/stores/lucid-pg-hot-path.d.ts +30 -0
  136. package/dist/src/stores/lucid-pg-hot-path.d.ts.map +1 -0
  137. package/dist/src/stores/lucid-pg-hot-path.js +67 -0
  138. package/dist/src/stores/lucid-pg-hot-path.js.map +1 -0
  139. package/dist/src/stores/lucid.d.ts +30 -1
  140. package/dist/src/stores/lucid.d.ts.map +1 -1
  141. package/dist/src/stores/lucid.js +460 -17
  142. package/dist/src/stores/lucid.js.map +1 -1
  143. package/dist/src/telescope/dashboard.d.ts.map +1 -1
  144. package/dist/src/telescope/dashboard.js +50 -0
  145. package/dist/src/telescope/dashboard.js.map +1 -1
  146. package/dist/src/telescope/data-providers.d.ts +19 -0
  147. package/dist/src/telescope/data-providers.d.ts.map +1 -1
  148. package/dist/src/telescope/data-providers.js +128 -2
  149. package/dist/src/telescope/data-providers.js.map +1 -1
  150. package/dist/src/telescope/extension.d.ts.map +1 -1
  151. package/dist/src/telescope/extension.js +4 -1
  152. package/dist/src/telescope/extension.js.map +1 -1
  153. package/dist/src/telescope/index.d.ts +3 -2
  154. package/dist/src/telescope/index.d.ts.map +1 -1
  155. package/dist/src/telescope/index.js +2 -1
  156. package/dist/src/telescope/index.js.map +1 -1
  157. package/dist/src/telescope/telescope-sdk.d.ts +16 -0
  158. package/dist/src/telescope/telescope-sdk.d.ts.map +1 -1
  159. package/dist/src/telescope/watcher.d.ts +66 -0
  160. package/dist/src/telescope/watcher.d.ts.map +1 -0
  161. package/dist/src/telescope/watcher.js +185 -0
  162. package/dist/src/telescope/watcher.js.map +1 -0
  163. package/dist/src/testing/in-memory-state-store.d.ts +12 -1
  164. package/dist/src/testing/in-memory-state-store.d.ts.map +1 -1
  165. package/dist/src/testing/in-memory-state-store.js +109 -1
  166. package/dist/src/testing/in-memory-state-store.js.map +1 -1
  167. package/dist/src/testing-kit/harness.d.ts +1 -1
  168. package/dist/src/testing-kit/harness.d.ts.map +1 -1
  169. package/dist/src/testing-kit/harness.js +3 -1
  170. package/dist/src/testing-kit/harness.js.map +1 -1
  171. package/dist/src/testing-kit/state-store-conformance.d.ts.map +1 -1
  172. package/dist/src/testing-kit/state-store-conformance.js +258 -0
  173. package/dist/src/testing-kit/state-store-conformance.js.map +1 -1
  174. package/dist/src/transport-pool.d.ts +10 -0
  175. package/dist/src/transport-pool.d.ts.map +1 -1
  176. package/dist/src/transport-pool.js +23 -2
  177. package/dist/src/transport-pool.js.map +1 -1
  178. package/dist/src/transports/bullmq/bullmq-transport.d.ts +41 -1
  179. package/dist/src/transports/bullmq/bullmq-transport.d.ts.map +1 -1
  180. package/dist/src/transports/bullmq/bullmq-transport.js +283 -5
  181. package/dist/src/transports/bullmq/bullmq-transport.js.map +1 -1
  182. package/dist/src/transports/bullmq/deps.d.ts +12 -1
  183. package/dist/src/transports/bullmq/deps.d.ts.map +1 -1
  184. package/dist/src/transports/bullmq/deps.js.map +1 -1
  185. package/dist/src/transports/bullmq/naming.d.ts +10 -0
  186. package/dist/src/transports/bullmq/naming.d.ts.map +1 -1
  187. package/dist/src/transports/bullmq/naming.js +16 -0
  188. package/dist/src/transports/bullmq/naming.js.map +1 -1
  189. package/dist/src/transports/db.d.ts +3 -1
  190. package/dist/src/transports/db.d.ts.map +1 -1
  191. package/dist/src/transports/db.js +4 -4
  192. package/dist/src/transports/db.js.map +1 -1
  193. package/dist/src/transports/queue.d.ts +3 -1
  194. package/dist/src/transports/queue.d.ts.map +1 -1
  195. package/dist/src/transports/queue.js +2 -2
  196. package/dist/src/transports/queue.js.map +1 -1
  197. package/dist/src/transports/worker-keys.d.ts +2 -1
  198. package/dist/src/transports/worker-keys.d.ts.map +1 -1
  199. package/dist/src/transports/worker-keys.js +3 -2
  200. package/dist/src/transports/worker-keys.js.map +1 -1
  201. package/dist/src/workflow-ctx.d.ts +7 -0
  202. package/dist/src/workflow-ctx.d.ts.map +1 -1
  203. package/dist/src/workflow-ctx.js +90 -1
  204. package/dist/src/workflow-ctx.js.map +1 -1
  205. package/dist/src/workflow-discovery.d.ts.map +1 -1
  206. package/dist/src/workflow-discovery.js +1 -0
  207. package/dist/src/workflow-discovery.js.map +1 -1
  208. package/dist/src/workflow-events.d.ts +101 -0
  209. package/dist/src/workflow-events.d.ts.map +1 -1
  210. package/dist/src/workflow-events.js +79 -1
  211. package/dist/src/workflow-events.js.map +1 -1
  212. package/dist/src/workflow-ref.d.ts +17 -2
  213. package/dist/src/workflow-ref.d.ts.map +1 -1
  214. package/dist/src/workflow-ref.js.map +1 -1
  215. package/package.json +5 -5
  216. package/skills/durable-cluster/SKILL.md +4 -2
  217. package/dist/assets/spa/assets/index-6nmSTfT7.css +0 -1
  218. package/dist/assets/spa/assets/index-Dj0X-IAu.js +0 -16
  219. package/dist/assets/spa/assets/index-Dj0X-IAu.js.map +0 -1
@@ -12,28 +12,30 @@ import { controlPlaneDescriptor, planDispatch, WorkflowBlocked, } from './dispat
12
12
  import { parseDuration } from './duration.js';
13
13
  import { Entities } from './entities.js';
14
14
  import { ContinueAsNew, GatherError, NonDeterminismError, RemoteStepTimeout, RemoteWorkflowTimeout, WorkflowSuspended, } from './errors.js';
15
- import { EventAccumulators } from './event-accumulators.js';
16
- import { eventMatches, eventMatchOf, eventPrefix } from './events.js';
15
+ import { accumulatorRunId, EventAccumulators, } from './event-accumulators.js';
16
+ import { acceptingSubscription, eventMatches, eventMatchOf, eventPrefix, toEventSubscription, } from './events.js';
17
17
  import { childIdsOf, FOLDED_PREFIX, fanOfWaiter, parentRunIdOf, parseStepFanGroup, stepFanGroup, } from './fan-join.js';
18
- import { aggregateAnnouncements } from './handshake/announced.js';
18
+ import { aggregateAnnouncements, announcementsOf, } from './handshake/announced.js';
19
19
  import { buildWorkflowDirectory } from './handshake/directory.js';
20
20
  import { isNonEmptyRunScope, TERMINAL_RUN_STATUSES } from './interfaces.js';
21
- import { DEFAULT_NAMESPACE } from './namespaces.js';
21
+ import { chainPredecessorsOf, coalescedSubscribersOf, deliveredSubscribersOf, LIFECYCLE_HEAL_WINDOW_MS, LIFECYCLE_ORPHAN_MS, LIFECYCLE_OWED_TAG_PREFIX, LIFECYCLE_PENDING_TAG, LIFECYCLE_REGISTRY_HEARTBEAT_MS, LIFECYCLE_REGISTRY_STALE_MS, LIFECYCLE_REGISTRY_TAG, LIFECYCLE_REGISTRY_TTL_MS, LIFECYCLE_REGISTRY_WORKFLOW, LIFECYCLE_TRACE_TAG_PREFIX, LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX, LIFECYCLE_TRIGGERED_TAG_PREFIX, lastRunOfChain, lifecycleCoalescedTag, lifecycleDeliveriesOf, lifecycleEventName, lifecycleEventOf, lifecycleKindOf, lifecycleOwedTag, lifecycleRegistryNamespace, lifecycleRegistryRunId, lifecycleRegistrySubscriptions, lifecycleRunId, lifecycleSourceOf, lifecycleTraceparentOf, lifecycleTriggerFailedTag, nextContinuationId, owedSubscribersOf, parseLifecycleEvent, RUN_WAIT_TOKEN_PREFIX, retentionKeeps, runWaitPrefix, runWaitTargetOf, triggeredRunsOf, } from './lifecycle-events.js';
22
+ import { DEFAULT_NAMESPACE, normalizeNamespace } from './namespaces.js';
22
23
  import { asHeartbeat } from './observability-scope.js';
23
24
  import { assertRoutes, codeAndDetails, HANDOFF_SEQ_BASE, handoffHandlerId, handoffName, liveHandoff, onErrorRunId, retiredHandoffName, routeFor, } from './on-error.js';
24
25
  import { ORIGIN_RESOLVED, originOfCaller } from './origin.js';
25
26
  import { breakpointToken, stepId } from './protocol.js';
26
27
  import { markRemoteEvent } from './remote-event.js';
27
28
  import { RemoteWorkflowExecutor } from './remote-workflow-executor.js';
28
- import { loadFanMembers, loadRunDetail, } from './run-detail-summary.js';
29
+ import { etagMatches, extendEtag, loadFanMembers, loadRunDetail, } from './run-detail-summary.js';
29
30
  import { runFacetsOf } from './run-facets.js';
30
31
  import { scanRunValueFacets } from './run-value-facets.js';
31
- import { listSignalWaitersOfRuns } from './run-waiting.js';
32
+ import { getRunsByIds, listSignalWaitersOfRuns } from './run-waiting.js';
32
33
  import { resolveSchedule, scheduleWindow } from './scheduler.js';
33
34
  import { ScheduleClient } from './schedules.js';
34
35
  import { SINGLETON_RESERVED_TAGS, withoutAdmission, withSingletonTracked, } from './singleton-admission.js';
35
36
  import { SingletonGate } from './singleton-gate.js';
36
37
  import { validateStandard, validateStandardSync, } from './standard-schema.js';
38
+ import { StickyCache, StickyContinuation } from './sticky-continuation.js';
37
39
  import { sanitizeQueueToken, tenantGroup } from './tenant-group.js';
38
40
  import { TransportPool } from './transport-pool.js';
39
41
  import { workflowAls } from './workflow-als.js';
@@ -48,6 +50,10 @@ function mergeTags(staticTags, runTags) {
48
50
  /** A breakpoint checkpoint's `name` is `breakpoint` (or `breakpoint:<label>`). This name — not the
49
51
  * reused `signal` kind — is the explicit marker the dashboard and `continue()` detect it by. */
50
52
  const BREAKPOINT = 'breakpoint';
53
+ /** Default size of the sticky continuation cache (see {@link WorkflowEngineDeps.stickyRuns}). */
54
+ const DEFAULT_STICKY_RUNS = 2000;
55
+ /** Default idle limit of a parked body in the sticky cache. */
56
+ const DEFAULT_STICKY_IDLE_MS = 300_000;
51
57
  const isBreakpoint = (cp) => cp.status === 'pending' && cp.name.startsWith(BREAKPOINT);
52
58
  /**
53
59
  * Every `RunStatus` from which a turn's outcome (`completed` / `failed` / `suspended` / `blocked`) is a
@@ -161,6 +167,8 @@ const leaseRefused = new WeakMap();
161
167
  * parent's next turn.
162
168
  */
163
169
  const LOCAL_RUN_CONCURRENCY = 256;
170
+ /** Pending runs one {@link WorkflowEngine.runPending} poll takes at most. */
171
+ const PENDING_POLL_BATCH = 100;
164
172
  /** How many dispatched runs this instance takes from the transport's run queue at once. */
165
173
  const RUN_CONSUMER_CONCURRENCY = 64;
166
174
  /** The most runs handed to {@link Transport.dispatchRuns} in one call. */
@@ -168,6 +176,8 @@ const RUN_DISPATCH_BATCH = 500;
168
176
  /** Tries, and the first backoff, for telling a parent that its child finished (see `deliverToParent`). */
169
177
  const PARENT_NOTIFY_ATTEMPTS = 4;
170
178
  const PARENT_NOTIFY_BACKOFF_MS = 250;
179
+ /** What a lifecycle publish's run lease owner carries: `<instanceId>#lifecycle#<uuid>`. */
180
+ const LIFECYCLE_LEASE_MARK = '#lifecycle#';
171
181
  /** How often, at most, and how many times a wake dropped on a held lease looks again (see `recheckAfterLease`). */
172
182
  const LEASE_RECHECK_MAX_MS = 2_000;
173
183
  const LEASE_RECHECK_SLACK_MS = 100;
@@ -217,11 +227,6 @@ function shuffle(items) {
217
227
  function stepGroup(step, runNamespace) {
218
228
  return tenantGroup(sanitizeQueueToken(step.name), step.partition ?? runNamespace);
219
229
  }
220
- /** The id for the next continuation of a run: `r` → `r~1` → `r~2` … (stable, traceable lineage). */
221
- function nextContinuationId(runId) {
222
- const m = runId.match(/^(.*)~(\d+)$/);
223
- return m ? `${m[1]}~${Number(m[2]) + 1}` : `${runId}~1`;
224
- }
225
230
  /** True when version `a` is newer than `b` (numeric when both parse as numbers, else natural sort). */
226
231
  function isNewerVersion(a, b) {
227
232
  const na = Number(a);
@@ -287,6 +292,16 @@ export class WorkflowEngine {
287
292
  maxRecoveryAttempts;
288
293
  unregisteredWorkflowGraceMs;
289
294
  reconcileMs;
295
+ /** Parked workflow bodies kept between turns (see {@link WorkflowEngineDeps.stickyRuns}). */
296
+ sticky;
297
+ /** Pending runs claimed by {@link runPending} for the turn about to run them, as read by the claim. */
298
+ claimedRuns = new Map();
299
+ /** The continuation each turn executing here runs in, by run id, while the turn runs. */
300
+ turnContinuations = new Map();
301
+ /** Whether a transport routes this instance's step results back to it (see `routeReplies`). */
302
+ repliesRouted = false;
303
+ /** The run row stamp (`updatedAt`, epoch ms) a turn here wrote when it settled `suspended`. */
304
+ suspendStamps = new Map();
290
305
  heartbeatPersistThrottleMs;
291
306
  timerBatchSize;
292
307
  eventBufferTtlMs;
@@ -335,7 +350,7 @@ export class WorkflowEngine {
335
350
  workflows = new Map();
336
351
  /** The newest registered version per workflow name — used to `start` new runs. */
337
352
  latest = new Map();
338
- /** Event name → workflow names started when that event is published (see `onEvent`). */
353
+ /** Event name → subscribing workflow name → that workflow's subscriptions to it. See `onEvent`. */
339
354
  eventTriggers = new Map();
340
355
  /** Durable-entity subsystem (registers the `__entity` runner; see `registerEntity`). */
341
356
  entities;
@@ -546,6 +561,9 @@ export class WorkflowEngine {
546
561
  reconcile === undefined || !Number.isFinite(reconcile) || reconcile < 0
547
562
  ? 300_000
548
563
  : reconcile || undefined;
564
+ const stickyRuns = deps.stickyRuns ?? DEFAULT_STICKY_RUNS;
565
+ const stickyIdle = deps.stickyIdleMs ?? DEFAULT_STICKY_IDLE_MS;
566
+ this.sticky = new StickyCache(Number.isFinite(stickyRuns) ? Math.max(0, Math.floor(stickyRuns)) : DEFAULT_STICKY_RUNS, Number.isFinite(stickyIdle) && stickyIdle >= 0 ? stickyIdle : DEFAULT_STICKY_IDLE_MS);
549
567
  // Validated like `timerBatchSize`: a NaN/negative throttle would silently disable the
550
568
  // cross-instance liveness salvage (`timeoutMs + NaN` never compares true). `0` stays valid.
551
569
  const throttle = deps.heartbeatPersistThrottleMs;
@@ -675,6 +693,11 @@ export class WorkflowEngine {
675
693
  async (event) => {
676
694
  await this.persistStepEvent(event);
677
695
  });
696
+ // Step results come back to the instance holding the body parked on the step, where a transport
697
+ // can route them (see WorkflowEngineDeps.routeReplies).
698
+ if ((deps.routeReplies ?? true) && this.sticky.enabled) {
699
+ this.repliesRouted = this.pool.routeReplies(this.instanceId);
700
+ }
678
701
  // Control plane: re-broadcast lifecycle events from OTHER instances to this instance's
679
702
  // subscribers (cross-pod live-tail), and act on cancellations issued elsewhere. A broker may
680
703
  // echo our own publish back — ignore those (we already handled them locally) to avoid duplicates.
@@ -1109,10 +1132,11 @@ export class WorkflowEngine {
1109
1132
  const current = this.latest.get(name);
1110
1133
  if (!current || isNewerVersion(version, current.version))
1111
1134
  this.latest.set(name, registered);
1112
- for (const event of opts?.onEvent ?? []) {
1113
- const subscribers = this.eventTriggers.get(event) ?? new Set();
1114
- subscribers.add(name);
1115
- this.eventTriggers.set(event, subscribers);
1135
+ for (const entry of opts?.onEvent ?? []) {
1136
+ const subscription = toEventSubscription(entry);
1137
+ const subscribers = this.eventTriggers.get(subscription.event) ?? new Map();
1138
+ subscribers.set(name, [...(subscribers.get(name) ?? []), subscription]);
1139
+ this.eventTriggers.set(subscription.event, subscribers);
1116
1140
  }
1117
1141
  }
1118
1142
  /**
@@ -1409,17 +1433,1226 @@ export class WorkflowEngine {
1409
1433
  console.warn(`[adonis-durable] could not start dead-letter workflow ${target} for run ${run.id}: ${err instanceof Error ? err.message : String(err)}`);
1410
1434
  }));
1411
1435
  }
1412
- /** Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the
1413
- * control plane so other instances (e.g. a dashboard pod) can live-tail this run. */
1414
- emit(event) {
1436
+ /**
1437
+ * Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the control
1438
+ * plane so other instances (e.g. a dashboard pod) can live-tail this run. A `run.completed` or
1439
+ * `run.failed` also wakes the `ctx.waitForRun` waiters on the run and publishes its lifecycle
1440
+ * event (see {@link publishLifecycle}); `lifecycle: false` marks a `run.completed` that is not a
1441
+ * real completion (a continue-as-new handoff), which publishes no lifecycle event.
1442
+ */
1443
+ emit(event, opts) {
1415
1444
  const full = { ...event, at: new Date() };
1416
1445
  this.deliver(full);
1446
+ if (full.type === 'run.completed' || full.type === 'run.failed') {
1447
+ this.notifyRunWaiters(full.runId, full.workflow);
1448
+ }
1449
+ if (opts?.lifecycle !== false)
1450
+ this.publishLifecycle(full);
1417
1451
  if (!this.controlPlane)
1418
1452
  return;
1419
1453
  this.eventsToBroadcast.push(full);
1420
1454
  if (!this.broadcastingEvents)
1421
1455
  void this.broadcastEvents();
1422
1456
  }
1457
+ /**
1458
+ * Start every workflow subscribed to `name` that takes `payload` (see {@link acceptingSubscription}),
1459
+ * each under `runIdOf(workflow)` and with its subscription's `input` applied; only the ones in
1460
+ * `only` when given. Returns the subscribers reached, with the run each started (none when the
1461
+ * payload went into its debounce/batch accumulator) and whether that run is new, and the ones that
1462
+ * could not be started, with why.
1463
+ *
1464
+ * `remote` names subscribers known only from the fleet's announcements (see
1465
+ * {@link announcedRemoteSubscribers}): each is started by name with the payload as its input, with
1466
+ * no `where` or `input` mapping, unless it is also registered here.
1467
+ */
1468
+ async startSubscribers(name, payload, runIdOf, only, remote = []) {
1469
+ const reached = [];
1470
+ const failed = [];
1471
+ for (const [workflow, subscriptions] of this.eventTriggers.get(name) ?? []) {
1472
+ if (only && !only.has(workflow))
1473
+ continue;
1474
+ const { accepted, errors } = acceptingSubscription(subscriptions, payload);
1475
+ if (!accepted) {
1476
+ if (errors.length > 0)
1477
+ failed.push({ workflow, error: errors[0] });
1478
+ continue;
1479
+ }
1480
+ // A subscriber that rejects the payload (validateInput / inputSchema) must not block the
1481
+ // others or the waiters: its run never starts, and the caller decides how to report it.
1482
+ try {
1483
+ const input = accepted.input ? accepted.input(payload) : payload;
1484
+ const batch = this.latest.get(workflow)?.eventBatch;
1485
+ if (batch) {
1486
+ // Coalesce: route the event into the workflow's accumulator (one long-lived run per
1487
+ // target and key that debounces/batches and then starts the target), named by the run
1488
+ // id it would start, so a second route of the same event is dropped there.
1489
+ await this.accumulators.route(workflow, batch, input, runIdOf(workflow));
1490
+ reached.push({ workflow });
1491
+ }
1492
+ else {
1493
+ const runId = runIdOf(workflow);
1494
+ const existed = (await this.store.getRun(runId)) !== null;
1495
+ await this.start(workflow, input, runId);
1496
+ reached.push({ workflow, runId, created: !existed });
1497
+ }
1498
+ }
1499
+ catch (error) {
1500
+ failed.push({ workflow, error });
1501
+ }
1502
+ }
1503
+ const local = this.eventTriggers.get(name);
1504
+ for (const workflow of remote) {
1505
+ if (local?.has(workflow) || (only && !only.has(workflow)))
1506
+ continue;
1507
+ try {
1508
+ const runId = runIdOf(workflow);
1509
+ const existed = (await this.store.getRun(runId)) !== null;
1510
+ await this.start(workflow, payload, runId);
1511
+ reached.push({ workflow, runId, created: !existed });
1512
+ }
1513
+ catch (error) {
1514
+ failed.push({ workflow, error });
1515
+ }
1516
+ }
1517
+ return { reached, failed };
1518
+ }
1519
+ /** The workflows registered on this instance that subscribe to `event`. */
1520
+ localSubscribers(event) {
1521
+ return [...(this.eventTriggers.get(event)?.keys() ?? [])];
1522
+ }
1523
+ /**
1524
+ * Whether this engine's own subscribers are owed the lifecycle events of `namespace`'s runs: always
1525
+ * for an engine with no namespace (its sweep reads every namespace's runs), else only its own
1526
+ * namespace's. `undefined` asks without a namespace and is always true.
1527
+ */
1528
+ servesNamespace(namespace) {
1529
+ return (namespace === undefined ||
1530
+ this.namespace === undefined ||
1531
+ normalizeNamespace(this.namespace) === normalizeNamespace(namespace));
1532
+ }
1533
+ /** The workflows registered here that subscribe to `event` and serve `namespace`'s runs. */
1534
+ localSubscribersFor(event, namespace) {
1535
+ return this.servesNamespace(namespace) ? this.localSubscribers(event) : [];
1536
+ }
1537
+ /**
1538
+ * The subscribers of `event` this engine starts by name although it has no `onEvent` for them:
1539
+ * announced only by non-Node workers (a Python `@worker.workflow(..., on_workflow_completed=...)`),
1540
+ * and reachable from here, either registered (`registerRemote`) or served by convention on a live
1541
+ * worker group of the same name. A subscriber some Node engine announces is left to that engine,
1542
+ * which applies its own `where` and `input`; one this engine cannot reach stays owed, for an
1543
+ * instance that can. Read from the snapshot {@link announcedSubscribers} keeps. With `namespace`,
1544
+ * only the subscriptions announced for that namespace's runs (see {@link announcedFor}).
1545
+ */
1546
+ async announcedRemoteSubscribers(event, namespace) {
1547
+ const announced = this.announcedFor(event, namespace);
1548
+ if (announced.size === 0)
1549
+ return [];
1550
+ const local = new Set(this.localSubscribers(event));
1551
+ const byEngine = this.announcedFor(event, namespace, { node: true });
1552
+ const startable = [];
1553
+ for (const workflow of [...announced].sort()) {
1554
+ if (local.has(workflow) || byEngine?.has(workflow) || workflow.startsWith('__'))
1555
+ continue;
1556
+ const registered = this.latest.get(workflow);
1557
+ if (registered && !registered.convention) {
1558
+ startable.push(workflow);
1559
+ continue;
1560
+ }
1561
+ try {
1562
+ if (await this.pool.transportWithLiveGroup(conventionGroup(workflow, this.namespace))) {
1563
+ startable.push(workflow);
1564
+ }
1565
+ }
1566
+ catch {
1567
+ // An unreadable fleet leaves the subscriber owed, for the next sweep.
1568
+ }
1569
+ }
1570
+ return startable;
1571
+ }
1572
+ /**
1573
+ * The subscribers of `event` this engine can deliver to for a run of `namespace`: the ones
1574
+ * registered here that serve it, and the announced ones it starts by name (see
1575
+ * {@link announcedRemoteSubscribers}).
1576
+ */
1577
+ async deliverableSubscribers(event, namespace) {
1578
+ const local = new Set(this.localSubscribersFor(event, namespace));
1579
+ const remote = await this.announcedRemoteSubscribers(event, namespace);
1580
+ return { local, remote, all: new Set([...local, ...remote]) };
1581
+ }
1582
+ /**
1583
+ * Every workflow a run of `workflow` settling as `kind` owes its lifecycle event to: the ones
1584
+ * registered here plus the ones the fleet announces (from a snapshot of the lifecycle subscription
1585
+ * registry and the worker descriptors at most 5s old), sorted.
1586
+ *
1587
+ * With `namespace` (the settled run's), only the subscribers that serve that namespace's runs: the
1588
+ * ones registered here when this engine has no namespace or has that one, the registry rows of
1589
+ * engines in that namespace or with none, and the worker descriptors serving it. What the
1590
+ * terminal status write marks the run owed to.
1591
+ */
1592
+ lifecycleSubscribers(workflow, kind, namespace) {
1593
+ if (workflow.startsWith('__'))
1594
+ return [];
1595
+ const event = lifecycleEventName(workflow, kind);
1596
+ return [
1597
+ ...new Set([
1598
+ ...this.localSubscribersFor(event, namespace),
1599
+ ...this.announcedSubscribers(event, namespace),
1600
+ ]),
1601
+ ].sort();
1602
+ }
1603
+ /**
1604
+ * Who starts whom through lifecycle events, for `workflow`: the workflows that start when it
1605
+ * settles, and the workflows whose settling starts it. Built from this engine's registry and the
1606
+ * subscriptions live workers announce (read again when the snapshot is older than 5s); a failed
1607
+ * fleet read leaves the local registry alone. What the dashboard's workflow page shows.
1608
+ *
1609
+ * With `namespace`, only the subscriptions of that namespace: the ones announced by registry runs
1610
+ * and worker descriptors serving it, and this engine's own when it is that namespace's engine (an
1611
+ * engine with no namespace is `default`'s). What a tenant is shown.
1612
+ */
1613
+ async workflowLifecycle(workflow, opts) {
1614
+ await this.freshAnnouncedSubscriptions();
1615
+ const snapshot = this.announcedSubscriptionsSnapshot;
1616
+ const scope = opts?.namespace;
1617
+ let fleet = snapshot?.byEvent ?? new Map();
1618
+ if (scope !== undefined) {
1619
+ fleet = new Map();
1620
+ for (const entry of snapshot?.entries ?? []) {
1621
+ if ((entry.namespace ?? DEFAULT_NAMESPACE) !== scope)
1622
+ continue;
1623
+ const subscribers = fleet.get(entry.event) ?? new Set();
1624
+ subscribers.add(entry.workflow);
1625
+ fleet.set(entry.event, subscribers);
1626
+ }
1627
+ }
1628
+ const triggers = scope === undefined || (this.namespace || 'default') === scope
1629
+ ? this.eventTriggers
1630
+ : new Map();
1631
+ const startsOnSettle = new Map();
1632
+ const startedBy = new Map();
1633
+ const link = (into, other, kind, local) => {
1634
+ const key = `${other}\u0000${kind}`;
1635
+ const entry = into.get(key) ?? {
1636
+ workflow: other,
1637
+ kind,
1638
+ local: false,
1639
+ announced: false,
1640
+ where: false,
1641
+ input: false,
1642
+ };
1643
+ if (local) {
1644
+ entry.local = true;
1645
+ entry.where ||= local.subscriptions.some((sub) => sub.where !== undefined);
1646
+ entry.input ||= local.subscriptions.some((sub) => sub.input !== undefined);
1647
+ const batch = this.latest.get(local.subscriber)?.eventBatch;
1648
+ if (batch)
1649
+ entry.coalesce = { mode: batch.mode, keyed: batch.key !== undefined };
1650
+ }
1651
+ else {
1652
+ entry.announced = true;
1653
+ }
1654
+ into.set(key, entry);
1655
+ };
1656
+ if (!workflow.startsWith('__')) {
1657
+ for (const kind of ['completed', 'failed']) {
1658
+ const event = lifecycleEventName(workflow, kind);
1659
+ for (const [subscriber, subscriptions] of triggers.get(event) ?? []) {
1660
+ link(startsOnSettle, subscriber, kind, { subscriber, subscriptions });
1661
+ }
1662
+ for (const subscriber of fleet.get(event) ?? []) {
1663
+ link(startsOnSettle, subscriber, kind, undefined);
1664
+ }
1665
+ }
1666
+ }
1667
+ for (const [event, subscribers] of triggers) {
1668
+ const source = parseLifecycleEvent(event);
1669
+ const subscriptions = subscribers.get(workflow);
1670
+ if (source && subscriptions) {
1671
+ link(startedBy, source.workflow, source.kind, { subscriber: workflow, subscriptions });
1672
+ }
1673
+ }
1674
+ for (const [event, subscribers] of fleet) {
1675
+ const source = parseLifecycleEvent(event);
1676
+ if (source && subscribers.has(workflow)) {
1677
+ link(startedBy, source.workflow, source.kind, undefined);
1678
+ }
1679
+ }
1680
+ const sorted = (links) => [...links.values()].sort((a, b) => a.workflow.localeCompare(b.workflow) || a.kind.localeCompare(b.kind));
1681
+ return { workflow, startsOnSettle: sorted(startsOnSettle), startedBy: sorted(startedBy) };
1682
+ }
1683
+ /**
1684
+ * `patch` (a terminal status write of `run`) plus the {@link LIFECYCLE_PENDING_TAG} marker and one
1685
+ * {@link LIFECYCLE_OWED_TAG_PREFIX} tag per subscriber of that ending, and the run's span as a
1686
+ * {@link LIFECYCLE_TRACE_TAG_PREFIX} tag when a run trace provider knows it. Written in the same
1687
+ * update as the status, so a run can never be settled without a record of whom its lifecycle event
1688
+ * is owed to.
1689
+ *
1690
+ * While this instance has no successful read of the fleet's announced subscriptions (none yet, or
1691
+ * the last one failed), it cannot know who else subscribes, so the marker is the pending tag alone:
1692
+ * the run's owed set is unresolved, and the publish that next reads the fleet resolves it (see
1693
+ * {@link publishSettled}). Never waits on that read.
1694
+ */
1695
+ withLifecycleMarker(run, patch) {
1696
+ const kind = lifecycleKindOf(patch.status);
1697
+ if (!kind || run.workflow.startsWith('__'))
1698
+ return patch;
1699
+ const subscribers = this.lifecycleSubscribers(run.workflow, kind, normalizeNamespace(run.namespace));
1700
+ const unresolved = !this.announcedSubscriptionsSnapshot?.ok;
1701
+ if (subscribers.length === 0 && !unresolved)
1702
+ return patch;
1703
+ const tags = patch.tags ?? run.tags ?? [];
1704
+ let traceparent;
1705
+ try {
1706
+ traceparent = this.runTraceparent?.(run.id);
1707
+ }
1708
+ catch {
1709
+ traceparent = undefined;
1710
+ }
1711
+ const recordsTrace = traceparent !== undefined &&
1712
+ subscribers.length > 0 &&
1713
+ !tags.some((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX));
1714
+ const marker = [
1715
+ LIFECYCLE_PENDING_TAG,
1716
+ ...(unresolved ? [] : subscribers.map(lifecycleOwedTag)),
1717
+ ...(recordsTrace ? [`${LIFECYCLE_TRACE_TAG_PREFIX}${traceparent}`] : []),
1718
+ ];
1719
+ if (marker.every((tag) => tags.includes(tag)))
1720
+ return patch;
1721
+ return { ...patch, tags: [...new Set([...tags, ...marker])] };
1722
+ }
1723
+ /** Answers the traceparent of a run's span in this process; see {@link setRunTraceparent}. */
1724
+ runTraceparent;
1725
+ /**
1726
+ * Install the function that answers the W3C `traceparent` of a run's span in this process, or
1727
+ * `undefined` to remove it. `attachDurableOtel` installs one. With it, a run some workflow
1728
+ * subscribes to records its traceparent (a {@link LIFECYCLE_TRACE_TAG_PREFIX} tag) with its first
1729
+ * suspend (see {@link withRunTrace}), or else with its terminal write, and the `run.started` of a
1730
+ * run a lifecycle event started carries its source run's as `sourceTraceparent`, so the
1731
+ * subscriber's trace links to the source's whichever process ran either.
1732
+ */
1733
+ setRunTraceparent(provider) {
1734
+ this.runTraceparent = provider;
1735
+ }
1736
+ /**
1737
+ * `patch` (a suspend write of `run`) plus the run's {@link LIFECYCLE_TRACE_TAG_PREFIX} tag, when a
1738
+ * run trace provider is installed (see {@link setRunTraceparent}), some workflow subscribes to the
1739
+ * run's ending, the provider answers for the run and the run does not carry the tag yet. The first
1740
+ * turn's suspend records the run's root span this way, so a run that settles in a process that
1741
+ * never saw its first turn still links its subscribers' traces; a workflow nobody subscribes to
1742
+ * gets no tag and no extra read or write. The tags are read again before they are written, so a
1743
+ * tag the turn added is kept.
1744
+ */
1745
+ async withRunTrace(run, patch) {
1746
+ const provider = this.runTraceparent;
1747
+ if (!provider || run.workflow.startsWith('__'))
1748
+ return patch;
1749
+ if ((run.tags ?? []).some((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX)))
1750
+ return patch;
1751
+ const namespace = normalizeNamespace(run.namespace);
1752
+ if (this.lifecycleSubscribers(run.workflow, 'completed', namespace).length === 0 &&
1753
+ this.lifecycleSubscribers(run.workflow, 'failed', namespace).length === 0) {
1754
+ return patch;
1755
+ }
1756
+ let traceparent;
1757
+ try {
1758
+ traceparent = provider(run.id);
1759
+ }
1760
+ catch {
1761
+ return patch;
1762
+ }
1763
+ if (!traceparent)
1764
+ return patch;
1765
+ const tags = (await this.store.getRun(run.id))?.tags ?? run.tags ?? [];
1766
+ if (tags.some((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX)))
1767
+ return patch;
1768
+ return { ...patch, tags: [...tags, `${LIFECYCLE_TRACE_TAG_PREFIX}${traceparent}`] };
1769
+ }
1770
+ /**
1771
+ * `{ sourceTraceparent }` for the `run.started` of `run`, when a lifecycle event started it and a
1772
+ * run trace provider is installed: the source run's recorded traceparent, else this process's.
1773
+ */
1774
+ async sourceTraceOf(run) {
1775
+ const provider = this.runTraceparent;
1776
+ if (!provider)
1777
+ return {};
1778
+ const source = lifecycleSourceOf(run);
1779
+ if (!source)
1780
+ return {};
1781
+ try {
1782
+ const recorded = lifecycleTraceparentOf((await this.store.getRun(source.runId))?.tags);
1783
+ const traceparent = recorded ?? provider(source.runId);
1784
+ return traceparent ? { sourceTraceparent: traceparent } : {};
1785
+ }
1786
+ catch {
1787
+ return {};
1788
+ }
1789
+ }
1790
+ /** How many registry rows one page of a read of the lifecycle subscription registry lists. */
1791
+ static LIFECYCLE_REGISTRY_PAGE_SIZE = 1000;
1792
+ /** How long a read of the subscriptions the fleet announces is reused. */
1793
+ static ANNOUNCED_SUBSCRIPTIONS_TTL_MS = 5_000;
1794
+ /** The last read of the fleet's announced subscriptions; see {@link announcedSubscribers}. */
1795
+ announcedSubscriptionsSnapshot;
1796
+ announcedSubscriptionsRead;
1797
+ /**
1798
+ * The workflows the fleet announces as subscribers of `event`: the live rows of the lifecycle
1799
+ * subscription registry (see {@link LIFECYCLE_REGISTRY_WORKFLOW}) and the live worker descriptors'
1800
+ * `registrations[].subscribes`, from a snapshot refreshed in the background once it is older than
1801
+ * {@link ANNOUNCED_SUBSCRIPTIONS_TTL_MS}. Synchronous, so a terminal status write never waits on
1802
+ * the fleet; before the first read completes it answers with nothing.
1803
+ */
1804
+ announcedSubscribers(event, namespace) {
1805
+ const snapshot = this.announcedSubscriptionsSnapshot;
1806
+ if (!snapshot || this.clock() - snapshot.at > WorkflowEngine.ANNOUNCED_SUBSCRIPTIONS_TTL_MS) {
1807
+ void this.refreshAnnouncedSubscriptions();
1808
+ }
1809
+ return this.announcedFor(event, namespace);
1810
+ }
1811
+ /**
1812
+ * The subscribers of `event` in the current snapshot that serve `namespace`'s runs: announced for
1813
+ * that namespace, or by an engine with no namespace (see {@link LifecycleRegistryEntry}). Every
1814
+ * announced subscriber when `namespace` is undefined. `node` keeps only the ones Node engines
1815
+ * announce (registry rows and Node worker descriptors), which start their own subscribers.
1816
+ */
1817
+ announcedFor(event, namespace, opts) {
1818
+ const snapshot = this.announcedSubscriptionsSnapshot;
1819
+ if (!snapshot?.byEvent.has(event))
1820
+ return new Set();
1821
+ const scope = namespace === undefined ? undefined : normalizeNamespace(namespace);
1822
+ const out = new Set();
1823
+ for (const entry of snapshot.entries) {
1824
+ if (entry.event !== event || (opts?.node && !entry.node))
1825
+ continue;
1826
+ if (scope !== undefined && entry.namespace !== null && entry.namespace !== scope)
1827
+ continue;
1828
+ out.add(entry.workflow);
1829
+ }
1830
+ return out;
1831
+ }
1832
+ /**
1833
+ * Read the fleet's announced subscriptions into the snapshot: the registry rows whose heartbeat
1834
+ * is at most {@link LIFECYCLE_REGISTRY_TTL_MS} old (a range of the store's `created_at` index, read
1835
+ * page by page until a short page, so a large fleet is never cut off), and
1836
+ * the worker descriptors on a transport that can read them fleet-wide. Never throws: a failed read
1837
+ * of either leaves an empty snapshot marked not `ok`, which counts as the local registry only and
1838
+ * never orphans a subscriber. Concurrent callers share one read.
1839
+ */
1840
+ refreshAnnouncedSubscriptions() {
1841
+ this.announcedSubscriptionsRead ??= (async () => {
1842
+ const at = this.clock();
1843
+ const byEvent = new Map();
1844
+ const entries = [];
1845
+ const add = (event, workflow, node, namespace) => {
1846
+ entries.push({ event, workflow, namespace, node });
1847
+ const subscribers = byEvent.get(event) ?? new Set();
1848
+ subscribers.add(workflow);
1849
+ byEvent.set(event, subscribers);
1850
+ };
1851
+ const readRegistry = async () => {
1852
+ const size = WorkflowEngine.LIFECYCLE_REGISTRY_PAGE_SIZE;
1853
+ const out = [];
1854
+ for (let page = 1;; page += 1) {
1855
+ const rows = await asHeartbeat(() => this.store.listRuns({
1856
+ workflow: LIFECYCLE_REGISTRY_WORKFLOW,
1857
+ createdAfter: at - LIFECYCLE_REGISTRY_TTL_MS,
1858
+ // A namespaced engine also reads `default`, where an engine with no namespace writes
1859
+ // its row: those subscribers are owed this namespace's runs too.
1860
+ ...(this.namespace !== undefined
1861
+ ? { namespaces: [...new Set([this.namespace, 'default'])] }
1862
+ : {}),
1863
+ page,
1864
+ size,
1865
+ }));
1866
+ out.push(...rows);
1867
+ if (rows.length < size)
1868
+ return out;
1869
+ }
1870
+ };
1871
+ try {
1872
+ const [descriptors, rows] = await Promise.all([
1873
+ this.pool.introspectsFleet ? this.pool.readAllWorkerDescriptors() : [],
1874
+ readRegistry(),
1875
+ ]);
1876
+ for (const row of rows) {
1877
+ if (row.workflow !== LIFECYCLE_REGISTRY_WORKFLOW)
1878
+ continue;
1879
+ if (at - row.updatedAt.getTime() > LIFECYCLE_REGISTRY_TTL_MS)
1880
+ continue;
1881
+ const namespace = lifecycleRegistryNamespace(row.input, row.namespace);
1882
+ for (const { workflow, events } of lifecycleRegistrySubscriptions(row.input)) {
1883
+ for (const event of events)
1884
+ add(event, workflow, true, namespace);
1885
+ }
1886
+ }
1887
+ for (const descriptor of descriptors) {
1888
+ for (const announced of announcementsOf(descriptor)) {
1889
+ const namespace = servedNamespace(descriptor.namespace, announced.group);
1890
+ for (const event of announced.subscribes ?? []) {
1891
+ add(event, announced.name, descriptor.runtime === 'node', namespace);
1892
+ }
1893
+ }
1894
+ }
1895
+ this.announcedSubscriptionsSnapshot = { at, ok: true, byEvent, entries };
1896
+ }
1897
+ catch {
1898
+ this.announcedSubscriptionsSnapshot = { at, ok: false, byEvent: new Map(), entries: [] };
1899
+ }
1900
+ })().finally(() => {
1901
+ this.announcedSubscriptionsRead = undefined;
1902
+ });
1903
+ return this.announcedSubscriptionsRead;
1904
+ }
1905
+ /** Read the fleet's announced subscriptions again when the snapshot is older than the TTL. */
1906
+ async freshAnnouncedSubscriptions() {
1907
+ const snapshot = this.announcedSubscriptionsSnapshot;
1908
+ if (!snapshot || this.clock() - snapshot.at > WorkflowEngine.ANNOUNCED_SUBSCRIPTIONS_TTL_MS) {
1909
+ await this.refreshAnnouncedSubscriptions();
1910
+ }
1911
+ }
1912
+ /** When this engine last wrote its registry row; see {@link heartbeatLifecycleRegistry}. */
1913
+ lifecycleRegistryBeatAt;
1914
+ /** When this engine last deleted stale registry rows; undefined before its first pass. */
1915
+ lifecycleRegistryCleanedAt;
1916
+ /** How often an engine with a registry row looks for stale rows to delete. */
1917
+ static LIFECYCLE_REGISTRY_CLEANUP_MS = 60_000;
1918
+ /**
1919
+ * This engine's lifecycle registry entry: each workflow registered here that subscribes to an
1920
+ * event, with the events it subscribes to. Undefined when none does.
1921
+ */
1922
+ lifecycleRegistryEntry() {
1923
+ const subscribes = new Map();
1924
+ for (const [event, subscribers] of this.eventTriggers) {
1925
+ for (const workflow of subscribers.keys()) {
1926
+ if (workflow.startsWith('__'))
1927
+ continue;
1928
+ const events = subscribes.get(workflow) ?? new Set();
1929
+ events.add(event);
1930
+ subscribes.set(workflow, events);
1931
+ }
1932
+ }
1933
+ if (subscribes.size === 0)
1934
+ return undefined;
1935
+ return {
1936
+ instanceId: this.instanceId,
1937
+ runtime: 'node',
1938
+ engineNamespace: this.namespace ?? null,
1939
+ subscriptions: [...subscribes.keys()].sort().map((workflow) => ({
1940
+ workflow,
1941
+ events: [...(subscribes.get(workflow) ?? [])].sort(),
1942
+ })),
1943
+ };
1944
+ }
1945
+ /**
1946
+ * Write this engine's row of the lifecycle subscription registry (see
1947
+ * {@link LIFECYCLE_REGISTRY_WORKFLOW}), at most once per {@link LIFECYCLE_REGISTRY_HEARTBEAT_MS}:
1948
+ * an update of the existing row, or a create when it is missing (first beat, or deleted as stale
1949
+ * after a long pause). Once a minute it also deletes rows whose heartbeat stopped
1950
+ * {@link LIFECYCLE_REGISTRY_STALE_MS} ago: the ones that went stale in the last
1951
+ * {@link LIFECYCLE_REGISTRY_STALE_MS}, a bounded range of the `created_at` index, and on its first
1952
+ * pass every stale row, for the ones left while no engine ran. The row is created tagged
1953
+ * {@link LIFECYCLE_REGISTRY_TAG}, which retention keeps. A no-op for an engine without
1954
+ * subscriptions. Its store calls are marked as heartbeats for observability tools. Never throws.
1955
+ */
1956
+ async heartbeatLifecycleRegistry() {
1957
+ const nowMs = this.clock();
1958
+ if (this.lifecycleRegistryBeatAt !== undefined &&
1959
+ nowMs - this.lifecycleRegistryBeatAt < LIFECYCLE_REGISTRY_HEARTBEAT_MS) {
1960
+ return;
1961
+ }
1962
+ const entry = this.lifecycleRegistryEntry();
1963
+ if (!entry)
1964
+ return;
1965
+ this.lifecycleRegistryBeatAt = nowMs;
1966
+ const id = lifecycleRegistryRunId(this.instanceId);
1967
+ const at = new Date(nowMs);
1968
+ try {
1969
+ await asHeartbeat(async () => {
1970
+ const beat = { input: entry, createdAt: at, updatedAt: at };
1971
+ if (await this.store.updateRunIf(id, ['completed'], beat))
1972
+ return;
1973
+ try {
1974
+ await this.store.createRun({
1975
+ id,
1976
+ workflow: LIFECYCLE_REGISTRY_WORKFLOW,
1977
+ workflowVersion: '1',
1978
+ status: 'completed',
1979
+ input: entry,
1980
+ tags: [LIFECYCLE_REGISTRY_TAG],
1981
+ ...(this.namespace !== undefined ? { namespace: this.namespace } : {}),
1982
+ createdAt: at,
1983
+ updatedAt: at,
1984
+ });
1985
+ }
1986
+ catch {
1987
+ // Created by a concurrent beat under the same instance id.
1988
+ await this.store.updateRun(id, beat);
1989
+ }
1990
+ });
1991
+ }
1992
+ catch (error) {
1993
+ console.error('[adonis-durable] could not write the lifecycle subscription registry:', error);
1994
+ }
1995
+ const cleanedAt = this.lifecycleRegistryCleanedAt;
1996
+ if (cleanedAt !== undefined &&
1997
+ nowMs - cleanedAt < WorkflowEngine.LIFECYCLE_REGISTRY_CLEANUP_MS) {
1998
+ return;
1999
+ }
2000
+ this.lifecycleRegistryCleanedAt = nowMs;
2001
+ const staleBefore = nowMs - LIFECYCLE_REGISTRY_STALE_MS;
2002
+ try {
2003
+ await asHeartbeat(async () => {
2004
+ const stale = await this.store.listRuns({
2005
+ workflow: LIFECYCLE_REGISTRY_WORKFLOW,
2006
+ ...(this.namespace !== undefined ? { namespace: this.namespace } : {}),
2007
+ createdBefore: staleBefore,
2008
+ ...(cleanedAt !== undefined
2009
+ ? { createdAfter: staleBefore - LIFECYCLE_REGISTRY_STALE_MS }
2010
+ : {}),
2011
+ size: 100,
2012
+ });
2013
+ for (const row of stale) {
2014
+ if (row.workflow === LIFECYCLE_REGISTRY_WORKFLOW && row.id !== id) {
2015
+ await this.store.deleteRun(row.id);
2016
+ }
2017
+ }
2018
+ });
2019
+ }
2020
+ catch (error) {
2021
+ console.error('[adonis-durable] could not prune the lifecycle subscription registry:', error);
2022
+ }
2023
+ }
2024
+ /**
2025
+ * The runs started by `runId` settling (see {@link publishLifecycle}), read from the run's own
2026
+ * tags, so it needs no subscriber registry. Empty for a run that has not settled or started
2027
+ * nobody. Pass the run itself when it is already loaded, to skip reading it again.
2028
+ */
2029
+ async triggeredRuns(source) {
2030
+ const run = typeof source === 'string' ? await this.store.getRun(source) : source;
2031
+ return run ? triggeredRunsOf(run) : [];
2032
+ }
2033
+ /** The live path of {@link publishSettled}: right after this instance settled the run. */
2034
+ publishLifecycle(event) {
2035
+ if (event.type !== 'run.completed' && event.type !== 'run.failed')
2036
+ return;
2037
+ const kind = event.type === 'run.completed' ? 'completed' : 'failed';
2038
+ if (!event.workflow || event.workflow.startsWith('__'))
2039
+ return;
2040
+ const name = lifecycleEventName(event.workflow, kind);
2041
+ const snapshot = this.announcedSubscriptionsSnapshot;
2042
+ // Without a successful read of the fleet the terminal write left the owed set unresolved, and
2043
+ // this publish resolves it.
2044
+ if (snapshot?.ok &&
2045
+ this.localSubscribers(name).length === 0 &&
2046
+ !snapshot.byEvent.get(name)?.size) {
2047
+ return;
2048
+ }
2049
+ // The turn that settled the run still holds its lease: publish once it let go.
2050
+ const turns = [...this.inflight].filter(([, runId]) => runId === event.runId).map(([p]) => p);
2051
+ void (async () => {
2052
+ await Promise.allSettled(turns);
2053
+ const run = await this.store.getRun(event.runId);
2054
+ if (run)
2055
+ await this.publishSettled(run);
2056
+ })().catch(() => undefined);
2057
+ }
2058
+ /**
2059
+ * Take the run lease for this instance's turn, as `tryLockRun` at `nowMs`; when the lease is held
2060
+ * by a lifecycle publish (see {@link withLifecycleLease}), which only ever holds a settled run for
2061
+ * the length of its publish (an explicit retry can land in it), wait for that publish to let go,
2062
+ * at most `leaseMs`.
2063
+ */
2064
+ async lockRunPastLifecyclePublish(runId, nowMs) {
2065
+ if (await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
2066
+ return true;
2067
+ }
2068
+ // Real time, not `this.clock`: a test clock may stand still.
2069
+ const giveUpAt = Date.now() + this.leaseMs;
2070
+ let delayMs = 10;
2071
+ for (;;) {
2072
+ const held = await this.store.getRun(runId);
2073
+ if (!held?.lockedBy?.includes(LIFECYCLE_LEASE_MARK) || Date.now() >= giveUpAt)
2074
+ return false;
2075
+ await new Promise((r) => setTimeout(r, delayMs));
2076
+ delayMs = Math.min(delayMs * 2, 250);
2077
+ const now = this.clock();
2078
+ if (await this.store.tryLockRun(runId, this.instanceId, now + this.leaseMs, now))
2079
+ return true;
2080
+ }
2081
+ }
2082
+ /** The waits between attempts to take a settled run's lifecycle lease (see {@link withLifecycleLease}). */
2083
+ static LIFECYCLE_LEASE_RETRY_MS = [50, 100, 200, 400, 800];
2084
+ /**
2085
+ * Run `fn` holding the run lease of the settled run `runId` under an owner unique to this call, so
2086
+ * every read-start-write of a run's lifecycle tags (live publish, sweeps, heal, republish, on any
2087
+ * instance or twice in this one) runs one at a time. A refused lease is asked for again after each
2088
+ * of {@link LIFECYCLE_LEASE_RETRY_MS}; still refused, resolves `{ acquired: false }` without
2089
+ * running `fn`. The lease is released when `fn` settles, and expires after `leaseMs` if this
2090
+ * process dies first.
2091
+ */
2092
+ async withLifecycleLease(runId, fn) {
2093
+ const owner = `${this.instanceId}${LIFECYCLE_LEASE_MARK}${globalThis.crypto.randomUUID()}`;
2094
+ const delays = WorkflowEngine.LIFECYCLE_LEASE_RETRY_MS;
2095
+ for (let attempt = 0;; attempt += 1) {
2096
+ const nowMs = this.clock();
2097
+ if (await this.store.tryLockRun(runId, owner, nowMs + this.leaseMs, nowMs))
2098
+ break;
2099
+ const delay = delays[attempt];
2100
+ if (delay === undefined)
2101
+ return { acquired: false };
2102
+ await new Promise((resolve) => setTimeout(resolve, delay));
2103
+ }
2104
+ try {
2105
+ return { acquired: true, value: await fn() };
2106
+ }
2107
+ finally {
2108
+ await this.store.releaseRunLock(runId, owner).catch(() => undefined);
2109
+ }
2110
+ }
2111
+ /**
2112
+ * Publish a settled run's lifecycle event (`run.completed:<workflow>` / `run.failed:<workflow>`)
2113
+ * to the subscribers it owes it to that this instance can deliver to (registered here, or announced
2114
+ * and started by name; see {@link deliverableSubscribers}), and settle their part of the marker:
2115
+ * drop each one's {@link LIFECYCLE_OWED_TAG_PREFIX} tag, record each started one as a
2116
+ * `durable:triggered:<workflow>` tag (a coalescing one as `durable:coalesced:<workflow>`), and drop
2117
+ * {@link LIFECYCLE_PENDING_TAG} once no owed tag is left.
2118
+ *
2119
+ * A run marked pending with no owed tag has an unresolved owed set (see
2120
+ * {@link LIFECYCLE_PENDING_TAG}). With a successful read of the fleet's subscriptions it is resolved
2121
+ * here: every subscriber of the ending (see {@link lifecycleSubscribers}) the run's tags do not
2122
+ * record as started or failed becomes owed, and the ones nobody here delivers to keep an owed tag.
2123
+ * Without one, the subscribers registered here are started and the run stays unresolved.
2124
+ *
2125
+ * `republish` publishes to every subscriber of the ending (or only `subscriber`, which must be one),
2126
+ * owed or not: the ones this instance can deliver to are started, and the others are marked owed
2127
+ * again (with the pending tag) for the sweep of an instance that can. `heal` names subscribers of
2128
+ * the ending the run was never marked for (see {@link healLifecycleEvents}): they are owed now, and
2129
+ * the ones this instance delivers to are started.
2130
+ * `nowMs` (the sweep's clock) also drops the owed tag of a subscriber that is neither registered
2131
+ * here nor announced by a live worker once the run settled {@link LIFECYCLE_ORPHAN_MS} ago,
2132
+ * reporting it as `run.trigger_failed`.
2133
+ *
2134
+ * Runs under the run's lifecycle lease (see {@link withLifecycleLease}) and reads the run again
2135
+ * under it, so concurrent publishers never overwrite each other's tags; `busy` when the lease
2136
+ * stayed taken. Idempotent, since every subscriber run id is deterministic and a debounce/batch
2137
+ * accumulator drops an event it already holds. A subscriber whose `where` throws, whose input is
2138
+ * rejected or whose start fails, and an orphan the sweep drops, gets a
2139
+ * {@link LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX} tag in the same tag write, and is not retried until a
2140
+ * republish; the write that records a subscriber reached removes its trigger-failed tag.
2141
+ * A debounce/batch subscriber routed into its accumulator gets a
2142
+ * {@link LIFECYCLE_COALESCED_TAG_PREFIX} tag in the same write instead of a triggered one; a
2143
+ * republish routes it again only when the run also carries its trigger-failed tag.
2144
+ * `run.triggered` and `run.trigger_failed` are emitted only for the subscribers whose triggered,
2145
+ * coalesced or trigger-failed tag this write added. A failure to write the marker leaves it, and
2146
+ * the sweep publishes the run again.
2147
+ */
2148
+ async publishSettled(source, opts) {
2149
+ const leased = await this.withLifecycleLease(source.id, async () => {
2150
+ const run = await this.store.getRun(source.id);
2151
+ return run
2152
+ ? this.publishSettledLeased(run, opts)
2153
+ : { started: 0, failed: [] };
2154
+ });
2155
+ return leased.acquired ? leased.value : { started: 0, failed: [], busy: true };
2156
+ }
2157
+ /** {@link publishSettled} once it holds the run's lifecycle lease and has read `run` under it. */
2158
+ async publishSettledLeased(run, opts) {
2159
+ const kind = lifecycleKindOf(run.status);
2160
+ if (!kind)
2161
+ return { started: 0, failed: [] };
2162
+ const name = lifecycleEventName(run.workflow, kind);
2163
+ const namespace = normalizeNamespace(run.namespace);
2164
+ const before = run.tags ?? [];
2165
+ const delivered = deliveredSubscribersOf(run);
2166
+ let owed = owedSubscribersOf(run);
2167
+ const pending = before.includes(LIFECYCLE_PENDING_TAG);
2168
+ // Subscribers this write marks owed: a resolved or healed owed set, or a republish's re-mark.
2169
+ let newlyOwed = [];
2170
+ let unresolved = false;
2171
+ if (pending && owed.length === 0 && !opts?.republish) {
2172
+ await this.freshAnnouncedSubscriptions();
2173
+ if (this.announcedSubscriptionsSnapshot?.ok) {
2174
+ owed = this.lifecycleSubscribers(run.workflow, kind, namespace).filter((s) => !delivered.has(s));
2175
+ newlyOwed = owed;
2176
+ }
2177
+ else {
2178
+ unresolved = true;
2179
+ }
2180
+ }
2181
+ if (opts?.heal?.length) {
2182
+ const healed = opts.heal.filter((s) => !owed.includes(s) && !delivered.has(s));
2183
+ owed = [...owed, ...healed];
2184
+ newlyOwed = [...newlyOwed, ...healed];
2185
+ }
2186
+ const { local, remote, all: here } = await this.deliverableSubscribers(name, namespace);
2187
+ let targets;
2188
+ const refused = [];
2189
+ if (opts?.republish) {
2190
+ await this.freshAnnouncedSubscriptions();
2191
+ // Without a read of the fleet, an unresolved run stays unresolved for the next publish.
2192
+ unresolved = pending && owed.length === 0 && !this.announcedSubscriptionsSnapshot?.ok;
2193
+ const subscribers = this.lifecycleSubscribers(run.workflow, kind, namespace);
2194
+ let all = subscribers;
2195
+ if (opts.subscriber !== undefined) {
2196
+ all = subscribers.includes(opts.subscriber) ? [opts.subscriber] : [];
2197
+ if (all.length === 0) {
2198
+ refused.push({
2199
+ workflow: opts.subscriber,
2200
+ error: new Error(`subscriber ${opts.subscriber} does not subscribe to ${name}`),
2201
+ });
2202
+ }
2203
+ }
2204
+ // An accumulator that already fired would start its target again, so a subscriber the run's
2205
+ // tags record as coalesced is routed again (or owed again) only after a failed delivery.
2206
+ const settledCoalesced = new Set(coalescedSubscribersOf(before).filter((s) => !before.includes(lifecycleTriggerFailedTag(s))));
2207
+ all = all.filter((s) => !settledCoalesced.has(s));
2208
+ targets = all.filter((s) => here.has(s));
2209
+ newlyOwed = all.filter((s) => !here.has(s));
2210
+ }
2211
+ else if (unresolved) {
2212
+ targets = [...local].filter((s) => !delivered.has(s));
2213
+ }
2214
+ else {
2215
+ targets = owed.filter((s) => here.has(s));
2216
+ }
2217
+ const { reached, failed } = targets.length > 0
2218
+ ? await this.startSubscribers(name, lifecycleEventOf(run), (sub) => lifecycleRunId(run.id, kind, sub), new Set(targets), remote)
2219
+ : { reached: [], failed: [] };
2220
+ const orphaned = opts?.nowMs !== undefined ? await this.orphanedSubscribers(run, owed, here, opts.nowMs) : [];
2221
+ const done = new Set([...targets, ...orphaned]);
2222
+ // A started subscriber is recorded by its triggered tag, one routed into its accumulator by its
2223
+ // coalesced tag.
2224
+ const deliveryTag = (r) => r.runId !== undefined
2225
+ ? `${LIFECYCLE_TRIGGERED_TAG_PREFIX}${r.workflow}`
2226
+ : lifecycleCoalescedTag(r.workflow);
2227
+ const deliveryTags = reached.map(deliveryTag);
2228
+ const reachedNow = new Set(reached.map((r) => r.workflow));
2229
+ const failedNow = [...failed.map((f) => f.workflow), ...orphaned];
2230
+ const kept = [
2231
+ ...before.filter((t) => !((t.startsWith(LIFECYCLE_OWED_TAG_PREFIX) &&
2232
+ done.has(t.slice(LIFECYCLE_OWED_TAG_PREFIX.length))) ||
2233
+ (t.startsWith(LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX) &&
2234
+ reachedNow.has(t.slice(LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX.length))))),
2235
+ ...newlyOwed.filter((s) => !done.has(s)).map(lifecycleOwedTag),
2236
+ ];
2237
+ const owedLeft = kept.some((t) => t.startsWith(LIFECYCLE_OWED_TAG_PREFIX));
2238
+ const tags = [
2239
+ ...new Set([
2240
+ ...kept.filter((t) => t !== LIFECYCLE_PENDING_TAG),
2241
+ ...(owedLeft || unresolved ? [LIFECYCLE_PENDING_TAG] : []),
2242
+ ...deliveryTags,
2243
+ ...failedNow.map(lifecycleTriggerFailedTag),
2244
+ ]),
2245
+ ];
2246
+ if (tags.length !== before.length || tags.some((t) => !before.includes(t))) {
2247
+ await this.store.updateRun(run.id, { tags });
2248
+ }
2249
+ // Only the tag changes this write made are announced, so a publisher that repeats another's
2250
+ // delivery emits nothing.
2251
+ for (const r of reached) {
2252
+ if (before.includes(deliveryTag(r)))
2253
+ continue;
2254
+ const { workflow, runId } = r;
2255
+ this.emit({
2256
+ type: 'run.triggered',
2257
+ runId: run.id,
2258
+ workflow: run.workflow,
2259
+ namespace: run.namespace,
2260
+ name: workflow,
2261
+ triggeredRunId: runId,
2262
+ });
2263
+ }
2264
+ const unreached = [
2265
+ ...failed,
2266
+ ...orphaned.map((workflow) => ({
2267
+ workflow,
2268
+ error: new Error(`subscriber ${workflow} is not registered on any live worker`),
2269
+ })),
2270
+ ];
2271
+ for (const { workflow, error } of unreached) {
2272
+ if (before.includes(lifecycleTriggerFailedTag(workflow)))
2273
+ continue;
2274
+ this.emit({
2275
+ type: 'run.trigger_failed',
2276
+ runId: run.id,
2277
+ workflow: run.workflow,
2278
+ namespace: run.namespace,
2279
+ name: workflow,
2280
+ error: { message: error instanceof Error ? error.message : String(error) },
2281
+ });
2282
+ }
2283
+ return {
2284
+ started: reached.filter((r) => r.created).length,
2285
+ failed: [...refused, ...unreached],
2286
+ };
2287
+ }
2288
+ /**
2289
+ * The subscribers `run` owes its event to that nothing can deliver it to: not in `here` (what
2290
+ * this instance delivers to), announced by no live worker, and owed since
2291
+ * {@link LIFECYCLE_ORPHAN_MS} or longer. Empty while the fleet's announcements cannot be read,
2292
+ * since then nobody is known to be gone.
2293
+ */
2294
+ async orphanedSubscribers(run, owed, here, nowMs) {
2295
+ if (nowMs - run.updatedAt.getTime() < LIFECYCLE_ORPHAN_MS)
2296
+ return [];
2297
+ const candidates = owed.filter((s) => !here.has(s));
2298
+ if (candidates.length === 0)
2299
+ return [];
2300
+ await this.freshAnnouncedSubscriptions();
2301
+ const fresh = this.announcedSubscriptionsSnapshot;
2302
+ if (!fresh?.ok)
2303
+ return [];
2304
+ const kind = lifecycleKindOf(run.status);
2305
+ if (!kind)
2306
+ return [];
2307
+ const announced = this.announcedFor(lifecycleEventName(run.workflow, kind), normalizeNamespace(run.namespace));
2308
+ return candidates.filter((s) => !announced.has(s));
2309
+ }
2310
+ /** How long a settled run's marker is left to the live publish before the sweep takes it over. */
2311
+ static LIFECYCLE_SWEEP_GRACE_MS = 30_000;
2312
+ /** How many marked runs one lifecycle sweep reads at most, looking for ones it can act on. */
2313
+ static LIFECYCLE_SWEEP_SCAN_CAP = 1000;
2314
+ /**
2315
+ * Publish the lifecycle event of settled runs still marked {@link LIFECYCLE_PENDING_TAG} to the
2316
+ * subscribers this instance can deliver to that they still owe it to: a run whose settling process
2317
+ * died, or whose marker write failed, before publishing, or whose subscriber is registered only on
2318
+ * other instances. Also writes this instance's row of the lifecycle subscription registry (see
2319
+ * {@link heartbeatLifecycleRegistry}), refreshes the snapshot of the subscriptions the fleet
2320
+ * announces (registry rows and worker descriptors), drops
2321
+ * owed subscribers nothing can deliver to (see {@link orphanedSubscribers}) and wakes the
2322
+ * `ctx.waitForRun` waiters whose run already settled (see {@link sweepRunWaits}).
2323
+ *
2324
+ * Called by the worker poll loop. Reads the marked runs page by page and skips the ones it cannot
2325
+ * act on (every owed subscriber is delivered elsewhere and none is orphaned yet), so runs owed only
2326
+ * to other instances never hide the ones it can publish. Stops once it has published `limit` runs
2327
+ * or read {@link LIFECYCLE_SWEEP_SCAN_CAP} marked runs. A run settled in the last
2328
+ * {@link LIFECYCLE_SWEEP_GRACE_MS} is left to the live publish; a run another publisher holds
2329
+ * (see {@link withLifecycleLease}) is left to the next tick. Then heals the runs settled in the
2330
+ * last {@link LIFECYCLE_HEAL_WINDOW_MS} (see {@link healLifecycleEvents}). Returns how many marked
2331
+ * runs it published plus how many waiters it woke; healed runs are not counted.
2332
+ */
2333
+ async sweepLifecycleEvents(nowMs = this.clock(), limit = 100) {
2334
+ await this.heartbeatLifecycleRegistry();
2335
+ // A failed read is tried again on every tick, so unresolved runs resolve once the fleet is back.
2336
+ if (this.announcedSubscriptionsSnapshot?.ok === false)
2337
+ await this.refreshAnnouncedSubscriptions();
2338
+ else
2339
+ await this.freshAnnouncedSubscriptions();
2340
+ const actionable = await this.lifecycleActionableRuns(nowMs, limit);
2341
+ let published = 0;
2342
+ for (const run of actionable) {
2343
+ try {
2344
+ const { busy } = await this.publishSettled(run, { nowMs });
2345
+ if (!busy)
2346
+ published += 1;
2347
+ }
2348
+ catch (error) {
2349
+ console.error(`[adonis-durable] lifecycle sweep: run ${run.id} failed; retrying next tick:`, error);
2350
+ }
2351
+ }
2352
+ try {
2353
+ await this.healLifecycleEvents(nowMs);
2354
+ }
2355
+ catch (error) {
2356
+ console.error('[adonis-durable] lifecycle sweep: heal failed; retrying next tick:', error);
2357
+ }
2358
+ return published + (await this.sweepRunWaits(limit));
2359
+ }
2360
+ /** How many settled runs of one source workflow a heal pass reads at most. */
2361
+ static LIFECYCLE_HEAL_SCAN_CAP = 100;
2362
+ /**
2363
+ * The (run, subscriber) pairs this engine already healed, with when each entry lapses (epoch ms):
2364
+ * a subscriber whose `where` declines the event leaves no tag, so without it every sweep would
2365
+ * offer it the event again for the whole heal window.
2366
+ */
2367
+ lifecycleHealed = new Map();
2368
+ /**
2369
+ * Deliver lifecycle events the terminal write never marked: for every source workflow some
2370
+ * subscriber in the current snapshot of the fleet's subscriptions (or registered here) listens to,
2371
+ * read its runs settled in the last {@link LIFECYCLE_HEAL_WINDOW_MS} (at most
2372
+ * {@link LIFECYCLE_HEAL_SCAN_CAP}, one listing per workflow) and publish each one to the
2373
+ * subscribers of its ending its tags record as neither owed, started, coalesced nor failed (see
2374
+ * {@link publishSettled}'s `heal`). That covers a subscriber announced just after the run
2375
+ * settled, while the settling instance's snapshot was stale. Each pair is healed at most once per
2376
+ * engine. A run whose owed set is unresolved is left to {@link sweepLifecycleEvents}. Needs a
2377
+ * successful read of the fleet's subscriptions; returns how many runs it published.
2378
+ */
2379
+ async healLifecycleEvents(nowMs) {
2380
+ const snapshot = this.announcedSubscriptionsSnapshot;
2381
+ if (!snapshot?.ok)
2382
+ return 0;
2383
+ for (const [key, lapsesAt] of this.lifecycleHealed) {
2384
+ if (lapsesAt <= nowMs)
2385
+ this.lifecycleHealed.delete(key);
2386
+ }
2387
+ const sources = new Map();
2388
+ for (const event of new Set([...snapshot.byEvent.keys(), ...this.eventTriggers.keys()])) {
2389
+ const source = parseLifecycleEvent(event);
2390
+ if (!source || source.workflow.startsWith('__'))
2391
+ continue;
2392
+ const kinds = sources.get(source.workflow) ?? new Set();
2393
+ kinds.add(source.kind);
2394
+ sources.set(source.workflow, kinds);
2395
+ }
2396
+ let healed = 0;
2397
+ for (const [workflow, kinds] of sources) {
2398
+ const statuses = [
2399
+ ...(kinds.has('completed') ? ['completed'] : []),
2400
+ ...(kinds.has('failed') ? ['failed', 'cancelled', 'dead'] : []),
2401
+ ];
2402
+ const runs = await asHeartbeat(() => this.store.listRuns({
2403
+ workflow,
2404
+ statuses,
2405
+ ...(this.namespace !== undefined ? { namespace: this.namespace } : {}),
2406
+ updatedAfter: nowMs - LIFECYCLE_HEAL_WINDOW_MS,
2407
+ size: WorkflowEngine.LIFECYCLE_HEAL_SCAN_CAP,
2408
+ }));
2409
+ for (const run of runs) {
2410
+ const kind = lifecycleKindOf(run.status);
2411
+ if (run.workflow !== workflow || !kind || !kinds.has(kind))
2412
+ continue;
2413
+ if (nowMs - run.updatedAt.getTime() > LIFECYCLE_HEAL_WINDOW_MS)
2414
+ continue;
2415
+ const owed = owedSubscribersOf(run);
2416
+ if (owed.length === 0 && run.tags?.includes(LIFECYCLE_PENDING_TAG))
2417
+ continue;
2418
+ const delivered = deliveredSubscribersOf(run);
2419
+ const missing = this.lifecycleSubscribers(workflow, kind, normalizeNamespace(run.namespace)).filter((s) => !owed.includes(s) &&
2420
+ !delivered.has(s) &&
2421
+ !this.lifecycleHealed.has(`${run.id}\u0000${s}`));
2422
+ if (missing.length === 0)
2423
+ continue;
2424
+ const { busy } = await this.publishSettled(run, { heal: missing, nowMs });
2425
+ if (busy)
2426
+ continue;
2427
+ for (const s of missing) {
2428
+ this.lifecycleHealed.set(`${run.id}\u0000${s}`, nowMs + LIFECYCLE_HEAL_WINDOW_MS);
2429
+ }
2430
+ healed += 1;
2431
+ }
2432
+ }
2433
+ return healed;
2434
+ }
2435
+ /**
2436
+ * Up to `limit` settled runs marked {@link LIFECYCLE_PENDING_TAG}, past the sweep grace, that this
2437
+ * instance can act on: a run owing its event to a subscriber it delivers to, a run marked without
2438
+ * owed tags, or a run owing it to a subscriber nothing delivers to past
2439
+ * {@link LIFECYCLE_ORPHAN_MS}. Reads at most {@link LIFECYCLE_SWEEP_SCAN_CAP} marked runs, in
2440
+ * pages, all read before any is acted on, since publishing a run takes it out of the listing.
2441
+ */
2442
+ async lifecycleActionableRuns(nowMs, limit) {
2443
+ const size = Math.max(1, Math.min(limit, 100));
2444
+ const deliverable = new Map();
2445
+ const deliverableTo = (event, namespace) => {
2446
+ const key = `${namespace}\u0000${event}`;
2447
+ let entry = deliverable.get(key);
2448
+ if (!entry) {
2449
+ entry = this.deliverableSubscribers(event, namespace).then((d) => d.all);
2450
+ deliverable.set(key, entry);
2451
+ }
2452
+ return entry;
2453
+ };
2454
+ const out = [];
2455
+ const seen = new Set();
2456
+ let scanned = 0;
2457
+ for (let page = 1; out.length < limit && scanned < WorkflowEngine.LIFECYCLE_SWEEP_SCAN_CAP;) {
2458
+ const runs = await asHeartbeat(() => this.store.listRuns({
2459
+ tag: LIFECYCLE_PENDING_TAG,
2460
+ statuses: ['completed', 'failed', 'cancelled', 'dead'],
2461
+ ...(this.namespace !== undefined ? { namespace: this.namespace } : {}),
2462
+ updatedBefore: nowMs - WorkflowEngine.LIFECYCLE_SWEEP_GRACE_MS,
2463
+ page,
2464
+ size,
2465
+ }));
2466
+ for (const run of runs) {
2467
+ if (out.length >= limit || scanned >= WorkflowEngine.LIFECYCLE_SWEEP_SCAN_CAP)
2468
+ break;
2469
+ scanned += 1;
2470
+ if (seen.has(run.id))
2471
+ continue;
2472
+ seen.add(run.id);
2473
+ if (nowMs - run.updatedAt.getTime() < WorkflowEngine.LIFECYCLE_SWEEP_GRACE_MS)
2474
+ continue;
2475
+ const kind = lifecycleKindOf(run.status);
2476
+ if (!kind)
2477
+ continue;
2478
+ const owed = owedSubscribersOf(run);
2479
+ if (owed.length === 0) {
2480
+ out.push(run);
2481
+ continue;
2482
+ }
2483
+ const here = await deliverableTo(lifecycleEventName(run.workflow, kind), normalizeNamespace(run.namespace));
2484
+ if (owed.some((s) => here.has(s)) ||
2485
+ (await this.orphanedSubscribers(run, owed, here, nowMs)).length > 0) {
2486
+ out.push(run);
2487
+ }
2488
+ }
2489
+ if (runs.length < size)
2490
+ break;
2491
+ page += 1;
2492
+ }
2493
+ return out;
2494
+ }
2495
+ /**
2496
+ * Publish settled runs' lifecycle events again, whether or not they are still marked: one run
2497
+ * (`{ runId }`), or a workflow's settled runs (`{ workflow, kind, since, until, limit }`, read in
2498
+ * pages of the store's listing, filtered by when the run settled). `subscriber` restricts it to
2499
+ * one subscribing workflow, which must subscribe to the run's ending (registered here or announced
2500
+ * by a live worker for the run's namespace): otherwise the run is reported in `failed` with
2501
+ * `subscriber <name> does not subscribe to <event>` and nothing is marked. The subscribers
2502
+ * registered on this instance are started; any other (announced by a live worker) is marked owed
2503
+ * again, so the sweep of an instance that registers it starts it, and is not counted in
2504
+ * `started`. The same path as the live publish, so a subscriber whose run already exists is not
2505
+ * started again; a subscriber that debounces or batches is not routed again once the run carries
2506
+ * its coalesced tag, unless it also carries its trigger-failed tag, since an accumulator that
2507
+ * already fired would start the target again. A run another
2508
+ * publisher holds past the lease retries is reported in `failed` with subscriber `*`. A
2509
+ * subscriber registered after the runs settled (more than {@link LIFECYCLE_HEAL_WINDOW_MS} after,
2510
+ * past what the sweep heals) is backfilled this way. A workflow target's `namespace` narrows it to
2511
+ * that namespace; on an engine bound to another namespace it publishes nothing.
2512
+ */
2513
+ async republishLifecycle(target) {
2514
+ const result = { runs: 0, started: 0, failed: [] };
2515
+ // A backfill usually follows a deploy: read the fleet's subscriptions now, not from the cache.
2516
+ await this.refreshAnnouncedSubscriptions();
2517
+ const publish = async (run) => {
2518
+ if (!lifecycleKindOf(run.status))
2519
+ return;
2520
+ result.runs += 1;
2521
+ const { started, failed, busy } = await this.publishSettled(run, {
2522
+ republish: true,
2523
+ subscriber: target.subscriber,
2524
+ });
2525
+ if (busy) {
2526
+ result.failed.push({
2527
+ runId: run.id,
2528
+ subscriber: '*',
2529
+ error: `run ${run.id} is being published elsewhere; try again`,
2530
+ });
2531
+ return;
2532
+ }
2533
+ result.started += started;
2534
+ for (const { workflow, error } of failed) {
2535
+ result.failed.push({
2536
+ runId: run.id,
2537
+ subscriber: workflow,
2538
+ error: error instanceof Error ? error.message : String(error),
2539
+ });
2540
+ }
2541
+ };
2542
+ if ('runId' in target) {
2543
+ const run = await this.store.getRun(target.runId);
2544
+ if (run)
2545
+ await publish(run);
2546
+ return result;
2547
+ }
2548
+ if (target.namespace !== undefined &&
2549
+ this.namespace !== undefined &&
2550
+ target.namespace !== this.namespace) {
2551
+ return result;
2552
+ }
2553
+ const namespace = target.namespace ?? this.namespace;
2554
+ const limit = target.limit ?? 1000;
2555
+ const statuses = target.kind === 'completed'
2556
+ ? ['completed']
2557
+ : target.kind === 'failed'
2558
+ ? ['failed', 'cancelled', 'dead']
2559
+ : ['completed', 'failed', 'cancelled', 'dead'];
2560
+ const size = Math.min(limit, 100);
2561
+ let seen = 0;
2562
+ for (let page = 1; seen < limit; page += 1) {
2563
+ const runs = await this.store.listRuns({
2564
+ workflow: target.workflow,
2565
+ statuses,
2566
+ ...(namespace !== undefined ? { namespace } : {}),
2567
+ page,
2568
+ size,
2569
+ });
2570
+ for (const run of runs) {
2571
+ if (seen >= limit)
2572
+ break;
2573
+ const settledAt = run.updatedAt.getTime();
2574
+ if (target.since && settledAt < target.since.getTime())
2575
+ continue;
2576
+ if (target.until && settledAt > target.until.getTime())
2577
+ continue;
2578
+ seen += 1;
2579
+ await publish(run);
2580
+ }
2581
+ if (runs.length < size)
2582
+ break;
2583
+ }
2584
+ return result;
2585
+ }
2586
+ /**
2587
+ * Wake the `ctx.waitForRun` waiters on `runId`, which just settled, and on the earlier runs of its
2588
+ * continue-as-new chain, with the event of the run that ends the chain (none yet when `runId`
2589
+ * handed off). One waiter listing per settle. Best effort: a waiter missed here (this process dies
2590
+ * first) is woken by {@link sweepRunWaits}. Internal `__` runs have no waiters, so they are
2591
+ * skipped without a read.
2592
+ */
2593
+ notifyRunWaiters(runId, workflow) {
2594
+ if (workflow?.startsWith('__'))
2595
+ return;
2596
+ void (async () => {
2597
+ const predecessors = chainPredecessorsOf(runId);
2598
+ const targets = new Set([runId, ...predecessors]);
2599
+ // Every run of a chain shares the first one's id as a prefix.
2600
+ const prefix = predecessors[0] !== undefined
2601
+ ? `${RUN_WAIT_TOKEN_PREFIX}${predecessors[0]}`
2602
+ : runWaitPrefix(runId);
2603
+ const waiters = (await this.store.listSignalWaiters(prefix)).filter((w) => targets.has(runWaitTargetOf(w) ?? ''));
2604
+ if (waiters.length === 0)
2605
+ return;
2606
+ const run = await lastRunOfChain((id) => this.store.getRun(id), runId);
2607
+ if (!run || !lifecycleKindOf(run.status))
2608
+ return;
2609
+ const event = lifecycleEventOf(run);
2610
+ for (const waiter of waiters)
2611
+ await this.wakeSignalWaiter(waiter.token, event);
2612
+ })().catch(() => undefined);
2613
+ }
2614
+ /** How many waited-on runs one {@link sweepRunWaits} reads at most. */
2615
+ static RUN_WAIT_SWEEP_SCAN_CAP = 1000;
2616
+ /** Where the next {@link sweepRunWaits} starts in the list of waited-on runs. */
2617
+ runWaitSweepCursor = 0;
2618
+ /**
2619
+ * Wake the `ctx.waitForRun` waiters (up to `limit`) whose run, or the run ending its
2620
+ * continue-as-new chain, has already settled: the recovery for a process that died between
2621
+ * settling a run and waking its waiters. Waiters on runs still in flight are skipped, reading at
2622
+ * most {@link RUN_WAIT_SWEEP_SCAN_CAP} waited-on runs per call; the next call starts where this one
2623
+ * stopped, so waiters on long-running runs never hide the ones that can be woken. Returns how many
2624
+ * it woke.
2625
+ */
2626
+ async sweepRunWaits(limit = 100) {
2627
+ const waiters = await asHeartbeat(() => this.store.listSignalWaiters(RUN_WAIT_TOKEN_PREFIX));
2628
+ const byTarget = new Map();
2629
+ for (const waiter of waiters) {
2630
+ const target = runWaitTargetOf(waiter);
2631
+ if (target === undefined)
2632
+ continue;
2633
+ byTarget.set(target, [...(byTarget.get(target) ?? []), waiter]);
2634
+ }
2635
+ const targets = [...byTarget.keys()].sort();
2636
+ if (targets.length === 0)
2637
+ return 0;
2638
+ const from = this.runWaitSweepCursor % targets.length;
2639
+ const scan = Math.min(targets.length, WorkflowEngine.RUN_WAIT_SWEEP_SCAN_CAP);
2640
+ let woken = 0;
2641
+ let read = 0;
2642
+ for (; read < scan && woken < limit; read += 1) {
2643
+ const target = targets[(from + read) % targets.length];
2644
+ const run = await lastRunOfChain((id) => this.store.getRun(id), target);
2645
+ if (!run || !lifecycleKindOf(run.status))
2646
+ continue;
2647
+ const event = lifecycleEventOf(run);
2648
+ for (const waiter of byTarget.get(target) ?? []) {
2649
+ if (await this.wakeSignalWaiter(waiter.token, event))
2650
+ woken += 1;
2651
+ }
2652
+ }
2653
+ this.runWaitSweepCursor = read >= targets.length ? 0 : (from + read) % targets.length;
2654
+ return woken;
2655
+ }
1423
2656
  /** Events waiting for the control-plane publish in flight, in emit order (see {@link broadcastEvents}). */
1424
2657
  eventsToBroadcast = [];
1425
2658
  broadcastingEvents = false;
@@ -1686,8 +2919,11 @@ export class WorkflowEngine {
1686
2919
  * engine's read API for inspection surfaces (the dashboard, the `durable:runs` CLI) so they query
1687
2920
  * through the engine rather than reaching for its private store.
1688
2921
  */
1689
- listRuns(query) {
1690
- return this.store.listRuns(query);
2922
+ async listRuns(query) {
2923
+ const scoped = withoutBookkeepingRuns(query);
2924
+ const runs = await this.store.listRuns(scoped);
2925
+ const hidden = scoped.excludeWorkflows;
2926
+ return hidden?.length ? runs.filter((run) => !hidden.includes(run.workflow)) : runs;
1691
2927
  }
1692
2928
  /**
1693
2929
  * The distinct values of ONE filter axis over the runs matching `query`, with counts — what a
@@ -1696,10 +2932,11 @@ export class WorkflowEngine {
1696
2932
  * {@link listRuns}.
1697
2933
  */
1698
2934
  runValueFacets(axis, query, opts) {
2935
+ const scoped = withoutBookkeepingRuns(query);
1699
2936
  if (this.store.runValueFacets) {
1700
- return this.store.runValueFacets(axis, query, opts);
2937
+ return this.store.runValueFacets(axis, scoped, opts);
1701
2938
  }
1702
- return scanRunValueFacets(this.store, axis, query, opts);
2939
+ return scanRunValueFacets(this.store, axis, scoped, opts);
1703
2940
  }
1704
2941
  /** List a run's step checkpoints (its timeline). Part of the engine's read API; see {@link listRuns}. */
1705
2942
  listCheckpoints(runId) {
@@ -1710,15 +2947,149 @@ export class WorkflowEngine {
1710
2947
  * has {@link StateStore.runFacets} (else counted over a listing). Part of the engine's read API.
1711
2948
  */
1712
2949
  runFacets(query) {
1713
- return runFacetsOf(this.store, query);
2950
+ return runFacetsOf(this.store, withoutBookkeepingRuns(query));
2951
+ }
2952
+ /**
2953
+ * A run's detail (see `loadRunDetail`), built from the store's checkpoint aggregates when it has
2954
+ * `checkpointStats` + `queryCheckpoints` — a summarized detail then reads a few rows per wide fan
2955
+ * instead of the whole history, and carries an `etag` that `options.ifNoneMatch` can match.
2956
+ */
2957
+ async getRunDetail(runId, options) {
2958
+ const source = this.detailSource();
2959
+ const withExtras = async (detail) => {
2960
+ const extras = await this.lifecycleDetail(detail.run);
2961
+ return {
2962
+ ...detail,
2963
+ ...extras,
2964
+ ...(detail.etag ? { etag: extendEtag(detail.etag, nonEmptyObject(extras)) } : {}),
2965
+ };
2966
+ };
2967
+ const ifNoneMatch = options?.ifNoneMatch;
2968
+ if (ifNoneMatch === undefined) {
2969
+ const detail = await loadRunDetail(source, runId, options);
2970
+ return detail ? withExtras(detail) : null;
2971
+ }
2972
+ // A revalidation: read the run and its validator only ('*' always matches), add what the
2973
+ // detail reads from outside the run, and answer 304 when both are unchanged. A source that
2974
+ // cannot version the detail answers it in full instead.
2975
+ const head = await loadRunDetail(source, runId, { ...options, ifNoneMatch: '*' });
2976
+ if (!head)
2977
+ return null;
2978
+ if (!head.notModified || !head.etag)
2979
+ return withExtras(head);
2980
+ const extras = await this.lifecycleDetail(head.run);
2981
+ const etag = extendEtag(head.etag, nonEmptyObject(extras));
2982
+ if (etagMatches(ifNoneMatch, etag)) {
2983
+ return { run: head.run, timeline: [], children: [], notModified: true, etag };
2984
+ }
2985
+ const { ifNoneMatch: _matched, ...fresh } = options;
2986
+ const detail = await loadRunDetail(source, runId, fresh);
2987
+ return detail ? withExtras(detail) : null;
2988
+ }
2989
+ /**
2990
+ * What a run detail reads beyond the run row and its checkpoints: the subscribers its lifecycle
2991
+ * event goes to (of its namespace) and where it stands for each, its workflow's lifecycle links,
2992
+ * and the `ctx.waitForRun` waits on either side of it with the other run's status. A few reads per
2993
+ * detail: the run's own waiters while it is suspended, the waiters on it (one prefix read), and
2994
+ * one batched read of every linked run.
2995
+ */
2996
+ async lifecycleDetail(run) {
2997
+ const namespace = normalizeNamespace(run.namespace);
2998
+ const kind = lifecycleKindOf(run.status);
2999
+ const subscribers = kind ? this.lifecycleSubscribers(run.workflow, kind, namespace) : [];
3000
+ const links = await this.workflowLifecycle(run.workflow);
3001
+ const deliveries = kind
3002
+ ? lifecycleDeliveriesOf(run, {
3003
+ subscribers,
3004
+ live: new Set(subscribers),
3005
+ coalesced: this.coalescedSubscribers(run, kind),
3006
+ })
3007
+ : [];
3008
+ const [own, onIt] = await Promise.all([
3009
+ run.status === 'suspended' ? listSignalWaitersOfRuns(this.store, [run.id]) : [],
3010
+ this.store.listSignalWaiters(runWaitPrefix(run.id)),
3011
+ ]);
3012
+ const waitsFor = own.flatMap((w) => {
3013
+ const target = runWaitTargetOf(w);
3014
+ return target !== undefined ? [{ runId: target, seq: w.seq }] : [];
3015
+ });
3016
+ const awaitedBy = onIt
3017
+ .filter((w) => runWaitTargetOf(w) === run.id)
3018
+ .map((w) => ({ runId: w.runId, seq: w.seq }));
3019
+ const started = deliveries.flatMap((d) => d.state === 'started' && d.runId !== undefined ? [d.runId] : []);
3020
+ const source = lifecycleSourceOf(run);
3021
+ const linked = await getRunsByIds(this.store, [
3022
+ ...(source ? [source.runId] : []),
3023
+ ...started,
3024
+ ...waitsFor.map((l) => l.runId),
3025
+ ...awaitedBy.map((l) => l.runId),
3026
+ ]);
3027
+ const withRun = (link) => {
3028
+ const other = link.runId !== undefined ? linked.get(link.runId) : undefined;
3029
+ return other ? { ...link, status: other.status, namespace: other.namespace } : link;
3030
+ };
3031
+ const sourceRun = source ? linked.get(source.runId) : undefined;
3032
+ const sourceTrace = lifecycleTraceparentOf(sourceRun?.tags)?.split('-')[1];
3033
+ return {
3034
+ ...(source
3035
+ ? {
3036
+ triggeredBy: {
3037
+ ...source,
3038
+ ...(sourceRun ? { status: sourceRun.status, namespace: sourceRun.namespace } : {}),
3039
+ ...(sourceTrace ? { traceId: sourceTrace } : {}),
3040
+ },
3041
+ }
3042
+ : {}),
3043
+ ...(subscribers.length > 0 ? { lifecycleSubscribers: subscribers } : {}),
3044
+ ...(links.startsOnSettle.length > 0 || links.startedBy.length > 0
3045
+ ? { workflowLifecycle: links }
3046
+ : {}),
3047
+ ...(deliveries.length > 0
3048
+ ? {
3049
+ lifecycleDeliveries: deliveries.map((d) => (d.state === 'started' ? withRun(d) : d)),
3050
+ }
3051
+ : {}),
3052
+ ...(waitsFor.length > 0 ? { waitsFor: waitsFor.map(withRun) } : {}),
3053
+ ...(awaitedBy.length > 0 ? { awaitedBy: awaitedBy.map(withRun) } : {}),
3054
+ };
1714
3055
  }
1715
3056
  /**
1716
- * A run's detail (see `loadRunDetail`), built from the store's checkpoint aggregates when it has
1717
- * `checkpointStats` + `queryCheckpoints` — a summarized detail then reads a few rows per wide fan
1718
- * instead of the whole history, and carries an `etag` that `options.ifNoneMatch` can match.
1719
- */
1720
- getRunDetail(runId, options) {
1721
- return loadRunDetail(this.detailSource(), runId, options);
3057
+ * The subscribers registered here that coalesce `run`'s lifecycle event (debounce/batch) and
3058
+ * accept it, each with the accumulator run the event went to: the subscription's `where`, `input`
3059
+ * and `key` applied to the event as `startSubscribers` does. Undefined when the key cannot be
3060
+ * computed. A subscriber whose `where` declines the event is left out.
3061
+ */
3062
+ coalescedSubscribers(run, kind) {
3063
+ const out = new Map();
3064
+ const event = lifecycleEventName(run.workflow, kind);
3065
+ const triggers = this.eventTriggers.get(event);
3066
+ if (!triggers)
3067
+ return out;
3068
+ let payload;
3069
+ for (const [workflow, subscriptions] of triggers) {
3070
+ const batch = this.latest.get(workflow)?.eventBatch;
3071
+ if (!batch)
3072
+ continue;
3073
+ if (!batch.key && subscriptions.every((sub) => !sub.where)) {
3074
+ out.set(workflow, accumulatorRunId(workflow));
3075
+ continue;
3076
+ }
3077
+ try {
3078
+ payload ??= lifecycleEventOf(run);
3079
+ const { accepted } = acceptingSubscription(subscriptions, payload);
3080
+ if (!accepted)
3081
+ continue;
3082
+ const input = accepted.input ? accepted.input(payload) : payload;
3083
+ const key = batch.key ? batch.key(input) : undefined;
3084
+ out.set(workflow, key === undefined || typeof key === 'string'
3085
+ ? accumulatorRunId(workflow, key)
3086
+ : undefined);
3087
+ }
3088
+ catch {
3089
+ out.set(workflow, undefined);
3090
+ }
3091
+ }
3092
+ return out;
1722
3093
  }
1723
3094
  /** One page of a summarized fan's members (see `loadFanMembers`), versioned like {@link getRunDetail}. */
1724
3095
  getFanMembers(runId, group, query) {
@@ -1961,7 +3332,7 @@ export class WorkflowEngine {
1961
3332
  passes.push(scoped);
1962
3333
  }
1963
3334
  else {
1964
- const [lastKept] = await this.store.listRuns({ ...scoped, page: maxCount, size: 1 });
3335
+ const [lastKept] = await this.store.listRuns(withoutBookkeepingRuns({ ...scoped, page: maxCount, size: 1 }));
1965
3336
  if (lastKept) {
1966
3337
  passes.push({ ...scoped, createdBefore: lastKept.createdAt.getTime() - 1 });
1967
3338
  }
@@ -1969,9 +3340,13 @@ export class WorkflowEngine {
1969
3340
  }
1970
3341
  }
1971
3342
  for (const pass of passes) {
1972
- const candidates = await this.store.listRuns({ ...pass, size: 100 });
3343
+ const candidates = await this.store.listRuns(withoutBookkeepingRuns({ ...pass, size: 100 }));
1973
3344
  const archived = [];
1974
3345
  for (const run of candidates) {
3346
+ // A run whose lifecycle event is still owed is kept until it is published, and a registry
3347
+ // run is replaced by its engine's heartbeat instead.
3348
+ if (retentionKeeps(run.tags) || run.workflow === LIFECYCLE_REGISTRY_WORKFLOW)
3349
+ continue;
1975
3350
  try {
1976
3351
  if (this.evictListeners.size > 0) {
1977
3352
  const checkpoints = await this.store.listCheckpoints(run.id);
@@ -2109,7 +3484,7 @@ export class WorkflowEngine {
2109
3484
  let deleted = 0;
2110
3485
  let previous = '';
2111
3486
  for (;;) {
2112
- const batch = await this.store.listRuns({ ...scope, statuses, size });
3487
+ const batch = await this.store.listRuns(withoutBookkeepingRuns({ ...scope, statuses, size }));
2113
3488
  if (batch.length === 0)
2114
3489
  break;
2115
3490
  const fingerprint = batch.map((r) => r.id).join('\u0000');
@@ -2259,7 +3634,38 @@ export class WorkflowEngine {
2259
3634
  });
2260
3635
  }
2261
3636
  async resumeRunNow(runId, allowFailed) {
2262
- const run = await this.store.getRun(runId);
3637
+ // A run just claimed for this turn (runPending) is held, and was read by the claim.
3638
+ const claimed = this.claimedRuns.get(runId);
3639
+ if (claimed) {
3640
+ this.claimedRuns.delete(runId);
3641
+ return this.resumeLoadedRun(claimed, runId, allowFailed);
3642
+ }
3643
+ // Take the lease and read the run in one round trip when the store can: `execute` then finds
3644
+ // the lease held for this turn. Not when a caller already holds it for this turn (a leased
3645
+ // sweep) or a turn of the run is in flight here; a failed take reads the run as before.
3646
+ const lockAndGet = this.lockedForTurn.has(runId) || this.executionLeases.has(runId)
3647
+ ? undefined
3648
+ : this.store.tryLockRunAndGet;
3649
+ let run = null;
3650
+ if (lockAndGet) {
3651
+ const nowMs = this.clock();
3652
+ run = await lockAndGet.call(this.store, runId, this.instanceId, nowMs + this.leaseMs, nowMs);
3653
+ }
3654
+ if (!run)
3655
+ return this.resumeLoadedRun(await this.store.getRun(runId), runId, allowFailed);
3656
+ this.lockedForTurn.add(runId);
3657
+ try {
3658
+ return await this.resumeLoadedRun(run, runId, allowFailed);
3659
+ }
3660
+ finally {
3661
+ // Still marked: the turn never started (a finished run, another namespace, a registration
3662
+ // that failed), so the lease taken above goes back.
3663
+ if (this.lockedForTurn.delete(runId)) {
3664
+ await this.store.releaseRunLock(runId, this.instanceId).catch(() => undefined);
3665
+ }
3666
+ }
3667
+ }
3668
+ async resumeLoadedRun(run, runId, allowFailed) {
2263
3669
  if (!run)
2264
3670
  throw new Error(`run ${runId} not found`);
2265
3671
  // A definitively-finished run must not be re-executed (e.g. a worker result landing after the
@@ -2400,6 +3806,8 @@ export class WorkflowEngine {
2400
3806
  */
2401
3807
  beginDrain(spentMs = 0) {
2402
3808
  this.draining = true;
3809
+ // Parked bodies are only an optimization: the runs replay wherever they resume next.
3810
+ this.sticky.clear();
2403
3811
  // Tell the local steps still running: a long one can stop between units of work instead of
2404
3812
  // being cut off when the drain budget runs out.
2405
3813
  if (!this.drainAbort.signal.aborted)
@@ -2550,11 +3958,11 @@ export class WorkflowEngine {
2550
3958
  // (and the listRuns calls above it), so this run could have settled on its own — via its own
2551
3959
  // in-flight turn, an operator cancel, or another sweep — in the gap. Only clobber it into
2552
3960
  // `cancelled` if it's still genuinely `running`/`suspended`.
2553
- const applied = await this.store.updateRunIf(run.id, ['running', 'suspended'], {
3961
+ const applied = await this.store.updateRunIf(run.id, ['running', 'suspended'], this.withLifecycleMarker(run, {
2554
3962
  status: 'cancelled',
2555
3963
  error,
2556
3964
  updatedAt: new Date(),
2557
- });
3965
+ }));
2558
3966
  if (!applied)
2559
3967
  continue;
2560
3968
  this.emit({
@@ -2878,11 +4286,11 @@ export class WorkflowEngine {
2878
4286
  * (e.g. a lease-free `cancel()`), so it is not ours to touch.
2879
4287
  */
2880
4288
  async deadLetterLeased(run, from, error) {
2881
- const applied = await this.store.updateRunIf(run.id, from, {
4289
+ const applied = await this.store.updateRunIf(run.id, from, this.withLifecycleMarker(run, {
2882
4290
  status: 'dead',
2883
4291
  error,
2884
4292
  updatedAt: new Date(),
2885
- });
4293
+ }));
2886
4294
  await this.store.releaseRunLock(run.id, this.instanceId);
2887
4295
  if (!applied)
2888
4296
  return null;
@@ -3311,9 +4719,58 @@ export class WorkflowEngine {
3311
4719
  async runPending(nowMs = this.clock()) {
3312
4720
  // Oldest-first (FIFO), capped per call so a backlog drains over several polls without one sweep
3313
4721
  // fetching unboundedly. A run not picked up this tick is picked up the next.
3314
- const pending = await asHeartbeat(() => this.store.listPendingRuns(100, this.namespace));
4722
+ // A store that can claims a batch in one statement (rows another instance is claiming are
4723
+ // skipped, not raced for), and the batch runs concurrently, within this engine's run budget.
4724
+ const claim = this.store.claimPendingRuns;
4725
+ if (claim) {
4726
+ if (this.draining)
4727
+ return [];
4728
+ const room = Math.min(PENDING_POLL_BATCH, LOCAL_RUN_CONCURRENCY - this.inflight.size);
4729
+ if (room <= 0)
4730
+ return [];
4731
+ const claimed = await asHeartbeat(() => claim.call(this.store, room, this.instanceId, nowMs + this.leaseMs, nowMs, this.namespace));
4732
+ return this.runClaimed(claimed, nowMs);
4733
+ }
4734
+ const pending = await asHeartbeat(() => this.store.listPendingRuns(PENDING_POLL_BATCH, this.namespace));
3315
4735
  return this.resumeLeased(pending, nowMs);
3316
4736
  }
4737
+ /**
4738
+ * Run `claimed` (pending runs this instance just leased, read as locked): every turn starts at
4739
+ * once, and each re-enters the lease taken for it without reading the run again. A run this
4740
+ * instance cannot resume is skipped and logged, as {@link resumeLeased} does; its lease lapses.
4741
+ */
4742
+ async runClaimed(claimed, nowMs) {
4743
+ const turns = claimed.map(async (run) => {
4744
+ if (this.draining) {
4745
+ // Drain began after the claim: give the lease back; the run stays for another instance.
4746
+ await this.store.releaseRunLock(run.id, this.instanceId).catch(() => undefined);
4747
+ return undefined;
4748
+ }
4749
+ this.lockedForTurn.add(run.id);
4750
+ this.claimedRuns.set(run.id, run);
4751
+ try {
4752
+ return await this.resumeImplicit(run.id);
4753
+ }
4754
+ catch (error) {
4755
+ if (error instanceof NamespaceMismatch)
4756
+ return undefined;
4757
+ if (error instanceof WorkflowNotRegistered) {
4758
+ const settled = await this.settleUnregistered(run, nowMs).catch(() => undefined);
4759
+ if (settled)
4760
+ return settled;
4761
+ if (settled !== undefined)
4762
+ return undefined;
4763
+ }
4764
+ console.error(`[adonis-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance or a later tick:`, error);
4765
+ return undefined;
4766
+ }
4767
+ finally {
4768
+ this.lockedForTurn.delete(run.id);
4769
+ this.claimedRuns.delete(run.id);
4770
+ }
4771
+ });
4772
+ return (await Promise.all(turns)).filter((r) => r !== undefined);
4773
+ }
3317
4774
  /**
3318
4775
  * Resolve once `runId` reaches a settled state — terminal (completed/failed/cancelled/dead) or
3319
4776
  * suspended (handed off to a timer/signal/event). The async counterpart to dispatch: pair it with
@@ -3462,7 +4919,9 @@ export class WorkflowEngine {
3462
4919
  * match the payload satisfies (fan-out, vs `signal`'s point-to-point token).
3463
4920
  * 2. **Starts** a fresh run of every workflow registered with `onEvent: [name]`, passing the
3464
4921
  * payload as input. Idempotent by `evt:<id>:<workflow>` — pass `opts.id` to dedupe redeliveries
3465
- * of the same logical event (default: a fresh uuid, so each publish triggers once).
4922
+ * of the same logical event (default: a fresh uuid, so each publish triggers once). A workflow
4923
+ * a non-Node worker announces as subscribing to `name` (a Python `on_event=[...]`) is started
4924
+ * the same way, by name, when this engine can reach it (see {@link announcedRemoteSubscribers}).
3466
4925
  *
3467
4926
  * Reliable (buffered) delivery — mirrors `signalWithStart`'s reliability contract, so document it
3468
4927
  * exactly: a publish that resumes ≥1 live waiter, OR routes into an `eventBatch` accumulator /
@@ -3484,27 +4943,14 @@ export class WorkflowEngine {
3484
4943
  touched += 1;
3485
4944
  }
3486
4945
  }
3487
- const subscribers = this.eventTriggers.get(name);
3488
- if (subscribers?.size) {
4946
+ await this.freshAnnouncedSubscriptions();
4947
+ const remote = await this.announcedRemoteSubscribers(name);
4948
+ if (this.eventTriggers.get(name)?.size || remote.length > 0) {
3489
4949
  const eventId = opts?.id ?? globalThis.crypto.randomUUID();
3490
- for (const workflow of subscribers) {
3491
- // A subscriber that rejects the payload (validateInput) must not block the others or the
3492
- // waiters — its run simply never starts, mirroring fire-and-forget dead-letter routing.
3493
- try {
3494
- const batch = this.latest.get(workflow)?.eventBatch;
3495
- if (batch) {
3496
- // Coalesce: route the event into a per-workflow accumulator (one long-lived run that
3497
- // debounces/batches and then starts the target with the collected payload(s)).
3498
- await this.accumulators.route(workflow, batch, payload);
3499
- }
3500
- else {
3501
- await this.start(workflow, payload, `evt:${eventId}:${workflow}`);
3502
- }
3503
- touched += 1;
3504
- }
3505
- catch {
3506
- // skip this subscriber
3507
- }
4950
+ const { reached, failed } = await this.startSubscribers(name, payload, (workflow) => `evt:${eventId}:${workflow}`, undefined, remote);
4951
+ touched += reached.length;
4952
+ for (const { workflow, error } of failed) {
4953
+ console.warn(`[adonis-durable] event ${name}: subscriber ${workflow} not started:`, error);
3508
4954
  }
3509
4955
  }
3510
4956
  // Nobody received it live (no waiter matched, no subscriber exists/started) — buffer ONE copy
@@ -3852,12 +5298,12 @@ export class WorkflowEngine {
3852
5298
  }
3853
5299
  const error = { message: 'cancelled' };
3854
5300
  // Clears a remote turn's decision marker too: a cancelled run awaits no decision.
3855
- await this.store.updateRun(runId, {
5301
+ await this.store.updateRun(runId, this.withLifecycleMarker(run, {
3856
5302
  status: 'cancelled',
3857
5303
  error,
3858
5304
  awaitingDecisionTaskId: undefined,
3859
5305
  updatedAt: new Date(),
3860
- });
5306
+ }));
3861
5307
  this.emit({
3862
5308
  type: 'run.failed',
3863
5309
  runId,
@@ -3964,7 +5410,7 @@ export class WorkflowEngine {
3964
5410
  * the full listing for a store without one.
3965
5411
  */
3966
5412
  countRuns(query) {
3967
- return countRuns(this.store, query);
5413
+ return countRuns(this.store, withoutBookkeepingRuns(query));
3968
5414
  }
3969
5415
  /**
3970
5416
  * Every workflow the live fleet can execute in this engine's partition: what workers DECLARE in
@@ -4365,7 +5811,7 @@ export class WorkflowEngine {
4365
5811
  (run.lockedUntil ?? 0) > lockNow &&
4366
5812
  (lockedForTurn || (await this.proveOwnLease(run.id, lockNow)));
4367
5813
  if (!reentered) {
4368
- if (!(await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow))) {
5814
+ if (!(await this.lockRunPastLifecyclePublish(run.id, lockNow))) {
4369
5815
  const stamping = this.draining ? undefined : this.stampWakeRequest(run.id, reason);
4370
5816
  this.trackEffect(this.requestWake(run.id, run.lockedUntil, reason, stamping));
4371
5817
  const refused = { runId: run.id, status: run.status };
@@ -4639,12 +6085,12 @@ export class WorkflowEngine {
4639
6085
  // resurrect/clobber it — the conditional write (status still one of SETTLE_ELIGIBLE_STATUSES) is
4640
6086
  // the guard; the database, not a prior JS read, arbitrates. Also clears any error from an
4641
6087
  // earlier failed-then-retried attempt — a completed run is a success.
4642
- const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, {
6088
+ const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, this.withLifecycleMarker(run, {
4643
6089
  status: 'completed',
4644
6090
  output: outcome.output,
4645
6091
  error: undefined,
4646
6092
  updatedAt,
4647
- });
6093
+ }));
4648
6094
  if (!applied)
4649
6095
  return this.echoCurrentStatus(run.id);
4650
6096
  this.emit({
@@ -4670,11 +6116,11 @@ export class WorkflowEngine {
4670
6116
  // (observed: a cascade-requeued child's fresh success lost to the stale buffered failure). A
4671
6117
  // legitimate retry always transitions through `pending`/`running` first (requeue flips the
4672
6118
  // status), so it is unaffected.
4673
- const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES.filter((s) => s !== 'failed'), {
6119
+ const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES.filter((s) => s !== 'failed'), this.withLifecycleMarker(run, {
4674
6120
  status: 'failed',
4675
6121
  error,
4676
6122
  updatedAt,
4677
- });
6123
+ }));
4678
6124
  if (!applied)
4679
6125
  return this.echoCurrentStatus(run.id);
4680
6126
  this.emit({
@@ -4694,13 +6140,20 @@ export class WorkflowEngine {
4694
6140
  // conditional write IS the guard — it subsumes the read-then-write this branch used to do (that
4695
6141
  // was itself a narrower, TOCTOU-prone version of the same check, and only looked for `cancelled`;
4696
6142
  // this now also catches `completed`/`dead` reached via any other path mid-turn).
4697
- const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, {
6143
+ // Strictly later than the row this turn read: a parked body is recognized by the exact stamp its
6144
+ // suspend wrote (see takeContinuation), so no later settle, by any instance with any clock, may
6145
+ // write that same stamp again, even within the same millisecond.
6146
+ const stamp = new Date(Math.max(updatedAt.getTime(), run.updatedAt.getTime() + 1));
6147
+ const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, await this.withRunTrace(run, {
4698
6148
  status: 'suspended',
4699
6149
  wakeAt: this.reconcileWakeAt(outcome.wakeAt),
4700
- updatedAt,
4701
- });
6150
+ updatedAt: stamp,
6151
+ }));
4702
6152
  if (!applied)
4703
6153
  return this.echoCurrentStatus(run.id);
6154
+ run.updatedAt = stamp;
6155
+ if (this.turnContinuations.has(run.id))
6156
+ this.suspendStamps.set(run.id, stamp.getTime());
4704
6157
  this.emit({
4705
6158
  type: 'run.suspended',
4706
6159
  runId: run.id,
@@ -4745,7 +6198,8 @@ export class WorkflowEngine {
4745
6198
  * settled it), so report what is persisted rather than writing over it.
4746
6199
  */
4747
6200
  /**
4748
- * A run's first execution: `pending` -> `running`. A singleton run not yet admitted is stamped
6201
+ * A run's first execution: `pending` -> `running`, resolving whether it moved it (a run cancelled
6202
+ * or deleted since it was read did not move). A singleton run not yet admitted is stamped
4749
6203
  * {@link SINGLETON_TRACKED_TAG} in the same write, BEFORE it competes for a slot, so a run this
4750
6204
  * replica is about to admit or deny is never mistaken for a holder admitted by a replica older than
4751
6205
  * 0.51 (see `legacySingletonHolders`).
@@ -4753,14 +6207,19 @@ export class WorkflowEngine {
4753
6207
  async markStarted(run, singleton) {
4754
6208
  const tags = singleton ? withSingletonTracked(run.tags) : run.tags;
4755
6209
  const retag = tags !== run.tags;
4756
- await this.store.updateRun(run.id, {
6210
+ // A compare-and-set on `pending`: the run was read before this turn took it, and a cancel (or a
6211
+ // delete) landing in between must not be overwritten with `running`, which would resurrect it.
6212
+ const started = await this.store.updateRunIf(run.id, ['pending'], {
4757
6213
  status: 'running',
4758
6214
  ...(retag ? { tags } : {}),
4759
6215
  updatedAt: new Date(),
4760
6216
  });
6217
+ if (!started)
6218
+ return false;
4761
6219
  run.status = 'running';
4762
6220
  if (retag)
4763
6221
  run.tags = tags;
6222
+ return true;
4764
6223
  }
4765
6224
  async singletonWaitResult(run) {
4766
6225
  const current = await this.store.getRun(run.id);
@@ -4804,12 +6263,17 @@ export class WorkflowEngine {
4804
6263
  return { runId: run.id, status: 'suspended' };
4805
6264
  }
4806
6265
  if (run.status === 'pending') {
4807
- await this.markStarted(run, registered?.singleton !== undefined);
6266
+ if (!(await this.markStarted(run, registered?.singleton !== undefined))) {
6267
+ // No longer `pending` (cancelled or deleted since it was read): nothing to run.
6268
+ await this.store.releaseRunLock(run.id, this.instanceId).catch(() => undefined);
6269
+ return await this.echoCurrentStatus(run.id);
6270
+ }
4808
6271
  this.emit({
4809
6272
  type: 'run.started',
4810
6273
  runId: run.id,
4811
6274
  workflow: run.workflow,
4812
6275
  namespace: run.namespace,
6276
+ ...(await this.sourceTraceOf(run)),
4813
6277
  });
4814
6278
  }
4815
6279
  if (registered.singleton && !(await this.singletons.admit(run, registered.singleton))) {
@@ -4818,7 +6282,14 @@ export class WorkflowEngine {
4818
6282
  // Capability/protocol guard (design §7.5): a remote workflow's turn dispatches a WorkflowTask to
4819
6283
  // its group — if no live worker on that group can run it, park `blocked` before advancing rather
4820
6284
  // than handing the task to the executor to enqueue into a void. Re-checked each turn (recovery).
4821
- const routePlan = await this.planRoute(registered.convention ? conventionGroup(run.workflow, run.namespace) : remote.group, registered.requires);
6285
+ const routeToken = registered.convention
6286
+ ? conventionGroup(run.workflow, run.namespace)
6287
+ : remote.group;
6288
+ let routePlan = await this.planRoute(routeToken, registered.requires);
6289
+ // As in ensureRoutable: re-read before parking, never park on reused descriptors.
6290
+ if (routePlan?.status === 'blocked') {
6291
+ routePlan = await this.planRoute(routeToken, registered.requires, true);
6292
+ }
4822
6293
  if (routePlan?.status === 'blocked')
4823
6294
  return this.parkBlocked(run, routePlan);
4824
6295
  const checkpoints = await this.replayCheckpoints(run.id);
@@ -4883,7 +6354,7 @@ export class WorkflowEngine {
4883
6354
  message: err instanceof Error ? err.message : String(err),
4884
6355
  stack: err instanceof Error ? err.stack : undefined,
4885
6356
  };
4886
- await this.store.updateRun(run.id, { status: 'failed', error, updatedAt: new Date() });
6357
+ await this.store.updateRun(run.id, this.withLifecycleMarker(run, { status: 'failed', error, updatedAt: new Date() }));
4887
6358
  this.emit({
4888
6359
  type: 'run.failed',
4889
6360
  runId: run.id,
@@ -4965,12 +6436,12 @@ export class WorkflowEngine {
4965
6436
  // id (the `db` transport dedupes a turn by it, and the marker gates the decision on it).
4966
6437
  const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID()}`;
4967
6438
  const windowMs = this.decisionWindowMs(registered);
4968
- const marked = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, {
6439
+ const marked = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, await this.withRunTrace(run, {
4969
6440
  status: 'suspended',
4970
6441
  awaitingDecisionTaskId: taskId,
4971
6442
  wakeAt: windowMs != null ? this.clock() + Math.min(windowMs, MAX_BACKOFF_MS) : undefined,
4972
6443
  updatedAt: new Date(),
4973
- });
6444
+ }));
4974
6445
  if (!marked) {
4975
6446
  // Cancelled / completed / dead while this turn was being prepared: nothing to dispatch.
4976
6447
  await this.releaseTurnLease(run.id);
@@ -5004,13 +6475,13 @@ export class WorkflowEngine {
5004
6475
  const current = await this.store.getRun(run.id);
5005
6476
  if (current?.awaitingDecisionTaskId !== taskId)
5006
6477
  return this.echoCurrentStatus(run.id);
5007
- const failed = await this.store.updateRunIf(run.id, ['suspended'], {
6478
+ const failed = await this.store.updateRunIf(run.id, ['suspended'], this.withLifecycleMarker(run, {
5008
6479
  status: 'failed',
5009
6480
  error,
5010
6481
  awaitingDecisionTaskId: undefined,
5011
6482
  wakeAt: undefined,
5012
6483
  updatedAt: new Date(),
5013
- });
6484
+ }));
5014
6485
  if (!failed)
5015
6486
  return this.echoCurrentStatus(run.id);
5016
6487
  this.emit({
@@ -5822,12 +7293,12 @@ export class WorkflowEngine {
5822
7293
  await this.runCompensation(run, comp, i);
5823
7294
  }
5824
7295
  const error = { message: 'cancelled' };
5825
- await this.store.updateRun(run.id, {
7296
+ await this.store.updateRun(run.id, this.withLifecycleMarker(run, {
5826
7297
  status: 'cancelled',
5827
7298
  error,
5828
7299
  awaitingDecisionTaskId: undefined,
5829
7300
  updatedAt: new Date(),
5830
- });
7301
+ }));
5831
7302
  // The cancellation is durably recorded — consume the marker(s) so a later retry of this run id
5832
7303
  // doesn't get spuriously re-cancelled.
5833
7304
  for (;;) {
@@ -5864,12 +7335,17 @@ export class WorkflowEngine {
5864
7335
  // gate, so an admitted run's body never starts while it still reads `pending`. A resumed run is
5865
7336
  // already past `pending`, so this fires exactly once.
5866
7337
  if (run.status === 'pending') {
5867
- await this.markStarted(run, registered?.singleton !== undefined);
7338
+ if (!(await this.markStarted(run, registered?.singleton !== undefined))) {
7339
+ // No longer `pending` (cancelled or deleted since it was read): nothing to run.
7340
+ await this.store.releaseRunLock(run.id, this.instanceId).catch(() => undefined);
7341
+ return await this.echoCurrentStatus(run.id);
7342
+ }
5868
7343
  this.emit({
5869
7344
  type: 'run.started',
5870
7345
  runId: run.id,
5871
7346
  workflow: run.workflow,
5872
7347
  namespace: run.namespace,
7348
+ ...(await this.sourceTraceOf(run)),
5873
7349
  });
5874
7350
  }
5875
7351
  // Singleton admission gate: if this run shares its key with `limit` older in-flight runs, wait
@@ -5877,20 +7353,37 @@ export class WorkflowEngine {
5877
7353
  if (registered?.singleton && !(await this.singletons.admit(run, registered.singleton))) {
5878
7354
  return this.singletonWaitResult(run);
5879
7355
  }
7356
+ // A body this instance parked on dispatched steps in an earlier turn continues from there (see
7357
+ // `sticky-continuation.ts`), when nothing but this instance's own turn has settled the run since.
7358
+ const stickyOk = this.stickyEligible(run);
7359
+ let cont = stickyOk ? this.takeContinuation(run, fn) : undefined;
7360
+ const warm = cont !== undefined;
7361
+ // The parked steps' checkpoints, read before the body resumes: a failing read fails the turn
7362
+ // like a failing history read does, never the run.
7363
+ const parkedFresh = cont ? await this.readParked(run.id, cont) : undefined;
5880
7364
  // Saga compensations registered by completed steps; run in reverse if the run later fails.
5881
- const compensations = [];
7365
+ const compensations = cont ? cont.compensations : [];
5882
7366
  // Load this run's checkpoints ONCE and key them by seq, so replaying the completed prefix reads
5883
7367
  // from memory instead of one `getCheckpoint` SELECT per primitive (the O(N²) replay-reads fix).
5884
7368
  // Read once at execution start: a checkpoint written AFTER this snapshot (the signal/timer/child
5885
7369
  // this resume wakes on, or one written later in this same execution) is absent from the map, and
5886
7370
  // the ctx falls back to the live store for any absent seq — so replay semantics are unchanged.
5887
- const snapshot = await this.replayCheckpoints(run.id);
5888
- const replay = new Map();
7371
+ // A warm continuation already replayed its prefix: it reads only what lies past its park point.
7372
+ const snapshot = cont ? [] : await this.replayCheckpoints(run.id);
7373
+ const replay = cont ? cont.replay : new Map();
5889
7374
  for (const cp of snapshot)
5890
7375
  replay.set(cp.seq, cp);
7376
+ if (!cont && stickyOk) {
7377
+ cont = new StickyContinuation(run.id, fn, run.workflowVersion, compensations, replay, (c) => this.sticky.evict(c.runId));
7378
+ }
7379
+ if (cont)
7380
+ this.turnContinuations.set(run.id, cont);
7381
+ let parkedTurn = false;
5891
7382
  let suspendedOnFan;
5892
7383
  let settledSuspended = false;
5893
- const ctx = createWorkflowCtx(this.ctxHostFor(replay, run.namespace), run.id, compensations, run.workflow);
7384
+ const ctx = warm
7385
+ ? undefined
7386
+ : createWorkflowCtx(this.ctxHostFor(replay, run.namespace, cont), run.id, compensations, run.workflow);
5894
7387
  // Every settle below is `return await`ed, never a bare `return this.settleRun(...)`: inside
5895
7388
  // `try … finally` a bare return runs the `finally` — which RELEASES THE LEASE — as soon as the
5896
7389
  // return expression is evaluated, while the settle's own write is still in flight. A first turn
@@ -5907,8 +7400,33 @@ export class WorkflowEngine {
5907
7400
  // Establish the ambient ctx for the duration of this body turn so context-aware statics
5908
7401
  // (`BaseWorkflow.start`/`dispatch`) reachable from `fn` route through this run's `ctx.child`/
5909
7402
  // `ctx.startChild`. Re-set on every replay turn (each is its own async scope) — correct.
5910
- const input = await this.turnInput(run, registered);
5911
- const output = await workflowAls.run(ctx, () => fn(ctx, input));
7403
+ let output;
7404
+ if (cont) {
7405
+ if (parkedFresh) {
7406
+ cont.attached = true;
7407
+ await cont.resumeParked(parkedFresh);
7408
+ }
7409
+ else {
7410
+ const input = await this.turnInput(run, registered);
7411
+ const body = ctx;
7412
+ cont.begin(() => workflowAls.run(body, () => fn(body, input)));
7413
+ }
7414
+ const outcome = await cont.next();
7415
+ if (outcome.kind === 'parked') {
7416
+ // Parked on dispatched steps with nothing else in flight: suspend exactly as a thrown
7417
+ // suspension would, and keep the body for the run's next turn here.
7418
+ parkedTurn = true;
7419
+ throw new WorkflowSuspended(outcome.wakeAt, outcome.waitSeqs);
7420
+ }
7421
+ if (outcome.kind === 'threw')
7422
+ throw outcome.error;
7423
+ output = outcome.value;
7424
+ }
7425
+ else {
7426
+ const input = await this.turnInput(run, registered);
7427
+ const body = ctx;
7428
+ output = await workflowAls.run(body, () => fn(body, input));
7429
+ }
5912
7430
  // A compensating cancel requested while this turn ran (or one whose resume replays straight to
5913
7431
  // `return`, with no suspension point left) still wins: undo + settle cancelled, never completed.
5914
7432
  if (await this.takeCancelIntent(run.id)) {
@@ -5949,7 +7467,7 @@ export class WorkflowEngine {
5949
7467
  runId: run.id,
5950
7468
  workflow: run.workflow,
5951
7469
  namespace: run.namespace,
5952
- });
7470
+ }, { lifecycle: false });
5953
7471
  this.notifyParent(run.id, { ok: true, value: undefined });
5954
7472
  // Tracked handoff (see {@link handoffRun}): held in `postSettle` from this settle until the
5955
7473
  // continuation has entered `inflight` and settled, so `drain()` can't return before the next
@@ -5971,6 +7489,11 @@ export class WorkflowEngine {
5971
7489
  return await this.compensateAndCancel(run, compensations);
5972
7490
  }
5973
7491
  const settled = await this.settleRun(run, { kind: 'suspended', wakeAt: err.wakeAt });
7492
+ const stamp = this.suspendStamps.get(run.id);
7493
+ this.suspendStamps.delete(run.id);
7494
+ if (parkedTurn && cont && settled.status === 'suspended' && stamp !== undefined) {
7495
+ this.keepContinuation(cont, stamp);
7496
+ }
5974
7497
  // Close the settle-vs-delivery race: a signal/result that landed WHILE this turn was still
5975
7498
  // executing wrote its checkpoint durably, but its resume no-oped against our held lease —
5976
7499
  // and this settle just parked the run on nothing but the reconcile fallback (or nothing at
@@ -6029,6 +7552,9 @@ export class WorkflowEngine {
6029
7552
  //
6030
7553
  // A suspended turn keeps the lease instead when a wake it refused asked for another turn, and
6031
7554
  // runs that turn (see `requestWake`).
7555
+ if (cont && this.turnContinuations.get(run.id) === cont) {
7556
+ this.turnContinuations.delete(run.id);
7557
+ }
6032
7558
  let released = true;
6033
7559
  try {
6034
7560
  if (settledSuspended)
@@ -6279,10 +7805,11 @@ export class WorkflowEngine {
6279
7805
  /** The seam handed to {@link createWorkflowCtx}: the authoring API reaches durability + lifecycle
6280
7806
  * (checkpointing, dispatch, child start) through this, so the ctx primitives live in their own
6281
7807
  * module and the engine stays the orchestrator. */
6282
- ctxHostFor(replay, runNamespace) {
7808
+ ctxHostFor(replay, runNamespace, sticky) {
6283
7809
  return {
6284
7810
  store: this.store,
6285
7811
  replay,
7812
+ sticky,
6286
7813
  clock: this.clock,
6287
7814
  stepSignal: this.drainAbort.signal,
6288
7815
  eventBufferTtlMs: this.eventBufferTtlMs,
@@ -6290,7 +7817,18 @@ export class WorkflowEngine {
6290
7817
  startStep: (s) => this.startStep(s),
6291
7818
  completeStep: (s) => this.completeStep(s),
6292
7819
  failStep: (s) => this.failStep(s),
6293
- callRemote: (runId, seq, step, input, queue, transport, admission) => this.callRemote(runId, seq, step, input, queue, transport, replay, admission, runNamespace),
7820
+ callRemote: (runId, seq, step, input, queue, transport, admission) => {
7821
+ const call = this.callRemote(runId, seq, step, input, queue, transport, replay, admission, runNamespace);
7822
+ if (!sticky)
7823
+ return call;
7824
+ // In a sticky continuation a call that would suspend parks instead: a later turn
7825
+ // re-evaluates it (the same logic, against its freshly read checkpoint).
7826
+ return call.catch((err) => {
7827
+ if (!(err instanceof WorkflowSuspended) || !sticky.attached)
7828
+ throw err;
7829
+ return sticky.park(seq, err.wakeAt, (fresh) => this.callRemote(runId, seq, step, input, queue, transport, fresh ? new Map([[seq, fresh]]) : undefined, admission, runNamespace, undefined, true));
7830
+ });
7831
+ },
6294
7832
  callRemoteAll: (runId, positions, step, inputs, opts) => this.callRemoteAll(runId, positions, step, inputs, opts, runNamespace),
6295
7833
  // Defer so a fast child can't reentrantly resume a still-running parent. The child inherits the
6296
7834
  // PARENT RUN's namespace, not this engine's: an operator legitimately executes every tenant's
@@ -6421,12 +7959,12 @@ export class WorkflowEngine {
6421
7959
  * descriptor keyspace. Empty when no transport can introspect it (in-process / pre-handshake broker)
6422
7960
  * OR nobody is publishing — either way the guard is skipped and dispatch proceeds legacy
6423
7961
  * assume-compatible (design §7.7). Never throws (the transport degrades to `[]`). */
6424
- async liveDescriptorsFor(token) {
7962
+ async liveDescriptorsFor(token, fresh = false) {
6425
7963
  const read = this.primaryTransport?.listWorkerDescriptors;
6426
7964
  if (!read)
6427
7965
  return [];
6428
7966
  try {
6429
- return await read.call(this.primaryTransport, token);
7967
+ return await read.call(this.primaryTransport, token, fresh ? { fresh } : undefined);
6430
7968
  }
6431
7969
  catch {
6432
7970
  return [];
@@ -6439,8 +7977,8 @@ export class WorkflowEngine {
6439
7977
  * - a `routable` plan — ≥1 live worker is capability-capable AND protocol-compatible (dispatch);
6440
7978
  * - a `blocked` plan — descriptors exist but none can run it (park the run, never dispatch into it).
6441
7979
  */
6442
- async planRoute(token, requires) {
6443
- const descriptors = await this.liveDescriptorsFor(token);
7980
+ async planRoute(token, requires, fresh = false) {
7981
+ const descriptors = await this.liveDescriptorsFor(token, fresh);
6444
7982
  if (descriptors.length === 0)
6445
7983
  return undefined; // legacy/no-runtime path — skip the guard
6446
7984
  return planDispatch(requires ?? [], descriptors, this.cpDescriptor, token);
@@ -6450,7 +7988,10 @@ export class WorkflowEngine {
6450
7988
  * checkpoint + dispatching into a queue nobody consumes (design §7.5). A no-op when the guard does
6451
7989
  * not engage (no descriptors) or a capable+compatible worker exists. */
6452
7990
  async ensureRoutable(token, requires) {
6453
- const plan = await this.planRoute(token, requires);
7991
+ let plan = await this.planRoute(token, requires);
7992
+ // Never park a run on reused descriptors: a capable worker may have joined since they were read.
7993
+ if (plan?.status === 'blocked')
7994
+ plan = await this.planRoute(token, requires, true);
6454
7995
  if (plan?.status === 'blocked')
6455
7996
  throw new WorkflowBlocked(plan);
6456
7997
  }
@@ -6506,7 +8047,21 @@ export class WorkflowEngine {
6506
8047
  }));
6507
8048
  return blocked;
6508
8049
  }
6509
- async callRemote(runId, seq, step, input, queue, transport, replay, admission, runNamespace, parallelGroup) {
8050
+ async callRemote(runId, seq, step, input, queue, transport, replay, admission, runNamespace, parallelGroup,
8051
+ /** `replay` holds this step's checkpoint as just read from the store: no re-read needed. */
8052
+ fresh) {
8053
+ // A position the turn's history does not hold is normally a first dispatch: write its `pending`
8054
+ // row only if none is there, instead of reading the position first. A row that is there (a turn
8055
+ // this one did not see wrote it) is then read and handled below like any other.
8056
+ if (replay !== undefined &&
8057
+ !replay.has(seq) &&
8058
+ this.store.insertCheckpointIfAbsent &&
8059
+ !step.timeoutMs &&
8060
+ this.pool.size > 0 &&
8061
+ !(queue && this.admission.handles(queue)) &&
8062
+ (await this.dispatchIfAbsent(runId, seq, step, input, transport, admission, runNamespace, parallelGroup))) {
8063
+ throw new WorkflowSuspended(undefined, [seq]);
8064
+ }
6510
8065
  // Read the prefix from the per-execution snapshot (avoids the O(N²) replay SELECTs); a seq absent
6511
8066
  // from the snapshot — not yet dispatched, or written after the snapshot — falls back to the store.
6512
8067
  let existing = replay?.get(seq) ?? (await this.store.getCheckpoint(runId, seq));
@@ -6516,7 +8071,7 @@ export class WorkflowEngine {
6516
8071
  // the lost-dispatch lease, or a re-dispatch) — written from a stale row, that overwrites the landed
6517
8072
  // `completed` result with `pending` and parks the run on a lease an hour out. So a pending step is
6518
8073
  // re-read from the store before anything acts on it (one read, only at a suspension point).
6519
- if (existing?.status === 'pending' && replay?.has(seq)) {
8074
+ if (existing?.status === 'pending' && replay?.has(seq) && !fresh) {
6520
8075
  existing = (await this.store.getCheckpoint(runId, seq)) ?? existing;
6521
8076
  }
6522
8077
  if (existing && existing.name !== step.name) {
@@ -6597,6 +8152,8 @@ export class WorkflowEngine {
6597
8152
  priority: admission?.priority,
6598
8153
  attempt: reAttempt,
6599
8154
  transport,
8155
+ // A fan's steps are joined by its barrier, not a parked body: their results stay shared.
8156
+ replyHere: parallelGroup === undefined,
6600
8157
  });
6601
8158
  this.emit({
6602
8159
  type: 'step.started',
@@ -6701,10 +8258,66 @@ export class WorkflowEngine {
6701
8258
  attempt,
6702
8259
  transport,
6703
8260
  enqueuedAt,
8261
+ replyHere: parallelGroup === undefined,
6704
8262
  });
6705
8263
  this.emit({ type: 'step.started', runId, seq, name: step.name, kind: 'remote' });
6706
8264
  throw new WorkflowSuspended(undefined, [seq]);
6707
8265
  }
8266
+ /**
8267
+ * {@link callRemote}'s first dispatch of a step, written with
8268
+ * {@link StateStore.insertCheckpointIfAbsent}: resolves whether it wrote the `pending` row and
8269
+ * dispatched. `false` when the position already has a row, or when no live worker can take the
8270
+ * step (the read-first path then decides, so a step that is in fact done is never blocked).
8271
+ */
8272
+ async dispatchIfAbsent(runId, seq, step, input, transport, admission, runNamespace, parallelGroup) {
8273
+ const validInput = step.input
8274
+ ? await validateStandard(step.input, input, `step "${step.name}" input`, 'invalid_input')
8275
+ : input;
8276
+ const token = stepGroup(step, runNamespace);
8277
+ try {
8278
+ await this.ensureRoutable(token, step.requires);
8279
+ }
8280
+ catch (err) {
8281
+ if (err instanceof WorkflowBlocked)
8282
+ return false;
8283
+ throw err;
8284
+ }
8285
+ const id = stepId(runId, seq);
8286
+ const enqueuedAt = new Date();
8287
+ const insert = this.store.insertCheckpointIfAbsent;
8288
+ const wrote = await insert.call(this.store, {
8289
+ runId,
8290
+ seq,
8291
+ name: step.name,
8292
+ kind: 'remote',
8293
+ stepId: id,
8294
+ status: 'pending',
8295
+ input: validInput,
8296
+ attempts: 1,
8297
+ workerGroup: token,
8298
+ ...(parallelGroup !== undefined ? { parallelGroup } : {}),
8299
+ enqueuedAt,
8300
+ startedAt: enqueuedAt, // placeholders until the worker result lands
8301
+ finishedAt: enqueuedAt,
8302
+ });
8303
+ if (!wrote)
8304
+ return false;
8305
+ await this.dispatchRemoteTask({
8306
+ runId,
8307
+ seq,
8308
+ name: step.name,
8309
+ stepId: id,
8310
+ group: token,
8311
+ input: validInput,
8312
+ priority: admission?.priority,
8313
+ attempt: 1,
8314
+ transport,
8315
+ enqueuedAt,
8316
+ replyHere: parallelGroup === undefined,
8317
+ });
8318
+ this.emit({ type: 'step.started', runId, seq, name: step.name, kind: 'remote' });
8319
+ return true;
8320
+ }
6708
8321
  /**
6709
8322
  * `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
6710
8323
  * reserves the whole position block up front, so replay re-derives the same seqs.
@@ -6942,11 +8555,85 @@ export class WorkflowEngine {
6942
8555
  context: this.context?.(),
6943
8556
  priority: task.priority,
6944
8557
  attempt: task.attempt,
8558
+ ...(task.replyHere && this.repliesRouted && !this.draining
8559
+ ? { replyTo: this.instanceId }
8560
+ : {}),
6945
8561
  };
6946
8562
  if (task.enqueuedAt)
6947
8563
  remote[DISPATCH_KEY] = dispatchKey(task.stepId, task.attempt, task.enqueuedAt);
6948
8564
  return this.pool.dispatch(remote, task.transport);
6949
8565
  }
8566
+ /**
8567
+ * Whether this turn of `run` may run in a sticky continuation (see `sticky-continuation.ts`). Not
8568
+ * while the engine drains, nor with a compensating cancel requested for the run.
8569
+ */
8570
+ stickyEligible(run) {
8571
+ return this.sticky.enabled && !this.draining && !this.cancelRequested.has(run.id);
8572
+ }
8573
+ /**
8574
+ * The continuation this instance parked for `run`, if it is still the run's latest turn: the run
8575
+ * is `suspended` with the very stamp this instance's suspend wrote, on the same body and version.
8576
+ * Any other settle in between (another instance's turn, a retry, a cancel) changed the stamp, and
8577
+ * the run replays instead. Taking it removes it from the cache.
8578
+ */
8579
+ takeContinuation(run, fn) {
8580
+ const cont = this.sticky.take(run.id);
8581
+ if (!cont)
8582
+ return undefined;
8583
+ const valid = cont.fn === fn &&
8584
+ cont.workflowVersion === run.workflowVersion &&
8585
+ run.status === 'suspended' &&
8586
+ cont.stamp !== undefined &&
8587
+ run.updatedAt.getTime() === cont.stamp;
8588
+ if (!valid) {
8589
+ this.sticky.misses += 1;
8590
+ return undefined;
8591
+ }
8592
+ this.sticky.hits += 1;
8593
+ return cont;
8594
+ }
8595
+ /** Keep `cont` (parked, detached) for the run's next turn here, stamped with its suspend write. */
8596
+ keepContinuation(cont, stamp) {
8597
+ if (cont.unusable)
8598
+ return;
8599
+ cont.stamp = stamp;
8600
+ // Everything up to the park point is consumed; later positions read the store.
8601
+ cont.replay.clear();
8602
+ this.sticky.put(cont);
8603
+ }
8604
+ /**
8605
+ * The checkpoints of the steps `cont` is parked on, as a replay would see them: in one read when
8606
+ * the store batches, and without the input of a step that completed (what
8607
+ * {@link StateStore.getCheckpointsForSettle} leaves out). A step still open or failed is read
8608
+ * whole, since a re-dispatch or a retry sends its input again.
8609
+ */
8610
+ async readParked(runId, cont) {
8611
+ const fresh = new Map();
8612
+ for (const seq of cont.parked.keys())
8613
+ fresh.set(seq, cont.hints.get(seq) ?? null);
8614
+ cont.hints.clear();
8615
+ const seqs = [...fresh].flatMap(([seq, cp]) => (cp ? [] : [seq]));
8616
+ if (seqs.length === 0)
8617
+ return fresh;
8618
+ const lean = this.store.getCheckpointsForSettle ?? this.store.getCheckpoints;
8619
+ if (lean) {
8620
+ for (const cp of await lean.call(this.store, runId, seqs))
8621
+ fresh.set(cp.seq, cp);
8622
+ }
8623
+ else {
8624
+ for (const seq of seqs)
8625
+ fresh.set(seq, await this.store.getCheckpoint(runId, seq));
8626
+ }
8627
+ if (lean === this.store.getCheckpointsForSettle) {
8628
+ for (const seq of seqs) {
8629
+ const cp = fresh.get(seq);
8630
+ if (cp && cp.status !== 'completed') {
8631
+ fresh.set(seq, await this.store.getCheckpoint(runId, seq));
8632
+ }
8633
+ }
8634
+ }
8635
+ return fresh;
8636
+ }
6950
8637
  /** A turn's read of its run's checkpoints: {@link StateStore.listCheckpointsForReplay}, or every
6951
8638
  * column through {@link StateStore.listCheckpoints} on a store without it. */
6952
8639
  replayCheckpoints(runId) {
@@ -7217,6 +8904,9 @@ export class WorkflowEngine {
7217
8904
  * replays every write of the batch, so a second would only replay it again.
7218
8905
  */
7219
8906
  async completeRemoteResults(runId, results, queuedIn = () => false) {
8907
+ if (this.store.settleResultsIfPending) {
8908
+ return this.settleResultsReturning(runId, results, queuedIn);
8909
+ }
7220
8910
  const outcomes = results.map(() => true);
7221
8911
  const settleOneByOne = async (indexes) => {
7222
8912
  for (const i of indexes) {
@@ -7348,6 +9038,144 @@ export class WorkflowEngine {
7348
9038
  }
7349
9039
  return outcomes;
7350
9040
  }
9041
+ /**
9042
+ * {@link completeRemoteResults} on a store with {@link StateStore.settleResultsIfPending}: the
9043
+ * results settle their `pending` rows and read them back, with their run's status, in one call,
9044
+ * instead of reading the rows and the run and then writing. An entry that settled nothing (a
9045
+ * redelivery, a step gone, a late result for a finished run) takes the single-result path, which
9046
+ * reads first and re-drives, drops or salvages. A settled `completed` row is also handed to the
9047
+ * run's parked body, if this instance holds one, so its next turn does not read it again.
9048
+ */
9049
+ async settleResultsReturning(runId, results, queuedIn) {
9050
+ const outcomes = results.map(() => true);
9051
+ const settle = this.store.settleResultsIfPending.bind(this.store);
9052
+ // A second copy of a result in the same batch is a duplicate delivery: the first copy settles
9053
+ // the step (or re-drives the resume), so the copy is dropped like a late one.
9054
+ const seen = new Set();
9055
+ const live = results.flatMap((r, i) => {
9056
+ if (seen.has(r.seq))
9057
+ return [];
9058
+ seen.add(r.seq);
9059
+ return [i];
9060
+ });
9061
+ const finishedAt = new Date();
9062
+ const entries = live.map((i) => {
9063
+ const result = results[i];
9064
+ return {
9065
+ runId,
9066
+ seq: result.seq,
9067
+ status: result.status,
9068
+ output: result.status === 'completed' ? result.output : undefined,
9069
+ error: result.error,
9070
+ events: result.events,
9071
+ startedAt: result.startedAt ? new Date(result.startedAt) : undefined,
9072
+ finishedAt,
9073
+ };
9074
+ });
9075
+ // Never rejects as a whole: the batch call rejecting wrote nothing, so the entries are then
9076
+ // settled one by one and a failing one fails only its own result.
9077
+ let rows;
9078
+ try {
9079
+ rows = await settle(entries);
9080
+ }
9081
+ catch (err) {
9082
+ if (entries.length === 1) {
9083
+ rows = [{ error: err }];
9084
+ }
9085
+ else {
9086
+ const one = await Promise.allSettled(entries.map((e) => settle([e])));
9087
+ rows = one.map((o) => o.status === 'fulfilled' ? (o.value[0] ?? null) : { error: o.reason });
9088
+ }
9089
+ }
9090
+ const resumable = [];
9091
+ const unsettled = [];
9092
+ let run;
9093
+ // The run's parked body: in the cache, or in a turn that is parking it right now.
9094
+ const parked = this.sticky.peek(runId) ?? this.turnContinuations.get(runId);
9095
+ for (const [k, row] of rows.entries()) {
9096
+ const index = live[k];
9097
+ const result = results[index];
9098
+ if (row === null) {
9099
+ unsettled.push(index);
9100
+ continue;
9101
+ }
9102
+ if ('error' in row) {
9103
+ outcomes[index] = { error: row.error };
9104
+ continue;
9105
+ }
9106
+ const cp = row.checkpoint;
9107
+ // The run as it read when this result settled: what the resume owes it (see `resumeOwed`).
9108
+ run ??= { id: runId, status: row.run.status, wakeAt: row.run.wakeAt };
9109
+ try {
9110
+ await this.releaseQueueSlot(cp);
9111
+ }
9112
+ catch (error) {
9113
+ // Written but not acknowledged: the redelivery finds the step settled and re-drives.
9114
+ outcomes[index] = { error };
9115
+ }
9116
+ if (cp.status === 'completed')
9117
+ parked?.hint(cp);
9118
+ this.emit({
9119
+ type: result.status === 'completed' ? 'step.completed' : 'step.failed',
9120
+ runId,
9121
+ seq: result.seq,
9122
+ name: cp.name,
9123
+ kind: cp.kind,
9124
+ output: result.output,
9125
+ error: result.error,
9126
+ queueMs: cp.startedAt.getTime() - cp.enqueuedAt.getTime(),
9127
+ durationMs: finishedAt.getTime() - cp.startedAt.getTime(),
9128
+ });
9129
+ resumable.push({ cp, result });
9130
+ }
9131
+ // Nothing written for these: read first, then re-drive, drop or salvage, one at a time.
9132
+ for (const index of unsettled) {
9133
+ try {
9134
+ await this.completeOneRemoteResult(results[index], queuedIn);
9135
+ }
9136
+ catch (error) {
9137
+ outcomes[index] = { error };
9138
+ }
9139
+ }
9140
+ if (resumable.length === 0)
9141
+ return outcomes;
9142
+ // The barrier logic of the read-first batch path (see completeRemoteResults): one resume.
9143
+ const barriers = new Map();
9144
+ const highest = new Map();
9145
+ for (const { cp } of resumable) {
9146
+ const fan = parseStepFanGroup(cp.parallelGroup);
9147
+ if (fan)
9148
+ highest.set(fan.firstSeq, Math.max(highest.get(fan.firstSeq) ?? cp.seq, cp.seq));
9149
+ }
9150
+ const singles = resumable
9151
+ .filter(({ cp }) => !parseStepFanGroup(cp.parallelGroup))
9152
+ .map(({ cp }) => cp.seq);
9153
+ for (const { cp, result } of resumable) {
9154
+ const fan = parseStepFanGroup(cp.parallelGroup);
9155
+ if (!fan || result.status === 'failed') {
9156
+ this.noteSettledHere(runId, singles);
9157
+ await this.resumeOwed(run, runId);
9158
+ return outcomes;
9159
+ }
9160
+ let closed = barriers.get(fan.firstSeq);
9161
+ if (!closed) {
9162
+ const lastSeq = fan.firstSeq + fan.size - 1;
9163
+ closed = queuedIn(fan.firstSeq, lastSeq)
9164
+ ? Promise.resolve(true)
9165
+ : this.stepFanHasPending(runId, fan.firstSeq, lastSeq).catch(() => false);
9166
+ barriers.set(fan.firstSeq, closed);
9167
+ }
9168
+ if (!(await closed)) {
9169
+ this.noteSettledHere(runId, singles);
9170
+ await this.resumeOwed(run, runId, {
9171
+ first: fan.firstSeq,
9172
+ seq: highest.get(fan.firstSeq) ?? cp.seq,
9173
+ });
9174
+ return outcomes;
9175
+ }
9176
+ }
9177
+ return outcomes;
9178
+ }
7351
9179
  /**
7352
9180
  * Settle pending steps, each only while its row is still `pending`; per entry, whether it wrote
7353
9181
  * or the error that kept it from writing. Through {@link StateStore.settleCheckpointsIfPending}
@@ -7798,8 +9626,30 @@ export class WorkflowEngine {
7798
9626
  });
7799
9627
  }
7800
9628
  }
7801
- /** The worker group a convention-routed workflow's run dispatches to: its sanitized name, suffixed by
7802
- * the run's namespace (bare for absent/`''`/`default`) — the token a tenant worker serves. */
9629
+ /**
9630
+ * The namespace an announcement serves: the descriptor's own, else the `@<namespace>` suffix of its
9631
+ * routing group, else `default`.
9632
+ */
9633
+ function servedNamespace(namespace, group) {
9634
+ if (namespace)
9635
+ return namespace;
9636
+ const at = group?.lastIndexOf('@') ?? -1;
9637
+ return group && at >= 0 && at < group.length - 1 ? group.slice(at + 1) : 'default';
9638
+ }
9639
+ function withoutBookkeepingRuns(query) {
9640
+ if (query.workflow === LIFECYCLE_REGISTRY_WORKFLOW ||
9641
+ query.workflows?.includes(LIFECYCLE_REGISTRY_WORKFLOW)) {
9642
+ return query;
9643
+ }
9644
+ const excluded = query.excludeWorkflows ?? [];
9645
+ if (excluded.includes(LIFECYCLE_REGISTRY_WORKFLOW))
9646
+ return query;
9647
+ return { ...query, excludeWorkflows: [...excluded, LIFECYCLE_REGISTRY_WORKFLOW] };
9648
+ }
9649
+ /** `value` when it has at least one own key, else undefined. */
9650
+ function nonEmptyObject(value) {
9651
+ return Object.keys(value).length > 0 ? value : undefined;
9652
+ }
7803
9653
  function conventionGroup(name, namespace) {
7804
9654
  return tenantGroup(sanitizeQueueToken(name), namespace);
7805
9655
  }