@adonis-agora/durable 0.58.2 → 0.59.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/CHANGELOG.md +50 -0
  2. package/dist/assets/spa/assets/index-Co0ef3-q.css +1 -0
  3. package/dist/assets/spa/assets/index-DmgfShPp.js +16 -0
  4. package/dist/assets/spa/assets/index-DmgfShPp.js.map +1 -0
  5. package/dist/assets/spa/index.html +2 -2
  6. package/dist/commands/lifecycle_republish.d.ts +23 -0
  7. package/dist/commands/lifecycle_republish.d.ts.map +1 -0
  8. package/dist/commands/lifecycle_republish.js +64 -0
  9. package/dist/commands/lifecycle_republish.js.map +1 -0
  10. package/dist/commands/main.d.ts +4 -3
  11. package/dist/commands/main.d.ts.map +1 -1
  12. package/dist/commands/main.js +3 -1
  13. package/dist/commands/main.js.map +1 -1
  14. package/dist/providers/dashboard_provider.d.ts +2 -0
  15. package/dist/providers/dashboard_provider.d.ts.map +1 -1
  16. package/dist/providers/dashboard_provider.js +9 -1
  17. package/dist/providers/dashboard_provider.js.map +1 -1
  18. package/dist/src/commands/index.d.ts +1 -1
  19. package/dist/src/commands/index.d.ts.map +1 -1
  20. package/dist/src/commands/index.js +1 -1
  21. package/dist/src/commands/index.js.map +1 -1
  22. package/dist/src/commands/runs.d.ts +24 -1
  23. package/dist/src/commands/runs.d.ts.map +1 -1
  24. package/dist/src/commands/runs.js +50 -0
  25. package/dist/src/commands/runs.js.map +1 -1
  26. package/dist/src/commands/worker.d.ts.map +1 -1
  27. package/dist/src/commands/worker.js +3 -0
  28. package/dist/src/commands/worker.js.map +1 -1
  29. package/dist/src/dashboard/gateway-adapter.d.ts +5 -0
  30. package/dist/src/dashboard/gateway-adapter.d.ts.map +1 -1
  31. package/dist/src/dashboard/gateway-adapter.js +20 -0
  32. package/dist/src/dashboard/gateway-adapter.js.map +1 -1
  33. package/dist/src/dashboard/handlers.d.ts +29 -1
  34. package/dist/src/dashboard/handlers.d.ts.map +1 -1
  35. package/dist/src/dashboard/handlers.js +103 -0
  36. package/dist/src/dashboard/handlers.js.map +1 -1
  37. package/dist/src/dashboard/index.d.ts +2 -2
  38. package/dist/src/dashboard/index.d.ts.map +1 -1
  39. package/dist/src/dashboard/index.js +2 -2
  40. package/dist/src/dashboard/index.js.map +1 -1
  41. package/dist/src/dashboard/openapi.d.ts.map +1 -1
  42. package/dist/src/dashboard/openapi.js +39 -0
  43. package/dist/src/dashboard/openapi.js.map +1 -1
  44. package/dist/src/diagnostics-bridge.d.ts +5 -1
  45. package/dist/src/diagnostics-bridge.d.ts.map +1 -1
  46. package/dist/src/diagnostics-bridge.js +16 -2
  47. package/dist/src/diagnostics-bridge.js.map +1 -1
  48. package/dist/src/engine.d.ts +365 -5
  49. package/dist/src/engine.d.ts.map +1 -1
  50. package/dist/src/engine.js +1491 -76
  51. package/dist/src/engine.js.map +1 -1
  52. package/dist/src/event-accumulators.d.ts +28 -5
  53. package/dist/src/event-accumulators.d.ts.map +1 -1
  54. package/dist/src/event-accumulators.js +102 -21
  55. package/dist/src/event-accumulators.js.map +1 -1
  56. package/dist/src/events.d.ts +25 -0
  57. package/dist/src/events.d.ts.map +1 -1
  58. package/dist/src/events.js +24 -0
  59. package/dist/src/events.js.map +1 -1
  60. package/dist/src/handshake/announced.d.ts +6 -0
  61. package/dist/src/handshake/announced.d.ts.map +1 -1
  62. package/dist/src/handshake/announced.js +20 -0
  63. package/dist/src/handshake/announced.js.map +1 -1
  64. package/dist/src/handshake/descriptor.d.ts +6 -0
  65. package/dist/src/handshake/descriptor.d.ts.map +1 -1
  66. package/dist/src/handshake/descriptor.js +1 -0
  67. package/dist/src/handshake/descriptor.js.map +1 -1
  68. package/dist/src/index.d.ts +3 -2
  69. package/dist/src/index.d.ts.map +1 -1
  70. package/dist/src/index.js +3 -2
  71. package/dist/src/index.js.map +1 -1
  72. package/dist/src/interfaces.d.ts +45 -2
  73. package/dist/src/interfaces.d.ts.map +1 -1
  74. package/dist/src/interfaces.js.map +1 -1
  75. package/dist/src/lifecycle-events.d.ts +372 -0
  76. package/dist/src/lifecycle-events.d.ts.map +1 -0
  77. package/dist/src/lifecycle-events.js +383 -0
  78. package/dist/src/lifecycle-events.js.map +1 -0
  79. package/dist/src/metrics.d.ts +15 -0
  80. package/dist/src/metrics.d.ts.map +1 -1
  81. package/dist/src/metrics.js +36 -1
  82. package/dist/src/metrics.js.map +1 -1
  83. package/dist/src/otel/durable-metrics.d.ts +9 -0
  84. package/dist/src/otel/durable-metrics.d.ts.map +1 -1
  85. package/dist/src/otel/durable-metrics.js +17 -0
  86. package/dist/src/otel/durable-metrics.js.map +1 -1
  87. package/dist/src/otel/durable-otel.d.ts +7 -0
  88. package/dist/src/otel/durable-otel.d.ts.map +1 -1
  89. package/dist/src/otel/durable-otel.js +69 -3
  90. package/dist/src/otel/durable-otel.js.map +1 -1
  91. package/dist/src/run-detail-summary.d.ts +48 -1
  92. package/dist/src/run-detail-summary.d.ts.map +1 -1
  93. package/dist/src/run-detail-summary.js +30 -2
  94. package/dist/src/run-detail-summary.js.map +1 -1
  95. package/dist/src/run-gateway/interface.d.ts +16 -0
  96. package/dist/src/run-gateway/interface.d.ts.map +1 -1
  97. package/dist/src/run-gateway/proxy-run-gateway.d.ts +31 -1
  98. package/dist/src/run-gateway/proxy-run-gateway.d.ts.map +1 -1
  99. package/dist/src/run-gateway/proxy-run-gateway.js +59 -7
  100. package/dist/src/run-gateway/proxy-run-gateway.js.map +1 -1
  101. package/dist/src/run-gateway/run-request-responder.d.ts +4 -29
  102. package/dist/src/run-gateway/run-request-responder.d.ts.map +1 -1
  103. package/dist/src/run-gateway/run-request-responder.js +183 -6
  104. package/dist/src/run-gateway/run-request-responder.js.map +1 -1
  105. package/dist/src/run-gateway/store-run-gateway.d.ts +9 -0
  106. package/dist/src/run-gateway/store-run-gateway.d.ts.map +1 -1
  107. package/dist/src/run-gateway/store-run-gateway.js +8 -0
  108. package/dist/src/run-gateway/store-run-gateway.js.map +1 -1
  109. package/dist/src/run-waiting.d.ts +1 -0
  110. package/dist/src/run-waiting.d.ts.map +1 -1
  111. package/dist/src/run-waiting.js +6 -1
  112. package/dist/src/run-waiting.js.map +1 -1
  113. package/dist/src/stores/lucid.d.ts.map +1 -1
  114. package/dist/src/stores/lucid.js +4 -0
  115. package/dist/src/stores/lucid.js.map +1 -1
  116. package/dist/src/telescope/dashboard.d.ts.map +1 -1
  117. package/dist/src/telescope/dashboard.js +50 -0
  118. package/dist/src/telescope/dashboard.js.map +1 -1
  119. package/dist/src/telescope/data-providers.d.ts +19 -0
  120. package/dist/src/telescope/data-providers.d.ts.map +1 -1
  121. package/dist/src/telescope/data-providers.js +128 -2
  122. package/dist/src/telescope/data-providers.js.map +1 -1
  123. package/dist/src/telescope/extension.d.ts.map +1 -1
  124. package/dist/src/telescope/extension.js +4 -1
  125. package/dist/src/telescope/extension.js.map +1 -1
  126. package/dist/src/telescope/index.d.ts +3 -2
  127. package/dist/src/telescope/index.d.ts.map +1 -1
  128. package/dist/src/telescope/index.js +2 -1
  129. package/dist/src/telescope/index.js.map +1 -1
  130. package/dist/src/telescope/telescope-sdk.d.ts +16 -0
  131. package/dist/src/telescope/telescope-sdk.d.ts.map +1 -1
  132. package/dist/src/telescope/watcher.d.ts +66 -0
  133. package/dist/src/telescope/watcher.d.ts.map +1 -0
  134. package/dist/src/telescope/watcher.js +185 -0
  135. package/dist/src/telescope/watcher.js.map +1 -0
  136. package/dist/src/testing/in-memory-state-store.d.ts.map +1 -1
  137. package/dist/src/testing/in-memory-state-store.js +8 -0
  138. package/dist/src/testing/in-memory-state-store.js.map +1 -1
  139. package/dist/src/testing-kit/harness.d.ts +1 -1
  140. package/dist/src/testing-kit/harness.d.ts.map +1 -1
  141. package/dist/src/testing-kit/harness.js +3 -1
  142. package/dist/src/testing-kit/harness.js.map +1 -1
  143. package/dist/src/testing-kit/state-store-conformance.d.ts.map +1 -1
  144. package/dist/src/testing-kit/state-store-conformance.js +59 -0
  145. package/dist/src/testing-kit/state-store-conformance.js.map +1 -1
  146. package/dist/src/workflow-ctx.d.ts.map +1 -1
  147. package/dist/src/workflow-ctx.js +62 -0
  148. package/dist/src/workflow-ctx.js.map +1 -1
  149. package/dist/src/workflow-discovery.d.ts.map +1 -1
  150. package/dist/src/workflow-discovery.js +1 -0
  151. package/dist/src/workflow-discovery.js.map +1 -1
  152. package/dist/src/workflow-events.d.ts +101 -0
  153. package/dist/src/workflow-events.d.ts.map +1 -1
  154. package/dist/src/workflow-events.js +79 -1
  155. package/dist/src/workflow-events.js.map +1 -1
  156. package/dist/src/workflow-ref.d.ts +17 -2
  157. package/dist/src/workflow-ref.d.ts.map +1 -1
  158. package/dist/src/workflow-ref.js.map +1 -1
  159. package/package.json +4 -4
  160. package/skills/durable-cluster/SKILL.md +4 -2
  161. package/dist/assets/spa/assets/index-6nmSTfT7.css +0 -1
  162. package/dist/assets/spa/assets/index-DjdTutsN.js +0 -16
  163. package/dist/assets/spa/assets/index-DjdTutsN.js.map +0 -1
@@ -12,23 +12,24 @@ import { controlPlaneDescriptor, planDispatch, WorkflowBlocked, } from './dispat
12
12
  import { parseDuration } from './duration.js';
13
13
  import { Entities } from './entities.js';
14
14
  import { ContinueAsNew, GatherError, NonDeterminismError, RemoteStepTimeout, RemoteWorkflowTimeout, WorkflowSuspended, } from './errors.js';
15
- import { EventAccumulators } from './event-accumulators.js';
16
- import { eventMatches, eventMatchOf, eventPrefix } from './events.js';
15
+ import { accumulatorRunId, EventAccumulators, } from './event-accumulators.js';
16
+ import { acceptingSubscription, eventMatches, eventMatchOf, eventPrefix, toEventSubscription, } from './events.js';
17
17
  import { childIdsOf, FOLDED_PREFIX, fanOfWaiter, parentRunIdOf, parseStepFanGroup, stepFanGroup, } from './fan-join.js';
18
- import { aggregateAnnouncements } from './handshake/announced.js';
18
+ import { aggregateAnnouncements, announcementsOf, } from './handshake/announced.js';
19
19
  import { buildWorkflowDirectory } from './handshake/directory.js';
20
20
  import { isNonEmptyRunScope, TERMINAL_RUN_STATUSES } from './interfaces.js';
21
- import { DEFAULT_NAMESPACE } from './namespaces.js';
21
+ import { chainPredecessorsOf, coalescedSubscribersOf, deliveredSubscribersOf, LIFECYCLE_HEAL_WINDOW_MS, LIFECYCLE_ORPHAN_MS, LIFECYCLE_OWED_TAG_PREFIX, LIFECYCLE_PENDING_TAG, LIFECYCLE_REGISTRY_HEARTBEAT_MS, LIFECYCLE_REGISTRY_STALE_MS, LIFECYCLE_REGISTRY_TAG, LIFECYCLE_REGISTRY_TTL_MS, LIFECYCLE_REGISTRY_WORKFLOW, LIFECYCLE_TRACE_TAG_PREFIX, LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX, LIFECYCLE_TRIGGERED_TAG_PREFIX, lastRunOfChain, lifecycleCoalescedTag, lifecycleDeliveriesOf, lifecycleEventName, lifecycleEventOf, lifecycleKindOf, lifecycleOwedTag, lifecycleRegistryNamespace, lifecycleRegistryRunId, lifecycleRegistrySubscriptions, lifecycleRunId, lifecycleSourceOf, lifecycleTraceparentOf, lifecycleTriggerFailedTag, nextContinuationId, owedSubscribersOf, parseLifecycleEvent, RUN_WAIT_TOKEN_PREFIX, retentionKeeps, runWaitPrefix, runWaitTargetOf, triggeredRunsOf, } from './lifecycle-events.js';
22
+ import { DEFAULT_NAMESPACE, normalizeNamespace } from './namespaces.js';
22
23
  import { asHeartbeat } from './observability-scope.js';
23
24
  import { assertRoutes, codeAndDetails, HANDOFF_SEQ_BASE, handoffHandlerId, handoffName, liveHandoff, onErrorRunId, retiredHandoffName, routeFor, } from './on-error.js';
24
25
  import { ORIGIN_RESOLVED, originOfCaller } from './origin.js';
25
26
  import { breakpointToken, stepId } from './protocol.js';
26
27
  import { markRemoteEvent } from './remote-event.js';
27
28
  import { RemoteWorkflowExecutor } from './remote-workflow-executor.js';
28
- import { loadFanMembers, loadRunDetail, } from './run-detail-summary.js';
29
+ import { etagMatches, extendEtag, loadFanMembers, loadRunDetail, } from './run-detail-summary.js';
29
30
  import { runFacetsOf } from './run-facets.js';
30
31
  import { scanRunValueFacets } from './run-value-facets.js';
31
- import { listSignalWaitersOfRuns } from './run-waiting.js';
32
+ import { getRunsByIds, listSignalWaitersOfRuns } from './run-waiting.js';
32
33
  import { resolveSchedule, scheduleWindow } from './scheduler.js';
33
34
  import { ScheduleClient } from './schedules.js';
34
35
  import { SINGLETON_RESERVED_TAGS, withoutAdmission, withSingletonTracked, } from './singleton-admission.js';
@@ -168,6 +169,8 @@ const RUN_DISPATCH_BATCH = 500;
168
169
  /** Tries, and the first backoff, for telling a parent that its child finished (see `deliverToParent`). */
169
170
  const PARENT_NOTIFY_ATTEMPTS = 4;
170
171
  const PARENT_NOTIFY_BACKOFF_MS = 250;
172
+ /** What a lifecycle publish's run lease owner carries: `<instanceId>#lifecycle#<uuid>`. */
173
+ const LIFECYCLE_LEASE_MARK = '#lifecycle#';
171
174
  /** How often, at most, and how many times a wake dropped on a held lease looks again (see `recheckAfterLease`). */
172
175
  const LEASE_RECHECK_MAX_MS = 2_000;
173
176
  const LEASE_RECHECK_SLACK_MS = 100;
@@ -217,11 +220,6 @@ function shuffle(items) {
217
220
  function stepGroup(step, runNamespace) {
218
221
  return tenantGroup(sanitizeQueueToken(step.name), step.partition ?? runNamespace);
219
222
  }
220
- /** The id for the next continuation of a run: `r` → `r~1` → `r~2` … (stable, traceable lineage). */
221
- function nextContinuationId(runId) {
222
- const m = runId.match(/^(.*)~(\d+)$/);
223
- return m ? `${m[1]}~${Number(m[2]) + 1}` : `${runId}~1`;
224
- }
225
223
  /** True when version `a` is newer than `b` (numeric when both parse as numbers, else natural sort). */
226
224
  function isNewerVersion(a, b) {
227
225
  const na = Number(a);
@@ -335,7 +333,7 @@ export class WorkflowEngine {
335
333
  workflows = new Map();
336
334
  /** The newest registered version per workflow name — used to `start` new runs. */
337
335
  latest = new Map();
338
- /** Event name → workflow names started when that event is published (see `onEvent`). */
336
+ /** Event name → subscribing workflow name → that workflow's subscriptions to it. See `onEvent`. */
339
337
  eventTriggers = new Map();
340
338
  /** Durable-entity subsystem (registers the `__entity` runner; see `registerEntity`). */
341
339
  entities;
@@ -399,6 +397,12 @@ export class WorkflowEngine {
399
397
  * own release as a takeover: a late `false` renew would otherwise fence whoever applies the decision.
400
398
  */
401
399
  executionLeases = new Map();
400
+ /** Runs a caller just locked to execute now, so their turn has nothing to prove and no await to add. */
401
+ lockedForTurn = new Set();
402
+ /** Runs this instance holds for a moment and will release, not to execute (see {@link handOffDue}). */
403
+ claimsInFlight = new Map();
404
+ /** How many such claims each run has had, so a proof can tell one came and went (see {@link proveOwnLease}). */
405
+ claimsMade = new Map();
402
406
  /** Stable decision handler subscribed on every remote executor's transport (see registerRemote). */
403
407
  decisionHandler = (decision) => {
404
408
  // A transport that delivers the decision from INSIDE its own `dispatchWorkflowTask` (an in-process
@@ -1103,10 +1107,11 @@ export class WorkflowEngine {
1103
1107
  const current = this.latest.get(name);
1104
1108
  if (!current || isNewerVersion(version, current.version))
1105
1109
  this.latest.set(name, registered);
1106
- for (const event of opts?.onEvent ?? []) {
1107
- const subscribers = this.eventTriggers.get(event) ?? new Set();
1108
- subscribers.add(name);
1109
- this.eventTriggers.set(event, subscribers);
1110
+ for (const entry of opts?.onEvent ?? []) {
1111
+ const subscription = toEventSubscription(entry);
1112
+ const subscribers = this.eventTriggers.get(subscription.event) ?? new Map();
1113
+ subscribers.set(name, [...(subscribers.get(name) ?? []), subscription]);
1114
+ this.eventTriggers.set(subscription.event, subscribers);
1110
1115
  }
1111
1116
  }
1112
1117
  /**
@@ -1403,17 +1408,1226 @@ export class WorkflowEngine {
1403
1408
  console.warn(`[adonis-durable] could not start dead-letter workflow ${target} for run ${run.id}: ${err instanceof Error ? err.message : String(err)}`);
1404
1409
  }));
1405
1410
  }
1406
- /** Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the
1407
- * control plane so other instances (e.g. a dashboard pod) can live-tail this run. */
1408
- emit(event) {
1411
+ /**
1412
+ * Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the control
1413
+ * plane so other instances (e.g. a dashboard pod) can live-tail this run. A `run.completed` or
1414
+ * `run.failed` also wakes the `ctx.waitForRun` waiters on the run and publishes its lifecycle
1415
+ * event (see {@link publishLifecycle}); `lifecycle: false` marks a `run.completed` that is not a
1416
+ * real completion (a continue-as-new handoff), which publishes no lifecycle event.
1417
+ */
1418
+ emit(event, opts) {
1409
1419
  const full = { ...event, at: new Date() };
1410
1420
  this.deliver(full);
1421
+ if (full.type === 'run.completed' || full.type === 'run.failed') {
1422
+ this.notifyRunWaiters(full.runId, full.workflow);
1423
+ }
1424
+ if (opts?.lifecycle !== false)
1425
+ this.publishLifecycle(full);
1411
1426
  if (!this.controlPlane)
1412
1427
  return;
1413
1428
  this.eventsToBroadcast.push(full);
1414
1429
  if (!this.broadcastingEvents)
1415
1430
  void this.broadcastEvents();
1416
1431
  }
1432
+ /**
1433
+ * Start every workflow subscribed to `name` that takes `payload` (see {@link acceptingSubscription}),
1434
+ * each under `runIdOf(workflow)` and with its subscription's `input` applied; only the ones in
1435
+ * `only` when given. Returns the subscribers reached, with the run each started (none when the
1436
+ * payload went into its debounce/batch accumulator) and whether that run is new, and the ones that
1437
+ * could not be started, with why.
1438
+ *
1439
+ * `remote` names subscribers known only from the fleet's announcements (see
1440
+ * {@link announcedRemoteSubscribers}): each is started by name with the payload as its input, with
1441
+ * no `where` or `input` mapping, unless it is also registered here.
1442
+ */
1443
+ async startSubscribers(name, payload, runIdOf, only, remote = []) {
1444
+ const reached = [];
1445
+ const failed = [];
1446
+ for (const [workflow, subscriptions] of this.eventTriggers.get(name) ?? []) {
1447
+ if (only && !only.has(workflow))
1448
+ continue;
1449
+ const { accepted, errors } = acceptingSubscription(subscriptions, payload);
1450
+ if (!accepted) {
1451
+ if (errors.length > 0)
1452
+ failed.push({ workflow, error: errors[0] });
1453
+ continue;
1454
+ }
1455
+ // A subscriber that rejects the payload (validateInput / inputSchema) must not block the
1456
+ // others or the waiters: its run never starts, and the caller decides how to report it.
1457
+ try {
1458
+ const input = accepted.input ? accepted.input(payload) : payload;
1459
+ const batch = this.latest.get(workflow)?.eventBatch;
1460
+ if (batch) {
1461
+ // Coalesce: route the event into the workflow's accumulator (one long-lived run per
1462
+ // target and key that debounces/batches and then starts the target), named by the run
1463
+ // id it would start, so a second route of the same event is dropped there.
1464
+ await this.accumulators.route(workflow, batch, input, runIdOf(workflow));
1465
+ reached.push({ workflow });
1466
+ }
1467
+ else {
1468
+ const runId = runIdOf(workflow);
1469
+ const existed = (await this.store.getRun(runId)) !== null;
1470
+ await this.start(workflow, input, runId);
1471
+ reached.push({ workflow, runId, created: !existed });
1472
+ }
1473
+ }
1474
+ catch (error) {
1475
+ failed.push({ workflow, error });
1476
+ }
1477
+ }
1478
+ const local = this.eventTriggers.get(name);
1479
+ for (const workflow of remote) {
1480
+ if (local?.has(workflow) || (only && !only.has(workflow)))
1481
+ continue;
1482
+ try {
1483
+ const runId = runIdOf(workflow);
1484
+ const existed = (await this.store.getRun(runId)) !== null;
1485
+ await this.start(workflow, payload, runId);
1486
+ reached.push({ workflow, runId, created: !existed });
1487
+ }
1488
+ catch (error) {
1489
+ failed.push({ workflow, error });
1490
+ }
1491
+ }
1492
+ return { reached, failed };
1493
+ }
1494
+ /** The workflows registered on this instance that subscribe to `event`. */
1495
+ localSubscribers(event) {
1496
+ return [...(this.eventTriggers.get(event)?.keys() ?? [])];
1497
+ }
1498
+ /**
1499
+ * Whether this engine's own subscribers are owed the lifecycle events of `namespace`'s runs: always
1500
+ * for an engine with no namespace (its sweep reads every namespace's runs), else only its own
1501
+ * namespace's. `undefined` asks without a namespace and is always true.
1502
+ */
1503
+ servesNamespace(namespace) {
1504
+ return (namespace === undefined ||
1505
+ this.namespace === undefined ||
1506
+ normalizeNamespace(this.namespace) === normalizeNamespace(namespace));
1507
+ }
1508
+ /** The workflows registered here that subscribe to `event` and serve `namespace`'s runs. */
1509
+ localSubscribersFor(event, namespace) {
1510
+ return this.servesNamespace(namespace) ? this.localSubscribers(event) : [];
1511
+ }
1512
+ /**
1513
+ * The subscribers of `event` this engine starts by name although it has no `onEvent` for them:
1514
+ * announced only by non-Node workers (a Python `@worker.workflow(..., on_workflow_completed=...)`),
1515
+ * and reachable from here, either registered (`registerRemote`) or served by convention on a live
1516
+ * worker group of the same name. A subscriber some Node engine announces is left to that engine,
1517
+ * which applies its own `where` and `input`; one this engine cannot reach stays owed, for an
1518
+ * instance that can. Read from the snapshot {@link announcedSubscribers} keeps. With `namespace`,
1519
+ * only the subscriptions announced for that namespace's runs (see {@link announcedFor}).
1520
+ */
1521
+ async announcedRemoteSubscribers(event, namespace) {
1522
+ const announced = this.announcedFor(event, namespace);
1523
+ if (announced.size === 0)
1524
+ return [];
1525
+ const local = new Set(this.localSubscribers(event));
1526
+ const byEngine = this.announcedFor(event, namespace, { node: true });
1527
+ const startable = [];
1528
+ for (const workflow of [...announced].sort()) {
1529
+ if (local.has(workflow) || byEngine?.has(workflow) || workflow.startsWith('__'))
1530
+ continue;
1531
+ const registered = this.latest.get(workflow);
1532
+ if (registered && !registered.convention) {
1533
+ startable.push(workflow);
1534
+ continue;
1535
+ }
1536
+ try {
1537
+ if (await this.pool.transportWithLiveGroup(conventionGroup(workflow, this.namespace))) {
1538
+ startable.push(workflow);
1539
+ }
1540
+ }
1541
+ catch {
1542
+ // An unreadable fleet leaves the subscriber owed, for the next sweep.
1543
+ }
1544
+ }
1545
+ return startable;
1546
+ }
1547
+ /**
1548
+ * The subscribers of `event` this engine can deliver to for a run of `namespace`: the ones
1549
+ * registered here that serve it, and the announced ones it starts by name (see
1550
+ * {@link announcedRemoteSubscribers}).
1551
+ */
1552
+ async deliverableSubscribers(event, namespace) {
1553
+ const local = new Set(this.localSubscribersFor(event, namespace));
1554
+ const remote = await this.announcedRemoteSubscribers(event, namespace);
1555
+ return { local, remote, all: new Set([...local, ...remote]) };
1556
+ }
1557
+ /**
1558
+ * Every workflow a run of `workflow` settling as `kind` owes its lifecycle event to: the ones
1559
+ * registered here plus the ones the fleet announces (from a snapshot of the lifecycle subscription
1560
+ * registry and the worker descriptors at most 5s old), sorted.
1561
+ *
1562
+ * With `namespace` (the settled run's), only the subscribers that serve that namespace's runs: the
1563
+ * ones registered here when this engine has no namespace or has that one, the registry rows of
1564
+ * engines in that namespace or with none, and the worker descriptors serving it. What the
1565
+ * terminal status write marks the run owed to.
1566
+ */
1567
+ lifecycleSubscribers(workflow, kind, namespace) {
1568
+ if (workflow.startsWith('__'))
1569
+ return [];
1570
+ const event = lifecycleEventName(workflow, kind);
1571
+ return [
1572
+ ...new Set([
1573
+ ...this.localSubscribersFor(event, namespace),
1574
+ ...this.announcedSubscribers(event, namespace),
1575
+ ]),
1576
+ ].sort();
1577
+ }
1578
+ /**
1579
+ * Who starts whom through lifecycle events, for `workflow`: the workflows that start when it
1580
+ * settles, and the workflows whose settling starts it. Built from this engine's registry and the
1581
+ * subscriptions live workers announce (read again when the snapshot is older than 5s); a failed
1582
+ * fleet read leaves the local registry alone. What the dashboard's workflow page shows.
1583
+ *
1584
+ * With `namespace`, only the subscriptions of that namespace: the ones announced by registry runs
1585
+ * and worker descriptors serving it, and this engine's own when it is that namespace's engine (an
1586
+ * engine with no namespace is `default`'s). What a tenant is shown.
1587
+ */
1588
+ async workflowLifecycle(workflow, opts) {
1589
+ await this.freshAnnouncedSubscriptions();
1590
+ const snapshot = this.announcedSubscriptionsSnapshot;
1591
+ const scope = opts?.namespace;
1592
+ let fleet = snapshot?.byEvent ?? new Map();
1593
+ if (scope !== undefined) {
1594
+ fleet = new Map();
1595
+ for (const entry of snapshot?.entries ?? []) {
1596
+ if ((entry.namespace ?? DEFAULT_NAMESPACE) !== scope)
1597
+ continue;
1598
+ const subscribers = fleet.get(entry.event) ?? new Set();
1599
+ subscribers.add(entry.workflow);
1600
+ fleet.set(entry.event, subscribers);
1601
+ }
1602
+ }
1603
+ const triggers = scope === undefined || (this.namespace || 'default') === scope
1604
+ ? this.eventTriggers
1605
+ : new Map();
1606
+ const startsOnSettle = new Map();
1607
+ const startedBy = new Map();
1608
+ const link = (into, other, kind, local) => {
1609
+ const key = `${other}\u0000${kind}`;
1610
+ const entry = into.get(key) ?? {
1611
+ workflow: other,
1612
+ kind,
1613
+ local: false,
1614
+ announced: false,
1615
+ where: false,
1616
+ input: false,
1617
+ };
1618
+ if (local) {
1619
+ entry.local = true;
1620
+ entry.where ||= local.subscriptions.some((sub) => sub.where !== undefined);
1621
+ entry.input ||= local.subscriptions.some((sub) => sub.input !== undefined);
1622
+ const batch = this.latest.get(local.subscriber)?.eventBatch;
1623
+ if (batch)
1624
+ entry.coalesce = { mode: batch.mode, keyed: batch.key !== undefined };
1625
+ }
1626
+ else {
1627
+ entry.announced = true;
1628
+ }
1629
+ into.set(key, entry);
1630
+ };
1631
+ if (!workflow.startsWith('__')) {
1632
+ for (const kind of ['completed', 'failed']) {
1633
+ const event = lifecycleEventName(workflow, kind);
1634
+ for (const [subscriber, subscriptions] of triggers.get(event) ?? []) {
1635
+ link(startsOnSettle, subscriber, kind, { subscriber, subscriptions });
1636
+ }
1637
+ for (const subscriber of fleet.get(event) ?? []) {
1638
+ link(startsOnSettle, subscriber, kind, undefined);
1639
+ }
1640
+ }
1641
+ }
1642
+ for (const [event, subscribers] of triggers) {
1643
+ const source = parseLifecycleEvent(event);
1644
+ const subscriptions = subscribers.get(workflow);
1645
+ if (source && subscriptions) {
1646
+ link(startedBy, source.workflow, source.kind, { subscriber: workflow, subscriptions });
1647
+ }
1648
+ }
1649
+ for (const [event, subscribers] of fleet) {
1650
+ const source = parseLifecycleEvent(event);
1651
+ if (source && subscribers.has(workflow)) {
1652
+ link(startedBy, source.workflow, source.kind, undefined);
1653
+ }
1654
+ }
1655
+ const sorted = (links) => [...links.values()].sort((a, b) => a.workflow.localeCompare(b.workflow) || a.kind.localeCompare(b.kind));
1656
+ return { workflow, startsOnSettle: sorted(startsOnSettle), startedBy: sorted(startedBy) };
1657
+ }
1658
+ /**
1659
+ * `patch` (a terminal status write of `run`) plus the {@link LIFECYCLE_PENDING_TAG} marker and one
1660
+ * {@link LIFECYCLE_OWED_TAG_PREFIX} tag per subscriber of that ending, and the run's span as a
1661
+ * {@link LIFECYCLE_TRACE_TAG_PREFIX} tag when a run trace provider knows it. Written in the same
1662
+ * update as the status, so a run can never be settled without a record of whom its lifecycle event
1663
+ * is owed to.
1664
+ *
1665
+ * While this instance has no successful read of the fleet's announced subscriptions (none yet, or
1666
+ * the last one failed), it cannot know who else subscribes, so the marker is the pending tag alone:
1667
+ * the run's owed set is unresolved, and the publish that next reads the fleet resolves it (see
1668
+ * {@link publishSettled}). Never waits on that read.
1669
+ */
1670
+ withLifecycleMarker(run, patch) {
1671
+ const kind = lifecycleKindOf(patch.status);
1672
+ if (!kind || run.workflow.startsWith('__'))
1673
+ return patch;
1674
+ const subscribers = this.lifecycleSubscribers(run.workflow, kind, normalizeNamespace(run.namespace));
1675
+ const unresolved = !this.announcedSubscriptionsSnapshot?.ok;
1676
+ if (subscribers.length === 0 && !unresolved)
1677
+ return patch;
1678
+ const tags = patch.tags ?? run.tags ?? [];
1679
+ let traceparent;
1680
+ try {
1681
+ traceparent = this.runTraceparent?.(run.id);
1682
+ }
1683
+ catch {
1684
+ traceparent = undefined;
1685
+ }
1686
+ const recordsTrace = traceparent !== undefined &&
1687
+ subscribers.length > 0 &&
1688
+ !tags.some((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX));
1689
+ const marker = [
1690
+ LIFECYCLE_PENDING_TAG,
1691
+ ...(unresolved ? [] : subscribers.map(lifecycleOwedTag)),
1692
+ ...(recordsTrace ? [`${LIFECYCLE_TRACE_TAG_PREFIX}${traceparent}`] : []),
1693
+ ];
1694
+ if (marker.every((tag) => tags.includes(tag)))
1695
+ return patch;
1696
+ return { ...patch, tags: [...new Set([...tags, ...marker])] };
1697
+ }
1698
+ /** Answers the traceparent of a run's span in this process; see {@link setRunTraceparent}. */
1699
+ runTraceparent;
1700
+ /**
1701
+ * Install the function that answers the W3C `traceparent` of a run's span in this process, or
1702
+ * `undefined` to remove it. `attachDurableOtel` installs one. With it, a run some workflow
1703
+ * subscribes to records its traceparent (a {@link LIFECYCLE_TRACE_TAG_PREFIX} tag) with its first
1704
+ * suspend (see {@link withRunTrace}), or else with its terminal write, and the `run.started` of a
1705
+ * run a lifecycle event started carries its source run's as `sourceTraceparent`, so the
1706
+ * subscriber's trace links to the source's whichever process ran either.
1707
+ */
1708
+ setRunTraceparent(provider) {
1709
+ this.runTraceparent = provider;
1710
+ }
1711
+ /**
1712
+ * `patch` (a suspend write of `run`) plus the run's {@link LIFECYCLE_TRACE_TAG_PREFIX} tag, when a
1713
+ * run trace provider is installed (see {@link setRunTraceparent}), some workflow subscribes to the
1714
+ * run's ending, the provider answers for the run and the run does not carry the tag yet. The first
1715
+ * turn's suspend records the run's root span this way, so a run that settles in a process that
1716
+ * never saw its first turn still links its subscribers' traces; a workflow nobody subscribes to
1717
+ * gets no tag and no extra read or write. The tags are read again before they are written, so a
1718
+ * tag the turn added is kept.
1719
+ */
1720
+ async withRunTrace(run, patch) {
1721
+ const provider = this.runTraceparent;
1722
+ if (!provider || run.workflow.startsWith('__'))
1723
+ return patch;
1724
+ if ((run.tags ?? []).some((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX)))
1725
+ return patch;
1726
+ const namespace = normalizeNamespace(run.namespace);
1727
+ if (this.lifecycleSubscribers(run.workflow, 'completed', namespace).length === 0 &&
1728
+ this.lifecycleSubscribers(run.workflow, 'failed', namespace).length === 0) {
1729
+ return patch;
1730
+ }
1731
+ let traceparent;
1732
+ try {
1733
+ traceparent = provider(run.id);
1734
+ }
1735
+ catch {
1736
+ return patch;
1737
+ }
1738
+ if (!traceparent)
1739
+ return patch;
1740
+ const tags = (await this.store.getRun(run.id))?.tags ?? run.tags ?? [];
1741
+ if (tags.some((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX)))
1742
+ return patch;
1743
+ return { ...patch, tags: [...tags, `${LIFECYCLE_TRACE_TAG_PREFIX}${traceparent}`] };
1744
+ }
1745
+ /**
1746
+ * `{ sourceTraceparent }` for the `run.started` of `run`, when a lifecycle event started it and a
1747
+ * run trace provider is installed: the source run's recorded traceparent, else this process's.
1748
+ */
1749
+ async sourceTraceOf(run) {
1750
+ const provider = this.runTraceparent;
1751
+ if (!provider)
1752
+ return {};
1753
+ const source = lifecycleSourceOf(run);
1754
+ if (!source)
1755
+ return {};
1756
+ try {
1757
+ const recorded = lifecycleTraceparentOf((await this.store.getRun(source.runId))?.tags);
1758
+ const traceparent = recorded ?? provider(source.runId);
1759
+ return traceparent ? { sourceTraceparent: traceparent } : {};
1760
+ }
1761
+ catch {
1762
+ return {};
1763
+ }
1764
+ }
1765
+ /** How many registry rows one page of a read of the lifecycle subscription registry lists. */
1766
+ static LIFECYCLE_REGISTRY_PAGE_SIZE = 1000;
1767
+ /** How long a read of the subscriptions the fleet announces is reused. */
1768
+ static ANNOUNCED_SUBSCRIPTIONS_TTL_MS = 5_000;
1769
+ /** The last read of the fleet's announced subscriptions; see {@link announcedSubscribers}. */
1770
+ announcedSubscriptionsSnapshot;
1771
+ announcedSubscriptionsRead;
1772
+ /**
1773
+ * The workflows the fleet announces as subscribers of `event`: the live rows of the lifecycle
1774
+ * subscription registry (see {@link LIFECYCLE_REGISTRY_WORKFLOW}) and the live worker descriptors'
1775
+ * `registrations[].subscribes`, from a snapshot refreshed in the background once it is older than
1776
+ * {@link ANNOUNCED_SUBSCRIPTIONS_TTL_MS}. Synchronous, so a terminal status write never waits on
1777
+ * the fleet; before the first read completes it answers with nothing.
1778
+ */
1779
+ announcedSubscribers(event, namespace) {
1780
+ const snapshot = this.announcedSubscriptionsSnapshot;
1781
+ if (!snapshot || this.clock() - snapshot.at > WorkflowEngine.ANNOUNCED_SUBSCRIPTIONS_TTL_MS) {
1782
+ void this.refreshAnnouncedSubscriptions();
1783
+ }
1784
+ return this.announcedFor(event, namespace);
1785
+ }
1786
+ /**
1787
+ * The subscribers of `event` in the current snapshot that serve `namespace`'s runs: announced for
1788
+ * that namespace, or by an engine with no namespace (see {@link LifecycleRegistryEntry}). Every
1789
+ * announced subscriber when `namespace` is undefined. `node` keeps only the ones Node engines
1790
+ * announce (registry rows and Node worker descriptors), which start their own subscribers.
1791
+ */
1792
+ announcedFor(event, namespace, opts) {
1793
+ const snapshot = this.announcedSubscriptionsSnapshot;
1794
+ if (!snapshot?.byEvent.has(event))
1795
+ return new Set();
1796
+ const scope = namespace === undefined ? undefined : normalizeNamespace(namespace);
1797
+ const out = new Set();
1798
+ for (const entry of snapshot.entries) {
1799
+ if (entry.event !== event || (opts?.node && !entry.node))
1800
+ continue;
1801
+ if (scope !== undefined && entry.namespace !== null && entry.namespace !== scope)
1802
+ continue;
1803
+ out.add(entry.workflow);
1804
+ }
1805
+ return out;
1806
+ }
1807
+ /**
1808
+ * Read the fleet's announced subscriptions into the snapshot: the registry rows whose heartbeat
1809
+ * is at most {@link LIFECYCLE_REGISTRY_TTL_MS} old (a range of the store's `created_at` index, read
1810
+ * page by page until a short page, so a large fleet is never cut off), and
1811
+ * the worker descriptors on a transport that can read them fleet-wide. Never throws: a failed read
1812
+ * of either leaves an empty snapshot marked not `ok`, which counts as the local registry only and
1813
+ * never orphans a subscriber. Concurrent callers share one read.
1814
+ */
1815
+ refreshAnnouncedSubscriptions() {
1816
+ this.announcedSubscriptionsRead ??= (async () => {
1817
+ const at = this.clock();
1818
+ const byEvent = new Map();
1819
+ const entries = [];
1820
+ const add = (event, workflow, node, namespace) => {
1821
+ entries.push({ event, workflow, namespace, node });
1822
+ const subscribers = byEvent.get(event) ?? new Set();
1823
+ subscribers.add(workflow);
1824
+ byEvent.set(event, subscribers);
1825
+ };
1826
+ const readRegistry = async () => {
1827
+ const size = WorkflowEngine.LIFECYCLE_REGISTRY_PAGE_SIZE;
1828
+ const out = [];
1829
+ for (let page = 1;; page += 1) {
1830
+ const rows = await asHeartbeat(() => this.store.listRuns({
1831
+ workflow: LIFECYCLE_REGISTRY_WORKFLOW,
1832
+ createdAfter: at - LIFECYCLE_REGISTRY_TTL_MS,
1833
+ // A namespaced engine also reads `default`, where an engine with no namespace writes
1834
+ // its row: those subscribers are owed this namespace's runs too.
1835
+ ...(this.namespace !== undefined
1836
+ ? { namespaces: [...new Set([this.namespace, 'default'])] }
1837
+ : {}),
1838
+ page,
1839
+ size,
1840
+ }));
1841
+ out.push(...rows);
1842
+ if (rows.length < size)
1843
+ return out;
1844
+ }
1845
+ };
1846
+ try {
1847
+ const [descriptors, rows] = await Promise.all([
1848
+ this.pool.introspectsFleet ? this.pool.readAllWorkerDescriptors() : [],
1849
+ readRegistry(),
1850
+ ]);
1851
+ for (const row of rows) {
1852
+ if (row.workflow !== LIFECYCLE_REGISTRY_WORKFLOW)
1853
+ continue;
1854
+ if (at - row.updatedAt.getTime() > LIFECYCLE_REGISTRY_TTL_MS)
1855
+ continue;
1856
+ const namespace = lifecycleRegistryNamespace(row.input, row.namespace);
1857
+ for (const { workflow, events } of lifecycleRegistrySubscriptions(row.input)) {
1858
+ for (const event of events)
1859
+ add(event, workflow, true, namespace);
1860
+ }
1861
+ }
1862
+ for (const descriptor of descriptors) {
1863
+ for (const announced of announcementsOf(descriptor)) {
1864
+ const namespace = servedNamespace(descriptor.namespace, announced.group);
1865
+ for (const event of announced.subscribes ?? []) {
1866
+ add(event, announced.name, descriptor.runtime === 'node', namespace);
1867
+ }
1868
+ }
1869
+ }
1870
+ this.announcedSubscriptionsSnapshot = { at, ok: true, byEvent, entries };
1871
+ }
1872
+ catch {
1873
+ this.announcedSubscriptionsSnapshot = { at, ok: false, byEvent: new Map(), entries: [] };
1874
+ }
1875
+ })().finally(() => {
1876
+ this.announcedSubscriptionsRead = undefined;
1877
+ });
1878
+ return this.announcedSubscriptionsRead;
1879
+ }
1880
+ /** Read the fleet's announced subscriptions again when the snapshot is older than the TTL. */
1881
+ async freshAnnouncedSubscriptions() {
1882
+ const snapshot = this.announcedSubscriptionsSnapshot;
1883
+ if (!snapshot || this.clock() - snapshot.at > WorkflowEngine.ANNOUNCED_SUBSCRIPTIONS_TTL_MS) {
1884
+ await this.refreshAnnouncedSubscriptions();
1885
+ }
1886
+ }
1887
+ /** When this engine last wrote its registry row; see {@link heartbeatLifecycleRegistry}. */
1888
+ lifecycleRegistryBeatAt;
1889
+ /** When this engine last deleted stale registry rows; undefined before its first pass. */
1890
+ lifecycleRegistryCleanedAt;
1891
+ /** How often an engine with a registry row looks for stale rows to delete. */
1892
+ static LIFECYCLE_REGISTRY_CLEANUP_MS = 60_000;
1893
+ /**
1894
+ * This engine's lifecycle registry entry: each workflow registered here that subscribes to an
1895
+ * event, with the events it subscribes to. Undefined when none does.
1896
+ */
1897
+ lifecycleRegistryEntry() {
1898
+ const subscribes = new Map();
1899
+ for (const [event, subscribers] of this.eventTriggers) {
1900
+ for (const workflow of subscribers.keys()) {
1901
+ if (workflow.startsWith('__'))
1902
+ continue;
1903
+ const events = subscribes.get(workflow) ?? new Set();
1904
+ events.add(event);
1905
+ subscribes.set(workflow, events);
1906
+ }
1907
+ }
1908
+ if (subscribes.size === 0)
1909
+ return undefined;
1910
+ return {
1911
+ instanceId: this.instanceId,
1912
+ runtime: 'node',
1913
+ engineNamespace: this.namespace ?? null,
1914
+ subscriptions: [...subscribes.keys()].sort().map((workflow) => ({
1915
+ workflow,
1916
+ events: [...(subscribes.get(workflow) ?? [])].sort(),
1917
+ })),
1918
+ };
1919
+ }
1920
+ /**
1921
+ * Write this engine's row of the lifecycle subscription registry (see
1922
+ * {@link LIFECYCLE_REGISTRY_WORKFLOW}), at most once per {@link LIFECYCLE_REGISTRY_HEARTBEAT_MS}:
1923
+ * an update of the existing row, or a create when it is missing (first beat, or deleted as stale
1924
+ * after a long pause). Once a minute it also deletes rows whose heartbeat stopped
1925
+ * {@link LIFECYCLE_REGISTRY_STALE_MS} ago: the ones that went stale in the last
1926
+ * {@link LIFECYCLE_REGISTRY_STALE_MS}, a bounded range of the `created_at` index, and on its first
1927
+ * pass every stale row, for the ones left while no engine ran. The row is created tagged
1928
+ * {@link LIFECYCLE_REGISTRY_TAG}, which retention keeps. A no-op for an engine without
1929
+ * subscriptions. Its store calls are marked as heartbeats for observability tools. Never throws.
1930
+ */
1931
+ async heartbeatLifecycleRegistry() {
1932
+ const nowMs = this.clock();
1933
+ if (this.lifecycleRegistryBeatAt !== undefined &&
1934
+ nowMs - this.lifecycleRegistryBeatAt < LIFECYCLE_REGISTRY_HEARTBEAT_MS) {
1935
+ return;
1936
+ }
1937
+ const entry = this.lifecycleRegistryEntry();
1938
+ if (!entry)
1939
+ return;
1940
+ this.lifecycleRegistryBeatAt = nowMs;
1941
+ const id = lifecycleRegistryRunId(this.instanceId);
1942
+ const at = new Date(nowMs);
1943
+ try {
1944
+ await asHeartbeat(async () => {
1945
+ const beat = { input: entry, createdAt: at, updatedAt: at };
1946
+ if (await this.store.updateRunIf(id, ['completed'], beat))
1947
+ return;
1948
+ try {
1949
+ await this.store.createRun({
1950
+ id,
1951
+ workflow: LIFECYCLE_REGISTRY_WORKFLOW,
1952
+ workflowVersion: '1',
1953
+ status: 'completed',
1954
+ input: entry,
1955
+ tags: [LIFECYCLE_REGISTRY_TAG],
1956
+ ...(this.namespace !== undefined ? { namespace: this.namespace } : {}),
1957
+ createdAt: at,
1958
+ updatedAt: at,
1959
+ });
1960
+ }
1961
+ catch {
1962
+ // Created by a concurrent beat under the same instance id.
1963
+ await this.store.updateRun(id, beat);
1964
+ }
1965
+ });
1966
+ }
1967
+ catch (error) {
1968
+ console.error('[adonis-durable] could not write the lifecycle subscription registry:', error);
1969
+ }
1970
+ const cleanedAt = this.lifecycleRegistryCleanedAt;
1971
+ if (cleanedAt !== undefined &&
1972
+ nowMs - cleanedAt < WorkflowEngine.LIFECYCLE_REGISTRY_CLEANUP_MS) {
1973
+ return;
1974
+ }
1975
+ this.lifecycleRegistryCleanedAt = nowMs;
1976
+ const staleBefore = nowMs - LIFECYCLE_REGISTRY_STALE_MS;
1977
+ try {
1978
+ await asHeartbeat(async () => {
1979
+ const stale = await this.store.listRuns({
1980
+ workflow: LIFECYCLE_REGISTRY_WORKFLOW,
1981
+ ...(this.namespace !== undefined ? { namespace: this.namespace } : {}),
1982
+ createdBefore: staleBefore,
1983
+ ...(cleanedAt !== undefined
1984
+ ? { createdAfter: staleBefore - LIFECYCLE_REGISTRY_STALE_MS }
1985
+ : {}),
1986
+ size: 100,
1987
+ });
1988
+ for (const row of stale) {
1989
+ if (row.workflow === LIFECYCLE_REGISTRY_WORKFLOW && row.id !== id) {
1990
+ await this.store.deleteRun(row.id);
1991
+ }
1992
+ }
1993
+ });
1994
+ }
1995
+ catch (error) {
1996
+ console.error('[adonis-durable] could not prune the lifecycle subscription registry:', error);
1997
+ }
1998
+ }
1999
+ /**
2000
+ * The runs started by `runId` settling (see {@link publishLifecycle}), read from the run's own
2001
+ * tags, so it needs no subscriber registry. Empty for a run that has not settled or started
2002
+ * nobody. Pass the run itself when it is already loaded, to skip reading it again.
2003
+ */
2004
+ async triggeredRuns(source) {
2005
+ const run = typeof source === 'string' ? await this.store.getRun(source) : source;
2006
+ return run ? triggeredRunsOf(run) : [];
2007
+ }
2008
+ /** The live path of {@link publishSettled}: right after this instance settled the run. */
2009
+ publishLifecycle(event) {
2010
+ if (event.type !== 'run.completed' && event.type !== 'run.failed')
2011
+ return;
2012
+ const kind = event.type === 'run.completed' ? 'completed' : 'failed';
2013
+ if (!event.workflow || event.workflow.startsWith('__'))
2014
+ return;
2015
+ const name = lifecycleEventName(event.workflow, kind);
2016
+ const snapshot = this.announcedSubscriptionsSnapshot;
2017
+ // Without a successful read of the fleet the terminal write left the owed set unresolved, and
2018
+ // this publish resolves it.
2019
+ if (snapshot?.ok &&
2020
+ this.localSubscribers(name).length === 0 &&
2021
+ !snapshot.byEvent.get(name)?.size) {
2022
+ return;
2023
+ }
2024
+ // The turn that settled the run still holds its lease: publish once it let go.
2025
+ const turns = [...this.inflight].filter(([, runId]) => runId === event.runId).map(([p]) => p);
2026
+ void (async () => {
2027
+ await Promise.allSettled(turns);
2028
+ const run = await this.store.getRun(event.runId);
2029
+ if (run)
2030
+ await this.publishSettled(run);
2031
+ })().catch(() => undefined);
2032
+ }
2033
+ /**
2034
+ * Take the run lease for this instance's turn, as `tryLockRun` at `nowMs`; when the lease is held
2035
+ * by a lifecycle publish (see {@link withLifecycleLease}), which only ever holds a settled run for
2036
+ * the length of its publish (an explicit retry can land in it), wait for that publish to let go,
2037
+ * at most `leaseMs`.
2038
+ */
2039
+ async lockRunPastLifecyclePublish(runId, nowMs) {
2040
+ if (await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
2041
+ return true;
2042
+ }
2043
+ // Real time, not `this.clock`: a test clock may stand still.
2044
+ const giveUpAt = Date.now() + this.leaseMs;
2045
+ let delayMs = 10;
2046
+ for (;;) {
2047
+ const held = await this.store.getRun(runId);
2048
+ if (!held?.lockedBy?.includes(LIFECYCLE_LEASE_MARK) || Date.now() >= giveUpAt)
2049
+ return false;
2050
+ await new Promise((r) => setTimeout(r, delayMs));
2051
+ delayMs = Math.min(delayMs * 2, 250);
2052
+ const now = this.clock();
2053
+ if (await this.store.tryLockRun(runId, this.instanceId, now + this.leaseMs, now))
2054
+ return true;
2055
+ }
2056
+ }
2057
+ /** The waits between attempts to take a settled run's lifecycle lease (see {@link withLifecycleLease}). */
2058
+ static LIFECYCLE_LEASE_RETRY_MS = [50, 100, 200, 400, 800];
2059
+ /**
2060
+ * Run `fn` holding the run lease of the settled run `runId` under an owner unique to this call, so
2061
+ * every read-start-write of a run's lifecycle tags (live publish, sweeps, heal, republish, on any
2062
+ * instance or twice in this one) runs one at a time. A refused lease is asked for again after each
2063
+ * of {@link LIFECYCLE_LEASE_RETRY_MS}; still refused, resolves `{ acquired: false }` without
2064
+ * running `fn`. The lease is released when `fn` settles, and expires after `leaseMs` if this
2065
+ * process dies first.
2066
+ */
2067
+ async withLifecycleLease(runId, fn) {
2068
+ const owner = `${this.instanceId}${LIFECYCLE_LEASE_MARK}${globalThis.crypto.randomUUID()}`;
2069
+ const delays = WorkflowEngine.LIFECYCLE_LEASE_RETRY_MS;
2070
+ for (let attempt = 0;; attempt += 1) {
2071
+ const nowMs = this.clock();
2072
+ if (await this.store.tryLockRun(runId, owner, nowMs + this.leaseMs, nowMs))
2073
+ break;
2074
+ const delay = delays[attempt];
2075
+ if (delay === undefined)
2076
+ return { acquired: false };
2077
+ await new Promise((resolve) => setTimeout(resolve, delay));
2078
+ }
2079
+ try {
2080
+ return { acquired: true, value: await fn() };
2081
+ }
2082
+ finally {
2083
+ await this.store.releaseRunLock(runId, owner).catch(() => undefined);
2084
+ }
2085
+ }
2086
+ /**
2087
+ * Publish a settled run's lifecycle event (`run.completed:<workflow>` / `run.failed:<workflow>`)
2088
+ * to the subscribers it owes it to that this instance can deliver to (registered here, or announced
2089
+ * and started by name; see {@link deliverableSubscribers}), and settle their part of the marker:
2090
+ * drop each one's {@link LIFECYCLE_OWED_TAG_PREFIX} tag, record each started one as a
2091
+ * `durable:triggered:<workflow>` tag (a coalescing one as `durable:coalesced:<workflow>`), and drop
2092
+ * {@link LIFECYCLE_PENDING_TAG} once no owed tag is left.
2093
+ *
2094
+ * A run marked pending with no owed tag has an unresolved owed set (see
2095
+ * {@link LIFECYCLE_PENDING_TAG}). With a successful read of the fleet's subscriptions it is resolved
2096
+ * here: every subscriber of the ending (see {@link lifecycleSubscribers}) the run's tags do not
2097
+ * record as started or failed becomes owed, and the ones nobody here delivers to keep an owed tag.
2098
+ * Without one, the subscribers registered here are started and the run stays unresolved.
2099
+ *
2100
+ * `republish` publishes to every subscriber of the ending (or only `subscriber`, which must be one),
2101
+ * owed or not: the ones this instance can deliver to are started, and the others are marked owed
2102
+ * again (with the pending tag) for the sweep of an instance that can. `heal` names subscribers of
2103
+ * the ending the run was never marked for (see {@link healLifecycleEvents}): they are owed now, and
2104
+ * the ones this instance delivers to are started.
2105
+ * `nowMs` (the sweep's clock) also drops the owed tag of a subscriber that is neither registered
2106
+ * here nor announced by a live worker once the run settled {@link LIFECYCLE_ORPHAN_MS} ago,
2107
+ * reporting it as `run.trigger_failed`.
2108
+ *
2109
+ * Runs under the run's lifecycle lease (see {@link withLifecycleLease}) and reads the run again
2110
+ * under it, so concurrent publishers never overwrite each other's tags; `busy` when the lease
2111
+ * stayed taken. Idempotent, since every subscriber run id is deterministic and a debounce/batch
2112
+ * accumulator drops an event it already holds. A subscriber whose `where` throws, whose input is
2113
+ * rejected or whose start fails, and an orphan the sweep drops, gets a
2114
+ * {@link LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX} tag in the same tag write, and is not retried until a
2115
+ * republish; the write that records a subscriber reached removes its trigger-failed tag.
2116
+ * A debounce/batch subscriber routed into its accumulator gets a
2117
+ * {@link LIFECYCLE_COALESCED_TAG_PREFIX} tag in the same write instead of a triggered one; a
2118
+ * republish routes it again only when the run also carries its trigger-failed tag.
2119
+ * `run.triggered` and `run.trigger_failed` are emitted only for the subscribers whose triggered,
2120
+ * coalesced or trigger-failed tag this write added. A failure to write the marker leaves it, and
2121
+ * the sweep publishes the run again.
2122
+ */
2123
+ async publishSettled(source, opts) {
2124
+ const leased = await this.withLifecycleLease(source.id, async () => {
2125
+ const run = await this.store.getRun(source.id);
2126
+ return run
2127
+ ? this.publishSettledLeased(run, opts)
2128
+ : { started: 0, failed: [] };
2129
+ });
2130
+ return leased.acquired ? leased.value : { started: 0, failed: [], busy: true };
2131
+ }
2132
+ /** {@link publishSettled} once it holds the run's lifecycle lease and has read `run` under it. */
2133
+ async publishSettledLeased(run, opts) {
2134
+ const kind = lifecycleKindOf(run.status);
2135
+ if (!kind)
2136
+ return { started: 0, failed: [] };
2137
+ const name = lifecycleEventName(run.workflow, kind);
2138
+ const namespace = normalizeNamespace(run.namespace);
2139
+ const before = run.tags ?? [];
2140
+ const delivered = deliveredSubscribersOf(run);
2141
+ let owed = owedSubscribersOf(run);
2142
+ const pending = before.includes(LIFECYCLE_PENDING_TAG);
2143
+ // Subscribers this write marks owed: a resolved or healed owed set, or a republish's re-mark.
2144
+ let newlyOwed = [];
2145
+ let unresolved = false;
2146
+ if (pending && owed.length === 0 && !opts?.republish) {
2147
+ await this.freshAnnouncedSubscriptions();
2148
+ if (this.announcedSubscriptionsSnapshot?.ok) {
2149
+ owed = this.lifecycleSubscribers(run.workflow, kind, namespace).filter((s) => !delivered.has(s));
2150
+ newlyOwed = owed;
2151
+ }
2152
+ else {
2153
+ unresolved = true;
2154
+ }
2155
+ }
2156
+ if (opts?.heal?.length) {
2157
+ const healed = opts.heal.filter((s) => !owed.includes(s) && !delivered.has(s));
2158
+ owed = [...owed, ...healed];
2159
+ newlyOwed = [...newlyOwed, ...healed];
2160
+ }
2161
+ const { local, remote, all: here } = await this.deliverableSubscribers(name, namespace);
2162
+ let targets;
2163
+ const refused = [];
2164
+ if (opts?.republish) {
2165
+ await this.freshAnnouncedSubscriptions();
2166
+ // Without a read of the fleet, an unresolved run stays unresolved for the next publish.
2167
+ unresolved = pending && owed.length === 0 && !this.announcedSubscriptionsSnapshot?.ok;
2168
+ const subscribers = this.lifecycleSubscribers(run.workflow, kind, namespace);
2169
+ let all = subscribers;
2170
+ if (opts.subscriber !== undefined) {
2171
+ all = subscribers.includes(opts.subscriber) ? [opts.subscriber] : [];
2172
+ if (all.length === 0) {
2173
+ refused.push({
2174
+ workflow: opts.subscriber,
2175
+ error: new Error(`subscriber ${opts.subscriber} does not subscribe to ${name}`),
2176
+ });
2177
+ }
2178
+ }
2179
+ // An accumulator that already fired would start its target again, so a subscriber the run's
2180
+ // tags record as coalesced is routed again (or owed again) only after a failed delivery.
2181
+ const settledCoalesced = new Set(coalescedSubscribersOf(before).filter((s) => !before.includes(lifecycleTriggerFailedTag(s))));
2182
+ all = all.filter((s) => !settledCoalesced.has(s));
2183
+ targets = all.filter((s) => here.has(s));
2184
+ newlyOwed = all.filter((s) => !here.has(s));
2185
+ }
2186
+ else if (unresolved) {
2187
+ targets = [...local].filter((s) => !delivered.has(s));
2188
+ }
2189
+ else {
2190
+ targets = owed.filter((s) => here.has(s));
2191
+ }
2192
+ const { reached, failed } = targets.length > 0
2193
+ ? await this.startSubscribers(name, lifecycleEventOf(run), (sub) => lifecycleRunId(run.id, kind, sub), new Set(targets), remote)
2194
+ : { reached: [], failed: [] };
2195
+ const orphaned = opts?.nowMs !== undefined ? await this.orphanedSubscribers(run, owed, here, opts.nowMs) : [];
2196
+ const done = new Set([...targets, ...orphaned]);
2197
+ // A started subscriber is recorded by its triggered tag, one routed into its accumulator by its
2198
+ // coalesced tag.
2199
+ const deliveryTag = (r) => r.runId !== undefined
2200
+ ? `${LIFECYCLE_TRIGGERED_TAG_PREFIX}${r.workflow}`
2201
+ : lifecycleCoalescedTag(r.workflow);
2202
+ const deliveryTags = reached.map(deliveryTag);
2203
+ const reachedNow = new Set(reached.map((r) => r.workflow));
2204
+ const failedNow = [...failed.map((f) => f.workflow), ...orphaned];
2205
+ const kept = [
2206
+ ...before.filter((t) => !((t.startsWith(LIFECYCLE_OWED_TAG_PREFIX) &&
2207
+ done.has(t.slice(LIFECYCLE_OWED_TAG_PREFIX.length))) ||
2208
+ (t.startsWith(LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX) &&
2209
+ reachedNow.has(t.slice(LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX.length))))),
2210
+ ...newlyOwed.filter((s) => !done.has(s)).map(lifecycleOwedTag),
2211
+ ];
2212
+ const owedLeft = kept.some((t) => t.startsWith(LIFECYCLE_OWED_TAG_PREFIX));
2213
+ const tags = [
2214
+ ...new Set([
2215
+ ...kept.filter((t) => t !== LIFECYCLE_PENDING_TAG),
2216
+ ...(owedLeft || unresolved ? [LIFECYCLE_PENDING_TAG] : []),
2217
+ ...deliveryTags,
2218
+ ...failedNow.map(lifecycleTriggerFailedTag),
2219
+ ]),
2220
+ ];
2221
+ if (tags.length !== before.length || tags.some((t) => !before.includes(t))) {
2222
+ await this.store.updateRun(run.id, { tags });
2223
+ }
2224
+ // Only the tag changes this write made are announced, so a publisher that repeats another's
2225
+ // delivery emits nothing.
2226
+ for (const r of reached) {
2227
+ if (before.includes(deliveryTag(r)))
2228
+ continue;
2229
+ const { workflow, runId } = r;
2230
+ this.emit({
2231
+ type: 'run.triggered',
2232
+ runId: run.id,
2233
+ workflow: run.workflow,
2234
+ namespace: run.namespace,
2235
+ name: workflow,
2236
+ triggeredRunId: runId,
2237
+ });
2238
+ }
2239
+ const unreached = [
2240
+ ...failed,
2241
+ ...orphaned.map((workflow) => ({
2242
+ workflow,
2243
+ error: new Error(`subscriber ${workflow} is not registered on any live worker`),
2244
+ })),
2245
+ ];
2246
+ for (const { workflow, error } of unreached) {
2247
+ if (before.includes(lifecycleTriggerFailedTag(workflow)))
2248
+ continue;
2249
+ this.emit({
2250
+ type: 'run.trigger_failed',
2251
+ runId: run.id,
2252
+ workflow: run.workflow,
2253
+ namespace: run.namespace,
2254
+ name: workflow,
2255
+ error: { message: error instanceof Error ? error.message : String(error) },
2256
+ });
2257
+ }
2258
+ return {
2259
+ started: reached.filter((r) => r.created).length,
2260
+ failed: [...refused, ...unreached],
2261
+ };
2262
+ }
2263
+ /**
2264
+ * The subscribers `run` owes its event to that nothing can deliver it to: not in `here` (what
2265
+ * this instance delivers to), announced by no live worker, and owed since
2266
+ * {@link LIFECYCLE_ORPHAN_MS} or longer. Empty while the fleet's announcements cannot be read,
2267
+ * since then nobody is known to be gone.
2268
+ */
2269
+ async orphanedSubscribers(run, owed, here, nowMs) {
2270
+ if (nowMs - run.updatedAt.getTime() < LIFECYCLE_ORPHAN_MS)
2271
+ return [];
2272
+ const candidates = owed.filter((s) => !here.has(s));
2273
+ if (candidates.length === 0)
2274
+ return [];
2275
+ await this.freshAnnouncedSubscriptions();
2276
+ const fresh = this.announcedSubscriptionsSnapshot;
2277
+ if (!fresh?.ok)
2278
+ return [];
2279
+ const kind = lifecycleKindOf(run.status);
2280
+ if (!kind)
2281
+ return [];
2282
+ const announced = this.announcedFor(lifecycleEventName(run.workflow, kind), normalizeNamespace(run.namespace));
2283
+ return candidates.filter((s) => !announced.has(s));
2284
+ }
2285
+ /** How long a settled run's marker is left to the live publish before the sweep takes it over. */
2286
+ static LIFECYCLE_SWEEP_GRACE_MS = 30_000;
2287
+ /** How many marked runs one lifecycle sweep reads at most, looking for ones it can act on. */
2288
+ static LIFECYCLE_SWEEP_SCAN_CAP = 1000;
2289
+ /**
2290
+ * Publish the lifecycle event of settled runs still marked {@link LIFECYCLE_PENDING_TAG} to the
2291
+ * subscribers this instance can deliver to that they still owe it to: a run whose settling process
2292
+ * died, or whose marker write failed, before publishing, or whose subscriber is registered only on
2293
+ * other instances. Also writes this instance's row of the lifecycle subscription registry (see
2294
+ * {@link heartbeatLifecycleRegistry}), refreshes the snapshot of the subscriptions the fleet
2295
+ * announces (registry rows and worker descriptors), drops
2296
+ * owed subscribers nothing can deliver to (see {@link orphanedSubscribers}) and wakes the
2297
+ * `ctx.waitForRun` waiters whose run already settled (see {@link sweepRunWaits}).
2298
+ *
2299
+ * Called by the worker poll loop. Reads the marked runs page by page and skips the ones it cannot
2300
+ * act on (every owed subscriber is delivered elsewhere and none is orphaned yet), so runs owed only
2301
+ * to other instances never hide the ones it can publish. Stops once it has published `limit` runs
2302
+ * or read {@link LIFECYCLE_SWEEP_SCAN_CAP} marked runs. A run settled in the last
2303
+ * {@link LIFECYCLE_SWEEP_GRACE_MS} is left to the live publish; a run another publisher holds
2304
+ * (see {@link withLifecycleLease}) is left to the next tick. Then heals the runs settled in the
2305
+ * last {@link LIFECYCLE_HEAL_WINDOW_MS} (see {@link healLifecycleEvents}). Returns how many marked
2306
+ * runs it published plus how many waiters it woke; healed runs are not counted.
2307
+ */
2308
+ async sweepLifecycleEvents(nowMs = this.clock(), limit = 100) {
2309
+ await this.heartbeatLifecycleRegistry();
2310
+ // A failed read is tried again on every tick, so unresolved runs resolve once the fleet is back.
2311
+ if (this.announcedSubscriptionsSnapshot?.ok === false)
2312
+ await this.refreshAnnouncedSubscriptions();
2313
+ else
2314
+ await this.freshAnnouncedSubscriptions();
2315
+ const actionable = await this.lifecycleActionableRuns(nowMs, limit);
2316
+ let published = 0;
2317
+ for (const run of actionable) {
2318
+ try {
2319
+ const { busy } = await this.publishSettled(run, { nowMs });
2320
+ if (!busy)
2321
+ published += 1;
2322
+ }
2323
+ catch (error) {
2324
+ console.error(`[adonis-durable] lifecycle sweep: run ${run.id} failed; retrying next tick:`, error);
2325
+ }
2326
+ }
2327
+ try {
2328
+ await this.healLifecycleEvents(nowMs);
2329
+ }
2330
+ catch (error) {
2331
+ console.error('[adonis-durable] lifecycle sweep: heal failed; retrying next tick:', error);
2332
+ }
2333
+ return published + (await this.sweepRunWaits(limit));
2334
+ }
2335
+ /** How many settled runs of one source workflow a heal pass reads at most. */
2336
+ static LIFECYCLE_HEAL_SCAN_CAP = 100;
2337
+ /**
2338
+ * The (run, subscriber) pairs this engine already healed, with when each entry lapses (epoch ms):
2339
+ * a subscriber whose `where` declines the event leaves no tag, so without it every sweep would
2340
+ * offer it the event again for the whole heal window.
2341
+ */
2342
+ lifecycleHealed = new Map();
2343
+ /**
2344
+ * Deliver lifecycle events the terminal write never marked: for every source workflow some
2345
+ * subscriber in the current snapshot of the fleet's subscriptions (or registered here) listens to,
2346
+ * read its runs settled in the last {@link LIFECYCLE_HEAL_WINDOW_MS} (at most
2347
+ * {@link LIFECYCLE_HEAL_SCAN_CAP}, one listing per workflow) and publish each one to the
2348
+ * subscribers of its ending its tags record as neither owed, started, coalesced nor failed (see
2349
+ * {@link publishSettled}'s `heal`). That covers a subscriber announced just after the run
2350
+ * settled, while the settling instance's snapshot was stale. Each pair is healed at most once per
2351
+ * engine. A run whose owed set is unresolved is left to {@link sweepLifecycleEvents}. Needs a
2352
+ * successful read of the fleet's subscriptions; returns how many runs it published.
2353
+ */
2354
+ async healLifecycleEvents(nowMs) {
2355
+ const snapshot = this.announcedSubscriptionsSnapshot;
2356
+ if (!snapshot?.ok)
2357
+ return 0;
2358
+ for (const [key, lapsesAt] of this.lifecycleHealed) {
2359
+ if (lapsesAt <= nowMs)
2360
+ this.lifecycleHealed.delete(key);
2361
+ }
2362
+ const sources = new Map();
2363
+ for (const event of new Set([...snapshot.byEvent.keys(), ...this.eventTriggers.keys()])) {
2364
+ const source = parseLifecycleEvent(event);
2365
+ if (!source || source.workflow.startsWith('__'))
2366
+ continue;
2367
+ const kinds = sources.get(source.workflow) ?? new Set();
2368
+ kinds.add(source.kind);
2369
+ sources.set(source.workflow, kinds);
2370
+ }
2371
+ let healed = 0;
2372
+ for (const [workflow, kinds] of sources) {
2373
+ const statuses = [
2374
+ ...(kinds.has('completed') ? ['completed'] : []),
2375
+ ...(kinds.has('failed') ? ['failed', 'cancelled', 'dead'] : []),
2376
+ ];
2377
+ const runs = await asHeartbeat(() => this.store.listRuns({
2378
+ workflow,
2379
+ statuses,
2380
+ ...(this.namespace !== undefined ? { namespace: this.namespace } : {}),
2381
+ updatedAfter: nowMs - LIFECYCLE_HEAL_WINDOW_MS,
2382
+ size: WorkflowEngine.LIFECYCLE_HEAL_SCAN_CAP,
2383
+ }));
2384
+ for (const run of runs) {
2385
+ const kind = lifecycleKindOf(run.status);
2386
+ if (run.workflow !== workflow || !kind || !kinds.has(kind))
2387
+ continue;
2388
+ if (nowMs - run.updatedAt.getTime() > LIFECYCLE_HEAL_WINDOW_MS)
2389
+ continue;
2390
+ const owed = owedSubscribersOf(run);
2391
+ if (owed.length === 0 && run.tags?.includes(LIFECYCLE_PENDING_TAG))
2392
+ continue;
2393
+ const delivered = deliveredSubscribersOf(run);
2394
+ const missing = this.lifecycleSubscribers(workflow, kind, normalizeNamespace(run.namespace)).filter((s) => !owed.includes(s) &&
2395
+ !delivered.has(s) &&
2396
+ !this.lifecycleHealed.has(`${run.id}\u0000${s}`));
2397
+ if (missing.length === 0)
2398
+ continue;
2399
+ const { busy } = await this.publishSettled(run, { heal: missing, nowMs });
2400
+ if (busy)
2401
+ continue;
2402
+ for (const s of missing) {
2403
+ this.lifecycleHealed.set(`${run.id}\u0000${s}`, nowMs + LIFECYCLE_HEAL_WINDOW_MS);
2404
+ }
2405
+ healed += 1;
2406
+ }
2407
+ }
2408
+ return healed;
2409
+ }
2410
+ /**
2411
+ * Up to `limit` settled runs marked {@link LIFECYCLE_PENDING_TAG}, past the sweep grace, that this
2412
+ * instance can act on: a run owing its event to a subscriber it delivers to, a run marked without
2413
+ * owed tags, or a run owing it to a subscriber nothing delivers to past
2414
+ * {@link LIFECYCLE_ORPHAN_MS}. Reads at most {@link LIFECYCLE_SWEEP_SCAN_CAP} marked runs, in
2415
+ * pages, all read before any is acted on, since publishing a run takes it out of the listing.
2416
+ */
2417
+ async lifecycleActionableRuns(nowMs, limit) {
2418
+ const size = Math.max(1, Math.min(limit, 100));
2419
+ const deliverable = new Map();
2420
+ const deliverableTo = (event, namespace) => {
2421
+ const key = `${namespace}\u0000${event}`;
2422
+ let entry = deliverable.get(key);
2423
+ if (!entry) {
2424
+ entry = this.deliverableSubscribers(event, namespace).then((d) => d.all);
2425
+ deliverable.set(key, entry);
2426
+ }
2427
+ return entry;
2428
+ };
2429
+ const out = [];
2430
+ const seen = new Set();
2431
+ let scanned = 0;
2432
+ for (let page = 1; out.length < limit && scanned < WorkflowEngine.LIFECYCLE_SWEEP_SCAN_CAP;) {
2433
+ const runs = await asHeartbeat(() => this.store.listRuns({
2434
+ tag: LIFECYCLE_PENDING_TAG,
2435
+ statuses: ['completed', 'failed', 'cancelled', 'dead'],
2436
+ ...(this.namespace !== undefined ? { namespace: this.namespace } : {}),
2437
+ updatedBefore: nowMs - WorkflowEngine.LIFECYCLE_SWEEP_GRACE_MS,
2438
+ page,
2439
+ size,
2440
+ }));
2441
+ for (const run of runs) {
2442
+ if (out.length >= limit || scanned >= WorkflowEngine.LIFECYCLE_SWEEP_SCAN_CAP)
2443
+ break;
2444
+ scanned += 1;
2445
+ if (seen.has(run.id))
2446
+ continue;
2447
+ seen.add(run.id);
2448
+ if (nowMs - run.updatedAt.getTime() < WorkflowEngine.LIFECYCLE_SWEEP_GRACE_MS)
2449
+ continue;
2450
+ const kind = lifecycleKindOf(run.status);
2451
+ if (!kind)
2452
+ continue;
2453
+ const owed = owedSubscribersOf(run);
2454
+ if (owed.length === 0) {
2455
+ out.push(run);
2456
+ continue;
2457
+ }
2458
+ const here = await deliverableTo(lifecycleEventName(run.workflow, kind), normalizeNamespace(run.namespace));
2459
+ if (owed.some((s) => here.has(s)) ||
2460
+ (await this.orphanedSubscribers(run, owed, here, nowMs)).length > 0) {
2461
+ out.push(run);
2462
+ }
2463
+ }
2464
+ if (runs.length < size)
2465
+ break;
2466
+ page += 1;
2467
+ }
2468
+ return out;
2469
+ }
2470
+ /**
2471
+ * Publish settled runs' lifecycle events again, whether or not they are still marked: one run
2472
+ * (`{ runId }`), or a workflow's settled runs (`{ workflow, kind, since, until, limit }`, read in
2473
+ * pages of the store's listing, filtered by when the run settled). `subscriber` restricts it to
2474
+ * one subscribing workflow, which must subscribe to the run's ending (registered here or announced
2475
+ * by a live worker for the run's namespace): otherwise the run is reported in `failed` with
2476
+ * `subscriber <name> does not subscribe to <event>` and nothing is marked. The subscribers
2477
+ * registered on this instance are started; any other (announced by a live worker) is marked owed
2478
+ * again, so the sweep of an instance that registers it starts it, and is not counted in
2479
+ * `started`. The same path as the live publish, so a subscriber whose run already exists is not
2480
+ * started again; a subscriber that debounces or batches is not routed again once the run carries
2481
+ * its coalesced tag, unless it also carries its trigger-failed tag, since an accumulator that
2482
+ * already fired would start the target again. A run another
2483
+ * publisher holds past the lease retries is reported in `failed` with subscriber `*`. A
2484
+ * subscriber registered after the runs settled (more than {@link LIFECYCLE_HEAL_WINDOW_MS} after,
2485
+ * past what the sweep heals) is backfilled this way. A workflow target's `namespace` narrows it to
2486
+ * that namespace; on an engine bound to another namespace it publishes nothing.
2487
+ */
2488
+ async republishLifecycle(target) {
2489
+ const result = { runs: 0, started: 0, failed: [] };
2490
+ // A backfill usually follows a deploy: read the fleet's subscriptions now, not from the cache.
2491
+ await this.refreshAnnouncedSubscriptions();
2492
+ const publish = async (run) => {
2493
+ if (!lifecycleKindOf(run.status))
2494
+ return;
2495
+ result.runs += 1;
2496
+ const { started, failed, busy } = await this.publishSettled(run, {
2497
+ republish: true,
2498
+ subscriber: target.subscriber,
2499
+ });
2500
+ if (busy) {
2501
+ result.failed.push({
2502
+ runId: run.id,
2503
+ subscriber: '*',
2504
+ error: `run ${run.id} is being published elsewhere; try again`,
2505
+ });
2506
+ return;
2507
+ }
2508
+ result.started += started;
2509
+ for (const { workflow, error } of failed) {
2510
+ result.failed.push({
2511
+ runId: run.id,
2512
+ subscriber: workflow,
2513
+ error: error instanceof Error ? error.message : String(error),
2514
+ });
2515
+ }
2516
+ };
2517
+ if ('runId' in target) {
2518
+ const run = await this.store.getRun(target.runId);
2519
+ if (run)
2520
+ await publish(run);
2521
+ return result;
2522
+ }
2523
+ if (target.namespace !== undefined &&
2524
+ this.namespace !== undefined &&
2525
+ target.namespace !== this.namespace) {
2526
+ return result;
2527
+ }
2528
+ const namespace = target.namespace ?? this.namespace;
2529
+ const limit = target.limit ?? 1000;
2530
+ const statuses = target.kind === 'completed'
2531
+ ? ['completed']
2532
+ : target.kind === 'failed'
2533
+ ? ['failed', 'cancelled', 'dead']
2534
+ : ['completed', 'failed', 'cancelled', 'dead'];
2535
+ const size = Math.min(limit, 100);
2536
+ let seen = 0;
2537
+ for (let page = 1; seen < limit; page += 1) {
2538
+ const runs = await this.store.listRuns({
2539
+ workflow: target.workflow,
2540
+ statuses,
2541
+ ...(namespace !== undefined ? { namespace } : {}),
2542
+ page,
2543
+ size,
2544
+ });
2545
+ for (const run of runs) {
2546
+ if (seen >= limit)
2547
+ break;
2548
+ const settledAt = run.updatedAt.getTime();
2549
+ if (target.since && settledAt < target.since.getTime())
2550
+ continue;
2551
+ if (target.until && settledAt > target.until.getTime())
2552
+ continue;
2553
+ seen += 1;
2554
+ await publish(run);
2555
+ }
2556
+ if (runs.length < size)
2557
+ break;
2558
+ }
2559
+ return result;
2560
+ }
2561
+ /**
2562
+ * Wake the `ctx.waitForRun` waiters on `runId`, which just settled, and on the earlier runs of its
2563
+ * continue-as-new chain, with the event of the run that ends the chain (none yet when `runId`
2564
+ * handed off). One waiter listing per settle. Best effort: a waiter missed here (this process dies
2565
+ * first) is woken by {@link sweepRunWaits}. Internal `__` runs have no waiters, so they are
2566
+ * skipped without a read.
2567
+ */
2568
+ notifyRunWaiters(runId, workflow) {
2569
+ if (workflow?.startsWith('__'))
2570
+ return;
2571
+ void (async () => {
2572
+ const predecessors = chainPredecessorsOf(runId);
2573
+ const targets = new Set([runId, ...predecessors]);
2574
+ // Every run of a chain shares the first one's id as a prefix.
2575
+ const prefix = predecessors[0] !== undefined
2576
+ ? `${RUN_WAIT_TOKEN_PREFIX}${predecessors[0]}`
2577
+ : runWaitPrefix(runId);
2578
+ const waiters = (await this.store.listSignalWaiters(prefix)).filter((w) => targets.has(runWaitTargetOf(w) ?? ''));
2579
+ if (waiters.length === 0)
2580
+ return;
2581
+ const run = await lastRunOfChain((id) => this.store.getRun(id), runId);
2582
+ if (!run || !lifecycleKindOf(run.status))
2583
+ return;
2584
+ const event = lifecycleEventOf(run);
2585
+ for (const waiter of waiters)
2586
+ await this.wakeSignalWaiter(waiter.token, event);
2587
+ })().catch(() => undefined);
2588
+ }
2589
+ /** How many waited-on runs one {@link sweepRunWaits} reads at most. */
2590
+ static RUN_WAIT_SWEEP_SCAN_CAP = 1000;
2591
+ /** Where the next {@link sweepRunWaits} starts in the list of waited-on runs. */
2592
+ runWaitSweepCursor = 0;
2593
+ /**
2594
+ * Wake the `ctx.waitForRun` waiters (up to `limit`) whose run, or the run ending its
2595
+ * continue-as-new chain, has already settled: the recovery for a process that died between
2596
+ * settling a run and waking its waiters. Waiters on runs still in flight are skipped, reading at
2597
+ * most {@link RUN_WAIT_SWEEP_SCAN_CAP} waited-on runs per call; the next call starts where this one
2598
+ * stopped, so waiters on long-running runs never hide the ones that can be woken. Returns how many
2599
+ * it woke.
2600
+ */
2601
+ async sweepRunWaits(limit = 100) {
2602
+ const waiters = await asHeartbeat(() => this.store.listSignalWaiters(RUN_WAIT_TOKEN_PREFIX));
2603
+ const byTarget = new Map();
2604
+ for (const waiter of waiters) {
2605
+ const target = runWaitTargetOf(waiter);
2606
+ if (target === undefined)
2607
+ continue;
2608
+ byTarget.set(target, [...(byTarget.get(target) ?? []), waiter]);
2609
+ }
2610
+ const targets = [...byTarget.keys()].sort();
2611
+ if (targets.length === 0)
2612
+ return 0;
2613
+ const from = this.runWaitSweepCursor % targets.length;
2614
+ const scan = Math.min(targets.length, WorkflowEngine.RUN_WAIT_SWEEP_SCAN_CAP);
2615
+ let woken = 0;
2616
+ let read = 0;
2617
+ for (; read < scan && woken < limit; read += 1) {
2618
+ const target = targets[(from + read) % targets.length];
2619
+ const run = await lastRunOfChain((id) => this.store.getRun(id), target);
2620
+ if (!run || !lifecycleKindOf(run.status))
2621
+ continue;
2622
+ const event = lifecycleEventOf(run);
2623
+ for (const waiter of byTarget.get(target) ?? []) {
2624
+ if (await this.wakeSignalWaiter(waiter.token, event))
2625
+ woken += 1;
2626
+ }
2627
+ }
2628
+ this.runWaitSweepCursor = read >= targets.length ? 0 : (from + read) % targets.length;
2629
+ return woken;
2630
+ }
1417
2631
  /** Events waiting for the control-plane publish in flight, in emit order (see {@link broadcastEvents}). */
1418
2632
  eventsToBroadcast = [];
1419
2633
  broadcastingEvents = false;
@@ -1680,8 +2894,11 @@ export class WorkflowEngine {
1680
2894
  * engine's read API for inspection surfaces (the dashboard, the `durable:runs` CLI) so they query
1681
2895
  * through the engine rather than reaching for its private store.
1682
2896
  */
1683
- listRuns(query) {
1684
- return this.store.listRuns(query);
2897
+ async listRuns(query) {
2898
+ const scoped = withoutBookkeepingRuns(query);
2899
+ const runs = await this.store.listRuns(scoped);
2900
+ const hidden = scoped.excludeWorkflows;
2901
+ return hidden?.length ? runs.filter((run) => !hidden.includes(run.workflow)) : runs;
1685
2902
  }
1686
2903
  /**
1687
2904
  * The distinct values of ONE filter axis over the runs matching `query`, with counts — what a
@@ -1690,10 +2907,11 @@ export class WorkflowEngine {
1690
2907
  * {@link listRuns}.
1691
2908
  */
1692
2909
  runValueFacets(axis, query, opts) {
2910
+ const scoped = withoutBookkeepingRuns(query);
1693
2911
  if (this.store.runValueFacets) {
1694
- return this.store.runValueFacets(axis, query, opts);
2912
+ return this.store.runValueFacets(axis, scoped, opts);
1695
2913
  }
1696
- return scanRunValueFacets(this.store, axis, query, opts);
2914
+ return scanRunValueFacets(this.store, axis, scoped, opts);
1697
2915
  }
1698
2916
  /** List a run's step checkpoints (its timeline). Part of the engine's read API; see {@link listRuns}. */
1699
2917
  listCheckpoints(runId) {
@@ -1704,15 +2922,149 @@ export class WorkflowEngine {
1704
2922
  * has {@link StateStore.runFacets} (else counted over a listing). Part of the engine's read API.
1705
2923
  */
1706
2924
  runFacets(query) {
1707
- return runFacetsOf(this.store, query);
2925
+ return runFacetsOf(this.store, withoutBookkeepingRuns(query));
1708
2926
  }
1709
2927
  /**
1710
2928
  * A run's detail (see `loadRunDetail`), built from the store's checkpoint aggregates when it has
1711
2929
  * `checkpointStats` + `queryCheckpoints` — a summarized detail then reads a few rows per wide fan
1712
2930
  * instead of the whole history, and carries an `etag` that `options.ifNoneMatch` can match.
1713
2931
  */
1714
- getRunDetail(runId, options) {
1715
- return loadRunDetail(this.detailSource(), runId, options);
2932
+ async getRunDetail(runId, options) {
2933
+ const source = this.detailSource();
2934
+ const withExtras = async (detail) => {
2935
+ const extras = await this.lifecycleDetail(detail.run);
2936
+ return {
2937
+ ...detail,
2938
+ ...extras,
2939
+ ...(detail.etag ? { etag: extendEtag(detail.etag, nonEmptyObject(extras)) } : {}),
2940
+ };
2941
+ };
2942
+ const ifNoneMatch = options?.ifNoneMatch;
2943
+ if (ifNoneMatch === undefined) {
2944
+ const detail = await loadRunDetail(source, runId, options);
2945
+ return detail ? withExtras(detail) : null;
2946
+ }
2947
+ // A revalidation: read the run and its validator only ('*' always matches), add what the
2948
+ // detail reads from outside the run, and answer 304 when both are unchanged. A source that
2949
+ // cannot version the detail answers it in full instead.
2950
+ const head = await loadRunDetail(source, runId, { ...options, ifNoneMatch: '*' });
2951
+ if (!head)
2952
+ return null;
2953
+ if (!head.notModified || !head.etag)
2954
+ return withExtras(head);
2955
+ const extras = await this.lifecycleDetail(head.run);
2956
+ const etag = extendEtag(head.etag, nonEmptyObject(extras));
2957
+ if (etagMatches(ifNoneMatch, etag)) {
2958
+ return { run: head.run, timeline: [], children: [], notModified: true, etag };
2959
+ }
2960
+ const { ifNoneMatch: _matched, ...fresh } = options;
2961
+ const detail = await loadRunDetail(source, runId, fresh);
2962
+ return detail ? withExtras(detail) : null;
2963
+ }
2964
+ /**
2965
+ * What a run detail reads beyond the run row and its checkpoints: the subscribers its lifecycle
2966
+ * event goes to (of its namespace) and where it stands for each, its workflow's lifecycle links,
2967
+ * and the `ctx.waitForRun` waits on either side of it with the other run's status. A few reads per
2968
+ * detail: the run's own waiters while it is suspended, the waiters on it (one prefix read), and
2969
+ * one batched read of every linked run.
2970
+ */
2971
+ async lifecycleDetail(run) {
2972
+ const namespace = normalizeNamespace(run.namespace);
2973
+ const kind = lifecycleKindOf(run.status);
2974
+ const subscribers = kind ? this.lifecycleSubscribers(run.workflow, kind, namespace) : [];
2975
+ const links = await this.workflowLifecycle(run.workflow);
2976
+ const deliveries = kind
2977
+ ? lifecycleDeliveriesOf(run, {
2978
+ subscribers,
2979
+ live: new Set(subscribers),
2980
+ coalesced: this.coalescedSubscribers(run, kind),
2981
+ })
2982
+ : [];
2983
+ const [own, onIt] = await Promise.all([
2984
+ run.status === 'suspended' ? listSignalWaitersOfRuns(this.store, [run.id]) : [],
2985
+ this.store.listSignalWaiters(runWaitPrefix(run.id)),
2986
+ ]);
2987
+ const waitsFor = own.flatMap((w) => {
2988
+ const target = runWaitTargetOf(w);
2989
+ return target !== undefined ? [{ runId: target, seq: w.seq }] : [];
2990
+ });
2991
+ const awaitedBy = onIt
2992
+ .filter((w) => runWaitTargetOf(w) === run.id)
2993
+ .map((w) => ({ runId: w.runId, seq: w.seq }));
2994
+ const started = deliveries.flatMap((d) => d.state === 'started' && d.runId !== undefined ? [d.runId] : []);
2995
+ const source = lifecycleSourceOf(run);
2996
+ const linked = await getRunsByIds(this.store, [
2997
+ ...(source ? [source.runId] : []),
2998
+ ...started,
2999
+ ...waitsFor.map((l) => l.runId),
3000
+ ...awaitedBy.map((l) => l.runId),
3001
+ ]);
3002
+ const withRun = (link) => {
3003
+ const other = link.runId !== undefined ? linked.get(link.runId) : undefined;
3004
+ return other ? { ...link, status: other.status, namespace: other.namespace } : link;
3005
+ };
3006
+ const sourceRun = source ? linked.get(source.runId) : undefined;
3007
+ const sourceTrace = lifecycleTraceparentOf(sourceRun?.tags)?.split('-')[1];
3008
+ return {
3009
+ ...(source
3010
+ ? {
3011
+ triggeredBy: {
3012
+ ...source,
3013
+ ...(sourceRun ? { status: sourceRun.status, namespace: sourceRun.namespace } : {}),
3014
+ ...(sourceTrace ? { traceId: sourceTrace } : {}),
3015
+ },
3016
+ }
3017
+ : {}),
3018
+ ...(subscribers.length > 0 ? { lifecycleSubscribers: subscribers } : {}),
3019
+ ...(links.startsOnSettle.length > 0 || links.startedBy.length > 0
3020
+ ? { workflowLifecycle: links }
3021
+ : {}),
3022
+ ...(deliveries.length > 0
3023
+ ? {
3024
+ lifecycleDeliveries: deliveries.map((d) => (d.state === 'started' ? withRun(d) : d)),
3025
+ }
3026
+ : {}),
3027
+ ...(waitsFor.length > 0 ? { waitsFor: waitsFor.map(withRun) } : {}),
3028
+ ...(awaitedBy.length > 0 ? { awaitedBy: awaitedBy.map(withRun) } : {}),
3029
+ };
3030
+ }
3031
+ /**
3032
+ * The subscribers registered here that coalesce `run`'s lifecycle event (debounce/batch) and
3033
+ * accept it, each with the accumulator run the event went to: the subscription's `where`, `input`
3034
+ * and `key` applied to the event as `startSubscribers` does. Undefined when the key cannot be
3035
+ * computed. A subscriber whose `where` declines the event is left out.
3036
+ */
3037
+ coalescedSubscribers(run, kind) {
3038
+ const out = new Map();
3039
+ const event = lifecycleEventName(run.workflow, kind);
3040
+ const triggers = this.eventTriggers.get(event);
3041
+ if (!triggers)
3042
+ return out;
3043
+ let payload;
3044
+ for (const [workflow, subscriptions] of triggers) {
3045
+ const batch = this.latest.get(workflow)?.eventBatch;
3046
+ if (!batch)
3047
+ continue;
3048
+ if (!batch.key && subscriptions.every((sub) => !sub.where)) {
3049
+ out.set(workflow, accumulatorRunId(workflow));
3050
+ continue;
3051
+ }
3052
+ try {
3053
+ payload ??= lifecycleEventOf(run);
3054
+ const { accepted } = acceptingSubscription(subscriptions, payload);
3055
+ if (!accepted)
3056
+ continue;
3057
+ const input = accepted.input ? accepted.input(payload) : payload;
3058
+ const key = batch.key ? batch.key(input) : undefined;
3059
+ out.set(workflow, key === undefined || typeof key === 'string'
3060
+ ? accumulatorRunId(workflow, key)
3061
+ : undefined);
3062
+ }
3063
+ catch {
3064
+ out.set(workflow, undefined);
3065
+ }
3066
+ }
3067
+ return out;
1716
3068
  }
1717
3069
  /** One page of a summarized fan's members (see `loadFanMembers`), versioned like {@link getRunDetail}. */
1718
3070
  getFanMembers(runId, group, query) {
@@ -1955,7 +3307,7 @@ export class WorkflowEngine {
1955
3307
  passes.push(scoped);
1956
3308
  }
1957
3309
  else {
1958
- const [lastKept] = await this.store.listRuns({ ...scoped, page: maxCount, size: 1 });
3310
+ const [lastKept] = await this.store.listRuns(withoutBookkeepingRuns({ ...scoped, page: maxCount, size: 1 }));
1959
3311
  if (lastKept) {
1960
3312
  passes.push({ ...scoped, createdBefore: lastKept.createdAt.getTime() - 1 });
1961
3313
  }
@@ -1963,9 +3315,13 @@ export class WorkflowEngine {
1963
3315
  }
1964
3316
  }
1965
3317
  for (const pass of passes) {
1966
- const candidates = await this.store.listRuns({ ...pass, size: 100 });
3318
+ const candidates = await this.store.listRuns(withoutBookkeepingRuns({ ...pass, size: 100 }));
1967
3319
  const archived = [];
1968
3320
  for (const run of candidates) {
3321
+ // A run whose lifecycle event is still owed is kept until it is published, and a registry
3322
+ // run is replaced by its engine's heartbeat instead.
3323
+ if (retentionKeeps(run.tags) || run.workflow === LIFECYCLE_REGISTRY_WORKFLOW)
3324
+ continue;
1969
3325
  try {
1970
3326
  if (this.evictListeners.size > 0) {
1971
3327
  const checkpoints = await this.store.listCheckpoints(run.id);
@@ -2103,7 +3459,7 @@ export class WorkflowEngine {
2103
3459
  let deleted = 0;
2104
3460
  let previous = '';
2105
3461
  for (;;) {
2106
- const batch = await this.store.listRuns({ ...scope, statuses, size });
3462
+ const batch = await this.store.listRuns(withoutBookkeepingRuns({ ...scope, statuses, size }));
2107
3463
  if (batch.length === 0)
2108
3464
  break;
2109
3465
  const fingerprint = batch.map((r) => r.id).join('\u0000');
@@ -2544,11 +3900,11 @@ export class WorkflowEngine {
2544
3900
  // (and the listRuns calls above it), so this run could have settled on its own — via its own
2545
3901
  // in-flight turn, an operator cancel, or another sweep — in the gap. Only clobber it into
2546
3902
  // `cancelled` if it's still genuinely `running`/`suspended`.
2547
- const applied = await this.store.updateRunIf(run.id, ['running', 'suspended'], {
3903
+ const applied = await this.store.updateRunIf(run.id, ['running', 'suspended'], this.withLifecycleMarker(run, {
2548
3904
  status: 'cancelled',
2549
3905
  error,
2550
3906
  updatedAt: new Date(),
2551
- });
3907
+ }));
2552
3908
  if (!applied)
2553
3909
  continue;
2554
3910
  this.emit({
@@ -2872,11 +4228,11 @@ export class WorkflowEngine {
2872
4228
  * (e.g. a lease-free `cancel()`), so it is not ours to touch.
2873
4229
  */
2874
4230
  async deadLetterLeased(run, from, error) {
2875
- const applied = await this.store.updateRunIf(run.id, from, {
4231
+ const applied = await this.store.updateRunIf(run.id, from, this.withLifecycleMarker(run, {
2876
4232
  status: 'dead',
2877
4233
  error,
2878
4234
  updatedAt: new Date(),
2879
- });
4235
+ }));
2880
4236
  await this.store.releaseRunLock(run.id, this.instanceId);
2881
4237
  if (!applied)
2882
4238
  return null;
@@ -2969,6 +4325,32 @@ export class WorkflowEngine {
2969
4325
  * lease, held while the run's `wakeAt` moves {@link leaseMs} ahead) keeps every other instance's
2970
4326
  * poll from handing it off again meanwhile; a wake lost on the way comes due again then.
2971
4327
  */
4328
+ markClaim(runId) {
4329
+ this.claimsInFlight.set(runId, (this.claimsInFlight.get(runId) ?? 0) + 1);
4330
+ // Bounded by forgetting: a proof under way then reads a changed count, and locks the ordinary way.
4331
+ if (this.claimsMade.size >= 4096)
4332
+ this.claimsMade.clear();
4333
+ this.claimsMade.set(runId, (this.claimsMade.get(runId) ?? 0) + 1);
4334
+ }
4335
+ unmarkClaim(runId) {
4336
+ const left = (this.claimsInFlight.get(runId) ?? 1) - 1;
4337
+ if (left > 0)
4338
+ this.claimsInFlight.set(runId, left);
4339
+ else
4340
+ this.claimsInFlight.delete(runId);
4341
+ }
4342
+ /**
4343
+ * Whether this instance holds the run's lease for a turn, from the store and by owner. A renewal can land
4344
+ * on a claim marked after it was sent, and that claim is about to be released, so any claim begun while
4345
+ * the renewal was out, even one already over, leaves it unproved.
4346
+ */
4347
+ async proveOwnLease(runId, nowMs) {
4348
+ if (this.claimsInFlight.has(runId))
4349
+ return false;
4350
+ const before = this.claimsMade.get(runId);
4351
+ const renewed = await this.store.renewRunLock(runId, this.instanceId, nowMs + this.leaseMs);
4352
+ return renewed && !this.claimsInFlight.has(runId) && this.claimsMade.get(runId) === before;
4353
+ }
2972
4354
  async handOffDue(runs, nowMs) {
2973
4355
  if (this.draining)
2974
4356
  return [];
@@ -2980,6 +4362,8 @@ export class WorkflowEngine {
2980
4362
  next += 1;
2981
4363
  if (!run || this.draining)
2982
4364
  return;
4365
+ // Marked before the lock, so no turn of this instance can take this claim for its own lease.
4366
+ this.markClaim(run.id);
2983
4367
  try {
2984
4368
  if (!(await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs))) {
2985
4369
  continue;
@@ -3017,6 +4401,9 @@ export class WorkflowEngine {
3017
4401
  catch (error) {
3018
4402
  console.error(`[adonis-durable] could not hand off due run ${run.id}; leaving it for another instance or a later tick:`, error);
3019
4403
  }
4404
+ finally {
4405
+ this.unmarkClaim(run.id);
4406
+ }
3020
4407
  }
3021
4408
  };
3022
4409
  await Promise.all(Array.from({ length: Math.min(8, runs.length) }, () => worker()));
@@ -3051,7 +4438,10 @@ export class WorkflowEngine {
3051
4438
  }
3052
4439
  // resume() checks the namespace and throws NamespaceMismatch when it doesn't match (releasing the
3053
4440
  // lease) — a foreign run leased here is simply skipped rather than run.
3054
- return this.resumeImplicit(runId).catch(async (err) => {
4441
+ this.lockedForTurn.add(runId);
4442
+ return this.resumeImplicit(runId)
4443
+ .finally(() => this.lockedForTurn.delete(runId))
4444
+ .catch(async (err) => {
3055
4445
  if (err instanceof NamespaceMismatch)
3056
4446
  return null;
3057
4447
  if (err instanceof WorkflowNotRegistered) {
@@ -3385,6 +4775,8 @@ export class WorkflowEngine {
3385
4775
  await this.redispatchIfHolderCrashed(run, nowMs);
3386
4776
  // A per-run hook (recovery counting / dead-lettering) may settle the run terminally instead.
3387
4777
  const settled = onLocked ? await onLocked(run) : undefined;
4778
+ if (!settled)
4779
+ this.lockedForTurn.add(run.id);
3388
4780
  results.push(settled ?? (await this.resumeImplicit(run.id)));
3389
4781
  }
3390
4782
  catch (error) {
@@ -3402,6 +4794,9 @@ export class WorkflowEngine {
3402
4794
  // (a version skew, a store outage) is what tells an operator which.
3403
4795
  console.error(`[adonis-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance or a later tick:`, error);
3404
4796
  }
4797
+ finally {
4798
+ this.lockedForTurn.delete(run.id);
4799
+ }
3405
4800
  }
3406
4801
  };
3407
4802
  await Promise.all(Array.from({ length: Math.min(CONCURRENCY, runs.length) }, () => worker()));
@@ -3417,7 +4812,9 @@ export class WorkflowEngine {
3417
4812
  * match the payload satisfies (fan-out, vs `signal`'s point-to-point token).
3418
4813
  * 2. **Starts** a fresh run of every workflow registered with `onEvent: [name]`, passing the
3419
4814
  * payload as input. Idempotent by `evt:<id>:<workflow>` — pass `opts.id` to dedupe redeliveries
3420
- * of the same logical event (default: a fresh uuid, so each publish triggers once).
4815
+ * of the same logical event (default: a fresh uuid, so each publish triggers once). A workflow
4816
+ * a non-Node worker announces as subscribing to `name` (a Python `on_event=[...]`) is started
4817
+ * the same way, by name, when this engine can reach it (see {@link announcedRemoteSubscribers}).
3421
4818
  *
3422
4819
  * Reliable (buffered) delivery — mirrors `signalWithStart`'s reliability contract, so document it
3423
4820
  * exactly: a publish that resumes ≥1 live waiter, OR routes into an `eventBatch` accumulator /
@@ -3439,27 +4836,14 @@ export class WorkflowEngine {
3439
4836
  touched += 1;
3440
4837
  }
3441
4838
  }
3442
- const subscribers = this.eventTriggers.get(name);
3443
- if (subscribers?.size) {
4839
+ await this.freshAnnouncedSubscriptions();
4840
+ const remote = await this.announcedRemoteSubscribers(name);
4841
+ if (this.eventTriggers.get(name)?.size || remote.length > 0) {
3444
4842
  const eventId = opts?.id ?? globalThis.crypto.randomUUID();
3445
- for (const workflow of subscribers) {
3446
- // A subscriber that rejects the payload (validateInput) must not block the others or the
3447
- // waiters — its run simply never starts, mirroring fire-and-forget dead-letter routing.
3448
- try {
3449
- const batch = this.latest.get(workflow)?.eventBatch;
3450
- if (batch) {
3451
- // Coalesce: route the event into a per-workflow accumulator (one long-lived run that
3452
- // debounces/batches and then starts the target with the collected payload(s)).
3453
- await this.accumulators.route(workflow, batch, payload);
3454
- }
3455
- else {
3456
- await this.start(workflow, payload, `evt:${eventId}:${workflow}`);
3457
- }
3458
- touched += 1;
3459
- }
3460
- catch {
3461
- // skip this subscriber
3462
- }
4843
+ const { reached, failed } = await this.startSubscribers(name, payload, (workflow) => `evt:${eventId}:${workflow}`, undefined, remote);
4844
+ touched += reached.length;
4845
+ for (const { workflow, error } of failed) {
4846
+ console.warn(`[adonis-durable] event ${name}: subscriber ${workflow} not started:`, error);
3463
4847
  }
3464
4848
  }
3465
4849
  // Nobody received it live (no waiter matched, no subscriber exists/started) — buffer ONE copy
@@ -3807,12 +5191,12 @@ export class WorkflowEngine {
3807
5191
  }
3808
5192
  const error = { message: 'cancelled' };
3809
5193
  // Clears a remote turn's decision marker too: a cancelled run awaits no decision.
3810
- await this.store.updateRun(runId, {
5194
+ await this.store.updateRun(runId, this.withLifecycleMarker(run, {
3811
5195
  status: 'cancelled',
3812
5196
  error,
3813
5197
  awaitingDecisionTaskId: undefined,
3814
5198
  updatedAt: new Date(),
3815
- });
5199
+ }));
3816
5200
  this.emit({
3817
5201
  type: 'run.failed',
3818
5202
  runId,
@@ -3919,7 +5303,7 @@ export class WorkflowEngine {
3919
5303
  * the full listing for a store without one.
3920
5304
  */
3921
5305
  countRuns(query) {
3922
- return countRuns(this.store, query);
5306
+ return countRuns(this.store, withoutBookkeepingRuns(query));
3923
5307
  }
3924
5308
  /**
3925
5309
  * Every workflow the live fleet can execute in this engine's partition: what workers DECLARE in
@@ -4312,8 +5696,15 @@ export class WorkflowEngine {
4312
5696
  // The wakes this turn answers: it reads the store after they were recorded.
4313
5697
  const reason = this.wakeReasons.get(run.id) ?? { generic: true, fans: new Map() };
4314
5698
  this.wakeReasons.delete(run.id);
4315
- if (run.lockedBy !== this.instanceId || (run.lockedUntil ?? 0) <= lockNow) {
4316
- if (!(await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow))) {
5699
+ // `run` may have been read while this instance held only a short claim that is now released, or is
5700
+ // about to be. So a re-entry is proved against the store, by owner, never read off the snapshot: a
5701
+ // turn holding nothing runs beside whoever locks next.
5702
+ const lockedForTurn = this.lockedForTurn.delete(run.id);
5703
+ const reentered = run.lockedBy === this.instanceId &&
5704
+ (run.lockedUntil ?? 0) > lockNow &&
5705
+ (lockedForTurn || (await this.proveOwnLease(run.id, lockNow)));
5706
+ if (!reentered) {
5707
+ if (!(await this.lockRunPastLifecyclePublish(run.id, lockNow))) {
4317
5708
  const stamping = this.draining ? undefined : this.stampWakeRequest(run.id, reason);
4318
5709
  this.trackEffect(this.requestWake(run.id, run.lockedUntil, reason, stamping));
4319
5710
  const refused = { runId: run.id, status: run.status };
@@ -4587,12 +5978,12 @@ export class WorkflowEngine {
4587
5978
  // resurrect/clobber it — the conditional write (status still one of SETTLE_ELIGIBLE_STATUSES) is
4588
5979
  // the guard; the database, not a prior JS read, arbitrates. Also clears any error from an
4589
5980
  // earlier failed-then-retried attempt — a completed run is a success.
4590
- const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, {
5981
+ const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, this.withLifecycleMarker(run, {
4591
5982
  status: 'completed',
4592
5983
  output: outcome.output,
4593
5984
  error: undefined,
4594
5985
  updatedAt,
4595
- });
5986
+ }));
4596
5987
  if (!applied)
4597
5988
  return this.echoCurrentStatus(run.id);
4598
5989
  this.emit({
@@ -4618,11 +6009,11 @@ export class WorkflowEngine {
4618
6009
  // (observed: a cascade-requeued child's fresh success lost to the stale buffered failure). A
4619
6010
  // legitimate retry always transitions through `pending`/`running` first (requeue flips the
4620
6011
  // status), so it is unaffected.
4621
- const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES.filter((s) => s !== 'failed'), {
6012
+ const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES.filter((s) => s !== 'failed'), this.withLifecycleMarker(run, {
4622
6013
  status: 'failed',
4623
6014
  error,
4624
6015
  updatedAt,
4625
- });
6016
+ }));
4626
6017
  if (!applied)
4627
6018
  return this.echoCurrentStatus(run.id);
4628
6019
  this.emit({
@@ -4642,11 +6033,11 @@ export class WorkflowEngine {
4642
6033
  // conditional write IS the guard — it subsumes the read-then-write this branch used to do (that
4643
6034
  // was itself a narrower, TOCTOU-prone version of the same check, and only looked for `cancelled`;
4644
6035
  // this now also catches `completed`/`dead` reached via any other path mid-turn).
4645
- const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, {
6036
+ const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, await this.withRunTrace(run, {
4646
6037
  status: 'suspended',
4647
6038
  wakeAt: this.reconcileWakeAt(outcome.wakeAt),
4648
6039
  updatedAt,
4649
- });
6040
+ }));
4650
6041
  if (!applied)
4651
6042
  return this.echoCurrentStatus(run.id);
4652
6043
  this.emit({
@@ -4758,6 +6149,7 @@ export class WorkflowEngine {
4758
6149
  runId: run.id,
4759
6150
  workflow: run.workflow,
4760
6151
  namespace: run.namespace,
6152
+ ...(await this.sourceTraceOf(run)),
4761
6153
  });
4762
6154
  }
4763
6155
  if (registered.singleton && !(await this.singletons.admit(run, registered.singleton))) {
@@ -4831,7 +6223,7 @@ export class WorkflowEngine {
4831
6223
  message: err instanceof Error ? err.message : String(err),
4832
6224
  stack: err instanceof Error ? err.stack : undefined,
4833
6225
  };
4834
- await this.store.updateRun(run.id, { status: 'failed', error, updatedAt: new Date() });
6226
+ await this.store.updateRun(run.id, this.withLifecycleMarker(run, { status: 'failed', error, updatedAt: new Date() }));
4835
6227
  this.emit({
4836
6228
  type: 'run.failed',
4837
6229
  runId: run.id,
@@ -4913,12 +6305,12 @@ export class WorkflowEngine {
4913
6305
  // id (the `db` transport dedupes a turn by it, and the marker gates the decision on it).
4914
6306
  const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID()}`;
4915
6307
  const windowMs = this.decisionWindowMs(registered);
4916
- const marked = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, {
6308
+ const marked = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, await this.withRunTrace(run, {
4917
6309
  status: 'suspended',
4918
6310
  awaitingDecisionTaskId: taskId,
4919
6311
  wakeAt: windowMs != null ? this.clock() + Math.min(windowMs, MAX_BACKOFF_MS) : undefined,
4920
6312
  updatedAt: new Date(),
4921
- });
6313
+ }));
4922
6314
  if (!marked) {
4923
6315
  // Cancelled / completed / dead while this turn was being prepared: nothing to dispatch.
4924
6316
  await this.releaseTurnLease(run.id);
@@ -4952,13 +6344,13 @@ export class WorkflowEngine {
4952
6344
  const current = await this.store.getRun(run.id);
4953
6345
  if (current?.awaitingDecisionTaskId !== taskId)
4954
6346
  return this.echoCurrentStatus(run.id);
4955
- const failed = await this.store.updateRunIf(run.id, ['suspended'], {
6347
+ const failed = await this.store.updateRunIf(run.id, ['suspended'], this.withLifecycleMarker(run, {
4956
6348
  status: 'failed',
4957
6349
  error,
4958
6350
  awaitingDecisionTaskId: undefined,
4959
6351
  wakeAt: undefined,
4960
6352
  updatedAt: new Date(),
4961
- });
6353
+ }));
4962
6354
  if (!failed)
4963
6355
  return this.echoCurrentStatus(run.id);
4964
6356
  this.emit({
@@ -5770,12 +7162,12 @@ export class WorkflowEngine {
5770
7162
  await this.runCompensation(run, comp, i);
5771
7163
  }
5772
7164
  const error = { message: 'cancelled' };
5773
- await this.store.updateRun(run.id, {
7165
+ await this.store.updateRun(run.id, this.withLifecycleMarker(run, {
5774
7166
  status: 'cancelled',
5775
7167
  error,
5776
7168
  awaitingDecisionTaskId: undefined,
5777
7169
  updatedAt: new Date(),
5778
- });
7170
+ }));
5779
7171
  // The cancellation is durably recorded — consume the marker(s) so a later retry of this run id
5780
7172
  // doesn't get spuriously re-cancelled.
5781
7173
  for (;;) {
@@ -5818,6 +7210,7 @@ export class WorkflowEngine {
5818
7210
  runId: run.id,
5819
7211
  workflow: run.workflow,
5820
7212
  namespace: run.namespace,
7213
+ ...(await this.sourceTraceOf(run)),
5821
7214
  });
5822
7215
  }
5823
7216
  // Singleton admission gate: if this run shares its key with `limit` older in-flight runs, wait
@@ -5897,7 +7290,7 @@ export class WorkflowEngine {
5897
7290
  runId: run.id,
5898
7291
  workflow: run.workflow,
5899
7292
  namespace: run.namespace,
5900
- });
7293
+ }, { lifecycle: false });
5901
7294
  this.notifyParent(run.id, { ok: true, value: undefined });
5902
7295
  // Tracked handoff (see {@link handoffRun}): held in `postSettle` from this settle until the
5903
7296
  // continuation has entered `inflight` and settled, so `drain()` can't return before the next
@@ -7746,8 +9139,30 @@ export class WorkflowEngine {
7746
9139
  });
7747
9140
  }
7748
9141
  }
7749
- /** The worker group a convention-routed workflow's run dispatches to: its sanitized name, suffixed by
7750
- * the run's namespace (bare for absent/`''`/`default`) — the token a tenant worker serves. */
9142
+ /**
9143
+ * The namespace an announcement serves: the descriptor's own, else the `@<namespace>` suffix of its
9144
+ * routing group, else `default`.
9145
+ */
9146
+ function servedNamespace(namespace, group) {
9147
+ if (namespace)
9148
+ return namespace;
9149
+ const at = group?.lastIndexOf('@') ?? -1;
9150
+ return group && at >= 0 && at < group.length - 1 ? group.slice(at + 1) : 'default';
9151
+ }
9152
+ function withoutBookkeepingRuns(query) {
9153
+ if (query.workflow === LIFECYCLE_REGISTRY_WORKFLOW ||
9154
+ query.workflows?.includes(LIFECYCLE_REGISTRY_WORKFLOW)) {
9155
+ return query;
9156
+ }
9157
+ const excluded = query.excludeWorkflows ?? [];
9158
+ if (excluded.includes(LIFECYCLE_REGISTRY_WORKFLOW))
9159
+ return query;
9160
+ return { ...query, excludeWorkflows: [...excluded, LIFECYCLE_REGISTRY_WORKFLOW] };
9161
+ }
9162
+ /** `value` when it has at least one own key, else undefined. */
9163
+ function nonEmptyObject(value) {
9164
+ return Object.keys(value).length > 0 ? value : undefined;
9165
+ }
7751
9166
  function conventionGroup(name, namespace) {
7752
9167
  return tenantGroup(sanitizeQueueToken(name), namespace);
7753
9168
  }