@dudousxd/nestjs-durable-core 0.86.2 → 0.87.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -813,6 +813,243 @@ function eventMatches(payload, match) {
813
813
  return keys.every((k) => JSON.stringify(p[k]) === JSON.stringify(match[k]));
814
814
  }
815
815
  __name(eventMatches, "eventMatches");
816
+ function toEventSubscription(entry) {
817
+ return typeof entry === "string" ? {
818
+ event: entry
819
+ } : entry;
820
+ }
821
+ __name(toEventSubscription, "toEventSubscription");
822
+ function acceptingSubscription(subscriptions, payload) {
823
+ const errors = [];
824
+ for (const subscription of subscriptions) {
825
+ if (!subscription.where) return {
826
+ accepted: subscription,
827
+ errors
828
+ };
829
+ try {
830
+ if (subscription.where(payload)) return {
831
+ accepted: subscription,
832
+ errors
833
+ };
834
+ } catch (error2) {
835
+ errors.push(error2);
836
+ }
837
+ }
838
+ return {
839
+ errors
840
+ };
841
+ }
842
+ __name(acceptingSubscription, "acceptingSubscription");
843
+
844
+ // src/lifecycle-events.ts
845
+ function runCompletedEvent(workflow) {
846
+ return `run.completed:${workflowName(workflow)}`;
847
+ }
848
+ __name(runCompletedEvent, "runCompletedEvent");
849
+ function runFailedEvent(workflow) {
850
+ return `run.failed:${workflowName(workflow)}`;
851
+ }
852
+ __name(runFailedEvent, "runFailedEvent");
853
+ var LIFECYCLE_PENDING_TAG = "durable:lifecycle:pending";
854
+ function carriesLifecyclePending(tags) {
855
+ if (Array.isArray(tags)) return tags.includes(LIFECYCLE_PENDING_TAG);
856
+ if (typeof tags === "string") return tags.includes(`"${LIFECYCLE_PENDING_TAG}"`);
857
+ return false;
858
+ }
859
+ __name(carriesLifecyclePending, "carriesLifecyclePending");
860
+ var LIFECYCLE_TRIGGERED_TAG_PREFIX = "durable:triggered:";
861
+ var LIFECYCLE_OWED_TAG_PREFIX = "durable:lifecycle:owed:";
862
+ var LIFECYCLE_ORPHAN_MS = 24 * 60 * 60 * 1e3;
863
+ var LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX = "durable:trigger_failed:";
864
+ function triggerFailedSubscribersOf(tags) {
865
+ return (tags ?? []).filter((tag) => tag.startsWith(LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX)).map((tag) => tag.slice(LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX.length));
866
+ }
867
+ __name(triggerFailedSubscribersOf, "triggerFailedSubscribersOf");
868
+ function triggeredSubscribersOf(tags) {
869
+ return (tags ?? []).filter((tag) => tag.startsWith(LIFECYCLE_TRIGGERED_TAG_PREFIX)).map((tag) => tag.slice(LIFECYCLE_TRIGGERED_TAG_PREFIX.length));
870
+ }
871
+ __name(triggeredSubscribersOf, "triggeredSubscribersOf");
872
+ var LIFECYCLE_COALESCED_TAG_PREFIX = "durable:coalesced:";
873
+ function coalescedSubscribersOf(tags) {
874
+ return (tags ?? []).filter((tag) => tag.startsWith(LIFECYCLE_COALESCED_TAG_PREFIX)).map((tag) => tag.slice(LIFECYCLE_COALESCED_TAG_PREFIX.length));
875
+ }
876
+ __name(coalescedSubscribersOf, "coalescedSubscribersOf");
877
+ var LIFECYCLE_HEAL_WINDOW_MS = 2 * 60 * 1e3;
878
+ function owedSubscribersOf(tags) {
879
+ return (tags ?? []).filter((tag) => tag.startsWith(LIFECYCLE_OWED_TAG_PREFIX)).map((tag) => tag.slice(LIFECYCLE_OWED_TAG_PREFIX.length));
880
+ }
881
+ __name(owedSubscribersOf, "owedSubscribersOf");
882
+ var LIFECYCLE_TRACE_TAG_PREFIX = "durable:lifecycle:trace:";
883
+ function lifecycleTraceparentOf(tags) {
884
+ const tag = (tags ?? []).find((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX));
885
+ return tag?.slice(LIFECYCLE_TRACE_TAG_PREFIX.length) || void 0;
886
+ }
887
+ __name(lifecycleTraceparentOf, "lifecycleTraceparentOf");
888
+ function isLifecycleTag(tag) {
889
+ return tag === LIFECYCLE_PENDING_TAG || tag.startsWith(LIFECYCLE_OWED_TAG_PREFIX) || tag.startsWith(LIFECYCLE_TRIGGERED_TAG_PREFIX) || tag.startsWith(LIFECYCLE_COALESCED_TAG_PREFIX) || tag.startsWith(LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX) || tag.startsWith(LIFECYCLE_TRACE_TAG_PREFIX);
890
+ }
891
+ __name(isLifecycleTag, "isLifecycleTag");
892
+ function lifecycleRunId(sourceRunId, kind, subscriber) {
893
+ return `evt:${sourceRunId}:${kind}:${subscriber}`;
894
+ }
895
+ __name(lifecycleRunId, "lifecycleRunId");
896
+ function lifecycleKindOf(status) {
897
+ if (status === "completed") return "completed";
898
+ if (status === "failed" || status === "cancelled" || status === "dead") return "failed";
899
+ return void 0;
900
+ }
901
+ __name(lifecycleKindOf, "lifecycleKindOf");
902
+ function lifecycleEventName(workflow, kind) {
903
+ return kind === "completed" ? runCompletedEvent(workflow) : runFailedEvent(workflow);
904
+ }
905
+ __name(lifecycleEventName, "lifecycleEventName");
906
+ function parseLifecycleEvent(event) {
907
+ for (const kind of [
908
+ "completed",
909
+ "failed"
910
+ ]) {
911
+ const prefix = `run.${kind}:`;
912
+ if (event.startsWith(prefix) && event.length > prefix.length) {
913
+ return {
914
+ workflow: event.slice(prefix.length),
915
+ kind
916
+ };
917
+ }
918
+ }
919
+ return void 0;
920
+ }
921
+ __name(parseLifecycleEvent, "parseLifecycleEvent");
922
+ function lifecycleSourceOf(run) {
923
+ if (!run.id.startsWith("evt:")) return void 0;
924
+ for (const kind of [
925
+ "completed",
926
+ "failed"
927
+ ]) {
928
+ const suffix = `:${kind}:${run.workflow}`;
929
+ if (run.id.endsWith(suffix) && run.id.length > "evt:".length + suffix.length) {
930
+ return {
931
+ runId: run.id.slice("evt:".length, -suffix.length),
932
+ kind
933
+ };
934
+ }
935
+ }
936
+ return void 0;
937
+ }
938
+ __name(lifecycleSourceOf, "lifecycleSourceOf");
939
+ function triggeredRunsOf(run) {
940
+ const kind = lifecycleKindOf(run.status);
941
+ if (!kind) return [];
942
+ return (run.tags ?? []).filter((tag) => tag.startsWith(LIFECYCLE_TRIGGERED_TAG_PREFIX)).map((tag) => lifecycleRunId(run.id, kind, tag.slice(LIFECYCLE_TRIGGERED_TAG_PREFIX.length)));
943
+ }
944
+ __name(triggeredRunsOf, "triggeredRunsOf");
945
+ function lifecycleEventOf(run) {
946
+ const completed = run.status === "completed";
947
+ return {
948
+ runId: run.id,
949
+ workflow: run.workflow,
950
+ workflowVersion: run.workflowVersion,
951
+ status: run.status,
952
+ namespace: run.namespace,
953
+ input: run.input,
954
+ ...completed ? {
955
+ output: run.output
956
+ } : {
957
+ error: run.error
958
+ },
959
+ tags: run.tags?.filter((tag) => !isLifecycleTag(tag)),
960
+ searchAttributes: run.searchAttributes
961
+ };
962
+ }
963
+ __name(lifecycleEventOf, "lifecycleEventOf");
964
+ var LIFECYCLE_REGISTRY_WORKFLOW = "__lifecycle_registry";
965
+ var LIFECYCLE_REGISTRY_TAG = "durable:lifecycle:registry";
966
+ var LIFECYCLE_REGISTRY_HEARTBEAT_MS = 1e4;
967
+ var LIFECYCLE_REGISTRY_TTL_MS = 35e3;
968
+ var LIFECYCLE_REGISTRY_STALE_MS = 10 * 60 * 1e3;
969
+ function lifecycleRegistryRunId(instanceId) {
970
+ return `${LIFECYCLE_REGISTRY_WORKFLOW}:${instanceId}`;
971
+ }
972
+ __name(lifecycleRegistryRunId, "lifecycleRegistryRunId");
973
+ function lifecycleRegistryNamespace(input, rowNamespace) {
974
+ const engineNamespace = input?.engineNamespace;
975
+ if (engineNamespace === null) return null;
976
+ if (typeof engineNamespace === "string") return engineNamespace || "default";
977
+ const row = rowNamespace || "default";
978
+ return row === "default" ? null : row;
979
+ }
980
+ __name(lifecycleRegistryNamespace, "lifecycleRegistryNamespace");
981
+ function lifecycleRegistrySubscriptions(input) {
982
+ const subscriptions = input?.subscriptions;
983
+ if (!Array.isArray(subscriptions)) return [];
984
+ return subscriptions.flatMap((entry) => {
985
+ const { workflow, events } = entry ?? {};
986
+ if (typeof workflow !== "string" || !Array.isArray(events)) return [];
987
+ return [
988
+ {
989
+ workflow,
990
+ events: events.filter((e) => typeof e === "string")
991
+ }
992
+ ];
993
+ });
994
+ }
995
+ __name(lifecycleRegistrySubscriptions, "lifecycleRegistrySubscriptions");
996
+ function retentionKeeps(tags) {
997
+ if (carriesLifecyclePending(tags)) return true;
998
+ if (Array.isArray(tags)) return tags.includes(LIFECYCLE_REGISTRY_TAG);
999
+ if (typeof tags === "string") return tags.includes(`"${LIFECYCLE_REGISTRY_TAG}"`);
1000
+ return false;
1001
+ }
1002
+ __name(retentionKeeps, "retentionKeeps");
1003
+
1004
+ // src/run-wait.ts
1005
+ var RUN_WAIT_TOKEN_PREFIX = "runwait:";
1006
+ function runWaitPrefix(targetRunId) {
1007
+ return `${RUN_WAIT_TOKEN_PREFIX}${targetRunId}#`;
1008
+ }
1009
+ __name(runWaitPrefix, "runWaitPrefix");
1010
+ function runWaitToken(targetRunId, runId, seq) {
1011
+ return `${runWaitPrefix(targetRunId)}${runId}#${seq}`;
1012
+ }
1013
+ __name(runWaitToken, "runWaitToken");
1014
+ function runWaitTargetOf(waiter) {
1015
+ const suffix = `#${waiter.runId}#${waiter.seq}`;
1016
+ const { token } = waiter;
1017
+ if (!token.startsWith(RUN_WAIT_TOKEN_PREFIX) || !token.endsWith(suffix)) return void 0;
1018
+ const target = token.slice(RUN_WAIT_TOKEN_PREFIX.length, token.length - suffix.length);
1019
+ return target.length > 0 ? target : void 0;
1020
+ }
1021
+ __name(runWaitTargetOf, "runWaitTargetOf");
1022
+ function nextContinuationId(runId) {
1023
+ const m = runId.match(/^(.*)~(\d+)$/);
1024
+ return m ? `${m[1]}~${Number(m[2]) + 1}` : `${runId}~1`;
1025
+ }
1026
+ __name(nextContinuationId, "nextContinuationId");
1027
+ var MAX_CONTINUATION_HOPS = 100;
1028
+ async function settledRunOf(store, runId) {
1029
+ let run = await store.getRun(runId);
1030
+ if (!run) return {
1031
+ state: "missing"
1032
+ };
1033
+ for (let hop = 0; hop <= MAX_CONTINUATION_HOPS; hop += 1) {
1034
+ if (!lifecycleKindOf(run.status)) return {
1035
+ state: "running"
1036
+ };
1037
+ if (run.status !== "completed") return {
1038
+ state: "settled",
1039
+ run
1040
+ };
1041
+ const next = await store.getRun(nextContinuationId(run.id));
1042
+ if (!next) return {
1043
+ state: "settled",
1044
+ run
1045
+ };
1046
+ run = next;
1047
+ }
1048
+ return {
1049
+ state: "running"
1050
+ };
1051
+ }
1052
+ __name(settledRunOf, "settledRunOf");
816
1053
 
817
1054
  // src/run-waiting.ts
818
1055
  var WEBHOOK_PREFIX = "wh:";
@@ -832,7 +1069,17 @@ function parseWebhookToken(token) {
832
1069
  };
833
1070
  }
834
1071
  __name(parseWebhookToken, "parseWebhookToken");
835
- function classifyWaiterToken(token) {
1072
+ function classifyWaiterToken(token, waiter) {
1073
+ if (token.startsWith(RUN_WAIT_TOKEN_PREFIX)) {
1074
+ const target = waiter ? runWaitTargetOf({
1075
+ token,
1076
+ ...waiter
1077
+ }) : token.slice(RUN_WAIT_TOKEN_PREFIX.length).match(/^(.*)#[^#]*#\d+$/)?.[1];
1078
+ if (target) return {
1079
+ on: "run",
1080
+ name: target
1081
+ };
1082
+ }
836
1083
  if (token.startsWith(WEBHOOK_PREFIX)) return {
837
1084
  on: "webhook",
838
1085
  name: token
@@ -858,7 +1105,7 @@ __name(classifyWaiterToken, "classifyWaiterToken");
858
1105
  function resolveRunWaiting(run, waiterByRun) {
859
1106
  if (run.status !== "suspended") return void 0;
860
1107
  const waiter = waiterByRun.get(run.id);
861
- return waiter ? classifyWaiterToken(waiter.token) : void 0;
1108
+ return waiter ? classifyWaiterToken(waiter.token, waiter) : void 0;
862
1109
  }
863
1110
  __name(resolveRunWaiting, "resolveRunWaiting");
864
1111
  var RUN_ID_BATCH = 500;
@@ -1723,7 +1970,55 @@ var Entities = class {
1723
1970
  };
1724
1971
 
1725
1972
  // src/event-accumulators.ts
1726
- var accToken = /* @__PURE__ */ __name((target) => `__evtacc__:${target}`, "accToken");
1973
+ function accumulatorRunId(target, key) {
1974
+ if (key === void 0) return `__evtacc__:${target}`;
1975
+ const escaped = encodeURIComponent(key).replace(/[~!'()*.]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
1976
+ return `__evtacc__:${target}:${escaped}`;
1977
+ }
1978
+ __name(accumulatorRunId, "accumulatorRunId");
1979
+ var ACCUMULATOR_SEEN_MAX = 1e3;
1980
+ function isRoutedEvent(signal) {
1981
+ return typeof signal === "object" && signal !== null && signal.__evtacc === 1 && typeof signal.id === "string";
1982
+ }
1983
+ __name(isRoutedEvent, "isRoutedEvent");
1984
+ var SeenEvents = class SeenEvents2 {
1985
+ static {
1986
+ __name(this, "SeenEvents");
1987
+ }
1988
+ ids;
1989
+ order;
1990
+ constructor(seen) {
1991
+ this.order = [
1992
+ ...seen ?? []
1993
+ ];
1994
+ this.ids = new Set(this.order);
1995
+ }
1996
+ fresh(signal) {
1997
+ if (!isRoutedEvent(signal)) return {
1998
+ fresh: true,
1999
+ payload: signal
2000
+ };
2001
+ if (this.ids.has(signal.id)) return {
2002
+ fresh: false
2003
+ };
2004
+ this.ids.add(signal.id);
2005
+ this.order.push(signal.id);
2006
+ return {
2007
+ fresh: true,
2008
+ payload: signal.payload
2009
+ };
2010
+ }
2011
+ /** The input to re-arm with: `input` plus the newest ids, when there are any. */
2012
+ rearm(input) {
2013
+ const { seen: _seen, ...rest } = input;
2014
+ const seen = this.order.slice(-ACCUMULATOR_SEEN_MAX);
2015
+ return seen.length > 0 ? {
2016
+ ...rest,
2017
+ seen
2018
+ } : rest;
2019
+ }
2020
+ };
2021
+ var accToken = /* @__PURE__ */ __name((input) => accumulatorRunId(input.target, input.key), "accToken");
1727
2022
  var EventAccumulators = class {
1728
2023
  static {
1729
2024
  __name(this, "EventAccumulators");
@@ -1738,60 +2033,75 @@ var EventAccumulators = class {
1738
2033
  origin: CORE_ORIGIN
1739
2034
  });
1740
2035
  }
1741
- /** Route an `onEvent` payload into its target's debounce/batch accumulator. */
1742
- route(target, config, payload) {
1743
- const [workflow, input] = config.mode === "debounce" ? [
1744
- "__evt_debounce",
1745
- {
1746
- target,
1747
- windowMs: config.windowMs
1748
- }
1749
- ] : [
1750
- "__evt_batch",
1751
- {
1752
- target,
1753
- maxSize: config.maxSize,
1754
- windowMs: config.windowMs
1755
- }
1756
- ];
1757
- return this.host.signalWithStart(workflow, input, accToken(target), {
1758
- token: accToken(target),
2036
+ /**
2037
+ * Route an `onEvent` payload into its target's debounce/batch accumulator: the target's one, or
2038
+ * with a `key`, the one for `String(config.key(payload))`. A `key` that throws rejects the route.
2039
+ * With `id`, the accumulator counts the event once however many times it is routed.
2040
+ */
2041
+ route(target, config, payload, id) {
2042
+ const input = {
2043
+ target,
2044
+ windowMs: config.windowMs,
2045
+ ...config.mode === "batch" ? {
2046
+ maxSize: config.maxSize
2047
+ } : {},
2048
+ ...config.key ? {
2049
+ key: String(config.key(payload))
2050
+ } : {}
2051
+ };
2052
+ const workflow = config.mode === "debounce" ? "__evt_debounce" : "__evt_batch";
2053
+ const token = accToken(input);
2054
+ const signal = id === void 0 ? payload : {
2055
+ __evtacc: 1,
2056
+ id,
1759
2057
  payload
2058
+ };
2059
+ return this.host.signalWithStart(workflow, input, token, {
2060
+ token,
2061
+ payload: signal
1760
2062
  });
1761
2063
  }
1762
2064
  /** Fire the target with the LAST payload once events have been quiet for `windowMs`. */
1763
2065
  async debounce(ctx, input) {
1764
2066
  const { target, windowMs } = input;
1765
- const token = accToken(target);
1766
- let last = await ctx.waitForSignal(token);
2067
+ const token = accToken(input);
2068
+ const seen = new SeenEvents(input.seen);
2069
+ let first = seen.fresh(await ctx.waitForSignal(token));
2070
+ while (!first.fresh) first = seen.fresh(await ctx.waitForSignal(token));
2071
+ let last = first.payload;
1767
2072
  for (; ; ) {
1768
2073
  try {
1769
- last = await ctx.waitForSignal(token, {
2074
+ const next = seen.fresh(await ctx.waitForSignal(token, {
1770
2075
  timeoutMs: windowMs
1771
- });
2076
+ }));
2077
+ if (next.fresh) last = next.payload;
1772
2078
  } catch (e) {
1773
2079
  if (e instanceof SignalTimeoutError) break;
1774
2080
  throw e;
1775
2081
  }
1776
2082
  }
1777
2083
  await ctx.startChild(target, last);
1778
- return ctx.continueAsNew(input);
2084
+ return ctx.continueAsNew(seen.rearm(input));
1779
2085
  }
1780
2086
  /** Fire the target with all payloads (`{ events }`) once `maxSize` or `windowMs` from the first. */
1781
2087
  async batch(ctx, input) {
1782
2088
  const { target, maxSize, windowMs } = input;
1783
- const token = accToken(target);
2089
+ const token = accToken(input);
2090
+ const seen = new SeenEvents(input.seen);
2091
+ let first = seen.fresh(await ctx.waitForSignal(token));
2092
+ while (!first.fresh) first = seen.fresh(await ctx.waitForSignal(token));
1784
2093
  const events = [
1785
- await ctx.waitForSignal(token)
2094
+ first.payload
1786
2095
  ];
1787
2096
  const deadline = await ctx.now() + windowMs;
1788
2097
  while (events.length < maxSize) {
1789
2098
  const remaining = deadline - await ctx.now();
1790
2099
  if (remaining <= 0) break;
1791
2100
  try {
1792
- events.push(await ctx.waitForSignal(token, {
2101
+ const next = seen.fresh(await ctx.waitForSignal(token, {
1793
2102
  timeoutMs: remaining
1794
2103
  }));
2104
+ if (next.fresh) events.push(next.payload);
1795
2105
  } catch (e) {
1796
2106
  if (e instanceof SignalTimeoutError) break;
1797
2107
  throw e;
@@ -1800,7 +2110,7 @@ var EventAccumulators = class {
1800
2110
  await ctx.startChild(target, {
1801
2111
  events
1802
2112
  });
1803
- return ctx.continueAsNew(input);
2113
+ return ctx.continueAsNew(seen.rearm(input));
1804
2114
  }
1805
2115
  };
1806
2116
 
@@ -1871,6 +2181,11 @@ function canonicalizeRegistrations(regs) {
1871
2181
  // Conditional, so a registration that states no routes keeps the hash it had before routes existed.
1872
2182
  ...r.onError !== void 0 ? {
1873
2183
  onError: canonicalRoutes(r.onError)
2184
+ } : {},
2185
+ ...r.subscribes !== void 0 ? {
2186
+ subscribes: [
2187
+ ...new Set(r.subscribes)
2188
+ ].sort()
1874
2189
  } : {}
1875
2190
  }));
1876
2191
  const byJson = /* @__PURE__ */ new Map();
@@ -2100,6 +2415,9 @@ function aggregateAnnouncements(descriptors) {
2100
2415
  origins,
2101
2416
  requires,
2102
2417
  onError,
2418
+ subscribes: [
2419
+ ...new Set(announcements.flatMap((a) => a.subscribes ?? []))
2420
+ ].sort(),
2103
2421
  runtimes: [
2104
2422
  ...new Set(announcements.map((a) => a.runtime))
2105
2423
  ].sort(),
@@ -2134,6 +2452,7 @@ function observedAnnouncements(heartbeats, covered) {
2134
2452
  origins: [],
2135
2453
  requires: [],
2136
2454
  onError: [],
2455
+ subscribes: [],
2137
2456
  runtimes: [],
2138
2457
  instances: [
2139
2458
  ...instances
@@ -4176,6 +4495,71 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
4176
4495
  if (bufferedHit) return bufferedHit.value;
4177
4496
  throw new WorkflowSuspended(deadline);
4178
4497
  }, "waitForEvent");
4498
+ const settleRunWaitFromTarget = /* @__PURE__ */ __name(async (target, token, seq) => {
4499
+ const state = await settledRunOf(store, target);
4500
+ if (state.state === "missing") {
4501
+ await store.removeSignalWaiter({
4502
+ token,
4503
+ runId,
4504
+ seq
4505
+ });
4506
+ throw new FatalError(`run ${target} not found`, "run_not_found");
4507
+ }
4508
+ if (state.state === "running") return null;
4509
+ if (!await store.takeSignalWaiter(token)) return null;
4510
+ const event = lifecycleEventOf(state.run);
4511
+ await writeCheckpoint(instantCheckpoint({
4512
+ runId,
4513
+ seq,
4514
+ name: `signal:${token}`,
4515
+ kind: "signal",
4516
+ output: event
4517
+ }));
4518
+ return {
4519
+ value: event
4520
+ };
4521
+ }, "settleRunWaitFromTarget");
4522
+ const waitForRun = /* @__PURE__ */ __name(async (first, second, third) => {
4523
+ const target = typeof first === "string" ? first : second;
4524
+ const opts = typeof first === "string" ? second : third;
4525
+ if (opts?.timeoutMs == null) {
4526
+ const current = pos.next();
4527
+ const token2 = runWaitToken(target, runId, current);
4528
+ const existing = await readCheckpoint(current);
4529
+ if (existing && existing.status === "completed") return existing.output;
4530
+ await store.putSignalWaiter({
4531
+ token: token2,
4532
+ runId,
4533
+ seq: current
4534
+ });
4535
+ const settled2 = await settleRunWaitFromTarget(target, token2, current);
4536
+ if (settled2) return settled2.value;
4537
+ throw new WorkflowSuspended();
4538
+ }
4539
+ const timeoutMs = parseDuration(opts.timeoutMs);
4540
+ const deadlineSeq = pos.next();
4541
+ const waitSeq = pos.next();
4542
+ const token = runWaitToken(target, runId, waitSeq);
4543
+ const deadline = await stampDeadline(deadlineSeq, `run:${target}`, timeoutMs);
4544
+ const waited = await readCheckpoint(waitSeq);
4545
+ if (waited && waited.status === "completed") return waited.output;
4546
+ if (host.clock() >= deadline) {
4547
+ await store.removeSignalWaiter({
4548
+ token,
4549
+ runId,
4550
+ seq: waitSeq
4551
+ }).catch(() => void 0);
4552
+ throw new SignalTimeoutError(`run:${target}`, timeoutMs);
4553
+ }
4554
+ await store.putSignalWaiter({
4555
+ token,
4556
+ runId,
4557
+ seq: waitSeq
4558
+ });
4559
+ const settled = await settleRunWaitFromTarget(target, token, waitSeq);
4560
+ if (settled) return settled.value;
4561
+ throw new WorkflowSuspended(deadline);
4562
+ }, "waitForRun");
4179
4563
  const task = /* @__PURE__ */ __name(async (name, dispatch, options) => {
4180
4564
  await localStep(`task:dispatch:${name}`, dispatch, options);
4181
4565
  return unwrapCompletion(await waitForSignal(`task:${runId}:${name}`), `task "${name}"`);
@@ -4841,6 +5225,7 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
4841
5225
  continueAsNew,
4842
5226
  waitForSignal: guard("waitForSignal", waitForSignal),
4843
5227
  waitForEvent: guard("waitForEvent", waitForEvent),
5228
+ waitForRun: guard("waitForRun", waitForRun),
4844
5229
  task: guard("task", task),
4845
5230
  child: guard("child", child),
4846
5231
  all: guard("all", all),
@@ -4871,6 +5256,28 @@ __name(mergeTags, "mergeTags");
4871
5256
  var BREAKPOINT = "breakpoint";
4872
5257
  var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
4873
5258
  var ANNOUNCED_ROUTES_TTL_MS = 5e3;
5259
+ function servedNamespace(namespace, group) {
5260
+ if (namespace) return namespace;
5261
+ const at = group?.lastIndexOf("@") ?? -1;
5262
+ return group && at >= 0 && at < group.length - 1 ? group.slice(at + 1) : "default";
5263
+ }
5264
+ __name(servedNamespace, "servedNamespace");
5265
+ function announcedIn(snapshot, event, namespace, nodeOnly = false) {
5266
+ const scope = namespace === null ? null : runNamespace(namespace);
5267
+ const subscribers = /* @__PURE__ */ new Set();
5268
+ for (const entry of snapshot.entries) {
5269
+ if (entry.event !== event || nodeOnly && !entry.node) continue;
5270
+ if (scope === null || entry.namespace === null || entry.namespace === scope) {
5271
+ subscribers.add(entry.workflow);
5272
+ }
5273
+ }
5274
+ return subscribers;
5275
+ }
5276
+ __name(announcedIn, "announcedIn");
5277
+ function runNamespace(namespace) {
5278
+ return namespace || "default";
5279
+ }
5280
+ __name(runNamespace, "runNamespace");
4874
5281
  var VERSION_UNDECLARED_TAG = "version:undeclared";
4875
5282
  var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
4876
5283
  var STEP_FAN_CHUNK = 200;
@@ -4967,15 +5374,10 @@ function shuffle(items) {
4967
5374
  return items;
4968
5375
  }
4969
5376
  __name(shuffle, "shuffle");
4970
- function stepGroup(step, runNamespace) {
4971
- return tenantGroup(sanitizeQueueToken(step.name), step.partition ?? runNamespace);
5377
+ function stepGroup(step, runNamespace2) {
5378
+ return tenantGroup(sanitizeQueueToken(step.name), step.partition ?? runNamespace2);
4972
5379
  }
4973
5380
  __name(stepGroup, "stepGroup");
4974
- function nextContinuationId(runId) {
4975
- const m = runId.match(/^(.*)~(\d+)$/);
4976
- return m ? `${m[1]}~${Number(m[2]) + 1}` : `${runId}~1`;
4977
- }
4978
- __name(nextContinuationId, "nextContinuationId");
4979
5381
  function isNewerVersion(a, b) {
4980
5382
  const na = Number(a);
4981
5383
  const nb = Number(b);
@@ -5025,7 +5427,7 @@ var UNREGISTERED_DEAD_LETTER_FROM = [
5025
5427
  "cancelling",
5026
5428
  "blocked"
5027
5429
  ];
5028
- var WorkflowEngine = class {
5430
+ var WorkflowEngine = class _WorkflowEngine {
5029
5431
  static {
5030
5432
  __name(this, "WorkflowEngine");
5031
5433
  }
@@ -5077,7 +5479,7 @@ var WorkflowEngine = class {
5077
5479
  workflows = /* @__PURE__ */ new Map();
5078
5480
  /** The newest registered version per workflow name — used to `start` new runs. */
5079
5481
  latest = /* @__PURE__ */ new Map();
5080
- /** Event name → workflow names started when that event is published (see `onEvent`). */
5482
+ /** Event name → subscribing workflow name → that workflow's subscriptions to it. See `onEvent`. */
5081
5483
  eventTriggers = /* @__PURE__ */ new Map();
5082
5484
  /** Durable-entity subsystem (registers the `__entity` runner; see `registerEntity`). */
5083
5485
  entities;
@@ -5723,10 +6125,14 @@ var WorkflowEngine = class {
5723
6125
  this.workflows.set(versionKey(name, version), registered);
5724
6126
  const current = this.latest.get(name);
5725
6127
  if (!current || isNewerVersion(version, current.version)) this.latest.set(name, registered);
5726
- for (const event of opts?.onEvent ?? []) {
5727
- const subscribers = this.eventTriggers.get(event) ?? /* @__PURE__ */ new Set();
5728
- subscribers.add(name);
5729
- this.eventTriggers.set(event, subscribers);
6128
+ for (const entry of opts?.onEvent ?? []) {
6129
+ const subscription = toEventSubscription(entry);
6130
+ const subscribers = this.eventTriggers.get(subscription.event) ?? /* @__PURE__ */ new Map();
6131
+ subscribers.set(name, [
6132
+ ...subscribers.get(name) ?? [],
6133
+ subscription
6134
+ ]);
6135
+ this.eventTriggers.set(subscription.event, subscribers);
5730
6136
  }
5731
6137
  }
5732
6138
  /**
@@ -5864,136 +6270,1334 @@ var WorkflowEngine = class {
5864
6270
  return () => this.deadListeners.delete(listener);
5865
6271
  }
5866
6272
  /**
5867
- * Delete buffered child completions no parent will ever consume (see
5868
- * `childCompletionNeedsBuffer`): those of root runs, of children whose parent already settled them
5869
- * or is gone, or terminal for good. Such rows were written by earlier versions, which buffered every
5870
- * completion that found no waiter, and nothing else ever removes them.
6273
+ * Delete buffered child completions no parent will ever consume (see
6274
+ * `childCompletionNeedsBuffer`): those of root runs, of children whose parent already settled them
6275
+ * or is gone, or terminal for good. Such rows were written by earlier versions, which buffered every
6276
+ * completion that found no waiter, and nothing else ever removes them.
6277
+ *
6278
+ * Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
6279
+ * at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
6280
+ * few timer ticks without a long query or a big delete. Each page is decided set-based (see
6281
+ * `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
6282
+ * read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
6283
+ * {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
6284
+ * call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
6285
+ * before the next one. Call it every tick (the NestJS timer poller does); a call made while
6286
+ * another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
6287
+ * carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
6288
+ * store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
6289
+ */
6290
+ async sweepBufferedSignals(now = this.clock(), budgetMs = BUFFERED_SWEEP_BUDGET_MS) {
6291
+ const { listBufferedSignals, deleteBufferedSignals } = this.store;
6292
+ if (!listBufferedSignals || !deleteBufferedSignals) return 0;
6293
+ if (this.draining || this.namespace !== void 0 || this.bufferedSweepActive) return 0;
6294
+ if (this.bufferedSweepCursor === void 0 && now - this.lastBufferedSweepEndedAt < BUFFERED_SWEEP_REST_MS) {
6295
+ return 0;
6296
+ }
6297
+ this.bufferedSweepActive = true;
6298
+ const deadline = Date.now() + budgetMs;
6299
+ let deleted = 0;
6300
+ try {
6301
+ for (let page = 0; page < BUFFERED_SWEEP_PAGES_PER_CALL; page++) {
6302
+ if (page > 0 && (Date.now() >= deadline || this.draining)) break;
6303
+ const rows = await listBufferedSignals.call(this.store, CHILD_TOKEN_PREFIX, this.bufferedSweepCursor, BUFFERED_SWEEP_PAGE);
6304
+ const { doomed, decided } = await unneededChildSignals(this.store, rows, deadline);
6305
+ if (doomed.length > 0) {
6306
+ await deleteBufferedSignals.call(this.store, doomed.map((row) => row.id));
6307
+ deleted += doomed.length;
6308
+ }
6309
+ if (decided < rows.length) {
6310
+ if (decided > 0) this.bufferedSweepCursor = rows[decided - 1].id;
6311
+ break;
6312
+ }
6313
+ const last = rows.at(-1);
6314
+ if (rows.length < BUFFERED_SWEEP_PAGE || !last) {
6315
+ this.bufferedSweepCursor = void 0;
6316
+ this.lastBufferedSweepEndedAt = now;
6317
+ break;
6318
+ }
6319
+ this.bufferedSweepCursor = last.id;
6320
+ }
6321
+ } finally {
6322
+ this.bufferedSweepActive = false;
6323
+ }
6324
+ return deleted;
6325
+ }
6326
+ /**
6327
+ * Register a pager hook for STRANDED runs: an in-flight run untouched past `stalledAfterMs` that is
6328
+ * either suspended with NO wake timer (wake-forever: nothing will ever re-drive it) or parked on an
6329
+ * old pending remote step with no live lease (dispatched and, most likely, lost). The timer poller
6330
+ * sweeps for these ({@link sweepStalled}, throttled to one pass a minute) and notifies each episode
6331
+ * ONCE per instance: a run is paged again only after it makes progress and strands again. Wire your
6332
+ * alerting here like {@link onDead}. A listener that throws never breaks the sweep. Returns an
6333
+ * unsubscribe function.
6334
+ */
6335
+ onStalled(listener) {
6336
+ this.stalledListeners.add(listener);
6337
+ return () => this.stalledListeners.delete(listener);
6338
+ }
6339
+ /**
6340
+ * Detect stranded in-flight runs and notify the {@link onStalled} listeners. Self-throttled to one
6341
+ * pass a minute, skipped while draining, and a no-op without listeners — call it every tick (the
6342
+ * NestJS timer poller does). Scoped to this engine's `namespace` (an operator engine with no
6343
+ * namespace sweeps every namespace). Returns the runs notified THIS pass.
6344
+ */
6345
+ async sweepStalled(now = this.clock()) {
6346
+ if (this.stalledListeners.size === 0 || this.draining) return [];
6347
+ if (now - this.lastStalledSweepAt < 6e4) return [];
6348
+ this.lastStalledSweepAt = now;
6349
+ const notified = [];
6350
+ for (const info2 of await this.listStalledRuns({
6351
+ now
6352
+ })) {
6353
+ const episode = info2.run.updatedAt.getTime();
6354
+ if (this.notifiedStalled.get(info2.run.id) === episode) continue;
6355
+ this.notifiedStalled.set(info2.run.id, episode);
6356
+ if (this.notifiedStalled.size > 1e3) this.notifiedStalled.clear();
6357
+ for (const listener of this.stalledListeners) {
6358
+ try {
6359
+ listener(info2);
6360
+ } catch {
6361
+ }
6362
+ }
6363
+ notified.push(info2);
6364
+ }
6365
+ return notified;
6366
+ }
6367
+ /**
6368
+ * The stranded runs right now, without notifying anyone: the query behind {@link sweepStalled} and
6369
+ * the CLI's `durable inspect --stale`. A `running`/`suspended` run qualifies when it has been
6370
+ * untouched for `thresholdMs` (default `stalledAfterMs`) AND either it is suspended with no wake
6371
+ * timer, or its oldest pending remote step is older than the threshold with no lease in the future
6372
+ * (a step with a `timeoutMs`, a heartbeat or `remoteRedispatchMs` keeps a lease; a long
6373
+ * `ctx.sleep` has a future `wakeAt`; neither is stranded). Scans up to `scan` candidates (default
6374
+ * 1000), newest first, in this engine's namespace.
6375
+ */
6376
+ async listStalledRuns(options = {}) {
6377
+ return findStalledRuns(this.store, {
6378
+ now: options.now ?? this.clock(),
6379
+ thresholdMs: options.thresholdMs ?? this.stalledAfterMs,
6380
+ scan: options.scan,
6381
+ namespace: this.namespace
6382
+ });
6383
+ }
6384
+ notifyDead(run) {
6385
+ for (const fn of this.deadListeners) {
6386
+ try {
6387
+ fn(run);
6388
+ } catch {
6389
+ }
6390
+ }
6391
+ }
6392
+ /**
6393
+ * Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the control
6394
+ * plane so other instances (e.g. a dashboard pod) can live-tail this run. A `run.completed` or
6395
+ * `run.failed` also publishes the run's lifecycle event (see {@link publishLifecycle}) and resumes
6396
+ * the runs waiting on it with `ctx.waitForRun` (see {@link deliverRunWaits}); `lifecycle: false`
6397
+ * marks a `run.completed` that is not a real completion (a continue-as-new handoff).
6398
+ */
6399
+ emit(event, opts) {
6400
+ const full = {
6401
+ ...event,
6402
+ at: /* @__PURE__ */ new Date()
6403
+ };
6404
+ this.deliver(full);
6405
+ if (opts?.lifecycle !== false) {
6406
+ this.publishLifecycle(full);
6407
+ if ((full.type === "run.completed" || full.type === "run.failed") && !full.workflow?.startsWith("__")) {
6408
+ void this.deliverRunWaits(full.runId).catch(() => void 0);
6409
+ }
6410
+ }
6411
+ if (!this.controlPlane) return;
6412
+ this.eventsToBroadcast.push(full);
6413
+ if (!this.broadcastingEvents) void this.broadcastEvents();
6414
+ }
6415
+ /**
6416
+ * Start every workflow subscribed to `name` that takes `payload` (see {@link acceptingSubscription}),
6417
+ * each under `runIdOf(workflow)` and with its subscription's `input` applied. `only` restricts it
6418
+ * to those subscribers. `remote` are subscribers only live workers announce (see
6419
+ * {@link announcedRemoteSubscribers}): each is started by name with the payload as its input, since
6420
+ * an announcement carries no `where` or `input`. `routeIdOf` names the event for a subscriber's
6421
+ * debounce/batch accumulator, which counts an id once. Returns the subscribers reached, with the
6422
+ * run each started (none when the payload went into its debounce/batch accumulator), and the ones
6423
+ * that could not be started, with why.
6424
+ */
6425
+ async startSubscribers(name, payload, runIdOf, only, remote = [], routeIdOf) {
6426
+ const reached = [];
6427
+ const failed = [];
6428
+ for (const [workflow, subscriptions] of this.eventTriggers.get(name) ?? []) {
6429
+ if (only && !only.has(workflow)) continue;
6430
+ const { accepted, errors } = acceptingSubscription(subscriptions, payload);
6431
+ if (!accepted) {
6432
+ if (errors.length > 0) failed.push({
6433
+ workflow,
6434
+ error: errors[0]
6435
+ });
6436
+ continue;
6437
+ }
6438
+ try {
6439
+ const input = accepted.input ? accepted.input(payload) : payload;
6440
+ const batch = this.latest.get(workflow)?.eventBatch;
6441
+ if (batch) {
6442
+ await this.accumulators.route(workflow, batch, input, routeIdOf?.(workflow));
6443
+ reached.push({
6444
+ workflow
6445
+ });
6446
+ } else {
6447
+ const runId = runIdOf(workflow);
6448
+ await this.start(workflow, input, runId);
6449
+ reached.push({
6450
+ workflow,
6451
+ runId
6452
+ });
6453
+ }
6454
+ } catch (error2) {
6455
+ failed.push({
6456
+ workflow,
6457
+ error: error2
6458
+ });
6459
+ }
6460
+ }
6461
+ const local = this.eventTriggers.get(name);
6462
+ for (const workflow of remote) {
6463
+ if (local?.has(workflow) || only && !only.has(workflow)) continue;
6464
+ try {
6465
+ const runId = runIdOf(workflow);
6466
+ await this.start(workflow, payload, runId);
6467
+ reached.push({
6468
+ workflow,
6469
+ runId
6470
+ });
6471
+ } catch (error2) {
6472
+ failed.push({
6473
+ workflow,
6474
+ error: error2
6475
+ });
6476
+ }
6477
+ }
6478
+ return {
6479
+ reached,
6480
+ failed
6481
+ };
6482
+ }
6483
+ /**
6484
+ * The subscribers of `event` that live workers announce for the settled runs of `namespace` (`null`:
6485
+ * any namespace, for an event no run published), are not registered on this engine with an `onEvent` for it, and that this engine can start: registered here (a `registerRemote`), or served
6486
+ * by convention on a live worker group of the same name (a Python `@workflow(on_event=...)`). One
6487
+ * this engine cannot start is left to the instances that can, so it stays owed rather than failing.
6488
+ */
6489
+ async announcedRemoteSubscribers(event, namespace) {
6490
+ const snapshot = await this.announcedSubscriptions();
6491
+ const announced = announcedIn(snapshot, event, namespace);
6492
+ if (!announced.size) return [];
6493
+ const local = new Set(this.localSubscribersIn(event, namespace));
6494
+ const byEngine = announcedIn(snapshot, event, namespace, true);
6495
+ const startable = [];
6496
+ for (const workflow of [
6497
+ ...announced
6498
+ ].sort()) {
6499
+ if (local.has(workflow) || byEngine.has(workflow) || workflow.startsWith("__")) continue;
6500
+ if (this.latest.has(workflow)) {
6501
+ startable.push(workflow);
6502
+ continue;
6503
+ }
6504
+ const group = tenantGroup(sanitizeQueueToken(workflow), this.namespace);
6505
+ const live = await this.pool.transportWithLiveGroup(group).catch(() => void 0);
6506
+ if (live) startable.push(workflow);
6507
+ }
6508
+ return startable;
6509
+ }
6510
+ /** The workflows registered on this engine that subscribe to `event`. */
6511
+ localSubscribers(event) {
6512
+ return [
6513
+ ...this.eventTriggers.get(event)?.keys() ?? []
6514
+ ];
6515
+ }
6516
+ /**
6517
+ * Whether some workflow may subscribe to `workflow` settling as `kind`: one registered on this
6518
+ * engine, or one in the last read of the fleet's announcements. True while there is no usable read
6519
+ * (none yet, or the last one failed), since the terminal write then marks the run unresolved (see
6520
+ * {@link withLifecycleMarker}).
6521
+ */
6522
+ hasLifecycleSubscribers(workflow, kind, namespace) {
6523
+ if (workflow.startsWith("__")) return false;
6524
+ const event = lifecycleEventName(workflow, kind);
6525
+ if (this.servesNamespace(namespace) && this.localSubscribers(event).length > 0) return true;
6526
+ const snapshot = this.announcedSubscriptionsSnapshot;
6527
+ if (!snapshot?.ok) return true;
6528
+ return announcedIn(snapshot, event, namespace).size > 0;
6529
+ }
6530
+ /**
6531
+ * Whether this engine's own subscribers apply to the settled runs of `namespace`: an engine with
6532
+ * no namespace publishes the runs of every namespace, a namespaced one only its own.
6533
+ */
6534
+ servesNamespace(namespace) {
6535
+ return namespace === null || this.namespace === void 0 || this.namespace === runNamespace(namespace);
6536
+ }
6537
+ /** This engine's subscribers of `event` that apply to the settled runs of `namespace`. */
6538
+ localSubscribersIn(event, namespace) {
6539
+ return this.servesNamespace(namespace) ? this.localSubscribers(event) : [];
6540
+ }
6541
+ /** Answers the traceparent of a run's span in this process; see {@link setRunTraceparent}. */
6542
+ runTraceparent;
6543
+ /**
6544
+ * Install the function that answers the W3C `traceparent` of a run's span in this process, or
6545
+ * `undefined` to remove it. `attachDurableOtel` installs one. With it, the first suspend or the
6546
+ * terminal write of a run some workflow subscribes to also records the run's traceparent (a
6547
+ * {@link LIFECYCLE_TRACE_TAG_PREFIX} tag, see {@link withRunTrace}), and the `run.started` of a run
6548
+ * a lifecycle event started carries its source run's as `sourceTraceparent`, so the subscriber's
6549
+ * trace links to the source's.
6550
+ */
6551
+ setRunTraceparent(provider) {
6552
+ this.runTraceparent = provider;
6553
+ }
6554
+ /**
6555
+ * `{ sourceTraceparent }` for the `run.started` of `run`, when a lifecycle event started it and a
6556
+ * run trace provider is installed: the source run's recorded traceparent, else this process's.
6557
+ */
6558
+ async sourceTraceOf(run) {
6559
+ const provider = this.runTraceparent;
6560
+ if (!provider) return {};
6561
+ const source = lifecycleSourceOf(run);
6562
+ if (!source) return {};
6563
+ try {
6564
+ const recorded = lifecycleTraceparentOf((await this.store.getRun(source.runId))?.tags);
6565
+ const traceparent = recorded ?? provider(source.runId);
6566
+ return traceparent ? {
6567
+ sourceTraceparent: traceparent
6568
+ } : {};
6569
+ } catch {
6570
+ return {};
6571
+ }
6572
+ }
6573
+ /** The last read of the subscriptions live workers announce. See {@link announcedSubscriptions}. */
6574
+ announcedSubscriptionsSnapshot;
6575
+ announcedSubscriptionsRead;
6576
+ /**
6577
+ * The `onEvent` subscriptions the live fleet announces, by event name: the fresh lifecycle
6578
+ * registry runs in the state store (see {@link heartbeatLifecycleRegistry}) and, on a transport
6579
+ * that reads them, each worker descriptor's `subscribes`. Each entry records the namespace its
6580
+ * announcer serves (`null` for an engine with no namespace) and whether a Node engine announced it
6581
+ * (every registry run is one). Read at most once per
6582
+ * {@link ANNOUNCED_ROUTES_TTL_MS}, concurrent callers sharing one read. Never throws: `ok: false`
6583
+ * means a read failed and only the local registry is known.
6584
+ */
6585
+ async announcedSubscriptions() {
6586
+ const cached = this.announcedSubscriptionsSnapshot;
6587
+ if (cached && this.clock() - cached.at <= ANNOUNCED_ROUTES_TTL_MS) return cached;
6588
+ this.announcedSubscriptionsRead ??= this.readAnnouncedSubscriptions().finally(() => {
6589
+ this.announcedSubscriptionsRead = void 0;
6590
+ });
6591
+ return this.announcedSubscriptionsRead;
6592
+ }
6593
+ /**
6594
+ * The last read of the fleet's announcements as it is, without waiting: undefined before the first
6595
+ * read completes. Starts a read in the background when it is missing or older than
6596
+ * {@link ANNOUNCED_ROUTES_TTL_MS}. What the terminal write uses, so settling a run never waits on
6597
+ * the fleet.
6598
+ */
6599
+ cachedAnnouncedSubscriptions() {
6600
+ const cached = this.announcedSubscriptionsSnapshot;
6601
+ if (!cached || this.clock() - cached.at > ANNOUNCED_ROUTES_TTL_MS) {
6602
+ void this.announcedSubscriptions();
6603
+ }
6604
+ return cached;
6605
+ }
6606
+ /**
6607
+ * When this engine first saw each subscription (`<event>\u0000<subscriber>`), in its registry or in
6608
+ * a good read of the fleet's announcements. The sweep's heal pass looks only at the ones first seen
6609
+ * within {@link LIFECYCLE_HEAL_WINDOW_MS}; see {@link healLifecycleEvents}.
6610
+ */
6611
+ subscriptionFirstSeen = /* @__PURE__ */ new Map();
6612
+ /** Record the subscriptions of `snapshot` (a good read) and of this engine's registry as seen. */
6613
+ noteSubscriptionsSeen(snapshot) {
6614
+ const now = this.clock();
6615
+ const note = /* @__PURE__ */ __name((event, subscriber) => {
6616
+ const key = `${event}\0${subscriber}`;
6617
+ if (!this.subscriptionFirstSeen.has(key)) this.subscriptionFirstSeen.set(key, now);
6618
+ }, "note");
6619
+ for (const [event, subscribers] of snapshot.byEvent) {
6620
+ for (const subscriber of subscribers) note(event, subscriber);
6621
+ }
6622
+ for (const [event, subscribers] of this.eventTriggers) {
6623
+ for (const subscriber of subscribers.keys()) note(event, subscriber);
6624
+ }
6625
+ }
6626
+ async readAnnouncedSubscriptions() {
6627
+ const byEvent = /* @__PURE__ */ new Map();
6628
+ const entries = [];
6629
+ let ok = true;
6630
+ const add = /* @__PURE__ */ __name((event, workflow, node, namespace) => {
6631
+ entries.push({
6632
+ event,
6633
+ workflow,
6634
+ namespace,
6635
+ node
6636
+ });
6637
+ const subscribers = byEvent.get(event) ?? /* @__PURE__ */ new Set();
6638
+ subscribers.add(workflow);
6639
+ byEvent.set(event, subscribers);
6640
+ }, "add");
6641
+ try {
6642
+ const now = this.clock();
6643
+ for (const row of await this.listLifecycleRegistry()) {
6644
+ if (now - row.updatedAt.getTime() > LIFECYCLE_REGISTRY_TTL_MS) continue;
6645
+ const namespace = lifecycleRegistryNamespace(row.input, row.namespace);
6646
+ for (const { workflow, events } of lifecycleRegistrySubscriptions(row.input)) {
6647
+ for (const event of events) add(event, workflow, true, namespace);
6648
+ }
6649
+ }
6650
+ if (this.pool.introspectsFleet) {
6651
+ for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
6652
+ for (const announced of announcementsOf(descriptor)) {
6653
+ const namespace = servedNamespace(descriptor.namespace, announced.group);
6654
+ for (const event of announced.subscribes ?? []) {
6655
+ add(event, announced.name, descriptor.runtime === "node", namespace);
6656
+ }
6657
+ }
6658
+ }
6659
+ }
6660
+ } catch {
6661
+ ok = false;
6662
+ byEvent.clear();
6663
+ entries.length = 0;
6664
+ }
6665
+ const snapshot = {
6666
+ at: this.clock(),
6667
+ ok,
6668
+ byEvent,
6669
+ entries
6670
+ };
6671
+ this.announcedSubscriptionsSnapshot = snapshot;
6672
+ if (ok) this.noteSubscriptionsSeen(snapshot);
6673
+ return snapshot;
6674
+ }
6675
+ /** How many registry runs one page of {@link listLifecycleRegistry} reads. */
6676
+ static LIFECYCLE_REGISTRY_PAGE = 1e3;
6677
+ /**
6678
+ * The lifecycle registry runs this engine's namespace sees: its namespace's, plus `default`'s, where
6679
+ * the engines with no namespace keep theirs. Read a page of
6680
+ * {@link LIFECYCLE_REGISTRY_PAGE} at a time until a short page, so a large fleet is read whole.
6681
+ */
6682
+ async listLifecycleRegistry() {
6683
+ const pageSize = _WorkflowEngine.LIFECYCLE_REGISTRY_PAGE;
6684
+ const rows = [];
6685
+ for (; ; ) {
6686
+ const page = await this.store.listRuns({
6687
+ workflow: LIFECYCLE_REGISTRY_WORKFLOW,
6688
+ ...this.namespace !== void 0 ? {
6689
+ namespaces: [
6690
+ .../* @__PURE__ */ new Set([
6691
+ this.namespace,
6692
+ "default"
6693
+ ])
6694
+ ]
6695
+ } : {},
6696
+ limit: pageSize,
6697
+ offset: rows.length
6698
+ });
6699
+ rows.push(...page);
6700
+ if (page.length < pageSize) return rows;
6701
+ }
6702
+ }
6703
+ /** When this engine last wrote its registry run, or undefined before the first heartbeat. */
6704
+ lifecycleRegistryBeatAt;
6705
+ /**
6706
+ * Refresh this engine's lifecycle registry run and delete stale ones. At most once per
6707
+ * {@link LIFECYCLE_REGISTRY_HEARTBEAT_MS}; {@link sweepLifecycleEvents} calls it on every tick.
6708
+ *
6709
+ * An engine with `onEvent` subscriptions keeps one completed run of the internal
6710
+ * {@link LIFECYCLE_REGISTRY_WORKFLOW} workflow, id `__lifecycle_registry:<instanceId>`, holding
6711
+ * them in `input` and its heartbeat in `updatedAt`. Other instances read the fresh ones
6712
+ * ({@link LIFECYCLE_REGISTRY_TTL_MS}) as announced subscriptions, so a settled run is owed to a
6713
+ * subscriber registered only elsewhere whatever the transport. Any engine deletes the runs not
6714
+ * refreshed for {@link LIFECYCLE_REGISTRY_STALE_MS}. Retention never prunes them
6715
+ * ({@link LIFECYCLE_REGISTRY_TAG}), and the run gateway leaves them out of run lists.
6716
+ */
6717
+ async heartbeatLifecycleRegistry() {
6718
+ const now = this.clock();
6719
+ const last = this.lifecycleRegistryBeatAt;
6720
+ if (last !== void 0 && now - last < LIFECYCLE_REGISTRY_HEARTBEAT_MS) return;
6721
+ this.lifecycleRegistryBeatAt = now;
6722
+ try {
6723
+ const subscriptions = this.registrySubscriptions();
6724
+ if (subscriptions.length > 0) await this.writeLifecycleRegistry(subscriptions, now);
6725
+ const stale = (await this.listLifecycleRegistry()).filter((row) => now - row.updatedAt.getTime() > LIFECYCLE_REGISTRY_STALE_MS);
6726
+ for (const row of stale) await this.store.deleteRun(row.id);
6727
+ } catch (error2) {
6728
+ this.lifecycleRegistryBeatAt = last;
6729
+ throw error2;
6730
+ }
6731
+ }
6732
+ /** This engine's `onEvent` subscriptions, by subscribing workflow, as its registry run holds them. */
6733
+ registrySubscriptions() {
6734
+ const byWorkflow = /* @__PURE__ */ new Map();
6735
+ for (const [event, subscribers] of this.eventTriggers) {
6736
+ for (const workflow of subscribers.keys()) {
6737
+ if (workflow.startsWith("__")) continue;
6738
+ const events = byWorkflow.get(workflow) ?? /* @__PURE__ */ new Set();
6739
+ events.add(event);
6740
+ byWorkflow.set(workflow, events);
6741
+ }
6742
+ }
6743
+ return [
6744
+ ...byWorkflow
6745
+ ].sort(([a], [b]) => a.localeCompare(b)).map(([workflow, events]) => ({
6746
+ workflow,
6747
+ events: [
6748
+ ...events
6749
+ ].sort()
6750
+ }));
6751
+ }
6752
+ async writeLifecycleRegistry(subscriptions, nowMs) {
6753
+ const id = lifecycleRegistryRunId(this.instanceId);
6754
+ const input = {
6755
+ instanceId: this.instanceId,
6756
+ runtime: "node",
6757
+ engineNamespace: this.namespace ?? null,
6758
+ subscriptions
6759
+ };
6760
+ const at = new Date(nowMs);
6761
+ const refreshed = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, id, [
6762
+ "completed"
6763
+ ], {
6764
+ input,
6765
+ updatedAt: at
6766
+ });
6767
+ if (refreshed) return;
6768
+ await this.store.createRun({
6769
+ id,
6770
+ workflow: LIFECYCLE_REGISTRY_WORKFLOW,
6771
+ workflowVersion: "1",
6772
+ status: "completed",
6773
+ input,
6774
+ tags: [
6775
+ LIFECYCLE_REGISTRY_TAG
6776
+ ],
6777
+ ...this.namespace !== void 0 ? {
6778
+ namespace: this.namespace
6779
+ } : {},
6780
+ origin: CORE_ORIGIN,
6781
+ createdAt: at,
6782
+ updatedAt: at
6783
+ });
6784
+ }
6785
+ /**
6786
+ * The subscribers a run of `workflow` in `namespace` settling as `kind` owes its lifecycle event to:
6787
+ * the ones registered on this engine, when it publishes that namespace's runs (an engine with no
6788
+ * namespace publishes every namespace's), plus the ones live workers announce for that namespace
6789
+ * (see {@link announcedSubscriptions}): by announcers serving it, and by engines with no namespace.
6790
+ * Without `namespace`, every namespace's: what any run of `workflow` could owe. Empty for an
6791
+ * internal `__` workflow.
6792
+ */
6793
+ async lifecycleSubscribers(workflow, kind, namespace) {
6794
+ return this.lifecycleSubscribersIn(await this.announcedSubscriptions(), workflow, kind, namespace);
6795
+ }
6796
+ /** {@link lifecycleSubscribers} from one given read of the fleet's announcements. */
6797
+ lifecycleSubscribersIn(snapshot, workflow, kind, namespace) {
6798
+ if (workflow.startsWith("__")) return [];
6799
+ const scope = namespace === void 0 ? null : runNamespace(namespace);
6800
+ const event = lifecycleEventName(workflow, kind);
6801
+ const announced = announcedIn(snapshot, event, scope);
6802
+ return [
6803
+ .../* @__PURE__ */ new Set([
6804
+ ...this.localSubscribersIn(event, scope),
6805
+ ...announced
6806
+ ])
6807
+ ].sort();
6808
+ }
6809
+ /**
6810
+ * The subscribers registered here that coalesce `run`'s lifecycle event (debounce/batch) and
6811
+ * accept it, each with the accumulator run the event went to: the subscription's `where`, `input`
6812
+ * and `key` applied to the event as `startSubscribers` does. Undefined when the key cannot be
6813
+ * computed. A subscriber whose `where` declines the event is left out, and so is every one when
6814
+ * this engine does not publish the run's namespace. Empty for a run that has not settled.
6815
+ */
6816
+ coalescedSubscribers(run) {
6817
+ const out = /* @__PURE__ */ new Map();
6818
+ const kind = lifecycleKindOf(run.status);
6819
+ if (!kind) return out;
6820
+ const triggers = this.eventTriggers.get(lifecycleEventName(run.workflow, kind));
6821
+ if (!triggers || !this.servesNamespace(run.namespace)) return out;
6822
+ let payload;
6823
+ for (const [workflow, subscriptions] of triggers) {
6824
+ const batch = this.latest.get(workflow)?.eventBatch;
6825
+ if (!batch) continue;
6826
+ if (!batch.key && subscriptions.every((sub2) => !sub2.where)) {
6827
+ out.set(workflow, accumulatorRunId(workflow));
6828
+ continue;
6829
+ }
6830
+ try {
6831
+ payload ??= lifecycleEventOf(run);
6832
+ const { accepted } = acceptingSubscription(subscriptions, payload);
6833
+ if (!accepted) continue;
6834
+ const input = accepted.input ? accepted.input(payload) : payload;
6835
+ out.set(workflow, accumulatorRunId(workflow, batch.key ? String(batch.key(input)) : void 0));
6836
+ } catch {
6837
+ out.set(workflow, void 0);
6838
+ }
6839
+ }
6840
+ return out;
6841
+ }
6842
+ /**
6843
+ * Who starts whom through lifecycle events, for `workflow`: the workflows that start when it
6844
+ * settles, and the workflows whose settling starts it. Built from this engine's registry and the
6845
+ * subscriptions live workers announce (see {@link announcedSubscriptions}); a failed fleet read
6846
+ * leaves the local registry alone. What a dashboard's workflow page shows.
6847
+ *
6848
+ * With `namespace`, only the subscriptions of that namespace: the ones announced by registry runs
6849
+ * and worker descriptors serving it, and this engine's own when it is that namespace's engine (an
6850
+ * engine with no namespace is `default`'s). What a tenant is shown.
6851
+ */
6852
+ async workflowLifecycle(workflow, opts) {
6853
+ const announced = await this.announcedSubscriptions();
6854
+ const scope = opts?.namespace;
6855
+ let fleet = announced.byEvent;
6856
+ if (scope !== void 0) {
6857
+ fleet = /* @__PURE__ */ new Map();
6858
+ for (const entry of announced.entries) {
6859
+ if ((entry.namespace ?? "default") !== scope) continue;
6860
+ const subscribers = fleet.get(entry.event) ?? /* @__PURE__ */ new Set();
6861
+ subscribers.add(entry.workflow);
6862
+ fleet.set(entry.event, subscribers);
6863
+ }
6864
+ }
6865
+ const triggers = scope === void 0 || (this.namespace || "default") === scope ? this.eventTriggers : /* @__PURE__ */ new Map();
6866
+ const startsOnSettle = /* @__PURE__ */ new Map();
6867
+ const startedBy = /* @__PURE__ */ new Map();
6868
+ const link = /* @__PURE__ */ __name((into, other, kind, local) => {
6869
+ const key = `${other}\0${kind}`;
6870
+ const entry = into.get(key) ?? {
6871
+ workflow: other,
6872
+ kind,
6873
+ local: false,
6874
+ announced: false,
6875
+ where: false,
6876
+ input: false
6877
+ };
6878
+ if (local) {
6879
+ entry.local = true;
6880
+ entry.where ||= local.subscriptions.some((sub2) => sub2.where !== void 0);
6881
+ entry.input ||= local.subscriptions.some((sub2) => sub2.input !== void 0);
6882
+ const batch = this.latest.get(local.subscriber)?.eventBatch;
6883
+ if (batch) entry.coalesce = {
6884
+ mode: batch.mode,
6885
+ keyed: batch.key !== void 0
6886
+ };
6887
+ } else {
6888
+ entry.announced = true;
6889
+ }
6890
+ into.set(key, entry);
6891
+ }, "link");
6892
+ for (const kind of [
6893
+ "completed",
6894
+ "failed"
6895
+ ]) {
6896
+ const event = lifecycleEventName(workflow, kind);
6897
+ for (const [subscriber, subscriptions] of triggers.get(event) ?? []) {
6898
+ link(startsOnSettle, subscriber, kind, {
6899
+ subscriber,
6900
+ subscriptions
6901
+ });
6902
+ }
6903
+ for (const subscriber of fleet.get(event) ?? []) {
6904
+ link(startsOnSettle, subscriber, kind, void 0);
6905
+ }
6906
+ }
6907
+ for (const [event, subscribers] of triggers) {
6908
+ const source = parseLifecycleEvent(event);
6909
+ const subscriptions = subscribers.get(workflow);
6910
+ if (source && subscriptions) {
6911
+ link(startedBy, source.workflow, source.kind, {
6912
+ subscriber: workflow,
6913
+ subscriptions
6914
+ });
6915
+ }
6916
+ }
6917
+ for (const [event, subscribers] of fleet) {
6918
+ const source = parseLifecycleEvent(event);
6919
+ if (source && subscribers.has(workflow)) {
6920
+ link(startedBy, source.workflow, source.kind, void 0);
6921
+ }
6922
+ }
6923
+ const sorted = /* @__PURE__ */ __name((links) => [
6924
+ ...links.values()
6925
+ ].sort((a, b) => a.workflow.localeCompare(b.workflow) || a.kind.localeCompare(b.kind)), "sorted");
6926
+ return {
6927
+ workflow,
6928
+ startsOnSettle: sorted(startsOnSettle),
6929
+ startedBy: sorted(startedBy)
6930
+ };
6931
+ }
6932
+ /**
6933
+ * `patch` (a terminal status write of `run`) plus the lifecycle marker, when some workflow
6934
+ * subscribes to that ending: {@link LIFECYCLE_PENDING_TAG} and one owed tag per subscriber (see
6935
+ * {@link lifecycleSubscribers}). Written in the same update as the status, so a run can never be
6936
+ * settled without a record of who its lifecycle event is still owed to.
6937
+ *
6938
+ * Uses the last read of the fleet's announcements without waiting for a new one (see
6939
+ * {@link cachedAnnouncedSubscriptions}). With no usable read (none yet, or the last one failed) the
6940
+ * marker is {@link LIFECYCLE_PENDING_TAG} alone, for any workflow but an internal `__` one: the
6941
+ * owed set is unresolved, and {@link publishSettled} resolves it once a read succeeds.
6942
+ */
6943
+ async withLifecycleMarker(run, patch) {
6944
+ const kind = lifecycleKindOf(patch.status);
6945
+ if (!kind || run.workflow.startsWith("__")) return patch;
6946
+ const snapshot = this.cachedAnnouncedSubscriptions();
6947
+ const subscribers = snapshot?.ok ? this.lifecycleSubscribersIn(snapshot, run.workflow, kind, runNamespace(run.namespace)) : [];
6948
+ if (snapshot?.ok && subscribers.length === 0) return patch;
6949
+ const tags = patch.tags ?? run.tags ?? [];
6950
+ let traceparent;
6951
+ try {
6952
+ traceparent = this.runTraceparent?.(run.id);
6953
+ } catch {
6954
+ traceparent = void 0;
6955
+ }
6956
+ const recordsTrace = traceparent !== void 0 && !tags.some((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX));
6957
+ const marker = [
6958
+ LIFECYCLE_PENDING_TAG,
6959
+ ...subscribers.map((s) => `${LIFECYCLE_OWED_TAG_PREFIX}${s}`),
6960
+ ...recordsTrace ? [
6961
+ `${LIFECYCLE_TRACE_TAG_PREFIX}${traceparent}`
6962
+ ] : []
6963
+ ];
6964
+ if (marker.every((tag) => tags.includes(tag))) return patch;
6965
+ return {
6966
+ ...patch,
6967
+ tags: [
6968
+ .../* @__PURE__ */ new Set([
6969
+ ...tags,
6970
+ ...marker
6971
+ ])
6972
+ ]
6973
+ };
6974
+ }
6975
+ /**
6976
+ * `patch` (a suspend write of `run`) plus the run's {@link LIFECYCLE_TRACE_TAG_PREFIX} tag, when a
6977
+ * run trace provider is installed, some workflow subscribes to the run's ending, the provider
6978
+ * answers for the run and the run does not carry the tag yet. The first turn's suspend records the
6979
+ * run's root span this way, so a run that settles in a process that never saw its first turn still
6980
+ * links its subscribers' traces; a workflow nobody subscribes to gets no tag and no extra write.
6981
+ * Uses the last read of the fleet's announcements without waiting; with no usable read, any
6982
+ * workflow but an internal one may have subscribers, so it gets the tag.
6983
+ */
6984
+ async withRunTrace(run, patch) {
6985
+ if (!this.runTraceparent || run.workflow.startsWith("__")) return patch;
6986
+ const tags = patch.tags ?? run.tags ?? [];
6987
+ if (tags.some((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX))) return patch;
6988
+ let traceparent;
6989
+ try {
6990
+ traceparent = this.runTraceparent(run.id);
6991
+ } catch {
6992
+ return patch;
6993
+ }
6994
+ if (!traceparent) return patch;
6995
+ const snapshot = this.cachedAnnouncedSubscriptions();
6996
+ const namespace = runNamespace(run.namespace);
6997
+ const subscribed = !snapshot?.ok || this.lifecycleSubscribersIn(snapshot, run.workflow, "completed", namespace).length > 0 || this.lifecycleSubscribersIn(snapshot, run.workflow, "failed", namespace).length > 0;
6998
+ if (!subscribed) return patch;
6999
+ return {
7000
+ ...patch,
7001
+ tags: [
7002
+ ...tags,
7003
+ `${LIFECYCLE_TRACE_TAG_PREFIX}${traceparent}`
7004
+ ]
7005
+ };
7006
+ }
7007
+ /**
7008
+ * The runs started by `runId` settling (see {@link publishLifecycle}), read from the run's own
7009
+ * tags, so it needs no subscriber registry. Empty for a run that has not settled or started
7010
+ * nobody. Pass the run itself when it is already loaded, to skip reading it again.
7011
+ */
7012
+ async triggeredRuns(source) {
7013
+ const run = typeof source === "string" ? await this.store.getRun(source) : source;
7014
+ return run ? triggeredRunsOf(run) : [];
7015
+ }
7016
+ /** The live path of {@link publishSettled}: right after this instance settled the run. */
7017
+ publishLifecycle(event) {
7018
+ if (event.type !== "run.completed" && event.type !== "run.failed") return;
7019
+ const kind = event.type === "run.completed" ? "completed" : "failed";
7020
+ if (!event.workflow || !this.hasLifecycleSubscribers(event.workflow, kind, event.namespace)) {
7021
+ return;
7022
+ }
7023
+ void (async () => {
7024
+ const run = await this.store.getRun(event.runId);
7025
+ if (run) await this.publishSettled(run);
7026
+ })().catch(() => void 0);
7027
+ }
7028
+ /** How long to wait before each new try at a run's lifecycle lease another publisher holds. */
7029
+ static LIFECYCLE_LEASE_BACKOFF_MS = [
7030
+ 50,
7031
+ 100,
7032
+ 200,
7033
+ 400,
7034
+ 800
7035
+ ];
7036
+ /**
7037
+ * Run `fn` holding the lease on the settled run `runId` (`tryLockRun`, owner
7038
+ * `<instanceId>#lifecycle#<uuid>`, unique per call), so the live publish, the sweeps of every
7039
+ * instance and a republish update its lifecycle tags one at a time. Retried on the
7040
+ * {@link LIFECYCLE_LEASE_BACKOFF_MS} schedule while another holder has it (the settling execution
7041
+ * still holding its own, or another publisher); `busy` once those tries are spent.
7042
+ */
7043
+ async withLifecycleLease(runId, fn) {
7044
+ const owner = `${this.instanceId}#lifecycle#${globalThis.crypto.randomUUID()}`;
7045
+ for (let attempt = 0; ; attempt += 1) {
7046
+ const now = this.clock();
7047
+ if (await this.store.tryLockRun(runId, owner, now + this.leaseMs, now)) break;
7048
+ const wait = _WorkflowEngine.LIFECYCLE_LEASE_BACKOFF_MS[attempt];
7049
+ if (wait === void 0) return {
7050
+ busy: true
7051
+ };
7052
+ await new Promise((resolve) => setTimeout(resolve, wait));
7053
+ }
7054
+ try {
7055
+ return {
7056
+ busy: false,
7057
+ value: await fn()
7058
+ };
7059
+ } finally {
7060
+ await this.store.releaseRunLock(runId, owner).catch(() => void 0);
7061
+ }
7062
+ }
7063
+ /**
7064
+ * Publish a settled run's lifecycle event (`run.completed:<workflow>` / `run.failed:<workflow>`)
7065
+ * to the subscribers this engine has, and settle their part of the marker: drop each one's owed
7066
+ * tag, record each subscriber it started as a `durable:triggered:<workflow>` tag and each one it
7067
+ * routed into a debounce/batch accumulator as a `durable:coalesced:<workflow>` tag, and drop
7068
+ * {@link LIFECYCLE_PENDING_TAG} once no owed tag is left. Subscribers registered only elsewhere
7069
+ * keep their owed tags for the instances that have them.
7070
+ *
7071
+ * Which subscribers: `only` when given (a republish), plus `owe`, owed to the instances that have
7072
+ * them; `heal`, the ones a heal pass found missing (see {@link healLifecycleEvents}); else the owed
7073
+ * ones. A run marked with the pending tag alone is unresolved: with a good read of the fleet's
7074
+ * announcements it is owed to every subscriber (see {@link lifecycleSubscribers}) that has not
7075
+ * started, coalesced or failed, and published as such; without one, this engine starts its own subscribers
7076
+ * that do not coalesce and the run stays unresolved.
7077
+ *
7078
+ * Runs under the run's lifecycle lease (see {@link withLifecycleLease}), reading the run again
7079
+ * inside it; `busy` when the lease could not be taken. Idempotent, since every subscriber run id is
7080
+ * deterministic and a debounce/batch accumulator counts the event once, so the live path, the
7081
+ * sweeps and a republish may all reach the same run. `run.triggered` and `run.trigger_failed` are
7082
+ * emitted by the publish whose write recorded them: a subscriber it reached whose triggered or
7083
+ * coalesced tag the run did not have, and one it failed to start whose failed tag the run did not
7084
+ * have.
7085
+ *
7086
+ * A subscriber whose `where` throws or whose start fails is reported as `run.trigger_failed`,
7087
+ * tagged {@link LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX} in the same update, and not retried; a
7088
+ * subscriber it starts loses that tag. A failure to write the marker leaves it, and the sweep
7089
+ * publishes the run again. Returns the subscribers acted on, or undefined when there was nothing
7090
+ * for this engine to do.
7091
+ */
7092
+ async publishSettled(source, opts = {}) {
7093
+ if (!lifecycleKindOf(source.status)) return void 0;
7094
+ const directed = opts.only !== void 0 || opts.heal !== void 0 || (opts.owe?.length ?? 0) > 0;
7095
+ const tags = source.tags ?? [];
7096
+ const marked = tags.includes(LIFECYCLE_PENDING_TAG) || tags.some((t) => t.startsWith(LIFECYCLE_OWED_TAG_PREFIX));
7097
+ if (!directed && !marked) return void 0;
7098
+ const leased = await this.withLifecycleLease(source.id, () => this.publishSettledLocked(source.id, opts));
7099
+ return leased.busy ? "busy" : leased.value;
7100
+ }
7101
+ /** {@link publishSettled}, under the run's lifecycle lease. */
7102
+ async publishSettledLocked(runId, opts) {
7103
+ const run = await this.store.getRun(runId);
7104
+ const kind = lifecycleKindOf(run?.status);
7105
+ if (!run || !kind) return void 0;
7106
+ const name = lifecycleEventName(run.workflow, kind);
7107
+ const tags = run.tags ?? [];
7108
+ const owed = owedSubscribersOf(tags);
7109
+ const unresolved = tags.includes(LIFECYCLE_PENDING_TAG) && owed.length === 0;
7110
+ const done = /* @__PURE__ */ new Set([
7111
+ ...triggeredSubscribersOf(tags),
7112
+ ...coalescedSubscribersOf(tags),
7113
+ ...triggerFailedSubscribersOf(tags)
7114
+ ]);
7115
+ const local = this.localSubscribersIn(name, run.namespace);
7116
+ const remote = await this.announcedRemoteSubscribers(name, run.namespace);
7117
+ const startable = /* @__PURE__ */ new Set([
7118
+ ...local,
7119
+ ...remote
7120
+ ]);
7121
+ const owe = new Set(opts.owe ?? []);
7122
+ let targets;
7123
+ let resolved = false;
7124
+ if (opts.only !== void 0) {
7125
+ const only = opts.only;
7126
+ targets = [
7127
+ ...startable
7128
+ ].filter((s) => only.has(s));
7129
+ } else if (opts.heal !== void 0) {
7130
+ targets = opts.heal.filter((s) => startable.has(s));
7131
+ for (const s of opts.heal) if (!startable.has(s)) owe.add(s);
7132
+ } else if (unresolved) {
7133
+ const fleet = await this.announcedSubscriptions();
7134
+ if (fleet.ok) {
7135
+ resolved = true;
7136
+ const full = this.lifecycleSubscribersIn(fleet, run.workflow, kind, run.namespace).filter((s) => !done.has(s));
7137
+ targets = full.filter((s) => startable.has(s));
7138
+ for (const s of full) if (!startable.has(s)) owe.add(s);
7139
+ } else {
7140
+ targets = local.filter((s) => !done.has(s) && !this.latest.get(s)?.eventBatch);
7141
+ }
7142
+ } else {
7143
+ targets = owed.filter((s) => startable.has(s));
7144
+ }
7145
+ if (targets.length === 0 && owe.size === 0 && !resolved) return void 0;
7146
+ const runIdOf = /* @__PURE__ */ __name((sub2) => lifecycleRunId(run.id, kind, sub2), "runIdOf");
7147
+ const { reached, failed } = targets.length > 0 ? await this.startSubscribers(name, lifecycleEventOf(run), runIdOf, new Set(targets), remote, runIdOf) : {
7148
+ reached: [],
7149
+ failed: []
7150
+ };
7151
+ const current = await this.store.getRun(run.id) ?? run;
7152
+ const before = current.tags ?? [];
7153
+ const dropped = /* @__PURE__ */ new Set([
7154
+ ...targets.map((s) => `${LIFECYCLE_OWED_TAG_PREFIX}${s}`),
7155
+ ...reached.map((r) => `${LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX}${r.workflow}`)
7156
+ ]);
7157
+ const deliveredTag = /* @__PURE__ */ __name((r) => r.runId !== void 0 ? `${LIFECYCLE_TRIGGERED_TAG_PREFIX}${r.workflow}` : `${LIFECYCLE_COALESCED_TAG_PREFIX}${r.workflow}`, "deliveredTag");
7158
+ const added = [
7159
+ ...reached.map(deliveredTag),
7160
+ ...failed.map((f) => `${LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX}${f.workflow}`),
7161
+ ...[
7162
+ ...owe
7163
+ ].map((s) => `${LIFECYCLE_OWED_TAG_PREFIX}${s}`)
7164
+ ];
7165
+ const kept = before.filter((t) => !dropped.has(t));
7166
+ const keepPending = [
7167
+ ...kept,
7168
+ ...added
7169
+ ].some((t) => t.startsWith(LIFECYCLE_OWED_TAG_PREFIX)) || unresolved && !resolved;
7170
+ const marker = keepPending ? kept.includes(LIFECYCLE_PENDING_TAG) ? kept : [
7171
+ ...kept,
7172
+ LIFECYCLE_PENDING_TAG
7173
+ ] : kept.filter((t) => t !== LIFECYCLE_PENDING_TAG);
7174
+ const tagsAfter = [
7175
+ .../* @__PURE__ */ new Set([
7176
+ ...marker,
7177
+ ...added
7178
+ ])
7179
+ ];
7180
+ if (tagsAfter.length !== before.length || tagsAfter.some((t) => !before.includes(t))) {
7181
+ await this.store.updateRun(run.id, {
7182
+ tags: tagsAfter
7183
+ });
7184
+ }
7185
+ const had = new Set(before);
7186
+ for (const r of reached) {
7187
+ if (had.has(deliveredTag(r))) continue;
7188
+ const { workflow, runId: triggeredRunId } = r;
7189
+ this.emit({
7190
+ type: "run.triggered",
7191
+ runId: run.id,
7192
+ workflow: run.workflow,
7193
+ namespace: run.namespace,
7194
+ name: workflow,
7195
+ triggeredRunId
7196
+ });
7197
+ }
7198
+ for (const { workflow, error: error2 } of failed) {
7199
+ if (had.has(`${LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX}${workflow}`)) continue;
7200
+ this.reportTriggerFailed(run, workflow, error2 instanceof Error ? error2.message : String(error2));
7201
+ }
7202
+ return {
7203
+ reached,
7204
+ failed
7205
+ };
7206
+ }
7207
+ reportTriggerFailed(run, subscriber, message) {
7208
+ this.emit({
7209
+ type: "run.trigger_failed",
7210
+ runId: run.id,
7211
+ workflow: run.workflow,
7212
+ namespace: run.namespace,
7213
+ name: subscriber,
7214
+ error: {
7215
+ message
7216
+ }
7217
+ });
7218
+ }
7219
+ /**
7220
+ * Drop the owed tags of `run`'s subscribers that no live worker announces and this engine does
7221
+ * not have, once the run settled more than {@link LIFECYCLE_ORPHAN_MS} ago, reporting each as
7222
+ * `run.trigger_failed` and tagging it {@link LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX} in the same
7223
+ * update, under the run's lifecycle lease. Skipped when the fleet's announcements could not be
7224
+ * read, since a failed read would make every remote subscriber look orphaned.
7225
+ */
7226
+ async dropOrphanedSubscribers(run, nowMs) {
7227
+ if (nowMs - run.updatedAt.getTime() < LIFECYCLE_ORPHAN_MS) return false;
7228
+ const kind = lifecycleKindOf(run.status);
7229
+ if (!kind || owedSubscribersOf(run.tags).length === 0) return false;
7230
+ const fleet = await this.announcedSubscriptions();
7231
+ if (!fleet.ok) return false;
7232
+ const leased = await this.withLifecycleLease(run.id, async () => {
7233
+ const current = await this.store.getRun(run.id);
7234
+ const owed = owedSubscribersOf(current?.tags);
7235
+ if (!current || owed.length === 0) return false;
7236
+ const event = lifecycleEventName(run.workflow, kind);
7237
+ const live = /* @__PURE__ */ new Set([
7238
+ ...this.localSubscribersIn(event, run.namespace),
7239
+ ...announcedIn(fleet, event, run.namespace)
7240
+ ]);
7241
+ const orphans = owed.filter((s) => !live.has(s));
7242
+ if (orphans.length === 0) return false;
7243
+ const dropped = new Set(orphans.map((s) => `${LIFECYCLE_OWED_TAG_PREFIX}${s}`));
7244
+ const kept = (current.tags ?? []).filter((t) => !dropped.has(t));
7245
+ const owedLeft = kept.some((t) => t.startsWith(LIFECYCLE_OWED_TAG_PREFIX));
7246
+ await this.store.updateRun(run.id, {
7247
+ tags: [
7248
+ .../* @__PURE__ */ new Set([
7249
+ ...kept.filter((t) => owedLeft || t !== LIFECYCLE_PENDING_TAG),
7250
+ ...orphans.map((s) => `${LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX}${s}`)
7251
+ ])
7252
+ ]
7253
+ });
7254
+ for (const subscriber of orphans) {
7255
+ this.reportTriggerFailed(current, subscriber, `subscriber ${subscriber} is not registered on any live worker`);
7256
+ }
7257
+ return true;
7258
+ });
7259
+ return !leased.busy && leased.value;
7260
+ }
7261
+ /** How long a settled run's marker is left to the live publish before the sweep takes it over. */
7262
+ static LIFECYCLE_SWEEP_GRACE_MS = 3e4;
7263
+ /**
7264
+ * How many marked runs (and `ctx.waitForRun` waiters) one sweep examines at most, looking for
7265
+ * `limit` it can act on. Runs owed only to subscribers other instances have are skipped without a
7266
+ * store read, so they cannot fill the window and keep this instance from the ones it can publish.
7267
+ */
7268
+ static LIFECYCLE_SWEEP_SCAN_MAX = 1e3;
7269
+ /** How many recently settled runs of one workflow a heal pass reads at most. */
7270
+ static LIFECYCLE_HEAL_SCAN_CAP = 100;
7271
+ /**
7272
+ * Publish the lifecycle event of every settled run still marked {@link LIFECYCLE_PENDING_TAG}:
7273
+ * one whose process died, or whose marker write failed, between settling it and publishing, one
7274
+ * owed to a subscriber the settling process does not have, or one whose owed set is unresolved.
7275
+ * Each instance publishes to the subscribers it has, and drops owed subscribers nobody has anymore
7276
+ * (see {@link dropOrphanedSubscribers}). Then heals the recently settled runs that missed a newly
7277
+ * announced subscriber (see {@link healLifecycleEvents}). Also delivers the `ctx.waitForRun` waits
7278
+ * whose run already settled (see {@link sweepRunWaits}). Called by the timer poller; a run settled
7279
+ * in the last {@link LIFECYCLE_SWEEP_GRACE_MS} is left to the live publish. Returns how many runs
7280
+ * it published to a subscriber.
5871
7281
  *
5872
- * Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
5873
- * at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
5874
- * few timer ticks without a long query or a big delete. Each page is decided set-based (see
5875
- * `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
5876
- * read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
5877
- * {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
5878
- * call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
5879
- * before the next one. Call it every tick (the NestJS timer poller does); a call made while
5880
- * another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
5881
- * carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
5882
- * store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
5883
- */
5884
- async sweepBufferedSignals(now = this.clock(), budgetMs = BUFFERED_SWEEP_BUDGET_MS) {
5885
- const { listBufferedSignals, deleteBufferedSignals } = this.store;
5886
- if (!listBufferedSignals || !deleteBufferedSignals) return 0;
5887
- if (this.draining || this.namespace !== void 0 || this.bufferedSweepActive) return 0;
5888
- if (this.bufferedSweepCursor === void 0 && now - this.lastBufferedSweepEndedAt < BUFFERED_SWEEP_REST_MS) {
5889
- return 0;
5890
- }
5891
- this.bufferedSweepActive = true;
5892
- const deadline = Date.now() + budgetMs;
5893
- let deleted = 0;
5894
- try {
5895
- for (let page = 0; page < BUFFERED_SWEEP_PAGES_PER_CALL; page++) {
5896
- if (page > 0 && (Date.now() >= deadline || this.draining)) break;
5897
- const rows = await listBufferedSignals.call(this.store, CHILD_TOKEN_PREFIX, this.bufferedSweepCursor, BUFFERED_SWEEP_PAGE);
5898
- const { doomed, decided } = await unneededChildSignals(this.store, rows, deadline);
5899
- if (doomed.length > 0) {
5900
- await deleteBufferedSignals.call(this.store, doomed.map((row) => row.id));
5901
- deleted += doomed.length;
5902
- }
5903
- if (decided < rows.length) {
5904
- if (decided > 0) this.bufferedSweepCursor = rows[decided - 1].id;
5905
- break;
5906
- }
5907
- const last = rows.at(-1);
5908
- if (rows.length < BUFFERED_SWEEP_PAGE || !last) {
5909
- this.bufferedSweepCursor = void 0;
5910
- this.lastBufferedSweepEndedAt = now;
5911
- break;
7282
+ * Reads the marked runs a page of `limit` at a time, skipping the ones it cannot act on, until it
7283
+ * has acted on `limit` runs (published, or dropped orphans) or examined
7284
+ * {@link LIFECYCLE_SWEEP_SCAN_MAX}. Pages advance by offset over a set the sweep itself shrinks,
7285
+ * so a run that moves into an earlier page is reached on the next tick, which starts over. A run
7286
+ * whose lifecycle lease another publisher holds is left for the next tick.
7287
+ */
7288
+ async sweepLifecycleEvents(nowMs = this.clock(), limit = 100) {
7289
+ await this.heartbeatLifecycleRegistry().catch((error2) => {
7290
+ console.error("[nestjs-durable] lifecycle registry heartbeat failed; retrying next tick:", error2);
7291
+ });
7292
+ await this.sweepRunWaits(limit).catch((error2) => {
7293
+ console.error("[nestjs-durable] run-wait sweep failed; retrying next tick:", error2);
7294
+ });
7295
+ await this.announcedSubscriptions();
7296
+ const pageSize = Math.max(1, limit);
7297
+ const scanMax = Math.max(pageSize, _WorkflowEngine.LIFECYCLE_SWEEP_SCAN_MAX);
7298
+ let published = 0;
7299
+ let acted = 0;
7300
+ let examined = 0;
7301
+ while (acted < limit && examined < scanMax) {
7302
+ const requested = Math.min(pageSize, scanMax - examined);
7303
+ const page = await this.store.listRuns({
7304
+ tag: LIFECYCLE_PENDING_TAG,
7305
+ statuses: [
7306
+ "completed",
7307
+ "failed",
7308
+ "cancelled",
7309
+ "dead"
7310
+ ],
7311
+ ...this.namespace !== void 0 ? {
7312
+ namespace: this.namespace
7313
+ } : {},
7314
+ limit: requested,
7315
+ offset: examined
7316
+ });
7317
+ for (const run of page) {
7318
+ if (acted >= limit) break;
7319
+ examined += 1;
7320
+ if (nowMs - run.updatedAt.getTime() < _WorkflowEngine.LIFECYCLE_SWEEP_GRACE_MS) continue;
7321
+ try {
7322
+ const outcome = await this.publishSettled(run);
7323
+ if (outcome === "busy") continue;
7324
+ if (outcome && outcome.reached.length + outcome.failed.length > 0) published += 1;
7325
+ const dropped = await this.dropOrphanedSubscribers(run, nowMs);
7326
+ if (outcome || dropped) acted += 1;
7327
+ } catch (error2) {
7328
+ console.error(`[nestjs-durable] lifecycle sweep: run ${run.id} failed; retrying next tick:`, error2);
5912
7329
  }
5913
- this.bufferedSweepCursor = last.id;
5914
7330
  }
5915
- } finally {
5916
- this.bufferedSweepActive = false;
7331
+ if (page.length < requested) break;
5917
7332
  }
5918
- return deleted;
7333
+ try {
7334
+ published += await this.healLifecycleEvents(nowMs);
7335
+ } catch (error2) {
7336
+ console.error("[nestjs-durable] lifecycle heal failed; retrying next tick:", error2);
7337
+ }
7338
+ return published;
5919
7339
  }
5920
7340
  /**
5921
- * Register a pager hook for STRANDED runs: an in-flight run untouched past `stalledAfterMs` that is
5922
- * either suspended with NO wake timer (wake-forever: nothing will ever re-drive it) or parked on an
5923
- * old pending remote step with no live lease (dispatched and, most likely, lost). The timer poller
5924
- * sweeps for these ({@link sweepStalled}, throttled to one pass a minute) and notifies each episode
5925
- * ONCE per instance: a run is paged again only after it makes progress and strands again. Wire your
5926
- * alerting here like {@link onDead}. A listener that throws never breaks the sweep. Returns an
5927
- * unsubscribe function.
7341
+ * The (run, subscriber) pairs this engine already healed, until when it remembers them; see
7342
+ * {@link healLifecycleEvents}.
5928
7343
  */
5929
- onStalled(listener) {
5930
- this.stalledListeners.add(listener);
5931
- return () => this.stalledListeners.delete(listener);
5932
- }
7344
+ healedLifecycle = /* @__PURE__ */ new Map();
5933
7345
  /**
5934
- * Detect stranded in-flight runs and notify the {@link onStalled} listeners. Self-throttled to one
5935
- * pass a minute, skipped while draining, and a no-op without listeners — call it every tick (the
5936
- * NestJS timer poller does). Scoped to this engine's `namespace` (an operator engine with no
5937
- * namespace sweeps every namespace). Returns the runs notified THIS pass.
5938
- */
5939
- async sweepStalled(now = this.clock()) {
5940
- if (this.stalledListeners.size === 0 || this.draining) return [];
5941
- if (now - this.lastStalledSweepAt < 6e4) return [];
5942
- this.lastStalledSweepAt = now;
5943
- const notified = [];
5944
- for (const info2 of await this.listStalledRuns({
5945
- now
5946
- })) {
5947
- const episode = info2.run.updatedAt.getTime();
5948
- if (this.notifiedStalled.get(info2.run.id) === episode) continue;
5949
- this.notifiedStalled.set(info2.run.id, episode);
5950
- if (this.notifiedStalled.size > 1e3) this.notifiedStalled.clear();
5951
- for (const listener of this.stalledListeners) {
7346
+ * Heal the runs settled in the last {@link LIFECYCLE_HEAL_WINDOW_MS} that missed a subscriber: one
7347
+ * the fleet announced only after the settling process last read the announcements, so it is in
7348
+ * none of the run's owed, triggered, coalesced or failed tags. Only subscriptions this engine first saw within
7349
+ * that window are checked (see {@link subscriptionFirstSeen}), since one the fleet already
7350
+ * announced when the run settled was in its owed set. Needs a good read of the fleet's
7351
+ * announcements.
7352
+ *
7353
+ * One `listRuns` per source workflow (`updatedAfter` the window, at most
7354
+ * {@link LIFECYCLE_HEAL_SCAN_CAP} runs), skipping unresolved runs (the sweep resolves those) and
7355
+ * the ones in their {@link LIFECYCLE_SWEEP_GRACE_MS}. Each missed subscriber is published like an
7356
+ * owed one (see {@link publishSettled}): started here when this engine can, owed to the instances
7357
+ * that have it otherwise. A subscriber whose `where` declines the event leaves no tag, so each
7358
+ * engine heals a pair once. Returns how many runs it published to a subscriber.
7359
+ */
7360
+ async healLifecycleEvents(nowMs) {
7361
+ const fleet = await this.announcedSubscriptions();
7362
+ if (!fleet.ok) return 0;
7363
+ for (const [key, until] of this.healedLifecycle) {
7364
+ if (until <= nowMs) this.healedLifecycle.delete(key);
7365
+ }
7366
+ const windowStart = nowMs - LIFECYCLE_HEAL_WINDOW_MS;
7367
+ const sources = /* @__PURE__ */ new Map();
7368
+ for (const [key, firstSeen] of this.subscriptionFirstSeen) {
7369
+ if (firstSeen < windowStart) continue;
7370
+ const source = parseLifecycleEvent(key.slice(0, key.indexOf("\0")));
7371
+ if (!source || source.workflow.startsWith("__")) continue;
7372
+ const kinds = sources.get(source.workflow) ?? /* @__PURE__ */ new Set();
7373
+ kinds.add(source.kind);
7374
+ sources.set(source.workflow, kinds);
7375
+ }
7376
+ let published = 0;
7377
+ for (const [workflow, kinds] of sources) {
7378
+ const statuses = [];
7379
+ if (kinds.has("completed")) statuses.push("completed");
7380
+ if (kinds.has("failed")) statuses.push("failed", "cancelled", "dead");
7381
+ const runs = await this.store.listRuns({
7382
+ workflow,
7383
+ statuses,
7384
+ updatedAfter: windowStart,
7385
+ ...this.namespace !== void 0 ? {
7386
+ namespace: this.namespace
7387
+ } : {},
7388
+ limit: _WorkflowEngine.LIFECYCLE_HEAL_SCAN_CAP
7389
+ });
7390
+ for (const run of runs) {
7391
+ const kind = lifecycleKindOf(run.status);
7392
+ if (!kind) continue;
7393
+ if (nowMs - run.updatedAt.getTime() < _WorkflowEngine.LIFECYCLE_SWEEP_GRACE_MS) continue;
7394
+ const tags = run.tags ?? [];
7395
+ const owed = owedSubscribersOf(tags);
7396
+ if (tags.includes(LIFECYCLE_PENDING_TAG) && owed.length === 0) continue;
7397
+ const event = lifecycleEventName(workflow, kind);
7398
+ const known = /* @__PURE__ */ new Set([
7399
+ ...owed,
7400
+ ...triggeredSubscribersOf(tags),
7401
+ ...coalescedSubscribersOf(tags),
7402
+ ...triggerFailedSubscribersOf(tags)
7403
+ ]);
7404
+ const missing = this.lifecycleSubscribersIn(fleet, workflow, kind, run.namespace).filter((s) => {
7405
+ if (known.has(s) || this.healedLifecycle.has(`${run.id}\0${s}`)) return false;
7406
+ const firstSeen = this.subscriptionFirstSeen.get(`${event}\0${s}`);
7407
+ return firstSeen !== void 0 && firstSeen >= windowStart;
7408
+ });
7409
+ if (missing.length === 0) continue;
5952
7410
  try {
5953
- listener(info2);
5954
- } catch {
7411
+ const outcome = await this.publishSettled(run, {
7412
+ heal: missing
7413
+ });
7414
+ if (outcome === "busy") continue;
7415
+ for (const s of missing) {
7416
+ this.healedLifecycle.set(`${run.id}\0${s}`, run.updatedAt.getTime() + LIFECYCLE_HEAL_WINDOW_MS);
7417
+ }
7418
+ if (outcome && outcome.reached.length + outcome.failed.length > 0) published += 1;
7419
+ } catch (error2) {
7420
+ console.error(`[nestjs-durable] lifecycle heal: run ${run.id} failed; retrying next tick:`, error2);
5955
7421
  }
5956
7422
  }
5957
- notified.push(info2);
5958
7423
  }
5959
- return notified;
7424
+ return published;
5960
7425
  }
5961
7426
  /**
5962
- * The stranded runs right now, without notifying anyone: the query behind {@link sweepStalled} and
5963
- * the CLI's `durable inspect --stale`. A `running`/`suspended` run qualifies when it has been
5964
- * untouched for `thresholdMs` (default `stalledAfterMs`) AND either it is suspended with no wake
5965
- * timer, or its oldest pending remote step is older than the threshold with no lease in the future
5966
- * (a step with a `timeoutMs`, a heartbeat or `remoteRedispatchMs` keeps a lease; a long
5967
- * `ctx.sleep` has a future `wakeAt`; neither is stranded). Scans up to `scan` candidates (default
5968
- * 1000), newest first, in this engine's namespace.
5969
- */
5970
- async listStalledRuns(options = {}) {
5971
- return findStalledRuns(this.store, {
5972
- now: options.now ?? this.clock(),
5973
- thresholdMs: options.thresholdMs ?? this.stalledAfterMs,
5974
- scan: options.scan,
5975
- namespace: this.namespace
5976
- });
7427
+ * Publish the lifecycle events of settled runs again, to subscribers that missed them: one
7428
+ * registered after the runs settled, or one whose run was deleted. Unlike the sweep it needs no
7429
+ * marker. Each subscriber's run id is deterministic, so one already started is not started again.
7430
+ *
7431
+ * Subscribers registered on this engine are started here. A subscriber only live workers announce
7432
+ * is owed the event again (the run is re-marked), so the sweep on an instance that has it starts
7433
+ * it. Only subscribers of the run's lifecycle event in its namespace are published to; a
7434
+ * `subscriber` that is not one is reported in `failed`. Reads at most `limit` (a positive integer,
7435
+ * default 1000) settled runs of a workflow, a page at a time.
7436
+ */
7437
+ async republishLifecycle(target) {
7438
+ const result = {
7439
+ runs: 0,
7440
+ started: 0,
7441
+ failed: []
7442
+ };
7443
+ if ("runId" in target) {
7444
+ const run = await this.store.getRun(target.runId);
7445
+ if (!run) throw new FatalError(`run ${target.runId} not found`);
7446
+ if (!lifecycleKindOf(run.status)) {
7447
+ throw new FatalError(`run ${target.runId} has not settled (${run.status})`);
7448
+ }
7449
+ await this.republishRun(run, target.subscriber, result);
7450
+ return result;
7451
+ }
7452
+ const statuses = target.kind === "completed" ? [
7453
+ "completed"
7454
+ ] : target.kind === "failed" ? [
7455
+ "failed",
7456
+ "cancelled",
7457
+ "dead"
7458
+ ] : [
7459
+ "completed",
7460
+ "failed",
7461
+ "cancelled",
7462
+ "dead"
7463
+ ];
7464
+ if (target.namespace !== void 0 && this.namespace !== void 0 && target.namespace !== this.namespace) {
7465
+ throw new FatalError(`namespace ${target.namespace} is not this engine's namespace (${this.namespace})`);
7466
+ }
7467
+ const limit = target.limit ?? 1e3;
7468
+ if (!Number.isSafeInteger(limit) || limit < 1) {
7469
+ throw new FatalError(`limit must be a positive integer, got ${String(target.limit)}`);
7470
+ }
7471
+ const namespace = target.namespace ?? this.namespace;
7472
+ const pageSize = Math.min(limit, 200);
7473
+ let read = 0;
7474
+ while (read < limit) {
7475
+ const want = Math.min(pageSize, limit - read);
7476
+ const page = await this.store.listRuns({
7477
+ workflow: target.workflow,
7478
+ statuses,
7479
+ ...namespace !== void 0 ? {
7480
+ namespace
7481
+ } : {},
7482
+ limit: want,
7483
+ offset: read
7484
+ });
7485
+ for (const run of page) {
7486
+ const settledAt = run.updatedAt.getTime();
7487
+ if (target.since && settledAt < target.since.getTime()) continue;
7488
+ if (target.until && settledAt > target.until.getTime()) continue;
7489
+ await this.republishRun(run, target.subscriber, result);
7490
+ }
7491
+ read += page.length;
7492
+ if (page.length < want) break;
7493
+ }
7494
+ return result;
5977
7495
  }
5978
- notifyDead(run) {
5979
- for (const fn of this.deadListeners) {
5980
- try {
5981
- fn(run);
5982
- } catch {
7496
+ /** {@link republishLifecycle} for one settled run: start what is missing, report what failed. */
7497
+ async republishRun(run, subscriber, result) {
7498
+ const kind = lifecycleKindOf(run.status);
7499
+ if (!kind) return;
7500
+ const subscribers = (await this.lifecycleSubscribers(run.workflow, kind, runNamespace(run.namespace))).filter((s) => subscriber === void 0 || s === subscriber);
7501
+ if (subscriber !== void 0 && subscribers.length === 0) {
7502
+ result.runs += 1;
7503
+ result.failed.push({
7504
+ runId: run.id,
7505
+ subscriber,
7506
+ error: `subscriber ${subscriber} does not subscribe to ${lifecycleEventName(run.workflow, kind)}`
7507
+ });
7508
+ return;
7509
+ }
7510
+ if (subscribers.length === 0) return;
7511
+ result.runs += 1;
7512
+ const event = lifecycleEventName(run.workflow, kind);
7513
+ const local = /* @__PURE__ */ new Set([
7514
+ ...this.localSubscribersIn(event, run.namespace),
7515
+ ...await this.announcedRemoteSubscribers(event, run.namespace)
7516
+ ]);
7517
+ const triggerFailed = triggerFailedSubscribersOf(run.tags);
7518
+ const coalesced = coalescedSubscribersOf(run.tags);
7519
+ const missing = [];
7520
+ for (const s of subscribers) {
7521
+ if (coalesced.includes(s) && !triggerFailed.includes(s)) continue;
7522
+ if (local.has(s) && this.latest.get(s)?.eventBatch !== void 0) {
7523
+ missing.push(s);
7524
+ continue;
5983
7525
  }
7526
+ if (!await this.store.getRun(lifecycleRunId(run.id, kind, s))) missing.push(s);
7527
+ }
7528
+ const toStart = new Set(missing.filter((s) => local.has(s)));
7529
+ const toOwe = missing.filter((s) => !local.has(s));
7530
+ if (toStart.size === 0 && toOwe.length === 0) return;
7531
+ const outcome = await this.publishSettled(run, {
7532
+ only: toStart,
7533
+ owe: toOwe
7534
+ });
7535
+ if (outcome === "busy") {
7536
+ result.failed.push({
7537
+ runId: run.id,
7538
+ subscriber: "*",
7539
+ error: `run ${run.id} is being published elsewhere; try again`
7540
+ });
7541
+ return;
7542
+ }
7543
+ result.started += outcome?.reached.length ?? 0;
7544
+ for (const { workflow, error: error2 } of outcome?.failed ?? []) {
7545
+ result.failed.push({
7546
+ runId: run.id,
7547
+ subscriber: workflow,
7548
+ error: error2 instanceof Error ? error2.message : String(error2)
7549
+ });
5984
7550
  }
5985
7551
  }
5986
- /** Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the
5987
- * control plane so other instances (e.g. a dashboard pod) can live-tail this run. */
5988
- emit(event) {
5989
- const full = {
5990
- ...event,
5991
- at: /* @__PURE__ */ new Date()
5992
- };
5993
- this.deliver(full);
5994
- if (!this.controlPlane) return;
5995
- this.eventsToBroadcast.push(full);
5996
- if (!this.broadcastingEvents) void this.broadcastEvents();
7552
+ /**
7553
+ * Resume the runs waiting on `targetRunId` with `ctx.waitForRun`, once it has settled. Each waiter
7554
+ * is taken before it is delivered to, so the settle path, the sweep and the waiting run's own
7555
+ * check never resume it twice. Returns how many it resumed.
7556
+ */
7557
+ async deliverRunWaits(targetRunId) {
7558
+ const waiters = (await this.store.listSignalWaiters(runWaitPrefix(targetRunId))).filter((w) => runWaitTargetOf(w) === targetRunId);
7559
+ if (waiters.length === 0) return 0;
7560
+ const target = await settledRunOf(this.store, targetRunId);
7561
+ if (target.state !== "settled") return 0;
7562
+ const payload = lifecycleEventOf(target.run);
7563
+ let delivered = 0;
7564
+ for (const w of waiters) {
7565
+ const waiter = await this.store.takeSignalWaiter(w.token);
7566
+ if (!waiter) continue;
7567
+ await this.deliverSignal(waiter, payload);
7568
+ delivered += 1;
7569
+ }
7570
+ return delivered;
7571
+ }
7572
+ /**
7573
+ * Deliver the `ctx.waitForRun` waits whose run has settled but which were never resumed: a crash
7574
+ * between settling and delivering, or a run that settled by handing off to a continue-as-new
7575
+ * chain that has since ended. Checks the waited-on runs in waiter order, skipping the ones that
7576
+ * have not settled, until it has resumed waiters of `limit` runs or checked
7577
+ * {@link LIFECYCLE_SWEEP_SCAN_MAX}, so long waits on runs still in flight cannot keep it from a
7578
+ * wait owed a delivery. Returns how many waiters it resumed.
7579
+ */
7580
+ async sweepRunWaits(limit = 100) {
7581
+ const waiters = await this.store.listSignalWaiters(RUN_WAIT_TOKEN_PREFIX);
7582
+ const targets = /* @__PURE__ */ new Set();
7583
+ for (const w of waiters) {
7584
+ const target = runWaitTargetOf(w);
7585
+ if (target !== void 0) targets.add(target);
7586
+ }
7587
+ const scanMax = Math.max(limit, _WorkflowEngine.LIFECYCLE_SWEEP_SCAN_MAX);
7588
+ let delivered = 0;
7589
+ let acted = 0;
7590
+ let examined = 0;
7591
+ for (const target of targets) {
7592
+ if (acted >= limit || examined >= scanMax) break;
7593
+ examined += 1;
7594
+ const resumed = await this.deliverRunWaits(target);
7595
+ if (resumed > 0) {
7596
+ delivered += resumed;
7597
+ acted += 1;
7598
+ }
7599
+ }
7600
+ return delivered;
5997
7601
  }
5998
7602
  /** Events waiting for the control-plane publish in flight, in emit order (see {@link broadcastEvents}). */
5999
7603
  eventsToBroadcast = [];
@@ -6676,11 +8280,11 @@ var WorkflowEngine = class {
6676
8280
  message: "execution timeout",
6677
8281
  code: "execution_timeout"
6678
8282
  };
6679
- await this.store.updateRun(run.id, {
8283
+ await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
6680
8284
  status: "cancelled",
6681
8285
  error: error2,
6682
8286
  updatedAt: /* @__PURE__ */ new Date()
6683
- });
8287
+ }));
6684
8288
  this.emit({
6685
8289
  type: "run.failed",
6686
8290
  runId: run.id,
@@ -6856,11 +8460,11 @@ var WorkflowEngine = class {
6856
8460
  message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) \u2014 moved to dead-letter`,
6857
8461
  code: "max_recovery_attempts"
6858
8462
  };
6859
- await this.store.updateRun(run.id, {
8463
+ await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
6860
8464
  status: "dead",
6861
8465
  error: error2,
6862
8466
  updatedAt: /* @__PURE__ */ new Date()
6863
- });
8467
+ }));
6864
8468
  await this.store.releaseRunLock(run.id);
6865
8469
  return this.announceDead({
6866
8470
  ...run,
@@ -6938,11 +8542,11 @@ var WorkflowEngine = class {
6938
8542
  };
6939
8543
  const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
6940
8544
  run.status
6941
- ], {
8545
+ ], await this.withLifecycleMarker(run, {
6942
8546
  status: "dead",
6943
8547
  error: error2,
6944
8548
  updatedAt: /* @__PURE__ */ new Date()
6945
- });
8549
+ }));
6946
8550
  await this.store.releaseRunLock(run.id);
6947
8551
  if (!applied) return null;
6948
8552
  console.warn(`[nestjs-durable] run ${run.id}: ${error2.message}`);
@@ -7371,7 +8975,8 @@ var WorkflowEngine = class {
7371
8975
  * 1. **Resumes** every in-flight run waiting on it via `ctx.waitForEvent(name, { match })` whose
7372
8976
  * match the payload satisfies (fan-out, vs `signal`'s point-to-point token).
7373
8977
  * 2. **Starts** a fresh run of every workflow registered with `onEvent: [name]`, passing the
7374
- * payload as input. Idempotent by `evt:<id>:<workflow>` — pass `opts.id` to dedupe redeliveries
8978
+ * payload as input, and of every workflow live workers announce as subscribing to it that this
8979
+ * engine can start by name (see {@link announcedRemoteSubscribers}). Idempotent by `evt:<id>:<workflow>` — pass `opts.id` to dedupe redeliveries
7375
8980
  * of the same logical event (default: a fresh uuid, so each publish triggers once).
7376
8981
  *
7377
8982
  * Semantics (deliberate — mirrors `signalWithStart`'s reliability contract, so document it exactly):
@@ -7398,20 +9003,13 @@ var WorkflowEngine = class {
7398
9003
  touched += 1;
7399
9004
  }
7400
9005
  }
7401
- const subscribers = this.eventTriggers.get(name);
7402
- if (subscribers?.size) {
9006
+ const remote = await this.announcedRemoteSubscribers(name, null);
9007
+ if (this.eventTriggers.get(name)?.size || remote.length > 0) {
7403
9008
  const eventId = opts?.id ?? globalThis.crypto.randomUUID();
7404
- for (const workflow of subscribers) {
7405
- try {
7406
- const batch = this.latest.get(workflow)?.eventBatch;
7407
- if (batch) {
7408
- await this.accumulators.route(workflow, batch, payload);
7409
- } else {
7410
- await this.start(workflow, payload, `evt:${eventId}:${workflow}`);
7411
- }
7412
- touched += 1;
7413
- } catch {
7414
- }
9009
+ const { reached, failed } = await this.startSubscribers(name, payload, (workflow) => `evt:${eventId}:${workflow}`, void 0, remote);
9010
+ touched += reached.length;
9011
+ for (const { workflow, error: error2 } of failed) {
9012
+ console.warn(`[nestjs-durable] event ${name}: subscriber ${workflow} not started:`, error2);
7415
9013
  }
7416
9014
  }
7417
9015
  if (touched > 0 || opts?.buffer === false) return touched;
@@ -7868,11 +9466,11 @@ var WorkflowEngine = class {
7868
9466
  const error2 = {
7869
9467
  message: "cancelled"
7870
9468
  };
7871
- await this.store.updateRun(runId, {
9469
+ await this.store.updateRun(runId, await this.withLifecycleMarker(run, {
7872
9470
  status: "cancelled",
7873
9471
  error: error2,
7874
9472
  updatedAt: /* @__PURE__ */ new Date()
7875
- });
9473
+ }));
7876
9474
  this.emit({
7877
9475
  type: "run.failed",
7878
9476
  runId,
@@ -8357,6 +9955,7 @@ var WorkflowEngine = class {
8357
9955
  * than 0.73 (see `legacySingletonHolders`).
8358
9956
  */
8359
9957
  async markStarted(run, singleton) {
9958
+ void this.announcedSubscriptions();
8360
9959
  const tags = singleton ? withSingletonTracked(run.tags) : run.tags;
8361
9960
  const retag = tags !== run.tags;
8362
9961
  await this.store.updateRun(run.id, {
@@ -8878,12 +10477,12 @@ ${run.workflowVersion}`;
8878
10477
  if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
8879
10478
  const updatedAt = /* @__PURE__ */ new Date();
8880
10479
  if (outcome.kind === "completed") {
8881
- const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, {
10480
+ const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, await this.withLifecycleMarker(run, {
8882
10481
  status: "completed",
8883
10482
  output: outcome.output,
8884
10483
  error: void 0,
8885
10484
  updatedAt
8886
- });
10485
+ }));
8887
10486
  if (!applied2) return this.echoCurrentStatus(run.id);
8888
10487
  this.emit({
8889
10488
  type: "run.completed",
@@ -8906,11 +10505,11 @@ ${run.workflowVersion}`;
8906
10505
  const handed = await this.routeFailure(run, outcome.error);
8907
10506
  if (handed && handed.kind !== "failed") return this.settleRun(run, handed);
8908
10507
  const error2 = handed?.error ?? outcome.error;
8909
- const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, {
10508
+ const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, await this.withLifecycleMarker(run, {
8910
10509
  status: "failed",
8911
10510
  error: error2,
8912
10511
  updatedAt
8913
- });
10512
+ }));
8914
10513
  if (!applied2) return this.echoCurrentStatus(run.id);
8915
10514
  this.emit({
8916
10515
  type: "run.failed",
@@ -8927,11 +10526,11 @@ ${run.workflowVersion}`;
8927
10526
  };
8928
10527
  }
8929
10528
  if (outcome.kind === "blocked") return this.parkBlocked(run, outcome.blocked);
8930
- const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, {
10529
+ const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, await this.withRunTrace(run, {
8931
10530
  status: "suspended",
8932
10531
  wakeAt: this.reconcileWakeAt(outcome.wakeAt),
8933
10532
  updatedAt
8934
- });
10533
+ }));
8935
10534
  if (!applied) return this.echoCurrentStatus(run.id);
8936
10535
  await this.keepDueForKeptWakes(run.id);
8937
10536
  this.emit({
@@ -8961,11 +10560,11 @@ ${run.workflowVersion}`;
8961
10560
  const error2 = {
8962
10561
  message: "cancelled"
8963
10562
  };
8964
- await this.store.updateRun(run.id, {
10563
+ await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
8965
10564
  status: "cancelled",
8966
10565
  error: error2,
8967
10566
  updatedAt: /* @__PURE__ */ new Date()
8968
- });
10567
+ }));
8969
10568
  if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
8970
10569
  this.emit({
8971
10570
  type: "run.failed",
@@ -8997,7 +10596,8 @@ ${run.workflowVersion}`;
8997
10596
  type: "run.started",
8998
10597
  runId: run.id,
8999
10598
  workflow: run.workflow,
9000
- namespace: run.namespace
10599
+ namespace: run.namespace,
10600
+ ...await this.sourceTraceOf(run)
9001
10601
  });
9002
10602
  }
9003
10603
  if (registered.singleton && !await this.singletons.admit(run, registered.singleton)) {
@@ -9020,12 +10620,12 @@ ${run.workflowVersion}`;
9020
10620
  const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID().slice(0, 8)}`;
9021
10621
  const silenceMs = this.remoteAdvanceSilenceMs;
9022
10622
  const wakeAt = silenceMs != null ? this.clock() + silenceMs : void 0;
9023
- await this.store.updateRun(run.id, {
10623
+ await this.store.updateRun(run.id, await this.withRunTrace(run, {
9024
10624
  status: "suspended",
9025
10625
  awaitingDecisionTaskId: taskId,
9026
10626
  wakeAt,
9027
10627
  updatedAt: /* @__PURE__ */ new Date()
9028
- });
10628
+ }));
9029
10629
  this.emit({
9030
10630
  type: "run.suspended",
9031
10631
  runId: run.id,
@@ -9062,11 +10662,11 @@ ${run.workflowVersion}`;
9062
10662
  stack: err instanceof Error ? err.stack : void 0
9063
10663
  };
9064
10664
  if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
9065
- await this.store.updateRun(run.id, {
10665
+ await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
9066
10666
  status: "failed",
9067
10667
  error: error2,
9068
10668
  updatedAt: /* @__PURE__ */ new Date()
9069
- });
10669
+ }));
9070
10670
  this.emit({
9071
10671
  type: "run.failed",
9072
10672
  runId: run.id,
@@ -9604,11 +11204,11 @@ ${run.workflowVersion}`;
9604
11204
  const error2 = {
9605
11205
  message: "cancelled"
9606
11206
  };
9607
- await this.store.updateRun(run.id, {
11207
+ await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
9608
11208
  status: "cancelled",
9609
11209
  error: error2,
9610
11210
  updatedAt: /* @__PURE__ */ new Date()
9611
- });
11211
+ }));
9612
11212
  this.emit({
9613
11213
  type: "run.failed",
9614
11214
  runId: run.id,
@@ -9655,7 +11255,8 @@ ${run.workflowVersion}`;
9655
11255
  type: "run.started",
9656
11256
  runId: run.id,
9657
11257
  workflow: run.workflow,
9658
- namespace: run.namespace
11258
+ namespace: run.namespace,
11259
+ ...await this.sourceTraceOf(run)
9659
11260
  });
9660
11261
  }
9661
11262
  await this.noteServedWakes(run.id);
@@ -9704,6 +11305,8 @@ ${run.workflowVersion}`;
9704
11305
  runId: run.id,
9705
11306
  workflow: run.workflow,
9706
11307
  namespace: run.namespace
11308
+ }, {
11309
+ lifecycle: false
9707
11310
  });
9708
11311
  void this.notifyParent(run.id, {
9709
11312
  ok: true,
@@ -11695,6 +13298,13 @@ function runDetailEtag(run, stats, over) {
11695
13298
  return `"rd-${hash(`${over}${JSON.stringify(run)}${statsFingerprint(stats)}`)}"`;
11696
13299
  }
11697
13300
  __name(runDetailEtag, "runDetailEtag");
13301
+ function extendEtag(etag, extras) {
13302
+ if (extras === void 0) return etag;
13303
+ const weak = etag.startsWith("W/") ? "W/" : "";
13304
+ const bare = etag.slice(weak.length).replace(/^"|"$/g, "");
13305
+ return `${weak}"${bare}.${hash(JSON.stringify(extras))}"`;
13306
+ }
13307
+ __name(extendEtag, "extendEtag");
11698
13308
  function fanPageEtag(group, stats, query) {
11699
13309
  const { offset, limit } = fanPageBounds(query);
11700
13310
  const key = [
@@ -12083,7 +13693,82 @@ var CodecStateStore = class {
12083
13693
  }
12084
13694
  };
12085
13695
 
13696
+ // src/lifecycle-deliveries.ts
13697
+ function lifecycleDeliveriesOf(run, opts) {
13698
+ const kind = lifecycleKindOf(run.status);
13699
+ if (!kind) return [];
13700
+ const tags = run.tags ?? [];
13701
+ const owed = new Set(owedSubscribersOf(tags));
13702
+ const triggered = new Set(tags.filter((t) => t.startsWith(LIFECYCLE_TRIGGERED_TAG_PREFIX)).map((t) => t.slice(LIFECYCLE_TRIGGERED_TAG_PREFIX.length)));
13703
+ const coalescedTagged = new Set(coalescedSubscribersOf(tags));
13704
+ const triggerFailed = new Set(triggerFailedSubscribersOf(tags));
13705
+ const owedToAll = owed.size === 0 && tags.includes(LIFECYCLE_PENDING_TAG);
13706
+ const settledAt = new Date(run.updatedAt).getTime();
13707
+ const names = [
13708
+ .../* @__PURE__ */ new Set([
13709
+ ...opts.subscribers,
13710
+ ...owed,
13711
+ ...triggered,
13712
+ ...coalescedTagged,
13713
+ ...triggerFailed
13714
+ ])
13715
+ ].sort();
13716
+ const coalesced = /* @__PURE__ */ __name((subscriber) => {
13717
+ const runId = opts.coalesced?.get(subscriber);
13718
+ return {
13719
+ subscriber,
13720
+ state: "coalesced",
13721
+ ...runId !== void 0 ? {
13722
+ runId
13723
+ } : {}
13724
+ };
13725
+ }, "coalesced");
13726
+ return names.map((subscriber) => {
13727
+ if (triggered.has(subscriber)) {
13728
+ return {
13729
+ subscriber,
13730
+ state: "started",
13731
+ runId: lifecycleRunId(run.id, kind, subscriber)
13732
+ };
13733
+ }
13734
+ if (coalescedTagged.has(subscriber)) return coalesced(subscriber);
13735
+ if (triggerFailed.has(subscriber)) return {
13736
+ subscriber,
13737
+ state: "failed"
13738
+ };
13739
+ if (owed.has(subscriber) || owedToAll) {
13740
+ return {
13741
+ subscriber,
13742
+ state: "owed",
13743
+ owedSince: settledAt,
13744
+ ...opts.live.has(subscriber) ? {} : {
13745
+ orphanAt: settledAt + LIFECYCLE_ORPHAN_MS
13746
+ }
13747
+ };
13748
+ }
13749
+ if (opts.coalesced?.has(subscriber)) return coalesced(subscriber);
13750
+ return {
13751
+ subscriber,
13752
+ state: "not-started"
13753
+ };
13754
+ });
13755
+ }
13756
+ __name(lifecycleDeliveriesOf, "lifecycleDeliveriesOf");
13757
+ function traceIdOfTraceparent(traceparent) {
13758
+ const traceId = traceparent?.split("-")[1];
13759
+ return traceId && /^[0-9a-f]{32}$/.test(traceId) && !/^0+$/.test(traceId) ? traceId : void 0;
13760
+ }
13761
+ __name(traceIdOfTraceparent, "traceIdOfTraceparent");
13762
+
12086
13763
  // src/metrics.ts
13764
+ function countBySubscriber(counts, e) {
13765
+ const workflow = e.workflow ?? "unknown";
13766
+ const subscriber = e.name ?? "unknown";
13767
+ const bySubscriber = counts[workflow] ?? {};
13768
+ bySubscriber[subscriber] = (bySubscriber[subscriber] ?? 0) + 1;
13769
+ counts[workflow] = bySubscriber;
13770
+ }
13771
+ __name(countBySubscriber, "countBySubscriber");
12087
13772
  function collectMetrics(engine) {
12088
13773
  const m = {
12089
13774
  runs: {
@@ -12095,6 +13780,8 @@ function collectMetrics(engine) {
12095
13780
  unrouted: 0
12096
13781
  },
12097
13782
  routedByCode: {},
13783
+ triggeredBySubscriber: {},
13784
+ triggerFailedBySubscriber: {},
12098
13785
  steps: {
12099
13786
  started: 0,
12100
13787
  completed: 0,
@@ -12146,6 +13833,12 @@ function collectMetrics(engine) {
12146
13833
  case "run.unrouted":
12147
13834
  m.runs.unrouted += 1;
12148
13835
  break;
13836
+ case "run.triggered":
13837
+ countBySubscriber(m.triggeredBySubscriber, e);
13838
+ break;
13839
+ case "run.trigger_failed":
13840
+ countBySubscriber(m.triggerFailedBySubscriber, e);
13841
+ break;
12149
13842
  case "step.started":
12150
13843
  m.steps.started += 1;
12151
13844
  break;
@@ -12174,6 +13867,10 @@ function collectMetrics(engine) {
12174
13867
  for (const [name, v] of Object.entries(m.byWorkflow)) for (const [event, n] of Object.entries(v)) lines.push(`durable_runs_by_workflow_total{workflow="${name}",event="${event}"} ${n}`);
12175
13868
  lines.push("# TYPE durable_runs_routed_total counter");
12176
13869
  for (const [code, n] of Object.entries(m.routedByCode)) lines.push(`durable_runs_routed_total{code="${code}"} ${n}`);
13870
+ lines.push("# TYPE durable_runs_triggered_total counter");
13871
+ for (const [workflow, bySubscriber] of Object.entries(m.triggeredBySubscriber)) for (const [subscriber, n] of Object.entries(bySubscriber)) lines.push(`durable_runs_triggered_total{workflow="${workflow}",subscriber="${subscriber}"} ${n}`);
13872
+ lines.push("# TYPE durable_runs_trigger_failed_total counter");
13873
+ for (const [workflow, bySubscriber] of Object.entries(m.triggerFailedBySubscriber)) for (const [subscriber, n] of Object.entries(bySubscriber)) lines.push(`durable_runs_trigger_failed_total{workflow="${workflow}",subscriber="${subscriber}"} ${n}`);
12177
13874
  lines.push("# TYPE durable_step_duration_ms_sum counter");
12178
13875
  lines.push(`durable_step_duration_ms_sum ${m.stepDuration.sumMs}`);
12179
13876
  lines.push("# TYPE durable_step_duration_count counter");
@@ -12321,6 +14018,9 @@ var TRANSPORT_CANONICAL = Symbol.for("@dudousxd/nestjs-durable:transport");
12321
14018
  var DURABLE_OPTIONS_CANONICAL = Symbol.for("@dudousxd/nestjs-durable:options");
12322
14019
 
12323
14020
  // src/testing/in-memory-state-store.ts
14021
+ var RETENTION_EXCLUDED = [
14022
+ LIFECYCLE_REGISTRY_WORKFLOW
14023
+ ];
12324
14024
  var InMemoryStateStore = class {
12325
14025
  static {
12326
14026
  __name(this, "InMemoryStateStore");
@@ -12450,12 +14150,13 @@ var InMemoryStateStore = class {
12450
14150
  if (policy.maxAge == null && policy.maxCount == null) return 0;
12451
14151
  const inScope = this.matching({
12452
14152
  ...policy.scope ?? {},
12453
- statuses
14153
+ statuses,
14154
+ excludeWorkflows: RETENTION_EXCLUDED
12454
14155
  });
12455
14156
  const ids = /* @__PURE__ */ new Set();
12456
14157
  if (policy.maxAge != null) {
12457
14158
  const cutoff = nowMs - parseDuration(policy.maxAge);
12458
- const expired = inScope.filter((r) => r.updatedAt.getTime() < cutoff).sort((a, b) => a.updatedAt.getTime() - b.updatedAt.getTime()).slice(0, limit);
14159
+ const expired = inScope.filter((r) => r.updatedAt.getTime() < cutoff && !retentionKeeps(r.tags)).sort((a, b) => a.updatedAt.getTime() - b.updatedAt.getTime()).slice(0, limit);
12459
14160
  for (const r of expired) ids.add(r.id);
12460
14161
  }
12461
14162
  if (policy.maxCount != null && ids.size < limit) {
@@ -12464,6 +14165,7 @@ var InMemoryStateStore = class {
12464
14165
  ].sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime() || b.id.localeCompare(a.id));
12465
14166
  for (const r of byRecency.slice(policy.maxCount)) {
12466
14167
  if (ids.size >= limit) break;
14168
+ if (retentionKeeps(r.tags)) continue;
12467
14169
  ids.add(r.id);
12468
14170
  }
12469
14171
  }
@@ -12761,6 +14463,13 @@ var InMemoryStateStore = class {
12761
14463
  ];
12762
14464
  if (query.workflow) runs = runs.filter((r) => r.workflow === query.workflow);
12763
14465
  if (query.workflows) runs = runs.filter((r) => query.workflows?.includes(r.workflow));
14466
+ if (query.excludeWorkflows?.length) {
14467
+ runs = runs.filter((r) => !query.excludeWorkflows?.includes(r.workflow));
14468
+ }
14469
+ if (query.updatedAfter !== void 0) {
14470
+ const after = query.updatedAfter;
14471
+ runs = runs.filter((r) => r.updatedAt.getTime() >= after);
14472
+ }
12764
14473
  if (query.status) runs = runs.filter((r) => r.status === query.status);
12765
14474
  if (query.statuses) runs = runs.filter((r) => query.statuses?.includes(r.status));
12766
14475
  if (query.tag) runs = runs.filter((r) => r.tags?.includes(query.tag));
@@ -13056,6 +14765,7 @@ var InMemoryRunQueue = class {
13056
14765
  }
13057
14766
  };
13058
14767
  export {
14768
+ ACCUMULATOR_SEEN_MAX,
13059
14769
  CHILD_EDGE_PREFIXES,
13060
14770
  CONTROL_FLOW_SIGNAL,
13061
14771
  CORE_ORIGIN,
@@ -13087,6 +14797,19 @@ export {
13087
14797
  InMemoryTransport,
13088
14798
  LEGACY_V1_CAPABILITIES,
13089
14799
  LEGACY_V1_PROTOCOL,
14800
+ LIFECYCLE_COALESCED_TAG_PREFIX,
14801
+ LIFECYCLE_HEAL_WINDOW_MS,
14802
+ LIFECYCLE_ORPHAN_MS,
14803
+ LIFECYCLE_OWED_TAG_PREFIX,
14804
+ LIFECYCLE_PENDING_TAG,
14805
+ LIFECYCLE_REGISTRY_HEARTBEAT_MS,
14806
+ LIFECYCLE_REGISTRY_STALE_MS,
14807
+ LIFECYCLE_REGISTRY_TAG,
14808
+ LIFECYCLE_REGISTRY_TTL_MS,
14809
+ LIFECYCLE_REGISTRY_WORKFLOW,
14810
+ LIFECYCLE_TRACE_TAG_PREFIX,
14811
+ LIFECYCLE_TRIGGERED_TAG_PREFIX,
14812
+ LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX,
13090
14813
  NestedWorkflowCallError,
13091
14814
  NonDeterminismError,
13092
14815
  QueueController,
@@ -13094,6 +14817,7 @@ export {
13094
14817
  RUN_HISTORY_FIXTURE_VERSION,
13095
14818
  RUN_VALUE_FACET_LIMIT,
13096
14819
  RUN_VALUE_FACET_SCAN,
14820
+ RUN_WAIT_TOKEN_PREFIX,
13097
14821
  RemoteStepError,
13098
14822
  RemoteStepTimeout,
13099
14823
  RemoteWorkflowExecutor,
@@ -13122,6 +14846,8 @@ export {
13122
14846
  WorkflowEngine,
13123
14847
  WorkflowNotRegistered,
13124
14848
  WorkflowSuspended,
14849
+ acceptingSubscription,
14850
+ accumulatorRunId,
13125
14851
  aggregateAnnouncements,
13126
14852
  aggregatesCheckpoints,
13127
14853
  announcementsOf,
@@ -13137,12 +14863,14 @@ export {
13137
14863
  buildWorkflowDirectory,
13138
14864
  canRoute,
13139
14865
  capableWorkers,
14866
+ carriesLifecyclePending,
13140
14867
  checkpointNameKind,
13141
14868
  checkpointStatsOf,
13142
14869
  childIdsOf,
13143
14870
  classifyWaiterToken,
13144
14871
  clearDurableExecutionWrappers,
13145
14872
  clearTerminalSingletonAdmission,
14873
+ coalescedSubscribersOf,
13146
14874
  codeAndDetails,
13147
14875
  codeOf,
13148
14876
  collectMetrics,
@@ -13165,6 +14893,7 @@ export {
13165
14893
  eventNameOf,
13166
14894
  eventPrefix,
13167
14895
  eventToken,
14896
+ extendEtag,
13168
14897
  facetOrigin,
13169
14898
  fanPageBounds,
13170
14899
  fanPageEtag,
@@ -13182,10 +14911,21 @@ export {
13182
14911
  isEventToken,
13183
14912
  isFailedMember,
13184
14913
  isLegacyDescriptor,
14914
+ isLifecycleTag,
13185
14915
  isNonEmptyRunScope,
13186
14916
  isRemoteEvent,
13187
14917
  isWorkflowControlFlowSignal,
13188
14918
  legacySingletonHolders,
14919
+ lifecycleDeliveriesOf,
14920
+ lifecycleEventName,
14921
+ lifecycleEventOf,
14922
+ lifecycleKindOf,
14923
+ lifecycleRegistryNamespace,
14924
+ lifecycleRegistryRunId,
14925
+ lifecycleRegistrySubscriptions,
14926
+ lifecycleRunId,
14927
+ lifecycleSourceOf,
14928
+ lifecycleTraceparentOf,
13189
14929
  listSignalWaitersOfRuns,
13190
14930
  liveHandoff,
13191
14931
  log,
@@ -13200,10 +14940,12 @@ export {
13200
14940
  normalizeDescriptor,
13201
14941
  observedAnnouncements,
13202
14942
  onErrorRunId,
14943
+ owedSubscribersOf,
13203
14944
  pageFan,
13204
14945
  pageFanFromStore,
13205
14946
  parentRunIdOf,
13206
14947
  parseDuration,
14948
+ parseLifecycleEvent,
13207
14949
  parseWebhookToken,
13208
14950
  partitionOfToken,
13209
14951
  planDispatch,
@@ -13218,10 +14960,13 @@ export {
13218
14960
  resolveRunWaiting,
13219
14961
  resolveTenantToken,
13220
14962
  resolveWireTenant,
14963
+ retentionKeeps,
13221
14964
  retiredHandoffName,
13222
14965
  routeFor,
14966
+ runCompletedEvent,
13223
14967
  runDetailEtag,
13224
14968
  runDurableExecution,
14969
+ runFailedEvent,
13225
14970
  runInStepLogger,
13226
14971
  runInStepScope,
13227
14972
  runInWorkflowCtx,
@@ -13229,6 +14974,9 @@ export {
13229
14974
  runSchedules,
13230
14975
  runStepHandler,
13231
14976
  runValueFacetsFromRuns,
14977
+ runWaitPrefix,
14978
+ runWaitTargetOf,
14979
+ runWaitToken,
13232
14980
  sanitizeQueueToken,
13233
14981
  scheduleRunIdAt,
13234
14982
  scheduledRunId,
@@ -13247,6 +14995,11 @@ export {
13247
14995
  summarizeRunFromStore,
13248
14996
  summarizeTimeline,
13249
14997
  tenantGroup,
14998
+ toEventSubscription,
14999
+ traceIdOfTraceparent,
15000
+ triggerFailedSubscribersOf,
15001
+ triggeredRunsOf,
15002
+ triggeredSubscribersOf,
13250
15003
  useDurableExecution,
13251
15004
  warn,
13252
15005
  withSingletonTracked,