@dudousxd/nestjs-durable-core 0.86.1 → 0.87.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +2052 -217
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +913 -71
- package/dist/index.d.ts +913 -71
- package/dist/index.js +2008 -217
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -813,6 +813,243 @@ function eventMatches(payload, match) {
|
|
|
813
813
|
return keys.every((k) => JSON.stringify(p[k]) === JSON.stringify(match[k]));
|
|
814
814
|
}
|
|
815
815
|
__name(eventMatches, "eventMatches");
|
|
816
|
+
function toEventSubscription(entry) {
|
|
817
|
+
return typeof entry === "string" ? {
|
|
818
|
+
event: entry
|
|
819
|
+
} : entry;
|
|
820
|
+
}
|
|
821
|
+
__name(toEventSubscription, "toEventSubscription");
|
|
822
|
+
function acceptingSubscription(subscriptions, payload) {
|
|
823
|
+
const errors = [];
|
|
824
|
+
for (const subscription of subscriptions) {
|
|
825
|
+
if (!subscription.where) return {
|
|
826
|
+
accepted: subscription,
|
|
827
|
+
errors
|
|
828
|
+
};
|
|
829
|
+
try {
|
|
830
|
+
if (subscription.where(payload)) return {
|
|
831
|
+
accepted: subscription,
|
|
832
|
+
errors
|
|
833
|
+
};
|
|
834
|
+
} catch (error2) {
|
|
835
|
+
errors.push(error2);
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
return {
|
|
839
|
+
errors
|
|
840
|
+
};
|
|
841
|
+
}
|
|
842
|
+
__name(acceptingSubscription, "acceptingSubscription");
|
|
843
|
+
|
|
844
|
+
// src/lifecycle-events.ts
|
|
845
|
+
function runCompletedEvent(workflow) {
|
|
846
|
+
return `run.completed:${workflowName(workflow)}`;
|
|
847
|
+
}
|
|
848
|
+
__name(runCompletedEvent, "runCompletedEvent");
|
|
849
|
+
function runFailedEvent(workflow) {
|
|
850
|
+
return `run.failed:${workflowName(workflow)}`;
|
|
851
|
+
}
|
|
852
|
+
__name(runFailedEvent, "runFailedEvent");
|
|
853
|
+
var LIFECYCLE_PENDING_TAG = "durable:lifecycle:pending";
|
|
854
|
+
function carriesLifecyclePending(tags) {
|
|
855
|
+
if (Array.isArray(tags)) return tags.includes(LIFECYCLE_PENDING_TAG);
|
|
856
|
+
if (typeof tags === "string") return tags.includes(`"${LIFECYCLE_PENDING_TAG}"`);
|
|
857
|
+
return false;
|
|
858
|
+
}
|
|
859
|
+
__name(carriesLifecyclePending, "carriesLifecyclePending");
|
|
860
|
+
var LIFECYCLE_TRIGGERED_TAG_PREFIX = "durable:triggered:";
|
|
861
|
+
var LIFECYCLE_OWED_TAG_PREFIX = "durable:lifecycle:owed:";
|
|
862
|
+
var LIFECYCLE_ORPHAN_MS = 24 * 60 * 60 * 1e3;
|
|
863
|
+
var LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX = "durable:trigger_failed:";
|
|
864
|
+
function triggerFailedSubscribersOf(tags) {
|
|
865
|
+
return (tags ?? []).filter((tag) => tag.startsWith(LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX)).map((tag) => tag.slice(LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX.length));
|
|
866
|
+
}
|
|
867
|
+
__name(triggerFailedSubscribersOf, "triggerFailedSubscribersOf");
|
|
868
|
+
function triggeredSubscribersOf(tags) {
|
|
869
|
+
return (tags ?? []).filter((tag) => tag.startsWith(LIFECYCLE_TRIGGERED_TAG_PREFIX)).map((tag) => tag.slice(LIFECYCLE_TRIGGERED_TAG_PREFIX.length));
|
|
870
|
+
}
|
|
871
|
+
__name(triggeredSubscribersOf, "triggeredSubscribersOf");
|
|
872
|
+
var LIFECYCLE_COALESCED_TAG_PREFIX = "durable:coalesced:";
|
|
873
|
+
function coalescedSubscribersOf(tags) {
|
|
874
|
+
return (tags ?? []).filter((tag) => tag.startsWith(LIFECYCLE_COALESCED_TAG_PREFIX)).map((tag) => tag.slice(LIFECYCLE_COALESCED_TAG_PREFIX.length));
|
|
875
|
+
}
|
|
876
|
+
__name(coalescedSubscribersOf, "coalescedSubscribersOf");
|
|
877
|
+
var LIFECYCLE_HEAL_WINDOW_MS = 2 * 60 * 1e3;
|
|
878
|
+
function owedSubscribersOf(tags) {
|
|
879
|
+
return (tags ?? []).filter((tag) => tag.startsWith(LIFECYCLE_OWED_TAG_PREFIX)).map((tag) => tag.slice(LIFECYCLE_OWED_TAG_PREFIX.length));
|
|
880
|
+
}
|
|
881
|
+
__name(owedSubscribersOf, "owedSubscribersOf");
|
|
882
|
+
var LIFECYCLE_TRACE_TAG_PREFIX = "durable:lifecycle:trace:";
|
|
883
|
+
function lifecycleTraceparentOf(tags) {
|
|
884
|
+
const tag = (tags ?? []).find((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX));
|
|
885
|
+
return tag?.slice(LIFECYCLE_TRACE_TAG_PREFIX.length) || void 0;
|
|
886
|
+
}
|
|
887
|
+
__name(lifecycleTraceparentOf, "lifecycleTraceparentOf");
|
|
888
|
+
function isLifecycleTag(tag) {
|
|
889
|
+
return tag === LIFECYCLE_PENDING_TAG || tag.startsWith(LIFECYCLE_OWED_TAG_PREFIX) || tag.startsWith(LIFECYCLE_TRIGGERED_TAG_PREFIX) || tag.startsWith(LIFECYCLE_COALESCED_TAG_PREFIX) || tag.startsWith(LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX) || tag.startsWith(LIFECYCLE_TRACE_TAG_PREFIX);
|
|
890
|
+
}
|
|
891
|
+
__name(isLifecycleTag, "isLifecycleTag");
|
|
892
|
+
function lifecycleRunId(sourceRunId, kind, subscriber) {
|
|
893
|
+
return `evt:${sourceRunId}:${kind}:${subscriber}`;
|
|
894
|
+
}
|
|
895
|
+
__name(lifecycleRunId, "lifecycleRunId");
|
|
896
|
+
function lifecycleKindOf(status) {
|
|
897
|
+
if (status === "completed") return "completed";
|
|
898
|
+
if (status === "failed" || status === "cancelled" || status === "dead") return "failed";
|
|
899
|
+
return void 0;
|
|
900
|
+
}
|
|
901
|
+
__name(lifecycleKindOf, "lifecycleKindOf");
|
|
902
|
+
function lifecycleEventName(workflow, kind) {
|
|
903
|
+
return kind === "completed" ? runCompletedEvent(workflow) : runFailedEvent(workflow);
|
|
904
|
+
}
|
|
905
|
+
__name(lifecycleEventName, "lifecycleEventName");
|
|
906
|
+
function parseLifecycleEvent(event) {
|
|
907
|
+
for (const kind of [
|
|
908
|
+
"completed",
|
|
909
|
+
"failed"
|
|
910
|
+
]) {
|
|
911
|
+
const prefix = `run.${kind}:`;
|
|
912
|
+
if (event.startsWith(prefix) && event.length > prefix.length) {
|
|
913
|
+
return {
|
|
914
|
+
workflow: event.slice(prefix.length),
|
|
915
|
+
kind
|
|
916
|
+
};
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
return void 0;
|
|
920
|
+
}
|
|
921
|
+
__name(parseLifecycleEvent, "parseLifecycleEvent");
|
|
922
|
+
function lifecycleSourceOf(run) {
|
|
923
|
+
if (!run.id.startsWith("evt:")) return void 0;
|
|
924
|
+
for (const kind of [
|
|
925
|
+
"completed",
|
|
926
|
+
"failed"
|
|
927
|
+
]) {
|
|
928
|
+
const suffix = `:${kind}:${run.workflow}`;
|
|
929
|
+
if (run.id.endsWith(suffix) && run.id.length > "evt:".length + suffix.length) {
|
|
930
|
+
return {
|
|
931
|
+
runId: run.id.slice("evt:".length, -suffix.length),
|
|
932
|
+
kind
|
|
933
|
+
};
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
return void 0;
|
|
937
|
+
}
|
|
938
|
+
__name(lifecycleSourceOf, "lifecycleSourceOf");
|
|
939
|
+
function triggeredRunsOf(run) {
|
|
940
|
+
const kind = lifecycleKindOf(run.status);
|
|
941
|
+
if (!kind) return [];
|
|
942
|
+
return (run.tags ?? []).filter((tag) => tag.startsWith(LIFECYCLE_TRIGGERED_TAG_PREFIX)).map((tag) => lifecycleRunId(run.id, kind, tag.slice(LIFECYCLE_TRIGGERED_TAG_PREFIX.length)));
|
|
943
|
+
}
|
|
944
|
+
__name(triggeredRunsOf, "triggeredRunsOf");
|
|
945
|
+
function lifecycleEventOf(run) {
|
|
946
|
+
const completed = run.status === "completed";
|
|
947
|
+
return {
|
|
948
|
+
runId: run.id,
|
|
949
|
+
workflow: run.workflow,
|
|
950
|
+
workflowVersion: run.workflowVersion,
|
|
951
|
+
status: run.status,
|
|
952
|
+
namespace: run.namespace,
|
|
953
|
+
input: run.input,
|
|
954
|
+
...completed ? {
|
|
955
|
+
output: run.output
|
|
956
|
+
} : {
|
|
957
|
+
error: run.error
|
|
958
|
+
},
|
|
959
|
+
tags: run.tags?.filter((tag) => !isLifecycleTag(tag)),
|
|
960
|
+
searchAttributes: run.searchAttributes
|
|
961
|
+
};
|
|
962
|
+
}
|
|
963
|
+
__name(lifecycleEventOf, "lifecycleEventOf");
|
|
964
|
+
var LIFECYCLE_REGISTRY_WORKFLOW = "__lifecycle_registry";
|
|
965
|
+
var LIFECYCLE_REGISTRY_TAG = "durable:lifecycle:registry";
|
|
966
|
+
var LIFECYCLE_REGISTRY_HEARTBEAT_MS = 1e4;
|
|
967
|
+
var LIFECYCLE_REGISTRY_TTL_MS = 35e3;
|
|
968
|
+
var LIFECYCLE_REGISTRY_STALE_MS = 10 * 60 * 1e3;
|
|
969
|
+
function lifecycleRegistryRunId(instanceId) {
|
|
970
|
+
return `${LIFECYCLE_REGISTRY_WORKFLOW}:${instanceId}`;
|
|
971
|
+
}
|
|
972
|
+
__name(lifecycleRegistryRunId, "lifecycleRegistryRunId");
|
|
973
|
+
function lifecycleRegistryNamespace(input, rowNamespace) {
|
|
974
|
+
const engineNamespace = input?.engineNamespace;
|
|
975
|
+
if (engineNamespace === null) return null;
|
|
976
|
+
if (typeof engineNamespace === "string") return engineNamespace || "default";
|
|
977
|
+
const row = rowNamespace || "default";
|
|
978
|
+
return row === "default" ? null : row;
|
|
979
|
+
}
|
|
980
|
+
__name(lifecycleRegistryNamespace, "lifecycleRegistryNamespace");
|
|
981
|
+
function lifecycleRegistrySubscriptions(input) {
|
|
982
|
+
const subscriptions = input?.subscriptions;
|
|
983
|
+
if (!Array.isArray(subscriptions)) return [];
|
|
984
|
+
return subscriptions.flatMap((entry) => {
|
|
985
|
+
const { workflow, events } = entry ?? {};
|
|
986
|
+
if (typeof workflow !== "string" || !Array.isArray(events)) return [];
|
|
987
|
+
return [
|
|
988
|
+
{
|
|
989
|
+
workflow,
|
|
990
|
+
events: events.filter((e) => typeof e === "string")
|
|
991
|
+
}
|
|
992
|
+
];
|
|
993
|
+
});
|
|
994
|
+
}
|
|
995
|
+
__name(lifecycleRegistrySubscriptions, "lifecycleRegistrySubscriptions");
|
|
996
|
+
function retentionKeeps(tags) {
|
|
997
|
+
if (carriesLifecyclePending(tags)) return true;
|
|
998
|
+
if (Array.isArray(tags)) return tags.includes(LIFECYCLE_REGISTRY_TAG);
|
|
999
|
+
if (typeof tags === "string") return tags.includes(`"${LIFECYCLE_REGISTRY_TAG}"`);
|
|
1000
|
+
return false;
|
|
1001
|
+
}
|
|
1002
|
+
__name(retentionKeeps, "retentionKeeps");
|
|
1003
|
+
|
|
1004
|
+
// src/run-wait.ts
|
|
1005
|
+
var RUN_WAIT_TOKEN_PREFIX = "runwait:";
|
|
1006
|
+
function runWaitPrefix(targetRunId) {
|
|
1007
|
+
return `${RUN_WAIT_TOKEN_PREFIX}${targetRunId}#`;
|
|
1008
|
+
}
|
|
1009
|
+
__name(runWaitPrefix, "runWaitPrefix");
|
|
1010
|
+
function runWaitToken(targetRunId, runId, seq) {
|
|
1011
|
+
return `${runWaitPrefix(targetRunId)}${runId}#${seq}`;
|
|
1012
|
+
}
|
|
1013
|
+
__name(runWaitToken, "runWaitToken");
|
|
1014
|
+
function runWaitTargetOf(waiter) {
|
|
1015
|
+
const suffix = `#${waiter.runId}#${waiter.seq}`;
|
|
1016
|
+
const { token } = waiter;
|
|
1017
|
+
if (!token.startsWith(RUN_WAIT_TOKEN_PREFIX) || !token.endsWith(suffix)) return void 0;
|
|
1018
|
+
const target = token.slice(RUN_WAIT_TOKEN_PREFIX.length, token.length - suffix.length);
|
|
1019
|
+
return target.length > 0 ? target : void 0;
|
|
1020
|
+
}
|
|
1021
|
+
__name(runWaitTargetOf, "runWaitTargetOf");
|
|
1022
|
+
function nextContinuationId(runId) {
|
|
1023
|
+
const m = runId.match(/^(.*)~(\d+)$/);
|
|
1024
|
+
return m ? `${m[1]}~${Number(m[2]) + 1}` : `${runId}~1`;
|
|
1025
|
+
}
|
|
1026
|
+
__name(nextContinuationId, "nextContinuationId");
|
|
1027
|
+
var MAX_CONTINUATION_HOPS = 100;
|
|
1028
|
+
async function settledRunOf(store, runId) {
|
|
1029
|
+
let run = await store.getRun(runId);
|
|
1030
|
+
if (!run) return {
|
|
1031
|
+
state: "missing"
|
|
1032
|
+
};
|
|
1033
|
+
for (let hop = 0; hop <= MAX_CONTINUATION_HOPS; hop += 1) {
|
|
1034
|
+
if (!lifecycleKindOf(run.status)) return {
|
|
1035
|
+
state: "running"
|
|
1036
|
+
};
|
|
1037
|
+
if (run.status !== "completed") return {
|
|
1038
|
+
state: "settled",
|
|
1039
|
+
run
|
|
1040
|
+
};
|
|
1041
|
+
const next = await store.getRun(nextContinuationId(run.id));
|
|
1042
|
+
if (!next) return {
|
|
1043
|
+
state: "settled",
|
|
1044
|
+
run
|
|
1045
|
+
};
|
|
1046
|
+
run = next;
|
|
1047
|
+
}
|
|
1048
|
+
return {
|
|
1049
|
+
state: "running"
|
|
1050
|
+
};
|
|
1051
|
+
}
|
|
1052
|
+
__name(settledRunOf, "settledRunOf");
|
|
816
1053
|
|
|
817
1054
|
// src/run-waiting.ts
|
|
818
1055
|
var WEBHOOK_PREFIX = "wh:";
|
|
@@ -832,7 +1069,17 @@ function parseWebhookToken(token) {
|
|
|
832
1069
|
};
|
|
833
1070
|
}
|
|
834
1071
|
__name(parseWebhookToken, "parseWebhookToken");
|
|
835
|
-
function classifyWaiterToken(token) {
|
|
1072
|
+
function classifyWaiterToken(token, waiter) {
|
|
1073
|
+
if (token.startsWith(RUN_WAIT_TOKEN_PREFIX)) {
|
|
1074
|
+
const target = waiter ? runWaitTargetOf({
|
|
1075
|
+
token,
|
|
1076
|
+
...waiter
|
|
1077
|
+
}) : token.slice(RUN_WAIT_TOKEN_PREFIX.length).match(/^(.*)#[^#]*#\d+$/)?.[1];
|
|
1078
|
+
if (target) return {
|
|
1079
|
+
on: "run",
|
|
1080
|
+
name: target
|
|
1081
|
+
};
|
|
1082
|
+
}
|
|
836
1083
|
if (token.startsWith(WEBHOOK_PREFIX)) return {
|
|
837
1084
|
on: "webhook",
|
|
838
1085
|
name: token
|
|
@@ -858,7 +1105,7 @@ __name(classifyWaiterToken, "classifyWaiterToken");
|
|
|
858
1105
|
function resolveRunWaiting(run, waiterByRun) {
|
|
859
1106
|
if (run.status !== "suspended") return void 0;
|
|
860
1107
|
const waiter = waiterByRun.get(run.id);
|
|
861
|
-
return waiter ? classifyWaiterToken(waiter.token) : void 0;
|
|
1108
|
+
return waiter ? classifyWaiterToken(waiter.token, waiter) : void 0;
|
|
862
1109
|
}
|
|
863
1110
|
__name(resolveRunWaiting, "resolveRunWaiting");
|
|
864
1111
|
var RUN_ID_BATCH = 500;
|
|
@@ -1723,7 +1970,55 @@ var Entities = class {
|
|
|
1723
1970
|
};
|
|
1724
1971
|
|
|
1725
1972
|
// src/event-accumulators.ts
|
|
1726
|
-
|
|
1973
|
+
function accumulatorRunId(target, key) {
|
|
1974
|
+
if (key === void 0) return `__evtacc__:${target}`;
|
|
1975
|
+
const escaped = encodeURIComponent(key).replace(/[~!'()*.]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
|
|
1976
|
+
return `__evtacc__:${target}:${escaped}`;
|
|
1977
|
+
}
|
|
1978
|
+
__name(accumulatorRunId, "accumulatorRunId");
|
|
1979
|
+
var ACCUMULATOR_SEEN_MAX = 1e3;
|
|
1980
|
+
function isRoutedEvent(signal) {
|
|
1981
|
+
return typeof signal === "object" && signal !== null && signal.__evtacc === 1 && typeof signal.id === "string";
|
|
1982
|
+
}
|
|
1983
|
+
__name(isRoutedEvent, "isRoutedEvent");
|
|
1984
|
+
var SeenEvents = class SeenEvents2 {
|
|
1985
|
+
static {
|
|
1986
|
+
__name(this, "SeenEvents");
|
|
1987
|
+
}
|
|
1988
|
+
ids;
|
|
1989
|
+
order;
|
|
1990
|
+
constructor(seen) {
|
|
1991
|
+
this.order = [
|
|
1992
|
+
...seen ?? []
|
|
1993
|
+
];
|
|
1994
|
+
this.ids = new Set(this.order);
|
|
1995
|
+
}
|
|
1996
|
+
fresh(signal) {
|
|
1997
|
+
if (!isRoutedEvent(signal)) return {
|
|
1998
|
+
fresh: true,
|
|
1999
|
+
payload: signal
|
|
2000
|
+
};
|
|
2001
|
+
if (this.ids.has(signal.id)) return {
|
|
2002
|
+
fresh: false
|
|
2003
|
+
};
|
|
2004
|
+
this.ids.add(signal.id);
|
|
2005
|
+
this.order.push(signal.id);
|
|
2006
|
+
return {
|
|
2007
|
+
fresh: true,
|
|
2008
|
+
payload: signal.payload
|
|
2009
|
+
};
|
|
2010
|
+
}
|
|
2011
|
+
/** The input to re-arm with: `input` plus the newest ids, when there are any. */
|
|
2012
|
+
rearm(input) {
|
|
2013
|
+
const { seen: _seen, ...rest } = input;
|
|
2014
|
+
const seen = this.order.slice(-ACCUMULATOR_SEEN_MAX);
|
|
2015
|
+
return seen.length > 0 ? {
|
|
2016
|
+
...rest,
|
|
2017
|
+
seen
|
|
2018
|
+
} : rest;
|
|
2019
|
+
}
|
|
2020
|
+
};
|
|
2021
|
+
var accToken = /* @__PURE__ */ __name((input) => accumulatorRunId(input.target, input.key), "accToken");
|
|
1727
2022
|
var EventAccumulators = class {
|
|
1728
2023
|
static {
|
|
1729
2024
|
__name(this, "EventAccumulators");
|
|
@@ -1738,60 +2033,75 @@ var EventAccumulators = class {
|
|
|
1738
2033
|
origin: CORE_ORIGIN
|
|
1739
2034
|
});
|
|
1740
2035
|
}
|
|
1741
|
-
/**
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
"
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
}
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
2036
|
+
/**
|
|
2037
|
+
* Route an `onEvent` payload into its target's debounce/batch accumulator: the target's one, or
|
|
2038
|
+
* with a `key`, the one for `String(config.key(payload))`. A `key` that throws rejects the route.
|
|
2039
|
+
* With `id`, the accumulator counts the event once however many times it is routed.
|
|
2040
|
+
*/
|
|
2041
|
+
route(target, config, payload, id) {
|
|
2042
|
+
const input = {
|
|
2043
|
+
target,
|
|
2044
|
+
windowMs: config.windowMs,
|
|
2045
|
+
...config.mode === "batch" ? {
|
|
2046
|
+
maxSize: config.maxSize
|
|
2047
|
+
} : {},
|
|
2048
|
+
...config.key ? {
|
|
2049
|
+
key: String(config.key(payload))
|
|
2050
|
+
} : {}
|
|
2051
|
+
};
|
|
2052
|
+
const workflow = config.mode === "debounce" ? "__evt_debounce" : "__evt_batch";
|
|
2053
|
+
const token = accToken(input);
|
|
2054
|
+
const signal = id === void 0 ? payload : {
|
|
2055
|
+
__evtacc: 1,
|
|
2056
|
+
id,
|
|
1759
2057
|
payload
|
|
2058
|
+
};
|
|
2059
|
+
return this.host.signalWithStart(workflow, input, token, {
|
|
2060
|
+
token,
|
|
2061
|
+
payload: signal
|
|
1760
2062
|
});
|
|
1761
2063
|
}
|
|
1762
2064
|
/** Fire the target with the LAST payload once events have been quiet for `windowMs`. */
|
|
1763
2065
|
async debounce(ctx, input) {
|
|
1764
2066
|
const { target, windowMs } = input;
|
|
1765
|
-
const token = accToken(
|
|
1766
|
-
|
|
2067
|
+
const token = accToken(input);
|
|
2068
|
+
const seen = new SeenEvents(input.seen);
|
|
2069
|
+
let first = seen.fresh(await ctx.waitForSignal(token));
|
|
2070
|
+
while (!first.fresh) first = seen.fresh(await ctx.waitForSignal(token));
|
|
2071
|
+
let last = first.payload;
|
|
1767
2072
|
for (; ; ) {
|
|
1768
2073
|
try {
|
|
1769
|
-
|
|
2074
|
+
const next = seen.fresh(await ctx.waitForSignal(token, {
|
|
1770
2075
|
timeoutMs: windowMs
|
|
1771
|
-
});
|
|
2076
|
+
}));
|
|
2077
|
+
if (next.fresh) last = next.payload;
|
|
1772
2078
|
} catch (e) {
|
|
1773
2079
|
if (e instanceof SignalTimeoutError) break;
|
|
1774
2080
|
throw e;
|
|
1775
2081
|
}
|
|
1776
2082
|
}
|
|
1777
2083
|
await ctx.startChild(target, last);
|
|
1778
|
-
return ctx.continueAsNew(input);
|
|
2084
|
+
return ctx.continueAsNew(seen.rearm(input));
|
|
1779
2085
|
}
|
|
1780
2086
|
/** Fire the target with all payloads (`{ events }`) once `maxSize` or `windowMs` from the first. */
|
|
1781
2087
|
async batch(ctx, input) {
|
|
1782
2088
|
const { target, maxSize, windowMs } = input;
|
|
1783
|
-
const token = accToken(
|
|
2089
|
+
const token = accToken(input);
|
|
2090
|
+
const seen = new SeenEvents(input.seen);
|
|
2091
|
+
let first = seen.fresh(await ctx.waitForSignal(token));
|
|
2092
|
+
while (!first.fresh) first = seen.fresh(await ctx.waitForSignal(token));
|
|
1784
2093
|
const events = [
|
|
1785
|
-
|
|
2094
|
+
first.payload
|
|
1786
2095
|
];
|
|
1787
2096
|
const deadline = await ctx.now() + windowMs;
|
|
1788
2097
|
while (events.length < maxSize) {
|
|
1789
2098
|
const remaining = deadline - await ctx.now();
|
|
1790
2099
|
if (remaining <= 0) break;
|
|
1791
2100
|
try {
|
|
1792
|
-
|
|
2101
|
+
const next = seen.fresh(await ctx.waitForSignal(token, {
|
|
1793
2102
|
timeoutMs: remaining
|
|
1794
2103
|
}));
|
|
2104
|
+
if (next.fresh) events.push(next.payload);
|
|
1795
2105
|
} catch (e) {
|
|
1796
2106
|
if (e instanceof SignalTimeoutError) break;
|
|
1797
2107
|
throw e;
|
|
@@ -1800,7 +2110,7 @@ var EventAccumulators = class {
|
|
|
1800
2110
|
await ctx.startChild(target, {
|
|
1801
2111
|
events
|
|
1802
2112
|
});
|
|
1803
|
-
return ctx.continueAsNew(input);
|
|
2113
|
+
return ctx.continueAsNew(seen.rearm(input));
|
|
1804
2114
|
}
|
|
1805
2115
|
};
|
|
1806
2116
|
|
|
@@ -1871,6 +2181,11 @@ function canonicalizeRegistrations(regs) {
|
|
|
1871
2181
|
// Conditional, so a registration that states no routes keeps the hash it had before routes existed.
|
|
1872
2182
|
...r.onError !== void 0 ? {
|
|
1873
2183
|
onError: canonicalRoutes(r.onError)
|
|
2184
|
+
} : {},
|
|
2185
|
+
...r.subscribes !== void 0 ? {
|
|
2186
|
+
subscribes: [
|
|
2187
|
+
...new Set(r.subscribes)
|
|
2188
|
+
].sort()
|
|
1874
2189
|
} : {}
|
|
1875
2190
|
}));
|
|
1876
2191
|
const byJson = /* @__PURE__ */ new Map();
|
|
@@ -2100,6 +2415,9 @@ function aggregateAnnouncements(descriptors) {
|
|
|
2100
2415
|
origins,
|
|
2101
2416
|
requires,
|
|
2102
2417
|
onError,
|
|
2418
|
+
subscribes: [
|
|
2419
|
+
...new Set(announcements.flatMap((a) => a.subscribes ?? []))
|
|
2420
|
+
].sort(),
|
|
2103
2421
|
runtimes: [
|
|
2104
2422
|
...new Set(announcements.map((a) => a.runtime))
|
|
2105
2423
|
].sort(),
|
|
@@ -2134,6 +2452,7 @@ function observedAnnouncements(heartbeats, covered) {
|
|
|
2134
2452
|
origins: [],
|
|
2135
2453
|
requires: [],
|
|
2136
2454
|
onError: [],
|
|
2455
|
+
subscribes: [],
|
|
2137
2456
|
runtimes: [],
|
|
2138
2457
|
instances: [
|
|
2139
2458
|
...instances
|
|
@@ -4176,6 +4495,71 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
4176
4495
|
if (bufferedHit) return bufferedHit.value;
|
|
4177
4496
|
throw new WorkflowSuspended(deadline);
|
|
4178
4497
|
}, "waitForEvent");
|
|
4498
|
+
const settleRunWaitFromTarget = /* @__PURE__ */ __name(async (target, token, seq) => {
|
|
4499
|
+
const state = await settledRunOf(store, target);
|
|
4500
|
+
if (state.state === "missing") {
|
|
4501
|
+
await store.removeSignalWaiter({
|
|
4502
|
+
token,
|
|
4503
|
+
runId,
|
|
4504
|
+
seq
|
|
4505
|
+
});
|
|
4506
|
+
throw new FatalError(`run ${target} not found`, "run_not_found");
|
|
4507
|
+
}
|
|
4508
|
+
if (state.state === "running") return null;
|
|
4509
|
+
if (!await store.takeSignalWaiter(token)) return null;
|
|
4510
|
+
const event = lifecycleEventOf(state.run);
|
|
4511
|
+
await writeCheckpoint(instantCheckpoint({
|
|
4512
|
+
runId,
|
|
4513
|
+
seq,
|
|
4514
|
+
name: `signal:${token}`,
|
|
4515
|
+
kind: "signal",
|
|
4516
|
+
output: event
|
|
4517
|
+
}));
|
|
4518
|
+
return {
|
|
4519
|
+
value: event
|
|
4520
|
+
};
|
|
4521
|
+
}, "settleRunWaitFromTarget");
|
|
4522
|
+
const waitForRun = /* @__PURE__ */ __name(async (first, second, third) => {
|
|
4523
|
+
const target = typeof first === "string" ? first : second;
|
|
4524
|
+
const opts = typeof first === "string" ? second : third;
|
|
4525
|
+
if (opts?.timeoutMs == null) {
|
|
4526
|
+
const current = pos.next();
|
|
4527
|
+
const token2 = runWaitToken(target, runId, current);
|
|
4528
|
+
const existing = await readCheckpoint(current);
|
|
4529
|
+
if (existing && existing.status === "completed") return existing.output;
|
|
4530
|
+
await store.putSignalWaiter({
|
|
4531
|
+
token: token2,
|
|
4532
|
+
runId,
|
|
4533
|
+
seq: current
|
|
4534
|
+
});
|
|
4535
|
+
const settled2 = await settleRunWaitFromTarget(target, token2, current);
|
|
4536
|
+
if (settled2) return settled2.value;
|
|
4537
|
+
throw new WorkflowSuspended();
|
|
4538
|
+
}
|
|
4539
|
+
const timeoutMs = parseDuration(opts.timeoutMs);
|
|
4540
|
+
const deadlineSeq = pos.next();
|
|
4541
|
+
const waitSeq = pos.next();
|
|
4542
|
+
const token = runWaitToken(target, runId, waitSeq);
|
|
4543
|
+
const deadline = await stampDeadline(deadlineSeq, `run:${target}`, timeoutMs);
|
|
4544
|
+
const waited = await readCheckpoint(waitSeq);
|
|
4545
|
+
if (waited && waited.status === "completed") return waited.output;
|
|
4546
|
+
if (host.clock() >= deadline) {
|
|
4547
|
+
await store.removeSignalWaiter({
|
|
4548
|
+
token,
|
|
4549
|
+
runId,
|
|
4550
|
+
seq: waitSeq
|
|
4551
|
+
}).catch(() => void 0);
|
|
4552
|
+
throw new SignalTimeoutError(`run:${target}`, timeoutMs);
|
|
4553
|
+
}
|
|
4554
|
+
await store.putSignalWaiter({
|
|
4555
|
+
token,
|
|
4556
|
+
runId,
|
|
4557
|
+
seq: waitSeq
|
|
4558
|
+
});
|
|
4559
|
+
const settled = await settleRunWaitFromTarget(target, token, waitSeq);
|
|
4560
|
+
if (settled) return settled.value;
|
|
4561
|
+
throw new WorkflowSuspended(deadline);
|
|
4562
|
+
}, "waitForRun");
|
|
4179
4563
|
const task = /* @__PURE__ */ __name(async (name, dispatch, options) => {
|
|
4180
4564
|
await localStep(`task:dispatch:${name}`, dispatch, options);
|
|
4181
4565
|
return unwrapCompletion(await waitForSignal(`task:${runId}:${name}`), `task "${name}"`);
|
|
@@ -4841,6 +5225,7 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
4841
5225
|
continueAsNew,
|
|
4842
5226
|
waitForSignal: guard("waitForSignal", waitForSignal),
|
|
4843
5227
|
waitForEvent: guard("waitForEvent", waitForEvent),
|
|
5228
|
+
waitForRun: guard("waitForRun", waitForRun),
|
|
4844
5229
|
task: guard("task", task),
|
|
4845
5230
|
child: guard("child", child),
|
|
4846
5231
|
all: guard("all", all),
|
|
@@ -4871,6 +5256,28 @@ __name(mergeTags, "mergeTags");
|
|
|
4871
5256
|
var BREAKPOINT = "breakpoint";
|
|
4872
5257
|
var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
|
|
4873
5258
|
var ANNOUNCED_ROUTES_TTL_MS = 5e3;
|
|
5259
|
+
function servedNamespace(namespace, group) {
|
|
5260
|
+
if (namespace) return namespace;
|
|
5261
|
+
const at = group?.lastIndexOf("@") ?? -1;
|
|
5262
|
+
return group && at >= 0 && at < group.length - 1 ? group.slice(at + 1) : "default";
|
|
5263
|
+
}
|
|
5264
|
+
__name(servedNamespace, "servedNamespace");
|
|
5265
|
+
function announcedIn(snapshot, event, namespace, nodeOnly = false) {
|
|
5266
|
+
const scope = namespace === null ? null : runNamespace(namespace);
|
|
5267
|
+
const subscribers = /* @__PURE__ */ new Set();
|
|
5268
|
+
for (const entry of snapshot.entries) {
|
|
5269
|
+
if (entry.event !== event || nodeOnly && !entry.node) continue;
|
|
5270
|
+
if (scope === null || entry.namespace === null || entry.namespace === scope) {
|
|
5271
|
+
subscribers.add(entry.workflow);
|
|
5272
|
+
}
|
|
5273
|
+
}
|
|
5274
|
+
return subscribers;
|
|
5275
|
+
}
|
|
5276
|
+
__name(announcedIn, "announcedIn");
|
|
5277
|
+
function runNamespace(namespace) {
|
|
5278
|
+
return namespace || "default";
|
|
5279
|
+
}
|
|
5280
|
+
__name(runNamespace, "runNamespace");
|
|
4874
5281
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
4875
5282
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
4876
5283
|
var STEP_FAN_CHUNK = 200;
|
|
@@ -4967,15 +5374,10 @@ function shuffle(items) {
|
|
|
4967
5374
|
return items;
|
|
4968
5375
|
}
|
|
4969
5376
|
__name(shuffle, "shuffle");
|
|
4970
|
-
function stepGroup(step,
|
|
4971
|
-
return tenantGroup(sanitizeQueueToken(step.name), step.partition ??
|
|
5377
|
+
function stepGroup(step, runNamespace2) {
|
|
5378
|
+
return tenantGroup(sanitizeQueueToken(step.name), step.partition ?? runNamespace2);
|
|
4972
5379
|
}
|
|
4973
5380
|
__name(stepGroup, "stepGroup");
|
|
4974
|
-
function nextContinuationId(runId) {
|
|
4975
|
-
const m = runId.match(/^(.*)~(\d+)$/);
|
|
4976
|
-
return m ? `${m[1]}~${Number(m[2]) + 1}` : `${runId}~1`;
|
|
4977
|
-
}
|
|
4978
|
-
__name(nextContinuationId, "nextContinuationId");
|
|
4979
5381
|
function isNewerVersion(a, b) {
|
|
4980
5382
|
const na = Number(a);
|
|
4981
5383
|
const nb = Number(b);
|
|
@@ -5025,7 +5427,7 @@ var UNREGISTERED_DEAD_LETTER_FROM = [
|
|
|
5025
5427
|
"cancelling",
|
|
5026
5428
|
"blocked"
|
|
5027
5429
|
];
|
|
5028
|
-
var WorkflowEngine = class {
|
|
5430
|
+
var WorkflowEngine = class _WorkflowEngine {
|
|
5029
5431
|
static {
|
|
5030
5432
|
__name(this, "WorkflowEngine");
|
|
5031
5433
|
}
|
|
@@ -5077,7 +5479,7 @@ var WorkflowEngine = class {
|
|
|
5077
5479
|
workflows = /* @__PURE__ */ new Map();
|
|
5078
5480
|
/** The newest registered version per workflow name — used to `start` new runs. */
|
|
5079
5481
|
latest = /* @__PURE__ */ new Map();
|
|
5080
|
-
/** Event name → workflow
|
|
5482
|
+
/** Event name → subscribing workflow name → that workflow's subscriptions to it. See `onEvent`. */
|
|
5081
5483
|
eventTriggers = /* @__PURE__ */ new Map();
|
|
5082
5484
|
/** Durable-entity subsystem (registers the `__entity` runner; see `registerEntity`). */
|
|
5083
5485
|
entities;
|
|
@@ -5199,6 +5601,12 @@ var WorkflowEngine = class {
|
|
|
5199
5601
|
* releases when its budget runs out.
|
|
5200
5602
|
*/
|
|
5201
5603
|
executionLeases = /* @__PURE__ */ new Map();
|
|
5604
|
+
/** Runs a caller just locked to execute now, so their turn has nothing to prove and no await to add. */
|
|
5605
|
+
lockedForTurn = /* @__PURE__ */ new Set();
|
|
5606
|
+
/** Runs this instance holds for a moment and will release, not to execute (see {@link handOffDue}). */
|
|
5607
|
+
claimsInFlight = /* @__PURE__ */ new Map();
|
|
5608
|
+
/** How many such claims each run has had, so a proof can tell one came and went (see {@link proveOwnLease}). */
|
|
5609
|
+
claimsMade = /* @__PURE__ */ new Map();
|
|
5202
5610
|
/** Pager hooks for stranded runs (see {@link onStalled}). */
|
|
5203
5611
|
stalledListeners = /* @__PURE__ */ new Set();
|
|
5204
5612
|
/** Runs already paged about, by id → the `updatedAt` of the paged episode (see {@link sweepStalled}). */
|
|
@@ -5542,6 +5950,27 @@ var WorkflowEngine = class {
|
|
|
5542
5950
|
console.error(`[nestjs-durable] could not run queued run ${runId}:`, error2);
|
|
5543
5951
|
}
|
|
5544
5952
|
}
|
|
5953
|
+
markClaim(runId) {
|
|
5954
|
+
this.claimsInFlight.set(runId, (this.claimsInFlight.get(runId) ?? 0) + 1);
|
|
5955
|
+
if (this.claimsMade.size >= 4096) this.claimsMade.clear();
|
|
5956
|
+
this.claimsMade.set(runId, (this.claimsMade.get(runId) ?? 0) + 1);
|
|
5957
|
+
}
|
|
5958
|
+
/**
|
|
5959
|
+
* Whether this instance holds the run's lease for a turn, from the store and by owner. A renewal can land
|
|
5960
|
+
* on a claim marked after it was sent, and that claim is about to be released, so any claim begun while
|
|
5961
|
+
* the renewal was out, even one already over, leaves it unproved.
|
|
5962
|
+
*/
|
|
5963
|
+
async proveOwnLease(runId, nowMs) {
|
|
5964
|
+
if (this.claimsInFlight.has(runId)) return false;
|
|
5965
|
+
const before = this.claimsMade.get(runId);
|
|
5966
|
+
const renewed = await this.store.renewRunLock(runId, this.instanceId, nowMs + this.leaseMs);
|
|
5967
|
+
return renewed && !this.claimsInFlight.has(runId) && this.claimsMade.get(runId) === before;
|
|
5968
|
+
}
|
|
5969
|
+
unmarkClaim(runId) {
|
|
5970
|
+
const left = (this.claimsInFlight.get(runId) ?? 1) - 1;
|
|
5971
|
+
if (left > 0) this.claimsInFlight.set(runId, left);
|
|
5972
|
+
else this.claimsInFlight.delete(runId);
|
|
5973
|
+
}
|
|
5545
5974
|
/**
|
|
5546
5975
|
* Hand due runs (timers, blocked runs due for a fleet re-check) to whichever instance takes them
|
|
5547
5976
|
* off the run queue. Each is claimed first, under its lease, by pushing its `wakeAt` one lease
|
|
@@ -5553,29 +5982,34 @@ var WorkflowEngine = class {
|
|
|
5553
5982
|
const claimed = [];
|
|
5554
5983
|
for (const run of runs) {
|
|
5555
5984
|
if (this.draining) break;
|
|
5556
|
-
|
|
5557
|
-
continue;
|
|
5558
|
-
}
|
|
5559
|
-
if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) {
|
|
5560
|
-
const result = await this.settleUnregistered(run, nowMs);
|
|
5561
|
-
if (result) settled?.set(run.id, result);
|
|
5562
|
-
else if (result === void 0) this.logUnregisteredSkip(run);
|
|
5563
|
-
continue;
|
|
5564
|
-
}
|
|
5985
|
+
this.markClaim(run.id);
|
|
5565
5986
|
try {
|
|
5566
|
-
|
|
5567
|
-
|
|
5568
|
-
}
|
|
5569
|
-
if (this.
|
|
5570
|
-
|
|
5571
|
-
|
|
5572
|
-
|
|
5573
|
-
|
|
5574
|
-
|
|
5575
|
-
|
|
5987
|
+
if (!await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
5988
|
+
continue;
|
|
5989
|
+
}
|
|
5990
|
+
if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) {
|
|
5991
|
+
const result = await this.settleUnregistered(run, nowMs);
|
|
5992
|
+
if (result) settled?.set(run.id, result);
|
|
5993
|
+
else if (result === void 0) this.logUnregisteredSkip(run);
|
|
5994
|
+
continue;
|
|
5995
|
+
}
|
|
5996
|
+
try {
|
|
5997
|
+
const patch = {
|
|
5998
|
+
wakeAt: nowMs + this.leaseMs
|
|
5999
|
+
};
|
|
6000
|
+
if (this.store.updateRunIf) {
|
|
6001
|
+
if (await this.store.updateRunIf(run.id, [
|
|
6002
|
+
run.status
|
|
6003
|
+
], patch)) claimed.push(run);
|
|
6004
|
+
} else {
|
|
6005
|
+
await this.store.updateRun(run.id, patch);
|
|
6006
|
+
claimed.push(run);
|
|
6007
|
+
}
|
|
6008
|
+
} finally {
|
|
6009
|
+
await this.store.releaseRunLock(run.id, this.instanceId);
|
|
5576
6010
|
}
|
|
5577
6011
|
} finally {
|
|
5578
|
-
|
|
6012
|
+
this.unmarkClaim(run.id);
|
|
5579
6013
|
}
|
|
5580
6014
|
}
|
|
5581
6015
|
try {
|
|
@@ -5691,10 +6125,14 @@ var WorkflowEngine = class {
|
|
|
5691
6125
|
this.workflows.set(versionKey(name, version), registered);
|
|
5692
6126
|
const current = this.latest.get(name);
|
|
5693
6127
|
if (!current || isNewerVersion(version, current.version)) this.latest.set(name, registered);
|
|
5694
|
-
for (const
|
|
5695
|
-
const
|
|
5696
|
-
subscribers.
|
|
5697
|
-
|
|
6128
|
+
for (const entry of opts?.onEvent ?? []) {
|
|
6129
|
+
const subscription = toEventSubscription(entry);
|
|
6130
|
+
const subscribers = this.eventTriggers.get(subscription.event) ?? /* @__PURE__ */ new Map();
|
|
6131
|
+
subscribers.set(name, [
|
|
6132
|
+
...subscribers.get(name) ?? [],
|
|
6133
|
+
subscription
|
|
6134
|
+
]);
|
|
6135
|
+
this.eventTriggers.set(subscription.event, subscribers);
|
|
5698
6136
|
}
|
|
5699
6137
|
}
|
|
5700
6138
|
/**
|
|
@@ -5832,136 +6270,1334 @@ var WorkflowEngine = class {
|
|
|
5832
6270
|
return () => this.deadListeners.delete(listener);
|
|
5833
6271
|
}
|
|
5834
6272
|
/**
|
|
5835
|
-
* Delete buffered child completions no parent will ever consume (see
|
|
5836
|
-
* `childCompletionNeedsBuffer`): those of root runs, of children whose parent already settled them
|
|
5837
|
-
* or is gone, or terminal for good. Such rows were written by earlier versions, which buffered every
|
|
5838
|
-
* completion that found no waiter, and nothing else ever removes them.
|
|
6273
|
+
* Delete buffered child completions no parent will ever consume (see
|
|
6274
|
+
* `childCompletionNeedsBuffer`): those of root runs, of children whose parent already settled them
|
|
6275
|
+
* or is gone, or terminal for good. Such rows were written by earlier versions, which buffered every
|
|
6276
|
+
* completion that found no waiter, and nothing else ever removes them.
|
|
6277
|
+
*
|
|
6278
|
+
* Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
|
|
6279
|
+
* at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
|
|
6280
|
+
* few timer ticks without a long query or a big delete. Each page is decided set-based (see
|
|
6281
|
+
* `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
|
|
6282
|
+
* read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
|
|
6283
|
+
* {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
|
|
6284
|
+
* call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
|
|
6285
|
+
* before the next one. Call it every tick (the NestJS timer poller does); a call made while
|
|
6286
|
+
* another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
|
|
6287
|
+
* carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
|
|
6288
|
+
* store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
|
|
6289
|
+
*/
|
|
6290
|
+
async sweepBufferedSignals(now = this.clock(), budgetMs = BUFFERED_SWEEP_BUDGET_MS) {
|
|
6291
|
+
const { listBufferedSignals, deleteBufferedSignals } = this.store;
|
|
6292
|
+
if (!listBufferedSignals || !deleteBufferedSignals) return 0;
|
|
6293
|
+
if (this.draining || this.namespace !== void 0 || this.bufferedSweepActive) return 0;
|
|
6294
|
+
if (this.bufferedSweepCursor === void 0 && now - this.lastBufferedSweepEndedAt < BUFFERED_SWEEP_REST_MS) {
|
|
6295
|
+
return 0;
|
|
6296
|
+
}
|
|
6297
|
+
this.bufferedSweepActive = true;
|
|
6298
|
+
const deadline = Date.now() + budgetMs;
|
|
6299
|
+
let deleted = 0;
|
|
6300
|
+
try {
|
|
6301
|
+
for (let page = 0; page < BUFFERED_SWEEP_PAGES_PER_CALL; page++) {
|
|
6302
|
+
if (page > 0 && (Date.now() >= deadline || this.draining)) break;
|
|
6303
|
+
const rows = await listBufferedSignals.call(this.store, CHILD_TOKEN_PREFIX, this.bufferedSweepCursor, BUFFERED_SWEEP_PAGE);
|
|
6304
|
+
const { doomed, decided } = await unneededChildSignals(this.store, rows, deadline);
|
|
6305
|
+
if (doomed.length > 0) {
|
|
6306
|
+
await deleteBufferedSignals.call(this.store, doomed.map((row) => row.id));
|
|
6307
|
+
deleted += doomed.length;
|
|
6308
|
+
}
|
|
6309
|
+
if (decided < rows.length) {
|
|
6310
|
+
if (decided > 0) this.bufferedSweepCursor = rows[decided - 1].id;
|
|
6311
|
+
break;
|
|
6312
|
+
}
|
|
6313
|
+
const last = rows.at(-1);
|
|
6314
|
+
if (rows.length < BUFFERED_SWEEP_PAGE || !last) {
|
|
6315
|
+
this.bufferedSweepCursor = void 0;
|
|
6316
|
+
this.lastBufferedSweepEndedAt = now;
|
|
6317
|
+
break;
|
|
6318
|
+
}
|
|
6319
|
+
this.bufferedSweepCursor = last.id;
|
|
6320
|
+
}
|
|
6321
|
+
} finally {
|
|
6322
|
+
this.bufferedSweepActive = false;
|
|
6323
|
+
}
|
|
6324
|
+
return deleted;
|
|
6325
|
+
}
|
|
6326
|
+
/**
|
|
6327
|
+
* Register a pager hook for STRANDED runs: an in-flight run untouched past `stalledAfterMs` that is
|
|
6328
|
+
* either suspended with NO wake timer (wake-forever: nothing will ever re-drive it) or parked on an
|
|
6329
|
+
* old pending remote step with no live lease (dispatched and, most likely, lost). The timer poller
|
|
6330
|
+
* sweeps for these ({@link sweepStalled}, throttled to one pass a minute) and notifies each episode
|
|
6331
|
+
* ONCE per instance: a run is paged again only after it makes progress and strands again. Wire your
|
|
6332
|
+
* alerting here like {@link onDead}. A listener that throws never breaks the sweep. Returns an
|
|
6333
|
+
* unsubscribe function.
|
|
6334
|
+
*/
|
|
6335
|
+
onStalled(listener) {
|
|
6336
|
+
this.stalledListeners.add(listener);
|
|
6337
|
+
return () => this.stalledListeners.delete(listener);
|
|
6338
|
+
}
|
|
6339
|
+
/**
|
|
6340
|
+
* Detect stranded in-flight runs and notify the {@link onStalled} listeners. Self-throttled to one
|
|
6341
|
+
* pass a minute, skipped while draining, and a no-op without listeners — call it every tick (the
|
|
6342
|
+
* NestJS timer poller does). Scoped to this engine's `namespace` (an operator engine with no
|
|
6343
|
+
* namespace sweeps every namespace). Returns the runs notified THIS pass.
|
|
6344
|
+
*/
|
|
6345
|
+
async sweepStalled(now = this.clock()) {
|
|
6346
|
+
if (this.stalledListeners.size === 0 || this.draining) return [];
|
|
6347
|
+
if (now - this.lastStalledSweepAt < 6e4) return [];
|
|
6348
|
+
this.lastStalledSweepAt = now;
|
|
6349
|
+
const notified = [];
|
|
6350
|
+
for (const info2 of await this.listStalledRuns({
|
|
6351
|
+
now
|
|
6352
|
+
})) {
|
|
6353
|
+
const episode = info2.run.updatedAt.getTime();
|
|
6354
|
+
if (this.notifiedStalled.get(info2.run.id) === episode) continue;
|
|
6355
|
+
this.notifiedStalled.set(info2.run.id, episode);
|
|
6356
|
+
if (this.notifiedStalled.size > 1e3) this.notifiedStalled.clear();
|
|
6357
|
+
for (const listener of this.stalledListeners) {
|
|
6358
|
+
try {
|
|
6359
|
+
listener(info2);
|
|
6360
|
+
} catch {
|
|
6361
|
+
}
|
|
6362
|
+
}
|
|
6363
|
+
notified.push(info2);
|
|
6364
|
+
}
|
|
6365
|
+
return notified;
|
|
6366
|
+
}
|
|
6367
|
+
/**
|
|
6368
|
+
* The stranded runs right now, without notifying anyone: the query behind {@link sweepStalled} and
|
|
6369
|
+
* the CLI's `durable inspect --stale`. A `running`/`suspended` run qualifies when it has been
|
|
6370
|
+
* untouched for `thresholdMs` (default `stalledAfterMs`) AND either it is suspended with no wake
|
|
6371
|
+
* timer, or its oldest pending remote step is older than the threshold with no lease in the future
|
|
6372
|
+
* (a step with a `timeoutMs`, a heartbeat or `remoteRedispatchMs` keeps a lease; a long
|
|
6373
|
+
* `ctx.sleep` has a future `wakeAt`; neither is stranded). Scans up to `scan` candidates (default
|
|
6374
|
+
* 1000), newest first, in this engine's namespace.
|
|
6375
|
+
*/
|
|
6376
|
+
async listStalledRuns(options = {}) {
|
|
6377
|
+
return findStalledRuns(this.store, {
|
|
6378
|
+
now: options.now ?? this.clock(),
|
|
6379
|
+
thresholdMs: options.thresholdMs ?? this.stalledAfterMs,
|
|
6380
|
+
scan: options.scan,
|
|
6381
|
+
namespace: this.namespace
|
|
6382
|
+
});
|
|
6383
|
+
}
|
|
6384
|
+
notifyDead(run) {
|
|
6385
|
+
for (const fn of this.deadListeners) {
|
|
6386
|
+
try {
|
|
6387
|
+
fn(run);
|
|
6388
|
+
} catch {
|
|
6389
|
+
}
|
|
6390
|
+
}
|
|
6391
|
+
}
|
|
6392
|
+
/**
|
|
6393
|
+
* Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the control
|
|
6394
|
+
* plane so other instances (e.g. a dashboard pod) can live-tail this run. A `run.completed` or
|
|
6395
|
+
* `run.failed` also publishes the run's lifecycle event (see {@link publishLifecycle}) and resumes
|
|
6396
|
+
* the runs waiting on it with `ctx.waitForRun` (see {@link deliverRunWaits}); `lifecycle: false`
|
|
6397
|
+
* marks a `run.completed` that is not a real completion (a continue-as-new handoff).
|
|
6398
|
+
*/
|
|
6399
|
+
emit(event, opts) {
|
|
6400
|
+
const full = {
|
|
6401
|
+
...event,
|
|
6402
|
+
at: /* @__PURE__ */ new Date()
|
|
6403
|
+
};
|
|
6404
|
+
this.deliver(full);
|
|
6405
|
+
if (opts?.lifecycle !== false) {
|
|
6406
|
+
this.publishLifecycle(full);
|
|
6407
|
+
if ((full.type === "run.completed" || full.type === "run.failed") && !full.workflow?.startsWith("__")) {
|
|
6408
|
+
void this.deliverRunWaits(full.runId).catch(() => void 0);
|
|
6409
|
+
}
|
|
6410
|
+
}
|
|
6411
|
+
if (!this.controlPlane) return;
|
|
6412
|
+
this.eventsToBroadcast.push(full);
|
|
6413
|
+
if (!this.broadcastingEvents) void this.broadcastEvents();
|
|
6414
|
+
}
|
|
6415
|
+
/**
|
|
6416
|
+
* Start every workflow subscribed to `name` that takes `payload` (see {@link acceptingSubscription}),
|
|
6417
|
+
* each under `runIdOf(workflow)` and with its subscription's `input` applied. `only` restricts it
|
|
6418
|
+
* to those subscribers. `remote` are subscribers only live workers announce (see
|
|
6419
|
+
* {@link announcedRemoteSubscribers}): each is started by name with the payload as its input, since
|
|
6420
|
+
* an announcement carries no `where` or `input`. `routeIdOf` names the event for a subscriber's
|
|
6421
|
+
* debounce/batch accumulator, which counts an id once. Returns the subscribers reached, with the
|
|
6422
|
+
* run each started (none when the payload went into its debounce/batch accumulator), and the ones
|
|
6423
|
+
* that could not be started, with why.
|
|
6424
|
+
*/
|
|
6425
|
+
async startSubscribers(name, payload, runIdOf, only, remote = [], routeIdOf) {
|
|
6426
|
+
const reached = [];
|
|
6427
|
+
const failed = [];
|
|
6428
|
+
for (const [workflow, subscriptions] of this.eventTriggers.get(name) ?? []) {
|
|
6429
|
+
if (only && !only.has(workflow)) continue;
|
|
6430
|
+
const { accepted, errors } = acceptingSubscription(subscriptions, payload);
|
|
6431
|
+
if (!accepted) {
|
|
6432
|
+
if (errors.length > 0) failed.push({
|
|
6433
|
+
workflow,
|
|
6434
|
+
error: errors[0]
|
|
6435
|
+
});
|
|
6436
|
+
continue;
|
|
6437
|
+
}
|
|
6438
|
+
try {
|
|
6439
|
+
const input = accepted.input ? accepted.input(payload) : payload;
|
|
6440
|
+
const batch = this.latest.get(workflow)?.eventBatch;
|
|
6441
|
+
if (batch) {
|
|
6442
|
+
await this.accumulators.route(workflow, batch, input, routeIdOf?.(workflow));
|
|
6443
|
+
reached.push({
|
|
6444
|
+
workflow
|
|
6445
|
+
});
|
|
6446
|
+
} else {
|
|
6447
|
+
const runId = runIdOf(workflow);
|
|
6448
|
+
await this.start(workflow, input, runId);
|
|
6449
|
+
reached.push({
|
|
6450
|
+
workflow,
|
|
6451
|
+
runId
|
|
6452
|
+
});
|
|
6453
|
+
}
|
|
6454
|
+
} catch (error2) {
|
|
6455
|
+
failed.push({
|
|
6456
|
+
workflow,
|
|
6457
|
+
error: error2
|
|
6458
|
+
});
|
|
6459
|
+
}
|
|
6460
|
+
}
|
|
6461
|
+
const local = this.eventTriggers.get(name);
|
|
6462
|
+
for (const workflow of remote) {
|
|
6463
|
+
if (local?.has(workflow) || only && !only.has(workflow)) continue;
|
|
6464
|
+
try {
|
|
6465
|
+
const runId = runIdOf(workflow);
|
|
6466
|
+
await this.start(workflow, payload, runId);
|
|
6467
|
+
reached.push({
|
|
6468
|
+
workflow,
|
|
6469
|
+
runId
|
|
6470
|
+
});
|
|
6471
|
+
} catch (error2) {
|
|
6472
|
+
failed.push({
|
|
6473
|
+
workflow,
|
|
6474
|
+
error: error2
|
|
6475
|
+
});
|
|
6476
|
+
}
|
|
6477
|
+
}
|
|
6478
|
+
return {
|
|
6479
|
+
reached,
|
|
6480
|
+
failed
|
|
6481
|
+
};
|
|
6482
|
+
}
|
|
6483
|
+
/**
|
|
6484
|
+
* The subscribers of `event` that live workers announce for the settled runs of `namespace` (`null`:
|
|
6485
|
+
* any namespace, for an event no run published), are not registered on this engine with an `onEvent` for it, and that this engine can start: registered here (a `registerRemote`), or served
|
|
6486
|
+
* by convention on a live worker group of the same name (a Python `@workflow(on_event=...)`). One
|
|
6487
|
+
* this engine cannot start is left to the instances that can, so it stays owed rather than failing.
|
|
6488
|
+
*/
|
|
6489
|
+
async announcedRemoteSubscribers(event, namespace) {
|
|
6490
|
+
const snapshot = await this.announcedSubscriptions();
|
|
6491
|
+
const announced = announcedIn(snapshot, event, namespace);
|
|
6492
|
+
if (!announced.size) return [];
|
|
6493
|
+
const local = new Set(this.localSubscribersIn(event, namespace));
|
|
6494
|
+
const byEngine = announcedIn(snapshot, event, namespace, true);
|
|
6495
|
+
const startable = [];
|
|
6496
|
+
for (const workflow of [
|
|
6497
|
+
...announced
|
|
6498
|
+
].sort()) {
|
|
6499
|
+
if (local.has(workflow) || byEngine.has(workflow) || workflow.startsWith("__")) continue;
|
|
6500
|
+
if (this.latest.has(workflow)) {
|
|
6501
|
+
startable.push(workflow);
|
|
6502
|
+
continue;
|
|
6503
|
+
}
|
|
6504
|
+
const group = tenantGroup(sanitizeQueueToken(workflow), this.namespace);
|
|
6505
|
+
const live = await this.pool.transportWithLiveGroup(group).catch(() => void 0);
|
|
6506
|
+
if (live) startable.push(workflow);
|
|
6507
|
+
}
|
|
6508
|
+
return startable;
|
|
6509
|
+
}
|
|
6510
|
+
/** The workflows registered on this engine that subscribe to `event`. */
|
|
6511
|
+
localSubscribers(event) {
|
|
6512
|
+
return [
|
|
6513
|
+
...this.eventTriggers.get(event)?.keys() ?? []
|
|
6514
|
+
];
|
|
6515
|
+
}
|
|
6516
|
+
/**
|
|
6517
|
+
* Whether some workflow may subscribe to `workflow` settling as `kind`: one registered on this
|
|
6518
|
+
* engine, or one in the last read of the fleet's announcements. True while there is no usable read
|
|
6519
|
+
* (none yet, or the last one failed), since the terminal write then marks the run unresolved (see
|
|
6520
|
+
* {@link withLifecycleMarker}).
|
|
6521
|
+
*/
|
|
6522
|
+
hasLifecycleSubscribers(workflow, kind, namespace) {
|
|
6523
|
+
if (workflow.startsWith("__")) return false;
|
|
6524
|
+
const event = lifecycleEventName(workflow, kind);
|
|
6525
|
+
if (this.servesNamespace(namespace) && this.localSubscribers(event).length > 0) return true;
|
|
6526
|
+
const snapshot = this.announcedSubscriptionsSnapshot;
|
|
6527
|
+
if (!snapshot?.ok) return true;
|
|
6528
|
+
return announcedIn(snapshot, event, namespace).size > 0;
|
|
6529
|
+
}
|
|
6530
|
+
/**
|
|
6531
|
+
* Whether this engine's own subscribers apply to the settled runs of `namespace`: an engine with
|
|
6532
|
+
* no namespace publishes the runs of every namespace, a namespaced one only its own.
|
|
6533
|
+
*/
|
|
6534
|
+
servesNamespace(namespace) {
|
|
6535
|
+
return namespace === null || this.namespace === void 0 || this.namespace === runNamespace(namespace);
|
|
6536
|
+
}
|
|
6537
|
+
/** This engine's subscribers of `event` that apply to the settled runs of `namespace`. */
|
|
6538
|
+
localSubscribersIn(event, namespace) {
|
|
6539
|
+
return this.servesNamespace(namespace) ? this.localSubscribers(event) : [];
|
|
6540
|
+
}
|
|
6541
|
+
/** Answers the traceparent of a run's span in this process; see {@link setRunTraceparent}. */
|
|
6542
|
+
runTraceparent;
|
|
6543
|
+
/**
|
|
6544
|
+
* Install the function that answers the W3C `traceparent` of a run's span in this process, or
|
|
6545
|
+
* `undefined` to remove it. `attachDurableOtel` installs one. With it, the first suspend or the
|
|
6546
|
+
* terminal write of a run some workflow subscribes to also records the run's traceparent (a
|
|
6547
|
+
* {@link LIFECYCLE_TRACE_TAG_PREFIX} tag, see {@link withRunTrace}), and the `run.started` of a run
|
|
6548
|
+
* a lifecycle event started carries its source run's as `sourceTraceparent`, so the subscriber's
|
|
6549
|
+
* trace links to the source's.
|
|
6550
|
+
*/
|
|
6551
|
+
setRunTraceparent(provider) {
|
|
6552
|
+
this.runTraceparent = provider;
|
|
6553
|
+
}
|
|
6554
|
+
/**
|
|
6555
|
+
* `{ sourceTraceparent }` for the `run.started` of `run`, when a lifecycle event started it and a
|
|
6556
|
+
* run trace provider is installed: the source run's recorded traceparent, else this process's.
|
|
6557
|
+
*/
|
|
6558
|
+
async sourceTraceOf(run) {
|
|
6559
|
+
const provider = this.runTraceparent;
|
|
6560
|
+
if (!provider) return {};
|
|
6561
|
+
const source = lifecycleSourceOf(run);
|
|
6562
|
+
if (!source) return {};
|
|
6563
|
+
try {
|
|
6564
|
+
const recorded = lifecycleTraceparentOf((await this.store.getRun(source.runId))?.tags);
|
|
6565
|
+
const traceparent = recorded ?? provider(source.runId);
|
|
6566
|
+
return traceparent ? {
|
|
6567
|
+
sourceTraceparent: traceparent
|
|
6568
|
+
} : {};
|
|
6569
|
+
} catch {
|
|
6570
|
+
return {};
|
|
6571
|
+
}
|
|
6572
|
+
}
|
|
6573
|
+
/** The last read of the subscriptions live workers announce. See {@link announcedSubscriptions}. */
|
|
6574
|
+
announcedSubscriptionsSnapshot;
|
|
6575
|
+
announcedSubscriptionsRead;
|
|
6576
|
+
/**
|
|
6577
|
+
* The `onEvent` subscriptions the live fleet announces, by event name: the fresh lifecycle
|
|
6578
|
+
* registry runs in the state store (see {@link heartbeatLifecycleRegistry}) and, on a transport
|
|
6579
|
+
* that reads them, each worker descriptor's `subscribes`. Each entry records the namespace its
|
|
6580
|
+
* announcer serves (`null` for an engine with no namespace) and whether a Node engine announced it
|
|
6581
|
+
* (every registry run is one). Read at most once per
|
|
6582
|
+
* {@link ANNOUNCED_ROUTES_TTL_MS}, concurrent callers sharing one read. Never throws: `ok: false`
|
|
6583
|
+
* means a read failed and only the local registry is known.
|
|
6584
|
+
*/
|
|
6585
|
+
async announcedSubscriptions() {
|
|
6586
|
+
const cached = this.announcedSubscriptionsSnapshot;
|
|
6587
|
+
if (cached && this.clock() - cached.at <= ANNOUNCED_ROUTES_TTL_MS) return cached;
|
|
6588
|
+
this.announcedSubscriptionsRead ??= this.readAnnouncedSubscriptions().finally(() => {
|
|
6589
|
+
this.announcedSubscriptionsRead = void 0;
|
|
6590
|
+
});
|
|
6591
|
+
return this.announcedSubscriptionsRead;
|
|
6592
|
+
}
|
|
6593
|
+
/**
|
|
6594
|
+
* The last read of the fleet's announcements as it is, without waiting: undefined before the first
|
|
6595
|
+
* read completes. Starts a read in the background when it is missing or older than
|
|
6596
|
+
* {@link ANNOUNCED_ROUTES_TTL_MS}. What the terminal write uses, so settling a run never waits on
|
|
6597
|
+
* the fleet.
|
|
6598
|
+
*/
|
|
6599
|
+
cachedAnnouncedSubscriptions() {
|
|
6600
|
+
const cached = this.announcedSubscriptionsSnapshot;
|
|
6601
|
+
if (!cached || this.clock() - cached.at > ANNOUNCED_ROUTES_TTL_MS) {
|
|
6602
|
+
void this.announcedSubscriptions();
|
|
6603
|
+
}
|
|
6604
|
+
return cached;
|
|
6605
|
+
}
|
|
6606
|
+
/**
|
|
6607
|
+
* When this engine first saw each subscription (`<event>\u0000<subscriber>`), in its registry or in
|
|
6608
|
+
* a good read of the fleet's announcements. The sweep's heal pass looks only at the ones first seen
|
|
6609
|
+
* within {@link LIFECYCLE_HEAL_WINDOW_MS}; see {@link healLifecycleEvents}.
|
|
6610
|
+
*/
|
|
6611
|
+
subscriptionFirstSeen = /* @__PURE__ */ new Map();
|
|
6612
|
+
/** Record the subscriptions of `snapshot` (a good read) and of this engine's registry as seen. */
|
|
6613
|
+
noteSubscriptionsSeen(snapshot) {
|
|
6614
|
+
const now = this.clock();
|
|
6615
|
+
const note = /* @__PURE__ */ __name((event, subscriber) => {
|
|
6616
|
+
const key = `${event}\0${subscriber}`;
|
|
6617
|
+
if (!this.subscriptionFirstSeen.has(key)) this.subscriptionFirstSeen.set(key, now);
|
|
6618
|
+
}, "note");
|
|
6619
|
+
for (const [event, subscribers] of snapshot.byEvent) {
|
|
6620
|
+
for (const subscriber of subscribers) note(event, subscriber);
|
|
6621
|
+
}
|
|
6622
|
+
for (const [event, subscribers] of this.eventTriggers) {
|
|
6623
|
+
for (const subscriber of subscribers.keys()) note(event, subscriber);
|
|
6624
|
+
}
|
|
6625
|
+
}
|
|
6626
|
+
async readAnnouncedSubscriptions() {
|
|
6627
|
+
const byEvent = /* @__PURE__ */ new Map();
|
|
6628
|
+
const entries = [];
|
|
6629
|
+
let ok = true;
|
|
6630
|
+
const add = /* @__PURE__ */ __name((event, workflow, node, namespace) => {
|
|
6631
|
+
entries.push({
|
|
6632
|
+
event,
|
|
6633
|
+
workflow,
|
|
6634
|
+
namespace,
|
|
6635
|
+
node
|
|
6636
|
+
});
|
|
6637
|
+
const subscribers = byEvent.get(event) ?? /* @__PURE__ */ new Set();
|
|
6638
|
+
subscribers.add(workflow);
|
|
6639
|
+
byEvent.set(event, subscribers);
|
|
6640
|
+
}, "add");
|
|
6641
|
+
try {
|
|
6642
|
+
const now = this.clock();
|
|
6643
|
+
for (const row of await this.listLifecycleRegistry()) {
|
|
6644
|
+
if (now - row.updatedAt.getTime() > LIFECYCLE_REGISTRY_TTL_MS) continue;
|
|
6645
|
+
const namespace = lifecycleRegistryNamespace(row.input, row.namespace);
|
|
6646
|
+
for (const { workflow, events } of lifecycleRegistrySubscriptions(row.input)) {
|
|
6647
|
+
for (const event of events) add(event, workflow, true, namespace);
|
|
6648
|
+
}
|
|
6649
|
+
}
|
|
6650
|
+
if (this.pool.introspectsFleet) {
|
|
6651
|
+
for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
|
|
6652
|
+
for (const announced of announcementsOf(descriptor)) {
|
|
6653
|
+
const namespace = servedNamespace(descriptor.namespace, announced.group);
|
|
6654
|
+
for (const event of announced.subscribes ?? []) {
|
|
6655
|
+
add(event, announced.name, descriptor.runtime === "node", namespace);
|
|
6656
|
+
}
|
|
6657
|
+
}
|
|
6658
|
+
}
|
|
6659
|
+
}
|
|
6660
|
+
} catch {
|
|
6661
|
+
ok = false;
|
|
6662
|
+
byEvent.clear();
|
|
6663
|
+
entries.length = 0;
|
|
6664
|
+
}
|
|
6665
|
+
const snapshot = {
|
|
6666
|
+
at: this.clock(),
|
|
6667
|
+
ok,
|
|
6668
|
+
byEvent,
|
|
6669
|
+
entries
|
|
6670
|
+
};
|
|
6671
|
+
this.announcedSubscriptionsSnapshot = snapshot;
|
|
6672
|
+
if (ok) this.noteSubscriptionsSeen(snapshot);
|
|
6673
|
+
return snapshot;
|
|
6674
|
+
}
|
|
6675
|
+
/** How many registry runs one page of {@link listLifecycleRegistry} reads. */
|
|
6676
|
+
static LIFECYCLE_REGISTRY_PAGE = 1e3;
|
|
6677
|
+
/**
|
|
6678
|
+
* The lifecycle registry runs this engine's namespace sees: its namespace's, plus `default`'s, where
|
|
6679
|
+
* the engines with no namespace keep theirs. Read a page of
|
|
6680
|
+
* {@link LIFECYCLE_REGISTRY_PAGE} at a time until a short page, so a large fleet is read whole.
|
|
6681
|
+
*/
|
|
6682
|
+
async listLifecycleRegistry() {
|
|
6683
|
+
const pageSize = _WorkflowEngine.LIFECYCLE_REGISTRY_PAGE;
|
|
6684
|
+
const rows = [];
|
|
6685
|
+
for (; ; ) {
|
|
6686
|
+
const page = await this.store.listRuns({
|
|
6687
|
+
workflow: LIFECYCLE_REGISTRY_WORKFLOW,
|
|
6688
|
+
...this.namespace !== void 0 ? {
|
|
6689
|
+
namespaces: [
|
|
6690
|
+
.../* @__PURE__ */ new Set([
|
|
6691
|
+
this.namespace,
|
|
6692
|
+
"default"
|
|
6693
|
+
])
|
|
6694
|
+
]
|
|
6695
|
+
} : {},
|
|
6696
|
+
limit: pageSize,
|
|
6697
|
+
offset: rows.length
|
|
6698
|
+
});
|
|
6699
|
+
rows.push(...page);
|
|
6700
|
+
if (page.length < pageSize) return rows;
|
|
6701
|
+
}
|
|
6702
|
+
}
|
|
6703
|
+
/** When this engine last wrote its registry run, or undefined before the first heartbeat. */
|
|
6704
|
+
lifecycleRegistryBeatAt;
|
|
6705
|
+
/**
|
|
6706
|
+
* Refresh this engine's lifecycle registry run and delete stale ones. At most once per
|
|
6707
|
+
* {@link LIFECYCLE_REGISTRY_HEARTBEAT_MS}; {@link sweepLifecycleEvents} calls it on every tick.
|
|
6708
|
+
*
|
|
6709
|
+
* An engine with `onEvent` subscriptions keeps one completed run of the internal
|
|
6710
|
+
* {@link LIFECYCLE_REGISTRY_WORKFLOW} workflow, id `__lifecycle_registry:<instanceId>`, holding
|
|
6711
|
+
* them in `input` and its heartbeat in `updatedAt`. Other instances read the fresh ones
|
|
6712
|
+
* ({@link LIFECYCLE_REGISTRY_TTL_MS}) as announced subscriptions, so a settled run is owed to a
|
|
6713
|
+
* subscriber registered only elsewhere whatever the transport. Any engine deletes the runs not
|
|
6714
|
+
* refreshed for {@link LIFECYCLE_REGISTRY_STALE_MS}. Retention never prunes them
|
|
6715
|
+
* ({@link LIFECYCLE_REGISTRY_TAG}), and the run gateway leaves them out of run lists.
|
|
6716
|
+
*/
|
|
6717
|
+
async heartbeatLifecycleRegistry() {
|
|
6718
|
+
const now = this.clock();
|
|
6719
|
+
const last = this.lifecycleRegistryBeatAt;
|
|
6720
|
+
if (last !== void 0 && now - last < LIFECYCLE_REGISTRY_HEARTBEAT_MS) return;
|
|
6721
|
+
this.lifecycleRegistryBeatAt = now;
|
|
6722
|
+
try {
|
|
6723
|
+
const subscriptions = this.registrySubscriptions();
|
|
6724
|
+
if (subscriptions.length > 0) await this.writeLifecycleRegistry(subscriptions, now);
|
|
6725
|
+
const stale = (await this.listLifecycleRegistry()).filter((row) => now - row.updatedAt.getTime() > LIFECYCLE_REGISTRY_STALE_MS);
|
|
6726
|
+
for (const row of stale) await this.store.deleteRun(row.id);
|
|
6727
|
+
} catch (error2) {
|
|
6728
|
+
this.lifecycleRegistryBeatAt = last;
|
|
6729
|
+
throw error2;
|
|
6730
|
+
}
|
|
6731
|
+
}
|
|
6732
|
+
/** This engine's `onEvent` subscriptions, by subscribing workflow, as its registry run holds them. */
|
|
6733
|
+
registrySubscriptions() {
|
|
6734
|
+
const byWorkflow = /* @__PURE__ */ new Map();
|
|
6735
|
+
for (const [event, subscribers] of this.eventTriggers) {
|
|
6736
|
+
for (const workflow of subscribers.keys()) {
|
|
6737
|
+
if (workflow.startsWith("__")) continue;
|
|
6738
|
+
const events = byWorkflow.get(workflow) ?? /* @__PURE__ */ new Set();
|
|
6739
|
+
events.add(event);
|
|
6740
|
+
byWorkflow.set(workflow, events);
|
|
6741
|
+
}
|
|
6742
|
+
}
|
|
6743
|
+
return [
|
|
6744
|
+
...byWorkflow
|
|
6745
|
+
].sort(([a], [b]) => a.localeCompare(b)).map(([workflow, events]) => ({
|
|
6746
|
+
workflow,
|
|
6747
|
+
events: [
|
|
6748
|
+
...events
|
|
6749
|
+
].sort()
|
|
6750
|
+
}));
|
|
6751
|
+
}
|
|
6752
|
+
async writeLifecycleRegistry(subscriptions, nowMs) {
|
|
6753
|
+
const id = lifecycleRegistryRunId(this.instanceId);
|
|
6754
|
+
const input = {
|
|
6755
|
+
instanceId: this.instanceId,
|
|
6756
|
+
runtime: "node",
|
|
6757
|
+
engineNamespace: this.namespace ?? null,
|
|
6758
|
+
subscriptions
|
|
6759
|
+
};
|
|
6760
|
+
const at = new Date(nowMs);
|
|
6761
|
+
const refreshed = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, id, [
|
|
6762
|
+
"completed"
|
|
6763
|
+
], {
|
|
6764
|
+
input,
|
|
6765
|
+
updatedAt: at
|
|
6766
|
+
});
|
|
6767
|
+
if (refreshed) return;
|
|
6768
|
+
await this.store.createRun({
|
|
6769
|
+
id,
|
|
6770
|
+
workflow: LIFECYCLE_REGISTRY_WORKFLOW,
|
|
6771
|
+
workflowVersion: "1",
|
|
6772
|
+
status: "completed",
|
|
6773
|
+
input,
|
|
6774
|
+
tags: [
|
|
6775
|
+
LIFECYCLE_REGISTRY_TAG
|
|
6776
|
+
],
|
|
6777
|
+
...this.namespace !== void 0 ? {
|
|
6778
|
+
namespace: this.namespace
|
|
6779
|
+
} : {},
|
|
6780
|
+
origin: CORE_ORIGIN,
|
|
6781
|
+
createdAt: at,
|
|
6782
|
+
updatedAt: at
|
|
6783
|
+
});
|
|
6784
|
+
}
|
|
6785
|
+
/**
|
|
6786
|
+
* The subscribers a run of `workflow` in `namespace` settling as `kind` owes its lifecycle event to:
|
|
6787
|
+
* the ones registered on this engine, when it publishes that namespace's runs (an engine with no
|
|
6788
|
+
* namespace publishes every namespace's), plus the ones live workers announce for that namespace
|
|
6789
|
+
* (see {@link announcedSubscriptions}): by announcers serving it, and by engines with no namespace.
|
|
6790
|
+
* Without `namespace`, every namespace's: what any run of `workflow` could owe. Empty for an
|
|
6791
|
+
* internal `__` workflow.
|
|
6792
|
+
*/
|
|
6793
|
+
async lifecycleSubscribers(workflow, kind, namespace) {
|
|
6794
|
+
return this.lifecycleSubscribersIn(await this.announcedSubscriptions(), workflow, kind, namespace);
|
|
6795
|
+
}
|
|
6796
|
+
/** {@link lifecycleSubscribers} from one given read of the fleet's announcements. */
|
|
6797
|
+
lifecycleSubscribersIn(snapshot, workflow, kind, namespace) {
|
|
6798
|
+
if (workflow.startsWith("__")) return [];
|
|
6799
|
+
const scope = namespace === void 0 ? null : runNamespace(namespace);
|
|
6800
|
+
const event = lifecycleEventName(workflow, kind);
|
|
6801
|
+
const announced = announcedIn(snapshot, event, scope);
|
|
6802
|
+
return [
|
|
6803
|
+
.../* @__PURE__ */ new Set([
|
|
6804
|
+
...this.localSubscribersIn(event, scope),
|
|
6805
|
+
...announced
|
|
6806
|
+
])
|
|
6807
|
+
].sort();
|
|
6808
|
+
}
|
|
6809
|
+
/**
|
|
6810
|
+
* The subscribers registered here that coalesce `run`'s lifecycle event (debounce/batch) and
|
|
6811
|
+
* accept it, each with the accumulator run the event went to: the subscription's `where`, `input`
|
|
6812
|
+
* and `key` applied to the event as `startSubscribers` does. Undefined when the key cannot be
|
|
6813
|
+
* computed. A subscriber whose `where` declines the event is left out, and so is every one when
|
|
6814
|
+
* this engine does not publish the run's namespace. Empty for a run that has not settled.
|
|
6815
|
+
*/
|
|
6816
|
+
coalescedSubscribers(run) {
|
|
6817
|
+
const out = /* @__PURE__ */ new Map();
|
|
6818
|
+
const kind = lifecycleKindOf(run.status);
|
|
6819
|
+
if (!kind) return out;
|
|
6820
|
+
const triggers = this.eventTriggers.get(lifecycleEventName(run.workflow, kind));
|
|
6821
|
+
if (!triggers || !this.servesNamespace(run.namespace)) return out;
|
|
6822
|
+
let payload;
|
|
6823
|
+
for (const [workflow, subscriptions] of triggers) {
|
|
6824
|
+
const batch = this.latest.get(workflow)?.eventBatch;
|
|
6825
|
+
if (!batch) continue;
|
|
6826
|
+
if (!batch.key && subscriptions.every((sub2) => !sub2.where)) {
|
|
6827
|
+
out.set(workflow, accumulatorRunId(workflow));
|
|
6828
|
+
continue;
|
|
6829
|
+
}
|
|
6830
|
+
try {
|
|
6831
|
+
payload ??= lifecycleEventOf(run);
|
|
6832
|
+
const { accepted } = acceptingSubscription(subscriptions, payload);
|
|
6833
|
+
if (!accepted) continue;
|
|
6834
|
+
const input = accepted.input ? accepted.input(payload) : payload;
|
|
6835
|
+
out.set(workflow, accumulatorRunId(workflow, batch.key ? String(batch.key(input)) : void 0));
|
|
6836
|
+
} catch {
|
|
6837
|
+
out.set(workflow, void 0);
|
|
6838
|
+
}
|
|
6839
|
+
}
|
|
6840
|
+
return out;
|
|
6841
|
+
}
|
|
6842
|
+
/**
|
|
6843
|
+
* Who starts whom through lifecycle events, for `workflow`: the workflows that start when it
|
|
6844
|
+
* settles, and the workflows whose settling starts it. Built from this engine's registry and the
|
|
6845
|
+
* subscriptions live workers announce (see {@link announcedSubscriptions}); a failed fleet read
|
|
6846
|
+
* leaves the local registry alone. What a dashboard's workflow page shows.
|
|
6847
|
+
*
|
|
6848
|
+
* With `namespace`, only the subscriptions of that namespace: the ones announced by registry runs
|
|
6849
|
+
* and worker descriptors serving it, and this engine's own when it is that namespace's engine (an
|
|
6850
|
+
* engine with no namespace is `default`'s). What a tenant is shown.
|
|
6851
|
+
*/
|
|
6852
|
+
async workflowLifecycle(workflow, opts) {
|
|
6853
|
+
const announced = await this.announcedSubscriptions();
|
|
6854
|
+
const scope = opts?.namespace;
|
|
6855
|
+
let fleet = announced.byEvent;
|
|
6856
|
+
if (scope !== void 0) {
|
|
6857
|
+
fleet = /* @__PURE__ */ new Map();
|
|
6858
|
+
for (const entry of announced.entries) {
|
|
6859
|
+
if ((entry.namespace ?? "default") !== scope) continue;
|
|
6860
|
+
const subscribers = fleet.get(entry.event) ?? /* @__PURE__ */ new Set();
|
|
6861
|
+
subscribers.add(entry.workflow);
|
|
6862
|
+
fleet.set(entry.event, subscribers);
|
|
6863
|
+
}
|
|
6864
|
+
}
|
|
6865
|
+
const triggers = scope === void 0 || (this.namespace || "default") === scope ? this.eventTriggers : /* @__PURE__ */ new Map();
|
|
6866
|
+
const startsOnSettle = /* @__PURE__ */ new Map();
|
|
6867
|
+
const startedBy = /* @__PURE__ */ new Map();
|
|
6868
|
+
const link = /* @__PURE__ */ __name((into, other, kind, local) => {
|
|
6869
|
+
const key = `${other}\0${kind}`;
|
|
6870
|
+
const entry = into.get(key) ?? {
|
|
6871
|
+
workflow: other,
|
|
6872
|
+
kind,
|
|
6873
|
+
local: false,
|
|
6874
|
+
announced: false,
|
|
6875
|
+
where: false,
|
|
6876
|
+
input: false
|
|
6877
|
+
};
|
|
6878
|
+
if (local) {
|
|
6879
|
+
entry.local = true;
|
|
6880
|
+
entry.where ||= local.subscriptions.some((sub2) => sub2.where !== void 0);
|
|
6881
|
+
entry.input ||= local.subscriptions.some((sub2) => sub2.input !== void 0);
|
|
6882
|
+
const batch = this.latest.get(local.subscriber)?.eventBatch;
|
|
6883
|
+
if (batch) entry.coalesce = {
|
|
6884
|
+
mode: batch.mode,
|
|
6885
|
+
keyed: batch.key !== void 0
|
|
6886
|
+
};
|
|
6887
|
+
} else {
|
|
6888
|
+
entry.announced = true;
|
|
6889
|
+
}
|
|
6890
|
+
into.set(key, entry);
|
|
6891
|
+
}, "link");
|
|
6892
|
+
for (const kind of [
|
|
6893
|
+
"completed",
|
|
6894
|
+
"failed"
|
|
6895
|
+
]) {
|
|
6896
|
+
const event = lifecycleEventName(workflow, kind);
|
|
6897
|
+
for (const [subscriber, subscriptions] of triggers.get(event) ?? []) {
|
|
6898
|
+
link(startsOnSettle, subscriber, kind, {
|
|
6899
|
+
subscriber,
|
|
6900
|
+
subscriptions
|
|
6901
|
+
});
|
|
6902
|
+
}
|
|
6903
|
+
for (const subscriber of fleet.get(event) ?? []) {
|
|
6904
|
+
link(startsOnSettle, subscriber, kind, void 0);
|
|
6905
|
+
}
|
|
6906
|
+
}
|
|
6907
|
+
for (const [event, subscribers] of triggers) {
|
|
6908
|
+
const source = parseLifecycleEvent(event);
|
|
6909
|
+
const subscriptions = subscribers.get(workflow);
|
|
6910
|
+
if (source && subscriptions) {
|
|
6911
|
+
link(startedBy, source.workflow, source.kind, {
|
|
6912
|
+
subscriber: workflow,
|
|
6913
|
+
subscriptions
|
|
6914
|
+
});
|
|
6915
|
+
}
|
|
6916
|
+
}
|
|
6917
|
+
for (const [event, subscribers] of fleet) {
|
|
6918
|
+
const source = parseLifecycleEvent(event);
|
|
6919
|
+
if (source && subscribers.has(workflow)) {
|
|
6920
|
+
link(startedBy, source.workflow, source.kind, void 0);
|
|
6921
|
+
}
|
|
6922
|
+
}
|
|
6923
|
+
const sorted = /* @__PURE__ */ __name((links) => [
|
|
6924
|
+
...links.values()
|
|
6925
|
+
].sort((a, b) => a.workflow.localeCompare(b.workflow) || a.kind.localeCompare(b.kind)), "sorted");
|
|
6926
|
+
return {
|
|
6927
|
+
workflow,
|
|
6928
|
+
startsOnSettle: sorted(startsOnSettle),
|
|
6929
|
+
startedBy: sorted(startedBy)
|
|
6930
|
+
};
|
|
6931
|
+
}
|
|
6932
|
+
/**
|
|
6933
|
+
* `patch` (a terminal status write of `run`) plus the lifecycle marker, when some workflow
|
|
6934
|
+
* subscribes to that ending: {@link LIFECYCLE_PENDING_TAG} and one owed tag per subscriber (see
|
|
6935
|
+
* {@link lifecycleSubscribers}). Written in the same update as the status, so a run can never be
|
|
6936
|
+
* settled without a record of who its lifecycle event is still owed to.
|
|
6937
|
+
*
|
|
6938
|
+
* Uses the last read of the fleet's announcements without waiting for a new one (see
|
|
6939
|
+
* {@link cachedAnnouncedSubscriptions}). With no usable read (none yet, or the last one failed) the
|
|
6940
|
+
* marker is {@link LIFECYCLE_PENDING_TAG} alone, for any workflow but an internal `__` one: the
|
|
6941
|
+
* owed set is unresolved, and {@link publishSettled} resolves it once a read succeeds.
|
|
6942
|
+
*/
|
|
6943
|
+
async withLifecycleMarker(run, patch) {
|
|
6944
|
+
const kind = lifecycleKindOf(patch.status);
|
|
6945
|
+
if (!kind || run.workflow.startsWith("__")) return patch;
|
|
6946
|
+
const snapshot = this.cachedAnnouncedSubscriptions();
|
|
6947
|
+
const subscribers = snapshot?.ok ? this.lifecycleSubscribersIn(snapshot, run.workflow, kind, runNamespace(run.namespace)) : [];
|
|
6948
|
+
if (snapshot?.ok && subscribers.length === 0) return patch;
|
|
6949
|
+
const tags = patch.tags ?? run.tags ?? [];
|
|
6950
|
+
let traceparent;
|
|
6951
|
+
try {
|
|
6952
|
+
traceparent = this.runTraceparent?.(run.id);
|
|
6953
|
+
} catch {
|
|
6954
|
+
traceparent = void 0;
|
|
6955
|
+
}
|
|
6956
|
+
const recordsTrace = traceparent !== void 0 && !tags.some((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX));
|
|
6957
|
+
const marker = [
|
|
6958
|
+
LIFECYCLE_PENDING_TAG,
|
|
6959
|
+
...subscribers.map((s) => `${LIFECYCLE_OWED_TAG_PREFIX}${s}`),
|
|
6960
|
+
...recordsTrace ? [
|
|
6961
|
+
`${LIFECYCLE_TRACE_TAG_PREFIX}${traceparent}`
|
|
6962
|
+
] : []
|
|
6963
|
+
];
|
|
6964
|
+
if (marker.every((tag) => tags.includes(tag))) return patch;
|
|
6965
|
+
return {
|
|
6966
|
+
...patch,
|
|
6967
|
+
tags: [
|
|
6968
|
+
.../* @__PURE__ */ new Set([
|
|
6969
|
+
...tags,
|
|
6970
|
+
...marker
|
|
6971
|
+
])
|
|
6972
|
+
]
|
|
6973
|
+
};
|
|
6974
|
+
}
|
|
6975
|
+
/**
|
|
6976
|
+
* `patch` (a suspend write of `run`) plus the run's {@link LIFECYCLE_TRACE_TAG_PREFIX} tag, when a
|
|
6977
|
+
* run trace provider is installed, some workflow subscribes to the run's ending, the provider
|
|
6978
|
+
* answers for the run and the run does not carry the tag yet. The first turn's suspend records the
|
|
6979
|
+
* run's root span this way, so a run that settles in a process that never saw its first turn still
|
|
6980
|
+
* links its subscribers' traces; a workflow nobody subscribes to gets no tag and no extra write.
|
|
6981
|
+
* Uses the last read of the fleet's announcements without waiting; with no usable read, any
|
|
6982
|
+
* workflow but an internal one may have subscribers, so it gets the tag.
|
|
6983
|
+
*/
|
|
6984
|
+
async withRunTrace(run, patch) {
|
|
6985
|
+
if (!this.runTraceparent || run.workflow.startsWith("__")) return patch;
|
|
6986
|
+
const tags = patch.tags ?? run.tags ?? [];
|
|
6987
|
+
if (tags.some((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX))) return patch;
|
|
6988
|
+
let traceparent;
|
|
6989
|
+
try {
|
|
6990
|
+
traceparent = this.runTraceparent(run.id);
|
|
6991
|
+
} catch {
|
|
6992
|
+
return patch;
|
|
6993
|
+
}
|
|
6994
|
+
if (!traceparent) return patch;
|
|
6995
|
+
const snapshot = this.cachedAnnouncedSubscriptions();
|
|
6996
|
+
const namespace = runNamespace(run.namespace);
|
|
6997
|
+
const subscribed = !snapshot?.ok || this.lifecycleSubscribersIn(snapshot, run.workflow, "completed", namespace).length > 0 || this.lifecycleSubscribersIn(snapshot, run.workflow, "failed", namespace).length > 0;
|
|
6998
|
+
if (!subscribed) return patch;
|
|
6999
|
+
return {
|
|
7000
|
+
...patch,
|
|
7001
|
+
tags: [
|
|
7002
|
+
...tags,
|
|
7003
|
+
`${LIFECYCLE_TRACE_TAG_PREFIX}${traceparent}`
|
|
7004
|
+
]
|
|
7005
|
+
};
|
|
7006
|
+
}
|
|
7007
|
+
/**
|
|
7008
|
+
* The runs started by `runId` settling (see {@link publishLifecycle}), read from the run's own
|
|
7009
|
+
* tags, so it needs no subscriber registry. Empty for a run that has not settled or started
|
|
7010
|
+
* nobody. Pass the run itself when it is already loaded, to skip reading it again.
|
|
7011
|
+
*/
|
|
7012
|
+
async triggeredRuns(source) {
|
|
7013
|
+
const run = typeof source === "string" ? await this.store.getRun(source) : source;
|
|
7014
|
+
return run ? triggeredRunsOf(run) : [];
|
|
7015
|
+
}
|
|
7016
|
+
/** The live path of {@link publishSettled}: right after this instance settled the run. */
|
|
7017
|
+
publishLifecycle(event) {
|
|
7018
|
+
if (event.type !== "run.completed" && event.type !== "run.failed") return;
|
|
7019
|
+
const kind = event.type === "run.completed" ? "completed" : "failed";
|
|
7020
|
+
if (!event.workflow || !this.hasLifecycleSubscribers(event.workflow, kind, event.namespace)) {
|
|
7021
|
+
return;
|
|
7022
|
+
}
|
|
7023
|
+
void (async () => {
|
|
7024
|
+
const run = await this.store.getRun(event.runId);
|
|
7025
|
+
if (run) await this.publishSettled(run);
|
|
7026
|
+
})().catch(() => void 0);
|
|
7027
|
+
}
|
|
7028
|
+
/** How long to wait before each new try at a run's lifecycle lease another publisher holds. */
|
|
7029
|
+
static LIFECYCLE_LEASE_BACKOFF_MS = [
|
|
7030
|
+
50,
|
|
7031
|
+
100,
|
|
7032
|
+
200,
|
|
7033
|
+
400,
|
|
7034
|
+
800
|
|
7035
|
+
];
|
|
7036
|
+
/**
|
|
7037
|
+
* Run `fn` holding the lease on the settled run `runId` (`tryLockRun`, owner
|
|
7038
|
+
* `<instanceId>#lifecycle#<uuid>`, unique per call), so the live publish, the sweeps of every
|
|
7039
|
+
* instance and a republish update its lifecycle tags one at a time. Retried on the
|
|
7040
|
+
* {@link LIFECYCLE_LEASE_BACKOFF_MS} schedule while another holder has it (the settling execution
|
|
7041
|
+
* still holding its own, or another publisher); `busy` once those tries are spent.
|
|
7042
|
+
*/
|
|
7043
|
+
async withLifecycleLease(runId, fn) {
|
|
7044
|
+
const owner = `${this.instanceId}#lifecycle#${globalThis.crypto.randomUUID()}`;
|
|
7045
|
+
for (let attempt = 0; ; attempt += 1) {
|
|
7046
|
+
const now = this.clock();
|
|
7047
|
+
if (await this.store.tryLockRun(runId, owner, now + this.leaseMs, now)) break;
|
|
7048
|
+
const wait = _WorkflowEngine.LIFECYCLE_LEASE_BACKOFF_MS[attempt];
|
|
7049
|
+
if (wait === void 0) return {
|
|
7050
|
+
busy: true
|
|
7051
|
+
};
|
|
7052
|
+
await new Promise((resolve) => setTimeout(resolve, wait));
|
|
7053
|
+
}
|
|
7054
|
+
try {
|
|
7055
|
+
return {
|
|
7056
|
+
busy: false,
|
|
7057
|
+
value: await fn()
|
|
7058
|
+
};
|
|
7059
|
+
} finally {
|
|
7060
|
+
await this.store.releaseRunLock(runId, owner).catch(() => void 0);
|
|
7061
|
+
}
|
|
7062
|
+
}
|
|
7063
|
+
/**
|
|
7064
|
+
* Publish a settled run's lifecycle event (`run.completed:<workflow>` / `run.failed:<workflow>`)
|
|
7065
|
+
* to the subscribers this engine has, and settle their part of the marker: drop each one's owed
|
|
7066
|
+
* tag, record each subscriber it started as a `durable:triggered:<workflow>` tag and each one it
|
|
7067
|
+
* routed into a debounce/batch accumulator as a `durable:coalesced:<workflow>` tag, and drop
|
|
7068
|
+
* {@link LIFECYCLE_PENDING_TAG} once no owed tag is left. Subscribers registered only elsewhere
|
|
7069
|
+
* keep their owed tags for the instances that have them.
|
|
7070
|
+
*
|
|
7071
|
+
* Which subscribers: `only` when given (a republish), plus `owe`, owed to the instances that have
|
|
7072
|
+
* them; `heal`, the ones a heal pass found missing (see {@link healLifecycleEvents}); else the owed
|
|
7073
|
+
* ones. A run marked with the pending tag alone is unresolved: with a good read of the fleet's
|
|
7074
|
+
* announcements it is owed to every subscriber (see {@link lifecycleSubscribers}) that has not
|
|
7075
|
+
* started, coalesced or failed, and published as such; without one, this engine starts its own subscribers
|
|
7076
|
+
* that do not coalesce and the run stays unresolved.
|
|
7077
|
+
*
|
|
7078
|
+
* Runs under the run's lifecycle lease (see {@link withLifecycleLease}), reading the run again
|
|
7079
|
+
* inside it; `busy` when the lease could not be taken. Idempotent, since every subscriber run id is
|
|
7080
|
+
* deterministic and a debounce/batch accumulator counts the event once, so the live path, the
|
|
7081
|
+
* sweeps and a republish may all reach the same run. `run.triggered` and `run.trigger_failed` are
|
|
7082
|
+
* emitted by the publish whose write recorded them: a subscriber it reached whose triggered or
|
|
7083
|
+
* coalesced tag the run did not have, and one it failed to start whose failed tag the run did not
|
|
7084
|
+
* have.
|
|
7085
|
+
*
|
|
7086
|
+
* A subscriber whose `where` throws or whose start fails is reported as `run.trigger_failed`,
|
|
7087
|
+
* tagged {@link LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX} in the same update, and not retried; a
|
|
7088
|
+
* subscriber it starts loses that tag. A failure to write the marker leaves it, and the sweep
|
|
7089
|
+
* publishes the run again. Returns the subscribers acted on, or undefined when there was nothing
|
|
7090
|
+
* for this engine to do.
|
|
7091
|
+
*/
|
|
7092
|
+
async publishSettled(source, opts = {}) {
|
|
7093
|
+
if (!lifecycleKindOf(source.status)) return void 0;
|
|
7094
|
+
const directed = opts.only !== void 0 || opts.heal !== void 0 || (opts.owe?.length ?? 0) > 0;
|
|
7095
|
+
const tags = source.tags ?? [];
|
|
7096
|
+
const marked = tags.includes(LIFECYCLE_PENDING_TAG) || tags.some((t) => t.startsWith(LIFECYCLE_OWED_TAG_PREFIX));
|
|
7097
|
+
if (!directed && !marked) return void 0;
|
|
7098
|
+
const leased = await this.withLifecycleLease(source.id, () => this.publishSettledLocked(source.id, opts));
|
|
7099
|
+
return leased.busy ? "busy" : leased.value;
|
|
7100
|
+
}
|
|
7101
|
+
/** {@link publishSettled}, under the run's lifecycle lease. */
|
|
7102
|
+
async publishSettledLocked(runId, opts) {
|
|
7103
|
+
const run = await this.store.getRun(runId);
|
|
7104
|
+
const kind = lifecycleKindOf(run?.status);
|
|
7105
|
+
if (!run || !kind) return void 0;
|
|
7106
|
+
const name = lifecycleEventName(run.workflow, kind);
|
|
7107
|
+
const tags = run.tags ?? [];
|
|
7108
|
+
const owed = owedSubscribersOf(tags);
|
|
7109
|
+
const unresolved = tags.includes(LIFECYCLE_PENDING_TAG) && owed.length === 0;
|
|
7110
|
+
const done = /* @__PURE__ */ new Set([
|
|
7111
|
+
...triggeredSubscribersOf(tags),
|
|
7112
|
+
...coalescedSubscribersOf(tags),
|
|
7113
|
+
...triggerFailedSubscribersOf(tags)
|
|
7114
|
+
]);
|
|
7115
|
+
const local = this.localSubscribersIn(name, run.namespace);
|
|
7116
|
+
const remote = await this.announcedRemoteSubscribers(name, run.namespace);
|
|
7117
|
+
const startable = /* @__PURE__ */ new Set([
|
|
7118
|
+
...local,
|
|
7119
|
+
...remote
|
|
7120
|
+
]);
|
|
7121
|
+
const owe = new Set(opts.owe ?? []);
|
|
7122
|
+
let targets;
|
|
7123
|
+
let resolved = false;
|
|
7124
|
+
if (opts.only !== void 0) {
|
|
7125
|
+
const only = opts.only;
|
|
7126
|
+
targets = [
|
|
7127
|
+
...startable
|
|
7128
|
+
].filter((s) => only.has(s));
|
|
7129
|
+
} else if (opts.heal !== void 0) {
|
|
7130
|
+
targets = opts.heal.filter((s) => startable.has(s));
|
|
7131
|
+
for (const s of opts.heal) if (!startable.has(s)) owe.add(s);
|
|
7132
|
+
} else if (unresolved) {
|
|
7133
|
+
const fleet = await this.announcedSubscriptions();
|
|
7134
|
+
if (fleet.ok) {
|
|
7135
|
+
resolved = true;
|
|
7136
|
+
const full = this.lifecycleSubscribersIn(fleet, run.workflow, kind, run.namespace).filter((s) => !done.has(s));
|
|
7137
|
+
targets = full.filter((s) => startable.has(s));
|
|
7138
|
+
for (const s of full) if (!startable.has(s)) owe.add(s);
|
|
7139
|
+
} else {
|
|
7140
|
+
targets = local.filter((s) => !done.has(s) && !this.latest.get(s)?.eventBatch);
|
|
7141
|
+
}
|
|
7142
|
+
} else {
|
|
7143
|
+
targets = owed.filter((s) => startable.has(s));
|
|
7144
|
+
}
|
|
7145
|
+
if (targets.length === 0 && owe.size === 0 && !resolved) return void 0;
|
|
7146
|
+
const runIdOf = /* @__PURE__ */ __name((sub2) => lifecycleRunId(run.id, kind, sub2), "runIdOf");
|
|
7147
|
+
const { reached, failed } = targets.length > 0 ? await this.startSubscribers(name, lifecycleEventOf(run), runIdOf, new Set(targets), remote, runIdOf) : {
|
|
7148
|
+
reached: [],
|
|
7149
|
+
failed: []
|
|
7150
|
+
};
|
|
7151
|
+
const current = await this.store.getRun(run.id) ?? run;
|
|
7152
|
+
const before = current.tags ?? [];
|
|
7153
|
+
const dropped = /* @__PURE__ */ new Set([
|
|
7154
|
+
...targets.map((s) => `${LIFECYCLE_OWED_TAG_PREFIX}${s}`),
|
|
7155
|
+
...reached.map((r) => `${LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX}${r.workflow}`)
|
|
7156
|
+
]);
|
|
7157
|
+
const deliveredTag = /* @__PURE__ */ __name((r) => r.runId !== void 0 ? `${LIFECYCLE_TRIGGERED_TAG_PREFIX}${r.workflow}` : `${LIFECYCLE_COALESCED_TAG_PREFIX}${r.workflow}`, "deliveredTag");
|
|
7158
|
+
const added = [
|
|
7159
|
+
...reached.map(deliveredTag),
|
|
7160
|
+
...failed.map((f) => `${LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX}${f.workflow}`),
|
|
7161
|
+
...[
|
|
7162
|
+
...owe
|
|
7163
|
+
].map((s) => `${LIFECYCLE_OWED_TAG_PREFIX}${s}`)
|
|
7164
|
+
];
|
|
7165
|
+
const kept = before.filter((t) => !dropped.has(t));
|
|
7166
|
+
const keepPending = [
|
|
7167
|
+
...kept,
|
|
7168
|
+
...added
|
|
7169
|
+
].some((t) => t.startsWith(LIFECYCLE_OWED_TAG_PREFIX)) || unresolved && !resolved;
|
|
7170
|
+
const marker = keepPending ? kept.includes(LIFECYCLE_PENDING_TAG) ? kept : [
|
|
7171
|
+
...kept,
|
|
7172
|
+
LIFECYCLE_PENDING_TAG
|
|
7173
|
+
] : kept.filter((t) => t !== LIFECYCLE_PENDING_TAG);
|
|
7174
|
+
const tagsAfter = [
|
|
7175
|
+
.../* @__PURE__ */ new Set([
|
|
7176
|
+
...marker,
|
|
7177
|
+
...added
|
|
7178
|
+
])
|
|
7179
|
+
];
|
|
7180
|
+
if (tagsAfter.length !== before.length || tagsAfter.some((t) => !before.includes(t))) {
|
|
7181
|
+
await this.store.updateRun(run.id, {
|
|
7182
|
+
tags: tagsAfter
|
|
7183
|
+
});
|
|
7184
|
+
}
|
|
7185
|
+
const had = new Set(before);
|
|
7186
|
+
for (const r of reached) {
|
|
7187
|
+
if (had.has(deliveredTag(r))) continue;
|
|
7188
|
+
const { workflow, runId: triggeredRunId } = r;
|
|
7189
|
+
this.emit({
|
|
7190
|
+
type: "run.triggered",
|
|
7191
|
+
runId: run.id,
|
|
7192
|
+
workflow: run.workflow,
|
|
7193
|
+
namespace: run.namespace,
|
|
7194
|
+
name: workflow,
|
|
7195
|
+
triggeredRunId
|
|
7196
|
+
});
|
|
7197
|
+
}
|
|
7198
|
+
for (const { workflow, error: error2 } of failed) {
|
|
7199
|
+
if (had.has(`${LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX}${workflow}`)) continue;
|
|
7200
|
+
this.reportTriggerFailed(run, workflow, error2 instanceof Error ? error2.message : String(error2));
|
|
7201
|
+
}
|
|
7202
|
+
return {
|
|
7203
|
+
reached,
|
|
7204
|
+
failed
|
|
7205
|
+
};
|
|
7206
|
+
}
|
|
7207
|
+
reportTriggerFailed(run, subscriber, message) {
|
|
7208
|
+
this.emit({
|
|
7209
|
+
type: "run.trigger_failed",
|
|
7210
|
+
runId: run.id,
|
|
7211
|
+
workflow: run.workflow,
|
|
7212
|
+
namespace: run.namespace,
|
|
7213
|
+
name: subscriber,
|
|
7214
|
+
error: {
|
|
7215
|
+
message
|
|
7216
|
+
}
|
|
7217
|
+
});
|
|
7218
|
+
}
|
|
7219
|
+
/**
|
|
7220
|
+
* Drop the owed tags of `run`'s subscribers that no live worker announces and this engine does
|
|
7221
|
+
* not have, once the run settled more than {@link LIFECYCLE_ORPHAN_MS} ago, reporting each as
|
|
7222
|
+
* `run.trigger_failed` and tagging it {@link LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX} in the same
|
|
7223
|
+
* update, under the run's lifecycle lease. Skipped when the fleet's announcements could not be
|
|
7224
|
+
* read, since a failed read would make every remote subscriber look orphaned.
|
|
7225
|
+
*/
|
|
7226
|
+
async dropOrphanedSubscribers(run, nowMs) {
|
|
7227
|
+
if (nowMs - run.updatedAt.getTime() < LIFECYCLE_ORPHAN_MS) return false;
|
|
7228
|
+
const kind = lifecycleKindOf(run.status);
|
|
7229
|
+
if (!kind || owedSubscribersOf(run.tags).length === 0) return false;
|
|
7230
|
+
const fleet = await this.announcedSubscriptions();
|
|
7231
|
+
if (!fleet.ok) return false;
|
|
7232
|
+
const leased = await this.withLifecycleLease(run.id, async () => {
|
|
7233
|
+
const current = await this.store.getRun(run.id);
|
|
7234
|
+
const owed = owedSubscribersOf(current?.tags);
|
|
7235
|
+
if (!current || owed.length === 0) return false;
|
|
7236
|
+
const event = lifecycleEventName(run.workflow, kind);
|
|
7237
|
+
const live = /* @__PURE__ */ new Set([
|
|
7238
|
+
...this.localSubscribersIn(event, run.namespace),
|
|
7239
|
+
...announcedIn(fleet, event, run.namespace)
|
|
7240
|
+
]);
|
|
7241
|
+
const orphans = owed.filter((s) => !live.has(s));
|
|
7242
|
+
if (orphans.length === 0) return false;
|
|
7243
|
+
const dropped = new Set(orphans.map((s) => `${LIFECYCLE_OWED_TAG_PREFIX}${s}`));
|
|
7244
|
+
const kept = (current.tags ?? []).filter((t) => !dropped.has(t));
|
|
7245
|
+
const owedLeft = kept.some((t) => t.startsWith(LIFECYCLE_OWED_TAG_PREFIX));
|
|
7246
|
+
await this.store.updateRun(run.id, {
|
|
7247
|
+
tags: [
|
|
7248
|
+
.../* @__PURE__ */ new Set([
|
|
7249
|
+
...kept.filter((t) => owedLeft || t !== LIFECYCLE_PENDING_TAG),
|
|
7250
|
+
...orphans.map((s) => `${LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX}${s}`)
|
|
7251
|
+
])
|
|
7252
|
+
]
|
|
7253
|
+
});
|
|
7254
|
+
for (const subscriber of orphans) {
|
|
7255
|
+
this.reportTriggerFailed(current, subscriber, `subscriber ${subscriber} is not registered on any live worker`);
|
|
7256
|
+
}
|
|
7257
|
+
return true;
|
|
7258
|
+
});
|
|
7259
|
+
return !leased.busy && leased.value;
|
|
7260
|
+
}
|
|
7261
|
+
/** How long a settled run's marker is left to the live publish before the sweep takes it over. */
|
|
7262
|
+
static LIFECYCLE_SWEEP_GRACE_MS = 3e4;
|
|
7263
|
+
/**
|
|
7264
|
+
* How many marked runs (and `ctx.waitForRun` waiters) one sweep examines at most, looking for
|
|
7265
|
+
* `limit` it can act on. Runs owed only to subscribers other instances have are skipped without a
|
|
7266
|
+
* store read, so they cannot fill the window and keep this instance from the ones it can publish.
|
|
7267
|
+
*/
|
|
7268
|
+
static LIFECYCLE_SWEEP_SCAN_MAX = 1e3;
|
|
7269
|
+
/** How many recently settled runs of one workflow a heal pass reads at most. */
|
|
7270
|
+
static LIFECYCLE_HEAL_SCAN_CAP = 100;
|
|
7271
|
+
/**
|
|
7272
|
+
* Publish the lifecycle event of every settled run still marked {@link LIFECYCLE_PENDING_TAG}:
|
|
7273
|
+
* one whose process died, or whose marker write failed, between settling it and publishing, one
|
|
7274
|
+
* owed to a subscriber the settling process does not have, or one whose owed set is unresolved.
|
|
7275
|
+
* Each instance publishes to the subscribers it has, and drops owed subscribers nobody has anymore
|
|
7276
|
+
* (see {@link dropOrphanedSubscribers}). Then heals the recently settled runs that missed a newly
|
|
7277
|
+
* announced subscriber (see {@link healLifecycleEvents}). Also delivers the `ctx.waitForRun` waits
|
|
7278
|
+
* whose run already settled (see {@link sweepRunWaits}). Called by the timer poller; a run settled
|
|
7279
|
+
* in the last {@link LIFECYCLE_SWEEP_GRACE_MS} is left to the live publish. Returns how many runs
|
|
7280
|
+
* it published to a subscriber.
|
|
5839
7281
|
*
|
|
5840
|
-
*
|
|
5841
|
-
*
|
|
5842
|
-
*
|
|
5843
|
-
*
|
|
5844
|
-
*
|
|
5845
|
-
|
|
5846
|
-
|
|
5847
|
-
|
|
5848
|
-
|
|
5849
|
-
|
|
5850
|
-
|
|
5851
|
-
|
|
5852
|
-
|
|
5853
|
-
|
|
5854
|
-
|
|
5855
|
-
|
|
5856
|
-
|
|
5857
|
-
|
|
5858
|
-
|
|
5859
|
-
|
|
5860
|
-
|
|
5861
|
-
|
|
5862
|
-
|
|
5863
|
-
|
|
5864
|
-
|
|
5865
|
-
|
|
5866
|
-
|
|
5867
|
-
|
|
5868
|
-
|
|
5869
|
-
|
|
5870
|
-
|
|
5871
|
-
|
|
5872
|
-
|
|
5873
|
-
|
|
5874
|
-
|
|
5875
|
-
|
|
5876
|
-
if (
|
|
5877
|
-
|
|
5878
|
-
|
|
5879
|
-
|
|
7282
|
+
* Reads the marked runs a page of `limit` at a time, skipping the ones it cannot act on, until it
|
|
7283
|
+
* has acted on `limit` runs (published, or dropped orphans) or examined
|
|
7284
|
+
* {@link LIFECYCLE_SWEEP_SCAN_MAX}. Pages advance by offset over a set the sweep itself shrinks,
|
|
7285
|
+
* so a run that moves into an earlier page is reached on the next tick, which starts over. A run
|
|
7286
|
+
* whose lifecycle lease another publisher holds is left for the next tick.
|
|
7287
|
+
*/
|
|
7288
|
+
async sweepLifecycleEvents(nowMs = this.clock(), limit = 100) {
|
|
7289
|
+
await this.heartbeatLifecycleRegistry().catch((error2) => {
|
|
7290
|
+
console.error("[nestjs-durable] lifecycle registry heartbeat failed; retrying next tick:", error2);
|
|
7291
|
+
});
|
|
7292
|
+
await this.sweepRunWaits(limit).catch((error2) => {
|
|
7293
|
+
console.error("[nestjs-durable] run-wait sweep failed; retrying next tick:", error2);
|
|
7294
|
+
});
|
|
7295
|
+
await this.announcedSubscriptions();
|
|
7296
|
+
const pageSize = Math.max(1, limit);
|
|
7297
|
+
const scanMax = Math.max(pageSize, _WorkflowEngine.LIFECYCLE_SWEEP_SCAN_MAX);
|
|
7298
|
+
let published = 0;
|
|
7299
|
+
let acted = 0;
|
|
7300
|
+
let examined = 0;
|
|
7301
|
+
while (acted < limit && examined < scanMax) {
|
|
7302
|
+
const requested = Math.min(pageSize, scanMax - examined);
|
|
7303
|
+
const page = await this.store.listRuns({
|
|
7304
|
+
tag: LIFECYCLE_PENDING_TAG,
|
|
7305
|
+
statuses: [
|
|
7306
|
+
"completed",
|
|
7307
|
+
"failed",
|
|
7308
|
+
"cancelled",
|
|
7309
|
+
"dead"
|
|
7310
|
+
],
|
|
7311
|
+
...this.namespace !== void 0 ? {
|
|
7312
|
+
namespace: this.namespace
|
|
7313
|
+
} : {},
|
|
7314
|
+
limit: requested,
|
|
7315
|
+
offset: examined
|
|
7316
|
+
});
|
|
7317
|
+
for (const run of page) {
|
|
7318
|
+
if (acted >= limit) break;
|
|
7319
|
+
examined += 1;
|
|
7320
|
+
if (nowMs - run.updatedAt.getTime() < _WorkflowEngine.LIFECYCLE_SWEEP_GRACE_MS) continue;
|
|
7321
|
+
try {
|
|
7322
|
+
const outcome = await this.publishSettled(run);
|
|
7323
|
+
if (outcome === "busy") continue;
|
|
7324
|
+
if (outcome && outcome.reached.length + outcome.failed.length > 0) published += 1;
|
|
7325
|
+
const dropped = await this.dropOrphanedSubscribers(run, nowMs);
|
|
7326
|
+
if (outcome || dropped) acted += 1;
|
|
7327
|
+
} catch (error2) {
|
|
7328
|
+
console.error(`[nestjs-durable] lifecycle sweep: run ${run.id} failed; retrying next tick:`, error2);
|
|
5880
7329
|
}
|
|
5881
|
-
this.bufferedSweepCursor = last.id;
|
|
5882
7330
|
}
|
|
5883
|
-
|
|
5884
|
-
this.bufferedSweepActive = false;
|
|
7331
|
+
if (page.length < requested) break;
|
|
5885
7332
|
}
|
|
5886
|
-
|
|
7333
|
+
try {
|
|
7334
|
+
published += await this.healLifecycleEvents(nowMs);
|
|
7335
|
+
} catch (error2) {
|
|
7336
|
+
console.error("[nestjs-durable] lifecycle heal failed; retrying next tick:", error2);
|
|
7337
|
+
}
|
|
7338
|
+
return published;
|
|
5887
7339
|
}
|
|
5888
7340
|
/**
|
|
5889
|
-
*
|
|
5890
|
-
*
|
|
5891
|
-
* old pending remote step with no live lease (dispatched and, most likely, lost). The timer poller
|
|
5892
|
-
* sweeps for these ({@link sweepStalled}, throttled to one pass a minute) and notifies each episode
|
|
5893
|
-
* ONCE per instance: a run is paged again only after it makes progress and strands again. Wire your
|
|
5894
|
-
* alerting here like {@link onDead}. A listener that throws never breaks the sweep. Returns an
|
|
5895
|
-
* unsubscribe function.
|
|
7341
|
+
* The (run, subscriber) pairs this engine already healed, until when it remembers them; see
|
|
7342
|
+
* {@link healLifecycleEvents}.
|
|
5896
7343
|
*/
|
|
5897
|
-
|
|
5898
|
-
this.stalledListeners.add(listener);
|
|
5899
|
-
return () => this.stalledListeners.delete(listener);
|
|
5900
|
-
}
|
|
7344
|
+
healedLifecycle = /* @__PURE__ */ new Map();
|
|
5901
7345
|
/**
|
|
5902
|
-
*
|
|
5903
|
-
*
|
|
5904
|
-
*
|
|
5905
|
-
*
|
|
5906
|
-
|
|
5907
|
-
|
|
5908
|
-
|
|
5909
|
-
|
|
5910
|
-
|
|
5911
|
-
|
|
5912
|
-
|
|
5913
|
-
|
|
5914
|
-
|
|
5915
|
-
|
|
5916
|
-
|
|
5917
|
-
|
|
5918
|
-
|
|
5919
|
-
|
|
7346
|
+
* Heal the runs settled in the last {@link LIFECYCLE_HEAL_WINDOW_MS} that missed a subscriber: one
|
|
7347
|
+
* the fleet announced only after the settling process last read the announcements, so it is in
|
|
7348
|
+
* none of the run's owed, triggered, coalesced or failed tags. Only subscriptions this engine first saw within
|
|
7349
|
+
* that window are checked (see {@link subscriptionFirstSeen}), since one the fleet already
|
|
7350
|
+
* announced when the run settled was in its owed set. Needs a good read of the fleet's
|
|
7351
|
+
* announcements.
|
|
7352
|
+
*
|
|
7353
|
+
* One `listRuns` per source workflow (`updatedAfter` the window, at most
|
|
7354
|
+
* {@link LIFECYCLE_HEAL_SCAN_CAP} runs), skipping unresolved runs (the sweep resolves those) and
|
|
7355
|
+
* the ones in their {@link LIFECYCLE_SWEEP_GRACE_MS}. Each missed subscriber is published like an
|
|
7356
|
+
* owed one (see {@link publishSettled}): started here when this engine can, owed to the instances
|
|
7357
|
+
* that have it otherwise. A subscriber whose `where` declines the event leaves no tag, so each
|
|
7358
|
+
* engine heals a pair once. Returns how many runs it published to a subscriber.
|
|
7359
|
+
*/
|
|
7360
|
+
async healLifecycleEvents(nowMs) {
|
|
7361
|
+
const fleet = await this.announcedSubscriptions();
|
|
7362
|
+
if (!fleet.ok) return 0;
|
|
7363
|
+
for (const [key, until] of this.healedLifecycle) {
|
|
7364
|
+
if (until <= nowMs) this.healedLifecycle.delete(key);
|
|
7365
|
+
}
|
|
7366
|
+
const windowStart = nowMs - LIFECYCLE_HEAL_WINDOW_MS;
|
|
7367
|
+
const sources = /* @__PURE__ */ new Map();
|
|
7368
|
+
for (const [key, firstSeen] of this.subscriptionFirstSeen) {
|
|
7369
|
+
if (firstSeen < windowStart) continue;
|
|
7370
|
+
const source = parseLifecycleEvent(key.slice(0, key.indexOf("\0")));
|
|
7371
|
+
if (!source || source.workflow.startsWith("__")) continue;
|
|
7372
|
+
const kinds = sources.get(source.workflow) ?? /* @__PURE__ */ new Set();
|
|
7373
|
+
kinds.add(source.kind);
|
|
7374
|
+
sources.set(source.workflow, kinds);
|
|
7375
|
+
}
|
|
7376
|
+
let published = 0;
|
|
7377
|
+
for (const [workflow, kinds] of sources) {
|
|
7378
|
+
const statuses = [];
|
|
7379
|
+
if (kinds.has("completed")) statuses.push("completed");
|
|
7380
|
+
if (kinds.has("failed")) statuses.push("failed", "cancelled", "dead");
|
|
7381
|
+
const runs = await this.store.listRuns({
|
|
7382
|
+
workflow,
|
|
7383
|
+
statuses,
|
|
7384
|
+
updatedAfter: windowStart,
|
|
7385
|
+
...this.namespace !== void 0 ? {
|
|
7386
|
+
namespace: this.namespace
|
|
7387
|
+
} : {},
|
|
7388
|
+
limit: _WorkflowEngine.LIFECYCLE_HEAL_SCAN_CAP
|
|
7389
|
+
});
|
|
7390
|
+
for (const run of runs) {
|
|
7391
|
+
const kind = lifecycleKindOf(run.status);
|
|
7392
|
+
if (!kind) continue;
|
|
7393
|
+
if (nowMs - run.updatedAt.getTime() < _WorkflowEngine.LIFECYCLE_SWEEP_GRACE_MS) continue;
|
|
7394
|
+
const tags = run.tags ?? [];
|
|
7395
|
+
const owed = owedSubscribersOf(tags);
|
|
7396
|
+
if (tags.includes(LIFECYCLE_PENDING_TAG) && owed.length === 0) continue;
|
|
7397
|
+
const event = lifecycleEventName(workflow, kind);
|
|
7398
|
+
const known = /* @__PURE__ */ new Set([
|
|
7399
|
+
...owed,
|
|
7400
|
+
...triggeredSubscribersOf(tags),
|
|
7401
|
+
...coalescedSubscribersOf(tags),
|
|
7402
|
+
...triggerFailedSubscribersOf(tags)
|
|
7403
|
+
]);
|
|
7404
|
+
const missing = this.lifecycleSubscribersIn(fleet, workflow, kind, run.namespace).filter((s) => {
|
|
7405
|
+
if (known.has(s) || this.healedLifecycle.has(`${run.id}\0${s}`)) return false;
|
|
7406
|
+
const firstSeen = this.subscriptionFirstSeen.get(`${event}\0${s}`);
|
|
7407
|
+
return firstSeen !== void 0 && firstSeen >= windowStart;
|
|
7408
|
+
});
|
|
7409
|
+
if (missing.length === 0) continue;
|
|
5920
7410
|
try {
|
|
5921
|
-
|
|
5922
|
-
|
|
7411
|
+
const outcome = await this.publishSettled(run, {
|
|
7412
|
+
heal: missing
|
|
7413
|
+
});
|
|
7414
|
+
if (outcome === "busy") continue;
|
|
7415
|
+
for (const s of missing) {
|
|
7416
|
+
this.healedLifecycle.set(`${run.id}\0${s}`, run.updatedAt.getTime() + LIFECYCLE_HEAL_WINDOW_MS);
|
|
7417
|
+
}
|
|
7418
|
+
if (outcome && outcome.reached.length + outcome.failed.length > 0) published += 1;
|
|
7419
|
+
} catch (error2) {
|
|
7420
|
+
console.error(`[nestjs-durable] lifecycle heal: run ${run.id} failed; retrying next tick:`, error2);
|
|
5923
7421
|
}
|
|
5924
7422
|
}
|
|
5925
|
-
notified.push(info2);
|
|
5926
7423
|
}
|
|
5927
|
-
return
|
|
7424
|
+
return published;
|
|
5928
7425
|
}
|
|
5929
7426
|
/**
|
|
5930
|
-
*
|
|
5931
|
-
* the
|
|
5932
|
-
*
|
|
5933
|
-
*
|
|
5934
|
-
*
|
|
5935
|
-
*
|
|
5936
|
-
*
|
|
5937
|
-
|
|
5938
|
-
|
|
5939
|
-
|
|
5940
|
-
|
|
5941
|
-
|
|
5942
|
-
|
|
5943
|
-
|
|
5944
|
-
|
|
7427
|
+
* Publish the lifecycle events of settled runs again, to subscribers that missed them: one
|
|
7428
|
+
* registered after the runs settled, or one whose run was deleted. Unlike the sweep it needs no
|
|
7429
|
+
* marker. Each subscriber's run id is deterministic, so one already started is not started again.
|
|
7430
|
+
*
|
|
7431
|
+
* Subscribers registered on this engine are started here. A subscriber only live workers announce
|
|
7432
|
+
* is owed the event again (the run is re-marked), so the sweep on an instance that has it starts
|
|
7433
|
+
* it. Only subscribers of the run's lifecycle event in its namespace are published to; a
|
|
7434
|
+
* `subscriber` that is not one is reported in `failed`. Reads at most `limit` (a positive integer,
|
|
7435
|
+
* default 1000) settled runs of a workflow, a page at a time.
|
|
7436
|
+
*/
|
|
7437
|
+
async republishLifecycle(target) {
|
|
7438
|
+
const result = {
|
|
7439
|
+
runs: 0,
|
|
7440
|
+
started: 0,
|
|
7441
|
+
failed: []
|
|
7442
|
+
};
|
|
7443
|
+
if ("runId" in target) {
|
|
7444
|
+
const run = await this.store.getRun(target.runId);
|
|
7445
|
+
if (!run) throw new FatalError(`run ${target.runId} not found`);
|
|
7446
|
+
if (!lifecycleKindOf(run.status)) {
|
|
7447
|
+
throw new FatalError(`run ${target.runId} has not settled (${run.status})`);
|
|
7448
|
+
}
|
|
7449
|
+
await this.republishRun(run, target.subscriber, result);
|
|
7450
|
+
return result;
|
|
7451
|
+
}
|
|
7452
|
+
const statuses = target.kind === "completed" ? [
|
|
7453
|
+
"completed"
|
|
7454
|
+
] : target.kind === "failed" ? [
|
|
7455
|
+
"failed",
|
|
7456
|
+
"cancelled",
|
|
7457
|
+
"dead"
|
|
7458
|
+
] : [
|
|
7459
|
+
"completed",
|
|
7460
|
+
"failed",
|
|
7461
|
+
"cancelled",
|
|
7462
|
+
"dead"
|
|
7463
|
+
];
|
|
7464
|
+
if (target.namespace !== void 0 && this.namespace !== void 0 && target.namespace !== this.namespace) {
|
|
7465
|
+
throw new FatalError(`namespace ${target.namespace} is not this engine's namespace (${this.namespace})`);
|
|
7466
|
+
}
|
|
7467
|
+
const limit = target.limit ?? 1e3;
|
|
7468
|
+
if (!Number.isSafeInteger(limit) || limit < 1) {
|
|
7469
|
+
throw new FatalError(`limit must be a positive integer, got ${String(target.limit)}`);
|
|
7470
|
+
}
|
|
7471
|
+
const namespace = target.namespace ?? this.namespace;
|
|
7472
|
+
const pageSize = Math.min(limit, 200);
|
|
7473
|
+
let read = 0;
|
|
7474
|
+
while (read < limit) {
|
|
7475
|
+
const want = Math.min(pageSize, limit - read);
|
|
7476
|
+
const page = await this.store.listRuns({
|
|
7477
|
+
workflow: target.workflow,
|
|
7478
|
+
statuses,
|
|
7479
|
+
...namespace !== void 0 ? {
|
|
7480
|
+
namespace
|
|
7481
|
+
} : {},
|
|
7482
|
+
limit: want,
|
|
7483
|
+
offset: read
|
|
7484
|
+
});
|
|
7485
|
+
for (const run of page) {
|
|
7486
|
+
const settledAt = run.updatedAt.getTime();
|
|
7487
|
+
if (target.since && settledAt < target.since.getTime()) continue;
|
|
7488
|
+
if (target.until && settledAt > target.until.getTime()) continue;
|
|
7489
|
+
await this.republishRun(run, target.subscriber, result);
|
|
7490
|
+
}
|
|
7491
|
+
read += page.length;
|
|
7492
|
+
if (page.length < want) break;
|
|
7493
|
+
}
|
|
7494
|
+
return result;
|
|
5945
7495
|
}
|
|
5946
|
-
|
|
5947
|
-
|
|
5948
|
-
|
|
5949
|
-
|
|
5950
|
-
|
|
7496
|
+
/** {@link republishLifecycle} for one settled run: start what is missing, report what failed. */
|
|
7497
|
+
async republishRun(run, subscriber, result) {
|
|
7498
|
+
const kind = lifecycleKindOf(run.status);
|
|
7499
|
+
if (!kind) return;
|
|
7500
|
+
const subscribers = (await this.lifecycleSubscribers(run.workflow, kind, runNamespace(run.namespace))).filter((s) => subscriber === void 0 || s === subscriber);
|
|
7501
|
+
if (subscriber !== void 0 && subscribers.length === 0) {
|
|
7502
|
+
result.runs += 1;
|
|
7503
|
+
result.failed.push({
|
|
7504
|
+
runId: run.id,
|
|
7505
|
+
subscriber,
|
|
7506
|
+
error: `subscriber ${subscriber} does not subscribe to ${lifecycleEventName(run.workflow, kind)}`
|
|
7507
|
+
});
|
|
7508
|
+
return;
|
|
7509
|
+
}
|
|
7510
|
+
if (subscribers.length === 0) return;
|
|
7511
|
+
result.runs += 1;
|
|
7512
|
+
const event = lifecycleEventName(run.workflow, kind);
|
|
7513
|
+
const local = /* @__PURE__ */ new Set([
|
|
7514
|
+
...this.localSubscribersIn(event, run.namespace),
|
|
7515
|
+
...await this.announcedRemoteSubscribers(event, run.namespace)
|
|
7516
|
+
]);
|
|
7517
|
+
const triggerFailed = triggerFailedSubscribersOf(run.tags);
|
|
7518
|
+
const coalesced = coalescedSubscribersOf(run.tags);
|
|
7519
|
+
const missing = [];
|
|
7520
|
+
for (const s of subscribers) {
|
|
7521
|
+
if (coalesced.includes(s) && !triggerFailed.includes(s)) continue;
|
|
7522
|
+
if (local.has(s) && this.latest.get(s)?.eventBatch !== void 0) {
|
|
7523
|
+
missing.push(s);
|
|
7524
|
+
continue;
|
|
5951
7525
|
}
|
|
7526
|
+
if (!await this.store.getRun(lifecycleRunId(run.id, kind, s))) missing.push(s);
|
|
7527
|
+
}
|
|
7528
|
+
const toStart = new Set(missing.filter((s) => local.has(s)));
|
|
7529
|
+
const toOwe = missing.filter((s) => !local.has(s));
|
|
7530
|
+
if (toStart.size === 0 && toOwe.length === 0) return;
|
|
7531
|
+
const outcome = await this.publishSettled(run, {
|
|
7532
|
+
only: toStart,
|
|
7533
|
+
owe: toOwe
|
|
7534
|
+
});
|
|
7535
|
+
if (outcome === "busy") {
|
|
7536
|
+
result.failed.push({
|
|
7537
|
+
runId: run.id,
|
|
7538
|
+
subscriber: "*",
|
|
7539
|
+
error: `run ${run.id} is being published elsewhere; try again`
|
|
7540
|
+
});
|
|
7541
|
+
return;
|
|
7542
|
+
}
|
|
7543
|
+
result.started += outcome?.reached.length ?? 0;
|
|
7544
|
+
for (const { workflow, error: error2 } of outcome?.failed ?? []) {
|
|
7545
|
+
result.failed.push({
|
|
7546
|
+
runId: run.id,
|
|
7547
|
+
subscriber: workflow,
|
|
7548
|
+
error: error2 instanceof Error ? error2.message : String(error2)
|
|
7549
|
+
});
|
|
5952
7550
|
}
|
|
5953
7551
|
}
|
|
5954
|
-
/**
|
|
5955
|
-
*
|
|
5956
|
-
|
|
5957
|
-
|
|
5958
|
-
|
|
5959
|
-
|
|
5960
|
-
|
|
5961
|
-
|
|
5962
|
-
|
|
5963
|
-
|
|
5964
|
-
|
|
7552
|
+
/**
|
|
7553
|
+
* Resume the runs waiting on `targetRunId` with `ctx.waitForRun`, once it has settled. Each waiter
|
|
7554
|
+
* is taken before it is delivered to, so the settle path, the sweep and the waiting run's own
|
|
7555
|
+
* check never resume it twice. Returns how many it resumed.
|
|
7556
|
+
*/
|
|
7557
|
+
async deliverRunWaits(targetRunId) {
|
|
7558
|
+
const waiters = (await this.store.listSignalWaiters(runWaitPrefix(targetRunId))).filter((w) => runWaitTargetOf(w) === targetRunId);
|
|
7559
|
+
if (waiters.length === 0) return 0;
|
|
7560
|
+
const target = await settledRunOf(this.store, targetRunId);
|
|
7561
|
+
if (target.state !== "settled") return 0;
|
|
7562
|
+
const payload = lifecycleEventOf(target.run);
|
|
7563
|
+
let delivered = 0;
|
|
7564
|
+
for (const w of waiters) {
|
|
7565
|
+
const waiter = await this.store.takeSignalWaiter(w.token);
|
|
7566
|
+
if (!waiter) continue;
|
|
7567
|
+
await this.deliverSignal(waiter, payload);
|
|
7568
|
+
delivered += 1;
|
|
7569
|
+
}
|
|
7570
|
+
return delivered;
|
|
7571
|
+
}
|
|
7572
|
+
/**
|
|
7573
|
+
* Deliver the `ctx.waitForRun` waits whose run has settled but which were never resumed: a crash
|
|
7574
|
+
* between settling and delivering, or a run that settled by handing off to a continue-as-new
|
|
7575
|
+
* chain that has since ended. Checks the waited-on runs in waiter order, skipping the ones that
|
|
7576
|
+
* have not settled, until it has resumed waiters of `limit` runs or checked
|
|
7577
|
+
* {@link LIFECYCLE_SWEEP_SCAN_MAX}, so long waits on runs still in flight cannot keep it from a
|
|
7578
|
+
* wait owed a delivery. Returns how many waiters it resumed.
|
|
7579
|
+
*/
|
|
7580
|
+
async sweepRunWaits(limit = 100) {
|
|
7581
|
+
const waiters = await this.store.listSignalWaiters(RUN_WAIT_TOKEN_PREFIX);
|
|
7582
|
+
const targets = /* @__PURE__ */ new Set();
|
|
7583
|
+
for (const w of waiters) {
|
|
7584
|
+
const target = runWaitTargetOf(w);
|
|
7585
|
+
if (target !== void 0) targets.add(target);
|
|
7586
|
+
}
|
|
7587
|
+
const scanMax = Math.max(limit, _WorkflowEngine.LIFECYCLE_SWEEP_SCAN_MAX);
|
|
7588
|
+
let delivered = 0;
|
|
7589
|
+
let acted = 0;
|
|
7590
|
+
let examined = 0;
|
|
7591
|
+
for (const target of targets) {
|
|
7592
|
+
if (acted >= limit || examined >= scanMax) break;
|
|
7593
|
+
examined += 1;
|
|
7594
|
+
const resumed = await this.deliverRunWaits(target);
|
|
7595
|
+
if (resumed > 0) {
|
|
7596
|
+
delivered += resumed;
|
|
7597
|
+
acted += 1;
|
|
7598
|
+
}
|
|
7599
|
+
}
|
|
7600
|
+
return delivered;
|
|
5965
7601
|
}
|
|
5966
7602
|
/** Events waiting for the control-plane publish in flight, in emit order (see {@link broadcastEvents}). */
|
|
5967
7603
|
eventsToBroadcast = [];
|
|
@@ -6644,11 +8280,11 @@ var WorkflowEngine = class {
|
|
|
6644
8280
|
message: "execution timeout",
|
|
6645
8281
|
code: "execution_timeout"
|
|
6646
8282
|
};
|
|
6647
|
-
await this.store.updateRun(run.id, {
|
|
8283
|
+
await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
|
|
6648
8284
|
status: "cancelled",
|
|
6649
8285
|
error: error2,
|
|
6650
8286
|
updatedAt: /* @__PURE__ */ new Date()
|
|
6651
|
-
});
|
|
8287
|
+
}));
|
|
6652
8288
|
this.emit({
|
|
6653
8289
|
type: "run.failed",
|
|
6654
8290
|
runId: run.id,
|
|
@@ -6824,11 +8460,11 @@ var WorkflowEngine = class {
|
|
|
6824
8460
|
message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) \u2014 moved to dead-letter`,
|
|
6825
8461
|
code: "max_recovery_attempts"
|
|
6826
8462
|
};
|
|
6827
|
-
await this.store.updateRun(run.id, {
|
|
8463
|
+
await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
|
|
6828
8464
|
status: "dead",
|
|
6829
8465
|
error: error2,
|
|
6830
8466
|
updatedAt: /* @__PURE__ */ new Date()
|
|
6831
|
-
});
|
|
8467
|
+
}));
|
|
6832
8468
|
await this.store.releaseRunLock(run.id);
|
|
6833
8469
|
return this.announceDead({
|
|
6834
8470
|
...run,
|
|
@@ -6906,11 +8542,11 @@ var WorkflowEngine = class {
|
|
|
6906
8542
|
};
|
|
6907
8543
|
const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
|
|
6908
8544
|
run.status
|
|
6909
|
-
], {
|
|
8545
|
+
], await this.withLifecycleMarker(run, {
|
|
6910
8546
|
status: "dead",
|
|
6911
8547
|
error: error2,
|
|
6912
8548
|
updatedAt: /* @__PURE__ */ new Date()
|
|
6913
|
-
});
|
|
8549
|
+
}));
|
|
6914
8550
|
await this.store.releaseRunLock(run.id);
|
|
6915
8551
|
if (!applied) return null;
|
|
6916
8552
|
console.warn(`[nestjs-durable] run ${run.id}: ${error2.message}`);
|
|
@@ -7030,7 +8666,8 @@ var WorkflowEngine = class {
|
|
|
7030
8666
|
const nowMs = this.clock();
|
|
7031
8667
|
const acquired = await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs);
|
|
7032
8668
|
if (!acquired) return null;
|
|
7033
|
-
|
|
8669
|
+
this.lockedForTurn.add(runId);
|
|
8670
|
+
return this.resumeImplicit(runId).finally(() => this.lockedForTurn.delete(runId)).catch(async (err) => {
|
|
7034
8671
|
if (err instanceof NamespaceMismatch) return null;
|
|
7035
8672
|
if (err instanceof WorkflowNotRegistered) {
|
|
7036
8673
|
const run = await this.store.getRun(runId);
|
|
@@ -7314,6 +8951,7 @@ var WorkflowEngine = class {
|
|
|
7314
8951
|
}
|
|
7315
8952
|
try {
|
|
7316
8953
|
const settled = onLocked ? await onLocked(run) : void 0;
|
|
8954
|
+
if (!settled) this.lockedForTurn.add(run.id);
|
|
7317
8955
|
results.push(settled ?? await this.resumeImplicit(run.id));
|
|
7318
8956
|
} catch (error2) {
|
|
7319
8957
|
if (error2 instanceof WorkflowNotRegistered) {
|
|
@@ -7322,6 +8960,8 @@ var WorkflowEngine = class {
|
|
|
7322
8960
|
if (settled !== void 0) continue;
|
|
7323
8961
|
}
|
|
7324
8962
|
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
|
|
8963
|
+
} finally {
|
|
8964
|
+
this.lockedForTurn.delete(run.id);
|
|
7325
8965
|
}
|
|
7326
8966
|
}
|
|
7327
8967
|
return results;
|
|
@@ -7335,7 +8975,8 @@ var WorkflowEngine = class {
|
|
|
7335
8975
|
* 1. **Resumes** every in-flight run waiting on it via `ctx.waitForEvent(name, { match })` whose
|
|
7336
8976
|
* match the payload satisfies (fan-out, vs `signal`'s point-to-point token).
|
|
7337
8977
|
* 2. **Starts** a fresh run of every workflow registered with `onEvent: [name]`, passing the
|
|
7338
|
-
* payload as input
|
|
8978
|
+
* payload as input, and of every workflow live workers announce as subscribing to it that this
|
|
8979
|
+
* engine can start by name (see {@link announcedRemoteSubscribers}). Idempotent by `evt:<id>:<workflow>` — pass `opts.id` to dedupe redeliveries
|
|
7339
8980
|
* of the same logical event (default: a fresh uuid, so each publish triggers once).
|
|
7340
8981
|
*
|
|
7341
8982
|
* Semantics (deliberate — mirrors `signalWithStart`'s reliability contract, so document it exactly):
|
|
@@ -7362,20 +9003,13 @@ var WorkflowEngine = class {
|
|
|
7362
9003
|
touched += 1;
|
|
7363
9004
|
}
|
|
7364
9005
|
}
|
|
7365
|
-
const
|
|
7366
|
-
if (
|
|
9006
|
+
const remote = await this.announcedRemoteSubscribers(name, null);
|
|
9007
|
+
if (this.eventTriggers.get(name)?.size || remote.length > 0) {
|
|
7367
9008
|
const eventId = opts?.id ?? globalThis.crypto.randomUUID();
|
|
7368
|
-
|
|
7369
|
-
|
|
7370
|
-
|
|
7371
|
-
|
|
7372
|
-
await this.accumulators.route(workflow, batch, payload);
|
|
7373
|
-
} else {
|
|
7374
|
-
await this.start(workflow, payload, `evt:${eventId}:${workflow}`);
|
|
7375
|
-
}
|
|
7376
|
-
touched += 1;
|
|
7377
|
-
} catch {
|
|
7378
|
-
}
|
|
9009
|
+
const { reached, failed } = await this.startSubscribers(name, payload, (workflow) => `evt:${eventId}:${workflow}`, void 0, remote);
|
|
9010
|
+
touched += reached.length;
|
|
9011
|
+
for (const { workflow, error: error2 } of failed) {
|
|
9012
|
+
console.warn(`[nestjs-durable] event ${name}: subscriber ${workflow} not started:`, error2);
|
|
7379
9013
|
}
|
|
7380
9014
|
}
|
|
7381
9015
|
if (touched > 0 || opts?.buffer === false) return touched;
|
|
@@ -7832,11 +9466,11 @@ var WorkflowEngine = class {
|
|
|
7832
9466
|
const error2 = {
|
|
7833
9467
|
message: "cancelled"
|
|
7834
9468
|
};
|
|
7835
|
-
await this.store.updateRun(runId, {
|
|
9469
|
+
await this.store.updateRun(runId, await this.withLifecycleMarker(run, {
|
|
7836
9470
|
status: "cancelled",
|
|
7837
9471
|
error: error2,
|
|
7838
9472
|
updatedAt: /* @__PURE__ */ new Date()
|
|
7839
|
-
});
|
|
9473
|
+
}));
|
|
7840
9474
|
this.emit({
|
|
7841
9475
|
type: "run.failed",
|
|
7842
9476
|
runId,
|
|
@@ -8321,6 +9955,7 @@ var WorkflowEngine = class {
|
|
|
8321
9955
|
* than 0.73 (see `legacySingletonHolders`).
|
|
8322
9956
|
*/
|
|
8323
9957
|
async markStarted(run, singleton) {
|
|
9958
|
+
void this.announcedSubscriptions();
|
|
8324
9959
|
const tags = singleton ? withSingletonTracked(run.tags) : run.tags;
|
|
8325
9960
|
const retag = tags !== run.tags;
|
|
8326
9961
|
await this.store.updateRun(run.id, {
|
|
@@ -8363,7 +9998,9 @@ var WorkflowEngine = class {
|
|
|
8363
9998
|
fans: /* @__PURE__ */ new Map()
|
|
8364
9999
|
};
|
|
8365
10000
|
this.wakeReasons.delete(run.id);
|
|
8366
|
-
|
|
10001
|
+
const lockedForTurn = this.lockedForTurn.delete(run.id);
|
|
10002
|
+
const reentered = run.lockedBy === this.instanceId && (run.lockedUntil ?? 0) > lockNow && (lockedForTurn || await this.proveOwnLease(run.id, lockNow));
|
|
10003
|
+
if (!reentered) {
|
|
8367
10004
|
if (!await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow)) {
|
|
8368
10005
|
if (reason.generic) {
|
|
8369
10006
|
if (!this.genericWakeRequested.has(run.id)) {
|
|
@@ -8840,12 +10477,12 @@ ${run.workflowVersion}`;
|
|
|
8840
10477
|
if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
|
|
8841
10478
|
const updatedAt = /* @__PURE__ */ new Date();
|
|
8842
10479
|
if (outcome.kind === "completed") {
|
|
8843
|
-
const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, {
|
|
10480
|
+
const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, await this.withLifecycleMarker(run, {
|
|
8844
10481
|
status: "completed",
|
|
8845
10482
|
output: outcome.output,
|
|
8846
10483
|
error: void 0,
|
|
8847
10484
|
updatedAt
|
|
8848
|
-
});
|
|
10485
|
+
}));
|
|
8849
10486
|
if (!applied2) return this.echoCurrentStatus(run.id);
|
|
8850
10487
|
this.emit({
|
|
8851
10488
|
type: "run.completed",
|
|
@@ -8868,11 +10505,11 @@ ${run.workflowVersion}`;
|
|
|
8868
10505
|
const handed = await this.routeFailure(run, outcome.error);
|
|
8869
10506
|
if (handed && handed.kind !== "failed") return this.settleRun(run, handed);
|
|
8870
10507
|
const error2 = handed?.error ?? outcome.error;
|
|
8871
|
-
const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, {
|
|
10508
|
+
const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, await this.withLifecycleMarker(run, {
|
|
8872
10509
|
status: "failed",
|
|
8873
10510
|
error: error2,
|
|
8874
10511
|
updatedAt
|
|
8875
|
-
});
|
|
10512
|
+
}));
|
|
8876
10513
|
if (!applied2) return this.echoCurrentStatus(run.id);
|
|
8877
10514
|
this.emit({
|
|
8878
10515
|
type: "run.failed",
|
|
@@ -8889,11 +10526,11 @@ ${run.workflowVersion}`;
|
|
|
8889
10526
|
};
|
|
8890
10527
|
}
|
|
8891
10528
|
if (outcome.kind === "blocked") return this.parkBlocked(run, outcome.blocked);
|
|
8892
|
-
const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, {
|
|
10529
|
+
const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, await this.withRunTrace(run, {
|
|
8893
10530
|
status: "suspended",
|
|
8894
10531
|
wakeAt: this.reconcileWakeAt(outcome.wakeAt),
|
|
8895
10532
|
updatedAt
|
|
8896
|
-
});
|
|
10533
|
+
}));
|
|
8897
10534
|
if (!applied) return this.echoCurrentStatus(run.id);
|
|
8898
10535
|
await this.keepDueForKeptWakes(run.id);
|
|
8899
10536
|
this.emit({
|
|
@@ -8923,11 +10560,11 @@ ${run.workflowVersion}`;
|
|
|
8923
10560
|
const error2 = {
|
|
8924
10561
|
message: "cancelled"
|
|
8925
10562
|
};
|
|
8926
|
-
await this.store.updateRun(run.id, {
|
|
10563
|
+
await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
|
|
8927
10564
|
status: "cancelled",
|
|
8928
10565
|
error: error2,
|
|
8929
10566
|
updatedAt: /* @__PURE__ */ new Date()
|
|
8930
|
-
});
|
|
10567
|
+
}));
|
|
8931
10568
|
if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
|
|
8932
10569
|
this.emit({
|
|
8933
10570
|
type: "run.failed",
|
|
@@ -8959,7 +10596,8 @@ ${run.workflowVersion}`;
|
|
|
8959
10596
|
type: "run.started",
|
|
8960
10597
|
runId: run.id,
|
|
8961
10598
|
workflow: run.workflow,
|
|
8962
|
-
namespace: run.namespace
|
|
10599
|
+
namespace: run.namespace,
|
|
10600
|
+
...await this.sourceTraceOf(run)
|
|
8963
10601
|
});
|
|
8964
10602
|
}
|
|
8965
10603
|
if (registered.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
@@ -8982,12 +10620,12 @@ ${run.workflowVersion}`;
|
|
|
8982
10620
|
const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID().slice(0, 8)}`;
|
|
8983
10621
|
const silenceMs = this.remoteAdvanceSilenceMs;
|
|
8984
10622
|
const wakeAt = silenceMs != null ? this.clock() + silenceMs : void 0;
|
|
8985
|
-
await this.store.updateRun(run.id, {
|
|
10623
|
+
await this.store.updateRun(run.id, await this.withRunTrace(run, {
|
|
8986
10624
|
status: "suspended",
|
|
8987
10625
|
awaitingDecisionTaskId: taskId,
|
|
8988
10626
|
wakeAt,
|
|
8989
10627
|
updatedAt: /* @__PURE__ */ new Date()
|
|
8990
|
-
});
|
|
10628
|
+
}));
|
|
8991
10629
|
this.emit({
|
|
8992
10630
|
type: "run.suspended",
|
|
8993
10631
|
runId: run.id,
|
|
@@ -9024,11 +10662,11 @@ ${run.workflowVersion}`;
|
|
|
9024
10662
|
stack: err instanceof Error ? err.stack : void 0
|
|
9025
10663
|
};
|
|
9026
10664
|
if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
|
|
9027
|
-
await this.store.updateRun(run.id, {
|
|
10665
|
+
await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
|
|
9028
10666
|
status: "failed",
|
|
9029
10667
|
error: error2,
|
|
9030
10668
|
updatedAt: /* @__PURE__ */ new Date()
|
|
9031
|
-
});
|
|
10669
|
+
}));
|
|
9032
10670
|
this.emit({
|
|
9033
10671
|
type: "run.failed",
|
|
9034
10672
|
runId: run.id,
|
|
@@ -9566,11 +11204,11 @@ ${run.workflowVersion}`;
|
|
|
9566
11204
|
const error2 = {
|
|
9567
11205
|
message: "cancelled"
|
|
9568
11206
|
};
|
|
9569
|
-
await this.store.updateRun(run.id, {
|
|
11207
|
+
await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
|
|
9570
11208
|
status: "cancelled",
|
|
9571
11209
|
error: error2,
|
|
9572
11210
|
updatedAt: /* @__PURE__ */ new Date()
|
|
9573
|
-
});
|
|
11211
|
+
}));
|
|
9574
11212
|
this.emit({
|
|
9575
11213
|
type: "run.failed",
|
|
9576
11214
|
runId: run.id,
|
|
@@ -9617,7 +11255,8 @@ ${run.workflowVersion}`;
|
|
|
9617
11255
|
type: "run.started",
|
|
9618
11256
|
runId: run.id,
|
|
9619
11257
|
workflow: run.workflow,
|
|
9620
|
-
namespace: run.namespace
|
|
11258
|
+
namespace: run.namespace,
|
|
11259
|
+
...await this.sourceTraceOf(run)
|
|
9621
11260
|
});
|
|
9622
11261
|
}
|
|
9623
11262
|
await this.noteServedWakes(run.id);
|
|
@@ -9666,6 +11305,8 @@ ${run.workflowVersion}`;
|
|
|
9666
11305
|
runId: run.id,
|
|
9667
11306
|
workflow: run.workflow,
|
|
9668
11307
|
namespace: run.namespace
|
|
11308
|
+
}, {
|
|
11309
|
+
lifecycle: false
|
|
9669
11310
|
});
|
|
9670
11311
|
void this.notifyParent(run.id, {
|
|
9671
11312
|
ok: true,
|
|
@@ -11657,6 +13298,13 @@ function runDetailEtag(run, stats, over) {
|
|
|
11657
13298
|
return `"rd-${hash(`${over}${JSON.stringify(run)}${statsFingerprint(stats)}`)}"`;
|
|
11658
13299
|
}
|
|
11659
13300
|
__name(runDetailEtag, "runDetailEtag");
|
|
13301
|
+
function extendEtag(etag, extras) {
|
|
13302
|
+
if (extras === void 0) return etag;
|
|
13303
|
+
const weak = etag.startsWith("W/") ? "W/" : "";
|
|
13304
|
+
const bare = etag.slice(weak.length).replace(/^"|"$/g, "");
|
|
13305
|
+
return `${weak}"${bare}.${hash(JSON.stringify(extras))}"`;
|
|
13306
|
+
}
|
|
13307
|
+
__name(extendEtag, "extendEtag");
|
|
11660
13308
|
function fanPageEtag(group, stats, query) {
|
|
11661
13309
|
const { offset, limit } = fanPageBounds(query);
|
|
11662
13310
|
const key = [
|
|
@@ -12045,7 +13693,82 @@ var CodecStateStore = class {
|
|
|
12045
13693
|
}
|
|
12046
13694
|
};
|
|
12047
13695
|
|
|
13696
|
+
// src/lifecycle-deliveries.ts
|
|
13697
|
+
function lifecycleDeliveriesOf(run, opts) {
|
|
13698
|
+
const kind = lifecycleKindOf(run.status);
|
|
13699
|
+
if (!kind) return [];
|
|
13700
|
+
const tags = run.tags ?? [];
|
|
13701
|
+
const owed = new Set(owedSubscribersOf(tags));
|
|
13702
|
+
const triggered = new Set(tags.filter((t) => t.startsWith(LIFECYCLE_TRIGGERED_TAG_PREFIX)).map((t) => t.slice(LIFECYCLE_TRIGGERED_TAG_PREFIX.length)));
|
|
13703
|
+
const coalescedTagged = new Set(coalescedSubscribersOf(tags));
|
|
13704
|
+
const triggerFailed = new Set(triggerFailedSubscribersOf(tags));
|
|
13705
|
+
const owedToAll = owed.size === 0 && tags.includes(LIFECYCLE_PENDING_TAG);
|
|
13706
|
+
const settledAt = new Date(run.updatedAt).getTime();
|
|
13707
|
+
const names = [
|
|
13708
|
+
.../* @__PURE__ */ new Set([
|
|
13709
|
+
...opts.subscribers,
|
|
13710
|
+
...owed,
|
|
13711
|
+
...triggered,
|
|
13712
|
+
...coalescedTagged,
|
|
13713
|
+
...triggerFailed
|
|
13714
|
+
])
|
|
13715
|
+
].sort();
|
|
13716
|
+
const coalesced = /* @__PURE__ */ __name((subscriber) => {
|
|
13717
|
+
const runId = opts.coalesced?.get(subscriber);
|
|
13718
|
+
return {
|
|
13719
|
+
subscriber,
|
|
13720
|
+
state: "coalesced",
|
|
13721
|
+
...runId !== void 0 ? {
|
|
13722
|
+
runId
|
|
13723
|
+
} : {}
|
|
13724
|
+
};
|
|
13725
|
+
}, "coalesced");
|
|
13726
|
+
return names.map((subscriber) => {
|
|
13727
|
+
if (triggered.has(subscriber)) {
|
|
13728
|
+
return {
|
|
13729
|
+
subscriber,
|
|
13730
|
+
state: "started",
|
|
13731
|
+
runId: lifecycleRunId(run.id, kind, subscriber)
|
|
13732
|
+
};
|
|
13733
|
+
}
|
|
13734
|
+
if (coalescedTagged.has(subscriber)) return coalesced(subscriber);
|
|
13735
|
+
if (triggerFailed.has(subscriber)) return {
|
|
13736
|
+
subscriber,
|
|
13737
|
+
state: "failed"
|
|
13738
|
+
};
|
|
13739
|
+
if (owed.has(subscriber) || owedToAll) {
|
|
13740
|
+
return {
|
|
13741
|
+
subscriber,
|
|
13742
|
+
state: "owed",
|
|
13743
|
+
owedSince: settledAt,
|
|
13744
|
+
...opts.live.has(subscriber) ? {} : {
|
|
13745
|
+
orphanAt: settledAt + LIFECYCLE_ORPHAN_MS
|
|
13746
|
+
}
|
|
13747
|
+
};
|
|
13748
|
+
}
|
|
13749
|
+
if (opts.coalesced?.has(subscriber)) return coalesced(subscriber);
|
|
13750
|
+
return {
|
|
13751
|
+
subscriber,
|
|
13752
|
+
state: "not-started"
|
|
13753
|
+
};
|
|
13754
|
+
});
|
|
13755
|
+
}
|
|
13756
|
+
__name(lifecycleDeliveriesOf, "lifecycleDeliveriesOf");
|
|
13757
|
+
function traceIdOfTraceparent(traceparent) {
|
|
13758
|
+
const traceId = traceparent?.split("-")[1];
|
|
13759
|
+
return traceId && /^[0-9a-f]{32}$/.test(traceId) && !/^0+$/.test(traceId) ? traceId : void 0;
|
|
13760
|
+
}
|
|
13761
|
+
__name(traceIdOfTraceparent, "traceIdOfTraceparent");
|
|
13762
|
+
|
|
12048
13763
|
// src/metrics.ts
|
|
13764
|
+
function countBySubscriber(counts, e) {
|
|
13765
|
+
const workflow = e.workflow ?? "unknown";
|
|
13766
|
+
const subscriber = e.name ?? "unknown";
|
|
13767
|
+
const bySubscriber = counts[workflow] ?? {};
|
|
13768
|
+
bySubscriber[subscriber] = (bySubscriber[subscriber] ?? 0) + 1;
|
|
13769
|
+
counts[workflow] = bySubscriber;
|
|
13770
|
+
}
|
|
13771
|
+
__name(countBySubscriber, "countBySubscriber");
|
|
12049
13772
|
function collectMetrics(engine) {
|
|
12050
13773
|
const m = {
|
|
12051
13774
|
runs: {
|
|
@@ -12057,6 +13780,8 @@ function collectMetrics(engine) {
|
|
|
12057
13780
|
unrouted: 0
|
|
12058
13781
|
},
|
|
12059
13782
|
routedByCode: {},
|
|
13783
|
+
triggeredBySubscriber: {},
|
|
13784
|
+
triggerFailedBySubscriber: {},
|
|
12060
13785
|
steps: {
|
|
12061
13786
|
started: 0,
|
|
12062
13787
|
completed: 0,
|
|
@@ -12108,6 +13833,12 @@ function collectMetrics(engine) {
|
|
|
12108
13833
|
case "run.unrouted":
|
|
12109
13834
|
m.runs.unrouted += 1;
|
|
12110
13835
|
break;
|
|
13836
|
+
case "run.triggered":
|
|
13837
|
+
countBySubscriber(m.triggeredBySubscriber, e);
|
|
13838
|
+
break;
|
|
13839
|
+
case "run.trigger_failed":
|
|
13840
|
+
countBySubscriber(m.triggerFailedBySubscriber, e);
|
|
13841
|
+
break;
|
|
12111
13842
|
case "step.started":
|
|
12112
13843
|
m.steps.started += 1;
|
|
12113
13844
|
break;
|
|
@@ -12136,6 +13867,10 @@ function collectMetrics(engine) {
|
|
|
12136
13867
|
for (const [name, v] of Object.entries(m.byWorkflow)) for (const [event, n] of Object.entries(v)) lines.push(`durable_runs_by_workflow_total{workflow="${name}",event="${event}"} ${n}`);
|
|
12137
13868
|
lines.push("# TYPE durable_runs_routed_total counter");
|
|
12138
13869
|
for (const [code, n] of Object.entries(m.routedByCode)) lines.push(`durable_runs_routed_total{code="${code}"} ${n}`);
|
|
13870
|
+
lines.push("# TYPE durable_runs_triggered_total counter");
|
|
13871
|
+
for (const [workflow, bySubscriber] of Object.entries(m.triggeredBySubscriber)) for (const [subscriber, n] of Object.entries(bySubscriber)) lines.push(`durable_runs_triggered_total{workflow="${workflow}",subscriber="${subscriber}"} ${n}`);
|
|
13872
|
+
lines.push("# TYPE durable_runs_trigger_failed_total counter");
|
|
13873
|
+
for (const [workflow, bySubscriber] of Object.entries(m.triggerFailedBySubscriber)) for (const [subscriber, n] of Object.entries(bySubscriber)) lines.push(`durable_runs_trigger_failed_total{workflow="${workflow}",subscriber="${subscriber}"} ${n}`);
|
|
12139
13874
|
lines.push("# TYPE durable_step_duration_ms_sum counter");
|
|
12140
13875
|
lines.push(`durable_step_duration_ms_sum ${m.stepDuration.sumMs}`);
|
|
12141
13876
|
lines.push("# TYPE durable_step_duration_count counter");
|
|
@@ -12283,6 +14018,9 @@ var TRANSPORT_CANONICAL = Symbol.for("@dudousxd/nestjs-durable:transport");
|
|
|
12283
14018
|
var DURABLE_OPTIONS_CANONICAL = Symbol.for("@dudousxd/nestjs-durable:options");
|
|
12284
14019
|
|
|
12285
14020
|
// src/testing/in-memory-state-store.ts
|
|
14021
|
+
var RETENTION_EXCLUDED = [
|
|
14022
|
+
LIFECYCLE_REGISTRY_WORKFLOW
|
|
14023
|
+
];
|
|
12286
14024
|
var InMemoryStateStore = class {
|
|
12287
14025
|
static {
|
|
12288
14026
|
__name(this, "InMemoryStateStore");
|
|
@@ -12412,12 +14150,13 @@ var InMemoryStateStore = class {
|
|
|
12412
14150
|
if (policy.maxAge == null && policy.maxCount == null) return 0;
|
|
12413
14151
|
const inScope = this.matching({
|
|
12414
14152
|
...policy.scope ?? {},
|
|
12415
|
-
statuses
|
|
14153
|
+
statuses,
|
|
14154
|
+
excludeWorkflows: RETENTION_EXCLUDED
|
|
12416
14155
|
});
|
|
12417
14156
|
const ids = /* @__PURE__ */ new Set();
|
|
12418
14157
|
if (policy.maxAge != null) {
|
|
12419
14158
|
const cutoff = nowMs - parseDuration(policy.maxAge);
|
|
12420
|
-
const expired = inScope.filter((r) => r.updatedAt.getTime() < cutoff).sort((a, b) => a.updatedAt.getTime() - b.updatedAt.getTime()).slice(0, limit);
|
|
14159
|
+
const expired = inScope.filter((r) => r.updatedAt.getTime() < cutoff && !retentionKeeps(r.tags)).sort((a, b) => a.updatedAt.getTime() - b.updatedAt.getTime()).slice(0, limit);
|
|
12421
14160
|
for (const r of expired) ids.add(r.id);
|
|
12422
14161
|
}
|
|
12423
14162
|
if (policy.maxCount != null && ids.size < limit) {
|
|
@@ -12426,6 +14165,7 @@ var InMemoryStateStore = class {
|
|
|
12426
14165
|
].sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime() || b.id.localeCompare(a.id));
|
|
12427
14166
|
for (const r of byRecency.slice(policy.maxCount)) {
|
|
12428
14167
|
if (ids.size >= limit) break;
|
|
14168
|
+
if (retentionKeeps(r.tags)) continue;
|
|
12429
14169
|
ids.add(r.id);
|
|
12430
14170
|
}
|
|
12431
14171
|
}
|
|
@@ -12723,6 +14463,13 @@ var InMemoryStateStore = class {
|
|
|
12723
14463
|
];
|
|
12724
14464
|
if (query.workflow) runs = runs.filter((r) => r.workflow === query.workflow);
|
|
12725
14465
|
if (query.workflows) runs = runs.filter((r) => query.workflows?.includes(r.workflow));
|
|
14466
|
+
if (query.excludeWorkflows?.length) {
|
|
14467
|
+
runs = runs.filter((r) => !query.excludeWorkflows?.includes(r.workflow));
|
|
14468
|
+
}
|
|
14469
|
+
if (query.updatedAfter !== void 0) {
|
|
14470
|
+
const after = query.updatedAfter;
|
|
14471
|
+
runs = runs.filter((r) => r.updatedAt.getTime() >= after);
|
|
14472
|
+
}
|
|
12726
14473
|
if (query.status) runs = runs.filter((r) => r.status === query.status);
|
|
12727
14474
|
if (query.statuses) runs = runs.filter((r) => query.statuses?.includes(r.status));
|
|
12728
14475
|
if (query.tag) runs = runs.filter((r) => r.tags?.includes(query.tag));
|
|
@@ -13018,6 +14765,7 @@ var InMemoryRunQueue = class {
|
|
|
13018
14765
|
}
|
|
13019
14766
|
};
|
|
13020
14767
|
export {
|
|
14768
|
+
ACCUMULATOR_SEEN_MAX,
|
|
13021
14769
|
CHILD_EDGE_PREFIXES,
|
|
13022
14770
|
CONTROL_FLOW_SIGNAL,
|
|
13023
14771
|
CORE_ORIGIN,
|
|
@@ -13049,6 +14797,19 @@ export {
|
|
|
13049
14797
|
InMemoryTransport,
|
|
13050
14798
|
LEGACY_V1_CAPABILITIES,
|
|
13051
14799
|
LEGACY_V1_PROTOCOL,
|
|
14800
|
+
LIFECYCLE_COALESCED_TAG_PREFIX,
|
|
14801
|
+
LIFECYCLE_HEAL_WINDOW_MS,
|
|
14802
|
+
LIFECYCLE_ORPHAN_MS,
|
|
14803
|
+
LIFECYCLE_OWED_TAG_PREFIX,
|
|
14804
|
+
LIFECYCLE_PENDING_TAG,
|
|
14805
|
+
LIFECYCLE_REGISTRY_HEARTBEAT_MS,
|
|
14806
|
+
LIFECYCLE_REGISTRY_STALE_MS,
|
|
14807
|
+
LIFECYCLE_REGISTRY_TAG,
|
|
14808
|
+
LIFECYCLE_REGISTRY_TTL_MS,
|
|
14809
|
+
LIFECYCLE_REGISTRY_WORKFLOW,
|
|
14810
|
+
LIFECYCLE_TRACE_TAG_PREFIX,
|
|
14811
|
+
LIFECYCLE_TRIGGERED_TAG_PREFIX,
|
|
14812
|
+
LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX,
|
|
13052
14813
|
NestedWorkflowCallError,
|
|
13053
14814
|
NonDeterminismError,
|
|
13054
14815
|
QueueController,
|
|
@@ -13056,6 +14817,7 @@ export {
|
|
|
13056
14817
|
RUN_HISTORY_FIXTURE_VERSION,
|
|
13057
14818
|
RUN_VALUE_FACET_LIMIT,
|
|
13058
14819
|
RUN_VALUE_FACET_SCAN,
|
|
14820
|
+
RUN_WAIT_TOKEN_PREFIX,
|
|
13059
14821
|
RemoteStepError,
|
|
13060
14822
|
RemoteStepTimeout,
|
|
13061
14823
|
RemoteWorkflowExecutor,
|
|
@@ -13084,6 +14846,8 @@ export {
|
|
|
13084
14846
|
WorkflowEngine,
|
|
13085
14847
|
WorkflowNotRegistered,
|
|
13086
14848
|
WorkflowSuspended,
|
|
14849
|
+
acceptingSubscription,
|
|
14850
|
+
accumulatorRunId,
|
|
13087
14851
|
aggregateAnnouncements,
|
|
13088
14852
|
aggregatesCheckpoints,
|
|
13089
14853
|
announcementsOf,
|
|
@@ -13099,12 +14863,14 @@ export {
|
|
|
13099
14863
|
buildWorkflowDirectory,
|
|
13100
14864
|
canRoute,
|
|
13101
14865
|
capableWorkers,
|
|
14866
|
+
carriesLifecyclePending,
|
|
13102
14867
|
checkpointNameKind,
|
|
13103
14868
|
checkpointStatsOf,
|
|
13104
14869
|
childIdsOf,
|
|
13105
14870
|
classifyWaiterToken,
|
|
13106
14871
|
clearDurableExecutionWrappers,
|
|
13107
14872
|
clearTerminalSingletonAdmission,
|
|
14873
|
+
coalescedSubscribersOf,
|
|
13108
14874
|
codeAndDetails,
|
|
13109
14875
|
codeOf,
|
|
13110
14876
|
collectMetrics,
|
|
@@ -13127,6 +14893,7 @@ export {
|
|
|
13127
14893
|
eventNameOf,
|
|
13128
14894
|
eventPrefix,
|
|
13129
14895
|
eventToken,
|
|
14896
|
+
extendEtag,
|
|
13130
14897
|
facetOrigin,
|
|
13131
14898
|
fanPageBounds,
|
|
13132
14899
|
fanPageEtag,
|
|
@@ -13144,10 +14911,21 @@ export {
|
|
|
13144
14911
|
isEventToken,
|
|
13145
14912
|
isFailedMember,
|
|
13146
14913
|
isLegacyDescriptor,
|
|
14914
|
+
isLifecycleTag,
|
|
13147
14915
|
isNonEmptyRunScope,
|
|
13148
14916
|
isRemoteEvent,
|
|
13149
14917
|
isWorkflowControlFlowSignal,
|
|
13150
14918
|
legacySingletonHolders,
|
|
14919
|
+
lifecycleDeliveriesOf,
|
|
14920
|
+
lifecycleEventName,
|
|
14921
|
+
lifecycleEventOf,
|
|
14922
|
+
lifecycleKindOf,
|
|
14923
|
+
lifecycleRegistryNamespace,
|
|
14924
|
+
lifecycleRegistryRunId,
|
|
14925
|
+
lifecycleRegistrySubscriptions,
|
|
14926
|
+
lifecycleRunId,
|
|
14927
|
+
lifecycleSourceOf,
|
|
14928
|
+
lifecycleTraceparentOf,
|
|
13151
14929
|
listSignalWaitersOfRuns,
|
|
13152
14930
|
liveHandoff,
|
|
13153
14931
|
log,
|
|
@@ -13162,10 +14940,12 @@ export {
|
|
|
13162
14940
|
normalizeDescriptor,
|
|
13163
14941
|
observedAnnouncements,
|
|
13164
14942
|
onErrorRunId,
|
|
14943
|
+
owedSubscribersOf,
|
|
13165
14944
|
pageFan,
|
|
13166
14945
|
pageFanFromStore,
|
|
13167
14946
|
parentRunIdOf,
|
|
13168
14947
|
parseDuration,
|
|
14948
|
+
parseLifecycleEvent,
|
|
13169
14949
|
parseWebhookToken,
|
|
13170
14950
|
partitionOfToken,
|
|
13171
14951
|
planDispatch,
|
|
@@ -13180,10 +14960,13 @@ export {
|
|
|
13180
14960
|
resolveRunWaiting,
|
|
13181
14961
|
resolveTenantToken,
|
|
13182
14962
|
resolveWireTenant,
|
|
14963
|
+
retentionKeeps,
|
|
13183
14964
|
retiredHandoffName,
|
|
13184
14965
|
routeFor,
|
|
14966
|
+
runCompletedEvent,
|
|
13185
14967
|
runDetailEtag,
|
|
13186
14968
|
runDurableExecution,
|
|
14969
|
+
runFailedEvent,
|
|
13187
14970
|
runInStepLogger,
|
|
13188
14971
|
runInStepScope,
|
|
13189
14972
|
runInWorkflowCtx,
|
|
@@ -13191,6 +14974,9 @@ export {
|
|
|
13191
14974
|
runSchedules,
|
|
13192
14975
|
runStepHandler,
|
|
13193
14976
|
runValueFacetsFromRuns,
|
|
14977
|
+
runWaitPrefix,
|
|
14978
|
+
runWaitTargetOf,
|
|
14979
|
+
runWaitToken,
|
|
13194
14980
|
sanitizeQueueToken,
|
|
13195
14981
|
scheduleRunIdAt,
|
|
13196
14982
|
scheduledRunId,
|
|
@@ -13209,6 +14995,11 @@ export {
|
|
|
13209
14995
|
summarizeRunFromStore,
|
|
13210
14996
|
summarizeTimeline,
|
|
13211
14997
|
tenantGroup,
|
|
14998
|
+
toEventSubscription,
|
|
14999
|
+
traceIdOfTraceparent,
|
|
15000
|
+
triggerFailedSubscribersOf,
|
|
15001
|
+
triggeredRunsOf,
|
|
15002
|
+
triggeredSubscribersOf,
|
|
13212
15003
|
useDurableExecution,
|
|
13213
15004
|
warn,
|
|
13214
15005
|
withSingletonTracked,
|