@dudousxd/nestjs-durable-core 0.86.1 → 0.87.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +2052 -217
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +913 -71
- package/dist/index.d.ts +913 -71
- package/dist/index.js +2008 -217
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -22,6 +22,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
22
22
|
// src/index.ts
|
|
23
23
|
var index_exports = {};
|
|
24
24
|
__export(index_exports, {
|
|
25
|
+
ACCUMULATOR_SEEN_MAX: () => ACCUMULATOR_SEEN_MAX,
|
|
25
26
|
CHILD_EDGE_PREFIXES: () => CHILD_EDGE_PREFIXES,
|
|
26
27
|
CONTROL_FLOW_SIGNAL: () => CONTROL_FLOW_SIGNAL,
|
|
27
28
|
CORE_ORIGIN: () => CORE_ORIGIN,
|
|
@@ -53,6 +54,19 @@ __export(index_exports, {
|
|
|
53
54
|
InMemoryTransport: () => InMemoryTransport,
|
|
54
55
|
LEGACY_V1_CAPABILITIES: () => LEGACY_V1_CAPABILITIES,
|
|
55
56
|
LEGACY_V1_PROTOCOL: () => LEGACY_V1_PROTOCOL,
|
|
57
|
+
LIFECYCLE_COALESCED_TAG_PREFIX: () => LIFECYCLE_COALESCED_TAG_PREFIX,
|
|
58
|
+
LIFECYCLE_HEAL_WINDOW_MS: () => LIFECYCLE_HEAL_WINDOW_MS,
|
|
59
|
+
LIFECYCLE_ORPHAN_MS: () => LIFECYCLE_ORPHAN_MS,
|
|
60
|
+
LIFECYCLE_OWED_TAG_PREFIX: () => LIFECYCLE_OWED_TAG_PREFIX,
|
|
61
|
+
LIFECYCLE_PENDING_TAG: () => LIFECYCLE_PENDING_TAG,
|
|
62
|
+
LIFECYCLE_REGISTRY_HEARTBEAT_MS: () => LIFECYCLE_REGISTRY_HEARTBEAT_MS,
|
|
63
|
+
LIFECYCLE_REGISTRY_STALE_MS: () => LIFECYCLE_REGISTRY_STALE_MS,
|
|
64
|
+
LIFECYCLE_REGISTRY_TAG: () => LIFECYCLE_REGISTRY_TAG,
|
|
65
|
+
LIFECYCLE_REGISTRY_TTL_MS: () => LIFECYCLE_REGISTRY_TTL_MS,
|
|
66
|
+
LIFECYCLE_REGISTRY_WORKFLOW: () => LIFECYCLE_REGISTRY_WORKFLOW,
|
|
67
|
+
LIFECYCLE_TRACE_TAG_PREFIX: () => LIFECYCLE_TRACE_TAG_PREFIX,
|
|
68
|
+
LIFECYCLE_TRIGGERED_TAG_PREFIX: () => LIFECYCLE_TRIGGERED_TAG_PREFIX,
|
|
69
|
+
LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX: () => LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX,
|
|
56
70
|
NestedWorkflowCallError: () => NestedWorkflowCallError,
|
|
57
71
|
NonDeterminismError: () => NonDeterminismError,
|
|
58
72
|
QueueController: () => QueueController,
|
|
@@ -60,6 +74,7 @@ __export(index_exports, {
|
|
|
60
74
|
RUN_HISTORY_FIXTURE_VERSION: () => RUN_HISTORY_FIXTURE_VERSION,
|
|
61
75
|
RUN_VALUE_FACET_LIMIT: () => RUN_VALUE_FACET_LIMIT,
|
|
62
76
|
RUN_VALUE_FACET_SCAN: () => RUN_VALUE_FACET_SCAN,
|
|
77
|
+
RUN_WAIT_TOKEN_PREFIX: () => RUN_WAIT_TOKEN_PREFIX,
|
|
63
78
|
RemoteStepError: () => RemoteStepError,
|
|
64
79
|
RemoteStepTimeout: () => RemoteStepTimeout,
|
|
65
80
|
RemoteWorkflowExecutor: () => RemoteWorkflowExecutor,
|
|
@@ -88,6 +103,8 @@ __export(index_exports, {
|
|
|
88
103
|
WorkflowEngine: () => WorkflowEngine,
|
|
89
104
|
WorkflowNotRegistered: () => WorkflowNotRegistered,
|
|
90
105
|
WorkflowSuspended: () => WorkflowSuspended,
|
|
106
|
+
acceptingSubscription: () => acceptingSubscription,
|
|
107
|
+
accumulatorRunId: () => accumulatorRunId,
|
|
91
108
|
aggregateAnnouncements: () => aggregateAnnouncements,
|
|
92
109
|
aggregatesCheckpoints: () => aggregatesCheckpoints,
|
|
93
110
|
announcementsOf: () => announcementsOf,
|
|
@@ -103,12 +120,14 @@ __export(index_exports, {
|
|
|
103
120
|
buildWorkflowDirectory: () => buildWorkflowDirectory,
|
|
104
121
|
canRoute: () => canRoute,
|
|
105
122
|
capableWorkers: () => capableWorkers,
|
|
123
|
+
carriesLifecyclePending: () => carriesLifecyclePending,
|
|
106
124
|
checkpointNameKind: () => checkpointNameKind,
|
|
107
125
|
checkpointStatsOf: () => checkpointStatsOf,
|
|
108
126
|
childIdsOf: () => childIdsOf,
|
|
109
127
|
classifyWaiterToken: () => classifyWaiterToken,
|
|
110
128
|
clearDurableExecutionWrappers: () => clearDurableExecutionWrappers,
|
|
111
129
|
clearTerminalSingletonAdmission: () => clearTerminalSingletonAdmission,
|
|
130
|
+
coalescedSubscribersOf: () => coalescedSubscribersOf,
|
|
112
131
|
codeAndDetails: () => codeAndDetails,
|
|
113
132
|
codeOf: () => codeOf,
|
|
114
133
|
collectMetrics: () => collectMetrics,
|
|
@@ -131,6 +150,7 @@ __export(index_exports, {
|
|
|
131
150
|
eventNameOf: () => eventNameOf,
|
|
132
151
|
eventPrefix: () => eventPrefix,
|
|
133
152
|
eventToken: () => eventToken,
|
|
153
|
+
extendEtag: () => extendEtag,
|
|
134
154
|
facetOrigin: () => facetOrigin,
|
|
135
155
|
fanPageBounds: () => fanPageBounds,
|
|
136
156
|
fanPageEtag: () => fanPageEtag,
|
|
@@ -148,10 +168,21 @@ __export(index_exports, {
|
|
|
148
168
|
isEventToken: () => isEventToken,
|
|
149
169
|
isFailedMember: () => isFailedMember,
|
|
150
170
|
isLegacyDescriptor: () => isLegacyDescriptor,
|
|
171
|
+
isLifecycleTag: () => isLifecycleTag,
|
|
151
172
|
isNonEmptyRunScope: () => isNonEmptyRunScope,
|
|
152
173
|
isRemoteEvent: () => isRemoteEvent,
|
|
153
174
|
isWorkflowControlFlowSignal: () => isWorkflowControlFlowSignal,
|
|
154
175
|
legacySingletonHolders: () => legacySingletonHolders,
|
|
176
|
+
lifecycleDeliveriesOf: () => lifecycleDeliveriesOf,
|
|
177
|
+
lifecycleEventName: () => lifecycleEventName,
|
|
178
|
+
lifecycleEventOf: () => lifecycleEventOf,
|
|
179
|
+
lifecycleKindOf: () => lifecycleKindOf,
|
|
180
|
+
lifecycleRegistryNamespace: () => lifecycleRegistryNamespace,
|
|
181
|
+
lifecycleRegistryRunId: () => lifecycleRegistryRunId,
|
|
182
|
+
lifecycleRegistrySubscriptions: () => lifecycleRegistrySubscriptions,
|
|
183
|
+
lifecycleRunId: () => lifecycleRunId,
|
|
184
|
+
lifecycleSourceOf: () => lifecycleSourceOf,
|
|
185
|
+
lifecycleTraceparentOf: () => lifecycleTraceparentOf,
|
|
155
186
|
listSignalWaitersOfRuns: () => listSignalWaitersOfRuns,
|
|
156
187
|
liveHandoff: () => liveHandoff,
|
|
157
188
|
log: () => log,
|
|
@@ -166,10 +197,12 @@ __export(index_exports, {
|
|
|
166
197
|
normalizeDescriptor: () => normalizeDescriptor,
|
|
167
198
|
observedAnnouncements: () => observedAnnouncements,
|
|
168
199
|
onErrorRunId: () => onErrorRunId,
|
|
200
|
+
owedSubscribersOf: () => owedSubscribersOf,
|
|
169
201
|
pageFan: () => pageFan,
|
|
170
202
|
pageFanFromStore: () => pageFanFromStore,
|
|
171
203
|
parentRunIdOf: () => parentRunIdOf,
|
|
172
204
|
parseDuration: () => parseDuration,
|
|
205
|
+
parseLifecycleEvent: () => parseLifecycleEvent,
|
|
173
206
|
parseWebhookToken: () => parseWebhookToken,
|
|
174
207
|
partitionOfToken: () => partitionOfToken,
|
|
175
208
|
planDispatch: () => planDispatch,
|
|
@@ -184,10 +217,13 @@ __export(index_exports, {
|
|
|
184
217
|
resolveRunWaiting: () => resolveRunWaiting,
|
|
185
218
|
resolveTenantToken: () => resolveTenantToken,
|
|
186
219
|
resolveWireTenant: () => resolveWireTenant,
|
|
220
|
+
retentionKeeps: () => retentionKeeps,
|
|
187
221
|
retiredHandoffName: () => retiredHandoffName,
|
|
188
222
|
routeFor: () => routeFor,
|
|
223
|
+
runCompletedEvent: () => runCompletedEvent,
|
|
189
224
|
runDetailEtag: () => runDetailEtag,
|
|
190
225
|
runDurableExecution: () => runDurableExecution,
|
|
226
|
+
runFailedEvent: () => runFailedEvent,
|
|
191
227
|
runInStepLogger: () => runInStepLogger,
|
|
192
228
|
runInStepScope: () => runInStepScope,
|
|
193
229
|
runInWorkflowCtx: () => runInWorkflowCtx,
|
|
@@ -195,6 +231,9 @@ __export(index_exports, {
|
|
|
195
231
|
runSchedules: () => runSchedules,
|
|
196
232
|
runStepHandler: () => runStepHandler,
|
|
197
233
|
runValueFacetsFromRuns: () => runValueFacetsFromRuns,
|
|
234
|
+
runWaitPrefix: () => runWaitPrefix,
|
|
235
|
+
runWaitTargetOf: () => runWaitTargetOf,
|
|
236
|
+
runWaitToken: () => runWaitToken,
|
|
198
237
|
sanitizeQueueToken: () => sanitizeQueueToken,
|
|
199
238
|
scheduleRunIdAt: () => scheduleRunIdAt,
|
|
200
239
|
scheduledRunId: () => scheduledRunId,
|
|
@@ -213,6 +252,11 @@ __export(index_exports, {
|
|
|
213
252
|
summarizeRunFromStore: () => summarizeRunFromStore,
|
|
214
253
|
summarizeTimeline: () => summarizeTimeline,
|
|
215
254
|
tenantGroup: () => tenantGroup,
|
|
255
|
+
toEventSubscription: () => toEventSubscription,
|
|
256
|
+
traceIdOfTraceparent: () => traceIdOfTraceparent,
|
|
257
|
+
triggerFailedSubscribersOf: () => triggerFailedSubscribersOf,
|
|
258
|
+
triggeredRunsOf: () => triggeredRunsOf,
|
|
259
|
+
triggeredSubscribersOf: () => triggeredSubscribersOf,
|
|
216
260
|
useDurableExecution: () => useDurableExecution,
|
|
217
261
|
warn: () => warn,
|
|
218
262
|
withSingletonTracked: () => withSingletonTracked,
|
|
@@ -1033,6 +1077,243 @@ function eventMatches(payload, match) {
|
|
|
1033
1077
|
return keys.every((k) => JSON.stringify(p[k]) === JSON.stringify(match[k]));
|
|
1034
1078
|
}
|
|
1035
1079
|
__name(eventMatches, "eventMatches");
|
|
1080
|
+
function toEventSubscription(entry) {
|
|
1081
|
+
return typeof entry === "string" ? {
|
|
1082
|
+
event: entry
|
|
1083
|
+
} : entry;
|
|
1084
|
+
}
|
|
1085
|
+
__name(toEventSubscription, "toEventSubscription");
|
|
1086
|
+
function acceptingSubscription(subscriptions, payload) {
|
|
1087
|
+
const errors = [];
|
|
1088
|
+
for (const subscription of subscriptions) {
|
|
1089
|
+
if (!subscription.where) return {
|
|
1090
|
+
accepted: subscription,
|
|
1091
|
+
errors
|
|
1092
|
+
};
|
|
1093
|
+
try {
|
|
1094
|
+
if (subscription.where(payload)) return {
|
|
1095
|
+
accepted: subscription,
|
|
1096
|
+
errors
|
|
1097
|
+
};
|
|
1098
|
+
} catch (error2) {
|
|
1099
|
+
errors.push(error2);
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
return {
|
|
1103
|
+
errors
|
|
1104
|
+
};
|
|
1105
|
+
}
|
|
1106
|
+
__name(acceptingSubscription, "acceptingSubscription");
|
|
1107
|
+
|
|
1108
|
+
// src/lifecycle-events.ts
|
|
1109
|
+
function runCompletedEvent(workflow) {
|
|
1110
|
+
return `run.completed:${workflowName(workflow)}`;
|
|
1111
|
+
}
|
|
1112
|
+
__name(runCompletedEvent, "runCompletedEvent");
|
|
1113
|
+
function runFailedEvent(workflow) {
|
|
1114
|
+
return `run.failed:${workflowName(workflow)}`;
|
|
1115
|
+
}
|
|
1116
|
+
__name(runFailedEvent, "runFailedEvent");
|
|
1117
|
+
var LIFECYCLE_PENDING_TAG = "durable:lifecycle:pending";
|
|
1118
|
+
function carriesLifecyclePending(tags) {
|
|
1119
|
+
if (Array.isArray(tags)) return tags.includes(LIFECYCLE_PENDING_TAG);
|
|
1120
|
+
if (typeof tags === "string") return tags.includes(`"${LIFECYCLE_PENDING_TAG}"`);
|
|
1121
|
+
return false;
|
|
1122
|
+
}
|
|
1123
|
+
__name(carriesLifecyclePending, "carriesLifecyclePending");
|
|
1124
|
+
var LIFECYCLE_TRIGGERED_TAG_PREFIX = "durable:triggered:";
|
|
1125
|
+
var LIFECYCLE_OWED_TAG_PREFIX = "durable:lifecycle:owed:";
|
|
1126
|
+
var LIFECYCLE_ORPHAN_MS = 24 * 60 * 60 * 1e3;
|
|
1127
|
+
var LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX = "durable:trigger_failed:";
|
|
1128
|
+
function triggerFailedSubscribersOf(tags) {
|
|
1129
|
+
return (tags ?? []).filter((tag) => tag.startsWith(LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX)).map((tag) => tag.slice(LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX.length));
|
|
1130
|
+
}
|
|
1131
|
+
__name(triggerFailedSubscribersOf, "triggerFailedSubscribersOf");
|
|
1132
|
+
function triggeredSubscribersOf(tags) {
|
|
1133
|
+
return (tags ?? []).filter((tag) => tag.startsWith(LIFECYCLE_TRIGGERED_TAG_PREFIX)).map((tag) => tag.slice(LIFECYCLE_TRIGGERED_TAG_PREFIX.length));
|
|
1134
|
+
}
|
|
1135
|
+
__name(triggeredSubscribersOf, "triggeredSubscribersOf");
|
|
1136
|
+
var LIFECYCLE_COALESCED_TAG_PREFIX = "durable:coalesced:";
|
|
1137
|
+
function coalescedSubscribersOf(tags) {
|
|
1138
|
+
return (tags ?? []).filter((tag) => tag.startsWith(LIFECYCLE_COALESCED_TAG_PREFIX)).map((tag) => tag.slice(LIFECYCLE_COALESCED_TAG_PREFIX.length));
|
|
1139
|
+
}
|
|
1140
|
+
__name(coalescedSubscribersOf, "coalescedSubscribersOf");
|
|
1141
|
+
var LIFECYCLE_HEAL_WINDOW_MS = 2 * 60 * 1e3;
|
|
1142
|
+
function owedSubscribersOf(tags) {
|
|
1143
|
+
return (tags ?? []).filter((tag) => tag.startsWith(LIFECYCLE_OWED_TAG_PREFIX)).map((tag) => tag.slice(LIFECYCLE_OWED_TAG_PREFIX.length));
|
|
1144
|
+
}
|
|
1145
|
+
__name(owedSubscribersOf, "owedSubscribersOf");
|
|
1146
|
+
var LIFECYCLE_TRACE_TAG_PREFIX = "durable:lifecycle:trace:";
|
|
1147
|
+
function lifecycleTraceparentOf(tags) {
|
|
1148
|
+
const tag = (tags ?? []).find((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX));
|
|
1149
|
+
return tag?.slice(LIFECYCLE_TRACE_TAG_PREFIX.length) || void 0;
|
|
1150
|
+
}
|
|
1151
|
+
__name(lifecycleTraceparentOf, "lifecycleTraceparentOf");
|
|
1152
|
+
function isLifecycleTag(tag) {
|
|
1153
|
+
return tag === LIFECYCLE_PENDING_TAG || tag.startsWith(LIFECYCLE_OWED_TAG_PREFIX) || tag.startsWith(LIFECYCLE_TRIGGERED_TAG_PREFIX) || tag.startsWith(LIFECYCLE_COALESCED_TAG_PREFIX) || tag.startsWith(LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX) || tag.startsWith(LIFECYCLE_TRACE_TAG_PREFIX);
|
|
1154
|
+
}
|
|
1155
|
+
__name(isLifecycleTag, "isLifecycleTag");
|
|
1156
|
+
function lifecycleRunId(sourceRunId, kind, subscriber) {
|
|
1157
|
+
return `evt:${sourceRunId}:${kind}:${subscriber}`;
|
|
1158
|
+
}
|
|
1159
|
+
__name(lifecycleRunId, "lifecycleRunId");
|
|
1160
|
+
function lifecycleKindOf(status) {
|
|
1161
|
+
if (status === "completed") return "completed";
|
|
1162
|
+
if (status === "failed" || status === "cancelled" || status === "dead") return "failed";
|
|
1163
|
+
return void 0;
|
|
1164
|
+
}
|
|
1165
|
+
__name(lifecycleKindOf, "lifecycleKindOf");
|
|
1166
|
+
function lifecycleEventName(workflow, kind) {
|
|
1167
|
+
return kind === "completed" ? runCompletedEvent(workflow) : runFailedEvent(workflow);
|
|
1168
|
+
}
|
|
1169
|
+
__name(lifecycleEventName, "lifecycleEventName");
|
|
1170
|
+
function parseLifecycleEvent(event) {
|
|
1171
|
+
for (const kind of [
|
|
1172
|
+
"completed",
|
|
1173
|
+
"failed"
|
|
1174
|
+
]) {
|
|
1175
|
+
const prefix = `run.${kind}:`;
|
|
1176
|
+
if (event.startsWith(prefix) && event.length > prefix.length) {
|
|
1177
|
+
return {
|
|
1178
|
+
workflow: event.slice(prefix.length),
|
|
1179
|
+
kind
|
|
1180
|
+
};
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
return void 0;
|
|
1184
|
+
}
|
|
1185
|
+
__name(parseLifecycleEvent, "parseLifecycleEvent");
|
|
1186
|
+
function lifecycleSourceOf(run) {
|
|
1187
|
+
if (!run.id.startsWith("evt:")) return void 0;
|
|
1188
|
+
for (const kind of [
|
|
1189
|
+
"completed",
|
|
1190
|
+
"failed"
|
|
1191
|
+
]) {
|
|
1192
|
+
const suffix = `:${kind}:${run.workflow}`;
|
|
1193
|
+
if (run.id.endsWith(suffix) && run.id.length > "evt:".length + suffix.length) {
|
|
1194
|
+
return {
|
|
1195
|
+
runId: run.id.slice("evt:".length, -suffix.length),
|
|
1196
|
+
kind
|
|
1197
|
+
};
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
1200
|
+
return void 0;
|
|
1201
|
+
}
|
|
1202
|
+
__name(lifecycleSourceOf, "lifecycleSourceOf");
|
|
1203
|
+
function triggeredRunsOf(run) {
|
|
1204
|
+
const kind = lifecycleKindOf(run.status);
|
|
1205
|
+
if (!kind) return [];
|
|
1206
|
+
return (run.tags ?? []).filter((tag) => tag.startsWith(LIFECYCLE_TRIGGERED_TAG_PREFIX)).map((tag) => lifecycleRunId(run.id, kind, tag.slice(LIFECYCLE_TRIGGERED_TAG_PREFIX.length)));
|
|
1207
|
+
}
|
|
1208
|
+
__name(triggeredRunsOf, "triggeredRunsOf");
|
|
1209
|
+
function lifecycleEventOf(run) {
|
|
1210
|
+
const completed = run.status === "completed";
|
|
1211
|
+
return {
|
|
1212
|
+
runId: run.id,
|
|
1213
|
+
workflow: run.workflow,
|
|
1214
|
+
workflowVersion: run.workflowVersion,
|
|
1215
|
+
status: run.status,
|
|
1216
|
+
namespace: run.namespace,
|
|
1217
|
+
input: run.input,
|
|
1218
|
+
...completed ? {
|
|
1219
|
+
output: run.output
|
|
1220
|
+
} : {
|
|
1221
|
+
error: run.error
|
|
1222
|
+
},
|
|
1223
|
+
tags: run.tags?.filter((tag) => !isLifecycleTag(tag)),
|
|
1224
|
+
searchAttributes: run.searchAttributes
|
|
1225
|
+
};
|
|
1226
|
+
}
|
|
1227
|
+
__name(lifecycleEventOf, "lifecycleEventOf");
|
|
1228
|
+
var LIFECYCLE_REGISTRY_WORKFLOW = "__lifecycle_registry";
|
|
1229
|
+
var LIFECYCLE_REGISTRY_TAG = "durable:lifecycle:registry";
|
|
1230
|
+
var LIFECYCLE_REGISTRY_HEARTBEAT_MS = 1e4;
|
|
1231
|
+
var LIFECYCLE_REGISTRY_TTL_MS = 35e3;
|
|
1232
|
+
var LIFECYCLE_REGISTRY_STALE_MS = 10 * 60 * 1e3;
|
|
1233
|
+
function lifecycleRegistryRunId(instanceId) {
|
|
1234
|
+
return `${LIFECYCLE_REGISTRY_WORKFLOW}:${instanceId}`;
|
|
1235
|
+
}
|
|
1236
|
+
__name(lifecycleRegistryRunId, "lifecycleRegistryRunId");
|
|
1237
|
+
function lifecycleRegistryNamespace(input, rowNamespace) {
|
|
1238
|
+
const engineNamespace = input?.engineNamespace;
|
|
1239
|
+
if (engineNamespace === null) return null;
|
|
1240
|
+
if (typeof engineNamespace === "string") return engineNamespace || "default";
|
|
1241
|
+
const row = rowNamespace || "default";
|
|
1242
|
+
return row === "default" ? null : row;
|
|
1243
|
+
}
|
|
1244
|
+
__name(lifecycleRegistryNamespace, "lifecycleRegistryNamespace");
|
|
1245
|
+
function lifecycleRegistrySubscriptions(input) {
|
|
1246
|
+
const subscriptions = input?.subscriptions;
|
|
1247
|
+
if (!Array.isArray(subscriptions)) return [];
|
|
1248
|
+
return subscriptions.flatMap((entry) => {
|
|
1249
|
+
const { workflow, events } = entry ?? {};
|
|
1250
|
+
if (typeof workflow !== "string" || !Array.isArray(events)) return [];
|
|
1251
|
+
return [
|
|
1252
|
+
{
|
|
1253
|
+
workflow,
|
|
1254
|
+
events: events.filter((e) => typeof e === "string")
|
|
1255
|
+
}
|
|
1256
|
+
];
|
|
1257
|
+
});
|
|
1258
|
+
}
|
|
1259
|
+
__name(lifecycleRegistrySubscriptions, "lifecycleRegistrySubscriptions");
|
|
1260
|
+
function retentionKeeps(tags) {
|
|
1261
|
+
if (carriesLifecyclePending(tags)) return true;
|
|
1262
|
+
if (Array.isArray(tags)) return tags.includes(LIFECYCLE_REGISTRY_TAG);
|
|
1263
|
+
if (typeof tags === "string") return tags.includes(`"${LIFECYCLE_REGISTRY_TAG}"`);
|
|
1264
|
+
return false;
|
|
1265
|
+
}
|
|
1266
|
+
__name(retentionKeeps, "retentionKeeps");
|
|
1267
|
+
|
|
1268
|
+
// src/run-wait.ts
|
|
1269
|
+
var RUN_WAIT_TOKEN_PREFIX = "runwait:";
|
|
1270
|
+
function runWaitPrefix(targetRunId) {
|
|
1271
|
+
return `${RUN_WAIT_TOKEN_PREFIX}${targetRunId}#`;
|
|
1272
|
+
}
|
|
1273
|
+
__name(runWaitPrefix, "runWaitPrefix");
|
|
1274
|
+
function runWaitToken(targetRunId, runId, seq) {
|
|
1275
|
+
return `${runWaitPrefix(targetRunId)}${runId}#${seq}`;
|
|
1276
|
+
}
|
|
1277
|
+
__name(runWaitToken, "runWaitToken");
|
|
1278
|
+
function runWaitTargetOf(waiter) {
|
|
1279
|
+
const suffix = `#${waiter.runId}#${waiter.seq}`;
|
|
1280
|
+
const { token } = waiter;
|
|
1281
|
+
if (!token.startsWith(RUN_WAIT_TOKEN_PREFIX) || !token.endsWith(suffix)) return void 0;
|
|
1282
|
+
const target = token.slice(RUN_WAIT_TOKEN_PREFIX.length, token.length - suffix.length);
|
|
1283
|
+
return target.length > 0 ? target : void 0;
|
|
1284
|
+
}
|
|
1285
|
+
__name(runWaitTargetOf, "runWaitTargetOf");
|
|
1286
|
+
function nextContinuationId(runId) {
|
|
1287
|
+
const m = runId.match(/^(.*)~(\d+)$/);
|
|
1288
|
+
return m ? `${m[1]}~${Number(m[2]) + 1}` : `${runId}~1`;
|
|
1289
|
+
}
|
|
1290
|
+
__name(nextContinuationId, "nextContinuationId");
|
|
1291
|
+
var MAX_CONTINUATION_HOPS = 100;
|
|
1292
|
+
async function settledRunOf(store, runId) {
|
|
1293
|
+
let run = await store.getRun(runId);
|
|
1294
|
+
if (!run) return {
|
|
1295
|
+
state: "missing"
|
|
1296
|
+
};
|
|
1297
|
+
for (let hop = 0; hop <= MAX_CONTINUATION_HOPS; hop += 1) {
|
|
1298
|
+
if (!lifecycleKindOf(run.status)) return {
|
|
1299
|
+
state: "running"
|
|
1300
|
+
};
|
|
1301
|
+
if (run.status !== "completed") return {
|
|
1302
|
+
state: "settled",
|
|
1303
|
+
run
|
|
1304
|
+
};
|
|
1305
|
+
const next = await store.getRun(nextContinuationId(run.id));
|
|
1306
|
+
if (!next) return {
|
|
1307
|
+
state: "settled",
|
|
1308
|
+
run
|
|
1309
|
+
};
|
|
1310
|
+
run = next;
|
|
1311
|
+
}
|
|
1312
|
+
return {
|
|
1313
|
+
state: "running"
|
|
1314
|
+
};
|
|
1315
|
+
}
|
|
1316
|
+
__name(settledRunOf, "settledRunOf");
|
|
1036
1317
|
|
|
1037
1318
|
// src/run-waiting.ts
|
|
1038
1319
|
var WEBHOOK_PREFIX = "wh:";
|
|
@@ -1052,7 +1333,17 @@ function parseWebhookToken(token) {
|
|
|
1052
1333
|
};
|
|
1053
1334
|
}
|
|
1054
1335
|
__name(parseWebhookToken, "parseWebhookToken");
|
|
1055
|
-
function classifyWaiterToken(token) {
|
|
1336
|
+
function classifyWaiterToken(token, waiter) {
|
|
1337
|
+
if (token.startsWith(RUN_WAIT_TOKEN_PREFIX)) {
|
|
1338
|
+
const target = waiter ? runWaitTargetOf({
|
|
1339
|
+
token,
|
|
1340
|
+
...waiter
|
|
1341
|
+
}) : token.slice(RUN_WAIT_TOKEN_PREFIX.length).match(/^(.*)#[^#]*#\d+$/)?.[1];
|
|
1342
|
+
if (target) return {
|
|
1343
|
+
on: "run",
|
|
1344
|
+
name: target
|
|
1345
|
+
};
|
|
1346
|
+
}
|
|
1056
1347
|
if (token.startsWith(WEBHOOK_PREFIX)) return {
|
|
1057
1348
|
on: "webhook",
|
|
1058
1349
|
name: token
|
|
@@ -1078,7 +1369,7 @@ __name(classifyWaiterToken, "classifyWaiterToken");
|
|
|
1078
1369
|
function resolveRunWaiting(run, waiterByRun) {
|
|
1079
1370
|
if (run.status !== "suspended") return void 0;
|
|
1080
1371
|
const waiter = waiterByRun.get(run.id);
|
|
1081
|
-
return waiter ? classifyWaiterToken(waiter.token) : void 0;
|
|
1372
|
+
return waiter ? classifyWaiterToken(waiter.token, waiter) : void 0;
|
|
1082
1373
|
}
|
|
1083
1374
|
__name(resolveRunWaiting, "resolveRunWaiting");
|
|
1084
1375
|
var RUN_ID_BATCH = 500;
|
|
@@ -1943,7 +2234,55 @@ var Entities = class {
|
|
|
1943
2234
|
};
|
|
1944
2235
|
|
|
1945
2236
|
// src/event-accumulators.ts
|
|
1946
|
-
|
|
2237
|
+
function accumulatorRunId(target, key) {
|
|
2238
|
+
if (key === void 0) return `__evtacc__:${target}`;
|
|
2239
|
+
const escaped = encodeURIComponent(key).replace(/[~!'()*.]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
|
|
2240
|
+
return `__evtacc__:${target}:${escaped}`;
|
|
2241
|
+
}
|
|
2242
|
+
__name(accumulatorRunId, "accumulatorRunId");
|
|
2243
|
+
var ACCUMULATOR_SEEN_MAX = 1e3;
|
|
2244
|
+
function isRoutedEvent(signal) {
|
|
2245
|
+
return typeof signal === "object" && signal !== null && signal.__evtacc === 1 && typeof signal.id === "string";
|
|
2246
|
+
}
|
|
2247
|
+
__name(isRoutedEvent, "isRoutedEvent");
|
|
2248
|
+
var SeenEvents = class SeenEvents2 {
|
|
2249
|
+
static {
|
|
2250
|
+
__name(this, "SeenEvents");
|
|
2251
|
+
}
|
|
2252
|
+
ids;
|
|
2253
|
+
order;
|
|
2254
|
+
constructor(seen) {
|
|
2255
|
+
this.order = [
|
|
2256
|
+
...seen ?? []
|
|
2257
|
+
];
|
|
2258
|
+
this.ids = new Set(this.order);
|
|
2259
|
+
}
|
|
2260
|
+
fresh(signal) {
|
|
2261
|
+
if (!isRoutedEvent(signal)) return {
|
|
2262
|
+
fresh: true,
|
|
2263
|
+
payload: signal
|
|
2264
|
+
};
|
|
2265
|
+
if (this.ids.has(signal.id)) return {
|
|
2266
|
+
fresh: false
|
|
2267
|
+
};
|
|
2268
|
+
this.ids.add(signal.id);
|
|
2269
|
+
this.order.push(signal.id);
|
|
2270
|
+
return {
|
|
2271
|
+
fresh: true,
|
|
2272
|
+
payload: signal.payload
|
|
2273
|
+
};
|
|
2274
|
+
}
|
|
2275
|
+
/** The input to re-arm with: `input` plus the newest ids, when there are any. */
|
|
2276
|
+
rearm(input) {
|
|
2277
|
+
const { seen: _seen, ...rest } = input;
|
|
2278
|
+
const seen = this.order.slice(-ACCUMULATOR_SEEN_MAX);
|
|
2279
|
+
return seen.length > 0 ? {
|
|
2280
|
+
...rest,
|
|
2281
|
+
seen
|
|
2282
|
+
} : rest;
|
|
2283
|
+
}
|
|
2284
|
+
};
|
|
2285
|
+
var accToken = /* @__PURE__ */ __name((input) => accumulatorRunId(input.target, input.key), "accToken");
|
|
1947
2286
|
var EventAccumulators = class {
|
|
1948
2287
|
static {
|
|
1949
2288
|
__name(this, "EventAccumulators");
|
|
@@ -1958,60 +2297,75 @@ var EventAccumulators = class {
|
|
|
1958
2297
|
origin: CORE_ORIGIN
|
|
1959
2298
|
});
|
|
1960
2299
|
}
|
|
1961
|
-
/**
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
"
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
}
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
2300
|
+
/**
|
|
2301
|
+
* Route an `onEvent` payload into its target's debounce/batch accumulator: the target's one, or
|
|
2302
|
+
* with a `key`, the one for `String(config.key(payload))`. A `key` that throws rejects the route.
|
|
2303
|
+
* With `id`, the accumulator counts the event once however many times it is routed.
|
|
2304
|
+
*/
|
|
2305
|
+
route(target, config, payload, id) {
|
|
2306
|
+
const input = {
|
|
2307
|
+
target,
|
|
2308
|
+
windowMs: config.windowMs,
|
|
2309
|
+
...config.mode === "batch" ? {
|
|
2310
|
+
maxSize: config.maxSize
|
|
2311
|
+
} : {},
|
|
2312
|
+
...config.key ? {
|
|
2313
|
+
key: String(config.key(payload))
|
|
2314
|
+
} : {}
|
|
2315
|
+
};
|
|
2316
|
+
const workflow = config.mode === "debounce" ? "__evt_debounce" : "__evt_batch";
|
|
2317
|
+
const token = accToken(input);
|
|
2318
|
+
const signal = id === void 0 ? payload : {
|
|
2319
|
+
__evtacc: 1,
|
|
2320
|
+
id,
|
|
1979
2321
|
payload
|
|
2322
|
+
};
|
|
2323
|
+
return this.host.signalWithStart(workflow, input, token, {
|
|
2324
|
+
token,
|
|
2325
|
+
payload: signal
|
|
1980
2326
|
});
|
|
1981
2327
|
}
|
|
1982
2328
|
/** Fire the target with the LAST payload once events have been quiet for `windowMs`. */
|
|
1983
2329
|
async debounce(ctx, input) {
|
|
1984
2330
|
const { target, windowMs } = input;
|
|
1985
|
-
const token = accToken(
|
|
1986
|
-
|
|
2331
|
+
const token = accToken(input);
|
|
2332
|
+
const seen = new SeenEvents(input.seen);
|
|
2333
|
+
let first = seen.fresh(await ctx.waitForSignal(token));
|
|
2334
|
+
while (!first.fresh) first = seen.fresh(await ctx.waitForSignal(token));
|
|
2335
|
+
let last = first.payload;
|
|
1987
2336
|
for (; ; ) {
|
|
1988
2337
|
try {
|
|
1989
|
-
|
|
2338
|
+
const next = seen.fresh(await ctx.waitForSignal(token, {
|
|
1990
2339
|
timeoutMs: windowMs
|
|
1991
|
-
});
|
|
2340
|
+
}));
|
|
2341
|
+
if (next.fresh) last = next.payload;
|
|
1992
2342
|
} catch (e) {
|
|
1993
2343
|
if (e instanceof SignalTimeoutError) break;
|
|
1994
2344
|
throw e;
|
|
1995
2345
|
}
|
|
1996
2346
|
}
|
|
1997
2347
|
await ctx.startChild(target, last);
|
|
1998
|
-
return ctx.continueAsNew(input);
|
|
2348
|
+
return ctx.continueAsNew(seen.rearm(input));
|
|
1999
2349
|
}
|
|
2000
2350
|
/** Fire the target with all payloads (`{ events }`) once `maxSize` or `windowMs` from the first. */
|
|
2001
2351
|
async batch(ctx, input) {
|
|
2002
2352
|
const { target, maxSize, windowMs } = input;
|
|
2003
|
-
const token = accToken(
|
|
2353
|
+
const token = accToken(input);
|
|
2354
|
+
const seen = new SeenEvents(input.seen);
|
|
2355
|
+
let first = seen.fresh(await ctx.waitForSignal(token));
|
|
2356
|
+
while (!first.fresh) first = seen.fresh(await ctx.waitForSignal(token));
|
|
2004
2357
|
const events = [
|
|
2005
|
-
|
|
2358
|
+
first.payload
|
|
2006
2359
|
];
|
|
2007
2360
|
const deadline = await ctx.now() + windowMs;
|
|
2008
2361
|
while (events.length < maxSize) {
|
|
2009
2362
|
const remaining = deadline - await ctx.now();
|
|
2010
2363
|
if (remaining <= 0) break;
|
|
2011
2364
|
try {
|
|
2012
|
-
|
|
2365
|
+
const next = seen.fresh(await ctx.waitForSignal(token, {
|
|
2013
2366
|
timeoutMs: remaining
|
|
2014
2367
|
}));
|
|
2368
|
+
if (next.fresh) events.push(next.payload);
|
|
2015
2369
|
} catch (e) {
|
|
2016
2370
|
if (e instanceof SignalTimeoutError) break;
|
|
2017
2371
|
throw e;
|
|
@@ -2020,7 +2374,7 @@ var EventAccumulators = class {
|
|
|
2020
2374
|
await ctx.startChild(target, {
|
|
2021
2375
|
events
|
|
2022
2376
|
});
|
|
2023
|
-
return ctx.continueAsNew(input);
|
|
2377
|
+
return ctx.continueAsNew(seen.rearm(input));
|
|
2024
2378
|
}
|
|
2025
2379
|
};
|
|
2026
2380
|
|
|
@@ -2091,6 +2445,11 @@ function canonicalizeRegistrations(regs) {
|
|
|
2091
2445
|
// Conditional, so a registration that states no routes keeps the hash it had before routes existed.
|
|
2092
2446
|
...r.onError !== void 0 ? {
|
|
2093
2447
|
onError: canonicalRoutes(r.onError)
|
|
2448
|
+
} : {},
|
|
2449
|
+
...r.subscribes !== void 0 ? {
|
|
2450
|
+
subscribes: [
|
|
2451
|
+
...new Set(r.subscribes)
|
|
2452
|
+
].sort()
|
|
2094
2453
|
} : {}
|
|
2095
2454
|
}));
|
|
2096
2455
|
const byJson = /* @__PURE__ */ new Map();
|
|
@@ -2320,6 +2679,9 @@ function aggregateAnnouncements(descriptors) {
|
|
|
2320
2679
|
origins,
|
|
2321
2680
|
requires,
|
|
2322
2681
|
onError,
|
|
2682
|
+
subscribes: [
|
|
2683
|
+
...new Set(announcements.flatMap((a) => a.subscribes ?? []))
|
|
2684
|
+
].sort(),
|
|
2323
2685
|
runtimes: [
|
|
2324
2686
|
...new Set(announcements.map((a) => a.runtime))
|
|
2325
2687
|
].sort(),
|
|
@@ -2354,6 +2716,7 @@ function observedAnnouncements(heartbeats, covered) {
|
|
|
2354
2716
|
origins: [],
|
|
2355
2717
|
requires: [],
|
|
2356
2718
|
onError: [],
|
|
2719
|
+
subscribes: [],
|
|
2357
2720
|
runtimes: [],
|
|
2358
2721
|
instances: [
|
|
2359
2722
|
...instances
|
|
@@ -4396,6 +4759,71 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
4396
4759
|
if (bufferedHit) return bufferedHit.value;
|
|
4397
4760
|
throw new WorkflowSuspended(deadline);
|
|
4398
4761
|
}, "waitForEvent");
|
|
4762
|
+
const settleRunWaitFromTarget = /* @__PURE__ */ __name(async (target, token, seq) => {
|
|
4763
|
+
const state = await settledRunOf(store, target);
|
|
4764
|
+
if (state.state === "missing") {
|
|
4765
|
+
await store.removeSignalWaiter({
|
|
4766
|
+
token,
|
|
4767
|
+
runId,
|
|
4768
|
+
seq
|
|
4769
|
+
});
|
|
4770
|
+
throw new FatalError(`run ${target} not found`, "run_not_found");
|
|
4771
|
+
}
|
|
4772
|
+
if (state.state === "running") return null;
|
|
4773
|
+
if (!await store.takeSignalWaiter(token)) return null;
|
|
4774
|
+
const event = lifecycleEventOf(state.run);
|
|
4775
|
+
await writeCheckpoint(instantCheckpoint({
|
|
4776
|
+
runId,
|
|
4777
|
+
seq,
|
|
4778
|
+
name: `signal:${token}`,
|
|
4779
|
+
kind: "signal",
|
|
4780
|
+
output: event
|
|
4781
|
+
}));
|
|
4782
|
+
return {
|
|
4783
|
+
value: event
|
|
4784
|
+
};
|
|
4785
|
+
}, "settleRunWaitFromTarget");
|
|
4786
|
+
const waitForRun = /* @__PURE__ */ __name(async (first, second, third) => {
|
|
4787
|
+
const target = typeof first === "string" ? first : second;
|
|
4788
|
+
const opts = typeof first === "string" ? second : third;
|
|
4789
|
+
if (opts?.timeoutMs == null) {
|
|
4790
|
+
const current = pos.next();
|
|
4791
|
+
const token2 = runWaitToken(target, runId, current);
|
|
4792
|
+
const existing = await readCheckpoint(current);
|
|
4793
|
+
if (existing && existing.status === "completed") return existing.output;
|
|
4794
|
+
await store.putSignalWaiter({
|
|
4795
|
+
token: token2,
|
|
4796
|
+
runId,
|
|
4797
|
+
seq: current
|
|
4798
|
+
});
|
|
4799
|
+
const settled2 = await settleRunWaitFromTarget(target, token2, current);
|
|
4800
|
+
if (settled2) return settled2.value;
|
|
4801
|
+
throw new WorkflowSuspended();
|
|
4802
|
+
}
|
|
4803
|
+
const timeoutMs = parseDuration(opts.timeoutMs);
|
|
4804
|
+
const deadlineSeq = pos.next();
|
|
4805
|
+
const waitSeq = pos.next();
|
|
4806
|
+
const token = runWaitToken(target, runId, waitSeq);
|
|
4807
|
+
const deadline = await stampDeadline(deadlineSeq, `run:${target}`, timeoutMs);
|
|
4808
|
+
const waited = await readCheckpoint(waitSeq);
|
|
4809
|
+
if (waited && waited.status === "completed") return waited.output;
|
|
4810
|
+
if (host.clock() >= deadline) {
|
|
4811
|
+
await store.removeSignalWaiter({
|
|
4812
|
+
token,
|
|
4813
|
+
runId,
|
|
4814
|
+
seq: waitSeq
|
|
4815
|
+
}).catch(() => void 0);
|
|
4816
|
+
throw new SignalTimeoutError(`run:${target}`, timeoutMs);
|
|
4817
|
+
}
|
|
4818
|
+
await store.putSignalWaiter({
|
|
4819
|
+
token,
|
|
4820
|
+
runId,
|
|
4821
|
+
seq: waitSeq
|
|
4822
|
+
});
|
|
4823
|
+
const settled = await settleRunWaitFromTarget(target, token, waitSeq);
|
|
4824
|
+
if (settled) return settled.value;
|
|
4825
|
+
throw new WorkflowSuspended(deadline);
|
|
4826
|
+
}, "waitForRun");
|
|
4399
4827
|
const task = /* @__PURE__ */ __name(async (name, dispatch, options) => {
|
|
4400
4828
|
await localStep(`task:dispatch:${name}`, dispatch, options);
|
|
4401
4829
|
return unwrapCompletion(await waitForSignal(`task:${runId}:${name}`), `task "${name}"`);
|
|
@@ -5061,6 +5489,7 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
5061
5489
|
continueAsNew,
|
|
5062
5490
|
waitForSignal: guard("waitForSignal", waitForSignal),
|
|
5063
5491
|
waitForEvent: guard("waitForEvent", waitForEvent),
|
|
5492
|
+
waitForRun: guard("waitForRun", waitForRun),
|
|
5064
5493
|
task: guard("task", task),
|
|
5065
5494
|
child: guard("child", child),
|
|
5066
5495
|
all: guard("all", all),
|
|
@@ -5091,6 +5520,28 @@ __name(mergeTags, "mergeTags");
|
|
|
5091
5520
|
var BREAKPOINT = "breakpoint";
|
|
5092
5521
|
var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
|
|
5093
5522
|
var ANNOUNCED_ROUTES_TTL_MS = 5e3;
|
|
5523
|
+
function servedNamespace(namespace, group) {
|
|
5524
|
+
if (namespace) return namespace;
|
|
5525
|
+
const at = group?.lastIndexOf("@") ?? -1;
|
|
5526
|
+
return group && at >= 0 && at < group.length - 1 ? group.slice(at + 1) : "default";
|
|
5527
|
+
}
|
|
5528
|
+
__name(servedNamespace, "servedNamespace");
|
|
5529
|
+
function announcedIn(snapshot, event, namespace, nodeOnly = false) {
|
|
5530
|
+
const scope = namespace === null ? null : runNamespace(namespace);
|
|
5531
|
+
const subscribers = /* @__PURE__ */ new Set();
|
|
5532
|
+
for (const entry of snapshot.entries) {
|
|
5533
|
+
if (entry.event !== event || nodeOnly && !entry.node) continue;
|
|
5534
|
+
if (scope === null || entry.namespace === null || entry.namespace === scope) {
|
|
5535
|
+
subscribers.add(entry.workflow);
|
|
5536
|
+
}
|
|
5537
|
+
}
|
|
5538
|
+
return subscribers;
|
|
5539
|
+
}
|
|
5540
|
+
__name(announcedIn, "announcedIn");
|
|
5541
|
+
function runNamespace(namespace) {
|
|
5542
|
+
return namespace || "default";
|
|
5543
|
+
}
|
|
5544
|
+
__name(runNamespace, "runNamespace");
|
|
5094
5545
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
5095
5546
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
5096
5547
|
var STEP_FAN_CHUNK = 200;
|
|
@@ -5187,15 +5638,10 @@ function shuffle(items) {
|
|
|
5187
5638
|
return items;
|
|
5188
5639
|
}
|
|
5189
5640
|
__name(shuffle, "shuffle");
|
|
5190
|
-
function stepGroup(step,
|
|
5191
|
-
return tenantGroup(sanitizeQueueToken(step.name), step.partition ??
|
|
5641
|
+
function stepGroup(step, runNamespace2) {
|
|
5642
|
+
return tenantGroup(sanitizeQueueToken(step.name), step.partition ?? runNamespace2);
|
|
5192
5643
|
}
|
|
5193
5644
|
__name(stepGroup, "stepGroup");
|
|
5194
|
-
function nextContinuationId(runId) {
|
|
5195
|
-
const m = runId.match(/^(.*)~(\d+)$/);
|
|
5196
|
-
return m ? `${m[1]}~${Number(m[2]) + 1}` : `${runId}~1`;
|
|
5197
|
-
}
|
|
5198
|
-
__name(nextContinuationId, "nextContinuationId");
|
|
5199
5645
|
function isNewerVersion(a, b) {
|
|
5200
5646
|
const na = Number(a);
|
|
5201
5647
|
const nb = Number(b);
|
|
@@ -5245,7 +5691,7 @@ var UNREGISTERED_DEAD_LETTER_FROM = [
|
|
|
5245
5691
|
"cancelling",
|
|
5246
5692
|
"blocked"
|
|
5247
5693
|
];
|
|
5248
|
-
var WorkflowEngine = class {
|
|
5694
|
+
var WorkflowEngine = class _WorkflowEngine {
|
|
5249
5695
|
static {
|
|
5250
5696
|
__name(this, "WorkflowEngine");
|
|
5251
5697
|
}
|
|
@@ -5297,7 +5743,7 @@ var WorkflowEngine = class {
|
|
|
5297
5743
|
workflows = /* @__PURE__ */ new Map();
|
|
5298
5744
|
/** The newest registered version per workflow name — used to `start` new runs. */
|
|
5299
5745
|
latest = /* @__PURE__ */ new Map();
|
|
5300
|
-
/** Event name → workflow
|
|
5746
|
+
/** Event name → subscribing workflow name → that workflow's subscriptions to it. See `onEvent`. */
|
|
5301
5747
|
eventTriggers = /* @__PURE__ */ new Map();
|
|
5302
5748
|
/** Durable-entity subsystem (registers the `__entity` runner; see `registerEntity`). */
|
|
5303
5749
|
entities;
|
|
@@ -5419,6 +5865,12 @@ var WorkflowEngine = class {
|
|
|
5419
5865
|
* releases when its budget runs out.
|
|
5420
5866
|
*/
|
|
5421
5867
|
executionLeases = /* @__PURE__ */ new Map();
|
|
5868
|
+
/** Runs a caller just locked to execute now, so their turn has nothing to prove and no await to add. */
|
|
5869
|
+
lockedForTurn = /* @__PURE__ */ new Set();
|
|
5870
|
+
/** Runs this instance holds for a moment and will release, not to execute (see {@link handOffDue}). */
|
|
5871
|
+
claimsInFlight = /* @__PURE__ */ new Map();
|
|
5872
|
+
/** How many such claims each run has had, so a proof can tell one came and went (see {@link proveOwnLease}). */
|
|
5873
|
+
claimsMade = /* @__PURE__ */ new Map();
|
|
5422
5874
|
/** Pager hooks for stranded runs (see {@link onStalled}). */
|
|
5423
5875
|
stalledListeners = /* @__PURE__ */ new Set();
|
|
5424
5876
|
/** Runs already paged about, by id → the `updatedAt` of the paged episode (see {@link sweepStalled}). */
|
|
@@ -5762,6 +6214,27 @@ var WorkflowEngine = class {
|
|
|
5762
6214
|
console.error(`[nestjs-durable] could not run queued run ${runId}:`, error2);
|
|
5763
6215
|
}
|
|
5764
6216
|
}
|
|
6217
|
+
markClaim(runId) {
|
|
6218
|
+
this.claimsInFlight.set(runId, (this.claimsInFlight.get(runId) ?? 0) + 1);
|
|
6219
|
+
if (this.claimsMade.size >= 4096) this.claimsMade.clear();
|
|
6220
|
+
this.claimsMade.set(runId, (this.claimsMade.get(runId) ?? 0) + 1);
|
|
6221
|
+
}
|
|
6222
|
+
/**
|
|
6223
|
+
* Whether this instance holds the run's lease for a turn, from the store and by owner. A renewal can land
|
|
6224
|
+
* on a claim marked after it was sent, and that claim is about to be released, so any claim begun while
|
|
6225
|
+
* the renewal was out, even one already over, leaves it unproved.
|
|
6226
|
+
*/
|
|
6227
|
+
async proveOwnLease(runId, nowMs) {
|
|
6228
|
+
if (this.claimsInFlight.has(runId)) return false;
|
|
6229
|
+
const before = this.claimsMade.get(runId);
|
|
6230
|
+
const renewed = await this.store.renewRunLock(runId, this.instanceId, nowMs + this.leaseMs);
|
|
6231
|
+
return renewed && !this.claimsInFlight.has(runId) && this.claimsMade.get(runId) === before;
|
|
6232
|
+
}
|
|
6233
|
+
unmarkClaim(runId) {
|
|
6234
|
+
const left = (this.claimsInFlight.get(runId) ?? 1) - 1;
|
|
6235
|
+
if (left > 0) this.claimsInFlight.set(runId, left);
|
|
6236
|
+
else this.claimsInFlight.delete(runId);
|
|
6237
|
+
}
|
|
5765
6238
|
/**
|
|
5766
6239
|
* Hand due runs (timers, blocked runs due for a fleet re-check) to whichever instance takes them
|
|
5767
6240
|
* off the run queue. Each is claimed first, under its lease, by pushing its `wakeAt` one lease
|
|
@@ -5773,29 +6246,34 @@ var WorkflowEngine = class {
|
|
|
5773
6246
|
const claimed = [];
|
|
5774
6247
|
for (const run of runs) {
|
|
5775
6248
|
if (this.draining) break;
|
|
5776
|
-
|
|
5777
|
-
continue;
|
|
5778
|
-
}
|
|
5779
|
-
if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) {
|
|
5780
|
-
const result = await this.settleUnregistered(run, nowMs);
|
|
5781
|
-
if (result) settled?.set(run.id, result);
|
|
5782
|
-
else if (result === void 0) this.logUnregisteredSkip(run);
|
|
5783
|
-
continue;
|
|
5784
|
-
}
|
|
6249
|
+
this.markClaim(run.id);
|
|
5785
6250
|
try {
|
|
5786
|
-
|
|
5787
|
-
|
|
5788
|
-
}
|
|
5789
|
-
if (this.
|
|
5790
|
-
|
|
5791
|
-
|
|
5792
|
-
|
|
5793
|
-
|
|
5794
|
-
|
|
5795
|
-
|
|
6251
|
+
if (!await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
6252
|
+
continue;
|
|
6253
|
+
}
|
|
6254
|
+
if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) {
|
|
6255
|
+
const result = await this.settleUnregistered(run, nowMs);
|
|
6256
|
+
if (result) settled?.set(run.id, result);
|
|
6257
|
+
else if (result === void 0) this.logUnregisteredSkip(run);
|
|
6258
|
+
continue;
|
|
6259
|
+
}
|
|
6260
|
+
try {
|
|
6261
|
+
const patch = {
|
|
6262
|
+
wakeAt: nowMs + this.leaseMs
|
|
6263
|
+
};
|
|
6264
|
+
if (this.store.updateRunIf) {
|
|
6265
|
+
if (await this.store.updateRunIf(run.id, [
|
|
6266
|
+
run.status
|
|
6267
|
+
], patch)) claimed.push(run);
|
|
6268
|
+
} else {
|
|
6269
|
+
await this.store.updateRun(run.id, patch);
|
|
6270
|
+
claimed.push(run);
|
|
6271
|
+
}
|
|
6272
|
+
} finally {
|
|
6273
|
+
await this.store.releaseRunLock(run.id, this.instanceId);
|
|
5796
6274
|
}
|
|
5797
6275
|
} finally {
|
|
5798
|
-
|
|
6276
|
+
this.unmarkClaim(run.id);
|
|
5799
6277
|
}
|
|
5800
6278
|
}
|
|
5801
6279
|
try {
|
|
@@ -5911,10 +6389,14 @@ var WorkflowEngine = class {
|
|
|
5911
6389
|
this.workflows.set(versionKey(name, version), registered);
|
|
5912
6390
|
const current = this.latest.get(name);
|
|
5913
6391
|
if (!current || isNewerVersion(version, current.version)) this.latest.set(name, registered);
|
|
5914
|
-
for (const
|
|
5915
|
-
const
|
|
5916
|
-
subscribers.
|
|
5917
|
-
|
|
6392
|
+
for (const entry of opts?.onEvent ?? []) {
|
|
6393
|
+
const subscription = toEventSubscription(entry);
|
|
6394
|
+
const subscribers = this.eventTriggers.get(subscription.event) ?? /* @__PURE__ */ new Map();
|
|
6395
|
+
subscribers.set(name, [
|
|
6396
|
+
...subscribers.get(name) ?? [],
|
|
6397
|
+
subscription
|
|
6398
|
+
]);
|
|
6399
|
+
this.eventTriggers.set(subscription.event, subscribers);
|
|
5918
6400
|
}
|
|
5919
6401
|
}
|
|
5920
6402
|
/**
|
|
@@ -6052,136 +6534,1334 @@ var WorkflowEngine = class {
|
|
|
6052
6534
|
return () => this.deadListeners.delete(listener);
|
|
6053
6535
|
}
|
|
6054
6536
|
/**
|
|
6055
|
-
* Delete buffered child completions no parent will ever consume (see
|
|
6056
|
-
* `childCompletionNeedsBuffer`): those of root runs, of children whose parent already settled them
|
|
6057
|
-
* or is gone, or terminal for good. Such rows were written by earlier versions, which buffered every
|
|
6058
|
-
* completion that found no waiter, and nothing else ever removes them.
|
|
6537
|
+
* Delete buffered child completions no parent will ever consume (see
|
|
6538
|
+
* `childCompletionNeedsBuffer`): those of root runs, of children whose parent already settled them
|
|
6539
|
+
* or is gone, or terminal for good. Such rows were written by earlier versions, which buffered every
|
|
6540
|
+
* completion that found no waiter, and nothing else ever removes them.
|
|
6541
|
+
*
|
|
6542
|
+
* Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
|
|
6543
|
+
* at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
|
|
6544
|
+
* few timer ticks without a long query or a big delete. Each page is decided set-based (see
|
|
6545
|
+
* `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
|
|
6546
|
+
* read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
|
|
6547
|
+
* {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
|
|
6548
|
+
* call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
|
|
6549
|
+
* before the next one. Call it every tick (the NestJS timer poller does); a call made while
|
|
6550
|
+
* another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
|
|
6551
|
+
* carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
|
|
6552
|
+
* store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
|
|
6553
|
+
*/
|
|
6554
|
+
async sweepBufferedSignals(now = this.clock(), budgetMs = BUFFERED_SWEEP_BUDGET_MS) {
|
|
6555
|
+
const { listBufferedSignals, deleteBufferedSignals } = this.store;
|
|
6556
|
+
if (!listBufferedSignals || !deleteBufferedSignals) return 0;
|
|
6557
|
+
if (this.draining || this.namespace !== void 0 || this.bufferedSweepActive) return 0;
|
|
6558
|
+
if (this.bufferedSweepCursor === void 0 && now - this.lastBufferedSweepEndedAt < BUFFERED_SWEEP_REST_MS) {
|
|
6559
|
+
return 0;
|
|
6560
|
+
}
|
|
6561
|
+
this.bufferedSweepActive = true;
|
|
6562
|
+
const deadline = Date.now() + budgetMs;
|
|
6563
|
+
let deleted = 0;
|
|
6564
|
+
try {
|
|
6565
|
+
for (let page = 0; page < BUFFERED_SWEEP_PAGES_PER_CALL; page++) {
|
|
6566
|
+
if (page > 0 && (Date.now() >= deadline || this.draining)) break;
|
|
6567
|
+
const rows = await listBufferedSignals.call(this.store, CHILD_TOKEN_PREFIX, this.bufferedSweepCursor, BUFFERED_SWEEP_PAGE);
|
|
6568
|
+
const { doomed, decided } = await unneededChildSignals(this.store, rows, deadline);
|
|
6569
|
+
if (doomed.length > 0) {
|
|
6570
|
+
await deleteBufferedSignals.call(this.store, doomed.map((row) => row.id));
|
|
6571
|
+
deleted += doomed.length;
|
|
6572
|
+
}
|
|
6573
|
+
if (decided < rows.length) {
|
|
6574
|
+
if (decided > 0) this.bufferedSweepCursor = rows[decided - 1].id;
|
|
6575
|
+
break;
|
|
6576
|
+
}
|
|
6577
|
+
const last = rows.at(-1);
|
|
6578
|
+
if (rows.length < BUFFERED_SWEEP_PAGE || !last) {
|
|
6579
|
+
this.bufferedSweepCursor = void 0;
|
|
6580
|
+
this.lastBufferedSweepEndedAt = now;
|
|
6581
|
+
break;
|
|
6582
|
+
}
|
|
6583
|
+
this.bufferedSweepCursor = last.id;
|
|
6584
|
+
}
|
|
6585
|
+
} finally {
|
|
6586
|
+
this.bufferedSweepActive = false;
|
|
6587
|
+
}
|
|
6588
|
+
return deleted;
|
|
6589
|
+
}
|
|
6590
|
+
/**
|
|
6591
|
+
* Register a pager hook for STRANDED runs: an in-flight run untouched past `stalledAfterMs` that is
|
|
6592
|
+
* either suspended with NO wake timer (wake-forever: nothing will ever re-drive it) or parked on an
|
|
6593
|
+
* old pending remote step with no live lease (dispatched and, most likely, lost). The timer poller
|
|
6594
|
+
* sweeps for these ({@link sweepStalled}, throttled to one pass a minute) and notifies each episode
|
|
6595
|
+
* ONCE per instance: a run is paged again only after it makes progress and strands again. Wire your
|
|
6596
|
+
* alerting here like {@link onDead}. A listener that throws never breaks the sweep. Returns an
|
|
6597
|
+
* unsubscribe function.
|
|
6598
|
+
*/
|
|
6599
|
+
onStalled(listener) {
|
|
6600
|
+
this.stalledListeners.add(listener);
|
|
6601
|
+
return () => this.stalledListeners.delete(listener);
|
|
6602
|
+
}
|
|
6603
|
+
/**
|
|
6604
|
+
* Detect stranded in-flight runs and notify the {@link onStalled} listeners. Self-throttled to one
|
|
6605
|
+
* pass a minute, skipped while draining, and a no-op without listeners — call it every tick (the
|
|
6606
|
+
* NestJS timer poller does). Scoped to this engine's `namespace` (an operator engine with no
|
|
6607
|
+
* namespace sweeps every namespace). Returns the runs notified THIS pass.
|
|
6608
|
+
*/
|
|
6609
|
+
async sweepStalled(now = this.clock()) {
|
|
6610
|
+
if (this.stalledListeners.size === 0 || this.draining) return [];
|
|
6611
|
+
if (now - this.lastStalledSweepAt < 6e4) return [];
|
|
6612
|
+
this.lastStalledSweepAt = now;
|
|
6613
|
+
const notified = [];
|
|
6614
|
+
for (const info2 of await this.listStalledRuns({
|
|
6615
|
+
now
|
|
6616
|
+
})) {
|
|
6617
|
+
const episode = info2.run.updatedAt.getTime();
|
|
6618
|
+
if (this.notifiedStalled.get(info2.run.id) === episode) continue;
|
|
6619
|
+
this.notifiedStalled.set(info2.run.id, episode);
|
|
6620
|
+
if (this.notifiedStalled.size > 1e3) this.notifiedStalled.clear();
|
|
6621
|
+
for (const listener of this.stalledListeners) {
|
|
6622
|
+
try {
|
|
6623
|
+
listener(info2);
|
|
6624
|
+
} catch {
|
|
6625
|
+
}
|
|
6626
|
+
}
|
|
6627
|
+
notified.push(info2);
|
|
6628
|
+
}
|
|
6629
|
+
return notified;
|
|
6630
|
+
}
|
|
6631
|
+
/**
|
|
6632
|
+
* The stranded runs right now, without notifying anyone: the query behind {@link sweepStalled} and
|
|
6633
|
+
* the CLI's `durable inspect --stale`. A `running`/`suspended` run qualifies when it has been
|
|
6634
|
+
* untouched for `thresholdMs` (default `stalledAfterMs`) AND either it is suspended with no wake
|
|
6635
|
+
* timer, or its oldest pending remote step is older than the threshold with no lease in the future
|
|
6636
|
+
* (a step with a `timeoutMs`, a heartbeat or `remoteRedispatchMs` keeps a lease; a long
|
|
6637
|
+
* `ctx.sleep` has a future `wakeAt`; neither is stranded). Scans up to `scan` candidates (default
|
|
6638
|
+
* 1000), newest first, in this engine's namespace.
|
|
6639
|
+
*/
|
|
6640
|
+
async listStalledRuns(options = {}) {
|
|
6641
|
+
return findStalledRuns(this.store, {
|
|
6642
|
+
now: options.now ?? this.clock(),
|
|
6643
|
+
thresholdMs: options.thresholdMs ?? this.stalledAfterMs,
|
|
6644
|
+
scan: options.scan,
|
|
6645
|
+
namespace: this.namespace
|
|
6646
|
+
});
|
|
6647
|
+
}
|
|
6648
|
+
notifyDead(run) {
|
|
6649
|
+
for (const fn of this.deadListeners) {
|
|
6650
|
+
try {
|
|
6651
|
+
fn(run);
|
|
6652
|
+
} catch {
|
|
6653
|
+
}
|
|
6654
|
+
}
|
|
6655
|
+
}
|
|
6656
|
+
/**
|
|
6657
|
+
* Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the control
|
|
6658
|
+
* plane so other instances (e.g. a dashboard pod) can live-tail this run. A `run.completed` or
|
|
6659
|
+
* `run.failed` also publishes the run's lifecycle event (see {@link publishLifecycle}) and resumes
|
|
6660
|
+
* the runs waiting on it with `ctx.waitForRun` (see {@link deliverRunWaits}); `lifecycle: false`
|
|
6661
|
+
* marks a `run.completed` that is not a real completion (a continue-as-new handoff).
|
|
6662
|
+
*/
|
|
6663
|
+
emit(event, opts) {
|
|
6664
|
+
const full = {
|
|
6665
|
+
...event,
|
|
6666
|
+
at: /* @__PURE__ */ new Date()
|
|
6667
|
+
};
|
|
6668
|
+
this.deliver(full);
|
|
6669
|
+
if (opts?.lifecycle !== false) {
|
|
6670
|
+
this.publishLifecycle(full);
|
|
6671
|
+
if ((full.type === "run.completed" || full.type === "run.failed") && !full.workflow?.startsWith("__")) {
|
|
6672
|
+
void this.deliverRunWaits(full.runId).catch(() => void 0);
|
|
6673
|
+
}
|
|
6674
|
+
}
|
|
6675
|
+
if (!this.controlPlane) return;
|
|
6676
|
+
this.eventsToBroadcast.push(full);
|
|
6677
|
+
if (!this.broadcastingEvents) void this.broadcastEvents();
|
|
6678
|
+
}
|
|
6679
|
+
/**
|
|
6680
|
+
* Start every workflow subscribed to `name` that takes `payload` (see {@link acceptingSubscription}),
|
|
6681
|
+
* each under `runIdOf(workflow)` and with its subscription's `input` applied. `only` restricts it
|
|
6682
|
+
* to those subscribers. `remote` are subscribers only live workers announce (see
|
|
6683
|
+
* {@link announcedRemoteSubscribers}): each is started by name with the payload as its input, since
|
|
6684
|
+
* an announcement carries no `where` or `input`. `routeIdOf` names the event for a subscriber's
|
|
6685
|
+
* debounce/batch accumulator, which counts an id once. Returns the subscribers reached, with the
|
|
6686
|
+
* run each started (none when the payload went into its debounce/batch accumulator), and the ones
|
|
6687
|
+
* that could not be started, with why.
|
|
6688
|
+
*/
|
|
6689
|
+
async startSubscribers(name, payload, runIdOf, only, remote = [], routeIdOf) {
|
|
6690
|
+
const reached = [];
|
|
6691
|
+
const failed = [];
|
|
6692
|
+
for (const [workflow, subscriptions] of this.eventTriggers.get(name) ?? []) {
|
|
6693
|
+
if (only && !only.has(workflow)) continue;
|
|
6694
|
+
const { accepted, errors } = acceptingSubscription(subscriptions, payload);
|
|
6695
|
+
if (!accepted) {
|
|
6696
|
+
if (errors.length > 0) failed.push({
|
|
6697
|
+
workflow,
|
|
6698
|
+
error: errors[0]
|
|
6699
|
+
});
|
|
6700
|
+
continue;
|
|
6701
|
+
}
|
|
6702
|
+
try {
|
|
6703
|
+
const input = accepted.input ? accepted.input(payload) : payload;
|
|
6704
|
+
const batch = this.latest.get(workflow)?.eventBatch;
|
|
6705
|
+
if (batch) {
|
|
6706
|
+
await this.accumulators.route(workflow, batch, input, routeIdOf?.(workflow));
|
|
6707
|
+
reached.push({
|
|
6708
|
+
workflow
|
|
6709
|
+
});
|
|
6710
|
+
} else {
|
|
6711
|
+
const runId = runIdOf(workflow);
|
|
6712
|
+
await this.start(workflow, input, runId);
|
|
6713
|
+
reached.push({
|
|
6714
|
+
workflow,
|
|
6715
|
+
runId
|
|
6716
|
+
});
|
|
6717
|
+
}
|
|
6718
|
+
} catch (error2) {
|
|
6719
|
+
failed.push({
|
|
6720
|
+
workflow,
|
|
6721
|
+
error: error2
|
|
6722
|
+
});
|
|
6723
|
+
}
|
|
6724
|
+
}
|
|
6725
|
+
const local = this.eventTriggers.get(name);
|
|
6726
|
+
for (const workflow of remote) {
|
|
6727
|
+
if (local?.has(workflow) || only && !only.has(workflow)) continue;
|
|
6728
|
+
try {
|
|
6729
|
+
const runId = runIdOf(workflow);
|
|
6730
|
+
await this.start(workflow, payload, runId);
|
|
6731
|
+
reached.push({
|
|
6732
|
+
workflow,
|
|
6733
|
+
runId
|
|
6734
|
+
});
|
|
6735
|
+
} catch (error2) {
|
|
6736
|
+
failed.push({
|
|
6737
|
+
workflow,
|
|
6738
|
+
error: error2
|
|
6739
|
+
});
|
|
6740
|
+
}
|
|
6741
|
+
}
|
|
6742
|
+
return {
|
|
6743
|
+
reached,
|
|
6744
|
+
failed
|
|
6745
|
+
};
|
|
6746
|
+
}
|
|
6747
|
+
/**
|
|
6748
|
+
* The subscribers of `event` that live workers announce for the settled runs of `namespace` (`null`:
|
|
6749
|
+
* any namespace, for an event no run published), are not registered on this engine with an `onEvent` for it, and that this engine can start: registered here (a `registerRemote`), or served
|
|
6750
|
+
* by convention on a live worker group of the same name (a Python `@workflow(on_event=...)`). One
|
|
6751
|
+
* this engine cannot start is left to the instances that can, so it stays owed rather than failing.
|
|
6752
|
+
*/
|
|
6753
|
+
async announcedRemoteSubscribers(event, namespace) {
|
|
6754
|
+
const snapshot = await this.announcedSubscriptions();
|
|
6755
|
+
const announced = announcedIn(snapshot, event, namespace);
|
|
6756
|
+
if (!announced.size) return [];
|
|
6757
|
+
const local = new Set(this.localSubscribersIn(event, namespace));
|
|
6758
|
+
const byEngine = announcedIn(snapshot, event, namespace, true);
|
|
6759
|
+
const startable = [];
|
|
6760
|
+
for (const workflow of [
|
|
6761
|
+
...announced
|
|
6762
|
+
].sort()) {
|
|
6763
|
+
if (local.has(workflow) || byEngine.has(workflow) || workflow.startsWith("__")) continue;
|
|
6764
|
+
if (this.latest.has(workflow)) {
|
|
6765
|
+
startable.push(workflow);
|
|
6766
|
+
continue;
|
|
6767
|
+
}
|
|
6768
|
+
const group = tenantGroup(sanitizeQueueToken(workflow), this.namespace);
|
|
6769
|
+
const live = await this.pool.transportWithLiveGroup(group).catch(() => void 0);
|
|
6770
|
+
if (live) startable.push(workflow);
|
|
6771
|
+
}
|
|
6772
|
+
return startable;
|
|
6773
|
+
}
|
|
6774
|
+
/** The workflows registered on this engine that subscribe to `event`. */
|
|
6775
|
+
localSubscribers(event) {
|
|
6776
|
+
return [
|
|
6777
|
+
...this.eventTriggers.get(event)?.keys() ?? []
|
|
6778
|
+
];
|
|
6779
|
+
}
|
|
6780
|
+
/**
|
|
6781
|
+
* Whether some workflow may subscribe to `workflow` settling as `kind`: one registered on this
|
|
6782
|
+
* engine, or one in the last read of the fleet's announcements. True while there is no usable read
|
|
6783
|
+
* (none yet, or the last one failed), since the terminal write then marks the run unresolved (see
|
|
6784
|
+
* {@link withLifecycleMarker}).
|
|
6785
|
+
*/
|
|
6786
|
+
hasLifecycleSubscribers(workflow, kind, namespace) {
|
|
6787
|
+
if (workflow.startsWith("__")) return false;
|
|
6788
|
+
const event = lifecycleEventName(workflow, kind);
|
|
6789
|
+
if (this.servesNamespace(namespace) && this.localSubscribers(event).length > 0) return true;
|
|
6790
|
+
const snapshot = this.announcedSubscriptionsSnapshot;
|
|
6791
|
+
if (!snapshot?.ok) return true;
|
|
6792
|
+
return announcedIn(snapshot, event, namespace).size > 0;
|
|
6793
|
+
}
|
|
6794
|
+
/**
|
|
6795
|
+
* Whether this engine's own subscribers apply to the settled runs of `namespace`: an engine with
|
|
6796
|
+
* no namespace publishes the runs of every namespace, a namespaced one only its own.
|
|
6797
|
+
*/
|
|
6798
|
+
servesNamespace(namespace) {
|
|
6799
|
+
return namespace === null || this.namespace === void 0 || this.namespace === runNamespace(namespace);
|
|
6800
|
+
}
|
|
6801
|
+
/** This engine's subscribers of `event` that apply to the settled runs of `namespace`. */
|
|
6802
|
+
localSubscribersIn(event, namespace) {
|
|
6803
|
+
return this.servesNamespace(namespace) ? this.localSubscribers(event) : [];
|
|
6804
|
+
}
|
|
6805
|
+
/** Answers the traceparent of a run's span in this process; see {@link setRunTraceparent}. */
|
|
6806
|
+
runTraceparent;
|
|
6807
|
+
/**
|
|
6808
|
+
* Install the function that answers the W3C `traceparent` of a run's span in this process, or
|
|
6809
|
+
* `undefined` to remove it. `attachDurableOtel` installs one. With it, the first suspend or the
|
|
6810
|
+
* terminal write of a run some workflow subscribes to also records the run's traceparent (a
|
|
6811
|
+
* {@link LIFECYCLE_TRACE_TAG_PREFIX} tag, see {@link withRunTrace}), and the `run.started` of a run
|
|
6812
|
+
* a lifecycle event started carries its source run's as `sourceTraceparent`, so the subscriber's
|
|
6813
|
+
* trace links to the source's.
|
|
6814
|
+
*/
|
|
6815
|
+
setRunTraceparent(provider) {
|
|
6816
|
+
this.runTraceparent = provider;
|
|
6817
|
+
}
|
|
6818
|
+
/**
|
|
6819
|
+
* `{ sourceTraceparent }` for the `run.started` of `run`, when a lifecycle event started it and a
|
|
6820
|
+
* run trace provider is installed: the source run's recorded traceparent, else this process's.
|
|
6821
|
+
*/
|
|
6822
|
+
async sourceTraceOf(run) {
|
|
6823
|
+
const provider = this.runTraceparent;
|
|
6824
|
+
if (!provider) return {};
|
|
6825
|
+
const source = lifecycleSourceOf(run);
|
|
6826
|
+
if (!source) return {};
|
|
6827
|
+
try {
|
|
6828
|
+
const recorded = lifecycleTraceparentOf((await this.store.getRun(source.runId))?.tags);
|
|
6829
|
+
const traceparent = recorded ?? provider(source.runId);
|
|
6830
|
+
return traceparent ? {
|
|
6831
|
+
sourceTraceparent: traceparent
|
|
6832
|
+
} : {};
|
|
6833
|
+
} catch {
|
|
6834
|
+
return {};
|
|
6835
|
+
}
|
|
6836
|
+
}
|
|
6837
|
+
/** The last read of the subscriptions live workers announce. See {@link announcedSubscriptions}. */
|
|
6838
|
+
announcedSubscriptionsSnapshot;
|
|
6839
|
+
announcedSubscriptionsRead;
|
|
6840
|
+
/**
|
|
6841
|
+
* The `onEvent` subscriptions the live fleet announces, by event name: the fresh lifecycle
|
|
6842
|
+
* registry runs in the state store (see {@link heartbeatLifecycleRegistry}) and, on a transport
|
|
6843
|
+
* that reads them, each worker descriptor's `subscribes`. Each entry records the namespace its
|
|
6844
|
+
* announcer serves (`null` for an engine with no namespace) and whether a Node engine announced it
|
|
6845
|
+
* (every registry run is one). Read at most once per
|
|
6846
|
+
* {@link ANNOUNCED_ROUTES_TTL_MS}, concurrent callers sharing one read. Never throws: `ok: false`
|
|
6847
|
+
* means a read failed and only the local registry is known.
|
|
6848
|
+
*/
|
|
6849
|
+
async announcedSubscriptions() {
|
|
6850
|
+
const cached = this.announcedSubscriptionsSnapshot;
|
|
6851
|
+
if (cached && this.clock() - cached.at <= ANNOUNCED_ROUTES_TTL_MS) return cached;
|
|
6852
|
+
this.announcedSubscriptionsRead ??= this.readAnnouncedSubscriptions().finally(() => {
|
|
6853
|
+
this.announcedSubscriptionsRead = void 0;
|
|
6854
|
+
});
|
|
6855
|
+
return this.announcedSubscriptionsRead;
|
|
6856
|
+
}
|
|
6857
|
+
/**
|
|
6858
|
+
* The last read of the fleet's announcements as it is, without waiting: undefined before the first
|
|
6859
|
+
* read completes. Starts a read in the background when it is missing or older than
|
|
6860
|
+
* {@link ANNOUNCED_ROUTES_TTL_MS}. What the terminal write uses, so settling a run never waits on
|
|
6861
|
+
* the fleet.
|
|
6862
|
+
*/
|
|
6863
|
+
cachedAnnouncedSubscriptions() {
|
|
6864
|
+
const cached = this.announcedSubscriptionsSnapshot;
|
|
6865
|
+
if (!cached || this.clock() - cached.at > ANNOUNCED_ROUTES_TTL_MS) {
|
|
6866
|
+
void this.announcedSubscriptions();
|
|
6867
|
+
}
|
|
6868
|
+
return cached;
|
|
6869
|
+
}
|
|
6870
|
+
/**
|
|
6871
|
+
* When this engine first saw each subscription (`<event>\u0000<subscriber>`), in its registry or in
|
|
6872
|
+
* a good read of the fleet's announcements. The sweep's heal pass looks only at the ones first seen
|
|
6873
|
+
* within {@link LIFECYCLE_HEAL_WINDOW_MS}; see {@link healLifecycleEvents}.
|
|
6874
|
+
*/
|
|
6875
|
+
subscriptionFirstSeen = /* @__PURE__ */ new Map();
|
|
6876
|
+
/** Record the subscriptions of `snapshot` (a good read) and of this engine's registry as seen. */
|
|
6877
|
+
noteSubscriptionsSeen(snapshot) {
|
|
6878
|
+
const now = this.clock();
|
|
6879
|
+
const note = /* @__PURE__ */ __name((event, subscriber) => {
|
|
6880
|
+
const key = `${event}\0${subscriber}`;
|
|
6881
|
+
if (!this.subscriptionFirstSeen.has(key)) this.subscriptionFirstSeen.set(key, now);
|
|
6882
|
+
}, "note");
|
|
6883
|
+
for (const [event, subscribers] of snapshot.byEvent) {
|
|
6884
|
+
for (const subscriber of subscribers) note(event, subscriber);
|
|
6885
|
+
}
|
|
6886
|
+
for (const [event, subscribers] of this.eventTriggers) {
|
|
6887
|
+
for (const subscriber of subscribers.keys()) note(event, subscriber);
|
|
6888
|
+
}
|
|
6889
|
+
}
|
|
6890
|
+
async readAnnouncedSubscriptions() {
|
|
6891
|
+
const byEvent = /* @__PURE__ */ new Map();
|
|
6892
|
+
const entries = [];
|
|
6893
|
+
let ok = true;
|
|
6894
|
+
const add = /* @__PURE__ */ __name((event, workflow, node, namespace) => {
|
|
6895
|
+
entries.push({
|
|
6896
|
+
event,
|
|
6897
|
+
workflow,
|
|
6898
|
+
namespace,
|
|
6899
|
+
node
|
|
6900
|
+
});
|
|
6901
|
+
const subscribers = byEvent.get(event) ?? /* @__PURE__ */ new Set();
|
|
6902
|
+
subscribers.add(workflow);
|
|
6903
|
+
byEvent.set(event, subscribers);
|
|
6904
|
+
}, "add");
|
|
6905
|
+
try {
|
|
6906
|
+
const now = this.clock();
|
|
6907
|
+
for (const row of await this.listLifecycleRegistry()) {
|
|
6908
|
+
if (now - row.updatedAt.getTime() > LIFECYCLE_REGISTRY_TTL_MS) continue;
|
|
6909
|
+
const namespace = lifecycleRegistryNamespace(row.input, row.namespace);
|
|
6910
|
+
for (const { workflow, events } of lifecycleRegistrySubscriptions(row.input)) {
|
|
6911
|
+
for (const event of events) add(event, workflow, true, namespace);
|
|
6912
|
+
}
|
|
6913
|
+
}
|
|
6914
|
+
if (this.pool.introspectsFleet) {
|
|
6915
|
+
for (const descriptor of await this.pool.readAllWorkerDescriptors()) {
|
|
6916
|
+
for (const announced of announcementsOf(descriptor)) {
|
|
6917
|
+
const namespace = servedNamespace(descriptor.namespace, announced.group);
|
|
6918
|
+
for (const event of announced.subscribes ?? []) {
|
|
6919
|
+
add(event, announced.name, descriptor.runtime === "node", namespace);
|
|
6920
|
+
}
|
|
6921
|
+
}
|
|
6922
|
+
}
|
|
6923
|
+
}
|
|
6924
|
+
} catch {
|
|
6925
|
+
ok = false;
|
|
6926
|
+
byEvent.clear();
|
|
6927
|
+
entries.length = 0;
|
|
6928
|
+
}
|
|
6929
|
+
const snapshot = {
|
|
6930
|
+
at: this.clock(),
|
|
6931
|
+
ok,
|
|
6932
|
+
byEvent,
|
|
6933
|
+
entries
|
|
6934
|
+
};
|
|
6935
|
+
this.announcedSubscriptionsSnapshot = snapshot;
|
|
6936
|
+
if (ok) this.noteSubscriptionsSeen(snapshot);
|
|
6937
|
+
return snapshot;
|
|
6938
|
+
}
|
|
6939
|
+
/** How many registry runs one page of {@link listLifecycleRegistry} reads. */
|
|
6940
|
+
static LIFECYCLE_REGISTRY_PAGE = 1e3;
|
|
6941
|
+
/**
|
|
6942
|
+
* The lifecycle registry runs this engine's namespace sees: its namespace's, plus `default`'s, where
|
|
6943
|
+
* the engines with no namespace keep theirs. Read a page of
|
|
6944
|
+
* {@link LIFECYCLE_REGISTRY_PAGE} at a time until a short page, so a large fleet is read whole.
|
|
6945
|
+
*/
|
|
6946
|
+
async listLifecycleRegistry() {
|
|
6947
|
+
const pageSize = _WorkflowEngine.LIFECYCLE_REGISTRY_PAGE;
|
|
6948
|
+
const rows = [];
|
|
6949
|
+
for (; ; ) {
|
|
6950
|
+
const page = await this.store.listRuns({
|
|
6951
|
+
workflow: LIFECYCLE_REGISTRY_WORKFLOW,
|
|
6952
|
+
...this.namespace !== void 0 ? {
|
|
6953
|
+
namespaces: [
|
|
6954
|
+
.../* @__PURE__ */ new Set([
|
|
6955
|
+
this.namespace,
|
|
6956
|
+
"default"
|
|
6957
|
+
])
|
|
6958
|
+
]
|
|
6959
|
+
} : {},
|
|
6960
|
+
limit: pageSize,
|
|
6961
|
+
offset: rows.length
|
|
6962
|
+
});
|
|
6963
|
+
rows.push(...page);
|
|
6964
|
+
if (page.length < pageSize) return rows;
|
|
6965
|
+
}
|
|
6966
|
+
}
|
|
6967
|
+
/** When this engine last wrote its registry run, or undefined before the first heartbeat. */
|
|
6968
|
+
lifecycleRegistryBeatAt;
|
|
6969
|
+
/**
|
|
6970
|
+
* Refresh this engine's lifecycle registry run and delete stale ones. At most once per
|
|
6971
|
+
* {@link LIFECYCLE_REGISTRY_HEARTBEAT_MS}; {@link sweepLifecycleEvents} calls it on every tick.
|
|
6972
|
+
*
|
|
6973
|
+
* An engine with `onEvent` subscriptions keeps one completed run of the internal
|
|
6974
|
+
* {@link LIFECYCLE_REGISTRY_WORKFLOW} workflow, id `__lifecycle_registry:<instanceId>`, holding
|
|
6975
|
+
* them in `input` and its heartbeat in `updatedAt`. Other instances read the fresh ones
|
|
6976
|
+
* ({@link LIFECYCLE_REGISTRY_TTL_MS}) as announced subscriptions, so a settled run is owed to a
|
|
6977
|
+
* subscriber registered only elsewhere whatever the transport. Any engine deletes the runs not
|
|
6978
|
+
* refreshed for {@link LIFECYCLE_REGISTRY_STALE_MS}. Retention never prunes them
|
|
6979
|
+
* ({@link LIFECYCLE_REGISTRY_TAG}), and the run gateway leaves them out of run lists.
|
|
6980
|
+
*/
|
|
6981
|
+
async heartbeatLifecycleRegistry() {
|
|
6982
|
+
const now = this.clock();
|
|
6983
|
+
const last = this.lifecycleRegistryBeatAt;
|
|
6984
|
+
if (last !== void 0 && now - last < LIFECYCLE_REGISTRY_HEARTBEAT_MS) return;
|
|
6985
|
+
this.lifecycleRegistryBeatAt = now;
|
|
6986
|
+
try {
|
|
6987
|
+
const subscriptions = this.registrySubscriptions();
|
|
6988
|
+
if (subscriptions.length > 0) await this.writeLifecycleRegistry(subscriptions, now);
|
|
6989
|
+
const stale = (await this.listLifecycleRegistry()).filter((row) => now - row.updatedAt.getTime() > LIFECYCLE_REGISTRY_STALE_MS);
|
|
6990
|
+
for (const row of stale) await this.store.deleteRun(row.id);
|
|
6991
|
+
} catch (error2) {
|
|
6992
|
+
this.lifecycleRegistryBeatAt = last;
|
|
6993
|
+
throw error2;
|
|
6994
|
+
}
|
|
6995
|
+
}
|
|
6996
|
+
/** This engine's `onEvent` subscriptions, by subscribing workflow, as its registry run holds them. */
|
|
6997
|
+
registrySubscriptions() {
|
|
6998
|
+
const byWorkflow = /* @__PURE__ */ new Map();
|
|
6999
|
+
for (const [event, subscribers] of this.eventTriggers) {
|
|
7000
|
+
for (const workflow of subscribers.keys()) {
|
|
7001
|
+
if (workflow.startsWith("__")) continue;
|
|
7002
|
+
const events = byWorkflow.get(workflow) ?? /* @__PURE__ */ new Set();
|
|
7003
|
+
events.add(event);
|
|
7004
|
+
byWorkflow.set(workflow, events);
|
|
7005
|
+
}
|
|
7006
|
+
}
|
|
7007
|
+
return [
|
|
7008
|
+
...byWorkflow
|
|
7009
|
+
].sort(([a], [b]) => a.localeCompare(b)).map(([workflow, events]) => ({
|
|
7010
|
+
workflow,
|
|
7011
|
+
events: [
|
|
7012
|
+
...events
|
|
7013
|
+
].sort()
|
|
7014
|
+
}));
|
|
7015
|
+
}
|
|
7016
|
+
async writeLifecycleRegistry(subscriptions, nowMs) {
|
|
7017
|
+
const id = lifecycleRegistryRunId(this.instanceId);
|
|
7018
|
+
const input = {
|
|
7019
|
+
instanceId: this.instanceId,
|
|
7020
|
+
runtime: "node",
|
|
7021
|
+
engineNamespace: this.namespace ?? null,
|
|
7022
|
+
subscriptions
|
|
7023
|
+
};
|
|
7024
|
+
const at = new Date(nowMs);
|
|
7025
|
+
const refreshed = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, id, [
|
|
7026
|
+
"completed"
|
|
7027
|
+
], {
|
|
7028
|
+
input,
|
|
7029
|
+
updatedAt: at
|
|
7030
|
+
});
|
|
7031
|
+
if (refreshed) return;
|
|
7032
|
+
await this.store.createRun({
|
|
7033
|
+
id,
|
|
7034
|
+
workflow: LIFECYCLE_REGISTRY_WORKFLOW,
|
|
7035
|
+
workflowVersion: "1",
|
|
7036
|
+
status: "completed",
|
|
7037
|
+
input,
|
|
7038
|
+
tags: [
|
|
7039
|
+
LIFECYCLE_REGISTRY_TAG
|
|
7040
|
+
],
|
|
7041
|
+
...this.namespace !== void 0 ? {
|
|
7042
|
+
namespace: this.namespace
|
|
7043
|
+
} : {},
|
|
7044
|
+
origin: CORE_ORIGIN,
|
|
7045
|
+
createdAt: at,
|
|
7046
|
+
updatedAt: at
|
|
7047
|
+
});
|
|
7048
|
+
}
|
|
7049
|
+
/**
|
|
7050
|
+
* The subscribers a run of `workflow` in `namespace` settling as `kind` owes its lifecycle event to:
|
|
7051
|
+
* the ones registered on this engine, when it publishes that namespace's runs (an engine with no
|
|
7052
|
+
* namespace publishes every namespace's), plus the ones live workers announce for that namespace
|
|
7053
|
+
* (see {@link announcedSubscriptions}): by announcers serving it, and by engines with no namespace.
|
|
7054
|
+
* Without `namespace`, every namespace's: what any run of `workflow` could owe. Empty for an
|
|
7055
|
+
* internal `__` workflow.
|
|
7056
|
+
*/
|
|
7057
|
+
async lifecycleSubscribers(workflow, kind, namespace) {
|
|
7058
|
+
return this.lifecycleSubscribersIn(await this.announcedSubscriptions(), workflow, kind, namespace);
|
|
7059
|
+
}
|
|
7060
|
+
/** {@link lifecycleSubscribers} from one given read of the fleet's announcements. */
|
|
7061
|
+
lifecycleSubscribersIn(snapshot, workflow, kind, namespace) {
|
|
7062
|
+
if (workflow.startsWith("__")) return [];
|
|
7063
|
+
const scope = namespace === void 0 ? null : runNamespace(namespace);
|
|
7064
|
+
const event = lifecycleEventName(workflow, kind);
|
|
7065
|
+
const announced = announcedIn(snapshot, event, scope);
|
|
7066
|
+
return [
|
|
7067
|
+
.../* @__PURE__ */ new Set([
|
|
7068
|
+
...this.localSubscribersIn(event, scope),
|
|
7069
|
+
...announced
|
|
7070
|
+
])
|
|
7071
|
+
].sort();
|
|
7072
|
+
}
|
|
7073
|
+
/**
|
|
7074
|
+
* The subscribers registered here that coalesce `run`'s lifecycle event (debounce/batch) and
|
|
7075
|
+
* accept it, each with the accumulator run the event went to: the subscription's `where`, `input`
|
|
7076
|
+
* and `key` applied to the event as `startSubscribers` does. Undefined when the key cannot be
|
|
7077
|
+
* computed. A subscriber whose `where` declines the event is left out, and so is every one when
|
|
7078
|
+
* this engine does not publish the run's namespace. Empty for a run that has not settled.
|
|
7079
|
+
*/
|
|
7080
|
+
coalescedSubscribers(run) {
|
|
7081
|
+
const out = /* @__PURE__ */ new Map();
|
|
7082
|
+
const kind = lifecycleKindOf(run.status);
|
|
7083
|
+
if (!kind) return out;
|
|
7084
|
+
const triggers = this.eventTriggers.get(lifecycleEventName(run.workflow, kind));
|
|
7085
|
+
if (!triggers || !this.servesNamespace(run.namespace)) return out;
|
|
7086
|
+
let payload;
|
|
7087
|
+
for (const [workflow, subscriptions] of triggers) {
|
|
7088
|
+
const batch = this.latest.get(workflow)?.eventBatch;
|
|
7089
|
+
if (!batch) continue;
|
|
7090
|
+
if (!batch.key && subscriptions.every((sub2) => !sub2.where)) {
|
|
7091
|
+
out.set(workflow, accumulatorRunId(workflow));
|
|
7092
|
+
continue;
|
|
7093
|
+
}
|
|
7094
|
+
try {
|
|
7095
|
+
payload ??= lifecycleEventOf(run);
|
|
7096
|
+
const { accepted } = acceptingSubscription(subscriptions, payload);
|
|
7097
|
+
if (!accepted) continue;
|
|
7098
|
+
const input = accepted.input ? accepted.input(payload) : payload;
|
|
7099
|
+
out.set(workflow, accumulatorRunId(workflow, batch.key ? String(batch.key(input)) : void 0));
|
|
7100
|
+
} catch {
|
|
7101
|
+
out.set(workflow, void 0);
|
|
7102
|
+
}
|
|
7103
|
+
}
|
|
7104
|
+
return out;
|
|
7105
|
+
}
|
|
7106
|
+
/**
|
|
7107
|
+
* Who starts whom through lifecycle events, for `workflow`: the workflows that start when it
|
|
7108
|
+
* settles, and the workflows whose settling starts it. Built from this engine's registry and the
|
|
7109
|
+
* subscriptions live workers announce (see {@link announcedSubscriptions}); a failed fleet read
|
|
7110
|
+
* leaves the local registry alone. What a dashboard's workflow page shows.
|
|
7111
|
+
*
|
|
7112
|
+
* With `namespace`, only the subscriptions of that namespace: the ones announced by registry runs
|
|
7113
|
+
* and worker descriptors serving it, and this engine's own when it is that namespace's engine (an
|
|
7114
|
+
* engine with no namespace is `default`'s). What a tenant is shown.
|
|
7115
|
+
*/
|
|
7116
|
+
async workflowLifecycle(workflow, opts) {
|
|
7117
|
+
const announced = await this.announcedSubscriptions();
|
|
7118
|
+
const scope = opts?.namespace;
|
|
7119
|
+
let fleet = announced.byEvent;
|
|
7120
|
+
if (scope !== void 0) {
|
|
7121
|
+
fleet = /* @__PURE__ */ new Map();
|
|
7122
|
+
for (const entry of announced.entries) {
|
|
7123
|
+
if ((entry.namespace ?? "default") !== scope) continue;
|
|
7124
|
+
const subscribers = fleet.get(entry.event) ?? /* @__PURE__ */ new Set();
|
|
7125
|
+
subscribers.add(entry.workflow);
|
|
7126
|
+
fleet.set(entry.event, subscribers);
|
|
7127
|
+
}
|
|
7128
|
+
}
|
|
7129
|
+
const triggers = scope === void 0 || (this.namespace || "default") === scope ? this.eventTriggers : /* @__PURE__ */ new Map();
|
|
7130
|
+
const startsOnSettle = /* @__PURE__ */ new Map();
|
|
7131
|
+
const startedBy = /* @__PURE__ */ new Map();
|
|
7132
|
+
const link = /* @__PURE__ */ __name((into, other, kind, local) => {
|
|
7133
|
+
const key = `${other}\0${kind}`;
|
|
7134
|
+
const entry = into.get(key) ?? {
|
|
7135
|
+
workflow: other,
|
|
7136
|
+
kind,
|
|
7137
|
+
local: false,
|
|
7138
|
+
announced: false,
|
|
7139
|
+
where: false,
|
|
7140
|
+
input: false
|
|
7141
|
+
};
|
|
7142
|
+
if (local) {
|
|
7143
|
+
entry.local = true;
|
|
7144
|
+
entry.where ||= local.subscriptions.some((sub2) => sub2.where !== void 0);
|
|
7145
|
+
entry.input ||= local.subscriptions.some((sub2) => sub2.input !== void 0);
|
|
7146
|
+
const batch = this.latest.get(local.subscriber)?.eventBatch;
|
|
7147
|
+
if (batch) entry.coalesce = {
|
|
7148
|
+
mode: batch.mode,
|
|
7149
|
+
keyed: batch.key !== void 0
|
|
7150
|
+
};
|
|
7151
|
+
} else {
|
|
7152
|
+
entry.announced = true;
|
|
7153
|
+
}
|
|
7154
|
+
into.set(key, entry);
|
|
7155
|
+
}, "link");
|
|
7156
|
+
for (const kind of [
|
|
7157
|
+
"completed",
|
|
7158
|
+
"failed"
|
|
7159
|
+
]) {
|
|
7160
|
+
const event = lifecycleEventName(workflow, kind);
|
|
7161
|
+
for (const [subscriber, subscriptions] of triggers.get(event) ?? []) {
|
|
7162
|
+
link(startsOnSettle, subscriber, kind, {
|
|
7163
|
+
subscriber,
|
|
7164
|
+
subscriptions
|
|
7165
|
+
});
|
|
7166
|
+
}
|
|
7167
|
+
for (const subscriber of fleet.get(event) ?? []) {
|
|
7168
|
+
link(startsOnSettle, subscriber, kind, void 0);
|
|
7169
|
+
}
|
|
7170
|
+
}
|
|
7171
|
+
for (const [event, subscribers] of triggers) {
|
|
7172
|
+
const source = parseLifecycleEvent(event);
|
|
7173
|
+
const subscriptions = subscribers.get(workflow);
|
|
7174
|
+
if (source && subscriptions) {
|
|
7175
|
+
link(startedBy, source.workflow, source.kind, {
|
|
7176
|
+
subscriber: workflow,
|
|
7177
|
+
subscriptions
|
|
7178
|
+
});
|
|
7179
|
+
}
|
|
7180
|
+
}
|
|
7181
|
+
for (const [event, subscribers] of fleet) {
|
|
7182
|
+
const source = parseLifecycleEvent(event);
|
|
7183
|
+
if (source && subscribers.has(workflow)) {
|
|
7184
|
+
link(startedBy, source.workflow, source.kind, void 0);
|
|
7185
|
+
}
|
|
7186
|
+
}
|
|
7187
|
+
const sorted = /* @__PURE__ */ __name((links) => [
|
|
7188
|
+
...links.values()
|
|
7189
|
+
].sort((a, b) => a.workflow.localeCompare(b.workflow) || a.kind.localeCompare(b.kind)), "sorted");
|
|
7190
|
+
return {
|
|
7191
|
+
workflow,
|
|
7192
|
+
startsOnSettle: sorted(startsOnSettle),
|
|
7193
|
+
startedBy: sorted(startedBy)
|
|
7194
|
+
};
|
|
7195
|
+
}
|
|
7196
|
+
/**
|
|
7197
|
+
* `patch` (a terminal status write of `run`) plus the lifecycle marker, when some workflow
|
|
7198
|
+
* subscribes to that ending: {@link LIFECYCLE_PENDING_TAG} and one owed tag per subscriber (see
|
|
7199
|
+
* {@link lifecycleSubscribers}). Written in the same update as the status, so a run can never be
|
|
7200
|
+
* settled without a record of who its lifecycle event is still owed to.
|
|
7201
|
+
*
|
|
7202
|
+
* Uses the last read of the fleet's announcements without waiting for a new one (see
|
|
7203
|
+
* {@link cachedAnnouncedSubscriptions}). With no usable read (none yet, or the last one failed) the
|
|
7204
|
+
* marker is {@link LIFECYCLE_PENDING_TAG} alone, for any workflow but an internal `__` one: the
|
|
7205
|
+
* owed set is unresolved, and {@link publishSettled} resolves it once a read succeeds.
|
|
7206
|
+
*/
|
|
7207
|
+
async withLifecycleMarker(run, patch) {
|
|
7208
|
+
const kind = lifecycleKindOf(patch.status);
|
|
7209
|
+
if (!kind || run.workflow.startsWith("__")) return patch;
|
|
7210
|
+
const snapshot = this.cachedAnnouncedSubscriptions();
|
|
7211
|
+
const subscribers = snapshot?.ok ? this.lifecycleSubscribersIn(snapshot, run.workflow, kind, runNamespace(run.namespace)) : [];
|
|
7212
|
+
if (snapshot?.ok && subscribers.length === 0) return patch;
|
|
7213
|
+
const tags = patch.tags ?? run.tags ?? [];
|
|
7214
|
+
let traceparent;
|
|
7215
|
+
try {
|
|
7216
|
+
traceparent = this.runTraceparent?.(run.id);
|
|
7217
|
+
} catch {
|
|
7218
|
+
traceparent = void 0;
|
|
7219
|
+
}
|
|
7220
|
+
const recordsTrace = traceparent !== void 0 && !tags.some((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX));
|
|
7221
|
+
const marker = [
|
|
7222
|
+
LIFECYCLE_PENDING_TAG,
|
|
7223
|
+
...subscribers.map((s) => `${LIFECYCLE_OWED_TAG_PREFIX}${s}`),
|
|
7224
|
+
...recordsTrace ? [
|
|
7225
|
+
`${LIFECYCLE_TRACE_TAG_PREFIX}${traceparent}`
|
|
7226
|
+
] : []
|
|
7227
|
+
];
|
|
7228
|
+
if (marker.every((tag) => tags.includes(tag))) return patch;
|
|
7229
|
+
return {
|
|
7230
|
+
...patch,
|
|
7231
|
+
tags: [
|
|
7232
|
+
.../* @__PURE__ */ new Set([
|
|
7233
|
+
...tags,
|
|
7234
|
+
...marker
|
|
7235
|
+
])
|
|
7236
|
+
]
|
|
7237
|
+
};
|
|
7238
|
+
}
|
|
7239
|
+
/**
|
|
7240
|
+
* `patch` (a suspend write of `run`) plus the run's {@link LIFECYCLE_TRACE_TAG_PREFIX} tag, when a
|
|
7241
|
+
* run trace provider is installed, some workflow subscribes to the run's ending, the provider
|
|
7242
|
+
* answers for the run and the run does not carry the tag yet. The first turn's suspend records the
|
|
7243
|
+
* run's root span this way, so a run that settles in a process that never saw its first turn still
|
|
7244
|
+
* links its subscribers' traces; a workflow nobody subscribes to gets no tag and no extra write.
|
|
7245
|
+
* Uses the last read of the fleet's announcements without waiting; with no usable read, any
|
|
7246
|
+
* workflow but an internal one may have subscribers, so it gets the tag.
|
|
7247
|
+
*/
|
|
7248
|
+
async withRunTrace(run, patch) {
|
|
7249
|
+
if (!this.runTraceparent || run.workflow.startsWith("__")) return patch;
|
|
7250
|
+
const tags = patch.tags ?? run.tags ?? [];
|
|
7251
|
+
if (tags.some((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX))) return patch;
|
|
7252
|
+
let traceparent;
|
|
7253
|
+
try {
|
|
7254
|
+
traceparent = this.runTraceparent(run.id);
|
|
7255
|
+
} catch {
|
|
7256
|
+
return patch;
|
|
7257
|
+
}
|
|
7258
|
+
if (!traceparent) return patch;
|
|
7259
|
+
const snapshot = this.cachedAnnouncedSubscriptions();
|
|
7260
|
+
const namespace = runNamespace(run.namespace);
|
|
7261
|
+
const subscribed = !snapshot?.ok || this.lifecycleSubscribersIn(snapshot, run.workflow, "completed", namespace).length > 0 || this.lifecycleSubscribersIn(snapshot, run.workflow, "failed", namespace).length > 0;
|
|
7262
|
+
if (!subscribed) return patch;
|
|
7263
|
+
return {
|
|
7264
|
+
...patch,
|
|
7265
|
+
tags: [
|
|
7266
|
+
...tags,
|
|
7267
|
+
`${LIFECYCLE_TRACE_TAG_PREFIX}${traceparent}`
|
|
7268
|
+
]
|
|
7269
|
+
};
|
|
7270
|
+
}
|
|
7271
|
+
/**
|
|
7272
|
+
* The runs started by `runId` settling (see {@link publishLifecycle}), read from the run's own
|
|
7273
|
+
* tags, so it needs no subscriber registry. Empty for a run that has not settled or started
|
|
7274
|
+
* nobody. Pass the run itself when it is already loaded, to skip reading it again.
|
|
7275
|
+
*/
|
|
7276
|
+
async triggeredRuns(source) {
|
|
7277
|
+
const run = typeof source === "string" ? await this.store.getRun(source) : source;
|
|
7278
|
+
return run ? triggeredRunsOf(run) : [];
|
|
7279
|
+
}
|
|
7280
|
+
/** The live path of {@link publishSettled}: right after this instance settled the run. */
|
|
7281
|
+
publishLifecycle(event) {
|
|
7282
|
+
if (event.type !== "run.completed" && event.type !== "run.failed") return;
|
|
7283
|
+
const kind = event.type === "run.completed" ? "completed" : "failed";
|
|
7284
|
+
if (!event.workflow || !this.hasLifecycleSubscribers(event.workflow, kind, event.namespace)) {
|
|
7285
|
+
return;
|
|
7286
|
+
}
|
|
7287
|
+
void (async () => {
|
|
7288
|
+
const run = await this.store.getRun(event.runId);
|
|
7289
|
+
if (run) await this.publishSettled(run);
|
|
7290
|
+
})().catch(() => void 0);
|
|
7291
|
+
}
|
|
7292
|
+
/** How long to wait before each new try at a run's lifecycle lease another publisher holds. */
|
|
7293
|
+
static LIFECYCLE_LEASE_BACKOFF_MS = [
|
|
7294
|
+
50,
|
|
7295
|
+
100,
|
|
7296
|
+
200,
|
|
7297
|
+
400,
|
|
7298
|
+
800
|
|
7299
|
+
];
|
|
7300
|
+
/**
|
|
7301
|
+
* Run `fn` holding the lease on the settled run `runId` (`tryLockRun`, owner
|
|
7302
|
+
* `<instanceId>#lifecycle#<uuid>`, unique per call), so the live publish, the sweeps of every
|
|
7303
|
+
* instance and a republish update its lifecycle tags one at a time. Retried on the
|
|
7304
|
+
* {@link LIFECYCLE_LEASE_BACKOFF_MS} schedule while another holder has it (the settling execution
|
|
7305
|
+
* still holding its own, or another publisher); `busy` once those tries are spent.
|
|
7306
|
+
*/
|
|
7307
|
+
async withLifecycleLease(runId, fn) {
|
|
7308
|
+
const owner = `${this.instanceId}#lifecycle#${globalThis.crypto.randomUUID()}`;
|
|
7309
|
+
for (let attempt = 0; ; attempt += 1) {
|
|
7310
|
+
const now = this.clock();
|
|
7311
|
+
if (await this.store.tryLockRun(runId, owner, now + this.leaseMs, now)) break;
|
|
7312
|
+
const wait = _WorkflowEngine.LIFECYCLE_LEASE_BACKOFF_MS[attempt];
|
|
7313
|
+
if (wait === void 0) return {
|
|
7314
|
+
busy: true
|
|
7315
|
+
};
|
|
7316
|
+
await new Promise((resolve) => setTimeout(resolve, wait));
|
|
7317
|
+
}
|
|
7318
|
+
try {
|
|
7319
|
+
return {
|
|
7320
|
+
busy: false,
|
|
7321
|
+
value: await fn()
|
|
7322
|
+
};
|
|
7323
|
+
} finally {
|
|
7324
|
+
await this.store.releaseRunLock(runId, owner).catch(() => void 0);
|
|
7325
|
+
}
|
|
7326
|
+
}
|
|
7327
|
+
/**
|
|
7328
|
+
* Publish a settled run's lifecycle event (`run.completed:<workflow>` / `run.failed:<workflow>`)
|
|
7329
|
+
* to the subscribers this engine has, and settle their part of the marker: drop each one's owed
|
|
7330
|
+
* tag, record each subscriber it started as a `durable:triggered:<workflow>` tag and each one it
|
|
7331
|
+
* routed into a debounce/batch accumulator as a `durable:coalesced:<workflow>` tag, and drop
|
|
7332
|
+
* {@link LIFECYCLE_PENDING_TAG} once no owed tag is left. Subscribers registered only elsewhere
|
|
7333
|
+
* keep their owed tags for the instances that have them.
|
|
7334
|
+
*
|
|
7335
|
+
* Which subscribers: `only` when given (a republish), plus `owe`, owed to the instances that have
|
|
7336
|
+
* them; `heal`, the ones a heal pass found missing (see {@link healLifecycleEvents}); else the owed
|
|
7337
|
+
* ones. A run marked with the pending tag alone is unresolved: with a good read of the fleet's
|
|
7338
|
+
* announcements it is owed to every subscriber (see {@link lifecycleSubscribers}) that has not
|
|
7339
|
+
* started, coalesced or failed, and published as such; without one, this engine starts its own subscribers
|
|
7340
|
+
* that do not coalesce and the run stays unresolved.
|
|
7341
|
+
*
|
|
7342
|
+
* Runs under the run's lifecycle lease (see {@link withLifecycleLease}), reading the run again
|
|
7343
|
+
* inside it; `busy` when the lease could not be taken. Idempotent, since every subscriber run id is
|
|
7344
|
+
* deterministic and a debounce/batch accumulator counts the event once, so the live path, the
|
|
7345
|
+
* sweeps and a republish may all reach the same run. `run.triggered` and `run.trigger_failed` are
|
|
7346
|
+
* emitted by the publish whose write recorded them: a subscriber it reached whose triggered or
|
|
7347
|
+
* coalesced tag the run did not have, and one it failed to start whose failed tag the run did not
|
|
7348
|
+
* have.
|
|
7349
|
+
*
|
|
7350
|
+
* A subscriber whose `where` throws or whose start fails is reported as `run.trigger_failed`,
|
|
7351
|
+
* tagged {@link LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX} in the same update, and not retried; a
|
|
7352
|
+
* subscriber it starts loses that tag. A failure to write the marker leaves it, and the sweep
|
|
7353
|
+
* publishes the run again. Returns the subscribers acted on, or undefined when there was nothing
|
|
7354
|
+
* for this engine to do.
|
|
7355
|
+
*/
|
|
7356
|
+
async publishSettled(source, opts = {}) {
|
|
7357
|
+
if (!lifecycleKindOf(source.status)) return void 0;
|
|
7358
|
+
const directed = opts.only !== void 0 || opts.heal !== void 0 || (opts.owe?.length ?? 0) > 0;
|
|
7359
|
+
const tags = source.tags ?? [];
|
|
7360
|
+
const marked = tags.includes(LIFECYCLE_PENDING_TAG) || tags.some((t) => t.startsWith(LIFECYCLE_OWED_TAG_PREFIX));
|
|
7361
|
+
if (!directed && !marked) return void 0;
|
|
7362
|
+
const leased = await this.withLifecycleLease(source.id, () => this.publishSettledLocked(source.id, opts));
|
|
7363
|
+
return leased.busy ? "busy" : leased.value;
|
|
7364
|
+
}
|
|
7365
|
+
/** {@link publishSettled}, under the run's lifecycle lease. */
|
|
7366
|
+
async publishSettledLocked(runId, opts) {
|
|
7367
|
+
const run = await this.store.getRun(runId);
|
|
7368
|
+
const kind = lifecycleKindOf(run?.status);
|
|
7369
|
+
if (!run || !kind) return void 0;
|
|
7370
|
+
const name = lifecycleEventName(run.workflow, kind);
|
|
7371
|
+
const tags = run.tags ?? [];
|
|
7372
|
+
const owed = owedSubscribersOf(tags);
|
|
7373
|
+
const unresolved = tags.includes(LIFECYCLE_PENDING_TAG) && owed.length === 0;
|
|
7374
|
+
const done = /* @__PURE__ */ new Set([
|
|
7375
|
+
...triggeredSubscribersOf(tags),
|
|
7376
|
+
...coalescedSubscribersOf(tags),
|
|
7377
|
+
...triggerFailedSubscribersOf(tags)
|
|
7378
|
+
]);
|
|
7379
|
+
const local = this.localSubscribersIn(name, run.namespace);
|
|
7380
|
+
const remote = await this.announcedRemoteSubscribers(name, run.namespace);
|
|
7381
|
+
const startable = /* @__PURE__ */ new Set([
|
|
7382
|
+
...local,
|
|
7383
|
+
...remote
|
|
7384
|
+
]);
|
|
7385
|
+
const owe = new Set(opts.owe ?? []);
|
|
7386
|
+
let targets;
|
|
7387
|
+
let resolved = false;
|
|
7388
|
+
if (opts.only !== void 0) {
|
|
7389
|
+
const only = opts.only;
|
|
7390
|
+
targets = [
|
|
7391
|
+
...startable
|
|
7392
|
+
].filter((s) => only.has(s));
|
|
7393
|
+
} else if (opts.heal !== void 0) {
|
|
7394
|
+
targets = opts.heal.filter((s) => startable.has(s));
|
|
7395
|
+
for (const s of opts.heal) if (!startable.has(s)) owe.add(s);
|
|
7396
|
+
} else if (unresolved) {
|
|
7397
|
+
const fleet = await this.announcedSubscriptions();
|
|
7398
|
+
if (fleet.ok) {
|
|
7399
|
+
resolved = true;
|
|
7400
|
+
const full = this.lifecycleSubscribersIn(fleet, run.workflow, kind, run.namespace).filter((s) => !done.has(s));
|
|
7401
|
+
targets = full.filter((s) => startable.has(s));
|
|
7402
|
+
for (const s of full) if (!startable.has(s)) owe.add(s);
|
|
7403
|
+
} else {
|
|
7404
|
+
targets = local.filter((s) => !done.has(s) && !this.latest.get(s)?.eventBatch);
|
|
7405
|
+
}
|
|
7406
|
+
} else {
|
|
7407
|
+
targets = owed.filter((s) => startable.has(s));
|
|
7408
|
+
}
|
|
7409
|
+
if (targets.length === 0 && owe.size === 0 && !resolved) return void 0;
|
|
7410
|
+
const runIdOf = /* @__PURE__ */ __name((sub2) => lifecycleRunId(run.id, kind, sub2), "runIdOf");
|
|
7411
|
+
const { reached, failed } = targets.length > 0 ? await this.startSubscribers(name, lifecycleEventOf(run), runIdOf, new Set(targets), remote, runIdOf) : {
|
|
7412
|
+
reached: [],
|
|
7413
|
+
failed: []
|
|
7414
|
+
};
|
|
7415
|
+
const current = await this.store.getRun(run.id) ?? run;
|
|
7416
|
+
const before = current.tags ?? [];
|
|
7417
|
+
const dropped = /* @__PURE__ */ new Set([
|
|
7418
|
+
...targets.map((s) => `${LIFECYCLE_OWED_TAG_PREFIX}${s}`),
|
|
7419
|
+
...reached.map((r) => `${LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX}${r.workflow}`)
|
|
7420
|
+
]);
|
|
7421
|
+
const deliveredTag = /* @__PURE__ */ __name((r) => r.runId !== void 0 ? `${LIFECYCLE_TRIGGERED_TAG_PREFIX}${r.workflow}` : `${LIFECYCLE_COALESCED_TAG_PREFIX}${r.workflow}`, "deliveredTag");
|
|
7422
|
+
const added = [
|
|
7423
|
+
...reached.map(deliveredTag),
|
|
7424
|
+
...failed.map((f) => `${LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX}${f.workflow}`),
|
|
7425
|
+
...[
|
|
7426
|
+
...owe
|
|
7427
|
+
].map((s) => `${LIFECYCLE_OWED_TAG_PREFIX}${s}`)
|
|
7428
|
+
];
|
|
7429
|
+
const kept = before.filter((t) => !dropped.has(t));
|
|
7430
|
+
const keepPending = [
|
|
7431
|
+
...kept,
|
|
7432
|
+
...added
|
|
7433
|
+
].some((t) => t.startsWith(LIFECYCLE_OWED_TAG_PREFIX)) || unresolved && !resolved;
|
|
7434
|
+
const marker = keepPending ? kept.includes(LIFECYCLE_PENDING_TAG) ? kept : [
|
|
7435
|
+
...kept,
|
|
7436
|
+
LIFECYCLE_PENDING_TAG
|
|
7437
|
+
] : kept.filter((t) => t !== LIFECYCLE_PENDING_TAG);
|
|
7438
|
+
const tagsAfter = [
|
|
7439
|
+
.../* @__PURE__ */ new Set([
|
|
7440
|
+
...marker,
|
|
7441
|
+
...added
|
|
7442
|
+
])
|
|
7443
|
+
];
|
|
7444
|
+
if (tagsAfter.length !== before.length || tagsAfter.some((t) => !before.includes(t))) {
|
|
7445
|
+
await this.store.updateRun(run.id, {
|
|
7446
|
+
tags: tagsAfter
|
|
7447
|
+
});
|
|
7448
|
+
}
|
|
7449
|
+
const had = new Set(before);
|
|
7450
|
+
for (const r of reached) {
|
|
7451
|
+
if (had.has(deliveredTag(r))) continue;
|
|
7452
|
+
const { workflow, runId: triggeredRunId } = r;
|
|
7453
|
+
this.emit({
|
|
7454
|
+
type: "run.triggered",
|
|
7455
|
+
runId: run.id,
|
|
7456
|
+
workflow: run.workflow,
|
|
7457
|
+
namespace: run.namespace,
|
|
7458
|
+
name: workflow,
|
|
7459
|
+
triggeredRunId
|
|
7460
|
+
});
|
|
7461
|
+
}
|
|
7462
|
+
for (const { workflow, error: error2 } of failed) {
|
|
7463
|
+
if (had.has(`${LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX}${workflow}`)) continue;
|
|
7464
|
+
this.reportTriggerFailed(run, workflow, error2 instanceof Error ? error2.message : String(error2));
|
|
7465
|
+
}
|
|
7466
|
+
return {
|
|
7467
|
+
reached,
|
|
7468
|
+
failed
|
|
7469
|
+
};
|
|
7470
|
+
}
|
|
7471
|
+
reportTriggerFailed(run, subscriber, message) {
|
|
7472
|
+
this.emit({
|
|
7473
|
+
type: "run.trigger_failed",
|
|
7474
|
+
runId: run.id,
|
|
7475
|
+
workflow: run.workflow,
|
|
7476
|
+
namespace: run.namespace,
|
|
7477
|
+
name: subscriber,
|
|
7478
|
+
error: {
|
|
7479
|
+
message
|
|
7480
|
+
}
|
|
7481
|
+
});
|
|
7482
|
+
}
|
|
7483
|
+
/**
|
|
7484
|
+
* Drop the owed tags of `run`'s subscribers that no live worker announces and this engine does
|
|
7485
|
+
* not have, once the run settled more than {@link LIFECYCLE_ORPHAN_MS} ago, reporting each as
|
|
7486
|
+
* `run.trigger_failed` and tagging it {@link LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX} in the same
|
|
7487
|
+
* update, under the run's lifecycle lease. Skipped when the fleet's announcements could not be
|
|
7488
|
+
* read, since a failed read would make every remote subscriber look orphaned.
|
|
7489
|
+
*/
|
|
7490
|
+
async dropOrphanedSubscribers(run, nowMs) {
|
|
7491
|
+
if (nowMs - run.updatedAt.getTime() < LIFECYCLE_ORPHAN_MS) return false;
|
|
7492
|
+
const kind = lifecycleKindOf(run.status);
|
|
7493
|
+
if (!kind || owedSubscribersOf(run.tags).length === 0) return false;
|
|
7494
|
+
const fleet = await this.announcedSubscriptions();
|
|
7495
|
+
if (!fleet.ok) return false;
|
|
7496
|
+
const leased = await this.withLifecycleLease(run.id, async () => {
|
|
7497
|
+
const current = await this.store.getRun(run.id);
|
|
7498
|
+
const owed = owedSubscribersOf(current?.tags);
|
|
7499
|
+
if (!current || owed.length === 0) return false;
|
|
7500
|
+
const event = lifecycleEventName(run.workflow, kind);
|
|
7501
|
+
const live = /* @__PURE__ */ new Set([
|
|
7502
|
+
...this.localSubscribersIn(event, run.namespace),
|
|
7503
|
+
...announcedIn(fleet, event, run.namespace)
|
|
7504
|
+
]);
|
|
7505
|
+
const orphans = owed.filter((s) => !live.has(s));
|
|
7506
|
+
if (orphans.length === 0) return false;
|
|
7507
|
+
const dropped = new Set(orphans.map((s) => `${LIFECYCLE_OWED_TAG_PREFIX}${s}`));
|
|
7508
|
+
const kept = (current.tags ?? []).filter((t) => !dropped.has(t));
|
|
7509
|
+
const owedLeft = kept.some((t) => t.startsWith(LIFECYCLE_OWED_TAG_PREFIX));
|
|
7510
|
+
await this.store.updateRun(run.id, {
|
|
7511
|
+
tags: [
|
|
7512
|
+
.../* @__PURE__ */ new Set([
|
|
7513
|
+
...kept.filter((t) => owedLeft || t !== LIFECYCLE_PENDING_TAG),
|
|
7514
|
+
...orphans.map((s) => `${LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX}${s}`)
|
|
7515
|
+
])
|
|
7516
|
+
]
|
|
7517
|
+
});
|
|
7518
|
+
for (const subscriber of orphans) {
|
|
7519
|
+
this.reportTriggerFailed(current, subscriber, `subscriber ${subscriber} is not registered on any live worker`);
|
|
7520
|
+
}
|
|
7521
|
+
return true;
|
|
7522
|
+
});
|
|
7523
|
+
return !leased.busy && leased.value;
|
|
7524
|
+
}
|
|
7525
|
+
/** How long a settled run's marker is left to the live publish before the sweep takes it over. */
|
|
7526
|
+
static LIFECYCLE_SWEEP_GRACE_MS = 3e4;
|
|
7527
|
+
/**
|
|
7528
|
+
* How many marked runs (and `ctx.waitForRun` waiters) one sweep examines at most, looking for
|
|
7529
|
+
* `limit` it can act on. Runs owed only to subscribers other instances have are skipped without a
|
|
7530
|
+
* store read, so they cannot fill the window and keep this instance from the ones it can publish.
|
|
7531
|
+
*/
|
|
7532
|
+
static LIFECYCLE_SWEEP_SCAN_MAX = 1e3;
|
|
7533
|
+
/** How many recently settled runs of one workflow a heal pass reads at most. */
|
|
7534
|
+
static LIFECYCLE_HEAL_SCAN_CAP = 100;
|
|
7535
|
+
/**
|
|
7536
|
+
* Publish the lifecycle event of every settled run still marked {@link LIFECYCLE_PENDING_TAG}:
|
|
7537
|
+
* one whose process died, or whose marker write failed, between settling it and publishing, one
|
|
7538
|
+
* owed to a subscriber the settling process does not have, or one whose owed set is unresolved.
|
|
7539
|
+
* Each instance publishes to the subscribers it has, and drops owed subscribers nobody has anymore
|
|
7540
|
+
* (see {@link dropOrphanedSubscribers}). Then heals the recently settled runs that missed a newly
|
|
7541
|
+
* announced subscriber (see {@link healLifecycleEvents}). Also delivers the `ctx.waitForRun` waits
|
|
7542
|
+
* whose run already settled (see {@link sweepRunWaits}). Called by the timer poller; a run settled
|
|
7543
|
+
* in the last {@link LIFECYCLE_SWEEP_GRACE_MS} is left to the live publish. Returns how many runs
|
|
7544
|
+
* it published to a subscriber.
|
|
6059
7545
|
*
|
|
6060
|
-
*
|
|
6061
|
-
*
|
|
6062
|
-
*
|
|
6063
|
-
*
|
|
6064
|
-
*
|
|
6065
|
-
|
|
6066
|
-
|
|
6067
|
-
|
|
6068
|
-
|
|
6069
|
-
|
|
6070
|
-
|
|
6071
|
-
|
|
6072
|
-
|
|
6073
|
-
|
|
6074
|
-
|
|
6075
|
-
|
|
6076
|
-
|
|
6077
|
-
|
|
6078
|
-
|
|
6079
|
-
|
|
6080
|
-
|
|
6081
|
-
|
|
6082
|
-
|
|
6083
|
-
|
|
6084
|
-
|
|
6085
|
-
|
|
6086
|
-
|
|
6087
|
-
|
|
6088
|
-
|
|
6089
|
-
|
|
6090
|
-
|
|
6091
|
-
|
|
6092
|
-
|
|
6093
|
-
|
|
6094
|
-
|
|
6095
|
-
|
|
6096
|
-
if (
|
|
6097
|
-
|
|
6098
|
-
|
|
6099
|
-
|
|
7546
|
+
* Reads the marked runs a page of `limit` at a time, skipping the ones it cannot act on, until it
|
|
7547
|
+
* has acted on `limit` runs (published, or dropped orphans) or examined
|
|
7548
|
+
* {@link LIFECYCLE_SWEEP_SCAN_MAX}. Pages advance by offset over a set the sweep itself shrinks,
|
|
7549
|
+
* so a run that moves into an earlier page is reached on the next tick, which starts over. A run
|
|
7550
|
+
* whose lifecycle lease another publisher holds is left for the next tick.
|
|
7551
|
+
*/
|
|
7552
|
+
async sweepLifecycleEvents(nowMs = this.clock(), limit = 100) {
|
|
7553
|
+
await this.heartbeatLifecycleRegistry().catch((error2) => {
|
|
7554
|
+
console.error("[nestjs-durable] lifecycle registry heartbeat failed; retrying next tick:", error2);
|
|
7555
|
+
});
|
|
7556
|
+
await this.sweepRunWaits(limit).catch((error2) => {
|
|
7557
|
+
console.error("[nestjs-durable] run-wait sweep failed; retrying next tick:", error2);
|
|
7558
|
+
});
|
|
7559
|
+
await this.announcedSubscriptions();
|
|
7560
|
+
const pageSize = Math.max(1, limit);
|
|
7561
|
+
const scanMax = Math.max(pageSize, _WorkflowEngine.LIFECYCLE_SWEEP_SCAN_MAX);
|
|
7562
|
+
let published = 0;
|
|
7563
|
+
let acted = 0;
|
|
7564
|
+
let examined = 0;
|
|
7565
|
+
while (acted < limit && examined < scanMax) {
|
|
7566
|
+
const requested = Math.min(pageSize, scanMax - examined);
|
|
7567
|
+
const page = await this.store.listRuns({
|
|
7568
|
+
tag: LIFECYCLE_PENDING_TAG,
|
|
7569
|
+
statuses: [
|
|
7570
|
+
"completed",
|
|
7571
|
+
"failed",
|
|
7572
|
+
"cancelled",
|
|
7573
|
+
"dead"
|
|
7574
|
+
],
|
|
7575
|
+
...this.namespace !== void 0 ? {
|
|
7576
|
+
namespace: this.namespace
|
|
7577
|
+
} : {},
|
|
7578
|
+
limit: requested,
|
|
7579
|
+
offset: examined
|
|
7580
|
+
});
|
|
7581
|
+
for (const run of page) {
|
|
7582
|
+
if (acted >= limit) break;
|
|
7583
|
+
examined += 1;
|
|
7584
|
+
if (nowMs - run.updatedAt.getTime() < _WorkflowEngine.LIFECYCLE_SWEEP_GRACE_MS) continue;
|
|
7585
|
+
try {
|
|
7586
|
+
const outcome = await this.publishSettled(run);
|
|
7587
|
+
if (outcome === "busy") continue;
|
|
7588
|
+
if (outcome && outcome.reached.length + outcome.failed.length > 0) published += 1;
|
|
7589
|
+
const dropped = await this.dropOrphanedSubscribers(run, nowMs);
|
|
7590
|
+
if (outcome || dropped) acted += 1;
|
|
7591
|
+
} catch (error2) {
|
|
7592
|
+
console.error(`[nestjs-durable] lifecycle sweep: run ${run.id} failed; retrying next tick:`, error2);
|
|
6100
7593
|
}
|
|
6101
|
-
this.bufferedSweepCursor = last.id;
|
|
6102
7594
|
}
|
|
6103
|
-
|
|
6104
|
-
this.bufferedSweepActive = false;
|
|
7595
|
+
if (page.length < requested) break;
|
|
6105
7596
|
}
|
|
6106
|
-
|
|
7597
|
+
try {
|
|
7598
|
+
published += await this.healLifecycleEvents(nowMs);
|
|
7599
|
+
} catch (error2) {
|
|
7600
|
+
console.error("[nestjs-durable] lifecycle heal failed; retrying next tick:", error2);
|
|
7601
|
+
}
|
|
7602
|
+
return published;
|
|
6107
7603
|
}
|
|
6108
7604
|
/**
|
|
6109
|
-
*
|
|
6110
|
-
*
|
|
6111
|
-
* old pending remote step with no live lease (dispatched and, most likely, lost). The timer poller
|
|
6112
|
-
* sweeps for these ({@link sweepStalled}, throttled to one pass a minute) and notifies each episode
|
|
6113
|
-
* ONCE per instance: a run is paged again only after it makes progress and strands again. Wire your
|
|
6114
|
-
* alerting here like {@link onDead}. A listener that throws never breaks the sweep. Returns an
|
|
6115
|
-
* unsubscribe function.
|
|
7605
|
+
* The (run, subscriber) pairs this engine already healed, until when it remembers them; see
|
|
7606
|
+
* {@link healLifecycleEvents}.
|
|
6116
7607
|
*/
|
|
6117
|
-
|
|
6118
|
-
this.stalledListeners.add(listener);
|
|
6119
|
-
return () => this.stalledListeners.delete(listener);
|
|
6120
|
-
}
|
|
7608
|
+
healedLifecycle = /* @__PURE__ */ new Map();
|
|
6121
7609
|
/**
|
|
6122
|
-
*
|
|
6123
|
-
*
|
|
6124
|
-
*
|
|
6125
|
-
*
|
|
6126
|
-
|
|
6127
|
-
|
|
6128
|
-
|
|
6129
|
-
|
|
6130
|
-
|
|
6131
|
-
|
|
6132
|
-
|
|
6133
|
-
|
|
6134
|
-
|
|
6135
|
-
|
|
6136
|
-
|
|
6137
|
-
|
|
6138
|
-
|
|
6139
|
-
|
|
7610
|
+
* Heal the runs settled in the last {@link LIFECYCLE_HEAL_WINDOW_MS} that missed a subscriber: one
|
|
7611
|
+
* the fleet announced only after the settling process last read the announcements, so it is in
|
|
7612
|
+
* none of the run's owed, triggered, coalesced or failed tags. Only subscriptions this engine first saw within
|
|
7613
|
+
* that window are checked (see {@link subscriptionFirstSeen}), since one the fleet already
|
|
7614
|
+
* announced when the run settled was in its owed set. Needs a good read of the fleet's
|
|
7615
|
+
* announcements.
|
|
7616
|
+
*
|
|
7617
|
+
* One `listRuns` per source workflow (`updatedAfter` the window, at most
|
|
7618
|
+
* {@link LIFECYCLE_HEAL_SCAN_CAP} runs), skipping unresolved runs (the sweep resolves those) and
|
|
7619
|
+
* the ones in their {@link LIFECYCLE_SWEEP_GRACE_MS}. Each missed subscriber is published like an
|
|
7620
|
+
* owed one (see {@link publishSettled}): started here when this engine can, owed to the instances
|
|
7621
|
+
* that have it otherwise. A subscriber whose `where` declines the event leaves no tag, so each
|
|
7622
|
+
* engine heals a pair once. Returns how many runs it published to a subscriber.
|
|
7623
|
+
*/
|
|
7624
|
+
async healLifecycleEvents(nowMs) {
|
|
7625
|
+
const fleet = await this.announcedSubscriptions();
|
|
7626
|
+
if (!fleet.ok) return 0;
|
|
7627
|
+
for (const [key, until] of this.healedLifecycle) {
|
|
7628
|
+
if (until <= nowMs) this.healedLifecycle.delete(key);
|
|
7629
|
+
}
|
|
7630
|
+
const windowStart = nowMs - LIFECYCLE_HEAL_WINDOW_MS;
|
|
7631
|
+
const sources = /* @__PURE__ */ new Map();
|
|
7632
|
+
for (const [key, firstSeen] of this.subscriptionFirstSeen) {
|
|
7633
|
+
if (firstSeen < windowStart) continue;
|
|
7634
|
+
const source = parseLifecycleEvent(key.slice(0, key.indexOf("\0")));
|
|
7635
|
+
if (!source || source.workflow.startsWith("__")) continue;
|
|
7636
|
+
const kinds = sources.get(source.workflow) ?? /* @__PURE__ */ new Set();
|
|
7637
|
+
kinds.add(source.kind);
|
|
7638
|
+
sources.set(source.workflow, kinds);
|
|
7639
|
+
}
|
|
7640
|
+
let published = 0;
|
|
7641
|
+
for (const [workflow, kinds] of sources) {
|
|
7642
|
+
const statuses = [];
|
|
7643
|
+
if (kinds.has("completed")) statuses.push("completed");
|
|
7644
|
+
if (kinds.has("failed")) statuses.push("failed", "cancelled", "dead");
|
|
7645
|
+
const runs = await this.store.listRuns({
|
|
7646
|
+
workflow,
|
|
7647
|
+
statuses,
|
|
7648
|
+
updatedAfter: windowStart,
|
|
7649
|
+
...this.namespace !== void 0 ? {
|
|
7650
|
+
namespace: this.namespace
|
|
7651
|
+
} : {},
|
|
7652
|
+
limit: _WorkflowEngine.LIFECYCLE_HEAL_SCAN_CAP
|
|
7653
|
+
});
|
|
7654
|
+
for (const run of runs) {
|
|
7655
|
+
const kind = lifecycleKindOf(run.status);
|
|
7656
|
+
if (!kind) continue;
|
|
7657
|
+
if (nowMs - run.updatedAt.getTime() < _WorkflowEngine.LIFECYCLE_SWEEP_GRACE_MS) continue;
|
|
7658
|
+
const tags = run.tags ?? [];
|
|
7659
|
+
const owed = owedSubscribersOf(tags);
|
|
7660
|
+
if (tags.includes(LIFECYCLE_PENDING_TAG) && owed.length === 0) continue;
|
|
7661
|
+
const event = lifecycleEventName(workflow, kind);
|
|
7662
|
+
const known = /* @__PURE__ */ new Set([
|
|
7663
|
+
...owed,
|
|
7664
|
+
...triggeredSubscribersOf(tags),
|
|
7665
|
+
...coalescedSubscribersOf(tags),
|
|
7666
|
+
...triggerFailedSubscribersOf(tags)
|
|
7667
|
+
]);
|
|
7668
|
+
const missing = this.lifecycleSubscribersIn(fleet, workflow, kind, run.namespace).filter((s) => {
|
|
7669
|
+
if (known.has(s) || this.healedLifecycle.has(`${run.id}\0${s}`)) return false;
|
|
7670
|
+
const firstSeen = this.subscriptionFirstSeen.get(`${event}\0${s}`);
|
|
7671
|
+
return firstSeen !== void 0 && firstSeen >= windowStart;
|
|
7672
|
+
});
|
|
7673
|
+
if (missing.length === 0) continue;
|
|
6140
7674
|
try {
|
|
6141
|
-
|
|
6142
|
-
|
|
7675
|
+
const outcome = await this.publishSettled(run, {
|
|
7676
|
+
heal: missing
|
|
7677
|
+
});
|
|
7678
|
+
if (outcome === "busy") continue;
|
|
7679
|
+
for (const s of missing) {
|
|
7680
|
+
this.healedLifecycle.set(`${run.id}\0${s}`, run.updatedAt.getTime() + LIFECYCLE_HEAL_WINDOW_MS);
|
|
7681
|
+
}
|
|
7682
|
+
if (outcome && outcome.reached.length + outcome.failed.length > 0) published += 1;
|
|
7683
|
+
} catch (error2) {
|
|
7684
|
+
console.error(`[nestjs-durable] lifecycle heal: run ${run.id} failed; retrying next tick:`, error2);
|
|
6143
7685
|
}
|
|
6144
7686
|
}
|
|
6145
|
-
notified.push(info2);
|
|
6146
7687
|
}
|
|
6147
|
-
return
|
|
7688
|
+
return published;
|
|
6148
7689
|
}
|
|
6149
7690
|
/**
|
|
6150
|
-
*
|
|
6151
|
-
* the
|
|
6152
|
-
*
|
|
6153
|
-
*
|
|
6154
|
-
*
|
|
6155
|
-
*
|
|
6156
|
-
*
|
|
6157
|
-
|
|
6158
|
-
|
|
6159
|
-
|
|
6160
|
-
|
|
6161
|
-
|
|
6162
|
-
|
|
6163
|
-
|
|
6164
|
-
|
|
7691
|
+
* Publish the lifecycle events of settled runs again, to subscribers that missed them: one
|
|
7692
|
+
* registered after the runs settled, or one whose run was deleted. Unlike the sweep it needs no
|
|
7693
|
+
* marker. Each subscriber's run id is deterministic, so one already started is not started again.
|
|
7694
|
+
*
|
|
7695
|
+
* Subscribers registered on this engine are started here. A subscriber only live workers announce
|
|
7696
|
+
* is owed the event again (the run is re-marked), so the sweep on an instance that has it starts
|
|
7697
|
+
* it. Only subscribers of the run's lifecycle event in its namespace are published to; a
|
|
7698
|
+
* `subscriber` that is not one is reported in `failed`. Reads at most `limit` (a positive integer,
|
|
7699
|
+
* default 1000) settled runs of a workflow, a page at a time.
|
|
7700
|
+
*/
|
|
7701
|
+
async republishLifecycle(target) {
|
|
7702
|
+
const result = {
|
|
7703
|
+
runs: 0,
|
|
7704
|
+
started: 0,
|
|
7705
|
+
failed: []
|
|
7706
|
+
};
|
|
7707
|
+
if ("runId" in target) {
|
|
7708
|
+
const run = await this.store.getRun(target.runId);
|
|
7709
|
+
if (!run) throw new FatalError(`run ${target.runId} not found`);
|
|
7710
|
+
if (!lifecycleKindOf(run.status)) {
|
|
7711
|
+
throw new FatalError(`run ${target.runId} has not settled (${run.status})`);
|
|
7712
|
+
}
|
|
7713
|
+
await this.republishRun(run, target.subscriber, result);
|
|
7714
|
+
return result;
|
|
7715
|
+
}
|
|
7716
|
+
const statuses = target.kind === "completed" ? [
|
|
7717
|
+
"completed"
|
|
7718
|
+
] : target.kind === "failed" ? [
|
|
7719
|
+
"failed",
|
|
7720
|
+
"cancelled",
|
|
7721
|
+
"dead"
|
|
7722
|
+
] : [
|
|
7723
|
+
"completed",
|
|
7724
|
+
"failed",
|
|
7725
|
+
"cancelled",
|
|
7726
|
+
"dead"
|
|
7727
|
+
];
|
|
7728
|
+
if (target.namespace !== void 0 && this.namespace !== void 0 && target.namespace !== this.namespace) {
|
|
7729
|
+
throw new FatalError(`namespace ${target.namespace} is not this engine's namespace (${this.namespace})`);
|
|
7730
|
+
}
|
|
7731
|
+
const limit = target.limit ?? 1e3;
|
|
7732
|
+
if (!Number.isSafeInteger(limit) || limit < 1) {
|
|
7733
|
+
throw new FatalError(`limit must be a positive integer, got ${String(target.limit)}`);
|
|
7734
|
+
}
|
|
7735
|
+
const namespace = target.namespace ?? this.namespace;
|
|
7736
|
+
const pageSize = Math.min(limit, 200);
|
|
7737
|
+
let read = 0;
|
|
7738
|
+
while (read < limit) {
|
|
7739
|
+
const want = Math.min(pageSize, limit - read);
|
|
7740
|
+
const page = await this.store.listRuns({
|
|
7741
|
+
workflow: target.workflow,
|
|
7742
|
+
statuses,
|
|
7743
|
+
...namespace !== void 0 ? {
|
|
7744
|
+
namespace
|
|
7745
|
+
} : {},
|
|
7746
|
+
limit: want,
|
|
7747
|
+
offset: read
|
|
7748
|
+
});
|
|
7749
|
+
for (const run of page) {
|
|
7750
|
+
const settledAt = run.updatedAt.getTime();
|
|
7751
|
+
if (target.since && settledAt < target.since.getTime()) continue;
|
|
7752
|
+
if (target.until && settledAt > target.until.getTime()) continue;
|
|
7753
|
+
await this.republishRun(run, target.subscriber, result);
|
|
7754
|
+
}
|
|
7755
|
+
read += page.length;
|
|
7756
|
+
if (page.length < want) break;
|
|
7757
|
+
}
|
|
7758
|
+
return result;
|
|
6165
7759
|
}
|
|
6166
|
-
|
|
6167
|
-
|
|
6168
|
-
|
|
6169
|
-
|
|
6170
|
-
|
|
7760
|
+
/** {@link republishLifecycle} for one settled run: start what is missing, report what failed. */
|
|
7761
|
+
async republishRun(run, subscriber, result) {
|
|
7762
|
+
const kind = lifecycleKindOf(run.status);
|
|
7763
|
+
if (!kind) return;
|
|
7764
|
+
const subscribers = (await this.lifecycleSubscribers(run.workflow, kind, runNamespace(run.namespace))).filter((s) => subscriber === void 0 || s === subscriber);
|
|
7765
|
+
if (subscriber !== void 0 && subscribers.length === 0) {
|
|
7766
|
+
result.runs += 1;
|
|
7767
|
+
result.failed.push({
|
|
7768
|
+
runId: run.id,
|
|
7769
|
+
subscriber,
|
|
7770
|
+
error: `subscriber ${subscriber} does not subscribe to ${lifecycleEventName(run.workflow, kind)}`
|
|
7771
|
+
});
|
|
7772
|
+
return;
|
|
7773
|
+
}
|
|
7774
|
+
if (subscribers.length === 0) return;
|
|
7775
|
+
result.runs += 1;
|
|
7776
|
+
const event = lifecycleEventName(run.workflow, kind);
|
|
7777
|
+
const local = /* @__PURE__ */ new Set([
|
|
7778
|
+
...this.localSubscribersIn(event, run.namespace),
|
|
7779
|
+
...await this.announcedRemoteSubscribers(event, run.namespace)
|
|
7780
|
+
]);
|
|
7781
|
+
const triggerFailed = triggerFailedSubscribersOf(run.tags);
|
|
7782
|
+
const coalesced = coalescedSubscribersOf(run.tags);
|
|
7783
|
+
const missing = [];
|
|
7784
|
+
for (const s of subscribers) {
|
|
7785
|
+
if (coalesced.includes(s) && !triggerFailed.includes(s)) continue;
|
|
7786
|
+
if (local.has(s) && this.latest.get(s)?.eventBatch !== void 0) {
|
|
7787
|
+
missing.push(s);
|
|
7788
|
+
continue;
|
|
6171
7789
|
}
|
|
7790
|
+
if (!await this.store.getRun(lifecycleRunId(run.id, kind, s))) missing.push(s);
|
|
7791
|
+
}
|
|
7792
|
+
const toStart = new Set(missing.filter((s) => local.has(s)));
|
|
7793
|
+
const toOwe = missing.filter((s) => !local.has(s));
|
|
7794
|
+
if (toStart.size === 0 && toOwe.length === 0) return;
|
|
7795
|
+
const outcome = await this.publishSettled(run, {
|
|
7796
|
+
only: toStart,
|
|
7797
|
+
owe: toOwe
|
|
7798
|
+
});
|
|
7799
|
+
if (outcome === "busy") {
|
|
7800
|
+
result.failed.push({
|
|
7801
|
+
runId: run.id,
|
|
7802
|
+
subscriber: "*",
|
|
7803
|
+
error: `run ${run.id} is being published elsewhere; try again`
|
|
7804
|
+
});
|
|
7805
|
+
return;
|
|
7806
|
+
}
|
|
7807
|
+
result.started += outcome?.reached.length ?? 0;
|
|
7808
|
+
for (const { workflow, error: error2 } of outcome?.failed ?? []) {
|
|
7809
|
+
result.failed.push({
|
|
7810
|
+
runId: run.id,
|
|
7811
|
+
subscriber: workflow,
|
|
7812
|
+
error: error2 instanceof Error ? error2.message : String(error2)
|
|
7813
|
+
});
|
|
6172
7814
|
}
|
|
6173
7815
|
}
|
|
6174
|
-
/**
|
|
6175
|
-
*
|
|
6176
|
-
|
|
6177
|
-
|
|
6178
|
-
|
|
6179
|
-
|
|
6180
|
-
|
|
6181
|
-
|
|
6182
|
-
|
|
6183
|
-
|
|
6184
|
-
|
|
7816
|
+
/**
|
|
7817
|
+
* Resume the runs waiting on `targetRunId` with `ctx.waitForRun`, once it has settled. Each waiter
|
|
7818
|
+
* is taken before it is delivered to, so the settle path, the sweep and the waiting run's own
|
|
7819
|
+
* check never resume it twice. Returns how many it resumed.
|
|
7820
|
+
*/
|
|
7821
|
+
async deliverRunWaits(targetRunId) {
|
|
7822
|
+
const waiters = (await this.store.listSignalWaiters(runWaitPrefix(targetRunId))).filter((w) => runWaitTargetOf(w) === targetRunId);
|
|
7823
|
+
if (waiters.length === 0) return 0;
|
|
7824
|
+
const target = await settledRunOf(this.store, targetRunId);
|
|
7825
|
+
if (target.state !== "settled") return 0;
|
|
7826
|
+
const payload = lifecycleEventOf(target.run);
|
|
7827
|
+
let delivered = 0;
|
|
7828
|
+
for (const w of waiters) {
|
|
7829
|
+
const waiter = await this.store.takeSignalWaiter(w.token);
|
|
7830
|
+
if (!waiter) continue;
|
|
7831
|
+
await this.deliverSignal(waiter, payload);
|
|
7832
|
+
delivered += 1;
|
|
7833
|
+
}
|
|
7834
|
+
return delivered;
|
|
7835
|
+
}
|
|
7836
|
+
/**
|
|
7837
|
+
* Deliver the `ctx.waitForRun` waits whose run has settled but which were never resumed: a crash
|
|
7838
|
+
* between settling and delivering, or a run that settled by handing off to a continue-as-new
|
|
7839
|
+
* chain that has since ended. Checks the waited-on runs in waiter order, skipping the ones that
|
|
7840
|
+
* have not settled, until it has resumed waiters of `limit` runs or checked
|
|
7841
|
+
* {@link LIFECYCLE_SWEEP_SCAN_MAX}, so long waits on runs still in flight cannot keep it from a
|
|
7842
|
+
* wait owed a delivery. Returns how many waiters it resumed.
|
|
7843
|
+
*/
|
|
7844
|
+
async sweepRunWaits(limit = 100) {
|
|
7845
|
+
const waiters = await this.store.listSignalWaiters(RUN_WAIT_TOKEN_PREFIX);
|
|
7846
|
+
const targets = /* @__PURE__ */ new Set();
|
|
7847
|
+
for (const w of waiters) {
|
|
7848
|
+
const target = runWaitTargetOf(w);
|
|
7849
|
+
if (target !== void 0) targets.add(target);
|
|
7850
|
+
}
|
|
7851
|
+
const scanMax = Math.max(limit, _WorkflowEngine.LIFECYCLE_SWEEP_SCAN_MAX);
|
|
7852
|
+
let delivered = 0;
|
|
7853
|
+
let acted = 0;
|
|
7854
|
+
let examined = 0;
|
|
7855
|
+
for (const target of targets) {
|
|
7856
|
+
if (acted >= limit || examined >= scanMax) break;
|
|
7857
|
+
examined += 1;
|
|
7858
|
+
const resumed = await this.deliverRunWaits(target);
|
|
7859
|
+
if (resumed > 0) {
|
|
7860
|
+
delivered += resumed;
|
|
7861
|
+
acted += 1;
|
|
7862
|
+
}
|
|
7863
|
+
}
|
|
7864
|
+
return delivered;
|
|
6185
7865
|
}
|
|
6186
7866
|
/** Events waiting for the control-plane publish in flight, in emit order (see {@link broadcastEvents}). */
|
|
6187
7867
|
eventsToBroadcast = [];
|
|
@@ -6864,11 +8544,11 @@ var WorkflowEngine = class {
|
|
|
6864
8544
|
message: "execution timeout",
|
|
6865
8545
|
code: "execution_timeout"
|
|
6866
8546
|
};
|
|
6867
|
-
await this.store.updateRun(run.id, {
|
|
8547
|
+
await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
|
|
6868
8548
|
status: "cancelled",
|
|
6869
8549
|
error: error2,
|
|
6870
8550
|
updatedAt: /* @__PURE__ */ new Date()
|
|
6871
|
-
});
|
|
8551
|
+
}));
|
|
6872
8552
|
this.emit({
|
|
6873
8553
|
type: "run.failed",
|
|
6874
8554
|
runId: run.id,
|
|
@@ -7044,11 +8724,11 @@ var WorkflowEngine = class {
|
|
|
7044
8724
|
message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) \u2014 moved to dead-letter`,
|
|
7045
8725
|
code: "max_recovery_attempts"
|
|
7046
8726
|
};
|
|
7047
|
-
await this.store.updateRun(run.id, {
|
|
8727
|
+
await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
|
|
7048
8728
|
status: "dead",
|
|
7049
8729
|
error: error2,
|
|
7050
8730
|
updatedAt: /* @__PURE__ */ new Date()
|
|
7051
|
-
});
|
|
8731
|
+
}));
|
|
7052
8732
|
await this.store.releaseRunLock(run.id);
|
|
7053
8733
|
return this.announceDead({
|
|
7054
8734
|
...run,
|
|
@@ -7126,11 +8806,11 @@ var WorkflowEngine = class {
|
|
|
7126
8806
|
};
|
|
7127
8807
|
const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
|
|
7128
8808
|
run.status
|
|
7129
|
-
], {
|
|
8809
|
+
], await this.withLifecycleMarker(run, {
|
|
7130
8810
|
status: "dead",
|
|
7131
8811
|
error: error2,
|
|
7132
8812
|
updatedAt: /* @__PURE__ */ new Date()
|
|
7133
|
-
});
|
|
8813
|
+
}));
|
|
7134
8814
|
await this.store.releaseRunLock(run.id);
|
|
7135
8815
|
if (!applied) return null;
|
|
7136
8816
|
console.warn(`[nestjs-durable] run ${run.id}: ${error2.message}`);
|
|
@@ -7250,7 +8930,8 @@ var WorkflowEngine = class {
|
|
|
7250
8930
|
const nowMs = this.clock();
|
|
7251
8931
|
const acquired = await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs);
|
|
7252
8932
|
if (!acquired) return null;
|
|
7253
|
-
|
|
8933
|
+
this.lockedForTurn.add(runId);
|
|
8934
|
+
return this.resumeImplicit(runId).finally(() => this.lockedForTurn.delete(runId)).catch(async (err) => {
|
|
7254
8935
|
if (err instanceof NamespaceMismatch) return null;
|
|
7255
8936
|
if (err instanceof WorkflowNotRegistered) {
|
|
7256
8937
|
const run = await this.store.getRun(runId);
|
|
@@ -7534,6 +9215,7 @@ var WorkflowEngine = class {
|
|
|
7534
9215
|
}
|
|
7535
9216
|
try {
|
|
7536
9217
|
const settled = onLocked ? await onLocked(run) : void 0;
|
|
9218
|
+
if (!settled) this.lockedForTurn.add(run.id);
|
|
7537
9219
|
results.push(settled ?? await this.resumeImplicit(run.id));
|
|
7538
9220
|
} catch (error2) {
|
|
7539
9221
|
if (error2 instanceof WorkflowNotRegistered) {
|
|
@@ -7542,6 +9224,8 @@ var WorkflowEngine = class {
|
|
|
7542
9224
|
if (settled !== void 0) continue;
|
|
7543
9225
|
}
|
|
7544
9226
|
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
|
|
9227
|
+
} finally {
|
|
9228
|
+
this.lockedForTurn.delete(run.id);
|
|
7545
9229
|
}
|
|
7546
9230
|
}
|
|
7547
9231
|
return results;
|
|
@@ -7555,7 +9239,8 @@ var WorkflowEngine = class {
|
|
|
7555
9239
|
* 1. **Resumes** every in-flight run waiting on it via `ctx.waitForEvent(name, { match })` whose
|
|
7556
9240
|
* match the payload satisfies (fan-out, vs `signal`'s point-to-point token).
|
|
7557
9241
|
* 2. **Starts** a fresh run of every workflow registered with `onEvent: [name]`, passing the
|
|
7558
|
-
* payload as input
|
|
9242
|
+
* payload as input, and of every workflow live workers announce as subscribing to it that this
|
|
9243
|
+
* engine can start by name (see {@link announcedRemoteSubscribers}). Idempotent by `evt:<id>:<workflow>` — pass `opts.id` to dedupe redeliveries
|
|
7559
9244
|
* of the same logical event (default: a fresh uuid, so each publish triggers once).
|
|
7560
9245
|
*
|
|
7561
9246
|
* Semantics (deliberate — mirrors `signalWithStart`'s reliability contract, so document it exactly):
|
|
@@ -7582,20 +9267,13 @@ var WorkflowEngine = class {
|
|
|
7582
9267
|
touched += 1;
|
|
7583
9268
|
}
|
|
7584
9269
|
}
|
|
7585
|
-
const
|
|
7586
|
-
if (
|
|
9270
|
+
const remote = await this.announcedRemoteSubscribers(name, null);
|
|
9271
|
+
if (this.eventTriggers.get(name)?.size || remote.length > 0) {
|
|
7587
9272
|
const eventId = opts?.id ?? globalThis.crypto.randomUUID();
|
|
7588
|
-
|
|
7589
|
-
|
|
7590
|
-
|
|
7591
|
-
|
|
7592
|
-
await this.accumulators.route(workflow, batch, payload);
|
|
7593
|
-
} else {
|
|
7594
|
-
await this.start(workflow, payload, `evt:${eventId}:${workflow}`);
|
|
7595
|
-
}
|
|
7596
|
-
touched += 1;
|
|
7597
|
-
} catch {
|
|
7598
|
-
}
|
|
9273
|
+
const { reached, failed } = await this.startSubscribers(name, payload, (workflow) => `evt:${eventId}:${workflow}`, void 0, remote);
|
|
9274
|
+
touched += reached.length;
|
|
9275
|
+
for (const { workflow, error: error2 } of failed) {
|
|
9276
|
+
console.warn(`[nestjs-durable] event ${name}: subscriber ${workflow} not started:`, error2);
|
|
7599
9277
|
}
|
|
7600
9278
|
}
|
|
7601
9279
|
if (touched > 0 || opts?.buffer === false) return touched;
|
|
@@ -8052,11 +9730,11 @@ var WorkflowEngine = class {
|
|
|
8052
9730
|
const error2 = {
|
|
8053
9731
|
message: "cancelled"
|
|
8054
9732
|
};
|
|
8055
|
-
await this.store.updateRun(runId, {
|
|
9733
|
+
await this.store.updateRun(runId, await this.withLifecycleMarker(run, {
|
|
8056
9734
|
status: "cancelled",
|
|
8057
9735
|
error: error2,
|
|
8058
9736
|
updatedAt: /* @__PURE__ */ new Date()
|
|
8059
|
-
});
|
|
9737
|
+
}));
|
|
8060
9738
|
this.emit({
|
|
8061
9739
|
type: "run.failed",
|
|
8062
9740
|
runId,
|
|
@@ -8541,6 +10219,7 @@ var WorkflowEngine = class {
|
|
|
8541
10219
|
* than 0.73 (see `legacySingletonHolders`).
|
|
8542
10220
|
*/
|
|
8543
10221
|
async markStarted(run, singleton) {
|
|
10222
|
+
void this.announcedSubscriptions();
|
|
8544
10223
|
const tags = singleton ? withSingletonTracked(run.tags) : run.tags;
|
|
8545
10224
|
const retag = tags !== run.tags;
|
|
8546
10225
|
await this.store.updateRun(run.id, {
|
|
@@ -8583,7 +10262,9 @@ var WorkflowEngine = class {
|
|
|
8583
10262
|
fans: /* @__PURE__ */ new Map()
|
|
8584
10263
|
};
|
|
8585
10264
|
this.wakeReasons.delete(run.id);
|
|
8586
|
-
|
|
10265
|
+
const lockedForTurn = this.lockedForTurn.delete(run.id);
|
|
10266
|
+
const reentered = run.lockedBy === this.instanceId && (run.lockedUntil ?? 0) > lockNow && (lockedForTurn || await this.proveOwnLease(run.id, lockNow));
|
|
10267
|
+
if (!reentered) {
|
|
8587
10268
|
if (!await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow)) {
|
|
8588
10269
|
if (reason.generic) {
|
|
8589
10270
|
if (!this.genericWakeRequested.has(run.id)) {
|
|
@@ -9060,12 +10741,12 @@ ${run.workflowVersion}`;
|
|
|
9060
10741
|
if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
|
|
9061
10742
|
const updatedAt = /* @__PURE__ */ new Date();
|
|
9062
10743
|
if (outcome.kind === "completed") {
|
|
9063
|
-
const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, {
|
|
10744
|
+
const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, await this.withLifecycleMarker(run, {
|
|
9064
10745
|
status: "completed",
|
|
9065
10746
|
output: outcome.output,
|
|
9066
10747
|
error: void 0,
|
|
9067
10748
|
updatedAt
|
|
9068
|
-
});
|
|
10749
|
+
}));
|
|
9069
10750
|
if (!applied2) return this.echoCurrentStatus(run.id);
|
|
9070
10751
|
this.emit({
|
|
9071
10752
|
type: "run.completed",
|
|
@@ -9088,11 +10769,11 @@ ${run.workflowVersion}`;
|
|
|
9088
10769
|
const handed = await this.routeFailure(run, outcome.error);
|
|
9089
10770
|
if (handed && handed.kind !== "failed") return this.settleRun(run, handed);
|
|
9090
10771
|
const error2 = handed?.error ?? outcome.error;
|
|
9091
|
-
const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, {
|
|
10772
|
+
const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, await this.withLifecycleMarker(run, {
|
|
9092
10773
|
status: "failed",
|
|
9093
10774
|
error: error2,
|
|
9094
10775
|
updatedAt
|
|
9095
|
-
});
|
|
10776
|
+
}));
|
|
9096
10777
|
if (!applied2) return this.echoCurrentStatus(run.id);
|
|
9097
10778
|
this.emit({
|
|
9098
10779
|
type: "run.failed",
|
|
@@ -9109,11 +10790,11 @@ ${run.workflowVersion}`;
|
|
|
9109
10790
|
};
|
|
9110
10791
|
}
|
|
9111
10792
|
if (outcome.kind === "blocked") return this.parkBlocked(run, outcome.blocked);
|
|
9112
|
-
const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, {
|
|
10793
|
+
const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, await this.withRunTrace(run, {
|
|
9113
10794
|
status: "suspended",
|
|
9114
10795
|
wakeAt: this.reconcileWakeAt(outcome.wakeAt),
|
|
9115
10796
|
updatedAt
|
|
9116
|
-
});
|
|
10797
|
+
}));
|
|
9117
10798
|
if (!applied) return this.echoCurrentStatus(run.id);
|
|
9118
10799
|
await this.keepDueForKeptWakes(run.id);
|
|
9119
10800
|
this.emit({
|
|
@@ -9143,11 +10824,11 @@ ${run.workflowVersion}`;
|
|
|
9143
10824
|
const error2 = {
|
|
9144
10825
|
message: "cancelled"
|
|
9145
10826
|
};
|
|
9146
|
-
await this.store.updateRun(run.id, {
|
|
10827
|
+
await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
|
|
9147
10828
|
status: "cancelled",
|
|
9148
10829
|
error: error2,
|
|
9149
10830
|
updatedAt: /* @__PURE__ */ new Date()
|
|
9150
|
-
});
|
|
10831
|
+
}));
|
|
9151
10832
|
if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
|
|
9152
10833
|
this.emit({
|
|
9153
10834
|
type: "run.failed",
|
|
@@ -9179,7 +10860,8 @@ ${run.workflowVersion}`;
|
|
|
9179
10860
|
type: "run.started",
|
|
9180
10861
|
runId: run.id,
|
|
9181
10862
|
workflow: run.workflow,
|
|
9182
|
-
namespace: run.namespace
|
|
10863
|
+
namespace: run.namespace,
|
|
10864
|
+
...await this.sourceTraceOf(run)
|
|
9183
10865
|
});
|
|
9184
10866
|
}
|
|
9185
10867
|
if (registered.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
@@ -9202,12 +10884,12 @@ ${run.workflowVersion}`;
|
|
|
9202
10884
|
const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID().slice(0, 8)}`;
|
|
9203
10885
|
const silenceMs = this.remoteAdvanceSilenceMs;
|
|
9204
10886
|
const wakeAt = silenceMs != null ? this.clock() + silenceMs : void 0;
|
|
9205
|
-
await this.store.updateRun(run.id, {
|
|
10887
|
+
await this.store.updateRun(run.id, await this.withRunTrace(run, {
|
|
9206
10888
|
status: "suspended",
|
|
9207
10889
|
awaitingDecisionTaskId: taskId,
|
|
9208
10890
|
wakeAt,
|
|
9209
10891
|
updatedAt: /* @__PURE__ */ new Date()
|
|
9210
|
-
});
|
|
10892
|
+
}));
|
|
9211
10893
|
this.emit({
|
|
9212
10894
|
type: "run.suspended",
|
|
9213
10895
|
runId: run.id,
|
|
@@ -9244,11 +10926,11 @@ ${run.workflowVersion}`;
|
|
|
9244
10926
|
stack: err instanceof Error ? err.stack : void 0
|
|
9245
10927
|
};
|
|
9246
10928
|
if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
|
|
9247
|
-
await this.store.updateRun(run.id, {
|
|
10929
|
+
await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
|
|
9248
10930
|
status: "failed",
|
|
9249
10931
|
error: error2,
|
|
9250
10932
|
updatedAt: /* @__PURE__ */ new Date()
|
|
9251
|
-
});
|
|
10933
|
+
}));
|
|
9252
10934
|
this.emit({
|
|
9253
10935
|
type: "run.failed",
|
|
9254
10936
|
runId: run.id,
|
|
@@ -9786,11 +11468,11 @@ ${run.workflowVersion}`;
|
|
|
9786
11468
|
const error2 = {
|
|
9787
11469
|
message: "cancelled"
|
|
9788
11470
|
};
|
|
9789
|
-
await this.store.updateRun(run.id, {
|
|
11471
|
+
await this.store.updateRun(run.id, await this.withLifecycleMarker(run, {
|
|
9790
11472
|
status: "cancelled",
|
|
9791
11473
|
error: error2,
|
|
9792
11474
|
updatedAt: /* @__PURE__ */ new Date()
|
|
9793
|
-
});
|
|
11475
|
+
}));
|
|
9794
11476
|
this.emit({
|
|
9795
11477
|
type: "run.failed",
|
|
9796
11478
|
runId: run.id,
|
|
@@ -9837,7 +11519,8 @@ ${run.workflowVersion}`;
|
|
|
9837
11519
|
type: "run.started",
|
|
9838
11520
|
runId: run.id,
|
|
9839
11521
|
workflow: run.workflow,
|
|
9840
|
-
namespace: run.namespace
|
|
11522
|
+
namespace: run.namespace,
|
|
11523
|
+
...await this.sourceTraceOf(run)
|
|
9841
11524
|
});
|
|
9842
11525
|
}
|
|
9843
11526
|
await this.noteServedWakes(run.id);
|
|
@@ -9886,6 +11569,8 @@ ${run.workflowVersion}`;
|
|
|
9886
11569
|
runId: run.id,
|
|
9887
11570
|
workflow: run.workflow,
|
|
9888
11571
|
namespace: run.namespace
|
|
11572
|
+
}, {
|
|
11573
|
+
lifecycle: false
|
|
9889
11574
|
});
|
|
9890
11575
|
void this.notifyParent(run.id, {
|
|
9891
11576
|
ok: true,
|
|
@@ -11877,6 +13562,13 @@ function runDetailEtag(run, stats, over) {
|
|
|
11877
13562
|
return `"rd-${hash(`${over}${JSON.stringify(run)}${statsFingerprint(stats)}`)}"`;
|
|
11878
13563
|
}
|
|
11879
13564
|
__name(runDetailEtag, "runDetailEtag");
|
|
13565
|
+
function extendEtag(etag, extras) {
|
|
13566
|
+
if (extras === void 0) return etag;
|
|
13567
|
+
const weak = etag.startsWith("W/") ? "W/" : "";
|
|
13568
|
+
const bare = etag.slice(weak.length).replace(/^"|"$/g, "");
|
|
13569
|
+
return `${weak}"${bare}.${hash(JSON.stringify(extras))}"`;
|
|
13570
|
+
}
|
|
13571
|
+
__name(extendEtag, "extendEtag");
|
|
11880
13572
|
function fanPageEtag(group, stats, query) {
|
|
11881
13573
|
const { offset, limit } = fanPageBounds(query);
|
|
11882
13574
|
const key = [
|
|
@@ -12265,7 +13957,82 @@ var CodecStateStore = class {
|
|
|
12265
13957
|
}
|
|
12266
13958
|
};
|
|
12267
13959
|
|
|
13960
|
+
// src/lifecycle-deliveries.ts
|
|
13961
|
+
function lifecycleDeliveriesOf(run, opts) {
|
|
13962
|
+
const kind = lifecycleKindOf(run.status);
|
|
13963
|
+
if (!kind) return [];
|
|
13964
|
+
const tags = run.tags ?? [];
|
|
13965
|
+
const owed = new Set(owedSubscribersOf(tags));
|
|
13966
|
+
const triggered = new Set(tags.filter((t) => t.startsWith(LIFECYCLE_TRIGGERED_TAG_PREFIX)).map((t) => t.slice(LIFECYCLE_TRIGGERED_TAG_PREFIX.length)));
|
|
13967
|
+
const coalescedTagged = new Set(coalescedSubscribersOf(tags));
|
|
13968
|
+
const triggerFailed = new Set(triggerFailedSubscribersOf(tags));
|
|
13969
|
+
const owedToAll = owed.size === 0 && tags.includes(LIFECYCLE_PENDING_TAG);
|
|
13970
|
+
const settledAt = new Date(run.updatedAt).getTime();
|
|
13971
|
+
const names = [
|
|
13972
|
+
.../* @__PURE__ */ new Set([
|
|
13973
|
+
...opts.subscribers,
|
|
13974
|
+
...owed,
|
|
13975
|
+
...triggered,
|
|
13976
|
+
...coalescedTagged,
|
|
13977
|
+
...triggerFailed
|
|
13978
|
+
])
|
|
13979
|
+
].sort();
|
|
13980
|
+
const coalesced = /* @__PURE__ */ __name((subscriber) => {
|
|
13981
|
+
const runId = opts.coalesced?.get(subscriber);
|
|
13982
|
+
return {
|
|
13983
|
+
subscriber,
|
|
13984
|
+
state: "coalesced",
|
|
13985
|
+
...runId !== void 0 ? {
|
|
13986
|
+
runId
|
|
13987
|
+
} : {}
|
|
13988
|
+
};
|
|
13989
|
+
}, "coalesced");
|
|
13990
|
+
return names.map((subscriber) => {
|
|
13991
|
+
if (triggered.has(subscriber)) {
|
|
13992
|
+
return {
|
|
13993
|
+
subscriber,
|
|
13994
|
+
state: "started",
|
|
13995
|
+
runId: lifecycleRunId(run.id, kind, subscriber)
|
|
13996
|
+
};
|
|
13997
|
+
}
|
|
13998
|
+
if (coalescedTagged.has(subscriber)) return coalesced(subscriber);
|
|
13999
|
+
if (triggerFailed.has(subscriber)) return {
|
|
14000
|
+
subscriber,
|
|
14001
|
+
state: "failed"
|
|
14002
|
+
};
|
|
14003
|
+
if (owed.has(subscriber) || owedToAll) {
|
|
14004
|
+
return {
|
|
14005
|
+
subscriber,
|
|
14006
|
+
state: "owed",
|
|
14007
|
+
owedSince: settledAt,
|
|
14008
|
+
...opts.live.has(subscriber) ? {} : {
|
|
14009
|
+
orphanAt: settledAt + LIFECYCLE_ORPHAN_MS
|
|
14010
|
+
}
|
|
14011
|
+
};
|
|
14012
|
+
}
|
|
14013
|
+
if (opts.coalesced?.has(subscriber)) return coalesced(subscriber);
|
|
14014
|
+
return {
|
|
14015
|
+
subscriber,
|
|
14016
|
+
state: "not-started"
|
|
14017
|
+
};
|
|
14018
|
+
});
|
|
14019
|
+
}
|
|
14020
|
+
__name(lifecycleDeliveriesOf, "lifecycleDeliveriesOf");
|
|
14021
|
+
function traceIdOfTraceparent(traceparent) {
|
|
14022
|
+
const traceId = traceparent?.split("-")[1];
|
|
14023
|
+
return traceId && /^[0-9a-f]{32}$/.test(traceId) && !/^0+$/.test(traceId) ? traceId : void 0;
|
|
14024
|
+
}
|
|
14025
|
+
__name(traceIdOfTraceparent, "traceIdOfTraceparent");
|
|
14026
|
+
|
|
12268
14027
|
// src/metrics.ts
|
|
14028
|
+
function countBySubscriber(counts, e) {
|
|
14029
|
+
const workflow = e.workflow ?? "unknown";
|
|
14030
|
+
const subscriber = e.name ?? "unknown";
|
|
14031
|
+
const bySubscriber = counts[workflow] ?? {};
|
|
14032
|
+
bySubscriber[subscriber] = (bySubscriber[subscriber] ?? 0) + 1;
|
|
14033
|
+
counts[workflow] = bySubscriber;
|
|
14034
|
+
}
|
|
14035
|
+
__name(countBySubscriber, "countBySubscriber");
|
|
12269
14036
|
function collectMetrics(engine) {
|
|
12270
14037
|
const m = {
|
|
12271
14038
|
runs: {
|
|
@@ -12277,6 +14044,8 @@ function collectMetrics(engine) {
|
|
|
12277
14044
|
unrouted: 0
|
|
12278
14045
|
},
|
|
12279
14046
|
routedByCode: {},
|
|
14047
|
+
triggeredBySubscriber: {},
|
|
14048
|
+
triggerFailedBySubscriber: {},
|
|
12280
14049
|
steps: {
|
|
12281
14050
|
started: 0,
|
|
12282
14051
|
completed: 0,
|
|
@@ -12328,6 +14097,12 @@ function collectMetrics(engine) {
|
|
|
12328
14097
|
case "run.unrouted":
|
|
12329
14098
|
m.runs.unrouted += 1;
|
|
12330
14099
|
break;
|
|
14100
|
+
case "run.triggered":
|
|
14101
|
+
countBySubscriber(m.triggeredBySubscriber, e);
|
|
14102
|
+
break;
|
|
14103
|
+
case "run.trigger_failed":
|
|
14104
|
+
countBySubscriber(m.triggerFailedBySubscriber, e);
|
|
14105
|
+
break;
|
|
12331
14106
|
case "step.started":
|
|
12332
14107
|
m.steps.started += 1;
|
|
12333
14108
|
break;
|
|
@@ -12356,6 +14131,10 @@ function collectMetrics(engine) {
|
|
|
12356
14131
|
for (const [name, v] of Object.entries(m.byWorkflow)) for (const [event, n] of Object.entries(v)) lines.push(`durable_runs_by_workflow_total{workflow="${name}",event="${event}"} ${n}`);
|
|
12357
14132
|
lines.push("# TYPE durable_runs_routed_total counter");
|
|
12358
14133
|
for (const [code, n] of Object.entries(m.routedByCode)) lines.push(`durable_runs_routed_total{code="${code}"} ${n}`);
|
|
14134
|
+
lines.push("# TYPE durable_runs_triggered_total counter");
|
|
14135
|
+
for (const [workflow, bySubscriber] of Object.entries(m.triggeredBySubscriber)) for (const [subscriber, n] of Object.entries(bySubscriber)) lines.push(`durable_runs_triggered_total{workflow="${workflow}",subscriber="${subscriber}"} ${n}`);
|
|
14136
|
+
lines.push("# TYPE durable_runs_trigger_failed_total counter");
|
|
14137
|
+
for (const [workflow, bySubscriber] of Object.entries(m.triggerFailedBySubscriber)) for (const [subscriber, n] of Object.entries(bySubscriber)) lines.push(`durable_runs_trigger_failed_total{workflow="${workflow}",subscriber="${subscriber}"} ${n}`);
|
|
12359
14138
|
lines.push("# TYPE durable_step_duration_ms_sum counter");
|
|
12360
14139
|
lines.push(`durable_step_duration_ms_sum ${m.stepDuration.sumMs}`);
|
|
12361
14140
|
lines.push("# TYPE durable_step_duration_count counter");
|
|
@@ -12503,6 +14282,9 @@ var TRANSPORT_CANONICAL = Symbol.for("@dudousxd/nestjs-durable:transport");
|
|
|
12503
14282
|
var DURABLE_OPTIONS_CANONICAL = Symbol.for("@dudousxd/nestjs-durable:options");
|
|
12504
14283
|
|
|
12505
14284
|
// src/testing/in-memory-state-store.ts
|
|
14285
|
+
var RETENTION_EXCLUDED = [
|
|
14286
|
+
LIFECYCLE_REGISTRY_WORKFLOW
|
|
14287
|
+
];
|
|
12506
14288
|
var InMemoryStateStore = class {
|
|
12507
14289
|
static {
|
|
12508
14290
|
__name(this, "InMemoryStateStore");
|
|
@@ -12632,12 +14414,13 @@ var InMemoryStateStore = class {
|
|
|
12632
14414
|
if (policy.maxAge == null && policy.maxCount == null) return 0;
|
|
12633
14415
|
const inScope = this.matching({
|
|
12634
14416
|
...policy.scope ?? {},
|
|
12635
|
-
statuses
|
|
14417
|
+
statuses,
|
|
14418
|
+
excludeWorkflows: RETENTION_EXCLUDED
|
|
12636
14419
|
});
|
|
12637
14420
|
const ids = /* @__PURE__ */ new Set();
|
|
12638
14421
|
if (policy.maxAge != null) {
|
|
12639
14422
|
const cutoff = nowMs - parseDuration(policy.maxAge);
|
|
12640
|
-
const expired = inScope.filter((r) => r.updatedAt.getTime() < cutoff).sort((a, b) => a.updatedAt.getTime() - b.updatedAt.getTime()).slice(0, limit);
|
|
14423
|
+
const expired = inScope.filter((r) => r.updatedAt.getTime() < cutoff && !retentionKeeps(r.tags)).sort((a, b) => a.updatedAt.getTime() - b.updatedAt.getTime()).slice(0, limit);
|
|
12641
14424
|
for (const r of expired) ids.add(r.id);
|
|
12642
14425
|
}
|
|
12643
14426
|
if (policy.maxCount != null && ids.size < limit) {
|
|
@@ -12646,6 +14429,7 @@ var InMemoryStateStore = class {
|
|
|
12646
14429
|
].sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime() || b.id.localeCompare(a.id));
|
|
12647
14430
|
for (const r of byRecency.slice(policy.maxCount)) {
|
|
12648
14431
|
if (ids.size >= limit) break;
|
|
14432
|
+
if (retentionKeeps(r.tags)) continue;
|
|
12649
14433
|
ids.add(r.id);
|
|
12650
14434
|
}
|
|
12651
14435
|
}
|
|
@@ -12943,6 +14727,13 @@ var InMemoryStateStore = class {
|
|
|
12943
14727
|
];
|
|
12944
14728
|
if (query.workflow) runs = runs.filter((r) => r.workflow === query.workflow);
|
|
12945
14729
|
if (query.workflows) runs = runs.filter((r) => query.workflows?.includes(r.workflow));
|
|
14730
|
+
if (query.excludeWorkflows?.length) {
|
|
14731
|
+
runs = runs.filter((r) => !query.excludeWorkflows?.includes(r.workflow));
|
|
14732
|
+
}
|
|
14733
|
+
if (query.updatedAfter !== void 0) {
|
|
14734
|
+
const after = query.updatedAfter;
|
|
14735
|
+
runs = runs.filter((r) => r.updatedAt.getTime() >= after);
|
|
14736
|
+
}
|
|
12946
14737
|
if (query.status) runs = runs.filter((r) => r.status === query.status);
|
|
12947
14738
|
if (query.statuses) runs = runs.filter((r) => query.statuses?.includes(r.status));
|
|
12948
14739
|
if (query.tag) runs = runs.filter((r) => r.tags?.includes(query.tag));
|
|
@@ -13239,6 +15030,7 @@ var InMemoryRunQueue = class {
|
|
|
13239
15030
|
};
|
|
13240
15031
|
// Annotate the CommonJS export names for ESM import in node:
|
|
13241
15032
|
0 && (module.exports = {
|
|
15033
|
+
ACCUMULATOR_SEEN_MAX,
|
|
13242
15034
|
CHILD_EDGE_PREFIXES,
|
|
13243
15035
|
CONTROL_FLOW_SIGNAL,
|
|
13244
15036
|
CORE_ORIGIN,
|
|
@@ -13270,6 +15062,19 @@ var InMemoryRunQueue = class {
|
|
|
13270
15062
|
InMemoryTransport,
|
|
13271
15063
|
LEGACY_V1_CAPABILITIES,
|
|
13272
15064
|
LEGACY_V1_PROTOCOL,
|
|
15065
|
+
LIFECYCLE_COALESCED_TAG_PREFIX,
|
|
15066
|
+
LIFECYCLE_HEAL_WINDOW_MS,
|
|
15067
|
+
LIFECYCLE_ORPHAN_MS,
|
|
15068
|
+
LIFECYCLE_OWED_TAG_PREFIX,
|
|
15069
|
+
LIFECYCLE_PENDING_TAG,
|
|
15070
|
+
LIFECYCLE_REGISTRY_HEARTBEAT_MS,
|
|
15071
|
+
LIFECYCLE_REGISTRY_STALE_MS,
|
|
15072
|
+
LIFECYCLE_REGISTRY_TAG,
|
|
15073
|
+
LIFECYCLE_REGISTRY_TTL_MS,
|
|
15074
|
+
LIFECYCLE_REGISTRY_WORKFLOW,
|
|
15075
|
+
LIFECYCLE_TRACE_TAG_PREFIX,
|
|
15076
|
+
LIFECYCLE_TRIGGERED_TAG_PREFIX,
|
|
15077
|
+
LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX,
|
|
13273
15078
|
NestedWorkflowCallError,
|
|
13274
15079
|
NonDeterminismError,
|
|
13275
15080
|
QueueController,
|
|
@@ -13277,6 +15082,7 @@ var InMemoryRunQueue = class {
|
|
|
13277
15082
|
RUN_HISTORY_FIXTURE_VERSION,
|
|
13278
15083
|
RUN_VALUE_FACET_LIMIT,
|
|
13279
15084
|
RUN_VALUE_FACET_SCAN,
|
|
15085
|
+
RUN_WAIT_TOKEN_PREFIX,
|
|
13280
15086
|
RemoteStepError,
|
|
13281
15087
|
RemoteStepTimeout,
|
|
13282
15088
|
RemoteWorkflowExecutor,
|
|
@@ -13305,6 +15111,8 @@ var InMemoryRunQueue = class {
|
|
|
13305
15111
|
WorkflowEngine,
|
|
13306
15112
|
WorkflowNotRegistered,
|
|
13307
15113
|
WorkflowSuspended,
|
|
15114
|
+
acceptingSubscription,
|
|
15115
|
+
accumulatorRunId,
|
|
13308
15116
|
aggregateAnnouncements,
|
|
13309
15117
|
aggregatesCheckpoints,
|
|
13310
15118
|
announcementsOf,
|
|
@@ -13320,12 +15128,14 @@ var InMemoryRunQueue = class {
|
|
|
13320
15128
|
buildWorkflowDirectory,
|
|
13321
15129
|
canRoute,
|
|
13322
15130
|
capableWorkers,
|
|
15131
|
+
carriesLifecyclePending,
|
|
13323
15132
|
checkpointNameKind,
|
|
13324
15133
|
checkpointStatsOf,
|
|
13325
15134
|
childIdsOf,
|
|
13326
15135
|
classifyWaiterToken,
|
|
13327
15136
|
clearDurableExecutionWrappers,
|
|
13328
15137
|
clearTerminalSingletonAdmission,
|
|
15138
|
+
coalescedSubscribersOf,
|
|
13329
15139
|
codeAndDetails,
|
|
13330
15140
|
codeOf,
|
|
13331
15141
|
collectMetrics,
|
|
@@ -13348,6 +15158,7 @@ var InMemoryRunQueue = class {
|
|
|
13348
15158
|
eventNameOf,
|
|
13349
15159
|
eventPrefix,
|
|
13350
15160
|
eventToken,
|
|
15161
|
+
extendEtag,
|
|
13351
15162
|
facetOrigin,
|
|
13352
15163
|
fanPageBounds,
|
|
13353
15164
|
fanPageEtag,
|
|
@@ -13365,10 +15176,21 @@ var InMemoryRunQueue = class {
|
|
|
13365
15176
|
isEventToken,
|
|
13366
15177
|
isFailedMember,
|
|
13367
15178
|
isLegacyDescriptor,
|
|
15179
|
+
isLifecycleTag,
|
|
13368
15180
|
isNonEmptyRunScope,
|
|
13369
15181
|
isRemoteEvent,
|
|
13370
15182
|
isWorkflowControlFlowSignal,
|
|
13371
15183
|
legacySingletonHolders,
|
|
15184
|
+
lifecycleDeliveriesOf,
|
|
15185
|
+
lifecycleEventName,
|
|
15186
|
+
lifecycleEventOf,
|
|
15187
|
+
lifecycleKindOf,
|
|
15188
|
+
lifecycleRegistryNamespace,
|
|
15189
|
+
lifecycleRegistryRunId,
|
|
15190
|
+
lifecycleRegistrySubscriptions,
|
|
15191
|
+
lifecycleRunId,
|
|
15192
|
+
lifecycleSourceOf,
|
|
15193
|
+
lifecycleTraceparentOf,
|
|
13372
15194
|
listSignalWaitersOfRuns,
|
|
13373
15195
|
liveHandoff,
|
|
13374
15196
|
log,
|
|
@@ -13383,10 +15205,12 @@ var InMemoryRunQueue = class {
|
|
|
13383
15205
|
normalizeDescriptor,
|
|
13384
15206
|
observedAnnouncements,
|
|
13385
15207
|
onErrorRunId,
|
|
15208
|
+
owedSubscribersOf,
|
|
13386
15209
|
pageFan,
|
|
13387
15210
|
pageFanFromStore,
|
|
13388
15211
|
parentRunIdOf,
|
|
13389
15212
|
parseDuration,
|
|
15213
|
+
parseLifecycleEvent,
|
|
13390
15214
|
parseWebhookToken,
|
|
13391
15215
|
partitionOfToken,
|
|
13392
15216
|
planDispatch,
|
|
@@ -13401,10 +15225,13 @@ var InMemoryRunQueue = class {
|
|
|
13401
15225
|
resolveRunWaiting,
|
|
13402
15226
|
resolveTenantToken,
|
|
13403
15227
|
resolveWireTenant,
|
|
15228
|
+
retentionKeeps,
|
|
13404
15229
|
retiredHandoffName,
|
|
13405
15230
|
routeFor,
|
|
15231
|
+
runCompletedEvent,
|
|
13406
15232
|
runDetailEtag,
|
|
13407
15233
|
runDurableExecution,
|
|
15234
|
+
runFailedEvent,
|
|
13408
15235
|
runInStepLogger,
|
|
13409
15236
|
runInStepScope,
|
|
13410
15237
|
runInWorkflowCtx,
|
|
@@ -13412,6 +15239,9 @@ var InMemoryRunQueue = class {
|
|
|
13412
15239
|
runSchedules,
|
|
13413
15240
|
runStepHandler,
|
|
13414
15241
|
runValueFacetsFromRuns,
|
|
15242
|
+
runWaitPrefix,
|
|
15243
|
+
runWaitTargetOf,
|
|
15244
|
+
runWaitToken,
|
|
13415
15245
|
sanitizeQueueToken,
|
|
13416
15246
|
scheduleRunIdAt,
|
|
13417
15247
|
scheduledRunId,
|
|
@@ -13430,6 +15260,11 @@ var InMemoryRunQueue = class {
|
|
|
13430
15260
|
summarizeRunFromStore,
|
|
13431
15261
|
summarizeTimeline,
|
|
13432
15262
|
tenantGroup,
|
|
15263
|
+
toEventSubscription,
|
|
15264
|
+
traceIdOfTraceparent,
|
|
15265
|
+
triggerFailedSubscribersOf,
|
|
15266
|
+
triggeredRunsOf,
|
|
15267
|
+
triggeredSubscribersOf,
|
|
13433
15268
|
useDurableExecution,
|
|
13434
15269
|
warn,
|
|
13435
15270
|
withSingletonTracked,
|