@dudousxd/nestjs-durable-core 0.27.1 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/index.cjs +3923 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +2044 -0
- package/dist/index.d.ts +2044 -20
- package/dist/index.js +3857 -38
- package/dist/index.js.map +1 -1
- package/package.json +19 -5
- package/dist/backoff.d.ts +0 -5
- package/dist/backoff.d.ts.map +0 -1
- package/dist/backoff.js +0 -14
- package/dist/backoff.js.map +0 -1
- package/dist/checkpoints.d.ts +0 -17
- package/dist/checkpoints.d.ts.map +0 -1
- package/dist/checkpoints.js +0 -28
- package/dist/checkpoints.js.map +0 -1
- package/dist/codec-state-store.d.ts +0 -57
- package/dist/codec-state-store.d.ts.map +0 -1
- package/dist/codec-state-store.js +0 -111
- package/dist/codec-state-store.js.map +0 -1
- package/dist/completion.d.ts +0 -15
- package/dist/completion.d.ts.map +0 -1
- package/dist/completion.js +0 -13
- package/dist/completion.js.map +0 -1
- package/dist/duration.d.ts +0 -6
- package/dist/duration.d.ts.map +0 -1
- package/dist/duration.js +0 -42
- package/dist/duration.js.map +0 -1
- package/dist/engine.d.ts +0 -525
- package/dist/engine.d.ts.map +0 -1
- package/dist/engine.js +0 -1785
- package/dist/engine.js.map +0 -1
- package/dist/entities.d.ts +0 -44
- package/dist/entities.d.ts.map +0 -1
- package/dist/entities.js +0 -68
- package/dist/entities.js.map +0 -1
- package/dist/errors.d.ts +0 -56
- package/dist/errors.d.ts.map +0 -1
- package/dist/errors.js +0 -93
- package/dist/errors.js.map +0 -1
- package/dist/event-accumulators.d.ts +0 -39
- package/dist/event-accumulators.d.ts.map +0 -1
- package/dist/event-accumulators.js +0 -74
- package/dist/event-accumulators.js.map +0 -1
- package/dist/events.d.ts +0 -9
- package/dist/events.d.ts.map +0 -1
- package/dist/events.js +0 -41
- package/dist/events.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/interfaces.d.ts +0 -830
- package/dist/interfaces.d.ts.map +0 -1
- package/dist/interfaces.js +0 -3
- package/dist/interfaces.js.map +0 -1
- package/dist/metrics.d.ts +0 -44
- package/dist/metrics.d.ts.map +0 -1
- package/dist/metrics.js +0 -76
- package/dist/metrics.js.map +0 -1
- package/dist/protocol.d.ts +0 -17
- package/dist/protocol.d.ts.map +0 -1
- package/dist/protocol.js +0 -53
- package/dist/protocol.js.map +0 -1
- package/dist/queue.d.ts +0 -48
- package/dist/queue.d.ts.map +0 -1
- package/dist/queue.js +0 -45
- package/dist/queue.js.map +0 -1
- package/dist/remote-step-factory.d.ts +0 -15
- package/dist/remote-step-factory.d.ts.map +0 -1
- package/dist/remote-step-factory.js +0 -15
- package/dist/remote-step-factory.js.map +0 -1
- package/dist/remote-workflow-executor.d.ts +0 -24
- package/dist/remote-workflow-executor.d.ts.map +0 -1
- package/dist/remote-workflow-executor.js +0 -73
- package/dist/remote-workflow-executor.js.map +0 -1
- package/dist/scheduler.d.ts +0 -42
- package/dist/scheduler.d.ts.map +0 -1
- package/dist/scheduler.js +0 -70
- package/dist/scheduler.js.map +0 -1
- package/dist/search-attributes.d.ts +0 -17
- package/dist/search-attributes.d.ts.map +0 -1
- package/dist/search-attributes.js +0 -47
- package/dist/search-attributes.js.map +0 -1
- package/dist/step-logger.d.ts +0 -9
- package/dist/step-logger.d.ts.map +0 -1
- package/dist/step-logger.js +0 -92
- package/dist/step-logger.js.map +0 -1
- package/dist/test-helpers.d.ts +0 -11
- package/dist/test-helpers.d.ts.map +0 -1
- package/dist/test-helpers.js +0 -15
- package/dist/test-helpers.js.map +0 -1
- package/dist/testing/in-memory-state-store.d.ts +0 -37
- package/dist/testing/in-memory-state-store.d.ts.map +0 -1
- package/dist/testing/in-memory-state-store.js +0 -134
- package/dist/testing/in-memory-state-store.js.map +0 -1
- package/dist/testing/in-memory-transport.d.ts +0 -25
- package/dist/testing/in-memory-transport.d.ts.map +0 -1
- package/dist/testing/in-memory-transport.js +0 -53
- package/dist/testing/in-memory-transport.js.map +0 -1
- package/dist/tokens.d.ts +0 -8
- package/dist/tokens.d.ts.map +0 -1
- package/dist/tokens.js +0 -11
- package/dist/tokens.js.map +0 -1
- package/dist/transport-pool.d.ts +0 -30
- package/dist/transport-pool.d.ts.map +0 -1
- package/dist/transport-pool.js +0 -86
- package/dist/transport-pool.js.map +0 -1
- package/dist/workflow-ctx.d.ts +0 -61
- package/dist/workflow-ctx.d.ts.map +0 -1
- package/dist/workflow-ctx.js +0 -451
- package/dist/workflow-ctx.js.map +0 -1
- package/dist/workflow-ref.d.ts +0 -34
- package/dist/workflow-ref.d.ts.map +0 -1
- package/dist/workflow-ref.js +0 -32
- package/dist/workflow-ref.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,39 +1,3858 @@
|
|
|
1
|
-
|
|
2
|
-
var
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
1
|
+
var __defProp = Object.defineProperty;
|
|
2
|
+
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
3
|
+
|
|
4
|
+
// src/duration.ts
|
|
5
|
+
var UNIT_MS = {
|
|
6
|
+
ms: 1,
|
|
7
|
+
s: 1e3,
|
|
8
|
+
sec: 1e3,
|
|
9
|
+
second: 1e3,
|
|
10
|
+
seconds: 1e3,
|
|
11
|
+
m: 6e4,
|
|
12
|
+
min: 6e4,
|
|
13
|
+
minute: 6e4,
|
|
14
|
+
minutes: 6e4,
|
|
15
|
+
h: 36e5,
|
|
16
|
+
hr: 36e5,
|
|
17
|
+
hour: 36e5,
|
|
18
|
+
hours: 36e5,
|
|
19
|
+
d: 864e5,
|
|
20
|
+
day: 864e5,
|
|
21
|
+
days: 864e5,
|
|
22
|
+
w: 6048e5,
|
|
23
|
+
week: 6048e5,
|
|
24
|
+
weeks: 6048e5
|
|
25
|
+
};
|
|
26
|
+
function parseDuration(duration) {
|
|
27
|
+
if (typeof duration === "number") return duration;
|
|
28
|
+
const match = duration.trim().match(/^(\d+(?:\.\d+)?)\s*([a-z]+)$/i);
|
|
29
|
+
if (!match) throw new Error(`invalid duration: ${duration}`);
|
|
30
|
+
const value = Number(match[1]);
|
|
31
|
+
const unit = (match[2] ?? "").toLowerCase();
|
|
32
|
+
const factor = UNIT_MS[unit];
|
|
33
|
+
if (factor === void 0) throw new Error(`unknown duration unit: ${unit}`);
|
|
34
|
+
return value * factor;
|
|
35
|
+
}
|
|
36
|
+
__name(parseDuration, "parseDuration");
|
|
37
|
+
|
|
38
|
+
// src/backoff.ts
|
|
39
|
+
function backoffDelay(attempt, options) {
|
|
40
|
+
const base = options?.backoffMs ?? 0;
|
|
41
|
+
if (base <= 0) return 0;
|
|
42
|
+
const raw = options?.backoff === "exp" ? base * 2 ** (attempt - 1) : base;
|
|
43
|
+
const capped = options?.backoffMaxMs ? Math.min(raw, options.backoffMaxMs) : raw;
|
|
44
|
+
return options?.jitter ? Math.round(capped * (0.5 + Math.random() * 0.5)) : capped;
|
|
45
|
+
}
|
|
46
|
+
__name(backoffDelay, "backoffDelay");
|
|
47
|
+
|
|
48
|
+
// src/step-logger.ts
|
|
49
|
+
function withDuration(start, now, data) {
|
|
50
|
+
if (data && typeof data === "object" && "durationMs" in data) {
|
|
51
|
+
return data;
|
|
52
|
+
}
|
|
53
|
+
const base = data && typeof data === "object" ? data : {};
|
|
54
|
+
return {
|
|
55
|
+
...base,
|
|
56
|
+
durationMs: now() - start
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
__name(withDuration, "withDuration");
|
|
60
|
+
function createStepLogger(events, now) {
|
|
61
|
+
let currentSub;
|
|
62
|
+
const push = /* @__PURE__ */ __name((level, message, data) => events.push({
|
|
63
|
+
at: now(),
|
|
64
|
+
level,
|
|
65
|
+
message,
|
|
66
|
+
...currentSub ? {
|
|
67
|
+
subId: currentSub.id
|
|
68
|
+
} : {},
|
|
69
|
+
...data === void 0 ? {} : {
|
|
70
|
+
data
|
|
71
|
+
}
|
|
72
|
+
}), "push");
|
|
73
|
+
const subEvent = /* @__PURE__ */ __name((e) => events.push({
|
|
74
|
+
at: now(),
|
|
75
|
+
level: e.status === "failed" ? "error" : e.status === "skipped" ? "warn" : "info",
|
|
76
|
+
message: e.message ?? e.phase ?? e.name,
|
|
77
|
+
subId: e.id,
|
|
78
|
+
name: e.name,
|
|
79
|
+
...e.group === void 0 ? {} : {
|
|
80
|
+
group: e.group
|
|
81
|
+
},
|
|
82
|
+
...e.phase === void 0 ? {} : {
|
|
83
|
+
phase: e.phase
|
|
84
|
+
},
|
|
85
|
+
...e.status === void 0 ? {} : {
|
|
86
|
+
status: e.status
|
|
87
|
+
},
|
|
88
|
+
...e.data === void 0 ? {} : {
|
|
89
|
+
data: e.data
|
|
90
|
+
}
|
|
91
|
+
}), "subEvent");
|
|
92
|
+
const subProcess = /* @__PURE__ */ __name(async (name, body, opts) => {
|
|
93
|
+
const id = opts?.id ?? globalThis.crypto.randomUUID();
|
|
94
|
+
const group = opts?.group;
|
|
95
|
+
const start = now();
|
|
96
|
+
const prevSub = currentSub;
|
|
97
|
+
currentSub = {
|
|
98
|
+
id
|
|
99
|
+
};
|
|
100
|
+
let terminal = false;
|
|
101
|
+
const emit = /* @__PURE__ */ __name((status, message, data) => {
|
|
102
|
+
if (terminal) return;
|
|
103
|
+
terminal = true;
|
|
104
|
+
subEvent({
|
|
105
|
+
id,
|
|
106
|
+
name,
|
|
107
|
+
group,
|
|
108
|
+
status,
|
|
109
|
+
message,
|
|
110
|
+
data: withDuration(start, now, data)
|
|
111
|
+
});
|
|
112
|
+
}, "emit");
|
|
113
|
+
const handle = {
|
|
114
|
+
phase: /* @__PURE__ */ __name((phase, data) => {
|
|
115
|
+
if (!terminal) subEvent({
|
|
116
|
+
id,
|
|
117
|
+
name,
|
|
118
|
+
group,
|
|
119
|
+
phase,
|
|
120
|
+
data
|
|
121
|
+
});
|
|
122
|
+
return handle;
|
|
123
|
+
}, "phase"),
|
|
124
|
+
skip: /* @__PURE__ */ __name((reason, data) => emit("skipped", reason, data), "skip"),
|
|
125
|
+
fail: /* @__PURE__ */ __name((reason, data) => emit("failed", reason, data), "fail")
|
|
126
|
+
};
|
|
127
|
+
try {
|
|
128
|
+
const result = await body(handle);
|
|
129
|
+
emit("ok");
|
|
130
|
+
return result;
|
|
131
|
+
} catch (err) {
|
|
132
|
+
emit("failed", err instanceof Error ? err.message : String(err));
|
|
133
|
+
throw err;
|
|
134
|
+
} finally {
|
|
135
|
+
currentSub = prevSub;
|
|
136
|
+
}
|
|
137
|
+
}, "subProcess");
|
|
138
|
+
return {
|
|
139
|
+
debug: /* @__PURE__ */ __name((m, d) => push("debug", m, d), "debug"),
|
|
140
|
+
info: /* @__PURE__ */ __name((m, d) => push("info", m, d), "info"),
|
|
141
|
+
warn: /* @__PURE__ */ __name((m, d) => push("warn", m, d), "warn"),
|
|
142
|
+
error: /* @__PURE__ */ __name((m, d) => push("error", m, d), "error"),
|
|
143
|
+
sub: /* @__PURE__ */ __name((name, status, message, data) => events.push({
|
|
144
|
+
at: now(),
|
|
145
|
+
level: status === "failed" ? "error" : status === "skipped" ? "warn" : "info",
|
|
146
|
+
message: message ?? name,
|
|
147
|
+
name,
|
|
148
|
+
status,
|
|
149
|
+
...data === void 0 ? {} : {
|
|
150
|
+
data
|
|
151
|
+
}
|
|
152
|
+
}), "sub"),
|
|
153
|
+
subEvent,
|
|
154
|
+
subProcess
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
__name(createStepLogger, "createStepLogger");
|
|
158
|
+
|
|
159
|
+
// src/protocol.ts
|
|
160
|
+
function stepId(runId, seq) {
|
|
161
|
+
return `${runId}:${seq}`;
|
|
162
|
+
}
|
|
163
|
+
__name(stepId, "stepId");
|
|
164
|
+
var breakpointToken = /* @__PURE__ */ __name((runId, seq) => `bp:${runId}:${seq}`, "breakpointToken");
|
|
165
|
+
async function runStepHandler(task, handler) {
|
|
166
|
+
const base = {
|
|
167
|
+
runId: task.runId,
|
|
168
|
+
seq: task.seq,
|
|
169
|
+
stepId: task.stepId,
|
|
170
|
+
startedAt: Date.now()
|
|
171
|
+
};
|
|
172
|
+
if (!handler) {
|
|
173
|
+
return {
|
|
174
|
+
...base,
|
|
175
|
+
status: "failed",
|
|
176
|
+
error: {
|
|
177
|
+
message: `no handler for ${task.name}`,
|
|
178
|
+
retryable: false
|
|
179
|
+
}
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
const events = [];
|
|
183
|
+
const withEvents = /* @__PURE__ */ __name((result) => events.length > 0 ? {
|
|
184
|
+
...result,
|
|
185
|
+
events
|
|
186
|
+
} : result, "withEvents");
|
|
187
|
+
try {
|
|
188
|
+
const output = await handler(task.input, createStepLogger(events, Date.now));
|
|
189
|
+
return withEvents({
|
|
190
|
+
...base,
|
|
191
|
+
status: "completed",
|
|
192
|
+
output
|
|
193
|
+
});
|
|
194
|
+
} catch (err) {
|
|
195
|
+
const e = err;
|
|
196
|
+
return withEvents({
|
|
197
|
+
...base,
|
|
198
|
+
status: "failed",
|
|
199
|
+
error: {
|
|
200
|
+
message: err instanceof Error ? err.message : String(err),
|
|
201
|
+
...typeof e?.code === "string" ? {
|
|
202
|
+
code: e.code
|
|
203
|
+
} : {},
|
|
204
|
+
...typeof e?.retryable === "boolean" ? {
|
|
205
|
+
retryable: e.retryable
|
|
206
|
+
} : {}
|
|
207
|
+
}
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
__name(runStepHandler, "runStepHandler");
|
|
212
|
+
|
|
213
|
+
// src/checkpoints.ts
|
|
214
|
+
function instantCheckpoint(p) {
|
|
215
|
+
const at = /* @__PURE__ */ new Date();
|
|
216
|
+
return {
|
|
217
|
+
runId: p.runId,
|
|
218
|
+
seq: p.seq,
|
|
219
|
+
name: p.name,
|
|
220
|
+
kind: p.kind,
|
|
221
|
+
stepId: stepId(p.runId, p.seq),
|
|
222
|
+
status: p.status ?? "completed",
|
|
223
|
+
output: p.output,
|
|
224
|
+
wakeAt: p.wakeAt,
|
|
225
|
+
attempts: 1,
|
|
226
|
+
enqueuedAt: at,
|
|
227
|
+
startedAt: at,
|
|
228
|
+
finishedAt: at
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
__name(instantCheckpoint, "instantCheckpoint");
|
|
232
|
+
|
|
233
|
+
// src/errors.ts
|
|
234
|
+
var FatalError = class extends Error {
|
|
235
|
+
static {
|
|
236
|
+
__name(this, "FatalError");
|
|
237
|
+
}
|
|
238
|
+
code;
|
|
239
|
+
constructor(message, code) {
|
|
240
|
+
super(message);
|
|
241
|
+
this.name = "FatalError";
|
|
242
|
+
this.code = code;
|
|
243
|
+
}
|
|
244
|
+
};
|
|
245
|
+
var SingletonQueueFullError = class extends Error {
|
|
246
|
+
static {
|
|
247
|
+
__name(this, "SingletonQueueFullError");
|
|
248
|
+
}
|
|
249
|
+
workflow;
|
|
250
|
+
key;
|
|
251
|
+
maxQueueDepth;
|
|
252
|
+
constructor(workflow, key, maxQueueDepth) {
|
|
253
|
+
super(`singleton queue for ${workflow} key "${key}" is full (maxQueueDepth=${maxQueueDepth}); retry later`);
|
|
254
|
+
this.name = "SingletonQueueFullError";
|
|
255
|
+
this.workflow = workflow;
|
|
256
|
+
this.key = key;
|
|
257
|
+
this.maxQueueDepth = maxQueueDepth;
|
|
258
|
+
}
|
|
259
|
+
};
|
|
260
|
+
var SignalTimeoutError = class extends Error {
|
|
261
|
+
static {
|
|
262
|
+
__name(this, "SignalTimeoutError");
|
|
263
|
+
}
|
|
264
|
+
token;
|
|
265
|
+
timeoutMs;
|
|
266
|
+
constructor(token, timeoutMs) {
|
|
267
|
+
super(`timed out after ${timeoutMs}ms waiting for signal "${token}"`);
|
|
268
|
+
this.name = "SignalTimeoutError";
|
|
269
|
+
this.token = token;
|
|
270
|
+
this.timeoutMs = timeoutMs;
|
|
271
|
+
}
|
|
272
|
+
};
|
|
273
|
+
var RemoteStepTimeout = class extends Error {
|
|
274
|
+
static {
|
|
275
|
+
__name(this, "RemoteStepTimeout");
|
|
276
|
+
}
|
|
277
|
+
stepId;
|
|
278
|
+
timeoutMs;
|
|
279
|
+
constructor(stepId2, timeoutMs) {
|
|
280
|
+
super(`remote step ${stepId2} produced no result/heartbeat within ${timeoutMs}ms`);
|
|
281
|
+
this.name = "RemoteStepTimeout";
|
|
282
|
+
this.stepId = stepId2;
|
|
283
|
+
this.timeoutMs = timeoutMs;
|
|
284
|
+
}
|
|
285
|
+
};
|
|
286
|
+
var NonDeterminismError = class extends Error {
|
|
287
|
+
static {
|
|
288
|
+
__name(this, "NonDeterminismError");
|
|
289
|
+
}
|
|
290
|
+
runId;
|
|
291
|
+
seq;
|
|
292
|
+
constructor(runId, seq, expected, recorded) {
|
|
293
|
+
super(`non-determinism at ${runId}#${seq}: code expects "${expected}" but history recorded "${recorded}". The workflow changed under an in-flight run \u2014 register a new @Workflow version.`);
|
|
294
|
+
this.name = "NonDeterminismError";
|
|
295
|
+
this.runId = runId;
|
|
296
|
+
this.seq = seq;
|
|
297
|
+
}
|
|
298
|
+
};
|
|
299
|
+
var WorkflowSuspended = class extends Error {
|
|
300
|
+
static {
|
|
301
|
+
__name(this, "WorkflowSuspended");
|
|
302
|
+
}
|
|
303
|
+
/** Epoch ms to auto-resume (durable sleep), or undefined when waiting on an external signal. */
|
|
304
|
+
wakeAt;
|
|
305
|
+
constructor(wakeAt) {
|
|
306
|
+
super("workflow suspended");
|
|
307
|
+
this.name = "WorkflowSuspended";
|
|
308
|
+
this.wakeAt = wakeAt;
|
|
309
|
+
}
|
|
310
|
+
};
|
|
311
|
+
var ContinueAsNew = class extends Error {
|
|
312
|
+
static {
|
|
313
|
+
__name(this, "ContinueAsNew");
|
|
314
|
+
}
|
|
315
|
+
input;
|
|
316
|
+
constructor(input) {
|
|
317
|
+
super("workflow continued as new"), this.input = input;
|
|
318
|
+
this.name = "ContinueAsNew";
|
|
319
|
+
}
|
|
320
|
+
};
|
|
321
|
+
|
|
322
|
+
// src/entities.ts
|
|
323
|
+
var ENTITY_WORKFLOW = "__entity";
|
|
324
|
+
var entityRunId = /* @__PURE__ */ __name((name, key) => `entity:${name}:${key}`, "entityRunId");
|
|
325
|
+
var Entities = class {
|
|
326
|
+
static {
|
|
327
|
+
__name(this, "Entities");
|
|
328
|
+
}
|
|
329
|
+
host;
|
|
330
|
+
configs = /* @__PURE__ */ new Map();
|
|
331
|
+
constructor(host) {
|
|
332
|
+
this.host = host;
|
|
333
|
+
host.register(ENTITY_WORKFLOW, "1", (ctx, input) => this.run(ctx, input));
|
|
334
|
+
}
|
|
335
|
+
register(name, config) {
|
|
336
|
+
this.configs.set(name, config);
|
|
337
|
+
}
|
|
338
|
+
/** Deliver an op to an entity's runner, starting it on first contact. `reply` (a signal token)
|
|
339
|
+
* receives the handler's result, for `ctx.callEntity`. */
|
|
340
|
+
dispatch(name, key, op, arg, reply) {
|
|
341
|
+
const runId = entityRunId(name, key);
|
|
342
|
+
return this.host.signalWithStart(ENTITY_WORKFLOW, {
|
|
343
|
+
name,
|
|
344
|
+
key
|
|
345
|
+
}, runId, {
|
|
346
|
+
token: runId,
|
|
347
|
+
payload: {
|
|
348
|
+
op,
|
|
349
|
+
arg,
|
|
350
|
+
reply
|
|
351
|
+
}
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
async signal(name, key, op, arg) {
|
|
355
|
+
await this.dispatch(name, key, op, arg);
|
|
356
|
+
}
|
|
357
|
+
getState(name, key) {
|
|
358
|
+
return this.host.getEvent(entityRunId(name, key), "state");
|
|
359
|
+
}
|
|
360
|
+
/** The built-in runner: one long-lived run per key, processing ops serially over state. */
|
|
361
|
+
async run(ctx, input) {
|
|
362
|
+
const { name, key } = input;
|
|
363
|
+
const config = this.configs.get(name);
|
|
364
|
+
if (!config) throw new FatalError(`entity "${name}" is not registered`);
|
|
365
|
+
const token = entityRunId(name, key);
|
|
366
|
+
let state = config.initialState();
|
|
367
|
+
for (let i = 0; ; i += 1) {
|
|
368
|
+
const msg = await ctx.waitForSignal(token);
|
|
369
|
+
const out = await ctx.step(`op:${i}`, async () => {
|
|
370
|
+
const handler = config.handlers[msg.op];
|
|
371
|
+
if (!handler) throw new FatalError(`entity "${name}" has no handler for op "${msg.op}"`);
|
|
372
|
+
const result = await handler(state, msg.arg);
|
|
373
|
+
return {
|
|
374
|
+
result,
|
|
375
|
+
state
|
|
376
|
+
};
|
|
377
|
+
});
|
|
378
|
+
state = out.state;
|
|
379
|
+
await ctx.setEvent("state", state);
|
|
380
|
+
if (msg.reply) {
|
|
381
|
+
await ctx.step(`reply:${i}`, async () => {
|
|
382
|
+
await this.host.signal(msg.reply, out.result);
|
|
383
|
+
});
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
};
|
|
388
|
+
|
|
389
|
+
// src/event-accumulators.ts
|
|
390
|
+
var accToken = /* @__PURE__ */ __name((target) => `__evtacc__:${target}`, "accToken");
|
|
391
|
+
var EventAccumulators = class {
|
|
392
|
+
static {
|
|
393
|
+
__name(this, "EventAccumulators");
|
|
394
|
+
}
|
|
395
|
+
host;
|
|
396
|
+
constructor(host) {
|
|
397
|
+
this.host = host;
|
|
398
|
+
host.register("__evt_debounce", "1", (ctx, input) => this.debounce(ctx, input));
|
|
399
|
+
host.register("__evt_batch", "1", (ctx, input) => this.batch(ctx, input));
|
|
400
|
+
}
|
|
401
|
+
/** Route an `onEvent` payload into its target's debounce/batch accumulator. */
|
|
402
|
+
route(target, config, payload) {
|
|
403
|
+
const [workflow, input] = config.mode === "debounce" ? [
|
|
404
|
+
"__evt_debounce",
|
|
405
|
+
{
|
|
406
|
+
target,
|
|
407
|
+
windowMs: config.windowMs
|
|
408
|
+
}
|
|
409
|
+
] : [
|
|
410
|
+
"__evt_batch",
|
|
411
|
+
{
|
|
412
|
+
target,
|
|
413
|
+
maxSize: config.maxSize,
|
|
414
|
+
windowMs: config.windowMs
|
|
415
|
+
}
|
|
416
|
+
];
|
|
417
|
+
return this.host.signalWithStart(workflow, input, accToken(target), {
|
|
418
|
+
token: accToken(target),
|
|
419
|
+
payload
|
|
420
|
+
});
|
|
421
|
+
}
|
|
422
|
+
/** Fire the target with the LAST payload once events have been quiet for `windowMs`. */
|
|
423
|
+
async debounce(ctx, input) {
|
|
424
|
+
const { target, windowMs } = input;
|
|
425
|
+
const token = accToken(target);
|
|
426
|
+
let last = await ctx.waitForSignal(token);
|
|
427
|
+
for (; ; ) {
|
|
428
|
+
try {
|
|
429
|
+
last = await ctx.waitForSignal(token, {
|
|
430
|
+
timeoutMs: windowMs
|
|
431
|
+
});
|
|
432
|
+
} catch (e) {
|
|
433
|
+
if (e instanceof SignalTimeoutError) break;
|
|
434
|
+
throw e;
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
await ctx.startChild(target, last);
|
|
438
|
+
return ctx.continueAsNew(input);
|
|
439
|
+
}
|
|
440
|
+
/** Fire the target with all payloads (`{ events }`) once `maxSize` or `windowMs` from the first. */
|
|
441
|
+
async batch(ctx, input) {
|
|
442
|
+
const { target, maxSize, windowMs } = input;
|
|
443
|
+
const token = accToken(target);
|
|
444
|
+
const events = [
|
|
445
|
+
await ctx.waitForSignal(token)
|
|
446
|
+
];
|
|
447
|
+
const deadline = await ctx.now() + windowMs;
|
|
448
|
+
while (events.length < maxSize) {
|
|
449
|
+
const remaining = deadline - await ctx.now();
|
|
450
|
+
if (remaining <= 0) break;
|
|
451
|
+
try {
|
|
452
|
+
events.push(await ctx.waitForSignal(token, {
|
|
453
|
+
timeoutMs: remaining
|
|
454
|
+
}));
|
|
455
|
+
} catch (e) {
|
|
456
|
+
if (e instanceof SignalTimeoutError) break;
|
|
457
|
+
throw e;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
await ctx.startChild(target, {
|
|
461
|
+
events
|
|
462
|
+
});
|
|
463
|
+
return ctx.continueAsNew(input);
|
|
464
|
+
}
|
|
465
|
+
};
|
|
466
|
+
|
|
467
|
+
// src/events.ts
|
|
468
|
+
var enc = /* @__PURE__ */ __name((s) => Buffer.from(s, "utf8").toString("base64"), "enc");
|
|
469
|
+
var dec = /* @__PURE__ */ __name((s) => Buffer.from(s, "base64").toString("utf8"), "dec");
|
|
470
|
+
function eventPrefix(name) {
|
|
471
|
+
return `event:${enc(name)}:`;
|
|
472
|
+
}
|
|
473
|
+
__name(eventPrefix, "eventPrefix");
|
|
474
|
+
function eventToken(name, match, runId, seq) {
|
|
475
|
+
return `${eventPrefix(name)}${enc(JSON.stringify(match ?? {}))}:${runId}#${seq}`;
|
|
476
|
+
}
|
|
477
|
+
__name(eventToken, "eventToken");
|
|
478
|
+
function eventMatchOf(token) {
|
|
479
|
+
try {
|
|
480
|
+
return JSON.parse(dec(token.split(":")[2] ?? ""));
|
|
481
|
+
} catch {
|
|
482
|
+
return {};
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
__name(eventMatchOf, "eventMatchOf");
|
|
486
|
+
function eventMatches(payload, match) {
|
|
487
|
+
const keys = Object.keys(match);
|
|
488
|
+
if (keys.length === 0) return true;
|
|
489
|
+
if (!payload || typeof payload !== "object") return false;
|
|
490
|
+
const p = payload;
|
|
491
|
+
return keys.every((k) => JSON.stringify(p[k]) === JSON.stringify(match[k]));
|
|
492
|
+
}
|
|
493
|
+
__name(eventMatches, "eventMatches");
|
|
494
|
+
|
|
495
|
+
// src/queue.ts
|
|
496
|
+
var QueueController = class {
|
|
497
|
+
static {
|
|
498
|
+
__name(this, "QueueController");
|
|
499
|
+
}
|
|
500
|
+
config;
|
|
501
|
+
clock;
|
|
502
|
+
inFlight = 0;
|
|
503
|
+
windowStart = 0;
|
|
504
|
+
windowCount = 0;
|
|
505
|
+
/** Registered waiters by id (only used when priority/fairness ordering is in play). */
|
|
506
|
+
waiters = /* @__PURE__ */ new Map();
|
|
507
|
+
/** Monotonic counter stamping each newly-seen waiter for a stable FIFO tiebreak. */
|
|
508
|
+
waiterSeq = 0;
|
|
509
|
+
/** Per-key "last served" order for round-robin fairness — higher = more recently served. */
|
|
510
|
+
keyServed = /* @__PURE__ */ new Map();
|
|
511
|
+
servedTick = 0;
|
|
512
|
+
constructor(config, clock) {
|
|
513
|
+
this.config = config;
|
|
514
|
+
this.clock = clock;
|
|
515
|
+
}
|
|
516
|
+
/** Whether this controller does any priority/fairness ordering (else it's a plain FIFO gate). */
|
|
517
|
+
get ordered() {
|
|
518
|
+
return this.config.fairness === "key";
|
|
519
|
+
}
|
|
520
|
+
/**
|
|
521
|
+
* Try to admit one unit of work. On `ok`, the caller has taken a concurrency slot and must call
|
|
522
|
+
* {@link release} when the step settles. On block, `retryAt` is when admission may next succeed.
|
|
523
|
+
*
|
|
524
|
+
* `item` carries optional priority/fairness hints; omitting it is the legacy FIFO path. When
|
|
525
|
+
* ordering is configured (or a priority is given), a blocked call REGISTERS as a waiter and is only
|
|
526
|
+
* admitted on a later retry once it is the rightful next under (priority desc, fairness, FIFO).
|
|
527
|
+
*/
|
|
528
|
+
tryAdmit(item) {
|
|
529
|
+
const now = this.clock();
|
|
530
|
+
const rl = this.config.rateLimit;
|
|
531
|
+
if (rl) {
|
|
532
|
+
if (now - this.windowStart >= rl.periodMs) {
|
|
533
|
+
this.windowStart = now;
|
|
534
|
+
this.windowCount = 0;
|
|
535
|
+
}
|
|
536
|
+
if (this.windowCount >= rl.limit) return {
|
|
537
|
+
ok: false,
|
|
538
|
+
retryAt: this.windowStart + rl.periodMs
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
const usingOrder = this.ordered || item?.priority != null || item?.waiterId != null;
|
|
542
|
+
const waiterId = item?.waiterId;
|
|
543
|
+
const full = this.config.concurrency != null && this.inFlight >= this.config.concurrency;
|
|
544
|
+
if (full) {
|
|
545
|
+
if (usingOrder && waiterId != null) this.register(waiterId, item);
|
|
546
|
+
return {
|
|
547
|
+
ok: false,
|
|
548
|
+
retryAt: now + (this.config.retryMs ?? 1e3)
|
|
549
|
+
};
|
|
550
|
+
}
|
|
551
|
+
if (usingOrder && this.waiters.size > 0) {
|
|
552
|
+
if (waiterId != null) this.register(waiterId, item);
|
|
553
|
+
if (!this.isNext(waiterId, item)) {
|
|
554
|
+
return {
|
|
555
|
+
ok: false,
|
|
556
|
+
retryAt: now + (this.config.retryMs ?? 1e3)
|
|
557
|
+
};
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
if (rl) this.windowCount += 1;
|
|
561
|
+
this.inFlight += 1;
|
|
562
|
+
if (waiterId != null) this.waiters.delete(waiterId);
|
|
563
|
+
if (item?.key != null) this.keyServed.set(item.key, ++this.servedTick);
|
|
564
|
+
return {
|
|
565
|
+
ok: true
|
|
566
|
+
};
|
|
567
|
+
}
|
|
568
|
+
/** Release a concurrency slot taken by a successful {@link tryAdmit}. */
|
|
569
|
+
release() {
|
|
570
|
+
if (this.inFlight > 0) this.inFlight -= 1;
|
|
571
|
+
}
|
|
572
|
+
/** Track (or refresh) a blocked call as a waiter; its FIFO seq is stamped only on first sight. */
|
|
573
|
+
register(waiterId, item) {
|
|
574
|
+
const existing = this.waiters.get(waiterId);
|
|
575
|
+
if (existing) {
|
|
576
|
+
existing.priority = item?.priority ?? 0;
|
|
577
|
+
existing.key = item?.key;
|
|
578
|
+
return;
|
|
579
|
+
}
|
|
580
|
+
this.waiters.set(waiterId, {
|
|
581
|
+
priority: item?.priority ?? 0,
|
|
582
|
+
key: item?.key,
|
|
583
|
+
seq: this.waiterSeq++
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
/**
|
|
587
|
+
* Is the calling item the best candidate among all registered waiters? Ordering, best first:
|
|
588
|
+
* 1. higher `priority`;
|
|
589
|
+
* 2. fairness: with `fairness: 'key'`, the least-recently-served key wins (round-robin);
|
|
590
|
+
* 3. earliest registration (FIFO) — stable tiebreak.
|
|
591
|
+
* A bare item (no waiterId) is compared on its hints directly.
|
|
592
|
+
*/
|
|
593
|
+
isNext(waiterId, item) {
|
|
594
|
+
const self = (waiterId != null ? this.waiters.get(waiterId) : void 0) ?? {
|
|
595
|
+
priority: item?.priority ?? 0,
|
|
596
|
+
key: item?.key,
|
|
597
|
+
seq: Number.POSITIVE_INFINITY
|
|
598
|
+
};
|
|
599
|
+
let best = self;
|
|
600
|
+
for (const w of this.waiters.values()) {
|
|
601
|
+
if (this.better(w, best)) best = w;
|
|
602
|
+
}
|
|
603
|
+
return best === self;
|
|
604
|
+
}
|
|
605
|
+
/** Strict "a should be admitted before b" comparison under the active policy. */
|
|
606
|
+
better(a, b) {
|
|
607
|
+
if (a.priority !== b.priority) return a.priority > b.priority;
|
|
608
|
+
if (this.config.fairness === "key") {
|
|
609
|
+
const sa = this.servedOf(a.key);
|
|
610
|
+
const sb = this.servedOf(b.key);
|
|
611
|
+
if (sa !== sb) return sa < sb;
|
|
612
|
+
}
|
|
613
|
+
return a.seq < b.seq;
|
|
614
|
+
}
|
|
615
|
+
/** Last-served tick for a key (0 = never served, so an unseen key is admitted before a busy one). */
|
|
616
|
+
servedOf(key) {
|
|
617
|
+
if (key == null) return 0;
|
|
618
|
+
return this.keyServed.get(key) ?? 0;
|
|
619
|
+
}
|
|
620
|
+
};
|
|
621
|
+
|
|
622
|
+
// src/transport-pool.ts
|
|
623
|
+
var TransportPool = class {
|
|
624
|
+
static {
|
|
625
|
+
__name(this, "TransportPool");
|
|
626
|
+
}
|
|
627
|
+
transports;
|
|
628
|
+
constructor(transports) {
|
|
629
|
+
this.transports = transports;
|
|
630
|
+
}
|
|
631
|
+
get size() {
|
|
632
|
+
return this.transports.length;
|
|
633
|
+
}
|
|
634
|
+
/** Register the engine's result/heartbeat/step-event handlers on EVERY transport — a result can come
|
|
635
|
+
* back on whichever one delivered the task, so failover stays symmetric. `onStepEvent` is wired only
|
|
636
|
+
* on transports that carry streamed workflow step lifecycle (BullMQ); others simply skip it. */
|
|
637
|
+
bind(onResult, onHeartbeat, onStepEvent) {
|
|
638
|
+
for (const { transport } of this.transports) {
|
|
639
|
+
transport.onResult(onResult);
|
|
640
|
+
transport.onHeartbeat(onHeartbeat);
|
|
641
|
+
if (onStepEvent && transport.onStepEvent) {
|
|
642
|
+
transport.onStepEvent(onStepEvent);
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
/**
|
|
647
|
+
* Dispatch `task` over the pool: try the preferred (or first) transport, fail over to the next on
|
|
648
|
+
* a dispatch error, and stamp the task with the id of the transport that accepted it (so a worker
|
|
649
|
+
* replies on the matching one). Throws if every transport fails.
|
|
650
|
+
*/
|
|
651
|
+
async dispatch(task, preferId) {
|
|
652
|
+
let lastErr;
|
|
653
|
+
for (const { id, transport } of this.ordered(preferId)) {
|
|
654
|
+
try {
|
|
655
|
+
await transport.dispatch({
|
|
656
|
+
...task,
|
|
657
|
+
transport: id
|
|
658
|
+
});
|
|
659
|
+
return;
|
|
660
|
+
} catch (err) {
|
|
661
|
+
lastErr = err;
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
throw lastErr ?? new Error("no transport accepted the dispatch");
|
|
665
|
+
}
|
|
666
|
+
/** Worker-health for `group`, merged across every transport that can report it (a group's queue
|
|
667
|
+
* and its workers may live on any pinned transport). Undefined when no transport implements
|
|
668
|
+
* `groupHealth` (e.g. a pure in-process transport — nothing to introspect). */
|
|
669
|
+
async groupHealth(group) {
|
|
670
|
+
const reports = [];
|
|
671
|
+
for (const { transport } of this.transports) {
|
|
672
|
+
if (transport.groupHealth) reports.push(await transport.groupHealth(group));
|
|
673
|
+
}
|
|
674
|
+
if (reports.length === 0) return void 0;
|
|
675
|
+
return {
|
|
676
|
+
group,
|
|
677
|
+
depth: reports.reduce((sum, r) => sum + r.depth, 0),
|
|
678
|
+
liveWorkers: reports.flatMap((r) => r.liveWorkers)
|
|
679
|
+
};
|
|
680
|
+
}
|
|
681
|
+
/** Distinct worker groups with a live heartbeat, merged across every transport that can report it. */
|
|
682
|
+
async listWorkerGroups() {
|
|
683
|
+
const groups = /* @__PURE__ */ new Set();
|
|
684
|
+
for (const { transport } of this.transports) {
|
|
685
|
+
if (transport.listWorkerGroups) {
|
|
686
|
+
for (const g of await transport.listWorkerGroups()) groups.add(g);
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
return [
|
|
690
|
+
...groups
|
|
691
|
+
];
|
|
692
|
+
}
|
|
693
|
+
/** Pinned `preferId` first, then the rest (failover order). */
|
|
694
|
+
ordered(preferId) {
|
|
695
|
+
if (!preferId) return this.transports;
|
|
696
|
+
const pref = this.transports.filter((t) => t.id === preferId);
|
|
697
|
+
if (pref.length === 0) throw new Error(`transport "${preferId}" is not registered`);
|
|
698
|
+
return [
|
|
699
|
+
...pref,
|
|
700
|
+
...this.transports.filter((t) => t.id !== preferId)
|
|
701
|
+
];
|
|
702
|
+
}
|
|
703
|
+
};
|
|
704
|
+
|
|
705
|
+
// src/completion.ts
|
|
706
|
+
function unwrapCompletion(payload, label) {
|
|
707
|
+
const c = payload;
|
|
708
|
+
if (c && typeof c === "object" && "ok" in c && c.ok === false) {
|
|
709
|
+
throw new FatalError(`${label} failed: ${c.error}`);
|
|
710
|
+
}
|
|
711
|
+
return c?.value;
|
|
712
|
+
}
|
|
713
|
+
__name(unwrapCompletion, "unwrapCompletion");
|
|
714
|
+
|
|
715
|
+
// src/workflow-ref.ts
|
|
716
|
+
var WORKFLOW_NAME_KEY = Symbol.for("nestjs-durable:workflow-name");
|
|
717
|
+
function workflowName(ref) {
|
|
718
|
+
if (typeof ref === "string") return ref;
|
|
719
|
+
const name = ref[WORKFLOW_NAME_KEY];
|
|
720
|
+
if (!name) {
|
|
721
|
+
throw new Error(`workflow class ${ref.name} has no registered name \u2014 is it decorated with @Workflow({ name })?`);
|
|
722
|
+
}
|
|
723
|
+
return name;
|
|
724
|
+
}
|
|
725
|
+
__name(workflowName, "workflowName");
|
|
726
|
+
|
|
727
|
+
// src/workflow-ctx.ts
|
|
728
|
+
var Position = class Position2 {
|
|
729
|
+
static {
|
|
730
|
+
__name(this, "Position");
|
|
731
|
+
}
|
|
732
|
+
seq = -1;
|
|
733
|
+
next() {
|
|
734
|
+
this.seq += 1;
|
|
735
|
+
return this.seq;
|
|
736
|
+
}
|
|
737
|
+
rewind() {
|
|
738
|
+
this.seq -= 1;
|
|
739
|
+
}
|
|
740
|
+
};
|
|
741
|
+
function createWorkflowCtx(host, runId, compensations, workflow = "") {
|
|
742
|
+
const { store, replay } = host;
|
|
743
|
+
const pos = new Position();
|
|
744
|
+
const readCheckpoint = /* @__PURE__ */ __name((seq) => {
|
|
745
|
+
const hit = replay?.get(seq);
|
|
746
|
+
if (hit !== void 0) return hit;
|
|
747
|
+
return store.getCheckpoint(runId, seq);
|
|
748
|
+
}, "readCheckpoint");
|
|
749
|
+
const writeCheckpoint = /* @__PURE__ */ __name(async (cp) => {
|
|
750
|
+
await store.saveCheckpoint(cp);
|
|
751
|
+
replay?.set(cp.seq, cp);
|
|
752
|
+
}, "writeCheckpoint");
|
|
753
|
+
const step = /* @__PURE__ */ __name(async (name, fn, options) => {
|
|
754
|
+
const current = pos.next();
|
|
755
|
+
const existing = await readCheckpoint(current);
|
|
756
|
+
if (existing && existing.name !== name) {
|
|
757
|
+
throw new NonDeterminismError(runId, current, name, existing.name);
|
|
758
|
+
}
|
|
759
|
+
if (existing && existing.status === "completed") {
|
|
760
|
+
if (options?.compensate) compensations.push({
|
|
761
|
+
name,
|
|
762
|
+
fn: options.compensate
|
|
763
|
+
});
|
|
764
|
+
return existing.output;
|
|
765
|
+
}
|
|
766
|
+
const maxAttempts = Math.max(1, options?.retries ?? 1);
|
|
767
|
+
const startedAt = /* @__PURE__ */ new Date();
|
|
768
|
+
await host.startStep({
|
|
769
|
+
runId,
|
|
770
|
+
seq: current,
|
|
771
|
+
name,
|
|
772
|
+
kind: "local",
|
|
773
|
+
attempts: 1,
|
|
774
|
+
enqueuedAt: startedAt,
|
|
775
|
+
startedAt
|
|
776
|
+
});
|
|
777
|
+
for (let attempt = 1; ; attempt += 1) {
|
|
778
|
+
const events = [];
|
|
779
|
+
try {
|
|
780
|
+
const invocation = {
|
|
781
|
+
runId,
|
|
782
|
+
workflow,
|
|
783
|
+
stepName: name,
|
|
784
|
+
seq: current,
|
|
785
|
+
attempt
|
|
786
|
+
};
|
|
787
|
+
const body = /* @__PURE__ */ __name(() => fn(createStepLogger(events, host.clock)), "body");
|
|
788
|
+
const output = host.interceptStep ? await host.interceptStep(invocation, body) : await body();
|
|
789
|
+
await host.completeStep({
|
|
790
|
+
runId,
|
|
791
|
+
seq: current,
|
|
792
|
+
name,
|
|
793
|
+
kind: "local",
|
|
794
|
+
output,
|
|
795
|
+
events,
|
|
796
|
+
attempts: attempt,
|
|
797
|
+
enqueuedAt: startedAt,
|
|
798
|
+
startedAt
|
|
799
|
+
});
|
|
800
|
+
if (options?.compensate) compensations.push({
|
|
801
|
+
name,
|
|
802
|
+
fn: options.compensate
|
|
803
|
+
});
|
|
804
|
+
return output;
|
|
805
|
+
} catch (err) {
|
|
806
|
+
if (err instanceof FatalError || attempt >= maxAttempts) {
|
|
807
|
+
await host.failStep({
|
|
808
|
+
runId,
|
|
809
|
+
seq: current,
|
|
810
|
+
name,
|
|
811
|
+
kind: "local",
|
|
812
|
+
error: {
|
|
813
|
+
message: err instanceof Error ? err.message : String(err),
|
|
814
|
+
stack: err instanceof Error ? err.stack : void 0
|
|
815
|
+
},
|
|
816
|
+
events,
|
|
817
|
+
attempts: attempt,
|
|
818
|
+
enqueuedAt: startedAt,
|
|
819
|
+
startedAt
|
|
820
|
+
});
|
|
821
|
+
throw err;
|
|
822
|
+
}
|
|
823
|
+
const wait = backoffDelay(attempt, options);
|
|
824
|
+
if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait));
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
}, "step");
|
|
828
|
+
const transaction = /* @__PURE__ */ __name(async (name, fn) => {
|
|
829
|
+
if (!store.transaction) {
|
|
830
|
+
throw new Error("ctx.transaction needs a store that supports transactions (the SQL adapters do). Use ctx.step for non-transactional work.");
|
|
831
|
+
}
|
|
832
|
+
const current = pos.next();
|
|
833
|
+
const existing = await readCheckpoint(current);
|
|
834
|
+
if (existing && existing.name !== name) {
|
|
835
|
+
throw new NonDeterminismError(runId, current, name, existing.name);
|
|
836
|
+
}
|
|
837
|
+
if (existing && existing.status === "completed") return existing.output;
|
|
838
|
+
return store.transaction(async (tx) => {
|
|
839
|
+
const output = await fn(tx.raw);
|
|
840
|
+
const cp = instantCheckpoint({
|
|
841
|
+
runId,
|
|
842
|
+
seq: current,
|
|
843
|
+
name,
|
|
844
|
+
kind: "local",
|
|
845
|
+
output
|
|
846
|
+
});
|
|
847
|
+
await tx.saveCheckpoint(cp);
|
|
848
|
+
replay?.set(current, cp);
|
|
849
|
+
return output;
|
|
850
|
+
});
|
|
851
|
+
}, "transaction");
|
|
852
|
+
const suspendUntil = /* @__PURE__ */ __name(async (wakeAt) => {
|
|
853
|
+
const current = pos.next();
|
|
854
|
+
const now2 = host.clock();
|
|
855
|
+
const existing = await readCheckpoint(current);
|
|
856
|
+
if (existing) {
|
|
857
|
+
if (now2 >= (existing.wakeAt ?? 0)) return;
|
|
858
|
+
throw new WorkflowSuspended(existing.wakeAt ?? now2);
|
|
859
|
+
}
|
|
860
|
+
const at = wakeAt();
|
|
861
|
+
await writeCheckpoint(instantCheckpoint({
|
|
862
|
+
runId,
|
|
863
|
+
seq: current,
|
|
864
|
+
name: "sleep",
|
|
865
|
+
kind: "sleep",
|
|
866
|
+
wakeAt: at
|
|
867
|
+
}));
|
|
868
|
+
throw new WorkflowSuspended(at);
|
|
869
|
+
}, "suspendUntil");
|
|
870
|
+
const sleep = /* @__PURE__ */ __name((duration) => suspendUntil(() => host.clock() + parseDuration(duration)), "sleep");
|
|
871
|
+
const sleepUntil = /* @__PURE__ */ __name((when) => suspendUntil(() => typeof when === "number" ? when : when.getTime()), "sleepUntil");
|
|
872
|
+
const continueAsNew = /* @__PURE__ */ __name((input) => {
|
|
873
|
+
throw new ContinueAsNew(input);
|
|
874
|
+
}, "continueAsNew");
|
|
875
|
+
const consumeBuffered = /* @__PURE__ */ __name(async (token, seq) => {
|
|
876
|
+
const buffered = await store.takeBufferedSignal(token);
|
|
877
|
+
if (!buffered) return null;
|
|
878
|
+
await writeCheckpoint(instantCheckpoint({
|
|
879
|
+
runId,
|
|
880
|
+
seq,
|
|
881
|
+
name: `signal:${token}`,
|
|
882
|
+
kind: "signal",
|
|
883
|
+
output: buffered.payload
|
|
884
|
+
}));
|
|
885
|
+
return {
|
|
886
|
+
value: buffered.payload
|
|
887
|
+
};
|
|
888
|
+
}, "consumeBuffered");
|
|
889
|
+
const waitForSignal = /* @__PURE__ */ __name(async (token, opts) => {
|
|
890
|
+
if (opts?.timeoutMs == null) {
|
|
891
|
+
const current = pos.next();
|
|
892
|
+
const existing = await readCheckpoint(current);
|
|
893
|
+
if (existing && existing.status === "completed") return existing.output;
|
|
894
|
+
const buffered2 = await consumeBuffered(token, current);
|
|
895
|
+
if (buffered2) return buffered2.value;
|
|
896
|
+
await store.putSignalWaiter({
|
|
897
|
+
token,
|
|
898
|
+
runId,
|
|
899
|
+
seq: current
|
|
900
|
+
});
|
|
901
|
+
throw new WorkflowSuspended();
|
|
902
|
+
}
|
|
903
|
+
const timeoutMs = opts.timeoutMs;
|
|
904
|
+
const deadlineSeq = pos.next();
|
|
905
|
+
const waitSeq = pos.next();
|
|
906
|
+
const recorded = await readCheckpoint(deadlineSeq);
|
|
907
|
+
const deadline = recorded?.wakeAt ?? host.clock() + timeoutMs;
|
|
908
|
+
if (!recorded) {
|
|
909
|
+
await writeCheckpoint(instantCheckpoint({
|
|
910
|
+
runId,
|
|
911
|
+
seq: deadlineSeq,
|
|
912
|
+
name: `timeout:${token}`,
|
|
913
|
+
kind: "sleep",
|
|
914
|
+
wakeAt: deadline
|
|
915
|
+
}));
|
|
916
|
+
}
|
|
917
|
+
const waited = await readCheckpoint(waitSeq);
|
|
918
|
+
if (waited && waited.status === "completed") return waited.output;
|
|
919
|
+
const buffered = await consumeBuffered(token, waitSeq);
|
|
920
|
+
if (buffered) return buffered.value;
|
|
921
|
+
if (host.clock() >= deadline) {
|
|
922
|
+
await store.takeSignalWaiter(token).catch(() => void 0);
|
|
923
|
+
throw new SignalTimeoutError(token, timeoutMs);
|
|
924
|
+
}
|
|
925
|
+
await store.putSignalWaiter({
|
|
926
|
+
token,
|
|
927
|
+
runId,
|
|
928
|
+
seq: waitSeq
|
|
929
|
+
});
|
|
930
|
+
throw new WorkflowSuspended(deadline);
|
|
931
|
+
}, "waitForSignal");
|
|
932
|
+
const waitForEvent = /* @__PURE__ */ __name(async (name, opts) => {
|
|
933
|
+
if (opts?.timeoutMs == null) {
|
|
934
|
+
const current = pos.next();
|
|
935
|
+
const token2 = eventToken(name, opts?.match, runId, current);
|
|
936
|
+
const existing = await readCheckpoint(current);
|
|
937
|
+
if (existing && existing.status === "completed") return existing.output;
|
|
938
|
+
await store.putSignalWaiter({
|
|
939
|
+
token: token2,
|
|
940
|
+
runId,
|
|
941
|
+
seq: current
|
|
942
|
+
});
|
|
943
|
+
throw new WorkflowSuspended();
|
|
944
|
+
}
|
|
945
|
+
const timeoutMs = opts.timeoutMs;
|
|
946
|
+
const deadlineSeq = pos.next();
|
|
947
|
+
const waitSeq = pos.next();
|
|
948
|
+
const token = eventToken(name, opts.match, runId, waitSeq);
|
|
949
|
+
const recorded = await readCheckpoint(deadlineSeq);
|
|
950
|
+
const deadline = recorded?.wakeAt ?? host.clock() + timeoutMs;
|
|
951
|
+
if (!recorded) {
|
|
952
|
+
await writeCheckpoint(instantCheckpoint({
|
|
953
|
+
runId,
|
|
954
|
+
seq: deadlineSeq,
|
|
955
|
+
name: `timeout:event:${name}`,
|
|
956
|
+
kind: "sleep",
|
|
957
|
+
wakeAt: deadline
|
|
958
|
+
}));
|
|
959
|
+
}
|
|
960
|
+
const waited = await readCheckpoint(waitSeq);
|
|
961
|
+
if (waited && waited.status === "completed") return waited.output;
|
|
962
|
+
if (host.clock() >= deadline) {
|
|
963
|
+
await store.takeSignalWaiter(token).catch(() => void 0);
|
|
964
|
+
throw new SignalTimeoutError(`event:${name}`, timeoutMs);
|
|
965
|
+
}
|
|
966
|
+
await store.putSignalWaiter({
|
|
967
|
+
token,
|
|
968
|
+
runId,
|
|
969
|
+
seq: waitSeq
|
|
970
|
+
});
|
|
971
|
+
throw new WorkflowSuspended(deadline);
|
|
972
|
+
}, "waitForEvent");
|
|
973
|
+
const task = /* @__PURE__ */ __name(async (name, dispatch, options) => {
|
|
974
|
+
await step(`task:dispatch:${name}`, dispatch, options);
|
|
975
|
+
return unwrapCompletion(await waitForSignal(`task:${runId}:${name}`), `task "${name}"`);
|
|
976
|
+
}, "task");
|
|
977
|
+
const child = /* @__PURE__ */ __name(async (workflow2, input, childId) => {
|
|
978
|
+
const current = pos.next();
|
|
979
|
+
const id = childId ?? `${runId}.child.${current}`;
|
|
980
|
+
const existing = await readCheckpoint(current);
|
|
981
|
+
if (existing && existing.status === "completed") {
|
|
982
|
+
return unwrapCompletion(existing.output, `child "${id}"`);
|
|
983
|
+
}
|
|
984
|
+
await store.putSignalWaiter({
|
|
985
|
+
token: `child:${id}`,
|
|
986
|
+
runId,
|
|
987
|
+
seq: current
|
|
988
|
+
});
|
|
989
|
+
if (!await store.getRun(id)) host.startChild(workflowName(workflow2), input, id);
|
|
990
|
+
if (!existing) {
|
|
991
|
+
await writeCheckpoint(instantCheckpoint({
|
|
992
|
+
runId,
|
|
993
|
+
seq: current,
|
|
994
|
+
name: `signal:child:${id}`,
|
|
995
|
+
kind: "signal",
|
|
996
|
+
status: "running"
|
|
997
|
+
}));
|
|
998
|
+
}
|
|
999
|
+
throw new WorkflowSuspended();
|
|
1000
|
+
}, "child");
|
|
1001
|
+
const startChild = /* @__PURE__ */ __name(async (workflow2, input, childId) => {
|
|
1002
|
+
const current = pos.next();
|
|
1003
|
+
const id = childId ?? `${runId}.child.${current}`;
|
|
1004
|
+
const existing = await readCheckpoint(current);
|
|
1005
|
+
if (existing && existing.status === "completed") return existing.output;
|
|
1006
|
+
if (!await store.getRun(id)) host.startChild(workflowName(workflow2), input, id);
|
|
1007
|
+
await writeCheckpoint(instantCheckpoint({
|
|
1008
|
+
runId,
|
|
1009
|
+
seq: current,
|
|
1010
|
+
name: `spawn:${id}`,
|
|
1011
|
+
kind: "local",
|
|
1012
|
+
output: id
|
|
1013
|
+
}));
|
|
1014
|
+
return id;
|
|
1015
|
+
}, "startChild");
|
|
1016
|
+
const callEntity = /* @__PURE__ */ __name(async (name, key, op, arg) => {
|
|
1017
|
+
const current = pos.next();
|
|
1018
|
+
const existing = await readCheckpoint(current);
|
|
1019
|
+
if (existing && existing.status === "completed") return existing.output;
|
|
1020
|
+
const reply = `entityreply:${runId}:${current}`;
|
|
1021
|
+
await store.putSignalWaiter({
|
|
1022
|
+
token: reply,
|
|
1023
|
+
runId,
|
|
1024
|
+
seq: current
|
|
1025
|
+
});
|
|
1026
|
+
host.signalEntity?.(name, key, op, arg, reply);
|
|
1027
|
+
throw new WorkflowSuspended();
|
|
1028
|
+
}, "callEntity");
|
|
1029
|
+
const signalEntity = /* @__PURE__ */ __name(async (name, key, op, arg) => {
|
|
1030
|
+
const current = pos.next();
|
|
1031
|
+
const existing = await readCheckpoint(current);
|
|
1032
|
+
if (existing && existing.status === "completed") return;
|
|
1033
|
+
host.signalEntity?.(name, key, op, arg);
|
|
1034
|
+
await writeCheckpoint(instantCheckpoint({
|
|
1035
|
+
runId,
|
|
1036
|
+
seq: current,
|
|
1037
|
+
name: `entitysig:${name}:${key}`,
|
|
1038
|
+
kind: "local"
|
|
1039
|
+
}));
|
|
1040
|
+
}, "signalEntity");
|
|
1041
|
+
const breakpoint = /* @__PURE__ */ __name(async (label) => {
|
|
1042
|
+
const current = pos.next();
|
|
1043
|
+
const existing = await readCheckpoint(current);
|
|
1044
|
+
if (existing && existing.status === "completed") return;
|
|
1045
|
+
if (!existing) {
|
|
1046
|
+
await writeCheckpoint(instantCheckpoint({
|
|
1047
|
+
runId,
|
|
1048
|
+
seq: current,
|
|
1049
|
+
name: label ? `breakpoint:${label}` : "breakpoint",
|
|
1050
|
+
kind: "signal",
|
|
1051
|
+
status: "pending"
|
|
1052
|
+
}));
|
|
1053
|
+
await store.putSignalWaiter({
|
|
1054
|
+
token: breakpointToken(runId, current),
|
|
1055
|
+
runId,
|
|
1056
|
+
seq: current
|
|
1057
|
+
});
|
|
1058
|
+
}
|
|
1059
|
+
throw new WorkflowSuspended();
|
|
1060
|
+
}, "breakpoint");
|
|
1061
|
+
const patched = /* @__PURE__ */ __name(async (id) => {
|
|
1062
|
+
const marker = `patch:${id}`;
|
|
1063
|
+
const current = pos.next();
|
|
1064
|
+
const existing = await readCheckpoint(current);
|
|
1065
|
+
if (existing) {
|
|
1066
|
+
if (existing.name === marker) return true;
|
|
1067
|
+
if (existing.name.startsWith("patch:")) {
|
|
1068
|
+
throw new NonDeterminismError(runId, current, marker, existing.name);
|
|
1069
|
+
}
|
|
1070
|
+
pos.rewind();
|
|
1071
|
+
return false;
|
|
1072
|
+
}
|
|
1073
|
+
await writeCheckpoint(instantCheckpoint({
|
|
1074
|
+
runId,
|
|
1075
|
+
seq: current,
|
|
1076
|
+
name: marker,
|
|
1077
|
+
kind: "local",
|
|
1078
|
+
output: true
|
|
1079
|
+
}));
|
|
1080
|
+
return true;
|
|
1081
|
+
}, "patched");
|
|
1082
|
+
const onUpdate = /* @__PURE__ */ __name((name, opts) => waitForSignal(`update:${runId}:${name}`, opts), "onUpdate");
|
|
1083
|
+
const setEvent = /* @__PURE__ */ __name(async (key, value) => {
|
|
1084
|
+
const current = pos.next();
|
|
1085
|
+
const name = `event:${key}`;
|
|
1086
|
+
const existing = await readCheckpoint(current);
|
|
1087
|
+
if (existing && existing.name !== name) {
|
|
1088
|
+
throw new NonDeterminismError(runId, current, name, existing.name);
|
|
1089
|
+
}
|
|
1090
|
+
if (existing && existing.status === "completed") return;
|
|
1091
|
+
await writeCheckpoint(instantCheckpoint({
|
|
1092
|
+
runId,
|
|
1093
|
+
seq: current,
|
|
1094
|
+
name,
|
|
1095
|
+
kind: "local",
|
|
1096
|
+
output: value
|
|
1097
|
+
}));
|
|
1098
|
+
}, "setEvent");
|
|
1099
|
+
const webhook = /* @__PURE__ */ __name(() => {
|
|
1100
|
+
const current = pos.next();
|
|
1101
|
+
const token = `wh:${runId}:${current}`;
|
|
1102
|
+
const wait = /* @__PURE__ */ __name(async () => {
|
|
1103
|
+
const existing = await readCheckpoint(current);
|
|
1104
|
+
if (existing && existing.status === "completed") return existing.output;
|
|
1105
|
+
await store.putSignalWaiter({
|
|
1106
|
+
token,
|
|
1107
|
+
runId,
|
|
1108
|
+
seq: current
|
|
1109
|
+
});
|
|
1110
|
+
throw new WorkflowSuspended();
|
|
1111
|
+
}, "wait");
|
|
1112
|
+
return {
|
|
1113
|
+
token,
|
|
1114
|
+
url: host.webhookUrl?.(token),
|
|
1115
|
+
wait
|
|
1116
|
+
};
|
|
1117
|
+
}, "webhook");
|
|
1118
|
+
const now = /* @__PURE__ */ __name(() => step("now", async () => host.clock()), "now");
|
|
1119
|
+
const random = /* @__PURE__ */ __name(() => step("random", async () => Math.random()), "random");
|
|
1120
|
+
const uuid = /* @__PURE__ */ __name(() => step("uuid", async () => globalThis.crypto.randomUUID()), "uuid");
|
|
1121
|
+
return {
|
|
1122
|
+
runId,
|
|
1123
|
+
step,
|
|
1124
|
+
transaction,
|
|
1125
|
+
callEntity,
|
|
1126
|
+
signalEntity,
|
|
1127
|
+
sleep,
|
|
1128
|
+
sleepUntil,
|
|
1129
|
+
continueAsNew,
|
|
1130
|
+
waitForSignal,
|
|
1131
|
+
waitForEvent,
|
|
1132
|
+
task,
|
|
1133
|
+
child,
|
|
1134
|
+
startChild,
|
|
1135
|
+
breakpoint,
|
|
1136
|
+
webhook,
|
|
1137
|
+
setEvent,
|
|
1138
|
+
onUpdate,
|
|
1139
|
+
patched,
|
|
1140
|
+
now,
|
|
1141
|
+
random,
|
|
1142
|
+
uuid,
|
|
1143
|
+
call: /* @__PURE__ */ __name((remote, input, opts) => host.callRemote(runId, pos.next(), remote, input, opts?.queue, opts?.transport, {
|
|
1144
|
+
priority: opts?.priority,
|
|
1145
|
+
fairnessKey: opts?.fairnessKey
|
|
1146
|
+
}), "call")
|
|
1147
|
+
};
|
|
1148
|
+
}
|
|
1149
|
+
__name(createWorkflowCtx, "createWorkflowCtx");
|
|
1150
|
+
|
|
1151
|
+
// src/engine.ts
|
|
1152
|
+
var SINGLETON_RETRY_MS = 1e3;
|
|
1153
|
+
var SINGLETON_RETRY_JITTER_MS = 250;
|
|
1154
|
+
var singletonRetryWakeAt = /* @__PURE__ */ __name((nowMs) => nowMs + SINGLETON_RETRY_MS + Math.floor((Math.random() * 2 - 1) * SINGLETON_RETRY_JITTER_MS), "singletonRetryWakeAt");
|
|
1155
|
+
var singletonTag = /* @__PURE__ */ __name((cfg, input) => `singleton:${cfg.key(input)}`, "singletonTag");
|
|
1156
|
+
function mergeTags(staticTags, runTags) {
|
|
1157
|
+
if (!staticTags?.length && !runTags?.length) return void 0;
|
|
1158
|
+
return [
|
|
1159
|
+
.../* @__PURE__ */ new Set([
|
|
1160
|
+
...staticTags ?? [],
|
|
1161
|
+
...runTags ?? []
|
|
1162
|
+
])
|
|
1163
|
+
];
|
|
1164
|
+
}
|
|
1165
|
+
__name(mergeTags, "mergeTags");
|
|
1166
|
+
var BREAKPOINT = "breakpoint";
|
|
1167
|
+
var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
|
|
1168
|
+
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
1169
|
+
function nextContinuationId(runId) {
|
|
1170
|
+
const m = runId.match(/^(.*)~(\d+)$/);
|
|
1171
|
+
return m ? `${m[1]}~${Number(m[2]) + 1}` : `${runId}~1`;
|
|
1172
|
+
}
|
|
1173
|
+
__name(nextContinuationId, "nextContinuationId");
|
|
1174
|
+
function isNewerVersion(a, b) {
|
|
1175
|
+
const na = Number(a);
|
|
1176
|
+
const nb = Number(b);
|
|
1177
|
+
if (!Number.isNaN(na) && !Number.isNaN(nb)) return na > nb;
|
|
1178
|
+
return a.localeCompare(b, void 0, {
|
|
1179
|
+
numeric: true
|
|
1180
|
+
}) > 0;
|
|
1181
|
+
}
|
|
1182
|
+
__name(isNewerVersion, "isNewerVersion");
|
|
1183
|
+
var WorkflowEngine = class {
|
|
1184
|
+
static {
|
|
1185
|
+
__name(this, "WorkflowEngine");
|
|
1186
|
+
}
|
|
1187
|
+
store;
|
|
1188
|
+
/** Ordered transport pool (dispatch + failover). Empty = no remote steps. */
|
|
1189
|
+
pool;
|
|
1190
|
+
controlPlane;
|
|
1191
|
+
clock;
|
|
1192
|
+
instanceId;
|
|
1193
|
+
leaseMs;
|
|
1194
|
+
maxRecoveryAttempts;
|
|
1195
|
+
webhookUrl;
|
|
1196
|
+
traceparent;
|
|
1197
|
+
context;
|
|
1198
|
+
/** Establish the originating context ambiently around a local step body (see {@link WorkflowEngineDeps.rehydrate}). Default passthrough. */
|
|
1199
|
+
rehydrate;
|
|
1200
|
+
compensationRetries;
|
|
1201
|
+
/** Persist a `running` checkpoint at the start of a local step body (see {@link WorkflowEngineDeps.trackStepStart}). */
|
|
1202
|
+
trackStepStart;
|
|
1203
|
+
/** Where a freshly-started run executes — in-process by default (see {@link RunDispatcher}). */
|
|
1204
|
+
runDispatcher;
|
|
1205
|
+
/** Every registered workflow, keyed by `name@version` — so old versions stay runnable. */
|
|
1206
|
+
workflows = /* @__PURE__ */ new Map();
|
|
1207
|
+
/** The newest registered version per workflow name — used to `start` new runs. */
|
|
1208
|
+
latest = /* @__PURE__ */ new Map();
|
|
1209
|
+
/** Event name → workflow names started when that event is published (see `onEvent`). */
|
|
1210
|
+
eventTriggers = /* @__PURE__ */ new Map();
|
|
1211
|
+
/** Durable-entity subsystem (registers the `__entity` runner; see `registerEntity`). */
|
|
1212
|
+
entities;
|
|
1213
|
+
/** Event debounce/batch accumulators (register the `__evt_*` runners; see `accumulators.route`). */
|
|
1214
|
+
accumulators;
|
|
1215
|
+
/** In-flight remote steps awaiting a worker result, keyed by stepId. */
|
|
1216
|
+
pending = /* @__PURE__ */ new Map();
|
|
1217
|
+
/** Per-step "reset the liveness timer" callbacks, called when a heartbeat arrives. */
|
|
1218
|
+
heartbeatResets = /* @__PURE__ */ new Map();
|
|
1219
|
+
listeners = /* @__PURE__ */ new Set();
|
|
1220
|
+
/** Step interceptors (onion middleware around real local-step execution), first = outermost. */
|
|
1221
|
+
interceptors = [];
|
|
1222
|
+
/** Callbacks notified (on any instance) when a run is cancelled — for cooperative cancellation. */
|
|
1223
|
+
cancelListeners = /* @__PURE__ */ new Set();
|
|
1224
|
+
/** Callbacks notified when a run is enqueued elsewhere — for low-latency cross-pod dispatch. */
|
|
1225
|
+
enqueuedListeners = /* @__PURE__ */ new Set();
|
|
1226
|
+
/** Notified when a run is dead-lettered (moved to `dead`) — a hook for a DLQ handler. */
|
|
1227
|
+
deadListeners = /* @__PURE__ */ new Set();
|
|
1228
|
+
/** Validators gating `engine.update`, keyed by `<workflow>:<updateName>`. */
|
|
1229
|
+
updateValidators = /* @__PURE__ */ new Map();
|
|
1230
|
+
/** Runs being cancelled WITH saga compensation — see `cancel({ compensate: true })`. */
|
|
1231
|
+
cancelRequested = /* @__PURE__ */ new Set();
|
|
1232
|
+
/** Flow-control queues for remote steps, keyed by name (see {@link registerQueue}). */
|
|
1233
|
+
queues = /* @__PURE__ */ new Map();
|
|
1234
|
+
/** Which queue a dispatched step took a slot from, by stepId — so the result can release it. */
|
|
1235
|
+
stepQueue = /* @__PURE__ */ new Map();
|
|
1236
|
+
/** Executions currently in flight, so a graceful shutdown can wait for them to settle. */
|
|
1237
|
+
inflight = /* @__PURE__ */ new Set();
|
|
1238
|
+
draining = false;
|
|
1239
|
+
constructor(deps) {
|
|
1240
|
+
this.store = deps.store;
|
|
1241
|
+
this.pool = new TransportPool(deps.transports ?? (deps.transport ? [
|
|
1242
|
+
{
|
|
1243
|
+
id: "default",
|
|
1244
|
+
transport: deps.transport
|
|
1245
|
+
}
|
|
1246
|
+
] : []));
|
|
1247
|
+
this.controlPlane = deps.controlPlane;
|
|
1248
|
+
this.clock = deps.clock ?? Date.now;
|
|
1249
|
+
this.instanceId = deps.instanceId ?? globalThis.crypto.randomUUID();
|
|
1250
|
+
this.leaseMs = deps.leaseMs ?? 3e4;
|
|
1251
|
+
this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
|
|
1252
|
+
this.webhookUrl = deps.webhookUrl;
|
|
1253
|
+
this.traceparent = deps.traceparent;
|
|
1254
|
+
this.context = deps.context;
|
|
1255
|
+
this.rehydrate = deps.rehydrate ?? ((_carrier, fn) => fn());
|
|
1256
|
+
this.compensationRetries = Math.max(1, deps.compensationRetries ?? 1);
|
|
1257
|
+
this.trackStepStart = deps.trackStepStart ?? true;
|
|
1258
|
+
this.runDispatcher = deps.runDispatcher ?? {
|
|
1259
|
+
dispatch: /* @__PURE__ */ __name((runId) => queueMicrotask(() => void this.runOne(runId).catch(() => {
|
|
1260
|
+
})), "dispatch")
|
|
1261
|
+
};
|
|
1262
|
+
this.pool.bind(
|
|
1263
|
+
async (result) => {
|
|
1264
|
+
const waiter = this.pending.get(result.stepId);
|
|
1265
|
+
if (waiter) {
|
|
1266
|
+
this.pending.delete(result.stepId);
|
|
1267
|
+
if (result.status === "completed") {
|
|
1268
|
+
waiter.resolve({
|
|
1269
|
+
output: result.output,
|
|
1270
|
+
startedAt: result.startedAt,
|
|
1271
|
+
events: result.events
|
|
1272
|
+
});
|
|
1273
|
+
} else {
|
|
1274
|
+
waiter.reject(new RemoteStepError(result.error));
|
|
1275
|
+
}
|
|
1276
|
+
return;
|
|
1277
|
+
}
|
|
1278
|
+
await this.completeRemoteResult(result);
|
|
1279
|
+
},
|
|
1280
|
+
// A heartbeat for an in-flight long step resets its liveness window (see callRemote).
|
|
1281
|
+
async (beat) => {
|
|
1282
|
+
this.heartbeatResets.get(beat.stepId)?.();
|
|
1283
|
+
},
|
|
1284
|
+
// A remote workflow worker streams each local step's lifecycle (running → completed/failed) so
|
|
1285
|
+
// it's checkpointed live, not all-at-once when the long turn ends.
|
|
1286
|
+
async (event) => {
|
|
1287
|
+
await this.persistStepEvent(event);
|
|
1288
|
+
}
|
|
1289
|
+
);
|
|
1290
|
+
this.controlPlane?.onControl((msg) => {
|
|
1291
|
+
if (msg.from === this.instanceId) return;
|
|
1292
|
+
if (msg.kind === "event") {
|
|
1293
|
+
this.deliver({
|
|
1294
|
+
...msg.event,
|
|
1295
|
+
at: new Date(msg.event.at)
|
|
1296
|
+
});
|
|
1297
|
+
} else if (msg.kind === "cancel") {
|
|
1298
|
+
this.notifyCancelled(msg.runId);
|
|
1299
|
+
} else if (msg.kind === "enqueued") {
|
|
1300
|
+
for (const fn of this.enqueuedListeners) {
|
|
1301
|
+
try {
|
|
1302
|
+
fn(msg.runId);
|
|
1303
|
+
} catch {
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
});
|
|
1308
|
+
this.accumulators = new EventAccumulators(this);
|
|
1309
|
+
this.entities = new Entities(this);
|
|
1310
|
+
}
|
|
1311
|
+
/**
|
|
1312
|
+
* Register a **durable entity** (a virtual object): a keyed actor whose `handlers` run **serialized
|
|
1313
|
+
* per key** over **durable state**, exactly once. Drive it with `signalEntity` (fire) /
|
|
1314
|
+
* `ctx.callEntity` (call + await result) and read its state with `getEntityState`. See {@link Entities}.
|
|
1315
|
+
*/
|
|
1316
|
+
registerEntity(name, config) {
|
|
1317
|
+
this.entities.register(name, config);
|
|
1318
|
+
}
|
|
1319
|
+
/** Send an operation to an entity (fire-and-forget). Ordered + exactly-once per key. */
|
|
1320
|
+
signalEntity(name, key, op, arg) {
|
|
1321
|
+
return this.entities.signal(name, key, op, arg);
|
|
1322
|
+
}
|
|
1323
|
+
/** Read an entity's current durable state (published after each op), or undefined if it has none yet. */
|
|
1324
|
+
getEntityState(name, key) {
|
|
1325
|
+
return this.entities.getState(name, key);
|
|
1326
|
+
}
|
|
1327
|
+
/**
|
|
1328
|
+
* Be notified when a run is enqueued on ANOTHER instance (via the control plane), so a worker can
|
|
1329
|
+
* pick it up at once — e.g. `engine.onEnqueued((runId) => engine.runOne(runId))`. Returns an
|
|
1330
|
+
* unsubscribe function. Only wire this on instances that should execute runs (workers).
|
|
1331
|
+
*/
|
|
1332
|
+
onEnqueued(listener) {
|
|
1333
|
+
this.enqueuedListeners.add(listener);
|
|
1334
|
+
return () => this.enqueuedListeners.delete(listener);
|
|
1335
|
+
}
|
|
1336
|
+
/** Fire cooperative-cancellation listeners for `runId` (a worker bridge aborts in-flight work). */
|
|
1337
|
+
notifyCancelled(runId) {
|
|
1338
|
+
for (const fn of this.cancelListeners) {
|
|
1339
|
+
try {
|
|
1340
|
+
fn(runId);
|
|
1341
|
+
} catch {
|
|
1342
|
+
}
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
/**
|
|
1346
|
+
* Register a workflow version. Register multiple versions of the same name to keep in-flight
|
|
1347
|
+
* runs working across a breaking change: old runs resume on the version they started on, new
|
|
1348
|
+
* runs start on the newest registered version.
|
|
1349
|
+
*/
|
|
1350
|
+
register(name, version, fn, opts) {
|
|
1351
|
+
const registered = {
|
|
1352
|
+
name,
|
|
1353
|
+
version,
|
|
1354
|
+
fn,
|
|
1355
|
+
tags: opts?.tags,
|
|
1356
|
+
singleton: opts?.singleton,
|
|
1357
|
+
executionTimeoutMs: opts?.executionTimeout != null ? parseDuration(opts.executionTimeout) : void 0,
|
|
1358
|
+
validateInput: opts?.validateInput,
|
|
1359
|
+
onEvent: opts?.onEvent,
|
|
1360
|
+
eventBatch: opts?.eventBatch
|
|
1361
|
+
};
|
|
1362
|
+
this.workflows.set(versionKey(name, version), registered);
|
|
1363
|
+
const current = this.latest.get(name);
|
|
1364
|
+
if (!current || isNewerVersion(version, current.version)) this.latest.set(name, registered);
|
|
1365
|
+
for (const event of opts?.onEvent ?? []) {
|
|
1366
|
+
const subscribers = this.eventTriggers.get(event) ?? /* @__PURE__ */ new Set();
|
|
1367
|
+
subscribers.add(name);
|
|
1368
|
+
this.eventTriggers.set(event, subscribers);
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
/**
|
|
1372
|
+
* Register a workflow whose body runs in another SDK (e.g. Python). The engine owns the run exactly
|
|
1373
|
+
* as for a TS workflow — it persists checkpoints, recovers, runs timers — but advances it by handing
|
|
1374
|
+
* the run's history to `executor` (which dispatches a {@link WorkflowTask} to the worker) and applying
|
|
1375
|
+
* the {@link WorkflowDecision} the worker's replay returns. The worker never touches the store.
|
|
1376
|
+
*/
|
|
1377
|
+
registerRemote(name, version, opts) {
|
|
1378
|
+
const registered = {
|
|
1379
|
+
name,
|
|
1380
|
+
version,
|
|
1381
|
+
// A remote workflow has no in-process body; execute() branches on `remote` before this is read.
|
|
1382
|
+
fn: /* @__PURE__ */ __name(() => {
|
|
1383
|
+
throw new Error(`workflow ${name} is remote \u2014 it has no in-process body`);
|
|
1384
|
+
}, "fn"),
|
|
1385
|
+
tags: opts.tags,
|
|
1386
|
+
singleton: opts.singleton,
|
|
1387
|
+
executionTimeoutMs: opts.executionTimeout != null ? parseDuration(opts.executionTimeout) : void 0,
|
|
1388
|
+
validateInput: opts.validateInput,
|
|
1389
|
+
remote: {
|
|
1390
|
+
group: opts.group,
|
|
1391
|
+
executor: opts.executor
|
|
1392
|
+
}
|
|
1393
|
+
};
|
|
1394
|
+
this.workflows.set(versionKey(name, version), registered);
|
|
1395
|
+
const current = this.latest.get(name);
|
|
1396
|
+
if (!current || isNewerVersion(version, current.version)) this.latest.set(name, registered);
|
|
1397
|
+
}
|
|
1398
|
+
/**
|
|
1399
|
+
* Register a flow-control queue referenced by `ctx.call(step, input, { queue })`. Caps concurrent
|
|
1400
|
+
* in-flight steps and/or the admission rate; blocked calls re-suspend and retry, so the limit is
|
|
1401
|
+
* durable. Per engine instance (see {@link QueueConfig}). Registering the same name replaces it.
|
|
1402
|
+
*/
|
|
1403
|
+
registerQueue(config) {
|
|
1404
|
+
this.queues.set(config.name, new QueueController(config, this.clock));
|
|
1405
|
+
}
|
|
1406
|
+
/** Subscribe to lifecycle events. Returns an unsubscribe function. */
|
|
1407
|
+
subscribe(listener) {
|
|
1408
|
+
this.listeners.add(listener);
|
|
1409
|
+
return () => this.listeners.delete(listener);
|
|
1410
|
+
}
|
|
1411
|
+
/**
|
|
1412
|
+
* Register a {@link StepInterceptor} — onion middleware run around the real execution of every
|
|
1413
|
+
* local `ctx.step` (timing, logging, tracing, error enrichment, context propagation). First
|
|
1414
|
+
* registered is outermost; interceptors fire only when a step executes, never on replay. Returns
|
|
1415
|
+
* an unsubscribe function.
|
|
1416
|
+
*/
|
|
1417
|
+
use(interceptor) {
|
|
1418
|
+
this.interceptors.push(interceptor);
|
|
1419
|
+
return () => {
|
|
1420
|
+
const i = this.interceptors.indexOf(interceptor);
|
|
1421
|
+
if (i >= 0) this.interceptors.splice(i, 1);
|
|
1422
|
+
};
|
|
1423
|
+
}
|
|
1424
|
+
/**
|
|
1425
|
+
* Fold the registered interceptors around a local step body (identity when there are none), then
|
|
1426
|
+
* run the whole thing inside the re-hydrated originating context. The carrier is read at execution
|
|
1427
|
+
* time from {@link context} — the SAME reader stamped on dispatched remote tasks — so a local step
|
|
1428
|
+
* sees the live tenant / user / trace ids ambiently (via the {@link rehydrate} bridge). Default
|
|
1429
|
+
* `rehydrate` is a passthrough, so this is byte-identical to a bare body call when unwired.
|
|
1430
|
+
*/
|
|
1431
|
+
interceptStep(invocation, body) {
|
|
1432
|
+
const carrier = this.context?.();
|
|
1433
|
+
const run = /* @__PURE__ */ __name(() => {
|
|
1434
|
+
if (this.interceptors.length === 0) return body();
|
|
1435
|
+
const chain = this.interceptors.reduceRight((next, interceptor) => () => interceptor(invocation, next), body);
|
|
1436
|
+
return chain();
|
|
1437
|
+
}, "run");
|
|
1438
|
+
return this.rehydrate(carrier, run);
|
|
1439
|
+
}
|
|
1440
|
+
/**
|
|
1441
|
+
* Be notified when a run is cancelled — on ANY instance, via the transport control plane. A
|
|
1442
|
+
* worker bridge can use this for cooperative cancellation: abort the in-flight work for `runId`
|
|
1443
|
+
* instead of finishing it just to have the result discarded. Returns an unsubscribe function.
|
|
1444
|
+
*/
|
|
1445
|
+
onCancel(listener) {
|
|
1446
|
+
this.cancelListeners.add(listener);
|
|
1447
|
+
return () => this.cancelListeners.delete(listener);
|
|
1448
|
+
}
|
|
1449
|
+
/**
|
|
1450
|
+
* Be notified when a run is **dead-lettered** — moved to `dead` after exceeding
|
|
1451
|
+
* `maxRecoveryAttempts`. The listener receives the dead run (status `dead`, with its error), so a
|
|
1452
|
+
* DLQ handler can do something other than just leaving it parked: alert, push to a real queue, or
|
|
1453
|
+
* start a dead-letter workflow (e.g. `engine.onDead((run) => engine.start('pipeline-dlq', run, ...))`).
|
|
1454
|
+
* Returns an unsubscribe function.
|
|
1455
|
+
*/
|
|
1456
|
+
onDead(listener) {
|
|
1457
|
+
this.deadListeners.add(listener);
|
|
1458
|
+
return () => this.deadListeners.delete(listener);
|
|
1459
|
+
}
|
|
1460
|
+
notifyDead(run) {
|
|
1461
|
+
for (const fn of this.deadListeners) {
|
|
1462
|
+
try {
|
|
1463
|
+
fn(run);
|
|
1464
|
+
} catch {
|
|
1465
|
+
}
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
/** Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the
|
|
1469
|
+
* control plane so other instances (e.g. a dashboard pod) can live-tail this run. */
|
|
1470
|
+
emit(event) {
|
|
1471
|
+
const full = {
|
|
1472
|
+
...event,
|
|
1473
|
+
at: /* @__PURE__ */ new Date()
|
|
1474
|
+
};
|
|
1475
|
+
this.deliver(full);
|
|
1476
|
+
if (this.controlPlane) {
|
|
1477
|
+
void this.controlPlane.publishControl({
|
|
1478
|
+
kind: "event",
|
|
1479
|
+
event: full,
|
|
1480
|
+
from: this.instanceId
|
|
1481
|
+
}).catch(() => {
|
|
1482
|
+
});
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
/** Deliver an event to local subscribers only (no re-broadcast) — used for both locally-produced
|
|
1486
|
+
* events and ones received from the control plane, so an event shows up once on every instance. */
|
|
1487
|
+
deliver(event) {
|
|
1488
|
+
for (const listener of this.listeners) {
|
|
1489
|
+
try {
|
|
1490
|
+
listener(event);
|
|
1491
|
+
} catch {
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
}
|
|
1495
|
+
async start(workflow, input, runId, opts) {
|
|
1496
|
+
const name = workflowName(workflow);
|
|
1497
|
+
const registered = this.latest.get(name);
|
|
1498
|
+
if (!registered) throw new Error(`workflow ${name} is not registered`);
|
|
1499
|
+
await registered.validateInput?.(input);
|
|
1500
|
+
const prior = await this.store.getRun(runId);
|
|
1501
|
+
if (prior) {
|
|
1502
|
+
return {
|
|
1503
|
+
runId,
|
|
1504
|
+
status: prior.status,
|
|
1505
|
+
output: prior.output,
|
|
1506
|
+
error: prior.error
|
|
1507
|
+
};
|
|
1508
|
+
}
|
|
1509
|
+
const now = /* @__PURE__ */ new Date();
|
|
1510
|
+
const tags = mergeTags(registered.tags, registered.singleton ? [
|
|
1511
|
+
...opts?.tags ?? [],
|
|
1512
|
+
singletonTag(registered.singleton, input)
|
|
1513
|
+
] : opts?.tags);
|
|
1514
|
+
const maxQueueDepth = registered.singleton?.maxQueueDepth;
|
|
1515
|
+
if (registered.singleton && maxQueueDepth != null) {
|
|
1516
|
+
const cfg = registered.singleton;
|
|
1517
|
+
const key = cfg.key(input);
|
|
1518
|
+
const cap = (cfg.limit ?? 1) + maxQueueDepth;
|
|
1519
|
+
const queued = await this.store.listRuns({
|
|
1520
|
+
tag: singletonTag(cfg, input),
|
|
1521
|
+
workflow: name,
|
|
1522
|
+
statuses: [
|
|
1523
|
+
"pending",
|
|
1524
|
+
"running",
|
|
1525
|
+
"suspended"
|
|
1526
|
+
]
|
|
1527
|
+
});
|
|
1528
|
+
if (queued.length >= cap) {
|
|
1529
|
+
throw new SingletonQueueFullError(name, key, maxQueueDepth);
|
|
1530
|
+
}
|
|
1531
|
+
}
|
|
1532
|
+
const run = {
|
|
1533
|
+
id: runId,
|
|
1534
|
+
workflow: name,
|
|
1535
|
+
workflowVersion: registered.version,
|
|
1536
|
+
status: "pending",
|
|
1537
|
+
input,
|
|
1538
|
+
tags,
|
|
1539
|
+
searchAttributes: opts?.searchAttributes,
|
|
1540
|
+
createdAt: now,
|
|
1541
|
+
updatedAt: now
|
|
1542
|
+
};
|
|
1543
|
+
await this.store.createRun(run);
|
|
1544
|
+
await this.runDispatcher.dispatch(runId);
|
|
1545
|
+
if (this.controlPlane) {
|
|
1546
|
+
void this.controlPlane.publishControl({
|
|
1547
|
+
kind: "enqueued",
|
|
1548
|
+
runId,
|
|
1549
|
+
from: this.instanceId
|
|
1550
|
+
}).catch(() => void 0);
|
|
1551
|
+
}
|
|
1552
|
+
return {
|
|
1553
|
+
runId,
|
|
1554
|
+
status: "pending"
|
|
1555
|
+
};
|
|
1556
|
+
}
|
|
1557
|
+
/** Read a run's current persisted state (or null if unknown). A thin pass-through to the store. */
|
|
1558
|
+
getRun(runId) {
|
|
1559
|
+
return this.store.getRun(runId);
|
|
1560
|
+
}
|
|
1561
|
+
async resume(runId) {
|
|
1562
|
+
const run = await this.store.getRun(runId);
|
|
1563
|
+
if (!run) throw new Error(`run ${runId} not found`);
|
|
1564
|
+
if (run.status === "cancelled" || run.status === "completed" || run.status === "dead") {
|
|
1565
|
+
return {
|
|
1566
|
+
runId,
|
|
1567
|
+
status: run.status,
|
|
1568
|
+
output: run.output,
|
|
1569
|
+
error: run.error
|
|
1570
|
+
};
|
|
1571
|
+
}
|
|
1572
|
+
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
|
|
1573
|
+
if (!registered) {
|
|
1574
|
+
throw new Error(`workflow ${run.workflow}@${run.workflowVersion} is not registered \u2014 keep the prior version deployed so in-flight runs can drain (skew protection)`);
|
|
1575
|
+
}
|
|
1576
|
+
return this.track(this.execute(run, registered.fn));
|
|
1577
|
+
}
|
|
1578
|
+
/** Track an in-flight execution so {@link drain} can wait for it. */
|
|
1579
|
+
track(p) {
|
|
1580
|
+
this.inflight.add(p);
|
|
1581
|
+
void p.finally(() => this.inflight.delete(p));
|
|
1582
|
+
return p;
|
|
1583
|
+
}
|
|
1584
|
+
/**
|
|
1585
|
+
* Graceful shutdown: stop picking up new runs (recovery/timer become no-ops) and wait for
|
|
1586
|
+
* in-flight executions to settle, up to `timeoutMs`. Call from your app's shutdown hook so a
|
|
1587
|
+
* deploy hands off cleanly instead of leaving runs to the lease timeout.
|
|
1588
|
+
*/
|
|
1589
|
+
async drain(timeoutMs = 1e4) {
|
|
1590
|
+
this.draining = true;
|
|
1591
|
+
if (this.inflight.size === 0) return;
|
|
1592
|
+
const timer = new Promise((resolve) => {
|
|
1593
|
+
const t = setTimeout(resolve, timeoutMs);
|
|
1594
|
+
t.unref?.();
|
|
1595
|
+
});
|
|
1596
|
+
await Promise.race([
|
|
1597
|
+
Promise.allSettled([
|
|
1598
|
+
...this.inflight
|
|
1599
|
+
]),
|
|
1600
|
+
timer
|
|
1601
|
+
]);
|
|
1602
|
+
}
|
|
1603
|
+
/**
|
|
1604
|
+
* Cancel in-flight runs that have outlived their workflow's `executionTimeout`. Call it from the
|
|
1605
|
+
* timer poller alongside {@link resumeDueTimers}. A timed-out run is moved to `cancelled` with an
|
|
1606
|
+
* `execution_timeout` error (terminal, so a late step result can't resurrect it).
|
|
1607
|
+
*/
|
|
1608
|
+
async sweepTimeouts(now = this.clock()) {
|
|
1609
|
+
for (const reg of new Set(this.latest.values())) {
|
|
1610
|
+
if (reg.executionTimeoutMs == null) continue;
|
|
1611
|
+
const deadline = now - reg.executionTimeoutMs;
|
|
1612
|
+
const inflight = [
|
|
1613
|
+
...await this.store.listRuns({
|
|
1614
|
+
workflow: reg.name,
|
|
1615
|
+
status: "running"
|
|
1616
|
+
}),
|
|
1617
|
+
...await this.store.listRuns({
|
|
1618
|
+
workflow: reg.name,
|
|
1619
|
+
status: "suspended"
|
|
1620
|
+
})
|
|
1621
|
+
];
|
|
1622
|
+
for (const run of inflight) {
|
|
1623
|
+
if (run.createdAt.getTime() > deadline) continue;
|
|
1624
|
+
const error = {
|
|
1625
|
+
message: "execution timeout",
|
|
1626
|
+
code: "execution_timeout"
|
|
1627
|
+
};
|
|
1628
|
+
await this.store.updateRun(run.id, {
|
|
1629
|
+
status: "cancelled",
|
|
1630
|
+
error,
|
|
1631
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1632
|
+
});
|
|
1633
|
+
this.emit({
|
|
1634
|
+
type: "run.failed",
|
|
1635
|
+
runId: run.id,
|
|
1636
|
+
workflow: run.workflow,
|
|
1637
|
+
error
|
|
1638
|
+
});
|
|
1639
|
+
}
|
|
1640
|
+
}
|
|
1641
|
+
}
|
|
1642
|
+
/**
|
|
1643
|
+
* Resume every run left incomplete by a crash or deploy. Called on boot. Completed steps
|
|
1644
|
+
* replay from their checkpoints, so only the work that had not finished runs again.
|
|
1645
|
+
*/
|
|
1646
|
+
async recoverIncomplete(nowMs = this.clock()) {
|
|
1647
|
+
if (this.draining) return [];
|
|
1648
|
+
const results = [];
|
|
1649
|
+
for (const run of await this.store.listIncompleteRuns()) {
|
|
1650
|
+
const acquired = await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs);
|
|
1651
|
+
if (!acquired) continue;
|
|
1652
|
+
const settled = await this.countRecovery(run);
|
|
1653
|
+
if (settled) {
|
|
1654
|
+
results.push(settled);
|
|
1655
|
+
continue;
|
|
1656
|
+
}
|
|
1657
|
+
await this.store.releaseRunLock(run.id);
|
|
1658
|
+
await this.store.updateRun(run.id, {
|
|
1659
|
+
status: "pending",
|
|
1660
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1661
|
+
});
|
|
1662
|
+
await this.runDispatcher.dispatch(run.id);
|
|
1663
|
+
results.push({
|
|
1664
|
+
runId: run.id,
|
|
1665
|
+
status: "pending"
|
|
1666
|
+
});
|
|
1667
|
+
}
|
|
1668
|
+
return results;
|
|
1669
|
+
}
|
|
1670
|
+
/**
|
|
1671
|
+
* Per-recovery bookkeeping (called once the lease is held): count the attempt, or — past
|
|
1672
|
+
* `maxRecoveryAttempts` — move a poison pill to the `dead` dead-letter state. Returns a terminal
|
|
1673
|
+
* result to skip the resume, or `undefined` to proceed.
|
|
1674
|
+
*/
|
|
1675
|
+
async countRecovery(run) {
|
|
1676
|
+
const attempts = (run.recoveryAttempts ?? 0) + 1;
|
|
1677
|
+
if (this.maxRecoveryAttempts != null && attempts > this.maxRecoveryAttempts) {
|
|
1678
|
+
const error = {
|
|
1679
|
+
message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) \u2014 moved to dead-letter`,
|
|
1680
|
+
code: "max_recovery_attempts"
|
|
1681
|
+
};
|
|
1682
|
+
await this.store.updateRun(run.id, {
|
|
1683
|
+
status: "dead",
|
|
1684
|
+
error,
|
|
1685
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1686
|
+
});
|
|
1687
|
+
await this.store.releaseRunLock(run.id);
|
|
1688
|
+
this.emit({
|
|
1689
|
+
type: "run.failed",
|
|
1690
|
+
runId: run.id,
|
|
1691
|
+
workflow: run.workflow,
|
|
1692
|
+
error
|
|
1693
|
+
});
|
|
1694
|
+
this.notifyDead({
|
|
1695
|
+
...run,
|
|
1696
|
+
status: "dead",
|
|
1697
|
+
error,
|
|
1698
|
+
recoveryAttempts: attempts
|
|
1699
|
+
});
|
|
1700
|
+
return {
|
|
1701
|
+
runId: run.id,
|
|
1702
|
+
status: "dead",
|
|
1703
|
+
error
|
|
1704
|
+
};
|
|
1705
|
+
}
|
|
1706
|
+
await this.store.updateRun(run.id, {
|
|
1707
|
+
recoveryAttempts: attempts,
|
|
1708
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1709
|
+
});
|
|
1710
|
+
return void 0;
|
|
1711
|
+
}
|
|
1712
|
+
/**
|
|
1713
|
+
* Resume every suspended run whose durable timer is due. Call periodically (a poller) and on
|
|
1714
|
+
* boot. A run still not due re-suspends cheaply without running new work.
|
|
1715
|
+
*/
|
|
1716
|
+
async resumeDueTimers(nowMs = this.clock()) {
|
|
1717
|
+
return this.resumeLeased(await this.store.listDueTimers(nowMs), nowMs);
|
|
1718
|
+
}
|
|
1719
|
+
/**
|
|
1720
|
+
* Lease and execute one run by id — the worker side of dispatch. Acquires the recovery lease (so
|
|
1721
|
+
* exactly one instance runs it), then resumes the body. Returns the result, or null if another
|
|
1722
|
+
* instance holds the lease or the engine is draining. The default in-process dispatcher calls this;
|
|
1723
|
+
* a broker-backed worker calls it per consumed run id.
|
|
1724
|
+
*/
|
|
1725
|
+
async runOne(runId) {
|
|
1726
|
+
if (this.draining) return null;
|
|
1727
|
+
const nowMs = this.clock();
|
|
1728
|
+
const acquired = await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs);
|
|
1729
|
+
if (!acquired) return null;
|
|
1730
|
+
return this.resume(runId);
|
|
1731
|
+
}
|
|
1732
|
+
/**
|
|
1733
|
+
* Re-enqueue a run for a worker to (re-)execute — the dispatch-model **retry**. Sets it back to
|
|
1734
|
+
* `pending`, clears any stale lease, and dispatches; a worker resumes it (replaying its checkpoints,
|
|
1735
|
+
* re-attempting the failed step). Returns the enqueued state immediately — never runs the body
|
|
1736
|
+
* inline — or null if the run is unknown. The dashboard "retry" goes through here so it can't block
|
|
1737
|
+
* the HTTP request on workflow execution.
|
|
1738
|
+
*/
|
|
1739
|
+
async requeue(runId) {
|
|
1740
|
+
const run = await this.store.getRun(runId);
|
|
1741
|
+
if (!run) return null;
|
|
1742
|
+
await this.store.releaseRunLock(runId);
|
|
1743
|
+
await this.store.updateRun(runId, {
|
|
1744
|
+
status: "pending",
|
|
1745
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1746
|
+
});
|
|
1747
|
+
await this.runDispatcher.dispatch(runId);
|
|
1748
|
+
return {
|
|
1749
|
+
runId,
|
|
1750
|
+
status: "pending"
|
|
1751
|
+
};
|
|
1752
|
+
}
|
|
1753
|
+
/**
|
|
1754
|
+
* **Fix-and-replay**: re-run a run (typically a `dead`/`failed` one) with a corrected `input`, as a
|
|
1755
|
+
* fresh run with clean history. It's a NEW run — `newRunId` defaults to `<runId>~retry~<uuid>` — so
|
|
1756
|
+
* the original stays inspectable. Returns the new run's id, or null if `runId` is unknown.
|
|
1757
|
+
*/
|
|
1758
|
+
async retryWithInput(runId, input, newRunId) {
|
|
1759
|
+
const run = await this.store.getRun(runId);
|
|
1760
|
+
if (!run) return null;
|
|
1761
|
+
const id = newRunId ?? `${runId}~retry~${globalThis.crypto.randomUUID().slice(0, 8)}`;
|
|
1762
|
+
await this.start(run.workflow, input, id, {
|
|
1763
|
+
tags: run.tags
|
|
1764
|
+
});
|
|
1765
|
+
return {
|
|
1766
|
+
runId: id
|
|
1767
|
+
};
|
|
1768
|
+
}
|
|
1769
|
+
/**
|
|
1770
|
+
* Pick up and execute every `pending` run — the poll-based side of dispatch for a worker pod with
|
|
1771
|
+
* no broker. Runs enqueued by other pods (or by a caller using a no-op dispatcher) sit `pending`
|
|
1772
|
+
* in the store until polled; leasing ensures exactly one pod runs each. Call periodically alongside
|
|
1773
|
+
* {@link resumeDueTimers}.
|
|
1774
|
+
*/
|
|
1775
|
+
async runPending(nowMs = this.clock()) {
|
|
1776
|
+
return this.resumeLeased(await this.store.listPendingRuns(100), nowMs);
|
|
1777
|
+
}
|
|
1778
|
+
/**
|
|
1779
|
+
* Resolve once `runId` reaches a settled state — terminal (completed/failed/cancelled/dead) or
|
|
1780
|
+
* suspended (handed off to a timer/signal/event). The async counterpart to dispatch: pair it with
|
|
1781
|
+
* `start` when a call site needs the outcome — `await start(...); const r = await waitForRun(id)`.
|
|
1782
|
+
*/
|
|
1783
|
+
waitForRun(runId, opts) {
|
|
1784
|
+
const isSettled = /* @__PURE__ */ __name((s) => s !== "pending" && s !== "running", "isSettled");
|
|
1785
|
+
const toResult = /* @__PURE__ */ __name((run) => ({
|
|
1786
|
+
runId,
|
|
1787
|
+
status: run.status,
|
|
1788
|
+
output: run.output,
|
|
1789
|
+
error: run.error
|
|
1790
|
+
}), "toResult");
|
|
1791
|
+
return new Promise((resolve, reject) => {
|
|
1792
|
+
let done = false;
|
|
1793
|
+
let off = /* @__PURE__ */ __name(() => {
|
|
1794
|
+
}, "off");
|
|
1795
|
+
const timer = opts?.timeoutMs != null ? setTimeout(() => {
|
|
1796
|
+
if (done) return;
|
|
1797
|
+
done = true;
|
|
1798
|
+
off();
|
|
1799
|
+
reject(new Error(`waitForRun(${runId}) timed out after ${opts.timeoutMs}ms`));
|
|
1800
|
+
}, opts.timeoutMs) : void 0;
|
|
1801
|
+
const finish = /* @__PURE__ */ __name((run) => {
|
|
1802
|
+
if (done) return;
|
|
1803
|
+
done = true;
|
|
1804
|
+
if (timer) clearTimeout(timer);
|
|
1805
|
+
off();
|
|
1806
|
+
resolve(toResult(run));
|
|
1807
|
+
}, "finish");
|
|
1808
|
+
const check = /* @__PURE__ */ __name(() => {
|
|
1809
|
+
void this.store.getRun(runId).then((run) => {
|
|
1810
|
+
if (run && isSettled(run.status)) finish(run);
|
|
1811
|
+
});
|
|
1812
|
+
}, "check");
|
|
1813
|
+
off = this.subscribe((ev) => {
|
|
1814
|
+
if (ev.runId === runId && (ev.type === "run.completed" || ev.type === "run.failed" || ev.type === "run.suspended")) {
|
|
1815
|
+
check();
|
|
1816
|
+
}
|
|
1817
|
+
});
|
|
1818
|
+
check();
|
|
1819
|
+
});
|
|
1820
|
+
}
|
|
1821
|
+
/**
|
|
1822
|
+
* Resume each run only if this instance can acquire its recovery lease — so when several
|
|
1823
|
+
* replicas recover or poll at once, each run is picked up by exactly one of them.
|
|
1824
|
+
*/
|
|
1825
|
+
async resumeLeased(runs, nowMs = this.clock(), onLocked) {
|
|
1826
|
+
if (this.draining) return [];
|
|
1827
|
+
const results = [];
|
|
1828
|
+
for (const run of runs) {
|
|
1829
|
+
const acquired = await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs);
|
|
1830
|
+
if (!acquired) continue;
|
|
1831
|
+
const settled = onLocked ? await onLocked(run) : void 0;
|
|
1832
|
+
results.push(settled ?? await this.resume(run.id));
|
|
1833
|
+
}
|
|
1834
|
+
return results;
|
|
1835
|
+
}
|
|
1836
|
+
/**
|
|
1837
|
+
* Deliver an external signal to the run waiting on `token` and resume it with `payload`.
|
|
1838
|
+
* Returns the run result, or null if no run is waiting for that token.
|
|
1839
|
+
*/
|
|
1840
|
+
/**
|
|
1841
|
+
* Publish a named event. It does two things, and returns how many runs it touched (the sum):
|
|
1842
|
+
* 1. **Resumes** every in-flight run waiting on it via `ctx.waitForEvent(name, { match })` whose
|
|
1843
|
+
* match the payload satisfies (fan-out, vs `signal`'s point-to-point token).
|
|
1844
|
+
* 2. **Starts** a fresh run of every workflow registered with `onEvent: [name]`, passing the
|
|
1845
|
+
* payload as input. Idempotent by `evt:<id>:<workflow>` — pass `opts.id` to dedupe redeliveries
|
|
1846
|
+
* of the same logical event (default: a fresh uuid, so each publish triggers once).
|
|
1847
|
+
*/
|
|
1848
|
+
async publishEvent(name, payload, opts) {
|
|
1849
|
+
let touched = 0;
|
|
1850
|
+
const waiters = await this.store.listSignalWaiters(eventPrefix(name));
|
|
1851
|
+
for (const w of waiters) {
|
|
1852
|
+
if (eventMatches(payload, eventMatchOf(w.token))) {
|
|
1853
|
+
await this.signal(w.token, payload);
|
|
1854
|
+
touched += 1;
|
|
1855
|
+
}
|
|
1856
|
+
}
|
|
1857
|
+
const subscribers = this.eventTriggers.get(name);
|
|
1858
|
+
if (subscribers?.size) {
|
|
1859
|
+
const eventId = opts?.id ?? globalThis.crypto.randomUUID();
|
|
1860
|
+
for (const workflow of subscribers) {
|
|
1861
|
+
try {
|
|
1862
|
+
const batch = this.latest.get(workflow)?.eventBatch;
|
|
1863
|
+
if (batch) {
|
|
1864
|
+
await this.accumulators.route(workflow, batch, payload);
|
|
1865
|
+
} else {
|
|
1866
|
+
await this.start(workflow, payload, `evt:${eventId}:${workflow}`);
|
|
1867
|
+
}
|
|
1868
|
+
touched += 1;
|
|
1869
|
+
} catch {
|
|
1870
|
+
}
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
return touched;
|
|
1874
|
+
}
|
|
1875
|
+
async signal(token, payload) {
|
|
1876
|
+
const waiter = await this.store.takeSignalWaiter(token);
|
|
1877
|
+
if (!waiter) {
|
|
1878
|
+
await this.store.bufferSignal(token, payload);
|
|
1879
|
+
return null;
|
|
1880
|
+
}
|
|
1881
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
1882
|
+
runId: waiter.runId,
|
|
1883
|
+
seq: waiter.seq,
|
|
1884
|
+
name: `signal:${token}`,
|
|
1885
|
+
kind: "signal",
|
|
1886
|
+
output: payload
|
|
1887
|
+
}));
|
|
1888
|
+
return this.resume(waiter.runId);
|
|
1889
|
+
}
|
|
1890
|
+
/**
|
|
1891
|
+
* Ensure a run exists for `runId`, then deliver a signal to it — atomically race-free thanks to
|
|
1892
|
+
* signal buffering: if the run is new (or busy / not yet waiting), the signal is buffered and
|
|
1893
|
+
* consumed when it reaches `waitForSignal(token)`. The canonical **durable-entity / accumulator**
|
|
1894
|
+
* pattern: one long-lived run per key (the `runId`) that loops on `waitForSignal`, fed events by
|
|
1895
|
+
* many `signalWithStart` calls. `start` is idempotent by `runId`, so concurrent callers converge on
|
|
1896
|
+
* one run. (Use a per-run `token`, e.g. derived from `runId`, so the signal targets this entity.)
|
|
1897
|
+
*/
|
|
1898
|
+
async signalWithStart(workflow, input, runId, signal, opts) {
|
|
1899
|
+
await this.start(workflow, input, runId, opts);
|
|
1900
|
+
await this.signal(signal.token, signal.payload);
|
|
1901
|
+
return {
|
|
1902
|
+
runId
|
|
1903
|
+
};
|
|
1904
|
+
}
|
|
1905
|
+
/**
|
|
1906
|
+
* Report the result of a `ctx.task(name, …)` back to its run (async completion). The external
|
|
1907
|
+
* worker that the task dispatched to calls this when done; the run resumes with `result`. Returns
|
|
1908
|
+
* null if no run is waiting on the task (e.g. a duplicate/late delivery) — a safe no-op.
|
|
1909
|
+
*/
|
|
1910
|
+
async completeTask(runId, name, result) {
|
|
1911
|
+
return this.signal(`task:${runId}:${name}`, {
|
|
1912
|
+
ok: true,
|
|
1913
|
+
value: result
|
|
1914
|
+
});
|
|
1915
|
+
}
|
|
1916
|
+
/** Report that a `ctx.task` failed — the run resumes and throws a FatalError at the task. */
|
|
1917
|
+
async failTask(runId, name, error) {
|
|
1918
|
+
return this.signal(`task:${runId}:${name}`, {
|
|
1919
|
+
ok: false,
|
|
1920
|
+
error
|
|
1921
|
+
});
|
|
1922
|
+
}
|
|
1923
|
+
/**
|
|
1924
|
+
* Notify a parent that's waiting on `runId` as a child of its terminal outcome (the `ctx.child`
|
|
1925
|
+
* rendezvous). A no-op when no parent is waiting, so `execute()` can call it on every run without
|
|
1926
|
+
* knowing about the child feature.
|
|
1927
|
+
*/
|
|
1928
|
+
notifyParent(runId, completion) {
|
|
1929
|
+
void this.signal(`child:${runId}`, completion).catch(() => void 0);
|
|
1930
|
+
}
|
|
1931
|
+
/**
|
|
1932
|
+
* Cancel a run (e.g. from the dashboard). Returns null if the run does not exist. Pass
|
|
1933
|
+
* `{ compensate: true }` to undo the saga first: the suspended run is resumed so its completed
|
|
1934
|
+
* steps' compensations run in reverse (visible as `compensate:<step>` events), then it's marked
|
|
1935
|
+
* cancelled. Without it, cancellation is immediate (no undo).
|
|
1936
|
+
*/
|
|
1937
|
+
async cancel(runId, opts) {
|
|
1938
|
+
const run = await this.store.getRun(runId);
|
|
1939
|
+
if (!run) return null;
|
|
1940
|
+
if (run.status === "completed" || run.status === "cancelled" || run.status === "dead") {
|
|
1941
|
+
return {
|
|
1942
|
+
runId,
|
|
1943
|
+
status: run.status,
|
|
1944
|
+
output: run.output,
|
|
1945
|
+
error: run.error
|
|
1946
|
+
};
|
|
1947
|
+
}
|
|
1948
|
+
if (opts?.compensate && (run.status === "suspended" || run.status === "running")) {
|
|
1949
|
+
this.cancelRequested.add(runId);
|
|
1950
|
+
if (this.controlPlane) {
|
|
1951
|
+
void this.controlPlane.publishControl({
|
|
1952
|
+
kind: "cancel",
|
|
1953
|
+
runId,
|
|
1954
|
+
from: this.instanceId
|
|
1955
|
+
}).catch(() => void 0);
|
|
1956
|
+
}
|
|
1957
|
+
queueMicrotask(() => {
|
|
1958
|
+
void this.resume(runId).then(() => this.notifyCancelled(runId)).catch(() => void 0);
|
|
1959
|
+
});
|
|
1960
|
+
await this.cancelChildren(runId, opts);
|
|
1961
|
+
return {
|
|
1962
|
+
runId,
|
|
1963
|
+
status: run.status
|
|
1964
|
+
};
|
|
1965
|
+
}
|
|
1966
|
+
const error = {
|
|
1967
|
+
message: "cancelled"
|
|
1968
|
+
};
|
|
1969
|
+
await this.store.updateRun(runId, {
|
|
1970
|
+
status: "cancelled",
|
|
1971
|
+
error,
|
|
1972
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1973
|
+
});
|
|
1974
|
+
this.emit({
|
|
1975
|
+
type: "run.failed",
|
|
1976
|
+
runId,
|
|
1977
|
+
workflow: run.workflow,
|
|
1978
|
+
error
|
|
1979
|
+
});
|
|
1980
|
+
await this.cancelChildren(runId, opts);
|
|
1981
|
+
this.notifyCancelled(runId);
|
|
1982
|
+
if (this.controlPlane) {
|
|
1983
|
+
void this.controlPlane.publishControl({
|
|
1984
|
+
kind: "cancel",
|
|
1985
|
+
runId,
|
|
1986
|
+
from: this.instanceId
|
|
1987
|
+
}).catch(() => void 0);
|
|
1988
|
+
}
|
|
1989
|
+
void this.wakeNextSingletons(run).catch(() => void 0);
|
|
1990
|
+
return {
|
|
1991
|
+
runId,
|
|
1992
|
+
status: "cancelled",
|
|
1993
|
+
error
|
|
1994
|
+
};
|
|
1995
|
+
}
|
|
1996
|
+
/**
|
|
1997
|
+
* Bulk-cancel every run matching a filter — e.g. cancel all `order` runs tagged `vip`, or every run
|
|
1998
|
+
* whose `tier` search attribute is `free`. The filter is a {@link RunQuery} (workflow / status / tag
|
|
1999
|
+
* / search-attribute predicates), so it reuses the same matching the dashboard list uses. Each match
|
|
2000
|
+
* is run through {@link cancel}, so the same plumbing applies per run: child cascade, the optional
|
|
2001
|
+
* saga `compensate`, local cancel listeners, and the control-plane broadcast that tells the owning
|
|
2002
|
+
* worker to abort. Returns one {@link RunResult} per matched run (already-finished matches report
|
|
2003
|
+
* their terminal status — `cancel` is a no-op on them, never clobbering a completed/dead run).
|
|
2004
|
+
*/
|
|
2005
|
+
async cancelWhere(filter, opts) {
|
|
2006
|
+
const runs = await this.store.listRuns(filter);
|
|
2007
|
+
const results = [];
|
|
2008
|
+
for (const run of runs) {
|
|
2009
|
+
const r = await this.cancel(run.id, opts);
|
|
2010
|
+
if (r) results.push(r);
|
|
2011
|
+
}
|
|
2012
|
+
return results;
|
|
2013
|
+
}
|
|
2014
|
+
/**
|
|
2015
|
+
* Cascade cancellation to a run's children — both awaited (`ctx.child`, found via its live
|
|
2016
|
+
* `child:<id>` waiter) and fire-and-forget (`ctx.startChild`, found via its `spawn:<id>`
|
|
2017
|
+
* checkpoint). Recursive, so a whole subtree is cancelled; the terminal guard in `cancel` stops it
|
|
2018
|
+
* at finished / already-cancelled runs (no loops, no re-cancel).
|
|
2019
|
+
*/
|
|
2020
|
+
async cancelChildren(parentRunId, opts) {
|
|
2021
|
+
for (const id of await this.getRunChildren(parentRunId)) {
|
|
2022
|
+
await this.cancel(id, opts).catch(() => void 0);
|
|
2023
|
+
}
|
|
2024
|
+
}
|
|
2025
|
+
/**
|
|
2026
|
+
* The ids of the runs a run spawned — both awaited (`ctx.child`) and fire-and-forget
|
|
2027
|
+
* (`ctx.startChild`, found via its `spawn:<id>` checkpoint). The canonical parent→children edge,
|
|
2028
|
+
* used for both cancellation cascades and the dashboard run-tree.
|
|
2029
|
+
*
|
|
2030
|
+
* An awaited child is discovered two ways, because the live `child:<id>` waiter only exists WHILE
|
|
2031
|
+
* the parent is suspended on it: the waiter resolves and is consumed the moment the child settles,
|
|
2032
|
+
* so a completed parent (or a completed awaited child) would otherwise drop out of the tree. The
|
|
2033
|
+
* `signal:child:<id>` checkpoint (the placeholder written when the parent first awaits the child,
|
|
2034
|
+
* overwritten as completed/failed when it settles) persists across completion, so we read both and
|
|
2035
|
+
* dedupe — the edge stays stable for a finished run, not just a live one.
|
|
2036
|
+
*/
|
|
2037
|
+
async getRunChildren(parentRunId) {
|
|
2038
|
+
const childIds = /* @__PURE__ */ new Set();
|
|
2039
|
+
for (const w of await this.store.listSignalWaiters("child:")) {
|
|
2040
|
+
if (w.runId === parentRunId) childIds.add(w.token.slice("child:".length));
|
|
2041
|
+
}
|
|
2042
|
+
for (const cp of await this.store.listCheckpoints(parentRunId)) {
|
|
2043
|
+
if (cp.name.startsWith("signal:child:")) childIds.add(cp.name.slice("signal:child:".length));
|
|
2044
|
+
if (cp.name.startsWith("spawn:") && typeof cp.output === "string") childIds.add(cp.output);
|
|
2045
|
+
}
|
|
2046
|
+
return [
|
|
2047
|
+
...childIds
|
|
2048
|
+
];
|
|
2049
|
+
}
|
|
2050
|
+
/** The worker groups this engine dispatches to: every registered remote workflow's group, plus any
|
|
2051
|
+
* `extra` the caller declares. Local-step groups (a group consumed by in-process `@DurableStep`
|
|
2052
|
+
* workers, e.g. `pipeline`) aren't derivable from registrations — pass them via `extra` so a group
|
|
2053
|
+
* with backlog and ZERO workers is still reported (the alert case has no heartbeat to discover). */
|
|
2054
|
+
knownGroups(extra = []) {
|
|
2055
|
+
const groups = new Set(extra);
|
|
2056
|
+
for (const def of this.workflows.values()) {
|
|
2057
|
+
if (def.remote?.group) groups.add(def.remote.group);
|
|
2058
|
+
}
|
|
2059
|
+
return [
|
|
2060
|
+
...groups
|
|
2061
|
+
];
|
|
2062
|
+
}
|
|
2063
|
+
/** Per-group worker health (queue backlog + live worker heartbeats). Covers {@link knownGroups}
|
|
2064
|
+
* (so a registered group with backlog and ZERO workers still reports — the alert case) UNION the
|
|
2065
|
+
* groups discovered from live heartbeats (so a local-step group like `pipeline`, not derivable
|
|
2066
|
+
* from registrations, shows once its workers beat). Empty when no transport can introspect health
|
|
2067
|
+
* (only the BullMQ transport implements `groupHealth`). */
|
|
2068
|
+
async workerHealth(extra = []) {
|
|
2069
|
+
const groups = /* @__PURE__ */ new Set([
|
|
2070
|
+
...this.knownGroups(extra),
|
|
2071
|
+
...await this.pool.listWorkerGroups()
|
|
2072
|
+
]);
|
|
2073
|
+
const out = [];
|
|
2074
|
+
for (const group of groups) {
|
|
2075
|
+
const health = await this.pool.groupHealth(group);
|
|
2076
|
+
if (health) out.push(health);
|
|
2077
|
+
}
|
|
2078
|
+
return out;
|
|
2079
|
+
}
|
|
2080
|
+
/**
|
|
2081
|
+
* Persist a streamed local-step lifecycle event from a remote workflow worker (see
|
|
2082
|
+
* {@link WorkflowStepEvent}). A Python `@workflow` runs its `ctx.step`s inline over one turn that
|
|
2083
|
+
* can last minutes; the worker streams each step's start/finish so the engine checkpoints it LIVE —
|
|
2084
|
+
* a step shows `running` the moment its body begins, then resolves to `completed`/`failed` with its
|
|
2085
|
+
* real wall-clock window and sub-process events — instead of every step appearing at once when the
|
|
2086
|
+
* turn ends. The turn's final `recordStep` command re-persists the same (runId, seq) checkpoint
|
|
2087
|
+
* idempotently, so this is purely additive observability and never changes the run's outcome.
|
|
2088
|
+
*/
|
|
2089
|
+
async persistStepEvent(event) {
|
|
2090
|
+
const startedAt = new Date(event.startedAt);
|
|
2091
|
+
const events = event.events && event.events.length > 0 ? event.events : void 0;
|
|
2092
|
+
if (event.phase === "running") {
|
|
2093
|
+
await this.store.saveCheckpoint({
|
|
2094
|
+
runId: event.runId,
|
|
2095
|
+
seq: event.seq,
|
|
2096
|
+
name: event.name,
|
|
2097
|
+
kind: "local",
|
|
2098
|
+
stepId: stepId(event.runId, event.seq),
|
|
2099
|
+
status: "running",
|
|
2100
|
+
events,
|
|
2101
|
+
attempts: 1,
|
|
2102
|
+
enqueuedAt: startedAt,
|
|
2103
|
+
startedAt,
|
|
2104
|
+
finishedAt: startedAt
|
|
2105
|
+
});
|
|
2106
|
+
this.emit({
|
|
2107
|
+
type: "step.started",
|
|
2108
|
+
runId: event.runId,
|
|
2109
|
+
seq: event.seq,
|
|
2110
|
+
name: event.name,
|
|
2111
|
+
kind: "local"
|
|
2112
|
+
});
|
|
2113
|
+
return;
|
|
2114
|
+
}
|
|
2115
|
+
const failed = event.phase === "failed";
|
|
2116
|
+
await this.store.saveCheckpoint({
|
|
2117
|
+
runId: event.runId,
|
|
2118
|
+
seq: event.seq,
|
|
2119
|
+
name: event.name,
|
|
2120
|
+
kind: "local",
|
|
2121
|
+
stepId: stepId(event.runId, event.seq),
|
|
2122
|
+
status: failed ? "failed" : "completed",
|
|
2123
|
+
output: failed ? void 0 : event.output,
|
|
2124
|
+
error: failed ? event.error : void 0,
|
|
2125
|
+
events,
|
|
2126
|
+
attempts: 1,
|
|
2127
|
+
enqueuedAt: startedAt,
|
|
2128
|
+
startedAt,
|
|
2129
|
+
finishedAt: event.finishedAt != null ? new Date(event.finishedAt) : /* @__PURE__ */ new Date()
|
|
2130
|
+
});
|
|
2131
|
+
this.emit({
|
|
2132
|
+
type: failed ? "step.failed" : "step.completed",
|
|
2133
|
+
runId: event.runId,
|
|
2134
|
+
seq: event.seq,
|
|
2135
|
+
name: event.name,
|
|
2136
|
+
kind: "local",
|
|
2137
|
+
output: failed ? void 0 : event.output,
|
|
2138
|
+
error: failed ? event.error : void 0
|
|
2139
|
+
});
|
|
2140
|
+
}
|
|
2141
|
+
/**
|
|
2142
|
+
* Announce a local step's body has begun and (when `trackStepStart`) checkpoint it as `running`,
|
|
2143
|
+
* so it's visible in flight rather than appearing only on completion. The checkpoint is a
|
|
2144
|
+
* placeholder overwritten by {@link completeStep}/{@link failStep}; it never short-circuits replay
|
|
2145
|
+
* (only `completed` does), so a crash mid-body just re-runs the step. The `step.started` event
|
|
2146
|
+
* fires regardless — the live SSE view sees the start even with persistence off.
|
|
2147
|
+
*/
|
|
2148
|
+
async startStep(step) {
|
|
2149
|
+
if (this.trackStepStart) {
|
|
2150
|
+
await this.store.saveCheckpoint({
|
|
2151
|
+
runId: step.runId,
|
|
2152
|
+
seq: step.seq,
|
|
2153
|
+
name: step.name,
|
|
2154
|
+
kind: step.kind,
|
|
2155
|
+
stepId: stepId(step.runId, step.seq),
|
|
2156
|
+
status: "running",
|
|
2157
|
+
input: step.input,
|
|
2158
|
+
events: step.events && step.events.length > 0 ? step.events : void 0,
|
|
2159
|
+
attempts: step.attempts,
|
|
2160
|
+
workerGroup: step.workerGroup,
|
|
2161
|
+
enqueuedAt: step.enqueuedAt,
|
|
2162
|
+
startedAt: step.startedAt,
|
|
2163
|
+
finishedAt: step.startedAt
|
|
2164
|
+
});
|
|
2165
|
+
}
|
|
2166
|
+
this.emit({
|
|
2167
|
+
type: "step.started",
|
|
2168
|
+
runId: step.runId,
|
|
2169
|
+
seq: step.seq,
|
|
2170
|
+
name: step.name,
|
|
2171
|
+
kind: step.kind
|
|
2172
|
+
});
|
|
2173
|
+
}
|
|
2174
|
+
/** Checkpoint a finished step and announce it — the two things that must always happen together. */
|
|
2175
|
+
async completeStep(step) {
|
|
2176
|
+
await this.store.saveCheckpoint({
|
|
2177
|
+
runId: step.runId,
|
|
2178
|
+
seq: step.seq,
|
|
2179
|
+
name: step.name,
|
|
2180
|
+
kind: step.kind,
|
|
2181
|
+
stepId: stepId(step.runId, step.seq),
|
|
2182
|
+
status: "completed",
|
|
2183
|
+
input: step.input,
|
|
2184
|
+
output: step.output,
|
|
2185
|
+
events: step.events && step.events.length > 0 ? step.events : void 0,
|
|
2186
|
+
attempts: step.attempts,
|
|
2187
|
+
workerGroup: step.workerGroup,
|
|
2188
|
+
enqueuedAt: step.enqueuedAt,
|
|
2189
|
+
startedAt: step.startedAt,
|
|
2190
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
2191
|
+
});
|
|
2192
|
+
this.emit({
|
|
2193
|
+
type: "step.completed",
|
|
2194
|
+
runId: step.runId,
|
|
2195
|
+
seq: step.seq,
|
|
2196
|
+
name: step.name,
|
|
2197
|
+
kind: step.kind,
|
|
2198
|
+
output: step.output,
|
|
2199
|
+
queueMs: step.startedAt.getTime() - step.enqueuedAt.getTime(),
|
|
2200
|
+
durationMs: Date.now() - step.startedAt.getTime()
|
|
2201
|
+
});
|
|
2202
|
+
}
|
|
2203
|
+
/** Checkpoint a step that failed terminally, so the failure point is visible (not just the run). */
|
|
2204
|
+
async failStep(step) {
|
|
2205
|
+
await this.store.saveCheckpoint({
|
|
2206
|
+
runId: step.runId,
|
|
2207
|
+
seq: step.seq,
|
|
2208
|
+
name: step.name,
|
|
2209
|
+
kind: step.kind,
|
|
2210
|
+
stepId: stepId(step.runId, step.seq),
|
|
2211
|
+
status: "failed",
|
|
2212
|
+
input: step.input,
|
|
2213
|
+
error: step.error,
|
|
2214
|
+
events: step.events && step.events.length > 0 ? step.events : void 0,
|
|
2215
|
+
attempts: step.attempts,
|
|
2216
|
+
workerGroup: step.workerGroup,
|
|
2217
|
+
enqueuedAt: step.enqueuedAt,
|
|
2218
|
+
startedAt: step.startedAt,
|
|
2219
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
2220
|
+
});
|
|
2221
|
+
this.emit({
|
|
2222
|
+
type: "step.failed",
|
|
2223
|
+
runId: step.runId,
|
|
2224
|
+
seq: step.seq,
|
|
2225
|
+
name: step.name,
|
|
2226
|
+
kind: step.kind,
|
|
2227
|
+
error: step.error,
|
|
2228
|
+
queueMs: step.startedAt.getTime() - step.enqueuedAt.getTime(),
|
|
2229
|
+
durationMs: Date.now() - step.startedAt.getTime()
|
|
2230
|
+
});
|
|
2231
|
+
}
|
|
2232
|
+
/**
|
|
2233
|
+
* Whether `run` may run now under its singleton key: it's among the `limit` oldest in-flight runs
|
|
2234
|
+
* (running or suspended) sharing the key, by `(createdAt, id)` order. A consistent store gives every
|
|
2235
|
+
* instance the same ordering, so admission is race-free + FIFO.
|
|
2236
|
+
*/
|
|
2237
|
+
async admitSingleton(run, cfg) {
|
|
2238
|
+
const tag = singletonTag(cfg, run.input);
|
|
2239
|
+
const inflight = (await this.store.listRuns({
|
|
2240
|
+
tag,
|
|
2241
|
+
workflow: run.workflow,
|
|
2242
|
+
statuses: [
|
|
2243
|
+
"running",
|
|
2244
|
+
"suspended"
|
|
2245
|
+
]
|
|
2246
|
+
})).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
|
|
2247
|
+
const idx = inflight.findIndex((r) => r.id === run.id);
|
|
2248
|
+
return idx >= 0 && idx < (cfg.limit ?? 1);
|
|
2249
|
+
}
|
|
2250
|
+
/**
|
|
2251
|
+
* Notify-on-release: a singleton run just settled (freeing a slot), so wake the gated waiters that
|
|
2252
|
+
* can now be admitted — immediately, instead of letting them sleep until their ~1s retry timer.
|
|
2253
|
+
* Dispatches up to `limit` of the oldest gated (`suspended` + same singleton tag) waiters by
|
|
2254
|
+
* `(createdAt, id)`; each re-checks admission in `runExecution` and runs only if it actually wins a
|
|
2255
|
+
* slot, so this never breaks the FIFO + race-free guarantees (the durable timer remains the
|
|
2256
|
+
* cross-instance / crash fallback — this just removes the poll latency on the happy path).
|
|
2257
|
+
*/
|
|
2258
|
+
async wakeNextSingletons(settled) {
|
|
2259
|
+
const registered = this.workflows.get(versionKey(settled.workflow, settled.workflowVersion)) ?? this.latest.get(settled.workflow);
|
|
2260
|
+
const cfg = registered?.singleton;
|
|
2261
|
+
if (!cfg) return;
|
|
2262
|
+
const tag = settled.tags?.find((t) => t.startsWith("singleton:"));
|
|
2263
|
+
if (!tag) return;
|
|
2264
|
+
const gated = (await this.store.listRuns({
|
|
2265
|
+
tag,
|
|
2266
|
+
workflow: settled.workflow,
|
|
2267
|
+
statuses: [
|
|
2268
|
+
"suspended"
|
|
2269
|
+
]
|
|
2270
|
+
})).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
|
|
2271
|
+
for (const next of gated.slice(0, cfg.limit ?? 1)) {
|
|
2272
|
+
if (next.wakeAt != null) {
|
|
2273
|
+
await this.store.updateRun(next.id, {
|
|
2274
|
+
wakeAt: void 0,
|
|
2275
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2276
|
+
}).catch(() => void 0);
|
|
2277
|
+
}
|
|
2278
|
+
this.runDispatcher.dispatch(next.id);
|
|
2279
|
+
}
|
|
2280
|
+
}
|
|
2281
|
+
async execute(run, fn) {
|
|
2282
|
+
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
|
|
2283
|
+
const lockNow = this.clock();
|
|
2284
|
+
if (run.lockedBy !== this.instanceId || (run.lockedUntil ?? 0) <= lockNow) {
|
|
2285
|
+
if (!await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow)) {
|
|
2286
|
+
return {
|
|
2287
|
+
runId: run.id,
|
|
2288
|
+
status: run.status
|
|
2289
|
+
};
|
|
2290
|
+
}
|
|
2291
|
+
}
|
|
2292
|
+
const renew = setInterval(() => {
|
|
2293
|
+
void this.store.renewRunLock(run.id, this.instanceId, this.clock() + this.leaseMs).catch(() => void 0);
|
|
2294
|
+
}, Math.max(50, Math.floor(this.leaseMs / 2)));
|
|
2295
|
+
renew.unref?.();
|
|
2296
|
+
try {
|
|
2297
|
+
const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
|
|
2298
|
+
if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
|
|
2299
|
+
void this.wakeNextSingletons(run).catch(() => void 0);
|
|
2300
|
+
}
|
|
2301
|
+
return result;
|
|
2302
|
+
} finally {
|
|
2303
|
+
clearInterval(renew);
|
|
2304
|
+
}
|
|
2305
|
+
}
|
|
2306
|
+
/**
|
|
2307
|
+
* Advance a remote (cross-SDK) workflow one turn: hand its history to the executor (which dispatches
|
|
2308
|
+
* a workflow task to the worker and awaits its replay) and apply the decision. Mirrors
|
|
2309
|
+
* {@link runExecution}'s settle/suspend; the lease is held by {@link execute}. The result that lands
|
|
2310
|
+
* us back here (a remote step finished, a timer fired) goes through `resume` like any TS workflow.
|
|
2311
|
+
*/
|
|
2312
|
+
async runRemoteExecution(run, registered) {
|
|
2313
|
+
const remote = registered.remote;
|
|
2314
|
+
if (run.status === "pending") {
|
|
2315
|
+
await this.store.updateRun(run.id, {
|
|
2316
|
+
status: "running",
|
|
2317
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2318
|
+
});
|
|
2319
|
+
run.status = "running";
|
|
2320
|
+
this.emit({
|
|
2321
|
+
type: "run.started",
|
|
2322
|
+
runId: run.id,
|
|
2323
|
+
workflow: run.workflow
|
|
2324
|
+
});
|
|
2325
|
+
}
|
|
2326
|
+
if (registered.singleton && !await this.admitSingleton(run, registered.singleton)) {
|
|
2327
|
+
const wakeAt2 = singletonRetryWakeAt(this.clock());
|
|
2328
|
+
await this.store.updateRun(run.id, {
|
|
2329
|
+
status: "suspended",
|
|
2330
|
+
wakeAt: wakeAt2,
|
|
2331
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2332
|
+
});
|
|
2333
|
+
this.emit({
|
|
2334
|
+
type: "run.suspended",
|
|
2335
|
+
runId: run.id,
|
|
2336
|
+
workflow: run.workflow
|
|
2337
|
+
});
|
|
2338
|
+
await this.store.releaseRunLock(run.id);
|
|
2339
|
+
return {
|
|
2340
|
+
runId: run.id,
|
|
2341
|
+
status: "suspended"
|
|
2342
|
+
};
|
|
2343
|
+
}
|
|
2344
|
+
const history = await this.remoteHistory(run.id);
|
|
2345
|
+
let decision;
|
|
2346
|
+
try {
|
|
2347
|
+
decision = await remote.executor.advance(run, history);
|
|
2348
|
+
} catch (err) {
|
|
2349
|
+
const error = {
|
|
2350
|
+
message: err instanceof Error ? err.message : String(err),
|
|
2351
|
+
stack: err instanceof Error ? err.stack : void 0
|
|
2352
|
+
};
|
|
2353
|
+
await this.store.updateRun(run.id, {
|
|
2354
|
+
status: "failed",
|
|
2355
|
+
error,
|
|
2356
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2357
|
+
});
|
|
2358
|
+
this.emit({
|
|
2359
|
+
type: "run.failed",
|
|
2360
|
+
runId: run.id,
|
|
2361
|
+
workflow: run.workflow,
|
|
2362
|
+
error
|
|
2363
|
+
});
|
|
2364
|
+
return {
|
|
2365
|
+
runId: run.id,
|
|
2366
|
+
status: "failed",
|
|
2367
|
+
error
|
|
2368
|
+
};
|
|
2369
|
+
}
|
|
2370
|
+
if (decision.status === "completed") {
|
|
2371
|
+
await this.applyCommands(run, decision.commands);
|
|
2372
|
+
await this.store.updateRun(run.id, {
|
|
2373
|
+
status: "completed",
|
|
2374
|
+
output: decision.output,
|
|
2375
|
+
error: void 0,
|
|
2376
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2377
|
+
});
|
|
2378
|
+
this.emit({
|
|
2379
|
+
type: "run.completed",
|
|
2380
|
+
runId: run.id,
|
|
2381
|
+
workflow: run.workflow,
|
|
2382
|
+
output: decision.output
|
|
2383
|
+
});
|
|
2384
|
+
void this.notifyParent(run.id, {
|
|
2385
|
+
ok: true,
|
|
2386
|
+
value: decision.output
|
|
2387
|
+
});
|
|
2388
|
+
return {
|
|
2389
|
+
runId: run.id,
|
|
2390
|
+
status: "completed",
|
|
2391
|
+
output: decision.output
|
|
2392
|
+
};
|
|
2393
|
+
}
|
|
2394
|
+
if (decision.status === "failed") {
|
|
2395
|
+
await this.applyCommands(run, decision.commands);
|
|
2396
|
+
const error = decision.error ?? {
|
|
2397
|
+
message: "workflow failed"
|
|
2398
|
+
};
|
|
2399
|
+
await this.store.updateRun(run.id, {
|
|
2400
|
+
status: "failed",
|
|
2401
|
+
error,
|
|
2402
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2403
|
+
});
|
|
2404
|
+
this.emit({
|
|
2405
|
+
type: "run.failed",
|
|
2406
|
+
runId: run.id,
|
|
2407
|
+
workflow: run.workflow,
|
|
2408
|
+
error
|
|
2409
|
+
});
|
|
2410
|
+
void this.notifyParent(run.id, {
|
|
2411
|
+
ok: false,
|
|
2412
|
+
error: error.message
|
|
2413
|
+
});
|
|
2414
|
+
return {
|
|
2415
|
+
runId: run.id,
|
|
2416
|
+
status: "failed",
|
|
2417
|
+
error
|
|
2418
|
+
};
|
|
2419
|
+
}
|
|
2420
|
+
const wakeAt = await this.applyCommands(run, decision.commands);
|
|
2421
|
+
await this.store.updateRun(run.id, {
|
|
2422
|
+
status: "suspended",
|
|
2423
|
+
wakeAt,
|
|
2424
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2425
|
+
});
|
|
2426
|
+
this.emit({
|
|
2427
|
+
type: "run.suspended",
|
|
2428
|
+
runId: run.id,
|
|
2429
|
+
workflow: run.workflow
|
|
2430
|
+
});
|
|
2431
|
+
return {
|
|
2432
|
+
runId: run.id,
|
|
2433
|
+
status: "suspended"
|
|
2434
|
+
};
|
|
2435
|
+
}
|
|
2436
|
+
/** The run's resolved durable ops as replay inputs: completed/failed steps + elapsed timers. */
|
|
2437
|
+
async remoteHistory(runId) {
|
|
2438
|
+
const checkpoints = await this.store.listCheckpoints(runId);
|
|
2439
|
+
const kindOf = {
|
|
2440
|
+
remote: "call",
|
|
2441
|
+
local: "step",
|
|
2442
|
+
sleep: "timer",
|
|
2443
|
+
signal: "signal"
|
|
2444
|
+
};
|
|
2445
|
+
const events = [];
|
|
2446
|
+
for (const cp of checkpoints) {
|
|
2447
|
+
if (cp.status === "completed" || cp.status === "failed") {
|
|
2448
|
+
if (cp.kind === "signal" && cp.name.startsWith("signal:child:")) {
|
|
2449
|
+
const completion = cp.output;
|
|
2450
|
+
events.push({
|
|
2451
|
+
seq: cp.seq,
|
|
2452
|
+
kind: "child",
|
|
2453
|
+
output: completion?.ok ? completion.value : void 0,
|
|
2454
|
+
error: completion && completion.ok === false ? {
|
|
2455
|
+
message: completion.error
|
|
2456
|
+
} : void 0
|
|
2457
|
+
});
|
|
2458
|
+
continue;
|
|
2459
|
+
}
|
|
2460
|
+
events.push({
|
|
2461
|
+
seq: cp.seq,
|
|
2462
|
+
kind: kindOf[cp.kind] ?? "step",
|
|
2463
|
+
// A signal checkpoint's name is the internal `signal:<token>`, not the workflow-level signal
|
|
2464
|
+
// name the replay used — omit it so the replay matches on seq + kind (its determinism anchor).
|
|
2465
|
+
name: cp.kind === "signal" ? void 0 : cp.name,
|
|
2466
|
+
output: cp.status === "completed" ? cp.output : void 0,
|
|
2467
|
+
error: cp.status === "failed" ? cp.error : void 0
|
|
2468
|
+
});
|
|
2469
|
+
} else if (cp.kind === "sleep" && cp.wakeAt != null && cp.wakeAt <= this.clock()) {
|
|
2470
|
+
events.push({
|
|
2471
|
+
seq: cp.seq,
|
|
2472
|
+
kind: "timer",
|
|
2473
|
+
name: cp.name
|
|
2474
|
+
});
|
|
2475
|
+
}
|
|
2476
|
+
}
|
|
2477
|
+
return events.sort((a, b) => a.seq - b.seq);
|
|
2478
|
+
}
|
|
2479
|
+
/** Apply a turn's commands: persist recorded local steps, dispatch remote calls, schedule timers.
|
|
2480
|
+
* Returns the earliest timer deadline to suspend on (or undefined — suspended on a result). */
|
|
2481
|
+
async applyCommands(run, commands) {
|
|
2482
|
+
let wakeAt;
|
|
2483
|
+
for (const cmd of commands) {
|
|
2484
|
+
const at = /* @__PURE__ */ new Date();
|
|
2485
|
+
const id = stepId(run.id, cmd.seq);
|
|
2486
|
+
if (cmd.kind === "recordStep") {
|
|
2487
|
+
const startedAt = cmd.startedAt != null ? new Date(cmd.startedAt) : at;
|
|
2488
|
+
const finishedAt = cmd.finishedAt != null ? new Date(cmd.finishedAt) : at;
|
|
2489
|
+
await this.store.saveCheckpoint({
|
|
2490
|
+
runId: run.id,
|
|
2491
|
+
seq: cmd.seq,
|
|
2492
|
+
name: cmd.name,
|
|
2493
|
+
kind: "local",
|
|
2494
|
+
stepId: id,
|
|
2495
|
+
status: cmd.error ? "failed" : "completed",
|
|
2496
|
+
output: cmd.output,
|
|
2497
|
+
error: cmd.error,
|
|
2498
|
+
events: cmd.events && cmd.events.length > 0 ? cmd.events : void 0,
|
|
2499
|
+
attempts: 1,
|
|
2500
|
+
enqueuedAt: startedAt,
|
|
2501
|
+
startedAt,
|
|
2502
|
+
finishedAt
|
|
2503
|
+
});
|
|
2504
|
+
this.emit({
|
|
2505
|
+
type: cmd.error ? "step.failed" : "step.completed",
|
|
2506
|
+
runId: run.id,
|
|
2507
|
+
seq: cmd.seq,
|
|
2508
|
+
name: cmd.name,
|
|
2509
|
+
kind: "local",
|
|
2510
|
+
output: cmd.output,
|
|
2511
|
+
error: cmd.error
|
|
2512
|
+
});
|
|
2513
|
+
} else if (cmd.kind === "call") {
|
|
2514
|
+
await this.store.saveCheckpoint({
|
|
2515
|
+
runId: run.id,
|
|
2516
|
+
seq: cmd.seq,
|
|
2517
|
+
name: cmd.name,
|
|
2518
|
+
kind: "remote",
|
|
2519
|
+
stepId: id,
|
|
2520
|
+
status: "pending",
|
|
2521
|
+
input: cmd.input,
|
|
2522
|
+
attempts: 1,
|
|
2523
|
+
workerGroup: cmd.group,
|
|
2524
|
+
enqueuedAt: at,
|
|
2525
|
+
startedAt: at,
|
|
2526
|
+
finishedAt: at
|
|
2527
|
+
});
|
|
2528
|
+
await this.pool.dispatch({
|
|
2529
|
+
runId: run.id,
|
|
2530
|
+
seq: cmd.seq,
|
|
2531
|
+
name: cmd.name,
|
|
2532
|
+
stepId: id,
|
|
2533
|
+
group: cmd.group,
|
|
2534
|
+
input: cmd.input,
|
|
2535
|
+
traceparent: this.traceparent?.(),
|
|
2536
|
+
context: this.context?.(),
|
|
2537
|
+
attempt: 1
|
|
2538
|
+
}, void 0);
|
|
2539
|
+
this.emit({
|
|
2540
|
+
type: "step.started",
|
|
2541
|
+
runId: run.id,
|
|
2542
|
+
seq: cmd.seq,
|
|
2543
|
+
name: cmd.name,
|
|
2544
|
+
kind: "remote"
|
|
2545
|
+
});
|
|
2546
|
+
} else if (cmd.kind === "sleep") {
|
|
2547
|
+
const deadline = this.clock() + cmd.ms;
|
|
2548
|
+
await this.store.saveCheckpoint({
|
|
2549
|
+
runId: run.id,
|
|
2550
|
+
seq: cmd.seq,
|
|
2551
|
+
name: `sleep:${cmd.seq}`,
|
|
2552
|
+
kind: "sleep",
|
|
2553
|
+
stepId: id,
|
|
2554
|
+
status: "pending",
|
|
2555
|
+
attempts: 1,
|
|
2556
|
+
wakeAt: deadline,
|
|
2557
|
+
enqueuedAt: at,
|
|
2558
|
+
startedAt: at,
|
|
2559
|
+
finishedAt: at
|
|
2560
|
+
});
|
|
2561
|
+
wakeAt = wakeAt == null ? deadline : Math.min(wakeAt, deadline);
|
|
2562
|
+
} else if (cmd.kind === "waitSignal") {
|
|
2563
|
+
const buffered = await this.store.takeBufferedSignal(cmd.signal);
|
|
2564
|
+
if (buffered) {
|
|
2565
|
+
await this.store.saveCheckpoint(instantCheckpoint({
|
|
2566
|
+
runId: run.id,
|
|
2567
|
+
seq: cmd.seq,
|
|
2568
|
+
name: `signal:${cmd.signal}`,
|
|
2569
|
+
kind: "signal",
|
|
2570
|
+
output: buffered.payload
|
|
2571
|
+
}));
|
|
2572
|
+
setTimeout(() => void this.resume(run.id).catch(() => void 0), 0);
|
|
2573
|
+
} else {
|
|
2574
|
+
await this.store.putSignalWaiter({
|
|
2575
|
+
token: cmd.signal,
|
|
2576
|
+
runId: run.id,
|
|
2577
|
+
seq: cmd.seq
|
|
2578
|
+
});
|
|
2579
|
+
}
|
|
2580
|
+
} else if (cmd.kind === "startChild") {
|
|
2581
|
+
const childId = `${run.id}.child.${cmd.seq}`;
|
|
2582
|
+
await this.store.putSignalWaiter({
|
|
2583
|
+
token: `child:${childId}`,
|
|
2584
|
+
runId: run.id,
|
|
2585
|
+
seq: cmd.seq
|
|
2586
|
+
});
|
|
2587
|
+
if (!await this.store.getRun(childId)) {
|
|
2588
|
+
queueMicrotask(() => void this.start(cmd.workflow, cmd.input, childId).catch(() => void 0));
|
|
2589
|
+
}
|
|
2590
|
+
} else {
|
|
2591
|
+
throw new Error(`remote workflow command '${cmd.kind}' is not supported yet`);
|
|
2592
|
+
}
|
|
2593
|
+
}
|
|
2594
|
+
return wakeAt;
|
|
2595
|
+
}
|
|
2596
|
+
/** The run body, lease held + renewed by {@link execute}. */
|
|
2597
|
+
async runExecution(run, fn, registered) {
|
|
2598
|
+
if (run.status === "pending") {
|
|
2599
|
+
await this.store.updateRun(run.id, {
|
|
2600
|
+
status: "running",
|
|
2601
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2602
|
+
});
|
|
2603
|
+
run.status = "running";
|
|
2604
|
+
this.emit({
|
|
2605
|
+
type: "run.started",
|
|
2606
|
+
runId: run.id,
|
|
2607
|
+
workflow: run.workflow
|
|
2608
|
+
});
|
|
2609
|
+
}
|
|
2610
|
+
if (registered?.singleton && !await this.admitSingleton(run, registered.singleton)) {
|
|
2611
|
+
const wakeAt = singletonRetryWakeAt(this.clock());
|
|
2612
|
+
await this.store.updateRun(run.id, {
|
|
2613
|
+
status: "suspended",
|
|
2614
|
+
wakeAt,
|
|
2615
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2616
|
+
});
|
|
2617
|
+
this.emit({
|
|
2618
|
+
type: "run.suspended",
|
|
2619
|
+
runId: run.id,
|
|
2620
|
+
workflow: run.workflow
|
|
2621
|
+
});
|
|
2622
|
+
await this.store.releaseRunLock(run.id);
|
|
2623
|
+
return {
|
|
2624
|
+
runId: run.id,
|
|
2625
|
+
status: "suspended"
|
|
2626
|
+
};
|
|
2627
|
+
}
|
|
2628
|
+
const compensations = [];
|
|
2629
|
+
const snapshot = await this.store.listCheckpoints(run.id);
|
|
2630
|
+
const replay = /* @__PURE__ */ new Map();
|
|
2631
|
+
for (const cp of snapshot) replay.set(cp.seq, cp);
|
|
2632
|
+
const ctx = createWorkflowCtx(this.ctxHostFor(replay), run.id, compensations, run.workflow);
|
|
2633
|
+
try {
|
|
2634
|
+
const output = await fn(ctx, run.input);
|
|
2635
|
+
await this.store.updateRun(run.id, {
|
|
2636
|
+
status: "completed",
|
|
2637
|
+
output,
|
|
2638
|
+
error: void 0,
|
|
2639
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2640
|
+
});
|
|
2641
|
+
this.emit({
|
|
2642
|
+
type: "run.completed",
|
|
2643
|
+
runId: run.id,
|
|
2644
|
+
workflow: run.workflow,
|
|
2645
|
+
output
|
|
2646
|
+
});
|
|
2647
|
+
void this.notifyParent(run.id, {
|
|
2648
|
+
ok: true,
|
|
2649
|
+
value: output
|
|
2650
|
+
});
|
|
2651
|
+
return {
|
|
2652
|
+
runId: run.id,
|
|
2653
|
+
status: "completed",
|
|
2654
|
+
output
|
|
2655
|
+
};
|
|
2656
|
+
} catch (err) {
|
|
2657
|
+
if (err instanceof ContinueAsNew) {
|
|
2658
|
+
await this.store.updateRun(run.id, {
|
|
2659
|
+
status: "completed",
|
|
2660
|
+
output: void 0,
|
|
2661
|
+
error: void 0,
|
|
2662
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2663
|
+
});
|
|
2664
|
+
this.emit({
|
|
2665
|
+
type: "run.completed",
|
|
2666
|
+
runId: run.id,
|
|
2667
|
+
workflow: run.workflow
|
|
2668
|
+
});
|
|
2669
|
+
void this.notifyParent(run.id, {
|
|
2670
|
+
ok: true,
|
|
2671
|
+
value: void 0
|
|
2672
|
+
});
|
|
2673
|
+
const nextId = nextContinuationId(run.id);
|
|
2674
|
+
queueMicrotask(() => void this.start(run.workflow, err.input, nextId).catch(() => void 0));
|
|
2675
|
+
return {
|
|
2676
|
+
runId: run.id,
|
|
2677
|
+
status: "completed"
|
|
2678
|
+
};
|
|
2679
|
+
}
|
|
2680
|
+
if (err instanceof WorkflowSuspended) {
|
|
2681
|
+
if (this.cancelRequested.has(run.id)) {
|
|
2682
|
+
this.cancelRequested.delete(run.id);
|
|
2683
|
+
for (let i = compensations.length - 1; i >= 0; i -= 1) {
|
|
2684
|
+
const comp = compensations[i];
|
|
2685
|
+
if (comp) await this.runCompensation(run, comp);
|
|
2686
|
+
}
|
|
2687
|
+
const error2 = {
|
|
2688
|
+
message: "cancelled"
|
|
2689
|
+
};
|
|
2690
|
+
await this.store.updateRun(run.id, {
|
|
2691
|
+
status: "cancelled",
|
|
2692
|
+
error: error2,
|
|
2693
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2694
|
+
});
|
|
2695
|
+
this.emit({
|
|
2696
|
+
type: "run.failed",
|
|
2697
|
+
runId: run.id,
|
|
2698
|
+
workflow: run.workflow,
|
|
2699
|
+
error: error2
|
|
2700
|
+
});
|
|
2701
|
+
return {
|
|
2702
|
+
runId: run.id,
|
|
2703
|
+
status: "cancelled",
|
|
2704
|
+
error: error2
|
|
2705
|
+
};
|
|
2706
|
+
}
|
|
2707
|
+
await this.store.updateRun(run.id, {
|
|
2708
|
+
status: "suspended",
|
|
2709
|
+
wakeAt: err.wakeAt,
|
|
2710
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2711
|
+
});
|
|
2712
|
+
this.emit({
|
|
2713
|
+
type: "run.suspended",
|
|
2714
|
+
runId: run.id,
|
|
2715
|
+
workflow: run.workflow
|
|
2716
|
+
});
|
|
2717
|
+
return {
|
|
2718
|
+
runId: run.id,
|
|
2719
|
+
status: "suspended"
|
|
2720
|
+
};
|
|
2721
|
+
}
|
|
2722
|
+
const error = {
|
|
2723
|
+
message: err instanceof Error ? err.message : String(err),
|
|
2724
|
+
stack: err instanceof Error ? err.stack : void 0
|
|
2725
|
+
};
|
|
2726
|
+
for (let i = compensations.length - 1; i >= 0; i -= 1) {
|
|
2727
|
+
const comp = compensations[i];
|
|
2728
|
+
if (!comp) continue;
|
|
2729
|
+
await this.runCompensation(run, comp);
|
|
2730
|
+
}
|
|
2731
|
+
await this.store.updateRun(run.id, {
|
|
2732
|
+
status: "failed",
|
|
2733
|
+
error,
|
|
2734
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
2735
|
+
});
|
|
2736
|
+
this.emit({
|
|
2737
|
+
type: "run.failed",
|
|
2738
|
+
runId: run.id,
|
|
2739
|
+
workflow: run.workflow,
|
|
2740
|
+
error
|
|
2741
|
+
});
|
|
2742
|
+
void this.notifyParent(run.id, {
|
|
2743
|
+
ok: false,
|
|
2744
|
+
error: error.message
|
|
2745
|
+
});
|
|
2746
|
+
return {
|
|
2747
|
+
runId: run.id,
|
|
2748
|
+
status: "failed",
|
|
2749
|
+
error
|
|
2750
|
+
};
|
|
2751
|
+
} finally {
|
|
2752
|
+
await this.store.releaseRunLock(run.id);
|
|
2753
|
+
}
|
|
2754
|
+
}
|
|
2755
|
+
/**
|
|
2756
|
+
* Run one saga compensation, retried up to `compensationRetries`, emitting a `compensate:<step>`
|
|
2757
|
+
* step event for its outcome so a stranded undo is visible. Never throws — a permanently-failing
|
|
2758
|
+
* compensation is skipped so it can't mask the original failure.
|
|
2759
|
+
*/
|
|
2760
|
+
async runCompensation(run, comp) {
|
|
2761
|
+
const name = `compensate:${comp.name}`;
|
|
2762
|
+
for (let attempt = 1; attempt <= this.compensationRetries; attempt += 1) {
|
|
2763
|
+
const startedAt = Date.now();
|
|
2764
|
+
try {
|
|
2765
|
+
await comp.fn();
|
|
2766
|
+
this.emit({
|
|
2767
|
+
type: "step.completed",
|
|
2768
|
+
runId: run.id,
|
|
2769
|
+
workflow: run.workflow,
|
|
2770
|
+
name,
|
|
2771
|
+
kind: "local",
|
|
2772
|
+
durationMs: Date.now() - startedAt
|
|
2773
|
+
});
|
|
2774
|
+
return;
|
|
2775
|
+
} catch (err) {
|
|
2776
|
+
if (attempt >= this.compensationRetries) {
|
|
2777
|
+
this.emit({
|
|
2778
|
+
type: "step.failed",
|
|
2779
|
+
runId: run.id,
|
|
2780
|
+
workflow: run.workflow,
|
|
2781
|
+
name,
|
|
2782
|
+
kind: "local",
|
|
2783
|
+
error: {
|
|
2784
|
+
message: err instanceof Error ? err.message : String(err)
|
|
2785
|
+
},
|
|
2786
|
+
durationMs: Date.now() - startedAt
|
|
2787
|
+
});
|
|
2788
|
+
}
|
|
2789
|
+
}
|
|
2790
|
+
}
|
|
2791
|
+
}
|
|
2792
|
+
/** The seam handed to {@link createWorkflowCtx}: the authoring API reaches durability + lifecycle
|
|
2793
|
+
* (checkpointing, dispatch, child start) through this, so the ctx primitives live in their own
|
|
2794
|
+
* module and the engine stays the orchestrator. */
|
|
2795
|
+
ctxHostFor(replay) {
|
|
2796
|
+
return {
|
|
2797
|
+
store: this.store,
|
|
2798
|
+
replay,
|
|
2799
|
+
clock: this.clock,
|
|
2800
|
+
webhookUrl: this.webhookUrl,
|
|
2801
|
+
startStep: /* @__PURE__ */ __name((s) => this.startStep(s), "startStep"),
|
|
2802
|
+
completeStep: /* @__PURE__ */ __name((s) => this.completeStep(s), "completeStep"),
|
|
2803
|
+
failStep: /* @__PURE__ */ __name((s) => this.failStep(s), "failStep"),
|
|
2804
|
+
callRemote: /* @__PURE__ */ __name((runId, seq, step, input, queue, transport, admission) => this.callRemote(runId, seq, step, input, queue, transport, replay, admission), "callRemote"),
|
|
2805
|
+
// Defer so a fast child can't reentrantly resume a still-running parent.
|
|
2806
|
+
startChild: /* @__PURE__ */ __name((workflow, input, id) => {
|
|
2807
|
+
queueMicrotask(() => void this.start(workflow, input, id).catch(() => void 0));
|
|
2808
|
+
}, "startChild"),
|
|
2809
|
+
signalEntity: /* @__PURE__ */ __name((name, key, op, arg, reply) => {
|
|
2810
|
+
queueMicrotask(() => void this.entities.dispatch(name, key, op, arg, reply).catch(() => void 0));
|
|
2811
|
+
}, "signalEntity"),
|
|
2812
|
+
interceptStep: /* @__PURE__ */ __name((invocation, body) => this.interceptStep(invocation, body), "interceptStep")
|
|
2813
|
+
};
|
|
2814
|
+
}
|
|
2815
|
+
/**
|
|
2816
|
+
* Resume a run paused at a {@link WorkflowCtx.breakpoint} (e.g. the dashboard "continue" button).
|
|
2817
|
+
* Finds the run's pending breakpoint checkpoint and signals it. Returns null if the run isn't
|
|
2818
|
+
* paused at a breakpoint.
|
|
2819
|
+
*/
|
|
2820
|
+
async continue(runId) {
|
|
2821
|
+
const checkpoints = await this.store.listCheckpoints(runId);
|
|
2822
|
+
const bp = checkpoints.find(isBreakpoint);
|
|
2823
|
+
if (!bp) return null;
|
|
2824
|
+
return this.signal(breakpointToken(runId, bp.seq), void 0);
|
|
2825
|
+
}
|
|
2826
|
+
/**
|
|
2827
|
+
* Read the latest value a run published for `key` via {@link WorkflowCtx.setEvent} — a
|
|
2828
|
+
* side-effect-free query of a live (or finished) run's state. Returns `undefined` if the run
|
|
2829
|
+
* never published that key. The suspend-model counterpart of a Temporal query.
|
|
2830
|
+
*/
|
|
2831
|
+
async getEvent(runId, key) {
|
|
2832
|
+
const name = `event:${key}`;
|
|
2833
|
+
const checkpoints = await this.store.listCheckpoints(runId);
|
|
2834
|
+
let latest;
|
|
2835
|
+
for (const cp of checkpoints) if (cp.name === name) latest = cp.output;
|
|
2836
|
+
return latest;
|
|
2837
|
+
}
|
|
2838
|
+
/**
|
|
2839
|
+
* Register a validator gating `engine.update(runId, name, …)` for runs of `workflow`. The
|
|
2840
|
+
* validator runs BEFORE the update is delivered, so a rejection leaves the run untouched. One
|
|
2841
|
+
* validator per (workflow, update name); registering again replaces it.
|
|
2842
|
+
*/
|
|
2843
|
+
registerUpdateValidator(workflow, name, validate) {
|
|
2844
|
+
this.updateValidators.set(`${workflow}:${name}`, validate);
|
|
2845
|
+
}
|
|
2846
|
+
/**
|
|
2847
|
+
* Deliver a validated update to the run waiting at `ctx.onUpdate(name)`. Runs the registered
|
|
2848
|
+
* validator (if any) first: on rejection returns `{ accepted: false, reason }` without disturbing
|
|
2849
|
+
* the run; otherwise delivers `arg` and resumes, returning `{ accepted: true, run }` (`run` is null
|
|
2850
|
+
* if nothing was waiting — a too-early or duplicate update).
|
|
2851
|
+
*/
|
|
2852
|
+
async update(runId, name, arg) {
|
|
2853
|
+
const run = await this.store.getRun(runId);
|
|
2854
|
+
if (!run) return {
|
|
2855
|
+
accepted: false,
|
|
2856
|
+
reason: `run ${runId} not found`
|
|
2857
|
+
};
|
|
2858
|
+
const validate = this.updateValidators.get(`${run.workflow}:${name}`);
|
|
2859
|
+
if (validate) {
|
|
2860
|
+
try {
|
|
2861
|
+
const reason = await validate(arg);
|
|
2862
|
+
if (typeof reason === "string" && reason.length > 0) return {
|
|
2863
|
+
accepted: false,
|
|
2864
|
+
reason
|
|
2865
|
+
};
|
|
2866
|
+
} catch (err) {
|
|
2867
|
+
return {
|
|
2868
|
+
accepted: false,
|
|
2869
|
+
reason: err instanceof Error ? err.message : String(err)
|
|
2870
|
+
};
|
|
2871
|
+
}
|
|
2872
|
+
}
|
|
2873
|
+
const result = await this.signal(`update:${runId}:${name}`, arg);
|
|
2874
|
+
return {
|
|
2875
|
+
accepted: true,
|
|
2876
|
+
run: result
|
|
2877
|
+
};
|
|
2878
|
+
}
|
|
2879
|
+
async callRemote(runId, seq, step, input, queue, transport, replay, admission) {
|
|
2880
|
+
const existing = replay?.get(seq) ?? await this.store.getCheckpoint(runId, seq);
|
|
2881
|
+
if (existing && existing.name !== step.name) {
|
|
2882
|
+
throw new NonDeterminismError(runId, seq, step.name, existing.name);
|
|
2883
|
+
}
|
|
2884
|
+
if (existing?.status === "completed") return existing.output;
|
|
2885
|
+
if (this.pool.size === 0) throw new Error("remote steps require a Transport");
|
|
2886
|
+
if (step.timeoutMs) return this.callRemoteInMemory(runId, seq, step, input, transport);
|
|
2887
|
+
if (existing?.status === "pending") throw new WorkflowSuspended();
|
|
2888
|
+
let attempt = 1;
|
|
2889
|
+
if (existing?.status === "failed") {
|
|
2890
|
+
const maxAttempts = Math.max(1, step.retries ?? 1);
|
|
2891
|
+
const retryable = existing.error?.retryable !== false;
|
|
2892
|
+
if (!retryable || existing.attempts >= maxAttempts) throw new RemoteStepError(existing.error);
|
|
2893
|
+
if (existing.wakeAt == null) {
|
|
2894
|
+
const wakeAt = this.clock() + backoffDelay(existing.attempts, step);
|
|
2895
|
+
await this.store.saveCheckpoint({
|
|
2896
|
+
...existing,
|
|
2897
|
+
wakeAt
|
|
2898
|
+
});
|
|
2899
|
+
throw new WorkflowSuspended(wakeAt);
|
|
2900
|
+
}
|
|
2901
|
+
if (this.clock() < existing.wakeAt) throw new WorkflowSuspended(existing.wakeAt);
|
|
2902
|
+
attempt = existing.attempts + 1;
|
|
2903
|
+
}
|
|
2904
|
+
const id = stepId(runId, seq);
|
|
2905
|
+
const controller = queue ? this.queues.get(queue) : void 0;
|
|
2906
|
+
if (controller) {
|
|
2907
|
+
const decision = controller.tryAdmit({
|
|
2908
|
+
priority: admission?.priority,
|
|
2909
|
+
key: admission?.fairnessKey ?? runId,
|
|
2910
|
+
waiterId: id
|
|
2911
|
+
});
|
|
2912
|
+
if (!decision.ok) throw new WorkflowSuspended(decision.retryAt);
|
|
2913
|
+
this.stepQueue.set(id, queue);
|
|
2914
|
+
}
|
|
2915
|
+
const validInput = step.input.parse(input);
|
|
2916
|
+
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
2917
|
+
await this.store.saveCheckpoint({
|
|
2918
|
+
runId,
|
|
2919
|
+
seq,
|
|
2920
|
+
name: step.name,
|
|
2921
|
+
kind: "remote",
|
|
2922
|
+
stepId: id,
|
|
2923
|
+
status: "pending",
|
|
2924
|
+
input: validInput,
|
|
2925
|
+
attempts: attempt,
|
|
2926
|
+
workerGroup: step.group,
|
|
2927
|
+
enqueuedAt,
|
|
2928
|
+
startedAt: enqueuedAt,
|
|
2929
|
+
finishedAt: enqueuedAt
|
|
2930
|
+
});
|
|
2931
|
+
await this.pool.dispatch({
|
|
2932
|
+
runId,
|
|
2933
|
+
seq,
|
|
2934
|
+
name: step.name,
|
|
2935
|
+
stepId: id,
|
|
2936
|
+
group: step.group,
|
|
2937
|
+
input: validInput,
|
|
2938
|
+
traceparent: this.traceparent?.(),
|
|
2939
|
+
context: this.context?.(),
|
|
2940
|
+
attempt
|
|
2941
|
+
}, transport);
|
|
2942
|
+
this.emit({
|
|
2943
|
+
type: "step.started",
|
|
2944
|
+
runId,
|
|
2945
|
+
seq,
|
|
2946
|
+
name: step.name,
|
|
2947
|
+
kind: "remote"
|
|
2948
|
+
});
|
|
2949
|
+
throw new WorkflowSuspended();
|
|
2950
|
+
}
|
|
2951
|
+
/**
|
|
2952
|
+
* Complete a durable remote step from its worker result and resume the run — runs on whichever
|
|
2953
|
+
* instance receives the result (the dispatching one may be gone), so the run is crash/scale-safe.
|
|
2954
|
+
* A no-op if the checkpoint isn't `pending` (a duplicate or late delivery).
|
|
2955
|
+
*/
|
|
2956
|
+
async completeRemoteResult(result) {
|
|
2957
|
+
const cp = await this.store.getCheckpoint(result.runId, result.seq);
|
|
2958
|
+
if (!cp || cp.status !== "pending") return;
|
|
2959
|
+
this.releaseQueueSlot(cp.stepId);
|
|
2960
|
+
const run = await this.store.getRun(result.runId);
|
|
2961
|
+
if (run && (run.status === "cancelled" || run.status === "completed")) return;
|
|
2962
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
2963
|
+
const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
|
|
2964
|
+
await this.store.saveCheckpoint({
|
|
2965
|
+
...cp,
|
|
2966
|
+
status: result.status,
|
|
2967
|
+
output: result.status === "completed" ? result.output : cp.output,
|
|
2968
|
+
error: result.error,
|
|
2969
|
+
events: result.events ?? cp.events,
|
|
2970
|
+
startedAt,
|
|
2971
|
+
finishedAt
|
|
2972
|
+
});
|
|
2973
|
+
this.emit({
|
|
2974
|
+
type: result.status === "completed" ? "step.completed" : "step.failed",
|
|
2975
|
+
runId: result.runId,
|
|
2976
|
+
seq: result.seq,
|
|
2977
|
+
name: cp.name,
|
|
2978
|
+
kind: cp.kind,
|
|
2979
|
+
output: result.output,
|
|
2980
|
+
error: result.error,
|
|
2981
|
+
queueMs: startedAt.getTime() - cp.enqueuedAt.getTime(),
|
|
2982
|
+
durationMs: finishedAt.getTime() - startedAt.getTime()
|
|
2983
|
+
});
|
|
2984
|
+
await this.resume(result.runId);
|
|
2985
|
+
}
|
|
2986
|
+
/** Release the flow-control slot a dispatched step held (if any), by its stepId. */
|
|
2987
|
+
releaseQueueSlot(id) {
|
|
2988
|
+
const queue = this.stepQueue.get(id);
|
|
2989
|
+
if (queue === void 0) return;
|
|
2990
|
+
this.stepQueue.delete(id);
|
|
2991
|
+
this.queues.get(queue)?.release();
|
|
2992
|
+
}
|
|
2993
|
+
/** In-memory await path for a remote step with a liveness `timeoutMs` (re-dispatch on timeout). */
|
|
2994
|
+
async callRemoteInMemory(runId, seq, step, input, transport) {
|
|
2995
|
+
if (this.pool.size === 0) throw new Error("remote steps require a Transport");
|
|
2996
|
+
const validInput = step.input.parse(input);
|
|
2997
|
+
const id = stepId(runId, seq);
|
|
2998
|
+
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
2999
|
+
this.emit({
|
|
3000
|
+
type: "step.started",
|
|
3001
|
+
runId,
|
|
3002
|
+
seq,
|
|
3003
|
+
name: step.name,
|
|
3004
|
+
kind: "remote"
|
|
3005
|
+
});
|
|
3006
|
+
const maxAttempts = step.timeoutMs ? Math.max(1, step.retries ?? 1) : 1;
|
|
3007
|
+
for (let attempt = 1; ; attempt += 1) {
|
|
3008
|
+
const resultPromise = new Promise((resolve, reject) => {
|
|
3009
|
+
this.pending.set(id, {
|
|
3010
|
+
resolve,
|
|
3011
|
+
reject
|
|
3012
|
+
});
|
|
3013
|
+
});
|
|
3014
|
+
await this.pool.dispatch({
|
|
3015
|
+
runId,
|
|
3016
|
+
seq,
|
|
3017
|
+
name: step.name,
|
|
3018
|
+
stepId: id,
|
|
3019
|
+
group: step.group,
|
|
3020
|
+
input: validInput,
|
|
3021
|
+
traceparent: this.traceparent?.(),
|
|
3022
|
+
context: this.context?.(),
|
|
3023
|
+
attempt
|
|
3024
|
+
}, transport);
|
|
3025
|
+
try {
|
|
3026
|
+
const resolution = step.timeoutMs ? await this.awaitWithHeartbeat(id, resultPromise, step.timeoutMs) : await resultPromise;
|
|
3027
|
+
const output = step.output.parse(resolution.output);
|
|
3028
|
+
const startedAt = resolution.startedAt ? new Date(resolution.startedAt) : enqueuedAt;
|
|
3029
|
+
await this.completeStep({
|
|
3030
|
+
runId,
|
|
3031
|
+
seq,
|
|
3032
|
+
name: step.name,
|
|
3033
|
+
kind: "remote",
|
|
3034
|
+
input: validInput,
|
|
3035
|
+
output,
|
|
3036
|
+
events: resolution.events,
|
|
3037
|
+
attempts: attempt,
|
|
3038
|
+
workerGroup: step.group,
|
|
3039
|
+
enqueuedAt,
|
|
3040
|
+
startedAt
|
|
3041
|
+
});
|
|
3042
|
+
return output;
|
|
3043
|
+
} catch (err) {
|
|
3044
|
+
this.pending.delete(id);
|
|
3045
|
+
if (err instanceof RemoteStepTimeout && attempt < maxAttempts) continue;
|
|
3046
|
+
throw err;
|
|
3047
|
+
}
|
|
3048
|
+
}
|
|
3049
|
+
}
|
|
3050
|
+
/**
|
|
3051
|
+
* Await a remote result, but reject with `RemoteStepTimeout` if neither the result nor a heartbeat
|
|
3052
|
+
* arrives within `timeoutMs`. Each heartbeat (delivered via `transport.onHeartbeat`) rearms the
|
|
3053
|
+
* window, so a worker that keeps beating stays alive past `timeoutMs`.
|
|
3054
|
+
*/
|
|
3055
|
+
awaitWithHeartbeat(id, resultPromise, timeoutMs) {
|
|
3056
|
+
return new Promise((resolve, reject) => {
|
|
3057
|
+
let timer;
|
|
3058
|
+
const cleanup = /* @__PURE__ */ __name(() => {
|
|
3059
|
+
clearTimeout(timer);
|
|
3060
|
+
this.heartbeatResets.delete(id);
|
|
3061
|
+
}, "cleanup");
|
|
3062
|
+
const arm = /* @__PURE__ */ __name(() => {
|
|
3063
|
+
timer = setTimeout(() => {
|
|
3064
|
+
cleanup();
|
|
3065
|
+
this.pending.delete(id);
|
|
3066
|
+
reject(new RemoteStepTimeout(id, timeoutMs));
|
|
3067
|
+
}, timeoutMs);
|
|
3068
|
+
timer.unref?.();
|
|
3069
|
+
}, "arm");
|
|
3070
|
+
this.heartbeatResets.set(id, () => {
|
|
3071
|
+
clearTimeout(timer);
|
|
3072
|
+
arm();
|
|
3073
|
+
});
|
|
3074
|
+
arm();
|
|
3075
|
+
resultPromise.then((value) => {
|
|
3076
|
+
cleanup();
|
|
3077
|
+
resolve(value);
|
|
3078
|
+
}, (err) => {
|
|
3079
|
+
cleanup();
|
|
3080
|
+
reject(err);
|
|
3081
|
+
});
|
|
3082
|
+
});
|
|
3083
|
+
}
|
|
3084
|
+
};
|
|
3085
|
+
var RemoteStepError = class extends Error {
|
|
3086
|
+
static {
|
|
3087
|
+
__name(this, "RemoteStepError");
|
|
3088
|
+
}
|
|
3089
|
+
stepError;
|
|
3090
|
+
constructor(stepError) {
|
|
3091
|
+
super(stepError?.message ?? "remote step failed");
|
|
3092
|
+
this.name = "RemoteStepError";
|
|
3093
|
+
this.stepError = stepError;
|
|
3094
|
+
}
|
|
3095
|
+
};
|
|
3096
|
+
|
|
3097
|
+
// src/remote-step-factory.ts
|
|
3098
|
+
function remoteStep(config) {
|
|
3099
|
+
return {
|
|
3100
|
+
...config,
|
|
3101
|
+
group: config.group ?? config.name.split(".")[0] ?? config.name,
|
|
3102
|
+
__remote: true
|
|
3103
|
+
};
|
|
3104
|
+
}
|
|
3105
|
+
__name(remoteStep, "remoteStep");
|
|
3106
|
+
|
|
3107
|
+
// src/remote-workflow-executor.ts
|
|
3108
|
+
var taskCounter = 0;
|
|
3109
|
+
var RemoteWorkflowExecutor = class {
|
|
3110
|
+
static {
|
|
3111
|
+
__name(this, "RemoteWorkflowExecutor");
|
|
3112
|
+
}
|
|
3113
|
+
transport;
|
|
3114
|
+
group;
|
|
3115
|
+
opts;
|
|
3116
|
+
pending = /* @__PURE__ */ new Map();
|
|
3117
|
+
subscribed = false;
|
|
3118
|
+
constructor(transport, group, opts = {}) {
|
|
3119
|
+
this.transport = transport;
|
|
3120
|
+
this.group = group;
|
|
3121
|
+
this.opts = opts;
|
|
3122
|
+
}
|
|
3123
|
+
ensureSubscribed() {
|
|
3124
|
+
if (this.subscribed) return;
|
|
3125
|
+
if (!this.transport.onDecision) {
|
|
3126
|
+
throw new Error("transport does not support workflow decisions (onDecision)");
|
|
3127
|
+
}
|
|
3128
|
+
this.subscribed = true;
|
|
3129
|
+
this.transport.onDecision(async (decision) => {
|
|
3130
|
+
const resolve = this.pending.get(decision.taskId);
|
|
3131
|
+
if (resolve) {
|
|
3132
|
+
this.pending.delete(decision.taskId);
|
|
3133
|
+
resolve(decision);
|
|
3134
|
+
}
|
|
3135
|
+
});
|
|
3136
|
+
}
|
|
3137
|
+
async advance(run, history) {
|
|
3138
|
+
if (!this.transport.dispatchWorkflowTask) {
|
|
3139
|
+
throw new Error("transport does not support workflow tasks (dispatchWorkflowTask)");
|
|
3140
|
+
}
|
|
3141
|
+
this.ensureSubscribed();
|
|
3142
|
+
taskCounter += 1;
|
|
3143
|
+
const taskId = `${run.id}:wf:${taskCounter}`;
|
|
3144
|
+
const task = {
|
|
3145
|
+
taskId,
|
|
3146
|
+
runId: run.id,
|
|
3147
|
+
workflow: run.workflow,
|
|
3148
|
+
workflowVersion: run.workflowVersion,
|
|
3149
|
+
input: run.input,
|
|
3150
|
+
history,
|
|
3151
|
+
group: this.group,
|
|
3152
|
+
attempt: 1
|
|
3153
|
+
};
|
|
3154
|
+
const decision = new Promise((resolve, reject) => {
|
|
3155
|
+
this.pending.set(taskId, resolve);
|
|
3156
|
+
if (this.opts.timeoutMs) {
|
|
3157
|
+
const timer = setTimeout(() => {
|
|
3158
|
+
this.pending.delete(taskId);
|
|
3159
|
+
reject(new Error(`workflow task ${taskId} timed out after ${this.opts.timeoutMs}ms`));
|
|
3160
|
+
}, this.opts.timeoutMs);
|
|
3161
|
+
timer.unref?.();
|
|
3162
|
+
}
|
|
3163
|
+
});
|
|
3164
|
+
await this.transport.dispatchWorkflowTask(task);
|
|
3165
|
+
return decision;
|
|
3166
|
+
}
|
|
3167
|
+
};
|
|
3168
|
+
|
|
3169
|
+
// src/codec-state-store.ts
|
|
3170
|
+
var CodecStateStore = class {
|
|
3171
|
+
static {
|
|
3172
|
+
__name(this, "CodecStateStore");
|
|
3173
|
+
}
|
|
3174
|
+
inner;
|
|
3175
|
+
codec;
|
|
3176
|
+
constructor(inner, codec) {
|
|
3177
|
+
this.inner = inner;
|
|
3178
|
+
this.codec = codec;
|
|
3179
|
+
}
|
|
3180
|
+
enc(v) {
|
|
3181
|
+
return v === void 0 ? void 0 : this.codec.encode(v);
|
|
3182
|
+
}
|
|
3183
|
+
dec(v) {
|
|
3184
|
+
return v === void 0 ? void 0 : this.codec.decode(v);
|
|
3185
|
+
}
|
|
3186
|
+
encRun(run) {
|
|
3187
|
+
return {
|
|
3188
|
+
...run,
|
|
3189
|
+
input: this.enc(run.input),
|
|
3190
|
+
output: this.enc(run.output)
|
|
3191
|
+
};
|
|
3192
|
+
}
|
|
3193
|
+
decRun(run) {
|
|
3194
|
+
return run && {
|
|
3195
|
+
...run,
|
|
3196
|
+
input: this.dec(run.input),
|
|
3197
|
+
output: this.dec(run.output)
|
|
3198
|
+
};
|
|
3199
|
+
}
|
|
3200
|
+
encCp(cp) {
|
|
3201
|
+
return {
|
|
3202
|
+
...cp,
|
|
3203
|
+
input: this.enc(cp.input),
|
|
3204
|
+
output: this.enc(cp.output)
|
|
3205
|
+
};
|
|
3206
|
+
}
|
|
3207
|
+
decCp(cp) {
|
|
3208
|
+
return cp && {
|
|
3209
|
+
...cp,
|
|
3210
|
+
input: this.dec(cp.input),
|
|
3211
|
+
output: this.dec(cp.output)
|
|
3212
|
+
};
|
|
3213
|
+
}
|
|
3214
|
+
ensureSchema() {
|
|
3215
|
+
return this.inner.ensureSchema?.() ?? Promise.resolve();
|
|
3216
|
+
}
|
|
3217
|
+
createRun(run) {
|
|
3218
|
+
return this.inner.createRun(this.encRun(run));
|
|
3219
|
+
}
|
|
3220
|
+
updateRun(runId, patch) {
|
|
3221
|
+
const next = {
|
|
3222
|
+
...patch
|
|
3223
|
+
};
|
|
3224
|
+
if ("input" in patch) next.input = this.enc(patch.input);
|
|
3225
|
+
if ("output" in patch) next.output = this.enc(patch.output);
|
|
3226
|
+
return this.inner.updateRun(runId, next);
|
|
3227
|
+
}
|
|
3228
|
+
async getRun(runId) {
|
|
3229
|
+
return this.decRun(await this.inner.getRun(runId));
|
|
3230
|
+
}
|
|
3231
|
+
async getCheckpoint(runId, seq) {
|
|
3232
|
+
return this.decCp(await this.inner.getCheckpoint(runId, seq));
|
|
3233
|
+
}
|
|
3234
|
+
saveCheckpoint(checkpoint) {
|
|
3235
|
+
return this.inner.saveCheckpoint(this.encCp(checkpoint));
|
|
3236
|
+
}
|
|
3237
|
+
transaction(work) {
|
|
3238
|
+
if (!this.inner.transaction) {
|
|
3239
|
+
throw new Error("the wrapped store does not support transactions");
|
|
3240
|
+
}
|
|
3241
|
+
return this.inner.transaction((tx) => work({
|
|
3242
|
+
raw: tx.raw,
|
|
3243
|
+
saveCheckpoint: /* @__PURE__ */ __name((cp) => tx.saveCheckpoint(this.encCp(cp)), "saveCheckpoint")
|
|
3244
|
+
}));
|
|
3245
|
+
}
|
|
3246
|
+
async listIncompleteRuns() {
|
|
3247
|
+
return (await this.inner.listIncompleteRuns()).map((r) => this.decRun(r));
|
|
3248
|
+
}
|
|
3249
|
+
async listPendingRuns(limit) {
|
|
3250
|
+
return (await this.inner.listPendingRuns(limit)).map((r) => this.decRun(r));
|
|
3251
|
+
}
|
|
3252
|
+
async listDueTimers(nowMs) {
|
|
3253
|
+
return (await this.inner.listDueTimers(nowMs)).map((r) => this.decRun(r));
|
|
3254
|
+
}
|
|
3255
|
+
tryLockRun(runId, owner, leaseUntilMs, nowMs) {
|
|
3256
|
+
return this.inner.tryLockRun(runId, owner, leaseUntilMs, nowMs);
|
|
3257
|
+
}
|
|
3258
|
+
releaseRunLock(runId) {
|
|
3259
|
+
return this.inner.releaseRunLock(runId);
|
|
3260
|
+
}
|
|
3261
|
+
renewRunLock(runId, owner, leaseUntilMs) {
|
|
3262
|
+
return this.inner.renewRunLock(runId, owner, leaseUntilMs);
|
|
3263
|
+
}
|
|
3264
|
+
putSignalWaiter(waiter) {
|
|
3265
|
+
return this.inner.putSignalWaiter(waiter);
|
|
3266
|
+
}
|
|
3267
|
+
takeSignalWaiter(token) {
|
|
3268
|
+
return this.inner.takeSignalWaiter(token);
|
|
3269
|
+
}
|
|
3270
|
+
listSignalWaiters(prefix) {
|
|
3271
|
+
return this.inner.listSignalWaiters(prefix);
|
|
3272
|
+
}
|
|
3273
|
+
bufferSignal(token, payload) {
|
|
3274
|
+
return this.inner.bufferSignal(token, this.enc(payload));
|
|
3275
|
+
}
|
|
3276
|
+
async takeBufferedSignal(token) {
|
|
3277
|
+
const buffered = await this.inner.takeBufferedSignal(token);
|
|
3278
|
+
return buffered && {
|
|
3279
|
+
payload: this.dec(buffered.payload)
|
|
3280
|
+
};
|
|
3281
|
+
}
|
|
3282
|
+
async listRuns(query) {
|
|
3283
|
+
return (await this.inner.listRuns(query)).map((r) => this.decRun(r));
|
|
3284
|
+
}
|
|
3285
|
+
async listCheckpoints(runId) {
|
|
3286
|
+
return (await this.inner.listCheckpoints(runId)).map((c) => this.decCp(c));
|
|
3287
|
+
}
|
|
3288
|
+
};
|
|
3289
|
+
|
|
3290
|
+
// src/metrics.ts
|
|
3291
|
+
function collectMetrics(engine) {
|
|
3292
|
+
const m = {
|
|
3293
|
+
runs: {
|
|
3294
|
+
started: 0,
|
|
3295
|
+
completed: 0,
|
|
3296
|
+
failed: 0,
|
|
3297
|
+
suspended: 0
|
|
3298
|
+
},
|
|
3299
|
+
steps: {
|
|
3300
|
+
started: 0,
|
|
3301
|
+
completed: 0,
|
|
3302
|
+
failed: 0
|
|
3303
|
+
},
|
|
3304
|
+
byWorkflow: {},
|
|
3305
|
+
stepDuration: {
|
|
3306
|
+
sumMs: 0,
|
|
3307
|
+
count: 0
|
|
3308
|
+
}
|
|
3309
|
+
};
|
|
3310
|
+
const wf = /* @__PURE__ */ __name((name) => {
|
|
3311
|
+
const k = name ?? "unknown";
|
|
3312
|
+
return m.byWorkflow[k] ??= {
|
|
3313
|
+
started: 0,
|
|
3314
|
+
completed: 0,
|
|
3315
|
+
failed: 0
|
|
3316
|
+
};
|
|
3317
|
+
}, "wf");
|
|
3318
|
+
const stop = engine.subscribe((e) => {
|
|
3319
|
+
switch (e.type) {
|
|
3320
|
+
case "run.started":
|
|
3321
|
+
m.runs.started += 1;
|
|
3322
|
+
wf(e.workflow).started += 1;
|
|
3323
|
+
break;
|
|
3324
|
+
case "run.completed":
|
|
3325
|
+
m.runs.completed += 1;
|
|
3326
|
+
wf(e.workflow).completed += 1;
|
|
3327
|
+
break;
|
|
3328
|
+
case "run.failed":
|
|
3329
|
+
m.runs.failed += 1;
|
|
3330
|
+
wf(e.workflow).failed += 1;
|
|
3331
|
+
break;
|
|
3332
|
+
case "run.suspended":
|
|
3333
|
+
m.runs.suspended += 1;
|
|
3334
|
+
break;
|
|
3335
|
+
case "step.started":
|
|
3336
|
+
m.steps.started += 1;
|
|
3337
|
+
break;
|
|
3338
|
+
case "step.completed":
|
|
3339
|
+
m.steps.completed += 1;
|
|
3340
|
+
if (e.durationMs != null) {
|
|
3341
|
+
m.stepDuration.sumMs += e.durationMs;
|
|
3342
|
+
m.stepDuration.count += 1;
|
|
3343
|
+
}
|
|
3344
|
+
break;
|
|
3345
|
+
case "step.failed":
|
|
3346
|
+
m.steps.failed += 1;
|
|
3347
|
+
break;
|
|
3348
|
+
}
|
|
3349
|
+
});
|
|
3350
|
+
return {
|
|
3351
|
+
snapshot: /* @__PURE__ */ __name(() => structuredClone(m), "snapshot"),
|
|
3352
|
+
stop,
|
|
3353
|
+
prometheus: /* @__PURE__ */ __name(() => {
|
|
3354
|
+
const lines = [];
|
|
3355
|
+
lines.push("# TYPE durable_runs_total counter");
|
|
3356
|
+
for (const [k, v] of Object.entries(m.runs)) lines.push(`durable_runs_total{event="${k}"} ${v}`);
|
|
3357
|
+
lines.push("# TYPE durable_steps_total counter");
|
|
3358
|
+
for (const [k, v] of Object.entries(m.steps)) lines.push(`durable_steps_total{event="${k}"} ${v}`);
|
|
3359
|
+
lines.push("# TYPE durable_runs_by_workflow_total counter");
|
|
3360
|
+
for (const [name, v] of Object.entries(m.byWorkflow)) for (const [event, n] of Object.entries(v)) lines.push(`durable_runs_by_workflow_total{workflow="${name}",event="${event}"} ${n}`);
|
|
3361
|
+
lines.push("# TYPE durable_step_duration_ms_sum counter");
|
|
3362
|
+
lines.push(`durable_step_duration_ms_sum ${m.stepDuration.sumMs}`);
|
|
3363
|
+
lines.push("# TYPE durable_step_duration_count counter");
|
|
3364
|
+
lines.push(`durable_step_duration_count ${m.stepDuration.count}`);
|
|
3365
|
+
return `${lines.join("\n")}
|
|
3366
|
+
`;
|
|
3367
|
+
}, "prometheus")
|
|
3368
|
+
};
|
|
3369
|
+
}
|
|
3370
|
+
__name(collectMetrics, "collectMetrics");
|
|
3371
|
+
|
|
3372
|
+
// src/scheduler.ts
|
|
3373
|
+
import { createRequire } from "node:module";
|
|
3374
|
+
var nodeRequire = createRequire(import.meta.url);
|
|
3375
|
+
var defaultSleep = /* @__PURE__ */ __name((ms) => new Promise((r) => setTimeout(r, ms)), "defaultSleep");
|
|
3376
|
+
function scheduledRunId(key, everyMs, nowMs) {
|
|
3377
|
+
return `sched:${key}:${Math.floor(nowMs / everyMs)}`;
|
|
3378
|
+
}
|
|
3379
|
+
__name(scheduledRunId, "scheduledRunId");
|
|
3380
|
+
var cronParser;
|
|
3381
|
+
function loadCronParser() {
|
|
3382
|
+
if (cronParser) return cronParser;
|
|
3383
|
+
try {
|
|
3384
|
+
cronParser = nodeRequire("cron-parser");
|
|
3385
|
+
} catch {
|
|
3386
|
+
throw new Error('cron schedules need the optional peer dependency "cron-parser" \u2014 install it (e.g. `npm i cron-parser`).');
|
|
3387
|
+
}
|
|
3388
|
+
return cronParser;
|
|
3389
|
+
}
|
|
3390
|
+
__name(loadCronParser, "loadCronParser");
|
|
3391
|
+
function prevCronFireMs(expr, nowMs, timezone = "UTC") {
|
|
3392
|
+
const parser = loadCronParser();
|
|
3393
|
+
const it = parser.parseExpression(expr, {
|
|
3394
|
+
currentDate: new Date(nowMs + 1),
|
|
3395
|
+
tz: timezone
|
|
3396
|
+
});
|
|
3397
|
+
return it.prev().toDate().getTime();
|
|
3398
|
+
}
|
|
3399
|
+
__name(prevCronFireMs, "prevCronFireMs");
|
|
3400
|
+
function scheduleRunIdAt(s, nowMs) {
|
|
3401
|
+
if (s.cron != null) return `sched:${s.key}:${prevCronFireMs(s.cron, nowMs, s.timezone)}`;
|
|
3402
|
+
if (s.everyMs != null) return scheduledRunId(s.key, s.everyMs, nowMs);
|
|
3403
|
+
throw new Error(`schedule "${s.key}" needs either "everyMs" or "cron"`);
|
|
3404
|
+
}
|
|
3405
|
+
__name(scheduleRunIdAt, "scheduleRunIdAt");
|
|
3406
|
+
function scheduleRunIdsAt(s, nowMs) {
|
|
3407
|
+
const current = scheduleRunIdAt(s, nowMs);
|
|
3408
|
+
if (!s.backfill || s.backfill.maxCatchup <= 0) return [
|
|
3409
|
+
current
|
|
3410
|
+
];
|
|
3411
|
+
const ids = [];
|
|
3412
|
+
if (s.everyMs != null) {
|
|
3413
|
+
const bucket = Math.floor(nowMs / s.everyMs);
|
|
3414
|
+
const oldest = Math.max(0, bucket - s.backfill.maxCatchup);
|
|
3415
|
+
for (let b = oldest; b < bucket; b += 1) ids.push(`sched:${s.key}:${b}`);
|
|
3416
|
+
} else if (s.cron != null) {
|
|
3417
|
+
const prior = [];
|
|
3418
|
+
let cursor = prevCronFireMs(s.cron, nowMs, s.timezone);
|
|
3419
|
+
for (let i = 0; i < s.backfill.maxCatchup; i += 1) {
|
|
3420
|
+
const before = prevCronFireMs(s.cron, cursor - 1, s.timezone);
|
|
3421
|
+
prior.push(`sched:${s.key}:${before}`);
|
|
3422
|
+
cursor = before;
|
|
3423
|
+
}
|
|
3424
|
+
ids.push(...prior.reverse());
|
|
3425
|
+
}
|
|
3426
|
+
ids.push(current);
|
|
3427
|
+
return ids;
|
|
3428
|
+
}
|
|
3429
|
+
__name(scheduleRunIdsAt, "scheduleRunIdsAt");
|
|
3430
|
+
async function runSchedules(engine, schedules, nowMs, opts) {
|
|
3431
|
+
const random = opts?.random ?? Math.random;
|
|
3432
|
+
const sleep = opts?.sleep ?? defaultSleep;
|
|
3433
|
+
const ids = [];
|
|
3434
|
+
for (const s of schedules) {
|
|
3435
|
+
if (s.paused) continue;
|
|
3436
|
+
if (s.overlap === "skip" && s.everyMs) {
|
|
3437
|
+
const prevBucket = Math.floor(nowMs / s.everyMs) - 1;
|
|
3438
|
+
if (prevBucket >= 0) {
|
|
3439
|
+
const prev = await engine.getRun(`sched:${s.key}:${prevBucket}`);
|
|
3440
|
+
if (prev && (prev.status === "running" || prev.status === "suspended")) continue;
|
|
3441
|
+
}
|
|
3442
|
+
}
|
|
3443
|
+
if (s.jitter && s.jitter > 0) await sleep(Math.floor(random() * s.jitter));
|
|
3444
|
+
for (const runId of scheduleRunIdsAt(s, nowMs)) {
|
|
3445
|
+
await engine.start(s.workflow, s.input, runId);
|
|
3446
|
+
ids.push(runId);
|
|
3447
|
+
}
|
|
3448
|
+
}
|
|
3449
|
+
return ids;
|
|
3450
|
+
}
|
|
3451
|
+
__name(runSchedules, "runSchedules");
|
|
3452
|
+
|
|
3453
|
+
// src/search-attributes.ts
|
|
3454
|
+
function compare(actual, op, expected) {
|
|
3455
|
+
switch (op) {
|
|
3456
|
+
case "eq":
|
|
3457
|
+
return actual === expected;
|
|
3458
|
+
case "ne":
|
|
3459
|
+
return actual !== expected;
|
|
3460
|
+
case "gt":
|
|
3461
|
+
return actual != null && actual > expected;
|
|
3462
|
+
case "gte":
|
|
3463
|
+
return actual != null && actual >= expected;
|
|
3464
|
+
case "lt":
|
|
3465
|
+
return actual != null && actual < expected;
|
|
3466
|
+
case "lte":
|
|
3467
|
+
return actual != null && actual <= expected;
|
|
3468
|
+
}
|
|
3469
|
+
}
|
|
3470
|
+
__name(compare, "compare");
|
|
3471
|
+
function matchesAttributes(attributes, filters) {
|
|
3472
|
+
if (!filters?.length) return true;
|
|
3473
|
+
if (!attributes) return false;
|
|
3474
|
+
return filters.every((f) => f.key in attributes && compare(attributes[f.key], f.op, f.value));
|
|
3475
|
+
}
|
|
3476
|
+
__name(matchesAttributes, "matchesAttributes");
|
|
3477
|
+
function applyAttributeQuery(rows, query) {
|
|
3478
|
+
const filtered = rows.filter((r) => matchesAttributes(r.searchAttributes, query.attributes));
|
|
3479
|
+
const offset = query.offset ?? 0;
|
|
3480
|
+
const limit = query.limit ?? filtered.length;
|
|
3481
|
+
return filtered.slice(offset, offset + limit);
|
|
3482
|
+
}
|
|
3483
|
+
__name(applyAttributeQuery, "applyAttributeQuery");
|
|
3484
|
+
function normalizeAttributeRows(runId, attributes) {
|
|
3485
|
+
if (!attributes) return [];
|
|
3486
|
+
const out = [];
|
|
3487
|
+
for (const [key, value] of Object.entries(attributes)) {
|
|
3488
|
+
if (typeof value === "number") {
|
|
3489
|
+
out.push({
|
|
3490
|
+
runId,
|
|
3491
|
+
key,
|
|
3492
|
+
strValue: null,
|
|
3493
|
+
numValue: value
|
|
3494
|
+
});
|
|
3495
|
+
} else if (typeof value === "boolean") {
|
|
3496
|
+
out.push({
|
|
3497
|
+
runId,
|
|
3498
|
+
key,
|
|
3499
|
+
strValue: value ? "true" : "false",
|
|
3500
|
+
numValue: null
|
|
3501
|
+
});
|
|
3502
|
+
} else {
|
|
3503
|
+
out.push({
|
|
3504
|
+
runId,
|
|
3505
|
+
key,
|
|
3506
|
+
strValue: value,
|
|
3507
|
+
numValue: null
|
|
3508
|
+
});
|
|
3509
|
+
}
|
|
3510
|
+
}
|
|
3511
|
+
return out;
|
|
3512
|
+
}
|
|
3513
|
+
__name(normalizeAttributeRows, "normalizeAttributeRows");
|
|
3514
|
+
function sqlComparator(op) {
|
|
3515
|
+
switch (op) {
|
|
3516
|
+
case "eq":
|
|
3517
|
+
return "=";
|
|
3518
|
+
case "ne":
|
|
3519
|
+
return "<>";
|
|
3520
|
+
case "gt":
|
|
3521
|
+
return ">";
|
|
3522
|
+
case "gte":
|
|
3523
|
+
return ">=";
|
|
3524
|
+
case "lt":
|
|
3525
|
+
return "<";
|
|
3526
|
+
case "lte":
|
|
3527
|
+
return "<=";
|
|
3528
|
+
}
|
|
3529
|
+
}
|
|
3530
|
+
__name(sqlComparator, "sqlComparator");
|
|
3531
|
+
function attributeColumnFor(filter) {
|
|
3532
|
+
return typeof filter.value === "number" ? "numValue" : "strValue";
|
|
3533
|
+
}
|
|
3534
|
+
__name(attributeColumnFor, "attributeColumnFor");
|
|
3535
|
+
function attributeOperand(filter) {
|
|
3536
|
+
if (typeof filter.value === "boolean") return filter.value ? "true" : "false";
|
|
3537
|
+
return filter.value;
|
|
3538
|
+
}
|
|
3539
|
+
__name(attributeOperand, "attributeOperand");
|
|
3540
|
+
|
|
3541
|
+
// src/tokens.ts
|
|
3542
|
+
var STATE_STORE = Symbol("nestjs-durable:STATE_STORE");
|
|
3543
|
+
var TRANSPORT = Symbol("nestjs-durable:TRANSPORT");
|
|
3544
|
+
var DURABLE_OPTIONS = Symbol("nestjs-durable:DURABLE_OPTIONS");
|
|
3545
|
+
|
|
3546
|
+
// src/testing/in-memory-state-store.ts
|
|
3547
|
+
var InMemoryStateStore = class {
|
|
3548
|
+
static {
|
|
3549
|
+
__name(this, "InMemoryStateStore");
|
|
3550
|
+
}
|
|
3551
|
+
runs = /* @__PURE__ */ new Map();
|
|
3552
|
+
checkpoints = /* @__PURE__ */ new Map();
|
|
3553
|
+
signalWaiters = /* @__PURE__ */ new Map();
|
|
3554
|
+
/**
|
|
3555
|
+
* Normalized search-attribute side-table, the in-memory analog of the SQL stores'
|
|
3556
|
+
* `durable_run_attributes`: `key -> (runId -> row)`. Maintained on every create/update so an
|
|
3557
|
+
* attribute query can be pushed DOWN to a key-indexed candidate set instead of scanning every run
|
|
3558
|
+
* and comparing in-process. Mirrors {@link normalizeAttributeRows} so behavior matches the SQL path.
|
|
3559
|
+
*/
|
|
3560
|
+
attributeIndex = /* @__PURE__ */ new Map();
|
|
3561
|
+
/** Test/observability hook: how many candidate runs the last attribute query's index produced. */
|
|
3562
|
+
lastAttributeCandidates = 0;
|
|
3563
|
+
key(runId, seq) {
|
|
3564
|
+
return `${runId}:${seq}`;
|
|
3565
|
+
}
|
|
3566
|
+
/** Drop then re-add a run's normalized attribute rows so the index tracks its current attributes. */
|
|
3567
|
+
reindexAttributes(runId, attributes) {
|
|
3568
|
+
for (const byRun of this.attributeIndex.values()) byRun.delete(runId);
|
|
3569
|
+
for (const row of normalizeAttributeRows(runId, attributes)) {
|
|
3570
|
+
let byRun = this.attributeIndex.get(row.key);
|
|
3571
|
+
if (!byRun) {
|
|
3572
|
+
byRun = /* @__PURE__ */ new Map();
|
|
3573
|
+
this.attributeIndex.set(row.key, byRun);
|
|
3574
|
+
}
|
|
3575
|
+
byRun.set(runId, row);
|
|
3576
|
+
}
|
|
3577
|
+
}
|
|
3578
|
+
async createRun(run) {
|
|
3579
|
+
this.runs.set(run.id, {
|
|
3580
|
+
...run
|
|
3581
|
+
});
|
|
3582
|
+
this.reindexAttributes(run.id, run.searchAttributes);
|
|
3583
|
+
}
|
|
3584
|
+
async updateRun(runId, patch) {
|
|
3585
|
+
const existing = this.runs.get(runId);
|
|
3586
|
+
if (!existing) throw new Error(`run ${runId} not found`);
|
|
3587
|
+
const next = {
|
|
3588
|
+
...existing,
|
|
3589
|
+
...patch
|
|
3590
|
+
};
|
|
3591
|
+
this.runs.set(runId, next);
|
|
3592
|
+
if ("searchAttributes" in patch) this.reindexAttributes(runId, next.searchAttributes);
|
|
3593
|
+
}
|
|
3594
|
+
async getRun(runId) {
|
|
3595
|
+
const run = this.runs.get(runId);
|
|
3596
|
+
return run ? {
|
|
3597
|
+
...run
|
|
3598
|
+
} : null;
|
|
3599
|
+
}
|
|
3600
|
+
async getCheckpoint(runId, seq) {
|
|
3601
|
+
const cp = this.checkpoints.get(this.key(runId, seq));
|
|
3602
|
+
return cp ? {
|
|
3603
|
+
...cp
|
|
3604
|
+
} : null;
|
|
3605
|
+
}
|
|
3606
|
+
async saveCheckpoint(checkpoint) {
|
|
3607
|
+
this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
|
|
3608
|
+
...checkpoint
|
|
3609
|
+
});
|
|
3610
|
+
}
|
|
3611
|
+
async listIncompleteRuns() {
|
|
3612
|
+
return [
|
|
3613
|
+
...this.runs.values()
|
|
3614
|
+
].filter((r) => r.status === "running").map((r) => ({
|
|
3615
|
+
...r
|
|
3616
|
+
}));
|
|
3617
|
+
}
|
|
3618
|
+
async listPendingRuns(limit) {
|
|
3619
|
+
return [
|
|
3620
|
+
...this.runs.values()
|
|
3621
|
+
].filter((r) => r.status === "pending").sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id)).slice(0, limit).map((r) => ({
|
|
3622
|
+
...r
|
|
3623
|
+
}));
|
|
3624
|
+
}
|
|
3625
|
+
async listDueTimers(nowMs) {
|
|
3626
|
+
return [
|
|
3627
|
+
...this.runs.values()
|
|
3628
|
+
].filter((r) => r.status === "suspended" && r.wakeAt !== void 0 && r.wakeAt <= nowMs).map((r) => ({
|
|
3629
|
+
...r
|
|
3630
|
+
}));
|
|
3631
|
+
}
|
|
3632
|
+
async tryLockRun(runId, owner, leaseUntilMs, nowMs) {
|
|
3633
|
+
const run = this.runs.get(runId);
|
|
3634
|
+
if (!run) return false;
|
|
3635
|
+
if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) return false;
|
|
3636
|
+
run.lockedBy = owner;
|
|
3637
|
+
run.lockedUntil = leaseUntilMs;
|
|
3638
|
+
return true;
|
|
3639
|
+
}
|
|
3640
|
+
async releaseRunLock(runId) {
|
|
3641
|
+
const run = this.runs.get(runId);
|
|
3642
|
+
if (run) {
|
|
3643
|
+
run.lockedBy = void 0;
|
|
3644
|
+
run.lockedUntil = void 0;
|
|
3645
|
+
}
|
|
3646
|
+
}
|
|
3647
|
+
async renewRunLock(runId, owner, leaseUntilMs) {
|
|
3648
|
+
const run = this.runs.get(runId);
|
|
3649
|
+
if (!run || run.lockedBy !== owner) return false;
|
|
3650
|
+
run.lockedUntil = leaseUntilMs;
|
|
3651
|
+
return true;
|
|
3652
|
+
}
|
|
3653
|
+
async putSignalWaiter(waiter) {
|
|
3654
|
+
this.signalWaiters.set(waiter.token, {
|
|
3655
|
+
...waiter
|
|
3656
|
+
});
|
|
3657
|
+
}
|
|
3658
|
+
async takeSignalWaiter(token) {
|
|
3659
|
+
const waiter = this.signalWaiters.get(token);
|
|
3660
|
+
if (!waiter) return null;
|
|
3661
|
+
this.signalWaiters.delete(token);
|
|
3662
|
+
return {
|
|
3663
|
+
...waiter
|
|
3664
|
+
};
|
|
3665
|
+
}
|
|
3666
|
+
async listSignalWaiters(prefix) {
|
|
3667
|
+
return [
|
|
3668
|
+
...this.signalWaiters.values()
|
|
3669
|
+
].filter((w) => w.token.startsWith(prefix)).map((w) => ({
|
|
3670
|
+
...w
|
|
3671
|
+
}));
|
|
3672
|
+
}
|
|
3673
|
+
// No real DB transaction (there's no DB) — run the work and save the checkpoint. Exactly-once
|
|
3674
|
+
// across a crash needs a SQL store; this keeps `ctx.transaction` usable in tests / local dev.
|
|
3675
|
+
async transaction(work) {
|
|
3676
|
+
return work({
|
|
3677
|
+
raw: this,
|
|
3678
|
+
saveCheckpoint: /* @__PURE__ */ __name((cp) => this.saveCheckpoint(cp), "saveCheckpoint")
|
|
3679
|
+
});
|
|
3680
|
+
}
|
|
3681
|
+
bufferedSignals = /* @__PURE__ */ new Map();
|
|
3682
|
+
async bufferSignal(token, payload) {
|
|
3683
|
+
const queue = this.bufferedSignals.get(token) ?? [];
|
|
3684
|
+
queue.push(payload);
|
|
3685
|
+
this.bufferedSignals.set(token, queue);
|
|
3686
|
+
}
|
|
3687
|
+
async takeBufferedSignal(token) {
|
|
3688
|
+
const queue = this.bufferedSignals.get(token);
|
|
3689
|
+
if (!queue || queue.length === 0) return null;
|
|
3690
|
+
const payload = queue.shift();
|
|
3691
|
+
if (queue.length === 0) this.bufferedSignals.delete(token);
|
|
3692
|
+
return {
|
|
3693
|
+
payload
|
|
3694
|
+
};
|
|
3695
|
+
}
|
|
3696
|
+
/**
|
|
3697
|
+
* Candidate run ids satisfying ONE attribute predicate, served from {@link attributeIndex} — the
|
|
3698
|
+
* pushdown step. Numeric predicates scan only `numValue` rows under the key; string/boolean ones
|
|
3699
|
+
* scan `strValue` rows. This is the in-memory analog of the SQL EXISTS on `durable_run_attributes`.
|
|
3700
|
+
*/
|
|
3701
|
+
candidatesForFilter(f) {
|
|
3702
|
+
const byRun = this.attributeIndex.get(f.key);
|
|
3703
|
+
const out = /* @__PURE__ */ new Set();
|
|
3704
|
+
if (!byRun) return out;
|
|
3705
|
+
const numeric = typeof f.value === "number";
|
|
3706
|
+
const operand = typeof f.value === "boolean" ? f.value ? "true" : "false" : f.value;
|
|
3707
|
+
for (const row of byRun.values()) {
|
|
3708
|
+
const actual = numeric ? row.numValue : row.strValue;
|
|
3709
|
+
if (actual == null) continue;
|
|
3710
|
+
let ok = false;
|
|
3711
|
+
switch (f.op) {
|
|
3712
|
+
case "eq":
|
|
3713
|
+
ok = actual === operand;
|
|
3714
|
+
break;
|
|
3715
|
+
case "ne":
|
|
3716
|
+
ok = actual !== operand;
|
|
3717
|
+
break;
|
|
3718
|
+
case "gt":
|
|
3719
|
+
ok = actual > operand;
|
|
3720
|
+
break;
|
|
3721
|
+
case "gte":
|
|
3722
|
+
ok = actual >= operand;
|
|
3723
|
+
break;
|
|
3724
|
+
case "lt":
|
|
3725
|
+
ok = actual < operand;
|
|
3726
|
+
break;
|
|
3727
|
+
case "lte":
|
|
3728
|
+
ok = actual <= operand;
|
|
3729
|
+
break;
|
|
3730
|
+
}
|
|
3731
|
+
if (ok) out.add(row.runId);
|
|
3732
|
+
}
|
|
3733
|
+
return out;
|
|
3734
|
+
}
|
|
3735
|
+
async listRuns(query) {
|
|
3736
|
+
let runs = [
|
|
3737
|
+
...this.runs.values()
|
|
3738
|
+
];
|
|
3739
|
+
if (query.workflow) runs = runs.filter((r) => r.workflow === query.workflow);
|
|
3740
|
+
if (query.status) runs = runs.filter((r) => r.status === query.status);
|
|
3741
|
+
if (query.statuses) runs = runs.filter((r) => query.statuses?.includes(r.status));
|
|
3742
|
+
if (query.tag) runs = runs.filter((r) => r.tags?.includes(query.tag));
|
|
3743
|
+
if (query.attributes?.length) {
|
|
3744
|
+
let candidates = null;
|
|
3745
|
+
for (const f of query.attributes) {
|
|
3746
|
+
const next = this.candidatesForFilter(f);
|
|
3747
|
+
if (candidates == null) {
|
|
3748
|
+
candidates = next;
|
|
3749
|
+
} else {
|
|
3750
|
+
const prev = candidates;
|
|
3751
|
+
candidates = new Set([
|
|
3752
|
+
...prev
|
|
3753
|
+
].filter((id) => next.has(id)));
|
|
3754
|
+
}
|
|
3755
|
+
if (candidates.size === 0) break;
|
|
3756
|
+
}
|
|
3757
|
+
const ids = candidates ?? /* @__PURE__ */ new Set();
|
|
3758
|
+
this.lastAttributeCandidates = ids.size;
|
|
3759
|
+
runs = runs.filter((r) => ids.has(r.id));
|
|
3760
|
+
}
|
|
3761
|
+
runs.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
|
|
3762
|
+
const offset = query.offset ?? 0;
|
|
3763
|
+
const limit = query.limit ?? runs.length;
|
|
3764
|
+
return runs.slice(offset, offset + limit).map((r) => ({
|
|
3765
|
+
...r
|
|
3766
|
+
}));
|
|
3767
|
+
}
|
|
3768
|
+
async listCheckpoints(runId) {
|
|
3769
|
+
return [
|
|
3770
|
+
...this.checkpoints.values()
|
|
3771
|
+
].filter((cp) => cp.runId === runId).sort((a, b) => a.seq - b.seq).map((cp) => ({
|
|
3772
|
+
...cp
|
|
3773
|
+
}));
|
|
3774
|
+
}
|
|
3775
|
+
};
|
|
3776
|
+
|
|
3777
|
+
// src/testing/in-memory-transport.ts
|
|
3778
|
+
var InMemoryTransport = class {
|
|
3779
|
+
static {
|
|
3780
|
+
__name(this, "InMemoryTransport");
|
|
3781
|
+
}
|
|
3782
|
+
handlers = /* @__PURE__ */ new Map();
|
|
3783
|
+
resultHandler;
|
|
3784
|
+
stepEventHandler;
|
|
3785
|
+
controlHandlers = /* @__PURE__ */ new Set();
|
|
3786
|
+
/** Register a fake worker handler for a step name. */
|
|
3787
|
+
handle(name, fn) {
|
|
3788
|
+
this.handlers.set(name, fn);
|
|
3789
|
+
}
|
|
3790
|
+
async dispatch(task) {
|
|
3791
|
+
if (!this.resultHandler) throw new Error("no result handler registered");
|
|
3792
|
+
const result = await runStepHandler(task, this.handlers.get(task.name));
|
|
3793
|
+
setImmediate(() => void this.resultHandler?.(result));
|
|
3794
|
+
}
|
|
3795
|
+
onResult(handler) {
|
|
3796
|
+
this.resultHandler = handler;
|
|
3797
|
+
}
|
|
3798
|
+
onHeartbeat(_handler) {
|
|
3799
|
+
}
|
|
3800
|
+
/** Workflow step lifecycle: deliver to the engine's handler (async, mirroring a real broker). */
|
|
3801
|
+
async dispatchStepEvent(event) {
|
|
3802
|
+
await this.stepEventHandler?.(event);
|
|
3803
|
+
}
|
|
3804
|
+
onStepEvent(handler) {
|
|
3805
|
+
this.stepEventHandler = handler;
|
|
3806
|
+
}
|
|
3807
|
+
// Control plane: broadcast to every registered handler (including the publisher's own — the
|
|
3808
|
+
// engine dedupes by `from`), mirroring how a real broker echoes a publish to all subscribers.
|
|
3809
|
+
async publishControl(msg) {
|
|
3810
|
+
for (const handler of this.controlHandlers) handler(msg);
|
|
3811
|
+
}
|
|
3812
|
+
onControl(handler) {
|
|
3813
|
+
this.controlHandlers.add(handler);
|
|
3814
|
+
}
|
|
3815
|
+
};
|
|
3816
|
+
export {
|
|
3817
|
+
CodecStateStore,
|
|
3818
|
+
ContinueAsNew,
|
|
3819
|
+
DURABLE_OPTIONS,
|
|
3820
|
+
Entities,
|
|
3821
|
+
EventAccumulators,
|
|
3822
|
+
FatalError,
|
|
3823
|
+
InMemoryStateStore,
|
|
3824
|
+
InMemoryTransport,
|
|
3825
|
+
NonDeterminismError,
|
|
3826
|
+
QueueController,
|
|
3827
|
+
RemoteStepError,
|
|
3828
|
+
RemoteStepTimeout,
|
|
3829
|
+
RemoteWorkflowExecutor,
|
|
3830
|
+
STATE_STORE,
|
|
3831
|
+
SignalTimeoutError,
|
|
3832
|
+
SingletonQueueFullError,
|
|
3833
|
+
TRANSPORT,
|
|
3834
|
+
WORKFLOW_NAME_KEY,
|
|
3835
|
+
WorkflowEngine,
|
|
3836
|
+
WorkflowSuspended,
|
|
3837
|
+
applyAttributeQuery,
|
|
3838
|
+
attributeColumnFor,
|
|
3839
|
+
attributeOperand,
|
|
3840
|
+
breakpointToken,
|
|
3841
|
+
collectMetrics,
|
|
3842
|
+
eventMatchOf,
|
|
3843
|
+
eventMatches,
|
|
3844
|
+
eventPrefix,
|
|
3845
|
+
eventToken,
|
|
3846
|
+
matchesAttributes,
|
|
3847
|
+
normalizeAttributeRows,
|
|
3848
|
+
parseDuration,
|
|
3849
|
+
prevCronFireMs,
|
|
3850
|
+
remoteStep,
|
|
3851
|
+
runSchedules,
|
|
3852
|
+
runStepHandler,
|
|
3853
|
+
scheduledRunId,
|
|
3854
|
+
sqlComparator,
|
|
3855
|
+
stepId,
|
|
3856
|
+
workflowName
|
|
3857
|
+
};
|
|
39
3858
|
//# sourceMappingURL=index.js.map
|