@dudousxd/nestjs-durable-core 0.60.0 → 0.62.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +223 -66
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +86 -4
- package/dist/index.d.ts +86 -4
- package/dist/index.js +213 -66
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -71,8 +71,11 @@ __export(index_exports, {
|
|
|
71
71
|
collectMetrics: () => collectMetrics,
|
|
72
72
|
controlPlaneDescriptor: () => controlPlaneDescriptor,
|
|
73
73
|
createStepLogger: () => createStepLogger,
|
|
74
|
+
currentStep: () => currentStep,
|
|
74
75
|
currentWorkflowCtx: () => currentWorkflowCtx,
|
|
76
|
+
debug: () => debug,
|
|
75
77
|
descriptorHash: () => descriptorHash,
|
|
78
|
+
error: () => error,
|
|
76
79
|
eventMatchOf: () => eventMatchOf,
|
|
77
80
|
eventMatches: () => eventMatches,
|
|
78
81
|
eventNameOf: () => eventNameOf,
|
|
@@ -80,9 +83,11 @@ __export(index_exports, {
|
|
|
80
83
|
eventToken: () => eventToken,
|
|
81
84
|
heartbeatStatus: () => heartbeatStatus,
|
|
82
85
|
indexWaitersByRun: () => indexWaitersByRun,
|
|
86
|
+
info: () => info,
|
|
83
87
|
isEventToken: () => isEventToken,
|
|
84
88
|
isLegacyDescriptor: () => isLegacyDescriptor,
|
|
85
89
|
isWorkflowControlFlowSignal: () => isWorkflowControlFlowSignal,
|
|
90
|
+
log: () => log,
|
|
86
91
|
matchesAttributes: () => matchesAttributes,
|
|
87
92
|
missingCapabilities: () => missingCapabilities,
|
|
88
93
|
negotiate: () => negotiate,
|
|
@@ -95,6 +100,7 @@ __export(index_exports, {
|
|
|
95
100
|
requiredCapabilities: () => requiredCapabilities,
|
|
96
101
|
resolveRouting: () => resolveRouting,
|
|
97
102
|
resolveRunWaiting: () => resolveRunWaiting,
|
|
103
|
+
runInStepLogger: () => runInStepLogger,
|
|
98
104
|
runInWorkflowCtx: () => runInWorkflowCtx,
|
|
99
105
|
runSchedules: () => runSchedules,
|
|
100
106
|
runStepHandler: () => runStepHandler,
|
|
@@ -104,7 +110,11 @@ __export(index_exports, {
|
|
|
104
110
|
stepConfigOf: () => stepConfigOf,
|
|
105
111
|
stepId: () => stepId,
|
|
106
112
|
stepNameOf: () => stepNameOf,
|
|
113
|
+
sub: () => sub,
|
|
114
|
+
subEvent: () => subEvent,
|
|
115
|
+
subProcess: () => subProcess,
|
|
107
116
|
tenantGroup: () => tenantGroup,
|
|
117
|
+
warn: () => warn,
|
|
108
118
|
workflowName: () => workflowName
|
|
109
119
|
});
|
|
110
120
|
module.exports = __toCommonJS(index_exports);
|
|
@@ -281,10 +291,68 @@ function currentWorkflowCtx() {
|
|
|
281
291
|
}
|
|
282
292
|
__name(currentWorkflowCtx, "currentWorkflowCtx");
|
|
283
293
|
|
|
294
|
+
// src/ambient-step.ts
|
|
295
|
+
var import_node_async_hooks2 = require("async_hooks");
|
|
296
|
+
var STORAGE_KEY2 = Symbol.for("nestjs-durable:ambient-step-logger");
|
|
297
|
+
var globalRef2 = globalThis;
|
|
298
|
+
if (!globalRef2[STORAGE_KEY2]) {
|
|
299
|
+
globalRef2[STORAGE_KEY2] = new import_node_async_hooks2.AsyncLocalStorage();
|
|
300
|
+
}
|
|
301
|
+
var storage2 = globalRef2[STORAGE_KEY2];
|
|
302
|
+
function runInStepLogger(logger, fn) {
|
|
303
|
+
return storage2.run(logger, fn);
|
|
304
|
+
}
|
|
305
|
+
__name(runInStepLogger, "runInStepLogger");
|
|
306
|
+
function currentStep() {
|
|
307
|
+
return storage2.getStore();
|
|
308
|
+
}
|
|
309
|
+
__name(currentStep, "currentStep");
|
|
310
|
+
function log(level, message, data) {
|
|
311
|
+
currentStep()?.[level](message, data);
|
|
312
|
+
}
|
|
313
|
+
__name(log, "log");
|
|
314
|
+
function debug(message, data) {
|
|
315
|
+
currentStep()?.debug(message, data);
|
|
316
|
+
}
|
|
317
|
+
__name(debug, "debug");
|
|
318
|
+
function info(message, data) {
|
|
319
|
+
currentStep()?.info(message, data);
|
|
320
|
+
}
|
|
321
|
+
__name(info, "info");
|
|
322
|
+
function warn(message, data) {
|
|
323
|
+
currentStep()?.warn(message, data);
|
|
324
|
+
}
|
|
325
|
+
__name(warn, "warn");
|
|
326
|
+
function error(message, data) {
|
|
327
|
+
currentStep()?.error(message, data);
|
|
328
|
+
}
|
|
329
|
+
__name(error, "error");
|
|
330
|
+
function sub(name, status, message, data) {
|
|
331
|
+
currentStep()?.sub(name, status, message, data);
|
|
332
|
+
}
|
|
333
|
+
__name(sub, "sub");
|
|
334
|
+
function subEvent(e) {
|
|
335
|
+
currentStep()?.subEvent(e);
|
|
336
|
+
}
|
|
337
|
+
__name(subEvent, "subEvent");
|
|
338
|
+
var noopSubProcessHandle = {
|
|
339
|
+
phase: /* @__PURE__ */ __name(() => noopSubProcessHandle, "phase"),
|
|
340
|
+
skip: /* @__PURE__ */ __name(() => {
|
|
341
|
+
}, "skip"),
|
|
342
|
+
fail: /* @__PURE__ */ __name(() => {
|
|
343
|
+
}, "fail")
|
|
344
|
+
};
|
|
345
|
+
function subProcess(name, body, opts) {
|
|
346
|
+
const logger = currentStep();
|
|
347
|
+
if (!logger) return Promise.resolve(body(noopSubProcessHandle));
|
|
348
|
+
return logger.subProcess(name, body, opts);
|
|
349
|
+
}
|
|
350
|
+
__name(subProcess, "subProcess");
|
|
351
|
+
|
|
284
352
|
// src/control-flow-signal.ts
|
|
285
353
|
var CONTROL_FLOW_SIGNAL = Symbol.for("aviary:durable:control-flow");
|
|
286
|
-
function isWorkflowControlFlowSignal(
|
|
287
|
-
return typeof
|
|
354
|
+
function isWorkflowControlFlowSignal(error2) {
|
|
355
|
+
return typeof error2 === "object" && error2 !== null && error2[CONTROL_FLOW_SIGNAL] === true;
|
|
288
356
|
}
|
|
289
357
|
__name(isWorkflowControlFlowSignal, "isWorkflowControlFlowSignal");
|
|
290
358
|
|
|
@@ -421,11 +489,11 @@ __name(workflowName, "workflowName");
|
|
|
421
489
|
|
|
422
490
|
// src/durable-workflow.ts
|
|
423
491
|
var BINDINGS_KEY = Symbol.for("nestjs-durable:workflow-class-bindings");
|
|
424
|
-
var
|
|
425
|
-
if (!
|
|
426
|
-
|
|
492
|
+
var globalRef3 = globalThis;
|
|
493
|
+
if (!globalRef3[BINDINGS_KEY]) {
|
|
494
|
+
globalRef3[BINDINGS_KEY] = /* @__PURE__ */ new WeakMap();
|
|
427
495
|
}
|
|
428
|
-
var bindings =
|
|
496
|
+
var bindings = globalRef3[BINDINGS_KEY];
|
|
429
497
|
function bindWorkflowClass(ctor, client) {
|
|
430
498
|
bindings.set(ctor, client);
|
|
431
499
|
}
|
|
@@ -556,7 +624,7 @@ function createStepLogger(events, now) {
|
|
|
556
624
|
data
|
|
557
625
|
}
|
|
558
626
|
}), "push");
|
|
559
|
-
const
|
|
627
|
+
const subEvent2 = /* @__PURE__ */ __name((e) => events.push({
|
|
560
628
|
at: now(),
|
|
561
629
|
level: e.status === "failed" ? "error" : e.status === "skipped" ? "warn" : "info",
|
|
562
630
|
message: e.message ?? e.phase ?? e.name,
|
|
@@ -575,7 +643,7 @@ function createStepLogger(events, now) {
|
|
|
575
643
|
data: e.data
|
|
576
644
|
}
|
|
577
645
|
}), "subEvent");
|
|
578
|
-
const
|
|
646
|
+
const subProcess2 = /* @__PURE__ */ __name(async (name, body, opts) => {
|
|
579
647
|
const id = opts?.id ?? globalThis.crypto.randomUUID();
|
|
580
648
|
const group = opts?.group;
|
|
581
649
|
const start = now();
|
|
@@ -587,7 +655,7 @@ function createStepLogger(events, now) {
|
|
|
587
655
|
const emit = /* @__PURE__ */ __name((status, message, data) => {
|
|
588
656
|
if (terminal) return;
|
|
589
657
|
terminal = true;
|
|
590
|
-
|
|
658
|
+
subEvent2({
|
|
591
659
|
id,
|
|
592
660
|
name,
|
|
593
661
|
group,
|
|
@@ -598,7 +666,7 @@ function createStepLogger(events, now) {
|
|
|
598
666
|
}, "emit");
|
|
599
667
|
const handle = {
|
|
600
668
|
phase: /* @__PURE__ */ __name((phase, data) => {
|
|
601
|
-
if (!terminal)
|
|
669
|
+
if (!terminal) subEvent2({
|
|
602
670
|
id,
|
|
603
671
|
name,
|
|
604
672
|
group,
|
|
@@ -636,8 +704,8 @@ function createStepLogger(events, now) {
|
|
|
636
704
|
data
|
|
637
705
|
}
|
|
638
706
|
}), "sub"),
|
|
639
|
-
subEvent,
|
|
640
|
-
subProcess
|
|
707
|
+
subEvent: subEvent2,
|
|
708
|
+
subProcess: subProcess2
|
|
641
709
|
};
|
|
642
710
|
}
|
|
643
711
|
__name(createStepLogger, "createStepLogger");
|
|
@@ -671,7 +739,8 @@ async function runStepHandler(task, handler) {
|
|
|
671
739
|
events
|
|
672
740
|
} : result, "withEvents");
|
|
673
741
|
try {
|
|
674
|
-
const
|
|
742
|
+
const logger = createStepLogger(events, Date.now);
|
|
743
|
+
const output = await runInStepLogger(logger, () => handler(task.input, logger));
|
|
675
744
|
return withEvents({
|
|
676
745
|
...base,
|
|
677
746
|
status: "completed",
|
|
@@ -1721,7 +1790,10 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
1721
1790
|
seq: current,
|
|
1722
1791
|
attempt
|
|
1723
1792
|
};
|
|
1724
|
-
const body = /* @__PURE__ */ __name(() =>
|
|
1793
|
+
const body = /* @__PURE__ */ __name(() => {
|
|
1794
|
+
const logger = createStepLogger(events, host.clock);
|
|
1795
|
+
return runInStepLogger(logger, () => fn(logger));
|
|
1796
|
+
}, "body");
|
|
1725
1797
|
const output = host.interceptStep ? await host.interceptStep(invocation, body) : await body();
|
|
1726
1798
|
await host.completeStep({
|
|
1727
1799
|
runId,
|
|
@@ -2324,6 +2396,10 @@ __name(mergeTags, "mergeTags");
|
|
|
2324
2396
|
var BREAKPOINT = "breakpoint";
|
|
2325
2397
|
var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
|
|
2326
2398
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
2399
|
+
function stepGroup(step, runNamespace) {
|
|
2400
|
+
return tenantGroup(sanitizeQueueToken(step.name), step.partition ?? runNamespace);
|
|
2401
|
+
}
|
|
2402
|
+
__name(stepGroup, "stepGroup");
|
|
2327
2403
|
function nextContinuationId(runId) {
|
|
2328
2404
|
const m = runId.match(/^(.*)~(\d+)$/);
|
|
2329
2405
|
return m ? `${m[1]}~${Number(m[2]) + 1}` : `${runId}~1`;
|
|
@@ -3023,13 +3099,13 @@ var WorkflowEngine = class {
|
|
|
3023
3099
|
];
|
|
3024
3100
|
for (const run of inflight) {
|
|
3025
3101
|
if (run.createdAt.getTime() > deadline) continue;
|
|
3026
|
-
const
|
|
3102
|
+
const error2 = {
|
|
3027
3103
|
message: "execution timeout",
|
|
3028
3104
|
code: "execution_timeout"
|
|
3029
3105
|
};
|
|
3030
3106
|
await this.store.updateRun(run.id, {
|
|
3031
3107
|
status: "cancelled",
|
|
3032
|
-
error,
|
|
3108
|
+
error: error2,
|
|
3033
3109
|
updatedAt: /* @__PURE__ */ new Date()
|
|
3034
3110
|
});
|
|
3035
3111
|
this.emit({
|
|
@@ -3037,7 +3113,7 @@ var WorkflowEngine = class {
|
|
|
3037
3113
|
runId: run.id,
|
|
3038
3114
|
workflow: run.workflow,
|
|
3039
3115
|
namespace: run.namespace,
|
|
3040
|
-
error
|
|
3116
|
+
error: error2
|
|
3041
3117
|
});
|
|
3042
3118
|
}
|
|
3043
3119
|
}
|
|
@@ -3052,6 +3128,22 @@ var WorkflowEngine = class {
|
|
|
3052
3128
|
for (const run of await this.store.listIncompleteRuns(this.namespace)) {
|
|
3053
3129
|
const acquired = await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs);
|
|
3054
3130
|
if (!acquired) continue;
|
|
3131
|
+
if (run.status === "running") {
|
|
3132
|
+
const dispatched = await this.inFlightRemoteStep(run.id);
|
|
3133
|
+
if (dispatched) {
|
|
3134
|
+
await this.store.updateRun(run.id, {
|
|
3135
|
+
status: "suspended",
|
|
3136
|
+
wakeAt: this.reconcileWakeAt(dispatched.wakeAt ?? void 0),
|
|
3137
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3138
|
+
});
|
|
3139
|
+
await this.store.releaseRunLock(run.id);
|
|
3140
|
+
results.push({
|
|
3141
|
+
runId: run.id,
|
|
3142
|
+
status: "suspended"
|
|
3143
|
+
});
|
|
3144
|
+
continue;
|
|
3145
|
+
}
|
|
3146
|
+
}
|
|
3055
3147
|
const settled = await this.countRecovery(run);
|
|
3056
3148
|
if (settled) {
|
|
3057
3149
|
results.push(settled);
|
|
@@ -3073,6 +3165,16 @@ var WorkflowEngine = class {
|
|
|
3073
3165
|
return results;
|
|
3074
3166
|
}
|
|
3075
3167
|
/**
|
|
3168
|
+
* The remote step this run dispatched and is still awaiting a result for, if any — the signature
|
|
3169
|
+
* that tells {@link recoverIncomplete} "this run's work is ON THE TRANSPORT, not in a crashed
|
|
3170
|
+
* process". A `pending` remote checkpoint is written BEFORE the dispatch and settled by the
|
|
3171
|
+
* worker's result, so its presence is the durable marker of an in-flight dispatched step.
|
|
3172
|
+
*/
|
|
3173
|
+
async inFlightRemoteStep(runId) {
|
|
3174
|
+
const checkpoints = await this.store.listCheckpoints(runId);
|
|
3175
|
+
return checkpoints.find((cp) => cp.kind === "remote" && cp.status === "pending");
|
|
3176
|
+
}
|
|
3177
|
+
/**
|
|
3076
3178
|
* Per-recovery bookkeeping (called once the lease is held): count the attempt, or — past
|
|
3077
3179
|
* `maxRecoveryAttempts` — move a poison pill to the `dead` dead-letter state. Returns a terminal
|
|
3078
3180
|
* result to skip the resume, or `undefined` to proceed.
|
|
@@ -3080,13 +3182,13 @@ var WorkflowEngine = class {
|
|
|
3080
3182
|
async countRecovery(run) {
|
|
3081
3183
|
const attempts = (run.recoveryAttempts ?? 0) + 1;
|
|
3082
3184
|
if (this.maxRecoveryAttempts != null && attempts > this.maxRecoveryAttempts) {
|
|
3083
|
-
const
|
|
3185
|
+
const error2 = {
|
|
3084
3186
|
message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) \u2014 moved to dead-letter`,
|
|
3085
3187
|
code: "max_recovery_attempts"
|
|
3086
3188
|
};
|
|
3087
3189
|
await this.store.updateRun(run.id, {
|
|
3088
3190
|
status: "dead",
|
|
3089
|
-
error,
|
|
3191
|
+
error: error2,
|
|
3090
3192
|
updatedAt: /* @__PURE__ */ new Date()
|
|
3091
3193
|
});
|
|
3092
3194
|
await this.store.releaseRunLock(run.id);
|
|
@@ -3095,18 +3197,18 @@ var WorkflowEngine = class {
|
|
|
3095
3197
|
runId: run.id,
|
|
3096
3198
|
workflow: run.workflow,
|
|
3097
3199
|
namespace: run.namespace,
|
|
3098
|
-
error
|
|
3200
|
+
error: error2
|
|
3099
3201
|
});
|
|
3100
3202
|
this.notifyDead({
|
|
3101
3203
|
...run,
|
|
3102
3204
|
status: "dead",
|
|
3103
|
-
error,
|
|
3205
|
+
error: error2,
|
|
3104
3206
|
recoveryAttempts: attempts
|
|
3105
3207
|
});
|
|
3106
3208
|
return {
|
|
3107
3209
|
runId: run.id,
|
|
3108
3210
|
status: "dead",
|
|
3109
|
-
error
|
|
3211
|
+
error: error2
|
|
3110
3212
|
};
|
|
3111
3213
|
}
|
|
3112
3214
|
await this.store.updateRun(run.id, {
|
|
@@ -3227,7 +3329,8 @@ var WorkflowEngine = class {
|
|
|
3227
3329
|
seen.add(runId);
|
|
3228
3330
|
const run = await this.store.getRun(runId);
|
|
3229
3331
|
if (!run) return null;
|
|
3230
|
-
|
|
3332
|
+
const resurrecting = run.status === "failed" || run.status === "dead";
|
|
3333
|
+
if (resurrecting) {
|
|
3231
3334
|
const isFailureCompletion = /* @__PURE__ */ __name((v) => typeof v === "object" && v !== null && v.ok === false, "isFailureCompletion");
|
|
3232
3335
|
for (const cp of await this.store.listCheckpoints(runId)) {
|
|
3233
3336
|
if (cp.status === "failed") {
|
|
@@ -3270,6 +3373,9 @@ var WorkflowEngine = class {
|
|
|
3270
3373
|
await this.store.updateRun(runId, {
|
|
3271
3374
|
status: "pending",
|
|
3272
3375
|
error: void 0,
|
|
3376
|
+
...resurrecting ? {
|
|
3377
|
+
recoveryAttempts: 0
|
|
3378
|
+
} : {},
|
|
3273
3379
|
updatedAt: /* @__PURE__ */ new Date()
|
|
3274
3380
|
});
|
|
3275
3381
|
await this.runDispatcher.dispatch(runId);
|
|
@@ -3282,9 +3388,11 @@ var WorkflowEngine = class {
|
|
|
3282
3388
|
* Explicitly re-dispatch every remote step of `runId` stuck `pending` — the OPERATOR escape hatch for
|
|
3283
3389
|
* a run whose dispatched step job was LOST (worker crashed with no result, or a transport that dropped
|
|
3284
3390
|
* the job). No automatic recovery re-drives these: a reconcile re-drive re-suspends a still-pending
|
|
3285
|
-
* step by design (unless `remoteRedispatchMs` is set), `recoverIncomplete`
|
|
3286
|
-
*
|
|
3287
|
-
*
|
|
3391
|
+
* step by design (unless `remoteRedispatchMs` is set), `recoverIncomplete` explicitly SKIPS a run with
|
|
3392
|
+
* an in-flight `pending` remote step (it re-asserts the durably-suspended invariant instead of counting
|
|
3393
|
+
* a recovery attempt — see there), and `requeue` just replays back to the pending guard. This
|
|
3394
|
+
* re-enqueues the same `stepId` so the (idempotent) step re-runs and its result resumes the run.
|
|
3395
|
+
* Safe to call on a healthy run — it re-runs
|
|
3288
3396
|
* only in-flight `pending` remote steps, which are idempotent by the durable contract. Returns the
|
|
3289
3397
|
* run's current status (with the count re-dispatched), or null if the run is unknown.
|
|
3290
3398
|
*/
|
|
@@ -3576,10 +3684,10 @@ var WorkflowEngine = class {
|
|
|
3576
3684
|
});
|
|
3577
3685
|
}
|
|
3578
3686
|
/** Report that a `ctx.task` failed — the run resumes and throws a FatalError at the task. */
|
|
3579
|
-
async failTask(runId, name,
|
|
3687
|
+
async failTask(runId, name, error2) {
|
|
3580
3688
|
return this.signal(`task:${runId}:${name}`, {
|
|
3581
3689
|
ok: false,
|
|
3582
|
-
error
|
|
3690
|
+
error: error2
|
|
3583
3691
|
});
|
|
3584
3692
|
}
|
|
3585
3693
|
/**
|
|
@@ -3661,12 +3769,12 @@ var WorkflowEngine = class {
|
|
|
3661
3769
|
status: "cancelling"
|
|
3662
3770
|
};
|
|
3663
3771
|
}
|
|
3664
|
-
const
|
|
3772
|
+
const error2 = {
|
|
3665
3773
|
message: "cancelled"
|
|
3666
3774
|
};
|
|
3667
3775
|
await this.store.updateRun(runId, {
|
|
3668
3776
|
status: "cancelled",
|
|
3669
|
-
error,
|
|
3777
|
+
error: error2,
|
|
3670
3778
|
updatedAt: /* @__PURE__ */ new Date()
|
|
3671
3779
|
});
|
|
3672
3780
|
this.emit({
|
|
@@ -3674,7 +3782,7 @@ var WorkflowEngine = class {
|
|
|
3674
3782
|
runId,
|
|
3675
3783
|
workflow: run.workflow,
|
|
3676
3784
|
namespace: run.namespace,
|
|
3677
|
-
error
|
|
3785
|
+
error: error2
|
|
3678
3786
|
});
|
|
3679
3787
|
await this.cancelChildren(runId, opts);
|
|
3680
3788
|
this.notifyCancelled(runId);
|
|
@@ -3689,7 +3797,7 @@ var WorkflowEngine = class {
|
|
|
3689
3797
|
return {
|
|
3690
3798
|
runId,
|
|
3691
3799
|
status: "cancelled",
|
|
3692
|
-
error
|
|
3800
|
+
error: error2
|
|
3693
3801
|
};
|
|
3694
3802
|
}
|
|
3695
3803
|
/**
|
|
@@ -3792,7 +3900,8 @@ var WorkflowEngine = class {
|
|
|
3792
3900
|
async workerHealth(extra = []) {
|
|
3793
3901
|
const groups = /* @__PURE__ */ new Set([
|
|
3794
3902
|
...this.knownGroups(extra),
|
|
3795
|
-
...await this.pool.listWorkerGroups()
|
|
3903
|
+
...await this.pool.listWorkerGroups(),
|
|
3904
|
+
...await this.inflightStepGroups()
|
|
3796
3905
|
]);
|
|
3797
3906
|
const workflowTokens = /* @__PURE__ */ new Set();
|
|
3798
3907
|
for (const def of this.workflows.values()) {
|
|
@@ -3813,6 +3922,38 @@ var WorkflowEngine = class {
|
|
|
3813
3922
|
return out;
|
|
3814
3923
|
}
|
|
3815
3924
|
/**
|
|
3925
|
+
* The routing groups of every in-flight run's still-`pending` remote step — the queues where work is
|
|
3926
|
+
* dispatched and awaiting a worker. Folded into {@link workerHealth} so a step waiting on a group
|
|
3927
|
+
* that is NEITHER a registration NOR heartbeating (the exact shape of a run stuck on an offline
|
|
3928
|
+
* tenant pool) still reports its backlog: without this the health scan can't "see" that queue, and
|
|
3929
|
+
* the dashboard shows the run as running instead of no-worker. Only the BullMQ transport introspects
|
|
3930
|
+
* health, so on any other transport this is harmless (the groups just resolve to no `groupHealth`).
|
|
3931
|
+
*
|
|
3932
|
+
* Cost: one `listRuns` over the non-terminal runs plus a checkpoint read each — a diagnostic-path
|
|
3933
|
+
* scan, not a hot path. Scoped to `running`/`suspended`/`blocked` (a `pending` run has dispatched no
|
|
3934
|
+
* step yet) and deduped to distinct groups.
|
|
3935
|
+
*/
|
|
3936
|
+
async inflightStepGroups() {
|
|
3937
|
+
const inflight = await this.store.listRuns({
|
|
3938
|
+
statuses: [
|
|
3939
|
+
"running",
|
|
3940
|
+
"suspended",
|
|
3941
|
+
"blocked"
|
|
3942
|
+
]
|
|
3943
|
+
});
|
|
3944
|
+
const groups = /* @__PURE__ */ new Set();
|
|
3945
|
+
for (const run of inflight) {
|
|
3946
|
+
for (const cp of await this.store.listCheckpoints(run.id)) {
|
|
3947
|
+
if (cp.kind === "remote" && cp.status === "pending" && cp.workerGroup) {
|
|
3948
|
+
groups.add(cp.workerGroup);
|
|
3949
|
+
}
|
|
3950
|
+
}
|
|
3951
|
+
}
|
|
3952
|
+
return [
|
|
3953
|
+
...groups
|
|
3954
|
+
];
|
|
3955
|
+
}
|
|
3956
|
+
/**
|
|
3816
3957
|
* Persist a streamed local-step lifecycle event from a remote workflow worker (see
|
|
3817
3958
|
* {@link WorkflowStepEvent}). A Python `@workflow` runs its `ctx.step`s inline over one turn that
|
|
3818
3959
|
* can last minutes; the worker streams each step's start/finish so the engine checkpoints it LIVE —
|
|
@@ -4084,12 +4225,12 @@ var WorkflowEngine = class {
|
|
|
4084
4225
|
}
|
|
4085
4226
|
async runRemoteExecution(run, registered) {
|
|
4086
4227
|
if (run.status === "cancelling") {
|
|
4087
|
-
const
|
|
4228
|
+
const error2 = {
|
|
4088
4229
|
message: "cancelled"
|
|
4089
4230
|
};
|
|
4090
4231
|
await this.store.updateRun(run.id, {
|
|
4091
4232
|
status: "cancelled",
|
|
4092
|
-
error,
|
|
4233
|
+
error: error2,
|
|
4093
4234
|
updatedAt: /* @__PURE__ */ new Date()
|
|
4094
4235
|
});
|
|
4095
4236
|
this.emit({
|
|
@@ -4097,7 +4238,7 @@ var WorkflowEngine = class {
|
|
|
4097
4238
|
runId: run.id,
|
|
4098
4239
|
workflow: run.workflow,
|
|
4099
4240
|
namespace: run.namespace,
|
|
4100
|
-
error
|
|
4241
|
+
error: error2
|
|
4101
4242
|
});
|
|
4102
4243
|
this.notifyCancelled(run.id);
|
|
4103
4244
|
if (this.controlPlane) {
|
|
@@ -4111,7 +4252,7 @@ var WorkflowEngine = class {
|
|
|
4111
4252
|
return {
|
|
4112
4253
|
runId: run.id,
|
|
4113
4254
|
status: "cancelled",
|
|
4114
|
-
error
|
|
4255
|
+
error: error2
|
|
4115
4256
|
};
|
|
4116
4257
|
}
|
|
4117
4258
|
const remote = registered.remote;
|
|
@@ -4190,13 +4331,13 @@ var WorkflowEngine = class {
|
|
|
4190
4331
|
status: "running"
|
|
4191
4332
|
};
|
|
4192
4333
|
}
|
|
4193
|
-
const
|
|
4334
|
+
const error2 = {
|
|
4194
4335
|
message: err instanceof Error ? err.message : String(err),
|
|
4195
4336
|
stack: err instanceof Error ? err.stack : void 0
|
|
4196
4337
|
};
|
|
4197
4338
|
await this.store.updateRun(run.id, {
|
|
4198
4339
|
status: "failed",
|
|
4199
|
-
error,
|
|
4340
|
+
error: error2,
|
|
4200
4341
|
updatedAt: /* @__PURE__ */ new Date()
|
|
4201
4342
|
});
|
|
4202
4343
|
this.emit({
|
|
@@ -4204,12 +4345,12 @@ var WorkflowEngine = class {
|
|
|
4204
4345
|
runId: run.id,
|
|
4205
4346
|
workflow: run.workflow,
|
|
4206
4347
|
namespace: run.namespace,
|
|
4207
|
-
error
|
|
4348
|
+
error: error2
|
|
4208
4349
|
});
|
|
4209
4350
|
return {
|
|
4210
4351
|
runId: run.id,
|
|
4211
4352
|
status: "failed",
|
|
4212
|
-
error
|
|
4353
|
+
error: error2
|
|
4213
4354
|
};
|
|
4214
4355
|
}
|
|
4215
4356
|
return this.applyDecision(run, decision);
|
|
@@ -4474,12 +4615,12 @@ var WorkflowEngine = class {
|
|
|
4474
4615
|
status: "cancelling"
|
|
4475
4616
|
};
|
|
4476
4617
|
this.cancelRequested.delete(run.id);
|
|
4477
|
-
const
|
|
4618
|
+
const error2 = {
|
|
4478
4619
|
message: "cancelled"
|
|
4479
4620
|
};
|
|
4480
4621
|
await this.store.updateRun(run.id, {
|
|
4481
4622
|
status: "cancelled",
|
|
4482
|
-
error,
|
|
4623
|
+
error: error2,
|
|
4483
4624
|
updatedAt: /* @__PURE__ */ new Date()
|
|
4484
4625
|
});
|
|
4485
4626
|
this.emit({
|
|
@@ -4487,12 +4628,12 @@ var WorkflowEngine = class {
|
|
|
4487
4628
|
runId: run.id,
|
|
4488
4629
|
workflow: run.workflow,
|
|
4489
4630
|
namespace: run.namespace,
|
|
4490
|
-
error
|
|
4631
|
+
error: error2
|
|
4491
4632
|
});
|
|
4492
4633
|
return {
|
|
4493
4634
|
runId: run.id,
|
|
4494
4635
|
status: "cancelled",
|
|
4495
|
-
error
|
|
4636
|
+
error: error2
|
|
4496
4637
|
};
|
|
4497
4638
|
}
|
|
4498
4639
|
/** The run body, lease held + renewed by {@link execute}. */
|
|
@@ -4593,7 +4734,7 @@ var WorkflowEngine = class {
|
|
|
4593
4734
|
if (this.cancelRequested.has(run.id)) {
|
|
4594
4735
|
return this.compensateAndCancel(run, compensations, replay);
|
|
4595
4736
|
}
|
|
4596
|
-
const
|
|
4737
|
+
const error2 = {
|
|
4597
4738
|
message: err instanceof Error ? err.message : String(err),
|
|
4598
4739
|
stack: err instanceof Error ? err.stack : void 0
|
|
4599
4740
|
};
|
|
@@ -4604,7 +4745,7 @@ var WorkflowEngine = class {
|
|
|
4604
4745
|
});
|
|
4605
4746
|
return this.settleRun(run, {
|
|
4606
4747
|
kind: "failed",
|
|
4607
|
-
error
|
|
4748
|
+
error: error2
|
|
4608
4749
|
});
|
|
4609
4750
|
} finally {
|
|
4610
4751
|
await this.store.releaseRunLock(run.id);
|
|
@@ -4693,7 +4834,7 @@ var WorkflowEngine = class {
|
|
|
4693
4834
|
return;
|
|
4694
4835
|
} catch (err) {
|
|
4695
4836
|
if (attempt >= this.compensationRetries) {
|
|
4696
|
-
const
|
|
4837
|
+
const error2 = {
|
|
4697
4838
|
message: err instanceof Error ? err.message : String(err)
|
|
4698
4839
|
};
|
|
4699
4840
|
this.emit({
|
|
@@ -4702,7 +4843,7 @@ var WorkflowEngine = class {
|
|
|
4702
4843
|
workflow: run.workflow,
|
|
4703
4844
|
name,
|
|
4704
4845
|
kind: "local",
|
|
4705
|
-
error,
|
|
4846
|
+
error: error2,
|
|
4706
4847
|
durationMs: Date.now() - startedAt.getTime()
|
|
4707
4848
|
});
|
|
4708
4849
|
const cp = stepCheckpoint({
|
|
@@ -4712,7 +4853,7 @@ var WorkflowEngine = class {
|
|
|
4712
4853
|
kind: "local",
|
|
4713
4854
|
status: "failed",
|
|
4714
4855
|
attempts: attempt,
|
|
4715
|
-
error,
|
|
4856
|
+
error: error2,
|
|
4716
4857
|
enqueuedAt,
|
|
4717
4858
|
startedAt,
|
|
4718
4859
|
finishedAt: /* @__PURE__ */ new Date()
|
|
@@ -4776,7 +4917,7 @@ var WorkflowEngine = class {
|
|
|
4776
4917
|
*/
|
|
4777
4918
|
async dispatchCompensationAttempt(run, seq, name, def, args, attempt, replay) {
|
|
4778
4919
|
if (this.pool.size === 0) throw new Error("remote steps require a Transport");
|
|
4779
|
-
const group =
|
|
4920
|
+
const group = stepGroup(def, run.namespace);
|
|
4780
4921
|
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
4781
4922
|
const cp = stepCheckpoint({
|
|
4782
4923
|
runId: run.id,
|
|
@@ -4824,7 +4965,12 @@ var WorkflowEngine = class {
|
|
|
4824
4965
|
startStep: /* @__PURE__ */ __name((s) => this.startStep(s), "startStep"),
|
|
4825
4966
|
completeStep: /* @__PURE__ */ __name((s) => this.completeStep(s), "completeStep"),
|
|
4826
4967
|
failStep: /* @__PURE__ */ __name((s) => this.failStep(s), "failStep"),
|
|
4827
|
-
|
|
4968
|
+
// Route the dispatch by the RUN's tenant (same reason as `startChild` below): the executing
|
|
4969
|
+
// engine may be an operator driving every namespace, so the token must come from the run, not
|
|
4970
|
+
// from this engine. See {@link stepGroup}.
|
|
4971
|
+
callRemote: /* @__PURE__ */ __name((runId, seq, step, input, queue, transport, admission) => this.callRemote(runId, seq, step, input, queue, transport, replay, admission, {
|
|
4972
|
+
namespace: parentNamespace
|
|
4973
|
+
}), "callRemote"),
|
|
4828
4974
|
// Defer so a fast child can't reentrantly resume a still-running parent. Stamp the child with
|
|
4829
4975
|
// the PARENT run's namespace (not this engine's) so a child belongs to the tenant/partition of
|
|
4830
4976
|
// the run it was spawned from — even when an operator (namespace: undefined, drives every
|
|
@@ -4968,14 +5114,14 @@ var WorkflowEngine = class {
|
|
|
4968
5114
|
error: current.error
|
|
4969
5115
|
};
|
|
4970
5116
|
}
|
|
4971
|
-
const
|
|
5117
|
+
const error2 = {
|
|
4972
5118
|
message: blocked.reason,
|
|
4973
5119
|
code: blocked.code,
|
|
4974
5120
|
retryable: true
|
|
4975
5121
|
};
|
|
4976
5122
|
await this.store.updateRun(run.id, {
|
|
4977
5123
|
status: "blocked",
|
|
4978
|
-
error,
|
|
5124
|
+
error: error2,
|
|
4979
5125
|
wakeAt: this.clock() + this.blockedPollMs,
|
|
4980
5126
|
updatedAt: /* @__PURE__ */ new Date()
|
|
4981
5127
|
});
|
|
@@ -4984,7 +5130,7 @@ var WorkflowEngine = class {
|
|
|
4984
5130
|
runId: run.id,
|
|
4985
5131
|
workflow: run.workflow,
|
|
4986
5132
|
namespace: run.namespace,
|
|
4987
|
-
error,
|
|
5133
|
+
error: error2,
|
|
4988
5134
|
requires: blocked.requires,
|
|
4989
5135
|
diagnostics: blocked.diagnostics
|
|
4990
5136
|
});
|
|
@@ -4992,7 +5138,7 @@ var WorkflowEngine = class {
|
|
|
4992
5138
|
return {
|
|
4993
5139
|
runId: run.id,
|
|
4994
5140
|
status: "blocked",
|
|
4995
|
-
error
|
|
5141
|
+
error: error2
|
|
4996
5142
|
};
|
|
4997
5143
|
}
|
|
4998
5144
|
/** Runs parked `blocked` whose recovery poll is due (`wakeAt <= nowMs`), namespace-scoped like
|
|
@@ -5006,14 +5152,15 @@ var WorkflowEngine = class {
|
|
|
5006
5152
|
});
|
|
5007
5153
|
return blocked.filter((r) => (this.namespace === void 0 || r.namespace === void 0 || r.namespace === this.namespace) && r.wakeAt !== void 0 && r.wakeAt <= nowMs);
|
|
5008
5154
|
}
|
|
5009
|
-
async callRemote(runId, seq, step, input, queue, transport, replay, admission) {
|
|
5155
|
+
async callRemote(runId, seq, step, input, queue, transport, replay, admission, routing) {
|
|
5156
|
+
const group = stepGroup(step, routing?.namespace);
|
|
5010
5157
|
const existing = replay?.get(seq) ?? await this.store.getCheckpoint(runId, seq);
|
|
5011
5158
|
if (existing && existing.name !== step.name) {
|
|
5012
5159
|
throw new NonDeterminismError(runId, seq, step.name, existing.name);
|
|
5013
5160
|
}
|
|
5014
5161
|
if (existing?.status === "completed") return existing.output;
|
|
5015
5162
|
if (this.pool.size === 0) throw new Error("remote steps require a Transport");
|
|
5016
|
-
if (step.timeoutMs) return this.callRemoteInMemory(runId, seq, step, input, transport);
|
|
5163
|
+
if (step.timeoutMs) return this.callRemoteInMemory(runId, seq, step, input, transport, group);
|
|
5017
5164
|
if (existing?.status === "pending") {
|
|
5018
5165
|
if (this.remoteRedispatchMs == null) throw new WorkflowSuspended();
|
|
5019
5166
|
if (existing.wakeAt == null) {
|
|
@@ -5047,7 +5194,7 @@ var WorkflowEngine = class {
|
|
|
5047
5194
|
seq,
|
|
5048
5195
|
name: step.name,
|
|
5049
5196
|
stepId: existing.stepId ?? stepId(runId, seq),
|
|
5050
|
-
group: existing.workerGroup ??
|
|
5197
|
+
group: existing.workerGroup ?? group,
|
|
5051
5198
|
input: existing.input,
|
|
5052
5199
|
priority: admission?.priority,
|
|
5053
5200
|
attempt: reAttempt,
|
|
@@ -5071,7 +5218,7 @@ var WorkflowEngine = class {
|
|
|
5071
5218
|
if (this.clock() < existing.wakeAt) throw new WorkflowSuspended(existing.wakeAt);
|
|
5072
5219
|
attempt = existing.attempts + 1;
|
|
5073
5220
|
}
|
|
5074
|
-
await this.ensureRoutable(
|
|
5221
|
+
await this.ensureRoutable(group, step.requires);
|
|
5075
5222
|
const id = stepId(runId, seq);
|
|
5076
5223
|
if (queue && this.admission.handles(queue)) {
|
|
5077
5224
|
const decision = await this.admission.tryAdmit(queue, {
|
|
@@ -5098,7 +5245,7 @@ var WorkflowEngine = class {
|
|
|
5098
5245
|
status: "pending",
|
|
5099
5246
|
input,
|
|
5100
5247
|
attempts: attempt,
|
|
5101
|
-
workerGroup:
|
|
5248
|
+
workerGroup: group,
|
|
5102
5249
|
enqueuedAt,
|
|
5103
5250
|
startedAt: enqueuedAt,
|
|
5104
5251
|
finishedAt: enqueuedAt
|
|
@@ -5108,7 +5255,7 @@ var WorkflowEngine = class {
|
|
|
5108
5255
|
seq,
|
|
5109
5256
|
name: step.name,
|
|
5110
5257
|
stepId: id,
|
|
5111
|
-
group
|
|
5258
|
+
group,
|
|
5112
5259
|
input,
|
|
5113
5260
|
priority: admission?.priority,
|
|
5114
5261
|
attempt,
|
|
@@ -5256,9 +5403,9 @@ var WorkflowEngine = class {
|
|
|
5256
5403
|
await this.admission.release(queue, id);
|
|
5257
5404
|
}
|
|
5258
5405
|
/** In-memory await path for a remote step with a liveness `timeoutMs` (re-dispatch on timeout). */
|
|
5259
|
-
async callRemoteInMemory(runId, seq, step, input, transport) {
|
|
5406
|
+
async callRemoteInMemory(runId, seq, step, input, transport, group = stepGroup(step, void 0)) {
|
|
5260
5407
|
if (this.pool.size === 0) throw new Error("remote steps require a Transport");
|
|
5261
|
-
await this.ensureRoutable(
|
|
5408
|
+
await this.ensureRoutable(group, step.requires);
|
|
5262
5409
|
const validInput = step.input ? step.input.parse(input) : input;
|
|
5263
5410
|
const id = stepId(runId, seq);
|
|
5264
5411
|
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
@@ -5282,7 +5429,7 @@ var WorkflowEngine = class {
|
|
|
5282
5429
|
seq,
|
|
5283
5430
|
name: step.name,
|
|
5284
5431
|
stepId: id,
|
|
5285
|
-
group
|
|
5432
|
+
group,
|
|
5286
5433
|
input: validInput,
|
|
5287
5434
|
traceparent: this.traceparent?.(),
|
|
5288
5435
|
context: this.context?.(),
|
|
@@ -5301,7 +5448,7 @@ var WorkflowEngine = class {
|
|
|
5301
5448
|
output,
|
|
5302
5449
|
events: resolution.events,
|
|
5303
5450
|
attempts: attempt,
|
|
5304
|
-
workerGroup:
|
|
5451
|
+
workerGroup: group,
|
|
5305
5452
|
enqueuedAt,
|
|
5306
5453
|
startedAt
|
|
5307
5454
|
});
|
|
@@ -6213,8 +6360,11 @@ var InMemoryTransport = class {
|
|
|
6213
6360
|
collectMetrics,
|
|
6214
6361
|
controlPlaneDescriptor,
|
|
6215
6362
|
createStepLogger,
|
|
6363
|
+
currentStep,
|
|
6216
6364
|
currentWorkflowCtx,
|
|
6365
|
+
debug,
|
|
6217
6366
|
descriptorHash,
|
|
6367
|
+
error,
|
|
6218
6368
|
eventMatchOf,
|
|
6219
6369
|
eventMatches,
|
|
6220
6370
|
eventNameOf,
|
|
@@ -6222,9 +6372,11 @@ var InMemoryTransport = class {
|
|
|
6222
6372
|
eventToken,
|
|
6223
6373
|
heartbeatStatus,
|
|
6224
6374
|
indexWaitersByRun,
|
|
6375
|
+
info,
|
|
6225
6376
|
isEventToken,
|
|
6226
6377
|
isLegacyDescriptor,
|
|
6227
6378
|
isWorkflowControlFlowSignal,
|
|
6379
|
+
log,
|
|
6228
6380
|
matchesAttributes,
|
|
6229
6381
|
missingCapabilities,
|
|
6230
6382
|
negotiate,
|
|
@@ -6237,6 +6389,7 @@ var InMemoryTransport = class {
|
|
|
6237
6389
|
requiredCapabilities,
|
|
6238
6390
|
resolveRouting,
|
|
6239
6391
|
resolveRunWaiting,
|
|
6392
|
+
runInStepLogger,
|
|
6240
6393
|
runInWorkflowCtx,
|
|
6241
6394
|
runSchedules,
|
|
6242
6395
|
runStepHandler,
|
|
@@ -6246,7 +6399,11 @@ var InMemoryTransport = class {
|
|
|
6246
6399
|
stepConfigOf,
|
|
6247
6400
|
stepId,
|
|
6248
6401
|
stepNameOf,
|
|
6402
|
+
sub,
|
|
6403
|
+
subEvent,
|
|
6404
|
+
subProcess,
|
|
6249
6405
|
tenantGroup,
|
|
6406
|
+
warn,
|
|
6250
6407
|
workflowName
|
|
6251
6408
|
});
|
|
6252
6409
|
//# sourceMappingURL=index.cjs.map
|