@dudousxd/nestjs-durable-core 0.60.0 → 0.62.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +223 -66
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +86 -4
- package/dist/index.d.ts +86 -4
- package/dist/index.js +213 -66
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -173,10 +173,68 @@ function currentWorkflowCtx() {
|
|
|
173
173
|
}
|
|
174
174
|
__name(currentWorkflowCtx, "currentWorkflowCtx");
|
|
175
175
|
|
|
176
|
+
// src/ambient-step.ts
|
|
177
|
+
import { AsyncLocalStorage as AsyncLocalStorage2 } from "node:async_hooks";
|
|
178
|
+
var STORAGE_KEY2 = Symbol.for("nestjs-durable:ambient-step-logger");
|
|
179
|
+
var globalRef2 = globalThis;
|
|
180
|
+
if (!globalRef2[STORAGE_KEY2]) {
|
|
181
|
+
globalRef2[STORAGE_KEY2] = new AsyncLocalStorage2();
|
|
182
|
+
}
|
|
183
|
+
var storage2 = globalRef2[STORAGE_KEY2];
|
|
184
|
+
function runInStepLogger(logger, fn) {
|
|
185
|
+
return storage2.run(logger, fn);
|
|
186
|
+
}
|
|
187
|
+
__name(runInStepLogger, "runInStepLogger");
|
|
188
|
+
function currentStep() {
|
|
189
|
+
return storage2.getStore();
|
|
190
|
+
}
|
|
191
|
+
__name(currentStep, "currentStep");
|
|
192
|
+
function log(level, message, data) {
|
|
193
|
+
currentStep()?.[level](message, data);
|
|
194
|
+
}
|
|
195
|
+
__name(log, "log");
|
|
196
|
+
function debug(message, data) {
|
|
197
|
+
currentStep()?.debug(message, data);
|
|
198
|
+
}
|
|
199
|
+
__name(debug, "debug");
|
|
200
|
+
function info(message, data) {
|
|
201
|
+
currentStep()?.info(message, data);
|
|
202
|
+
}
|
|
203
|
+
__name(info, "info");
|
|
204
|
+
function warn(message, data) {
|
|
205
|
+
currentStep()?.warn(message, data);
|
|
206
|
+
}
|
|
207
|
+
__name(warn, "warn");
|
|
208
|
+
function error(message, data) {
|
|
209
|
+
currentStep()?.error(message, data);
|
|
210
|
+
}
|
|
211
|
+
__name(error, "error");
|
|
212
|
+
function sub(name, status, message, data) {
|
|
213
|
+
currentStep()?.sub(name, status, message, data);
|
|
214
|
+
}
|
|
215
|
+
__name(sub, "sub");
|
|
216
|
+
function subEvent(e) {
|
|
217
|
+
currentStep()?.subEvent(e);
|
|
218
|
+
}
|
|
219
|
+
__name(subEvent, "subEvent");
|
|
220
|
+
var noopSubProcessHandle = {
|
|
221
|
+
phase: /* @__PURE__ */ __name(() => noopSubProcessHandle, "phase"),
|
|
222
|
+
skip: /* @__PURE__ */ __name(() => {
|
|
223
|
+
}, "skip"),
|
|
224
|
+
fail: /* @__PURE__ */ __name(() => {
|
|
225
|
+
}, "fail")
|
|
226
|
+
};
|
|
227
|
+
function subProcess(name, body, opts) {
|
|
228
|
+
const logger = currentStep();
|
|
229
|
+
if (!logger) return Promise.resolve(body(noopSubProcessHandle));
|
|
230
|
+
return logger.subProcess(name, body, opts);
|
|
231
|
+
}
|
|
232
|
+
__name(subProcess, "subProcess");
|
|
233
|
+
|
|
176
234
|
// src/control-flow-signal.ts
|
|
177
235
|
var CONTROL_FLOW_SIGNAL = Symbol.for("aviary:durable:control-flow");
|
|
178
|
-
function isWorkflowControlFlowSignal(
|
|
179
|
-
return typeof
|
|
236
|
+
function isWorkflowControlFlowSignal(error2) {
|
|
237
|
+
return typeof error2 === "object" && error2 !== null && error2[CONTROL_FLOW_SIGNAL] === true;
|
|
180
238
|
}
|
|
181
239
|
__name(isWorkflowControlFlowSignal, "isWorkflowControlFlowSignal");
|
|
182
240
|
|
|
@@ -313,11 +371,11 @@ __name(workflowName, "workflowName");
|
|
|
313
371
|
|
|
314
372
|
// src/durable-workflow.ts
|
|
315
373
|
var BINDINGS_KEY = Symbol.for("nestjs-durable:workflow-class-bindings");
|
|
316
|
-
var
|
|
317
|
-
if (!
|
|
318
|
-
|
|
374
|
+
var globalRef3 = globalThis;
|
|
375
|
+
if (!globalRef3[BINDINGS_KEY]) {
|
|
376
|
+
globalRef3[BINDINGS_KEY] = /* @__PURE__ */ new WeakMap();
|
|
319
377
|
}
|
|
320
|
-
var bindings =
|
|
378
|
+
var bindings = globalRef3[BINDINGS_KEY];
|
|
321
379
|
function bindWorkflowClass(ctor, client) {
|
|
322
380
|
bindings.set(ctor, client);
|
|
323
381
|
}
|
|
@@ -448,7 +506,7 @@ function createStepLogger(events, now) {
|
|
|
448
506
|
data
|
|
449
507
|
}
|
|
450
508
|
}), "push");
|
|
451
|
-
const
|
|
509
|
+
const subEvent2 = /* @__PURE__ */ __name((e) => events.push({
|
|
452
510
|
at: now(),
|
|
453
511
|
level: e.status === "failed" ? "error" : e.status === "skipped" ? "warn" : "info",
|
|
454
512
|
message: e.message ?? e.phase ?? e.name,
|
|
@@ -467,7 +525,7 @@ function createStepLogger(events, now) {
|
|
|
467
525
|
data: e.data
|
|
468
526
|
}
|
|
469
527
|
}), "subEvent");
|
|
470
|
-
const
|
|
528
|
+
const subProcess2 = /* @__PURE__ */ __name(async (name, body, opts) => {
|
|
471
529
|
const id = opts?.id ?? globalThis.crypto.randomUUID();
|
|
472
530
|
const group = opts?.group;
|
|
473
531
|
const start = now();
|
|
@@ -479,7 +537,7 @@ function createStepLogger(events, now) {
|
|
|
479
537
|
const emit = /* @__PURE__ */ __name((status, message, data) => {
|
|
480
538
|
if (terminal) return;
|
|
481
539
|
terminal = true;
|
|
482
|
-
|
|
540
|
+
subEvent2({
|
|
483
541
|
id,
|
|
484
542
|
name,
|
|
485
543
|
group,
|
|
@@ -490,7 +548,7 @@ function createStepLogger(events, now) {
|
|
|
490
548
|
}, "emit");
|
|
491
549
|
const handle = {
|
|
492
550
|
phase: /* @__PURE__ */ __name((phase, data) => {
|
|
493
|
-
if (!terminal)
|
|
551
|
+
if (!terminal) subEvent2({
|
|
494
552
|
id,
|
|
495
553
|
name,
|
|
496
554
|
group,
|
|
@@ -528,8 +586,8 @@ function createStepLogger(events, now) {
|
|
|
528
586
|
data
|
|
529
587
|
}
|
|
530
588
|
}), "sub"),
|
|
531
|
-
subEvent,
|
|
532
|
-
subProcess
|
|
589
|
+
subEvent: subEvent2,
|
|
590
|
+
subProcess: subProcess2
|
|
533
591
|
};
|
|
534
592
|
}
|
|
535
593
|
__name(createStepLogger, "createStepLogger");
|
|
@@ -563,7 +621,8 @@ async function runStepHandler(task, handler) {
|
|
|
563
621
|
events
|
|
564
622
|
} : result, "withEvents");
|
|
565
623
|
try {
|
|
566
|
-
const
|
|
624
|
+
const logger = createStepLogger(events, Date.now);
|
|
625
|
+
const output = await runInStepLogger(logger, () => handler(task.input, logger));
|
|
567
626
|
return withEvents({
|
|
568
627
|
...base,
|
|
569
628
|
status: "completed",
|
|
@@ -1613,7 +1672,10 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
1613
1672
|
seq: current,
|
|
1614
1673
|
attempt
|
|
1615
1674
|
};
|
|
1616
|
-
const body = /* @__PURE__ */ __name(() =>
|
|
1675
|
+
const body = /* @__PURE__ */ __name(() => {
|
|
1676
|
+
const logger = createStepLogger(events, host.clock);
|
|
1677
|
+
return runInStepLogger(logger, () => fn(logger));
|
|
1678
|
+
}, "body");
|
|
1617
1679
|
const output = host.interceptStep ? await host.interceptStep(invocation, body) : await body();
|
|
1618
1680
|
await host.completeStep({
|
|
1619
1681
|
runId,
|
|
@@ -2216,6 +2278,10 @@ __name(mergeTags, "mergeTags");
|
|
|
2216
2278
|
var BREAKPOINT = "breakpoint";
|
|
2217
2279
|
var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
|
|
2218
2280
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
2281
|
+
function stepGroup(step, runNamespace) {
|
|
2282
|
+
return tenantGroup(sanitizeQueueToken(step.name), step.partition ?? runNamespace);
|
|
2283
|
+
}
|
|
2284
|
+
__name(stepGroup, "stepGroup");
|
|
2219
2285
|
function nextContinuationId(runId) {
|
|
2220
2286
|
const m = runId.match(/^(.*)~(\d+)$/);
|
|
2221
2287
|
return m ? `${m[1]}~${Number(m[2]) + 1}` : `${runId}~1`;
|
|
@@ -2915,13 +2981,13 @@ var WorkflowEngine = class {
|
|
|
2915
2981
|
];
|
|
2916
2982
|
for (const run of inflight) {
|
|
2917
2983
|
if (run.createdAt.getTime() > deadline) continue;
|
|
2918
|
-
const
|
|
2984
|
+
const error2 = {
|
|
2919
2985
|
message: "execution timeout",
|
|
2920
2986
|
code: "execution_timeout"
|
|
2921
2987
|
};
|
|
2922
2988
|
await this.store.updateRun(run.id, {
|
|
2923
2989
|
status: "cancelled",
|
|
2924
|
-
error,
|
|
2990
|
+
error: error2,
|
|
2925
2991
|
updatedAt: /* @__PURE__ */ new Date()
|
|
2926
2992
|
});
|
|
2927
2993
|
this.emit({
|
|
@@ -2929,7 +2995,7 @@ var WorkflowEngine = class {
|
|
|
2929
2995
|
runId: run.id,
|
|
2930
2996
|
workflow: run.workflow,
|
|
2931
2997
|
namespace: run.namespace,
|
|
2932
|
-
error
|
|
2998
|
+
error: error2
|
|
2933
2999
|
});
|
|
2934
3000
|
}
|
|
2935
3001
|
}
|
|
@@ -2944,6 +3010,22 @@ var WorkflowEngine = class {
|
|
|
2944
3010
|
for (const run of await this.store.listIncompleteRuns(this.namespace)) {
|
|
2945
3011
|
const acquired = await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs);
|
|
2946
3012
|
if (!acquired) continue;
|
|
3013
|
+
if (run.status === "running") {
|
|
3014
|
+
const dispatched = await this.inFlightRemoteStep(run.id);
|
|
3015
|
+
if (dispatched) {
|
|
3016
|
+
await this.store.updateRun(run.id, {
|
|
3017
|
+
status: "suspended",
|
|
3018
|
+
wakeAt: this.reconcileWakeAt(dispatched.wakeAt ?? void 0),
|
|
3019
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
3020
|
+
});
|
|
3021
|
+
await this.store.releaseRunLock(run.id);
|
|
3022
|
+
results.push({
|
|
3023
|
+
runId: run.id,
|
|
3024
|
+
status: "suspended"
|
|
3025
|
+
});
|
|
3026
|
+
continue;
|
|
3027
|
+
}
|
|
3028
|
+
}
|
|
2947
3029
|
const settled = await this.countRecovery(run);
|
|
2948
3030
|
if (settled) {
|
|
2949
3031
|
results.push(settled);
|
|
@@ -2965,6 +3047,16 @@ var WorkflowEngine = class {
|
|
|
2965
3047
|
return results;
|
|
2966
3048
|
}
|
|
2967
3049
|
/**
|
|
3050
|
+
* The remote step this run dispatched and is still awaiting a result for, if any — the signature
|
|
3051
|
+
* that tells {@link recoverIncomplete} "this run's work is ON THE TRANSPORT, not in a crashed
|
|
3052
|
+
* process". A `pending` remote checkpoint is written BEFORE the dispatch and settled by the
|
|
3053
|
+
* worker's result, so its presence is the durable marker of an in-flight dispatched step.
|
|
3054
|
+
*/
|
|
3055
|
+
async inFlightRemoteStep(runId) {
|
|
3056
|
+
const checkpoints = await this.store.listCheckpoints(runId);
|
|
3057
|
+
return checkpoints.find((cp) => cp.kind === "remote" && cp.status === "pending");
|
|
3058
|
+
}
|
|
3059
|
+
/**
|
|
2968
3060
|
* Per-recovery bookkeeping (called once the lease is held): count the attempt, or — past
|
|
2969
3061
|
* `maxRecoveryAttempts` — move a poison pill to the `dead` dead-letter state. Returns a terminal
|
|
2970
3062
|
* result to skip the resume, or `undefined` to proceed.
|
|
@@ -2972,13 +3064,13 @@ var WorkflowEngine = class {
|
|
|
2972
3064
|
async countRecovery(run) {
|
|
2973
3065
|
const attempts = (run.recoveryAttempts ?? 0) + 1;
|
|
2974
3066
|
if (this.maxRecoveryAttempts != null && attempts > this.maxRecoveryAttempts) {
|
|
2975
|
-
const
|
|
3067
|
+
const error2 = {
|
|
2976
3068
|
message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) \u2014 moved to dead-letter`,
|
|
2977
3069
|
code: "max_recovery_attempts"
|
|
2978
3070
|
};
|
|
2979
3071
|
await this.store.updateRun(run.id, {
|
|
2980
3072
|
status: "dead",
|
|
2981
|
-
error,
|
|
3073
|
+
error: error2,
|
|
2982
3074
|
updatedAt: /* @__PURE__ */ new Date()
|
|
2983
3075
|
});
|
|
2984
3076
|
await this.store.releaseRunLock(run.id);
|
|
@@ -2987,18 +3079,18 @@ var WorkflowEngine = class {
|
|
|
2987
3079
|
runId: run.id,
|
|
2988
3080
|
workflow: run.workflow,
|
|
2989
3081
|
namespace: run.namespace,
|
|
2990
|
-
error
|
|
3082
|
+
error: error2
|
|
2991
3083
|
});
|
|
2992
3084
|
this.notifyDead({
|
|
2993
3085
|
...run,
|
|
2994
3086
|
status: "dead",
|
|
2995
|
-
error,
|
|
3087
|
+
error: error2,
|
|
2996
3088
|
recoveryAttempts: attempts
|
|
2997
3089
|
});
|
|
2998
3090
|
return {
|
|
2999
3091
|
runId: run.id,
|
|
3000
3092
|
status: "dead",
|
|
3001
|
-
error
|
|
3093
|
+
error: error2
|
|
3002
3094
|
};
|
|
3003
3095
|
}
|
|
3004
3096
|
await this.store.updateRun(run.id, {
|
|
@@ -3119,7 +3211,8 @@ var WorkflowEngine = class {
|
|
|
3119
3211
|
seen.add(runId);
|
|
3120
3212
|
const run = await this.store.getRun(runId);
|
|
3121
3213
|
if (!run) return null;
|
|
3122
|
-
|
|
3214
|
+
const resurrecting = run.status === "failed" || run.status === "dead";
|
|
3215
|
+
if (resurrecting) {
|
|
3123
3216
|
const isFailureCompletion = /* @__PURE__ */ __name((v) => typeof v === "object" && v !== null && v.ok === false, "isFailureCompletion");
|
|
3124
3217
|
for (const cp of await this.store.listCheckpoints(runId)) {
|
|
3125
3218
|
if (cp.status === "failed") {
|
|
@@ -3162,6 +3255,9 @@ var WorkflowEngine = class {
|
|
|
3162
3255
|
await this.store.updateRun(runId, {
|
|
3163
3256
|
status: "pending",
|
|
3164
3257
|
error: void 0,
|
|
3258
|
+
...resurrecting ? {
|
|
3259
|
+
recoveryAttempts: 0
|
|
3260
|
+
} : {},
|
|
3165
3261
|
updatedAt: /* @__PURE__ */ new Date()
|
|
3166
3262
|
});
|
|
3167
3263
|
await this.runDispatcher.dispatch(runId);
|
|
@@ -3174,9 +3270,11 @@ var WorkflowEngine = class {
|
|
|
3174
3270
|
* Explicitly re-dispatch every remote step of `runId` stuck `pending` — the OPERATOR escape hatch for
|
|
3175
3271
|
* a run whose dispatched step job was LOST (worker crashed with no result, or a transport that dropped
|
|
3176
3272
|
* the job). No automatic recovery re-drives these: a reconcile re-drive re-suspends a still-pending
|
|
3177
|
-
* step by design (unless `remoteRedispatchMs` is set), `recoverIncomplete`
|
|
3178
|
-
*
|
|
3179
|
-
*
|
|
3273
|
+
* step by design (unless `remoteRedispatchMs` is set), `recoverIncomplete` explicitly SKIPS a run with
|
|
3274
|
+
* an in-flight `pending` remote step (it re-asserts the durably-suspended invariant instead of counting
|
|
3275
|
+
* a recovery attempt — see there), and `requeue` just replays back to the pending guard. This
|
|
3276
|
+
* re-enqueues the same `stepId` so the (idempotent) step re-runs and its result resumes the run.
|
|
3277
|
+
* Safe to call on a healthy run — it re-runs
|
|
3180
3278
|
* only in-flight `pending` remote steps, which are idempotent by the durable contract. Returns the
|
|
3181
3279
|
* run's current status (with the count re-dispatched), or null if the run is unknown.
|
|
3182
3280
|
*/
|
|
@@ -3468,10 +3566,10 @@ var WorkflowEngine = class {
|
|
|
3468
3566
|
});
|
|
3469
3567
|
}
|
|
3470
3568
|
/** Report that a `ctx.task` failed — the run resumes and throws a FatalError at the task. */
|
|
3471
|
-
async failTask(runId, name,
|
|
3569
|
+
async failTask(runId, name, error2) {
|
|
3472
3570
|
return this.signal(`task:${runId}:${name}`, {
|
|
3473
3571
|
ok: false,
|
|
3474
|
-
error
|
|
3572
|
+
error: error2
|
|
3475
3573
|
});
|
|
3476
3574
|
}
|
|
3477
3575
|
/**
|
|
@@ -3553,12 +3651,12 @@ var WorkflowEngine = class {
|
|
|
3553
3651
|
status: "cancelling"
|
|
3554
3652
|
};
|
|
3555
3653
|
}
|
|
3556
|
-
const
|
|
3654
|
+
const error2 = {
|
|
3557
3655
|
message: "cancelled"
|
|
3558
3656
|
};
|
|
3559
3657
|
await this.store.updateRun(runId, {
|
|
3560
3658
|
status: "cancelled",
|
|
3561
|
-
error,
|
|
3659
|
+
error: error2,
|
|
3562
3660
|
updatedAt: /* @__PURE__ */ new Date()
|
|
3563
3661
|
});
|
|
3564
3662
|
this.emit({
|
|
@@ -3566,7 +3664,7 @@ var WorkflowEngine = class {
|
|
|
3566
3664
|
runId,
|
|
3567
3665
|
workflow: run.workflow,
|
|
3568
3666
|
namespace: run.namespace,
|
|
3569
|
-
error
|
|
3667
|
+
error: error2
|
|
3570
3668
|
});
|
|
3571
3669
|
await this.cancelChildren(runId, opts);
|
|
3572
3670
|
this.notifyCancelled(runId);
|
|
@@ -3581,7 +3679,7 @@ var WorkflowEngine = class {
|
|
|
3581
3679
|
return {
|
|
3582
3680
|
runId,
|
|
3583
3681
|
status: "cancelled",
|
|
3584
|
-
error
|
|
3682
|
+
error: error2
|
|
3585
3683
|
};
|
|
3586
3684
|
}
|
|
3587
3685
|
/**
|
|
@@ -3684,7 +3782,8 @@ var WorkflowEngine = class {
|
|
|
3684
3782
|
async workerHealth(extra = []) {
|
|
3685
3783
|
const groups = /* @__PURE__ */ new Set([
|
|
3686
3784
|
...this.knownGroups(extra),
|
|
3687
|
-
...await this.pool.listWorkerGroups()
|
|
3785
|
+
...await this.pool.listWorkerGroups(),
|
|
3786
|
+
...await this.inflightStepGroups()
|
|
3688
3787
|
]);
|
|
3689
3788
|
const workflowTokens = /* @__PURE__ */ new Set();
|
|
3690
3789
|
for (const def of this.workflows.values()) {
|
|
@@ -3705,6 +3804,38 @@ var WorkflowEngine = class {
|
|
|
3705
3804
|
return out;
|
|
3706
3805
|
}
|
|
3707
3806
|
/**
|
|
3807
|
+
* The routing groups of every in-flight run's still-`pending` remote step — the queues where work is
|
|
3808
|
+
* dispatched and awaiting a worker. Folded into {@link workerHealth} so a step waiting on a group
|
|
3809
|
+
* that is NEITHER a registration NOR heartbeating (the exact shape of a run stuck on an offline
|
|
3810
|
+
* tenant pool) still reports its backlog: without this the health scan can't "see" that queue, and
|
|
3811
|
+
* the dashboard shows the run as running instead of no-worker. Only the BullMQ transport introspects
|
|
3812
|
+
* health, so on any other transport this is harmless (the groups just resolve to no `groupHealth`).
|
|
3813
|
+
*
|
|
3814
|
+
* Cost: one `listRuns` over the non-terminal runs plus a checkpoint read each — a diagnostic-path
|
|
3815
|
+
* scan, not a hot path. Scoped to `running`/`suspended`/`blocked` (a `pending` run has dispatched no
|
|
3816
|
+
* step yet) and deduped to distinct groups.
|
|
3817
|
+
*/
|
|
3818
|
+
async inflightStepGroups() {
|
|
3819
|
+
const inflight = await this.store.listRuns({
|
|
3820
|
+
statuses: [
|
|
3821
|
+
"running",
|
|
3822
|
+
"suspended",
|
|
3823
|
+
"blocked"
|
|
3824
|
+
]
|
|
3825
|
+
});
|
|
3826
|
+
const groups = /* @__PURE__ */ new Set();
|
|
3827
|
+
for (const run of inflight) {
|
|
3828
|
+
for (const cp of await this.store.listCheckpoints(run.id)) {
|
|
3829
|
+
if (cp.kind === "remote" && cp.status === "pending" && cp.workerGroup) {
|
|
3830
|
+
groups.add(cp.workerGroup);
|
|
3831
|
+
}
|
|
3832
|
+
}
|
|
3833
|
+
}
|
|
3834
|
+
return [
|
|
3835
|
+
...groups
|
|
3836
|
+
];
|
|
3837
|
+
}
|
|
3838
|
+
/**
|
|
3708
3839
|
* Persist a streamed local-step lifecycle event from a remote workflow worker (see
|
|
3709
3840
|
* {@link WorkflowStepEvent}). A Python `@workflow` runs its `ctx.step`s inline over one turn that
|
|
3710
3841
|
* can last minutes; the worker streams each step's start/finish so the engine checkpoints it LIVE —
|
|
@@ -3976,12 +4107,12 @@ var WorkflowEngine = class {
|
|
|
3976
4107
|
}
|
|
3977
4108
|
async runRemoteExecution(run, registered) {
|
|
3978
4109
|
if (run.status === "cancelling") {
|
|
3979
|
-
const
|
|
4110
|
+
const error2 = {
|
|
3980
4111
|
message: "cancelled"
|
|
3981
4112
|
};
|
|
3982
4113
|
await this.store.updateRun(run.id, {
|
|
3983
4114
|
status: "cancelled",
|
|
3984
|
-
error,
|
|
4115
|
+
error: error2,
|
|
3985
4116
|
updatedAt: /* @__PURE__ */ new Date()
|
|
3986
4117
|
});
|
|
3987
4118
|
this.emit({
|
|
@@ -3989,7 +4120,7 @@ var WorkflowEngine = class {
|
|
|
3989
4120
|
runId: run.id,
|
|
3990
4121
|
workflow: run.workflow,
|
|
3991
4122
|
namespace: run.namespace,
|
|
3992
|
-
error
|
|
4123
|
+
error: error2
|
|
3993
4124
|
});
|
|
3994
4125
|
this.notifyCancelled(run.id);
|
|
3995
4126
|
if (this.controlPlane) {
|
|
@@ -4003,7 +4134,7 @@ var WorkflowEngine = class {
|
|
|
4003
4134
|
return {
|
|
4004
4135
|
runId: run.id,
|
|
4005
4136
|
status: "cancelled",
|
|
4006
|
-
error
|
|
4137
|
+
error: error2
|
|
4007
4138
|
};
|
|
4008
4139
|
}
|
|
4009
4140
|
const remote = registered.remote;
|
|
@@ -4082,13 +4213,13 @@ var WorkflowEngine = class {
|
|
|
4082
4213
|
status: "running"
|
|
4083
4214
|
};
|
|
4084
4215
|
}
|
|
4085
|
-
const
|
|
4216
|
+
const error2 = {
|
|
4086
4217
|
message: err instanceof Error ? err.message : String(err),
|
|
4087
4218
|
stack: err instanceof Error ? err.stack : void 0
|
|
4088
4219
|
};
|
|
4089
4220
|
await this.store.updateRun(run.id, {
|
|
4090
4221
|
status: "failed",
|
|
4091
|
-
error,
|
|
4222
|
+
error: error2,
|
|
4092
4223
|
updatedAt: /* @__PURE__ */ new Date()
|
|
4093
4224
|
});
|
|
4094
4225
|
this.emit({
|
|
@@ -4096,12 +4227,12 @@ var WorkflowEngine = class {
|
|
|
4096
4227
|
runId: run.id,
|
|
4097
4228
|
workflow: run.workflow,
|
|
4098
4229
|
namespace: run.namespace,
|
|
4099
|
-
error
|
|
4230
|
+
error: error2
|
|
4100
4231
|
});
|
|
4101
4232
|
return {
|
|
4102
4233
|
runId: run.id,
|
|
4103
4234
|
status: "failed",
|
|
4104
|
-
error
|
|
4235
|
+
error: error2
|
|
4105
4236
|
};
|
|
4106
4237
|
}
|
|
4107
4238
|
return this.applyDecision(run, decision);
|
|
@@ -4366,12 +4497,12 @@ var WorkflowEngine = class {
|
|
|
4366
4497
|
status: "cancelling"
|
|
4367
4498
|
};
|
|
4368
4499
|
this.cancelRequested.delete(run.id);
|
|
4369
|
-
const
|
|
4500
|
+
const error2 = {
|
|
4370
4501
|
message: "cancelled"
|
|
4371
4502
|
};
|
|
4372
4503
|
await this.store.updateRun(run.id, {
|
|
4373
4504
|
status: "cancelled",
|
|
4374
|
-
error,
|
|
4505
|
+
error: error2,
|
|
4375
4506
|
updatedAt: /* @__PURE__ */ new Date()
|
|
4376
4507
|
});
|
|
4377
4508
|
this.emit({
|
|
@@ -4379,12 +4510,12 @@ var WorkflowEngine = class {
|
|
|
4379
4510
|
runId: run.id,
|
|
4380
4511
|
workflow: run.workflow,
|
|
4381
4512
|
namespace: run.namespace,
|
|
4382
|
-
error
|
|
4513
|
+
error: error2
|
|
4383
4514
|
});
|
|
4384
4515
|
return {
|
|
4385
4516
|
runId: run.id,
|
|
4386
4517
|
status: "cancelled",
|
|
4387
|
-
error
|
|
4518
|
+
error: error2
|
|
4388
4519
|
};
|
|
4389
4520
|
}
|
|
4390
4521
|
/** The run body, lease held + renewed by {@link execute}. */
|
|
@@ -4485,7 +4616,7 @@ var WorkflowEngine = class {
|
|
|
4485
4616
|
if (this.cancelRequested.has(run.id)) {
|
|
4486
4617
|
return this.compensateAndCancel(run, compensations, replay);
|
|
4487
4618
|
}
|
|
4488
|
-
const
|
|
4619
|
+
const error2 = {
|
|
4489
4620
|
message: err instanceof Error ? err.message : String(err),
|
|
4490
4621
|
stack: err instanceof Error ? err.stack : void 0
|
|
4491
4622
|
};
|
|
@@ -4496,7 +4627,7 @@ var WorkflowEngine = class {
|
|
|
4496
4627
|
});
|
|
4497
4628
|
return this.settleRun(run, {
|
|
4498
4629
|
kind: "failed",
|
|
4499
|
-
error
|
|
4630
|
+
error: error2
|
|
4500
4631
|
});
|
|
4501
4632
|
} finally {
|
|
4502
4633
|
await this.store.releaseRunLock(run.id);
|
|
@@ -4585,7 +4716,7 @@ var WorkflowEngine = class {
|
|
|
4585
4716
|
return;
|
|
4586
4717
|
} catch (err) {
|
|
4587
4718
|
if (attempt >= this.compensationRetries) {
|
|
4588
|
-
const
|
|
4719
|
+
const error2 = {
|
|
4589
4720
|
message: err instanceof Error ? err.message : String(err)
|
|
4590
4721
|
};
|
|
4591
4722
|
this.emit({
|
|
@@ -4594,7 +4725,7 @@ var WorkflowEngine = class {
|
|
|
4594
4725
|
workflow: run.workflow,
|
|
4595
4726
|
name,
|
|
4596
4727
|
kind: "local",
|
|
4597
|
-
error,
|
|
4728
|
+
error: error2,
|
|
4598
4729
|
durationMs: Date.now() - startedAt.getTime()
|
|
4599
4730
|
});
|
|
4600
4731
|
const cp = stepCheckpoint({
|
|
@@ -4604,7 +4735,7 @@ var WorkflowEngine = class {
|
|
|
4604
4735
|
kind: "local",
|
|
4605
4736
|
status: "failed",
|
|
4606
4737
|
attempts: attempt,
|
|
4607
|
-
error,
|
|
4738
|
+
error: error2,
|
|
4608
4739
|
enqueuedAt,
|
|
4609
4740
|
startedAt,
|
|
4610
4741
|
finishedAt: /* @__PURE__ */ new Date()
|
|
@@ -4668,7 +4799,7 @@ var WorkflowEngine = class {
|
|
|
4668
4799
|
*/
|
|
4669
4800
|
async dispatchCompensationAttempt(run, seq, name, def, args, attempt, replay) {
|
|
4670
4801
|
if (this.pool.size === 0) throw new Error("remote steps require a Transport");
|
|
4671
|
-
const group =
|
|
4802
|
+
const group = stepGroup(def, run.namespace);
|
|
4672
4803
|
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
4673
4804
|
const cp = stepCheckpoint({
|
|
4674
4805
|
runId: run.id,
|
|
@@ -4716,7 +4847,12 @@ var WorkflowEngine = class {
|
|
|
4716
4847
|
startStep: /* @__PURE__ */ __name((s) => this.startStep(s), "startStep"),
|
|
4717
4848
|
completeStep: /* @__PURE__ */ __name((s) => this.completeStep(s), "completeStep"),
|
|
4718
4849
|
failStep: /* @__PURE__ */ __name((s) => this.failStep(s), "failStep"),
|
|
4719
|
-
|
|
4850
|
+
// Route the dispatch by the RUN's tenant (same reason as `startChild` below): the executing
|
|
4851
|
+
// engine may be an operator driving every namespace, so the token must come from the run, not
|
|
4852
|
+
// from this engine. See {@link stepGroup}.
|
|
4853
|
+
callRemote: /* @__PURE__ */ __name((runId, seq, step, input, queue, transport, admission) => this.callRemote(runId, seq, step, input, queue, transport, replay, admission, {
|
|
4854
|
+
namespace: parentNamespace
|
|
4855
|
+
}), "callRemote"),
|
|
4720
4856
|
// Defer so a fast child can't reentrantly resume a still-running parent. Stamp the child with
|
|
4721
4857
|
// the PARENT run's namespace (not this engine's) so a child belongs to the tenant/partition of
|
|
4722
4858
|
// the run it was spawned from — even when an operator (namespace: undefined, drives every
|
|
@@ -4860,14 +4996,14 @@ var WorkflowEngine = class {
|
|
|
4860
4996
|
error: current.error
|
|
4861
4997
|
};
|
|
4862
4998
|
}
|
|
4863
|
-
const
|
|
4999
|
+
const error2 = {
|
|
4864
5000
|
message: blocked.reason,
|
|
4865
5001
|
code: blocked.code,
|
|
4866
5002
|
retryable: true
|
|
4867
5003
|
};
|
|
4868
5004
|
await this.store.updateRun(run.id, {
|
|
4869
5005
|
status: "blocked",
|
|
4870
|
-
error,
|
|
5006
|
+
error: error2,
|
|
4871
5007
|
wakeAt: this.clock() + this.blockedPollMs,
|
|
4872
5008
|
updatedAt: /* @__PURE__ */ new Date()
|
|
4873
5009
|
});
|
|
@@ -4876,7 +5012,7 @@ var WorkflowEngine = class {
|
|
|
4876
5012
|
runId: run.id,
|
|
4877
5013
|
workflow: run.workflow,
|
|
4878
5014
|
namespace: run.namespace,
|
|
4879
|
-
error,
|
|
5015
|
+
error: error2,
|
|
4880
5016
|
requires: blocked.requires,
|
|
4881
5017
|
diagnostics: blocked.diagnostics
|
|
4882
5018
|
});
|
|
@@ -4884,7 +5020,7 @@ var WorkflowEngine = class {
|
|
|
4884
5020
|
return {
|
|
4885
5021
|
runId: run.id,
|
|
4886
5022
|
status: "blocked",
|
|
4887
|
-
error
|
|
5023
|
+
error: error2
|
|
4888
5024
|
};
|
|
4889
5025
|
}
|
|
4890
5026
|
/** Runs parked `blocked` whose recovery poll is due (`wakeAt <= nowMs`), namespace-scoped like
|
|
@@ -4898,14 +5034,15 @@ var WorkflowEngine = class {
|
|
|
4898
5034
|
});
|
|
4899
5035
|
return blocked.filter((r) => (this.namespace === void 0 || r.namespace === void 0 || r.namespace === this.namespace) && r.wakeAt !== void 0 && r.wakeAt <= nowMs);
|
|
4900
5036
|
}
|
|
4901
|
-
async callRemote(runId, seq, step, input, queue, transport, replay, admission) {
|
|
5037
|
+
async callRemote(runId, seq, step, input, queue, transport, replay, admission, routing) {
|
|
5038
|
+
const group = stepGroup(step, routing?.namespace);
|
|
4902
5039
|
const existing = replay?.get(seq) ?? await this.store.getCheckpoint(runId, seq);
|
|
4903
5040
|
if (existing && existing.name !== step.name) {
|
|
4904
5041
|
throw new NonDeterminismError(runId, seq, step.name, existing.name);
|
|
4905
5042
|
}
|
|
4906
5043
|
if (existing?.status === "completed") return existing.output;
|
|
4907
5044
|
if (this.pool.size === 0) throw new Error("remote steps require a Transport");
|
|
4908
|
-
if (step.timeoutMs) return this.callRemoteInMemory(runId, seq, step, input, transport);
|
|
5045
|
+
if (step.timeoutMs) return this.callRemoteInMemory(runId, seq, step, input, transport, group);
|
|
4909
5046
|
if (existing?.status === "pending") {
|
|
4910
5047
|
if (this.remoteRedispatchMs == null) throw new WorkflowSuspended();
|
|
4911
5048
|
if (existing.wakeAt == null) {
|
|
@@ -4939,7 +5076,7 @@ var WorkflowEngine = class {
|
|
|
4939
5076
|
seq,
|
|
4940
5077
|
name: step.name,
|
|
4941
5078
|
stepId: existing.stepId ?? stepId(runId, seq),
|
|
4942
|
-
group: existing.workerGroup ??
|
|
5079
|
+
group: existing.workerGroup ?? group,
|
|
4943
5080
|
input: existing.input,
|
|
4944
5081
|
priority: admission?.priority,
|
|
4945
5082
|
attempt: reAttempt,
|
|
@@ -4963,7 +5100,7 @@ var WorkflowEngine = class {
|
|
|
4963
5100
|
if (this.clock() < existing.wakeAt) throw new WorkflowSuspended(existing.wakeAt);
|
|
4964
5101
|
attempt = existing.attempts + 1;
|
|
4965
5102
|
}
|
|
4966
|
-
await this.ensureRoutable(
|
|
5103
|
+
await this.ensureRoutable(group, step.requires);
|
|
4967
5104
|
const id = stepId(runId, seq);
|
|
4968
5105
|
if (queue && this.admission.handles(queue)) {
|
|
4969
5106
|
const decision = await this.admission.tryAdmit(queue, {
|
|
@@ -4990,7 +5127,7 @@ var WorkflowEngine = class {
|
|
|
4990
5127
|
status: "pending",
|
|
4991
5128
|
input,
|
|
4992
5129
|
attempts: attempt,
|
|
4993
|
-
workerGroup:
|
|
5130
|
+
workerGroup: group,
|
|
4994
5131
|
enqueuedAt,
|
|
4995
5132
|
startedAt: enqueuedAt,
|
|
4996
5133
|
finishedAt: enqueuedAt
|
|
@@ -5000,7 +5137,7 @@ var WorkflowEngine = class {
|
|
|
5000
5137
|
seq,
|
|
5001
5138
|
name: step.name,
|
|
5002
5139
|
stepId: id,
|
|
5003
|
-
group
|
|
5140
|
+
group,
|
|
5004
5141
|
input,
|
|
5005
5142
|
priority: admission?.priority,
|
|
5006
5143
|
attempt,
|
|
@@ -5148,9 +5285,9 @@ var WorkflowEngine = class {
|
|
|
5148
5285
|
await this.admission.release(queue, id);
|
|
5149
5286
|
}
|
|
5150
5287
|
/** In-memory await path for a remote step with a liveness `timeoutMs` (re-dispatch on timeout). */
|
|
5151
|
-
async callRemoteInMemory(runId, seq, step, input, transport) {
|
|
5288
|
+
async callRemoteInMemory(runId, seq, step, input, transport, group = stepGroup(step, void 0)) {
|
|
5152
5289
|
if (this.pool.size === 0) throw new Error("remote steps require a Transport");
|
|
5153
|
-
await this.ensureRoutable(
|
|
5290
|
+
await this.ensureRoutable(group, step.requires);
|
|
5154
5291
|
const validInput = step.input ? step.input.parse(input) : input;
|
|
5155
5292
|
const id = stepId(runId, seq);
|
|
5156
5293
|
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
@@ -5174,7 +5311,7 @@ var WorkflowEngine = class {
|
|
|
5174
5311
|
seq,
|
|
5175
5312
|
name: step.name,
|
|
5176
5313
|
stepId: id,
|
|
5177
|
-
group
|
|
5314
|
+
group,
|
|
5178
5315
|
input: validInput,
|
|
5179
5316
|
traceparent: this.traceparent?.(),
|
|
5180
5317
|
context: this.context?.(),
|
|
@@ -5193,7 +5330,7 @@ var WorkflowEngine = class {
|
|
|
5193
5330
|
output,
|
|
5194
5331
|
events: resolution.events,
|
|
5195
5332
|
attempts: attempt,
|
|
5196
|
-
workerGroup:
|
|
5333
|
+
workerGroup: group,
|
|
5197
5334
|
enqueuedAt,
|
|
5198
5335
|
startedAt
|
|
5199
5336
|
});
|
|
@@ -6104,8 +6241,11 @@ export {
|
|
|
6104
6241
|
collectMetrics,
|
|
6105
6242
|
controlPlaneDescriptor,
|
|
6106
6243
|
createStepLogger,
|
|
6244
|
+
currentStep,
|
|
6107
6245
|
currentWorkflowCtx,
|
|
6246
|
+
debug,
|
|
6108
6247
|
descriptorHash,
|
|
6248
|
+
error,
|
|
6109
6249
|
eventMatchOf,
|
|
6110
6250
|
eventMatches,
|
|
6111
6251
|
eventNameOf,
|
|
@@ -6113,9 +6253,11 @@ export {
|
|
|
6113
6253
|
eventToken,
|
|
6114
6254
|
heartbeatStatus,
|
|
6115
6255
|
indexWaitersByRun,
|
|
6256
|
+
info,
|
|
6116
6257
|
isEventToken,
|
|
6117
6258
|
isLegacyDescriptor,
|
|
6118
6259
|
isWorkflowControlFlowSignal,
|
|
6260
|
+
log,
|
|
6119
6261
|
matchesAttributes,
|
|
6120
6262
|
missingCapabilities,
|
|
6121
6263
|
negotiate,
|
|
@@ -6128,6 +6270,7 @@ export {
|
|
|
6128
6270
|
requiredCapabilities,
|
|
6129
6271
|
resolveRouting,
|
|
6130
6272
|
resolveRunWaiting,
|
|
6273
|
+
runInStepLogger,
|
|
6131
6274
|
runInWorkflowCtx,
|
|
6132
6275
|
runSchedules,
|
|
6133
6276
|
runStepHandler,
|
|
@@ -6137,7 +6280,11 @@ export {
|
|
|
6137
6280
|
stepConfigOf,
|
|
6138
6281
|
stepId,
|
|
6139
6282
|
stepNameOf,
|
|
6283
|
+
sub,
|
|
6284
|
+
subEvent,
|
|
6285
|
+
subProcess,
|
|
6140
6286
|
tenantGroup,
|
|
6287
|
+
warn,
|
|
6141
6288
|
workflowName
|
|
6142
6289
|
};
|
|
6143
6290
|
//# sourceMappingURL=index.js.map
|