@dudousxd/nestjs-durable-core 0.76.0 → 0.77.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +324 -17
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +110 -8
- package/dist/index.d.ts +110 -8
- package/dist/index.js +324 -17
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1087,6 +1087,20 @@ function allGroup(firstSeq, mode) {
|
|
|
1087
1087
|
return mode === "failFast" ? `all:${firstSeq}:failFast` : `all:${firstSeq}`;
|
|
1088
1088
|
}
|
|
1089
1089
|
__name(allGroup, "allGroup");
|
|
1090
|
+
function stepFanGroup(firstSeq, size, mode) {
|
|
1091
|
+
const group = `steps:${firstSeq}:${size}`;
|
|
1092
|
+
return mode === "failFast" ? `${group}:failFast` : group;
|
|
1093
|
+
}
|
|
1094
|
+
__name(stepFanGroup, "stepFanGroup");
|
|
1095
|
+
function parseStepFanGroup(group) {
|
|
1096
|
+
const m = group ? /^steps:(\d+):(\d+)(?::failFast)?$/.exec(group) : null;
|
|
1097
|
+
if (!m) return void 0;
|
|
1098
|
+
return {
|
|
1099
|
+
firstSeq: Number(m[1]),
|
|
1100
|
+
size: Number(m[2])
|
|
1101
|
+
};
|
|
1102
|
+
}
|
|
1103
|
+
__name(parseStepFanGroup, "parseStepFanGroup");
|
|
1090
1104
|
function allWait(runId, firstSeq, wave) {
|
|
1091
1105
|
const base = allBase(runId, firstSeq, wave);
|
|
1092
1106
|
return {
|
|
@@ -3128,6 +3142,9 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3128
3142
|
}, "child");
|
|
3129
3143
|
const all = /* @__PURE__ */ __name(async (workflow2, inputs, opts) => {
|
|
3130
3144
|
if (inputs.length === 0) return [];
|
|
3145
|
+
if (stepNameOf(workflow2) !== void 0) {
|
|
3146
|
+
return allSteps(workflow2, inputs, opts);
|
|
3147
|
+
}
|
|
3131
3148
|
const mode = opts?.mode ?? "waitAll";
|
|
3132
3149
|
const name = workflowName(workflow2);
|
|
3133
3150
|
const waves = allWaves(inputs.length, opts?.concurrency);
|
|
@@ -3267,6 +3284,26 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3267
3284
|
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
3268
3285
|
return outputs;
|
|
3269
3286
|
}, "all");
|
|
3287
|
+
const allSteps = /* @__PURE__ */ __name(async (ref, inputs, opts) => {
|
|
3288
|
+
if (opts?.queue !== void 0) {
|
|
3289
|
+
throw new Error("ctx.all: a flow-control queue is not supported on a step fan");
|
|
3290
|
+
}
|
|
3291
|
+
const name = stepNameOf(ref);
|
|
3292
|
+
const def = {
|
|
3293
|
+
name,
|
|
3294
|
+
...resolveStepPolicy(stepConfigOf(ref), opts)
|
|
3295
|
+
};
|
|
3296
|
+
const positions = inputs.map(() => pos.next());
|
|
3297
|
+
return host.callRemoteAll(runId, positions, def, inputs, {
|
|
3298
|
+
mode: opts?.mode ?? "waitAll",
|
|
3299
|
+
concurrency: opts?.concurrency,
|
|
3300
|
+
transport: opts?.transport,
|
|
3301
|
+
admission: {
|
|
3302
|
+
priority: opts?.priority,
|
|
3303
|
+
fairnessKey: opts?.fairnessKey
|
|
3304
|
+
}
|
|
3305
|
+
});
|
|
3306
|
+
}, "allSteps");
|
|
3270
3307
|
const readFan = /* @__PURE__ */ __name(async (positions, prefixes) => {
|
|
3271
3308
|
const checkpoints = positions.map((seq) => replay?.get(seq) ?? null);
|
|
3272
3309
|
const folded = positions.map(() => false);
|
|
@@ -3603,6 +3640,18 @@ var BREAKPOINT = "breakpoint";
|
|
|
3603
3640
|
var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
|
|
3604
3641
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
3605
3642
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
3643
|
+
var STEP_FAN_CHUNK = 200;
|
|
3644
|
+
function shuffle(items) {
|
|
3645
|
+
for (let i = items.length - 1; i > 0; i -= 1) {
|
|
3646
|
+
const j = Math.floor(Math.random() * (i + 1));
|
|
3647
|
+
[items[i], items[j]] = [
|
|
3648
|
+
items[j],
|
|
3649
|
+
items[i]
|
|
3650
|
+
];
|
|
3651
|
+
}
|
|
3652
|
+
return items;
|
|
3653
|
+
}
|
|
3654
|
+
__name(shuffle, "shuffle");
|
|
3606
3655
|
function stepGroup(step, runNamespace) {
|
|
3607
3656
|
return tenantGroup(sanitizeQueueToken(step.name), step.partition ?? runNamespace);
|
|
3608
3657
|
}
|
|
@@ -3644,6 +3693,7 @@ var WorkflowEngine = class {
|
|
|
3644
3693
|
maxRecoveryAttempts;
|
|
3645
3694
|
remoteAdvanceSilenceMs;
|
|
3646
3695
|
reconcileMs;
|
|
3696
|
+
timerBatchSize;
|
|
3647
3697
|
eventBufferTtlMs;
|
|
3648
3698
|
remoteRedispatchMs;
|
|
3649
3699
|
remoteRedispatchMax;
|
|
@@ -3704,6 +3754,21 @@ var WorkflowEngine = class {
|
|
|
3704
3754
|
* {@link redriveSettledSteps}).
|
|
3705
3755
|
*/
|
|
3706
3756
|
awaitedRemoteSteps = /* @__PURE__ */ new Map();
|
|
3757
|
+
/**
|
|
3758
|
+
* When THIS instance settled a remote step from its result, by `runId:seq`. That result's resume
|
|
3759
|
+
* runs here, and `execute` lets it into a lease this instance holds, so its wake is never the
|
|
3760
|
+
* dropped one {@link redriveSettledSteps} recovers. Pruned by age (see {@link noteSettledHere}).
|
|
3761
|
+
*/
|
|
3762
|
+
settledHere = /* @__PURE__ */ new Map();
|
|
3763
|
+
/**
|
|
3764
|
+
* The step fans (`ctx.all` over a `@Step`) a turn suspended on, by run, while that turn executes on
|
|
3765
|
+
* THIS instance — re-checked once its lease is released (see {@link redriveSettledStepFans}).
|
|
3766
|
+
*/
|
|
3767
|
+
awaitedStepFans = /* @__PURE__ */ new Map();
|
|
3768
|
+
/** Step fans whose join barrier this instance opened, by `runId:firstSeq` — see {@link settledHere}. */
|
|
3769
|
+
stepFansFiredHere = /* @__PURE__ */ new Map();
|
|
3770
|
+
/** The tail of each run's queue of results being handled on this instance — see {@link serializedPerRun}. */
|
|
3771
|
+
resultChains = /* @__PURE__ */ new Map();
|
|
3707
3772
|
/** Flow-control admission backend for remote steps (see {@link registerQueue}). */
|
|
3708
3773
|
admission;
|
|
3709
3774
|
/** Runs on THIS instance blocked on admission, by queue — woken early on a freed-slot signal. */
|
|
@@ -3735,6 +3800,8 @@ var WorkflowEngine = class {
|
|
|
3735
3800
|
this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
|
|
3736
3801
|
this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
|
|
3737
3802
|
this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
|
|
3803
|
+
const batch = deps.timerBatchSize;
|
|
3804
|
+
this.timerBatchSize = batch !== void 0 && Number.isFinite(batch) && batch >= 1 ? Math.floor(batch) : 500;
|
|
3738
3805
|
this.eventBufferTtlMs = deps.eventBufferTtlMs || void 0;
|
|
3739
3806
|
this.remoteRedispatchMs = deps.remoteRedispatchMs || void 0;
|
|
3740
3807
|
this.remoteRedispatchMax = Math.max(1, deps.remoteRedispatchMax ?? 10);
|
|
@@ -3787,7 +3854,11 @@ var WorkflowEngine = class {
|
|
|
3787
3854
|
}
|
|
3788
3855
|
return;
|
|
3789
3856
|
}
|
|
3790
|
-
await this.completeRemoteResult(result);
|
|
3857
|
+
if (!await this.completeRemoteResult(result)) return;
|
|
3858
|
+
await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId).then(() => void 0, (err) => {
|
|
3859
|
+
if (err instanceof NamespaceMismatch) return;
|
|
3860
|
+
throw err;
|
|
3861
|
+
}));
|
|
3791
3862
|
},
|
|
3792
3863
|
// A heartbeat resets the liveness window for whatever it targets: an in-flight long STEP (keyed
|
|
3793
3864
|
// by stepId, see callRemote) or — when stepId is absent — the in-flight workflow TURN (keyed by
|
|
@@ -4647,7 +4718,7 @@ var WorkflowEngine = class {
|
|
|
4647
4718
|
* scan, filtered to just this tick's batch, keeps it cheap regardless of store size.
|
|
4648
4719
|
*/
|
|
4649
4720
|
async resumeDueTimers(nowMs = this.clock()) {
|
|
4650
|
-
const due = await this.store.listDueTimers(nowMs, this.namespace);
|
|
4721
|
+
const due = shuffle(await this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize));
|
|
4651
4722
|
const blocked = await this.dueBlockedRuns(nowMs);
|
|
4652
4723
|
if (due.length === 0 && blocked.length === 0) return [];
|
|
4653
4724
|
const waiterByRun = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
|
|
@@ -5131,11 +5202,47 @@ var WorkflowEngine = class {
|
|
|
5131
5202
|
for (const seq of seqs) {
|
|
5132
5203
|
const cp = await this.store.getCheckpoint(runId, seq);
|
|
5133
5204
|
if (cp && (cp.status === "completed" || cp.status === "failed")) {
|
|
5205
|
+
if (this.settledHere.has(`${runId}:${seq}`)) return;
|
|
5206
|
+
const nowMs = this.clock();
|
|
5207
|
+
if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
5208
|
+
return;
|
|
5209
|
+
}
|
|
5210
|
+
const run = await this.store.getRun(runId);
|
|
5211
|
+
if (run?.status !== "suspended") {
|
|
5212
|
+
await this.store.releaseRunLock(runId);
|
|
5213
|
+
return;
|
|
5214
|
+
}
|
|
5134
5215
|
await this.resumeImplicit(runId);
|
|
5135
5216
|
return;
|
|
5136
5217
|
}
|
|
5137
5218
|
}
|
|
5138
5219
|
}
|
|
5220
|
+
/**
|
|
5221
|
+
* {@link redriveSettledSteps} for a step fan: once the turn that suspended on it released its lease,
|
|
5222
|
+
* re-drive the run if no step of the fan is still `pending` — the last result landed on another
|
|
5223
|
+
* instance while this one held the lease, and its wake was dropped there. A fan whose barrier this
|
|
5224
|
+
* instance opened is skipped, for the reason {@link settledHere} gives.
|
|
5225
|
+
*/
|
|
5226
|
+
async redriveSettledStepFans(runId, fans) {
|
|
5227
|
+
for (const fan of fans) {
|
|
5228
|
+
if (this.stepFansFiredHere.has(`${runId}:${fan.from}`)) continue;
|
|
5229
|
+
const pending = this.store.countCheckpoints ? await this.store.countCheckpoints(runId, fan.from, fan.to, [
|
|
5230
|
+
"pending"
|
|
5231
|
+
]) : (await this.store.listCheckpoints(runId)).filter((c) => c.seq >= fan.from && c.seq <= fan.to && c.status === "pending").length;
|
|
5232
|
+
if (pending > 0) continue;
|
|
5233
|
+
const nowMs = this.clock();
|
|
5234
|
+
if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
5235
|
+
return;
|
|
5236
|
+
}
|
|
5237
|
+
const run = await this.store.getRun(runId);
|
|
5238
|
+
if (run?.status !== "suspended") {
|
|
5239
|
+
await this.store.releaseRunLock(runId);
|
|
5240
|
+
return;
|
|
5241
|
+
}
|
|
5242
|
+
await this.resumeImplicit(runId);
|
|
5243
|
+
return;
|
|
5244
|
+
}
|
|
5245
|
+
}
|
|
5139
5246
|
async redriveSettledChildren(runId, fan) {
|
|
5140
5247
|
if (!this.store.hasSignalWaiter) return;
|
|
5141
5248
|
for (const wait of fan.waits) {
|
|
@@ -6593,6 +6700,8 @@ var WorkflowEngine = class {
|
|
|
6593
6700
|
let suspendedOnSteps = false;
|
|
6594
6701
|
const awaitedSteps = /* @__PURE__ */ new Set();
|
|
6595
6702
|
this.awaitedRemoteSteps.set(run.id, awaitedSteps);
|
|
6703
|
+
const awaitedFans = [];
|
|
6704
|
+
this.awaitedStepFans.set(run.id, awaitedFans);
|
|
6596
6705
|
const ctx = createWorkflowCtx(this.ctxHostFor(replay, run.namespace), run.id, compensations, run.workflow, registered?.searchAttributesSchema);
|
|
6597
6706
|
try {
|
|
6598
6707
|
const output = await runInWorkflowCtx(ctx, () => fn(ctx, run.input));
|
|
@@ -6672,6 +6781,7 @@ var WorkflowEngine = class {
|
|
|
6672
6781
|
if (this.awaitedRemoteSteps.get(run.id) === awaitedSteps) {
|
|
6673
6782
|
this.awaitedRemoteSteps.delete(run.id);
|
|
6674
6783
|
}
|
|
6784
|
+
if (this.awaitedStepFans.get(run.id) === awaitedFans) this.awaitedStepFans.delete(run.id);
|
|
6675
6785
|
await this.store.releaseRunLock(run.id);
|
|
6676
6786
|
if (suspendedOnFan) {
|
|
6677
6787
|
void this.redriveSettledChildren(run.id, suspendedOnFan).catch(() => void 0);
|
|
@@ -6679,6 +6789,9 @@ var WorkflowEngine = class {
|
|
|
6679
6789
|
if (suspendedOnSteps && awaitedSteps.size > 0) {
|
|
6680
6790
|
void this.redriveSettledSteps(run.id, awaitedSteps).catch(() => void 0);
|
|
6681
6791
|
}
|
|
6792
|
+
if (suspendedOnSteps && awaitedFans.length > 0) {
|
|
6793
|
+
void this.redriveSettledStepFans(run.id, awaitedFans).catch(() => void 0);
|
|
6794
|
+
}
|
|
6682
6795
|
}
|
|
6683
6796
|
}
|
|
6684
6797
|
/**
|
|
@@ -6901,6 +7014,9 @@ var WorkflowEngine = class {
|
|
|
6901
7014
|
callRemote: /* @__PURE__ */ __name((runId, seq, step, input, queue, transport, admission) => this.callRemote(runId, seq, step, input, queue, transport, replay, admission, {
|
|
6902
7015
|
namespace: parentNamespace
|
|
6903
7016
|
}), "callRemote"),
|
|
7017
|
+
callRemoteAll: /* @__PURE__ */ __name((runId, positions, step, inputs, opts) => this.callRemoteAll(runId, positions, step, inputs, opts, {
|
|
7018
|
+
namespace: parentNamespace
|
|
7019
|
+
}), "callRemoteAll"),
|
|
6904
7020
|
// Defer so a fast child can't reentrantly resume a still-running parent. Stamp the child with
|
|
6905
7021
|
// the PARENT run's namespace (not this engine's) so a child belongs to the tenant/partition of
|
|
6906
7022
|
// the run it was spawned from — even when an operator (namespace: undefined, drives every
|
|
@@ -7202,6 +7318,9 @@ var WorkflowEngine = class {
|
|
|
7202
7318
|
input,
|
|
7203
7319
|
attempts: attempt,
|
|
7204
7320
|
workerGroup: group,
|
|
7321
|
+
...routing?.parallelGroup !== void 0 ? {
|
|
7322
|
+
parallelGroup: routing.parallelGroup
|
|
7323
|
+
} : {},
|
|
7205
7324
|
enqueuedAt,
|
|
7206
7325
|
startedAt: enqueuedAt,
|
|
7207
7326
|
finishedAt: enqueuedAt
|
|
@@ -7227,6 +7346,149 @@ var WorkflowEngine = class {
|
|
|
7227
7346
|
this.awaitedRemoteSteps.get(runId)?.add(seq);
|
|
7228
7347
|
throw new WorkflowSuspended();
|
|
7229
7348
|
}
|
|
7349
|
+
/**
|
|
7350
|
+
* `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
|
|
7351
|
+
* reserves the whole position block up front, so replay re-derives the same seqs.
|
|
7352
|
+
*
|
|
7353
|
+
* Each turn reads the fan's checkpoints fresh, hands every step that already has one to
|
|
7354
|
+
* {@link callRemote} (which owns the completed / pending-lease / failed-retry rules), and sends
|
|
7355
|
+
* the steps nobody has sent yet: all of them, or with `concurrency`, the next wave once the last
|
|
7356
|
+
* one has drained. New steps are written `pending` BEFORE any is dispatched, so a turn that runs
|
|
7357
|
+
* concurrently with this one finds them already sent instead of sending them again. The join
|
|
7358
|
+
* barrier ({@link stepFanStillRunning}) then wakes the parent once per wave, not once per result.
|
|
7359
|
+
*/
|
|
7360
|
+
async callRemoteAll(runId, positions, step, inputs, opts, routing) {
|
|
7361
|
+
if (step.timeoutMs) {
|
|
7362
|
+
throw new Error(`ctx.all: step "${step.name}" has a liveness timeoutMs; a step fan needs durable steps`);
|
|
7363
|
+
}
|
|
7364
|
+
if (opts.concurrency !== void 0 && (!Number.isInteger(opts.concurrency) || opts.concurrency < 1)) {
|
|
7365
|
+
throw new Error(`ctx.all: concurrency must be a positive integer, got ${opts.concurrency}`);
|
|
7366
|
+
}
|
|
7367
|
+
if (this.pool.size === 0) throw new Error("remote steps require a Transport");
|
|
7368
|
+
const first = positions[0];
|
|
7369
|
+
const last = positions[positions.length - 1];
|
|
7370
|
+
const group = stepFanGroup(first, positions.length, opts.mode);
|
|
7371
|
+
const bySeq = /* @__PURE__ */ new Map();
|
|
7372
|
+
for (const cp of await this.store.listCheckpoints(runId)) {
|
|
7373
|
+
if (cp.seq >= first && cp.seq <= last) bySeq.set(cp.seq, cp);
|
|
7374
|
+
}
|
|
7375
|
+
const outputs = new Array(inputs.length);
|
|
7376
|
+
const failures = [];
|
|
7377
|
+
const unsent = [];
|
|
7378
|
+
let inFlight = 0;
|
|
7379
|
+
let wakeAt;
|
|
7380
|
+
for (let i = 0; i < inputs.length; i += 1) {
|
|
7381
|
+
const seq = positions[i];
|
|
7382
|
+
const cp = bySeq.get(seq);
|
|
7383
|
+
if (!cp) {
|
|
7384
|
+
unsent.push(i);
|
|
7385
|
+
continue;
|
|
7386
|
+
}
|
|
7387
|
+
if (cp.name !== step.name) throw new NonDeterminismError(runId, seq, step.name, cp.name);
|
|
7388
|
+
if (cp.status === "completed") {
|
|
7389
|
+
outputs[i] = cp.output;
|
|
7390
|
+
continue;
|
|
7391
|
+
}
|
|
7392
|
+
try {
|
|
7393
|
+
outputs[i] = await this.callRemote(runId, seq, step, inputs[i], void 0, opts.transport, void 0, opts.admission, {
|
|
7394
|
+
...routing,
|
|
7395
|
+
parallelGroup: group
|
|
7396
|
+
});
|
|
7397
|
+
} catch (err) {
|
|
7398
|
+
if (err instanceof WorkflowSuspended) {
|
|
7399
|
+
inFlight += 1;
|
|
7400
|
+
if (err.wakeAt !== void 0) wakeAt = Math.min(wakeAt ?? err.wakeAt, err.wakeAt);
|
|
7401
|
+
} else if (err instanceof RemoteStepError) {
|
|
7402
|
+
const failure = {
|
|
7403
|
+
index: i,
|
|
7404
|
+
id: stepId(runId, seq),
|
|
7405
|
+
error: err.message
|
|
7406
|
+
};
|
|
7407
|
+
if (opts.mode === "failFast") throw new GatherError([
|
|
7408
|
+
failure
|
|
7409
|
+
], inputs.length);
|
|
7410
|
+
failures.push(failure);
|
|
7411
|
+
} else {
|
|
7412
|
+
throw err;
|
|
7413
|
+
}
|
|
7414
|
+
} finally {
|
|
7415
|
+
this.awaitedRemoteSteps.get(runId)?.delete(seq);
|
|
7416
|
+
}
|
|
7417
|
+
}
|
|
7418
|
+
const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
|
|
7419
|
+
if (batch.length > 0) await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
|
|
7420
|
+
if (inFlight + batch.length > 0) {
|
|
7421
|
+
this.awaitedStepFans.get(runId)?.push({
|
|
7422
|
+
from: first,
|
|
7423
|
+
to: last
|
|
7424
|
+
});
|
|
7425
|
+
throw new WorkflowSuspended(wakeAt);
|
|
7426
|
+
}
|
|
7427
|
+
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
7428
|
+
return outputs;
|
|
7429
|
+
}
|
|
7430
|
+
/** Write the `pending` checkpoints of a step fan's new steps, then dispatch them. */
|
|
7431
|
+
async sendStepFan(runId, positions, step, inputs, batch, group, opts, routing) {
|
|
7432
|
+
const workerGroup = stepGroup(step, routing?.namespace);
|
|
7433
|
+
await this.ensureRoutable(workerGroup, step.requires);
|
|
7434
|
+
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
7435
|
+
for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
|
|
7436
|
+
await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map((i) => {
|
|
7437
|
+
const seq = positions[i];
|
|
7438
|
+
return this.store.saveCheckpoint({
|
|
7439
|
+
runId,
|
|
7440
|
+
seq,
|
|
7441
|
+
name: step.name,
|
|
7442
|
+
kind: "remote",
|
|
7443
|
+
stepId: stepId(runId, seq),
|
|
7444
|
+
status: "pending",
|
|
7445
|
+
input: inputs[i],
|
|
7446
|
+
attempts: 1,
|
|
7447
|
+
workerGroup,
|
|
7448
|
+
parallelGroup: group,
|
|
7449
|
+
enqueuedAt,
|
|
7450
|
+
startedAt: enqueuedAt,
|
|
7451
|
+
finishedAt: enqueuedAt
|
|
7452
|
+
});
|
|
7453
|
+
}));
|
|
7454
|
+
}
|
|
7455
|
+
for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
|
|
7456
|
+
await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map(async (i) => {
|
|
7457
|
+
const seq = positions[i];
|
|
7458
|
+
await this.dispatchRemoteTask({
|
|
7459
|
+
runId,
|
|
7460
|
+
seq,
|
|
7461
|
+
name: step.name,
|
|
7462
|
+
stepId: stepId(runId, seq),
|
|
7463
|
+
group: workerGroup,
|
|
7464
|
+
input: inputs[i],
|
|
7465
|
+
priority: opts.admission?.priority,
|
|
7466
|
+
attempt: 1,
|
|
7467
|
+
transport: opts.transport
|
|
7468
|
+
});
|
|
7469
|
+
this.emit({
|
|
7470
|
+
type: "step.started",
|
|
7471
|
+
runId,
|
|
7472
|
+
seq,
|
|
7473
|
+
name: step.name,
|
|
7474
|
+
kind: "remote"
|
|
7475
|
+
});
|
|
7476
|
+
}));
|
|
7477
|
+
}
|
|
7478
|
+
}
|
|
7479
|
+
/** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
|
|
7480
|
+
noteSettledHere(key) {
|
|
7481
|
+
this.noteRecent(this.settledHere, key);
|
|
7482
|
+
}
|
|
7483
|
+
/** Record `key` in a "happened here recently" map, pruning entries older than a lease. */
|
|
7484
|
+
noteRecent(map, key) {
|
|
7485
|
+
const now = this.clock();
|
|
7486
|
+
map.set(key, now);
|
|
7487
|
+
if (map.size < 1024) return;
|
|
7488
|
+
for (const [k, at] of map) {
|
|
7489
|
+
if (now - at > this.leaseMs) map.delete(k);
|
|
7490
|
+
}
|
|
7491
|
+
}
|
|
7230
7492
|
/** {@link StateStore.saveCheckpointIfPending}, or a plain save on a store without it. */
|
|
7231
7493
|
async savePendingCheckpoint(checkpoint) {
|
|
7232
7494
|
if (this.store.saveCheckpointIfPending) return this.store.saveCheckpointIfPending(checkpoint);
|
|
@@ -7251,16 +7513,33 @@ var WorkflowEngine = class {
|
|
|
7251
7513
|
}, task.transport);
|
|
7252
7514
|
}
|
|
7253
7515
|
/**
|
|
7254
|
-
*
|
|
7255
|
-
*
|
|
7256
|
-
*
|
|
7516
|
+
* Run `work` after every earlier `work` for the same run on this instance has settled. A result
|
|
7517
|
+
* resumes its run inline, and `execute` lets this instance into a lease it already holds — so two
|
|
7518
|
+
* results of one run handled at once here would run two turns of it concurrently. Results of
|
|
7519
|
+
* different runs still go in parallel, which is what a results consumer with concurrency > 1 is for.
|
|
7520
|
+
*/
|
|
7521
|
+
serializedPerRun(runId, work) {
|
|
7522
|
+
const previous = this.resultChains.get(runId) ?? Promise.resolve();
|
|
7523
|
+
const next = previous.then(work, work);
|
|
7524
|
+
const tail = next.catch(() => void 0);
|
|
7525
|
+
this.resultChains.set(runId, tail);
|
|
7526
|
+
void tail.then(() => {
|
|
7527
|
+
if (this.resultChains.get(runId) === tail) this.resultChains.delete(runId);
|
|
7528
|
+
});
|
|
7529
|
+
return next;
|
|
7530
|
+
}
|
|
7531
|
+
/**
|
|
7532
|
+
* Complete a durable remote step from its worker result — runs on whichever instance receives the
|
|
7533
|
+
* result (the dispatching one may be gone), so the run is crash/scale-safe. Resolves whether the
|
|
7534
|
+
* run should now be resumed: not for a duplicate or late delivery (the checkpoint isn't `pending`),
|
|
7535
|
+
* a run already finished, or a step fan whose barrier is still closed.
|
|
7257
7536
|
*/
|
|
7258
7537
|
async completeRemoteResult(result) {
|
|
7259
7538
|
const cp = await this.store.getCheckpoint(result.runId, result.seq);
|
|
7260
|
-
if (!cp || cp.status !== "pending") return;
|
|
7539
|
+
if (!cp || cp.status !== "pending") return false;
|
|
7261
7540
|
void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
|
|
7262
7541
|
const run = await this.store.getRun(result.runId);
|
|
7263
|
-
if (run && (run.status === "cancelled" || run.status === "completed")) return;
|
|
7542
|
+
if (run && (run.status === "cancelled" || run.status === "completed")) return false;
|
|
7264
7543
|
const finishedAt = /* @__PURE__ */ new Date();
|
|
7265
7544
|
const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
|
|
7266
7545
|
const claimed = await this.savePendingCheckpoint({
|
|
@@ -7272,7 +7551,8 @@ var WorkflowEngine = class {
|
|
|
7272
7551
|
startedAt,
|
|
7273
7552
|
finishedAt
|
|
7274
7553
|
});
|
|
7275
|
-
if (!claimed) return;
|
|
7554
|
+
if (!claimed) return false;
|
|
7555
|
+
this.noteSettledHere(`${result.runId}:${result.seq}`);
|
|
7276
7556
|
this.emit({
|
|
7277
7557
|
type: result.status === "completed" ? "step.completed" : "step.failed",
|
|
7278
7558
|
runId: result.runId,
|
|
@@ -7284,10 +7564,25 @@ var WorkflowEngine = class {
|
|
|
7284
7564
|
queueMs: startedAt.getTime() - cp.enqueuedAt.getTime(),
|
|
7285
7565
|
durationMs: finishedAt.getTime() - startedAt.getTime()
|
|
7286
7566
|
});
|
|
7287
|
-
await this.
|
|
7288
|
-
|
|
7289
|
-
|
|
7290
|
-
|
|
7567
|
+
return !await this.stepFanStillRunning(cp, result);
|
|
7568
|
+
}
|
|
7569
|
+
/**
|
|
7570
|
+
* The join barrier of a step fan: a result for one of its steps resumes the parent only once no
|
|
7571
|
+
* step of the fan is still `pending` (a settled wave), or on a failure (failFast surfaces it, and a
|
|
7572
|
+
* retry needs the parent to schedule it). Without it every result would replay the whole parent,
|
|
7573
|
+
* which makes a fan of N steps cost O(N²) checkpoint reads.
|
|
7574
|
+
*/
|
|
7575
|
+
async stepFanStillRunning(cp, result) {
|
|
7576
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
7577
|
+
if (!fan) return false;
|
|
7578
|
+
if (result.status === "failed") return false;
|
|
7579
|
+
const to = fan.firstSeq + fan.size - 1;
|
|
7580
|
+
const pending = this.store.countCheckpoints ? await this.store.countCheckpoints(result.runId, fan.firstSeq, to, [
|
|
7581
|
+
"pending"
|
|
7582
|
+
]) : (await this.store.listCheckpoints(result.runId)).filter((c) => c.seq >= fan.firstSeq && c.seq <= to && c.status === "pending").length;
|
|
7583
|
+
if (pending > 0) return true;
|
|
7584
|
+
this.noteRecent(this.stepFansFiredHere, `${result.runId}:${fan.firstSeq}`);
|
|
7585
|
+
return false;
|
|
7291
7586
|
}
|
|
7292
7587
|
/**
|
|
7293
7588
|
* Apply a remote workflow-turn {@link WorkflowDecision} durably — runs on WHATEVER instance receives
|
|
@@ -7668,6 +7963,8 @@ var CodecStateStore = class {
|
|
|
7668
7963
|
/** Encodes like {@link saveCheckpoint}. Bound only when the inner store has it, so the engine's
|
|
7669
7964
|
* plain-save fallback still kicks in through the wrapper. */
|
|
7670
7965
|
saveCheckpointIfPending;
|
|
7966
|
+
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
|
|
7967
|
+
countCheckpoints;
|
|
7671
7968
|
/** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
|
|
7672
7969
|
* when the inner store prunes, so the retention poller still sees (and warns about) its absence. */
|
|
7673
7970
|
pruneTerminalRuns;
|
|
@@ -7697,6 +7994,8 @@ var CodecStateStore = class {
|
|
|
7697
7994
|
if (deleteRuns) this.deleteRuns = (ids) => deleteRuns.call(inner, ids);
|
|
7698
7995
|
const saveIfPending = inner.saveCheckpointIfPending;
|
|
7699
7996
|
if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
|
|
7997
|
+
const countCps = inner.countCheckpoints;
|
|
7998
|
+
if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
|
|
7700
7999
|
const prune = inner.pruneTerminalRuns;
|
|
7701
8000
|
if (prune) this.pruneTerminalRuns = (policy, now, limit) => prune.call(inner, policy, now, limit);
|
|
7702
8001
|
const count = inner.countRuns;
|
|
@@ -7827,8 +8126,8 @@ var CodecStateStore = class {
|
|
|
7827
8126
|
async listPendingRuns(limit, namespace) {
|
|
7828
8127
|
return (await this.inner.listPendingRuns(limit, namespace)).map((r) => this.decRun(r));
|
|
7829
8128
|
}
|
|
7830
|
-
async listDueTimers(nowMs, namespace) {
|
|
7831
|
-
return (await this.inner.listDueTimers(nowMs, namespace)).map((r) => this.decRun(r));
|
|
8129
|
+
async listDueTimers(nowMs, namespace, limit) {
|
|
8130
|
+
return (await this.inner.listDueTimers(nowMs, namespace, limit)).map((r) => this.decRun(r));
|
|
7832
8131
|
}
|
|
7833
8132
|
tryLockRun(runId, owner, leaseUntilMs, nowMs) {
|
|
7834
8133
|
return this.inner.tryLockRun(runId, owner, leaseUntilMs, nowMs);
|
|
@@ -8282,10 +8581,11 @@ var InMemoryStateStore = class {
|
|
|
8282
8581
|
...r
|
|
8283
8582
|
}));
|
|
8284
8583
|
}
|
|
8285
|
-
async listDueTimers(nowMs, namespace) {
|
|
8286
|
-
|
|
8584
|
+
async listDueTimers(nowMs, namespace, limit) {
|
|
8585
|
+
const due = [
|
|
8287
8586
|
...this.runs.values()
|
|
8288
|
-
].filter((r) => r.status === "suspended" && r.wakeAt !== void 0 && r.wakeAt <= nowMs && (namespace === void 0 || r.namespace === namespace)).
|
|
8587
|
+
].filter((r) => r.status === "suspended" && r.wakeAt !== void 0 && r.wakeAt <= nowMs && (namespace === void 0 || r.namespace === namespace)).sort((a, b) => (a.wakeAt ?? 0) - (b.wakeAt ?? 0));
|
|
8588
|
+
return (limit === void 0 ? due : due.slice(0, limit)).map((r) => ({
|
|
8289
8589
|
...r
|
|
8290
8590
|
}));
|
|
8291
8591
|
}
|
|
@@ -8559,6 +8859,13 @@ var InMemoryStateStore = class {
|
|
|
8559
8859
|
...latest
|
|
8560
8860
|
} : void 0;
|
|
8561
8861
|
}
|
|
8862
|
+
async countCheckpoints(runId, fromSeq, toSeq, statuses) {
|
|
8863
|
+
let n = 0;
|
|
8864
|
+
for (const cp of this.checkpoints.values()) {
|
|
8865
|
+
if (cp.runId === runId && cp.seq >= fromSeq && cp.seq <= toSeq && statuses.includes(cp.status)) n += 1;
|
|
8866
|
+
}
|
|
8867
|
+
return n;
|
|
8868
|
+
}
|
|
8562
8869
|
async listCheckpointsByNamePrefix(runId, prefixes) {
|
|
8563
8870
|
const rows = [
|
|
8564
8871
|
...this.checkpoints.values()
|