@dudousxd/nestjs-durable-core 0.76.0 → 0.77.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +324 -17
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +110 -8
- package/dist/index.d.ts +110 -8
- package/dist/index.js +324 -17
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -1250,6 +1250,20 @@ function allGroup(firstSeq, mode) {
|
|
|
1250
1250
|
return mode === "failFast" ? `all:${firstSeq}:failFast` : `all:${firstSeq}`;
|
|
1251
1251
|
}
|
|
1252
1252
|
__name(allGroup, "allGroup");
|
|
1253
|
+
function stepFanGroup(firstSeq, size, mode) {
|
|
1254
|
+
const group = `steps:${firstSeq}:${size}`;
|
|
1255
|
+
return mode === "failFast" ? `${group}:failFast` : group;
|
|
1256
|
+
}
|
|
1257
|
+
__name(stepFanGroup, "stepFanGroup");
|
|
1258
|
+
function parseStepFanGroup(group) {
|
|
1259
|
+
const m = group ? /^steps:(\d+):(\d+)(?::failFast)?$/.exec(group) : null;
|
|
1260
|
+
if (!m) return void 0;
|
|
1261
|
+
return {
|
|
1262
|
+
firstSeq: Number(m[1]),
|
|
1263
|
+
size: Number(m[2])
|
|
1264
|
+
};
|
|
1265
|
+
}
|
|
1266
|
+
__name(parseStepFanGroup, "parseStepFanGroup");
|
|
1253
1267
|
function allWait(runId, firstSeq, wave) {
|
|
1254
1268
|
const base = allBase(runId, firstSeq, wave);
|
|
1255
1269
|
return {
|
|
@@ -3291,6 +3305,9 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3291
3305
|
}, "child");
|
|
3292
3306
|
const all = /* @__PURE__ */ __name(async (workflow2, inputs, opts) => {
|
|
3293
3307
|
if (inputs.length === 0) return [];
|
|
3308
|
+
if (stepNameOf(workflow2) !== void 0) {
|
|
3309
|
+
return allSteps(workflow2, inputs, opts);
|
|
3310
|
+
}
|
|
3294
3311
|
const mode = opts?.mode ?? "waitAll";
|
|
3295
3312
|
const name = workflowName(workflow2);
|
|
3296
3313
|
const waves = allWaves(inputs.length, opts?.concurrency);
|
|
@@ -3430,6 +3447,26 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3430
3447
|
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
3431
3448
|
return outputs;
|
|
3432
3449
|
}, "all");
|
|
3450
|
+
const allSteps = /* @__PURE__ */ __name(async (ref, inputs, opts) => {
|
|
3451
|
+
if (opts?.queue !== void 0) {
|
|
3452
|
+
throw new Error("ctx.all: a flow-control queue is not supported on a step fan");
|
|
3453
|
+
}
|
|
3454
|
+
const name = stepNameOf(ref);
|
|
3455
|
+
const def = {
|
|
3456
|
+
name,
|
|
3457
|
+
...resolveStepPolicy(stepConfigOf(ref), opts)
|
|
3458
|
+
};
|
|
3459
|
+
const positions = inputs.map(() => pos.next());
|
|
3460
|
+
return host.callRemoteAll(runId, positions, def, inputs, {
|
|
3461
|
+
mode: opts?.mode ?? "waitAll",
|
|
3462
|
+
concurrency: opts?.concurrency,
|
|
3463
|
+
transport: opts?.transport,
|
|
3464
|
+
admission: {
|
|
3465
|
+
priority: opts?.priority,
|
|
3466
|
+
fairnessKey: opts?.fairnessKey
|
|
3467
|
+
}
|
|
3468
|
+
});
|
|
3469
|
+
}, "allSteps");
|
|
3433
3470
|
const readFan = /* @__PURE__ */ __name(async (positions, prefixes) => {
|
|
3434
3471
|
const checkpoints = positions.map((seq) => replay?.get(seq) ?? null);
|
|
3435
3472
|
const folded = positions.map(() => false);
|
|
@@ -3766,6 +3803,18 @@ var BREAKPOINT = "breakpoint";
|
|
|
3766
3803
|
var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
|
|
3767
3804
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
3768
3805
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
3806
|
+
var STEP_FAN_CHUNK = 200;
|
|
3807
|
+
function shuffle(items) {
|
|
3808
|
+
for (let i = items.length - 1; i > 0; i -= 1) {
|
|
3809
|
+
const j = Math.floor(Math.random() * (i + 1));
|
|
3810
|
+
[items[i], items[j]] = [
|
|
3811
|
+
items[j],
|
|
3812
|
+
items[i]
|
|
3813
|
+
];
|
|
3814
|
+
}
|
|
3815
|
+
return items;
|
|
3816
|
+
}
|
|
3817
|
+
__name(shuffle, "shuffle");
|
|
3769
3818
|
function stepGroup(step, runNamespace) {
|
|
3770
3819
|
return tenantGroup(sanitizeQueueToken(step.name), step.partition ?? runNamespace);
|
|
3771
3820
|
}
|
|
@@ -3807,6 +3856,7 @@ var WorkflowEngine = class {
|
|
|
3807
3856
|
maxRecoveryAttempts;
|
|
3808
3857
|
remoteAdvanceSilenceMs;
|
|
3809
3858
|
reconcileMs;
|
|
3859
|
+
timerBatchSize;
|
|
3810
3860
|
eventBufferTtlMs;
|
|
3811
3861
|
remoteRedispatchMs;
|
|
3812
3862
|
remoteRedispatchMax;
|
|
@@ -3867,6 +3917,21 @@ var WorkflowEngine = class {
|
|
|
3867
3917
|
* {@link redriveSettledSteps}).
|
|
3868
3918
|
*/
|
|
3869
3919
|
awaitedRemoteSteps = /* @__PURE__ */ new Map();
|
|
3920
|
+
/**
|
|
3921
|
+
* When THIS instance settled a remote step from its result, by `runId:seq`. That result's resume
|
|
3922
|
+
* runs here, and `execute` lets it into a lease this instance holds, so its wake is never the
|
|
3923
|
+
* dropped one {@link redriveSettledSteps} recovers. Pruned by age (see {@link noteSettledHere}).
|
|
3924
|
+
*/
|
|
3925
|
+
settledHere = /* @__PURE__ */ new Map();
|
|
3926
|
+
/**
|
|
3927
|
+
* The step fans (`ctx.all` over a `@Step`) a turn suspended on, by run, while that turn executes on
|
|
3928
|
+
* THIS instance — re-checked once its lease is released (see {@link redriveSettledStepFans}).
|
|
3929
|
+
*/
|
|
3930
|
+
awaitedStepFans = /* @__PURE__ */ new Map();
|
|
3931
|
+
/** Step fans whose join barrier this instance opened, by `runId:firstSeq` — see {@link settledHere}. */
|
|
3932
|
+
stepFansFiredHere = /* @__PURE__ */ new Map();
|
|
3933
|
+
/** The tail of each run's queue of results being handled on this instance — see {@link serializedPerRun}. */
|
|
3934
|
+
resultChains = /* @__PURE__ */ new Map();
|
|
3870
3935
|
/** Flow-control admission backend for remote steps (see {@link registerQueue}). */
|
|
3871
3936
|
admission;
|
|
3872
3937
|
/** Runs on THIS instance blocked on admission, by queue — woken early on a freed-slot signal. */
|
|
@@ -3898,6 +3963,8 @@ var WorkflowEngine = class {
|
|
|
3898
3963
|
this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
|
|
3899
3964
|
this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
|
|
3900
3965
|
this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
|
|
3966
|
+
const batch = deps.timerBatchSize;
|
|
3967
|
+
this.timerBatchSize = batch !== void 0 && Number.isFinite(batch) && batch >= 1 ? Math.floor(batch) : 500;
|
|
3901
3968
|
this.eventBufferTtlMs = deps.eventBufferTtlMs || void 0;
|
|
3902
3969
|
this.remoteRedispatchMs = deps.remoteRedispatchMs || void 0;
|
|
3903
3970
|
this.remoteRedispatchMax = Math.max(1, deps.remoteRedispatchMax ?? 10);
|
|
@@ -3950,7 +4017,11 @@ var WorkflowEngine = class {
|
|
|
3950
4017
|
}
|
|
3951
4018
|
return;
|
|
3952
4019
|
}
|
|
3953
|
-
await this.completeRemoteResult(result);
|
|
4020
|
+
if (!await this.completeRemoteResult(result)) return;
|
|
4021
|
+
await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId).then(() => void 0, (err) => {
|
|
4022
|
+
if (err instanceof NamespaceMismatch) return;
|
|
4023
|
+
throw err;
|
|
4024
|
+
}));
|
|
3954
4025
|
},
|
|
3955
4026
|
// A heartbeat resets the liveness window for whatever it targets: an in-flight long STEP (keyed
|
|
3956
4027
|
// by stepId, see callRemote) or — when stepId is absent — the in-flight workflow TURN (keyed by
|
|
@@ -4810,7 +4881,7 @@ var WorkflowEngine = class {
|
|
|
4810
4881
|
* scan, filtered to just this tick's batch, keeps it cheap regardless of store size.
|
|
4811
4882
|
*/
|
|
4812
4883
|
async resumeDueTimers(nowMs = this.clock()) {
|
|
4813
|
-
const due = await this.store.listDueTimers(nowMs, this.namespace);
|
|
4884
|
+
const due = shuffle(await this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize));
|
|
4814
4885
|
const blocked = await this.dueBlockedRuns(nowMs);
|
|
4815
4886
|
if (due.length === 0 && blocked.length === 0) return [];
|
|
4816
4887
|
const waiterByRun = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
|
|
@@ -5294,11 +5365,47 @@ var WorkflowEngine = class {
|
|
|
5294
5365
|
for (const seq of seqs) {
|
|
5295
5366
|
const cp = await this.store.getCheckpoint(runId, seq);
|
|
5296
5367
|
if (cp && (cp.status === "completed" || cp.status === "failed")) {
|
|
5368
|
+
if (this.settledHere.has(`${runId}:${seq}`)) return;
|
|
5369
|
+
const nowMs = this.clock();
|
|
5370
|
+
if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
5371
|
+
return;
|
|
5372
|
+
}
|
|
5373
|
+
const run = await this.store.getRun(runId);
|
|
5374
|
+
if (run?.status !== "suspended") {
|
|
5375
|
+
await this.store.releaseRunLock(runId);
|
|
5376
|
+
return;
|
|
5377
|
+
}
|
|
5297
5378
|
await this.resumeImplicit(runId);
|
|
5298
5379
|
return;
|
|
5299
5380
|
}
|
|
5300
5381
|
}
|
|
5301
5382
|
}
|
|
5383
|
+
/**
|
|
5384
|
+
* {@link redriveSettledSteps} for a step fan: once the turn that suspended on it released its lease,
|
|
5385
|
+
* re-drive the run if no step of the fan is still `pending` — the last result landed on another
|
|
5386
|
+
* instance while this one held the lease, and its wake was dropped there. A fan whose barrier this
|
|
5387
|
+
* instance opened is skipped, for the reason {@link settledHere} gives.
|
|
5388
|
+
*/
|
|
5389
|
+
async redriveSettledStepFans(runId, fans) {
|
|
5390
|
+
for (const fan of fans) {
|
|
5391
|
+
if (this.stepFansFiredHere.has(`${runId}:${fan.from}`)) continue;
|
|
5392
|
+
const pending = this.store.countCheckpoints ? await this.store.countCheckpoints(runId, fan.from, fan.to, [
|
|
5393
|
+
"pending"
|
|
5394
|
+
]) : (await this.store.listCheckpoints(runId)).filter((c) => c.seq >= fan.from && c.seq <= fan.to && c.status === "pending").length;
|
|
5395
|
+
if (pending > 0) continue;
|
|
5396
|
+
const nowMs = this.clock();
|
|
5397
|
+
if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
5398
|
+
return;
|
|
5399
|
+
}
|
|
5400
|
+
const run = await this.store.getRun(runId);
|
|
5401
|
+
if (run?.status !== "suspended") {
|
|
5402
|
+
await this.store.releaseRunLock(runId);
|
|
5403
|
+
return;
|
|
5404
|
+
}
|
|
5405
|
+
await this.resumeImplicit(runId);
|
|
5406
|
+
return;
|
|
5407
|
+
}
|
|
5408
|
+
}
|
|
5302
5409
|
async redriveSettledChildren(runId, fan) {
|
|
5303
5410
|
if (!this.store.hasSignalWaiter) return;
|
|
5304
5411
|
for (const wait of fan.waits) {
|
|
@@ -6756,6 +6863,8 @@ var WorkflowEngine = class {
|
|
|
6756
6863
|
let suspendedOnSteps = false;
|
|
6757
6864
|
const awaitedSteps = /* @__PURE__ */ new Set();
|
|
6758
6865
|
this.awaitedRemoteSteps.set(run.id, awaitedSteps);
|
|
6866
|
+
const awaitedFans = [];
|
|
6867
|
+
this.awaitedStepFans.set(run.id, awaitedFans);
|
|
6759
6868
|
const ctx = createWorkflowCtx(this.ctxHostFor(replay, run.namespace), run.id, compensations, run.workflow, registered?.searchAttributesSchema);
|
|
6760
6869
|
try {
|
|
6761
6870
|
const output = await runInWorkflowCtx(ctx, () => fn(ctx, run.input));
|
|
@@ -6835,6 +6944,7 @@ var WorkflowEngine = class {
|
|
|
6835
6944
|
if (this.awaitedRemoteSteps.get(run.id) === awaitedSteps) {
|
|
6836
6945
|
this.awaitedRemoteSteps.delete(run.id);
|
|
6837
6946
|
}
|
|
6947
|
+
if (this.awaitedStepFans.get(run.id) === awaitedFans) this.awaitedStepFans.delete(run.id);
|
|
6838
6948
|
await this.store.releaseRunLock(run.id);
|
|
6839
6949
|
if (suspendedOnFan) {
|
|
6840
6950
|
void this.redriveSettledChildren(run.id, suspendedOnFan).catch(() => void 0);
|
|
@@ -6842,6 +6952,9 @@ var WorkflowEngine = class {
|
|
|
6842
6952
|
if (suspendedOnSteps && awaitedSteps.size > 0) {
|
|
6843
6953
|
void this.redriveSettledSteps(run.id, awaitedSteps).catch(() => void 0);
|
|
6844
6954
|
}
|
|
6955
|
+
if (suspendedOnSteps && awaitedFans.length > 0) {
|
|
6956
|
+
void this.redriveSettledStepFans(run.id, awaitedFans).catch(() => void 0);
|
|
6957
|
+
}
|
|
6845
6958
|
}
|
|
6846
6959
|
}
|
|
6847
6960
|
/**
|
|
@@ -7064,6 +7177,9 @@ var WorkflowEngine = class {
|
|
|
7064
7177
|
callRemote: /* @__PURE__ */ __name((runId, seq, step, input, queue, transport, admission) => this.callRemote(runId, seq, step, input, queue, transport, replay, admission, {
|
|
7065
7178
|
namespace: parentNamespace
|
|
7066
7179
|
}), "callRemote"),
|
|
7180
|
+
callRemoteAll: /* @__PURE__ */ __name((runId, positions, step, inputs, opts) => this.callRemoteAll(runId, positions, step, inputs, opts, {
|
|
7181
|
+
namespace: parentNamespace
|
|
7182
|
+
}), "callRemoteAll"),
|
|
7067
7183
|
// Defer so a fast child can't reentrantly resume a still-running parent. Stamp the child with
|
|
7068
7184
|
// the PARENT run's namespace (not this engine's) so a child belongs to the tenant/partition of
|
|
7069
7185
|
// the run it was spawned from — even when an operator (namespace: undefined, drives every
|
|
@@ -7365,6 +7481,9 @@ var WorkflowEngine = class {
|
|
|
7365
7481
|
input,
|
|
7366
7482
|
attempts: attempt,
|
|
7367
7483
|
workerGroup: group,
|
|
7484
|
+
...routing?.parallelGroup !== void 0 ? {
|
|
7485
|
+
parallelGroup: routing.parallelGroup
|
|
7486
|
+
} : {},
|
|
7368
7487
|
enqueuedAt,
|
|
7369
7488
|
startedAt: enqueuedAt,
|
|
7370
7489
|
finishedAt: enqueuedAt
|
|
@@ -7390,6 +7509,149 @@ var WorkflowEngine = class {
|
|
|
7390
7509
|
this.awaitedRemoteSteps.get(runId)?.add(seq);
|
|
7391
7510
|
throw new WorkflowSuspended();
|
|
7392
7511
|
}
|
|
7512
|
+
/**
|
|
7513
|
+
* `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
|
|
7514
|
+
* reserves the whole position block up front, so replay re-derives the same seqs.
|
|
7515
|
+
*
|
|
7516
|
+
* Each turn reads the fan's checkpoints fresh, hands every step that already has one to
|
|
7517
|
+
* {@link callRemote} (which owns the completed / pending-lease / failed-retry rules), and sends
|
|
7518
|
+
* the steps nobody has sent yet: all of them, or with `concurrency`, the next wave once the last
|
|
7519
|
+
* one has drained. New steps are written `pending` BEFORE any is dispatched, so a turn that runs
|
|
7520
|
+
* concurrently with this one finds them already sent instead of sending them again. The join
|
|
7521
|
+
* barrier ({@link stepFanStillRunning}) then wakes the parent once per wave, not once per result.
|
|
7522
|
+
*/
|
|
7523
|
+
async callRemoteAll(runId, positions, step, inputs, opts, routing) {
|
|
7524
|
+
if (step.timeoutMs) {
|
|
7525
|
+
throw new Error(`ctx.all: step "${step.name}" has a liveness timeoutMs; a step fan needs durable steps`);
|
|
7526
|
+
}
|
|
7527
|
+
if (opts.concurrency !== void 0 && (!Number.isInteger(opts.concurrency) || opts.concurrency < 1)) {
|
|
7528
|
+
throw new Error(`ctx.all: concurrency must be a positive integer, got ${opts.concurrency}`);
|
|
7529
|
+
}
|
|
7530
|
+
if (this.pool.size === 0) throw new Error("remote steps require a Transport");
|
|
7531
|
+
const first = positions[0];
|
|
7532
|
+
const last = positions[positions.length - 1];
|
|
7533
|
+
const group = stepFanGroup(first, positions.length, opts.mode);
|
|
7534
|
+
const bySeq = /* @__PURE__ */ new Map();
|
|
7535
|
+
for (const cp of await this.store.listCheckpoints(runId)) {
|
|
7536
|
+
if (cp.seq >= first && cp.seq <= last) bySeq.set(cp.seq, cp);
|
|
7537
|
+
}
|
|
7538
|
+
const outputs = new Array(inputs.length);
|
|
7539
|
+
const failures = [];
|
|
7540
|
+
const unsent = [];
|
|
7541
|
+
let inFlight = 0;
|
|
7542
|
+
let wakeAt;
|
|
7543
|
+
for (let i = 0; i < inputs.length; i += 1) {
|
|
7544
|
+
const seq = positions[i];
|
|
7545
|
+
const cp = bySeq.get(seq);
|
|
7546
|
+
if (!cp) {
|
|
7547
|
+
unsent.push(i);
|
|
7548
|
+
continue;
|
|
7549
|
+
}
|
|
7550
|
+
if (cp.name !== step.name) throw new NonDeterminismError(runId, seq, step.name, cp.name);
|
|
7551
|
+
if (cp.status === "completed") {
|
|
7552
|
+
outputs[i] = cp.output;
|
|
7553
|
+
continue;
|
|
7554
|
+
}
|
|
7555
|
+
try {
|
|
7556
|
+
outputs[i] = await this.callRemote(runId, seq, step, inputs[i], void 0, opts.transport, void 0, opts.admission, {
|
|
7557
|
+
...routing,
|
|
7558
|
+
parallelGroup: group
|
|
7559
|
+
});
|
|
7560
|
+
} catch (err) {
|
|
7561
|
+
if (err instanceof WorkflowSuspended) {
|
|
7562
|
+
inFlight += 1;
|
|
7563
|
+
if (err.wakeAt !== void 0) wakeAt = Math.min(wakeAt ?? err.wakeAt, err.wakeAt);
|
|
7564
|
+
} else if (err instanceof RemoteStepError) {
|
|
7565
|
+
const failure = {
|
|
7566
|
+
index: i,
|
|
7567
|
+
id: stepId(runId, seq),
|
|
7568
|
+
error: err.message
|
|
7569
|
+
};
|
|
7570
|
+
if (opts.mode === "failFast") throw new GatherError([
|
|
7571
|
+
failure
|
|
7572
|
+
], inputs.length);
|
|
7573
|
+
failures.push(failure);
|
|
7574
|
+
} else {
|
|
7575
|
+
throw err;
|
|
7576
|
+
}
|
|
7577
|
+
} finally {
|
|
7578
|
+
this.awaitedRemoteSteps.get(runId)?.delete(seq);
|
|
7579
|
+
}
|
|
7580
|
+
}
|
|
7581
|
+
const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
|
|
7582
|
+
if (batch.length > 0) await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
|
|
7583
|
+
if (inFlight + batch.length > 0) {
|
|
7584
|
+
this.awaitedStepFans.get(runId)?.push({
|
|
7585
|
+
from: first,
|
|
7586
|
+
to: last
|
|
7587
|
+
});
|
|
7588
|
+
throw new WorkflowSuspended(wakeAt);
|
|
7589
|
+
}
|
|
7590
|
+
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
7591
|
+
return outputs;
|
|
7592
|
+
}
|
|
7593
|
+
/** Write the `pending` checkpoints of a step fan's new steps, then dispatch them. */
|
|
7594
|
+
async sendStepFan(runId, positions, step, inputs, batch, group, opts, routing) {
|
|
7595
|
+
const workerGroup = stepGroup(step, routing?.namespace);
|
|
7596
|
+
await this.ensureRoutable(workerGroup, step.requires);
|
|
7597
|
+
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
7598
|
+
for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
|
|
7599
|
+
await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map((i) => {
|
|
7600
|
+
const seq = positions[i];
|
|
7601
|
+
return this.store.saveCheckpoint({
|
|
7602
|
+
runId,
|
|
7603
|
+
seq,
|
|
7604
|
+
name: step.name,
|
|
7605
|
+
kind: "remote",
|
|
7606
|
+
stepId: stepId(runId, seq),
|
|
7607
|
+
status: "pending",
|
|
7608
|
+
input: inputs[i],
|
|
7609
|
+
attempts: 1,
|
|
7610
|
+
workerGroup,
|
|
7611
|
+
parallelGroup: group,
|
|
7612
|
+
enqueuedAt,
|
|
7613
|
+
startedAt: enqueuedAt,
|
|
7614
|
+
finishedAt: enqueuedAt
|
|
7615
|
+
});
|
|
7616
|
+
}));
|
|
7617
|
+
}
|
|
7618
|
+
for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
|
|
7619
|
+
await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map(async (i) => {
|
|
7620
|
+
const seq = positions[i];
|
|
7621
|
+
await this.dispatchRemoteTask({
|
|
7622
|
+
runId,
|
|
7623
|
+
seq,
|
|
7624
|
+
name: step.name,
|
|
7625
|
+
stepId: stepId(runId, seq),
|
|
7626
|
+
group: workerGroup,
|
|
7627
|
+
input: inputs[i],
|
|
7628
|
+
priority: opts.admission?.priority,
|
|
7629
|
+
attempt: 1,
|
|
7630
|
+
transport: opts.transport
|
|
7631
|
+
});
|
|
7632
|
+
this.emit({
|
|
7633
|
+
type: "step.started",
|
|
7634
|
+
runId,
|
|
7635
|
+
seq,
|
|
7636
|
+
name: step.name,
|
|
7637
|
+
kind: "remote"
|
|
7638
|
+
});
|
|
7639
|
+
}));
|
|
7640
|
+
}
|
|
7641
|
+
}
|
|
7642
|
+
/** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
|
|
7643
|
+
noteSettledHere(key) {
|
|
7644
|
+
this.noteRecent(this.settledHere, key);
|
|
7645
|
+
}
|
|
7646
|
+
/** Record `key` in a "happened here recently" map, pruning entries older than a lease. */
|
|
7647
|
+
noteRecent(map, key) {
|
|
7648
|
+
const now = this.clock();
|
|
7649
|
+
map.set(key, now);
|
|
7650
|
+
if (map.size < 1024) return;
|
|
7651
|
+
for (const [k, at] of map) {
|
|
7652
|
+
if (now - at > this.leaseMs) map.delete(k);
|
|
7653
|
+
}
|
|
7654
|
+
}
|
|
7393
7655
|
/** {@link StateStore.saveCheckpointIfPending}, or a plain save on a store without it. */
|
|
7394
7656
|
async savePendingCheckpoint(checkpoint) {
|
|
7395
7657
|
if (this.store.saveCheckpointIfPending) return this.store.saveCheckpointIfPending(checkpoint);
|
|
@@ -7414,16 +7676,33 @@ var WorkflowEngine = class {
|
|
|
7414
7676
|
}, task.transport);
|
|
7415
7677
|
}
|
|
7416
7678
|
/**
|
|
7417
|
-
*
|
|
7418
|
-
*
|
|
7419
|
-
*
|
|
7679
|
+
* Run `work` after every earlier `work` for the same run on this instance has settled. A result
|
|
7680
|
+
* resumes its run inline, and `execute` lets this instance into a lease it already holds — so two
|
|
7681
|
+
* results of one run handled at once here would run two turns of it concurrently. Results of
|
|
7682
|
+
* different runs still go in parallel, which is what a results consumer with concurrency > 1 is for.
|
|
7683
|
+
*/
|
|
7684
|
+
serializedPerRun(runId, work) {
|
|
7685
|
+
const previous = this.resultChains.get(runId) ?? Promise.resolve();
|
|
7686
|
+
const next = previous.then(work, work);
|
|
7687
|
+
const tail = next.catch(() => void 0);
|
|
7688
|
+
this.resultChains.set(runId, tail);
|
|
7689
|
+
void tail.then(() => {
|
|
7690
|
+
if (this.resultChains.get(runId) === tail) this.resultChains.delete(runId);
|
|
7691
|
+
});
|
|
7692
|
+
return next;
|
|
7693
|
+
}
|
|
7694
|
+
/**
|
|
7695
|
+
* Complete a durable remote step from its worker result — runs on whichever instance receives the
|
|
7696
|
+
* result (the dispatching one may be gone), so the run is crash/scale-safe. Resolves whether the
|
|
7697
|
+
* run should now be resumed: not for a duplicate or late delivery (the checkpoint isn't `pending`),
|
|
7698
|
+
* a run already finished, or a step fan whose barrier is still closed.
|
|
7420
7699
|
*/
|
|
7421
7700
|
async completeRemoteResult(result) {
|
|
7422
7701
|
const cp = await this.store.getCheckpoint(result.runId, result.seq);
|
|
7423
|
-
if (!cp || cp.status !== "pending") return;
|
|
7702
|
+
if (!cp || cp.status !== "pending") return false;
|
|
7424
7703
|
void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
|
|
7425
7704
|
const run = await this.store.getRun(result.runId);
|
|
7426
|
-
if (run && (run.status === "cancelled" || run.status === "completed")) return;
|
|
7705
|
+
if (run && (run.status === "cancelled" || run.status === "completed")) return false;
|
|
7427
7706
|
const finishedAt = /* @__PURE__ */ new Date();
|
|
7428
7707
|
const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
|
|
7429
7708
|
const claimed = await this.savePendingCheckpoint({
|
|
@@ -7435,7 +7714,8 @@ var WorkflowEngine = class {
|
|
|
7435
7714
|
startedAt,
|
|
7436
7715
|
finishedAt
|
|
7437
7716
|
});
|
|
7438
|
-
if (!claimed) return;
|
|
7717
|
+
if (!claimed) return false;
|
|
7718
|
+
this.noteSettledHere(`${result.runId}:${result.seq}`);
|
|
7439
7719
|
this.emit({
|
|
7440
7720
|
type: result.status === "completed" ? "step.completed" : "step.failed",
|
|
7441
7721
|
runId: result.runId,
|
|
@@ -7447,10 +7727,25 @@ var WorkflowEngine = class {
|
|
|
7447
7727
|
queueMs: startedAt.getTime() - cp.enqueuedAt.getTime(),
|
|
7448
7728
|
durationMs: finishedAt.getTime() - startedAt.getTime()
|
|
7449
7729
|
});
|
|
7450
|
-
await this.
|
|
7451
|
-
|
|
7452
|
-
|
|
7453
|
-
|
|
7730
|
+
return !await this.stepFanStillRunning(cp, result);
|
|
7731
|
+
}
|
|
7732
|
+
/**
|
|
7733
|
+
* The join barrier of a step fan: a result for one of its steps resumes the parent only once no
|
|
7734
|
+
* step of the fan is still `pending` (a settled wave), or on a failure (failFast surfaces it, and a
|
|
7735
|
+
* retry needs the parent to schedule it). Without it every result would replay the whole parent,
|
|
7736
|
+
* which makes a fan of N steps cost O(N²) checkpoint reads.
|
|
7737
|
+
*/
|
|
7738
|
+
async stepFanStillRunning(cp, result) {
|
|
7739
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
7740
|
+
if (!fan) return false;
|
|
7741
|
+
if (result.status === "failed") return false;
|
|
7742
|
+
const to = fan.firstSeq + fan.size - 1;
|
|
7743
|
+
const pending = this.store.countCheckpoints ? await this.store.countCheckpoints(result.runId, fan.firstSeq, to, [
|
|
7744
|
+
"pending"
|
|
7745
|
+
]) : (await this.store.listCheckpoints(result.runId)).filter((c) => c.seq >= fan.firstSeq && c.seq <= to && c.status === "pending").length;
|
|
7746
|
+
if (pending > 0) return true;
|
|
7747
|
+
this.noteRecent(this.stepFansFiredHere, `${result.runId}:${fan.firstSeq}`);
|
|
7748
|
+
return false;
|
|
7454
7749
|
}
|
|
7455
7750
|
/**
|
|
7456
7751
|
* Apply a remote workflow-turn {@link WorkflowDecision} durably — runs on WHATEVER instance receives
|
|
@@ -7831,6 +8126,8 @@ var CodecStateStore = class {
|
|
|
7831
8126
|
/** Encodes like {@link saveCheckpoint}. Bound only when the inner store has it, so the engine's
|
|
7832
8127
|
* plain-save fallback still kicks in through the wrapper. */
|
|
7833
8128
|
saveCheckpointIfPending;
|
|
8129
|
+
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
|
|
8130
|
+
countCheckpoints;
|
|
7834
8131
|
/** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
|
|
7835
8132
|
* when the inner store prunes, so the retention poller still sees (and warns about) its absence. */
|
|
7836
8133
|
pruneTerminalRuns;
|
|
@@ -7860,6 +8157,8 @@ var CodecStateStore = class {
|
|
|
7860
8157
|
if (deleteRuns) this.deleteRuns = (ids) => deleteRuns.call(inner, ids);
|
|
7861
8158
|
const saveIfPending = inner.saveCheckpointIfPending;
|
|
7862
8159
|
if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
|
|
8160
|
+
const countCps = inner.countCheckpoints;
|
|
8161
|
+
if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
|
|
7863
8162
|
const prune = inner.pruneTerminalRuns;
|
|
7864
8163
|
if (prune) this.pruneTerminalRuns = (policy, now, limit) => prune.call(inner, policy, now, limit);
|
|
7865
8164
|
const count = inner.countRuns;
|
|
@@ -7990,8 +8289,8 @@ var CodecStateStore = class {
|
|
|
7990
8289
|
async listPendingRuns(limit, namespace) {
|
|
7991
8290
|
return (await this.inner.listPendingRuns(limit, namespace)).map((r) => this.decRun(r));
|
|
7992
8291
|
}
|
|
7993
|
-
async listDueTimers(nowMs, namespace) {
|
|
7994
|
-
return (await this.inner.listDueTimers(nowMs, namespace)).map((r) => this.decRun(r));
|
|
8292
|
+
async listDueTimers(nowMs, namespace, limit) {
|
|
8293
|
+
return (await this.inner.listDueTimers(nowMs, namespace, limit)).map((r) => this.decRun(r));
|
|
7995
8294
|
}
|
|
7996
8295
|
tryLockRun(runId, owner, leaseUntilMs, nowMs) {
|
|
7997
8296
|
return this.inner.tryLockRun(runId, owner, leaseUntilMs, nowMs);
|
|
@@ -8445,10 +8744,11 @@ var InMemoryStateStore = class {
|
|
|
8445
8744
|
...r
|
|
8446
8745
|
}));
|
|
8447
8746
|
}
|
|
8448
|
-
async listDueTimers(nowMs, namespace) {
|
|
8449
|
-
|
|
8747
|
+
async listDueTimers(nowMs, namespace, limit) {
|
|
8748
|
+
const due = [
|
|
8450
8749
|
...this.runs.values()
|
|
8451
|
-
].filter((r) => r.status === "suspended" && r.wakeAt !== void 0 && r.wakeAt <= nowMs && (namespace === void 0 || r.namespace === namespace)).
|
|
8750
|
+
].filter((r) => r.status === "suspended" && r.wakeAt !== void 0 && r.wakeAt <= nowMs && (namespace === void 0 || r.namespace === namespace)).sort((a, b) => (a.wakeAt ?? 0) - (b.wakeAt ?? 0));
|
|
8751
|
+
return (limit === void 0 ? due : due.slice(0, limit)).map((r) => ({
|
|
8452
8752
|
...r
|
|
8453
8753
|
}));
|
|
8454
8754
|
}
|
|
@@ -8722,6 +9022,13 @@ var InMemoryStateStore = class {
|
|
|
8722
9022
|
...latest
|
|
8723
9023
|
} : void 0;
|
|
8724
9024
|
}
|
|
9025
|
+
async countCheckpoints(runId, fromSeq, toSeq, statuses) {
|
|
9026
|
+
let n = 0;
|
|
9027
|
+
for (const cp of this.checkpoints.values()) {
|
|
9028
|
+
if (cp.runId === runId && cp.seq >= fromSeq && cp.seq <= toSeq && statuses.includes(cp.status)) n += 1;
|
|
9029
|
+
}
|
|
9030
|
+
return n;
|
|
9031
|
+
}
|
|
8725
9032
|
async listCheckpointsByNamePrefix(runId, prefixes) {
|
|
8726
9033
|
const rows = [
|
|
8727
9034
|
...this.checkpoints.values()
|