@dudousxd/nestjs-durable-core 0.76.1 → 0.78.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +437 -25
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +302 -113
- package/dist/index.d.ts +302 -113
- package/dist/index.js +430 -25
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1083,10 +1083,37 @@ function allChildId(runId, firstSeq, index, wave) {
|
|
|
1083
1083
|
return `${allBase(runId, firstSeq, wave)}${index}`;
|
|
1084
1084
|
}
|
|
1085
1085
|
__name(allChildId, "allChildId");
|
|
1086
|
-
function allGroup(firstSeq, mode) {
|
|
1087
|
-
|
|
1086
|
+
function allGroup(firstSeq, size, mode) {
|
|
1087
|
+
const group = `all:${firstSeq}:n${size}`;
|
|
1088
|
+
return mode === "failFast" ? `${group}:failFast` : group;
|
|
1088
1089
|
}
|
|
1089
1090
|
__name(allGroup, "allGroup");
|
|
1091
|
+
function stepFanGroup(firstSeq, size, mode) {
|
|
1092
|
+
const group = `steps:${firstSeq}:${size}`;
|
|
1093
|
+
return mode === "failFast" ? `${group}:failFast` : group;
|
|
1094
|
+
}
|
|
1095
|
+
__name(stepFanGroup, "stepFanGroup");
|
|
1096
|
+
function parseStepFanGroup(group) {
|
|
1097
|
+
const fan = parseFanGroup(group);
|
|
1098
|
+
if (fan?.kind !== "steps" || fan.size === void 0) return void 0;
|
|
1099
|
+
return {
|
|
1100
|
+
firstSeq: fan.firstSeq,
|
|
1101
|
+
size: fan.size
|
|
1102
|
+
};
|
|
1103
|
+
}
|
|
1104
|
+
__name(parseStepFanGroup, "parseStepFanGroup");
|
|
1105
|
+
function parseFanGroup(group) {
|
|
1106
|
+
const m = group ? /^(all|steps):(\d+)(?::n?(\d+))?(:failFast)?$/.exec(group) : null;
|
|
1107
|
+
if (!m) return void 0;
|
|
1108
|
+
if (m[1] === "steps" && m[3] === void 0) return void 0;
|
|
1109
|
+
return {
|
|
1110
|
+
kind: m[1],
|
|
1111
|
+
firstSeq: Number(m[2]),
|
|
1112
|
+
size: m[3] === void 0 ? void 0 : Number(m[3]),
|
|
1113
|
+
failFast: m[4] !== void 0
|
|
1114
|
+
};
|
|
1115
|
+
}
|
|
1116
|
+
__name(parseFanGroup, "parseFanGroup");
|
|
1090
1117
|
function allWait(runId, firstSeq, wave) {
|
|
1091
1118
|
const base = allBase(runId, firstSeq, wave);
|
|
1092
1119
|
return {
|
|
@@ -1136,7 +1163,7 @@ function parentRunIdOf(runId) {
|
|
|
1136
1163
|
return ENGINE_CHILD_ID.exec(runId)?.[1];
|
|
1137
1164
|
}
|
|
1138
1165
|
__name(parentRunIdOf, "parentRunIdOf");
|
|
1139
|
-
var GROUP = /^all:(\d+)(:failFast)?$/;
|
|
1166
|
+
var GROUP = /^all:(\d+)(?::n\d+)?(:failFast)?$/;
|
|
1140
1167
|
var REST = /^(?:w(\d+)\.)?\d+$/;
|
|
1141
1168
|
function fanOfWaiter(waiter) {
|
|
1142
1169
|
const group = waiter.parallelGroup ? GROUP.exec(waiter.parallelGroup) : null;
|
|
@@ -3128,12 +3155,15 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3128
3155
|
}, "child");
|
|
3129
3156
|
const all = /* @__PURE__ */ __name(async (workflow2, inputs, opts) => {
|
|
3130
3157
|
if (inputs.length === 0) return [];
|
|
3158
|
+
if (stepNameOf(workflow2) !== void 0) {
|
|
3159
|
+
return allSteps(workflow2, inputs, opts);
|
|
3160
|
+
}
|
|
3131
3161
|
const mode = opts?.mode ?? "waitAll";
|
|
3132
3162
|
const name = workflowName(workflow2);
|
|
3133
3163
|
const waves = allWaves(inputs.length, opts?.concurrency);
|
|
3134
3164
|
const positions = inputs.map(() => pos.next());
|
|
3135
3165
|
const firstSeq = positions[0];
|
|
3136
|
-
const group = allGroup(firstSeq, mode);
|
|
3166
|
+
const group = allGroup(firstSeq, inputs.length, mode);
|
|
3137
3167
|
const waveOf = /* @__PURE__ */ __name((i) => waves ? Math.floor(i / waves.waveSize) : void 0, "waveOf");
|
|
3138
3168
|
const id = /* @__PURE__ */ __name((i) => allChildId(runId, firstSeq, i, waveOf(i)), "id");
|
|
3139
3169
|
const { checkpoints: existing, folded } = await readFan(positions, [
|
|
@@ -3267,6 +3297,26 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3267
3297
|
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
3268
3298
|
return outputs;
|
|
3269
3299
|
}, "all");
|
|
3300
|
+
const allSteps = /* @__PURE__ */ __name(async (ref, inputs, opts) => {
|
|
3301
|
+
if (opts?.queue !== void 0) {
|
|
3302
|
+
throw new Error("ctx.all: a flow-control queue is not supported on a step fan");
|
|
3303
|
+
}
|
|
3304
|
+
const name = stepNameOf(ref);
|
|
3305
|
+
const def = {
|
|
3306
|
+
name,
|
|
3307
|
+
...resolveStepPolicy(stepConfigOf(ref), opts)
|
|
3308
|
+
};
|
|
3309
|
+
const positions = inputs.map(() => pos.next());
|
|
3310
|
+
return host.callRemoteAll(runId, positions, def, inputs, {
|
|
3311
|
+
mode: opts?.mode ?? "waitAll",
|
|
3312
|
+
concurrency: opts?.concurrency,
|
|
3313
|
+
transport: opts?.transport,
|
|
3314
|
+
admission: {
|
|
3315
|
+
priority: opts?.priority,
|
|
3316
|
+
fairnessKey: opts?.fairnessKey
|
|
3317
|
+
}
|
|
3318
|
+
});
|
|
3319
|
+
}, "allSteps");
|
|
3270
3320
|
const readFan = /* @__PURE__ */ __name(async (positions, prefixes) => {
|
|
3271
3321
|
const checkpoints = positions.map((seq) => replay?.get(seq) ?? null);
|
|
3272
3322
|
const folded = positions.map(() => false);
|
|
@@ -3603,6 +3653,18 @@ var BREAKPOINT = "breakpoint";
|
|
|
3603
3653
|
var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
|
|
3604
3654
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
3605
3655
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
3656
|
+
var STEP_FAN_CHUNK = 200;
|
|
3657
|
+
function shuffle(items) {
|
|
3658
|
+
for (let i = items.length - 1; i > 0; i -= 1) {
|
|
3659
|
+
const j = Math.floor(Math.random() * (i + 1));
|
|
3660
|
+
[items[i], items[j]] = [
|
|
3661
|
+
items[j],
|
|
3662
|
+
items[i]
|
|
3663
|
+
];
|
|
3664
|
+
}
|
|
3665
|
+
return items;
|
|
3666
|
+
}
|
|
3667
|
+
__name(shuffle, "shuffle");
|
|
3606
3668
|
function stepGroup(step, runNamespace) {
|
|
3607
3669
|
return tenantGroup(sanitizeQueueToken(step.name), step.partition ?? runNamespace);
|
|
3608
3670
|
}
|
|
@@ -3644,6 +3706,7 @@ var WorkflowEngine = class {
|
|
|
3644
3706
|
maxRecoveryAttempts;
|
|
3645
3707
|
remoteAdvanceSilenceMs;
|
|
3646
3708
|
reconcileMs;
|
|
3709
|
+
timerBatchSize;
|
|
3647
3710
|
eventBufferTtlMs;
|
|
3648
3711
|
remoteRedispatchMs;
|
|
3649
3712
|
remoteRedispatchMax;
|
|
@@ -3710,6 +3773,15 @@ var WorkflowEngine = class {
|
|
|
3710
3773
|
* dropped one {@link redriveSettledSteps} recovers. Pruned by age (see {@link noteSettledHere}).
|
|
3711
3774
|
*/
|
|
3712
3775
|
settledHere = /* @__PURE__ */ new Map();
|
|
3776
|
+
/**
|
|
3777
|
+
* The step fans (`ctx.all` over a `@Step`) a turn suspended on, by run, while that turn executes on
|
|
3778
|
+
* THIS instance — re-checked once its lease is released (see {@link redriveSettledStepFans}).
|
|
3779
|
+
*/
|
|
3780
|
+
awaitedStepFans = /* @__PURE__ */ new Map();
|
|
3781
|
+
/** Step fans whose join barrier this instance opened, by `runId:firstSeq` — see {@link settledHere}. */
|
|
3782
|
+
stepFansFiredHere = /* @__PURE__ */ new Map();
|
|
3783
|
+
/** The tail of each run's queue of results being handled on this instance — see {@link serializedPerRun}. */
|
|
3784
|
+
resultChains = /* @__PURE__ */ new Map();
|
|
3713
3785
|
/** Flow-control admission backend for remote steps (see {@link registerQueue}). */
|
|
3714
3786
|
admission;
|
|
3715
3787
|
/** Runs on THIS instance blocked on admission, by queue — woken early on a freed-slot signal. */
|
|
@@ -3741,6 +3813,8 @@ var WorkflowEngine = class {
|
|
|
3741
3813
|
this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
|
|
3742
3814
|
this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
|
|
3743
3815
|
this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
|
|
3816
|
+
const batch = deps.timerBatchSize;
|
|
3817
|
+
this.timerBatchSize = batch !== void 0 && Number.isFinite(batch) && batch >= 1 ? Math.floor(batch) : 500;
|
|
3744
3818
|
this.eventBufferTtlMs = deps.eventBufferTtlMs || void 0;
|
|
3745
3819
|
this.remoteRedispatchMs = deps.remoteRedispatchMs || void 0;
|
|
3746
3820
|
this.remoteRedispatchMax = Math.max(1, deps.remoteRedispatchMax ?? 10);
|
|
@@ -3793,7 +3867,11 @@ var WorkflowEngine = class {
|
|
|
3793
3867
|
}
|
|
3794
3868
|
return;
|
|
3795
3869
|
}
|
|
3796
|
-
await this.completeRemoteResult(result);
|
|
3870
|
+
if (!await this.completeRemoteResult(result)) return;
|
|
3871
|
+
await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId).then(() => void 0, (err) => {
|
|
3872
|
+
if (err instanceof NamespaceMismatch) return;
|
|
3873
|
+
throw err;
|
|
3874
|
+
}));
|
|
3797
3875
|
},
|
|
3798
3876
|
// A heartbeat resets the liveness window for whatever it targets: an in-flight long STEP (keyed
|
|
3799
3877
|
// by stepId, see callRemote) or — when stepId is absent — the in-flight workflow TURN (keyed by
|
|
@@ -4653,7 +4731,7 @@ var WorkflowEngine = class {
|
|
|
4653
4731
|
* scan, filtered to just this tick's batch, keeps it cheap regardless of store size.
|
|
4654
4732
|
*/
|
|
4655
4733
|
async resumeDueTimers(nowMs = this.clock()) {
|
|
4656
|
-
const due = await this.store.listDueTimers(nowMs, this.namespace);
|
|
4734
|
+
const due = shuffle(await this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize));
|
|
4657
4735
|
const blocked = await this.dueBlockedRuns(nowMs);
|
|
4658
4736
|
if (due.length === 0 && blocked.length === 0) return [];
|
|
4659
4737
|
const waiterByRun = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
|
|
@@ -5152,6 +5230,32 @@ var WorkflowEngine = class {
|
|
|
5152
5230
|
}
|
|
5153
5231
|
}
|
|
5154
5232
|
}
|
|
5233
|
+
/**
|
|
5234
|
+
* {@link redriveSettledSteps} for a step fan: once the turn that suspended on it released its lease,
|
|
5235
|
+
* re-drive the run if no step of the fan is still `pending` — the last result landed on another
|
|
5236
|
+
* instance while this one held the lease, and its wake was dropped there. A fan whose barrier this
|
|
5237
|
+
* instance opened is skipped, for the reason {@link settledHere} gives.
|
|
5238
|
+
*/
|
|
5239
|
+
async redriveSettledStepFans(runId, fans) {
|
|
5240
|
+
for (const fan of fans) {
|
|
5241
|
+
if (this.stepFansFiredHere.has(`${runId}:${fan.from}`)) continue;
|
|
5242
|
+
const pending = this.store.countCheckpoints ? await this.store.countCheckpoints(runId, fan.from, fan.to, [
|
|
5243
|
+
"pending"
|
|
5244
|
+
]) : (await this.store.listCheckpoints(runId)).filter((c) => c.seq >= fan.from && c.seq <= fan.to && c.status === "pending").length;
|
|
5245
|
+
if (pending > 0) continue;
|
|
5246
|
+
const nowMs = this.clock();
|
|
5247
|
+
if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
5248
|
+
return;
|
|
5249
|
+
}
|
|
5250
|
+
const run = await this.store.getRun(runId);
|
|
5251
|
+
if (run?.status !== "suspended") {
|
|
5252
|
+
await this.store.releaseRunLock(runId);
|
|
5253
|
+
return;
|
|
5254
|
+
}
|
|
5255
|
+
await this.resumeImplicit(runId);
|
|
5256
|
+
return;
|
|
5257
|
+
}
|
|
5258
|
+
}
|
|
5155
5259
|
async redriveSettledChildren(runId, fan) {
|
|
5156
5260
|
if (!this.store.hasSignalWaiter) return;
|
|
5157
5261
|
for (const wait of fan.waits) {
|
|
@@ -6609,6 +6713,8 @@ var WorkflowEngine = class {
|
|
|
6609
6713
|
let suspendedOnSteps = false;
|
|
6610
6714
|
const awaitedSteps = /* @__PURE__ */ new Set();
|
|
6611
6715
|
this.awaitedRemoteSteps.set(run.id, awaitedSteps);
|
|
6716
|
+
const awaitedFans = [];
|
|
6717
|
+
this.awaitedStepFans.set(run.id, awaitedFans);
|
|
6612
6718
|
const ctx = createWorkflowCtx(this.ctxHostFor(replay, run.namespace), run.id, compensations, run.workflow, registered?.searchAttributesSchema);
|
|
6613
6719
|
try {
|
|
6614
6720
|
const output = await runInWorkflowCtx(ctx, () => fn(ctx, run.input));
|
|
@@ -6688,6 +6794,7 @@ var WorkflowEngine = class {
|
|
|
6688
6794
|
if (this.awaitedRemoteSteps.get(run.id) === awaitedSteps) {
|
|
6689
6795
|
this.awaitedRemoteSteps.delete(run.id);
|
|
6690
6796
|
}
|
|
6797
|
+
if (this.awaitedStepFans.get(run.id) === awaitedFans) this.awaitedStepFans.delete(run.id);
|
|
6691
6798
|
await this.store.releaseRunLock(run.id);
|
|
6692
6799
|
if (suspendedOnFan) {
|
|
6693
6800
|
void this.redriveSettledChildren(run.id, suspendedOnFan).catch(() => void 0);
|
|
@@ -6695,6 +6802,9 @@ var WorkflowEngine = class {
|
|
|
6695
6802
|
if (suspendedOnSteps && awaitedSteps.size > 0) {
|
|
6696
6803
|
void this.redriveSettledSteps(run.id, awaitedSteps).catch(() => void 0);
|
|
6697
6804
|
}
|
|
6805
|
+
if (suspendedOnSteps && awaitedFans.length > 0) {
|
|
6806
|
+
void this.redriveSettledStepFans(run.id, awaitedFans).catch(() => void 0);
|
|
6807
|
+
}
|
|
6698
6808
|
}
|
|
6699
6809
|
}
|
|
6700
6810
|
/**
|
|
@@ -6917,6 +7027,9 @@ var WorkflowEngine = class {
|
|
|
6917
7027
|
callRemote: /* @__PURE__ */ __name((runId, seq, step, input, queue, transport, admission) => this.callRemote(runId, seq, step, input, queue, transport, replay, admission, {
|
|
6918
7028
|
namespace: parentNamespace
|
|
6919
7029
|
}), "callRemote"),
|
|
7030
|
+
callRemoteAll: /* @__PURE__ */ __name((runId, positions, step, inputs, opts) => this.callRemoteAll(runId, positions, step, inputs, opts, {
|
|
7031
|
+
namespace: parentNamespace
|
|
7032
|
+
}), "callRemoteAll"),
|
|
6920
7033
|
// Defer so a fast child can't reentrantly resume a still-running parent. Stamp the child with
|
|
6921
7034
|
// the PARENT run's namespace (not this engine's) so a child belongs to the tenant/partition of
|
|
6922
7035
|
// the run it was spawned from — even when an operator (namespace: undefined, drives every
|
|
@@ -7218,6 +7331,9 @@ var WorkflowEngine = class {
|
|
|
7218
7331
|
input,
|
|
7219
7332
|
attempts: attempt,
|
|
7220
7333
|
workerGroup: group,
|
|
7334
|
+
...routing?.parallelGroup !== void 0 ? {
|
|
7335
|
+
parallelGroup: routing.parallelGroup
|
|
7336
|
+
} : {},
|
|
7221
7337
|
enqueuedAt,
|
|
7222
7338
|
startedAt: enqueuedAt,
|
|
7223
7339
|
finishedAt: enqueuedAt
|
|
@@ -7243,13 +7359,147 @@ var WorkflowEngine = class {
|
|
|
7243
7359
|
this.awaitedRemoteSteps.get(runId)?.add(seq);
|
|
7244
7360
|
throw new WorkflowSuspended();
|
|
7245
7361
|
}
|
|
7362
|
+
/**
|
|
7363
|
+
* `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
|
|
7364
|
+
* reserves the whole position block up front, so replay re-derives the same seqs.
|
|
7365
|
+
*
|
|
7366
|
+
* Each turn reads the fan's checkpoints fresh, hands every step that already has one to
|
|
7367
|
+
* {@link callRemote} (which owns the completed / pending-lease / failed-retry rules), and sends
|
|
7368
|
+
* the steps nobody has sent yet: all of them, or with `concurrency`, the next wave once the last
|
|
7369
|
+
* one has drained. New steps are written `pending` BEFORE any is dispatched, so a turn that runs
|
|
7370
|
+
* concurrently with this one finds them already sent instead of sending them again. The join
|
|
7371
|
+
* barrier ({@link stepFanStillRunning}) then wakes the parent once per wave, not once per result.
|
|
7372
|
+
*/
|
|
7373
|
+
async callRemoteAll(runId, positions, step, inputs, opts, routing) {
|
|
7374
|
+
if (step.timeoutMs) {
|
|
7375
|
+
throw new Error(`ctx.all: step "${step.name}" has a liveness timeoutMs; a step fan needs durable steps`);
|
|
7376
|
+
}
|
|
7377
|
+
if (opts.concurrency !== void 0 && (!Number.isInteger(opts.concurrency) || opts.concurrency < 1)) {
|
|
7378
|
+
throw new Error(`ctx.all: concurrency must be a positive integer, got ${opts.concurrency}`);
|
|
7379
|
+
}
|
|
7380
|
+
if (this.pool.size === 0) throw new Error("remote steps require a Transport");
|
|
7381
|
+
const first = positions[0];
|
|
7382
|
+
const last = positions[positions.length - 1];
|
|
7383
|
+
const group = stepFanGroup(first, positions.length, opts.mode);
|
|
7384
|
+
const bySeq = /* @__PURE__ */ new Map();
|
|
7385
|
+
for (const cp of await this.store.listCheckpoints(runId)) {
|
|
7386
|
+
if (cp.seq >= first && cp.seq <= last) bySeq.set(cp.seq, cp);
|
|
7387
|
+
}
|
|
7388
|
+
const outputs = new Array(inputs.length);
|
|
7389
|
+
const failures = [];
|
|
7390
|
+
const unsent = [];
|
|
7391
|
+
let inFlight = 0;
|
|
7392
|
+
let wakeAt;
|
|
7393
|
+
for (let i = 0; i < inputs.length; i += 1) {
|
|
7394
|
+
const seq = positions[i];
|
|
7395
|
+
const cp = bySeq.get(seq);
|
|
7396
|
+
if (!cp) {
|
|
7397
|
+
unsent.push(i);
|
|
7398
|
+
continue;
|
|
7399
|
+
}
|
|
7400
|
+
if (cp.name !== step.name) throw new NonDeterminismError(runId, seq, step.name, cp.name);
|
|
7401
|
+
if (cp.status === "completed") {
|
|
7402
|
+
outputs[i] = cp.output;
|
|
7403
|
+
continue;
|
|
7404
|
+
}
|
|
7405
|
+
try {
|
|
7406
|
+
outputs[i] = await this.callRemote(runId, seq, step, inputs[i], void 0, opts.transport, void 0, opts.admission, {
|
|
7407
|
+
...routing,
|
|
7408
|
+
parallelGroup: group
|
|
7409
|
+
});
|
|
7410
|
+
} catch (err) {
|
|
7411
|
+
if (err instanceof WorkflowSuspended) {
|
|
7412
|
+
inFlight += 1;
|
|
7413
|
+
if (err.wakeAt !== void 0) wakeAt = Math.min(wakeAt ?? err.wakeAt, err.wakeAt);
|
|
7414
|
+
} else if (err instanceof RemoteStepError) {
|
|
7415
|
+
const failure = {
|
|
7416
|
+
index: i,
|
|
7417
|
+
id: stepId(runId, seq),
|
|
7418
|
+
error: err.message
|
|
7419
|
+
};
|
|
7420
|
+
if (opts.mode === "failFast") throw new GatherError([
|
|
7421
|
+
failure
|
|
7422
|
+
], inputs.length);
|
|
7423
|
+
failures.push(failure);
|
|
7424
|
+
} else {
|
|
7425
|
+
throw err;
|
|
7426
|
+
}
|
|
7427
|
+
} finally {
|
|
7428
|
+
this.awaitedRemoteSteps.get(runId)?.delete(seq);
|
|
7429
|
+
}
|
|
7430
|
+
}
|
|
7431
|
+
const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
|
|
7432
|
+
if (batch.length > 0) await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
|
|
7433
|
+
if (inFlight + batch.length > 0) {
|
|
7434
|
+
this.awaitedStepFans.get(runId)?.push({
|
|
7435
|
+
from: first,
|
|
7436
|
+
to: last
|
|
7437
|
+
});
|
|
7438
|
+
throw new WorkflowSuspended(wakeAt);
|
|
7439
|
+
}
|
|
7440
|
+
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
7441
|
+
return outputs;
|
|
7442
|
+
}
|
|
7443
|
+
/** Write the `pending` checkpoints of a step fan's new steps, then dispatch them. */
|
|
7444
|
+
async sendStepFan(runId, positions, step, inputs, batch, group, opts, routing) {
|
|
7445
|
+
const workerGroup = stepGroup(step, routing?.namespace);
|
|
7446
|
+
await this.ensureRoutable(workerGroup, step.requires);
|
|
7447
|
+
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
7448
|
+
for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
|
|
7449
|
+
await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map((i) => {
|
|
7450
|
+
const seq = positions[i];
|
|
7451
|
+
return this.store.saveCheckpoint({
|
|
7452
|
+
runId,
|
|
7453
|
+
seq,
|
|
7454
|
+
name: step.name,
|
|
7455
|
+
kind: "remote",
|
|
7456
|
+
stepId: stepId(runId, seq),
|
|
7457
|
+
status: "pending",
|
|
7458
|
+
input: inputs[i],
|
|
7459
|
+
attempts: 1,
|
|
7460
|
+
workerGroup,
|
|
7461
|
+
parallelGroup: group,
|
|
7462
|
+
enqueuedAt,
|
|
7463
|
+
startedAt: enqueuedAt,
|
|
7464
|
+
finishedAt: enqueuedAt
|
|
7465
|
+
});
|
|
7466
|
+
}));
|
|
7467
|
+
}
|
|
7468
|
+
for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
|
|
7469
|
+
await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map(async (i) => {
|
|
7470
|
+
const seq = positions[i];
|
|
7471
|
+
await this.dispatchRemoteTask({
|
|
7472
|
+
runId,
|
|
7473
|
+
seq,
|
|
7474
|
+
name: step.name,
|
|
7475
|
+
stepId: stepId(runId, seq),
|
|
7476
|
+
group: workerGroup,
|
|
7477
|
+
input: inputs[i],
|
|
7478
|
+
priority: opts.admission?.priority,
|
|
7479
|
+
attempt: 1,
|
|
7480
|
+
transport: opts.transport
|
|
7481
|
+
});
|
|
7482
|
+
this.emit({
|
|
7483
|
+
type: "step.started",
|
|
7484
|
+
runId,
|
|
7485
|
+
seq,
|
|
7486
|
+
name: step.name,
|
|
7487
|
+
kind: "remote"
|
|
7488
|
+
});
|
|
7489
|
+
}));
|
|
7490
|
+
}
|
|
7491
|
+
}
|
|
7246
7492
|
/** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
|
|
7247
7493
|
noteSettledHere(key) {
|
|
7494
|
+
this.noteRecent(this.settledHere, key);
|
|
7495
|
+
}
|
|
7496
|
+
/** Record `key` in a "happened here recently" map, pruning entries older than a lease. */
|
|
7497
|
+
noteRecent(map, key) {
|
|
7248
7498
|
const now = this.clock();
|
|
7249
|
-
|
|
7250
|
-
if (
|
|
7251
|
-
for (const [k, at] of
|
|
7252
|
-
if (now - at > this.leaseMs)
|
|
7499
|
+
map.set(key, now);
|
|
7500
|
+
if (map.size < 1024) return;
|
|
7501
|
+
for (const [k, at] of map) {
|
|
7502
|
+
if (now - at > this.leaseMs) map.delete(k);
|
|
7253
7503
|
}
|
|
7254
7504
|
}
|
|
7255
7505
|
/** {@link StateStore.saveCheckpointIfPending}, or a plain save on a store without it. */
|
|
@@ -7276,16 +7526,33 @@ var WorkflowEngine = class {
|
|
|
7276
7526
|
}, task.transport);
|
|
7277
7527
|
}
|
|
7278
7528
|
/**
|
|
7279
|
-
*
|
|
7280
|
-
*
|
|
7281
|
-
*
|
|
7529
|
+
* Run `work` after every earlier `work` for the same run on this instance has settled. A result
|
|
7530
|
+
* resumes its run inline, and `execute` lets this instance into a lease it already holds — so two
|
|
7531
|
+
* results of one run handled at once here would run two turns of it concurrently. Results of
|
|
7532
|
+
* different runs still go in parallel, which is what a results consumer with concurrency > 1 is for.
|
|
7533
|
+
*/
|
|
7534
|
+
serializedPerRun(runId, work) {
|
|
7535
|
+
const previous = this.resultChains.get(runId) ?? Promise.resolve();
|
|
7536
|
+
const next = previous.then(work, work);
|
|
7537
|
+
const tail = next.catch(() => void 0);
|
|
7538
|
+
this.resultChains.set(runId, tail);
|
|
7539
|
+
void tail.then(() => {
|
|
7540
|
+
if (this.resultChains.get(runId) === tail) this.resultChains.delete(runId);
|
|
7541
|
+
});
|
|
7542
|
+
return next;
|
|
7543
|
+
}
|
|
7544
|
+
/**
|
|
7545
|
+
* Complete a durable remote step from its worker result — runs on whichever instance receives the
|
|
7546
|
+
* result (the dispatching one may be gone), so the run is crash/scale-safe. Resolves whether the
|
|
7547
|
+
* run should now be resumed: not for a duplicate or late delivery (the checkpoint isn't `pending`),
|
|
7548
|
+
* a run already finished, or a step fan whose barrier is still closed.
|
|
7282
7549
|
*/
|
|
7283
7550
|
async completeRemoteResult(result) {
|
|
7284
7551
|
const cp = await this.store.getCheckpoint(result.runId, result.seq);
|
|
7285
|
-
if (!cp || cp.status !== "pending") return;
|
|
7552
|
+
if (!cp || cp.status !== "pending") return false;
|
|
7286
7553
|
void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
|
|
7287
7554
|
const run = await this.store.getRun(result.runId);
|
|
7288
|
-
if (run && (run.status === "cancelled" || run.status === "completed")) return;
|
|
7555
|
+
if (run && (run.status === "cancelled" || run.status === "completed")) return false;
|
|
7289
7556
|
const finishedAt = /* @__PURE__ */ new Date();
|
|
7290
7557
|
const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
|
|
7291
7558
|
const claimed = await this.savePendingCheckpoint({
|
|
@@ -7297,7 +7564,7 @@ var WorkflowEngine = class {
|
|
|
7297
7564
|
startedAt,
|
|
7298
7565
|
finishedAt
|
|
7299
7566
|
});
|
|
7300
|
-
if (!claimed) return;
|
|
7567
|
+
if (!claimed) return false;
|
|
7301
7568
|
this.noteSettledHere(`${result.runId}:${result.seq}`);
|
|
7302
7569
|
this.emit({
|
|
7303
7570
|
type: result.status === "completed" ? "step.completed" : "step.failed",
|
|
@@ -7310,10 +7577,25 @@ var WorkflowEngine = class {
|
|
|
7310
7577
|
queueMs: startedAt.getTime() - cp.enqueuedAt.getTime(),
|
|
7311
7578
|
durationMs: finishedAt.getTime() - startedAt.getTime()
|
|
7312
7579
|
});
|
|
7313
|
-
await this.
|
|
7314
|
-
|
|
7315
|
-
|
|
7316
|
-
|
|
7580
|
+
return !await this.stepFanStillRunning(cp, result);
|
|
7581
|
+
}
|
|
7582
|
+
/**
|
|
7583
|
+
* The join barrier of a step fan: a result for one of its steps resumes the parent only once no
|
|
7584
|
+
* step of the fan is still `pending` (a settled wave), or on a failure (failFast surfaces it, and a
|
|
7585
|
+
* retry needs the parent to schedule it). Without it every result would replay the whole parent,
|
|
7586
|
+
* which makes a fan of N steps cost O(N²) checkpoint reads.
|
|
7587
|
+
*/
|
|
7588
|
+
async stepFanStillRunning(cp, result) {
|
|
7589
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
7590
|
+
if (!fan) return false;
|
|
7591
|
+
if (result.status === "failed") return false;
|
|
7592
|
+
const to = fan.firstSeq + fan.size - 1;
|
|
7593
|
+
const pending = this.store.countCheckpoints ? await this.store.countCheckpoints(result.runId, fan.firstSeq, to, [
|
|
7594
|
+
"pending"
|
|
7595
|
+
]) : (await this.store.listCheckpoints(result.runId)).filter((c) => c.seq >= fan.firstSeq && c.seq <= to && c.status === "pending").length;
|
|
7596
|
+
if (pending > 0) return true;
|
|
7597
|
+
this.noteRecent(this.stepFansFiredHere, `${result.runId}:${fan.firstSeq}`);
|
|
7598
|
+
return false;
|
|
7317
7599
|
}
|
|
7318
7600
|
/**
|
|
7319
7601
|
* Apply a remote workflow-turn {@link WorkflowDecision} durably — runs on WHATEVER instance receives
|
|
@@ -7666,6 +7948,110 @@ var RunGateway = class {
|
|
|
7666
7948
|
}
|
|
7667
7949
|
};
|
|
7668
7950
|
|
|
7951
|
+
// src/run-detail-summary.ts
|
|
7952
|
+
var FAN_SUMMARY_FAILURES = 20;
|
|
7953
|
+
var SUMMARY_CHILD_LIMIT = 200;
|
|
7954
|
+
var FAN_PAGE_MAX = 500;
|
|
7955
|
+
function foldedCount(cp) {
|
|
7956
|
+
if (!cp.name.startsWith(FOLDED_PREFIX)) return void 0;
|
|
7957
|
+
const values = cp.output?.values;
|
|
7958
|
+
return Array.isArray(values) ? values.length : 0;
|
|
7959
|
+
}
|
|
7960
|
+
__name(foldedCount, "foldedCount");
|
|
7961
|
+
function isFailedMember(cp) {
|
|
7962
|
+
if (cp.status === "failed") return true;
|
|
7963
|
+
if (!cp.name.startsWith("signal:child:") || cp.status !== "completed") return false;
|
|
7964
|
+
return cp.output?.ok === false;
|
|
7965
|
+
}
|
|
7966
|
+
__name(isFailedMember, "isFailedMember");
|
|
7967
|
+
function fanLabel(members) {
|
|
7968
|
+
if (members.some((cp) => cp.name.startsWith("signal:child:") || foldedCount(cp) !== void 0)) {
|
|
7969
|
+
return "child runs";
|
|
7970
|
+
}
|
|
7971
|
+
const first = members[0]?.name ?? "";
|
|
7972
|
+
return members.every((cp) => cp.name === first) ? first : "steps";
|
|
7973
|
+
}
|
|
7974
|
+
__name(fanLabel, "fanLabel");
|
|
7975
|
+
function summarizeFan(group, members) {
|
|
7976
|
+
const counts = {};
|
|
7977
|
+
const failures = [];
|
|
7978
|
+
let size = 0;
|
|
7979
|
+
let startedAt;
|
|
7980
|
+
let finishedAt;
|
|
7981
|
+
let firstSeq = Number.POSITIVE_INFINITY;
|
|
7982
|
+
let lastSeq = Number.NEGATIVE_INFINITY;
|
|
7983
|
+
for (const cp of members) {
|
|
7984
|
+
firstSeq = Math.min(firstSeq, cp.seq);
|
|
7985
|
+
lastSeq = Math.max(lastSeq, cp.seq);
|
|
7986
|
+
const folded = foldedCount(cp);
|
|
7987
|
+
const n = folded ?? 1;
|
|
7988
|
+
size += n;
|
|
7989
|
+
const status = isFailedMember(cp) ? "failed" : cp.status;
|
|
7990
|
+
counts[status] = (counts[status] ?? 0) + n;
|
|
7991
|
+
if (status === "failed" && failures.length < FAN_SUMMARY_FAILURES) failures.push(cp);
|
|
7992
|
+
if (cp.status !== "pending" && cp.status !== "running") {
|
|
7993
|
+
if (!startedAt || cp.startedAt < startedAt) startedAt = cp.startedAt;
|
|
7994
|
+
if (!finishedAt || cp.finishedAt > finishedAt) finishedAt = cp.finishedAt;
|
|
7995
|
+
} else if (!startedAt || cp.enqueuedAt < startedAt) {
|
|
7996
|
+
startedAt = cp.enqueuedAt;
|
|
7997
|
+
}
|
|
7998
|
+
}
|
|
7999
|
+
const parsed = parseFanGroup(group);
|
|
8000
|
+
const total = parsed?.size;
|
|
8001
|
+
return {
|
|
8002
|
+
group,
|
|
8003
|
+
label: fanLabel(members),
|
|
8004
|
+
firstSeq,
|
|
8005
|
+
lastSeq,
|
|
8006
|
+
size,
|
|
8007
|
+
total,
|
|
8008
|
+
notStarted: total === void 0 ? void 0 : Math.max(0, total - size),
|
|
8009
|
+
failFast: parsed?.failFast ?? false,
|
|
8010
|
+
counts,
|
|
8011
|
+
startedAt,
|
|
8012
|
+
finishedAt,
|
|
8013
|
+
failures
|
|
8014
|
+
};
|
|
8015
|
+
}
|
|
8016
|
+
__name(summarizeFan, "summarizeFan");
|
|
8017
|
+
function summarizeTimeline(timeline, over) {
|
|
8018
|
+
const byGroup = /* @__PURE__ */ new Map();
|
|
8019
|
+
for (const cp of timeline) {
|
|
8020
|
+
if (cp.parallelGroup === void 0) continue;
|
|
8021
|
+
const members = byGroup.get(cp.parallelGroup);
|
|
8022
|
+
if (members) members.push(cp);
|
|
8023
|
+
else byGroup.set(cp.parallelGroup, [
|
|
8024
|
+
cp
|
|
8025
|
+
]);
|
|
8026
|
+
}
|
|
8027
|
+
const summarized = /* @__PURE__ */ new Set();
|
|
8028
|
+
const fans = [];
|
|
8029
|
+
for (const [group, members] of byGroup) {
|
|
8030
|
+
if (members.length <= over && (parseFanGroup(group)?.size ?? 0) <= over) continue;
|
|
8031
|
+
summarized.add(group);
|
|
8032
|
+
fans.push(summarizeFan(group, members));
|
|
8033
|
+
}
|
|
8034
|
+
if (fans.length === 0) return {
|
|
8035
|
+
timeline,
|
|
8036
|
+
fans
|
|
8037
|
+
};
|
|
8038
|
+
return {
|
|
8039
|
+
timeline: timeline.filter((cp) => cp.parallelGroup === void 0 || !summarized.has(cp.parallelGroup)),
|
|
8040
|
+
fans: fans.sort((a, b) => a.firstSeq - b.firstSeq)
|
|
8041
|
+
};
|
|
8042
|
+
}
|
|
8043
|
+
__name(summarizeTimeline, "summarizeTimeline");
|
|
8044
|
+
function pageFan(timeline, group, query) {
|
|
8045
|
+
const matching = timeline.filter((cp) => cp.parallelGroup === group && (!query.failedOnly || isFailedMember(cp)));
|
|
8046
|
+
const offset = query.offset !== void 0 && Number.isFinite(query.offset) ? Math.max(0, Math.floor(query.offset)) : 0;
|
|
8047
|
+
const limit = query.limit !== void 0 && Number.isFinite(query.limit) ? Math.min(FAN_PAGE_MAX, Math.max(1, Math.floor(query.limit))) : 100;
|
|
8048
|
+
return {
|
|
8049
|
+
items: matching.slice(offset, offset + limit),
|
|
8050
|
+
total: matching.length
|
|
8051
|
+
};
|
|
8052
|
+
}
|
|
8053
|
+
__name(pageFan, "pageFan");
|
|
8054
|
+
|
|
7669
8055
|
// src/codec-state-store.ts
|
|
7670
8056
|
var CodecStateStore = class {
|
|
7671
8057
|
static {
|
|
@@ -7694,6 +8080,8 @@ var CodecStateStore = class {
|
|
|
7694
8080
|
/** Encodes like {@link saveCheckpoint}. Bound only when the inner store has it, so the engine's
|
|
7695
8081
|
* plain-save fallback still kicks in through the wrapper. */
|
|
7696
8082
|
saveCheckpointIfPending;
|
|
8083
|
+
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
|
|
8084
|
+
countCheckpoints;
|
|
7697
8085
|
/** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
|
|
7698
8086
|
* when the inner store prunes, so the retention poller still sees (and warns about) its absence. */
|
|
7699
8087
|
pruneTerminalRuns;
|
|
@@ -7723,6 +8111,8 @@ var CodecStateStore = class {
|
|
|
7723
8111
|
if (deleteRuns) this.deleteRuns = (ids) => deleteRuns.call(inner, ids);
|
|
7724
8112
|
const saveIfPending = inner.saveCheckpointIfPending;
|
|
7725
8113
|
if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
|
|
8114
|
+
const countCps = inner.countCheckpoints;
|
|
8115
|
+
if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
|
|
7726
8116
|
const prune = inner.pruneTerminalRuns;
|
|
7727
8117
|
if (prune) this.pruneTerminalRuns = (policy, now, limit) => prune.call(inner, policy, now, limit);
|
|
7728
8118
|
const count = inner.countRuns;
|
|
@@ -7853,8 +8243,8 @@ var CodecStateStore = class {
|
|
|
7853
8243
|
async listPendingRuns(limit, namespace) {
|
|
7854
8244
|
return (await this.inner.listPendingRuns(limit, namespace)).map((r) => this.decRun(r));
|
|
7855
8245
|
}
|
|
7856
|
-
async listDueTimers(nowMs, namespace) {
|
|
7857
|
-
return (await this.inner.listDueTimers(nowMs, namespace)).map((r) => this.decRun(r));
|
|
8246
|
+
async listDueTimers(nowMs, namespace, limit) {
|
|
8247
|
+
return (await this.inner.listDueTimers(nowMs, namespace, limit)).map((r) => this.decRun(r));
|
|
7858
8248
|
}
|
|
7859
8249
|
tryLockRun(runId, owner, leaseUntilMs, nowMs) {
|
|
7860
8250
|
return this.inner.tryLockRun(runId, owner, leaseUntilMs, nowMs);
|
|
@@ -8308,10 +8698,11 @@ var InMemoryStateStore = class {
|
|
|
8308
8698
|
...r
|
|
8309
8699
|
}));
|
|
8310
8700
|
}
|
|
8311
|
-
async listDueTimers(nowMs, namespace) {
|
|
8312
|
-
|
|
8701
|
+
async listDueTimers(nowMs, namespace, limit) {
|
|
8702
|
+
const due = [
|
|
8313
8703
|
...this.runs.values()
|
|
8314
|
-
].filter((r) => r.status === "suspended" && r.wakeAt !== void 0 && r.wakeAt <= nowMs && (namespace === void 0 || r.namespace === namespace)).
|
|
8704
|
+
].filter((r) => r.status === "suspended" && r.wakeAt !== void 0 && r.wakeAt <= nowMs && (namespace === void 0 || r.namespace === namespace)).sort((a, b) => (a.wakeAt ?? 0) - (b.wakeAt ?? 0));
|
|
8705
|
+
return (limit === void 0 ? due : due.slice(0, limit)).map((r) => ({
|
|
8315
8706
|
...r
|
|
8316
8707
|
}));
|
|
8317
8708
|
}
|
|
@@ -8585,6 +8976,13 @@ var InMemoryStateStore = class {
|
|
|
8585
8976
|
...latest
|
|
8586
8977
|
} : void 0;
|
|
8587
8978
|
}
|
|
8979
|
+
async countCheckpoints(runId, fromSeq, toSeq, statuses) {
|
|
8980
|
+
let n = 0;
|
|
8981
|
+
for (const cp of this.checkpoints.values()) {
|
|
8982
|
+
if (cp.runId === runId && cp.seq >= fromSeq && cp.seq <= toSeq && statuses.includes(cp.status)) n += 1;
|
|
8983
|
+
}
|
|
8984
|
+
return n;
|
|
8985
|
+
}
|
|
8588
8986
|
async listCheckpointsByNamePrefix(runId, prefixes) {
|
|
8589
8987
|
const rows = [
|
|
8590
8988
|
...this.checkpoints.values()
|
|
@@ -8659,6 +9057,7 @@ export {
|
|
|
8659
9057
|
EVENT_TOKEN_PREFIX,
|
|
8660
9058
|
Entities,
|
|
8661
9059
|
EventAccumulators,
|
|
9060
|
+
FAN_SUMMARY_FAILURES,
|
|
8662
9061
|
FatalError,
|
|
8663
9062
|
GatherError,
|
|
8664
9063
|
InMemoryAdmissionBackend,
|
|
@@ -8680,6 +9079,7 @@ export {
|
|
|
8680
9079
|
SINGLETON_ADMITTED_TAG,
|
|
8681
9080
|
STATE_STORE,
|
|
8682
9081
|
STATE_STORE_CANONICAL,
|
|
9082
|
+
SUMMARY_CHILD_LIMIT,
|
|
8683
9083
|
ScheduleAlreadyExistsError,
|
|
8684
9084
|
ScheduleClient,
|
|
8685
9085
|
ScheduleNotFoundError,
|
|
@@ -8706,6 +9106,7 @@ export {
|
|
|
8706
9106
|
buildWorkflowDirectory,
|
|
8707
9107
|
canRoute,
|
|
8708
9108
|
capableWorkers,
|
|
9109
|
+
childIdsOf,
|
|
8709
9110
|
classifyWaiterToken,
|
|
8710
9111
|
clearDurableExecutionWrappers,
|
|
8711
9112
|
clearTerminalSingletonAdmission,
|
|
@@ -8734,6 +9135,7 @@ export {
|
|
|
8734
9135
|
info,
|
|
8735
9136
|
isEngineMintedTag,
|
|
8736
9137
|
isEventToken,
|
|
9138
|
+
isFailedMember,
|
|
8737
9139
|
isLegacyDescriptor,
|
|
8738
9140
|
isNonEmptyRunScope,
|
|
8739
9141
|
isWorkflowControlFlowSignal,
|
|
@@ -8748,6 +9150,7 @@ export {
|
|
|
8748
9150
|
normalizeAttributeRows,
|
|
8749
9151
|
normalizeDescriptor,
|
|
8750
9152
|
observedAnnouncements,
|
|
9153
|
+
pageFan,
|
|
8751
9154
|
parentRunIdOf,
|
|
8752
9155
|
parseDuration,
|
|
8753
9156
|
partitionOfToken,
|
|
@@ -8777,6 +9180,8 @@ export {
|
|
|
8777
9180
|
sub,
|
|
8778
9181
|
subEvent,
|
|
8779
9182
|
subProcess,
|
|
9183
|
+
summarizeFan,
|
|
9184
|
+
summarizeTimeline,
|
|
8780
9185
|
tenantGroup,
|
|
8781
9186
|
useDurableExecution,
|
|
8782
9187
|
warn,
|