@dudousxd/nestjs-durable-core 0.85.8 → 0.85.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +264 -29
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +123 -4
- package/dist/index.d.ts +123 -4
- package/dist/index.js +261 -29
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -2561,6 +2561,20 @@ function codeAndDetails(err) {
|
|
|
2561
2561
|
}
|
|
2562
2562
|
__name(codeAndDetails, "codeAndDetails");
|
|
2563
2563
|
|
|
2564
|
+
// src/remote-event.ts
|
|
2565
|
+
var REMOTE_EVENT = Symbol.for("@dudousxd/nestjs-durable:remote-event");
|
|
2566
|
+
function markRemoteEvent(event) {
|
|
2567
|
+
Object.defineProperty(event, REMOTE_EVENT, {
|
|
2568
|
+
value: true
|
|
2569
|
+
});
|
|
2570
|
+
return event;
|
|
2571
|
+
}
|
|
2572
|
+
__name(markRemoteEvent, "markRemoteEvent");
|
|
2573
|
+
function isRemoteEvent(event) {
|
|
2574
|
+
return event[REMOTE_EVENT] === true;
|
|
2575
|
+
}
|
|
2576
|
+
__name(isRemoteEvent, "isRemoteEvent");
|
|
2577
|
+
|
|
2564
2578
|
// src/tenant-group.ts
|
|
2565
2579
|
function tenantGroup(baseGroup, tenant) {
|
|
2566
2580
|
return tenant !== void 0 && tenant !== "" && tenant !== "default" ? `${baseGroup}@${tenant}` : baseGroup;
|
|
@@ -4800,6 +4814,7 @@ var ANNOUNCED_ROUTES_TTL_MS = 5e3;
|
|
|
4800
4814
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
4801
4815
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
4802
4816
|
var STEP_FAN_CHUNK = 200;
|
|
4817
|
+
var FAN_REREAD_CHUNK = 1e3;
|
|
4803
4818
|
var PARENT_NOTIFY_ATTEMPTS = 4;
|
|
4804
4819
|
var PARENT_NOTIFY_BACKOFF_MS = 250;
|
|
4805
4820
|
var LEASE_RECHECK_MAX_MS = 2e3;
|
|
@@ -5264,10 +5279,10 @@ var WorkflowEngine = class {
|
|
|
5264
5279
|
this.controlPlane?.onControl((msg) => {
|
|
5265
5280
|
if (msg.from === this.instanceId) return;
|
|
5266
5281
|
if (msg.kind === "event") {
|
|
5267
|
-
this.deliver({
|
|
5282
|
+
this.deliver(markRemoteEvent({
|
|
5268
5283
|
...msg.event,
|
|
5269
5284
|
at: new Date(msg.event.at)
|
|
5270
|
-
});
|
|
5285
|
+
}));
|
|
5271
5286
|
} else if (msg.kind === "cancel") {
|
|
5272
5287
|
this.notifyCancelled(msg.runId);
|
|
5273
5288
|
} else if (msg.kind === "enqueued") {
|
|
@@ -7236,8 +7251,7 @@ var WorkflowEngine = class {
|
|
|
7236
7251
|
*/
|
|
7237
7252
|
async redriveSettledSteps(runId, seqs) {
|
|
7238
7253
|
for (const seq of seqs) {
|
|
7239
|
-
|
|
7240
|
-
if (cp && (cp.status === "completed" || cp.status === "failed")) {
|
|
7254
|
+
if (await this.stepSettled(runId, seq)) {
|
|
7241
7255
|
if (this.settledHere.has(`${runId}:${seq}`)) return;
|
|
7242
7256
|
const nowMs = this.clock();
|
|
7243
7257
|
if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
@@ -7253,6 +7267,17 @@ var WorkflowEngine = class {
|
|
|
7253
7267
|
}
|
|
7254
7268
|
}
|
|
7255
7269
|
}
|
|
7270
|
+
/** Whether the step at `seq` is settled: a `COUNT` when the store has one, else its checkpoint. */
|
|
7271
|
+
async stepSettled(runId, seq) {
|
|
7272
|
+
if (this.store.countCheckpoints) {
|
|
7273
|
+
return await this.store.countCheckpoints(runId, seq, seq, [
|
|
7274
|
+
"completed",
|
|
7275
|
+
"failed"
|
|
7276
|
+
]) > 0;
|
|
7277
|
+
}
|
|
7278
|
+
const cp = await this.store.getCheckpoint(runId, seq);
|
|
7279
|
+
return cp?.status === "completed" || cp?.status === "failed";
|
|
7280
|
+
}
|
|
7256
7281
|
/**
|
|
7257
7282
|
* {@link redriveSettledSteps} for a step fan: once the turn that suspended on it released its lease,
|
|
7258
7283
|
* re-drive the run if no step of the fan is still `pending` — the last result landed on another
|
|
@@ -7262,7 +7287,7 @@ var WorkflowEngine = class {
|
|
|
7262
7287
|
async redriveSettledStepFans(runId, fans) {
|
|
7263
7288
|
for (const fan of fans) {
|
|
7264
7289
|
if (this.stepFansFiredHere.has(`${runId}:${fan.from}`)) continue;
|
|
7265
|
-
if (await this.
|
|
7290
|
+
if (await this.stepFanHasPending(runId, fan.from, fan.to)) continue;
|
|
7266
7291
|
const nowMs = this.clock();
|
|
7267
7292
|
if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
7268
7293
|
return;
|
|
@@ -8724,7 +8749,7 @@ ${run.workflowVersion}`;
|
|
|
8724
8749
|
}
|
|
8725
8750
|
const wfPlan = await this.planRoute(tenantGroup(sanitizeQueueToken(remote.group), void 0), registered.requires);
|
|
8726
8751
|
if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
|
|
8727
|
-
const checkpoints = await this.
|
|
8752
|
+
const checkpoints = await this.replayCheckpoints(run.id);
|
|
8728
8753
|
await this.deliverFinishedChildren(run.id, checkpoints);
|
|
8729
8754
|
await this.deliverFinishedChildWaiters(run.id, checkpoints);
|
|
8730
8755
|
const handoff = liveHandoff(run.id, checkpoints);
|
|
@@ -9000,6 +9025,37 @@ ${run.workflowVersion}`;
|
|
|
9000
9025
|
* throttle for {@link rearmStepLease}, the step-scoped sibling of {@link decisionDeadlineRearms}. */
|
|
9001
9026
|
stepLeaseRearms = /* @__PURE__ */ new Map();
|
|
9002
9027
|
/**
|
|
9028
|
+
* {@link stepLeaseRearms} for the steps of a fan this instance sent, by run: one entry per sent
|
|
9029
|
+
* block of consecutive seqs instead of one per step, since they share the lease stamped at send.
|
|
9030
|
+
*/
|
|
9031
|
+
fanLeaseRearms = /* @__PURE__ */ new Map();
|
|
9032
|
+
noteFanLease(runId, seqs, lease) {
|
|
9033
|
+
if (this.remoteRedispatchMs == null || seqs.length === 0) return;
|
|
9034
|
+
let from = seqs[0];
|
|
9035
|
+
let to = from;
|
|
9036
|
+
for (const seq of seqs) {
|
|
9037
|
+
if (seq < from) from = seq;
|
|
9038
|
+
if (seq > to) to = seq;
|
|
9039
|
+
}
|
|
9040
|
+
if (to - from + 1 !== seqs.length) return;
|
|
9041
|
+
if (this.fanLeaseRearms.size > 1024) this.fanLeaseRearms.clear();
|
|
9042
|
+
const blocks = this.fanLeaseRearms.get(runId) ?? [];
|
|
9043
|
+
blocks.push({
|
|
9044
|
+
from,
|
|
9045
|
+
to,
|
|
9046
|
+
lookAgainAt: lease - Math.ceil(this.remoteRedispatchMs / 2)
|
|
9047
|
+
});
|
|
9048
|
+
this.fanLeaseRearms.set(runId, blocks);
|
|
9049
|
+
}
|
|
9050
|
+
fanLeaseLookAgainAt(runId, seq, now) {
|
|
9051
|
+
const blocks = this.fanLeaseRearms.get(runId);
|
|
9052
|
+
if (!blocks) return void 0;
|
|
9053
|
+
const live = blocks.filter((b) => now < b.lookAgainAt);
|
|
9054
|
+
if (live.length === 0) this.fanLeaseRearms.delete(runId);
|
|
9055
|
+
else if (live.length < blocks.length) this.fanLeaseRearms.set(runId, live);
|
|
9056
|
+
return live.find((b) => seq >= b.from && seq <= b.to)?.lookAgainAt;
|
|
9057
|
+
}
|
|
9058
|
+
/**
|
|
9003
9059
|
* A step-scoped heartbeat for a durably-suspended remote step: push its LEASE forward so a worker
|
|
9004
9060
|
* that is still working on it is never presumed dead and double-dispatched. The durable counterpart
|
|
9005
9061
|
* of {@link callRemoteInMemory}'s in-memory `heartbeatResets` rearm (which only exists on the
|
|
@@ -9028,7 +9084,7 @@ ${run.workflowVersion}`;
|
|
|
9028
9084
|
if (this.remoteRedispatchMs == null) return;
|
|
9029
9085
|
const id = stepId(runId, seq);
|
|
9030
9086
|
const now = this.clock();
|
|
9031
|
-
const lookAgainAt = this.stepLeaseRearms.get(id);
|
|
9087
|
+
const lookAgainAt = this.stepLeaseRearms.get(id) ?? this.fanLeaseLookAgainAt(runId, seq, now);
|
|
9032
9088
|
if (lookAgainAt !== void 0 && now < lookAgainAt) return;
|
|
9033
9089
|
try {
|
|
9034
9090
|
const cp = await this.store.getCheckpoint(runId, seq);
|
|
@@ -9351,7 +9407,7 @@ ${run.workflowVersion}`;
|
|
|
9351
9407
|
return this.singletonWaitResult(run);
|
|
9352
9408
|
}
|
|
9353
9409
|
const compensations = [];
|
|
9354
|
-
const snapshot = await this.
|
|
9410
|
+
const snapshot = await this.replayCheckpoints(run.id);
|
|
9355
9411
|
await this.deliverFinishedChildren(run.id, snapshot);
|
|
9356
9412
|
const replay = /* @__PURE__ */ new Map();
|
|
9357
9413
|
for (const cp of snapshot) replay.set(cp.seq, cp);
|
|
@@ -9681,7 +9737,7 @@ ${run.workflowVersion}`;
|
|
|
9681
9737
|
}), "callRemote"),
|
|
9682
9738
|
callRemoteAll: /* @__PURE__ */ __name((runId, positions, step, inputs, opts) => this.callRemoteAll(runId, positions, step, inputs, opts, {
|
|
9683
9739
|
namespace: parentNamespace
|
|
9684
|
-
}), "callRemoteAll"),
|
|
9740
|
+
}, replay), "callRemoteAll"),
|
|
9685
9741
|
// Defer so a fast child can't reentrantly resume a still-running parent. Stamp the child with
|
|
9686
9742
|
// the PARENT run's namespace (not this engine's) so a child belongs to the tenant/partition of
|
|
9687
9743
|
// the run it was spawned from — even when an operator (namespace: undefined, drives every
|
|
@@ -10038,7 +10094,7 @@ ${run.workflowVersion}`;
|
|
|
10038
10094
|
* concurrently with this one finds them already sent instead of sending them again. The join
|
|
10039
10095
|
* barrier ({@link stepFanStillRunning}) then wakes the parent once per wave, not once per result.
|
|
10040
10096
|
*/
|
|
10041
|
-
async callRemoteAll(runId, positions, step, inputs, opts, routing) {
|
|
10097
|
+
async callRemoteAll(runId, positions, step, inputs, opts, routing, replay) {
|
|
10042
10098
|
if (step.timeoutMs) {
|
|
10043
10099
|
throw new Error(`ctx.all: step "${step.name}" has a liveness timeoutMs; a step fan needs durable steps`);
|
|
10044
10100
|
}
|
|
@@ -10049,10 +10105,7 @@ ${run.workflowVersion}`;
|
|
|
10049
10105
|
const first = positions[0];
|
|
10050
10106
|
const last = positions[positions.length - 1];
|
|
10051
10107
|
const group = stepFanGroup(first, positions.length, opts.mode);
|
|
10052
|
-
const bySeq =
|
|
10053
|
-
for (const cp of await this.store.listCheckpoints(runId)) {
|
|
10054
|
-
if (cp.seq >= first && cp.seq <= last) bySeq.set(cp.seq, cp);
|
|
10055
|
-
}
|
|
10108
|
+
const bySeq = await this.fanCheckpoints(runId, positions, replay);
|
|
10056
10109
|
const outputs = new Array(inputs.length);
|
|
10057
10110
|
const failures = [];
|
|
10058
10111
|
const unsent = [];
|
|
@@ -10109,7 +10162,13 @@ ${run.workflowVersion}`;
|
|
|
10109
10162
|
}
|
|
10110
10163
|
if (inFlight === 0 && highest >= 0) this.turnSettledFans.get(runId)?.set(first, highest);
|
|
10111
10164
|
const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
|
|
10112
|
-
if (batch.length > 0)
|
|
10165
|
+
if (batch.length > 0) {
|
|
10166
|
+
const lease = await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
|
|
10167
|
+
if (lease !== void 0) {
|
|
10168
|
+
const at = this.stepLeaseWakeAt(lease);
|
|
10169
|
+
wakeAt = Math.min(wakeAt ?? at, at);
|
|
10170
|
+
}
|
|
10171
|
+
}
|
|
10113
10172
|
if (inFlight + batch.length > 0) {
|
|
10114
10173
|
this.awaitedStepFans.get(runId)?.push({
|
|
10115
10174
|
from: first,
|
|
@@ -10139,6 +10198,34 @@ ${run.workflowVersion}`;
|
|
|
10139
10198
|
return outputs;
|
|
10140
10199
|
}
|
|
10141
10200
|
/**
|
|
10201
|
+
* The checkpoints of a step fan's seqs, by seq. A settled one is taken from the turn's snapshot
|
|
10202
|
+
* (`replay`), since nothing but a turn of the run changes it once settled; one the snapshot holds
|
|
10203
|
+
* `pending`, or does not hold, is read again, as {@link callRemote} does for a single step: its
|
|
10204
|
+
* result may have landed since. The fan's last turn so reads nothing it already read.
|
|
10205
|
+
*/
|
|
10206
|
+
async fanCheckpoints(runId, positions, replay) {
|
|
10207
|
+
const bySeq = /* @__PURE__ */ new Map();
|
|
10208
|
+
const reread = [];
|
|
10209
|
+
for (const seq of positions) {
|
|
10210
|
+
const cp = replay?.get(seq);
|
|
10211
|
+
if (cp && (cp.status === "completed" || cp.status === "failed")) bySeq.set(seq, cp);
|
|
10212
|
+
else reread.push(seq);
|
|
10213
|
+
}
|
|
10214
|
+
if (reread.length === 0) return bySeq;
|
|
10215
|
+
if (reread.length === positions.length || !this.store.getCheckpoints) {
|
|
10216
|
+
const wanted = new Set(reread);
|
|
10217
|
+
for (const cp of await this.store.listCheckpoints(runId)) {
|
|
10218
|
+
if (wanted.has(cp.seq)) bySeq.set(cp.seq, cp);
|
|
10219
|
+
}
|
|
10220
|
+
return bySeq;
|
|
10221
|
+
}
|
|
10222
|
+
for (let at = 0; at < reread.length; at += FAN_REREAD_CHUNK) {
|
|
10223
|
+
const read = await this.store.getCheckpoints(runId, reread.slice(at, at + FAN_REREAD_CHUNK));
|
|
10224
|
+
for (const cp of read) bySeq.set(cp.seq, cp);
|
|
10225
|
+
}
|
|
10226
|
+
return bySeq;
|
|
10227
|
+
}
|
|
10228
|
+
/**
|
|
10142
10229
|
* Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
|
|
10143
10230
|
* {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
|
|
10144
10231
|
* batch, where the store and transport have them.
|
|
@@ -10147,6 +10234,7 @@ ${run.workflowVersion}`;
|
|
|
10147
10234
|
const workerGroup = stepGroup(step, routing?.namespace);
|
|
10148
10235
|
await this.ensureRoutable(workerGroup, step.requires);
|
|
10149
10236
|
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
10237
|
+
const lease = this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
|
|
10150
10238
|
const checkpoints = batch.map((i) => {
|
|
10151
10239
|
const seq = positions[i];
|
|
10152
10240
|
return {
|
|
@@ -10162,7 +10250,10 @@ ${run.workflowVersion}`;
|
|
|
10162
10250
|
parallelGroup: group,
|
|
10163
10251
|
enqueuedAt,
|
|
10164
10252
|
startedAt: enqueuedAt,
|
|
10165
|
-
finishedAt: enqueuedAt
|
|
10253
|
+
finishedAt: enqueuedAt,
|
|
10254
|
+
...lease !== void 0 ? {
|
|
10255
|
+
wakeAt: lease
|
|
10256
|
+
} : {}
|
|
10166
10257
|
};
|
|
10167
10258
|
});
|
|
10168
10259
|
if (this.store.saveCheckpoints) {
|
|
@@ -10195,20 +10286,36 @@ ${run.workflowVersion}`;
|
|
|
10195
10286
|
kind: "remote"
|
|
10196
10287
|
});
|
|
10197
10288
|
}
|
|
10289
|
+
if (lease !== void 0) {
|
|
10290
|
+
this.noteFanLease(runId, checkpoints.map((cp) => cp.seq), lease);
|
|
10291
|
+
}
|
|
10292
|
+
return lease;
|
|
10198
10293
|
}
|
|
10199
10294
|
/** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
|
|
10200
10295
|
noteSettledHere(key) {
|
|
10201
10296
|
this.noteRecent(this.settledHere, key);
|
|
10202
10297
|
}
|
|
10203
|
-
/**
|
|
10298
|
+
/**
|
|
10299
|
+
* Record `key` in a "happened here recently" map and drop the entries older than a lease. A key is
|
|
10300
|
+
* re-inserted at the end, so the map stays in the order its keys were noted and the stale ones are
|
|
10301
|
+
* all at its front: pruning stops at the first fresh entry instead of walking the whole map.
|
|
10302
|
+
*/
|
|
10204
10303
|
noteRecent(map, key) {
|
|
10205
10304
|
const now = this.clock();
|
|
10305
|
+
map.delete(key);
|
|
10206
10306
|
map.set(key, now);
|
|
10207
|
-
if (map.size < 1024) return;
|
|
10208
10307
|
for (const [k, at] of map) {
|
|
10209
|
-
if (now - at
|
|
10308
|
+
if (now - at <= this.leaseMs) break;
|
|
10309
|
+
map.delete(k);
|
|
10210
10310
|
}
|
|
10211
10311
|
}
|
|
10312
|
+
/**
|
|
10313
|
+
* The run's checkpoints as a turn replays them: {@link StateStore.listCheckpointsForReplay} when
|
|
10314
|
+
* the store has it (no input on a completed row), else {@link StateStore.listCheckpoints}.
|
|
10315
|
+
*/
|
|
10316
|
+
replayCheckpoints(runId) {
|
|
10317
|
+
return this.store.listCheckpointsForReplay ? this.store.listCheckpointsForReplay(runId) : this.store.listCheckpoints(runId);
|
|
10318
|
+
}
|
|
10212
10319
|
/** {@link StateStore.saveCheckpointIfPending}, or a plain save on a store without it. */
|
|
10213
10320
|
async savePendingCheckpoint(checkpoint) {
|
|
10214
10321
|
if (this.store.saveCheckpointIfPending) return this.store.saveCheckpointIfPending(checkpoint);
|
|
@@ -10235,11 +10342,59 @@ ${run.workflowVersion}`;
|
|
|
10235
10342
|
});
|
|
10236
10343
|
}
|
|
10237
10344
|
/**
|
|
10345
|
+
* Settle steps of one run at distinct seqs from their results ({@link StepSettlement}s, `settled[i]`
|
|
10346
|
+
* being the `pending` row `settlements[i]` settles): one
|
|
10347
|
+
* {@link StateStore.settleCheckpointsIfPending} call when the store has it, which writes only what
|
|
10348
|
+
* a result changes, else the whole rows through {@link savePendingCheckpoints}. Never rejects as a
|
|
10349
|
+
* whole, the same way.
|
|
10350
|
+
*/
|
|
10351
|
+
async settlePendingCheckpoints(settlements, settled) {
|
|
10352
|
+
const settle = this.store.settleCheckpointsIfPending?.bind(this.store);
|
|
10353
|
+
if (!settle) {
|
|
10354
|
+
return this.savePendingCheckpoints(settlements.map((s, i) => ({
|
|
10355
|
+
...settled[i],
|
|
10356
|
+
...s
|
|
10357
|
+
})));
|
|
10358
|
+
}
|
|
10359
|
+
if (settlements.length > 1) {
|
|
10360
|
+
try {
|
|
10361
|
+
const wrote = await settle(settlements);
|
|
10362
|
+
return settlements.map((_, i) => wrote[i] === true);
|
|
10363
|
+
} catch {
|
|
10364
|
+
}
|
|
10365
|
+
}
|
|
10366
|
+
const one = await Promise.allSettled(settlements.map((s) => settle([
|
|
10367
|
+
s
|
|
10368
|
+
])));
|
|
10369
|
+
return one.map((s) => s.status === "fulfilled" ? s.value[0] === true : {
|
|
10370
|
+
error: s.reason
|
|
10371
|
+
});
|
|
10372
|
+
}
|
|
10373
|
+
/**
|
|
10238
10374
|
* The checkpoints of one run at distinct `seqs`, each as its own outcome: one
|
|
10239
10375
|
* {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
|
|
10240
10376
|
* call rejecting falls back to one read each, so a failing read fails only its own entry.
|
|
10241
10377
|
*/
|
|
10242
10378
|
async readCheckpoints(runId, seqs) {
|
|
10379
|
+
const forSettle = this.store.settleCheckpointsIfPending ? this.store.getCheckpointsForSettle?.bind(this.store) : void 0;
|
|
10380
|
+
if (forSettle) {
|
|
10381
|
+
if (seqs.length > 1) {
|
|
10382
|
+
try {
|
|
10383
|
+
const bySeq = new Map((await forSettle(runId, seqs)).map((cp) => [
|
|
10384
|
+
cp.seq,
|
|
10385
|
+
cp
|
|
10386
|
+
]));
|
|
10387
|
+
return seqs.map((seq) => ({
|
|
10388
|
+
status: "fulfilled",
|
|
10389
|
+
value: bySeq.get(seq) ?? null
|
|
10390
|
+
}));
|
|
10391
|
+
} catch {
|
|
10392
|
+
}
|
|
10393
|
+
}
|
|
10394
|
+
return Promise.allSettled(seqs.map(async (seq) => (await forSettle(runId, [
|
|
10395
|
+
seq
|
|
10396
|
+
]))[0] ?? null));
|
|
10397
|
+
}
|
|
10243
10398
|
if (seqs.length > 1 && this.store.getCheckpoints) {
|
|
10244
10399
|
try {
|
|
10245
10400
|
const bySeq = new Map((await this.store.getCheckpoints(runId, seqs)).map((cp) => [
|
|
@@ -10388,10 +10543,11 @@ ${run.workflowVersion}`;
|
|
|
10388
10543
|
const run = await this.store.getRun(runId);
|
|
10389
10544
|
if (run && (run.status === "cancelled" || run.status === "completed")) return outcomes;
|
|
10390
10545
|
const finishedAt = /* @__PURE__ */ new Date();
|
|
10391
|
-
const
|
|
10546
|
+
const settlements = pending.map(({ index, cp }) => {
|
|
10392
10547
|
const result = results[index];
|
|
10393
10548
|
return {
|
|
10394
|
-
|
|
10549
|
+
runId,
|
|
10550
|
+
seq: cp.seq,
|
|
10395
10551
|
status: result.status,
|
|
10396
10552
|
output: result.status === "completed" ? result.output : cp.output,
|
|
10397
10553
|
error: result.error,
|
|
@@ -10400,7 +10556,7 @@ ${run.workflowVersion}`;
|
|
|
10400
10556
|
finishedAt
|
|
10401
10557
|
};
|
|
10402
10558
|
});
|
|
10403
|
-
const claims = await this.
|
|
10559
|
+
const claims = await this.settlePendingCheckpoints(settlements, pending.map(({ cp }) => cp));
|
|
10404
10560
|
const settled = [];
|
|
10405
10561
|
claims.forEach((claim, k) => {
|
|
10406
10562
|
const { index, cp } = pending[k];
|
|
@@ -10414,7 +10570,7 @@ ${run.workflowVersion}`;
|
|
|
10414
10570
|
};
|
|
10415
10571
|
});
|
|
10416
10572
|
for (const { cp, result } of settled) {
|
|
10417
|
-
this.noteSettledHere(`${runId}:${result.seq}`);
|
|
10573
|
+
if (!parseStepFanGroup(cp.parallelGroup)) this.noteSettledHere(`${runId}:${result.seq}`);
|
|
10418
10574
|
const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
|
|
10419
10575
|
this.emit({
|
|
10420
10576
|
type: result.status === "completed" ? "step.completed" : "step.failed",
|
|
@@ -10479,21 +10635,26 @@ ${run.workflowVersion}`;
|
|
|
10479
10635
|
let closed = barriers.get(fan.firstSeq);
|
|
10480
10636
|
if (!closed) {
|
|
10481
10637
|
const lastSeq = fan.firstSeq + fan.size - 1;
|
|
10482
|
-
closed = queuedIn(fan.firstSeq, lastSeq) ? Promise.resolve(true) : this.
|
|
10638
|
+
closed = queuedIn(fan.firstSeq, lastSeq) ? Promise.resolve(true) : this.stepFanHasPending(result.runId, fan.firstSeq, lastSeq).catch(() => false);
|
|
10483
10639
|
barriers.set(fan.firstSeq, closed);
|
|
10484
10640
|
}
|
|
10485
10641
|
if (await closed) return true;
|
|
10486
10642
|
this.noteRecent(this.stepFansFiredHere, `${result.runId}:${fan.firstSeq}`);
|
|
10487
10643
|
return false;
|
|
10488
10644
|
}
|
|
10489
|
-
/**
|
|
10490
|
-
async
|
|
10491
|
-
if (this.store.
|
|
10492
|
-
return this.store.
|
|
10645
|
+
/** Whether any step of a step fan's seq range is still `pending`. */
|
|
10646
|
+
async stepFanHasPending(runId, from, to) {
|
|
10647
|
+
if (this.store.hasCheckpoints) {
|
|
10648
|
+
return this.store.hasCheckpoints(runId, from, to, [
|
|
10493
10649
|
"pending"
|
|
10494
10650
|
]);
|
|
10495
10651
|
}
|
|
10496
|
-
|
|
10652
|
+
if (this.store.countCheckpoints) {
|
|
10653
|
+
return await this.store.countCheckpoints(runId, from, to, [
|
|
10654
|
+
"pending"
|
|
10655
|
+
]) > 0;
|
|
10656
|
+
}
|
|
10657
|
+
return (await this.store.listCheckpoints(runId)).some((c) => c.seq >= from && c.seq <= to && c.status === "pending");
|
|
10497
10658
|
}
|
|
10498
10659
|
/**
|
|
10499
10660
|
* Apply a remote workflow-turn {@link WorkflowDecision} durably — runs on WHATEVER instance receives
|
|
@@ -11313,10 +11474,20 @@ var CodecStateStore = class {
|
|
|
11313
11474
|
saveCheckpointIfPending;
|
|
11314
11475
|
/** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
|
|
11315
11476
|
saveCheckpointsIfPending;
|
|
11477
|
+
/** Encodes each settlement's payloads like {@link saveCheckpoint}. Bound only when the inner store
|
|
11478
|
+
* has it, so the engine's whole-row fallback still kicks in through the wrapper. */
|
|
11479
|
+
settleCheckpointsIfPending;
|
|
11316
11480
|
/** Encodes each checkpoint written. Bound only when the inner store has it. */
|
|
11317
11481
|
saveCheckpoints;
|
|
11482
|
+
/** Decodes each checkpoint read like {@link listCheckpoints}. Bound only when the inner store has
|
|
11483
|
+
* it, so the engine's full-read fallback still kicks in through the wrapper. */
|
|
11484
|
+
listCheckpointsForReplay;
|
|
11485
|
+
/** Decodes each checkpoint read. Bound only when the inner store has it. */
|
|
11486
|
+
getCheckpointsForSettle;
|
|
11318
11487
|
/** Decodes each checkpoint read. Bound only when the inner store has it. */
|
|
11319
11488
|
getCheckpoints;
|
|
11489
|
+
/** Forwarded verbatim — reads no payload. Bound only when the inner store has it. */
|
|
11490
|
+
hasCheckpoints;
|
|
11320
11491
|
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
|
|
11321
11492
|
countCheckpoints;
|
|
11322
11493
|
/** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
|
|
@@ -11367,10 +11538,18 @@ var CodecStateStore = class {
|
|
|
11367
11538
|
if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
|
|
11368
11539
|
const saveManyIfPending = inner.saveCheckpointsIfPending;
|
|
11369
11540
|
if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11541
|
+
const settleMany = inner.settleCheckpointsIfPending;
|
|
11542
|
+
if (settleMany) this.settleCheckpointsIfPending = (settlements) => settleMany.call(inner, settlements.map((s) => this.encSettlement(s)));
|
|
11370
11543
|
const saveMany = inner.saveCheckpoints;
|
|
11371
11544
|
if (saveMany) this.saveCheckpoints = (cps) => saveMany.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11545
|
+
const listForReplay = inner.listCheckpointsForReplay;
|
|
11546
|
+
if (listForReplay) this.listCheckpointsForReplay = async (runId) => (await listForReplay.call(inner, runId)).map((cp) => this.decCp(cp));
|
|
11547
|
+
const getForSettle = inner.getCheckpointsForSettle;
|
|
11548
|
+
if (getForSettle) this.getCheckpointsForSettle = async (runId, seqs) => (await getForSettle.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
|
|
11372
11549
|
const getMany = inner.getCheckpoints;
|
|
11373
11550
|
if (getMany) this.getCheckpoints = async (runId, seqs) => (await getMany.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
|
|
11551
|
+
const hasCps = inner.hasCheckpoints;
|
|
11552
|
+
if (hasCps) this.hasCheckpoints = (runId, from, to, statuses) => hasCps.call(inner, runId, from, to, statuses);
|
|
11374
11553
|
const countCps = inner.countCheckpoints;
|
|
11375
11554
|
if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
|
|
11376
11555
|
const stats = inner.checkpointStats;
|
|
@@ -11460,6 +11639,21 @@ var CodecStateStore = class {
|
|
|
11460
11639
|
}
|
|
11461
11640
|
return next;
|
|
11462
11641
|
}
|
|
11642
|
+
encSettlement(settlement) {
|
|
11643
|
+
const next = {
|
|
11644
|
+
...settlement,
|
|
11645
|
+
output: this.enc(settlement.output)
|
|
11646
|
+
};
|
|
11647
|
+
if (this.extended) {
|
|
11648
|
+
if (settlement.events !== void 0) {
|
|
11649
|
+
next.events = this.enc(settlement.events);
|
|
11650
|
+
}
|
|
11651
|
+
if (settlement.error !== void 0) {
|
|
11652
|
+
next.error = this.enc(settlement.error);
|
|
11653
|
+
}
|
|
11654
|
+
}
|
|
11655
|
+
return next;
|
|
11656
|
+
}
|
|
11463
11657
|
decCp(cp) {
|
|
11464
11658
|
if (!cp) return cp;
|
|
11465
11659
|
const next = {
|
|
@@ -11649,6 +11843,7 @@ function collectMetrics(engine) {
|
|
|
11649
11843
|
return entry;
|
|
11650
11844
|
}, "wf");
|
|
11651
11845
|
const stop = engine.subscribe((e) => {
|
|
11846
|
+
if (isRemoteEvent(e)) return;
|
|
11652
11847
|
switch (e.type) {
|
|
11653
11848
|
case "run.started":
|
|
11654
11849
|
m.runs.started += 1;
|
|
@@ -12019,6 +12214,18 @@ var InMemoryStateStore = class {
|
|
|
12019
12214
|
] : [];
|
|
12020
12215
|
});
|
|
12021
12216
|
}
|
|
12217
|
+
async getCheckpointsForSettle(runId, seqs) {
|
|
12218
|
+
return [
|
|
12219
|
+
...new Set(seqs)
|
|
12220
|
+
].sort((a, b) => a - b).flatMap((seq) => {
|
|
12221
|
+
const cp = this.checkpoints.get(this.key(runId, seq));
|
|
12222
|
+
if (!cp) return [];
|
|
12223
|
+
const { input: _input, ...rest } = cp;
|
|
12224
|
+
return [
|
|
12225
|
+
rest
|
|
12226
|
+
];
|
|
12227
|
+
});
|
|
12228
|
+
}
|
|
12022
12229
|
async saveCheckpoint(checkpoint) {
|
|
12023
12230
|
this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
|
|
12024
12231
|
...checkpoint
|
|
@@ -12049,6 +12256,18 @@ var InMemoryStateStore = class {
|
|
|
12049
12256
|
return true;
|
|
12050
12257
|
});
|
|
12051
12258
|
}
|
|
12259
|
+
async settleCheckpointsIfPending(settlements) {
|
|
12260
|
+
return settlements.map((settlement) => {
|
|
12261
|
+
const key = this.key(settlement.runId, settlement.seq);
|
|
12262
|
+
const cp = this.checkpoints.get(key);
|
|
12263
|
+
if (cp?.status !== "pending") return false;
|
|
12264
|
+
this.checkpoints.set(key, {
|
|
12265
|
+
...cp,
|
|
12266
|
+
...settlement
|
|
12267
|
+
});
|
|
12268
|
+
return true;
|
|
12269
|
+
});
|
|
12270
|
+
}
|
|
12052
12271
|
async listOrphanedRuns(nowMs, limit, namespace, ownedBy) {
|
|
12053
12272
|
return [
|
|
12054
12273
|
...this.runs.values()
|
|
@@ -12365,6 +12584,13 @@ var InMemoryStateStore = class {
|
|
|
12365
12584
|
this.checkpointRowsRead += rows.length;
|
|
12366
12585
|
return rows;
|
|
12367
12586
|
}
|
|
12587
|
+
async listCheckpointsForReplay(runId) {
|
|
12588
|
+
return (await this.listCheckpoints(runId)).map((cp) => {
|
|
12589
|
+
if (cp.status !== "completed") return cp;
|
|
12590
|
+
const { input: _input, ...rest } = cp;
|
|
12591
|
+
return rest;
|
|
12592
|
+
});
|
|
12593
|
+
}
|
|
12368
12594
|
runCheckpoints(runId) {
|
|
12369
12595
|
return [
|
|
12370
12596
|
...this.checkpoints.values()
|
|
@@ -12392,6 +12618,9 @@ var InMemoryStateStore = class {
|
|
|
12392
12618
|
...latest
|
|
12393
12619
|
} : void 0;
|
|
12394
12620
|
}
|
|
12621
|
+
async hasCheckpoints(runId, fromSeq, toSeq, statuses) {
|
|
12622
|
+
return await this.countCheckpoints(runId, fromSeq, toSeq, statuses) > 0;
|
|
12623
|
+
}
|
|
12395
12624
|
async countCheckpoints(runId, fromSeq, toSeq, statuses) {
|
|
12396
12625
|
let n = 0;
|
|
12397
12626
|
for (const cp of this.checkpoints.values()) {
|
|
@@ -12584,6 +12813,7 @@ export {
|
|
|
12584
12813
|
NestedWorkflowCallError,
|
|
12585
12814
|
NonDeterminismError,
|
|
12586
12815
|
QueueController,
|
|
12816
|
+
REMOTE_EVENT,
|
|
12587
12817
|
RUN_HISTORY_FIXTURE_VERSION,
|
|
12588
12818
|
RUN_VALUE_FACET_LIMIT,
|
|
12589
12819
|
RUN_VALUE_FACET_SCAN,
|
|
@@ -12674,11 +12904,13 @@ export {
|
|
|
12674
12904
|
isFailedMember,
|
|
12675
12905
|
isLegacyDescriptor,
|
|
12676
12906
|
isNonEmptyRunScope,
|
|
12907
|
+
isRemoteEvent,
|
|
12677
12908
|
isWorkflowControlFlowSignal,
|
|
12678
12909
|
legacySingletonHolders,
|
|
12679
12910
|
listSignalWaitersOfRuns,
|
|
12680
12911
|
liveHandoff,
|
|
12681
12912
|
log,
|
|
12913
|
+
markRemoteEvent,
|
|
12682
12914
|
matchesAttributes,
|
|
12683
12915
|
mergeRunFacetRows,
|
|
12684
12916
|
mergeRunValueFacetRows,
|