@dudousxd/nestjs-durable-core 0.85.8 → 0.85.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +264 -29
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +123 -4
- package/dist/index.d.ts +123 -4
- package/dist/index.js +261 -29
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -56,6 +56,7 @@ __export(index_exports, {
|
|
|
56
56
|
NestedWorkflowCallError: () => NestedWorkflowCallError,
|
|
57
57
|
NonDeterminismError: () => NonDeterminismError,
|
|
58
58
|
QueueController: () => QueueController,
|
|
59
|
+
REMOTE_EVENT: () => REMOTE_EVENT,
|
|
59
60
|
RUN_HISTORY_FIXTURE_VERSION: () => RUN_HISTORY_FIXTURE_VERSION,
|
|
60
61
|
RUN_VALUE_FACET_LIMIT: () => RUN_VALUE_FACET_LIMIT,
|
|
61
62
|
RUN_VALUE_FACET_SCAN: () => RUN_VALUE_FACET_SCAN,
|
|
@@ -146,11 +147,13 @@ __export(index_exports, {
|
|
|
146
147
|
isFailedMember: () => isFailedMember,
|
|
147
148
|
isLegacyDescriptor: () => isLegacyDescriptor,
|
|
148
149
|
isNonEmptyRunScope: () => isNonEmptyRunScope,
|
|
150
|
+
isRemoteEvent: () => isRemoteEvent,
|
|
149
151
|
isWorkflowControlFlowSignal: () => isWorkflowControlFlowSignal,
|
|
150
152
|
legacySingletonHolders: () => legacySingletonHolders,
|
|
151
153
|
listSignalWaitersOfRuns: () => listSignalWaitersOfRuns,
|
|
152
154
|
liveHandoff: () => liveHandoff,
|
|
153
155
|
log: () => log,
|
|
156
|
+
markRemoteEvent: () => markRemoteEvent,
|
|
154
157
|
matchesAttributes: () => matchesAttributes,
|
|
155
158
|
mergeRunFacetRows: () => mergeRunFacetRows,
|
|
156
159
|
mergeRunValueFacetRows: () => mergeRunValueFacetRows,
|
|
@@ -2775,6 +2778,20 @@ function codeAndDetails(err) {
|
|
|
2775
2778
|
}
|
|
2776
2779
|
__name(codeAndDetails, "codeAndDetails");
|
|
2777
2780
|
|
|
2781
|
+
// src/remote-event.ts
|
|
2782
|
+
var REMOTE_EVENT = Symbol.for("@dudousxd/nestjs-durable:remote-event");
|
|
2783
|
+
function markRemoteEvent(event) {
|
|
2784
|
+
Object.defineProperty(event, REMOTE_EVENT, {
|
|
2785
|
+
value: true
|
|
2786
|
+
});
|
|
2787
|
+
return event;
|
|
2788
|
+
}
|
|
2789
|
+
__name(markRemoteEvent, "markRemoteEvent");
|
|
2790
|
+
function isRemoteEvent(event) {
|
|
2791
|
+
return event[REMOTE_EVENT] === true;
|
|
2792
|
+
}
|
|
2793
|
+
__name(isRemoteEvent, "isRemoteEvent");
|
|
2794
|
+
|
|
2778
2795
|
// src/tenant-group.ts
|
|
2779
2796
|
function tenantGroup(baseGroup, tenant) {
|
|
2780
2797
|
return tenant !== void 0 && tenant !== "" && tenant !== "default" ? `${baseGroup}@${tenant}` : baseGroup;
|
|
@@ -5014,6 +5031,7 @@ var ANNOUNCED_ROUTES_TTL_MS = 5e3;
|
|
|
5014
5031
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
5015
5032
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
5016
5033
|
var STEP_FAN_CHUNK = 200;
|
|
5034
|
+
var FAN_REREAD_CHUNK = 1e3;
|
|
5017
5035
|
var PARENT_NOTIFY_ATTEMPTS = 4;
|
|
5018
5036
|
var PARENT_NOTIFY_BACKOFF_MS = 250;
|
|
5019
5037
|
var LEASE_RECHECK_MAX_MS = 2e3;
|
|
@@ -5478,10 +5496,10 @@ var WorkflowEngine = class {
|
|
|
5478
5496
|
this.controlPlane?.onControl((msg) => {
|
|
5479
5497
|
if (msg.from === this.instanceId) return;
|
|
5480
5498
|
if (msg.kind === "event") {
|
|
5481
|
-
this.deliver({
|
|
5499
|
+
this.deliver(markRemoteEvent({
|
|
5482
5500
|
...msg.event,
|
|
5483
5501
|
at: new Date(msg.event.at)
|
|
5484
|
-
});
|
|
5502
|
+
}));
|
|
5485
5503
|
} else if (msg.kind === "cancel") {
|
|
5486
5504
|
this.notifyCancelled(msg.runId);
|
|
5487
5505
|
} else if (msg.kind === "enqueued") {
|
|
@@ -7450,8 +7468,7 @@ var WorkflowEngine = class {
|
|
|
7450
7468
|
*/
|
|
7451
7469
|
async redriveSettledSteps(runId, seqs) {
|
|
7452
7470
|
for (const seq of seqs) {
|
|
7453
|
-
|
|
7454
|
-
if (cp && (cp.status === "completed" || cp.status === "failed")) {
|
|
7471
|
+
if (await this.stepSettled(runId, seq)) {
|
|
7455
7472
|
if (this.settledHere.has(`${runId}:${seq}`)) return;
|
|
7456
7473
|
const nowMs = this.clock();
|
|
7457
7474
|
if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
@@ -7467,6 +7484,17 @@ var WorkflowEngine = class {
|
|
|
7467
7484
|
}
|
|
7468
7485
|
}
|
|
7469
7486
|
}
|
|
7487
|
+
/** Whether the step at `seq` is settled: a `COUNT` when the store has one, else its checkpoint. */
|
|
7488
|
+
async stepSettled(runId, seq) {
|
|
7489
|
+
if (this.store.countCheckpoints) {
|
|
7490
|
+
return await this.store.countCheckpoints(runId, seq, seq, [
|
|
7491
|
+
"completed",
|
|
7492
|
+
"failed"
|
|
7493
|
+
]) > 0;
|
|
7494
|
+
}
|
|
7495
|
+
const cp = await this.store.getCheckpoint(runId, seq);
|
|
7496
|
+
return cp?.status === "completed" || cp?.status === "failed";
|
|
7497
|
+
}
|
|
7470
7498
|
/**
|
|
7471
7499
|
* {@link redriveSettledSteps} for a step fan: once the turn that suspended on it released its lease,
|
|
7472
7500
|
* re-drive the run if no step of the fan is still `pending` — the last result landed on another
|
|
@@ -7476,7 +7504,7 @@ var WorkflowEngine = class {
|
|
|
7476
7504
|
async redriveSettledStepFans(runId, fans) {
|
|
7477
7505
|
for (const fan of fans) {
|
|
7478
7506
|
if (this.stepFansFiredHere.has(`${runId}:${fan.from}`)) continue;
|
|
7479
|
-
if (await this.
|
|
7507
|
+
if (await this.stepFanHasPending(runId, fan.from, fan.to)) continue;
|
|
7480
7508
|
const nowMs = this.clock();
|
|
7481
7509
|
if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
7482
7510
|
return;
|
|
@@ -8938,7 +8966,7 @@ ${run.workflowVersion}`;
|
|
|
8938
8966
|
}
|
|
8939
8967
|
const wfPlan = await this.planRoute(tenantGroup(sanitizeQueueToken(remote.group), void 0), registered.requires);
|
|
8940
8968
|
if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
|
|
8941
|
-
const checkpoints = await this.
|
|
8969
|
+
const checkpoints = await this.replayCheckpoints(run.id);
|
|
8942
8970
|
await this.deliverFinishedChildren(run.id, checkpoints);
|
|
8943
8971
|
await this.deliverFinishedChildWaiters(run.id, checkpoints);
|
|
8944
8972
|
const handoff = liveHandoff(run.id, checkpoints);
|
|
@@ -9214,6 +9242,37 @@ ${run.workflowVersion}`;
|
|
|
9214
9242
|
* throttle for {@link rearmStepLease}, the step-scoped sibling of {@link decisionDeadlineRearms}. */
|
|
9215
9243
|
stepLeaseRearms = /* @__PURE__ */ new Map();
|
|
9216
9244
|
/**
|
|
9245
|
+
* {@link stepLeaseRearms} for the steps of a fan this instance sent, by run: one entry per sent
|
|
9246
|
+
* block of consecutive seqs instead of one per step, since they share the lease stamped at send.
|
|
9247
|
+
*/
|
|
9248
|
+
fanLeaseRearms = /* @__PURE__ */ new Map();
|
|
9249
|
+
noteFanLease(runId, seqs, lease) {
|
|
9250
|
+
if (this.remoteRedispatchMs == null || seqs.length === 0) return;
|
|
9251
|
+
let from = seqs[0];
|
|
9252
|
+
let to = from;
|
|
9253
|
+
for (const seq of seqs) {
|
|
9254
|
+
if (seq < from) from = seq;
|
|
9255
|
+
if (seq > to) to = seq;
|
|
9256
|
+
}
|
|
9257
|
+
if (to - from + 1 !== seqs.length) return;
|
|
9258
|
+
if (this.fanLeaseRearms.size > 1024) this.fanLeaseRearms.clear();
|
|
9259
|
+
const blocks = this.fanLeaseRearms.get(runId) ?? [];
|
|
9260
|
+
blocks.push({
|
|
9261
|
+
from,
|
|
9262
|
+
to,
|
|
9263
|
+
lookAgainAt: lease - Math.ceil(this.remoteRedispatchMs / 2)
|
|
9264
|
+
});
|
|
9265
|
+
this.fanLeaseRearms.set(runId, blocks);
|
|
9266
|
+
}
|
|
9267
|
+
fanLeaseLookAgainAt(runId, seq, now) {
|
|
9268
|
+
const blocks = this.fanLeaseRearms.get(runId);
|
|
9269
|
+
if (!blocks) return void 0;
|
|
9270
|
+
const live = blocks.filter((b) => now < b.lookAgainAt);
|
|
9271
|
+
if (live.length === 0) this.fanLeaseRearms.delete(runId);
|
|
9272
|
+
else if (live.length < blocks.length) this.fanLeaseRearms.set(runId, live);
|
|
9273
|
+
return live.find((b) => seq >= b.from && seq <= b.to)?.lookAgainAt;
|
|
9274
|
+
}
|
|
9275
|
+
/**
|
|
9217
9276
|
* A step-scoped heartbeat for a durably-suspended remote step: push its LEASE forward so a worker
|
|
9218
9277
|
* that is still working on it is never presumed dead and double-dispatched. The durable counterpart
|
|
9219
9278
|
* of {@link callRemoteInMemory}'s in-memory `heartbeatResets` rearm (which only exists on the
|
|
@@ -9242,7 +9301,7 @@ ${run.workflowVersion}`;
|
|
|
9242
9301
|
if (this.remoteRedispatchMs == null) return;
|
|
9243
9302
|
const id = stepId(runId, seq);
|
|
9244
9303
|
const now = this.clock();
|
|
9245
|
-
const lookAgainAt = this.stepLeaseRearms.get(id);
|
|
9304
|
+
const lookAgainAt = this.stepLeaseRearms.get(id) ?? this.fanLeaseLookAgainAt(runId, seq, now);
|
|
9246
9305
|
if (lookAgainAt !== void 0 && now < lookAgainAt) return;
|
|
9247
9306
|
try {
|
|
9248
9307
|
const cp = await this.store.getCheckpoint(runId, seq);
|
|
@@ -9565,7 +9624,7 @@ ${run.workflowVersion}`;
|
|
|
9565
9624
|
return this.singletonWaitResult(run);
|
|
9566
9625
|
}
|
|
9567
9626
|
const compensations = [];
|
|
9568
|
-
const snapshot = await this.
|
|
9627
|
+
const snapshot = await this.replayCheckpoints(run.id);
|
|
9569
9628
|
await this.deliverFinishedChildren(run.id, snapshot);
|
|
9570
9629
|
const replay = /* @__PURE__ */ new Map();
|
|
9571
9630
|
for (const cp of snapshot) replay.set(cp.seq, cp);
|
|
@@ -9895,7 +9954,7 @@ ${run.workflowVersion}`;
|
|
|
9895
9954
|
}), "callRemote"),
|
|
9896
9955
|
callRemoteAll: /* @__PURE__ */ __name((runId, positions, step, inputs, opts) => this.callRemoteAll(runId, positions, step, inputs, opts, {
|
|
9897
9956
|
namespace: parentNamespace
|
|
9898
|
-
}), "callRemoteAll"),
|
|
9957
|
+
}, replay), "callRemoteAll"),
|
|
9899
9958
|
// Defer so a fast child can't reentrantly resume a still-running parent. Stamp the child with
|
|
9900
9959
|
// the PARENT run's namespace (not this engine's) so a child belongs to the tenant/partition of
|
|
9901
9960
|
// the run it was spawned from — even when an operator (namespace: undefined, drives every
|
|
@@ -10252,7 +10311,7 @@ ${run.workflowVersion}`;
|
|
|
10252
10311
|
* concurrently with this one finds them already sent instead of sending them again. The join
|
|
10253
10312
|
* barrier ({@link stepFanStillRunning}) then wakes the parent once per wave, not once per result.
|
|
10254
10313
|
*/
|
|
10255
|
-
async callRemoteAll(runId, positions, step, inputs, opts, routing) {
|
|
10314
|
+
async callRemoteAll(runId, positions, step, inputs, opts, routing, replay) {
|
|
10256
10315
|
if (step.timeoutMs) {
|
|
10257
10316
|
throw new Error(`ctx.all: step "${step.name}" has a liveness timeoutMs; a step fan needs durable steps`);
|
|
10258
10317
|
}
|
|
@@ -10263,10 +10322,7 @@ ${run.workflowVersion}`;
|
|
|
10263
10322
|
const first = positions[0];
|
|
10264
10323
|
const last = positions[positions.length - 1];
|
|
10265
10324
|
const group = stepFanGroup(first, positions.length, opts.mode);
|
|
10266
|
-
const bySeq =
|
|
10267
|
-
for (const cp of await this.store.listCheckpoints(runId)) {
|
|
10268
|
-
if (cp.seq >= first && cp.seq <= last) bySeq.set(cp.seq, cp);
|
|
10269
|
-
}
|
|
10325
|
+
const bySeq = await this.fanCheckpoints(runId, positions, replay);
|
|
10270
10326
|
const outputs = new Array(inputs.length);
|
|
10271
10327
|
const failures = [];
|
|
10272
10328
|
const unsent = [];
|
|
@@ -10323,7 +10379,13 @@ ${run.workflowVersion}`;
|
|
|
10323
10379
|
}
|
|
10324
10380
|
if (inFlight === 0 && highest >= 0) this.turnSettledFans.get(runId)?.set(first, highest);
|
|
10325
10381
|
const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
|
|
10326
|
-
if (batch.length > 0)
|
|
10382
|
+
if (batch.length > 0) {
|
|
10383
|
+
const lease = await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
|
|
10384
|
+
if (lease !== void 0) {
|
|
10385
|
+
const at = this.stepLeaseWakeAt(lease);
|
|
10386
|
+
wakeAt = Math.min(wakeAt ?? at, at);
|
|
10387
|
+
}
|
|
10388
|
+
}
|
|
10327
10389
|
if (inFlight + batch.length > 0) {
|
|
10328
10390
|
this.awaitedStepFans.get(runId)?.push({
|
|
10329
10391
|
from: first,
|
|
@@ -10353,6 +10415,34 @@ ${run.workflowVersion}`;
|
|
|
10353
10415
|
return outputs;
|
|
10354
10416
|
}
|
|
10355
10417
|
/**
|
|
10418
|
+
* The checkpoints of a step fan's seqs, by seq. A settled one is taken from the turn's snapshot
|
|
10419
|
+
* (`replay`), since nothing but a turn of the run changes it once settled; one the snapshot holds
|
|
10420
|
+
* `pending`, or does not hold, is read again, as {@link callRemote} does for a single step: its
|
|
10421
|
+
* result may have landed since. The fan's last turn so reads nothing it already read.
|
|
10422
|
+
*/
|
|
10423
|
+
async fanCheckpoints(runId, positions, replay) {
|
|
10424
|
+
const bySeq = /* @__PURE__ */ new Map();
|
|
10425
|
+
const reread = [];
|
|
10426
|
+
for (const seq of positions) {
|
|
10427
|
+
const cp = replay?.get(seq);
|
|
10428
|
+
if (cp && (cp.status === "completed" || cp.status === "failed")) bySeq.set(seq, cp);
|
|
10429
|
+
else reread.push(seq);
|
|
10430
|
+
}
|
|
10431
|
+
if (reread.length === 0) return bySeq;
|
|
10432
|
+
if (reread.length === positions.length || !this.store.getCheckpoints) {
|
|
10433
|
+
const wanted = new Set(reread);
|
|
10434
|
+
for (const cp of await this.store.listCheckpoints(runId)) {
|
|
10435
|
+
if (wanted.has(cp.seq)) bySeq.set(cp.seq, cp);
|
|
10436
|
+
}
|
|
10437
|
+
return bySeq;
|
|
10438
|
+
}
|
|
10439
|
+
for (let at = 0; at < reread.length; at += FAN_REREAD_CHUNK) {
|
|
10440
|
+
const read = await this.store.getCheckpoints(runId, reread.slice(at, at + FAN_REREAD_CHUNK));
|
|
10441
|
+
for (const cp of read) bySeq.set(cp.seq, cp);
|
|
10442
|
+
}
|
|
10443
|
+
return bySeq;
|
|
10444
|
+
}
|
|
10445
|
+
/**
|
|
10356
10446
|
* Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
|
|
10357
10447
|
* {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
|
|
10358
10448
|
* batch, where the store and transport have them.
|
|
@@ -10361,6 +10451,7 @@ ${run.workflowVersion}`;
|
|
|
10361
10451
|
const workerGroup = stepGroup(step, routing?.namespace);
|
|
10362
10452
|
await this.ensureRoutable(workerGroup, step.requires);
|
|
10363
10453
|
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
10454
|
+
const lease = this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
|
|
10364
10455
|
const checkpoints = batch.map((i) => {
|
|
10365
10456
|
const seq = positions[i];
|
|
10366
10457
|
return {
|
|
@@ -10376,7 +10467,10 @@ ${run.workflowVersion}`;
|
|
|
10376
10467
|
parallelGroup: group,
|
|
10377
10468
|
enqueuedAt,
|
|
10378
10469
|
startedAt: enqueuedAt,
|
|
10379
|
-
finishedAt: enqueuedAt
|
|
10470
|
+
finishedAt: enqueuedAt,
|
|
10471
|
+
...lease !== void 0 ? {
|
|
10472
|
+
wakeAt: lease
|
|
10473
|
+
} : {}
|
|
10380
10474
|
};
|
|
10381
10475
|
});
|
|
10382
10476
|
if (this.store.saveCheckpoints) {
|
|
@@ -10409,20 +10503,36 @@ ${run.workflowVersion}`;
|
|
|
10409
10503
|
kind: "remote"
|
|
10410
10504
|
});
|
|
10411
10505
|
}
|
|
10506
|
+
if (lease !== void 0) {
|
|
10507
|
+
this.noteFanLease(runId, checkpoints.map((cp) => cp.seq), lease);
|
|
10508
|
+
}
|
|
10509
|
+
return lease;
|
|
10412
10510
|
}
|
|
10413
10511
|
/** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
|
|
10414
10512
|
noteSettledHere(key) {
|
|
10415
10513
|
this.noteRecent(this.settledHere, key);
|
|
10416
10514
|
}
|
|
10417
|
-
/**
|
|
10515
|
+
/**
|
|
10516
|
+
* Record `key` in a "happened here recently" map and drop the entries older than a lease. A key is
|
|
10517
|
+
* re-inserted at the end, so the map stays in the order its keys were noted and the stale ones are
|
|
10518
|
+
* all at its front: pruning stops at the first fresh entry instead of walking the whole map.
|
|
10519
|
+
*/
|
|
10418
10520
|
noteRecent(map, key) {
|
|
10419
10521
|
const now = this.clock();
|
|
10522
|
+
map.delete(key);
|
|
10420
10523
|
map.set(key, now);
|
|
10421
|
-
if (map.size < 1024) return;
|
|
10422
10524
|
for (const [k, at] of map) {
|
|
10423
|
-
if (now - at
|
|
10525
|
+
if (now - at <= this.leaseMs) break;
|
|
10526
|
+
map.delete(k);
|
|
10424
10527
|
}
|
|
10425
10528
|
}
|
|
10529
|
+
/**
|
|
10530
|
+
* The run's checkpoints as a turn replays them: {@link StateStore.listCheckpointsForReplay} when
|
|
10531
|
+
* the store has it (no input on a completed row), else {@link StateStore.listCheckpoints}.
|
|
10532
|
+
*/
|
|
10533
|
+
replayCheckpoints(runId) {
|
|
10534
|
+
return this.store.listCheckpointsForReplay ? this.store.listCheckpointsForReplay(runId) : this.store.listCheckpoints(runId);
|
|
10535
|
+
}
|
|
10426
10536
|
/** {@link StateStore.saveCheckpointIfPending}, or a plain save on a store without it. */
|
|
10427
10537
|
async savePendingCheckpoint(checkpoint) {
|
|
10428
10538
|
if (this.store.saveCheckpointIfPending) return this.store.saveCheckpointIfPending(checkpoint);
|
|
@@ -10449,11 +10559,59 @@ ${run.workflowVersion}`;
|
|
|
10449
10559
|
});
|
|
10450
10560
|
}
|
|
10451
10561
|
/**
|
|
10562
|
+
* Settle steps of one run at distinct seqs from their results ({@link StepSettlement}s, `settled[i]`
|
|
10563
|
+
* being the `pending` row `settlements[i]` settles): one
|
|
10564
|
+
* {@link StateStore.settleCheckpointsIfPending} call when the store has it, which writes only what
|
|
10565
|
+
* a result changes, else the whole rows through {@link savePendingCheckpoints}. Never rejects as a
|
|
10566
|
+
* whole, the same way.
|
|
10567
|
+
*/
|
|
10568
|
+
async settlePendingCheckpoints(settlements, settled) {
|
|
10569
|
+
const settle = this.store.settleCheckpointsIfPending?.bind(this.store);
|
|
10570
|
+
if (!settle) {
|
|
10571
|
+
return this.savePendingCheckpoints(settlements.map((s, i) => ({
|
|
10572
|
+
...settled[i],
|
|
10573
|
+
...s
|
|
10574
|
+
})));
|
|
10575
|
+
}
|
|
10576
|
+
if (settlements.length > 1) {
|
|
10577
|
+
try {
|
|
10578
|
+
const wrote = await settle(settlements);
|
|
10579
|
+
return settlements.map((_, i) => wrote[i] === true);
|
|
10580
|
+
} catch {
|
|
10581
|
+
}
|
|
10582
|
+
}
|
|
10583
|
+
const one = await Promise.allSettled(settlements.map((s) => settle([
|
|
10584
|
+
s
|
|
10585
|
+
])));
|
|
10586
|
+
return one.map((s) => s.status === "fulfilled" ? s.value[0] === true : {
|
|
10587
|
+
error: s.reason
|
|
10588
|
+
});
|
|
10589
|
+
}
|
|
10590
|
+
/**
|
|
10452
10591
|
* The checkpoints of one run at distinct `seqs`, each as its own outcome: one
|
|
10453
10592
|
* {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
|
|
10454
10593
|
* call rejecting falls back to one read each, so a failing read fails only its own entry.
|
|
10455
10594
|
*/
|
|
10456
10595
|
async readCheckpoints(runId, seqs) {
|
|
10596
|
+
const forSettle = this.store.settleCheckpointsIfPending ? this.store.getCheckpointsForSettle?.bind(this.store) : void 0;
|
|
10597
|
+
if (forSettle) {
|
|
10598
|
+
if (seqs.length > 1) {
|
|
10599
|
+
try {
|
|
10600
|
+
const bySeq = new Map((await forSettle(runId, seqs)).map((cp) => [
|
|
10601
|
+
cp.seq,
|
|
10602
|
+
cp
|
|
10603
|
+
]));
|
|
10604
|
+
return seqs.map((seq) => ({
|
|
10605
|
+
status: "fulfilled",
|
|
10606
|
+
value: bySeq.get(seq) ?? null
|
|
10607
|
+
}));
|
|
10608
|
+
} catch {
|
|
10609
|
+
}
|
|
10610
|
+
}
|
|
10611
|
+
return Promise.allSettled(seqs.map(async (seq) => (await forSettle(runId, [
|
|
10612
|
+
seq
|
|
10613
|
+
]))[0] ?? null));
|
|
10614
|
+
}
|
|
10457
10615
|
if (seqs.length > 1 && this.store.getCheckpoints) {
|
|
10458
10616
|
try {
|
|
10459
10617
|
const bySeq = new Map((await this.store.getCheckpoints(runId, seqs)).map((cp) => [
|
|
@@ -10602,10 +10760,11 @@ ${run.workflowVersion}`;
|
|
|
10602
10760
|
const run = await this.store.getRun(runId);
|
|
10603
10761
|
if (run && (run.status === "cancelled" || run.status === "completed")) return outcomes;
|
|
10604
10762
|
const finishedAt = /* @__PURE__ */ new Date();
|
|
10605
|
-
const
|
|
10763
|
+
const settlements = pending.map(({ index, cp }) => {
|
|
10606
10764
|
const result = results[index];
|
|
10607
10765
|
return {
|
|
10608
|
-
|
|
10766
|
+
runId,
|
|
10767
|
+
seq: cp.seq,
|
|
10609
10768
|
status: result.status,
|
|
10610
10769
|
output: result.status === "completed" ? result.output : cp.output,
|
|
10611
10770
|
error: result.error,
|
|
@@ -10614,7 +10773,7 @@ ${run.workflowVersion}`;
|
|
|
10614
10773
|
finishedAt
|
|
10615
10774
|
};
|
|
10616
10775
|
});
|
|
10617
|
-
const claims = await this.
|
|
10776
|
+
const claims = await this.settlePendingCheckpoints(settlements, pending.map(({ cp }) => cp));
|
|
10618
10777
|
const settled = [];
|
|
10619
10778
|
claims.forEach((claim, k) => {
|
|
10620
10779
|
const { index, cp } = pending[k];
|
|
@@ -10628,7 +10787,7 @@ ${run.workflowVersion}`;
|
|
|
10628
10787
|
};
|
|
10629
10788
|
});
|
|
10630
10789
|
for (const { cp, result } of settled) {
|
|
10631
|
-
this.noteSettledHere(`${runId}:${result.seq}`);
|
|
10790
|
+
if (!parseStepFanGroup(cp.parallelGroup)) this.noteSettledHere(`${runId}:${result.seq}`);
|
|
10632
10791
|
const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
|
|
10633
10792
|
this.emit({
|
|
10634
10793
|
type: result.status === "completed" ? "step.completed" : "step.failed",
|
|
@@ -10693,21 +10852,26 @@ ${run.workflowVersion}`;
|
|
|
10693
10852
|
let closed = barriers.get(fan.firstSeq);
|
|
10694
10853
|
if (!closed) {
|
|
10695
10854
|
const lastSeq = fan.firstSeq + fan.size - 1;
|
|
10696
|
-
closed = queuedIn(fan.firstSeq, lastSeq) ? Promise.resolve(true) : this.
|
|
10855
|
+
closed = queuedIn(fan.firstSeq, lastSeq) ? Promise.resolve(true) : this.stepFanHasPending(result.runId, fan.firstSeq, lastSeq).catch(() => false);
|
|
10697
10856
|
barriers.set(fan.firstSeq, closed);
|
|
10698
10857
|
}
|
|
10699
10858
|
if (await closed) return true;
|
|
10700
10859
|
this.noteRecent(this.stepFansFiredHere, `${result.runId}:${fan.firstSeq}`);
|
|
10701
10860
|
return false;
|
|
10702
10861
|
}
|
|
10703
|
-
/**
|
|
10704
|
-
async
|
|
10705
|
-
if (this.store.
|
|
10706
|
-
return this.store.
|
|
10862
|
+
/** Whether any step of a step fan's seq range is still `pending`. */
|
|
10863
|
+
async stepFanHasPending(runId, from, to) {
|
|
10864
|
+
if (this.store.hasCheckpoints) {
|
|
10865
|
+
return this.store.hasCheckpoints(runId, from, to, [
|
|
10707
10866
|
"pending"
|
|
10708
10867
|
]);
|
|
10709
10868
|
}
|
|
10710
|
-
|
|
10869
|
+
if (this.store.countCheckpoints) {
|
|
10870
|
+
return await this.store.countCheckpoints(runId, from, to, [
|
|
10871
|
+
"pending"
|
|
10872
|
+
]) > 0;
|
|
10873
|
+
}
|
|
10874
|
+
return (await this.store.listCheckpoints(runId)).some((c) => c.seq >= from && c.seq <= to && c.status === "pending");
|
|
10711
10875
|
}
|
|
10712
10876
|
/**
|
|
10713
10877
|
* Apply a remote workflow-turn {@link WorkflowDecision} durably — runs on WHATEVER instance receives
|
|
@@ -11527,10 +11691,20 @@ var CodecStateStore = class {
|
|
|
11527
11691
|
saveCheckpointIfPending;
|
|
11528
11692
|
/** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
|
|
11529
11693
|
saveCheckpointsIfPending;
|
|
11694
|
+
/** Encodes each settlement's payloads like {@link saveCheckpoint}. Bound only when the inner store
|
|
11695
|
+
* has it, so the engine's whole-row fallback still kicks in through the wrapper. */
|
|
11696
|
+
settleCheckpointsIfPending;
|
|
11530
11697
|
/** Encodes each checkpoint written. Bound only when the inner store has it. */
|
|
11531
11698
|
saveCheckpoints;
|
|
11699
|
+
/** Decodes each checkpoint read like {@link listCheckpoints}. Bound only when the inner store has
|
|
11700
|
+
* it, so the engine's full-read fallback still kicks in through the wrapper. */
|
|
11701
|
+
listCheckpointsForReplay;
|
|
11702
|
+
/** Decodes each checkpoint read. Bound only when the inner store has it. */
|
|
11703
|
+
getCheckpointsForSettle;
|
|
11532
11704
|
/** Decodes each checkpoint read. Bound only when the inner store has it. */
|
|
11533
11705
|
getCheckpoints;
|
|
11706
|
+
/** Forwarded verbatim — reads no payload. Bound only when the inner store has it. */
|
|
11707
|
+
hasCheckpoints;
|
|
11534
11708
|
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
|
|
11535
11709
|
countCheckpoints;
|
|
11536
11710
|
/** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
|
|
@@ -11581,10 +11755,18 @@ var CodecStateStore = class {
|
|
|
11581
11755
|
if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
|
|
11582
11756
|
const saveManyIfPending = inner.saveCheckpointsIfPending;
|
|
11583
11757
|
if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11758
|
+
const settleMany = inner.settleCheckpointsIfPending;
|
|
11759
|
+
if (settleMany) this.settleCheckpointsIfPending = (settlements) => settleMany.call(inner, settlements.map((s) => this.encSettlement(s)));
|
|
11584
11760
|
const saveMany = inner.saveCheckpoints;
|
|
11585
11761
|
if (saveMany) this.saveCheckpoints = (cps) => saveMany.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11762
|
+
const listForReplay = inner.listCheckpointsForReplay;
|
|
11763
|
+
if (listForReplay) this.listCheckpointsForReplay = async (runId) => (await listForReplay.call(inner, runId)).map((cp) => this.decCp(cp));
|
|
11764
|
+
const getForSettle = inner.getCheckpointsForSettle;
|
|
11765
|
+
if (getForSettle) this.getCheckpointsForSettle = async (runId, seqs) => (await getForSettle.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
|
|
11586
11766
|
const getMany = inner.getCheckpoints;
|
|
11587
11767
|
if (getMany) this.getCheckpoints = async (runId, seqs) => (await getMany.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
|
|
11768
|
+
const hasCps = inner.hasCheckpoints;
|
|
11769
|
+
if (hasCps) this.hasCheckpoints = (runId, from, to, statuses) => hasCps.call(inner, runId, from, to, statuses);
|
|
11588
11770
|
const countCps = inner.countCheckpoints;
|
|
11589
11771
|
if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
|
|
11590
11772
|
const stats = inner.checkpointStats;
|
|
@@ -11674,6 +11856,21 @@ var CodecStateStore = class {
|
|
|
11674
11856
|
}
|
|
11675
11857
|
return next;
|
|
11676
11858
|
}
|
|
11859
|
+
encSettlement(settlement) {
|
|
11860
|
+
const next = {
|
|
11861
|
+
...settlement,
|
|
11862
|
+
output: this.enc(settlement.output)
|
|
11863
|
+
};
|
|
11864
|
+
if (this.extended) {
|
|
11865
|
+
if (settlement.events !== void 0) {
|
|
11866
|
+
next.events = this.enc(settlement.events);
|
|
11867
|
+
}
|
|
11868
|
+
if (settlement.error !== void 0) {
|
|
11869
|
+
next.error = this.enc(settlement.error);
|
|
11870
|
+
}
|
|
11871
|
+
}
|
|
11872
|
+
return next;
|
|
11873
|
+
}
|
|
11677
11874
|
decCp(cp) {
|
|
11678
11875
|
if (!cp) return cp;
|
|
11679
11876
|
const next = {
|
|
@@ -11863,6 +12060,7 @@ function collectMetrics(engine) {
|
|
|
11863
12060
|
return entry;
|
|
11864
12061
|
}, "wf");
|
|
11865
12062
|
const stop = engine.subscribe((e) => {
|
|
12063
|
+
if (isRemoteEvent(e)) return;
|
|
11866
12064
|
switch (e.type) {
|
|
11867
12065
|
case "run.started":
|
|
11868
12066
|
m.runs.started += 1;
|
|
@@ -12233,6 +12431,18 @@ var InMemoryStateStore = class {
|
|
|
12233
12431
|
] : [];
|
|
12234
12432
|
});
|
|
12235
12433
|
}
|
|
12434
|
+
async getCheckpointsForSettle(runId, seqs) {
|
|
12435
|
+
return [
|
|
12436
|
+
...new Set(seqs)
|
|
12437
|
+
].sort((a, b) => a - b).flatMap((seq) => {
|
|
12438
|
+
const cp = this.checkpoints.get(this.key(runId, seq));
|
|
12439
|
+
if (!cp) return [];
|
|
12440
|
+
const { input: _input, ...rest } = cp;
|
|
12441
|
+
return [
|
|
12442
|
+
rest
|
|
12443
|
+
];
|
|
12444
|
+
});
|
|
12445
|
+
}
|
|
12236
12446
|
async saveCheckpoint(checkpoint) {
|
|
12237
12447
|
this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
|
|
12238
12448
|
...checkpoint
|
|
@@ -12263,6 +12473,18 @@ var InMemoryStateStore = class {
|
|
|
12263
12473
|
return true;
|
|
12264
12474
|
});
|
|
12265
12475
|
}
|
|
12476
|
+
async settleCheckpointsIfPending(settlements) {
|
|
12477
|
+
return settlements.map((settlement) => {
|
|
12478
|
+
const key = this.key(settlement.runId, settlement.seq);
|
|
12479
|
+
const cp = this.checkpoints.get(key);
|
|
12480
|
+
if (cp?.status !== "pending") return false;
|
|
12481
|
+
this.checkpoints.set(key, {
|
|
12482
|
+
...cp,
|
|
12483
|
+
...settlement
|
|
12484
|
+
});
|
|
12485
|
+
return true;
|
|
12486
|
+
});
|
|
12487
|
+
}
|
|
12266
12488
|
async listOrphanedRuns(nowMs, limit, namespace, ownedBy) {
|
|
12267
12489
|
return [
|
|
12268
12490
|
...this.runs.values()
|
|
@@ -12579,6 +12801,13 @@ var InMemoryStateStore = class {
|
|
|
12579
12801
|
this.checkpointRowsRead += rows.length;
|
|
12580
12802
|
return rows;
|
|
12581
12803
|
}
|
|
12804
|
+
async listCheckpointsForReplay(runId) {
|
|
12805
|
+
return (await this.listCheckpoints(runId)).map((cp) => {
|
|
12806
|
+
if (cp.status !== "completed") return cp;
|
|
12807
|
+
const { input: _input, ...rest } = cp;
|
|
12808
|
+
return rest;
|
|
12809
|
+
});
|
|
12810
|
+
}
|
|
12582
12811
|
runCheckpoints(runId) {
|
|
12583
12812
|
return [
|
|
12584
12813
|
...this.checkpoints.values()
|
|
@@ -12606,6 +12835,9 @@ var InMemoryStateStore = class {
|
|
|
12606
12835
|
...latest
|
|
12607
12836
|
} : void 0;
|
|
12608
12837
|
}
|
|
12838
|
+
async hasCheckpoints(runId, fromSeq, toSeq, statuses) {
|
|
12839
|
+
return await this.countCheckpoints(runId, fromSeq, toSeq, statuses) > 0;
|
|
12840
|
+
}
|
|
12609
12841
|
async countCheckpoints(runId, fromSeq, toSeq, statuses) {
|
|
12610
12842
|
let n = 0;
|
|
12611
12843
|
for (const cp of this.checkpoints.values()) {
|
|
@@ -12799,6 +13031,7 @@ var InMemoryRunQueue = class {
|
|
|
12799
13031
|
NestedWorkflowCallError,
|
|
12800
13032
|
NonDeterminismError,
|
|
12801
13033
|
QueueController,
|
|
13034
|
+
REMOTE_EVENT,
|
|
12802
13035
|
RUN_HISTORY_FIXTURE_VERSION,
|
|
12803
13036
|
RUN_VALUE_FACET_LIMIT,
|
|
12804
13037
|
RUN_VALUE_FACET_SCAN,
|
|
@@ -12889,11 +13122,13 @@ var InMemoryRunQueue = class {
|
|
|
12889
13122
|
isFailedMember,
|
|
12890
13123
|
isLegacyDescriptor,
|
|
12891
13124
|
isNonEmptyRunScope,
|
|
13125
|
+
isRemoteEvent,
|
|
12892
13126
|
isWorkflowControlFlowSignal,
|
|
12893
13127
|
legacySingletonHolders,
|
|
12894
13128
|
listSignalWaitersOfRuns,
|
|
12895
13129
|
liveHandoff,
|
|
12896
13130
|
log,
|
|
13131
|
+
markRemoteEvent,
|
|
12897
13132
|
matchesAttributes,
|
|
12898
13133
|
mergeRunFacetRows,
|
|
12899
13134
|
mergeRunValueFacetRows,
|