@dudousxd/nestjs-durable-core 0.85.6 → 0.85.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +397 -94
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +114 -12
- package/dist/index.d.ts +114 -12
- package/dist/index.js +397 -94
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -3347,6 +3347,7 @@ function constantTimeEquals(a, b) {
|
|
|
3347
3347
|
__name(constantTimeEquals, "constantTimeEquals");
|
|
3348
3348
|
|
|
3349
3349
|
// src/transport-pool.ts
|
|
3350
|
+
var DISPATCH_CHUNK = 200;
|
|
3350
3351
|
var TransportPool = class {
|
|
3351
3352
|
static {
|
|
3352
3353
|
__name(this, "TransportPool");
|
|
@@ -3406,6 +3407,28 @@ var TransportPool = class {
|
|
|
3406
3407
|
}
|
|
3407
3408
|
throw lastErr ?? new Error("no transport accepted the dispatch");
|
|
3408
3409
|
}
|
|
3410
|
+
/**
|
|
3411
|
+
* {@link dispatch} for many tasks: one {@link Transport.dispatchMany} call on the preferred (or
|
|
3412
|
+
* first) transport when it has one, else (or when that call rejects) each task through
|
|
3413
|
+
* {@link dispatch}, a bounded number at a time.
|
|
3414
|
+
*/
|
|
3415
|
+
async dispatchMany(tasks, preferId) {
|
|
3416
|
+
if (tasks.length === 0) return;
|
|
3417
|
+
const first = this.ordered(preferId)[0];
|
|
3418
|
+
if (first?.transport.dispatchMany) {
|
|
3419
|
+
try {
|
|
3420
|
+
await first.transport.dispatchMany(tasks.map((task) => ({
|
|
3421
|
+
...task,
|
|
3422
|
+
transport: first.id
|
|
3423
|
+
})));
|
|
3424
|
+
return;
|
|
3425
|
+
} catch {
|
|
3426
|
+
}
|
|
3427
|
+
}
|
|
3428
|
+
for (let at = 0; at < tasks.length; at += DISPATCH_CHUNK) {
|
|
3429
|
+
await Promise.all(tasks.slice(at, at + DISPATCH_CHUNK).map((task) => this.dispatch(task, preferId)));
|
|
3430
|
+
}
|
|
3431
|
+
}
|
|
3409
3432
|
/** Worker-health for `group`, merged across every transport that can report it (a group's queue
|
|
3410
3433
|
* and its workers may live on any pinned transport). Undefined when no transport implements
|
|
3411
3434
|
* `groupHealth` (e.g. a pure in-process transport — nothing to introspect). */
|
|
@@ -4625,6 +4648,15 @@ var PARENT_NOTIFY_BACKOFF_MS = 250;
|
|
|
4625
4648
|
var LEASE_RECHECK_MAX_MS = 2e3;
|
|
4626
4649
|
var LEASE_RECHECK_SLACK_MS = 100;
|
|
4627
4650
|
var LEASE_RECHECK_ATTEMPTS = 30;
|
|
4651
|
+
function wakeSeen(request, settledFans, served) {
|
|
4652
|
+
if (request.id !== void 0 && served.has(request.id)) return true;
|
|
4653
|
+
return isFanWakeRequest(request.payload) && request.payload.fans.every(([first, seq]) => (settledFans.get(first) ?? -1) >= seq);
|
|
4654
|
+
}
|
|
4655
|
+
__name(wakeSeen, "wakeSeen");
|
|
4656
|
+
function isFanWakeRequest(payload) {
|
|
4657
|
+
return typeof payload === "object" && payload !== null && Array.isArray(payload.fans);
|
|
4658
|
+
}
|
|
4659
|
+
__name(isFanWakeRequest, "isFanWakeRequest");
|
|
4628
4660
|
function deferred() {
|
|
4629
4661
|
let resolve;
|
|
4630
4662
|
let reject;
|
|
@@ -4643,12 +4675,51 @@ __name(deferred, "deferred");
|
|
|
4643
4675
|
var pendingStateKey = /* @__PURE__ */ __name((at) => String(new Date(at).getTime()), "pendingStateKey");
|
|
4644
4676
|
var freshPendingKey = /* @__PURE__ */ __name(() => `${Date.now()}-${globalThis.crypto.randomUUID().slice(0, 8)}`, "freshPendingKey");
|
|
4645
4677
|
var wakeRequestToken = /* @__PURE__ */ __name((runId) => `durable:wake-after-turn:${runId}`, "wakeRequestToken");
|
|
4678
|
+
var WAKE_REQUEST_SCAN = 100;
|
|
4646
4679
|
var CHILD_SIGNAL_PREFIX = "signal:child:";
|
|
4647
4680
|
var BUFFERED_SWEEP_PAGE = 500;
|
|
4648
4681
|
var BUFFERED_SWEEP_PAGES_PER_CALL = 4;
|
|
4649
4682
|
var BUFFERED_SWEEP_REST_MS = 10 * 6e4;
|
|
4650
4683
|
var DELIVER_CHUNK = 50;
|
|
4651
4684
|
var RESULT_BATCH_MAX = 256;
|
|
4685
|
+
function isResultError(outcome) {
|
|
4686
|
+
return typeof outcome === "object" && "error" in outcome;
|
|
4687
|
+
}
|
|
4688
|
+
__name(isResultError, "isResultError");
|
|
4689
|
+
var FanWakeCollector = class FanWakeCollector2 {
|
|
4690
|
+
static {
|
|
4691
|
+
__name(this, "FanWakeCollector");
|
|
4692
|
+
}
|
|
4693
|
+
carrier;
|
|
4694
|
+
generic = false;
|
|
4695
|
+
fan;
|
|
4696
|
+
/** The highest seq the batch settled, per fan. */
|
|
4697
|
+
highest = /* @__PURE__ */ new Map();
|
|
4698
|
+
/** A checkpoint the batch settled. */
|
|
4699
|
+
note(cp) {
|
|
4700
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
4701
|
+
if (fan) this.highest.set(fan.firstSeq, Math.max(this.highest.get(fan.firstSeq) ?? cp.seq, cp.seq));
|
|
4702
|
+
}
|
|
4703
|
+
/** A settled checkpoint that resumes the run. */
|
|
4704
|
+
add(index, cp, status) {
|
|
4705
|
+
this.carrier = index;
|
|
4706
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
4707
|
+
if (!fan || status === "failed" || this.fan !== void 0 && this.fan !== fan.firstSeq) {
|
|
4708
|
+
this.generic = true;
|
|
4709
|
+
return;
|
|
4710
|
+
}
|
|
4711
|
+
this.fan = fan.firstSeq;
|
|
4712
|
+
}
|
|
4713
|
+
applyTo(outcomes) {
|
|
4714
|
+
if (this.carrier === void 0) return outcomes;
|
|
4715
|
+
const seq = this.fan === void 0 ? void 0 : this.highest.get(this.fan);
|
|
4716
|
+
outcomes[this.carrier] = this.generic || this.fan === void 0 || seq === void 0 ? true : {
|
|
4717
|
+
first: this.fan,
|
|
4718
|
+
seq
|
|
4719
|
+
};
|
|
4720
|
+
return outcomes;
|
|
4721
|
+
}
|
|
4722
|
+
};
|
|
4652
4723
|
function shuffle(items) {
|
|
4653
4724
|
for (let i = items.length - 1; i > 0; i -= 1) {
|
|
4654
4725
|
const j = Math.floor(Math.random() * (i + 1));
|
|
@@ -4837,6 +4908,23 @@ var WorkflowEngine = class {
|
|
|
4837
4908
|
/** Re-checks scheduled for runs whose wake found another instance's lease (see {@link execute}). */
|
|
4838
4909
|
leaseRetries = /* @__PURE__ */ new Map();
|
|
4839
4910
|
/**
|
|
4911
|
+
* Runs whose scheduled re-check resumes them whatever happened meanwhile. Without an entry the
|
|
4912
|
+
* re-check only stood for fan wakes kept as requests, and resumes the run only if a request is
|
|
4913
|
+
* still there: the turn that held the lease takes them once it ends (see {@link retryDroppedWake}).
|
|
4914
|
+
*/
|
|
4915
|
+
unconditionalRechecks = /* @__PURE__ */ new Set();
|
|
4916
|
+
/** Runs a generic wake request was written for in the current re-check cycle (see {@link execute}). */
|
|
4917
|
+
genericWakeRequested = /* @__PURE__ */ new Set();
|
|
4918
|
+
/** Why each run is being woken on this instance, until the turn that answers it starts. */
|
|
4919
|
+
wakeReasons = /* @__PURE__ */ new Map();
|
|
4920
|
+
/**
|
|
4921
|
+
* The step fans each executing turn on this instance read with nothing in flight: first seq -> the
|
|
4922
|
+
* highest seq read. Written by {@link callRemoteAll}, read once the turn ends.
|
|
4923
|
+
*/
|
|
4924
|
+
turnSettledFans = /* @__PURE__ */ new Map();
|
|
4925
|
+
/** The wake requests each executing turn on this instance serves (see {@link noteServedWakes}). */
|
|
4926
|
+
turnServedWakes = /* @__PURE__ */ new Map();
|
|
4927
|
+
/**
|
|
4840
4928
|
* Every run turn and child start this instance takes on itself, under one budget of
|
|
4841
4929
|
* {@link LOCAL_RUN_CONCURRENCY}: the runs it dispatches to itself, a fan's deferred child starts,
|
|
4842
4930
|
* enqueued pickups and the runs it takes off the transport's run queue.
|
|
@@ -4960,8 +5048,9 @@ var WorkflowEngine = class {
|
|
|
4960
5048
|
}
|
|
4961
5049
|
return;
|
|
4962
5050
|
}
|
|
4963
|
-
|
|
4964
|
-
|
|
5051
|
+
const wake = await this.completeRemoteResult(result);
|
|
5052
|
+
if (!wake) return;
|
|
5053
|
+
await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId, wake === true ? void 0 : wake).then(() => void 0, (err) => {
|
|
4965
5054
|
if (err instanceof NamespaceMismatch) return;
|
|
4966
5055
|
throw err;
|
|
4967
5056
|
}));
|
|
@@ -5178,7 +5267,10 @@ var WorkflowEngine = class {
|
|
|
5178
5267
|
const resumed = await this.resumeLeased([
|
|
5179
5268
|
run
|
|
5180
5269
|
], nowMs, (locked) => this.reconcileSignalWaiter(locked, waiters.get(locked.id)));
|
|
5181
|
-
if (resumed.length === 0)
|
|
5270
|
+
if (resumed.length === 0) {
|
|
5271
|
+
this.unconditionalRechecks.add(runId);
|
|
5272
|
+
this.recheckAfterLease(runId, run.lockedUntil);
|
|
5273
|
+
}
|
|
5182
5274
|
return;
|
|
5183
5275
|
}
|
|
5184
5276
|
await this.resumeImplicit(runId).catch((err) => {
|
|
@@ -5745,15 +5837,27 @@ var WorkflowEngine = class {
|
|
|
5745
5837
|
* terminal too.
|
|
5746
5838
|
*/
|
|
5747
5839
|
resume(runId) {
|
|
5840
|
+
this.noteWakeReason(runId, void 0);
|
|
5748
5841
|
return this.resumeRun(runId, true);
|
|
5749
5842
|
}
|
|
5843
|
+
/** Record why `runId` is being woken: a step fan's barrier opening, or (`undefined`) anything else. */
|
|
5844
|
+
noteWakeReason(runId, fan) {
|
|
5845
|
+
const reason = this.wakeReasons.get(runId) ?? {
|
|
5846
|
+
generic: false,
|
|
5847
|
+
fans: /* @__PURE__ */ new Map()
|
|
5848
|
+
};
|
|
5849
|
+
if (fan) reason.fans.set(fan.first, Math.max(reason.fans.get(fan.first) ?? fan.seq, fan.seq));
|
|
5850
|
+
else reason.generic = true;
|
|
5851
|
+
this.wakeReasons.set(runId, reason);
|
|
5852
|
+
}
|
|
5750
5853
|
/**
|
|
5751
5854
|
* {@link resume} for every IMPLICIT wake-up — a delivered signal/child completion, a late step
|
|
5752
5855
|
* result, a due timer, a redelivered dispatch, a queue-slot wake. A `failed` run is terminal to
|
|
5753
5856
|
* these: only an explicit retry ({@link requeue} / {@link resume}) may re-drive it. Otherwise a
|
|
5754
5857
|
* stale waiter or a late result left behind by a failed run silently re-executes its body.
|
|
5755
5858
|
*/
|
|
5756
|
-
resumeImplicit(runId) {
|
|
5859
|
+
resumeImplicit(runId, fan) {
|
|
5860
|
+
if (fan !== "coalesced") this.noteWakeReason(runId, fan);
|
|
5757
5861
|
const turn = this.wakeTurns.get(runId);
|
|
5758
5862
|
if (turn) {
|
|
5759
5863
|
turn.again ??= deferred();
|
|
@@ -5778,7 +5882,7 @@ var WorkflowEngine = class {
|
|
|
5778
5882
|
});
|
|
5779
5883
|
return;
|
|
5780
5884
|
}
|
|
5781
|
-
const next = this.resumeImplicit(runId);
|
|
5885
|
+
const next = this.resumeImplicit(runId, "coalesced");
|
|
5782
5886
|
next.then(again.resolve, again.reject);
|
|
5783
5887
|
this.inBackground(next);
|
|
5784
5888
|
}, "finish");
|
|
@@ -7843,9 +7947,26 @@ var WorkflowEngine = class {
|
|
|
7843
7947
|
async execute(run, fn) {
|
|
7844
7948
|
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
|
|
7845
7949
|
const lockNow = this.clock();
|
|
7950
|
+
const reason = this.wakeReasons.get(run.id) ?? {
|
|
7951
|
+
generic: true,
|
|
7952
|
+
fans: /* @__PURE__ */ new Map()
|
|
7953
|
+
};
|
|
7954
|
+
this.wakeReasons.delete(run.id);
|
|
7846
7955
|
if (run.lockedBy !== this.instanceId || (run.lockedUntil ?? 0) <= lockNow) {
|
|
7847
7956
|
if (!await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow)) {
|
|
7848
|
-
if (
|
|
7957
|
+
if (reason.generic) {
|
|
7958
|
+
if (!this.genericWakeRequested.has(run.id)) {
|
|
7959
|
+
this.genericWakeRequested.add(run.id);
|
|
7960
|
+
this.inBackground(this.requestWakeAfterTurn(run.id, null));
|
|
7961
|
+
}
|
|
7962
|
+
} else {
|
|
7963
|
+
const fans = {
|
|
7964
|
+
fans: [
|
|
7965
|
+
...reason.fans
|
|
7966
|
+
]
|
|
7967
|
+
};
|
|
7968
|
+
this.inBackground(this.requestWakeAfterTurn(run.id, fans));
|
|
7969
|
+
}
|
|
7849
7970
|
this.recheckAfterLease(run.id, run.lockedUntil);
|
|
7850
7971
|
return {
|
|
7851
7972
|
runId: run.id,
|
|
@@ -7853,56 +7974,142 @@ var WorkflowEngine = class {
|
|
|
7853
7974
|
};
|
|
7854
7975
|
}
|
|
7855
7976
|
}
|
|
7977
|
+
const settledFans = /* @__PURE__ */ new Map();
|
|
7978
|
+
this.turnSettledFans.set(run.id, settledFans);
|
|
7856
7979
|
const held = this.holdExecutionLease(run.id);
|
|
7857
7980
|
try {
|
|
7981
|
+
const served = /* @__PURE__ */ new Set();
|
|
7982
|
+
this.turnServedWakes.set(run.id, served);
|
|
7983
|
+
if (registered?.remote) await this.noteServedWakes(run.id);
|
|
7858
7984
|
const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
|
|
7859
7985
|
if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
|
|
7860
7986
|
void this.singletons.wakeNext(run).catch(() => void 0);
|
|
7861
7987
|
}
|
|
7862
|
-
if (result.status === "suspended")
|
|
7863
|
-
|
|
7864
|
-
|
|
7988
|
+
if (result.status === "suspended") {
|
|
7989
|
+
this.inBackground(this.honorWakeRequests(run.id, settledFans, served));
|
|
7990
|
+
} else if (TERMINAL_RUN_STATUSES.includes(result.status)) {
|
|
7991
|
+
this.inBackground(this.listWakeRequests(run.id, true).then((kept) => this.dropWakeRequests(kept)));
|
|
7865
7992
|
}
|
|
7866
7993
|
return result;
|
|
7867
7994
|
} finally {
|
|
7868
7995
|
held.stop();
|
|
7996
|
+
if (this.turnSettledFans.get(run.id) === settledFans) this.turnSettledFans.delete(run.id);
|
|
7997
|
+
this.turnServedWakes.delete(run.id);
|
|
7869
7998
|
}
|
|
7870
7999
|
}
|
|
7871
8000
|
/**
|
|
7872
8001
|
* Keep a wake that found `runId`'s lease held by a turn executing elsewhere, durably, so that turn
|
|
7873
8002
|
* runs again once it ends instead of suspending on a snapshot that predates what the wake brought
|
|
7874
|
-
* (a signal, a child's completion). The request is a buffered
|
|
7875
|
-
* on
|
|
8003
|
+
* (a signal, a child's completion). The request is a buffered event under a name no workflow waits
|
|
8004
|
+
* on. It stays until a turn that started after it has settled: the holder looks at it after it
|
|
8005
|
+
* releases the lease ({@link honorWakeRequests}), and the turn that serves it drops it.
|
|
7876
8006
|
*
|
|
7877
8007
|
* Written first and looked at second, against a holder that releases first and looks second:
|
|
7878
8008
|
* whichever look runs last sees the other side's write. So when the run is no longer executing by
|
|
7879
|
-
* the time this re-reads it, the holder may have looked already, and this side
|
|
7880
|
-
*
|
|
8009
|
+
* the time this re-reads it, the holder may have looked already, and this side wakes the run
|
|
8010
|
+
* itself once the lease is free ({@link retryDroppedWake}). While the holder still runs, the run is
|
|
8011
|
+
* made due when the holder's lease runs out, so a holder that goes down before it settles leaves
|
|
8012
|
+
* the run to any instance's timer poll; a holder that settles sets the run's timer itself.
|
|
7881
8013
|
*/
|
|
7882
|
-
async requestWakeAfterTurn(runId) {
|
|
7883
|
-
|
|
8014
|
+
async requestWakeAfterTurn(runId, request) {
|
|
8015
|
+
const id = globalThis.crypto.randomUUID();
|
|
8016
|
+
await this.store.bufferEvent({
|
|
8017
|
+
name: wakeRequestToken(runId),
|
|
8018
|
+
payload: request,
|
|
8019
|
+
id,
|
|
8020
|
+
publishedAt: this.clock()
|
|
8021
|
+
});
|
|
7884
8022
|
const run = await this.store.getRun(runId);
|
|
7885
|
-
if (run
|
|
7886
|
-
|
|
7887
|
-
|
|
8023
|
+
if (!run || TERMINAL_RUN_STATUSES.includes(run.status)) {
|
|
8024
|
+
await this.store.removeBufferedEvent(id);
|
|
8025
|
+
return;
|
|
8026
|
+
}
|
|
8027
|
+
if (run.status === "running") return;
|
|
8028
|
+
if (run.lockedBy !== void 0 && run.lockedBy !== this.instanceId) {
|
|
8029
|
+
await this.makeDueBy(runId, (run.lockedUntil ?? this.clock()) + LEASE_RECHECK_SLACK_MS);
|
|
7888
8030
|
return;
|
|
7889
8031
|
}
|
|
7890
|
-
if (!await this.takeWakeRequests(runId)) return;
|
|
7891
|
-
if (TERMINAL_RUN_STATUSES.includes(run.status)) return;
|
|
7892
8032
|
this.recheckAfterLease(runId, run.lockedUntil);
|
|
7893
8033
|
}
|
|
7894
|
-
/**
|
|
7895
|
-
|
|
7896
|
-
|
|
8034
|
+
/**
|
|
8035
|
+
* After a turn that suspended released its lease: drop the wakes it served, and run again if a
|
|
8036
|
+
* wake was kept for it meanwhile, unless every such wake was a fan wake this turn already saw: it
|
|
8037
|
+
* read that fan with nothing in flight, at least as far as the batch that opened its barrier.
|
|
8038
|
+
* The run is made due before it is woken, so if this instance goes down before that turn runs,
|
|
8039
|
+
* the timer poll of another instance runs it once this instance's hold on it would have lapsed.
|
|
8040
|
+
*/
|
|
8041
|
+
async honorWakeRequests(runId, settledFans, served) {
|
|
8042
|
+
const kept = await this.listWakeRequests(runId, true);
|
|
8043
|
+
const seen = /* @__PURE__ */ __name((request) => wakeSeen(request, settledFans, served), "seen");
|
|
8044
|
+
const needed = kept.filter((request) => !seen(request));
|
|
8045
|
+
await this.dropWakeRequests(kept.filter(seen));
|
|
8046
|
+
if (needed.length === 0) return;
|
|
8047
|
+
await this.makeDueBy(runId, this.clock() + this.leaseMs);
|
|
7897
8048
|
await this.wake(runId).catch((err) => {
|
|
7898
8049
|
if (!(err instanceof NamespaceMismatch)) throw err;
|
|
7899
8050
|
});
|
|
7900
8051
|
}
|
|
7901
|
-
/**
|
|
7902
|
-
|
|
7903
|
-
|
|
7904
|
-
|
|
7905
|
-
|
|
8052
|
+
/**
|
|
8053
|
+
* The wake requests kept for `runId`. With `legacy`, also those an older engine kept as buffered
|
|
8054
|
+
* signals, which are taken as they are read.
|
|
8055
|
+
*/
|
|
8056
|
+
async listWakeRequests(runId, legacy = false) {
|
|
8057
|
+
const token = wakeRequestToken(runId);
|
|
8058
|
+
const kept = (await this.store.listBufferedEvents(token, WAKE_REQUEST_SCAN)).map((request) => ({
|
|
8059
|
+
id: request.id,
|
|
8060
|
+
payload: request.payload
|
|
8061
|
+
}));
|
|
8062
|
+
if (!legacy) return kept;
|
|
8063
|
+
for (; ; ) {
|
|
8064
|
+
const signal = await this.store.takeBufferedSignal(token);
|
|
8065
|
+
if (!signal) return kept;
|
|
8066
|
+
kept.push({
|
|
8067
|
+
payload: signal.payload
|
|
8068
|
+
});
|
|
8069
|
+
}
|
|
8070
|
+
}
|
|
8071
|
+
/** Delete `requests` (the ones kept as buffered events). */
|
|
8072
|
+
async dropWakeRequests(requests) {
|
|
8073
|
+
await Promise.all(requests.flatMap((request) => request.id === void 0 ? [] : [
|
|
8074
|
+
this.store.removeBufferedEvent(request.id)
|
|
8075
|
+
]));
|
|
8076
|
+
}
|
|
8077
|
+
/**
|
|
8078
|
+
* Record, for the turn executing `runId` here, the wakes kept for the run so far. Called before the
|
|
8079
|
+
* turn reads the run's state, so the turn serves them: they are dropped once it has settled, and
|
|
8080
|
+
* until then stay for whoever runs the run next.
|
|
8081
|
+
*/
|
|
8082
|
+
async noteServedWakes(runId) {
|
|
8083
|
+
const served = this.turnServedWakes.get(runId);
|
|
8084
|
+
if (!served) return;
|
|
8085
|
+
for (const request of await this.store.listBufferedEvents(wakeRequestToken(runId), WAKE_REQUEST_SCAN)) {
|
|
8086
|
+
served.add(request.id);
|
|
8087
|
+
}
|
|
8088
|
+
}
|
|
8089
|
+
/**
|
|
8090
|
+
* Right after a turn of `runId` wrote its suspension, while it still holds the lease: if a wake
|
|
8091
|
+
* was kept for the run during the turn that the turn did not see, keep the run due when the lease
|
|
8092
|
+
* runs out. The suspension overwrote the timer the instance keeping the wake set, and this instance
|
|
8093
|
+
* may go down before it looks at the wake once it has released the lease.
|
|
8094
|
+
*/
|
|
8095
|
+
async keepDueForKeptWakes(runId) {
|
|
8096
|
+
const served = this.turnServedWakes.get(runId);
|
|
8097
|
+
if (!served) return;
|
|
8098
|
+
const settledFans = this.turnSettledFans.get(runId) ?? /* @__PURE__ */ new Map();
|
|
8099
|
+
const kept = await this.listWakeRequests(runId);
|
|
8100
|
+
if (kept.every((request) => wakeSeen(request, settledFans, served))) return;
|
|
8101
|
+
await this.makeDueBy(runId, this.clock() + this.leaseMs);
|
|
8102
|
+
}
|
|
8103
|
+
/** Bring a suspended run's timer forward to `at` (never later than the one it has). */
|
|
8104
|
+
async makeDueBy(runId, at) {
|
|
8105
|
+
const run = await this.store.getRun(runId);
|
|
8106
|
+
if (run?.status !== "suspended") return;
|
|
8107
|
+
if (run.wakeAt !== void 0 && run.wakeAt <= at) return;
|
|
8108
|
+
await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, runId, [
|
|
8109
|
+
"suspended"
|
|
8110
|
+
], {
|
|
8111
|
+
wakeAt: at
|
|
8112
|
+
});
|
|
7906
8113
|
}
|
|
7907
8114
|
/**
|
|
7908
8115
|
* A wake that found the run leased by another instance is dropped, and normally that is fine: the
|
|
@@ -7921,6 +8128,7 @@ var WorkflowEngine = class {
|
|
|
7921
8128
|
const remaining = Math.max(0, (lockedUntil ?? this.clock()) - this.clock());
|
|
7922
8129
|
const timer = setTimeout(() => {
|
|
7923
8130
|
this.leaseRetries.delete(runId);
|
|
8131
|
+
this.genericWakeRequested.delete(runId);
|
|
7924
8132
|
this.inBackground(this.retryDroppedWake(runId, attempt));
|
|
7925
8133
|
}, Math.min(remaining, LEASE_RECHECK_MAX_MS) + LEASE_RECHECK_SLACK_MS);
|
|
7926
8134
|
timer.unref?.();
|
|
@@ -7929,12 +8137,15 @@ var WorkflowEngine = class {
|
|
|
7929
8137
|
async retryDroppedWake(runId, attempt) {
|
|
7930
8138
|
if (this.draining) return;
|
|
7931
8139
|
const run = await this.store.getRun(runId);
|
|
7932
|
-
if (run?.status !== "suspended") return;
|
|
7933
8140
|
const nowMs = this.clock();
|
|
7934
|
-
if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) {
|
|
7935
|
-
|
|
8141
|
+
if (run?.status === "suspended" && run.lockedUntil !== void 0 && run.lockedUntil > nowMs && attempt < LEASE_RECHECK_ATTEMPTS) {
|
|
8142
|
+
this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
|
|
7936
8143
|
return;
|
|
7937
8144
|
}
|
|
8145
|
+
const unconditional = this.unconditionalRechecks.delete(runId);
|
|
8146
|
+
if (run?.status !== "suspended") return;
|
|
8147
|
+
if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) return;
|
|
8148
|
+
if (!unconditional && (await this.listWakeRequests(runId)).length === 0) return;
|
|
7938
8149
|
if (this.queuesRunsOverTransport) {
|
|
7939
8150
|
await this.wake(runId);
|
|
7940
8151
|
return;
|
|
@@ -8273,6 +8484,7 @@ ${run.workflowVersion}`;
|
|
|
8273
8484
|
updatedAt
|
|
8274
8485
|
});
|
|
8275
8486
|
if (!applied) return this.echoCurrentStatus(run.id);
|
|
8487
|
+
await this.keepDueForKeptWakes(run.id);
|
|
8276
8488
|
this.emit({
|
|
8277
8489
|
type: "run.suspended",
|
|
8278
8490
|
runId: run.id,
|
|
@@ -8966,6 +9178,7 @@ ${run.workflowVersion}`;
|
|
|
8966
9178
|
namespace: run.namespace
|
|
8967
9179
|
});
|
|
8968
9180
|
}
|
|
9181
|
+
await this.noteServedWakes(run.id);
|
|
8969
9182
|
if (run.status !== "cancelling" && registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
8970
9183
|
return this.singletonWaitResult(run);
|
|
8971
9184
|
}
|
|
@@ -9639,6 +9852,14 @@ ${run.workflowVersion}`;
|
|
|
9639
9852
|
throw new WorkflowSuspended();
|
|
9640
9853
|
}
|
|
9641
9854
|
/**
|
|
9855
|
+
* Whether a `pending` step a turn just read only waits for its result: {@link callRemote} would
|
|
9856
|
+
* suspend on it without writing (no redispatch deadline to stamp, the one stamped not yet due).
|
|
9857
|
+
*/
|
|
9858
|
+
stepIsWaiting(cp) {
|
|
9859
|
+
if (this.remoteRedispatchMs == null) return true;
|
|
9860
|
+
return cp.wakeAt != null && this.clock() < cp.wakeAt;
|
|
9861
|
+
}
|
|
9862
|
+
/**
|
|
9642
9863
|
* `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
|
|
9643
9864
|
* reserves the whole position block up front, so replay re-derives the same seqs.
|
|
9644
9865
|
*
|
|
@@ -9668,6 +9889,7 @@ ${run.workflowVersion}`;
|
|
|
9668
9889
|
const failures = [];
|
|
9669
9890
|
const unsent = [];
|
|
9670
9891
|
let inFlight = 0;
|
|
9892
|
+
let highest = -1;
|
|
9671
9893
|
let wakeAt;
|
|
9672
9894
|
for (let i = 0; i < inputs.length; i += 1) {
|
|
9673
9895
|
const seq = positions[i];
|
|
@@ -9677,10 +9899,19 @@ ${run.workflowVersion}`;
|
|
|
9677
9899
|
continue;
|
|
9678
9900
|
}
|
|
9679
9901
|
if (cp.name !== step.name) throw new NonDeterminismError(runId, seq, step.name, cp.name);
|
|
9902
|
+
if (seq > highest) highest = seq;
|
|
9680
9903
|
if (cp.status === "completed") {
|
|
9681
9904
|
outputs[i] = cp.output;
|
|
9682
9905
|
continue;
|
|
9683
9906
|
}
|
|
9907
|
+
if (cp.status === "pending" && this.stepIsWaiting(cp)) {
|
|
9908
|
+
inFlight += 1;
|
|
9909
|
+
if (cp.wakeAt != null) {
|
|
9910
|
+
const at = this.stepLeaseWakeAt(cp.wakeAt);
|
|
9911
|
+
wakeAt = Math.min(wakeAt ?? at, at);
|
|
9912
|
+
}
|
|
9913
|
+
continue;
|
|
9914
|
+
}
|
|
9684
9915
|
try {
|
|
9685
9916
|
outputs[i] = await this.callRemote(runId, seq, step, inputs[i], void 0, opts.transport, void 0, opts.admission, {
|
|
9686
9917
|
...routing,
|
|
@@ -9708,6 +9939,7 @@ ${run.workflowVersion}`;
|
|
|
9708
9939
|
this.awaitedRemoteSteps.get(runId)?.delete(seq);
|
|
9709
9940
|
}
|
|
9710
9941
|
}
|
|
9942
|
+
if (inFlight === 0 && highest >= 0) this.turnSettledFans.get(runId)?.set(first, highest);
|
|
9711
9943
|
const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
|
|
9712
9944
|
if (batch.length > 0) await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
|
|
9713
9945
|
if (inFlight + batch.length > 0) {
|
|
@@ -9738,53 +9970,62 @@ ${run.workflowVersion}`;
|
|
|
9738
9970
|
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
9739
9971
|
return outputs;
|
|
9740
9972
|
}
|
|
9741
|
-
/**
|
|
9973
|
+
/**
|
|
9974
|
+
* Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
|
|
9975
|
+
* {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
|
|
9976
|
+
* batch, where the store and transport have them.
|
|
9977
|
+
*/
|
|
9742
9978
|
async sendStepFan(runId, positions, step, inputs, batch, group, opts, routing) {
|
|
9743
9979
|
const workerGroup = stepGroup(step, routing?.namespace);
|
|
9744
9980
|
await this.ensureRoutable(workerGroup, step.requires);
|
|
9745
9981
|
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
9746
|
-
|
|
9747
|
-
|
|
9748
|
-
|
|
9749
|
-
|
|
9750
|
-
|
|
9751
|
-
|
|
9752
|
-
|
|
9753
|
-
|
|
9754
|
-
|
|
9755
|
-
|
|
9756
|
-
|
|
9757
|
-
|
|
9758
|
-
|
|
9759
|
-
|
|
9760
|
-
|
|
9761
|
-
|
|
9762
|
-
|
|
9763
|
-
|
|
9764
|
-
|
|
9982
|
+
const checkpoints = batch.map((i) => {
|
|
9983
|
+
const seq = positions[i];
|
|
9984
|
+
return {
|
|
9985
|
+
runId,
|
|
9986
|
+
seq,
|
|
9987
|
+
name: step.name,
|
|
9988
|
+
kind: "remote",
|
|
9989
|
+
stepId: stepId(runId, seq),
|
|
9990
|
+
status: "pending",
|
|
9991
|
+
input: inputs[i],
|
|
9992
|
+
attempts: 1,
|
|
9993
|
+
workerGroup,
|
|
9994
|
+
parallelGroup: group,
|
|
9995
|
+
enqueuedAt,
|
|
9996
|
+
startedAt: enqueuedAt,
|
|
9997
|
+
finishedAt: enqueuedAt
|
|
9998
|
+
};
|
|
9999
|
+
});
|
|
10000
|
+
if (this.store.saveCheckpoints) {
|
|
10001
|
+
await this.store.saveCheckpoints(checkpoints);
|
|
10002
|
+
} else {
|
|
10003
|
+
for (let at = 0; at < checkpoints.length; at += STEP_FAN_CHUNK) {
|
|
10004
|
+
await Promise.all(checkpoints.slice(at, at + STEP_FAN_CHUNK).map((cp) => this.store.saveCheckpoint(cp)));
|
|
10005
|
+
}
|
|
9765
10006
|
}
|
|
9766
|
-
|
|
9767
|
-
|
|
9768
|
-
|
|
9769
|
-
|
|
9770
|
-
|
|
9771
|
-
|
|
9772
|
-
|
|
9773
|
-
|
|
9774
|
-
|
|
9775
|
-
|
|
9776
|
-
|
|
9777
|
-
|
|
9778
|
-
|
|
9779
|
-
|
|
9780
|
-
|
|
9781
|
-
|
|
9782
|
-
|
|
9783
|
-
|
|
9784
|
-
|
|
9785
|
-
|
|
9786
|
-
|
|
9787
|
-
})
|
|
10007
|
+
const traceparent = this.traceparent?.();
|
|
10008
|
+
const context = this.context?.();
|
|
10009
|
+
await this.pool.dispatchMany(checkpoints.map((cp, k) => ({
|
|
10010
|
+
runId,
|
|
10011
|
+
seq: cp.seq,
|
|
10012
|
+
name: step.name,
|
|
10013
|
+
stepId: cp.stepId,
|
|
10014
|
+
group: workerGroup,
|
|
10015
|
+
input: inputs[batch[k]],
|
|
10016
|
+
traceparent,
|
|
10017
|
+
context,
|
|
10018
|
+
priority: opts.admission?.priority,
|
|
10019
|
+
attempt: 1
|
|
10020
|
+
})), opts.transport);
|
|
10021
|
+
for (const cp of checkpoints) {
|
|
10022
|
+
this.emit({
|
|
10023
|
+
type: "step.started",
|
|
10024
|
+
runId,
|
|
10025
|
+
seq: cp.seq,
|
|
10026
|
+
name: step.name,
|
|
10027
|
+
kind: "remote"
|
|
10028
|
+
});
|
|
9788
10029
|
}
|
|
9789
10030
|
}
|
|
9790
10031
|
/** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
|
|
@@ -9825,6 +10066,27 @@ ${run.workflowVersion}`;
|
|
|
9825
10066
|
error: s.reason
|
|
9826
10067
|
});
|
|
9827
10068
|
}
|
|
10069
|
+
/**
|
|
10070
|
+
* The checkpoints of one run at distinct `seqs`, each as its own outcome: one
|
|
10071
|
+
* {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
|
|
10072
|
+
* call rejecting falls back to one read each, so a failing read fails only its own entry.
|
|
10073
|
+
*/
|
|
10074
|
+
async readCheckpoints(runId, seqs) {
|
|
10075
|
+
if (seqs.length > 1 && this.store.getCheckpoints) {
|
|
10076
|
+
try {
|
|
10077
|
+
const bySeq = new Map((await this.store.getCheckpoints(runId, seqs)).map((cp) => [
|
|
10078
|
+
cp.seq,
|
|
10079
|
+
cp
|
|
10080
|
+
]));
|
|
10081
|
+
return seqs.map((seq) => ({
|
|
10082
|
+
status: "fulfilled",
|
|
10083
|
+
value: bySeq.get(seq) ?? null
|
|
10084
|
+
}));
|
|
10085
|
+
} catch {
|
|
10086
|
+
}
|
|
10087
|
+
}
|
|
10088
|
+
return Promise.allSettled(seqs.map((seq) => this.store.getCheckpoint(runId, seq)));
|
|
10089
|
+
}
|
|
9828
10090
|
/** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
|
|
9829
10091
|
* dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
|
|
9830
10092
|
* {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
|
|
@@ -9866,9 +10128,10 @@ ${run.workflowVersion}`;
|
|
|
9866
10128
|
*
|
|
9867
10129
|
* Results of one run are settled in batches: the first starts a batch, and the ones that arrive
|
|
9868
10130
|
* while it is being written form the next. Under load a wide fan's results are written by one
|
|
9869
|
-
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked
|
|
9870
|
-
*
|
|
9871
|
-
* transport acknowledges nothing that is not durable yet.
|
|
10131
|
+
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked only
|
|
10132
|
+
* by a batch with no more of the fan's results queued behind it. A result resolves only once its
|
|
10133
|
+
* own write has committed, so the transport acknowledges nothing that is not durable yet. How
|
|
10134
|
+
* large a batch gets is bounded by how many results the transport hands over at once.
|
|
9872
10135
|
*/
|
|
9873
10136
|
completeRemoteResult(result) {
|
|
9874
10137
|
return new Promise((resolve, reject) => {
|
|
@@ -9896,7 +10159,7 @@ ${run.workflowVersion}`;
|
|
|
9896
10159
|
const batch = queue.splice(0, RESULT_BATCH_MAX);
|
|
9897
10160
|
let outcomes;
|
|
9898
10161
|
try {
|
|
9899
|
-
outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result));
|
|
10162
|
+
outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result), (from, to) => queue.some(({ result }) => result.seq >= from && result.seq <= to));
|
|
9900
10163
|
} catch (err) {
|
|
9901
10164
|
outcomes = batch.map(() => ({
|
|
9902
10165
|
error: err
|
|
@@ -9904,8 +10167,8 @@ ${run.workflowVersion}`;
|
|
|
9904
10167
|
}
|
|
9905
10168
|
batch.forEach((q, i) => {
|
|
9906
10169
|
const outcome = outcomes[i] ?? false;
|
|
9907
|
-
if (
|
|
9908
|
-
else q.
|
|
10170
|
+
if (isResultError(outcome)) q.reject(outcome.error);
|
|
10171
|
+
else q.resolve(outcome);
|
|
9909
10172
|
});
|
|
9910
10173
|
}
|
|
9911
10174
|
this.resultQueues.delete(runId);
|
|
@@ -9914,8 +10177,13 @@ ${run.workflowVersion}`;
|
|
|
9914
10177
|
* Settle a batch of one run's step results. At most one of them resolves `true` (resume the run):
|
|
9915
10178
|
* that resume replays every write of the batch, so a second would only replay it again. A result
|
|
9916
10179
|
* whose read or write failed resolves its error, and the transport redelivers just that one.
|
|
10180
|
+
*
|
|
10181
|
+
* `queuedIn(from, to)` says whether a result for a seq in that range is already waiting for this
|
|
10182
|
+
* run's next batch on this instance. A fan with one waiting is not asked its barrier: that result
|
|
10183
|
+
* asks it once its own batch is written (as a settle, or as a redelivery of a settled step), so
|
|
10184
|
+
* under a backlog the barrier is read once per drained queue rather than once per batch.
|
|
9917
10185
|
*/
|
|
9918
|
-
async completeRemoteResults(runId, results) {
|
|
10186
|
+
async completeRemoteResults(runId, results, queuedIn = () => false) {
|
|
9919
10187
|
const outcomes = results.map(() => false);
|
|
9920
10188
|
const seen = /* @__PURE__ */ new Set();
|
|
9921
10189
|
const live = results.flatMap((r, i) => {
|
|
@@ -9925,7 +10193,7 @@ ${run.workflowVersion}`;
|
|
|
9925
10193
|
i
|
|
9926
10194
|
];
|
|
9927
10195
|
});
|
|
9928
|
-
const reads = await
|
|
10196
|
+
const reads = await this.readCheckpoints(runId, live.map((i) => results[i].seq));
|
|
9929
10197
|
const pending = [];
|
|
9930
10198
|
const resettled = [];
|
|
9931
10199
|
reads.forEach((read, k) => {
|
|
@@ -9946,7 +10214,7 @@ ${run.workflowVersion}`;
|
|
|
9946
10214
|
});
|
|
9947
10215
|
if (pending.length === 0) {
|
|
9948
10216
|
if (resettled.length === 0) return outcomes;
|
|
9949
|
-
return this.resumeForRedelivered(runId, resettled, outcomes);
|
|
10217
|
+
return this.resumeForRedelivered(runId, resettled, outcomes, queuedIn);
|
|
9950
10218
|
}
|
|
9951
10219
|
for (const { cp } of pending) void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
|
|
9952
10220
|
const run = await this.store.getRun(runId);
|
|
@@ -9973,7 +10241,9 @@ ${run.workflowVersion}`;
|
|
|
9973
10241
|
cp,
|
|
9974
10242
|
result: results[index]
|
|
9975
10243
|
});
|
|
9976
|
-
else if (claim !== false) outcomes[index] =
|
|
10244
|
+
else if (claim !== false) outcomes[index] = {
|
|
10245
|
+
error: claim.error
|
|
10246
|
+
};
|
|
9977
10247
|
});
|
|
9978
10248
|
for (const { cp, result } of settled) {
|
|
9979
10249
|
this.noteSettledHere(`${runId}:${result.seq}`);
|
|
@@ -9991,19 +10261,21 @@ ${run.workflowVersion}`;
|
|
|
9991
10261
|
});
|
|
9992
10262
|
}
|
|
9993
10263
|
const barriers = /* @__PURE__ */ new Map();
|
|
9994
|
-
|
|
10264
|
+
const wake = new FanWakeCollector();
|
|
10265
|
+
for (const { cp } of settled) wake.note(cp);
|
|
9995
10266
|
for (const { index, cp, result } of settled) {
|
|
9996
|
-
if (!await this.stepFanStillRunning(cp, result, barriers))
|
|
10267
|
+
if (!await this.stepFanStillRunning(cp, result, barriers, queuedIn)) {
|
|
10268
|
+
wake.add(index, cp, result.status);
|
|
10269
|
+
}
|
|
9997
10270
|
}
|
|
9998
|
-
|
|
9999
|
-
return outcomes;
|
|
10271
|
+
return wake.applyTo(outcomes);
|
|
10000
10272
|
}
|
|
10001
10273
|
/**
|
|
10002
10274
|
* The outcomes for a batch of redelivered results (their steps already settled): one of them
|
|
10003
10275
|
* resumes the run if it is `suspended` and, for a step of a fan, the fan's barrier is open. A
|
|
10004
10276
|
* resume of a run that had in fact moved on replays to its current wait and re-parks.
|
|
10005
10277
|
*/
|
|
10006
|
-
async resumeForRedelivered(runId, resettled, outcomes) {
|
|
10278
|
+
async resumeForRedelivered(runId, resettled, outcomes, queuedIn) {
|
|
10007
10279
|
const run = await this.store.getRun(runId);
|
|
10008
10280
|
if (run?.status !== "suspended") return outcomes;
|
|
10009
10281
|
const barriers = /* @__PURE__ */ new Map();
|
|
@@ -10014,9 +10286,11 @@ ${run.workflowVersion}`;
|
|
|
10014
10286
|
stepId: cp.stepId ?? stepId(runId, cp.seq),
|
|
10015
10287
|
status: cp.status === "failed" ? "failed" : "completed"
|
|
10016
10288
|
};
|
|
10017
|
-
if (!await this.stepFanStillRunning(cp, asResult, barriers)) {
|
|
10018
|
-
|
|
10019
|
-
|
|
10289
|
+
if (!await this.stepFanStillRunning(cp, asResult, barriers, queuedIn)) {
|
|
10290
|
+
const wake = new FanWakeCollector();
|
|
10291
|
+
for (const settled of resettled) wake.note(settled.cp);
|
|
10292
|
+
wake.add(index, cp, asResult.status);
|
|
10293
|
+
return wake.applyTo(outcomes);
|
|
10020
10294
|
}
|
|
10021
10295
|
}
|
|
10022
10296
|
return outcomes;
|
|
@@ -10027,15 +10301,17 @@ ${run.workflowVersion}`;
|
|
|
10027
10301
|
* retry needs the parent to schedule it). Without it every result would replay the whole parent,
|
|
10028
10302
|
* which makes a fan of N steps cost O(N²) checkpoint reads. `barriers` holds the answer per fan
|
|
10029
10303
|
* for the rest of a batch. A barrier that cannot be read counts as open: a spare resume replays and
|
|
10030
|
-
* re-suspends, a lost one strands the run.
|
|
10304
|
+
* re-suspends, a lost one strands the run. A fan with a result waiting in `queuedIn` is left to
|
|
10305
|
+
* that result (see {@link completeRemoteResults}).
|
|
10031
10306
|
*/
|
|
10032
|
-
async stepFanStillRunning(cp, result, barriers) {
|
|
10307
|
+
async stepFanStillRunning(cp, result, barriers, queuedIn) {
|
|
10033
10308
|
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
10034
10309
|
if (!fan) return false;
|
|
10035
10310
|
if (result.status === "failed") return false;
|
|
10036
10311
|
let closed = barriers.get(fan.firstSeq);
|
|
10037
10312
|
if (!closed) {
|
|
10038
|
-
|
|
10313
|
+
const lastSeq = fan.firstSeq + fan.size - 1;
|
|
10314
|
+
closed = queuedIn(fan.firstSeq, lastSeq) ? Promise.resolve(true) : this.stepFanPending(result.runId, fan.firstSeq, lastSeq).then((pending) => pending > 0, () => false);
|
|
10039
10315
|
barriers.set(fan.firstSeq, closed);
|
|
10040
10316
|
}
|
|
10041
10317
|
if (await closed) return true;
|
|
@@ -10869,6 +11145,10 @@ var CodecStateStore = class {
|
|
|
10869
11145
|
saveCheckpointIfPending;
|
|
10870
11146
|
/** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
|
|
10871
11147
|
saveCheckpointsIfPending;
|
|
11148
|
+
/** Encodes each checkpoint written. Bound only when the inner store has it. */
|
|
11149
|
+
saveCheckpoints;
|
|
11150
|
+
/** Decodes each checkpoint read. Bound only when the inner store has it. */
|
|
11151
|
+
getCheckpoints;
|
|
10872
11152
|
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
|
|
10873
11153
|
countCheckpoints;
|
|
10874
11154
|
/** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
|
|
@@ -10919,6 +11199,10 @@ var CodecStateStore = class {
|
|
|
10919
11199
|
if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
|
|
10920
11200
|
const saveManyIfPending = inner.saveCheckpointsIfPending;
|
|
10921
11201
|
if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11202
|
+
const saveMany = inner.saveCheckpoints;
|
|
11203
|
+
if (saveMany) this.saveCheckpoints = (cps) => saveMany.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11204
|
+
const getMany = inner.getCheckpoints;
|
|
11205
|
+
if (getMany) this.getCheckpoints = async (runId, seqs) => (await getMany.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
|
|
10922
11206
|
const countCps = inner.countCheckpoints;
|
|
10923
11207
|
if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
|
|
10924
11208
|
const stats = inner.checkpointStats;
|
|
@@ -11555,11 +11839,30 @@ var InMemoryStateStore = class {
|
|
|
11555
11839
|
...cp
|
|
11556
11840
|
} : null;
|
|
11557
11841
|
}
|
|
11842
|
+
async getCheckpoints(runId, seqs) {
|
|
11843
|
+
return [
|
|
11844
|
+
...new Set(seqs)
|
|
11845
|
+
].sort((a, b) => a - b).flatMap((seq) => {
|
|
11846
|
+
const cp = this.checkpoints.get(this.key(runId, seq));
|
|
11847
|
+
return cp ? [
|
|
11848
|
+
{
|
|
11849
|
+
...cp
|
|
11850
|
+
}
|
|
11851
|
+
] : [];
|
|
11852
|
+
});
|
|
11853
|
+
}
|
|
11558
11854
|
async saveCheckpoint(checkpoint) {
|
|
11559
11855
|
this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
|
|
11560
11856
|
...checkpoint
|
|
11561
11857
|
});
|
|
11562
11858
|
}
|
|
11859
|
+
async saveCheckpoints(checkpoints) {
|
|
11860
|
+
for (const checkpoint of checkpoints) {
|
|
11861
|
+
this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
|
|
11862
|
+
...checkpoint
|
|
11863
|
+
});
|
|
11864
|
+
}
|
|
11865
|
+
}
|
|
11563
11866
|
async saveCheckpointIfPending(checkpoint) {
|
|
11564
11867
|
const key = this.key(checkpoint.runId, checkpoint.seq);
|
|
11565
11868
|
if (this.checkpoints.get(key)?.status !== "pending") return false;
|