@dudousxd/nestjs-durable-core 0.85.6 → 0.85.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +397 -94
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +114 -12
- package/dist/index.d.ts +114 -12
- package/dist/index.js +397 -94
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -3561,6 +3561,7 @@ function constantTimeEquals(a, b) {
|
|
|
3561
3561
|
__name(constantTimeEquals, "constantTimeEquals");
|
|
3562
3562
|
|
|
3563
3563
|
// src/transport-pool.ts
|
|
3564
|
+
var DISPATCH_CHUNK = 200;
|
|
3564
3565
|
var TransportPool = class {
|
|
3565
3566
|
static {
|
|
3566
3567
|
__name(this, "TransportPool");
|
|
@@ -3620,6 +3621,28 @@ var TransportPool = class {
|
|
|
3620
3621
|
}
|
|
3621
3622
|
throw lastErr ?? new Error("no transport accepted the dispatch");
|
|
3622
3623
|
}
|
|
3624
|
+
/**
|
|
3625
|
+
* {@link dispatch} for many tasks: one {@link Transport.dispatchMany} call on the preferred (or
|
|
3626
|
+
* first) transport when it has one, else (or when that call rejects) each task through
|
|
3627
|
+
* {@link dispatch}, a bounded number at a time.
|
|
3628
|
+
*/
|
|
3629
|
+
async dispatchMany(tasks, preferId) {
|
|
3630
|
+
if (tasks.length === 0) return;
|
|
3631
|
+
const first = this.ordered(preferId)[0];
|
|
3632
|
+
if (first?.transport.dispatchMany) {
|
|
3633
|
+
try {
|
|
3634
|
+
await first.transport.dispatchMany(tasks.map((task) => ({
|
|
3635
|
+
...task,
|
|
3636
|
+
transport: first.id
|
|
3637
|
+
})));
|
|
3638
|
+
return;
|
|
3639
|
+
} catch {
|
|
3640
|
+
}
|
|
3641
|
+
}
|
|
3642
|
+
for (let at = 0; at < tasks.length; at += DISPATCH_CHUNK) {
|
|
3643
|
+
await Promise.all(tasks.slice(at, at + DISPATCH_CHUNK).map((task) => this.dispatch(task, preferId)));
|
|
3644
|
+
}
|
|
3645
|
+
}
|
|
3623
3646
|
/** Worker-health for `group`, merged across every transport that can report it (a group's queue
|
|
3624
3647
|
* and its workers may live on any pinned transport). Undefined when no transport implements
|
|
3625
3648
|
* `groupHealth` (e.g. a pure in-process transport — nothing to introspect). */
|
|
@@ -4839,6 +4862,15 @@ var PARENT_NOTIFY_BACKOFF_MS = 250;
|
|
|
4839
4862
|
var LEASE_RECHECK_MAX_MS = 2e3;
|
|
4840
4863
|
var LEASE_RECHECK_SLACK_MS = 100;
|
|
4841
4864
|
var LEASE_RECHECK_ATTEMPTS = 30;
|
|
4865
|
+
function wakeSeen(request, settledFans, served) {
|
|
4866
|
+
if (request.id !== void 0 && served.has(request.id)) return true;
|
|
4867
|
+
return isFanWakeRequest(request.payload) && request.payload.fans.every(([first, seq]) => (settledFans.get(first) ?? -1) >= seq);
|
|
4868
|
+
}
|
|
4869
|
+
__name(wakeSeen, "wakeSeen");
|
|
4870
|
+
function isFanWakeRequest(payload) {
|
|
4871
|
+
return typeof payload === "object" && payload !== null && Array.isArray(payload.fans);
|
|
4872
|
+
}
|
|
4873
|
+
__name(isFanWakeRequest, "isFanWakeRequest");
|
|
4842
4874
|
function deferred() {
|
|
4843
4875
|
let resolve;
|
|
4844
4876
|
let reject;
|
|
@@ -4857,12 +4889,51 @@ __name(deferred, "deferred");
|
|
|
4857
4889
|
var pendingStateKey = /* @__PURE__ */ __name((at) => String(new Date(at).getTime()), "pendingStateKey");
|
|
4858
4890
|
var freshPendingKey = /* @__PURE__ */ __name(() => `${Date.now()}-${globalThis.crypto.randomUUID().slice(0, 8)}`, "freshPendingKey");
|
|
4859
4891
|
var wakeRequestToken = /* @__PURE__ */ __name((runId) => `durable:wake-after-turn:${runId}`, "wakeRequestToken");
|
|
4892
|
+
var WAKE_REQUEST_SCAN = 100;
|
|
4860
4893
|
var CHILD_SIGNAL_PREFIX = "signal:child:";
|
|
4861
4894
|
var BUFFERED_SWEEP_PAGE = 500;
|
|
4862
4895
|
var BUFFERED_SWEEP_PAGES_PER_CALL = 4;
|
|
4863
4896
|
var BUFFERED_SWEEP_REST_MS = 10 * 6e4;
|
|
4864
4897
|
var DELIVER_CHUNK = 50;
|
|
4865
4898
|
var RESULT_BATCH_MAX = 256;
|
|
4899
|
+
function isResultError(outcome) {
|
|
4900
|
+
return typeof outcome === "object" && "error" in outcome;
|
|
4901
|
+
}
|
|
4902
|
+
__name(isResultError, "isResultError");
|
|
4903
|
+
var FanWakeCollector = class FanWakeCollector2 {
|
|
4904
|
+
static {
|
|
4905
|
+
__name(this, "FanWakeCollector");
|
|
4906
|
+
}
|
|
4907
|
+
carrier;
|
|
4908
|
+
generic = false;
|
|
4909
|
+
fan;
|
|
4910
|
+
/** The highest seq the batch settled, per fan. */
|
|
4911
|
+
highest = /* @__PURE__ */ new Map();
|
|
4912
|
+
/** A checkpoint the batch settled. */
|
|
4913
|
+
note(cp) {
|
|
4914
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
4915
|
+
if (fan) this.highest.set(fan.firstSeq, Math.max(this.highest.get(fan.firstSeq) ?? cp.seq, cp.seq));
|
|
4916
|
+
}
|
|
4917
|
+
/** A settled checkpoint that resumes the run. */
|
|
4918
|
+
add(index, cp, status) {
|
|
4919
|
+
this.carrier = index;
|
|
4920
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
4921
|
+
if (!fan || status === "failed" || this.fan !== void 0 && this.fan !== fan.firstSeq) {
|
|
4922
|
+
this.generic = true;
|
|
4923
|
+
return;
|
|
4924
|
+
}
|
|
4925
|
+
this.fan = fan.firstSeq;
|
|
4926
|
+
}
|
|
4927
|
+
applyTo(outcomes) {
|
|
4928
|
+
if (this.carrier === void 0) return outcomes;
|
|
4929
|
+
const seq = this.fan === void 0 ? void 0 : this.highest.get(this.fan);
|
|
4930
|
+
outcomes[this.carrier] = this.generic || this.fan === void 0 || seq === void 0 ? true : {
|
|
4931
|
+
first: this.fan,
|
|
4932
|
+
seq
|
|
4933
|
+
};
|
|
4934
|
+
return outcomes;
|
|
4935
|
+
}
|
|
4936
|
+
};
|
|
4866
4937
|
function shuffle(items) {
|
|
4867
4938
|
for (let i = items.length - 1; i > 0; i -= 1) {
|
|
4868
4939
|
const j = Math.floor(Math.random() * (i + 1));
|
|
@@ -5051,6 +5122,23 @@ var WorkflowEngine = class {
|
|
|
5051
5122
|
/** Re-checks scheduled for runs whose wake found another instance's lease (see {@link execute}). */
|
|
5052
5123
|
leaseRetries = /* @__PURE__ */ new Map();
|
|
5053
5124
|
/**
|
|
5125
|
+
* Runs whose scheduled re-check resumes them whatever happened meanwhile. Without an entry the
|
|
5126
|
+
* re-check only stood for fan wakes kept as requests, and resumes the run only if a request is
|
|
5127
|
+
* still there: the turn that held the lease takes them once it ends (see {@link retryDroppedWake}).
|
|
5128
|
+
*/
|
|
5129
|
+
unconditionalRechecks = /* @__PURE__ */ new Set();
|
|
5130
|
+
/** Runs a generic wake request was written for in the current re-check cycle (see {@link execute}). */
|
|
5131
|
+
genericWakeRequested = /* @__PURE__ */ new Set();
|
|
5132
|
+
/** Why each run is being woken on this instance, until the turn that answers it starts. */
|
|
5133
|
+
wakeReasons = /* @__PURE__ */ new Map();
|
|
5134
|
+
/**
|
|
5135
|
+
* The step fans each executing turn on this instance read with nothing in flight: first seq -> the
|
|
5136
|
+
* highest seq read. Written by {@link callRemoteAll}, read once the turn ends.
|
|
5137
|
+
*/
|
|
5138
|
+
turnSettledFans = /* @__PURE__ */ new Map();
|
|
5139
|
+
/** The wake requests each executing turn on this instance serves (see {@link noteServedWakes}). */
|
|
5140
|
+
turnServedWakes = /* @__PURE__ */ new Map();
|
|
5141
|
+
/**
|
|
5054
5142
|
* Every run turn and child start this instance takes on itself, under one budget of
|
|
5055
5143
|
* {@link LOCAL_RUN_CONCURRENCY}: the runs it dispatches to itself, a fan's deferred child starts,
|
|
5056
5144
|
* enqueued pickups and the runs it takes off the transport's run queue.
|
|
@@ -5174,8 +5262,9 @@ var WorkflowEngine = class {
|
|
|
5174
5262
|
}
|
|
5175
5263
|
return;
|
|
5176
5264
|
}
|
|
5177
|
-
|
|
5178
|
-
|
|
5265
|
+
const wake = await this.completeRemoteResult(result);
|
|
5266
|
+
if (!wake) return;
|
|
5267
|
+
await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId, wake === true ? void 0 : wake).then(() => void 0, (err) => {
|
|
5179
5268
|
if (err instanceof NamespaceMismatch) return;
|
|
5180
5269
|
throw err;
|
|
5181
5270
|
}));
|
|
@@ -5392,7 +5481,10 @@ var WorkflowEngine = class {
|
|
|
5392
5481
|
const resumed = await this.resumeLeased([
|
|
5393
5482
|
run
|
|
5394
5483
|
], nowMs, (locked) => this.reconcileSignalWaiter(locked, waiters.get(locked.id)));
|
|
5395
|
-
if (resumed.length === 0)
|
|
5484
|
+
if (resumed.length === 0) {
|
|
5485
|
+
this.unconditionalRechecks.add(runId);
|
|
5486
|
+
this.recheckAfterLease(runId, run.lockedUntil);
|
|
5487
|
+
}
|
|
5396
5488
|
return;
|
|
5397
5489
|
}
|
|
5398
5490
|
await this.resumeImplicit(runId).catch((err) => {
|
|
@@ -5959,15 +6051,27 @@ var WorkflowEngine = class {
|
|
|
5959
6051
|
* terminal too.
|
|
5960
6052
|
*/
|
|
5961
6053
|
resume(runId) {
|
|
6054
|
+
this.noteWakeReason(runId, void 0);
|
|
5962
6055
|
return this.resumeRun(runId, true);
|
|
5963
6056
|
}
|
|
6057
|
+
/** Record why `runId` is being woken: a step fan's barrier opening, or (`undefined`) anything else. */
|
|
6058
|
+
noteWakeReason(runId, fan) {
|
|
6059
|
+
const reason = this.wakeReasons.get(runId) ?? {
|
|
6060
|
+
generic: false,
|
|
6061
|
+
fans: /* @__PURE__ */ new Map()
|
|
6062
|
+
};
|
|
6063
|
+
if (fan) reason.fans.set(fan.first, Math.max(reason.fans.get(fan.first) ?? fan.seq, fan.seq));
|
|
6064
|
+
else reason.generic = true;
|
|
6065
|
+
this.wakeReasons.set(runId, reason);
|
|
6066
|
+
}
|
|
5964
6067
|
/**
|
|
5965
6068
|
* {@link resume} for every IMPLICIT wake-up — a delivered signal/child completion, a late step
|
|
5966
6069
|
* result, a due timer, a redelivered dispatch, a queue-slot wake. A `failed` run is terminal to
|
|
5967
6070
|
* these: only an explicit retry ({@link requeue} / {@link resume}) may re-drive it. Otherwise a
|
|
5968
6071
|
* stale waiter or a late result left behind by a failed run silently re-executes its body.
|
|
5969
6072
|
*/
|
|
5970
|
-
resumeImplicit(runId) {
|
|
6073
|
+
resumeImplicit(runId, fan) {
|
|
6074
|
+
if (fan !== "coalesced") this.noteWakeReason(runId, fan);
|
|
5971
6075
|
const turn = this.wakeTurns.get(runId);
|
|
5972
6076
|
if (turn) {
|
|
5973
6077
|
turn.again ??= deferred();
|
|
@@ -5992,7 +6096,7 @@ var WorkflowEngine = class {
|
|
|
5992
6096
|
});
|
|
5993
6097
|
return;
|
|
5994
6098
|
}
|
|
5995
|
-
const next = this.resumeImplicit(runId);
|
|
6099
|
+
const next = this.resumeImplicit(runId, "coalesced");
|
|
5996
6100
|
next.then(again.resolve, again.reject);
|
|
5997
6101
|
this.inBackground(next);
|
|
5998
6102
|
}, "finish");
|
|
@@ -8057,9 +8161,26 @@ var WorkflowEngine = class {
|
|
|
8057
8161
|
async execute(run, fn) {
|
|
8058
8162
|
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
|
|
8059
8163
|
const lockNow = this.clock();
|
|
8164
|
+
const reason = this.wakeReasons.get(run.id) ?? {
|
|
8165
|
+
generic: true,
|
|
8166
|
+
fans: /* @__PURE__ */ new Map()
|
|
8167
|
+
};
|
|
8168
|
+
this.wakeReasons.delete(run.id);
|
|
8060
8169
|
if (run.lockedBy !== this.instanceId || (run.lockedUntil ?? 0) <= lockNow) {
|
|
8061
8170
|
if (!await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow)) {
|
|
8062
|
-
if (
|
|
8171
|
+
if (reason.generic) {
|
|
8172
|
+
if (!this.genericWakeRequested.has(run.id)) {
|
|
8173
|
+
this.genericWakeRequested.add(run.id);
|
|
8174
|
+
this.inBackground(this.requestWakeAfterTurn(run.id, null));
|
|
8175
|
+
}
|
|
8176
|
+
} else {
|
|
8177
|
+
const fans = {
|
|
8178
|
+
fans: [
|
|
8179
|
+
...reason.fans
|
|
8180
|
+
]
|
|
8181
|
+
};
|
|
8182
|
+
this.inBackground(this.requestWakeAfterTurn(run.id, fans));
|
|
8183
|
+
}
|
|
8063
8184
|
this.recheckAfterLease(run.id, run.lockedUntil);
|
|
8064
8185
|
return {
|
|
8065
8186
|
runId: run.id,
|
|
@@ -8067,56 +8188,142 @@ var WorkflowEngine = class {
|
|
|
8067
8188
|
};
|
|
8068
8189
|
}
|
|
8069
8190
|
}
|
|
8191
|
+
const settledFans = /* @__PURE__ */ new Map();
|
|
8192
|
+
this.turnSettledFans.set(run.id, settledFans);
|
|
8070
8193
|
const held = this.holdExecutionLease(run.id);
|
|
8071
8194
|
try {
|
|
8195
|
+
const served = /* @__PURE__ */ new Set();
|
|
8196
|
+
this.turnServedWakes.set(run.id, served);
|
|
8197
|
+
if (registered?.remote) await this.noteServedWakes(run.id);
|
|
8072
8198
|
const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
|
|
8073
8199
|
if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
|
|
8074
8200
|
void this.singletons.wakeNext(run).catch(() => void 0);
|
|
8075
8201
|
}
|
|
8076
|
-
if (result.status === "suspended")
|
|
8077
|
-
|
|
8078
|
-
|
|
8202
|
+
if (result.status === "suspended") {
|
|
8203
|
+
this.inBackground(this.honorWakeRequests(run.id, settledFans, served));
|
|
8204
|
+
} else if (TERMINAL_RUN_STATUSES.includes(result.status)) {
|
|
8205
|
+
this.inBackground(this.listWakeRequests(run.id, true).then((kept) => this.dropWakeRequests(kept)));
|
|
8079
8206
|
}
|
|
8080
8207
|
return result;
|
|
8081
8208
|
} finally {
|
|
8082
8209
|
held.stop();
|
|
8210
|
+
if (this.turnSettledFans.get(run.id) === settledFans) this.turnSettledFans.delete(run.id);
|
|
8211
|
+
this.turnServedWakes.delete(run.id);
|
|
8083
8212
|
}
|
|
8084
8213
|
}
|
|
8085
8214
|
/**
|
|
8086
8215
|
* Keep a wake that found `runId`'s lease held by a turn executing elsewhere, durably, so that turn
|
|
8087
8216
|
* runs again once it ends instead of suspending on a snapshot that predates what the wake brought
|
|
8088
|
-
* (a signal, a child's completion). The request is a buffered
|
|
8089
|
-
* on
|
|
8217
|
+
* (a signal, a child's completion). The request is a buffered event under a name no workflow waits
|
|
8218
|
+
* on. It stays until a turn that started after it has settled: the holder looks at it after it
|
|
8219
|
+
* releases the lease ({@link honorWakeRequests}), and the turn that serves it drops it.
|
|
8090
8220
|
*
|
|
8091
8221
|
* Written first and looked at second, against a holder that releases first and looks second:
|
|
8092
8222
|
* whichever look runs last sees the other side's write. So when the run is no longer executing by
|
|
8093
|
-
* the time this re-reads it, the holder may have looked already, and this side
|
|
8094
|
-
*
|
|
8223
|
+
* the time this re-reads it, the holder may have looked already, and this side wakes the run
|
|
8224
|
+
* itself once the lease is free ({@link retryDroppedWake}). While the holder still runs, the run is
|
|
8225
|
+
* made due when the holder's lease runs out, so a holder that goes down before it settles leaves
|
|
8226
|
+
* the run to any instance's timer poll; a holder that settles sets the run's timer itself.
|
|
8095
8227
|
*/
|
|
8096
|
-
async requestWakeAfterTurn(runId) {
|
|
8097
|
-
|
|
8228
|
+
async requestWakeAfterTurn(runId, request) {
|
|
8229
|
+
const id = globalThis.crypto.randomUUID();
|
|
8230
|
+
await this.store.bufferEvent({
|
|
8231
|
+
name: wakeRequestToken(runId),
|
|
8232
|
+
payload: request,
|
|
8233
|
+
id,
|
|
8234
|
+
publishedAt: this.clock()
|
|
8235
|
+
});
|
|
8098
8236
|
const run = await this.store.getRun(runId);
|
|
8099
|
-
if (run
|
|
8100
|
-
|
|
8101
|
-
|
|
8237
|
+
if (!run || TERMINAL_RUN_STATUSES.includes(run.status)) {
|
|
8238
|
+
await this.store.removeBufferedEvent(id);
|
|
8239
|
+
return;
|
|
8240
|
+
}
|
|
8241
|
+
if (run.status === "running") return;
|
|
8242
|
+
if (run.lockedBy !== void 0 && run.lockedBy !== this.instanceId) {
|
|
8243
|
+
await this.makeDueBy(runId, (run.lockedUntil ?? this.clock()) + LEASE_RECHECK_SLACK_MS);
|
|
8102
8244
|
return;
|
|
8103
8245
|
}
|
|
8104
|
-
if (!await this.takeWakeRequests(runId)) return;
|
|
8105
|
-
if (TERMINAL_RUN_STATUSES.includes(run.status)) return;
|
|
8106
8246
|
this.recheckAfterLease(runId, run.lockedUntil);
|
|
8107
8247
|
}
|
|
8108
|
-
/**
|
|
8109
|
-
|
|
8110
|
-
|
|
8248
|
+
/**
|
|
8249
|
+
* After a turn that suspended released its lease: drop the wakes it served, and run again if a
|
|
8250
|
+
* wake was kept for it meanwhile, unless every such wake was a fan wake this turn already saw: it
|
|
8251
|
+
* read that fan with nothing in flight, at least as far as the batch that opened its barrier.
|
|
8252
|
+
* The run is made due before it is woken, so if this instance goes down before that turn runs,
|
|
8253
|
+
* the timer poll of another instance runs it once this instance's hold on it would have lapsed.
|
|
8254
|
+
*/
|
|
8255
|
+
async honorWakeRequests(runId, settledFans, served) {
|
|
8256
|
+
const kept = await this.listWakeRequests(runId, true);
|
|
8257
|
+
const seen = /* @__PURE__ */ __name((request) => wakeSeen(request, settledFans, served), "seen");
|
|
8258
|
+
const needed = kept.filter((request) => !seen(request));
|
|
8259
|
+
await this.dropWakeRequests(kept.filter(seen));
|
|
8260
|
+
if (needed.length === 0) return;
|
|
8261
|
+
await this.makeDueBy(runId, this.clock() + this.leaseMs);
|
|
8111
8262
|
await this.wake(runId).catch((err) => {
|
|
8112
8263
|
if (!(err instanceof NamespaceMismatch)) throw err;
|
|
8113
8264
|
});
|
|
8114
8265
|
}
|
|
8115
|
-
/**
|
|
8116
|
-
|
|
8117
|
-
|
|
8118
|
-
|
|
8119
|
-
|
|
8266
|
+
/**
|
|
8267
|
+
* The wake requests kept for `runId`. With `legacy`, also those an older engine kept as buffered
|
|
8268
|
+
* signals, which are taken as they are read.
|
|
8269
|
+
*/
|
|
8270
|
+
async listWakeRequests(runId, legacy = false) {
|
|
8271
|
+
const token = wakeRequestToken(runId);
|
|
8272
|
+
const kept = (await this.store.listBufferedEvents(token, WAKE_REQUEST_SCAN)).map((request) => ({
|
|
8273
|
+
id: request.id,
|
|
8274
|
+
payload: request.payload
|
|
8275
|
+
}));
|
|
8276
|
+
if (!legacy) return kept;
|
|
8277
|
+
for (; ; ) {
|
|
8278
|
+
const signal = await this.store.takeBufferedSignal(token);
|
|
8279
|
+
if (!signal) return kept;
|
|
8280
|
+
kept.push({
|
|
8281
|
+
payload: signal.payload
|
|
8282
|
+
});
|
|
8283
|
+
}
|
|
8284
|
+
}
|
|
8285
|
+
/** Delete `requests` (the ones kept as buffered events). */
|
|
8286
|
+
async dropWakeRequests(requests) {
|
|
8287
|
+
await Promise.all(requests.flatMap((request) => request.id === void 0 ? [] : [
|
|
8288
|
+
this.store.removeBufferedEvent(request.id)
|
|
8289
|
+
]));
|
|
8290
|
+
}
|
|
8291
|
+
/**
|
|
8292
|
+
* Record, for the turn executing `runId` here, the wakes kept for the run so far. Called before the
|
|
8293
|
+
* turn reads the run's state, so the turn serves them: they are dropped once it has settled, and
|
|
8294
|
+
* until then stay for whoever runs the run next.
|
|
8295
|
+
*/
|
|
8296
|
+
async noteServedWakes(runId) {
|
|
8297
|
+
const served = this.turnServedWakes.get(runId);
|
|
8298
|
+
if (!served) return;
|
|
8299
|
+
for (const request of await this.store.listBufferedEvents(wakeRequestToken(runId), WAKE_REQUEST_SCAN)) {
|
|
8300
|
+
served.add(request.id);
|
|
8301
|
+
}
|
|
8302
|
+
}
|
|
8303
|
+
/**
|
|
8304
|
+
* Right after a turn of `runId` wrote its suspension, while it still holds the lease: if a wake
|
|
8305
|
+
* was kept for the run during the turn that the turn did not see, keep the run due when the lease
|
|
8306
|
+
* runs out. The suspension overwrote the timer the instance keeping the wake set, and this instance
|
|
8307
|
+
* may go down before it looks at the wake once it has released the lease.
|
|
8308
|
+
*/
|
|
8309
|
+
async keepDueForKeptWakes(runId) {
|
|
8310
|
+
const served = this.turnServedWakes.get(runId);
|
|
8311
|
+
if (!served) return;
|
|
8312
|
+
const settledFans = this.turnSettledFans.get(runId) ?? /* @__PURE__ */ new Map();
|
|
8313
|
+
const kept = await this.listWakeRequests(runId);
|
|
8314
|
+
if (kept.every((request) => wakeSeen(request, settledFans, served))) return;
|
|
8315
|
+
await this.makeDueBy(runId, this.clock() + this.leaseMs);
|
|
8316
|
+
}
|
|
8317
|
+
/** Bring a suspended run's timer forward to `at` (never later than the one it has). */
|
|
8318
|
+
async makeDueBy(runId, at) {
|
|
8319
|
+
const run = await this.store.getRun(runId);
|
|
8320
|
+
if (run?.status !== "suspended") return;
|
|
8321
|
+
if (run.wakeAt !== void 0 && run.wakeAt <= at) return;
|
|
8322
|
+
await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, runId, [
|
|
8323
|
+
"suspended"
|
|
8324
|
+
], {
|
|
8325
|
+
wakeAt: at
|
|
8326
|
+
});
|
|
8120
8327
|
}
|
|
8121
8328
|
/**
|
|
8122
8329
|
* A wake that found the run leased by another instance is dropped, and normally that is fine: the
|
|
@@ -8135,6 +8342,7 @@ var WorkflowEngine = class {
|
|
|
8135
8342
|
const remaining = Math.max(0, (lockedUntil ?? this.clock()) - this.clock());
|
|
8136
8343
|
const timer = setTimeout(() => {
|
|
8137
8344
|
this.leaseRetries.delete(runId);
|
|
8345
|
+
this.genericWakeRequested.delete(runId);
|
|
8138
8346
|
this.inBackground(this.retryDroppedWake(runId, attempt));
|
|
8139
8347
|
}, Math.min(remaining, LEASE_RECHECK_MAX_MS) + LEASE_RECHECK_SLACK_MS);
|
|
8140
8348
|
timer.unref?.();
|
|
@@ -8143,12 +8351,15 @@ var WorkflowEngine = class {
|
|
|
8143
8351
|
async retryDroppedWake(runId, attempt) {
|
|
8144
8352
|
if (this.draining) return;
|
|
8145
8353
|
const run = await this.store.getRun(runId);
|
|
8146
|
-
if (run?.status !== "suspended") return;
|
|
8147
8354
|
const nowMs = this.clock();
|
|
8148
|
-
if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) {
|
|
8149
|
-
|
|
8355
|
+
if (run?.status === "suspended" && run.lockedUntil !== void 0 && run.lockedUntil > nowMs && attempt < LEASE_RECHECK_ATTEMPTS) {
|
|
8356
|
+
this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
|
|
8150
8357
|
return;
|
|
8151
8358
|
}
|
|
8359
|
+
const unconditional = this.unconditionalRechecks.delete(runId);
|
|
8360
|
+
if (run?.status !== "suspended") return;
|
|
8361
|
+
if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) return;
|
|
8362
|
+
if (!unconditional && (await this.listWakeRequests(runId)).length === 0) return;
|
|
8152
8363
|
if (this.queuesRunsOverTransport) {
|
|
8153
8364
|
await this.wake(runId);
|
|
8154
8365
|
return;
|
|
@@ -8487,6 +8698,7 @@ ${run.workflowVersion}`;
|
|
|
8487
8698
|
updatedAt
|
|
8488
8699
|
});
|
|
8489
8700
|
if (!applied) return this.echoCurrentStatus(run.id);
|
|
8701
|
+
await this.keepDueForKeptWakes(run.id);
|
|
8490
8702
|
this.emit({
|
|
8491
8703
|
type: "run.suspended",
|
|
8492
8704
|
runId: run.id,
|
|
@@ -9180,6 +9392,7 @@ ${run.workflowVersion}`;
|
|
|
9180
9392
|
namespace: run.namespace
|
|
9181
9393
|
});
|
|
9182
9394
|
}
|
|
9395
|
+
await this.noteServedWakes(run.id);
|
|
9183
9396
|
if (run.status !== "cancelling" && registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
9184
9397
|
return this.singletonWaitResult(run);
|
|
9185
9398
|
}
|
|
@@ -9853,6 +10066,14 @@ ${run.workflowVersion}`;
|
|
|
9853
10066
|
throw new WorkflowSuspended();
|
|
9854
10067
|
}
|
|
9855
10068
|
/**
|
|
10069
|
+
* Whether a `pending` step a turn just read only waits for its result: {@link callRemote} would
|
|
10070
|
+
* suspend on it without writing (no redispatch deadline to stamp, the one stamped not yet due).
|
|
10071
|
+
*/
|
|
10072
|
+
stepIsWaiting(cp) {
|
|
10073
|
+
if (this.remoteRedispatchMs == null) return true;
|
|
10074
|
+
return cp.wakeAt != null && this.clock() < cp.wakeAt;
|
|
10075
|
+
}
|
|
10076
|
+
/**
|
|
9856
10077
|
* `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
|
|
9857
10078
|
* reserves the whole position block up front, so replay re-derives the same seqs.
|
|
9858
10079
|
*
|
|
@@ -9882,6 +10103,7 @@ ${run.workflowVersion}`;
|
|
|
9882
10103
|
const failures = [];
|
|
9883
10104
|
const unsent = [];
|
|
9884
10105
|
let inFlight = 0;
|
|
10106
|
+
let highest = -1;
|
|
9885
10107
|
let wakeAt;
|
|
9886
10108
|
for (let i = 0; i < inputs.length; i += 1) {
|
|
9887
10109
|
const seq = positions[i];
|
|
@@ -9891,10 +10113,19 @@ ${run.workflowVersion}`;
|
|
|
9891
10113
|
continue;
|
|
9892
10114
|
}
|
|
9893
10115
|
if (cp.name !== step.name) throw new NonDeterminismError(runId, seq, step.name, cp.name);
|
|
10116
|
+
if (seq > highest) highest = seq;
|
|
9894
10117
|
if (cp.status === "completed") {
|
|
9895
10118
|
outputs[i] = cp.output;
|
|
9896
10119
|
continue;
|
|
9897
10120
|
}
|
|
10121
|
+
if (cp.status === "pending" && this.stepIsWaiting(cp)) {
|
|
10122
|
+
inFlight += 1;
|
|
10123
|
+
if (cp.wakeAt != null) {
|
|
10124
|
+
const at = this.stepLeaseWakeAt(cp.wakeAt);
|
|
10125
|
+
wakeAt = Math.min(wakeAt ?? at, at);
|
|
10126
|
+
}
|
|
10127
|
+
continue;
|
|
10128
|
+
}
|
|
9898
10129
|
try {
|
|
9899
10130
|
outputs[i] = await this.callRemote(runId, seq, step, inputs[i], void 0, opts.transport, void 0, opts.admission, {
|
|
9900
10131
|
...routing,
|
|
@@ -9922,6 +10153,7 @@ ${run.workflowVersion}`;
|
|
|
9922
10153
|
this.awaitedRemoteSteps.get(runId)?.delete(seq);
|
|
9923
10154
|
}
|
|
9924
10155
|
}
|
|
10156
|
+
if (inFlight === 0 && highest >= 0) this.turnSettledFans.get(runId)?.set(first, highest);
|
|
9925
10157
|
const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
|
|
9926
10158
|
if (batch.length > 0) await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
|
|
9927
10159
|
if (inFlight + batch.length > 0) {
|
|
@@ -9952,53 +10184,62 @@ ${run.workflowVersion}`;
|
|
|
9952
10184
|
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
9953
10185
|
return outputs;
|
|
9954
10186
|
}
|
|
9955
|
-
/**
|
|
10187
|
+
/**
|
|
10188
|
+
* Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
|
|
10189
|
+
* {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
|
|
10190
|
+
* batch, where the store and transport have them.
|
|
10191
|
+
*/
|
|
9956
10192
|
async sendStepFan(runId, positions, step, inputs, batch, group, opts, routing) {
|
|
9957
10193
|
const workerGroup = stepGroup(step, routing?.namespace);
|
|
9958
10194
|
await this.ensureRoutable(workerGroup, step.requires);
|
|
9959
10195
|
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
9960
|
-
|
|
9961
|
-
|
|
9962
|
-
|
|
9963
|
-
|
|
9964
|
-
|
|
9965
|
-
|
|
9966
|
-
|
|
9967
|
-
|
|
9968
|
-
|
|
9969
|
-
|
|
9970
|
-
|
|
9971
|
-
|
|
9972
|
-
|
|
9973
|
-
|
|
9974
|
-
|
|
9975
|
-
|
|
9976
|
-
|
|
9977
|
-
|
|
9978
|
-
|
|
10196
|
+
const checkpoints = batch.map((i) => {
|
|
10197
|
+
const seq = positions[i];
|
|
10198
|
+
return {
|
|
10199
|
+
runId,
|
|
10200
|
+
seq,
|
|
10201
|
+
name: step.name,
|
|
10202
|
+
kind: "remote",
|
|
10203
|
+
stepId: stepId(runId, seq),
|
|
10204
|
+
status: "pending",
|
|
10205
|
+
input: inputs[i],
|
|
10206
|
+
attempts: 1,
|
|
10207
|
+
workerGroup,
|
|
10208
|
+
parallelGroup: group,
|
|
10209
|
+
enqueuedAt,
|
|
10210
|
+
startedAt: enqueuedAt,
|
|
10211
|
+
finishedAt: enqueuedAt
|
|
10212
|
+
};
|
|
10213
|
+
});
|
|
10214
|
+
if (this.store.saveCheckpoints) {
|
|
10215
|
+
await this.store.saveCheckpoints(checkpoints);
|
|
10216
|
+
} else {
|
|
10217
|
+
for (let at = 0; at < checkpoints.length; at += STEP_FAN_CHUNK) {
|
|
10218
|
+
await Promise.all(checkpoints.slice(at, at + STEP_FAN_CHUNK).map((cp) => this.store.saveCheckpoint(cp)));
|
|
10219
|
+
}
|
|
9979
10220
|
}
|
|
9980
|
-
|
|
9981
|
-
|
|
9982
|
-
|
|
9983
|
-
|
|
9984
|
-
|
|
9985
|
-
|
|
9986
|
-
|
|
9987
|
-
|
|
9988
|
-
|
|
9989
|
-
|
|
9990
|
-
|
|
9991
|
-
|
|
9992
|
-
|
|
9993
|
-
|
|
9994
|
-
|
|
9995
|
-
|
|
9996
|
-
|
|
9997
|
-
|
|
9998
|
-
|
|
9999
|
-
|
|
10000
|
-
|
|
10001
|
-
})
|
|
10221
|
+
const traceparent = this.traceparent?.();
|
|
10222
|
+
const context = this.context?.();
|
|
10223
|
+
await this.pool.dispatchMany(checkpoints.map((cp, k) => ({
|
|
10224
|
+
runId,
|
|
10225
|
+
seq: cp.seq,
|
|
10226
|
+
name: step.name,
|
|
10227
|
+
stepId: cp.stepId,
|
|
10228
|
+
group: workerGroup,
|
|
10229
|
+
input: inputs[batch[k]],
|
|
10230
|
+
traceparent,
|
|
10231
|
+
context,
|
|
10232
|
+
priority: opts.admission?.priority,
|
|
10233
|
+
attempt: 1
|
|
10234
|
+
})), opts.transport);
|
|
10235
|
+
for (const cp of checkpoints) {
|
|
10236
|
+
this.emit({
|
|
10237
|
+
type: "step.started",
|
|
10238
|
+
runId,
|
|
10239
|
+
seq: cp.seq,
|
|
10240
|
+
name: step.name,
|
|
10241
|
+
kind: "remote"
|
|
10242
|
+
});
|
|
10002
10243
|
}
|
|
10003
10244
|
}
|
|
10004
10245
|
/** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
|
|
@@ -10039,6 +10280,27 @@ ${run.workflowVersion}`;
|
|
|
10039
10280
|
error: s.reason
|
|
10040
10281
|
});
|
|
10041
10282
|
}
|
|
10283
|
+
/**
|
|
10284
|
+
* The checkpoints of one run at distinct `seqs`, each as its own outcome: one
|
|
10285
|
+
* {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
|
|
10286
|
+
* call rejecting falls back to one read each, so a failing read fails only its own entry.
|
|
10287
|
+
*/
|
|
10288
|
+
async readCheckpoints(runId, seqs) {
|
|
10289
|
+
if (seqs.length > 1 && this.store.getCheckpoints) {
|
|
10290
|
+
try {
|
|
10291
|
+
const bySeq = new Map((await this.store.getCheckpoints(runId, seqs)).map((cp) => [
|
|
10292
|
+
cp.seq,
|
|
10293
|
+
cp
|
|
10294
|
+
]));
|
|
10295
|
+
return seqs.map((seq) => ({
|
|
10296
|
+
status: "fulfilled",
|
|
10297
|
+
value: bySeq.get(seq) ?? null
|
|
10298
|
+
}));
|
|
10299
|
+
} catch {
|
|
10300
|
+
}
|
|
10301
|
+
}
|
|
10302
|
+
return Promise.allSettled(seqs.map((seq) => this.store.getCheckpoint(runId, seq)));
|
|
10303
|
+
}
|
|
10042
10304
|
/** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
|
|
10043
10305
|
* dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
|
|
10044
10306
|
* {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
|
|
@@ -10080,9 +10342,10 @@ ${run.workflowVersion}`;
|
|
|
10080
10342
|
*
|
|
10081
10343
|
* Results of one run are settled in batches: the first starts a batch, and the ones that arrive
|
|
10082
10344
|
* while it is being written form the next. Under load a wide fan's results are written by one
|
|
10083
|
-
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked
|
|
10084
|
-
*
|
|
10085
|
-
* transport acknowledges nothing that is not durable yet.
|
|
10345
|
+
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked only
|
|
10346
|
+
* by a batch with no more of the fan's results queued behind it. A result resolves only once its
|
|
10347
|
+
* own write has committed, so the transport acknowledges nothing that is not durable yet. How
|
|
10348
|
+
* large a batch gets is bounded by how many results the transport hands over at once.
|
|
10086
10349
|
*/
|
|
10087
10350
|
completeRemoteResult(result) {
|
|
10088
10351
|
return new Promise((resolve, reject) => {
|
|
@@ -10110,7 +10373,7 @@ ${run.workflowVersion}`;
|
|
|
10110
10373
|
const batch = queue.splice(0, RESULT_BATCH_MAX);
|
|
10111
10374
|
let outcomes;
|
|
10112
10375
|
try {
|
|
10113
|
-
outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result));
|
|
10376
|
+
outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result), (from, to) => queue.some(({ result }) => result.seq >= from && result.seq <= to));
|
|
10114
10377
|
} catch (err) {
|
|
10115
10378
|
outcomes = batch.map(() => ({
|
|
10116
10379
|
error: err
|
|
@@ -10118,8 +10381,8 @@ ${run.workflowVersion}`;
|
|
|
10118
10381
|
}
|
|
10119
10382
|
batch.forEach((q, i) => {
|
|
10120
10383
|
const outcome = outcomes[i] ?? false;
|
|
10121
|
-
if (
|
|
10122
|
-
else q.
|
|
10384
|
+
if (isResultError(outcome)) q.reject(outcome.error);
|
|
10385
|
+
else q.resolve(outcome);
|
|
10123
10386
|
});
|
|
10124
10387
|
}
|
|
10125
10388
|
this.resultQueues.delete(runId);
|
|
@@ -10128,8 +10391,13 @@ ${run.workflowVersion}`;
|
|
|
10128
10391
|
* Settle a batch of one run's step results. At most one of them resolves `true` (resume the run):
|
|
10129
10392
|
* that resume replays every write of the batch, so a second would only replay it again. A result
|
|
10130
10393
|
* whose read or write failed resolves its error, and the transport redelivers just that one.
|
|
10394
|
+
*
|
|
10395
|
+
* `queuedIn(from, to)` says whether a result for a seq in that range is already waiting for this
|
|
10396
|
+
* run's next batch on this instance. A fan with one waiting is not asked its barrier: that result
|
|
10397
|
+
* asks it once its own batch is written (as a settle, or as a redelivery of a settled step), so
|
|
10398
|
+
* under a backlog the barrier is read once per drained queue rather than once per batch.
|
|
10131
10399
|
*/
|
|
10132
|
-
async completeRemoteResults(runId, results) {
|
|
10400
|
+
async completeRemoteResults(runId, results, queuedIn = () => false) {
|
|
10133
10401
|
const outcomes = results.map(() => false);
|
|
10134
10402
|
const seen = /* @__PURE__ */ new Set();
|
|
10135
10403
|
const live = results.flatMap((r, i) => {
|
|
@@ -10139,7 +10407,7 @@ ${run.workflowVersion}`;
|
|
|
10139
10407
|
i
|
|
10140
10408
|
];
|
|
10141
10409
|
});
|
|
10142
|
-
const reads = await
|
|
10410
|
+
const reads = await this.readCheckpoints(runId, live.map((i) => results[i].seq));
|
|
10143
10411
|
const pending = [];
|
|
10144
10412
|
const resettled = [];
|
|
10145
10413
|
reads.forEach((read, k) => {
|
|
@@ -10160,7 +10428,7 @@ ${run.workflowVersion}`;
|
|
|
10160
10428
|
});
|
|
10161
10429
|
if (pending.length === 0) {
|
|
10162
10430
|
if (resettled.length === 0) return outcomes;
|
|
10163
|
-
return this.resumeForRedelivered(runId, resettled, outcomes);
|
|
10431
|
+
return this.resumeForRedelivered(runId, resettled, outcomes, queuedIn);
|
|
10164
10432
|
}
|
|
10165
10433
|
for (const { cp } of pending) void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
|
|
10166
10434
|
const run = await this.store.getRun(runId);
|
|
@@ -10187,7 +10455,9 @@ ${run.workflowVersion}`;
|
|
|
10187
10455
|
cp,
|
|
10188
10456
|
result: results[index]
|
|
10189
10457
|
});
|
|
10190
|
-
else if (claim !== false) outcomes[index] =
|
|
10458
|
+
else if (claim !== false) outcomes[index] = {
|
|
10459
|
+
error: claim.error
|
|
10460
|
+
};
|
|
10191
10461
|
});
|
|
10192
10462
|
for (const { cp, result } of settled) {
|
|
10193
10463
|
this.noteSettledHere(`${runId}:${result.seq}`);
|
|
@@ -10205,19 +10475,21 @@ ${run.workflowVersion}`;
|
|
|
10205
10475
|
});
|
|
10206
10476
|
}
|
|
10207
10477
|
const barriers = /* @__PURE__ */ new Map();
|
|
10208
|
-
|
|
10478
|
+
const wake = new FanWakeCollector();
|
|
10479
|
+
for (const { cp } of settled) wake.note(cp);
|
|
10209
10480
|
for (const { index, cp, result } of settled) {
|
|
10210
|
-
if (!await this.stepFanStillRunning(cp, result, barriers))
|
|
10481
|
+
if (!await this.stepFanStillRunning(cp, result, barriers, queuedIn)) {
|
|
10482
|
+
wake.add(index, cp, result.status);
|
|
10483
|
+
}
|
|
10211
10484
|
}
|
|
10212
|
-
|
|
10213
|
-
return outcomes;
|
|
10485
|
+
return wake.applyTo(outcomes);
|
|
10214
10486
|
}
|
|
10215
10487
|
/**
|
|
10216
10488
|
* The outcomes for a batch of redelivered results (their steps already settled): one of them
|
|
10217
10489
|
* resumes the run if it is `suspended` and, for a step of a fan, the fan's barrier is open. A
|
|
10218
10490
|
* resume of a run that had in fact moved on replays to its current wait and re-parks.
|
|
10219
10491
|
*/
|
|
10220
|
-
async resumeForRedelivered(runId, resettled, outcomes) {
|
|
10492
|
+
async resumeForRedelivered(runId, resettled, outcomes, queuedIn) {
|
|
10221
10493
|
const run = await this.store.getRun(runId);
|
|
10222
10494
|
if (run?.status !== "suspended") return outcomes;
|
|
10223
10495
|
const barriers = /* @__PURE__ */ new Map();
|
|
@@ -10228,9 +10500,11 @@ ${run.workflowVersion}`;
|
|
|
10228
10500
|
stepId: cp.stepId ?? stepId(runId, cp.seq),
|
|
10229
10501
|
status: cp.status === "failed" ? "failed" : "completed"
|
|
10230
10502
|
};
|
|
10231
|
-
if (!await this.stepFanStillRunning(cp, asResult, barriers)) {
|
|
10232
|
-
|
|
10233
|
-
|
|
10503
|
+
if (!await this.stepFanStillRunning(cp, asResult, barriers, queuedIn)) {
|
|
10504
|
+
const wake = new FanWakeCollector();
|
|
10505
|
+
for (const settled of resettled) wake.note(settled.cp);
|
|
10506
|
+
wake.add(index, cp, asResult.status);
|
|
10507
|
+
return wake.applyTo(outcomes);
|
|
10234
10508
|
}
|
|
10235
10509
|
}
|
|
10236
10510
|
return outcomes;
|
|
@@ -10241,15 +10515,17 @@ ${run.workflowVersion}`;
|
|
|
10241
10515
|
* retry needs the parent to schedule it). Without it every result would replay the whole parent,
|
|
10242
10516
|
* which makes a fan of N steps cost O(N²) checkpoint reads. `barriers` holds the answer per fan
|
|
10243
10517
|
* for the rest of a batch. A barrier that cannot be read counts as open: a spare resume replays and
|
|
10244
|
-
* re-suspends, a lost one strands the run.
|
|
10518
|
+
* re-suspends, a lost one strands the run. A fan with a result waiting in `queuedIn` is left to
|
|
10519
|
+
* that result (see {@link completeRemoteResults}).
|
|
10245
10520
|
*/
|
|
10246
|
-
async stepFanStillRunning(cp, result, barriers) {
|
|
10521
|
+
async stepFanStillRunning(cp, result, barriers, queuedIn) {
|
|
10247
10522
|
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
10248
10523
|
if (!fan) return false;
|
|
10249
10524
|
if (result.status === "failed") return false;
|
|
10250
10525
|
let closed = barriers.get(fan.firstSeq);
|
|
10251
10526
|
if (!closed) {
|
|
10252
|
-
|
|
10527
|
+
const lastSeq = fan.firstSeq + fan.size - 1;
|
|
10528
|
+
closed = queuedIn(fan.firstSeq, lastSeq) ? Promise.resolve(true) : this.stepFanPending(result.runId, fan.firstSeq, lastSeq).then((pending) => pending > 0, () => false);
|
|
10253
10529
|
barriers.set(fan.firstSeq, closed);
|
|
10254
10530
|
}
|
|
10255
10531
|
if (await closed) return true;
|
|
@@ -11083,6 +11359,10 @@ var CodecStateStore = class {
|
|
|
11083
11359
|
saveCheckpointIfPending;
|
|
11084
11360
|
/** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
|
|
11085
11361
|
saveCheckpointsIfPending;
|
|
11362
|
+
/** Encodes each checkpoint written. Bound only when the inner store has it. */
|
|
11363
|
+
saveCheckpoints;
|
|
11364
|
+
/** Decodes each checkpoint read. Bound only when the inner store has it. */
|
|
11365
|
+
getCheckpoints;
|
|
11086
11366
|
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
|
|
11087
11367
|
countCheckpoints;
|
|
11088
11368
|
/** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
|
|
@@ -11133,6 +11413,10 @@ var CodecStateStore = class {
|
|
|
11133
11413
|
if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
|
|
11134
11414
|
const saveManyIfPending = inner.saveCheckpointsIfPending;
|
|
11135
11415
|
if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11416
|
+
const saveMany = inner.saveCheckpoints;
|
|
11417
|
+
if (saveMany) this.saveCheckpoints = (cps) => saveMany.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11418
|
+
const getMany = inner.getCheckpoints;
|
|
11419
|
+
if (getMany) this.getCheckpoints = async (runId, seqs) => (await getMany.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
|
|
11136
11420
|
const countCps = inner.countCheckpoints;
|
|
11137
11421
|
if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
|
|
11138
11422
|
const stats = inner.checkpointStats;
|
|
@@ -11769,11 +12053,30 @@ var InMemoryStateStore = class {
|
|
|
11769
12053
|
...cp
|
|
11770
12054
|
} : null;
|
|
11771
12055
|
}
|
|
12056
|
+
async getCheckpoints(runId, seqs) {
|
|
12057
|
+
return [
|
|
12058
|
+
...new Set(seqs)
|
|
12059
|
+
].sort((a, b) => a - b).flatMap((seq) => {
|
|
12060
|
+
const cp = this.checkpoints.get(this.key(runId, seq));
|
|
12061
|
+
return cp ? [
|
|
12062
|
+
{
|
|
12063
|
+
...cp
|
|
12064
|
+
}
|
|
12065
|
+
] : [];
|
|
12066
|
+
});
|
|
12067
|
+
}
|
|
11772
12068
|
async saveCheckpoint(checkpoint) {
|
|
11773
12069
|
this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
|
|
11774
12070
|
...checkpoint
|
|
11775
12071
|
});
|
|
11776
12072
|
}
|
|
12073
|
+
async saveCheckpoints(checkpoints) {
|
|
12074
|
+
for (const checkpoint of checkpoints) {
|
|
12075
|
+
this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
|
|
12076
|
+
...checkpoint
|
|
12077
|
+
});
|
|
12078
|
+
}
|
|
12079
|
+
}
|
|
11777
12080
|
async saveCheckpointIfPending(checkpoint) {
|
|
11778
12081
|
const key = this.key(checkpoint.runId, checkpoint.seq);
|
|
11779
12082
|
if (this.checkpoints.get(key)?.status !== "pending") return false;
|