@dudousxd/nestjs-durable-core 0.85.6 → 0.85.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -3561,6 +3561,7 @@ function constantTimeEquals(a, b) {
3561
3561
  __name(constantTimeEquals, "constantTimeEquals");
3562
3562
 
3563
3563
  // src/transport-pool.ts
3564
+ var DISPATCH_CHUNK = 200;
3564
3565
  var TransportPool = class {
3565
3566
  static {
3566
3567
  __name(this, "TransportPool");
@@ -3620,6 +3621,28 @@ var TransportPool = class {
3620
3621
  }
3621
3622
  throw lastErr ?? new Error("no transport accepted the dispatch");
3622
3623
  }
3624
+ /**
3625
+ * {@link dispatch} for many tasks: one {@link Transport.dispatchMany} call on the preferred (or
3626
+ * first) transport when it has one, else (or when that call rejects) each task through
3627
+ * {@link dispatch}, a bounded number at a time.
3628
+ */
3629
+ async dispatchMany(tasks, preferId) {
3630
+ if (tasks.length === 0) return;
3631
+ const first = this.ordered(preferId)[0];
3632
+ if (first?.transport.dispatchMany) {
3633
+ try {
3634
+ await first.transport.dispatchMany(tasks.map((task) => ({
3635
+ ...task,
3636
+ transport: first.id
3637
+ })));
3638
+ return;
3639
+ } catch {
3640
+ }
3641
+ }
3642
+ for (let at = 0; at < tasks.length; at += DISPATCH_CHUNK) {
3643
+ await Promise.all(tasks.slice(at, at + DISPATCH_CHUNK).map((task) => this.dispatch(task, preferId)));
3644
+ }
3645
+ }
3623
3646
  /** Worker-health for `group`, merged across every transport that can report it (a group's queue
3624
3647
  * and its workers may live on any pinned transport). Undefined when no transport implements
3625
3648
  * `groupHealth` (e.g. a pure in-process transport — nothing to introspect). */
@@ -4839,6 +4862,15 @@ var PARENT_NOTIFY_BACKOFF_MS = 250;
4839
4862
  var LEASE_RECHECK_MAX_MS = 2e3;
4840
4863
  var LEASE_RECHECK_SLACK_MS = 100;
4841
4864
  var LEASE_RECHECK_ATTEMPTS = 30;
4865
+ function wakeSeen(request, settledFans, served) {
4866
+ if (request.id !== void 0 && served.has(request.id)) return true;
4867
+ return isFanWakeRequest(request.payload) && request.payload.fans.every(([first, seq]) => (settledFans.get(first) ?? -1) >= seq);
4868
+ }
4869
+ __name(wakeSeen, "wakeSeen");
4870
+ function isFanWakeRequest(payload) {
4871
+ return typeof payload === "object" && payload !== null && Array.isArray(payload.fans);
4872
+ }
4873
+ __name(isFanWakeRequest, "isFanWakeRequest");
4842
4874
  function deferred() {
4843
4875
  let resolve;
4844
4876
  let reject;
@@ -4857,12 +4889,51 @@ __name(deferred, "deferred");
4857
4889
  var pendingStateKey = /* @__PURE__ */ __name((at) => String(new Date(at).getTime()), "pendingStateKey");
4858
4890
  var freshPendingKey = /* @__PURE__ */ __name(() => `${Date.now()}-${globalThis.crypto.randomUUID().slice(0, 8)}`, "freshPendingKey");
4859
4891
  var wakeRequestToken = /* @__PURE__ */ __name((runId) => `durable:wake-after-turn:${runId}`, "wakeRequestToken");
4892
+ var WAKE_REQUEST_SCAN = 100;
4860
4893
  var CHILD_SIGNAL_PREFIX = "signal:child:";
4861
4894
  var BUFFERED_SWEEP_PAGE = 500;
4862
4895
  var BUFFERED_SWEEP_PAGES_PER_CALL = 4;
4863
4896
  var BUFFERED_SWEEP_REST_MS = 10 * 6e4;
4864
4897
  var DELIVER_CHUNK = 50;
4865
4898
  var RESULT_BATCH_MAX = 256;
4899
+ function isResultError(outcome) {
4900
+ return typeof outcome === "object" && "error" in outcome;
4901
+ }
4902
+ __name(isResultError, "isResultError");
4903
+ var FanWakeCollector = class FanWakeCollector2 {
4904
+ static {
4905
+ __name(this, "FanWakeCollector");
4906
+ }
4907
+ carrier;
4908
+ generic = false;
4909
+ fan;
4910
+ /** The highest seq the batch settled, per fan. */
4911
+ highest = /* @__PURE__ */ new Map();
4912
+ /** A checkpoint the batch settled. */
4913
+ note(cp) {
4914
+ const fan = parseStepFanGroup(cp.parallelGroup);
4915
+ if (fan) this.highest.set(fan.firstSeq, Math.max(this.highest.get(fan.firstSeq) ?? cp.seq, cp.seq));
4916
+ }
4917
+ /** A settled checkpoint that resumes the run. */
4918
+ add(index, cp, status) {
4919
+ this.carrier = index;
4920
+ const fan = parseStepFanGroup(cp.parallelGroup);
4921
+ if (!fan || status === "failed" || this.fan !== void 0 && this.fan !== fan.firstSeq) {
4922
+ this.generic = true;
4923
+ return;
4924
+ }
4925
+ this.fan = fan.firstSeq;
4926
+ }
4927
+ applyTo(outcomes) {
4928
+ if (this.carrier === void 0) return outcomes;
4929
+ const seq = this.fan === void 0 ? void 0 : this.highest.get(this.fan);
4930
+ outcomes[this.carrier] = this.generic || this.fan === void 0 || seq === void 0 ? true : {
4931
+ first: this.fan,
4932
+ seq
4933
+ };
4934
+ return outcomes;
4935
+ }
4936
+ };
4866
4937
  function shuffle(items) {
4867
4938
  for (let i = items.length - 1; i > 0; i -= 1) {
4868
4939
  const j = Math.floor(Math.random() * (i + 1));
@@ -5051,6 +5122,23 @@ var WorkflowEngine = class {
5051
5122
  /** Re-checks scheduled for runs whose wake found another instance's lease (see {@link execute}). */
5052
5123
  leaseRetries = /* @__PURE__ */ new Map();
5053
5124
  /**
5125
+ * Runs whose scheduled re-check resumes them whatever happened meanwhile. Without an entry the
5126
+ * re-check only stood for fan wakes kept as requests, and resumes the run only if a request is
5127
+ * still there: the turn that held the lease takes them once it ends (see {@link retryDroppedWake}).
5128
+ */
5129
+ unconditionalRechecks = /* @__PURE__ */ new Set();
5130
+ /** Runs a generic wake request was written for in the current re-check cycle (see {@link execute}). */
5131
+ genericWakeRequested = /* @__PURE__ */ new Set();
5132
+ /** Why each run is being woken on this instance, until the turn that answers it starts. */
5133
+ wakeReasons = /* @__PURE__ */ new Map();
5134
+ /**
5135
+ * The step fans each executing turn on this instance read with nothing in flight: first seq -> the
5136
+ * highest seq read. Written by {@link callRemoteAll}, read once the turn ends.
5137
+ */
5138
+ turnSettledFans = /* @__PURE__ */ new Map();
5139
+ /** The wake requests each executing turn on this instance serves (see {@link noteServedWakes}). */
5140
+ turnServedWakes = /* @__PURE__ */ new Map();
5141
+ /**
5054
5142
  * Every run turn and child start this instance takes on itself, under one budget of
5055
5143
  * {@link LOCAL_RUN_CONCURRENCY}: the runs it dispatches to itself, a fan's deferred child starts,
5056
5144
  * enqueued pickups and the runs it takes off the transport's run queue.
@@ -5174,8 +5262,9 @@ var WorkflowEngine = class {
5174
5262
  }
5175
5263
  return;
5176
5264
  }
5177
- if (!await this.completeRemoteResult(result)) return;
5178
- await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId).then(() => void 0, (err) => {
5265
+ const wake = await this.completeRemoteResult(result);
5266
+ if (!wake) return;
5267
+ await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId, wake === true ? void 0 : wake).then(() => void 0, (err) => {
5179
5268
  if (err instanceof NamespaceMismatch) return;
5180
5269
  throw err;
5181
5270
  }));
@@ -5392,7 +5481,10 @@ var WorkflowEngine = class {
5392
5481
  const resumed = await this.resumeLeased([
5393
5482
  run
5394
5483
  ], nowMs, (locked) => this.reconcileSignalWaiter(locked, waiters.get(locked.id)));
5395
- if (resumed.length === 0) this.recheckAfterLease(runId, run.lockedUntil);
5484
+ if (resumed.length === 0) {
5485
+ this.unconditionalRechecks.add(runId);
5486
+ this.recheckAfterLease(runId, run.lockedUntil);
5487
+ }
5396
5488
  return;
5397
5489
  }
5398
5490
  await this.resumeImplicit(runId).catch((err) => {
@@ -5959,15 +6051,27 @@ var WorkflowEngine = class {
5959
6051
  * terminal too.
5960
6052
  */
5961
6053
  resume(runId) {
6054
+ this.noteWakeReason(runId, void 0);
5962
6055
  return this.resumeRun(runId, true);
5963
6056
  }
6057
+ /** Record why `runId` is being woken: a step fan's barrier opening, or (`undefined`) anything else. */
6058
+ noteWakeReason(runId, fan) {
6059
+ const reason = this.wakeReasons.get(runId) ?? {
6060
+ generic: false,
6061
+ fans: /* @__PURE__ */ new Map()
6062
+ };
6063
+ if (fan) reason.fans.set(fan.first, Math.max(reason.fans.get(fan.first) ?? fan.seq, fan.seq));
6064
+ else reason.generic = true;
6065
+ this.wakeReasons.set(runId, reason);
6066
+ }
5964
6067
  /**
5965
6068
  * {@link resume} for every IMPLICIT wake-up — a delivered signal/child completion, a late step
5966
6069
  * result, a due timer, a redelivered dispatch, a queue-slot wake. A `failed` run is terminal to
5967
6070
  * these: only an explicit retry ({@link requeue} / {@link resume}) may re-drive it. Otherwise a
5968
6071
  * stale waiter or a late result left behind by a failed run silently re-executes its body.
5969
6072
  */
5970
- resumeImplicit(runId) {
6073
+ resumeImplicit(runId, fan) {
6074
+ if (fan !== "coalesced") this.noteWakeReason(runId, fan);
5971
6075
  const turn = this.wakeTurns.get(runId);
5972
6076
  if (turn) {
5973
6077
  turn.again ??= deferred();
@@ -5992,7 +6096,7 @@ var WorkflowEngine = class {
5992
6096
  });
5993
6097
  return;
5994
6098
  }
5995
- const next = this.resumeImplicit(runId);
6099
+ const next = this.resumeImplicit(runId, "coalesced");
5996
6100
  next.then(again.resolve, again.reject);
5997
6101
  this.inBackground(next);
5998
6102
  }, "finish");
@@ -8057,9 +8161,26 @@ var WorkflowEngine = class {
8057
8161
  async execute(run, fn) {
8058
8162
  const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
8059
8163
  const lockNow = this.clock();
8164
+ const reason = this.wakeReasons.get(run.id) ?? {
8165
+ generic: true,
8166
+ fans: /* @__PURE__ */ new Map()
8167
+ };
8168
+ this.wakeReasons.delete(run.id);
8060
8169
  if (run.lockedBy !== this.instanceId || (run.lockedUntil ?? 0) <= lockNow) {
8061
8170
  if (!await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow)) {
8062
- if (!this.leaseRetries.has(run.id)) this.inBackground(this.requestWakeAfterTurn(run.id));
8171
+ if (reason.generic) {
8172
+ if (!this.genericWakeRequested.has(run.id)) {
8173
+ this.genericWakeRequested.add(run.id);
8174
+ this.inBackground(this.requestWakeAfterTurn(run.id, null));
8175
+ }
8176
+ } else {
8177
+ const fans = {
8178
+ fans: [
8179
+ ...reason.fans
8180
+ ]
8181
+ };
8182
+ this.inBackground(this.requestWakeAfterTurn(run.id, fans));
8183
+ }
8063
8184
  this.recheckAfterLease(run.id, run.lockedUntil);
8064
8185
  return {
8065
8186
  runId: run.id,
@@ -8067,56 +8188,142 @@ var WorkflowEngine = class {
8067
8188
  };
8068
8189
  }
8069
8190
  }
8191
+ const settledFans = /* @__PURE__ */ new Map();
8192
+ this.turnSettledFans.set(run.id, settledFans);
8070
8193
  const held = this.holdExecutionLease(run.id);
8071
8194
  try {
8195
+ const served = /* @__PURE__ */ new Set();
8196
+ this.turnServedWakes.set(run.id, served);
8197
+ if (registered?.remote) await this.noteServedWakes(run.id);
8072
8198
  const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
8073
8199
  if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
8074
8200
  void this.singletons.wakeNext(run).catch(() => void 0);
8075
8201
  }
8076
- if (result.status === "suspended") this.inBackground(this.honorWakeRequests(run.id));
8077
- else if (TERMINAL_RUN_STATUSES.includes(result.status)) {
8078
- this.inBackground(this.takeWakeRequests(run.id));
8202
+ if (result.status === "suspended") {
8203
+ this.inBackground(this.honorWakeRequests(run.id, settledFans, served));
8204
+ } else if (TERMINAL_RUN_STATUSES.includes(result.status)) {
8205
+ this.inBackground(this.listWakeRequests(run.id, true).then((kept) => this.dropWakeRequests(kept)));
8079
8206
  }
8080
8207
  return result;
8081
8208
  } finally {
8082
8209
  held.stop();
8210
+ if (this.turnSettledFans.get(run.id) === settledFans) this.turnSettledFans.delete(run.id);
8211
+ this.turnServedWakes.delete(run.id);
8083
8212
  }
8084
8213
  }
8085
8214
  /**
8086
8215
  * Keep a wake that found `runId`'s lease held by a turn executing elsewhere, durably, so that turn
8087
8216
  * runs again once it ends instead of suspending on a snapshot that predates what the wake brought
8088
- * (a signal, a child's completion). The request is a buffered signal on a token no workflow waits
8089
- * on; the holder takes it after it releases the lease ({@link honorWakeRequests}).
8217
+ * (a signal, a child's completion). The request is a buffered event under a name no workflow waits
8218
+ * on. It stays until a turn that started after it has settled: the holder looks at it after it
8219
+ * releases the lease ({@link honorWakeRequests}), and the turn that serves it drops it.
8090
8220
  *
8091
8221
  * Written first and looked at second, against a holder that releases first and looks second:
8092
8222
  * whichever look runs last sees the other side's write. So when the run is no longer executing by
8093
- * the time this re-reads it, the holder may have looked already, and this side takes the request
8094
- * and wakes the run itself once the lease is free.
8223
+ * the time this re-reads it, the holder may have looked already, and this side wakes the run
8224
+ * itself once the lease is free ({@link retryDroppedWake}). While the holder still runs, the run is
8225
+ * made due when the holder's lease runs out, so a holder that goes down before it settles leaves
8226
+ * the run to any instance's timer poll; a holder that settles sets the run's timer itself.
8095
8227
  */
8096
- async requestWakeAfterTurn(runId) {
8097
- await this.store.bufferSignal(wakeRequestToken(runId), null);
8228
+ async requestWakeAfterTurn(runId, request) {
8229
+ const id = globalThis.crypto.randomUUID();
8230
+ await this.store.bufferEvent({
8231
+ name: wakeRequestToken(runId),
8232
+ payload: request,
8233
+ id,
8234
+ publishedAt: this.clock()
8235
+ });
8098
8236
  const run = await this.store.getRun(runId);
8099
- if (run?.status === "running") return;
8100
- if (!run) {
8101
- await this.takeWakeRequests(runId);
8237
+ if (!run || TERMINAL_RUN_STATUSES.includes(run.status)) {
8238
+ await this.store.removeBufferedEvent(id);
8239
+ return;
8240
+ }
8241
+ if (run.status === "running") return;
8242
+ if (run.lockedBy !== void 0 && run.lockedBy !== this.instanceId) {
8243
+ await this.makeDueBy(runId, (run.lockedUntil ?? this.clock()) + LEASE_RECHECK_SLACK_MS);
8102
8244
  return;
8103
8245
  }
8104
- if (!await this.takeWakeRequests(runId)) return;
8105
- if (TERMINAL_RUN_STATUSES.includes(run.status)) return;
8106
8246
  this.recheckAfterLease(runId, run.lockedUntil);
8107
8247
  }
8108
- /** After a turn that suspended released its lease: run again if a wake was kept for it. */
8109
- async honorWakeRequests(runId) {
8110
- if (!await this.takeWakeRequests(runId)) return;
8248
+ /**
8249
+ * After a turn that suspended released its lease: drop the wakes it served, and run again if a
8250
+ * wake was kept for it meanwhile, unless every such wake was a fan wake this turn already saw: it
8251
+ * read that fan with nothing in flight, at least as far as the batch that opened its barrier.
8252
+ * The run is made due before it is woken, so if this instance goes down before that turn runs,
8253
+ * the timer poll of another instance runs it once this instance's hold on it would have lapsed.
8254
+ */
8255
+ async honorWakeRequests(runId, settledFans, served) {
8256
+ const kept = await this.listWakeRequests(runId, true);
8257
+ const seen = /* @__PURE__ */ __name((request) => wakeSeen(request, settledFans, served), "seen");
8258
+ const needed = kept.filter((request) => !seen(request));
8259
+ await this.dropWakeRequests(kept.filter(seen));
8260
+ if (needed.length === 0) return;
8261
+ await this.makeDueBy(runId, this.clock() + this.leaseMs);
8111
8262
  await this.wake(runId).catch((err) => {
8112
8263
  if (!(err instanceof NamespaceMismatch)) throw err;
8113
8264
  });
8114
8265
  }
8115
- /** Take every wake request kept for `runId`; true when there was one. */
8116
- async takeWakeRequests(runId) {
8117
- let taken = false;
8118
- while (await this.store.takeBufferedSignal(wakeRequestToken(runId))) taken = true;
8119
- return taken;
8266
+ /**
8267
+ * The wake requests kept for `runId`. With `legacy`, also those an older engine kept as buffered
8268
+ * signals, which are taken as they are read.
8269
+ */
8270
+ async listWakeRequests(runId, legacy = false) {
8271
+ const token = wakeRequestToken(runId);
8272
+ const kept = (await this.store.listBufferedEvents(token, WAKE_REQUEST_SCAN)).map((request) => ({
8273
+ id: request.id,
8274
+ payload: request.payload
8275
+ }));
8276
+ if (!legacy) return kept;
8277
+ for (; ; ) {
8278
+ const signal = await this.store.takeBufferedSignal(token);
8279
+ if (!signal) return kept;
8280
+ kept.push({
8281
+ payload: signal.payload
8282
+ });
8283
+ }
8284
+ }
8285
+ /** Delete `requests` (the ones kept as buffered events). */
8286
+ async dropWakeRequests(requests) {
8287
+ await Promise.all(requests.flatMap((request) => request.id === void 0 ? [] : [
8288
+ this.store.removeBufferedEvent(request.id)
8289
+ ]));
8290
+ }
8291
+ /**
8292
+ * Record, for the turn executing `runId` here, the wakes kept for the run so far. Called before the
8293
+ * turn reads the run's state, so the turn serves them: they are dropped once it has settled, and
8294
+ * until then stay for whoever runs the run next.
8295
+ */
8296
+ async noteServedWakes(runId) {
8297
+ const served = this.turnServedWakes.get(runId);
8298
+ if (!served) return;
8299
+ for (const request of await this.store.listBufferedEvents(wakeRequestToken(runId), WAKE_REQUEST_SCAN)) {
8300
+ served.add(request.id);
8301
+ }
8302
+ }
8303
+ /**
8304
+ * Right after a turn of `runId` wrote its suspension, while it still holds the lease: if a wake
8305
+ * was kept for the run during the turn that the turn did not see, keep the run due when the lease
8306
+ * runs out. The suspension overwrote the timer the instance keeping the wake set, and this instance
8307
+ * may go down before it looks at the wake once it has released the lease.
8308
+ */
8309
+ async keepDueForKeptWakes(runId) {
8310
+ const served = this.turnServedWakes.get(runId);
8311
+ if (!served) return;
8312
+ const settledFans = this.turnSettledFans.get(runId) ?? /* @__PURE__ */ new Map();
8313
+ const kept = await this.listWakeRequests(runId);
8314
+ if (kept.every((request) => wakeSeen(request, settledFans, served))) return;
8315
+ await this.makeDueBy(runId, this.clock() + this.leaseMs);
8316
+ }
8317
+ /** Bring a suspended run's timer forward to `at` (never later than the one it has). */
8318
+ async makeDueBy(runId, at) {
8319
+ const run = await this.store.getRun(runId);
8320
+ if (run?.status !== "suspended") return;
8321
+ if (run.wakeAt !== void 0 && run.wakeAt <= at) return;
8322
+ await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, runId, [
8323
+ "suspended"
8324
+ ], {
8325
+ wakeAt: at
8326
+ });
8120
8327
  }
8121
8328
  /**
8122
8329
  * A wake that found the run leased by another instance is dropped, and normally that is fine: the
@@ -8135,6 +8342,7 @@ var WorkflowEngine = class {
8135
8342
  const remaining = Math.max(0, (lockedUntil ?? this.clock()) - this.clock());
8136
8343
  const timer = setTimeout(() => {
8137
8344
  this.leaseRetries.delete(runId);
8345
+ this.genericWakeRequested.delete(runId);
8138
8346
  this.inBackground(this.retryDroppedWake(runId, attempt));
8139
8347
  }, Math.min(remaining, LEASE_RECHECK_MAX_MS) + LEASE_RECHECK_SLACK_MS);
8140
8348
  timer.unref?.();
@@ -8143,12 +8351,15 @@ var WorkflowEngine = class {
8143
8351
  async retryDroppedWake(runId, attempt) {
8144
8352
  if (this.draining) return;
8145
8353
  const run = await this.store.getRun(runId);
8146
- if (run?.status !== "suspended") return;
8147
8354
  const nowMs = this.clock();
8148
- if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) {
8149
- if (attempt < LEASE_RECHECK_ATTEMPTS) this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
8355
+ if (run?.status === "suspended" && run.lockedUntil !== void 0 && run.lockedUntil > nowMs && attempt < LEASE_RECHECK_ATTEMPTS) {
8356
+ this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
8150
8357
  return;
8151
8358
  }
8359
+ const unconditional = this.unconditionalRechecks.delete(runId);
8360
+ if (run?.status !== "suspended") return;
8361
+ if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) return;
8362
+ if (!unconditional && (await this.listWakeRequests(runId)).length === 0) return;
8152
8363
  if (this.queuesRunsOverTransport) {
8153
8364
  await this.wake(runId);
8154
8365
  return;
@@ -8487,6 +8698,7 @@ ${run.workflowVersion}`;
8487
8698
  updatedAt
8488
8699
  });
8489
8700
  if (!applied) return this.echoCurrentStatus(run.id);
8701
+ await this.keepDueForKeptWakes(run.id);
8490
8702
  this.emit({
8491
8703
  type: "run.suspended",
8492
8704
  runId: run.id,
@@ -9180,6 +9392,7 @@ ${run.workflowVersion}`;
9180
9392
  namespace: run.namespace
9181
9393
  });
9182
9394
  }
9395
+ await this.noteServedWakes(run.id);
9183
9396
  if (run.status !== "cancelling" && registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
9184
9397
  return this.singletonWaitResult(run);
9185
9398
  }
@@ -9853,6 +10066,14 @@ ${run.workflowVersion}`;
9853
10066
  throw new WorkflowSuspended();
9854
10067
  }
9855
10068
  /**
10069
+ * Whether a `pending` step a turn just read only waits for its result: {@link callRemote} would
10070
+ * suspend on it without writing (no redispatch deadline to stamp, the one stamped not yet due).
10071
+ */
10072
+ stepIsWaiting(cp) {
10073
+ if (this.remoteRedispatchMs == null) return true;
10074
+ return cp.wakeAt != null && this.clock() < cp.wakeAt;
10075
+ }
10076
+ /**
9856
10077
  * `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
9857
10078
  * reserves the whole position block up front, so replay re-derives the same seqs.
9858
10079
  *
@@ -9882,6 +10103,7 @@ ${run.workflowVersion}`;
9882
10103
  const failures = [];
9883
10104
  const unsent = [];
9884
10105
  let inFlight = 0;
10106
+ let highest = -1;
9885
10107
  let wakeAt;
9886
10108
  for (let i = 0; i < inputs.length; i += 1) {
9887
10109
  const seq = positions[i];
@@ -9891,10 +10113,19 @@ ${run.workflowVersion}`;
9891
10113
  continue;
9892
10114
  }
9893
10115
  if (cp.name !== step.name) throw new NonDeterminismError(runId, seq, step.name, cp.name);
10116
+ if (seq > highest) highest = seq;
9894
10117
  if (cp.status === "completed") {
9895
10118
  outputs[i] = cp.output;
9896
10119
  continue;
9897
10120
  }
10121
+ if (cp.status === "pending" && this.stepIsWaiting(cp)) {
10122
+ inFlight += 1;
10123
+ if (cp.wakeAt != null) {
10124
+ const at = this.stepLeaseWakeAt(cp.wakeAt);
10125
+ wakeAt = Math.min(wakeAt ?? at, at);
10126
+ }
10127
+ continue;
10128
+ }
9898
10129
  try {
9899
10130
  outputs[i] = await this.callRemote(runId, seq, step, inputs[i], void 0, opts.transport, void 0, opts.admission, {
9900
10131
  ...routing,
@@ -9922,6 +10153,7 @@ ${run.workflowVersion}`;
9922
10153
  this.awaitedRemoteSteps.get(runId)?.delete(seq);
9923
10154
  }
9924
10155
  }
10156
+ if (inFlight === 0 && highest >= 0) this.turnSettledFans.get(runId)?.set(first, highest);
9925
10157
  const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
9926
10158
  if (batch.length > 0) await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
9927
10159
  if (inFlight + batch.length > 0) {
@@ -9952,53 +10184,62 @@ ${run.workflowVersion}`;
9952
10184
  if (failures.length > 0) throw new GatherError(failures, inputs.length);
9953
10185
  return outputs;
9954
10186
  }
9955
- /** Write the `pending` checkpoints of a step fan's new steps, then dispatch them. */
10187
+ /**
10188
+ * Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
10189
+ * {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
10190
+ * batch, where the store and transport have them.
10191
+ */
9956
10192
  async sendStepFan(runId, positions, step, inputs, batch, group, opts, routing) {
9957
10193
  const workerGroup = stepGroup(step, routing?.namespace);
9958
10194
  await this.ensureRoutable(workerGroup, step.requires);
9959
10195
  const enqueuedAt = /* @__PURE__ */ new Date();
9960
- for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
9961
- await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map((i) => {
9962
- const seq = positions[i];
9963
- return this.store.saveCheckpoint({
9964
- runId,
9965
- seq,
9966
- name: step.name,
9967
- kind: "remote",
9968
- stepId: stepId(runId, seq),
9969
- status: "pending",
9970
- input: inputs[i],
9971
- attempts: 1,
9972
- workerGroup,
9973
- parallelGroup: group,
9974
- enqueuedAt,
9975
- startedAt: enqueuedAt,
9976
- finishedAt: enqueuedAt
9977
- });
9978
- }));
10196
+ const checkpoints = batch.map((i) => {
10197
+ const seq = positions[i];
10198
+ return {
10199
+ runId,
10200
+ seq,
10201
+ name: step.name,
10202
+ kind: "remote",
10203
+ stepId: stepId(runId, seq),
10204
+ status: "pending",
10205
+ input: inputs[i],
10206
+ attempts: 1,
10207
+ workerGroup,
10208
+ parallelGroup: group,
10209
+ enqueuedAt,
10210
+ startedAt: enqueuedAt,
10211
+ finishedAt: enqueuedAt
10212
+ };
10213
+ });
10214
+ if (this.store.saveCheckpoints) {
10215
+ await this.store.saveCheckpoints(checkpoints);
10216
+ } else {
10217
+ for (let at = 0; at < checkpoints.length; at += STEP_FAN_CHUNK) {
10218
+ await Promise.all(checkpoints.slice(at, at + STEP_FAN_CHUNK).map((cp) => this.store.saveCheckpoint(cp)));
10219
+ }
9979
10220
  }
9980
- for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
9981
- await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map(async (i) => {
9982
- const seq = positions[i];
9983
- await this.dispatchRemoteTask({
9984
- runId,
9985
- seq,
9986
- name: step.name,
9987
- stepId: stepId(runId, seq),
9988
- group: workerGroup,
9989
- input: inputs[i],
9990
- priority: opts.admission?.priority,
9991
- attempt: 1,
9992
- transport: opts.transport
9993
- });
9994
- this.emit({
9995
- type: "step.started",
9996
- runId,
9997
- seq,
9998
- name: step.name,
9999
- kind: "remote"
10000
- });
10001
- }));
10221
+ const traceparent = this.traceparent?.();
10222
+ const context = this.context?.();
10223
+ await this.pool.dispatchMany(checkpoints.map((cp, k) => ({
10224
+ runId,
10225
+ seq: cp.seq,
10226
+ name: step.name,
10227
+ stepId: cp.stepId,
10228
+ group: workerGroup,
10229
+ input: inputs[batch[k]],
10230
+ traceparent,
10231
+ context,
10232
+ priority: opts.admission?.priority,
10233
+ attempt: 1
10234
+ })), opts.transport);
10235
+ for (const cp of checkpoints) {
10236
+ this.emit({
10237
+ type: "step.started",
10238
+ runId,
10239
+ seq: cp.seq,
10240
+ name: step.name,
10241
+ kind: "remote"
10242
+ });
10002
10243
  }
10003
10244
  }
10004
10245
  /** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
@@ -10039,6 +10280,27 @@ ${run.workflowVersion}`;
10039
10280
  error: s.reason
10040
10281
  });
10041
10282
  }
10283
+ /**
10284
+ * The checkpoints of one run at distinct `seqs`, each as its own outcome: one
10285
+ * {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
10286
+ * call rejecting falls back to one read each, so a failing read fails only its own entry.
10287
+ */
10288
+ async readCheckpoints(runId, seqs) {
10289
+ if (seqs.length > 1 && this.store.getCheckpoints) {
10290
+ try {
10291
+ const bySeq = new Map((await this.store.getCheckpoints(runId, seqs)).map((cp) => [
10292
+ cp.seq,
10293
+ cp
10294
+ ]));
10295
+ return seqs.map((seq) => ({
10296
+ status: "fulfilled",
10297
+ value: bySeq.get(seq) ?? null
10298
+ }));
10299
+ } catch {
10300
+ }
10301
+ }
10302
+ return Promise.allSettled(seqs.map((seq) => this.store.getCheckpoint(runId, seq)));
10303
+ }
10042
10304
  /** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
10043
10305
  * dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
10044
10306
  * {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
@@ -10080,9 +10342,10 @@ ${run.workflowVersion}`;
10080
10342
  *
10081
10343
  * Results of one run are settled in batches: the first starts a batch, and the ones that arrive
10082
10344
  * while it is being written form the next. Under load a wide fan's results are written by one
10083
- * {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked once
10084
- * per batch, not once per result. A result resolves only once its own write has committed, so the
10085
- * transport acknowledges nothing that is not durable yet.
10345
+ * {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked only
10346
+ * by a batch with no more of the fan's results queued behind it. A result resolves only once its
10347
+ * own write has committed, so the transport acknowledges nothing that is not durable yet. How
10348
+ * large a batch gets is bounded by how many results the transport hands over at once.
10086
10349
  */
10087
10350
  completeRemoteResult(result) {
10088
10351
  return new Promise((resolve, reject) => {
@@ -10110,7 +10373,7 @@ ${run.workflowVersion}`;
10110
10373
  const batch = queue.splice(0, RESULT_BATCH_MAX);
10111
10374
  let outcomes;
10112
10375
  try {
10113
- outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result));
10376
+ outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result), (from, to) => queue.some(({ result }) => result.seq >= from && result.seq <= to));
10114
10377
  } catch (err) {
10115
10378
  outcomes = batch.map(() => ({
10116
10379
  error: err
@@ -10118,8 +10381,8 @@ ${run.workflowVersion}`;
10118
10381
  }
10119
10382
  batch.forEach((q, i) => {
10120
10383
  const outcome = outcomes[i] ?? false;
10121
- if (typeof outcome === "boolean") q.resolve(outcome);
10122
- else q.reject(outcome.error);
10384
+ if (isResultError(outcome)) q.reject(outcome.error);
10385
+ else q.resolve(outcome);
10123
10386
  });
10124
10387
  }
10125
10388
  this.resultQueues.delete(runId);
@@ -10128,8 +10391,13 @@ ${run.workflowVersion}`;
10128
10391
  * Settle a batch of one run's step results. At most one of them resolves `true` (resume the run):
10129
10392
  * that resume replays every write of the batch, so a second would only replay it again. A result
10130
10393
  * whose read or write failed resolves its error, and the transport redelivers just that one.
10394
+ *
10395
+ * `queuedIn(from, to)` says whether a result for a seq in that range is already waiting for this
10396
+ * run's next batch on this instance. A fan with one waiting is not asked its barrier: that result
10397
+ * asks it once its own batch is written (as a settle, or as a redelivery of a settled step), so
10398
+ * under a backlog the barrier is read once per drained queue rather than once per batch.
10131
10399
  */
10132
- async completeRemoteResults(runId, results) {
10400
+ async completeRemoteResults(runId, results, queuedIn = () => false) {
10133
10401
  const outcomes = results.map(() => false);
10134
10402
  const seen = /* @__PURE__ */ new Set();
10135
10403
  const live = results.flatMap((r, i) => {
@@ -10139,7 +10407,7 @@ ${run.workflowVersion}`;
10139
10407
  i
10140
10408
  ];
10141
10409
  });
10142
- const reads = await Promise.allSettled(live.map((i) => this.store.getCheckpoint(runId, results[i].seq)));
10410
+ const reads = await this.readCheckpoints(runId, live.map((i) => results[i].seq));
10143
10411
  const pending = [];
10144
10412
  const resettled = [];
10145
10413
  reads.forEach((read, k) => {
@@ -10160,7 +10428,7 @@ ${run.workflowVersion}`;
10160
10428
  });
10161
10429
  if (pending.length === 0) {
10162
10430
  if (resettled.length === 0) return outcomes;
10163
- return this.resumeForRedelivered(runId, resettled, outcomes);
10431
+ return this.resumeForRedelivered(runId, resettled, outcomes, queuedIn);
10164
10432
  }
10165
10433
  for (const { cp } of pending) void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
10166
10434
  const run = await this.store.getRun(runId);
@@ -10187,7 +10455,9 @@ ${run.workflowVersion}`;
10187
10455
  cp,
10188
10456
  result: results[index]
10189
10457
  });
10190
- else if (claim !== false) outcomes[index] = claim;
10458
+ else if (claim !== false) outcomes[index] = {
10459
+ error: claim.error
10460
+ };
10191
10461
  });
10192
10462
  for (const { cp, result } of settled) {
10193
10463
  this.noteSettledHere(`${runId}:${result.seq}`);
@@ -10205,19 +10475,21 @@ ${run.workflowVersion}`;
10205
10475
  });
10206
10476
  }
10207
10477
  const barriers = /* @__PURE__ */ new Map();
10208
- let carrier;
10478
+ const wake = new FanWakeCollector();
10479
+ for (const { cp } of settled) wake.note(cp);
10209
10480
  for (const { index, cp, result } of settled) {
10210
- if (!await this.stepFanStillRunning(cp, result, barriers)) carrier = index;
10481
+ if (!await this.stepFanStillRunning(cp, result, barriers, queuedIn)) {
10482
+ wake.add(index, cp, result.status);
10483
+ }
10211
10484
  }
10212
- if (carrier !== void 0) outcomes[carrier] = true;
10213
- return outcomes;
10485
+ return wake.applyTo(outcomes);
10214
10486
  }
10215
10487
  /**
10216
10488
  * The outcomes for a batch of redelivered results (their steps already settled): one of them
10217
10489
  * resumes the run if it is `suspended` and, for a step of a fan, the fan's barrier is open. A
10218
10490
  * resume of a run that had in fact moved on replays to its current wait and re-parks.
10219
10491
  */
10220
- async resumeForRedelivered(runId, resettled, outcomes) {
10492
+ async resumeForRedelivered(runId, resettled, outcomes, queuedIn) {
10221
10493
  const run = await this.store.getRun(runId);
10222
10494
  if (run?.status !== "suspended") return outcomes;
10223
10495
  const barriers = /* @__PURE__ */ new Map();
@@ -10228,9 +10500,11 @@ ${run.workflowVersion}`;
10228
10500
  stepId: cp.stepId ?? stepId(runId, cp.seq),
10229
10501
  status: cp.status === "failed" ? "failed" : "completed"
10230
10502
  };
10231
- if (!await this.stepFanStillRunning(cp, asResult, barriers)) {
10232
- outcomes[index] = true;
10233
- break;
10503
+ if (!await this.stepFanStillRunning(cp, asResult, barriers, queuedIn)) {
10504
+ const wake = new FanWakeCollector();
10505
+ for (const settled of resettled) wake.note(settled.cp);
10506
+ wake.add(index, cp, asResult.status);
10507
+ return wake.applyTo(outcomes);
10234
10508
  }
10235
10509
  }
10236
10510
  return outcomes;
@@ -10241,15 +10515,17 @@ ${run.workflowVersion}`;
10241
10515
  * retry needs the parent to schedule it). Without it every result would replay the whole parent,
10242
10516
  * which makes a fan of N steps cost O(N²) checkpoint reads. `barriers` holds the answer per fan
10243
10517
  * for the rest of a batch. A barrier that cannot be read counts as open: a spare resume replays and
10244
- * re-suspends, a lost one strands the run.
10518
+ * re-suspends, a lost one strands the run. A fan with a result waiting in `queuedIn` is left to
10519
+ * that result (see {@link completeRemoteResults}).
10245
10520
  */
10246
- async stepFanStillRunning(cp, result, barriers) {
10521
+ async stepFanStillRunning(cp, result, barriers, queuedIn) {
10247
10522
  const fan = parseStepFanGroup(cp.parallelGroup);
10248
10523
  if (!fan) return false;
10249
10524
  if (result.status === "failed") return false;
10250
10525
  let closed = barriers.get(fan.firstSeq);
10251
10526
  if (!closed) {
10252
- closed = this.stepFanPending(result.runId, fan.firstSeq, fan.firstSeq + fan.size - 1).then((pending) => pending > 0, () => false);
10527
+ const lastSeq = fan.firstSeq + fan.size - 1;
10528
+ closed = queuedIn(fan.firstSeq, lastSeq) ? Promise.resolve(true) : this.stepFanPending(result.runId, fan.firstSeq, lastSeq).then((pending) => pending > 0, () => false);
10253
10529
  barriers.set(fan.firstSeq, closed);
10254
10530
  }
10255
10531
  if (await closed) return true;
@@ -11083,6 +11359,10 @@ var CodecStateStore = class {
11083
11359
  saveCheckpointIfPending;
11084
11360
  /** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
11085
11361
  saveCheckpointsIfPending;
11362
+ /** Encodes each checkpoint written. Bound only when the inner store has it. */
11363
+ saveCheckpoints;
11364
+ /** Decodes each checkpoint read. Bound only when the inner store has it. */
11365
+ getCheckpoints;
11086
11366
  /** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
11087
11367
  countCheckpoints;
11088
11368
  /** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
@@ -11133,6 +11413,10 @@ var CodecStateStore = class {
11133
11413
  if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
11134
11414
  const saveManyIfPending = inner.saveCheckpointsIfPending;
11135
11415
  if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
11416
+ const saveMany = inner.saveCheckpoints;
11417
+ if (saveMany) this.saveCheckpoints = (cps) => saveMany.call(inner, cps.map((cp) => this.encCp(cp)));
11418
+ const getMany = inner.getCheckpoints;
11419
+ if (getMany) this.getCheckpoints = async (runId, seqs) => (await getMany.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
11136
11420
  const countCps = inner.countCheckpoints;
11137
11421
  if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
11138
11422
  const stats = inner.checkpointStats;
@@ -11769,11 +12053,30 @@ var InMemoryStateStore = class {
11769
12053
  ...cp
11770
12054
  } : null;
11771
12055
  }
12056
+ async getCheckpoints(runId, seqs) {
12057
+ return [
12058
+ ...new Set(seqs)
12059
+ ].sort((a, b) => a - b).flatMap((seq) => {
12060
+ const cp = this.checkpoints.get(this.key(runId, seq));
12061
+ return cp ? [
12062
+ {
12063
+ ...cp
12064
+ }
12065
+ ] : [];
12066
+ });
12067
+ }
11772
12068
  async saveCheckpoint(checkpoint) {
11773
12069
  this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
11774
12070
  ...checkpoint
11775
12071
  });
11776
12072
  }
12073
+ async saveCheckpoints(checkpoints) {
12074
+ for (const checkpoint of checkpoints) {
12075
+ this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
12076
+ ...checkpoint
12077
+ });
12078
+ }
12079
+ }
11777
12080
  async saveCheckpointIfPending(checkpoint) {
11778
12081
  const key = this.key(checkpoint.runId, checkpoint.seq);
11779
12082
  if (this.checkpoints.get(key)?.status !== "pending") return false;