@dudousxd/nestjs-durable-core 0.85.6 → 0.85.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -3347,6 +3347,7 @@ function constantTimeEquals(a, b) {
3347
3347
  __name(constantTimeEquals, "constantTimeEquals");
3348
3348
 
3349
3349
  // src/transport-pool.ts
3350
+ var DISPATCH_CHUNK = 200;
3350
3351
  var TransportPool = class {
3351
3352
  static {
3352
3353
  __name(this, "TransportPool");
@@ -3406,6 +3407,28 @@ var TransportPool = class {
3406
3407
  }
3407
3408
  throw lastErr ?? new Error("no transport accepted the dispatch");
3408
3409
  }
3410
+ /**
3411
+ * {@link dispatch} for many tasks: one {@link Transport.dispatchMany} call on the preferred (or
3412
+ * first) transport when it has one, else (or when that call rejects) each task through
3413
+ * {@link dispatch}, a bounded number at a time.
3414
+ */
3415
+ async dispatchMany(tasks, preferId) {
3416
+ if (tasks.length === 0) return;
3417
+ const first = this.ordered(preferId)[0];
3418
+ if (first?.transport.dispatchMany) {
3419
+ try {
3420
+ await first.transport.dispatchMany(tasks.map((task) => ({
3421
+ ...task,
3422
+ transport: first.id
3423
+ })));
3424
+ return;
3425
+ } catch {
3426
+ }
3427
+ }
3428
+ for (let at = 0; at < tasks.length; at += DISPATCH_CHUNK) {
3429
+ await Promise.all(tasks.slice(at, at + DISPATCH_CHUNK).map((task) => this.dispatch(task, preferId)));
3430
+ }
3431
+ }
3409
3432
  /** Worker-health for `group`, merged across every transport that can report it (a group's queue
3410
3433
  * and its workers may live on any pinned transport). Undefined when no transport implements
3411
3434
  * `groupHealth` (e.g. a pure in-process transport — nothing to introspect). */
@@ -4625,6 +4648,15 @@ var PARENT_NOTIFY_BACKOFF_MS = 250;
4625
4648
  var LEASE_RECHECK_MAX_MS = 2e3;
4626
4649
  var LEASE_RECHECK_SLACK_MS = 100;
4627
4650
  var LEASE_RECHECK_ATTEMPTS = 30;
4651
+ function wakeSeen(request, settledFans, served) {
4652
+ if (request.id !== void 0 && served.has(request.id)) return true;
4653
+ return isFanWakeRequest(request.payload) && request.payload.fans.every(([first, seq]) => (settledFans.get(first) ?? -1) >= seq);
4654
+ }
4655
+ __name(wakeSeen, "wakeSeen");
4656
+ function isFanWakeRequest(payload) {
4657
+ return typeof payload === "object" && payload !== null && Array.isArray(payload.fans);
4658
+ }
4659
+ __name(isFanWakeRequest, "isFanWakeRequest");
4628
4660
  function deferred() {
4629
4661
  let resolve;
4630
4662
  let reject;
@@ -4643,12 +4675,51 @@ __name(deferred, "deferred");
4643
4675
  var pendingStateKey = /* @__PURE__ */ __name((at) => String(new Date(at).getTime()), "pendingStateKey");
4644
4676
  var freshPendingKey = /* @__PURE__ */ __name(() => `${Date.now()}-${globalThis.crypto.randomUUID().slice(0, 8)}`, "freshPendingKey");
4645
4677
  var wakeRequestToken = /* @__PURE__ */ __name((runId) => `durable:wake-after-turn:${runId}`, "wakeRequestToken");
4678
+ var WAKE_REQUEST_SCAN = 100;
4646
4679
  var CHILD_SIGNAL_PREFIX = "signal:child:";
4647
4680
  var BUFFERED_SWEEP_PAGE = 500;
4648
4681
  var BUFFERED_SWEEP_PAGES_PER_CALL = 4;
4649
4682
  var BUFFERED_SWEEP_REST_MS = 10 * 6e4;
4650
4683
  var DELIVER_CHUNK = 50;
4651
4684
  var RESULT_BATCH_MAX = 256;
4685
+ function isResultError(outcome) {
4686
+ return typeof outcome === "object" && "error" in outcome;
4687
+ }
4688
+ __name(isResultError, "isResultError");
4689
+ var FanWakeCollector = class FanWakeCollector2 {
4690
+ static {
4691
+ __name(this, "FanWakeCollector");
4692
+ }
4693
+ carrier;
4694
+ generic = false;
4695
+ fan;
4696
+ /** The highest seq the batch settled, per fan. */
4697
+ highest = /* @__PURE__ */ new Map();
4698
+ /** A checkpoint the batch settled. */
4699
+ note(cp) {
4700
+ const fan = parseStepFanGroup(cp.parallelGroup);
4701
+ if (fan) this.highest.set(fan.firstSeq, Math.max(this.highest.get(fan.firstSeq) ?? cp.seq, cp.seq));
4702
+ }
4703
+ /** A settled checkpoint that resumes the run. */
4704
+ add(index, cp, status) {
4705
+ this.carrier = index;
4706
+ const fan = parseStepFanGroup(cp.parallelGroup);
4707
+ if (!fan || status === "failed" || this.fan !== void 0 && this.fan !== fan.firstSeq) {
4708
+ this.generic = true;
4709
+ return;
4710
+ }
4711
+ this.fan = fan.firstSeq;
4712
+ }
4713
+ applyTo(outcomes) {
4714
+ if (this.carrier === void 0) return outcomes;
4715
+ const seq = this.fan === void 0 ? void 0 : this.highest.get(this.fan);
4716
+ outcomes[this.carrier] = this.generic || this.fan === void 0 || seq === void 0 ? true : {
4717
+ first: this.fan,
4718
+ seq
4719
+ };
4720
+ return outcomes;
4721
+ }
4722
+ };
4652
4723
  function shuffle(items) {
4653
4724
  for (let i = items.length - 1; i > 0; i -= 1) {
4654
4725
  const j = Math.floor(Math.random() * (i + 1));
@@ -4837,6 +4908,23 @@ var WorkflowEngine = class {
4837
4908
  /** Re-checks scheduled for runs whose wake found another instance's lease (see {@link execute}). */
4838
4909
  leaseRetries = /* @__PURE__ */ new Map();
4839
4910
  /**
4911
+ * Runs whose scheduled re-check resumes them whatever happened meanwhile. Without an entry the
4912
+ * re-check only stood for fan wakes kept as requests, and resumes the run only if a request is
4913
+ * still there: the turn that held the lease takes them once it ends (see {@link retryDroppedWake}).
4914
+ */
4915
+ unconditionalRechecks = /* @__PURE__ */ new Set();
4916
+ /** Runs a generic wake request was written for in the current re-check cycle (see {@link execute}). */
4917
+ genericWakeRequested = /* @__PURE__ */ new Set();
4918
+ /** Why each run is being woken on this instance, until the turn that answers it starts. */
4919
+ wakeReasons = /* @__PURE__ */ new Map();
4920
+ /**
4921
+ * The step fans each executing turn on this instance read with nothing in flight: first seq -> the
4922
+ * highest seq read. Written by {@link callRemoteAll}, read once the turn ends.
4923
+ */
4924
+ turnSettledFans = /* @__PURE__ */ new Map();
4925
+ /** The wake requests each executing turn on this instance serves (see {@link noteServedWakes}). */
4926
+ turnServedWakes = /* @__PURE__ */ new Map();
4927
+ /**
4840
4928
  * Every run turn and child start this instance takes on itself, under one budget of
4841
4929
  * {@link LOCAL_RUN_CONCURRENCY}: the runs it dispatches to itself, a fan's deferred child starts,
4842
4930
  * enqueued pickups and the runs it takes off the transport's run queue.
@@ -4960,8 +5048,9 @@ var WorkflowEngine = class {
4960
5048
  }
4961
5049
  return;
4962
5050
  }
4963
- if (!await this.completeRemoteResult(result)) return;
4964
- await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId).then(() => void 0, (err) => {
5051
+ const wake = await this.completeRemoteResult(result);
5052
+ if (!wake) return;
5053
+ await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId, wake === true ? void 0 : wake).then(() => void 0, (err) => {
4965
5054
  if (err instanceof NamespaceMismatch) return;
4966
5055
  throw err;
4967
5056
  }));
@@ -5178,7 +5267,10 @@ var WorkflowEngine = class {
5178
5267
  const resumed = await this.resumeLeased([
5179
5268
  run
5180
5269
  ], nowMs, (locked) => this.reconcileSignalWaiter(locked, waiters.get(locked.id)));
5181
- if (resumed.length === 0) this.recheckAfterLease(runId, run.lockedUntil);
5270
+ if (resumed.length === 0) {
5271
+ this.unconditionalRechecks.add(runId);
5272
+ this.recheckAfterLease(runId, run.lockedUntil);
5273
+ }
5182
5274
  return;
5183
5275
  }
5184
5276
  await this.resumeImplicit(runId).catch((err) => {
@@ -5745,15 +5837,27 @@ var WorkflowEngine = class {
5745
5837
  * terminal too.
5746
5838
  */
5747
5839
  resume(runId) {
5840
+ this.noteWakeReason(runId, void 0);
5748
5841
  return this.resumeRun(runId, true);
5749
5842
  }
5843
+ /** Record why `runId` is being woken: a step fan's barrier opening, or (`undefined`) anything else. */
5844
+ noteWakeReason(runId, fan) {
5845
+ const reason = this.wakeReasons.get(runId) ?? {
5846
+ generic: false,
5847
+ fans: /* @__PURE__ */ new Map()
5848
+ };
5849
+ if (fan) reason.fans.set(fan.first, Math.max(reason.fans.get(fan.first) ?? fan.seq, fan.seq));
5850
+ else reason.generic = true;
5851
+ this.wakeReasons.set(runId, reason);
5852
+ }
5750
5853
  /**
5751
5854
  * {@link resume} for every IMPLICIT wake-up — a delivered signal/child completion, a late step
5752
5855
  * result, a due timer, a redelivered dispatch, a queue-slot wake. A `failed` run is terminal to
5753
5856
  * these: only an explicit retry ({@link requeue} / {@link resume}) may re-drive it. Otherwise a
5754
5857
  * stale waiter or a late result left behind by a failed run silently re-executes its body.
5755
5858
  */
5756
- resumeImplicit(runId) {
5859
+ resumeImplicit(runId, fan) {
5860
+ if (fan !== "coalesced") this.noteWakeReason(runId, fan);
5757
5861
  const turn = this.wakeTurns.get(runId);
5758
5862
  if (turn) {
5759
5863
  turn.again ??= deferred();
@@ -5778,7 +5882,7 @@ var WorkflowEngine = class {
5778
5882
  });
5779
5883
  return;
5780
5884
  }
5781
- const next = this.resumeImplicit(runId);
5885
+ const next = this.resumeImplicit(runId, "coalesced");
5782
5886
  next.then(again.resolve, again.reject);
5783
5887
  this.inBackground(next);
5784
5888
  }, "finish");
@@ -7843,9 +7947,26 @@ var WorkflowEngine = class {
7843
7947
  async execute(run, fn) {
7844
7948
  const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
7845
7949
  const lockNow = this.clock();
7950
+ const reason = this.wakeReasons.get(run.id) ?? {
7951
+ generic: true,
7952
+ fans: /* @__PURE__ */ new Map()
7953
+ };
7954
+ this.wakeReasons.delete(run.id);
7846
7955
  if (run.lockedBy !== this.instanceId || (run.lockedUntil ?? 0) <= lockNow) {
7847
7956
  if (!await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow)) {
7848
- if (!this.leaseRetries.has(run.id)) this.inBackground(this.requestWakeAfterTurn(run.id));
7957
+ if (reason.generic) {
7958
+ if (!this.genericWakeRequested.has(run.id)) {
7959
+ this.genericWakeRequested.add(run.id);
7960
+ this.inBackground(this.requestWakeAfterTurn(run.id, null));
7961
+ }
7962
+ } else {
7963
+ const fans = {
7964
+ fans: [
7965
+ ...reason.fans
7966
+ ]
7967
+ };
7968
+ this.inBackground(this.requestWakeAfterTurn(run.id, fans));
7969
+ }
7849
7970
  this.recheckAfterLease(run.id, run.lockedUntil);
7850
7971
  return {
7851
7972
  runId: run.id,
@@ -7853,56 +7974,142 @@ var WorkflowEngine = class {
7853
7974
  };
7854
7975
  }
7855
7976
  }
7977
+ const settledFans = /* @__PURE__ */ new Map();
7978
+ this.turnSettledFans.set(run.id, settledFans);
7856
7979
  const held = this.holdExecutionLease(run.id);
7857
7980
  try {
7981
+ const served = /* @__PURE__ */ new Set();
7982
+ this.turnServedWakes.set(run.id, served);
7983
+ if (registered?.remote) await this.noteServedWakes(run.id);
7858
7984
  const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
7859
7985
  if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
7860
7986
  void this.singletons.wakeNext(run).catch(() => void 0);
7861
7987
  }
7862
- if (result.status === "suspended") this.inBackground(this.honorWakeRequests(run.id));
7863
- else if (TERMINAL_RUN_STATUSES.includes(result.status)) {
7864
- this.inBackground(this.takeWakeRequests(run.id));
7988
+ if (result.status === "suspended") {
7989
+ this.inBackground(this.honorWakeRequests(run.id, settledFans, served));
7990
+ } else if (TERMINAL_RUN_STATUSES.includes(result.status)) {
7991
+ this.inBackground(this.listWakeRequests(run.id, true).then((kept) => this.dropWakeRequests(kept)));
7865
7992
  }
7866
7993
  return result;
7867
7994
  } finally {
7868
7995
  held.stop();
7996
+ if (this.turnSettledFans.get(run.id) === settledFans) this.turnSettledFans.delete(run.id);
7997
+ this.turnServedWakes.delete(run.id);
7869
7998
  }
7870
7999
  }
7871
8000
  /**
7872
8001
  * Keep a wake that found `runId`'s lease held by a turn executing elsewhere, durably, so that turn
7873
8002
  * runs again once it ends instead of suspending on a snapshot that predates what the wake brought
7874
- * (a signal, a child's completion). The request is a buffered signal on a token no workflow waits
7875
- * on; the holder takes it after it releases the lease ({@link honorWakeRequests}).
8003
+ * (a signal, a child's completion). The request is a buffered event under a name no workflow waits
8004
+ * on. It stays until a turn that started after it has settled: the holder looks at it after it
8005
+ * releases the lease ({@link honorWakeRequests}), and the turn that serves it drops it.
7876
8006
  *
7877
8007
  * Written first and looked at second, against a holder that releases first and looks second:
7878
8008
  * whichever look runs last sees the other side's write. So when the run is no longer executing by
7879
- * the time this re-reads it, the holder may have looked already, and this side takes the request
7880
- * and wakes the run itself once the lease is free.
8009
+ * the time this re-reads it, the holder may have looked already, and this side wakes the run
8010
+ * itself once the lease is free ({@link retryDroppedWake}). While the holder still runs, the run is
8011
+ * made due when the holder's lease runs out, so a holder that goes down before it settles leaves
8012
+ * the run to any instance's timer poll; a holder that settles sets the run's timer itself.
7881
8013
  */
7882
- async requestWakeAfterTurn(runId) {
7883
- await this.store.bufferSignal(wakeRequestToken(runId), null);
8014
+ async requestWakeAfterTurn(runId, request) {
8015
+ const id = globalThis.crypto.randomUUID();
8016
+ await this.store.bufferEvent({
8017
+ name: wakeRequestToken(runId),
8018
+ payload: request,
8019
+ id,
8020
+ publishedAt: this.clock()
8021
+ });
7884
8022
  const run = await this.store.getRun(runId);
7885
- if (run?.status === "running") return;
7886
- if (!run) {
7887
- await this.takeWakeRequests(runId);
8023
+ if (!run || TERMINAL_RUN_STATUSES.includes(run.status)) {
8024
+ await this.store.removeBufferedEvent(id);
8025
+ return;
8026
+ }
8027
+ if (run.status === "running") return;
8028
+ if (run.lockedBy !== void 0 && run.lockedBy !== this.instanceId) {
8029
+ await this.makeDueBy(runId, (run.lockedUntil ?? this.clock()) + LEASE_RECHECK_SLACK_MS);
7888
8030
  return;
7889
8031
  }
7890
- if (!await this.takeWakeRequests(runId)) return;
7891
- if (TERMINAL_RUN_STATUSES.includes(run.status)) return;
7892
8032
  this.recheckAfterLease(runId, run.lockedUntil);
7893
8033
  }
7894
- /** After a turn that suspended released its lease: run again if a wake was kept for it. */
7895
- async honorWakeRequests(runId) {
7896
- if (!await this.takeWakeRequests(runId)) return;
8034
+ /**
8035
+ * After a turn that suspended released its lease: drop the wakes it served, and run again if a
8036
+ * wake was kept for it meanwhile, unless every such wake was a fan wake this turn already saw: it
8037
+ * read that fan with nothing in flight, at least as far as the batch that opened its barrier.
8038
+ * The run is made due before it is woken, so if this instance goes down before that turn runs,
8039
+ * the timer poll of another instance runs it once this instance's hold on it would have lapsed.
8040
+ */
8041
+ async honorWakeRequests(runId, settledFans, served) {
8042
+ const kept = await this.listWakeRequests(runId, true);
8043
+ const seen = /* @__PURE__ */ __name((request) => wakeSeen(request, settledFans, served), "seen");
8044
+ const needed = kept.filter((request) => !seen(request));
8045
+ await this.dropWakeRequests(kept.filter(seen));
8046
+ if (needed.length === 0) return;
8047
+ await this.makeDueBy(runId, this.clock() + this.leaseMs);
7897
8048
  await this.wake(runId).catch((err) => {
7898
8049
  if (!(err instanceof NamespaceMismatch)) throw err;
7899
8050
  });
7900
8051
  }
7901
- /** Take every wake request kept for `runId`; true when there was one. */
7902
- async takeWakeRequests(runId) {
7903
- let taken = false;
7904
- while (await this.store.takeBufferedSignal(wakeRequestToken(runId))) taken = true;
7905
- return taken;
8052
+ /**
8053
+ * The wake requests kept for `runId`. With `legacy`, also those an older engine kept as buffered
8054
+ * signals, which are taken as they are read.
8055
+ */
8056
+ async listWakeRequests(runId, legacy = false) {
8057
+ const token = wakeRequestToken(runId);
8058
+ const kept = (await this.store.listBufferedEvents(token, WAKE_REQUEST_SCAN)).map((request) => ({
8059
+ id: request.id,
8060
+ payload: request.payload
8061
+ }));
8062
+ if (!legacy) return kept;
8063
+ for (; ; ) {
8064
+ const signal = await this.store.takeBufferedSignal(token);
8065
+ if (!signal) return kept;
8066
+ kept.push({
8067
+ payload: signal.payload
8068
+ });
8069
+ }
8070
+ }
8071
+ /** Delete `requests` (the ones kept as buffered events). */
8072
+ async dropWakeRequests(requests) {
8073
+ await Promise.all(requests.flatMap((request) => request.id === void 0 ? [] : [
8074
+ this.store.removeBufferedEvent(request.id)
8075
+ ]));
8076
+ }
8077
+ /**
8078
+ * Record, for the turn executing `runId` here, the wakes kept for the run so far. Called before the
8079
+ * turn reads the run's state, so the turn serves them: they are dropped once it has settled, and
8080
+ * until then stay for whoever runs the run next.
8081
+ */
8082
+ async noteServedWakes(runId) {
8083
+ const served = this.turnServedWakes.get(runId);
8084
+ if (!served) return;
8085
+ for (const request of await this.store.listBufferedEvents(wakeRequestToken(runId), WAKE_REQUEST_SCAN)) {
8086
+ served.add(request.id);
8087
+ }
8088
+ }
8089
+ /**
8090
+ * Right after a turn of `runId` wrote its suspension, while it still holds the lease: if a wake
8091
+ * was kept for the run during the turn that the turn did not see, keep the run due when the lease
8092
+ * runs out. The suspension overwrote the timer the instance keeping the wake set, and this instance
8093
+ * may go down before it looks at the wake once it has released the lease.
8094
+ */
8095
+ async keepDueForKeptWakes(runId) {
8096
+ const served = this.turnServedWakes.get(runId);
8097
+ if (!served) return;
8098
+ const settledFans = this.turnSettledFans.get(runId) ?? /* @__PURE__ */ new Map();
8099
+ const kept = await this.listWakeRequests(runId);
8100
+ if (kept.every((request) => wakeSeen(request, settledFans, served))) return;
8101
+ await this.makeDueBy(runId, this.clock() + this.leaseMs);
8102
+ }
8103
+ /** Bring a suspended run's timer forward to `at` (never later than the one it has). */
8104
+ async makeDueBy(runId, at) {
8105
+ const run = await this.store.getRun(runId);
8106
+ if (run?.status !== "suspended") return;
8107
+ if (run.wakeAt !== void 0 && run.wakeAt <= at) return;
8108
+ await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, runId, [
8109
+ "suspended"
8110
+ ], {
8111
+ wakeAt: at
8112
+ });
7906
8113
  }
7907
8114
  /**
7908
8115
  * A wake that found the run leased by another instance is dropped, and normally that is fine: the
@@ -7921,6 +8128,7 @@ var WorkflowEngine = class {
7921
8128
  const remaining = Math.max(0, (lockedUntil ?? this.clock()) - this.clock());
7922
8129
  const timer = setTimeout(() => {
7923
8130
  this.leaseRetries.delete(runId);
8131
+ this.genericWakeRequested.delete(runId);
7924
8132
  this.inBackground(this.retryDroppedWake(runId, attempt));
7925
8133
  }, Math.min(remaining, LEASE_RECHECK_MAX_MS) + LEASE_RECHECK_SLACK_MS);
7926
8134
  timer.unref?.();
@@ -7929,12 +8137,15 @@ var WorkflowEngine = class {
7929
8137
  async retryDroppedWake(runId, attempt) {
7930
8138
  if (this.draining) return;
7931
8139
  const run = await this.store.getRun(runId);
7932
- if (run?.status !== "suspended") return;
7933
8140
  const nowMs = this.clock();
7934
- if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) {
7935
- if (attempt < LEASE_RECHECK_ATTEMPTS) this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
8141
+ if (run?.status === "suspended" && run.lockedUntil !== void 0 && run.lockedUntil > nowMs && attempt < LEASE_RECHECK_ATTEMPTS) {
8142
+ this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
7936
8143
  return;
7937
8144
  }
8145
+ const unconditional = this.unconditionalRechecks.delete(runId);
8146
+ if (run?.status !== "suspended") return;
8147
+ if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) return;
8148
+ if (!unconditional && (await this.listWakeRequests(runId)).length === 0) return;
7938
8149
  if (this.queuesRunsOverTransport) {
7939
8150
  await this.wake(runId);
7940
8151
  return;
@@ -8273,6 +8484,7 @@ ${run.workflowVersion}`;
8273
8484
  updatedAt
8274
8485
  });
8275
8486
  if (!applied) return this.echoCurrentStatus(run.id);
8487
+ await this.keepDueForKeptWakes(run.id);
8276
8488
  this.emit({
8277
8489
  type: "run.suspended",
8278
8490
  runId: run.id,
@@ -8966,6 +9178,7 @@ ${run.workflowVersion}`;
8966
9178
  namespace: run.namespace
8967
9179
  });
8968
9180
  }
9181
+ await this.noteServedWakes(run.id);
8969
9182
  if (run.status !== "cancelling" && registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
8970
9183
  return this.singletonWaitResult(run);
8971
9184
  }
@@ -9639,6 +9852,14 @@ ${run.workflowVersion}`;
9639
9852
  throw new WorkflowSuspended();
9640
9853
  }
9641
9854
  /**
9855
+ * Whether a `pending` step a turn just read only waits for its result: {@link callRemote} would
9856
+ * suspend on it without writing (no redispatch deadline to stamp, the one stamped not yet due).
9857
+ */
9858
+ stepIsWaiting(cp) {
9859
+ if (this.remoteRedispatchMs == null) return true;
9860
+ return cp.wakeAt != null && this.clock() < cp.wakeAt;
9861
+ }
9862
+ /**
9642
9863
  * `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
9643
9864
  * reserves the whole position block up front, so replay re-derives the same seqs.
9644
9865
  *
@@ -9668,6 +9889,7 @@ ${run.workflowVersion}`;
9668
9889
  const failures = [];
9669
9890
  const unsent = [];
9670
9891
  let inFlight = 0;
9892
+ let highest = -1;
9671
9893
  let wakeAt;
9672
9894
  for (let i = 0; i < inputs.length; i += 1) {
9673
9895
  const seq = positions[i];
@@ -9677,10 +9899,19 @@ ${run.workflowVersion}`;
9677
9899
  continue;
9678
9900
  }
9679
9901
  if (cp.name !== step.name) throw new NonDeterminismError(runId, seq, step.name, cp.name);
9902
+ if (seq > highest) highest = seq;
9680
9903
  if (cp.status === "completed") {
9681
9904
  outputs[i] = cp.output;
9682
9905
  continue;
9683
9906
  }
9907
+ if (cp.status === "pending" && this.stepIsWaiting(cp)) {
9908
+ inFlight += 1;
9909
+ if (cp.wakeAt != null) {
9910
+ const at = this.stepLeaseWakeAt(cp.wakeAt);
9911
+ wakeAt = Math.min(wakeAt ?? at, at);
9912
+ }
9913
+ continue;
9914
+ }
9684
9915
  try {
9685
9916
  outputs[i] = await this.callRemote(runId, seq, step, inputs[i], void 0, opts.transport, void 0, opts.admission, {
9686
9917
  ...routing,
@@ -9708,6 +9939,7 @@ ${run.workflowVersion}`;
9708
9939
  this.awaitedRemoteSteps.get(runId)?.delete(seq);
9709
9940
  }
9710
9941
  }
9942
+ if (inFlight === 0 && highest >= 0) this.turnSettledFans.get(runId)?.set(first, highest);
9711
9943
  const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
9712
9944
  if (batch.length > 0) await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
9713
9945
  if (inFlight + batch.length > 0) {
@@ -9738,53 +9970,62 @@ ${run.workflowVersion}`;
9738
9970
  if (failures.length > 0) throw new GatherError(failures, inputs.length);
9739
9971
  return outputs;
9740
9972
  }
9741
- /** Write the `pending` checkpoints of a step fan's new steps, then dispatch them. */
9973
+ /**
9974
+ * Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
9975
+ * {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
9976
+ * batch, where the store and transport have them.
9977
+ */
9742
9978
  async sendStepFan(runId, positions, step, inputs, batch, group, opts, routing) {
9743
9979
  const workerGroup = stepGroup(step, routing?.namespace);
9744
9980
  await this.ensureRoutable(workerGroup, step.requires);
9745
9981
  const enqueuedAt = /* @__PURE__ */ new Date();
9746
- for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
9747
- await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map((i) => {
9748
- const seq = positions[i];
9749
- return this.store.saveCheckpoint({
9750
- runId,
9751
- seq,
9752
- name: step.name,
9753
- kind: "remote",
9754
- stepId: stepId(runId, seq),
9755
- status: "pending",
9756
- input: inputs[i],
9757
- attempts: 1,
9758
- workerGroup,
9759
- parallelGroup: group,
9760
- enqueuedAt,
9761
- startedAt: enqueuedAt,
9762
- finishedAt: enqueuedAt
9763
- });
9764
- }));
9982
+ const checkpoints = batch.map((i) => {
9983
+ const seq = positions[i];
9984
+ return {
9985
+ runId,
9986
+ seq,
9987
+ name: step.name,
9988
+ kind: "remote",
9989
+ stepId: stepId(runId, seq),
9990
+ status: "pending",
9991
+ input: inputs[i],
9992
+ attempts: 1,
9993
+ workerGroup,
9994
+ parallelGroup: group,
9995
+ enqueuedAt,
9996
+ startedAt: enqueuedAt,
9997
+ finishedAt: enqueuedAt
9998
+ };
9999
+ });
10000
+ if (this.store.saveCheckpoints) {
10001
+ await this.store.saveCheckpoints(checkpoints);
10002
+ } else {
10003
+ for (let at = 0; at < checkpoints.length; at += STEP_FAN_CHUNK) {
10004
+ await Promise.all(checkpoints.slice(at, at + STEP_FAN_CHUNK).map((cp) => this.store.saveCheckpoint(cp)));
10005
+ }
9765
10006
  }
9766
- for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
9767
- await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map(async (i) => {
9768
- const seq = positions[i];
9769
- await this.dispatchRemoteTask({
9770
- runId,
9771
- seq,
9772
- name: step.name,
9773
- stepId: stepId(runId, seq),
9774
- group: workerGroup,
9775
- input: inputs[i],
9776
- priority: opts.admission?.priority,
9777
- attempt: 1,
9778
- transport: opts.transport
9779
- });
9780
- this.emit({
9781
- type: "step.started",
9782
- runId,
9783
- seq,
9784
- name: step.name,
9785
- kind: "remote"
9786
- });
9787
- }));
10007
+ const traceparent = this.traceparent?.();
10008
+ const context = this.context?.();
10009
+ await this.pool.dispatchMany(checkpoints.map((cp, k) => ({
10010
+ runId,
10011
+ seq: cp.seq,
10012
+ name: step.name,
10013
+ stepId: cp.stepId,
10014
+ group: workerGroup,
10015
+ input: inputs[batch[k]],
10016
+ traceparent,
10017
+ context,
10018
+ priority: opts.admission?.priority,
10019
+ attempt: 1
10020
+ })), opts.transport);
10021
+ for (const cp of checkpoints) {
10022
+ this.emit({
10023
+ type: "step.started",
10024
+ runId,
10025
+ seq: cp.seq,
10026
+ name: step.name,
10027
+ kind: "remote"
10028
+ });
9788
10029
  }
9789
10030
  }
9790
10031
  /** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
@@ -9825,6 +10066,27 @@ ${run.workflowVersion}`;
9825
10066
  error: s.reason
9826
10067
  });
9827
10068
  }
10069
+ /**
10070
+ * The checkpoints of one run at distinct `seqs`, each as its own outcome: one
10071
+ * {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
10072
+ * call rejecting falls back to one read each, so a failing read fails only its own entry.
10073
+ */
10074
+ async readCheckpoints(runId, seqs) {
10075
+ if (seqs.length > 1 && this.store.getCheckpoints) {
10076
+ try {
10077
+ const bySeq = new Map((await this.store.getCheckpoints(runId, seqs)).map((cp) => [
10078
+ cp.seq,
10079
+ cp
10080
+ ]));
10081
+ return seqs.map((seq) => ({
10082
+ status: "fulfilled",
10083
+ value: bySeq.get(seq) ?? null
10084
+ }));
10085
+ } catch {
10086
+ }
10087
+ }
10088
+ return Promise.allSettled(seqs.map((seq) => this.store.getCheckpoint(runId, seq)));
10089
+ }
9828
10090
  /** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
9829
10091
  * dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
9830
10092
  * {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
@@ -9866,9 +10128,10 @@ ${run.workflowVersion}`;
9866
10128
  *
9867
10129
  * Results of one run are settled in batches: the first starts a batch, and the ones that arrive
9868
10130
  * while it is being written form the next. Under load a wide fan's results are written by one
9869
- * {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked once
9870
- * per batch, not once per result. A result resolves only once its own write has committed, so the
9871
- * transport acknowledges nothing that is not durable yet.
10131
+ * {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked only
10132
+ * by a batch with no more of the fan's results queued behind it. A result resolves only once its
10133
+ * own write has committed, so the transport acknowledges nothing that is not durable yet. How
10134
+ * large a batch gets is bounded by how many results the transport hands over at once.
9872
10135
  */
9873
10136
  completeRemoteResult(result) {
9874
10137
  return new Promise((resolve, reject) => {
@@ -9896,7 +10159,7 @@ ${run.workflowVersion}`;
9896
10159
  const batch = queue.splice(0, RESULT_BATCH_MAX);
9897
10160
  let outcomes;
9898
10161
  try {
9899
- outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result));
10162
+ outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result), (from, to) => queue.some(({ result }) => result.seq >= from && result.seq <= to));
9900
10163
  } catch (err) {
9901
10164
  outcomes = batch.map(() => ({
9902
10165
  error: err
@@ -9904,8 +10167,8 @@ ${run.workflowVersion}`;
9904
10167
  }
9905
10168
  batch.forEach((q, i) => {
9906
10169
  const outcome = outcomes[i] ?? false;
9907
- if (typeof outcome === "boolean") q.resolve(outcome);
9908
- else q.reject(outcome.error);
10170
+ if (isResultError(outcome)) q.reject(outcome.error);
10171
+ else q.resolve(outcome);
9909
10172
  });
9910
10173
  }
9911
10174
  this.resultQueues.delete(runId);
@@ -9914,8 +10177,13 @@ ${run.workflowVersion}`;
9914
10177
  * Settle a batch of one run's step results. At most one of them resolves `true` (resume the run):
9915
10178
  * that resume replays every write of the batch, so a second would only replay it again. A result
9916
10179
  * whose read or write failed resolves its error, and the transport redelivers just that one.
10180
+ *
10181
+ * `queuedIn(from, to)` says whether a result for a seq in that range is already waiting for this
10182
+ * run's next batch on this instance. A fan with one waiting is not asked its barrier: that result
10183
+ * asks it once its own batch is written (as a settle, or as a redelivery of a settled step), so
10184
+ * under a backlog the barrier is read once per drained queue rather than once per batch.
9917
10185
  */
9918
- async completeRemoteResults(runId, results) {
10186
+ async completeRemoteResults(runId, results, queuedIn = () => false) {
9919
10187
  const outcomes = results.map(() => false);
9920
10188
  const seen = /* @__PURE__ */ new Set();
9921
10189
  const live = results.flatMap((r, i) => {
@@ -9925,7 +10193,7 @@ ${run.workflowVersion}`;
9925
10193
  i
9926
10194
  ];
9927
10195
  });
9928
- const reads = await Promise.allSettled(live.map((i) => this.store.getCheckpoint(runId, results[i].seq)));
10196
+ const reads = await this.readCheckpoints(runId, live.map((i) => results[i].seq));
9929
10197
  const pending = [];
9930
10198
  const resettled = [];
9931
10199
  reads.forEach((read, k) => {
@@ -9946,7 +10214,7 @@ ${run.workflowVersion}`;
9946
10214
  });
9947
10215
  if (pending.length === 0) {
9948
10216
  if (resettled.length === 0) return outcomes;
9949
- return this.resumeForRedelivered(runId, resettled, outcomes);
10217
+ return this.resumeForRedelivered(runId, resettled, outcomes, queuedIn);
9950
10218
  }
9951
10219
  for (const { cp } of pending) void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
9952
10220
  const run = await this.store.getRun(runId);
@@ -9973,7 +10241,9 @@ ${run.workflowVersion}`;
9973
10241
  cp,
9974
10242
  result: results[index]
9975
10243
  });
9976
- else if (claim !== false) outcomes[index] = claim;
10244
+ else if (claim !== false) outcomes[index] = {
10245
+ error: claim.error
10246
+ };
9977
10247
  });
9978
10248
  for (const { cp, result } of settled) {
9979
10249
  this.noteSettledHere(`${runId}:${result.seq}`);
@@ -9991,19 +10261,21 @@ ${run.workflowVersion}`;
9991
10261
  });
9992
10262
  }
9993
10263
  const barriers = /* @__PURE__ */ new Map();
9994
- let carrier;
10264
+ const wake = new FanWakeCollector();
10265
+ for (const { cp } of settled) wake.note(cp);
9995
10266
  for (const { index, cp, result } of settled) {
9996
- if (!await this.stepFanStillRunning(cp, result, barriers)) carrier = index;
10267
+ if (!await this.stepFanStillRunning(cp, result, barriers, queuedIn)) {
10268
+ wake.add(index, cp, result.status);
10269
+ }
9997
10270
  }
9998
- if (carrier !== void 0) outcomes[carrier] = true;
9999
- return outcomes;
10271
+ return wake.applyTo(outcomes);
10000
10272
  }
10001
10273
  /**
10002
10274
  * The outcomes for a batch of redelivered results (their steps already settled): one of them
10003
10275
  * resumes the run if it is `suspended` and, for a step of a fan, the fan's barrier is open. A
10004
10276
  * resume of a run that had in fact moved on replays to its current wait and re-parks.
10005
10277
  */
10006
- async resumeForRedelivered(runId, resettled, outcomes) {
10278
+ async resumeForRedelivered(runId, resettled, outcomes, queuedIn) {
10007
10279
  const run = await this.store.getRun(runId);
10008
10280
  if (run?.status !== "suspended") return outcomes;
10009
10281
  const barriers = /* @__PURE__ */ new Map();
@@ -10014,9 +10286,11 @@ ${run.workflowVersion}`;
10014
10286
  stepId: cp.stepId ?? stepId(runId, cp.seq),
10015
10287
  status: cp.status === "failed" ? "failed" : "completed"
10016
10288
  };
10017
- if (!await this.stepFanStillRunning(cp, asResult, barriers)) {
10018
- outcomes[index] = true;
10019
- break;
10289
+ if (!await this.stepFanStillRunning(cp, asResult, barriers, queuedIn)) {
10290
+ const wake = new FanWakeCollector();
10291
+ for (const settled of resettled) wake.note(settled.cp);
10292
+ wake.add(index, cp, asResult.status);
10293
+ return wake.applyTo(outcomes);
10020
10294
  }
10021
10295
  }
10022
10296
  return outcomes;
@@ -10027,15 +10301,17 @@ ${run.workflowVersion}`;
10027
10301
  * retry needs the parent to schedule it). Without it every result would replay the whole parent,
10028
10302
  * which makes a fan of N steps cost O(N²) checkpoint reads. `barriers` holds the answer per fan
10029
10303
  * for the rest of a batch. A barrier that cannot be read counts as open: a spare resume replays and
10030
- * re-suspends, a lost one strands the run.
10304
+ * re-suspends, a lost one strands the run. A fan with a result waiting in `queuedIn` is left to
10305
+ * that result (see {@link completeRemoteResults}).
10031
10306
  */
10032
- async stepFanStillRunning(cp, result, barriers) {
10307
+ async stepFanStillRunning(cp, result, barriers, queuedIn) {
10033
10308
  const fan = parseStepFanGroup(cp.parallelGroup);
10034
10309
  if (!fan) return false;
10035
10310
  if (result.status === "failed") return false;
10036
10311
  let closed = barriers.get(fan.firstSeq);
10037
10312
  if (!closed) {
10038
- closed = this.stepFanPending(result.runId, fan.firstSeq, fan.firstSeq + fan.size - 1).then((pending) => pending > 0, () => false);
10313
+ const lastSeq = fan.firstSeq + fan.size - 1;
10314
+ closed = queuedIn(fan.firstSeq, lastSeq) ? Promise.resolve(true) : this.stepFanPending(result.runId, fan.firstSeq, lastSeq).then((pending) => pending > 0, () => false);
10039
10315
  barriers.set(fan.firstSeq, closed);
10040
10316
  }
10041
10317
  if (await closed) return true;
@@ -10869,6 +11145,10 @@ var CodecStateStore = class {
10869
11145
  saveCheckpointIfPending;
10870
11146
  /** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
10871
11147
  saveCheckpointsIfPending;
11148
+ /** Encodes each checkpoint written. Bound only when the inner store has it. */
11149
+ saveCheckpoints;
11150
+ /** Decodes each checkpoint read. Bound only when the inner store has it. */
11151
+ getCheckpoints;
10872
11152
  /** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
10873
11153
  countCheckpoints;
10874
11154
  /** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
@@ -10919,6 +11199,10 @@ var CodecStateStore = class {
10919
11199
  if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
10920
11200
  const saveManyIfPending = inner.saveCheckpointsIfPending;
10921
11201
  if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
11202
+ const saveMany = inner.saveCheckpoints;
11203
+ if (saveMany) this.saveCheckpoints = (cps) => saveMany.call(inner, cps.map((cp) => this.encCp(cp)));
11204
+ const getMany = inner.getCheckpoints;
11205
+ if (getMany) this.getCheckpoints = async (runId, seqs) => (await getMany.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
10922
11206
  const countCps = inner.countCheckpoints;
10923
11207
  if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
10924
11208
  const stats = inner.checkpointStats;
@@ -11555,11 +11839,30 @@ var InMemoryStateStore = class {
11555
11839
  ...cp
11556
11840
  } : null;
11557
11841
  }
11842
+ async getCheckpoints(runId, seqs) {
11843
+ return [
11844
+ ...new Set(seqs)
11845
+ ].sort((a, b) => a - b).flatMap((seq) => {
11846
+ const cp = this.checkpoints.get(this.key(runId, seq));
11847
+ return cp ? [
11848
+ {
11849
+ ...cp
11850
+ }
11851
+ ] : [];
11852
+ });
11853
+ }
11558
11854
  async saveCheckpoint(checkpoint) {
11559
11855
  this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
11560
11856
  ...checkpoint
11561
11857
  });
11562
11858
  }
11859
+ async saveCheckpoints(checkpoints) {
11860
+ for (const checkpoint of checkpoints) {
11861
+ this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
11862
+ ...checkpoint
11863
+ });
11864
+ }
11865
+ }
11563
11866
  async saveCheckpointIfPending(checkpoint) {
11564
11867
  const key = this.key(checkpoint.runId, checkpoint.seq);
11565
11868
  if (this.checkpoints.get(key)?.status !== "pending") return false;