@dudousxd/nestjs-durable-core 0.85.6 → 0.85.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -937,7 +937,6 @@ var RETRYABLE = /* @__PURE__ */ new Set([
937
937
  "failed",
938
938
  "dead"
939
939
  ]);
940
- var FAN_CHILD = /^(.+\.all\.\d+\.(?:w\d+\.)?)(\d+)$/;
941
940
  function canonicalJson(value) {
942
941
  const plain = JSON.stringify(value);
943
942
  return plain === void 0 ? "null" : sortedJson(JSON.parse(plain));
@@ -952,77 +951,235 @@ function sortedJson(value) {
952
951
  return JSON.stringify(value);
953
952
  }
954
953
  __name(sortedJson, "sortedJson");
955
- function settledNames(childId) {
954
+ var FAN_CHILD_PARTS = /^(.+\.all\.(\d+)\.)(?:w(\d+)\.)?(\d+)$/;
955
+ var SEQ_CHILD = /\.child\.(\d+)$/;
956
+ function slotOf(childId) {
956
957
  const signal = `${CHILD_SIGNAL_NAME_PREFIX}${childId}`;
957
- const fan = FAN_CHILD.exec(childId);
958
- if (!fan) return {
959
- signal
960
- };
961
- return {
962
- signal,
963
- fold: {
964
- name: `${FOLDED_PREFIX}${fan[1]}`,
965
- index: Number(fan[2])
958
+ const fan = FAN_CHILD_PARTS.exec(childId);
959
+ if (fan) {
960
+ const root = fan[1];
961
+ const wave = fan[3];
962
+ const firstSeq = Number(fan[2]);
963
+ const i = Number(fan[4]);
964
+ if (wave === void 0) {
965
+ return {
966
+ signal,
967
+ signalSeq: firstSeq + i,
968
+ fan: {
969
+ fold: `${FOLDED_PREFIX}${root}`,
970
+ firstSeq,
971
+ index: i
972
+ }
973
+ };
966
974
  }
975
+ return {
976
+ signal,
977
+ signalSeq: firstSeq + i,
978
+ fan: {
979
+ fold: `${FOLDED_PREFIX}${root}w${wave}.`,
980
+ firstSeq,
981
+ index: i,
982
+ wave: Number(wave),
983
+ waveZeroFold: `${FOLDED_PREFIX}${root}w0.`
984
+ }
985
+ };
986
+ }
987
+ const seq = SEQ_CHILD.exec(childId);
988
+ return seq ? {
989
+ signal,
990
+ signalSeq: Number(seq[1])
991
+ } : {
992
+ signal
967
993
  };
968
994
  }
969
- __name(settledNames, "settledNames");
970
- function parentHoldsChild(checkpoints, childId, completion) {
971
- const { signal, fold } = settledNames(childId);
972
- const expected = canonicalJson(completion);
973
- return checkpoints.some((cp) => {
974
- if (cp.status !== "completed") return false;
975
- if (cp.name === signal) return canonicalJson(cp.output) === expected;
976
- if (fold && cp.name === fold.name) {
977
- const { from, values } = cp.output ?? {};
978
- return typeof from === "number" && Array.isArray(values) && fold.index >= from && fold.index < from + values.length;
995
+ __name(slotOf, "slotOf");
996
+ function foldOf(cp, name) {
997
+ if (!cp || cp.status !== "completed" || cp.name !== name) return void 0;
998
+ const { from, values } = cp.output ?? {};
999
+ return typeof from === "number" && Array.isArray(values) ? {
1000
+ from,
1001
+ values
1002
+ } : void 0;
1003
+ }
1004
+ __name(foldOf, "foldOf");
1005
+ function covers(fold, index) {
1006
+ return !!fold && index >= fold.from && index < fold.from + fold.values.length;
1007
+ }
1008
+ __name(covers, "covers");
1009
+ function holdsSignal(cp, signal, expected) {
1010
+ return cp.status === "completed" && cp.name === signal && canonicalJson(cp.output) === expected;
1011
+ }
1012
+ __name(holdsSignal, "holdsSignal");
1013
+ async function checkpointsAt(store, runId, seqs) {
1014
+ const unique = [
1015
+ ...new Set(seqs)
1016
+ ];
1017
+ const bySeq = /* @__PURE__ */ new Map();
1018
+ if (unique.length === 0) return bySeq;
1019
+ const rows = store.getCheckpoints ? await store.getCheckpoints(runId, unique) : (await Promise.all(unique.map((seq) => store.getCheckpoint(runId, seq)))).filter((cp) => cp !== null);
1020
+ for (const cp of rows) bySeq.set(cp.seq, cp);
1021
+ return bySeq;
1022
+ }
1023
+ __name(checkpointsAt, "checkpointsAt");
1024
+ async function checkpointsNamed(store, runId, names) {
1025
+ const wanted = new Set(names);
1026
+ const rows = store.listCheckpointsByNamePrefix ? await store.listCheckpointsByNamePrefix(runId, [
1027
+ ...wanted
1028
+ ]) : await store.listCheckpoints(runId);
1029
+ return rows.filter((cp) => wanted.has(cp.name));
1030
+ }
1031
+ __name(checkpointsNamed, "checkpointsNamed");
1032
+ async function parentSettles(store, parentId, children) {
1033
+ const entries = children.map(({ childId, completion }) => ({
1034
+ slot: slotOf(childId),
1035
+ expected: canonicalJson(completion)
1036
+ }));
1037
+ const held = entries.map(() => void 0);
1038
+ const first = await checkpointsAt(store, parentId, entries.flatMap(({ slot }) => [
1039
+ ...slot.signalSeq === void 0 ? [] : [
1040
+ slot.signalSeq
1041
+ ],
1042
+ ...slot.fan ? [
1043
+ slot.fan.firstSeq
1044
+ ] : []
1045
+ ]));
1046
+ const waveFolds = [];
1047
+ const byName = [];
1048
+ entries.forEach(({ slot, expected }, at) => {
1049
+ const own = slot.signalSeq === void 0 ? void 0 : first.get(slot.signalSeq);
1050
+ if (own?.name === slot.signal) {
1051
+ held[at] = holdsSignal(own, slot.signal, expected);
1052
+ return;
979
1053
  }
980
- return false;
1054
+ const fan = slot.fan;
1055
+ if (fan) {
1056
+ if (covers(foldOf(own, fan.fold), fan.index)) {
1057
+ held[at] = true;
1058
+ return;
1059
+ }
1060
+ const atFirst = first.get(fan.firstSeq);
1061
+ if (fan.wave === void 0 || fan.wave === 0) {
1062
+ held[at] = covers(foldOf(atFirst, fan.fold), fan.index);
1063
+ return;
1064
+ }
1065
+ const waveZero = foldOf(atFirst, fan.waveZeroFold);
1066
+ if (waveZero && waveZero.values.length > 0) {
1067
+ waveFolds.push({
1068
+ at,
1069
+ seq: fan.firstSeq + fan.wave * waveZero.values.length
1070
+ });
1071
+ return;
1072
+ }
1073
+ byName.push(at);
1074
+ return;
1075
+ }
1076
+ if (slot.signalSeq !== void 0 && !own) {
1077
+ held[at] = false;
1078
+ return;
1079
+ }
1080
+ byName.push(at);
981
1081
  });
1082
+ if (waveFolds.length > 0) {
1083
+ const second = await checkpointsAt(store, parentId, waveFolds.map((w) => w.seq));
1084
+ for (const { at, seq } of waveFolds) {
1085
+ const fan = entries[at].slot.fan;
1086
+ held[at] = covers(foldOf(second.get(seq), fan.fold), fan.index);
1087
+ }
1088
+ }
1089
+ if (byName.length > 0) {
1090
+ const names = byName.flatMap((at) => {
1091
+ const { slot } = entries[at];
1092
+ return slot.fan ? [
1093
+ slot.signal,
1094
+ slot.fan.fold
1095
+ ] : [
1096
+ slot.signal
1097
+ ];
1098
+ });
1099
+ const named = await checkpointsNamed(store, parentId, names);
1100
+ for (const at of byName) {
1101
+ const { slot, expected } = entries[at];
1102
+ held[at] = named.some((cp) => holdsSignal(cp, slot.signal, expected) || !!slot.fan && covers(foldOf(cp, slot.fan.fold), slot.fan.index));
1103
+ }
1104
+ }
1105
+ return held.map((h) => h === true);
982
1106
  }
983
- __name(parentHoldsChild, "parentHoldsChild");
984
- async function parentCheckpoints(store, parentId, childId) {
985
- const { signal, fold } = settledNames(childId);
986
- const prefixes = fold ? [
987
- signal,
988
- fold.name
989
- ] : [
990
- signal
991
- ];
992
- return store.listCheckpointsByNamePrefix ? store.listCheckpointsByNamePrefix(parentId, prefixes) : store.listCheckpoints(parentId);
1107
+ __name(parentSettles, "parentSettles");
1108
+ function parentClosed(run) {
1109
+ return !run || TERMINAL_RUN_STATUSES.includes(run.status) && !RETRYABLE.has(run.status);
993
1110
  }
994
- __name(parentCheckpoints, "parentCheckpoints");
1111
+ __name(parentClosed, "parentClosed");
995
1112
  async function childCompletionNeedsBuffer(store, childId, completion, parent) {
996
1113
  const parentId = parentRunIdOf(childId);
997
1114
  if (!parentId) return false;
998
1115
  const run = parent === void 0 ? await store.getRun(parentId) : parent;
999
- if (!run) return false;
1000
- if (TERMINAL_RUN_STATUSES.includes(run.status) && !RETRYABLE.has(run.status)) return false;
1001
- return !parentHoldsChild(await parentCheckpoints(store, parentId, childId), childId, completion);
1116
+ if (parentClosed(run)) return false;
1117
+ const [held] = await parentSettles(store, parentId, [
1118
+ {
1119
+ childId,
1120
+ completion
1121
+ }
1122
+ ]);
1123
+ return !held;
1002
1124
  }
1003
1125
  __name(childCompletionNeedsBuffer, "childCompletionNeedsBuffer");
1004
- async function unneededChildSignals(store, rows) {
1126
+ async function unneededChildSignals(store, rows, deadline = Number.POSITIVE_INFINITY) {
1127
+ const childOf = /* @__PURE__ */ __name((row) => row.token.startsWith(CHILD_TOKEN_PREFIX) ? row.token.slice(CHILD_TOKEN_PREFIX.length) : void 0, "childOf");
1005
1128
  const parentIds = rows.flatMap((row) => {
1006
- const parentId = parentRunIdOf(row.token.slice(CHILD_TOKEN_PREFIX.length));
1129
+ const childId = childOf(row);
1130
+ const parentId = childId === void 0 ? void 0 : parentRunIdOf(childId);
1007
1131
  return parentId ? [
1008
1132
  parentId
1009
1133
  ] : [];
1010
1134
  });
1011
1135
  const { runs, unread } = await readRunsByIds(store, parentIds);
1012
1136
  const unreadable = new Set(unread);
1013
- const doomed = [];
1014
- for (const row of rows) {
1015
- if (!row.token.startsWith(CHILD_TOKEN_PREFIX)) continue;
1016
- const childId = row.token.slice(CHILD_TOKEN_PREFIX.length);
1137
+ const doomed = /* @__PURE__ */ new Set();
1138
+ const open = /* @__PURE__ */ new Map();
1139
+ rows.forEach((row, at) => {
1140
+ const childId = childOf(row);
1141
+ if (childId === void 0) return;
1017
1142
  const parentId = parentRunIdOf(childId);
1018
- if (parentId && unreadable.has(parentId)) continue;
1019
- const parent = parentId ? runs.get(parentId) ?? null : null;
1143
+ if (!parentId) {
1144
+ doomed.add(row);
1145
+ return;
1146
+ }
1147
+ if (unreadable.has(parentId)) return;
1148
+ if (parentClosed(runs.get(parentId) ?? null)) {
1149
+ doomed.add(row);
1150
+ return;
1151
+ }
1152
+ const group = open.get(parentId) ?? [];
1153
+ group.push({
1154
+ at,
1155
+ row,
1156
+ childId
1157
+ });
1158
+ open.set(parentId, group);
1159
+ });
1160
+ let decided = rows.length;
1161
+ let checked = 0;
1162
+ for (const [parentId, group] of open) {
1163
+ if (checked > 0 && Date.now() >= deadline) {
1164
+ decided = group[0].at;
1165
+ break;
1166
+ }
1167
+ checked += 1;
1020
1168
  try {
1021
- if (!await childCompletionNeedsBuffer(store, childId, row.payload, parent)) doomed.push(row);
1169
+ const held = await parentSettles(store, parentId, group.map(({ row, childId }) => ({
1170
+ childId,
1171
+ completion: row.payload
1172
+ })));
1173
+ group.forEach(({ row }, k) => {
1174
+ if (held[k]) doomed.add(row);
1175
+ });
1022
1176
  } catch {
1023
1177
  }
1024
1178
  }
1025
- return doomed;
1179
+ return {
1180
+ doomed: rows.filter((row) => doomed.has(row)),
1181
+ decided
1182
+ };
1026
1183
  }
1027
1184
  __name(unneededChildSignals, "unneededChildSignals");
1028
1185
 
@@ -3347,6 +3504,7 @@ function constantTimeEquals(a, b) {
3347
3504
  __name(constantTimeEquals, "constantTimeEquals");
3348
3505
 
3349
3506
  // src/transport-pool.ts
3507
+ var DISPATCH_CHUNK = 200;
3350
3508
  var TransportPool = class {
3351
3509
  static {
3352
3510
  __name(this, "TransportPool");
@@ -3406,6 +3564,28 @@ var TransportPool = class {
3406
3564
  }
3407
3565
  throw lastErr ?? new Error("no transport accepted the dispatch");
3408
3566
  }
3567
+ /**
3568
+ * {@link dispatch} for many tasks: one {@link Transport.dispatchMany} call on the preferred (or
3569
+ * first) transport when it has one, else (or when that call rejects) each task through
3570
+ * {@link dispatch}, a bounded number at a time.
3571
+ */
3572
+ async dispatchMany(tasks, preferId) {
3573
+ if (tasks.length === 0) return;
3574
+ const first = this.ordered(preferId)[0];
3575
+ if (first?.transport.dispatchMany) {
3576
+ try {
3577
+ await first.transport.dispatchMany(tasks.map((task) => ({
3578
+ ...task,
3579
+ transport: first.id
3580
+ })));
3581
+ return;
3582
+ } catch {
3583
+ }
3584
+ }
3585
+ for (let at = 0; at < tasks.length; at += DISPATCH_CHUNK) {
3586
+ await Promise.all(tasks.slice(at, at + DISPATCH_CHUNK).map((task) => this.dispatch(task, preferId)));
3587
+ }
3588
+ }
3409
3589
  /** Worker-health for `group`, merged across every transport that can report it (a group's queue
3410
3590
  * and its workers may live on any pinned transport). Undefined when no transport implements
3411
3591
  * `groupHealth` (e.g. a pure in-process transport — nothing to introspect). */
@@ -4625,6 +4805,15 @@ var PARENT_NOTIFY_BACKOFF_MS = 250;
4625
4805
  var LEASE_RECHECK_MAX_MS = 2e3;
4626
4806
  var LEASE_RECHECK_SLACK_MS = 100;
4627
4807
  var LEASE_RECHECK_ATTEMPTS = 30;
4808
+ function wakeSeen(request, settledFans, served) {
4809
+ if (request.id !== void 0 && served.has(request.id)) return true;
4810
+ return isFanWakeRequest(request.payload) && request.payload.fans.every(([first, seq]) => (settledFans.get(first) ?? -1) >= seq);
4811
+ }
4812
+ __name(wakeSeen, "wakeSeen");
4813
+ function isFanWakeRequest(payload) {
4814
+ return typeof payload === "object" && payload !== null && Array.isArray(payload.fans);
4815
+ }
4816
+ __name(isFanWakeRequest, "isFanWakeRequest");
4628
4817
  function deferred() {
4629
4818
  let resolve;
4630
4819
  let reject;
@@ -4643,12 +4832,52 @@ __name(deferred, "deferred");
4643
4832
  var pendingStateKey = /* @__PURE__ */ __name((at) => String(new Date(at).getTime()), "pendingStateKey");
4644
4833
  var freshPendingKey = /* @__PURE__ */ __name(() => `${Date.now()}-${globalThis.crypto.randomUUID().slice(0, 8)}`, "freshPendingKey");
4645
4834
  var wakeRequestToken = /* @__PURE__ */ __name((runId) => `durable:wake-after-turn:${runId}`, "wakeRequestToken");
4835
+ var WAKE_REQUEST_SCAN = 100;
4646
4836
  var CHILD_SIGNAL_PREFIX = "signal:child:";
4647
4837
  var BUFFERED_SWEEP_PAGE = 500;
4648
4838
  var BUFFERED_SWEEP_PAGES_PER_CALL = 4;
4839
+ var BUFFERED_SWEEP_BUDGET_MS = 250;
4649
4840
  var BUFFERED_SWEEP_REST_MS = 10 * 6e4;
4650
4841
  var DELIVER_CHUNK = 50;
4651
4842
  var RESULT_BATCH_MAX = 256;
4843
+ function isResultError(outcome) {
4844
+ return typeof outcome === "object" && "error" in outcome;
4845
+ }
4846
+ __name(isResultError, "isResultError");
4847
+ var FanWakeCollector = class FanWakeCollector2 {
4848
+ static {
4849
+ __name(this, "FanWakeCollector");
4850
+ }
4851
+ carrier;
4852
+ generic = false;
4853
+ fan;
4854
+ /** The highest seq the batch settled, per fan. */
4855
+ highest = /* @__PURE__ */ new Map();
4856
+ /** A checkpoint the batch settled. */
4857
+ note(cp) {
4858
+ const fan = parseStepFanGroup(cp.parallelGroup);
4859
+ if (fan) this.highest.set(fan.firstSeq, Math.max(this.highest.get(fan.firstSeq) ?? cp.seq, cp.seq));
4860
+ }
4861
+ /** A settled checkpoint that resumes the run. */
4862
+ add(index, cp, status) {
4863
+ this.carrier = index;
4864
+ const fan = parseStepFanGroup(cp.parallelGroup);
4865
+ if (!fan || status === "failed" || this.fan !== void 0 && this.fan !== fan.firstSeq) {
4866
+ this.generic = true;
4867
+ return;
4868
+ }
4869
+ this.fan = fan.firstSeq;
4870
+ }
4871
+ applyTo(outcomes) {
4872
+ if (this.carrier === void 0) return outcomes;
4873
+ const seq = this.fan === void 0 ? void 0 : this.highest.get(this.fan);
4874
+ outcomes[this.carrier] = this.generic || this.fan === void 0 || seq === void 0 ? true : {
4875
+ first: this.fan,
4876
+ seq
4877
+ };
4878
+ return outcomes;
4879
+ }
4880
+ };
4652
4881
  function shuffle(items) {
4653
4882
  for (let i = items.length - 1; i > 0; i -= 1) {
4654
4883
  const j = Math.floor(Math.random() * (i + 1));
@@ -4837,6 +5066,23 @@ var WorkflowEngine = class {
4837
5066
  /** Re-checks scheduled for runs whose wake found another instance's lease (see {@link execute}). */
4838
5067
  leaseRetries = /* @__PURE__ */ new Map();
4839
5068
  /**
5069
+ * Runs whose scheduled re-check resumes them whatever happened meanwhile. Without an entry the
5070
+ * re-check only stood for fan wakes kept as requests, and resumes the run only if a request is
5071
+ * still there: the turn that held the lease takes them once it ends (see {@link retryDroppedWake}).
5072
+ */
5073
+ unconditionalRechecks = /* @__PURE__ */ new Set();
5074
+ /** Runs a generic wake request was written for in the current re-check cycle (see {@link execute}). */
5075
+ genericWakeRequested = /* @__PURE__ */ new Set();
5076
+ /** Why each run is being woken on this instance, until the turn that answers it starts. */
5077
+ wakeReasons = /* @__PURE__ */ new Map();
5078
+ /**
5079
+ * The step fans each executing turn on this instance read with nothing in flight: first seq -> the
5080
+ * highest seq read. Written by {@link callRemoteAll}, read once the turn ends.
5081
+ */
5082
+ turnSettledFans = /* @__PURE__ */ new Map();
5083
+ /** The wake requests each executing turn on this instance serves (see {@link noteServedWakes}). */
5084
+ turnServedWakes = /* @__PURE__ */ new Map();
5085
+ /**
4840
5086
  * Every run turn and child start this instance takes on itself, under one budget of
4841
5087
  * {@link LOCAL_RUN_CONCURRENCY}: the runs it dispatches to itself, a fan's deferred child starts,
4842
5088
  * enqueued pickups and the runs it takes off the transport's run queue.
@@ -4960,8 +5206,9 @@ var WorkflowEngine = class {
4960
5206
  }
4961
5207
  return;
4962
5208
  }
4963
- if (!await this.completeRemoteResult(result)) return;
4964
- await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId).then(() => void 0, (err) => {
5209
+ const wake = await this.completeRemoteResult(result);
5210
+ if (!wake) return;
5211
+ await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId, wake === true ? void 0 : wake).then(() => void 0, (err) => {
4965
5212
  if (err instanceof NamespaceMismatch) return;
4966
5213
  throw err;
4967
5214
  }));
@@ -5178,7 +5425,10 @@ var WorkflowEngine = class {
5178
5425
  const resumed = await this.resumeLeased([
5179
5426
  run
5180
5427
  ], nowMs, (locked) => this.reconcileSignalWaiter(locked, waiters.get(locked.id)));
5181
- if (resumed.length === 0) this.recheckAfterLease(runId, run.lockedUntil);
5428
+ if (resumed.length === 0) {
5429
+ this.unconditionalRechecks.add(runId);
5430
+ this.recheckAfterLease(runId, run.lockedUntil);
5431
+ }
5182
5432
  return;
5183
5433
  }
5184
5434
  await this.resumeImplicit(runId).catch((err) => {
@@ -5479,13 +5729,17 @@ var WorkflowEngine = class {
5479
5729
  *
5480
5730
  * Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
5481
5731
  * at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
5482
- * few timer ticks without a long query or a big delete. A pass that reaches the end of the table
5483
- * rests {@link BUFFERED_SWEEP_REST_MS} before the next one. Call it every tick (the NestJS timer
5484
- * poller does). Skipped while draining, on a namespaced engine (a buffered signal carries no
5485
- * namespace, and a scoped read would report another tenant's parent as gone), and on a store
5486
- * without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
5487
- */
5488
- async sweepBufferedSignals(now = this.clock()) {
5732
+ * few timer ticks without a long query or a big delete. Each page is decided set-based (see
5733
+ * `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
5734
+ * read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
5735
+ * {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
5736
+ * call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
5737
+ * before the next one. Call it every tick (the NestJS timer poller does); a call made while
5738
+ * another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
5739
+ * carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
5740
+ * store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
5741
+ */
5742
+ async sweepBufferedSignals(now = this.clock(), budgetMs = BUFFERED_SWEEP_BUDGET_MS) {
5489
5743
  const { listBufferedSignals, deleteBufferedSignals } = this.store;
5490
5744
  if (!listBufferedSignals || !deleteBufferedSignals) return 0;
5491
5745
  if (this.draining || this.namespace !== void 0 || this.bufferedSweepActive) return 0;
@@ -5493,15 +5747,21 @@ var WorkflowEngine = class {
5493
5747
  return 0;
5494
5748
  }
5495
5749
  this.bufferedSweepActive = true;
5750
+ const deadline = Date.now() + budgetMs;
5496
5751
  let deleted = 0;
5497
5752
  try {
5498
5753
  for (let page = 0; page < BUFFERED_SWEEP_PAGES_PER_CALL; page++) {
5754
+ if (page > 0 && (Date.now() >= deadline || this.draining)) break;
5499
5755
  const rows = await listBufferedSignals.call(this.store, CHILD_TOKEN_PREFIX, this.bufferedSweepCursor, BUFFERED_SWEEP_PAGE);
5500
- const doomed = await unneededChildSignals(this.store, rows);
5756
+ const { doomed, decided } = await unneededChildSignals(this.store, rows, deadline);
5501
5757
  if (doomed.length > 0) {
5502
5758
  await deleteBufferedSignals.call(this.store, doomed.map((row) => row.id));
5503
5759
  deleted += doomed.length;
5504
5760
  }
5761
+ if (decided < rows.length) {
5762
+ if (decided > 0) this.bufferedSweepCursor = rows[decided - 1].id;
5763
+ break;
5764
+ }
5505
5765
  const last = rows.at(-1);
5506
5766
  if (rows.length < BUFFERED_SWEEP_PAGE || !last) {
5507
5767
  this.bufferedSweepCursor = void 0;
@@ -5745,15 +6005,27 @@ var WorkflowEngine = class {
5745
6005
  * terminal too.
5746
6006
  */
5747
6007
  resume(runId) {
6008
+ this.noteWakeReason(runId, void 0);
5748
6009
  return this.resumeRun(runId, true);
5749
6010
  }
6011
+ /** Record why `runId` is being woken: a step fan's barrier opening, or (`undefined`) anything else. */
6012
+ noteWakeReason(runId, fan) {
6013
+ const reason = this.wakeReasons.get(runId) ?? {
6014
+ generic: false,
6015
+ fans: /* @__PURE__ */ new Map()
6016
+ };
6017
+ if (fan) reason.fans.set(fan.first, Math.max(reason.fans.get(fan.first) ?? fan.seq, fan.seq));
6018
+ else reason.generic = true;
6019
+ this.wakeReasons.set(runId, reason);
6020
+ }
5750
6021
  /**
5751
6022
  * {@link resume} for every IMPLICIT wake-up — a delivered signal/child completion, a late step
5752
6023
  * result, a due timer, a redelivered dispatch, a queue-slot wake. A `failed` run is terminal to
5753
6024
  * these: only an explicit retry ({@link requeue} / {@link resume}) may re-drive it. Otherwise a
5754
6025
  * stale waiter or a late result left behind by a failed run silently re-executes its body.
5755
6026
  */
5756
- resumeImplicit(runId) {
6027
+ resumeImplicit(runId, fan) {
6028
+ if (fan !== "coalesced") this.noteWakeReason(runId, fan);
5757
6029
  const turn = this.wakeTurns.get(runId);
5758
6030
  if (turn) {
5759
6031
  turn.again ??= deferred();
@@ -5778,7 +6050,7 @@ var WorkflowEngine = class {
5778
6050
  });
5779
6051
  return;
5780
6052
  }
5781
- const next = this.resumeImplicit(runId);
6053
+ const next = this.resumeImplicit(runId, "coalesced");
5782
6054
  next.then(again.resolve, again.reject);
5783
6055
  this.inBackground(next);
5784
6056
  }, "finish");
@@ -7843,9 +8115,26 @@ var WorkflowEngine = class {
7843
8115
  async execute(run, fn) {
7844
8116
  const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
7845
8117
  const lockNow = this.clock();
8118
+ const reason = this.wakeReasons.get(run.id) ?? {
8119
+ generic: true,
8120
+ fans: /* @__PURE__ */ new Map()
8121
+ };
8122
+ this.wakeReasons.delete(run.id);
7846
8123
  if (run.lockedBy !== this.instanceId || (run.lockedUntil ?? 0) <= lockNow) {
7847
8124
  if (!await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow)) {
7848
- if (!this.leaseRetries.has(run.id)) this.inBackground(this.requestWakeAfterTurn(run.id));
8125
+ if (reason.generic) {
8126
+ if (!this.genericWakeRequested.has(run.id)) {
8127
+ this.genericWakeRequested.add(run.id);
8128
+ this.inBackground(this.requestWakeAfterTurn(run.id, null));
8129
+ }
8130
+ } else {
8131
+ const fans = {
8132
+ fans: [
8133
+ ...reason.fans
8134
+ ]
8135
+ };
8136
+ this.inBackground(this.requestWakeAfterTurn(run.id, fans));
8137
+ }
7849
8138
  this.recheckAfterLease(run.id, run.lockedUntil);
7850
8139
  return {
7851
8140
  runId: run.id,
@@ -7853,56 +8142,142 @@ var WorkflowEngine = class {
7853
8142
  };
7854
8143
  }
7855
8144
  }
8145
+ const settledFans = /* @__PURE__ */ new Map();
8146
+ this.turnSettledFans.set(run.id, settledFans);
7856
8147
  const held = this.holdExecutionLease(run.id);
7857
8148
  try {
8149
+ const served = /* @__PURE__ */ new Set();
8150
+ this.turnServedWakes.set(run.id, served);
8151
+ if (registered?.remote) await this.noteServedWakes(run.id);
7858
8152
  const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
7859
8153
  if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
7860
8154
  void this.singletons.wakeNext(run).catch(() => void 0);
7861
8155
  }
7862
- if (result.status === "suspended") this.inBackground(this.honorWakeRequests(run.id));
7863
- else if (TERMINAL_RUN_STATUSES.includes(result.status)) {
7864
- this.inBackground(this.takeWakeRequests(run.id));
8156
+ if (result.status === "suspended") {
8157
+ this.inBackground(this.honorWakeRequests(run.id, settledFans, served));
8158
+ } else if (TERMINAL_RUN_STATUSES.includes(result.status)) {
8159
+ this.inBackground(this.listWakeRequests(run.id, true).then((kept) => this.dropWakeRequests(kept)));
7865
8160
  }
7866
8161
  return result;
7867
8162
  } finally {
7868
8163
  held.stop();
8164
+ if (this.turnSettledFans.get(run.id) === settledFans) this.turnSettledFans.delete(run.id);
8165
+ this.turnServedWakes.delete(run.id);
7869
8166
  }
7870
8167
  }
7871
8168
  /**
7872
8169
  * Keep a wake that found `runId`'s lease held by a turn executing elsewhere, durably, so that turn
7873
8170
  * runs again once it ends instead of suspending on a snapshot that predates what the wake brought
7874
- * (a signal, a child's completion). The request is a buffered signal on a token no workflow waits
7875
- * on; the holder takes it after it releases the lease ({@link honorWakeRequests}).
8171
+ * (a signal, a child's completion). The request is a buffered event under a name no workflow waits
8172
+ * on. It stays until a turn that started after it has settled: the holder looks at it after it
8173
+ * releases the lease ({@link honorWakeRequests}), and the turn that serves it drops it.
7876
8174
  *
7877
8175
  * Written first and looked at second, against a holder that releases first and looks second:
7878
8176
  * whichever look runs last sees the other side's write. So when the run is no longer executing by
7879
- * the time this re-reads it, the holder may have looked already, and this side takes the request
7880
- * and wakes the run itself once the lease is free.
8177
+ * the time this re-reads it, the holder may have looked already, and this side wakes the run
8178
+ * itself once the lease is free ({@link retryDroppedWake}). While the holder still runs, the run is
8179
+ * made due when the holder's lease runs out, so a holder that goes down before it settles leaves
8180
+ * the run to any instance's timer poll; a holder that settles sets the run's timer itself.
7881
8181
  */
7882
- async requestWakeAfterTurn(runId) {
7883
- await this.store.bufferSignal(wakeRequestToken(runId), null);
8182
+ async requestWakeAfterTurn(runId, request) {
8183
+ const id = globalThis.crypto.randomUUID();
8184
+ await this.store.bufferEvent({
8185
+ name: wakeRequestToken(runId),
8186
+ payload: request,
8187
+ id,
8188
+ publishedAt: this.clock()
8189
+ });
7884
8190
  const run = await this.store.getRun(runId);
7885
- if (run?.status === "running") return;
7886
- if (!run) {
7887
- await this.takeWakeRequests(runId);
8191
+ if (!run || TERMINAL_RUN_STATUSES.includes(run.status)) {
8192
+ await this.store.removeBufferedEvent(id);
8193
+ return;
8194
+ }
8195
+ if (run.status === "running") return;
8196
+ if (run.lockedBy !== void 0 && run.lockedBy !== this.instanceId) {
8197
+ await this.makeDueBy(runId, (run.lockedUntil ?? this.clock()) + LEASE_RECHECK_SLACK_MS);
7888
8198
  return;
7889
8199
  }
7890
- if (!await this.takeWakeRequests(runId)) return;
7891
- if (TERMINAL_RUN_STATUSES.includes(run.status)) return;
7892
8200
  this.recheckAfterLease(runId, run.lockedUntil);
7893
8201
  }
7894
- /** After a turn that suspended released its lease: run again if a wake was kept for it. */
7895
- async honorWakeRequests(runId) {
7896
- if (!await this.takeWakeRequests(runId)) return;
8202
+ /**
8203
+ * After a turn that suspended released its lease: drop the wakes it served, and run again if a
8204
+ * wake was kept for it meanwhile, unless every such wake was a fan wake this turn already saw: it
8205
+ * read that fan with nothing in flight, at least as far as the batch that opened its barrier.
8206
+ * The run is made due before it is woken, so if this instance goes down before that turn runs,
8207
+ * the timer poll of another instance runs it once this instance's hold on it would have lapsed.
8208
+ */
8209
+ async honorWakeRequests(runId, settledFans, served) {
8210
+ const kept = await this.listWakeRequests(runId, true);
8211
+ const seen = /* @__PURE__ */ __name((request) => wakeSeen(request, settledFans, served), "seen");
8212
+ const needed = kept.filter((request) => !seen(request));
8213
+ await this.dropWakeRequests(kept.filter(seen));
8214
+ if (needed.length === 0) return;
8215
+ await this.makeDueBy(runId, this.clock() + this.leaseMs);
7897
8216
  await this.wake(runId).catch((err) => {
7898
8217
  if (!(err instanceof NamespaceMismatch)) throw err;
7899
8218
  });
7900
8219
  }
7901
- /** Take every wake request kept for `runId`; true when there was one. */
7902
- async takeWakeRequests(runId) {
7903
- let taken = false;
7904
- while (await this.store.takeBufferedSignal(wakeRequestToken(runId))) taken = true;
7905
- return taken;
8220
+ /**
8221
+ * The wake requests kept for `runId`. With `legacy`, also those an older engine kept as buffered
8222
+ * signals, which are taken as they are read.
8223
+ */
8224
+ async listWakeRequests(runId, legacy = false) {
8225
+ const token = wakeRequestToken(runId);
8226
+ const kept = (await this.store.listBufferedEvents(token, WAKE_REQUEST_SCAN)).map((request) => ({
8227
+ id: request.id,
8228
+ payload: request.payload
8229
+ }));
8230
+ if (!legacy) return kept;
8231
+ for (; ; ) {
8232
+ const signal = await this.store.takeBufferedSignal(token);
8233
+ if (!signal) return kept;
8234
+ kept.push({
8235
+ payload: signal.payload
8236
+ });
8237
+ }
8238
+ }
8239
+ /** Delete `requests` (the ones kept as buffered events). */
8240
+ async dropWakeRequests(requests) {
8241
+ await Promise.all(requests.flatMap((request) => request.id === void 0 ? [] : [
8242
+ this.store.removeBufferedEvent(request.id)
8243
+ ]));
8244
+ }
8245
+ /**
8246
+ * Record, for the turn executing `runId` here, the wakes kept for the run so far. Called before the
8247
+ * turn reads the run's state, so the turn serves them: they are dropped once it has settled, and
8248
+ * until then stay for whoever runs the run next.
8249
+ */
8250
+ async noteServedWakes(runId) {
8251
+ const served = this.turnServedWakes.get(runId);
8252
+ if (!served) return;
8253
+ for (const request of await this.store.listBufferedEvents(wakeRequestToken(runId), WAKE_REQUEST_SCAN)) {
8254
+ served.add(request.id);
8255
+ }
8256
+ }
8257
+ /**
8258
+ * Right after a turn of `runId` wrote its suspension, while it still holds the lease: if a wake
8259
+ * was kept for the run during the turn that the turn did not see, keep the run due when the lease
8260
+ * runs out. The suspension overwrote the timer the instance keeping the wake set, and this instance
8261
+ * may go down before it looks at the wake once it has released the lease.
8262
+ */
8263
+ async keepDueForKeptWakes(runId) {
8264
+ const served = this.turnServedWakes.get(runId);
8265
+ if (!served) return;
8266
+ const settledFans = this.turnSettledFans.get(runId) ?? /* @__PURE__ */ new Map();
8267
+ const kept = await this.listWakeRequests(runId);
8268
+ if (kept.every((request) => wakeSeen(request, settledFans, served))) return;
8269
+ await this.makeDueBy(runId, this.clock() + this.leaseMs);
8270
+ }
8271
+ /** Bring a suspended run's timer forward to `at` (never later than the one it has). */
8272
+ async makeDueBy(runId, at) {
8273
+ const run = await this.store.getRun(runId);
8274
+ if (run?.status !== "suspended") return;
8275
+ if (run.wakeAt !== void 0 && run.wakeAt <= at) return;
8276
+ await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, runId, [
8277
+ "suspended"
8278
+ ], {
8279
+ wakeAt: at
8280
+ });
7906
8281
  }
7907
8282
  /**
7908
8283
  * A wake that found the run leased by another instance is dropped, and normally that is fine: the
@@ -7921,6 +8296,7 @@ var WorkflowEngine = class {
7921
8296
  const remaining = Math.max(0, (lockedUntil ?? this.clock()) - this.clock());
7922
8297
  const timer = setTimeout(() => {
7923
8298
  this.leaseRetries.delete(runId);
8299
+ this.genericWakeRequested.delete(runId);
7924
8300
  this.inBackground(this.retryDroppedWake(runId, attempt));
7925
8301
  }, Math.min(remaining, LEASE_RECHECK_MAX_MS) + LEASE_RECHECK_SLACK_MS);
7926
8302
  timer.unref?.();
@@ -7929,12 +8305,15 @@ var WorkflowEngine = class {
7929
8305
  async retryDroppedWake(runId, attempt) {
7930
8306
  if (this.draining) return;
7931
8307
  const run = await this.store.getRun(runId);
7932
- if (run?.status !== "suspended") return;
7933
8308
  const nowMs = this.clock();
7934
- if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) {
7935
- if (attempt < LEASE_RECHECK_ATTEMPTS) this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
8309
+ if (run?.status === "suspended" && run.lockedUntil !== void 0 && run.lockedUntil > nowMs && attempt < LEASE_RECHECK_ATTEMPTS) {
8310
+ this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
7936
8311
  return;
7937
8312
  }
8313
+ const unconditional = this.unconditionalRechecks.delete(runId);
8314
+ if (run?.status !== "suspended") return;
8315
+ if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) return;
8316
+ if (!unconditional && (await this.listWakeRequests(runId)).length === 0) return;
7938
8317
  if (this.queuesRunsOverTransport) {
7939
8318
  await this.wake(runId);
7940
8319
  return;
@@ -8273,6 +8652,7 @@ ${run.workflowVersion}`;
8273
8652
  updatedAt
8274
8653
  });
8275
8654
  if (!applied) return this.echoCurrentStatus(run.id);
8655
+ await this.keepDueForKeptWakes(run.id);
8276
8656
  this.emit({
8277
8657
  type: "run.suspended",
8278
8658
  runId: run.id,
@@ -8966,6 +9346,7 @@ ${run.workflowVersion}`;
8966
9346
  namespace: run.namespace
8967
9347
  });
8968
9348
  }
9349
+ await this.noteServedWakes(run.id);
8969
9350
  if (run.status !== "cancelling" && registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
8970
9351
  return this.singletonWaitResult(run);
8971
9352
  }
@@ -9639,6 +10020,14 @@ ${run.workflowVersion}`;
9639
10020
  throw new WorkflowSuspended();
9640
10021
  }
9641
10022
  /**
10023
+ * Whether a `pending` step a turn just read only waits for its result: {@link callRemote} would
10024
+ * suspend on it without writing (no redispatch deadline to stamp, the one stamped not yet due).
10025
+ */
10026
+ stepIsWaiting(cp) {
10027
+ if (this.remoteRedispatchMs == null) return true;
10028
+ return cp.wakeAt != null && this.clock() < cp.wakeAt;
10029
+ }
10030
+ /**
9642
10031
  * `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
9643
10032
  * reserves the whole position block up front, so replay re-derives the same seqs.
9644
10033
  *
@@ -9668,6 +10057,7 @@ ${run.workflowVersion}`;
9668
10057
  const failures = [];
9669
10058
  const unsent = [];
9670
10059
  let inFlight = 0;
10060
+ let highest = -1;
9671
10061
  let wakeAt;
9672
10062
  for (let i = 0; i < inputs.length; i += 1) {
9673
10063
  const seq = positions[i];
@@ -9677,10 +10067,19 @@ ${run.workflowVersion}`;
9677
10067
  continue;
9678
10068
  }
9679
10069
  if (cp.name !== step.name) throw new NonDeterminismError(runId, seq, step.name, cp.name);
10070
+ if (seq > highest) highest = seq;
9680
10071
  if (cp.status === "completed") {
9681
10072
  outputs[i] = cp.output;
9682
10073
  continue;
9683
10074
  }
10075
+ if (cp.status === "pending" && this.stepIsWaiting(cp)) {
10076
+ inFlight += 1;
10077
+ if (cp.wakeAt != null) {
10078
+ const at = this.stepLeaseWakeAt(cp.wakeAt);
10079
+ wakeAt = Math.min(wakeAt ?? at, at);
10080
+ }
10081
+ continue;
10082
+ }
9684
10083
  try {
9685
10084
  outputs[i] = await this.callRemote(runId, seq, step, inputs[i], void 0, opts.transport, void 0, opts.admission, {
9686
10085
  ...routing,
@@ -9708,6 +10107,7 @@ ${run.workflowVersion}`;
9708
10107
  this.awaitedRemoteSteps.get(runId)?.delete(seq);
9709
10108
  }
9710
10109
  }
10110
+ if (inFlight === 0 && highest >= 0) this.turnSettledFans.get(runId)?.set(first, highest);
9711
10111
  const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
9712
10112
  if (batch.length > 0) await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
9713
10113
  if (inFlight + batch.length > 0) {
@@ -9738,53 +10138,62 @@ ${run.workflowVersion}`;
9738
10138
  if (failures.length > 0) throw new GatherError(failures, inputs.length);
9739
10139
  return outputs;
9740
10140
  }
9741
- /** Write the `pending` checkpoints of a step fan's new steps, then dispatch them. */
10141
+ /**
10142
+ * Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
10143
+ * {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
10144
+ * batch, where the store and transport have them.
10145
+ */
9742
10146
  async sendStepFan(runId, positions, step, inputs, batch, group, opts, routing) {
9743
10147
  const workerGroup = stepGroup(step, routing?.namespace);
9744
10148
  await this.ensureRoutable(workerGroup, step.requires);
9745
10149
  const enqueuedAt = /* @__PURE__ */ new Date();
9746
- for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
9747
- await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map((i) => {
9748
- const seq = positions[i];
9749
- return this.store.saveCheckpoint({
9750
- runId,
9751
- seq,
9752
- name: step.name,
9753
- kind: "remote",
9754
- stepId: stepId(runId, seq),
9755
- status: "pending",
9756
- input: inputs[i],
9757
- attempts: 1,
9758
- workerGroup,
9759
- parallelGroup: group,
9760
- enqueuedAt,
9761
- startedAt: enqueuedAt,
9762
- finishedAt: enqueuedAt
9763
- });
9764
- }));
10150
+ const checkpoints = batch.map((i) => {
10151
+ const seq = positions[i];
10152
+ return {
10153
+ runId,
10154
+ seq,
10155
+ name: step.name,
10156
+ kind: "remote",
10157
+ stepId: stepId(runId, seq),
10158
+ status: "pending",
10159
+ input: inputs[i],
10160
+ attempts: 1,
10161
+ workerGroup,
10162
+ parallelGroup: group,
10163
+ enqueuedAt,
10164
+ startedAt: enqueuedAt,
10165
+ finishedAt: enqueuedAt
10166
+ };
10167
+ });
10168
+ if (this.store.saveCheckpoints) {
10169
+ await this.store.saveCheckpoints(checkpoints);
10170
+ } else {
10171
+ for (let at = 0; at < checkpoints.length; at += STEP_FAN_CHUNK) {
10172
+ await Promise.all(checkpoints.slice(at, at + STEP_FAN_CHUNK).map((cp) => this.store.saveCheckpoint(cp)));
10173
+ }
9765
10174
  }
9766
- for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
9767
- await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map(async (i) => {
9768
- const seq = positions[i];
9769
- await this.dispatchRemoteTask({
9770
- runId,
9771
- seq,
9772
- name: step.name,
9773
- stepId: stepId(runId, seq),
9774
- group: workerGroup,
9775
- input: inputs[i],
9776
- priority: opts.admission?.priority,
9777
- attempt: 1,
9778
- transport: opts.transport
9779
- });
9780
- this.emit({
9781
- type: "step.started",
9782
- runId,
9783
- seq,
9784
- name: step.name,
9785
- kind: "remote"
9786
- });
9787
- }));
10175
+ const traceparent = this.traceparent?.();
10176
+ const context = this.context?.();
10177
+ await this.pool.dispatchMany(checkpoints.map((cp, k) => ({
10178
+ runId,
10179
+ seq: cp.seq,
10180
+ name: step.name,
10181
+ stepId: cp.stepId,
10182
+ group: workerGroup,
10183
+ input: inputs[batch[k]],
10184
+ traceparent,
10185
+ context,
10186
+ priority: opts.admission?.priority,
10187
+ attempt: 1
10188
+ })), opts.transport);
10189
+ for (const cp of checkpoints) {
10190
+ this.emit({
10191
+ type: "step.started",
10192
+ runId,
10193
+ seq: cp.seq,
10194
+ name: step.name,
10195
+ kind: "remote"
10196
+ });
9788
10197
  }
9789
10198
  }
9790
10199
  /** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
@@ -9825,6 +10234,27 @@ ${run.workflowVersion}`;
9825
10234
  error: s.reason
9826
10235
  });
9827
10236
  }
10237
+ /**
10238
+ * The checkpoints of one run at distinct `seqs`, each as its own outcome: one
10239
+ * {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
10240
+ * call rejecting falls back to one read each, so a failing read fails only its own entry.
10241
+ */
10242
+ async readCheckpoints(runId, seqs) {
10243
+ if (seqs.length > 1 && this.store.getCheckpoints) {
10244
+ try {
10245
+ const bySeq = new Map((await this.store.getCheckpoints(runId, seqs)).map((cp) => [
10246
+ cp.seq,
10247
+ cp
10248
+ ]));
10249
+ return seqs.map((seq) => ({
10250
+ status: "fulfilled",
10251
+ value: bySeq.get(seq) ?? null
10252
+ }));
10253
+ } catch {
10254
+ }
10255
+ }
10256
+ return Promise.allSettled(seqs.map((seq) => this.store.getCheckpoint(runId, seq)));
10257
+ }
9828
10258
  /** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
9829
10259
  * dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
9830
10260
  * {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
@@ -9866,9 +10296,10 @@ ${run.workflowVersion}`;
9866
10296
  *
9867
10297
  * Results of one run are settled in batches: the first starts a batch, and the ones that arrive
9868
10298
  * while it is being written form the next. Under load a wide fan's results are written by one
9869
- * {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked once
9870
- * per batch, not once per result. A result resolves only once its own write has committed, so the
9871
- * transport acknowledges nothing that is not durable yet.
10299
+ * {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked only
10300
+ * by a batch with no more of the fan's results queued behind it. A result resolves only once its
10301
+ * own write has committed, so the transport acknowledges nothing that is not durable yet. How
10302
+ * large a batch gets is bounded by how many results the transport hands over at once.
9872
10303
  */
9873
10304
  completeRemoteResult(result) {
9874
10305
  return new Promise((resolve, reject) => {
@@ -9896,7 +10327,7 @@ ${run.workflowVersion}`;
9896
10327
  const batch = queue.splice(0, RESULT_BATCH_MAX);
9897
10328
  let outcomes;
9898
10329
  try {
9899
- outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result));
10330
+ outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result), (from, to) => queue.some(({ result }) => result.seq >= from && result.seq <= to));
9900
10331
  } catch (err) {
9901
10332
  outcomes = batch.map(() => ({
9902
10333
  error: err
@@ -9904,8 +10335,8 @@ ${run.workflowVersion}`;
9904
10335
  }
9905
10336
  batch.forEach((q, i) => {
9906
10337
  const outcome = outcomes[i] ?? false;
9907
- if (typeof outcome === "boolean") q.resolve(outcome);
9908
- else q.reject(outcome.error);
10338
+ if (isResultError(outcome)) q.reject(outcome.error);
10339
+ else q.resolve(outcome);
9909
10340
  });
9910
10341
  }
9911
10342
  this.resultQueues.delete(runId);
@@ -9914,8 +10345,13 @@ ${run.workflowVersion}`;
9914
10345
  * Settle a batch of one run's step results. At most one of them resolves `true` (resume the run):
9915
10346
  * that resume replays every write of the batch, so a second would only replay it again. A result
9916
10347
  * whose read or write failed resolves its error, and the transport redelivers just that one.
10348
+ *
10349
+ * `queuedIn(from, to)` says whether a result for a seq in that range is already waiting for this
10350
+ * run's next batch on this instance. A fan with one waiting is not asked its barrier: that result
10351
+ * asks it once its own batch is written (as a settle, or as a redelivery of a settled step), so
10352
+ * under a backlog the barrier is read once per drained queue rather than once per batch.
9917
10353
  */
9918
- async completeRemoteResults(runId, results) {
10354
+ async completeRemoteResults(runId, results, queuedIn = () => false) {
9919
10355
  const outcomes = results.map(() => false);
9920
10356
  const seen = /* @__PURE__ */ new Set();
9921
10357
  const live = results.flatMap((r, i) => {
@@ -9925,7 +10361,7 @@ ${run.workflowVersion}`;
9925
10361
  i
9926
10362
  ];
9927
10363
  });
9928
- const reads = await Promise.allSettled(live.map((i) => this.store.getCheckpoint(runId, results[i].seq)));
10364
+ const reads = await this.readCheckpoints(runId, live.map((i) => results[i].seq));
9929
10365
  const pending = [];
9930
10366
  const resettled = [];
9931
10367
  reads.forEach((read, k) => {
@@ -9946,7 +10382,7 @@ ${run.workflowVersion}`;
9946
10382
  });
9947
10383
  if (pending.length === 0) {
9948
10384
  if (resettled.length === 0) return outcomes;
9949
- return this.resumeForRedelivered(runId, resettled, outcomes);
10385
+ return this.resumeForRedelivered(runId, resettled, outcomes, queuedIn);
9950
10386
  }
9951
10387
  for (const { cp } of pending) void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
9952
10388
  const run = await this.store.getRun(runId);
@@ -9973,7 +10409,9 @@ ${run.workflowVersion}`;
9973
10409
  cp,
9974
10410
  result: results[index]
9975
10411
  });
9976
- else if (claim !== false) outcomes[index] = claim;
10412
+ else if (claim !== false) outcomes[index] = {
10413
+ error: claim.error
10414
+ };
9977
10415
  });
9978
10416
  for (const { cp, result } of settled) {
9979
10417
  this.noteSettledHere(`${runId}:${result.seq}`);
@@ -9991,19 +10429,21 @@ ${run.workflowVersion}`;
9991
10429
  });
9992
10430
  }
9993
10431
  const barriers = /* @__PURE__ */ new Map();
9994
- let carrier;
10432
+ const wake = new FanWakeCollector();
10433
+ for (const { cp } of settled) wake.note(cp);
9995
10434
  for (const { index, cp, result } of settled) {
9996
- if (!await this.stepFanStillRunning(cp, result, barriers)) carrier = index;
10435
+ if (!await this.stepFanStillRunning(cp, result, barriers, queuedIn)) {
10436
+ wake.add(index, cp, result.status);
10437
+ }
9997
10438
  }
9998
- if (carrier !== void 0) outcomes[carrier] = true;
9999
- return outcomes;
10439
+ return wake.applyTo(outcomes);
10000
10440
  }
10001
10441
  /**
10002
10442
  * The outcomes for a batch of redelivered results (their steps already settled): one of them
10003
10443
  * resumes the run if it is `suspended` and, for a step of a fan, the fan's barrier is open. A
10004
10444
  * resume of a run that had in fact moved on replays to its current wait and re-parks.
10005
10445
  */
10006
- async resumeForRedelivered(runId, resettled, outcomes) {
10446
+ async resumeForRedelivered(runId, resettled, outcomes, queuedIn) {
10007
10447
  const run = await this.store.getRun(runId);
10008
10448
  if (run?.status !== "suspended") return outcomes;
10009
10449
  const barriers = /* @__PURE__ */ new Map();
@@ -10014,9 +10454,11 @@ ${run.workflowVersion}`;
10014
10454
  stepId: cp.stepId ?? stepId(runId, cp.seq),
10015
10455
  status: cp.status === "failed" ? "failed" : "completed"
10016
10456
  };
10017
- if (!await this.stepFanStillRunning(cp, asResult, barriers)) {
10018
- outcomes[index] = true;
10019
- break;
10457
+ if (!await this.stepFanStillRunning(cp, asResult, barriers, queuedIn)) {
10458
+ const wake = new FanWakeCollector();
10459
+ for (const settled of resettled) wake.note(settled.cp);
10460
+ wake.add(index, cp, asResult.status);
10461
+ return wake.applyTo(outcomes);
10020
10462
  }
10021
10463
  }
10022
10464
  return outcomes;
@@ -10027,15 +10469,17 @@ ${run.workflowVersion}`;
10027
10469
  * retry needs the parent to schedule it). Without it every result would replay the whole parent,
10028
10470
  * which makes a fan of N steps cost O(N²) checkpoint reads. `barriers` holds the answer per fan
10029
10471
  * for the rest of a batch. A barrier that cannot be read counts as open: a spare resume replays and
10030
- * re-suspends, a lost one strands the run.
10472
+ * re-suspends, a lost one strands the run. A fan with a result waiting in `queuedIn` is left to
10473
+ * that result (see {@link completeRemoteResults}).
10031
10474
  */
10032
- async stepFanStillRunning(cp, result, barriers) {
10475
+ async stepFanStillRunning(cp, result, barriers, queuedIn) {
10033
10476
  const fan = parseStepFanGroup(cp.parallelGroup);
10034
10477
  if (!fan) return false;
10035
10478
  if (result.status === "failed") return false;
10036
10479
  let closed = barriers.get(fan.firstSeq);
10037
10480
  if (!closed) {
10038
- closed = this.stepFanPending(result.runId, fan.firstSeq, fan.firstSeq + fan.size - 1).then((pending) => pending > 0, () => false);
10481
+ const lastSeq = fan.firstSeq + fan.size - 1;
10482
+ closed = queuedIn(fan.firstSeq, lastSeq) ? Promise.resolve(true) : this.stepFanPending(result.runId, fan.firstSeq, lastSeq).then((pending) => pending > 0, () => false);
10039
10483
  barriers.set(fan.firstSeq, closed);
10040
10484
  }
10041
10485
  if (await closed) return true;
@@ -10869,6 +11313,10 @@ var CodecStateStore = class {
10869
11313
  saveCheckpointIfPending;
10870
11314
  /** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
10871
11315
  saveCheckpointsIfPending;
11316
+ /** Encodes each checkpoint written. Bound only when the inner store has it. */
11317
+ saveCheckpoints;
11318
+ /** Decodes each checkpoint read. Bound only when the inner store has it. */
11319
+ getCheckpoints;
10872
11320
  /** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
10873
11321
  countCheckpoints;
10874
11322
  /** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
@@ -10919,6 +11367,10 @@ var CodecStateStore = class {
10919
11367
  if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
10920
11368
  const saveManyIfPending = inner.saveCheckpointsIfPending;
10921
11369
  if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
11370
+ const saveMany = inner.saveCheckpoints;
11371
+ if (saveMany) this.saveCheckpoints = (cps) => saveMany.call(inner, cps.map((cp) => this.encCp(cp)));
11372
+ const getMany = inner.getCheckpoints;
11373
+ if (getMany) this.getCheckpoints = async (runId, seqs) => (await getMany.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
10922
11374
  const countCps = inner.countCheckpoints;
10923
11375
  if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
10924
11376
  const stats = inner.checkpointStats;
@@ -11555,11 +12007,30 @@ var InMemoryStateStore = class {
11555
12007
  ...cp
11556
12008
  } : null;
11557
12009
  }
12010
+ async getCheckpoints(runId, seqs) {
12011
+ return [
12012
+ ...new Set(seqs)
12013
+ ].sort((a, b) => a - b).flatMap((seq) => {
12014
+ const cp = this.checkpoints.get(this.key(runId, seq));
12015
+ return cp ? [
12016
+ {
12017
+ ...cp
12018
+ }
12019
+ ] : [];
12020
+ });
12021
+ }
11558
12022
  async saveCheckpoint(checkpoint) {
11559
12023
  this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
11560
12024
  ...checkpoint
11561
12025
  });
11562
12026
  }
12027
+ async saveCheckpoints(checkpoints) {
12028
+ for (const checkpoint of checkpoints) {
12029
+ this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
12030
+ ...checkpoint
12031
+ });
12032
+ }
12033
+ }
11563
12034
  async saveCheckpointIfPending(checkpoint) {
11564
12035
  const key = this.key(checkpoint.runId, checkpoint.seq);
11565
12036
  if (this.checkpoints.get(key)?.status !== "pending") return false;