@dudousxd/nestjs-durable-core 0.85.7 → 0.85.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -937,7 +937,6 @@ var RETRYABLE = /* @__PURE__ */ new Set([
937
937
  "failed",
938
938
  "dead"
939
939
  ]);
940
- var FAN_CHILD = /^(.+\.all\.\d+\.(?:w\d+\.)?)(\d+)$/;
941
940
  function canonicalJson(value) {
942
941
  const plain = JSON.stringify(value);
943
942
  return plain === void 0 ? "null" : sortedJson(JSON.parse(plain));
@@ -952,77 +951,235 @@ function sortedJson(value) {
952
951
  return JSON.stringify(value);
953
952
  }
954
953
  __name(sortedJson, "sortedJson");
955
- function settledNames(childId) {
954
+ var FAN_CHILD_PARTS = /^(.+\.all\.(\d+)\.)(?:w(\d+)\.)?(\d+)$/;
955
+ var SEQ_CHILD = /\.child\.(\d+)$/;
956
+ function slotOf(childId) {
956
957
  const signal = `${CHILD_SIGNAL_NAME_PREFIX}${childId}`;
957
- const fan = FAN_CHILD.exec(childId);
958
- if (!fan) return {
959
- signal
960
- };
961
- return {
962
- signal,
963
- fold: {
964
- name: `${FOLDED_PREFIX}${fan[1]}`,
965
- index: Number(fan[2])
958
+ const fan = FAN_CHILD_PARTS.exec(childId);
959
+ if (fan) {
960
+ const root = fan[1];
961
+ const wave = fan[3];
962
+ const firstSeq = Number(fan[2]);
963
+ const i = Number(fan[4]);
964
+ if (wave === void 0) {
965
+ return {
966
+ signal,
967
+ signalSeq: firstSeq + i,
968
+ fan: {
969
+ fold: `${FOLDED_PREFIX}${root}`,
970
+ firstSeq,
971
+ index: i
972
+ }
973
+ };
966
974
  }
975
+ return {
976
+ signal,
977
+ signalSeq: firstSeq + i,
978
+ fan: {
979
+ fold: `${FOLDED_PREFIX}${root}w${wave}.`,
980
+ firstSeq,
981
+ index: i,
982
+ wave: Number(wave),
983
+ waveZeroFold: `${FOLDED_PREFIX}${root}w0.`
984
+ }
985
+ };
986
+ }
987
+ const seq = SEQ_CHILD.exec(childId);
988
+ return seq ? {
989
+ signal,
990
+ signalSeq: Number(seq[1])
991
+ } : {
992
+ signal
967
993
  };
968
994
  }
969
- __name(settledNames, "settledNames");
970
- function parentHoldsChild(checkpoints, childId, completion) {
971
- const { signal, fold } = settledNames(childId);
972
- const expected = canonicalJson(completion);
973
- return checkpoints.some((cp) => {
974
- if (cp.status !== "completed") return false;
975
- if (cp.name === signal) return canonicalJson(cp.output) === expected;
976
- if (fold && cp.name === fold.name) {
977
- const { from, values } = cp.output ?? {};
978
- return typeof from === "number" && Array.isArray(values) && fold.index >= from && fold.index < from + values.length;
995
+ __name(slotOf, "slotOf");
996
+ function foldOf(cp, name) {
997
+ if (!cp || cp.status !== "completed" || cp.name !== name) return void 0;
998
+ const { from, values } = cp.output ?? {};
999
+ return typeof from === "number" && Array.isArray(values) ? {
1000
+ from,
1001
+ values
1002
+ } : void 0;
1003
+ }
1004
+ __name(foldOf, "foldOf");
1005
+ function covers(fold, index) {
1006
+ return !!fold && index >= fold.from && index < fold.from + fold.values.length;
1007
+ }
1008
+ __name(covers, "covers");
1009
+ function holdsSignal(cp, signal, expected) {
1010
+ return cp.status === "completed" && cp.name === signal && canonicalJson(cp.output) === expected;
1011
+ }
1012
+ __name(holdsSignal, "holdsSignal");
1013
+ async function checkpointsAt(store, runId, seqs) {
1014
+ const unique = [
1015
+ ...new Set(seqs)
1016
+ ];
1017
+ const bySeq = /* @__PURE__ */ new Map();
1018
+ if (unique.length === 0) return bySeq;
1019
+ const rows = store.getCheckpoints ? await store.getCheckpoints(runId, unique) : (await Promise.all(unique.map((seq) => store.getCheckpoint(runId, seq)))).filter((cp) => cp !== null);
1020
+ for (const cp of rows) bySeq.set(cp.seq, cp);
1021
+ return bySeq;
1022
+ }
1023
+ __name(checkpointsAt, "checkpointsAt");
1024
+ async function checkpointsNamed(store, runId, names) {
1025
+ const wanted = new Set(names);
1026
+ const rows = store.listCheckpointsByNamePrefix ? await store.listCheckpointsByNamePrefix(runId, [
1027
+ ...wanted
1028
+ ]) : await store.listCheckpoints(runId);
1029
+ return rows.filter((cp) => wanted.has(cp.name));
1030
+ }
1031
+ __name(checkpointsNamed, "checkpointsNamed");
1032
+ async function parentSettles(store, parentId, children) {
1033
+ const entries = children.map(({ childId, completion }) => ({
1034
+ slot: slotOf(childId),
1035
+ expected: canonicalJson(completion)
1036
+ }));
1037
+ const held = entries.map(() => void 0);
1038
+ const first = await checkpointsAt(store, parentId, entries.flatMap(({ slot }) => [
1039
+ ...slot.signalSeq === void 0 ? [] : [
1040
+ slot.signalSeq
1041
+ ],
1042
+ ...slot.fan ? [
1043
+ slot.fan.firstSeq
1044
+ ] : []
1045
+ ]));
1046
+ const waveFolds = [];
1047
+ const byName = [];
1048
+ entries.forEach(({ slot, expected }, at) => {
1049
+ const own = slot.signalSeq === void 0 ? void 0 : first.get(slot.signalSeq);
1050
+ if (own?.name === slot.signal) {
1051
+ held[at] = holdsSignal(own, slot.signal, expected);
1052
+ return;
979
1053
  }
980
- return false;
1054
+ const fan = slot.fan;
1055
+ if (fan) {
1056
+ if (covers(foldOf(own, fan.fold), fan.index)) {
1057
+ held[at] = true;
1058
+ return;
1059
+ }
1060
+ const atFirst = first.get(fan.firstSeq);
1061
+ if (fan.wave === void 0 || fan.wave === 0) {
1062
+ held[at] = covers(foldOf(atFirst, fan.fold), fan.index);
1063
+ return;
1064
+ }
1065
+ const waveZero = foldOf(atFirst, fan.waveZeroFold);
1066
+ if (waveZero && waveZero.values.length > 0) {
1067
+ waveFolds.push({
1068
+ at,
1069
+ seq: fan.firstSeq + fan.wave * waveZero.values.length
1070
+ });
1071
+ return;
1072
+ }
1073
+ byName.push(at);
1074
+ return;
1075
+ }
1076
+ if (slot.signalSeq !== void 0 && !own) {
1077
+ held[at] = false;
1078
+ return;
1079
+ }
1080
+ byName.push(at);
981
1081
  });
1082
+ if (waveFolds.length > 0) {
1083
+ const second = await checkpointsAt(store, parentId, waveFolds.map((w) => w.seq));
1084
+ for (const { at, seq } of waveFolds) {
1085
+ const fan = entries[at].slot.fan;
1086
+ held[at] = covers(foldOf(second.get(seq), fan.fold), fan.index);
1087
+ }
1088
+ }
1089
+ if (byName.length > 0) {
1090
+ const names = byName.flatMap((at) => {
1091
+ const { slot } = entries[at];
1092
+ return slot.fan ? [
1093
+ slot.signal,
1094
+ slot.fan.fold
1095
+ ] : [
1096
+ slot.signal
1097
+ ];
1098
+ });
1099
+ const named = await checkpointsNamed(store, parentId, names);
1100
+ for (const at of byName) {
1101
+ const { slot, expected } = entries[at];
1102
+ held[at] = named.some((cp) => holdsSignal(cp, slot.signal, expected) || !!slot.fan && covers(foldOf(cp, slot.fan.fold), slot.fan.index));
1103
+ }
1104
+ }
1105
+ return held.map((h) => h === true);
982
1106
  }
983
- __name(parentHoldsChild, "parentHoldsChild");
984
- async function parentCheckpoints(store, parentId, childId) {
985
- const { signal, fold } = settledNames(childId);
986
- const prefixes = fold ? [
987
- signal,
988
- fold.name
989
- ] : [
990
- signal
991
- ];
992
- return store.listCheckpointsByNamePrefix ? store.listCheckpointsByNamePrefix(parentId, prefixes) : store.listCheckpoints(parentId);
1107
+ __name(parentSettles, "parentSettles");
1108
+ function parentClosed(run) {
1109
+ return !run || TERMINAL_RUN_STATUSES.includes(run.status) && !RETRYABLE.has(run.status);
993
1110
  }
994
- __name(parentCheckpoints, "parentCheckpoints");
1111
+ __name(parentClosed, "parentClosed");
995
1112
  async function childCompletionNeedsBuffer(store, childId, completion, parent) {
996
1113
  const parentId = parentRunIdOf(childId);
997
1114
  if (!parentId) return false;
998
1115
  const run = parent === void 0 ? await store.getRun(parentId) : parent;
999
- if (!run) return false;
1000
- if (TERMINAL_RUN_STATUSES.includes(run.status) && !RETRYABLE.has(run.status)) return false;
1001
- return !parentHoldsChild(await parentCheckpoints(store, parentId, childId), childId, completion);
1116
+ if (parentClosed(run)) return false;
1117
+ const [held] = await parentSettles(store, parentId, [
1118
+ {
1119
+ childId,
1120
+ completion
1121
+ }
1122
+ ]);
1123
+ return !held;
1002
1124
  }
1003
1125
  __name(childCompletionNeedsBuffer, "childCompletionNeedsBuffer");
1004
- async function unneededChildSignals(store, rows) {
1126
+ async function unneededChildSignals(store, rows, deadline = Number.POSITIVE_INFINITY) {
1127
+ const childOf = /* @__PURE__ */ __name((row) => row.token.startsWith(CHILD_TOKEN_PREFIX) ? row.token.slice(CHILD_TOKEN_PREFIX.length) : void 0, "childOf");
1005
1128
  const parentIds = rows.flatMap((row) => {
1006
- const parentId = parentRunIdOf(row.token.slice(CHILD_TOKEN_PREFIX.length));
1129
+ const childId = childOf(row);
1130
+ const parentId = childId === void 0 ? void 0 : parentRunIdOf(childId);
1007
1131
  return parentId ? [
1008
1132
  parentId
1009
1133
  ] : [];
1010
1134
  });
1011
1135
  const { runs, unread } = await readRunsByIds(store, parentIds);
1012
1136
  const unreadable = new Set(unread);
1013
- const doomed = [];
1014
- for (const row of rows) {
1015
- if (!row.token.startsWith(CHILD_TOKEN_PREFIX)) continue;
1016
- const childId = row.token.slice(CHILD_TOKEN_PREFIX.length);
1137
+ const doomed = /* @__PURE__ */ new Set();
1138
+ const open = /* @__PURE__ */ new Map();
1139
+ rows.forEach((row, at) => {
1140
+ const childId = childOf(row);
1141
+ if (childId === void 0) return;
1017
1142
  const parentId = parentRunIdOf(childId);
1018
- if (parentId && unreadable.has(parentId)) continue;
1019
- const parent = parentId ? runs.get(parentId) ?? null : null;
1143
+ if (!parentId) {
1144
+ doomed.add(row);
1145
+ return;
1146
+ }
1147
+ if (unreadable.has(parentId)) return;
1148
+ if (parentClosed(runs.get(parentId) ?? null)) {
1149
+ doomed.add(row);
1150
+ return;
1151
+ }
1152
+ const group = open.get(parentId) ?? [];
1153
+ group.push({
1154
+ at,
1155
+ row,
1156
+ childId
1157
+ });
1158
+ open.set(parentId, group);
1159
+ });
1160
+ let decided = rows.length;
1161
+ let checked = 0;
1162
+ for (const [parentId, group] of open) {
1163
+ if (checked > 0 && Date.now() >= deadline) {
1164
+ decided = group[0].at;
1165
+ break;
1166
+ }
1167
+ checked += 1;
1020
1168
  try {
1021
- if (!await childCompletionNeedsBuffer(store, childId, row.payload, parent)) doomed.push(row);
1169
+ const held = await parentSettles(store, parentId, group.map(({ row, childId }) => ({
1170
+ childId,
1171
+ completion: row.payload
1172
+ })));
1173
+ group.forEach(({ row }, k) => {
1174
+ if (held[k]) doomed.add(row);
1175
+ });
1022
1176
  } catch {
1023
1177
  }
1024
1178
  }
1025
- return doomed;
1179
+ return {
1180
+ doomed: rows.filter((row) => doomed.has(row)),
1181
+ decided
1182
+ };
1026
1183
  }
1027
1184
  __name(unneededChildSignals, "unneededChildSignals");
1028
1185
 
@@ -4643,6 +4800,7 @@ var ANNOUNCED_ROUTES_TTL_MS = 5e3;
4643
4800
  var VERSION_UNDECLARED_TAG = "version:undeclared";
4644
4801
  var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
4645
4802
  var STEP_FAN_CHUNK = 200;
4803
+ var FAN_REREAD_CHUNK = 1e3;
4646
4804
  var PARENT_NOTIFY_ATTEMPTS = 4;
4647
4805
  var PARENT_NOTIFY_BACKOFF_MS = 250;
4648
4806
  var LEASE_RECHECK_MAX_MS = 2e3;
@@ -4679,6 +4837,7 @@ var WAKE_REQUEST_SCAN = 100;
4679
4837
  var CHILD_SIGNAL_PREFIX = "signal:child:";
4680
4838
  var BUFFERED_SWEEP_PAGE = 500;
4681
4839
  var BUFFERED_SWEEP_PAGES_PER_CALL = 4;
4840
+ var BUFFERED_SWEEP_BUDGET_MS = 250;
4682
4841
  var BUFFERED_SWEEP_REST_MS = 10 * 6e4;
4683
4842
  var DELIVER_CHUNK = 50;
4684
4843
  var RESULT_BATCH_MAX = 256;
@@ -5571,13 +5730,17 @@ var WorkflowEngine = class {
5571
5730
  *
5572
5731
  * Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
5573
5732
  * at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
5574
- * few timer ticks without a long query or a big delete. A pass that reaches the end of the table
5575
- * rests {@link BUFFERED_SWEEP_REST_MS} before the next one. Call it every tick (the NestJS timer
5576
- * poller does). Skipped while draining, on a namespaced engine (a buffered signal carries no
5577
- * namespace, and a scoped read would report another tenant's parent as gone), and on a store
5578
- * without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
5579
- */
5580
- async sweepBufferedSignals(now = this.clock()) {
5733
+ * few timer ticks without a long query or a big delete. Each page is decided set-based (see
5734
+ * `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
5735
+ * read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
5736
+ * {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
5737
+ * call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
5738
+ * before the next one. Call it every tick (the NestJS timer poller does); a call made while
5739
+ * another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
5740
+ * carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
5741
+ * store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
5742
+ */
5743
+ async sweepBufferedSignals(now = this.clock(), budgetMs = BUFFERED_SWEEP_BUDGET_MS) {
5581
5744
  const { listBufferedSignals, deleteBufferedSignals } = this.store;
5582
5745
  if (!listBufferedSignals || !deleteBufferedSignals) return 0;
5583
5746
  if (this.draining || this.namespace !== void 0 || this.bufferedSweepActive) return 0;
@@ -5585,15 +5748,21 @@ var WorkflowEngine = class {
5585
5748
  return 0;
5586
5749
  }
5587
5750
  this.bufferedSweepActive = true;
5751
+ const deadline = Date.now() + budgetMs;
5588
5752
  let deleted = 0;
5589
5753
  try {
5590
5754
  for (let page = 0; page < BUFFERED_SWEEP_PAGES_PER_CALL; page++) {
5755
+ if (page > 0 && (Date.now() >= deadline || this.draining)) break;
5591
5756
  const rows = await listBufferedSignals.call(this.store, CHILD_TOKEN_PREFIX, this.bufferedSweepCursor, BUFFERED_SWEEP_PAGE);
5592
- const doomed = await unneededChildSignals(this.store, rows);
5757
+ const { doomed, decided } = await unneededChildSignals(this.store, rows, deadline);
5593
5758
  if (doomed.length > 0) {
5594
5759
  await deleteBufferedSignals.call(this.store, doomed.map((row) => row.id));
5595
5760
  deleted += doomed.length;
5596
5761
  }
5762
+ if (decided < rows.length) {
5763
+ if (decided > 0) this.bufferedSweepCursor = rows[decided - 1].id;
5764
+ break;
5765
+ }
5597
5766
  const last = rows.at(-1);
5598
5767
  if (rows.length < BUFFERED_SWEEP_PAGE || !last) {
5599
5768
  this.bufferedSweepCursor = void 0;
@@ -7068,8 +7237,7 @@ var WorkflowEngine = class {
7068
7237
  */
7069
7238
  async redriveSettledSteps(runId, seqs) {
7070
7239
  for (const seq of seqs) {
7071
- const cp = await this.store.getCheckpoint(runId, seq);
7072
- if (cp && (cp.status === "completed" || cp.status === "failed")) {
7240
+ if (await this.stepSettled(runId, seq)) {
7073
7241
  if (this.settledHere.has(`${runId}:${seq}`)) return;
7074
7242
  const nowMs = this.clock();
7075
7243
  if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
@@ -7085,6 +7253,17 @@ var WorkflowEngine = class {
7085
7253
  }
7086
7254
  }
7087
7255
  }
7256
+ /** Whether the step at `seq` is settled: a `COUNT` when the store has one, else its checkpoint. */
7257
+ async stepSettled(runId, seq) {
7258
+ if (this.store.countCheckpoints) {
7259
+ return await this.store.countCheckpoints(runId, seq, seq, [
7260
+ "completed",
7261
+ "failed"
7262
+ ]) > 0;
7263
+ }
7264
+ const cp = await this.store.getCheckpoint(runId, seq);
7265
+ return cp?.status === "completed" || cp?.status === "failed";
7266
+ }
7088
7267
  /**
7089
7268
  * {@link redriveSettledSteps} for a step fan: once the turn that suspended on it released its lease,
7090
7269
  * re-drive the run if no step of the fan is still `pending` — the last result landed on another
@@ -8556,7 +8735,7 @@ ${run.workflowVersion}`;
8556
8735
  }
8557
8736
  const wfPlan = await this.planRoute(tenantGroup(sanitizeQueueToken(remote.group), void 0), registered.requires);
8558
8737
  if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
8559
- const checkpoints = await this.store.listCheckpoints(run.id);
8738
+ const checkpoints = await this.replayCheckpoints(run.id);
8560
8739
  await this.deliverFinishedChildren(run.id, checkpoints);
8561
8740
  await this.deliverFinishedChildWaiters(run.id, checkpoints);
8562
8741
  const handoff = liveHandoff(run.id, checkpoints);
@@ -8832,6 +9011,37 @@ ${run.workflowVersion}`;
8832
9011
  * throttle for {@link rearmStepLease}, the step-scoped sibling of {@link decisionDeadlineRearms}. */
8833
9012
  stepLeaseRearms = /* @__PURE__ */ new Map();
8834
9013
  /**
9014
+ * {@link stepLeaseRearms} for the steps of a fan this instance sent, by run: one entry per sent
9015
+ * block of consecutive seqs instead of one per step, since they share the lease stamped at send.
9016
+ */
9017
+ fanLeaseRearms = /* @__PURE__ */ new Map();
9018
+ noteFanLease(runId, seqs, lease) {
9019
+ if (this.remoteRedispatchMs == null || seqs.length === 0) return;
9020
+ let from = seqs[0];
9021
+ let to = from;
9022
+ for (const seq of seqs) {
9023
+ if (seq < from) from = seq;
9024
+ if (seq > to) to = seq;
9025
+ }
9026
+ if (to - from + 1 !== seqs.length) return;
9027
+ if (this.fanLeaseRearms.size > 1024) this.fanLeaseRearms.clear();
9028
+ const blocks = this.fanLeaseRearms.get(runId) ?? [];
9029
+ blocks.push({
9030
+ from,
9031
+ to,
9032
+ lookAgainAt: lease - Math.ceil(this.remoteRedispatchMs / 2)
9033
+ });
9034
+ this.fanLeaseRearms.set(runId, blocks);
9035
+ }
9036
+ fanLeaseLookAgainAt(runId, seq, now) {
9037
+ const blocks = this.fanLeaseRearms.get(runId);
9038
+ if (!blocks) return void 0;
9039
+ const live = blocks.filter((b) => now < b.lookAgainAt);
9040
+ if (live.length === 0) this.fanLeaseRearms.delete(runId);
9041
+ else if (live.length < blocks.length) this.fanLeaseRearms.set(runId, live);
9042
+ return live.find((b) => seq >= b.from && seq <= b.to)?.lookAgainAt;
9043
+ }
9044
+ /**
8835
9045
  * A step-scoped heartbeat for a durably-suspended remote step: push its LEASE forward so a worker
8836
9046
  * that is still working on it is never presumed dead and double-dispatched. The durable counterpart
8837
9047
  * of {@link callRemoteInMemory}'s in-memory `heartbeatResets` rearm (which only exists on the
@@ -8860,7 +9070,7 @@ ${run.workflowVersion}`;
8860
9070
  if (this.remoteRedispatchMs == null) return;
8861
9071
  const id = stepId(runId, seq);
8862
9072
  const now = this.clock();
8863
- const lookAgainAt = this.stepLeaseRearms.get(id);
9073
+ const lookAgainAt = this.stepLeaseRearms.get(id) ?? this.fanLeaseLookAgainAt(runId, seq, now);
8864
9074
  if (lookAgainAt !== void 0 && now < lookAgainAt) return;
8865
9075
  try {
8866
9076
  const cp = await this.store.getCheckpoint(runId, seq);
@@ -9183,7 +9393,7 @@ ${run.workflowVersion}`;
9183
9393
  return this.singletonWaitResult(run);
9184
9394
  }
9185
9395
  const compensations = [];
9186
- const snapshot = await this.store.listCheckpoints(run.id);
9396
+ const snapshot = await this.replayCheckpoints(run.id);
9187
9397
  await this.deliverFinishedChildren(run.id, snapshot);
9188
9398
  const replay = /* @__PURE__ */ new Map();
9189
9399
  for (const cp of snapshot) replay.set(cp.seq, cp);
@@ -9513,7 +9723,7 @@ ${run.workflowVersion}`;
9513
9723
  }), "callRemote"),
9514
9724
  callRemoteAll: /* @__PURE__ */ __name((runId, positions, step, inputs, opts) => this.callRemoteAll(runId, positions, step, inputs, opts, {
9515
9725
  namespace: parentNamespace
9516
- }), "callRemoteAll"),
9726
+ }, replay), "callRemoteAll"),
9517
9727
  // Defer so a fast child can't reentrantly resume a still-running parent. Stamp the child with
9518
9728
  // the PARENT run's namespace (not this engine's) so a child belongs to the tenant/partition of
9519
9729
  // the run it was spawned from — even when an operator (namespace: undefined, drives every
@@ -9870,7 +10080,7 @@ ${run.workflowVersion}`;
9870
10080
  * concurrently with this one finds them already sent instead of sending them again. The join
9871
10081
  * barrier ({@link stepFanStillRunning}) then wakes the parent once per wave, not once per result.
9872
10082
  */
9873
- async callRemoteAll(runId, positions, step, inputs, opts, routing) {
10083
+ async callRemoteAll(runId, positions, step, inputs, opts, routing, replay) {
9874
10084
  if (step.timeoutMs) {
9875
10085
  throw new Error(`ctx.all: step "${step.name}" has a liveness timeoutMs; a step fan needs durable steps`);
9876
10086
  }
@@ -9881,10 +10091,7 @@ ${run.workflowVersion}`;
9881
10091
  const first = positions[0];
9882
10092
  const last = positions[positions.length - 1];
9883
10093
  const group = stepFanGroup(first, positions.length, opts.mode);
9884
- const bySeq = /* @__PURE__ */ new Map();
9885
- for (const cp of await this.store.listCheckpoints(runId)) {
9886
- if (cp.seq >= first && cp.seq <= last) bySeq.set(cp.seq, cp);
9887
- }
10094
+ const bySeq = await this.fanCheckpoints(runId, positions, replay);
9888
10095
  const outputs = new Array(inputs.length);
9889
10096
  const failures = [];
9890
10097
  const unsent = [];
@@ -9941,7 +10148,13 @@ ${run.workflowVersion}`;
9941
10148
  }
9942
10149
  if (inFlight === 0 && highest >= 0) this.turnSettledFans.get(runId)?.set(first, highest);
9943
10150
  const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
9944
- if (batch.length > 0) await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
10151
+ if (batch.length > 0) {
10152
+ const lease = await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
10153
+ if (lease !== void 0) {
10154
+ const at = this.stepLeaseWakeAt(lease);
10155
+ wakeAt = Math.min(wakeAt ?? at, at);
10156
+ }
10157
+ }
9945
10158
  if (inFlight + batch.length > 0) {
9946
10159
  this.awaitedStepFans.get(runId)?.push({
9947
10160
  from: first,
@@ -9971,6 +10184,34 @@ ${run.workflowVersion}`;
9971
10184
  return outputs;
9972
10185
  }
9973
10186
  /**
10187
+ * The checkpoints of a step fan's seqs, by seq. A settled one is taken from the turn's snapshot
10188
+ * (`replay`), since nothing but a turn of the run changes it once settled; one the snapshot holds
10189
+ * `pending`, or does not hold, is read again, as {@link callRemote} does for a single step: its
10190
+ * result may have landed since. The fan's last turn so reads nothing it already read.
10191
+ */
10192
+ async fanCheckpoints(runId, positions, replay) {
10193
+ const bySeq = /* @__PURE__ */ new Map();
10194
+ const reread = [];
10195
+ for (const seq of positions) {
10196
+ const cp = replay?.get(seq);
10197
+ if (cp && (cp.status === "completed" || cp.status === "failed")) bySeq.set(seq, cp);
10198
+ else reread.push(seq);
10199
+ }
10200
+ if (reread.length === 0) return bySeq;
10201
+ if (reread.length === positions.length || !this.store.getCheckpoints) {
10202
+ const wanted = new Set(reread);
10203
+ for (const cp of await this.store.listCheckpoints(runId)) {
10204
+ if (wanted.has(cp.seq)) bySeq.set(cp.seq, cp);
10205
+ }
10206
+ return bySeq;
10207
+ }
10208
+ for (let at = 0; at < reread.length; at += FAN_REREAD_CHUNK) {
10209
+ const read = await this.store.getCheckpoints(runId, reread.slice(at, at + FAN_REREAD_CHUNK));
10210
+ for (const cp of read) bySeq.set(cp.seq, cp);
10211
+ }
10212
+ return bySeq;
10213
+ }
10214
+ /**
9974
10215
  * Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
9975
10216
  * {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
9976
10217
  * batch, where the store and transport have them.
@@ -9979,6 +10220,7 @@ ${run.workflowVersion}`;
9979
10220
  const workerGroup = stepGroup(step, routing?.namespace);
9980
10221
  await this.ensureRoutable(workerGroup, step.requires);
9981
10222
  const enqueuedAt = /* @__PURE__ */ new Date();
10223
+ const lease = this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
9982
10224
  const checkpoints = batch.map((i) => {
9983
10225
  const seq = positions[i];
9984
10226
  return {
@@ -9994,7 +10236,10 @@ ${run.workflowVersion}`;
9994
10236
  parallelGroup: group,
9995
10237
  enqueuedAt,
9996
10238
  startedAt: enqueuedAt,
9997
- finishedAt: enqueuedAt
10239
+ finishedAt: enqueuedAt,
10240
+ ...lease !== void 0 ? {
10241
+ wakeAt: lease
10242
+ } : {}
9998
10243
  };
9999
10244
  });
10000
10245
  if (this.store.saveCheckpoints) {
@@ -10027,20 +10272,36 @@ ${run.workflowVersion}`;
10027
10272
  kind: "remote"
10028
10273
  });
10029
10274
  }
10275
+ if (lease !== void 0) {
10276
+ this.noteFanLease(runId, checkpoints.map((cp) => cp.seq), lease);
10277
+ }
10278
+ return lease;
10030
10279
  }
10031
10280
  /** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
10032
10281
  noteSettledHere(key) {
10033
10282
  this.noteRecent(this.settledHere, key);
10034
10283
  }
10035
- /** Record `key` in a "happened here recently" map, pruning entries older than a lease. */
10284
+ /**
10285
+ * Record `key` in a "happened here recently" map and drop the entries older than a lease. A key is
10286
+ * re-inserted at the end, so the map stays in the order its keys were noted and the stale ones are
10287
+ * all at its front: pruning stops at the first fresh entry instead of walking the whole map.
10288
+ */
10036
10289
  noteRecent(map, key) {
10037
10290
  const now = this.clock();
10291
+ map.delete(key);
10038
10292
  map.set(key, now);
10039
- if (map.size < 1024) return;
10040
10293
  for (const [k, at] of map) {
10041
- if (now - at > this.leaseMs) map.delete(k);
10294
+ if (now - at <= this.leaseMs) break;
10295
+ map.delete(k);
10042
10296
  }
10043
10297
  }
10298
+ /**
10299
+ * The run's checkpoints as a turn replays them: {@link StateStore.listCheckpointsForReplay} when
10300
+ * the store has it (no input on a completed row), else {@link StateStore.listCheckpoints}.
10301
+ */
10302
+ replayCheckpoints(runId) {
10303
+ return this.store.listCheckpointsForReplay ? this.store.listCheckpointsForReplay(runId) : this.store.listCheckpoints(runId);
10304
+ }
10044
10305
  /** {@link StateStore.saveCheckpointIfPending}, or a plain save on a store without it. */
10045
10306
  async savePendingCheckpoint(checkpoint) {
10046
10307
  if (this.store.saveCheckpointIfPending) return this.store.saveCheckpointIfPending(checkpoint);
@@ -10067,11 +10328,59 @@ ${run.workflowVersion}`;
10067
10328
  });
10068
10329
  }
10069
10330
  /**
10331
+ * Settle steps of one run at distinct seqs from their results ({@link StepSettlement}s, `settled[i]`
10332
+ * being the `pending` row `settlements[i]` settles): one
10333
+ * {@link StateStore.settleCheckpointsIfPending} call when the store has it, which writes only what
10334
+ * a result changes, else the whole rows through {@link savePendingCheckpoints}. Never rejects as a
10335
+ * whole, the same way.
10336
+ */
10337
+ async settlePendingCheckpoints(settlements, settled) {
10338
+ const settle = this.store.settleCheckpointsIfPending?.bind(this.store);
10339
+ if (!settle) {
10340
+ return this.savePendingCheckpoints(settlements.map((s, i) => ({
10341
+ ...settled[i],
10342
+ ...s
10343
+ })));
10344
+ }
10345
+ if (settlements.length > 1) {
10346
+ try {
10347
+ const wrote = await settle(settlements);
10348
+ return settlements.map((_, i) => wrote[i] === true);
10349
+ } catch {
10350
+ }
10351
+ }
10352
+ const one = await Promise.allSettled(settlements.map((s) => settle([
10353
+ s
10354
+ ])));
10355
+ return one.map((s) => s.status === "fulfilled" ? s.value[0] === true : {
10356
+ error: s.reason
10357
+ });
10358
+ }
10359
+ /**
10070
10360
  * The checkpoints of one run at distinct `seqs`, each as its own outcome: one
10071
10361
  * {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
10072
10362
  * call rejecting falls back to one read each, so a failing read fails only its own entry.
10073
10363
  */
10074
10364
  async readCheckpoints(runId, seqs) {
10365
+ const forSettle = this.store.settleCheckpointsIfPending ? this.store.getCheckpointsForSettle?.bind(this.store) : void 0;
10366
+ if (forSettle) {
10367
+ if (seqs.length > 1) {
10368
+ try {
10369
+ const bySeq = new Map((await forSettle(runId, seqs)).map((cp) => [
10370
+ cp.seq,
10371
+ cp
10372
+ ]));
10373
+ return seqs.map((seq) => ({
10374
+ status: "fulfilled",
10375
+ value: bySeq.get(seq) ?? null
10376
+ }));
10377
+ } catch {
10378
+ }
10379
+ }
10380
+ return Promise.allSettled(seqs.map(async (seq) => (await forSettle(runId, [
10381
+ seq
10382
+ ]))[0] ?? null));
10383
+ }
10075
10384
  if (seqs.length > 1 && this.store.getCheckpoints) {
10076
10385
  try {
10077
10386
  const bySeq = new Map((await this.store.getCheckpoints(runId, seqs)).map((cp) => [
@@ -10220,10 +10529,11 @@ ${run.workflowVersion}`;
10220
10529
  const run = await this.store.getRun(runId);
10221
10530
  if (run && (run.status === "cancelled" || run.status === "completed")) return outcomes;
10222
10531
  const finishedAt = /* @__PURE__ */ new Date();
10223
- const writes = pending.map(({ index, cp }) => {
10532
+ const settlements = pending.map(({ index, cp }) => {
10224
10533
  const result = results[index];
10225
10534
  return {
10226
- ...cp,
10535
+ runId,
10536
+ seq: cp.seq,
10227
10537
  status: result.status,
10228
10538
  output: result.status === "completed" ? result.output : cp.output,
10229
10539
  error: result.error,
@@ -10232,7 +10542,7 @@ ${run.workflowVersion}`;
10232
10542
  finishedAt
10233
10543
  };
10234
10544
  });
10235
- const claims = await this.savePendingCheckpoints(writes);
10545
+ const claims = await this.settlePendingCheckpoints(settlements, pending.map(({ cp }) => cp));
10236
10546
  const settled = [];
10237
10547
  claims.forEach((claim, k) => {
10238
10548
  const { index, cp } = pending[k];
@@ -10246,7 +10556,7 @@ ${run.workflowVersion}`;
10246
10556
  };
10247
10557
  });
10248
10558
  for (const { cp, result } of settled) {
10249
- this.noteSettledHere(`${runId}:${result.seq}`);
10559
+ if (!parseStepFanGroup(cp.parallelGroup)) this.noteSettledHere(`${runId}:${result.seq}`);
10250
10560
  const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
10251
10561
  this.emit({
10252
10562
  type: result.status === "completed" ? "step.completed" : "step.failed",
@@ -11145,8 +11455,16 @@ var CodecStateStore = class {
11145
11455
  saveCheckpointIfPending;
11146
11456
  /** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
11147
11457
  saveCheckpointsIfPending;
11458
+ /** Encodes each settlement's payloads like {@link saveCheckpoint}. Bound only when the inner store
11459
+ * has it, so the engine's whole-row fallback still kicks in through the wrapper. */
11460
+ settleCheckpointsIfPending;
11148
11461
  /** Encodes each checkpoint written. Bound only when the inner store has it. */
11149
11462
  saveCheckpoints;
11463
+ /** Decodes each checkpoint read like {@link listCheckpoints}. Bound only when the inner store has
11464
+ * it, so the engine's full-read fallback still kicks in through the wrapper. */
11465
+ listCheckpointsForReplay;
11466
+ /** Decodes each checkpoint read. Bound only when the inner store has it. */
11467
+ getCheckpointsForSettle;
11150
11468
  /** Decodes each checkpoint read. Bound only when the inner store has it. */
11151
11469
  getCheckpoints;
11152
11470
  /** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
@@ -11199,8 +11517,14 @@ var CodecStateStore = class {
11199
11517
  if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
11200
11518
  const saveManyIfPending = inner.saveCheckpointsIfPending;
11201
11519
  if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
11520
+ const settleMany = inner.settleCheckpointsIfPending;
11521
+ if (settleMany) this.settleCheckpointsIfPending = (settlements) => settleMany.call(inner, settlements.map((s) => this.encSettlement(s)));
11202
11522
  const saveMany = inner.saveCheckpoints;
11203
11523
  if (saveMany) this.saveCheckpoints = (cps) => saveMany.call(inner, cps.map((cp) => this.encCp(cp)));
11524
+ const listForReplay = inner.listCheckpointsForReplay;
11525
+ if (listForReplay) this.listCheckpointsForReplay = async (runId) => (await listForReplay.call(inner, runId)).map((cp) => this.decCp(cp));
11526
+ const getForSettle = inner.getCheckpointsForSettle;
11527
+ if (getForSettle) this.getCheckpointsForSettle = async (runId, seqs) => (await getForSettle.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
11204
11528
  const getMany = inner.getCheckpoints;
11205
11529
  if (getMany) this.getCheckpoints = async (runId, seqs) => (await getMany.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
11206
11530
  const countCps = inner.countCheckpoints;
@@ -11292,6 +11616,21 @@ var CodecStateStore = class {
11292
11616
  }
11293
11617
  return next;
11294
11618
  }
11619
+ encSettlement(settlement) {
11620
+ const next = {
11621
+ ...settlement,
11622
+ output: this.enc(settlement.output)
11623
+ };
11624
+ if (this.extended) {
11625
+ if (settlement.events !== void 0) {
11626
+ next.events = this.enc(settlement.events);
11627
+ }
11628
+ if (settlement.error !== void 0) {
11629
+ next.error = this.enc(settlement.error);
11630
+ }
11631
+ }
11632
+ return next;
11633
+ }
11295
11634
  decCp(cp) {
11296
11635
  if (!cp) return cp;
11297
11636
  const next = {
@@ -11851,6 +12190,18 @@ var InMemoryStateStore = class {
11851
12190
  ] : [];
11852
12191
  });
11853
12192
  }
12193
+ async getCheckpointsForSettle(runId, seqs) {
12194
+ return [
12195
+ ...new Set(seqs)
12196
+ ].sort((a, b) => a - b).flatMap((seq) => {
12197
+ const cp = this.checkpoints.get(this.key(runId, seq));
12198
+ if (!cp) return [];
12199
+ const { input: _input, ...rest } = cp;
12200
+ return [
12201
+ rest
12202
+ ];
12203
+ });
12204
+ }
11854
12205
  async saveCheckpoint(checkpoint) {
11855
12206
  this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
11856
12207
  ...checkpoint
@@ -11881,6 +12232,18 @@ var InMemoryStateStore = class {
11881
12232
  return true;
11882
12233
  });
11883
12234
  }
12235
+ async settleCheckpointsIfPending(settlements) {
12236
+ return settlements.map((settlement) => {
12237
+ const key = this.key(settlement.runId, settlement.seq);
12238
+ const cp = this.checkpoints.get(key);
12239
+ if (cp?.status !== "pending") return false;
12240
+ this.checkpoints.set(key, {
12241
+ ...cp,
12242
+ ...settlement
12243
+ });
12244
+ return true;
12245
+ });
12246
+ }
11884
12247
  async listOrphanedRuns(nowMs, limit, namespace, ownedBy) {
11885
12248
  return [
11886
12249
  ...this.runs.values()
@@ -12197,6 +12560,13 @@ var InMemoryStateStore = class {
12197
12560
  this.checkpointRowsRead += rows.length;
12198
12561
  return rows;
12199
12562
  }
12563
+ async listCheckpointsForReplay(runId) {
12564
+ return (await this.listCheckpoints(runId)).map((cp) => {
12565
+ if (cp.status !== "completed") return cp;
12566
+ const { input: _input, ...rest } = cp;
12567
+ return rest;
12568
+ });
12569
+ }
12200
12570
  runCheckpoints(runId) {
12201
12571
  return [
12202
12572
  ...this.checkpoints.values()