@dudousxd/nestjs-durable-core 0.85.6 → 0.85.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -1151,7 +1151,6 @@ var RETRYABLE = /* @__PURE__ */ new Set([
1151
1151
  "failed",
1152
1152
  "dead"
1153
1153
  ]);
1154
- var FAN_CHILD = /^(.+\.all\.\d+\.(?:w\d+\.)?)(\d+)$/;
1155
1154
  function canonicalJson(value) {
1156
1155
  const plain = JSON.stringify(value);
1157
1156
  return plain === void 0 ? "null" : sortedJson(JSON.parse(plain));
@@ -1166,77 +1165,235 @@ function sortedJson(value) {
1166
1165
  return JSON.stringify(value);
1167
1166
  }
1168
1167
  __name(sortedJson, "sortedJson");
1169
- function settledNames(childId) {
1168
+ var FAN_CHILD_PARTS = /^(.+\.all\.(\d+)\.)(?:w(\d+)\.)?(\d+)$/;
1169
+ var SEQ_CHILD = /\.child\.(\d+)$/;
1170
+ function slotOf(childId) {
1170
1171
  const signal = `${CHILD_SIGNAL_NAME_PREFIX}${childId}`;
1171
- const fan = FAN_CHILD.exec(childId);
1172
- if (!fan) return {
1173
- signal
1174
- };
1175
- return {
1176
- signal,
1177
- fold: {
1178
- name: `${FOLDED_PREFIX}${fan[1]}`,
1179
- index: Number(fan[2])
1172
+ const fan = FAN_CHILD_PARTS.exec(childId);
1173
+ if (fan) {
1174
+ const root = fan[1];
1175
+ const wave = fan[3];
1176
+ const firstSeq = Number(fan[2]);
1177
+ const i = Number(fan[4]);
1178
+ if (wave === void 0) {
1179
+ return {
1180
+ signal,
1181
+ signalSeq: firstSeq + i,
1182
+ fan: {
1183
+ fold: `${FOLDED_PREFIX}${root}`,
1184
+ firstSeq,
1185
+ index: i
1186
+ }
1187
+ };
1180
1188
  }
1189
+ return {
1190
+ signal,
1191
+ signalSeq: firstSeq + i,
1192
+ fan: {
1193
+ fold: `${FOLDED_PREFIX}${root}w${wave}.`,
1194
+ firstSeq,
1195
+ index: i,
1196
+ wave: Number(wave),
1197
+ waveZeroFold: `${FOLDED_PREFIX}${root}w0.`
1198
+ }
1199
+ };
1200
+ }
1201
+ const seq = SEQ_CHILD.exec(childId);
1202
+ return seq ? {
1203
+ signal,
1204
+ signalSeq: Number(seq[1])
1205
+ } : {
1206
+ signal
1181
1207
  };
1182
1208
  }
1183
- __name(settledNames, "settledNames");
1184
- function parentHoldsChild(checkpoints, childId, completion) {
1185
- const { signal, fold } = settledNames(childId);
1186
- const expected = canonicalJson(completion);
1187
- return checkpoints.some((cp) => {
1188
- if (cp.status !== "completed") return false;
1189
- if (cp.name === signal) return canonicalJson(cp.output) === expected;
1190
- if (fold && cp.name === fold.name) {
1191
- const { from, values } = cp.output ?? {};
1192
- return typeof from === "number" && Array.isArray(values) && fold.index >= from && fold.index < from + values.length;
1209
+ __name(slotOf, "slotOf");
1210
+ function foldOf(cp, name) {
1211
+ if (!cp || cp.status !== "completed" || cp.name !== name) return void 0;
1212
+ const { from, values } = cp.output ?? {};
1213
+ return typeof from === "number" && Array.isArray(values) ? {
1214
+ from,
1215
+ values
1216
+ } : void 0;
1217
+ }
1218
+ __name(foldOf, "foldOf");
1219
+ function covers(fold, index) {
1220
+ return !!fold && index >= fold.from && index < fold.from + fold.values.length;
1221
+ }
1222
+ __name(covers, "covers");
1223
+ function holdsSignal(cp, signal, expected) {
1224
+ return cp.status === "completed" && cp.name === signal && canonicalJson(cp.output) === expected;
1225
+ }
1226
+ __name(holdsSignal, "holdsSignal");
1227
+ async function checkpointsAt(store, runId, seqs) {
1228
+ const unique = [
1229
+ ...new Set(seqs)
1230
+ ];
1231
+ const bySeq = /* @__PURE__ */ new Map();
1232
+ if (unique.length === 0) return bySeq;
1233
+ const rows = store.getCheckpoints ? await store.getCheckpoints(runId, unique) : (await Promise.all(unique.map((seq) => store.getCheckpoint(runId, seq)))).filter((cp) => cp !== null);
1234
+ for (const cp of rows) bySeq.set(cp.seq, cp);
1235
+ return bySeq;
1236
+ }
1237
+ __name(checkpointsAt, "checkpointsAt");
1238
+ async function checkpointsNamed(store, runId, names) {
1239
+ const wanted = new Set(names);
1240
+ const rows = store.listCheckpointsByNamePrefix ? await store.listCheckpointsByNamePrefix(runId, [
1241
+ ...wanted
1242
+ ]) : await store.listCheckpoints(runId);
1243
+ return rows.filter((cp) => wanted.has(cp.name));
1244
+ }
1245
+ __name(checkpointsNamed, "checkpointsNamed");
1246
+ async function parentSettles(store, parentId, children) {
1247
+ const entries = children.map(({ childId, completion }) => ({
1248
+ slot: slotOf(childId),
1249
+ expected: canonicalJson(completion)
1250
+ }));
1251
+ const held = entries.map(() => void 0);
1252
+ const first = await checkpointsAt(store, parentId, entries.flatMap(({ slot }) => [
1253
+ ...slot.signalSeq === void 0 ? [] : [
1254
+ slot.signalSeq
1255
+ ],
1256
+ ...slot.fan ? [
1257
+ slot.fan.firstSeq
1258
+ ] : []
1259
+ ]));
1260
+ const waveFolds = [];
1261
+ const byName = [];
1262
+ entries.forEach(({ slot, expected }, at) => {
1263
+ const own = slot.signalSeq === void 0 ? void 0 : first.get(slot.signalSeq);
1264
+ if (own?.name === slot.signal) {
1265
+ held[at] = holdsSignal(own, slot.signal, expected);
1266
+ return;
1193
1267
  }
1194
- return false;
1268
+ const fan = slot.fan;
1269
+ if (fan) {
1270
+ if (covers(foldOf(own, fan.fold), fan.index)) {
1271
+ held[at] = true;
1272
+ return;
1273
+ }
1274
+ const atFirst = first.get(fan.firstSeq);
1275
+ if (fan.wave === void 0 || fan.wave === 0) {
1276
+ held[at] = covers(foldOf(atFirst, fan.fold), fan.index);
1277
+ return;
1278
+ }
1279
+ const waveZero = foldOf(atFirst, fan.waveZeroFold);
1280
+ if (waveZero && waveZero.values.length > 0) {
1281
+ waveFolds.push({
1282
+ at,
1283
+ seq: fan.firstSeq + fan.wave * waveZero.values.length
1284
+ });
1285
+ return;
1286
+ }
1287
+ byName.push(at);
1288
+ return;
1289
+ }
1290
+ if (slot.signalSeq !== void 0 && !own) {
1291
+ held[at] = false;
1292
+ return;
1293
+ }
1294
+ byName.push(at);
1195
1295
  });
1296
+ if (waveFolds.length > 0) {
1297
+ const second = await checkpointsAt(store, parentId, waveFolds.map((w) => w.seq));
1298
+ for (const { at, seq } of waveFolds) {
1299
+ const fan = entries[at].slot.fan;
1300
+ held[at] = covers(foldOf(second.get(seq), fan.fold), fan.index);
1301
+ }
1302
+ }
1303
+ if (byName.length > 0) {
1304
+ const names = byName.flatMap((at) => {
1305
+ const { slot } = entries[at];
1306
+ return slot.fan ? [
1307
+ slot.signal,
1308
+ slot.fan.fold
1309
+ ] : [
1310
+ slot.signal
1311
+ ];
1312
+ });
1313
+ const named = await checkpointsNamed(store, parentId, names);
1314
+ for (const at of byName) {
1315
+ const { slot, expected } = entries[at];
1316
+ held[at] = named.some((cp) => holdsSignal(cp, slot.signal, expected) || !!slot.fan && covers(foldOf(cp, slot.fan.fold), slot.fan.index));
1317
+ }
1318
+ }
1319
+ return held.map((h) => h === true);
1196
1320
  }
1197
- __name(parentHoldsChild, "parentHoldsChild");
1198
- async function parentCheckpoints(store, parentId, childId) {
1199
- const { signal, fold } = settledNames(childId);
1200
- const prefixes = fold ? [
1201
- signal,
1202
- fold.name
1203
- ] : [
1204
- signal
1205
- ];
1206
- return store.listCheckpointsByNamePrefix ? store.listCheckpointsByNamePrefix(parentId, prefixes) : store.listCheckpoints(parentId);
1321
+ __name(parentSettles, "parentSettles");
1322
+ function parentClosed(run) {
1323
+ return !run || TERMINAL_RUN_STATUSES.includes(run.status) && !RETRYABLE.has(run.status);
1207
1324
  }
1208
- __name(parentCheckpoints, "parentCheckpoints");
1325
+ __name(parentClosed, "parentClosed");
1209
1326
  async function childCompletionNeedsBuffer(store, childId, completion, parent) {
1210
1327
  const parentId = parentRunIdOf(childId);
1211
1328
  if (!parentId) return false;
1212
1329
  const run = parent === void 0 ? await store.getRun(parentId) : parent;
1213
- if (!run) return false;
1214
- if (TERMINAL_RUN_STATUSES.includes(run.status) && !RETRYABLE.has(run.status)) return false;
1215
- return !parentHoldsChild(await parentCheckpoints(store, parentId, childId), childId, completion);
1330
+ if (parentClosed(run)) return false;
1331
+ const [held] = await parentSettles(store, parentId, [
1332
+ {
1333
+ childId,
1334
+ completion
1335
+ }
1336
+ ]);
1337
+ return !held;
1216
1338
  }
1217
1339
  __name(childCompletionNeedsBuffer, "childCompletionNeedsBuffer");
1218
- async function unneededChildSignals(store, rows) {
1340
+ async function unneededChildSignals(store, rows, deadline = Number.POSITIVE_INFINITY) {
1341
+ const childOf = /* @__PURE__ */ __name((row) => row.token.startsWith(CHILD_TOKEN_PREFIX) ? row.token.slice(CHILD_TOKEN_PREFIX.length) : void 0, "childOf");
1219
1342
  const parentIds = rows.flatMap((row) => {
1220
- const parentId = parentRunIdOf(row.token.slice(CHILD_TOKEN_PREFIX.length));
1343
+ const childId = childOf(row);
1344
+ const parentId = childId === void 0 ? void 0 : parentRunIdOf(childId);
1221
1345
  return parentId ? [
1222
1346
  parentId
1223
1347
  ] : [];
1224
1348
  });
1225
1349
  const { runs, unread } = await readRunsByIds(store, parentIds);
1226
1350
  const unreadable = new Set(unread);
1227
- const doomed = [];
1228
- for (const row of rows) {
1229
- if (!row.token.startsWith(CHILD_TOKEN_PREFIX)) continue;
1230
- const childId = row.token.slice(CHILD_TOKEN_PREFIX.length);
1351
+ const doomed = /* @__PURE__ */ new Set();
1352
+ const open = /* @__PURE__ */ new Map();
1353
+ rows.forEach((row, at) => {
1354
+ const childId = childOf(row);
1355
+ if (childId === void 0) return;
1231
1356
  const parentId = parentRunIdOf(childId);
1232
- if (parentId && unreadable.has(parentId)) continue;
1233
- const parent = parentId ? runs.get(parentId) ?? null : null;
1357
+ if (!parentId) {
1358
+ doomed.add(row);
1359
+ return;
1360
+ }
1361
+ if (unreadable.has(parentId)) return;
1362
+ if (parentClosed(runs.get(parentId) ?? null)) {
1363
+ doomed.add(row);
1364
+ return;
1365
+ }
1366
+ const group = open.get(parentId) ?? [];
1367
+ group.push({
1368
+ at,
1369
+ row,
1370
+ childId
1371
+ });
1372
+ open.set(parentId, group);
1373
+ });
1374
+ let decided = rows.length;
1375
+ let checked = 0;
1376
+ for (const [parentId, group] of open) {
1377
+ if (checked > 0 && Date.now() >= deadline) {
1378
+ decided = group[0].at;
1379
+ break;
1380
+ }
1381
+ checked += 1;
1234
1382
  try {
1235
- if (!await childCompletionNeedsBuffer(store, childId, row.payload, parent)) doomed.push(row);
1383
+ const held = await parentSettles(store, parentId, group.map(({ row, childId }) => ({
1384
+ childId,
1385
+ completion: row.payload
1386
+ })));
1387
+ group.forEach(({ row }, k) => {
1388
+ if (held[k]) doomed.add(row);
1389
+ });
1236
1390
  } catch {
1237
1391
  }
1238
1392
  }
1239
- return doomed;
1393
+ return {
1394
+ doomed: rows.filter((row) => doomed.has(row)),
1395
+ decided
1396
+ };
1240
1397
  }
1241
1398
  __name(unneededChildSignals, "unneededChildSignals");
1242
1399
 
@@ -3561,6 +3718,7 @@ function constantTimeEquals(a, b) {
3561
3718
  __name(constantTimeEquals, "constantTimeEquals");
3562
3719
 
3563
3720
  // src/transport-pool.ts
3721
+ var DISPATCH_CHUNK = 200;
3564
3722
  var TransportPool = class {
3565
3723
  static {
3566
3724
  __name(this, "TransportPool");
@@ -3620,6 +3778,28 @@ var TransportPool = class {
3620
3778
  }
3621
3779
  throw lastErr ?? new Error("no transport accepted the dispatch");
3622
3780
  }
3781
+ /**
3782
+ * {@link dispatch} for many tasks: one {@link Transport.dispatchMany} call on the preferred (or
3783
+ * first) transport when it has one, else (or when that call rejects) each task through
3784
+ * {@link dispatch}, a bounded number at a time.
3785
+ */
3786
+ async dispatchMany(tasks, preferId) {
3787
+ if (tasks.length === 0) return;
3788
+ const first = this.ordered(preferId)[0];
3789
+ if (first?.transport.dispatchMany) {
3790
+ try {
3791
+ await first.transport.dispatchMany(tasks.map((task) => ({
3792
+ ...task,
3793
+ transport: first.id
3794
+ })));
3795
+ return;
3796
+ } catch {
3797
+ }
3798
+ }
3799
+ for (let at = 0; at < tasks.length; at += DISPATCH_CHUNK) {
3800
+ await Promise.all(tasks.slice(at, at + DISPATCH_CHUNK).map((task) => this.dispatch(task, preferId)));
3801
+ }
3802
+ }
3623
3803
  /** Worker-health for `group`, merged across every transport that can report it (a group's queue
3624
3804
  * and its workers may live on any pinned transport). Undefined when no transport implements
3625
3805
  * `groupHealth` (e.g. a pure in-process transport — nothing to introspect). */
@@ -4839,6 +5019,15 @@ var PARENT_NOTIFY_BACKOFF_MS = 250;
4839
5019
  var LEASE_RECHECK_MAX_MS = 2e3;
4840
5020
  var LEASE_RECHECK_SLACK_MS = 100;
4841
5021
  var LEASE_RECHECK_ATTEMPTS = 30;
5022
+ function wakeSeen(request, settledFans, served) {
5023
+ if (request.id !== void 0 && served.has(request.id)) return true;
5024
+ return isFanWakeRequest(request.payload) && request.payload.fans.every(([first, seq]) => (settledFans.get(first) ?? -1) >= seq);
5025
+ }
5026
+ __name(wakeSeen, "wakeSeen");
5027
+ function isFanWakeRequest(payload) {
5028
+ return typeof payload === "object" && payload !== null && Array.isArray(payload.fans);
5029
+ }
5030
+ __name(isFanWakeRequest, "isFanWakeRequest");
4842
5031
  function deferred() {
4843
5032
  let resolve;
4844
5033
  let reject;
@@ -4857,12 +5046,52 @@ __name(deferred, "deferred");
4857
5046
  var pendingStateKey = /* @__PURE__ */ __name((at) => String(new Date(at).getTime()), "pendingStateKey");
4858
5047
  var freshPendingKey = /* @__PURE__ */ __name(() => `${Date.now()}-${globalThis.crypto.randomUUID().slice(0, 8)}`, "freshPendingKey");
4859
5048
  var wakeRequestToken = /* @__PURE__ */ __name((runId) => `durable:wake-after-turn:${runId}`, "wakeRequestToken");
5049
+ var WAKE_REQUEST_SCAN = 100;
4860
5050
  var CHILD_SIGNAL_PREFIX = "signal:child:";
4861
5051
  var BUFFERED_SWEEP_PAGE = 500;
4862
5052
  var BUFFERED_SWEEP_PAGES_PER_CALL = 4;
5053
+ var BUFFERED_SWEEP_BUDGET_MS = 250;
4863
5054
  var BUFFERED_SWEEP_REST_MS = 10 * 6e4;
4864
5055
  var DELIVER_CHUNK = 50;
4865
5056
  var RESULT_BATCH_MAX = 256;
5057
+ function isResultError(outcome) {
5058
+ return typeof outcome === "object" && "error" in outcome;
5059
+ }
5060
+ __name(isResultError, "isResultError");
5061
+ var FanWakeCollector = class FanWakeCollector2 {
5062
+ static {
5063
+ __name(this, "FanWakeCollector");
5064
+ }
5065
+ carrier;
5066
+ generic = false;
5067
+ fan;
5068
+ /** The highest seq the batch settled, per fan. */
5069
+ highest = /* @__PURE__ */ new Map();
5070
+ /** A checkpoint the batch settled. */
5071
+ note(cp) {
5072
+ const fan = parseStepFanGroup(cp.parallelGroup);
5073
+ if (fan) this.highest.set(fan.firstSeq, Math.max(this.highest.get(fan.firstSeq) ?? cp.seq, cp.seq));
5074
+ }
5075
+ /** A settled checkpoint that resumes the run. */
5076
+ add(index, cp, status) {
5077
+ this.carrier = index;
5078
+ const fan = parseStepFanGroup(cp.parallelGroup);
5079
+ if (!fan || status === "failed" || this.fan !== void 0 && this.fan !== fan.firstSeq) {
5080
+ this.generic = true;
5081
+ return;
5082
+ }
5083
+ this.fan = fan.firstSeq;
5084
+ }
5085
+ applyTo(outcomes) {
5086
+ if (this.carrier === void 0) return outcomes;
5087
+ const seq = this.fan === void 0 ? void 0 : this.highest.get(this.fan);
5088
+ outcomes[this.carrier] = this.generic || this.fan === void 0 || seq === void 0 ? true : {
5089
+ first: this.fan,
5090
+ seq
5091
+ };
5092
+ return outcomes;
5093
+ }
5094
+ };
4866
5095
  function shuffle(items) {
4867
5096
  for (let i = items.length - 1; i > 0; i -= 1) {
4868
5097
  const j = Math.floor(Math.random() * (i + 1));
@@ -5051,6 +5280,23 @@ var WorkflowEngine = class {
5051
5280
  /** Re-checks scheduled for runs whose wake found another instance's lease (see {@link execute}). */
5052
5281
  leaseRetries = /* @__PURE__ */ new Map();
5053
5282
  /**
5283
+ * Runs whose scheduled re-check resumes them whatever happened meanwhile. Without an entry the
5284
+ * re-check only stood for fan wakes kept as requests, and resumes the run only if a request is
5285
+ * still there: the turn that held the lease takes them once it ends (see {@link retryDroppedWake}).
5286
+ */
5287
+ unconditionalRechecks = /* @__PURE__ */ new Set();
5288
+ /** Runs a generic wake request was written for in the current re-check cycle (see {@link execute}). */
5289
+ genericWakeRequested = /* @__PURE__ */ new Set();
5290
+ /** Why each run is being woken on this instance, until the turn that answers it starts. */
5291
+ wakeReasons = /* @__PURE__ */ new Map();
5292
+ /**
5293
+ * The step fans each executing turn on this instance read with nothing in flight: first seq -> the
5294
+ * highest seq read. Written by {@link callRemoteAll}, read once the turn ends.
5295
+ */
5296
+ turnSettledFans = /* @__PURE__ */ new Map();
5297
+ /** The wake requests each executing turn on this instance serves (see {@link noteServedWakes}). */
5298
+ turnServedWakes = /* @__PURE__ */ new Map();
5299
+ /**
5054
5300
  * Every run turn and child start this instance takes on itself, under one budget of
5055
5301
  * {@link LOCAL_RUN_CONCURRENCY}: the runs it dispatches to itself, a fan's deferred child starts,
5056
5302
  * enqueued pickups and the runs it takes off the transport's run queue.
@@ -5174,8 +5420,9 @@ var WorkflowEngine = class {
5174
5420
  }
5175
5421
  return;
5176
5422
  }
5177
- if (!await this.completeRemoteResult(result)) return;
5178
- await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId).then(() => void 0, (err) => {
5423
+ const wake = await this.completeRemoteResult(result);
5424
+ if (!wake) return;
5425
+ await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId, wake === true ? void 0 : wake).then(() => void 0, (err) => {
5179
5426
  if (err instanceof NamespaceMismatch) return;
5180
5427
  throw err;
5181
5428
  }));
@@ -5392,7 +5639,10 @@ var WorkflowEngine = class {
5392
5639
  const resumed = await this.resumeLeased([
5393
5640
  run
5394
5641
  ], nowMs, (locked) => this.reconcileSignalWaiter(locked, waiters.get(locked.id)));
5395
- if (resumed.length === 0) this.recheckAfterLease(runId, run.lockedUntil);
5642
+ if (resumed.length === 0) {
5643
+ this.unconditionalRechecks.add(runId);
5644
+ this.recheckAfterLease(runId, run.lockedUntil);
5645
+ }
5396
5646
  return;
5397
5647
  }
5398
5648
  await this.resumeImplicit(runId).catch((err) => {
@@ -5693,13 +5943,17 @@ var WorkflowEngine = class {
5693
5943
  *
5694
5944
  * Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
5695
5945
  * at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
5696
- * few timer ticks without a long query or a big delete. A pass that reaches the end of the table
5697
- * rests {@link BUFFERED_SWEEP_REST_MS} before the next one. Call it every tick (the NestJS timer
5698
- * poller does). Skipped while draining, on a namespaced engine (a buffered signal carries no
5699
- * namespace, and a scoped read would report another tenant's parent as gone), and on a store
5700
- * without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
5701
- */
5702
- async sweepBufferedSignals(now = this.clock()) {
5946
+ * few timer ticks without a long query or a big delete. Each page is decided set-based (see
5947
+ * `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
5948
+ * read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
5949
+ * {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
5950
+ * call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
5951
+ * before the next one. Call it every tick (the NestJS timer poller does); a call made while
5952
+ * another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
5953
+ * carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
5954
+ * store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
5955
+ */
5956
+ async sweepBufferedSignals(now = this.clock(), budgetMs = BUFFERED_SWEEP_BUDGET_MS) {
5703
5957
  const { listBufferedSignals, deleteBufferedSignals } = this.store;
5704
5958
  if (!listBufferedSignals || !deleteBufferedSignals) return 0;
5705
5959
  if (this.draining || this.namespace !== void 0 || this.bufferedSweepActive) return 0;
@@ -5707,15 +5961,21 @@ var WorkflowEngine = class {
5707
5961
  return 0;
5708
5962
  }
5709
5963
  this.bufferedSweepActive = true;
5964
+ const deadline = Date.now() + budgetMs;
5710
5965
  let deleted = 0;
5711
5966
  try {
5712
5967
  for (let page = 0; page < BUFFERED_SWEEP_PAGES_PER_CALL; page++) {
5968
+ if (page > 0 && (Date.now() >= deadline || this.draining)) break;
5713
5969
  const rows = await listBufferedSignals.call(this.store, CHILD_TOKEN_PREFIX, this.bufferedSweepCursor, BUFFERED_SWEEP_PAGE);
5714
- const doomed = await unneededChildSignals(this.store, rows);
5970
+ const { doomed, decided } = await unneededChildSignals(this.store, rows, deadline);
5715
5971
  if (doomed.length > 0) {
5716
5972
  await deleteBufferedSignals.call(this.store, doomed.map((row) => row.id));
5717
5973
  deleted += doomed.length;
5718
5974
  }
5975
+ if (decided < rows.length) {
5976
+ if (decided > 0) this.bufferedSweepCursor = rows[decided - 1].id;
5977
+ break;
5978
+ }
5719
5979
  const last = rows.at(-1);
5720
5980
  if (rows.length < BUFFERED_SWEEP_PAGE || !last) {
5721
5981
  this.bufferedSweepCursor = void 0;
@@ -5959,15 +6219,27 @@ var WorkflowEngine = class {
5959
6219
  * terminal too.
5960
6220
  */
5961
6221
  resume(runId) {
6222
+ this.noteWakeReason(runId, void 0);
5962
6223
  return this.resumeRun(runId, true);
5963
6224
  }
6225
+ /** Record why `runId` is being woken: a step fan's barrier opening, or (`undefined`) anything else. */
6226
+ noteWakeReason(runId, fan) {
6227
+ const reason = this.wakeReasons.get(runId) ?? {
6228
+ generic: false,
6229
+ fans: /* @__PURE__ */ new Map()
6230
+ };
6231
+ if (fan) reason.fans.set(fan.first, Math.max(reason.fans.get(fan.first) ?? fan.seq, fan.seq));
6232
+ else reason.generic = true;
6233
+ this.wakeReasons.set(runId, reason);
6234
+ }
5964
6235
  /**
5965
6236
  * {@link resume} for every IMPLICIT wake-up — a delivered signal/child completion, a late step
5966
6237
  * result, a due timer, a redelivered dispatch, a queue-slot wake. A `failed` run is terminal to
5967
6238
  * these: only an explicit retry ({@link requeue} / {@link resume}) may re-drive it. Otherwise a
5968
6239
  * stale waiter or a late result left behind by a failed run silently re-executes its body.
5969
6240
  */
5970
- resumeImplicit(runId) {
6241
+ resumeImplicit(runId, fan) {
6242
+ if (fan !== "coalesced") this.noteWakeReason(runId, fan);
5971
6243
  const turn = this.wakeTurns.get(runId);
5972
6244
  if (turn) {
5973
6245
  turn.again ??= deferred();
@@ -5992,7 +6264,7 @@ var WorkflowEngine = class {
5992
6264
  });
5993
6265
  return;
5994
6266
  }
5995
- const next = this.resumeImplicit(runId);
6267
+ const next = this.resumeImplicit(runId, "coalesced");
5996
6268
  next.then(again.resolve, again.reject);
5997
6269
  this.inBackground(next);
5998
6270
  }, "finish");
@@ -8057,9 +8329,26 @@ var WorkflowEngine = class {
8057
8329
  async execute(run, fn) {
8058
8330
  const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
8059
8331
  const lockNow = this.clock();
8332
+ const reason = this.wakeReasons.get(run.id) ?? {
8333
+ generic: true,
8334
+ fans: /* @__PURE__ */ new Map()
8335
+ };
8336
+ this.wakeReasons.delete(run.id);
8060
8337
  if (run.lockedBy !== this.instanceId || (run.lockedUntil ?? 0) <= lockNow) {
8061
8338
  if (!await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow)) {
8062
- if (!this.leaseRetries.has(run.id)) this.inBackground(this.requestWakeAfterTurn(run.id));
8339
+ if (reason.generic) {
8340
+ if (!this.genericWakeRequested.has(run.id)) {
8341
+ this.genericWakeRequested.add(run.id);
8342
+ this.inBackground(this.requestWakeAfterTurn(run.id, null));
8343
+ }
8344
+ } else {
8345
+ const fans = {
8346
+ fans: [
8347
+ ...reason.fans
8348
+ ]
8349
+ };
8350
+ this.inBackground(this.requestWakeAfterTurn(run.id, fans));
8351
+ }
8063
8352
  this.recheckAfterLease(run.id, run.lockedUntil);
8064
8353
  return {
8065
8354
  runId: run.id,
@@ -8067,56 +8356,142 @@ var WorkflowEngine = class {
8067
8356
  };
8068
8357
  }
8069
8358
  }
8359
+ const settledFans = /* @__PURE__ */ new Map();
8360
+ this.turnSettledFans.set(run.id, settledFans);
8070
8361
  const held = this.holdExecutionLease(run.id);
8071
8362
  try {
8363
+ const served = /* @__PURE__ */ new Set();
8364
+ this.turnServedWakes.set(run.id, served);
8365
+ if (registered?.remote) await this.noteServedWakes(run.id);
8072
8366
  const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
8073
8367
  if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
8074
8368
  void this.singletons.wakeNext(run).catch(() => void 0);
8075
8369
  }
8076
- if (result.status === "suspended") this.inBackground(this.honorWakeRequests(run.id));
8077
- else if (TERMINAL_RUN_STATUSES.includes(result.status)) {
8078
- this.inBackground(this.takeWakeRequests(run.id));
8370
+ if (result.status === "suspended") {
8371
+ this.inBackground(this.honorWakeRequests(run.id, settledFans, served));
8372
+ } else if (TERMINAL_RUN_STATUSES.includes(result.status)) {
8373
+ this.inBackground(this.listWakeRequests(run.id, true).then((kept) => this.dropWakeRequests(kept)));
8079
8374
  }
8080
8375
  return result;
8081
8376
  } finally {
8082
8377
  held.stop();
8378
+ if (this.turnSettledFans.get(run.id) === settledFans) this.turnSettledFans.delete(run.id);
8379
+ this.turnServedWakes.delete(run.id);
8083
8380
  }
8084
8381
  }
8085
8382
  /**
8086
8383
  * Keep a wake that found `runId`'s lease held by a turn executing elsewhere, durably, so that turn
8087
8384
  * runs again once it ends instead of suspending on a snapshot that predates what the wake brought
8088
- * (a signal, a child's completion). The request is a buffered signal on a token no workflow waits
8089
- * on; the holder takes it after it releases the lease ({@link honorWakeRequests}).
8385
+ * (a signal, a child's completion). The request is a buffered event under a name no workflow waits
8386
+ * on. It stays until a turn that started after it has settled: the holder looks at it after it
8387
+ * releases the lease ({@link honorWakeRequests}), and the turn that serves it drops it.
8090
8388
  *
8091
8389
  * Written first and looked at second, against a holder that releases first and looks second:
8092
8390
  * whichever look runs last sees the other side's write. So when the run is no longer executing by
8093
- * the time this re-reads it, the holder may have looked already, and this side takes the request
8094
- * and wakes the run itself once the lease is free.
8391
+ * the time this re-reads it, the holder may have looked already, and this side wakes the run
8392
+ * itself once the lease is free ({@link retryDroppedWake}). While the holder still runs, the run is
8393
+ * made due when the holder's lease runs out, so a holder that goes down before it settles leaves
8394
+ * the run to any instance's timer poll; a holder that settles sets the run's timer itself.
8095
8395
  */
8096
- async requestWakeAfterTurn(runId) {
8097
- await this.store.bufferSignal(wakeRequestToken(runId), null);
8396
+ async requestWakeAfterTurn(runId, request) {
8397
+ const id = globalThis.crypto.randomUUID();
8398
+ await this.store.bufferEvent({
8399
+ name: wakeRequestToken(runId),
8400
+ payload: request,
8401
+ id,
8402
+ publishedAt: this.clock()
8403
+ });
8098
8404
  const run = await this.store.getRun(runId);
8099
- if (run?.status === "running") return;
8100
- if (!run) {
8101
- await this.takeWakeRequests(runId);
8405
+ if (!run || TERMINAL_RUN_STATUSES.includes(run.status)) {
8406
+ await this.store.removeBufferedEvent(id);
8407
+ return;
8408
+ }
8409
+ if (run.status === "running") return;
8410
+ if (run.lockedBy !== void 0 && run.lockedBy !== this.instanceId) {
8411
+ await this.makeDueBy(runId, (run.lockedUntil ?? this.clock()) + LEASE_RECHECK_SLACK_MS);
8102
8412
  return;
8103
8413
  }
8104
- if (!await this.takeWakeRequests(runId)) return;
8105
- if (TERMINAL_RUN_STATUSES.includes(run.status)) return;
8106
8414
  this.recheckAfterLease(runId, run.lockedUntil);
8107
8415
  }
8108
- /** After a turn that suspended released its lease: run again if a wake was kept for it. */
8109
- async honorWakeRequests(runId) {
8110
- if (!await this.takeWakeRequests(runId)) return;
8416
+ /**
8417
+ * After a turn that suspended released its lease: drop the wakes it served, and run again if a
8418
+ * wake was kept for it meanwhile, unless every such wake was a fan wake this turn already saw: it
8419
+ * read that fan with nothing in flight, at least as far as the batch that opened its barrier.
8420
+ * The run is made due before it is woken, so if this instance goes down before that turn runs,
8421
+ * the timer poll of another instance runs it once this instance's hold on it would have lapsed.
8422
+ */
8423
+ async honorWakeRequests(runId, settledFans, served) {
8424
+ const kept = await this.listWakeRequests(runId, true);
8425
+ const seen = /* @__PURE__ */ __name((request) => wakeSeen(request, settledFans, served), "seen");
8426
+ const needed = kept.filter((request) => !seen(request));
8427
+ await this.dropWakeRequests(kept.filter(seen));
8428
+ if (needed.length === 0) return;
8429
+ await this.makeDueBy(runId, this.clock() + this.leaseMs);
8111
8430
  await this.wake(runId).catch((err) => {
8112
8431
  if (!(err instanceof NamespaceMismatch)) throw err;
8113
8432
  });
8114
8433
  }
8115
- /** Take every wake request kept for `runId`; true when there was one. */
8116
- async takeWakeRequests(runId) {
8117
- let taken = false;
8118
- while (await this.store.takeBufferedSignal(wakeRequestToken(runId))) taken = true;
8119
- return taken;
8434
+ /**
8435
+ * The wake requests kept for `runId`. With `legacy`, also those an older engine kept as buffered
8436
+ * signals, which are taken as they are read.
8437
+ */
8438
+ async listWakeRequests(runId, legacy = false) {
8439
+ const token = wakeRequestToken(runId);
8440
+ const kept = (await this.store.listBufferedEvents(token, WAKE_REQUEST_SCAN)).map((request) => ({
8441
+ id: request.id,
8442
+ payload: request.payload
8443
+ }));
8444
+ if (!legacy) return kept;
8445
+ for (; ; ) {
8446
+ const signal = await this.store.takeBufferedSignal(token);
8447
+ if (!signal) return kept;
8448
+ kept.push({
8449
+ payload: signal.payload
8450
+ });
8451
+ }
8452
+ }
8453
+ /** Delete `requests` (the ones kept as buffered events). */
8454
+ async dropWakeRequests(requests) {
8455
+ await Promise.all(requests.flatMap((request) => request.id === void 0 ? [] : [
8456
+ this.store.removeBufferedEvent(request.id)
8457
+ ]));
8458
+ }
8459
+ /**
8460
+ * Record, for the turn executing `runId` here, the wakes kept for the run so far. Called before the
8461
+ * turn reads the run's state, so the turn serves them: they are dropped once it has settled, and
8462
+ * until then stay for whoever runs the run next.
8463
+ */
8464
+ async noteServedWakes(runId) {
8465
+ const served = this.turnServedWakes.get(runId);
8466
+ if (!served) return;
8467
+ for (const request of await this.store.listBufferedEvents(wakeRequestToken(runId), WAKE_REQUEST_SCAN)) {
8468
+ served.add(request.id);
8469
+ }
8470
+ }
8471
+ /**
8472
+ * Right after a turn of `runId` wrote its suspension, while it still holds the lease: if a wake
8473
+ * was kept for the run during the turn that the turn did not see, keep the run due when the lease
8474
+ * runs out. The suspension overwrote the timer the instance keeping the wake set, and this instance
8475
+ * may go down before it looks at the wake once it has released the lease.
8476
+ */
8477
+ async keepDueForKeptWakes(runId) {
8478
+ const served = this.turnServedWakes.get(runId);
8479
+ if (!served) return;
8480
+ const settledFans = this.turnSettledFans.get(runId) ?? /* @__PURE__ */ new Map();
8481
+ const kept = await this.listWakeRequests(runId);
8482
+ if (kept.every((request) => wakeSeen(request, settledFans, served))) return;
8483
+ await this.makeDueBy(runId, this.clock() + this.leaseMs);
8484
+ }
8485
+ /** Bring a suspended run's timer forward to `at` (never later than the one it has). */
8486
+ async makeDueBy(runId, at) {
8487
+ const run = await this.store.getRun(runId);
8488
+ if (run?.status !== "suspended") return;
8489
+ if (run.wakeAt !== void 0 && run.wakeAt <= at) return;
8490
+ await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, runId, [
8491
+ "suspended"
8492
+ ], {
8493
+ wakeAt: at
8494
+ });
8120
8495
  }
8121
8496
  /**
8122
8497
  * A wake that found the run leased by another instance is dropped, and normally that is fine: the
@@ -8135,6 +8510,7 @@ var WorkflowEngine = class {
8135
8510
  const remaining = Math.max(0, (lockedUntil ?? this.clock()) - this.clock());
8136
8511
  const timer = setTimeout(() => {
8137
8512
  this.leaseRetries.delete(runId);
8513
+ this.genericWakeRequested.delete(runId);
8138
8514
  this.inBackground(this.retryDroppedWake(runId, attempt));
8139
8515
  }, Math.min(remaining, LEASE_RECHECK_MAX_MS) + LEASE_RECHECK_SLACK_MS);
8140
8516
  timer.unref?.();
@@ -8143,12 +8519,15 @@ var WorkflowEngine = class {
8143
8519
  async retryDroppedWake(runId, attempt) {
8144
8520
  if (this.draining) return;
8145
8521
  const run = await this.store.getRun(runId);
8146
- if (run?.status !== "suspended") return;
8147
8522
  const nowMs = this.clock();
8148
- if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) {
8149
- if (attempt < LEASE_RECHECK_ATTEMPTS) this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
8523
+ if (run?.status === "suspended" && run.lockedUntil !== void 0 && run.lockedUntil > nowMs && attempt < LEASE_RECHECK_ATTEMPTS) {
8524
+ this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
8150
8525
  return;
8151
8526
  }
8527
+ const unconditional = this.unconditionalRechecks.delete(runId);
8528
+ if (run?.status !== "suspended") return;
8529
+ if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) return;
8530
+ if (!unconditional && (await this.listWakeRequests(runId)).length === 0) return;
8152
8531
  if (this.queuesRunsOverTransport) {
8153
8532
  await this.wake(runId);
8154
8533
  return;
@@ -8487,6 +8866,7 @@ ${run.workflowVersion}`;
8487
8866
  updatedAt
8488
8867
  });
8489
8868
  if (!applied) return this.echoCurrentStatus(run.id);
8869
+ await this.keepDueForKeptWakes(run.id);
8490
8870
  this.emit({
8491
8871
  type: "run.suspended",
8492
8872
  runId: run.id,
@@ -9180,6 +9560,7 @@ ${run.workflowVersion}`;
9180
9560
  namespace: run.namespace
9181
9561
  });
9182
9562
  }
9563
+ await this.noteServedWakes(run.id);
9183
9564
  if (run.status !== "cancelling" && registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
9184
9565
  return this.singletonWaitResult(run);
9185
9566
  }
@@ -9853,6 +10234,14 @@ ${run.workflowVersion}`;
9853
10234
  throw new WorkflowSuspended();
9854
10235
  }
9855
10236
  /**
10237
+ * Whether a `pending` step a turn just read only waits for its result: {@link callRemote} would
10238
+ * suspend on it without writing (no redispatch deadline to stamp, the one stamped not yet due).
10239
+ */
10240
+ stepIsWaiting(cp) {
10241
+ if (this.remoteRedispatchMs == null) return true;
10242
+ return cp.wakeAt != null && this.clock() < cp.wakeAt;
10243
+ }
10244
+ /**
9856
10245
  * `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
9857
10246
  * reserves the whole position block up front, so replay re-derives the same seqs.
9858
10247
  *
@@ -9882,6 +10271,7 @@ ${run.workflowVersion}`;
9882
10271
  const failures = [];
9883
10272
  const unsent = [];
9884
10273
  let inFlight = 0;
10274
+ let highest = -1;
9885
10275
  let wakeAt;
9886
10276
  for (let i = 0; i < inputs.length; i += 1) {
9887
10277
  const seq = positions[i];
@@ -9891,10 +10281,19 @@ ${run.workflowVersion}`;
9891
10281
  continue;
9892
10282
  }
9893
10283
  if (cp.name !== step.name) throw new NonDeterminismError(runId, seq, step.name, cp.name);
10284
+ if (seq > highest) highest = seq;
9894
10285
  if (cp.status === "completed") {
9895
10286
  outputs[i] = cp.output;
9896
10287
  continue;
9897
10288
  }
10289
+ if (cp.status === "pending" && this.stepIsWaiting(cp)) {
10290
+ inFlight += 1;
10291
+ if (cp.wakeAt != null) {
10292
+ const at = this.stepLeaseWakeAt(cp.wakeAt);
10293
+ wakeAt = Math.min(wakeAt ?? at, at);
10294
+ }
10295
+ continue;
10296
+ }
9898
10297
  try {
9899
10298
  outputs[i] = await this.callRemote(runId, seq, step, inputs[i], void 0, opts.transport, void 0, opts.admission, {
9900
10299
  ...routing,
@@ -9922,6 +10321,7 @@ ${run.workflowVersion}`;
9922
10321
  this.awaitedRemoteSteps.get(runId)?.delete(seq);
9923
10322
  }
9924
10323
  }
10324
+ if (inFlight === 0 && highest >= 0) this.turnSettledFans.get(runId)?.set(first, highest);
9925
10325
  const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
9926
10326
  if (batch.length > 0) await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
9927
10327
  if (inFlight + batch.length > 0) {
@@ -9952,53 +10352,62 @@ ${run.workflowVersion}`;
9952
10352
  if (failures.length > 0) throw new GatherError(failures, inputs.length);
9953
10353
  return outputs;
9954
10354
  }
9955
- /** Write the `pending` checkpoints of a step fan's new steps, then dispatch them. */
10355
+ /**
10356
+ * Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
10357
+ * {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
10358
+ * batch, where the store and transport have them.
10359
+ */
9956
10360
  async sendStepFan(runId, positions, step, inputs, batch, group, opts, routing) {
9957
10361
  const workerGroup = stepGroup(step, routing?.namespace);
9958
10362
  await this.ensureRoutable(workerGroup, step.requires);
9959
10363
  const enqueuedAt = /* @__PURE__ */ new Date();
9960
- for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
9961
- await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map((i) => {
9962
- const seq = positions[i];
9963
- return this.store.saveCheckpoint({
9964
- runId,
9965
- seq,
9966
- name: step.name,
9967
- kind: "remote",
9968
- stepId: stepId(runId, seq),
9969
- status: "pending",
9970
- input: inputs[i],
9971
- attempts: 1,
9972
- workerGroup,
9973
- parallelGroup: group,
9974
- enqueuedAt,
9975
- startedAt: enqueuedAt,
9976
- finishedAt: enqueuedAt
9977
- });
9978
- }));
10364
+ const checkpoints = batch.map((i) => {
10365
+ const seq = positions[i];
10366
+ return {
10367
+ runId,
10368
+ seq,
10369
+ name: step.name,
10370
+ kind: "remote",
10371
+ stepId: stepId(runId, seq),
10372
+ status: "pending",
10373
+ input: inputs[i],
10374
+ attempts: 1,
10375
+ workerGroup,
10376
+ parallelGroup: group,
10377
+ enqueuedAt,
10378
+ startedAt: enqueuedAt,
10379
+ finishedAt: enqueuedAt
10380
+ };
10381
+ });
10382
+ if (this.store.saveCheckpoints) {
10383
+ await this.store.saveCheckpoints(checkpoints);
10384
+ } else {
10385
+ for (let at = 0; at < checkpoints.length; at += STEP_FAN_CHUNK) {
10386
+ await Promise.all(checkpoints.slice(at, at + STEP_FAN_CHUNK).map((cp) => this.store.saveCheckpoint(cp)));
10387
+ }
9979
10388
  }
9980
- for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
9981
- await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map(async (i) => {
9982
- const seq = positions[i];
9983
- await this.dispatchRemoteTask({
9984
- runId,
9985
- seq,
9986
- name: step.name,
9987
- stepId: stepId(runId, seq),
9988
- group: workerGroup,
9989
- input: inputs[i],
9990
- priority: opts.admission?.priority,
9991
- attempt: 1,
9992
- transport: opts.transport
9993
- });
9994
- this.emit({
9995
- type: "step.started",
9996
- runId,
9997
- seq,
9998
- name: step.name,
9999
- kind: "remote"
10000
- });
10001
- }));
10389
+ const traceparent = this.traceparent?.();
10390
+ const context = this.context?.();
10391
+ await this.pool.dispatchMany(checkpoints.map((cp, k) => ({
10392
+ runId,
10393
+ seq: cp.seq,
10394
+ name: step.name,
10395
+ stepId: cp.stepId,
10396
+ group: workerGroup,
10397
+ input: inputs[batch[k]],
10398
+ traceparent,
10399
+ context,
10400
+ priority: opts.admission?.priority,
10401
+ attempt: 1
10402
+ })), opts.transport);
10403
+ for (const cp of checkpoints) {
10404
+ this.emit({
10405
+ type: "step.started",
10406
+ runId,
10407
+ seq: cp.seq,
10408
+ name: step.name,
10409
+ kind: "remote"
10410
+ });
10002
10411
  }
10003
10412
  }
10004
10413
  /** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
@@ -10039,6 +10448,27 @@ ${run.workflowVersion}`;
10039
10448
  error: s.reason
10040
10449
  });
10041
10450
  }
10451
+ /**
10452
+ * The checkpoints of one run at distinct `seqs`, each as its own outcome: one
10453
+ * {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
10454
+ * call rejecting falls back to one read each, so a failing read fails only its own entry.
10455
+ */
10456
+ async readCheckpoints(runId, seqs) {
10457
+ if (seqs.length > 1 && this.store.getCheckpoints) {
10458
+ try {
10459
+ const bySeq = new Map((await this.store.getCheckpoints(runId, seqs)).map((cp) => [
10460
+ cp.seq,
10461
+ cp
10462
+ ]));
10463
+ return seqs.map((seq) => ({
10464
+ status: "fulfilled",
10465
+ value: bySeq.get(seq) ?? null
10466
+ }));
10467
+ } catch {
10468
+ }
10469
+ }
10470
+ return Promise.allSettled(seqs.map((seq) => this.store.getCheckpoint(runId, seq)));
10471
+ }
10042
10472
  /** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
10043
10473
  * dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
10044
10474
  * {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
@@ -10080,9 +10510,10 @@ ${run.workflowVersion}`;
10080
10510
  *
10081
10511
  * Results of one run are settled in batches: the first starts a batch, and the ones that arrive
10082
10512
  * while it is being written form the next. Under load a wide fan's results are written by one
10083
- * {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked once
10084
- * per batch, not once per result. A result resolves only once its own write has committed, so the
10085
- * transport acknowledges nothing that is not durable yet.
10513
+ * {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked only
10514
+ * by a batch with no more of the fan's results queued behind it. A result resolves only once its
10515
+ * own write has committed, so the transport acknowledges nothing that is not durable yet. How
10516
+ * large a batch gets is bounded by how many results the transport hands over at once.
10086
10517
  */
10087
10518
  completeRemoteResult(result) {
10088
10519
  return new Promise((resolve, reject) => {
@@ -10110,7 +10541,7 @@ ${run.workflowVersion}`;
10110
10541
  const batch = queue.splice(0, RESULT_BATCH_MAX);
10111
10542
  let outcomes;
10112
10543
  try {
10113
- outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result));
10544
+ outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result), (from, to) => queue.some(({ result }) => result.seq >= from && result.seq <= to));
10114
10545
  } catch (err) {
10115
10546
  outcomes = batch.map(() => ({
10116
10547
  error: err
@@ -10118,8 +10549,8 @@ ${run.workflowVersion}`;
10118
10549
  }
10119
10550
  batch.forEach((q, i) => {
10120
10551
  const outcome = outcomes[i] ?? false;
10121
- if (typeof outcome === "boolean") q.resolve(outcome);
10122
- else q.reject(outcome.error);
10552
+ if (isResultError(outcome)) q.reject(outcome.error);
10553
+ else q.resolve(outcome);
10123
10554
  });
10124
10555
  }
10125
10556
  this.resultQueues.delete(runId);
@@ -10128,8 +10559,13 @@ ${run.workflowVersion}`;
10128
10559
  * Settle a batch of one run's step results. At most one of them resolves `true` (resume the run):
10129
10560
  * that resume replays every write of the batch, so a second would only replay it again. A result
10130
10561
  * whose read or write failed resolves its error, and the transport redelivers just that one.
10562
+ *
10563
+ * `queuedIn(from, to)` says whether a result for a seq in that range is already waiting for this
10564
+ * run's next batch on this instance. A fan with one waiting is not asked its barrier: that result
10565
+ * asks it once its own batch is written (as a settle, or as a redelivery of a settled step), so
10566
+ * under a backlog the barrier is read once per drained queue rather than once per batch.
10131
10567
  */
10132
- async completeRemoteResults(runId, results) {
10568
+ async completeRemoteResults(runId, results, queuedIn = () => false) {
10133
10569
  const outcomes = results.map(() => false);
10134
10570
  const seen = /* @__PURE__ */ new Set();
10135
10571
  const live = results.flatMap((r, i) => {
@@ -10139,7 +10575,7 @@ ${run.workflowVersion}`;
10139
10575
  i
10140
10576
  ];
10141
10577
  });
10142
- const reads = await Promise.allSettled(live.map((i) => this.store.getCheckpoint(runId, results[i].seq)));
10578
+ const reads = await this.readCheckpoints(runId, live.map((i) => results[i].seq));
10143
10579
  const pending = [];
10144
10580
  const resettled = [];
10145
10581
  reads.forEach((read, k) => {
@@ -10160,7 +10596,7 @@ ${run.workflowVersion}`;
10160
10596
  });
10161
10597
  if (pending.length === 0) {
10162
10598
  if (resettled.length === 0) return outcomes;
10163
- return this.resumeForRedelivered(runId, resettled, outcomes);
10599
+ return this.resumeForRedelivered(runId, resettled, outcomes, queuedIn);
10164
10600
  }
10165
10601
  for (const { cp } of pending) void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
10166
10602
  const run = await this.store.getRun(runId);
@@ -10187,7 +10623,9 @@ ${run.workflowVersion}`;
10187
10623
  cp,
10188
10624
  result: results[index]
10189
10625
  });
10190
- else if (claim !== false) outcomes[index] = claim;
10626
+ else if (claim !== false) outcomes[index] = {
10627
+ error: claim.error
10628
+ };
10191
10629
  });
10192
10630
  for (const { cp, result } of settled) {
10193
10631
  this.noteSettledHere(`${runId}:${result.seq}`);
@@ -10205,19 +10643,21 @@ ${run.workflowVersion}`;
10205
10643
  });
10206
10644
  }
10207
10645
  const barriers = /* @__PURE__ */ new Map();
10208
- let carrier;
10646
+ const wake = new FanWakeCollector();
10647
+ for (const { cp } of settled) wake.note(cp);
10209
10648
  for (const { index, cp, result } of settled) {
10210
- if (!await this.stepFanStillRunning(cp, result, barriers)) carrier = index;
10649
+ if (!await this.stepFanStillRunning(cp, result, barriers, queuedIn)) {
10650
+ wake.add(index, cp, result.status);
10651
+ }
10211
10652
  }
10212
- if (carrier !== void 0) outcomes[carrier] = true;
10213
- return outcomes;
10653
+ return wake.applyTo(outcomes);
10214
10654
  }
10215
10655
  /**
10216
10656
  * The outcomes for a batch of redelivered results (their steps already settled): one of them
10217
10657
  * resumes the run if it is `suspended` and, for a step of a fan, the fan's barrier is open. A
10218
10658
  * resume of a run that had in fact moved on replays to its current wait and re-parks.
10219
10659
  */
10220
- async resumeForRedelivered(runId, resettled, outcomes) {
10660
+ async resumeForRedelivered(runId, resettled, outcomes, queuedIn) {
10221
10661
  const run = await this.store.getRun(runId);
10222
10662
  if (run?.status !== "suspended") return outcomes;
10223
10663
  const barriers = /* @__PURE__ */ new Map();
@@ -10228,9 +10668,11 @@ ${run.workflowVersion}`;
10228
10668
  stepId: cp.stepId ?? stepId(runId, cp.seq),
10229
10669
  status: cp.status === "failed" ? "failed" : "completed"
10230
10670
  };
10231
- if (!await this.stepFanStillRunning(cp, asResult, barriers)) {
10232
- outcomes[index] = true;
10233
- break;
10671
+ if (!await this.stepFanStillRunning(cp, asResult, barriers, queuedIn)) {
10672
+ const wake = new FanWakeCollector();
10673
+ for (const settled of resettled) wake.note(settled.cp);
10674
+ wake.add(index, cp, asResult.status);
10675
+ return wake.applyTo(outcomes);
10234
10676
  }
10235
10677
  }
10236
10678
  return outcomes;
@@ -10241,15 +10683,17 @@ ${run.workflowVersion}`;
10241
10683
  * retry needs the parent to schedule it). Without it every result would replay the whole parent,
10242
10684
  * which makes a fan of N steps cost O(N²) checkpoint reads. `barriers` holds the answer per fan
10243
10685
  * for the rest of a batch. A barrier that cannot be read counts as open: a spare resume replays and
10244
- * re-suspends, a lost one strands the run.
10686
+ * re-suspends, a lost one strands the run. A fan with a result waiting in `queuedIn` is left to
10687
+ * that result (see {@link completeRemoteResults}).
10245
10688
  */
10246
- async stepFanStillRunning(cp, result, barriers) {
10689
+ async stepFanStillRunning(cp, result, barriers, queuedIn) {
10247
10690
  const fan = parseStepFanGroup(cp.parallelGroup);
10248
10691
  if (!fan) return false;
10249
10692
  if (result.status === "failed") return false;
10250
10693
  let closed = barriers.get(fan.firstSeq);
10251
10694
  if (!closed) {
10252
- closed = this.stepFanPending(result.runId, fan.firstSeq, fan.firstSeq + fan.size - 1).then((pending) => pending > 0, () => false);
10695
+ const lastSeq = fan.firstSeq + fan.size - 1;
10696
+ closed = queuedIn(fan.firstSeq, lastSeq) ? Promise.resolve(true) : this.stepFanPending(result.runId, fan.firstSeq, lastSeq).then((pending) => pending > 0, () => false);
10253
10697
  barriers.set(fan.firstSeq, closed);
10254
10698
  }
10255
10699
  if (await closed) return true;
@@ -11083,6 +11527,10 @@ var CodecStateStore = class {
11083
11527
  saveCheckpointIfPending;
11084
11528
  /** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
11085
11529
  saveCheckpointsIfPending;
11530
+ /** Encodes each checkpoint written. Bound only when the inner store has it. */
11531
+ saveCheckpoints;
11532
+ /** Decodes each checkpoint read. Bound only when the inner store has it. */
11533
+ getCheckpoints;
11086
11534
  /** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
11087
11535
  countCheckpoints;
11088
11536
  /** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
@@ -11133,6 +11581,10 @@ var CodecStateStore = class {
11133
11581
  if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
11134
11582
  const saveManyIfPending = inner.saveCheckpointsIfPending;
11135
11583
  if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
11584
+ const saveMany = inner.saveCheckpoints;
11585
+ if (saveMany) this.saveCheckpoints = (cps) => saveMany.call(inner, cps.map((cp) => this.encCp(cp)));
11586
+ const getMany = inner.getCheckpoints;
11587
+ if (getMany) this.getCheckpoints = async (runId, seqs) => (await getMany.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
11136
11588
  const countCps = inner.countCheckpoints;
11137
11589
  if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
11138
11590
  const stats = inner.checkpointStats;
@@ -11769,11 +12221,30 @@ var InMemoryStateStore = class {
11769
12221
  ...cp
11770
12222
  } : null;
11771
12223
  }
12224
+ async getCheckpoints(runId, seqs) {
12225
+ return [
12226
+ ...new Set(seqs)
12227
+ ].sort((a, b) => a - b).flatMap((seq) => {
12228
+ const cp = this.checkpoints.get(this.key(runId, seq));
12229
+ return cp ? [
12230
+ {
12231
+ ...cp
12232
+ }
12233
+ ] : [];
12234
+ });
12235
+ }
11772
12236
  async saveCheckpoint(checkpoint) {
11773
12237
  this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
11774
12238
  ...checkpoint
11775
12239
  });
11776
12240
  }
12241
+ async saveCheckpoints(checkpoints) {
12242
+ for (const checkpoint of checkpoints) {
12243
+ this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
12244
+ ...checkpoint
12245
+ });
12246
+ }
12247
+ }
11777
12248
  async saveCheckpointIfPending(checkpoint) {
11778
12249
  const key = this.key(checkpoint.runId, checkpoint.seq);
11779
12250
  if (this.checkpoints.get(key)?.status !== "pending") return false;