@dudousxd/nestjs-durable-core 0.85.7 → 0.85.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -1151,7 +1151,6 @@ var RETRYABLE = /* @__PURE__ */ new Set([
1151
1151
  "failed",
1152
1152
  "dead"
1153
1153
  ]);
1154
- var FAN_CHILD = /^(.+\.all\.\d+\.(?:w\d+\.)?)(\d+)$/;
1155
1154
  function canonicalJson(value) {
1156
1155
  const plain = JSON.stringify(value);
1157
1156
  return plain === void 0 ? "null" : sortedJson(JSON.parse(plain));
@@ -1166,77 +1165,235 @@ function sortedJson(value) {
1166
1165
  return JSON.stringify(value);
1167
1166
  }
1168
1167
  __name(sortedJson, "sortedJson");
1169
- function settledNames(childId) {
1168
+ var FAN_CHILD_PARTS = /^(.+\.all\.(\d+)\.)(?:w(\d+)\.)?(\d+)$/;
1169
+ var SEQ_CHILD = /\.child\.(\d+)$/;
1170
+ function slotOf(childId) {
1170
1171
  const signal = `${CHILD_SIGNAL_NAME_PREFIX}${childId}`;
1171
- const fan = FAN_CHILD.exec(childId);
1172
- if (!fan) return {
1173
- signal
1174
- };
1175
- return {
1176
- signal,
1177
- fold: {
1178
- name: `${FOLDED_PREFIX}${fan[1]}`,
1179
- index: Number(fan[2])
1172
+ const fan = FAN_CHILD_PARTS.exec(childId);
1173
+ if (fan) {
1174
+ const root = fan[1];
1175
+ const wave = fan[3];
1176
+ const firstSeq = Number(fan[2]);
1177
+ const i = Number(fan[4]);
1178
+ if (wave === void 0) {
1179
+ return {
1180
+ signal,
1181
+ signalSeq: firstSeq + i,
1182
+ fan: {
1183
+ fold: `${FOLDED_PREFIX}${root}`,
1184
+ firstSeq,
1185
+ index: i
1186
+ }
1187
+ };
1180
1188
  }
1189
+ return {
1190
+ signal,
1191
+ signalSeq: firstSeq + i,
1192
+ fan: {
1193
+ fold: `${FOLDED_PREFIX}${root}w${wave}.`,
1194
+ firstSeq,
1195
+ index: i,
1196
+ wave: Number(wave),
1197
+ waveZeroFold: `${FOLDED_PREFIX}${root}w0.`
1198
+ }
1199
+ };
1200
+ }
1201
+ const seq = SEQ_CHILD.exec(childId);
1202
+ return seq ? {
1203
+ signal,
1204
+ signalSeq: Number(seq[1])
1205
+ } : {
1206
+ signal
1181
1207
  };
1182
1208
  }
1183
- __name(settledNames, "settledNames");
1184
- function parentHoldsChild(checkpoints, childId, completion) {
1185
- const { signal, fold } = settledNames(childId);
1186
- const expected = canonicalJson(completion);
1187
- return checkpoints.some((cp) => {
1188
- if (cp.status !== "completed") return false;
1189
- if (cp.name === signal) return canonicalJson(cp.output) === expected;
1190
- if (fold && cp.name === fold.name) {
1191
- const { from, values } = cp.output ?? {};
1192
- return typeof from === "number" && Array.isArray(values) && fold.index >= from && fold.index < from + values.length;
1209
+ __name(slotOf, "slotOf");
1210
+ function foldOf(cp, name) {
1211
+ if (!cp || cp.status !== "completed" || cp.name !== name) return void 0;
1212
+ const { from, values } = cp.output ?? {};
1213
+ return typeof from === "number" && Array.isArray(values) ? {
1214
+ from,
1215
+ values
1216
+ } : void 0;
1217
+ }
1218
+ __name(foldOf, "foldOf");
1219
+ function covers(fold, index) {
1220
+ return !!fold && index >= fold.from && index < fold.from + fold.values.length;
1221
+ }
1222
+ __name(covers, "covers");
1223
+ function holdsSignal(cp, signal, expected) {
1224
+ return cp.status === "completed" && cp.name === signal && canonicalJson(cp.output) === expected;
1225
+ }
1226
+ __name(holdsSignal, "holdsSignal");
1227
+ async function checkpointsAt(store, runId, seqs) {
1228
+ const unique = [
1229
+ ...new Set(seqs)
1230
+ ];
1231
+ const bySeq = /* @__PURE__ */ new Map();
1232
+ if (unique.length === 0) return bySeq;
1233
+ const rows = store.getCheckpoints ? await store.getCheckpoints(runId, unique) : (await Promise.all(unique.map((seq) => store.getCheckpoint(runId, seq)))).filter((cp) => cp !== null);
1234
+ for (const cp of rows) bySeq.set(cp.seq, cp);
1235
+ return bySeq;
1236
+ }
1237
+ __name(checkpointsAt, "checkpointsAt");
1238
+ async function checkpointsNamed(store, runId, names) {
1239
+ const wanted = new Set(names);
1240
+ const rows = store.listCheckpointsByNamePrefix ? await store.listCheckpointsByNamePrefix(runId, [
1241
+ ...wanted
1242
+ ]) : await store.listCheckpoints(runId);
1243
+ return rows.filter((cp) => wanted.has(cp.name));
1244
+ }
1245
+ __name(checkpointsNamed, "checkpointsNamed");
1246
+ async function parentSettles(store, parentId, children) {
1247
+ const entries = children.map(({ childId, completion }) => ({
1248
+ slot: slotOf(childId),
1249
+ expected: canonicalJson(completion)
1250
+ }));
1251
+ const held = entries.map(() => void 0);
1252
+ const first = await checkpointsAt(store, parentId, entries.flatMap(({ slot }) => [
1253
+ ...slot.signalSeq === void 0 ? [] : [
1254
+ slot.signalSeq
1255
+ ],
1256
+ ...slot.fan ? [
1257
+ slot.fan.firstSeq
1258
+ ] : []
1259
+ ]));
1260
+ const waveFolds = [];
1261
+ const byName = [];
1262
+ entries.forEach(({ slot, expected }, at) => {
1263
+ const own = slot.signalSeq === void 0 ? void 0 : first.get(slot.signalSeq);
1264
+ if (own?.name === slot.signal) {
1265
+ held[at] = holdsSignal(own, slot.signal, expected);
1266
+ return;
1193
1267
  }
1194
- return false;
1268
+ const fan = slot.fan;
1269
+ if (fan) {
1270
+ if (covers(foldOf(own, fan.fold), fan.index)) {
1271
+ held[at] = true;
1272
+ return;
1273
+ }
1274
+ const atFirst = first.get(fan.firstSeq);
1275
+ if (fan.wave === void 0 || fan.wave === 0) {
1276
+ held[at] = covers(foldOf(atFirst, fan.fold), fan.index);
1277
+ return;
1278
+ }
1279
+ const waveZero = foldOf(atFirst, fan.waveZeroFold);
1280
+ if (waveZero && waveZero.values.length > 0) {
1281
+ waveFolds.push({
1282
+ at,
1283
+ seq: fan.firstSeq + fan.wave * waveZero.values.length
1284
+ });
1285
+ return;
1286
+ }
1287
+ byName.push(at);
1288
+ return;
1289
+ }
1290
+ if (slot.signalSeq !== void 0 && !own) {
1291
+ held[at] = false;
1292
+ return;
1293
+ }
1294
+ byName.push(at);
1195
1295
  });
1296
+ if (waveFolds.length > 0) {
1297
+ const second = await checkpointsAt(store, parentId, waveFolds.map((w) => w.seq));
1298
+ for (const { at, seq } of waveFolds) {
1299
+ const fan = entries[at].slot.fan;
1300
+ held[at] = covers(foldOf(second.get(seq), fan.fold), fan.index);
1301
+ }
1302
+ }
1303
+ if (byName.length > 0) {
1304
+ const names = byName.flatMap((at) => {
1305
+ const { slot } = entries[at];
1306
+ return slot.fan ? [
1307
+ slot.signal,
1308
+ slot.fan.fold
1309
+ ] : [
1310
+ slot.signal
1311
+ ];
1312
+ });
1313
+ const named = await checkpointsNamed(store, parentId, names);
1314
+ for (const at of byName) {
1315
+ const { slot, expected } = entries[at];
1316
+ held[at] = named.some((cp) => holdsSignal(cp, slot.signal, expected) || !!slot.fan && covers(foldOf(cp, slot.fan.fold), slot.fan.index));
1317
+ }
1318
+ }
1319
+ return held.map((h) => h === true);
1196
1320
  }
1197
- __name(parentHoldsChild, "parentHoldsChild");
1198
- async function parentCheckpoints(store, parentId, childId) {
1199
- const { signal, fold } = settledNames(childId);
1200
- const prefixes = fold ? [
1201
- signal,
1202
- fold.name
1203
- ] : [
1204
- signal
1205
- ];
1206
- return store.listCheckpointsByNamePrefix ? store.listCheckpointsByNamePrefix(parentId, prefixes) : store.listCheckpoints(parentId);
1321
+ __name(parentSettles, "parentSettles");
1322
+ function parentClosed(run) {
1323
+ return !run || TERMINAL_RUN_STATUSES.includes(run.status) && !RETRYABLE.has(run.status);
1207
1324
  }
1208
- __name(parentCheckpoints, "parentCheckpoints");
1325
+ __name(parentClosed, "parentClosed");
1209
1326
  async function childCompletionNeedsBuffer(store, childId, completion, parent) {
1210
1327
  const parentId = parentRunIdOf(childId);
1211
1328
  if (!parentId) return false;
1212
1329
  const run = parent === void 0 ? await store.getRun(parentId) : parent;
1213
- if (!run) return false;
1214
- if (TERMINAL_RUN_STATUSES.includes(run.status) && !RETRYABLE.has(run.status)) return false;
1215
- return !parentHoldsChild(await parentCheckpoints(store, parentId, childId), childId, completion);
1330
+ if (parentClosed(run)) return false;
1331
+ const [held] = await parentSettles(store, parentId, [
1332
+ {
1333
+ childId,
1334
+ completion
1335
+ }
1336
+ ]);
1337
+ return !held;
1216
1338
  }
1217
1339
  __name(childCompletionNeedsBuffer, "childCompletionNeedsBuffer");
1218
- async function unneededChildSignals(store, rows) {
1340
+ async function unneededChildSignals(store, rows, deadline = Number.POSITIVE_INFINITY) {
1341
+ const childOf = /* @__PURE__ */ __name((row) => row.token.startsWith(CHILD_TOKEN_PREFIX) ? row.token.slice(CHILD_TOKEN_PREFIX.length) : void 0, "childOf");
1219
1342
  const parentIds = rows.flatMap((row) => {
1220
- const parentId = parentRunIdOf(row.token.slice(CHILD_TOKEN_PREFIX.length));
1343
+ const childId = childOf(row);
1344
+ const parentId = childId === void 0 ? void 0 : parentRunIdOf(childId);
1221
1345
  return parentId ? [
1222
1346
  parentId
1223
1347
  ] : [];
1224
1348
  });
1225
1349
  const { runs, unread } = await readRunsByIds(store, parentIds);
1226
1350
  const unreadable = new Set(unread);
1227
- const doomed = [];
1228
- for (const row of rows) {
1229
- if (!row.token.startsWith(CHILD_TOKEN_PREFIX)) continue;
1230
- const childId = row.token.slice(CHILD_TOKEN_PREFIX.length);
1351
+ const doomed = /* @__PURE__ */ new Set();
1352
+ const open = /* @__PURE__ */ new Map();
1353
+ rows.forEach((row, at) => {
1354
+ const childId = childOf(row);
1355
+ if (childId === void 0) return;
1231
1356
  const parentId = parentRunIdOf(childId);
1232
- if (parentId && unreadable.has(parentId)) continue;
1233
- const parent = parentId ? runs.get(parentId) ?? null : null;
1357
+ if (!parentId) {
1358
+ doomed.add(row);
1359
+ return;
1360
+ }
1361
+ if (unreadable.has(parentId)) return;
1362
+ if (parentClosed(runs.get(parentId) ?? null)) {
1363
+ doomed.add(row);
1364
+ return;
1365
+ }
1366
+ const group = open.get(parentId) ?? [];
1367
+ group.push({
1368
+ at,
1369
+ row,
1370
+ childId
1371
+ });
1372
+ open.set(parentId, group);
1373
+ });
1374
+ let decided = rows.length;
1375
+ let checked = 0;
1376
+ for (const [parentId, group] of open) {
1377
+ if (checked > 0 && Date.now() >= deadline) {
1378
+ decided = group[0].at;
1379
+ break;
1380
+ }
1381
+ checked += 1;
1234
1382
  try {
1235
- if (!await childCompletionNeedsBuffer(store, childId, row.payload, parent)) doomed.push(row);
1383
+ const held = await parentSettles(store, parentId, group.map(({ row, childId }) => ({
1384
+ childId,
1385
+ completion: row.payload
1386
+ })));
1387
+ group.forEach(({ row }, k) => {
1388
+ if (held[k]) doomed.add(row);
1389
+ });
1236
1390
  } catch {
1237
1391
  }
1238
1392
  }
1239
- return doomed;
1393
+ return {
1394
+ doomed: rows.filter((row) => doomed.has(row)),
1395
+ decided
1396
+ };
1240
1397
  }
1241
1398
  __name(unneededChildSignals, "unneededChildSignals");
1242
1399
 
@@ -4857,6 +5014,7 @@ var ANNOUNCED_ROUTES_TTL_MS = 5e3;
4857
5014
  var VERSION_UNDECLARED_TAG = "version:undeclared";
4858
5015
  var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
4859
5016
  var STEP_FAN_CHUNK = 200;
5017
+ var FAN_REREAD_CHUNK = 1e3;
4860
5018
  var PARENT_NOTIFY_ATTEMPTS = 4;
4861
5019
  var PARENT_NOTIFY_BACKOFF_MS = 250;
4862
5020
  var LEASE_RECHECK_MAX_MS = 2e3;
@@ -4893,6 +5051,7 @@ var WAKE_REQUEST_SCAN = 100;
4893
5051
  var CHILD_SIGNAL_PREFIX = "signal:child:";
4894
5052
  var BUFFERED_SWEEP_PAGE = 500;
4895
5053
  var BUFFERED_SWEEP_PAGES_PER_CALL = 4;
5054
+ var BUFFERED_SWEEP_BUDGET_MS = 250;
4896
5055
  var BUFFERED_SWEEP_REST_MS = 10 * 6e4;
4897
5056
  var DELIVER_CHUNK = 50;
4898
5057
  var RESULT_BATCH_MAX = 256;
@@ -5785,13 +5944,17 @@ var WorkflowEngine = class {
5785
5944
  *
5786
5945
  * Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
5787
5946
  * at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
5788
- * few timer ticks without a long query or a big delete. A pass that reaches the end of the table
5789
- * rests {@link BUFFERED_SWEEP_REST_MS} before the next one. Call it every tick (the NestJS timer
5790
- * poller does). Skipped while draining, on a namespaced engine (a buffered signal carries no
5791
- * namespace, and a scoped read would report another tenant's parent as gone), and on a store
5792
- * without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
5793
- */
5794
- async sweepBufferedSignals(now = this.clock()) {
5947
+ * few timer ticks without a long query or a big delete. Each page is decided set-based (see
5948
+ * `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
5949
+ * read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
5950
+ * {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
5951
+ * call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
5952
+ * before the next one. Call it every tick (the NestJS timer poller does); a call made while
5953
+ * another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
5954
+ * carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
5955
+ * store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
5956
+ */
5957
+ async sweepBufferedSignals(now = this.clock(), budgetMs = BUFFERED_SWEEP_BUDGET_MS) {
5795
5958
  const { listBufferedSignals, deleteBufferedSignals } = this.store;
5796
5959
  if (!listBufferedSignals || !deleteBufferedSignals) return 0;
5797
5960
  if (this.draining || this.namespace !== void 0 || this.bufferedSweepActive) return 0;
@@ -5799,15 +5962,21 @@ var WorkflowEngine = class {
5799
5962
  return 0;
5800
5963
  }
5801
5964
  this.bufferedSweepActive = true;
5965
+ const deadline = Date.now() + budgetMs;
5802
5966
  let deleted = 0;
5803
5967
  try {
5804
5968
  for (let page = 0; page < BUFFERED_SWEEP_PAGES_PER_CALL; page++) {
5969
+ if (page > 0 && (Date.now() >= deadline || this.draining)) break;
5805
5970
  const rows = await listBufferedSignals.call(this.store, CHILD_TOKEN_PREFIX, this.bufferedSweepCursor, BUFFERED_SWEEP_PAGE);
5806
- const doomed = await unneededChildSignals(this.store, rows);
5971
+ const { doomed, decided } = await unneededChildSignals(this.store, rows, deadline);
5807
5972
  if (doomed.length > 0) {
5808
5973
  await deleteBufferedSignals.call(this.store, doomed.map((row) => row.id));
5809
5974
  deleted += doomed.length;
5810
5975
  }
5976
+ if (decided < rows.length) {
5977
+ if (decided > 0) this.bufferedSweepCursor = rows[decided - 1].id;
5978
+ break;
5979
+ }
5811
5980
  const last = rows.at(-1);
5812
5981
  if (rows.length < BUFFERED_SWEEP_PAGE || !last) {
5813
5982
  this.bufferedSweepCursor = void 0;
@@ -7282,8 +7451,7 @@ var WorkflowEngine = class {
7282
7451
  */
7283
7452
  async redriveSettledSteps(runId, seqs) {
7284
7453
  for (const seq of seqs) {
7285
- const cp = await this.store.getCheckpoint(runId, seq);
7286
- if (cp && (cp.status === "completed" || cp.status === "failed")) {
7454
+ if (await this.stepSettled(runId, seq)) {
7287
7455
  if (this.settledHere.has(`${runId}:${seq}`)) return;
7288
7456
  const nowMs = this.clock();
7289
7457
  if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
@@ -7299,6 +7467,17 @@ var WorkflowEngine = class {
7299
7467
  }
7300
7468
  }
7301
7469
  }
7470
+ /** Whether the step at `seq` is settled: a `COUNT` when the store has one, else its checkpoint. */
7471
+ async stepSettled(runId, seq) {
7472
+ if (this.store.countCheckpoints) {
7473
+ return await this.store.countCheckpoints(runId, seq, seq, [
7474
+ "completed",
7475
+ "failed"
7476
+ ]) > 0;
7477
+ }
7478
+ const cp = await this.store.getCheckpoint(runId, seq);
7479
+ return cp?.status === "completed" || cp?.status === "failed";
7480
+ }
7302
7481
  /**
7303
7482
  * {@link redriveSettledSteps} for a step fan: once the turn that suspended on it released its lease,
7304
7483
  * re-drive the run if no step of the fan is still `pending` — the last result landed on another
@@ -8770,7 +8949,7 @@ ${run.workflowVersion}`;
8770
8949
  }
8771
8950
  const wfPlan = await this.planRoute(tenantGroup(sanitizeQueueToken(remote.group), void 0), registered.requires);
8772
8951
  if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
8773
- const checkpoints = await this.store.listCheckpoints(run.id);
8952
+ const checkpoints = await this.replayCheckpoints(run.id);
8774
8953
  await this.deliverFinishedChildren(run.id, checkpoints);
8775
8954
  await this.deliverFinishedChildWaiters(run.id, checkpoints);
8776
8955
  const handoff = liveHandoff(run.id, checkpoints);
@@ -9046,6 +9225,37 @@ ${run.workflowVersion}`;
9046
9225
  * throttle for {@link rearmStepLease}, the step-scoped sibling of {@link decisionDeadlineRearms}. */
9047
9226
  stepLeaseRearms = /* @__PURE__ */ new Map();
9048
9227
  /**
9228
+ * {@link stepLeaseRearms} for the steps of a fan this instance sent, by run: one entry per sent
9229
+ * block of consecutive seqs instead of one per step, since they share the lease stamped at send.
9230
+ */
9231
+ fanLeaseRearms = /* @__PURE__ */ new Map();
9232
+ noteFanLease(runId, seqs, lease) {
9233
+ if (this.remoteRedispatchMs == null || seqs.length === 0) return;
9234
+ let from = seqs[0];
9235
+ let to = from;
9236
+ for (const seq of seqs) {
9237
+ if (seq < from) from = seq;
9238
+ if (seq > to) to = seq;
9239
+ }
9240
+ if (to - from + 1 !== seqs.length) return;
9241
+ if (this.fanLeaseRearms.size > 1024) this.fanLeaseRearms.clear();
9242
+ const blocks = this.fanLeaseRearms.get(runId) ?? [];
9243
+ blocks.push({
9244
+ from,
9245
+ to,
9246
+ lookAgainAt: lease - Math.ceil(this.remoteRedispatchMs / 2)
9247
+ });
9248
+ this.fanLeaseRearms.set(runId, blocks);
9249
+ }
9250
+ fanLeaseLookAgainAt(runId, seq, now) {
9251
+ const blocks = this.fanLeaseRearms.get(runId);
9252
+ if (!blocks) return void 0;
9253
+ const live = blocks.filter((b) => now < b.lookAgainAt);
9254
+ if (live.length === 0) this.fanLeaseRearms.delete(runId);
9255
+ else if (live.length < blocks.length) this.fanLeaseRearms.set(runId, live);
9256
+ return live.find((b) => seq >= b.from && seq <= b.to)?.lookAgainAt;
9257
+ }
9258
+ /**
9049
9259
  * A step-scoped heartbeat for a durably-suspended remote step: push its LEASE forward so a worker
9050
9260
  * that is still working on it is never presumed dead and double-dispatched. The durable counterpart
9051
9261
  * of {@link callRemoteInMemory}'s in-memory `heartbeatResets` rearm (which only exists on the
@@ -9074,7 +9284,7 @@ ${run.workflowVersion}`;
9074
9284
  if (this.remoteRedispatchMs == null) return;
9075
9285
  const id = stepId(runId, seq);
9076
9286
  const now = this.clock();
9077
- const lookAgainAt = this.stepLeaseRearms.get(id);
9287
+ const lookAgainAt = this.stepLeaseRearms.get(id) ?? this.fanLeaseLookAgainAt(runId, seq, now);
9078
9288
  if (lookAgainAt !== void 0 && now < lookAgainAt) return;
9079
9289
  try {
9080
9290
  const cp = await this.store.getCheckpoint(runId, seq);
@@ -9397,7 +9607,7 @@ ${run.workflowVersion}`;
9397
9607
  return this.singletonWaitResult(run);
9398
9608
  }
9399
9609
  const compensations = [];
9400
- const snapshot = await this.store.listCheckpoints(run.id);
9610
+ const snapshot = await this.replayCheckpoints(run.id);
9401
9611
  await this.deliverFinishedChildren(run.id, snapshot);
9402
9612
  const replay = /* @__PURE__ */ new Map();
9403
9613
  for (const cp of snapshot) replay.set(cp.seq, cp);
@@ -9727,7 +9937,7 @@ ${run.workflowVersion}`;
9727
9937
  }), "callRemote"),
9728
9938
  callRemoteAll: /* @__PURE__ */ __name((runId, positions, step, inputs, opts) => this.callRemoteAll(runId, positions, step, inputs, opts, {
9729
9939
  namespace: parentNamespace
9730
- }), "callRemoteAll"),
9940
+ }, replay), "callRemoteAll"),
9731
9941
  // Defer so a fast child can't reentrantly resume a still-running parent. Stamp the child with
9732
9942
  // the PARENT run's namespace (not this engine's) so a child belongs to the tenant/partition of
9733
9943
  // the run it was spawned from — even when an operator (namespace: undefined, drives every
@@ -10084,7 +10294,7 @@ ${run.workflowVersion}`;
10084
10294
  * concurrently with this one finds them already sent instead of sending them again. The join
10085
10295
  * barrier ({@link stepFanStillRunning}) then wakes the parent once per wave, not once per result.
10086
10296
  */
10087
- async callRemoteAll(runId, positions, step, inputs, opts, routing) {
10297
+ async callRemoteAll(runId, positions, step, inputs, opts, routing, replay) {
10088
10298
  if (step.timeoutMs) {
10089
10299
  throw new Error(`ctx.all: step "${step.name}" has a liveness timeoutMs; a step fan needs durable steps`);
10090
10300
  }
@@ -10095,10 +10305,7 @@ ${run.workflowVersion}`;
10095
10305
  const first = positions[0];
10096
10306
  const last = positions[positions.length - 1];
10097
10307
  const group = stepFanGroup(first, positions.length, opts.mode);
10098
- const bySeq = /* @__PURE__ */ new Map();
10099
- for (const cp of await this.store.listCheckpoints(runId)) {
10100
- if (cp.seq >= first && cp.seq <= last) bySeq.set(cp.seq, cp);
10101
- }
10308
+ const bySeq = await this.fanCheckpoints(runId, positions, replay);
10102
10309
  const outputs = new Array(inputs.length);
10103
10310
  const failures = [];
10104
10311
  const unsent = [];
@@ -10155,7 +10362,13 @@ ${run.workflowVersion}`;
10155
10362
  }
10156
10363
  if (inFlight === 0 && highest >= 0) this.turnSettledFans.get(runId)?.set(first, highest);
10157
10364
  const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
10158
- if (batch.length > 0) await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
10365
+ if (batch.length > 0) {
10366
+ const lease = await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
10367
+ if (lease !== void 0) {
10368
+ const at = this.stepLeaseWakeAt(lease);
10369
+ wakeAt = Math.min(wakeAt ?? at, at);
10370
+ }
10371
+ }
10159
10372
  if (inFlight + batch.length > 0) {
10160
10373
  this.awaitedStepFans.get(runId)?.push({
10161
10374
  from: first,
@@ -10185,6 +10398,34 @@ ${run.workflowVersion}`;
10185
10398
  return outputs;
10186
10399
  }
10187
10400
  /**
10401
+ * The checkpoints of a step fan's seqs, by seq. A settled one is taken from the turn's snapshot
10402
+ * (`replay`), since nothing but a turn of the run changes it once settled; one the snapshot holds
10403
+ * `pending`, or does not hold, is read again, as {@link callRemote} does for a single step: its
10404
+ * result may have landed since. The fan's last turn so reads nothing it already read.
10405
+ */
10406
+ async fanCheckpoints(runId, positions, replay) {
10407
+ const bySeq = /* @__PURE__ */ new Map();
10408
+ const reread = [];
10409
+ for (const seq of positions) {
10410
+ const cp = replay?.get(seq);
10411
+ if (cp && (cp.status === "completed" || cp.status === "failed")) bySeq.set(seq, cp);
10412
+ else reread.push(seq);
10413
+ }
10414
+ if (reread.length === 0) return bySeq;
10415
+ if (reread.length === positions.length || !this.store.getCheckpoints) {
10416
+ const wanted = new Set(reread);
10417
+ for (const cp of await this.store.listCheckpoints(runId)) {
10418
+ if (wanted.has(cp.seq)) bySeq.set(cp.seq, cp);
10419
+ }
10420
+ return bySeq;
10421
+ }
10422
+ for (let at = 0; at < reread.length; at += FAN_REREAD_CHUNK) {
10423
+ const read = await this.store.getCheckpoints(runId, reread.slice(at, at + FAN_REREAD_CHUNK));
10424
+ for (const cp of read) bySeq.set(cp.seq, cp);
10425
+ }
10426
+ return bySeq;
10427
+ }
10428
+ /**
10188
10429
  * Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
10189
10430
  * {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
10190
10431
  * batch, where the store and transport have them.
@@ -10193,6 +10434,7 @@ ${run.workflowVersion}`;
10193
10434
  const workerGroup = stepGroup(step, routing?.namespace);
10194
10435
  await this.ensureRoutable(workerGroup, step.requires);
10195
10436
  const enqueuedAt = /* @__PURE__ */ new Date();
10437
+ const lease = this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
10196
10438
  const checkpoints = batch.map((i) => {
10197
10439
  const seq = positions[i];
10198
10440
  return {
@@ -10208,7 +10450,10 @@ ${run.workflowVersion}`;
10208
10450
  parallelGroup: group,
10209
10451
  enqueuedAt,
10210
10452
  startedAt: enqueuedAt,
10211
- finishedAt: enqueuedAt
10453
+ finishedAt: enqueuedAt,
10454
+ ...lease !== void 0 ? {
10455
+ wakeAt: lease
10456
+ } : {}
10212
10457
  };
10213
10458
  });
10214
10459
  if (this.store.saveCheckpoints) {
@@ -10241,20 +10486,36 @@ ${run.workflowVersion}`;
10241
10486
  kind: "remote"
10242
10487
  });
10243
10488
  }
10489
+ if (lease !== void 0) {
10490
+ this.noteFanLease(runId, checkpoints.map((cp) => cp.seq), lease);
10491
+ }
10492
+ return lease;
10244
10493
  }
10245
10494
  /** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
10246
10495
  noteSettledHere(key) {
10247
10496
  this.noteRecent(this.settledHere, key);
10248
10497
  }
10249
- /** Record `key` in a "happened here recently" map, pruning entries older than a lease. */
10498
+ /**
10499
+ * Record `key` in a "happened here recently" map and drop the entries older than a lease. A key is
10500
+ * re-inserted at the end, so the map stays in the order its keys were noted and the stale ones are
10501
+ * all at its front: pruning stops at the first fresh entry instead of walking the whole map.
10502
+ */
10250
10503
  noteRecent(map, key) {
10251
10504
  const now = this.clock();
10505
+ map.delete(key);
10252
10506
  map.set(key, now);
10253
- if (map.size < 1024) return;
10254
10507
  for (const [k, at] of map) {
10255
- if (now - at > this.leaseMs) map.delete(k);
10508
+ if (now - at <= this.leaseMs) break;
10509
+ map.delete(k);
10256
10510
  }
10257
10511
  }
10512
+ /**
10513
+ * The run's checkpoints as a turn replays them: {@link StateStore.listCheckpointsForReplay} when
10514
+ * the store has it (no input on a completed row), else {@link StateStore.listCheckpoints}.
10515
+ */
10516
+ replayCheckpoints(runId) {
10517
+ return this.store.listCheckpointsForReplay ? this.store.listCheckpointsForReplay(runId) : this.store.listCheckpoints(runId);
10518
+ }
10258
10519
  /** {@link StateStore.saveCheckpointIfPending}, or a plain save on a store without it. */
10259
10520
  async savePendingCheckpoint(checkpoint) {
10260
10521
  if (this.store.saveCheckpointIfPending) return this.store.saveCheckpointIfPending(checkpoint);
@@ -10281,11 +10542,59 @@ ${run.workflowVersion}`;
10281
10542
  });
10282
10543
  }
10283
10544
  /**
10545
+ * Settle steps of one run at distinct seqs from their results ({@link StepSettlement}s, `settled[i]`
10546
+ * being the `pending` row `settlements[i]` settles): one
10547
+ * {@link StateStore.settleCheckpointsIfPending} call when the store has it, which writes only what
10548
+ * a result changes, else the whole rows through {@link savePendingCheckpoints}. Never rejects as a
10549
+ * whole, the same way.
10550
+ */
10551
+ async settlePendingCheckpoints(settlements, settled) {
10552
+ const settle = this.store.settleCheckpointsIfPending?.bind(this.store);
10553
+ if (!settle) {
10554
+ return this.savePendingCheckpoints(settlements.map((s, i) => ({
10555
+ ...settled[i],
10556
+ ...s
10557
+ })));
10558
+ }
10559
+ if (settlements.length > 1) {
10560
+ try {
10561
+ const wrote = await settle(settlements);
10562
+ return settlements.map((_, i) => wrote[i] === true);
10563
+ } catch {
10564
+ }
10565
+ }
10566
+ const one = await Promise.allSettled(settlements.map((s) => settle([
10567
+ s
10568
+ ])));
10569
+ return one.map((s) => s.status === "fulfilled" ? s.value[0] === true : {
10570
+ error: s.reason
10571
+ });
10572
+ }
10573
+ /**
10284
10574
  * The checkpoints of one run at distinct `seqs`, each as its own outcome: one
10285
10575
  * {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
10286
10576
  * call rejecting falls back to one read each, so a failing read fails only its own entry.
10287
10577
  */
10288
10578
  async readCheckpoints(runId, seqs) {
10579
+ const forSettle = this.store.settleCheckpointsIfPending ? this.store.getCheckpointsForSettle?.bind(this.store) : void 0;
10580
+ if (forSettle) {
10581
+ if (seqs.length > 1) {
10582
+ try {
10583
+ const bySeq = new Map((await forSettle(runId, seqs)).map((cp) => [
10584
+ cp.seq,
10585
+ cp
10586
+ ]));
10587
+ return seqs.map((seq) => ({
10588
+ status: "fulfilled",
10589
+ value: bySeq.get(seq) ?? null
10590
+ }));
10591
+ } catch {
10592
+ }
10593
+ }
10594
+ return Promise.allSettled(seqs.map(async (seq) => (await forSettle(runId, [
10595
+ seq
10596
+ ]))[0] ?? null));
10597
+ }
10289
10598
  if (seqs.length > 1 && this.store.getCheckpoints) {
10290
10599
  try {
10291
10600
  const bySeq = new Map((await this.store.getCheckpoints(runId, seqs)).map((cp) => [
@@ -10434,10 +10743,11 @@ ${run.workflowVersion}`;
10434
10743
  const run = await this.store.getRun(runId);
10435
10744
  if (run && (run.status === "cancelled" || run.status === "completed")) return outcomes;
10436
10745
  const finishedAt = /* @__PURE__ */ new Date();
10437
- const writes = pending.map(({ index, cp }) => {
10746
+ const settlements = pending.map(({ index, cp }) => {
10438
10747
  const result = results[index];
10439
10748
  return {
10440
- ...cp,
10749
+ runId,
10750
+ seq: cp.seq,
10441
10751
  status: result.status,
10442
10752
  output: result.status === "completed" ? result.output : cp.output,
10443
10753
  error: result.error,
@@ -10446,7 +10756,7 @@ ${run.workflowVersion}`;
10446
10756
  finishedAt
10447
10757
  };
10448
10758
  });
10449
- const claims = await this.savePendingCheckpoints(writes);
10759
+ const claims = await this.settlePendingCheckpoints(settlements, pending.map(({ cp }) => cp));
10450
10760
  const settled = [];
10451
10761
  claims.forEach((claim, k) => {
10452
10762
  const { index, cp } = pending[k];
@@ -10460,7 +10770,7 @@ ${run.workflowVersion}`;
10460
10770
  };
10461
10771
  });
10462
10772
  for (const { cp, result } of settled) {
10463
- this.noteSettledHere(`${runId}:${result.seq}`);
10773
+ if (!parseStepFanGroup(cp.parallelGroup)) this.noteSettledHere(`${runId}:${result.seq}`);
10464
10774
  const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
10465
10775
  this.emit({
10466
10776
  type: result.status === "completed" ? "step.completed" : "step.failed",
@@ -11359,8 +11669,16 @@ var CodecStateStore = class {
11359
11669
  saveCheckpointIfPending;
11360
11670
  /** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
11361
11671
  saveCheckpointsIfPending;
11672
+ /** Encodes each settlement's payloads like {@link saveCheckpoint}. Bound only when the inner store
11673
+ * has it, so the engine's whole-row fallback still kicks in through the wrapper. */
11674
+ settleCheckpointsIfPending;
11362
11675
  /** Encodes each checkpoint written. Bound only when the inner store has it. */
11363
11676
  saveCheckpoints;
11677
+ /** Decodes each checkpoint read like {@link listCheckpoints}. Bound only when the inner store has
11678
+ * it, so the engine's full-read fallback still kicks in through the wrapper. */
11679
+ listCheckpointsForReplay;
11680
+ /** Decodes each checkpoint read. Bound only when the inner store has it. */
11681
+ getCheckpointsForSettle;
11364
11682
  /** Decodes each checkpoint read. Bound only when the inner store has it. */
11365
11683
  getCheckpoints;
11366
11684
  /** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
@@ -11413,8 +11731,14 @@ var CodecStateStore = class {
11413
11731
  if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
11414
11732
  const saveManyIfPending = inner.saveCheckpointsIfPending;
11415
11733
  if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
11734
+ const settleMany = inner.settleCheckpointsIfPending;
11735
+ if (settleMany) this.settleCheckpointsIfPending = (settlements) => settleMany.call(inner, settlements.map((s) => this.encSettlement(s)));
11416
11736
  const saveMany = inner.saveCheckpoints;
11417
11737
  if (saveMany) this.saveCheckpoints = (cps) => saveMany.call(inner, cps.map((cp) => this.encCp(cp)));
11738
+ const listForReplay = inner.listCheckpointsForReplay;
11739
+ if (listForReplay) this.listCheckpointsForReplay = async (runId) => (await listForReplay.call(inner, runId)).map((cp) => this.decCp(cp));
11740
+ const getForSettle = inner.getCheckpointsForSettle;
11741
+ if (getForSettle) this.getCheckpointsForSettle = async (runId, seqs) => (await getForSettle.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
11418
11742
  const getMany = inner.getCheckpoints;
11419
11743
  if (getMany) this.getCheckpoints = async (runId, seqs) => (await getMany.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
11420
11744
  const countCps = inner.countCheckpoints;
@@ -11506,6 +11830,21 @@ var CodecStateStore = class {
11506
11830
  }
11507
11831
  return next;
11508
11832
  }
11833
+ encSettlement(settlement) {
11834
+ const next = {
11835
+ ...settlement,
11836
+ output: this.enc(settlement.output)
11837
+ };
11838
+ if (this.extended) {
11839
+ if (settlement.events !== void 0) {
11840
+ next.events = this.enc(settlement.events);
11841
+ }
11842
+ if (settlement.error !== void 0) {
11843
+ next.error = this.enc(settlement.error);
11844
+ }
11845
+ }
11846
+ return next;
11847
+ }
11509
11848
  decCp(cp) {
11510
11849
  if (!cp) return cp;
11511
11850
  const next = {
@@ -12065,6 +12404,18 @@ var InMemoryStateStore = class {
12065
12404
  ] : [];
12066
12405
  });
12067
12406
  }
12407
+ async getCheckpointsForSettle(runId, seqs) {
12408
+ return [
12409
+ ...new Set(seqs)
12410
+ ].sort((a, b) => a - b).flatMap((seq) => {
12411
+ const cp = this.checkpoints.get(this.key(runId, seq));
12412
+ if (!cp) return [];
12413
+ const { input: _input, ...rest } = cp;
12414
+ return [
12415
+ rest
12416
+ ];
12417
+ });
12418
+ }
12068
12419
  async saveCheckpoint(checkpoint) {
12069
12420
  this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
12070
12421
  ...checkpoint
@@ -12095,6 +12446,18 @@ var InMemoryStateStore = class {
12095
12446
  return true;
12096
12447
  });
12097
12448
  }
12449
+ async settleCheckpointsIfPending(settlements) {
12450
+ return settlements.map((settlement) => {
12451
+ const key = this.key(settlement.runId, settlement.seq);
12452
+ const cp = this.checkpoints.get(key);
12453
+ if (cp?.status !== "pending") return false;
12454
+ this.checkpoints.set(key, {
12455
+ ...cp,
12456
+ ...settlement
12457
+ });
12458
+ return true;
12459
+ });
12460
+ }
12098
12461
  async listOrphanedRuns(nowMs, limit, namespace, ownedBy) {
12099
12462
  return [
12100
12463
  ...this.runs.values()
@@ -12411,6 +12774,13 @@ var InMemoryStateStore = class {
12411
12774
  this.checkpointRowsRead += rows.length;
12412
12775
  return rows;
12413
12776
  }
12777
+ async listCheckpointsForReplay(runId) {
12778
+ return (await this.listCheckpoints(runId)).map((cp) => {
12779
+ if (cp.status !== "completed") return cp;
12780
+ const { input: _input, ...rest } = cp;
12781
+ return rest;
12782
+ });
12783
+ }
12414
12784
  runCheckpoints(runId) {
12415
12785
  return [
12416
12786
  ...this.checkpoints.values()