@dudousxd/nestjs-durable-core 0.73.0 → 0.75.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -117,6 +117,7 @@ __export(index_exports, {
117
117
  isLegacyDescriptor: () => isLegacyDescriptor,
118
118
  isNonEmptyRunScope: () => isNonEmptyRunScope,
119
119
  isWorkflowControlFlowSignal: () => isWorkflowControlFlowSignal,
120
+ listSignalWaitersOfRuns: () => listSignalWaitersOfRuns,
120
121
  log: () => log,
121
122
  matchesAttributes: () => matchesAttributes,
122
123
  mergeRunFacetRows: () => mergeRunFacetRows,
@@ -127,6 +128,7 @@ __export(index_exports, {
127
128
  normalizeAttributeRows: () => normalizeAttributeRows,
128
129
  normalizeDescriptor: () => normalizeDescriptor,
129
130
  observedAnnouncements: () => observedAnnouncements,
131
+ parentRunIdOf: () => parentRunIdOf,
130
132
  parseDuration: () => parseDuration,
131
133
  partitionOfToken: () => partitionOfToken,
132
134
  planDispatch: () => planDispatch,
@@ -550,12 +552,15 @@ var WorkflowSuspended = class extends Error {
550
552
  }
551
553
  /** Epoch ms to auto-resume (durable sleep), or undefined when waiting on an external signal. */
552
554
  wakeAt;
555
+ /** Set when awaited children are what the run is waiting on — see {@link FanJoin}. */
556
+ fan;
553
557
  /** Marks this as a control-flow signal — see {@link isWorkflowControlFlowSignal}. */
554
558
  [CONTROL_FLOW_SIGNAL] = true;
555
- constructor(wakeAt) {
559
+ constructor(wakeAt, fan) {
556
560
  super("workflow suspended");
557
561
  this.name = "WorkflowSuspended";
558
562
  this.wakeAt = wakeAt;
563
+ this.fan = fan;
559
564
  }
560
565
  };
561
566
  var ContinueAsNew = class extends Error {
@@ -982,6 +987,21 @@ function instantCheckpoint(p) {
982
987
  }
983
988
  __name(instantCheckpoint, "instantCheckpoint");
984
989
 
990
+ // src/completion.ts
991
+ function isFailedCompletion(payload) {
992
+ const c = payload;
993
+ return !!c && typeof c === "object" && "ok" in c && c.ok === false;
994
+ }
995
+ __name(isFailedCompletion, "isFailedCompletion");
996
+ function unwrapCompletion(payload, label) {
997
+ const c = payload;
998
+ if (c && typeof c === "object" && "ok" in c && c.ok === false) {
999
+ throw new FatalError(`${label} failed: ${c.error}`);
1000
+ }
1001
+ return c?.value;
1002
+ }
1003
+ __name(unwrapCompletion, "unwrapCompletion");
1004
+
985
1005
  // src/concurrency.ts
986
1006
  var DEFAULT_CONCURRENCY_STATUSES = [
987
1007
  "pending",
@@ -1221,6 +1241,81 @@ function eventMatches(payload, match) {
1221
1241
  }
1222
1242
  __name(eventMatches, "eventMatches");
1223
1243
 
1244
+ // src/fan-join.ts
1245
+ function allChildId(runId, firstSeq, index, wave) {
1246
+ return `${allBase(runId, firstSeq, wave)}${index}`;
1247
+ }
1248
+ __name(allChildId, "allChildId");
1249
+ function allGroup(firstSeq, mode) {
1250
+ return mode === "failFast" ? `all:${firstSeq}:failFast` : `all:${firstSeq}`;
1251
+ }
1252
+ __name(allGroup, "allGroup");
1253
+ function allWait(runId, firstSeq, wave) {
1254
+ const base = allBase(runId, firstSeq, wave);
1255
+ return {
1256
+ waiterPrefix: `child:${base}`,
1257
+ checkpointPrefix: `signal:child:${base}`
1258
+ };
1259
+ }
1260
+ __name(allWait, "allWait");
1261
+ function fanJoin(waits) {
1262
+ return {
1263
+ waits
1264
+ };
1265
+ }
1266
+ __name(fanJoin, "fanJoin");
1267
+ var FOLDED_PREFIX = "fan:";
1268
+ function allFoldedName(runId, firstSeq, wave) {
1269
+ return `${FOLDED_PREFIX}${allBase(runId, firstSeq, wave)}`;
1270
+ }
1271
+ __name(allFoldedName, "allFoldedName");
1272
+ function foldedChildIds(checkpoint) {
1273
+ const base = checkpoint.name.slice(FOLDED_PREFIX.length);
1274
+ const { from, values } = checkpoint.output;
1275
+ return values.map((_, j) => `${base}${from + j}`);
1276
+ }
1277
+ __name(foldedChildIds, "foldedChildIds");
1278
+ function allBase(runId, firstSeq, wave) {
1279
+ return wave === void 0 ? `${runId}.all.${firstSeq}.` : `${runId}.all.${firstSeq}.w${wave}.`;
1280
+ }
1281
+ __name(allBase, "allBase");
1282
+ function allWaves(size, concurrency) {
1283
+ if (concurrency === void 0 || concurrency >= size) return void 0;
1284
+ if (!Number.isInteger(concurrency) || concurrency < 1) {
1285
+ throw new Error(`ctx.all: concurrency must be a positive integer, got ${concurrency}`);
1286
+ }
1287
+ if (concurrency === 1) return {
1288
+ waveSize: 1,
1289
+ maxWaves: 1
1290
+ };
1291
+ return {
1292
+ waveSize: Math.floor(concurrency / 2),
1293
+ maxWaves: 2
1294
+ };
1295
+ }
1296
+ __name(allWaves, "allWaves");
1297
+ var ENGINE_CHILD_ID = /^(.+)\.(?:all\.\d+\.(?:w\d+\.)?\d+|child\.\d+)$/;
1298
+ function parentRunIdOf(runId) {
1299
+ return ENGINE_CHILD_ID.exec(runId)?.[1];
1300
+ }
1301
+ __name(parentRunIdOf, "parentRunIdOf");
1302
+ var GROUP = /^all:(\d+)(:failFast)?$/;
1303
+ var REST = /^(?:w(\d+)\.)?\d+$/;
1304
+ function fanOfWaiter(waiter) {
1305
+ const group = waiter.parallelGroup ? GROUP.exec(waiter.parallelGroup) : null;
1306
+ if (!group) return void 0;
1307
+ const base = `child:${waiter.runId}.all.${group[1]}.`;
1308
+ if (!waiter.token.startsWith(base)) return void 0;
1309
+ const rest = REST.exec(waiter.token.slice(base.length));
1310
+ if (!rest) return void 0;
1311
+ const waiterPrefix = rest[1] === void 0 ? base : `${base}w${rest[1]}.`;
1312
+ return {
1313
+ waiterPrefix,
1314
+ failFast: group[2] !== void 0
1315
+ };
1316
+ }
1317
+ __name(fanOfWaiter, "fanOfWaiter");
1318
+
1224
1319
  // src/handshake/descriptor.ts
1225
1320
  var CURRENT_PROTOCOL_VERSION = 1;
1226
1321
  var LEGACY_V1_PROTOCOL = {
@@ -1962,6 +2057,23 @@ function resolveRunWaiting(run, waiterByRun) {
1962
2057
  return waiter ? classifyWaiterToken(waiter.token) : void 0;
1963
2058
  }
1964
2059
  __name(resolveRunWaiting, "resolveRunWaiting");
2060
+ var RUN_ID_BATCH = 500;
2061
+ async function listSignalWaitersOfRuns(store, runIds) {
2062
+ const unique = [
2063
+ ...new Set(runIds)
2064
+ ];
2065
+ if (unique.length === 0) return [];
2066
+ if (!store.listSignalWaitersByRun) {
2067
+ const ids = new Set(unique);
2068
+ return (await store.listSignalWaiters("")).filter((w) => ids.has(w.runId));
2069
+ }
2070
+ const out = [];
2071
+ for (let at = 0; at < unique.length; at += RUN_ID_BATCH) {
2072
+ out.push(...await store.listSignalWaitersByRun(unique.slice(at, at + RUN_ID_BATCH)));
2073
+ }
2074
+ return out;
2075
+ }
2076
+ __name(listSignalWaitersOfRuns, "listSignalWaitersOfRuns");
1965
2077
  function indexWaitersByRun(waiters) {
1966
2078
  const byRun = /* @__PURE__ */ new Map();
1967
2079
  for (const w of waiters) if (!byRun.has(w.runId)) byRun.set(w.runId, w);
@@ -2238,12 +2350,14 @@ var ScheduleClient = class {
2238
2350
  const store = this.requireStore();
2239
2351
  const started = [];
2240
2352
  for (let round = 0; round < 50; round++) {
2353
+ if (this.host.isDraining?.()) break;
2241
2354
  const due = await store.listSchedules({
2242
2355
  dueBy: nowMs,
2243
2356
  namespace: this.options.namespace,
2244
2357
  limit: TICK_BATCH
2245
2358
  });
2246
2359
  for (const record of due) {
2360
+ if (this.host.isDraining?.()) return started;
2247
2361
  const runId = await this.fire(record, nowMs);
2248
2362
  if (runId) started.push(runId);
2249
2363
  }
@@ -2772,16 +2886,6 @@ var TransportPool = class {
2772
2886
  }
2773
2887
  };
2774
2888
 
2775
- // src/completion.ts
2776
- function unwrapCompletion(payload, label) {
2777
- const c = payload;
2778
- if (c && typeof c === "object" && "ok" in c && c.ok === false) {
2779
- throw new FatalError(`${label} failed: ${c.error}`);
2780
- }
2781
- return c?.value;
2782
- }
2783
- __name(unwrapCompletion, "unwrapCompletion");
2784
-
2785
2889
  // src/step-name-symbol.ts
2786
2890
  var DURABLE_STEP_NAME = Symbol.for("durable.step.name");
2787
2891
  var DURABLE_STEP_CONFIG = Symbol.for("durable.step.config");
@@ -2803,6 +2907,8 @@ function stepConfigOf(ref) {
2803
2907
  __name(stepConfigOf, "stepConfigOf");
2804
2908
 
2805
2909
  // src/workflow-ctx.ts
2910
+ var FAN_CHUNK = 500;
2911
+ var FOLD_MIN_FAN = 100;
2806
2912
  function normalizeChildOptions(options) {
2807
2913
  return typeof options === "string" ? {
2808
2914
  childId: options
@@ -3145,7 +3251,7 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
3145
3251
  if (existing && existing.status === "completed") {
3146
3252
  return unwrapCompletion(existing.output, `child "${id}"`);
3147
3253
  }
3148
- await store.putSignalWaiter({
3254
+ if (!existing) await store.putSignalWaiter({
3149
3255
  token: `child:${id}`,
3150
3256
  runId,
3151
3257
  seq: current
@@ -3176,16 +3282,33 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
3176
3282
  status: "running"
3177
3283
  }));
3178
3284
  }
3179
- throw new WorkflowSuspended();
3285
+ throw new WorkflowSuspended(void 0, fanJoin([
3286
+ {
3287
+ waiterPrefix: `child:${id}`,
3288
+ checkpointPrefix: `signal:child:${id}`
3289
+ }
3290
+ ]));
3180
3291
  }, "child");
3181
3292
  const all = /* @__PURE__ */ __name(async (workflow2, inputs, opts) => {
3182
3293
  if (inputs.length === 0) return [];
3183
3294
  const mode = opts?.mode ?? "waitAll";
3184
3295
  const name = workflowName(workflow2);
3296
+ const waves = allWaves(inputs.length, opts?.concurrency);
3185
3297
  const positions = inputs.map(() => pos.next());
3186
- const group = `all:${positions[0]}`;
3187
- const id = /* @__PURE__ */ __name((i) => `${runId}.all.${positions[0]}.${i}`, "id");
3188
- const existing = await Promise.all(positions.map((seq) => readCheckpoint(seq)));
3298
+ const firstSeq = positions[0];
3299
+ const group = allGroup(firstSeq, mode);
3300
+ const waveOf = /* @__PURE__ */ __name((i) => waves ? Math.floor(i / waves.waveSize) : void 0, "waveOf");
3301
+ const id = /* @__PURE__ */ __name((i) => allChildId(runId, firstSeq, i, waveOf(i)), "id");
3302
+ const { checkpoints: existing, folded } = await readFan(positions, [
3303
+ allWait(runId, firstSeq).checkpointPrefix,
3304
+ allFoldedName(runId, firstSeq)
3305
+ ]);
3306
+ const waveCount = waves ? Math.ceil(inputs.length / waves.waveSize) : 1;
3307
+ const waveItems = /* @__PURE__ */ __name((k) => {
3308
+ if (!waves) return inputs.map((_, i) => i);
3309
+ const from = k * waves.waveSize;
3310
+ return inputs.map((_, i) => i).slice(from, from + waves.waveSize);
3311
+ }, "waveItems");
3189
3312
  if (mode === "failFast") {
3190
3313
  for (let i = 0; i < inputs.length; i += 1) {
3191
3314
  const cp = existing[i];
@@ -3204,32 +3327,91 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
3204
3327
  }
3205
3328
  }
3206
3329
  }
3207
- let pending = false;
3208
- for (let i = 0; i < inputs.length; i += 1) {
3209
- const cp = existing[i];
3210
- if (cp?.status === "completed") continue;
3211
- pending = true;
3212
- const seq = positions[i];
3213
- const childId = id(i);
3214
- await store.putSignalWaiter({
3215
- token: `child:${childId}`,
3216
- runId,
3217
- seq,
3218
- parallelGroup: group
3219
- });
3220
- if (!await store.getRun(childId)) host.startChild(name, inputs[i], childId);
3221
- if (!cp) {
3222
- await writeCheckpoint(instantCheckpoint({
3330
+ if (store.compactCheckpoints && inputs.length >= FOLD_MIN_FAN) {
3331
+ for (let k = 0; k < waveCount; k += 1) {
3332
+ const items = waveItems(k);
3333
+ const first = items[0];
3334
+ const last = items[items.length - 1];
3335
+ if (folded[first]) continue;
3336
+ const values = [];
3337
+ for (const i of items) {
3338
+ const c = existing[i]?.status === "completed" ? existing[i]?.output : void 0;
3339
+ if (!c || typeof c !== "object" || c.ok !== true) break;
3340
+ values.push(c.value);
3341
+ }
3342
+ if (values.length < items.length) continue;
3343
+ const into = instantCheckpoint({
3223
3344
  runId,
3224
- seq,
3225
- name: `signal:child:${childId}`,
3345
+ seq: positions[first],
3346
+ name: allFoldedName(runId, firstSeq, waves ? k : void 0),
3226
3347
  kind: "signal",
3227
- status: "running",
3348
+ output: {
3349
+ from: first,
3350
+ values
3351
+ },
3228
3352
  parallelGroup: group
3353
+ });
3354
+ await store.compactCheckpoints(runId, positions[first], positions[last], into);
3355
+ for (const i of items) replay?.delete(positions[i]);
3356
+ replay?.set(into.seq, into);
3357
+ }
3358
+ }
3359
+ const sent = /* @__PURE__ */ __name((k) => waveItems(k).some((i) => existing[i]), "sent");
3360
+ const settled = /* @__PURE__ */ __name((k) => waveItems(k).every((i) => existing[i]?.status === "completed"), "settled");
3361
+ const active = [];
3362
+ let unsettled = 0;
3363
+ for (let k = 0; k < waveCount; k += 1) {
3364
+ if (settled(k)) continue;
3365
+ if (sent(k) || unsettled < (waves?.maxWaves ?? 1)) {
3366
+ active.push(k);
3367
+ unsettled += 1;
3368
+ }
3369
+ }
3370
+ if (active.length > 0) {
3371
+ const toRegister = [];
3372
+ const toEnsure = [];
3373
+ for (const k of active) {
3374
+ for (const i of waveItems(k)) {
3375
+ const cp = existing[i];
3376
+ if (cp?.status === "completed") continue;
3377
+ (cp ? toEnsure : toRegister).push(i);
3378
+ }
3379
+ }
3380
+ for (let at = 0; at < toRegister.length; at += FAN_CHUNK) {
3381
+ const chunk = toRegister.slice(at, at + FAN_CHUNK);
3382
+ const entries = chunk.map((i) => ({
3383
+ // Carry the fan `group` onto the waiter too: the child's terminal `signal:child:`
3384
+ // checkpoint is (re)written by engine.signal when the child notifies the parent,
3385
+ // OVERWRITING the running placeholder at the same seq — so without this the resolved
3386
+ // checkpoint would lose the group and the dashboard would render the resolved fan as a
3387
+ // sequential chain.
3388
+ waiter: {
3389
+ token: `child:${id(i)}`,
3390
+ runId,
3391
+ seq: positions[i],
3392
+ parallelGroup: group
3393
+ },
3394
+ checkpoint: instantCheckpoint({
3395
+ runId,
3396
+ seq: positions[i],
3397
+ name: `signal:child:${id(i)}`,
3398
+ kind: "signal",
3399
+ status: "running",
3400
+ parallelGroup: group
3401
+ })
3402
+ }));
3403
+ await registerChildWaits(entries);
3404
+ for (const i of chunk) host.startChild(name, inputs[i], id(i));
3405
+ }
3406
+ for (let at = 0; at < toEnsure.length; at += FAN_CHUNK) {
3407
+ await Promise.all(toEnsure.slice(at, at + FAN_CHUNK).map(async (i) => {
3408
+ if (!await store.getRun(id(i))) host.startChild(name, inputs[i], id(i));
3229
3409
  }));
3230
3410
  }
3231
3411
  }
3232
- if (pending) throw new WorkflowSuspended();
3412
+ if (active.length > 0) {
3413
+ throw new WorkflowSuspended(void 0, fanJoin(active.map((k) => allWait(runId, firstSeq, waves ? k : void 0))));
3414
+ }
3233
3415
  const outputs = [];
3234
3416
  const failures = [];
3235
3417
  for (let i = 0; i < inputs.length; i += 1) {
@@ -3248,6 +3430,67 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
3248
3430
  if (failures.length > 0) throw new GatherError(failures, inputs.length);
3249
3431
  return outputs;
3250
3432
  }, "all");
3433
+ const readFan = /* @__PURE__ */ __name(async (positions, prefixes) => {
3434
+ const checkpoints = positions.map((seq) => replay?.get(seq) ?? null);
3435
+ const folded = positions.map(() => false);
3436
+ const expand = /* @__PURE__ */ __name(() => {
3437
+ for (let i = 0; i < checkpoints.length; i += 1) {
3438
+ const cp = checkpoints[i];
3439
+ if (!cp?.name.startsWith(FOLDED_PREFIX) || folded[i]) continue;
3440
+ const fold = cp.output;
3441
+ if (!fold || typeof fold.from !== "number" || !Array.isArray(fold.values)) {
3442
+ throw new Error(`ctx.all: folded checkpoint "${cp.name}" at seq ${cp.seq} is malformed`);
3443
+ }
3444
+ const { from, values } = fold;
3445
+ values.forEach((value, j) => {
3446
+ checkpoints[from + j] = {
3447
+ ...cp,
3448
+ seq: positions[from + j],
3449
+ output: {
3450
+ ok: true,
3451
+ value
3452
+ }
3453
+ };
3454
+ folded[from + j] = true;
3455
+ });
3456
+ }
3457
+ }, "expand");
3458
+ expand();
3459
+ const missing = positions.filter((_, i) => !checkpoints[i]);
3460
+ if (missing.length === 0) return {
3461
+ checkpoints,
3462
+ folded
3463
+ };
3464
+ if (missing.length > 1 && store.listCheckpointsByNamePrefix) {
3465
+ const bySeq = /* @__PURE__ */ new Map();
3466
+ for (const cp of await store.listCheckpointsByNamePrefix(runId, prefixes)) {
3467
+ bySeq.set(cp.seq, cp);
3468
+ }
3469
+ positions.forEach((seq, i) => {
3470
+ checkpoints[i] ??= bySeq.get(seq) ?? null;
3471
+ });
3472
+ } else {
3473
+ await Promise.all(positions.map(async (seq, i) => {
3474
+ checkpoints[i] ??= await readCheckpoint(seq);
3475
+ }));
3476
+ }
3477
+ expand();
3478
+ return {
3479
+ checkpoints,
3480
+ folded
3481
+ };
3482
+ }, "readFan");
3483
+ const registerChildWaits = /* @__PURE__ */ __name(async (entries) => {
3484
+ if (store.registerChildWaits) {
3485
+ await store.registerChildWaits(entries);
3486
+ } else {
3487
+ await Promise.all(entries.map(async (e) => {
3488
+ await store.putSignalWaiter(e.waiter);
3489
+ await store.saveCheckpoint(e.checkpoint);
3490
+ }));
3491
+ }
3492
+ for (const e of entries) replay?.set(e.checkpoint.seq, e.checkpoint);
3493
+ }, "registerChildWaits");
3251
3494
  const startChild = /* @__PURE__ */ __name(async (workflow2, input, options) => {
3252
3495
  const { childId, priority, version } = normalizeChildOptions(options);
3253
3496
  const current = pos.next();
@@ -3627,6 +3870,8 @@ var WorkflowEngine = class {
3627
3870
  /** Executions currently in flight, so a graceful shutdown can wait for them to settle. */
3628
3871
  inflight = /* @__PURE__ */ new Set();
3629
3872
  draining = false;
3873
+ /** Wait already spent on this shutdown before `drain()` (see `beginDrain`). */
3874
+ drainSpentMs = 0;
3630
3875
  constructor(deps) {
3631
3876
  this.store = deps.store;
3632
3877
  this.pool = new TransportPool(deps.transports ?? (deps.transport ? [
@@ -3675,7 +3920,8 @@ var WorkflowEngine = class {
3675
3920
  });
3676
3921
  this.schedules = new ScheduleClient(this.store, {
3677
3922
  start: /* @__PURE__ */ __name((workflow, input, runId, opts) => this.start(workflow, input, runId, opts), "start"),
3678
- getRun: /* @__PURE__ */ __name((runId) => this.store.getRun(runId), "getRun")
3923
+ getRun: /* @__PURE__ */ __name((runId) => this.store.getRun(runId), "getRun"),
3924
+ isDraining: /* @__PURE__ */ __name(() => this.draining, "isDraining")
3679
3925
  }, {
3680
3926
  namespace: this.namespace,
3681
3927
  clock: this.clock
@@ -4287,10 +4533,8 @@ var WorkflowEngine = class {
4287
4533
  */
4288
4534
  async findParentRunId(childRunId) {
4289
4535
  const token = `child:${childRunId}`;
4290
- for (const waiter of await this.store.listSignalWaiters("child:")) {
4291
- if (waiter.token === token) return waiter.runId;
4292
- }
4293
- return void 0;
4536
+ const waiters = await this.store.listSignalWaiters(token);
4537
+ return waiters.find((waiter) => waiter.token === token)?.runId;
4294
4538
  }
4295
4539
  /** Track an in-flight execution so {@link drain} can wait for it. */
4296
4540
  track(p) {
@@ -4302,15 +4546,34 @@ var WorkflowEngine = class {
4302
4546
  return p;
4303
4547
  }
4304
4548
  /**
4549
+ * Stop picking up new work without waiting: recovery, timer, schedule and timeout sweeps (and
4550
+ * their per-run lock loops) start nothing once this is called, and {@link runOne} returns null. A
4551
+ * poller calls it on its own shutdown so an in-flight sweep stops locking runs before it is awaited.
4552
+ *
4553
+ * `spentMs` is wait time ALREADY spent on this shutdown (the poller awaiting its in-flight sweep).
4554
+ * It is deducted from the budget of a later {@link drain}, so that wait plus the drain stay within
4555
+ * one `shutdownTimeoutMs`. Only time actually spent waiting is charged: time that merely passes
4556
+ * between this call and `drain()` (other hooks, the HTTP server closing) never is. Calls add up.
4557
+ */
4558
+ beginDrain(spentMs = 0) {
4559
+ this.draining = true;
4560
+ this.drainSpentMs += Math.max(0, spentMs);
4561
+ }
4562
+ /**
4305
4563
  * Graceful shutdown: stop picking up new runs (recovery/timer become no-ops) and wait for
4306
4564
  * in-flight executions to settle, up to `timeoutMs`. Call from your app's shutdown hook so a
4307
4565
  * deploy hands off cleanly instead of leaving runs to the lease timeout.
4566
+ *
4567
+ * `timeoutMs` is the total budget for waiting on in-flight runs; any wait already reported through
4568
+ * {@link beginDrain} (`spentMs`) is deducted, so a shutdown that already waited on the timer poll
4569
+ * for part of it waits only for the remainder. With nothing reported, the full `timeoutMs` applies.
4308
4570
  */
4309
4571
  async drain(timeoutMs = 1e4) {
4310
- this.draining = true;
4572
+ this.beginDrain();
4573
+ const budget = Math.max(0, timeoutMs - this.drainSpentMs);
4311
4574
  if (this.inflight.size === 0) return;
4312
4575
  const timer = new Promise((resolve) => {
4313
- const t = setTimeout(resolve, timeoutMs);
4576
+ const t = setTimeout(resolve, budget);
4314
4577
  t.unref?.();
4315
4578
  });
4316
4579
  await Promise.race([
@@ -4352,6 +4615,7 @@ var WorkflowEngine = class {
4352
4615
  */
4353
4616
  async sweepTimeouts(now = this.clock()) {
4354
4617
  for (const reg of new Set(this.latest.values())) {
4618
+ if (this.draining) return;
4355
4619
  if (reg.executionTimeoutMs == null) continue;
4356
4620
  const deadline = now - reg.executionTimeoutMs;
4357
4621
  const inflight = [
@@ -4367,6 +4631,7 @@ var WorkflowEngine = class {
4367
4631
  })
4368
4632
  ];
4369
4633
  for (const run of inflight) {
4634
+ if (this.draining) return;
4370
4635
  if (run.createdAt.getTime() > deadline) continue;
4371
4636
  const error2 = {
4372
4637
  message: "execution timeout",
@@ -4396,6 +4661,7 @@ var WorkflowEngine = class {
4396
4661
  if (this.draining) return [];
4397
4662
  const results = [];
4398
4663
  for (const listed of await this.store.listIncompleteRuns(this.namespace)) {
4664
+ if (this.draining) break;
4399
4665
  const acquired = await this.store.tryLockRun(listed.id, this.instanceId, nowMs + this.leaseMs, nowMs);
4400
4666
  if (!acquired) continue;
4401
4667
  const run = await this.store.getRun(listed.id);
@@ -4511,7 +4777,10 @@ var WorkflowEngine = class {
4511
4777
  const due = await this.store.listDueTimers(nowMs, this.namespace);
4512
4778
  const blocked = await this.dueBlockedRuns(nowMs);
4513
4779
  if (due.length === 0 && blocked.length === 0) return [];
4514
- const waiterByRun = indexWaitersByRun(await this.store.listSignalWaiters(""));
4780
+ const waiterByRun = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
4781
+ ...due,
4782
+ ...blocked
4783
+ ].map((run) => run.id)));
4515
4784
  return this.resumeLeased([
4516
4785
  ...due,
4517
4786
  ...blocked
@@ -4621,11 +4890,6 @@ var WorkflowEngine = class {
4621
4890
  } : {}
4622
4891
  });
4623
4892
  } else if (cp.kind === "signal" && cp.name.startsWith("signal:child:") && cp.status === "completed" && isFailureCompletion(cp.output)) {
4624
- await this.store.saveCheckpoint({
4625
- ...cp,
4626
- status: "running",
4627
- output: void 0
4628
- });
4629
4893
  const childId = cp.name.slice("signal:child:".length);
4630
4894
  const parked = [];
4631
4895
  for (; ; ) {
@@ -4633,8 +4897,28 @@ var WorkflowEngine = class {
4633
4897
  if (!buffered) break;
4634
4898
  parked.push(buffered.payload);
4635
4899
  }
4900
+ const successAt = parked.findIndex((payload) => typeof payload === "object" && payload !== null && payload.ok === true);
4901
+ const success = successAt === -1 ? void 0 : parked.splice(successAt, 1)[0];
4636
4902
  for (const payload of parked) await this.store.bufferSignal(`child:${childId}`, payload);
4637
- const bufferedSuccess = parked.some((payload) => typeof payload === "object" && payload !== null && payload.ok === true);
4903
+ const bufferedSuccess = success !== void 0;
4904
+ if (bufferedSuccess) {
4905
+ await this.store.saveCheckpoint({
4906
+ ...cp,
4907
+ output: success
4908
+ });
4909
+ } else {
4910
+ await this.store.putSignalWaiter({
4911
+ token: `child:${childId}`,
4912
+ runId,
4913
+ seq: cp.seq,
4914
+ parallelGroup: cp.parallelGroup
4915
+ });
4916
+ await this.store.saveCheckpoint({
4917
+ ...cp,
4918
+ status: "running",
4919
+ output: void 0
4920
+ });
4921
+ }
4638
4922
  if (!bufferedSuccess) {
4639
4923
  const child = await this.store.getRun(childId);
4640
4924
  if (child && (child.status === "failed" || child.status === "dead")) {
@@ -4813,8 +5097,13 @@ var WorkflowEngine = class {
4813
5097
  if (this.draining) return [];
4814
5098
  const results = [];
4815
5099
  for (const run of runs) {
5100
+ if (this.draining) break;
4816
5101
  const acquired = await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs);
4817
5102
  if (!acquired) continue;
5103
+ if (this.draining) {
5104
+ await this.store.releaseRunLock(run.id);
5105
+ break;
5106
+ }
4818
5107
  try {
4819
5108
  const settled = onLocked ? await onLocked(run) : void 0;
4820
5109
  results.push(settled ?? await this.resumeImplicit(run.id));
@@ -4914,12 +5203,62 @@ var WorkflowEngine = class {
4914
5203
  // for an ordinary (non-fan) signal/child await.
4915
5204
  parallelGroup: waiter.parallelGroup
4916
5205
  }));
5206
+ if (await this.fanStillWaiting(waiter, payload)) {
5207
+ return {
5208
+ runId: waiter.runId,
5209
+ status: "suspended"
5210
+ };
5211
+ }
4917
5212
  return this.resumeImplicit(waiter.runId).catch((err) => {
4918
5213
  if (err instanceof NamespaceMismatch) return null;
4919
5214
  throw err;
4920
5215
  });
4921
5216
  }
4922
5217
  /**
5218
+ * The `ctx.all` join barrier: true when `waiter` was one child of a fan (or of one wave of it)
5219
+ * whose siblings are still out, so waking the parent now would only replay the fan and re-suspend.
5220
+ * Skipping that wake is what keeps a fan of N children at O(N) work instead of N replays of N items.
5221
+ *
5222
+ * Safe because the LAST sibling to settle always wakes the parent: each child consumes its own
5223
+ * waiter before it asks, so whichever asks last sees none left. A failed child of a failFast fan
5224
+ * always wakes it. Anything that is not a `ctx.all` waiter, or a store that cannot answer the
5225
+ * question cheaply, wakes the parent as before.
5226
+ */
5227
+ async fanStillWaiting(waiter, payload) {
5228
+ if (!this.store.hasSignalWaiter) return false;
5229
+ const fan = fanOfWaiter(waiter);
5230
+ if (!fan) return false;
5231
+ if (fan.failFast && isFailedCompletion(payload)) return false;
5232
+ return this.store.hasSignalWaiter(waiter.runId, fan.waiterPrefix);
5233
+ }
5234
+ /**
5235
+ * Re-drive a run that just suspended on awaited children (a `ctx.all` fan or wave, a `ctx.child`)
5236
+ * which, by now, have settled.
5237
+ *
5238
+ * {@link execute} drops a wake that finds the run's lease held. So if a child settles while the
5239
+ * parent is still executing (the turn that dispatched it, or one woken by an earlier child), that
5240
+ * wake is spent, and the turn suspends on a snapshot that predates the child. With the `ctx.all`
5241
+ * barrier that wake was the only one the fan would get. Nothing would wake the run until the
5242
+ * `reconcileMs` sweep, minutes later.
5243
+ *
5244
+ * Called after the turn released its lease, so any child that settles from here on wakes the run
5245
+ * itself. Only re-drives for a group with no waiter left whose checkpoints are all settled, which
5246
+ * the next turn is then guaranteed to see — so it cannot loop.
5247
+ */
5248
+ async redriveSettledChildren(runId, fan) {
5249
+ if (!this.store.hasSignalWaiter) return;
5250
+ for (const wait of fan.waits) {
5251
+ if (await this.store.hasSignalWaiter(runId, wait.waiterPrefix)) continue;
5252
+ const checkpoints = this.store.listCheckpointsByNamePrefix ? await this.store.listCheckpointsByNamePrefix(runId, [
5253
+ wait.checkpointPrefix
5254
+ ]) : (await this.store.listCheckpoints(runId)).filter((cp) => cp.name.startsWith(wait.checkpointPrefix));
5255
+ if (checkpoints.length > 0 && checkpoints.every((cp) => cp.status === "completed")) {
5256
+ await this.resumeImplicit(runId);
5257
+ return;
5258
+ }
5259
+ }
5260
+ }
5261
+ /**
4923
5262
  * Deliver an external signal to the run waiting on `token`, resume it with `payload`, and return
4924
5263
  * the run result — or null if no run is (or ends up) waiting for it.
4925
5264
  *
@@ -5212,23 +5551,11 @@ var WorkflowEngine = class {
5212
5551
  }
5213
5552
  /** The given runs plus every descendant, children before parents. One waiter scan for the batch. */
5214
5553
  async collectSubtrees(rootIds) {
5215
- const waitersByParent = /* @__PURE__ */ new Map();
5216
- for (const w of await this.store.listSignalWaiters("child:")) {
5217
- const list = waitersByParent.get(w.runId) ?? [];
5218
- list.push(w.token.slice("child:".length));
5219
- waitersByParent.set(w.runId, list);
5220
- }
5221
- const prefixes = [
5222
- "signal:child:",
5223
- "spawn:"
5224
- ];
5554
+ const prefixes = CHILD_EDGE_PREFIXES;
5225
5555
  const childrenOf = /* @__PURE__ */ __name(async (parent) => {
5226
- const ids = new Set(waitersByParent.get(parent) ?? []);
5556
+ const ids = new Set(await this.awaitedChildIds(parent));
5227
5557
  const cps = this.store.listCheckpointsByNamePrefix ? await this.store.listCheckpointsByNamePrefix(parent, prefixes) : (await this.store.listCheckpoints(parent)).filter((cp) => prefixes.some((p) => cp.name.startsWith(p)));
5228
- for (const cp of cps) {
5229
- if (cp.name.startsWith("signal:child:")) ids.add(cp.name.slice("signal:child:".length));
5230
- if (cp.name.startsWith("spawn:") && typeof cp.output === "string") ids.add(cp.output);
5231
- }
5558
+ for (const id of childIdsOf(cps)) ids.add(id);
5232
5559
  return [
5233
5560
  ...ids
5234
5561
  ];
@@ -5278,22 +5605,20 @@ var WorkflowEngine = class {
5278
5605
  * so a completed parent (or a completed awaited child) would otherwise drop out of the tree. The
5279
5606
  * `signal:child:<id>` checkpoint (the placeholder written when the parent first awaits the child,
5280
5607
  * overwritten as completed/failed when it settles) persists across completion, so we read both and
5281
- * dedupe — the edge stays stable for a finished run, not just a live one.
5608
+ * dedupe — the edge stays stable for a finished run, not just a live one. A settled `ctx.all`
5609
+ * group folded into one `fan:` checkpoint stands for every child in it.
5282
5610
  */
5611
+ /** The children `parentRunId` is awaiting right now, read off its own live `child:` waiters. */
5612
+ async awaitedChildIds(parentRunId) {
5613
+ return (await listSignalWaitersOfRuns(this.store, [
5614
+ parentRunId
5615
+ ])).filter((w) => w.token.startsWith("child:")).map((w) => w.token.slice("child:".length));
5616
+ }
5283
5617
  async getRunChildren(parentRunId) {
5284
- const childIds = /* @__PURE__ */ new Set();
5285
- for (const w of await this.store.listSignalWaiters("child:")) {
5286
- if (w.runId === parentRunId) childIds.add(w.token.slice("child:".length));
5287
- }
5288
- const prefixes = [
5289
- "signal:child:",
5290
- "spawn:"
5291
- ];
5618
+ const childIds = new Set(await this.awaitedChildIds(parentRunId));
5619
+ const prefixes = CHILD_EDGE_PREFIXES;
5292
5620
  const childCheckpoints = this.store.listCheckpointsByNamePrefix ? await this.store.listCheckpointsByNamePrefix(parentRunId, prefixes) : (await this.store.listCheckpoints(parentRunId)).filter((cp) => prefixes.some((p) => cp.name.startsWith(p)));
5293
- for (const cp of childCheckpoints) {
5294
- if (cp.name.startsWith("signal:child:")) childIds.add(cp.name.slice("signal:child:".length));
5295
- if (cp.name.startsWith("spawn:") && typeof cp.output === "string") childIds.add(cp.output);
5296
- }
5621
+ for (const id of childIdsOf(childCheckpoints)) childIds.add(id);
5297
5622
  return [
5298
5623
  ...childIds
5299
5624
  ];
@@ -6373,6 +6698,7 @@ var WorkflowEngine = class {
6373
6698
  const snapshot = await this.store.listCheckpoints(run.id);
6374
6699
  const replay = /* @__PURE__ */ new Map();
6375
6700
  for (const cp of snapshot) replay.set(cp.seq, cp);
6701
+ let suspendedOnFan;
6376
6702
  const ctx = createWorkflowCtx(this.ctxHostFor(replay, run.namespace), run.id, compensations, run.workflow, registered?.searchAttributesSchema);
6377
6703
  try {
6378
6704
  const output = await runInWorkflowCtx(ctx, () => fn(ctx, run.input));
@@ -6424,10 +6750,12 @@ var WorkflowEngine = class {
6424
6750
  if (this.cancelRequested.has(run.id)) {
6425
6751
  return await this.compensateAndCancel(run, compensations, replay);
6426
6752
  }
6427
- return await this.settleRun(run, {
6753
+ const suspended = await this.settleRun(run, {
6428
6754
  kind: "suspended",
6429
6755
  wakeAt: err.wakeAt
6430
6756
  });
6757
+ suspendedOnFan = err.fan;
6758
+ return suspended;
6431
6759
  }
6432
6760
  if (this.cancelRequested.has(run.id)) {
6433
6761
  return await this.compensateAndCancel(run, compensations, replay);
@@ -6447,6 +6775,9 @@ var WorkflowEngine = class {
6447
6775
  });
6448
6776
  } finally {
6449
6777
  await this.store.releaseRunLock(run.id);
6778
+ if (suspendedOnFan) {
6779
+ void this.redriveSettledChildren(run.id, suspendedOnFan).catch(() => void 0);
6780
+ }
6450
6781
  }
6451
6782
  }
6452
6783
  /**
@@ -7261,6 +7592,21 @@ var RemoteStepError = class extends Error {
7261
7592
  this.stepError = stepError;
7262
7593
  }
7263
7594
  };
7595
+ var CHILD_EDGE_PREFIXES = [
7596
+ "signal:child:",
7597
+ FOLDED_PREFIX,
7598
+ "spawn:"
7599
+ ];
7600
+ function childIdsOf(checkpoints) {
7601
+ const ids = [];
7602
+ for (const cp of checkpoints) {
7603
+ if (cp.name.startsWith("signal:child:")) ids.push(cp.name.slice("signal:child:".length));
7604
+ if (cp.name.startsWith(FOLDED_PREFIX)) ids.push(...foldedChildIds(cp));
7605
+ if (cp.name.startsWith("spawn:") && typeof cp.output === "string") ids.push(cp.output);
7606
+ }
7607
+ return ids;
7608
+ }
7609
+ __name(childIdsOf, "childIdsOf");
7264
7610
 
7265
7611
  // src/run-facets.ts
7266
7612
  function facetOrigin(origin) {
@@ -7410,6 +7756,14 @@ var CodecStateStore = class {
7410
7756
  /** Forwarded verbatim — a count reads no payload. Bound only when the inner store counts, so the
7411
7757
  * engine's facet/listing fallback still applies through the wrapper. */
7412
7758
  countRuns;
7759
+ /** The placeholder checkpoints are encoded like any other checkpoint. */
7760
+ registerChildWaits;
7761
+ /** The folded checkpoint carries the children's outputs, so it is encoded. */
7762
+ compactCheckpoints;
7763
+ /** Forwarded verbatim — a waiter carries no payload. */
7764
+ listSignalWaitersByRun;
7765
+ /** Forwarded verbatim — a waiter carries no payload. */
7766
+ hasSignalWaiter;
7413
7767
  /** Schedules persist their run `input` inside `spec` — encoded like any other payload. Bound only
7414
7768
  * when the inner store persists schedules, so `engine.schedules` reports absence faithfully. */
7415
7769
  saveSchedule;
@@ -7427,6 +7781,17 @@ var CodecStateStore = class {
7427
7781
  if (prune) this.pruneTerminalRuns = (policy, now, limit) => prune.call(inner, policy, now, limit);
7428
7782
  const count = inner.countRuns;
7429
7783
  if (count) this.countRuns = (query) => count.call(inner, query);
7784
+ const registerChildWaits = inner.registerChildWaits;
7785
+ if (registerChildWaits) this.registerChildWaits = (entries) => registerChildWaits.call(inner, entries.map((e) => ({
7786
+ waiter: e.waiter,
7787
+ checkpoint: this.encCp(e.checkpoint)
7788
+ })));
7789
+ const compact = inner.compactCheckpoints;
7790
+ if (compact) this.compactCheckpoints = (runId, fromSeq, toSeq, into) => compact.call(inner, runId, fromSeq, toSeq, this.encCp(into));
7791
+ const byRun = inner.listSignalWaitersByRun;
7792
+ if (byRun) this.listSignalWaitersByRun = (runIds) => byRun.call(inner, runIds);
7793
+ const hasWaiter = inner.hasSignalWaiter;
7794
+ if (hasWaiter) this.hasSignalWaiter = (runId, tokenPrefix) => hasWaiter.call(inner, runId, tokenPrefix);
7430
7795
  const { saveSchedule, getSchedule, updateSchedule, deleteSchedule, listSchedules } = inner;
7431
7796
  if (saveSchedule && getSchedule && updateSchedule && deleteSchedule && listSchedules) {
7432
7797
  this.saveSchedule = (r) => saveSchedule.call(inner, this.encSchedule(r));
@@ -8037,6 +8402,33 @@ var InMemoryStateStore = class {
8037
8402
  ...w
8038
8403
  }));
8039
8404
  }
8405
+ async registerChildWaits(entries) {
8406
+ for (const { waiter } of entries) this.signalWaiters.set(waiter.token, {
8407
+ ...waiter
8408
+ });
8409
+ for (const { checkpoint } of entries) await this.saveCheckpoint(checkpoint);
8410
+ }
8411
+ async compactCheckpoints(runId, fromSeq, toSeq, into) {
8412
+ for (let seq = fromSeq; seq <= toSeq; seq += 1) this.checkpoints.delete(this.key(runId, seq));
8413
+ this.checkpoints.set(this.key(runId, into.seq), {
8414
+ ...into,
8415
+ runId
8416
+ });
8417
+ }
8418
+ async listSignalWaitersByRun(runIds) {
8419
+ const ids = new Set(runIds);
8420
+ return [
8421
+ ...this.signalWaiters.values()
8422
+ ].filter((w) => ids.has(w.runId)).map((w) => ({
8423
+ ...w
8424
+ }));
8425
+ }
8426
+ async hasSignalWaiter(runId, tokenPrefix) {
8427
+ for (const w of this.signalWaiters.values()) {
8428
+ if (w.runId === runId && w.token.startsWith(tokenPrefix)) return true;
8429
+ }
8430
+ return false;
8431
+ }
8040
8432
  async removeSignalWaiter(waiter) {
8041
8433
  const current = this.signalWaiters.get(waiter.token);
8042
8434
  if (current && current.runId === waiter.runId && current.seq === waiter.seq) {
@@ -8392,6 +8784,7 @@ var InMemoryTransport = class {
8392
8784
  isLegacyDescriptor,
8393
8785
  isNonEmptyRunScope,
8394
8786
  isWorkflowControlFlowSignal,
8787
+ listSignalWaitersOfRuns,
8395
8788
  log,
8396
8789
  matchesAttributes,
8397
8790
  mergeRunFacetRows,
@@ -8402,6 +8795,7 @@ var InMemoryTransport = class {
8402
8795
  normalizeAttributeRows,
8403
8796
  normalizeDescriptor,
8404
8797
  observedAnnouncements,
8798
+ parentRunIdOf,
8405
8799
  parseDuration,
8406
8800
  partitionOfToken,
8407
8801
  planDispatch,