@dudousxd/nestjs-durable-core 0.84.1 → 0.85.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -47,6 +47,7 @@ __export(index_exports, {
47
47
  GatherError: () => GatherError,
48
48
  HANDOFF_SEQ_BASE: () => HANDOFF_SEQ_BASE,
49
49
  InMemoryAdmissionBackend: () => InMemoryAdmissionBackend,
50
+ InMemoryRunQueue: () => InMemoryRunQueue,
50
51
  InMemoryStateStore: () => InMemoryStateStore,
51
52
  InMemoryTransport: () => InMemoryTransport,
52
53
  LEGACY_V1_CAPABILITIES: () => LEGACY_V1_CAPABILITIES,
@@ -131,6 +132,7 @@ __export(index_exports, {
131
132
  fanPageEtag: () => fanPageEtag,
132
133
  findStalledRuns: () => findStalledRuns,
133
134
  foldedCount: () => foldedCount,
135
+ getRunsByIds: () => getRunsByIds,
134
136
  handoffHandlerId: () => handoffHandlerId,
135
137
  handoffName: () => handoffName,
136
138
  hasDurableExecutionWrappers: () => hasDurableExecutionWrappers,
@@ -168,6 +170,7 @@ __export(index_exports, {
168
170
  poolNamespace: () => poolNamespace,
169
171
  prevCronFireMs: () => prevCronFireMs,
170
172
  queryCheckpointsOf: () => queryCheckpointsOf,
173
+ readRunsByIds: () => readRunsByIds,
171
174
  readSearchAttributes: () => readSearchAttributes,
172
175
  requiredCapabilities: () => requiredCapabilities,
173
176
  resolveConcurrency: () => resolveConcurrency,
@@ -780,6 +783,82 @@ function backoffDelay(attempt, options) {
780
783
  }
781
784
  __name(backoffDelay, "backoffDelay");
782
785
 
786
+ // src/bounded-runner.ts
787
+ var BoundedRunner = class {
788
+ static {
789
+ __name(this, "BoundedRunner");
790
+ }
791
+ limit;
792
+ active = 0;
793
+ waiting = /* @__PURE__ */ new Map();
794
+ seq = 0;
795
+ constructor(limit) {
796
+ this.limit = limit;
797
+ }
798
+ /** Tasks running now plus tasks waiting for a slot. */
799
+ get size() {
800
+ return this.active + this.waiting.size;
801
+ }
802
+ /** Tasks running now. */
803
+ get running() {
804
+ return this.active;
805
+ }
806
+ submit(key, task) {
807
+ if (this.active < this.limit) {
808
+ this.active += 1;
809
+ queueMicrotask(() => this.execute(task));
810
+ return;
811
+ }
812
+ if (!this.waiting.has(key)) this.waiting.set(key, {
813
+ task
814
+ });
815
+ }
816
+ /**
817
+ * Run `task` under the bound and resolve once it settled, or once it was dropped by
818
+ * {@link clear} before it got a slot. Never deduplicated.
819
+ */
820
+ run(task) {
821
+ return new Promise((resolve) => {
822
+ const done = /* @__PURE__ */ __name(() => resolve(), "done");
823
+ const wrapped = /* @__PURE__ */ __name(() => task().then(done, done), "wrapped");
824
+ if (this.active < this.limit) {
825
+ this.active += 1;
826
+ queueMicrotask(() => this.execute(wrapped));
827
+ return;
828
+ }
829
+ this.seq += 1;
830
+ this.waiting.set(`\0run:${this.seq}`, {
831
+ task: wrapped,
832
+ dropped: done
833
+ });
834
+ });
835
+ }
836
+ /** Forget every task still waiting for a slot. */
837
+ clear() {
838
+ for (const { dropped } of this.waiting.values()) dropped?.();
839
+ this.waiting.clear();
840
+ }
841
+ execute(task) {
842
+ let settled;
843
+ try {
844
+ settled = task();
845
+ } catch {
846
+ settled = Promise.resolve();
847
+ }
848
+ void settled.catch(() => void 0).then(() => {
849
+ const next = this.waiting.entries().next();
850
+ if (next.done) {
851
+ this.active -= 1;
852
+ return;
853
+ }
854
+ const [key, waiting] = next.value;
855
+ this.waiting.delete(key);
856
+ this.execute(waiting.task);
857
+ });
858
+ }
859
+ };
860
+ var LOCAL_RUN_CONCURRENCY = 256;
861
+
783
862
  // src/execution-hooks.ts
784
863
  var wrappers = [];
785
864
  function useDurableExecution(wrapper) {
@@ -2340,6 +2419,61 @@ var RemoteWorkflowExecutor = class {
2340
2419
  }
2341
2420
  };
2342
2421
 
2422
+ // src/run-dispatch-batcher.ts
2423
+ var RUN_DISPATCH_BATCH = 500;
2424
+ var RUN_DISPATCH_WINDOW_MS = 5;
2425
+ var RunDispatchBatcher = class {
2426
+ static {
2427
+ __name(this, "RunDispatchBatcher");
2428
+ }
2429
+ send;
2430
+ onError;
2431
+ buffer = /* @__PURE__ */ new Map();
2432
+ timer;
2433
+ sending = /* @__PURE__ */ new Set();
2434
+ constructor(send, onError) {
2435
+ this.send = send;
2436
+ this.onError = onError;
2437
+ }
2438
+ /** Nothing buffered and nothing being sent. */
2439
+ get idle() {
2440
+ return this.buffer.size === 0 && this.sending.size === 0;
2441
+ }
2442
+ /** Buffer `run`. A wake for a run already buffered replaces its start: a wake also runs a `pending` run. */
2443
+ add(run) {
2444
+ const prior = this.buffer.get(run.runId);
2445
+ if (prior && run.reason !== "wake") return;
2446
+ this.buffer.set(run.runId, run);
2447
+ if (this.buffer.size >= RUN_DISPATCH_BATCH) {
2448
+ this.flush();
2449
+ return;
2450
+ }
2451
+ if (!this.timer) {
2452
+ this.timer = setTimeout(() => this.flush(), RUN_DISPATCH_WINDOW_MS);
2453
+ this.timer.unref?.();
2454
+ }
2455
+ }
2456
+ /** Send what is buffered now. */
2457
+ flush() {
2458
+ if (this.timer) clearTimeout(this.timer);
2459
+ this.timer = void 0;
2460
+ if (this.buffer.size === 0) return;
2461
+ const runs = [
2462
+ ...this.buffer.values()
2463
+ ];
2464
+ this.buffer = /* @__PURE__ */ new Map();
2465
+ const sent = this.send(runs).catch((err) => this.onError(runs, err)).finally(() => this.sending.delete(sent));
2466
+ this.sending.add(sent);
2467
+ }
2468
+ /** Send what is buffered and wait for every send in flight. */
2469
+ async settle() {
2470
+ this.flush();
2471
+ await Promise.allSettled([
2472
+ ...this.sending
2473
+ ]);
2474
+ }
2475
+ };
2476
+
2343
2477
  // src/run-waiting.ts
2344
2478
  var WEBHOOK_PREFIX = "wh:";
2345
2479
  var CHILD_PREFIX = "child:";
@@ -2410,6 +2544,51 @@ function indexWaitersByRun(waiters) {
2410
2544
  return byRun;
2411
2545
  }
2412
2546
  __name(indexWaitersByRun, "indexWaitersByRun");
2547
+ var GET_RUN_FALLBACK_PARALLEL = 50;
2548
+ async function getRunsByIds(store, runIds) {
2549
+ const { runs, unread, error: error2 } = await readRunsByIds(store, runIds);
2550
+ if (unread.length > 0) throw error2;
2551
+ return runs;
2552
+ }
2553
+ __name(getRunsByIds, "getRunsByIds");
2554
+ async function readRunsByIds(store, runIds) {
2555
+ const unique = [
2556
+ ...new Set(runIds)
2557
+ ];
2558
+ const runs = /* @__PURE__ */ new Map();
2559
+ const unread = [];
2560
+ let error2;
2561
+ const batch = store.getRuns ? RUN_ID_BATCH : GET_RUN_FALLBACK_PARALLEL;
2562
+ for (let at = 0; at < unique.length; at += batch) {
2563
+ const ids = unique.slice(at, at + batch);
2564
+ if (store.getRuns) {
2565
+ try {
2566
+ for (const run of await store.getRuns(ids)) runs.set(run.id, run);
2567
+ } catch (err) {
2568
+ error2 ??= err;
2569
+ unread.push(...ids);
2570
+ }
2571
+ continue;
2572
+ }
2573
+ const reads = await Promise.allSettled(ids.map((id) => store.getRun(id)));
2574
+ reads.forEach((read, k) => {
2575
+ if (read.status === "fulfilled") {
2576
+ if (read.value) runs.set(read.value.id, read.value);
2577
+ return;
2578
+ }
2579
+ error2 ??= read.reason;
2580
+ unread.push(ids[k]);
2581
+ });
2582
+ }
2583
+ return {
2584
+ runs,
2585
+ unread,
2586
+ ...unread.length > 0 ? {
2587
+ error: error2
2588
+ } : {}
2589
+ };
2590
+ }
2591
+ __name(readRunsByIds, "readRunsByIds");
2413
2592
 
2414
2593
  // src/scheduler.ts
2415
2594
  var import_node_module = require("module");
@@ -3302,6 +3481,39 @@ var TransportPool = class {
3302
3481
  transport.onStartRun?.(handler);
3303
3482
  }
3304
3483
  }
3484
+ /** Does any transport here carry a run queue ({@link Transport.dispatchRuns} + `onDispatchedRun`)? */
3485
+ get dispatchesRuns() {
3486
+ return this.transports.some(({ transport }) => transport.dispatchRuns && transport.onDispatchedRun);
3487
+ }
3488
+ /**
3489
+ * Queue `runs` on the first transport that carries a run queue, failing over to the next on an
3490
+ * error, like {@link dispatch}. Throws if none accepted them.
3491
+ */
3492
+ async dispatchRuns(runs, opts) {
3493
+ let lastErr;
3494
+ for (const { transport } of this.transports) {
3495
+ if (!transport.dispatchRuns || !transport.onDispatchedRun) continue;
3496
+ try {
3497
+ await transport.dispatchRuns(runs, opts);
3498
+ return;
3499
+ } catch (err) {
3500
+ lastErr = err;
3501
+ }
3502
+ }
3503
+ throw lastErr ?? new Error("no transport carries a run queue");
3504
+ }
3505
+ /**
3506
+ * Consume the run queue of every transport that carries one (a run may have been queued on any of
3507
+ * them after a failover). Returns one stop function per consumer.
3508
+ */
3509
+ onDispatchedRun(handler, opts) {
3510
+ const stops = [];
3511
+ for (const { transport } of this.transports) {
3512
+ if (!transport.dispatchRuns || !transport.onDispatchedRun) continue;
3513
+ stops.push(transport.onDispatchedRun(handler, opts));
3514
+ }
3515
+ return stops;
3516
+ }
3305
3517
  /**
3306
3518
  * The live handshake descriptors advertised for `group` (design §7.2), merged across every transport
3307
3519
  * that can report them. Returns `[]` when NO transport implements
@@ -3834,21 +4046,28 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
3834
4046
  return inputs.map((_, i) => i).slice(from, from + waves.waveSize);
3835
4047
  }, "waveItems");
3836
4048
  if (mode === "failFast") {
3837
- for (let i = 0; i < inputs.length; i += 1) {
3838
- const cp = existing[i];
3839
- const c = cp?.status === "completed" ? cp.output : null;
3840
- if (c && c.ok === false) {
3841
- for (let j = 0; j < inputs.length; j += 1) {
3842
- if (j !== i && existing[j]?.status !== "completed") host.cancelChild?.(id(j));
3843
- }
3844
- throw new GatherError([
3845
- {
3846
- index: i,
3847
- id: id(i),
3848
- error: c.error ?? "unknown"
3849
- }
3850
- ], inputs.length);
4049
+ const failure = /* @__PURE__ */ __name((i2) => {
4050
+ const cp = existing[i2];
4051
+ const c2 = cp?.status === "completed" ? cp.output : null;
4052
+ return c2 && c2.ok === false ? c2 : null;
4053
+ }, "failure");
4054
+ const first = /* @__PURE__ */ __name((cancelled) => inputs.findIndex((_, i2) => {
4055
+ const c2 = failure(i2);
4056
+ return c2 !== null && c2.code === "cancelled" === cancelled;
4057
+ }), "first");
4058
+ const i = first(false) !== -1 ? first(false) : first(true);
4059
+ const c = i === -1 ? null : failure(i);
4060
+ if (c) {
4061
+ for (let j = 0; j < inputs.length; j += 1) {
4062
+ if (j !== i && existing[j]?.status !== "completed") host.cancelChild?.(id(j));
3851
4063
  }
4064
+ throw new GatherError([
4065
+ {
4066
+ index: i,
4067
+ id: id(i),
4068
+ error: c.error ?? "unknown"
4069
+ }
4070
+ ], inputs.length);
3852
4071
  }
3853
4072
  }
3854
4073
  if (store.compactCheckpoints && inputs.length >= FOLD_MIN_FAN) {
@@ -3927,10 +4146,12 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
3927
4146
  await registerChildWaits(entries);
3928
4147
  for (const i of chunk) host.startChild(name, inputs[i], id(i));
3929
4148
  }
3930
- for (let at = 0; at < toEnsure.length; at += FAN_CHUNK) {
3931
- await Promise.all(toEnsure.slice(at, at + FAN_CHUNK).map(async (i) => {
3932
- if (!await store.getRun(id(i))) host.startChild(name, inputs[i], id(i));
3933
- }));
4149
+ if (toEnsure.length > 0) {
4150
+ const { runs: started, unread } = await readRunsByIds(store, toEnsure.map((i) => id(i)));
4151
+ const skip = new Set(unread);
4152
+ for (const i of toEnsure) {
4153
+ if (!started.has(id(i)) && !skip.has(id(i))) host.startChild(name, inputs[i], id(i));
4154
+ }
3934
4155
  }
3935
4156
  }
3936
4157
  if (active.length > 0) {
@@ -4390,6 +4611,56 @@ var ANNOUNCED_ROUTES_TTL_MS = 5e3;
4390
4611
  var VERSION_UNDECLARED_TAG = "version:undeclared";
4391
4612
  var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
4392
4613
  var STEP_FAN_CHUNK = 200;
4614
+ var PARENT_NOTIFY_ATTEMPTS = 4;
4615
+ var PARENT_NOTIFY_BACKOFF_MS = 250;
4616
+ var LEASE_RECHECK_MAX_MS = 2e3;
4617
+ var LEASE_RECHECK_SLACK_MS = 100;
4618
+ var LEASE_RECHECK_ATTEMPTS = 30;
4619
+ function deferred() {
4620
+ let resolve;
4621
+ let reject;
4622
+ const promise = new Promise((res, rej) => {
4623
+ resolve = res;
4624
+ reject = rej;
4625
+ });
4626
+ promise.catch(() => void 0);
4627
+ return {
4628
+ promise,
4629
+ resolve,
4630
+ reject
4631
+ };
4632
+ }
4633
+ __name(deferred, "deferred");
4634
+ var pendingStateKey = /* @__PURE__ */ __name((at) => String(new Date(at).getTime()), "pendingStateKey");
4635
+ var freshPendingKey = /* @__PURE__ */ __name(() => `${Date.now()}-${globalThis.crypto.randomUUID().slice(0, 8)}`, "freshPendingKey");
4636
+ var CHILD_SIGNAL_PREFIX = "signal:child:";
4637
+ var DELIVER_CHUNK = 50;
4638
+ function childCompletion(child) {
4639
+ if (child.status === "completed") return {
4640
+ ok: true,
4641
+ value: child.output
4642
+ };
4643
+ if (child.status === "failed") return failedCompletion(child.error ?? {
4644
+ message: "failed"
4645
+ });
4646
+ if (child.status === "dead") return failedCompletion(child.error ?? {
4647
+ message: "dead"
4648
+ });
4649
+ if (child.status === "cancelled") return cancelledCompletion(child.error);
4650
+ return void 0;
4651
+ }
4652
+ __name(childCompletion, "childCompletion");
4653
+ var CHILD_CANCELLED = "cancelled";
4654
+ function cancelledCompletion(error2) {
4655
+ const base = error2 ?? {
4656
+ message: "cancelled"
4657
+ };
4658
+ return failedCompletion({
4659
+ ...base,
4660
+ code: base.code ?? CHILD_CANCELLED
4661
+ });
4662
+ }
4663
+ __name(cancelledCompletion, "cancelledCompletion");
4393
4664
  var RESULT_BATCH_MAX = 256;
4394
4665
  function shuffle(items) {
4395
4666
  for (let i = items.length - 1; i > 0; i -= 1) {
@@ -4566,6 +4837,35 @@ var WorkflowEngine = class {
4566
4837
  /** Executions currently in flight, so a graceful shutdown can wait for them to settle. */
4567
4838
  inflight = /* @__PURE__ */ new Set();
4568
4839
  /**
4840
+ * Work a settled turn leaves running in the background (a parent notification, a post-release
4841
+ * re-drive), so a graceful shutdown waits for it too: dropping one on exit strands the run until
4842
+ * its reconcile timer.
4843
+ */
4844
+ background = /* @__PURE__ */ new Set();
4845
+ /**
4846
+ * Runs with an implicit wake executing on this instance, and whether another wake arrived for the
4847
+ * run meanwhile (see {@link resumeImplicit}).
4848
+ */
4849
+ wakeTurns = /* @__PURE__ */ new Map();
4850
+ /** Re-checks scheduled for runs whose wake found another instance's lease (see {@link execute}). */
4851
+ leaseRetries = /* @__PURE__ */ new Map();
4852
+ /**
4853
+ * Every run turn and child start this instance takes on itself, under one budget of
4854
+ * {@link LOCAL_RUN_CONCURRENCY}: the runs it dispatches to itself, a fan's deferred child starts,
4855
+ * enqueued pickups and the runs it takes off the transport's run queue.
4856
+ */
4857
+ localWork;
4858
+ /** The dispatcher the caller supplied, if any; without one the engine picks (see {@link dispatchRun}). */
4859
+ customDispatcher;
4860
+ /** Runs on their way to the transport's run queue (see {@link consumeRuns}). */
4861
+ runBatcher;
4862
+ /** Set while this engine consumes a transport's run queue, and so dispatches its runs to it. */
4863
+ dispatchesOverTransport = false;
4864
+ /** Stops for the run-queue consumers started by {@link consumeRuns}. */
4865
+ runConsumerStops = [];
4866
+ /** The consumers being stopped, so {@link drain} waits for their runs in flight. */
4867
+ runConsumersStopping;
4868
+ /**
4569
4869
  * The run lease of each execution in flight on THIS instance, by run id — renewed while it runs
4570
4870
  * and checked by every settle (the zombie fence, see {@link ExecutionLease}). Also what `drain()`
4571
4871
  * releases when its budget runs out.
@@ -4621,15 +4921,27 @@ var WorkflowEngine = class {
4621
4921
  this.rehydrate = deps.rehydrate ?? ((_carrier, fn) => fn());
4622
4922
  this.compensationRetries = Math.max(1, deps.compensationRetries ?? 1);
4623
4923
  this.trackStepStart = deps.trackStepStart ?? true;
4624
- this.runDispatcher = deps.runDispatcher ?? {
4625
- dispatch: /* @__PURE__ */ __name((runId) => queueMicrotask(() => void this.runOne(runId).catch(() => {
4626
- })), "dispatch")
4924
+ this.localWork = new BoundedRunner(LOCAL_RUN_CONCURRENCY);
4925
+ this.customDispatcher = deps.runDispatcher;
4926
+ this.runBatcher = new RunDispatchBatcher((runs) => this.pool.dispatchRuns(runs, {
4927
+ namespace: this.namespace
4928
+ }), (runs, err) => {
4929
+ for (const run of runs) {
4930
+ if (run.reason === "wake" && !this.draining) {
4931
+ this.inBackground(this.resumeImplicit(run.runId));
4932
+ }
4933
+ }
4934
+ console.error(`[nestjs-durable] could not queue ${runs.length} run(s) on the transport; starts stay pending for the next poll, wakes resume here:`, err);
4935
+ });
4936
+ this.runDispatcher = {
4937
+ dispatch: /* @__PURE__ */ __name((runId) => this.dispatchRun(runId), "dispatch")
4627
4938
  };
4628
4939
  this.singletons = new SingletonGate({
4629
4940
  store: this.store,
4630
4941
  clock: this.clock,
4631
4942
  dispatch: /* @__PURE__ */ __name((runId) => {
4632
- this.runDispatcher.dispatch(runId);
4943
+ if (this.queuesRunsOverTransport) void this.wake(runId);
4944
+ else this.runDispatcher.dispatch(runId);
4633
4945
  }, "dispatch"),
4634
4946
  configFor: /* @__PURE__ */ __name((run) => (this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? this.latest.get(run.workflow))?.singleton, "configFor")
4635
4947
  });
@@ -4754,13 +5066,224 @@ var WorkflowEngine = class {
4754
5066
  }
4755
5067
  /**
4756
5068
  * Be notified when a run is enqueued on ANOTHER instance (via the control plane), so a worker can
4757
- * pick it up at once — e.g. `engine.onEnqueued((runId) => engine.runOne(runId))`. Returns an
5069
+ * pick it up at once — e.g. `engine.onEnqueued((runId) => engine.runEnqueued(runId))`. Returns an
4758
5070
  * unsubscribe function. Only wire this on instances that should execute runs (workers).
4759
5071
  */
4760
5072
  onEnqueued(listener) {
4761
5073
  this.enqueuedListeners.add(listener);
4762
5074
  return () => this.enqueuedListeners.delete(listener);
4763
5075
  }
5076
+ /**
5077
+ * Pick up a run another instance announced as enqueued: run it here if it is still `pending`.
5078
+ * Bounded by {@link LOCAL_RUN_CONCURRENCY}, and a run another instance already took is skipped
5079
+ * after one read, instead of being leased and replayed here once more.
5080
+ */
5081
+ runEnqueued(runId) {
5082
+ this.localWork.submit(`run:${runId}`, () => this.runIfPending(runId));
5083
+ }
5084
+ /**
5085
+ * Consume the run queue of every transport that carries one ({@link Transport.dispatchRuns} and
5086
+ * {@link Transport.onDispatchedRun}), and from then on dispatch the runs this engine starts there
5087
+ * rather than executing them in-process: a run, a `ctx.all` fan's children included, executes on
5088
+ * whichever consuming instance takes it, at most {@link LOCAL_RUN_CONCURRENCY} at once per
5089
+ * instance. Call it on every instance that drives runs; the Nest module does on a driving operator.
5090
+ *
5091
+ * Returns a function that stops consuming and resolves once the runs in flight settled, after
5092
+ * which runs are executed in-process again; {@link beginDrain} stops consuming too. Undefined
5093
+ * when no transport carries a run queue: runs keep executing in-process.
5094
+ */
5095
+ consumeRuns() {
5096
+ if (!this.pool.dispatchesRuns) return void 0;
5097
+ if (this.runConsumerStops.length === 0) {
5098
+ this.runConsumerStops = this.pool.onDispatchedRun((run) => this.localWork.run(() => this.runFromQueue(run)), {
5099
+ concurrency: LOCAL_RUN_CONCURRENCY,
5100
+ namespace: this.namespace
5101
+ });
5102
+ }
5103
+ this.dispatchesOverTransport = true;
5104
+ return async () => {
5105
+ this.dispatchesOverTransport = false;
5106
+ await this.runBatcher.settle();
5107
+ await this.stopRunConsumers();
5108
+ };
5109
+ }
5110
+ /** Stop every run-queue consumer; resolves once their runs in flight settled. */
5111
+ stopRunConsumers() {
5112
+ if (this.runConsumerStops.length > 0) {
5113
+ const stops = this.runConsumerStops;
5114
+ this.runConsumerStops = [];
5115
+ this.runConsumersStopping = Promise.allSettled(stops.map((stop) => stop()));
5116
+ }
5117
+ return this.runConsumersStopping ?? Promise.resolve();
5118
+ }
5119
+ /** Whether {@link dispatchRun} queues runs on the transport (see {@link consumeRuns}). */
5120
+ get queuesRunsOverTransport() {
5121
+ return this.customDispatcher === void 0 && this.dispatchesOverTransport;
5122
+ }
5123
+ /**
5124
+ * Hand a `pending` run to whatever executes it: the caller's {@link WorkflowEngineDeps.runDispatcher};
5125
+ * else the transport's run queue while this engine consumes it, batched; else this instance,
5126
+ * bounded by {@link LOCAL_RUN_CONCURRENCY}. A run whose queueing fails stays `pending` for a
5127
+ * `runPending` poll.
5128
+ */
5129
+ dispatchRun(runId, priority, key) {
5130
+ if (this.customDispatcher) return this.customDispatcher.dispatch(runId);
5131
+ if (this.dispatchesOverTransport) {
5132
+ this.runBatcher.add({
5133
+ runId,
5134
+ reason: "start",
5135
+ ...priority !== void 0 ? {
5136
+ priority
5137
+ } : {},
5138
+ ...key !== void 0 ? {
5139
+ key
5140
+ } : {}
5141
+ });
5142
+ return;
5143
+ }
5144
+ this.localWork.submit(`run:${runId}`, () => this.runOne(runId));
5145
+ }
5146
+ /**
5147
+ * Wake a suspended run that has something new to see (a delivered signal or child completion, a
5148
+ * lease that came free, a freed slot): while this engine consumes the transport's run queue, on
5149
+ * whichever instance takes the wake (the result then only says it was handed over); otherwise
5150
+ * here, now.
5151
+ */
5152
+ wake(runId) {
5153
+ if (this.queuesRunsOverTransport) {
5154
+ this.runBatcher.add({
5155
+ runId,
5156
+ reason: "wake"
5157
+ });
5158
+ return Promise.resolve({
5159
+ runId,
5160
+ status: "suspended"
5161
+ });
5162
+ }
5163
+ return this.resumeImplicit(runId);
5164
+ }
5165
+ /**
5166
+ * Take a turn of a run taken off the transport's run queue. A `start` runs only while the run is
5167
+ * still `pending` (or `cancelling`, a recovered cancel): any other state means another instance
5168
+ * already took it. A `wake` leases and resumes the run unless it is finished. The run lease admits
5169
+ * one executor either way. Never rejects.
5170
+ */
5171
+ async runFromQueue(dispatch) {
5172
+ if (this.draining) return;
5173
+ const { runId } = dispatch;
5174
+ try {
5175
+ const run = await this.store.getRun(runId);
5176
+ if (!run) return;
5177
+ if (run.status === "pending" || run.status === "cancelling") {
5178
+ await this.runOne(runId);
5179
+ return;
5180
+ }
5181
+ if (dispatch.reason !== "wake" || TERMINAL_RUN_STATUSES.includes(run.status)) return;
5182
+ const nowMs = this.clock();
5183
+ if (run.wakeAt !== void 0 && run.wakeAt <= nowMs) {
5184
+ const waiters = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
5185
+ runId
5186
+ ]));
5187
+ const resumed = await this.resumeLeased([
5188
+ run
5189
+ ], nowMs, (locked) => this.reconcileSignalWaiter(locked, waiters.get(locked.id)));
5190
+ if (resumed.length === 0) this.recheckAfterLease(runId, run.lockedUntil);
5191
+ return;
5192
+ }
5193
+ await this.resumeImplicit(runId).catch((err) => {
5194
+ if (!(err instanceof NamespaceMismatch)) throw err;
5195
+ });
5196
+ } catch (error2) {
5197
+ console.error(`[nestjs-durable] could not run queued run ${runId}:`, error2);
5198
+ }
5199
+ }
5200
+ /**
5201
+ * Hand due runs (timers, blocked runs due for a fleet re-check) to whichever instance takes them
5202
+ * off the run queue. Each is claimed first, under its lease, by pushing its `wakeAt` one lease
5203
+ * period out, so the next poll (here or elsewhere) does not queue it again while it waits; a wake
5204
+ * that is lost brings it back due after that period. False when the transport refused the wakes,
5205
+ * so the caller resumes them itself.
5206
+ */
5207
+ async handOffDue(runs, nowMs) {
5208
+ const claimed = [];
5209
+ for (const run of runs) {
5210
+ if (this.draining) break;
5211
+ if (!await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs)) {
5212
+ continue;
5213
+ }
5214
+ try {
5215
+ const patch = {
5216
+ wakeAt: nowMs + this.leaseMs
5217
+ };
5218
+ if (this.store.updateRunIf) {
5219
+ if (await this.store.updateRunIf(run.id, [
5220
+ run.status
5221
+ ], patch)) claimed.push(run);
5222
+ } else {
5223
+ await this.store.updateRun(run.id, patch);
5224
+ claimed.push(run);
5225
+ }
5226
+ } finally {
5227
+ await this.store.releaseRunLock(run.id);
5228
+ }
5229
+ }
5230
+ try {
5231
+ for (let at = 0; at < claimed.length; at += RUN_DISPATCH_BATCH) {
5232
+ await this.pool.dispatchRuns(claimed.slice(at, at + RUN_DISPATCH_BATCH).map((run) => ({
5233
+ runId: run.id,
5234
+ reason: "wake",
5235
+ ...run.priority !== void 0 ? {
5236
+ priority: run.priority
5237
+ } : {}
5238
+ })), {
5239
+ namespace: this.namespace
5240
+ });
5241
+ }
5242
+ return true;
5243
+ } catch (error2) {
5244
+ console.error("[nestjs-durable] could not queue due runs on the transport:", error2);
5245
+ return false;
5246
+ }
5247
+ }
5248
+ /**
5249
+ * Queue `runs` on the transport for whichever consuming instance takes them, in batches. False
5250
+ * when the transport refused them, so the caller executes them itself instead.
5251
+ */
5252
+ async queueOnTransport(runs) {
5253
+ try {
5254
+ for (let at = 0; at < runs.length; at += RUN_DISPATCH_BATCH) {
5255
+ await this.pool.dispatchRuns(runs.slice(at, at + RUN_DISPATCH_BATCH).map((run) => ({
5256
+ runId: run.id,
5257
+ reason: "start",
5258
+ key: pendingStateKey(run.updatedAt),
5259
+ ...run.priority !== void 0 ? {
5260
+ priority: run.priority
5261
+ } : {}
5262
+ })), {
5263
+ namespace: this.namespace
5264
+ });
5265
+ }
5266
+ return true;
5267
+ } catch (error2) {
5268
+ console.error("[nestjs-durable] could not queue runs on the transport; running them here:", error2);
5269
+ return false;
5270
+ }
5271
+ }
5272
+ /**
5273
+ * Execute a run announced by another instance, if it is still `pending`: a
5274
+ * run another instance already took is skipped after one read instead of being leased and
5275
+ * replayed here once more. Never rejects.
5276
+ */
5277
+ async runIfPending(runId) {
5278
+ if (this.draining) return;
5279
+ try {
5280
+ const run = await this.store.getRun(runId);
5281
+ if (run?.status !== "pending") return;
5282
+ await this.runOne(runId);
5283
+ } catch (error2) {
5284
+ console.error(`[nestjs-durable] could not run enqueued run ${runId}:`, error2);
5285
+ }
5286
+ }
4764
5287
  /** Fire cooperative-cancellation listeners for `runId` (a worker bridge aborts in-flight work). */
4765
5288
  notifyCancelled(runId) {
4766
5289
  for (const fn of this.cancelListeners) {
@@ -5156,8 +5679,8 @@ var WorkflowEngine = class {
5156
5679
  status: "suspended"
5157
5680
  };
5158
5681
  }
5159
- await this.runDispatcher.dispatch(runId);
5160
- if (this.controlPlane) {
5682
+ await this.dispatchRun(runId, run.priority, pendingStateKey(run.updatedAt));
5683
+ if (this.controlPlane && !this.queuesRunsOverTransport) {
5161
5684
  void this.controlPlane.publishControl({
5162
5685
  kind: "enqueued",
5163
5686
  runId,
@@ -5196,7 +5719,43 @@ var WorkflowEngine = class {
5196
5719
  * stale waiter or a late result left behind by a failed run silently re-executes its body.
5197
5720
  */
5198
5721
  resumeImplicit(runId) {
5199
- return this.resumeRun(runId, false);
5722
+ const turn = this.wakeTurns.get(runId);
5723
+ if (turn) {
5724
+ turn.again ??= deferred();
5725
+ if (currentWorkflowCtx()?.runId === runId) {
5726
+ return Promise.resolve({
5727
+ runId,
5728
+ status: "running"
5729
+ });
5730
+ }
5731
+ return turn.again.promise;
5732
+ }
5733
+ const mine = {};
5734
+ this.wakeTurns.set(runId, mine);
5735
+ const finish = /* @__PURE__ */ __name(() => {
5736
+ if (this.wakeTurns.get(runId) === mine) this.wakeTurns.delete(runId);
5737
+ const again = mine.again;
5738
+ if (!again) return;
5739
+ if (this.draining) {
5740
+ again.resolve({
5741
+ runId,
5742
+ status: "running"
5743
+ });
5744
+ return;
5745
+ }
5746
+ const next = this.resumeImplicit(runId);
5747
+ next.then(again.resolve, again.reject);
5748
+ this.inBackground(next);
5749
+ }, "finish");
5750
+ const result = this.resumeRun(runId, false);
5751
+ void result.then(finish, finish);
5752
+ return result;
5753
+ }
5754
+ /** Keep `work` in {@link background} until it settles; its failure is swallowed. */
5755
+ inBackground(work) {
5756
+ const tracked = work.catch(() => void 0);
5757
+ this.background.add(tracked);
5758
+ void tracked.then(() => this.background.delete(tracked));
5200
5759
  }
5201
5760
  /**
5202
5761
  * One execution of a run at a time in this process. A resume that lands while the run executes
@@ -5483,6 +6042,10 @@ var WorkflowEngine = class {
5483
6042
  */
5484
6043
  beginDrain(spentMs = 0) {
5485
6044
  this.draining = true;
6045
+ if (this.runConsumerStops.length > 0) {
6046
+ this.runBatcher.flush();
6047
+ void this.stopRunConsumers();
6048
+ }
5486
6049
  if (Number.isFinite(spentMs) && spentMs > 0) {
5487
6050
  this.drainSpentMs += spentMs;
5488
6051
  for (const rearm of this.drainRearms) rearm();
@@ -5503,7 +6066,14 @@ var WorkflowEngine = class {
5503
6066
  */
5504
6067
  async drain(timeoutMs = 1e4) {
5505
6068
  this.beginDrain();
5506
- if (this.inflight.size === 0) return;
6069
+ this.localWork.clear();
6070
+ for (const timer2 of this.leaseRetries.values()) clearTimeout(timer2);
6071
+ this.leaseRetries.clear();
6072
+ const consumers = this.runConsumersStopping;
6073
+ if (this.inflight.size === 0 && this.background.size === 0 && !consumers && this.runBatcher.idle) {
6074
+ return;
6075
+ }
6076
+ const queued = this.runBatcher.settle();
5507
6077
  const start = Date.now();
5508
6078
  let rearm = /* @__PURE__ */ __name(() => {
5509
6079
  }, "rearm");
@@ -5522,7 +6092,10 @@ var WorkflowEngine = class {
5522
6092
  try {
5523
6093
  await Promise.race([
5524
6094
  Promise.allSettled([
5525
- ...this.inflight
6095
+ ...this.inflight,
6096
+ ...this.background,
6097
+ queued,
6098
+ consumers
5526
6099
  ]).then(() => {
5527
6100
  settled = true;
5528
6101
  }),
@@ -5629,6 +6202,7 @@ var WorkflowEngine = class {
5629
6202
  error: error2
5630
6203
  });
5631
6204
  await this.cancelChildren(run.id).catch(() => void 0);
6205
+ this.notifyParent(run.id, cancelledCompletion(error2));
5632
6206
  }
5633
6207
  }
5634
6208
  }
@@ -5643,7 +6217,9 @@ var WorkflowEngine = class {
5643
6217
  if (this.draining) break;
5644
6218
  const outcome = this.isOwnLiveLease(listed, nowMs) ? await this.reclaimOwnLease(listed, nowMs) : await this.recoverOrphan(listed, nowMs);
5645
6219
  if (!outcome) continue;
5646
- if (outcome.dispatch) await this.runDispatcher.dispatch(outcome.result.runId);
6220
+ if (outcome.dispatch) {
6221
+ await this.dispatchRun(outcome.result.runId, listed.priority, freshPendingKey());
6222
+ }
5647
6223
  results.push(outcome.result);
5648
6224
  }
5649
6225
  return results;
@@ -5835,6 +6411,18 @@ var WorkflowEngine = class {
5835
6411
  const due = shuffle(await this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize));
5836
6412
  const blocked = await this.dueBlockedRuns(nowMs);
5837
6413
  if (due.length === 0 && blocked.length === 0) return [];
6414
+ if (this.queuesRunsOverTransport && !this.draining) {
6415
+ const runs = [
6416
+ ...due,
6417
+ ...blocked
6418
+ ];
6419
+ if (await this.handOffDue(runs, nowMs)) {
6420
+ return runs.map((run) => ({
6421
+ runId: run.id,
6422
+ status: run.status
6423
+ }));
6424
+ }
6425
+ }
5838
6426
  const waiterByRun = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
5839
6427
  ...due,
5840
6428
  ...blocked
@@ -5868,7 +6456,7 @@ var WorkflowEngine = class {
5868
6456
  const buffered = await this.store.takeBufferedSignal(waiter.token);
5869
6457
  if (!buffered) return void 0;
5870
6458
  await this.store.removeSignalWaiter(waiter);
5871
- const settled = await this.deliverSignal(waiter, buffered.payload);
6459
+ const settled = await this.deliverSignal(waiter, buffered.payload, true);
5872
6460
  return settled ?? {
5873
6461
  runId: run.id,
5874
6462
  status: run.status
@@ -5894,7 +6482,7 @@ var WorkflowEngine = class {
5894
6482
  if (!eventMatches(candidate.payload, match)) continue;
5895
6483
  if (!await this.store.removeBufferedEvent(candidate.id)) continue;
5896
6484
  await this.store.removeSignalWaiter(waiter);
5897
- const settled = await this.deliverSignal(waiter, candidate.payload);
6485
+ const settled = await this.deliverSignal(waiter, candidate.payload, true);
5898
6486
  return settled ?? {
5899
6487
  runId: run.id,
5900
6488
  status: run.status
@@ -6014,7 +6602,7 @@ var WorkflowEngine = class {
6014
6602
  } : {},
6015
6603
  updatedAt: /* @__PURE__ */ new Date()
6016
6604
  });
6017
- await this.runDispatcher.dispatch(runId);
6605
+ await this.dispatchRun(runId, run.priority, freshPendingKey());
6018
6606
  return {
6019
6607
  runId,
6020
6608
  status: "pending"
@@ -6089,7 +6677,16 @@ var WorkflowEngine = class {
6089
6677
  * {@link resumeDueTimers}.
6090
6678
  */
6091
6679
  async runPending(nowMs = this.clock()) {
6092
- return this.resumeLeased(await this.store.listPendingRuns(100, this.namespace), nowMs);
6680
+ const pending = await this.store.listPendingRuns(100, this.namespace);
6681
+ if (this.queuesRunsOverTransport && !this.draining && pending.length > 0) {
6682
+ if (await this.queueOnTransport(pending)) {
6683
+ return pending.map((run) => ({
6684
+ runId: run.id,
6685
+ status: run.status
6686
+ }));
6687
+ }
6688
+ }
6689
+ return this.resumeLeased(pending, nowMs);
6093
6690
  }
6094
6691
  /**
6095
6692
  * Resolve once `runId` reaches a settled state — terminal (completed/failed/cancelled/dead) or
@@ -6266,7 +6863,7 @@ var WorkflowEngine = class {
6266
6863
  * due-timer reconcile sweep ({@link reconcileSignalWaiter}) — all three resolve a waiter the exact
6267
6864
  * same way.
6268
6865
  */
6269
- async deliverSignal(waiter, payload) {
6866
+ async deliverSignal(waiter, payload, resumeHere = false) {
6270
6867
  await this.store.saveCheckpoint(instantCheckpoint({
6271
6868
  runId: waiter.runId,
6272
6869
  seq: waiter.seq,
@@ -6285,7 +6882,7 @@ var WorkflowEngine = class {
6285
6882
  status: "suspended"
6286
6883
  };
6287
6884
  }
6288
- return this.resumeImplicit(waiter.runId).catch((err) => {
6885
+ return (resumeHere ? this.resumeImplicit(waiter.runId) : this.wake(waiter.runId)).catch((err) => {
6289
6886
  if (err instanceof NamespaceMismatch) return null;
6290
6887
  throw err;
6291
6888
  });
@@ -6372,6 +6969,101 @@ var WorkflowEngine = class {
6372
6969
  return;
6373
6970
  }
6374
6971
  }
6972
+ /**
6973
+ * Deliver the outcome of every awaited child that already finished but whose completion never
6974
+ * reached this run, into `checkpoints` (this turn's snapshot) and the store.
6975
+ *
6976
+ * A child persists its terminal status first and tells its parent after ({@link notifyParent}). A
6977
+ * crash, a scale-down or a store error between the two leaves the parent's `running` placeholder
6978
+ * and `child:<id>` waiter behind for a child that is done, and nothing else would ever settle
6979
+ * them: the parent's reconcile wake replays straight back to the same wait. So each turn looks up
6980
+ * the children of its `running` placeholders (one batched read per 500) and settles the finished
6981
+ * ones exactly as the child's own notification would have: the `signal:child:<id>` checkpoint
6982
+ * with the child's completion, then the waiter removed. A notification arriving late writes the
6983
+ * same checkpoint and finds no waiter. Best effort: a store error leaves the rest to the next turn.
6984
+ */
6985
+ async deliverFinishedChildren(runId, checkpoints) {
6986
+ const awaited = [];
6987
+ for (const cp of checkpoints) {
6988
+ if (cp.kind !== "signal" || cp.status !== "running") continue;
6989
+ if (!cp.name.startsWith(CHILD_SIGNAL_PREFIX)) continue;
6990
+ awaited.push({
6991
+ childId: cp.name.slice(CHILD_SIGNAL_PREFIX.length),
6992
+ seq: cp.seq,
6993
+ parallelGroup: cp.parallelGroup
6994
+ });
6995
+ }
6996
+ await this.settleFinishedChildren(runId, awaited, checkpoints);
6997
+ }
6998
+ /**
6999
+ * {@link deliverFinishedChildren} for a remote workflow, whose awaited children (`startChild`
7000
+ * commands) leave a `child:<id>` waiter but no placeholder checkpoint: the run's own waiters say
7001
+ * which children it is waiting on. One indexed read per turn of a remote run.
7002
+ */
7003
+ async deliverFinishedChildWaiters(runId, checkpoints) {
7004
+ let waiters;
7005
+ try {
7006
+ waiters = await listSignalWaitersOfRuns(this.store, [
7007
+ runId
7008
+ ]);
7009
+ } catch {
7010
+ return;
7011
+ }
7012
+ const settled = new Set(checkpoints.filter((cp) => cp.status === "completed").map((cp) => cp.seq));
7013
+ const awaited = waiters.filter((w) => w.token.startsWith("child:") && !settled.has(w.seq)).map((w) => ({
7014
+ childId: w.token.slice("child:".length),
7015
+ seq: w.seq,
7016
+ parallelGroup: w.parallelGroup
7017
+ }));
7018
+ await this.settleFinishedChildren(runId, awaited, checkpoints);
7019
+ }
7020
+ /**
7021
+ * Settle every child in `awaited` that has finished: its `signal:child:<id>` checkpoint with the
7022
+ * child's completion, then its waiter removed, exactly as the child's own notification does, and
7023
+ * the checkpoint put into `checkpoints`. Children are read in batches of 500.
7024
+ */
7025
+ async settleFinishedChildren(runId, awaited, checkpoints) {
7026
+ if (awaited.length === 0) return;
7027
+ try {
7028
+ const { runs: children } = await readRunsByIds(this.store, awaited.map((a) => a.childId));
7029
+ const finished = awaited.flatMap((a) => {
7030
+ const child = children.get(a.childId);
7031
+ const completion = child ? childCompletion(child) : void 0;
7032
+ return completion ? [
7033
+ {
7034
+ ...a,
7035
+ completion
7036
+ }
7037
+ ] : [];
7038
+ });
7039
+ for (let at = 0; at < finished.length; at += DELIVER_CHUNK) {
7040
+ await Promise.all(finished.slice(at, at + DELIVER_CHUNK).map(async (a) => {
7041
+ const settled = instantCheckpoint({
7042
+ runId,
7043
+ seq: a.seq,
7044
+ name: `${CHILD_SIGNAL_PREFIX}${a.childId}`,
7045
+ kind: "signal",
7046
+ output: a.completion,
7047
+ parallelGroup: a.parallelGroup
7048
+ });
7049
+ await this.store.saveCheckpoint(settled);
7050
+ await this.store.removeSignalWaiter({
7051
+ token: `child:${a.childId}`,
7052
+ runId,
7053
+ seq: a.seq,
7054
+ ...a.parallelGroup !== void 0 ? {
7055
+ parallelGroup: a.parallelGroup
7056
+ } : {}
7057
+ });
7058
+ const index = checkpoints.findIndex((cp) => cp.seq === a.seq);
7059
+ if (index === -1) checkpoints.push(settled);
7060
+ else checkpoints[index] = settled;
7061
+ }));
7062
+ }
7063
+ checkpoints.sort((a, b) => a.seq - b.seq);
7064
+ } catch {
7065
+ }
7066
+ }
6375
7067
  async redriveSettledChildren(runId, fan) {
6376
7068
  if (!this.store.hasSignalWaiter) return;
6377
7069
  for (const wait of fan.waits) {
@@ -6470,13 +7162,63 @@ var WorkflowEngine = class {
6470
7162
  * knowing about the child feature.
6471
7163
  */
6472
7164
  notifyParent(runId, completion) {
6473
- void this.signal(`child:${runId}`, completion).catch(() => void 0);
7165
+ this.inBackground(this.deliverToParent(`child:${runId}`, completion));
6474
7166
  const at = runId.lastIndexOf("~retry~");
6475
7167
  if (at !== -1 && completion.ok) {
6476
- void this.signal(`child:${runId.slice(0, at)}`, completion).catch(() => void 0);
7168
+ this.inBackground(this.deliverToParent(`child:${runId.slice(0, at)}`, completion));
6477
7169
  }
6478
7170
  }
6479
7171
  /**
7172
+ * {@link signal} a child's completion to its parent, retrying a store error a few times. The child
7173
+ * is already persisted terminal, so a failure here would leave the parent waiting on a child that
7174
+ * finished; the parent's own turns reconcile that ({@link deliverFinishedChildren}), but only on
7175
+ * its next wake. A waiter already taken is delivered to directly on a retry, so the completion is
7176
+ * not buffered for a waiter that no longer exists.
7177
+ */
7178
+ async deliverToParent(token, completion) {
7179
+ let waiter;
7180
+ const cancelled = !completion.ok && completion.code === CHILD_CANCELLED;
7181
+ for (let attempt = 1; ; attempt += 1) {
7182
+ try {
7183
+ if (waiter === void 0) waiter = await this.store.takeSignalWaiter(token);
7184
+ if (waiter) {
7185
+ const parent = cancelled ? await this.store.getRun(waiter.runId) : void 0;
7186
+ if (parent && TERMINAL_RUN_STATUSES.includes(parent.status)) return;
7187
+ await this.deliverSignal(waiter, completion);
7188
+ return;
7189
+ }
7190
+ if (cancelled || await this.parentHoldsCompletion(token, completion)) return;
7191
+ await this.signal(token, completion);
7192
+ return;
7193
+ } catch (err) {
7194
+ if (err instanceof NamespaceMismatch) return;
7195
+ if (attempt >= PARENT_NOTIFY_ATTEMPTS || this.draining) return;
7196
+ await new Promise((resolve) => {
7197
+ const timer = setTimeout(resolve, PARENT_NOTIFY_BACKOFF_MS * 4 ** (attempt - 1));
7198
+ timer.unref?.();
7199
+ });
7200
+ }
7201
+ }
7202
+ }
7203
+ /**
7204
+ * Whether the parent of the child behind `token` (`child:<id>`) already holds `completion` as that
7205
+ * child's settled `signal:child:<id>` checkpoint: its own turn settled the child from the child's
7206
+ * row, so this notification is a duplicate. A different completion (a fix-and-replay run's success
7207
+ * for a child the parent saw fail) is not, and is buffered for the parent's retry. Only for a child
7208
+ * whose id the engine minted (its parent is derivable from it).
7209
+ */
7210
+ async parentHoldsCompletion(token, completion) {
7211
+ const childId = token.slice("child:".length);
7212
+ const parentId = parentRunIdOf(childId);
7213
+ if (!parentId) return false;
7214
+ const name = `${CHILD_SIGNAL_PREFIX}${childId}`;
7215
+ const checkpoints = this.store.listCheckpointsByNamePrefix ? await this.store.listCheckpointsByNamePrefix(parentId, [
7216
+ name
7217
+ ]) : await this.store.listCheckpoints(parentId);
7218
+ const expected = JSON.stringify(completion);
7219
+ return checkpoints.some((cp) => cp.name === name && cp.status === "completed" && JSON.stringify(cp.output) === expected);
7220
+ }
7221
+ /**
6480
7222
  * Deferred child start shared by the in-process ctx host and the remote `startChild` command.
6481
7223
  * Deferred (microtask) so a fast child can't reentrantly resume a still-suspending parent. A start
6482
7224
  * that THROWS (unregistered/unroutable workflow, input validation, singleton back-pressure, store
@@ -6489,13 +7231,16 @@ var WorkflowEngine = class {
6489
7231
  * correctly observes the failed start.
6490
7232
  */
6491
7233
  startChildDeferred(workflow, input, childId, opts) {
6492
- queueMicrotask(() => void this.start(workflow, input, childId, opts).catch((err) => {
6493
- const message = err instanceof Error ? err.message : String(err);
6494
- this.notifyParent(childId, {
6495
- ok: false,
6496
- error: `child workflow "${workflow}" failed to start: ${message}`
7234
+ this.localWork.submit(`start:${childId}`, async () => {
7235
+ if (this.draining) return;
7236
+ await this.start(workflow, input, childId, opts).catch((err) => {
7237
+ const message = err instanceof Error ? err.message : String(err);
7238
+ this.notifyParent(childId, {
7239
+ ok: false,
7240
+ error: `child workflow "${workflow}" failed to start: ${message}`
7241
+ });
6497
7242
  });
6498
- }));
7243
+ });
6499
7244
  }
6500
7245
  /**
6501
7246
  * Cancel a run (e.g. from the dashboard). Returns null if the run does not exist. Pass
@@ -6559,6 +7304,7 @@ var WorkflowEngine = class {
6559
7304
  error: error2
6560
7305
  });
6561
7306
  await this.cancelChildren(runId, opts);
7307
+ this.notifyParent(runId, cancelledCompletion(error2));
6562
7308
  this.notifyCancelled(runId);
6563
7309
  if (this.controlPlane) {
6564
7310
  void this.controlPlane.publishControl({
@@ -7073,6 +7819,7 @@ var WorkflowEngine = class {
7073
7819
  const lockNow = this.clock();
7074
7820
  if (run.lockedBy !== this.instanceId || (run.lockedUntil ?? 0) <= lockNow) {
7075
7821
  if (!await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow)) {
7822
+ this.recheckAfterLease(run.id, run.lockedUntil);
7076
7823
  return {
7077
7824
  runId: run.id,
7078
7825
  status: run.status
@@ -7091,6 +7838,51 @@ var WorkflowEngine = class {
7091
7838
  }
7092
7839
  }
7093
7840
  /**
7841
+ * A wake that found the run leased by another instance is dropped, and normally that is fine: the
7842
+ * holder's turn either sees what the wake wrote, or re-checks once it releases the lease
7843
+ * ({@link redriveSettledSteps}, {@link redriveSettledChildren}). Not if the holder goes down first
7844
+ * (a scale-down mid-turn, an OOM): its lease lapses with the run `suspended` on a step that has
7845
+ * finished, and nothing wakes it before its timer.
7846
+ *
7847
+ * So the dropped wake is kept here: once the lease is free (looked at every
7848
+ * {@link LEASE_RECHECK_MAX_MS} at most, for a bounded number of times), a run still `suspended` is
7849
+ * leased and resumed. A holder that already moved the run on makes this one spare replay that
7850
+ * re-parks. Best effort; the run's timer remains the fallback if this instance goes down too.
7851
+ */
7852
+ recheckAfterLease(runId, lockedUntil, attempt = 1) {
7853
+ if (this.draining || this.leaseRetries.has(runId)) return;
7854
+ const remaining = Math.max(0, (lockedUntil ?? this.clock()) - this.clock());
7855
+ const timer = setTimeout(() => {
7856
+ this.leaseRetries.delete(runId);
7857
+ this.inBackground(this.retryDroppedWake(runId, attempt));
7858
+ }, Math.min(remaining, LEASE_RECHECK_MAX_MS) + LEASE_RECHECK_SLACK_MS);
7859
+ timer.unref?.();
7860
+ this.leaseRetries.set(runId, timer);
7861
+ }
7862
+ async retryDroppedWake(runId, attempt) {
7863
+ if (this.draining) return;
7864
+ const run = await this.store.getRun(runId);
7865
+ if (run?.status !== "suspended") return;
7866
+ const nowMs = this.clock();
7867
+ if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) {
7868
+ if (attempt < LEASE_RECHECK_ATTEMPTS) this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
7869
+ return;
7870
+ }
7871
+ if (this.queuesRunsOverTransport) {
7872
+ await this.wake(runId);
7873
+ return;
7874
+ }
7875
+ if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) return;
7876
+ const fresh = await this.store.getRun(runId);
7877
+ if (fresh?.status !== "suspended") {
7878
+ await this.store.releaseRunLock(runId);
7879
+ return;
7880
+ }
7881
+ await this.resumeImplicit(runId).catch((err) => {
7882
+ if (!(err instanceof NamespaceMismatch)) throw err;
7883
+ });
7884
+ }
7885
+ /**
7094
7886
  * Renew `runId`'s lease for as long as this execution holds it, and register it in
7095
7887
  * {@link executionLeases} so settles can check the zombie fence and `drain()` can release it.
7096
7888
  *
@@ -7454,6 +8246,7 @@ ${run.workflowVersion}`;
7454
8246
  namespace: run.namespace,
7455
8247
  error: error2
7456
8248
  });
8249
+ this.notifyParent(run.id, cancelledCompletion(error2));
7457
8250
  this.notifyCancelled(run.id);
7458
8251
  if (this.controlPlane) {
7459
8252
  void this.controlPlane.publishControl({
@@ -7485,6 +8278,8 @@ ${run.workflowVersion}`;
7485
8278
  const wfPlan = await this.planRoute(tenantGroup(sanitizeQueueToken(remote.group), void 0), registered.requires);
7486
8279
  if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
7487
8280
  const checkpoints = await this.store.listCheckpoints(run.id);
8281
+ await this.deliverFinishedChildren(run.id, checkpoints);
8282
+ await this.deliverFinishedChildWaiters(run.id, checkpoints);
7488
8283
  const handoff = liveHandoff(run.id, checkpoints);
7489
8284
  if (handoff) {
7490
8285
  const result = await this.settleRun(run, await this.followHandoff(run, handoff));
@@ -8059,6 +8854,7 @@ ${run.workflowVersion}`;
8059
8854
  namespace: run.namespace,
8060
8855
  error: error2
8061
8856
  });
8857
+ this.notifyParent(run.id, cancelledCompletion(error2));
8062
8858
  return {
8063
8859
  runId: run.id,
8064
8860
  status: "cancelled",
@@ -8105,6 +8901,7 @@ ${run.workflowVersion}`;
8105
8901
  }
8106
8902
  const compensations = [];
8107
8903
  const snapshot = await this.store.listCheckpoints(run.id);
8904
+ await this.deliverFinishedChildren(run.id, snapshot);
8108
8905
  const replay = /* @__PURE__ */ new Map();
8109
8906
  for (const cp of snapshot) replay.set(cp.seq, cp);
8110
8907
  let suspendedOnFan;
@@ -8201,13 +8998,13 @@ ${run.workflowVersion}`;
8201
8998
  if (this.awaitedStepFans.get(run.id) === awaitedFans) this.awaitedStepFans.delete(run.id);
8202
8999
  await this.releaseLeaseBestEffort(run.id);
8203
9000
  if (suspendedOnFan) {
8204
- void this.redriveSettledChildren(run.id, suspendedOnFan).catch(() => void 0);
9001
+ this.inBackground(this.redriveSettledChildren(run.id, suspendedOnFan));
8205
9002
  }
8206
9003
  if (suspendedOnSteps && awaitedSteps.size > 0) {
8207
- void this.redriveSettledSteps(run.id, awaitedSteps).catch(() => void 0);
9004
+ this.inBackground(this.redriveSettledSteps(run.id, awaitedSteps));
8208
9005
  }
8209
9006
  if (suspendedOnSteps && awaitedFans.length > 0) {
8210
- void this.redriveSettledStepFans(run.id, awaitedFans).catch(() => void 0);
9007
+ this.inBackground(this.redriveSettledStepFans(run.id, awaitedFans));
8211
9008
  }
8212
9009
  }
8213
9010
  }
@@ -8647,9 +9444,11 @@ ${run.workflowVersion}`;
8647
9444
  })) {
8648
9445
  return this.callRemote(runId, seq, step, input, queue, transport, void 0, admission, routing);
8649
9446
  }
8650
- throw new WorkflowSuspended(wakeAt);
9447
+ throw new WorkflowSuspended(this.stepLeaseWakeAt(wakeAt));
9448
+ }
9449
+ if (this.clock() < existing.wakeAt) {
9450
+ throw new WorkflowSuspended(this.stepLeaseWakeAt(existing.wakeAt));
8651
9451
  }
8652
- if (this.clock() < existing.wakeAt) throw new WorkflowSuspended(existing.wakeAt);
8653
9452
  if (existing.attempts >= this.remoteRedispatchMax) {
8654
9453
  throw new RemoteStepError({
8655
9454
  message: `remote step "${step.name}" lost \u2014 no result after ${existing.attempts} re-dispatch(es)`,
@@ -8695,7 +9494,7 @@ ${run.workflowVersion}`;
8695
9494
  kind: "remote",
8696
9495
  redispatched: true
8697
9496
  });
8698
- throw new WorkflowSuspended(nextDeadline);
9497
+ throw new WorkflowSuspended(this.stepLeaseWakeAt(nextDeadline));
8699
9498
  }
8700
9499
  let attempt = 1;
8701
9500
  if (existing?.status === "failed") {
@@ -9058,6 +9857,7 @@ ${run.workflowVersion}`;
9058
9857
  });
9059
9858
  const reads = await Promise.allSettled(live.map((i) => this.store.getCheckpoint(runId, results[i].seq)));
9060
9859
  const pending = [];
9860
+ const resettled = [];
9061
9861
  reads.forEach((read, k) => {
9062
9862
  const index = live[k];
9063
9863
  if (read.status === "rejected") outcomes[index] = {
@@ -9067,8 +9867,17 @@ ${run.workflowVersion}`;
9067
9867
  index,
9068
9868
  cp: read.value
9069
9869
  });
9870
+ else if (read.value?.status === "completed" || read.value?.status === "failed") {
9871
+ resettled.push({
9872
+ index,
9873
+ cp: read.value
9874
+ });
9875
+ }
9070
9876
  });
9071
- if (pending.length === 0) return outcomes;
9877
+ if (pending.length === 0) {
9878
+ if (resettled.length === 0) return outcomes;
9879
+ return this.resumeForRedelivered(runId, resettled, outcomes);
9880
+ }
9072
9881
  for (const { cp } of pending) void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
9073
9882
  const run = await this.store.getRun(runId);
9074
9883
  if (run && (run.status === "cancelled" || run.status === "completed")) return outcomes;
@@ -9120,6 +9929,29 @@ ${run.workflowVersion}`;
9120
9929
  return outcomes;
9121
9930
  }
9122
9931
  /**
9932
+ * The outcomes for a batch of redelivered results (their steps already settled): one of them
9933
+ * resumes the run if it is `suspended` and, for a step of a fan, the fan's barrier is open. A
9934
+ * resume of a run that had in fact moved on replays to its current wait and re-parks.
9935
+ */
9936
+ async resumeForRedelivered(runId, resettled, outcomes) {
9937
+ const run = await this.store.getRun(runId);
9938
+ if (run?.status !== "suspended") return outcomes;
9939
+ const barriers = /* @__PURE__ */ new Map();
9940
+ for (const { index, cp } of resettled) {
9941
+ const asResult = {
9942
+ runId,
9943
+ seq: cp.seq,
9944
+ stepId: cp.stepId ?? stepId(runId, cp.seq),
9945
+ status: cp.status === "failed" ? "failed" : "completed"
9946
+ };
9947
+ if (!await this.stepFanStillRunning(cp, asResult, barriers)) {
9948
+ outcomes[index] = true;
9949
+ break;
9950
+ }
9951
+ }
9952
+ return outcomes;
9953
+ }
9954
+ /**
9123
9955
  * The join barrier of a step fan: a result for one of its steps resumes the parent only once no
9124
9956
  * step of the fan is still `pending` (a settled wave), or on a failure (failFast surfaces it, and a
9125
9957
  * retry needs the parent to schedule it). Without it every result would replay the whole parent,
@@ -9243,7 +10075,7 @@ ${run.workflowVersion}`;
9243
10075
  ...waiters
9244
10076
  ];
9245
10077
  waiters.clear();
9246
- for (const runId of runIds) void this.resumeImplicit(runId).catch(() => void 0);
10078
+ for (const runId of runIds) void this.wake(runId).catch(() => void 0);
9247
10079
  }
9248
10080
  /** Release the flow-control slot a dispatched step held (if any), by its stepId. */
9249
10081
  async releaseQueueSlot(id) {
@@ -9985,6 +10817,8 @@ var CodecStateStore = class {
9985
10817
  registerChildWaits;
9986
10818
  /** The folded checkpoint carries the children's outputs, so it is encoded. */
9987
10819
  compactCheckpoints;
10820
+ /** Decodes each run like {@link getRun}. Bound only when the inner store has it. */
10821
+ getRuns;
9988
10822
  /** Forwarded verbatim — a waiter carries no payload. */
9989
10823
  listSignalWaitersByRun;
9990
10824
  /** Forwarded verbatim — a waiter carries no payload. */
@@ -10028,6 +10862,8 @@ var CodecStateStore = class {
10028
10862
  })));
10029
10863
  const compact = inner.compactCheckpoints;
10030
10864
  if (compact) this.compactCheckpoints = (runId, fromSeq, toSeq, into) => compact.call(inner, runId, fromSeq, toSeq, this.encCp(into));
10865
+ const getRuns = inner.getRuns;
10866
+ if (getRuns) this.getRuns = async (runIds) => (await getRuns.call(inner, runIds)).map((r) => this.decRun(r));
10031
10867
  const byRun = inner.listSignalWaitersByRun;
10032
10868
  if (byRun) this.listSignalWaitersByRun = (runIds) => byRun.call(inner, runIds);
10033
10869
  const hasWaiter = inner.hasSignalWaiter;
@@ -10580,6 +11416,16 @@ var InMemoryStateStore = class {
10580
11416
  ...run
10581
11417
  } : null;
10582
11418
  }
11419
+ async getRuns(runIds) {
11420
+ const out = [];
11421
+ for (const id of new Set(runIds)) {
11422
+ const run = this.runs.get(id);
11423
+ if (run) out.push({
11424
+ ...run
11425
+ });
11426
+ }
11427
+ return out;
11428
+ }
10583
11429
  async deleteRun(runId) {
10584
11430
  this.runs.delete(runId);
10585
11431
  for (const [key, cp] of this.checkpoints) {
@@ -11031,6 +11877,99 @@ var InMemoryTransport = class {
11031
11877
  this.controlHandlers.add(handler);
11032
11878
  }
11033
11879
  };
11880
+
11881
+ // src/testing/in-memory-run-queue.ts
11882
+ var InMemoryRunQueue = class {
11883
+ static {
11884
+ __name(this, "InMemoryRunQueue");
11885
+ }
11886
+ lanes = /* @__PURE__ */ new Map();
11887
+ /** A transport on this queue. */
11888
+ transport() {
11889
+ return {
11890
+ dispatch: /* @__PURE__ */ __name(async (_task) => {
11891
+ throw new Error("InMemoryRunQueue carries no remote steps");
11892
+ }, "dispatch"),
11893
+ onResult: /* @__PURE__ */ __name((_handler) => {
11894
+ }, "onResult"),
11895
+ onHeartbeat: /* @__PURE__ */ __name((_handler) => {
11896
+ }, "onHeartbeat"),
11897
+ dispatchRuns: /* @__PURE__ */ __name(async (runs, opts) => this.add(runs, opts), "dispatchRuns"),
11898
+ onDispatchedRun: /* @__PURE__ */ __name((handler, opts) => this.consume(handler, opts), "onDispatchedRun")
11899
+ };
11900
+ }
11901
+ /** Runs waiting in the queue of `namespace`. */
11902
+ waiting(namespace) {
11903
+ return (this.lanes.get(namespace ?? "")?.waiting ?? []).map((r) => r.runId);
11904
+ }
11905
+ lane(namespace) {
11906
+ const key = namespace ?? "";
11907
+ let lane = this.lanes.get(key);
11908
+ if (!lane) {
11909
+ lane = {
11910
+ waiting: [],
11911
+ queued: /* @__PURE__ */ new Map(),
11912
+ consumers: [],
11913
+ next: 0
11914
+ };
11915
+ this.lanes.set(key, lane);
11916
+ }
11917
+ return lane;
11918
+ }
11919
+ add(runs, opts) {
11920
+ const lane = this.lane(opts?.namespace);
11921
+ for (const run of runs) {
11922
+ const waiting = lane.queued.get(run.runId);
11923
+ if (waiting) {
11924
+ if (run.reason === "wake") waiting.reason = "wake";
11925
+ continue;
11926
+ }
11927
+ const entry = {
11928
+ ...run
11929
+ };
11930
+ lane.queued.set(run.runId, entry);
11931
+ lane.waiting.push(entry);
11932
+ }
11933
+ lane.waiting.sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0));
11934
+ setImmediate(() => this.pump(lane));
11935
+ }
11936
+ consume(handler, opts) {
11937
+ const lane = this.lane(opts.namespace);
11938
+ const consumer = {
11939
+ handler,
11940
+ concurrency: Math.max(1, opts.concurrency),
11941
+ active: 0,
11942
+ stopped: false,
11943
+ idle: []
11944
+ };
11945
+ lane.consumers.push(consumer);
11946
+ setImmediate(() => this.pump(lane));
11947
+ return () => {
11948
+ consumer.stopped = true;
11949
+ lane.consumers = lane.consumers.filter((c) => c !== consumer);
11950
+ if (consumer.active === 0) return Promise.resolve();
11951
+ return new Promise((resolve) => consumer.idle.push(resolve));
11952
+ };
11953
+ }
11954
+ pump(lane) {
11955
+ while (lane.waiting.length > 0) {
11956
+ const free = lane.consumers.filter((c) => !c.stopped && c.active < c.concurrency);
11957
+ if (free.length === 0) return;
11958
+ const consumer = free[lane.next % free.length];
11959
+ lane.next += 1;
11960
+ const run = lane.waiting.shift();
11961
+ lane.queued.delete(run.runId);
11962
+ consumer.active += 1;
11963
+ void consumer.handler(run).catch(() => void 0).then(() => {
11964
+ consumer.active -= 1;
11965
+ if (consumer.active === 0 && consumer.stopped) {
11966
+ for (const resolve of consumer.idle.splice(0)) resolve();
11967
+ }
11968
+ this.pump(lane);
11969
+ });
11970
+ }
11971
+ }
11972
+ };
11034
11973
  // Annotate the CommonJS export names for ESM import in node:
11035
11974
  0 && (module.exports = {
11036
11975
  CHILD_EDGE_PREFIXES,
@@ -11058,6 +11997,7 @@ var InMemoryTransport = class {
11058
11997
  GatherError,
11059
11998
  HANDOFF_SEQ_BASE,
11060
11999
  InMemoryAdmissionBackend,
12000
+ InMemoryRunQueue,
11061
12001
  InMemoryStateStore,
11062
12002
  InMemoryTransport,
11063
12003
  LEGACY_V1_CAPABILITIES,
@@ -11142,6 +12082,7 @@ var InMemoryTransport = class {
11142
12082
  fanPageEtag,
11143
12083
  findStalledRuns,
11144
12084
  foldedCount,
12085
+ getRunsByIds,
11145
12086
  handoffHandlerId,
11146
12087
  handoffName,
11147
12088
  hasDurableExecutionWrappers,
@@ -11179,6 +12120,7 @@ var InMemoryTransport = class {
11179
12120
  poolNamespace,
11180
12121
  prevCronFireMs,
11181
12122
  queryCheckpointsOf,
12123
+ readRunsByIds,
11182
12124
  readSearchAttributes,
11183
12125
  requiredCapabilities,
11184
12126
  resolveConcurrency,