@dudousxd/nestjs-durable-core 0.76.1 → 0.77.0

package/dist/index.cjs

@@ -1250,6 +1250,20 @@ function allGroup(firstSeq, mode) {
   return mode === "failFast" ? `all:${firstSeq}:failFast` : `all:${firstSeq}`;
 }
 __name(allGroup, "allGroup");
+function stepFanGroup(firstSeq, size, mode) {
+  const group = `steps:${firstSeq}:${size}`;
+  return mode === "failFast" ? `${group}:failFast` : group;
+}
+__name(stepFanGroup, "stepFanGroup");
+function parseStepFanGroup(group) {
+  const m = group ? /^steps:(\d+):(\d+)(?::failFast)?$/.exec(group) : null;
+  if (!m) return void 0;
+  return {
+    firstSeq: Number(m[1]),
+    size: Number(m[2])
+  };
+}
+__name(parseStepFanGroup, "parseStepFanGroup");
 function allWait(runId, firstSeq, wave) {
   const base = allBase(runId, firstSeq, wave);
   return {
@@ -3291,6 +3305,9 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
   }, "child");
   const all = /* @__PURE__ */ __name(async (workflow2, inputs, opts) => {
     if (inputs.length === 0) return [];
+    if (stepNameOf(workflow2) !== void 0) {
+      return allSteps(workflow2, inputs, opts);
+    }
     const mode = opts?.mode ?? "waitAll";
     const name = workflowName(workflow2);
     const waves = allWaves(inputs.length, opts?.concurrency);
@@ -3430,6 +3447,26 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
     if (failures.length > 0) throw new GatherError(failures, inputs.length);
     return outputs;
   }, "all");
+  const allSteps = /* @__PURE__ */ __name(async (ref, inputs, opts) => {
+    if (opts?.queue !== void 0) {
+      throw new Error("ctx.all: a flow-control queue is not supported on a step fan");
+    }
+    const name = stepNameOf(ref);
+    const def = {
+      name,
+      ...resolveStepPolicy(stepConfigOf(ref), opts)
+    };
+    const positions = inputs.map(() => pos.next());
+    return host.callRemoteAll(runId, positions, def, inputs, {
+      mode: opts?.mode ?? "waitAll",
+      concurrency: opts?.concurrency,
+      transport: opts?.transport,
+      admission: {
+        priority: opts?.priority,
+        fairnessKey: opts?.fairnessKey
+      }
+    });
+  }, "allSteps");
   const readFan = /* @__PURE__ */ __name(async (positions, prefixes) => {
     const checkpoints = positions.map((seq) => replay?.get(seq) ?? null);
     const folded = positions.map(() => false);
@@ -3766,6 +3803,18 @@ var BREAKPOINT = "breakpoint";
 var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
 var VERSION_UNDECLARED_TAG = "version:undeclared";
 var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
+var STEP_FAN_CHUNK = 200;
+function shuffle(items) {
+  for (let i = items.length - 1; i > 0; i -= 1) {
+    const j = Math.floor(Math.random() * (i + 1));
+    [items[i], items[j]] = [
+      items[j],
+      items[i]
+    ];
+  }
+  return items;
+}
+__name(shuffle, "shuffle");
 function stepGroup(step, runNamespace) {
   return tenantGroup(sanitizeQueueToken(step.name), step.partition ?? runNamespace);
 }
@@ -3807,6 +3856,7 @@ var WorkflowEngine = class {
   maxRecoveryAttempts;
   remoteAdvanceSilenceMs;
   reconcileMs;
+  timerBatchSize;
   eventBufferTtlMs;
   remoteRedispatchMs;
   remoteRedispatchMax;
@@ -3873,6 +3923,15 @@ var WorkflowEngine = class {
   * dropped one {@link redriveSettledSteps} recovers. Pruned by age (see {@link noteSettledHere}).
   */
   settledHere = /* @__PURE__ */ new Map();
+  /**
+  * The step fans (`ctx.all` over a `@Step`) a turn suspended on, by run, while that turn executes on
+  * THIS instance — re-checked once its lease is released (see {@link redriveSettledStepFans}).
+  */
+  awaitedStepFans = /* @__PURE__ */ new Map();
+  /** Step fans whose join barrier this instance opened, by `runId:firstSeq` — see {@link settledHere}. */
+  stepFansFiredHere = /* @__PURE__ */ new Map();
+  /** The tail of each run's queue of results being handled on this instance — see {@link serializedPerRun}. */
+  resultChains = /* @__PURE__ */ new Map();
   /** Flow-control admission backend for remote steps (see {@link registerQueue}). */
   admission;
   /** Runs on THIS instance blocked on admission, by queue — woken early on a freed-slot signal. */
@@ -3904,6 +3963,8 @@ var WorkflowEngine = class {
     this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
     this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
     this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
+    const batch = deps.timerBatchSize;
+    this.timerBatchSize = batch !== void 0 && Number.isFinite(batch) && batch >= 1 ? Math.floor(batch) : 500;
     this.eventBufferTtlMs = deps.eventBufferTtlMs || void 0;
     this.remoteRedispatchMs = deps.remoteRedispatchMs || void 0;
     this.remoteRedispatchMax = Math.max(1, deps.remoteRedispatchMax ?? 10);
@@ -3956,7 +4017,11 @@ var WorkflowEngine = class {
           }
           return;
         }
-        await this.completeRemoteResult(result);
+        if (!await this.completeRemoteResult(result)) return;
+        await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId).then(() => void 0, (err) => {
+          if (err instanceof NamespaceMismatch) return;
+          throw err;
+        }));
       },
       // A heartbeat resets the liveness window for whatever it targets: an in-flight long STEP (keyed
       // by stepId, see callRemote) or — when stepId is absent — the in-flight workflow TURN (keyed by
@@ -4816,7 +4881,7 @@ var WorkflowEngine = class {
   * scan, filtered to just this tick's batch, keeps it cheap regardless of store size.
   */
   async resumeDueTimers(nowMs = this.clock()) {
-    const due = await this.store.listDueTimers(nowMs, this.namespace);
+    const due = shuffle(await this.store.listDueTimers(nowMs, this.namespace, this.timerBatchSize));
     const blocked = await this.dueBlockedRuns(nowMs);
     if (due.length === 0 && blocked.length === 0) return [];
     const waiterByRun = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
@@ -5315,6 +5380,32 @@ var WorkflowEngine = class {
       }
     }
   }
+  /**
+  * {@link redriveSettledSteps} for a step fan: once the turn that suspended on it released its lease,
+  * re-drive the run if no step of the fan is still `pending` — the last result landed on another
+  * instance while this one held the lease, and its wake was dropped there. A fan whose barrier this
+  * instance opened is skipped, for the reason {@link settledHere} gives.
+  */
+  async redriveSettledStepFans(runId, fans) {
+    for (const fan of fans) {
+      if (this.stepFansFiredHere.has(`${runId}:${fan.from}`)) continue;
+      const pending = this.store.countCheckpoints ? await this.store.countCheckpoints(runId, fan.from, fan.to, [
+        "pending"
+      ]) : (await this.store.listCheckpoints(runId)).filter((c) => c.seq >= fan.from && c.seq <= fan.to && c.status === "pending").length;
+      if (pending > 0) continue;
+      const nowMs = this.clock();
+      if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
+        return;
+      }
+      const run = await this.store.getRun(runId);
+      if (run?.status !== "suspended") {
+        await this.store.releaseRunLock(runId);
+        return;
+      }
+      await this.resumeImplicit(runId);
+      return;
+    }
+  }
   async redriveSettledChildren(runId, fan) {
     if (!this.store.hasSignalWaiter) return;
     for (const wait of fan.waits) {
@@ -6772,6 +6863,8 @@ var WorkflowEngine = class {
     let suspendedOnSteps = false;
     const awaitedSteps = /* @__PURE__ */ new Set();
     this.awaitedRemoteSteps.set(run.id, awaitedSteps);
+    const awaitedFans = [];
+    this.awaitedStepFans.set(run.id, awaitedFans);
     const ctx = createWorkflowCtx(this.ctxHostFor(replay, run.namespace), run.id, compensations, run.workflow, registered?.searchAttributesSchema);
     try {
       const output = await runInWorkflowCtx(ctx, () => fn(ctx, run.input));
@@ -6851,6 +6944,7 @@ var WorkflowEngine = class {
       if (this.awaitedRemoteSteps.get(run.id) === awaitedSteps) {
         this.awaitedRemoteSteps.delete(run.id);
       }
+      if (this.awaitedStepFans.get(run.id) === awaitedFans) this.awaitedStepFans.delete(run.id);
       await this.store.releaseRunLock(run.id);
       if (suspendedOnFan) {
         void this.redriveSettledChildren(run.id, suspendedOnFan).catch(() => void 0);
@@ -6858,6 +6952,9 @@ var WorkflowEngine = class {
       if (suspendedOnSteps && awaitedSteps.size > 0) {
         void this.redriveSettledSteps(run.id, awaitedSteps).catch(() => void 0);
       }
+      if (suspendedOnSteps && awaitedFans.length > 0) {
+        void this.redriveSettledStepFans(run.id, awaitedFans).catch(() => void 0);
+      }
     }
   }
   /**
@@ -7080,6 +7177,9 @@ var WorkflowEngine = class {
       callRemote: /* @__PURE__ */ __name((runId, seq, step, input, queue, transport, admission) => this.callRemote(runId, seq, step, input, queue, transport, replay, admission, {
         namespace: parentNamespace
       }), "callRemote"),
+      callRemoteAll: /* @__PURE__ */ __name((runId, positions, step, inputs, opts) => this.callRemoteAll(runId, positions, step, inputs, opts, {
+        namespace: parentNamespace
+      }), "callRemoteAll"),
       // Defer so a fast child can't reentrantly resume a still-running parent. Stamp the child with
       // the PARENT run's namespace (not this engine's) so a child belongs to the tenant/partition of
       // the run it was spawned from — even when an operator (namespace: undefined, drives every
@@ -7381,6 +7481,9 @@ var WorkflowEngine = class {
       input,
       attempts: attempt,
       workerGroup: group,
+      ...routing?.parallelGroup !== void 0 ? {
+        parallelGroup: routing.parallelGroup
+      } : {},
       enqueuedAt,
       startedAt: enqueuedAt,
       finishedAt: enqueuedAt
@@ -7406,13 +7509,147 @@ var WorkflowEngine = class {
     this.awaitedRemoteSteps.get(runId)?.add(seq);
     throw new WorkflowSuspended();
   }
+  /**
+  * `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
+  * reserves the whole position block up front, so replay re-derives the same seqs.
+  *
+  * Each turn reads the fan's checkpoints fresh, hands every step that already has one to
+  * {@link callRemote} (which owns the completed / pending-lease / failed-retry rules), and sends
+  * the steps nobody has sent yet: all of them, or with `concurrency`, the next wave once the last
+  * one has drained. New steps are written `pending` BEFORE any is dispatched, so a turn that runs
+  * concurrently with this one finds them already sent instead of sending them again. The join
+  * barrier ({@link stepFanStillRunning}) then wakes the parent once per wave, not once per result.
+  */
+  async callRemoteAll(runId, positions, step, inputs, opts, routing) {
+    if (step.timeoutMs) {
+      throw new Error(`ctx.all: step "${step.name}" has a liveness timeoutMs; a step fan needs durable steps`);
+    }
+    if (opts.concurrency !== void 0 && (!Number.isInteger(opts.concurrency) || opts.concurrency < 1)) {
+      throw new Error(`ctx.all: concurrency must be a positive integer, got ${opts.concurrency}`);
+    }
+    if (this.pool.size === 0) throw new Error("remote steps require a Transport");
+    const first = positions[0];
+    const last = positions[positions.length - 1];
+    const group = stepFanGroup(first, positions.length, opts.mode);
+    const bySeq = /* @__PURE__ */ new Map();
+    for (const cp of await this.store.listCheckpoints(runId)) {
+      if (cp.seq >= first && cp.seq <= last) bySeq.set(cp.seq, cp);
+    }
+    const outputs = new Array(inputs.length);
+    const failures = [];
+    const unsent = [];
+    let inFlight = 0;
+    let wakeAt;
+    for (let i = 0; i < inputs.length; i += 1) {
+      const seq = positions[i];
+      const cp = bySeq.get(seq);
+      if (!cp) {
+        unsent.push(i);
+        continue;
+      }
+      if (cp.name !== step.name) throw new NonDeterminismError(runId, seq, step.name, cp.name);
+      if (cp.status === "completed") {
+        outputs[i] = cp.output;
+        continue;
+      }
+      try {
+        outputs[i] = await this.callRemote(runId, seq, step, inputs[i], void 0, opts.transport, void 0, opts.admission, {
+          ...routing,
+          parallelGroup: group
+        });
+      } catch (err) {
+        if (err instanceof WorkflowSuspended) {
+          inFlight += 1;
+          if (err.wakeAt !== void 0) wakeAt = Math.min(wakeAt ?? err.wakeAt, err.wakeAt);
+        } else if (err instanceof RemoteStepError) {
+          const failure = {
+            index: i,
+            id: stepId(runId, seq),
+            error: err.message
+          };
+          if (opts.mode === "failFast") throw new GatherError([
+            failure
+          ], inputs.length);
+          failures.push(failure);
+        } else {
+          throw err;
+        }
+      } finally {
+        this.awaitedRemoteSteps.get(runId)?.delete(seq);
+      }
+    }
+    const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
+    if (batch.length > 0) await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
+    if (inFlight + batch.length > 0) {
+      this.awaitedStepFans.get(runId)?.push({
+        from: first,
+        to: last
+      });
+      throw new WorkflowSuspended(wakeAt);
+    }
+    if (failures.length > 0) throw new GatherError(failures, inputs.length);
+    return outputs;
+  }
+  /** Write the `pending` checkpoints of a step fan's new steps, then dispatch them. */
+  async sendStepFan(runId, positions, step, inputs, batch, group, opts, routing) {
+    const workerGroup = stepGroup(step, routing?.namespace);
+    await this.ensureRoutable(workerGroup, step.requires);
+    const enqueuedAt = /* @__PURE__ */ new Date();
+    for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
+      await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map((i) => {
+        const seq = positions[i];
+        return this.store.saveCheckpoint({
+          runId,
+          seq,
+          name: step.name,
+          kind: "remote",
+          stepId: stepId(runId, seq),
+          status: "pending",
+          input: inputs[i],
+          attempts: 1,
+          workerGroup,
+          parallelGroup: group,
+          enqueuedAt,
+          startedAt: enqueuedAt,
+          finishedAt: enqueuedAt
+        });
+      }));
+    }
+    for (let at = 0; at < batch.length; at += STEP_FAN_CHUNK) {
+      await Promise.all(batch.slice(at, at + STEP_FAN_CHUNK).map(async (i) => {
+        const seq = positions[i];
+        await this.dispatchRemoteTask({
+          runId,
+          seq,
+          name: step.name,
+          stepId: stepId(runId, seq),
+          group: workerGroup,
+          input: inputs[i],
+          priority: opts.admission?.priority,
+          attempt: 1,
+          transport: opts.transport
+        });
+        this.emit({
+          type: "step.started",
+          runId,
+          seq,
+          name: step.name,
+          kind: "remote"
+        });
+      }));
+    }
+  }
   /** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
   noteSettledHere(key) {
+    this.noteRecent(this.settledHere, key);
+  }
+  /** Record `key` in a "happened here recently" map, pruning entries older than a lease. */
+  noteRecent(map, key) {
     const now = this.clock();
-    this.settledHere.set(key, now);
-    if (this.settledHere.size < 1024) return;
-    for (const [k, at] of this.settledHere) {
-      if (now - at > this.leaseMs) this.settledHere.delete(k);
+    map.set(key, now);
+    if (map.size < 1024) return;
+    for (const [k, at] of map) {
+      if (now - at > this.leaseMs) map.delete(k);
     }
   }
   /** {@link StateStore.saveCheckpointIfPending}, or a plain save on a store without it. */
@@ -7439,16 +7676,33 @@ var WorkflowEngine = class {
     }, task.transport);
   }
   /**
-  * Complete a durable remote step from its worker result and resume the run — runs on whichever
-  * instance receives the result (the dispatching one may be gone), so the run is crash/scale-safe.
-  * A no-op if the checkpoint isn't `pending` (a duplicate or late delivery).
+  * Run `work` after every earlier `work` for the same run on this instance has settled. A result
+  * resumes its run inline, and `execute` lets this instance into a lease it already holds — so two
+  * results of one run handled at once here would run two turns of it concurrently. Results of
+  * different runs still go in parallel, which is what a results consumer with concurrency > 1 is for.
+  */
+  serializedPerRun(runId, work) {
+    const previous = this.resultChains.get(runId) ?? Promise.resolve();
+    const next = previous.then(work, work);
+    const tail = next.catch(() => void 0);
+    this.resultChains.set(runId, tail);
+    void tail.then(() => {
+      if (this.resultChains.get(runId) === tail) this.resultChains.delete(runId);
+    });
+    return next;
+  }
+  /**
+  * Complete a durable remote step from its worker result — runs on whichever instance receives the
+  * result (the dispatching one may be gone), so the run is crash/scale-safe. Resolves whether the
+  * run should now be resumed: not for a duplicate or late delivery (the checkpoint isn't `pending`),
+  * a run already finished, or a step fan whose barrier is still closed.
   */
   async completeRemoteResult(result) {
     const cp = await this.store.getCheckpoint(result.runId, result.seq);
-    if (!cp || cp.status !== "pending") return;
+    if (!cp || cp.status !== "pending") return false;
     void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
     const run = await this.store.getRun(result.runId);
-    if (run && (run.status === "cancelled" || run.status === "completed")) return;
+    if (run && (run.status === "cancelled" || run.status === "completed")) return false;
     const finishedAt = /* @__PURE__ */ new Date();
     const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
     const claimed = await this.savePendingCheckpoint({
@@ -7460,7 +7714,7 @@ var WorkflowEngine = class {
       startedAt,
       finishedAt
     });
-    if (!claimed) return;
+    if (!claimed) return false;
     this.noteSettledHere(`${result.runId}:${result.seq}`);
     this.emit({
       type: result.status === "completed" ? "step.completed" : "step.failed",
@@ -7473,10 +7727,25 @@ var WorkflowEngine = class {
       queueMs: startedAt.getTime() - cp.enqueuedAt.getTime(),
       durationMs: finishedAt.getTime() - startedAt.getTime()
     });
-    await this.resumeImplicit(result.runId).catch((err) => {
-      if (err instanceof NamespaceMismatch) return null;
-      throw err;
-    });
+    return !await this.stepFanStillRunning(cp, result);
+  }
+  /**
+  * The join barrier of a step fan: a result for one of its steps resumes the parent only once no
+  * step of the fan is still `pending` (a settled wave), or on a failure (failFast surfaces it, and a
+  * retry needs the parent to schedule it). Without it every result would replay the whole parent,
+  * which makes a fan of N steps cost O(N²) checkpoint reads.
+  */
+  async stepFanStillRunning(cp, result) {
+    const fan = parseStepFanGroup(cp.parallelGroup);
+    if (!fan) return false;
+    if (result.status === "failed") return false;
+    const to = fan.firstSeq + fan.size - 1;
+    const pending = this.store.countCheckpoints ? await this.store.countCheckpoints(result.runId, fan.firstSeq, to, [
+      "pending"
+    ]) : (await this.store.listCheckpoints(result.runId)).filter((c) => c.seq >= fan.firstSeq && c.seq <= to && c.status === "pending").length;
+    if (pending > 0) return true;
+    this.noteRecent(this.stepFansFiredHere, `${result.runId}:${fan.firstSeq}`);
+    return false;
   }
   /**
   * Apply a remote workflow-turn {@link WorkflowDecision} durably — runs on WHATEVER instance receives
@@ -7857,6 +8126,8 @@ var CodecStateStore = class {
   /** Encodes like {@link saveCheckpoint}. Bound only when the inner store has it, so the engine's
   *  plain-save fallback still kicks in through the wrapper. */
   saveCheckpointIfPending;
+  /** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
+  countCheckpoints;
   /** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
   *  when the inner store prunes, so the retention poller still sees (and warns about) its absence. */
   pruneTerminalRuns;
@@ -7886,6 +8157,8 @@ var CodecStateStore = class {
     if (deleteRuns) this.deleteRuns = (ids) => deleteRuns.call(inner, ids);
     const saveIfPending = inner.saveCheckpointIfPending;
     if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
+    const countCps = inner.countCheckpoints;
+    if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
     const prune = inner.pruneTerminalRuns;
     if (prune) this.pruneTerminalRuns = (policy, now, limit) => prune.call(inner, policy, now, limit);
     const count = inner.countRuns;
@@ -8016,8 +8289,8 @@ var CodecStateStore = class {
   async listPendingRuns(limit, namespace) {
     return (await this.inner.listPendingRuns(limit, namespace)).map((r) => this.decRun(r));
   }
-  async listDueTimers(nowMs, namespace) {
-    return (await this.inner.listDueTimers(nowMs, namespace)).map((r) => this.decRun(r));
+  async listDueTimers(nowMs, namespace, limit) {
+    return (await this.inner.listDueTimers(nowMs, namespace, limit)).map((r) => this.decRun(r));
   }
   tryLockRun(runId, owner, leaseUntilMs, nowMs) {
     return this.inner.tryLockRun(runId, owner, leaseUntilMs, nowMs);
@@ -8471,10 +8744,11 @@ var InMemoryStateStore = class {
       ...r
     }));
   }
-  async listDueTimers(nowMs, namespace) {
-    return [
+  async listDueTimers(nowMs, namespace, limit) {
+    const due = [
       ...this.runs.values()
-    ].filter((r) => r.status === "suspended" && r.wakeAt !== void 0 && r.wakeAt <= nowMs && (namespace === void 0 || r.namespace === namespace)).map((r) => ({
+    ].filter((r) => r.status === "suspended" && r.wakeAt !== void 0 && r.wakeAt <= nowMs && (namespace === void 0 || r.namespace === namespace)).sort((a, b) => (a.wakeAt ?? 0) - (b.wakeAt ?? 0));
+    return (limit === void 0 ? due : due.slice(0, limit)).map((r) => ({
       ...r
     }));
   }
@@ -8748,6 +9022,13 @@ var InMemoryStateStore = class {
       ...latest
     } : void 0;
   }
+  async countCheckpoints(runId, fromSeq, toSeq, statuses) {
+    let n = 0;
+    for (const cp of this.checkpoints.values()) {
+      if (cp.runId === runId && cp.seq >= fromSeq && cp.seq <= toSeq && statuses.includes(cp.status)) n += 1;
+    }
+    return n;
+  }
   async listCheckpointsByNamePrefix(runId, prefixes) {
     const rows = [
       ...this.checkpoints.values()