@dudousxd/nestjs-durable-core 0.61.0 → 0.62.0

package/dist/index.cjs

@@ -2396,6 +2396,10 @@ __name(mergeTags, "mergeTags");
 var BREAKPOINT = "breakpoint";
 var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
 var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
+function stepGroup(step, runNamespace) {
+  return tenantGroup(sanitizeQueueToken(step.name), step.partition ?? runNamespace);
+}
+__name(stepGroup, "stepGroup");
 function nextContinuationId(runId) {
   const m = runId.match(/^(.*)~(\d+)$/);
   return m ? `${m[1]}~${Number(m[2]) + 1}` : `${runId}~1`;
@@ -3896,7 +3900,8 @@ var WorkflowEngine = class {
   async workerHealth(extra = []) {
     const groups = /* @__PURE__ */ new Set([
       ...this.knownGroups(extra),
-      ...await this.pool.listWorkerGroups()
+      ...await this.pool.listWorkerGroups(),
+      ...await this.inflightStepGroups()
     ]);
     const workflowTokens = /* @__PURE__ */ new Set();
     for (const def of this.workflows.values()) {
@@ -3917,6 +3922,38 @@ var WorkflowEngine = class {
     return out;
   }
   /**
+  * The routing groups of every in-flight run's still-`pending` remote step — the queues where work is
+  * dispatched and awaiting a worker. Folded into {@link workerHealth} so a step waiting on a group
+  * that is NEITHER a registration NOR heartbeating (the exact shape of a run stuck on an offline
+  * tenant pool) still reports its backlog: without this the health scan can't "see" that queue, and
+  * the dashboard shows the run as running instead of no-worker. Only the BullMQ transport introspects
+  * health, so on any other transport this is harmless (the groups just resolve to no `groupHealth`).
+  *
+  * Cost: one `listRuns` over the non-terminal runs plus a checkpoint read each — a diagnostic-path
+  * scan, not a hot path. Scoped to `running`/`suspended`/`blocked` (a `pending` run has dispatched no
+  * step yet) and deduped to distinct groups.
+  */
+  async inflightStepGroups() {
+    const inflight = await this.store.listRuns({
+      statuses: [
+        "running",
+        "suspended",
+        "blocked"
+      ]
+    });
+    const groups = /* @__PURE__ */ new Set();
+    for (const run of inflight) {
+      for (const cp of await this.store.listCheckpoints(run.id)) {
+        if (cp.kind === "remote" && cp.status === "pending" && cp.workerGroup) {
+          groups.add(cp.workerGroup);
+        }
+      }
+    }
+    return [
+      ...groups
+    ];
+  }
+  /**
   * Persist a streamed local-step lifecycle event from a remote workflow worker (see
   * {@link WorkflowStepEvent}). A Python `@workflow` runs its `ctx.step`s inline over one turn that
   * can last minutes; the worker streams each step's start/finish so the engine checkpoints it LIVE —
@@ -4880,7 +4917,7 @@ var WorkflowEngine = class {
   */
   async dispatchCompensationAttempt(run, seq, name, def, args, attempt, replay) {
     if (this.pool.size === 0) throw new Error("remote steps require a Transport");
-    const group = tenantGroup(sanitizeQueueToken(def.name), def.partition);
+    const group = stepGroup(def, run.namespace);
     const enqueuedAt = /* @__PURE__ */ new Date();
     const cp = stepCheckpoint({
       runId: run.id,
@@ -4928,7 +4965,12 @@ var WorkflowEngine = class {
       startStep: /* @__PURE__ */ __name((s) => this.startStep(s), "startStep"),
       completeStep: /* @__PURE__ */ __name((s) => this.completeStep(s), "completeStep"),
       failStep: /* @__PURE__ */ __name((s) => this.failStep(s), "failStep"),
-      callRemote: /* @__PURE__ */ __name((runId, seq, step, input, queue, transport, admission) => this.callRemote(runId, seq, step, input, queue, transport, replay, admission), "callRemote"),
+      // Route the dispatch by the RUN's tenant (same reason as `startChild` below): the executing
+      // engine may be an operator driving every namespace, so the token must come from the run, not
+      // from this engine. See {@link stepGroup}.
+      callRemote: /* @__PURE__ */ __name((runId, seq, step, input, queue, transport, admission) => this.callRemote(runId, seq, step, input, queue, transport, replay, admission, {
+        namespace: parentNamespace
+      }), "callRemote"),
       // Defer so a fast child can't reentrantly resume a still-running parent. Stamp the child with
       // the PARENT run's namespace (not this engine's) so a child belongs to the tenant/partition of
       // the run it was spawned from — even when an operator (namespace: undefined, drives every
@@ -5110,14 +5152,15 @@ var WorkflowEngine = class {
     });
     return blocked.filter((r) => (this.namespace === void 0 || r.namespace === void 0 || r.namespace === this.namespace) && r.wakeAt !== void 0 && r.wakeAt <= nowMs);
   }
-  async callRemote(runId, seq, step, input, queue, transport, replay, admission) {
+  async callRemote(runId, seq, step, input, queue, transport, replay, admission, routing) {
+    const group = stepGroup(step, routing?.namespace);
     const existing = replay?.get(seq) ?? await this.store.getCheckpoint(runId, seq);
     if (existing && existing.name !== step.name) {
       throw new NonDeterminismError(runId, seq, step.name, existing.name);
     }
     if (existing?.status === "completed") return existing.output;
     if (this.pool.size === 0) throw new Error("remote steps require a Transport");
-    if (step.timeoutMs) return this.callRemoteInMemory(runId, seq, step, input, transport);
+    if (step.timeoutMs) return this.callRemoteInMemory(runId, seq, step, input, transport, group);
     if (existing?.status === "pending") {
       if (this.remoteRedispatchMs == null) throw new WorkflowSuspended();
       if (existing.wakeAt == null) {
@@ -5151,7 +5194,7 @@ var WorkflowEngine = class {
         seq,
         name: step.name,
         stepId: existing.stepId ?? stepId(runId, seq),
-        group: existing.workerGroup ?? tenantGroup(sanitizeQueueToken(step.name), step.partition),
+        group: existing.workerGroup ?? group,
         input: existing.input,
         priority: admission?.priority,
         attempt: reAttempt,
@@ -5175,7 +5218,7 @@ var WorkflowEngine = class {
       if (this.clock() < existing.wakeAt) throw new WorkflowSuspended(existing.wakeAt);
       attempt = existing.attempts + 1;
     }
-    await this.ensureRoutable(tenantGroup(sanitizeQueueToken(step.name), step.partition), step.requires);
+    await this.ensureRoutable(group, step.requires);
     const id = stepId(runId, seq);
     if (queue && this.admission.handles(queue)) {
       const decision = await this.admission.tryAdmit(queue, {
@@ -5202,7 +5245,7 @@ var WorkflowEngine = class {
       status: "pending",
       input,
       attempts: attempt,
-      workerGroup: tenantGroup(sanitizeQueueToken(step.name), step.partition),
+      workerGroup: group,
       enqueuedAt,
       startedAt: enqueuedAt,
       finishedAt: enqueuedAt
@@ -5212,7 +5255,7 @@ var WorkflowEngine = class {
       seq,
       name: step.name,
       stepId: id,
-      group: tenantGroup(sanitizeQueueToken(step.name), step.partition),
+      group,
       input,
       priority: admission?.priority,
       attempt,
@@ -5360,9 +5403,9 @@ var WorkflowEngine = class {
     await this.admission.release(queue, id);
   }
   /** In-memory await path for a remote step with a liveness `timeoutMs` (re-dispatch on timeout). */
-  async callRemoteInMemory(runId, seq, step, input, transport) {
+  async callRemoteInMemory(runId, seq, step, input, transport, group = stepGroup(step, void 0)) {
     if (this.pool.size === 0) throw new Error("remote steps require a Transport");
-    await this.ensureRoutable(tenantGroup(sanitizeQueueToken(step.name), step.partition), step.requires);
+    await this.ensureRoutable(group, step.requires);
     const validInput = step.input ? step.input.parse(input) : input;
     const id = stepId(runId, seq);
     const enqueuedAt = /* @__PURE__ */ new Date();
@@ -5386,7 +5429,7 @@ var WorkflowEngine = class {
         seq,
         name: step.name,
         stepId: id,
-        group: tenantGroup(sanitizeQueueToken(step.name), step.partition),
+        group,
         input: validInput,
         traceparent: this.traceparent?.(),
         context: this.context?.(),
@@ -5405,7 +5448,7 @@ var WorkflowEngine = class {
           output,
           events: resolution.events,
           attempts: attempt,
-          workerGroup: tenantGroup(sanitizeQueueToken(step.name), step.partition),
+          workerGroup: group,
           enqueuedAt,
           startedAt
         });