@dudousxd/nestjs-durable-core 0.42.0 → 0.44.0

package/dist/index.js

@@ -700,6 +700,49 @@ function eventMatches(payload, match) {
 }
 __name(eventMatches, "eventMatches");
 
+// src/remote-workflow-executor.ts
+var taskCounter = 0;
+var RemoteWorkflowExecutor = class {
+  static {
+    __name(this, "RemoteWorkflowExecutor");
+  }
+  transport;
+  group;
+  constructor(transport, group, _opts = {}) {
+    this.transport = transport;
+    this.group = group;
+  }
+  /**
+  * Dispatch one workflow turn and return its `taskId`. Does NOT await the decision — the engine
+  * suspends the run on the returned `taskId` and applies the decision durably via `onDecision`.
+  */
+  async dispatch(run, history, pendingSignals) {
+    if (!this.transport.dispatchWorkflowTask) {
+      throw new Error("transport does not support workflow tasks (dispatchWorkflowTask)");
+    }
+    taskCounter += 1;
+    const taskId = `${run.id}:wf:${taskCounter}`;
+    const task = {
+      taskId,
+      runId: run.id,
+      workflow: run.workflow,
+      workflowVersion: run.workflowVersion,
+      input: run.input,
+      history,
+      ...pendingSignals ? {
+        pendingSignals
+      } : {},
+      group: this.group,
+      priority: run.priority,
+      attempt: 1
+    };
+    await this.transport.dispatchWorkflowTask(task);
+    return {
+      taskId
+    };
+  }
+};
+
 // src/singleton-gate.ts
 var SINGLETON_RETRY_MS = 1e3;
 var SINGLETON_RETRY_JITTER_MS = 250;
@@ -801,16 +844,32 @@ var TransportPool = class {
   get size() {
     return this.transports.length;
   }
+  /**
+  * The primary (first) transport — the one `dispatch` prefers when no `preferId` is pinned. A
+  * {@link RemoteWorkflowExecutor} built by `engine.remote()` rides it (it needs a single
+  * `dispatchWorkflowTask`/`onDecision` Transport, not the whole failover pool). Throws if the pool
+  * is empty — there is nothing to dispatch a remote workflow over.
+  */
+  get primary() {
+    const first = this.transports[0];
+    if (!first) {
+      throw new Error("no transport configured \u2014 engine.remote() needs a transport (or transports)");
+    }
+    return first.transport;
+  }
   /** Register the engine's result/heartbeat/step-event handlers on EVERY transport — a result can come
   *  back on whichever one delivered the task, so failover stays symmetric. `onStepEvent` is wired only
   *  on transports that carry streamed workflow step lifecycle (BullMQ); others simply skip it. */
-  bind(onResult, onHeartbeat, onStepEvent) {
+  bind(onResult, onHeartbeat, onStepEvent, onDecision) {
     for (const { transport } of this.transports) {
       transport.onResult(onResult);
       transport.onHeartbeat(onHeartbeat);
       if (onStepEvent && transport.onStepEvent) {
         transport.onStepEvent(onStepEvent);
       }
+      if (onDecision && transport.onDecision) {
+        transport.onDecision(onDecision);
+      }
     }
   }
   /**
@@ -1568,14 +1627,27 @@ var WorkflowEngine = class {
       },
       // A heartbeat resets the liveness window for whatever it targets: an in-flight long STEP (keyed
       // by stepId, see callRemote) or — when stepId is absent — the in-flight workflow TURN (keyed by
-      // runId, see runRemoteExecution's heartbeat-rearmed advance).
+      // runId). For an INLINE-advance turn the reset is an in-memory callback; for a DISPATCH-suspend
+      // turn there is no in-memory waiter, so a run-scoped beat rearms the suspended turn's durable
+      // re-drive deadline (`wakeAt`) instead, keeping a live worker's turn from being re-dispatched.
       async (beat) => {
-        this.heartbeatResets.get(beat.stepId ?? beat.runId)?.();
+        const reset = this.heartbeatResets.get(beat.stepId ?? beat.runId);
+        if (reset) {
+          reset();
+          return;
+        }
+        if (beat.stepId === void 0) await this.rearmDecisionDeadline(beat.runId);
       },
       // A remote workflow worker streams each local step's lifecycle (running → completed/failed) so
       // it's checkpointed live, not all-at-once when the long turn ends.
       async (event) => {
         await this.persistStepEvent(event);
+      },
+      // A workflow-turn DECISION delivered over the broker — applied durably (by run id) on whatever
+      // instance consumes it, NOT correlated to an in-memory promise on the dispatcher. This is the
+      // multi-instance fix: a point-to-point broker may hand the decision to a non-dispatching instance.
+      async (decision) => {
+        await this.completeRemoteDecision(decision);
       }
     );
     this.controlPlane?.onControl((msg) => {
@@ -1720,6 +1792,28 @@ var WorkflowEngine = class {
     if (!current || isNewerVersion(version, current.version)) this.latest.set(name, registered);
   }
   /**
+  * The convenience form of {@link registerRemote}: register a workflow whose body runs on a worker
+  * `group` (e.g. the Python `durable-worker`, or a separate TS worker), without hand-building a
+  * {@link RemoteWorkflowExecutor}. It wires the standard broker executor — a `RemoteWorkflowExecutor`
+  * over the engine's own transport, dispatching to `opts.group` — for you, so you write
+  * `engine.remote('processing', { group: 'processing' })` instead of
+  * `engine.registerRemote('processing', '1', { group: 'processing', executor: new RemoteWorkflowExecutor(transport, 'processing') })`.
+  *
+  * Defaults the version to `'1'`. All the other {@link registerRemote} options
+  * (`tags` / `singleton` / `executionTimeout` / `validateInput`) pass straight through. Reach for the
+  * low-level {@link registerRemote} when you need a custom {@link WorkflowExecutor} (a different
+  * transport, a fixed `timeoutMs`, a polyglot stub) — `remote()` is the standard wiring on the rails.
+  */
+  remote(name, opts) {
+    const { group, version, ...rest } = opts;
+    const executor = new RemoteWorkflowExecutor(this.pool.primary, group);
+    this.registerRemote(name, version ?? "1", {
+      group,
+      executor,
+      ...rest
+    });
+  }
+  /**
   * Register a flow-control queue referenced by `ctx.call(step, input, { queue })`. Caps concurrent
   * in-flight steps and/or the admission rate; blocked calls re-suspend and retry, so the limit is
   * durable. Per engine instance (see {@link QueueConfig}). Registering the same name replaces it.
@@ -2182,7 +2276,7 @@ var WorkflowEngine = class {
   * `start` when a call site needs the outcome — `await start(...); const r = await waitForRun(id)`.
   */
   waitForRun(runId, opts) {
-    const isSettled = /* @__PURE__ */ __name((s) => s !== "pending" && s !== "running" && s !== "cancelling", "isSettled");
+    const isSettled = /* @__PURE__ */ __name((run) => run.status !== "pending" && run.status !== "running" && run.status !== "cancelling" && !(run.status === "suspended" && run.awaitingDecisionTaskId !== void 0), "isSettled");
     const toResult = /* @__PURE__ */ __name((run) => ({
       runId,
       status: run.status,
@@ -2208,7 +2302,7 @@ var WorkflowEngine = class {
       }, "finish");
       const check = /* @__PURE__ */ __name(() => {
         void this.store.getRun(runId).then((run) => {
-          if (run && isSettled(run.status)) finish(run);
+          if (run && isSettled(run)) finish(run);
         });
       }, "check");
       off = this.subscribe((ev) => {
@@ -2815,10 +2909,10 @@ var WorkflowEngine = class {
       });
     }
     if (registered.singleton && !await this.singletons.admit(run, registered.singleton)) {
-      const wakeAt2 = this.singletons.retryWakeAt();
+      const wakeAt = this.singletons.retryWakeAt();
       await this.store.updateRun(run.id, {
         status: "suspended",
-        wakeAt: wakeAt2,
+        wakeAt,
         updatedAt: /* @__PURE__ */ new Date()
       });
       this.emit({
@@ -2833,10 +2927,45 @@ var WorkflowEngine = class {
       };
     }
     const history = await this.remoteHistory(run.id);
+    if (remote.executor.dispatch) {
+      const { taskId } = await remote.executor.dispatch(run, history);
+      const silenceMs = this.remoteAdvanceSilenceMs;
+      const wakeAt = silenceMs != null ? this.clock() + silenceMs : void 0;
+      const fresh = await this.store.getRun(run.id);
+      if (fresh && (fresh.status === "cancelled" || fresh.status === "completed" || fresh.status === "dead")) {
+        await this.store.releaseRunLock(run.id);
+        return {
+          runId: run.id,
+          status: fresh.status,
+          output: fresh.output,
+          error: fresh.error
+        };
+      }
+      await this.store.updateRun(run.id, {
+        status: "suspended",
+        awaitingDecisionTaskId: taskId,
+        wakeAt,
+        updatedAt: /* @__PURE__ */ new Date()
+      });
+      this.emit({
+        type: "run.suspended",
+        runId: run.id,
+        workflow: run.workflow
+      });
+      await this.store.releaseRunLock(run.id);
+      return {
+        runId: run.id,
+        status: "suspended"
+      };
+    }
+    const advanceExecutor = remote.executor.advance;
+    if (!advanceExecutor) {
+      throw new Error(`workflow ${run.workflow}@${run.workflowVersion}: executor implements neither advance nor dispatch`);
+    }
     let decision;
     try {
       const silenceMs = this.remoteAdvanceSilenceMs;
-      const advance = remote.executor.advance(run, history);
+      const advance = advanceExecutor.call(remote.executor, run, history);
       decision = silenceMs != null ? await this.awaitWithLivenessDeadline(run.id, advance, silenceMs, () => new RemoteWorkflowTimeout(run.id, silenceMs)) : await advance;
     } catch (err) {
       if (err instanceof RemoteWorkflowTimeout) {
@@ -2867,6 +2996,16 @@ var WorkflowEngine = class {
         error
       };
     }
+    return this.applyDecision(run, decision);
+  }
+  /**
+  * Apply a remote workflow turn's {@link WorkflowDecision}: persist the local steps it ran, then
+  * settle (completed/failed/cancelled) or dispatch its blocking ops and suspend (continue). Shared by
+  * the INLINE-advance path ({@link runRemoteExecution}, under the held lease) and the durable
+  * DISPATCH path ({@link completeRemoteDecision}, which holds the lease itself). Idempotent transitions
+  * — the caller has already ensured this is the decision for the currently-awaited turn.
+  */
+  async applyDecision(run, decision) {
     if (decision.status === "completed") {
       await this.applyCommands(run, decision.commands);
       return this.settleRun(run, {
@@ -3464,6 +3603,60 @@ var WorkflowEngine = class {
     });
   }
   /**
+  * Apply a remote workflow-turn {@link WorkflowDecision} durably — runs on WHATEVER instance receives
+  * it over the broker (the dispatching one may be gone, or a point-to-point broker handed it to a
+  * non-dispatcher). The durable, multi-instance-safe twin of {@link completeRemoteResult} for the
+  * workflow TURN: look the run up by `decision.runId`, verify the decision matches the run's
+  * CURRENTLY-awaited turn (`awaitingDecisionTaskId`) — so a stale / duplicate / foreign / re-driven
+  * decision is dropped (applied AT MOST ONCE) — lease it, clear the marker, and apply the decision.
+  */
+  async completeRemoteDecision(decision) {
+    const run = await this.store.getRun(decision.runId);
+    if (!run || run.awaitingDecisionTaskId !== decision.taskId) return;
+    if (run.status === "cancelled" || run.status === "completed" || run.status === "dead") return;
+    if (run.namespace !== void 0 && run.namespace !== this.namespace) return;
+    const nowMs = this.clock();
+    if (!await this.store.tryLockRun(decision.runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
+      return;
+    }
+    const renew = setInterval(() => {
+      void this.store.renewRunLock(decision.runId, this.instanceId, this.clock() + this.leaseMs).catch(() => void 0);
+    }, Math.max(50, Math.floor(this.leaseMs / 2)));
+    renew.unref?.();
+    try {
+      const fresh = await this.store.getRun(decision.runId);
+      if (!fresh || fresh.awaitingDecisionTaskId !== decision.taskId) return;
+      if (fresh.status === "cancelled" || fresh.status === "completed" || fresh.status === "dead") {
+        return;
+      }
+      await this.store.updateRun(decision.runId, {
+        awaitingDecisionTaskId: void 0,
+        wakeAt: void 0,
+        updatedAt: /* @__PURE__ */ new Date()
+      });
+      fresh.awaitingDecisionTaskId = void 0;
+      await this.applyDecision(fresh, decision);
+    } finally {
+      clearInterval(renew);
+      await this.store.releaseRunLock(decision.runId);
+    }
+  }
+  /**
+  * A run-scoped heartbeat for a run suspended awaiting a workflow-turn decision (the dispatch path):
+  * push its durable re-drive deadline (`wakeAt`) forward so a still-working worker's turn is not
+  * re-dispatched. The durable counterpart of an inline turn's in-memory `heartbeatResets` rearm.
+  */
+  async rearmDecisionDeadline(runId) {
+    const silenceMs = this.remoteAdvanceSilenceMs;
+    if (silenceMs == null) return;
+    const run = await this.store.getRun(runId);
+    if (!run || run.status !== "suspended" || run.awaitingDecisionTaskId === void 0) return;
+    await this.store.updateRun(runId, {
+      wakeAt: this.clock() + silenceMs,
+      updatedAt: /* @__PURE__ */ new Date()
+    });
+  }
+  /**
   * A slot freed on `queue` (a fleet-wide signal): resume this instance's runs blocked on it so they
   * re-contend now instead of at their retry tick. Snapshot-and-clear — a run still blocked after the
   * retry re-registers itself, and one that's gone (cancelled/admitted) is dropped. Best-effort.
@@ -3615,70 +3808,6 @@ function remoteStep(config) {
 }
 __name(remoteStep, "remoteStep");
 
-// src/remote-workflow-executor.ts
-var taskCounter = 0;
-var RemoteWorkflowExecutor = class {
-  static {
-    __name(this, "RemoteWorkflowExecutor");
-  }
-  transport;
-  group;
-  opts;
-  pending = /* @__PURE__ */ new Map();
-  subscribed = false;
-  constructor(transport, group, opts = {}) {
-    this.transport = transport;
-    this.group = group;
-    this.opts = opts;
-  }
-  ensureSubscribed() {
-    if (this.subscribed) return;
-    if (!this.transport.onDecision) {
-      throw new Error("transport does not support workflow decisions (onDecision)");
-    }
-    this.subscribed = true;
-    this.transport.onDecision(async (decision) => {
-      const resolve = this.pending.get(decision.taskId);
-      if (resolve) {
-        this.pending.delete(decision.taskId);
-        resolve(decision);
-      }
-    });
-  }
-  async advance(run, history) {
-    if (!this.transport.dispatchWorkflowTask) {
-      throw new Error("transport does not support workflow tasks (dispatchWorkflowTask)");
-    }
-    this.ensureSubscribed();
-    taskCounter += 1;
-    const taskId = `${run.id}:wf:${taskCounter}`;
-    const task = {
-      taskId,
-      runId: run.id,
-      workflow: run.workflow,
-      workflowVersion: run.workflowVersion,
-      input: run.input,
-      history,
-      group: this.group,
-      priority: run.priority,
-      attempt: 1
-    };
-    const decision = new Promise((resolve, reject) => {
-      this.pending.set(taskId, resolve);
-      const { timeoutMs } = this.opts;
-      if (timeoutMs) {
-        const timer = setTimeout(() => {
-          this.pending.delete(taskId);
-          reject(new RemoteWorkflowTimeout(taskId, timeoutMs));
-        }, timeoutMs);
-        timer.unref?.();
-      }
-    });
-    await this.transport.dispatchWorkflowTask(task);
-    return decision;
-  }
-};
-
 // src/codec-state-store.ts
 var CodecStateStore = class {
   static {