@dudousxd/nestjs-durable-core 0.49.3 → 0.49.4

package/dist/index.d.cts

@@ -730,6 +730,9 @@ type RunRequestKind = {
     kind: 'retryWithInput';
     runId: string;
     input: unknown;
+} | {
+    kind: 'redispatch';
+    runId: string;
 };
 /** The control plane's answer to a {@link RunRequest}, correlated by `requestId`. */
 interface RunReply {
@@ -1818,6 +1821,27 @@ interface WorkflowEngineDeps {
      * for a genuinely-orphaned run. Default 5min; set `0` to disable (the prior wake-forever behavior).
      */
     reconcileMs?: number | undefined;
+    /**
+     * Opt-in self-heal window (ms) for a remote step with NO `timeoutMs` whose dispatched job was LOST —
+     * the worker crashed mid-step leaving no result, or the transport dropped the job (a Redis
+     * flush/eviction, or BullMQ moving a stalled job to `failed` and removing it). The `pending`
+     * checkpoint then has no result to resume on, and — BY DESIGN (see {@link reconcileMs}) — the
+     * reconcile re-drive re-suspends a still-pending step rather than re-dispatching, so a slow-but-live
+     * worker is never double-run. That safety means a genuinely-lost dispatch would otherwise hang
+     * forever. When set, a reconcile re-drive that finds a remote step still `pending` PAST this window
+     * re-dispatches it (bumping `attempts`, bounded by {@link remoteRedispatchMax}). Off by default:
+     * re-dispatch can double-run a step whose original job is merely slow, so the window MUST exceed the
+     * longest such step and the step MUST be idempotent. Prefer a per-step `timeoutMs` (tighter,
+     * heartbeat-aware) where you can; this is the store-driven net for no-timeout steps that must
+     * survive a lost dispatch. {@link WorkflowEngine.redispatchPending} is the manual counterpart.
+     */
+    remoteRedispatchMs?: number | undefined;
+    /**
+     * Max times {@link remoteRedispatchMs} re-dispatches one lost remote step before giving up and
+     * failing it as a {@link RemoteStepError} (`code: 'remote_step_lost'`), so the run fails / dead-letters
+     * instead of re-dispatching forever. Default 10. Ignored when `remoteRedispatchMs` is unset.
+     */
+    remoteRedispatchMax?: number | undefined;
     /**
      * Build the public callback URL for a `ctx.webhook()` token (e.g.
      * ``(t) => `https://api.example.com/durable/webhooks/${t}` ``). Populates
@@ -1894,6 +1918,8 @@ declare class WorkflowEngine {
     private readonly maxRecoveryAttempts?;
     private readonly remoteAdvanceSilenceMs?;
     private readonly reconcileMs;
+    private readonly remoteRedispatchMs;
+    private readonly remoteRedispatchMax;
     private readonly webhookUrl?;
     private readonly traceparent?;
     private readonly context?;
@@ -2186,6 +2212,19 @@ declare class WorkflowEngine {
      * the HTTP request on workflow execution.
      */
     requeue(runId: string): Promise<RunResult | null>;
+    /**
+     * Explicitly re-dispatch every remote step of `runId` stuck `pending` — the OPERATOR escape hatch for
+     * a run whose dispatched step job was LOST (worker crashed with no result, or a transport that dropped
+     * the job). No automatic recovery re-drives these: a reconcile re-drive re-suspends a still-pending
+     * step by design (unless `remoteRedispatchMs` is set), `recoverIncomplete` only reclaims LEASED runs,
+     * and `requeue` just replays back to the pending guard. This re-enqueues the same `stepId` so the
+     * (idempotent) step re-runs and its result resumes the run. Safe to call on a healthy run — it re-runs
+     * only in-flight `pending` remote steps, which are idempotent by the durable contract. Returns the
+     * run's current status (with the count re-dispatched), or null if the run is unknown.
+     */
+    redispatchPending(runId: string): Promise<(RunResult & {
+        redispatched: number;
+    }) | null>;
     /**
      * **Fix-and-replay**: re-run a run (typically a `dead`/`failed` one) with a corrected `input`, as a
      * fresh run with clean history. It's a NEW run — `newRunId` defaults to `<runId>~retry~<uuid>` — so
@@ -2488,6 +2527,10 @@ declare class WorkflowEngine {
      */
     update(runId: string, name: string, arg: unknown): Promise<UpdateResult>;
     private callRemote;
+    /** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
+     *  dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
+     *  {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
+    private dispatchRemoteTask;
     /**
      * Complete a durable remote step from its worker result and resume the run — runs on whichever
      * instance receives the result (the dispatching one may be gone), so the run is crash/scale-safe.
@@ -2820,6 +2863,12 @@ interface RunGateway {
     }): Promise<RunResult | null>;
     retry(runId: string): Promise<RunResult | null>;
     continue(runId: string): Promise<RunResult | null>;
+    /** Re-dispatch every remote step of a run stuck `pending` — the operator recovery for a LOST step
+     *  dispatch (crashed worker / dropped job) that no automatic path re-drives. Returns the run's status
+     *  plus how many steps were re-dispatched, or null if the run is unknown. */
+    redispatchPending(runId: string): Promise<(RunResult & {
+        redispatched: number;
+    }) | null>;
     retryWithInput(runId: string, input: unknown): Promise<{
         runId: string;
     } | null>;

package/dist/index.d.ts

@@ -730,6 +730,9 @@ type RunRequestKind = {
     kind: 'retryWithInput';
     runId: string;
     input: unknown;
+} | {
+    kind: 'redispatch';
+    runId: string;
 };
 /** The control plane's answer to a {@link RunRequest}, correlated by `requestId`. */
 interface RunReply {
@@ -1818,6 +1821,27 @@ interface WorkflowEngineDeps {
      * for a genuinely-orphaned run. Default 5min; set `0` to disable (the prior wake-forever behavior).
      */
     reconcileMs?: number | undefined;
+    /**
+     * Opt-in self-heal window (ms) for a remote step with NO `timeoutMs` whose dispatched job was LOST —
+     * the worker crashed mid-step leaving no result, or the transport dropped the job (a Redis
+     * flush/eviction, or BullMQ moving a stalled job to `failed` and removing it). The `pending`
+     * checkpoint then has no result to resume on, and — BY DESIGN (see {@link reconcileMs}) — the
+     * reconcile re-drive re-suspends a still-pending step rather than re-dispatching, so a slow-but-live
+     * worker is never double-run. That safety means a genuinely-lost dispatch would otherwise hang
+     * forever. When set, a reconcile re-drive that finds a remote step still `pending` PAST this window
+     * re-dispatches it (bumping `attempts`, bounded by {@link remoteRedispatchMax}). Off by default:
+     * re-dispatch can double-run a step whose original job is merely slow, so the window MUST exceed the
+     * longest such step and the step MUST be idempotent. Prefer a per-step `timeoutMs` (tighter,
+     * heartbeat-aware) where you can; this is the store-driven net for no-timeout steps that must
+     * survive a lost dispatch. {@link WorkflowEngine.redispatchPending} is the manual counterpart.
+     */
+    remoteRedispatchMs?: number | undefined;
+    /**
+     * Max times {@link remoteRedispatchMs} re-dispatches one lost remote step before giving up and
+     * failing it as a {@link RemoteStepError} (`code: 'remote_step_lost'`), so the run fails / dead-letters
+     * instead of re-dispatching forever. Default 10. Ignored when `remoteRedispatchMs` is unset.
+     */
+    remoteRedispatchMax?: number | undefined;
     /**
      * Build the public callback URL for a `ctx.webhook()` token (e.g.
      * ``(t) => `https://api.example.com/durable/webhooks/${t}` ``). Populates
@@ -1894,6 +1918,8 @@ declare class WorkflowEngine {
     private readonly maxRecoveryAttempts?;
     private readonly remoteAdvanceSilenceMs?;
     private readonly reconcileMs;
+    private readonly remoteRedispatchMs;
+    private readonly remoteRedispatchMax;
     private readonly webhookUrl?;
     private readonly traceparent?;
     private readonly context?;
@@ -2186,6 +2212,19 @@ declare class WorkflowEngine {
      * the HTTP request on workflow execution.
      */
     requeue(runId: string): Promise<RunResult | null>;
+    /**
+     * Explicitly re-dispatch every remote step of `runId` stuck `pending` — the OPERATOR escape hatch for
+     * a run whose dispatched step job was LOST (worker crashed with no result, or a transport that dropped
+     * the job). No automatic recovery re-drives these: a reconcile re-drive re-suspends a still-pending
+     * step by design (unless `remoteRedispatchMs` is set), `recoverIncomplete` only reclaims LEASED runs,
+     * and `requeue` just replays back to the pending guard. This re-enqueues the same `stepId` so the
+     * (idempotent) step re-runs and its result resumes the run. Safe to call on a healthy run — it re-runs
+     * only in-flight `pending` remote steps, which are idempotent by the durable contract. Returns the
+     * run's current status (with the count re-dispatched), or null if the run is unknown.
+     */
+    redispatchPending(runId: string): Promise<(RunResult & {
+        redispatched: number;
+    }) | null>;
     /**
      * **Fix-and-replay**: re-run a run (typically a `dead`/`failed` one) with a corrected `input`, as a
      * fresh run with clean history. It's a NEW run — `newRunId` defaults to `<runId>~retry~<uuid>` — so
@@ -2488,6 +2527,10 @@ declare class WorkflowEngine {
      */
     update(runId: string, name: string, arg: unknown): Promise<UpdateResult>;
     private callRemote;
+    /** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
+     *  dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
+     *  {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
+    private dispatchRemoteTask;
     /**
      * Complete a durable remote step from its worker result and resume the run — runs on whichever
      * instance receives the result (the dispatching one may be gone), so the run is crash/scale-safe.
@@ -2820,6 +2863,12 @@ interface RunGateway {
     }): Promise<RunResult | null>;
     retry(runId: string): Promise<RunResult | null>;
     continue(runId: string): Promise<RunResult | null>;
+    /** Re-dispatch every remote step of a run stuck `pending` — the operator recovery for a LOST step
+     *  dispatch (crashed worker / dropped job) that no automatic path re-drives. Returns the run's status
+     *  plus how many steps were re-dispatched, or null if the run is unknown. */
+    redispatchPending(runId: string): Promise<(RunResult & {
+        redispatched: number;
+    }) | null>;
     retryWithInput(runId: string, input: unknown): Promise<{
         runId: string;
     } | null>;

package/dist/index.js

@@ -1729,6 +1729,8 @@ var WorkflowEngine = class {
   maxRecoveryAttempts;
   remoteAdvanceSilenceMs;
   reconcileMs;
+  remoteRedispatchMs;
+  remoteRedispatchMax;
   webhookUrl;
   traceparent;
   context;
@@ -1800,6 +1802,8 @@ var WorkflowEngine = class {
     this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
     this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
     this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
+    this.remoteRedispatchMs = deps.remoteRedispatchMs || void 0;
+    this.remoteRedispatchMax = Math.max(1, deps.remoteRedispatchMax ?? 10);
     this.webhookUrl = deps.webhookUrl;
     this.traceparent = deps.traceparent;
     this.context = deps.context;
@@ -2511,6 +2515,49 @@ var WorkflowEngine = class {
     };
   }
   /**
+  * Explicitly re-dispatch every remote step of `runId` stuck `pending` — the OPERATOR escape hatch for
+  * a run whose dispatched step job was LOST (worker crashed with no result, or a transport that dropped
+  * the job). No automatic recovery re-drives these: a reconcile re-drive re-suspends a still-pending
+  * step by design (unless `remoteRedispatchMs` is set), `recoverIncomplete` only reclaims LEASED runs,
+  * and `requeue` just replays back to the pending guard. This re-enqueues the same `stepId` so the
+  * (idempotent) step re-runs and its result resumes the run. Safe to call on a healthy run — it re-runs
+  * only in-flight `pending` remote steps, which are idempotent by the durable contract. Returns the
+  * run's current status (with the count re-dispatched), or null if the run is unknown.
+  */
+  async redispatchPending(runId) {
+    const run = await this.store.getRun(runId);
+    if (!run) return null;
+    const pending = (await this.store.listCheckpoints(runId)).filter((cp) => cp.kind === "remote" && cp.status === "pending");
+    let redispatched = 0;
+    for (const cp of pending) {
+      if (!cp.stepId) continue;
+      const reAttempt = cp.attempts + 1;
+      const reEnqueuedAt = /* @__PURE__ */ new Date();
+      await this.store.saveCheckpoint({
+        ...cp,
+        attempts: reAttempt,
+        enqueuedAt: reEnqueuedAt,
+        startedAt: reEnqueuedAt,
+        finishedAt: reEnqueuedAt
+      });
+      await this.dispatchRemoteTask({
+        runId,
+        seq: cp.seq,
+        name: cp.name,
+        stepId: cp.stepId,
+        group: cp.workerGroup ?? sanitizeQueueToken(cp.name),
+        input: cp.input,
+        attempt: reAttempt
+      });
+      redispatched += 1;
+    }
+    return {
+      runId,
+      status: run.status,
+      redispatched
+    };
+  }
+  /**
   * **Fix-and-replay**: re-run a run (typically a `dead`/`failed` one) with a corrected `input`, as a
   * fresh run with clean history. It's a NEW run — `newRunId` defaults to `<runId>~retry~<uuid>` — so
   * the original stays inspectable. Returns the new run's id, or null if `runId` is unknown.
@@ -3998,7 +4045,47 @@ var WorkflowEngine = class {
     if (existing?.status === "completed") return existing.output;
     if (this.pool.size === 0) throw new Error("remote steps require a Transport");
     if (step.timeoutMs) return this.callRemoteInMemory(runId, seq, step, input, transport);
-    if (existing?.status === "pending") throw new WorkflowSuspended();
+    if (existing?.status === "pending") {
+      if (this.remoteRedispatchMs == null) throw new WorkflowSuspended();
+      if (existing.wakeAt == null) {
+        const wakeAt = this.clock() + this.remoteRedispatchMs;
+        await this.store.saveCheckpoint({
+          ...existing,
+          wakeAt
+        });
+        throw new WorkflowSuspended(wakeAt);
+      }
+      if (this.clock() < existing.wakeAt) throw new WorkflowSuspended(existing.wakeAt);
+      if (existing.attempts >= this.remoteRedispatchMax) {
+        throw new RemoteStepError({
+          message: `remote step "${step.name}" lost \u2014 no result after ${existing.attempts} re-dispatch(es)`,
+          code: "remote_step_lost"
+        });
+      }
+      const reAttempt = existing.attempts + 1;
+      const nextDeadline = this.clock() + this.remoteRedispatchMs;
+      const reEnqueuedAt = /* @__PURE__ */ new Date();
+      await this.store.saveCheckpoint({
+        ...existing,
+        attempts: reAttempt,
+        wakeAt: nextDeadline,
+        enqueuedAt: reEnqueuedAt,
+        startedAt: reEnqueuedAt,
+        finishedAt: reEnqueuedAt
+      });
+      await this.dispatchRemoteTask({
+        runId,
+        seq,
+        name: step.name,
+        stepId: existing.stepId ?? stepId(runId, seq),
+        group: existing.workerGroup ?? tenantGroup(sanitizeQueueToken(step.name), step.partition),
+        input: existing.input,
+        priority: admission?.priority,
+        attempt: reAttempt,
+        transport
+      });
+      throw new WorkflowSuspended(nextDeadline);
+    }
     let attempt = 1;
     if (existing?.status === "failed") {
       const maxAttempts = Math.max(1, step.retries ?? 1);
@@ -4046,18 +4133,17 @@ var WorkflowEngine = class {
       startedAt: enqueuedAt,
       finishedAt: enqueuedAt
     });
-    await this.pool.dispatch({
+    await this.dispatchRemoteTask({
       runId,
       seq,
       name: step.name,
       stepId: id,
       group: tenantGroup(sanitizeQueueToken(step.name), step.partition),
       input,
-      traceparent: this.traceparent?.(),
-      context: this.context?.(),
       priority: admission?.priority,
-      attempt
-    }, transport);
+      attempt,
+      transport
+    });
     this.emit({
       type: "step.started",
       runId,
@@ -4067,6 +4153,23 @@ var WorkflowEngine = class {
     });
     throw new WorkflowSuspended();
   }
+  /** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
+  *  dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
+  *  {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
+  dispatchRemoteTask(task) {
+    return this.pool.dispatch({
+      runId: task.runId,
+      seq: task.seq,
+      name: task.name,
+      stepId: task.stepId,
+      group: task.group,
+      input: task.input,
+      traceparent: this.traceparent?.(),
+      context: this.context?.(),
+      priority: task.priority,
+      attempt: task.attempt
+    }, task.transport);
+  }
   /**
   * Complete a durable remote step from its worker result and resume the run — runs on whichever
   * instance receives the result (the dispatching one may be gone), so the run is crash/scale-safe.