@dudousxd/nestjs-durable-core 0.49.3 → 0.49.4

package/dist/index.cjs

@@ -1814,6 +1814,8 @@ var WorkflowEngine = class {
   maxRecoveryAttempts;
   remoteAdvanceSilenceMs;
   reconcileMs;
+  remoteRedispatchMs;
+  remoteRedispatchMax;
   webhookUrl;
   traceparent;
   context;
@@ -1885,6 +1887,8 @@ var WorkflowEngine = class {
     this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
     this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
     this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
+    this.remoteRedispatchMs = deps.remoteRedispatchMs || void 0;
+    this.remoteRedispatchMax = Math.max(1, deps.remoteRedispatchMax ?? 10);
     this.webhookUrl = deps.webhookUrl;
     this.traceparent = deps.traceparent;
     this.context = deps.context;
@@ -2596,6 +2600,49 @@ var WorkflowEngine = class {
     };
   }
   /**
+  * Explicitly re-dispatch every remote step of `runId` stuck `pending` — the OPERATOR escape hatch for
+  * a run whose dispatched step job was LOST (worker crashed with no result, or a transport that dropped
+  * the job). No automatic recovery re-drives these: a reconcile re-drive re-suspends a still-pending
+  * step by design (unless `remoteRedispatchMs` is set), `recoverIncomplete` only reclaims LEASED runs,
+  * and `requeue` just replays back to the pending guard. This re-enqueues the same `stepId` so the
+  * (idempotent) step re-runs and its result resumes the run. Safe to call on a healthy run — it re-runs
+  * only in-flight `pending` remote steps, which are idempotent by the durable contract. Returns the
+  * run's current status (with the count re-dispatched), or null if the run is unknown.
+  */
+  async redispatchPending(runId) {
+    const run = await this.store.getRun(runId);
+    if (!run) return null;
+    const pending = (await this.store.listCheckpoints(runId)).filter((cp) => cp.kind === "remote" && cp.status === "pending");
+    let redispatched = 0;
+    for (const cp of pending) {
+      if (!cp.stepId) continue;
+      const reAttempt = cp.attempts + 1;
+      const reEnqueuedAt = /* @__PURE__ */ new Date();
+      await this.store.saveCheckpoint({
+        ...cp,
+        attempts: reAttempt,
+        enqueuedAt: reEnqueuedAt,
+        startedAt: reEnqueuedAt,
+        finishedAt: reEnqueuedAt
+      });
+      await this.dispatchRemoteTask({
+        runId,
+        seq: cp.seq,
+        name: cp.name,
+        stepId: cp.stepId,
+        group: cp.workerGroup ?? sanitizeQueueToken(cp.name),
+        input: cp.input,
+        attempt: reAttempt
+      });
+      redispatched += 1;
+    }
+    return {
+      runId,
+      status: run.status,
+      redispatched
+    };
+  }
+  /**
   * **Fix-and-replay**: re-run a run (typically a `dead`/`failed` one) with a corrected `input`, as a
   * fresh run with clean history. It's a NEW run — `newRunId` defaults to `<runId>~retry~<uuid>` — so
   * the original stays inspectable. Returns the new run's id, or null if `runId` is unknown.
@@ -4083,7 +4130,47 @@ var WorkflowEngine = class {
     if (existing?.status === "completed") return existing.output;
     if (this.pool.size === 0) throw new Error("remote steps require a Transport");
     if (step.timeoutMs) return this.callRemoteInMemory(runId, seq, step, input, transport);
-    if (existing?.status === "pending") throw new WorkflowSuspended();
+    if (existing?.status === "pending") {
+      if (this.remoteRedispatchMs == null) throw new WorkflowSuspended();
+      if (existing.wakeAt == null) {
+        const wakeAt = this.clock() + this.remoteRedispatchMs;
+        await this.store.saveCheckpoint({
+          ...existing,
+          wakeAt
+        });
+        throw new WorkflowSuspended(wakeAt);
+      }
+      if (this.clock() < existing.wakeAt) throw new WorkflowSuspended(existing.wakeAt);
+      if (existing.attempts >= this.remoteRedispatchMax) {
+        throw new RemoteStepError({
+          message: `remote step "${step.name}" lost \u2014 no result after ${existing.attempts} re-dispatch(es)`,
+          code: "remote_step_lost"
+        });
+      }
+      const reAttempt = existing.attempts + 1;
+      const nextDeadline = this.clock() + this.remoteRedispatchMs;
+      const reEnqueuedAt = /* @__PURE__ */ new Date();
+      await this.store.saveCheckpoint({
+        ...existing,
+        attempts: reAttempt,
+        wakeAt: nextDeadline,
+        enqueuedAt: reEnqueuedAt,
+        startedAt: reEnqueuedAt,
+        finishedAt: reEnqueuedAt
+      });
+      await this.dispatchRemoteTask({
+        runId,
+        seq,
+        name: step.name,
+        stepId: existing.stepId ?? stepId(runId, seq),
+        group: existing.workerGroup ?? tenantGroup(sanitizeQueueToken(step.name), step.partition),
+        input: existing.input,
+        priority: admission?.priority,
+        attempt: reAttempt,
+        transport
+      });
+      throw new WorkflowSuspended(nextDeadline);
+    }
     let attempt = 1;
     if (existing?.status === "failed") {
       const maxAttempts = Math.max(1, step.retries ?? 1);
@@ -4131,18 +4218,17 @@ var WorkflowEngine = class {
       startedAt: enqueuedAt,
       finishedAt: enqueuedAt
     });
-    await this.pool.dispatch({
+    await this.dispatchRemoteTask({
       runId,
       seq,
       name: step.name,
       stepId: id,
       group: tenantGroup(sanitizeQueueToken(step.name), step.partition),
       input,
-      traceparent: this.traceparent?.(),
-      context: this.context?.(),
       priority: admission?.priority,
-      attempt
-    }, transport);
+      attempt,
+      transport
+    });
     this.emit({
       type: "step.started",
       runId,
@@ -4152,6 +4238,23 @@ var WorkflowEngine = class {
     });
     throw new WorkflowSuspended();
   }
+  /** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
+  *  dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
+  *  {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
+  dispatchRemoteTask(task) {
+    return this.pool.dispatch({
+      runId: task.runId,
+      seq: task.seq,
+      name: task.name,
+      stepId: task.stepId,
+      group: task.group,
+      input: task.input,
+      traceparent: this.traceparent?.(),
+      context: this.context?.(),
+      priority: task.priority,
+      attempt: task.attempt
+    }, task.transport);
+  }
   /**
   * Complete a durable remote step from its worker result and resume the run — runs on whichever
   * instance receives the result (the dispatching one may be gone), so the run is crash/scale-safe.