@dudousxd/nestjs-durable-core 0.49.2 → 0.49.4

package/dist/index.cjs

@@ -57,17 +57,21 @@ __export(index_exports, {
   attributeOperand: () => attributeOperand,
   bindWorkflowClass: () => bindWorkflowClass,
   breakpointToken: () => breakpointToken,
+  classifyWaiterToken: () => classifyWaiterToken,
   collectMetrics: () => collectMetrics,
   createStepLogger: () => createStepLogger,
   currentWorkflowCtx: () => currentWorkflowCtx,
   eventMatchOf: () => eventMatchOf,
   eventMatches: () => eventMatches,
+  eventNameOf: () => eventNameOf,
   eventPrefix: () => eventPrefix,
   eventToken: () => eventToken,
+  indexWaitersByRun: () => indexWaitersByRun,
   matchesAttributes: () => matchesAttributes,
   normalizeAttributeRows: () => normalizeAttributeRows,
   parseDuration: () => parseDuration,
   prevCronFireMs: () => prevCronFireMs,
+  resolveRunWaiting: () => resolveRunWaiting,
   runInWorkflowCtx: () => runInWorkflowCtx,
   runSchedules: () => runSchedules,
   runStepHandler: () => runStepHandler,
@@ -861,6 +865,14 @@ function eventToken(name, match, runId, seq) {
   return `${eventPrefix(name)}${enc(JSON.stringify(match ?? {}))}:${runId}#${seq}`;
 }
 __name(eventToken, "eventToken");
+function eventNameOf(token) {
+  try {
+    return dec(token.split(":")[1] ?? "");
+  } catch {
+    return token;
+  }
+}
+__name(eventNameOf, "eventNameOf");
 function eventMatchOf(token) {
   try {
     return JSON.parse(dec(token.split(":")[2] ?? ""));
@@ -1802,6 +1814,8 @@ var WorkflowEngine = class {
   maxRecoveryAttempts;
   remoteAdvanceSilenceMs;
   reconcileMs;
+  remoteRedispatchMs;
+  remoteRedispatchMax;
   webhookUrl;
   traceparent;
   context;
@@ -1873,6 +1887,8 @@ var WorkflowEngine = class {
     this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
     this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
     this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
+    this.remoteRedispatchMs = deps.remoteRedispatchMs || void 0;
+    this.remoteRedispatchMax = Math.max(1, deps.remoteRedispatchMax ?? 10);
     this.webhookUrl = deps.webhookUrl;
     this.traceparent = deps.traceparent;
     this.context = deps.context;
@@ -2584,6 +2600,49 @@ var WorkflowEngine = class {
     };
   }
   /**
+  * Explicitly re-dispatch every remote step of `runId` stuck `pending` — the OPERATOR escape hatch for
+  * a run whose dispatched step job was LOST (worker crashed with no result, or a transport that dropped
+  * the job). No automatic recovery re-drives these: a reconcile re-drive re-suspends a still-pending
+  * step by design (unless `remoteRedispatchMs` is set), `recoverIncomplete` only reclaims LEASED runs,
+  * and `requeue` just replays back to the pending guard. This re-enqueues the same `stepId` so the
+  * (idempotent) step re-runs and its result resumes the run. Safe to call on a healthy run — it re-runs
+  * only in-flight `pending` remote steps, which are idempotent by the durable contract. Returns the
+  * run's current status (with the count re-dispatched), or null if the run is unknown.
+  */
+  async redispatchPending(runId) {
+    const run = await this.store.getRun(runId);
+    if (!run) return null;
+    const pending = (await this.store.listCheckpoints(runId)).filter((cp) => cp.kind === "remote" && cp.status === "pending");
+    let redispatched = 0;
+    for (const cp of pending) {
+      if (!cp.stepId) continue;
+      const reAttempt = cp.attempts + 1;
+      const reEnqueuedAt = /* @__PURE__ */ new Date();
+      await this.store.saveCheckpoint({
+        ...cp,
+        attempts: reAttempt,
+        enqueuedAt: reEnqueuedAt,
+        startedAt: reEnqueuedAt,
+        finishedAt: reEnqueuedAt
+      });
+      await this.dispatchRemoteTask({
+        runId,
+        seq: cp.seq,
+        name: cp.name,
+        stepId: cp.stepId,
+        group: cp.workerGroup ?? sanitizeQueueToken(cp.name),
+        input: cp.input,
+        attempt: reAttempt
+      });
+      redispatched += 1;
+    }
+    return {
+      runId,
+      status: run.status,
+      redispatched
+    };
+  }
+  /**
   * **Fix-and-replay**: re-run a run (typically a `dead`/`failed` one) with a corrected `input`, as a
   * fresh run with clean history. It's a NEW run — `newRunId` defaults to `<runId>~retry~<uuid>` — so
   * the original stays inspectable. Returns the new run's id, or null if `runId` is unknown.
@@ -4071,7 +4130,47 @@ var WorkflowEngine = class {
     if (existing?.status === "completed") return existing.output;
     if (this.pool.size === 0) throw new Error("remote steps require a Transport");
     if (step.timeoutMs) return this.callRemoteInMemory(runId, seq, step, input, transport);
-    if (existing?.status === "pending") throw new WorkflowSuspended();
+    if (existing?.status === "pending") {
+      if (this.remoteRedispatchMs == null) throw new WorkflowSuspended();
+      if (existing.wakeAt == null) {
+        const wakeAt = this.clock() + this.remoteRedispatchMs;
+        await this.store.saveCheckpoint({
+          ...existing,
+          wakeAt
+        });
+        throw new WorkflowSuspended(wakeAt);
+      }
+      if (this.clock() < existing.wakeAt) throw new WorkflowSuspended(existing.wakeAt);
+      if (existing.attempts >= this.remoteRedispatchMax) {
+        throw new RemoteStepError({
+          message: `remote step "${step.name}" lost \u2014 no result after ${existing.attempts} re-dispatch(es)`,
+          code: "remote_step_lost"
+        });
+      }
+      const reAttempt = existing.attempts + 1;
+      const nextDeadline = this.clock() + this.remoteRedispatchMs;
+      const reEnqueuedAt = /* @__PURE__ */ new Date();
+      await this.store.saveCheckpoint({
+        ...existing,
+        attempts: reAttempt,
+        wakeAt: nextDeadline,
+        enqueuedAt: reEnqueuedAt,
+        startedAt: reEnqueuedAt,
+        finishedAt: reEnqueuedAt
+      });
+      await this.dispatchRemoteTask({
+        runId,
+        seq,
+        name: step.name,
+        stepId: existing.stepId ?? stepId(runId, seq),
+        group: existing.workerGroup ?? tenantGroup(sanitizeQueueToken(step.name), step.partition),
+        input: existing.input,
+        priority: admission?.priority,
+        attempt: reAttempt,
+        transport
+      });
+      throw new WorkflowSuspended(nextDeadline);
+    }
     let attempt = 1;
     if (existing?.status === "failed") {
       const maxAttempts = Math.max(1, step.retries ?? 1);
@@ -4119,18 +4218,17 @@ var WorkflowEngine = class {
       startedAt: enqueuedAt,
       finishedAt: enqueuedAt
     });
-    await this.pool.dispatch({
+    await this.dispatchRemoteTask({
       runId,
       seq,
       name: step.name,
       stepId: id,
       group: tenantGroup(sanitizeQueueToken(step.name), step.partition),
       input,
-      traceparent: this.traceparent?.(),
-      context: this.context?.(),
       priority: admission?.priority,
-      attempt
-    }, transport);
+      attempt,
+      transport
+    });
     this.emit({
       type: "step.started",
       runId,
@@ -4140,6 +4238,23 @@ var WorkflowEngine = class {
     });
     throw new WorkflowSuspended();
   }
+  /** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
+  *  dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
+  *  {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
+  dispatchRemoteTask(task) {
+    return this.pool.dispatch({
+      runId: task.runId,
+      seq: task.seq,
+      name: task.name,
+      stepId: task.stepId,
+      group: task.group,
+      input: task.input,
+      traceparent: this.traceparent?.(),
+      context: this.context?.(),
+      priority: task.priority,
+      attempt: task.attempt
+    }, task.transport);
+  }
   /**
   * Complete a durable remote step from its worker result and resume the run — runs on whichever
   * instance receives the result (the dispatching one may be gone), so the run is crash/scale-safe.
@@ -4376,6 +4491,42 @@ var TERMINAL_RUN_STATUSES = [
   "dead"
 ];
 
+// src/run-waiting.ts
+var WEBHOOK_PREFIX = "wh:";
+var CHILD_PREFIX = "child:";
+var EVENT_PREFIX = "event:";
+function classifyWaiterToken(token) {
+  if (token.startsWith(WEBHOOK_PREFIX)) return {
+    on: "webhook",
+    name: token
+  };
+  if (token.startsWith(CHILD_PREFIX)) return {
+    on: "child",
+    name: token.slice(CHILD_PREFIX.length)
+  };
+  if (token.startsWith(EVENT_PREFIX)) return {
+    on: "signal",
+    name: eventNameOf(token)
+  };
+  return {
+    on: "signal",
+    name: token
+  };
+}
+__name(classifyWaiterToken, "classifyWaiterToken");
+function resolveRunWaiting(run, waiterByRun) {
+  if (run.status !== "suspended") return void 0;
+  const waiter = waiterByRun.get(run.id);
+  return waiter ? classifyWaiterToken(waiter.token) : void 0;
+}
+__name(resolveRunWaiting, "resolveRunWaiting");
+function indexWaitersByRun(waiters) {
+  const byRun = /* @__PURE__ */ new Map();
+  for (const w of waiters) if (!byRun.has(w.runId)) byRun.set(w.runId, w);
+  return byRun;
+}
+__name(indexWaitersByRun, "indexWaitersByRun");
+
 // src/codec-state-store.ts
 var CodecStateStore = class {
   static {
@@ -5131,17 +5282,21 @@ var InMemoryTransport = class {
   attributeOperand,
   bindWorkflowClass,
   breakpointToken,
+  classifyWaiterToken,
   collectMetrics,
   createStepLogger,
   currentWorkflowCtx,
   eventMatchOf,
   eventMatches,
+  eventNameOf,
   eventPrefix,
   eventToken,
+  indexWaitersByRun,
   matchesAttributes,
   normalizeAttributeRows,
   parseDuration,
   prevCronFireMs,
+  resolveRunWaiting,
   runInWorkflowCtx,
   runSchedules,
   runStepHandler,