@dudousxd/nestjs-durable-core 0.70.3 → 0.71.1

package/dist/index.js

@@ -706,6 +706,17 @@ async function runStepHandler(task, handler) {
 __name(runStepHandler, "runStepHandler");
 
 // src/checkpoints.ts
+var REDISPATCHED_STEP_EVENT = "step.redispatched";
+var LOST_STEP_EVENT = "step.lost";
+function lostStepEvent(at, attempts, outcome) {
+  return {
+    at,
+    level: "warn",
+    name: outcome === "lost" ? LOST_STEP_EVENT : REDISPATCHED_STEP_EVENT,
+    message: outcome === "lost" ? `dispatch lost and the re-drive bound is spent after ${attempts} dispatch(es) \u2014 failing the step` : `re-dispatched after a lost dispatch (attempt ${attempts}): the step's lease lapsed with no result and no heartbeat`
+  };
+}
+__name(lostStepEvent, "lostStepEvent");
 function nonEmptyEvents(events) {
   return events && events.length > 0 ? events : void 0;
 }
@@ -2846,16 +2857,21 @@ var WorkflowEngine = class {
       },
       // A heartbeat resets the liveness window for whatever it targets: an in-flight long STEP (keyed
       // by stepId, see callRemote) or — when stepId is absent — the in-flight workflow TURN (keyed by
-      // runId). For an INLINE-advance turn the reset is an in-memory callback; for a DISPATCH-suspend
-      // turn there is no in-memory waiter, so a run-scoped beat rearms the suspended turn's durable
-      // re-drive deadline (`wakeAt`) instead, keeping a live worker's turn from being re-dispatched.
+      // runId). For an INLINE-advance turn the reset is an in-memory callback; without one (the step
+      // or turn suspended durably, possibly on another instance) the beat rearms the DURABLE deadline
+      // instead: the suspended turn's `wakeAt` for a run-scoped beat, the step's lease for a
+      // step-scoped one — either way keeping a live worker's work from being presumed lost.
       async (beat) => {
         const reset = this.heartbeatResets.get(beat.stepId ?? beat.runId);
         if (reset) {
           reset();
           return;
         }
-        if (beat.stepId === void 0) await this.rearmDecisionDeadline(beat.runId);
+        if (beat.stepId === void 0) {
+          await this.rearmDecisionDeadline(beat.runId);
+          return;
+        }
+        await this.rearmStepLease(beat.runId, beat.seq);
       },
       // A remote workflow worker streams each local step's lifecycle (running → completed/failed) so
       // it's checkpointed live, not all-at-once when the long turn ends.
@@ -3500,9 +3516,14 @@ var WorkflowEngine = class {
   async recoverIncomplete(nowMs = this.clock()) {
     if (this.draining) return [];
     const results = [];
-    for (const run of await this.store.listIncompleteRuns(this.namespace)) {
-      const acquired = await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs);
+    for (const listed of await this.store.listIncompleteRuns(this.namespace)) {
+      const acquired = await this.store.tryLockRun(listed.id, this.instanceId, nowMs + this.leaseMs, nowMs);
       if (!acquired) continue;
+      const run = await this.store.getRun(listed.id);
+      if (!run || run.status !== "running" && run.status !== "cancelling") {
+        await this.store.releaseRunLock(listed.id);
+        continue;
+      }
       if (run.status === "running") {
         const dispatched = await this.inFlightRemoteStep(run.id);
         if (dispatched) {
@@ -4835,7 +4856,7 @@ var WorkflowEngine = class {
         error: fresh.error
       };
     }
-    const wakeAt = await this.applyCommands(run, decision.commands);
+    const wakeAt = await this.applyCommands(run, decision.commands, true);
     const suspended = await this.settleRun(run, {
       kind: "suspended",
       wakeAt
@@ -4873,6 +4894,182 @@ var WorkflowEngine = class {
       }
     }
   }
+  /**
+  * A polyglot workflow's re-emitted `call` whose checkpoint already exists: decide whether the step
+  * is still out for delivery, or whether its dispatch was LOST and has to be re-driven. Returns the
+  * lease deadline to park the run on (so the next decision lands exactly when it's due), or
+  * `undefined` when there is nothing to wait for.
+  *
+  * WHY THIS EXISTS. A worker process killed mid-step (OOM, exit 137 — a pod's memory ceiling, not a
+  * bug in anyone's code) takes its in-flight job with it: BullMQ's stalled-check eventually fails the
+  * job, and for a consumer that can't bridge a terminal job failure back into a `StepResult` (any
+  * non-TS SDK) NOTHING settles the checkpoint. It stays `pending` — "out for delivery" — and the
+  * `applyCommands` idempotency guard, which exists so a `ctx.gather_calls` fan-out doesn't
+  * double-dispatch its still-in-flight siblings on every partial resume, skips it on every later turn.
+  * The run then wakes on the reconcile sweep, dispatches a turn, has the call re-emitted, skips it
+  * again, and sleeps. Forever. Observed in the wild on a six-run `gather_calls` fan-out: `pending`,
+  * `attempts = 1`, `wakeAt` NULL, over an hour, no retry.
+  *
+  * THE POLICY (mirrors {@link callRemote}'s `ctx.step` self-heal, which this path silently did not
+  * share — so `remoteRedispatchMs` simply did not apply to a fan-out):
+  *  - a SETTLED checkpoint always wins: a step that completed just before the crash is never re-run;
+  *  - `remoteRedispatchMs` unset (the default) keeps the by-design "re-suspend, never re-dispatch";
+  *  - otherwise the pending checkpoint carries a LEASE in `wakeAt` — a result or a step heartbeat
+  *    ({@link rearmStepLease}) renews it, so a live worker still holding the step is never
+  *    double-dispatched — and only a LAPSED lease re-dispatches, bounded by `remoteRedispatchMax`;
+  *  - past the bound the step is failed `remote_step_lost` and the failure enters the run's history,
+  *    so the workflow's own error path surfaces it instead of the engine looping.
+  */
+  async redriveLostCall(run, cp) {
+    if (cp.kind !== "remote" || cp.status !== "pending") return void 0;
+    if (this.remoteRedispatchMs == null) return void 0;
+    if (cp.wakeAt == null) {
+      const lease2 = this.clock() + this.remoteRedispatchMs;
+      await this.store.saveCheckpoint({
+        ...cp,
+        wakeAt: lease2
+      });
+      return this.stepLeaseWakeAt(lease2);
+    }
+    if (this.clock() < cp.wakeAt) return this.stepLeaseWakeAt(cp.wakeAt);
+    if (cp.attempts >= this.remoteRedispatchMax) {
+      const error2 = {
+        message: `remote step "${cp.name}" lost \u2014 no result after ${cp.attempts} dispatch(es)`,
+        code: "remote_step_lost",
+        retryable: false
+      };
+      const now = /* @__PURE__ */ new Date();
+      await this.store.saveCheckpoint({
+        ...cp,
+        status: "failed",
+        error: error2,
+        wakeAt: void 0,
+        finishedAt: now,
+        events: [
+          ...cp.events ?? [],
+          lostStepEvent(this.clock(), cp.attempts, "lost")
+        ]
+      });
+      this.emit({
+        type: "step.failed",
+        runId: run.id,
+        seq: cp.seq,
+        name: cp.name,
+        kind: "remote",
+        error: error2
+      });
+      return void 0;
+    }
+    const attempts = cp.attempts + 1;
+    const lease = this.clock() + this.remoteRedispatchMs;
+    const at = /* @__PURE__ */ new Date();
+    await this.store.saveCheckpoint({
+      ...cp,
+      attempts,
+      wakeAt: lease,
+      // The re-dispatch restarts the step's wall-clock window; the old one measured a worker that died.
+      enqueuedAt: at,
+      startedAt: at,
+      finishedAt: at,
+      // Observability: the step's own trail says WHY it ran again — a lost worker, not a failure retry.
+      events: [
+        ...cp.events ?? [],
+        lostStepEvent(this.clock(), attempts, "redispatched")
+      ]
+    });
+    await this.dispatchRemoteTask({
+      runId: run.id,
+      seq: cp.seq,
+      name: cp.name,
+      stepId: cp.stepId ?? stepId(run.id, cp.seq),
+      group: cp.workerGroup ?? sanitizeQueueToken(cp.name),
+      input: cp.input,
+      attempt: attempts
+    });
+    this.emit({
+      type: "step.started",
+      runId: run.id,
+      seq: cp.seq,
+      name: cp.name,
+      kind: "remote",
+      redispatched: true
+    });
+    return this.stepLeaseWakeAt(lease);
+  }
+  /**
+  * The run `wakeAt` a step lease should park on: the lease itself, but never LATER than the
+  * `reconcileMs` safety net would have woken the run anyway. A long lease (a fan-out step that
+  * legitimately runs for half an hour) must not stretch the orphan sweep's cadence for this run —
+  * an early wake just re-parks on the un-lapsed lease, which is cheap and convergent.
+  */
+  stepLeaseWakeAt(lease) {
+    const reconcile = this.reconcileMs;
+    return reconcile == null ? lease : Math.min(lease, this.clock() + reconcile);
+  }
+  /** Earliest clock time a step's lease is worth LOOKING at again (`stepId` → clock ms) — the read
+  *  throttle for {@link rearmStepLease}, the step-scoped sibling of {@link decisionDeadlineRearms}. */
+  stepLeaseRearms = /* @__PURE__ */ new Map();
+  /**
+  * A step-scoped heartbeat for a durably-suspended remote step: push its LEASE forward so a worker
+  * that is still working on it is never presumed dead and double-dispatched. The durable counterpart
+  * of {@link callRemoteInMemory}'s in-memory `heartbeatResets` rearm (which only exists on the
+  * instance that dispatched a `timeoutMs` step), and the step-scoped sibling of
+  * {@link rearmDecisionDeadline}. No-op unless the lost-dispatch self-heal is on — without
+  * `remoteRedispatchMs` nothing ever re-dispatches, so there is no lease to defend.
+  *
+  * THROTTLED BY THE LEASE ITSELF, not a fixed interval: a renewal buys at most ONE window, so it is
+  * only WORTH a write once the window is half spent. Beats arrive every couple of seconds from a hot
+  * worker loop — and a `gather_calls` fan-out multiplies that by its in-flight steps — so an
+  * unthrottled rearm is a per-beat SELECT + UPDATE storm (plus a `getRun` + `updateRun` whenever the
+  * run is parked exactly on this step's lease). The `cp.wakeAt >= renewed` guard it replaces never
+  * short-circuited anything, because `renewed` advances with the clock. So:
+  *  - a beat that finds more than half the window still on the clock writes nothing, and records the
+  *    half-spent mark so subsequent beats skip even the `getCheckpoint` READ until then;
+  *  - a beat at or past the mark renews, leaving a full half-window of headroom — the lease can never
+  *    lapse under a worker whose beats are anywhere near their normal cadence.
+  *
+  * That bounds the cost at ONE read + ONE write per half `remoteRedispatchMs` per in-flight step,
+  * independent of beat frequency (with a 30-minute window: two writes an hour). The in-memory mark is
+  * a pure optimisation — cold on a fresh instance, or on the one that didn't dispatch the step — and
+  * the durable half-spent check below re-derives the same answer from the checkpoint, so nothing
+  * depends on it surviving. Cleared wholesale past 1024 entries: a rare extra read, never a leak.
+  */
+  async rearmStepLease(runId, seq) {
+    if (this.remoteRedispatchMs == null) return;
+    const id = stepId(runId, seq);
+    const now = this.clock();
+    const lookAgainAt = this.stepLeaseRearms.get(id);
+    if (lookAgainAt !== void 0 && now < lookAgainAt) return;
+    try {
+      const cp = await this.store.getCheckpoint(runId, seq);
+      if (!cp || cp.kind !== "remote" || cp.status !== "pending") {
+        this.stepLeaseRearms.delete(id);
+        return;
+      }
+      const halfWindow = Math.ceil(this.remoteRedispatchMs / 2);
+      if (cp.wakeAt != null && cp.wakeAt > now + halfWindow) {
+        this.stepLeaseRearms.set(id, cp.wakeAt - halfWindow);
+        return;
+      }
+      const renewed = now + this.remoteRedispatchMs;
+      const previous = cp.wakeAt;
+      await this.store.saveCheckpoint({
+        ...cp,
+        wakeAt: renewed
+      });
+      if (this.stepLeaseRearms.size > 1024) this.stepLeaseRearms.clear();
+      this.stepLeaseRearms.set(id, renewed - halfWindow);
+      if (previous == null) return;
+      const run = await this.store.getRun(runId);
+      if (run?.status === "suspended" && run.wakeAt === previous) {
+        await this.store.updateRun(runId, {
+          wakeAt: renewed,
+          updatedAt: /* @__PURE__ */ new Date()
+        });
+      }
+    } catch {
+    }
+  }
   /** The run's resolved durable ops as replay inputs: completed/failed steps + elapsed timers. */
   async remoteHistory(runId) {
     const checkpoints = await this.store.listCheckpoints(runId);
@@ -4917,8 +5114,13 @@ var WorkflowEngine = class {
     return events.sort((a, b) => a.seq - b.seq);
   }
   /** Apply a turn's commands: persist recorded local steps, dispatch remote calls, schedule timers.
-  *  Returns the earliest timer deadline to suspend on (or undefined — suspended on a result). */
-  async applyCommands(run, commands) {
+  *  Returns the earliest timer deadline to suspend on (or undefined — suspended on a result).
+  *
+  *  `redriveLost` is set only by the `continue` path: a turn that PARKS on its blocking ops is the
+  *  one that may find a re-emitted `call` whose dispatch was lost and needs re-driving (see
+  *  {@link redriveLostCall}). A terminal turn (completed/failed/cancelled) applies its commands only
+  *  to record what already ran, and must never dispatch anything new. */
+  async applyCommands(run, commands, redriveLost = false) {
     let wakeAt;
     for (const cmd of commands) {
       const at = /* @__PURE__ */ new Date();
@@ -4954,7 +5156,13 @@ var WorkflowEngine = class {
           error: cmd.error
         });
       } else if (cmd.kind === "call") {
-        if (await this.store.getCheckpoint(run.id, cmd.seq)) continue;
+        const existing = await this.store.getCheckpoint(run.id, cmd.seq);
+        if (existing) {
+          const leaseAt = redriveLost ? await this.redriveLostCall(run, existing) : void 0;
+          if (leaseAt != null) wakeAt = wakeAt == null ? leaseAt : Math.min(wakeAt, leaseAt);
+          continue;
+        }
+        const lease = this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
         await this.store.saveCheckpoint(stepCheckpoint({
           runId: run.id,
           seq: cmd.seq,
@@ -4963,6 +5171,7 @@ var WorkflowEngine = class {
           status: "pending",
           input: cmd.input,
           attempts: 1,
+          wakeAt: lease,
           // `cmd.group` is the routing declaration the worker's replay emitted (cross-SDK, or a
           // group-served TS body) — treated as an isolation partition on top of the step's own
           // name, mirroring the StepDef formula below (identical expression everywhere).
@@ -4993,6 +5202,10 @@ var WorkflowEngine = class {
           name: cmd.name,
           kind: "remote"
         });
+        if (lease != null) {
+          const leaseWake = this.stepLeaseWakeAt(lease);
+          wakeAt = wakeAt == null ? leaseWake : Math.min(wakeAt, leaseWake);
+        }
       } else if (cmd.kind === "sleep") {
         const deadline = this.clock() + cmd.ms;
         await this.store.saveCheckpoint(stepCheckpoint({
@@ -5170,16 +5383,16 @@ var WorkflowEngine = class {
     try {
       const output = await runInWorkflowCtx(ctx, () => fn(ctx, run.input));
       if (this.cancelRequested.has(run.id)) {
-        return this.compensateAndCancel(run, compensations, replay);
+        return await this.compensateAndCancel(run, compensations, replay);
       }
-      return this.settleRun(run, {
+      return await this.settleRun(run, {
         kind: "completed",
         output
       });
     } catch (err) {
       if (err instanceof ContinueAsNew) {
         if (this.cancelRequested.has(run.id)) {
-          return this.compensateAndCancel(run, compensations, replay);
+          return await this.compensateAndCancel(run, compensations, replay);
         }
         await this.store.updateRun(run.id, {
           status: "completed",
@@ -5206,35 +5419,35 @@ var WorkflowEngine = class {
       }
       if (err instanceof WorkflowBlocked) {
         if (this.cancelRequested.has(run.id)) {
-          return this.compensateAndCancel(run, compensations, replay);
+          return await this.compensateAndCancel(run, compensations, replay);
         }
-        return this.settleRun(run, {
+        return await this.settleRun(run, {
           kind: "blocked",
           blocked: err.plan
         });
       }
       if (err instanceof WorkflowSuspended) {
         if (this.cancelRequested.has(run.id)) {
-          return this.compensateAndCancel(run, compensations, replay);
+          return await this.compensateAndCancel(run, compensations, replay);
         }
-        return this.settleRun(run, {
+        return await this.settleRun(run, {
           kind: "suspended",
           wakeAt: err.wakeAt
         });
       }
       if (this.cancelRequested.has(run.id)) {
-        return this.compensateAndCancel(run, compensations, replay);
+        return await this.compensateAndCancel(run, compensations, replay);
       }
       const error2 = {
         message: err instanceof Error ? err.message : String(err),
         stack: err instanceof Error ? err.stack : void 0
       };
       const pending = await this.unwindCompensations(run, compensations, replay);
-      if (pending) return this.settleRun(run, {
+      if (pending) return await this.settleRun(run, {
         kind: "suspended",
         wakeAt: pending.wakeAt
       });
-      return this.settleRun(run, {
+      return await this.settleRun(run, {
         kind: "failed",
         error: error2
       });
@@ -5648,7 +5861,10 @@ var WorkflowEngine = class {
   }
   async callRemote(runId, seq, step, input, queue, transport, replay, admission, routing) {
     const group = stepGroup(step, routing?.namespace);
-    const existing = replay?.get(seq) ?? await this.store.getCheckpoint(runId, seq);
+    let existing = replay?.get(seq) ?? await this.store.getCheckpoint(runId, seq);
+    if (existing?.status === "pending" && replay?.has(seq)) {
+      existing = await this.store.getCheckpoint(runId, seq) ?? existing;
+    }
     if (existing && existing.name !== step.name) {
       throw new NonDeterminismError(runId, seq, step.name, existing.name);
     }
@@ -5681,7 +5897,13 @@ var WorkflowEngine = class {
         wakeAt: nextDeadline,
         enqueuedAt: reEnqueuedAt,
         startedAt: reEnqueuedAt,
-        finishedAt: reEnqueuedAt
+        finishedAt: reEnqueuedAt,
+        // Same observable marker the polyglot `call` path stamps (see redriveLostCall): the step's own
+        // trail distinguishes a lost-dispatch re-drive from a failure retry.
+        events: [
+          ...existing.events ?? [],
+          lostStepEvent(this.clock(), reAttempt, "redispatched")
+        ]
       });
       await this.dispatchRemoteTask({
         runId,
@@ -5694,6 +5916,14 @@ var WorkflowEngine = class {
         attempt: reAttempt,
         transport
       });
+      this.emit({
+        type: "step.started",
+        runId,
+        seq,
+        name: step.name,
+        kind: "remote",
+        redispatched: true
+      });
       throw new WorkflowSuspended(nextDeadline);
     }
     let attempt = 1;
@@ -5860,20 +6090,48 @@ var WorkflowEngine = class {
       await this.store.releaseRunLock(decision.runId);
     }
   }
+  /** Earliest clock time a suspended turn's deadline is worth LOOKING at again (`runId` → clock ms) —
+  *  the read throttle for {@link rearmDecisionDeadline}, the run-scoped sibling of
+  *  {@link stepLeaseRearms}. */
+  decisionDeadlineRearms = /* @__PURE__ */ new Map();
   /**
   * A run-scoped heartbeat for a run suspended awaiting a workflow-turn decision (the dispatch path):
   * push its durable re-drive deadline (`wakeAt`) forward so a still-working worker's turn is not
   * re-dispatched. The durable counterpart of an inline turn's in-memory `heartbeatResets` rearm.
+  *
+  * THROTTLED exactly as {@link rearmStepLease} is, and for the same reason: a long workflow turn's
+  * worker beats every couple of seconds, and each beat otherwise cost a `getRun` + `updateRun`.
+  * The window here is `remoteAdvanceSilenceMs`, and a renewal buys at most one of them — so it is
+  * only worth a write once the window is half spent, which leaves a full half-window of headroom
+  * before the timer poller would re-drive the turn. The in-memory mark is a pure optimisation: a
+  * cold (or different) instance re-derives the same answer from the run's own `wakeAt`.
   */
   async rearmDecisionDeadline(runId) {
     const silenceMs = this.remoteAdvanceSilenceMs;
     if (silenceMs == null) return;
-    const run = await this.store.getRun(runId);
-    if (!run || run.status !== "suspended" || run.awaitingDecisionTaskId === void 0) return;
-    await this.store.updateRun(runId, {
-      wakeAt: this.clock() + silenceMs,
-      updatedAt: /* @__PURE__ */ new Date()
-    });
+    const now = this.clock();
+    const lookAgainAt = this.decisionDeadlineRearms.get(runId);
+    if (lookAgainAt !== void 0 && now < lookAgainAt) return;
+    try {
+      const run = await this.store.getRun(runId);
+      if (!run || run.status !== "suspended" || run.awaitingDecisionTaskId === void 0) {
+        this.decisionDeadlineRearms.delete(runId);
+        return;
+      }
+      const halfWindow = Math.ceil(silenceMs / 2);
+      if (run.wakeAt != null && run.wakeAt > now + halfWindow) {
+        this.decisionDeadlineRearms.set(runId, run.wakeAt - halfWindow);
+        return;
+      }
+      const renewed = now + silenceMs;
+      await this.store.updateRun(runId, {
+        wakeAt: renewed,
+        updatedAt: /* @__PURE__ */ new Date()
+      });
+      if (this.decisionDeadlineRearms.size > 1024) this.decisionDeadlineRearms.clear();
+      this.decisionDeadlineRearms.set(runId, renewed - halfWindow);
+    } catch {
+    }
   }
   /**
   * A slot freed on `queue` (a fleet-wide signal): resume this instance's runs blocked on it so they