@dudousxd/nestjs-durable-core 0.70.3 → 0.71.1

package/dist/index.d.cts

@@ -2326,6 +2326,15 @@ interface EngineEvent {
     /** The live step event carried by a `step.progress` (the single log line / sub-process outcome a
      *  running step just emitted). Absent on lifecycle events. */
     event?: StepEvent | undefined;
+    /**
+     * On a `step.started`: this dispatch is a lost-dispatch RE-DRIVE, not a first attempt — the step's
+     * lease lapsed with no result and no heartbeat (the worker holding it is presumed gone), so the
+     * engine re-enqueued it (see `WorkflowEngineDeps.remoteRedispatchMs`). Absent on a normal dispatch
+     * and on a failure retry, so a dashboard/alert can tell "re-driven after a lost worker" apart from
+     * "retried after a failure". The step's persisted event trail carries the same fact durably
+     * (`step.redispatched`).
+     */
+    redispatched?: boolean | undefined;
     /** Capabilities the blocked dispatch required (on a `run.blocked` event). */
     requires?: string[] | undefined;
     /** The structured routing delta on a `run.blocked` event (design §7.6): which capability/protocol
@@ -4022,10 +4031,78 @@ declare class WorkflowEngine {
      * Scheduled off the current stack so it runs after the caller released the run lease.
      */
     private redriveIfTurnSawStaleHistory;
+    /**
+     * A polyglot workflow's re-emitted `call` whose checkpoint already exists: decide whether the step
+     * is still out for delivery, or whether its dispatch was LOST and has to be re-driven. Returns the
+     * lease deadline to park the run on (so the next decision lands exactly when it's due), or
+     * `undefined` when there is nothing to wait for.
+     *
+     * WHY THIS EXISTS. A worker process killed mid-step (OOM, exit 137 — a pod's memory ceiling, not a
+     * bug in anyone's code) takes its in-flight job with it: BullMQ's stalled-check eventually fails the
+     * job, and for a consumer that can't bridge a terminal job failure back into a `StepResult` (any
+     * non-TS SDK) NOTHING settles the checkpoint. It stays `pending` — "out for delivery" — and the
+     * `applyCommands` idempotency guard, which exists so a `ctx.gather_calls` fan-out doesn't
+     * double-dispatch its still-in-flight siblings on every partial resume, skips it on every later turn.
+     * The run then wakes on the reconcile sweep, dispatches a turn, has the call re-emitted, skips it
+     * again, and sleeps. Forever. Observed in the wild on a six-run `gather_calls` fan-out: `pending`,
+     * `attempts = 1`, `wakeAt` NULL, over an hour, no retry.
+     *
+     * THE POLICY (mirrors {@link callRemote}'s `ctx.step` self-heal, which this path silently did not
+     * share — so `remoteRedispatchMs` simply did not apply to a fan-out):
+     *  - a SETTLED checkpoint always wins: a step that completed just before the crash is never re-run;
+     *  - `remoteRedispatchMs` unset (the default) keeps the by-design "re-suspend, never re-dispatch";
+     *  - otherwise the pending checkpoint carries a LEASE in `wakeAt` — a result or a step heartbeat
+     *    ({@link rearmStepLease}) renews it, so a live worker still holding the step is never
+     *    double-dispatched — and only a LAPSED lease re-dispatches, bounded by `remoteRedispatchMax`;
+     *  - past the bound the step is failed `remote_step_lost` and the failure enters the run's history,
+     *    so the workflow's own error path surfaces it instead of the engine looping.
+     */
+    private redriveLostCall;
+    /**
+     * The run `wakeAt` a step lease should park on: the lease itself, but never LATER than the
+     * `reconcileMs` safety net would have woken the run anyway. A long lease (a fan-out step that
+     * legitimately runs for half an hour) must not stretch the orphan sweep's cadence for this run —
+     * an early wake just re-parks on the un-lapsed lease, which is cheap and convergent.
+     */
+    private stepLeaseWakeAt;
+    /** Earliest clock time a step's lease is worth LOOKING at again (`stepId` → clock ms) — the read
+     *  throttle for {@link rearmStepLease}, the step-scoped sibling of {@link decisionDeadlineRearms}. */
+    private readonly stepLeaseRearms;
+    /**
+     * A step-scoped heartbeat for a durably-suspended remote step: push its LEASE forward so a worker
+     * that is still working on it is never presumed dead and double-dispatched. The durable counterpart
+     * of {@link callRemoteInMemory}'s in-memory `heartbeatResets` rearm (which only exists on the
+     * instance that dispatched a `timeoutMs` step), and the step-scoped sibling of
+     * {@link rearmDecisionDeadline}. No-op unless the lost-dispatch self-heal is on — without
+     * `remoteRedispatchMs` nothing ever re-dispatches, so there is no lease to defend.
+     *
+     * THROTTLED BY THE LEASE ITSELF, not a fixed interval: a renewal buys at most ONE window, so it is
+     * only WORTH a write once the window is half spent. Beats arrive every couple of seconds from a hot
+     * worker loop — and a `gather_calls` fan-out multiplies that by its in-flight steps — so an
+     * unthrottled rearm is a per-beat SELECT + UPDATE storm (plus a `getRun` + `updateRun` whenever the
+     * run is parked exactly on this step's lease). The `cp.wakeAt >= renewed` guard it replaces never
+     * short-circuited anything, because `renewed` advances with the clock. So:
+     *  - a beat that finds more than half the window still on the clock writes nothing, and records the
+     *    half-spent mark so subsequent beats skip even the `getCheckpoint` READ until then;
+     *  - a beat at or past the mark renews, leaving a full half-window of headroom — the lease can never
+     *    lapse under a worker whose beats are anywhere near their normal cadence.
+     *
+     * That bounds the cost at ONE read + ONE write per half `remoteRedispatchMs` per in-flight step,
+     * independent of beat frequency (with a 30-minute window: two writes an hour). The in-memory mark is
+     * a pure optimisation — cold on a fresh instance, or on the one that didn't dispatch the step — and
+     * the durable half-spent check below re-derives the same answer from the checkpoint, so nothing
+     * depends on it surviving. Cleared wholesale past 1024 entries: a rare extra read, never a leak.
+     */
+    private rearmStepLease;
     /** The run's resolved durable ops as replay inputs: completed/failed steps + elapsed timers. */
     private remoteHistory;
     /** Apply a turn's commands: persist recorded local steps, dispatch remote calls, schedule timers.
-     *  Returns the earliest timer deadline to suspend on (or undefined — suspended on a result). */
+     *  Returns the earliest timer deadline to suspend on (or undefined — suspended on a result).
+     *
+     *  `redriveLost` is set only by the `continue` path: a turn that PARKS on its blocking ops is the
+     *  one that may find a re-emitted `call` whose dispatch was lost and needs re-driving (see
+     *  {@link redriveLostCall}). A terminal turn (completed/failed/cancelled) applies its commands only
+     *  to record what already ran, and must never dispatch anything new. */
     private applyCommands;
     /**
      * Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
@@ -4196,10 +4273,21 @@ declare class WorkflowEngine {
      * decision is dropped (applied AT MOST ONCE) — lease it, clear the marker, and apply the decision.
      */
     private completeRemoteDecision;
+    /** Earliest clock time a suspended turn's deadline is worth LOOKING at again (`runId` → clock ms) —
+     *  the read throttle for {@link rearmDecisionDeadline}, the run-scoped sibling of
+     *  {@link stepLeaseRearms}. */
+    private readonly decisionDeadlineRearms;
     /**
      * A run-scoped heartbeat for a run suspended awaiting a workflow-turn decision (the dispatch path):
      * push its durable re-drive deadline (`wakeAt`) forward so a still-working worker's turn is not
      * re-dispatched. The durable counterpart of an inline turn's in-memory `heartbeatResets` rearm.
+     *
+     * THROTTLED exactly as {@link rearmStepLease} is, and for the same reason: a long workflow turn's
+     * worker beats every couple of seconds, and each beat otherwise cost a `getRun` + `updateRun`.
+     * The window here is `remoteAdvanceSilenceMs`, and a renewal buys at most one of them — so it is
+     * only worth a write once the window is half spent, which leaves a full half-window of headroom
+     * before the timer poller would re-drive the turn. The in-memory mark is a pure optimisation: a
+     * cold (or different) instance re-derives the same answer from the run's own `wakeAt`.
      */
     private rearmDecisionDeadline;
     /**

package/dist/index.d.ts

@@ -2326,6 +2326,15 @@ interface EngineEvent {
     /** The live step event carried by a `step.progress` (the single log line / sub-process outcome a
      *  running step just emitted). Absent on lifecycle events. */
     event?: StepEvent | undefined;
+    /**
+     * On a `step.started`: this dispatch is a lost-dispatch RE-DRIVE, not a first attempt — the step's
+     * lease lapsed with no result and no heartbeat (the worker holding it is presumed gone), so the
+     * engine re-enqueued it (see `WorkflowEngineDeps.remoteRedispatchMs`). Absent on a normal dispatch
+     * and on a failure retry, so a dashboard/alert can tell "re-driven after a lost worker" apart from
+     * "retried after a failure". The step's persisted event trail carries the same fact durably
+     * (`step.redispatched`).
+     */
+    redispatched?: boolean | undefined;
     /** Capabilities the blocked dispatch required (on a `run.blocked` event). */
     requires?: string[] | undefined;
     /** The structured routing delta on a `run.blocked` event (design §7.6): which capability/protocol
@@ -4022,10 +4031,78 @@ declare class WorkflowEngine {
      * Scheduled off the current stack so it runs after the caller released the run lease.
      */
     private redriveIfTurnSawStaleHistory;
+    /**
+     * A polyglot workflow's re-emitted `call` whose checkpoint already exists: decide whether the step
+     * is still out for delivery, or whether its dispatch was LOST and has to be re-driven. Returns the
+     * lease deadline to park the run on (so the next decision lands exactly when it's due), or
+     * `undefined` when there is nothing to wait for.
+     *
+     * WHY THIS EXISTS. A worker process killed mid-step (OOM, exit 137 — a pod's memory ceiling, not a
+     * bug in anyone's code) takes its in-flight job with it: BullMQ's stalled-check eventually fails the
+     * job, and for a consumer that can't bridge a terminal job failure back into a `StepResult` (any
+     * non-TS SDK) NOTHING settles the checkpoint. It stays `pending` — "out for delivery" — and the
+     * `applyCommands` idempotency guard, which exists so a `ctx.gather_calls` fan-out doesn't
+     * double-dispatch its still-in-flight siblings on every partial resume, skips it on every later turn.
+     * The run then wakes on the reconcile sweep, dispatches a turn, has the call re-emitted, skips it
+     * again, and sleeps. Forever. Observed in the wild on a six-run `gather_calls` fan-out: `pending`,
+     * `attempts = 1`, `wakeAt` NULL, over an hour, no retry.
+     *
+     * THE POLICY (mirrors {@link callRemote}'s `ctx.step` self-heal, which this path silently did not
+     * share — so `remoteRedispatchMs` simply did not apply to a fan-out):
+     *  - a SETTLED checkpoint always wins: a step that completed just before the crash is never re-run;
+     *  - `remoteRedispatchMs` unset (the default) keeps the by-design "re-suspend, never re-dispatch";
+     *  - otherwise the pending checkpoint carries a LEASE in `wakeAt` — a result or a step heartbeat
+     *    ({@link rearmStepLease}) renews it, so a live worker still holding the step is never
+     *    double-dispatched — and only a LAPSED lease re-dispatches, bounded by `remoteRedispatchMax`;
+     *  - past the bound the step is failed `remote_step_lost` and the failure enters the run's history,
+     *    so the workflow's own error path surfaces it instead of the engine looping.
+     */
+    private redriveLostCall;
+    /**
+     * The run `wakeAt` a step lease should park on: the lease itself, but never LATER than the
+     * `reconcileMs` safety net would have woken the run anyway. A long lease (a fan-out step that
+     * legitimately runs for half an hour) must not stretch the orphan sweep's cadence for this run —
+     * an early wake just re-parks on the un-lapsed lease, which is cheap and convergent.
+     */
+    private stepLeaseWakeAt;
+    /** Earliest clock time a step's lease is worth LOOKING at again (`stepId` → clock ms) — the read
+     *  throttle for {@link rearmStepLease}, the step-scoped sibling of {@link decisionDeadlineRearms}. */
+    private readonly stepLeaseRearms;
+    /**
+     * A step-scoped heartbeat for a durably-suspended remote step: push its LEASE forward so a worker
+     * that is still working on it is never presumed dead and double-dispatched. The durable counterpart
+     * of {@link callRemoteInMemory}'s in-memory `heartbeatResets` rearm (which only exists on the
+     * instance that dispatched a `timeoutMs` step), and the step-scoped sibling of
+     * {@link rearmDecisionDeadline}. No-op unless the lost-dispatch self-heal is on — without
+     * `remoteRedispatchMs` nothing ever re-dispatches, so there is no lease to defend.
+     *
+     * THROTTLED BY THE LEASE ITSELF, not a fixed interval: a renewal buys at most ONE window, so it is
+     * only WORTH a write once the window is half spent. Beats arrive every couple of seconds from a hot
+     * worker loop — and a `gather_calls` fan-out multiplies that by its in-flight steps — so an
+     * unthrottled rearm is a per-beat SELECT + UPDATE storm (plus a `getRun` + `updateRun` whenever the
+     * run is parked exactly on this step's lease). The `cp.wakeAt >= renewed` guard it replaces never
+     * short-circuited anything, because `renewed` advances with the clock. So:
+     *  - a beat that finds more than half the window still on the clock writes nothing, and records the
+     *    half-spent mark so subsequent beats skip even the `getCheckpoint` READ until then;
+     *  - a beat at or past the mark renews, leaving a full half-window of headroom — the lease can never
+     *    lapse under a worker whose beats are anywhere near their normal cadence.
+     *
+     * That bounds the cost at ONE read + ONE write per half `remoteRedispatchMs` per in-flight step,
+     * independent of beat frequency (with a 30-minute window: two writes an hour). The in-memory mark is
+     * a pure optimisation — cold on a fresh instance, or on the one that didn't dispatch the step — and
+     * the durable half-spent check below re-derives the same answer from the checkpoint, so nothing
+     * depends on it surviving. Cleared wholesale past 1024 entries: a rare extra read, never a leak.
+     */
+    private rearmStepLease;
     /** The run's resolved durable ops as replay inputs: completed/failed steps + elapsed timers. */
     private remoteHistory;
     /** Apply a turn's commands: persist recorded local steps, dispatch remote calls, schedule timers.
-     *  Returns the earliest timer deadline to suspend on (or undefined — suspended on a result). */
+     *  Returns the earliest timer deadline to suspend on (or undefined — suspended on a result).
+     *
+     *  `redriveLost` is set only by the `continue` path: a turn that PARKS on its blocking ops is the
+     *  one that may find a re-emitted `call` whose dispatch was lost and needs re-driving (see
+     *  {@link redriveLostCall}). A terminal turn (completed/failed/cancelled) applies its commands only
+     *  to record what already ran, and must never dispatch anything new. */
     private applyCommands;
     /**
      * Undo the registered saga compensations in reverse and settle the run `cancelled`. Shared by EVERY
@@ -4196,10 +4273,21 @@ declare class WorkflowEngine {
      * decision is dropped (applied AT MOST ONCE) — lease it, clear the marker, and apply the decision.
      */
     private completeRemoteDecision;
+    /** Earliest clock time a suspended turn's deadline is worth LOOKING at again (`runId` → clock ms) —
+     *  the read throttle for {@link rearmDecisionDeadline}, the run-scoped sibling of
+     *  {@link stepLeaseRearms}. */
+    private readonly decisionDeadlineRearms;
     /**
      * A run-scoped heartbeat for a run suspended awaiting a workflow-turn decision (the dispatch path):
      * push its durable re-drive deadline (`wakeAt`) forward so a still-working worker's turn is not
      * re-dispatched. The durable counterpart of an inline turn's in-memory `heartbeatResets` rearm.
+     *
+     * THROTTLED exactly as {@link rearmStepLease} is, and for the same reason: a long workflow turn's
+     * worker beats every couple of seconds, and each beat otherwise cost a `getRun` + `updateRun`.
+     * The window here is `remoteAdvanceSilenceMs`, and a renewal buys at most one of them — so it is
+     * only worth a write once the window is half spent, which leaves a full half-window of headroom
+     * before the timer poller would re-drive the turn. The in-memory mark is a pure optimisation: a
+     * cold (or different) instance re-derives the same answer from the run's own `wakeAt`.
      */
     private rearmDecisionDeadline;
     /**