@dudousxd/nestjs-durable-core 0.79.0 → 0.80.0

package/dist/index.d.cts

@@ -1169,6 +1169,21 @@ interface StateStore {
     tryAdmitSingleton?(runId: string, tag: string, workflow: string, limit: number, retryWakeAt?: number): Promise<boolean>;
     createRun(run: WorkflowRun): Promise<void>;
     updateRun(runId: string, patch: Partial<WorkflowRun>): Promise<void>;
+    /**
+     * {@link updateRun}, but only if the run's CURRENT status is one of `expectedStatuses`: a
+     * compare-and-set on the status. Resolves whether the patch applied; never throws on a mismatch or
+     * a missing run (both resolve `false`). The engine settles a turn's outcome (completed, failed,
+     * suspended, blocked) through it, so a run that reached `cancelled`/`completed`/`dead` through
+     * another path while the turn executed (an operator's `cancel()`, a timeout sweep, a dead-letter)
+     * is never overwritten by the turn's stale outcome.
+     *
+     * For a SQL store the predicate must be part of the `UPDATE ... WHERE` itself; a read-then-write
+     * in application code reintroduces the race this exists to close.
+     *
+     * Optional: a store that omits it gets the engine's read-then-write fallback, which narrows the
+     * race to the gap between that read and the write but cannot close it.
+     */
+    updateRunIf?(runId: string, expectedStatuses: RunStatus[], patch: Partial<WorkflowRun>): Promise<boolean>;
     getRun(runId: string): Promise<WorkflowRun | null>;
     /**
      * Hard-delete a run and all of its rows — the run plus its checkpoints, signal waiters, and
@@ -1228,6 +1243,16 @@ interface StateStore {
     saveCheckpointsIfPending?(checkpoints: StepCheckpoint[]): Promise<boolean[]>;
     /** Used by recovery on boot to find runs to resume (crashed, left `running`). */
     listIncompleteRuns(namespace?: string): Promise<WorkflowRun[]>;
+    /**
+     * The ORPHANS a periodic recovery pass should reclaim: {@link listIncompleteRuns}'s runs
+     * (`running` or `cancelling`) whose lease is free (`lockedUntil` unset or `<= nowMs`), oldest
+     * (`createdAt`) first, at most `limit`. When `namespace` is given, restricted to that partition.
+     *
+     * Optional but recommended: without it the engine lists every incomplete run and filters the
+     * leases in process, which on a busy fleet reads every healthy running run once per tick per
+     * worker.
+     */
+    listOrphanedRuns?(nowMs: number, limit: number, namespace?: string): Promise<WorkflowRun[]>;
     /** The oldest `pending` runs awaiting dispatch (FIFO, by `createdAt`), capped at `limit`. */
     listPendingRuns(limit: number, namespace?: string): Promise<WorkflowRun[]>;
     /**
@@ -3969,6 +3994,18 @@ interface StartOptions {
      * workflow's registered `concurrency`. See {@link ConcurrencyQuota}.
      */
     concurrency?: ConcurrencyQuota | undefined;
+    /**
+     * Delayed start: don't execute before this time (epoch ms, or a `Date`), compared against the
+     * engine's clock. The run is created `suspended` with its `wakeAt` set to it, which is the same
+     * durable timer a `ctx.sleep` parks on: nothing is held in memory, it survives restarts, and the
+     * timer poller (`resumeDueTimers`) starts it once due, on any instance. A time that is not in the
+     * future starts the run at once, as if it were omitted.
+     *
+     * A delayed run reads `suspended` until it fires (so `start` returns `suspended`, and `waitForRun`
+     * settles on that right away), and skips the `run.started` event's `pending → running` hop. It
+     * counts as in flight for singleton back-pressure and concurrency quotas while it waits.
+     */
+    startAt?: number | Date | undefined;
 }
 /**
  * Serialize runs of a workflow that share a key — a durable, FIFO mutex (e.g. one pipeline per base).
@@ -4283,6 +4320,18 @@ declare class WorkflowEngine {
     private readonly awaitedStepFans;
     /** Step fans whose join barrier this instance opened, by `runId:firstSeq` — see {@link settledHere}. */
     private readonly stepFansFiredHere;
+    /**
+     * The in-process execution gate, by run id: the execution of the run currently in flight in THIS
+     * process (`tail`), and at most one resume queued behind it (`queued`). See {@link resumeRun}.
+     *
+     * The run lease is keyed by engine instance, so it fences OTHER processes only: to the store, a
+     * second execution in this same process holds the very same lease (and {@link execute} skips
+     * `tryLockRun` when its snapshot says this instance holds it). Without this gate, a resume landing
+     * while this process was still executing the run (a signal, a late step result, a child wake, an
+     * explicit `resume()`) went straight past the lease and ran the body concurrently: a
+     * not-yet-checkpointed step executed twice.
+     */
+    private readonly runGates;
     /** The tail of each run's queue of results being handled on this instance — see {@link serializedPerRun}. */
     private readonly resultChains;
     /** Each run's results waiting to be settled on this instance — see {@link completeRemoteResult}. */
@@ -4487,7 +4536,49 @@ declare class WorkflowEngine {
      * stale waiter or a late result left behind by a failed run silently re-executes its body.
      */
     private resumeImplicit;
+    /**
+     * One execution of a run at a time in this process. A resume that lands while the run executes
+     * here neither joins that execution (the double-run) nor no-ops against its held lease (a lost
+     * wake: its checkpoint may postdate the running turn's snapshot): it is QUEUED, and re-drives the
+     * run once the running execution has settled and released its lease, re-reading the run then
+     * (terminal: echoed, never re-executed). Resumes landing while one is already queued coalesce
+     * into it: it has not started yet, so it will observe every write they made. Their callers get
+     * the queued execution's result, i.e. the run's state after their own wake was applied.
+     *
+     * Across processes, exclusion stays the lease's job: {@link execute} still `tryLockRun`s when it
+     * does not hold the lease.
+     */
     private resumeRun;
+    /** Drop `gate` once `tail` settles, unless another execution queued behind it meanwhile. */
+    private closeGateAfter;
+    /**
+     * Release a run's lease AFTER its outcome is decided (persisted, or about to be returned). Best
+     * effort, and that matters most on the failure path: the thing that failed the run is frequently
+     * the very store this release talks to (a step that violated a DB constraint leaves Postgres
+     * refusing every further statement on that connection), so the release throws too. A throwing
+     * release in a `finally` REPLACES the run's real outcome with a lease write nobody asked about,
+     * skips the post-turn re-drives after it, and, for a turn driven as a background resume, surfaces
+     * as an unhandled rejection instead of a failed run.
+     *
+     * Letting it go is safe: the lease expires on its own and `recoverIncomplete` re-drives whatever
+     * is still held. A lost release costs one lease window of latency; a masked error costs the
+     * diagnosis.
+     */
+    private releaseLeaseBestEffort;
+    /**
+     * The run's persisted state as a {@link RunResult}, without executing anything: the answer to a
+     * resume issued from inside the run's own execution, and to a settle whose compare-and-set lost
+     * (the path that won already emitted and notified). A run that is gone reads as `cancelled`, the
+     * safest terminal reading, never a fabricated success.
+     */
+    private echoCurrentStatus;
+    /**
+     * The engine's stand-in for {@link StateStore.updateRunIf} on a store without it, called with the
+     * store as `this`: read-then-write, which narrows the race to the gap between the read and the
+     * write (the pre-CAS behavior, now applied to every settle) but cannot close it.
+     */
+    private updateRunIfFallback;
+    private resumeRunNow;
     /**
      * The registration for an UNREGISTERED run, INHERITED from its nearest REMOTE ancestor — or
      * `undefined` if it has none (the skew-protection error case in {@link resume}). Callers check the
@@ -4616,6 +4707,16 @@ declare class WorkflowEngine {
      * replay from their checkpoints, so only the work that had not finished runs again.
      */
     recoverIncomplete(nowMs?: number): Promise<RunResult[]>;
+    /**
+     * The runs {@link recoverIncomplete} should try to reclaim: incomplete runs whose lease is free or
+     * expired, never the healthy ones a live turn is renewing. Every worker runs recovery on every
+     * timer tick, so listing every running run and lock-probing each (one doomed UPDATE per healthy
+     * run per worker per tick) was pure database load. The store answers it directly when it can
+     * ({@link StateStore.listOrphanedRuns}, bounded by {@link RECOVERY_BATCH}); else the full list is
+     * filtered here, which still spares the probes. Shuffled, like the due-timer pass, so instances
+     * recovering at once spread over the batch instead of contending for the same leases.
+     */
+    private orphanedRuns;
     /**
      * The remote step this run dispatched and is still awaiting a result for, if any — the signature
      * that tells {@link recoverIncomplete} "this run's work is ON THE TRANSPORT, not in a crashed
@@ -6063,6 +6164,12 @@ declare class CodecStateStore implements StateStore {
     /** Forwarded verbatim (deletes touch no payload). Bound only when the inner store has it, so the
      *  engine's per-run fallback still kicks in through the wrapper. */
     readonly deleteRuns?: (runIds: string[]) => Promise<void>;
+    /** Decodes the runs like {@link listIncompleteRuns}. Bound only when the inner store has it, so
+     *  the engine's list-and-filter fallback still kicks in through the wrapper. */
+    readonly listOrphanedRuns?: (nowMs: number, limit: number, namespace?: string) => Promise<WorkflowRun[]>;
+    /** Encodes the patch like {@link updateRun}. Bound only when the inner store has it, so the
+     *  engine's read-then-write fallback still kicks in through the wrapper. */
+    readonly updateRunIf?: (runId: string, expectedStatuses: RunStatus[], patch: Partial<WorkflowRun>) => Promise<boolean>;
     /** Encodes like {@link saveCheckpoint}. Bound only when the inner store has it, so the engine's
      *  plain-save fallback still kicks in through the wrapper. */
     readonly saveCheckpointIfPending?: (checkpoint: StepCheckpoint) => Promise<boolean>;
@@ -6112,6 +6219,7 @@ declare class CodecStateStore implements StateStore {
     ensureSchema(): Promise<void>;
     createRun(run: WorkflowRun): Promise<void>;
     updateRun(runId: string, patch: Partial<WorkflowRun>): Promise<void>;
+    private encRunPatch;
     getRun(runId: string): Promise<WorkflowRun | null>;
     deleteRun(runId: string): Promise<void>;
     getCheckpoint(runId: string, seq: number): Promise<StepCheckpoint | null>;
@@ -6551,6 +6659,8 @@ declare class InMemoryStateStore implements StateStore {
     tryAdmitSingleton(runId: string, tag: string, workflow: string, limit: number, retryWakeAt?: number): Promise<boolean>;
     createRun(run: WorkflowRun): Promise<void>;
     updateRun(runId: string, inputPatch: Partial<WorkflowRun>): Promise<void>;
+    updateRunIf(runId: string, expectedStatuses: RunStatus[], patch: Partial<WorkflowRun>): Promise<boolean>;
+    private applyRunPatch;
     getRun(runId: string): Promise<WorkflowRun | null>;
     deleteRun(runId: string): Promise<void>;
     deleteRuns(runIds: string[]): Promise<void>;
@@ -6559,6 +6669,7 @@ declare class InMemoryStateStore implements StateStore {
     saveCheckpoint(checkpoint: StepCheckpoint): Promise<void>;
     saveCheckpointIfPending(checkpoint: StepCheckpoint): Promise<boolean>;
     saveCheckpointsIfPending(checkpoints: StepCheckpoint[]): Promise<boolean[]>;
+    listOrphanedRuns(nowMs: number, limit: number, namespace?: string): Promise<WorkflowRun[]>;
     listIncompleteRuns(namespace?: string): Promise<WorkflowRun[]>;
     listPendingRuns(limit: number, namespace?: string): Promise<WorkflowRun[]>;
     listDueTimers(nowMs: number, namespace?: string, limit?: number): Promise<WorkflowRun[]>;

package/dist/index.d.ts

@@ -1169,6 +1169,21 @@ interface StateStore {
     tryAdmitSingleton?(runId: string, tag: string, workflow: string, limit: number, retryWakeAt?: number): Promise<boolean>;
     createRun(run: WorkflowRun): Promise<void>;
     updateRun(runId: string, patch: Partial<WorkflowRun>): Promise<void>;
+    /**
+     * {@link updateRun}, but only if the run's CURRENT status is one of `expectedStatuses`: a
+     * compare-and-set on the status. Resolves whether the patch applied; never throws on a mismatch or
+     * a missing run (both resolve `false`). The engine settles a turn's outcome (completed, failed,
+     * suspended, blocked) through it, so a run that reached `cancelled`/`completed`/`dead` through
+     * another path while the turn executed (an operator's `cancel()`, a timeout sweep, a dead-letter)
+     * is never overwritten by the turn's stale outcome.
+     *
+     * For a SQL store the predicate must be part of the `UPDATE ... WHERE` itself; a read-then-write
+     * in application code reintroduces the race this exists to close.
+     *
+     * Optional: a store that omits it gets the engine's read-then-write fallback, which narrows the
+     * race to the gap between that read and the write but cannot close it.
+     */
+    updateRunIf?(runId: string, expectedStatuses: RunStatus[], patch: Partial<WorkflowRun>): Promise<boolean>;
     getRun(runId: string): Promise<WorkflowRun | null>;
     /**
      * Hard-delete a run and all of its rows — the run plus its checkpoints, signal waiters, and
@@ -1228,6 +1243,16 @@ interface StateStore {
     saveCheckpointsIfPending?(checkpoints: StepCheckpoint[]): Promise<boolean[]>;
     /** Used by recovery on boot to find runs to resume (crashed, left `running`). */
     listIncompleteRuns(namespace?: string): Promise<WorkflowRun[]>;
+    /**
+     * The ORPHANS a periodic recovery pass should reclaim: {@link listIncompleteRuns}'s runs
+     * (`running` or `cancelling`) whose lease is free (`lockedUntil` unset or `<= nowMs`), oldest
+     * (`createdAt`) first, at most `limit`. When `namespace` is given, restricted to that partition.
+     *
+     * Optional but recommended: without it the engine lists every incomplete run and filters the
+     * leases in process, which on a busy fleet reads every healthy running run once per tick per
+     * worker.
+     */
+    listOrphanedRuns?(nowMs: number, limit: number, namespace?: string): Promise<WorkflowRun[]>;
     /** The oldest `pending` runs awaiting dispatch (FIFO, by `createdAt`), capped at `limit`. */
     listPendingRuns(limit: number, namespace?: string): Promise<WorkflowRun[]>;
     /**
@@ -3969,6 +3994,18 @@ interface StartOptions {
      * workflow's registered `concurrency`. See {@link ConcurrencyQuota}.
      */
     concurrency?: ConcurrencyQuota | undefined;
+    /**
+     * Delayed start: don't execute before this time (epoch ms, or a `Date`), compared against the
+     * engine's clock. The run is created `suspended` with its `wakeAt` set to it, which is the same
+     * durable timer a `ctx.sleep` parks on: nothing is held in memory, it survives restarts, and the
+     * timer poller (`resumeDueTimers`) starts it once due, on any instance. A time that is not in the
+     * future starts the run at once, as if it were omitted.
+     *
+     * A delayed run reads `suspended` until it fires (so `start` returns `suspended`, and `waitForRun`
+     * settles on that right away), and skips the `run.started` event's `pending → running` hop. It
+     * counts as in flight for singleton back-pressure and concurrency quotas while it waits.
+     */
+    startAt?: number | Date | undefined;
 }
 /**
  * Serialize runs of a workflow that share a key — a durable, FIFO mutex (e.g. one pipeline per base).
@@ -4283,6 +4320,18 @@ declare class WorkflowEngine {
     private readonly awaitedStepFans;
     /** Step fans whose join barrier this instance opened, by `runId:firstSeq` — see {@link settledHere}. */
     private readonly stepFansFiredHere;
+    /**
+     * The in-process execution gate, by run id: the execution of the run currently in flight in THIS
+     * process (`tail`), and at most one resume queued behind it (`queued`). See {@link resumeRun}.
+     *
+     * The run lease is keyed by engine instance, so it fences OTHER processes only: to the store, a
+     * second execution in this same process holds the very same lease (and {@link execute} skips
+     * `tryLockRun` when its snapshot says this instance holds it). Without this gate, a resume landing
+     * while this process was still executing the run (a signal, a late step result, a child wake, an
+     * explicit `resume()`) went straight past the lease and ran the body concurrently: a
+     * not-yet-checkpointed step executed twice.
+     */
+    private readonly runGates;
     /** The tail of each run's queue of results being handled on this instance — see {@link serializedPerRun}. */
     private readonly resultChains;
     /** Each run's results waiting to be settled on this instance — see {@link completeRemoteResult}. */
@@ -4487,7 +4536,49 @@ declare class WorkflowEngine {
      * stale waiter or a late result left behind by a failed run silently re-executes its body.
      */
     private resumeImplicit;
+    /**
+     * One execution of a run at a time in this process. A resume that lands while the run executes
+     * here neither joins that execution (the double-run) nor no-ops against its held lease (a lost
+     * wake: its checkpoint may postdate the running turn's snapshot): it is QUEUED, and re-drives the
+     * run once the running execution has settled and released its lease, re-reading the run then
+     * (terminal: echoed, never re-executed). Resumes landing while one is already queued coalesce
+     * into it: it has not started yet, so it will observe every write they made. Their callers get
+     * the queued execution's result, i.e. the run's state after their own wake was applied.
+     *
+     * Across processes, exclusion stays the lease's job: {@link execute} still `tryLockRun`s when it
+     * does not hold the lease.
+     */
     private resumeRun;
+    /** Drop `gate` once `tail` settles, unless another execution queued behind it meanwhile. */
+    private closeGateAfter;
+    /**
+     * Release a run's lease AFTER its outcome is decided (persisted, or about to be returned). Best
+     * effort, and that matters most on the failure path: the thing that failed the run is frequently
+     * the very store this release talks to (a step that violated a DB constraint leaves Postgres
+     * refusing every further statement on that connection), so the release throws too. A throwing
+     * release in a `finally` REPLACES the run's real outcome with a lease write nobody asked about,
+     * skips the post-turn re-drives after it, and, for a turn driven as a background resume, surfaces
+     * as an unhandled rejection instead of a failed run.
+     *
+     * Letting it go is safe: the lease expires on its own and `recoverIncomplete` re-drives whatever
+     * is still held. A lost release costs one lease window of latency; a masked error costs the
+     * diagnosis.
+     */
+    private releaseLeaseBestEffort;
+    /**
+     * The run's persisted state as a {@link RunResult}, without executing anything: the answer to a
+     * resume issued from inside the run's own execution, and to a settle whose compare-and-set lost
+     * (the path that won already emitted and notified). A run that is gone reads as `cancelled`, the
+     * safest terminal reading, never a fabricated success.
+     */
+    private echoCurrentStatus;
+    /**
+     * The engine's stand-in for {@link StateStore.updateRunIf} on a store without it, called with the
+     * store as `this`: read-then-write, which narrows the race to the gap between the read and the
+     * write (the pre-CAS behavior, now applied to every settle) but cannot close it.
+     */
+    private updateRunIfFallback;
+    private resumeRunNow;
     /**
      * The registration for an UNREGISTERED run, INHERITED from its nearest REMOTE ancestor — or
      * `undefined` if it has none (the skew-protection error case in {@link resume}). Callers check the
@@ -4616,6 +4707,16 @@ declare class WorkflowEngine {
      * replay from their checkpoints, so only the work that had not finished runs again.
      */
     recoverIncomplete(nowMs?: number): Promise<RunResult[]>;
+    /**
+     * The runs {@link recoverIncomplete} should try to reclaim: incomplete runs whose lease is free or
+     * expired, never the healthy ones a live turn is renewing. Every worker runs recovery on every
+     * timer tick, so listing every running run and lock-probing each (one doomed UPDATE per healthy
+     * run per worker per tick) was pure database load. The store answers it directly when it can
+     * ({@link StateStore.listOrphanedRuns}, bounded by {@link RECOVERY_BATCH}); else the full list is
+     * filtered here, which still spares the probes. Shuffled, like the due-timer pass, so instances
+     * recovering at once spread over the batch instead of contending for the same leases.
+     */
+    private orphanedRuns;
     /**
      * The remote step this run dispatched and is still awaiting a result for, if any — the signature
      * that tells {@link recoverIncomplete} "this run's work is ON THE TRANSPORT, not in a crashed
@@ -6063,6 +6164,12 @@ declare class CodecStateStore implements StateStore {
     /** Forwarded verbatim (deletes touch no payload). Bound only when the inner store has it, so the
      *  engine's per-run fallback still kicks in through the wrapper. */
     readonly deleteRuns?: (runIds: string[]) => Promise<void>;
+    /** Decodes the runs like {@link listIncompleteRuns}. Bound only when the inner store has it, so
+     *  the engine's list-and-filter fallback still kicks in through the wrapper. */
+    readonly listOrphanedRuns?: (nowMs: number, limit: number, namespace?: string) => Promise<WorkflowRun[]>;
+    /** Encodes the patch like {@link updateRun}. Bound only when the inner store has it, so the
+     *  engine's read-then-write fallback still kicks in through the wrapper. */
+    readonly updateRunIf?: (runId: string, expectedStatuses: RunStatus[], patch: Partial<WorkflowRun>) => Promise<boolean>;
     /** Encodes like {@link saveCheckpoint}. Bound only when the inner store has it, so the engine's
      *  plain-save fallback still kicks in through the wrapper. */
     readonly saveCheckpointIfPending?: (checkpoint: StepCheckpoint) => Promise<boolean>;
@@ -6112,6 +6219,7 @@ declare class CodecStateStore implements StateStore {
     ensureSchema(): Promise<void>;
     createRun(run: WorkflowRun): Promise<void>;
     updateRun(runId: string, patch: Partial<WorkflowRun>): Promise<void>;
+    private encRunPatch;
     getRun(runId: string): Promise<WorkflowRun | null>;
     deleteRun(runId: string): Promise<void>;
     getCheckpoint(runId: string, seq: number): Promise<StepCheckpoint | null>;
@@ -6551,6 +6659,8 @@ declare class InMemoryStateStore implements StateStore {
     tryAdmitSingleton(runId: string, tag: string, workflow: string, limit: number, retryWakeAt?: number): Promise<boolean>;
     createRun(run: WorkflowRun): Promise<void>;
     updateRun(runId: string, inputPatch: Partial<WorkflowRun>): Promise<void>;
+    updateRunIf(runId: string, expectedStatuses: RunStatus[], patch: Partial<WorkflowRun>): Promise<boolean>;
+    private applyRunPatch;
     getRun(runId: string): Promise<WorkflowRun | null>;
     deleteRun(runId: string): Promise<void>;
     deleteRuns(runIds: string[]): Promise<void>;
@@ -6559,6 +6669,7 @@ declare class InMemoryStateStore implements StateStore {
     saveCheckpoint(checkpoint: StepCheckpoint): Promise<void>;
     saveCheckpointIfPending(checkpoint: StepCheckpoint): Promise<boolean>;
     saveCheckpointsIfPending(checkpoints: StepCheckpoint[]): Promise<boolean[]>;
+    listOrphanedRuns(nowMs: number, limit: number, namespace?: string): Promise<WorkflowRun[]>;
     listIncompleteRuns(namespace?: string): Promise<WorkflowRun[]>;
     listPendingRuns(limit: number, namespace?: string): Promise<WorkflowRun[]>;
     listDueTimers(nowMs: number, namespace?: string, limit?: number): Promise<WorkflowRun[]>;