@dudousxd/nestjs-durable-core 0.81.0 → 0.82.0

package/dist/index.js

@@ -2816,6 +2816,66 @@ var SingletonGate = class {
   }
 };
 
+// src/stalled-runs.ts
+var DEFAULT_STALLED_AFTER_MS = 15 * 6e4;
+async function findStalledRuns(store, options = {}) {
+  const now = options.now ?? Date.now();
+  const thresholdMs = options.thresholdMs ?? DEFAULT_STALLED_AFTER_MS;
+  const scan = Math.max(1, options.scan ?? 1e3);
+  const pageSize = Math.min(200, scan);
+  const stalled = [];
+  for (let offset = 0; offset < scan; offset += pageSize) {
+    const limit = Math.min(pageSize, scan - offset);
+    const page = await store.listRuns({
+      statuses: [
+        "running",
+        "suspended"
+      ],
+      ...options.namespace !== void 0 ? {
+        namespace: options.namespace
+      } : {},
+      limit,
+      offset
+    });
+    for (const run of page) {
+      const ageMs = now - run.updatedAt.getTime();
+      if (ageMs < thresholdMs) continue;
+      const info2 = await stalledInfo(store, run, now, thresholdMs, ageMs);
+      if (info2) stalled.push(info2);
+    }
+    if (page.length < limit) break;
+  }
+  return stalled;
+}
+__name(findStalledRuns, "findStalledRuns");
+async function stalledInfo(store, run, now, thresholdMs, ageMs) {
+  let stalePending = null;
+  for (const cp of await store.listCheckpoints(run.id)) {
+    if (cp.kind !== "remote" || cp.status !== "pending") continue;
+    const stepAgeMs = now - cp.enqueuedAt.getTime();
+    if (stepAgeMs < thresholdMs) continue;
+    if (cp.wakeAt != null && cp.wakeAt > now) continue;
+    if (!stalePending || stepAgeMs > stalePending.ageMs) {
+      stalePending = {
+        seq: cp.seq,
+        name: cp.name,
+        attempts: cp.attempts,
+        ageMs: stepAgeMs,
+        leaseUntil: cp.wakeAt ?? null
+      };
+    }
+  }
+  const wakeForever = run.status === "suspended" && run.wakeAt == null;
+  if (!wakeForever && !stalePending) return null;
+  return {
+    run,
+    ageMs,
+    wakeForever,
+    stalePending
+  };
+}
+__name(stalledInfo, "stalledInfo");
+
 // src/transport-pool.ts
 var TransportPool = class {
   static {
@@ -4173,6 +4233,18 @@ var WorkflowEngine = class {
   stepQueue = /* @__PURE__ */ new Map();
   /** Executions currently in flight, so a graceful shutdown can wait for them to settle. */
   inflight = /* @__PURE__ */ new Set();
+  /**
+  * The run lease of each execution in flight on THIS instance, by run id — renewed while it runs
+  * and checked by every settle (the zombie fence, see {@link ExecutionLease}). Also what `drain()`
+  * releases when its budget runs out.
+  */
+  executionLeases = /* @__PURE__ */ new Map();
+  /** Pager hooks for stranded runs (see {@link onStalled}). */
+  stalledListeners = /* @__PURE__ */ new Set();
+  /** Runs already paged about, by id → the `updatedAt` of the paged episode (see {@link sweepStalled}). */
+  notifiedStalled = /* @__PURE__ */ new Map();
+  lastStalledSweepAt = Number.NEGATIVE_INFINITY;
+  stalledAfterMs;
   draining = false;
   /** Wait already spent on this shutdown before `drain()` (see `beginDrain`). */
   drainSpentMs = 0;
@@ -4197,6 +4269,8 @@ var WorkflowEngine = class {
     this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
     this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
     this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
+    const stalledAfter = deps.stalledAfterMs;
+    this.stalledAfterMs = stalledAfter !== void 0 && Number.isFinite(stalledAfter) ? Math.max(6e4, stalledAfter) : DEFAULT_STALLED_AFTER_MS;
     const batch = deps.timerBatchSize;
     this.timerBatchSize = batch !== void 0 && Number.isFinite(batch) && batch >= 1 ? Math.floor(batch) : 500;
     this.eventBufferTtlMs = deps.eventBufferTtlMs || void 0;
@@ -4540,6 +4614,64 @@ var WorkflowEngine = class {
     this.deadListeners.add(listener);
     return () => this.deadListeners.delete(listener);
   }
+  /**
+  * Register a pager hook for STRANDED runs: an in-flight run untouched past `stalledAfterMs` that is
+  * either suspended with NO wake timer (wake-forever: nothing will ever re-drive it) or parked on an
+  * old pending remote step with no live lease (dispatched and, most likely, lost). The timer poller
+  * sweeps for these ({@link sweepStalled}, throttled to one pass a minute) and notifies each episode
+  * ONCE per instance: a run is paged again only after it makes progress and strands again. Wire your
+  * alerting here like {@link onDead}. A listener that throws never breaks the sweep. Returns an
+  * unsubscribe function.
+  */
+  onStalled(listener) {
+    this.stalledListeners.add(listener);
+    return () => this.stalledListeners.delete(listener);
+  }
+  /**
+  * Detect stranded in-flight runs and notify the {@link onStalled} listeners. Self-throttled to one
+  * pass a minute, skipped while draining, and a no-op without listeners — call it every tick (the
+  * NestJS timer poller does). Scoped to this engine's `namespace` (an operator engine with no
+  * namespace sweeps every namespace). Returns the runs notified THIS pass.
+  */
+  async sweepStalled(now = this.clock()) {
+    if (this.stalledListeners.size === 0 || this.draining) return [];
+    if (now - this.lastStalledSweepAt < 6e4) return [];
+    this.lastStalledSweepAt = now;
+    const notified = [];
+    for (const info2 of await this.listStalledRuns({
+      now
+    })) {
+      const episode = info2.run.updatedAt.getTime();
+      if (this.notifiedStalled.get(info2.run.id) === episode) continue;
+      this.notifiedStalled.set(info2.run.id, episode);
+      if (this.notifiedStalled.size > 1e3) this.notifiedStalled.clear();
+      for (const listener of this.stalledListeners) {
+        try {
+          listener(info2);
+        } catch {
+        }
+      }
+      notified.push(info2);
+    }
+    return notified;
+  }
+  /**
+  * The stranded runs right now, without notifying anyone: the query behind {@link sweepStalled} and
+  * the CLI's `durable inspect --stale`. A `running`/`suspended` run qualifies when it has been
+  * untouched for `thresholdMs` (default `stalledAfterMs`) AND either it is suspended with no wake
+  * timer, or its oldest pending remote step is older than the threshold with no lease in the future
+  * (a step with a `timeoutMs`, a heartbeat or `remoteRedispatchMs` keeps a lease; a long
+  * `ctx.sleep` has a future `wakeAt`; neither is stranded). Scans up to `scan` candidates (default
+  * 1000), newest first, in this engine's namespace.
+  */
+  async listStalledRuns(options = {}) {
+    return findStalledRuns(this.store, {
+      now: options.now ?? this.clock(),
+      thresholdMs: options.thresholdMs ?? this.stalledAfterMs,
+      scan: options.scan,
+      namespace: this.namespace
+    });
+  }
   notifyDead(run) {
     for (const fn of this.deadListeners) {
       try {
@@ -4698,6 +4830,11 @@ var WorkflowEngine = class {
   getRun(runId) {
     return this.store.getRun(runId);
   }
+  /** A run's full checkpoint timeline, ordered by seq. A thin pass-through to the store (it is what
+  *  the testing kit's `captureHistory` reads, next to {@link getRun}). */
+  listCheckpoints(runId) {
+    return this.store.listCheckpoints(runId);
+  }
   /**
   * Re-execute a run's body (replaying its checkpoints). This is the EXPLICIT entry point: besides
   * driving a live run, it re-drives a `failed` run — the manual retry (prefer {@link requeue}, which
@@ -4783,6 +4920,8 @@ var WorkflowEngine = class {
   * diagnosis.
   */
   async releaseLeaseBestEffort(runId) {
+    if (this.isFenced(runId)) return;
+    this.handOffLease(runId);
     try {
       await this.store.releaseRunLock(runId);
     } catch (err) {
@@ -5032,17 +5171,47 @@ var WorkflowEngine = class {
       rearm();
     });
     this.drainRearms.add(rearm);
+    let settled = false;
     try {
       await Promise.race([
         Promise.allSettled([
           ...this.inflight
-        ]),
+        ]).then(() => {
+          settled = true;
+        }),
         timer
       ]);
     } finally {
       this.drainRearms.delete(rearm);
       clearTimeout(timeout);
     }
+    if (!settled) await this.releaseInflightLeases();
+  }
+  /**
+  * The drain budget ran out with executions still in flight: release their recovery leases so
+  * another instance re-drives those runs on its next recovery pass, instead of waiting out
+  * `leaseMs` after this process exits. Each still-running turn is FENCED first, so whatever it
+  * computes after this point is never written over the new owner's work (see settleRun). Its
+  * completed steps are checkpointed, so the new owner replays them; a step still executing here
+  * may run again there, exactly as after a crash. The release is owner-scoped where the store
+  * supports it (only a lease this instance still holds is cleared). Best effort: never throws.
+  */
+  async releaseInflightLeases() {
+    const released = [];
+    for (const [runId, lease] of [
+      ...this.executionLeases
+    ]) {
+      if (lease.lost || lease.handedOff || lease.done) continue;
+      lease.lost = true;
+      try {
+        await this.store.releaseRunLock(runId, this.instanceId);
+        released.push(runId);
+      } catch {
+      }
+    }
+    if (released.length > 0) {
+      console.warn(`[nestjs-durable] drain timed out with ${released.length} run(s) still executing; released their leases so another instance recovers them now: ${released.join(", ")}`);
+    }
   }
   /**
   * Cancel in-flight runs that have outlived their workflow's `executionTimeout`. Call it from the
@@ -6532,7 +6701,10 @@ var WorkflowEngine = class {
   }
   async singletonWaitResult(run) {
     const current = await this.store.getRun(run.id);
-    await this.store.releaseRunLock(run.id);
+    if (!this.isFenced(run.id)) {
+      this.handOffLease(run.id);
+      await this.store.releaseRunLock(run.id);
+    }
     if (!current) throw new Error(`run ${run.id} was deleted during singleton admission`);
     if (current.status === "suspended") {
       this.emit({
@@ -6560,10 +6732,7 @@ var WorkflowEngine = class {
         };
       }
     }
-    const renew = setInterval(() => {
-      void this.store.renewRunLock(run.id, this.instanceId, this.clock() + this.leaseMs).catch(() => void 0);
-    }, Math.max(50, Math.floor(this.leaseMs / 2)));
-    renew.unref?.();
+    const held = this.holdExecutionLease(run.id);
     try {
       const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
       if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
@@ -6571,10 +6740,59 @@ var WorkflowEngine = class {
       }
       return result;
     } finally {
-      clearInterval(renew);
+      held.stop();
     }
   }
   /**
+  * Renew `runId`'s lease for as long as this execution holds it, and register it in
+  * {@link executionLeases} so settles can check the zombie fence and `drain()` can release it.
+  *
+  * A renewal that returns `false` means the lease is gone: this turn stalled past `leaseMs` and
+  * another instance reclaimed the run, so this executor is now a zombie. It is FENCED (`lost`): its
+  * settles degrade to an echo of the store's state instead of clobbering the new owner's writes, and
+  * its own lease release is skipped (it would free the new owner's lease). A renewal that THROWS (a
+  * transient store error) is not evidence of a takeover, so it stays swallowed and the next
+  * interval retries. A lease this execution released on purpose (`handedOff`) is not renewed, and a
+  * renewal resolving after the execution ended (`done`) fences nothing.
+  */
+  holdExecutionLease(runId) {
+    const lease = {
+      lost: false,
+      handedOff: false,
+      done: false
+    };
+    this.executionLeases.set(runId, lease);
+    const renew = setInterval(() => {
+      if (lease.handedOff || lease.lost || lease.done) return;
+      void this.store.renewRunLock(runId, this.instanceId, this.clock() + this.leaseMs).then((renewed) => {
+        if (renewed || lease.handedOff || lease.lost || lease.done) return;
+        lease.lost = true;
+        console.warn(`[nestjs-durable] lost the lease of run ${runId} while executing it (another instance took it over after it expired); this instance will not write the run's outcome`);
+      }).catch(() => void 0);
+    }, Math.max(50, Math.floor(this.leaseMs / 2)));
+    renew.unref?.();
+    return {
+      lease,
+      stop: /* @__PURE__ */ __name(() => {
+        clearInterval(renew);
+        lease.done = true;
+        if (this.executionLeases.get(runId) === lease) this.executionLeases.delete(runId);
+      }, "stop")
+    };
+  }
+  /** Whether `runId`'s execution on this instance lost its lease (see {@link holdExecutionLease}). */
+  isFenced(runId) {
+    return this.executionLeases.get(runId)?.lost === true;
+  }
+  /**
+  * Mark `runId`'s lease as given away on purpose by its own execution (released mid-turn), so a
+  * renewal that now fails is not read as a takeover.
+  */
+  handOffLease(runId) {
+    const lease = this.executionLeases.get(runId);
+    if (lease) lease.handedOff = true;
+  }
+  /**
   * Advance a remote (cross-SDK) workflow one turn: hand its history to the executor (which dispatches
   * a workflow task to the worker and awaits its replay) and apply the decision. Mirrors
   * {@link runExecution}'s settle/suspend; the lease is held by {@link execute}. The result that lands
@@ -6791,6 +7009,7 @@ ${run.workflowVersion}`;
   * keeps its own lease handling (the TS executor's `finally`, the remote path's caller).
   */
   async settleRun(run, outcome) {
+    if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
     const updatedAt = /* @__PURE__ */ new Date();
     if (outcome.kind === "completed") {
       const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, {
@@ -6871,6 +7090,7 @@ ${run.workflowVersion}`;
   }
   async runRemoteExecution(run, registered) {
     if (run.status === "cancelling") {
+      if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
       const error2 = {
         message: "cancelled"
       };
@@ -6879,6 +7099,7 @@ ${run.workflowVersion}`;
         error: error2,
         updatedAt: /* @__PURE__ */ new Date()
       });
+      if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
       this.emit({
         type: "run.failed",
         runId: run.id,
@@ -6925,6 +7146,7 @@ ${run.workflowVersion}`;
     }
     const history = await this.remoteHistory(run.id, checkpoints);
     if (remote.executor.dispatch) {
+      if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
       const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID().slice(0, 8)}`;
       const silenceMs = this.remoteAdvanceSilenceMs;
       const wakeAt = silenceMs != null ? this.clock() + silenceMs : void 0;
@@ -6940,6 +7162,7 @@ ${run.workflowVersion}`;
         workflow: run.workflow,
         namespace: run.namespace
       });
+      this.handOffLease(run.id);
       await this.store.releaseRunLock(run.id);
       await remote.executor.dispatch(run, history, taskId);
       return {
@@ -6968,6 +7191,7 @@ ${run.workflowVersion}`;
         message: err instanceof Error ? err.message : String(err),
         stack: err instanceof Error ? err.stack : void 0
       };
+      if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
       await this.store.updateRun(run.id, {
         status: "failed",
         error: error2,
@@ -6996,6 +7220,7 @@ ${run.workflowVersion}`;
   * — the caller has already ensured this is the decision for the currently-awaited turn.
   */
   async applyDecision(run, decision) {
+    if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
     if (decision.status === "completed") {
       await this.applyCommands(run, decision.commands);
       return this.settleRun(run, {
@@ -7465,6 +7690,7 @@ ${run.workflowVersion}`;
   * cancel flag stays too, re-derived from the status on any later re-drive regardless.
   */
   async compensateAndCancel(run, compensations, replay) {
+    if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
     const pending = await this.unwindCompensations(run, compensations, replay);
     if (pending) return {
       runId: run.id,
@@ -7556,6 +7782,7 @@ ${run.workflowVersion}`;
       });
     } catch (err) {
       if (err instanceof ContinueAsNew) {
+        if (this.isFenced(run.id)) return await this.echoCurrentStatus(run.id);
         if (this.cancelRequested.has(run.id)) {
           return await this.compensateAndCancel(run, compensations, replay);
         }
@@ -8007,6 +8234,7 @@ ${run.workflowVersion}`;
   * own `finally` also releases (the run-execution path) is safe.
   */
   async parkBlocked(run, blocked) {
+    if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
     const error2 = {
       message: blocked.reason,
       code: blocked.code,
@@ -8593,10 +8821,7 @@ ${run.workflowVersion}`;
     if (!await this.store.tryLockRun(decision.runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
       return;
     }
-    const renew = setInterval(() => {
-      void this.store.renewRunLock(decision.runId, this.instanceId, this.clock() + this.leaseMs).catch(() => void 0);
-    }, Math.max(50, Math.floor(this.leaseMs / 2)));
-    renew.unref?.();
+    const held = this.holdExecutionLease(decision.runId);
     try {
       const fresh = await this.store.getRun(decision.runId);
       if (!fresh || fresh.awaitingDecisionTaskId !== decision.taskId) return;
@@ -8611,8 +8836,9 @@ ${run.workflowVersion}`;
       fresh.awaitingDecisionTaskId = void 0;
       await this.applyDecision(fresh, decision);
     } finally {
-      clearInterval(renew);
-      await this.store.releaseRunLock(decision.runId);
+      const fenced = held.lease.lost;
+      held.stop();
+      if (!fenced) await this.store.releaseRunLock(decision.runId);
     }
   }
   /** Earliest clock time a suspended turn's deadline is worth LOOKING at again (`runId` → clock ms) —
@@ -8808,6 +9034,9 @@ function childIdsOf(checkpoints) {
 }
 __name(childIdsOf, "childIdsOf");
 
+// src/run-history.ts
+var RUN_HISTORY_FIXTURE_VERSION = 1;
+
 // src/run-facets.ts
 function facetOrigin(origin) {
   return origin?.trim() ? origin : null;
@@ -9361,13 +9590,14 @@ var CodecStateStore = class {
   }
   inner;
   codec;
+  extended;
   /** Atomic claims touch only searchable metadata and bypass payload encoding. */
   tryAdmitSingleton;
   /**
   * Forwarded verbatim — a facet cell counts `status`/`origin`, neither of which the codec touches
-  * (it encodes payloads: input/output/error). Bound as a PROPERTY, and only when the inner store
-  * actually aggregates, so `store.runFacets?.()` still reports absence through the wrapper instead
-  * of resolving to a silently wrong empty result.
+  * (it encodes payloads: input/output, plus events/error under `extended` coverage). Bound as a
+  * PROPERTY, and only when the inner store actually aggregates, so `store.runFacets?.()` still
+  * reports absence through the wrapper instead of resolving to a silently wrong empty result.
   */
   runFacets;
   /**
@@ -9419,9 +9649,10 @@ var CodecStateStore = class {
   updateSchedule;
   deleteSchedule;
   listSchedules;
-  constructor(inner, codec) {
+  constructor(inner, codec, options = {}) {
     this.inner = inner;
     this.codec = codec;
+    this.extended = options.coverage === "extended";
     if (inner.tryAdmitSingleton) this.tryAdmitSingleton = inner.tryAdmitSingleton.bind(inner);
     const deleteRuns = inner.deleteRuns;
     if (deleteRuns) this.deleteRuns = (ids) => deleteRuns.call(inner, ids);
@@ -9479,32 +9710,52 @@ var CodecStateStore = class {
     return v === void 0 ? void 0 : this.codec.decode(v);
   }
   encRun(run) {
-    return {
+    const next = {
       ...run,
       input: this.enc(run.input),
       output: this.enc(run.output)
     };
+    if (this.extended && run.error !== void 0) {
+      next.error = this.enc(run.error);
+    }
+    return next;
   }
   decRun(run) {
-    return run && {
+    if (!run) return run;
+    const next = {
       ...run,
       input: this.dec(run.input),
       output: this.dec(run.output)
     };
+    if (this.extended && run.error !== void 0) {
+      next.error = this.dec(run.error);
+    }
+    return next;
   }
   encCp(cp) {
-    return {
+    const next = {
       ...cp,
       input: this.enc(cp.input),
       output: this.enc(cp.output)
     };
+    if (this.extended) {
+      if (cp.events !== void 0) next.events = this.enc(cp.events);
+      if (cp.error !== void 0) next.error = this.enc(cp.error);
+    }
+    return next;
   }
   decCp(cp) {
-    return cp && {
+    if (!cp) return cp;
+    const next = {
       ...cp,
       input: this.dec(cp.input),
       output: this.dec(cp.output)
     };
+    if (this.extended) {
+      if (cp.events !== void 0) next.events = this.dec(cp.events);
+      if (cp.error !== void 0) next.error = this.dec(cp.error);
+    }
+    return next;
   }
   encSpec(spec) {
     return "input" in spec ? {
@@ -9543,6 +9794,9 @@ var CodecStateStore = class {
     };
     if ("input" in patch) next.input = this.enc(patch.input);
     if ("output" in patch) next.output = this.enc(patch.output);
+    if (this.extended && patch.error !== void 0) {
+      next.error = this.enc(patch.error);
+    }
     return next;
   }
   async getRun(runId) {
@@ -9578,8 +9832,8 @@ var CodecStateStore = class {
   tryLockRun(runId, owner, leaseUntilMs, nowMs) {
     return this.inner.tryLockRun(runId, owner, leaseUntilMs, nowMs);
   }
-  releaseRunLock(runId) {
-    return this.inner.releaseRunLock(runId);
+  releaseRunLock(runId, owner) {
+    return owner === void 0 ? this.inner.releaseRunLock(runId) : this.inner.releaseRunLock(runId, owner);
   }
   renewRunLock(runId, owner, leaseUntilMs) {
     return this.inner.renewRunLock(runId, owner, leaseUntilMs);
@@ -10087,9 +10341,9 @@ var InMemoryStateStore = class {
     run.lockedUntil = leaseUntilMs;
     return true;
   }
-  async releaseRunLock(runId) {
+  async releaseRunLock(runId, owner) {
     const run = this.runs.get(runId);
-    if (run) {
+    if (run && (owner === void 0 || run.lockedBy === owner)) {
       run.lockedBy = void 0;
       run.lockedUntil = void 0;
     }
@@ -10439,6 +10693,7 @@ export {
   ContinueAsNew,
   DEFAULT_CONCURRENCY_STATUSES,
   DEFAULT_PARTITION,
+  DEFAULT_STALLED_AFTER_MS,
   DURABLE_OPTIONS,
   DURABLE_OPTIONS_CANONICAL,
   DURABLE_STEP_CONFIG,
@@ -10462,6 +10717,7 @@ export {
   NestedWorkflowCallError,
   NonDeterminismError,
   QueueController,
+  RUN_HISTORY_FIXTURE_VERSION,
   RUN_VALUE_FACET_LIMIT,
   RUN_VALUE_FACET_SCAN,
   RemoteStepError,
@@ -10535,6 +10791,7 @@ export {
   facetOrigin,
   fanPageBounds,
   fanPageEtag,
+  findStalledRuns,
   foldedCount,
   handoffHandlerId,
   handoffName,