@dudousxd/nestjs-durable-core 0.80.1 → 0.82.0

package/dist/index.js

@@ -928,6 +928,9 @@ async function assertConcurrency(store, workflow, quota) {
 __name(assertConcurrency, "assertConcurrency");
 
 // src/entities.ts
+var NUDGE = {
+  nudge: true
+};
 var ENTITY_WORKFLOW = "__entity";
 var entityRunId = /* @__PURE__ */ __name((name, key) => `entity:${name}:${key}`, "entityRunId");
 var Entities = class {
@@ -935,9 +938,11 @@ var Entities = class {
     __name(this, "Entities");
   }
   host;
+  workflowHost;
   configs = /* @__PURE__ */ new Map();
-  constructor(host) {
+  constructor(host, workflowHost) {
     this.host = host;
+    this.workflowHost = workflowHost;
     host.register(ENTITY_WORKFLOW, "1", (ctx, input) => this.run(ctx, input));
   }
   register(name, config) {
@@ -959,6 +964,52 @@ var Entities = class {
       }
     });
   }
+  /**
+  * The durable half of a send from INSIDE a workflow (`ctx.callEntity` / `ctx.signalEntity`): write
+  * the op to the entity's signal buffer and return once it is stored, so the caller can checkpoint
+  * after it. The op carries `id`, the caller's stable `<runId>:<seq>`, so if the caller dies between
+  * this write and its checkpoint and sends again on replay, the entity skips the second copy. The
+  * entity's run is also ensured here, awaited (`start` only enqueues, it never runs the body
+  * inline), so an op sent on first contact can't be stranded in a buffer no run will ever read if
+  * the process dies right after. The wake-up ({@link kick}) is separate and may be lost: a busy
+  * entity reads the buffer at its next wait, and a parked one is paired with it by the engine's
+  * `reconcileMs` sweep. Returns `false` on a host without a workflow host.
+  */
+  async enqueue(name, key, op, arg, id, reply) {
+    if (!this.workflowHost) return false;
+    const message = {
+      op,
+      arg,
+      id,
+      ...reply ? {
+        reply
+      } : {}
+    };
+    const runId = entityRunId(name, key);
+    await this.workflowHost.bufferSignal(runId, message);
+    await this.workflowHost.ensureStarted(ENTITY_WORKFLOW, {
+      name,
+      key
+    }, runId);
+    return true;
+  }
+  /**
+  * Make the entity act on what {@link enqueue} buffered: wake its run if it is parked on its wait
+  * (the run itself was ensured by `enqueue`; ensured again here, idempotently, for a caller replay
+  * that only re-kicks). A run that is busy needs nothing: its next wait reads the buffer before
+  * suspending. The wake carries a no-op `nudge`, never the op itself, so the op stays in
+  * the buffer in arrival order and a lost or repeated kick costs at most one empty iteration.
+  */
+  async kick(name, key) {
+    const host = this.workflowHost;
+    if (!host) return;
+    const runId = entityRunId(name, key);
+    await host.ensureStarted(ENTITY_WORKFLOW, {
+      name,
+      key
+    }, runId);
+    await host.wakeWaiter(runId, NUDGE);
+  }
   async signal(name, key, op, arg) {
     await this.dispatch(name, key, op, arg);
   }
@@ -972,11 +1023,18 @@ var Entities = class {
     if (!config) throw new FatalError(`entity "${name}" is not registered`);
     const token = entityRunId(name, key);
     let state = config.initialState();
+    const seen = /* @__PURE__ */ new Set();
     for (let i = 0; ; i += 1) {
       const msg = await ctx.waitForSignal(token);
+      if (msg.nudge) continue;
+      if (msg.id !== void 0) {
+        if (seen.has(msg.id)) continue;
+        seen.add(msg.id);
+      }
+      const op = msg.op;
       const out = await ctx.localStep(`op:${i}`, async () => {
-        const handler = config.handlers[msg.op];
-        if (!handler) throw new FatalError(`entity "${name}" has no handler for op "${msg.op}"`);
+        const handler = config.handlers[op];
+        if (!handler) throw new FatalError(`entity "${name}" has no handler for op "${op}"`);
         const result = await handler(state, msg.arg);
         return {
           result,
@@ -986,8 +1044,9 @@ var Entities = class {
       state = out.state;
       await ctx.setEvent("state", state);
       if (msg.reply) {
+        const reply = msg.reply;
         await ctx.localStep(`reply:${i}`, async () => {
-          await this.host.signal(msg.reply, out.result);
+          await this.host.signal(reply, out.result);
         });
       }
     }
@@ -1911,6 +1970,10 @@ function checkpointNameKind(name) {
   return "step";
 }
 __name(checkpointNameKind, "checkpointNameKind");
+function poolNamespace(namespace) {
+  return namespace === "" ? "default" : namespace;
+}
+__name(poolNamespace, "poolNamespace");
 var TERMINAL_RUN_STATUSES = [
   "completed",
   "failed",
@@ -2042,6 +2105,19 @@ var WEBHOOK_PREFIX = "wh:";
 var CHILD_PREFIX = "child:";
 var BREAKPOINT_PREFIX = "bp:";
 var EVENT_PREFIX = EVENT_TOKEN_PREFIX;
+function parseWebhookToken(token) {
+  if (!token.startsWith(WEBHOOK_PREFIX)) return null;
+  const rest = token.slice(WEBHOOK_PREFIX.length);
+  const at = rest.lastIndexOf(":");
+  if (at <= 0) return null;
+  const seqText = rest.slice(at + 1);
+  if (!/^\d+$/.test(seqText)) return null;
+  return {
+    runId: rest.slice(0, at),
+    seq: Number(seqText)
+  };
+}
+__name(parseWebhookToken, "parseWebhookToken");
 function classifyWaiterToken(token) {
   if (token.startsWith(WEBHOOK_PREFIX)) return {
     on: "webhook",
@@ -2579,6 +2655,11 @@ __name(errorMessage, "errorMessage");
 
 // src/singleton-admission.ts
 var SINGLETON_ADMITTED_TAG = "durable:singleton:admitted";
+var SINGLETON_TRACKED_TAG = "durable:singleton:tracked";
+var SINGLETON_RESERVED_TAGS = [
+  SINGLETON_ADMITTED_TAG,
+  SINGLETON_TRACKED_TAG
+];
 var SINGLETON_ACTIVE_STATUSES = [
   "pending",
   "running",
@@ -2586,6 +2667,33 @@ var SINGLETON_ACTIVE_STATUSES = [
   "blocked",
   "cancelling"
 ];
+var LEGACY_SCAN_STATUSES = [
+  "running",
+  "suspended",
+  "cancelling"
+];
+var byAdmissionOrder = /* @__PURE__ */ __name((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id), "byAdmissionOrder");
+function withoutSingletonMarkers(tags) {
+  return tags?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
+}
+__name(withoutSingletonMarkers, "withoutSingletonMarkers");
+function withSingletonTracked(tags) {
+  const current = tags ?? [];
+  if (current.includes(SINGLETON_ADMITTED_TAG) || current.includes(SINGLETON_TRACKED_TAG)) {
+    return current;
+  }
+  return [
+    ...current,
+    SINGLETON_TRACKED_TAG
+  ];
+}
+__name(withSingletonTracked, "withSingletonTracked");
+function legacySingletonHolders(active, runId, limit) {
+  const scanned = active.filter((run) => LEGACY_SCAN_STATUSES.includes(run.status)).sort(byAdmissionOrder);
+  const window = new Set(scanned.slice(0, limit).map((run) => run.id));
+  return scanned.filter((run) => run.id !== runId && !run.tags?.includes(SINGLETON_ADMITTED_TAG) && (!run.tags?.includes(SINGLETON_TRACKED_TAG) || window.has(run.id)));
+}
+__name(legacySingletonHolders, "legacySingletonHolders");
 function singletonAdmissionTags(runs, runId, tag, workflow, limit) {
   if (!Number.isInteger(limit) || limit < 1) throw new Error("singleton limit must be a positive integer");
   const active = runs.filter((run) => run.workflow === workflow && SINGLETON_ACTIVE_STATUSES.includes(run.status) && run.tags?.includes(tag));
@@ -2594,12 +2702,13 @@ function singletonAdmissionTags(runs, runId, tag, workflow, limit) {
   if (candidate.tags?.includes(SINGLETON_ADMITTED_TAG)) return candidate.tags;
   if (candidate.status === "cancelling") return null;
   const holders = active.filter((run) => run.tags?.includes(SINGLETON_ADMITTED_TAG));
-  const remaining = limit - holders.length;
+  const legacy = new Set(legacySingletonHolders(active, runId, limit).map((run) => run.id));
+  const remaining = limit - holders.length - legacy.size;
   if (remaining <= 0) return null;
-  const waiters = active.filter((run) => !run.tags?.includes(SINGLETON_ADMITTED_TAG)).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
+  const waiters = active.filter((run) => !run.tags?.includes(SINGLETON_ADMITTED_TAG) && !legacy.has(run.id)).sort(byAdmissionOrder);
   if (!waiters.slice(0, remaining).some((run) => run.id === runId)) return null;
   return [
-    ...candidate.tags ?? [],
+    ...(candidate.tags ?? []).filter((t) => t !== SINGLETON_TRACKED_TAG),
     SINGLETON_ADMITTED_TAG
   ];
 }
@@ -2609,7 +2718,7 @@ async function clearTerminalSingletonAdmission(runId, patch, getRun) {
   const tags = "tags" in patch ? patch.tags : (await getRun(runId))?.tags;
   return {
     ...patch,
-    tags: tags?.filter((tag) => tag !== SINGLETON_ADMITTED_TAG)
+    tags: withoutSingletonMarkers(tags)
   };
 }
 __name(clearTerminalSingletonAdmission, "clearTerminalSingletonAdmission");
@@ -2620,6 +2729,7 @@ function singletonAdmissionDenialPatch(runs, runId, tag, workflow, retryWakeAt)
   return {
     status: "suspended",
     wakeAt: retryWakeAt,
+    tags: withSingletonTracked(candidate.tags),
     updatedAt: /* @__PURE__ */ new Date()
   };
 }
@@ -2672,7 +2782,7 @@ var SingletonGate = class {
     const admitted = await this.deps.store.tryAdmitSingleton(run.id, this.tag(cfg, run.input), run.workflow, cfg.limit ?? 1, this.retryWakeAt());
     if (admitted && !run.tags?.includes(SINGLETON_ADMITTED_TAG)) {
       run.tags = [
-        ...run.tags ?? [],
+        ...(run.tags ?? []).filter((tag) => tag !== SINGLETON_TRACKED_TAG),
         SINGLETON_ADMITTED_TAG
       ];
     }
@@ -2706,6 +2816,66 @@ var SingletonGate = class {
   }
 };
 
+// src/stalled-runs.ts
+var DEFAULT_STALLED_AFTER_MS = 15 * 6e4;
+async function findStalledRuns(store, options = {}) {
+  const now = options.now ?? Date.now();
+  const thresholdMs = options.thresholdMs ?? DEFAULT_STALLED_AFTER_MS;
+  const scan = Math.max(1, options.scan ?? 1e3);
+  const pageSize = Math.min(200, scan);
+  const stalled = [];
+  for (let offset = 0; offset < scan; offset += pageSize) {
+    const limit = Math.min(pageSize, scan - offset);
+    const page = await store.listRuns({
+      statuses: [
+        "running",
+        "suspended"
+      ],
+      ...options.namespace !== void 0 ? {
+        namespace: options.namespace
+      } : {},
+      limit,
+      offset
+    });
+    for (const run of page) {
+      const ageMs = now - run.updatedAt.getTime();
+      if (ageMs < thresholdMs) continue;
+      const info2 = await stalledInfo(store, run, now, thresholdMs, ageMs);
+      if (info2) stalled.push(info2);
+    }
+    if (page.length < limit) break;
+  }
+  return stalled;
+}
+__name(findStalledRuns, "findStalledRuns");
+async function stalledInfo(store, run, now, thresholdMs, ageMs) {
+  let stalePending = null;
+  for (const cp of await store.listCheckpoints(run.id)) {
+    if (cp.kind !== "remote" || cp.status !== "pending") continue;
+    const stepAgeMs = now - cp.enqueuedAt.getTime();
+    if (stepAgeMs < thresholdMs) continue;
+    if (cp.wakeAt != null && cp.wakeAt > now) continue;
+    if (!stalePending || stepAgeMs > stalePending.ageMs) {
+      stalePending = {
+        seq: cp.seq,
+        name: cp.name,
+        attempts: cp.attempts,
+        ageMs: stepAgeMs,
+        leaseUntil: cp.wakeAt ?? null
+      };
+    }
+  }
+  const wakeForever = run.status === "suspended" && run.wakeAt == null;
+  if (!wakeForever && !stalePending) return null;
+  return {
+    run,
+    ageMs,
+    wakeForever,
+    stalePending
+  };
+}
+__name(stalledInfo, "stalledInfo");
+
 // src/transport-pool.ts
 var TransportPool = class {
   static {
@@ -3567,30 +3737,82 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
     }));
     return id;
   }, "startChild");
+  const sendEntityOp = /* @__PURE__ */ __name(async (name, key, op, arg, seq, reply) => await host.enqueueEntityOp?.(name, key, op, arg, `${runId}:${seq}`, reply) ?? false, "sendEntityOp");
   const callEntity = /* @__PURE__ */ __name(async (name, key, op, arg) => {
     const current = pos.next();
-    const existing = await readCheckpoint(current);
+    let existing = await readCheckpoint(current);
     if (existing && existing.status === "completed") return existing.output;
     const reply = `entityreply:${runId}:${current}`;
+    const intentName = `signal:${reply}`;
+    if (existing && existing.name !== intentName) {
+      throw new NonDeterminismError(runId, current, intentName, existing.name);
+    }
+    if (existing && replay?.has(current)) {
+      existing = await store.getCheckpoint(runId, current) ?? existing;
+      if (existing.status === "completed") return existing.output;
+    }
+    const markSent = /* @__PURE__ */ __name(() => store.saveCheckpointIfPending?.(instantCheckpoint({
+      runId,
+      seq: current,
+      name: intentName,
+      kind: "signal",
+      status: "pending",
+      output: {
+        sent: true
+      }
+    })) ?? Promise.resolve(false), "markSent");
+    if (existing) {
+      const buffered = await consumeBuffered(reply, current);
+      if (buffered) {
+        await store.removeSignalWaiter({
+          token: reply,
+          runId,
+          seq: current
+        });
+        return buffered.value;
+      }
+      if (existing.output?.sent !== true) {
+        if (await sendEntityOp(name, key, op, arg, current, reply)) await markSent();
+      }
+      host.kickEntity?.(name, key);
+      throw new WorkflowSuspended();
+    }
     await store.putSignalWaiter({
       token: reply,
       runId,
       seq: current
     });
-    host.signalEntity?.(name, key, op, arg, reply);
+    if (!host.enqueueEntityOp) {
+      host.signalEntity?.(name, key, op, arg, reply);
+      throw new WorkflowSuspended();
+    }
+    await writeCheckpoint(instantCheckpoint({
+      runId,
+      seq: current,
+      name: intentName,
+      kind: "signal",
+      status: "pending",
+      output: {
+        sent: false
+      }
+    }));
+    if (await sendEntityOp(name, key, op, arg, current, reply)) await markSent();
+    host.kickEntity?.(name, key);
     throw new WorkflowSuspended();
   }, "callEntity");
   const signalEntity = /* @__PURE__ */ __name(async (name, key, op, arg) => {
     const current = pos.next();
     const existing = await readCheckpoint(current);
     if (existing && existing.status === "completed") return;
-    host.signalEntity?.(name, key, op, arg);
+    const durable = await sendEntityOp(name, key, op, arg, current);
+    if (!durable) host.signalEntity?.(name, key, op, arg);
     await writeCheckpoint(instantCheckpoint({
       runId,
       seq: current,
       name: `entitysig:${name}:${key}`,
       kind: "local"
     }));
+    if (durable) host.kickEntity?.(name, key);
   }, "signalEntity");
   const breakpoint = /* @__PURE__ */ __name(async (label) => {
     const current = pos.next();
@@ -3658,17 +3880,15 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
       const deadlineSeq = timeoutMs == null ? -1 : pos.next();
       const existing = await readCheckpoint(current);
       if (existing && existing.status === "completed") return existing.output;
-      if (timeoutMs == null) {
-        await store.putSignalWaiter({
+      const buffered = await consumeBuffered(token, current);
+      if (buffered) return buffered.value;
+      const deadline = timeoutMs == null ? void 0 : await stampDeadline(deadlineSeq, token, timeoutMs);
+      if (deadline !== void 0 && host.clock() >= deadline) {
+        await store.removeSignalWaiter({
           token,
           runId,
           seq: current
-        });
-        throw new WorkflowSuspended();
-      }
-      const deadline = await stampDeadline(deadlineSeq, token, timeoutMs);
-      if (host.clock() >= deadline) {
-        await store.takeSignalWaiter(token).catch(() => void 0);
+        }).catch(() => void 0);
         throw new SignalTimeoutError(token, timeoutMs);
       }
       await store.putSignalWaiter({
@@ -3676,6 +3896,15 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
         runId,
         seq: current
       });
+      const lateBuffered = await consumeBuffered(token, current);
+      if (lateBuffered) {
+        await store.removeSignalWaiter({
+          token,
+          runId,
+          seq: current
+        });
+        return lateBuffered.value;
+      }
       throw new WorkflowSuspended(deadline);
     }, "wait");
     return {
@@ -3867,6 +4096,12 @@ var SETTLE_ELIGIBLE_STATUSES = [
   "failed",
   "cancelling"
 ];
+var TIMEOUT_SWEPT_STATUSES = [
+  "pending",
+  "running",
+  "suspended",
+  "blocked"
+];
 var SUSPEND_ELIGIBLE_STATUSES = SETTLE_ELIGIBLE_STATUSES.filter((status) => status !== "cancelling");
 var RECOVERY_BATCH = 500;
 var NamespaceMismatch = class NamespaceMismatch2 extends Error {
@@ -3889,6 +4124,12 @@ var WorkflowEngine = class {
   instanceId;
   namespace;
   leaseMs;
+  /**
+  * Leases held by this `instanceId` that expire strictly before this epoch ms were taken by a
+  * PREVIOUS process with the same id (every lease this one takes or renews expires at
+  * `clock() + leaseMs`, so at or after it). See {@link orphanedRuns}.
+  */
+  ownLeaseCutoffMs;
   maxRecoveryAttempts;
   remoteAdvanceSilenceMs;
   reconcileMs;
@@ -3992,6 +4233,18 @@ var WorkflowEngine = class {
   stepQueue = /* @__PURE__ */ new Map();
   /** Executions currently in flight, so a graceful shutdown can wait for them to settle. */
   inflight = /* @__PURE__ */ new Set();
+  /**
+  * The run lease of each execution in flight on THIS instance, by run id — renewed while it runs
+  * and checked by every settle (the zombie fence, see {@link ExecutionLease}). Also what `drain()`
+  * releases when its budget runs out.
+  */
+  executionLeases = /* @__PURE__ */ new Map();
+  /** Pager hooks for stranded runs (see {@link onStalled}). */
+  stalledListeners = /* @__PURE__ */ new Set();
+  /** Runs already paged about, by id → the `updatedAt` of the paged episode (see {@link sweepStalled}). */
+  notifiedStalled = /* @__PURE__ */ new Map();
+  lastStalledSweepAt = Number.NEGATIVE_INFINITY;
+  stalledAfterMs;
   draining = false;
   /** Wait already spent on this shutdown before `drain()` (see `beginDrain`). */
   drainSpentMs = 0;
@@ -4010,11 +4263,14 @@ var WorkflowEngine = class {
     this.admission = deps.admission ?? new InMemoryAdmissionBackend(this.clock);
     this.admission.onFreed?.((queue) => this.wakeQueueWaiters(queue));
     this.instanceId = deps.instanceId ?? globalThis.crypto.randomUUID();
-    this.namespace = deps.namespace;
+    this.namespace = poolNamespace(deps.namespace);
     this.leaseMs = deps.leaseMs ?? 3e4;
+    this.ownLeaseCutoffMs = this.clock() + this.leaseMs;
     this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
     this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
     this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
+    const stalledAfter = deps.stalledAfterMs;
+    this.stalledAfterMs = stalledAfter !== void 0 && Number.isFinite(stalledAfter) ? Math.max(6e4, stalledAfter) : DEFAULT_STALLED_AFTER_MS;
     const batch = deps.timerBatchSize;
     this.timerBatchSize = batch !== void 0 && Number.isFinite(batch) && batch >= 1 ? Math.floor(batch) : 500;
     this.eventBufferTtlMs = deps.eventBufferTtlMs || void 0;
@@ -4131,7 +4387,11 @@ var WorkflowEngine = class {
       }
     });
     this.accumulators = new EventAccumulators(this);
-    this.entities = new Entities(this);
+    this.entities = new Entities(this, {
+      bufferSignal: /* @__PURE__ */ __name((token, payload) => this.store.bufferSignal(token, payload), "bufferSignal"),
+      wakeWaiter: /* @__PURE__ */ __name((token, payload) => this.wakeSignalWaiter(token, payload), "wakeWaiter"),
+      ensureStarted: /* @__PURE__ */ __name((workflow, input, runId) => this.start(workflow, input, runId), "ensureStarted")
+    });
   }
   /**
   * Register a **durable entity** (a virtual object): a keyed actor whose `handlers` run **serialized
@@ -4354,6 +4614,64 @@ var WorkflowEngine = class {
     this.deadListeners.add(listener);
     return () => this.deadListeners.delete(listener);
   }
+  /**
+  * Register a pager hook for STRANDED runs: an in-flight run untouched past `stalledAfterMs` that is
+  * either suspended with NO wake timer (wake-forever: nothing will ever re-drive it) or parked on an
+  * old pending remote step with no live lease (dispatched and, most likely, lost). The timer poller
+  * sweeps for these ({@link sweepStalled}, throttled to one pass a minute) and notifies each episode
+  * ONCE per instance: a run is paged again only after it makes progress and strands again. Wire your
+  * alerting here like {@link onDead}. A listener that throws never breaks the sweep. Returns an
+  * unsubscribe function.
+  */
+  onStalled(listener) {
+    this.stalledListeners.add(listener);
+    return () => this.stalledListeners.delete(listener);
+  }
+  /**
+  * Detect stranded in-flight runs and notify the {@link onStalled} listeners. Self-throttled to one
+  * pass a minute, skipped while draining, and a no-op without listeners — call it every tick (the
+  * NestJS timer poller does). Scoped to this engine's `namespace` (an operator engine with no
+  * namespace sweeps every namespace). Returns the runs notified THIS pass.
+  */
+  async sweepStalled(now = this.clock()) {
+    if (this.stalledListeners.size === 0 || this.draining) return [];
+    if (now - this.lastStalledSweepAt < 6e4) return [];
+    this.lastStalledSweepAt = now;
+    const notified = [];
+    for (const info2 of await this.listStalledRuns({
+      now
+    })) {
+      const episode = info2.run.updatedAt.getTime();
+      if (this.notifiedStalled.get(info2.run.id) === episode) continue;
+      this.notifiedStalled.set(info2.run.id, episode);
+      if (this.notifiedStalled.size > 1e3) this.notifiedStalled.clear();
+      for (const listener of this.stalledListeners) {
+        try {
+          listener(info2);
+        } catch {
+        }
+      }
+      notified.push(info2);
+    }
+    return notified;
+  }
+  /**
+  * The stranded runs right now, without notifying anyone: the query behind {@link sweepStalled} and
+  * the CLI's `durable inspect --stale`. A `running`/`suspended` run qualifies when it has been
+  * untouched for `thresholdMs` (default `stalledAfterMs`) AND either it is suspended with no wake
+  * timer, or its oldest pending remote step is older than the threshold with no lease in the future
+  * (a step with a `timeoutMs`, a heartbeat or `remoteRedispatchMs` keeps a lease; a long
+  * `ctx.sleep` has a future `wakeAt`; neither is stranded). Scans up to `scan` candidates (default
+  * 1000), newest first, in this engine's namespace.
+  */
+  async listStalledRuns(options = {}) {
+    return findStalledRuns(this.store, {
+      now: options.now ?? this.clock(),
+      thresholdMs: options.thresholdMs ?? this.stalledAfterMs,
+      scan: options.scan,
+      namespace: this.namespace
+    });
+  }
   notifyDead(run) {
     for (const fn of this.deadListeners) {
       try {
@@ -4407,7 +4725,7 @@ var WorkflowEngine = class {
         // tenant-suffixed group the run will dispatch to — an operator starting a tenant's run via the
         // `onStartRun` wire (opts.namespace = tenant) must find the live `<workflow>@<tenant>` group,
         // not the bare group. Without this the start-run entry path can't reach a suffixed-only worker.
-        namespace: opts?.namespace ?? this.namespace,
+        namespace: poolNamespace(opts?.namespace) ?? this.namespace,
         input,
         createdAt: /* @__PURE__ */ new Date(),
         updatedAt: /* @__PURE__ */ new Date()
@@ -4444,7 +4762,7 @@ var WorkflowEngine = class {
         VERSION_UNDECLARED_TAG
       ] : []
     ];
-    const tags = mergeTags(registered.tags, stampedTags)?.filter((tag) => tag !== SINGLETON_ADMITTED_TAG);
+    const tags = mergeTags(registered.tags, stampedTags)?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
     if (registered.singleton) {
       await this.singletons.assertCapacity(name, registered.singleton, input);
     }
@@ -4458,7 +4776,7 @@ var WorkflowEngine = class {
       ...delayed ? {
         wakeAt: startAtMs
       } : {},
-      namespace: opts?.namespace ?? this.namespace,
+      namespace: poolNamespace(opts?.namespace) ?? this.namespace,
       // DERIVED from the registration, exactly like `namespace` is derived from this engine — there is
       // deliberately no `opts.origin`, so a caller cannot label its run as another package's work.
       // `undefined` when the registration carried none (remote/convention/synthesized child, or a
@@ -4512,6 +4830,11 @@ var WorkflowEngine = class {
   getRun(runId) {
     return this.store.getRun(runId);
   }
+  /** A run's full checkpoint timeline, ordered by seq. A thin pass-through to the store (it is what
+  *  the testing kit's `captureHistory` reads, next to {@link getRun}). */
+  listCheckpoints(runId) {
+    return this.store.listCheckpoints(runId);
+  }
   /**
   * Re-execute a run's body (replaying its checkpoints). This is the EXPLICIT entry point: besides
   * driving a live run, it re-drives a `failed` run — the manual retry (prefer {@link requeue}, which
@@ -4597,6 +4920,8 @@ var WorkflowEngine = class {
   * diagnosis.
   */
   async releaseLeaseBestEffort(runId) {
+    if (this.isFenced(runId)) return;
+    this.handOffLease(runId);
     try {
       await this.store.releaseRunLock(runId);
     } catch (err) {
@@ -4640,12 +4965,12 @@ var WorkflowEngine = class {
         error: run.error
       };
     }
-    if (this.namespace !== void 0 && run.namespace !== void 0 && run.namespace !== this.namespace) {
+    if (this.namespace !== void 0 && run.namespace !== void 0 && poolNamespace(run.namespace) !== this.namespace) {
       await this.store.releaseRunLock(runId);
       throw new NamespaceMismatch();
     }
-    if (run.status === "failed" && run.tags?.includes(SINGLETON_ADMITTED_TAG)) {
-      run.tags = run.tags.filter((tag) => tag !== SINGLETON_ADMITTED_TAG);
+    if (run.status === "failed" && (run.tags?.includes(SINGLETON_ADMITTED_TAG) || run.tags?.includes(SINGLETON_TRACKED_TAG))) {
+      run.tags = withoutSingletonMarkers(run.tags);
       await this.store.updateRun(run.id, {
         tags: run.tags
       });
@@ -4846,17 +5171,47 @@ var WorkflowEngine = class {
       rearm();
     });
     this.drainRearms.add(rearm);
+    let settled = false;
     try {
       await Promise.race([
         Promise.allSettled([
           ...this.inflight
-        ]),
+        ]).then(() => {
+          settled = true;
+        }),
         timer
       ]);
     } finally {
       this.drainRearms.delete(rearm);
       clearTimeout(timeout);
     }
+    if (!settled) await this.releaseInflightLeases();
+  }
+  /**
+  * The drain budget ran out with executions still in flight: release their recovery leases so
+  * another instance re-drives those runs on its next recovery pass, instead of waiting out
+  * `leaseMs` after this process exits. Each still-running turn is FENCED first, so whatever it
+  * computes after this point is never written over the new owner's work (see settleRun). Its
+  * completed steps are checkpointed, so the new owner replays them; a step still executing here
+  * may run again there, exactly as after a crash. The release is owner-scoped where the store
+  * supports it (only a lease this instance still holds is cleared). Best effort: never throws.
+  */
+  async releaseInflightLeases() {
+    const released = [];
+    for (const [runId, lease] of [
+      ...this.executionLeases
+    ]) {
+      if (lease.lost || lease.handedOff || lease.done) continue;
+      lease.lost = true;
+      try {
+        await this.store.releaseRunLock(runId, this.instanceId);
+        released.push(runId);
+      } catch {
+      }
+    }
+    if (released.length > 0) {
+      console.warn(`[nestjs-durable] drain timed out with ${released.length} run(s) still executing; released their leases so another instance recovers them now: ${released.join(", ")}`);
+    }
   }
   /**
   * Cancel in-flight runs that have outlived their workflow's `executionTimeout`. Call it from the
@@ -4864,6 +5219,15 @@ var WorkflowEngine = class {
   * `execution_timeout` error (terminal, so a late step result can't resurrect it), AND its children
   * are cascaded exactly as an explicit {@link cancel} cascades them.
   *
+  * WHICH RUNS. Every run not yet finished, counted from `createdAt`: `pending` and `blocked` as well
+  * as `running` and `suspended`. A run nobody picks up (no worker on its queue) or nobody can run (no
+  * worker with its capabilities) is as stuck as one that hangs mid-body, and the cap is the backstop
+  * for exactly that — leaving them out meant a run that never started lived forever while one that
+  * started and hung was cut off. Cancelling one a worker is picking up at that moment is safe: the
+  * worker's settle only writes over a live status ({@link SETTLE_ELIGIBLE_STATUSES}), so it can't
+  * bring the run back. `cancelling` is left alone on purpose — a compensating cancel's saga undo is
+  * running, it ends `cancelled` by itself, and cutting it short would skip the undo.
+  *
   * WHY THE TERMINAL WRITE IS DIRECT, not a call to {@link cancel}. Two reasons, both still standing:
   * the run must land with the `execution_timeout` error `cancel` knows nothing about (`cancel` writes
   * a plain `cancelled`, and the code is what a dashboard/alert distinguishes a timeout by), and this
@@ -4876,8 +5240,8 @@ var WorkflowEngine = class {
   * children kept running with the parent gone and nothing pointing at them — a silent orphan you
   * could only find by reading the runs table by hand. {@link cancelChildren} is the SAME recursive
   * walk `cancel` uses, so children of children go too, and it inherits that path's two guards:
-  * `cancel` returns early on a run that is already `completed`/`cancelled`/`dead` (a child that beat
-  * the sweep to the finish is never clobbered, and a re-cancel is a no-op), and it writes the
+  * `cancel` returns early on a run that is already terminal (a child that beat the sweep to the
+  * finish is never clobbered, and a re-cancel is a no-op), and it writes the
   * parent's terminal status BEFORE recursing, so a cyclic parent→child graph terminates on the
   * second visit instead of looping. Two pollers sweeping the same run concurrently therefore write
   * the same terminal state twice and cascade into already-cancelled children — idempotent, not
@@ -4893,18 +5257,11 @@ var WorkflowEngine = class {
       if (this.draining) return;
       if (reg.executionTimeoutMs == null) continue;
       const deadline = now - reg.executionTimeoutMs;
-      const inflight = [
-        ...await this.store.listRuns({
-          workflow: reg.name,
-          status: "running",
-          namespace: this.namespace
-        }),
-        ...await this.store.listRuns({
-          workflow: reg.name,
-          status: "suspended",
-          namespace: this.namespace
-        })
-      ];
+      const inflight = await this.store.listRuns({
+        workflow: reg.name,
+        statuses: TIMEOUT_SWEPT_STATUSES,
+        namespace: this.namespace
+      });
       for (const run of inflight) {
         if (this.draining) return;
         if (run.createdAt.getTime() > deadline) continue;
@@ -4937,48 +5294,103 @@ var WorkflowEngine = class {
     const results = [];
     for (const listed of await this.orphanedRuns(nowMs)) {
       if (this.draining) break;
-      const acquired = await this.store.tryLockRun(listed.id, this.instanceId, nowMs + this.leaseMs, nowMs);
-      if (!acquired) continue;
-      const run = await this.store.getRun(listed.id);
-      if (!run || run.status !== "running" && run.status !== "cancelling") {
-        await this.store.releaseRunLock(listed.id);
-        continue;
-      }
-      if (run.status === "running") {
-        const dispatched = await this.inFlightRemoteStep(run.id);
-        if (dispatched) {
-          await this.store.updateRun(run.id, {
-            status: "suspended",
-            wakeAt: this.reconcileWakeAt(dispatched.wakeAt ?? void 0),
-            updatedAt: /* @__PURE__ */ new Date()
-          });
-          await this.store.releaseRunLock(run.id);
-          results.push({
-            runId: run.id,
-            status: "suspended"
-          });
-          continue;
-        }
-      }
-      const settled = await this.countRecovery(run);
-      if (settled) {
-        results.push(settled);
-        continue;
-      }
-      await this.store.releaseRunLock(run.id);
-      if (run.status !== "cancelling") {
+      const outcome = this.isOwnLiveLease(listed, nowMs) ? await this.reclaimOwnLease(listed, nowMs) : await this.recoverOrphan(listed, nowMs);
+      if (!outcome) continue;
+      if (outcome.dispatch) await this.runDispatcher.dispatch(outcome.result.runId);
+      results.push(outcome.result);
+    }
+    return results;
+  }
+  /** A run whose still-live lease is held by this very `instanceId` (see {@link reclaimOwnLease}). */
+  isOwnLiveLease(run, nowMs) {
+    return run.lockedBy === this.instanceId && (run.lockedUntil ?? 0) > nowMs;
+  }
+  /**
+  * Recover a run whose lease is still live but held by THIS `instanceId` and predates this process
+  * (listed by {@link orphanedRuns}): the previous process with this stable id died without a
+  * graceful shutdown (OOM, SIGKILL), so nothing is renewing that lease and waiting for it to expire
+  * only delays the run. The lease is taken back with `renewRunLock` (a compare-and-set on the
+  * owner; `tryLockRun` refuses a live lease), so the lease alone cannot fence a live execution in
+  * THIS process from its own recovery: the in-process run gate does. A run with a
+  * gate is executing (or about to) here and is skipped; otherwise a gate is held while the run is
+  * recovered, so a resume landing meanwhile queues behind the recovery instead of running beside it.
+  * The gate is released before the re-dispatch, which may resume the run on this instance.
+  */
+  async reclaimOwnLease(listed, nowMs) {
+    if (this.runGates.has(listed.id)) return void 0;
+    let release;
+    const tail = new Promise((resolve) => {
+      release = resolve;
+    });
+    const gate = {
+      tail
+    };
+    this.runGates.set(listed.id, gate);
+    this.closeGateAfter(listed.id, gate, tail);
+    let outcome;
+    try {
+      const fresh = await this.store.getRun(listed.id);
+      const liveHere = fresh?.lockedBy === this.instanceId && (fresh.lockedUntil ?? 0) > nowMs && (fresh.lockedUntil ?? 0) >= this.ownLeaseCutoffMs;
+      if (fresh && !liveHere) outcome = await this.recoverOrphan(listed, nowMs, true);
+    } finally {
+      release(outcome?.result ?? {
+        runId: listed.id,
+        status: listed.status
+      });
+    }
+    return outcome;
+  }
+  /**
+  * One run of a {@link recoverIncomplete} pass: take its lease and decide on it. Resolves the
+  * pass's result for it and whether it must be re-dispatched, or `undefined` when it was not ours.
+  * `ownLease`: the run's live lease is this instance id's own, left by a previous process (see
+  * {@link reclaimOwnLease}), so it is taken back by owner rather than acquired as a free one.
+  */
+  async recoverOrphan(listed, nowMs, ownLease = false) {
+    const acquired = ownLease ? await this.store.renewRunLock(listed.id, this.instanceId, nowMs + this.leaseMs) : await this.store.tryLockRun(listed.id, this.instanceId, nowMs + this.leaseMs, nowMs);
+    if (!acquired) return void 0;
+    const run = await this.store.getRun(listed.id);
+    if (!run || run.status !== "running" && run.status !== "cancelling") {
+      await this.store.releaseRunLock(listed.id);
+      return void 0;
+    }
+    if (run.status === "running") {
+      const dispatched = await this.inFlightRemoteStep(run.id);
+      if (dispatched) {
         await this.store.updateRun(run.id, {
-          status: "pending",
+          status: "suspended",
+          wakeAt: this.reconcileWakeAt(dispatched.wakeAt ?? void 0),
           updatedAt: /* @__PURE__ */ new Date()
         });
+        await this.store.releaseRunLock(run.id);
+        return {
+          result: {
+            runId: run.id,
+            status: "suspended"
+          },
+          dispatch: false
+        };
       }
-      await this.runDispatcher.dispatch(run.id);
-      results.push({
-        runId: run.id,
-        status: run.status === "cancelling" ? "cancelling" : "pending"
+    }
+    const settled = await this.countRecovery(run);
+    if (settled) return {
+      result: settled,
+      dispatch: false
+    };
+    await this.store.releaseRunLock(run.id);
+    if (run.status !== "cancelling") {
+      await this.store.updateRun(run.id, {
+        status: "pending",
+        updatedAt: /* @__PURE__ */ new Date()
       });
     }
-    return results;
+    return {
+      result: {
+        runId: run.id,
+        status: run.status === "cancelling" ? "cancelling" : "pending"
+      },
+      dispatch: true
+    };
   }
   /**
   * The runs {@link recoverIncomplete} should try to reclaim: incomplete runs whose lease is free or
@@ -4988,12 +5400,20 @@ var WorkflowEngine = class {
   * ({@link StateStore.listOrphanedRuns}, bounded by {@link RECOVERY_BATCH}); else the full list is
   * filtered here, which still spares the probes. Shuffled, like the due-timer pass, so instances
   * recovering at once spread over the batch instead of contending for the same leases.
+  *
+  * Also listed: runs still leased by THIS `instanceId` with a lease taken before this process
+  * booted ({@link ownLeaseCutoffMs}). With a stable `instanceId` (pinned to the pod identity), that
+  * is the previous process, killed without a graceful shutdown; see {@link reclaimOwnLease}.
   */
   async orphanedRuns(nowMs) {
+    const ownedBy = {
+      owner: this.instanceId,
+      lockedBefore: this.ownLeaseCutoffMs
+    };
     if (this.store.listOrphanedRuns) {
-      return shuffle(await this.store.listOrphanedRuns(nowMs, RECOVERY_BATCH, this.namespace));
+      return shuffle(await this.store.listOrphanedRuns(nowMs, RECOVERY_BATCH, this.namespace, ownedBy));
     }
-    return shuffle((await this.store.listIncompleteRuns(this.namespace)).filter((run) => run.lockedUntil == null || run.lockedUntil <= nowMs).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()).slice(0, RECOVERY_BATCH));
+    return shuffle((await this.store.listIncompleteRuns(this.namespace)).filter((run) => run.lockedUntil == null || run.lockedUntil <= nowMs || run.lockedBy === ownedBy.owner && run.lockedUntil < ownedBy.lockedBefore).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()).slice(0, RECOVERY_BATCH));
   }
   /**
   * The remote step this run dispatched and is still awaiting a result for, if any — the signature
@@ -5036,6 +5456,7 @@ var WorkflowEngine = class {
         error: error2,
         recoveryAttempts: attempts
       });
+      this.notifyParent(run.id, failedCompletion(error2));
       return {
         runId: run.id,
         status: "dead",
@@ -5242,7 +5663,7 @@ var WorkflowEngine = class {
       error: void 0,
       ...resurrecting ? {
         recoveryAttempts: 0,
-        tags: run.tags?.filter((tag) => tag !== SINGLETON_ADMITTED_TAG)
+        tags: withoutSingletonMarkers(run.tags)
       } : {},
       updatedAt: /* @__PURE__ */ new Date()
     });
@@ -5654,6 +6075,16 @@ var WorkflowEngine = class {
     return this.deliverSignal(lateWaiter, reclaimed.payload);
   }
   /**
+  * Wake the run parked on `token` with `payload`, if there is one; otherwise do nothing — unlike
+  * {@link signal}, it never buffers. For a wake-up whose real payload is ALREADY buffered on the
+  * token (an entity op, see `Entities.kick`): a run that isn't parked reads that buffer at its next
+  * wait, so there is nothing to keep.
+  */
+  async wakeSignalWaiter(token, payload) {
+    const waiter = await this.store.takeSignalWaiter(token);
+    return waiter ? this.deliverSignal(waiter, payload) : null;
+  }
+  /**
   * Ensure a run exists for `runId`, then deliver a signal to it — atomically race-free thanks to
   * signal buffering: if the run is new (or busy / not yet waiting), the signal is buffered and
   * consumed when it reaches `waitForSignal(token)`. The canonical **durable-entity / accumulator**
@@ -6249,9 +6680,31 @@ var WorkflowEngine = class {
     });
   }
   /** Admission persists the wait atomically; never overwrite a cancellation after it returns. */
+  /**
+  * A run's first execution: `pending` -> `running`. A singleton run not yet admitted is stamped
+  * {@link SINGLETON_TRACKED_TAG} in the same write, BEFORE it competes for a slot, so a run this
+  * replica is about to admit or deny is never mistaken for a holder admitted by a replica older
+  * than 0.73 (see `legacySingletonHolders`).
+  */
+  async markStarted(run, singleton) {
+    const tags = singleton ? withSingletonTracked(run.tags) : run.tags;
+    const retag = tags !== run.tags;
+    await this.store.updateRun(run.id, {
+      status: "running",
+      ...retag ? {
+        tags
+      } : {},
+      updatedAt: /* @__PURE__ */ new Date()
+    });
+    run.status = "running";
+    if (retag) run.tags = tags;
+  }
   async singletonWaitResult(run) {
     const current = await this.store.getRun(run.id);
-    await this.store.releaseRunLock(run.id);
+    if (!this.isFenced(run.id)) {
+      this.handOffLease(run.id);
+      await this.store.releaseRunLock(run.id);
+    }
     if (!current) throw new Error(`run ${run.id} was deleted during singleton admission`);
     if (current.status === "suspended") {
       this.emit({
@@ -6279,10 +6732,7 @@ var WorkflowEngine = class {
         };
       }
     }
-    const renew = setInterval(() => {
-      void this.store.renewRunLock(run.id, this.instanceId, this.clock() + this.leaseMs).catch(() => void 0);
-    }, Math.max(50, Math.floor(this.leaseMs / 2)));
-    renew.unref?.();
+    const held = this.holdExecutionLease(run.id);
     try {
       const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
       if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
@@ -6290,10 +6740,59 @@ var WorkflowEngine = class {
       }
       return result;
     } finally {
-      clearInterval(renew);
+      held.stop();
     }
   }
   /**
+  * Renew `runId`'s lease for as long as this execution holds it, and register it in
+  * {@link executionLeases} so settles can check the zombie fence and `drain()` can release it.
+  *
+  * A renewal that returns `false` means the lease is gone: this turn stalled past `leaseMs` and
+  * another instance reclaimed the run, so this executor is now a zombie. It is FENCED (`lost`): its
+  * settles degrade to an echo of the store's state instead of clobbering the new owner's writes, and
+  * its own lease release is skipped (it would free the new owner's lease). A renewal that THROWS (a
+  * transient store error) is not evidence of a takeover, so it stays swallowed and the next
+  * interval retries. A lease this execution released on purpose (`handedOff`) is not renewed, and a
+  * renewal resolving after the execution ended (`done`) fences nothing.
+  */
+  holdExecutionLease(runId) {
+    const lease = {
+      lost: false,
+      handedOff: false,
+      done: false
+    };
+    this.executionLeases.set(runId, lease);
+    const renew = setInterval(() => {
+      if (lease.handedOff || lease.lost || lease.done) return;
+      void this.store.renewRunLock(runId, this.instanceId, this.clock() + this.leaseMs).then((renewed) => {
+        if (renewed || lease.handedOff || lease.lost || lease.done) return;
+        lease.lost = true;
+        console.warn(`[nestjs-durable] lost the lease of run ${runId} while executing it (another instance took it over after it expired); this instance will not write the run's outcome`);
+      }).catch(() => void 0);
+    }, Math.max(50, Math.floor(this.leaseMs / 2)));
+    renew.unref?.();
+    return {
+      lease,
+      stop: /* @__PURE__ */ __name(() => {
+        clearInterval(renew);
+        lease.done = true;
+        if (this.executionLeases.get(runId) === lease) this.executionLeases.delete(runId);
+      }, "stop")
+    };
+  }
+  /** Whether `runId`'s execution on this instance lost its lease (see {@link holdExecutionLease}). */
+  isFenced(runId) {
+    return this.executionLeases.get(runId)?.lost === true;
+  }
+  /**
+  * Mark `runId`'s lease as given away on purpose by its own execution (released mid-turn), so a
+  * renewal that now fails is not read as a takeover.
+  */
+  handOffLease(runId) {
+    const lease = this.executionLeases.get(runId);
+    if (lease) lease.handedOff = true;
+  }
+  /**
   * Advance a remote (cross-SDK) workflow one turn: hand its history to the executor (which dispatches
   * a workflow task to the worker and awaits its replay) and apply the decision. Mirrors
   * {@link runExecution}'s settle/suspend; the lease is held by {@link execute}. The result that lands
@@ -6510,6 +7009,7 @@ ${run.workflowVersion}`;
   * keeps its own lease handling (the TS executor's `finally`, the remote path's caller).
   */
   async settleRun(run, outcome) {
+    if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
     const updatedAt = /* @__PURE__ */ new Date();
     if (outcome.kind === "completed") {
       const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, {
@@ -6590,6 +7090,7 @@ ${run.workflowVersion}`;
   }
   async runRemoteExecution(run, registered) {
     if (run.status === "cancelling") {
+      if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
       const error2 = {
         message: "cancelled"
       };
@@ -6598,6 +7099,7 @@ ${run.workflowVersion}`;
         error: error2,
         updatedAt: /* @__PURE__ */ new Date()
       });
+      if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
       this.emit({
         type: "run.failed",
         runId: run.id,
@@ -6622,11 +7124,7 @@ ${run.workflowVersion}`;
     }
     const remote = registered.remote;
     if (run.status === "pending") {
-      await this.store.updateRun(run.id, {
-        status: "running",
-        updatedAt: /* @__PURE__ */ new Date()
-      });
-      run.status = "running";
+      await this.markStarted(run, registered?.singleton !== void 0);
       this.emit({
         type: "run.started",
         runId: run.id,
@@ -6648,6 +7146,7 @@ ${run.workflowVersion}`;
     }
     const history = await this.remoteHistory(run.id, checkpoints);
     if (remote.executor.dispatch) {
+      if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
       const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID().slice(0, 8)}`;
       const silenceMs = this.remoteAdvanceSilenceMs;
       const wakeAt = silenceMs != null ? this.clock() + silenceMs : void 0;
@@ -6663,6 +7162,7 @@ ${run.workflowVersion}`;
         workflow: run.workflow,
         namespace: run.namespace
       });
+      this.handOffLease(run.id);
       await this.store.releaseRunLock(run.id);
       await remote.executor.dispatch(run, history, taskId);
       return {
@@ -6691,6 +7191,7 @@ ${run.workflowVersion}`;
         message: err instanceof Error ? err.message : String(err),
         stack: err instanceof Error ? err.stack : void 0
       };
+      if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
       await this.store.updateRun(run.id, {
         status: "failed",
         error: error2,
@@ -6719,6 +7220,7 @@ ${run.workflowVersion}`;
   * — the caller has already ensured this is the decision for the currently-awaited turn.
   */
   async applyDecision(run, decision) {
+    if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
     if (decision.status === "completed") {
       await this.applyCommands(run, decision.commands);
       return this.settleRun(run, {
@@ -7188,6 +7690,7 @@ ${run.workflowVersion}`;
   * cancel flag stays too, re-derived from the status on any later re-drive regardless.
   */
   async compensateAndCancel(run, compensations, replay) {
+    if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
     const pending = await this.unwindCompensations(run, compensations, replay);
     if (pending) return {
       runId: run.id,
@@ -7242,11 +7745,7 @@ ${run.workflowVersion}`;
   async runExecutionTurn(run, fn, registered) {
     if (run.status === "cancelling") this.cancelRequested.add(run.id);
     if (run.status === "pending") {
-      await this.store.updateRun(run.id, {
-        status: "running",
-        updatedAt: /* @__PURE__ */ new Date()
-      });
-      run.status = "running";
+      await this.markStarted(run, registered?.singleton !== void 0);
       this.emit({
         type: "run.started",
         runId: run.id,
@@ -7283,6 +7782,7 @@ ${run.workflowVersion}`;
       });
     } catch (err) {
       if (err instanceof ContinueAsNew) {
+        if (this.isFenced(run.id)) return await this.echoCurrentStatus(run.id);
         if (this.cancelRequested.has(run.id)) {
           return await this.compensateAndCancel(run, compensations, replay);
         }
@@ -7622,6 +8122,17 @@ ${run.workflowVersion}`;
       signalEntity: /* @__PURE__ */ __name((name, key, op, arg, reply) => {
         queueMicrotask(() => void this.entities.dispatch(name, key, op, arg, reply).catch(() => void 0));
       }, "signalEntity"),
+      // The durable half of a workflow-side entity send: awaited inside the turn (a store write
+      // only, never an inline entity turn), so the ctx can checkpoint after it. See Entities.enqueue.
+      enqueueEntityOp: /* @__PURE__ */ __name((name, key, op, arg, id, reply) => this.entities.enqueue(name, key, op, arg, id, reply), "enqueueEntityOp"),
+      // Start/wake the entity for what enqueueEntityOp buffered — deferred like signalEntity above,
+      // so it never re-enters a still-executing turn. A failed wake loses nothing: the op stays
+      // buffered for the entity's next wait (and recovery's buffered-entity wake).
+      kickEntity: /* @__PURE__ */ __name((name, key) => {
+        queueMicrotask(() => void this.entities.kick(name, key).catch((err) => {
+          console.warn(`[nestjs-durable] waking entity "${name}" key "${key}" failed (its op stays buffered for the next wake):`, err);
+        }));
+      }, "kickEntity"),
       interceptStep: /* @__PURE__ */ __name((invocation, body) => this.interceptStep(invocation, body), "interceptStep")
     };
   }
@@ -7723,6 +8234,7 @@ ${run.workflowVersion}`;
   * own `finally` also releases (the run-execution path) is safe.
   */
   async parkBlocked(run, blocked) {
+    if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
     const error2 = {
       message: blocked.reason,
       code: blocked.code,
@@ -7763,7 +8275,7 @@ ${run.workflowVersion}`;
         "blocked"
       ]
     });
-    return blocked.filter((r) => (this.namespace === void 0 || r.namespace === void 0 || r.namespace === this.namespace) && r.wakeAt !== void 0 && r.wakeAt <= nowMs);
+    return blocked.filter((r) => (this.namespace === void 0 || r.namespace === void 0 || poolNamespace(r.namespace) === this.namespace) && r.wakeAt !== void 0 && r.wakeAt <= nowMs);
   }
   async callRemote(runId, seq, step, input, queue, transport, replay, admission, routing) {
     const group = stepGroup(step, routing?.namespace);
@@ -8302,17 +8814,14 @@ ${run.workflowVersion}`;
     const run = await this.store.getRun(decision.runId);
     if (!run || run.awaitingDecisionTaskId !== decision.taskId) return;
     if (run.status === "cancelled" || run.status === "completed" || run.status === "dead") return;
-    if (this.namespace !== void 0 && run.namespace !== void 0 && run.namespace !== this.namespace) {
+    if (this.namespace !== void 0 && run.namespace !== void 0 && poolNamespace(run.namespace) !== this.namespace) {
       return;
     }
     const nowMs = this.clock();
     if (!await this.store.tryLockRun(decision.runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
       return;
     }
-    const renew = setInterval(() => {
-      void this.store.renewRunLock(decision.runId, this.instanceId, this.clock() + this.leaseMs).catch(() => void 0);
-    }, Math.max(50, Math.floor(this.leaseMs / 2)));
-    renew.unref?.();
+    const held = this.holdExecutionLease(decision.runId);
     try {
       const fresh = await this.store.getRun(decision.runId);
       if (!fresh || fresh.awaitingDecisionTaskId !== decision.taskId) return;
@@ -8327,8 +8836,9 @@ ${run.workflowVersion}`;
       fresh.awaitingDecisionTaskId = void 0;
       await this.applyDecision(fresh, decision);
     } finally {
-      clearInterval(renew);
-      await this.store.releaseRunLock(decision.runId);
+      const fenced = held.lease.lost;
+      held.stop();
+      if (!fenced) await this.store.releaseRunLock(decision.runId);
     }
   }
   /** Earliest clock time a suspended turn's deadline is worth LOOKING at again (`runId` → clock ms) —
@@ -8524,6 +9034,9 @@ function childIdsOf(checkpoints) {
 }
 __name(childIdsOf, "childIdsOf");
 
+// src/run-history.ts
+var RUN_HISTORY_FIXTURE_VERSION = 1;
+
 // src/run-facets.ts
 function facetOrigin(origin) {
   return origin?.trim() ? origin : null;
@@ -9077,13 +9590,14 @@ var CodecStateStore = class {
   }
   inner;
   codec;
+  extended;
   /** Atomic claims touch only searchable metadata and bypass payload encoding. */
   tryAdmitSingleton;
   /**
   * Forwarded verbatim — a facet cell counts `status`/`origin`, neither of which the codec touches
-  * (it encodes payloads: input/output/error). Bound as a PROPERTY, and only when the inner store
-  * actually aggregates, so `store.runFacets?.()` still reports absence through the wrapper instead
-  * of resolving to a silently wrong empty result.
+  * (it encodes payloads: input/output, plus events/error under `extended` coverage). Bound as a
+  * PROPERTY, and only when the inner store actually aggregates, so `store.runFacets?.()` still
+  * reports absence through the wrapper instead of resolving to a silently wrong empty result.
   */
   runFacets;
   /**
@@ -9135,14 +9649,15 @@ var CodecStateStore = class {
   updateSchedule;
   deleteSchedule;
   listSchedules;
-  constructor(inner, codec) {
+  constructor(inner, codec, options = {}) {
     this.inner = inner;
     this.codec = codec;
+    this.extended = options.coverage === "extended";
     if (inner.tryAdmitSingleton) this.tryAdmitSingleton = inner.tryAdmitSingleton.bind(inner);
     const deleteRuns = inner.deleteRuns;
     if (deleteRuns) this.deleteRuns = (ids) => deleteRuns.call(inner, ids);
     const orphans = inner.listOrphanedRuns;
-    if (orphans) this.listOrphanedRuns = async (nowMs, limit, namespace) => (await orphans.call(inner, nowMs, limit, namespace)).map((r) => this.decRun(r));
+    if (orphans) this.listOrphanedRuns = async (nowMs, limit, namespace, ownedBy) => (await orphans.call(inner, nowMs, limit, namespace, ownedBy)).map((r) => this.decRun(r));
     const updateIf = inner.updateRunIf;
     if (updateIf) this.updateRunIf = (runId, expected, patch) => updateIf.call(inner, runId, expected, this.encRunPatch(patch));
     const saveIfPending = inner.saveCheckpointIfPending;
@@ -9195,32 +9710,52 @@ var CodecStateStore = class {
     return v === void 0 ? void 0 : this.codec.decode(v);
   }
   encRun(run) {
-    return {
+    const next = {
       ...run,
       input: this.enc(run.input),
       output: this.enc(run.output)
     };
+    if (this.extended && run.error !== void 0) {
+      next.error = this.enc(run.error);
+    }
+    return next;
   }
   decRun(run) {
-    return run && {
+    if (!run) return run;
+    const next = {
       ...run,
       input: this.dec(run.input),
       output: this.dec(run.output)
     };
+    if (this.extended && run.error !== void 0) {
+      next.error = this.dec(run.error);
+    }
+    return next;
   }
   encCp(cp) {
-    return {
+    const next = {
       ...cp,
       input: this.enc(cp.input),
       output: this.enc(cp.output)
     };
+    if (this.extended) {
+      if (cp.events !== void 0) next.events = this.enc(cp.events);
+      if (cp.error !== void 0) next.error = this.enc(cp.error);
+    }
+    return next;
   }
   decCp(cp) {
-    return cp && {
+    if (!cp) return cp;
+    const next = {
       ...cp,
       input: this.dec(cp.input),
       output: this.dec(cp.output)
     };
+    if (this.extended) {
+      if (cp.events !== void 0) next.events = this.dec(cp.events);
+      if (cp.error !== void 0) next.error = this.dec(cp.error);
+    }
+    return next;
   }
   encSpec(spec) {
     return "input" in spec ? {
@@ -9259,6 +9794,9 @@ var CodecStateStore = class {
     };
     if ("input" in patch) next.input = this.enc(patch.input);
     if ("output" in patch) next.output = this.enc(patch.output);
+    if (this.extended && patch.error !== void 0) {
+      next.error = this.enc(patch.error);
+    }
     return next;
   }
   async getRun(runId) {
@@ -9294,8 +9832,8 @@ var CodecStateStore = class {
   tryLockRun(runId, owner, leaseUntilMs, nowMs) {
     return this.inner.tryLockRun(runId, owner, leaseUntilMs, nowMs);
   }
-  releaseRunLock(runId) {
-    return this.inner.releaseRunLock(runId);
+  releaseRunLock(runId, owner) {
+    return owner === void 0 ? this.inner.releaseRunLock(runId) : this.inner.releaseRunLock(runId, owner);
   }
   renewRunLock(runId, owner, leaseUntilMs) {
     return this.inner.renewRunLock(runId, owner, leaseUntilMs);
@@ -9663,7 +10201,7 @@ var InMemoryStateStore = class {
   async createRun(run) {
     this.runs.set(run.id, {
       ...run,
-      namespace: run.namespace ?? "default"
+      namespace: run.namespace || "default"
     });
     this.reindexAttributes(run.id, run.searchAttributes);
   }
@@ -9678,7 +10216,7 @@ var InMemoryStateStore = class {
   applyRunPatch(runId, inputPatch) {
     const patch = inputPatch.status && TERMINAL_RUN_STATUSES.includes(inputPatch.status) ? {
       ...inputPatch,
-      tags: ("tags" in inputPatch ? inputPatch.tags : this.runs.get(runId)?.tags)?.filter((tag) => tag !== SINGLETON_ADMITTED_TAG)
+      tags: withoutSingletonMarkers("tags" in inputPatch ? inputPatch.tags : this.runs.get(runId)?.tags)
     } : inputPatch;
     const existing = this.runs.get(runId);
     if (!existing) throw new Error(`run ${runId} not found`);
@@ -9766,10 +10304,10 @@ var InMemoryStateStore = class {
       return true;
     });
   }
-  async listOrphanedRuns(nowMs, limit, namespace) {
+  async listOrphanedRuns(nowMs, limit, namespace, ownedBy) {
     return [
       ...this.runs.values()
-    ].filter((r) => (r.status === "running" || r.status === "cancelling") && (r.lockedUntil == null || r.lockedUntil <= nowMs) && (namespace === void 0 || r.namespace === namespace)).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()).slice(0, Math.max(0, limit)).map((r) => ({
+    ].filter((r) => (r.status === "running" || r.status === "cancelling") && (r.lockedUntil == null || r.lockedUntil <= nowMs || ownedBy !== void 0 && r.lockedBy === ownedBy.owner && r.lockedUntil < ownedBy.lockedBefore) && (namespace === void 0 || r.namespace === namespace)).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()).slice(0, Math.max(0, limit)).map((r) => ({
       ...r
     }));
   }
@@ -9803,9 +10341,9 @@ var InMemoryStateStore = class {
     run.lockedUntil = leaseUntilMs;
     return true;
   }
-  async releaseRunLock(runId) {
+  async releaseRunLock(runId, owner) {
     const run = this.runs.get(runId);
-    if (run) {
+    if (run && (owner === void 0 || run.lockedBy === owner)) {
       run.lockedBy = void 0;
       run.lockedUntil = void 0;
     }
@@ -10155,12 +10693,14 @@ export {
   ContinueAsNew,
   DEFAULT_CONCURRENCY_STATUSES,
   DEFAULT_PARTITION,
+  DEFAULT_STALLED_AFTER_MS,
   DURABLE_OPTIONS,
   DURABLE_OPTIONS_CANONICAL,
   DURABLE_STEP_CONFIG,
   DURABLE_STEP_NAME,
   DurableWorkflow,
   ENGINE_MINTED_TAG_PREFIXES,
+  ENTITY_WORKFLOW,
   EVENT_BUFFER_SCAN_LIMIT,
   EVENT_TOKEN_PREFIX,
   Entities,
@@ -10177,6 +10717,7 @@ export {
   NestedWorkflowCallError,
   NonDeterminismError,
   QueueController,
+  RUN_HISTORY_FIXTURE_VERSION,
   RUN_VALUE_FACET_LIMIT,
   RUN_VALUE_FACET_SCAN,
   RemoteStepError,
@@ -10186,6 +10727,8 @@ export {
   RunGateway,
   SINGLETON_ACTIVE_STATUSES,
   SINGLETON_ADMITTED_TAG,
+  SINGLETON_RESERVED_TAGS,
+  SINGLETON_TRACKED_TAG,
   STATE_STORE,
   STATE_STORE_CANONICAL,
   SUMMARY_CHILD_LIMIT,
@@ -10248,6 +10791,7 @@ export {
   facetOrigin,
   fanPageBounds,
   fanPageEtag,
+  findStalledRuns,
   foldedCount,
   handoffHandlerId,
   handoffName,
@@ -10261,6 +10805,7 @@ export {
   isLegacyDescriptor,
   isNonEmptyRunScope,
   isWorkflowControlFlowSignal,
+  legacySingletonHolders,
   listSignalWaitersOfRuns,
   liveHandoff,
   log,
@@ -10278,8 +10823,10 @@ export {
   pageFanFromStore,
   parentRunIdOf,
   parseDuration,
+  parseWebhookToken,
   partitionOfToken,
   planDispatch,
+  poolNamespace,
   prevCronFireMs,
   queryCheckpointsOf,
   readSearchAttributes,
@@ -10315,6 +10862,8 @@ export {
   tenantGroup,
   useDurableExecution,
   warn,
+  withSingletonTracked,
+  withoutSingletonMarkers,
   workflowName
 };
 //# sourceMappingURL=index.js.map