@dudousxd/nestjs-durable-core 0.80.1 → 0.81.0

package/dist/index.js

@@ -928,6 +928,9 @@ async function assertConcurrency(store, workflow, quota) {
 __name(assertConcurrency, "assertConcurrency");
 
 // src/entities.ts
+var NUDGE = {
+  nudge: true
+};
 var ENTITY_WORKFLOW = "__entity";
 var entityRunId = /* @__PURE__ */ __name((name, key) => `entity:${name}:${key}`, "entityRunId");
 var Entities = class {
@@ -935,9 +938,11 @@ var Entities = class {
     __name(this, "Entities");
   }
   host;
+  workflowHost;
   configs = /* @__PURE__ */ new Map();
-  constructor(host) {
+  constructor(host, workflowHost) {
     this.host = host;
+    this.workflowHost = workflowHost;
     host.register(ENTITY_WORKFLOW, "1", (ctx, input) => this.run(ctx, input));
   }
   register(name, config) {
@@ -959,6 +964,52 @@ var Entities = class {
       }
     });
   }
+  /**
+  * The durable half of a send from INSIDE a workflow (`ctx.callEntity` / `ctx.signalEntity`): write
+  * the op to the entity's signal buffer and return once it is stored, so the caller can checkpoint
+  * after it. The op carries `id`, the caller's stable `<runId>:<seq>`, so if the caller dies between
+  * this write and its checkpoint and sends again on replay, the entity skips the second copy. The
+  * entity's run is also ensured here, awaited (`start` only enqueues, it never runs the body
+  * inline), so an op sent on first contact can't be stranded in a buffer no run will ever read if
+  * the process dies right after. The wake-up ({@link kick}) is separate and may be lost: a busy
+  * entity reads the buffer at its next wait, and a parked one is paired with it by the engine's
+  * `reconcileMs` sweep. Returns `false` on a host without a workflow host.
+  */
+  async enqueue(name, key, op, arg, id, reply) {
+    if (!this.workflowHost) return false;
+    const message = {
+      op,
+      arg,
+      id,
+      ...reply ? {
+        reply
+      } : {}
+    };
+    const runId = entityRunId(name, key);
+    await this.workflowHost.bufferSignal(runId, message);
+    await this.workflowHost.ensureStarted(ENTITY_WORKFLOW, {
+      name,
+      key
+    }, runId);
+    return true;
+  }
+  /**
+  * Make the entity act on what {@link enqueue} buffered: wake its run if it is parked on its wait
+  * (the run itself was ensured by `enqueue`; ensured again here, idempotently, for a caller replay
+  * that only re-kicks). A run that is busy needs nothing: its next wait reads the buffer before
+  * suspending. The wake carries a no-op `nudge`, never the op itself, so the op stays in
+  * the buffer in arrival order and a lost or repeated kick costs at most one empty iteration.
+  */
+  async kick(name, key) {
+    const host = this.workflowHost;
+    if (!host) return;
+    const runId = entityRunId(name, key);
+    await host.ensureStarted(ENTITY_WORKFLOW, {
+      name,
+      key
+    }, runId);
+    await host.wakeWaiter(runId, NUDGE);
+  }
   async signal(name, key, op, arg) {
     await this.dispatch(name, key, op, arg);
   }
@@ -972,11 +1023,18 @@ var Entities = class {
     if (!config) throw new FatalError(`entity "${name}" is not registered`);
     const token = entityRunId(name, key);
     let state = config.initialState();
+    const seen = /* @__PURE__ */ new Set();
     for (let i = 0; ; i += 1) {
       const msg = await ctx.waitForSignal(token);
+      if (msg.nudge) continue;
+      if (msg.id !== void 0) {
+        if (seen.has(msg.id)) continue;
+        seen.add(msg.id);
+      }
+      const op = msg.op;
       const out = await ctx.localStep(`op:${i}`, async () => {
-        const handler = config.handlers[msg.op];
-        if (!handler) throw new FatalError(`entity "${name}" has no handler for op "${msg.op}"`);
+        const handler = config.handlers[op];
+        if (!handler) throw new FatalError(`entity "${name}" has no handler for op "${op}"`);
         const result = await handler(state, msg.arg);
         return {
           result,
@@ -986,8 +1044,9 @@ var Entities = class {
       state = out.state;
       await ctx.setEvent("state", state);
       if (msg.reply) {
+        const reply = msg.reply;
         await ctx.localStep(`reply:${i}`, async () => {
-          await this.host.signal(msg.reply, out.result);
+          await this.host.signal(reply, out.result);
         });
       }
     }
@@ -1911,6 +1970,10 @@ function checkpointNameKind(name) {
   return "step";
 }
 __name(checkpointNameKind, "checkpointNameKind");
+function poolNamespace(namespace) {
+  return namespace === "" ? "default" : namespace;
+}
+__name(poolNamespace, "poolNamespace");
 var TERMINAL_RUN_STATUSES = [
   "completed",
   "failed",
@@ -2042,6 +2105,19 @@ var WEBHOOK_PREFIX = "wh:";
 var CHILD_PREFIX = "child:";
 var BREAKPOINT_PREFIX = "bp:";
 var EVENT_PREFIX = EVENT_TOKEN_PREFIX;
+function parseWebhookToken(token) {
+  if (!token.startsWith(WEBHOOK_PREFIX)) return null;
+  const rest = token.slice(WEBHOOK_PREFIX.length);
+  const at = rest.lastIndexOf(":");
+  if (at <= 0) return null;
+  const seqText = rest.slice(at + 1);
+  if (!/^\d+$/.test(seqText)) return null;
+  return {
+    runId: rest.slice(0, at),
+    seq: Number(seqText)
+  };
+}
+__name(parseWebhookToken, "parseWebhookToken");
 function classifyWaiterToken(token) {
   if (token.startsWith(WEBHOOK_PREFIX)) return {
     on: "webhook",
@@ -2579,6 +2655,11 @@ __name(errorMessage, "errorMessage");
 
 // src/singleton-admission.ts
 var SINGLETON_ADMITTED_TAG = "durable:singleton:admitted";
+var SINGLETON_TRACKED_TAG = "durable:singleton:tracked";
+var SINGLETON_RESERVED_TAGS = [
+  SINGLETON_ADMITTED_TAG,
+  SINGLETON_TRACKED_TAG
+];
 var SINGLETON_ACTIVE_STATUSES = [
   "pending",
   "running",
@@ -2586,6 +2667,33 @@ var SINGLETON_ACTIVE_STATUSES = [
   "blocked",
   "cancelling"
 ];
+var LEGACY_SCAN_STATUSES = [
+  "running",
+  "suspended",
+  "cancelling"
+];
+var byAdmissionOrder = /* @__PURE__ */ __name((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id), "byAdmissionOrder");
+function withoutSingletonMarkers(tags) {
+  return tags?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
+}
+__name(withoutSingletonMarkers, "withoutSingletonMarkers");
+function withSingletonTracked(tags) {
+  const current = tags ?? [];
+  if (current.includes(SINGLETON_ADMITTED_TAG) || current.includes(SINGLETON_TRACKED_TAG)) {
+    return current;
+  }
+  return [
+    ...current,
+    SINGLETON_TRACKED_TAG
+  ];
+}
+__name(withSingletonTracked, "withSingletonTracked");
+function legacySingletonHolders(active, runId, limit) {
+  const scanned = active.filter((run) => LEGACY_SCAN_STATUSES.includes(run.status)).sort(byAdmissionOrder);
+  const window = new Set(scanned.slice(0, limit).map((run) => run.id));
+  return scanned.filter((run) => run.id !== runId && !run.tags?.includes(SINGLETON_ADMITTED_TAG) && (!run.tags?.includes(SINGLETON_TRACKED_TAG) || window.has(run.id)));
+}
+__name(legacySingletonHolders, "legacySingletonHolders");
 function singletonAdmissionTags(runs, runId, tag, workflow, limit) {
   if (!Number.isInteger(limit) || limit < 1) throw new Error("singleton limit must be a positive integer");
   const active = runs.filter((run) => run.workflow === workflow && SINGLETON_ACTIVE_STATUSES.includes(run.status) && run.tags?.includes(tag));
@@ -2594,12 +2702,13 @@ function singletonAdmissionTags(runs, runId, tag, workflow, limit) {
   if (candidate.tags?.includes(SINGLETON_ADMITTED_TAG)) return candidate.tags;
   if (candidate.status === "cancelling") return null;
   const holders = active.filter((run) => run.tags?.includes(SINGLETON_ADMITTED_TAG));
-  const remaining = limit - holders.length;
+  const legacy = new Set(legacySingletonHolders(active, runId, limit).map((run) => run.id));
+  const remaining = limit - holders.length - legacy.size;
   if (remaining <= 0) return null;
-  const waiters = active.filter((run) => !run.tags?.includes(SINGLETON_ADMITTED_TAG)).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime() || a.id.localeCompare(b.id));
+  const waiters = active.filter((run) => !run.tags?.includes(SINGLETON_ADMITTED_TAG) && !legacy.has(run.id)).sort(byAdmissionOrder);
   if (!waiters.slice(0, remaining).some((run) => run.id === runId)) return null;
   return [
-    ...candidate.tags ?? [],
+    ...(candidate.tags ?? []).filter((t) => t !== SINGLETON_TRACKED_TAG),
     SINGLETON_ADMITTED_TAG
   ];
 }
@@ -2609,7 +2718,7 @@ async function clearTerminalSingletonAdmission(runId, patch, getRun) {
   const tags = "tags" in patch ? patch.tags : (await getRun(runId))?.tags;
   return {
     ...patch,
-    tags: tags?.filter((tag) => tag !== SINGLETON_ADMITTED_TAG)
+    tags: withoutSingletonMarkers(tags)
   };
 }
 __name(clearTerminalSingletonAdmission, "clearTerminalSingletonAdmission");
@@ -2620,6 +2729,7 @@ function singletonAdmissionDenialPatch(runs, runId, tag, workflow, retryWakeAt)
   return {
     status: "suspended",
     wakeAt: retryWakeAt,
+    tags: withSingletonTracked(candidate.tags),
     updatedAt: /* @__PURE__ */ new Date()
   };
 }
@@ -2672,7 +2782,7 @@ var SingletonGate = class {
     const admitted = await this.deps.store.tryAdmitSingleton(run.id, this.tag(cfg, run.input), run.workflow, cfg.limit ?? 1, this.retryWakeAt());
     if (admitted && !run.tags?.includes(SINGLETON_ADMITTED_TAG)) {
       run.tags = [
-        ...run.tags ?? [],
+        ...(run.tags ?? []).filter((tag) => tag !== SINGLETON_TRACKED_TAG),
         SINGLETON_ADMITTED_TAG
       ];
     }
@@ -3567,30 +3677,82 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
     }));
     return id;
   }, "startChild");
+  const sendEntityOp = /* @__PURE__ */ __name(async (name, key, op, arg, seq, reply) => await host.enqueueEntityOp?.(name, key, op, arg, `${runId}:${seq}`, reply) ?? false, "sendEntityOp");
   const callEntity = /* @__PURE__ */ __name(async (name, key, op, arg) => {
     const current = pos.next();
-    const existing = await readCheckpoint(current);
+    let existing = await readCheckpoint(current);
     if (existing && existing.status === "completed") return existing.output;
     const reply = `entityreply:${runId}:${current}`;
+    const intentName = `signal:${reply}`;
+    if (existing && existing.name !== intentName) {
+      throw new NonDeterminismError(runId, current, intentName, existing.name);
+    }
+    if (existing && replay?.has(current)) {
+      existing = await store.getCheckpoint(runId, current) ?? existing;
+      if (existing.status === "completed") return existing.output;
+    }
+    const markSent = /* @__PURE__ */ __name(() => store.saveCheckpointIfPending?.(instantCheckpoint({
+      runId,
+      seq: current,
+      name: intentName,
+      kind: "signal",
+      status: "pending",
+      output: {
+        sent: true
+      }
+    })) ?? Promise.resolve(false), "markSent");
+    if (existing) {
+      const buffered = await consumeBuffered(reply, current);
+      if (buffered) {
+        await store.removeSignalWaiter({
+          token: reply,
+          runId,
+          seq: current
+        });
+        return buffered.value;
+      }
+      if (existing.output?.sent !== true) {
+        if (await sendEntityOp(name, key, op, arg, current, reply)) await markSent();
+      }
+      host.kickEntity?.(name, key);
+      throw new WorkflowSuspended();
+    }
     await store.putSignalWaiter({
       token: reply,
       runId,
       seq: current
     });
-    host.signalEntity?.(name, key, op, arg, reply);
+    if (!host.enqueueEntityOp) {
+      host.signalEntity?.(name, key, op, arg, reply);
+      throw new WorkflowSuspended();
+    }
+    await writeCheckpoint(instantCheckpoint({
+      runId,
+      seq: current,
+      name: intentName,
+      kind: "signal",
+      status: "pending",
+      output: {
+        sent: false
+      }
+    }));
+    if (await sendEntityOp(name, key, op, arg, current, reply)) await markSent();
+    host.kickEntity?.(name, key);
     throw new WorkflowSuspended();
   }, "callEntity");
   const signalEntity = /* @__PURE__ */ __name(async (name, key, op, arg) => {
     const current = pos.next();
     const existing = await readCheckpoint(current);
     if (existing && existing.status === "completed") return;
-    host.signalEntity?.(name, key, op, arg);
+    const durable = await sendEntityOp(name, key, op, arg, current);
+    if (!durable) host.signalEntity?.(name, key, op, arg);
     await writeCheckpoint(instantCheckpoint({
       runId,
       seq: current,
       name: `entitysig:${name}:${key}`,
       kind: "local"
     }));
+    if (durable) host.kickEntity?.(name, key);
   }, "signalEntity");
   const breakpoint = /* @__PURE__ */ __name(async (label) => {
     const current = pos.next();
@@ -3658,17 +3820,15 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
       const deadlineSeq = timeoutMs == null ? -1 : pos.next();
       const existing = await readCheckpoint(current);
       if (existing && existing.status === "completed") return existing.output;
-      if (timeoutMs == null) {
-        await store.putSignalWaiter({
+      const buffered = await consumeBuffered(token, current);
+      if (buffered) return buffered.value;
+      const deadline = timeoutMs == null ? void 0 : await stampDeadline(deadlineSeq, token, timeoutMs);
+      if (deadline !== void 0 && host.clock() >= deadline) {
+        await store.removeSignalWaiter({
           token,
           runId,
           seq: current
-        });
-        throw new WorkflowSuspended();
-      }
-      const deadline = await stampDeadline(deadlineSeq, token, timeoutMs);
-      if (host.clock() >= deadline) {
-        await store.takeSignalWaiter(token).catch(() => void 0);
+        }).catch(() => void 0);
         throw new SignalTimeoutError(token, timeoutMs);
       }
       await store.putSignalWaiter({
@@ -3676,6 +3836,15 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
         runId,
         seq: current
       });
+      const lateBuffered = await consumeBuffered(token, current);
+      if (lateBuffered) {
+        await store.removeSignalWaiter({
+          token,
+          runId,
+          seq: current
+        });
+        return lateBuffered.value;
+      }
       throw new WorkflowSuspended(deadline);
     }, "wait");
     return {
@@ -3867,6 +4036,12 @@ var SETTLE_ELIGIBLE_STATUSES = [
   "failed",
   "cancelling"
 ];
+var TIMEOUT_SWEPT_STATUSES = [
+  "pending",
+  "running",
+  "suspended",
+  "blocked"
+];
 var SUSPEND_ELIGIBLE_STATUSES = SETTLE_ELIGIBLE_STATUSES.filter((status) => status !== "cancelling");
 var RECOVERY_BATCH = 500;
 var NamespaceMismatch = class NamespaceMismatch2 extends Error {
@@ -3889,6 +4064,12 @@ var WorkflowEngine = class {
   instanceId;
   namespace;
   leaseMs;
+  /**
+  * Leases held by this `instanceId` that expire strictly before this epoch ms were taken by a
+  * PREVIOUS process with the same id (every lease this one takes or renews expires at
+  * `clock() + leaseMs`, so at or after it). See {@link orphanedRuns}.
+  */
+  ownLeaseCutoffMs;
   maxRecoveryAttempts;
   remoteAdvanceSilenceMs;
   reconcileMs;
@@ -4010,8 +4191,9 @@ var WorkflowEngine = class {
     this.admission = deps.admission ?? new InMemoryAdmissionBackend(this.clock);
     this.admission.onFreed?.((queue) => this.wakeQueueWaiters(queue));
     this.instanceId = deps.instanceId ?? globalThis.crypto.randomUUID();
-    this.namespace = deps.namespace;
+    this.namespace = poolNamespace(deps.namespace);
     this.leaseMs = deps.leaseMs ?? 3e4;
+    this.ownLeaseCutoffMs = this.clock() + this.leaseMs;
     this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
     this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
     this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
@@ -4131,7 +4313,11 @@ var WorkflowEngine = class {
       }
     });
     this.accumulators = new EventAccumulators(this);
-    this.entities = new Entities(this);
+    this.entities = new Entities(this, {
+      bufferSignal: /* @__PURE__ */ __name((token, payload) => this.store.bufferSignal(token, payload), "bufferSignal"),
+      wakeWaiter: /* @__PURE__ */ __name((token, payload) => this.wakeSignalWaiter(token, payload), "wakeWaiter"),
+      ensureStarted: /* @__PURE__ */ __name((workflow, input, runId) => this.start(workflow, input, runId), "ensureStarted")
+    });
   }
   /**
   * Register a **durable entity** (a virtual object): a keyed actor whose `handlers` run **serialized
@@ -4407,7 +4593,7 @@ var WorkflowEngine = class {
         // tenant-suffixed group the run will dispatch to — an operator starting a tenant's run via the
         // `onStartRun` wire (opts.namespace = tenant) must find the live `<workflow>@<tenant>` group,
         // not the bare group. Without this the start-run entry path can't reach a suffixed-only worker.
-        namespace: opts?.namespace ?? this.namespace,
+        namespace: poolNamespace(opts?.namespace) ?? this.namespace,
         input,
         createdAt: /* @__PURE__ */ new Date(),
         updatedAt: /* @__PURE__ */ new Date()
@@ -4444,7 +4630,7 @@ var WorkflowEngine = class {
         VERSION_UNDECLARED_TAG
       ] : []
     ];
-    const tags = mergeTags(registered.tags, stampedTags)?.filter((tag) => tag !== SINGLETON_ADMITTED_TAG);
+    const tags = mergeTags(registered.tags, stampedTags)?.filter((tag) => !SINGLETON_RESERVED_TAGS.includes(tag));
     if (registered.singleton) {
       await this.singletons.assertCapacity(name, registered.singleton, input);
     }
@@ -4458,7 +4644,7 @@ var WorkflowEngine = class {
       ...delayed ? {
         wakeAt: startAtMs
       } : {},
-      namespace: opts?.namespace ?? this.namespace,
+      namespace: poolNamespace(opts?.namespace) ?? this.namespace,
       // DERIVED from the registration, exactly like `namespace` is derived from this engine — there is
       // deliberately no `opts.origin`, so a caller cannot label its run as another package's work.
       // `undefined` when the registration carried none (remote/convention/synthesized child, or a
@@ -4640,12 +4826,12 @@ var WorkflowEngine = class {
         error: run.error
       };
     }
-    if (this.namespace !== void 0 && run.namespace !== void 0 && run.namespace !== this.namespace) {
+    if (this.namespace !== void 0 && run.namespace !== void 0 && poolNamespace(run.namespace) !== this.namespace) {
       await this.store.releaseRunLock(runId);
       throw new NamespaceMismatch();
     }
-    if (run.status === "failed" && run.tags?.includes(SINGLETON_ADMITTED_TAG)) {
-      run.tags = run.tags.filter((tag) => tag !== SINGLETON_ADMITTED_TAG);
+    if (run.status === "failed" && (run.tags?.includes(SINGLETON_ADMITTED_TAG) || run.tags?.includes(SINGLETON_TRACKED_TAG))) {
+      run.tags = withoutSingletonMarkers(run.tags);
       await this.store.updateRun(run.id, {
         tags: run.tags
       });
@@ -4864,6 +5050,15 @@ var WorkflowEngine = class {
   * `execution_timeout` error (terminal, so a late step result can't resurrect it), AND its children
   * are cascaded exactly as an explicit {@link cancel} cascades them.
   *
+  * WHICH RUNS. Every run not yet finished, counted from `createdAt`: `pending` and `blocked` as well
+  * as `running` and `suspended`. A run nobody picks up (no worker on its queue) or nobody can run (no
+  * worker with its capabilities) is as stuck as one that hangs mid-body, and the cap is the backstop
+  * for exactly that — leaving them out meant a run that never started lived forever while one that
+  * started and hung was cut off. Cancelling one a worker is picking up at that moment is safe: the
+  * worker's settle only writes over a live status ({@link SETTLE_ELIGIBLE_STATUSES}), so it can't
+  * bring the run back. `cancelling` is left alone on purpose — a compensating cancel's saga undo is
+  * running, it ends `cancelled` by itself, and cutting it short would skip the undo.
+  *
   * WHY THE TERMINAL WRITE IS DIRECT, not a call to {@link cancel}. Two reasons, both still standing:
   * the run must land with the `execution_timeout` error `cancel` knows nothing about (`cancel` writes
   * a plain `cancelled`, and the code is what a dashboard/alert distinguishes a timeout by), and this
@@ -4876,8 +5071,8 @@ var WorkflowEngine = class {
   * children kept running with the parent gone and nothing pointing at them — a silent orphan you
   * could only find by reading the runs table by hand. {@link cancelChildren} is the SAME recursive
   * walk `cancel` uses, so children of children go too, and it inherits that path's two guards:
-  * `cancel` returns early on a run that is already `completed`/`cancelled`/`dead` (a child that beat
-  * the sweep to the finish is never clobbered, and a re-cancel is a no-op), and it writes the
+  * `cancel` returns early on a run that is already terminal (a child that beat the sweep to the
+  * finish is never clobbered, and a re-cancel is a no-op), and it writes the
   * parent's terminal status BEFORE recursing, so a cyclic parent→child graph terminates on the
   * second visit instead of looping. Two pollers sweeping the same run concurrently therefore write
   * the same terminal state twice and cascade into already-cancelled children — idempotent, not
@@ -4893,18 +5088,11 @@ var WorkflowEngine = class {
       if (this.draining) return;
       if (reg.executionTimeoutMs == null) continue;
       const deadline = now - reg.executionTimeoutMs;
-      const inflight = [
-        ...await this.store.listRuns({
-          workflow: reg.name,
-          status: "running",
-          namespace: this.namespace
-        }),
-        ...await this.store.listRuns({
-          workflow: reg.name,
-          status: "suspended",
-          namespace: this.namespace
-        })
-      ];
+      const inflight = await this.store.listRuns({
+        workflow: reg.name,
+        statuses: TIMEOUT_SWEPT_STATUSES,
+        namespace: this.namespace
+      });
       for (const run of inflight) {
         if (this.draining) return;
         if (run.createdAt.getTime() > deadline) continue;
@@ -4937,48 +5125,103 @@ var WorkflowEngine = class {
     const results = [];
     for (const listed of await this.orphanedRuns(nowMs)) {
       if (this.draining) break;
-      const acquired = await this.store.tryLockRun(listed.id, this.instanceId, nowMs + this.leaseMs, nowMs);
-      if (!acquired) continue;
-      const run = await this.store.getRun(listed.id);
-      if (!run || run.status !== "running" && run.status !== "cancelling") {
-        await this.store.releaseRunLock(listed.id);
-        continue;
-      }
-      if (run.status === "running") {
-        const dispatched = await this.inFlightRemoteStep(run.id);
-        if (dispatched) {
-          await this.store.updateRun(run.id, {
-            status: "suspended",
-            wakeAt: this.reconcileWakeAt(dispatched.wakeAt ?? void 0),
-            updatedAt: /* @__PURE__ */ new Date()
-          });
-          await this.store.releaseRunLock(run.id);
-          results.push({
-            runId: run.id,
-            status: "suspended"
-          });
-          continue;
-        }
-      }
-      const settled = await this.countRecovery(run);
-      if (settled) {
-        results.push(settled);
-        continue;
-      }
-      await this.store.releaseRunLock(run.id);
-      if (run.status !== "cancelling") {
+      const outcome = this.isOwnLiveLease(listed, nowMs) ? await this.reclaimOwnLease(listed, nowMs) : await this.recoverOrphan(listed, nowMs);
+      if (!outcome) continue;
+      if (outcome.dispatch) await this.runDispatcher.dispatch(outcome.result.runId);
+      results.push(outcome.result);
+    }
+    return results;
+  }
+  /** A run whose still-live lease is held by this very `instanceId` (see {@link reclaimOwnLease}). */
+  isOwnLiveLease(run, nowMs) {
+    return run.lockedBy === this.instanceId && (run.lockedUntil ?? 0) > nowMs;
+  }
+  /**
+  * Recover a run whose lease is still live but held by THIS `instanceId` and predates this process
+  * (listed by {@link orphanedRuns}): the previous process with this stable id died without a
+  * graceful shutdown (OOM, SIGKILL), so nothing is renewing that lease and waiting for it to expire
+  * only delays the run. The lease is taken back with `renewRunLock` (a compare-and-set on the
+  * owner; `tryLockRun` refuses a live lease), so the lease alone cannot fence a live execution in
+  * THIS process from its own recovery: the in-process run gate does. A run with a
+  * gate is executing (or about to) here and is skipped; otherwise a gate is held while the run is
+  * recovered, so a resume landing meanwhile queues behind the recovery instead of running beside it.
+  * The gate is released before the re-dispatch, which may resume the run on this instance.
+  */
+  async reclaimOwnLease(listed, nowMs) {
+    if (this.runGates.has(listed.id)) return void 0;
+    let release;
+    const tail = new Promise((resolve) => {
+      release = resolve;
+    });
+    const gate = {
+      tail
+    };
+    this.runGates.set(listed.id, gate);
+    this.closeGateAfter(listed.id, gate, tail);
+    let outcome;
+    try {
+      const fresh = await this.store.getRun(listed.id);
+      const liveHere = fresh?.lockedBy === this.instanceId && (fresh.lockedUntil ?? 0) > nowMs && (fresh.lockedUntil ?? 0) >= this.ownLeaseCutoffMs;
+      if (fresh && !liveHere) outcome = await this.recoverOrphan(listed, nowMs, true);
+    } finally {
+      release(outcome?.result ?? {
+        runId: listed.id,
+        status: listed.status
+      });
+    }
+    return outcome;
+  }
+  /**
+  * One run of a {@link recoverIncomplete} pass: take its lease and decide on it. Resolves the
+  * pass's result for it and whether it must be re-dispatched, or `undefined` when it was not ours.
+  * `ownLease`: the run's live lease is this instance id's own, left by a previous process (see
+  * {@link reclaimOwnLease}), so it is taken back by owner rather than acquired as a free one.
+  */
+  async recoverOrphan(listed, nowMs, ownLease = false) {
+    const acquired = ownLease ? await this.store.renewRunLock(listed.id, this.instanceId, nowMs + this.leaseMs) : await this.store.tryLockRun(listed.id, this.instanceId, nowMs + this.leaseMs, nowMs);
+    if (!acquired) return void 0;
+    const run = await this.store.getRun(listed.id);
+    if (!run || run.status !== "running" && run.status !== "cancelling") {
+      await this.store.releaseRunLock(listed.id);
+      return void 0;
+    }
+    if (run.status === "running") {
+      const dispatched = await this.inFlightRemoteStep(run.id);
+      if (dispatched) {
         await this.store.updateRun(run.id, {
-          status: "pending",
+          status: "suspended",
+          wakeAt: this.reconcileWakeAt(dispatched.wakeAt ?? void 0),
           updatedAt: /* @__PURE__ */ new Date()
         });
+        await this.store.releaseRunLock(run.id);
+        return {
+          result: {
+            runId: run.id,
+            status: "suspended"
+          },
+          dispatch: false
+        };
       }
-      await this.runDispatcher.dispatch(run.id);
-      results.push({
-        runId: run.id,
-        status: run.status === "cancelling" ? "cancelling" : "pending"
+    }
+    const settled = await this.countRecovery(run);
+    if (settled) return {
+      result: settled,
+      dispatch: false
+    };
+    await this.store.releaseRunLock(run.id);
+    if (run.status !== "cancelling") {
+      await this.store.updateRun(run.id, {
+        status: "pending",
+        updatedAt: /* @__PURE__ */ new Date()
       });
     }
-    return results;
+    return {
+      result: {
+        runId: run.id,
+        status: run.status === "cancelling" ? "cancelling" : "pending"
+      },
+      dispatch: true
+    };
   }
   /**
   * The runs {@link recoverIncomplete} should try to reclaim: incomplete runs whose lease is free or
@@ -4988,12 +5231,20 @@ var WorkflowEngine = class {
   * ({@link StateStore.listOrphanedRuns}, bounded by {@link RECOVERY_BATCH}); else the full list is
   * filtered here, which still spares the probes. Shuffled, like the due-timer pass, so instances
   * recovering at once spread over the batch instead of contending for the same leases.
+  *
+  * Also listed: runs still leased by THIS `instanceId` with a lease taken before this process
+  * booted ({@link ownLeaseCutoffMs}). With a stable `instanceId` (pinned to the pod identity), that
+  * is the previous process, killed without a graceful shutdown; see {@link reclaimOwnLease}.
   */
   async orphanedRuns(nowMs) {
+    const ownedBy = {
+      owner: this.instanceId,
+      lockedBefore: this.ownLeaseCutoffMs
+    };
     if (this.store.listOrphanedRuns) {
-      return shuffle(await this.store.listOrphanedRuns(nowMs, RECOVERY_BATCH, this.namespace));
+      return shuffle(await this.store.listOrphanedRuns(nowMs, RECOVERY_BATCH, this.namespace, ownedBy));
     }
-    return shuffle((await this.store.listIncompleteRuns(this.namespace)).filter((run) => run.lockedUntil == null || run.lockedUntil <= nowMs).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()).slice(0, RECOVERY_BATCH));
+    return shuffle((await this.store.listIncompleteRuns(this.namespace)).filter((run) => run.lockedUntil == null || run.lockedUntil <= nowMs || run.lockedBy === ownedBy.owner && run.lockedUntil < ownedBy.lockedBefore).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()).slice(0, RECOVERY_BATCH));
   }
   /**
   * The remote step this run dispatched and is still awaiting a result for, if any — the signature
@@ -5036,6 +5287,7 @@ var WorkflowEngine = class {
         error: error2,
         recoveryAttempts: attempts
       });
+      this.notifyParent(run.id, failedCompletion(error2));
       return {
         runId: run.id,
         status: "dead",
@@ -5242,7 +5494,7 @@ var WorkflowEngine = class {
       error: void 0,
       ...resurrecting ? {
         recoveryAttempts: 0,
-        tags: run.tags?.filter((tag) => tag !== SINGLETON_ADMITTED_TAG)
+        tags: withoutSingletonMarkers(run.tags)
       } : {},
       updatedAt: /* @__PURE__ */ new Date()
     });
@@ -5654,6 +5906,16 @@ var WorkflowEngine = class {
     return this.deliverSignal(lateWaiter, reclaimed.payload);
   }
   /**
+  * Wake the run parked on `token` with `payload`, if there is one; otherwise do nothing — unlike
+  * {@link signal}, it never buffers. For a wake-up whose real payload is ALREADY buffered on the
+  * token (an entity op, see `Entities.kick`): a run that isn't parked reads that buffer at its next
+  * wait, so there is nothing to keep.
+  */
+  async wakeSignalWaiter(token, payload) {
+    const waiter = await this.store.takeSignalWaiter(token);
+    return waiter ? this.deliverSignal(waiter, payload) : null;
+  }
+  /**
   * Ensure a run exists for `runId`, then deliver a signal to it — atomically race-free thanks to
   * signal buffering: if the run is new (or busy / not yet waiting), the signal is buffered and
   * consumed when it reaches `waitForSignal(token)`. The canonical **durable-entity / accumulator**
@@ -6249,6 +6511,25 @@ var WorkflowEngine = class {
     });
   }
   /** Admission persists the wait atomically; never overwrite a cancellation after it returns. */
+  /**
+  * A run's first execution: `pending` -> `running`. A singleton run not yet admitted is stamped
+  * {@link SINGLETON_TRACKED_TAG} in the same write, BEFORE it competes for a slot, so a run this
+  * replica is about to admit or deny is never mistaken for a holder admitted by a replica older
+  * than 0.73 (see `legacySingletonHolders`).
+  */
+  async markStarted(run, singleton) {
+    const tags = singleton ? withSingletonTracked(run.tags) : run.tags;
+    const retag = tags !== run.tags;
+    await this.store.updateRun(run.id, {
+      status: "running",
+      ...retag ? {
+        tags
+      } : {},
+      updatedAt: /* @__PURE__ */ new Date()
+    });
+    run.status = "running";
+    if (retag) run.tags = tags;
+  }
   async singletonWaitResult(run) {
     const current = await this.store.getRun(run.id);
     await this.store.releaseRunLock(run.id);
@@ -6622,11 +6903,7 @@ ${run.workflowVersion}`;
     }
     const remote = registered.remote;
     if (run.status === "pending") {
-      await this.store.updateRun(run.id, {
-        status: "running",
-        updatedAt: /* @__PURE__ */ new Date()
-      });
-      run.status = "running";
+      await this.markStarted(run, registered?.singleton !== void 0);
       this.emit({
         type: "run.started",
         runId: run.id,
@@ -7242,11 +7519,7 @@ ${run.workflowVersion}`;
   async runExecutionTurn(run, fn, registered) {
     if (run.status === "cancelling") this.cancelRequested.add(run.id);
     if (run.status === "pending") {
-      await this.store.updateRun(run.id, {
-        status: "running",
-        updatedAt: /* @__PURE__ */ new Date()
-      });
-      run.status = "running";
+      await this.markStarted(run, registered?.singleton !== void 0);
       this.emit({
         type: "run.started",
         runId: run.id,
@@ -7622,6 +7895,17 @@ ${run.workflowVersion}`;
       signalEntity: /* @__PURE__ */ __name((name, key, op, arg, reply) => {
         queueMicrotask(() => void this.entities.dispatch(name, key, op, arg, reply).catch(() => void 0));
       }, "signalEntity"),
+      // The durable half of a workflow-side entity send: awaited inside the turn (a store write
+      // only, never an inline entity turn), so the ctx can checkpoint after it. See Entities.enqueue.
+      enqueueEntityOp: /* @__PURE__ */ __name((name, key, op, arg, id, reply) => this.entities.enqueue(name, key, op, arg, id, reply), "enqueueEntityOp"),
+      // Start/wake the entity for what enqueueEntityOp buffered — deferred like signalEntity above,
+      // so it never re-enters a still-executing turn. A failed wake loses nothing: the op stays
+      // buffered for the entity's next wait (and recovery's buffered-entity wake).
+      kickEntity: /* @__PURE__ */ __name((name, key) => {
+        queueMicrotask(() => void this.entities.kick(name, key).catch((err) => {
+          console.warn(`[nestjs-durable] waking entity "${name}" key "${key}" failed (its op stays buffered for the next wake):`, err);
+        }));
+      }, "kickEntity"),
       interceptStep: /* @__PURE__ */ __name((invocation, body) => this.interceptStep(invocation, body), "interceptStep")
     };
   }
@@ -7763,7 +8047,7 @@ ${run.workflowVersion}`;
         "blocked"
       ]
     });
-    return blocked.filter((r) => (this.namespace === void 0 || r.namespace === void 0 || r.namespace === this.namespace) && r.wakeAt !== void 0 && r.wakeAt <= nowMs);
+    return blocked.filter((r) => (this.namespace === void 0 || r.namespace === void 0 || poolNamespace(r.namespace) === this.namespace) && r.wakeAt !== void 0 && r.wakeAt <= nowMs);
   }
   async callRemote(runId, seq, step, input, queue, transport, replay, admission, routing) {
     const group = stepGroup(step, routing?.namespace);
@@ -8302,7 +8586,7 @@ ${run.workflowVersion}`;
     const run = await this.store.getRun(decision.runId);
     if (!run || run.awaitingDecisionTaskId !== decision.taskId) return;
     if (run.status === "cancelled" || run.status === "completed" || run.status === "dead") return;
-    if (this.namespace !== void 0 && run.namespace !== void 0 && run.namespace !== this.namespace) {
+    if (this.namespace !== void 0 && run.namespace !== void 0 && poolNamespace(run.namespace) !== this.namespace) {
       return;
     }
     const nowMs = this.clock();
@@ -9142,7 +9426,7 @@ var CodecStateStore = class {
     const deleteRuns = inner.deleteRuns;
     if (deleteRuns) this.deleteRuns = (ids) => deleteRuns.call(inner, ids);
     const orphans = inner.listOrphanedRuns;
-    if (orphans) this.listOrphanedRuns = async (nowMs, limit, namespace) => (await orphans.call(inner, nowMs, limit, namespace)).map((r) => this.decRun(r));
+    if (orphans) this.listOrphanedRuns = async (nowMs, limit, namespace, ownedBy) => (await orphans.call(inner, nowMs, limit, namespace, ownedBy)).map((r) => this.decRun(r));
     const updateIf = inner.updateRunIf;
     if (updateIf) this.updateRunIf = (runId, expected, patch) => updateIf.call(inner, runId, expected, this.encRunPatch(patch));
     const saveIfPending = inner.saveCheckpointIfPending;
@@ -9663,7 +9947,7 @@ var InMemoryStateStore = class {
   async createRun(run) {
     this.runs.set(run.id, {
       ...run,
-      namespace: run.namespace ?? "default"
+      namespace: run.namespace || "default"
     });
     this.reindexAttributes(run.id, run.searchAttributes);
   }
@@ -9678,7 +9962,7 @@ var InMemoryStateStore = class {
   applyRunPatch(runId, inputPatch) {
     const patch = inputPatch.status && TERMINAL_RUN_STATUSES.includes(inputPatch.status) ? {
       ...inputPatch,
-      tags: ("tags" in inputPatch ? inputPatch.tags : this.runs.get(runId)?.tags)?.filter((tag) => tag !== SINGLETON_ADMITTED_TAG)
+      tags: withoutSingletonMarkers("tags" in inputPatch ? inputPatch.tags : this.runs.get(runId)?.tags)
     } : inputPatch;
     const existing = this.runs.get(runId);
     if (!existing) throw new Error(`run ${runId} not found`);
@@ -9766,10 +10050,10 @@ var InMemoryStateStore = class {
       return true;
     });
   }
-  async listOrphanedRuns(nowMs, limit, namespace) {
+  async listOrphanedRuns(nowMs, limit, namespace, ownedBy) {
     return [
       ...this.runs.values()
-    ].filter((r) => (r.status === "running" || r.status === "cancelling") && (r.lockedUntil == null || r.lockedUntil <= nowMs) && (namespace === void 0 || r.namespace === namespace)).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()).slice(0, Math.max(0, limit)).map((r) => ({
+    ].filter((r) => (r.status === "running" || r.status === "cancelling") && (r.lockedUntil == null || r.lockedUntil <= nowMs || ownedBy !== void 0 && r.lockedBy === ownedBy.owner && r.lockedUntil < ownedBy.lockedBefore) && (namespace === void 0 || r.namespace === namespace)).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()).slice(0, Math.max(0, limit)).map((r) => ({
       ...r
     }));
   }
@@ -10161,6 +10445,7 @@ export {
   DURABLE_STEP_NAME,
   DurableWorkflow,
   ENGINE_MINTED_TAG_PREFIXES,
+  ENTITY_WORKFLOW,
   EVENT_BUFFER_SCAN_LIMIT,
   EVENT_TOKEN_PREFIX,
   Entities,
@@ -10186,6 +10471,8 @@ export {
   RunGateway,
   SINGLETON_ACTIVE_STATUSES,
   SINGLETON_ADMITTED_TAG,
+  SINGLETON_RESERVED_TAGS,
+  SINGLETON_TRACKED_TAG,
   STATE_STORE,
   STATE_STORE_CANONICAL,
   SUMMARY_CHILD_LIMIT,
@@ -10261,6 +10548,7 @@ export {
   isLegacyDescriptor,
   isNonEmptyRunScope,
   isWorkflowControlFlowSignal,
+  legacySingletonHolders,
   listSignalWaitersOfRuns,
   liveHandoff,
   log,
@@ -10278,8 +10566,10 @@ export {
   pageFanFromStore,
   parentRunIdOf,
   parseDuration,
+  parseWebhookToken,
   partitionOfToken,
   planDispatch,
+  poolNamespace,
   prevCronFireMs,
   queryCheckpointsOf,
   readSearchAttributes,
@@ -10315,6 +10605,8 @@ export {
   tenantGroup,
   useDurableExecution,
   warn,
+  withSingletonTracked,
+  withoutSingletonMarkers,
   workflowName
 };
 //# sourceMappingURL=index.js.map