@remnic/core 9.3.699 → 9.3.701

package/src/namespaces/catalog.ts

@@ -1,21 +1,21 @@
 import path from "node:path";
-import { randomUUID } from "node:crypto";
 import type { Dirent } from "node:fs";
 import {
   appendFile,
   lstat,
   mkdir,
-  open,
   readdir,
   readFile,
   realpath,
   rename,
   stat,
-  unlink,
-  utimes,
   writeFile,
 } from "node:fs/promises";
 import type { PluginConfig } from "../types.js";
+import {
+  MutationSerializer,
+  withHeldFileLock,
+} from "../utils/serialize-mutations.js";
 import { isSafeRouteNamespace } from "../routing/engine.js";
 import { namespaceIdentityFromToken, namespaceIdentityToken, normalizeNamespaceIdentity } from "./identity.js";
 import { resolveDefaultNamespaceRoot, resolveNamespaceStorageRoot } from "./storage.js";
@@ -430,15 +430,16 @@ export class NamespaceCatalog {
   private readonly stateDir: string;
   private readonly catalogPath: string;
   private readonly rebuildLockPath: string;
-  // Per-INSTANCE lock owner id (round 6, codex P2 — NBsGP). The rebuild lock
-  // file records this id, not just `process.pid`, so two NamespaceCatalog
-  // instances in the SAME process sharing a memoryDir are NOT mistaken for each
-  // other: a touch on instance B must still wait for instance A's rebuild lock
-  // (different owner id, same PID) instead of skipping as "self-held".
-  private readonly lockOwnerId: string = randomUUID();
-  // Serialized write chain that recovers from rejection (CLAUDE.md rule #40)
-  // so a single failed append cannot permanently poison subsequent writes.
-  private writeChain: Promise<void> = Promise.resolve();
+  // In-process serialization for catalog mutations (issue #1524 adoption).
+  // Replaces the bespoke `writeChain` field: every touch/rebuild runs through
+  // this serializer so a single failed section never poisons subsequent ones
+  // (CLAUDE.md rule #40 — recovery is the util's contract, not re-implemented
+  // here). Cross-process identity (the lock file's owner-uuid) is now per-CALL
+  // inside the shared util, which is STRONGER than the previous per-instance
+  // `lockOwnerId` — two calls on the SAME instance get different ids, so
+  // neither mistakes the other's lock for self-held (round 6, codex P2 — NBsGP
+  // invariant preserved and tightened).
+  private readonly criticalSection = new MutationSerializer();
   // Test-only seam (round 7 — NEZkA): fires inside a touch's HELD-lock critical
   // section, after the lock is acquired but BEFORE the read→merge→append. A
   // deterministic concurrency test installs a hook here to widen the (otherwise
@@ -1826,24 +1827,23 @@ export class NamespaceCatalog {
    * their respective critical sections — closing the check-then-append gap where a
    * polled-only touch could append into a rebuild's load→rename window.
    *
-   * Acquisition is atomic via `open(..., "wx")`. A lock older than
-   * `REBUILD_LOCK_STALE_MS` is treated as a crashed holder and broken. After
-   * `REBUILD_LOCK_MAX_WAIT_MS` of contention we proceed best-effort WITHOUT the
-   * lock rather than block forever. The lock is always released in `finally`.
+   * Issue #1524 adoption: this is now a thin delegation to the shared
+   * `withHeldFileLock` utility. The acquire loop, mtime heartbeat, stale-break
+   * (NG7Bg replacement-safe), and ownership-checked release (NCzT6) all live in
+   * ONE place — the util — so this module no longer re-implements them. The
+   * catalog's `REBUILD_LOCK_*` constants and the `onBeforeBreakStaleUnlinkForTest`
+   * seam flow straight through. The util generates a per-CALL owner uuid, which
+   * is stricter than the previous per-instance `lockOwnerId` (two calls on the
+   * same instance get different ids, so neither mistakes the other's lock as
+   * self-held — the NBsGP invariant, preserved and tightened).
    *
    * IN-PROCESS SAFETY: every caller invokes this from inside (or wrapping) the
    * per-process `queueCritical` chain, which serializes all catalog mutations in
-   * THIS process. So within one process only one logical holder attempts OS-lock
-   * acquisition at a time — the file lock is never self-contended in-process, and
-   * the lock is acquired and released within a single in-process turn. The file
-   * lock adds only the missing CROSS-process exclusion.
-   *
-   * HEARTBEAT (round 5, cursor/codex Medium/P2): while WE hold the lock a timer
-   * refreshes its mtime every `REBUILD_LOCK_HEARTBEAT_MS`, so a legitimately long
-   * holder (> `REBUILD_LOCK_STALE_MS`) is not treated as a crashed holder and
-   * unlinked by another process — which would let overlapping windows lose
-   * appends. Heartbeat failures are swallowed; the timer is always cleared in
-   * `finally`.
+   * THIS process (now via `MutationSerializer`). So within one process only one
+   * logical holder attempts OS-lock acquisition at a time — the file lock is never
+   * self-contended in-process, and the lock is acquired and released within a
+   * single in-process turn. The file lock adds only the missing CROSS-process
+   * exclusion.
    *
    * ACQUISITION RESULT (round 6, codex P2 — NBPmY): `fn` receives whether WE
    * actually hold the lock. When acquisition TIMED OUT (another holder is active),
@@ -1852,152 +1852,20 @@ export class NamespaceCatalog {
    * caller uses `acquired` to run compute-only (rebuild) or DROP the append
    * (touch) when unlocked.
    */
-  private async withHeldCatalogLock<T>(fn: (acquired: boolean) => Promise<T>): Promise<T> {
-    const acquired = await this.acquireRebuildLock();
-    let heartbeat: ReturnType<typeof setInterval> | undefined;
-    if (acquired) {
-      heartbeat = setInterval(() => {
-        const now = new Date();
-        // Refresh mtime so age-based stale detection sees an active holder.
-        utimes(this.rebuildLockPath, now, now).catch(() => undefined);
-      }, REBUILD_LOCK_HEARTBEAT_MS);
-      // Don't keep the event loop alive solely for the heartbeat.
-      heartbeat.unref?.();
-    }
-    try {
-      return await fn(acquired);
-    } finally {
-      if (heartbeat) clearInterval(heartbeat);
-      if (acquired) {
-        try {
-          // Release ONLY the lock still owned by THIS instance (round 6, codex
-          // P2 — NCzT6). If this rebuild paused long enough that another process
-          // treated our lock as stale, unlinked it, and acquired a REPLACEMENT,
-          // an unconditional unlink here would delete that other holder's active
-          // lock — letting writers/another rebuild proceed during its load/rename
-          // window and recreating the lost-append race. Verify ownership first.
-          if (await this.rebuildLockHeldBySelf()) {
-            await unlink(this.rebuildLockPath);
-          }
-        } catch {
-          // Best-effort release; a stale lock will be broken on next rebuild.
-        }
-      }
-    }
-  }
-
-  /** Try to acquire the rebuild lock; returns true if WE created it. */
-  private async acquireRebuildLock(): Promise<boolean> {
-    const deadline = Date.now() + REBUILD_LOCK_MAX_WAIT_MS;
-    await mkdir(this.stateDir, { recursive: true });
-    for (;;) {
-      try {
-        const handle = await open(this.rebuildLockPath, "wx");
-        try {
-          // Record PID, this instance's owner id, and a timestamp. The owner id
-          // distinguishes same-process instances (NBsGP).
-          await handle.writeFile(
-            `${process.pid} ${this.lockOwnerId} ${new Date().toISOString()}\n`,
-            "utf8",
-          );
-        } catch {
-          // Ignore write failures — the exclusive create already gave us the lock.
-        } finally {
-          await handle.close();
-        }
-        return true;
-      } catch (err) {
-        if ((err as NodeJS.ErrnoException)?.code !== "EEXIST") {
-          // Unexpected FS error — proceed best-effort without the lock.
-          return false;
-        }
-        // Lock exists: break it if stale, otherwise wait briefly.
-        await this.breakStaleRebuildLock();
-        if (Date.now() >= deadline) return false;
-        await new Promise((r) => setTimeout(r, REBUILD_LOCK_POLL_MS));
-      }
-    }
-  }
-
-  /**
-   * Remove the lock file if its mtime is older than the stale threshold.
-   *
-   * REPLACEMENT-SAFE (NG7Bg, codex P2): a plain `stat` → `unlink` has a TOCTOU
-   * window — two processes can both observe the SAME stale lock; one removes it and
-   * creates a FRESH lock, and the other's later `unlink` then deletes that fresh
-   * holder's ACTIVE lock based on the stale identity it read earlier, leaving the
-   * fresh holder running its critical section with no visible lock and reopening the
-   * lost-update race the mutex prevents. We therefore capture the lock's IDENTITY
-   * (its full content line: `<pid> <owner-uuid> <iso>`) when we judge it stale, then
-   * RE-READ immediately before unlinking and only remove it when the content is
-   * byte-identical AND still stale. A replacement lock has a different owner id /
-   * timestamp, so its content differs and we leave it untouched. We never unlink a
-   * lock whose mtime is now fresh (a heartbeat refreshed it) or whose identity
-   * changed (a replacement was created). This is best-effort: any mismatch/vanish
-   * simply skips the break and the caller polls again.
-   */
-  private async breakStaleRebuildLock(): Promise<void> {
-    let staleIdentity: string;
-    try {
-      const info = await stat(this.rebuildLockPath);
-      if (Date.now() - info.mtimeMs <= REBUILD_LOCK_STALE_MS) {
-        // Not stale (e.g. a live holder's heartbeat keeps it fresh) — leave it.
-        return;
-      }
-      // Capture the exact identity we judged stale, so we can confirm it has not
-      // been replaced before we unlink.
-      staleIdentity = await readFile(this.rebuildLockPath, "utf8");
-    } catch {
-      // Lock vanished (released by holder) or stat/read failed — nothing to do.
-      return;
-    }
-    // Test-only seam: simulate a replacement lock being created in the race window
-    // between the staleness judgment and the unlink (NG7Bg). No-op in production.
-    if (this.onBeforeBreakStaleUnlinkForTest) {
-      await this.onBeforeBreakStaleUnlinkForTest();
-    }
-    try {
-      // Re-validate immediately before unlinking: the lock must still carry the
-      // SAME identity AND still be stale. If a replacement lock was created in the
-      // window (different owner/timestamp) or a heartbeat refreshed the mtime, do
-      // NOT unlink — that would delete another process's ACTIVE lock.
-      const current = await readFile(this.rebuildLockPath, "utf8");
-      if (current !== staleIdentity) return; // replaced — leave the fresh lock
-      const recheck = await stat(this.rebuildLockPath);
-      if (Date.now() - recheck.mtimeMs <= REBUILD_LOCK_STALE_MS) return; // refreshed
-      await unlink(this.rebuildLockPath).catch(() => undefined);
-    } catch {
-      // The lock changed/vanished between checks — another process handled it.
-    }
-  }
-
-  /**
-   * Whether the rebuild lock file was written by THIS instance (round 6, codex
-   * P2 — NBsGP). Matches the per-instance owner id, NOT just `process.pid`: two
-   * NamespaceCatalog instances in the same process share a PID, so a PID-only
-   * check would wrongly treat instance A's lock as self-held by instance B and
-   * let B's touch skip the wait and append into A's rebuild window. Falls back to
-   * the legacy PID-only form for lock files written before owner ids existed.
-   */
-  private async rebuildLockHeldBySelf(): Promise<boolean> {
-    try {
-      const body = await readFile(this.rebuildLockPath, "utf8");
-      const parts = body.trim().split(/\s+/);
-      const pid = Number.parseInt(parts[0] ?? "", 10);
-      const ownerId = parts[1];
-      // New format: "<pid> <uuid> <iso>". A UUID at parts[1] uniquely identifies
-      // the writing INSTANCE; only the same instance is self. The strict UUID
-      // shape avoids mistaking a legacy "<pid> <iso>" timestamp (also hyphenated)
-      // for an owner id.
-      const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
-      if (ownerId && UUID_RE.test(ownerId)) {
-        return ownerId === this.lockOwnerId;
-      }
-      // Legacy format: "<pid> <iso>" (no owner id). Best-effort PID match.
-      return Number.isFinite(pid) && pid === process.pid;
-    } catch {
-      return false;
-    }
+  private withHeldCatalogLock<T>(fn: (acquired: boolean) => Promise<T>): Promise<T> {
+    return withHeldFileLock(
+      this.rebuildLockPath,
+      {
+        staleMs: REBUILD_LOCK_STALE_MS,
+        maxWaitMs: REBUILD_LOCK_MAX_WAIT_MS,
+        pollMs: REBUILD_LOCK_POLL_MS,
+        heartbeatMs: REBUILD_LOCK_HEARTBEAT_MS,
+        // NG7Bg seam: fires inside the util's breakStaleLock after it judges the
+        // lock stale and captures its identity, before the atomic rename+verify.
+        onBeforeBreakStaleUnlinkForTest: this.onBeforeBreakStaleUnlinkForTest,
+      },
+      fn,
+    );
   }
 
   /**
@@ -2071,21 +1939,20 @@ export class NamespaceCatalog {
 
   /**
    * Serialize an arbitrary read-modify-write critical section through the single
-   * write chain. Every catalog mutation (touch read+merge+append, full rewrite)
-   * runs through this so they are mutually exclusive: a touch always reads the
-   * latest persisted state before appending, and a rebuild rewrite cannot
-   * interleave with a touch's append. The chain recovers from rejection
-   * (CLAUDE.md rule #40) — one failed section never poisons subsequent ones —
-   * while still surfacing the error to that section's awaited promise.
+   * per-instance chain. Every catalog mutation (touch read+merge+append, full
+   * rewrite) runs through this so they are mutually exclusive: a touch always
+   * reads the latest persisted state before appending, and a rebuild rewrite
+   * cannot interleave with a touch's append.
+   *
+   * Issue #1524 adoption: delegates to the shared `MutationSerializer` (stored
+   * as `criticalSection`). The util owns the rejection-recovery invariant
+   * (CLAUDE.md rule #40 — one failed section never poisons subsequent ones, but
+   * the failing section's error still surfaces to ITS awaited promise) and the
+   * no-unbounded-growth cleanup. The key is constant: the catalog has ONE
+   * logical mutation queue (touches and rebuilds mutually exclude in-process).
    */
   private queueCritical<T>(fn: () => Promise<T>): Promise<T> {
-    const run = this.writeChain.then(fn);
-    // Keep the chain alive after a rejection so later sections still run.
-    this.writeChain = run.then(
-      () => undefined,
-      () => undefined,
-    );
-    return run;
+    return this.criticalSection.serialize("catalog", fn);
   }
 
   /**

package/src/namespaces/storage.ts

@@ -6,6 +6,7 @@ import type { PluginConfig } from "../types.js";
 import { ALL_CATEGORY_DIRS } from "../utils/category-dir.js";
 import { namespaceIdentityToken, normalizeNamespaceIdentity } from "./identity.js";
 import type { NamespaceCatalog } from "./catalog.js";
+import { MutationSerializer } from "../utils/serialize-mutations.js";
 
 async function exists(p: string): Promise<boolean> {
   try {
@@ -209,21 +210,22 @@ export class NamespaceStorageRouter {
   // rebuilds. We fire the hook only when the (namespace, storageDir) pair is new
   // or its dir changed, so a steady-state cache hit is a no-op for the catalog.
   private readonly notifiedResolved = new Map<string, string>();
-  // In-flight resolve-hook dedup (NFJV-, codex P2). The catalog's `onResolve`
-  // hook is ASYNC (it returns `registerResolved(...)`), so `notifiedResolved` is
-  // only set after the hook's promise SETTLES. Without tracking the in-flight
-  // window, a burst of `storageFor()` cache hits for the SAME namespace before
-  // the first registration finishes would each pass the `notifiedResolved` guard
-  // and fire their OWN `onResolve` — queueing N duplicate catalog touches + lock
-  // acquisitions despite the once-per-namespace intent. We therefore record the
-  // (namespace → storageDir) being registered BEFORE awaiting the hook so a
-  // concurrent call for the same pair skips firing. On SUCCESS the pair is
-  // promoted to `notifiedResolved` (future calls skip permanently); on `false`
-  // (dropped touch — e.g. rebuild-lock timeout) OR rejection the in-flight marker
-  // is CLEARED so a later `storageFor()` can RETRY the dropped registration. The
-  // entry is always removed when the promise settles, so the map cannot grow
-  // unbounded (one transient entry per concurrently-resolving namespace).
-  private readonly inFlightResolved = new Map<string, string>();
+  // Instance-scoped serializer for the resolve hook (issue #1524 adoption).
+  // Replaces the bespoke `inFlightResolved` marker-then-clear pattern: the
+  // serializer's per-key chain strictly orders concurrent notifications for the
+  // SAME namespace, and the task re-checks `notifiedResolved` once it runs, so a
+  // burst of cache hits before the first hook settles collapses to a single
+  // hook invocation (the once-per-namespace intent). Recovery is preserved —
+  // one rejected hook never poisons subsequent notifications for that key.
+  private readonly resolveSerializer = new MutationSerializer();
+  // (namespace, storageDir) pairs whose resolve hook is currently pending. Set SYNCHRONOUSLY in
+  // notifyResolved (before queueing) so a burst of concurrent storageFor()
+  // cache hits collapses onto the one in-flight hook instead of each enqueuing
+  // its own task. Cleared when the queued hook settles. Without this, a dropped
+  // hook (returns false on a rebuild-lock timeout) leaves notifiedResolved
+  // unset, so every queued sibling task would re-run the hook — N serial lock
+  // waits (cursor Medium 06f58a7c, codex P2).
+  private readonly inFlightResolveHooks = new Set<string>();
   // Tracks every in-flight resolve-hook promise so callers can deterministically
   // await the fire-and-forget registrations that `storageFor()` kicks off (see
   // `whenResolveHooksSettled`). Entries are removed as each hook settles, so the
@@ -297,6 +299,9 @@ export class NamespaceStorageRouter {
     // #1522: install the post-write catalog touch at the chokepoint — every
     // successful write on this StorageManager records the namespace touch.
     this.bindCatalogWriteHook(sm, ns);
+    // #1579: apply the tombstone non-resurrection config so every namespace
+    // storage enforces the invariant at its own writeMemory chokepoint.
+    this.applyTombstonesConfig(sm, ns);
     this.cache.set(ns, sm);
     this.notifyResolved(ns, root);
     return sm;
@@ -305,77 +310,82 @@ export class NamespaceStorageRouter {
   /**
    * Fire the resolve hook defensively. A hook failure (e.g. a catalog write
    * error) MUST NOT crash storage resolution — see CLAUDE.md gotcha #13.
+   *
+   * Issue #1524 adoption: hook invocations for the SAME namespace are now
+   * strictly ordered through the shared `MutationSerializer` rather than the
+   * bespoke `inFlightResolved` marker-then-clear pattern. The serializer
+   * guarantees one in-flight hook per namespace; a concurrent burst of
+   * `storageFor()` cache hits collapses to a single hook invocation because
+   * each queued task re-checks `notifiedResolved` before invoking the hook. A
+   * dropped (`false`) or rejected hook leaves `notifiedResolved` unset so the
+   * next `storageFor()` retries — the serializer recovers from the rejection,
+   * so a failed hook never poisons later notifications.
    */
   private notifyResolved(namespace: string, storageDir: string): void {
     const hook = this.hooks.onResolve;
     if (!hook) return;
-    // Skip when we've already SUCCESSFULLY notified this exact (namespace,
-    // storageDir) — a steady-state cache hit must not re-append to the catalog
-    // log (NCNL2). A changed dir (rare: migration/realignment) still re-fires
-    // once. We mark the pair as notified ONLY AFTER the hook succeeds, and CLEAR
-    // it on failure, so a dropped registration (e.g. rebuild-lock timeout) is
-    // RETRIED on the next cache hit instead of being suppressed forever (round 6,
-    // cursor Medium — ND3EJ).
+    // Permanent dedup: skip once we've SUCCESSFULLY notified this exact
+    // (namespace, storageDir). A changed dir (rare: migration/realignment)
+    // still re-fires once. The mark is set ONLY AFTER the hook succeeds, so a
+    // dropped registration (e.g. rebuild-lock timeout) is RETRIED on the next
+    // cache hit instead of being suppressed forever (round 6, cursor Medium —
+    // ND3EJ).
     if (this.notifiedResolved.get(namespace) === storageDir) return;
-    // In-flight dedup (NFJV-, codex P2): if a registration for this exact
-    // (namespace, storageDir) is already AWAITING its async hook, do not fire a
-    // second one. Without this, concurrent cache-hit bursts before the first
-    // append settles each pass the `notifiedResolved` guard above and queue
-    // duplicate catalog touches/lock acquisitions. A pair with a DIFFERENT
-    // in-flight dir (rare mid-migration realignment) still fires once.
-    if (this.inFlightResolved.get(namespace) === storageDir) return;
-    try {
-      // Handle BOTH synchronous throws and asynchronous rejections (round 6,
-      // codex P2 — NDo8C). The hook may be `async`; its rejected promise would
-      // bypass this try/catch and, where unhandled rejections are fatal, crash
-      // storage resolution. Mark the dedup pair as notified ONLY when the hook
-      // resolves to a PERSISTED result (round 6, codex P2 — NEFoX): a result of
-      // `false` means the registration was dropped/no-op (e.g. rebuild-lock
-      // timeout), so we must NOT suppress its retry. `void`/`undefined` is treated
-      // as success for legacy hooks. On rejection we leave it un-notified to retry.
-      //
-      // Record the in-flight marker BEFORE awaiting so concurrent calls for the
-      // same pair skip (NFJV-). It is always cleared once the promise settles, so
-      // the map holds at most one transient entry per concurrently-resolving
-      // namespace and cannot grow unbounded.
-      this.inFlightResolved.set(namespace, storageDir);
-      const hookResult = Promise.resolve(hook(namespace, storageDir));
-      // Track the in-flight promise so `whenResolveHooksSettled()` can await it.
-      this.pendingResolveHooks.add(hookResult);
-      hookResult.then(
-        (persisted) => {
-          // Clear the in-flight marker ONLY if it is still ours (a newer resolve
-          // for a different dir may have replaced it).
-          if (this.inFlightResolved.get(namespace) === storageDir) {
-            this.inFlightResolved.delete(namespace);
-          }
-          if (persisted !== false) {
-            this.notifiedResolved.set(namespace, storageDir);
-          }
-          // On `false` (dropped touch) we intentionally do NOT mark notified, so
-          // a later `storageFor()` retries the registration. Clearing the
-          // in-flight marker above is what re-enables that retry.
-          this.pendingResolveHooks.delete(hookResult);
-        },
-        () => {
-          // Registration failed — clear in-flight AND do NOT mark as notified, so
-          // it is retried on the next cache hit.
-          if (this.inFlightResolved.get(namespace) === storageDir) {
-            this.inFlightResolved.delete(namespace);
-          }
-          if (this.notifiedResolved.get(namespace) === storageDir) {
-            this.notifiedResolved.delete(namespace);
-          }
-          this.pendingResolveHooks.delete(hookResult);
-        },
-      );
-    } catch {
-      // Synchronous throw: clear any in-flight marker we just set and leave the
-      // pair un-notified so a later resolve retries.
-      if (this.inFlightResolved.get(namespace) === storageDir) {
-        this.inFlightResolved.delete(namespace);
+    // In-flight dedup (cursor Medium 06f58a7c, codex P2): if a hook for THIS
+    // namespace is already pending, collapse this call onto it instead of
+    // enqueueing another task. The serializer strictly orders queued tasks, but
+    // ordering alone is not enough — when the first hook returns `false`
+    // (dropped touch, e.g. rebuild-lock timeout), `notifiedResolved` stays
+    // unset, so each queued sibling task would pass its re-check and re-run the
+    // hook. With the real catalog hook each retry can spend the full lock wait,
+    // so N cache hits during a rebuild leave N serial background lock attempts.
+    // Collapsing here means at most ONE hook runs per in-flight registration;
+    // its result (set `notifiedResolved` on success, leave unset on drop)
+    // decides whether a LATER `storageFor()` retries — collapsing loses
+    // nothing because the drop is already retried on the next cache hit.
+    // Keyed by the composite (namespace, storageDir) so a CHANGED dir
+    // (migration/realignment) for the same namespace still gets its own hook —
+    // it is NOT collapsed onto the old dir's pending registration (cursor
+    // Medium, codex P2).
+    const inFlightKey = namespace + "\u0000" + storageDir;
+    if (this.inFlightResolveHooks.has(inFlightKey)) return;
+    this.inFlightResolveHooks.add(inFlightKey);
+    // Queue through the serializer. Concurrent calls for the same namespace
+    // strictly order here; the 2nd call's task runs only after the 1st's hook
+    // settles, by which point `notifiedResolved` is either set (no-op) or still
+    // unset (retry). The returned promise is tracked for
+    // `whenResolveHooksSettled()`. Rejections from the task body are caught so
+    // they never reach the caller (best-effort hook contract); the serializer's
+    // recovered tail still lets subsequent notifications run.
+    const task = async (): Promise<void> => {
+      // Re-check after queueing: a prior task in this same chain may have just
+      // marked the pair as notified.
+      if (this.notifiedResolved.get(namespace) === storageDir) return;
+      try {
+        // Hook may be sync or async; Promise.resolve normalizes both. A result
+        // of `false` means the registration was dropped/no-op (e.g. rebuild-lock
+        // timeout) — leave notifiedResolved UNSET so the next storageFor retries
+        // (round 6, codex P2 — NEFoX). `void`/`undefined` is success for legacy
+        // hooks. A rejection leaves notifiedResolved unset for the same reason.
+        const persisted = await Promise.resolve(hook(namespace, storageDir));
+        if (persisted !== false) {
+          this.notifiedResolved.set(namespace, storageDir);
+        }
+      } catch {
+        // Best-effort: a hook failure MUST NOT crash storage resolution. Leave
+        // notifiedResolved unset so a later storageFor retries the registration.
       }
-    }
+    };
+    const queued = this.resolveSerializer.serialize(namespace, task);
+    this.pendingResolveHooks.add(queued);
+    // Clear the in-flight marker when the hook settles so a subsequent
+    // storageFor() can retry after a drop, or short-circuit via notifiedResolved
+    // after a success.
+    const cleanup = (): void => {
+      this.inFlightResolveHooks.delete(inFlightKey);
+      this.pendingResolveHooks.delete(queued);
+    };
+    void queued.then(cleanup, cleanup);
   }
 
   /**
@@ -387,6 +397,36 @@ export class NamespaceStorageRouter {
     sm.onCatalogWrite = () => this.touchCatalogWrite(namespace, sm.dir);
   }
 
+  /**
+   * Install the tombstone config on an externally-constructed StorageManager
+   * (issue #1579). Mirrors `bindCatalogWriteHook` — used for the legacy
+   * default-namespace storage that bypasses the router. Router-created
+   * storages are wired inline in `storageFor()` via `applyTombstonesConfig`.
+   */
+  bindTombstonesConfig(
+    sm: StorageManager,
+    namespace: string,
+    config: { enabled: boolean; semanticMatch: boolean; semanticThreshold: number },
+  ): void {
+    this.tombstonesGlobalConfig = { ...config };
+    sm.setTombstonesConfig({ ...config, namespace });
+  }
+
+  private tombstonesGlobalConfig: {
+    enabled: boolean;
+    semanticMatch: boolean;
+    semanticThreshold: number;
+  } = { enabled: false, semanticMatch: false, semanticThreshold: 0.9 };
+
+  /**
+   * Apply the tombstone config to a router-created StorageManager. Called
+   * inline in `storageFor()` so every namespace storage enforces the
+   * non-resurrection invariant, namespace-scoped (rule 42).
+   */
+  private applyTombstonesConfig(sm: StorageManager, namespace: string): void {
+    sm.setTombstonesConfig({ ...this.tombstonesGlobalConfig, namespace });
+  }
+
   /**
    * Post-write catalog touch (issue #1522 chokepoint). Called by every
    * StorageManager's post-write hook AFTER a successful write. Best-effort

package/src/operator-toolkit.ts

@@ -1326,6 +1326,12 @@ export async function runOperatorDoctor(options: OperatorDoctorOptions): Promise
     details: namespaceMaintenanceHealth,
   });
 
+  // Tombstone non-resurrection invariant (issue #1579). Reports active
+  // tombstone count, last append, corrupted-line count, and rebuild
+  // staleness so operators can verify the invariant is enforced and know
+  // when a rebuild is warranted. Informational: never errors on its own.
+  checks.push(await summarizeTombstoneStatus(options.orchestrator.storage));
+
   const summary = checks.reduce(
     (acc, check) => {
       acc[check.status] += 1;
@@ -1871,6 +1877,77 @@ export async function summarizeObservationThroughput(
   }
 }
 
+/**
+ * Tombstone non-resurrection invariant status for `remnic doctor`
+ * (issue #1579).
+ *
+ * Reports:
+ *   - active tombstone count (excluding revoked)
+ *   - revoked count (re-allowed via the review queue)
+ *   - last append timestamp
+ *   - corrupted-line count (skipped, not crashed — rule 34/18)
+ *   - rebuild staleness (whether the in-memory index loaded cleanly)
+ *
+ * Always informational: a disabled or empty tombstone store is the expected
+ * cold-install state and is never an error. A non-zero corrupted-line count
+ * surfaces as `warn` so operators know a rebuild (`remnic doctor
+ * --rebuild-tombstones`) will repair the log.
+ */
+export async function summarizeTombstoneStatus(
+  storage: StorageManager,
+): Promise<OperatorDoctorCheck> {
+  try {
+    const stats = await storage.getTombstoneStats();
+    if (stats === null) {
+      return {
+        key: "tombstones",
+        status: "ok",
+        summary: "Tombstone non-resurrection invariant is disabled (tombstonesEnabled=false).",
+        details: { enabled: false },
+      };
+    }
+    const corrupted = stats.corruptedLines > 0;
+    const parts = [
+      `${stats.count} active tombstone(s)`,
+      ...(stats.revoked > 0 ? [`${stats.revoked} revoked`] : []),
+    ];
+    if (stats.lastAppendAt) {
+      parts.push(`last append ${stats.lastAppendAt}`);
+    }
+    if (corrupted) {
+      parts.push(`${stats.corruptedLines} corrupted line(s) skipped`);
+    }
+    return {
+      key: "tombstones",
+      status: corrupted ? "warn" : "ok",
+      summary:
+        `Tombstone invariant enabled. ${parts.join(", ")}. ` +
+        (corrupted
+          ? "Run `remnic doctor --rebuild-tombstones` to repair the log."
+          : "Re-observation of retired facts is blocked at the storage chokepoint."),
+      remediation: corrupted
+        ? "Rebuild the tombstone log from retired memories on disk."
+        : undefined,
+      details: {
+        enabled: true,
+        count: stats.count,
+        revoked: stats.revoked,
+        lastAppendAt: stats.lastAppendAt,
+        corruptedLines: stats.corruptedLines,
+        loaded: stats.loaded,
+      },
+    };
+  } catch (err) {
+    return {
+      key: "tombstones",
+      status: "warn",
+      summary: "Could not read tombstone store stats.",
+      remediation: "Ensure the memory state directory is readable and rerun `remnic doctor`.",
+      details: { error: String(err) },
+    };
+  }
+}
+
 export async function summarizeConsolidationProvenance(
   storage: StorageManager,
   config: Pick<PluginConfig, "memoryDir"> & { versioningSidecarDir?: string },