@remnic/core 9.3.699 → 9.3.701

package/src/maintenance/forget.ts

@@ -22,6 +22,8 @@
  */
 
 import type { StorageManager } from "../storage.js";
+import { buildRetiredFactTombstoneInputs } from "../lifecycle/tombstones.js";
+import { supersessionKeysForFact } from "../temporal-supersession.js";
 import type { MemoryFile } from "../types.js";
 
 export interface ForgetMemoryRequest {
@@ -100,6 +102,45 @@ export async function forgetMemory(
     actor: "remnic-forget",
     reasonCode: "operator_forget",
   });
+  // Issue #1579 thread OchiF: emit a LIVE tombstone at forget time so a
+  // forgotten FACT cannot resurrect via re-extraction/import before an
+  // operator runs `remnic doctor --rebuild-tombstones`. Without this, the
+  // non-resurrection chokepoint in writeMemory has nothing to match — the
+  // forgotten fact's content hash is gone from the active index but no
+  // tombstone exists until a manual rebuild derives one — so the same fact
+  // written again through writeMemory becomes active. Mirrors supersedeMemory
+  // (contradiction) and applyTemporalSupersession via the shared
+  // buildRetiredFactTombstoneInputs helper. Best-effort (rule 34): a tombstone
+  // append failure must not fail the forget, which already succeeded on disk.
+  if (memory.frontmatter.category === "fact") {
+    for (const input of buildRetiredFactTombstoneInputs(
+      {
+        id,
+        content: memory.content,
+        ...(memory.frontmatter.contentHash
+          ? { contentHash: memory.frontmatter.contentHash }
+          : {}),
+        ...(memory.frontmatter.entityRef
+          ? { entityRef: memory.frontmatter.entityRef }
+          : {}),
+        ...(memory.frontmatter.structuredAttributes
+          ? { structuredAttributes: memory.frontmatter.structuredAttributes }
+          : {}),
+      },
+      {
+        reason: "retraction",
+        createdBy: "user_correction",
+        createdAt: forgottenAt,
+        supersessionKeysForFact,
+      },
+    )) {
+      try {
+        await storage.appendTombstone(input);
+      } catch {
+        // Best-effort — the forget already succeeded on disk.
+      }
+    }
+  }
   return {
     id,
     path: memory.path,

package/src/maintenance/pattern-reinforcement.ts

@@ -33,6 +33,8 @@
  */
 
 import { clusterByKey } from "../procedural/reinforcement-core.js";
+import { buildRetiredFactTombstoneInputs, type TombstoneReason, type TombstoneCreatedBy } from "../lifecycle/tombstones.js";
+import { supersessionKeysForFact } from "../temporal-supersession.js";
 import type {
   MemoryFile,
   MemoryFrontmatter,
@@ -50,6 +52,21 @@ export interface PatternReinforcementStorage {
     memory: MemoryFile,
     patch: Partial<MemoryFrontmatter>,
   ): Promise<boolean>;
+  /**
+   * Best-effort tombstone append for retired duplicates (issue #1579 thread
+   * Oblq9). Optional so unit-test stubs need not implement it; the production
+   * StorageManager satisfies it via its appendTombstone chokepoint.
+   */
+  appendTombstone?(input: {
+    reason: TombstoneReason;
+    createdBy: TombstoneCreatedBy;
+    sourceMemoryId: string;
+    contentHash?: string;
+    rawContent: string;
+    entityRef?: string;
+    supersessionKey?: string;
+    createdAt: string;
+  }): Promise<string | null>;
 }
 
 export interface PatternReinforcementOptions {
@@ -345,6 +362,36 @@ export async function runPatternReinforcement(
       await storage.writeMemoryFrontmatter(dup, patch);
       supersededIds.push(dup.frontmatter.id);
       result.duplicatesSuperseded += 1;
+      // Issue #1579 thread Oblq9 / Ocn0C / OcoPs: emit a tombstone so the
+      // retired duplicate cannot resurrect through re-extraction / import.
+      // Pattern reinforcement retires via writeMemoryFrontmatter which
+      // bypasses the writeMemory chokepoint; without this emit, the same
+      // content re-extracted before a manual rebuild would be stored active.
+      // Threads Ocn0C/OcoPs: emit one tombstone PER derived supersession key
+      // via buildRetiredFactTombstoneInputs (same helper as supersedeMemory +
+      // forgetMemory), not a single entityRef-only record — a paraphrased
+      // re-extraction with the same entity/attribute can only be caught by
+      // the keyed tier. Best-effort (the interface method is optional; a
+      // failure must not fail the reinforcement job).
+      if (storage.appendTombstone) {
+        for (const input of buildRetiredFactTombstoneInputs(
+          {
+            id: dup.frontmatter.id,
+            content: dup.content,
+            contentHash: dup.frontmatter.contentHash,
+            entityRef: dup.frontmatter.entityRef,
+            structuredAttributes: dup.frontmatter.structuredAttributes,
+          },
+          { reason: "supersession", createdBy: "supersession", createdAt: nowIso, supersessionKeysForFact },
+        )) {
+          try {
+            await storage.appendTombstone(input);
+          } catch {
+            // Best-effort: a tombstone append failure must not fail the
+            // reinforcement job (gotcha #13 / rule 34 spirit).
+          }
+        }
+      }
     }
 
     result.clusters.push({

package/src/memory-provenance.ts

@@ -328,6 +328,18 @@ function inferSafetyReasons(input: {
   if (frontmatter.verificationState === "disputed") {
     reasons.push("verification=disputed");
   }
+  // Issue #1579 — x-ray visibility (rule 34: never a silent drop). When a
+  // memory carries `blockedBy`, it was tombstone-blocked at the storage
+  // chokepoint. Surface the tombstone id (and the matched tier when known)
+  // in the safety reasons so every recall x-ray consumer (CLI / HTTP / MCP)
+  // can attribute the pending_review state to a tombstone rather than a
+  // generic low-confidence review.
+  if (typeof frontmatter.blockedBy === "string" && frontmatter.blockedBy.length > 0) {
+    reasons.push(`tombstone-blocked=${frontmatter.blockedBy}`);
+    if (frontmatter.tombstoneBlockTier) {
+      reasons.push(`tombstone-tier=${frontmatter.tombstoneBlockTier}`);
+    }
+  }
   if (stale) reasons.push("stale=true");
 
   const boundaryScopes = userContextScopes.filter(isUserBoundaryScope);

package/src/namespaces/catalog.test.ts

@@ -11,6 +11,7 @@ import {
   resolveDefaultNamespaceRoot,
   resolveNamespaceStorageRoot,
 } from "./storage.js";
+import { withHeldFileLock } from "../utils/serialize-mutations.js";
 
 function makeConfig(memoryDir: string, overrides: Partial<PluginConfig> = {}): PluginConfig {
   return {
@@ -2171,8 +2172,16 @@ test("registerConfiguredNamespaces skips an unsafe configured name without abort
 
 // ── Round 7 (codex P2 — NBsGP): two catalog instances in the SAME process
 // sharing a memoryDir must not treat each other's rebuild lock as self-held. A
-// touch on instance B must DROP its append while instance A holds the lock,
-// instead of skipping the wait (same PID) and appending into A's window.
+// touch on instance B must DROP its append while a foreign lock is held,
+// instead of skipping the wait (same PID) and appending into the holder's
+// window.
+//
+// Issue #1524 note: the catalog now delegates to the shared withHeldFileLock
+// utility, which generates a per-CALL owner uuid (stronger than the previous
+// per-instance lockOwnerId). ANY foreign lock — including one written by
+// another call on the SAME instance — is therefore not self-held. The test
+// seeds a foreign lock with an arbitrary uuid + a heartbeat that keeps it
+// fresh, then asserts a touch on a fresh instance waits then DROPS.
 test("a same-process second instance does not treat another instance's lock as self-held", async () => {
   const memoryDir = await mkMemoryDir();
   try {
@@ -2183,10 +2192,12 @@ test("a same-process second instance does not treat another instance's lock as s
     await mkdir(stateDir, { recursive: true });
     const lockPath = path.join(stateDir, "namespaces.rebuild.lock");
 
-    // Instance A writes a lock with ITS OWN owner id (a UUID) — same PID as B.
-    const instanceA = new NamespaceCatalog(makeConfig(memoryDir));
-    const aOwnerId = (instanceA as unknown as { lockOwnerId: string }).lockOwnerId;
-    await writeFile(lockPath, `${process.pid} ${aOwnerId} ${new Date().toISOString()}\n`, "utf8");
+    // A foreign held lock: a different PID + a real UUID owner id (so it is
+    // NEVER mistaken for self by any catalog instance in any process) + a fresh
+    // mtime. A heartbeat keeps it fresh so it is never broken as stale during
+    // the touch's bounded wait.
+    const foreignOwner = "00000000-0000-4000-8000-000000000000";
+    await writeFile(lockPath, `999999 ${foreignOwner} ${new Date().toISOString()}\n`, "utf8");
     const hb = setInterval(() => {
       const now = new Date();
       utimes(lockPath, now, now).catch(() => undefined);
@@ -2194,17 +2205,17 @@ test("a same-process second instance does not treat another instance's lock as s
     hb.unref?.();
 
     try {
-      // Instance B (different owner id, same PID) must NOT consider A's lock
-      // self-held; its touch waits then DROPS on timeout (no append).
-      const instanceB = new NamespaceCatalog(makeConfig(memoryDir));
+      // The catalog instance must NOT consider the foreign lock self-held; its
+      // touch waits then DROPS on timeout (no append).
+      const instance = new NamespaceCatalog(makeConfig(memoryDir));
       const started = Date.now();
-      await instanceB.markWrite(ns, { discoveredBy: "write", storageDir: tokenDir });
+      await instance.markWrite(ns, { discoveredBy: "write", storageDir: tokenDir });
       const waited = Date.now() - started;
-      assert.ok(waited >= 4_000, "instance B must wait on instance A's lock, not skip it as self");
+      assert.ok(waited >= 4_000, "the catalog must wait on the foreign lock, not skip it as self");
       assert.equal(
-        await instanceB.getNamespaceRecord(ns),
+        await instance.getNamespaceRecord(ns),
         null,
-        "instance B's touch must DROP while instance A's lock is held (no overwrite race)",
+        "the touch must DROP while the foreign lock is held (no overwrite race)",
       );
     } finally {
       clearInterval(hb);
@@ -2609,6 +2620,98 @@ test("a dropped resolve registration (hook returns false) is retried on a later
   }
 });
 
+// ── In-flight dedup under a DROPPED hook (cursor Medium 06f58a7c, codex P2).
+// The serializer strictly orders queued tasks, but ordering alone does not
+// collapse a burst when the hook returns `false` (dropped registration, e.g. a
+// rebuild-lock timeout): `notifiedResolved` stays unset, so every queued sibling
+// task passes its re-check and re-invokes the hook — N serial lock waits. The
+// `inFlightResolveHooks` marker (set synchronously before queueing) collapses the
+// burst to a single hook invocation; the drop stays retryable on a LATER
+// storageFor(). This test PROVE-FAILS without the marker (calls === N) and PASSES
+// with it (calls === 1).
+test("a burst of concurrent storageFor() with a DROPPED hook fires it ONCE (not once per queued task)", async () => {
+  const memoryDir = await mkMemoryDir();
+  try {
+    let calls = 0;
+    // Gated hook that DROPS (returns false) once released. This is the real
+    // scenario the reviewers flagged: the catalog's onResolve waits on the
+    // rebuild lock (slow), times out, and returns false. While that hook is
+    // in-flight, a burst of cache hits must collapse to the one in-flight
+    // registration instead of each queueing its own serial lock wait.
+    let release!: () => void;
+    const gate = new Promise<void>((r) => {
+      release = r;
+    });
+    const router = new NamespaceStorageRouter(makeConfig(memoryDir), {
+      onResolve: async () => {
+        calls += 1;
+        await gate;
+        return false; // dropped (rebuild-lock timeout analogue)
+      },
+    });
+
+    // A burst of concurrent cache hits while the hook is IN-FLIGHT. Without the
+    // in-flight dedup marker each enqueued task would re-run the dropped hook
+    // once the first settles (N serial lock waits); with it they collapse.
+    const N = 8;
+    await Promise.all(Array.from({ length: N }, () => router.storageFor("project-origin-burst-drop")));
+    // Let the in-flight hook settle as DROPPED.
+    release();
+    await router.whenResolveHooksSettled();
+    assert.equal(calls, 1, "a dropped hook fires ONCE under a burst, not once per queued task");
+
+    // The drop must remain retryable: a later storageFor() re-fires the hook.
+    await router.storageFor("project-origin-burst-drop");
+    await router.whenResolveHooksSettled();
+    assert.equal(calls, 2, "a dropped registration is retried on the next storageFor()");
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+// ── Composite-key in-flight dedup (cursor Medium, codex P2): the in-flight
+// marker must be keyed by (namespace, storageDir), NOT namespace alone. A
+// CHANGED storageDir (migration/realignment) for the same namespace while
+// another dir's hook is pending must still get its OWN hook invocation — it is
+// not collapsed onto the old dir's pending registration. A namespace-only key
+// would silently drop the new-dir notification.
+test("a CHANGED storageDir for the same namespace is not collapsed onto a pending hook", async () => {
+  const memoryDir = await mkMemoryDir();
+  try {
+    const seen: string[] = [];
+    let release!: () => void;
+    const gate = new Promise<void>((r) => {
+      release = r;
+    });
+    const router = new NamespaceStorageRouter(makeConfig(memoryDir), {
+      onResolve: async (_ns, dir) => {
+        seen.push(dir);
+        await gate;
+      },
+    });
+    // notifyResolved is private; reach it via the same cast pattern other tests
+    // use for router internals, so we can drive two distinct dirs directly.
+    const internals = router as unknown as {
+      notifyResolved(namespace: string, storageDir: string): void;
+    };
+    const dirA = path.join(memoryDir, "dir-a");
+    const dirB = path.join(memoryDir, "dir-b");
+    // dirA's hook is IN-FLIGHT (gated). dirB is a DIFFERENT dir for the same
+    // namespace — it must NOT be collapsed onto dirA's pending registration.
+    internals.notifyResolved("project-origin-dir-change", dirA);
+    internals.notifyResolved("project-origin-dir-change", dirB);
+    release();
+    await router.whenResolveHooksSettled();
+    assert.deepEqual(
+      seen.sort(),
+      [dirA, dirB].sort(),
+      "both distinct dirs fire their own hook; the new dir is not collapsed onto the pending one",
+    );
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
 // ── Round 7 (codex P2 — NDxiS): a configured non-default namespace must be seeded
 // with the ROUTER-resolved root, not a blanket tokenized dir. When a legacy raw
 // root (`namespaces/<rawname>`) already exists, the router serves it, so the
@@ -2777,8 +2880,35 @@ class SeamCatalog extends NamespaceCatalog {
     (this as unknown as { onBeforeBreakStaleUnlinkForTest?: () => Promise<void> }).onBeforeBreakStaleUnlinkForTest =
       fn;
   }
+  /**
+   * Drive the catalog's break-stale path through the shared util (issue #1524
+   * adoption). The catalog no longer owns a private breakStaleRebuildLock; the
+   * util's breakStaleLock fires inside its acquire loop, which is what
+   * withHeldCatalogLock now invokes. We trigger that path with a SHORT maxWaitMs
+   * so a surviving replacement lock is observed quickly (the production
+   * REBUILD_LOCK_MAX_WAIT_MS would force a 5s wait on the NG7Bg-replacement
+   * case). The seam is forwarded exactly as production does.
+   */
   async callBreakStaleRebuildLock(): Promise<void> {
-    await (this as unknown as { breakStaleRebuildLock: () => Promise<void> }).breakStaleRebuildLock();
+    const seam = (this as unknown as { onBeforeBreakStaleUnlinkForTest?: () => Promise<void> })
+      .onBeforeBreakStaleUnlinkForTest;
+    // Match the catalog's lock config (stale/heartbeat/poll); only maxWaitMs
+    // is shortened for test speed. The break-stale invariant does not depend
+    // on maxWaitMs — the seam fires inside breakStaleLock regardless.
+    await withHeldFileLock(
+      (this as unknown as { rebuildLockPath: string }).rebuildLockPath,
+      {
+        staleMs: 30_000,
+        maxWaitMs: 200,
+        pollMs: 10,
+        heartbeatMs: 10_000,
+        onBeforeBreakStaleUnlinkForTest: seam,
+      },
+      async () => {
+        // No-op: we only need the acquire loop to invoke breakStaleLock so the
+        // seam fires and the replacement/stale-lock invariant is exercised.
+      },
+    );
   }
 }
 
@@ -3354,3 +3484,81 @@ test("listNamespaces prefers configured token owners over stale literal token al
     await rm(memoryDir, { recursive: true, force: true });
   }
 });
+
+// ── Issue #1524 adoption prove-fail: catalog mutations route through the
+// shared MutationSerializer (instance-scoped `criticalSection`). The defect
+// class is a naive bare-.then(fn) chain that silently drops subsequent sections
+// after a rejection — exactly the poison-chain bug the shared util prevents.
+// We force the FIRST serialized section to reject, then assert the SECOND
+// section STILL runs (its record lands in the catalog). Pre-fix (a poison
+// chain) the second section would be skipped.
+test("catalog queueCritical recovers after a prior section rejects (issue #1524 poison-chain prove-fail)", async () => {
+  const memoryDir = await mkMemoryDir();
+  try {
+    const catalog = new NamespaceCatalog(makeConfig(memoryDir));
+    // Reach the private serializer to inject a failing section ahead of a real
+    // one. Both target the SAME key ("catalog") so they share a chain.
+    const serializer = (catalog as unknown as {
+      criticalSection: {
+        serialize<T>(key: string, task: () => Promise<T>): Promise<T>;
+      };
+    }).criticalSection;
+
+    let secondRan = false;
+    const [, second] = await Promise.allSettled([
+      serializer.serialize("catalog", async () => {
+        throw new Error("intentional first-section failure");
+      }),
+      serializer.serialize("catalog", async () => {
+        secondRan = true;
+      }),
+    ]);
+    assert.equal(second.status, "fulfilled", "second section settled (ran or skipped?)");
+    assert.equal(secondRan, true, "second section MUST run after the first rejected (chain recovered)");
+    // And a real catalog op still works through the same chain after the failure.
+    await catalog.markWrite("project-origin-poison-recovery", { discoveredBy: "write" });
+    const record = await catalog.getNamespaceRecord("project-origin-poison-recovery");
+    assert.ok(record, "the catalog is fully usable after a rejected section (chain not poisoned)");
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+// ── Issue #1524 adoption prove-fail: NamespaceStorageRouter resolve-hooks
+// route through the shared MutationSerializer (`resolveSerializer`). Same
+// defect class — a poison chain would skip the second hook after the first
+// rejected. We fire two notifications for the SAME namespace; the first hook
+// rejects, the second MUST still run.
+test("router resolve-hook serializer recovers after a prior hook rejects (issue #1524 poison-chain prove-fail)", async () => {
+  const memoryDir = await mkMemoryDir();
+  try {
+    let secondCalls = 0;
+    let firstCalls = 0;
+    let rejectNext = true;
+    const router = new NamespaceStorageRouter(makeConfig(memoryDir), {
+      onResolve: async () => {
+        if (rejectNext) {
+          firstCalls += 1;
+          rejectNext = false;
+          throw new Error("intentional first-hook failure");
+        }
+        secondCalls += 1;
+      },
+    });
+    // Two storageFor calls for the same namespace. The first triggers the hook
+    // (which rejects); the second queues behind it through the serializer.
+    // Both calls themselves must resolve (the rejection is best-effort inside
+    // the hook wrapper, never surfaced to the storage caller).
+    await router.storageFor("project-origin-router-poison");
+    await router.whenResolveHooksSettled();
+    await router.storageFor("project-origin-router-poison");
+    await router.whenResolveHooksSettled();
+    assert.ok(firstCalls >= 1, "the first (rejecting) hook fired");
+    assert.ok(
+      secondCalls >= 1,
+      "the second hook MUST fire after the first rejected (serializer recovered, not poisoned)",
+    );
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});