@remnic/core 9.63.4 → 9.63.6

package/src/memory-cache.ts

@@ -33,26 +33,41 @@ function sweepExpiredHotEntries(now: number): void {
   }
 }
 
-export function getCachedMemories(
-  baseDir: string,
+/**
+ * Shared read gate for every versioned cache layer (issue #2481): serve an
+ * entry only when its version and secure-store keyId match the caller.
+ * `currentVersion === 0` means version tracking is unavailable (tests, fresh
+ * installs) and always misses so disk is read. The TTL is a safety net for
+ * external filesystem edits that bypass the version sentinel (issue #1902):
+ * ttlMs <= 0 disables it, and on expiry the entry is evicted — not just
+ * skipped — so a stale dir releases its resident corpus instead of lingering
+ * until an explicit clear.
+ */
+function getVersionedCacheEntry<K, E extends { version: number; keyId: string; loadedAt: number }>(
+  cache: Map<K, E>,
+  key: K,
   currentVersion: number,
-  keyId = "",
-  ttlMs = 0,
-): MemoryFile[] | null {
-  // Don't serve from cache when version tracking is unavailable (version=0).
-  // This ensures tests and fresh installs without a version file always read disk.
+  keyId: string,
+  ttlMs: number,
+): E | null {
   if (currentVersion === 0) return null;
-  const entry = hotCacheByDir.get(baseDir);
+  const entry = cache.get(key);
   if (!entry || entry.version !== currentVersion || entry.keyId !== keyId) return null;
-  // TTL safety net (issue #1902): bound staleness from external filesystem edits
-  // that bypass the version sentinel. ttlMs <= 0 disables the check. On expiry
-  // evict the entry (not just return null) so a revisited-but-stale dir releases
-  // its resident corpus instead of lingering until an explicit clear.
   if (ttlMs > 0 && Date.now() - entry.loadedAt > ttlMs) {
-    hotCacheByDir.delete(baseDir);
+    cache.delete(key);
     return null;
   }
-  return [...entry.memories.values()];
+  return entry;
+}
+
+export function getCachedMemories(
+  baseDir: string,
+  currentVersion: number,
+  keyId = "",
+  ttlMs = 0,
+): MemoryFile[] | null {
+  const entry = getVersionedCacheEntry(hotCacheByDir, baseDir, currentVersion, keyId, ttlMs);
+  return entry ? [...entry.memories.values()] : null;
 }
 
 export function setCachedMemories(
@@ -89,10 +104,10 @@ export function getCachedArchivedMemories(
   currentVersion: number,
   keyId = "",
 ): MemoryFile[] | null {
-  if (currentVersion === 0) return null;
-  const entry = archiveCacheByDir.get(baseDir);
-  if (!entry || entry.version !== currentVersion || entry.keyId !== keyId) return null;
-  return [...entry.memories.values()];
+  // ttlMs 0: the archive cache has no age TTL; it is version-keyed and cleared
+  // via invalidation.
+  const entry = getVersionedCacheEntry(archiveCacheByDir, baseDir, currentVersion, keyId, 0);
+  return entry ? [...entry.memories.values()] : null;
 }
 
 export function setCachedArchivedMemories(
@@ -165,7 +180,7 @@ export function invalidateCachedEntities(baseDir: string): void {
 // These avoid O(146K) filter+map on every verified recall/rules call.
 interface DerivedCacheEntry<T> {
   data: T;
-  sourceVersion: number; // matches the hot cache version it was derived from
+  version: number; // matches the hot cache version it was derived from
   keyId: string; // secure-store key identity of the corpus it was derived from (#1902)
   loadedAt: number; // build time — bounds staleness from external edits via ttlMs (#1902)
 }
@@ -179,17 +194,8 @@ export function getCachedEpisodeMap(
   keyId = "",
   ttlMs = 0,
 ): Map<string, MemoryFile> | null {
-  if (currentVersion === 0) return null;
-  const entry = episodeMapByDir.get(baseDir);
-  if (!entry || entry.sourceVersion !== currentVersion || entry.keyId !== keyId) return null;
-  // TTL safety net (issue #1902): the version sentinel doesn't move on a direct
-  // external edit, so bound how long a derived view is served before a rebuild
-  // (which re-reads via readAllMemories, itself TTL-bounded). ttlMs <= 0 disables.
-  if (ttlMs > 0 && Date.now() - entry.loadedAt > ttlMs) {
-    episodeMapByDir.delete(baseDir);
-    return null;
-  }
-  return entry.data;
+  const entry = getVersionedCacheEntry(episodeMapByDir, baseDir, currentVersion, keyId, ttlMs);
+  return entry ? entry.data : null;
 }
 
 /** Build and cache the episode memory map from the full memory list. */
@@ -206,7 +212,7 @@ export function setCachedEpisodeMap(
     if (m.frontmatter.memoryKind !== "episode") continue;
     map.set(m.frontmatter.id, m);
   }
-  if (cache) episodeMapByDir.set(baseDir, { data: map, sourceVersion: version, keyId, loadedAt: Date.now() });
+  if (cache) episodeMapByDir.set(baseDir, { data: map, version, keyId, loadedAt: Date.now() });
   return map;
 }
 
@@ -217,16 +223,8 @@ export function getCachedRuleMemories(
   keyId = "",
   ttlMs = 0,
 ): { all: MemoryFile[]; byId: Map<string, MemoryFile> } | null {
-  if (currentVersion === 0) return null;
-  const entry = ruleMemoriesByDir.get(baseDir);
-  if (!entry || entry.sourceVersion !== currentVersion || entry.keyId !== keyId) return null;
-  // TTL safety net (issue #1902): bound staleness from direct external edits
-  // that don't move the version sentinel. ttlMs <= 0 disables.
-  if (ttlMs > 0 && Date.now() - entry.loadedAt > ttlMs) {
-    ruleMemoriesByDir.delete(baseDir);
-    return null;
-  }
-  return entry.data;
+  const entry = getVersionedCacheEntry(ruleMemoriesByDir, baseDir, currentVersion, keyId, ttlMs);
+  return entry ? entry.data : null;
 }
 
 /** Build and cache the rule memories from the full memory list. */
@@ -250,7 +248,7 @@ export function setCachedRuleMemories(
     }
   }
   const result = { all, byId };
-  if (cache) ruleMemoriesByDir.set(baseDir, { data: result, sourceVersion: version, keyId, loadedAt: Date.now() });
+  if (cache) ruleMemoriesByDir.set(baseDir, { data: result, version, keyId, loadedAt: Date.now() });
   return result;
 }
 

package/src/storage/content-hash-index.ts

@@ -110,6 +110,20 @@ export interface ContentHashPathEntry {
  */
 export class ContentHashIndex {
   private hashes: Set<string> = new Set();
+  /**
+   * Fact-ONLY hash partition (issue #2474). `hashes` above is category-blind
+   * (every write category registers there); this subset carries only
+   * `category === "fact"` hashes so fact-only membership checks
+   * (`hasFact*`) can never be satisfied by an unrelated decision/preference
+   * sharing the same normalized body. In-memory only — never serialized to
+   * fact-hashes.txt; `clear()` drops it and every authoritative rebuild
+   * repopulates it via `addFactByHash`. Subset invariant: `factHashes ⊆
+   * hashes` — `addFact*` registers in both, and the removal sites that drop
+   * a shared hash also drop the partition entry (removal stays caller-paired
+   * because shared-index removal is category-blind: archiving a non-fact
+   * must not evict an active fact's fact-only membership).
+   */
+  private factHashes: Set<string> = new Set();
   private dirty = false;
   /** True when load() found the pre-Unicode, markerless on-disk format. */
   private formatMigrationRequired = false;
@@ -290,6 +304,9 @@ export class ContentHashIndex {
     if (this.hashes.size > 0) {
       this.hashes.clear();
     }
+    // The fact-only partition is rebuilt with the shared set on every
+    // authoritative corpus rebuild, so it clears here too (issue #2474).
+    this.factHashes.clear();
     // A full clear is always paired with a plain overwrite save() (rebuild);
     // pending per-hash add/remove deltas are subsumed by writing the rebuilt set.
     // Cancel any armed deferred-reconcile retry (PR #2016) so it cannot fire in
@@ -693,6 +710,43 @@ export class ContentHashIndex {
     this.dirty = true;
   }
 
+  // ── Fact-only partition (issue #2474) ─────────────────────────────────
+  // Folded here from StorageManager's parallel `factOnlyHashes` set, which had
+  // to be kept in lockstep by hand on every add/remove/rebuild site. The
+  // partition lives on the index instance, so every path that mutates the
+  // shared set and needs fact-only truth mutates it in the same breath.
+
+  /** Register fact content in BOTH the shared index and the fact-only partition. */
+  addFact(content: string): void {
+    this.addFactByHash(ContentHashIndex.computeHash(content));
+  }
+
+  /** Register a pre-computed fact hash in both the shared index and the fact-only partition. */
+  addFactByHash(hash: string): void {
+    this.addByHash(hash);
+    this.factHashes.add(hash);
+  }
+
+  /** Fact-ONLY membership: has this content been registered as a fact? */
+  hasFact(content: string): boolean {
+    return this.factHashes.has(ContentHashIndex.computeHash(content));
+  }
+
+  /** Fact-ONLY membership for a pre-computed SHA-256 hash. */
+  hasFactByHash(hash: string): boolean {
+    return this.factHashes.has(hash);
+  }
+
+  /**
+   * Drop fact-only partition membership WITHOUT touching the shared index.
+   * The shared removal stays caller-owned: a shared hash is dropped only when
+   * no other active memory holds it, while fact-only membership follows the
+   * fact's own lifecycle (supersede/archive/promotion).
+   */
+  removeFactByHash(hash: string): void {
+    this.factHashes.delete(hash);
+  }
+
   /**
    * Resolve a pre-computed semantic hash to one deterministic corpus path.
    *

package/src/storage-contract/content-hash.test.ts

@@ -11,7 +11,7 @@
 import assert from "node:assert/strict";
 import test from "node:test";
 
-import { ContentHashIndex } from "../storage.js";
+import { ContentHashIndex, StorageManager } from "../storage.js";
 import { makeStorage } from "./harness.js";
 
 test("content-hash: ContentHashIndex.computeHash is deterministic for the same input", () => {
@@ -158,3 +158,61 @@ test("content-hash: the frontmatter contentHash matches computeHash of the raw c
     await cleanup();
   }
 });
+
+test("content-hash: fact-only membership across write, supersede, and rebuild — no stale true (#2474)", async () => {
+  const { storage, baseDir, cleanup } = await makeStorage();
+  try {
+    const supersededBody = "the staging region retires on friday";
+    const activeBody = "the production region stays";
+
+    // Write: both facts register fact-only membership on the shared index.
+    await storage.writeMemory("fact", supersededBody);
+    await storage.writeMemory("fact", activeBody);
+    assert.equal(
+      await storage.hasFactContentHash(supersededBody),
+      true,
+      "write registers the fact-only hash",
+    );
+    assert.equal(await storage.hasFactContentHash(activeBody), true);
+
+    // Supersede: the frontmatter rewrite runs syncFactHashIndexAfterRewrite →
+    // removeFactContentHashesForMemories, which must drop BOTH the shared hash
+    // and the fact-only partition. A stale partition entry is the PR #2016
+    // stale-true bug class: wearable / explicit-capture / promotion callers
+    // would skip a valid re-write.
+    const mem = (await storage.readAllMemories()).find((m) =>
+      (m.content ?? "").includes(supersededBody),
+    );
+    assert.ok(mem, "the fact to supersede must exist");
+    const ok = await storage.writeMemoryFrontmatter(mem!, { status: "superseded" });
+    assert.equal(ok, true, "frontmatter rewrite must succeed");
+    assert.equal(
+      await storage.hasFactContentHash(supersededBody),
+      false,
+      "no stale hasFactContentHash true after supersede",
+    );
+    assert.equal(
+      await storage.hasFactContentHash(activeBody),
+      true,
+      "the untouched fact keeps its fact-only membership",
+    );
+
+    // Rebuild: a fresh instance repopulates the index from the corpus. The
+    // superseded fact must stay out of the fact-only partition; the active
+    // fact must survive.
+    const reopened = new StorageManager(baseDir);
+    await reopened.ensureDirectories();
+    assert.equal(
+      await reopened.hasFactContentHash(supersededBody),
+      false,
+      "superseded fact stays out of fact-only membership after rebuild",
+    );
+    assert.equal(
+      await reopened.hasFactContentHash(activeBody),
+      true,
+      "active fact survives the rebuild",
+    );
+  } finally {
+    await cleanup();
+  }
+});

package/src/storage-fact-hash-index-defer.test.ts

@@ -308,10 +308,10 @@ test("direct writeMemory('fact') without the flag persists the hash via the LOCK
 });
 
 test("#2016 thread SDzOT: a fact hash added during the authoritative rebuild's corpus scan is not lost", async () => {
-  // The rebuild used to build a fresh `factOnly` set and publish it with
-  // `this.factOnlyHashes = factOnly`, dropping any concurrent in-process
-  // writeMemory that added to the LIVE set during the readAllMemories /
-  // readAllColdMemories awaits. The rebuild now repopulates the live set in place
+  // The rebuild used to build a fresh fact-only set and publish it with a
+  // reassignment, dropping any concurrent in-process writeMemory that added to
+  // the LIVE membership during the readAllMemories / readAllColdMemories
+  // awaits. The fact-only partition on ContentHashIndex repopulates in place
   // (clear + add), so a concurrent add survives publication — exactly as the
   // shared index preserves concurrent adds by mutating its `hashes` set in place.
   await withMemoryDir(async (dir) => {
@@ -320,7 +320,7 @@ test("#2016 thread SDzOT: a fact hash added during the authoritative rebuild's c
     await storage.writeMemory("fact", "alpha established fact", { source: "extraction" });
 
     const s = storage as unknown as {
-      factOnlyHashes: Set<string>;
+      factHashIndex: ContentHashIndex | null;
       factHashIndexAuthoritative: boolean | null;
       readAllColdMemories: () => Promise<unknown[]>;
       ensureFactHashIndexAuthoritative: () => Promise<boolean>;
@@ -329,13 +329,13 @@ test("#2016 thread SDzOT: a fact hash added during the authoritative rebuild's c
     const realCold = s.readAllColdMemories.bind(storage);
     let injected = false;
     // Interpose on the SECOND corpus await inside the rebuild (after clear()):
-    // simulate a concurrent writeMemory adding a fact hash to the live set while
-    // the rebuild is mid-scan.
+    // simulate a concurrent writeMemory adding a fact hash to the live
+    // partition while the rebuild is mid-scan.
     s.readAllColdMemories = async () => {
       const res = await realCold();
       if (!injected) {
         injected = true;
-        s.factOnlyHashes.add(sentinel);
+        s.factHashIndex?.addFactByHash(sentinel);
       }
       return res;
     };
@@ -345,7 +345,7 @@ test("#2016 thread SDzOT: a fact hash added during the authoritative rebuild's c
     assert.equal(await s.ensureFactHashIndexAuthoritative(), true, "rebuild published");
 
     assert.equal(
-      s.factOnlyHashes.has(sentinel),
+      s.factHashIndex?.hasFactByHash(sentinel),
       true,
       "a fact hash added during the rebuild scan must survive publication (lost under reassign)"
     );

package/src/storage.ts

@@ -1867,19 +1867,15 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   private factHashIndexAuthoritative: boolean | null = null;
   private factHashIndexAuthoritativePromise: Promise<boolean> | null = null;
   /**
-   * Fact-ONLY hash membership (PR #2016). The shared `factHashIndex` above is
-   * category-agnostic (the round-15 corpus rebuild + addContentHashDedup index
-   * every category), so it cannot answer a fact-only question. This set carries
-   * only `category === "fact"` hashes and is the sole source for
-   * `hasFactContentHash`, so an over-included non-fact body can never suppress a
-   * real fact candidate. Kept in lockstep with the shared index on EVERY path:
-   * the authoritative corpus rebuild (repopulated in place), the write path
-   * (`writeMemory` / `addActiveFactContentHash`), the storage-owned removal
-   * (`removeFactContentHashesForMemories`), AND the orchestrator's archival /
-   * consolidation removal (`removeContentHashForMemory` ->
-   * `removeFactOnlyHashForMemory`). In-memory only; never persisted.
+   * Fact-ONLY hash membership (PR #2016 / issue #2474) lives on the shared
+   * `ContentHashIndex` fact-only partition (`addFact*` / `hasFact*` /
+   * `removeFactByHash`), not a parallel set here. The partition carries only
+   * `category === "fact"` hashes and is the sole source for
+   * `hasFactContentHash`, so an over-included non-fact body can never
+   * suppress a real fact candidate. In-memory only; never persisted — every
+   * authoritative corpus rebuild repopulates it via
+   * `ContentHashIndex.clear()` + `addFactByHash`.
    */
-  private factOnlyHashes: Set<string> = new Set();
   /** Optional lock/retry tuning for the fact-hash index cross-process lock (PR #2016; tests inject tight budgets). */
   factHashIndexLockOptions: ContentHashIndexLockOptions = {};
   private readonly secureAppendChains = new Map<string, Promise<void>>();
@@ -3328,15 +3324,14 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
    */
   private async rebuildFactHashIndexFromCorpus(factHashIndex: ContentHashIndex): Promise<void> {
     factHashIndex.clear();
-    // Fact-ONLY membership rebuilt in lockstep (PR #2016): hasFactContentHash
-    // reads THIS set, never the category-agnostic shared index below, so an
-    // over-included non-fact body can never satisfy a fact-hash check. Repopulate
-    // the LIVE set in place (clear + add) rather than building a fresh set and
-    // reassigning: a concurrent in-process writeMemory that adds a fact hash
-    // during the corpus-read awaits below would be lost by a publish-time
-    // reassignment (PR #2016 thread SDzOT), exactly as the shared index avoids
-    // by mutating its `hashes` set in place.
-    this.factOnlyHashes.clear();
+    // Fact-ONLY membership is rebuilt in lockstep (PR #2016 / issue #2474):
+    // hasFactContentHash reads the index's fact-only partition, never the
+    // category-agnostic shared set, so an over-included non-fact body can
+    // never satisfy a fact-hash check. `factHashIndex.clear()` above cleared
+    // the partition too, and it repopulates in place below — a concurrent
+    // in-process writeMemory that adds a fact hash during the corpus-read
+    // awaits survives publication (PR #2016 thread SDzOT), exactly as the
+    // shared index preserves by mutating its `hashes` set in place.
     // #1909 review round 14: index the HOT and COLD tiers together. A fact or
     // procedure demoted to cold/ is still active and its content-hash must
     // survive the corpus rebuild, or a restart would drop the hash and let the
@@ -3371,8 +3366,11 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
         continue;
       }
       for (const hash of hashes) {
-        factHashIndex.addByHash(hash);
-        if (memory.frontmatter.category === "fact") this.factOnlyHashes.add(hash);
+        if (memory.frontmatter.category === "fact") {
+          factHashIndex.addFactByHash(hash);
+        } else {
+          factHashIndex.addByHash(hash);
+        }
       }
     }
     if (legacyRecovered > 0) {
@@ -3896,8 +3894,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
           options.contentHashSource !== undefined && options.contentHashSource.length > 0
             ? sanitizeMemoryContent(options.contentHashSource).text
             : sanitized.text;
-        factHashIndex.add(hashText);
-        this.factOnlyHashes.add(ContentHashIndex.computeHash(hashText));
+        factHashIndex.addFact(hashText);
         // Gate only the flush (issue #1909): the `.add(...)` above already set
         // dirty=true. When the caller defers, it owns the batch save
         // (extraction persist -> saveContentHashIndexes()). Single-write callers
@@ -3973,7 +3970,8 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     // upkeep); otherwise the snapshot may be stale, so verify against the durable
     // fact corpus (ground truth) so a lock-contended read never suppresses a fact.
     if (authoritative) {
-      return this.factOnlyHashes.has(hash);
+      const factHashIndex = await this.getFactHashIndex();
+      return factHashIndex.hasFact(sanitized.text);
     }
     return await this.factContentHashPresentInCorpus(hash);
   }
@@ -4002,8 +4000,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     await this.ensureFactHashIndexAuthoritative();
     const factHashIndex = await this.getFactHashIndex();
     for (const hash of hashes) {
-      factHashIndex.addByHash(hash);
-      this.factOnlyHashes.add(hash);
+      factHashIndex.addFactByHash(hash);
     }
     // PR #2016 thread SDzOP: flush through the SAME cross-process locked
     // reconcile the write/batch/rebuild paths use, never the unlocked whole-file
@@ -4089,7 +4086,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     for (const hash of removedHashes) {
       if (!remainingActiveHashes.has(hash)) {
         factHashIndex.removeByHash(hash);
-        this.factOnlyHashes.delete(hash);
+        factHashIndex.removeFactByHash(hash);
       }
     }
     // PR #2016 thread SDzOP: serialize the removal with the per-index lock via
@@ -4102,21 +4099,23 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   }
 
   /**
-   * Remove a memory's fact-ONLY hash membership in lockstep with the shared,
-   * category-agnostic index removal the orchestrator's
-   * `removeContentHashForMemory` performs on archival / semantic consolidation
-   * (PR #2016 threads SDzOP / SDzOR). Without this, `factOnlyHashes` kept a
-   * removed/superseded fact's hash and `hasFactContentHash` returned a stale
-   * `true` until the next corpus rebuild, so wearable / explicit-capture /
-   * promotion callers skipped a valid write. In-memory only — `factOnlyHashes`
-   * is never persisted; it is rebuilt from the corpus. No-op for non-facts.
-   * Matches the shared index's unconditional per-hash removal so the two stay
-   * coherent.
+   * Remove a memory's fact-ONLY partition membership in lockstep with the
+   * shared, category-agnostic index removal the orchestrator's
+   * `removeContentHashForMemory` performs on archival / semantic
+   * consolidation (PR #2016 threads SDzOP / SDzOR). Without this, the
+   * partition kept a removed/superseded fact's hash and `hasFactContentHash`
+   * returned a stale `true` until the next corpus rebuild, so wearable /
+   * explicit-capture / promotion callers skipped a valid write. No-op when
+   * the fact-hash index has not been loaded in this process yet (the
+   * partition is empty then and the corpus rebuild excludes the removed
+   * memory). No-op for non-facts.
    */
   removeFactOnlyHashForMemory(memory: MemoryFile): void {
     if (memory.frontmatter.category !== "fact") return;
+    const factHashIndex = this.factHashIndex;
+    if (!factHashIndex) return;
     for (const hash of this.factContentHashesForRemoval(memory)) {
-      this.factOnlyHashes.delete(hash);
+      factHashIndex.removeFactByHash(hash);
     }
   }