@remnic/core 9.63.3 → 9.63.5

package/src/orchestration/turn-ingestion-snapshot.test.ts

@@ -0,0 +1,167 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { SmartBuffer } from "../buffer.js";
+import { parseConfig } from "../config.js";
+import type { BufferState, BufferTurn } from "../types.js";
+import {
+  TurnIngestionCoordinator,
+  type TurnIngestionDeps,
+} from "./turn-ingestion.js";
+
+class FakeStorage {
+  constructor(private readonly initial: BufferState) {}
+  async loadBuffer(): Promise<BufferState> {
+    return structuredClone(this.initial);
+  }
+  async saveBuffer(_state: BufferState): Promise<void> {}
+}
+
+function makeTurn(content: string): BufferTurn {
+  return {
+    role: "user",
+    content,
+    timestamp: "2026-08-17T00:00:00.000Z",
+    sessionKey: "k",
+  };
+}
+
+test("#2468 extract trigger returns the mutation snapshot", async () => {
+  const buffer = new SmartBuffer(
+    parseConfig({ triggerMode: "every_n", bufferMaxTurns: 1 }),
+    new FakeStorage({ turns: [], lastExtractionAt: null, extractionCount: 0 }) as never,
+  );
+  const outcome = await buffer.addTurnWithOutcome("k", makeTurn("only"));
+  assert.equal(outcome.decision, "extract_batch");
+  assert.equal(outcome.extractionTurns?.length, 1);
+  assert.equal(outcome.extractionTurns?.[0]?.content, "only");
+});
+
+test("#2468 built-in trigger includes retained turns in the snapshot", async () => {
+  const buffer = new SmartBuffer(
+    parseConfig({ triggerMode: "every_n", bufferMaxTurns: 1 }),
+    new FakeStorage({ turns: [], lastExtractionAt: null, extractionCount: 0 }) as never,
+  );
+  await buffer.retainDeferredTurns("k", [makeTurn("retained")], 10);
+  const outcome = await buffer.addTurnWithOutcome("k", makeTurn("live"));
+  assert.equal(outcome.decision, "extract_batch");
+  assert.deepEqual(
+    outcome.extractionTurns?.map((turn) => turn.content),
+    ["retained", "live"],
+  );
+});
+
+test("#2468 processTurn does not adopt a later getTurns() batch", async () => {
+  const snapshot = [makeTurn("first")];
+  const later = [makeTurn("first"), makeTurn("later")];
+  const queued: BufferTurn[][] = [];
+  const buffer = {
+    addTurnWithOutcome: async () => ({
+      decision: "extract_now" as const,
+      extractionTurns: snapshot,
+    }),
+    getTurns: () => later,
+  };
+  const deps = {
+    buffer,
+    config: parseConfig({}),
+    getStorage: async () => {
+      throw new Error("unexpected getStorage");
+    },
+    bulkImportWriteNamespace: () => "default",
+    get extractionQueueCoordinator(): never {
+      throw new Error("unexpected extractionQueueCoordinator");
+    },
+    heartbeatObserverChains: new Map(),
+    lcmEngine: null,
+    passiveCorrectionDedup: new Set(),
+    passiveCorrectionService: () => {
+      throw new Error("unexpected passiveCorrectionService");
+    },
+    passiveCorrectionTelemetry: {
+      detected: 0,
+      queued: 0,
+      autoApplied: 0,
+      suppressedReasonCounts: {},
+    },
+    queueBufferedExtraction: async (turns: BufferTurn[]) => {
+      queued.push(turns);
+    },
+    resolveMemoryIdOrHandle: () => {
+      throw new Error("unexpected resolveMemoryIdOrHandle");
+    },
+    runExtraction: async () => {
+      throw new Error("unexpected runExtraction");
+    },
+    get sessionObserver(): never {
+      throw new Error("unexpected sessionObserver");
+    },
+    shouldQueueExtraction: () => {
+      throw new Error("unexpected shouldQueueExtraction");
+    },
+    get transcript(): never {
+      throw new Error("unexpected transcript");
+    },
+  } as unknown as TurnIngestionDeps;
+  await new TurnIngestionCoordinator(deps).processTurn("user", "first", "k");
+  assert.equal(queued.length, 1);
+  assert.equal(queued[0]?.length, 1);
+  assert.equal(queued[0]?.[0]?.content, "first");
+});
+test("#2468 concurrent processTurn first trigger keeps its mutation snapshot", async () => {
+  const queued: BufferTurn[][] = [];
+  const buffer = new SmartBuffer(
+    parseConfig({ triggerMode: "every_n", bufferMaxTurns: 1, bufferSaveDebounceMs: 0 }),
+    new FakeStorage({ turns: [], lastExtractionAt: null, extractionCount: 0 }) as never,
+  );
+  const deps = {
+    buffer,
+    config: parseConfig({ triggerMode: "every_n", bufferMaxTurns: 1 }),
+    getStorage: async () => {
+      throw new Error("unexpected getStorage");
+    },
+    bulkImportWriteNamespace: () => "default",
+    get extractionQueueCoordinator(): never {
+      throw new Error("unexpected extractionQueueCoordinator");
+    },
+    heartbeatObserverChains: new Map(),
+    lcmEngine: null,
+    passiveCorrectionDedup: new Set(),
+    passiveCorrectionService: () => {
+      throw new Error("unexpected passiveCorrectionService");
+    },
+    passiveCorrectionTelemetry: {
+      detected: 0,
+      queued: 0,
+      autoApplied: 0,
+      suppressedReasonCounts: {},
+    },
+    queueBufferedExtraction: async (turns: BufferTurn[]) => {
+      queued.push(turns);
+    },
+    resolveMemoryIdOrHandle: () => {
+      throw new Error("unexpected resolveMemoryIdOrHandle");
+    },
+    runExtraction: async () => {
+      throw new Error("unexpected runExtraction");
+    },
+    get sessionObserver(): never {
+      throw new Error("unexpected sessionObserver");
+    },
+    shouldQueueExtraction: () => {
+      throw new Error("unexpected shouldQueueExtraction");
+    },
+    get transcript(): never {
+      throw new Error("unexpected transcript");
+    },
+  } as unknown as TurnIngestionDeps;
+  const coordinator = new TurnIngestionCoordinator(deps);
+  await Promise.all([
+    coordinator.processTurn("user", "alpha", "shared-key"),
+    coordinator.processTurn("user", "beta", "shared-key"),
+  ]);
+  assert.equal(queued.length, 2);
+  assert.ok(
+    queued.some((batch) => batch.length === 1),
+    "the first trigger must extract only its mutation snapshot",
+  );
+});

package/src/orchestration/turn-ingestion.ts

@@ -201,8 +201,9 @@ export class TurnIngestionCoordinator {
         : { decision: await this.deps.buffer.addTurn(bufferKey, turn) };
 
     if (outcome.decision === "keep_buffering") return;
+    if (!outcome.extractionTurns) return;
     await this.deps.queueBufferedExtraction(
-      outcome.extractionTurns ?? this.deps.buffer.getTurns(bufferKey),
+      outcome.extractionTurns,
       "trigger_mode",
       { bufferKey },
     );

package/src/storage/content-hash-index.ts

@@ -110,6 +110,20 @@ export interface ContentHashPathEntry {
  */
 export class ContentHashIndex {
   private hashes: Set<string> = new Set();
+  /**
+   * Fact-ONLY hash partition (issue #2474). `hashes` above is category-blind
+   * (every write category registers there); this subset carries only
+   * `category === "fact"` hashes so fact-only membership checks
+   * (`hasFact*`) can never be satisfied by an unrelated decision/preference
+   * sharing the same normalized body. In-memory only — never serialized to
+   * fact-hashes.txt; `clear()` drops it and every authoritative rebuild
+   * repopulates it via `addFactByHash`. Subset invariant: `factHashes ⊆
+   * hashes` — `addFact*` registers in both, and the removal sites that drop
+   * a shared hash also drop the partition entry (removal stays caller-paired
+   * because shared-index removal is category-blind: archiving a non-fact
+   * must not evict an active fact's fact-only membership).
+   */
+  private factHashes: Set<string> = new Set();
   private dirty = false;
   /** True when load() found the pre-Unicode, markerless on-disk format. */
   private formatMigrationRequired = false;
@@ -290,6 +304,9 @@ export class ContentHashIndex {
     if (this.hashes.size > 0) {
       this.hashes.clear();
     }
+    // The fact-only partition is rebuilt with the shared set on every
+    // authoritative corpus rebuild, so it clears here too (issue #2474).
+    this.factHashes.clear();
     // A full clear is always paired with a plain overwrite save() (rebuild);
     // pending per-hash add/remove deltas are subsumed by writing the rebuilt set.
     // Cancel any armed deferred-reconcile retry (PR #2016) so it cannot fire in
@@ -693,6 +710,43 @@ export class ContentHashIndex {
     this.dirty = true;
   }
 
+  // ── Fact-only partition (issue #2474) ─────────────────────────────────
+  // Folded here from StorageManager's parallel `factOnlyHashes` set, which had
+  // to be kept in lockstep by hand on every add/remove/rebuild site. The
+  // partition lives on the index instance, so every path that mutates the
+  // shared set and needs fact-only truth mutates it in the same breath.
+
+  /** Register fact content in BOTH the shared index and the fact-only partition. */
+  addFact(content: string): void {
+    this.addFactByHash(ContentHashIndex.computeHash(content));
+  }
+
+  /** Register a pre-computed fact hash in both the shared index and the fact-only partition. */
+  addFactByHash(hash: string): void {
+    this.addByHash(hash);
+    this.factHashes.add(hash);
+  }
+
+  /** Fact-ONLY membership: has this content been registered as a fact? */
+  hasFact(content: string): boolean {
+    return this.factHashes.has(ContentHashIndex.computeHash(content));
+  }
+
+  /** Fact-ONLY membership for a pre-computed SHA-256 hash. */
+  hasFactByHash(hash: string): boolean {
+    return this.factHashes.has(hash);
+  }
+
+  /**
+   * Drop fact-only partition membership WITHOUT touching the shared index.
+   * The shared removal stays caller-owned: a shared hash is dropped only when
+   * no other active memory holds it, while fact-only membership follows the
+   * fact's own lifecycle (supersede/archive/promotion).
+   */
+  removeFactByHash(hash: string): void {
+    this.factHashes.delete(hash);
+  }
+
   /**
    * Resolve a pre-computed semantic hash to one deterministic corpus path.
    *

package/src/storage-contract/content-hash.test.ts

@@ -11,7 +11,7 @@
 import assert from "node:assert/strict";
 import test from "node:test";
 
-import { ContentHashIndex } from "../storage.js";
+import { ContentHashIndex, StorageManager } from "../storage.js";
 import { makeStorage } from "./harness.js";
 
 test("content-hash: ContentHashIndex.computeHash is deterministic for the same input", () => {
@@ -158,3 +158,61 @@ test("content-hash: the frontmatter contentHash matches computeHash of the raw c
     await cleanup();
   }
 });
+
+test("content-hash: fact-only membership across write, supersede, and rebuild — no stale true (#2474)", async () => {
+  const { storage, baseDir, cleanup } = await makeStorage();
+  try {
+    const supersededBody = "the staging region retires on friday";
+    const activeBody = "the production region stays";
+
+    // Write: both facts register fact-only membership on the shared index.
+    await storage.writeMemory("fact", supersededBody);
+    await storage.writeMemory("fact", activeBody);
+    assert.equal(
+      await storage.hasFactContentHash(supersededBody),
+      true,
+      "write registers the fact-only hash",
+    );
+    assert.equal(await storage.hasFactContentHash(activeBody), true);
+
+    // Supersede: the frontmatter rewrite runs syncFactHashIndexAfterRewrite →
+    // removeFactContentHashesForMemories, which must drop BOTH the shared hash
+    // and the fact-only partition. A stale partition entry is the PR #2016
+    // stale-true bug class: wearable / explicit-capture / promotion callers
+    // would skip a valid re-write.
+    const mem = (await storage.readAllMemories()).find((m) =>
+      (m.content ?? "").includes(supersededBody),
+    );
+    assert.ok(mem, "the fact to supersede must exist");
+    const ok = await storage.writeMemoryFrontmatter(mem!, { status: "superseded" });
+    assert.equal(ok, true, "frontmatter rewrite must succeed");
+    assert.equal(
+      await storage.hasFactContentHash(supersededBody),
+      false,
+      "no stale hasFactContentHash true after supersede",
+    );
+    assert.equal(
+      await storage.hasFactContentHash(activeBody),
+      true,
+      "the untouched fact keeps its fact-only membership",
+    );
+
+    // Rebuild: a fresh instance repopulates the index from the corpus. The
+    // superseded fact must stay out of the fact-only partition; the active
+    // fact must survive.
+    const reopened = new StorageManager(baseDir);
+    await reopened.ensureDirectories();
+    assert.equal(
+      await reopened.hasFactContentHash(supersededBody),
+      false,
+      "superseded fact stays out of fact-only membership after rebuild",
+    );
+    assert.equal(
+      await reopened.hasFactContentHash(activeBody),
+      true,
+      "active fact survives the rebuild",
+    );
+  } finally {
+    await cleanup();
+  }
+});

package/src/storage-fact-hash-index-defer.test.ts

@@ -308,10 +308,10 @@ test("direct writeMemory('fact') without the flag persists the hash via the LOCK
 });
 
 test("#2016 thread SDzOT: a fact hash added during the authoritative rebuild's corpus scan is not lost", async () => {
-  // The rebuild used to build a fresh `factOnly` set and publish it with
-  // `this.factOnlyHashes = factOnly`, dropping any concurrent in-process
-  // writeMemory that added to the LIVE set during the readAllMemories /
-  // readAllColdMemories awaits. The rebuild now repopulates the live set in place
+  // The rebuild used to build a fresh fact-only set and publish it with a
+  // reassignment, dropping any concurrent in-process writeMemory that added to
+  // the LIVE membership during the readAllMemories / readAllColdMemories
+  // awaits. The fact-only partition on ContentHashIndex repopulates in place
   // (clear + add), so a concurrent add survives publication — exactly as the
   // shared index preserves concurrent adds by mutating its `hashes` set in place.
   await withMemoryDir(async (dir) => {
@@ -320,7 +320,7 @@ test("#2016 thread SDzOT: a fact hash added during the authoritative rebuild's c
     await storage.writeMemory("fact", "alpha established fact", { source: "extraction" });
 
     const s = storage as unknown as {
-      factOnlyHashes: Set<string>;
+      factHashIndex: ContentHashIndex | null;
       factHashIndexAuthoritative: boolean | null;
       readAllColdMemories: () => Promise<unknown[]>;
       ensureFactHashIndexAuthoritative: () => Promise<boolean>;
@@ -329,13 +329,13 @@ test("#2016 thread SDzOT: a fact hash added during the authoritative rebuild's c
     const realCold = s.readAllColdMemories.bind(storage);
     let injected = false;
     // Interpose on the SECOND corpus await inside the rebuild (after clear()):
-    // simulate a concurrent writeMemory adding a fact hash to the live set while
-    // the rebuild is mid-scan.
+    // simulate a concurrent writeMemory adding a fact hash to the live
+    // partition while the rebuild is mid-scan.
     s.readAllColdMemories = async () => {
       const res = await realCold();
       if (!injected) {
         injected = true;
-        s.factOnlyHashes.add(sentinel);
+        s.factHashIndex?.addFactByHash(sentinel);
       }
       return res;
     };
@@ -345,7 +345,7 @@ test("#2016 thread SDzOT: a fact hash added during the authoritative rebuild's c
     assert.equal(await s.ensureFactHashIndexAuthoritative(), true, "rebuild published");
 
     assert.equal(
-      s.factOnlyHashes.has(sentinel),
+      s.factHashIndex?.hasFactByHash(sentinel),
       true,
       "a fact hash added during the rebuild scan must survive publication (lost under reassign)"
     );

package/src/storage.ts

@@ -1867,19 +1867,15 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   private factHashIndexAuthoritative: boolean | null = null;
   private factHashIndexAuthoritativePromise: Promise<boolean> | null = null;
   /**
-   * Fact-ONLY hash membership (PR #2016). The shared `factHashIndex` above is
-   * category-agnostic (the round-15 corpus rebuild + addContentHashDedup index
-   * every category), so it cannot answer a fact-only question. This set carries
-   * only `category === "fact"` hashes and is the sole source for
-   * `hasFactContentHash`, so an over-included non-fact body can never suppress a
-   * real fact candidate. Kept in lockstep with the shared index on EVERY path:
-   * the authoritative corpus rebuild (repopulated in place), the write path
-   * (`writeMemory` / `addActiveFactContentHash`), the storage-owned removal
-   * (`removeFactContentHashesForMemories`), AND the orchestrator's archival /
-   * consolidation removal (`removeContentHashForMemory` ->
-   * `removeFactOnlyHashForMemory`). In-memory only; never persisted.
+   * Fact-ONLY hash membership (PR #2016 / issue #2474) lives on the shared
+   * `ContentHashIndex` fact-only partition (`addFact*` / `hasFact*` /
+   * `removeFactByHash`), not a parallel set here. The partition carries only
+   * `category === "fact"` hashes and is the sole source for
+   * `hasFactContentHash`, so an over-included non-fact body can never
+   * suppress a real fact candidate. In-memory only; never persisted — every
+   * authoritative corpus rebuild repopulates it via
+   * `ContentHashIndex.clear()` + `addFactByHash`.
    */
-  private factOnlyHashes: Set<string> = new Set();
   /** Optional lock/retry tuning for the fact-hash index cross-process lock (PR #2016; tests inject tight budgets). */
   factHashIndexLockOptions: ContentHashIndexLockOptions = {};
   private readonly secureAppendChains = new Map<string, Promise<void>>();
@@ -3328,15 +3324,14 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
    */
   private async rebuildFactHashIndexFromCorpus(factHashIndex: ContentHashIndex): Promise<void> {
     factHashIndex.clear();
-    // Fact-ONLY membership rebuilt in lockstep (PR #2016): hasFactContentHash
-    // reads THIS set, never the category-agnostic shared index below, so an
-    // over-included non-fact body can never satisfy a fact-hash check. Repopulate
-    // the LIVE set in place (clear + add) rather than building a fresh set and
-    // reassigning: a concurrent in-process writeMemory that adds a fact hash
-    // during the corpus-read awaits below would be lost by a publish-time
-    // reassignment (PR #2016 thread SDzOT), exactly as the shared index avoids
-    // by mutating its `hashes` set in place.
-    this.factOnlyHashes.clear();
+    // Fact-ONLY membership is rebuilt in lockstep (PR #2016 / issue #2474):
+    // hasFactContentHash reads the index's fact-only partition, never the
+    // category-agnostic shared set, so an over-included non-fact body can
+    // never satisfy a fact-hash check. `factHashIndex.clear()` above cleared
+    // the partition too, and it repopulates in place below — a concurrent
+    // in-process writeMemory that adds a fact hash during the corpus-read
+    // awaits survives publication (PR #2016 thread SDzOT), exactly as the
+    // shared index preserves by mutating its `hashes` set in place.
     // #1909 review round 14: index the HOT and COLD tiers together. A fact or
     // procedure demoted to cold/ is still active and its content-hash must
     // survive the corpus rebuild, or a restart would drop the hash and let the
@@ -3371,8 +3366,11 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
         continue;
       }
       for (const hash of hashes) {
-        factHashIndex.addByHash(hash);
-        if (memory.frontmatter.category === "fact") this.factOnlyHashes.add(hash);
+        if (memory.frontmatter.category === "fact") {
+          factHashIndex.addFactByHash(hash);
+        } else {
+          factHashIndex.addByHash(hash);
+        }
       }
     }
     if (legacyRecovered > 0) {
@@ -3896,8 +3894,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
           options.contentHashSource !== undefined && options.contentHashSource.length > 0
             ? sanitizeMemoryContent(options.contentHashSource).text
             : sanitized.text;
-        factHashIndex.add(hashText);
-        this.factOnlyHashes.add(ContentHashIndex.computeHash(hashText));
+        factHashIndex.addFact(hashText);
         // Gate only the flush (issue #1909): the `.add(...)` above already set
         // dirty=true. When the caller defers, it owns the batch save
         // (extraction persist -> saveContentHashIndexes()). Single-write callers
@@ -3973,7 +3970,8 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     // upkeep); otherwise the snapshot may be stale, so verify against the durable
     // fact corpus (ground truth) so a lock-contended read never suppresses a fact.
     if (authoritative) {
-      return this.factOnlyHashes.has(hash);
+      const factHashIndex = await this.getFactHashIndex();
+      return factHashIndex.hasFact(sanitized.text);
     }
     return await this.factContentHashPresentInCorpus(hash);
   }
@@ -4002,8 +4000,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     await this.ensureFactHashIndexAuthoritative();
     const factHashIndex = await this.getFactHashIndex();
     for (const hash of hashes) {
-      factHashIndex.addByHash(hash);
-      this.factOnlyHashes.add(hash);
+      factHashIndex.addFactByHash(hash);
     }
     // PR #2016 thread SDzOP: flush through the SAME cross-process locked
     // reconcile the write/batch/rebuild paths use, never the unlocked whole-file
@@ -4089,7 +4086,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     for (const hash of removedHashes) {
       if (!remainingActiveHashes.has(hash)) {
         factHashIndex.removeByHash(hash);
-        this.factOnlyHashes.delete(hash);
+        factHashIndex.removeFactByHash(hash);
       }
     }
     // PR #2016 thread SDzOP: serialize the removal with the per-index lock via
@@ -4102,21 +4099,23 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   }
 
   /**
-   * Remove a memory's fact-ONLY hash membership in lockstep with the shared,
-   * category-agnostic index removal the orchestrator's
-   * `removeContentHashForMemory` performs on archival / semantic consolidation
-   * (PR #2016 threads SDzOP / SDzOR). Without this, `factOnlyHashes` kept a
-   * removed/superseded fact's hash and `hasFactContentHash` returned a stale
-   * `true` until the next corpus rebuild, so wearable / explicit-capture /
-   * promotion callers skipped a valid write. In-memory only — `factOnlyHashes`
-   * is never persisted; it is rebuilt from the corpus. No-op for non-facts.
-   * Matches the shared index's unconditional per-hash removal so the two stay
-   * coherent.
+   * Remove a memory's fact-ONLY partition membership in lockstep with the
+   * shared, category-agnostic index removal the orchestrator's
+   * `removeContentHashForMemory` performs on archival / semantic
+   * consolidation (PR #2016 threads SDzOP / SDzOR). Without this, the
+   * partition kept a removed/superseded fact's hash and `hasFactContentHash`
+   * returned a stale `true` until the next corpus rebuild, so wearable /
+   * explicit-capture / promotion callers skipped a valid write. No-op when
+   * the fact-hash index has not been loaded in this process yet (the
+   * partition is empty then and the corpus rebuild excludes the removed
+   * memory). No-op for non-facts.
    */
   removeFactOnlyHashForMemory(memory: MemoryFile): void {
     if (memory.frontmatter.category !== "fact") return;
+    const factHashIndex = this.factHashIndex;
+    if (!factHashIndex) return;
     for (const hash of this.factContentHashesForRemoval(memory)) {
-      this.factOnlyHashes.delete(hash);
+      factHashIndex.removeFactByHash(hash);
     }
   }