@remnic/core 1.1.31 → 9.3.515

package/src/contradiction/contradiction.test.ts

@@ -90,6 +90,7 @@ function makeResolutionStorage(options: {
   failSupersedeFor?: string;
   failRollbackFor?: string;
   partialSupersedeBeforeFailureFor?: string;
+  onSupersede?: (oldId: string, newId: string) => void;
 } = {}) {
   const memories = new Map<string, MemoryFile>([
     ["mem-a-001", makeMemory("mem-a-001")],
@@ -132,6 +133,7 @@ function makeResolutionStorage(options: {
     },
     async supersedeMemory(oldId: string, newId: string, reason: string) {
       supersedeCalls.push({ oldId, newId, reason });
+      options.onSupersede?.(oldId, newId);
       if (oldId === options.failSupersedeFor) return false;
       const memory = memories.get(oldId);
       if (!memory) return false;
@@ -992,6 +994,50 @@ test("runContradictionScan caps candidates during generation and preserves strat
   }
 });
 
+test("runContradictionScan pairs topic-overlap memories with different entity refs", async () => {
+  const { dir, cleanup } = await makeTempDir();
+  try {
+    const apples = makeMemory("topic-apples");
+    apples.frontmatter.entityRef = "entity:apples";
+    apples.frontmatter.tags = ["pricing"];
+    apples.content = "Apples should be priced at $2 per pound.";
+
+    const oranges = makeMemory("topic-oranges");
+    oranges.frontmatter.entityRef = "entity:oranges";
+    oranges.frontmatter.tags = ["pricing"];
+    oranges.content = "Oranges should be priced at $3 per pound.";
+
+    const storage = makeScanStorage([oranges, apples]);
+    const config = parseConfig({
+      memoryDir: dir,
+      contradictionScan: {
+        enabled: true,
+        maxPairsPerRun: 10,
+        topicOverlapFloor: 0.5,
+        similarityFloor: 0.9,
+      },
+    });
+
+    const result = await runContradictionScan({
+      storage,
+      config,
+      memoryDir: dir,
+      localLlm: null,
+      fallbackLlm: null,
+    });
+
+    assert.equal(result.candidates, 1);
+    assert.equal(result.judged, 1);
+    assert.equal(result.queued, 1);
+
+    const queued = listPairs(dir, { filter: "all" }).pairs;
+    assert.equal(queued.length, 1);
+    assert.deepEqual([...queued[0]!.memoryIds].sort(), ["topic-apples", "topic-oranges"]);
+  } finally {
+    await cleanup();
+  }
+});
+
 test("runContradictionScan resolves default namespace scans through namespace storage resolver", async () => {
   const { dir, cleanup } = await makeTempDir();
   try {
@@ -1262,6 +1308,83 @@ test("runContradictionScan rejudges cooling pairs when referenced memory content
   }
 });
 
+test("runContradictionScan finds cooling pairs beyond the review list page limit", async () => {
+  const { dir, cleanup } = await makeTempDir();
+  try {
+    const defaultA = makeMemory("default-a");
+    defaultA.frontmatter.entityRef = "entity:shared";
+    const defaultB = makeMemory("default-b");
+    defaultB.frontmatter.entityRef = "entity:shared";
+    const targetPairId = computePairId("default-a", "default-b");
+    const reviewPath = path.join(dir, ".review", "contradictions");
+    fs.mkdirSync(reviewPath, { recursive: true });
+
+    const dummyEntries: string[] = [];
+    for (let i = 0; i < 10_000; i++) {
+      const memoryIds: [string, string] = [`dummy-a-${i}`, `dummy-b-${i}`];
+      const pairId = computePairId(memoryIds[0], memoryIds[1]);
+      dummyEntries.push(`${pairId}.json`);
+      fs.writeFileSync(
+        path.join(reviewPath, `${pairId}.json`),
+        JSON.stringify({
+          ...makePair({ memoryIds, verdict: "independent", lastReviewedAt: new Date().toISOString() }),
+          pairId,
+        }),
+        "utf-8",
+      );
+    }
+
+    const targetPair: ContradictionPair = {
+      ...makePair({
+        memoryIds: ["default-a", "default-b"],
+        verdict: "independent",
+        lastReviewedAt: new Date().toISOString(),
+      }),
+      pairId: targetPairId,
+    };
+    fs.writeFileSync(path.join(reviewPath, `${targetPairId}.json`), JSON.stringify(targetPair), "utf-8");
+
+    const originalReaddirSync = fs.readdirSync;
+    const fsMutable = fs as unknown as { readdirSync: typeof fs.readdirSync };
+    try {
+      fsMutable.readdirSync = ((target: fs.PathLike, options?: BufferEncoding | { encoding?: BufferEncoding | null; withFileTypes?: false }) => {
+        if (String(target) === reviewPath) {
+          return [...dummyEntries, `${targetPairId}.json`];
+        }
+        return originalReaddirSync(target, options as never);
+      }) as typeof fs.readdirSync;
+
+      const storage = makeScanStorage([defaultA, defaultB]);
+      const config = parseConfig({
+        memoryDir: dir,
+        contradictionScan: {
+          enabled: true,
+          cooldownDays: 14,
+          maxPairsPerRun: 10,
+          topicOverlapFloor: 0,
+          similarityFloor: 0,
+        },
+      });
+
+      const result = await runContradictionScan({
+        storage,
+        config,
+        memoryDir: dir,
+        localLlm: null,
+        fallbackLlm: null,
+      });
+
+      assert.equal(result.cooledDown, 1);
+      assert.equal(result.judged, 0);
+      assert.equal(result.queued, 0);
+    } finally {
+      fsMutable.readdirSync = originalReaddirSync;
+    }
+  } finally {
+    await cleanup();
+  }
+});
+
 test("runContradictionScan rejects namespace scans without an access-checked storage resolver", async () => {
   const { dir, cleanup } = await makeTempDir();
   try {
@@ -1639,7 +1762,8 @@ test("executeResolution merge can create and verify a merged memory from content
     assert.ok(mergedId);
     assert.deepEqual(result.affectedIds, ["mem-a-001", "mem-b-002"]);
     assert.equal(storage.memories.get(mergedId)?.content, "merged canonical fact");
-    assert.deepEqual(storage.memories.get(mergedId)?.frontmatter.derived_from, ["mem-a-001", "mem-b-002"]);
+    assert.deepEqual(storage.memories.get(mergedId)?.frontmatter.lineage, ["mem-a-001", "mem-b-002"]);
+    assert.equal(storage.memories.get(mergedId)?.frontmatter.derived_from, undefined);
     assert.equal(storage.memories.get(mergedId)?.frontmatter.derived_via, "merge");
     assert.equal(readPair(dir, written.pairId)?.resolution, "merge");
   } finally {
@@ -1647,6 +1771,29 @@ test("executeResolution merge can create and verify a merged memory from content
   }
 });
 
+test("executeResolution rolls back memory changes when pair resolution persistence fails", async () => {
+  const { dir, cleanup } = await makeTempDir();
+  try {
+    const written = writePair(dir, makePair());
+    const pairFile = path.join(dir, ".review", "contradictions", `${written.pairId}.json`);
+    const storage = makeResolutionStorage({
+      onSupersede: () => {
+        fs.rmSync(pairFile, { force: true });
+      },
+    });
+
+    const result = await executeResolution(dir, storage, written.pairId, "keep-a");
+
+    assert.match(result.message, /Resolution persistence failed; rolled back memory changes/);
+    assert.deepEqual(result.affectedIds, []);
+    assert.equal(storage.memories.get("mem-b-002")?.frontmatter.status, undefined);
+    assert.equal(storage.memories.get("mem-b-002")?.frontmatter.supersededBy, undefined);
+    assert.equal(readPair(dir, written.pairId), null);
+  } finally {
+    await cleanup();
+  }
+});
+
 test("executeResolution merge rolls back the first supersession when the second fails", async () => {
   const { dir, cleanup } = await makeTempDir();
   try {
@@ -2068,6 +2215,75 @@ test("expired needs-more-context deferral returns to unresolved and can be refre
   }
 });
 
+test("writePair refreshes active needs-more-context deferrals when memory content changed", async () => {
+  const { dir, cleanup } = await makeTempDir();
+  try {
+    const staleAHash = computeMemoryContentHash("old content for mem-a-001", "fact");
+    const currentAHash = computeMemoryContentHash("new content for mem-a-001", "fact");
+    const bHash = computeMemoryContentHash("content for mem-b-002", "fact");
+    const written = writePair(dir, makePair({
+      confidence: 0.95,
+      memoryContentHashes: {
+        "mem-a-001": staleAHash,
+        "mem-b-002": bHash,
+      },
+    }));
+    const deferred = resolvePair(dir, written.pairId, "needs-more-context");
+    assert.ok(deferred?.deferredUntil);
+
+    const refreshed = writePair(dir, makePair({
+      confidence: 0.5,
+      verdict: "duplicates",
+      rationale: "Fresh judge result after source memory changed",
+      memoryContentHashes: {
+        "mem-a-001": currentAHash,
+        "mem-b-002": bHash,
+      },
+    }));
+
+    assert.equal(refreshed.pairId, written.pairId);
+    assert.equal(refreshed.verdict, "duplicates");
+    assert.equal(refreshed.confidence, 0.5);
+    assert.equal(refreshed.resolution, undefined);
+    assert.equal(refreshed.deferredUntil, undefined);
+  } finally {
+    await cleanup();
+  }
+});
+
+test("writePair refreshes unresolved higher-confidence entries when memory content changed", async () => {
+  const { dir, cleanup } = await makeTempDir();
+  try {
+    const staleAHash = computeMemoryContentHash("old content for mem-a-001", "fact");
+    const currentAHash = computeMemoryContentHash("new content for mem-a-001", "fact");
+    const bHash = computeMemoryContentHash("content for mem-b-002", "fact");
+    const written = writePair(dir, makePair({
+      confidence: 0.95,
+      memoryContentHashes: {
+        "mem-a-001": staleAHash,
+        "mem-b-002": bHash,
+      },
+    }));
+
+    const refreshed = writePair(dir, makePair({
+      confidence: 0.5,
+      verdict: "duplicates",
+      rationale: "Fresh lower-confidence result after source memory changed",
+      memoryContentHashes: {
+        "mem-a-001": currentAHash,
+        "mem-b-002": bHash,
+      },
+    }));
+
+    assert.equal(refreshed.pairId, written.pairId);
+    assert.equal(refreshed.verdict, "duplicates");
+    assert.equal(refreshed.confidence, 0.5);
+    assert.equal(refreshed.rationale, "Fresh lower-confidence result after source memory changed");
+  } finally {
+    await cleanup();
+  }
+});
+
 test("legacy needs-more-context resolutions honor cooldown before returning to unresolved", async () => {
   const { dir, cleanup } = await makeTempDir();
   try {
@@ -2203,6 +2419,26 @@ test("contentHash differs for different content", () => {
   assert.notEqual(_contentHash(a), _contentHash(b));
 });
 
+test("contentHash does not collide when text and category contain delimiters", () => {
+  const a = {
+    memoryIdA: "1",
+    memoryIdB: "2",
+    textA: "alpha|fact",
+    categoryA: "beta",
+    textB: "gamma",
+    categoryB: "delta",
+  };
+  const b = {
+    memoryIdA: "1",
+    memoryIdB: "2",
+    textA: "alpha",
+    categoryA: "fact|beta",
+    textB: "gamma",
+    categoryB: "delta",
+  };
+  assert.notEqual(_contentHash(a), _contentHash(b));
+});
+
 // ── JSON parse safety (rule 18) ────────────────────────────────────────────────
 
 test("readPair returns null for invalid JSON", async () => {

package/src/contradiction/resolution.ts

@@ -80,6 +80,7 @@ export async function executeResolution(
   const affectedIds: string[] = [];
   let message = "";
   let supersedeFailed = false;
+  let rollbackAfterResolveFailure: (() => Promise<boolean>) | null = null;
 
   switch (verb) {
     case "keep-a": {
@@ -93,7 +94,12 @@ export async function executeResolution(
       const ok = sourceB
         ? await supersedeSafe(resolutionStorage, idB, idA, "contradiction-resolution:keep-a")
         : false;
-      if (ok) { affectedIds.push(idB); message = `Kept ${idA}, superseded ${idB}`; }
+      if (ok) {
+        affectedIds.push(idB);
+        rollbackAfterResolveFailure = async () =>
+          restoreMemorySnapshot(resolutionStorage, sourceB!, "contradiction-resolution:keep-a-rollback");
+        message = `Kept ${idA}, superseded ${idB}`;
+      }
       else {
         supersedeFailed = true;
         const rolledBack = sourceB
@@ -116,7 +122,12 @@ export async function executeResolution(
       const ok = sourceA
         ? await supersedeSafe(resolutionStorage, idA, idB, "contradiction-resolution:keep-b")
         : false;
-      if (ok) { affectedIds.push(idA); message = `Kept ${idB}, superseded ${idA}`; }
+      if (ok) {
+        affectedIds.push(idA);
+        rollbackAfterResolveFailure = async () =>
+          restoreMemorySnapshot(resolutionStorage, sourceA!, "contradiction-resolution:keep-b-rollback");
+        message = `Kept ${idB}, superseded ${idA}`;
+      }
       else {
         supersedeFailed = true;
         const rolledBack = sourceA
@@ -167,6 +178,14 @@ export async function executeResolution(
       }
 
       affectedIds.push(idA, idB);
+      rollbackAfterResolveFailure = async () => {
+        const rolledBackA = await restoreMemorySnapshot(resolutionStorage, replacement.sourceA);
+        const rolledBackB = await restoreMemorySnapshot(resolutionStorage, replacement.sourceB);
+        if (rolledBackA && rolledBackB) {
+          await cleanupCreatedReplacement(resolutionStorage, replacement);
+        }
+        return rolledBackA && rolledBackB;
+      };
       message = `Both memories superseded by merged ${replacement.mergedId}`;
       break;
     }
@@ -181,7 +200,28 @@ export async function executeResolution(
   }
 
   if (!supersedeFailed) {
-    resolvePair(memoryDir, pairId, verb);
+    let resolved = false;
+    try {
+      resolved = resolvePair(memoryDir, pairId, verb) !== null;
+    } catch (err) {
+      log.warn(
+        "[contradiction-resolution] failed to persist pair=%s verb=%s: %s",
+        pairId,
+        verb,
+        err instanceof Error ? err.message : err,
+      );
+    }
+    if (!resolved) {
+      if (rollbackAfterResolveFailure) {
+        const rolledBack = await rollbackAfterResolveFailure();
+        affectedIds.length = 0;
+        message = rolledBack
+          ? `Resolution persistence failed; rolled back memory changes and did not resolve ${pairId}`
+          : `Resolution persistence failed; rollback incomplete and pair is not resolved`;
+      } else {
+        message = `Resolution persistence failed; pair is not resolved`;
+      }
+    }
   }
   log.info("[contradiction-resolution] pair=%s verb=%s affected=%d", pairId, verb, affectedIds.length);
   return { pairId, verb, affectedIds, message };
@@ -249,7 +289,6 @@ async function prepareMergeReplacement(
       tags: ["contradiction-resolution", "merge"],
       source: "contradiction-resolution",
       lineage: [idA, idB],
-      derivedFrom: [idA, idB],
       derivedVia: "merge",
     });
   } catch (err) {

package/src/conversation-index/backend.ts

@@ -77,7 +77,10 @@ export interface ConversationIndexBackend {
   readonly kind: "qmd" | "faiss";
   initialize(): Promise<ConversationIndexBackendInitResult>;
   search(query: string, maxResults: number): Promise<ConversationSearchResult[]>;
-  update(chunks: ConversationChunk[], options: { embed: boolean }): Promise<{ embedded: boolean }>;
+  update(
+    chunks: ConversationChunk[],
+    options: { embed: boolean; retentionCutoffMs?: number },
+  ): Promise<{ embedded: boolean }>;
   rebuild(chunks: ConversationChunk[], options: { embed: boolean }): Promise<{ embedded: boolean; rebuilt: boolean }>;
   health(): Promise<ConversationIndexBackendHealth>;
   inspect(): Promise<ConversationIndexBackendInspection>;
@@ -160,7 +163,7 @@ function createQmdBackend(
       if (!qmd || !qmd.isAvailable()) return [];
       return searchConversationIndex(qmd, query, maxResults);
     },
-    async update(_chunks: ConversationChunk[], options: { embed: boolean }) {
+    async update(_chunks: ConversationChunk[], options: { embed: boolean; retentionCutoffMs?: number }) {
       const qmd = getQmd();
       if (!qmd || !qmd.isAvailable()) return { embedded: false };
       await qmd.update();
@@ -247,13 +250,18 @@ function createFaissBackend(
     async search(query: string, maxResults: number) {
       return searchConversationIndexFaissFailOpen(getFaiss(), query, maxResults);
     },
-    async update(chunks: ConversationChunk[], _options: { embed: boolean }) {
-      await upsertConversationChunksFailOpen(getFaiss(), chunks);
+    async update(chunks: ConversationChunk[], options: { embed: boolean; retentionCutoffMs?: number }) {
+      await upsertConversationChunksFailOpen(getFaiss(), chunks, {
+        retentionCutoffMs: options.retentionCutoffMs,
+      });
       return { embedded: false };
     },
     async rebuild(chunks: ConversationChunk[], _options: { embed: boolean }) {
       const result = await rebuildConversationChunksFailOpen(getFaiss(), chunks);
-      return { embedded: false, rebuilt: result.skipped !== true };
+      return {
+        embedded: false,
+        rebuilt: result.skipped !== true && result.rebuilt === chunks.length,
+      };
     },
     async health() {
       const faiss = await failOpenFaissHealth(getFaiss());

package/src/conversation-index/cleanup.ts

@@ -1,4 +1,4 @@
-import { readdir, rm } from "node:fs/promises";
+import { lstat, readdir, realpath, rm } from "node:fs/promises";
 import path from "node:path";
 import { log } from "../logger.js";
 
@@ -19,10 +19,11 @@ export async function cleanupConversationChunks(
   const cutoffMs = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
 
   try {
+    const root = await assertCleanupRoot(rootDir);
     const sessions = await readdir(rootDir, { withFileTypes: true });
     for (const s of sessions) {
       if (!s.isDirectory()) continue;
-      const sessionDir = path.join(rootDir, s.name);
+      const sessionDir = await assertCleanupChild(root, path.join(rootDir, s.name));
       const dayDirs = await readdir(sessionDir, { withFileTypes: true });
 
       for (const d of dayDirs) {
@@ -32,7 +33,7 @@ export async function cleanupConversationChunks(
         const dayMs = new Date(d.name + "T00:00:00.000Z").getTime();
         if (!Number.isFinite(dayMs)) continue;
         if (dayMs < cutoffMs) {
-          await rm(path.join(sessionDir, d.name), { recursive: true, force: true });
+          await rm(await assertCleanupChild(root, path.join(sessionDir, d.name)), { recursive: true, force: true });
         }
       }
 
@@ -40,7 +41,7 @@ export async function cleanupConversationChunks(
       try {
         const remaining = await readdir(sessionDir);
         if (remaining.length === 0) {
-          await rm(sessionDir, { recursive: true, force: true });
+          await rm(await assertCleanupChild(root, sessionDir), { recursive: true, force: true });
         }
       } catch {
         // ignore
@@ -51,3 +52,23 @@ export async function cleanupConversationChunks(
   }
 }
 
+async function assertCleanupRoot(rootDir: string): Promise<string> {
+  const stat = await lstat(rootDir);
+  if (stat.isSymbolicLink()) {
+    throw new Error("conversation chunk cleanup root must not be a symlink");
+  }
+  return realpath(rootDir);
+}
+
+async function assertCleanupChild(rootReal: string, candidate: string): Promise<string> {
+  const stat = await lstat(candidate);
+  if (stat.isSymbolicLink()) {
+    throw new Error("conversation chunk cleanup path must not contain symlinks");
+  }
+  const real = await realpath(candidate);
+  const relative = path.relative(rootReal, real);
+  if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) {
+    return real;
+  }
+  throw new Error("conversation chunk cleanup path escapes index root");
+}

package/src/conversation-index/faiss-adapter.ts

@@ -124,9 +124,13 @@ export class FaissConversationIndexAdapter {
   private readonly pythonBin: string;
   private readonly scriptPath: string;
   private readonly indexPath: string;
+  private readonly maxBatchSize: number;
   private readonly spawnFn: typeof launchProcess;
 
   constructor(private readonly config: FaissAdapterConfig) {
+    if (!Number.isInteger(config.maxBatchSize) || config.maxBatchSize <= 0) {
+      throw new RangeError("FAISS maxBatchSize must be a positive integer");
+    }
     this.pythonBin = config.pythonBin && config.pythonBin.trim().length > 0 ? config.pythonBin.trim() : "python3";
     this.scriptPath = config.scriptPath && config.scriptPath.trim().length > 0
       ? config.scriptPath.trim()
@@ -134,14 +138,17 @@ export class FaissConversationIndexAdapter {
     this.indexPath = path.isAbsolute(config.indexDir)
       ? config.indexDir
       : path.join(config.memoryDir, config.indexDir);
+    this.maxBatchSize = config.maxBatchSize;
     this.spawnFn = config.spawnFn ?? launchProcess;
   }
 
-  async upsertChunks(chunks: ConversationChunk[]): Promise<number> {
-    if (this.config.maxBatchSize <= 0) return 0;
+  async upsertChunks(
+    chunks: ConversationChunk[],
+    options: { retentionCutoffMs?: number } = {},
+  ): Promise<number> {
     let totalUpserted = 0;
-    for (let offset = 0; offset < chunks.length; offset += this.config.maxBatchSize) {
-      const batch = chunks.slice(offset, offset + this.config.maxBatchSize);
+    for (let offset = 0; offset < chunks.length; offset += this.maxBatchSize) {
+      const batch = chunks.slice(offset, offset + this.maxBatchSize);
       if (batch.length === 0) continue;
       const payload = {
         modelId: this.config.modelId,
@@ -153,6 +160,10 @@ export class FaissConversationIndexAdapter {
           startTs: chunk.startTs,
           endTs: chunk.endTs,
         })),
+        ...(typeof options.retentionCutoffMs === "number" &&
+        Number.isFinite(options.retentionCutoffMs)
+          ? { retentionCutoffMs: options.retentionCutoffMs }
+          : {}),
       };
       const result = await this.runCommand("upsert", payload, this.config.upsertTimeoutMs);
       const upserted = result.upserted;
@@ -237,13 +248,10 @@ export class FaissConversationIndexAdapter {
   }
 
   async rebuildChunks(chunks: ConversationChunk[]): Promise<number> {
-    if (this.config.maxBatchSize <= 0) return 0;
-
-    const firstBatch = chunks.slice(0, this.config.maxBatchSize);
     const rebuildPayload = {
       modelId: this.config.modelId,
       indexPath: this.indexPath,
-      chunks: firstBatch.map((chunk) => ({
+      chunks: chunks.map((chunk) => ({
         id: chunk.id,
         sessionKey: chunk.sessionKey,
         text: chunk.text,
@@ -256,13 +264,14 @@ export class FaissConversationIndexAdapter {
     if (typeof rebuilt !== "number" || !Number.isFinite(rebuilt)) {
       throw new FaissAdapterError("FAISS sidecar produced malformed rebuild response", "malformed_output");
     }
-
-    const rebuildCount = Math.max(0, Math.floor(rebuilt));
-    const remaining = chunks.slice(firstBatch.length);
-    if (remaining.length === 0) return rebuildCount;
-
-    const upserted = await this.upsertChunks(remaining);
-    return rebuildCount + upserted;
+    const rebuildCount = Math.floor(rebuilt);
+    if (rebuildCount !== chunks.length) {
+      throw new FaissAdapterError(
+        `FAISS sidecar rebuilt ${rebuildCount} chunk(s), expected ${chunks.length}`,
+        "malformed_output",
+      );
+    }
+    return rebuildCount;
   }
 
   private async runCommand(command: SidecarCommand, payload: object, timeoutMs: number): Promise<SidecarResult> {