@remnic/core 1.1.12 → 1.1.14

package/src/contradiction/contradiction-scan.ts

@@ -0,0 +1,504 @@
+/**
+ * Contradiction Scan — pair generator + scan driver (issue #520).
+ *
+ * Nightly cron that pairs semantically-similar active memories within
+ * the same namespace, sends candidate pairs to the LLM-as-judge
+ * contradiction classifier, and drops contradicting pairs into a
+ * review queue for user resolution.
+ *
+ * Pair generation (cheap pre-filter):
+ *   1. Find candidate peers via embedding cosine >= similarityFloor.
+ *   2. Restrict to pairs sharing at least one entityRef OR overlapping
+ *      topic tokens >= topicOverlapFloor.
+ *   3. Skip pairs already judged independent/both-valid within cooldown.
+ *   4. Cap per-run work at maxPairsPerRun.
+ */
+
+import type { StorageManager } from "../storage.js";
+import type { PluginConfig, MemoryFile, MemoryStatus } from "../types.js";
+import type { SemanticDedupLookup } from "../dedup/semantic.js";
+import { log } from "../logger.js";
+import { judgeContradictionPairs, type ContradictionJudgeInput } from "./contradiction-judge.js";
+import {
+  writePairs,
+  listPairs,
+  isCoolingDown,
+  memoryHashesChanged,
+  computeMemoryContentHash,
+  computePairId,
+  migrateUnscopedPairsToNamespace,
+  type ContradictionPair,
+} from "./contradiction-review.js";
+import type { LocalLlmClient } from "../local-llm.js";
+import type { FallbackLlmClient } from "../fallback-llm.js";
+
+// ── Types ──────────────────────────────────────────────────────────────────────
+
+/** The set of statuses that represent "live" memories eligible for scanning. */
+export const ACTIVE_STATUSES: Set<MemoryStatus> = new Set(["active"]);
+
+/** High-value taxonomy buckets to scan. */
+const SCAN_CATEGORIES = new Set([
+  "decision",
+  "principle",
+  "rule",
+  "entity",
+  "fact",
+  "preference",
+]);
+
+export interface ScanResult {
+  /** Total active memories scanned. */
+  scanned: number;
+  /** Candidate pairs generated by the pre-filter. */
+  candidates: number;
+  /** Pairs sent to judge. */
+  judged: number;
+  /** Pairs written to review queue. */
+  queued: number;
+  /** Pairs skipped due to cooldown. */
+  cooledDown: number;
+  /** Total wall-clock time in ms. */
+  elapsedMs: number;
+}
+
+export interface ScanStorageResolution {
+  storage: StorageManager;
+  namespace?: string;
+}
+
+export interface ScanDependencies {
+  storage: StorageManager;
+  config: PluginConfig;
+  memoryDir: string;
+  /** Pre-built embedding lookup. When provided, used as-is for Strategy 3. */
+  embeddingLookup?: SemanticDedupLookup;
+  /**
+   * Factory to build a namespace-scoped embedding lookup.
+   * When provided, takes precedence over `embeddingLookup`.
+   * The scan driver passes its own `storage` so the lookup queries the correct index.
+   */
+  embeddingLookupFactory?: (storage: StorageManager) => SemanticDedupLookup | undefined;
+  /** Resolver for namespace-scoped storage. Required for scans when namespaces are enabled. */
+  storageForNamespace?: (
+    namespace: string | undefined,
+  ) => StorageManager | ScanStorageResolution | Promise<StorageManager | ScanStorageResolution>;
+  localLlm: LocalLlmClient | null;
+  fallbackLlm: FallbackLlmClient | null;
+  namespace?: string;
+}
+
+// ── Main scan driver ───────────────────────────────────────────────────────────
+
+/**
+ * Run a contradiction scan over the memory corpus.
+ *
+ * This is the entry point called by the nightly cron and by the CLI.
+ */
+export async function runContradictionScan(deps: ScanDependencies): Promise<ScanResult> {
+  const startTime = Date.now();
+  const { config, memoryDir, embeddingLookup, embeddingLookupFactory, localLlm, fallbackLlm } = deps;
+  const requestedNamespace = normalizeScanNamespace(deps.namespace);
+  const scanConfig = config.contradictionScan;
+
+  if (!scanConfig.enabled) {
+    log.info("[contradiction-scan] disabled by config");
+    return { scanned: 0, candidates: 0, judged: 0, queued: 0, cooledDown: 0, elapsedMs: 0 };
+  }
+
+  const { storage: scanStorage, namespace } = await resolveScanStorage(deps, requestedNamespace);
+  if (config.namespacesEnabled && namespace !== undefined && isDefaultNamespaceScan(config, requestedNamespace, namespace)) {
+    const migrated = migrateUnscopedPairsToNamespace(memoryDir, namespace, { cooldownDays: scanConfig.cooldownDays });
+    if (migrated > 0) {
+      log.info("[contradiction-scan] migrated %d legacy unscoped review pairs to namespace %s", migrated, namespace);
+    }
+  }
+
+  // Prefer the factory (which uses the scan's own storage for correct namespace scoping)
+  // over a pre-built lookup (which may use default-namespace storage).
+  const scopedEmbeddingLookup = embeddingLookupFactory
+    ? embeddingLookupFactory(scanStorage)
+    : embeddingLookup;
+
+  // 1. Load active memories in scan categories
+  const memories = await loadEligibleMemories(scanStorage);
+  log.info("[contradiction-scan] loaded %d eligible memories", memories.length);
+
+  if (memories.length < 2) {
+    return { scanned: memories.length, candidates: 0, judged: 0, queued: 0, cooledDown: 0, elapsedMs: Date.now() - startTime };
+  }
+
+  // 2. Load existing review pairs for cooldown checking
+  const existingPairs = listPairs(memoryDir, { filter: "all", namespace, limit: 10000 }).pairs;
+  const existingMap = new Map<string, ContradictionPair>();
+  for (const p of existingPairs) {
+    existingMap.set(p.pairId, p);
+  }
+
+  // 3. Generate candidate pairs
+  const candidates = await generatePairs(memories, existingMap, scanConfig, namespace, scopedEmbeddingLookup);
+  const cooledDown = candidates.skipped;
+  log.info("[contradiction-scan] generated %d candidates (%d cooled down)", candidates.pairs.length, cooledDown);
+
+  if (candidates.pairs.length === 0) {
+    return {
+      scanned: memories.length,
+      candidates: 0,
+      judged: 0,
+      queued: 0,
+      cooledDown,
+      elapsedMs: Date.now() - startTime,
+    };
+  }
+
+  // 4. Build judge inputs. Candidate generation already preserves strategy
+  // priority and caps work at maxPairsPerRun.
+  const judgeInputs: ContradictionJudgeInput[] = candidates.pairs.map((pair) => ({
+    memoryIdA: pair.idA,
+    memoryIdB: pair.idB,
+    textA: pair.textA,
+    textB: pair.textB,
+    categoryA: pair.categoryA,
+    categoryB: pair.categoryB,
+  }));
+
+  // 6. Send to judge (per-scan cache avoids module-level singleton leak)
+  const scanCache = new Map<string, import("./contradiction-judge.js").ContradictionJudgeResult>();
+  const judgeResult = await judgeContradictionPairs(judgeInputs, config, localLlm, fallbackLlm, scanCache);
+  log.info("[contradiction-scan] judge completed: %d judged, %d cached in %dms", judgeResult.judged, judgeResult.cached, judgeResult.elapsed);
+
+  // 7. Write to review queue
+  const queueEntries: Array<Omit<ContradictionPair, "pairId"> & { memoryIds: [string, string] }> = [];
+  const currentMemoryHashes = memoryContentHashMap(memories);
+  for (const [key, result] of judgeResult.results) {
+    queueEntries.push({
+      memoryIds: [result.memoryIdA, result.memoryIdB],
+      verdict: result.verdict,
+      rationale: result.rationale,
+      confidence: result.confidence,
+      detectedAt: new Date().toISOString(),
+      // Set lastReviewedAt for non-actionable verdicts so cooldown prevents re-judging
+      lastReviewedAt: result.verdict === "independent" ? new Date().toISOString() : undefined,
+      memoryContentHashes: pairMemoryContentHashes(result.memoryIdA, result.memoryIdB, currentMemoryHashes),
+      namespace,
+    });
+  }
+
+  const written = writePairs(memoryDir, queueEntries, { cooldownDays: scanConfig.cooldownDays });
+  const elapsed = Date.now() - startTime;
+  log.info("[contradiction-scan] complete: %d queued in %dms", written.length, elapsed);
+
+  return {
+    scanned: memories.length,
+    candidates: candidates.pairs.length,
+    judged: judgeResult.judged,
+    queued: written.length,
+    cooledDown,
+    elapsedMs: elapsed,
+  };
+}
+
+// ── Pair generation ────────────────────────────────────────────────────────────
+
+interface CandidatePair {
+  idA: string;
+  idB: string;
+  textA: string;
+  textB: string;
+  categoryA?: string;
+  categoryB?: string;
+}
+
+interface PairGenResult {
+  pairs: CandidatePair[];
+  skipped: number;
+}
+
+async function generatePairs(
+  memories: MemoryFile[],
+  existingPairs: Map<string, ContradictionPair>,
+  scanConfig: PluginConfig["contradictionScan"],
+  namespace: string | undefined,
+  embeddingLookup?: SemanticDedupLookup,
+): Promise<PairGenResult> {
+  const pairs: CandidatePair[] = [];
+  let skipped = 0;
+  const seen = new Set<string>();
+  const maxPairs = Math.max(0, Math.floor(scanConfig.maxPairsPerRun));
+  if (maxPairs === 0) return { pairs, skipped };
+
+  const orderedMemories = [...memories].sort(compareMemoryId);
+  const currentMemoryHashes = memoryContentHashMap(orderedMemories);
+
+  const pushCandidate = (pair: CandidatePair): void => {
+    if (pairs.length < maxPairs) pairs.push(pair);
+  };
+
+  const hasCapacity = (): boolean => pairs.length < maxPairs;
+  const isSkippedByCooldown = (existing: ContradictionPair | undefined): boolean => {
+    if (!existing || !isCoolingDown(existing, scanConfig.cooldownDays)) return false;
+    return !memoryHashesChanged("", existing, (memoryId) => currentMemoryHashes.get(memoryId) ?? null);
+  };
+
+  // Build index by entityRef for fast lookup
+  const byEntity = new Map<string, MemoryFile[]>();
+  for (const mem of orderedMemories) {
+    const entity = mem.frontmatter.entityRef;
+    if (entity) {
+      const existing = byEntity.get(entity) ?? [];
+      existing.push(mem);
+      byEntity.set(entity, existing);
+    }
+  }
+
+  // Strategy 1: Entity-ref based pairing (high precision)
+  entityPairs:
+  for (const entity of [...byEntity.keys()].sort()) {
+    const group = byEntity.get(entity) ?? [];
+    if (group.length < 2) continue;
+    for (let i = 0; i < group.length; i++) {
+      for (let j = i + 1; j < group.length; j++) {
+        if (!hasCapacity()) break entityPairs;
+        const a = group[i];
+        const b = group[j];
+        const pairId = computePairId(a.frontmatter.id!, b.frontmatter.id!, namespace);
+
+        if (seen.has(pairId)) continue;
+        seen.add(pairId);
+
+        // Check cooldown
+        const existing = existingPairs.get(pairId);
+        if (isSkippedByCooldown(existing)) {
+          skipped++;
+          continue;
+        }
+
+        pushCandidate({
+          idA: a.frontmatter.id!,
+          idB: b.frontmatter.id!,
+          textA: a.content,
+          textB: b.content,
+          categoryA: a.frontmatter.category as string | undefined,
+          categoryB: b.frontmatter.category as string | undefined,
+        });
+      }
+    }
+  }
+
+  // Strategy 2: Tag/topic overlap for memories without shared entityRef
+  const noEntity = orderedMemories.filter((m) => !m.frontmatter.entityRef);
+  topicPairs:
+  for (let i = 0; i < noEntity.length; i++) {
+    for (let j = i + 1; j < noEntity.length; j++) {
+      if (!hasCapacity()) break topicPairs;
+      const a = noEntity[i];
+      const b = noEntity[j];
+      const overlap = jaccardOverlap(
+        (a.frontmatter.tags as string[]) ?? [],
+        (b.frontmatter.tags as string[]) ?? [],
+      );
+
+      if (overlap < scanConfig.topicOverlapFloor) continue;
+
+      const pairId = computePairId(a.frontmatter.id!, b.frontmatter.id!, namespace);
+      if (seen.has(pairId)) continue;
+      seen.add(pairId);
+
+      const existing = existingPairs.get(pairId);
+      if (isSkippedByCooldown(existing)) {
+        skipped++;
+        continue;
+      }
+
+      pushCandidate({
+        idA: a.frontmatter.id!,
+        idB: b.frontmatter.id!,
+        textA: a.content,
+        textB: b.content,
+        categoryA: a.frontmatter.category as string | undefined,
+        categoryB: b.frontmatter.category as string | undefined,
+      });
+    }
+  }
+
+  // Strategy 3: Embedding cosine similarity (enforces similarityFloor config)
+  if (embeddingLookup && hasCapacity()) {
+    const memoryById = new Map(orderedMemories.map((m) => [m.frontmatter.id!, m]));
+    embeddingPairs:
+    for (const mem of orderedMemories) {
+      if (!hasCapacity()) break;
+      const id = mem.frontmatter.id!;
+      try {
+        const hits = (await embeddingLookup(mem.content, 20)).sort((a, b) =>
+          b.score - a.score || a.id.localeCompare(b.id),
+        );
+        for (const hit of hits) {
+          if (!hasCapacity()) break embeddingPairs;
+          if (hit.score < scanConfig.similarityFloor) continue;
+          if (hit.id === id) continue;
+          const peer = memoryById.get(hit.id);
+          if (!peer) continue;
+
+          const pairId = computePairId(id, hit.id, namespace);
+          if (seen.has(pairId)) continue;
+          seen.add(pairId);
+
+          const existing = existingPairs.get(pairId);
+          if (isSkippedByCooldown(existing)) {
+            skipped++;
+            continue;
+          }
+
+          pushCandidate({
+            idA: id,
+            idB: hit.id,
+            textA: mem.content,
+            textB: peer.content,
+            categoryA: mem.frontmatter.category as string | undefined,
+            categoryB: peer.frontmatter.category as string | undefined,
+          });
+        }
+      } catch {
+        // Embedding backend unavailable — skip, entity-ref/tag strategies already covered
+      }
+    }
+  }
+
+  return { pairs, skipped };
+}
+
+// ── Helpers ────────────────────────────────────────────────────────────────────
+
+function normalizeScanNamespace(namespace: string | undefined): string | undefined {
+  const trimmed = namespace?.trim();
+  return trimmed ? trimmed : undefined;
+}
+
+async function resolveScanStorage(
+  deps: ScanDependencies,
+  namespace: string | undefined,
+): Promise<ScanStorageResolution> {
+  if (!deps.config.namespacesEnabled) {
+    const defaultNamespace = normalizeScanNamespace(deps.config.defaultNamespace);
+    if (namespace && namespace !== defaultNamespace) {
+      throw new Error(`unsupported namespace: ${namespace}`);
+    }
+    return { storage: deps.storage, namespace: undefined };
+  }
+
+  if (deps.storageForNamespace) {
+    const resolved = await deps.storageForNamespace(namespace);
+    if (isScanStorageResolution(resolved)) {
+      return {
+        storage: resolved.storage,
+        namespace: normalizeScanNamespace(resolved.namespace) ?? fallbackResolvedNamespace(deps, namespace),
+      };
+    }
+    if (!isStorageManagerLike(resolved)) {
+      throw new Error("storageForNamespace must return a StorageManager or { storage: StorageManager, namespace?: string }");
+    }
+    return {
+      storage: resolved,
+      namespace: fallbackResolvedNamespace(deps, namespace),
+    };
+  }
+
+  throw new Error(
+    "contradiction scans require storageForNamespace when namespaces are enabled so callers can enforce namespace access",
+  );
+}
+
+function isScanStorageResolution(value: StorageManager | ScanStorageResolution): value is ScanStorageResolution {
+  if (isStorageManagerLike(value)) return false;
+  if (typeof value !== "object" || value === null) return false;
+  const candidate = value as { storage?: unknown };
+  return isStorageManagerLike(candidate.storage);
+}
+
+function isStorageManagerLike(value: unknown): value is StorageManager {
+  if (typeof value !== "object" || value === null) return false;
+  const candidate = value as { readAllMemories?: unknown };
+  return typeof candidate.readAllMemories === "function";
+}
+
+function fallbackResolvedNamespace(deps: ScanDependencies, namespace: string | undefined): string | undefined {
+  if (namespace) return namespace;
+  return deps.config.namespacesEnabled ? deps.config.defaultNamespace : undefined;
+}
+
+function isDefaultNamespaceScan(
+  config: PluginConfig,
+  requestedNamespace: string | undefined,
+  resolvedNamespace: string,
+): boolean {
+  if (requestedNamespace === undefined) return true;
+  return requestedNamespace === config.defaultNamespace || resolvedNamespace === config.defaultNamespace;
+}
+
+async function loadEligibleMemories(storage: StorageManager): Promise<MemoryFile[]> {
+  let all: MemoryFile[];
+  try {
+    all = await storage.readAllMemories();
+  } catch {
+    return [];
+  }
+
+  return all.filter((mem) => {
+    const fm = mem.frontmatter;
+    // Only active memories (rule 53: explicit ACTIVE_STATUSES set)
+    const status = (fm.status as MemoryStatus) ?? "active";
+    if (!ACTIVE_STATUSES.has(status)) return false;
+
+    // Only scan high-value categories
+    const category = fm.category as string | undefined;
+    if (category && !SCAN_CATEGORIES.has(category)) return false;
+
+    // Must have content
+    if (!mem.content || mem.content.trim().length === 0) return false;
+
+    // Must have an ID
+    if (!fm.id) return false;
+
+    return true;
+  });
+}
+
+function jaccardOverlap(a: string[], b: string[]): number {
+  if (a.length === 0 && b.length === 0) return 0;
+  const setA = new Set(a.map((t) => t.toLowerCase()));
+  const setB = new Set(b.map((t) => t.toLowerCase()));
+  let intersection = 0;
+  for (const item of setA) {
+    if (setB.has(item)) intersection++;
+  }
+  const union = setA.size + setB.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
+
+function compareMemoryId(a: MemoryFile, b: MemoryFile): number {
+  return (a.frontmatter.id ?? "").localeCompare(b.frontmatter.id ?? "");
+}
+
+function memoryContentHashMap(memories: MemoryFile[]): Map<string, string> {
+  const hashes = new Map<string, string>();
+  for (const memory of memories) {
+    const id = memory.frontmatter.id;
+    if (!id) continue;
+    hashes.set(id, computeMemoryContentHash(memory.content, memory.frontmatter.category as string | undefined));
+  }
+  return hashes;
+}
+
+function pairMemoryContentHashes(
+  memoryIdA: string,
+  memoryIdB: string,
+  currentMemoryHashes: Map<string, string>,
+): Record<string, string> | undefined {
+  const hashA = currentMemoryHashes.get(memoryIdA);
+  const hashB = currentMemoryHashes.get(memoryIdB);
+  if (!hashA || !hashB) return undefined;
+  return {
+    [memoryIdA]: hashA,
+    [memoryIdB]: hashB,
+  };
+}