@totalreclaw/totalreclaw 1.0.5 → 1.2.0

package/consolidation.test.ts

@@ -0,0 +1,356 @@
+/**
+ * Unit tests for memory consolidation & near-duplicate detection.
+ *
+ * Run with: npx tsx consolidation.test.ts
+ *
+ * Uses TAP-style output (no test framework dependency).
+ */
+
+import {
+  findNearDuplicate,
+  shouldSupersede,
+  clusterFacts,
+  getStoreDedupThreshold,
+  getConsolidationThreshold,
+  STORE_DEDUP_MAX_CANDIDATES,
+} from './consolidation.js';
+import type { DecryptedCandidate } from './consolidation.js';
+
+let passed = 0;
+let failed = 0;
+let testNum = 0;
+
+function assert(condition: boolean, message: string): void {
+  testNum++;
+  if (condition) {
+    passed++;
+    console.log(`ok ${testNum} - ${message}`);
+  } else {
+    failed++;
+    console.log(`not ok ${testNum} - ${message}`);
+  }
+}
+
+function assertClose(actual: number, expected: number, epsilon: number, message: string): void {
+  const diff = Math.abs(actual - expected);
+  assert(diff < epsilon, `${message} (expected ~${expected}, got ${actual}, diff=${diff})`);
+}
+
+// Helper: create a DecryptedCandidate
+function makeCandidate(
+  overrides: Partial<DecryptedCandidate> & { id: string },
+): DecryptedCandidate {
+  return {
+    text: `fact ${overrides.id}`,
+    embedding: null,
+    importance: 5,
+    decayScore: 1.0,
+    createdAt: 1000,
+    version: 1,
+    ...overrides,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// getStoreDedupThreshold tests
+// ---------------------------------------------------------------------------
+
+console.log('# getStoreDedupThreshold');
+
+{
+  // Default threshold should be 0.85 (no env var set)
+  const orig = process.env.TOTALRECLAW_STORE_DEDUP_THRESHOLD;
+  delete process.env.TOTALRECLAW_STORE_DEDUP_THRESHOLD;
+  assertClose(getStoreDedupThreshold(), 0.85, 1e-10, 'default threshold is 0.85');
+  if (orig !== undefined) process.env.TOTALRECLAW_STORE_DEDUP_THRESHOLD = orig;
+}
+
+{
+  // Custom threshold via env var
+  const orig = process.env.TOTALRECLAW_STORE_DEDUP_THRESHOLD;
+  process.env.TOTALRECLAW_STORE_DEDUP_THRESHOLD = '0.75';
+  assertClose(getStoreDedupThreshold(), 0.75, 1e-10, 'custom threshold 0.75 from env');
+  if (orig !== undefined) {
+    process.env.TOTALRECLAW_STORE_DEDUP_THRESHOLD = orig;
+  } else {
+    delete process.env.TOTALRECLAW_STORE_DEDUP_THRESHOLD;
+  }
+}
+
+{
+  // Invalid env var falls back to default
+  const orig = process.env.TOTALRECLAW_STORE_DEDUP_THRESHOLD;
+  process.env.TOTALRECLAW_STORE_DEDUP_THRESHOLD = 'not-a-number';
+  assertClose(getStoreDedupThreshold(), 0.85, 1e-10, 'invalid env var falls back to 0.85');
+  if (orig !== undefined) {
+    process.env.TOTALRECLAW_STORE_DEDUP_THRESHOLD = orig;
+  } else {
+    delete process.env.TOTALRECLAW_STORE_DEDUP_THRESHOLD;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// getConsolidationThreshold tests
+// ---------------------------------------------------------------------------
+
+console.log('# getConsolidationThreshold');
+
+{
+  // Default threshold should be 0.88 (no env var set)
+  const orig = process.env.TOTALRECLAW_CONSOLIDATION_THRESHOLD;
+  delete process.env.TOTALRECLAW_CONSOLIDATION_THRESHOLD;
+  assertClose(getConsolidationThreshold(), 0.88, 1e-10, 'default threshold is 0.88');
+  if (orig !== undefined) process.env.TOTALRECLAW_CONSOLIDATION_THRESHOLD = orig;
+}
+
+{
+  // Custom threshold via env var
+  const orig = process.env.TOTALRECLAW_CONSOLIDATION_THRESHOLD;
+  process.env.TOTALRECLAW_CONSOLIDATION_THRESHOLD = '0.95';
+  assertClose(getConsolidationThreshold(), 0.95, 1e-10, 'custom threshold 0.95 from env');
+  if (orig !== undefined) {
+    process.env.TOTALRECLAW_CONSOLIDATION_THRESHOLD = orig;
+  } else {
+    delete process.env.TOTALRECLAW_CONSOLIDATION_THRESHOLD;
+  }
+}
+
+{
+  // Invalid env var falls back to default
+  const orig = process.env.TOTALRECLAW_CONSOLIDATION_THRESHOLD;
+  process.env.TOTALRECLAW_CONSOLIDATION_THRESHOLD = 'garbage';
+  assertClose(getConsolidationThreshold(), 0.88, 1e-10, 'invalid env var falls back to 0.88');
+  if (orig !== undefined) {
+    process.env.TOTALRECLAW_CONSOLIDATION_THRESHOLD = orig;
+  } else {
+    delete process.env.TOTALRECLAW_CONSOLIDATION_THRESHOLD;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// STORE_DEDUP_MAX_CANDIDATES constant
+// ---------------------------------------------------------------------------
+
+console.log('# STORE_DEDUP_MAX_CANDIDATES');
+
+assert(STORE_DEDUP_MAX_CANDIDATES === 200, 'STORE_DEDUP_MAX_CANDIDATES is 200');
+
+// ---------------------------------------------------------------------------
+// findNearDuplicate tests
+// ---------------------------------------------------------------------------
+
+console.log('# findNearDuplicate');
+
+{
+  // Empty candidates -> null
+  const result = findNearDuplicate([1, 0, 0], [], 0.85);
+  assert(result === null, 'empty candidates returns null');
+}
+
+{
+  // No embeddings on candidates -> null
+  const candidates = [
+    makeCandidate({ id: 'a', embedding: null }),
+    makeCandidate({ id: 'b', embedding: null }),
+  ];
+  const result = findNearDuplicate([1, 0, 0], candidates, 0.85);
+  assert(result === null, 'candidates without embeddings returns null');
+}
+
+{
+  // Below threshold -> null
+  const candidates = [
+    makeCandidate({ id: 'a', embedding: [0, 1, 0] }), // orthogonal, cosine = 0
+  ];
+  const result = findNearDuplicate([1, 0, 0], candidates, 0.85);
+  assert(result === null, 'below threshold returns null');
+}
+
+{
+  // Above threshold -> returns match
+  const candidates = [
+    makeCandidate({ id: 'a', embedding: [1, 0, 0] }), // cosine = 1.0
+  ];
+  const result = findNearDuplicate([1, 0, 0], candidates, 0.85);
+  assert(result !== null, 'above threshold returns match');
+  assert(result!.existingFact.id === 'a', 'match is the correct candidate');
+  assertClose(result!.similarity, 1.0, 1e-6, 'similarity is ~1.0');
+}
+
+{
+  // Multiple matches -> returns highest similarity
+  const candidates = [
+    makeCandidate({ id: 'low', embedding: [0.86, Math.sqrt(1 - 0.86 * 0.86), 0] }), // cosine ~ 0.86
+    makeCandidate({ id: 'high', embedding: [0.99, Math.sqrt(1 - 0.99 * 0.99), 0] }), // cosine ~ 0.99
+    makeCandidate({ id: 'mid', embedding: [0.90, Math.sqrt(1 - 0.90 * 0.90), 0] }),  // cosine ~ 0.90
+  ];
+  const result = findNearDuplicate([1, 0, 0], candidates, 0.85);
+  assert(result !== null, 'multiple matches: returns a match');
+  assert(result!.existingFact.id === 'high', 'multiple matches: returns highest similarity');
+}
+
+{
+  // Parallel vectors (cosine = 1.0) -> match
+  const candidates = [
+    makeCandidate({ id: 'parallel', embedding: [3, 6, 9] }), // parallel to [1, 2, 3]
+  ];
+  const result = findNearDuplicate([1, 2, 3], candidates, 0.85);
+  assert(result !== null, 'parallel vectors: returns match');
+  assertClose(result!.similarity, 1.0, 1e-6, 'parallel vectors: cosine is ~1.0');
+}
+
+{
+  // Orthogonal vectors (cosine = 0) -> null
+  const candidates = [
+    makeCandidate({ id: 'ortho', embedding: [0, 1, 0] }),
+  ];
+  const result = findNearDuplicate([1, 0, 0], candidates, 0.85);
+  assert(result === null, 'orthogonal vectors: returns null');
+}
+
+// ---------------------------------------------------------------------------
+// shouldSupersede tests
+// ---------------------------------------------------------------------------
+
+console.log('# shouldSupersede');
+
+{
+  // Higher new importance -> supersede
+  const existing = makeCandidate({ id: 'old', importance: 5 });
+  const result = shouldSupersede(8, existing);
+  assert(result === 'supersede', 'higher new importance -> supersede');
+}
+
+{
+  // Lower new importance -> skip
+  const existing = makeCandidate({ id: 'old', importance: 8 });
+  const result = shouldSupersede(3, existing);
+  assert(result === 'skip', 'lower new importance -> skip');
+}
+
+{
+  // Equal importance -> supersede (newer wins)
+  const existing = makeCandidate({ id: 'old', importance: 5 });
+  const result = shouldSupersede(5, existing);
+  assert(result === 'supersede', 'equal importance -> supersede (newer wins)');
+}
+
+// ---------------------------------------------------------------------------
+// clusterFacts tests
+// ---------------------------------------------------------------------------
+
+console.log('# clusterFacts');
+
+{
+  // Empty facts -> empty clusters
+  const clusters = clusterFacts([], 0.88);
+  assert(clusters.length === 0, 'empty facts -> no clusters');
+}
+
+{
+  // Single fact -> no clusters (needs at least 2 to form a cluster)
+  const facts = [
+    makeCandidate({ id: 'a', embedding: [1, 0, 0] }),
+  ];
+  const clusters = clusterFacts(facts, 0.88);
+  assert(clusters.length === 0, 'single fact -> no clusters');
+}
+
+{
+  // Two identical embeddings -> one cluster
+  const facts = [
+    makeCandidate({ id: 'a', embedding: [1, 0, 0] }),
+    makeCandidate({ id: 'b', embedding: [1, 0, 0] }),
+  ];
+  const clusters = clusterFacts(facts, 0.88);
+  assert(clusters.length === 1, 'two identical -> one cluster');
+  assert(clusters[0].duplicates.length === 1, 'two identical -> one duplicate');
+}
+
+{
+  // Two dissimilar embeddings -> no clusters
+  const facts = [
+    makeCandidate({ id: 'a', embedding: [1, 0, 0] }),
+    makeCandidate({ id: 'b', embedding: [0, 1, 0] }), // orthogonal
+  ];
+  const clusters = clusterFacts(facts, 0.88);
+  assert(clusters.length === 0, 'two dissimilar -> no clusters');
+}
+
+{
+  // Multiple clusters: two groups of duplicates + one unique
+  const facts = [
+    makeCandidate({ id: 'a1', embedding: [1, 0, 0] }),
+    makeCandidate({ id: 'a2', embedding: [1, 0, 0] }),    // dup of a1
+    makeCandidate({ id: 'b1', embedding: [0, 1, 0] }),
+    makeCandidate({ id: 'b2', embedding: [0, 1, 0] }),    // dup of b1
+    makeCandidate({ id: 'c1', embedding: [0, 0, 1] }),    // unique
+  ];
+  const clusters = clusterFacts(facts, 0.88);
+  assert(clusters.length === 2, 'multiple clusters: two groups found');
+}
+
+{
+  // Facts without embeddings are not clustered
+  const facts = [
+    makeCandidate({ id: 'a', embedding: [1, 0, 0] }),
+    makeCandidate({ id: 'b', embedding: null }),           // no embedding
+    makeCandidate({ id: 'c', embedding: [1, 0, 0] }),     // dup of a
+  ];
+  const clusters = clusterFacts(facts, 0.88);
+  assert(clusters.length === 1, 'no-embedding facts skipped, one cluster of a+c');
+  // b should not appear in any cluster
+  const allIds = clusters.flatMap(c => [c.representative.id, ...c.duplicates.map(d => d.id)]);
+  assert(!allIds.includes('b'), 'no-embedding fact not in any cluster');
+}
+
+{
+  // Representative = highest importance (via decayScore tiebreak)
+  const facts = [
+    makeCandidate({ id: 'low', embedding: [1, 0, 0], decayScore: 0.5, importance: 3 }),
+    makeCandidate({ id: 'high', embedding: [1, 0, 0], decayScore: 0.9, importance: 8 }),
+    makeCandidate({ id: 'mid', embedding: [1, 0, 0], decayScore: 0.7, importance: 5 }),
+  ];
+  const clusters = clusterFacts(facts, 0.88);
+  assert(clusters.length === 1, 'representative test: one cluster');
+  assert(clusters[0].representative.id === 'high', 'representative = highest decayScore');
+  assert(clusters[0].duplicates.length === 2, 'two duplicates');
+}
+
+{
+  // Tiebreak: same decayScore -> most recent (highest createdAt)
+  const facts = [
+    makeCandidate({ id: 'old', embedding: [1, 0, 0], decayScore: 1.0, createdAt: 1000 }),
+    makeCandidate({ id: 'new', embedding: [1, 0, 0], decayScore: 1.0, createdAt: 2000 }),
+  ];
+  const clusters = clusterFacts(facts, 0.88);
+  assert(clusters.length === 1, 'tiebreak test: one cluster');
+  assert(clusters[0].representative.id === 'new', 'tiebreak: most recent is representative');
+}
+
+{
+  // Tiebreak: same decayScore + createdAt -> longest text
+  const facts = [
+    makeCandidate({ id: 'short', text: 'abc', embedding: [1, 0, 0], decayScore: 1.0, createdAt: 1000 }),
+    makeCandidate({ id: 'long', text: 'abcdefghij', embedding: [1, 0, 0], decayScore: 1.0, createdAt: 1000 }),
+  ];
+  const clusters = clusterFacts(facts, 0.88);
+  assert(clusters.length === 1, 'tiebreak longest text: one cluster');
+  assert(clusters[0].representative.id === 'long', 'tiebreak: longest text is representative');
+}
+
+// ---------------------------------------------------------------------------
+// Summary
+// ---------------------------------------------------------------------------
+
+console.log(`\n1..${testNum}`);
+console.log(`# pass: ${passed}`);
+console.log(`# fail: ${failed}`);
+
+if (failed > 0) {
+  console.log('\nFAILED');
+  process.exit(1);
+} else {
+  console.log('\nALL TESTS PASSED');
+  process.exit(0);
+}

package/consolidation.ts

@@ -0,0 +1,227 @@
+/**
+ * TotalReclaw Plugin - Memory Consolidation & Near-Duplicate Detection
+ *
+ * Provides cross-session / cross-vault deduplication of stored facts using
+ * cosine similarity on their embeddings. Unlike semantic-dedup.ts (which
+ * handles within-batch dedup at threshold 0.9), this module handles:
+ *
+ *   1. Store-time dedup — before writing a new fact, check whether a
+ *      near-duplicate already exists in the vault (findNearDuplicate).
+ *   2. Supersede logic — when a near-duplicate is found, decide whether
+ *      the new fact should replace or be skipped (shouldSupersede).
+ *   3. Bulk consolidation — cluster all facts in the vault and identify
+ *      groups of near-duplicates for cleanup (clusterFacts).
+ *
+ * This module intentionally has minimal dependencies (only reranker for
+ * cosineSimilarity) so it can be tested without pulling in the full
+ * plugin dependency graph.
+ */
+
+import { cosineSimilarity } from './reranker.js';
+
+// ---------------------------------------------------------------------------
+// Configuration
+// ---------------------------------------------------------------------------
+
+/**
+ * Get the cosine similarity threshold for store-time dedup.
+ *
+ * Configurable via TOTALRECLAW_STORE_DEDUP_THRESHOLD env var.
+ * Must be a number in [0, 1]. Falls back to 0.85 if invalid or unset.
+ */
+export function getStoreDedupThreshold(): number {
+  const envVal = process.env.TOTALRECLAW_STORE_DEDUP_THRESHOLD;
+  if (envVal !== undefined) {
+    const parsed = parseFloat(envVal);
+    if (!isNaN(parsed) && parsed >= 0 && parsed <= 1) return parsed;
+  }
+  return 0.85;
+}
+
+/**
+ * Get the cosine similarity threshold for bulk consolidation clustering.
+ *
+ * Configurable via TOTALRECLAW_CONSOLIDATION_THRESHOLD env var.
+ * Must be a number in [0, 1]. Falls back to 0.88 if invalid or unset.
+ */
+export function getConsolidationThreshold(): number {
+  const envVal = process.env.TOTALRECLAW_CONSOLIDATION_THRESHOLD;
+  if (envVal !== undefined) {
+    const parsed = parseFloat(envVal);
+    if (!isNaN(parsed) && parsed >= 0 && parsed <= 1) return parsed;
+  }
+  return 0.88;
+}
+
+/** Maximum candidates to compare against during store-time dedup. */
+export const STORE_DEDUP_MAX_CANDIDATES = 200;
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+/** A decrypted fact candidate from the vault, with metadata for ranking. */
+export interface DecryptedCandidate {
+  id: string;
+  text: string;
+  embedding: number[] | null;
+  importance: number;
+  decayScore: number;
+  createdAt: number;
+  version: number;
+}
+
+/** A match result from near-duplicate detection. */
+export interface NearDuplicateMatch {
+  existingFact: DecryptedCandidate;
+  similarity: number;
+}
+
+/** A cluster of near-duplicate facts for consolidation. */
+export interface ConsolidationCluster {
+  representative: DecryptedCandidate;
+  duplicates: DecryptedCandidate[];
+}
+
+// ---------------------------------------------------------------------------
+// Store-time dedup
+// ---------------------------------------------------------------------------
+
+/**
+ * Find the best near-duplicate match for a new fact among existing candidates.
+ *
+ * Compares the new fact's embedding against all candidates using cosine
+ * similarity. Returns the candidate with the highest similarity above the
+ * threshold, or null if no match is found.
+ *
+ * Candidates without embeddings are skipped (fail-safe).
+ *
+ * @param newFactEmbedding - Embedding vector for the new fact
+ * @param candidates       - Existing facts to compare against
+ * @param threshold        - Cosine similarity threshold (e.g. 0.85)
+ * @returns                - Best match above threshold, or null
+ */
+export function findNearDuplicate(
+  newFactEmbedding: number[],
+  candidates: DecryptedCandidate[],
+  threshold: number,
+): NearDuplicateMatch | null {
+  let bestMatch: NearDuplicateMatch | null = null;
+
+  for (const candidate of candidates) {
+    if (!candidate.embedding || candidate.embedding.length === 0) continue;
+
+    const similarity = cosineSimilarity(newFactEmbedding, candidate.embedding);
+    if (similarity >= threshold) {
+      if (!bestMatch || similarity > bestMatch.similarity) {
+        bestMatch = { existingFact: candidate, similarity };
+      }
+    }
+  }
+
+  return bestMatch;
+}
+
+// ---------------------------------------------------------------------------
+// Supersede logic
+// ---------------------------------------------------------------------------
+
+/**
+ * Decide whether a new fact should supersede an existing near-duplicate.
+ *
+ * - Higher importance wins.
+ * - Equal importance: new fact supersedes (newer is preferred).
+ *
+ * @param newImportance - Importance score of the new fact
+ * @param existingFact  - The existing near-duplicate candidate
+ * @returns             - 'supersede' if new fact should replace, 'skip' otherwise
+ */
+export function shouldSupersede(
+  newImportance: number,
+  existingFact: DecryptedCandidate,
+): 'supersede' | 'skip' {
+  if (newImportance >= existingFact.importance) return 'supersede';
+  return 'skip';
+}
+
+// ---------------------------------------------------------------------------
+// Bulk consolidation
+// ---------------------------------------------------------------------------
+
+/**
+ * Pick the best representative from a group of near-duplicate facts.
+ *
+ * Tiebreak order:
+ *   1. Highest decayScore
+ *   2. Most recent (highest createdAt)
+ *   3. Longest text
+ */
+function pickRepresentative(facts: DecryptedCandidate[]): DecryptedCandidate {
+  let best = facts[0];
+  for (let i = 1; i < facts.length; i++) {
+    const f = facts[i];
+    if (
+      f.decayScore > best.decayScore ||
+      (f.decayScore === best.decayScore && f.createdAt > best.createdAt) ||
+      (f.decayScore === best.decayScore && f.createdAt === best.createdAt && f.text.length > best.text.length)
+    ) {
+      best = f;
+    }
+  }
+  return best;
+}
+
+/**
+ * Cluster facts by semantic similarity using greedy single-pass clustering.
+ *
+ * For each fact (in order), assigns it to the first existing cluster whose
+ * representative has cosine similarity >= threshold. If no cluster matches,
+ * a new cluster is started.
+ *
+ * Only returns clusters that have duplicates (i.e. more than one member).
+ * Facts without embeddings are not clustered.
+ *
+ * @param facts     - All facts to cluster
+ * @param threshold - Cosine similarity threshold (e.g. 0.88)
+ * @returns         - Clusters with duplicates (representative + duplicates)
+ */
+export function clusterFacts(
+  facts: DecryptedCandidate[],
+  threshold: number,
+): ConsolidationCluster[] {
+  const clusters: { members: DecryptedCandidate[] }[] = [];
+
+  for (const fact of facts) {
+    if (!fact.embedding || fact.embedding.length === 0) continue;
+
+    let assigned = false;
+    for (const cluster of clusters) {
+      // Compare against the first member's embedding (cluster seed)
+      const seed = cluster.members[0];
+      if (!seed.embedding) continue;
+
+      const similarity = cosineSimilarity(fact.embedding, seed.embedding);
+      if (similarity >= threshold) {
+        cluster.members.push(fact);
+        assigned = true;
+        break;
+      }
+    }
+
+    if (!assigned) {
+      clusters.push({ members: [fact] });
+    }
+  }
+
+  // Only return clusters with duplicates, pick representative for each
+  const result: ConsolidationCluster[] = [];
+  for (const cluster of clusters) {
+    if (cluster.members.length < 2) continue;
+
+    const representative = pickRepresentative(cluster.members);
+    const duplicates = cluster.members.filter((m) => m !== representative);
+    result.push({ representative, duplicates });
+  }
+
+  return result;
+}