hippo-memory 0.19.0 → 0.20.0

package/README.md

@@ -45,10 +45,12 @@ hippo init --scan ~
 
 `--scan` finds every git repo under your home directory, creates a `.hippo/` store in each one, and seeds it with lessons from your commit history. One command, instant memory across all your projects.
 
-After setup, `hippo sleep` runs automatically at session end (via agent hooks) and does three things:
+After setup, `hippo sleep` runs automatically at session end (via agent hooks) and does five things:
 1. **Learns** from today's git commits
-2. **Consolidates** memories (decay, merge, prune)
-3. **Shares** high-value lessons to the global store so they surface in every project
+2. **Imports** new entries from Claude Code MEMORY.md files
+3. **Consolidates** memories (decay, merge, prune)
+4. **Deduplicates** near-identical memories (keeps the stronger copy)
+5. **Shares** high-value lessons to the global store so they surface in every project
 
 ```bash
 # Manual usage
@@ -58,6 +60,16 @@ hippo recall "data pipeline issues" --budget 2000
 
 ---
 
+### What's new in v0.20
+
+- **`hippo dedup`.** Scans for near-duplicate memories, shows you what's duplicated and why (redundant semantic patterns, same lesson from multiple sources, cross-layer overlap), and removes the weaker copy. Runs automatically during `hippo sleep`.
+- **MEMORY.md import.** `hippo init` and `hippo sleep` now scan Claude Code memory files and import new entries. Your agent memories from Claude Code flow into hippo automatically.
+
+### What's new in v0.19.1
+
+- **Configured embedding models now work end to end.** `hippo embed`, hybrid search, and physics search all respect `embeddings.model` from `.hippo/config.json`.
+- **Safe rebuild on model change.** If you switch embedding models, rerun `hippo embed`. Hippo now rebuilds cached embeddings and resets physics state so old vectors are not mixed with the new model.
+
 ### What's new in v0.18
 
 - **Multi-project auto-discovery.** `hippo init --scan [dir]` finds all git repos under a directory and initializes each one. Seeds with a full year of git history by default. One command to set up memory across all your projects.
@@ -126,6 +138,7 @@ hippo recall "data pipeline issues" --budget 2000
 ### What's new in v0.8.0
 
 - **Hybrid search** blends BM25 keywords with cosine embedding similarity. Install `@xenova/transformers`, run `hippo embed`, recall quality jumps. Falls back to BM25 otherwise.
+- Configure a custom embedding model with `embeddings.model` in `.hippo/config.json`. If you change models later, rerun `hippo embed` so Hippo rebuilds cached embeddings and physics state for the new vector space.
 - **Schema acceleration** auto-computes how well new memories fit existing patterns. Familiar memories consolidate faster; novel ones decay faster if unused.
 - **Multi-agent shared memory** with `hippo share`, `hippo peers`, and transfer scoring. Universal lessons travel between projects; project-specific config stays local.
 - **Conflict resolution** via `hippo resolve <id> --keep <mem_id>`. Closes the detect-inspect-resolve loop.

package/dist/cli.js

@@ -30,9 +30,9 @@ import * as os from 'os';
 import { execSync } from 'child_process';
 import { createMemory, calculateStrength, calculateRewardFactor, deriveHalfLife, resolveConfidence, applyOutcome, computeSchemaFit, Layer, DECISION_HALF_LIFE_DAYS, } from './memory.js';
 import { getHippoRoot, isInitialized, initStore, writeEntry, readEntry, deleteEntry, loadAllEntries, loadSearchEntries, loadIndex, saveIndex, loadStats, updateStats, saveActiveTaskSnapshot, loadActiveTaskSnapshot, clearActiveTaskSnapshot, appendSessionEvent, listSessionEvents, listMemoryConflicts, resolveConflict, saveSessionHandoff, loadLatestHandoff, loadHandoffById, } from './store.js';
-import { markRetrieved, estimateTokens, hybridSearch, physicsSearch, explainMatch } from './search.js';
+import { markRetrieved, estimateTokens, hybridSearch, physicsSearch, explainMatch, textOverlap } from './search.js';
 import { consolidate } from './consolidate.js';
-import { isEmbeddingAvailable, embedAll, embedMemory, loadEmbeddingIndex, } from './embeddings.js';
+import { isEmbeddingAvailable, embedAll, embedMemory, loadEmbeddingIndex, resolveEmbeddingModel, } from './embeddings.js';
 import { loadPhysicsState, resetAllPhysicsState } from './physics-state.js';
 import { computeSystemEnergy, vecNorm } from './physics.js';
 import { loadConfig } from './config.js';
@@ -218,6 +218,11 @@ function cmdInit(hippoRoot, flags) {
                 console.log(`   No matching commits found in git history.`);
             }
         }
+        // Also import from Claude Code / agent MEMORY.md files
+        const memImported = learnFromMemoryMd(hippoRoot);
+        if (memImported > 0) {
+            console.log(`   Imported ${memImported} memories from agent MEMORY.md files.`);
+        }
     }
 }
 /**
@@ -492,6 +497,149 @@ async function cmdRecall(hippoRoot, query, flags) {
         console.log();
     }
 }
+/**
+ * Scan for Claude Code MEMORY.md files and import new entries into hippo.
+ * Looks in ~/.claude/projects/<project>/memory/ for .md files with YAML frontmatter.
+ */
+function learnFromMemoryMd(hippoRoot) {
+    const home = os.homedir();
+    const memoryDirs = [];
+    // Claude Code project memories
+    const claudeProjectsDir = path.join(home, '.claude', 'projects');
+    if (fs.existsSync(claudeProjectsDir)) {
+        try {
+            for (const project of fs.readdirSync(claudeProjectsDir)) {
+                const memDir = path.join(claudeProjectsDir, project, 'memory');
+                if (fs.existsSync(memDir))
+                    memoryDirs.push(memDir);
+            }
+        }
+        catch { /* permission denied */ }
+    }
+    if (memoryDirs.length === 0)
+        return 0;
+    const existing = loadAllEntries(hippoRoot);
+    let imported = 0;
+    for (const memDir of memoryDirs) {
+        try {
+            const files = fs.readdirSync(memDir).filter(f => f.endsWith('.md') && f !== 'MEMORY.md');
+            for (const file of files) {
+                const raw = fs.readFileSync(path.join(memDir, file), 'utf8');
+                // Parse YAML frontmatter
+                const fmMatch = raw.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/);
+                if (!fmMatch)
+                    continue;
+                const body = fmMatch[2].trim();
+                if (!body || body.length < 10)
+                    continue;
+                // Truncate to reasonable size
+                const content = body.length > 1500 ? body.slice(0, 1500) + ' [truncated]' : body;
+                // Dedup: check if substantially similar content already exists
+                const isDup = existing.some(e => {
+                    const overlap = textOverlap(content.slice(0, 200), e.content.slice(0, 200));
+                    return overlap > 0.6;
+                });
+                if (isDup)
+                    continue;
+                const entry = createMemory(content, {
+                    layer: Layer.Episodic,
+                    tags: ['claude-code-memory'],
+                    source: `claude-memory:${file}`,
+                    confidence: 'observed',
+                });
+                writeEntry(hippoRoot, entry);
+                existing.push(entry); // prevent self-dedup within batch
+                imported++;
+            }
+        }
+        catch { /* skip broken dirs */ }
+    }
+    return imported;
+}
+function deduplicateStore(hippoRoot, options = {}) {
+    const threshold = options.threshold ?? 0.7;
+    const dryRun = options.dryRun ?? false;
+    const entries = loadAllEntries(hippoRoot);
+    // Sort by strength desc, then retrieval count, so we keep the most valuable copy
+    entries.sort((a, b) => {
+        const sDiff = (b.strength ?? 0) - (a.strength ?? 0);
+        if (Math.abs(sDiff) > 0.01)
+            return sDiff;
+        return (b.retrieval_count ?? 0) - (a.retrieval_count ?? 0);
+    });
+    const removed = new Set();
+    const pairs = [];
+    for (let i = 0; i < entries.length; i++) {
+        if (removed.has(entries[i].id))
+            continue;
+        for (let j = i + 1; j < entries.length; j++) {
+            if (removed.has(entries[j].id))
+                continue;
+            const similarity = textOverlap(entries[i].content, entries[j].content);
+            if (similarity <= threshold)
+                continue;
+            removed.add(entries[j].id);
+            pairs.push({
+                kept: entries[i].id,
+                keptContent: entries[i].content,
+                keptLayer: entries[i].layer,
+                keptStrength: entries[i].strength ?? 0,
+                removed: entries[j].id,
+                removedContent: entries[j].content,
+                removedLayer: entries[j].layer,
+                removedStrength: entries[j].strength ?? 0,
+                similarity,
+            });
+        }
+    }
+    if (!dryRun) {
+        for (const id of removed) {
+            deleteEntry(hippoRoot, id);
+        }
+    }
+    return { removed: removed.size, pairs };
+}
+function cmdDedup(hippoRoot, flags) {
+    requireInit(hippoRoot);
+    const dryRun = Boolean(flags['dry-run']);
+    const threshold = parseFloat(String(flags['threshold'] ?? '0.7'));
+    const entries = loadAllEntries(hippoRoot);
+    console.log(`Scanning ${entries.length} memories for duplicates (>=${(threshold * 100).toFixed(0)}% text overlap)${dryRun ? ' (dry run)' : ''}...\n`);
+    const result = deduplicateStore(hippoRoot, { threshold, dryRun });
+    if (result.removed === 0) {
+        console.log('No duplicates found.');
+        return;
+    }
+    // Group by reason
+    const sameLayerSem = result.pairs.filter(p => p.keptLayer === 'semantic' && p.removedLayer === 'semantic');
+    const sameLayerEpi = result.pairs.filter(p => p.keptLayer === 'episodic' && p.removedLayer === 'episodic');
+    const crossLayer = result.pairs.filter(p => p.keptLayer !== p.removedLayer);
+    console.log(`${dryRun ? 'Would remove' : 'Removed'} ${result.removed} duplicates:`);
+    if (sameLayerSem.length > 0) {
+        console.log(`  ${sameLayerSem.length} redundant semantic memories (consolidation regenerated near-identical patterns)`);
+    }
+    if (sameLayerEpi.length > 0) {
+        console.log(`  ${sameLayerEpi.length} duplicate episodic memories (same lesson learned from multiple sources)`);
+    }
+    if (crossLayer.length > 0) {
+        console.log(`  ${crossLayer.length} cross-layer duplicates (episodic content already consolidated into semantic)`);
+    }
+    // Show detailed pairs
+    console.log('');
+    const shown = result.pairs.slice(0, 15);
+    for (const pair of shown) {
+        const simPct = (pair.similarity * 100).toFixed(0);
+        const action = dryRun ? 'Would remove' : 'Removed';
+        console.log(`  ${simPct}% similar | kept [${pair.keptLayer}] strength=${pair.keptStrength.toFixed(2)}`);
+        console.log(`    ${pair.keptContent.slice(0, 90)}`);
+        console.log(`  ${action} [${pair.removedLayer}] strength=${pair.removedStrength.toFixed(2)}`);
+        console.log(`    ${pair.removedContent.slice(0, 90)}`);
+        console.log('');
+    }
+    if (result.pairs.length > 15) {
+        console.log(`  ... and ${result.pairs.length - 15} more (run with --dry-run to see all)`);
+    }
+}
 function cmdSleep(hippoRoot, flags) {
     requireInit(hippoRoot);
     // Auto-learn from git before consolidating (unless --no-learn)
@@ -502,6 +650,10 @@ function cmdSleep(hippoRoot, flags) {
             if (added > 0)
                 console.log(`Auto-learned ${added} lessons from today's git commits.`);
         }
+        // Also learn from Claude Code MEMORY.md files
+        const memImported = learnFromMemoryMd(hippoRoot);
+        if (memImported > 0)
+            console.log(`Imported ${memImported} memories from Claude Code MEMORY.md files.`);
     }
     const dryRun = Boolean(flags['dry-run']);
     console.log(`Running consolidation${dryRun ? ' (dry run)' : ''}...`);
@@ -519,6 +671,23 @@ function cmdSleep(hippoRoot, flags) {
     }
     if (dryRun)
         console.log('\n(dry run  - nothing written)');
+    // Auto-dedup after consolidation (unless dry-run)
+    if (!dryRun) {
+        const dedupResult = deduplicateStore(hippoRoot);
+        if (dedupResult.removed > 0) {
+            const semDups = dedupResult.pairs.filter(p => p.keptLayer === 'semantic' && p.removedLayer === 'semantic').length;
+            const epiDups = dedupResult.pairs.filter(p => p.keptLayer === 'episodic' && p.removedLayer === 'episodic').length;
+            const crossDups = dedupResult.pairs.filter(p => p.keptLayer !== p.removedLayer).length;
+            const parts = [];
+            if (semDups > 0)
+                parts.push(`${semDups} redundant semantic patterns`);
+            if (epiDups > 0)
+                parts.push(`${epiDups} duplicate episodic lessons`);
+            if (crossDups > 0)
+                parts.push(`${crossDups} cross-layer duplicates`);
+            console.log(`\nDeduped ${dedupResult.removed} duplicates (${parts.join(', ')}). Kept stronger copies.`);
+        }
+    }
     // Auto-share high-transfer-score memories to global (unless --no-share or dry-run)
     if (!dryRun && !flags['no-share']) {
         const sleepConfig = loadConfig(hippoRoot);
@@ -1358,7 +1527,7 @@ async function cmdEmbed(hippoRoot, flags) {
         return;
     }
     console.log('Embedding all memories (this may take a moment on first run to download model)...');
-    const count = await embedAll(hippoRoot);
+    const count = await embedAll(hippoRoot, resolveEmbeddingModel(hippoRoot));
     const entriesAfter = loadAllEntries(hippoRoot);
     const embIndexAfter = loadEmbeddingIndex(hippoRoot);
     console.log(`Done. ${count} new embeddings created. ${Object.keys(embIndexAfter).length}/${entriesAfter.length} total.`);
@@ -2040,10 +2209,13 @@ Commands:
     --budget <n>           Token budget (default: 1500)
     --format <fmt>         Output format: markdown (default) or json
     --framing <mode>       Framing: observe (default), suggest, assert
-  sleep                    Run consolidation pass (auto-learns + auto-shares)
+  sleep                    Run consolidation pass (auto-learns + dedup + auto-shares)
     --dry-run              Preview without writing
     --no-learn             Skip auto git-learn before consolidation
     --no-share             Skip auto-sharing to global store
+  dedup                    Remove duplicate memories (keeps stronger copy)
+    --dry-run              Preview without removing
+    --threshold <n>        Overlap threshold 0-1 (default: 0.7)
   status                   Show memory health stats
   outcome                  Apply feedback to last recall
     --good                 Memories were helpful
@@ -2220,6 +2392,9 @@ async function main() {
         case 'sleep':
             cmdSleep(hippoRoot, flags);
             break;
+        case 'dedup':
+            cmdDedup(hippoRoot, flags);
+            break;
         case 'status':
             cmdStatus(hippoRoot);
             break;