clawmem 0.2.0 → 0.2.2

package/AGENTS.md

@@ -634,9 +634,20 @@ Symptom: embed --force with new model produces 3 docs stuck as "Unembedded" but
   → Fix: Delete partial content_vectors + vectors_vec for the stuck hashes, then re-run embed (no --force).
     The vec0 DELETE try-catch prevents cascading failures during the re-embed.
 
+Symptom: reindex --force after v0.2.0 upgrade shows no entity extraction
+  → `reindex --force` treats existing docs as updates (isNew=false). The A-MEM pipeline
+    skips entity extraction, link generation, and evolution for updates to avoid churn.
+  → Fix: Use `clawmem reindex --enrich` instead. The `--enrich` flag forces the full
+    enrichment pipeline (entity extraction + links + evolution) on all documents.
+  → `--force` alone only refreshes A-MEM notes (keywords, tags, context). `--enrich`
+    is needed after major upgrades that add new enrichment stages.
+
 Symptom: CLI reindex/update falls back to node-llama-cpp Vulkan (not GPU server)
   → GPU env vars only in systemd drop-in, not in wrapper script. CLI invocations missed them.
   → Fixed 2026-02-12: bin/clawmem wrapper exports CLAWMEM_EMBED_URL/LLM_URL/RERANK_URL defaults.
+  → Always run ClawMem via the `bin/clawmem` wrapper, not `bun run src/clawmem.ts` directly.
+    The wrapper sets CLAWMEM_EMBED_URL/LLM_URL/RERANK_URL defaults. Scripts or inline bun
+    commands that bypass the wrapper will fall back to in-process node-llama-cpp (slow, CPU).
 
 Symptom: "UserPromptSubmit hook error" on context-surfacing hook (intermittent)
   → SQLite contention between the watcher and the hook. The watcher processes filesystem events

package/CLAUDE.md

@@ -634,9 +634,20 @@ Symptom: embed --force with new model produces 3 docs stuck as "Unembedded" but
   → Fix: Delete partial content_vectors + vectors_vec for the stuck hashes, then re-run embed (no --force).
     The vec0 DELETE try-catch prevents cascading failures during the re-embed.
 
+Symptom: reindex --force after v0.2.0 upgrade shows no entity extraction
+  → `reindex --force` treats existing docs as updates (isNew=false). The A-MEM pipeline
+    skips entity extraction, link generation, and evolution for updates to avoid churn.
+  → Fix: Use `clawmem reindex --enrich` instead. The `--enrich` flag forces the full
+    enrichment pipeline (entity extraction + links + evolution) on all documents.
+  → `--force` alone only refreshes A-MEM notes (keywords, tags, context). `--enrich`
+    is needed after major upgrades that add new enrichment stages.
+
 Symptom: CLI reindex/update falls back to node-llama-cpp Vulkan (not GPU server)
   → GPU env vars only in systemd drop-in, not in wrapper script. CLI invocations missed them.
   → Fixed 2026-02-12: bin/clawmem wrapper exports CLAWMEM_EMBED_URL/LLM_URL/RERANK_URL defaults.
+  → Always run ClawMem via the `bin/clawmem` wrapper, not `bun run src/clawmem.ts` directly.
+    The wrapper sets CLAWMEM_EMBED_URL/LLM_URL/RERANK_URL defaults. Scripts or inline bun
+    commands that bypass the wrapper will fall back to in-process node-llama-cpp (slow, CPU).
 
 Symptom: "UserPromptSubmit hook error" on context-surfacing hook (intermittent)
   → SQLite contention between the watcher and the hook. The watcher processes filesystem events

package/README.md

@@ -44,8 +44,6 @@ Runs fully local with no API keys and no cloud services. Integrates via Claude C
 
 ### v0.2.0 Enhancements
 
-Seven patterns extracted from competitor analysis ([Hindsight](https://github.com/vectorize-io/hindsight), [Hermes Agent](https://github.com/NousResearch/hermes-agent), [claude-mem](https://github.com/thedotmack/claude-mem)):
-
 - **Entity resolution + co-occurrence graph** — LLM entity extraction during A-MEM enrichment, canonical normalization via FTS5 + Levenshtein fuzzy matching, co-occurrence tracking, entity graph traversal for ENTITY intent queries
 - **MPFP graph retrieval** — Multi-Path Fact Propagation with meta-path patterns per intent, hop-synchronized edge cache, Forward Push with α=0.15 teleport probability. Replaces single-beam traversal for causal/entity/temporal queries.
 - **Temporal query extraction** — regex-based date range extraction from natural language queries ("last week", "March 2026"), wired as WHERE filters into BM25 and vector search
@@ -149,6 +147,25 @@ clawmem update --embed
 clawmem doctor
 ```
 
+### Upgrading
+
+```bash
+bun update -g clawmem   # or: npm update -g clawmem
+```
+
+Database schema migrates automatically on next startup (new tables and columns are added via `CREATE IF NOT EXISTS` / `ALTER TABLE ADD COLUMN`).
+
+After **major version updates** (e.g. 0.1.x → 0.2.0) that add new enrichment pipelines, run a full enrichment pass to backfill existing documents:
+
+```bash
+clawmem reindex --enrich  # Full enrichment: entity extraction + links + evolution for all docs
+clawmem embed             # Re-embed if upgrading embedding models (not needed for most updates)
+```
+
+`--enrich` forces the complete A-MEM pipeline (entity extraction, link generation, memory evolution) on all documents, not just new ones. Without it, reindex only refreshes metadata for existing docs.
+
+Routine patch updates (e.g. 0.2.0 → 0.2.1) do not require reindexing.
+
 ### Integration
 
 #### Claude Code
@@ -1027,6 +1044,8 @@ Built on the shoulders of:
 - [Beads](https://github.com/steveyegge/beads) — Dolt-backed issue tracker for AI agents
 - [claude-mem](https://github.com/thedotmack/claude-mem) — Claude Code memory integration reference
 - [Engram](https://github.com/Gentleman-Programming/engram) — observation dedup window, topic-key upsert pattern, temporal timeline navigation, duplicate metadata scoring signals
+- [Hermes Agent](https://github.com/NousResearch/hermes-agent) — memory nudge system (periodic lifecycle tool prompting)
+- [Hindsight](https://github.com/vectorize-io/hindsight) — entity resolution, MPFP graph traversal, temporal extraction, 3-tier consolidation, observation invalidation, 4-way parallel retrieval
 - [MAGMA](https://arxiv.org/abs/2501.13956) — multi-graph memory agent
 - [memory-lancedb-pro](https://github.com/CortexReach/memory-lancedb-pro) — retrieval gate, length normalization, MMR diversity, access reinforcement algorithms
 - [OpenViking](https://github.com/volcengine/OpenViking) — query decomposition patterns, collection-scoped retrieval, transaction-safe indexing

package/SKILL.md

@@ -272,7 +272,7 @@ Once escalated, route by query type:
 | `beads_sync` | Sync Beads issues from Dolt backend into memory. |
 | `index_stats` | Doc counts, embedding coverage, content type distribution. |
 | `status` | Quick index health. |
-| `reindex` | Force re-index (BM25 only, does NOT embed). |
+| `reindex` | Force re-index (BM25 only, does NOT embed). Use `--enrich` after major upgrades to backfill entity extraction + links on existing docs. |
 | `memory_evolution_status` | Track how a doc's A-MEM metadata evolved over time. |
 | `timeline` | Temporal neighborhood around a document — what was modified before/after. Progressive disclosure: search → timeline → get. Supports same-collection scoping and session correlation. |
 | `list_vaults` | Show configured vault names and paths. Empty in single-vault mode. |
@@ -660,6 +660,14 @@ echo "user query" | clawmem surface --context --stdin
 echo "session-id" | clawmem surface --bootstrap --stdin
 ```
 
+### Enrichment Commands
+
+```bash
+clawmem reindex --enrich        # Full A-MEM pipeline on ALL docs (entity extraction,
+                                # link generation, memory evolution). Use after major upgrades.
+                                # Without --enrich, reindex only refreshes metadata for changed docs.
+```
+
 ### Analysis Commands
 
 ```bash

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clawmem",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "description": "On-device context engine and memory for AI agents. Claude Code and OpenClaw. Hooks + MCP server + hybrid RAG search.",
   "type": "module",
   "bin": {

package/src/clawmem.ts

@@ -1269,6 +1269,7 @@ async function cmdWatch() {
 
 async function cmdReindex(args: string[]) {
   const force = args.includes("--force") || args.includes("-f");
+  const enrich = args.includes("--enrich");
   const collections = collectionsList();
 
   if (collections.length === 0) {
@@ -1283,9 +1284,13 @@ async function cmdReindex(args: string[]) {
     console.log(`${c.yellow}Force reindex: deactivated all documents${c.reset}`);
   }
 
+  if (enrich) {
+    console.log(`${c.cyan}Full enrichment: entity extraction + links + evolution for all documents${c.reset}`);
+  }
+
   for (const col of collections) {
     console.log(`Indexing ${c.bold}${col.name}${c.reset} (${col.path})...`);
-    const stats = await indexCollection(s, col.name, col.path, col.pattern);
+    const stats = await indexCollection(s, col.name, col.path, col.pattern, { forceEnrich: enrich });
     console.log(`  +${stats.added} added, ~${stats.updated} updated, =${stats.unchanged} unchanged, -${stats.removed} removed`);
   }
 }
@@ -2202,7 +2207,7 @@ ${c.bold}Setup:${c.reset}
 ${c.bold}Indexing:${c.reset}
   clawmem update [--pull] [--embed]    Re-scan collections (--embed auto-embeds)
   clawmem embed [-f]                   Generate fragment embeddings
-  clawmem reindex [--force]            Full re-index
+  clawmem reindex [--force] [--enrich]  Full re-index (--enrich: run entity extraction + links on all docs)
   clawmem watch                        File watcher daemon
   clawmem status                       Show index status
 

package/src/entity.ts

@@ -8,6 +8,7 @@
  */
 
 import type { Database } from "bun:sqlite";
+import { createHash } from "crypto";
 import type { LlamaCpp } from "./llm.ts";
 import { extractJsonFromLLM } from "./amem.ts";
 
@@ -299,12 +300,63 @@ export function trackCoOccurrences(
 // Entity Enrichment Pipeline (called during A-MEM postIndexEnrich)
 // =============================================================================
 
+/**
+ * Compute extraction input hash from title + body.
+ * Captures the actual input to the LLM prompt — changes to either trigger re-extraction.
+ */
+function computeInputHash(title: string, body: string): string {
+  return createHash('sha256').update(title + '\0' + body).digest('hex');
+}
+
+/**
+ * Clear all derived entity state for a document:
+ * mentions, co-occurrence contributions, entity edges, and mention counts.
+ */
+function clearDocEntityState(db: Database, docId: number): void {
+  // Get entity IDs this doc mentions (before deletion)
+  const oldMentions = db.prepare(
+    `SELECT entity_id FROM entity_mentions WHERE doc_id = ?`
+  ).all(docId) as { entity_id: string }[];
+
+  // Delete mentions
+  db.prepare(`DELETE FROM entity_mentions WHERE doc_id = ?`).run(docId);
+
+  // Decrement mention_count for each entity
+  for (const m of oldMentions) {
+    db.prepare(`
+      UPDATE entity_nodes SET mention_count = MAX(0, mention_count - 1) WHERE entity_id = ?
+    `).run(m.entity_id);
+  }
+
+  // Remove entity edges involving this doc
+  db.prepare(`
+    DELETE FROM memory_relations WHERE (source_id = ? OR target_id = ?) AND relation_type = 'entity'
+  `).run(docId, docId);
+
+  // Decrement co-occurrence counts for entity pairs from this doc
+  if (oldMentions.length >= 2) {
+    const ids = oldMentions.map(m => m.entity_id);
+    for (let i = 0; i < ids.length; i++) {
+      for (let j = i + 1; j < ids.length; j++) {
+        const sorted = [ids[i]!, ids[j]!].sort();
+        db.prepare(`
+          UPDATE entity_cooccurrences SET count = MAX(0, count - 1)
+          WHERE entity_a = ? AND entity_b = ?
+        `).run(sorted[0]!, sorted[1]!);
+      }
+    }
+    // Clean up zero-count rows
+    db.prepare(`DELETE FROM entity_cooccurrences WHERE count <= 0`).run();
+  }
+}
+
 /**
  * Full entity enrichment for a document:
- * 1. Extract entities via LLM
- * 2. Resolve each to canonical form
- * 3. Record mentions
- * 4. Track co-occurrences
+ * 1. Check enrichment state (skip if input unchanged)
+ * 2. Extract entities via LLM
+ * 3. Resolve each to canonical form
+ * 4. Record mentions + co-occurrences + entity edges
+ * 5. Persist enrichment state for idempotency
  *
  * @returns Number of entities resolved
  */
@@ -315,7 +367,7 @@ export async function enrichDocumentEntities(
   vault: string = 'default'
 ): Promise<number> {
   try {
-    // Get document content
+    // Get document content (snapshot for extraction)
     const doc = db.prepare(`
       SELECT d.title, c.doc as body
       FROM documents d
@@ -328,14 +380,34 @@ export async function enrichDocumentEntities(
       return 0;
     }
 
-    // Step 1: Extract entities
+    // Compute extraction input hash (title + body — the actual LLM prompt input)
+    const inputHash = computeInputHash(doc.title, doc.body);
+
+    // Check enrichment state — skip if already enriched with same input
+    const existingState = db.prepare(
+      `SELECT input_hash FROM entity_enrichment_state WHERE doc_id = ?`
+    ).get(docId) as { input_hash: string } | undefined;
+
+    if (existingState?.input_hash === inputHash) {
+      return 0; // Same input, already enriched — skip
+    }
+
+    // Step 1: Extract entities via LLM
     const entities = await extractEntities(llm, doc.title, doc.body);
-    if (entities.length === 0) {
-      console.log(`[entity] No entities found in docId ${docId}`);
+
+    // Recheck input hash before writing — abort if content changed during LLM call
+    const recheckHash = db.prepare(`
+      SELECT d.title, c.doc as body FROM documents d
+      JOIN content c ON c.hash = d.hash WHERE d.id = ? AND d.active = 1
+    `).get(docId) as { title: string; body: string } | null;
+
+    if (!recheckHash || computeInputHash(recheckHash.title, recheckHash.body) !== inputHash) {
+      console.log(`[entity] Document ${docId} changed during extraction — aborting`);
       return 0;
     }
 
-    // Step 2-3: Deduplicate entities by name+type, then resolve and record mentions
+    // Step 3: Deduplicate entities by surface form, then resolve canonical IDs
+    // Done BEFORE transaction to avoid calling upsertEntity (which mutates counters) for dupes
     const seenKeys = new Set<string>();
     const uniqueEntities: ExtractedEntity[] = [];
     for (const entity of entities) {
@@ -346,36 +418,87 @@ export async function enrichDocumentEntities(
       }
     }
 
-    const resolvedIds: string[] = [];
+    // Resolve canonical IDs first (read-only lookups, no counter mutation yet)
+    const resolvedPairs: { entity: ExtractedEntity; canonicalId: string }[] = [];
+    const seenCanonicalIds = new Set<string>();
     for (const entity of uniqueEntities) {
-      const entityId = upsertEntity(db, entity.name, entity.type, vault);
-      resolvedIds.push(entityId);
-      recordEntityMention(db, entityId, docId, entity.name);
+      const canonicalId = resolveEntityCanonical(db, entity.name, entity.type, vault)
+        || makeEntityId(entity.name, entity.type, vault);
+      if (!seenCanonicalIds.has(canonicalId)) {
+        seenCanonicalIds.add(canonicalId);
+        resolvedPairs.push({ entity, canonicalId });
+      }
     }
 
-    // Step 4: Track co-occurrences (deduplicated IDs prevent inflated pair counts)
-    trackCoOccurrences(db, resolvedIds);
+    // All writes in a transaction — partial failure rolls back cleanly
+    try {
+      db.exec("BEGIN");
+
+      // Re-check enrichment state inside transaction (prevents concurrent overcount)
+      const txState = db.prepare(
+        `SELECT input_hash FROM entity_enrichment_state WHERE doc_id = ?`
+      ).get(docId) as { input_hash: string } | undefined;
 
-    // Step 5: Create entity edges in memory_relations
-    for (const entityId of resolvedIds) {
-      // Find other documents mentioning this entity
-      const otherDocs = db.prepare(`
-        SELECT doc_id FROM entity_mentions
-        WHERE entity_id = ? AND doc_id != ?
-        LIMIT 10
-      `).all(entityId, docId) as { doc_id: number }[];
+      if (txState?.input_hash === inputHash) {
+        db.exec("ROLLBACK");
+        return 0; // Another worker already committed this exact enrichment
+      }
+
+      // Clear old derived state if re-enriching (content changed)
+      if (txState || existingState) {
+        clearDocEntityState(db, docId);
+      }
 
-      for (const other of otherDocs) {
-        // Insert entity relation (unidirectional; graph traversal handles inbound for entity/semantic types)
+      if (entities.length === 0) {
         db.prepare(`
-          INSERT OR IGNORE INTO memory_relations (source_id, target_id, relation_type, weight, metadata, created_at)
-          VALUES (?, ?, 'entity', 0.7, ?, datetime('now'))
-        `).run(docId, other.doc_id, JSON.stringify({ entity: entityId }));
+          INSERT OR REPLACE INTO entity_enrichment_state (doc_id, input_hash, enriched_at)
+          VALUES (?, ?, datetime('now'))
+        `).run(docId, inputHash);
+        db.exec("COMMIT");
+        console.log(`[entity] No entities found in docId ${docId}`);
+        return 0;
       }
-    }
 
-    console.log(`[entity] Enriched docId ${docId}: ${resolvedIds.length} entities, ${entities.length} extracted`);
-    return resolvedIds.length;
+      // Now mutate counters — one upsert per unique canonical ID (no inflation)
+      const resolvedIds: string[] = [];
+      for (const { entity, canonicalId } of resolvedPairs) {
+        const entityId = upsertEntity(db, entity.name, entity.type, vault);
+        resolvedIds.push(entityId);
+        recordEntityMention(db, entityId, docId, entity.name);
+      }
+
+      // Step 4: Track co-occurrences (deduplicated by canonical ID)
+      trackCoOccurrences(db, resolvedIds);
+
+      // Step 5: Create entity edges in memory_relations
+      for (const entityId of resolvedIds) {
+        const otherDocs = db.prepare(`
+          SELECT doc_id FROM entity_mentions
+          WHERE entity_id = ? AND doc_id != ?
+          LIMIT 10
+        `).all(entityId, docId) as { doc_id: number }[];
+
+        for (const other of otherDocs) {
+          db.prepare(`
+            INSERT OR IGNORE INTO memory_relations (source_id, target_id, relation_type, weight, metadata, created_at)
+            VALUES (?, ?, 'entity', 0.7, ?, datetime('now'))
+          `).run(docId, other.doc_id, JSON.stringify({ entity: entityId }));
+        }
+      }
+
+      // Persist enrichment state LAST — only after all derived data written
+      db.prepare(`
+        INSERT OR REPLACE INTO entity_enrichment_state (doc_id, input_hash, enriched_at)
+        VALUES (?, ?, datetime('now'))
+      `).run(docId, inputHash);
+
+      db.exec("COMMIT");
+      console.log(`[entity] Enriched docId ${docId}: ${resolvedIds.length} entities, ${entities.length} extracted`);
+      return resolvedIds.length;
+    } catch (txErr) {
+      try { db.exec("ROLLBACK"); } catch { /* already rolled back */ }
+      throw txErr; // re-throw to outer catch
+    }
   } catch (err) {
     console.log(`[entity] Error enriching docId ${docId}:`, err);
     return 0;

package/src/indexer.ts

@@ -166,7 +166,8 @@ export async function indexCollection(
   store: Store,
   collectionName: string,
   collectionPath: string,
-  pattern: string = "**/*.md"
+  pattern: string = "**/*.md",
+  options?: { forceEnrich?: boolean }
 ): Promise<IndexStats> {
   const stats: IndexStats = { added: 0, updated: 0, unchanged: 0, removed: 0 };
   const activePaths = new Set<string>();
@@ -223,6 +224,10 @@ export async function indexCollection(
 
         if (existingRow?.content_hash === contentHash) {
           stats.unchanged++;
+          if (options?.forceEnrich) {
+            // --enrich: queue unchanged docs for full enrichment (entity extraction, links)
+            enrichQueue.push({ docId: existing.id, isNew: true });
+          }
           continue;
         }
 
@@ -319,8 +324,9 @@ export async function indexCollection(
   }
 
   // A-MEM enrichment runs after successful commit (LLM calls should not block transaction)
+  // forceEnrich overrides isNew to true — triggers full pipeline (entity extraction, links, evolution)
   for (const { docId, isNew } of enrichQueue) {
-    await store.postIndexEnrich(llm, docId, isNew);
+    await store.postIndexEnrich(llm, docId, options?.forceEnrich ? true : isNew);
   }
 
   return stats;

package/src/store.ts

@@ -711,6 +711,16 @@ function initializeDatabase(db: Database): void {
   // Entity FTS5 for fuzzy name lookup
   db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS entities_fts USING fts5(entity_id, name, entity_type)`);
 
+  // Entity enrichment state: tracks what input was used for extraction (idempotent --enrich)
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS entity_enrichment_state (
+      doc_id INTEGER PRIMARY KEY,
+      input_hash TEXT NOT NULL,
+      enriched_at TEXT NOT NULL,
+      FOREIGN KEY (doc_id) REFERENCES documents(id) ON DELETE CASCADE
+    )
+  `);
+
   // 3-tier consolidation: observations synthesized from clusters of related facts
   db.exec(`
     CREATE TABLE IF NOT EXISTS consolidated_observations (