clawmem 0.2.1 → 0.2.2

package/README.md

@@ -44,8 +44,6 @@ Runs fully local with no API keys and no cloud services. Integrates via Claude C
 
 ### v0.2.0 Enhancements
 
-Seven patterns extracted from competitor analysis ([Hindsight](https://github.com/vectorize-io/hindsight), [Hermes Agent](https://github.com/NousResearch/hermes-agent), [claude-mem](https://github.com/thedotmack/claude-mem)):
-
 - **Entity resolution + co-occurrence graph** — LLM entity extraction during A-MEM enrichment, canonical normalization via FTS5 + Levenshtein fuzzy matching, co-occurrence tracking, entity graph traversal for ENTITY intent queries
 - **MPFP graph retrieval** — Multi-Path Fact Propagation with meta-path patterns per intent, hop-synchronized edge cache, Forward Push with α=0.15 teleport probability. Replaces single-beam traversal for causal/entity/temporal queries.
 - **Temporal query extraction** — regex-based date range extraction from natural language queries ("last week", "March 2026"), wired as WHERE filters into BM25 and vector search
@@ -1046,6 +1044,8 @@ Built on the shoulders of:
 - [Beads](https://github.com/steveyegge/beads) — Dolt-backed issue tracker for AI agents
 - [claude-mem](https://github.com/thedotmack/claude-mem) — Claude Code memory integration reference
 - [Engram](https://github.com/Gentleman-Programming/engram) — observation dedup window, topic-key upsert pattern, temporal timeline navigation, duplicate metadata scoring signals
+- [Hermes Agent](https://github.com/NousResearch/hermes-agent) — memory nudge system (periodic lifecycle tool prompting)
+- [Hindsight](https://github.com/vectorize-io/hindsight) — entity resolution, MPFP graph traversal, temporal extraction, 3-tier consolidation, observation invalidation, 4-way parallel retrieval
 - [MAGMA](https://arxiv.org/abs/2501.13956) — multi-graph memory agent
 - [memory-lancedb-pro](https://github.com/CortexReach/memory-lancedb-pro) — retrieval gate, length normalization, MMR diversity, access reinforcement algorithms
 - [OpenViking](https://github.com/volcengine/OpenViking) — query decomposition patterns, collection-scoped retrieval, transaction-safe indexing

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clawmem",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "On-device context engine and memory for AI agents. Claude Code and OpenClaw. Hooks + MCP server + hybrid RAG search.",
   "type": "module",
   "bin": {

package/src/entity.ts

@@ -8,6 +8,7 @@
  */
 
 import type { Database } from "bun:sqlite";
+import { createHash } from "crypto";
 import type { LlamaCpp } from "./llm.ts";
 import { extractJsonFromLLM } from "./amem.ts";
 
@@ -299,12 +300,63 @@ export function trackCoOccurrences(
 // Entity Enrichment Pipeline (called during A-MEM postIndexEnrich)
 // =============================================================================
 
+/**
+ * Compute extraction input hash from title + body.
+ * Captures the actual input to the LLM prompt — changes to either trigger re-extraction.
+ */
+function computeInputHash(title: string, body: string): string {
+  return createHash('sha256').update(title + '\0' + body).digest('hex');
+}
+
+/**
+ * Clear all derived entity state for a document:
+ * mentions, co-occurrence contributions, entity edges, and mention counts.
+ */
+function clearDocEntityState(db: Database, docId: number): void {
+  // Get entity IDs this doc mentions (before deletion)
+  const oldMentions = db.prepare(
+    `SELECT entity_id FROM entity_mentions WHERE doc_id = ?`
+  ).all(docId) as { entity_id: string }[];
+
+  // Delete mentions
+  db.prepare(`DELETE FROM entity_mentions WHERE doc_id = ?`).run(docId);
+
+  // Decrement mention_count for each entity
+  for (const m of oldMentions) {
+    db.prepare(`
+      UPDATE entity_nodes SET mention_count = MAX(0, mention_count - 1) WHERE entity_id = ?
+    `).run(m.entity_id);
+  }
+
+  // Remove entity edges involving this doc
+  db.prepare(`
+    DELETE FROM memory_relations WHERE (source_id = ? OR target_id = ?) AND relation_type = 'entity'
+  `).run(docId, docId);
+
+  // Decrement co-occurrence counts for entity pairs from this doc
+  if (oldMentions.length >= 2) {
+    const ids = oldMentions.map(m => m.entity_id);
+    for (let i = 0; i < ids.length; i++) {
+      for (let j = i + 1; j < ids.length; j++) {
+        const sorted = [ids[i]!, ids[j]!].sort();
+        db.prepare(`
+          UPDATE entity_cooccurrences SET count = MAX(0, count - 1)
+          WHERE entity_a = ? AND entity_b = ?
+        `).run(sorted[0]!, sorted[1]!);
+      }
+    }
+    // Clean up zero-count rows
+    db.prepare(`DELETE FROM entity_cooccurrences WHERE count <= 0`).run();
+  }
+}
+
 /**
  * Full entity enrichment for a document:
- * 1. Extract entities via LLM
- * 2. Resolve each to canonical form
- * 3. Record mentions
- * 4. Track co-occurrences
+ * 1. Check enrichment state (skip if input unchanged)
+ * 2. Extract entities via LLM
+ * 3. Resolve each to canonical form
+ * 4. Record mentions + co-occurrences + entity edges
+ * 5. Persist enrichment state for idempotency
  *
  * @returns Number of entities resolved
  */
@@ -315,7 +367,7 @@ export async function enrichDocumentEntities(
   vault: string = 'default'
 ): Promise<number> {
   try {
-    // Get document content
+    // Get document content (snapshot for extraction)
     const doc = db.prepare(`
       SELECT d.title, c.doc as body
       FROM documents d
@@ -328,14 +380,34 @@ export async function enrichDocumentEntities(
       return 0;
     }
 
-    // Step 1: Extract entities
+    // Compute extraction input hash (title + body — the actual LLM prompt input)
+    const inputHash = computeInputHash(doc.title, doc.body);
+
+    // Check enrichment state — skip if already enriched with same input
+    const existingState = db.prepare(
+      `SELECT input_hash FROM entity_enrichment_state WHERE doc_id = ?`
+    ).get(docId) as { input_hash: string } | undefined;
+
+    if (existingState?.input_hash === inputHash) {
+      return 0; // Same input, already enriched — skip
+    }
+
+    // Step 1: Extract entities via LLM
     const entities = await extractEntities(llm, doc.title, doc.body);
-    if (entities.length === 0) {
-      console.log(`[entity] No entities found in docId ${docId}`);
+
+    // Recheck input hash before writing — abort if content changed during LLM call
+    const recheckHash = db.prepare(`
+      SELECT d.title, c.doc as body FROM documents d
+      JOIN content c ON c.hash = d.hash WHERE d.id = ? AND d.active = 1
+    `).get(docId) as { title: string; body: string } | null;
+
+    if (!recheckHash || computeInputHash(recheckHash.title, recheckHash.body) !== inputHash) {
+      console.log(`[entity] Document ${docId} changed during extraction — aborting`);
       return 0;
     }
 
-    // Step 2-3: Deduplicate entities by name+type, then resolve and record mentions
+    // Step 3: Deduplicate entities by surface form, then resolve canonical IDs
+    // Done BEFORE transaction to avoid calling upsertEntity (which mutates counters) for dupes
     const seenKeys = new Set<string>();
     const uniqueEntities: ExtractedEntity[] = [];
     for (const entity of entities) {
@@ -346,36 +418,87 @@ export async function enrichDocumentEntities(
       }
     }
 
-    const resolvedIds: string[] = [];
+    // Resolve canonical IDs first (read-only lookups, no counter mutation yet)
+    const resolvedPairs: { entity: ExtractedEntity; canonicalId: string }[] = [];
+    const seenCanonicalIds = new Set<string>();
     for (const entity of uniqueEntities) {
-      const entityId = upsertEntity(db, entity.name, entity.type, vault);
-      resolvedIds.push(entityId);
-      recordEntityMention(db, entityId, docId, entity.name);
+      const canonicalId = resolveEntityCanonical(db, entity.name, entity.type, vault)
+        || makeEntityId(entity.name, entity.type, vault);
+      if (!seenCanonicalIds.has(canonicalId)) {
+        seenCanonicalIds.add(canonicalId);
+        resolvedPairs.push({ entity, canonicalId });
+      }
     }
 
-    // Step 4: Track co-occurrences (deduplicated IDs prevent inflated pair counts)
-    trackCoOccurrences(db, resolvedIds);
+    // All writes in a transaction — partial failure rolls back cleanly
+    try {
+      db.exec("BEGIN");
+
+      // Re-check enrichment state inside transaction (prevents concurrent overcount)
+      const txState = db.prepare(
+        `SELECT input_hash FROM entity_enrichment_state WHERE doc_id = ?`
+      ).get(docId) as { input_hash: string } | undefined;
 
-    // Step 5: Create entity edges in memory_relations
-    for (const entityId of resolvedIds) {
-      // Find other documents mentioning this entity
-      const otherDocs = db.prepare(`
-        SELECT doc_id FROM entity_mentions
-        WHERE entity_id = ? AND doc_id != ?
-        LIMIT 10
-      `).all(entityId, docId) as { doc_id: number }[];
+      if (txState?.input_hash === inputHash) {
+        db.exec("ROLLBACK");
+        return 0; // Another worker already committed this exact enrichment
+      }
+
+      // Clear old derived state if re-enriching (content changed)
+      if (txState || existingState) {
+        clearDocEntityState(db, docId);
+      }
 
-      for (const other of otherDocs) {
-        // Insert entity relation (unidirectional; graph traversal handles inbound for entity/semantic types)
+      if (entities.length === 0) {
         db.prepare(`
-          INSERT OR IGNORE INTO memory_relations (source_id, target_id, relation_type, weight, metadata, created_at)
-          VALUES (?, ?, 'entity', 0.7, ?, datetime('now'))
-        `).run(docId, other.doc_id, JSON.stringify({ entity: entityId }));
+          INSERT OR REPLACE INTO entity_enrichment_state (doc_id, input_hash, enriched_at)
+          VALUES (?, ?, datetime('now'))
+        `).run(docId, inputHash);
+        db.exec("COMMIT");
+        console.log(`[entity] No entities found in docId ${docId}`);
+        return 0;
       }
-    }
 
-    console.log(`[entity] Enriched docId ${docId}: ${resolvedIds.length} entities, ${entities.length} extracted`);
-    return resolvedIds.length;
+      // Now mutate counters — one upsert per unique canonical ID (no inflation)
+      const resolvedIds: string[] = [];
+      for (const { entity, canonicalId } of resolvedPairs) {
+        const entityId = upsertEntity(db, entity.name, entity.type, vault);
+        resolvedIds.push(entityId);
+        recordEntityMention(db, entityId, docId, entity.name);
+      }
+
+      // Step 4: Track co-occurrences (deduplicated by canonical ID)
+      trackCoOccurrences(db, resolvedIds);
+
+      // Step 5: Create entity edges in memory_relations
+      for (const entityId of resolvedIds) {
+        const otherDocs = db.prepare(`
+          SELECT doc_id FROM entity_mentions
+          WHERE entity_id = ? AND doc_id != ?
+          LIMIT 10
+        `).all(entityId, docId) as { doc_id: number }[];
+
+        for (const other of otherDocs) {
+          db.prepare(`
+            INSERT OR IGNORE INTO memory_relations (source_id, target_id, relation_type, weight, metadata, created_at)
+            VALUES (?, ?, 'entity', 0.7, ?, datetime('now'))
+          `).run(docId, other.doc_id, JSON.stringify({ entity: entityId }));
+        }
+      }
+
+      // Persist enrichment state LAST — only after all derived data written
+      db.prepare(`
+        INSERT OR REPLACE INTO entity_enrichment_state (doc_id, input_hash, enriched_at)
+        VALUES (?, ?, datetime('now'))
+      `).run(docId, inputHash);
+
+      db.exec("COMMIT");
+      console.log(`[entity] Enriched docId ${docId}: ${resolvedIds.length} entities, ${entities.length} extracted`);
+      return resolvedIds.length;
+    } catch (txErr) {
+      try { db.exec("ROLLBACK"); } catch { /* already rolled back */ }
+      throw txErr; // re-throw to outer catch
+    }
   } catch (err) {
     console.log(`[entity] Error enriching docId ${docId}:`, err);
     return 0;

package/src/store.ts

@@ -711,6 +711,16 @@ function initializeDatabase(db: Database): void {
   // Entity FTS5 for fuzzy name lookup
   db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS entities_fts USING fts5(entity_id, name, entity_type)`);
 
+  // Entity enrichment state: tracks what input was used for extraction (idempotent --enrich)
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS entity_enrichment_state (
+      doc_id INTEGER PRIMARY KEY,
+      input_hash TEXT NOT NULL,
+      enriched_at TEXT NOT NULL,
+      FOREIGN KEY (doc_id) REFERENCES documents(id) ON DELETE CASCADE
+    )
+  `);
+
   // 3-tier consolidation: observations synthesized from clusters of related facts
   db.exec(`
     CREATE TABLE IF NOT EXISTS consolidated_observations (