clawmem 0.5.1 → 0.6.0

package/AGENTS.md

@@ -250,7 +250,7 @@ ClawMem hooks handle ~90% of retrieval automatically. Agent-initiated MCP calls
 | `postcompact-inject` | SessionStart (compact) | 1200 tokens | re-injects authoritative context after compaction: precompact state (600) + recent decisions (400) + antipatterns (150) + vault context (200) → `<vault-postcompact>` |
 | `curator-nudge` | SessionStart | 200 tokens | surfaces curator report actions, nudges when report is stale (>7 days) |
 | `precompact-extract` | PreCompact | — | extracts decisions, file paths, open questions → writes `precompact-state.md` to auto-memory. Query-aware decision ranking. Reindexes auto-memory collection. |
-| `decision-extractor` | Stop | — | LLM extracts observations → `_clawmem/agent/observations/`, infers causal links, detects contradictions with prior decisions |
+| `decision-extractor` | Stop | — | LLM extracts observations → `_clawmem/agent/observations/`, infers causal links, detects contradictions, extracts SPO triples from decision/preference/milestone/problem facts. Background consolidation worker synthesizes deductive observations from related facts (Phase 3, every ~15 min). |
 | `handoff-generator` | Stop | — | LLM summarizes session → `_clawmem/agent/handoffs/` |
 | `feedback-loop` | Stop | — | tracks referenced notes → boosts confidence, records usage relations + co-activations between co-referenced docs, tracks utility signals (surfaced vs referenced ratio for lifecycle automation) |
 
@@ -447,7 +447,7 @@ compositeScore = (0.10 × searchScore + 0.70 × recencyScore + 0.20 × confidenc
 
 | Content Type | Half-Life | Effect |
 |--------------|-----------|--------|
-| decision, preference, hub | ∞ | Never decay |
+| decision, deductive, preference, hub | ∞ | Never decay |
 | antipattern | ∞ | Never decay — accumulated negative patterns persist |
 | project | 120 days | Slow decay |
 | research | 90 days | Moderate decay |
@@ -456,7 +456,7 @@ compositeScore = (0.10 × searchScore + 0.70 × recencyScore + 0.20 × confidenc
 | handoff | 30 days | Fast — recent matters most |
 
 Half-lives extend up to 3× for frequently-accessed memories (access reinforcement decays over 90 days).
-Attention decay: non-durable types (handoff, progress, conversation, note, project) lose 5% confidence per week without access. Decision/preference/hub/research/antipattern are exempt.
+Attention decay: non-durable types (handoff, progress, conversation, note, project) lose 5% confidence per week without access. Decision/deductive/preference/hub/research/antipattern are exempt.
 
 ## Indexing & Graph Building
 
@@ -499,6 +499,7 @@ The `memory_relations` table is populated by multiple independent sources:
 | `buildSemanticGraph()` | semantic | `build_graphs` MCP tool (manual) | Pure cosine similarity. PK collision: `INSERT OR IGNORE` means A-MEM semantic edges take precedence if they exist first. |
 | Entity co-occurrence graph | entity | A-MEM enrichment (indexing) | LLM entity extraction → quality filters (title/length/blocklist/location validation) → type-agnostic canonical resolution within compatibility buckets (person, org, location, tech=project/service/tool/concept) → `entity_mentions` + `entity_cooccurrences` tables. Entity edges use IDF-based specificity scoring. Feeds ENTITY intent queries and MPFP `[entity, semantic]` patterns. |
 | `consolidated_observations` | supporting | Consolidation worker (background) | 3-tier consolidation: facts → observations → mental models. Observations track `proof_count`, `trend` (STABLE/STRENGTHENING/WEAKENING/STALE), and source links. |
+| Deductive synthesis | supporting | Consolidation worker Phase 3 (background, every ~15 min) | Combines 2-3 related recent observations (decision/preference/milestone/problem, last 7 days) into `content_type='deductive'` documents with `source_doc_ids` provenance. First-class searchable docs with ∞ half-life. |
 
 **Edge collision:** Both `generateMemoryLinks()` and `buildSemanticGraph()` insert `relation_type='semantic'`. PK is `(source_id, target_id, relation_type)` — first writer wins.
 

package/CLAUDE.md

@@ -250,7 +250,7 @@ ClawMem hooks handle ~90% of retrieval automatically. Agent-initiated MCP calls
 | `postcompact-inject` | SessionStart (compact) | 1200 tokens | re-injects authoritative context after compaction: precompact state (600) + recent decisions (400) + antipatterns (150) + vault context (200) → `<vault-postcompact>` |
 | `curator-nudge` | SessionStart | 200 tokens | surfaces curator report actions, nudges when report is stale (>7 days) |
 | `precompact-extract` | PreCompact | — | extracts decisions, file paths, open questions → writes `precompact-state.md` to auto-memory. Query-aware decision ranking. Reindexes auto-memory collection. |
-| `decision-extractor` | Stop | — | LLM extracts observations → `_clawmem/agent/observations/`, infers causal links, detects contradictions with prior decisions |
+| `decision-extractor` | Stop | — | LLM extracts observations → `_clawmem/agent/observations/`, infers causal links, detects contradictions, extracts SPO triples from decision/preference/milestone/problem facts. Background consolidation worker synthesizes deductive observations from related facts (Phase 3, every ~15 min). |
 | `handoff-generator` | Stop | — | LLM summarizes session → `_clawmem/agent/handoffs/` |
 | `feedback-loop` | Stop | — | tracks referenced notes → boosts confidence, records usage relations + co-activations between co-referenced docs, tracks utility signals (surfaced vs referenced ratio for lifecycle automation) |
 
@@ -447,7 +447,7 @@ compositeScore = (0.10 × searchScore + 0.70 × recencyScore + 0.20 × confidenc
 
 | Content Type | Half-Life | Effect |
 |--------------|-----------|--------|
-| decision, preference, hub | ∞ | Never decay |
+| decision, deductive, preference, hub | ∞ | Never decay |
 | antipattern | ∞ | Never decay — accumulated negative patterns persist |
 | project | 120 days | Slow decay |
 | research | 90 days | Moderate decay |
@@ -456,7 +456,7 @@ compositeScore = (0.10 × searchScore + 0.70 × recencyScore + 0.20 × confidenc
 | handoff | 30 days | Fast — recent matters most |
 
 Half-lives extend up to 3× for frequently-accessed memories (access reinforcement decays over 90 days).
-Attention decay: non-durable types (handoff, progress, conversation, note, project) lose 5% confidence per week without access. Decision/preference/hub/research/antipattern are exempt.
+Attention decay: non-durable types (handoff, progress, conversation, note, project) lose 5% confidence per week without access. Decision/deductive/preference/hub/research/antipattern are exempt.
 
 ## Indexing & Graph Building
 
@@ -499,6 +499,7 @@ The `memory_relations` table is populated by multiple independent sources:
 | `buildSemanticGraph()` | semantic | `build_graphs` MCP tool (manual) | Pure cosine similarity. PK collision: `INSERT OR IGNORE` means A-MEM semantic edges take precedence if they exist first. |
 | Entity co-occurrence graph | entity | A-MEM enrichment (indexing) | LLM entity extraction → quality filters (title/length/blocklist/location validation) → type-agnostic canonical resolution within compatibility buckets (person, org, location, tech=project/service/tool/concept) → `entity_mentions` + `entity_cooccurrences` tables. Entity edges use IDF-based specificity scoring. Feeds ENTITY intent queries and MPFP `[entity, semantic]` patterns. |
 | `consolidated_observations` | supporting | Consolidation worker (background) | 3-tier consolidation: facts → observations → mental models. Observations track `proof_count`, `trend` (STABLE/STRENGTHENING/WEAKENING/STALE), and source links. |
+| Deductive synthesis | supporting | Consolidation worker Phase 3 (background, every ~15 min) | Combines 2-3 related recent observations (decision/preference/milestone/problem, last 7 days) into `content_type='deductive'` documents with `source_doc_ids` provenance. First-class searchable docs with ∞ half-life. |
 
 **Edge collision:** Both `generateMemoryLinks()` and `buildSemanticGraph()` insert `relation_type='semantic'`. PK is `(source_id, target_id, relation_type)` — first writer wins.
 

package/README.md

@@ -823,6 +823,7 @@ For WHY and ENTITY queries, the search pipeline expands results through the memo
 | Type | Half-life | Baseline | Notes |
 |---|---|---|---|
 | `decision` | ∞ | 0.85 | Never decays |
+| `deductive` | ∞ | 0.85 | Never decays — cross-session derived insights with source provenance |
 | `preference` | ∞ | 0.80 | Never decays — user preferences are durable facts |
 | `hub` | ∞ | 0.80 | Never decays |
 | `antipattern` | ∞ | 0.75 | Never decays — accumulated negative patterns persist |
@@ -835,7 +836,7 @@ For WHY and ENTITY queries, the search pipeline expands results through the memo
 | `progress` | 45 days | 0.50 | |
 | `note` | 60 days | 0.50 | Default |
 
-Content types are inferred from frontmatter or file path patterns. Half-lives extend up to 3× for frequently-accessed memories (access reinforcement, decays over 90 days). Non-durable types (handoff, progress, conversation, note, project) lose 5% confidence per week without access (attention decay). Decision/preference/hub/research/antipattern are exempt.
+Content types are inferred from frontmatter or file path patterns. Half-lives extend up to 3× for frequently-accessed memories (access reinforcement, decays over 90 days). Non-durable types (handoff, progress, conversation, note, project) lose 5% confidence per week without access (attention decay). Decision/deductive/preference/hub/research/antipattern are exempt.
 
 **Quality scoring:** Each document gets a `quality_score` (0.0–1.0) computed during indexing based on length, structure (headings, lists), decision/correction keywords, and frontmatter richness. Applied as `qualityMultiplier = 0.7 + 0.6 × qualityScore` (range: 0.7× penalty to 1.3× boost).
 
@@ -1119,6 +1120,7 @@ Built on the shoulders of:
 - [Engram](https://github.com/Gentleman-Programming/engram) — observation dedup window, topic-key upsert pattern, temporal timeline navigation, duplicate metadata scoring signals
 - [Hermes Agent](https://github.com/NousResearch/hermes-agent) — MemoryProvider plugin integration, memory nudge system (periodic lifecycle tool prompting)
 - [Hindsight](https://github.com/vectorize-io/hindsight) — entity resolution, MPFP graph traversal, temporal extraction, 3-tier consolidation, observation invalidation, 4-way parallel retrieval
+- [Honcho](https://github.com/plastic-labs/honcho) — deductive observation synthesis patterns, surprisal-based anomaly scoring concept, embed-state self-healing, retrieval separation (raw vs derived)
 - [MAGMA](https://arxiv.org/abs/2501.13956) — multi-graph memory agent
 - [MemPalace](https://github.com/milla-jovovich/mempalace) — conversation import patterns, broadened observation taxonomy (preference/milestone/problem), session-bootstrap synthesis
 - [memory-lancedb-pro](https://github.com/CortexReach/memory-lancedb-pro) — retrieval gate, length normalization, MMR diversity, access reinforcement algorithms

package/SKILL.md

@@ -451,7 +451,7 @@ compositeScore = (0.10 x searchScore + 0.70 x recencyScore + 0.20 x confidenceSc
 
 | Content Type | Half-Life | Effect |
 |--------------|-----------|--------|
-| decision, preference, hub | infinity | Never decay |
+| decision, deductive, preference, hub | infinity | Never decay |
 | antipattern | infinity | Never decay — accumulated negative patterns persist |
 | project | 120 days | Slow decay |
 | research | 90 days | Moderate decay |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clawmem",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "description": "On-device context engine and memory for AI agents. Claude Code and OpenClaw. Hooks + MCP server + hybrid RAG search.",
   "type": "module",
   "bin": {

package/src/clawmem.ts

@@ -410,6 +410,9 @@ async function cmdEmbed(args: string[]) {
 
     const fragments = splitDocument(body, frontmatter);
     const docStart = Date.now();
+    const prevTotalFragments = totalFragments;
+    const prevFailedFragments = failedFragments;
+    let seq0Succeeded = false;
     console.error(`  [${docIdx + 1}/${hashes.length}] ${basename(path)} (${fragments.length} frags, ${body.length} chars)`);
 
     if (isCloudEmbed) {
@@ -463,6 +466,7 @@ async function cmdEmbed(args: string[]) {
                 result.model, new Date().toISOString(), frag.type, frag.label ?? undefined, canId
               );
               totalFragments++;
+              if (seq === 0) seq0Succeeded = true;
             } else {
               failedFragments++;
             }
@@ -491,6 +495,7 @@ async function cmdEmbed(args: string[]) {
               result.model, new Date().toISOString(), frag.type, frag.label ?? undefined, canId
             );
             totalFragments++;
+            if (seq === 0) seq0Succeeded = true;
             if (seq === 0 || (seq + 1) % 5 === 0 || seq === fragments.length - 1) {
               console.error(`    frag ${seq + 1}/${fragments.length} (${frag.type}) ${fragMs}ms [${text.length} chars]`);
             }
@@ -505,6 +510,18 @@ async function cmdEmbed(args: string[]) {
       }
     }
 
+    // Track embed state per document — seq=0 (primary) must succeed for synced status
+    const docFragsOk = totalFragments - prevTotalFragments;
+    const docFragsFail = failedFragments - prevFailedFragments;
+    if (seq0Succeeded) {
+      s.markEmbedSynced(hash);
+    } else if (docFragsOk === 0 && docFragsFail > 0) {
+      s.markEmbedFailed(hash, "all fragments failed");
+    } else {
+      // seq=0 failed but some later fragments succeeded — mark failed so seq=0 gets retried
+      s.markEmbedFailed(hash, "primary fragment (seq=0) failed");
+    }
+
     embedded++;
     const docMs = Date.now() - docStart;
     const elapsed = ((Date.now() - batchStart) / 1000).toFixed(0);

package/src/consolidation.ts

@@ -1,17 +1,21 @@
 /**
  * ClawMem Consolidation Worker
  *
- * Two-phase background worker:
+ * Three-phase background worker:
  * 1. A-MEM backfill: enriches documents missing memory notes
  * 2. 3-tier consolidation: synthesizes clusters of related observations
  *    into higher-order consolidated observations with proof counts and trends
+ * 3. Deductive synthesis: combines related recent observations into
+ *    first-class deductive documents with source provenance
  *
  * Pattern H from ENHANCEMENT-PLAN.md (source: Hindsight consolidator.py)
+ * Deductive synthesis inspired by Honcho's Dreamer deduction specialist.
  */
 
 import type { Store } from "./store.ts";
 import type { LlamaCpp } from "./llm.ts";
 import { extractJsonFromLLM } from "./amem.ts";
+import { hashContent } from "./indexer.ts";
 
 // =============================================================================
 // Types
@@ -115,6 +119,11 @@ async function tick(store: Store, llm: LlamaCpp): Promise<void> {
     if (tickCount % 6 === 0) {
       await consolidateObservations(store, llm);
     }
+
+    // Phase 3: Deductive synthesis (every 3rd tick, ~15 min at default interval)
+    if (tickCount % 3 === 0) {
+      await generateDeductiveObservations(store, llm);
+    }
   } catch (err) {
     console.error("[consolidation] Tick failed:", err);
   } finally {
@@ -375,6 +384,308 @@ function updateTrends(store: Store): void {
   }
 }
 
+// =============================================================================
+// Phase 3: Deductive Observation Synthesis
+// =============================================================================
+
+/**
+ * Find pairs/groups of recent high-confidence observations that can be combined
+ * into higher-level deductive conclusions. Creates first-class documents with
+ * content_type='deductive' and source_doc_ids provenance.
+ *
+ * Only considers decision/preference/milestone/problem observations from the
+ * last 7 days that haven't already been used as sources for deductions.
+ */
+async function generateDeductiveObservations(store: Store, llm: LlamaCpp): Promise<number> {
+  // Find recent high-value observations not yet used in deductions
+  const DEDUCTIVE_TYPES = ['decision', 'preference', 'milestone', 'problem'];
+  const recentObs = store.db.prepare(`
+    SELECT d.id, d.title, d.facts, d.narrative, d.observation_type, d.content_type,
+           d.collection, d.path, d.modified_at
+    FROM documents d
+    WHERE d.active = 1
+      AND d.content_type IN (${DEDUCTIVE_TYPES.map(() => '?').join(',')})
+      AND d.observation_type IS NOT NULL
+      AND d.facts IS NOT NULL
+      AND d.modified_at >= datetime('now', '-7 days')
+      AND d.id NOT IN (
+        SELECT value FROM (
+          SELECT json_each.value as value
+          FROM documents dd, json_each(dd.source_doc_ids)
+          WHERE dd.content_type = 'deductive' AND dd.active = 1
+        )
+      )
+    ORDER BY d.modified_at DESC
+    LIMIT 20
+  `).all(...DEDUCTIVE_TYPES) as {
+    id: number; title: string; facts: string; narrative: string;
+    observation_type: string; content_type: string; collection: string;
+    path: string; modified_at: string;
+  }[];
+
+  if (recentObs.length < 2) return 0;
+
+  // Build context for LLM
+  const obsText = recentObs.map((o, i) =>
+    `[${i + 1}] (${o.content_type}/${o.observation_type}) "${o.title}"\n   Facts: ${(o.facts || '').slice(0, 300)}\n   Narrative: ${(o.narrative || '').slice(0, 200)}`
+  ).join('\n\n');
+
+  const prompt = `You are analyzing recent observations from a developer's work sessions. Find logical deductions that can be drawn by combining 2-3 observations.
+
+A deduction combines facts from different observations into a NEW conclusion that isn't stated in any single observation alone.
+
+Observations:
+${obsText}
+
+For each valid deduction:
+1. State the conclusion clearly (1-2 sentences)
+2. List the premises (which observations support it)
+3. List the source indices (1-indexed)
+
+Return ONLY valid JSON array:
+[
+  {
+    "conclusion": "Clear deductive statement",
+    "premises": ["Premise from obs 1", "Premise from obs 3"],
+    "source_indices": [1, 3]
+  }
+]
+
+Rules:
+- Each deduction MUST combine 2+ different observations (not restate a single one)
+- Only include conclusions with genuine logical basis
+- Maximum 3 deductions
+- If no valid deductions exist, return []
+Return ONLY the JSON array. /no_think`;
+
+  const result = await llm.generate(prompt, { temperature: 0.3, maxTokens: 500 });
+  if (!result?.text) return 0;
+
+  const parsed = extractJsonFromLLM(result.text) as Array<{
+    conclusion: string;
+    premises: string[];
+    source_indices: number[];
+  }> | null;
+
+  if (!Array.isArray(parsed)) return 0;
+
+  let created = 0;
+  const timestamp = new Date().toISOString();
+  const dateStr = timestamp.slice(0, 10);
+
+  for (const deduction of parsed) {
+    if (!deduction.conclusion || !Array.isArray(deduction.source_indices) || deduction.source_indices.length < 2) continue;
+
+    const sourceDocIds = deduction.source_indices
+      .filter(i => i >= 1 && i <= recentObs.length)
+      .map(i => recentObs[i - 1]!.id);
+
+    if (sourceDocIds.length < 2) continue;
+
+    // Check for duplicate deduction (Jaccard on conclusion text)
+    const existingDedups = store.db.prepare(`
+      SELECT id, title FROM documents
+      WHERE content_type = 'deductive' AND active = 1
+      ORDER BY created_at DESC LIMIT 20
+    `).all() as { id: number; title: string }[];
+
+    const conclusionWords = new Set(deduction.conclusion.toLowerCase().split(/\s+/).filter(w => w.length > 3));
+    const isDuplicate = existingDedups.some(d => {
+      const titleWords = new Set(d.title.toLowerCase().split(/\s+/).filter(w => w.length > 3));
+      const intersection = [...conclusionWords].filter(w => titleWords.has(w)).length;
+      const union = new Set([...conclusionWords, ...titleWords]).size;
+      return union > 0 && intersection / union > 0.5;
+    });
+
+    if (isDuplicate) continue;
+
+    // Build the deductive document
+    const premisesText = (deduction.premises || []).map(p => `- ${p}`).join('\n');
+    const sourceRefs = sourceDocIds.map(id => {
+      const obs = recentObs.find(o => o.id === id);
+      return obs ? `- "${obs.title}" (${obs.content_type})` : `- doc#${id}`;
+    }).join('\n');
+
+    const body = [
+      `---`,
+      `content_type: deductive`,
+      `tags: [auto-deduced, consolidation]`,
+      `---`,
+      ``,
+      `# ${deduction.conclusion.slice(0, 80)}`,
+      ``,
+      deduction.conclusion,
+      ``,
+      `## Premises`,
+      ``,
+      premisesText,
+      ``,
+      `## Sources`,
+      ``,
+      sourceRefs,
+      ``,
+    ].join('\n');
+
+    const dedPath = `deductions/${dateStr}-${sourceDocIds.join('-')}.md`;
+    const hash = hashContent(body);
+
+    try {
+      store.insertContent(hash, body, timestamp);
+      store.insertDocument("_clawmem", dedPath, deduction.conclusion.slice(0, 80), hash, timestamp, timestamp);
+
+      const doc = store.findActiveDocument("_clawmem", dedPath);
+      if (doc) {
+        store.updateDocumentMeta(doc.id, {
+          content_type: "deductive",
+          confidence: 0.85,
+        });
+        store.updateObservationFields(dedPath, "_clawmem", {
+          observation_type: "deductive",
+          facts: JSON.stringify(deduction.premises || []),
+          narrative: deduction.conclusion,
+        });
+        // Store source provenance
+        store.db.prepare(`UPDATE documents SET source_doc_ids = ? WHERE id = ?`)
+          .run(JSON.stringify(sourceDocIds), doc.id);
+
+        // Create supporting edges in memory_relations
+        for (const sourceId of sourceDocIds) {
+          try {
+            store.db.prepare(`
+              INSERT OR IGNORE INTO memory_relations (source_id, target_id, relation_type, weight, created_at)
+              VALUES (?, ?, 'supporting', 0.85, datetime('now'))
+            `).run(sourceId, doc.id);
+          } catch { /* non-fatal */ }
+        }
+
+        created++;
+        console.log(`[deductive] Created: "${deduction.conclusion.slice(0, 60)}..." from ${sourceDocIds.length} sources`);
+      }
+    } catch (err) {
+      console.error(`[deductive] Failed to create deduction:`, err);
+    }
+  }
+
+  return created;
+}
+
+/**
+ * Manually trigger deductive synthesis (for CLI or MCP tool).
+ */
+export async function runDeductiveSynthesis(
+  store: Store,
+  llm: LlamaCpp,
+): Promise<{ created: number }> {
+  const created = await generateDeductiveObservations(store, llm);
+  return { created };
+}
+
+// =============================================================================
+// Surprisal Scoring (k-NN density anomaly detection)
+// =============================================================================
+
+export interface SurprisalResult {
+  docId: number;
+  title: string;
+  path: string;
+  collection: string;
+  contentType: string;
+  avgNeighborDistance: number;  // higher = more anomalous
+  neighborCount: number;
+}
+
+/**
+ * Compute surprisal scores for observation documents using k-NN average
+ * neighbor distance in embedding space. High-surprisal observations are
+ * anomalous — they don't fit existing patterns and deserve curator attention.
+ *
+ * Uses sqlite-vec's built-in KNN query (vec0 virtual table) for efficiency.
+ * Only scores documents that have embeddings (content_vectors + vectors_vec).
+ */
+export function computeSurprisalScores(
+  store: Store,
+  options?: { collection?: string; limit?: number; k?: number; minScore?: number }
+): SurprisalResult[] {
+  const k = options?.k ?? 5;
+  const limit = options?.limit ?? 20;
+  const minScore = options?.minScore ?? 0;
+
+  // Get observation documents with embeddings (seq=0 = primary fragment)
+  let sql = `
+    SELECT d.id, d.title, d.path, d.collection, d.content_type,
+           cv.hash || '_0' as hash_seq
+    FROM documents d
+    JOIN content_vectors cv ON d.hash = cv.hash AND cv.seq = 0
+    WHERE d.active = 1
+      AND d.observation_type IS NOT NULL
+  `;
+  const params: any[] = [];
+  if (options?.collection) {
+    sql += ` AND d.collection = ?`;
+    params.push(options.collection);
+  }
+  sql += ` ORDER BY d.modified_at DESC LIMIT 100`;
+
+  const docs = store.db.prepare(sql).all(...params) as {
+    id: number; title: string; path: string; collection: string;
+    content_type: string; hash_seq: string;
+  }[];
+
+  if (docs.length < k + 1) return []; // Not enough docs for meaningful k-NN
+
+  // For each doc, query its k nearest neighbors and compute average distance
+  const results: SurprisalResult[] = [];
+
+  // Check if vectors_vec exists
+  const vecTable = store.db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get();
+  if (!vecTable) return [];
+
+  for (const doc of docs) {
+    try {
+      // Get this doc's embedding from vectors_vec
+      const vecRow = store.db.prepare(
+        `SELECT embedding FROM vectors_vec WHERE hash_seq = ?`
+      ).get(doc.hash_seq) as { embedding: Float32Array | number[] } | null;
+
+      if (!vecRow?.embedding) continue;
+
+      // Query k+1 nearest neighbors (first result is the doc itself)
+      const neighbors = store.db.prepare(`
+        SELECT distance
+        FROM vectors_vec
+        WHERE embedding MATCH ?
+        ORDER BY distance
+        LIMIT ?
+      `).all(vecRow.embedding, k + 1) as { distance: number }[];
+
+      // Skip the first result (self, distance ≈ 0) and compute average
+      const nonSelf = neighbors.filter(n => n.distance > 0.001);
+      if (nonSelf.length === 0) continue;
+
+      const avgDist = nonSelf.reduce((sum, n) => sum + n.distance, 0) / nonSelf.length;
+
+      if (avgDist >= minScore) {
+        results.push({
+          docId: doc.id,
+          title: doc.title,
+          path: doc.path,
+          collection: doc.collection,
+          contentType: doc.content_type,
+          avgNeighborDistance: avgDist,
+          neighborCount: nonSelf.length,
+        });
+      }
+    } catch {
+      // Skip docs that fail vector lookup (missing embedding, dimension mismatch)
+      continue;
+    }
+  }
+
+  // Sort by surprisal (highest first) and limit
+  results.sort((a, b) => b.avgNeighborDistance - a.avgNeighborDistance);
+  return results.slice(0, limit);
+}
+
 // =============================================================================
 // Public API for MCP / CLI
 // =============================================================================

package/src/hooks/session-bootstrap.ts

@@ -260,10 +260,11 @@ function getCurrentFocus(
   cutoff.setDate(cutoff.getDate() - DECISION_LOOKBACK_DAYS);
   const cutoffStr = cutoff.toISOString();
 
-  // Gather recent decisions, preferences, and active problems
+  // Gather recent decisions, preferences, active problems, and deductive insights
   const decisions = store.getDocumentsByType("decision", 10);
   const preferences = store.getDocumentsByType("preference", 5);
   const problems = store.getDocumentsByType("problem", 5);
+  const deductions = store.getDocumentsByType("deductive", 5);
 
   // Rank by: pinned first, then recency, then access_count
   const now = Date.now();
@@ -285,7 +286,11 @@ function getCurrentFocus(
   // Preferences are durable — no date filter, just rank
   const rankedPrefs = [...preferences].sort((a, b) => rankDoc(b) - rankDoc(a));
 
-  if (recentDecisions.length === 0 && rankedPrefs.length === 0 && activeProblems.length === 0) {
+  const recentDeductions = deductions
+    .filter(d => d.modifiedAt >= cutoffStr)
+    .sort((a, b) => rankDoc(b) - rankDoc(a));
+
+  if (recentDecisions.length === 0 && rankedPrefs.length === 0 && activeProblems.length === 0 && recentDeductions.length === 0) {
     return null;
   }
 
@@ -338,6 +343,19 @@ function getCurrentFocus(
     }
   }
 
+  // Cross-session deductions (derived insights with source provenance)
+  if (recentDeductions.length > 0) {
+    lines.push("**Derived Insights:**");
+    charCount += 24;
+    for (const d of recentDeductions) {
+      if (charCount >= maxChars) break;
+      const entry = `- ${d.title} (${d.modifiedAt.slice(0, 10)})`;
+      lines.push(entry);
+      paths.push(`${d.collection}/${d.path}`);
+      charCount += entry.length + 2;
+    }
+  }
+
   return lines.length > 1 ? { text: lines.join("\n"), paths } : null;
 }
 

package/src/memory.ts

@@ -20,6 +20,7 @@ export const HALF_LIVES: Record<string, number> = {
   project: 120,
   preference: Infinity,
   decision: Infinity,
+  deductive: Infinity,
   hub: Infinity,
 };
 
@@ -29,6 +30,7 @@ export const HALF_LIVES: Record<string, number> = {
 
 export const TYPE_BASELINES: Record<string, number> = {
   decision: 0.85,
+  deductive: 0.85,
   preference: 0.80,
   hub: 0.80,
   problem: 0.75,
@@ -45,7 +47,7 @@ export const TYPE_BASELINES: Record<string, number> = {
 // Content Type Inference
 // =============================================================================
 
-export type ContentType = "decision" | "preference" | "hub" | "research" | "project" | "handoff" | "conversation" | "progress" | "milestone" | "problem" | "note";
+export type ContentType = "decision" | "deductive" | "preference" | "hub" | "research" | "project" | "handoff" | "conversation" | "progress" | "milestone" | "problem" | "note";
 
 export function inferContentType(path: string, explicitType?: string): ContentType {
   if (explicitType && explicitType in TYPE_BASELINES) return explicitType as ContentType;
@@ -75,7 +77,7 @@ export type MemoryType = "episodic" | "semantic" | "procedural";
  */
 export function inferMemoryType(path: string, contentType: string, body?: string): MemoryType {
   if (["handoff", "progress", "conversation"].includes(contentType)) return "episodic";
-  if (["decision", "hub", "research"].includes(contentType)) return "semantic";
+  if (["decision", "deductive", "hub", "research"].includes(contentType)) return "semantic";
   if (body && /\b(step\s+\d|workflow|recipe|how\s+to|procedure|runbook|playbook)\b/i.test(body)) return "procedural";
   if (path.includes("sop") || path.includes("runbook") || path.includes("playbook")) return "procedural";
   if (contentType === "antipattern") return "semantic";
@@ -150,7 +152,7 @@ export function confidenceScore(
   // Attention decay: reduce confidence if not accessed recently (5% per week)
   // Only apply to episodic/progress content — skip for durable types (decision, hub, research)
   // Also skip if last_accessed_at was backfilled from modified_at (no real access yet)
-  const DECAY_EXEMPT_TYPES = new Set(["decision", "hub", "research", "antipattern", "preference"]);
+  const DECAY_EXEMPT_TYPES = new Set(["decision", "deductive", "hub", "research", "antipattern", "preference"]);
   let attentionDecay = 1.0;
   if (lastAccessedAt && !DECAY_EXEMPT_TYPES.has(contentType)) {
     const lastAccess = typeof lastAccessedAt === "string" ? new Date(lastAccessedAt) : lastAccessedAt;

package/src/store.ts

@@ -544,6 +544,10 @@ function initializeDatabase(db: Database): void {
     ["skill_name", "ALTER TABLE documents ADD COLUMN skill_name TEXT"],
     ["obs_quality_score", "ALTER TABLE documents ADD COLUMN obs_quality_score REAL"],
     ["failure_reason", "ALTER TABLE documents ADD COLUMN failure_reason TEXT"],
+    ["source_doc_ids", "ALTER TABLE documents ADD COLUMN source_doc_ids TEXT"],
+    ["embed_state", "ALTER TABLE documents ADD COLUMN embed_state TEXT DEFAULT 'pending'"],
+    ["embed_error", "ALTER TABLE documents ADD COLUMN embed_error TEXT"],
+    ["embed_attempts", "ALTER TABLE documents ADD COLUMN embed_attempts INTEGER DEFAULT 0"],
   ];
   for (const [col, sql] of obsMigrations) {
     if (!colNames.has(col)) {
@@ -906,6 +910,11 @@ export type Store = {
   pinDocument: (collection: string, path: string, pinned: boolean) => void;
   snoozeDocument: (collection: string, path: string, until: string | null) => void;
 
+  // Embed state tracking
+  markEmbedSynced: (hash: string) => void;
+  markEmbedFailed: (hash: string, error: string) => void;
+  getEmbedStats: () => { pending: number; synced: number; failed: number };
+
   // Beads integration
   syncBeadsIssues: (projectDir: string) => Promise<{ synced: number; created: number; newDocIds: number[] }>;
   detectBeadsProject: (cwd: string) => string | null;
@@ -1078,6 +1087,24 @@ export function createStore(dbPath?: string, opts?: { readonly?: boolean; busyTi
     pinDocument: (collection: string, path: string, pinned: boolean) => pinDocumentFn(db, collection, path, pinned),
     snoozeDocument: (collection: string, path: string, until: string | null) => snoozeDocumentFn(db, collection, path, until),
 
+    // Embed state tracking
+    markEmbedSynced: (hash: string) => {
+      db.prepare(`UPDATE documents SET embed_state = 'synced' WHERE hash = ? AND active = 1`).run(hash);
+    },
+    markEmbedFailed: (hash: string, error: string) => {
+      db.prepare(`UPDATE documents SET embed_state = 'failed', embed_error = ?, embed_attempts = COALESCE(embed_attempts, 0) + 1 WHERE hash = ? AND active = 1`).run(error, hash);
+    },
+    getEmbedStats: () => {
+      const stats = db.prepare(`
+        SELECT
+          SUM(CASE WHEN embed_state = 'pending' OR embed_state IS NULL THEN 1 ELSE 0 END) as pending,
+          SUM(CASE WHEN embed_state = 'synced' THEN 1 ELSE 0 END) as synced,
+          SUM(CASE WHEN embed_state = 'failed' THEN 1 ELSE 0 END) as failed
+        FROM documents WHERE active = 1
+      `).get() as { pending: number; synced: number; failed: number };
+      return { pending: stats.pending || 0, synced: stats.synced || 0, failed: stats.failed || 0 };
+    },
+
     // Beads integration
     syncBeadsIssues: (projectDir: string) => syncBeadsIssues(db, projectDir),
     detectBeadsProject,
@@ -2924,12 +2951,17 @@ export function getHashesForEmbedding(db: Database): { hash: string; body: strin
  * Returns hashes that have no content_vectors row with fragment_type set.
  */
 export function getHashesNeedingFragments(db: Database): { hash: string; body: string; path: string; title: string; collection: string }[] {
+  // Select docs that either have no fragments at all OR are missing the primary (seq=0) fragment.
+  // The seq=0 embedding is critical — surprisal scoring, semantic graph, and health checks depend on it.
   return db.prepare(`
     SELECT d.hash, c.doc as body, MIN(d.path) as path, MIN(d.title) as title, MIN(d.collection) as collection
     FROM documents d
     JOIN content c ON d.hash = c.hash
     LEFT JOIN content_vectors v ON d.hash = v.hash AND v.fragment_type IS NOT NULL
-    WHERE d.active = 1 AND v.hash IS NULL
+    LEFT JOIN content_vectors v0 ON d.hash = v0.hash AND v0.seq = 0
+    WHERE d.active = 1
+      AND (v.hash IS NULL OR v0.hash IS NULL)
+      AND COALESCE(d.embed_attempts, 0) < 3
     GROUP BY d.hash
   `).all() as { hash: string; body: string; path: string; title: string; collection: string }[];
 }
@@ -2941,6 +2973,8 @@ export function getHashesNeedingFragments(db: Database): { hash: string; body: s
 export function clearAllEmbeddings(db: Database): void {
   db.exec(`DELETE FROM content_vectors`);
   db.exec(`DROP TABLE IF EXISTS vectors_vec`);
+  // Reset embed state so failed docs get retried after force re-embed
+  try { db.exec(`UPDATE documents SET embed_state = 'pending', embed_error = NULL, embed_attempts = 0 WHERE active = 1`); } catch { /* column may not exist yet */ }
   vecTableDimsCache.delete(db);
 }