@199-bio/engram 0.5.1 → 0.6.0

package/src/consolidation/consolidator.ts

@@ -12,7 +12,9 @@
  */
 
 import Anthropic from "@anthropic-ai/sdk";
-import { EngramDatabase, Memory, Digest } from "../storage/database.js";
+import { EngramDatabase, Memory, Digest, Episode } from "../storage/database.js";
+import { KnowledgeGraph } from "../graph/knowledge-graph.js";
+import { HybridSearch } from "../retrieval/hybrid.js";
 
 const CONSOLIDATION_SYSTEM = `You are a high-quality memory consolidation system for a personal AI assistant. Your goal is to create comprehensive, nuanced digests that preserve the richness of human experience and relationships.
 
@@ -49,6 +51,37 @@ const CONSOLIDATION_SYSTEM = `You are a high-quality memory consolidation system
 - If memories span different time periods, note the evolution
 - Only flag true contradictions, not incomplete information or natural life changes`;
 
+const EPISODE_EXTRACTION_SYSTEM = `You are extracting structured memories from a conversation. Your goal is to identify facts, preferences, events, and relationships worth remembering.
+
+## What to Extract
+- Key facts about people, places, organizations
+- User preferences and opinions
+- Important events and their dates
+- Relationships between entities
+- Decisions made or plans discussed
+
+## What to Skip
+- Small talk and pleasantries
+- Repetitive information
+- Context that's only relevant to the immediate task
+- Technical details that don't reveal user preferences
+
+## Output Format (JSON)
+{
+  "memories": [
+    {
+      "content": "The actual memory to store (clear, standalone statement)",
+      "importance": 0.5,
+      "emotional_weight": 0.5,
+      "event_time": "2024-12-01 or null if not mentioned",
+      "entities": [{"name": "Boris", "type": "person"}],
+      "relationships": [{"from": "Boris", "to": "Google", "type": "works_at"}]
+    }
+  ]
+}
+
+Extract 0-5 memories. Quality over quantity. If nothing worth remembering, return empty memories array.`;
+
 interface ConsolidationResult {
   digest: string;
   topic: string;
@@ -58,6 +91,19 @@ interface ConsolidationResult {
   }>;
 }
 
+interface ExtractedMemory {
+  content: string;
+  importance: number;
+  emotional_weight: number;
+  event_time: string | null;
+  entities: Array<{ name: string; type: string }>;
+  relationships: Array<{ from: string; to: string; type: string }>;
+}
+
+interface EpisodeExtractionResult {
+  memories: ExtractedMemory[];
+}
+
 interface ConsolidateOptions {
   batchSize?: number;
   minMemoriesForConsolidation?: number;
@@ -66,9 +112,17 @@ interface ConsolidateOptions {
 export class Consolidator {
   private client: Anthropic | null = null;
   private db: EngramDatabase;
-
-  constructor(db: EngramDatabase) {
+  private graph: KnowledgeGraph | null = null;
+  private search: HybridSearch | null = null;
+
+  constructor(
+    db: EngramDatabase,
+    graph?: KnowledgeGraph,
+    search?: HybridSearch
+  ) {
     this.db = db;
+    this.graph = graph || null;
+    this.search = search || null;
 
     const apiKey = process.env.ANTHROPIC_API_KEY;
     if (apiKey) {
@@ -359,18 +413,204 @@ Create a rich, detailed profile. Do not summarize away important nuances. Respon
   getStatus(): {
     configured: boolean;
     unconsolidatedMemories: number;
+    unconsolidatedEpisodes: number;
     totalDigests: number;
     unresolvedContradictions: number;
   } {
-    const unconsolidated = this.db.getUnconsolidatedMemories(undefined, 1000);
+    const unconsolidatedMem = this.db.getUnconsolidatedMemories(undefined, 1000);
+    const unconsolidatedEp = this.db.getUnconsolidatedEpisodes(1000);
     const digests = this.db.getDigests(undefined, 1000);
     const contradictions = this.db.getContradictions(false, 1000);
 
     return {
       configured: this.isConfigured(),
-      unconsolidatedMemories: unconsolidated.length,
+      unconsolidatedMemories: unconsolidatedMem.length,
+      unconsolidatedEpisodes: unconsolidatedEp.length,
       totalDigests: digests.length,
       unresolvedContradictions: contradictions.length,
     };
   }
+
+  /**
+   * Process unconsolidated episodes into memories
+   * This is the "working memory → long-term memory" transfer
+   */
+  async consolidateEpisodes(options: {
+    minEpisodes?: number;
+    batchSize?: number;
+  } = {}): Promise<{
+    episodesProcessed: number;
+    memoriesCreated: number;
+    entitiesCreated: number;
+  }> {
+    if (!this.client) {
+      throw new Error("Consolidator not configured - set ANTHROPIC_API_KEY");
+    }
+
+    const { minEpisodes = 4, batchSize = 20 } = options;
+
+    // Get unconsolidated episodes
+    const episodes = this.db.getUnconsolidatedEpisodes(batchSize);
+
+    if (episodes.length < minEpisodes) {
+      return { episodesProcessed: 0, memoriesCreated: 0, entitiesCreated: 0 };
+    }
+
+    // Group by session for context
+    const sessionGroups = new Map<string, Episode[]>();
+    for (const ep of episodes) {
+      const existing = sessionGroups.get(ep.session_id) || [];
+      existing.push(ep);
+      sessionGroups.set(ep.session_id, existing);
+    }
+
+    let episodesProcessed = 0;
+    let memoriesCreated = 0;
+    let entitiesCreated = 0;
+
+    // Process each session
+    for (const [sessionId, sessionEpisodes] of sessionGroups) {
+      if (sessionEpisodes.length < 2) continue;
+
+      try {
+        const result = await this.extractMemoriesFromEpisodes(sessionEpisodes);
+
+        if (result && result.memories.length > 0) {
+          for (const mem of result.memories) {
+            // Create the memory
+            const memory = this.db.createMemory(
+              mem.content,
+              "episode_consolidation",
+              mem.importance,
+              {
+                eventTime: mem.event_time ? new Date(mem.event_time) : undefined,
+                emotionalWeight: mem.emotional_weight,
+              }
+            );
+            memoriesCreated++;
+
+            // Index for search
+            if (this.search) {
+              await this.search.indexMemory(memory);
+            }
+
+            // Create entities and relationships
+            if (this.graph) {
+              for (const ent of mem.entities || []) {
+                const entity = this.graph.getOrCreateEntity(
+                  ent.name,
+                  ent.type as "person" | "place" | "concept" | "event" | "organization"
+                );
+                this.db.addObservation(entity.id, mem.content, memory.id, 1.0);
+                entitiesCreated++;
+              }
+
+              for (const rel of mem.relationships || []) {
+                try {
+                  const fromEntity = this.graph.getOrCreateEntity(rel.from, "person");
+                  const toEntity = this.graph.getOrCreateEntity(rel.to, "person");
+                  this.graph.relate(fromEntity.name, toEntity.name, rel.type);
+                } catch {
+                  // Skip invalid relationships
+                }
+              }
+            }
+          }
+        }
+
+        // Mark episodes as consolidated
+        this.db.markEpisodesConsolidated(sessionEpisodes.map(e => e.id));
+        episodesProcessed += sessionEpisodes.length;
+
+      } catch (error) {
+        console.error("[Consolidator] Episode consolidation failed:", error);
+      }
+    }
+
+    return { episodesProcessed, memoriesCreated, entitiesCreated };
+  }
+
+  /**
+   * Extract memories from conversation episodes using Haiku (fast, cheap)
+   */
+  private async extractMemoriesFromEpisodes(
+    episodes: Episode[]
+  ): Promise<EpisodeExtractionResult | null> {
+    if (!this.client) return null;
+
+    // Format conversation
+    const conversationText = episodes
+      .sort((a, b) => a.turn_index - b.turn_index)
+      .map(ep => `${ep.role.toUpperCase()}: ${ep.content}`)
+      .join("\n\n");
+
+    const userPrompt = `Extract memorable facts from this conversation.
+
+CONVERSATION:
+${conversationText}
+
+Remember: Only extract information worth remembering long-term. Skip transient task details.
+Respond with JSON only.`;
+
+    try {
+      // Use Haiku for speed/cost (no extended thinking needed)
+      const response = await this.client.messages.create({
+        model: "claude-haiku-4-5-20251201",
+        max_tokens: 4000,
+        messages: [
+          {
+            role: "user",
+            content: userPrompt,
+          },
+        ],
+        system: EPISODE_EXTRACTION_SYSTEM,
+      });
+
+      let text = "";
+      for (const block of response.content) {
+        if (block.type === "text") {
+          text = block.text;
+          break;
+        }
+      }
+
+      if (!text) return null;
+
+      const jsonMatch = text.match(/\{[\s\S]*\}/);
+      if (!jsonMatch) return null;
+
+      return JSON.parse(jsonMatch[0]) as EpisodeExtractionResult;
+    } catch (error) {
+      console.error("[Consolidator] Episode extraction failed:", error);
+      return null;
+    }
+  }
+
+  /**
+   * Run full consolidation cycle (episodes → memories → digests)
+   * This is the "sleep cycle" that should run periodically
+   */
+  async runSleepCycle(): Promise<{
+    episodesProcessed: number;
+    memoriesCreated: number;
+    digestsCreated: number;
+    contradictionsFound: number;
+  }> {
+    console.error("[Consolidator] Starting sleep cycle...");
+
+    // Step 1: Process episodes into memories
+    const episodeResult = await this.consolidateEpisodes();
+    console.error(`[Consolidator] Episodes: ${episodeResult.episodesProcessed} → ${episodeResult.memoriesCreated} memories`);
+
+    // Step 2: Consolidate memories into digests
+    const memoryResult = await this.consolidate();
+    console.error(`[Consolidator] Memories: ${memoryResult.memoriesProcessed} → ${memoryResult.digestsCreated} digests`);
+
+    return {
+      episodesProcessed: episodeResult.episodesProcessed,
+      memoriesCreated: episodeResult.memoriesCreated,
+      digestsCreated: memoryResult.digestsCreated,
+      contradictionsFound: memoryResult.contradictionsFound,
+    };
+  }
 }

package/src/index.ts

@@ -20,6 +20,7 @@ import { KnowledgeGraph } from "./graph/knowledge-graph.js";
 import { createRetriever } from "./retrieval/colbert.js";
 import { HybridSearch } from "./retrieval/hybrid.js";
 import { EngramWebServer } from "./web/server.js";
+import { Consolidator } from "./consolidation/consolidator.js";
 
 // ============ Configuration ============
 
@@ -34,6 +35,7 @@ const DB_FILE = path.join(DB_PATH, "engram.db");
 let db: EngramDatabase;
 let graph: KnowledgeGraph;
 let search: HybridSearch;
+let consolidator: Consolidator;
 let webServer: EngramWebServer | null = null;
 
 async function initialize(): Promise<void> {
@@ -44,6 +46,7 @@ async function initialize(): Promise<void> {
 
   const retriever = await createRetriever(DB_PATH);
   search = new HybridSearch(db, graph, retriever);
+  consolidator = new Consolidator(db, graph, search);
 
   // Rebuild index with existing memories
   const stats = db.getStats();
@@ -53,6 +56,9 @@ async function initialize(): Promise<void> {
   }
 
   console.error(`[Engram] Ready. Stats: ${JSON.stringify(stats)}`);
+  if (consolidator.isConfigured()) {
+    console.error(`[Engram] Consolidation enabled (ANTHROPIC_API_KEY found)`);
+  }
 }
 
 // ============ MCP Server ============
@@ -60,7 +66,7 @@ async function initialize(): Promise<void> {
 const server = new Server(
   {
     name: "engram",
-    version: "0.5.1",
+    version: "0.6.0",
   },
   {
     capabilities: {
@@ -90,6 +96,17 @@ const TOOLS = [
           maximum: 1,
           default: 0.5,
         },
+        emotional_weight: {
+          type: "number",
+          description: "0-1 emotional significance. Use 0.8+ for emotionally charged content, celebrations, losses. Affects memory retention.",
+          minimum: 0,
+          maximum: 1,
+          default: 0.5,
+        },
+        event_time: {
+          type: "string",
+          description: "When the event actually happened (ISO 8601), if different from now. E.g., 'Last week I went to Paris' → set event_time to that date.",
+        },
         entities: {
           type: "array",
           description: "Key entities mentioned (people, organizations, places). Only include clear, specific named entities.",
@@ -200,6 +217,28 @@ const TOOLS = [
       openWorldHint: true,
     },
   },
+  {
+    name: "consolidate",
+    description: "Run memory consolidation to compress episodes into memories and memories into digests. Like sleep for the memory system. Use periodically or when requested.",
+    inputSchema: {
+      type: "object" as const,
+      properties: {
+        mode: {
+          type: "string",
+          enum: ["full", "episodes_only", "memories_only"],
+          description: "What to consolidate: 'full' (default) runs everything, 'episodes_only' just processes conversation history, 'memories_only' creates digests",
+          default: "full",
+        },
+      },
+    },
+    annotations: {
+      title: "Consolidate Memories",
+      readOnlyHint: false,
+      destructiveHint: false,
+      idempotentHint: false,
+      openWorldHint: false,
+    },
+  },
 ];
 
 // List available tools
@@ -218,18 +257,25 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
           content,
           source = "conversation",
           importance = 0.5,
+          emotional_weight = 0.5,
+          event_time,
           entities: providedEntities = [],
           relationships: providedRelationships = [],
         } = args as {
           content: string;
           source?: string;
           importance?: number;
+          emotional_weight?: number;
+          event_time?: string;
           entities?: Array<{ name: string; type: "person" | "organization" | "place" }>;
           relationships?: Array<{ from: string; to: string; type: string }>;
         };
 
-        // Create memory
-        const memory = db.createMemory(content, source, importance);
+        // Create memory with new temporal and salience fields
+        const memory = db.createMemory(content, source, importance, {
+          eventTime: event_time ? new Date(event_time) : undefined,
+          emotionalWeight: emotional_weight,
+        });
 
         // Index for semantic search
         await search.indexMemory(memory);
@@ -290,6 +336,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
           source: r.memory.source,
           timestamp: r.memory.timestamp.toISOString(),
           relevance_score: r.score.toFixed(4),
+          retention: r.retention.toFixed(2),  // How well-retained (0-1)
           matched_via: Object.entries(r.sources)
             .filter(([, v]) => v !== undefined)
             .map(([k]) => k)
@@ -365,6 +412,52 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         };
       }
 
+      case "consolidate": {
+        const { mode = "full" } = args as { mode?: "full" | "episodes_only" | "memories_only" };
+
+        if (!consolidator.isConfigured()) {
+          return {
+            content: [
+              {
+                type: "text" as const,
+                text: JSON.stringify({
+                  success: false,
+                  error: "Consolidation requires ANTHROPIC_API_KEY environment variable",
+                }),
+              },
+            ],
+            isError: true,
+          };
+        }
+
+        let result;
+        switch (mode) {
+          case "episodes_only":
+            result = await consolidator.consolidateEpisodes();
+            break;
+          case "memories_only":
+            result = await consolidator.consolidate();
+            break;
+          case "full":
+          default:
+            result = await consolidator.runSleepCycle();
+            break;
+        }
+
+        return {
+          content: [
+            {
+              type: "text" as const,
+              text: JSON.stringify({
+                success: true,
+                mode,
+                ...result,
+              }, null, 2),
+            },
+          ],
+        };
+      }
+
       default:
         throw new Error(`Unknown tool: ${name}`);
     }

package/src/retrieval/hybrid.ts

@@ -1,6 +1,7 @@
 /**
  * Hybrid Search with Reciprocal Rank Fusion (RRF)
  * Combines BM25 (keyword) and ColBERT (semantic) search
+ * Enhanced with temporal decay and salience scoring
  */
 
 import { EngramDatabase, Memory } from "../storage/database.js";
@@ -10,6 +11,7 @@ import { ColBERTRetriever, SimpleRetriever, SearchResult, Document } from "./col
 export interface HybridSearchResult {
   memory: Memory;
   score: number;
+  retention: number;  // 0-1 how well-retained this memory is
   sources: {
     bm25?: number;
     semantic?: number;
@@ -17,6 +19,60 @@ export interface HybridSearchResult {
   };
 }
 
+/**
+ * Calculate Ebbinghaus forgetting curve retention
+ * R = e^(-t/S) where t=time since last access, S=stability
+ *
+ * Higher stability = slower forgetting
+ * Recent access = higher retention
+ */
+function calculateRetention(memory: Memory, now: Date): number {
+  // Use last_accessed if available, otherwise timestamp
+  const lastActive = memory.last_accessed || memory.timestamp;
+  const daysSinceAccess = (now.getTime() - lastActive.getTime()) / (1000 * 60 * 60 * 24);
+
+  // Stability is our memory strength (default 1.0, increases with recalls)
+  const stability = memory.stability || 1.0;
+
+  // Half-life in days = stability * 7 (so stability=1 means 7-day half-life)
+  const halfLife = stability * 7;
+
+  // Exponential decay: R = e^(-0.693 * t / halfLife)
+  const retention = Math.exp(-0.693 * daysSinceAccess / halfLife);
+
+  return Math.max(0, Math.min(1, retention));
+}
+
+/**
+ * Calculate salience score - how important/memorable is this?
+ * Combines emotional weight, importance, and access patterns
+ */
+function calculateSalience(memory: Memory): number {
+  const importance = memory.importance || 0.5;
+  const emotionalWeight = memory.emotional_weight || 0.5;
+  const accessBonus = Math.min(1, Math.log(1 + (memory.access_count || 0)) / 5);
+
+  // Weighted combination
+  return (importance * 0.4) + (emotionalWeight * 0.4) + (accessBonus * 0.2);
+}
+
+/**
+ * Apply temporal and salience adjustments to search results
+ */
+function adjustScore(memory: Memory, baseScore: number, now: Date): { adjusted: number; retention: number } {
+  const retention = calculateRetention(memory, now);
+  const salience = calculateSalience(memory);
+
+  // Final score = base * (0.5 + 0.3*retention + 0.2*salience)
+  // This means: 50% retrieval match, 30% recency/stability, 20% importance
+  const multiplier = 0.5 + (0.3 * retention) + (0.2 * salience);
+
+  return {
+    adjusted: baseScore * multiplier,
+    retention,
+  };
+}
+
 export class HybridSearch {
   constructor(
     private db: EngramDatabase,
@@ -133,18 +189,21 @@ export class HybridSearch {
     // Sort by RRF score
     rrfScores.sort((a, b) => b.score - a.score);
 
-    // Get top results with full memory data
-    const results: HybridSearchResult[] = [];
+    // Get results with full memory data and apply temporal adjustments
+    const now = new Date();
+    const adjustedResults: Array<HybridSearchResult & { originalScore: number }> = [];
 
-    for (const { id, score, sources } of rrfScores.slice(0, limit)) {
+    for (const { id, score, sources } of rrfScores) {
       const memory = this.db.getMemory(id);
       if (memory) {
-        // Update access count
-        this.db.touchMemory(id);
+        // Apply Ebbinghaus decay and salience scoring
+        const { adjusted, retention } = adjustScore(memory, score, now);
 
-        results.push({
+        adjustedResults.push({
           memory,
-          score,
+          score: adjusted,
+          retention,
+          originalScore: score,
           sources: {
             bm25: sources.bm25,
             semantic: sources.semantic,
@@ -154,6 +213,23 @@ export class HybridSearch {
       }
     }
 
+    // Re-sort by adjusted score (accounts for recency/stability)
+    adjustedResults.sort((a, b) => b.score - a.score);
+
+    // Take top results and update access counts
+    const results: HybridSearchResult[] = [];
+    for (const result of adjustedResults.slice(0, limit)) {
+      // Update access count (which also increases stability for future searches)
+      this.db.touchMemory(result.memory.id);
+
+      results.push({
+        memory: result.memory,
+        score: result.score,
+        retention: result.retention,
+        sources: result.sources,
+      });
+    }
+
     return results;
   }