@199-bio/engram 0.8.1 → 0.10.0

package/src/storage/database.ts

@@ -119,6 +119,37 @@ export interface RetrievalLog {
   processed: boolean;      // Has this been used for learning?
 }
 
+/**
+ * Consolidation checkpoint for safe backlog processing
+ * Enables resume capability and progress tracking
+ */
+export interface ConsolidationCheckpoint {
+  id: string;
+  run_id: string;              // Unique ID for this consolidation run
+  phase: "episodes" | "memories" | "decay" | "cleanup" | "complete";
+  batches_completed: number;
+  batches_total: number;
+  memories_processed: number;
+  episodes_processed: number;
+  digests_created: number;
+  contradictions_found: number;
+  tokens_used: number;
+  estimated_cost_usd: number;
+  started_at: Date;
+  updated_at: Date;
+  completed_at: Date | null;
+  error: string | null;
+}
+
+/**
+ * Consolidation configuration for budget and rate limits
+ */
+export interface ConsolidationConfig {
+  key: string;
+  value: string;
+  updated_at: Date;
+}
+
 export class EngramDatabase {
   private db: Database.Database;
   private stmtCache: Map<string, Database.Statement> = new Map();
@@ -284,6 +315,34 @@ export class EngramDatabase {
       CREATE INDEX IF NOT EXISTS idx_digest_sources_memory ON digest_sources(memory_id);
     `);
 
+    // FTS5 for digest BM25 search
+    this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS digests_fts USING fts5(
+        content,
+        topic,
+        content='digests',
+        content_rowid='rowid'
+      );
+
+      -- Triggers to keep FTS in sync
+      CREATE TRIGGER IF NOT EXISTS digests_ai AFTER INSERT ON digests BEGIN
+        INSERT INTO digests_fts(rowid, content, topic) VALUES (NEW.rowid, NEW.content, COALESCE(NEW.topic, ''));
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS digests_ad AFTER DELETE ON digests BEGIN
+        INSERT INTO digests_fts(digests_fts, rowid, content, topic) VALUES('delete', OLD.rowid, OLD.content, COALESCE(OLD.topic, ''));
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS digests_au AFTER UPDATE ON digests BEGIN
+        INSERT INTO digests_fts(digests_fts, rowid, content, topic) VALUES('delete', OLD.rowid, OLD.content, COALESCE(OLD.topic, ''));
+        INSERT INTO digests_fts(rowid, content, topic) VALUES (NEW.rowid, NEW.content, COALESCE(NEW.topic, ''));
+      END;
+    `);
+
+    // Rebuild FTS5 index to sync with content table (necessary for existing databases)
+    // This is idempotent and fast for small tables
+    this.db.exec(`INSERT INTO digests_fts(digests_fts) VALUES('rebuild');`);
+
     // Contradictions table (detected conflicts)
     this.db.exec(`
       CREATE TABLE IF NOT EXISTS contradictions (
@@ -340,6 +399,48 @@ export class EngramDatabase {
       CREATE INDEX IF NOT EXISTS idx_retrieval_logs_recall ON retrieval_logs(recall_id);
       CREATE INDEX IF NOT EXISTS idx_retrieval_logs_processed ON retrieval_logs(processed);
     `);
+
+    // Consolidation checkpoints for safe backlog processing
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS consolidation_checkpoints (
+        id TEXT PRIMARY KEY,
+        run_id TEXT NOT NULL UNIQUE,
+        phase TEXT NOT NULL CHECK(phase IN ('episodes', 'memories', 'decay', 'cleanup', 'complete')),
+        batches_completed INTEGER DEFAULT 0,
+        batches_total INTEGER DEFAULT 0,
+        memories_processed INTEGER DEFAULT 0,
+        episodes_processed INTEGER DEFAULT 0,
+        digests_created INTEGER DEFAULT 0,
+        contradictions_found INTEGER DEFAULT 0,
+        tokens_used INTEGER DEFAULT 0,
+        estimated_cost_usd REAL DEFAULT 0,
+        started_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+        updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+        completed_at DATETIME,
+        error TEXT
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_checkpoints_run ON consolidation_checkpoints(run_id);
+      CREATE INDEX IF NOT EXISTS idx_checkpoints_phase ON consolidation_checkpoints(phase);
+    `);
+
+    // Consolidation configuration for budget and rate limits
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS consolidation_config (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL,
+        updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
+      );
+
+      -- Default configuration values
+      INSERT OR IGNORE INTO consolidation_config (key, value) VALUES
+        ('daily_budget_usd', '5.0'),
+        ('max_batches_per_run', '5'),
+        ('delay_between_calls_ms', '2000'),
+        ('recovery_mode_threshold', '100'),
+        ('error_rate_threshold', '0.3'),
+        ('empty_digest_threshold', '0.2');
+    `);
   }
 
   /**
@@ -568,6 +669,23 @@ export class EngramDatabase {
     }));
   }
 
+  searchDigestsBM25(query: string, limit: number = 10): Array<Digest & { score: number }> {
+    const escapedQuery = this.escapeFTS5Query(query);
+    const rows = this.stmt(`
+      SELECT d.*, bm25(digests_fts) as score
+      FROM digests_fts fts
+      JOIN digests d ON fts.rowid = d.rowid
+      WHERE digests_fts MATCH ?
+      ORDER BY score
+      LIMIT ?
+    `).all(escapedQuery, limit) as Array<Record<string, unknown>>;
+
+    return rows.map((row) => ({
+      ...this.rowToDigest(row),
+      score: row.score as number,
+    }));
+  }
+
   private escapeFTS5Query(query: string): string {
     // Simple tokenization - split on whitespace, escape special chars
     const tokens = query
@@ -1341,6 +1459,195 @@ export class EngramDatabase {
     return result.changes;
   }
 
+  // ============ Consolidation Checkpoints ============
+
+  /**
+   * Create a new consolidation checkpoint
+   */
+  createCheckpoint(runId: string, phase: ConsolidationCheckpoint["phase"], batchesTotal: number): ConsolidationCheckpoint {
+    const id = randomUUID();
+    this.db.prepare(`
+      INSERT INTO consolidation_checkpoints (id, run_id, phase, batches_total)
+      VALUES (?, ?, ?, ?)
+    `).run(id, runId, phase, batchesTotal);
+    return this.getCheckpoint(runId)!;
+  }
+
+  /**
+   * Get checkpoint by run_id
+   */
+  getCheckpoint(runId: string): ConsolidationCheckpoint | null {
+    const row = this.stmt("SELECT * FROM consolidation_checkpoints WHERE run_id = ?").get(runId) as Record<string, unknown> | undefined;
+    return row ? this.rowToCheckpoint(row) : null;
+  }
+
+  /**
+   * Get the latest incomplete checkpoint (for resume)
+   */
+  getIncompleteCheckpoint(): ConsolidationCheckpoint | null {
+    const row = this.stmt(`
+      SELECT * FROM consolidation_checkpoints
+      WHERE completed_at IS NULL AND error IS NULL
+      ORDER BY started_at DESC LIMIT 1
+    `).get() as Record<string, unknown> | undefined;
+    return row ? this.rowToCheckpoint(row) : null;
+  }
+
+  /**
+   * Update checkpoint progress
+   */
+  updateCheckpoint(
+    runId: string,
+    updates: Partial<Pick<ConsolidationCheckpoint,
+      "phase" | "batches_completed" | "batches_total" | "memories_processed" |
+      "episodes_processed" | "digests_created" | "contradictions_found" |
+      "tokens_used" | "estimated_cost_usd" | "error"
+    >>
+  ): ConsolidationCheckpoint | null {
+    const sets: string[] = ["updated_at = CURRENT_TIMESTAMP"];
+    const values: unknown[] = [];
+
+    if (updates.phase !== undefined) {
+      sets.push("phase = ?");
+      values.push(updates.phase);
+    }
+    if (updates.batches_completed !== undefined) {
+      sets.push("batches_completed = ?");
+      values.push(updates.batches_completed);
+    }
+    if (updates.batches_total !== undefined) {
+      sets.push("batches_total = ?");
+      values.push(updates.batches_total);
+    }
+    if (updates.memories_processed !== undefined) {
+      sets.push("memories_processed = ?");
+      values.push(updates.memories_processed);
+    }
+    if (updates.episodes_processed !== undefined) {
+      sets.push("episodes_processed = ?");
+      values.push(updates.episodes_processed);
+    }
+    if (updates.digests_created !== undefined) {
+      sets.push("digests_created = ?");
+      values.push(updates.digests_created);
+    }
+    if (updates.contradictions_found !== undefined) {
+      sets.push("contradictions_found = ?");
+      values.push(updates.contradictions_found);
+    }
+    if (updates.tokens_used !== undefined) {
+      sets.push("tokens_used = ?");
+      values.push(updates.tokens_used);
+    }
+    if (updates.estimated_cost_usd !== undefined) {
+      sets.push("estimated_cost_usd = ?");
+      values.push(updates.estimated_cost_usd);
+    }
+    if (updates.error !== undefined) {
+      sets.push("error = ?");
+      values.push(updates.error);
+    }
+
+    values.push(runId);
+    this.db.prepare(`UPDATE consolidation_checkpoints SET ${sets.join(", ")} WHERE run_id = ?`).run(...values);
+    return this.getCheckpoint(runId);
+  }
+
+  /**
+   * Mark checkpoint as complete
+   */
+  completeCheckpoint(runId: string): void {
+    this.db.prepare(`
+      UPDATE consolidation_checkpoints
+      SET phase = 'complete', completed_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP
+      WHERE run_id = ?
+    `).run(runId);
+  }
+
+  /**
+   * Get recent checkpoints for reporting
+   */
+  getRecentCheckpoints(limit: number = 10): ConsolidationCheckpoint[] {
+    const rows = this.stmt(`
+      SELECT * FROM consolidation_checkpoints
+      ORDER BY started_at DESC LIMIT ?
+    `).all(limit) as Record<string, unknown>[];
+    return rows.map(r => this.rowToCheckpoint(r));
+  }
+
+  /**
+   * Get total spending in the last 24 hours
+   */
+  getDailySpending(): number {
+    const row = this.stmt(`
+      SELECT COALESCE(SUM(estimated_cost_usd), 0) as total
+      FROM consolidation_checkpoints
+      WHERE started_at >= datetime('now', '-1 day')
+    `).get() as { total: number };
+    return row.total;
+  }
+
+  // ============ Consolidation Config ============
+
+  /**
+   * Get a config value
+   */
+  getConfig(key: string): string | null {
+    const row = this.stmt("SELECT value FROM consolidation_config WHERE key = ?").get(key) as { value: string } | undefined;
+    return row?.value ?? null;
+  }
+
+  /**
+   * Get a config value as number
+   */
+  getConfigNumber(key: string, defaultValue: number): number {
+    const value = this.getConfig(key);
+    return value ? parseFloat(value) : defaultValue;
+  }
+
+  /**
+   * Set a config value
+   */
+  setConfig(key: string, value: string): void {
+    this.db.prepare(`
+      INSERT INTO consolidation_config (key, value, updated_at)
+      VALUES (?, ?, CURRENT_TIMESTAMP)
+      ON CONFLICT(key) DO UPDATE SET value = ?, updated_at = CURRENT_TIMESTAMP
+    `).run(key, value, value);
+  }
+
+  /**
+   * Get all config values
+   */
+  getAllConfig(): Record<string, string> {
+    const rows = this.stmt("SELECT key, value FROM consolidation_config").all() as Array<{ key: string; value: string }>;
+    const config: Record<string, string> = {};
+    for (const row of rows) {
+      config[row.key] = row.value;
+    }
+    return config;
+  }
+
+  private rowToCheckpoint(row: Record<string, unknown>): ConsolidationCheckpoint {
+    return {
+      id: row.id as string,
+      run_id: row.run_id as string,
+      phase: row.phase as ConsolidationCheckpoint["phase"],
+      batches_completed: row.batches_completed as number,
+      batches_total: row.batches_total as number,
+      memories_processed: row.memories_processed as number,
+      episodes_processed: row.episodes_processed as number,
+      digests_created: row.digests_created as number,
+      contradictions_found: row.contradictions_found as number,
+      tokens_used: row.tokens_used as number,
+      estimated_cost_usd: row.estimated_cost_usd as number,
+      started_at: new Date(row.started_at as string),
+      updated_at: new Date(row.updated_at as string),
+      completed_at: row.completed_at ? new Date(row.completed_at as string) : null,
+      error: row.error as string | null,
+    };
+  }
+
   // ============ Statistics ============
 
   getStats(): {

package/src/transport/http.ts

@@ -0,0 +1,111 @@
+/**
+ * HTTP Transport for Engram MCP Server
+ * Uses StreamableHTTPServerTransport in stateless mode for Railway deployment
+ */
+
+import http from "http";
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+
+interface HttpServerOptions {
+  port: number;
+  server: Server;
+}
+
+/**
+ * Start HTTP server with MCP transport
+ * Returns a promise that resolves when server is listening
+ */
+export async function startHttpServer(options: HttpServerOptions): Promise<http.Server> {
+  const { port, server } = options;
+
+  // Create stateless transport (sessionIdGenerator: undefined)
+  // Stateless mode is perfect for Railway - no session management needed
+  const transport = new StreamableHTTPServerTransport({
+    sessionIdGenerator: undefined,
+  });
+
+  // Connect MCP server to transport
+  await server.connect(transport);
+
+  // Create HTTP server
+  const httpServer = http.createServer(async (req, res) => {
+    // CORS headers for remote clients (ElevenLabs, etc.)
+    res.setHeader("Access-Control-Allow-Origin", "*");
+    res.setHeader("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS");
+    res.setHeader("Access-Control-Allow-Headers", "Content-Type, mcp-session-id");
+
+    if (req.method === "OPTIONS") {
+      res.writeHead(204);
+      res.end();
+      return;
+    }
+
+    const url = new URL(req.url || "/", `http://localhost:${port}`);
+
+    // Health check endpoint (Railway uses this)
+    if (url.pathname === "/health" && req.method === "GET") {
+      res.writeHead(200, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({
+        status: "ok",
+        transport: "http",
+        version: "0.10.0"
+      }));
+      return;
+    }
+
+    // Root endpoint - service info
+    if (url.pathname === "/" && req.method === "GET") {
+      res.writeHead(200, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({
+        name: "engram",
+        description: "MCP memory server with hybrid search",
+        version: "0.10.0",
+        transport: "streamable-http",
+        endpoints: {
+          mcp: "/mcp",
+          health: "/health",
+        },
+      }));
+      return;
+    }
+
+    // MCP endpoint - handles both POST (messages) and GET (SSE stream)
+    if (url.pathname === "/mcp") {
+      try {
+        await transport.handleRequest(req, res);
+      } catch (error) {
+        console.error("[Engram HTTP] Error handling MCP request:", error);
+        if (!res.headersSent) {
+          res.writeHead(500, { "Content-Type": "application/json" });
+          res.end(JSON.stringify({ error: "Internal server error" }));
+        }
+      }
+      return;
+    }
+
+    // SSE alias (for clients expecting /sse endpoint)
+    if (url.pathname === "/sse") {
+      // Redirect to /mcp which handles SSE via GET
+      res.writeHead(307, { "Location": "/mcp" });
+      res.end();
+      return;
+    }
+
+    // 404 for unknown paths
+    res.writeHead(404, { "Content-Type": "application/json" });
+    res.end(JSON.stringify({ error: "Not found" }));
+  });
+
+  // Start listening
+  return new Promise((resolve, reject) => {
+    httpServer.once("error", reject);
+    httpServer.listen(port, () => {
+      console.error(`[Engram] MCP HTTP server running on port ${port}`);
+      console.error(`[Engram] Endpoints:`);
+      console.error(`[Engram]   POST http://localhost:${port}/mcp - MCP protocol`);
+      console.error(`[Engram]   GET  http://localhost:${port}/health - Health check`);
+      resolve(httpServer);
+    });
+  });
+}

package/src/transport/index.ts

@@ -0,0 +1,24 @@
+/**
+ * Transport layer for Engram MCP Server
+ * Supports both stdio (local) and HTTP (remote/Railway) transports
+ */
+
+export type TransportMode = "stdio" | "http";
+
+/**
+ * Detect transport mode from environment variables
+ * Default: stdio (preserves existing behavior)
+ */
+export function getTransportMode(): TransportMode {
+  const mode = process.env.ENGRAM_TRANSPORT?.toLowerCase();
+  if (mode === "http" || mode === "sse") return "http";
+  return "stdio";
+}
+
+/**
+ * Get HTTP port from environment
+ * Railway provides PORT, we also support ENGRAM_MCP_PORT
+ */
+export function getHttpPort(): number {
+  return parseInt(process.env.PORT || process.env.ENGRAM_MCP_PORT || "3000", 10);
+}

package/src/web/chat-handler.ts

@@ -253,16 +253,34 @@ const TOOLS: Anthropic.Tool[] = [
   },
 ];
 
-const SYSTEM_PROMPT = `You are a helpful assistant for managing Engram, a personal memory system. You help users:
-- View and search their memories
-- Manage entities (people, organizations, places)
-- Fix incorrect relationships
-- Merge duplicate entities
-- Delete incorrect data
-
-Be concise and helpful. When making changes, confirm what you did. If asked to do something destructive, confirm first unless the user is explicit.
-
-When listing entities or memories, format them clearly. Use the tools available to you.`;
+const SYSTEM_PROMPT = `You are a helpful assistant for managing Engram, a personal memory system. You have extended thinking capabilities - use them to reason carefully about complex requests.
+
+## Your Capabilities
+- Search and retrieve memories using semantic + keyword hybrid search
+- Manage entities (people, organizations, places) - create, rename, merge, delete
+- Manage relationships between entities
+- Create, edit, and delete memories
+- Find and auto-merge duplicate entities
+
+## Critical Behaviors
+1. **Always search first**: When asked about anything that might be in memory, use search_memories FIRST before answering. Don't assume you know the answer.
+2. **Multi-step reasoning**: For complex requests, break them into steps. Search, analyze results, then act.
+3. **Confirm destructive actions**: Unless the user is explicit, ask before deleting or merging data.
+4. **Be precise**: Use exact entity names when making changes. Check spelling.
+5. **Context awareness**: Remember what the user discussed earlier in this conversation.
+
+## Response Style
+- Be concise but thorough
+- Format lists and results clearly using markdown
+- When you find relevant memories, quote the key parts
+- If you're uncertain, say so and explain your reasoning
+
+## Tool Usage
+- search_memories: Use liberally - hybrid search is fast and effective
+- list_entities: Good for getting an overview before specific operations
+- get_entity: Get full details including observations and relationships
+- find_duplicates: Run this when asked about data quality or cleanup
+- auto_tidy: Only use when user explicitly wants automatic cleanup`;
 
 interface ChatMessage {
   role: "user" | "assistant";
@@ -271,7 +289,7 @@ interface ChatMessage {
 
 // Stream event types for SSE
 export interface StreamEvent {
-  type: "text" | "tool_start" | "tool_end" | "error" | "done";
+  type: "text" | "thinking" | "tool_start" | "tool_end" | "error" | "done";
   content?: string;
   tool?: string;
   result?: unknown;
@@ -308,7 +326,12 @@ export class ChatHandler {
       if (!this.client) {
         console.error("[Engram] ChatHandler: API key configured");
       }
-      this.client = new Anthropic({ apiKey });
+      this.client = new Anthropic({
+        apiKey,
+        defaultHeaders: {
+          "anthropic-beta": "interleaved-thinking-2025-05-14",
+        },
+      });
     } else {
       this.client = null;
     }
@@ -392,13 +415,18 @@ export class ChatHandler {
       while (continueLoop) {
         const stream = this.client.messages.stream({
           model: "claude-opus-4-5-20251101",
-          max_tokens: 1024,
+          max_tokens: 16000,
           system: SYSTEM_PROMPT,
           tools: TOOLS,
           messages: this.conversationHistory,
+          thinking: {
+            type: "enabled",
+            budget_tokens: 8000,
+          },
         });
 
         let currentToolUse: { id: string; name: string; input: string } | null = null;
+        let isThinking = false;
 
         for await (const event of stream) {
           if (event.type === "content_block_start") {
@@ -409,16 +437,23 @@ export class ChatHandler {
                 input: "",
               };
               yield { type: "tool_start", tool: event.content_block.name };
+            } else if (event.content_block.type === "thinking") {
+              isThinking = true;
+              yield { type: "thinking", content: "" };
             }
           } else if (event.type === "content_block_delta") {
             if (event.delta.type === "text_delta") {
               yield { type: "text", content: event.delta.text };
+            } else if (event.delta.type === "thinking_delta") {
+              // Stream thinking content for transparency
+              yield { type: "thinking", content: event.delta.thinking };
             } else if (event.delta.type === "input_json_delta" && currentToolUse) {
               currentToolUse.input += event.delta.partial_json;
             }
           } else if (event.type === "content_block_stop") {
             // Don't execute tools here - wait for finalMessage to avoid double execution
             currentToolUse = null;
+            isThinking = false;
           }
         }
 
@@ -494,10 +529,14 @@ export class ChatHandler {
 
       let response = await this.client.messages.create({
         model: "claude-opus-4-5-20251101",
-        max_tokens: 1024,
+        max_tokens: 16000,
         system: SYSTEM_PROMPT,
         tools: TOOLS,
         messages: this.conversationHistory,
+        thinking: {
+          type: "enabled",
+          budget_tokens: 8000,
+        },
       });
 
       // Handle tool use loop
@@ -531,10 +570,14 @@ export class ChatHandler {
         // Continue the conversation
         response = await this.client.messages.create({
           model: "claude-opus-4-5-20251101",
-          max_tokens: 1024,
+          max_tokens: 16000,
           system: SYSTEM_PROMPT,
           tools: TOOLS,
           messages: this.conversationHistory,
+          thinking: {
+            type: "enabled",
+            budget_tokens: 8000,
+          },
         });
       }