@199-bio/engram 0.8.0 → 0.10.0

package/src/consolidation/plan.ts

@@ -0,0 +1,444 @@
+/**
+ * Consolidation Plan
+ *
+ * Implements a Standard Operating Procedure for safe consolidation of large backlogs.
+ * Prevents damage through:
+ * - Assessment and recovery mode detection
+ * - Prioritization (recent + high importance first)
+ * - Rate limiting with delays between API calls
+ * - Budget tracking and cost caps
+ * - Checkpointing for resume capability
+ * - Validation with soft rollback triggers
+ */
+
+import { randomUUID } from "crypto";
+import { EngramDatabase, Memory, Episode, ConsolidationCheckpoint } from "../storage/database.js";
+
+// Token pricing (Opus 4.5 with extended thinking)
+const PRICING = {
+  opus: {
+    input: 15 / 1_000_000,      // $15 per 1M input tokens
+    output: 75 / 1_000_000,     // $75 per 1M output tokens
+    thinking: 15 / 1_000_000,   // $15 per 1M thinking tokens (same as input)
+  },
+  haiku: {
+    input: 0.80 / 1_000_000,    // $0.80 per 1M input tokens
+    output: 4.00 / 1_000_000,   // $4.00 per 1M output tokens
+  },
+};
+
+// Estimated tokens per operation (conservative estimates)
+const TOKEN_ESTIMATES = {
+  episodeBatch: {
+    input: 2000,    // Conversation text + system prompt
+    output: 1000,   // Extracted memories JSON
+  },
+  memoryBatch: {
+    input: 3000,     // Memories + system prompt
+    output: 2000,    // Digest + contradictions
+    thinking: 10000, // Extended thinking budget
+  },
+  entityProfile: {
+    input: 4000,
+    output: 3000,
+    thinking: 16000,
+  },
+};
+
+export interface BacklogAssessment {
+  unconsolidatedMemories: number;
+  unconsolidatedEpisodes: number;
+  isRecoveryMode: boolean;
+  estimatedBatches: number;
+  estimatedCost: number;
+  dailyBudget: number;
+  dailySpent: number;
+  budgetRemaining: number;
+  canProceed: boolean;
+  recommendedBatches: number;
+  phases: PhasePlan[];
+}
+
+export interface PhasePlan {
+  phase: "episodes" | "memories" | "decay" | "cleanup";
+  itemCount: number;
+  batchCount: number;
+  estimatedCost: number;
+  estimatedTimeMs: number;
+}
+
+export interface ConsolidationProgress {
+  runId: string;
+  phase: ConsolidationCheckpoint["phase"];
+  batchesCompleted: number;
+  batchesTotal: number;
+  memoriesProcessed: number;
+  episodesProcessed: number;
+  digestsCreated: number;
+  contradictionsFound: number;
+  tokensUsed: number;
+  estimatedCost: number;
+  errors: string[];
+  startedAt: Date;
+  elapsedMs: number;
+}
+
+export interface RollbackTrigger {
+  type: "error_rate" | "empty_digests" | "contradiction_rate" | "budget_exceeded";
+  threshold: number;
+  current: number;
+  triggered: boolean;
+  message: string;
+}
+
+export class ConsolidationPlan {
+  private db: EngramDatabase;
+  private runId: string;
+  private errors: string[] = [];
+  private emptyDigests: number = 0;
+  private totalDigests: number = 0;
+  private apiCalls: number = 0;
+  private apiErrors: number = 0;
+
+  constructor(db: EngramDatabase) {
+    this.db = db;
+    this.runId = randomUUID();
+  }
+
+  /**
+   * Assess the current backlog and create a consolidation plan
+   */
+  assessBacklog(): BacklogAssessment {
+    const unconsolidatedMem = this.db.getUnconsolidatedMemories(undefined, 10000);
+    const unconsolidatedEp = this.db.getUnconsolidatedEpisodes(10000);
+
+    const recoveryThreshold = this.db.getConfigNumber("recovery_mode_threshold", 100);
+    const isRecoveryMode = unconsolidatedMem.length > recoveryThreshold ||
+                          unconsolidatedEp.length > recoveryThreshold;
+
+    const dailyBudget = this.db.getConfigNumber("daily_budget_usd", 5.0);
+    const dailySpent = this.db.getDailySpending();
+    const budgetRemaining = Math.max(0, dailyBudget - dailySpent);
+
+    const maxBatchesPerRun = this.db.getConfigNumber("max_batches_per_run", 5);
+
+    // Calculate phase plans
+    const episodeBatches = Math.ceil(unconsolidatedEp.length / 20);
+    const memoryBatches = Math.ceil(unconsolidatedMem.length / 15);
+    const totalBatches = episodeBatches + memoryBatches;
+
+    const phases: PhasePlan[] = [];
+    const delayMs = this.db.getConfigNumber("delay_between_calls_ms", 2000);
+
+    // Episode phase (Haiku - cheap)
+    if (unconsolidatedEp.length >= 4) {
+      const batchCount = Math.min(episodeBatches, maxBatchesPerRun);
+      const cost = batchCount * this.estimateEpisodeBatchCost();
+      phases.push({
+        phase: "episodes",
+        itemCount: Math.min(unconsolidatedEp.length, batchCount * 20),
+        batchCount,
+        estimatedCost: cost,
+        estimatedTimeMs: batchCount * (2000 + delayMs), // ~2s per Haiku call + delay
+      });
+    }
+
+    // Memory phase (Opus with thinking - expensive)
+    if (unconsolidatedMem.length >= 5) {
+      const batchCount = Math.min(memoryBatches, maxBatchesPerRun);
+      const cost = batchCount * this.estimateMemoryBatchCost();
+      phases.push({
+        phase: "memories",
+        itemCount: Math.min(unconsolidatedMem.length, batchCount * 15),
+        batchCount,
+        estimatedCost: cost,
+        estimatedTimeMs: batchCount * (15000 + delayMs), // ~15s per Opus call + delay
+      });
+    }
+
+    // Decay and cleanup phases (no API calls)
+    phases.push({ phase: "decay", itemCount: 0, batchCount: 0, estimatedCost: 0, estimatedTimeMs: 100 });
+    phases.push({ phase: "cleanup", itemCount: 0, batchCount: 0, estimatedCost: 0, estimatedTimeMs: 100 });
+
+    const estimatedCost = phases.reduce((sum, p) => sum + p.estimatedCost, 0);
+    const canProceed = estimatedCost <= budgetRemaining;
+
+    // In recovery mode, be more conservative
+    const recommendedBatches = isRecoveryMode
+      ? Math.min(3, maxBatchesPerRun)
+      : maxBatchesPerRun;
+
+    return {
+      unconsolidatedMemories: unconsolidatedMem.length,
+      unconsolidatedEpisodes: unconsolidatedEp.length,
+      isRecoveryMode,
+      estimatedBatches: totalBatches,
+      estimatedCost,
+      dailyBudget,
+      dailySpent,
+      budgetRemaining,
+      canProceed,
+      recommendedBatches,
+      phases,
+    };
+  }
+
+  /**
+   * Get prioritized memories for consolidation
+   * Priority: recent + high importance first, then older chronologically
+   */
+  getPrioritizedMemories(limit: number): Memory[] {
+    const allMemories = this.db.getUnconsolidatedMemories(undefined, 10000);
+
+    // Score each memory
+    const now = Date.now();
+    const dayMs = 24 * 60 * 60 * 1000;
+
+    const scored = allMemories.map(m => {
+      const ageHours = (now - m.timestamp.getTime()) / (60 * 60 * 1000);
+      const ageDays = ageHours / 24;
+
+      // Recency score: 1.0 for today, decays over 7 days
+      const recencyScore = Math.max(0, 1 - (ageDays / 7));
+
+      // Importance score: 0-1
+      const importanceScore = m.importance;
+
+      // Emotional weight: 0-1
+      const emotionalScore = m.emotional_weight;
+
+      // Access frequency bonus
+      const accessBonus = Math.min(0.2, m.access_count * 0.05);
+
+      // Combined priority (weights: recency 40%, importance 30%, emotional 20%, access 10%)
+      const priority = (recencyScore * 0.4) +
+                      (importanceScore * 0.3) +
+                      (emotionalScore * 0.2) +
+                      (accessBonus * 0.1);
+
+      return { memory: m, priority };
+    });
+
+    // Sort by priority (highest first)
+    scored.sort((a, b) => b.priority - a.priority);
+
+    return scored.slice(0, limit).map(s => s.memory);
+  }
+
+  /**
+   * Get prioritized episodes for consolidation
+   */
+  getPrioritizedEpisodes(limit: number): Episode[] {
+    const episodes = this.db.getUnconsolidatedEpisodes(limit);
+
+    // Sort by timestamp (oldest first for episodes - process in order)
+    episodes.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
+
+    return episodes;
+  }
+
+  /**
+   * Create checkpoint for this run
+   */
+  createCheckpoint(phase: ConsolidationCheckpoint["phase"], batchesTotal: number): ConsolidationCheckpoint {
+    return this.db.createCheckpoint(this.runId, phase, batchesTotal);
+  }
+
+  /**
+   * Update checkpoint progress
+   */
+  updateProgress(updates: Partial<{
+    phase: ConsolidationCheckpoint["phase"];
+    batchesCompleted: number;
+    batchesTotal: number;
+    memoriesProcessed: number;
+    episodesProcessed: number;
+    digestsCreated: number;
+    contradictionsFound: number;
+    tokensUsed: number;
+    estimatedCost: number;
+  }>): void {
+    this.db.updateCheckpoint(this.runId, {
+      phase: updates.phase,
+      batches_completed: updates.batchesCompleted,
+      batches_total: updates.batchesTotal,
+      memories_processed: updates.memoriesProcessed,
+      episodes_processed: updates.episodesProcessed,
+      digests_created: updates.digestsCreated,
+      contradictions_found: updates.contradictionsFound,
+      tokens_used: updates.tokensUsed,
+      estimated_cost_usd: updates.estimatedCost,
+    });
+  }
+
+  /**
+   * Mark run as complete
+   */
+  complete(): void {
+    this.db.completeCheckpoint(this.runId);
+  }
+
+  /**
+   * Mark run as failed
+   */
+  fail(error: string): void {
+    this.db.updateCheckpoint(this.runId, { error });
+  }
+
+  /**
+   * Get current progress
+   */
+  getProgress(): ConsolidationProgress {
+    const checkpoint = this.db.getCheckpoint(this.runId);
+
+    return {
+      runId: this.runId,
+      phase: checkpoint?.phase || "episodes",
+      batchesCompleted: checkpoint?.batches_completed || 0,
+      batchesTotal: checkpoint?.batches_total || 0,
+      memoriesProcessed: checkpoint?.memories_processed || 0,
+      episodesProcessed: checkpoint?.episodes_processed || 0,
+      digestsCreated: checkpoint?.digests_created || 0,
+      contradictionsFound: checkpoint?.contradictions_found || 0,
+      tokensUsed: checkpoint?.tokens_used || 0,
+      estimatedCost: checkpoint?.estimated_cost_usd || 0,
+      errors: this.errors,
+      startedAt: checkpoint?.started_at || new Date(),
+      elapsedMs: checkpoint ? Date.now() - checkpoint.started_at.getTime() : 0,
+    };
+  }
+
+  /**
+   * Check if we should resume a previous incomplete run
+   */
+  checkForResume(): ConsolidationCheckpoint | null {
+    return this.db.getIncompleteCheckpoint();
+  }
+
+  /**
+   * Resume from a previous checkpoint
+   */
+  resumeFrom(checkpoint: ConsolidationCheckpoint): void {
+    this.runId = checkpoint.run_id;
+  }
+
+  /**
+   * Record an API call result for tracking
+   */
+  recordApiCall(success: boolean, tokensUsed?: number): void {
+    this.apiCalls++;
+    if (!success) {
+      this.apiErrors++;
+    }
+  }
+
+  /**
+   * Record a digest creation result
+   */
+  recordDigest(isEmpty: boolean): void {
+    this.totalDigests++;
+    if (isEmpty) {
+      this.emptyDigests++;
+    }
+  }
+
+  /**
+   * Record an error
+   */
+  recordError(error: string): void {
+    this.errors.push(error);
+  }
+
+  /**
+   * Check rollback triggers and return any that fired
+   */
+  checkRollbackTriggers(): RollbackTrigger[] {
+    const triggers: RollbackTrigger[] = [];
+
+    // Error rate threshold
+    const errorRateThreshold = this.db.getConfigNumber("error_rate_threshold", 0.3);
+    if (this.apiCalls >= 3) {
+      const errorRate = this.apiErrors / this.apiCalls;
+      triggers.push({
+        type: "error_rate",
+        threshold: errorRateThreshold,
+        current: errorRate,
+        triggered: errorRate > errorRateThreshold,
+        message: `API error rate ${(errorRate * 100).toFixed(1)}% exceeds ${(errorRateThreshold * 100).toFixed(0)}%`,
+      });
+    }
+
+    // Empty digest threshold
+    const emptyDigestThreshold = this.db.getConfigNumber("empty_digest_threshold", 0.2);
+    if (this.totalDigests >= 3) {
+      const emptyRate = this.emptyDigests / this.totalDigests;
+      triggers.push({
+        type: "empty_digests",
+        threshold: emptyDigestThreshold,
+        current: emptyRate,
+        triggered: emptyRate > emptyDigestThreshold,
+        message: `Empty digest rate ${(emptyRate * 100).toFixed(1)}% exceeds ${(emptyDigestThreshold * 100).toFixed(0)}%`,
+      });
+    }
+
+    // Budget exceeded
+    const dailyBudget = this.db.getConfigNumber("daily_budget_usd", 5.0);
+    const dailySpent = this.db.getDailySpending();
+    triggers.push({
+      type: "budget_exceeded",
+      threshold: dailyBudget,
+      current: dailySpent,
+      triggered: dailySpent > dailyBudget,
+      message: `Daily spending $${dailySpent.toFixed(2)} exceeds budget $${dailyBudget.toFixed(2)}`,
+    });
+
+    return triggers;
+  }
+
+  /**
+   * Delay between API calls (rate limiting)
+   */
+  async delay(): Promise<void> {
+    const delayMs = this.db.getConfigNumber("delay_between_calls_ms", 2000);
+    await new Promise(resolve => setTimeout(resolve, delayMs));
+  }
+
+  /**
+   * Estimate cost for an episode batch (Haiku)
+   */
+  private estimateEpisodeBatchCost(): number {
+    const { input, output } = TOKEN_ESTIMATES.episodeBatch;
+    return (input * PRICING.haiku.input) + (output * PRICING.haiku.output);
+  }
+
+  /**
+   * Estimate cost for a memory batch (Opus with thinking)
+   */
+  private estimateMemoryBatchCost(): number {
+    const { input, output, thinking } = TOKEN_ESTIMATES.memoryBatch;
+    return (input * PRICING.opus.input) +
+           (output * PRICING.opus.output) +
+           (thinking * PRICING.opus.thinking);
+  }
+
+  /**
+   * Calculate actual cost from token usage
+   */
+  calculateCost(model: "opus" | "haiku", inputTokens: number, outputTokens: number, thinkingTokens?: number): number {
+    const pricing = PRICING[model];
+    let cost = (inputTokens * pricing.input) + (outputTokens * pricing.output);
+
+    if (model === "opus" && thinkingTokens) {
+      cost += thinkingTokens * PRICING.opus.thinking;
+    }
+
+    return cost;
+  }
+
+  /**
+   * Get the run ID
+   */
+  getRunId(): string {
+    return this.runId;
+  }
+}

package/src/index.ts

@@ -14,6 +14,10 @@ import {
 } from "@modelcontextprotocol/sdk/types.js";
 import path from "path";
 import os from "os";
+import fs from "fs";
+
+import { getTransportMode, getHttpPort } from "./transport/index.js";
+import { startHttpServer } from "./transport/http.js";
 
 import { EngramDatabase } from "./storage/database.js";
 import { KnowledgeGraph } from "./graph/knowledge-graph.js";
@@ -29,6 +33,102 @@ const DB_PATH = process.env.ENGRAM_DB_PATH
   : path.join(os.homedir(), ".engram");
 
 const DB_FILE = path.join(DB_PATH, "engram.db");
+const PID_FILE = path.join(DB_PATH, "engram.pid");
+
+// ============ Zombie Prevention ============
+
+/**
+ * Kill any existing engram process and clean up stale PID file
+ */
+function cleanupZombies(): void {
+  try {
+    if (fs.existsSync(PID_FILE)) {
+      const oldPid = parseInt(fs.readFileSync(PID_FILE, "utf-8").trim(), 10);
+      if (oldPid && oldPid !== process.pid) {
+        try {
+          // Check if process exists
+          process.kill(oldPid, 0);
+          // It exists, kill it
+          console.error(`[Engram] Killing old instance (PID ${oldPid})`);
+          process.kill(oldPid, "SIGTERM");
+        } catch {
+          // Process doesn't exist, that's fine
+        }
+      }
+      fs.unlinkSync(PID_FILE);
+    }
+  } catch (error) {
+    console.error("[Engram] Error cleaning up zombies:", error);
+  }
+}
+
+/**
+ * Write our PID file
+ */
+function writePidFile(): void {
+  try {
+    // Ensure directory exists
+    if (!fs.existsSync(DB_PATH)) {
+      fs.mkdirSync(DB_PATH, { recursive: true });
+    }
+    fs.writeFileSync(PID_FILE, String(process.pid));
+  } catch (error) {
+    console.error("[Engram] Error writing PID file:", error);
+  }
+}
+
+/**
+ * Clean up on exit
+ */
+function cleanup(): void {
+  try {
+    if (fs.existsSync(PID_FILE)) {
+      const storedPid = parseInt(fs.readFileSync(PID_FILE, "utf-8").trim(), 10);
+      if (storedPid === process.pid) {
+        fs.unlinkSync(PID_FILE);
+      }
+    }
+    if (webServer) {
+      webServer.stop();
+    }
+    if (db) {
+      db.close();
+    }
+  } catch {
+    // Ignore cleanup errors
+  }
+}
+
+// Register signal handlers early
+process.on("SIGTERM", () => {
+  console.error("[Engram] Received SIGTERM, shutting down...");
+  cleanup();
+  process.exit(0);
+});
+
+process.on("SIGINT", () => {
+  console.error("[Engram] Received SIGINT, shutting down...");
+  cleanup();
+  process.exit(0);
+});
+
+process.on("exit", cleanup);
+
+// Detect when parent process (Claude) dies by monitoring stdin
+// Only needed in stdio mode
+if (getTransportMode() === "stdio") {
+  process.stdin.on("end", () => {
+    console.error("[Engram] stdin closed, parent process likely died. Shutting down...");
+    cleanup();
+    process.exit(0);
+  });
+
+  process.stdin.on("close", () => {
+    console.error("[Engram] stdin closed, shutting down...");
+    cleanup();
+    process.exit(0);
+  });
+}
 
 // ============ Initialize Components ============
 
@@ -402,7 +502,29 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
           includeGraph: include_graph,
         });
 
+        // Format digests (synthesized context - these provide broad understanding)
+        const digestsFormatted = response.digests.map((d) => ({
+          type: "digest" as const,
+          id: d.digest.id,
+          level: d.digest.level,  // 1=session, 2=topic, 3=entity
+          topic: d.digest.topic,
+          content: d.digest.content,
+          source_count: d.digest.source_count,
+          period: {
+            start: d.digest.period_start.toISOString(),
+            end: d.digest.period_end.toISOString(),
+          },
+          relevance_score: d.score.toFixed(4),
+          // Key evidence - specific memories supporting this synthesis
+          key_memories: d.key_memories.map((m) => ({
+            id: m.id,
+            content: m.content,
+            timestamp: m.timestamp.toISOString(),
+          })),
+        }));
+
         const formatted = response.results.map((r) => ({
+          type: "memory" as const,
           id: r.memory.id,
           content: r.memory.content,
           source: r.memory.source,
@@ -417,6 +539,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
 
         // Format connected memories (Hebbian associations)
         const connectedFormatted = response.connected_memories.map((c) => ({
+          type: "connected" as const,
           id: c.memory.id,
           content: c.memory.content,
           connected_to: c.connected_to,
@@ -430,10 +553,16 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
               text: JSON.stringify({
                 recall_id: response.recall_id,  // For memory_feedback
                 query,
+                // Digests first - they provide synthesized context
+                digests: digestsFormatted,
+                digests_count: digestsFormatted.length,
+                // Then individual memories for specific evidence
                 results: formatted,
                 count: formatted.length,
                 connected_memories: connectedFormatted,
-                hint: formatted.length > 0 ? "Call memory_feedback with useful_memory_ids after answering" : undefined,
+                hint: formatted.length > 0 || digestsFormatted.length > 0
+                  ? "Call memory_feedback with useful_memory_ids after answering"
+                  : undefined,
               }, null, 2),
             },
           ],
@@ -624,17 +753,44 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
           need_more?: boolean;
         };
 
-        // Update the retrieval log with feedback
-        const updated = db.updateRetrievalFeedback(recall_id, useful_memory_ids, need_more);
+        // First, get the original recall to validate useful_memory_ids
+        const retrievalLog = db.getRetrievalLog(recall_id);
+        if (!retrievalLog) {
+          return {
+            content: [
+              {
+                type: "text" as const,
+                text: JSON.stringify({
+                  success: false,
+                  error: `Recall ID not found: ${recall_id}`,
+                }),
+              },
+            ],
+          };
+        }
+
+        // Validate: only accept IDs that were in the original recall
+        const originalIdSet = new Set(retrievalLog.memory_ids);
+        const validUsefulIds = useful_memory_ids.filter(id => originalIdSet.has(id));
+        const invalidIds = useful_memory_ids.filter(id => !originalIdSet.has(id));
+
+        if (invalidIds.length > 0) {
+          console.error(`[Engram] memory_feedback: ${invalidIds.length} IDs not in original recall, ignored: ${invalidIds.join(", ")}`);
+        }
+
+        // Update the retrieval log with validated feedback
+        const updated = db.updateRetrievalFeedback(recall_id, validUsefulIds, need_more);
 
         if (!updated) {
+          // Should not happen since we already checked above, but handle gracefully
+          console.error(`[Engram] memory_feedback: failed to update retrieval log ${recall_id}`);
           return {
             content: [
               {
                 type: "text" as const,
                 text: JSON.stringify({
                   success: false,
-                  error: `Recall ID not found: ${recall_id}`,
+                  error: `Failed to update feedback for: ${recall_id}`,
                 }),
               },
             ],
@@ -664,7 +820,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                 text: JSON.stringify({
                   success: true,
                   feedback_recorded: true,
-                  useful_count: useful_memory_ids.length,
+                  useful_count: validUsefulIds.length,
                   expanded_search: true,
                   additional_results: formatted,
                   additional_count: formatted.length,
@@ -697,7 +853,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
               text: JSON.stringify({
                 success: true,
                 feedback_recorded: true,
-                useful_count: useful_memory_ids.length,
+                useful_count: validUsefulIds.length,
                 learning_applied: learningApplied > 0,
                 connections_strengthened: learningApplied,
               }, null, 2),
@@ -726,12 +882,27 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
 // ============ Main ============
 
 async function main() {
-  await initialize();
+  const transportMode = getTransportMode();
 
-  const transport = new StdioServerTransport();
-  await server.connect(transport);
+  // Zombie cleanup only needed in stdio mode (local usage)
+  if (transportMode === "stdio") {
+    cleanupZombies();
+    writePidFile();
+  }
 
-  console.error("[Engram] MCP server running on stdio");
+  await initialize();
+
+  if (transportMode === "http") {
+    // HTTP mode - for Railway/remote deployment
+    const port = getHttpPort();
+    await startHttpServer({ port, server });
+    console.error(`[Engram] MCP server running in HTTP mode (PID ${process.pid})`);
+  } else {
+    // Stdio mode (default) - for local Claude Desktop/Cursor
+    const transport = new StdioServerTransport();
+    await server.connect(transport);
+    console.error(`[Engram] MCP server running on stdio (PID ${process.pid})`);
+  }
 }
 
 main().catch((error) => {