@199-bio/engram 0.8.1 → 0.10.0

package/src/consolidation/plan.ts

@@ -0,0 +1,444 @@
+/**
+ * Consolidation Plan
+ *
+ * Implements a Standard Operating Procedure for safe consolidation of large backlogs.
+ * Prevents damage through:
+ * - Assessment and recovery mode detection
+ * - Prioritization (recent + high importance first)
+ * - Rate limiting with delays between API calls
+ * - Budget tracking and cost caps
+ * - Checkpointing for resume capability
+ * - Validation with soft rollback triggers
+ */
+
+import { randomUUID } from "crypto";
+import { EngramDatabase, Memory, Episode, ConsolidationCheckpoint } from "../storage/database.js";
+
+// Token pricing (Opus 4.5 with extended thinking)
+const PRICING = {
+  opus: {
+    input: 15 / 1_000_000,      // $15 per 1M input tokens
+    output: 75 / 1_000_000,     // $75 per 1M output tokens
+    thinking: 15 / 1_000_000,   // $15 per 1M thinking tokens (same as input)
+  },
+  haiku: {
+    input: 0.80 / 1_000_000,    // $0.80 per 1M input tokens
+    output: 4.00 / 1_000_000,   // $4.00 per 1M output tokens
+  },
+};
+
+// Estimated tokens per operation (conservative estimates)
+const TOKEN_ESTIMATES = {
+  episodeBatch: {
+    input: 2000,    // Conversation text + system prompt
+    output: 1000,   // Extracted memories JSON
+  },
+  memoryBatch: {
+    input: 3000,     // Memories + system prompt
+    output: 2000,    // Digest + contradictions
+    thinking: 10000, // Extended thinking budget
+  },
+  entityProfile: {
+    input: 4000,
+    output: 3000,
+    thinking: 16000,
+  },
+};
+
+export interface BacklogAssessment {
+  unconsolidatedMemories: number;
+  unconsolidatedEpisodes: number;
+  isRecoveryMode: boolean;
+  estimatedBatches: number;
+  estimatedCost: number;
+  dailyBudget: number;
+  dailySpent: number;
+  budgetRemaining: number;
+  canProceed: boolean;
+  recommendedBatches: number;
+  phases: PhasePlan[];
+}
+
+export interface PhasePlan {
+  phase: "episodes" | "memories" | "decay" | "cleanup";
+  itemCount: number;
+  batchCount: number;
+  estimatedCost: number;
+  estimatedTimeMs: number;
+}
+
+export interface ConsolidationProgress {
+  runId: string;
+  phase: ConsolidationCheckpoint["phase"];
+  batchesCompleted: number;
+  batchesTotal: number;
+  memoriesProcessed: number;
+  episodesProcessed: number;
+  digestsCreated: number;
+  contradictionsFound: number;
+  tokensUsed: number;
+  estimatedCost: number;
+  errors: string[];
+  startedAt: Date;
+  elapsedMs: number;
+}
+
+export interface RollbackTrigger {
+  type: "error_rate" | "empty_digests" | "contradiction_rate" | "budget_exceeded";
+  threshold: number;
+  current: number;
+  triggered: boolean;
+  message: string;
+}
+
+export class ConsolidationPlan {
+  private db: EngramDatabase;
+  private runId: string;
+  private errors: string[] = [];
+  private emptyDigests: number = 0;
+  private totalDigests: number = 0;
+  private apiCalls: number = 0;
+  private apiErrors: number = 0;
+
+  constructor(db: EngramDatabase) {
+    this.db = db;
+    this.runId = randomUUID();
+  }
+
+  /**
+   * Assess the current backlog and create a consolidation plan
+   */
+  assessBacklog(): BacklogAssessment {
+    const unconsolidatedMem = this.db.getUnconsolidatedMemories(undefined, 10000);
+    const unconsolidatedEp = this.db.getUnconsolidatedEpisodes(10000);
+
+    const recoveryThreshold = this.db.getConfigNumber("recovery_mode_threshold", 100);
+    const isRecoveryMode = unconsolidatedMem.length > recoveryThreshold ||
+                          unconsolidatedEp.length > recoveryThreshold;
+
+    const dailyBudget = this.db.getConfigNumber("daily_budget_usd", 5.0);
+    const dailySpent = this.db.getDailySpending();
+    const budgetRemaining = Math.max(0, dailyBudget - dailySpent);
+
+    const maxBatchesPerRun = this.db.getConfigNumber("max_batches_per_run", 5);
+
+    // Calculate phase plans
+    const episodeBatches = Math.ceil(unconsolidatedEp.length / 20);
+    const memoryBatches = Math.ceil(unconsolidatedMem.length / 15);
+    const totalBatches = episodeBatches + memoryBatches;
+
+    const phases: PhasePlan[] = [];
+    const delayMs = this.db.getConfigNumber("delay_between_calls_ms", 2000);
+
+    // Episode phase (Haiku - cheap)
+    if (unconsolidatedEp.length >= 4) {
+      const batchCount = Math.min(episodeBatches, maxBatchesPerRun);
+      const cost = batchCount * this.estimateEpisodeBatchCost();
+      phases.push({
+        phase: "episodes",
+        itemCount: Math.min(unconsolidatedEp.length, batchCount * 20),
+        batchCount,
+        estimatedCost: cost,
+        estimatedTimeMs: batchCount * (2000 + delayMs), // ~2s per Haiku call + delay
+      });
+    }
+
+    // Memory phase (Opus with thinking - expensive)
+    if (unconsolidatedMem.length >= 5) {
+      const batchCount = Math.min(memoryBatches, maxBatchesPerRun);
+      const cost = batchCount * this.estimateMemoryBatchCost();
+      phases.push({
+        phase: "memories",
+        itemCount: Math.min(unconsolidatedMem.length, batchCount * 15),
+        batchCount,
+        estimatedCost: cost,
+        estimatedTimeMs: batchCount * (15000 + delayMs), // ~15s per Opus call + delay
+      });
+    }
+
+    // Decay and cleanup phases (no API calls)
+    phases.push({ phase: "decay", itemCount: 0, batchCount: 0, estimatedCost: 0, estimatedTimeMs: 100 });
+    phases.push({ phase: "cleanup", itemCount: 0, batchCount: 0, estimatedCost: 0, estimatedTimeMs: 100 });
+
+    const estimatedCost = phases.reduce((sum, p) => sum + p.estimatedCost, 0);
+    const canProceed = estimatedCost <= budgetRemaining;
+
+    // In recovery mode, be more conservative
+    const recommendedBatches = isRecoveryMode
+      ? Math.min(3, maxBatchesPerRun)
+      : maxBatchesPerRun;
+
+    return {
+      unconsolidatedMemories: unconsolidatedMem.length,
+      unconsolidatedEpisodes: unconsolidatedEp.length,
+      isRecoveryMode,
+      estimatedBatches: totalBatches,
+      estimatedCost,
+      dailyBudget,
+      dailySpent,
+      budgetRemaining,
+      canProceed,
+      recommendedBatches,
+      phases,
+    };
+  }
+
+  /**
+   * Get prioritized memories for consolidation
+   * Priority: recent + high importance first, then older chronologically
+   */
+  getPrioritizedMemories(limit: number): Memory[] {
+    const allMemories = this.db.getUnconsolidatedMemories(undefined, 10000);
+
+    // Score each memory
+    const now = Date.now();
+    const dayMs = 24 * 60 * 60 * 1000;
+
+    const scored = allMemories.map(m => {
+      const ageHours = (now - m.timestamp.getTime()) / (60 * 60 * 1000);
+      const ageDays = ageHours / 24;
+
+      // Recency score: 1.0 for today, decays over 7 days
+      const recencyScore = Math.max(0, 1 - (ageDays / 7));
+
+      // Importance score: 0-1
+      const importanceScore = m.importance;
+
+      // Emotional weight: 0-1
+      const emotionalScore = m.emotional_weight;
+
+      // Access frequency bonus
+      const accessBonus = Math.min(0.2, m.access_count * 0.05);
+
+      // Combined priority (weights: recency 40%, importance 30%, emotional 20%, access 10%)
+      const priority = (recencyScore * 0.4) +
+                      (importanceScore * 0.3) +
+                      (emotionalScore * 0.2) +
+                      (accessBonus * 0.1);
+
+      return { memory: m, priority };
+    });
+
+    // Sort by priority (highest first)
+    scored.sort((a, b) => b.priority - a.priority);
+
+    return scored.slice(0, limit).map(s => s.memory);
+  }
+
+  /**
+   * Get prioritized episodes for consolidation
+   */
+  getPrioritizedEpisodes(limit: number): Episode[] {
+    const episodes = this.db.getUnconsolidatedEpisodes(limit);
+
+    // Sort by timestamp (oldest first for episodes - process in order)
+    episodes.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
+
+    return episodes;
+  }
+
+  /**
+   * Create checkpoint for this run
+   */
+  createCheckpoint(phase: ConsolidationCheckpoint["phase"], batchesTotal: number): ConsolidationCheckpoint {
+    return this.db.createCheckpoint(this.runId, phase, batchesTotal);
+  }
+
+  /**
+   * Update checkpoint progress
+   */
+  updateProgress(updates: Partial<{
+    phase: ConsolidationCheckpoint["phase"];
+    batchesCompleted: number;
+    batchesTotal: number;
+    memoriesProcessed: number;
+    episodesProcessed: number;
+    digestsCreated: number;
+    contradictionsFound: number;
+    tokensUsed: number;
+    estimatedCost: number;
+  }>): void {
+    this.db.updateCheckpoint(this.runId, {
+      phase: updates.phase,
+      batches_completed: updates.batchesCompleted,
+      batches_total: updates.batchesTotal,
+      memories_processed: updates.memoriesProcessed,
+      episodes_processed: updates.episodesProcessed,
+      digests_created: updates.digestsCreated,
+      contradictions_found: updates.contradictionsFound,
+      tokens_used: updates.tokensUsed,
+      estimated_cost_usd: updates.estimatedCost,
+    });
+  }
+
+  /**
+   * Mark run as complete
+   */
+  complete(): void {
+    this.db.completeCheckpoint(this.runId);
+  }
+
+  /**
+   * Mark run as failed
+   */
+  fail(error: string): void {
+    this.db.updateCheckpoint(this.runId, { error });
+  }
+
+  /**
+   * Get current progress
+   */
+  getProgress(): ConsolidationProgress {
+    const checkpoint = this.db.getCheckpoint(this.runId);
+
+    return {
+      runId: this.runId,
+      phase: checkpoint?.phase || "episodes",
+      batchesCompleted: checkpoint?.batches_completed || 0,
+      batchesTotal: checkpoint?.batches_total || 0,
+      memoriesProcessed: checkpoint?.memories_processed || 0,
+      episodesProcessed: checkpoint?.episodes_processed || 0,
+      digestsCreated: checkpoint?.digests_created || 0,
+      contradictionsFound: checkpoint?.contradictions_found || 0,
+      tokensUsed: checkpoint?.tokens_used || 0,
+      estimatedCost: checkpoint?.estimated_cost_usd || 0,
+      errors: this.errors,
+      startedAt: checkpoint?.started_at || new Date(),
+      elapsedMs: checkpoint ? Date.now() - checkpoint.started_at.getTime() : 0,
+    };
+  }
+
+  /**
+   * Check if we should resume a previous incomplete run
+   */
+  checkForResume(): ConsolidationCheckpoint | null {
+    return this.db.getIncompleteCheckpoint();
+  }
+
+  /**
+   * Resume from a previous checkpoint
+   */
+  resumeFrom(checkpoint: ConsolidationCheckpoint): void {
+    this.runId = checkpoint.run_id;
+  }
+
+  /**
+   * Record an API call result for tracking
+   */
+  recordApiCall(success: boolean, tokensUsed?: number): void {
+    this.apiCalls++;
+    if (!success) {
+      this.apiErrors++;
+    }
+  }
+
+  /**
+   * Record a digest creation result
+   */
+  recordDigest(isEmpty: boolean): void {
+    this.totalDigests++;
+    if (isEmpty) {
+      this.emptyDigests++;
+    }
+  }
+
+  /**
+   * Record an error
+   */
+  recordError(error: string): void {
+    this.errors.push(error);
+  }
+
+  /**
+   * Check rollback triggers and return any that fired
+   */
+  checkRollbackTriggers(): RollbackTrigger[] {
+    const triggers: RollbackTrigger[] = [];
+
+    // Error rate threshold
+    const errorRateThreshold = this.db.getConfigNumber("error_rate_threshold", 0.3);
+    if (this.apiCalls >= 3) {
+      const errorRate = this.apiErrors / this.apiCalls;
+      triggers.push({
+        type: "error_rate",
+        threshold: errorRateThreshold,
+        current: errorRate,
+        triggered: errorRate > errorRateThreshold,
+        message: `API error rate ${(errorRate * 100).toFixed(1)}% exceeds ${(errorRateThreshold * 100).toFixed(0)}%`,
+      });
+    }
+
+    // Empty digest threshold
+    const emptyDigestThreshold = this.db.getConfigNumber("empty_digest_threshold", 0.2);
+    if (this.totalDigests >= 3) {
+      const emptyRate = this.emptyDigests / this.totalDigests;
+      triggers.push({
+        type: "empty_digests",
+        threshold: emptyDigestThreshold,
+        current: emptyRate,
+        triggered: emptyRate > emptyDigestThreshold,
+        message: `Empty digest rate ${(emptyRate * 100).toFixed(1)}% exceeds ${(emptyDigestThreshold * 100).toFixed(0)}%`,
+      });
+    }
+
+    // Budget exceeded
+    const dailyBudget = this.db.getConfigNumber("daily_budget_usd", 5.0);
+    const dailySpent = this.db.getDailySpending();
+    triggers.push({
+      type: "budget_exceeded",
+      threshold: dailyBudget,
+      current: dailySpent,
+      triggered: dailySpent > dailyBudget,
+      message: `Daily spending $${dailySpent.toFixed(2)} exceeds budget $${dailyBudget.toFixed(2)}`,
+    });
+
+    return triggers;
+  }
+
+  /**
+   * Delay between API calls (rate limiting)
+   */
+  async delay(): Promise<void> {
+    const delayMs = this.db.getConfigNumber("delay_between_calls_ms", 2000);
+    await new Promise(resolve => setTimeout(resolve, delayMs));
+  }
+
+  /**
+   * Estimate cost for an episode batch (Haiku)
+   */
+  private estimateEpisodeBatchCost(): number {
+    const { input, output } = TOKEN_ESTIMATES.episodeBatch;
+    return (input * PRICING.haiku.input) + (output * PRICING.haiku.output);
+  }
+
+  /**
+   * Estimate cost for a memory batch (Opus with thinking)
+   */
+  private estimateMemoryBatchCost(): number {
+    const { input, output, thinking } = TOKEN_ESTIMATES.memoryBatch;
+    return (input * PRICING.opus.input) +
+           (output * PRICING.opus.output) +
+           (thinking * PRICING.opus.thinking);
+  }
+
+  /**
+   * Calculate actual cost from token usage
+   */
+  calculateCost(model: "opus" | "haiku", inputTokens: number, outputTokens: number, thinkingTokens?: number): number {
+    const pricing = PRICING[model];
+    let cost = (inputTokens * pricing.input) + (outputTokens * pricing.output);
+
+    if (model === "opus" && thinkingTokens) {
+      cost += thinkingTokens * PRICING.opus.thinking;
+    }
+
+    return cost;
+  }
+
+  /**
+   * Get the run ID
+   */
+  getRunId(): string {
+    return this.runId;
+  }
+}

package/src/index.ts

@@ -16,6 +16,9 @@ import path from "path";
 import os from "os";
 import fs from "fs";
 
+import { getTransportMode, getHttpPort } from "./transport/index.js";
+import { startHttpServer } from "./transport/http.js";
+
 import { EngramDatabase } from "./storage/database.js";
 import { KnowledgeGraph } from "./graph/knowledge-graph.js";
 import { createRetriever } from "./retrieval/colbert.js";
@@ -112,17 +115,20 @@ process.on("SIGINT", () => {
 process.on("exit", cleanup);
 
 // Detect when parent process (Claude) dies by monitoring stdin
-process.stdin.on("end", () => {
-  console.error("[Engram] stdin closed, parent process likely died. Shutting down...");
-  cleanup();
-  process.exit(0);
-});
-
-process.stdin.on("close", () => {
-  console.error("[Engram] stdin closed, shutting down...");
-  cleanup();
-  process.exit(0);
-});
+// Only needed in stdio mode
+if (getTransportMode() === "stdio") {
+  process.stdin.on("end", () => {
+    console.error("[Engram] stdin closed, parent process likely died. Shutting down...");
+    cleanup();
+    process.exit(0);
+  });
+
+  process.stdin.on("close", () => {
+    console.error("[Engram] stdin closed, shutting down...");
+    cleanup();
+    process.exit(0);
+  });
+}
 
 // ============ Initialize Components ============
 
@@ -496,7 +502,29 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
           includeGraph: include_graph,
         });
 
+        // Format digests (synthesized context - these provide broad understanding)
+        const digestsFormatted = response.digests.map((d) => ({
+          type: "digest" as const,
+          id: d.digest.id,
+          level: d.digest.level,  // 1=session, 2=topic, 3=entity
+          topic: d.digest.topic,
+          content: d.digest.content,
+          source_count: d.digest.source_count,
+          period: {
+            start: d.digest.period_start.toISOString(),
+            end: d.digest.period_end.toISOString(),
+          },
+          relevance_score: d.score.toFixed(4),
+          // Key evidence - specific memories supporting this synthesis
+          key_memories: d.key_memories.map((m) => ({
+            id: m.id,
+            content: m.content,
+            timestamp: m.timestamp.toISOString(),
+          })),
+        }));
+
         const formatted = response.results.map((r) => ({
+          type: "memory" as const,
           id: r.memory.id,
           content: r.memory.content,
           source: r.memory.source,
@@ -511,6 +539,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
 
         // Format connected memories (Hebbian associations)
         const connectedFormatted = response.connected_memories.map((c) => ({
+          type: "connected" as const,
           id: c.memory.id,
           content: c.memory.content,
           connected_to: c.connected_to,
@@ -524,10 +553,16 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
               text: JSON.stringify({
                 recall_id: response.recall_id,  // For memory_feedback
                 query,
+                // Digests first - they provide synthesized context
+                digests: digestsFormatted,
+                digests_count: digestsFormatted.length,
+                // Then individual memories for specific evidence
                 results: formatted,
                 count: formatted.length,
                 connected_memories: connectedFormatted,
-                hint: formatted.length > 0 ? "Call memory_feedback with useful_memory_ids after answering" : undefined,
+                hint: formatted.length > 0 || digestsFormatted.length > 0
+                  ? "Call memory_feedback with useful_memory_ids after answering"
+                  : undefined,
               }, null, 2),
             },
           ],
@@ -847,16 +882,27 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
 // ============ Main ============
 
 async function main() {
-  // Kill any zombie instances before starting
-  cleanupZombies();
-  writePidFile();
+  const transportMode = getTransportMode();
 
-  await initialize();
+  // Zombie cleanup only needed in stdio mode (local usage)
+  if (transportMode === "stdio") {
+    cleanupZombies();
+    writePidFile();
+  }
 
-  const transport = new StdioServerTransport();
-  await server.connect(transport);
+  await initialize();
 
-  console.error(`[Engram] MCP server running on stdio (PID ${process.pid})`);
+  if (transportMode === "http") {
+    // HTTP mode - for Railway/remote deployment
+    const port = getHttpPort();
+    await startHttpServer({ port, server });
+    console.error(`[Engram] MCP server running in HTTP mode (PID ${process.pid})`);
+  } else {
+    // Stdio mode (default) - for local Claude Desktop/Cursor
+    const transport = new StdioServerTransport();
+    await server.connect(transport);
+    console.error(`[Engram] MCP server running on stdio (PID ${process.pid})`);
+  }
 }
 
 main().catch((error) => {

package/src/retrieval/hybrid.ts

@@ -4,7 +4,7 @@
  * Enhanced with temporal decay and salience scoring
  */
 
-import { EngramDatabase, Memory } from "../storage/database.js";
+import { EngramDatabase, Memory, Digest } from "../storage/database.js";
 import { KnowledgeGraph } from "../graph/knowledge-graph.js";
 import { ColBERTRetriever, SimpleRetriever, SearchResult, Document } from "./colbert.js";
 
@@ -20,8 +20,15 @@ export interface HybridSearchResult {
   };
 }
 
+export interface DigestSearchResult {
+  digest: Digest;
+  score: number;
+  key_memories: Memory[];  // 2-3 source memories that best support this digest
+}
+
 export interface HybridSearchResponse {
   results: HybridSearchResult[];
+  digests: DigestSearchResult[];  // Relevant synthesized context
   recall_id: string;  // For LLM feedback
   connected_memories: Array<{
     memory: Memory;
@@ -179,11 +186,15 @@ export class HybridSearch {
     if (allCandidateIds.size === 0) {
       return {
         results: [],
+        digests: [],
         recall_id: recallId,
         connected_memories: [],
       };
     }
 
+    // Search digests via BM25 (top 3 relevant digests)
+    const digestResults = this.searchDigests(query, 3);
+
     // Create rankings for RRF
     const rankings: Map<string, { bm25?: number; semantic?: number; graph?: number; connected?: number }> = new Map();
 
@@ -335,13 +346,62 @@ export class HybridSearch {
       }
     }
 
+    // TOKEN EFFICIENCY: If digests are returned, reduce memory count
+    // Digest provides context (synthesis), memories provide evidence (specifics)
+    // Return fewer memories when we have good digest coverage
+    let finalResults = results;
+    if (digestResults.length > 0) {
+      // Get IDs of memories already covered by digests as key_memories
+      const coveredByDigests = new Set<string>();
+      digestResults.forEach(d => d.key_memories.forEach(m => coveredByDigests.add(m.id)));
+
+      // Keep memories not already shown as key_memories in digests
+      // Also limit to fewer since digests provide the context
+      const maxMemoriesWithDigests = Math.max(2, Math.floor(limit / 2));
+      finalResults = results
+        .filter(r => !coveredByDigests.has(r.memory.id))
+        .slice(0, maxMemoriesWithDigests);
+    }
+
     return {
-      results,
+      results: finalResults,
+      digests: digestResults,
       recall_id: recallId,
       connected_memories: connectedMemories,
     };
   }
 
+  /**
+   * Search digests via BM25 and return with key source memories
+   * Returns top N digests with 2-3 representative source memories each
+   */
+  private searchDigests(query: string, limit: number): DigestSearchResult[] {
+    try {
+      const digestHits = this.db.searchDigestsBM25(query, limit);
+
+      return digestHits.map(hit => {
+        // Get source memories for this digest, take top 3 most relevant
+        const sources = this.db.getDigestSources(hit.id);
+        // Sort by importance and recency, take best 3
+        const keyMemories = sources
+          .sort((a, b) => {
+            const scoreA = (a.importance || 0.5) + (a.access_count || 0) * 0.1;
+            const scoreB = (b.importance || 0.5) + (b.access_count || 0) * 0.1;
+            return scoreB - scoreA;
+          })
+          .slice(0, 3);
+
+        return {
+          digest: hit,
+          score: Math.abs(hit.score),  // BM25 returns negative scores
+          key_memories: keyMemories,
+        };
+      });
+    } catch {
+      return [];
+    }
+  }
+
   /**
    * Expanded search when LLM needs more memories
    * Relaxes constraints and follows weaker connections
@@ -355,7 +415,7 @@ export class HybridSearch {
     // Get the original retrieval log
     const log = this.db.getRetrievalLog(recallId);
     if (!log) {
-      return { results: [], recall_id: recallId, connected_memories: [] };
+      return { results: [], digests: [], recall_id: recallId, connected_memories: [] };
     }
 
     // Search again with relaxed parameters