@betterdb/memory 0.1.2 → 0.4.0

package/src/mcp/server.ts

@@ -2,43 +2,72 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
 import { getValkeyClient } from "../client/valkey.js";
+import { getPluginMemoryStore } from "../client/memory-store.js";
 import { createModelClient } from "../client/model.js";
-import { formatForInjection } from "../memory/retrieval.js";
-import { getCwdProject } from "../memory/capture.js";
+import { formatSearchResult } from "../memory/retrieval.js";
+import { escalatingRecall } from "../memory/recall.js";
+import { getCwdProject, getGitBranch } from "../memory/capture.js";
+import { isConfigured } from "../config.js";
 import type { EpisodicMemory, KnowledgeEntry } from "../memory/schema.js";
 
+const SETUP_MESSAGE =
+  "BetterDB Memory is not configured yet. Run /betterdb-memory:setup to connect to Valkey and create the search index.";
+
 const server = new McpServer({
   name: "betterdb-memory",
-  version: "0.1.0",
+  version: "0.4.0",
 });
 
 // --- Tool: search_context ---
 
 server.tool(
   "search_context",
-  "Search your past Claude Code sessions for relevant context, decisions, or patterns",
+  "Search your past Claude Code sessions for relevant context, decisions, or patterns. " +
+    "Escalates automatically (project → wider → cross-project) and gates by relevance, " +
+    "so a miss means nothing relevant is stored — never fabricate to fill a miss.",
   {
     query: z.string().describe("The search query"),
-    top_k: z.number().int().min(1).max(20).optional().describe("Max results (default: 5)"),
+    top_k: z.number().int().min(1).max(20).optional().describe("Max results shown (default: 5)"),
+    scope: z
+      .enum(["project", "all"])
+      .optional()
+      .describe(
+        "Search scope. 'project' (default) stays in the current project; " +
+          "'all' also searches across every project — use when a project-scoped search found nothing.",
+      ),
+    tags: z
+      .array(z.enum(["decision", "pattern", "problem", "open-thread"]))
+      .optional()
+      .describe(
+        "Filter to memories of these content types — e.g. ['decision'] to " +
+          "recall only decisions, ['open-thread'] for unresolved items.",
+      ),
   },
-  async ({ query, top_k }) => {
-    const valkeyClient = await getValkeyClient();
+  async ({ query, top_k, scope, tags }) => {
+    if (!isConfigured()) {
+      return { content: [{ type: "text" as const, text: SETUP_MESSAGE }] };
+    }
+
     const modelClient = await createModelClient();
+    const store = await getPluginMemoryStore((t) => modelClient.embed(t));
 
-    const embedding = await modelClient.embed(query);
     const project = getCwdProject();
+    const branch = getGitBranch();
     const k = top_k ?? 5;
-
-    const memories = await valkeyClient.searchMemories(embedding, project, k);
-    const formatted = formatForInjection(memories);
+    // Default (project) scope stays in-project so a miss can *offer* to widen
+    // to all projects — the two-step consent flow. An explicit scope="all"
+    // requests the cross-project rung; escalatingRecall still gates it on
+    // BETTERDB_ALLOW_CROSS_PROJECT and flags the miss honestly if it's off.
+    const result = await escalatingRecall(store, query, {
+      project,
+      ...(branch !== "unknown" ? { branch } : {}),
+      ...(tags !== undefined ? { tags } : {}),
+      crossProjectRequested: scope === "all",
+    });
+    const formatted = formatSearchResult(query, result, k);
 
     return {
-      content: [
-        {
-          type: "text" as const,
-          text: formatted || "No matching memories found.",
-        },
-      ],
+      content: [{ type: "text" as const, text: formatted }],
     };
   },
 );
@@ -56,25 +85,17 @@ server.tool(
     project: z.string().optional().describe("Project name (auto-detected if omitted)"),
   },
   async ({ content, category, project: projectInput }) => {
+    if (!isConfigured()) {
+      return { content: [{ type: "text" as const, text: SETUP_MESSAGE }] };
+    }
+
     const valkeyClient = await getValkeyClient();
     const modelClient = await createModelClient();
+    const store = await getPluginMemoryStore((t) => modelClient.embed(t));
     const project = projectInput ?? getCwdProject();
 
-    // Store as KnowledgeEntry
-    const entry: KnowledgeEntry = {
-      entryId: crypto.randomUUID(),
-      project,
-      topic: category,
-      fact: content,
-      confidence: 0.9,
-      sourceMemoryIds: [],
-      lastUpdated: new Date().toISOString(),
-      accessCount: 0,
-    };
-    await valkeyClient.storeKnowledge(entry);
-
-    // Also store as EpisodicMemory for vector searchability
-    const embedding = await modelClient.embed(content);
+    // Store as EpisodicMemory for vector searchability. MemoryStore mints the
+    // id, so capture it for the knowledge link and the user-facing response.
     const memory: EpisodicMemory = {
       memoryId: crypto.randomUUID(),
       project,
@@ -92,13 +113,26 @@ server.tool(
       accessCount: 0,
       lastAccessed: new Date().toISOString(),
     };
-    await valkeyClient.storeMemory(memory, embedding);
+    const memoryId = await store.storeMemory(memory);
+
+    // Store as KnowledgeEntry, linked to the episodic memory just written.
+    const entry: KnowledgeEntry = {
+      entryId: crypto.randomUUID(),
+      project,
+      topic: category,
+      fact: content,
+      confidence: 0.9,
+      sourceMemoryIds: [memoryId],
+      lastUpdated: new Date().toISOString(),
+      accessCount: 0,
+    };
+    await valkeyClient.storeKnowledge(entry);
 
     return {
       content: [
         {
           type: "text" as const,
-          text: `Stored ${category}: "${content}" (memory: ${memory.memoryId})`,
+          text: `Stored ${category}: "${content}" (memory: ${memoryId})`,
         },
       ],
     };
@@ -114,15 +148,17 @@ server.tool(
     project: z.string().optional().describe("Project name (auto-detected if omitted)"),
   },
   async ({ project: projectInput }) => {
-    const valkeyClient = await getValkeyClient();
+    if (!isConfigured()) {
+      return { content: [{ type: "text" as const, text: SETUP_MESSAGE }] };
+    }
+
+    const store = await getPluginMemoryStore();
     const project = projectInput ?? getCwdProject();
 
-    const memoryIds = await valkeyClient.listMemoryIds(project, 0.5);
+    const memories = await store.listMemories(project, 0.5);
     const threads = new Set<string>();
 
-    for (const id of memoryIds) {
-      const memory = await valkeyClient.getMemory(id);
-      if (!memory) continue;
+    for (const memory of memories) {
       for (const thread of memory.summary.openThreads) {
         threads.add(thread);
       }
@@ -154,10 +190,14 @@ server.tool(
     confirmed: z.boolean().optional().describe("Set to true to confirm deletion"),
   },
   async ({ memory_id, confirmed }) => {
-    const valkeyClient = await getValkeyClient();
+    if (!isConfigured()) {
+      return { content: [{ type: "text" as const, text: SETUP_MESSAGE }] };
+    }
+
+    const store = await getPluginMemoryStore();
 
     if (!confirmed) {
-      const memory = await valkeyClient.getMemory(memory_id);
+      const memory = await store.getMemory(memory_id);
       if (!memory) {
         return {
           content: [
@@ -182,7 +222,7 @@ server.tool(
       };
     }
 
-    await valkeyClient.deleteMemory(memory_id);
+    await store.deleteMemory(memory_id);
 
     return {
       content: [

package/src/memory/aging.ts

@@ -1,199 +1,86 @@
+import type { MemoryItem } from "@betterdb/agent-memory";
 import { config } from "../config.js";
 import type { ModelClient } from "../client/model.js";
 import type { ValkeyClient } from "../client/valkey.js";
-import { SessionSummarySchema, type EpisodicMemory } from "./schema.js";
-import { computeInitialImportance, SessionCapture } from "./capture.js";
-
-// --- Cosine Similarity ---
-
-export function cosineSimilarity(a: number[], b: number[]): number {
-  let dot = 0;
-  let magA = 0;
-  let magB = 0;
-  for (let i = 0; i < a.length; i++) {
-    const av = a[i] ?? 0;
-    const bv = b[i] ?? 0;
-    dot += av * bv;
-    magA += av * av;
-    magB += bv * bv;
-  }
-  const denom = Math.sqrt(magA) * Math.sqrt(magB);
-  return denom === 0 ? 0 : dot / denom;
-}
+import {
+  itemToEpisodic,
+  type PluginMemoryStore,
+} from "../client/memory-store.js";
+import type { EpisodicMemory } from "./schema.js";
+import { computeInitialImportance } from "./capture.js";
 
 // --- Aging Pipeline ---
+//
+// Recency decay and similarity clustering used to live here as bespoke code;
+// both are now provided by @betterdb/agent-memory's MemoryStore — composite
+// recall scoring handles recency at query time, and consolidate() merges a
+// scope's low-value memories into a single summary. What remains here is the
+// plugin-specific glue: ingest-queue processing, LLM-driven consolidation
+// summarization, and pattern distillation into KnowledgeEntries.
+//
+// Consolidation only runs when a project has at least this many low-importance
+// memories, so a lone low-value memory isn't pointlessly re-summarized (which
+// would discard its structured summary and reset its access stats).
+const CONSOLIDATE_MIN_CANDIDATES = 3;
 
 export class AgingPipeline {
   private valkeyClient: ValkeyClient;
+  private store: PluginMemoryStore;
   private modelClient: ModelClient;
 
-  constructor(valkeyClient: ValkeyClient, modelClient: ModelClient) {
+  constructor(
+    valkeyClient: ValkeyClient,
+    store: PluginMemoryStore,
+    modelClient: ModelClient,
+  ) {
     this.valkeyClient = valkeyClient;
+    this.store = store;
     this.modelClient = modelClient;
   }
 
-  // --- Decay ---
-
-  async runDecay(
-    project?: string,
-  ): Promise<{ processed: number; flagged: number }> {
-    const memoryIds = await this.valkeyClient.listMemoryIds(project);
-    let processed = 0;
-    let flagged = 0;
-
-    for (const id of memoryIds) {
-      const memory = await this.valkeyClient.getMemory(id);
-      if (!memory) continue;
-
-      const daysSince =
-        (Date.now() - new Date(memory.lastAccessed).getTime()) /
-        (1000 * 60 * 60 * 24);
-      const newScore =
-        memory.importanceScore *
-        Math.pow(config.memory.decayRate, daysSince);
+  // --- Consolidation ---
 
-      await this.valkeyClient.updateImportance(id, newScore);
-      processed++;
+  async runConsolidation(
+    project: string,
+  ): Promise<{ consolidated: number; created: number; deleted: number }> {
+    const threshold = config.memory.compressThreshold;
+    const candidates = (await this.store.listMemories(project)).filter(
+      (m) => m.importanceScore <= threshold,
+    );
 
-      if (newScore < config.memory.compressThreshold) {
-        await this.valkeyClient.pushCompressQueue(id);
-        flagged++;
-      }
+    if (candidates.length < CONSOLIDATE_MIN_CANDIDATES) {
+      return { consolidated: 0, created: 0, deleted: 0 };
     }
 
-    return { processed, flagged };
+    const result = await this.store.consolidate({
+      namespace: project,
+      maxImportance: threshold,
+      summaryImportance: threshold,
+      summarize: (items) => this.summarizeCluster(items),
+    });
+
+    return {
+      consolidated: result.consolidated,
+      created: result.created.length,
+      deleted: result.deleted,
+    };
   }
 
-  // --- Compression ---
-
-  async runCompression(): Promise<{ merged: number; deleted: number }> {
-    const ids = await this.valkeyClient.popCompressQueue(50);
-    if (ids.length === 0) return { merged: 0, deleted: 0 };
-
-    // Fetch memories with embeddings
-    const entries: Array<{
-      memory: EpisodicMemory;
-      embedding: number[];
-    }> = [];
-
-    for (const id of ids) {
-      const memory = await this.valkeyClient.getMemory(id);
-      const embedding = await this.valkeyClient.getMemoryEmbedding(id);
-      if (memory && embedding) {
-        entries.push({ memory, embedding });
-      }
-    }
-
-    // Group by project
-    const byProject = new Map<
-      string,
-      Array<{ memory: EpisodicMemory; embedding: number[] }>
-    >();
-    for (const entry of entries) {
-      const group = byProject.get(entry.memory.project) ?? [];
-      group.push(entry);
-      byProject.set(entry.memory.project, group);
-    }
-
-    let merged = 0;
-    let deleted = 0;
-
-    for (const [, group] of byProject) {
-      // Batch size guard: only process 100 lowest-importance per project
-      const sorted = group
-        .sort((a, b) => a.memory.importanceScore - b.memory.importanceScore)
-        .slice(0, 100);
-
-      if (sorted.length < group.length) {
-        console.error(
-          `[betterdb] Project group exceeds 100 memories, processing only lowest-importance 100. Additional runs needed.`,
+  private async summarizeCluster(items: MemoryItem[]): Promise<string> {
+    const transcript = items
+      .map((item) => {
+        const memory = itemToEpisodic(item);
+        if (!memory) return item.content;
+        return (
+          `Session: ${memory.summary.oneLineSummary}\n` +
+          `Decisions: ${memory.summary.decisions.join("; ")}\n` +
+          `Patterns: ${memory.summary.patterns.join("; ")}`
         );
-      }
-
-      // Find clusters of similar memories
-      const used = new Set<number>();
-      const clusters: Array<
-        Array<{ memory: EpisodicMemory; embedding: number[] }>
-      > = [];
-
-      for (let i = 0; i < sorted.length; i++) {
-        if (used.has(i)) continue;
-        const cluster = [sorted[i]!];
-        used.add(i);
-
-        for (let j = i + 1; j < sorted.length; j++) {
-          if (used.has(j)) continue;
-          // Check if similar to all cluster members
-          const similar = cluster.every(
-            (c) =>
-              cosineSimilarity(c.embedding, sorted[j]!.embedding) > 0.85,
-          );
-          if (similar) {
-            cluster.push(sorted[j]!);
-            used.add(j);
-          }
-        }
-
-        clusters.push(cluster);
-      }
-
-      // Process clusters
-      for (const cluster of clusters) {
-        if (cluster.length >= 3) {
-          // Merge cluster into a single memory
-          const combinedTranscript = cluster
-            .map(
-              (c) =>
-                `Session ${c.memory.memoryId}: ${c.memory.summary.oneLineSummary}\n` +
-                `Decisions: ${c.memory.summary.decisions.join("; ")}\n` +
-                `Patterns: ${c.memory.summary.patterns.join("; ")}`,
-            )
-            .join("\n\n");
-
-          const mergedSummary =
-            await this.modelClient.summarize(combinedTranscript);
-          const mergedEmbedding = await this.modelClient.embed(
-            mergedSummary.oneLineSummary,
-          );
-
-          const avgImportance =
-            cluster.reduce((sum, c) => sum + c.memory.importanceScore, 0) /
-            cluster.length;
-
-          const newMemory: EpisodicMemory = {
-            memoryId: crypto.randomUUID(),
-            project: cluster[0]!.memory.project,
-            branch: cluster[0]!.memory.branch,
-            timestamp: new Date().toISOString(),
-            summary: mergedSummary,
-            importanceScore: avgImportance,
-            accessCount: 0,
-            lastAccessed: new Date().toISOString(),
-          };
-
-          await this.valkeyClient.storeMemory(newMemory, mergedEmbedding);
-
-          // Delete originals
-          for (const c of cluster) {
-            await this.valkeyClient.deleteMemory(c.memory.memoryId);
-          }
-
-          merged += cluster.length;
-        } else if (cluster.length === 1) {
-          const m = cluster[0]!.memory;
-          const daysSince =
-            (Date.now() - new Date(m.lastAccessed).getTime()) /
-            (1000 * 60 * 60 * 24);
-
-          if (m.importanceScore < 0.05 && daysSince > 90) {
-            await this.valkeyClient.deleteMemory(m.memoryId);
-            deleted++;
-          }
-        }
-      }
-    }
+      })
+      .join("\n\n");
 
-    return { merged, deleted };
+    const summary = await this.modelClient.summarize(transcript);
+    return summary.oneLineSummary;
   }
 
   // --- Distillation ---
@@ -201,13 +88,7 @@ export class AgingPipeline {
   async runDistillation(
     project: string,
   ): Promise<{ distilled: number }> {
-    const memoryIds = await this.valkeyClient.listMemoryIds(project, 0.5);
-    const memories: EpisodicMemory[] = [];
-
-    for (const id of memoryIds) {
-      const memory = await this.valkeyClient.getMemory(id);
-      if (memory) memories.push(memory);
-    }
+    const memories = await this.store.listMemories(project, 0.5);
 
     if (memories.length < config.memory.distillMinSessions) {
       return { distilled: 0 };
@@ -259,9 +140,6 @@ export class AgingPipeline {
     for (const item of items) {
       try {
         const summary = await this.modelClient.summarize(item.transcript);
-        const embedding = await this.modelClient.embed(
-          summary.oneLineSummary,
-        );
         const importance = computeInitialImportance(summary);
 
         const meta = item.meta as Record<string, string>;
@@ -276,7 +154,7 @@ export class AgingPipeline {
           lastAccessed: new Date().toISOString(),
         };
 
-        await this.valkeyClient.storeMemory(memory, embedding);
+        await this.store.storeMemory(memory);
         processed++;
       } catch (err) {
         console.error("[betterdb] Failed to process queued transcript:", err);
@@ -300,22 +178,26 @@ export class AgingPipeline {
     const { processed: ingested } = await this.processIngestQueue();
     console.error(`[betterdb] Ingest queue: processed ${ingested} items`);
 
-    const { processed, flagged } = await this.runDecay(project);
-    console.error(
-      `[betterdb] Decay: processed ${processed}, flagged ${flagged} for compression`,
-    );
+    const projects = project
+      ? [project]
+      : await this.allProjects();
 
-    const { merged, deleted } = await this.runCompression();
-    console.error(
-      `[betterdb] Compression: merged ${merged}, deleted ${deleted}`,
-    );
+    for (const p of projects) {
+      const { consolidated, created, deleted } = await this.runConsolidation(p);
+      console.error(
+        `[betterdb] Consolidation (${p}): merged ${consolidated} into ${created}, deleted ${deleted}`,
+      );
 
-    if (project) {
-      const { distilled } = await this.runDistillation(project);
-      console.error(`[betterdb] Distillation: distilled ${distilled} entries`);
+      const { distilled } = await this.runDistillation(p);
+      console.error(`[betterdb] Distillation (${p}): distilled ${distilled} entries`);
     }
 
     await this.valkeyClient.setLastAgingRun(new Date());
     console.error("[betterdb] Aging pipeline complete.");
   }
+
+  private async allProjects(): Promise<string[]> {
+    const memories = await this.store.listMemories();
+    return [...new Set(memories.map((m) => m.project))];
+  }
 }