@betterdb/memory 0.2.0 → 0.4.0

package/src/mcp/server.ts

@@ -2,9 +2,11 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
 import { getValkeyClient } from "../client/valkey.js";
+import { getPluginMemoryStore } from "../client/memory-store.js";
 import { createModelClient } from "../client/model.js";
-import { formatForInjection } from "../memory/retrieval.js";
-import { getCwdProject } from "../memory/capture.js";
+import { formatSearchResult } from "../memory/retrieval.js";
+import { escalatingRecall } from "../memory/recall.js";
+import { getCwdProject, getGitBranch } from "../memory/capture.js";
 import { isConfigured } from "../config.js";
 import type { EpisodicMemory, KnowledgeEntry } from "../memory/schema.js";
 
@@ -13,40 +15,59 @@ const SETUP_MESSAGE =
 
 const server = new McpServer({
   name: "betterdb-memory",
-  version: "0.2.0",
+  version: "0.4.0",
 });
 
 // --- Tool: search_context ---
 
 server.tool(
   "search_context",
-  "Search your past Claude Code sessions for relevant context, decisions, or patterns",
+  "Search your past Claude Code sessions for relevant context, decisions, or patterns. " +
+    "Escalates automatically (project → wider → cross-project) and gates by relevance, " +
+    "so a miss means nothing relevant is stored — never fabricate to fill a miss.",
   {
     query: z.string().describe("The search query"),
-    top_k: z.number().int().min(1).max(20).optional().describe("Max results (default: 5)"),
+    top_k: z.number().int().min(1).max(20).optional().describe("Max results shown (default: 5)"),
+    scope: z
+      .enum(["project", "all"])
+      .optional()
+      .describe(
+        "Search scope. 'project' (default) stays in the current project; " +
+          "'all' also searches across every project — use when a project-scoped search found nothing.",
+      ),
+    tags: z
+      .array(z.enum(["decision", "pattern", "problem", "open-thread"]))
+      .optional()
+      .describe(
+        "Filter to memories of these content types — e.g. ['decision'] to " +
+          "recall only decisions, ['open-thread'] for unresolved items.",
+      ),
   },
-  async ({ query, top_k }) => {
+  async ({ query, top_k, scope, tags }) => {
     if (!isConfigured()) {
       return { content: [{ type: "text" as const, text: SETUP_MESSAGE }] };
     }
 
-    const valkeyClient = await getValkeyClient();
     const modelClient = await createModelClient();
+    const store = await getPluginMemoryStore((t) => modelClient.embed(t));
 
-    const embedding = await modelClient.embed(query);
     const project = getCwdProject();
+    const branch = getGitBranch();
     const k = top_k ?? 5;
-
-    const memories = await valkeyClient.searchMemories(embedding, project, k);
-    const formatted = formatForInjection(memories);
+    // Default (project) scope stays in-project so a miss can *offer* to widen
+    // to all projects — the two-step consent flow. An explicit scope="all"
+    // requests the cross-project rung; escalatingRecall still gates it on
+    // BETTERDB_ALLOW_CROSS_PROJECT and flags the miss honestly if it's off.
+    const result = await escalatingRecall(store, query, {
+      project,
+      ...(branch !== "unknown" ? { branch } : {}),
+      ...(tags !== undefined ? { tags } : {}),
+      crossProjectRequested: scope === "all",
+    });
+    const formatted = formatSearchResult(query, result, k);
 
     return {
-      content: [
-        {
-          type: "text" as const,
-          text: formatted || "No matching memories found.",
-        },
-      ],
+      content: [{ type: "text" as const, text: formatted }],
     };
   },
 );
@@ -70,23 +91,11 @@ server.tool(
 
     const valkeyClient = await getValkeyClient();
     const modelClient = await createModelClient();
+    const store = await getPluginMemoryStore((t) => modelClient.embed(t));
     const project = projectInput ?? getCwdProject();
 
-    // Store as KnowledgeEntry
-    const entry: KnowledgeEntry = {
-      entryId: crypto.randomUUID(),
-      project,
-      topic: category,
-      fact: content,
-      confidence: 0.9,
-      sourceMemoryIds: [],
-      lastUpdated: new Date().toISOString(),
-      accessCount: 0,
-    };
-    await valkeyClient.storeKnowledge(entry);
-
-    // Also store as EpisodicMemory for vector searchability
-    const embedding = await modelClient.embed(content);
+    // Store as EpisodicMemory for vector searchability. MemoryStore mints the
+    // id, so capture it for the knowledge link and the user-facing response.
     const memory: EpisodicMemory = {
       memoryId: crypto.randomUUID(),
       project,
@@ -104,13 +113,26 @@ server.tool(
       accessCount: 0,
       lastAccessed: new Date().toISOString(),
     };
-    await valkeyClient.storeMemory(memory, embedding);
+    const memoryId = await store.storeMemory(memory);
+
+    // Store as KnowledgeEntry, linked to the episodic memory just written.
+    const entry: KnowledgeEntry = {
+      entryId: crypto.randomUUID(),
+      project,
+      topic: category,
+      fact: content,
+      confidence: 0.9,
+      sourceMemoryIds: [memoryId],
+      lastUpdated: new Date().toISOString(),
+      accessCount: 0,
+    };
+    await valkeyClient.storeKnowledge(entry);
 
     return {
       content: [
         {
           type: "text" as const,
-          text: `Stored ${category}: "${content}" (memory: ${memory.memoryId})`,
+          text: `Stored ${category}: "${content}" (memory: ${memoryId})`,
         },
       ],
     };
@@ -130,15 +152,13 @@ server.tool(
       return { content: [{ type: "text" as const, text: SETUP_MESSAGE }] };
     }
 
-    const valkeyClient = await getValkeyClient();
+    const store = await getPluginMemoryStore();
     const project = projectInput ?? getCwdProject();
 
-    const memoryIds = await valkeyClient.listMemoryIds(project, 0.5);
+    const memories = await store.listMemories(project, 0.5);
     const threads = new Set<string>();
 
-    for (const id of memoryIds) {
-      const memory = await valkeyClient.getMemory(id);
-      if (!memory) continue;
+    for (const memory of memories) {
       for (const thread of memory.summary.openThreads) {
         threads.add(thread);
       }
@@ -174,10 +194,10 @@ server.tool(
       return { content: [{ type: "text" as const, text: SETUP_MESSAGE }] };
     }
 
-    const valkeyClient = await getValkeyClient();
+    const store = await getPluginMemoryStore();
 
     if (!confirmed) {
-      const memory = await valkeyClient.getMemory(memory_id);
+      const memory = await store.getMemory(memory_id);
       if (!memory) {
         return {
           content: [
@@ -202,7 +222,7 @@ server.tool(
       };
     }
 
-    await valkeyClient.deleteMemory(memory_id);
+    await store.deleteMemory(memory_id);
 
     return {
       content: [

package/src/memory/aging.ts

@@ -1,199 +1,86 @@
+import type { MemoryItem } from "@betterdb/agent-memory";
 import { config } from "../config.js";
 import type { ModelClient } from "../client/model.js";
 import type { ValkeyClient } from "../client/valkey.js";
-import { SessionSummarySchema, type EpisodicMemory } from "./schema.js";
-import { computeInitialImportance, SessionCapture } from "./capture.js";
-
-// --- Cosine Similarity ---
-
-export function cosineSimilarity(a: number[], b: number[]): number {
-  let dot = 0;
-  let magA = 0;
-  let magB = 0;
-  for (let i = 0; i < a.length; i++) {
-    const av = a[i] ?? 0;
-    const bv = b[i] ?? 0;
-    dot += av * bv;
-    magA += av * av;
-    magB += bv * bv;
-  }
-  const denom = Math.sqrt(magA) * Math.sqrt(magB);
-  return denom === 0 ? 0 : dot / denom;
-}
+import {
+  itemToEpisodic,
+  type PluginMemoryStore,
+} from "../client/memory-store.js";
+import type { EpisodicMemory } from "./schema.js";
+import { computeInitialImportance } from "./capture.js";
 
 // --- Aging Pipeline ---
+//
+// Recency decay and similarity clustering used to live here as bespoke code;
+// both are now provided by @betterdb/agent-memory's MemoryStore — composite
+// recall scoring handles recency at query time, and consolidate() merges a
+// scope's low-value memories into a single summary. What remains here is the
+// plugin-specific glue: ingest-queue processing, LLM-driven consolidation
+// summarization, and pattern distillation into KnowledgeEntries.
+//
+// Consolidation only runs when a project has at least this many low-importance
+// memories, so a lone low-value memory isn't pointlessly re-summarized (which
+// would discard its structured summary and reset its access stats).
+const CONSOLIDATE_MIN_CANDIDATES = 3;
 
 export class AgingPipeline {
   private valkeyClient: ValkeyClient;
+  private store: PluginMemoryStore;
   private modelClient: ModelClient;
 
-  constructor(valkeyClient: ValkeyClient, modelClient: ModelClient) {
+  constructor(
+    valkeyClient: ValkeyClient,
+    store: PluginMemoryStore,
+    modelClient: ModelClient,
+  ) {
     this.valkeyClient = valkeyClient;
+    this.store = store;
     this.modelClient = modelClient;
   }
 
-  // --- Decay ---
-
-  async runDecay(
-    project?: string,
-  ): Promise<{ processed: number; flagged: number }> {
-    const memoryIds = await this.valkeyClient.listMemoryIds(project);
-    let processed = 0;
-    let flagged = 0;
-
-    for (const id of memoryIds) {
-      const memory = await this.valkeyClient.getMemory(id);
-      if (!memory) continue;
-
-      const daysSince =
-        (Date.now() - new Date(memory.lastAccessed).getTime()) /
-        (1000 * 60 * 60 * 24);
-      const newScore =
-        memory.importanceScore *
-        Math.pow(config.memory.decayRate, daysSince);
+  // --- Consolidation ---
 
-      await this.valkeyClient.updateImportance(id, newScore);
-      processed++;
+  async runConsolidation(
+    project: string,
+  ): Promise<{ consolidated: number; created: number; deleted: number }> {
+    const threshold = config.memory.compressThreshold;
+    const candidates = (await this.store.listMemories(project)).filter(
+      (m) => m.importanceScore <= threshold,
+    );
 
-      if (newScore < config.memory.compressThreshold) {
-        await this.valkeyClient.pushCompressQueue(id);
-        flagged++;
-      }
+    if (candidates.length < CONSOLIDATE_MIN_CANDIDATES) {
+      return { consolidated: 0, created: 0, deleted: 0 };
     }
 
-    return { processed, flagged };
+    const result = await this.store.consolidate({
+      namespace: project,
+      maxImportance: threshold,
+      summaryImportance: threshold,
+      summarize: (items) => this.summarizeCluster(items),
+    });
+
+    return {
+      consolidated: result.consolidated,
+      created: result.created.length,
+      deleted: result.deleted,
+    };
   }
 
-  // --- Compression ---
-
-  async runCompression(): Promise<{ merged: number; deleted: number }> {
-    const ids = await this.valkeyClient.popCompressQueue(50);
-    if (ids.length === 0) return { merged: 0, deleted: 0 };
-
-    // Fetch memories with embeddings
-    const entries: Array<{
-      memory: EpisodicMemory;
-      embedding: number[];
-    }> = [];
-
-    for (const id of ids) {
-      const memory = await this.valkeyClient.getMemory(id);
-      const embedding = await this.valkeyClient.getMemoryEmbedding(id);
-      if (memory && embedding) {
-        entries.push({ memory, embedding });
-      }
-    }
-
-    // Group by project
-    const byProject = new Map<
-      string,
-      Array<{ memory: EpisodicMemory; embedding: number[] }>
-    >();
-    for (const entry of entries) {
-      const group = byProject.get(entry.memory.project) ?? [];
-      group.push(entry);
-      byProject.set(entry.memory.project, group);
-    }
-
-    let merged = 0;
-    let deleted = 0;
-
-    for (const [, group] of byProject) {
-      // Batch size guard: only process 100 lowest-importance per project
-      const sorted = group
-        .sort((a, b) => a.memory.importanceScore - b.memory.importanceScore)
-        .slice(0, 100);
-
-      if (sorted.length < group.length) {
-        console.error(
-          `[betterdb] Project group exceeds 100 memories, processing only lowest-importance 100. Additional runs needed.`,
+  private async summarizeCluster(items: MemoryItem[]): Promise<string> {
+    const transcript = items
+      .map((item) => {
+        const memory = itemToEpisodic(item);
+        if (!memory) return item.content;
+        return (
+          `Session: ${memory.summary.oneLineSummary}\n` +
+          `Decisions: ${memory.summary.decisions.join("; ")}\n` +
+          `Patterns: ${memory.summary.patterns.join("; ")}`
         );
-      }
-
-      // Find clusters of similar memories
-      const used = new Set<number>();
-      const clusters: Array<
-        Array<{ memory: EpisodicMemory; embedding: number[] }>
-      > = [];
-
-      for (let i = 0; i < sorted.length; i++) {
-        if (used.has(i)) continue;
-        const cluster = [sorted[i]!];
-        used.add(i);
-
-        for (let j = i + 1; j < sorted.length; j++) {
-          if (used.has(j)) continue;
-          // Check if similar to all cluster members
-          const similar = cluster.every(
-            (c) =>
-              cosineSimilarity(c.embedding, sorted[j]!.embedding) > 0.85,
-          );
-          if (similar) {
-            cluster.push(sorted[j]!);
-            used.add(j);
-          }
-        }
-
-        clusters.push(cluster);
-      }
-
-      // Process clusters
-      for (const cluster of clusters) {
-        if (cluster.length >= 3) {
-          // Merge cluster into a single memory
-          const combinedTranscript = cluster
-            .map(
-              (c) =>
-                `Session ${c.memory.memoryId}: ${c.memory.summary.oneLineSummary}\n` +
-                `Decisions: ${c.memory.summary.decisions.join("; ")}\n` +
-                `Patterns: ${c.memory.summary.patterns.join("; ")}`,
-            )
-            .join("\n\n");
-
-          const mergedSummary =
-            await this.modelClient.summarize(combinedTranscript);
-          const mergedEmbedding = await this.modelClient.embed(
-            mergedSummary.oneLineSummary,
-          );
-
-          const avgImportance =
-            cluster.reduce((sum, c) => sum + c.memory.importanceScore, 0) /
-            cluster.length;
-
-          const newMemory: EpisodicMemory = {
-            memoryId: crypto.randomUUID(),
-            project: cluster[0]!.memory.project,
-            branch: cluster[0]!.memory.branch,
-            timestamp: new Date().toISOString(),
-            summary: mergedSummary,
-            importanceScore: avgImportance,
-            accessCount: 0,
-            lastAccessed: new Date().toISOString(),
-          };
-
-          await this.valkeyClient.storeMemory(newMemory, mergedEmbedding);
-
-          // Delete originals
-          for (const c of cluster) {
-            await this.valkeyClient.deleteMemory(c.memory.memoryId);
-          }
-
-          merged += cluster.length;
-        } else if (cluster.length === 1) {
-          const m = cluster[0]!.memory;
-          const daysSince =
-            (Date.now() - new Date(m.lastAccessed).getTime()) /
-            (1000 * 60 * 60 * 24);
-
-          if (m.importanceScore < 0.05 && daysSince > 90) {
-            await this.valkeyClient.deleteMemory(m.memoryId);
-            deleted++;
-          }
-        }
-      }
-    }
+      })
+      .join("\n\n");
 
-    return { merged, deleted };
+    const summary = await this.modelClient.summarize(transcript);
+    return summary.oneLineSummary;
   }
 
   // --- Distillation ---
@@ -201,13 +88,7 @@ export class AgingPipeline {
   async runDistillation(
     project: string,
   ): Promise<{ distilled: number }> {
-    const memoryIds = await this.valkeyClient.listMemoryIds(project, 0.5);
-    const memories: EpisodicMemory[] = [];
-
-    for (const id of memoryIds) {
-      const memory = await this.valkeyClient.getMemory(id);
-      if (memory) memories.push(memory);
-    }
+    const memories = await this.store.listMemories(project, 0.5);
 
     if (memories.length < config.memory.distillMinSessions) {
       return { distilled: 0 };
@@ -259,9 +140,6 @@ export class AgingPipeline {
     for (const item of items) {
       try {
         const summary = await this.modelClient.summarize(item.transcript);
-        const embedding = await this.modelClient.embed(
-          summary.oneLineSummary,
-        );
         const importance = computeInitialImportance(summary);
 
         const meta = item.meta as Record<string, string>;
@@ -276,7 +154,7 @@ export class AgingPipeline {
           lastAccessed: new Date().toISOString(),
         };
 
-        await this.valkeyClient.storeMemory(memory, embedding);
+        await this.store.storeMemory(memory);
         processed++;
       } catch (err) {
         console.error("[betterdb] Failed to process queued transcript:", err);
@@ -300,22 +178,26 @@ export class AgingPipeline {
     const { processed: ingested } = await this.processIngestQueue();
     console.error(`[betterdb] Ingest queue: processed ${ingested} items`);
 
-    const { processed, flagged } = await this.runDecay(project);
-    console.error(
-      `[betterdb] Decay: processed ${processed}, flagged ${flagged} for compression`,
-    );
+    const projects = project
+      ? [project]
+      : await this.allProjects();
 
-    const { merged, deleted } = await this.runCompression();
-    console.error(
-      `[betterdb] Compression: merged ${merged}, deleted ${deleted}`,
-    );
+    for (const p of projects) {
+      const { consolidated, created, deleted } = await this.runConsolidation(p);
+      console.error(
+        `[betterdb] Consolidation (${p}): merged ${consolidated} into ${created}, deleted ${deleted}`,
+      );
 
-    if (project) {
-      const { distilled } = await this.runDistillation(project);
-      console.error(`[betterdb] Distillation: distilled ${distilled} entries`);
+      const { distilled } = await this.runDistillation(p);
+      console.error(`[betterdb] Distillation (${p}): distilled ${distilled} entries`);
     }
 
     await this.valkeyClient.setLastAgingRun(new Date());
     console.error("[betterdb] Aging pipeline complete.");
   }
+
+  private async allProjects(): Promise<string[]> {
+    const memories = await this.store.listMemories();
+    return [...new Set(memories.map((m) => m.project))];
+  }
 }

package/src/memory/recall.ts

@@ -0,0 +1,169 @@
+import { config } from "../config.js";
+import type { PluginMemoryStore, ScoredMemory } from "../client/memory-store.js";
+
+// Over-fetch → gate → narrow, mirroring the LongMemEval harness. Dense recall
+// is already ~95%; the gain is a real candidate set plus an honest gate, so
+// "found nothing" means "nothing cleared the gate" rather than an empty KNN.
+//
+// The gate is RELATIVE, not an absolute similarity threshold. Embed models
+// compress cosine similarity into different, narrow bands (mxbai-embed-large
+// packs everything into ~0.7–0.88; all-MiniLM differs), so a fixed tau doesn't
+// transfer across models. Instead: loosen the store's own distance gate to a
+// generous floor, drop genuine noise below that floor, then keep only the hits
+// within `margin` of the top match. Confidence comes from the top-vs-next gap,
+// which is scale-independent.
+
+export interface RecallResult {
+  hits: ScoredMemory[];
+  scope: "project" | "all";
+  /** 1: project+branch (or project) · 2: project · 3: cross-project · 0: nothing. */
+  rung: 0 | 1 | 2 | 3;
+  confidence: "high" | "low" | "none";
+  /**
+   * True on a miss when the caller asked to widen (`crossProjectRequested`) but
+   * `BETTERDB_ALLOW_CROSS_PROJECT` is off, so the cross-project rung never ran.
+   * Lets the formatter say "cross-project is disabled" instead of falsely
+   * offering a scope="all" retry the config would also refuse.
+   */
+  crossProjectBlocked: boolean;
+}
+
+/** Scoping for {@link escalatingRecall}. */
+export interface RecallQuery {
+  project: string;
+  /** Git branch (native thread scope). Rung 1 narrows to it when present. */
+  branch?: string;
+  /** Content-type filter (e.g. `["decision"]`) applied at every rung. */
+  tags?: string[];
+  /**
+   * Whether the caller wants to widen past the project (user consent / an
+   * explicit scope="all"). The cross-project rung *also* requires
+   * `BETTERDB_ALLOW_CROSS_PROJECT`; when requested but globally disabled the
+   * result is flagged {@link RecallResult.crossProjectBlocked}.
+   */
+  crossProjectRequested: boolean;
+}
+
+interface Gated {
+  hits: ScoredMemory[];
+  confidence: "high" | "low" | "none";
+}
+
+/** Keep hits within `margin` of the top match above `floor`; grade by gap. */
+function gate(pool: ScoredMemory[]): Gated {
+  const { floor, margin, separation } = config.recall;
+  const eligible = pool
+    .filter((h) => h.relevance >= floor)
+    .sort((a, b) => b.relevance - a.relevance);
+  if (eligible.length === 0) return { hits: [], confidence: "none" };
+
+  const top = eligible[0]!.relevance;
+  const hits = eligible.filter((h) => h.relevance >= top - margin);
+  const second = eligible[1]?.relevance ?? -Infinity;
+  // A clear peak above the rest → confident; a bunched cluster (e.g. many
+  // near-duplicate file-history entries) → low, honestly.
+  const confidence =
+    eligible.length === 1 || top - second >= separation ? "high" : "low";
+  return { hits, confidence };
+}
+
+/** Distance gate to hand the store so its strict default (0.25) doesn't
+ * pre-filter everything: similarity `floor` ↔ distance `2·(1 − floor)`. */
+function storeThreshold(): number {
+  return 2 * (1 - config.recall.floor);
+}
+
+/**
+ * Escalating recall, narrow → wide:
+ *   rung 1 — project + `branch` (when given), pool `poolK`. Same project and
+ *            branch is the most relevant scope; without a branch this is just
+ *            project scope.
+ *   rung 2 — project, any branch, wider pool `poolKWide`.
+ *   rung 3 — cross-project probe. Only when the caller requested widening AND
+ *            `BETTERDB_ALLOW_CROSS_PROJECT` is on, since another project's
+ *            memory is often noise or privacy-sensitive.
+ * Every rung is a speculative over-fetch, so all recalls set `reinforce:false`:
+ * the store reinforces its whole returned pool, but the gate then drops most of
+ * it, so reinforcing pre-gate would bump access counts on candidates the user
+ * never sees (and on entire pools of a miss), skewing composite ranking.
+ * A `tags` filter, when present, applies at every rung. Stops at the first rung
+ * that yields gated hits.
+ */
+export async function escalatingRecall(
+  store: PluginMemoryStore,
+  query: string,
+  q: RecallQuery,
+): Promise<RecallResult> {
+  const { poolK, poolKWide } = config.recall;
+  const threshold = storeThreshold();
+  const { project, branch, tags, crossProjectRequested } = q;
+  const crossProjectEnabled =
+    crossProjectRequested && config.recall.allowCrossProject;
+
+  // rung 1 — project + branch (most specific).
+  let pool = await store.recall(query, {
+    project,
+    ...(branch !== undefined ? { branch } : {}),
+    tags,
+    k: poolK,
+    threshold,
+    reinforce: false,
+  });
+  let g = gate(pool);
+  if (g.hits.length > 0) {
+    return {
+      hits: g.hits,
+      scope: "project",
+      rung: 1,
+      confidence: g.confidence,
+      crossProjectBlocked: false,
+    };
+  }
+
+  // rung 2 — project, any branch, wider pool.
+  pool = await store.recall(query, {
+    project,
+    tags,
+    k: poolKWide,
+    threshold,
+    reinforce: false,
+  });
+  g = gate(pool);
+  if (g.hits.length > 0) {
+    return {
+      hits: g.hits,
+      scope: "project",
+      rung: 2,
+      confidence: g.confidence,
+      crossProjectBlocked: false,
+    };
+  }
+
+  // rung 3 — cross-project probe.
+  if (crossProjectEnabled) {
+    pool = await store.recall(query, {
+      tags,
+      k: poolKWide,
+      threshold,
+      reinforce: false,
+    });
+    g = gate(pool);
+    if (g.hits.length > 0) {
+      return {
+        hits: g.hits,
+        scope: "all",
+        rung: 3,
+        confidence: g.confidence,
+        crossProjectBlocked: false,
+      };
+    }
+  }
+
+  return {
+    hits: [],
+    scope: crossProjectEnabled ? "all" : "project",
+    rung: 0,
+    confidence: "none",
+    crossProjectBlocked: crossProjectRequested && !config.recall.allowCrossProject,
+  };
+}