@betterdb/memory 0.1.0

package/src/memory/aging.ts

@@ -0,0 +1,321 @@
+import { config } from "../config.js";
+import type { ModelClient } from "../client/model.js";
+import type { ValkeyClient } from "../client/valkey.js";
+import { SessionSummarySchema, type EpisodicMemory } from "./schema.js";
+import { computeInitialImportance, SessionCapture } from "./capture.js";
+
+// --- Cosine Similarity ---
+
+export function cosineSimilarity(a: number[], b: number[]): number {
+  let dot = 0;
+  let magA = 0;
+  let magB = 0;
+  for (let i = 0; i < a.length; i++) {
+    const av = a[i] ?? 0;
+    const bv = b[i] ?? 0;
+    dot += av * bv;
+    magA += av * av;
+    magB += bv * bv;
+  }
+  const denom = Math.sqrt(magA) * Math.sqrt(magB);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+// --- Aging Pipeline ---
+
+export class AgingPipeline {
+  private valkeyClient: ValkeyClient;
+  private modelClient: ModelClient;
+
+  constructor(valkeyClient: ValkeyClient, modelClient: ModelClient) {
+    this.valkeyClient = valkeyClient;
+    this.modelClient = modelClient;
+  }
+
+  // --- Decay ---
+
+  async runDecay(
+    project?: string,
+  ): Promise<{ processed: number; flagged: number }> {
+    const memoryIds = await this.valkeyClient.listMemoryIds(project);
+    let processed = 0;
+    let flagged = 0;
+
+    for (const id of memoryIds) {
+      const memory = await this.valkeyClient.getMemory(id);
+      if (!memory) continue;
+
+      const daysSince =
+        (Date.now() - new Date(memory.lastAccessed).getTime()) /
+        (1000 * 60 * 60 * 24);
+      const newScore =
+        memory.importanceScore *
+        Math.pow(config.memory.decayRate, daysSince);
+
+      await this.valkeyClient.updateImportance(id, newScore);
+      processed++;
+
+      if (newScore < config.memory.compressThreshold) {
+        await this.valkeyClient.pushCompressQueue(id);
+        flagged++;
+      }
+    }
+
+    return { processed, flagged };
+  }
+
+  // --- Compression ---
+
+  async runCompression(): Promise<{ merged: number; deleted: number }> {
+    const ids = await this.valkeyClient.popCompressQueue(50);
+    if (ids.length === 0) return { merged: 0, deleted: 0 };
+
+    // Fetch memories with embeddings
+    const entries: Array<{
+      memory: EpisodicMemory;
+      embedding: number[];
+    }> = [];
+
+    for (const id of ids) {
+      const memory = await this.valkeyClient.getMemory(id);
+      const embedding = await this.valkeyClient.getMemoryEmbedding(id);
+      if (memory && embedding) {
+        entries.push({ memory, embedding });
+      }
+    }
+
+    // Group by project
+    const byProject = new Map<
+      string,
+      Array<{ memory: EpisodicMemory; embedding: number[] }>
+    >();
+    for (const entry of entries) {
+      const group = byProject.get(entry.memory.project) ?? [];
+      group.push(entry);
+      byProject.set(entry.memory.project, group);
+    }
+
+    let merged = 0;
+    let deleted = 0;
+
+    for (const [, group] of byProject) {
+      // Batch size guard: only process 100 lowest-importance per project
+      const sorted = group
+        .sort((a, b) => a.memory.importanceScore - b.memory.importanceScore)
+        .slice(0, 100);
+
+      if (sorted.length < group.length) {
+        console.error(
+          `[betterdb] Project group exceeds 100 memories, processing only lowest-importance 100. Additional runs needed.`,
+        );
+      }
+
+      // Find clusters of similar memories
+      const used = new Set<number>();
+      const clusters: Array<
+        Array<{ memory: EpisodicMemory; embedding: number[] }>
+      > = [];
+
+      for (let i = 0; i < sorted.length; i++) {
+        if (used.has(i)) continue;
+        const cluster = [sorted[i]!];
+        used.add(i);
+
+        for (let j = i + 1; j < sorted.length; j++) {
+          if (used.has(j)) continue;
+          // Check if similar to all cluster members
+          const similar = cluster.every(
+            (c) =>
+              cosineSimilarity(c.embedding, sorted[j]!.embedding) > 0.85,
+          );
+          if (similar) {
+            cluster.push(sorted[j]!);
+            used.add(j);
+          }
+        }
+
+        clusters.push(cluster);
+      }
+
+      // Process clusters
+      for (const cluster of clusters) {
+        if (cluster.length >= 3) {
+          // Merge cluster into a single memory
+          const combinedTranscript = cluster
+            .map(
+              (c) =>
+                `Session ${c.memory.memoryId}: ${c.memory.summary.oneLineSummary}\n` +
+                `Decisions: ${c.memory.summary.decisions.join("; ")}\n` +
+                `Patterns: ${c.memory.summary.patterns.join("; ")}`,
+            )
+            .join("\n\n");
+
+          const mergedSummary =
+            await this.modelClient.summarize(combinedTranscript);
+          const mergedEmbedding = await this.modelClient.embed(
+            mergedSummary.oneLineSummary,
+          );
+
+          const avgImportance =
+            cluster.reduce((sum, c) => sum + c.memory.importanceScore, 0) /
+            cluster.length;
+
+          const newMemory: EpisodicMemory = {
+            memoryId: crypto.randomUUID(),
+            project: cluster[0]!.memory.project,
+            branch: cluster[0]!.memory.branch,
+            timestamp: new Date().toISOString(),
+            summary: mergedSummary,
+            importanceScore: avgImportance,
+            accessCount: 0,
+            lastAccessed: new Date().toISOString(),
+          };
+
+          await this.valkeyClient.storeMemory(newMemory, mergedEmbedding);
+
+          // Delete originals
+          for (const c of cluster) {
+            await this.valkeyClient.deleteMemory(c.memory.memoryId);
+          }
+
+          merged += cluster.length;
+        } else if (cluster.length === 1) {
+          const m = cluster[0]!.memory;
+          const daysSince =
+            (Date.now() - new Date(m.lastAccessed).getTime()) /
+            (1000 * 60 * 60 * 24);
+
+          if (m.importanceScore < 0.05 && daysSince > 90) {
+            await this.valkeyClient.deleteMemory(m.memoryId);
+            deleted++;
+          }
+        }
+      }
+    }
+
+    return { merged, deleted };
+  }
+
+  // --- Distillation ---
+
+  async runDistillation(
+    project: string,
+  ): Promise<{ distilled: number }> {
+    const memoryIds = await this.valkeyClient.listMemoryIds(project, 0.5);
+    const memories: EpisodicMemory[] = [];
+
+    for (const id of memoryIds) {
+      const memory = await this.valkeyClient.getMemory(id);
+      if (memory) memories.push(memory);
+    }
+
+    if (memories.length < config.memory.distillMinSessions) {
+      return { distilled: 0 };
+    }
+
+    // Count pattern occurrences
+    const patternCounts = new Map<string, string[]>();
+    for (const m of memories) {
+      for (const pattern of m.summary.patterns) {
+        const normalized = pattern.toLowerCase().trim();
+        const sources = patternCounts.get(normalized) ?? [];
+        sources.push(m.memoryId);
+        patternCounts.set(normalized, sources);
+      }
+    }
+
+    let distilled = 0;
+
+    for (const [pattern, sourceIds] of patternCounts) {
+      if (sourceIds.length < config.memory.distillMinSessions) continue;
+
+      const distillPrompt = `Distill this recurring pattern into a single factual sentence:\nPattern: "${pattern}"\nAppeared in ${sourceIds.length} sessions.`;
+      const summary = await this.modelClient.summarize(distillPrompt);
+
+      const entry = {
+        entryId: crypto.randomUUID(),
+        project,
+        topic: pattern.slice(0, 100),
+        fact: summary.oneLineSummary,
+        confidence: Math.min(0.5 + sourceIds.length * 0.1, 1.0),
+        sourceMemoryIds: sourceIds.slice(0, 10),
+        lastUpdated: new Date().toISOString(),
+        accessCount: 0,
+      };
+
+      await this.valkeyClient.storeKnowledge(entry);
+      distilled++;
+    }
+
+    return { distilled };
+  }
+
+  // --- Ingest Queue Processing ---
+
+  async processIngestQueue(): Promise<{ processed: number }> {
+    const items = await this.valkeyClient.popIngestQueue(20);
+    let processed = 0;
+
+    for (const item of items) {
+      try {
+        const summary = await this.modelClient.summarize(item.transcript);
+        const embedding = await this.modelClient.embed(
+          summary.oneLineSummary,
+        );
+        const importance = computeInitialImportance(summary);
+
+        const meta = item.meta as Record<string, string>;
+        const memory: EpisodicMemory = {
+          memoryId: crypto.randomUUID(),
+          project: meta["project"] ?? "unknown",
+          branch: meta["branch"] ?? "unknown",
+          timestamp: meta["timestamp"] ?? new Date().toISOString(),
+          summary,
+          importanceScore: importance,
+          accessCount: 0,
+          lastAccessed: new Date().toISOString(),
+        };
+
+        await this.valkeyClient.storeMemory(memory, embedding);
+        processed++;
+      } catch (err) {
+        console.error("[betterdb] Failed to process queued transcript:", err);
+        // Re-queue on failure
+        await this.valkeyClient.pushIngestQueue(
+          item.transcript,
+          item.meta,
+        );
+        break;
+      }
+    }
+
+    return { processed };
+  }
+
+  // --- Full Pipeline ---
+
+  async runFullPipeline(project?: string): Promise<void> {
+    console.error("[betterdb] Starting aging pipeline...");
+
+    const { processed: ingested } = await this.processIngestQueue();
+    console.error(`[betterdb] Ingest queue: processed ${ingested} items`);
+
+    const { processed, flagged } = await this.runDecay(project);
+    console.error(
+      `[betterdb] Decay: processed ${processed}, flagged ${flagged} for compression`,
+    );
+
+    const { merged, deleted } = await this.runCompression();
+    console.error(
+      `[betterdb] Compression: merged ${merged}, deleted ${deleted}`,
+    );
+
+    if (project) {
+      const { distilled } = await this.runDistillation(project);
+      console.error(`[betterdb] Distillation: distilled ${distilled} entries`);
+    }
+
+    await this.valkeyClient.setLastAgingRun(new Date());
+    console.error("[betterdb] Aging pipeline complete.");
+  }
+}

package/src/memory/capture.ts

@@ -0,0 +1,122 @@
+import type { ModelClient } from "../client/model.js";
+import type { SessionEvent, SessionSummary } from "./schema.js";
+
+// --- Git Helpers ---
+
+export function getGitBranch(): string {
+  try {
+    const result = Bun.spawnSync(["git", "rev-parse", "--abbrev-ref", "HEAD"]);
+    const output = result.stdout.toString().trim();
+    return output || "unknown";
+  } catch {
+    return "unknown";
+  }
+}
+
+export function getGitLog(n = 5): string {
+  try {
+    const result = Bun.spawnSync([
+      "git",
+      "log",
+      `--oneline`,
+      `-n`,
+      String(n),
+      "--format=%s",
+    ]);
+    return result.stdout.toString().trim();
+  } catch {
+    return "";
+  }
+}
+
+export function getStagedFiles(): string[] {
+  try {
+    const result = Bun.spawnSync([
+      "git",
+      "diff",
+      "--cached",
+      "--name-only",
+    ]);
+    const output = result.stdout.toString().trim();
+    return output ? output.split("\n") : [];
+  } catch {
+    return [];
+  }
+}
+
+export function getCwdProject(): string {
+  try {
+    const parts = process.cwd().split("/");
+    return parts[parts.length - 1] ?? "unknown";
+  } catch {
+    return "unknown";
+  }
+}
+
+// --- Importance Scoring ---
+
+export function computeInitialImportance(summary: SessionSummary): number {
+  let score = 0.3;
+  score += Math.min(summary.problemsSolved.length * 0.15, 0.3);
+  score += Math.min(summary.decisions.length * 0.05, 0.15);
+  score += Math.min(summary.filesChanged.length * 0.02, 0.1);
+  score += summary.openThreads.length > 0 ? 0.1 : 0;
+  score += summary.patterns.length > 0 ? 0.05 : 0;
+  return Math.min(score, 1.0);
+}
+
+// --- Session Capture ---
+
+export class SessionCapture {
+  private events: SessionEvent[] = [];
+
+  addEvent(event: SessionEvent): void {
+    this.events.push(event);
+  }
+
+  buildTranscript(): string {
+    return this.events
+      .map((e) => {
+        const filePart = e.filePath ? ` [${e.filePath}]` : "";
+        return `[${e.timestamp}] ${e.eventType}${filePart}: ${e.content}`;
+      })
+      .join("\n");
+  }
+
+  async capture(client: ModelClient): Promise<SessionSummary> {
+    const transcript = this.buildTranscript();
+    if (!transcript.trim()) {
+      const { SessionSummarySchema } = await import("./schema.js");
+      return SessionSummarySchema.parse({});
+    }
+    return client.summarize(transcript);
+  }
+
+  async getQueryContext(): Promise<string> {
+    const parts: string[] = [];
+
+    const project = getCwdProject();
+    parts.push(`Project: ${project}`);
+
+    const branch = getGitBranch();
+    if (branch !== "unknown") {
+      parts.push(`Branch: ${branch}`);
+    }
+
+    const gitLog = getGitLog(5);
+    if (gitLog) {
+      parts.push(`Recent commits:\n${gitLog}`);
+    }
+
+    const staged = getStagedFiles();
+    if (staged.length > 0) {
+      parts.push(`Staged files: ${staged.join(", ")}`);
+    }
+
+    return parts.join("\n");
+  }
+
+  get eventCount(): number {
+    return this.events.length;
+  }
+}

package/src/memory/retrieval.ts

@@ -0,0 +1,114 @@
+import { config } from "../config.js";
+import type { ModelClient } from "../client/model.js";
+import type { ValkeyClient } from "../client/valkey.js";
+import type { EpisodicMemory } from "./schema.js";
+import { AgingPipeline } from "./aging.js";
+
+// --- Memory Retriever ---
+
+export class MemoryRetriever {
+  private valkeyClient: ValkeyClient;
+  private modelClient: ModelClient;
+  private agingPipeline: AgingPipeline;
+
+  constructor(valkeyClient: ValkeyClient, modelClient: ModelClient) {
+    this.valkeyClient = valkeyClient;
+    this.modelClient = modelClient;
+    this.agingPipeline = new AgingPipeline(valkeyClient, modelClient);
+  }
+
+  async retrieve(
+    queryContext: string,
+    project: string,
+  ): Promise<EpisodicMemory[]> {
+    await this.maybeRunAging(project);
+
+    const embedding = await this.modelClient.embed(queryContext);
+
+    const topK = config.memory.maxContextMemories * 2;
+    const candidates = await this.valkeyClient.searchMemories(
+      embedding,
+      project,
+      topK,
+    );
+
+    const now = Date.now();
+    const scored = candidates
+      .filter((m) => m.importanceScore >= 0.1)
+      .map((m) => {
+        const daysSince =
+          (now - new Date(m.lastAccessed).getTime()) / (1000 * 60 * 60 * 24);
+        const recencyFactor = Math.pow(
+          config.memory.decayRate,
+          Math.max(daysSince, 0),
+        );
+        return {
+          memory: m,
+          score: m.importanceScore * recencyFactor,
+        };
+      })
+      .sort((a, b) => b.score - a.score)
+      .slice(0, config.memory.maxContextMemories);
+
+    // Fire-and-forget access increments
+    for (const { memory } of scored) {
+      this.valkeyClient.incrementAccess(memory.memoryId).catch(() => {});
+    }
+
+    return scored.map((s) => s.memory);
+  }
+
+  async maybeRunAging(project: string): Promise<void> {
+    const lastRun = await this.valkeyClient.getLastAgingRun();
+    const hoursAgo = lastRun
+      ? (Date.now() - lastRun.getTime()) / (1000 * 60 * 60)
+      : Infinity;
+
+    if (hoursAgo >= config.memory.agingIntervalHours) {
+      await this.agingPipeline.runDecay(project);
+      await this.valkeyClient.setLastAgingRun(new Date());
+    }
+  }
+}
+
+// --- Format for Injection ---
+
+export function formatForInjection(memories: EpisodicMemory[]): string {
+  if (memories.length === 0) return "";
+
+  const sections: string[] = [
+    `# BetterDB Session Context`,
+    `_Retrieved ${memories.length} memories. Auto-generated — do not edit._`,
+  ];
+
+  // Per-memory summaries — this is the most important section
+  sections.push(`\n## Session Memories`);
+  for (const m of memories) {
+    const date = m.timestamp.split("T")[0];
+    sections.push(`- **[${date}]** ${m.summary.oneLineSummary}`);
+    for (const d of m.summary.decisions) {
+      sections.push(`  - Decision: ${d}`);
+    }
+    for (const p of m.summary.problemsSolved) {
+      sections.push(`  - Solved: ${p.problem} → ${p.resolution}`);
+    }
+    for (const t of m.summary.openThreads) {
+      sections.push(`  - Open: ${t}`);
+    }
+  }
+
+  // Aggregated files across all memories
+  const files = new Set<string>();
+  for (const m of memories) {
+    for (const f of m.summary.filesChanged) files.add(f);
+  }
+
+  if (files.size > 0) {
+    sections.push(
+      `\n## Files with History`,
+      ...[...files].slice(0, 10).map((f) => `- ${f}`),
+    );
+  }
+
+  return sections.join("\n");
+}

package/src/memory/schema.ts

@@ -0,0 +1,111 @@
+import { z } from "zod";
+
+// --- Session Events ---
+
+export const SessionEventSchema = z.object({
+  sessionId: z.string(),
+  timestamp: z.string().datetime(),
+  eventType: z.enum(["tool_call", "tool_result", "error", "file_change"]),
+  content: z.string(),
+  filePath: z.string().optional(),
+});
+
+export type SessionEvent = z.infer<typeof SessionEventSchema>;
+
+// --- Session Summary (output of summarizer) ---
+
+export const SessionSummarySchema = z.object({
+  decisions: z.array(z.string()).max(10).default([]),
+  patterns: z.array(z.string()).max(5).default([]),
+  problemsSolved: z
+    .array(
+      z.object({
+        problem: z.string(),
+        resolution: z.string(),
+      }),
+    )
+    .max(5)
+    .default([]),
+  openThreads: z.array(z.string()).max(5).default([]),
+  filesChanged: z.array(z.string()).default([]),
+  oneLineSummary: z
+    .string()
+    .default("Session recorded — summary unavailable"),
+});
+
+export type SessionSummary = z.infer<typeof SessionSummarySchema>;
+
+// --- Episodic Memory ---
+
+export const EpisodicMemorySchema = z.object({
+  memoryId: z.string().uuid(),
+  project: z.string(),
+  branch: z.string(),
+  timestamp: z.string().datetime(),
+  summary: SessionSummarySchema,
+  importanceScore: z.number().min(0).max(1),
+  accessCount: z.number().int().nonnegative(),
+  lastAccessed: z.string().datetime(),
+});
+
+export type EpisodicMemory = z.infer<typeof EpisodicMemorySchema>;
+
+// --- Knowledge Entry ---
+
+export const KnowledgeEntrySchema = z.object({
+  entryId: z.string().uuid(),
+  project: z.string(),
+  topic: z.string(),
+  fact: z.string(),
+  confidence: z.number().min(0).max(1),
+  sourceMemoryIds: z.array(z.string().uuid()),
+  lastUpdated: z.string().datetime(),
+  accessCount: z.number().int().nonnegative(),
+});
+
+export type KnowledgeEntry = z.infer<typeof KnowledgeEntrySchema>;
+
+// --- Hook Payloads (Claude Code Hooks API) ---
+// See: https://docs.anthropic.com/en/docs/claude-code/hooks
+
+const BaseHookPayload = z.object({
+  session_id: z.string(),
+  transcript_path: z.string().optional(),
+  cwd: z.string().optional(),
+  hook_event_name: z.string(),
+});
+
+export const SessionStartPayload = BaseHookPayload.extend({
+  hook_event_name: z.literal("SessionStart"),
+  source: z.enum(["startup", "resume", "clear", "compact"]).optional(),
+});
+
+export const StopPayload = BaseHookPayload.extend({
+  hook_event_name: z.literal("Stop"),
+});
+
+export const PreToolUsePayload = BaseHookPayload.extend({
+  hook_event_name: z.literal("PreToolUse"),
+  tool_name: z.string(),
+  tool_input: z.record(z.unknown()).optional(),
+});
+
+export const PostToolUsePayload = BaseHookPayload.extend({
+  hook_event_name: z.literal("PostToolUse"),
+  tool_name: z.string(),
+  tool_input: z.record(z.unknown()).optional(),
+  tool_result: z.unknown().optional(),
+});
+
+export const HookPayloadSchema = z.discriminatedUnion("hook_event_name", [
+  SessionStartPayload,
+  StopPayload,
+  PreToolUsePayload,
+  PostToolUsePayload,
+]);
+
+export type HookPayload = z.infer<typeof HookPayloadSchema>;
+export type SessionStartHookPayload = z.infer<typeof SessionStartPayload>;
+export type StopHookPayload = z.infer<typeof StopPayload>;
+export type PreToolUseHookPayload = z.infer<typeof PreToolUsePayload>;
+export type PostToolUseHookPayload = z.infer<typeof PostToolUsePayload>;

package/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "strict": true,
+    "noUncheckedIndexedAccess": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "outDir": "./dist",
+    "rootDir": ".",
+    "types": ["bun-types"]
+  },
+  "include": ["src/**/*.ts", "scripts/**/*.ts", "tests/**/*.ts"],
+  "exclude": ["node_modules", "dist"]
+}