@agentmemory/agentmemory 0.7.0

package/benchmark/quality-eval.ts

@@ -0,0 +1,643 @@
+import { SearchIndex } from "../src/state/search-index.js";
+import { VectorIndex } from "../src/state/vector-index.js";
+import { HybridSearch } from "../src/state/hybrid-search.js";
+import { GraphRetrieval } from "../src/functions/graph-retrieval.js";
+import { extractEntitiesFromQuery } from "../src/functions/query-expansion.js";
+import type { CompressedObservation, GraphNode, GraphEdge, GraphEdgeType } from "../src/types.js";
+import { generateDataset, type LabeledQuery } from "./dataset.js";
+import { writeFileSync } from "node:fs";
+
+interface QualityMetrics {
+  query: string;
+  category: string;
+  recall_at_5: number;
+  recall_at_10: number;
+  recall_at_20: number;
+  precision_at_5: number;
+  precision_at_10: number;
+  ndcg_at_10: number;
+  mrr: number;
+  relevant_count: number;
+  retrieved_count: number;
+  latency_ms: number;
+}
+
+interface SystemMetrics {
+  system: string;
+  avg_recall_at_5: number;
+  avg_recall_at_10: number;
+  avg_recall_at_20: number;
+  avg_precision_at_5: number;
+  avg_precision_at_10: number;
+  avg_ndcg_at_10: number;
+  avg_mrr: number;
+  avg_latency_ms: number;
+  total_tokens_per_query: number;
+  per_query: QualityMetrics[];
+}
+
+function dcg(relevances: boolean[], k: number): number {
+  let sum = 0;
+  for (let i = 0; i < Math.min(k, relevances.length); i++) {
+    sum += (relevances[i] ? 1 : 0) / Math.log2(i + 2);
+  }
+  return sum;
+}
+
+function ndcg(retrieved: string[], relevant: Set<string>, k: number): number {
+  const actualRelevances = retrieved.slice(0, k).map(id => relevant.has(id));
+  const idealRelevances = Array.from({ length: Math.min(k, relevant.size) }, () => true);
+  const idealDCG = dcg(idealRelevances, k);
+  if (idealDCG === 0) return 0;
+  return dcg(actualRelevances, k) / idealDCG;
+}
+
+function recall(retrieved: string[], relevant: Set<string>, k: number): number {
+  if (relevant.size === 0) return 1;
+  const topK = new Set(retrieved.slice(0, k));
+  let hits = 0;
+  for (const id of relevant) {
+    if (topK.has(id)) hits++;
+  }
+  return hits / relevant.size;
+}
+
+function precision(retrieved: string[], relevant: Set<string>, k: number): number {
+  const topK = retrieved.slice(0, k);
+  if (topK.length === 0) return 0;
+  let hits = 0;
+  for (const id of topK) {
+    if (relevant.has(id)) hits++;
+  }
+  return hits / topK.length;
+}
+
+function mrr(retrieved: string[], relevant: Set<string>): number {
+  for (let i = 0; i < retrieved.length; i++) {
+    if (relevant.has(retrieved[i])) return 1 / (i + 1);
+  }
+  return 0;
+}
+
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+function mockKV() {
+  const store = new Map<string, Map<string, unknown>>();
+  return {
+    get: async <T>(scope: string, key: string): Promise<T | null> => {
+      return (store.get(scope)?.get(key) as T) ?? null;
+    },
+    set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+      if (!store.has(scope)) store.set(scope, new Map());
+      store.get(scope)!.set(key, data);
+      return data;
+    },
+    delete: async (scope: string, key: string): Promise<void> => {
+      store.get(scope)?.delete(key);
+    },
+    list: async <T>(scope: string): Promise<T[]> => {
+      const entries = store.get(scope);
+      return entries ? (Array.from(entries.values()) as T[]) : [];
+    },
+  };
+}
+
+function deterministicEmbedding(text: string, dims = 384): Float32Array {
+  const arr = new Float32Array(dims);
+  const words = text.toLowerCase().split(/\W+/).filter(w => w.length > 2);
+  for (const word of words) {
+    for (let i = 0; i < word.length; i++) {
+      const idx = (word.charCodeAt(i) * 31 + i * 17) % dims;
+      arr[idx] += 1;
+      const idx2 = (word.charCodeAt(i) * 37 + i * 13 + word.length * 7) % dims;
+      arr[idx2] += 0.5;
+    }
+  }
+  const norm = Math.sqrt(arr.reduce((s, v) => s + v * v, 0));
+  if (norm > 0) for (let i = 0; i < dims; i++) arr[i] /= norm;
+  return arr;
+}
+
+async function evalBm25Only(
+  observations: CompressedObservation[],
+  queries: LabeledQuery[],
+): Promise<SystemMetrics> {
+  const index = new SearchIndex();
+  for (const obs of observations) index.add(obs);
+
+  const perQuery: QualityMetrics[] = [];
+
+  for (const q of queries) {
+    const relevant = new Set(q.relevantObsIds);
+    const start = performance.now();
+    const results = index.search(q.query, 20);
+    const latency = performance.now() - start;
+
+    const retrieved = results.map(r => r.obsId);
+    perQuery.push({
+      query: q.query,
+      category: q.category,
+      recall_at_5: recall(retrieved, relevant, 5),
+      recall_at_10: recall(retrieved, relevant, 10),
+      recall_at_20: recall(retrieved, relevant, 20),
+      precision_at_5: precision(retrieved, relevant, 5),
+      precision_at_10: precision(retrieved, relevant, 10),
+      ndcg_at_10: ndcg(retrieved, relevant, 10),
+      mrr: mrr(retrieved, relevant),
+      relevant_count: relevant.size,
+      retrieved_count: results.length,
+      latency_ms: latency,
+    });
+  }
+
+  const avgTokens = perQuery.reduce((sum, q) => sum + q.retrieved_count, 0) / perQuery.length;
+  const avgObsTokens = observations.slice(0, 50).reduce((s, o) => s + estimateTokens(JSON.stringify(o)), 0) / 50;
+
+  return {
+    system: "BM25-only",
+    avg_recall_at_5: avg(perQuery.map(q => q.recall_at_5)),
+    avg_recall_at_10: avg(perQuery.map(q => q.recall_at_10)),
+    avg_recall_at_20: avg(perQuery.map(q => q.recall_at_20)),
+    avg_precision_at_5: avg(perQuery.map(q => q.precision_at_5)),
+    avg_precision_at_10: avg(perQuery.map(q => q.precision_at_10)),
+    avg_ndcg_at_10: avg(perQuery.map(q => q.ndcg_at_10)),
+    avg_mrr: avg(perQuery.map(q => q.mrr)),
+    avg_latency_ms: avg(perQuery.map(q => q.latency_ms)),
+    total_tokens_per_query: Math.round(avgObsTokens * avgTokens),
+    per_query: perQuery,
+  };
+}
+
+async function evalDualStream(
+  observations: CompressedObservation[],
+  queries: LabeledQuery[],
+): Promise<SystemMetrics> {
+  const kv = mockKV();
+  const bm25 = new SearchIndex();
+  const vector = new VectorIndex();
+  const dims = 384;
+
+  for (const obs of observations) {
+    bm25.add(obs);
+    const text = [obs.title, obs.narrative, ...obs.concepts, ...obs.facts].join(" ");
+    vector.add(obs.id, obs.sessionId, deterministicEmbedding(text, dims));
+    await kv.set(`mem:obs:${obs.sessionId}`, obs.id, obs);
+  }
+
+  const mockEmbed: any = {
+    name: "deterministic",
+    dimensions: dims,
+    embed: async (text: string) => deterministicEmbedding(text, dims),
+    embedBatch: async (texts: string[]) => texts.map(t => deterministicEmbedding(t, dims)),
+  };
+
+  const hybrid = new HybridSearch(bm25, vector, mockEmbed, kv as never, 0.4, 0.6, 0);
+  const perQuery: QualityMetrics[] = [];
+
+  for (const q of queries) {
+    const relevant = new Set(q.relevantObsIds);
+    const start = performance.now();
+    const results = await hybrid.search(q.query, 20);
+    const latency = performance.now() - start;
+
+    const retrieved = results.map(r => r.observation.id);
+    perQuery.push({
+      query: q.query,
+      category: q.category,
+      recall_at_5: recall(retrieved, relevant, 5),
+      recall_at_10: recall(retrieved, relevant, 10),
+      recall_at_20: recall(retrieved, relevant, 20),
+      precision_at_5: precision(retrieved, relevant, 5),
+      precision_at_10: precision(retrieved, relevant, 10),
+      ndcg_at_10: ndcg(retrieved, relevant, 10),
+      mrr: mrr(retrieved, relevant),
+      relevant_count: relevant.size,
+      retrieved_count: results.length,
+      latency_ms: latency,
+    });
+  }
+
+  const avgResultTokens = perQuery.reduce((sum, q) => {
+    return sum + q.retrieved_count;
+  }, 0) / perQuery.length;
+  const avgObsTokens2 = observations.slice(0, 50).reduce((s, o) => s + estimateTokens(JSON.stringify(o)), 0) / 50;
+
+  return {
+    system: "Dual-stream (BM25+Vector)",
+    avg_recall_at_5: avg(perQuery.map(q => q.recall_at_5)),
+    avg_recall_at_10: avg(perQuery.map(q => q.recall_at_10)),
+    avg_recall_at_20: avg(perQuery.map(q => q.recall_at_20)),
+    avg_precision_at_5: avg(perQuery.map(q => q.precision_at_5)),
+    avg_precision_at_10: avg(perQuery.map(q => q.precision_at_10)),
+    avg_ndcg_at_10: avg(perQuery.map(q => q.ndcg_at_10)),
+    avg_mrr: avg(perQuery.map(q => q.mrr)),
+    avg_latency_ms: avg(perQuery.map(q => q.latency_ms)),
+    total_tokens_per_query: Math.round(avgObsTokens2 * avgResultTokens),
+    per_query: perQuery,
+  };
+}
+
+async function evalTripleStream(
+  observations: CompressedObservation[],
+  queries: LabeledQuery[],
+): Promise<SystemMetrics> {
+  const kv = mockKV();
+  const bm25 = new SearchIndex();
+  const vector = new VectorIndex();
+  const dims = 384;
+
+  for (const obs of observations) {
+    bm25.add(obs);
+    const text = [obs.title, obs.narrative, ...obs.concepts, ...obs.facts].join(" ");
+    vector.add(obs.id, obs.sessionId, deterministicEmbedding(text, dims));
+    await kv.set(`mem:obs:${obs.sessionId}`, obs.id, obs);
+  }
+
+  const conceptToNodes = new Map<string, string>();
+  const nodeTypes: GraphNode["type"][] = ["concept", "library", "file", "pattern"];
+  const edgeTypes: GraphEdgeType[] = ["uses", "related_to", "depends_on", "modifies"];
+  const now = new Date().toISOString();
+  let nodeId = 0;
+
+  for (const obs of observations) {
+    for (const concept of obs.concepts) {
+      if (!conceptToNodes.has(concept)) {
+        const nid = `gn_${nodeId++}`;
+        conceptToNodes.set(concept, nid);
+        await kv.set("mem:graph:nodes", nid, {
+          id: nid,
+          type: nodeTypes[nodeId % nodeTypes.length],
+          name: concept,
+          properties: {},
+          sourceObservationIds: [],
+          createdAt: now,
+        } as GraphNode);
+      }
+      const nid = conceptToNodes.get(concept)!;
+      const existing = await kv.get<GraphNode>("mem:graph:nodes", nid);
+      if (existing && !existing.sourceObservationIds.includes(obs.id)) {
+        existing.sourceObservationIds.push(obs.id);
+        await kv.set("mem:graph:nodes", nid, existing);
+      }
+    }
+
+    const capped = obs.concepts.slice(0, 10);
+    for (let i = 0; i < capped.length; i++) {
+      for (let j = i + 1; j < capped.length; j++) {
+        const srcNid = conceptToNodes.get(capped[i])!;
+        const tgtNid = conceptToNodes.get(capped[j])!;
+        if (srcNid && tgtNid && srcNid !== tgtNid) {
+          const eid = `ge_${srcNid}_${tgtNid}`;
+          const existing = await kv.get<GraphEdge>("mem:graph:edges", eid);
+          const weight = existing ? Math.min(1.0, existing.weight + 0.1) : 0.5;
+          await kv.set("mem:graph:edges", eid, {
+            id: eid,
+            type: edgeTypes[(i + j) % edgeTypes.length],
+            sourceNodeId: srcNid,
+            targetNodeId: tgtNid,
+            weight,
+            sourceObservationIds: existing
+              ? [...new Set([...existing.sourceObservationIds, obs.id])]
+              : [obs.id],
+            createdAt: now,
+            tcommit: now,
+            version: 1,
+            isLatest: true,
+          } as GraphEdge);
+        }
+      }
+    }
+  }
+
+  const mockEmbed: any = {
+    name: "deterministic",
+    dimensions: dims,
+    embed: async (text: string) => deterministicEmbedding(text, dims),
+    embedBatch: async (texts: string[]) => texts.map(t => deterministicEmbedding(t, dims)),
+  };
+
+  const hybrid = new HybridSearch(bm25, vector, mockEmbed, kv as never, 0.4, 0.6, 0.3);
+  const perQuery: QualityMetrics[] = [];
+
+  for (const q of queries) {
+    const relevant = new Set(q.relevantObsIds);
+    const start = performance.now();
+    const results = await hybrid.search(q.query, 20);
+    const latency = performance.now() - start;
+
+    const retrieved = results.map(r => r.observation.id);
+    perQuery.push({
+      query: q.query,
+      category: q.category,
+      recall_at_5: recall(retrieved, relevant, 5),
+      recall_at_10: recall(retrieved, relevant, 10),
+      recall_at_20: recall(retrieved, relevant, 20),
+      precision_at_5: precision(retrieved, relevant, 5),
+      precision_at_10: precision(retrieved, relevant, 10),
+      ndcg_at_10: ndcg(retrieved, relevant, 10),
+      mrr: mrr(retrieved, relevant),
+      relevant_count: relevant.size,
+      retrieved_count: results.length,
+      latency_ms: latency,
+    });
+  }
+
+  const avgResultTokens3 = perQuery.reduce((sum, q) => {
+    return sum + q.retrieved_count;
+  }, 0) / perQuery.length;
+  const avgObsTokens3 = observations.slice(0, 50).reduce((s, o) => s + estimateTokens(JSON.stringify(o)), 0) / 50;
+
+  return {
+    system: "Triple-stream (BM25+Vector+Graph)",
+    avg_recall_at_5: avg(perQuery.map(q => q.recall_at_5)),
+    avg_recall_at_10: avg(perQuery.map(q => q.recall_at_10)),
+    avg_recall_at_20: avg(perQuery.map(q => q.recall_at_20)),
+    avg_precision_at_5: avg(perQuery.map(q => q.precision_at_5)),
+    avg_precision_at_10: avg(perQuery.map(q => q.precision_at_10)),
+    avg_ndcg_at_10: avg(perQuery.map(q => q.ndcg_at_10)),
+    avg_mrr: avg(perQuery.map(q => q.mrr)),
+    avg_latency_ms: avg(perQuery.map(q => q.latency_ms)),
+    total_tokens_per_query: Math.round(avgObsTokens3 * avgResultTokens3),
+    per_query: perQuery,
+  };
+}
+
+async function evalBuiltinMemory(
+  observations: CompressedObservation[],
+  queries: LabeledQuery[],
+): Promise<SystemMetrics> {
+  const allText = observations.map(o =>
+    `## ${o.title}\n${o.narrative}\nConcepts: ${o.concepts.join(", ")}\nFiles: ${o.files.join(", ")}`
+  ).join("\n\n");
+
+  const totalTokens = estimateTokens(allText);
+
+  const perQuery: QualityMetrics[] = [];
+
+  for (const q of queries) {
+    const relevant = new Set(q.relevantObsIds);
+    const start = performance.now();
+
+    const queryTerms = q.query.toLowerCase().split(/\W+/).filter(w => w.length > 2);
+    const scored: Array<{ id: string; score: number }> = [];
+
+    for (const obs of observations) {
+      const text = [obs.title, obs.narrative, ...obs.concepts, ...obs.facts].join(" ").toLowerCase();
+      let score = 0;
+      for (const term of queryTerms) {
+        if (text.includes(term)) score++;
+      }
+      if (score > 0) scored.push({ id: obs.id, score });
+    }
+
+    scored.sort((a, b) => b.score - a.score);
+    const latency = performance.now() - start;
+
+    const retrieved = scored.map(s => s.id).slice(0, 20);
+    perQuery.push({
+      query: q.query,
+      category: q.category,
+      recall_at_5: recall(retrieved, relevant, 5),
+      recall_at_10: recall(retrieved, relevant, 10),
+      recall_at_20: recall(retrieved, relevant, 20),
+      precision_at_5: precision(retrieved, relevant, 5),
+      precision_at_10: precision(retrieved, relevant, 10),
+      ndcg_at_10: ndcg(retrieved, relevant, 10),
+      mrr: mrr(retrieved, relevant),
+      relevant_count: relevant.size,
+      retrieved_count: Math.min(scored.length, 20),
+      latency_ms: latency,
+    });
+  }
+
+  return {
+    system: "Built-in (CLAUDE.md / grep)",
+    avg_recall_at_5: avg(perQuery.map(q => q.recall_at_5)),
+    avg_recall_at_10: avg(perQuery.map(q => q.recall_at_10)),
+    avg_recall_at_20: avg(perQuery.map(q => q.recall_at_20)),
+    avg_precision_at_5: avg(perQuery.map(q => q.precision_at_5)),
+    avg_precision_at_10: avg(perQuery.map(q => q.precision_at_10)),
+    avg_ndcg_at_10: avg(perQuery.map(q => q.ndcg_at_10)),
+    avg_mrr: avg(perQuery.map(q => q.mrr)),
+    avg_latency_ms: avg(perQuery.map(q => q.latency_ms)),
+    total_tokens_per_query: totalTokens,
+    per_query: perQuery,
+  };
+}
+
+async function evalBuiltinMemoryTruncated(
+  observations: CompressedObservation[],
+  queries: LabeledQuery[],
+): Promise<SystemMetrics> {
+  const MAX_LINES = 200;
+  const lines = observations.map(o =>
+    `- ${o.title}: ${o.narrative.slice(0, 80)}... [${o.concepts.slice(0, 3).join(", ")}]`
+  );
+  const truncated = lines.slice(0, MAX_LINES);
+  const truncatedIds = new Set(observations.slice(0, MAX_LINES).map(o => o.id));
+  const totalTokens = estimateTokens(truncated.join("\n"));
+
+  const perQuery: QualityMetrics[] = [];
+
+  for (const q of queries) {
+    const relevant = new Set(q.relevantObsIds);
+    const start = performance.now();
+
+    const queryTerms = q.query.toLowerCase().split(/\W+/).filter(w => w.length > 2);
+    const scored: Array<{ id: string; score: number }> = [];
+
+    for (let i = 0; i < Math.min(MAX_LINES, observations.length); i++) {
+      const obs = observations[i];
+      const line = truncated[i];
+      let score = 0;
+      for (const term of queryTerms) {
+        if (line.toLowerCase().includes(term)) score++;
+      }
+      if (score > 0) scored.push({ id: obs.id, score });
+    }
+
+    scored.sort((a, b) => b.score - a.score);
+    const latency = performance.now() - start;
+
+    const retrieved = scored.map(s => s.id).slice(0, 20);
+
+    const reachableRelevant = new Set(
+      [...relevant].filter(id => truncatedIds.has(id))
+    );
+
+    perQuery.push({
+      query: q.query,
+      category: q.category,
+      recall_at_5: recall(retrieved, relevant, 5),
+      recall_at_10: recall(retrieved, relevant, 10),
+      recall_at_20: recall(retrieved, relevant, 20),
+      precision_at_5: precision(retrieved, relevant, 5),
+      precision_at_10: precision(retrieved, relevant, 10),
+      ndcg_at_10: ndcg(retrieved, relevant, 10),
+      mrr: mrr(retrieved, relevant),
+      relevant_count: relevant.size,
+      retrieved_count: Math.min(scored.length, 20),
+      latency_ms: latency,
+    });
+  }
+
+  return {
+    system: "Built-in (200-line MEMORY.md)",
+    avg_recall_at_5: avg(perQuery.map(q => q.recall_at_5)),
+    avg_recall_at_10: avg(perQuery.map(q => q.recall_at_10)),
+    avg_recall_at_20: avg(perQuery.map(q => q.recall_at_20)),
+    avg_precision_at_5: avg(perQuery.map(q => q.precision_at_5)),
+    avg_precision_at_10: avg(perQuery.map(q => q.precision_at_10)),
+    avg_ndcg_at_10: avg(perQuery.map(q => q.ndcg_at_10)),
+    avg_mrr: avg(perQuery.map(q => q.mrr)),
+    avg_latency_ms: avg(perQuery.map(q => q.latency_ms)),
+    total_tokens_per_query: totalTokens,
+    per_query: perQuery,
+  };
+}
+
+function avg(nums: number[]): number {
+  return nums.length ? nums.reduce((a, b) => a + b, 0) / nums.length : 0;
+}
+
+function pct(n: number): string {
+  return (n * 100).toFixed(1) + "%";
+}
+
+function generateReport(systems: SystemMetrics[], obsCount: number, queryCount: number): string {
+  const lines: string[] = [];
+  const w = (s: string) => lines.push(s);
+
+  w("# agentmemory v0.6.0 — Search Quality Evaluation");
+  w("");
+  w(`**Date:** ${new Date().toISOString()}`);
+  w(`**Dataset:** ${obsCount} observations across 30 sessions (realistic coding project)`);
+  w(`**Queries:** ${queryCount} labeled queries with ground-truth relevance`);
+  w(`**Metric definitions:** Recall@K (fraction of relevant docs in top K), Precision@K (fraction of top K that are relevant), NDCG@10 (ranking quality), MRR (position of first relevant result)`);
+  w("");
+
+  w("## Head-to-Head Comparison");
+  w("");
+  w("| System | Recall@5 | Recall@10 | Precision@5 | NDCG@10 | MRR | Latency | Tokens/query |");
+  w("|--------|----------|-----------|-------------|---------|-----|---------|--------------|");
+  for (const s of systems) {
+    w(`| ${s.system} | ${pct(s.avg_recall_at_5)} | ${pct(s.avg_recall_at_10)} | ${pct(s.avg_precision_at_5)} | ${pct(s.avg_ndcg_at_10)} | ${pct(s.avg_mrr)} | ${s.avg_latency_ms.toFixed(2)}ms | ${s.total_tokens_per_query.toLocaleString()} |`);
+  }
+
+  w("");
+  w("## Why This Matters");
+  w("");
+
+  const builtin = systems.find(s => s.system.includes("CLAUDE.md / grep"));
+  const truncated = systems.find(s => s.system.includes("200-line"));
+  const triple = systems.find(s => s.system.includes("Triple"));
+  const bm25 = systems.find(s => s.system === "BM25-only");
+
+  if (builtin && triple) {
+    const recallLift = ((triple.avg_recall_at_10 - builtin.avg_recall_at_10) / Math.max(0.001, builtin.avg_recall_at_10) * 100);
+    const tokenSaving = ((1 - triple.total_tokens_per_query / builtin.total_tokens_per_query) * 100);
+    w(`**Recall improvement:** agentmemory triple-stream finds ${pct(triple.avg_recall_at_10)} of relevant memories at K=10 vs ${pct(builtin.avg_recall_at_10)} for keyword grep (${recallLift > 0 ? "+" : ""}${recallLift.toFixed(0)}%)`);
+    w(`**Token savings:** agentmemory returns only the top 10 results (${triple.total_tokens_per_query.toLocaleString()} tokens) vs loading everything into context (${builtin.total_tokens_per_query.toLocaleString()} tokens) — ${tokenSaving.toFixed(0)}% reduction`);
+  }
+
+  if (truncated && triple) {
+    w(`**200-line cap:** Claude Code's MEMORY.md is capped at 200 lines. With ${obsCount} observations, ${pct(truncated.avg_recall_at_10)} recall at K=10 — memories from later sessions are simply invisible.`);
+  }
+
+  w("");
+  w("## Per-Query Breakdown (Triple-Stream)");
+  w("");
+
+  if (triple) {
+    w("| Query | Category | Recall@10 | NDCG@10 | MRR | Relevant | Latency |");
+    w("|-------|----------|-----------|---------|-----|----------|---------|");
+    for (const q of triple.per_query) {
+      w(`| ${q.query.slice(0, 45)}${q.query.length > 45 ? "..." : ""} | ${q.category} | ${pct(q.recall_at_10)} | ${pct(q.ndcg_at_10)} | ${pct(q.mrr)} | ${q.relevant_count} | ${q.latency_ms.toFixed(1)}ms |`);
+    }
+  }
+
+  w("");
+  w("## By Query Category");
+  w("");
+
+  const categories = ["exact", "semantic", "cross-session", "entity"];
+  if (triple) {
+    w("| Category | Avg Recall@10 | Avg NDCG@10 | Avg MRR | Queries |");
+    w("|----------|---------------|-------------|---------|---------|");
+    for (const cat of categories) {
+      const qs = triple.per_query.filter(q => q.category === cat);
+      if (qs.length === 0) continue;
+      w(`| ${cat} | ${pct(avg(qs.map(q => q.recall_at_10)))} | ${pct(avg(qs.map(q => q.ndcg_at_10)))} | ${pct(avg(qs.map(q => q.mrr)))} | ${qs.length} |`);
+    }
+  }
+
+  w("");
+  w("## Context Window Analysis");
+  w("");
+  w("The fundamental problem with built-in agent memory:");
+  w("");
+  w("| Observations | MEMORY.md tokens | agentmemory tokens (top 10) | Savings | MEMORY.md reachable |");
+  w("|-------------|-----------------|---------------------------|---------|-------------------|");
+
+  for (const count of [240, 500, 1000, 5000]) {
+    const memTokens = Math.round(count * 50);
+    const amTokens = triple ? triple.total_tokens_per_query : 500;
+    const saving = ((1 - amTokens / memTokens) * 100);
+    const reachable = count <= 200 ? "100%" : `${((200 / count) * 100).toFixed(0)}%`;
+    w(`| ${count.toLocaleString()} | ${memTokens.toLocaleString()} | ${amTokens.toLocaleString()} | ${saving.toFixed(0)}% | ${reachable} |`);
+  }
+
+  w("");
+  w("At 240 observations (our dataset), MEMORY.md already hits its 200-line cap and loses access to the most recent 40 observations. At 1,000 observations, 80% of memories are invisible. agentmemory always searches the full corpus.");
+
+  w("");
+  w("---");
+  w("");
+  w(`*${systems.reduce((s, sys) => s + sys.per_query.length, 0)} evaluations across ${systems.length} systems. Ground-truth labels assigned by concept matching against observation metadata.*`);
+
+  return lines.join("\n");
+}
+
+async function main() {
+  console.log("Generating labeled dataset...");
+  const { observations, queries, sessions } = generateDataset();
+  console.log(`Dataset: ${observations.length} observations, ${sessions.size} sessions, ${queries.length} queries`);
+  console.log(`Avg relevant docs per query: ${(queries.reduce((s, q) => s + q.relevantObsIds.length, 0) / queries.length).toFixed(1)}`);
+  console.log("");
+
+  console.log("Evaluating: Built-in (CLAUDE.md / grep)...");
+  const builtinResults = await evalBuiltinMemory(observations, queries);
+  console.log(`  Recall@10: ${pct(builtinResults.avg_recall_at_10)}, NDCG@10: ${pct(builtinResults.avg_ndcg_at_10)}`);
+
+  console.log("Evaluating: Built-in (200-line MEMORY.md)...");
+  const truncatedResults = await evalBuiltinMemoryTruncated(observations, queries);
+  console.log(`  Recall@10: ${pct(truncatedResults.avg_recall_at_10)}, NDCG@10: ${pct(truncatedResults.avg_ndcg_at_10)}`);
+
+  console.log("Evaluating: BM25-only...");
+  const bm25Results = await evalBm25Only(observations, queries);
+  console.log(`  Recall@10: ${pct(bm25Results.avg_recall_at_10)}, NDCG@10: ${pct(bm25Results.avg_ndcg_at_10)}`);
+
+  console.log("Evaluating: Dual-stream (BM25+Vector)...");
+  const dualResults = await evalDualStream(observations, queries);
+  console.log(`  Recall@10: ${pct(dualResults.avg_recall_at_10)}, NDCG@10: ${pct(dualResults.avg_ndcg_at_10)}`);
+
+  console.log("Evaluating: Triple-stream (BM25+Vector+Graph)...");
+  const tripleResults = await evalTripleStream(observations, queries);
+  console.log(`  Recall@10: ${pct(tripleResults.avg_recall_at_10)}, NDCG@10: ${pct(tripleResults.avg_ndcg_at_10)}`);
+
+  console.log("");
+
+  const report = generateReport(
+    [builtinResults, truncatedResults, bm25Results, dualResults, tripleResults],
+    observations.length,
+    queries.length,
+  );
+
+  writeFileSync("benchmark/QUALITY.md", report);
+  console.log(report);
+  console.log(`\nReport written to benchmark/QUALITY.md`);
+}
+
+main().catch(console.error);