engram-sdk 0.4.2 → 0.4.4

package/rescore-codebase.ts

@@ -1,113 +1,184 @@
 #!/usr/bin/env npx tsx
 /**
- * Re-score existing codebase eval results using LLM-as-judge
+ * rescore-codebase.ts — Re-score existing codebase eval results using LLM judge
+ * Uses the saved answers + ground truth, just re-runs scoring
  */
 
 import { readFileSync, writeFileSync } from 'fs';
-import { join, dirname } from 'path';
-import { homedir } from 'os';
+import { resolve, dirname } from 'path';
+import { fileURLToPath } from 'url';
 
-const GEMINI_KEY = readFileSync(join(homedir(), '.config/engram/gemini-key'), 'utf8').trim();
-const RESULTS_PATH = join(homedir(), '.openclaw/workspace/engram/eval-scale-data/codebase-results-openclaw.json');
-const RATE_LIMIT_MS = 6000;
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const GEMINI_KEY = readFileSync(resolve(process.env.HOME!, '.config/engram/gemini-key'), 'utf8').trim();
+const RESULTS_PATH = resolve(__dirname, 'eval-scale-data/codebase-results-vscode.json');
+const REPORT_PATH = resolve(__dirname, 'eval-scale-data/codebase-report-vscode-v2.json');
 
-function sleep(ms: number) { return new Promise(r => setTimeout(r, ms)); }
-
-async function geminiCall(prompt: string, maxTokens = 200, retries = 3, jsonMode = false): Promise<string> {
-  for (let attempt = 0; attempt < retries; attempt++) {
-    const response = await fetch(
-      `https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-lite:generateContent?key=${GEMINI_KEY}`,
-      {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({
-          contents: [{ parts: [{ text: prompt }] }],
-          generationConfig: {
-            maxOutputTokens: maxTokens,
-            ...(jsonMode ? { responseMimeType: 'application/json' } : {}),
-          },
-        }),
-      },
-    );
-    if (response.status === 429) {
-      console.log(`    Rate limited, waiting ${(attempt + 1) * 10}s...`);
-      await sleep((attempt + 1) * 10000);
-      continue;
+async function geminiCall(prompt: string, maxTokens = 100): Promise<string> {
+  for (let attempt = 0; attempt < 3; attempt++) {
+    try {
+      const response = await fetch(
+        `https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key=${GEMINI_KEY}`,
+        {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({
+            contents: [{ parts: [{ text: prompt }] }],
+            generationConfig: { maxOutputTokens: maxTokens, temperature: 0 },
+          }),
+        }
+      );
+      if (response.status === 429) {
+        const retryAfter = parseInt(response.headers.get('retry-after') || '10');
+        console.log(`    Rate limited, waiting ${retryAfter}s...`);
+        await new Promise(r => setTimeout(r, retryAfter * 1000));
+        continue;
+      }
+      if (!response.ok) {
+        console.log(`    API error ${response.status}, retrying...`);
+        await new Promise(r => setTimeout(r, 2000));
+        continue;
+      }
+      const data = await response.json() as any;
+      return data.candidates?.[0]?.content?.parts?.[0]?.text ?? '';
+    } catch (e: any) {
+      console.log(`    Fetch error: ${e.message}, retrying...`);
+      await new Promise(r => setTimeout(r, 2000));
     }
-    if (!response.ok) throw new Error(`Gemini API error: ${response.status}`);
-    const data = await response.json() as any;
-    return data.candidates?.[0]?.content?.parts?.[0]?.text ?? '';
   }
-  throw new Error('Max retries exceeded');
+  return '';
+}
+
+async function scoreAnswer(question: string, truth: string, answer: string): Promise<number> {
+  const prompt = `You are evaluating an AI's answer about a codebase. Score it from 0.0 to 1.0.
+
+- 1.0 = Correct and complete
+- 0.7 = Mostly correct, minor gaps
+- 0.5 = Partially correct
+- 0.3 = Mentions something relevant but mostly wrong
+- 0.0 = Wrong or "I don't know"
+
+Question: ${question}
+Ground Truth: ${truth}
+AI's Answer: ${answer}
+
+Respond with ONLY a decimal number (e.g. 0.7). Nothing else.`;
+
+  const response = await geminiCall(prompt);
+  const cleaned = response.trim();
+  
+  // Try direct float parse first
+  const direct = parseFloat(cleaned);
+  if (!isNaN(direct) && direct >= 0 && direct <= 1) return direct;
+  
+  // Try regex
+  const match = cleaned.match(/(0\.\d+|1\.0|0|1)/);
+  if (match) return parseFloat(match[1]);
+  
+  console.log(`    Failed to parse score: "${cleaned}"`);
+  return -1; // Mark as failed, don't default to 0
 }
 
 async function main() {
   const results = JSON.parse(readFileSync(RESULTS_PATH, 'utf8'));
-  console.log(`Rescoring ${results.length} codebase questions...\n`);
-
-  let scored = 0;
-  for (const r of results) {
-    // Skip if already scored
-    if (r.engram.correct > 0 || r.fullContext.correct > 0 || r.grepSearch?.correct > 0) {
-      console.log(`  [${r.index}] Already scored, skipping`);
-      scored++;
-      continue;
-    }
-
-    try {
-      await sleep(RATE_LIMIT_MS);
-      const scorePrompt = `Score these three answers to a codebase question on a scale of 0.0 to 1.0.
-0.0 = completely wrong or irrelevant
-0.5 = partially correct, missing key details
-1.0 = fully correct and complete
-
-Question: ${r.question}
-Ground Truth: ${r.groundTruth}
-
-Answer A (Engram): ${r.engram.answer}
-Answer B (Full Context): ${r.fullContext.answer}
-Answer C (Grep Search): ${r.grepSearch.answer}
-
-Output ONLY a JSON object: {"a": <score>, "b": <score>, "c": <score>}`;
-
-      const scoreResponse = await geminiCall(scorePrompt, 200, 3, true);
-      // Aggressively extract JSON from any preamble/wrapper text
-      const firstBrace = scoreResponse.indexOf('{');
-      const lastBrace = scoreResponse.lastIndexOf('}');
-      const scores = (firstBrace >= 0 && lastBrace > firstBrace)
-        ? JSON.parse(scoreResponse.slice(firstBrace, lastBrace + 1))
-        : null;
-
-      if (!scores) {
-        console.log(`  [${r.index}] Failed to parse scores: ${scoreResponse.slice(0, 100)}`);
-        continue;
-      }
-
-      r.engram.correct = scores.a;
-      r.fullContext.correct = scores.b;
-      if (r.grepSearch) r.grepSearch.correct = scores.c;
-
-      scored++;
-      console.log(`  [${r.index}] E: ${scores.a.toFixed(1)} | F: ${scores.b.toFixed(1)} | G: ${scores.c.toFixed(1)} — ${r.question.slice(0, 60)}...`);
-
-      // Auto-save every 5
-      if (scored % 5 === 0) {
-        writeFileSync(RESULTS_PATH, JSON.stringify(results, null, 2));
+  console.log(`Rescoring ${results.length} results...\n`);
+  
+  const systems = ['engram', 'cappedContext', 'naiveRag', 'grepSearch'] as const;
+  let totalScored = 0;
+  let totalFailed = 0;
+  
+  for (let i = 0; i < results.length; i++) {
+    const r = results[i];
+    console.log(`[${i+1}/${results.length}] (${r.category}/${r.difficulty}) ${r.question.slice(0, 70)}...`);
+    
+    const scores: Record<string, number> = {};
+    for (const sys of systems) {
+      if (!r[sys]?.answer) { scores[sys] = 0; continue; }
+      const score = await scoreAnswer(r.question, r.groundTruth, r[sys].answer);
+      if (score === -1) {
+        totalFailed++;
+        scores[sys] = 0;
+      } else {
+        scores[sys] = score;
       }
-    } catch (err: any) {
-      console.error(`  [${r.index}] Error: ${err.message}`);
-      await sleep(15000);
+      r[sys].score = scores[sys];
+    }
+    
+    totalScored += systems.length;
+    const line = systems.map(s => `${s[0].toUpperCase()}:${scores[s].toFixed(2)}`).join(' ');
+    console.log(`  ${line}`);
+    
+    // Save progress every 5 questions
+    if ((i + 1) % 5 === 0 || i === results.length - 1) {
+      writeFileSync(RESULTS_PATH.replace('.json', '-rescored2.json'), JSON.stringify(results, null, 2));
     }
   }
-
-  writeFileSync(RESULTS_PATH, JSON.stringify(results, null, 2));
-  console.log(`\n✅ Rescored ${scored}/${results.length} questions`);
-
-  // Quick summary
-  const avgE = results.reduce((s: number, r: any) => s + r.engram.correct, 0) / results.length * 100;
-  const avgF = results.reduce((s: number, r: any) => s + r.fullContext.correct, 0) / results.length * 100;
-  const avgG = results.reduce((s: number, r: any) => s + (r.grepSearch?.correct || 0), 0) / results.length * 100;
-  console.log(`\nEngram: ${avgE.toFixed(1)}% | Full Context: ${avgF.toFixed(1)}% | Grep: ${avgG.toFixed(1)}%`);
+  
+  console.log(`\nScored: ${totalScored}, Failed parses: ${totalFailed}\n`);
+  
+  // Generate report
+  const avg = (sys: string) => {
+    const vals = results.map((r: any) => r[sys]?.score ?? 0);
+    return vals.reduce((a: number, b: number) => a + b, 0) / vals.length;
+  };
+  const avgTokens = (sys: string) => {
+    const vals = results.map((r: any) => r[sys]?.tokensUsed ?? 0);
+    return Math.round(vals.reduce((a: number, b: number) => a + b, 0) / vals.length);
+  };
+  
+  console.log('=== VS Code Codebase Evaluation Report ===\n');
+  console.log('OVERALL (50 questions)');
+  console.log(`${'System'.padEnd(20)} ${'Accuracy'.padEnd(12)} Avg Tokens`);
+  for (const sys of systems) {
+    const acc = (avg(sys) * 100).toFixed(1);
+    console.log(`${sys.padEnd(20)} ${(acc + '%').padEnd(12)} ${avgTokens(sys)}`);
+  }
+  
+  // Per category
+  const categories = [...new Set(results.map((r: any) => r.category))];
+  for (const cat of categories) {
+    const catResults = results.filter((r: any) => r.category === cat);
+    const catAvg = (sys: string) => {
+      const vals = catResults.map((r: any) => r[sys]?.score ?? 0);
+      return (vals.reduce((a: number, b: number) => a + b, 0) / vals.length * 100).toFixed(1);
+    };
+    console.log(`\n  ${cat.toUpperCase()} (n=${catResults.length}): ${systems.map(s => `${s[0].toUpperCase()}:${catAvg(s)}%`).join(' ')}`);
+  }
+  
+  // Per difficulty
+  const diffs = [...new Set(results.map((r: any) => r.difficulty))];
+  for (const diff of diffs) {
+    const diffResults = results.filter((r: any) => r.difficulty === diff);
+    const diffAvg = (sys: string) => {
+      const vals = diffResults.map((r: any) => r[sys]?.score ?? 0);
+      return (vals.reduce((a: number, b: number) => a + b, 0) / vals.length * 100).toFixed(1);
+    };
+    console.log(`\n  ${diff.toUpperCase()} (n=${diffResults.length}): ${systems.map(s => `${s[0].toUpperCase()}:${diffAvg(s)}%`).join(' ')}`);
+  }
+  
+  const tokenSavings = (1 - avgTokens('engram') / avgTokens('cappedContext')) * 100;
+  console.log(`\n  Token savings vs capped context: ${tokenSavings.toFixed(1)}%`);
+  
+  // Save report
+  const report = {
+    timestamp: new Date().toISOString(),
+    totalQuestions: results.length,
+    failedParses: totalFailed,
+    overall: Object.fromEntries(systems.map(s => [s, {
+      accuracy: (avg(s) * 100).toFixed(1),
+      avgTokens: avgTokens(s),
+    }])),
+    byCategory: Object.fromEntries(categories.map(c => {
+      const cr = results.filter((r: any) => r.category === c);
+      return [c, Object.fromEntries(systems.map(s => [s, (cr.reduce((a: number, r: any) => a + (r[s]?.score ?? 0), 0) / cr.length * 100).toFixed(1)]))];
+    })),
+    byDifficulty: Object.fromEntries(diffs.map(d => {
+      const dr = results.filter((r: any) => r.difficulty === d);
+      return [d, Object.fromEntries(systems.map(s => [s, (dr.reduce((a: number, r: any) => a + (r[s]?.score ?? 0), 0) / dr.length * 100).toFixed(1)]))];
+    })),
+    tokenSavingsVsCapped: tokenSavings.toFixed(1) + '%',
+  };
+  writeFileSync(REPORT_PATH, JSON.stringify(report, null, 2));
+  console.log(`\nReport saved: ${REPORT_PATH}`);
 }
 
 main().catch(console.error);

package/rescore-vscode.ts

@@ -0,0 +1,142 @@
+#!/usr/bin/env npx tsx
+/**
+ * rescore-vscode.ts -- Re-score the VS Code codebase eval results
+ * 
+ * The original eval generated good answers but the judge scoring returned
+ * unparseable responses (all 0s). This script re-runs ONLY the scoring step.
+ */
+
+import { readFileSync, writeFileSync } from 'fs';
+import { homedir } from 'os';
+import { join } from 'path';
+
+const GEMINI_KEY = readFileSync(join(homedir(), '.config/engram/gemini-key'), 'utf8').trim();
+const EVAL_DIR = join(homedir(), '.openclaw/workspace/engram/eval-scale-data');
+const RESULTS_PATH = join(EVAL_DIR, 'codebase-results-vscode.json');
+const RESCORED_PATH = join(EVAL_DIR, 'codebase-results-vscode-rescored.json');
+const RATE_LIMIT_MS = 1500;
+
+async function sleep(ms: number) { return new Promise(r => setTimeout(r, ms)); }
+
+async function withRetry<T>(fn: () => Promise<T>, retries = 5): Promise<T> {
+  for (let i = 0; i < retries; i++) {
+    try {
+      return await fn();
+    } catch (err: any) {
+      if (err.message?.includes('429') && i < retries - 1) {
+        const backoff = Math.min(1500 * Math.pow(2, i + 1), 60000);
+        console.log(`  [Retry ${i + 1}/${retries}] 429, waiting ${backoff}ms...`);
+        await sleep(backoff);
+      } else {
+        throw err;
+      }
+    }
+  }
+  throw new Error('Exhausted retries');
+}
+
+async function geminiCall(prompt: string, maxTokens = 200): Promise<string> {
+  const res = await fetch(
+    `https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key=${GEMINI_KEY}`,
+    {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        contents: [{ parts: [{ text: prompt }] }],
+        generationConfig: { maxOutputTokens: maxTokens, temperature: 0.0 },
+      }),
+    }
+  );
+  if (!res.ok) {
+    const body = await res.text();
+    throw new Error(`Gemini ${res.status}: ${body.slice(0, 200)}`);
+  }
+  const data = await res.json() as any;
+  return data.candidates?.[0]?.content?.parts?.[0]?.text?.trim() || '';
+}
+
+async function scoreAnswer(question: string, groundTruth: string, answer: string): Promise<number> {
+  const prompt = `You are a strict code knowledge evaluator. Score this answer about the VS Code codebase.
+
+Question: ${question}
+
+Correct Answer: ${groundTruth}
+
+Given Answer: ${answer}
+
+Score from 0.0 to 1.0:
+- 1.0 = Completely correct, mentions the right files/classes/functions
+- 0.7 = Mostly correct, minor details missing
+- 0.5 = Partially correct, gets the general area but misses specifics
+- 0.3 = Vaguely related but mostly wrong
+- 0.0 = Completely wrong or says "insufficient context"
+
+Respond with ONLY a single number between 0.0 and 1.0. Nothing else.`;
+
+  const response = await withRetry(() => geminiCall(prompt, 10));
+  const score = parseFloat(response);
+  if (isNaN(score) || score < 0 || score > 1) {
+    console.log(`  Warning: unparseable score "${response}", defaulting to 0`);
+    return 0;
+  }
+  return score;
+}
+
+async function main() {
+  console.log('=== Re-scoring VS Code Codebase Eval ===\n');
+
+  const results = JSON.parse(readFileSync(RESULTS_PATH, 'utf8'));
+  console.log(`Loaded ${results.length} results\n`);
+
+  for (let i = 0; i < results.length; i++) {
+    const r = results[i];
+    console.log(`[${i + 1}/${results.length}] (${r.category}/${r.difficulty}) ${r.question.slice(0, 65)}...`);
+
+    // Score each system individually (more reliable than 4-at-once)
+    await sleep(RATE_LIMIT_MS);
+    r.engram.score = await scoreAnswer(r.question, r.groundTruth, r.engram.answer);
+
+    await sleep(RATE_LIMIT_MS);
+    r.cappedContext.score = await scoreAnswer(r.question, r.groundTruth, r.cappedContext.answer);
+
+    await sleep(RATE_LIMIT_MS);
+    r.naiveRag.score = await scoreAnswer(r.question, r.groundTruth, r.naiveRag.answer);
+
+    await sleep(RATE_LIMIT_MS);
+    r.grepSearch.score = await scoreAnswer(r.question, r.groundTruth, r.grepSearch.answer);
+
+    console.log(`  E:${r.engram.score.toFixed(2)} C:${r.cappedContext.score.toFixed(2)} R:${r.naiveRag.score.toFixed(2)} G:${r.grepSearch.score.toFixed(2)}`);
+
+    // Save every 5
+    if ((i + 1) % 5 === 0) {
+      writeFileSync(RESCORED_PATH, JSON.stringify(results, null, 2));
+    }
+  }
+
+  writeFileSync(RESCORED_PATH, JSON.stringify(results, null, 2));
+  console.log(`\nSaved to ${RESCORED_PATH}`);
+
+  // Print summary
+  const n = results.length;
+  const avg = (key: string) => (results.reduce((s: number, r: any) => s + r[key].score, 0) / n * 100).toFixed(1);
+  const avgTok = (key: string) => Math.round(results.reduce((s: number, r: any) => s + r[key].tokensUsed, 0) / n);
+
+  console.log(`\n=== RESULTS (${n} questions) ===`);
+  console.log(`Engram:         ${avg('engram')}%  (${avgTok('engram')} tok/q)`);
+  console.log(`Capped Context: ${avg('cappedContext')}%  (${avgTok('cappedContext')} tok/q)`);
+  console.log(`Naive RAG:      ${avg('naiveRag')}%  (${avgTok('naiveRag')} tok/q)`);
+  console.log(`Grep Search:    ${avg('grepSearch')}%  (${avgTok('grepSearch')} tok/q)`);
+
+  // By category
+  const cats = [...new Set(results.map((r: any) => r.category))];
+  for (const cat of cats) {
+    const cr = results.filter((r: any) => r.category === cat);
+    const catAvg = (key: string) => (cr.reduce((s: number, r: any) => s + r[key].score, 0) / cr.length * 100).toFixed(1);
+    console.log(`\n${(cat as string).toUpperCase()} (n=${cr.length}): E:${catAvg('engram')}% C:${catAvg('cappedContext')}% R:${catAvg('naiveRag')}% G:${catAvg('grepSearch')}%`);
+  }
+}
+
+main().catch(err => {
+  console.error('Fatal:', err);
+  process.exit(1);
+});

package/server.json

@@ -0,0 +1,29 @@
+{
+  "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
+  "name": "io.github.tstockham96/engram",
+  "description": "Intelligent memory layer for AI agents. Automatic extraction, consolidation, bi-temporal recall, and proactive context surfacing. Scores 80% on LOCOMO benchmark at 93% fewer tokens than full context.",
+  "repository": {
+    "url": "https://github.com/tstockham96/engram",
+    "source": "github"
+  },
+  "version": "0.4.4",
+  "packages": [
+    {
+      "registryType": "npm",
+      "identifier": "engram-sdk",
+      "version": "0.4.4",
+      "transport": {
+        "type": "stdio"
+      },
+      "environmentVariables": [
+        {
+          "description": "Google Gemini API key for embeddings and LLM operations",
+          "isRequired": true,
+          "format": "string",
+          "isSecret": true,
+          "name": "GEMINI_API_KEY"
+        }
+      ]
+    }
+  ]
+}

package/EVAL-FRAMEWORK.md

@@ -1,70 +0,0 @@
-# Engram Eval Framework
-
-## The Question
-Does Engram make an AI agent meaningfully better at its job compared to flat-file memory (MEMORY.md + daily notes + vector search)?
-
-## What "Better" Means
-1. **Fewer repeated questions** — Agent doesn't ask things it should already know
-2. **Surprise recall** — Agent surfaces relevant context the human didn't prompt for
-3. **Contradiction prevention** — Agent catches itself before giving conflicting info
-4. **Reduced curation toil** — Human spends less time maintaining memory files
-5. **Token efficiency** — Same or better context quality with fewer tokens
-
-## Daily Automated Eval (runs via cron)
-
-### Recall Accuracy Test
-Run a fixed set of 20 questions with known answers. Score: % of correct top-3 results.
-Track over time as memories accumulate and consolidation runs.
-
-Questions should cover:
-- Factual (What is Thomas's job? → Senior PM at BambooHR)
-- Procedural (How do I deploy the site? → cd engram-site && npx vercel --prod)
-- Relational (Who are Engram's competitors? → Mem0, Zep, Letta, LangMem)
-- Temporal (What did we build yesterday? → depends on date)
-- Personal preference (How does Thomas like communication? → direct, no fluff)
-
-### Token Comparison
-- Measure: bytes of MEMORY.md + loaded daily files vs Engram briefing() output
-- Track both as they grow over time
-- Calculate: tokens saved per request if using Engram briefing instead of file dump
-
-### Consolidation Quality
-After each consolidation run, evaluate:
-- Are new semantic memories factually accurate?
-- Are they non-redundant with existing memories?
-- Do they capture something the raw episodes didn't explicitly state? (insight generation)
-
-### Memory Freshness
-- Count: memories with status "pending" that are actually fulfilled
-- Count: memories that contradict each other
-- Track whether lifecycle management is happening
-
-## Weekly Human Eval
-
-Thomas rates (1-5) on a weekly basis:
-1. Did the agent remember things it should have? 
-2. Did the agent forget things it shouldn't have?
-3. Did the agent surprise you with relevant context?
-4. Was there less repetitive Q&A this week vs last?
-
-## Success Criteria
-
-### Engram is WORKING when:
-- Recall accuracy consistently >80% on the fixed question set
-- Agent surfaces relevant context Thomas didn't ask for at least 2x/week
-- Agent catches a contradiction or stale commitment at least 1x/week
-- MEMORY.md stops growing because Engram captures what it used to
-
-### Engram is NOT WORKING when:
-- MEMORY.md remains the primary useful memory source
-- Recall accuracy stays flat or degrades as memories accumulate
-- Consolidation produces redundant summaries instead of insights
-- Token cost increases without quality improvement
-
-## Scale Milestones
-Track value at each milestone:
-- 220 memories (current) — baseline
-- 500 memories — first consolidation cycle quality check
-- 1,000 memories — graph density threshold
-- 5,000 memories — flat-file memory should be breaking down by here
-- 10,000+ memories — Engram must be clearly better or it's not working

package/EVAL.md

@@ -1,127 +0,0 @@
-# Engram Self-Eval: Agent Performance With vs Without Memory
-
-*Designed Feb 16, 2026. Running over 72 hours.*
-
-## Hypothesis
-
-An AI agent with Engram memory performs measurably better at tasks requiring context from prior conversations than the same agent without it.
-
-## Design
-
-**Subject:** Jarvis (Claude Opus via OpenClaw) — the same agent, tested in two conditions:
-
-1. **Baseline (no Engram):** Agent uses only OpenClaw's default memory system (MEMORY.md flat files + vector search over markdown)
-2. **Engram-enhanced:** Agent uses Engram vault (auto-ingest, semantic recall, consolidation, knowledge graph)
-
-**Interface:** REST API + auto-ingest pipeline — the same way any OpenClaw user would run it. No MCP needed. Just the Engram server running alongside the agent, auto-ingesting conversations, and serving recall queries via HTTP.
-
-**Method:** Over 72 hours of normal use with Thomas, I'll log specific moments where memory matters. At each moment, I'll:
-1. Query Engram's REST API for relevant context
-2. Note what my flat-file memory (MEMORY.md + vector search) would have returned
-3. Compare the quality of recall
-
-This isn't a synthetic benchmark — it's real-world usage on the same setup any OpenClaw user would have. More ecologically valid than a lab test.
-
-## Metrics
-
-### 1. Recall Accuracy (quantitative)
-At each memory-dependent moment, score:
-- **Hit**: Correct, relevant memory surfaced → 1 point
-- **Partial**: Related but incomplete or slightly wrong → 0.5 points
-- **Miss**: Relevant memory existed but wasn't recalled → 0 points
-- **Hallucination**: Confidently wrong memory → -1 point
-
-Score both systems independently. Compare hit rates.
-
-### 2. Context Richness (qualitative, 1-5 scale)
-When recalling context, rate the depth:
-- 1: No useful context
-- 2: Basic facts only ("Thomas works at BambooHR")
-- 3: Facts + preferences ("Thomas works at BambooHR, prefers direct communication")
-- 4: Facts + preferences + history ("...and he pivoted from Kin to Engram because thin-margin hosting has no moat")
-- 5: Full situational awareness including decisions, reasoning, and emotional context
-
-### 3. Consolidation Quality (qualitative)
-After each consolidation run:
-- How many genuinely useful semantic memories were created?
-- Were any contradictions correctly identified?
-- Were any stale memories correctly superseded?
-- Did the knowledge graph capture meaningful relationships?
-
-### 4. Time-to-Context (qualitative)
-How quickly does the agent reach useful context?
-- Engram: single API call, structured results
-- Flat files: grep through markdown, parse manually, hope the right section is there
-
-### 5. "Surprise" Moments
-Log any moment where Engram surfaced context the agent wouldn't have thought to look for. These are the moments that demonstrate proactive value.
-
-## Test Scenarios
-
-Run these naturally over the 72-hour period:
-
-### Day 1 (Feb 16-17): Foundation
-- [x] Seed vault from existing conversations
-- [x] Run auto-ingest on current session
-- [x] First consolidation run
-- [ ] Evening: Ask about decisions made in morning (test same-day recall)
-- [ ] Evening: Run consolidation, note what it produces
-
-### Day 2 (Feb 17-18): Cross-Session Recall
-- [ ] Morning: Can Engram recall yesterday's context without being told?
-- [ ] Ask about pricing decisions (tests semantic memory from consolidation)
-- [ ] Ask about Thomas's preferences (tests across multiple conversations)
-- [ ] Work on a task that references past decisions — compare with/without
-- [ ] Run consolidation, note cross-day patterns
-
-### Day 3 (Feb 18-19): Stress Tests
-- [ ] Ask a question that requires connecting 3+ separate memories
-- [ ] Ask about something mentioned once 3 days ago (tests decay/persistence)
-- [ ] Introduce contradictory information — does consolidation catch it?
-- [ ] Ask an ambiguous question — does the knowledge graph help disambiguate?
-- [ ] Final consolidation + full analysis
-
-## Logging Format
-
-Each test moment gets logged as:
-
-```markdown
-### Test #N: [Description]
-**Time:** YYYY-MM-DD HH:MM
-**Query:** "What I asked / needed to recall"
-**Engram result:** [what Engram returned]
-**Flat-file result:** [what MEMORY.md search would return]
-**Recall accuracy:** Hit / Partial / Miss / Hallucination (for each system)
-**Context richness:** X/5 (for each system)
-**Notes:** [observations]
-```
-
-## Success Criteria
-
-Engram is **better** if:
-- Recall accuracy is ≥20% higher than flat files
-- Context richness averages ≥1 point higher
-- At least 3 "surprise" moments where Engram surfaced unexpected relevant context
-- Consolidation produces ≥10 genuinely useful semantic memories over 72 hours
-- Zero hallucinated memories (confidence should prevent false recalls)
-
-Engram is **equivalent** if scores are within 10%. Even equivalence is a win if Engram requires less manual effort (no hand-editing MEMORY.md).
-
-Engram **fails** if:
-- Recall accuracy is lower than flat files
-- Consolidation produces mostly noise
-- The agent performs worse on tasks due to bad memory surfacing
-
-## Results
-
-*Will be filled in as the eval progresses.*
-
-### Summary (to be completed Feb 19)
-- Total test moments: 
-- Engram recall accuracy: 
-- Flat-file recall accuracy: 
-- Engram context richness avg: 
-- Flat-file context richness avg: 
-- Surprise moments: 
-- Semantic memories from consolidation: 
-- Verdict: