engram-sdk 0.4.3 → 0.4.4

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "engram-sdk",
-  "version": "0.4.3",
+  "mcpName": "io.github.tstockham96/engram",
+  "version": "0.4.4",
   "description": "Universal memory layer for AI agents. Remember, recall, consolidate.",
   "type": "module",
   "main": "dist/index.js",

package/rescore-codebase.ts

@@ -1,113 +1,184 @@
 #!/usr/bin/env npx tsx
 /**
- * Re-score existing codebase eval results using LLM-as-judge
+ * rescore-codebase.ts — Re-score existing codebase eval results using LLM judge
+ * Uses the saved answers + ground truth, just re-runs scoring
  */
 
 import { readFileSync, writeFileSync } from 'fs';
-import { join, dirname } from 'path';
-import { homedir } from 'os';
+import { resolve, dirname } from 'path';
+import { fileURLToPath } from 'url';
 
-const GEMINI_KEY = readFileSync(join(homedir(), '.config/engram/gemini-key'), 'utf8').trim();
-const RESULTS_PATH = join(homedir(), '.openclaw/workspace/engram/eval-scale-data/codebase-results-openclaw.json');
-const RATE_LIMIT_MS = 6000;
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const GEMINI_KEY = readFileSync(resolve(process.env.HOME!, '.config/engram/gemini-key'), 'utf8').trim();
+const RESULTS_PATH = resolve(__dirname, 'eval-scale-data/codebase-results-vscode.json');
+const REPORT_PATH = resolve(__dirname, 'eval-scale-data/codebase-report-vscode-v2.json');
 
-function sleep(ms: number) { return new Promise(r => setTimeout(r, ms)); }
-
-async function geminiCall(prompt: string, maxTokens = 200, retries = 3, jsonMode = false): Promise<string> {
-  for (let attempt = 0; attempt < retries; attempt++) {
-    const response = await fetch(
-      `https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-lite:generateContent?key=${GEMINI_KEY}`,
-      {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({
-          contents: [{ parts: [{ text: prompt }] }],
-          generationConfig: {
-            maxOutputTokens: maxTokens,
-            ...(jsonMode ? { responseMimeType: 'application/json' } : {}),
-          },
-        }),
-      },
-    );
-    if (response.status === 429) {
-      console.log(`    Rate limited, waiting ${(attempt + 1) * 10}s...`);
-      await sleep((attempt + 1) * 10000);
-      continue;
+async function geminiCall(prompt: string, maxTokens = 100): Promise<string> {
+  for (let attempt = 0; attempt < 3; attempt++) {
+    try {
+      const response = await fetch(
+        `https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key=${GEMINI_KEY}`,
+        {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({
+            contents: [{ parts: [{ text: prompt }] }],
+            generationConfig: { maxOutputTokens: maxTokens, temperature: 0 },
+          }),
+        }
+      );
+      if (response.status === 429) {
+        const retryAfter = parseInt(response.headers.get('retry-after') || '10');
+        console.log(`    Rate limited, waiting ${retryAfter}s...`);
+        await new Promise(r => setTimeout(r, retryAfter * 1000));
+        continue;
+      }
+      if (!response.ok) {
+        console.log(`    API error ${response.status}, retrying...`);
+        await new Promise(r => setTimeout(r, 2000));
+        continue;
+      }
+      const data = await response.json() as any;
+      return data.candidates?.[0]?.content?.parts?.[0]?.text ?? '';
+    } catch (e: any) {
+      console.log(`    Fetch error: ${e.message}, retrying...`);
+      await new Promise(r => setTimeout(r, 2000));
     }
-    if (!response.ok) throw new Error(`Gemini API error: ${response.status}`);
-    const data = await response.json() as any;
-    return data.candidates?.[0]?.content?.parts?.[0]?.text ?? '';
   }
-  throw new Error('Max retries exceeded');
+  return '';
+}
+
+async function scoreAnswer(question: string, truth: string, answer: string): Promise<number> {
+  const prompt = `You are evaluating an AI's answer about a codebase. Score it from 0.0 to 1.0.
+
+- 1.0 = Correct and complete
+- 0.7 = Mostly correct, minor gaps
+- 0.5 = Partially correct
+- 0.3 = Mentions something relevant but mostly wrong
+- 0.0 = Wrong or "I don't know"
+
+Question: ${question}
+Ground Truth: ${truth}
+AI's Answer: ${answer}
+
+Respond with ONLY a decimal number (e.g. 0.7). Nothing else.`;
+
+  const response = await geminiCall(prompt);
+  const cleaned = response.trim();
+  
+  // Try direct float parse first
+  const direct = parseFloat(cleaned);
+  if (!isNaN(direct) && direct >= 0 && direct <= 1) return direct;
+  
+  // Try regex
+  const match = cleaned.match(/(0\.\d+|1\.0|0|1)/);
+  if (match) return parseFloat(match[1]);
+  
+  console.log(`    Failed to parse score: "${cleaned}"`);
+  return -1; // Mark as failed, don't default to 0
 }
 
 async function main() {
   const results = JSON.parse(readFileSync(RESULTS_PATH, 'utf8'));
-  console.log(`Rescoring ${results.length} codebase questions...\n`);
-
-  let scored = 0;
-  for (const r of results) {
-    // Skip if already scored
-    if (r.engram.correct > 0 || r.fullContext.correct > 0 || r.grepSearch?.correct > 0) {
-      console.log(`  [${r.index}] Already scored, skipping`);
-      scored++;
-      continue;
-    }
-
-    try {
-      await sleep(RATE_LIMIT_MS);
-      const scorePrompt = `Score these three answers to a codebase question on a scale of 0.0 to 1.0.
-0.0 = completely wrong or irrelevant
-0.5 = partially correct, missing key details
-1.0 = fully correct and complete
-
-Question: ${r.question}
-Ground Truth: ${r.groundTruth}
-
-Answer A (Engram): ${r.engram.answer}
-Answer B (Full Context): ${r.fullContext.answer}
-Answer C (Grep Search): ${r.grepSearch.answer}
-
-Output ONLY a JSON object: {"a": <score>, "b": <score>, "c": <score>}`;
-
-      const scoreResponse = await geminiCall(scorePrompt, 200, 3, true);
-      // Aggressively extract JSON from any preamble/wrapper text
-      const firstBrace = scoreResponse.indexOf('{');
-      const lastBrace = scoreResponse.lastIndexOf('}');
-      const scores = (firstBrace >= 0 && lastBrace > firstBrace)
-        ? JSON.parse(scoreResponse.slice(firstBrace, lastBrace + 1))
-        : null;
-
-      if (!scores) {
-        console.log(`  [${r.index}] Failed to parse scores: ${scoreResponse.slice(0, 100)}`);
-        continue;
-      }
-
-      r.engram.correct = scores.a;
-      r.fullContext.correct = scores.b;
-      if (r.grepSearch) r.grepSearch.correct = scores.c;
-
-      scored++;
-      console.log(`  [${r.index}] E: ${scores.a.toFixed(1)} | F: ${scores.b.toFixed(1)} | G: ${scores.c.toFixed(1)} — ${r.question.slice(0, 60)}...`);
-
-      // Auto-save every 5
-      if (scored % 5 === 0) {
-        writeFileSync(RESULTS_PATH, JSON.stringify(results, null, 2));
+  console.log(`Rescoring ${results.length} results...\n`);
+  
+  const systems = ['engram', 'cappedContext', 'naiveRag', 'grepSearch'] as const;
+  let totalScored = 0;
+  let totalFailed = 0;
+  
+  for (let i = 0; i < results.length; i++) {
+    const r = results[i];
+    console.log(`[${i+1}/${results.length}] (${r.category}/${r.difficulty}) ${r.question.slice(0, 70)}...`);
+    
+    const scores: Record<string, number> = {};
+    for (const sys of systems) {
+      if (!r[sys]?.answer) { scores[sys] = 0; continue; }
+      const score = await scoreAnswer(r.question, r.groundTruth, r[sys].answer);
+      if (score === -1) {
+        totalFailed++;
+        scores[sys] = 0;
+      } else {
+        scores[sys] = score;
       }
-    } catch (err: any) {
-      console.error(`  [${r.index}] Error: ${err.message}`);
-      await sleep(15000);
+      r[sys].score = scores[sys];
+    }
+    
+    totalScored += systems.length;
+    const line = systems.map(s => `${s[0].toUpperCase()}:${scores[s].toFixed(2)}`).join(' ');
+    console.log(`  ${line}`);
+    
+    // Save progress every 5 questions
+    if ((i + 1) % 5 === 0 || i === results.length - 1) {
+      writeFileSync(RESULTS_PATH.replace('.json', '-rescored2.json'), JSON.stringify(results, null, 2));
     }
   }
-
-  writeFileSync(RESULTS_PATH, JSON.stringify(results, null, 2));
-  console.log(`\n✅ Rescored ${scored}/${results.length} questions`);
-
-  // Quick summary
-  const avgE = results.reduce((s: number, r: any) => s + r.engram.correct, 0) / results.length * 100;
-  const avgF = results.reduce((s: number, r: any) => s + r.fullContext.correct, 0) / results.length * 100;
-  const avgG = results.reduce((s: number, r: any) => s + (r.grepSearch?.correct || 0), 0) / results.length * 100;
-  console.log(`\nEngram: ${avgE.toFixed(1)}% | Full Context: ${avgF.toFixed(1)}% | Grep: ${avgG.toFixed(1)}%`);
+  
+  console.log(`\nScored: ${totalScored}, Failed parses: ${totalFailed}\n`);
+  
+  // Generate report
+  const avg = (sys: string) => {
+    const vals = results.map((r: any) => r[sys]?.score ?? 0);
+    return vals.reduce((a: number, b: number) => a + b, 0) / vals.length;
+  };
+  const avgTokens = (sys: string) => {
+    const vals = results.map((r: any) => r[sys]?.tokensUsed ?? 0);
+    return Math.round(vals.reduce((a: number, b: number) => a + b, 0) / vals.length);
+  };
+  
+  console.log('=== VS Code Codebase Evaluation Report ===\n');
+  console.log('OVERALL (50 questions)');
+  console.log(`${'System'.padEnd(20)} ${'Accuracy'.padEnd(12)} Avg Tokens`);
+  for (const sys of systems) {
+    const acc = (avg(sys) * 100).toFixed(1);
+    console.log(`${sys.padEnd(20)} ${(acc + '%').padEnd(12)} ${avgTokens(sys)}`);
+  }
+  
+  // Per category
+  const categories = [...new Set(results.map((r: any) => r.category))];
+  for (const cat of categories) {
+    const catResults = results.filter((r: any) => r.category === cat);
+    const catAvg = (sys: string) => {
+      const vals = catResults.map((r: any) => r[sys]?.score ?? 0);
+      return (vals.reduce((a: number, b: number) => a + b, 0) / vals.length * 100).toFixed(1);
+    };
+    console.log(`\n  ${cat.toUpperCase()} (n=${catResults.length}): ${systems.map(s => `${s[0].toUpperCase()}:${catAvg(s)}%`).join(' ')}`);
+  }
+  
+  // Per difficulty
+  const diffs = [...new Set(results.map((r: any) => r.difficulty))];
+  for (const diff of diffs) {
+    const diffResults = results.filter((r: any) => r.difficulty === diff);
+    const diffAvg = (sys: string) => {
+      const vals = diffResults.map((r: any) => r[sys]?.score ?? 0);
+      return (vals.reduce((a: number, b: number) => a + b, 0) / vals.length * 100).toFixed(1);
+    };
+    console.log(`\n  ${diff.toUpperCase()} (n=${diffResults.length}): ${systems.map(s => `${s[0].toUpperCase()}:${diffAvg(s)}%`).join(' ')}`);
+  }
+  
+  const tokenSavings = (1 - avgTokens('engram') / avgTokens('cappedContext')) * 100;
+  console.log(`\n  Token savings vs capped context: ${tokenSavings.toFixed(1)}%`);
+  
+  // Save report
+  const report = {
+    timestamp: new Date().toISOString(),
+    totalQuestions: results.length,
+    failedParses: totalFailed,
+    overall: Object.fromEntries(systems.map(s => [s, {
+      accuracy: (avg(s) * 100).toFixed(1),
+      avgTokens: avgTokens(s),
+    }])),
+    byCategory: Object.fromEntries(categories.map(c => {
+      const cr = results.filter((r: any) => r.category === c);
+      return [c, Object.fromEntries(systems.map(s => [s, (cr.reduce((a: number, r: any) => a + (r[s]?.score ?? 0), 0) / cr.length * 100).toFixed(1)]))];
+    })),
+    byDifficulty: Object.fromEntries(diffs.map(d => {
+      const dr = results.filter((r: any) => r.difficulty === d);
+      return [d, Object.fromEntries(systems.map(s => [s, (dr.reduce((a: number, r: any) => a + (r[s]?.score ?? 0), 0) / dr.length * 100).toFixed(1)]))];
+    })),
+    tokenSavingsVsCapped: tokenSavings.toFixed(1) + '%',
+  };
+  writeFileSync(REPORT_PATH, JSON.stringify(report, null, 2));
+  console.log(`\nReport saved: ${REPORT_PATH}`);
 }
 
 main().catch(console.error);

package/server.json

@@ -0,0 +1,29 @@
+{
+  "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
+  "name": "io.github.tstockham96/engram",
+  "description": "Intelligent memory layer for AI agents. Automatic extraction, consolidation, bi-temporal recall, and proactive context surfacing. Scores 80% on LOCOMO benchmark at 93% fewer tokens than full context.",
+  "repository": {
+    "url": "https://github.com/tstockham96/engram",
+    "source": "github"
+  },
+  "version": "0.4.4",
+  "packages": [
+    {
+      "registryType": "npm",
+      "identifier": "engram-sdk",
+      "version": "0.4.4",
+      "transport": {
+        "type": "stdio"
+      },
+      "environmentVariables": [
+        {
+          "description": "Google Gemini API key for embeddings and LLM operations",
+          "isRequired": true,
+          "format": "string",
+          "isSecret": true,
+          "name": "GEMINI_API_KEY"
+        }
+      ]
+    }
+  ]
+}