engram-sdk 0.5.5 → 0.5.6

package/eval-rescore-judge.mjs

@@ -0,0 +1,158 @@
+#!/usr/bin/env node
+/**
+ * Re-score existing LOCOMO results with GPT-4o-mini judge (matches Mem0 methodology).
+ * 
+ * Usage:
+ *   OPENAI_API_KEY=sk-... node eval-rescore-judge.mjs          # Run re-judge
+ *   node eval-rescore-judge.mjs --report                        # Compare judges
+ */
+
+import { readFileSync, writeFileSync, existsSync } from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+
+const EVAL_DIR = join(homedir(), '.openclaw/workspace/engram/eval-scale-data');
+const RESULTS_PATH = join(EVAL_DIR, 'locomo-results.json');
+const OPENAI_KEY = process.env.OPENAI_API_KEY || readFileSync(join(homedir(), '.config/engram/openai-key'), 'utf8').trim();
+const OUTPUT_PATH = join(EVAL_DIR, 'locomo-results-rejudge-openai.json');
+
+const sleep = ms => new Promise(r => setTimeout(r, ms));
+
+async function judgeOpenAI(question, groundTruth, systemAnswer) {
+  const prompt = `You are evaluating the quality of an AI system's answer about a conversation.
+
+Question: ${question}
+Ground Truth Answer: ${groundTruth}
+System Answer: ${systemAnswer}
+
+Rate the system's answer on a scale from 0.0 to 1.0 based on:
+- Factual accuracy compared to the ground truth
+- Relevance to the question asked  
+- Completeness of the answer
+- Whether it contains any incorrect information
+
+Provide your evaluation as a JSON object:
+{"score": 0.85, "reason": "The answer correctly identifies..."}`;
+
+  for (let attempt = 1; attempt <= 5; attempt++) {
+    try {
+      const resp = await fetch('https://api.openai.com/v1/chat/completions', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Authorization': `Bearer ${OPENAI_KEY}`,
+        },
+        body: JSON.stringify({
+          model: 'gpt-4o-mini',
+          messages: [{ role: 'user', content: prompt }],
+          temperature: 0.1,
+          max_tokens: 500,
+          response_format: { type: 'json_object' },
+        }),
+      });
+      if (resp.status === 429) {
+        const wait = attempt * 15;
+        console.warn(`  [429] Retry in ${wait}s...`);
+        await sleep(wait * 1000);
+        continue;
+      }
+      const data = await resp.json();
+      const text = data.choices?.[0]?.message?.content || '';
+      const parsed = JSON.parse(text);
+      return Math.max(0, Math.min(1, parsed.score || 0));
+    } catch (err) {
+      console.warn(`  [ERR attempt ${attempt}]`, err.message);
+      if (attempt === 5) return 0;
+      await sleep(attempt * 5000);
+    }
+  }
+  return 0;
+}
+
+async function report() {
+  const cats = { '1': 'Single-Hop', '2': 'Temporal', '3': 'Open-Domain', '4': 'Multi-Hop' };
+  
+  for (const [label, path] of [['Original (Gemini judge)', RESULTS_PATH], ['GPT-4o-mini judge', OUTPUT_PATH]]) {
+    if (!existsSync(path)) { console.log(`${label}: not found`); continue; }
+    const data = JSON.parse(readFileSync(path, 'utf8'));
+    
+    const catScores = {};
+    for (const r of data) {
+      const cat = cats[String(r.category)] || String(r.category);
+      if (!catScores[cat]) catScores[cat] = { engram: [], full: [], md: [] };
+      catScores[cat].engram.push(r.results.engram.score);
+      catScores[cat].full.push(r.results.fullContext.score);
+      catScores[cat].md.push(r.results.memoryMd.score);
+    }
+    
+    const avg = arr => arr.length ? (arr.reduce((a, b) => a + b, 0) / arr.length * 100).toFixed(1) : 'N/A';
+    
+    console.log(`\n=== ${label} (${data.length} questions) ===`);
+    console.log(`${'Category'.padEnd(15)} ${'Engram'.padStart(8)} ${'Full'.padStart(8)} ${'MD'.padStart(8)} ${'Count'.padStart(6)}`);
+    console.log('-'.repeat(50));
+    
+    let allE = [], allF = [], allM = [];
+    for (const cat of ['Single-Hop', 'Temporal', 'Multi-Hop', 'Open-Domain']) {
+      const d = catScores[cat];
+      if (!d) continue;
+      console.log(`${cat.padEnd(15)} ${avg(d.engram).padStart(8)} ${avg(d.full).padStart(8)} ${avg(d.md).padStart(8)} ${String(d.engram.length).padStart(6)}`);
+      allE.push(...d.engram); allF.push(...d.full); allM.push(...d.md);
+    }
+    console.log('-'.repeat(50));
+    console.log(`${'Overall'.padEnd(15)} ${avg(allE).padStart(8)} ${avg(allF).padStart(8)} ${avg(allM).padStart(8)} ${String(allE.length).padStart(6)}`);
+  }
+}
+
+async function main() {
+  if (process.argv.includes('--report')) return report();
+  
+  console.log('Re-judging LOCOMO results with GPT-4o-mini...');
+  console.log(`Input: ${RESULTS_PATH}`);
+  console.log(`Output: ${OUTPUT_PATH}\n`);
+  
+  const results = JSON.parse(readFileSync(RESULTS_PATH, 'utf8'));
+  
+  // Resume support
+  let rescored = [];
+  const doneSet = new Set();
+  if (existsSync(OUTPUT_PATH)) {
+    rescored = JSON.parse(readFileSync(OUTPUT_PATH, 'utf8'));
+    for (const r of rescored) doneSet.add(r.questionId);
+    console.log(`Resuming: ${rescored.length}/${results.length} already done\n`);
+  }
+  
+  let done = rescored.length;
+  const total = results.length;
+  
+  for (const r of results) {
+    if (doneSet.has(r.questionId)) continue;
+    
+    const eScore = await judgeOpenAI(r.question, r.groundTruth, r.results.engram.answer);
+    await sleep(200);
+    const fScore = await judgeOpenAI(r.question, r.groundTruth, r.results.fullContext.answer);
+    await sleep(200);
+    const mScore = await judgeOpenAI(r.question, r.groundTruth, r.results.memoryMd.answer);
+    await sleep(200);
+    
+    rescored.push({
+      ...r,
+      results: {
+        engram: { ...r.results.engram, score: eScore },
+        fullContext: { ...r.results.fullContext, score: fScore },
+        memoryMd: { ...r.results.memoryMd, score: mScore },
+      },
+    });
+    done++;
+    
+    if (done % 10 === 0 || done === total) {
+      writeFileSync(OUTPUT_PATH, JSON.stringify(rescored, null, 2));
+      const eAvg = (rescored.reduce((s, x) => s + x.results.engram.score, 0) / rescored.length * 100).toFixed(1);
+      const fAvg = (rescored.reduce((s, x) => s + x.results.fullContext.score, 0) / rescored.length * 100).toFixed(1);
+      console.log(`[${done}/${total}] Engram: ${eAvg}% | Full: ${fAvg}% | Last: E=${eScore.toFixed(2)} F=${fScore.toFixed(2)} M=${mScore.toFixed(2)}`);
+    }
+  }
+  
+  console.log('\nDone! Run with --report to compare.');
+}
+
+main().catch(err => { console.error(err); process.exit(1); });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "engram-sdk",
   "mcpName": "io.github.tstockham96/engram",
-  "version": "0.5.5",
+  "version": "0.5.6",
   "description": "Universal memory layer for AI agents. Remember, recall, consolidate.",
   "type": "module",
   "main": "dist/index.js",