@eidentic/bench 0.1.0 → 0.1.2

package/dist/index.cjs

@@ -5,6 +5,9 @@ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
 var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
 var __export = (target, all) => {
   for (var name in all)
     __defProp(target, name, { get: all[name], enumerable: true });
@@ -27,17 +30,238 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
+// src/locomo-loader.ts
+var locomo_loader_exports = {};
+__export(locomo_loader_exports, {
+  LOCOMO_SOURCE_SHA: () => LOCOMO_SOURCE_SHA,
+  loadLoCoMo: () => loadLoCoMo2
+});
+async function assertFileSize2(filePath, maxBytes) {
+  let fileSize;
+  try {
+    const s = await (0, import_promises2.stat)(filePath);
+    fileSize = s.size;
+  } catch (err) {
+    throw new Error(
+      `bench loader: cannot stat file "${filePath}": ${err.message}`
+    );
+  }
+  if (fileSize > maxBytes) {
+    const mb = (fileSize / (1024 * 1024)).toFixed(1);
+    const capMb = (maxBytes / (1024 * 1024)).toFixed(0);
+    throw new Error(
+      `bench loader: file "${filePath}" is ${mb} MiB, which exceeds the ${capMb} MiB cap. Pass a larger maxBytes option if this is intentional.`
+    );
+  }
+}
+function parseLocomoDateTime(raw) {
+  const cleaned = raw.replace(/\s*on\s+/, " ").trim();
+  const ms = Date.parse(cleaned);
+  if (Number.isNaN(ms)) {
+    const match = /^(\d{1,2}:\d{2}\s*(?:am|pm))\s+(.+)$/i.exec(cleaned);
+    if (match) {
+      const ms2 = Date.parse(`${match[2]} ${match[1]}`);
+      return Number.isNaN(ms2) ? 0 : ms2;
+    }
+    return 0;
+  }
+  return ms;
+}
+function parseSessions(conv) {
+  const indices = [];
+  for (const key of Object.keys(conv)) {
+    const m = /^session_(\d+)$/.exec(key);
+    if (m) indices.push(parseInt(m[1], 10));
+  }
+  indices.sort((a, b) => a - b);
+  const sessions = [];
+  for (const idx of indices) {
+    const turnsRaw = conv[`session_${idx}`];
+    if (!Array.isArray(turnsRaw)) continue;
+    const dateTimeRaw = typeof conv[`session_${idx}_date_time`] === "string" ? conv[`session_${idx}_date_time`] : "";
+    const turns = turnsRaw.map((t) => ({
+      speaker: t.speaker ?? "",
+      diaId: t.dia_id ?? "",
+      text: t.text ?? ""
+    }));
+    sessions.push({
+      index: idx,
+      dateTime: dateTimeRaw,
+      dateTimeMs: dateTimeRaw ? parseLocomoDateTime(dateTimeRaw) : 0,
+      turns
+    });
+  }
+  return sessions;
+}
+async function loadLoCoMo2(jsonPath, opts) {
+  await assertFileSize2(jsonPath, opts?.maxBytes ?? DEFAULT_MAX_BYTES2);
+  const raw = JSON.parse(await (0, import_promises2.readFile)(jsonPath, "utf-8"));
+  if (!Array.isArray(raw)) {
+    throw new Error(
+      `bench loader: expected the LoCoMo JSON root to be an array, but got ${typeof raw}. Did you pass the correct file?`
+    );
+  }
+  const rawSamples = raw;
+  const samples = rawSamples.map((s, i) => {
+    const sampleId = s.sample_id ?? String(i);
+    const conv = s.conversation ?? {};
+    const sessions = parseSessions(conv);
+    const qa = (s.qa ?? []).map((q) => {
+      const answer = q.answer !== void 0 ? String(q.answer) : void 0;
+      return {
+        question: q.question ?? "",
+        answer,
+        category: q.category,
+        evidence: Array.isArray(q.evidence) ? q.evidence : [],
+        adversarialAnswer: q.adversarial_answer
+      };
+    });
+    return {
+      sampleId,
+      speakerA: String(conv.speaker_a ?? ""),
+      speakerB: String(conv.speaker_b ?? ""),
+      sessions,
+      qa
+    };
+  });
+  return { samples };
+}
+var import_promises2, LOCOMO_SOURCE_SHA, DEFAULT_MAX_BYTES2;
+var init_locomo_loader = __esm({
+  "src/locomo-loader.ts"() {
+    "use strict";
+    import_promises2 = require("node:fs/promises");
+    LOCOMO_SOURCE_SHA = "3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376";
+    DEFAULT_MAX_BYTES2 = 256 * 1024 * 1024;
+  }
+});
+
+// src/lme-loader.ts
+var lme_loader_exports = {};
+__export(lme_loader_exports, {
+  LONGMEMEVAL_SOURCE: () => LONGMEMEVAL_SOURCE,
+  loadLongMemEval: () => loadLongMemEval2,
+  parseLmeDateTimeString: () => parseLmeDateTimeString
+});
+async function assertFileSize3(filePath, maxBytes) {
+  let fileSize;
+  try {
+    const s = await (0, import_promises4.stat)(filePath);
+    fileSize = s.size;
+  } catch (err) {
+    throw new Error(
+      `bench loader: cannot stat file "${filePath}": ${err.message}`
+    );
+  }
+  if (fileSize > maxBytes) {
+    const mb = (fileSize / (1024 * 1024)).toFixed(1);
+    const capMb = (maxBytes / (1024 * 1024)).toFixed(0);
+    throw new Error(
+      `bench loader: file "${filePath}" is ${mb} MiB, which exceeds the ${capMb} MiB cap. Pass a larger maxBytes option if this is intentional.`
+    );
+  }
+}
+function parseLmeDateTimeString(raw) {
+  if (!raw) return 0;
+  const cleaned = raw.replace(/\s*\([A-Za-z]+\)\s*/, " ").trim();
+  const iso = cleaned.replace(/^(\d{4})\/(\d{2})\/(\d{2})/, "$1-$2-$3");
+  const ms = Date.parse(iso);
+  if (Number.isNaN(ms)) return 0;
+  return ms;
+}
+function extractBaseType(rawType) {
+  if (rawType.endsWith("_abs")) {
+    return rawType.slice(0, -4);
+  }
+  return rawType;
+}
+function parseSession(id, dateTime, rawTurns) {
+  const turns = rawTurns.map((t) => ({
+    role: t.role === "assistant" ? "assistant" : "user",
+    content: t.content ?? "",
+    hasAnswer: t.has_answer === true
+  }));
+  return {
+    id,
+    dateTime,
+    dateTimeMs: parseLmeDateTimeString(dateTime),
+    turns
+  };
+}
+async function loadLongMemEval2(jsonPath, opts) {
+  await assertFileSize3(jsonPath, opts?.maxBytes ?? DEFAULT_MAX_BYTES3);
+  const raw = JSON.parse(await (0, import_promises4.readFile)(jsonPath, "utf-8"));
+  if (!Array.isArray(raw)) {
+    throw new Error(
+      `bench loader: expected the LongMemEval JSON root to be an array, but got ${typeof raw}. Did you pass the correct file?`
+    );
+  }
+  const rawQuestions = raw;
+  const questions = rawQuestions.map((q, i) => {
+    const id = q.question_id ?? String(i);
+    const rawType = q.question_type ?? "single-session-user";
+    const baseType = extractBaseType(rawType);
+    const isAbstention = rawType.endsWith("_abs");
+    const rawSessions = Array.isArray(q.haystack_sessions) ? q.haystack_sessions : [];
+    const dates = Array.isArray(q.haystack_dates) ? q.haystack_dates : [];
+    const sessionIds = Array.isArray(q.haystack_session_ids) ? q.haystack_session_ids : [];
+    const sessions = rawSessions.map((turns, idx) => {
+      const sessId = sessionIds[idx] ?? `sess-${idx}`;
+      const dateTime = dates[idx] ?? "";
+      return parseSession(sessId, dateTime, Array.isArray(turns) ? turns : []);
+    });
+    sessions.sort((a, b) => a.dateTimeMs - b.dateTimeMs);
+    return {
+      id,
+      type: rawType,
+      baseType,
+      isAbstention,
+      question: q.question ?? "",
+      answer: q.answer ?? "",
+      questionDate: q.question_date ?? "",
+      questionDateMs: parseLmeDateTimeString(q.question_date ?? ""),
+      sessions,
+      answerSessionIds: Array.isArray(q.answer_session_ids) ? q.answer_session_ids : []
+    };
+  });
+  return { questions };
+}
+var import_promises4, LONGMEMEVAL_SOURCE, DEFAULT_MAX_BYTES3;
+var init_lme_loader = __esm({
+  "src/lme-loader.ts"() {
+    "use strict";
+    import_promises4 = require("node:fs/promises");
+    LONGMEMEVAL_SOURCE = {
+      url: "https://huggingface.co/datasets/xiaowu0162/longmemeval",
+      snapshotSha: "2ec2a557f339b6c0369619b1ed5793734cc87533",
+      file: "longmemeval_s",
+      license: "MIT"
+    };
+    DEFAULT_MAX_BYTES3 = 512 * 1024 * 1024;
+  }
+});
+
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
   CONTRADICTION_FIXTURES: () => CONTRADICTION_FIXTURES,
   JUNK_STREAM_FIXTURES: () => JUNK_STREAM_FIXTURES,
+  LOCOMO_SOURCE_SHA: () => LOCOMO_SOURCE_SHA,
+  LONGMEMEVAL_SOURCE: () => LONGMEMEVAL_SOURCE,
   factRecall: () => factRecall,
-  loadLoCoMo: () => loadLoCoMo,
-  loadLongMemEval: () => loadLongMemEval,
+  loadLoCoMo: () => loadLoCoMo2,
+  loadLoCoMoLegacy: () => loadLoCoMo,
+  loadLongMemEval: () => loadLongMemEval2,
+  loadLongMemEvalLegacy: () => loadLongMemEval,
   normalizeText: () => normalizeText,
   normalizedIncludes: () => normalizedIncludes,
+  parseLmeDateTimeString: () => parseLmeDateTimeString,
   recallAtK: () => recallAtK,
+  renderLocomoReportMarkdown: () => renderLocomoReportMarkdown,
+  renderLongMemEvalReportMarkdown: () => renderLongMemEvalReportMarkdown,
+  resolveEvidence: () => resolveEvidence,
+  runLocomoBench: () => runLocomoBench,
+  runLongMemEvalBench: () => runLongMemEvalBench,
   runMemoryBench: () => runMemoryBench,
   runTemporalBench: () => runTemporalBench,
   runWriteQualityBench: () => runWriteQualityBench,
@@ -445,6 +669,1107 @@ async function loadLoCoMo(jsonPath, opts) {
   return { name: "LoCoMo", cases: benchCases };
 }
 
+// src/index.ts
+init_locomo_loader();
+
+// src/locomo-run.ts
+var import_promises3 = require("node:fs/promises");
+var import_node_fs = require("node:fs");
+init_locomo_loader();
+function makeRng(seed) {
+  let s = seed >>> 0;
+  if (s === 0) s = 1;
+  return () => {
+    s ^= s << 13;
+    s ^= s >>> 17;
+    s ^= s << 5;
+    s = s >>> 0;
+    return s / 4294967296;
+  };
+}
+function seededShuffle(arr, rng) {
+  const out = [...arr];
+  for (let i = out.length - 1; i > 0; i--) {
+    const j = Math.floor(rng() * (i + 1));
+    [out[i], out[j]] = [out[j], out[i]];
+  }
+  return out;
+}
+var ANSWER_INSTRUCTION = "Answer the question concisely and specifically based only on the information provided. If the provided context contains no answer to the question, reply exactly: No information available";
+function buildMemoryAnswerPrompt(snippets, question) {
+  const context = snippets.length > 0 ? snippets.map((s, i) => `[${i + 1}] ${s}`).join("\n\n") : "(no relevant context retrieved)";
+  return `Context from conversation:
+${context}
+
+Question: ${question}
+
+${ANSWER_INSTRUCTION}`;
+}
+function buildFullContextAnswerPrompt(conversationText, question) {
+  return `Conversation:
+${conversationText}
+
+Question: ${question}
+
+${ANSWER_INSTRUCTION}`;
+}
+function buildJudgePrompt(question, goldAnswer, modelAnswer, adversarialAnswer, isCategory5) {
+  if (isCategory5) {
+    const trapNote = adversarialAnswer ? `
+WARNING \u2014 trap answer to watch for: "${adversarialAnswer}". If the model answer closely matches this trap, mark incorrect.` : "";
+    return `You are a strict judge. For this unanswerable question, a CORRECT response means the model refused to answer or said something equivalent to "No information available" or "I don't know".${trapNote}
+
+Question: ${question}
+Model answer: ${modelAnswer}
+
+Respond with JSON only: {"correct": true} if the model declined (correct), {"correct": false} if it made a claim.`;
+  }
+  return `You are a strict judge evaluating whether a model answer is correct.
+CORRECT: the model answer contains the specific information in the gold answer (exact match or clear paraphrase).
+WRONG: the model answer is vague, only topically related, or contradicts the gold answer.
+For date/time questions: equivalent date expressions referring to the same date are correct.
+
+Question: ${question}
+Gold answer: ${goldAnswer ?? "(none)"}
+Model answer: ${modelAnswer}
+
+Respond with JSON only: {"correct": true} or {"correct": false}`;
+}
+function renderConversation(sample) {
+  const lines = [];
+  for (const sess of sample.sessions) {
+    const dateLabel = sess.dateTime ? ` \u2014 ${sess.dateTime}` : "";
+    lines.push(`Session ${sess.index}${dateLabel}`);
+    for (const turn of sess.turns) {
+      lines.push(`[${turn.speaker}]: ${turn.text}`);
+    }
+    lines.push("");
+  }
+  return lines.join("\n").trim();
+}
+async function ingestSampleIntoMemory(sample, memory, scope) {
+  const events = [];
+  for (const sess of sample.sessions) {
+    const dateLabel = sess.dateTime ? ` \u2014 ${sess.dateTime}` : "";
+    const header = `Session ${sess.index}${dateLabel}`;
+    for (const turn of sess.turns) {
+      events.push({
+        id: `${sample.sampleId}:turn:${turn.diaId}`,
+        scope,
+        text: `[${header}] [${turn.speaker}]: ${turn.text}`,
+        metadata: {
+          diaId: turn.diaId,
+          sessionIndex: sess.index,
+          ingestedAt: sess.dateTimeMs || void 0
+        }
+      });
+    }
+    const sessionText = [
+      header,
+      ...sess.turns.map((t) => `[${t.speaker}]: ${t.text}`)
+    ].join("\n");
+    events.push({
+      id: `${sample.sampleId}:sess:${sess.index}:chunk`,
+      scope,
+      text: sessionText,
+      metadata: { sessionIndex: sess.index, ingestedAt: sess.dateTimeMs || void 0 }
+    });
+  }
+  await memory.ingest(events);
+}
+async function callJudge(judgeModel, prompt) {
+  const response = await judgeModel.complete({
+    messages: [{ role: "user", content: prompt }],
+    tools: [],
+    outputSchema: {
+      type: "object",
+      properties: { correct: { type: "boolean" } },
+      required: ["correct"],
+      // OpenAI strict structured-output mode requires this to be explicit.
+      additionalProperties: false
+    }
+  });
+  let correct = false;
+  if (response.object && typeof response.object.correct === "boolean") {
+    correct = response.object.correct;
+  } else {
+    const text = response.content.filter((b) => b.type === "text").map((b) => b.text ?? "").join("").toLowerCase().trim();
+    if (/"correct"\s*:\s*true/i.test(text)) correct = true;
+    else if (/"correct"\s*:\s*false/i.test(text)) correct = false;
+    else correct = text.includes("true");
+  }
+  return {
+    correct,
+    inputTokens: response.usage?.inputTokens ?? 0,
+    outputTokens: response.usage?.outputTokens ?? 0
+  };
+}
+async function loadCheckpoint(path) {
+  const done = /* @__PURE__ */ new Set();
+  if (!(0, import_node_fs.existsSync)(path)) return done;
+  const raw = await (0, import_promises3.readFile)(path, "utf-8");
+  for (const line of raw.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      const row = JSON.parse(trimmed);
+      if (row.sampleId && row.questionIndex !== void 0) {
+        done.add(`${row.sampleId}:${row.questionIndex}`);
+      }
+    } catch {
+    }
+  }
+  return done;
+}
+async function appendCheckpointRow(path, row) {
+  await (0, import_promises3.appendFile)(path, JSON.stringify(row) + "\n", "utf-8");
+}
+var DECLINE_PATTERNS = /\bno information available\b|\bi (don'?t|do not) know\b|\bcannot (find|answer|determine)\b|\bunavailable\b/i;
+function appearsToDecline(answer) {
+  return DECLINE_PATTERNS.test(answer);
+}
+async function runLocomoBench(opts) {
+  const {
+    answerModel,
+    judgeModel,
+    mode,
+    categories = [1, 2, 3, 4, 5],
+    sampleLimit,
+    questionLimit,
+    seed = 42,
+    topK: rawTopK = 10,
+    concurrency = 1,
+    onProgress,
+    checkpointPath
+  } = opts;
+  const topK = Math.min(rawTopK, 10);
+  const rng = makeRng(seed);
+  let dataset;
+  if (opts.dataset) {
+    dataset = opts.dataset;
+  } else {
+    const { loadLoCoMo: loadLoCoMo3 } = await Promise.resolve().then(() => (init_locomo_loader(), locomo_loader_exports));
+    dataset = await loadLoCoMo3(opts.dataPath);
+  }
+  let samples = [...dataset.samples];
+  if (sampleLimit !== void 0 && sampleLimit < samples.length) {
+    samples = seededShuffle(samples, makeRng(seed)).slice(0, sampleLimit);
+  }
+  const queue = [];
+  for (const sample of samples) {
+    let qaList = sample.qa.filter((q) => categories.includes(q.category));
+    if (questionLimit !== void 0 && questionLimit < qaList.length) {
+      qaList = seededShuffle(qaList, makeRng(seed + 1)).slice(0, questionLimit);
+    }
+    for (const qa of qaList) {
+      queue.push({ sample, qaIndex: sample.qa.indexOf(qa), qa });
+    }
+  }
+  const checkpoint = checkpointPath ? await loadCheckpoint(checkpointPath) : /* @__PURE__ */ new Set();
+  const conversationText = /* @__PURE__ */ new Map();
+  if (mode === "full-context") {
+    for (const sample of samples) {
+      conversationText.set(sample.sampleId, renderConversation(sample));
+    }
+  }
+  const memories = /* @__PURE__ */ new Map();
+  let ingestInputTokens = 0;
+  let ingestOutputTokens = 0;
+  if (mode === "memory") {
+    if (!opts.memoryFactory) {
+      throw new Error("runLocomoBench: memoryFactory is required when mode='memory'");
+    }
+    for (const sample of samples) {
+      const memory = await opts.memoryFactory(sample.sampleId);
+      const scope = { kind: "agent", agentId: `locomo:${sample.sampleId}` };
+      await ingestSampleIntoMemory(sample, memory, scope);
+      memories.set(sample.sampleId, memory);
+    }
+  }
+  const results = [];
+  let totalAnswerInputTokens = 0;
+  let totalAnswerOutputTokens = 0;
+  let totalJudgeInputTokens = 0;
+  let totalJudgeOutputTokens = 0;
+  let errorCount = 0;
+  let done = 0;
+  const total = queue.length - checkpoint.size;
+  const startTime = Date.now();
+  const processItem = async (item) => {
+    const key = `${item.sample.sampleId}:${item.qaIndex}`;
+    if (checkpoint.has(key)) return;
+    let modelAnswer = "";
+    let answerIn = 0;
+    let answerOut = 0;
+    let judgeIn = 0;
+    let judgeOut = 0;
+    let correct = false;
+    let trapTriggered;
+    let errorMsg;
+    try {
+      if (mode === "memory") {
+        const memory = memories.get(item.sample.sampleId);
+        const scope = { kind: "agent", agentId: `locomo:${item.sample.sampleId}` };
+        const retrieved = await memory.retrieve({ text: item.qa.question, scope, topK });
+        const snippets = retrieved.snippets.map((s) => s.text);
+        const prompt = buildMemoryAnswerPrompt(snippets, item.qa.question);
+        const resp = await answerModel.complete({
+          messages: [{ role: "user", content: prompt }],
+          tools: []
+        });
+        const textBlocks = resp.content.filter((b) => b.type === "text");
+        modelAnswer = textBlocks.map((b) => b.text).join("").trim();
+        answerIn = resp.usage?.inputTokens ?? 0;
+        answerOut = resp.usage?.outputTokens ?? 0;
+      } else {
+        const convText = conversationText.get(item.sample.sampleId) ?? "";
+        const prompt = buildFullContextAnswerPrompt(convText, item.qa.question);
+        const resp = await answerModel.complete({
+          messages: [{ role: "user", content: prompt }],
+          tools: []
+        });
+        const textBlocks = resp.content.filter((b) => b.type === "text");
+        modelAnswer = textBlocks.map((b) => b.text).join("").trim();
+        answerIn = resp.usage?.inputTokens ?? 0;
+        answerOut = resp.usage?.outputTokens ?? 0;
+      }
+      const isCategory5 = item.qa.category === 5;
+      const judgePrompt = buildJudgePrompt(
+        item.qa.question,
+        item.qa.answer,
+        modelAnswer,
+        item.qa.adversarialAnswer,
+        isCategory5
+      );
+      const judgeResult = await callJudge(judgeModel, judgePrompt);
+      correct = judgeResult.correct;
+      judgeIn = judgeResult.inputTokens;
+      judgeOut = judgeResult.outputTokens;
+      if (isCategory5 && item.qa.adversarialAnswer) {
+        const adversarialLower = item.qa.adversarialAnswer.toLowerCase();
+        const answerLower = modelAnswer.toLowerCase();
+        trapTriggered = answerLower.includes(adversarialLower) || adversarialLower.includes(answerLower.slice(0, Math.min(answerLower.length, 30)));
+        if (trapTriggered && correct) correct = false;
+        if (!correct && appearsToDecline(modelAnswer)) {
+          correct = true;
+          trapTriggered = false;
+        }
+      }
+    } catch (err) {
+      errorMsg = err.message;
+      errorCount++;
+      correct = false;
+    }
+    const row = {
+      sampleId: item.sample.sampleId,
+      questionIndex: item.qaIndex,
+      question: item.qa.question,
+      goldAnswer: item.qa.answer,
+      category: item.qa.category,
+      modelAnswer,
+      correct,
+      ...trapTriggered !== void 0 ? { trapTriggered } : {},
+      ...errorMsg !== void 0 ? { error: errorMsg } : {},
+      answerInputTokens: answerIn,
+      answerOutputTokens: answerOut,
+      judgeInputTokens: judgeIn,
+      judgeOutputTokens: judgeOut
+    };
+    results.push(row);
+    totalAnswerInputTokens += answerIn;
+    totalAnswerOutputTokens += answerOut;
+    totalJudgeInputTokens += judgeIn;
+    totalJudgeOutputTokens += judgeOut;
+    if (checkpointPath) {
+      await appendCheckpointRow(checkpointPath, row);
+    }
+    done++;
+    if (onProgress) onProgress(done, total);
+  };
+  const concurrencyLimit = Math.max(1, concurrency);
+  const pending = [];
+  for (const item of queue) {
+    const p = processItem(item);
+    pending.push(p);
+    if (pending.length >= concurrencyLimit) {
+      await Promise.all(pending.splice(0, concurrencyLimit));
+    }
+  }
+  if (pending.length > 0) await Promise.all(pending);
+  if (checkpointPath && checkpoint.size > 0) {
+    const raw = await (0, import_promises3.readFile)(checkpointPath, "utf-8").catch(() => "");
+    for (const line of raw.split("\n")) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      try {
+        const row = JSON.parse(trimmed);
+        const key = `${row.sampleId}:${row.questionIndex}`;
+        if (checkpoint.has(key)) {
+          results.push(row);
+          totalAnswerInputTokens += row.answerInputTokens ?? 0;
+          totalAnswerOutputTokens += row.answerOutputTokens ?? 0;
+          totalJudgeInputTokens += row.judgeInputTokens ?? 0;
+          totalJudgeOutputTokens += row.judgeOutputTokens ?? 0;
+        }
+      } catch {
+      }
+    }
+  }
+  const byCategoryMap = {};
+  for (const row of results) {
+    const cat = String(row.category);
+    if (!byCategoryMap[cat]) byCategoryMap[cat] = { correct: 0, total: 0 };
+    byCategoryMap[cat].total++;
+    if (row.correct) byCategoryMap[cat].correct++;
+  }
+  const byCategory = {};
+  for (const [cat, stats] of Object.entries(byCategoryMap)) {
+    byCategory[cat] = {
+      ...stats,
+      accuracy: stats.total > 0 ? stats.correct / stats.total : 0
+    };
+  }
+  let j14Correct = 0;
+  let j14Total = 0;
+  for (const [cat, stats] of Object.entries(byCategory)) {
+    const n = parseInt(cat);
+    if (n >= 1 && n <= 4) {
+      j14Correct += stats.correct;
+      j14Total += stats.total;
+    }
+  }
+  const cat5Stats = byCategory["5"];
+  const cat5RefusalRate = cat5Stats ? { correct: cat5Stats.correct, total: cat5Stats.total, rate: cat5Stats.accuracy } : void 0;
+  const wallClockMs = Date.now() - startTime;
+  return {
+    config: {
+      mode,
+      topK,
+      answerModelId: answerModel.modelId ?? "(unknown)",
+      judgeModelId: judgeModel.modelId ?? "(unknown)",
+      datasetSha: LOCOMO_SOURCE_SHA,
+      seed,
+      categories: [...categories].sort((a, b) => a - b),
+      samplesRun: samples.length,
+      questionsRun: results.length
+    },
+    overallJ14: {
+      correct: j14Correct,
+      total: j14Total,
+      accuracy: j14Total > 0 ? j14Correct / j14Total : 0
+    },
+    byCategory,
+    cat5RefusalRate,
+    tokens: {
+      ingestInputTokens,
+      ingestOutputTokens,
+      answerInputTokens: totalAnswerInputTokens,
+      answerOutputTokens: totalAnswerOutputTokens,
+      judgeInputTokens: totalJudgeInputTokens,
+      judgeOutputTokens: totalJudgeOutputTokens,
+      totalInputTokens: ingestInputTokens + totalAnswerInputTokens + totalJudgeInputTokens,
+      totalOutputTokens: ingestOutputTokens + totalAnswerOutputTokens + totalJudgeOutputTokens
+    },
+    wallClockMs,
+    questions: results,
+    errorCount
+  };
+}
+
+// src/locomo-render.ts
+function pct(n) {
+  return (n * 100).toFixed(1) + "%";
+}
+function fmtNum(n) {
+  return n.toLocaleString("en-US");
+}
+function estimateCost(tokens, prices) {
+  if (!prices) return "\u2014";
+  const cost = tokens.totalInputTokens / 1e6 * prices.inputPer1M + tokens.totalOutputTokens / 1e6 * prices.outputPer1M;
+  return `$${cost.toFixed(4)}`;
+}
+function renderLocomoReportMarkdown(reports, prices) {
+  const lines = [];
+  lines.push("# LoCoMo Benchmark Results");
+  lines.push("");
+  lines.push("Dataset: [LoCoMo](https://github.com/snap-research/locomo) (Snap Research) \xB7 CC BY-NC 4.0");
+  lines.push("Raw data is not redistributed. Only aggregate results are published here.");
+  lines.push("");
+  if (reports.length === 0) {
+    lines.push("_No results yet._");
+    return lines.join("\n");
+  }
+  const headers = [
+    "System / Mode",
+    "Cat1 (multi-hop)",
+    "Cat2 (temporal)",
+    "Cat3 (open-domain)",
+    "Cat4 (single-hop)",
+    "J(1\u20134) overall",
+    "Cat5 refusal rate",
+    "Tokens/query",
+    "Est. cost/run",
+    "Answer model",
+    "Judge model",
+    "topK",
+    "n-Q",
+    "Seed",
+    "Dataset SHA"
+  ];
+  lines.push("## Results");
+  lines.push("");
+  lines.push("| " + headers.join(" | ") + " |");
+  lines.push("| " + headers.map(() => "---").join(" | ") + " |");
+  for (const r of reports) {
+    const c = r.config;
+    const cat = (n) => {
+      const s = r.byCategory[String(n)];
+      if (!s) return "\u2014";
+      return `${pct(s.accuracy)} (${s.correct}/${s.total})`;
+    };
+    const cat5 = r.cat5RefusalRate ? `${pct(r.cat5RefusalRate.rate)} (${r.cat5RefusalRate.correct}/${r.cat5RefusalRate.total})` : "\u2014";
+    const totalQ = r.questions.length;
+    const tokensPerQuery = totalQ > 0 ? Math.round((r.tokens.totalInputTokens + r.tokens.totalOutputTokens) / totalQ) : 0;
+    const row = [
+      `${c.answerModelId} / ${c.mode}`,
+      cat(1),
+      cat(2),
+      cat(3),
+      cat(4),
+      `${pct(r.overallJ14.accuracy)} (${r.overallJ14.correct}/${r.overallJ14.total})`,
+      cat5,
+      fmtNum(tokensPerQuery),
+      estimateCost(r.tokens, prices),
+      c.answerModelId,
+      c.judgeModelId,
+      String(c.topK),
+      fmtNum(r.config.questionsRun),
+      String(c.seed),
+      c.datasetSha.slice(0, 8)
+    ];
+    lines.push("| " + row.join(" | ") + " |");
+  }
+  lines.push("");
+  lines.push("## Run Configuration");
+  lines.push("");
+  for (const r of reports) {
+    const c = r.config;
+    lines.push(`### ${c.answerModelId} / ${c.mode}`);
+    lines.push("");
+    lines.push(`- **Mode**: ${c.mode}`);
+    lines.push(`- **Answer model**: ${c.answerModelId}`);
+    lines.push(`- **Judge model**: ${c.judgeModelId}`);
+    lines.push(`- **topK**: ${c.topK}`);
+    lines.push(`- **Dataset SHA**: \`${c.datasetSha}\``);
+    lines.push(`- **Seed**: ${c.seed}`);
+    lines.push(`- **Categories**: ${c.categories.join(", ")}`);
+    lines.push(`- **Samples run**: ${c.samplesRun}`);
+    lines.push(`- **Questions run**: ${c.questionsRun}`);
+    lines.push(`- **Wall-clock**: ${(r.wallClockMs / 1e3).toFixed(1)}s`);
+    lines.push(`- **Errors**: ${r.errorCount}`);
+    lines.push(`- **Tokens** (in/out): ${fmtNum(r.tokens.totalInputTokens)} / ${fmtNum(r.tokens.totalOutputTokens)}`);
+    lines.push("");
+  }
+  lines.push("## Methodology Notes");
+  lines.push("");
+  lines.push("These results were produced using the Eidentic LoCoMo fair-run harness. The following rules apply:");
+  lines.push("");
+  lines.push("1. **Both speakers are treated as humans.** Turns are ingested as `[SpeakerName]: text` \u2014 never mapped to user/assistant roles.");
+  lines.push("2. **Timestamps are structural.** Each session is prefixed with a header line `Session N \u2014 <date>` and `ingestedAt` metadata carries the epoch-ms.");
+  lines.push("3. **topK \u2264 10 in memory mode.** Larger topK values trivialise retrieval quality and are not permitted.");
+  lines.push("4. **Full-context baseline is required** alongside any memory-mode result.");
+  lines.push("5. **Judge is strict**: a model answer is correct only when it contains the gold answer's specific information. Vague/topical-only answers are wrong.");
+  lines.push("6. **Category 5 (adversarial)**: correct = model declined; adversarial-trap match = wrong.");
+  lines.push("7. **Primary metric J(1\u20134)**: denominator is the number of cat 1\u20134 questions actually run (max 1540 on full dataset).");
+  lines.push("8. **Dataset license**: CC BY-NC 4.0 \u2014 raw data is not redistributed; only aggregate results are published.");
+  lines.push("");
+  lines.push("> Category mapping in locomo10.json: 1=multi-hop (282), 2=temporal (321), 3=open-domain (96), 4=single-hop (841), 5=adversarial (446).");
+  lines.push("");
+  return lines.join("\n");
+}
+
+// src/locomo-types.ts
+function resolveEvidence(sample, diaIds) {
+  const turnMap = /* @__PURE__ */ new Map();
+  for (const sess of sample.sessions) {
+    for (const turn of sess.turns) {
+      turnMap.set(turn.diaId, turn);
+    }
+  }
+  const results = [];
+  for (const id of diaIds) {
+    const t = turnMap.get(id);
+    if (t) results.push(t.text);
+  }
+  return results;
+}
+
+// src/index.ts
+init_lme_loader();
+
+// src/lme-run.ts
+var import_promises5 = require("node:fs/promises");
+var import_node_fs2 = require("node:fs");
+init_lme_loader();
+function makeRng2(seed) {
+  let s = seed >>> 0;
+  if (s === 0) s = 1;
+  return () => {
+    s ^= s << 13;
+    s ^= s >>> 17;
+    s ^= s << 5;
+    s = s >>> 0;
+    return s / 4294967296;
+  };
+}
+function seededShuffle2(arr, rng) {
+  const out = [...arr];
+  for (let i = out.length - 1; i > 0; i--) {
+    const j = Math.floor(rng() * (i + 1));
+    [out[i], out[j]] = [out[j], out[i]];
+  }
+  return out;
+}
+var DEFAULT_FULL_CONTEXT_MAX_CHARS = 48e4;
+var DECLINE_PATTERNS2 = /\bno information available\b|\bi (don'?t|do not) (know|have)\b|\bcannot (find|answer|determine|provide)\b|\bnot (mentioned|stated|provided|found|available)\b|\bunable to (find|answer|determine)\b|\bno relevant (information|data)\b/i;
+function appearsToDecline2(answer) {
+  return DECLINE_PATTERNS2.test(answer);
+}
+var ANSWER_INSTRUCTION2 = "Answer the question concisely and specifically based only on the information provided. If the provided context does not contain the answer, reply exactly: No information available";
+function buildMemoryAnswerPrompt2(snippets, question, currentDate) {
+  const context = snippets.length > 0 ? snippets.map((s, i) => `[${i + 1}] ${s}`).join("\n\n") : "(no relevant context retrieved)";
+  const dateNote = currentDate ? `
+Current date: ${currentDate}` : "";
+  return `Context from conversation history:
+${context}${dateNote}
+
+Question: ${question}
+
+${ANSWER_INSTRUCTION2}`;
+}
+function buildFullContextAnswerPrompt2(haystackText, question, currentDate) {
+  const dateNote = currentDate ? `
+Current date: ${currentDate}` : "";
+  return `Conversation history:
+${haystackText}${dateNote}
+
+Question: ${question}
+
+${ANSWER_INSTRUCTION2}`;
+}
+function buildJudgePrompt2(question, goldAnswer, modelAnswer, questionType, isAbstention) {
+  if (isAbstention) {
+    return `You are a strict judge. This question requires the model to recognize that the information is not present in the conversation history, or that the premise is flawed. A CORRECT response means the model abstained, said something equivalent to "No information available", "I don't know", or identified that the premise is wrong. An INCORRECT response means the model fabricated a specific, concrete answer.
+
+Question: ${question}
+Model answer: ${modelAnswer}
+
+Respond with JSON only: {"correct": true} if the model appropriately abstained, {"correct": false} if it fabricated a specific answer.`;
+  }
+  const temporalNote = questionType === "temporal-reasoning" ? "\nFor time/date questions: equivalent date expressions referring to the same date or duration are correct." : "";
+  return `You are a strict judge evaluating whether a model answer is correct.
+CORRECT: the model answer contains the specific information in the gold answer (exact match or clear paraphrase is fine).${temporalNote}
+WRONG: the model answer is vague, only topically related, contradicts the gold answer, or says "no information" when a specific answer exists.
+
+Question: ${question}
+Gold answer: ${goldAnswer}
+Model answer: ${modelAnswer}
+
+Respond with JSON only: {"correct": true} or {"correct": false}`;
+}
+function renderHaystack(sessions) {
+  const lines = [];
+  for (let i = 0; i < sessions.length; i++) {
+    const sess = sessions[i];
+    const label = `Session ${i + 1} \u2014 ${sess.dateTime || sess.id}`;
+    lines.push(label);
+    for (const turn of sess.turns) {
+      const roleLabel = turn.role === "user" ? "User" : "Assistant";
+      lines.push(`[${roleLabel}]: ${turn.content}`);
+    }
+    lines.push("");
+  }
+  return lines.join("\n").trim();
+}
+function renderHaystackCapped(sessions, maxChars) {
+  const full = renderHaystack(sessions);
+  if (full.length <= maxChars) return { text: full, truncated: false };
+  let kept = sessions.slice();
+  while (kept.length > 1) {
+    kept = kept.slice(1);
+    const t = renderHaystack(kept);
+    if (t.length <= maxChars) return { text: t, truncated: true };
+  }
+  return { text: renderHaystack(kept).slice(0, maxChars), truncated: true };
+}
+var EMBED_CHAR_CAP = 2e4;
+function capForEmbedding(text) {
+  return text.length <= EMBED_CHAR_CAP ? text : text.slice(0, EMBED_CHAR_CAP);
+}
+async function ingestQuestionIntoMemory(question, memory, scope) {
+  const events = [];
+  for (let i = 0; i < question.sessions.length; i++) {
+    const sess = question.sessions[i];
+    const sessLabel = `Session ${i + 1} \u2014 ${sess.dateTime || sess.id}`;
+    for (let t = 0; t < sess.turns.length; t++) {
+      const turn = sess.turns[t];
+      const roleLabel = turn.role === "user" ? "User" : "Assistant";
+      events.push({
+        id: `${question.id}:sess${i}:turn${t}`,
+        scope,
+        text: capForEmbedding(`[${sessLabel}] [${roleLabel}]: ${turn.content}`),
+        metadata: {
+          sessionId: sess.id,
+          sessionIndex: i,
+          turnRole: turn.role,
+          ingestedAt: sess.dateTimeMs || void 0
+        }
+      });
+    }
+    const sessionText = [
+      sessLabel,
+      ...sess.turns.map((t) => `[${t.role === "user" ? "User" : "Assistant"}]: ${t.content}`)
+    ].join("\n");
+    events.push({
+      id: `${question.id}:sess${i}:chunk`,
+      scope,
+      text: capForEmbedding(sessionText),
+      metadata: {
+        sessionId: sess.id,
+        sessionIndex: i,
+        ingestedAt: sess.dateTimeMs || void 0
+      }
+    });
+  }
+  await memory.ingest(events);
+}
+async function callJudge2(judgeModel, prompt) {
+  const response = await judgeModel.complete({
+    messages: [{ role: "user", content: prompt }],
+    tools: [],
+    outputSchema: {
+      type: "object",
+      properties: { correct: { type: "boolean" } },
+      required: ["correct"],
+      // OpenAI strict structured-output mode requires this to be explicit.
+      additionalProperties: false
+    }
+  });
+  let correct = false;
+  if (response.object && typeof response.object.correct === "boolean") {
+    correct = response.object.correct;
+  } else {
+    const text = response.content.filter((b) => b.type === "text").map((b) => b.text ?? "").join("").toLowerCase().trim();
+    if (/"correct"\s*:\s*true/i.test(text)) correct = true;
+    else if (/"correct"\s*:\s*false/i.test(text)) correct = false;
+    else correct = text.includes("true");
+  }
+  return {
+    correct,
+    inputTokens: response.usage?.inputTokens ?? 0,
+    outputTokens: response.usage?.outputTokens ?? 0
+  };
+}
+async function loadCheckpoint2(path) {
+  const done = /* @__PURE__ */ new Set();
+  if (!(0, import_node_fs2.existsSync)(path)) return done;
+  const raw = await (0, import_promises5.readFile)(path, "utf-8");
+  for (const line of raw.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      const row = JSON.parse(trimmed);
+      if (row.questionId) done.add(row.questionId);
+    } catch {
+    }
+  }
+  return done;
+}
+async function appendCheckpointRow2(path, row) {
+  await (0, import_promises5.appendFile)(path, JSON.stringify(row) + "\n", "utf-8");
+}
+async function runLongMemEvalBench(opts) {
+  const {
+    answerModel,
+    judgeModel,
+    mode,
+    types,
+    questionLimit,
+    seed = 42,
+    concurrency = 1,
+    onProgress,
+    checkpointPath,
+    fullContextMaxChars = DEFAULT_FULL_CONTEXT_MAX_CHARS
+  } = opts;
+  const topK = Math.min(opts.topK ?? 10, 10);
+  const rng = makeRng2(seed);
+  if (mode === "memory" && !opts.memoryFactory) {
+    throw new Error("runLongMemEvalBench: memoryFactory is required when mode='memory'");
+  }
+  let dataset;
+  if (opts.dataset) {
+    dataset = opts.dataset;
+  } else if (opts.dataPath) {
+    const { loadLongMemEval: loader } = await Promise.resolve().then(() => (init_lme_loader(), lme_loader_exports));
+    dataset = await loader(opts.dataPath);
+  } else {
+    throw new Error("runLongMemEvalBench: either dataPath or dataset must be provided");
+  }
+  let questions = dataset.questions;
+  if (types && types.length > 0) {
+    questions = questions.filter((q) => types.includes(q.type) || types.includes(q.baseType));
+  }
+  if (questionLimit !== void 0 && questionLimit < questions.length) {
+    questions = seededShuffle2(questions, makeRng2(seed)).slice(0, questionLimit);
+  }
+  const checkpoint = checkpointPath ? await loadCheckpoint2(checkpointPath) : /* @__PURE__ */ new Set();
+  const results = [];
+  let totalAnswerInputTokens = 0;
+  let totalAnswerOutputTokens = 0;
+  let totalJudgeInputTokens = 0;
+  let totalJudgeOutputTokens = 0;
+  let ingestEmbedTokens = 0;
+  let errorCount = 0;
+  let done = 0;
+  const total = questions.length - checkpoint.size;
+  const startTime = Date.now();
+  const processQuestion = async (q) => {
+    if (checkpoint.has(q.id)) return;
+    let modelAnswer = "";
+    let answerIn = 0;
+    let answerOut = 0;
+    let judgeIn = 0;
+    let judgeOut = 0;
+    let correct = false;
+    let contextTruncated = false;
+    let errorMsg;
+    try {
+      if (mode === "memory") {
+        const memory = await opts.memoryFactory(q.id);
+        const scope = { kind: "agent", agentId: `lme:${q.id}` };
+        await ingestQuestionIntoMemory(q, memory, scope);
+        const retrieved = await memory.retrieve({ text: q.question, scope, topK });
+        const snippets = retrieved.snippets.map((s) => s.text);
+        const prompt = buildMemoryAnswerPrompt2(snippets, q.question, q.questionDate);
+        const resp = await answerModel.complete({
+          messages: [{ role: "user", content: prompt }],
+          tools: []
+        });
+        const textBlocks = resp.content.filter((b) => b.type === "text");
+        modelAnswer = textBlocks.map((b) => b.text).join("").trim();
+        answerIn = resp.usage?.inputTokens ?? 0;
+        answerOut = resp.usage?.outputTokens ?? 0;
+      } else {
+        const { text, truncated } = renderHaystackCapped(q.sessions, fullContextMaxChars);
+        contextTruncated = truncated;
+        const prompt = buildFullContextAnswerPrompt2(text, q.question, q.questionDate);
+        const resp = await answerModel.complete({
+          messages: [{ role: "user", content: prompt }],
+          tools: []
+        });
+        const textBlocks = resp.content.filter((b) => b.type === "text");
+        modelAnswer = textBlocks.map((b) => b.text).join("").trim();
+        answerIn = resp.usage?.inputTokens ?? 0;
+        answerOut = resp.usage?.outputTokens ?? 0;
+      }
+      const judgePrompt = buildJudgePrompt2(
+        q.question,
+        q.answer,
+        modelAnswer,
+        q.baseType,
+        q.isAbstention
+      );
+      const judgeResult = await callJudge2(judgeModel, judgePrompt);
+      correct = judgeResult.correct;
+      judgeIn = judgeResult.inputTokens;
+      judgeOut = judgeResult.outputTokens;
+    } catch (err) {
+      errorMsg = err.message;
+      errorCount++;
+      correct = false;
+    }
+    const appearedToAbstain = appearsToDecline2(modelAnswer);
+    const row = {
+      questionId: q.id,
+      questionType: q.type,
+      isAbstention: q.isAbstention,
+      question: q.question,
+      goldAnswer: q.answer,
+      modelAnswer,
+      correct,
+      appearedToAbstain,
+      ...contextTruncated ? { contextTruncated } : {},
+      ...errorMsg !== void 0 ? { error: errorMsg } : {},
+      answerInputTokens: answerIn,
+      answerOutputTokens: answerOut,
+      judgeInputTokens: judgeIn,
+      judgeOutputTokens: judgeOut
+    };
+    results.push(row);
+    totalAnswerInputTokens += answerIn;
+    totalAnswerOutputTokens += answerOut;
+    totalJudgeInputTokens += judgeIn;
+    totalJudgeOutputTokens += judgeOut;
+    if (checkpointPath) {
+      await appendCheckpointRow2(checkpointPath, row);
+    }
+    done++;
+    if (onProgress) onProgress(done, total);
+  };
+  const concurrencyLimit = Math.max(1, concurrency);
+  const pending = [];
+  for (const q of questions) {
+    const p = processQuestion(q);
+    pending.push(p);
+    if (pending.length >= concurrencyLimit) {
+      await Promise.all(pending.splice(0, concurrencyLimit));
+    }
+  }
+  if (pending.length > 0) await Promise.all(pending);
+  if (checkpointPath && checkpoint.size > 0) {
+    const raw = await (0, import_promises5.readFile)(checkpointPath, "utf-8").catch(() => "");
+    for (const line of raw.split("\n")) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      try {
+        const row = JSON.parse(trimmed);
+        if (checkpoint.has(row.questionId)) {
+          results.push(row);
+          totalAnswerInputTokens += row.answerInputTokens ?? 0;
+          totalAnswerOutputTokens += row.answerOutputTokens ?? 0;
+          totalJudgeInputTokens += row.judgeInputTokens ?? 0;
+          totalJudgeOutputTokens += row.judgeOutputTokens ?? 0;
+        }
+      } catch {
+      }
+    }
+  }
+  const byTypeMap = {};
+  let overallCorrect = 0;
+  let overallTotal = 0;
+  let abstentionCorrect = 0;
+  let abstentionTotal = 0;
+  for (const row of results) {
+    if (row.isAbstention) {
+      abstentionTotal++;
+      if (row.correct) abstentionCorrect++;
+    } else {
+      const bt = row.questionType.endsWith("_abs") ? row.questionType.slice(0, -4) : row.questionType;
+      if (!byTypeMap[bt]) byTypeMap[bt] = { correct: 0, total: 0 };
+      overallTotal++;
+      if (row.correct) overallCorrect++;
+      byTypeMap[bt].total++;
+      if (row.correct) byTypeMap[bt].correct++;
+    }
+  }
+  const byType = {};
+  for (const [t, stats] of Object.entries(byTypeMap)) {
+    byType[t] = {
+      ...stats,
+      accuracy: stats.total > 0 ? stats.correct / stats.total : 0
+    };
+  }
+  const abstentionAccuracy = abstentionTotal > 0 ? {
+    correct: abstentionCorrect,
+    total: abstentionTotal,
+    accuracy: abstentionCorrect / abstentionTotal
+  } : void 0;
+  const wallClockMs = Date.now() - startTime;
+  const allTypes = [...new Set(questions.map((q) => q.type))].sort();
+  return {
+    config: {
+      mode,
+      topK,
+      answerModelId: answerModel.modelId ?? "(unknown)",
+      judgeModelId: judgeModel.modelId ?? "(unknown)",
+      datasetSource: LONGMEMEVAL_SOURCE,
+      seed,
+      types: allTypes,
+      questionsRun: results.length
+    },
+    overall: {
+      correct: overallCorrect,
+      total: overallTotal,
+      accuracy: overallTotal > 0 ? overallCorrect / overallTotal : 0
+    },
+    byType,
+    ...abstentionAccuracy !== void 0 ? { abstentionAccuracy } : {},
+    tokens: {
+      ingestEmbedTokens,
+      answerInputTokens: totalAnswerInputTokens,
+      answerOutputTokens: totalAnswerOutputTokens,
+      judgeInputTokens: totalJudgeInputTokens,
+      judgeOutputTokens: totalJudgeOutputTokens,
+      totalInputTokens: totalAnswerInputTokens + totalJudgeInputTokens,
+      totalOutputTokens: totalAnswerOutputTokens + totalJudgeOutputTokens
+    },
+    wallClockMs,
+    questions: results,
+    errorCount
+  };
+}
+
+// src/lme-render.ts
+var QUESTION_TYPE_LABELS = {
+  "single-session-user": "Single-session (user)",
+  "single-session-assistant": "Single-session (asst.)",
+  "single-session-preference": "Single-session (pref.)",
+  "multi-session": "Multi-session",
+  "temporal-reasoning": "Temporal reasoning",
+  "knowledge-update": "Knowledge update"
+};
+function pct2(n) {
+  return (n * 100).toFixed(1) + "%";
+}
+function fmtNum2(n) {
+  return n.toLocaleString("en-US");
+}
+function estimateCost2(tokens, prices) {
+  if (!prices) return "\u2014";
+  const cost = tokens.totalInputTokens / 1e6 * prices.inputPer1M + tokens.totalOutputTokens / 1e6 * prices.outputPer1M;
+  return `$${cost.toFixed(4)}`;
+}
+function fmtStat(s) {
+  if (!s || s.total === 0) return "\u2014";
+  return `${pct2(s.accuracy)} (${s.correct}/${s.total})`;
+}
+function renderLongMemEvalReportMarkdown(reports, prices) {
+  const lines = [];
+  lines.push("# LongMemEval Benchmark Results");
+  lines.push("");
+  lines.push(
+    "Dataset: [LongMemEval](https://github.com/xiaowu0162/LongMemEval) (Wu et al.) \xB7 MIT License"
+  );
+  lines.push("Raw data is not redistributed. Only aggregate results are published here.");
+  lines.push("");
+  if (reports.length === 0) {
+    lines.push("_No results yet._");
+    return lines.join("\n");
+  }
+  const allBaseTypes = /* @__PURE__ */ new Set();
+  for (const r of reports) {
+    for (const t of Object.keys(r.byType)) allBaseTypes.add(t);
+  }
+  const sortedTypes = [
+    "single-session-user",
+    "single-session-assistant",
+    "single-session-preference",
+    "multi-session",
+    "temporal-reasoning",
+    "knowledge-update"
+  ].filter((t) => allBaseTypes.has(t));
+  for (const t of [...allBaseTypes].sort()) {
+    if (!sortedTypes.includes(t)) sortedTypes.push(t);
+  }
+  const typeHeaders = sortedTypes.map(
+    (t) => QUESTION_TYPE_LABELS[t] ?? t
+  );
+  const headers = [
+    "System / Mode",
+    ...typeHeaders,
+    "Overall accuracy",
+    "Abstention accuracy",
+    "Tokens/query",
+    "Est. cost/run",
+    "Answer model",
+    "Judge model",
+    "topK",
+    "n-Q",
+    "Seed",
+    "Dataset provenance"
+  ];
+  lines.push("## Results");
+  lines.push("");
+  lines.push("| " + headers.join(" | ") + " |");
+  lines.push("| " + headers.map(() => "---").join(" | ") + " |");
+  for (const r of reports) {
+    const c = r.config;
+    const typeRow = sortedTypes.map((t) => fmtStat(r.byType[t]));
+    const totalQ = r.questions.length;
+    const tokensPerQuery = totalQ > 0 ? Math.round(
+      (r.tokens.totalInputTokens + r.tokens.totalOutputTokens) / totalQ
+    ) : 0;
+    const provenance = `${c.datasetSource.url.replace("https://", "")} @ ${c.datasetSource.snapshotSha.slice(0, 8)}`;
+    const row = [
+      `${c.answerModelId} / ${c.mode}`,
+      ...typeRow,
+      fmtStat(r.overall),
+      fmtStat(r.abstentionAccuracy),
+      fmtNum2(tokensPerQuery),
+      estimateCost2(r.tokens, prices),
+      c.answerModelId,
+      c.judgeModelId,
+      c.mode === "memory" ? String(c.topK) : "\u2014",
+      fmtNum2(r.config.questionsRun),
+      String(c.seed),
+      provenance
+    ];
+    lines.push("| " + row.join(" | ") + " |");
+  }
+  lines.push("");
+  lines.push("## Run Configuration");
+  lines.push("");
+  for (const r of reports) {
+    const c = r.config;
+    lines.push(`### ${c.answerModelId} / ${c.mode}`);
+    lines.push("");
+    lines.push(`- **Mode**: ${c.mode}`);
+    lines.push(`- **Answer model**: ${c.answerModelId}`);
+    lines.push(`- **Judge model**: ${c.judgeModelId}`);
+    if (c.mode === "memory") lines.push(`- **topK**: ${c.topK}`);
+    lines.push(`- **Dataset source**: ${c.datasetSource.url}`);
+    lines.push(`- **Dataset snapshot SHA**: \`${c.datasetSource.snapshotSha}\``);
+    lines.push(`- **Dataset file**: ${c.datasetSource.file}`);
+    lines.push(`- **Dataset license**: ${c.datasetSource.license}`);
+    lines.push(`- **Seed**: ${c.seed}`);
+    lines.push(`- **Types**: ${c.types.join(", ") || "all"}`);
+    lines.push(`- **Questions run**: ${c.questionsRun}`);
+    lines.push(`- **Wall-clock**: ${(r.wallClockMs / 1e3).toFixed(1)}s`);
+    lines.push(`- **Errors**: ${r.errorCount}`);
+    lines.push(
+      `- **Tokens** (in/out): ${fmtNum2(r.tokens.totalInputTokens)} / ${fmtNum2(r.tokens.totalOutputTokens)}`
+    );
+    lines.push("");
+  }
+  lines.push("## Methodology Notes");
+  lines.push("");
+  lines.push(
+    "These results were produced using the Eidentic LongMemEval fair-run harness. The following rules apply:"
+  );
+  lines.push("");
+  lines.push(
+    "1. **Per-question memory scope.** Each question has its own haystack (~50 sessions on average). A fresh Memory instance is created per question; no cross-question contamination."
+  );
+  lines.push(
+    "2. **Dual-granularity ingest.** Each turn is ingested with its session date in the text (temporally anchored). An additional session-level chunk entry captures multi-turn context."
+  );
+  lines.push(
+    "3. **Current date in prompt.** The `question_date` is passed to the answer prompt so temporal questions can reason about recency."
+  );
+  lines.push(
+    "4. **topK \u2264 10 in memory mode.** Larger topK values trivialise retrieval quality and are not permitted."
+  );
+  lines.push(
+    "5. **Full-context baseline is required** alongside any memory-mode result."
+  );
+  lines.push(
+    "6. **Judge is strict**: a model answer is correct only when it contains the gold answer's specific information. Vague/topical-only answers are wrong. Equivalent date expressions for the same date/duration are correct (temporal-reasoning type)."
+  );
+  lines.push(
+    "7. **Abstention questions** (not present in longmemeval_s.json standard split): correct = model declined / said no-info / identified a flawed premise; fabricating a specific answer = wrong. Abstention accuracy is reported separately and not folded into overall accuracy."
+  );
+  lines.push(
+    "8. **Dataset license**: MIT \u2014 raw data is not redistributed; only aggregate results are published."
+  );
+  lines.push("");
+  lines.push("> Per-type question counts in longmemeval_s.json (500 total):");
+  lines.push("> single-session-user 70, single-session-assistant 56, single-session-preference 30,");
+  lines.push("> multi-session 133, temporal-reasoning 133, knowledge-update 78.");
+  lines.push("> No abstention variants in the standard _s split.");
+  lines.push("");
+  return lines.join("\n");
+}
+
 // src/write-quality.ts
 var CONTRADICTION_FIXTURES = [
   {
@@ -822,7 +2147,7 @@ async function runTemporalBench(memory, dataset, opts = {}) {
 }
 
 // src/datasets/temporal.ts
-function makeRng(seed) {
+function makeRng3(seed) {
   let s = seed >>> 0;
   if (s === 0) s = 1;
   return () => {
@@ -905,7 +2230,7 @@ function syntheticTemporalDataset(opts = {}) {
   const entityCount = opts.entityCount ?? 4;
   const seed = opts.seed ?? 42;
   const changesPerProperty = opts.changesPerProperty ?? 3;
-  const rng = makeRng(seed);
+  const rng = makeRng3(seed);
   const entities = [];
   const asserts = [];
   const questions = [];
@@ -992,12 +2317,22 @@ function syntheticTemporalDataset(opts = {}) {
 0 && (module.exports = {
   CONTRADICTION_FIXTURES,
   JUNK_STREAM_FIXTURES,
+  LOCOMO_SOURCE_SHA,
+  LONGMEMEVAL_SOURCE,
   factRecall,
   loadLoCoMo,
+  loadLoCoMoLegacy,
   loadLongMemEval,
+  loadLongMemEvalLegacy,
   normalizeText,
   normalizedIncludes,
+  parseLmeDateTimeString,
   recallAtK,
+  renderLocomoReportMarkdown,
+  renderLongMemEvalReportMarkdown,
+  resolveEvidence,
+  runLocomoBench,
+  runLongMemEvalBench,
   runMemoryBench,
   runTemporalBench,
   runWriteQualityBench,