@eidentic/bench 0.1.0 → 0.1.2

package/README.md

@@ -103,10 +103,47 @@ const report = await runWriteQualityBench(memory, {
 });
 ```
 
-### Real dataset loaders (gated)
+### LoCoMo fair-run harness
+
+The LoCoMo harness is a rigorous end-to-end benchmark: ingest → retrieve → answer → LLM judge.
+It enforces strict fair-run rules (both speakers as humans, structural timestamps, topK ≤ 10,
+mandatory full-context baseline). See [BASELINES.md](./BASELINES.md) for the full methodology.
+
+```ts
+import { loadLoCoMo, runLocomoBench, renderLocomoReportMarkdown } from "@eidentic/bench";
+import { Memory } from "@eidentic/memory";
+import { InMemoryStore, InMemoryVectorStore, FakeEmbedder } from "@eidentic/types/testing";
+
+// Download data/locomo10.json first — CC BY-NC 4.0, do not commit
+const dataset = await loadLoCoMo("data/locomo10.json");
+
+const report = await runLocomoBench({
+  dataPath: "data/locomo10.json",
+  dataset,
+  answerModel: myModel,
+  judgeModel: myJudgeModel,
+  mode: "full-context",  // or "memory" with memoryFactory
+  categories: [1, 2, 3, 4, 5],
+  memoryFactory: (sampleId) => new Memory({
+    store: new InMemoryStore(),
+    vector: new InMemoryVectorStore(),
+    embedder: new FakeEmbedder(32),
+  }),
+});
+
+console.log("J(1-4):", report.overallJ14.accuracy);
+console.log(renderLocomoReportMarkdown([report]));
+```
+
+Run the full pilot:
+```bash
+ANTHROPIC_API_KEY=... pnpm --filter eidentic-examples bench:locomo -- --mode full-context --samples 2
+```
+
+### LongMemEval loader (retrieval benchmark, gated)
 
 ```ts
-import { loadLongMemEval, loadLoCoMo } from "@eidentic/bench";
+import { loadLongMemEval } from "@eidentic/bench";
 
 const dataset = await loadLongMemEval("/path/to/longmemeval.json");
 const report = await runMemoryBench(() => myMemory(), dataset, { topK: 10 });

package/dist/chunk-KOMVTEKE.js

@@ -0,0 +1,98 @@
+// src/lme-loader.ts
+import { readFile, stat } from "node:fs/promises";
+var LONGMEMEVAL_SOURCE = {
+  url: "https://huggingface.co/datasets/xiaowu0162/longmemeval",
+  snapshotSha: "2ec2a557f339b6c0369619b1ed5793734cc87533",
+  file: "longmemeval_s",
+  license: "MIT"
+};
+var DEFAULT_MAX_BYTES = 512 * 1024 * 1024;
+async function assertFileSize(filePath, maxBytes) {
+  let fileSize;
+  try {
+    const s = await stat(filePath);
+    fileSize = s.size;
+  } catch (err) {
+    throw new Error(
+      `bench loader: cannot stat file "${filePath}": ${err.message}`
+    );
+  }
+  if (fileSize > maxBytes) {
+    const mb = (fileSize / (1024 * 1024)).toFixed(1);
+    const capMb = (maxBytes / (1024 * 1024)).toFixed(0);
+    throw new Error(
+      `bench loader: file "${filePath}" is ${mb} MiB, which exceeds the ${capMb} MiB cap. Pass a larger maxBytes option if this is intentional.`
+    );
+  }
+}
+function parseLmeDateTimeString(raw) {
+  if (!raw) return 0;
+  const cleaned = raw.replace(/\s*\([A-Za-z]+\)\s*/, " ").trim();
+  const iso = cleaned.replace(/^(\d{4})\/(\d{2})\/(\d{2})/, "$1-$2-$3");
+  const ms = Date.parse(iso);
+  if (Number.isNaN(ms)) return 0;
+  return ms;
+}
+function extractBaseType(rawType) {
+  if (rawType.endsWith("_abs")) {
+    return rawType.slice(0, -4);
+  }
+  return rawType;
+}
+function parseSession(id, dateTime, rawTurns) {
+  const turns = rawTurns.map((t) => ({
+    role: t.role === "assistant" ? "assistant" : "user",
+    content: t.content ?? "",
+    hasAnswer: t.has_answer === true
+  }));
+  return {
+    id,
+    dateTime,
+    dateTimeMs: parseLmeDateTimeString(dateTime),
+    turns
+  };
+}
+async function loadLongMemEval(jsonPath, opts) {
+  await assertFileSize(jsonPath, opts?.maxBytes ?? DEFAULT_MAX_BYTES);
+  const raw = JSON.parse(await readFile(jsonPath, "utf-8"));
+  if (!Array.isArray(raw)) {
+    throw new Error(
+      `bench loader: expected the LongMemEval JSON root to be an array, but got ${typeof raw}. Did you pass the correct file?`
+    );
+  }
+  const rawQuestions = raw;
+  const questions = rawQuestions.map((q, i) => {
+    const id = q.question_id ?? String(i);
+    const rawType = q.question_type ?? "single-session-user";
+    const baseType = extractBaseType(rawType);
+    const isAbstention = rawType.endsWith("_abs");
+    const rawSessions = Array.isArray(q.haystack_sessions) ? q.haystack_sessions : [];
+    const dates = Array.isArray(q.haystack_dates) ? q.haystack_dates : [];
+    const sessionIds = Array.isArray(q.haystack_session_ids) ? q.haystack_session_ids : [];
+    const sessions = rawSessions.map((turns, idx) => {
+      const sessId = sessionIds[idx] ?? `sess-${idx}`;
+      const dateTime = dates[idx] ?? "";
+      return parseSession(sessId, dateTime, Array.isArray(turns) ? turns : []);
+    });
+    sessions.sort((a, b) => a.dateTimeMs - b.dateTimeMs);
+    return {
+      id,
+      type: rawType,
+      baseType,
+      isAbstention,
+      question: q.question ?? "",
+      answer: q.answer ?? "",
+      questionDate: q.question_date ?? "",
+      questionDateMs: parseLmeDateTimeString(q.question_date ?? ""),
+      sessions,
+      answerSessionIds: Array.isArray(q.answer_session_ids) ? q.answer_session_ids : []
+    };
+  });
+  return { questions };
+}
+
+export {
+  LONGMEMEVAL_SOURCE,
+  parseLmeDateTimeString,
+  loadLongMemEval
+};

package/dist/chunk-PVIWNXCY.js

@@ -0,0 +1,99 @@
+// src/locomo-loader.ts
+import { readFile, stat } from "node:fs/promises";
+var LOCOMO_SOURCE_SHA = "3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376";
+var DEFAULT_MAX_BYTES = 256 * 1024 * 1024;
+async function assertFileSize(filePath, maxBytes) {
+  let fileSize;
+  try {
+    const s = await stat(filePath);
+    fileSize = s.size;
+  } catch (err) {
+    throw new Error(
+      `bench loader: cannot stat file "${filePath}": ${err.message}`
+    );
+  }
+  if (fileSize > maxBytes) {
+    const mb = (fileSize / (1024 * 1024)).toFixed(1);
+    const capMb = (maxBytes / (1024 * 1024)).toFixed(0);
+    throw new Error(
+      `bench loader: file "${filePath}" is ${mb} MiB, which exceeds the ${capMb} MiB cap. Pass a larger maxBytes option if this is intentional.`
+    );
+  }
+}
+function parseLocomoDateTime(raw) {
+  const cleaned = raw.replace(/\s*on\s+/, " ").trim();
+  const ms = Date.parse(cleaned);
+  if (Number.isNaN(ms)) {
+    const match = /^(\d{1,2}:\d{2}\s*(?:am|pm))\s+(.+)$/i.exec(cleaned);
+    if (match) {
+      const ms2 = Date.parse(`${match[2]} ${match[1]}`);
+      return Number.isNaN(ms2) ? 0 : ms2;
+    }
+    return 0;
+  }
+  return ms;
+}
+function parseSessions(conv) {
+  const indices = [];
+  for (const key of Object.keys(conv)) {
+    const m = /^session_(\d+)$/.exec(key);
+    if (m) indices.push(parseInt(m[1], 10));
+  }
+  indices.sort((a, b) => a - b);
+  const sessions = [];
+  for (const idx of indices) {
+    const turnsRaw = conv[`session_${idx}`];
+    if (!Array.isArray(turnsRaw)) continue;
+    const dateTimeRaw = typeof conv[`session_${idx}_date_time`] === "string" ? conv[`session_${idx}_date_time`] : "";
+    const turns = turnsRaw.map((t) => ({
+      speaker: t.speaker ?? "",
+      diaId: t.dia_id ?? "",
+      text: t.text ?? ""
+    }));
+    sessions.push({
+      index: idx,
+      dateTime: dateTimeRaw,
+      dateTimeMs: dateTimeRaw ? parseLocomoDateTime(dateTimeRaw) : 0,
+      turns
+    });
+  }
+  return sessions;
+}
+async function loadLoCoMo(jsonPath, opts) {
+  await assertFileSize(jsonPath, opts?.maxBytes ?? DEFAULT_MAX_BYTES);
+  const raw = JSON.parse(await readFile(jsonPath, "utf-8"));
+  if (!Array.isArray(raw)) {
+    throw new Error(
+      `bench loader: expected the LoCoMo JSON root to be an array, but got ${typeof raw}. Did you pass the correct file?`
+    );
+  }
+  const rawSamples = raw;
+  const samples = rawSamples.map((s, i) => {
+    const sampleId = s.sample_id ?? String(i);
+    const conv = s.conversation ?? {};
+    const sessions = parseSessions(conv);
+    const qa = (s.qa ?? []).map((q) => {
+      const answer = q.answer !== void 0 ? String(q.answer) : void 0;
+      return {
+        question: q.question ?? "",
+        answer,
+        category: q.category,
+        evidence: Array.isArray(q.evidence) ? q.evidence : [],
+        adversarialAnswer: q.adversarial_answer
+      };
+    });
+    return {
+      sampleId,
+      speakerA: String(conv.speaker_a ?? ""),
+      speakerB: String(conv.speaker_b ?? ""),
+      sessions,
+      qa
+    };
+  });
+  return { samples };
+}
+
+export {
+  LOCOMO_SOURCE_SHA,
+  loadLoCoMo
+};