@eidentic/bench 0.1.0 → 0.1.1

package/README.md

@@ -103,10 +103,47 @@ const report = await runWriteQualityBench(memory, {
 });
 ```
 
-### Real dataset loaders (gated)
+### LoCoMo fair-run harness
+
+The LoCoMo harness is a rigorous end-to-end benchmark: ingest → retrieve → answer → LLM judge.
+It enforces strict fair-run rules (both speakers as humans, structural timestamps, topK ≤ 10,
+mandatory full-context baseline). See [BASELINES.md](./BASELINES.md) for the full methodology.
+
+```ts
+import { loadLoCoMo, runLocomoBench, renderLocomoReportMarkdown } from "@eidentic/bench";
+import { Memory } from "@eidentic/memory";
+import { InMemoryStore, InMemoryVectorStore, FakeEmbedder } from "@eidentic/types/testing";
+
+// Download data/locomo10.json first — CC BY-NC 4.0, do not commit
+const dataset = await loadLoCoMo("data/locomo10.json");
+
+const report = await runLocomoBench({
+  dataPath: "data/locomo10.json",
+  dataset,
+  answerModel: myModel,
+  judgeModel: myJudgeModel,
+  mode: "full-context",  // or "memory" with memoryFactory
+  categories: [1, 2, 3, 4, 5],
+  memoryFactory: (sampleId) => new Memory({
+    store: new InMemoryStore(),
+    vector: new InMemoryVectorStore(),
+    embedder: new FakeEmbedder(32),
+  }),
+});
+
+console.log("J(1-4):", report.overallJ14.accuracy);
+console.log(renderLocomoReportMarkdown([report]));
+```
+
+Run the full pilot:
+```bash
+ANTHROPIC_API_KEY=... pnpm --filter eidentic-examples bench:locomo -- --mode full-context --samples 2
+```
+
+### LongMemEval loader (retrieval benchmark, gated)
 
 ```ts
-import { loadLongMemEval, loadLoCoMo } from "@eidentic/bench";
+import { loadLongMemEval } from "@eidentic/bench";
 
 const dataset = await loadLongMemEval("/path/to/longmemeval.json");
 const report = await runMemoryBench(() => myMemory(), dataset, { topK: 10 });

package/dist/chunk-PVIWNXCY.js

@@ -0,0 +1,99 @@
+// src/locomo-loader.ts
+import { readFile, stat } from "node:fs/promises";
+var LOCOMO_SOURCE_SHA = "3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376";
+var DEFAULT_MAX_BYTES = 256 * 1024 * 1024;
+async function assertFileSize(filePath, maxBytes) {
+  let fileSize;
+  try {
+    const s = await stat(filePath);
+    fileSize = s.size;
+  } catch (err) {
+    throw new Error(
+      `bench loader: cannot stat file "${filePath}": ${err.message}`
+    );
+  }
+  if (fileSize > maxBytes) {
+    const mb = (fileSize / (1024 * 1024)).toFixed(1);
+    const capMb = (maxBytes / (1024 * 1024)).toFixed(0);
+    throw new Error(
+      `bench loader: file "${filePath}" is ${mb} MiB, which exceeds the ${capMb} MiB cap. Pass a larger maxBytes option if this is intentional.`
+    );
+  }
+}
+function parseLocomoDateTime(raw) {
+  const cleaned = raw.replace(/\s*on\s+/, " ").trim();
+  const ms = Date.parse(cleaned);
+  if (Number.isNaN(ms)) {
+    const match = /^(\d{1,2}:\d{2}\s*(?:am|pm))\s+(.+)$/i.exec(cleaned);
+    if (match) {
+      const ms2 = Date.parse(`${match[2]} ${match[1]}`);
+      return Number.isNaN(ms2) ? 0 : ms2;
+    }
+    return 0;
+  }
+  return ms;
+}
+function parseSessions(conv) {
+  const indices = [];
+  for (const key of Object.keys(conv)) {
+    const m = /^session_(\d+)$/.exec(key);
+    if (m) indices.push(parseInt(m[1], 10));
+  }
+  indices.sort((a, b) => a - b);
+  const sessions = [];
+  for (const idx of indices) {
+    const turnsRaw = conv[`session_${idx}`];
+    if (!Array.isArray(turnsRaw)) continue;
+    const dateTimeRaw = typeof conv[`session_${idx}_date_time`] === "string" ? conv[`session_${idx}_date_time`] : "";
+    const turns = turnsRaw.map((t) => ({
+      speaker: t.speaker ?? "",
+      diaId: t.dia_id ?? "",
+      text: t.text ?? ""
+    }));
+    sessions.push({
+      index: idx,
+      dateTime: dateTimeRaw,
+      dateTimeMs: dateTimeRaw ? parseLocomoDateTime(dateTimeRaw) : 0,
+      turns
+    });
+  }
+  return sessions;
+}
+async function loadLoCoMo(jsonPath, opts) {
+  await assertFileSize(jsonPath, opts?.maxBytes ?? DEFAULT_MAX_BYTES);
+  const raw = JSON.parse(await readFile(jsonPath, "utf-8"));
+  if (!Array.isArray(raw)) {
+    throw new Error(
+      `bench loader: expected the LoCoMo JSON root to be an array, but got ${typeof raw}. Did you pass the correct file?`
+    );
+  }
+  const rawSamples = raw;
+  const samples = rawSamples.map((s, i) => {
+    const sampleId = s.sample_id ?? String(i);
+    const conv = s.conversation ?? {};
+    const sessions = parseSessions(conv);
+    const qa = (s.qa ?? []).map((q) => {
+      const answer = q.answer !== void 0 ? String(q.answer) : void 0;
+      return {
+        question: q.question ?? "",
+        answer,
+        category: q.category,
+        evidence: Array.isArray(q.evidence) ? q.evidence : [],
+        adversarialAnswer: q.adversarial_answer
+      };
+    });
+    return {
+      sampleId,
+      speakerA: String(conv.speaker_a ?? ""),
+      speakerB: String(conv.speaker_b ?? ""),
+      sessions,
+      qa
+    };
+  });
+  return { samples };
+}
+
+export {
+  LOCOMO_SOURCE_SHA,
+  loadLoCoMo
+};