@graphorin/evals 0.5.0

package/dist/loaders/locomo.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"locomo.js","names":["parsed: unknown","out: Case<MemoryEvalInput, string>[]","indices: number[]","sessions: MemoryEvalSession[]","turns: MemoryEvalTurn[]","role: 'user' | 'assistant'"],"sources":["../../src/loaders/locomo.ts"],"sourcesContent":["/**\n * Real LOCOMO dataset loader (arXiv:2402.17753 —\n * https://snap-research.github.io/locomo/). LOCOMO ships as a JSON\n * **array** of samples; each sample bundles one multi-session\n * conversation with many QA pairs over it:\n *\n * ```jsonc\n * {\n *   \"sample_id\": \"conv-26\",\n *   \"conversation\": {\n *     \"speaker_a\": \"Caroline\",\n *     \"speaker_b\": \"Melanie\",\n *     \"session_1_date_time\": \"1:56 pm on 8 May, 2023\",\n *     \"session_1\": [ { \"speaker\": \"Caroline\", \"dia_id\": \"D1:1\", \"text\": \"...\" }, ... ],\n *     \"session_2_date_time\": \"...\",\n *     \"session_2\": [ ... ]\n *   },\n *   \"qa\": [\n *     { \"question\": \"...\", \"answer\": \"...\", \"evidence\": [\"D1:1\"], \"category\": 1 },\n *     { \"question\": \"...\", \"category\": 5, \"adversarial_answer\": \"Not mentioned ...\" }\n *   ]\n * }\n * ```\n *\n * Each QA pair becomes one {@link MemoryEvalInput} case that shares the\n * sample's sessions. LOCOMO categories map to abilities as:\n * `1`→multi-session (multi-hop), `2`→temporal, `5`→abstention\n * (adversarial), everything else (`3` open-domain, `4` single-hop)\n * →info-extraction.\n *\n * @packageDocumentation\n */\n\nimport { readFile } from 'node:fs/promises';\n\nimport type { Case, Dataset } from '@graphorin/observability/eval';\n\nimport type {\n  MemoryEvalAbility,\n  MemoryEvalInput,\n  MemoryEvalSession,\n  MemoryEvalTurn,\n} from './memory-eval.js';\n\n/** @stable */\nexport interface LoadLocomoOptions {\n  /** Local path to the LOCOMO JSON (under `benchmarks/.datasets/`). */\n  readonly path: string;\n  /** Optional dataset name surfaced in `Dataset.metadata.name`. */\n  readonly name?: string;\n  /** Optional description surfaced in `Dataset.metadata.description`. */\n  readonly description?: string;\n}\n\n/**\n * Read a LOCOMO JSON file and return a fully-materialised\n * {@link Dataset} — one case per QA pair, scored against the reference\n * answer string (LLM-judge \"J\").\n *\n * @stable\n */\nexport async function loadLocomoDataset(\n  options: LoadLocomoOptions,\n): Promise<Dataset<MemoryEvalInput, string>> {\n  const text = await readFile(options.path, 'utf8');\n  const cases = parseLocomo(text);\n  return {\n    cases,\n    metadata: {\n      name: options.name ?? 'locomo',\n      ...(options.description !== undefined ? { description: options.description } : {}),\n      createdAt: new Date(),\n    },\n  };\n}\n\n/**\n * Pure parser. Exported so tests can exercise the mapping without\n * touching the filesystem.\n *\n * @stable\n */\nexport function parseLocomo(text: string): ReadonlyArray<Case<MemoryEvalInput, string>> {\n  const parsed: unknown = JSON.parse(text);\n  if (!Array.isArray(parsed)) {\n    throw new Error('[graphorin/evals] LOCOMO dataset must be a JSON array of samples.');\n  }\n  const samples = parsed as ReadonlyArray<unknown>;\n  const out: Case<MemoryEvalInput, string>[] = [];\n  for (let i = 0; i < samples.length; i++) {\n    const sample = samples[i];\n    if (sample === null || typeof sample !== 'object') {\n      throw new Error(`[graphorin/evals] LOCOMO sample ${i} is not an object.`);\n    }\n    appendSampleCases(sample as Record<string, unknown>, i, out);\n  }\n  return out;\n}\n\nfunction appendSampleCases(\n  sample: Record<string, unknown>,\n  index: number,\n  out: Case<MemoryEvalInput, string>[],\n): void {\n  const sampleId = asString(sample.sample_id) ?? `locomo-${index}`;\n  const conversation = isRecord(sample.conversation) ? sample.conversation : {};\n  const sessions = extractSessions(conversation);\n  const qa = asArray(sample.qa);\n  for (let q = 0; q < qa.length; q++) {\n    const item = qa[q];\n    if (!isRecord(item)) continue;\n    const question = asString(item.question);\n    if (question === undefined) continue;\n    const category = typeof item.category === 'number' ? item.category : undefined;\n    const ability = mapCategory(category);\n    const answer = asString(item.answer) ?? asString(item.adversarial_answer) ?? '';\n    const evidence = asArray(item.evidence).filter((e): e is string => typeof e === 'string');\n    out.push({\n      id: `${sampleId}-q${q}`,\n      input: { haystackSessions: sessions, question, ability },\n      expected: answer,\n      metadata: {\n        datasetName: 'locomo',\n        sampleId,\n        ability,\n        ...(category !== undefined ? { category } : {}),\n        evidence,\n      },\n    });\n  }\n}\n\nfunction extractSessions(conversation: Record<string, unknown>): MemoryEvalSession[] {\n  const speakerA = asString(conversation.speaker_a);\n  const indices: number[] = [];\n  for (const key of Object.keys(conversation)) {\n    const m = /^session_(\\d+)$/.exec(key);\n    if (m?.[1] !== undefined) indices.push(Number.parseInt(m[1], 10));\n  }\n  indices.sort((a, b) => a - b);\n  const sessions: MemoryEvalSession[] = [];\n  for (const n of indices) {\n    const raw = asArray(conversation[`session_${n}`]);\n    const date = asString(conversation[`session_${n}_date_time`]);\n    const turns: MemoryEvalTurn[] = [];\n    for (const t of raw) {\n      if (!isRecord(t)) continue;\n      const content = asString(t.text) ?? asString(t.content);\n      if (content === undefined) continue;\n      const speaker = asString(t.speaker);\n      const role: 'user' | 'assistant' =\n        speakerA !== undefined && speaker !== undefined && speaker !== speakerA\n          ? 'assistant'\n          : 'user';\n      turns.push({ role, content, ...(date !== undefined ? { timestamp: date } : {}) });\n    }\n    sessions.push({ id: `session_${n}`, turns });\n  }\n  return sessions;\n}\n\n/** Maps a LOCOMO numeric `category` onto a {@link MemoryEvalAbility}. */\nfunction mapCategory(category: number | undefined): MemoryEvalAbility {\n  switch (category) {\n    case 1:\n      return 'multi-session';\n    case 2:\n      return 'temporal';\n    case 5:\n      return 'abstention';\n    default:\n      return 'info-extraction';\n  }\n}\n\nfunction asString(value: unknown): string | undefined {\n  return typeof value === 'string' ? value : undefined;\n}\n\nfunction asArray(value: unknown): ReadonlyArray<unknown> {\n  return Array.isArray(value) ? (value as ReadonlyArray<unknown>) : [];\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n  return value !== null && typeof value === 'object' && !Array.isArray(value);\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6DA,eAAsB,kBACpB,SAC2C;AAG3C,QAAO;EACL,OAFY,YADD,MAAM,SAAS,QAAQ,MAAM,OAAO,CAClB;EAG7B,UAAU;GACR,MAAM,QAAQ,QAAQ;GACtB,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;GACjF,2BAAW,IAAI,MAAM;GACtB;EACF;;;;;;;;AASH,SAAgB,YAAY,MAA4D;CACtF,MAAMA,SAAkB,KAAK,MAAM,KAAK;AACxC,KAAI,CAAC,MAAM,QAAQ,OAAO,CACxB,OAAM,IAAI,MAAM,oEAAoE;CAEtF,MAAM,UAAU;CAChB,MAAMC,MAAuC,EAAE;AAC/C,MAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;EACvC,MAAM,SAAS,QAAQ;AACvB,MAAI,WAAW,QAAQ,OAAO,WAAW,SACvC,OAAM,IAAI,MAAM,mCAAmC,EAAE,oBAAoB;AAE3E,oBAAkB,QAAmC,GAAG,IAAI;;AAE9D,QAAO;;AAGT,SAAS,kBACP,QACA,OACA,KACM;CACN,MAAM,WAAW,SAAS,OAAO,UAAU,IAAI,UAAU;CAEzD,MAAM,WAAW,gBADI,SAAS,OAAO,aAAa,GAAG,OAAO,eAAe,EAAE,CAC/B;CAC9C,MAAM,KAAK,QAAQ,OAAO,GAAG;AAC7B,MAAK,IAAI,IAAI,GAAG,IAAI,GAAG,QAAQ,KAAK;EAClC,MAAM,OAAO,GAAG;AAChB,MAAI,CAAC,SAAS,KAAK,CAAE;EACrB,MAAM,WAAW,SAAS,KAAK,SAAS;AACxC,MAAI,aAAa,OAAW;EAC5B,MAAM,WAAW,OAAO,KAAK,aAAa,WAAW,KAAK,WAAW;EACrE,MAAM,UAAU,YAAY,SAAS;EACrC,MAAM,SAAS,SAAS,KAAK,OAAO,IAAI,SAAS,KAAK,mBAAmB,IAAI;EAC7E,MAAM,WAAW,QAAQ,KAAK,SAAS,CAAC,QAAQ,MAAmB,OAAO,MAAM,SAAS;AACzF,MAAI,KAAK;GACP,IAAI,GAAG,SAAS,IAAI;GACpB,OAAO;IAAE,kBAAkB;IAAU;IAAU;IAAS;GACxD,UAAU;GACV,UAAU;IACR,aAAa;IACb;IACA;IACA,GAAI,aAAa,SAAY,EAAE,UAAU,GAAG,EAAE;IAC9C;IACD;GACF,CAAC;;;AAIN,SAAS,gBAAgB,cAA4D;CACnF,MAAM,WAAW,SAAS,aAAa,UAAU;CACjD,MAAMC,UAAoB,EAAE;AAC5B,MAAK,MAAM,OAAO,OAAO,KAAK,aAAa,EAAE;EAC3C,MAAM,IAAI,kBAAkB,KAAK,IAAI;AACrC,MAAI,IAAI,OAAO,OAAW,SAAQ,KAAK,OAAO,SAAS,EAAE,IAAI,GAAG,CAAC;;AAEnE,SAAQ,MAAM,GAAG,MAAM,IAAI,EAAE;CAC7B,MAAMC,WAAgC,EAAE;AACxC,MAAK,MAAM,KAAK,SAAS;EACvB,MAAM,MAAM,QAAQ,aAAa,WAAW,KAAK;EACjD,MAAM,OAAO,SAAS,aAAa,WAAW,EAAE,aAAa;EAC7D,MAAMC,QAA0B,EAAE;AAClC,OAAK,MAAM,KAAK,KAAK;AACnB,OAAI,CAAC,SAAS,EAAE,CAAE;GAClB,MAAM,UAAU,SAAS,EAAE,KAAK,IAAI,SAAS,EAAE,QAAQ;AACvD,OAAI,YAAY,OAAW;GAC3B,MAAM,UAAU,SAAS,EAAE,QAAQ;GACnC,MAAMC,OACJ,aAAa,UAAa,YAAY,UAAa,YAAY,WAC3D,cACA;AACN,SAAM,KAAK;IAAE;IAAM;IAAS,GAAI,SAAS,SAAY,EAAE,WAAW,MAAM,GAAG,EAAE;IAAG,CAAC;;AAEnF,WAAS,KAAK;GAAE,IAAI,WAAW;GAAK;GAAO,CAAC;;AAE9C,QAAO;;;AAIT,SAAS,YAAY,UAAiD;AACpE,SAAQ,UAAR;EACE,KAAK,EACH,QAAO;EACT,KAAK,EACH,QAAO;EACT,KAAK,EACH,QAAO;EACT,QACE,QAAO;;;AAIb,SAAS,SAAS,OAAoC;AACpD,QAAO,OAAO,UAAU,WAAW,QAAQ;;AAG7C,SAAS,QAAQ,OAAwC;AACvD,QAAO,MAAM,QAAQ,MAAM,GAAI,QAAmC,EAAE;;AAGtE,SAAS,SAAS,OAAkD;AAClE,QAAO,UAAU,QAAQ,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,MAAM"}

package/dist/loaders/longmemeval.d.ts

@@ -0,0 +1,36 @@
+import { MemoryEvalAbility, MemoryEvalInput } from "./memory-eval.js";
+import { Case, Dataset } from "@graphorin/observability/eval";
+
+//#region src/loaders/longmemeval.d.ts
+
+/** @stable */
+interface LoadLongMemEvalOptions {
+  /** Local path to the dataset JSON (under `benchmarks/.datasets/`). */
+  readonly path: string;
+  /** Which release this file is. `'S'` (~115 sessions) is the default target. */
+  readonly variant?: 'S' | 'M';
+  /** When set, keep only cases whose mapped ability is in this list. */
+  readonly abilities?: ReadonlyArray<MemoryEvalAbility>;
+  /** Optional dataset name surfaced in `Dataset.metadata.name`. */
+  readonly name?: string;
+  /** Optional description surfaced in `Dataset.metadata.description`. */
+  readonly description?: string;
+}
+/**
+ * Read a LongMemEval JSON file and return a fully-materialised
+ * {@link Dataset} of {@link MemoryEvalInput} cases scored against the
+ * reference `answer` string.
+ *
+ * @stable
+ */
+declare function loadLongMemEvalDataset(options: LoadLongMemEvalOptions): Promise<Dataset<MemoryEvalInput, string>>;
+/**
+ * Pure parser. Exported so tests can exercise the mapping without
+ * touching the filesystem.
+ *
+ * @stable
+ */
+declare function parseLongMemEval(text: string, abilities?: ReadonlyArray<MemoryEvalAbility>): ReadonlyArray<Case<MemoryEvalInput, string>>;
+//#endregion
+export { LoadLongMemEvalOptions, loadLongMemEvalDataset, parseLongMemEval };
+//# sourceMappingURL=longmemeval.d.ts.map

package/dist/loaders/longmemeval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"longmemeval.d.ts","names":[],"sources":["../../src/loaders/longmemeval.ts"],"sourcesContent":[],"mappings":";;;;;;UAwCiB,sBAAA;;;;;;uBAMM,cAAc;;;;;;;;;;;;;iBAcf,sBAAA,UACX,yBACR,QAAQ,QAAQ;;;;;;;iBAoBH,gBAAA,2BAEF,cAAc,qBACzB,cAAc,KAAK"}

package/dist/loaders/longmemeval.js

@@ -0,0 +1,135 @@
+import { readFile } from "node:fs/promises";
+
+//#region src/loaders/longmemeval.ts
+/**
+* LongMemEval dataset loader (ICLR 2025, arXiv:2410.10813 —
+* https://github.com/xiaowu0162/LongMemEval). LongMemEval ships as a
+* JSON **array** of question objects:
+*
+* ```jsonc
+* {
+*   "question_id": "abc_abs",          // `_abs` suffix marks abstention
+*   "question_type": "temporal-reasoning",
+*   "question": "Where did I live in 2021?",
+*   "answer": "Berlin",
+*   "question_date": "2023/05/20 (Sat) 14:00",
+*   "haystack_session_ids": ["s1", "s2"],
+*   "haystack_dates": ["2021/03/01 ...", "2023/01/05 ..."],
+*   "haystack_sessions": [
+*     [ { "role": "user", "content": "..." }, { "role": "assistant", "content": "...", "has_answer": true } ],
+*     [ ... ]
+*   ]
+* }
+* ```
+*
+* The three parallel arrays (`haystack_session_ids` / `haystack_dates`
+* / `haystack_sessions`) are zipped by index; one date applies to every
+* turn in its session.
+*
+* @packageDocumentation
+*/
+/**
+* Read a LongMemEval JSON file and return a fully-materialised
+* {@link Dataset} of {@link MemoryEvalInput} cases scored against the
+* reference `answer` string.
+*
+* @stable
+*/
+async function loadLongMemEvalDataset(options) {
+	const cases = parseLongMemEval(await readFile(options.path, "utf8"), options.abilities);
+	const variantSuffix = options.variant !== void 0 ? `_${options.variant.toLowerCase()}` : "";
+	return {
+		cases,
+		metadata: {
+			name: options.name ?? `longmemeval${variantSuffix}`,
+			...options.description !== void 0 ? { description: options.description } : {},
+			createdAt: /* @__PURE__ */ new Date()
+		}
+	};
+}
+/**
+* Pure parser. Exported so tests can exercise the mapping without
+* touching the filesystem.
+*
+* @stable
+*/
+function parseLongMemEval(text, abilities) {
+	const parsed = JSON.parse(text);
+	if (!Array.isArray(parsed)) throw new Error("[graphorin/evals] LongMemEval dataset must be a JSON array of questions.");
+	const rows = parsed;
+	const out = [];
+	for (let i = 0; i < rows.length; i++) {
+		const row = rows[i];
+		if (row === null || typeof row !== "object") throw new Error(`[graphorin/evals] LongMemEval row ${i} is not an object.`);
+		const mapped = mapRow(row, i);
+		if (abilities !== void 0 && !abilities.includes(mapped.input.ability ?? "info-extraction")) continue;
+		out.push(mapped);
+	}
+	return out;
+}
+function mapRow(row, index) {
+	const question = asString(row.question);
+	if (question === void 0) throw new Error(`[graphorin/evals] LongMemEval row ${index} is missing a 'question'.`);
+	const questionId = asString(row.question_id) ?? `longmemeval-${index}`;
+	const questionType = asString(row.question_type) ?? "";
+	const askedAt = asString(row.question_date);
+	const ability = mapAbility(questionType, questionId);
+	const rawSessions = asArray(row.haystack_sessions);
+	const sessionIds = asArray(row.haystack_session_ids);
+	const sessionDates = asArray(row.haystack_dates);
+	const haystackSessions = [];
+	for (let s = 0; s < rawSessions.length; s++) {
+		const rawTurns = asArray(rawSessions[s]);
+		const date = asString(sessionDates[s]);
+		const turns = [];
+		for (const t of rawTurns) {
+			if (t === null || typeof t !== "object") continue;
+			const content = asString(t.content);
+			if (content === void 0) continue;
+			const role = t.role === "assistant" ? "assistant" : "user";
+			turns.push({
+				role,
+				content,
+				...date !== void 0 ? { timestamp: date } : {}
+			});
+		}
+		haystackSessions.push({
+			id: asString(sessionIds[s]) ?? `session-${s}`,
+			turns
+		});
+	}
+	return {
+		id: questionId,
+		input: {
+			haystackSessions,
+			question,
+			...askedAt !== void 0 ? { askedAt } : {},
+			ability
+		},
+		expected: asString(row.answer) ?? "",
+		metadata: {
+			datasetName: "longmemeval",
+			ability,
+			questionType
+		}
+	};
+}
+/** Maps a LongMemEval `question_type` / `_abs` suffix onto a {@link MemoryEvalAbility}. */
+function mapAbility(questionType, questionId) {
+	const qt = questionType.toLowerCase();
+	if (questionId.endsWith("_abs") || qt.includes("abstention")) return "abstention";
+	if (qt.includes("temporal")) return "temporal";
+	if (qt.includes("knowledge-update") || qt.includes("knowledge_update")) return "knowledge-update";
+	if (qt.includes("multi-session") || qt.includes("multi_session")) return "multi-session";
+	return "info-extraction";
+}
+function asString(value) {
+	return typeof value === "string" ? value : void 0;
+}
+function asArray(value) {
+	return Array.isArray(value) ? value : [];
+}
+
+//#endregion
+export { loadLongMemEvalDataset, parseLongMemEval };
+//# sourceMappingURL=longmemeval.js.map

package/dist/loaders/longmemeval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"longmemeval.js","names":["parsed: unknown","out: Case<MemoryEvalInput, string>[]","haystackSessions: MemoryEvalSession[]","turns: MemoryEvalTurn[]"],"sources":["../../src/loaders/longmemeval.ts"],"sourcesContent":["/**\n * LongMemEval dataset loader (ICLR 2025, arXiv:2410.10813 —\n * https://github.com/xiaowu0162/LongMemEval). LongMemEval ships as a\n * JSON **array** of question objects:\n *\n * ```jsonc\n * {\n *   \"question_id\": \"abc_abs\",          // `_abs` suffix marks abstention\n *   \"question_type\": \"temporal-reasoning\",\n *   \"question\": \"Where did I live in 2021?\",\n *   \"answer\": \"Berlin\",\n *   \"question_date\": \"2023/05/20 (Sat) 14:00\",\n *   \"haystack_session_ids\": [\"s1\", \"s2\"],\n *   \"haystack_dates\": [\"2021/03/01 ...\", \"2023/01/05 ...\"],\n *   \"haystack_sessions\": [\n *     [ { \"role\": \"user\", \"content\": \"...\" }, { \"role\": \"assistant\", \"content\": \"...\", \"has_answer\": true } ],\n *     [ ... ]\n *   ]\n * }\n * ```\n *\n * The three parallel arrays (`haystack_session_ids` / `haystack_dates`\n * / `haystack_sessions`) are zipped by index; one date applies to every\n * turn in its session.\n *\n * @packageDocumentation\n */\n\nimport { readFile } from 'node:fs/promises';\n\nimport type { Case, Dataset } from '@graphorin/observability/eval';\n\nimport type {\n  MemoryEvalAbility,\n  MemoryEvalInput,\n  MemoryEvalSession,\n  MemoryEvalTurn,\n} from './memory-eval.js';\n\n/** @stable */\nexport interface LoadLongMemEvalOptions {\n  /** Local path to the dataset JSON (under `benchmarks/.datasets/`). */\n  readonly path: string;\n  /** Which release this file is. `'S'` (~115 sessions) is the default target. */\n  readonly variant?: 'S' | 'M';\n  /** When set, keep only cases whose mapped ability is in this list. */\n  readonly abilities?: ReadonlyArray<MemoryEvalAbility>;\n  /** Optional dataset name surfaced in `Dataset.metadata.name`. */\n  readonly name?: string;\n  /** Optional description surfaced in `Dataset.metadata.description`. */\n  readonly description?: string;\n}\n\n/**\n * Read a LongMemEval JSON file and return a fully-materialised\n * {@link Dataset} of {@link MemoryEvalInput} cases scored against the\n * reference `answer` string.\n *\n * @stable\n */\nexport async function loadLongMemEvalDataset(\n  options: LoadLongMemEvalOptions,\n): Promise<Dataset<MemoryEvalInput, string>> {\n  const text = await readFile(options.path, 'utf8');\n  const cases = parseLongMemEval(text, options.abilities);\n  const variantSuffix = options.variant !== undefined ? `_${options.variant.toLowerCase()}` : '';\n  return {\n    cases,\n    metadata: {\n      name: options.name ?? `longmemeval${variantSuffix}`,\n      ...(options.description !== undefined ? { description: options.description } : {}),\n      createdAt: new Date(),\n    },\n  };\n}\n\n/**\n * Pure parser. Exported so tests can exercise the mapping without\n * touching the filesystem.\n *\n * @stable\n */\nexport function parseLongMemEval(\n  text: string,\n  abilities?: ReadonlyArray<MemoryEvalAbility>,\n): ReadonlyArray<Case<MemoryEvalInput, string>> {\n  const parsed: unknown = JSON.parse(text);\n  if (!Array.isArray(parsed)) {\n    throw new Error('[graphorin/evals] LongMemEval dataset must be a JSON array of questions.');\n  }\n  const rows = parsed as ReadonlyArray<unknown>;\n  const out: Case<MemoryEvalInput, string>[] = [];\n  for (let i = 0; i < rows.length; i++) {\n    const row = rows[i];\n    if (row === null || typeof row !== 'object') {\n      throw new Error(`[graphorin/evals] LongMemEval row ${i} is not an object.`);\n    }\n    const mapped = mapRow(row as Record<string, unknown>, i);\n    if (abilities !== undefined && !abilities.includes(mapped.input.ability ?? 'info-extraction')) {\n      continue;\n    }\n    out.push(mapped);\n  }\n  return out;\n}\n\nfunction mapRow(row: Record<string, unknown>, index: number): Case<MemoryEvalInput, string> {\n  const question = asString(row.question);\n  if (question === undefined) {\n    throw new Error(`[graphorin/evals] LongMemEval row ${index} is missing a 'question'.`);\n  }\n  const questionId = asString(row.question_id) ?? `longmemeval-${index}`;\n  const questionType = asString(row.question_type) ?? '';\n  const askedAt = asString(row.question_date);\n  const ability = mapAbility(questionType, questionId);\n\n  const rawSessions = asArray(row.haystack_sessions);\n  const sessionIds = asArray(row.haystack_session_ids);\n  const sessionDates = asArray(row.haystack_dates);\n  const haystackSessions: MemoryEvalSession[] = [];\n  for (let s = 0; s < rawSessions.length; s++) {\n    const rawTurns = asArray(rawSessions[s]);\n    const date = asString(sessionDates[s]);\n    const turns: MemoryEvalTurn[] = [];\n    for (const t of rawTurns) {\n      if (t === null || typeof t !== 'object') continue;\n      const content = asString((t as Record<string, unknown>).content);\n      if (content === undefined) continue;\n      const role = (t as Record<string, unknown>).role === 'assistant' ? 'assistant' : 'user';\n      turns.push({ role, content, ...(date !== undefined ? { timestamp: date } : {}) });\n    }\n    haystackSessions.push({ id: asString(sessionIds[s]) ?? `session-${s}`, turns });\n  }\n\n  return {\n    id: questionId,\n    input: {\n      haystackSessions,\n      question,\n      ...(askedAt !== undefined ? { askedAt } : {}),\n      ability,\n    },\n    expected: asString(row.answer) ?? '',\n    metadata: { datasetName: 'longmemeval', ability, questionType },\n  };\n}\n\n/** Maps a LongMemEval `question_type` / `_abs` suffix onto a {@link MemoryEvalAbility}. */\nfunction mapAbility(questionType: string, questionId: string): MemoryEvalAbility {\n  const qt = questionType.toLowerCase();\n  if (questionId.endsWith('_abs') || qt.includes('abstention')) return 'abstention';\n  if (qt.includes('temporal')) return 'temporal';\n  if (qt.includes('knowledge-update') || qt.includes('knowledge_update')) return 'knowledge-update';\n  if (qt.includes('multi-session') || qt.includes('multi_session')) return 'multi-session';\n  return 'info-extraction';\n}\n\nfunction asString(value: unknown): string | undefined {\n  return typeof value === 'string' ? value : undefined;\n}\n\nfunction asArray(value: unknown): ReadonlyArray<unknown> {\n  return Array.isArray(value) ? (value as ReadonlyArray<unknown>) : [];\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4DA,eAAsB,uBACpB,SAC2C;CAE3C,MAAM,QAAQ,iBADD,MAAM,SAAS,QAAQ,MAAM,OAAO,EACZ,QAAQ,UAAU;CACvD,MAAM,gBAAgB,QAAQ,YAAY,SAAY,IAAI,QAAQ,QAAQ,aAAa,KAAK;AAC5F,QAAO;EACL;EACA,UAAU;GACR,MAAM,QAAQ,QAAQ,cAAc;GACpC,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;GACjF,2BAAW,IAAI,MAAM;GACtB;EACF;;;;;;;;AASH,SAAgB,iBACd,MACA,WAC8C;CAC9C,MAAMA,SAAkB,KAAK,MAAM,KAAK;AACxC,KAAI,CAAC,MAAM,QAAQ,OAAO,CACxB,OAAM,IAAI,MAAM,2EAA2E;CAE7F,MAAM,OAAO;CACb,MAAMC,MAAuC,EAAE;AAC/C,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,MAAM,KAAK;AACjB,MAAI,QAAQ,QAAQ,OAAO,QAAQ,SACjC,OAAM,IAAI,MAAM,qCAAqC,EAAE,oBAAoB;EAE7E,MAAM,SAAS,OAAO,KAAgC,EAAE;AACxD,MAAI,cAAc,UAAa,CAAC,UAAU,SAAS,OAAO,MAAM,WAAW,kBAAkB,CAC3F;AAEF,MAAI,KAAK,OAAO;;AAElB,QAAO;;AAGT,SAAS,OAAO,KAA8B,OAA8C;CAC1F,MAAM,WAAW,SAAS,IAAI,SAAS;AACvC,KAAI,aAAa,OACf,OAAM,IAAI,MAAM,qCAAqC,MAAM,2BAA2B;CAExF,MAAM,aAAa,SAAS,IAAI,YAAY,IAAI,eAAe;CAC/D,MAAM,eAAe,SAAS,IAAI,cAAc,IAAI;CACpD,MAAM,UAAU,SAAS,IAAI,cAAc;CAC3C,MAAM,UAAU,WAAW,cAAc,WAAW;CAEpD,MAAM,cAAc,QAAQ,IAAI,kBAAkB;CAClD,MAAM,aAAa,QAAQ,IAAI,qBAAqB;CACpD,MAAM,eAAe,QAAQ,IAAI,eAAe;CAChD,MAAMC,mBAAwC,EAAE;AAChD,MAAK,IAAI,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;EAC3C,MAAM,WAAW,QAAQ,YAAY,GAAG;EACxC,MAAM,OAAO,SAAS,aAAa,GAAG;EACtC,MAAMC,QAA0B,EAAE;AAClC,OAAK,MAAM,KAAK,UAAU;AACxB,OAAI,MAAM,QAAQ,OAAO,MAAM,SAAU;GACzC,MAAM,UAAU,SAAU,EAA8B,QAAQ;AAChE,OAAI,YAAY,OAAW;GAC3B,MAAM,OAAQ,EAA8B,SAAS,cAAc,cAAc;AACjF,SAAM,KAAK;IAAE;IAAM;IAAS,GAAI,SAAS,SAAY,EAAE,WAAW,MAAM,GAAG,EAAE;IAAG,CAAC;;AAEnF,mBAAiB,KAAK;GAAE,IAAI,SAAS,WAAW,GAAG,IAAI,WAAW;GAAK;GAAO,CAAC;;AAGjF,QAAO;EACL,IAAI;EACJ,OAAO;GACL;GACA;GACA,GAAI,YAAY,SAAY,EAAE,SAAS,GAAG,EAAE;GAC5C;GACD;EACD,UAAU,SAAS,IAAI,OAAO,IAAI;EAClC,UAAU;GAAE,aAAa;GAAe;GAAS;GAAc;EAChE;;;AAIH,SAAS,WAAW,cAAsB,YAAuC;CAC/E,MAAM,KAAK,aAAa,aAAa;AACrC,KAAI,WAAW,SAAS,OAAO,IAAI,GAAG,SAAS,aAAa,CAAE,QAAO;AACrE,KAAI,GAAG,SAAS,WAAW,CAAE,QAAO;AACpC,KAAI,GAAG,SAAS,mBAAmB,IAAI,GAAG,SAAS,mBAAmB,CAAE,QAAO;AAC/E,KAAI,GAAG,SAAS,gBAAgB,IAAI,GAAG,SAAS,gBAAgB,CAAE,QAAO;AACzE,QAAO;;AAGT,SAAS,SAAS,OAAoC;AACpD,QAAO,OAAO,UAAU,WAAW,QAAQ;;AAG7C,SAAS,QAAQ,OAAwC;AACvD,QAAO,MAAM,QAAQ,MAAM,GAAI,QAAmC,EAAE"}

package/dist/loaders/memory-eval.d.ts

@@ -0,0 +1,67 @@
+//#region src/loaders/memory-eval.d.ts
+/**
+ * Shared "memory system under test" contract used by the long-term
+ * memory eval loaders ({@link loadLongMemEvalDataset},
+ * {@link loadLocomoDataset}, {@link loadDmrDataset}) and the benchmark
+ * runner. A loader turns a dataset's native JSON into
+ * `Case<MemoryEvalInput, string>`: the input carries the haystack of
+ * prior sessions plus the question; the expected output is the
+ * reference answer string, graded by the `llmJudge` "J" scorer.
+ *
+ * This module is intentionally **type-only** and carries no dependency
+ * on `@graphorin/memory` — the concrete system-under-test agent lives
+ * in the benchmark package so `@graphorin/evals` stays a generic
+ * harness.
+ *
+ * @packageDocumentation
+ */
+/**
+ * One conversational turn inside a haystack session.
+ *
+ * @stable
+ */
+interface MemoryEvalTurn {
+  readonly role: 'user' | 'assistant';
+  readonly content: string;
+  /** Dataset-native (often ISO-8601) timestamp, when the dataset provides one. */
+  readonly timestamp?: string;
+}
+/**
+ * One prior session in the haystack the memory system must ingest
+ * before the question is asked.
+ *
+ * @stable
+ */
+interface MemoryEvalSession {
+  readonly id: string;
+  readonly turns: ReadonlyArray<MemoryEvalTurn>;
+}
+/**
+ * The five LongMemEval abilities. Every loader maps its dataset-native
+ * category onto one of these so per-ability scoring is comparable
+ * across datasets; the raw category is preserved in `Case.metadata`.
+ *
+ * @stable
+ */
+type MemoryEvalAbility = 'info-extraction' | 'multi-session' | 'temporal' | 'knowledge-update' | 'abstention';
+/**
+ * Input handed to the memory system under test for one eval case.
+ *
+ * @stable
+ */
+interface MemoryEvalInput {
+  /** Prior sessions to ingest before the question is asked. */
+  readonly haystackSessions: ReadonlyArray<MemoryEvalSession>;
+  /** The question to answer from memory. */
+  readonly question: string;
+  /**
+   * When the question is asked. Drives temporal / knowledge-update
+   * reasoning; dataset-native string (not necessarily ISO-8601).
+   */
+  readonly askedAt?: string;
+  /** Mapped ability ({@link MemoryEvalAbility}); the raw category lives in metadata. */
+  readonly ability?: MemoryEvalAbility;
+}
+//#endregion
+export { MemoryEvalAbility, MemoryEvalInput, MemoryEvalSession, MemoryEvalTurn };
+//# sourceMappingURL=memory-eval.d.ts.map

package/dist/loaders/memory-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-eval.d.ts","names":[],"sources":["../../src/loaders/memory-eval.ts"],"sourcesContent":[],"mappings":";;AAsBA;AAaA;AAYA;AAYA;;;;;;;;;;;;;;;;;UArCiB,cAAA;;;;;;;;;;;;UAaA,iBAAA;;kBAEC,cAAc;;;;;;;;;KAUpB,iBAAA;;;;;;UAYK,eAAA;;6BAEY,cAAc;;;;;;;;;qBAStB"}

package/dist/regression.d.ts

@@ -0,0 +1,13 @@
+import { EvalReport, RegressionOptions, RegressionReport } from "./types.js";
+
+//#region src/regression.d.ts
+
+/**
+ * Detect regressions between `current` and `baseline` reports.
+ *
+ * @stable
+ */
+declare function detectRegressions<I, O>(current: EvalReport<I, O>, baseline: EvalReport<I, O>, options?: RegressionOptions): RegressionReport<I, O>;
+//#endregion
+export { detectRegressions };
+//# sourceMappingURL=regression.d.ts.map

package/dist/regression.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"regression.d.ts","names":[],"sources":["../src/regression.ts"],"sourcesContent":[],"mappings":";;;;;;;;;AAiCG,iBAJa,iBAIb,CAAA,CAAA,EAAA,CAAA,CAAA,CAAA,OAAA,EAHQ,UAGR,CAHmB,CAGnB,EAHsB,CAGtB,CAAA,EAAA,QAAA,EAFS,UAET,CAFoB,CAEpB,EAFuB,CAEvB,CAAA,EAAA,OAAA,CAAA,EADQ,iBACR,CAAA,EAAA,gBAAA,CAAiB,CAAjB,EAAoB,CAApB,CAAA"}

package/dist/regression.js

@@ -0,0 +1,63 @@
+//#region src/regression.ts
+const DEFAULT_PASS_RATE_DROP_PCT = 5;
+const DEFAULT_AVG_SCORE_DROP = .05;
+const DEFAULT_AVG_DURATION_INCREASE_MS = Number.POSITIVE_INFINITY;
+/**
+* Detect regressions between `current` and `baseline` reports.
+*
+* @stable
+*/
+function detectRegressions(current, baseline, options = {}) {
+	const maxPassRateDropPct = options.maxPassRateDropPct ?? DEFAULT_PASS_RATE_DROP_PCT;
+	const maxAvgScoreDrop = options.maxAvgScoreDrop ?? DEFAULT_AVG_SCORE_DROP;
+	const maxAvgDurationIncreaseMs = options.maxAvgDurationIncreaseMs ?? DEFAULT_AVG_DURATION_INCREASE_MS;
+	const findings = [];
+	const currentPassRate = total(current) === 0 ? 0 : current.summary.passed / total(current);
+	const baselinePassRate = total(baseline) === 0 ? 0 : baseline.summary.passed / total(baseline);
+	const passRateDropPct = (baselinePassRate - currentPassRate) * 100;
+	if (passRateDropPct > maxPassRateDropPct) findings.push({
+		kind: "pass-rate-drop",
+		message: `pass rate dropped by ${passRateDropPct.toFixed(2)}% (${(currentPassRate * 100).toFixed(2)}% < baseline ${(baselinePassRate * 100).toFixed(2)}% − tolerance ${maxPassRateDropPct.toFixed(2)}%).`,
+		delta: -passRateDropPct
+	});
+	const durationDelta = current.summary.avgDurationMs - baseline.summary.avgDurationMs;
+	if (durationDelta > maxAvgDurationIncreaseMs) findings.push({
+		kind: "avg-duration-increase",
+		message: `avg duration increased by ${durationDelta.toFixed(2)} ms (${current.summary.avgDurationMs.toFixed(2)} ms > baseline ${baseline.summary.avgDurationMs.toFixed(2)} ms − tolerance ${maxAvgDurationIncreaseMs} ms).`,
+		delta: durationDelta
+	});
+	for (const [scorer, baselineRow] of Object.entries(baseline.summary.byScorer)) {
+		const currentRow = current.summary.byScorer[scorer];
+		if (currentRow === void 0) {
+			findings.push({
+				kind: "scorer-removed",
+				scorer,
+				message: `scorer '${scorer}' is in the baseline but missing from the current run.`,
+				delta: NaN
+			});
+			continue;
+		}
+		if (typeof baselineRow.avgScore === "number" && typeof currentRow.avgScore === "number") {
+			const drop = baselineRow.avgScore - currentRow.avgScore;
+			if (drop > maxAvgScoreDrop) findings.push({
+				kind: "avg-score-drop",
+				scorer,
+				message: `'${scorer}' avg score dropped by ${drop.toFixed(4)} (${currentRow.avgScore.toFixed(4)} < baseline ${baselineRow.avgScore.toFixed(4)} − tolerance ${maxAvgScoreDrop}).`,
+				delta: -drop
+			});
+		}
+	}
+	return {
+		hasRegressions: findings.length > 0,
+		findings,
+		current,
+		baseline
+	};
+}
+function total(r) {
+	return r.summary.total;
+}
+
+//#endregion
+export { detectRegressions };
+//# sourceMappingURL=regression.js.map

package/dist/regression.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"regression.js","names":["findings: RegressionFinding[]"],"sources":["../src/regression.ts"],"sourcesContent":["/**\n * Compares two `EvalReport`s — typically the freshly-produced report\n * vs a stored baseline — and lists every metric that regressed beyond\n * the configured tolerance.\n *\n * @packageDocumentation\n */\n\nimport type {\n  EvalReport,\n  RegressionFinding,\n  RegressionOptions,\n  RegressionReport,\n} from './types.js';\n\nconst DEFAULT_PASS_RATE_DROP_PCT = 5;\nconst DEFAULT_AVG_SCORE_DROP = 0.05;\n// EB-4: the avg-duration gate is OPT-IN (default off). Absolute wall-clock\n// budgets are environment-sensitive — a baseline recorded on a fast\n// workstation vs a loaded CI runner, or real LLM-latency jitter of whole\n// seconds, would flag spurious \"regressions\". Callers that genuinely want a\n// duration budget pass an explicit finite `maxAvgDurationIncreaseMs`.\nconst DEFAULT_AVG_DURATION_INCREASE_MS = Number.POSITIVE_INFINITY;\n\n/**\n * Detect regressions between `current` and `baseline` reports.\n *\n * @stable\n */\nexport function detectRegressions<I, O>(\n  current: EvalReport<I, O>,\n  baseline: EvalReport<I, O>,\n  options: RegressionOptions = {},\n): RegressionReport<I, O> {\n  const maxPassRateDropPct = options.maxPassRateDropPct ?? DEFAULT_PASS_RATE_DROP_PCT;\n  const maxAvgScoreDrop = options.maxAvgScoreDrop ?? DEFAULT_AVG_SCORE_DROP;\n  const maxAvgDurationIncreaseMs =\n    options.maxAvgDurationIncreaseMs ?? DEFAULT_AVG_DURATION_INCREASE_MS;\n  const findings: RegressionFinding[] = [];\n\n  const currentPassRate = total(current) === 0 ? 0 : current.summary.passed / total(current);\n  const baselinePassRate = total(baseline) === 0 ? 0 : baseline.summary.passed / total(baseline);\n  const passRateDropPct = (baselinePassRate - currentPassRate) * 100;\n  if (passRateDropPct > maxPassRateDropPct) {\n    findings.push({\n      kind: 'pass-rate-drop',\n      message:\n        `pass rate dropped by ${passRateDropPct.toFixed(2)}% ` +\n        `(${(currentPassRate * 100).toFixed(2)}% < baseline ${(baselinePassRate * 100).toFixed(2)}% ` +\n        `− tolerance ${maxPassRateDropPct.toFixed(2)}%).`,\n      delta: -passRateDropPct,\n    });\n  }\n\n  const durationDelta = current.summary.avgDurationMs - baseline.summary.avgDurationMs;\n  if (durationDelta > maxAvgDurationIncreaseMs) {\n    findings.push({\n      kind: 'avg-duration-increase',\n      message:\n        `avg duration increased by ${durationDelta.toFixed(2)} ms ` +\n        `(${current.summary.avgDurationMs.toFixed(2)} ms > baseline ` +\n        `${baseline.summary.avgDurationMs.toFixed(2)} ms − tolerance ${maxAvgDurationIncreaseMs} ms).`,\n      delta: durationDelta,\n    });\n  }\n\n  for (const [scorer, baselineRow] of Object.entries(baseline.summary.byScorer)) {\n    const currentRow = current.summary.byScorer[scorer];\n    if (currentRow === undefined) {\n      findings.push({\n        kind: 'scorer-removed',\n        scorer,\n        message: `scorer '${scorer}' is in the baseline but missing from the current run.`,\n        delta: Number.NaN,\n      });\n      continue;\n    }\n    if (typeof baselineRow.avgScore === 'number' && typeof currentRow.avgScore === 'number') {\n      const drop = baselineRow.avgScore - currentRow.avgScore;\n      if (drop > maxAvgScoreDrop) {\n        findings.push({\n          kind: 'avg-score-drop',\n          scorer,\n          message:\n            `'${scorer}' avg score dropped by ${drop.toFixed(4)} ` +\n            `(${currentRow.avgScore.toFixed(4)} < baseline ${baselineRow.avgScore.toFixed(4)} ` +\n            `− tolerance ${maxAvgScoreDrop}).`,\n          delta: -drop,\n        });\n      }\n    }\n  }\n\n  return {\n    hasRegressions: findings.length > 0,\n    findings,\n    current,\n    baseline,\n  };\n}\n\nfunction total<I, O>(r: EvalReport<I, O>): number {\n  return r.summary.total;\n}\n"],"mappings":";AAeA,MAAM,6BAA6B;AACnC,MAAM,yBAAyB;AAM/B,MAAM,mCAAmC,OAAO;;;;;;AAOhD,SAAgB,kBACd,SACA,UACA,UAA6B,EAAE,EACP;CACxB,MAAM,qBAAqB,QAAQ,sBAAsB;CACzD,MAAM,kBAAkB,QAAQ,mBAAmB;CACnD,MAAM,2BACJ,QAAQ,4BAA4B;CACtC,MAAMA,WAAgC,EAAE;CAExC,MAAM,kBAAkB,MAAM,QAAQ,KAAK,IAAI,IAAI,QAAQ,QAAQ,SAAS,MAAM,QAAQ;CAC1F,MAAM,mBAAmB,MAAM,SAAS,KAAK,IAAI,IAAI,SAAS,QAAQ,SAAS,MAAM,SAAS;CAC9F,MAAM,mBAAmB,mBAAmB,mBAAmB;AAC/D,KAAI,kBAAkB,mBACpB,UAAS,KAAK;EACZ,MAAM;EACN,SACE,wBAAwB,gBAAgB,QAAQ,EAAE,CAAC,MAC9C,kBAAkB,KAAK,QAAQ,EAAE,CAAC,gBAAgB,mBAAmB,KAAK,QAAQ,EAAE,CAAC,gBAC3E,mBAAmB,QAAQ,EAAE,CAAC;EAC/C,OAAO,CAAC;EACT,CAAC;CAGJ,MAAM,gBAAgB,QAAQ,QAAQ,gBAAgB,SAAS,QAAQ;AACvE,KAAI,gBAAgB,yBAClB,UAAS,KAAK;EACZ,MAAM;EACN,SACE,6BAA6B,cAAc,QAAQ,EAAE,CAAC,OAClD,QAAQ,QAAQ,cAAc,QAAQ,EAAE,CAAC,iBAC1C,SAAS,QAAQ,cAAc,QAAQ,EAAE,CAAC,kBAAkB,yBAAyB;EAC1F,OAAO;EACR,CAAC;AAGJ,MAAK,MAAM,CAAC,QAAQ,gBAAgB,OAAO,QAAQ,SAAS,QAAQ,SAAS,EAAE;EAC7E,MAAM,aAAa,QAAQ,QAAQ,SAAS;AAC5C,MAAI,eAAe,QAAW;AAC5B,YAAS,KAAK;IACZ,MAAM;IACN;IACA,SAAS,WAAW,OAAO;IAC3B,OAAO;IACR,CAAC;AACF;;AAEF,MAAI,OAAO,YAAY,aAAa,YAAY,OAAO,WAAW,aAAa,UAAU;GACvF,MAAM,OAAO,YAAY,WAAW,WAAW;AAC/C,OAAI,OAAO,gBACT,UAAS,KAAK;IACZ,MAAM;IACN;IACA,SACE,IAAI,OAAO,yBAAyB,KAAK,QAAQ,EAAE,CAAC,IAChD,WAAW,SAAS,QAAQ,EAAE,CAAC,cAAc,YAAY,SAAS,QAAQ,EAAE,CAAC,eAClE,gBAAgB;IACjC,OAAO,CAAC;IACT,CAAC;;;AAKR,QAAO;EACL,gBAAgB,SAAS,SAAS;EAClC;EACA;EACA;EACD;;AAGH,SAAS,MAAY,GAA6B;AAChD,QAAO,EAAE,QAAQ"}

package/dist/reporters/html.d.ts

@@ -0,0 +1,11 @@
+import { EvalReport } from "@graphorin/observability/eval";
+
+//#region src/reporters/html.d.ts
+
+/** @stable */
+declare function renderHtmlReport<I, O>(report: EvalReport<I, O>, options?: {
+  readonly title?: string;
+}): string;
+//#endregion
+export { renderHtmlReport };
+//# sourceMappingURL=html.d.ts.map

package/dist/reporters/html.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html.d.ts","names":[],"sources":["../../src/reporters/html.ts"],"sourcesContent":[],"mappings":";;;;;iBAYgB,+BACN,WAAW,GAAG"}

package/dist/reporters/html.js

@@ -0,0 +1,60 @@
+//#region src/reporters/html.ts
+/** @stable */
+function renderHtmlReport(report, options = {}) {
+	const title = escapeHtml(options.title ?? "graphorin/evals report");
+	const summary = report.summary;
+	const passRate = summary.total === 0 ? 0 : summary.passed / summary.total * 100;
+	const scorerRows = Object.entries(summary.byScorer).map(([scorer, row]) => {
+		const avg = row.avgScore === null ? "n/a" : row.avgScore.toFixed(4);
+		return `<tr><td><code>${escapeHtml(scorer)}</code></td><td>${row.passed}</td><td>${row.failed}</td><td>${avg}</td></tr>`;
+	}).join("");
+	const failureRows = report.results.flatMap((r) => {
+		return r.scores.filter((s) => !s.result.pass).map((f) => `<tr><td><code>${escapeHtml(r.caseId)}</code></td><td><code>${escapeHtml(f.scorer)}</code></td><td>${escapeHtml(f.result.reason ?? "")}</td></tr>`);
+	}).join("");
+	return `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>${title}</title>
+<style>
+  body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; margin: 2rem; max-width: 64rem; color: #1a1a1a; }
+  h1 { margin-top: 0; }
+  table { border-collapse: collapse; width: 100%; margin: 1rem 0; }
+  th, td { padding: 0.4rem 0.6rem; text-align: left; border-bottom: 1px solid #e5e5e5; }
+  th { background: #f5f5f5; }
+  .summary-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(8rem, 1fr)); gap: 0.6rem; }
+  .summary-card { background: #f5f5f5; padding: 0.8rem; border-radius: 6px; }
+  .summary-card .label { font-size: 0.8rem; text-transform: uppercase; color: #666; letter-spacing: 0.04em; }
+  .summary-card .value { font-size: 1.4rem; font-weight: 600; }
+  code { font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; font-size: 0.9rem; }
+</style>
+</head>
+<body>
+<h1>${title}</h1>
+<div class="summary-grid">
+  <div class="summary-card"><div class="label">Total</div><div class="value">${summary.total}</div></div>
+  <div class="summary-card"><div class="label">Passed</div><div class="value">${summary.passed}</div></div>
+  <div class="summary-card"><div class="label">Failed</div><div class="value">${summary.failed}</div></div>
+  <div class="summary-card"><div class="label">Pass rate</div><div class="value">${passRate.toFixed(1)}%</div></div>
+  <div class="summary-card"><div class="label">Avg ms</div><div class="value">${summary.avgDurationMs.toFixed(0)}</div></div>
+</div>
+<h2>Per-scorer</h2>
+<table>
+  <thead><tr><th>Scorer</th><th>Pass</th><th>Fail</th><th>Avg score</th></tr></thead>
+  <tbody>${scorerRows || "<tr><td colspan=\"4\">No scorers.</td></tr>"}</tbody>
+</table>
+<h2>Failures</h2>
+<table>
+  <thead><tr><th>Case</th><th>Scorer</th><th>Reason</th></tr></thead>
+  <tbody>${failureRows || "<tr><td colspan=\"3\">No failures.</td></tr>"}</tbody>
+</table>
+</body>
+</html>`;
+}
+function escapeHtml(value) {
+	return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
+}
+
+//#endregion
+export { renderHtmlReport };
+//# sourceMappingURL=html.js.map

package/dist/reporters/html.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html.js","names":[],"sources":["../../src/reporters/html.ts"],"sourcesContent":["/**\n * HTML reporter. Renders an {@link EvalReport} as a self-contained\n * HTML document — no external CSS / JS — so artifact viewers can\n * render it directly. Designed to fit on a single page; for richer\n * dashboards consume the JSON reporter instead.\n *\n * @packageDocumentation\n */\n\nimport type { EvalReport } from '@graphorin/observability/eval';\n\n/** @stable */\nexport function renderHtmlReport<I, O>(\n  report: EvalReport<I, O>,\n  options: { readonly title?: string } = {},\n): string {\n  const title = escapeHtml(options.title ?? 'graphorin/evals report');\n  const summary = report.summary;\n  const passRate = summary.total === 0 ? 0 : (summary.passed / summary.total) * 100;\n  const scorerRows = Object.entries(summary.byScorer)\n    .map(([scorer, row]) => {\n      const avg = row.avgScore === null ? 'n/a' : row.avgScore.toFixed(4);\n      return `<tr><td><code>${escapeHtml(scorer)}</code></td><td>${row.passed}</td><td>${row.failed}</td><td>${avg}</td></tr>`;\n    })\n    .join('');\n  const failureRows = report.results\n    .flatMap((r) => {\n      const failures = r.scores.filter((s) => !s.result.pass);\n      return failures.map(\n        (f) =>\n          `<tr><td><code>${escapeHtml(r.caseId)}</code></td><td><code>${escapeHtml(f.scorer)}</code></td><td>${escapeHtml(f.result.reason ?? '')}</td></tr>`,\n      );\n    })\n    .join('');\n  return `<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\">\n<title>${title}</title>\n<style>\n  body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; margin: 2rem; max-width: 64rem; color: #1a1a1a; }\n  h1 { margin-top: 0; }\n  table { border-collapse: collapse; width: 100%; margin: 1rem 0; }\n  th, td { padding: 0.4rem 0.6rem; text-align: left; border-bottom: 1px solid #e5e5e5; }\n  th { background: #f5f5f5; }\n  .summary-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(8rem, 1fr)); gap: 0.6rem; }\n  .summary-card { background: #f5f5f5; padding: 0.8rem; border-radius: 6px; }\n  .summary-card .label { font-size: 0.8rem; text-transform: uppercase; color: #666; letter-spacing: 0.04em; }\n  .summary-card .value { font-size: 1.4rem; font-weight: 600; }\n  code { font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; font-size: 0.9rem; }\n</style>\n</head>\n<body>\n<h1>${title}</h1>\n<div class=\"summary-grid\">\n  <div class=\"summary-card\"><div class=\"label\">Total</div><div class=\"value\">${summary.total}</div></div>\n  <div class=\"summary-card\"><div class=\"label\">Passed</div><div class=\"value\">${summary.passed}</div></div>\n  <div class=\"summary-card\"><div class=\"label\">Failed</div><div class=\"value\">${summary.failed}</div></div>\n  <div class=\"summary-card\"><div class=\"label\">Pass rate</div><div class=\"value\">${passRate.toFixed(1)}%</div></div>\n  <div class=\"summary-card\"><div class=\"label\">Avg ms</div><div class=\"value\">${summary.avgDurationMs.toFixed(0)}</div></div>\n</div>\n<h2>Per-scorer</h2>\n<table>\n  <thead><tr><th>Scorer</th><th>Pass</th><th>Fail</th><th>Avg score</th></tr></thead>\n  <tbody>${scorerRows || '<tr><td colspan=\"4\">No scorers.</td></tr>'}</tbody>\n</table>\n<h2>Failures</h2>\n<table>\n  <thead><tr><th>Case</th><th>Scorer</th><th>Reason</th></tr></thead>\n  <tbody>${failureRows || '<tr><td colspan=\"3\">No failures.</td></tr>'}</tbody>\n</table>\n</body>\n</html>`;\n}\n\nfunction escapeHtml(value: string): string {\n  return value\n    .replace(/&/g, '&amp;')\n    .replace(/</g, '&lt;')\n    .replace(/>/g, '&gt;')\n    .replace(/\"/g, '&quot;')\n    .replace(/'/g, '&#39;');\n}\n"],"mappings":";;AAYA,SAAgB,iBACd,QACA,UAAuC,EAAE,EACjC;CACR,MAAM,QAAQ,WAAW,QAAQ,SAAS,yBAAyB;CACnE,MAAM,UAAU,OAAO;CACvB,MAAM,WAAW,QAAQ,UAAU,IAAI,IAAK,QAAQ,SAAS,QAAQ,QAAS;CAC9E,MAAM,aAAa,OAAO,QAAQ,QAAQ,SAAS,CAChD,KAAK,CAAC,QAAQ,SAAS;EACtB,MAAM,MAAM,IAAI,aAAa,OAAO,QAAQ,IAAI,SAAS,QAAQ,EAAE;AACnE,SAAO,iBAAiB,WAAW,OAAO,CAAC,kBAAkB,IAAI,OAAO,WAAW,IAAI,OAAO,WAAW,IAAI;GAC7G,CACD,KAAK,GAAG;CACX,MAAM,cAAc,OAAO,QACxB,SAAS,MAAM;AAEd,SADiB,EAAE,OAAO,QAAQ,MAAM,CAAC,EAAE,OAAO,KAAK,CACvC,KACb,MACC,iBAAiB,WAAW,EAAE,OAAO,CAAC,wBAAwB,WAAW,EAAE,OAAO,CAAC,kBAAkB,WAAW,EAAE,OAAO,UAAU,GAAG,CAAC,YAC1I;GACD,CACD,KAAK,GAAG;AACX,QAAO;;;;SAIA,MAAM;;;;;;;;;;;;;;;MAeT,MAAM;;+EAEmE,QAAQ,MAAM;gFACb,QAAQ,OAAO;gFACf,QAAQ,OAAO;mFACZ,SAAS,QAAQ,EAAE,CAAC;gFACvB,QAAQ,cAAc,QAAQ,EAAE,CAAC;;;;;WAKtG,cAAc,8CAA4C;;;;;WAK1D,eAAe,+CAA6C;;;;;AAMvE,SAAS,WAAW,OAAuB;AACzC,QAAO,MACJ,QAAQ,MAAM,QAAQ,CACtB,QAAQ,MAAM,OAAO,CACrB,QAAQ,MAAM,OAAO,CACrB,QAAQ,MAAM,SAAS,CACvB,QAAQ,MAAM,QAAQ"}

package/dist/reporters/index.d.ts

@@ -0,0 +1,6 @@
+import { renderHtmlReport } from "./html.js";
+import { renderJsonReport } from "./json.js";
+import { renderJunitReport } from "./junit.js";
+import { renderMarkdownReport } from "./markdown.js";
+import { renderTerminalReport } from "./terminal.js";
+export { renderHtmlReport, renderJsonReport, renderJunitReport, renderMarkdownReport, renderTerminalReport };

package/dist/reporters/index.js

@@ -0,0 +1,7 @@
+import { renderHtmlReport } from "./html.js";
+import { renderJsonReport } from "./json.js";
+import { renderJunitReport } from "./junit.js";
+import { renderMarkdownReport } from "./markdown.js";
+import { renderTerminalReport } from "./terminal.js";
+
+export { renderHtmlReport, renderJsonReport, renderJunitReport, renderMarkdownReport, renderTerminalReport };

package/dist/reporters/json.d.ts

@@ -0,0 +1,11 @@
+import { EvalReport } from "@graphorin/observability/eval";
+
+//#region src/reporters/json.d.ts
+
+/** @stable */
+declare function renderJsonReport<I, O>(report: EvalReport<I, O>, options?: {
+  readonly pretty?: boolean;
+}): string;
+//#endregion
+export { renderJsonReport };
+//# sourceMappingURL=json.d.ts.map

package/dist/reporters/json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.d.ts","names":[],"sources":["../../src/reporters/json.ts"],"sourcesContent":[],"mappings":";;;;;iBAWgB,+BACN,WAAW,GAAG"}

package/dist/reporters/json.js

@@ -0,0 +1,10 @@
+//#region src/reporters/json.ts
+/** @stable */
+function renderJsonReport(report, options = {}) {
+	const indent = options.pretty === true ? 2 : 0;
+	return JSON.stringify(report, null, indent);
+}
+
+//#endregion
+export { renderJsonReport };
+//# sourceMappingURL=json.js.map

package/dist/reporters/json.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.js","names":[],"sources":["../../src/reporters/json.ts"],"sourcesContent":["/**\n * JSON reporter. Returns the canonical JSON-serialised representation\n * of an {@link EvalReport} so downstream tooling (dashboards,\n * spreadsheets, regression checkers) can consume it deterministically.\n *\n * @packageDocumentation\n */\n\nimport type { EvalReport } from '@graphorin/observability/eval';\n\n/** @stable */\nexport function renderJsonReport<I, O>(\n  report: EvalReport<I, O>,\n  options: { readonly pretty?: boolean } = {},\n): string {\n  const indent = options.pretty === true ? 2 : 0;\n  return JSON.stringify(report, null, indent);\n}\n"],"mappings":";;AAWA,SAAgB,iBACd,QACA,UAAyC,EAAE,EACnC;CACR,MAAM,SAAS,QAAQ,WAAW,OAAO,IAAI;AAC7C,QAAO,KAAK,UAAU,QAAQ,MAAM,OAAO"}

package/dist/reporters/junit.d.ts

@@ -0,0 +1,11 @@
+import { EvalReport } from "@graphorin/observability/eval";
+
+//#region src/reporters/junit.d.ts
+
+/** @stable */
+declare function renderJunitReport<I, O>(report: EvalReport<I, O>, options?: {
+  readonly suiteName?: string;
+}): string;
+//#endregion
+export { renderJunitReport };
+//# sourceMappingURL=junit.d.ts.map

package/dist/reporters/junit.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"junit.d.ts","names":[],"sources":["../../src/reporters/junit.ts"],"sourcesContent":[],"mappings":";;;;;iBAYgB,gCACN,WAAW,GAAG"}

package/dist/reporters/junit.js

@@ -0,0 +1,29 @@
+//#region src/reporters/junit.ts
+/** @stable */
+function renderJunitReport(report, options = {}) {
+	const suiteName = options.suiteName ?? "graphorin-evals";
+	const totalTests = report.results.reduce((acc, r) => acc + r.scores.length, 0);
+	const failures = report.results.reduce((acc, r) => acc + r.scores.filter((s) => !s.result.pass).length, 0);
+	const totalTime = report.summary.avgDurationMs * report.summary.total / 1e3;
+	const lines = [];
+	lines.push("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+	lines.push(`<testsuite name="${escapeXml(suiteName)}" tests="${totalTests}" failures="${failures}" time="${totalTime.toFixed(3)}">`);
+	for (const r of report.results) for (const { scorer, result } of r.scores) {
+		const time = (r.durationMs / 1e3).toFixed(3);
+		lines.push(`  <testcase classname="${escapeXml(scorer)}" name="${escapeXml(r.caseId)}" time="${time}">`);
+		if (!result.pass) {
+			const message = escapeXml(result.reason ?? "failed");
+			lines.push(`    <failure message="${message}">${message}</failure>`);
+		}
+		lines.push("  </testcase>");
+	}
+	lines.push("</testsuite>");
+	return lines.join("\n");
+}
+function escapeXml(value) {
+	return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
+}
+
+//#endregion
+export { renderJunitReport };
+//# sourceMappingURL=junit.js.map

package/dist/reporters/junit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"junit.js","names":["lines: string[]"],"sources":["../../src/reporters/junit.ts"],"sourcesContent":["/**\n * JUnit XML reporter. Renders an {@link EvalReport} as a JUnit\n * `<testsuite>` document so CI systems (GitHub Actions, GitLab,\n * CircleCI, Jenkins) can render pass / fail counts in the standard\n * test-report widget.\n *\n * @packageDocumentation\n */\n\nimport type { EvalReport } from '@graphorin/observability/eval';\n\n/** @stable */\nexport function renderJunitReport<I, O>(\n  report: EvalReport<I, O>,\n  options: { readonly suiteName?: string } = {},\n): string {\n  const suiteName = options.suiteName ?? 'graphorin-evals';\n  const totalTests = report.results.reduce((acc, r) => acc + r.scores.length, 0);\n  const failures = report.results.reduce(\n    (acc, r) => acc + r.scores.filter((s) => !s.result.pass).length,\n    0,\n  );\n  const totalTime = (report.summary.avgDurationMs * report.summary.total) / 1000;\n  const lines: string[] = [];\n  lines.push('<?xml version=\"1.0\" encoding=\"UTF-8\"?>');\n  lines.push(\n    `<testsuite name=\"${escapeXml(suiteName)}\" tests=\"${totalTests}\" failures=\"${failures}\" time=\"${totalTime.toFixed(3)}\">`,\n  );\n  for (const r of report.results) {\n    for (const { scorer, result } of r.scores) {\n      const time = (r.durationMs / 1000).toFixed(3);\n      lines.push(\n        `  <testcase classname=\"${escapeXml(scorer)}\" name=\"${escapeXml(r.caseId)}\" time=\"${time}\">`,\n      );\n      if (!result.pass) {\n        const message = escapeXml(result.reason ?? 'failed');\n        lines.push(`    <failure message=\"${message}\">${message}</failure>`);\n      }\n      lines.push('  </testcase>');\n    }\n  }\n  lines.push('</testsuite>');\n  return lines.join('\\n');\n}\n\nfunction escapeXml(value: string): string {\n  return value\n    .replace(/&/g, '&amp;')\n    .replace(/</g, '&lt;')\n    .replace(/>/g, '&gt;')\n    .replace(/\"/g, '&quot;')\n    .replace(/'/g, '&apos;');\n}\n"],"mappings":";;AAYA,SAAgB,kBACd,QACA,UAA2C,EAAE,EACrC;CACR,MAAM,YAAY,QAAQ,aAAa;CACvC,MAAM,aAAa,OAAO,QAAQ,QAAQ,KAAK,MAAM,MAAM,EAAE,OAAO,QAAQ,EAAE;CAC9E,MAAM,WAAW,OAAO,QAAQ,QAC7B,KAAK,MAAM,MAAM,EAAE,OAAO,QAAQ,MAAM,CAAC,EAAE,OAAO,KAAK,CAAC,QACzD,EACD;CACD,MAAM,YAAa,OAAO,QAAQ,gBAAgB,OAAO,QAAQ,QAAS;CAC1E,MAAMA,QAAkB,EAAE;AAC1B,OAAM,KAAK,6CAAyC;AACpD,OAAM,KACJ,oBAAoB,UAAU,UAAU,CAAC,WAAW,WAAW,cAAc,SAAS,UAAU,UAAU,QAAQ,EAAE,CAAC,IACtH;AACD,MAAK,MAAM,KAAK,OAAO,QACrB,MAAK,MAAM,EAAE,QAAQ,YAAY,EAAE,QAAQ;EACzC,MAAM,QAAQ,EAAE,aAAa,KAAM,QAAQ,EAAE;AAC7C,QAAM,KACJ,0BAA0B,UAAU,OAAO,CAAC,UAAU,UAAU,EAAE,OAAO,CAAC,UAAU,KAAK,IAC1F;AACD,MAAI,CAAC,OAAO,MAAM;GAChB,MAAM,UAAU,UAAU,OAAO,UAAU,SAAS;AACpD,SAAM,KAAK,yBAAyB,QAAQ,IAAI,QAAQ,YAAY;;AAEtE,QAAM,KAAK,gBAAgB;;AAG/B,OAAM,KAAK,eAAe;AAC1B,QAAO,MAAM,KAAK,KAAK;;AAGzB,SAAS,UAAU,OAAuB;AACxC,QAAO,MACJ,QAAQ,MAAM,QAAQ,CACtB,QAAQ,MAAM,OAAO,CACrB,QAAQ,MAAM,OAAO,CACrB,QAAQ,MAAM,SAAS,CACvB,QAAQ,MAAM,SAAS"}

package/dist/reporters/markdown.d.ts

@@ -0,0 +1,9 @@
+import { EvalReport } from "@graphorin/observability/eval";
+
+//#region src/reporters/markdown.d.ts
+
+/** @stable */
+declare function renderMarkdownReport<I, O>(report: EvalReport<I, O>): string;
+//#endregion
+export { renderMarkdownReport };
+//# sourceMappingURL=markdown.d.ts.map

package/dist/reporters/markdown.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.d.ts","names":[],"sources":["../../src/reporters/markdown.ts"],"sourcesContent":[],"mappings":";;;;;iBAWgB,mCAAmC,WAAW,GAAG"}