@graphorin/evals 0.5.0

package/dist/loaders/dmr.d.ts

@@ -0,0 +1,31 @@
+import { MemoryEvalInput } from "./memory-eval.js";
+import { Case, Dataset } from "@graphorin/observability/eval";
+
+//#region src/loaders/dmr.d.ts
+
+/** @stable */
+interface LoadDmrOptions {
+  /** Local path to the DMR JSON (under `benchmarks/.datasets/`). */
+  readonly path: string;
+  /** Optional dataset name surfaced in `Dataset.metadata.name`. */
+  readonly name?: string;
+  /** Optional description surfaced in `Dataset.metadata.description`. */
+  readonly description?: string;
+}
+/**
+ * Read a DMR JSON file and return a fully-materialised {@link Dataset}
+ * of multi-session retrieval cases scored against the reference answer.
+ *
+ * @stable
+ */
+declare function loadDmrDataset(options: LoadDmrOptions): Promise<Dataset<MemoryEvalInput, string>>;
+/**
+ * Pure parser. Exported so tests can exercise the mapping without
+ * touching the filesystem.
+ *
+ * @stable
+ */
+declare function parseDmr(text: string): ReadonlyArray<Case<MemoryEvalInput, string>>;
+//#endregion
+export { LoadDmrOptions, loadDmrDataset, parseDmr };
+//# sourceMappingURL=dmr.d.ts.map

package/dist/loaders/dmr.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dmr.d.ts","names":[],"sources":["../../src/loaders/dmr.ts"],"sourcesContent":[],"mappings":";;;;;;UAqCiB,cAAA;;;;;;;;;;;;;;iBAeK,cAAA,UACX,iBACR,QAAQ,QAAQ;;;;;;;iBAmBH,QAAA,gBAAwB,cAAc,KAAK"}

package/dist/loaders/dmr.js

@@ -0,0 +1,121 @@
+import { readFile } from "node:fs/promises";
+
+//#region src/loaders/dmr.ts
+/**
+* DMR (Deep Memory Retrieval) loader — the MemGPT/Letta continuity set
+* derived from Multi-Session Chat (arXiv:2310.08560). DMR's published
+* shape is less standardised than LongMemEval/LOCOMO, so this loader is
+* deliberately tolerant: it accepts a JSON **array** of items, each
+* exposing a `question`, an `answer`, and the prior conversation under
+* either `sessions` or `haystack_sessions`. Sessions may be a bare
+* array of turns or an object wrapping the turns under
+* `turns` / `messages` / `dialog`; a turn may use `role` or `speaker`
+* and `content` or `text`.
+*
+* ```jsonc
+* {
+*   "id": "dmr-1",
+*   "question": "What is the user's sister's name?",
+*   "answer": "Anna",
+*   "sessions": [
+*     { "id": "s1", "turns": [ { "role": "user", "content": "..." } ] },
+*     [ { "speaker": "assistant", "text": "..." } ]
+*   ]
+* }
+* ```
+*
+* Every DMR case is treated as `multi-session` (its defining ability).
+* The exact field names should be reconciled against the downloaded
+* file the first time `scripts/fetch-eval-datasets.mjs` is run.
+*
+* @packageDocumentation
+*/
+/**
+* Read a DMR JSON file and return a fully-materialised {@link Dataset}
+* of multi-session retrieval cases scored against the reference answer.
+*
+* @stable
+*/
+async function loadDmrDataset(options) {
+	return {
+		cases: parseDmr(await readFile(options.path, "utf8")),
+		metadata: {
+			name: options.name ?? "dmr",
+			...options.description !== void 0 ? { description: options.description } : {},
+			createdAt: /* @__PURE__ */ new Date()
+		}
+	};
+}
+/**
+* Pure parser. Exported so tests can exercise the mapping without
+* touching the filesystem.
+*
+* @stable
+*/
+function parseDmr(text) {
+	const parsed = JSON.parse(text);
+	if (!Array.isArray(parsed)) throw new Error("[graphorin/evals] DMR dataset must be a JSON array of items.");
+	const items = parsed;
+	const out = [];
+	for (let i = 0; i < items.length; i++) {
+		const item = items[i];
+		if (!isRecord(item)) throw new Error(`[graphorin/evals] DMR item ${i} is not an object.`);
+		const question = asString(item.question);
+		if (question === void 0) throw new Error(`[graphorin/evals] DMR item ${i} is missing a 'question'.`);
+		const sessions = coerceSessions(item.sessions ?? item.haystack_sessions);
+		out.push({
+			id: asString(item.id) ?? `dmr-${i}`,
+			input: {
+				haystackSessions: sessions,
+				question,
+				ability: "multi-session"
+			},
+			expected: asString(item.answer) ?? "",
+			metadata: {
+				datasetName: "dmr",
+				ability: "multi-session"
+			}
+		});
+	}
+	return out;
+}
+function coerceSessions(value) {
+	const rawSessions = asArray(value);
+	const sessions = [];
+	for (let s = 0; s < rawSessions.length; s++) {
+		const raw = rawSessions[s];
+		const rawTurns = Array.isArray(raw) ? raw : isRecord(raw) ? asArray(raw.turns ?? raw.messages ?? raw.dialog) : [];
+		const id = isRecord(raw) ? asString(raw.id) ?? `session-${s}` : `session-${s}`;
+		const turns = [];
+		for (const t of rawTurns) {
+			if (!isRecord(t)) continue;
+			const content = asString(t.content) ?? asString(t.text);
+			if (content === void 0) continue;
+			const role = (asString(t.role) ?? asString(t.speaker)) === "assistant" ? "assistant" : "user";
+			const timestamp = asString(t.timestamp);
+			turns.push({
+				role,
+				content,
+				...timestamp !== void 0 ? { timestamp } : {}
+			});
+		}
+		sessions.push({
+			id,
+			turns
+		});
+	}
+	return sessions;
+}
+function asString(value) {
+	return typeof value === "string" ? value : void 0;
+}
+function asArray(value) {
+	return Array.isArray(value) ? value : [];
+}
+function isRecord(value) {
+	return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+//#endregion
+export { loadDmrDataset, parseDmr };
+//# sourceMappingURL=dmr.js.map

package/dist/loaders/dmr.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dmr.js","names":["parsed: unknown","out: Case<MemoryEvalInput, string>[]","sessions: MemoryEvalSession[]","turns: MemoryEvalTurn[]"],"sources":["../../src/loaders/dmr.ts"],"sourcesContent":["/**\n * DMR (Deep Memory Retrieval) loader — the MemGPT/Letta continuity set\n * derived from Multi-Session Chat (arXiv:2310.08560). DMR's published\n * shape is less standardised than LongMemEval/LOCOMO, so this loader is\n * deliberately tolerant: it accepts a JSON **array** of items, each\n * exposing a `question`, an `answer`, and the prior conversation under\n * either `sessions` or `haystack_sessions`. Sessions may be a bare\n * array of turns or an object wrapping the turns under\n * `turns` / `messages` / `dialog`; a turn may use `role` or `speaker`\n * and `content` or `text`.\n *\n * ```jsonc\n * {\n *   \"id\": \"dmr-1\",\n *   \"question\": \"What is the user's sister's name?\",\n *   \"answer\": \"Anna\",\n *   \"sessions\": [\n *     { \"id\": \"s1\", \"turns\": [ { \"role\": \"user\", \"content\": \"...\" } ] },\n *     [ { \"speaker\": \"assistant\", \"text\": \"...\" } ]\n *   ]\n * }\n * ```\n *\n * Every DMR case is treated as `multi-session` (its defining ability).\n * The exact field names should be reconciled against the downloaded\n * file the first time `scripts/fetch-eval-datasets.mjs` is run.\n *\n * @packageDocumentation\n */\n\nimport { readFile } from 'node:fs/promises';\n\nimport type { Case, Dataset } from '@graphorin/observability/eval';\n\nimport type { MemoryEvalInput, MemoryEvalSession, MemoryEvalTurn } from './memory-eval.js';\n\n/** @stable */\nexport interface LoadDmrOptions {\n  /** Local path to the DMR JSON (under `benchmarks/.datasets/`). */\n  readonly path: string;\n  /** Optional dataset name surfaced in `Dataset.metadata.name`. */\n  readonly name?: string;\n  /** Optional description surfaced in `Dataset.metadata.description`. */\n  readonly description?: string;\n}\n\n/**\n * Read a DMR JSON file and return a fully-materialised {@link Dataset}\n * of multi-session retrieval cases scored against the reference answer.\n *\n * @stable\n */\nexport async function loadDmrDataset(\n  options: LoadDmrOptions,\n): Promise<Dataset<MemoryEvalInput, string>> {\n  const text = await readFile(options.path, 'utf8');\n  const cases = parseDmr(text);\n  return {\n    cases,\n    metadata: {\n      name: options.name ?? 'dmr',\n      ...(options.description !== undefined ? { description: options.description } : {}),\n      createdAt: new Date(),\n    },\n  };\n}\n\n/**\n * Pure parser. Exported so tests can exercise the mapping without\n * touching the filesystem.\n *\n * @stable\n */\nexport function parseDmr(text: string): ReadonlyArray<Case<MemoryEvalInput, string>> {\n  const parsed: unknown = JSON.parse(text);\n  if (!Array.isArray(parsed)) {\n    throw new Error('[graphorin/evals] DMR dataset must be a JSON array of items.');\n  }\n  const items = parsed as ReadonlyArray<unknown>;\n  const out: Case<MemoryEvalInput, string>[] = [];\n  for (let i = 0; i < items.length; i++) {\n    const item = items[i];\n    if (!isRecord(item)) {\n      throw new Error(`[graphorin/evals] DMR item ${i} is not an object.`);\n    }\n    const question = asString(item.question);\n    if (question === undefined) {\n      throw new Error(`[graphorin/evals] DMR item ${i} is missing a 'question'.`);\n    }\n    const sessions = coerceSessions(item.sessions ?? item.haystack_sessions);\n    out.push({\n      id: asString(item.id) ?? `dmr-${i}`,\n      input: { haystackSessions: sessions, question, ability: 'multi-session' },\n      expected: asString(item.answer) ?? '',\n      metadata: { datasetName: 'dmr', ability: 'multi-session' },\n    });\n  }\n  return out;\n}\n\nfunction coerceSessions(value: unknown): MemoryEvalSession[] {\n  const rawSessions = asArray(value);\n  const sessions: MemoryEvalSession[] = [];\n  for (let s = 0; s < rawSessions.length; s++) {\n    const raw = rawSessions[s];\n    const rawTurns = Array.isArray(raw)\n      ? (raw as ReadonlyArray<unknown>)\n      : isRecord(raw)\n        ? asArray(raw.turns ?? raw.messages ?? raw.dialog)\n        : [];\n    const id = isRecord(raw) ? (asString(raw.id) ?? `session-${s}`) : `session-${s}`;\n    const turns: MemoryEvalTurn[] = [];\n    for (const t of rawTurns) {\n      if (!isRecord(t)) continue;\n      const content = asString(t.content) ?? asString(t.text);\n      if (content === undefined) continue;\n      const speaker = asString(t.role) ?? asString(t.speaker);\n      const role = speaker === 'assistant' ? 'assistant' : 'user';\n      const timestamp = asString(t.timestamp);\n      turns.push({ role, content, ...(timestamp !== undefined ? { timestamp } : {}) });\n    }\n    sessions.push({ id, turns });\n  }\n  return sessions;\n}\n\nfunction asString(value: unknown): string | undefined {\n  return typeof value === 'string' ? value : undefined;\n}\n\nfunction asArray(value: unknown): ReadonlyArray<unknown> {\n  return Array.isArray(value) ? (value as ReadonlyArray<unknown>) : [];\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n  return value !== null && typeof value === 'object' && !Array.isArray(value);\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoDA,eAAsB,eACpB,SAC2C;AAG3C,QAAO;EACL,OAFY,SADD,MAAM,SAAS,QAAQ,MAAM,OAAO,CACrB;EAG1B,UAAU;GACR,MAAM,QAAQ,QAAQ;GACtB,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;GACjF,2BAAW,IAAI,MAAM;GACtB;EACF;;;;;;;;AASH,SAAgB,SAAS,MAA4D;CACnF,MAAMA,SAAkB,KAAK,MAAM,KAAK;AACxC,KAAI,CAAC,MAAM,QAAQ,OAAO,CACxB,OAAM,IAAI,MAAM,+DAA+D;CAEjF,MAAM,QAAQ;CACd,MAAMC,MAAuC,EAAE;AAC/C,MAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;EACrC,MAAM,OAAO,MAAM;AACnB,MAAI,CAAC,SAAS,KAAK,CACjB,OAAM,IAAI,MAAM,8BAA8B,EAAE,oBAAoB;EAEtE,MAAM,WAAW,SAAS,KAAK,SAAS;AACxC,MAAI,aAAa,OACf,OAAM,IAAI,MAAM,8BAA8B,EAAE,2BAA2B;EAE7E,MAAM,WAAW,eAAe,KAAK,YAAY,KAAK,kBAAkB;AACxE,MAAI,KAAK;GACP,IAAI,SAAS,KAAK,GAAG,IAAI,OAAO;GAChC,OAAO;IAAE,kBAAkB;IAAU;IAAU,SAAS;IAAiB;GACzE,UAAU,SAAS,KAAK,OAAO,IAAI;GACnC,UAAU;IAAE,aAAa;IAAO,SAAS;IAAiB;GAC3D,CAAC;;AAEJ,QAAO;;AAGT,SAAS,eAAe,OAAqC;CAC3D,MAAM,cAAc,QAAQ,MAAM;CAClC,MAAMC,WAAgC,EAAE;AACxC,MAAK,IAAI,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;EAC3C,MAAM,MAAM,YAAY;EACxB,MAAM,WAAW,MAAM,QAAQ,IAAI,GAC9B,MACD,SAAS,IAAI,GACX,QAAQ,IAAI,SAAS,IAAI,YAAY,IAAI,OAAO,GAChD,EAAE;EACR,MAAM,KAAK,SAAS,IAAI,GAAI,SAAS,IAAI,GAAG,IAAI,WAAW,MAAO,WAAW;EAC7E,MAAMC,QAA0B,EAAE;AAClC,OAAK,MAAM,KAAK,UAAU;AACxB,OAAI,CAAC,SAAS,EAAE,CAAE;GAClB,MAAM,UAAU,SAAS,EAAE,QAAQ,IAAI,SAAS,EAAE,KAAK;AACvD,OAAI,YAAY,OAAW;GAE3B,MAAM,QADU,SAAS,EAAE,KAAK,IAAI,SAAS,EAAE,QAAQ,MAC9B,cAAc,cAAc;GACrD,MAAM,YAAY,SAAS,EAAE,UAAU;AACvC,SAAM,KAAK;IAAE;IAAM;IAAS,GAAI,cAAc,SAAY,EAAE,WAAW,GAAG,EAAE;IAAG,CAAC;;AAElF,WAAS,KAAK;GAAE;GAAI;GAAO,CAAC;;AAE9B,QAAO;;AAGT,SAAS,SAAS,OAAoC;AACpD,QAAO,OAAO,UAAU,WAAW,QAAQ;;AAG7C,SAAS,QAAQ,OAAwC;AACvD,QAAO,MAAM,QAAQ,MAAM,GAAI,QAAmC,EAAE;;AAGtE,SAAS,SAAS,OAAkD;AAClE,QAAO,UAAU,QAAQ,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,MAAM"}

package/dist/loaders/from-traces.d.ts

@@ -0,0 +1,32 @@
+import { Case, Dataset } from "@graphorin/observability/eval";
+
+//#region src/loaders/from-traces.d.ts
+
+/** @stable */
+interface TraceEvent {
+  readonly runId: string;
+  readonly type: string;
+  readonly payload: Readonly<Record<string, unknown>>;
+  readonly timestamp?: string;
+}
+/** @stable */
+interface FromTracesOptions<I, O> {
+  /** Distil one `Case<I, O>` from every group of events sharing a `runId`. */
+  readonly extract: (events: ReadonlyArray<TraceEvent>) => Case<I, O> | null;
+  /** Optional name surfaced in `Dataset.metadata.name`. */
+  readonly name?: string;
+  /** Optional description surfaced in `Dataset.metadata.description`. */
+  readonly description?: string;
+}
+/** @stable */
+declare function loadDatasetFromTraces<I, O>(path: string, options: FromTracesOptions<I, O>): Promise<Dataset<I, O>>;
+/**
+ * Pure parser for the trace JSONL format. Exported so tests can
+ * exercise the extraction without touching the filesystem.
+ *
+ * @stable
+ */
+declare function groupAndExtract<I, O>(text: string, options: FromTracesOptions<I, O>): Dataset<I, O>;
+//#endregion
+export { FromTracesOptions, TraceEvent, groupAndExtract, loadDatasetFromTraces };
+//# sourceMappingURL=from-traces.d.ts.map

package/dist/loaders/from-traces.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"from-traces.d.ts","names":[],"sources":["../../src/loaders/from-traces.ts"],"sourcesContent":[],"mappings":";;;;;AAiCsB,UAlBL,UAAA,CAkB0B;EAEd,SAAA,KAAA,EAAA,MAAA;EAAG,SAAA,IAAA,EAAA,MAAA;EAArB,SAAA,OAAA,EAjBS,QAiBT,CAjBkB,MAiBlB,CAAA,MAAA,EAAA,OAAA,CAAA,CAAA;EACQ,SAAA,SAAA,CAAA,EAAA,MAAA;;;AAAhB,UAbc,iBAad,CAAA,CAAA,EAAA,CAAA,CAAA,CAAA;EAAO;EAWM,SAAA,OAAA,EAAe,CAAA,MAAA,EAtBF,aAsBE,CAtBY,UAsBZ,CAAA,EAAA,GAtB4B,IAsB5B,CAtBiC,CAsBjC,EAtBoC,CAsBpC,CAAA,GAAA,IAAA;EAEF;EAAG,SAAA,IAAA,CAAA,EAAA,MAAA;EAArB;EACA,SAAA,WAAA,CAAA,EAAA,MAAA;;;AAAD,iBAjBY,qBAiBZ,CAAA,CAAA,EAAA,CAAA,CAAA,CAAA,IAAA,EAAA,MAAA,EAAA,OAAA,EAfC,iBAeD,CAfmB,CAenB,EAfsB,CAetB,CAAA,CAAA,EAdP,OAcO,CAdC,OAcD,CAdS,CAcT,EAdY,CAcZ,CAAA,CAAA;;;;;;;iBAHM,6CAEL,kBAAkB,GAAG,KAC7B,QAAQ,GAAG"}

package/dist/loaders/from-traces.js

@@ -0,0 +1,66 @@
+import { readFile } from "node:fs/promises";
+
+//#region src/loaders/from-traces.ts
+/**
+* `loadDatasetFromTraces` — distil a dataset from the framework's
+* replay log. The caller supplies the JSONL replay file and a small
+* extraction function that pulls the `(input, output)` pair out of
+* each event group; the loader handles the JSONL parsing + grouping
+* by `runId`.
+*
+* @packageDocumentation
+*/
+/** @stable */
+async function loadDatasetFromTraces(path, options) {
+	return groupAndExtract(await readFile(path, "utf8"), options);
+}
+/**
+* Pure parser for the trace JSONL format. Exported so tests can
+* exercise the extraction without touching the filesystem.
+*
+* @stable
+*/
+function groupAndExtract(text, options) {
+	const lines = text.split(/\r?\n/);
+	const groups = /* @__PURE__ */ new Map();
+	for (const raw of lines) {
+		if (raw.trim().length === 0) continue;
+		let parsed;
+		try {
+			parsed = JSON.parse(raw);
+		} catch {
+			continue;
+		}
+		if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) continue;
+		const obj = parsed;
+		const runId = typeof obj.runId === "string" ? obj.runId : null;
+		const type = typeof obj.type === "string" ? obj.type : null;
+		if (runId === null || type === null) continue;
+		const event = {
+			runId,
+			type,
+			payload: obj.payload !== null && typeof obj.payload === "object" && !Array.isArray(obj.payload) ? obj.payload : {},
+			...typeof obj.timestamp === "string" ? { timestamp: obj.timestamp } : {}
+		};
+		const existing = groups.get(runId);
+		if (existing === void 0) groups.set(runId, [event]);
+		else existing.push(event);
+	}
+	const cases = [];
+	for (const events of groups.values()) {
+		const sample = options.extract(events);
+		if (sample !== null) cases.push(sample);
+	}
+	return {
+		cases,
+		metadata: {
+			...options.name !== void 0 ? { name: options.name } : {},
+			...options.description !== void 0 ? { description: options.description } : {},
+			createdAt: /* @__PURE__ */ new Date()
+		}
+	};
+}
+
+//#endregion
+export { groupAndExtract, loadDatasetFromTraces };
+//# sourceMappingURL=from-traces.js.map

package/dist/loaders/from-traces.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"from-traces.js","names":["parsed: unknown","event: TraceEvent","cases: Case<I, O>[]"],"sources":["../../src/loaders/from-traces.ts"],"sourcesContent":["/**\n * `loadDatasetFromTraces` — distil a dataset from the framework's\n * replay log. The caller supplies the JSONL replay file and a small\n * extraction function that pulls the `(input, output)` pair out of\n * each event group; the loader handles the JSONL parsing + grouping\n * by `runId`.\n *\n * @packageDocumentation\n */\n\nimport { readFile } from 'node:fs/promises';\n\nimport type { Case, Dataset } from '@graphorin/observability/eval';\n\n/** @stable */\nexport interface TraceEvent {\n  readonly runId: string;\n  readonly type: string;\n  readonly payload: Readonly<Record<string, unknown>>;\n  readonly timestamp?: string;\n}\n\n/** @stable */\nexport interface FromTracesOptions<I, O> {\n  /** Distil one `Case<I, O>` from every group of events sharing a `runId`. */\n  readonly extract: (events: ReadonlyArray<TraceEvent>) => Case<I, O> | null;\n  /** Optional name surfaced in `Dataset.metadata.name`. */\n  readonly name?: string;\n  /** Optional description surfaced in `Dataset.metadata.description`. */\n  readonly description?: string;\n}\n\n/** @stable */\nexport async function loadDatasetFromTraces<I, O>(\n  path: string,\n  options: FromTracesOptions<I, O>,\n): Promise<Dataset<I, O>> {\n  const text = await readFile(path, 'utf8');\n  return groupAndExtract<I, O>(text, options);\n}\n\n/**\n * Pure parser for the trace JSONL format. Exported so tests can\n * exercise the extraction without touching the filesystem.\n *\n * @stable\n */\nexport function groupAndExtract<I, O>(\n  text: string,\n  options: FromTracesOptions<I, O>,\n): Dataset<I, O> {\n  const lines = text.split(/\\r?\\n/);\n  const groups = new Map<string, TraceEvent[]>();\n  for (const raw of lines) {\n    if (raw.trim().length === 0) continue;\n    let parsed: unknown;\n    try {\n      parsed = JSON.parse(raw);\n    } catch {\n      continue;\n    }\n    if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) continue;\n    const obj = parsed as Record<string, unknown>;\n    const runId = typeof obj.runId === 'string' ? obj.runId : null;\n    const type = typeof obj.type === 'string' ? obj.type : null;\n    if (runId === null || type === null) continue;\n    const payload =\n      obj.payload !== null && typeof obj.payload === 'object' && !Array.isArray(obj.payload)\n        ? (obj.payload as Readonly<Record<string, unknown>>)\n        : ({} as Readonly<Record<string, unknown>>);\n    const event: TraceEvent = {\n      runId,\n      type,\n      payload,\n      ...(typeof obj.timestamp === 'string' ? { timestamp: obj.timestamp } : {}),\n    };\n    const existing = groups.get(runId);\n    if (existing === undefined) groups.set(runId, [event]);\n    else existing.push(event);\n  }\n  const cases: Case<I, O>[] = [];\n  for (const events of groups.values()) {\n    const sample = options.extract(events);\n    if (sample !== null) cases.push(sample);\n  }\n  const meta: Dataset<I, O>['metadata'] = {\n    ...(options.name !== undefined ? { name: options.name } : {}),\n    ...(options.description !== undefined ? { description: options.description } : {}),\n    createdAt: new Date(),\n  };\n  return { cases, metadata: meta };\n}\n"],"mappings":";;;;;;;;;;;;;AAiCA,eAAsB,sBACpB,MACA,SACwB;AAExB,QAAO,gBADM,MAAM,SAAS,MAAM,OAAO,EACN,QAAQ;;;;;;;;AAS7C,SAAgB,gBACd,MACA,SACe;CACf,MAAM,QAAQ,KAAK,MAAM,QAAQ;CACjC,MAAM,yBAAS,IAAI,KAA2B;AAC9C,MAAK,MAAM,OAAO,OAAO;AACvB,MAAI,IAAI,MAAM,CAAC,WAAW,EAAG;EAC7B,IAAIA;AACJ,MAAI;AACF,YAAS,KAAK,MAAM,IAAI;UAClB;AACN;;AAEF,MAAI,WAAW,QAAQ,OAAO,WAAW,YAAY,MAAM,QAAQ,OAAO,CAAE;EAC5E,MAAM,MAAM;EACZ,MAAM,QAAQ,OAAO,IAAI,UAAU,WAAW,IAAI,QAAQ;EAC1D,MAAM,OAAO,OAAO,IAAI,SAAS,WAAW,IAAI,OAAO;AACvD,MAAI,UAAU,QAAQ,SAAS,KAAM;EAKrC,MAAMC,QAAoB;GACxB;GACA;GACA,SANA,IAAI,YAAY,QAAQ,OAAO,IAAI,YAAY,YAAY,CAAC,MAAM,QAAQ,IAAI,QAAQ,GACjF,IAAI,UACJ,EAAE;GAKP,GAAI,OAAO,IAAI,cAAc,WAAW,EAAE,WAAW,IAAI,WAAW,GAAG,EAAE;GAC1E;EACD,MAAM,WAAW,OAAO,IAAI,MAAM;AAClC,MAAI,aAAa,OAAW,QAAO,IAAI,OAAO,CAAC,MAAM,CAAC;MACjD,UAAS,KAAK,MAAM;;CAE3B,MAAMC,QAAsB,EAAE;AAC9B,MAAK,MAAM,UAAU,OAAO,QAAQ,EAAE;EACpC,MAAM,SAAS,QAAQ,QAAQ,OAAO;AACtC,MAAI,WAAW,KAAM,OAAM,KAAK,OAAO;;AAOzC,QAAO;EAAE;EAAO,UALwB;GACtC,GAAI,QAAQ,SAAS,SAAY,EAAE,MAAM,QAAQ,MAAM,GAAG,EAAE;GAC5D,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;GACjF,2BAAW,IAAI,MAAM;GACtB;EAC+B"}

package/dist/loaders/index.d.ts

@@ -0,0 +1,9 @@
+import { LoadCsvOptions, loadCsvDataset, parseCsv } from "./csv.js";
+import { MemoryEvalAbility, MemoryEvalInput, MemoryEvalSession, MemoryEvalTurn } from "./memory-eval.js";
+import { LoadDmrOptions, loadDmrDataset, parseDmr } from "./dmr.js";
+import { FromTracesOptions, TraceEvent, groupAndExtract, loadDatasetFromTraces } from "./from-traces.js";
+import { fromIterable } from "./iterable.js";
+import { LoadJsonlOptions, loadJsonlDataset, parseJsonl } from "./jsonl.js";
+import { LoadLocomoOptions, loadLocomoDataset, parseLocomo } from "./locomo.js";
+import { LoadLongMemEvalOptions, loadLongMemEvalDataset, parseLongMemEval } from "./longmemeval.js";
+export { type FromTracesOptions, type LoadCsvOptions, type LoadDmrOptions, type LoadJsonlOptions, type LoadLocomoOptions, type LoadLongMemEvalOptions, type MemoryEvalAbility, type MemoryEvalInput, type MemoryEvalSession, type MemoryEvalTurn, type TraceEvent, fromIterable, groupAndExtract, loadCsvDataset, loadDatasetFromTraces, loadDmrDataset, loadJsonlDataset, loadLocomoDataset, loadLongMemEvalDataset, parseCsv, parseDmr, parseJsonl, parseLocomo, parseLongMemEval };

package/dist/loaders/index.js

@@ -0,0 +1,9 @@
+import { loadCsvDataset, parseCsv } from "./csv.js";
+import { loadDmrDataset, parseDmr } from "./dmr.js";
+import { groupAndExtract, loadDatasetFromTraces } from "./from-traces.js";
+import { fromIterable } from "./iterable.js";
+import { loadJsonlDataset, parseJsonl } from "./jsonl.js";
+import { loadLocomoDataset, parseLocomo } from "./locomo.js";
+import { loadLongMemEvalDataset, parseLongMemEval } from "./longmemeval.js";
+
+export { fromIterable, groupAndExtract, loadCsvDataset, loadDatasetFromTraces, loadDmrDataset, loadJsonlDataset, loadLocomoDataset, loadLongMemEvalDataset, parseCsv, parseDmr, parseJsonl, parseLocomo, parseLongMemEval };

package/dist/loaders/iterable.d.ts

@@ -0,0 +1,12 @@
+import { Case, Dataset } from "@graphorin/observability/eval";
+
+//#region src/loaders/iterable.d.ts
+
+/** @stable */
+declare function fromIterable<I, O = unknown>(cases: Iterable<Case<I, O>> | ReadonlyArray<Case<I, O>>, options?: {
+  readonly name?: string;
+  readonly description?: string;
+}): Dataset<I, O>;
+//#endregion
+export { fromIterable };
+//# sourceMappingURL=iterable.d.ts.map

package/dist/loaders/iterable.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"iterable.d.ts","names":[],"sources":["../../src/loaders/iterable.ts"],"sourcesContent":[],"mappings":";;;;;AAWmD,iBADnC,YACmC,CAAA,CAAA,EAAA,IAAA,OAAA,CAAA,CAAA,KAAA,EAA1C,QAA0C,CAAjC,IAAiC,CAA5B,CAA4B,EAAzB,CAAyB,CAAA,CAAA,GAAnB,aAAmB,CAAL,IAAK,CAAA,CAAA,EAAG,CAAH,CAAA,CAAA,EAAA,OAAnB,CAAmB,EAAA;EAAG,SAAA,IAAA,CAAA,EAAA,MAAA;EAAR,SAAA,WAAA,CAAA,EAAA,MAAA;CAAd,CAAA,EAE7B,OAF6B,CAErB,CAFqB,EAElB,CAFkB,CAAA"}

package/dist/loaders/iterable.js

@@ -0,0 +1,16 @@
+//#region src/loaders/iterable.ts
+/** @stable */
+function fromIterable(cases, options = {}) {
+	return {
+		cases: Array.isArray(cases) ? cases : Array.from(cases),
+		metadata: {
+			...options.name !== void 0 ? { name: options.name } : {},
+			...options.description !== void 0 ? { description: options.description } : {},
+			createdAt: /* @__PURE__ */ new Date()
+		}
+	};
+}
+
+//#endregion
+export { fromIterable };
+//# sourceMappingURL=iterable.js.map

package/dist/loaders/iterable.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"iterable.js","names":[],"sources":["../../src/loaders/iterable.ts"],"sourcesContent":["/**\n * `fromIterable` — wrap an in-memory array / iterable as a\n * {@link Dataset}. Useful for tests + ad-hoc data.\n *\n * @packageDocumentation\n */\n\nimport type { Case, Dataset } from '@graphorin/observability/eval';\n\n/** @stable */\nexport function fromIterable<I, O = unknown>(\n  cases: Iterable<Case<I, O>> | ReadonlyArray<Case<I, O>>,\n  options: { readonly name?: string; readonly description?: string } = {},\n): Dataset<I, O> {\n  const arr = Array.isArray(cases) ? (cases as ReadonlyArray<Case<I, O>>) : Array.from(cases);\n  const meta: Dataset<I, O>['metadata'] = {\n    ...(options.name !== undefined ? { name: options.name } : {}),\n    ...(options.description !== undefined ? { description: options.description } : {}),\n    createdAt: new Date(),\n  };\n  return { cases: arr, metadata: meta };\n}\n"],"mappings":";;AAUA,SAAgB,aACd,OACA,UAAqE,EAAE,EACxD;AAOf,QAAO;EAAE,OANG,MAAM,QAAQ,MAAM,GAAI,QAAsC,MAAM,KAAK,MAAM;EAMtE,UALmB;GACtC,GAAI,QAAQ,SAAS,SAAY,EAAE,MAAM,QAAQ,MAAM,GAAG,EAAE;GAC5D,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;GACjF,2BAAW,IAAI,MAAM;GACtB;EACoC"}

package/dist/loaders/jsonl.d.ts

@@ -0,0 +1,34 @@
+import { Case, Dataset } from "@graphorin/observability/eval";
+
+//#region src/loaders/jsonl.d.ts
+
+/** @stable */
+interface LoadJsonlOptions {
+  /** Optional dataset name surfaced in `Dataset.metadata.name`. */
+  readonly name?: string;
+  /** Optional description surfaced in `Dataset.metadata.description`. */
+  readonly description?: string;
+  /**
+   * Map a parsed line into a `Case`. Default forwards the line
+   * verbatim. Override to translate column names or coerce types.
+   */
+  readonly mapper?: (line: Record<string, unknown>, index: number) => Case<unknown, unknown>;
+}
+/**
+ * Read a JSONL file and return a fully-materialised {@link Dataset}.
+ * Empty lines are skipped; malformed lines throw with the line
+ * number.
+ *
+ * @stable
+ */
+declare function loadJsonlDataset(path: string, options?: LoadJsonlOptions): Promise<Dataset<unknown, unknown>>;
+/**
+ * Pure parser. Exported separately so tests can exercise the line-by-
+ * line behaviour without touching the filesystem.
+ *
+ * @stable
+ */
+declare function parseJsonl(text: string, mapper?: LoadJsonlOptions['mapper']): ReadonlyArray<Case<unknown, unknown>>;
+//#endregion
+export { LoadJsonlOptions, loadJsonlDataset, parseJsonl };
+//# sourceMappingURL=jsonl.d.ts.map

package/dist/loaders/jsonl.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonl.d.ts","names":[],"sources":["../../src/loaders/jsonl.ts"],"sourcesContent":[],"mappings":";;;;;AAkCU,UAtBO,gBAAA,CAsBP;EAiBM;EAEL,SAAA,IAAA,CAAA,EAAA,MAAA;EACM;EAAd,SAAA,WAAA,CAAA,EAAA,MAAA;EAAa;;;;2BAjCW,2CAA2C;;;;;;;;;iBAUhD,gBAAA,yBAEX,mBACR,QAAQ;;;;;;;iBAiBK,UAAA,wBAEL,6BACR,cAAc"}

package/dist/loaders/jsonl.js

@@ -0,0 +1,64 @@
+import { readFile } from "node:fs/promises";
+
+//#region src/loaders/jsonl.ts
+/**
+* JSONL dataset loader. Each line is a JSON object with `input` and
+* optional `expected` / `metadata` / `id` fields.
+*
+* @packageDocumentation
+*/
+/**
+* Read a JSONL file and return a fully-materialised {@link Dataset}.
+* Empty lines are skipped; malformed lines throw with the line
+* number.
+*
+* @stable
+*/
+async function loadJsonlDataset(path, options = {}) {
+	return {
+		cases: parseJsonl(await readFile(path, "utf8"), options.mapper),
+		metadata: {
+			...options.name !== void 0 ? { name: options.name } : {},
+			...options.description !== void 0 ? { description: options.description } : {},
+			createdAt: /* @__PURE__ */ new Date()
+		}
+	};
+}
+/**
+* Pure parser. Exported separately so tests can exercise the line-by-
+* line behaviour without touching the filesystem.
+*
+* @stable
+*/
+function parseJsonl(text, mapper) {
+	const out = [];
+	const lines = text.split(/\r?\n/);
+	for (let i = 0; i < lines.length; i++) {
+		const raw = lines[i] ?? "";
+		if (raw.trim().length === 0) continue;
+		let parsed;
+		try {
+			parsed = JSON.parse(raw);
+		} catch (err) {
+			throw new Error(`[graphorin/evals] failed to parse JSONL at line ${i + 1}: ${err.message}`, { cause: err });
+		}
+		if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error(`[graphorin/evals] JSONL line ${i + 1} must be a JSON object, got ${typeof parsed}.`);
+		const obj = parsed;
+		const mapped = mapper !== void 0 ? mapper(obj, i) : defaultMapper(obj, i);
+		out.push(mapped);
+	}
+	return out;
+}
+function defaultMapper(line, _index) {
+	if (!("input" in line)) throw new Error(`[graphorin/evals] JSONL row missing required 'input' field.`);
+	return {
+		...typeof line.id === "string" ? { id: line.id } : {},
+		input: line.input,
+		..."expected" in line ? { expected: line.expected } : {},
+		...line.metadata !== void 0 && typeof line.metadata === "object" && line.metadata !== null ? { metadata: line.metadata } : {}
+	};
+}
+
+//#endregion
+export { loadJsonlDataset, parseJsonl };
+//# sourceMappingURL=jsonl.js.map

package/dist/loaders/jsonl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonl.js","names":["out: Case<unknown, unknown>[]","parsed: unknown"],"sources":["../../src/loaders/jsonl.ts"],"sourcesContent":["/**\n * JSONL dataset loader. Each line is a JSON object with `input` and\n * optional `expected` / `metadata` / `id` fields.\n *\n * @packageDocumentation\n */\n\nimport { readFile } from 'node:fs/promises';\n\nimport type { Case, Dataset } from '@graphorin/observability/eval';\n\n/** @stable */\nexport interface LoadJsonlOptions {\n  /** Optional dataset name surfaced in `Dataset.metadata.name`. */\n  readonly name?: string;\n  /** Optional description surfaced in `Dataset.metadata.description`. */\n  readonly description?: string;\n  /**\n   * Map a parsed line into a `Case`. Default forwards the line\n   * verbatim. Override to translate column names or coerce types.\n   */\n  readonly mapper?: (line: Record<string, unknown>, index: number) => Case<unknown, unknown>;\n}\n\n/**\n * Read a JSONL file and return a fully-materialised {@link Dataset}.\n * Empty lines are skipped; malformed lines throw with the line\n * number.\n *\n * @stable\n */\nexport async function loadJsonlDataset(\n  path: string,\n  options: LoadJsonlOptions = {},\n): Promise<Dataset<unknown, unknown>> {\n  const text = await readFile(path, 'utf8');\n  const cases = parseJsonl(text, options.mapper);\n  const meta: Dataset<unknown, unknown>['metadata'] = {\n    ...(options.name !== undefined ? { name: options.name } : {}),\n    ...(options.description !== undefined ? { description: options.description } : {}),\n    createdAt: new Date(),\n  };\n  return { cases, metadata: meta };\n}\n\n/**\n * Pure parser. Exported separately so tests can exercise the line-by-\n * line behaviour without touching the filesystem.\n *\n * @stable\n */\nexport function parseJsonl(\n  text: string,\n  mapper?: LoadJsonlOptions['mapper'],\n): ReadonlyArray<Case<unknown, unknown>> {\n  const out: Case<unknown, unknown>[] = [];\n  const lines = text.split(/\\r?\\n/);\n  for (let i = 0; i < lines.length; i++) {\n    const raw = lines[i] ?? '';\n    if (raw.trim().length === 0) continue;\n    let parsed: unknown;\n    try {\n      parsed = JSON.parse(raw);\n    } catch (err) {\n      throw new Error(\n        `[graphorin/evals] failed to parse JSONL at line ${i + 1}: ${(err as Error).message}`,\n        { cause: err },\n      );\n    }\n    if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {\n      throw new Error(\n        `[graphorin/evals] JSONL line ${i + 1} must be a JSON object, got ${typeof parsed}.`,\n      );\n    }\n    const obj = parsed as Record<string, unknown>;\n    const mapped = mapper !== undefined ? mapper(obj, i) : defaultMapper(obj, i);\n    out.push(mapped);\n  }\n  return out;\n}\n\nfunction defaultMapper(line: Record<string, unknown>, _index: number): Case<unknown, unknown> {\n  if (!('input' in line)) {\n    throw new Error(`[graphorin/evals] JSONL row missing required 'input' field.`);\n  }\n  return {\n    ...(typeof line.id === 'string' ? { id: line.id } : {}),\n    input: line.input,\n    ...('expected' in line ? { expected: line.expected } : {}),\n    ...(line.metadata !== undefined && typeof line.metadata === 'object' && line.metadata !== null\n      ? { metadata: line.metadata as Readonly<Record<string, unknown>> }\n      : {}),\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA+BA,eAAsB,iBACpB,MACA,UAA4B,EAAE,EACM;AAQpC,QAAO;EAAE,OANK,WADD,MAAM,SAAS,MAAM,OAAO,EACV,QAAQ,OAAO;EAM9B,UALoC;GAClD,GAAI,QAAQ,SAAS,SAAY,EAAE,MAAM,QAAQ,MAAM,GAAG,EAAE;GAC5D,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;GACjF,2BAAW,IAAI,MAAM;GACtB;EAC+B;;;;;;;;AASlC,SAAgB,WACd,MACA,QACuC;CACvC,MAAMA,MAAgC,EAAE;CACxC,MAAM,QAAQ,KAAK,MAAM,QAAQ;AACjC,MAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;EACrC,MAAM,MAAM,MAAM,MAAM;AACxB,MAAI,IAAI,MAAM,CAAC,WAAW,EAAG;EAC7B,IAAIC;AACJ,MAAI;AACF,YAAS,KAAK,MAAM,IAAI;WACjB,KAAK;AACZ,SAAM,IAAI,MACR,mDAAmD,IAAI,EAAE,IAAK,IAAc,WAC5E,EAAE,OAAO,KAAK,CACf;;AAEH,MAAI,WAAW,QAAQ,OAAO,WAAW,YAAY,MAAM,QAAQ,OAAO,CACxE,OAAM,IAAI,MACR,gCAAgC,IAAI,EAAE,8BAA8B,OAAO,OAAO,GACnF;EAEH,MAAM,MAAM;EACZ,MAAM,SAAS,WAAW,SAAY,OAAO,KAAK,EAAE,GAAG,cAAc,KAAK,EAAE;AAC5E,MAAI,KAAK,OAAO;;AAElB,QAAO;;AAGT,SAAS,cAAc,MAA+B,QAAwC;AAC5F,KAAI,EAAE,WAAW,MACf,OAAM,IAAI,MAAM,8DAA8D;AAEhF,QAAO;EACL,GAAI,OAAO,KAAK,OAAO,WAAW,EAAE,IAAI,KAAK,IAAI,GAAG,EAAE;EACtD,OAAO,KAAK;EACZ,GAAI,cAAc,OAAO,EAAE,UAAU,KAAK,UAAU,GAAG,EAAE;EACzD,GAAI,KAAK,aAAa,UAAa,OAAO,KAAK,aAAa,YAAY,KAAK,aAAa,OACtF,EAAE,UAAU,KAAK,UAA+C,GAChE,EAAE;EACP"}

package/dist/loaders/locomo.d.ts

@@ -0,0 +1,32 @@
+import { MemoryEvalInput } from "./memory-eval.js";
+import { Case, Dataset } from "@graphorin/observability/eval";
+
+//#region src/loaders/locomo.d.ts
+
+/** @stable */
+interface LoadLocomoOptions {
+  /** Local path to the LOCOMO JSON (under `benchmarks/.datasets/`). */
+  readonly path: string;
+  /** Optional dataset name surfaced in `Dataset.metadata.name`. */
+  readonly name?: string;
+  /** Optional description surfaced in `Dataset.metadata.description`. */
+  readonly description?: string;
+}
+/**
+ * Read a LOCOMO JSON file and return a fully-materialised
+ * {@link Dataset} — one case per QA pair, scored against the reference
+ * answer string (LLM-judge "J").
+ *
+ * @stable
+ */
+declare function loadLocomoDataset(options: LoadLocomoOptions): Promise<Dataset<MemoryEvalInput, string>>;
+/**
+ * Pure parser. Exported so tests can exercise the mapping without
+ * touching the filesystem.
+ *
+ * @stable
+ */
+declare function parseLocomo(text: string): ReadonlyArray<Case<MemoryEvalInput, string>>;
+//#endregion
+export { LoadLocomoOptions, loadLocomoDataset, parseLocomo };
+//# sourceMappingURL=locomo.d.ts.map

package/dist/loaders/locomo.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"locomo.d.ts","names":[],"sources":["../../src/loaders/locomo.ts"],"sourcesContent":[],"mappings":";;;;;;UA6CiB,iBAAA;;;;;;;;;;;;;;;iBAgBK,iBAAA,UACX,oBACR,QAAQ,QAAQ;;;;;;;iBAmBH,WAAA,gBAA2B,cAAc,KAAK"}

package/dist/loaders/locomo.js

@@ -0,0 +1,155 @@
+import { readFile } from "node:fs/promises";
+
+//#region src/loaders/locomo.ts
+/**
+* Real LOCOMO dataset loader (arXiv:2402.17753 —
+* https://snap-research.github.io/locomo/). LOCOMO ships as a JSON
+* **array** of samples; each sample bundles one multi-session
+* conversation with many QA pairs over it:
+*
+* ```jsonc
+* {
+*   "sample_id": "conv-26",
+*   "conversation": {
+*     "speaker_a": "Caroline",
+*     "speaker_b": "Melanie",
+*     "session_1_date_time": "1:56 pm on 8 May, 2023",
+*     "session_1": [ { "speaker": "Caroline", "dia_id": "D1:1", "text": "..." }, ... ],
+*     "session_2_date_time": "...",
+*     "session_2": [ ... ]
+*   },
+*   "qa": [
+*     { "question": "...", "answer": "...", "evidence": ["D1:1"], "category": 1 },
+*     { "question": "...", "category": 5, "adversarial_answer": "Not mentioned ..." }
+*   ]
+* }
+* ```
+*
+* Each QA pair becomes one {@link MemoryEvalInput} case that shares the
+* sample's sessions. LOCOMO categories map to abilities as:
+* `1`→multi-session (multi-hop), `2`→temporal, `5`→abstention
+* (adversarial), everything else (`3` open-domain, `4` single-hop)
+* →info-extraction.
+*
+* @packageDocumentation
+*/
+/**
+* Read a LOCOMO JSON file and return a fully-materialised
+* {@link Dataset} — one case per QA pair, scored against the reference
+* answer string (LLM-judge "J").
+*
+* @stable
+*/
+async function loadLocomoDataset(options) {
+	return {
+		cases: parseLocomo(await readFile(options.path, "utf8")),
+		metadata: {
+			name: options.name ?? "locomo",
+			...options.description !== void 0 ? { description: options.description } : {},
+			createdAt: /* @__PURE__ */ new Date()
+		}
+	};
+}
+/**
+* Pure parser. Exported so tests can exercise the mapping without
+* touching the filesystem.
+*
+* @stable
+*/
+function parseLocomo(text) {
+	const parsed = JSON.parse(text);
+	if (!Array.isArray(parsed)) throw new Error("[graphorin/evals] LOCOMO dataset must be a JSON array of samples.");
+	const samples = parsed;
+	const out = [];
+	for (let i = 0; i < samples.length; i++) {
+		const sample = samples[i];
+		if (sample === null || typeof sample !== "object") throw new Error(`[graphorin/evals] LOCOMO sample ${i} is not an object.`);
+		appendSampleCases(sample, i, out);
+	}
+	return out;
+}
+function appendSampleCases(sample, index, out) {
+	const sampleId = asString(sample.sample_id) ?? `locomo-${index}`;
+	const sessions = extractSessions(isRecord(sample.conversation) ? sample.conversation : {});
+	const qa = asArray(sample.qa);
+	for (let q = 0; q < qa.length; q++) {
+		const item = qa[q];
+		if (!isRecord(item)) continue;
+		const question = asString(item.question);
+		if (question === void 0) continue;
+		const category = typeof item.category === "number" ? item.category : void 0;
+		const ability = mapCategory(category);
+		const answer = asString(item.answer) ?? asString(item.adversarial_answer) ?? "";
+		const evidence = asArray(item.evidence).filter((e) => typeof e === "string");
+		out.push({
+			id: `${sampleId}-q${q}`,
+			input: {
+				haystackSessions: sessions,
+				question,
+				ability
+			},
+			expected: answer,
+			metadata: {
+				datasetName: "locomo",
+				sampleId,
+				ability,
+				...category !== void 0 ? { category } : {},
+				evidence
+			}
+		});
+	}
+}
+function extractSessions(conversation) {
+	const speakerA = asString(conversation.speaker_a);
+	const indices = [];
+	for (const key of Object.keys(conversation)) {
+		const m = /^session_(\d+)$/.exec(key);
+		if (m?.[1] !== void 0) indices.push(Number.parseInt(m[1], 10));
+	}
+	indices.sort((a, b) => a - b);
+	const sessions = [];
+	for (const n of indices) {
+		const raw = asArray(conversation[`session_${n}`]);
+		const date = asString(conversation[`session_${n}_date_time`]);
+		const turns = [];
+		for (const t of raw) {
+			if (!isRecord(t)) continue;
+			const content = asString(t.text) ?? asString(t.content);
+			if (content === void 0) continue;
+			const speaker = asString(t.speaker);
+			const role = speakerA !== void 0 && speaker !== void 0 && speaker !== speakerA ? "assistant" : "user";
+			turns.push({
+				role,
+				content,
+				...date !== void 0 ? { timestamp: date } : {}
+			});
+		}
+		sessions.push({
+			id: `session_${n}`,
+			turns
+		});
+	}
+	return sessions;
+}
+/** Maps a LOCOMO numeric `category` onto a {@link MemoryEvalAbility}. */
+function mapCategory(category) {
+	switch (category) {
+		case 1: return "multi-session";
+		case 2: return "temporal";
+		case 5: return "abstention";
+		default: return "info-extraction";
+	}
+}
+function asString(value) {
+	return typeof value === "string" ? value : void 0;
+}
+function asArray(value) {
+	return Array.isArray(value) ? value : [];
+}
+function isRecord(value) {
+	return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+//#endregion
+export { loadLocomoDataset, parseLocomo };
+//# sourceMappingURL=locomo.js.map