@graphorin/evals 0.6.0 → 0.7.0

package/src/loaders/csv.ts

@@ -0,0 +1,147 @@
+/**
+ * CSV dataset loader. Implements RFC 4180 strict subset:
+ *
+ *  - Comma separator (configurable).
+ *  - Optional `"`-quoted cells; doubled `""` to embed a literal `"`.
+ *  - Header row required by default.
+ *
+ * Columns map to `Case` fields by name: `input`, `expected`, `id`,
+ * `metadata` (parsed as JSON when present). Unknown columns are
+ * ignored unless a `mapper` is supplied.
+ *
+ * @packageDocumentation
+ */
+
+import { readFile } from 'node:fs/promises';
+
+import type { Case, Dataset } from '@graphorin/observability/eval';
+
+/** @stable */
+export interface LoadCsvOptions {
+  readonly delimiter?: string;
+  readonly name?: string;
+  readonly description?: string;
+  readonly mapper?: (row: Record<string, string>, index: number) => Case<unknown, unknown>;
+}
+
+/** @stable */
+export async function loadCsvDataset(
+  path: string,
+  options: LoadCsvOptions = {},
+): Promise<Dataset<unknown, unknown>> {
+  const text = await readFile(path, 'utf8');
+  const cases = parseCsv(text, options);
+  const meta: Dataset<unknown, unknown>['metadata'] = {
+    ...(options.name !== undefined ? { name: options.name } : {}),
+    ...(options.description !== undefined ? { description: options.description } : {}),
+    createdAt: new Date(),
+  };
+  return { cases, metadata: meta };
+}
+
+/**
+ * Pure parser. Exported separately so tests can exercise the
+ * column-mapping behaviour without touching the filesystem.
+ *
+ * @stable
+ */
+export function parseCsv(
+  text: string,
+  options: LoadCsvOptions = {},
+): ReadonlyArray<Case<unknown, unknown>> {
+  const delimiter = options.delimiter ?? ',';
+  const rows = parseCsvRows(text, delimiter);
+  if (rows.length === 0) return [];
+  const header = rows[0];
+  if (header === undefined) return [];
+  const dataRows = rows.slice(1);
+  const out: Case<unknown, unknown>[] = [];
+  for (let i = 0; i < dataRows.length; i++) {
+    const row = dataRows[i] ?? [];
+    if (row.every((cell) => cell.length === 0)) continue;
+    const record: Record<string, string> = {};
+    for (let c = 0; c < header.length; c++) {
+      const key = header[c] ?? '';
+      const value = row[c] ?? '';
+      record[key] = value;
+    }
+    if (options.mapper !== undefined) {
+      out.push(options.mapper(record, i));
+      continue;
+    }
+    if (record.input === undefined) {
+      throw new Error(`[graphorin/evals] CSV row ${i + 1} missing required 'input' column.`);
+    }
+    const sampleCase: Case<unknown, unknown> = {
+      ...(record.id !== undefined && record.id.length > 0 ? { id: record.id } : {}),
+      input: record.input,
+      ...('expected' in record && record.expected !== undefined && record.expected.length > 0
+        ? { expected: record.expected }
+        : {}),
+      ...(record.metadata !== undefined && record.metadata.length > 0
+        ? { metadata: parseMetadata(record.metadata) }
+        : {}),
+    };
+    out.push(sampleCase);
+  }
+  return out;
+}
+
+function parseMetadata(raw: string): Readonly<Record<string, unknown>> {
+  try {
+    const parsed = JSON.parse(raw);
+    if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) {
+      return parsed as Readonly<Record<string, unknown>>;
+    }
+    return { value: parsed } as Readonly<Record<string, unknown>>;
+  } catch {
+    return { raw } as Readonly<Record<string, unknown>>;
+  }
+}
+
+function parseCsvRows(text: string, delimiter: string): string[][] {
+  const rows: string[][] = [];
+  let current: string[] = [];
+  let cell = '';
+  let inQuotes = false;
+  for (let i = 0; i < text.length; i++) {
+    const ch = text[i];
+    if (inQuotes) {
+      if (ch === '"') {
+        const next = text[i + 1];
+        if (next === '"') {
+          cell += '"';
+          i += 1;
+        } else {
+          inQuotes = false;
+        }
+      } else {
+        cell += ch;
+      }
+      continue;
+    }
+    if (ch === '"') {
+      inQuotes = true;
+      continue;
+    }
+    if (ch === delimiter) {
+      current.push(cell);
+      cell = '';
+      continue;
+    }
+    if (ch === '\n' || ch === '\r') {
+      if (ch === '\r' && text[i + 1] === '\n') i += 1;
+      current.push(cell);
+      rows.push(current);
+      current = [];
+      cell = '';
+      continue;
+    }
+    cell += ch;
+  }
+  if (cell.length > 0 || current.length > 0) {
+    current.push(cell);
+    rows.push(current);
+  }
+  return rows;
+}

package/src/loaders/dmr.ts

@@ -0,0 +1,137 @@
+/**
+ * DMR (Deep Memory Retrieval) loader - the MemGPT/Letta continuity set
+ * derived from Multi-Session Chat (arXiv:2310.08560). DMR's published
+ * shape is less standardised than LongMemEval/LOCOMO, so this loader is
+ * deliberately tolerant: it accepts a JSON **array** of items, each
+ * exposing a `question`, an `answer`, and the prior conversation under
+ * either `sessions` or `haystack_sessions`. Sessions may be a bare
+ * array of turns or an object wrapping the turns under
+ * `turns` / `messages` / `dialog`; a turn may use `role` or `speaker`
+ * and `content` or `text`.
+ *
+ * ```jsonc
+ * {
+ *   "id": "dmr-1",
+ *   "question": "What is the user's sister's name?",
+ *   "answer": "Anna",
+ *   "sessions": [
+ *     { "id": "s1", "turns": [ { "role": "user", "content": "..." } ] },
+ *     [ { "speaker": "assistant", "text": "..." } ]
+ *   ]
+ * }
+ * ```
+ *
+ * Every DMR case is treated as `multi-session` (its defining ability).
+ * The exact field names should be reconciled against the downloaded
+ * file the first time `scripts/fetch-eval-datasets.mjs` is run.
+ *
+ * @packageDocumentation
+ */
+
+import { readFile } from 'node:fs/promises';
+
+import type { Case, Dataset } from '@graphorin/observability/eval';
+
+import type { MemoryEvalInput, MemoryEvalSession, MemoryEvalTurn } from './memory-eval.js';
+
+/** @stable */
+export interface LoadDmrOptions {
+  /** Local path to the DMR JSON (under `benchmarks/.datasets/`). */
+  readonly path: string;
+  /** Optional dataset name surfaced in `Dataset.metadata.name`. */
+  readonly name?: string;
+  /** Optional description surfaced in `Dataset.metadata.description`. */
+  readonly description?: string;
+}
+
+/**
+ * Read a DMR JSON file and return a fully-materialised {@link Dataset}
+ * of multi-session retrieval cases scored against the reference answer.
+ *
+ * @stable
+ */
+export async function loadDmrDataset(
+  options: LoadDmrOptions,
+): Promise<Dataset<MemoryEvalInput, string>> {
+  const text = await readFile(options.path, 'utf8');
+  const cases = parseDmr(text);
+  return {
+    cases,
+    metadata: {
+      name: options.name ?? 'dmr',
+      ...(options.description !== undefined ? { description: options.description } : {}),
+      createdAt: new Date(),
+    },
+  };
+}
+
+/**
+ * Pure parser. Exported so tests can exercise the mapping without
+ * touching the filesystem.
+ *
+ * @stable
+ */
+export function parseDmr(text: string): ReadonlyArray<Case<MemoryEvalInput, string>> {
+  const parsed: unknown = JSON.parse(text);
+  if (!Array.isArray(parsed)) {
+    throw new Error('[graphorin/evals] DMR dataset must be a JSON array of items.');
+  }
+  const items = parsed as ReadonlyArray<unknown>;
+  const out: Case<MemoryEvalInput, string>[] = [];
+  for (let i = 0; i < items.length; i++) {
+    const item = items[i];
+    if (!isRecord(item)) {
+      throw new Error(`[graphorin/evals] DMR item ${i} is not an object.`);
+    }
+    const question = asString(item.question);
+    if (question === undefined) {
+      throw new Error(`[graphorin/evals] DMR item ${i} is missing a 'question'.`);
+    }
+    const sessions = coerceSessions(item.sessions ?? item.haystack_sessions);
+    out.push({
+      id: asString(item.id) ?? `dmr-${i}`,
+      input: { haystackSessions: sessions, question, ability: 'multi-session' },
+      expected: asString(item.answer) ?? '',
+      metadata: { datasetName: 'dmr', ability: 'multi-session' },
+    });
+  }
+  return out;
+}
+
+function coerceSessions(value: unknown): MemoryEvalSession[] {
+  const rawSessions = asArray(value);
+  const sessions: MemoryEvalSession[] = [];
+  for (let s = 0; s < rawSessions.length; s++) {
+    const raw = rawSessions[s];
+    const rawTurns = Array.isArray(raw)
+      ? (raw as ReadonlyArray<unknown>)
+      : isRecord(raw)
+        ? asArray(raw.turns ?? raw.messages ?? raw.dialog)
+        : [];
+    const id = isRecord(raw) ? (asString(raw.id) ?? `session-${s}`) : `session-${s}`;
+    const turns: MemoryEvalTurn[] = [];
+    for (const t of rawTurns) {
+      if (!isRecord(t)) continue;
+      const content = asString(t.content) ?? asString(t.text);
+      if (content === undefined) continue;
+      const speaker = asString(t.role) ?? asString(t.speaker);
+      const role = speaker === 'assistant' ? 'assistant' : 'user';
+      const timestamp = asString(t.timestamp);
+      turns.push({ role, content, ...(timestamp !== undefined ? { timestamp } : {}) });
+    }
+    sessions.push({ id, turns });
+  }
+  return sessions;
+}
+
+function asString(value: unknown): string | undefined {
+  return typeof value === 'string' ? value : undefined;
+}
+
+function asArray(value: unknown): ReadonlyArray<unknown> {
+  return Array.isArray(value) ? (value as ReadonlyArray<unknown>) : [];
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}

package/src/loaders/from-traces.ts

@@ -0,0 +1,92 @@
+/**
+ * `loadDatasetFromTraces` - distil a dataset from the framework's
+ * replay log. The caller supplies the JSONL replay file and a small
+ * extraction function that pulls the `(input, output)` pair out of
+ * each event group; the loader handles the JSONL parsing + grouping
+ * by `runId`.
+ *
+ * @packageDocumentation
+ */
+
+import { readFile } from 'node:fs/promises';
+
+import type { Case, Dataset } from '@graphorin/observability/eval';
+
+/** @stable */
+export interface TraceEvent {
+  readonly runId: string;
+  readonly type: string;
+  readonly payload: Readonly<Record<string, unknown>>;
+  readonly timestamp?: string;
+}
+
+/** @stable */
+export interface FromTracesOptions<I, O> {
+  /** Distil one `Case<I, O>` from every group of events sharing a `runId`. */
+  readonly extract: (events: ReadonlyArray<TraceEvent>) => Case<I, O> | null;
+  /** Optional name surfaced in `Dataset.metadata.name`. */
+  readonly name?: string;
+  /** Optional description surfaced in `Dataset.metadata.description`. */
+  readonly description?: string;
+}
+
+/** @stable */
+export async function loadDatasetFromTraces<I, O>(
+  path: string,
+  options: FromTracesOptions<I, O>,
+): Promise<Dataset<I, O>> {
+  const text = await readFile(path, 'utf8');
+  return groupAndExtract<I, O>(text, options);
+}
+
+/**
+ * Pure parser for the trace JSONL format. Exported so tests can
+ * exercise the extraction without touching the filesystem.
+ *
+ * @stable
+ */
+export function groupAndExtract<I, O>(
+  text: string,
+  options: FromTracesOptions<I, O>,
+): Dataset<I, O> {
+  const lines = text.split(/\r?\n/);
+  const groups = new Map<string, TraceEvent[]>();
+  for (const raw of lines) {
+    if (raw.trim().length === 0) continue;
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(raw);
+    } catch {
+      continue;
+    }
+    if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) continue;
+    const obj = parsed as Record<string, unknown>;
+    const runId = typeof obj.runId === 'string' ? obj.runId : null;
+    const type = typeof obj.type === 'string' ? obj.type : null;
+    if (runId === null || type === null) continue;
+    const payload =
+      obj.payload !== null && typeof obj.payload === 'object' && !Array.isArray(obj.payload)
+        ? (obj.payload as Readonly<Record<string, unknown>>)
+        : ({} as Readonly<Record<string, unknown>>);
+    const event: TraceEvent = {
+      runId,
+      type,
+      payload,
+      ...(typeof obj.timestamp === 'string' ? { timestamp: obj.timestamp } : {}),
+    };
+    const existing = groups.get(runId);
+    if (existing === undefined) groups.set(runId, [event]);
+    else existing.push(event);
+  }
+  const cases: Case<I, O>[] = [];
+  for (const events of groups.values()) {
+    const sample = options.extract(events);
+    if (sample !== null) cases.push(sample);
+  }
+  const meta: Dataset<I, O>['metadata'] = {
+    ...(options.name !== undefined ? { name: options.name } : {}),
+    ...(options.description !== undefined ? { description: options.description } : {}),
+    createdAt: new Date(),
+  };
+  return { cases, metadata: meta };
+}

package/src/loaders/index.ts

@@ -0,0 +1,46 @@
+/**
+ * Dataset loaders. Every loader returns a fully-materialised
+ * `Dataset` that the runner can iterate over without further
+ * I/O. Streaming loaders are a post-MVP follow-up.
+ *
+ * @packageDocumentation
+ */
+
+export {
+  type LoadCsvOptions,
+  loadCsvDataset,
+  parseCsv,
+} from './csv.js';
+export {
+  type LoadDmrOptions,
+  loadDmrDataset,
+  parseDmr,
+} from './dmr.js';
+export {
+  type FromTracesOptions,
+  groupAndExtract,
+  loadDatasetFromTraces,
+  type TraceEvent,
+} from './from-traces.js';
+export { fromIterable } from './iterable.js';
+export {
+  type LoadJsonlOptions,
+  loadJsonlDataset,
+  parseJsonl,
+} from './jsonl.js';
+export {
+  type LoadLocomoOptions,
+  loadLocomoDataset,
+  parseLocomo,
+} from './locomo.js';
+export {
+  type LoadLongMemEvalOptions,
+  loadLongMemEvalDataset,
+  parseLongMemEval,
+} from './longmemeval.js';
+export type {
+  MemoryEvalAbility,
+  MemoryEvalInput,
+  MemoryEvalSession,
+  MemoryEvalTurn,
+} from './memory-eval.js';

package/src/loaders/iterable.ts

@@ -0,0 +1,22 @@
+/**
+ * `fromIterable` - wrap an in-memory array / iterable as a
+ * {@link Dataset}. Useful for tests + ad-hoc data.
+ *
+ * @packageDocumentation
+ */
+
+import type { Case, Dataset } from '@graphorin/observability/eval';
+
+/** @stable */
+export function fromIterable<I, O = unknown>(
+  cases: Iterable<Case<I, O>> | ReadonlyArray<Case<I, O>>,
+  options: { readonly name?: string; readonly description?: string } = {},
+): Dataset<I, O> {
+  const arr = Array.isArray(cases) ? (cases as ReadonlyArray<Case<I, O>>) : Array.from(cases);
+  const meta: Dataset<I, O>['metadata'] = {
+    ...(options.name !== undefined ? { name: options.name } : {}),
+    ...(options.description !== undefined ? { description: options.description } : {}),
+    createdAt: new Date(),
+  };
+  return { cases: arr, metadata: meta };
+}

package/src/loaders/jsonl.ts

@@ -0,0 +1,94 @@
+/**
+ * JSONL dataset loader. Each line is a JSON object with `input` and
+ * optional `expected` / `metadata` / `id` fields.
+ *
+ * @packageDocumentation
+ */
+
+import { readFile } from 'node:fs/promises';
+
+import type { Case, Dataset } from '@graphorin/observability/eval';
+
+/** @stable */
+export interface LoadJsonlOptions {
+  /** Optional dataset name surfaced in `Dataset.metadata.name`. */
+  readonly name?: string;
+  /** Optional description surfaced in `Dataset.metadata.description`. */
+  readonly description?: string;
+  /**
+   * Map a parsed line into a `Case`. Default forwards the line
+   * verbatim. Override to translate column names or coerce types.
+   */
+  readonly mapper?: (line: Record<string, unknown>, index: number) => Case<unknown, unknown>;
+}
+
+/**
+ * Read a JSONL file and return a fully-materialised {@link Dataset}.
+ * Empty lines are skipped; malformed lines throw with the line
+ * number.
+ *
+ * @stable
+ */
+export async function loadJsonlDataset(
+  path: string,
+  options: LoadJsonlOptions = {},
+): Promise<Dataset<unknown, unknown>> {
+  const text = await readFile(path, 'utf8');
+  const cases = parseJsonl(text, options.mapper);
+  const meta: Dataset<unknown, unknown>['metadata'] = {
+    ...(options.name !== undefined ? { name: options.name } : {}),
+    ...(options.description !== undefined ? { description: options.description } : {}),
+    createdAt: new Date(),
+  };
+  return { cases, metadata: meta };
+}
+
+/**
+ * Pure parser. Exported separately so tests can exercise the line-by-
+ * line behaviour without touching the filesystem.
+ *
+ * @stable
+ */
+export function parseJsonl(
+  text: string,
+  mapper?: LoadJsonlOptions['mapper'],
+): ReadonlyArray<Case<unknown, unknown>> {
+  const out: Case<unknown, unknown>[] = [];
+  const lines = text.split(/\r?\n/);
+  for (let i = 0; i < lines.length; i++) {
+    const raw = lines[i] ?? '';
+    if (raw.trim().length === 0) continue;
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(raw);
+    } catch (err) {
+      throw new Error(
+        `[graphorin/evals] failed to parse JSONL at line ${i + 1}: ${(err as Error).message}`,
+        { cause: err },
+      );
+    }
+    if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+      throw new Error(
+        `[graphorin/evals] JSONL line ${i + 1} must be a JSON object, got ${typeof parsed}.`,
+      );
+    }
+    const obj = parsed as Record<string, unknown>;
+    const mapped = mapper !== undefined ? mapper(obj, i) : defaultMapper(obj, i);
+    out.push(mapped);
+  }
+  return out;
+}
+
+function defaultMapper(line: Record<string, unknown>, _index: number): Case<unknown, unknown> {
+  if (!('input' in line)) {
+    throw new Error(`[graphorin/evals] JSONL row missing required 'input' field.`);
+  }
+  return {
+    ...(typeof line.id === 'string' ? { id: line.id } : {}),
+    input: line.input,
+    ...('expected' in line ? { expected: line.expected } : {}),
+    ...(line.metadata !== undefined && typeof line.metadata === 'object' && line.metadata !== null
+      ? { metadata: line.metadata as Readonly<Record<string, unknown>> }
+      : {}),
+  };
+}