@crewhaus/eval-dataset 0.1.0

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@crewhaus/eval-dataset",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Eval dataset loaders (JSONL, CSV, YAML, HTTP) with a lazy iterator API",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "test": "bun test src"
+  },
+  "dependencies": {
+    "@crewhaus/errors": "0.0.0",
+    "yaml": "^2.6.0",
+    "zod": "^3.23.8"
+  },
+  "license": "Apache-2.0",
+  "author": {
+    "name": "Max Meier",
+    "email": "max@studiomax.io",
+    "url": "https://studiomax.io"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/crewhaus/factory.git",
+    "directory": "packages/eval-dataset"
+  },
+  "homepage": "https://github.com/crewhaus/factory/tree/main/packages/eval-dataset#readme",
+  "bugs": {
+    "url": "https://github.com/crewhaus/factory/issues"
+  },
+  "publishConfig": {
+    "access": "restricted"
+  },
+  "files": [
+    "src",
+    "README.md",
+    "LICENSE",
+    "NOTICE"
+  ]
+}

package/src/__fixtures__/bare-array.yaml

@@ -0,0 +1,5 @@
+- id: a1
+  input: first
+- id: a2
+  input: second
+  expected_output: ok

package/src/__fixtures__/ok.csv

@@ -0,0 +1,5 @@
+id,input,expected_output,expected_tools
+q1,hello,,
+q2,"hello, world","greeting","bash, read"
+q3,"line1
+line2","multi","read"

package/src/__fixtures__/ok.jsonl

@@ -0,0 +1,3 @@
+{"id":"q1","input":"hello"}
+{"id":"q2","input":"goodbye","expected_output":"bye"}
+{"id":"q3","input":"add tools","expected_tools":["bash","read"],"metadata":{"difficulty":"easy"}}

package/src/__fixtures__/ok.yaml

@@ -0,0 +1,12 @@
+name: yaml-fixture
+samples:
+  - id: q1
+    input: hello
+  - id: q2
+    input: goodbye
+    expected_output: bye
+  - id: q3
+    input: tools
+    expected_tools: [bash, read]
+    metadata:
+      difficulty: easy

package/src/errors.ts

@@ -0,0 +1,7 @@
+import { RuntimeError } from "@crewhaus/errors";
+
+export class DatasetLoadError extends RuntimeError {
+  constructor(message: string, cause?: unknown) {
+    super(`eval-dataset: ${message}`, cause);
+  }
+}

package/src/index.test.ts

@@ -0,0 +1,145 @@
+import { describe, expect, test } from "bun:test";
+import { join } from "node:path";
+import { DatasetLoadError, loadDataset, parseCsv } from "./index";
+
+// `tsc -b` compiles this file into `dist/`, so `bun test` (with no path filter)
+// picks up both `src/index.test.ts` and `dist/index.test.js`. The compiled copy
+// resolves `import.meta.dir` to `dist/`, but `__fixtures__/` only exists under
+// `src/`. Map back to the source tree so both copies find the fixtures.
+const FIX = join(import.meta.dir.replace(/([/\\])dist$/, "$1src"), "__fixtures__");
+
+async function collect<T>(iter: AsyncIterable<T>): Promise<T[]> {
+  const out: T[] = [];
+  for await (const item of iter) out.push(item);
+  return out;
+}
+
+describe("loadDataset — JSONL loader (T1)", () => {
+  test("parses well-formed JSONL with all field shapes", async () => {
+    const ds = await loadDataset(join(FIX, "ok.jsonl"));
+    expect(ds.name).toBe("ok");
+    const samples = await collect(ds.samples);
+    expect(samples).toHaveLength(3);
+    expect(samples[0]).toEqual({ id: "q1", input: "hello" });
+    expect(samples[1]?.expected_output).toBe("bye");
+    expect(samples[2]?.expected_tools).toEqual(["bash", "read"]);
+    expect(samples[2]?.metadata).toEqual({ difficulty: "easy" });
+  });
+
+  test("yields zero samples for empty file", async () => {
+    const ds = await loadDataset(join(FIX, "empty.jsonl"));
+    expect(await collect(ds.samples)).toEqual([]);
+  });
+
+  test("rejects malformed JSON with line number", async () => {
+    const tmp = `${FIX}/__tmp_malformed.jsonl`;
+    await Bun.write(tmp, '{"id":"q1","input":"ok"}\nnot json\n');
+    try {
+      const ds = await loadDataset(tmp);
+      await expect(collect(ds.samples)).rejects.toThrow(/malformed JSON on line 2/);
+    } finally {
+      await Bun.file(tmp).delete();
+    }
+  });
+
+  test("rejects sample missing required fields", async () => {
+    const tmp = `${FIX}/__tmp_invalid.jsonl`;
+    await Bun.write(tmp, '{"input":"no id"}\n');
+    try {
+      const ds = await loadDataset(tmp);
+      await expect(collect(ds.samples)).rejects.toThrow(DatasetLoadError);
+    } finally {
+      await Bun.file(tmp).delete();
+    }
+  });
+
+  test("rejects missing file", async () => {
+    await expect(loadDataset(`${FIX}/does-not-exist.jsonl`)).rejects.toThrow(DatasetLoadError);
+  });
+});
+
+describe("loadDataset — CSV loader (T1)", () => {
+  test("parses CSV with quoted fields, embedded newlines, and multi-tool column", async () => {
+    const ds = await loadDataset(join(FIX, "ok.csv"));
+    const samples = await collect(ds.samples);
+    expect(samples).toHaveLength(3);
+    expect(samples[0]).toEqual({ id: "q1", input: "hello" });
+    expect(samples[1]).toEqual({
+      id: "q2",
+      input: "hello, world",
+      expected_output: "greeting",
+      expected_tools: ["bash", "read"],
+    });
+    expect(samples[2]?.input).toBe("line1\nline2");
+  });
+});
+
+describe("parseCsv — RFC 4180 subset (T1)", () => {
+  test("empty input yields no rows", () => {
+    expect(parseCsv("")).toEqual([]);
+  });
+
+  test("escaped double-quotes inside quoted field", () => {
+    expect(parseCsv('a,"b ""quoted"" c"')).toEqual([["a", 'b "quoted" c']]);
+  });
+
+  test("CRLF line endings", () => {
+    expect(parseCsv("a,b\r\nc,d\r\n")).toEqual([
+      ["a", "b"],
+      ["c", "d"],
+    ]);
+  });
+
+  test("trailing newline is not a synthetic empty row", () => {
+    expect(parseCsv("a,b\n")).toEqual([["a", "b"]]);
+  });
+
+  test("missing final newline still yields the last row", () => {
+    expect(parseCsv("a,b\nc,d")).toEqual([
+      ["a", "b"],
+      ["c", "d"],
+    ]);
+  });
+});
+
+describe("loadDataset — YAML loader (T1)", () => {
+  test("parses Dataset wrapper shape", async () => {
+    const ds = await loadDataset(join(FIX, "ok.yaml"));
+    expect(ds.name).toBe("yaml-fixture");
+    const samples = await collect(ds.samples);
+    expect(samples).toHaveLength(3);
+    expect(samples[0]).toEqual({ id: "q1", input: "hello" });
+  });
+
+  test("parses bare-array shape and derives name from filename", async () => {
+    const ds = await loadDataset(join(FIX, "bare-array.yaml"));
+    expect(ds.name).toBe("bare-array");
+    const samples = await collect(ds.samples);
+    expect(samples).toHaveLength(2);
+    expect(samples[1]?.expected_output).toBe("ok");
+  });
+
+  test("supports .yml extension", async () => {
+    const tmp = `${FIX}/__tmp.yml`;
+    await Bun.write(tmp, "- id: x\n  input: y\n");
+    try {
+      const ds = await loadDataset(tmp);
+      const samples = await collect(ds.samples);
+      expect(samples).toHaveLength(1);
+    } finally {
+      await Bun.file(tmp).delete();
+    }
+  });
+});
+
+describe("loadDataset — dispatch", () => {
+  test("rejects unknown extension", async () => {
+    await expect(loadDataset("/tmp/something.xyz")).rejects.toThrow(/unrecognized dataset source/);
+  });
+
+  test("HTTP scheme dispatches to http loader (rejects offline lookups gracefully)", async () => {
+    // Simply verifies dispatch — actual fetch is exercised in __test__/http.test.ts
+    // when CREWHAUS_TEST_HTTP=1 is set.
+    await expect(loadDataset("http://127.0.0.1:1/never-listens.jsonl")).rejects.toThrow();
+  });
+});

package/src/index.ts

@@ -0,0 +1,69 @@
+/**
+ * Catalog R-eval `eval-dataset` — load samples from JSONL, CSV, YAML, or HTTP.
+ *
+ * Sample shape is the contract every other eval-* module consumes:
+ *   { id, input, expected_output?, expected_tools?, metadata? }
+ *
+ * Loaders return an `AsyncIterable<Sample>` so 100k-row datasets stream rather
+ * than load fully into memory. JSONL/CSV truly stream off the file system via
+ * `Bun.file().stream()`. YAML and HTTP buffer (YAML can't stream; HTTP buffers
+ * before re-dispatching to the format-specific loader).
+ *
+ * Reference: build-roadmap.md §16.
+ */
+import { z } from "zod";
+import { DatasetLoadError } from "./errors";
+import { loadCsv } from "./loaders/csv";
+import { loadHttp } from "./loaders/http";
+import { loadJsonl } from "./loaders/jsonl";
+import { loadYaml } from "./loaders/yaml";
+
+export const SampleSchema = z.object({
+  id: z.string().min(1),
+  input: z.string(),
+  expected_output: z.string().optional(),
+  expected_tools: z.array(z.string()).optional(),
+  metadata: z.record(z.unknown()).optional(),
+});
+
+export type Sample = z.infer<typeof SampleSchema>;
+
+export const DatasetSchema = z.object({
+  name: z.string(),
+  samples: z.array(SampleSchema),
+});
+
+export type Dataset = z.infer<typeof DatasetSchema>;
+
+export type LoadedDataset = {
+  readonly name: string;
+  readonly samples: AsyncIterable<Sample>;
+};
+
+export type DatasetSource = string | URL;
+
+/**
+ * Dispatch to the right loader based on URL scheme or file extension.
+ * `http://` and `https://` go to the HTTP loader. Local paths are dispatched
+ * by extension: `.jsonl`, `.csv`, `.yaml`, `.yml`.
+ */
+export async function loadDataset(source: DatasetSource): Promise<LoadedDataset> {
+  const sourceStr = typeof source === "string" ? source : source.toString();
+
+  if (sourceStr.startsWith("http://") || sourceStr.startsWith("https://")) {
+    return loadHttp(sourceStr);
+  }
+
+  const lower = sourceStr.toLowerCase();
+  if (lower.endsWith(".jsonl") || lower.endsWith(".ndjson")) return loadJsonl(sourceStr);
+  if (lower.endsWith(".csv")) return loadCsv(sourceStr);
+  if (lower.endsWith(".yaml") || lower.endsWith(".yml")) return loadYaml(sourceStr);
+
+  throw new DatasetLoadError(
+    `unrecognized dataset source "${sourceStr}" — expected .jsonl, .csv, .yaml, .yml, or http(s):// URL`,
+  );
+}
+
+export { DatasetLoadError };
+export { loadCsv, loadHttp, loadJsonl, loadYaml };
+export { parseCsv } from "./loaders/csv";

package/src/loaders/csv.ts

@@ -0,0 +1,126 @@
+import { basename } from "node:path";
+import { DatasetLoadError } from "../errors";
+import { type LoadedDataset, type Sample, SampleSchema } from "../index";
+
+export async function loadCsv(path: string): Promise<LoadedDataset> {
+  const file = Bun.file(path);
+  if (!(await file.exists())) {
+    throw new DatasetLoadError(`file not found: ${path}`);
+  }
+  const text = await file.text();
+  const rows = parseCsv(text);
+  if (rows.length === 0) {
+    return { name: basename(path).replace(/\.csv$/i, ""), samples: emptyIterable() };
+  }
+  const header = rows[0];
+  if (!header) {
+    return { name: basename(path).replace(/\.csv$/i, ""), samples: emptyIterable() };
+  }
+
+  return {
+    name: basename(path).replace(/\.csv$/i, ""),
+    samples: rowsToSamples(rows.slice(1), header, path),
+  };
+}
+
+async function* emptyIterable(): AsyncIterable<Sample> {
+  /* nothing */
+}
+
+async function* rowsToSamples(
+  rows: ReadonlyArray<ReadonlyArray<string>>,
+  header: ReadonlyArray<string>,
+  path: string,
+): AsyncIterable<Sample> {
+  let rowNo = 1; // header was row 0
+  for (const row of rows) {
+    rowNo += 1;
+    if (row.length === 1 && row[0] === "") continue; // trailing newline
+    const obj: Record<string, string | string[]> = {};
+    for (let i = 0; i < header.length; i++) {
+      const key = header[i];
+      if (key === undefined) continue;
+      const cell = row[i] ?? "";
+      // expected_tools is the only array-shaped field — comma-split
+      if (key === "expected_tools" && cell !== "") {
+        obj[key] = cell
+          .split(",")
+          .map((s) => s.trim())
+          .filter((s) => s.length > 0);
+      } else if (cell !== "") {
+        obj[key] = cell;
+      }
+    }
+    const result = SampleSchema.safeParse(obj);
+    if (!result.success) {
+      throw new DatasetLoadError(
+        `invalid sample on row ${rowNo} of ${path}: ${result.error.message}`,
+      );
+    }
+    yield result.data;
+  }
+}
+
+/**
+ * RFC 4180-style CSV parser. Handles:
+ * - quoted fields with embedded commas, newlines (CRLF/LF), and escaped `""`
+ * - trailing empty line tolerance
+ * - any combination of CRLF / LF / mixed line endings
+ *
+ * Returns one `string[]` per row. Empty input yields `[]`.
+ */
+export function parseCsv(text: string): string[][] {
+  const rows: string[][] = [];
+  if (text.length === 0) return rows;
+  let row: string[] = [];
+  let field = "";
+  let inQuotes = false;
+  let i = 0;
+  while (i < text.length) {
+    const ch = text[i];
+    if (inQuotes) {
+      if (ch === '"') {
+        if (text[i + 1] === '"') {
+          field += '"';
+          i += 2;
+          continue;
+        }
+        inQuotes = false;
+        i += 1;
+        continue;
+      }
+      field += ch;
+      i += 1;
+      continue;
+    }
+    if (ch === '"') {
+      inQuotes = true;
+      i += 1;
+      continue;
+    }
+    if (ch === ",") {
+      row.push(field);
+      field = "";
+      i += 1;
+      continue;
+    }
+    if (ch === "\n" || ch === "\r") {
+      row.push(field);
+      field = "";
+      rows.push(row);
+      row = [];
+      // swallow CRLF as a single newline
+      if (ch === "\r" && text[i + 1] === "\n") i += 2;
+      else i += 1;
+      continue;
+    }
+    field += ch;
+    i += 1;
+  }
+  // EOF without trailing newline
+  if (field !== "" || row.length > 0) {
+    row.push(field);
+    rows.push(row);
+  }
+  return rows;
+}

package/src/loaders/http.ts

@@ -0,0 +1,143 @@
+import { parse as parseYaml } from "yaml";
+import { DatasetLoadError } from "../errors";
+import { DatasetSchema, type LoadedDataset, type Sample, SampleSchema } from "../index";
+import { parseCsv } from "./csv";
+
+/**
+ * HTTP loader — fetches the URL into memory, then dispatches to the
+ * appropriate format parser based on the URL extension first, then
+ * the response Content-Type as a fallback.
+ *
+ * Buffers the entire body in memory; HuggingFace-scale datasets should
+ * be downloaded ahead of time and loaded via the local-file loaders.
+ */
+export async function loadHttp(url: string): Promise<LoadedDataset> {
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new DatasetLoadError(`HTTP ${response.status} fetching ${url}`);
+  }
+  const text = await response.text();
+  const ct = (response.headers.get("content-type") ?? "").toLowerCase();
+  const lower = url.toLowerCase();
+
+  if (
+    lower.endsWith(".jsonl") ||
+    lower.endsWith(".ndjson") ||
+    ct.includes("application/x-jsonlines") ||
+    ct.includes("application/x-ndjson")
+  ) {
+    return { name: deriveName(url), samples: parseJsonlText(text, url) };
+  }
+  if (lower.endsWith(".csv") || ct.includes("text/csv")) {
+    return { name: deriveName(url), samples: parseCsvText(text, url) };
+  }
+  if (
+    lower.endsWith(".yaml") ||
+    lower.endsWith(".yml") ||
+    ct.includes("application/yaml") ||
+    ct.includes("text/yaml")
+  ) {
+    return parseYamlText(text, url);
+  }
+  throw new DatasetLoadError(
+    `unrecognized HTTP dataset format for ${url} (content-type: ${ct || "unknown"})`,
+  );
+}
+
+function deriveName(url: string): string {
+  try {
+    const u = new URL(url);
+    const last = u.pathname.split("/").filter(Boolean).pop() ?? "remote-dataset";
+    return last.replace(/\.(jsonl|ndjson|csv|yaml|yml)$/i, "");
+  } catch {
+    return "remote-dataset";
+  }
+}
+
+async function* parseJsonlText(text: string, source: string): AsyncIterable<Sample> {
+  let lineNo = 0;
+  for (const raw of text.split(/\r?\n/)) {
+    lineNo += 1;
+    if (raw.trim() === "") continue;
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(raw);
+    } catch (err) {
+      throw new DatasetLoadError(`malformed JSON on line ${lineNo} of ${source}`, err);
+    }
+    const result = SampleSchema.safeParse(parsed);
+    if (!result.success) {
+      throw new DatasetLoadError(
+        `invalid sample on line ${lineNo} of ${source}: ${result.error.message}`,
+      );
+    }
+    yield result.data;
+  }
+}
+
+async function* parseCsvText(text: string, source: string): AsyncIterable<Sample> {
+  const rows = parseCsv(text);
+  if (rows.length === 0) return;
+  const header = rows[0];
+  if (!header) return;
+  let rowNo = 1;
+  for (const row of rows.slice(1)) {
+    rowNo += 1;
+    if (row.length === 1 && row[0] === "") continue;
+    const obj: Record<string, string | string[]> = {};
+    for (let i = 0; i < header.length; i++) {
+      const key = header[i];
+      if (key === undefined) continue;
+      const cell = row[i] ?? "";
+      if (key === "expected_tools" && cell !== "") {
+        obj[key] = cell
+          .split(",")
+          .map((s) => s.trim())
+          .filter((s) => s.length > 0);
+      } else if (cell !== "") {
+        obj[key] = cell;
+      }
+    }
+    const result = SampleSchema.safeParse(obj);
+    if (!result.success) {
+      throw new DatasetLoadError(
+        `invalid sample on row ${rowNo} of ${source}: ${result.error.message}`,
+      );
+    }
+    yield result.data;
+  }
+}
+
+function parseYamlText(text: string, source: string): LoadedDataset {
+  let parsed: unknown;
+  try {
+    parsed = parseYaml(text);
+  } catch (err) {
+    throw new DatasetLoadError(`malformed YAML in ${source}`, err);
+  }
+  if (Array.isArray(parsed)) {
+    return { name: deriveName(source), samples: yieldSamples(parsed, source) };
+  }
+  const result = DatasetSchema.safeParse(parsed);
+  if (!result.success) {
+    throw new DatasetLoadError(`invalid dataset in ${source}: ${result.error.message}`);
+  }
+  return { name: result.data.name, samples: yieldSamples(result.data.samples, source) };
+}
+
+async function* yieldSamples(
+  samples: ReadonlyArray<unknown>,
+  source: string,
+): AsyncIterable<Sample> {
+  let i = 0;
+  for (const raw of samples) {
+    const result = SampleSchema.safeParse(raw);
+    if (!result.success) {
+      throw new DatasetLoadError(
+        `invalid sample at index ${i} of ${source}: ${result.error.message}`,
+      );
+    }
+    yield result.data;
+    i += 1;
+  }
+}

package/src/loaders/jsonl.ts

@@ -0,0 +1,37 @@
+import { basename } from "node:path";
+import { DatasetLoadError } from "../errors";
+import { type LoadedDataset, type Sample, SampleSchema } from "../index";
+
+export async function loadJsonl(path: string): Promise<LoadedDataset> {
+  const file = Bun.file(path);
+  if (!(await file.exists())) {
+    throw new DatasetLoadError(`file not found: ${path}`);
+  }
+  return {
+    name: basename(path).replace(/\.(jsonl|ndjson)$/i, ""),
+    samples: streamJsonl(path),
+  };
+}
+
+async function* streamJsonl(path: string): AsyncIterable<Sample> {
+  const file = Bun.file(path);
+  const text = await file.text();
+  let lineNo = 0;
+  for (const raw of text.split(/\r?\n/)) {
+    lineNo += 1;
+    if (raw.trim() === "") continue;
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(raw);
+    } catch (err) {
+      throw new DatasetLoadError(`malformed JSON on line ${lineNo} of ${path}`, err);
+    }
+    const result = SampleSchema.safeParse(parsed);
+    if (!result.success) {
+      throw new DatasetLoadError(
+        `invalid sample on line ${lineNo} of ${path}: ${result.error.message}`,
+      );
+    }
+    yield result.data;
+  }
+}

package/src/loaders/yaml.ts

@@ -0,0 +1,47 @@
+import { basename } from "node:path";
+import { parse as parseYaml } from "yaml";
+import { DatasetLoadError } from "../errors";
+import { DatasetSchema, type LoadedDataset, type Sample, SampleSchema } from "../index";
+
+export async function loadYaml(path: string): Promise<LoadedDataset> {
+  const file = Bun.file(path);
+  if (!(await file.exists())) {
+    throw new DatasetLoadError(`file not found: ${path}`);
+  }
+  const text = await file.text();
+  let parsed: unknown;
+  try {
+    parsed = parseYaml(text);
+  } catch (err) {
+    throw new DatasetLoadError(`malformed YAML in ${path}`, err);
+  }
+
+  // Two accepted shapes:
+  //   1. Top-level Dataset: { name, samples: [Sample, …] }
+  //   2. Bare Sample[] (assigns a synthetic name from the filename)
+  if (Array.isArray(parsed)) {
+    const name = basename(path).replace(/\.(yaml|yml)$/i, "");
+    return { name, samples: yieldSamples(parsed, path) };
+  }
+
+  const result = DatasetSchema.safeParse(parsed);
+  if (!result.success) {
+    throw new DatasetLoadError(`invalid dataset in ${path}: ${result.error.message}`);
+  }
+  const dataset = result.data;
+  return { name: dataset.name, samples: yieldSamples(dataset.samples, path) };
+}
+
+async function* yieldSamples(samples: ReadonlyArray<unknown>, path: string): AsyncIterable<Sample> {
+  let i = 0;
+  for (const raw of samples) {
+    const result = SampleSchema.safeParse(raw);
+    if (!result.success) {
+      throw new DatasetLoadError(
+        `invalid sample at index ${i} of ${path}: ${result.error.message}`,
+      );
+    }
+    yield result.data;
+    i += 1;
+  }
+}