@crewhaus/eval-dataset 0.1.3 → 0.1.5

package/src/index.test.ts

@@ -1,145 +0,0 @@
-import { describe, expect, test } from "bun:test";
-import { join } from "node:path";
-import { DatasetLoadError, loadDataset, parseCsv } from "./index";
-
-// `tsc -b` compiles this file into `dist/`, so `bun test` (with no path filter)
-// picks up both `src/index.test.ts` and `dist/index.test.js`. The compiled copy
-// resolves `import.meta.dir` to `dist/`, but `__fixtures__/` only exists under
-// `src/`. Map back to the source tree so both copies find the fixtures.
-const FIX = join(import.meta.dir.replace(/([/\\])dist$/, "$1src"), "__fixtures__");
-
-async function collect<T>(iter: AsyncIterable<T>): Promise<T[]> {
-  const out: T[] = [];
-  for await (const item of iter) out.push(item);
-  return out;
-}
-
-describe("loadDataset — JSONL loader (T1)", () => {
-  test("parses well-formed JSONL with all field shapes", async () => {
-    const ds = await loadDataset(join(FIX, "ok.jsonl"));
-    expect(ds.name).toBe("ok");
-    const samples = await collect(ds.samples);
-    expect(samples).toHaveLength(3);
-    expect(samples[0]).toEqual({ id: "q1", input: "hello" });
-    expect(samples[1]?.expected_output).toBe("bye");
-    expect(samples[2]?.expected_tools).toEqual(["bash", "read"]);
-    expect(samples[2]?.metadata).toEqual({ difficulty: "easy" });
-  });
-
-  test("yields zero samples for empty file", async () => {
-    const ds = await loadDataset(join(FIX, "empty.jsonl"));
-    expect(await collect(ds.samples)).toEqual([]);
-  });
-
-  test("rejects malformed JSON with line number", async () => {
-    const tmp = `${FIX}/__tmp_malformed.jsonl`;
-    await Bun.write(tmp, '{"id":"q1","input":"ok"}\nnot json\n');
-    try {
-      const ds = await loadDataset(tmp);
-      await expect(collect(ds.samples)).rejects.toThrow(/malformed JSON on line 2/);
-    } finally {
-      await Bun.file(tmp).delete();
-    }
-  });
-
-  test("rejects sample missing required fields", async () => {
-    const tmp = `${FIX}/__tmp_invalid.jsonl`;
-    await Bun.write(tmp, '{"input":"no id"}\n');
-    try {
-      const ds = await loadDataset(tmp);
-      await expect(collect(ds.samples)).rejects.toThrow(DatasetLoadError);
-    } finally {
-      await Bun.file(tmp).delete();
-    }
-  });
-
-  test("rejects missing file", async () => {
-    await expect(loadDataset(`${FIX}/does-not-exist.jsonl`)).rejects.toThrow(DatasetLoadError);
-  });
-});
-
-describe("loadDataset — CSV loader (T1)", () => {
-  test("parses CSV with quoted fields, embedded newlines, and multi-tool column", async () => {
-    const ds = await loadDataset(join(FIX, "ok.csv"));
-    const samples = await collect(ds.samples);
-    expect(samples).toHaveLength(3);
-    expect(samples[0]).toEqual({ id: "q1", input: "hello" });
-    expect(samples[1]).toEqual({
-      id: "q2",
-      input: "hello, world",
-      expected_output: "greeting",
-      expected_tools: ["bash", "read"],
-    });
-    expect(samples[2]?.input).toBe("line1\nline2");
-  });
-});
-
-describe("parseCsv — RFC 4180 subset (T1)", () => {
-  test("empty input yields no rows", () => {
-    expect(parseCsv("")).toEqual([]);
-  });
-
-  test("escaped double-quotes inside quoted field", () => {
-    expect(parseCsv('a,"b ""quoted"" c"')).toEqual([["a", 'b "quoted" c']]);
-  });
-
-  test("CRLF line endings", () => {
-    expect(parseCsv("a,b\r\nc,d\r\n")).toEqual([
-      ["a", "b"],
-      ["c", "d"],
-    ]);
-  });
-
-  test("trailing newline is not a synthetic empty row", () => {
-    expect(parseCsv("a,b\n")).toEqual([["a", "b"]]);
-  });
-
-  test("missing final newline still yields the last row", () => {
-    expect(parseCsv("a,b\nc,d")).toEqual([
-      ["a", "b"],
-      ["c", "d"],
-    ]);
-  });
-});
-
-describe("loadDataset — YAML loader (T1)", () => {
-  test("parses Dataset wrapper shape", async () => {
-    const ds = await loadDataset(join(FIX, "ok.yaml"));
-    expect(ds.name).toBe("yaml-fixture");
-    const samples = await collect(ds.samples);
-    expect(samples).toHaveLength(3);
-    expect(samples[0]).toEqual({ id: "q1", input: "hello" });
-  });
-
-  test("parses bare-array shape and derives name from filename", async () => {
-    const ds = await loadDataset(join(FIX, "bare-array.yaml"));
-    expect(ds.name).toBe("bare-array");
-    const samples = await collect(ds.samples);
-    expect(samples).toHaveLength(2);
-    expect(samples[1]?.expected_output).toBe("ok");
-  });
-
-  test("supports .yml extension", async () => {
-    const tmp = `${FIX}/__tmp.yml`;
-    await Bun.write(tmp, "- id: x\n  input: y\n");
-    try {
-      const ds = await loadDataset(tmp);
-      const samples = await collect(ds.samples);
-      expect(samples).toHaveLength(1);
-    } finally {
-      await Bun.file(tmp).delete();
-    }
-  });
-});
-
-describe("loadDataset — dispatch", () => {
-  test("rejects unknown extension", async () => {
-    await expect(loadDataset("/tmp/something.xyz")).rejects.toThrow(/unrecognized dataset source/);
-  });
-
-  test("HTTP scheme dispatches to http loader (rejects offline lookups gracefully)", async () => {
-    // Simply verifies dispatch — actual fetch is exercised in __test__/http.test.ts
-    // when CREWHAUS_TEST_HTTP=1 is set.
-    await expect(loadDataset("http://127.0.0.1:1/never-listens.jsonl")).rejects.toThrow();
-  });
-});

package/src/loaders/csv.ts

@@ -1,123 +0,0 @@
-import { basename } from "node:path";
-import { DatasetLoadError } from "../errors";
-import { type LoadedDataset, type Sample, SampleSchema } from "../index";
-
-export async function loadCsv(path: string): Promise<LoadedDataset> {
-  const file = Bun.file(path);
-  if (!(await file.exists())) {
-    throw new DatasetLoadError(`file not found: ${path}`);
-  }
-  const text = await file.text();
-  const [header, ...dataRows] = parseCsv(text);
-  const name = basename(path).replace(/\.csv$/i, "");
-  if (!header) {
-    return { name, samples: emptyIterable() };
-  }
-
-  return {
-    name,
-    samples: rowsToSamples(dataRows, header, path),
-  };
-}
-
-async function* emptyIterable(): AsyncIterable<Sample> {
-  /* nothing */
-}
-
-async function* rowsToSamples(
-  rows: ReadonlyArray<ReadonlyArray<string>>,
-  header: ReadonlyArray<string>,
-  path: string,
-): AsyncIterable<Sample> {
-  let rowNo = 1; // header was row 0
-  for (const row of rows) {
-    rowNo += 1;
-    if (row.length === 1 && row[0] === "") continue; // trailing newline
-    const obj: Record<string, string | string[]> = {};
-    for (let i = 0; i < header.length; i++) {
-      const key = header[i];
-      if (key === undefined) continue;
-      const cell = row[i] ?? "";
-      // expected_tools is the only array-shaped field — comma-split
-      if (key === "expected_tools" && cell !== "") {
-        obj[key] = cell
-          .split(",")
-          .map((s) => s.trim())
-          .filter((s) => s.length > 0);
-      } else if (cell !== "") {
-        obj[key] = cell;
-      }
-    }
-    const result = SampleSchema.safeParse(obj);
-    if (!result.success) {
-      throw new DatasetLoadError(
-        `invalid sample on row ${rowNo} of ${path}: ${result.error.message}`,
-      );
-    }
-    yield result.data;
-  }
-}
-
-/**
- * RFC 4180-style CSV parser. Handles:
- * - quoted fields with embedded commas, newlines (CRLF/LF), and escaped `""`
- * - trailing empty line tolerance
- * - any combination of CRLF / LF / mixed line endings
- *
- * Returns one `string[]` per row. Empty input yields `[]`.
- */
-export function parseCsv(text: string): string[][] {
-  const rows: string[][] = [];
-  if (text.length === 0) return rows;
-  let row: string[] = [];
-  let field = "";
-  let inQuotes = false;
-  let i = 0;
-  while (i < text.length) {
-    const ch = text[i];
-    if (inQuotes) {
-      if (ch === '"') {
-        if (text[i + 1] === '"') {
-          field += '"';
-          i += 2;
-          continue;
-        }
-        inQuotes = false;
-        i += 1;
-        continue;
-      }
-      field += ch;
-      i += 1;
-      continue;
-    }
-    if (ch === '"') {
-      inQuotes = true;
-      i += 1;
-      continue;
-    }
-    if (ch === ",") {
-      row.push(field);
-      field = "";
-      i += 1;
-      continue;
-    }
-    if (ch === "\n" || ch === "\r") {
-      row.push(field);
-      field = "";
-      rows.push(row);
-      row = [];
-      // swallow CRLF as a single newline
-      if (ch === "\r" && text[i + 1] === "\n") i += 2;
-      else i += 1;
-      continue;
-    }
-    field += ch;
-    i += 1;
-  }
-  // EOF without trailing newline
-  if (field !== "" || row.length > 0) {
-    row.push(field);
-    rows.push(row);
-  }
-  return rows;
-}

package/src/loaders/http.ts

@@ -1,141 +0,0 @@
-import { parse as parseYaml } from "yaml";
-import { DatasetLoadError } from "../errors";
-import { DatasetSchema, type LoadedDataset, type Sample, SampleSchema } from "../index";
-import { parseCsv } from "./csv";
-
-/**
- * HTTP loader — fetches the URL into memory, then dispatches to the
- * appropriate format parser based on the URL extension first, then
- * the response Content-Type as a fallback.
- *
- * Buffers the entire body in memory; HuggingFace-scale datasets should
- * be downloaded ahead of time and loaded via the local-file loaders.
- */
-export async function loadHttp(url: string): Promise<LoadedDataset> {
-  const response = await fetch(url);
-  if (!response.ok) {
-    throw new DatasetLoadError(`HTTP ${response.status} fetching ${url}`);
-  }
-  const text = await response.text();
-  const ct = (response.headers.get("content-type") ?? "").toLowerCase();
-  const lower = url.toLowerCase();
-
-  if (
-    lower.endsWith(".jsonl") ||
-    lower.endsWith(".ndjson") ||
-    ct.includes("application/x-jsonlines") ||
-    ct.includes("application/x-ndjson")
-  ) {
-    return { name: deriveName(url), samples: parseJsonlText(text, url) };
-  }
-  if (lower.endsWith(".csv") || ct.includes("text/csv")) {
-    return { name: deriveName(url), samples: parseCsvText(text, url) };
-  }
-  if (
-    lower.endsWith(".yaml") ||
-    lower.endsWith(".yml") ||
-    ct.includes("application/yaml") ||
-    ct.includes("text/yaml")
-  ) {
-    return parseYamlText(text, url);
-  }
-  throw new DatasetLoadError(
-    `unrecognized HTTP dataset format for ${url} (content-type: ${ct || "unknown"})`,
-  );
-}
-
-function deriveName(url: string): string {
-  try {
-    const u = new URL(url);
-    const last = u.pathname.split("/").filter(Boolean).pop() ?? "remote-dataset";
-    return last.replace(/\.(jsonl|ndjson|csv|yaml|yml)$/i, "");
-  } catch {
-    return "remote-dataset";
-  }
-}
-
-async function* parseJsonlText(text: string, source: string): AsyncIterable<Sample> {
-  let lineNo = 0;
-  for (const raw of text.split(/\r?\n/)) {
-    lineNo += 1;
-    if (raw.trim() === "") continue;
-    let parsed: unknown;
-    try {
-      parsed = JSON.parse(raw);
-    } catch (err) {
-      throw new DatasetLoadError(`malformed JSON on line ${lineNo} of ${source}`, err);
-    }
-    const result = SampleSchema.safeParse(parsed);
-    if (!result.success) {
-      throw new DatasetLoadError(
-        `invalid sample on line ${lineNo} of ${source}: ${result.error.message}`,
-      );
-    }
-    yield result.data;
-  }
-}
-
-async function* parseCsvText(text: string, source: string): AsyncIterable<Sample> {
-  const [header, ...dataRows] = parseCsv(text);
-  if (!header) return;
-  let rowNo = 1;
-  for (const row of dataRows) {
-    rowNo += 1;
-    if (row.length === 1 && row[0] === "") continue;
-    const obj: Record<string, string | string[]> = {};
-    for (let i = 0; i < header.length; i++) {
-      const key = header[i];
-      if (key === undefined) continue;
-      const cell = row[i] ?? "";
-      if (key === "expected_tools" && cell !== "") {
-        obj[key] = cell
-          .split(",")
-          .map((s) => s.trim())
-          .filter((s) => s.length > 0);
-      } else if (cell !== "") {
-        obj[key] = cell;
-      }
-    }
-    const result = SampleSchema.safeParse(obj);
-    if (!result.success) {
-      throw new DatasetLoadError(
-        `invalid sample on row ${rowNo} of ${source}: ${result.error.message}`,
-      );
-    }
-    yield result.data;
-  }
-}
-
-function parseYamlText(text: string, source: string): LoadedDataset {
-  let parsed: unknown;
-  try {
-    parsed = parseYaml(text);
-  } catch (err) {
-    throw new DatasetLoadError(`malformed YAML in ${source}`, err);
-  }
-  if (Array.isArray(parsed)) {
-    return { name: deriveName(source), samples: yieldSamples(parsed, source) };
-  }
-  const result = DatasetSchema.safeParse(parsed);
-  if (!result.success) {
-    throw new DatasetLoadError(`invalid dataset in ${source}: ${result.error.message}`);
-  }
-  return { name: result.data.name, samples: yieldSamples(result.data.samples, source) };
-}
-
-async function* yieldSamples(
-  samples: ReadonlyArray<unknown>,
-  source: string,
-): AsyncIterable<Sample> {
-  let i = 0;
-  for (const raw of samples) {
-    const result = SampleSchema.safeParse(raw);
-    if (!result.success) {
-      throw new DatasetLoadError(
-        `invalid sample at index ${i} of ${source}: ${result.error.message}`,
-      );
-    }
-    yield result.data;
-    i += 1;
-  }
-}

package/src/loaders/jsonl.ts

@@ -1,37 +0,0 @@
-import { basename } from "node:path";
-import { DatasetLoadError } from "../errors";
-import { type LoadedDataset, type Sample, SampleSchema } from "../index";
-
-export async function loadJsonl(path: string): Promise<LoadedDataset> {
-  const file = Bun.file(path);
-  if (!(await file.exists())) {
-    throw new DatasetLoadError(`file not found: ${path}`);
-  }
-  return {
-    name: basename(path).replace(/\.(jsonl|ndjson)$/i, ""),
-    samples: streamJsonl(path),
-  };
-}
-
-async function* streamJsonl(path: string): AsyncIterable<Sample> {
-  const file = Bun.file(path);
-  const text = await file.text();
-  let lineNo = 0;
-  for (const raw of text.split(/\r?\n/)) {
-    lineNo += 1;
-    if (raw.trim() === "") continue;
-    let parsed: unknown;
-    try {
-      parsed = JSON.parse(raw);
-    } catch (err) {
-      throw new DatasetLoadError(`malformed JSON on line ${lineNo} of ${path}`, err);
-    }
-    const result = SampleSchema.safeParse(parsed);
-    if (!result.success) {
-      throw new DatasetLoadError(
-        `invalid sample on line ${lineNo} of ${path}: ${result.error.message}`,
-      );
-    }
-    yield result.data;
-  }
-}