@crewhaus/eval-dataset 0.3.2 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -2,12 +2,14 @@
2
2
  * Catalog R-eval `eval-dataset` — load samples from JSONL, CSV, YAML, or HTTP.
3
3
  *
4
4
  * Sample shape is the contract every other eval-* module consumes:
5
- * { id, input, expected_output?, expected_tools?, metadata? }
5
+ * { id, input, history?, expected_output?, expected_tools?, metadata? }
6
6
  *
7
7
  * Loaders return an `AsyncIterable<Sample>` so 100k-row datasets stream rather
8
- * than load fully into memory. JSONL/CSV truly stream off the file system via
9
- * `Bun.file().stream()`. YAML and HTTP buffer (YAML can't stream; HTTP buffers
10
- * before re-dispatching to the format-specific loader).
8
+ * than load fully into memory. JSONL truly streams — local files via
9
+ * `Bun.file().stream()`, HTTP bodies via the fetch reader (B24) — through one
10
+ * shared incremental line parser, memory bounded by the longest line. CSV and
11
+ * YAML buffer their source whole, locally and over HTTP alike (YAML can't
12
+ * stream, CSV quoting spans physical lines).
11
13
  *
12
14
  * Reference: build-roadmap.md §16.
13
15
  */
@@ -17,21 +19,70 @@ import { loadCsv } from "./loaders/csv";
17
19
  import { loadHttp } from "./loaders/http";
18
20
  import { loadJsonl } from "./loaders/jsonl";
19
21
  import { loadYaml } from "./loaders/yaml";
22
+ /**
23
+ * B14 — one prior conversation turn of a multi-turn sample. STRICT item
24
+ * shape: a history message is seeded into the eval session transcript
25
+ * VERBATIM (no model call runs for it), so an unknown key is an authoring
26
+ * mistake to reject loudly, not an extension point.
27
+ */
28
+ export declare const HistoryMessageSchema: z.ZodObject<{
29
+ role: z.ZodEnum<["user", "assistant"]>;
30
+ content: z.ZodString;
31
+ }, "strict", z.ZodTypeAny, {
32
+ role: "user" | "assistant";
33
+ content: string;
34
+ }, {
35
+ role: "user" | "assistant";
36
+ content: string;
37
+ }>;
38
+ export type HistoryMessage = z.infer<typeof HistoryMessageSchema>;
20
39
  export declare const SampleSchema: z.ZodObject<{
21
40
  id: z.ZodString;
22
41
  input: z.ZodString;
42
+ /**
43
+ * B14 — optional prior conversation turns (MT-Bench-style multi-turn
44
+ * evaluation). `input` stays the required FINAL user message — every
45
+ * history-less dataset parses byte-identically — and when `history` is
46
+ * present the eval invoker seeds these messages into the session
47
+ * transcript verbatim before running `input`. Non-empty when present:
48
+ * `history: []` says "multi-turn" while carrying no turns, an authoring
49
+ * error worth rejecting at load. One DELIBERATE break rides along: a
50
+ * hand-authored dataset whose samples already carried a free-form
51
+ * `history` key was silently ignored before (Zod strip mode dropped the
52
+ * unknown key) — it now validates, so a shape-mismatched value fails the
53
+ * load loudly, and a shape-matched one starts seeding turns (changing
54
+ * that dataset's sample hashes). Rename the key or fix its shape.
55
+ */
56
+ history: z.ZodOptional<z.ZodArray<z.ZodObject<{
57
+ role: z.ZodEnum<["user", "assistant"]>;
58
+ content: z.ZodString;
59
+ }, "strict", z.ZodTypeAny, {
60
+ role: "user" | "assistant";
61
+ content: string;
62
+ }, {
63
+ role: "user" | "assistant";
64
+ content: string;
65
+ }>, "many">>;
23
66
  expected_output: z.ZodOptional<z.ZodString>;
24
67
  expected_tools: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
25
68
  metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
26
69
  }, "strip", z.ZodTypeAny, {
27
70
  id: string;
28
71
  input: string;
72
+ history?: {
73
+ role: "user" | "assistant";
74
+ content: string;
75
+ }[] | undefined;
29
76
  expected_output?: string | undefined;
30
77
  expected_tools?: string[] | undefined;
31
78
  metadata?: Record<string, unknown> | undefined;
32
79
  }, {
33
80
  id: string;
34
81
  input: string;
82
+ history?: {
83
+ role: "user" | "assistant";
84
+ content: string;
85
+ }[] | undefined;
35
86
  expected_output?: string | undefined;
36
87
  expected_tools?: string[] | undefined;
37
88
  metadata?: Record<string, unknown> | undefined;
@@ -42,18 +93,50 @@ export declare const DatasetSchema: z.ZodObject<{
42
93
  samples: z.ZodArray<z.ZodObject<{
43
94
  id: z.ZodString;
44
95
  input: z.ZodString;
96
+ /**
97
+ * B14 — optional prior conversation turns (MT-Bench-style multi-turn
98
+ * evaluation). `input` stays the required FINAL user message — every
99
+ * history-less dataset parses byte-identically — and when `history` is
100
+ * present the eval invoker seeds these messages into the session
101
+ * transcript verbatim before running `input`. Non-empty when present:
102
+ * `history: []` says "multi-turn" while carrying no turns, an authoring
103
+ * error worth rejecting at load. One DELIBERATE break rides along: a
104
+ * hand-authored dataset whose samples already carried a free-form
105
+ * `history` key was silently ignored before (Zod strip mode dropped the
106
+ * unknown key) — it now validates, so a shape-mismatched value fails the
107
+ * load loudly, and a shape-matched one starts seeding turns (changing
108
+ * that dataset's sample hashes). Rename the key or fix its shape.
109
+ */
110
+ history: z.ZodOptional<z.ZodArray<z.ZodObject<{
111
+ role: z.ZodEnum<["user", "assistant"]>;
112
+ content: z.ZodString;
113
+ }, "strict", z.ZodTypeAny, {
114
+ role: "user" | "assistant";
115
+ content: string;
116
+ }, {
117
+ role: "user" | "assistant";
118
+ content: string;
119
+ }>, "many">>;
45
120
  expected_output: z.ZodOptional<z.ZodString>;
46
121
  expected_tools: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
47
122
  metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
48
123
  }, "strip", z.ZodTypeAny, {
49
124
  id: string;
50
125
  input: string;
126
+ history?: {
127
+ role: "user" | "assistant";
128
+ content: string;
129
+ }[] | undefined;
51
130
  expected_output?: string | undefined;
52
131
  expected_tools?: string[] | undefined;
53
132
  metadata?: Record<string, unknown> | undefined;
54
133
  }, {
55
134
  id: string;
56
135
  input: string;
136
+ history?: {
137
+ role: "user" | "assistant";
138
+ content: string;
139
+ }[] | undefined;
57
140
  expected_output?: string | undefined;
58
141
  expected_tools?: string[] | undefined;
59
142
  metadata?: Record<string, unknown> | undefined;
@@ -63,6 +146,10 @@ export declare const DatasetSchema: z.ZodObject<{
63
146
  samples: {
64
147
  id: string;
65
148
  input: string;
149
+ history?: {
150
+ role: "user" | "assistant";
151
+ content: string;
152
+ }[] | undefined;
66
153
  expected_output?: string | undefined;
67
154
  expected_tools?: string[] | undefined;
68
155
  metadata?: Record<string, unknown> | undefined;
@@ -72,6 +159,10 @@ export declare const DatasetSchema: z.ZodObject<{
72
159
  samples: {
73
160
  id: string;
74
161
  input: string;
162
+ history?: {
163
+ role: "user" | "assistant";
164
+ content: string;
165
+ }[] | undefined;
75
166
  expected_output?: string | undefined;
76
167
  expected_tools?: string[] | undefined;
77
168
  metadata?: Record<string, unknown> | undefined;
package/dist/index.js CHANGED
@@ -2,12 +2,14 @@
2
2
  * Catalog R-eval `eval-dataset` — load samples from JSONL, CSV, YAML, or HTTP.
3
3
  *
4
4
  * Sample shape is the contract every other eval-* module consumes:
5
- * { id, input, expected_output?, expected_tools?, metadata? }
5
+ * { id, input, history?, expected_output?, expected_tools?, metadata? }
6
6
  *
7
7
  * Loaders return an `AsyncIterable<Sample>` so 100k-row datasets stream rather
8
- * than load fully into memory. JSONL/CSV truly stream off the file system via
9
- * `Bun.file().stream()`. YAML and HTTP buffer (YAML can't stream; HTTP buffers
10
- * before re-dispatching to the format-specific loader).
8
+ * than load fully into memory. JSONL truly streams — local files via
9
+ * `Bun.file().stream()`, HTTP bodies via the fetch reader (B24) — through one
10
+ * shared incremental line parser, memory bounded by the longest line. CSV and
11
+ * YAML buffer their source whole, locally and over HTTP alike (YAML can't
12
+ * stream, CSV quoting spans physical lines).
11
13
  *
12
14
  * Reference: build-roadmap.md §16.
13
15
  */
@@ -17,9 +19,36 @@ import { loadCsv } from "./loaders/csv";
17
19
  import { loadHttp } from "./loaders/http";
18
20
  import { loadJsonl } from "./loaders/jsonl";
19
21
  import { loadYaml } from "./loaders/yaml";
22
+ /**
23
+ * B14 — one prior conversation turn of a multi-turn sample. STRICT item
24
+ * shape: a history message is seeded into the eval session transcript
25
+ * VERBATIM (no model call runs for it), so an unknown key is an authoring
26
+ * mistake to reject loudly, not an extension point.
27
+ */
28
+ export const HistoryMessageSchema = z
29
+ .object({
30
+ role: z.enum(["user", "assistant"]),
31
+ content: z.string(),
32
+ })
33
+ .strict();
20
34
  export const SampleSchema = z.object({
21
35
  id: z.string().min(1),
22
36
  input: z.string(),
37
+ /**
38
+ * B14 — optional prior conversation turns (MT-Bench-style multi-turn
39
+ * evaluation). `input` stays the required FINAL user message — every
40
+ * history-less dataset parses byte-identically — and when `history` is
41
+ * present the eval invoker seeds these messages into the session
42
+ * transcript verbatim before running `input`. Non-empty when present:
43
+ * `history: []` says "multi-turn" while carrying no turns, an authoring
44
+ * error worth rejecting at load. One DELIBERATE break rides along: a
45
+ * hand-authored dataset whose samples already carried a free-form
46
+ * `history` key was silently ignored before (Zod strip mode dropped the
47
+ * unknown key) — it now validates, so a shape-mismatched value fails the
48
+ * load loudly, and a shape-matched one starts seeding turns (changing
49
+ * that dataset's sample hashes). Rename the key or fix its shape.
50
+ */
51
+ history: z.array(HistoryMessageSchema).min(1).optional(),
23
52
  expected_output: z.string().optional(),
24
53
  expected_tools: z.array(z.string()).optional(),
25
54
  metadata: z.record(z.unknown()).optional(),
@@ -1,5 +1,19 @@
1
- import { type LoadedDataset } from "../index";
1
+ import { type LoadedDataset, type Sample } from "../index";
2
2
  export declare function loadCsv(path: string): Promise<LoadedDataset>;
3
+ /**
4
+ * Shared CSV row → Sample generator (the file loader and the HTTP CSV path
5
+ * both dispatch here, so structured-column handling can never drift between
6
+ * them). Empty cells are omitted — a blank column keeps the sample
7
+ * byte-identical to one authored without it. Structured columns:
8
+ * - `expected_tools` — comma-split list.
9
+ * - `metadata` (NEW-formats-1) / `history` (B14) — JSON-encoded cells,
10
+ * parsed here so CSV datasets can carry the fields the rest of the
11
+ * eval system leans on (slicing, provenance, multi-turn). Malformed
12
+ * JSON is a hard `DatasetLoadError` naming the row — the raw string
13
+ * would otherwise reach schema validation and fail with a far more
14
+ * confusing type error (previously EVERY non-empty metadata cell did).
15
+ */
16
+ export declare function csvRowsToSamples(rows: ReadonlyArray<ReadonlyArray<string>>, header: ReadonlyArray<string>, source: string): AsyncIterable<Sample>;
3
17
  /**
4
18
  * RFC 4180-style CSV parser. Handles:
5
19
  * - quoted fields with embedded commas, newlines (CRLF/LF), and escaped `""`
@@ -14,13 +14,26 @@ export async function loadCsv(path) {
14
14
  }
15
15
  return {
16
16
  name,
17
- samples: rowsToSamples(dataRows, header, path),
17
+ samples: csvRowsToSamples(dataRows, header, path),
18
18
  };
19
19
  }
20
20
  async function* emptyIterable() {
21
21
  /* nothing */
22
22
  }
23
- async function* rowsToSamples(rows, header, path) {
23
+ /**
24
+ * Shared CSV row → Sample generator (the file loader and the HTTP CSV path
25
+ * both dispatch here, so structured-column handling can never drift between
26
+ * them). Empty cells are omitted — a blank column keeps the sample
27
+ * byte-identical to one authored without it. Structured columns:
28
+ * - `expected_tools` — comma-split list.
29
+ * - `metadata` (NEW-formats-1) / `history` (B14) — JSON-encoded cells,
30
+ * parsed here so CSV datasets can carry the fields the rest of the
31
+ * eval system leans on (slicing, provenance, multi-turn). Malformed
32
+ * JSON is a hard `DatasetLoadError` naming the row — the raw string
33
+ * would otherwise reach schema validation and fail with a far more
34
+ * confusing type error (previously EVERY non-empty metadata cell did).
35
+ */
36
+ export async function* csvRowsToSamples(rows, header, source) {
24
37
  let rowNo = 1; // header was row 0
25
38
  for (const row of rows) {
26
39
  rowNo += 1;
@@ -32,20 +45,30 @@ async function* rowsToSamples(rows, header, path) {
32
45
  if (key === undefined)
33
46
  continue;
34
47
  const cell = row[i] ?? "";
35
- // expected_tools is the only array-shaped field — comma-split
36
- if (key === "expected_tools" && cell !== "") {
48
+ if (cell === "")
49
+ continue;
50
+ if (key === "expected_tools") {
51
+ // expected_tools is the only comma-split field.
37
52
  obj[key] = cell
38
53
  .split(",")
39
54
  .map((s) => s.trim())
40
55
  .filter((s) => s.length > 0);
41
56
  }
42
- else if (cell !== "") {
57
+ else if (key === "metadata" || key === "history") {
58
+ try {
59
+ obj[key] = JSON.parse(cell);
60
+ }
61
+ catch (err) {
62
+ throw new DatasetLoadError(`malformed JSON in "${key}" column on row ${rowNo} of ${source}`, err);
63
+ }
64
+ }
65
+ else {
43
66
  obj[key] = cell;
44
67
  }
45
68
  }
46
69
  const result = SampleSchema.safeParse(obj);
47
70
  if (!result.success) {
48
- throw new DatasetLoadError(`invalid sample on row ${rowNo} of ${path}: ${result.error.message}`);
71
+ throw new DatasetLoadError(`invalid sample on row ${rowNo} of ${source}: ${result.error.message}`);
49
72
  }
50
73
  yield result.data;
51
74
  }
@@ -1,10 +1,14 @@
1
1
  import { type LoadedDataset } from "../index";
2
2
  /**
3
- * HTTP loader — fetches the URL into memory, then dispatches to the
4
- * appropriate format parser based on the URL extension first, then
5
- * the response Content-Type as a fallback.
3
+ * HTTP loader — dispatches on the URL extension first, then the response
4
+ * Content-Type as a fallback.
6
5
  *
7
- * Buffers the entire body in memory; HuggingFace-scale datasets should
8
- * be downloaded ahead of time and loaded via the local-file loaders.
6
+ * B24 — `.jsonl`/`.ndjson` bodies STREAM line-by-line off the fetch body
7
+ * reader through the SAME incremental parser as the local file loader:
8
+ * memory stays bounded by the longest line, not the body, so a multi-GB
9
+ * JSONL URL parses as it downloads. CSV and YAML buffer the entire body via
10
+ * `response.text()` — YAML cannot be parsed incrementally and CSV quoting
11
+ * can span physical lines — matching the (also buffered) local CSV/YAML
12
+ * loaders.
9
13
  */
10
14
  export declare function loadHttp(url: string): Promise<LoadedDataset>;
@@ -1,37 +1,45 @@
1
1
  import { parse as parseYaml } from "yaml";
2
2
  import { DatasetLoadError } from "../errors";
3
3
  import { DatasetSchema, SampleSchema } from "../index";
4
- import { parseCsv } from "./csv";
4
+ import { csvRowsToSamples, parseCsv } from "./csv";
5
+ import { samplesFromJsonlStream, samplesFromJsonlText } from "./jsonl";
5
6
  /**
6
- * HTTP loader — fetches the URL into memory, then dispatches to the
7
- * appropriate format parser based on the URL extension first, then
8
- * the response Content-Type as a fallback.
7
+ * HTTP loader — dispatches on the URL extension first, then the response
8
+ * Content-Type as a fallback.
9
9
  *
10
- * Buffers the entire body in memory; HuggingFace-scale datasets should
11
- * be downloaded ahead of time and loaded via the local-file loaders.
10
+ * B24 — `.jsonl`/`.ndjson` bodies STREAM line-by-line off the fetch body
11
+ * reader through the SAME incremental parser as the local file loader:
12
+ * memory stays bounded by the longest line, not the body, so a multi-GB
13
+ * JSONL URL parses as it downloads. CSV and YAML buffer the entire body via
14
+ * `response.text()` — YAML cannot be parsed incrementally and CSV quoting
15
+ * can span physical lines — matching the (also buffered) local CSV/YAML
16
+ * loaders.
12
17
  */
13
18
  export async function loadHttp(url) {
14
19
  const response = await fetch(url);
15
20
  if (!response.ok) {
16
21
  throw new DatasetLoadError(`HTTP ${response.status} fetching ${url}`);
17
22
  }
18
- const text = await response.text();
19
23
  const ct = (response.headers.get("content-type") ?? "").toLowerCase();
20
24
  const lower = url.toLowerCase();
21
25
  if (lower.endsWith(".jsonl") ||
22
26
  lower.endsWith(".ndjson") ||
23
27
  ct.includes("application/x-jsonlines") ||
24
28
  ct.includes("application/x-ndjson")) {
25
- return { name: deriveName(url), samples: parseJsonlText(text, url) };
29
+ return { name: deriveName(url), samples: streamJsonlBody(response, url) };
26
30
  }
27
31
  if (lower.endsWith(".csv") || ct.includes("text/csv")) {
28
- return { name: deriveName(url), samples: parseCsvText(text, url) };
32
+ // Buffered: CSV quoting can span physical lines. Rows then flow through
33
+ // the SAME row → Sample generator as the file loader (metadata/history
34
+ // JSON columns included).
35
+ const [header, ...dataRows] = parseCsv(await response.text());
36
+ return { name: deriveName(url), samples: csvRowsToSamples(dataRows, header ?? [], url) };
29
37
  }
30
38
  if (lower.endsWith(".yaml") ||
31
39
  lower.endsWith(".yml") ||
32
40
  ct.includes("application/yaml") ||
33
41
  ct.includes("text/yaml")) {
34
- return parseYamlText(text, url);
42
+ return parseYamlText(await response.text(), url);
35
43
  }
36
44
  throw new DatasetLoadError(`unrecognized HTTP dataset format for ${url} (content-type: ${ct || "unknown"})`);
37
45
  }
@@ -45,57 +53,19 @@ function deriveName(url) {
45
53
  return "remote-dataset";
46
54
  }
47
55
  }
48
- async function* parseJsonlText(text, source) {
49
- let lineNo = 0;
50
- for (const raw of text.split(/\r?\n/)) {
51
- lineNo += 1;
52
- if (raw.trim() === "")
53
- continue;
54
- let parsed;
55
- try {
56
- parsed = JSON.parse(raw);
57
- }
58
- catch (err) {
59
- throw new DatasetLoadError(`malformed JSON on line ${lineNo} of ${source}`, err);
60
- }
61
- const result = SampleSchema.safeParse(parsed);
62
- if (!result.success) {
63
- throw new DatasetLoadError(`invalid sample on line ${lineNo} of ${source}: ${result.error.message}`);
64
- }
65
- yield result.data;
66
- }
67
- }
68
- async function* parseCsvText(text, source) {
69
- const [header, ...dataRows] = parseCsv(text);
70
- if (!header)
56
+ /**
57
+ * B24 — HTTP `.jsonl` delegates to the shared incremental JSONL parser (see
58
+ * `loaders/jsonl.ts`) over the fetch body reader. A null body (null-body
59
+ * statuses, minimal Response stubs) falls back to buffering — there is
60
+ * nothing to stream.
61
+ */
62
+ async function* streamJsonlBody(response, source) {
63
+ const body = response.body;
64
+ if (body === null || body === undefined) {
65
+ yield* samplesFromJsonlText(await response.text(), source);
71
66
  return;
72
- let rowNo = 1;
73
- for (const row of dataRows) {
74
- rowNo += 1;
75
- if (row.length === 1 && row[0] === "")
76
- continue;
77
- const obj = {};
78
- for (let i = 0; i < header.length; i++) {
79
- const key = header[i];
80
- if (key === undefined)
81
- continue;
82
- const cell = row[i] ?? "";
83
- if (key === "expected_tools" && cell !== "") {
84
- obj[key] = cell
85
- .split(",")
86
- .map((s) => s.trim())
87
- .filter((s) => s.length > 0);
88
- }
89
- else if (cell !== "") {
90
- obj[key] = cell;
91
- }
92
- }
93
- const result = SampleSchema.safeParse(obj);
94
- if (!result.success) {
95
- throw new DatasetLoadError(`invalid sample on row ${rowNo} of ${source}: ${result.error.message}`);
96
- }
97
- yield result.data;
98
67
  }
68
+ yield* samplesFromJsonlStream(body, source);
99
69
  }
100
70
  function parseYamlText(text, source) {
101
71
  let parsed;
@@ -1,2 +1,15 @@
1
- import { type LoadedDataset } from "../index";
1
+ import { type LoadedDataset, type Sample } from "../index";
2
2
  export declare function loadJsonl(path: string): Promise<LoadedDataset>;
3
+ /**
4
+ * Buffered-text JSONL → Sample generator (the HTTP loader's null-body
5
+ * fallback). Same parser, same line numbering as the streaming path.
6
+ */
7
+ export declare function samplesFromJsonlText(text: string, source: string): AsyncIterable<Sample>;
8
+ /**
9
+ * B24 — incremental JSONL over any byte stream (local file or fetch body):
10
+ * pull chunks off the reader, split on newlines (tolerating CRLF and JSON
11
+ * lines split across chunk boundaries), and validate each line as it
12
+ * arrives. Line numbering matches the buffered parser's exactly (blank lines
13
+ * count, then skip).
14
+ */
15
+ export declare function samplesFromJsonlStream(stream: ReadableStream<Uint8Array>, source: string): AsyncIterable<Sample>;
@@ -8,28 +8,99 @@ export async function loadJsonl(path) {
8
8
  }
9
9
  return {
10
10
  name: basename(path).replace(/\.(jsonl|ndjson)$/i, ""),
11
- samples: streamJsonl(path),
11
+ samples: streamJsonlFile(path),
12
12
  };
13
13
  }
14
- async function* streamJsonl(path) {
15
- const file = Bun.file(path);
16
- const text = await file.text();
14
+ /**
15
+ * B24 — local `.jsonl` files stream through the SAME incremental line parser
16
+ * as HTTP bodies: `Bun.file().stream()` is a `ReadableStream<Uint8Array>`
17
+ * exactly like a fetch body, so memory stays bounded by the longest line, not
18
+ * the file. The generator wrapper keeps the file unopened until the caller
19
+ * actually iterates (loaders hand back lazy iterables, not open handles).
20
+ */
21
+ async function* streamJsonlFile(path) {
22
+ yield* samplesFromJsonlStream(Bun.file(path).stream(), path);
23
+ }
24
+ /**
25
+ * One line-numbering + parse + validate closure shared by every JSONL entry
26
+ * point (local file, HTTP body, buffered fallback), so error messages
27
+ * ("malformed JSON on line N of SOURCE") can never drift between loaders.
28
+ * Blank lines count toward numbering, then skip; a trailing `\r` (CRLF input
29
+ * split on `\n`) is tolerated.
30
+ */
31
+ function makeLineParser(source) {
17
32
  let lineNo = 0;
18
- for (const raw of text.split(/\r?\n/)) {
33
+ return (line) => {
19
34
  lineNo += 1;
35
+ const raw = line.endsWith("\r") ? line.slice(0, -1) : line;
20
36
  if (raw.trim() === "")
21
- continue;
37
+ return undefined;
22
38
  let parsed;
23
39
  try {
24
40
  parsed = JSON.parse(raw);
25
41
  }
26
42
  catch (err) {
27
- throw new DatasetLoadError(`malformed JSON on line ${lineNo} of ${path}`, err);
43
+ throw new DatasetLoadError(`malformed JSON on line ${lineNo} of ${source}`, err);
28
44
  }
29
45
  const result = SampleSchema.safeParse(parsed);
30
46
  if (!result.success) {
31
- throw new DatasetLoadError(`invalid sample on line ${lineNo} of ${path}: ${result.error.message}`);
47
+ throw new DatasetLoadError(`invalid sample on line ${lineNo} of ${source}: ${result.error.message}`);
48
+ }
49
+ return result.data;
50
+ };
51
+ }
52
+ /**
53
+ * Buffered-text JSONL → Sample generator (the HTTP loader's null-body
54
+ * fallback). Same parser, same line numbering as the streaming path.
55
+ */
56
+ export async function* samplesFromJsonlText(text, source) {
57
+ const parseLine = makeLineParser(source);
58
+ for (const line of text.split("\n")) {
59
+ const sample = parseLine(line);
60
+ if (sample !== undefined)
61
+ yield sample;
62
+ }
63
+ }
64
+ /**
65
+ * B24 — incremental JSONL over any byte stream (local file or fetch body):
66
+ * pull chunks off the reader, split on newlines (tolerating CRLF and JSON
67
+ * lines split across chunk boundaries), and validate each line as it
68
+ * arrives. Line numbering matches the buffered parser's exactly (blank lines
69
+ * count, then skip).
70
+ */
71
+ export async function* samplesFromJsonlStream(stream, source) {
72
+ const parseLine = makeLineParser(source);
73
+ const reader = stream.getReader();
74
+ const decoder = new TextDecoder();
75
+ let carry = "";
76
+ try {
77
+ while (true) {
78
+ const { done, value } = await reader.read();
79
+ if (done)
80
+ break;
81
+ carry += decoder.decode(value, { stream: true });
82
+ let nl = carry.indexOf("\n");
83
+ while (nl !== -1) {
84
+ const line = carry.slice(0, nl);
85
+ carry = carry.slice(nl + 1);
86
+ const sample = parseLine(line);
87
+ if (sample !== undefined)
88
+ yield sample;
89
+ nl = carry.indexOf("\n");
90
+ }
32
91
  }
33
- yield result.data;
92
+ // Flush any multi-byte tail the decoder held back, then the final
93
+ // (newline-less) line — or the trailing "" a newline-terminated source
94
+ // leaves, which parseLine counts and skips like the buffered parser.
95
+ carry += decoder.decode();
96
+ const last = parseLine(carry);
97
+ if (last !== undefined)
98
+ yield last;
99
+ }
100
+ finally {
101
+ // Early exit (consumer break, invalid line mid-stream) must not leave a
102
+ // connection or file handle dangling; on normal completion this is a
103
+ // no-op.
104
+ await reader.cancel().catch(() => { });
34
105
  }
35
106
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crewhaus/eval-dataset",
3
- "version": "0.3.2",
3
+ "version": "0.4.2",
4
4
  "type": "module",
5
5
  "description": "Eval dataset loaders (JSONL, CSV, YAML, HTTP) with a lazy iterator API",
6
6
  "main": "dist/index.js",
@@ -15,7 +15,7 @@
15
15
  "test": "bun test src"
16
16
  },
17
17
  "dependencies": {
18
- "@crewhaus/errors": "0.3.2",
18
+ "@crewhaus/errors": "0.4.2",
19
19
  "yaml": "^2.6.0",
20
20
  "zod": "^3.23.8"
21
21
  },