@crewhaus/eval-dataset 0.3.2 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +95 -4
- package/dist/index.js +33 -4
- package/dist/loaders/csv.d.ts +15 -1
- package/dist/loaders/csv.js +29 -6
- package/dist/loaders/http.d.ts +9 -5
- package/dist/loaders/http.js +29 -59
- package/dist/loaders/jsonl.d.ts +14 -1
- package/dist/loaders/jsonl.js +80 -9
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -2,12 +2,14 @@
|
|
|
2
2
|
* Catalog R-eval `eval-dataset` — load samples from JSONL, CSV, YAML, or HTTP.
|
|
3
3
|
*
|
|
4
4
|
* Sample shape is the contract every other eval-* module consumes:
|
|
5
|
-
* { id, input, expected_output?, expected_tools?, metadata? }
|
|
5
|
+
* { id, input, history?, expected_output?, expected_tools?, metadata? }
|
|
6
6
|
*
|
|
7
7
|
* Loaders return an `AsyncIterable<Sample>` so 100k-row datasets stream rather
|
|
8
|
-
* than load fully into memory. JSONL
|
|
9
|
-
* `Bun.file().stream()
|
|
10
|
-
*
|
|
8
|
+
* than load fully into memory. JSONL truly streams — local files via
|
|
9
|
+
* `Bun.file().stream()`, HTTP bodies via the fetch reader (B24) — through one
|
|
10
|
+
* shared incremental line parser, memory bounded by the longest line. CSV and
|
|
11
|
+
* YAML buffer their source whole, locally and over HTTP alike (YAML can't
|
|
12
|
+
* stream, CSV quoting spans physical lines).
|
|
11
13
|
*
|
|
12
14
|
* Reference: build-roadmap.md §16.
|
|
13
15
|
*/
|
|
@@ -17,21 +19,70 @@ import { loadCsv } from "./loaders/csv";
|
|
|
17
19
|
import { loadHttp } from "./loaders/http";
|
|
18
20
|
import { loadJsonl } from "./loaders/jsonl";
|
|
19
21
|
import { loadYaml } from "./loaders/yaml";
|
|
22
|
+
/**
|
|
23
|
+
* B14 — one prior conversation turn of a multi-turn sample. STRICT item
|
|
24
|
+
* shape: a history message is seeded into the eval session transcript
|
|
25
|
+
* VERBATIM (no model call runs for it), so an unknown key is an authoring
|
|
26
|
+
* mistake to reject loudly, not an extension point.
|
|
27
|
+
*/
|
|
28
|
+
export declare const HistoryMessageSchema: z.ZodObject<{
|
|
29
|
+
role: z.ZodEnum<["user", "assistant"]>;
|
|
30
|
+
content: z.ZodString;
|
|
31
|
+
}, "strict", z.ZodTypeAny, {
|
|
32
|
+
role: "user" | "assistant";
|
|
33
|
+
content: string;
|
|
34
|
+
}, {
|
|
35
|
+
role: "user" | "assistant";
|
|
36
|
+
content: string;
|
|
37
|
+
}>;
|
|
38
|
+
export type HistoryMessage = z.infer<typeof HistoryMessageSchema>;
|
|
20
39
|
export declare const SampleSchema: z.ZodObject<{
|
|
21
40
|
id: z.ZodString;
|
|
22
41
|
input: z.ZodString;
|
|
42
|
+
/**
|
|
43
|
+
* B14 — optional prior conversation turns (MT-Bench-style multi-turn
|
|
44
|
+
* evaluation). `input` stays the required FINAL user message — every
|
|
45
|
+
* history-less dataset parses byte-identically — and when `history` is
|
|
46
|
+
* present the eval invoker seeds these messages into the session
|
|
47
|
+
* transcript verbatim before running `input`. Non-empty when present:
|
|
48
|
+
* `history: []` says "multi-turn" while carrying no turns, an authoring
|
|
49
|
+
* error worth rejecting at load. One DELIBERATE break rides along: a
|
|
50
|
+
* hand-authored dataset whose samples already carried a free-form
|
|
51
|
+
* `history` key was silently ignored before (Zod strip mode dropped the
|
|
52
|
+
* unknown key) — it now validates, so a shape-mismatched value fails the
|
|
53
|
+
* load loudly, and a shape-matched one starts seeding turns (changing
|
|
54
|
+
* that dataset's sample hashes). Rename the key or fix its shape.
|
|
55
|
+
*/
|
|
56
|
+
history: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
57
|
+
role: z.ZodEnum<["user", "assistant"]>;
|
|
58
|
+
content: z.ZodString;
|
|
59
|
+
}, "strict", z.ZodTypeAny, {
|
|
60
|
+
role: "user" | "assistant";
|
|
61
|
+
content: string;
|
|
62
|
+
}, {
|
|
63
|
+
role: "user" | "assistant";
|
|
64
|
+
content: string;
|
|
65
|
+
}>, "many">>;
|
|
23
66
|
expected_output: z.ZodOptional<z.ZodString>;
|
|
24
67
|
expected_tools: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
25
68
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
26
69
|
}, "strip", z.ZodTypeAny, {
|
|
27
70
|
id: string;
|
|
28
71
|
input: string;
|
|
72
|
+
history?: {
|
|
73
|
+
role: "user" | "assistant";
|
|
74
|
+
content: string;
|
|
75
|
+
}[] | undefined;
|
|
29
76
|
expected_output?: string | undefined;
|
|
30
77
|
expected_tools?: string[] | undefined;
|
|
31
78
|
metadata?: Record<string, unknown> | undefined;
|
|
32
79
|
}, {
|
|
33
80
|
id: string;
|
|
34
81
|
input: string;
|
|
82
|
+
history?: {
|
|
83
|
+
role: "user" | "assistant";
|
|
84
|
+
content: string;
|
|
85
|
+
}[] | undefined;
|
|
35
86
|
expected_output?: string | undefined;
|
|
36
87
|
expected_tools?: string[] | undefined;
|
|
37
88
|
metadata?: Record<string, unknown> | undefined;
|
|
@@ -42,18 +93,50 @@ export declare const DatasetSchema: z.ZodObject<{
|
|
|
42
93
|
samples: z.ZodArray<z.ZodObject<{
|
|
43
94
|
id: z.ZodString;
|
|
44
95
|
input: z.ZodString;
|
|
96
|
+
/**
|
|
97
|
+
* B14 — optional prior conversation turns (MT-Bench-style multi-turn
|
|
98
|
+
* evaluation). `input` stays the required FINAL user message — every
|
|
99
|
+
* history-less dataset parses byte-identically — and when `history` is
|
|
100
|
+
* present the eval invoker seeds these messages into the session
|
|
101
|
+
* transcript verbatim before running `input`. Non-empty when present:
|
|
102
|
+
* `history: []` says "multi-turn" while carrying no turns, an authoring
|
|
103
|
+
* error worth rejecting at load. One DELIBERATE break rides along: a
|
|
104
|
+
* hand-authored dataset whose samples already carried a free-form
|
|
105
|
+
* `history` key was silently ignored before (Zod strip mode dropped the
|
|
106
|
+
* unknown key) — it now validates, so a shape-mismatched value fails the
|
|
107
|
+
* load loudly, and a shape-matched one starts seeding turns (changing
|
|
108
|
+
* that dataset's sample hashes). Rename the key or fix its shape.
|
|
109
|
+
*/
|
|
110
|
+
history: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
111
|
+
role: z.ZodEnum<["user", "assistant"]>;
|
|
112
|
+
content: z.ZodString;
|
|
113
|
+
}, "strict", z.ZodTypeAny, {
|
|
114
|
+
role: "user" | "assistant";
|
|
115
|
+
content: string;
|
|
116
|
+
}, {
|
|
117
|
+
role: "user" | "assistant";
|
|
118
|
+
content: string;
|
|
119
|
+
}>, "many">>;
|
|
45
120
|
expected_output: z.ZodOptional<z.ZodString>;
|
|
46
121
|
expected_tools: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
47
122
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
48
123
|
}, "strip", z.ZodTypeAny, {
|
|
49
124
|
id: string;
|
|
50
125
|
input: string;
|
|
126
|
+
history?: {
|
|
127
|
+
role: "user" | "assistant";
|
|
128
|
+
content: string;
|
|
129
|
+
}[] | undefined;
|
|
51
130
|
expected_output?: string | undefined;
|
|
52
131
|
expected_tools?: string[] | undefined;
|
|
53
132
|
metadata?: Record<string, unknown> | undefined;
|
|
54
133
|
}, {
|
|
55
134
|
id: string;
|
|
56
135
|
input: string;
|
|
136
|
+
history?: {
|
|
137
|
+
role: "user" | "assistant";
|
|
138
|
+
content: string;
|
|
139
|
+
}[] | undefined;
|
|
57
140
|
expected_output?: string | undefined;
|
|
58
141
|
expected_tools?: string[] | undefined;
|
|
59
142
|
metadata?: Record<string, unknown> | undefined;
|
|
@@ -63,6 +146,10 @@ export declare const DatasetSchema: z.ZodObject<{
|
|
|
63
146
|
samples: {
|
|
64
147
|
id: string;
|
|
65
148
|
input: string;
|
|
149
|
+
history?: {
|
|
150
|
+
role: "user" | "assistant";
|
|
151
|
+
content: string;
|
|
152
|
+
}[] | undefined;
|
|
66
153
|
expected_output?: string | undefined;
|
|
67
154
|
expected_tools?: string[] | undefined;
|
|
68
155
|
metadata?: Record<string, unknown> | undefined;
|
|
@@ -72,6 +159,10 @@ export declare const DatasetSchema: z.ZodObject<{
|
|
|
72
159
|
samples: {
|
|
73
160
|
id: string;
|
|
74
161
|
input: string;
|
|
162
|
+
history?: {
|
|
163
|
+
role: "user" | "assistant";
|
|
164
|
+
content: string;
|
|
165
|
+
}[] | undefined;
|
|
75
166
|
expected_output?: string | undefined;
|
|
76
167
|
expected_tools?: string[] | undefined;
|
|
77
168
|
metadata?: Record<string, unknown> | undefined;
|
package/dist/index.js
CHANGED
|
@@ -2,12 +2,14 @@
|
|
|
2
2
|
* Catalog R-eval `eval-dataset` — load samples from JSONL, CSV, YAML, or HTTP.
|
|
3
3
|
*
|
|
4
4
|
* Sample shape is the contract every other eval-* module consumes:
|
|
5
|
-
* { id, input, expected_output?, expected_tools?, metadata? }
|
|
5
|
+
* { id, input, history?, expected_output?, expected_tools?, metadata? }
|
|
6
6
|
*
|
|
7
7
|
* Loaders return an `AsyncIterable<Sample>` so 100k-row datasets stream rather
|
|
8
|
-
* than load fully into memory. JSONL
|
|
9
|
-
* `Bun.file().stream()
|
|
10
|
-
*
|
|
8
|
+
* than load fully into memory. JSONL truly streams — local files via
|
|
9
|
+
* `Bun.file().stream()`, HTTP bodies via the fetch reader (B24) — through one
|
|
10
|
+
* shared incremental line parser, memory bounded by the longest line. CSV and
|
|
11
|
+
* YAML buffer their source whole, locally and over HTTP alike (YAML can't
|
|
12
|
+
* stream, CSV quoting spans physical lines).
|
|
11
13
|
*
|
|
12
14
|
* Reference: build-roadmap.md §16.
|
|
13
15
|
*/
|
|
@@ -17,9 +19,36 @@ import { loadCsv } from "./loaders/csv";
|
|
|
17
19
|
import { loadHttp } from "./loaders/http";
|
|
18
20
|
import { loadJsonl } from "./loaders/jsonl";
|
|
19
21
|
import { loadYaml } from "./loaders/yaml";
|
|
22
|
+
/**
|
|
23
|
+
* B14 — one prior conversation turn of a multi-turn sample. STRICT item
|
|
24
|
+
* shape: a history message is seeded into the eval session transcript
|
|
25
|
+
* VERBATIM (no model call runs for it), so an unknown key is an authoring
|
|
26
|
+
* mistake to reject loudly, not an extension point.
|
|
27
|
+
*/
|
|
28
|
+
export const HistoryMessageSchema = z
|
|
29
|
+
.object({
|
|
30
|
+
role: z.enum(["user", "assistant"]),
|
|
31
|
+
content: z.string(),
|
|
32
|
+
})
|
|
33
|
+
.strict();
|
|
20
34
|
export const SampleSchema = z.object({
|
|
21
35
|
id: z.string().min(1),
|
|
22
36
|
input: z.string(),
|
|
37
|
+
/**
|
|
38
|
+
* B14 — optional prior conversation turns (MT-Bench-style multi-turn
|
|
39
|
+
* evaluation). `input` stays the required FINAL user message — every
|
|
40
|
+
* history-less dataset parses byte-identically — and when `history` is
|
|
41
|
+
* present the eval invoker seeds these messages into the session
|
|
42
|
+
* transcript verbatim before running `input`. Non-empty when present:
|
|
43
|
+
* `history: []` says "multi-turn" while carrying no turns, an authoring
|
|
44
|
+
* error worth rejecting at load. One DELIBERATE break rides along: a
|
|
45
|
+
* hand-authored dataset whose samples already carried a free-form
|
|
46
|
+
* `history` key was silently ignored before (Zod strip mode dropped the
|
|
47
|
+
* unknown key) — it now validates, so a shape-mismatched value fails the
|
|
48
|
+
* load loudly, and a shape-matched one starts seeding turns (changing
|
|
49
|
+
* that dataset's sample hashes). Rename the key or fix its shape.
|
|
50
|
+
*/
|
|
51
|
+
history: z.array(HistoryMessageSchema).min(1).optional(),
|
|
23
52
|
expected_output: z.string().optional(),
|
|
24
53
|
expected_tools: z.array(z.string()).optional(),
|
|
25
54
|
metadata: z.record(z.unknown()).optional(),
|
package/dist/loaders/csv.d.ts
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
|
-
import { type LoadedDataset } from "../index";
|
|
1
|
+
import { type LoadedDataset, type Sample } from "../index";
|
|
2
2
|
export declare function loadCsv(path: string): Promise<LoadedDataset>;
|
|
3
|
+
/**
|
|
4
|
+
* Shared CSV row → Sample generator (the file loader and the HTTP CSV path
|
|
5
|
+
* both dispatch here, so structured-column handling can never drift between
|
|
6
|
+
* them). Empty cells are omitted — a blank column keeps the sample
|
|
7
|
+
* byte-identical to one authored without it. Structured columns:
|
|
8
|
+
* - `expected_tools` — comma-split list.
|
|
9
|
+
* - `metadata` (NEW-formats-1) / `history` (B14) — JSON-encoded cells,
|
|
10
|
+
* parsed here so CSV datasets can carry the fields the rest of the
|
|
11
|
+
* eval system leans on (slicing, provenance, multi-turn). Malformed
|
|
12
|
+
* JSON is a hard `DatasetLoadError` naming the row — the raw string
|
|
13
|
+
* would otherwise reach schema validation and fail with a far more
|
|
14
|
+
* confusing type error (previously EVERY non-empty metadata cell did).
|
|
15
|
+
*/
|
|
16
|
+
export declare function csvRowsToSamples(rows: ReadonlyArray<ReadonlyArray<string>>, header: ReadonlyArray<string>, source: string): AsyncIterable<Sample>;
|
|
3
17
|
/**
|
|
4
18
|
* RFC 4180-style CSV parser. Handles:
|
|
5
19
|
* - quoted fields with embedded commas, newlines (CRLF/LF), and escaped `""`
|
package/dist/loaders/csv.js
CHANGED
|
@@ -14,13 +14,26 @@ export async function loadCsv(path) {
|
|
|
14
14
|
}
|
|
15
15
|
return {
|
|
16
16
|
name,
|
|
17
|
-
samples:
|
|
17
|
+
samples: csvRowsToSamples(dataRows, header, path),
|
|
18
18
|
};
|
|
19
19
|
}
|
|
20
20
|
async function* emptyIterable() {
|
|
21
21
|
/* nothing */
|
|
22
22
|
}
|
|
23
|
-
|
|
23
|
+
/**
|
|
24
|
+
* Shared CSV row → Sample generator (the file loader and the HTTP CSV path
|
|
25
|
+
* both dispatch here, so structured-column handling can never drift between
|
|
26
|
+
* them). Empty cells are omitted — a blank column keeps the sample
|
|
27
|
+
* byte-identical to one authored without it. Structured columns:
|
|
28
|
+
* - `expected_tools` — comma-split list.
|
|
29
|
+
* - `metadata` (NEW-formats-1) / `history` (B14) — JSON-encoded cells,
|
|
30
|
+
* parsed here so CSV datasets can carry the fields the rest of the
|
|
31
|
+
* eval system leans on (slicing, provenance, multi-turn). Malformed
|
|
32
|
+
* JSON is a hard `DatasetLoadError` naming the row — the raw string
|
|
33
|
+
* would otherwise reach schema validation and fail with a far more
|
|
34
|
+
* confusing type error (previously EVERY non-empty metadata cell did).
|
|
35
|
+
*/
|
|
36
|
+
export async function* csvRowsToSamples(rows, header, source) {
|
|
24
37
|
let rowNo = 1; // header was row 0
|
|
25
38
|
for (const row of rows) {
|
|
26
39
|
rowNo += 1;
|
|
@@ -32,20 +45,30 @@ async function* rowsToSamples(rows, header, path) {
|
|
|
32
45
|
if (key === undefined)
|
|
33
46
|
continue;
|
|
34
47
|
const cell = row[i] ?? "";
|
|
35
|
-
|
|
36
|
-
|
|
48
|
+
if (cell === "")
|
|
49
|
+
continue;
|
|
50
|
+
if (key === "expected_tools") {
|
|
51
|
+
// expected_tools is the only comma-split field.
|
|
37
52
|
obj[key] = cell
|
|
38
53
|
.split(",")
|
|
39
54
|
.map((s) => s.trim())
|
|
40
55
|
.filter((s) => s.length > 0);
|
|
41
56
|
}
|
|
42
|
-
else if (
|
|
57
|
+
else if (key === "metadata" || key === "history") {
|
|
58
|
+
try {
|
|
59
|
+
obj[key] = JSON.parse(cell);
|
|
60
|
+
}
|
|
61
|
+
catch (err) {
|
|
62
|
+
throw new DatasetLoadError(`malformed JSON in "${key}" column on row ${rowNo} of ${source}`, err);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
else {
|
|
43
66
|
obj[key] = cell;
|
|
44
67
|
}
|
|
45
68
|
}
|
|
46
69
|
const result = SampleSchema.safeParse(obj);
|
|
47
70
|
if (!result.success) {
|
|
48
|
-
throw new DatasetLoadError(`invalid sample on row ${rowNo} of ${
|
|
71
|
+
throw new DatasetLoadError(`invalid sample on row ${rowNo} of ${source}: ${result.error.message}`);
|
|
49
72
|
}
|
|
50
73
|
yield result.data;
|
|
51
74
|
}
|
package/dist/loaders/http.d.ts
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import { type LoadedDataset } from "../index";
|
|
2
2
|
/**
|
|
3
|
-
* HTTP loader —
|
|
4
|
-
*
|
|
5
|
-
* the response Content-Type as a fallback.
|
|
3
|
+
* HTTP loader — dispatches on the URL extension first, then the response
|
|
4
|
+
* Content-Type as a fallback.
|
|
6
5
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
6
|
+
* B24 — `.jsonl`/`.ndjson` bodies STREAM line-by-line off the fetch body
|
|
7
|
+
* reader through the SAME incremental parser as the local file loader:
|
|
8
|
+
* memory stays bounded by the longest line, not the body, so a multi-GB
|
|
9
|
+
* JSONL URL parses as it downloads. CSV and YAML buffer the entire body via
|
|
10
|
+
* `response.text()` — YAML cannot be parsed incrementally and CSV quoting
|
|
11
|
+
* can span physical lines — matching the (also buffered) local CSV/YAML
|
|
12
|
+
* loaders.
|
|
9
13
|
*/
|
|
10
14
|
export declare function loadHttp(url: string): Promise<LoadedDataset>;
|
package/dist/loaders/http.js
CHANGED
|
@@ -1,37 +1,45 @@
|
|
|
1
1
|
import { parse as parseYaml } from "yaml";
|
|
2
2
|
import { DatasetLoadError } from "../errors";
|
|
3
3
|
import { DatasetSchema, SampleSchema } from "../index";
|
|
4
|
-
import { parseCsv } from "./csv";
|
|
4
|
+
import { csvRowsToSamples, parseCsv } from "./csv";
|
|
5
|
+
import { samplesFromJsonlStream, samplesFromJsonlText } from "./jsonl";
|
|
5
6
|
/**
|
|
6
|
-
* HTTP loader —
|
|
7
|
-
*
|
|
8
|
-
* the response Content-Type as a fallback.
|
|
7
|
+
* HTTP loader — dispatches on the URL extension first, then the response
|
|
8
|
+
* Content-Type as a fallback.
|
|
9
9
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
10
|
+
* B24 — `.jsonl`/`.ndjson` bodies STREAM line-by-line off the fetch body
|
|
11
|
+
* reader through the SAME incremental parser as the local file loader:
|
|
12
|
+
* memory stays bounded by the longest line, not the body, so a multi-GB
|
|
13
|
+
* JSONL URL parses as it downloads. CSV and YAML buffer the entire body via
|
|
14
|
+
* `response.text()` — YAML cannot be parsed incrementally and CSV quoting
|
|
15
|
+
* can span physical lines — matching the (also buffered) local CSV/YAML
|
|
16
|
+
* loaders.
|
|
12
17
|
*/
|
|
13
18
|
export async function loadHttp(url) {
|
|
14
19
|
const response = await fetch(url);
|
|
15
20
|
if (!response.ok) {
|
|
16
21
|
throw new DatasetLoadError(`HTTP ${response.status} fetching ${url}`);
|
|
17
22
|
}
|
|
18
|
-
const text = await response.text();
|
|
19
23
|
const ct = (response.headers.get("content-type") ?? "").toLowerCase();
|
|
20
24
|
const lower = url.toLowerCase();
|
|
21
25
|
if (lower.endsWith(".jsonl") ||
|
|
22
26
|
lower.endsWith(".ndjson") ||
|
|
23
27
|
ct.includes("application/x-jsonlines") ||
|
|
24
28
|
ct.includes("application/x-ndjson")) {
|
|
25
|
-
return { name: deriveName(url), samples:
|
|
29
|
+
return { name: deriveName(url), samples: streamJsonlBody(response, url) };
|
|
26
30
|
}
|
|
27
31
|
if (lower.endsWith(".csv") || ct.includes("text/csv")) {
|
|
28
|
-
|
|
32
|
+
// Buffered: CSV quoting can span physical lines. Rows then flow through
|
|
33
|
+
// the SAME row → Sample generator as the file loader (metadata/history
|
|
34
|
+
// JSON columns included).
|
|
35
|
+
const [header, ...dataRows] = parseCsv(await response.text());
|
|
36
|
+
return { name: deriveName(url), samples: csvRowsToSamples(dataRows, header ?? [], url) };
|
|
29
37
|
}
|
|
30
38
|
if (lower.endsWith(".yaml") ||
|
|
31
39
|
lower.endsWith(".yml") ||
|
|
32
40
|
ct.includes("application/yaml") ||
|
|
33
41
|
ct.includes("text/yaml")) {
|
|
34
|
-
return parseYamlText(text, url);
|
|
42
|
+
return parseYamlText(await response.text(), url);
|
|
35
43
|
}
|
|
36
44
|
throw new DatasetLoadError(`unrecognized HTTP dataset format for ${url} (content-type: ${ct || "unknown"})`);
|
|
37
45
|
}
|
|
@@ -45,57 +53,19 @@ function deriveName(url) {
|
|
|
45
53
|
return "remote-dataset";
|
|
46
54
|
}
|
|
47
55
|
}
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
catch (err) {
|
|
59
|
-
throw new DatasetLoadError(`malformed JSON on line ${lineNo} of ${source}`, err);
|
|
60
|
-
}
|
|
61
|
-
const result = SampleSchema.safeParse(parsed);
|
|
62
|
-
if (!result.success) {
|
|
63
|
-
throw new DatasetLoadError(`invalid sample on line ${lineNo} of ${source}: ${result.error.message}`);
|
|
64
|
-
}
|
|
65
|
-
yield result.data;
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
async function* parseCsvText(text, source) {
|
|
69
|
-
const [header, ...dataRows] = parseCsv(text);
|
|
70
|
-
if (!header)
|
|
56
|
+
/**
|
|
57
|
+
* B24 — HTTP `.jsonl` delegates to the shared incremental JSONL parser (see
|
|
58
|
+
* `loaders/jsonl.ts`) over the fetch body reader. A null body (null-body
|
|
59
|
+
* statuses, minimal Response stubs) falls back to buffering — there is
|
|
60
|
+
* nothing to stream.
|
|
61
|
+
*/
|
|
62
|
+
async function* streamJsonlBody(response, source) {
|
|
63
|
+
const body = response.body;
|
|
64
|
+
if (body === null || body === undefined) {
|
|
65
|
+
yield* samplesFromJsonlText(await response.text(), source);
|
|
71
66
|
return;
|
|
72
|
-
let rowNo = 1;
|
|
73
|
-
for (const row of dataRows) {
|
|
74
|
-
rowNo += 1;
|
|
75
|
-
if (row.length === 1 && row[0] === "")
|
|
76
|
-
continue;
|
|
77
|
-
const obj = {};
|
|
78
|
-
for (let i = 0; i < header.length; i++) {
|
|
79
|
-
const key = header[i];
|
|
80
|
-
if (key === undefined)
|
|
81
|
-
continue;
|
|
82
|
-
const cell = row[i] ?? "";
|
|
83
|
-
if (key === "expected_tools" && cell !== "") {
|
|
84
|
-
obj[key] = cell
|
|
85
|
-
.split(",")
|
|
86
|
-
.map((s) => s.trim())
|
|
87
|
-
.filter((s) => s.length > 0);
|
|
88
|
-
}
|
|
89
|
-
else if (cell !== "") {
|
|
90
|
-
obj[key] = cell;
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
const result = SampleSchema.safeParse(obj);
|
|
94
|
-
if (!result.success) {
|
|
95
|
-
throw new DatasetLoadError(`invalid sample on row ${rowNo} of ${source}: ${result.error.message}`);
|
|
96
|
-
}
|
|
97
|
-
yield result.data;
|
|
98
67
|
}
|
|
68
|
+
yield* samplesFromJsonlStream(body, source);
|
|
99
69
|
}
|
|
100
70
|
function parseYamlText(text, source) {
|
|
101
71
|
let parsed;
|
package/dist/loaders/jsonl.d.ts
CHANGED
|
@@ -1,2 +1,15 @@
|
|
|
1
|
-
import { type LoadedDataset } from "../index";
|
|
1
|
+
import { type LoadedDataset, type Sample } from "../index";
|
|
2
2
|
export declare function loadJsonl(path: string): Promise<LoadedDataset>;
|
|
3
|
+
/**
|
|
4
|
+
* Buffered-text JSONL → Sample generator (the HTTP loader's null-body
|
|
5
|
+
* fallback). Same parser, same line numbering as the streaming path.
|
|
6
|
+
*/
|
|
7
|
+
export declare function samplesFromJsonlText(text: string, source: string): AsyncIterable<Sample>;
|
|
8
|
+
/**
|
|
9
|
+
* B24 — incremental JSONL over any byte stream (local file or fetch body):
|
|
10
|
+
* pull chunks off the reader, split on newlines (tolerating CRLF and JSON
|
|
11
|
+
* lines split across chunk boundaries), and validate each line as it
|
|
12
|
+
* arrives. Line numbering matches the buffered parser's exactly (blank lines
|
|
13
|
+
* count, then skip).
|
|
14
|
+
*/
|
|
15
|
+
export declare function samplesFromJsonlStream(stream: ReadableStream<Uint8Array>, source: string): AsyncIterable<Sample>;
|
package/dist/loaders/jsonl.js
CHANGED
|
@@ -8,28 +8,99 @@ export async function loadJsonl(path) {
|
|
|
8
8
|
}
|
|
9
9
|
return {
|
|
10
10
|
name: basename(path).replace(/\.(jsonl|ndjson)$/i, ""),
|
|
11
|
-
samples:
|
|
11
|
+
samples: streamJsonlFile(path),
|
|
12
12
|
};
|
|
13
13
|
}
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
/**
|
|
15
|
+
* B24 — local `.jsonl` files stream through the SAME incremental line parser
|
|
16
|
+
* as HTTP bodies: `Bun.file().stream()` is a `ReadableStream<Uint8Array>`
|
|
17
|
+
* exactly like a fetch body, so memory stays bounded by the longest line, not
|
|
18
|
+
* the file. The generator wrapper keeps the file unopened until the caller
|
|
19
|
+
* actually iterates (loaders hand back lazy iterables, not open handles).
|
|
20
|
+
*/
|
|
21
|
+
async function* streamJsonlFile(path) {
|
|
22
|
+
yield* samplesFromJsonlStream(Bun.file(path).stream(), path);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* One line-numbering + parse + validate closure shared by every JSONL entry
|
|
26
|
+
* point (local file, HTTP body, buffered fallback), so error messages
|
|
27
|
+
* ("malformed JSON on line N of SOURCE") can never drift between loaders.
|
|
28
|
+
* Blank lines count toward numbering, then skip; a trailing `\r` (CRLF input
|
|
29
|
+
* split on `\n`) is tolerated.
|
|
30
|
+
*/
|
|
31
|
+
function makeLineParser(source) {
|
|
17
32
|
let lineNo = 0;
|
|
18
|
-
|
|
33
|
+
return (line) => {
|
|
19
34
|
lineNo += 1;
|
|
35
|
+
const raw = line.endsWith("\r") ? line.slice(0, -1) : line;
|
|
20
36
|
if (raw.trim() === "")
|
|
21
|
-
|
|
37
|
+
return undefined;
|
|
22
38
|
let parsed;
|
|
23
39
|
try {
|
|
24
40
|
parsed = JSON.parse(raw);
|
|
25
41
|
}
|
|
26
42
|
catch (err) {
|
|
27
|
-
throw new DatasetLoadError(`malformed JSON on line ${lineNo} of ${
|
|
43
|
+
throw new DatasetLoadError(`malformed JSON on line ${lineNo} of ${source}`, err);
|
|
28
44
|
}
|
|
29
45
|
const result = SampleSchema.safeParse(parsed);
|
|
30
46
|
if (!result.success) {
|
|
31
|
-
throw new DatasetLoadError(`invalid sample on line ${lineNo} of ${
|
|
47
|
+
throw new DatasetLoadError(`invalid sample on line ${lineNo} of ${source}: ${result.error.message}`);
|
|
48
|
+
}
|
|
49
|
+
return result.data;
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Buffered-text JSONL → Sample generator (the HTTP loader's null-body
|
|
54
|
+
* fallback). Same parser, same line numbering as the streaming path.
|
|
55
|
+
*/
|
|
56
|
+
export async function* samplesFromJsonlText(text, source) {
|
|
57
|
+
const parseLine = makeLineParser(source);
|
|
58
|
+
for (const line of text.split("\n")) {
|
|
59
|
+
const sample = parseLine(line);
|
|
60
|
+
if (sample !== undefined)
|
|
61
|
+
yield sample;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* B24 — incremental JSONL over any byte stream (local file or fetch body):
|
|
66
|
+
* pull chunks off the reader, split on newlines (tolerating CRLF and JSON
|
|
67
|
+
* lines split across chunk boundaries), and validate each line as it
|
|
68
|
+
* arrives. Line numbering matches the buffered parser's exactly (blank lines
|
|
69
|
+
* count, then skip).
|
|
70
|
+
*/
|
|
71
|
+
export async function* samplesFromJsonlStream(stream, source) {
|
|
72
|
+
const parseLine = makeLineParser(source);
|
|
73
|
+
const reader = stream.getReader();
|
|
74
|
+
const decoder = new TextDecoder();
|
|
75
|
+
let carry = "";
|
|
76
|
+
try {
|
|
77
|
+
while (true) {
|
|
78
|
+
const { done, value } = await reader.read();
|
|
79
|
+
if (done)
|
|
80
|
+
break;
|
|
81
|
+
carry += decoder.decode(value, { stream: true });
|
|
82
|
+
let nl = carry.indexOf("\n");
|
|
83
|
+
while (nl !== -1) {
|
|
84
|
+
const line = carry.slice(0, nl);
|
|
85
|
+
carry = carry.slice(nl + 1);
|
|
86
|
+
const sample = parseLine(line);
|
|
87
|
+
if (sample !== undefined)
|
|
88
|
+
yield sample;
|
|
89
|
+
nl = carry.indexOf("\n");
|
|
90
|
+
}
|
|
32
91
|
}
|
|
33
|
-
|
|
92
|
+
// Flush any multi-byte tail the decoder held back, then the final
|
|
93
|
+
// (newline-less) line — or the trailing "" a newline-terminated source
|
|
94
|
+
// leaves, which parseLine counts and skips like the buffered parser.
|
|
95
|
+
carry += decoder.decode();
|
|
96
|
+
const last = parseLine(carry);
|
|
97
|
+
if (last !== undefined)
|
|
98
|
+
yield last;
|
|
99
|
+
}
|
|
100
|
+
finally {
|
|
101
|
+
// Early exit (consumer break, invalid line mid-stream) must not leave a
|
|
102
|
+
// connection or file handle dangling; on normal completion this is a
|
|
103
|
+
// no-op.
|
|
104
|
+
await reader.cancel().catch(() => { });
|
|
34
105
|
}
|
|
35
106
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crewhaus/eval-dataset",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.2",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Eval dataset loaders (JSONL, CSV, YAML, HTTP) with a lazy iterator API",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
"test": "bun test src"
|
|
16
16
|
},
|
|
17
17
|
"dependencies": {
|
|
18
|
-
"@crewhaus/errors": "0.
|
|
18
|
+
"@crewhaus/errors": "0.4.2",
|
|
19
19
|
"yaml": "^2.6.0",
|
|
20
20
|
"zod": "^3.23.8"
|
|
21
21
|
},
|