@deepagents/evals 0.19.0

package/README.md

@@ -0,0 +1,218 @@
+# @deepagents/evals
+
+A general-purpose LLM evaluation framework with dataset loading, scoring, run persistence, model comparison, and console reporting.
+
+## Installation
+
+```bash
+npm install @deepagents/evals
+```
+
+## Quick Start
+
+```typescript
+import { dataset, evaluate, exactMatch } from '@deepagents/evals';
+
+const summary = await evaluate({
+  name: 'my-eval',
+  model: 'gpt-4o',
+  dataset: dataset([
+    { input: 'What is 2+2?', expected: '4' },
+    { input: 'What is 3+3?', expected: '6' },
+  ]),
+  task: async (item) => {
+    const response = await callMyLLM(item.input);
+    return { output: response };
+  },
+  scorers: { exact: exactMatch },
+});
+```
+
+## Modules
+
+The package is organized into subpath exports for granular imports:
+
+| Import                         | Description                             |
+| ------------------------------ | --------------------------------------- |
+| `@deepagents/evals`            | Top-level convenience API (`evaluate`)  |
+| `@deepagents/evals/dataset`    | Dataset loading and transforms          |
+| `@deepagents/evals/scorers`    | Scorer functions and combinators        |
+| `@deepagents/evals/store`      | SQLite run persistence                  |
+| `@deepagents/evals/engine`     | Eval engine with concurrency and events |
+| `@deepagents/evals/comparison` | Run diffing and regression detection    |
+| `@deepagents/evals/reporters`  | Console reporter                        |
+
+## Dataset
+
+Load data from inline arrays or local files (JSON, JSONL, CSV):
+
+```typescript
+import { dataset } from '@deepagents/evals/dataset';
+
+// Inline array
+const ds = dataset([{ input: 'hello', expected: 'world' }]);
+
+// From file
+const ds = dataset('./data/questions.json');
+const ds = dataset('./data/questions.jsonl');
+const ds = dataset('./data/questions.csv');
+```
+
+### Transforms
+
+Chainable, lazy transforms on datasets:
+
+```typescript
+dataset('./large-dataset.jsonl')
+  .filter((row) => row.difficulty === 'hard')
+  .map((row) => ({ input: row.question, expected: row.answer }))
+  .shuffle()
+  .limit(100);
+```
+
+| Transform    | Behavior                                     |
+| ------------ | -------------------------------------------- |
+| `map(fn)`    | Lazy — transforms each element               |
+| `filter(fn)` | Lazy — excludes non-matching elements        |
+| `limit(n)`   | Lazy — caps output at n elements             |
+| `shuffle()`  | Eager — buffers all, randomizes order        |
+| `sample(n)`  | Eager — buffers all, picks n random elements |
+| `toArray()`  | Consumes into a plain array                  |
+
+## Scorers
+
+All scorers return `{ score: number (0..1), reason?: string }`.
+
+### Deterministic Scorers
+
+```typescript
+import {
+  exactMatch,
+  includes,
+  jsonMatch,
+  levenshtein,
+  regex,
+} from '@deepagents/evals/scorers';
+```
+
+| Scorer           | Description                         |
+| ---------------- | ----------------------------------- |
+| `exactMatch`     | Strict string equality              |
+| `includes`       | Substring check                     |
+| `regex(pattern)` | RegExp test                         |
+| `levenshtein`    | Normalized edit distance similarity |
+| `jsonMatch`      | Deep JSON structural equality       |
+
+### LLM-Based Scorers
+
+```typescript
+import { factuality, llmJudge } from '@deepagents/evals/scorers';
+
+const judge = llmJudge({ model: myModel, criteria: 'Is the answer helpful?' });
+const fact = factuality({ model: myModel });
+```
+
+### Combinators
+
+```typescript
+import { all, any, weighted } from '@deepagents/evals/scorers';
+
+// Weakest-link (minimum score)
+const strict = all(exactMatch, includes);
+
+// Best-of (maximum score)
+const lenient = any(exactMatch, includes);
+
+// Weighted average
+const balanced = weighted({
+  accuracy: { scorer: exactMatch, weight: 2 },
+  style: { scorer: llmJudge({ model, criteria: '...' }), weight: 1 },
+});
+```
+
+## Run Store
+
+SQLite-backed persistence for runs, cases, and scores:
+
+```typescript
+import { RunStore } from '@deepagents/evals/store';
+
+const store = new RunStore('.evals/store.db');
+
+// Create a suite for grouping runs
+const suite = store.createSuite('text2sql-accuracy');
+
+// Query results
+const runs = store.listRuns(suite.id);
+const failing = store.getFailingCases(runId, 0.5);
+const summary = store.getRunSummary(runId);
+```
+
+## Engine
+
+The engine orchestrates dataset iteration, task execution, scoring, and persistence:
+
+```typescript
+import { EvalEmitter, runEval } from '@deepagents/evals/engine';
+
+const emitter = new EvalEmitter();
+emitter.on('case:scored', (data) => console.log(data.index, data.scores));
+
+const summary = await runEval({
+  name: 'my-eval',
+  model: 'gpt-4o',
+  dataset: ds,
+  task: myTask,
+  scorers: { exact: exactMatch },
+  store,
+  emitter,
+  maxConcurrency: 10,
+  timeout: 30_000,
+  trials: 1,
+  threshold: 0.5,
+});
+```
+
+### Events
+
+| Event         | Payload                               | When                                      |
+| ------------- | ------------------------------------- | ----------------------------------------- |
+| `run:start`   | `{ runId, totalCases, name, model }`  | Run begins                                |
+| `case:start`  | `{ runId, index, input }`             | Case execution starts                     |
+| `case:scored` | `{ runId, index, scores, latencyMs }` | Case scored (always fires, even on error) |
+| `case:error`  | `{ runId, index, error }`             | Task threw an error                       |
+| `run:end`     | `{ runId, summary }`                  | All cases complete                        |
+
+## Comparison
+
+Compare two runs case-by-case to detect improvements and regressions:
+
+```typescript
+import { compareRuns } from '@deepagents/evals/comparison';
+
+const result = compareRuns(store, baselineRunId, candidateRunId, {
+  tolerance: 0.01,
+  regressionThreshold: 0.05,
+});
+
+console.log(result.regression.regressed); // true if any scorer regressed
+console.log(result.scorerSummaries); // per-scorer mean deltas and counts
+console.log(result.costDelta); // latency and token differences
+```
+
+## Console Reporter
+
+Subscribe to engine events for terminal output:
+
+```typescript
+import { consoleReporter } from '@deepagents/evals/reporters';
+
+consoleReporter(emitter, {
+  verbosity: 'normal', // 'quiet' | 'normal' | 'verbose'
+  threshold: 0.5,
+});
+```
+
+## License
+
+MIT

package/dist/comparison/index.d.ts

@@ -0,0 +1,41 @@
+import type { RunStore } from '../store/index.ts';
+export type ChangeType = 'improved' | 'regressed' | 'unchanged';
+export interface CaseDiff {
+    index: number;
+    scorerDeltas: Record<string, {
+        baseline: number;
+        candidate: number;
+        delta: number;
+        change: ChangeType;
+    }>;
+}
+export interface ScorerSummary {
+    meanDelta: number;
+    improvedCount: number;
+    regressedCount: number;
+    unchangedCount: number;
+}
+export interface CostDelta {
+    latencyDeltaMs: number;
+    tokenInDelta: number;
+    tokenOutDelta: number;
+}
+export interface ComparisonResult {
+    caseDiffs: CaseDiff[];
+    scorerSummaries: Record<string, ScorerSummary>;
+    costDelta: CostDelta;
+    totalCasesCompared: number;
+    regression: {
+        regressed: boolean;
+        details: Record<string, {
+            meanDelta: number;
+            exceeds: boolean;
+        }>;
+    };
+}
+export interface CompareOptions {
+    tolerance?: number;
+    regressionThreshold?: number;
+}
+export declare function compareRuns(store: RunStore, baselineRunId: string, candidateRunId: string, options?: CompareOptions): ComparisonResult;
+//# sourceMappingURL=index.d.ts.map

package/dist/comparison/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/comparison/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAkB,QAAQ,EAAc,MAAM,mBAAmB,CAAC;AAE9E,MAAM,MAAM,UAAU,GAAG,UAAU,GAAG,WAAW,GAAG,WAAW,CAAC;AAEhE,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAClB,MAAM,EACN;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,CAAA;KAAE,CAC3E,CAAC;CACH;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,SAAS;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC/C,SAAS,EAAE,SAAS,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE;QACV,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,OAAO,CAAA;SAAE,CAAC,CAAC;KAClE,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AA+BD,wBAAgB,WAAW,CACzB,KAAK,EAAE,QAAQ,EACf,aAAa,EAAE,MAAM,EACrB,cAAc,EAAE,MAAM,EACtB,OAAO,CAAC,EAAE,cAAc,GACvB,gBAAgB,CA0GlB"}

package/dist/comparison/index.js

@@ -0,0 +1,106 @@
+// packages/evals/src/comparison/index.ts
+function categorize(delta, tolerance) {
+  if (Math.abs(delta) <= tolerance) return "unchanged";
+  return delta > 0 ? "improved" : "regressed";
+}
+function buildScoreMap(cases) {
+  const map = /* @__PURE__ */ new Map();
+  for (const c of cases) {
+    const scores = {};
+    for (const s of c.scores) {
+      scores[s.scorer_name] = s.score;
+    }
+    map.set(c.idx, scores);
+  }
+  return map;
+}
+function getAllCasesWithScores(store, runId) {
+  const cases = store.getCases(runId);
+  const withScores = store.getFailingCases(runId, Infinity);
+  const scoredMap = new Map(withScores.map((c) => [c.id, c]));
+  return cases.map((c) => scoredMap.get(c.id) ?? { ...c, scores: [] });
+}
+function compareRuns(store, baselineRunId, candidateRunId, options) {
+  const tolerance = options?.tolerance ?? 0.01;
+  const regressionThreshold = options?.regressionThreshold ?? 0.05;
+  const baselineCases = getAllCasesWithScores(store, baselineRunId);
+  const candidateCases = getAllCasesWithScores(store, candidateRunId);
+  if (baselineCases.length !== candidateCases.length) {
+    console.warn(
+      `Run case count mismatch: baseline=${baselineCases.length}, candidate=${candidateCases.length}. Comparing intersection only.`
+    );
+  }
+  const baselineMap = buildScoreMap(baselineCases);
+  const candidateMap = buildScoreMap(candidateCases);
+  const allScorerNames = /* @__PURE__ */ new Set();
+  for (const scores of baselineMap.values()) {
+    for (const name of Object.keys(scores)) allScorerNames.add(name);
+  }
+  for (const scores of candidateMap.values()) {
+    for (const name of Object.keys(scores)) allScorerNames.add(name);
+  }
+  const commonIndices = [...baselineMap.keys()].filter(
+    (idx) => candidateMap.has(idx)
+  );
+  commonIndices.sort((a, b) => a - b);
+  const caseDiffs = [];
+  const scorerDeltas = {};
+  const scorerCounts = {};
+  for (const name of allScorerNames) {
+    scorerDeltas[name] = [];
+    scorerCounts[name] = { improved: 0, regressed: 0, unchanged: 0 };
+  }
+  for (const idx of commonIndices) {
+    const baseScores = baselineMap.get(idx);
+    const candScores = candidateMap.get(idx);
+    const diff = { index: idx, scorerDeltas: {} };
+    for (const name of allScorerNames) {
+      const baseline = baseScores[name] ?? 0;
+      const candidate = candScores[name] ?? 0;
+      const delta = candidate - baseline;
+      const change = categorize(delta, tolerance);
+      diff.scorerDeltas[name] = { baseline, candidate, delta, change };
+      scorerDeltas[name].push(delta);
+      if (change === "improved") scorerCounts[name].improved++;
+      else if (change === "regressed") scorerCounts[name].regressed++;
+      else scorerCounts[name].unchanged++;
+    }
+    caseDiffs.push(diff);
+  }
+  const scorerSummaries = {};
+  for (const name of allScorerNames) {
+    const deltas = scorerDeltas[name];
+    const meanDelta = deltas.length > 0 ? deltas.reduce((a, b) => a + b, 0) / deltas.length : 0;
+    scorerSummaries[name] = {
+      meanDelta,
+      improvedCount: scorerCounts[name].improved,
+      regressedCount: scorerCounts[name].regressed,
+      unchangedCount: scorerCounts[name].unchanged
+    };
+  }
+  const baselineSummary = store.getRunSummary(baselineRunId);
+  const candidateSummary = store.getRunSummary(candidateRunId);
+  const costDelta = {
+    latencyDeltaMs: candidateSummary.totalLatencyMs - baselineSummary.totalLatencyMs,
+    tokenInDelta: candidateSummary.totalTokensIn - baselineSummary.totalTokensIn,
+    tokenOutDelta: candidateSummary.totalTokensOut - baselineSummary.totalTokensOut
+  };
+  const regressionDetails = {};
+  let anyRegressed = false;
+  for (const [name, summary] of Object.entries(scorerSummaries)) {
+    const exceeds = summary.meanDelta < -regressionThreshold;
+    regressionDetails[name] = { meanDelta: summary.meanDelta, exceeds };
+    if (exceeds) anyRegressed = true;
+  }
+  return {
+    caseDiffs,
+    scorerSummaries,
+    costDelta,
+    totalCasesCompared: commonIndices.length,
+    regression: { regressed: anyRegressed, details: regressionDetails }
+  };
+}
+export {
+  compareRuns
+};
+//# sourceMappingURL=index.js.map

package/dist/comparison/index.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/comparison/index.ts"],
+  "sourcesContent": ["import type { CaseWithScores, RunStore, RunSummary } from '../store/index.ts';\n\nexport type ChangeType = 'improved' | 'regressed' | 'unchanged';\n\nexport interface CaseDiff {\n  index: number;\n  scorerDeltas: Record<\n    string,\n    { baseline: number; candidate: number; delta: number; change: ChangeType }\n  >;\n}\n\nexport interface ScorerSummary {\n  meanDelta: number;\n  improvedCount: number;\n  regressedCount: number;\n  unchangedCount: number;\n}\n\nexport interface CostDelta {\n  latencyDeltaMs: number;\n  tokenInDelta: number;\n  tokenOutDelta: number;\n}\n\nexport interface ComparisonResult {\n  caseDiffs: CaseDiff[];\n  scorerSummaries: Record<string, ScorerSummary>;\n  costDelta: CostDelta;\n  totalCasesCompared: number;\n  regression: {\n    regressed: boolean;\n    details: Record<string, { meanDelta: number; exceeds: boolean }>;\n  };\n}\n\nexport interface CompareOptions {\n  tolerance?: number;\n  regressionThreshold?: number;\n}\n\nfunction categorize(delta: number, tolerance: number): ChangeType {\n  if (Math.abs(delta) <= tolerance) return 'unchanged';\n  return delta > 0 ? 'improved' : 'regressed';\n}\n\nfunction buildScoreMap(\n  cases: CaseWithScores[],\n): Map<number, Record<string, number>> {\n  const map = new Map<number, Record<string, number>>();\n  for (const c of cases) {\n    const scores: Record<string, number> = {};\n    for (const s of c.scores) {\n      scores[s.scorer_name] = s.score;\n    }\n    map.set(c.idx, scores);\n  }\n  return map;\n}\n\nfunction getAllCasesWithScores(\n  store: RunStore,\n  runId: string,\n): CaseWithScores[] {\n  const cases = store.getCases(runId);\n  const withScores = store.getFailingCases(runId, Infinity);\n  const scoredMap = new Map(withScores.map((c) => [c.id, c]));\n  return cases.map((c) => scoredMap.get(c.id) ?? { ...c, scores: [] });\n}\n\nexport function compareRuns(\n  store: RunStore,\n  baselineRunId: string,\n  candidateRunId: string,\n  options?: CompareOptions,\n): ComparisonResult {\n  const tolerance = options?.tolerance ?? 0.01;\n  const regressionThreshold = options?.regressionThreshold ?? 0.05;\n\n  const baselineCases = getAllCasesWithScores(store, baselineRunId);\n  const candidateCases = getAllCasesWithScores(store, candidateRunId);\n\n  if (baselineCases.length !== candidateCases.length) {\n    console.warn(\n      `Run case count mismatch: baseline=${baselineCases.length}, candidate=${candidateCases.length}. Comparing intersection only.`,\n    );\n  }\n\n  const baselineMap = buildScoreMap(baselineCases);\n  const candidateMap = buildScoreMap(candidateCases);\n\n  const allScorerNames = new Set<string>();\n  for (const scores of baselineMap.values()) {\n    for (const name of Object.keys(scores)) allScorerNames.add(name);\n  }\n  for (const scores of candidateMap.values()) {\n    for (const name of Object.keys(scores)) allScorerNames.add(name);\n  }\n\n  const commonIndices = [...baselineMap.keys()].filter((idx) =>\n    candidateMap.has(idx),\n  );\n  commonIndices.sort((a, b) => a - b);\n\n  const caseDiffs: CaseDiff[] = [];\n  const scorerDeltas: Record<string, number[]> = {};\n  const scorerCounts: Record<\n    string,\n    { improved: number; regressed: number; unchanged: number }\n  > = {};\n\n  for (const name of allScorerNames) {\n    scorerDeltas[name] = [];\n    scorerCounts[name] = { improved: 0, regressed: 0, unchanged: 0 };\n  }\n\n  for (const idx of commonIndices) {\n    const baseScores = baselineMap.get(idx)!;\n    const candScores = candidateMap.get(idx)!;\n    const diff: CaseDiff = { index: idx, scorerDeltas: {} };\n\n    for (const name of allScorerNames) {\n      const baseline = baseScores[name] ?? 0;\n      const candidate = candScores[name] ?? 0;\n      const delta = candidate - baseline;\n      const change = categorize(delta, tolerance);\n\n      diff.scorerDeltas[name] = { baseline, candidate, delta, change };\n      scorerDeltas[name]!.push(delta);\n\n      if (change === 'improved') scorerCounts[name]!.improved++;\n      else if (change === 'regressed') scorerCounts[name]!.regressed++;\n      else scorerCounts[name]!.unchanged++;\n    }\n\n    caseDiffs.push(diff);\n  }\n\n  const scorerSummaries: Record<string, ScorerSummary> = {};\n  for (const name of allScorerNames) {\n    const deltas = scorerDeltas[name]!;\n    const meanDelta =\n      deltas.length > 0 ? deltas.reduce((a, b) => a + b, 0) / deltas.length : 0;\n    scorerSummaries[name] = {\n      meanDelta,\n      improvedCount: scorerCounts[name]!.improved,\n      regressedCount: scorerCounts[name]!.regressed,\n      unchangedCount: scorerCounts[name]!.unchanged,\n    };\n  }\n\n  const baselineSummary = store.getRunSummary(baselineRunId);\n  const candidateSummary = store.getRunSummary(candidateRunId);\n\n  const costDelta: CostDelta = {\n    latencyDeltaMs:\n      candidateSummary.totalLatencyMs - baselineSummary.totalLatencyMs,\n    tokenInDelta:\n      candidateSummary.totalTokensIn - baselineSummary.totalTokensIn,\n    tokenOutDelta:\n      candidateSummary.totalTokensOut - baselineSummary.totalTokensOut,\n  };\n\n  const regressionDetails: Record<\n    string,\n    { meanDelta: number; exceeds: boolean }\n  > = {};\n  let anyRegressed = false;\n  for (const [name, summary] of Object.entries(scorerSummaries)) {\n    const exceeds = summary.meanDelta < -regressionThreshold;\n    regressionDetails[name] = { meanDelta: summary.meanDelta, exceeds };\n    if (exceeds) anyRegressed = true;\n  }\n\n  return {\n    caseDiffs,\n    scorerSummaries,\n    costDelta,\n    totalCasesCompared: commonIndices.length,\n    regression: { regressed: anyRegressed, details: regressionDetails },\n  };\n}\n"],
+  "mappings": ";AAyCA,SAAS,WAAW,OAAe,WAA+B;AAChE,MAAI,KAAK,IAAI,KAAK,KAAK,UAAW,QAAO;AACzC,SAAO,QAAQ,IAAI,aAAa;AAClC;AAEA,SAAS,cACP,OACqC;AACrC,QAAM,MAAM,oBAAI,IAAoC;AACpD,aAAW,KAAK,OAAO;AACrB,UAAM,SAAiC,CAAC;AACxC,eAAW,KAAK,EAAE,QAAQ;AACxB,aAAO,EAAE,WAAW,IAAI,EAAE;AAAA,IAC5B;AACA,QAAI,IAAI,EAAE,KAAK,MAAM;AAAA,EACvB;AACA,SAAO;AACT;AAEA,SAAS,sBACP,OACA,OACkB;AAClB,QAAM,QAAQ,MAAM,SAAS,KAAK;AAClC,QAAM,aAAa,MAAM,gBAAgB,OAAO,QAAQ;AACxD,QAAM,YAAY,IAAI,IAAI,WAAW,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AAC1D,SAAO,MAAM,IAAI,CAAC,MAAM,UAAU,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,GAAG,QAAQ,CAAC,EAAE,CAAC;AACrE;AAEO,SAAS,YACd,OACA,eACA,gBACA,SACkB;AAClB,QAAM,YAAY,SAAS,aAAa;AACxC,QAAM,sBAAsB,SAAS,uBAAuB;AAE5D,QAAM,gBAAgB,sBAAsB,OAAO,aAAa;AAChE,QAAM,iBAAiB,sBAAsB,OAAO,cAAc;AAElE,MAAI,cAAc,WAAW,eAAe,QAAQ;AAClD,YAAQ;AAAA,MACN,qCAAqC,cAAc,MAAM,eAAe,eAAe,MAAM;AAAA,IAC/F;AAAA,EACF;AAEA,QAAM,cAAc,cAAc,aAAa;AAC/C,QAAM,eAAe,cAAc,cAAc;AAEjD,QAAM,iBAAiB,oBAAI,IAAY;AACvC,aAAW,UAAU,YAAY,OAAO,GAAG;AACzC,eAAW,QAAQ,OAAO,KAAK,MAAM,EAAG,gBAAe,IAAI,IAAI;AAAA,EACjE;AACA,aAAW,UAAU,aAAa,OAAO,GAAG;AAC1C,eAAW,QAAQ,OAAO,KAAK,MAAM,EAAG,gBAAe,IAAI,IAAI;AAAA,EACjE;AAEA,QAAM,gBAAgB,CAAC,GAAG,YAAY,KAAK,CAAC,EAAE;AAAA,IAAO,CAAC,QACpD,aAAa,IAAI,GAAG;AAAA,EACtB;AACA,gBAAc,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAElC,QAAM,YAAwB,CAAC;AAC/B,QAAM,eAAyC,CAAC;AAChD,QAAM,eAGF,CAAC;AAEL,aAAW,QAAQ,gBAAgB;AACjC,iBAAa,IAAI,IAAI,CAAC;AACtB,iBAAa,IAAI,IAAI,EAAE,UAAU,GAAG,WAAW,GAAG,WAAW,EAAE;AAAA,EACjE;AAEA,aAAW,OAAO,eAAe;AAC/B,UAAM,aAAa,YAAY,IAAI,GAAG;AACtC,UAAM,aAAa,aAAa,IAAI,GAAG;AACvC,UAAM,OAAiB,EAAE,OAAO,KAAK,cAAc,CAAC,EAAE;AAEtD,eAAW,QAAQ,gBAAgB;AACjC,YAAM,WAAW,WAAW,IAAI,KAAK;AACrC,YAAM,YAAY,WAAW,IAAI,KAAK;AACtC,YAAM,QAAQ,YAAY;AAC1B,YAAM,SAAS,WAAW,OAAO,SAAS;AAE1C,WAAK,aAAa,IAAI,IAAI,EAAE,UAAU,WAAW,OAAO,OAAO;AAC/D,mBAAa,IAAI,EAAG,KAAK,KAAK;AAE9B,UAAI,WAAW,WAAY,cAAa,IAAI,EAAG;AAAA,eACtC,WAAW,YAAa,cAAa,IAAI,EAAG;AAAA,UAChD,cAAa,IAAI,EAAG;AAAA,IAC3B;AAEA,cAAU,KAAK,IAAI;AAAA,EACrB;AAEA,QAAM,kBAAiD,CAAC;AACxD,aAAW,QAAQ,gBAAgB;AACjC,UAAM,SAAS,aAAa,IAAI;AAChC,UAAM,YACJ,OAAO,SAAS,IAAI,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO,SAAS;AAC1E,oBAAgB,IAAI,IAAI;AAAA,MACtB;AAAA,MACA,eAAe,aAAa,IAAI,EAAG;AAAA,MACnC,gBAAgB,aAAa,IAAI,EAAG;AAAA,MACpC,gBAAgB,aAAa,IAAI,EAAG;AAAA,IACtC;AAAA,EACF;AAEA,QAAM,kBAAkB,MAAM,cAAc,aAAa;AACzD,QAAM,mBAAmB,MAAM,cAAc,cAAc;AAE3D,QAAM,YAAuB;AAAA,IAC3B,gBACE,iBAAiB,iBAAiB,gBAAgB;AAAA,IACpD,cACE,iBAAiB,gBAAgB,gBAAgB;AAAA,IACnD,eACE,iBAAiB,iBAAiB,gBAAgB;AAAA,EACtD;AAEA,QAAM,oBAGF,CAAC;AACL,MAAI,eAAe;AACnB,aAAW,CAAC,MAAM,OAAO,KAAK,OAAO,QAAQ,eAAe,GAAG;AAC7D,UAAM,UAAU,QAAQ,YAAY,CAAC;AACrC,sBAAkB,IAAI,IAAI,EAAE,WAAW,QAAQ,WAAW,QAAQ;AAClE,QAAI,QAAS,gBAAe;AAAA,EAC9B;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,oBAAoB,cAAc;AAAA,IAClC,YAAY,EAAE,WAAW,cAAc,SAAS,kBAAkB;AAAA,EACpE;AACF;",
+  "names": []
+}

package/dist/dataset/hf.d.ts

@@ -0,0 +1,16 @@
+export interface HfOptions {
+    dataset: string;
+    config: string;
+    split: string;
+    rows?: number;
+}
+export declare function hf<T = Record<string, unknown>>(options: HfOptions): AsyncIterable<T>;
+export declare function fetchHfRows(options: {
+    dataset: string;
+    config: string;
+    split: string;
+}, offset: number, length: number): Promise<{
+    rows: Record<string, unknown>[];
+    total: number;
+}>;
+//# sourceMappingURL=hf.d.ts.map

package/dist/dataset/hf.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hf.d.ts","sourceRoot":"","sources":["../../src/dataset/hf.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAUD,wBAAgB,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC5C,OAAO,EAAE,SAAS,GACjB,aAAa,CAAC,CAAC,CAAC,CAMlB;AA2CD,wBAAsB,WAAW,CAC/B,OAAO,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,EAC3D,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,GACb,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAa7D"}

package/dist/dataset/index.d.ts

@@ -0,0 +1,17 @@
+export { hf } from './hf.ts';
+export type { HfOptions } from './hf.ts';
+export type TransformFn<T, U> = (item: T) => U;
+export type PredicateFn<T> = (item: T) => boolean;
+export declare class Dataset<T> implements AsyncIterable<T> {
+    #private;
+    constructor(source: () => AsyncIterable<T>);
+    map<U>(fn: TransformFn<T, U>): Dataset<U>;
+    filter(fn: PredicateFn<T>): Dataset<T>;
+    limit(n: number): Dataset<T>;
+    shuffle(): Dataset<T>;
+    sample(n: number): Dataset<T>;
+    toArray(): Promise<T[]>;
+    [Symbol.asyncIterator](): AsyncIterator<T>;
+}
+export declare function dataset<T>(source: T[] | string | AsyncIterable<T>): Dataset<T>;
+//# sourceMappingURL=index.d.ts.map

package/dist/dataset/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/dataset/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,EAAE,EAAE,MAAM,SAAS,CAAC;AAC7B,YAAY,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEzC,MAAM,MAAM,WAAW,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC;AAC/C,MAAM,MAAM,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC;AAElD,qBAAa,OAAO,CAAC,CAAC,CAAE,YAAW,aAAa,CAAC,CAAC,CAAC;;gBAGrC,MAAM,EAAE,MAAM,aAAa,CAAC,CAAC,CAAC;IAI1C,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IASzC,MAAM,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAStC,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;IAY5B,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC;IAiBrB,MAAM,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;IAoBvB,OAAO,IAAI,OAAO,CAAC,CAAC,EAAE,CAAC;IAQ7B,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC;CAG3C;AA+FD,wBAAgB,OAAO,CAAC,CAAC,EACvB,MAAM,EAAE,CAAC,EAAE,GAAG,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,GACtC,OAAO,CAAC,CAAC,CAAC,CAwBZ"}

package/dist/dataset/index.js

@@ -0,0 +1,256 @@
+// packages/evals/src/dataset/index.ts
+import { createReadStream } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { extname } from "node:path";
+import { createInterface } from "node:readline";
+
+// packages/evals/src/dataset/hf.ts
+var HF_BASE_URL = "https://datasets-server.huggingface.co/rows";
+var PAGE_SIZE = 100;
+function hf(options) {
+  return {
+    [Symbol.asyncIterator]() {
+      return paginate(options);
+    }
+  };
+}
+async function* paginate(options) {
+  const { dataset: dataset2, config, split, rows } = options;
+  const limit = rows ?? Infinity;
+  let offset = 0;
+  let yielded = 0;
+  while (yielded < limit) {
+    const pageSize = limit === Infinity ? PAGE_SIZE : Math.min(PAGE_SIZE, limit - yielded);
+    const url = buildUrl(dataset2, config, split, offset, pageSize);
+    const page = await fetchPage(url);
+    if (page.rows.length === 0) return;
+    for (const entry of page.rows) {
+      yield entry.row;
+      yielded++;
+      if (yielded >= limit) return;
+    }
+    offset += page.rows.length;
+    if (page.rows.length < pageSize || offset >= page.num_rows_total) return;
+  }
+}
+function buildUrl(dataset2, config, split, offset, length) {
+  const url = new URL(HF_BASE_URL);
+  url.searchParams.set("dataset", dataset2);
+  url.searchParams.set("config", config);
+  url.searchParams.set("split", split);
+  url.searchParams.set("offset", String(offset));
+  url.searchParams.set("length", String(length));
+  return url.toString();
+}
+async function fetchPage(url) {
+  const response = await fetch(url);
+  if (!response.ok) {
+    const body = await response.text().catch(() => "");
+    throw new Error(
+      `HuggingFace API error ${response.status}: ${body || response.statusText}`
+    );
+  }
+  const text = await response.text();
+  try {
+    return JSON.parse(text);
+  } catch {
+    throw new Error(
+      `HuggingFace API returned non-JSON response from ${url}: ${text.slice(0, 200)}`
+    );
+  }
+}
+
+// packages/evals/src/dataset/index.ts
+var Dataset = class _Dataset {
+  #source;
+  constructor(source) {
+    this.#source = source;
+  }
+  map(fn) {
+    const source = this.#source;
+    return new _Dataset(async function* () {
+      for await (const item of source()) {
+        yield fn(item);
+      }
+    });
+  }
+  filter(fn) {
+    const source = this.#source;
+    return new _Dataset(async function* () {
+      for await (const item of source()) {
+        if (fn(item)) yield item;
+      }
+    });
+  }
+  limit(n) {
+    const source = this.#source;
+    return new _Dataset(async function* () {
+      let count = 0;
+      for await (const item of source()) {
+        if (count >= n) return;
+        yield item;
+        count++;
+      }
+    });
+  }
+  shuffle() {
+    const source = this.#source;
+    return new _Dataset(async function* () {
+      const items = [];
+      for await (const item of source()) {
+        items.push(item);
+      }
+      for (let i = items.length - 1; i > 0; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        const temp = items[i];
+        items[i] = items[j];
+        items[j] = temp;
+      }
+      yield* items;
+    });
+  }
+  sample(n) {
+    const source = this.#source;
+    return new _Dataset(async function* () {
+      const items = [];
+      for await (const item of source()) {
+        items.push(item);
+      }
+      const count = Math.min(Math.max(0, n), items.length);
+      for (let i = items.length - 1; i > items.length - count - 1; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        const temp = items[i];
+        items[i] = items[j];
+        items[j] = temp;
+      }
+      for (let i = items.length - count; i < items.length; i++) {
+        yield items[i];
+      }
+    });
+  }
+  async toArray() {
+    const result = [];
+    for await (const item of this.#source()) {
+      result.push(item);
+    }
+    return result;
+  }
+  [Symbol.asyncIterator]() {
+    return this.#source()[Symbol.asyncIterator]();
+  }
+};
+function parseCSVLine(line) {
+  const fields = [];
+  let current = "";
+  let inQuotes = false;
+  for (let i = 0; i < line.length; i++) {
+    const char = line[i];
+    if (inQuotes) {
+      if (char === '"') {
+        if (i + 1 < line.length && line[i + 1] === '"') {
+          current += '"';
+          i++;
+        } else {
+          inQuotes = false;
+        }
+      } else {
+        current += char;
+      }
+    } else {
+      if (char === '"' && current === "") {
+        inQuotes = true;
+      } else if (char === ",") {
+        fields.push(current);
+        current = "";
+      } else {
+        current += char;
+      }
+    }
+  }
+  fields.push(current);
+  return fields;
+}
+function loadJSON(filePath) {
+  return async function* () {
+    const content = await readFile(filePath, "utf-8");
+    const data = JSON.parse(content);
+    if (!Array.isArray(data)) {
+      throw new Error(`JSON file "${filePath}" does not contain an array`);
+    }
+    yield* data;
+  };
+}
+function loadJSONL(filePath) {
+  return async function* () {
+    const rl = createInterface({
+      input: createReadStream(filePath, "utf-8"),
+      crlfDelay: Infinity
+    });
+    try {
+      for await (const line of rl) {
+        const trimmed = line.trim();
+        if (trimmed) {
+          yield JSON.parse(trimmed);
+        }
+      }
+    } finally {
+      rl.close();
+    }
+  };
+}
+function loadCSV(filePath) {
+  return async function* () {
+    const rl = createInterface({
+      input: createReadStream(filePath, "utf-8"),
+      crlfDelay: Infinity
+    });
+    try {
+      let headers;
+      for await (const line of rl) {
+        const trimmed = line.trim();
+        if (!trimmed) continue;
+        const fields = parseCSVLine(trimmed);
+        if (!headers) {
+          headers = fields;
+          continue;
+        }
+        const row = {};
+        for (let i = 0; i < headers.length; i++) {
+          row[headers[i]] = fields[i] ?? "";
+        }
+        yield row;
+      }
+    } finally {
+      rl.close();
+    }
+  };
+}
+function dataset(source) {
+  if (Array.isArray(source)) {
+    return new Dataset(async function* () {
+      yield* source;
+    });
+  }
+  if (typeof source === "object" && Symbol.asyncIterator in source) {
+    return new Dataset(() => source);
+  }
+  const ext = extname(source).toLowerCase();
+  switch (ext) {
+    case ".json":
+      return new Dataset(loadJSON(source));
+    case ".jsonl":
+      return new Dataset(loadJSONL(source));
+    case ".csv":
+      return new Dataset(loadCSV(source));
+    default:
+      throw new Error(
+        `Unsupported file extension "${ext}" for dataset file "${source}". Supported: .json, .jsonl, .csv`
+      );
+  }
+}
+export {
+  Dataset,
+  dataset,
+  hf
+};
+//# sourceMappingURL=index.js.map