@graphorin/evals 0.5.0

package/CHANGELOG.md

@@ -0,0 +1,10 @@
+# @graphorin/evals
+
+## 0.1.0
+
+### Minor Changes
+
+- Initial release of the eval framework for the Graphorin framework.
+  Ships scorer libraries, dataset loaders (JSONL/CSV/from-traces),
+  reporters (terminal/markdown/JSON/JUnit), parallel runner with
+  regression detection, and CLI integration helpers.

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Oleksiy Stepurenko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,157 @@
+# @graphorin/evals
+
+> Eval framework for the [Graphorin](https://github.com/o-stepper/graphorin)
+> framework. Ships scorer libraries (code, LLM-judge, prebuilt rubrics),
+> dataset loaders (JSONL / CSV / from-traces / iterable), reporters
+> (terminal / markdown / JSON / JUnit / HTML), a parallel runner with
+> bounded concurrency, and regression detection that compares the
+> current run against a stored baseline.
+>
+> Project Graphorin · v0.5.0 · MIT License · © 2026 Oleksiy Stepurenko ·
+> <https://github.com/o-stepper/graphorin>
+
+---
+
+## Status
+
+- **Published:** v0.5.0 (optional sub-pack; the full orchestrator is
+  decoupled from `@graphorin/observability` per RB-17 / DEC-152).
+
+---
+
+## Install
+
+```bash
+pnpm add @graphorin/evals
+```
+
+The package depends only on `@graphorin/core` and
+`@graphorin/observability`; reporters / loaders are part of the same
+bundle so consumers do not need additional installs.
+
+---
+
+## Quickstart
+
+```ts
+import {
+  runEvals,
+  loadJsonlDataset,
+  exactMatch,
+  renderTerminalReport,
+  exitOnFailures,
+} from '@graphorin/evals';
+
+const dataset = await loadJsonlDataset('./fixtures/golden.jsonl');
+const report = await runEvals({
+  agent,                       // anything with `run(input)`
+  dataset,
+  scorers: [exactMatch({ caseInsensitive: true })],
+  concurrency: 4,
+});
+console.log(renderTerminalReport(report));
+exitOnFailures(report);
+```
+
+---
+
+## Scorers
+
+| Scorer family | Identifiers | Notes |
+|---|---|---|
+| `code/`     | `exactMatch`, `regexMatch`, `jsonPath`, `predicate` | Pure-code grading. No provider call.|
+| `llm/`      | `llmJudge`                                          | LLM-as-judge. Default `gpt-4o-mini`-class judge with `temperature: 0`. |
+| `prebuilt/` | `toxicityScorer`, `factualityScorer`, `helpfulnessScorer` | Wrap `llmJudge` with a project-tested rubric. |
+| `trajectory/` | `correctToolSelected`, `argumentValidity`, `redundantCallDetection`, `recoveryAfterError`, `finalStateCorrect` | Pure-code, offline scorers over a `Trajectory` (the tool calls a harness made). Measure harness reliability — tool selection, argument validity, redundant work, error recovery, goal state. |
+
+---
+
+## Dataset loaders
+
+| Loader | Use |
+|---|---|
+| `loadJsonlDataset(path)`                       | Read a JSONL file. Each line is a JSON object with `input` + optional `expected` / `id` / `metadata`.|
+| `loadCsvDataset(path)`                         | Read a CSV file (RFC 4180 strict subset). Columns map by name.|
+| `loadDatasetFromTraces(path, { extract })`     | Distil a dataset from the framework's replay log.|
+| `fromIterable(cases)`                          | Wrap an in-memory array as a dataset (tests / ad-hoc data).|
+
+---
+
+## Reporters
+
+| Reporter | Output | Best for |
+|---|---|---|
+| `renderTerminalReport(report)`  | Plain text (no ANSI). | CI logs, local dev. |
+| `renderMarkdownReport(report)`  | Markdown.             | PR descriptions, doc sites. |
+| `renderJsonReport(report)`      | Canonical JSON.       | Dashboards, regression checkers. |
+| `renderJunitReport(report)`     | JUnit XML.            | GitHub Actions / GitLab / CircleCI. |
+| `renderHtmlReport(report)`      | Self-contained HTML.  | Artifact viewers. |
+
+---
+
+## Parallel runner
+
+```ts
+const report = await runEvals({
+  agent,
+  dataset,
+  scorers,
+  iterations: 3,            // each case run 3 times for variance estimation
+  concurrency: 8,           // up to 8 parallel agent.run() calls
+  signal: controller.signal,
+  onProgress: (e) => console.log(`${e.index}/${e.total} ${e.caseId}`),
+});
+```
+
+---
+
+## Regression detection
+
+```ts
+import { detectRegressions, exitOnFailures } from '@graphorin/evals';
+
+const baseline = JSON.parse(await readFile('./baselines/golden.json', 'utf8'));
+const report = await runEvals(...);
+const regression = detectRegressions(report, baseline, {
+  maxPassRateDropPct: 5,
+  maxAvgScoreDrop: 0.05,
+  maxAvgDurationIncreaseMs: 250,
+});
+if (regression.hasRegressions) {
+  for (const f of regression.findings) {
+    console.error(`regression — ${f.kind}: ${f.message}`);
+  }
+}
+exitOnFailures(report, regression);
+```
+
+---
+
+## Multi-format report writing
+
+```ts
+import { writeReports } from '@graphorin/evals';
+
+await writeReports({
+  report,
+  outDir: './eval-out',
+  formats: ['terminal', 'markdown', 'json', 'junit', 'html'],
+  basename: 'golden',
+});
+```
+
+---
+
+## Related decisions
+
+- DEC-152 — Eval split: keep evaluation interfaces in `@graphorin/observability`, ship the full eval framework as `@graphorin/evals`.
+
+---
+
+## License
+
+MIT © 2026 Oleksiy Stepurenko
+
+---
+
+**Project Graphorin** · v0.5.0 · MIT License · © 2026 Oleksiy Stepurenko · <https://github.com/o-stepper/graphorin>

package/dist/cli/index.d.ts

@@ -0,0 +1,43 @@
+import { RegressionOptions, RegressionReport } from "../types.js";
+import { detectRegressions } from "../regression.js";
+import { EvalReport } from "@graphorin/observability/eval";
+
+//#region src/cli/index.d.ts
+
+/**
+ * Set `process.exitCode` to `1` when at least one case failed, or
+ * when a regression report contains findings. Uses `exitCode` rather
+ * than `process.exit` so other async tasks finish cleanly.
+ *
+ * @stable
+ */
+declare function exitOnFailures<I, O>(report: EvalReport<I, O>, regression?: RegressionReport<I, O>): void;
+/**
+ * Reporter ids accepted by {@link writeReports}.
+ *
+ * @stable
+ */
+type ReporterFormat = 'terminal' | 'markdown' | 'json' | 'junit' | 'html';
+/** @stable */
+interface WriteReportsOptions<I, O> {
+  readonly report: EvalReport<I, O>;
+  readonly outDir?: string;
+  readonly formats: ReadonlyArray<ReporterFormat>;
+  readonly basename?: string;
+}
+/** @stable */
+interface WrittenReport {
+  readonly format: ReporterFormat;
+  readonly path: string;
+  readonly bytes: number;
+}
+/**
+ * Render the report in every requested format and write each one to a
+ * file. Returns the manifest of written files.
+ *
+ * @stable
+ */
+declare function writeReports<I, O>(options: WriteReportsOptions<I, O>): Promise<ReadonlyArray<WrittenReport>>;
+//#endregion
+export { type RegressionOptions, ReporterFormat, WriteReportsOptions, WrittenReport, detectRegressions, exitOnFailures, writeReports };
+//# sourceMappingURL=index.d.ts.map

package/dist/cli/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/cli/index.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;;;iBAkDgB,6BACN,WAAW,GAAG,iBACT,iBAAiB,GAAG;;;;;;KAWvB,cAAA;;UAGK;mBACE,WAAW,GAAG;;oBAEb,cAAc;;;;UAKjB,aAAA;mBACE;;;;;;;;;;iBAWG,4BACX,oBAAoB,GAAG,KAC/B,QAAQ,cAAc"}

package/dist/cli/index.js

@@ -0,0 +1,102 @@
+import { detectRegressions } from "../regression.js";
+import { renderHtmlReport } from "../reporters/html.js";
+import { renderJsonReport } from "../reporters/json.js";
+import { renderJunitReport } from "../reporters/junit.js";
+import { renderMarkdownReport } from "../reporters/markdown.js";
+import { renderTerminalReport } from "../reporters/terminal.js";
+import { writeFile } from "node:fs/promises";
+import process from "node:process";
+
+//#region src/cli/index.ts
+/**
+* CLI integration helpers. Convenience wrappers that combine the
+* runner + a reporter + an exit-code mapping so consumer scripts can
+* stay short.
+*
+* Typical use from a `package.json` script:
+*
+* ```jsonc
+* {
+*   "scripts": {
+*     "eval": "node ./scripts/run-evals.mjs"
+*   }
+* }
+* ```
+*
+* Where `run-evals.mjs` looks like:
+*
+* ```ts
+* import { runEvals, exitOnFailures, renderTerminalReport } from '@graphorin/evals';
+*
+* const report = await runEvals({...});
+* console.log(renderTerminalReport(report));
+* exitOnFailures(report);
+* ```
+*
+* @packageDocumentation
+*/
+/**
+* Set `process.exitCode` to `1` when at least one case failed, or
+* when a regression report contains findings. Uses `exitCode` rather
+* than `process.exit` so other async tasks finish cleanly.
+*
+* @stable
+*/
+function exitOnFailures(report, regression) {
+	if (report.summary.failed > 0) process.exitCode = 1;
+	if (regression?.hasRegressions) process.exitCode = 1;
+}
+/**
+* Render the report in every requested format and write each one to a
+* file. Returns the manifest of written files.
+*
+* @stable
+*/
+async function writeReports(options) {
+	const out = [];
+	const dir = options.outDir ?? ".";
+	const basename = options.basename ?? "eval-report";
+	for (const format of options.formats) {
+		const { extension, body } = renderForFormat(format, options.report);
+		const path = `${dir}/${basename}.${extension}`;
+		await writeFile(path, body, "utf8");
+		out.push({
+			format,
+			path,
+			bytes: Buffer.byteLength(body, "utf8")
+		});
+	}
+	return out;
+}
+function renderForFormat(format, report) {
+	switch (format) {
+		case "terminal": return {
+			extension: "txt",
+			body: renderTerminalReport(report)
+		};
+		case "markdown": return {
+			extension: "md",
+			body: renderMarkdownReport(report)
+		};
+		case "json": return {
+			extension: "json",
+			body: renderJsonReport(report, { pretty: true })
+		};
+		case "junit": return {
+			extension: "xml",
+			body: renderJunitReport(report)
+		};
+		case "html": return {
+			extension: "html",
+			body: renderHtmlReport(report)
+		};
+		default: {
+			const exhaustive = format;
+			throw new Error(`unknown reporter format '${exhaustive}'`);
+		}
+	}
+}
+
+//#endregion
+export { detectRegressions, exitOnFailures, writeReports };
+//# sourceMappingURL=index.js.map

package/dist/cli/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","names":["out: WrittenReport[]","exhaustive: never"],"sources":["../../src/cli/index.ts"],"sourcesContent":["/**\n * CLI integration helpers. Convenience wrappers that combine the\n * runner + a reporter + an exit-code mapping so consumer scripts can\n * stay short.\n *\n * Typical use from a `package.json` script:\n *\n * ```jsonc\n * {\n *   \"scripts\": {\n *     \"eval\": \"node ./scripts/run-evals.mjs\"\n *   }\n * }\n * ```\n *\n * Where `run-evals.mjs` looks like:\n *\n * ```ts\n * import { runEvals, exitOnFailures, renderTerminalReport } from '@graphorin/evals';\n *\n * const report = await runEvals({...});\n * console.log(renderTerminalReport(report));\n * exitOnFailures(report);\n * ```\n *\n * @packageDocumentation\n */\n\nimport { writeFile } from 'node:fs/promises';\nimport process from 'node:process';\n\nimport type { EvalReport } from '@graphorin/observability/eval';\n\nimport { detectRegressions } from '../regression.js';\nimport {\n  renderHtmlReport,\n  renderJsonReport,\n  renderJunitReport,\n  renderMarkdownReport,\n  renderTerminalReport,\n} from '../reporters/index.js';\nimport type { RegressionOptions, RegressionReport } from '../types.js';\n\n/**\n * Set `process.exitCode` to `1` when at least one case failed, or\n * when a regression report contains findings. Uses `exitCode` rather\n * than `process.exit` so other async tasks finish cleanly.\n *\n * @stable\n */\nexport function exitOnFailures<I, O>(\n  report: EvalReport<I, O>,\n  regression?: RegressionReport<I, O>,\n): void {\n  if (report.summary.failed > 0) process.exitCode = 1;\n  if (regression?.hasRegressions) process.exitCode = 1;\n}\n\n/**\n * Reporter ids accepted by {@link writeReports}.\n *\n * @stable\n */\nexport type ReporterFormat = 'terminal' | 'markdown' | 'json' | 'junit' | 'html';\n\n/** @stable */\nexport interface WriteReportsOptions<I, O> {\n  readonly report: EvalReport<I, O>;\n  readonly outDir?: string;\n  readonly formats: ReadonlyArray<ReporterFormat>;\n  readonly basename?: string;\n}\n\n/** @stable */\nexport interface WrittenReport {\n  readonly format: ReporterFormat;\n  readonly path: string;\n  readonly bytes: number;\n}\n\n/**\n * Render the report in every requested format and write each one to a\n * file. Returns the manifest of written files.\n *\n * @stable\n */\nexport async function writeReports<I, O>(\n  options: WriteReportsOptions<I, O>,\n): Promise<ReadonlyArray<WrittenReport>> {\n  const out: WrittenReport[] = [];\n  const dir = options.outDir ?? '.';\n  const basename = options.basename ?? 'eval-report';\n  for (const format of options.formats) {\n    const { extension, body } = renderForFormat(format, options.report);\n    const path = `${dir}/${basename}.${extension}`;\n    await writeFile(path, body, 'utf8');\n    out.push({ format, path, bytes: Buffer.byteLength(body, 'utf8') });\n  }\n  return out;\n}\n\nfunction renderForFormat<I, O>(\n  format: ReporterFormat,\n  report: EvalReport<I, O>,\n): { extension: string; body: string } {\n  switch (format) {\n    case 'terminal':\n      return { extension: 'txt', body: renderTerminalReport(report) };\n    case 'markdown':\n      return { extension: 'md', body: renderMarkdownReport(report) };\n    case 'json':\n      return { extension: 'json', body: renderJsonReport(report, { pretty: true }) };\n    case 'junit':\n      return { extension: 'xml', body: renderJunitReport(report) };\n    case 'html':\n      return { extension: 'html', body: renderHtmlReport(report) };\n    default: {\n      const exhaustive: never = format;\n      throw new Error(`unknown reporter format '${exhaustive as string}'`);\n    }\n  }\n}\n\nexport { detectRegressions, type RegressionOptions };\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkDA,SAAgB,eACd,QACA,YACM;AACN,KAAI,OAAO,QAAQ,SAAS,EAAG,SAAQ,WAAW;AAClD,KAAI,YAAY,eAAgB,SAAQ,WAAW;;;;;;;;AA+BrD,eAAsB,aACpB,SACuC;CACvC,MAAMA,MAAuB,EAAE;CAC/B,MAAM,MAAM,QAAQ,UAAU;CAC9B,MAAM,WAAW,QAAQ,YAAY;AACrC,MAAK,MAAM,UAAU,QAAQ,SAAS;EACpC,MAAM,EAAE,WAAW,SAAS,gBAAgB,QAAQ,QAAQ,OAAO;EACnE,MAAM,OAAO,GAAG,IAAI,GAAG,SAAS,GAAG;AACnC,QAAM,UAAU,MAAM,MAAM,OAAO;AACnC,MAAI,KAAK;GAAE;GAAQ;GAAM,OAAO,OAAO,WAAW,MAAM,OAAO;GAAE,CAAC;;AAEpE,QAAO;;AAGT,SAAS,gBACP,QACA,QACqC;AACrC,SAAQ,QAAR;EACE,KAAK,WACH,QAAO;GAAE,WAAW;GAAO,MAAM,qBAAqB,OAAO;GAAE;EACjE,KAAK,WACH,QAAO;GAAE,WAAW;GAAM,MAAM,qBAAqB,OAAO;GAAE;EAChE,KAAK,OACH,QAAO;GAAE,WAAW;GAAQ,MAAM,iBAAiB,QAAQ,EAAE,QAAQ,MAAM,CAAC;GAAE;EAChF,KAAK,QACH,QAAO;GAAE,WAAW;GAAO,MAAM,kBAAkB,OAAO;GAAE;EAC9D,KAAK,OACH,QAAO;GAAE,WAAW;GAAQ,MAAM,iBAAiB,OAAO;GAAE;EAC9D,SAAS;GACP,MAAMC,aAAoB;AAC1B,SAAM,IAAI,MAAM,4BAA4B,WAAqB,GAAG"}

package/dist/index.d.ts

@@ -0,0 +1,66 @@
+import { AgentLike, Case, Dataset, EvalCaseResult, EvalReport, ProgressEvent, RegressionFinding, RegressionOptions, RegressionReport, RunEvalOptions, RunOptions, ScoreResult, Scorer } from "./types.js";
+import { detectRegressions } from "./regression.js";
+import { ReporterFormat, WriteReportsOptions, WrittenReport, exitOnFailures, writeReports } from "./cli/index.js";
+import { LoadCsvOptions, loadCsvDataset, parseCsv } from "./loaders/csv.js";
+import { MemoryEvalAbility, MemoryEvalInput, MemoryEvalSession, MemoryEvalTurn } from "./loaders/memory-eval.js";
+import { LoadDmrOptions, loadDmrDataset, parseDmr } from "./loaders/dmr.js";
+import { FromTracesOptions, TraceEvent, loadDatasetFromTraces } from "./loaders/from-traces.js";
+import { fromIterable } from "./loaders/iterable.js";
+import { LoadJsonlOptions, loadJsonlDataset, parseJsonl } from "./loaders/jsonl.js";
+import { LoadLocomoOptions, loadLocomoDataset, parseLocomo } from "./loaders/locomo.js";
+import { LoadLongMemEvalOptions, loadLongMemEvalDataset, parseLongMemEval } from "./loaders/longmemeval.js";
+import { renderHtmlReport } from "./reporters/html.js";
+import { renderJsonReport } from "./reporters/json.js";
+import { renderJunitReport } from "./reporters/junit.js";
+import { renderMarkdownReport } from "./reporters/markdown.js";
+import { renderTerminalReport } from "./reporters/terminal.js";
+import { runEvals } from "./runner.js";
+import { ExactMatchOptions, exactMatch } from "./scorers/code/exact-match.js";
+import { JsonPathOptions, jsonPath } from "./scorers/code/json-path.js";
+import { PredicateOptions, predicate } from "./scorers/code/predicate.js";
+import { RegexMatchOptions, regexMatch } from "./scorers/code/regex.js";
+import { LlmJudgeOptions, fenceForJudge, llmJudge, parseScore, scoreContract } from "./scorers/llm/judge.js";
+import { PrebuiltScorerOptions, factualityScorer, helpfulnessScorer, toxicityScorer } from "./scorers/prebuilt/index.js";
+import { Trajectory, TrajectoryToolCall } from "./scorers/trajectory/types.js";
+import { ArgumentValidityOptions, argumentValidity } from "./scorers/trajectory/argument-validity.js";
+import { CorrectToolSelectedOptions, correctToolSelected } from "./scorers/trajectory/correct-tool-selected.js";
+import { FinalStateCorrectOptions, finalStateCorrect } from "./scorers/trajectory/final-state-correct.js";
+import { RecoveryAfterErrorOptions, recoveryAfterError } from "./scorers/trajectory/recovery-after-error.js";
+import { RedundantCallDetectionOptions, redundantCallDetection } from "./scorers/trajectory/redundant-call-detection.js";
+
+//#region src/index.d.ts
+/**
+ * @graphorin/evals — eval framework for the Graphorin framework.
+ *
+ * Builds on the eval primitives shipped from `@graphorin/observability`
+ * (RB-17 / DEC-152) — full orchestrator lives here, post-MVP, decoupled
+ * from the core observability bundle so consumers do not pay the
+ * dataset / reporter cost when only the inline runner is needed.
+ *
+ * ```ts
+ * import {
+ *   runEvals,
+ *   loadJsonlDataset,
+ *   exactMatch,
+ *   renderTerminalReport,
+ *   exitOnFailures,
+ * } from '@graphorin/evals';
+ *
+ * const dataset = await loadJsonlDataset('./fixtures/golden.jsonl');
+ * const report = await runEvals({
+ *   agent,
+ *   dataset,
+ *   scorers: [exactMatch()],
+ *   concurrency: 4,
+ * });
+ * console.log(renderTerminalReport(report));
+ * exitOnFailures(report);
+ * ```
+ *
+ * @packageDocumentation
+ */
+/** Canonical version constant. Mirrors the `package.json` version. */
+declare const VERSION = "0.5.0";
+//#endregion
+export { type AgentLike, type ArgumentValidityOptions, type Case, type CorrectToolSelectedOptions, type Dataset, type EvalCaseResult, type EvalReport, type ExactMatchOptions, type FinalStateCorrectOptions, type FromTracesOptions, type JsonPathOptions, type LlmJudgeOptions, type LoadCsvOptions, type LoadDmrOptions, type LoadJsonlOptions, type LoadLocomoOptions, type LoadLongMemEvalOptions, type MemoryEvalAbility, type MemoryEvalInput, type MemoryEvalSession, type MemoryEvalTurn, type PrebuiltScorerOptions, type PredicateOptions, type ProgressEvent, type RecoveryAfterErrorOptions, type RedundantCallDetectionOptions, type RegexMatchOptions, type RegressionFinding, type RegressionOptions, type RegressionReport, type ReporterFormat, type RunEvalOptions, type RunOptions, type ScoreResult, type Scorer, type TraceEvent, type Trajectory, type TrajectoryToolCall, VERSION, type WriteReportsOptions, type WrittenReport, argumentValidity, correctToolSelected, detectRegressions, detectRegressions as detectRegressionsFromReports, exactMatch, exitOnFailures, factualityScorer, fenceForJudge, finalStateCorrect, fromIterable, helpfulnessScorer, jsonPath, llmJudge, loadCsvDataset, loadDatasetFromTraces, loadDmrDataset, loadJsonlDataset, loadLocomoDataset, loadLongMemEvalDataset, parseCsv, parseDmr, parseJsonl, parseLocomo, parseLongMemEval, parseScore, predicate, recoveryAfterError, redundantCallDetection, regexMatch, renderHtmlReport, renderJsonReport, renderJunitReport, renderMarkdownReport, renderTerminalReport, runEvals, scoreContract, toxicityScorer, writeReports };
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","names":[],"sources":["../src/index.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgCa,cAAA,OAAA,GAAO,OAAA"}

package/dist/index.js

@@ -0,0 +1,64 @@
+import { detectRegressions } from "./regression.js";
+import { renderHtmlReport } from "./reporters/html.js";
+import { renderJsonReport } from "./reporters/json.js";
+import { renderJunitReport } from "./reporters/junit.js";
+import { renderMarkdownReport } from "./reporters/markdown.js";
+import { renderTerminalReport } from "./reporters/terminal.js";
+import { exitOnFailures, writeReports } from "./cli/index.js";
+import { loadCsvDataset, parseCsv } from "./loaders/csv.js";
+import { loadDmrDataset, parseDmr } from "./loaders/dmr.js";
+import { loadDatasetFromTraces } from "./loaders/from-traces.js";
+import { fromIterable } from "./loaders/iterable.js";
+import { loadJsonlDataset, parseJsonl } from "./loaders/jsonl.js";
+import { loadLocomoDataset, parseLocomo } from "./loaders/locomo.js";
+import { loadLongMemEvalDataset, parseLongMemEval } from "./loaders/longmemeval.js";
+import { runEvals } from "./runner.js";
+import { exactMatch } from "./scorers/code/exact-match.js";
+import { jsonPath } from "./scorers/code/json-path.js";
+import { predicate } from "./scorers/code/predicate.js";
+import { regexMatch } from "./scorers/code/regex.js";
+import { fenceForJudge, llmJudge, parseScore, scoreContract } from "./scorers/llm/judge.js";
+import { factualityScorer, helpfulnessScorer, toxicityScorer } from "./scorers/prebuilt/index.js";
+import { argumentValidity } from "./scorers/trajectory/argument-validity.js";
+import { correctToolSelected } from "./scorers/trajectory/correct-tool-selected.js";
+import { finalStateCorrect } from "./scorers/trajectory/final-state-correct.js";
+import { recoveryAfterError } from "./scorers/trajectory/recovery-after-error.js";
+import { redundantCallDetection } from "./scorers/trajectory/redundant-call-detection.js";
+
+//#region src/index.ts
+/**
+* @graphorin/evals — eval framework for the Graphorin framework.
+*
+* Builds on the eval primitives shipped from `@graphorin/observability`
+* (RB-17 / DEC-152) — full orchestrator lives here, post-MVP, decoupled
+* from the core observability bundle so consumers do not pay the
+* dataset / reporter cost when only the inline runner is needed.
+*
+* ```ts
+* import {
+*   runEvals,
+*   loadJsonlDataset,
+*   exactMatch,
+*   renderTerminalReport,
+*   exitOnFailures,
+* } from '@graphorin/evals';
+*
+* const dataset = await loadJsonlDataset('./fixtures/golden.jsonl');
+* const report = await runEvals({
+*   agent,
+*   dataset,
+*   scorers: [exactMatch()],
+*   concurrency: 4,
+* });
+* console.log(renderTerminalReport(report));
+* exitOnFailures(report);
+* ```
+*
+* @packageDocumentation
+*/
+/** Canonical version constant. Mirrors the `package.json` version. */
+const VERSION = "0.5.0";
+
+//#endregion
+export { VERSION, argumentValidity, correctToolSelected, detectRegressions, detectRegressions as detectRegressionsFromReports, exactMatch, exitOnFailures, factualityScorer, fenceForJudge, finalStateCorrect, fromIterable, helpfulnessScorer, jsonPath, llmJudge, loadCsvDataset, loadDatasetFromTraces, loadDmrDataset, loadJsonlDataset, loadLocomoDataset, loadLongMemEvalDataset, parseCsv, parseDmr, parseJsonl, parseLocomo, parseLongMemEval, parseScore, predicate, recoveryAfterError, redundantCallDetection, regexMatch, renderHtmlReport, renderJsonReport, renderJunitReport, renderMarkdownReport, renderTerminalReport, runEvals, scoreContract, toxicityScorer, writeReports };
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","names":[],"sources":["../src/index.ts"],"sourcesContent":["/**\n * @graphorin/evals — eval framework for the Graphorin framework.\n *\n * Builds on the eval primitives shipped from `@graphorin/observability`\n * (RB-17 / DEC-152) — full orchestrator lives here, post-MVP, decoupled\n * from the core observability bundle so consumers do not pay the\n * dataset / reporter cost when only the inline runner is needed.\n *\n * ```ts\n * import {\n *   runEvals,\n *   loadJsonlDataset,\n *   exactMatch,\n *   renderTerminalReport,\n *   exitOnFailures,\n * } from '@graphorin/evals';\n *\n * const dataset = await loadJsonlDataset('./fixtures/golden.jsonl');\n * const report = await runEvals({\n *   agent,\n *   dataset,\n *   scorers: [exactMatch()],\n *   concurrency: 4,\n * });\n * console.log(renderTerminalReport(report));\n * exitOnFailures(report);\n * ```\n *\n * @packageDocumentation\n */\n\n/** Canonical version constant. Mirrors the `package.json` version. */\nexport const VERSION = '0.5.0';\n\nexport {\n  detectRegressions,\n  exitOnFailures,\n  type ReporterFormat,\n  type WriteReportsOptions,\n  type WrittenReport,\n  writeReports,\n} from './cli/index.js';\nexport type {\n  FromTracesOptions,\n  LoadDmrOptions,\n  LoadLocomoOptions,\n  LoadLongMemEvalOptions,\n  MemoryEvalAbility,\n  MemoryEvalInput,\n  MemoryEvalSession,\n  MemoryEvalTurn,\n  TraceEvent,\n} from './loaders/index.js';\nexport {\n  fromIterable,\n  type LoadCsvOptions,\n  type LoadJsonlOptions,\n  loadCsvDataset,\n  loadDatasetFromTraces,\n  loadDmrDataset,\n  loadJsonlDataset,\n  loadLocomoDataset,\n  loadLongMemEvalDataset,\n  parseCsv,\n  parseDmr,\n  parseJsonl,\n  parseLocomo,\n  parseLongMemEval,\n} from './loaders/index.js';\nexport { detectRegressions as detectRegressionsFromReports } from './regression.js';\nexport {\n  renderHtmlReport,\n  renderJsonReport,\n  renderJunitReport,\n  renderMarkdownReport,\n  renderTerminalReport,\n} from './reporters/index.js';\nexport { runEvals } from './runner.js';\nexport {\n  type ArgumentValidityOptions,\n  argumentValidity,\n  type CorrectToolSelectedOptions,\n  correctToolSelected,\n  type ExactMatchOptions,\n  exactMatch,\n  type FinalStateCorrectOptions,\n  factualityScorer,\n  fenceForJudge,\n  finalStateCorrect,\n  helpfulnessScorer,\n  type JsonPathOptions,\n  jsonPath,\n  type LlmJudgeOptions,\n  llmJudge,\n  type PrebuiltScorerOptions,\n  type PredicateOptions,\n  parseScore,\n  predicate,\n  type RecoveryAfterErrorOptions,\n  type RedundantCallDetectionOptions,\n  type RegexMatchOptions,\n  recoveryAfterError,\n  redundantCallDetection,\n  regexMatch,\n  scoreContract,\n  type Trajectory,\n  type TrajectoryToolCall,\n  toxicityScorer,\n} from './scorers/index.js';\nexport type {\n  AgentLike,\n  Case,\n  Dataset,\n  EvalCaseResult,\n  EvalReport,\n  ProgressEvent,\n  RegressionFinding,\n  RegressionOptions,\n  RegressionReport,\n  RunEvalOptions,\n  RunOptions,\n  ScoreResult,\n  Scorer,\n} from './types.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgCA,MAAa,UAAU"}

package/dist/loaders/csv.d.ts

@@ -0,0 +1,23 @@
+import { Case, Dataset } from "@graphorin/observability/eval";
+
+//#region src/loaders/csv.d.ts
+
+/** @stable */
+interface LoadCsvOptions {
+  readonly delimiter?: string;
+  readonly name?: string;
+  readonly description?: string;
+  readonly mapper?: (row: Record<string, string>, index: number) => Case<unknown, unknown>;
+}
+/** @stable */
+declare function loadCsvDataset(path: string, options?: LoadCsvOptions): Promise<Dataset<unknown, unknown>>;
+/**
+ * Pure parser. Exported separately so tests can exercise the
+ * column-mapping behaviour without touching the filesystem.
+ *
+ * @stable
+ */
+declare function parseCsv(text: string, options?: LoadCsvOptions): ReadonlyArray<Case<unknown, unknown>>;
+//#endregion
+export { LoadCsvOptions, loadCsvDataset, parseCsv };
+//# sourceMappingURL=csv.d.ts.map

package/dist/loaders/csv.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv.d.ts","names":[],"sources":["../../src/loaders/csv.ts"],"sourcesContent":[],"mappings":";;;;;UAmBiB,cAAA;;;;0BAIS,0CAA0C;;;iBAI9C,cAAA,yBAEX,iBACR,QAAQ;;;;;;;iBAiBK,QAAA,yBAEL,iBACR,cAAc"}

package/dist/loaders/csv.js

@@ -0,0 +1,116 @@
+import { readFile } from "node:fs/promises";
+
+//#region src/loaders/csv.ts
+/**
+* CSV dataset loader. Implements RFC 4180 strict subset:
+*
+*  - Comma separator (configurable).
+*  - Optional `"`-quoted cells; doubled `""` to embed a literal `"`.
+*  - Header row required by default.
+*
+* Columns map to `Case` fields by name: `input`, `expected`, `id`,
+* `metadata` (parsed as JSON when present). Unknown columns are
+* ignored unless a `mapper` is supplied.
+*
+* @packageDocumentation
+*/
+/** @stable */
+async function loadCsvDataset(path, options = {}) {
+	return {
+		cases: parseCsv(await readFile(path, "utf8"), options),
+		metadata: {
+			...options.name !== void 0 ? { name: options.name } : {},
+			...options.description !== void 0 ? { description: options.description } : {},
+			createdAt: /* @__PURE__ */ new Date()
+		}
+	};
+}
+/**
+* Pure parser. Exported separately so tests can exercise the
+* column-mapping behaviour without touching the filesystem.
+*
+* @stable
+*/
+function parseCsv(text, options = {}) {
+	const rows = parseCsvRows(text, options.delimiter ?? ",");
+	if (rows.length === 0) return [];
+	const header = rows[0];
+	if (header === void 0) return [];
+	const dataRows = rows.slice(1);
+	const out = [];
+	for (let i = 0; i < dataRows.length; i++) {
+		const row = dataRows[i] ?? [];
+		if (row.every((cell) => cell.length === 0)) continue;
+		const record = {};
+		for (let c = 0; c < header.length; c++) {
+			const key = header[c] ?? "";
+			record[key] = row[c] ?? "";
+		}
+		if (options.mapper !== void 0) {
+			out.push(options.mapper(record, i));
+			continue;
+		}
+		if (record.input === void 0) throw new Error(`[graphorin/evals] CSV row ${i + 1} missing required 'input' column.`);
+		const sampleCase = {
+			...record.id !== void 0 && record.id.length > 0 ? { id: record.id } : {},
+			input: record.input,
+			..."expected" in record && record.expected !== void 0 && record.expected.length > 0 ? { expected: record.expected } : {},
+			...record.metadata !== void 0 && record.metadata.length > 0 ? { metadata: parseMetadata(record.metadata) } : {}
+		};
+		out.push(sampleCase);
+	}
+	return out;
+}
+function parseMetadata(raw) {
+	try {
+		const parsed = JSON.parse(raw);
+		if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) return parsed;
+		return { value: parsed };
+	} catch {
+		return { raw };
+	}
+}
+function parseCsvRows(text, delimiter) {
+	const rows = [];
+	let current = [];
+	let cell = "";
+	let inQuotes = false;
+	for (let i = 0; i < text.length; i++) {
+		const ch = text[i];
+		if (inQuotes) {
+			if (ch === "\"") if (text[i + 1] === "\"") {
+				cell += "\"";
+				i += 1;
+			} else inQuotes = false;
+			else cell += ch;
+			continue;
+		}
+		if (ch === "\"") {
+			inQuotes = true;
+			continue;
+		}
+		if (ch === delimiter) {
+			current.push(cell);
+			cell = "";
+			continue;
+		}
+		if (ch === "\n" || ch === "\r") {
+			if (ch === "\r" && text[i + 1] === "\n") i += 1;
+			current.push(cell);
+			rows.push(current);
+			current = [];
+			cell = "";
+			continue;
+		}
+		cell += ch;
+	}
+	if (cell.length > 0 || current.length > 0) {
+		current.push(cell);
+		rows.push(current);
+	}
+	return rows;
+}
+
+//#endregion
+export { loadCsvDataset, parseCsv };
+//# sourceMappingURL=csv.js.map

package/dist/loaders/csv.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv.js","names":["out: Case<unknown, unknown>[]","record: Record<string, string>","sampleCase: Case<unknown, unknown>","rows: string[][]","current: string[]"],"sources":["../../src/loaders/csv.ts"],"sourcesContent":["/**\n * CSV dataset loader. Implements RFC 4180 strict subset:\n *\n *  - Comma separator (configurable).\n *  - Optional `\"`-quoted cells; doubled `\"\"` to embed a literal `\"`.\n *  - Header row required by default.\n *\n * Columns map to `Case` fields by name: `input`, `expected`, `id`,\n * `metadata` (parsed as JSON when present). Unknown columns are\n * ignored unless a `mapper` is supplied.\n *\n * @packageDocumentation\n */\n\nimport { readFile } from 'node:fs/promises';\n\nimport type { Case, Dataset } from '@graphorin/observability/eval';\n\n/** @stable */\nexport interface LoadCsvOptions {\n  readonly delimiter?: string;\n  readonly name?: string;\n  readonly description?: string;\n  readonly mapper?: (row: Record<string, string>, index: number) => Case<unknown, unknown>;\n}\n\n/** @stable */\nexport async function loadCsvDataset(\n  path: string,\n  options: LoadCsvOptions = {},\n): Promise<Dataset<unknown, unknown>> {\n  const text = await readFile(path, 'utf8');\n  const cases = parseCsv(text, options);\n  const meta: Dataset<unknown, unknown>['metadata'] = {\n    ...(options.name !== undefined ? { name: options.name } : {}),\n    ...(options.description !== undefined ? { description: options.description } : {}),\n    createdAt: new Date(),\n  };\n  return { cases, metadata: meta };\n}\n\n/**\n * Pure parser. Exported separately so tests can exercise the\n * column-mapping behaviour without touching the filesystem.\n *\n * @stable\n */\nexport function parseCsv(\n  text: string,\n  options: LoadCsvOptions = {},\n): ReadonlyArray<Case<unknown, unknown>> {\n  const delimiter = options.delimiter ?? ',';\n  const rows = parseCsvRows(text, delimiter);\n  if (rows.length === 0) return [];\n  const header = rows[0];\n  if (header === undefined) return [];\n  const dataRows = rows.slice(1);\n  const out: Case<unknown, unknown>[] = [];\n  for (let i = 0; i < dataRows.length; i++) {\n    const row = dataRows[i] ?? [];\n    if (row.every((cell) => cell.length === 0)) continue;\n    const record: Record<string, string> = {};\n    for (let c = 0; c < header.length; c++) {\n      const key = header[c] ?? '';\n      const value = row[c] ?? '';\n      record[key] = value;\n    }\n    if (options.mapper !== undefined) {\n      out.push(options.mapper(record, i));\n      continue;\n    }\n    if (record.input === undefined) {\n      throw new Error(`[graphorin/evals] CSV row ${i + 1} missing required 'input' column.`);\n    }\n    const sampleCase: Case<unknown, unknown> = {\n      ...(record.id !== undefined && record.id.length > 0 ? { id: record.id } : {}),\n      input: record.input,\n      ...('expected' in record && record.expected !== undefined && record.expected.length > 0\n        ? { expected: record.expected }\n        : {}),\n      ...(record.metadata !== undefined && record.metadata.length > 0\n        ? { metadata: parseMetadata(record.metadata) }\n        : {}),\n    };\n    out.push(sampleCase);\n  }\n  return out;\n}\n\nfunction parseMetadata(raw: string): Readonly<Record<string, unknown>> {\n  try {\n    const parsed = JSON.parse(raw);\n    if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) {\n      return parsed as Readonly<Record<string, unknown>>;\n    }\n    return { value: parsed } as Readonly<Record<string, unknown>>;\n  } catch {\n    return { raw } as Readonly<Record<string, unknown>>;\n  }\n}\n\nfunction parseCsvRows(text: string, delimiter: string): string[][] {\n  const rows: string[][] = [];\n  let current: string[] = [];\n  let cell = '';\n  let inQuotes = false;\n  for (let i = 0; i < text.length; i++) {\n    const ch = text[i];\n    if (inQuotes) {\n      if (ch === '\"') {\n        const next = text[i + 1];\n        if (next === '\"') {\n          cell += '\"';\n          i += 1;\n        } else {\n          inQuotes = false;\n        }\n      } else {\n        cell += ch;\n      }\n      continue;\n    }\n    if (ch === '\"') {\n      inQuotes = true;\n      continue;\n    }\n    if (ch === delimiter) {\n      current.push(cell);\n      cell = '';\n      continue;\n    }\n    if (ch === '\\n' || ch === '\\r') {\n      if (ch === '\\r' && text[i + 1] === '\\n') i += 1;\n      current.push(cell);\n      rows.push(current);\n      current = [];\n      cell = '';\n      continue;\n    }\n    cell += ch;\n  }\n  if (cell.length > 0 || current.length > 0) {\n    current.push(cell);\n    rows.push(current);\n  }\n  return rows;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AA2BA,eAAsB,eACpB,MACA,UAA0B,EAAE,EACQ;AAQpC,QAAO;EAAE,OANK,SADD,MAAM,SAAS,MAAM,OAAO,EACZ,QAAQ;EAMrB,UALoC;GAClD,GAAI,QAAQ,SAAS,SAAY,EAAE,MAAM,QAAQ,MAAM,GAAG,EAAE;GAC5D,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;GACjF,2BAAW,IAAI,MAAM;GACtB;EAC+B;;;;;;;;AASlC,SAAgB,SACd,MACA,UAA0B,EAAE,EACW;CAEvC,MAAM,OAAO,aAAa,MADR,QAAQ,aAAa,IACG;AAC1C,KAAI,KAAK,WAAW,EAAG,QAAO,EAAE;CAChC,MAAM,SAAS,KAAK;AACpB,KAAI,WAAW,OAAW,QAAO,EAAE;CACnC,MAAM,WAAW,KAAK,MAAM,EAAE;CAC9B,MAAMA,MAAgC,EAAE;AACxC,MAAK,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;EACxC,MAAM,MAAM,SAAS,MAAM,EAAE;AAC7B,MAAI,IAAI,OAAO,SAAS,KAAK,WAAW,EAAE,CAAE;EAC5C,MAAMC,SAAiC,EAAE;AACzC,OAAK,IAAI,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;GACtC,MAAM,MAAM,OAAO,MAAM;AAEzB,UAAO,OADO,IAAI,MAAM;;AAG1B,MAAI,QAAQ,WAAW,QAAW;AAChC,OAAI,KAAK,QAAQ,OAAO,QAAQ,EAAE,CAAC;AACnC;;AAEF,MAAI,OAAO,UAAU,OACnB,OAAM,IAAI,MAAM,6BAA6B,IAAI,EAAE,mCAAmC;EAExF,MAAMC,aAAqC;GACzC,GAAI,OAAO,OAAO,UAAa,OAAO,GAAG,SAAS,IAAI,EAAE,IAAI,OAAO,IAAI,GAAG,EAAE;GAC5E,OAAO,OAAO;GACd,GAAI,cAAc,UAAU,OAAO,aAAa,UAAa,OAAO,SAAS,SAAS,IAClF,EAAE,UAAU,OAAO,UAAU,GAC7B,EAAE;GACN,GAAI,OAAO,aAAa,UAAa,OAAO,SAAS,SAAS,IAC1D,EAAE,UAAU,cAAc,OAAO,SAAS,EAAE,GAC5C,EAAE;GACP;AACD,MAAI,KAAK,WAAW;;AAEtB,QAAO;;AAGT,SAAS,cAAc,KAAgD;AACrE,KAAI;EACF,MAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,MAAI,WAAW,QAAQ,OAAO,WAAW,YAAY,CAAC,MAAM,QAAQ,OAAO,CACzE,QAAO;AAET,SAAO,EAAE,OAAO,QAAQ;SAClB;AACN,SAAO,EAAE,KAAK;;;AAIlB,SAAS,aAAa,MAAc,WAA+B;CACjE,MAAMC,OAAmB,EAAE;CAC3B,IAAIC,UAAoB,EAAE;CAC1B,IAAI,OAAO;CACX,IAAI,WAAW;AACf,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,KAAK,KAAK;AAChB,MAAI,UAAU;AACZ,OAAI,OAAO,KAET,KADa,KAAK,IAAI,OACT,MAAK;AAChB,YAAQ;AACR,SAAK;SAEL,YAAW;OAGb,SAAQ;AAEV;;AAEF,MAAI,OAAO,MAAK;AACd,cAAW;AACX;;AAEF,MAAI,OAAO,WAAW;AACpB,WAAQ,KAAK,KAAK;AAClB,UAAO;AACP;;AAEF,MAAI,OAAO,QAAQ,OAAO,MAAM;AAC9B,OAAI,OAAO,QAAQ,KAAK,IAAI,OAAO,KAAM,MAAK;AAC9C,WAAQ,KAAK,KAAK;AAClB,QAAK,KAAK,QAAQ;AAClB,aAAU,EAAE;AACZ,UAAO;AACP;;AAEF,UAAQ;;AAEV,KAAI,KAAK,SAAS,KAAK,QAAQ,SAAS,GAAG;AACzC,UAAQ,KAAK,KAAK;AAClB,OAAK,KAAK,QAAQ;;AAEpB,QAAO"}