@warlock.js/ai 4.3.0 → 4.5.0

package/esm/eval/dataset.mjs

@@ -0,0 +1,112 @@
+import { InvalidRequestError } from "../errors/invalid-request-error.mjs";
+import "../errors/index.mjs";
+import { readFileSync } from "node:fs";
+
+//#region ../@warlock.js/ai/src/eval/dataset.ts
+/**
+* Parse a JSONL file's contents into {@link DatasetEntry} rows. Blank
+* lines (and trailing whitespace-only lines) are skipped; every other
+* line must be a JSON object. A malformed line throws an
+* `InvalidRequestError` naming the 1-based line number — failing loud at
+* construction, like `SystemPrompt.fromFile`.
+*/
+function parseJsonl(path, contents) {
+	const entries = [];
+	const lines = contents.split(/\r?\n/);
+	for (let index = 0; index < lines.length; index++) {
+		const line = lines[index].trim();
+		if (line === "") continue;
+		let parsed;
+		try {
+			parsed = JSON.parse(line);
+		} catch (error) {
+			throw new InvalidRequestError(`Failed to parse dataset file "${path}" — line ${index + 1} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`, {
+				context: {
+					path,
+					line: index + 1
+				},
+				cause: error
+			});
+		}
+		if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) throw new InvalidRequestError(`Failed to parse dataset file "${path}" — line ${index + 1} is not a JSON object`, { context: {
+			path,
+			line: index + 1
+		} });
+		entries.push(parsed);
+	}
+	return entries;
+}
+/**
+* Read a JSONL dataset file once, synchronously, at construction.
+* Mirrors `SystemPrompt.fromFile`: a read failure (missing path,
+* permission denied) throws an `InvalidRequestError` surfacing the
+* underlying cause.
+*/
+function readDatasetFile(path) {
+	let contents;
+	try {
+		contents = readFileSync(path, "utf8");
+	} catch (error) {
+		throw new InvalidRequestError(`Failed to read dataset file "${path}" — ${error instanceof Error ? error.message : String(error)}`, {
+			context: { path },
+			cause: error
+		});
+	}
+	return parseJsonl(path, contents);
+}
+/**
+* Build an immutable {@link DatasetContract} over the given entries.
+* Shared by the {@link dataset} factory and by `filter` / `shard`, which
+* each return a fresh dataset built from a derived case list.
+*/
+function makeDataset(name, cases) {
+	return {
+		name,
+		cases,
+		filter(predicate) {
+			return makeDataset(name, cases.filter(predicate));
+		},
+		shard(index, total) {
+			if (!Number.isInteger(total) || total <= 0) throw new InvalidRequestError(`dataset.shard: "total" must be a positive integer, received ${total}`, { context: {
+				name,
+				total
+			} });
+			if (!Number.isInteger(index) || index < 0 || index >= total) throw new InvalidRequestError(`dataset.shard: "index" must be an integer in [0, ${total}), received ${index}`, { context: {
+				name,
+				index,
+				total
+			} });
+			return makeDataset(name, cases.filter((_, position) => position % total === index));
+		}
+	};
+}
+/**
+* Create an immutable evaluation dataset that feeds `agent.eval({ cases })`
+* directly.
+*
+* **Role.** A taggable, filterable, shardable wrapper around a list of
+* {@link DatasetEntry} rows. `agent.eval` accepts a `DatasetContract` in
+* place of a raw `EvalCase[]`, reading `.cases` off it.
+*
+* Sources (combinable — file entries append after inline `cases`):
+* - `cases` → inline entries.
+* - `fromFile` → a JSONL file read once, synchronously, at construction
+*   (one JSON object per line). A malformed line throws an
+*   `InvalidRequestError` naming the 1-based line number.
+*
+* @example
+* const ds = dataset({ name: "support", fromFile: "./eval/support.jsonl" });
+* const smoke = ds.filter((entry) => entry.tags?.includes("smoke"));
+* const shard = ds.shard(0, 4); // first of four parallel CI shards
+*
+* const report = await agent.eval({ cases: ds, scorers: [contains()] });
+*/
+function dataset(options) {
+	const cases = [...options.cases ?? []];
+	if (options.fromFile !== void 0) cases.push(...readDatasetFile(options.fromFile));
+	return makeDataset(options.name, cases);
+}
+
+//#endregion
+export { dataset };
+//# sourceMappingURL=dataset.mjs.map

package/esm/eval/dataset.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/dataset.ts"],"sourcesContent":["import { readFileSync } from \"node:fs\";\nimport type { DatasetContract, DatasetEntry, DatasetOptions } from \"./dataset.type\";\nimport { InvalidRequestError } from \"../errors\";\n\n/**\n * Parse a JSONL file's contents into {@link DatasetEntry} rows. Blank\n * lines (and trailing whitespace-only lines) are skipped; every other\n * line must be a JSON object. A malformed line throws an\n * `InvalidRequestError` naming the 1-based line number — failing loud at\n * construction, like `SystemPrompt.fromFile`.\n */\nfunction parseJsonl<TOutput>(path: string, contents: string): DatasetEntry<TOutput>[] {\n  const entries: DatasetEntry<TOutput>[] = [];\n  const lines = contents.split(/\\r?\\n/);\n\n  for (let index = 0; index < lines.length; index++) {\n    const line = lines[index].trim();\n\n    if (line === \"\") {\n      continue;\n    }\n\n    let parsed: unknown;\n\n    try {\n      parsed = JSON.parse(line);\n    } catch (error) {\n      throw new InvalidRequestError(\n        `Failed to parse dataset file \"${path}\" — line ${index + 1} is not valid JSON: ${\n          error instanceof Error ? error.message : String(error)\n        }`,\n        { context: { path, line: index + 1 }, cause: error },\n      );\n    }\n\n    if (parsed === null || typeof parsed !== \"object\" || Array.isArray(parsed)) {\n      throw new InvalidRequestError(\n        `Failed to parse dataset file \"${path}\" — line ${index + 1} is not a JSON object`,\n        { context: { path, line: index + 1 } },\n      );\n    }\n\n    entries.push(parsed as DatasetEntry<TOutput>);\n  }\n\n  return entries;\n}\n\n/**\n * Read a JSONL dataset file once, synchronously, at construction.\n * Mirrors `SystemPrompt.fromFile`: a read failure (missing path,\n * permission denied) throws an `InvalidRequestError` surfacing the\n * underlying cause.\n */\nfunction readDatasetFile<TOutput>(path: string): DatasetEntry<TOutput>[] {\n  let contents: string;\n\n  try {\n    contents = readFileSync(path, \"utf8\");\n  } catch (error) {\n    throw new InvalidRequestError(\n      `Failed to read dataset file \"${path}\" — ${\n        error instanceof Error ? error.message : String(error)\n      }`,\n      { context: { path }, cause: error },\n    );\n  }\n\n  return parseJsonl<TOutput>(path, contents);\n}\n\n/**\n * Build an immutable {@link DatasetContract} over the given entries.\n * Shared by the {@link dataset} factory and by `filter` / `shard`, which\n * each return a fresh dataset built from a derived case list.\n */\nfunction makeDataset<TOutput>(\n  name: string,\n  cases: DatasetEntry<TOutput>[],\n): DatasetContract<TOutput> {\n  return {\n    name,\n    cases,\n    filter(predicate) {\n      return makeDataset(name, cases.filter(predicate));\n    },\n    shard(index, total) {\n      if (!Number.isInteger(total) || total <= 0) {\n        throw new InvalidRequestError(\n          `dataset.shard: \"total\" must be a positive integer, received ${total}`,\n          { context: { name, total } },\n        );\n      }\n\n      if (!Number.isInteger(index) || index < 0 || index >= total) {\n        throw new InvalidRequestError(\n          `dataset.shard: \"index\" must be an integer in [0, ${total}), received ${index}`,\n          { context: { name, index, total } },\n        );\n      }\n\n      return makeDataset(\n        name,\n        cases.filter((_, position) => position % total === index),\n      );\n    },\n  };\n}\n\n/**\n * Create an immutable evaluation dataset that feeds `agent.eval({ cases })`\n * directly.\n *\n * **Role.** A taggable, filterable, shardable wrapper around a list of\n * {@link DatasetEntry} rows. `agent.eval` accepts a `DatasetContract` in\n * place of a raw `EvalCase[]`, reading `.cases` off it.\n *\n * Sources (combinable — file entries append after inline `cases`):\n * - `cases` → inline entries.\n * - `fromFile` → a JSONL file read once, synchronously, at construction\n *   (one JSON object per line). A malformed line throws an\n *   `InvalidRequestError` naming the 1-based line number.\n *\n * @example\n * const ds = dataset({ name: \"support\", fromFile: \"./eval/support.jsonl\" });\n * const smoke = ds.filter((entry) => entry.tags?.includes(\"smoke\"));\n * const shard = ds.shard(0, 4); // first of four parallel CI shards\n *\n * const report = await agent.eval({ cases: ds, scorers: [contains()] });\n */\nexport function dataset<TOutput = unknown>(\n  options: DatasetOptions<TOutput>,\n): DatasetContract<TOutput> {\n  const cases: DatasetEntry<TOutput>[] = [...(options.cases ?? [])];\n\n  if (options.fromFile !== undefined) {\n    cases.push(...readDatasetFile<TOutput>(options.fromFile));\n  }\n\n  return makeDataset(options.name, cases);\n}\n"],"mappings":";;;;;;;;;;;;AAWA,SAAS,WAAoB,MAAc,UAA2C;CACpF,MAAM,UAAmC,CAAC;CAC1C,MAAM,QAAQ,SAAS,MAAM,OAAO;CAEpC,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,QAAQ,SAAS;EACjD,MAAM,OAAO,MAAM,MAAM,CAAC,KAAK;EAE/B,IAAI,SAAS,IACX;EAGF,IAAI;EAEJ,IAAI;GACF,SAAS,KAAK,MAAM,IAAI;EAC1B,SAAS,OAAO;GACd,MAAM,IAAI,oBACR,iCAAiC,KAAK,WAAW,QAAQ,EAAE,sBACzD,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,KAEvD;IAAE,SAAS;KAAE;KAAM,MAAM,QAAQ;IAAE;IAAG,OAAO;GAAM,CACrD;EACF;EAEA,IAAI,WAAW,QAAQ,OAAO,WAAW,YAAY,MAAM,QAAQ,MAAM,GACvE,MAAM,IAAI,oBACR,iCAAiC,KAAK,WAAW,QAAQ,EAAE,wBAC3D,EAAE,SAAS;GAAE;GAAM,MAAM,QAAQ;EAAE,EAAE,CACvC;EAGF,QAAQ,KAAK,MAA+B;CAC9C;CAEA,OAAO;AACT;;;;;;;AAQA,SAAS,gBAAyB,MAAuC;CACvE,IAAI;CAEJ,IAAI;EACF,WAAW,aAAa,MAAM,MAAM;CACtC,SAAS,OAAO;EACd,MAAM,IAAI,oBACR,gCAAgC,KAAK,MACnC,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,KAEvD;GAAE,SAAS,EAAE,KAAK;GAAG,OAAO;EAAM,CACpC;CACF;CAEA,OAAO,WAAoB,MAAM,QAAQ;AAC3C;;;;;;AAOA,SAAS,YACP,MACA,OAC0B;CAC1B,OAAO;EACL;EACA;EACA,OAAO,WAAW;GAChB,OAAO,YAAY,MAAM,MAAM,OAAO,SAAS,CAAC;EAClD;EACA,MAAM,OAAO,OAAO;GAClB,IAAI,CAAC,OAAO,UAAU,KAAK,KAAK,SAAS,GACvC,MAAM,IAAI,oBACR,+DAA+D,SAC/D,EAAE,SAAS;IAAE;IAAM;GAAM,EAAE,CAC7B;GAGF,IAAI,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,KAAK,SAAS,OACpD,MAAM,IAAI,oBACR,oDAAoD,MAAM,cAAc,SACxE,EAAE,SAAS;IAAE;IAAM;IAAO;GAAM,EAAE,CACpC;GAGF,OAAO,YACL,MACA,MAAM,QAAQ,GAAG,aAAa,WAAW,UAAU,KAAK,CAC1D;EACF;CACF;AACF;;;;;;;;;;;;;;;;;;;;;;AAuBA,SAAgB,QACd,SAC0B;CAC1B,MAAM,QAAiC,CAAC,GAAI,QAAQ,SAAS,CAAC,CAAE;CAEhE,IAAI,QAAQ,aAAa,QACvB,MAAM,KAAK,GAAG,gBAAyB,QAAQ,QAAQ,CAAC;CAG1D,OAAO,YAAY,QAAQ,MAAM,KAAK;AACxC"}

package/esm/eval/dataset.type.d.mts

@@ -0,0 +1,53 @@
+import { EvalCase } from "../contracts/agent/eval.type.mjs";
+
+//#region ../@warlock.js/ai/src/eval/dataset.type.d.ts
+/**
+ * A single dataset row. A superset of {@link EvalCase} — every dataset
+ * entry is a valid eval case, plus optional `tags` for filtering and
+ * sharding a large suite.
+ */
+type DatasetEntry<TOutput = unknown> = EvalCase<TOutput> & {
+  /** Optional tags for filtering / sharding a large dataset. */tags?: string[];
+};
+/**
+ * An immutable, filterable, shardable collection of {@link DatasetEntry}
+ * rows that feeds `agent.eval({ cases })` directly. Built via the
+ * {@link dataset} factory — never `new`.
+ */
+type DatasetContract<TOutput = unknown> = {
+  /** Stable identifier for the dataset — surfaced for logging / artifacts. */readonly name: string; /** All entries (after any filter / shard applied at construction). */
+  readonly cases: DatasetEntry<TOutput>[];
+  /**
+   * Narrow by an arbitrary predicate (typically a tag check) — returns a
+   * new dataset sharing nothing mutable with this one.
+   */
+  filter(predicate: (entry: DatasetEntry<TOutput>) => boolean): DatasetContract<TOutput>;
+  /**
+   * Deterministic shard `index`-of-`total` for parallel CI jobs. Every
+   * entry lands in exactly one shard (round-robin by position), so the
+   * union of all `total` shards reproduces the full case list with no
+   * gaps or overlaps.
+   */
+  shard(index: number, total: number): DatasetContract<TOutput>;
+};
+/**
+ * Options for the {@link dataset} factory. Supply `cases` inline, or
+ * `fromFile` to load a JSONL file (one JSON-encoded {@link DatasetEntry}
+ * per line) synchronously at construction — mirroring
+ * `SystemPrompt.fromFile`'s one-shot sync read. Both may be combined;
+ * file entries are appended after inline `cases`.
+ */
+type DatasetOptions<TOutput = unknown> = {
+  /** Stable identifier for the dataset. */name: string; /** Inline entries. */
+  cases?: DatasetEntry<TOutput>[];
+  /**
+   * Path to a JSONL file read once, synchronously, at construction. Each
+   * non-blank line must be a JSON object matching {@link DatasetEntry}; a
+   * malformed line throws an `InvalidRequestError` naming the 1-based line
+   * number.
+   */
+  fromFile?: string;
+};
+//#endregion
+export { DatasetContract, DatasetEntry, DatasetOptions };
+//# sourceMappingURL=dataset.type.d.mts.map

package/esm/eval/dataset.type.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.type.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/dataset.type.ts"],"mappings":";;;;;AAOA;;;KAAY,YAAA,sBAAkC,QAAQ,CAAC,OAAA;EAA9B,8DAEvB,IAAA;AAAA;;;AAAI;AAQN;;KAAY,eAAA;EAImB,qFAFpB,IAAA,UAQyB;EAAA,SANzB,KAAA,EAAO,YAAA,CAAa,OAAA;EAOV;;;;EAFnB,MAAA,CACE,SAAA,GAAY,KAAA,EAAO,YAAA,CAAa,OAAA,gBAC/B,eAAA,CAAgB,OAAA;EAOiC;;;;;;EAApD,KAAA,CAAM,KAAA,UAAe,KAAA,WAAgB,eAAA,CAAgB,OAAA;AAAA;;;;;;;;KAU3C,cAAA;EAV2B,yCAYrC,IAAA,UAZ4D;EAc5D,KAAA,GAAQ,YAAY,CAAC,OAAA;EAJX;;;;;;EAWV,QAAA;AAAA"}

package/esm/eval/eval-runner.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-runner.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/eval-runner.ts"],"mappings":";;;;;;AAmIA;;;;;;;iBAAsB,OAAA,UACpB,KAAA,EAAO,aAAA,CAAc,OAAA,GACrB,OAAA,EAAS,WAAA,CAAY,OAAA,IACpB,OAAA,CAAQ,UAAA,CAAW,OAAA"}
+{"version":3,"file":"eval-runner.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/eval-runner.ts"],"mappings":";;;;;;AAqJA;;;;;;;iBAAsB,OAAA,UACpB,KAAA,EAAO,aAAA,CAAc,OAAA,GACrB,OAAA,EAAS,WAAA,CAAY,OAAA,IACpB,OAAA,CAAQ,UAAA,CAAW,OAAA"}

package/esm/eval/eval-runner.mjs

@@ -1,9 +1,20 @@
 import { AgentExecutionError } from "../errors/agent-execution-error.mjs";
 import "../errors/index.mjs";
 import { judge } from "./judge-scorer.mjs";
+import { diff } from "./regression.mjs";
 import { log } from "@warlock.js/logger";
 
 //#region ../@warlock.js/ai/src/eval/eval-runner.ts
+/**
+* Narrow `EvalOptions.cases` to the underlying `EvalCase[]`. A
+* `DatasetContract` is identified structurally by its `cases` property
+* (an array carried alongside `name` / `filter` / `shard`); a raw
+* `EvalCase[]` is used as-is.
+*/
+function resolveCases(cases) {
+	if (Array.isArray(cases)) return cases;
+	return cases.cases;
+}
 const LOG_MODULE = "ai.eval";
 const DEFAULT_PASS_THRESHOLD = .5;
 /**
@@ -86,8 +97,9 @@ async function runCase(agent, evalCase, options, passThreshold) {
 async function runEval(agent, options) {
 	const passThreshold = options.passThreshold ?? DEFAULT_PASS_THRESHOLD;
 	const start = performance.now();
+	const suiteCases = resolveCases(options.cases);
 	const cases = [];
-	for (const evalCase of options.cases) {
+	for (const evalCase of suiteCases) {
 		const caseResult = await runCase(agent, evalCase, options, passThreshold);
 		cases.push(caseResult);
 		if (!caseResult.passed && options.onFailure) try {
@@ -103,7 +115,7 @@ async function runEval(agent, options) {
 	const passedCount = cases.filter((entry) => entry.passed).length;
 	const total = cases.length;
 	const meanScore = total > 0 ? cases.reduce((sum, entry) => sum + entry.score, 0) / total : 0;
-	return {
+	const report = {
 		agentName: agent.name,
 		total,
 		passedCount,
@@ -114,6 +126,8 @@ async function runEval(agent, options) {
 		cases,
 		duration: performance.now() - start
 	};
+	if (options.baseline) report.regression = diff(report, options.baseline, options.tolerance);
+	return report;
 }
 
 //#endregion

package/esm/eval/eval-runner.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"eval-runner.mjs","names":["judgeScorer"],"sources":["../../../../../../../@warlock.js/ai/src/eval/eval-runner.ts"],"sourcesContent":["import type { AgentContract } from \"../contracts/agent/agent.contract\";\nimport type { AgentExecuteOptions } from \"../contracts/agent/agent-options.type\";\nimport type {\n  EvalCase,\n  EvalCaseResult,\n  EvalOptions,\n  EvalReport,\n  EvalScore,\n  EvalScorer,\n  EvalScorerContext,\n} from \"../contracts/agent/eval.type\";\nimport { AgentExecutionError } from \"../errors\";\nimport { log } from \"@warlock.js/logger\";\nimport { judge as judgeScorer } from \"./judge-scorer\";\n\nconst LOG_MODULE = \"ai.eval\";\nconst DEFAULT_PASS_THRESHOLD = 0.5;\n\n/**\n * Resolve the scorer list for a single case. Precedence: the case's\n * own `scorers` → the suite `scorers` → a synthesized judge scorer\n * when `judge` is configured. Throws an authoring-time\n * `AgentExecutionError` when a case can resolve none — an eval suite\n * with no way to score a case is a config bug worth surfacing at the\n * call site, not a silent pass.\n */\nfunction resolveScorers<TOutput>(\n  evalCase: EvalCase<TOutput>,\n  options: EvalOptions<TOutput>,\n  passThreshold: number,\n): EvalScorer<TOutput>[] {\n  if (evalCase.scorers && evalCase.scorers.length > 0) {\n    return evalCase.scorers;\n  }\n\n  if (options.scorers && options.scorers.length > 0) {\n    return options.scorers;\n  }\n\n  if (options.judge) {\n    return [judgeScorer<TOutput>(options.judge, passThreshold)];\n  }\n\n  throw new AgentExecutionError(\n    `eval case \"${evalCase.name}\" has no scorer — supply per-case \"scorers\", suite \"scorers\", or a \"judge\"`,\n    { context: { authoring: true, case: evalCase.name } },\n  );\n}\n\n/**\n * Decide a single scorer verdict's pass/fail. Honors an explicit\n * `passed` from the scorer; otherwise derives it from\n * `score >= passThreshold`.\n */\nfunction isScorePassing(score: EvalScore, passThreshold: number): boolean {\n  if (typeof score.passed === \"boolean\") {\n    return score.passed;\n  }\n\n  return score.score >= passThreshold;\n}\n\n/**\n * Merge suite-level execute options with the case's own override.\n * Per-case wins on conflict (shallow merge).\n */\nfunction mergeOptions<TOutput>(\n  suite: AgentExecuteOptions<TOutput> | undefined,\n  perCase: AgentExecuteOptions<TOutput> | undefined,\n): AgentExecuteOptions<TOutput> | undefined {\n  if (!suite) return perCase;\n  if (!perCase) return suite;\n  return { ...suite, ...perCase };\n}\n\n/**\n * Run one case end-to-end: execute the agent, run every resolved\n * scorer, aggregate into an {@link EvalCaseResult}. A case passes only\n * when the agent did not error AND every scorer passed.\n */\nasync function runCase<TOutput>(\n  agent: AgentContract<TOutput>,\n  evalCase: EvalCase<TOutput>,\n  options: EvalOptions<TOutput>,\n  passThreshold: number,\n): Promise<EvalCaseResult<TOutput>> {\n  const scorers = resolveScorers(evalCase, options, passThreshold);\n  const executeOptions = mergeOptions(options.executeOptions, evalCase.options);\n\n  const start = performance.now();\n  const result = await agent.execute(evalCase.input, executeOptions);\n  const duration = performance.now() - start;\n\n  const context: EvalScorerContext<TOutput> = {\n    case: evalCase,\n    result,\n    output: result.data,\n    text: result.text,\n  };\n\n  const scores: EvalScore[] = [];\n\n  for (const scorer of scorers) {\n    scores.push(await scorer(context));\n  }\n\n  const meanScore =\n    scores.length > 0 ? scores.reduce((sum, score) => sum + score.score, 0) / scores.length : 0;\n\n  const allScorersPassed = scores.every((score) => isScorePassing(score, passThreshold));\n  const passed = result.error === undefined && allScorersPassed;\n\n  return {\n    case: evalCase,\n    result,\n    scores,\n    score: meanScore,\n    passed,\n    duration,\n  };\n}\n\n/**\n * Core implementation of `agent.eval`. Runs every case sequentially\n * (cases share the agent and may carry side effects — ordering must be\n * deterministic), scores each, fires `onFailure` for failed cases, and\n * assembles the aggregate {@link EvalReport}.\n *\n * Never throws on a case-level failure; the only throw is the\n * authoring-time \"no scorer\" guard from {@link resolveScorers}.\n */\nexport async function runEval<TOutput>(\n  agent: AgentContract<TOutput>,\n  options: EvalOptions<TOutput>,\n): Promise<EvalReport<TOutput>> {\n  const passThreshold = options.passThreshold ?? DEFAULT_PASS_THRESHOLD;\n  const start = performance.now();\n\n  const cases: EvalCaseResult<TOutput>[] = [];\n\n  for (const evalCase of options.cases) {\n    const caseResult = await runCase(agent, evalCase, options, passThreshold);\n\n    cases.push(caseResult);\n\n    if (!caseResult.passed && options.onFailure) {\n      try {\n        await options.onFailure(caseResult);\n      } catch (error) {\n        log.warn(LOG_MODULE, \"onFailure.hook.error\", \"eval onFailure handler threw\", {\n          agent: agent.name,\n          case: evalCase.name,\n          error: error instanceof Error ? error.message : String(error),\n        });\n      }\n    }\n  }\n\n  const passedCount = cases.filter((entry) => entry.passed).length;\n  const total = cases.length;\n  const meanScore =\n    total > 0 ? cases.reduce((sum, entry) => sum + entry.score, 0) / total : 0;\n\n  return {\n    agentName: agent.name,\n    total,\n    passedCount,\n    failedCount: total - passedCount,\n    passRate: total > 0 ? passedCount / total : 0,\n    meanScore,\n    passed: total > 0 && passedCount === total,\n    cases,\n    duration: performance.now() - start,\n  };\n}\n"],"mappings":";;;;;;AAeA,MAAM,aAAa;AACnB,MAAM,yBAAyB;;;;;;;;;AAU/B,SAAS,eACP,UACA,SACA,eACuB;CACvB,IAAI,SAAS,WAAW,SAAS,QAAQ,SAAS,GAChD,OAAO,SAAS;CAGlB,IAAI,QAAQ,WAAW,QAAQ,QAAQ,SAAS,GAC9C,OAAO,QAAQ;CAGjB,IAAI,QAAQ,OACV,OAAO,CAACA,MAAqB,QAAQ,OAAO,aAAa,CAAC;CAG5D,MAAM,IAAI,oBACR,cAAc,SAAS,KAAK,6EAC5B,EAAE,SAAS;EAAE,WAAW;EAAM,MAAM,SAAS;CAAK,EAAE,CACtD;AACF;;;;;;AAOA,SAAS,eAAe,OAAkB,eAAgC;CACxE,IAAI,OAAO,MAAM,WAAW,WAC1B,OAAO,MAAM;CAGf,OAAO,MAAM,SAAS;AACxB;;;;;AAMA,SAAS,aACP,OACA,SAC0C;CAC1C,IAAI,CAAC,OAAO,OAAO;CACnB,IAAI,CAAC,SAAS,OAAO;CACrB,OAAO;EAAE,GAAG;EAAO,GAAG;CAAQ;AAChC;;;;;;AAOA,eAAe,QACb,OACA,UACA,SACA,eACkC;CAClC,MAAM,UAAU,eAAe,UAAU,SAAS,aAAa;CAC/D,MAAM,iBAAiB,aAAa,QAAQ,gBAAgB,SAAS,OAAO;CAE5E,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,SAAS,MAAM,MAAM,QAAQ,SAAS,OAAO,cAAc;CACjE,MAAM,WAAW,YAAY,IAAI,IAAI;CAErC,MAAM,UAAsC;EAC1C,MAAM;EACN;EACA,QAAQ,OAAO;EACf,MAAM,OAAO;CACf;CAEA,MAAM,SAAsB,CAAC;CAE7B,KAAK,MAAM,UAAU,SACnB,OAAO,KAAK,MAAM,OAAO,OAAO,CAAC;CAGnC,MAAM,YACJ,OAAO,SAAS,IAAI,OAAO,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,CAAC,IAAI,OAAO,SAAS;CAE5F,MAAM,mBAAmB,OAAO,OAAO,UAAU,eAAe,OAAO,aAAa,CAAC;CAGrF,OAAO;EACL,MAAM;EACN;EACA;EACA,OAAO;EACP,QAPa,OAAO,UAAU,UAAa;EAQ3C;CACF;AACF;;;;;;;;;;AAWA,eAAsB,QACpB,OACA,SAC8B;CAC9B,MAAM,gBAAgB,QAAQ,iBAAiB;CAC/C,MAAM,QAAQ,YAAY,IAAI;CAE9B,MAAM,QAAmC,CAAC;CAE1C,KAAK,MAAM,YAAY,QAAQ,OAAO;EACpC,MAAM,aAAa,MAAM,QAAQ,OAAO,UAAU,SAAS,aAAa;EAExE,MAAM,KAAK,UAAU;EAErB,IAAI,CAAC,WAAW,UAAU,QAAQ,WAChC,IAAI;GACF,MAAM,QAAQ,UAAU,UAAU;EACpC,SAAS,OAAO;GACd,IAAI,KAAK,YAAY,wBAAwB,gCAAgC;IAC3E,OAAO,MAAM;IACb,MAAM,SAAS;IACf,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D,CAAC;EACH;CAEJ;CAEA,MAAM,cAAc,MAAM,QAAQ,UAAU,MAAM,MAAM,CAAC,CAAC;CAC1D,MAAM,QAAQ,MAAM;CACpB,MAAM,YACJ,QAAQ,IAAI,MAAM,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,CAAC,IAAI,QAAQ;CAE3E,OAAO;EACL,WAAW,MAAM;EACjB;EACA;EACA,aAAa,QAAQ;EACrB,UAAU,QAAQ,IAAI,cAAc,QAAQ;EAC5C;EACA,QAAQ,QAAQ,KAAK,gBAAgB;EACrC;EACA,UAAU,YAAY,IAAI,IAAI;CAChC;AACF"}
+{"version":3,"file":"eval-runner.mjs","names":["judgeScorer"],"sources":["../../../../../../../@warlock.js/ai/src/eval/eval-runner.ts"],"sourcesContent":["import type { AgentContract } from \"../contracts/agent/agent.contract\";\nimport type { AgentExecuteOptions } from \"../contracts/agent/agent-options.type\";\nimport type {\n  EvalCase,\n  EvalCaseResult,\n  EvalOptions,\n  EvalReport,\n  EvalScore,\n  EvalScorer,\n  EvalScorerContext,\n} from \"../contracts/agent/eval.type\";\nimport type { EvalCase as EvalCaseType } from \"../contracts/agent/eval.type\";\nimport { AgentExecutionError } from \"../errors\";\nimport { log } from \"@warlock.js/logger\";\nimport { judge as judgeScorer } from \"./judge-scorer\";\nimport { diff } from \"./regression\";\n\n/**\n * Narrow `EvalOptions.cases` to the underlying `EvalCase[]`. A\n * `DatasetContract` is identified structurally by its `cases` property\n * (an array carried alongside `name` / `filter` / `shard`); a raw\n * `EvalCase[]` is used as-is.\n */\nfunction resolveCases<TOutput>(\n  cases: EvalOptions<TOutput>[\"cases\"],\n): EvalCaseType<TOutput>[] {\n  if (Array.isArray(cases)) {\n    return cases;\n  }\n\n  return cases.cases;\n}\n\nconst LOG_MODULE = \"ai.eval\";\nconst DEFAULT_PASS_THRESHOLD = 0.5;\n\n/**\n * Resolve the scorer list for a single case. Precedence: the case's\n * own `scorers` → the suite `scorers` → a synthesized judge scorer\n * when `judge` is configured. Throws an authoring-time\n * `AgentExecutionError` when a case can resolve none — an eval suite\n * with no way to score a case is a config bug worth surfacing at the\n * call site, not a silent pass.\n */\nfunction resolveScorers<TOutput>(\n  evalCase: EvalCase<TOutput>,\n  options: EvalOptions<TOutput>,\n  passThreshold: number,\n): EvalScorer<TOutput>[] {\n  if (evalCase.scorers && evalCase.scorers.length > 0) {\n    return evalCase.scorers;\n  }\n\n  if (options.scorers && options.scorers.length > 0) {\n    return options.scorers;\n  }\n\n  if (options.judge) {\n    return [judgeScorer<TOutput>(options.judge, passThreshold)];\n  }\n\n  throw new AgentExecutionError(\n    `eval case \"${evalCase.name}\" has no scorer — supply per-case \"scorers\", suite \"scorers\", or a \"judge\"`,\n    { context: { authoring: true, case: evalCase.name } },\n  );\n}\n\n/**\n * Decide a single scorer verdict's pass/fail. Honors an explicit\n * `passed` from the scorer; otherwise derives it from\n * `score >= passThreshold`.\n */\nfunction isScorePassing(score: EvalScore, passThreshold: number): boolean {\n  if (typeof score.passed === \"boolean\") {\n    return score.passed;\n  }\n\n  return score.score >= passThreshold;\n}\n\n/**\n * Merge suite-level execute options with the case's own override.\n * Per-case wins on conflict (shallow merge).\n */\nfunction mergeOptions<TOutput>(\n  suite: AgentExecuteOptions<TOutput> | undefined,\n  perCase: AgentExecuteOptions<TOutput> | undefined,\n): AgentExecuteOptions<TOutput> | undefined {\n  if (!suite) return perCase;\n  if (!perCase) return suite;\n  return { ...suite, ...perCase };\n}\n\n/**\n * Run one case end-to-end: execute the agent, run every resolved\n * scorer, aggregate into an {@link EvalCaseResult}. A case passes only\n * when the agent did not error AND every scorer passed.\n */\nasync function runCase<TOutput>(\n  agent: AgentContract<TOutput>,\n  evalCase: EvalCase<TOutput>,\n  options: EvalOptions<TOutput>,\n  passThreshold: number,\n): Promise<EvalCaseResult<TOutput>> {\n  const scorers = resolveScorers(evalCase, options, passThreshold);\n  const executeOptions = mergeOptions(options.executeOptions, evalCase.options);\n\n  const start = performance.now();\n  const result = await agent.execute(evalCase.input, executeOptions);\n  const duration = performance.now() - start;\n\n  const context: EvalScorerContext<TOutput> = {\n    case: evalCase,\n    result,\n    output: result.data,\n    text: result.text,\n  };\n\n  const scores: EvalScore[] = [];\n\n  for (const scorer of scorers) {\n    scores.push(await scorer(context));\n  }\n\n  const meanScore =\n    scores.length > 0 ? scores.reduce((sum, score) => sum + score.score, 0) / scores.length : 0;\n\n  const allScorersPassed = scores.every((score) => isScorePassing(score, passThreshold));\n  const passed = result.error === undefined && allScorersPassed;\n\n  return {\n    case: evalCase,\n    result,\n    scores,\n    score: meanScore,\n    passed,\n    duration,\n  };\n}\n\n/**\n * Core implementation of `agent.eval`. Runs every case sequentially\n * (cases share the agent and may carry side effects — ordering must be\n * deterministic), scores each, fires `onFailure` for failed cases, and\n * assembles the aggregate {@link EvalReport}.\n *\n * Never throws on a case-level failure; the only throw is the\n * authoring-time \"no scorer\" guard from {@link resolveScorers}.\n */\nexport async function runEval<TOutput>(\n  agent: AgentContract<TOutput>,\n  options: EvalOptions<TOutput>,\n): Promise<EvalReport<TOutput>> {\n  const passThreshold = options.passThreshold ?? DEFAULT_PASS_THRESHOLD;\n  const start = performance.now();\n\n  const suiteCases = resolveCases(options.cases);\n  const cases: EvalCaseResult<TOutput>[] = [];\n\n  for (const evalCase of suiteCases) {\n    const caseResult = await runCase(agent, evalCase, options, passThreshold);\n\n    cases.push(caseResult);\n\n    if (!caseResult.passed && options.onFailure) {\n      try {\n        await options.onFailure(caseResult);\n      } catch (error) {\n        log.warn(LOG_MODULE, \"onFailure.hook.error\", \"eval onFailure handler threw\", {\n          agent: agent.name,\n          case: evalCase.name,\n          error: error instanceof Error ? error.message : String(error),\n        });\n      }\n    }\n  }\n\n  const passedCount = cases.filter((entry) => entry.passed).length;\n  const total = cases.length;\n  const meanScore =\n    total > 0 ? cases.reduce((sum, entry) => sum + entry.score, 0) / total : 0;\n\n  const report: EvalReport<TOutput> = {\n    agentName: agent.name,\n    total,\n    passedCount,\n    failedCount: total - passedCount,\n    passRate: total > 0 ? passedCount / total : 0,\n    meanScore,\n    passed: total > 0 && passedCount === total,\n    cases,\n    duration: performance.now() - start,\n  };\n\n  if (options.baseline) {\n    report.regression = diff(report, options.baseline, options.tolerance);\n  }\n\n  return report;\n}\n"],"mappings":";;;;;;;;;;;;;AAuBA,SAAS,aACP,OACyB;CACzB,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO;CAGT,OAAO,MAAM;AACf;AAEA,MAAM,aAAa;AACnB,MAAM,yBAAyB;;;;;;;;;AAU/B,SAAS,eACP,UACA,SACA,eACuB;CACvB,IAAI,SAAS,WAAW,SAAS,QAAQ,SAAS,GAChD,OAAO,SAAS;CAGlB,IAAI,QAAQ,WAAW,QAAQ,QAAQ,SAAS,GAC9C,OAAO,QAAQ;CAGjB,IAAI,QAAQ,OACV,OAAO,CAACA,MAAqB,QAAQ,OAAO,aAAa,CAAC;CAG5D,MAAM,IAAI,oBACR,cAAc,SAAS,KAAK,6EAC5B,EAAE,SAAS;EAAE,WAAW;EAAM,MAAM,SAAS;CAAK,EAAE,CACtD;AACF;;;;;;AAOA,SAAS,eAAe,OAAkB,eAAgC;CACxE,IAAI,OAAO,MAAM,WAAW,WAC1B,OAAO,MAAM;CAGf,OAAO,MAAM,SAAS;AACxB;;;;;AAMA,SAAS,aACP,OACA,SAC0C;CAC1C,IAAI,CAAC,OAAO,OAAO;CACnB,IAAI,CAAC,SAAS,OAAO;CACrB,OAAO;EAAE,GAAG;EAAO,GAAG;CAAQ;AAChC;;;;;;AAOA,eAAe,QACb,OACA,UACA,SACA,eACkC;CAClC,MAAM,UAAU,eAAe,UAAU,SAAS,aAAa;CAC/D,MAAM,iBAAiB,aAAa,QAAQ,gBAAgB,SAAS,OAAO;CAE5E,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,SAAS,MAAM,MAAM,QAAQ,SAAS,OAAO,cAAc;CACjE,MAAM,WAAW,YAAY,IAAI,IAAI;CAErC,MAAM,UAAsC;EAC1C,MAAM;EACN;EACA,QAAQ,OAAO;EACf,MAAM,OAAO;CACf;CAEA,MAAM,SAAsB,CAAC;CAE7B,KAAK,MAAM,UAAU,SACnB,OAAO,KAAK,MAAM,OAAO,OAAO,CAAC;CAGnC,MAAM,YACJ,OAAO,SAAS,IAAI,OAAO,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,CAAC,IAAI,OAAO,SAAS;CAE5F,MAAM,mBAAmB,OAAO,OAAO,UAAU,eAAe,OAAO,aAAa,CAAC;CAGrF,OAAO;EACL,MAAM;EACN;EACA;EACA,OAAO;EACP,QAPa,OAAO,UAAU,UAAa;EAQ3C;CACF;AACF;;;;;;;;;;AAWA,eAAsB,QACpB,OACA,SAC8B;CAC9B,MAAM,gBAAgB,QAAQ,iBAAiB;CAC/C,MAAM,QAAQ,YAAY,IAAI;CAE9B,MAAM,aAAa,aAAa,QAAQ,KAAK;CAC7C,MAAM,QAAmC,CAAC;CAE1C,KAAK,MAAM,YAAY,YAAY;EACjC,MAAM,aAAa,MAAM,QAAQ,OAAO,UAAU,SAAS,aAAa;EAExE,MAAM,KAAK,UAAU;EAErB,IAAI,CAAC,WAAW,UAAU,QAAQ,WAChC,IAAI;GACF,MAAM,QAAQ,UAAU,UAAU;EACpC,SAAS,OAAO;GACd,IAAI,KAAK,YAAY,wBAAwB,gCAAgC;IAC3E,OAAO,MAAM;IACb,MAAM,SAAS;IACf,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D,CAAC;EACH;CAEJ;CAEA,MAAM,cAAc,MAAM,QAAQ,UAAU,MAAM,MAAM,CAAC,CAAC;CAC1D,MAAM,QAAQ,MAAM;CACpB,MAAM,YACJ,QAAQ,IAAI,MAAM,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,CAAC,IAAI,QAAQ;CAE3E,MAAM,SAA8B;EAClC,WAAW,MAAM;EACjB;EACA;EACA,aAAa,QAAQ;EACrB,UAAU,QAAQ,IAAI,cAAc,QAAQ;EAC5C;EACA,QAAQ,QAAQ,KAAK,gBAAgB;EACrC;EACA,UAAU,YAAY,IAAI,IAAI;CAChC;CAEA,IAAI,QAAQ,UACV,OAAO,aAAa,KAAK,QAAQ,QAAQ,UAAU,QAAQ,SAAS;CAGtE,OAAO;AACT"}

package/esm/eval/index.d.mts

@@ -1,28 +1,47 @@
+import { DatasetContract, DatasetEntry, DatasetOptions } from "./dataset.type.mjs";
 import { judge } from "./judge-scorer.mjs";
+import { fromJSON, toJSON } from "./report-json.mjs";
+import { toJUnit } from "./report-junit.mjs";
 import { EvalPredicate, contains, exact, predicate } from "./scorers.mjs";
 import { runEval } from "./eval-runner.mjs";
+import { dataset } from "./dataset.mjs";
+import { diff } from "./regression.mjs";
 
 //#region ../@warlock.js/ai/src/eval/index.d.ts
 /**
- * Built-in eval scorer factories, surfaced on `ai.eval.*`.
+ * Built-in eval scorer factories plus the CI reporters, surfaced on
+ * `ai.eval.*`.
  *
+ * Scorers:
  * - `exact()` — pass when output equals the case `expected` (trimmed,
  *   case-insensitive; structured values compared by canonical JSON).
  * - `contains()` — pass when `expected` appears as a substring.
  * - `predicate(fn)` — wrap an arbitrary boolean assertion.
  * - `judge(config)` — LLM-as-judge scoring against a rubric.
  *
+ * Reporters / serialization (pure functions over a finished `EvalReport`):
+ * - `toJUnit(report)` — JUnit-XML artifact for CI ingestion.
+ * - `toJSON(report)` / `fromJSON(serialized)` — round-trippable snapshot;
+ *   today's report becomes tomorrow's `baseline`.
+ *
  * @example
  * await myAgent.eval({
  *   cases: [{ name: "q", input: "Capital of Egypt?", expected: "Cairo" }],
  *   scorers: [ai.eval.contains()],
  * });
+ *
+ * @example
+ * const report = await myAgent.eval({ cases: ds, scorers: [ai.eval.exact()] });
+ * await writeFile("./report.junit.xml", ai.eval.toJUnit(report));
  */
 declare const evalScorers: {
   exact: typeof exact;
   contains: typeof contains;
   predicate: typeof predicate;
   judge: typeof judge;
+  toJUnit: typeof toJUnit;
+  toJSON: typeof toJSON;
+  fromJSON: typeof fromJSON;
 };
 //#endregion
 export { evalScorers };

package/esm/eval/index.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/index.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;cA0Ba,WAAA"}
+{"version":3,"file":"index.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/index.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAqDa,WAAA"}

package/esm/eval/index.mjs

@@ -1,28 +1,46 @@
 import { judge } from "./judge-scorer.mjs";
+import { diff } from "./regression.mjs";
 import { runEval } from "./eval-runner.mjs";
+import { fromJSON, toJSON } from "./report-json.mjs";
+import { toJUnit } from "./report-junit.mjs";
 import { contains, exact, predicate } from "./scorers.mjs";
+import { dataset } from "./dataset.mjs";
 
 //#region ../@warlock.js/ai/src/eval/index.ts
 /**
-* Built-in eval scorer factories, surfaced on `ai.eval.*`.
+* Built-in eval scorer factories plus the CI reporters, surfaced on
+* `ai.eval.*`.
 *
+* Scorers:
 * - `exact()` — pass when output equals the case `expected` (trimmed,
 *   case-insensitive; structured values compared by canonical JSON).
 * - `contains()` — pass when `expected` appears as a substring.
 * - `predicate(fn)` — wrap an arbitrary boolean assertion.
 * - `judge(config)` — LLM-as-judge scoring against a rubric.
 *
+* Reporters / serialization (pure functions over a finished `EvalReport`):
+* - `toJUnit(report)` — JUnit-XML artifact for CI ingestion.
+* - `toJSON(report)` / `fromJSON(serialized)` — round-trippable snapshot;
+*   today's report becomes tomorrow's `baseline`.
+*
 * @example
 * await myAgent.eval({
 *   cases: [{ name: "q", input: "Capital of Egypt?", expected: "Cairo" }],
 *   scorers: [ai.eval.contains()],
 * });
+*
+* @example
+* const report = await myAgent.eval({ cases: ds, scorers: [ai.eval.exact()] });
+* await writeFile("./report.junit.xml", ai.eval.toJUnit(report));
 */
 const evalScorers = {
 	exact,
 	contains,
 	predicate,
-	judge
+	judge,
+	toJUnit,
+	toJSON,
+	fromJSON
 };
 
 //#endregion

package/esm/eval/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/index.ts"],"sourcesContent":["import { judge } from \"./judge-scorer\";\nimport { contains, exact, predicate } from \"./scorers\";\n\n// Runner — wired into AgentContract.eval() by the agent factory.\nexport { runEval } from \"./eval-runner\";\n\n// Scorer factories — re-exported individually for direct import.\nexport { contains, exact, predicate } from \"./scorers\";\nexport type { EvalPredicate } from \"./scorers\";\nexport { judge } from \"./judge-scorer\";\n\n/**\n * Built-in eval scorer factories, surfaced on `ai.eval.*`.\n *\n * - `exact()` — pass when output equals the case `expected` (trimmed,\n *   case-insensitive; structured values compared by canonical JSON).\n * - `contains()` — pass when `expected` appears as a substring.\n * - `predicate(fn)` — wrap an arbitrary boolean assertion.\n * - `judge(config)` — LLM-as-judge scoring against a rubric.\n *\n * @example\n * await myAgent.eval({\n *   cases: [{ name: \"q\", input: \"Capital of Egypt?\", expected: \"Cairo\" }],\n *   scorers: [ai.eval.contains()],\n * });\n */\nexport const evalScorers = {\n  exact,\n  contains,\n  predicate,\n  judge,\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AA0BA,MAAa,cAAc;CACzB;CACA;CACA;CACA;AACF"}
+{"version":3,"file":"index.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/index.ts"],"sourcesContent":["import { dataset } from \"./dataset\";\nimport { judge } from \"./judge-scorer\";\nimport { fromJSON, toJSON } from \"./report-json\";\nimport { toJUnit } from \"./report-junit\";\nimport { contains, exact, predicate } from \"./scorers\";\n\n// Runner — wired into AgentContract.eval() by the agent factory.\nexport { runEval } from \"./eval-runner\";\n\n// Scorer factories — re-exported individually for direct import.\nexport { contains, exact, predicate } from \"./scorers\";\nexport type { EvalPredicate } from \"./scorers\";\nexport { judge } from \"./judge-scorer\";\n\n// Dataset primitive — feeds `agent.eval({ cases })`.\nexport { dataset } from \"./dataset\";\nexport type {\n  DatasetContract,\n  DatasetEntry,\n  DatasetOptions,\n} from \"./dataset.type\";\n\n// Regression diff + CI reporters (pure, runner-decoupled).\nexport { diff } from \"./regression\";\nexport { toJSON, fromJSON } from \"./report-json\";\nexport { toJUnit } from \"./report-junit\";\n\n/**\n * Built-in eval scorer factories plus the CI reporters, surfaced on\n * `ai.eval.*`.\n *\n * Scorers:\n * - `exact()` — pass when output equals the case `expected` (trimmed,\n *   case-insensitive; structured values compared by canonical JSON).\n * - `contains()` — pass when `expected` appears as a substring.\n * - `predicate(fn)` — wrap an arbitrary boolean assertion.\n * - `judge(config)` — LLM-as-judge scoring against a rubric.\n *\n * Reporters / serialization (pure functions over a finished `EvalReport`):\n * - `toJUnit(report)` — JUnit-XML artifact for CI ingestion.\n * - `toJSON(report)` / `fromJSON(serialized)` — round-trippable snapshot;\n *   today's report becomes tomorrow's `baseline`.\n *\n * @example\n * await myAgent.eval({\n *   cases: [{ name: \"q\", input: \"Capital of Egypt?\", expected: \"Cairo\" }],\n *   scorers: [ai.eval.contains()],\n * });\n *\n * @example\n * const report = await myAgent.eval({ cases: ds, scorers: [ai.eval.exact()] });\n * await writeFile(\"./report.junit.xml\", ai.eval.toJUnit(report));\n */\nexport const evalScorers = {\n  exact,\n  contains,\n  predicate,\n  judge,\n  toJUnit,\n  toJSON,\n  fromJSON,\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqDA,MAAa,cAAc;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;AACF"}

package/esm/eval/regression.d.mts

@@ -0,0 +1,30 @@
+import { EvalRegression, EvalReport } from "../contracts/agent/eval.type.mjs";
+
+//#region ../@warlock.js/ai/src/eval/regression.d.ts
+/**
+ * Diff a fresh {@link EvalReport} against a `baseline`, joining cases by
+ * name, to produce an {@link EvalRegression} verdict.
+ *
+ * A case **regresses** when its new aggregate `score` is more than
+ * `tolerance` below its baseline score (`before - after > tolerance`).
+ * Cases that improved, held steady, or moved within `tolerance` are not
+ * flagged. Cases present in only one of the two reports are surfaced
+ * under `added` / `removed` rather than treated as regressions, so adding
+ * or dropping a case never fails the gate by itself.
+ *
+ * Pure — depends only on the two reports and the tolerance; attaches no
+ * state and mutates neither input.
+ *
+ * @param report - The newly produced report.
+ * @param baseline - A prior report to compare against.
+ * @param tolerance - Max allowed score drop before a case counts as a
+ *   regression. Defaults to `0` (any drop regresses).
+ *
+ * @example
+ * const regression = diff(report, baseline, 0.05);
+ * expect(regression.passed).toBe(true);
+ */
+declare function diff<TOutput = unknown>(report: EvalReport<TOutput>, baseline: EvalReport<TOutput>, tolerance?: number): EvalRegression;
+//#endregion
+export { diff };
+//# sourceMappingURL=regression.d.mts.map

package/esm/eval/regression.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"regression.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/regression.ts"],"mappings":";;;;;AAyBA;;;;;;;;;;;;;;;;;;;;AAIiB;iBAJD,IAAA,oBACd,MAAA,EAAQ,UAAA,CAAW,OAAA,GACnB,QAAA,EAAU,UAAA,CAAW,OAAA,GACrB,SAAA,YACC,cAAA"}

package/esm/eval/regression.mjs

@@ -0,0 +1,51 @@
+//#region ../@warlock.js/ai/src/eval/regression.ts
+/**
+* Diff a fresh {@link EvalReport} against a `baseline`, joining cases by
+* name, to produce an {@link EvalRegression} verdict.
+*
+* A case **regresses** when its new aggregate `score` is more than
+* `tolerance` below its baseline score (`before - after > tolerance`).
+* Cases that improved, held steady, or moved within `tolerance` are not
+* flagged. Cases present in only one of the two reports are surfaced
+* under `added` / `removed` rather than treated as regressions, so adding
+* or dropping a case never fails the gate by itself.
+*
+* Pure — depends only on the two reports and the tolerance; attaches no
+* state and mutates neither input.
+*
+* @param report - The newly produced report.
+* @param baseline - A prior report to compare against.
+* @param tolerance - Max allowed score drop before a case counts as a
+*   regression. Defaults to `0` (any drop regresses).
+*
+* @example
+* const regression = diff(report, baseline, 0.05);
+* expect(regression.passed).toBe(true);
+*/
+function diff(report, baseline, tolerance = 0) {
+	const baselineScores = /* @__PURE__ */ new Map();
+	for (const entry of baseline.cases) baselineScores.set(entry.case.name, entry.score);
+	const currentNames = /* @__PURE__ */ new Set();
+	const regressed = [];
+	for (const entry of report.cases) {
+		const name = entry.case.name;
+		currentNames.add(name);
+		const before = baselineScores.get(name);
+		if (before === void 0) continue;
+		if (before - entry.score > tolerance) regressed.push({
+			name,
+			before,
+			after: entry.score
+		});
+	}
+	return {
+		regressed,
+		removed: baseline.cases.map((entry) => entry.case.name).filter((name) => !currentNames.has(name)),
+		added: report.cases.map((entry) => entry.case.name).filter((name) => !baselineScores.has(name)),
+		passed: regressed.length === 0
+	};
+}
+
+//#endregion
+export { diff };
+//# sourceMappingURL=regression.mjs.map

package/esm/eval/regression.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"regression.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/regression.ts"],"sourcesContent":["import type { EvalRegression, EvalReport } from \"../contracts/agent/eval.type\";\n\n/**\n * Diff a fresh {@link EvalReport} against a `baseline`, joining cases by\n * name, to produce an {@link EvalRegression} verdict.\n *\n * A case **regresses** when its new aggregate `score` is more than\n * `tolerance` below its baseline score (`before - after > tolerance`).\n * Cases that improved, held steady, or moved within `tolerance` are not\n * flagged. Cases present in only one of the two reports are surfaced\n * under `added` / `removed` rather than treated as regressions, so adding\n * or dropping a case never fails the gate by itself.\n *\n * Pure — depends only on the two reports and the tolerance; attaches no\n * state and mutates neither input.\n *\n * @param report - The newly produced report.\n * @param baseline - A prior report to compare against.\n * @param tolerance - Max allowed score drop before a case counts as a\n *   regression. Defaults to `0` (any drop regresses).\n *\n * @example\n * const regression = diff(report, baseline, 0.05);\n * expect(regression.passed).toBe(true);\n */\nexport function diff<TOutput = unknown>(\n  report: EvalReport<TOutput>,\n  baseline: EvalReport<TOutput>,\n  tolerance = 0,\n): EvalRegression {\n  const baselineScores = new Map<string, number>();\n\n  for (const entry of baseline.cases) {\n    baselineScores.set(entry.case.name, entry.score);\n  }\n\n  const currentNames = new Set<string>();\n  const regressed: EvalRegression[\"regressed\"] = [];\n\n  for (const entry of report.cases) {\n    const name = entry.case.name;\n    currentNames.add(name);\n\n    const before = baselineScores.get(name);\n\n    if (before === undefined) {\n      continue;\n    }\n\n    if (before - entry.score > tolerance) {\n      regressed.push({ name, before, after: entry.score });\n    }\n  }\n\n  const removed = baseline.cases\n    .map((entry) => entry.case.name)\n    .filter((name) => !currentNames.has(name));\n\n  const added = report.cases\n    .map((entry) => entry.case.name)\n    .filter((name) => !baselineScores.has(name));\n\n  return {\n    regressed,\n    removed,\n    added,\n    passed: regressed.length === 0,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAyBA,SAAgB,KACd,QACA,UACA,YAAY,GACI;CAChB,MAAM,iCAAiB,IAAI,IAAoB;CAE/C,KAAK,MAAM,SAAS,SAAS,OAC3B,eAAe,IAAI,MAAM,KAAK,MAAM,MAAM,KAAK;CAGjD,MAAM,+BAAe,IAAI,IAAY;CACrC,MAAM,YAAyC,CAAC;CAEhD,KAAK,MAAM,SAAS,OAAO,OAAO;EAChC,MAAM,OAAO,MAAM,KAAK;EACxB,aAAa,IAAI,IAAI;EAErB,MAAM,SAAS,eAAe,IAAI,IAAI;EAEtC,IAAI,WAAW,QACb;EAGF,IAAI,SAAS,MAAM,QAAQ,WACzB,UAAU,KAAK;GAAE;GAAM;GAAQ,OAAO,MAAM;EAAM,CAAC;CAEvD;CAUA,OAAO;EACL;EACA,SAVc,SAAS,MACtB,KAAK,UAAU,MAAM,KAAK,IAAI,CAAC,CAC/B,QAAQ,SAAS,CAAC,aAAa,IAAI,IAAI,CAQlC;EACN,OAPY,OAAO,MAClB,KAAK,UAAU,MAAM,KAAK,IAAI,CAAC,CAC/B,QAAQ,SAAS,CAAC,eAAe,IAAI,IAAI,CAKtC;EACJ,QAAQ,UAAU,WAAW;CAC/B;AACF"}

package/esm/eval/report-json.d.mts

@@ -0,0 +1,30 @@
+import { EvalReport } from "../contracts/agent/eval.type.mjs";
+
+//#region ../@warlock.js/ai/src/eval/report-json.d.ts
+/**
+ * Serialize an {@link EvalReport} to a pretty-printed JSON string — a
+ * round-trippable snapshot suitable for committing as the next run's
+ * baseline (`agent.eval({ baseline: fromJSON(...) })`).
+ *
+ * Pure. The whole report is emitted verbatim; `result` payloads,
+ * per-case `scores`, timings, and any attached `regression` block are all
+ * preserved, so a parsed report drives regression diffing exactly as the
+ * in-memory one would.
+ *
+ * @example
+ * await writeFile("./eval/baseline.json", toJSON(report));
+ */
+declare function toJSON(report: EvalReport): string;
+/**
+ * Parse a string produced by {@link toJSON} back into an
+ * {@link EvalReport}. The inverse of `toJSON` — `fromJSON(toJSON(r))`
+ * reproduces `r`'s data (functions such as scorers were never part of the
+ * serialized report, so the round-trip is over plain data only).
+ *
+ * @example
+ * const baseline = fromJSON(await readFile("./eval/baseline.json", "utf8"));
+ */
+declare function fromJSON<TOutput = unknown>(serialized: string): EvalReport<TOutput>;
+//#endregion
+export { fromJSON, toJSON };
+//# sourceMappingURL=report-json.d.mts.map

package/esm/eval/report-json.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-json.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/report-json.ts"],"mappings":";;;;;AAeA;;;;AAAyC;AAazC;;;;;;iBAbgB,MAAA,CAAO,MAAkB,EAAV,UAAU;;;AAa0C;;;;;;;iBAAnE,QAAA,oBAA4B,UAAA,WAAqB,UAAU,CAAC,OAAA"}

package/esm/eval/report-json.mjs

@@ -0,0 +1,33 @@
+//#region ../@warlock.js/ai/src/eval/report-json.ts
+/**
+* Serialize an {@link EvalReport} to a pretty-printed JSON string — a
+* round-trippable snapshot suitable for committing as the next run's
+* baseline (`agent.eval({ baseline: fromJSON(...) })`).
+*
+* Pure. The whole report is emitted verbatim; `result` payloads,
+* per-case `scores`, timings, and any attached `regression` block are all
+* preserved, so a parsed report drives regression diffing exactly as the
+* in-memory one would.
+*
+* @example
+* await writeFile("./eval/baseline.json", toJSON(report));
+*/
+function toJSON(report) {
+	return JSON.stringify(report, void 0, 2);
+}
+/**
+* Parse a string produced by {@link toJSON} back into an
+* {@link EvalReport}. The inverse of `toJSON` — `fromJSON(toJSON(r))`
+* reproduces `r`'s data (functions such as scorers were never part of the
+* serialized report, so the round-trip is over plain data only).
+*
+* @example
+* const baseline = fromJSON(await readFile("./eval/baseline.json", "utf8"));
+*/
+function fromJSON(serialized) {
+	return JSON.parse(serialized);
+}
+
+//#endregion
+export { fromJSON, toJSON };
+//# sourceMappingURL=report-json.mjs.map

package/esm/eval/report-json.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-json.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/report-json.ts"],"sourcesContent":["import type { EvalReport } from \"../contracts/agent/eval.type\";\n\n/**\n * Serialize an {@link EvalReport} to a pretty-printed JSON string — a\n * round-trippable snapshot suitable for committing as the next run's\n * baseline (`agent.eval({ baseline: fromJSON(...) })`).\n *\n * Pure. The whole report is emitted verbatim; `result` payloads,\n * per-case `scores`, timings, and any attached `regression` block are all\n * preserved, so a parsed report drives regression diffing exactly as the\n * in-memory one would.\n *\n * @example\n * await writeFile(\"./eval/baseline.json\", toJSON(report));\n */\nexport function toJSON(report: EvalReport): string {\n  return JSON.stringify(report, undefined, 2);\n}\n\n/**\n * Parse a string produced by {@link toJSON} back into an\n * {@link EvalReport}. The inverse of `toJSON` — `fromJSON(toJSON(r))`\n * reproduces `r`'s data (functions such as scorers were never part of the\n * serialized report, so the round-trip is over plain data only).\n *\n * @example\n * const baseline = fromJSON(await readFile(\"./eval/baseline.json\", \"utf8\"));\n */\nexport function fromJSON<TOutput = unknown>(serialized: string): EvalReport<TOutput> {\n  return JSON.parse(serialized) as EvalReport<TOutput>;\n}\n"],"mappings":";;;;;;;;;;;;;;AAeA,SAAgB,OAAO,QAA4B;CACjD,OAAO,KAAK,UAAU,QAAQ,QAAW,CAAC;AAC5C;;;;;;;;;;AAWA,SAAgB,SAA4B,YAAyC;CACnF,OAAO,KAAK,MAAM,UAAU;AAC9B"}

package/esm/eval/report-junit.d.mts

@@ -0,0 +1,22 @@
+import { EvalReport } from "../contracts/agent/eval.type.mjs";
+
+//#region ../@warlock.js/ai/src/eval/report-junit.d.ts
+/**
+ * Serialize an {@link EvalReport} to a JUnit-XML string for CI ingestion.
+ *
+ * **Role.** A pure, runner-decoupled reporter: one `<testsuite>` whose
+ * name is the agent, one `<testcase>` per eval case, a `<failure>` child
+ * on each case that did not pass (with the joined scorer reasons), and a
+ * `time` attribute carrying the case / suite duration in **seconds**
+ * (JUnit's unit; the report stores milliseconds).
+ *
+ * XML is hand-emitted (no `xml` dependency) and every dynamic value is
+ * entity-escaped via {@link escapeXml}.
+ *
+ * @example
+ * await writeFile("./report.junit.xml", toJUnit(report));
+ */
+declare function toJUnit(report: EvalReport): string;
+//#endregion
+export { toJUnit };
+//# sourceMappingURL=report-junit.d.mts.map

package/esm/eval/report-junit.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-junit.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/report-junit.ts"],"mappings":";;;;;AAqDA;;;;AAA0C;;;;;;;;;iBAA1B,OAAA,CAAQ,MAAkB,EAAV,UAAU"}