@warlock.js/ai 4.4.0 → 4.6.0

package/esm/eval/eval-runner.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"eval-runner.mjs","names":["judgeScorer"],"sources":["../../../../../../../@warlock.js/ai/src/eval/eval-runner.ts"],"sourcesContent":["import type { AgentContract } from \"../contracts/agent/agent.contract\";\nimport type { AgentExecuteOptions } from \"../contracts/agent/agent-options.type\";\nimport type {\n  EvalCase,\n  EvalCaseResult,\n  EvalOptions,\n  EvalReport,\n  EvalScore,\n  EvalScorer,\n  EvalScorerContext,\n} from \"../contracts/agent/eval.type\";\nimport { AgentExecutionError } from \"../errors\";\nimport { log } from \"@warlock.js/logger\";\nimport { judge as judgeScorer } from \"./judge-scorer\";\n\nconst LOG_MODULE = \"ai.eval\";\nconst DEFAULT_PASS_THRESHOLD = 0.5;\n\n/**\n * Resolve the scorer list for a single case. Precedence: the case's\n * own `scorers` → the suite `scorers` → a synthesized judge scorer\n * when `judge` is configured. Throws an authoring-time\n * `AgentExecutionError` when a case can resolve none — an eval suite\n * with no way to score a case is a config bug worth surfacing at the\n * call site, not a silent pass.\n */\nfunction resolveScorers<TOutput>(\n  evalCase: EvalCase<TOutput>,\n  options: EvalOptions<TOutput>,\n  passThreshold: number,\n): EvalScorer<TOutput>[] {\n  if (evalCase.scorers && evalCase.scorers.length > 0) {\n    return evalCase.scorers;\n  }\n\n  if (options.scorers && options.scorers.length > 0) {\n    return options.scorers;\n  }\n\n  if (options.judge) {\n    return [judgeScorer<TOutput>(options.judge, passThreshold)];\n  }\n\n  throw new AgentExecutionError(\n    `eval case \"${evalCase.name}\" has no scorer — supply per-case \"scorers\", suite \"scorers\", or a \"judge\"`,\n    { context: { authoring: true, case: evalCase.name } },\n  );\n}\n\n/**\n * Decide a single scorer verdict's pass/fail. Honors an explicit\n * `passed` from the scorer; otherwise derives it from\n * `score >= passThreshold`.\n */\nfunction isScorePassing(score: EvalScore, passThreshold: number): boolean {\n  if (typeof score.passed === \"boolean\") {\n    return score.passed;\n  }\n\n  return score.score >= passThreshold;\n}\n\n/**\n * Merge suite-level execute options with the case's own override.\n * Per-case wins on conflict (shallow merge).\n */\nfunction mergeOptions<TOutput>(\n  suite: AgentExecuteOptions<TOutput> | undefined,\n  perCase: AgentExecuteOptions<TOutput> | undefined,\n): AgentExecuteOptions<TOutput> | undefined {\n  if (!suite) return perCase;\n  if (!perCase) return suite;\n  return { ...suite, ...perCase };\n}\n\n/**\n * Run one case end-to-end: execute the agent, run every resolved\n * scorer, aggregate into an {@link EvalCaseResult}. A case passes only\n * when the agent did not error AND every scorer passed.\n */\nasync function runCase<TOutput>(\n  agent: AgentContract<TOutput>,\n  evalCase: EvalCase<TOutput>,\n  options: EvalOptions<TOutput>,\n  passThreshold: number,\n): Promise<EvalCaseResult<TOutput>> {\n  const scorers = resolveScorers(evalCase, options, passThreshold);\n  const executeOptions = mergeOptions(options.executeOptions, evalCase.options);\n\n  const start = performance.now();\n  const result = await agent.execute(evalCase.input, executeOptions);\n  const duration = performance.now() - start;\n\n  const context: EvalScorerContext<TOutput> = {\n    case: evalCase,\n    result,\n    output: result.data,\n    text: result.text,\n  };\n\n  const scores: EvalScore[] = [];\n\n  for (const scorer of scorers) {\n    scores.push(await scorer(context));\n  }\n\n  const meanScore =\n    scores.length > 0 ? scores.reduce((sum, score) => sum + score.score, 0) / scores.length : 0;\n\n  const allScorersPassed = scores.every((score) => isScorePassing(score, passThreshold));\n  const passed = result.error === undefined && allScorersPassed;\n\n  return {\n    case: evalCase,\n    result,\n    scores,\n    score: meanScore,\n    passed,\n    duration,\n  };\n}\n\n/**\n * Core implementation of `agent.eval`. Runs every case sequentially\n * (cases share the agent and may carry side effects — ordering must be\n * deterministic), scores each, fires `onFailure` for failed cases, and\n * assembles the aggregate {@link EvalReport}.\n *\n * Never throws on a case-level failure; the only throw is the\n * authoring-time \"no scorer\" guard from {@link resolveScorers}.\n */\nexport async function runEval<TOutput>(\n  agent: AgentContract<TOutput>,\n  options: EvalOptions<TOutput>,\n): Promise<EvalReport<TOutput>> {\n  const passThreshold = options.passThreshold ?? DEFAULT_PASS_THRESHOLD;\n  const start = performance.now();\n\n  const cases: EvalCaseResult<TOutput>[] = [];\n\n  for (const evalCase of options.cases) {\n    const caseResult = await runCase(agent, evalCase, options, passThreshold);\n\n    cases.push(caseResult);\n\n    if (!caseResult.passed && options.onFailure) {\n      try {\n        await options.onFailure(caseResult);\n      } catch (error) {\n        log.warn(LOG_MODULE, \"onFailure.hook.error\", \"eval onFailure handler threw\", {\n          agent: agent.name,\n          case: evalCase.name,\n          error: error instanceof Error ? error.message : String(error),\n        });\n      }\n    }\n  }\n\n  const passedCount = cases.filter((entry) => entry.passed).length;\n  const total = cases.length;\n  const meanScore =\n    total > 0 ? cases.reduce((sum, entry) => sum + entry.score, 0) / total : 0;\n\n  return {\n    agentName: agent.name,\n    total,\n    passedCount,\n    failedCount: total - passedCount,\n    passRate: total > 0 ? passedCount / total : 0,\n    meanScore,\n    passed: total > 0 && passedCount === total,\n    cases,\n    duration: performance.now() - start,\n  };\n}\n"],"mappings":";;;;;;AAeA,MAAM,aAAa;AACnB,MAAM,yBAAyB;;;;;;;;;AAU/B,SAAS,eACP,UACA,SACA,eACuB;CACvB,IAAI,SAAS,WAAW,SAAS,QAAQ,SAAS,GAChD,OAAO,SAAS;CAGlB,IAAI,QAAQ,WAAW,QAAQ,QAAQ,SAAS,GAC9C,OAAO,QAAQ;CAGjB,IAAI,QAAQ,OACV,OAAO,CAACA,MAAqB,QAAQ,OAAO,aAAa,CAAC;CAG5D,MAAM,IAAI,oBACR,cAAc,SAAS,KAAK,6EAC5B,EAAE,SAAS;EAAE,WAAW;EAAM,MAAM,SAAS;CAAK,EAAE,CACtD;AACF;;;;;;AAOA,SAAS,eAAe,OAAkB,eAAgC;CACxE,IAAI,OAAO,MAAM,WAAW,WAC1B,OAAO,MAAM;CAGf,OAAO,MAAM,SAAS;AACxB;;;;;AAMA,SAAS,aACP,OACA,SAC0C;CAC1C,IAAI,CAAC,OAAO,OAAO;CACnB,IAAI,CAAC,SAAS,OAAO;CACrB,OAAO;EAAE,GAAG;EAAO,GAAG;CAAQ;AAChC;;;;;;AAOA,eAAe,QACb,OACA,UACA,SACA,eACkC;CAClC,MAAM,UAAU,eAAe,UAAU,SAAS,aAAa;CAC/D,MAAM,iBAAiB,aAAa,QAAQ,gBAAgB,SAAS,OAAO;CAE5E,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,SAAS,MAAM,MAAM,QAAQ,SAAS,OAAO,cAAc;CACjE,MAAM,WAAW,YAAY,IAAI,IAAI;CAErC,MAAM,UAAsC;EAC1C,MAAM;EACN;EACA,QAAQ,OAAO;EACf,MAAM,OAAO;CACf;CAEA,MAAM,SAAsB,CAAC;CAE7B,KAAK,MAAM,UAAU,SACnB,OAAO,KAAK,MAAM,OAAO,OAAO,CAAC;CAGnC,MAAM,YACJ,OAAO,SAAS,IAAI,OAAO,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,CAAC,IAAI,OAAO,SAAS;CAE5F,MAAM,mBAAmB,OAAO,OAAO,UAAU,eAAe,OAAO,aAAa,CAAC;CAGrF,OAAO;EACL,MAAM;EACN;EACA;EACA,OAAO;EACP,QAPa,OAAO,UAAU,UAAa;EAQ3C;CACF;AACF;;;;;;;;;;AAWA,eAAsB,QACpB,OACA,SAC8B;CAC9B,MAAM,gBAAgB,QAAQ,iBAAiB;CAC/C,MAAM,QAAQ,YAAY,IAAI;CAE9B,MAAM,QAAmC,CAAC;CAE1C,KAAK,MAAM,YAAY,QAAQ,OAAO;EACpC,MAAM,aAAa,MAAM,QAAQ,OAAO,UAAU,SAAS,aAAa;EAExE,MAAM,KAAK,UAAU;EAErB,IAAI,CAAC,WAAW,UAAU,QAAQ,WAChC,IAAI;GACF,MAAM,QAAQ,UAAU,UAAU;EACpC,SAAS,OAAO;GACd,IAAI,KAAK,YAAY,wBAAwB,gCAAgC;IAC3E,OAAO,MAAM;IACb,MAAM,SAAS;IACf,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D,CAAC;EACH;CAEJ;CAEA,MAAM,cAAc,MAAM,QAAQ,UAAU,MAAM,MAAM,CAAC,CAAC;CAC1D,MAAM,QAAQ,MAAM;CACpB,MAAM,YACJ,QAAQ,IAAI,MAAM,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,CAAC,IAAI,QAAQ;CAE3E,OAAO;EACL,WAAW,MAAM;EACjB;EACA;EACA,aAAa,QAAQ;EACrB,UAAU,QAAQ,IAAI,cAAc,QAAQ;EAC5C;EACA,QAAQ,QAAQ,KAAK,gBAAgB;EACrC;EACA,UAAU,YAAY,IAAI,IAAI;CAChC;AACF"}
+{"version":3,"file":"eval-runner.mjs","names":["judgeScorer"],"sources":["../../../../../../../@warlock.js/ai/src/eval/eval-runner.ts"],"sourcesContent":["import type { AgentContract } from \"../contracts/agent/agent.contract\";\nimport type { AgentExecuteOptions } from \"../contracts/agent/agent-options.type\";\nimport type {\n  EvalCase,\n  EvalCaseResult,\n  EvalOptions,\n  EvalReport,\n  EvalScore,\n  EvalScorer,\n  EvalScorerContext,\n} from \"../contracts/agent/eval.type\";\nimport type { EvalCase as EvalCaseType } from \"../contracts/agent/eval.type\";\nimport { AgentExecutionError } from \"../errors\";\nimport { log } from \"@warlock.js/logger\";\nimport { judge as judgeScorer } from \"./judge-scorer\";\nimport { diff } from \"./regression\";\n\n/**\n * Narrow `EvalOptions.cases` to the underlying `EvalCase[]`. A\n * `DatasetContract` is identified structurally by its `cases` property\n * (an array carried alongside `name` / `filter` / `shard`); a raw\n * `EvalCase[]` is used as-is.\n */\nfunction resolveCases<TOutput>(\n  cases: EvalOptions<TOutput>[\"cases\"],\n): EvalCaseType<TOutput>[] {\n  if (Array.isArray(cases)) {\n    return cases;\n  }\n\n  return cases.cases;\n}\n\nconst LOG_MODULE = \"ai.eval\";\nconst DEFAULT_PASS_THRESHOLD = 0.5;\n\n/**\n * Resolve the scorer list for a single case. Precedence: the case's\n * own `scorers` → the suite `scorers` → a synthesized judge scorer\n * when `judge` is configured. Throws an authoring-time\n * `AgentExecutionError` when a case can resolve none — an eval suite\n * with no way to score a case is a config bug worth surfacing at the\n * call site, not a silent pass.\n */\nfunction resolveScorers<TOutput>(\n  evalCase: EvalCase<TOutput>,\n  options: EvalOptions<TOutput>,\n  passThreshold: number,\n): EvalScorer<TOutput>[] {\n  if (evalCase.scorers && evalCase.scorers.length > 0) {\n    return evalCase.scorers;\n  }\n\n  if (options.scorers && options.scorers.length > 0) {\n    return options.scorers;\n  }\n\n  if (options.judge) {\n    return [judgeScorer<TOutput>(options.judge, passThreshold)];\n  }\n\n  throw new AgentExecutionError(\n    `eval case \"${evalCase.name}\" has no scorer — supply per-case \"scorers\", suite \"scorers\", or a \"judge\"`,\n    { context: { authoring: true, case: evalCase.name } },\n  );\n}\n\n/**\n * Decide a single scorer verdict's pass/fail. Honors an explicit\n * `passed` from the scorer; otherwise derives it from\n * `score >= passThreshold`.\n */\nfunction isScorePassing(score: EvalScore, passThreshold: number): boolean {\n  if (typeof score.passed === \"boolean\") {\n    return score.passed;\n  }\n\n  return score.score >= passThreshold;\n}\n\n/**\n * Merge suite-level execute options with the case's own override.\n * Per-case wins on conflict (shallow merge).\n */\nfunction mergeOptions<TOutput>(\n  suite: AgentExecuteOptions<TOutput> | undefined,\n  perCase: AgentExecuteOptions<TOutput> | undefined,\n): AgentExecuteOptions<TOutput> | undefined {\n  if (!suite) return perCase;\n  if (!perCase) return suite;\n  return { ...suite, ...perCase };\n}\n\n/**\n * Run one case end-to-end: execute the agent, run every resolved\n * scorer, aggregate into an {@link EvalCaseResult}. A case passes only\n * when the agent did not error AND every scorer passed.\n */\nasync function runCase<TOutput>(\n  agent: AgentContract<TOutput>,\n  evalCase: EvalCase<TOutput>,\n  options: EvalOptions<TOutput>,\n  passThreshold: number,\n): Promise<EvalCaseResult<TOutput>> {\n  const scorers = resolveScorers(evalCase, options, passThreshold);\n  const executeOptions = mergeOptions(options.executeOptions, evalCase.options);\n\n  const start = performance.now();\n  const result = await agent.execute(evalCase.input, executeOptions);\n  const duration = performance.now() - start;\n\n  const context: EvalScorerContext<TOutput> = {\n    case: evalCase,\n    result,\n    output: result.data,\n    text: result.text,\n  };\n\n  const scores: EvalScore[] = [];\n\n  for (const scorer of scorers) {\n    scores.push(await scorer(context));\n  }\n\n  const meanScore =\n    scores.length > 0 ? scores.reduce((sum, score) => sum + score.score, 0) / scores.length : 0;\n\n  const allScorersPassed = scores.every((score) => isScorePassing(score, passThreshold));\n  const passed = result.error === undefined && allScorersPassed;\n\n  return {\n    case: evalCase,\n    result,\n    scores,\n    score: meanScore,\n    passed,\n    duration,\n  };\n}\n\n/**\n * Core implementation of `agent.eval`. Runs every case sequentially\n * (cases share the agent and may carry side effects — ordering must be\n * deterministic), scores each, fires `onFailure` for failed cases, and\n * assembles the aggregate {@link EvalReport}.\n *\n * Never throws on a case-level failure; the only throw is the\n * authoring-time \"no scorer\" guard from {@link resolveScorers}.\n */\nexport async function runEval<TOutput>(\n  agent: AgentContract<TOutput>,\n  options: EvalOptions<TOutput>,\n): Promise<EvalReport<TOutput>> {\n  const passThreshold = options.passThreshold ?? DEFAULT_PASS_THRESHOLD;\n  const start = performance.now();\n\n  const suiteCases = resolveCases(options.cases);\n  const cases: EvalCaseResult<TOutput>[] = [];\n\n  for (const evalCase of suiteCases) {\n    const caseResult = await runCase(agent, evalCase, options, passThreshold);\n\n    cases.push(caseResult);\n\n    if (!caseResult.passed && options.onFailure) {\n      try {\n        await options.onFailure(caseResult);\n      } catch (error) {\n        log.warn(LOG_MODULE, \"onFailure.hook.error\", \"eval onFailure handler threw\", {\n          agent: agent.name,\n          case: evalCase.name,\n          error: error instanceof Error ? error.message : String(error),\n        });\n      }\n    }\n  }\n\n  const passedCount = cases.filter((entry) => entry.passed).length;\n  const total = cases.length;\n  const meanScore =\n    total > 0 ? cases.reduce((sum, entry) => sum + entry.score, 0) / total : 0;\n\n  const report: EvalReport<TOutput> = {\n    agentName: agent.name,\n    total,\n    passedCount,\n    failedCount: total - passedCount,\n    passRate: total > 0 ? passedCount / total : 0,\n    meanScore,\n    passed: total > 0 && passedCount === total,\n    cases,\n    duration: performance.now() - start,\n  };\n\n  if (options.baseline) {\n    report.regression = diff(report, options.baseline, options.tolerance);\n  }\n\n  return report;\n}\n"],"mappings":";;;;;;;;;;;;;AAuBA,SAAS,aACP,OACyB;CACzB,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO;CAGT,OAAO,MAAM;AACf;AAEA,MAAM,aAAa;AACnB,MAAM,yBAAyB;;;;;;;;;AAU/B,SAAS,eACP,UACA,SACA,eACuB;CACvB,IAAI,SAAS,WAAW,SAAS,QAAQ,SAAS,GAChD,OAAO,SAAS;CAGlB,IAAI,QAAQ,WAAW,QAAQ,QAAQ,SAAS,GAC9C,OAAO,QAAQ;CAGjB,IAAI,QAAQ,OACV,OAAO,CAACA,MAAqB,QAAQ,OAAO,aAAa,CAAC;CAG5D,MAAM,IAAI,oBACR,cAAc,SAAS,KAAK,6EAC5B,EAAE,SAAS;EAAE,WAAW;EAAM,MAAM,SAAS;CAAK,EAAE,CACtD;AACF;;;;;;AAOA,SAAS,eAAe,OAAkB,eAAgC;CACxE,IAAI,OAAO,MAAM,WAAW,WAC1B,OAAO,MAAM;CAGf,OAAO,MAAM,SAAS;AACxB;;;;;AAMA,SAAS,aACP,OACA,SAC0C;CAC1C,IAAI,CAAC,OAAO,OAAO;CACnB,IAAI,CAAC,SAAS,OAAO;CACrB,OAAO;EAAE,GAAG;EAAO,GAAG;CAAQ;AAChC;;;;;;AAOA,eAAe,QACb,OACA,UACA,SACA,eACkC;CAClC,MAAM,UAAU,eAAe,UAAU,SAAS,aAAa;CAC/D,MAAM,iBAAiB,aAAa,QAAQ,gBAAgB,SAAS,OAAO;CAE5E,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,SAAS,MAAM,MAAM,QAAQ,SAAS,OAAO,cAAc;CACjE,MAAM,WAAW,YAAY,IAAI,IAAI;CAErC,MAAM,UAAsC;EAC1C,MAAM;EACN;EACA,QAAQ,OAAO;EACf,MAAM,OAAO;CACf;CAEA,MAAM,SAAsB,CAAC;CAE7B,KAAK,MAAM,UAAU,SACnB,OAAO,KAAK,MAAM,OAAO,OAAO,CAAC;CAGnC,MAAM,YACJ,OAAO,SAAS,IAAI,OAAO,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,CAAC,IAAI,OAAO,SAAS;CAE5F,MAAM,mBAAmB,OAAO,OAAO,UAAU,eAAe,OAAO,aAAa,CAAC;CAGrF,OAAO;EACL,MAAM;EACN;EACA;EACA,OAAO;EACP,QAPa,OAAO,UAAU,UAAa;EAQ3C;CACF;AACF;;;;;;;;;;AAWA,eAAsB,QACpB,OACA,SAC8B;CAC9B,MAAM,gBAAgB,QAAQ,iBAAiB;CAC/C,MAAM,QAAQ,YAAY,IAAI;CAE9B,MAAM,aAAa,aAAa,QAAQ,KAAK;CAC7C,MAAM,QAAmC,CAAC;CAE1C,KAAK,MAAM,YAAY,YAAY;EACjC,MAAM,aAAa,MAAM,QAAQ,OAAO,UAAU,SAAS,aAAa;EAExE,MAAM,KAAK,UAAU;EAErB,IAAI,CAAC,WAAW,UAAU,QAAQ,WAChC,IAAI;GACF,MAAM,QAAQ,UAAU,UAAU;EACpC,SAAS,OAAO;GACd,IAAI,KAAK,YAAY,wBAAwB,gCAAgC;IAC3E,OAAO,MAAM;IACb,MAAM,SAAS;IACf,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D,CAAC;EACH;CAEJ;CAEA,MAAM,cAAc,MAAM,QAAQ,UAAU,MAAM,MAAM,CAAC,CAAC;CAC1D,MAAM,QAAQ,MAAM;CACpB,MAAM,YACJ,QAAQ,IAAI,MAAM,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,CAAC,IAAI,QAAQ;CAE3E,MAAM,SAA8B;EAClC,WAAW,MAAM;EACjB;EACA;EACA,aAAa,QAAQ;EACrB,UAAU,QAAQ,IAAI,cAAc,QAAQ;EAC5C;EACA,QAAQ,QAAQ,KAAK,gBAAgB;EACrC;EACA,UAAU,YAAY,IAAI,IAAI;CAChC;CAEA,IAAI,QAAQ,UACV,OAAO,aAAa,KAAK,QAAQ,QAAQ,UAAU,QAAQ,SAAS;CAGtE,OAAO;AACT"}

package/esm/eval/index.d.mts

@@ -1,28 +1,47 @@
+import { DatasetContract, DatasetEntry, DatasetOptions } from "./dataset.type.mjs";
 import { judge } from "./judge-scorer.mjs";
+import { fromJSON, toJSON } from "./report-json.mjs";
+import { toJUnit } from "./report-junit.mjs";
 import { EvalPredicate, contains, exact, predicate } from "./scorers.mjs";
 import { runEval } from "./eval-runner.mjs";
+import { dataset } from "./dataset.mjs";
+import { diff } from "./regression.mjs";
 
 //#region ../@warlock.js/ai/src/eval/index.d.ts
 /**
- * Built-in eval scorer factories, surfaced on `ai.eval.*`.
+ * Built-in eval scorer factories plus the CI reporters, surfaced on
+ * `ai.eval.*`.
  *
+ * Scorers:
  * - `exact()` — pass when output equals the case `expected` (trimmed,
  *   case-insensitive; structured values compared by canonical JSON).
  * - `contains()` — pass when `expected` appears as a substring.
  * - `predicate(fn)` — wrap an arbitrary boolean assertion.
  * - `judge(config)` — LLM-as-judge scoring against a rubric.
  *
+ * Reporters / serialization (pure functions over a finished `EvalReport`):
+ * - `toJUnit(report)` — JUnit-XML artifact for CI ingestion.
+ * - `toJSON(report)` / `fromJSON(serialized)` — round-trippable snapshot;
+ *   today's report becomes tomorrow's `baseline`.
+ *
  * @example
  * await myAgent.eval({
  *   cases: [{ name: "q", input: "Capital of Egypt?", expected: "Cairo" }],
  *   scorers: [ai.eval.contains()],
  * });
+ *
+ * @example
+ * const report = await myAgent.eval({ cases: ds, scorers: [ai.eval.exact()] });
+ * await writeFile("./report.junit.xml", ai.eval.toJUnit(report));
  */
 declare const evalScorers: {
   exact: typeof exact;
   contains: typeof contains;
   predicate: typeof predicate;
   judge: typeof judge;
+  toJUnit: typeof toJUnit;
+  toJSON: typeof toJSON;
+  fromJSON: typeof fromJSON;
 };
 //#endregion
 export { evalScorers };

package/esm/eval/index.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/index.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;cA0Ba,WAAA"}
+{"version":3,"file":"index.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/index.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAqDa,WAAA"}

package/esm/eval/index.mjs

@@ -1,28 +1,46 @@
 import { judge } from "./judge-scorer.mjs";
+import { diff } from "./regression.mjs";
 import { runEval } from "./eval-runner.mjs";
+import { fromJSON, toJSON } from "./report-json.mjs";
+import { toJUnit } from "./report-junit.mjs";
 import { contains, exact, predicate } from "./scorers.mjs";
+import { dataset } from "./dataset.mjs";
 
 //#region ../@warlock.js/ai/src/eval/index.ts
 /**
-* Built-in eval scorer factories, surfaced on `ai.eval.*`.
+* Built-in eval scorer factories plus the CI reporters, surfaced on
+* `ai.eval.*`.
 *
+* Scorers:
 * - `exact()` — pass when output equals the case `expected` (trimmed,
 *   case-insensitive; structured values compared by canonical JSON).
 * - `contains()` — pass when `expected` appears as a substring.
 * - `predicate(fn)` — wrap an arbitrary boolean assertion.
 * - `judge(config)` — LLM-as-judge scoring against a rubric.
 *
+* Reporters / serialization (pure functions over a finished `EvalReport`):
+* - `toJUnit(report)` — JUnit-XML artifact for CI ingestion.
+* - `toJSON(report)` / `fromJSON(serialized)` — round-trippable snapshot;
+*   today's report becomes tomorrow's `baseline`.
+*
 * @example
 * await myAgent.eval({
 *   cases: [{ name: "q", input: "Capital of Egypt?", expected: "Cairo" }],
 *   scorers: [ai.eval.contains()],
 * });
+*
+* @example
+* const report = await myAgent.eval({ cases: ds, scorers: [ai.eval.exact()] });
+* await writeFile("./report.junit.xml", ai.eval.toJUnit(report));
 */
 const evalScorers = {
 	exact,
 	contains,
 	predicate,
-	judge
+	judge,
+	toJUnit,
+	toJSON,
+	fromJSON
 };
 
 //#endregion

package/esm/eval/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/index.ts"],"sourcesContent":["import { judge } from \"./judge-scorer\";\nimport { contains, exact, predicate } from \"./scorers\";\n\n// Runner — wired into AgentContract.eval() by the agent factory.\nexport { runEval } from \"./eval-runner\";\n\n// Scorer factories — re-exported individually for direct import.\nexport { contains, exact, predicate } from \"./scorers\";\nexport type { EvalPredicate } from \"./scorers\";\nexport { judge } from \"./judge-scorer\";\n\n/**\n * Built-in eval scorer factories, surfaced on `ai.eval.*`.\n *\n * - `exact()` — pass when output equals the case `expected` (trimmed,\n *   case-insensitive; structured values compared by canonical JSON).\n * - `contains()` — pass when `expected` appears as a substring.\n * - `predicate(fn)` — wrap an arbitrary boolean assertion.\n * - `judge(config)` — LLM-as-judge scoring against a rubric.\n *\n * @example\n * await myAgent.eval({\n *   cases: [{ name: \"q\", input: \"Capital of Egypt?\", expected: \"Cairo\" }],\n *   scorers: [ai.eval.contains()],\n * });\n */\nexport const evalScorers = {\n  exact,\n  contains,\n  predicate,\n  judge,\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AA0BA,MAAa,cAAc;CACzB;CACA;CACA;CACA;AACF"}
+{"version":3,"file":"index.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/index.ts"],"sourcesContent":["import { dataset } from \"./dataset\";\nimport { judge } from \"./judge-scorer\";\nimport { fromJSON, toJSON } from \"./report-json\";\nimport { toJUnit } from \"./report-junit\";\nimport { contains, exact, predicate } from \"./scorers\";\n\n// Runner — wired into AgentContract.eval() by the agent factory.\nexport { runEval } from \"./eval-runner\";\n\n// Scorer factories — re-exported individually for direct import.\nexport { contains, exact, predicate } from \"./scorers\";\nexport type { EvalPredicate } from \"./scorers\";\nexport { judge } from \"./judge-scorer\";\n\n// Dataset primitive — feeds `agent.eval({ cases })`.\nexport { dataset } from \"./dataset\";\nexport type {\n  DatasetContract,\n  DatasetEntry,\n  DatasetOptions,\n} from \"./dataset.type\";\n\n// Regression diff + CI reporters (pure, runner-decoupled).\nexport { diff } from \"./regression\";\nexport { toJSON, fromJSON } from \"./report-json\";\nexport { toJUnit } from \"./report-junit\";\n\n/**\n * Built-in eval scorer factories plus the CI reporters, surfaced on\n * `ai.eval.*`.\n *\n * Scorers:\n * - `exact()` — pass when output equals the case `expected` (trimmed,\n *   case-insensitive; structured values compared by canonical JSON).\n * - `contains()` — pass when `expected` appears as a substring.\n * - `predicate(fn)` — wrap an arbitrary boolean assertion.\n * - `judge(config)` — LLM-as-judge scoring against a rubric.\n *\n * Reporters / serialization (pure functions over a finished `EvalReport`):\n * - `toJUnit(report)` — JUnit-XML artifact for CI ingestion.\n * - `toJSON(report)` / `fromJSON(serialized)` — round-trippable snapshot;\n *   today's report becomes tomorrow's `baseline`.\n *\n * @example\n * await myAgent.eval({\n *   cases: [{ name: \"q\", input: \"Capital of Egypt?\", expected: \"Cairo\" }],\n *   scorers: [ai.eval.contains()],\n * });\n *\n * @example\n * const report = await myAgent.eval({ cases: ds, scorers: [ai.eval.exact()] });\n * await writeFile(\"./report.junit.xml\", ai.eval.toJUnit(report));\n */\nexport const evalScorers = {\n  exact,\n  contains,\n  predicate,\n  judge,\n  toJUnit,\n  toJSON,\n  fromJSON,\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqDA,MAAa,cAAc;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;AACF"}

package/esm/eval/regression.d.mts

@@ -0,0 +1,30 @@
+import { EvalRegression, EvalReport } from "../contracts/agent/eval.type.mjs";
+
+//#region ../@warlock.js/ai/src/eval/regression.d.ts
+/**
+ * Diff a fresh {@link EvalReport} against a `baseline`, joining cases by
+ * name, to produce an {@link EvalRegression} verdict.
+ *
+ * A case **regresses** when its new aggregate `score` is more than
+ * `tolerance` below its baseline score (`before - after > tolerance`).
+ * Cases that improved, held steady, or moved within `tolerance` are not
+ * flagged. Cases present in only one of the two reports are surfaced
+ * under `added` / `removed` rather than treated as regressions, so adding
+ * or dropping a case never fails the gate by itself.
+ *
+ * Pure — depends only on the two reports and the tolerance; attaches no
+ * state and mutates neither input.
+ *
+ * @param report - The newly produced report.
+ * @param baseline - A prior report to compare against.
+ * @param tolerance - Max allowed score drop before a case counts as a
+ *   regression. Defaults to `0` (any drop regresses).
+ *
+ * @example
+ * const regression = diff(report, baseline, 0.05);
+ * expect(regression.passed).toBe(true);
+ */
+declare function diff<TOutput = unknown>(report: EvalReport<TOutput>, baseline: EvalReport<TOutput>, tolerance?: number): EvalRegression;
+//#endregion
+export { diff };
+//# sourceMappingURL=regression.d.mts.map

package/esm/eval/regression.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"regression.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/regression.ts"],"mappings":";;;;;AAyBA;;;;;;;;;;;;;;;;;;;;AAIiB;iBAJD,IAAA,oBACd,MAAA,EAAQ,UAAA,CAAW,OAAA,GACnB,QAAA,EAAU,UAAA,CAAW,OAAA,GACrB,SAAA,YACC,cAAA"}

package/esm/eval/regression.mjs

@@ -0,0 +1,51 @@
+//#region ../@warlock.js/ai/src/eval/regression.ts
+/**
+* Diff a fresh {@link EvalReport} against a `baseline`, joining cases by
+* name, to produce an {@link EvalRegression} verdict.
+*
+* A case **regresses** when its new aggregate `score` is more than
+* `tolerance` below its baseline score (`before - after > tolerance`).
+* Cases that improved, held steady, or moved within `tolerance` are not
+* flagged. Cases present in only one of the two reports are surfaced
+* under `added` / `removed` rather than treated as regressions, so adding
+* or dropping a case never fails the gate by itself.
+*
+* Pure — depends only on the two reports and the tolerance; attaches no
+* state and mutates neither input.
+*
+* @param report - The newly produced report.
+* @param baseline - A prior report to compare against.
+* @param tolerance - Max allowed score drop before a case counts as a
+*   regression. Defaults to `0` (any drop regresses).
+*
+* @example
+* const regression = diff(report, baseline, 0.05);
+* expect(regression.passed).toBe(true);
+*/
+function diff(report, baseline, tolerance = 0) {
+	const baselineScores = /* @__PURE__ */ new Map();
+	for (const entry of baseline.cases) baselineScores.set(entry.case.name, entry.score);
+	const currentNames = /* @__PURE__ */ new Set();
+	const regressed = [];
+	for (const entry of report.cases) {
+		const name = entry.case.name;
+		currentNames.add(name);
+		const before = baselineScores.get(name);
+		if (before === void 0) continue;
+		if (before - entry.score > tolerance) regressed.push({
+			name,
+			before,
+			after: entry.score
+		});
+	}
+	return {
+		regressed,
+		removed: baseline.cases.map((entry) => entry.case.name).filter((name) => !currentNames.has(name)),
+		added: report.cases.map((entry) => entry.case.name).filter((name) => !baselineScores.has(name)),
+		passed: regressed.length === 0
+	};
+}
+
+//#endregion
+export { diff };
+//# sourceMappingURL=regression.mjs.map

package/esm/eval/regression.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"regression.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/regression.ts"],"sourcesContent":["import type { EvalRegression, EvalReport } from \"../contracts/agent/eval.type\";\n\n/**\n * Diff a fresh {@link EvalReport} against a `baseline`, joining cases by\n * name, to produce an {@link EvalRegression} verdict.\n *\n * A case **regresses** when its new aggregate `score` is more than\n * `tolerance` below its baseline score (`before - after > tolerance`).\n * Cases that improved, held steady, or moved within `tolerance` are not\n * flagged. Cases present in only one of the two reports are surfaced\n * under `added` / `removed` rather than treated as regressions, so adding\n * or dropping a case never fails the gate by itself.\n *\n * Pure — depends only on the two reports and the tolerance; attaches no\n * state and mutates neither input.\n *\n * @param report - The newly produced report.\n * @param baseline - A prior report to compare against.\n * @param tolerance - Max allowed score drop before a case counts as a\n *   regression. Defaults to `0` (any drop regresses).\n *\n * @example\n * const regression = diff(report, baseline, 0.05);\n * expect(regression.passed).toBe(true);\n */\nexport function diff<TOutput = unknown>(\n  report: EvalReport<TOutput>,\n  baseline: EvalReport<TOutput>,\n  tolerance = 0,\n): EvalRegression {\n  const baselineScores = new Map<string, number>();\n\n  for (const entry of baseline.cases) {\n    baselineScores.set(entry.case.name, entry.score);\n  }\n\n  const currentNames = new Set<string>();\n  const regressed: EvalRegression[\"regressed\"] = [];\n\n  for (const entry of report.cases) {\n    const name = entry.case.name;\n    currentNames.add(name);\n\n    const before = baselineScores.get(name);\n\n    if (before === undefined) {\n      continue;\n    }\n\n    if (before - entry.score > tolerance) {\n      regressed.push({ name, before, after: entry.score });\n    }\n  }\n\n  const removed = baseline.cases\n    .map((entry) => entry.case.name)\n    .filter((name) => !currentNames.has(name));\n\n  const added = report.cases\n    .map((entry) => entry.case.name)\n    .filter((name) => !baselineScores.has(name));\n\n  return {\n    regressed,\n    removed,\n    added,\n    passed: regressed.length === 0,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAyBA,SAAgB,KACd,QACA,UACA,YAAY,GACI;CAChB,MAAM,iCAAiB,IAAI,IAAoB;CAE/C,KAAK,MAAM,SAAS,SAAS,OAC3B,eAAe,IAAI,MAAM,KAAK,MAAM,MAAM,KAAK;CAGjD,MAAM,+BAAe,IAAI,IAAY;CACrC,MAAM,YAAyC,CAAC;CAEhD,KAAK,MAAM,SAAS,OAAO,OAAO;EAChC,MAAM,OAAO,MAAM,KAAK;EACxB,aAAa,IAAI,IAAI;EAErB,MAAM,SAAS,eAAe,IAAI,IAAI;EAEtC,IAAI,WAAW,QACb;EAGF,IAAI,SAAS,MAAM,QAAQ,WACzB,UAAU,KAAK;GAAE;GAAM;GAAQ,OAAO,MAAM;EAAM,CAAC;CAEvD;CAUA,OAAO;EACL;EACA,SAVc,SAAS,MACtB,KAAK,UAAU,MAAM,KAAK,IAAI,CAAC,CAC/B,QAAQ,SAAS,CAAC,aAAa,IAAI,IAAI,CAQlC;EACN,OAPY,OAAO,MAClB,KAAK,UAAU,MAAM,KAAK,IAAI,CAAC,CAC/B,QAAQ,SAAS,CAAC,eAAe,IAAI,IAAI,CAKtC;EACJ,QAAQ,UAAU,WAAW;CAC/B;AACF"}

package/esm/eval/report-json.d.mts

@@ -0,0 +1,30 @@
+import { EvalReport } from "../contracts/agent/eval.type.mjs";
+
+//#region ../@warlock.js/ai/src/eval/report-json.d.ts
+/**
+ * Serialize an {@link EvalReport} to a pretty-printed JSON string — a
+ * round-trippable snapshot suitable for committing as the next run's
+ * baseline (`agent.eval({ baseline: fromJSON(...) })`).
+ *
+ * Pure. The whole report is emitted verbatim; `result` payloads,
+ * per-case `scores`, timings, and any attached `regression` block are all
+ * preserved, so a parsed report drives regression diffing exactly as the
+ * in-memory one would.
+ *
+ * @example
+ * await writeFile("./eval/baseline.json", toJSON(report));
+ */
+declare function toJSON(report: EvalReport): string;
+/**
+ * Parse a string produced by {@link toJSON} back into an
+ * {@link EvalReport}. The inverse of `toJSON` — `fromJSON(toJSON(r))`
+ * reproduces `r`'s data (functions such as scorers were never part of the
+ * serialized report, so the round-trip is over plain data only).
+ *
+ * @example
+ * const baseline = fromJSON(await readFile("./eval/baseline.json", "utf8"));
+ */
+declare function fromJSON<TOutput = unknown>(serialized: string): EvalReport<TOutput>;
+//#endregion
+export { fromJSON, toJSON };
+//# sourceMappingURL=report-json.d.mts.map

package/esm/eval/report-json.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-json.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/report-json.ts"],"mappings":";;;;;AAeA;;;;AAAyC;AAazC;;;;;;iBAbgB,MAAA,CAAO,MAAkB,EAAV,UAAU;;;AAa0C;;;;;;;iBAAnE,QAAA,oBAA4B,UAAA,WAAqB,UAAU,CAAC,OAAA"}

package/esm/eval/report-json.mjs

@@ -0,0 +1,33 @@
+//#region ../@warlock.js/ai/src/eval/report-json.ts
+/**
+* Serialize an {@link EvalReport} to a pretty-printed JSON string — a
+* round-trippable snapshot suitable for committing as the next run's
+* baseline (`agent.eval({ baseline: fromJSON(...) })`).
+*
+* Pure. The whole report is emitted verbatim; `result` payloads,
+* per-case `scores`, timings, and any attached `regression` block are all
+* preserved, so a parsed report drives regression diffing exactly as the
+* in-memory one would.
+*
+* @example
+* await writeFile("./eval/baseline.json", toJSON(report));
+*/
+function toJSON(report) {
+	return JSON.stringify(report, void 0, 2);
+}
+/**
+* Parse a string produced by {@link toJSON} back into an
+* {@link EvalReport}. The inverse of `toJSON` — `fromJSON(toJSON(r))`
+* reproduces `r`'s data (functions such as scorers were never part of the
+* serialized report, so the round-trip is over plain data only).
+*
+* @example
+* const baseline = fromJSON(await readFile("./eval/baseline.json", "utf8"));
+*/
+function fromJSON(serialized) {
+	return JSON.parse(serialized);
+}
+
+//#endregion
+export { fromJSON, toJSON };
+//# sourceMappingURL=report-json.mjs.map

package/esm/eval/report-json.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-json.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/report-json.ts"],"sourcesContent":["import type { EvalReport } from \"../contracts/agent/eval.type\";\n\n/**\n * Serialize an {@link EvalReport} to a pretty-printed JSON string — a\n * round-trippable snapshot suitable for committing as the next run's\n * baseline (`agent.eval({ baseline: fromJSON(...) })`).\n *\n * Pure. The whole report is emitted verbatim; `result` payloads,\n * per-case `scores`, timings, and any attached `regression` block are all\n * preserved, so a parsed report drives regression diffing exactly as the\n * in-memory one would.\n *\n * @example\n * await writeFile(\"./eval/baseline.json\", toJSON(report));\n */\nexport function toJSON(report: EvalReport): string {\n  return JSON.stringify(report, undefined, 2);\n}\n\n/**\n * Parse a string produced by {@link toJSON} back into an\n * {@link EvalReport}. The inverse of `toJSON` — `fromJSON(toJSON(r))`\n * reproduces `r`'s data (functions such as scorers were never part of the\n * serialized report, so the round-trip is over plain data only).\n *\n * @example\n * const baseline = fromJSON(await readFile(\"./eval/baseline.json\", \"utf8\"));\n */\nexport function fromJSON<TOutput = unknown>(serialized: string): EvalReport<TOutput> {\n  return JSON.parse(serialized) as EvalReport<TOutput>;\n}\n"],"mappings":";;;;;;;;;;;;;;AAeA,SAAgB,OAAO,QAA4B;CACjD,OAAO,KAAK,UAAU,QAAQ,QAAW,CAAC;AAC5C;;;;;;;;;;AAWA,SAAgB,SAA4B,YAAyC;CACnF,OAAO,KAAK,MAAM,UAAU;AAC9B"}

package/esm/eval/report-junit.d.mts

@@ -0,0 +1,22 @@
+import { EvalReport } from "../contracts/agent/eval.type.mjs";
+
+//#region ../@warlock.js/ai/src/eval/report-junit.d.ts
+/**
+ * Serialize an {@link EvalReport} to a JUnit-XML string for CI ingestion.
+ *
+ * **Role.** A pure, runner-decoupled reporter: one `<testsuite>` whose
+ * name is the agent, one `<testcase>` per eval case, a `<failure>` child
+ * on each case that did not pass (with the joined scorer reasons), and a
+ * `time` attribute carrying the case / suite duration in **seconds**
+ * (JUnit's unit; the report stores milliseconds).
+ *
+ * XML is hand-emitted (no `xml` dependency) and every dynamic value is
+ * entity-escaped via {@link escapeXml}.
+ *
+ * @example
+ * await writeFile("./report.junit.xml", toJUnit(report));
+ */
+declare function toJUnit(report: EvalReport): string;
+//#endregion
+export { toJUnit };
+//# sourceMappingURL=report-junit.d.mts.map

package/esm/eval/report-junit.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-junit.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/report-junit.ts"],"mappings":";;;;;AAqDA;;;;AAA0C;;;;;;;;;iBAA1B,OAAA,CAAQ,MAAkB,EAAV,UAAU"}

package/esm/eval/report-junit.mjs

@@ -0,0 +1,60 @@
+//#region ../@warlock.js/ai/src/eval/report-junit.ts
+/**
+* Escape the five XML predefined entities so arbitrary text (case names,
+* failure reasons, agent names) is safe inside an attribute value or
+* element body. Covers `&`, `<`, `>`, `"`, and `'`.
+*/
+function escapeXml(value) {
+	return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
+}
+/**
+* Build the `<failure>` body for a failed case: the joined reasons of
+* every non-passing scorer, falling back to a generic message when a
+* scorer offered no reason (or the failure was an agent error).
+*/
+function failureMessage(entry) {
+	if (entry.result.error) return `agent error: ${entry.result.error.message}`;
+	const reasons = entry.scores.filter((score) => score.passed === false).map((score) => score.reason).filter((reason) => typeof reason === "string" && reason !== "");
+	if (reasons.length > 0) return reasons.join("; ");
+	return "case did not pass";
+}
+/**
+* Serialize an {@link EvalReport} to a JUnit-XML string for CI ingestion.
+*
+* **Role.** A pure, runner-decoupled reporter: one `<testsuite>` whose
+* name is the agent, one `<testcase>` per eval case, a `<failure>` child
+* on each case that did not pass (with the joined scorer reasons), and a
+* `time` attribute carrying the case / suite duration in **seconds**
+* (JUnit's unit; the report stores milliseconds).
+*
+* XML is hand-emitted (no `xml` dependency) and every dynamic value is
+* entity-escaped via {@link escapeXml}.
+*
+* @example
+* await writeFile("./report.junit.xml", toJUnit(report));
+*/
+function toJUnit(report) {
+	const suiteName = escapeXml(report.agentName);
+	const suiteTime = (report.duration / 1e3).toFixed(3);
+	const lines = [];
+	lines.push("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+	lines.push(`<testsuite name="${suiteName}" tests="${report.total}" failures="${report.failedCount}" time="${suiteTime}">`);
+	for (const entry of report.cases) {
+		const caseName = escapeXml(entry.case.name);
+		const caseTime = (entry.duration / 1e3).toFixed(3);
+		if (entry.passed) {
+			lines.push(`  <testcase name="${caseName}" classname="${suiteName}" time="${caseTime}"/>`);
+			continue;
+		}
+		const message = failureMessage(entry);
+		lines.push(`  <testcase name="${caseName}" classname="${suiteName}" time="${caseTime}">`);
+		lines.push(`    <failure message="${escapeXml(message)}">${escapeXml(message)}</failure>`);
+		lines.push("  </testcase>");
+	}
+	lines.push("</testsuite>");
+	return lines.join("\n");
+}
+
+//#endregion
+export { toJUnit };
+//# sourceMappingURL=report-junit.mjs.map

package/esm/eval/report-junit.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-junit.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/report-junit.ts"],"sourcesContent":["import type { EvalCaseResult, EvalReport } from \"../contracts/agent/eval.type\";\n\n/**\n * Escape the five XML predefined entities so arbitrary text (case names,\n * failure reasons, agent names) is safe inside an attribute value or\n * element body. Covers `&`, `<`, `>`, `\"`, and `'`.\n */\nfunction escapeXml(value: string): string {\n  return value\n    .replace(/&/g, \"&amp;\")\n    .replace(/</g, \"&lt;\")\n    .replace(/>/g, \"&gt;\")\n    .replace(/\"/g, \"&quot;\")\n    .replace(/'/g, \"&apos;\");\n}\n\n/**\n * Build the `<failure>` body for a failed case: the joined reasons of\n * every non-passing scorer, falling back to a generic message when a\n * scorer offered no reason (or the failure was an agent error).\n */\nfunction failureMessage(entry: EvalCaseResult): string {\n  if (entry.result.error) {\n    return `agent error: ${entry.result.error.message}`;\n  }\n\n  const reasons = entry.scores\n    .filter((score) => score.passed === false)\n    .map((score) => score.reason)\n    .filter((reason): reason is string => typeof reason === \"string\" && reason !== \"\");\n\n  if (reasons.length > 0) {\n    return reasons.join(\"; \");\n  }\n\n  return \"case did not pass\";\n}\n\n/**\n * Serialize an {@link EvalReport} to a JUnit-XML string for CI ingestion.\n *\n * **Role.** A pure, runner-decoupled reporter: one `<testsuite>` whose\n * name is the agent, one `<testcase>` per eval case, a `<failure>` child\n * on each case that did not pass (with the joined scorer reasons), and a\n * `time` attribute carrying the case / suite duration in **seconds**\n * (JUnit's unit; the report stores milliseconds).\n *\n * XML is hand-emitted (no `xml` dependency) and every dynamic value is\n * entity-escaped via {@link escapeXml}.\n *\n * @example\n * await writeFile(\"./report.junit.xml\", toJUnit(report));\n */\nexport function toJUnit(report: EvalReport): string {\n  const suiteName = escapeXml(report.agentName);\n  const suiteTime = (report.duration / 1000).toFixed(3);\n\n  const lines: string[] = [];\n\n  lines.push('<?xml version=\"1.0\" encoding=\"UTF-8\"?>');\n  lines.push(\n    `<testsuite name=\"${suiteName}\" tests=\"${report.total}\" failures=\"${report.failedCount}\" time=\"${suiteTime}\">`,\n  );\n\n  for (const entry of report.cases) {\n    const caseName = escapeXml(entry.case.name);\n    const caseTime = (entry.duration / 1000).toFixed(3);\n\n    if (entry.passed) {\n      lines.push(\n        `  <testcase name=\"${caseName}\" classname=\"${suiteName}\" time=\"${caseTime}\"/>`,\n      );\n\n      continue;\n    }\n\n    const message = failureMessage(entry);\n    lines.push(\n      `  <testcase name=\"${caseName}\" classname=\"${suiteName}\" time=\"${caseTime}\">`,\n    );\n    lines.push(\n      `    <failure message=\"${escapeXml(message)}\">${escapeXml(message)}</failure>`,\n    );\n    lines.push(\"  </testcase>\");\n  }\n\n  lines.push(\"</testsuite>\");\n\n  return lines.join(\"\\n\");\n}\n"],"mappings":";;;;;;AAOA,SAAS,UAAU,OAAuB;CACxC,OAAO,MACJ,QAAQ,MAAM,OAAO,CAAC,CACtB,QAAQ,MAAM,MAAM,CAAC,CACrB,QAAQ,MAAM,MAAM,CAAC,CACrB,QAAQ,MAAM,QAAQ,CAAC,CACvB,QAAQ,MAAM,QAAQ;AAC3B;;;;;;AAOA,SAAS,eAAe,OAA+B;CACrD,IAAI,MAAM,OAAO,OACf,OAAO,gBAAgB,MAAM,OAAO,MAAM;CAG5C,MAAM,UAAU,MAAM,OACnB,QAAQ,UAAU,MAAM,WAAW,KAAK,CAAC,CACzC,KAAK,UAAU,MAAM,MAAM,CAAC,CAC5B,QAAQ,WAA6B,OAAO,WAAW,YAAY,WAAW,EAAE;CAEnF,IAAI,QAAQ,SAAS,GACnB,OAAO,QAAQ,KAAK,IAAI;CAG1B,OAAO;AACT;;;;;;;;;;;;;;;;AAiBA,SAAgB,QAAQ,QAA4B;CAClD,MAAM,YAAY,UAAU,OAAO,SAAS;CAC5C,MAAM,aAAa,OAAO,WAAW,IAAI,CAAE,QAAQ,CAAC;CAEpD,MAAM,QAAkB,CAAC;CAEzB,MAAM,KAAK,4CAAwC;CACnD,MAAM,KACJ,oBAAoB,UAAU,WAAW,OAAO,MAAM,cAAc,OAAO,YAAY,UAAU,UAAU,GAC7G;CAEA,KAAK,MAAM,SAAS,OAAO,OAAO;EAChC,MAAM,WAAW,UAAU,MAAM,KAAK,IAAI;EAC1C,MAAM,YAAY,MAAM,WAAW,IAAI,CAAE,QAAQ,CAAC;EAElD,IAAI,MAAM,QAAQ;GAChB,MAAM,KACJ,qBAAqB,SAAS,eAAe,UAAU,UAAU,SAAS,IAC5E;GAEA;EACF;EAEA,MAAM,UAAU,eAAe,KAAK;EACpC,MAAM,KACJ,qBAAqB,SAAS,eAAe,UAAU,UAAU,SAAS,GAC5E;EACA,MAAM,KACJ,yBAAyB,UAAU,OAAO,EAAE,IAAI,UAAU,OAAO,EAAE,WACrE;EACA,MAAM,KAAK,eAAe;CAC5B;CAEA,MAAM,KAAK,cAAc;CAEzB,OAAO,MAAM,KAAK,IAAI;AACxB"}

package/esm/guard/contracts/guard-options.type.d.mts

@@ -0,0 +1,165 @@
+import { MiddlewareTripContext } from "../../contracts/middleware/middleware-context.type.mjs";
+import { GuardrailAction, GuardrailMatch, GuardrailPhase } from "./verdict.type.mjs";
+import { GuardrailDetector } from "./guardrail.contract.mjs";
+import { OpenAiClientLike } from "./openai-client.contract.mjs";
+
+//#region ../@warlock.js/ai/src/guard/contracts/guard-options.type.d.ts
+/**
+ * The payload handed to {@link GuardrailEscalation.onBlock} when a detector
+ * returns `{ type: "block", escalate: true }`. The factory builds it just
+ * before throwing the `GuardrailViolationError`, so a host can route the
+ * hard block to a human-review surface.
+ */
+interface GuardrailBlockEvent {
+  /** Where the block fired: input prompt, output, or tool args. */
+  readonly phase: GuardrailPhase;
+  /** The detector's human-readable reason. */
+  readonly reason: string;
+  /** The matches that triggered the block, when the detector reported them. */
+  readonly matches?: readonly GuardrailMatch[];
+  /** The live middleware trip context (state, messages, agent, model, signal). */
+  readonly ctx: MiddlewareTripContext;
+}
+/**
+ * Escalation seam for composing a hard block with a human-review surface.
+ *
+ * The callback fires **only** for a verdict of `{ type: "block", escalate:
+ * true }`, awaited *before* the `GuardrailViolationError` is thrown. It is a
+ * plain callback by design — `ai-guard` takes no dependency on the deferred
+ * human-step machinery; the host wires its own review queue / resume loop
+ * inside `onBlock`.
+ */
+interface GuardrailEscalation {
+  /** Route a `{ type: "block", escalate: true }` verdict to a human. */
+  onBlock?(event: GuardrailBlockEvent): void | Promise<void>;
+}
+/**
+ * Configuration for the {@link guard} factory (surfaced as
+ * `ai.guardrail(options)`). Each phase array runs in registration order;
+ * the first non-`allow` verdict decides the action for that phase.
+ *
+ * At least one of `input` / `output` / `tool` should be supplied — a guard
+ * with no detectors is a no-op middleware.
+ */
+interface GuardOptions {
+  /**
+   * Override the middleware name — surfaces on
+   * `GuardrailViolationError.guardrail` and as the `ctx.state` namespace so
+   * operators can tell two guards apart. Default `"guardrail"`.
+   */
+  name?: string;
+  /**
+   * Detectors run on the outbound prompt at `trip.before`.
+   *
+   * **Input redaction is not supported in v1:** the core `trip.before` hook
+   * can only short-circuit, not rewrite-and-continue, so a `redact` verdict
+   * here is treated as a `block`. Input detectors are effectively
+   * `block` / `flag` only.
+   */
+  input?: readonly GuardrailDetector[];
+  /** Detectors run on `response.content` at `trip.after`. Support `redact`. */
+  output?: readonly GuardrailDetector[];
+  /**
+   * Detectors run on `JSON.stringify(toolArgs)` at `tool.before`. A
+   * `redact` verdict here is treated as a `block` (silently rewriting tool
+   * arguments changes side-effects unpredictably).
+   */
+  tool?: readonly GuardrailDetector[];
+  /**
+   * The tool names the `tool` detectors apply to. Omit to apply to every
+   * tool. When set, the middleware's `tool` hooks are scoped via the core
+   * `forTool(toolNames, mw)` helper; `trip` hooks are unaffected.
+   */
+  toolNames?: string | readonly string[];
+  /** Compose a `{ type: "block", escalate: true }` verdict with a human. */
+  escalation?: GuardrailEscalation;
+}
+/**
+ * The PII categories the built-in `pii` detector can scan for. Each is a
+ * linear (anchored, no nested quantifiers) regex — safe against catastrophic
+ * backtracking.
+ */
+type PiiCategory = "ssn" | "email" | "phone" | "credit-card" | "ipv4";
+/**
+ * Options for the built-in `pii` detector (`ai.guardrail.pii`). All built-in
+ * PII matching is regex / exact-string only — zero runtime dependency.
+ */
+interface PiiDetectorOptions {
+  /** Which categories to scan for. Default: every {@link PiiCategory}. */
+  detect?: readonly PiiCategory[];
+  /** What to do on a match. Default `"redact"`. */
+  onMatch?: Extract<GuardrailAction, "redact" | "block" | "flag">;
+  /**
+   * Replacement template used on `redact`. Supports the `{label}` token,
+   * substituted with the matched category — e.g. `"[REDACTED:{label}]"`.
+   * Default masks the matched span with a fixed placeholder.
+   */
+  mask?: string;
+  /** Extra exact-string terms to treat as PII alongside the built-in regexes. */
+  dictionary?: readonly string[];
+}
+/**
+ * Options for the built-in `topic` filter (`ai.guardrail.topic`). Matches a
+ * case-insensitive substring or a `RegExp` against the inspected text.
+ */
+interface TopicFilterOptions {
+  /**
+   * Deny-list terms / phrases. A `string` matches case-insensitively as a
+   * substring; a `RegExp` is tested as-is. Any hit triggers `onMatch`.
+   */
+  deny?: readonly (string | RegExp)[];
+  /**
+   * Allow-list terms / phrases. When set, text matching **none** of these
+   * triggers `onMatch` (an allow-list miss).
+   */
+  allow?: readonly (string | RegExp)[];
+  /** Action on a deny hit (or an allow-list miss). Default `"block"`. */
+  onMatch?: Extract<GuardrailAction, "block" | "flag">;
+  /** Override the verdict's human-readable reason. */
+  reason?: string;
+}
+/**
+ * Options for the built-in `injection` detector (`ai.guardrail.injection`).
+ * Matches a built-in set of jailbreak / prompt-injection marker phrases,
+ * extensible with caller-supplied markers.
+ */
+interface InjectionDetectorOptions {
+  /**
+   * Extra marker phrases beyond the built-in jailbreak / prompt-injection
+   * set. A `string` matches case-insensitively as a substring; a `RegExp`
+   * is tested as-is.
+   */
+  markers?: readonly (string | RegExp)[];
+  /**
+   * Action on a match. Default `"flag"` — callers commonly escalate to
+   * `"block"` on the input phase.
+   */
+  onMatch?: Extract<GuardrailAction, "block" | "flag">;
+}
+/**
+ * Options for the optional `moderation` detector (`ai.guardrail.moderation`),
+ * backed by a lazily-imported `openai` peer. Importing `@warlock.js/ai`
+ * never forces `openai` to resolve; the detector throws a curated install
+ * string on first `check()` when the SDK is absent.
+ */
+interface OpenAiModerationOptions {
+  /**
+   * A pre-built OpenAI-compatible client (any object matching
+   * {@link OpenAiClientLike}, including a real `OpenAI` instance). When
+   * supplied, the detector calls it directly and never imports the SDK —
+   * the bring-your-own-client / test escape hatch.
+   */
+  client?: OpenAiClientLike;
+  /** OpenAI API key. Reads `OPENAI_API_KEY` from the environment when omitted. */
+  apiKey?: string;
+  /** Moderation model to call. Default `"omni-moderation-latest"`. */
+  model?: string;
+  /**
+   * Categories that escalate to `block`; every other flagged category
+   * produces a `flag` verdict instead. Omit to `flag` on any category.
+   */
+  blockOn?: readonly string[];
+}
+//#endregion
+export { GuardOptions, GuardrailBlockEvent, GuardrailEscalation, InjectionDetectorOptions, OpenAiModerationOptions, PiiCategory, PiiDetectorOptions, TopicFilterOptions };
+//# sourceMappingURL=guard-options.type.d.mts.map

package/esm/guard/contracts/guard-options.type.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"guard-options.type.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/guard/contracts/guard-options.type.ts"],"mappings":";;;;;;;;AAeA;;;;UAAiB,mBAAA;EAQD;EAAA,SANL,KAAA,EAAO,cAAA;EAMmB;EAAA,SAJ1B,MAAA;EAFO;EAAA,SAIP,OAAA,YAAmB,cAAA;EAAnB;EAAA,SAEA,GAAA,EAAK,qBAAA;AAAA;;;AAAqB;AAYrC;;;;;;UAAiB,mBAAA;EAE8B;EAA7C,OAAA,EAAS,KAAA,EAAO,mBAAA,UAA6B,OAAO;AAAA;AAWtD;;;;;;;;AAAA,UAAiB,YAAA;EAMf;;;;;EAAA,IAAA;EAiBgB;;;;;AAQgB;AAQlC;;EAxBE,KAAA,YAAiB,iBAAA;EAwBI;EAtBrB,MAAA,YAAkB,iBAAA;EA4BH;;;;;EAtBf,IAAA,YAAgB,iBAAA;EA0BC;;;;;EApBjB,SAAA;EAoBkB;EAlBlB,UAAA,GAAa,mBAAA;AAAA;;AA0BH;AAOZ;;;KAzBY,WAAA;;;;;UAMK,kBAAA;EAwBf;EAtBA,MAAA,YAAkB,WAAA;EA2BlB;EAzBA,OAAA,GAAU,OAAA,CAAQ,eAAA;EA2BlB;;;;;EArBA,IAAA;EA+Be;EA7Bf,UAAA;AAAA;;;;;UAOe,kBAAA;EA4Bf;;;;EAvBA,IAAA,sBAA0B,MAAA;EA4BO;AAAA;AASnC;;EAhCE,KAAA,sBAA2B,MAAA;EAuCF;EArCzB,OAAA,GAAU,OAAA,CAAQ,eAAA;EAqCT;EAnCT,MAAA;AAAA;;;AA4CO;;;UApCQ,wBAAA;;;;;;EAMf,OAAA,sBAA6B,MAAA;;;;;EAK7B,OAAA,GAAU,OAAA,CAAQ,eAAA;AAAA;;;;;;;UASH,uBAAA;;;;;;;EAOf,MAAA,GAAS,gBAAgB;;EAEzB,MAAA;;EAEA,KAAA;;;;;EAKA,OAAA;AAAA"}