@warlock.js/core 4.4.0 → 4.6.0

package/esm/ai/src/eval/judge-scorer.mjs

@@ -0,0 +1,87 @@
+import { extractJsonPayload } from "../utils/extract-json-payload.mjs";
+import { safeJsonParse } from "../utils/safe-json-parse.mjs";
+import "../utils/index.mjs";
+
+//#region ../@warlock.js/ai/src/eval/judge-scorer.ts
+/** Clamp an arbitrary numeric value into the `[0, 1]` score range. */
+function clampScore(value) {
+	if (Number.isNaN(value)) return 0;
+	if (value < 0) return 0;
+	if (value > 1) return 1;
+	return value;
+}
+/**
+* Render the prompt the judge agent receives for one case. Includes
+* the rubric (if any), the original question, the expected reference
+* (when supplied), and the actual answer — then asks for a strict JSON
+* verdict so the response is machine-parseable even without an output
+* schema.
+*/
+function buildJudgePrompt(context, judge) {
+	const lines = [];
+	if (judge.rubric) lines.push(`Grading rubric:\n${judge.rubric}`, "");
+	lines.push(`Question:\n${context.case.input}`, "");
+	if (context.case.expected !== void 0) {
+		const expectedText = typeof context.case.expected === "string" ? context.case.expected : JSON.stringify(context.case.expected);
+		lines.push(`Reference answer:\n${expectedText}`, "");
+	}
+	const actual = context.text ?? JSON.stringify(context.output ?? null);
+	lines.push(`Answer to grade:\n${actual}`, "");
+	lines.push("Respond with JSON only: { \"score\": <0..1>, \"passed\": <true|false>, \"reason\": \"<short explanation>\" }.");
+	return lines.join("\n");
+}
+/**
+* Coerce the judge agent's parsed/extracted verdict into a normalized
+* {@link EvalScore}. Defends against a judge that returns a string
+* score or omits `passed` — the caller-supplied `passThreshold`
+* derives `passed` from `score` when the judge didn't decide.
+*/
+function toEvalScore(verdict, passThreshold) {
+	const rawScore = typeof verdict.score === "string" ? Number(verdict.score) : verdict.score;
+	const score = clampScore(typeof rawScore === "number" ? rawScore : 0);
+	return {
+		score,
+		passed: typeof verdict.passed === "boolean" ? verdict.passed : score >= passThreshold,
+		reason: typeof verdict.reason === "string" ? verdict.reason : void 0
+	};
+}
+/**
+* LLM-as-judge scorer. Runs the judge agent on a prompt built from the
+* case (question + expected + actual answer + rubric) and parses its
+* `{ score, passed?, reason? }` verdict.
+*
+* Verdict source order: `result.data` (when the judge declares an
+* output schema), then `result.text` parsed as JSON. A judge that
+* errors or returns unparseable text scores `0` with the failure
+* reason attached — a broken judge fails the case rather than crashing
+* the suite.
+*
+* @example
+* scorers: [judge({ agent: judgeAgent, rubric: "Cite a source for full marks." })]
+*/
+function judge(config, passThreshold = .5) {
+	const threshold = config.passThreshold ?? passThreshold;
+	return async (context) => {
+		const prompt = buildJudgePrompt(context, config);
+		const verdictResult = await config.agent.execute(prompt);
+		if (verdictResult.error) return {
+			score: 0,
+			passed: false,
+			reason: `judge failed: ${verdictResult.error.message}`
+		};
+		if (verdictResult.data && typeof verdictResult.data === "object") return toEvalScore(verdictResult.data, threshold);
+		const text = verdictResult.text ?? "";
+		const sentinel = Symbol("judge-parse-failed");
+		const parsed = safeJsonParse(extractJsonPayload(text), sentinel);
+		if (parsed === sentinel || parsed === null || typeof parsed !== "object") return {
+			score: 0,
+			passed: false,
+			reason: "judge returned no parseable verdict"
+		};
+		return toEvalScore(parsed, threshold);
+	};
+}
+
+//#endregion
+export { judge };
+//# sourceMappingURL=judge-scorer.mjs.map

package/esm/ai/src/eval/judge-scorer.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge-scorer.mjs","names":[],"sources":["../../../../../../../../../@warlock.js/ai/src/eval/judge-scorer.ts"],"sourcesContent":["import type {\n  EvalJudge,\n  EvalScore,\n  EvalScorer,\n  EvalScorerContext,\n} from \"../contracts/agent/eval.type\";\nimport { extractJsonPayload, safeJsonParse } from \"../utils\";\n\n/**\n * Raw shape the judge agent is expected to return — either parsed from\n * `result.data` (when the judge has an output schema) or extracted\n * from `result.text`.\n */\ntype JudgeVerdict = {\n  score?: unknown;\n  passed?: unknown;\n  reason?: unknown;\n};\n\n/** Clamp an arbitrary numeric value into the `[0, 1]` score range. */\nfunction clampScore(value: number): number {\n  if (Number.isNaN(value)) return 0;\n  if (value < 0) return 0;\n  if (value > 1) return 1;\n  return value;\n}\n\n/**\n * Render the prompt the judge agent receives for one case. Includes\n * the rubric (if any), the original question, the expected reference\n * (when supplied), and the actual answer — then asks for a strict JSON\n * verdict so the response is machine-parseable even without an output\n * schema.\n */\nfunction buildJudgePrompt(context: EvalScorerContext, judge: EvalJudge): string {\n  const lines: string[] = [];\n\n  if (judge.rubric) {\n    lines.push(`Grading rubric:\\n${judge.rubric}`, \"\");\n  }\n\n  lines.push(`Question:\\n${context.case.input}`, \"\");\n\n  if (context.case.expected !== undefined) {\n    const expectedText =\n      typeof context.case.expected === \"string\"\n        ? context.case.expected\n        : JSON.stringify(context.case.expected);\n    lines.push(`Reference answer:\\n${expectedText}`, \"\");\n  }\n\n  const actual = context.text ?? JSON.stringify(context.output ?? null);\n  lines.push(`Answer to grade:\\n${actual}`, \"\");\n\n  lines.push(\n    'Respond with JSON only: { \"score\": <0..1>, \"passed\": <true|false>, \"reason\": \"<short explanation>\" }.',\n  );\n\n  return lines.join(\"\\n\");\n}\n\n/**\n * Coerce the judge agent's parsed/extracted verdict into a normalized\n * {@link EvalScore}. Defends against a judge that returns a string\n * score or omits `passed` — the caller-supplied `passThreshold`\n * derives `passed` from `score` when the judge didn't decide.\n */\nfunction toEvalScore(verdict: JudgeVerdict, passThreshold: number): EvalScore {\n  const rawScore = typeof verdict.score === \"string\" ? Number(verdict.score) : verdict.score;\n  const score = clampScore(typeof rawScore === \"number\" ? rawScore : 0);\n\n  const passed = typeof verdict.passed === \"boolean\" ? verdict.passed : score >= passThreshold;\n\n  const reason = typeof verdict.reason === \"string\" ? verdict.reason : undefined;\n\n  return { score, passed, reason };\n}\n\n/**\n * LLM-as-judge scorer. Runs the judge agent on a prompt built from the\n * case (question + expected + actual answer + rubric) and parses its\n * `{ score, passed?, reason? }` verdict.\n *\n * Verdict source order: `result.data` (when the judge declares an\n * output schema), then `result.text` parsed as JSON. A judge that\n * errors or returns unparseable text scores `0` with the failure\n * reason attached — a broken judge fails the case rather than crashing\n * the suite.\n *\n * @example\n * scorers: [judge({ agent: judgeAgent, rubric: \"Cite a source for full marks.\" })]\n */\nexport function judge<TOutput = unknown>(\n  config: EvalJudge,\n  passThreshold = 0.5,\n): EvalScorer<TOutput> {\n  const threshold = config.passThreshold ?? passThreshold;\n\n  return async (context: EvalScorerContext<TOutput>): Promise<EvalScore> => {\n    const prompt = buildJudgePrompt(context as EvalScorerContext, config);\n\n    const verdictResult = await config.agent.execute(prompt);\n\n    if (verdictResult.error) {\n      return {\n        score: 0,\n        passed: false,\n        reason: `judge failed: ${verdictResult.error.message}`,\n      };\n    }\n\n    if (verdictResult.data && typeof verdictResult.data === \"object\") {\n      return toEvalScore(verdictResult.data as JudgeVerdict, threshold);\n    }\n\n    const text = verdictResult.text ?? \"\";\n    const sentinel = Symbol(\"judge-parse-failed\");\n    const parsed = safeJsonParse<unknown>(extractJsonPayload(text), sentinel);\n\n    if (parsed === sentinel || parsed === null || typeof parsed !== \"object\") {\n      return {\n        score: 0,\n        passed: false,\n        reason: \"judge returned no parseable verdict\",\n      };\n    }\n\n    return toEvalScore(parsed as JudgeVerdict, threshold);\n  };\n}\n"],"mappings":";;;;;;AAoBA,SAAS,WAAW,OAAuB;CACzC,IAAI,OAAO,MAAM,KAAK,GAAG,OAAO;CAChC,IAAI,QAAQ,GAAG,OAAO;CACtB,IAAI,QAAQ,GAAG,OAAO;CACtB,OAAO;AACT;;;;;;;;AASA,SAAS,iBAAiB,SAA4B,OAA0B;CAC9E,MAAM,QAAkB,CAAC;CAEzB,IAAI,MAAM,QACR,MAAM,KAAK,oBAAoB,MAAM,UAAU,EAAE;CAGnD,MAAM,KAAK,cAAc,QAAQ,KAAK,SAAS,EAAE;CAEjD,IAAI,QAAQ,KAAK,aAAa,QAAW;EACvC,MAAM,eACJ,OAAO,QAAQ,KAAK,aAAa,WAC7B,QAAQ,KAAK,WACb,KAAK,UAAU,QAAQ,KAAK,QAAQ;EAC1C,MAAM,KAAK,sBAAsB,gBAAgB,EAAE;CACrD;CAEA,MAAM,SAAS,QAAQ,QAAQ,KAAK,UAAU,QAAQ,UAAU,IAAI;CACpE,MAAM,KAAK,qBAAqB,UAAU,EAAE;CAE5C,MAAM,KACJ,+GACF;CAEA,OAAO,MAAM,KAAK,IAAI;AACxB;;;;;;;AAQA,SAAS,YAAY,SAAuB,eAAkC;CAC5E,MAAM,WAAW,OAAO,QAAQ,UAAU,WAAW,OAAO,QAAQ,KAAK,IAAI,QAAQ;CACrF,MAAM,QAAQ,WAAW,OAAO,aAAa,WAAW,WAAW,CAAC;CAMpE,OAAO;EAAE;EAAO,QAJD,OAAO,QAAQ,WAAW,YAAY,QAAQ,SAAS,SAAS;EAIvD,QAFT,OAAO,QAAQ,WAAW,WAAW,QAAQ,SAAS;CAEtC;AACjC;;;;;;;;;;;;;;;AAgBA,SAAgB,MACd,QACA,gBAAgB,IACK;CACrB,MAAM,YAAY,OAAO,iBAAiB;CAE1C,OAAO,OAAO,YAA4D;EACxE,MAAM,SAAS,iBAAiB,SAA8B,MAAM;EAEpE,MAAM,gBAAgB,MAAM,OAAO,MAAM,QAAQ,MAAM;EAEvD,IAAI,cAAc,OAChB,OAAO;GACL,OAAO;GACP,QAAQ;GACR,QAAQ,iBAAiB,cAAc,MAAM;EAC/C;EAGF,IAAI,cAAc,QAAQ,OAAO,cAAc,SAAS,UACtD,OAAO,YAAY,cAAc,MAAsB,SAAS;EAGlE,MAAM,OAAO,cAAc,QAAQ;EACnC,MAAM,WAAW,OAAO,oBAAoB;EAC5C,MAAM,SAAS,cAAuB,mBAAmB,IAAI,GAAG,QAAQ;EAExE,IAAI,WAAW,YAAY,WAAW,QAAQ,OAAO,WAAW,UAC9D,OAAO;GACL,OAAO;GACP,QAAQ;GACR,QAAQ;EACV;EAGF,OAAO,YAAY,QAAwB,SAAS;CACtD;AACF"}

package/esm/ai/src/eval/regression.mjs

@@ -0,0 +1,51 @@
+//#region ../@warlock.js/ai/src/eval/regression.ts
+/**
+* Diff a fresh {@link EvalReport} against a `baseline`, joining cases by
+* name, to produce an {@link EvalRegression} verdict.
+*
+* A case **regresses** when its new aggregate `score` is more than
+* `tolerance` below its baseline score (`before - after > tolerance`).
+* Cases that improved, held steady, or moved within `tolerance` are not
+* flagged. Cases present in only one of the two reports are surfaced
+* under `added` / `removed` rather than treated as regressions, so adding
+* or dropping a case never fails the gate by itself.
+*
+* Pure — depends only on the two reports and the tolerance; attaches no
+* state and mutates neither input.
+*
+* @param report - The newly produced report.
+* @param baseline - A prior report to compare against.
+* @param tolerance - Max allowed score drop before a case counts as a
+*   regression. Defaults to `0` (any drop regresses).
+*
+* @example
+* const regression = diff(report, baseline, 0.05);
+* expect(regression.passed).toBe(true);
+*/
+function diff(report, baseline, tolerance = 0) {
+	const baselineScores = /* @__PURE__ */ new Map();
+	for (const entry of baseline.cases) baselineScores.set(entry.case.name, entry.score);
+	const currentNames = /* @__PURE__ */ new Set();
+	const regressed = [];
+	for (const entry of report.cases) {
+		const name = entry.case.name;
+		currentNames.add(name);
+		const before = baselineScores.get(name);
+		if (before === void 0) continue;
+		if (before - entry.score > tolerance) regressed.push({
+			name,
+			before,
+			after: entry.score
+		});
+	}
+	return {
+		regressed,
+		removed: baseline.cases.map((entry) => entry.case.name).filter((name) => !currentNames.has(name)),
+		added: report.cases.map((entry) => entry.case.name).filter((name) => !baselineScores.has(name)),
+		passed: regressed.length === 0
+	};
+}
+
+//#endregion
+export { diff };
+//# sourceMappingURL=regression.mjs.map

package/esm/ai/src/eval/regression.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"regression.mjs","names":[],"sources":["../../../../../../../../../@warlock.js/ai/src/eval/regression.ts"],"sourcesContent":["import type { EvalRegression, EvalReport } from \"../contracts/agent/eval.type\";\n\n/**\n * Diff a fresh {@link EvalReport} against a `baseline`, joining cases by\n * name, to produce an {@link EvalRegression} verdict.\n *\n * A case **regresses** when its new aggregate `score` is more than\n * `tolerance` below its baseline score (`before - after > tolerance`).\n * Cases that improved, held steady, or moved within `tolerance` are not\n * flagged. Cases present in only one of the two reports are surfaced\n * under `added` / `removed` rather than treated as regressions, so adding\n * or dropping a case never fails the gate by itself.\n *\n * Pure — depends only on the two reports and the tolerance; attaches no\n * state and mutates neither input.\n *\n * @param report - The newly produced report.\n * @param baseline - A prior report to compare against.\n * @param tolerance - Max allowed score drop before a case counts as a\n *   regression. Defaults to `0` (any drop regresses).\n *\n * @example\n * const regression = diff(report, baseline, 0.05);\n * expect(regression.passed).toBe(true);\n */\nexport function diff<TOutput = unknown>(\n  report: EvalReport<TOutput>,\n  baseline: EvalReport<TOutput>,\n  tolerance = 0,\n): EvalRegression {\n  const baselineScores = new Map<string, number>();\n\n  for (const entry of baseline.cases) {\n    baselineScores.set(entry.case.name, entry.score);\n  }\n\n  const currentNames = new Set<string>();\n  const regressed: EvalRegression[\"regressed\"] = [];\n\n  for (const entry of report.cases) {\n    const name = entry.case.name;\n    currentNames.add(name);\n\n    const before = baselineScores.get(name);\n\n    if (before === undefined) {\n      continue;\n    }\n\n    if (before - entry.score > tolerance) {\n      regressed.push({ name, before, after: entry.score });\n    }\n  }\n\n  const removed = baseline.cases\n    .map((entry) => entry.case.name)\n    .filter((name) => !currentNames.has(name));\n\n  const added = report.cases\n    .map((entry) => entry.case.name)\n    .filter((name) => !baselineScores.has(name));\n\n  return {\n    regressed,\n    removed,\n    added,\n    passed: regressed.length === 0,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAyBA,SAAgB,KACd,QACA,UACA,YAAY,GACI;CAChB,MAAM,iCAAiB,IAAI,IAAoB;CAE/C,KAAK,MAAM,SAAS,SAAS,OAC3B,eAAe,IAAI,MAAM,KAAK,MAAM,MAAM,KAAK;CAGjD,MAAM,+BAAe,IAAI,IAAY;CACrC,MAAM,YAAyC,CAAC;CAEhD,KAAK,MAAM,SAAS,OAAO,OAAO;EAChC,MAAM,OAAO,MAAM,KAAK;EACxB,aAAa,IAAI,IAAI;EAErB,MAAM,SAAS,eAAe,IAAI,IAAI;EAEtC,IAAI,WAAW,QACb;EAGF,IAAI,SAAS,MAAM,QAAQ,WACzB,UAAU,KAAK;GAAE;GAAM;GAAQ,OAAO,MAAM;EAAM,CAAC;CAEvD;CAUA,OAAO;EACL;EACA,SAVc,SAAS,MACtB,KAAK,UAAU,MAAM,KAAK,IAAI,CAAC,CAC/B,QAAQ,SAAS,CAAC,aAAa,IAAI,IAAI,CAQlC;EACN,OAPY,OAAO,MAClB,KAAK,UAAU,MAAM,KAAK,IAAI,CAAC,CAC/B,QAAQ,SAAS,CAAC,eAAe,IAAI,IAAI,CAKtC;EACJ,QAAQ,UAAU,WAAW;CAC/B;AACF"}

package/esm/ai/src/eval/report-json.mjs

@@ -0,0 +1,33 @@
+//#region ../@warlock.js/ai/src/eval/report-json.ts
+/**
+* Serialize an {@link EvalReport} to a pretty-printed JSON string — a
+* round-trippable snapshot suitable for committing as the next run's
+* baseline (`agent.eval({ baseline: fromJSON(...) })`).
+*
+* Pure. The whole report is emitted verbatim; `result` payloads,
+* per-case `scores`, timings, and any attached `regression` block are all
+* preserved, so a parsed report drives regression diffing exactly as the
+* in-memory one would.
+*
+* @example
+* await writeFile("./eval/baseline.json", toJSON(report));
+*/
+function toJSON(report) {
+	return JSON.stringify(report, void 0, 2);
+}
+/**
+* Parse a string produced by {@link toJSON} back into an
+* {@link EvalReport}. The inverse of `toJSON` — `fromJSON(toJSON(r))`
+* reproduces `r`'s data (functions such as scorers were never part of the
+* serialized report, so the round-trip is over plain data only).
+*
+* @example
+* const baseline = fromJSON(await readFile("./eval/baseline.json", "utf8"));
+*/
+function fromJSON(serialized) {
+	return JSON.parse(serialized);
+}
+
+//#endregion
+export { fromJSON, toJSON };
+//# sourceMappingURL=report-json.mjs.map

package/esm/ai/src/eval/report-json.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-json.mjs","names":[],"sources":["../../../../../../../../../@warlock.js/ai/src/eval/report-json.ts"],"sourcesContent":["import type { EvalReport } from \"../contracts/agent/eval.type\";\n\n/**\n * Serialize an {@link EvalReport} to a pretty-printed JSON string — a\n * round-trippable snapshot suitable for committing as the next run's\n * baseline (`agent.eval({ baseline: fromJSON(...) })`).\n *\n * Pure. The whole report is emitted verbatim; `result` payloads,\n * per-case `scores`, timings, and any attached `regression` block are all\n * preserved, so a parsed report drives regression diffing exactly as the\n * in-memory one would.\n *\n * @example\n * await writeFile(\"./eval/baseline.json\", toJSON(report));\n */\nexport function toJSON(report: EvalReport): string {\n  return JSON.stringify(report, undefined, 2);\n}\n\n/**\n * Parse a string produced by {@link toJSON} back into an\n * {@link EvalReport}. The inverse of `toJSON` — `fromJSON(toJSON(r))`\n * reproduces `r`'s data (functions such as scorers were never part of the\n * serialized report, so the round-trip is over plain data only).\n *\n * @example\n * const baseline = fromJSON(await readFile(\"./eval/baseline.json\", \"utf8\"));\n */\nexport function fromJSON<TOutput = unknown>(serialized: string): EvalReport<TOutput> {\n  return JSON.parse(serialized) as EvalReport<TOutput>;\n}\n"],"mappings":";;;;;;;;;;;;;;AAeA,SAAgB,OAAO,QAA4B;CACjD,OAAO,KAAK,UAAU,QAAQ,QAAW,CAAC;AAC5C;;;;;;;;;;AAWA,SAAgB,SAA4B,YAAyC;CACnF,OAAO,KAAK,MAAM,UAAU;AAC9B"}

package/esm/ai/src/eval/report-junit.mjs

@@ -0,0 +1,60 @@
+//#region ../@warlock.js/ai/src/eval/report-junit.ts
+/**
+* Escape the five XML predefined entities so arbitrary text (case names,
+* failure reasons, agent names) is safe inside an attribute value or
+* element body. Covers `&`, `<`, `>`, `"`, and `'`.
+*/
+function escapeXml(value) {
+	return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
+}
+/**
+* Build the `<failure>` body for a failed case: the joined reasons of
+* every non-passing scorer, falling back to a generic message when a
+* scorer offered no reason (or the failure was an agent error).
+*/
+function failureMessage(entry) {
+	if (entry.result.error) return `agent error: ${entry.result.error.message}`;
+	const reasons = entry.scores.filter((score) => score.passed === false).map((score) => score.reason).filter((reason) => typeof reason === "string" && reason !== "");
+	if (reasons.length > 0) return reasons.join("; ");
+	return "case did not pass";
+}
+/**
+* Serialize an {@link EvalReport} to a JUnit-XML string for CI ingestion.
+*
+* **Role.** A pure, runner-decoupled reporter: one `<testsuite>` whose
+* name is the agent, one `<testcase>` per eval case, a `<failure>` child
+* on each case that did not pass (with the joined scorer reasons), and a
+* `time` attribute carrying the case / suite duration in **seconds**
+* (JUnit's unit; the report stores milliseconds).
+*
+* XML is hand-emitted (no `xml` dependency) and every dynamic value is
+* entity-escaped via {@link escapeXml}.
+*
+* @example
+* await writeFile("./report.junit.xml", toJUnit(report));
+*/
+function toJUnit(report) {
+	const suiteName = escapeXml(report.agentName);
+	const suiteTime = (report.duration / 1e3).toFixed(3);
+	const lines = [];
+	lines.push("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+	lines.push(`<testsuite name="${suiteName}" tests="${report.total}" failures="${report.failedCount}" time="${suiteTime}">`);
+	for (const entry of report.cases) {
+		const caseName = escapeXml(entry.case.name);
+		const caseTime = (entry.duration / 1e3).toFixed(3);
+		if (entry.passed) {
+			lines.push(`  <testcase name="${caseName}" classname="${suiteName}" time="${caseTime}"/>`);
+			continue;
+		}
+		const message = failureMessage(entry);
+		lines.push(`  <testcase name="${caseName}" classname="${suiteName}" time="${caseTime}">`);
+		lines.push(`    <failure message="${escapeXml(message)}">${escapeXml(message)}</failure>`);
+		lines.push("  </testcase>");
+	}
+	lines.push("</testsuite>");
+	return lines.join("\n");
+}
+
+//#endregion
+export { toJUnit };
+//# sourceMappingURL=report-junit.mjs.map

package/esm/ai/src/eval/report-junit.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-junit.mjs","names":[],"sources":["../../../../../../../../../@warlock.js/ai/src/eval/report-junit.ts"],"sourcesContent":["import type { EvalCaseResult, EvalReport } from \"../contracts/agent/eval.type\";\n\n/**\n * Escape the five XML predefined entities so arbitrary text (case names,\n * failure reasons, agent names) is safe inside an attribute value or\n * element body. Covers `&`, `<`, `>`, `\"`, and `'`.\n */\nfunction escapeXml(value: string): string {\n  return value\n    .replace(/&/g, \"&amp;\")\n    .replace(/</g, \"&lt;\")\n    .replace(/>/g, \"&gt;\")\n    .replace(/\"/g, \"&quot;\")\n    .replace(/'/g, \"&apos;\");\n}\n\n/**\n * Build the `<failure>` body for a failed case: the joined reasons of\n * every non-passing scorer, falling back to a generic message when a\n * scorer offered no reason (or the failure was an agent error).\n */\nfunction failureMessage(entry: EvalCaseResult): string {\n  if (entry.result.error) {\n    return `agent error: ${entry.result.error.message}`;\n  }\n\n  const reasons = entry.scores\n    .filter((score) => score.passed === false)\n    .map((score) => score.reason)\n    .filter((reason): reason is string => typeof reason === \"string\" && reason !== \"\");\n\n  if (reasons.length > 0) {\n    return reasons.join(\"; \");\n  }\n\n  return \"case did not pass\";\n}\n\n/**\n * Serialize an {@link EvalReport} to a JUnit-XML string for CI ingestion.\n *\n * **Role.** A pure, runner-decoupled reporter: one `<testsuite>` whose\n * name is the agent, one `<testcase>` per eval case, a `<failure>` child\n * on each case that did not pass (with the joined scorer reasons), and a\n * `time` attribute carrying the case / suite duration in **seconds**\n * (JUnit's unit; the report stores milliseconds).\n *\n * XML is hand-emitted (no `xml` dependency) and every dynamic value is\n * entity-escaped via {@link escapeXml}.\n *\n * @example\n * await writeFile(\"./report.junit.xml\", toJUnit(report));\n */\nexport function toJUnit(report: EvalReport): string {\n  const suiteName = escapeXml(report.agentName);\n  const suiteTime = (report.duration / 1000).toFixed(3);\n\n  const lines: string[] = [];\n\n  lines.push('<?xml version=\"1.0\" encoding=\"UTF-8\"?>');\n  lines.push(\n    `<testsuite name=\"${suiteName}\" tests=\"${report.total}\" failures=\"${report.failedCount}\" time=\"${suiteTime}\">`,\n  );\n\n  for (const entry of report.cases) {\n    const caseName = escapeXml(entry.case.name);\n    const caseTime = (entry.duration / 1000).toFixed(3);\n\n    if (entry.passed) {\n      lines.push(\n        `  <testcase name=\"${caseName}\" classname=\"${suiteName}\" time=\"${caseTime}\"/>`,\n      );\n\n      continue;\n    }\n\n    const message = failureMessage(entry);\n    lines.push(\n      `  <testcase name=\"${caseName}\" classname=\"${suiteName}\" time=\"${caseTime}\">`,\n    );\n    lines.push(\n      `    <failure message=\"${escapeXml(message)}\">${escapeXml(message)}</failure>`,\n    );\n    lines.push(\"  </testcase>\");\n  }\n\n  lines.push(\"</testsuite>\");\n\n  return lines.join(\"\\n\");\n}\n"],"mappings":";;;;;;AAOA,SAAS,UAAU,OAAuB;CACxC,OAAO,MACJ,QAAQ,MAAM,OAAO,CAAC,CACtB,QAAQ,MAAM,MAAM,CAAC,CACrB,QAAQ,MAAM,MAAM,CAAC,CACrB,QAAQ,MAAM,QAAQ,CAAC,CACvB,QAAQ,MAAM,QAAQ;AAC3B;;;;;;AAOA,SAAS,eAAe,OAA+B;CACrD,IAAI,MAAM,OAAO,OACf,OAAO,gBAAgB,MAAM,OAAO,MAAM;CAG5C,MAAM,UAAU,MAAM,OACnB,QAAQ,UAAU,MAAM,WAAW,KAAK,CAAC,CACzC,KAAK,UAAU,MAAM,MAAM,CAAC,CAC5B,QAAQ,WAA6B,OAAO,WAAW,YAAY,WAAW,EAAE;CAEnF,IAAI,QAAQ,SAAS,GACnB,OAAO,QAAQ,KAAK,IAAI;CAG1B,OAAO;AACT;;;;;;;;;;;;;;;;AAiBA,SAAgB,QAAQ,QAA4B;CAClD,MAAM,YAAY,UAAU,OAAO,SAAS;CAC5C,MAAM,aAAa,OAAO,WAAW,IAAI,CAAE,QAAQ,CAAC;CAEpD,MAAM,QAAkB,CAAC;CAEzB,MAAM,KAAK,4CAAwC;CACnD,MAAM,KACJ,oBAAoB,UAAU,WAAW,OAAO,MAAM,cAAc,OAAO,YAAY,UAAU,UAAU,GAC7G;CAEA,KAAK,MAAM,SAAS,OAAO,OAAO;EAChC,MAAM,WAAW,UAAU,MAAM,KAAK,IAAI;EAC1C,MAAM,YAAY,MAAM,WAAW,IAAI,CAAE,QAAQ,CAAC;EAElD,IAAI,MAAM,QAAQ;GAChB,MAAM,KACJ,qBAAqB,SAAS,eAAe,UAAU,UAAU,SAAS,IAC5E;GAEA;EACF;EAEA,MAAM,UAAU,eAAe,KAAK;EACpC,MAAM,KACJ,qBAAqB,SAAS,eAAe,UAAU,UAAU,SAAS,GAC5E;EACA,MAAM,KACJ,yBAAyB,UAAU,OAAO,EAAE,IAAI,UAAU,OAAO,EAAE,WACrE;EACA,MAAM,KAAK,eAAe;CAC5B;CAEA,MAAM,KAAK,cAAc;CAEzB,OAAO,MAAM,KAAK,IAAI;AACxB"}

package/esm/ai/src/eval/scorers.mjs

@@ -0,0 +1,101 @@
+//#region ../@warlock.js/ai/src/eval/scorers.ts
+/**
+* Normalize a value for case-insensitive, whitespace-trimmed string
+* comparison. Non-string values are JSON-serialized first so a
+* structured `expected` can still be matched against structured
+* `output`.
+*/
+function normalizeForComparison(value) {
+	return (typeof value === "string" ? value : JSON.stringify(value)).trim().toLowerCase();
+}
+/**
+* Exact-match scorer. Compares the agent's output against the case's
+* `expected` reference. Prefers `result.data` (parsed structured
+* output) when present, falling back to `result.text`. Comparison is
+* trimmed and case-insensitive; structured values are compared by
+* canonical JSON.
+*
+* Scores `1` / `passed: true` on a match, `0` / `passed: false`
+* otherwise. A case with no `expected` always scores `0` — exact
+* matching is meaningless without a reference.
+*
+* @example
+* const report = await agent.eval({
+*   cases: [{ name: "q", input: "2+2?", expected: "4" }],
+*   scorers: [exact()],
+* });
+*/
+function exact() {
+	return (context) => {
+		if (context.case.expected === void 0) return {
+			score: 0,
+			passed: false,
+			reason: "no expected value supplied for exact match"
+		};
+		const actual = context.output ?? context.text;
+		if (actual === void 0) return {
+			score: 0,
+			passed: false,
+			reason: "agent produced no output"
+		};
+		const matches = normalizeForComparison(actual) === normalizeForComparison(context.case.expected);
+		return {
+			score: matches ? 1 : 0,
+			passed: matches,
+			reason: matches ? "exact match" : "output did not match expected"
+		};
+	};
+}
+/**
+* Substring / contains scorer. Passes when the normalized `expected`
+* string appears anywhere in the agent's normalized output. Useful
+* when the agent's phrasing varies but a key fact must be present.
+*
+* @example
+* scorers: [contains()] // expected "Cairo" passes "The capital is Cairo."
+*/
+function contains() {
+	return (context) => {
+		if (context.case.expected === void 0) return {
+			score: 0,
+			passed: false,
+			reason: "no expected value supplied for contains match"
+		};
+		const actual = context.output ?? context.text;
+		if (actual === void 0) return {
+			score: 0,
+			passed: false,
+			reason: "agent produced no output"
+		};
+		const found = normalizeForComparison(actual).includes(normalizeForComparison(context.case.expected));
+		return {
+			score: found ? 1 : 0,
+			passed: found,
+			reason: found ? "expected substring found" : "expected substring not found"
+		};
+	};
+}
+/**
+* Predicate scorer. Wraps a boolean-returning callback into a scorer —
+* `true` scores `1` / `passed`, `false` scores `0` / fails. The
+* escape hatch for arbitrary assertions ("output is valid JSON", "no
+* tool errored", "duration under budget") that don't fit exact or
+* judge scoring.
+*
+* @example
+* scorers: [predicate((ctx) => ctx.result.report.children.every(c => c.status === "completed"))]
+*/
+function predicate(fn) {
+	return async (context) => {
+		const result = await fn(context);
+		return {
+			score: result ? 1 : 0,
+			passed: result,
+			reason: result ? "predicate passed" : "predicate failed"
+		};
+	};
+}
+
+//#endregion
+export { contains, exact, predicate };
+//# sourceMappingURL=scorers.mjs.map

package/esm/ai/src/eval/scorers.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorers.mjs","names":[],"sources":["../../../../../../../../../@warlock.js/ai/src/eval/scorers.ts"],"sourcesContent":["import type {\n  EvalScore,\n  EvalScorer,\n  EvalScorerContext,\n} from \"../contracts/agent/eval.type\";\n\n/**\n * Predicate signature for {@link predicate}. Receives the same context\n * a full scorer does and returns a boolean (sync or async). A `true`\n * verdict scores `1`, `false` scores `0`.\n */\nexport type EvalPredicate<TOutput = unknown> = (\n  context: EvalScorerContext<TOutput>,\n) => boolean | Promise<boolean>;\n\n/**\n * Normalize a value for case-insensitive, whitespace-trimmed string\n * comparison. Non-string values are JSON-serialized first so a\n * structured `expected` can still be matched against structured\n * `output`.\n */\nfunction normalizeForComparison(value: unknown): string {\n  const text = typeof value === \"string\" ? value : JSON.stringify(value);\n  return text.trim().toLowerCase();\n}\n\n/**\n * Exact-match scorer. Compares the agent's output against the case's\n * `expected` reference. Prefers `result.data` (parsed structured\n * output) when present, falling back to `result.text`. Comparison is\n * trimmed and case-insensitive; structured values are compared by\n * canonical JSON.\n *\n * Scores `1` / `passed: true` on a match, `0` / `passed: false`\n * otherwise. A case with no `expected` always scores `0` — exact\n * matching is meaningless without a reference.\n *\n * @example\n * const report = await agent.eval({\n *   cases: [{ name: \"q\", input: \"2+2?\", expected: \"4\" }],\n *   scorers: [exact()],\n * });\n */\nexport function exact<TOutput = unknown>(): EvalScorer<TOutput> {\n  return (context: EvalScorerContext<TOutput>): EvalScore => {\n    if (context.case.expected === undefined) {\n      return {\n        score: 0,\n        passed: false,\n        reason: \"no expected value supplied for exact match\",\n      };\n    }\n\n    const actual = context.output ?? context.text;\n\n    if (actual === undefined) {\n      return { score: 0, passed: false, reason: \"agent produced no output\" };\n    }\n\n    const matches =\n      normalizeForComparison(actual) === normalizeForComparison(context.case.expected);\n\n    return {\n      score: matches ? 1 : 0,\n      passed: matches,\n      reason: matches ? \"exact match\" : \"output did not match expected\",\n    };\n  };\n}\n\n/**\n * Substring / contains scorer. Passes when the normalized `expected`\n * string appears anywhere in the agent's normalized output. Useful\n * when the agent's phrasing varies but a key fact must be present.\n *\n * @example\n * scorers: [contains()] // expected \"Cairo\" passes \"The capital is Cairo.\"\n */\nexport function contains<TOutput = unknown>(): EvalScorer<TOutput> {\n  return (context: EvalScorerContext<TOutput>): EvalScore => {\n    if (context.case.expected === undefined) {\n      return {\n        score: 0,\n        passed: false,\n        reason: \"no expected value supplied for contains match\",\n      };\n    }\n\n    const actual = context.output ?? context.text;\n\n    if (actual === undefined) {\n      return { score: 0, passed: false, reason: \"agent produced no output\" };\n    }\n\n    const found = normalizeForComparison(actual).includes(\n      normalizeForComparison(context.case.expected),\n    );\n\n    return {\n      score: found ? 1 : 0,\n      passed: found,\n      reason: found ? \"expected substring found\" : \"expected substring not found\",\n    };\n  };\n}\n\n/**\n * Predicate scorer. Wraps a boolean-returning callback into a scorer —\n * `true` scores `1` / `passed`, `false` scores `0` / fails. The\n * escape hatch for arbitrary assertions (\"output is valid JSON\", \"no\n * tool errored\", \"duration under budget\") that don't fit exact or\n * judge scoring.\n *\n * @example\n * scorers: [predicate((ctx) => ctx.result.report.children.every(c => c.status === \"completed\"))]\n */\nexport function predicate<TOutput = unknown>(\n  fn: EvalPredicate<TOutput>,\n): EvalScorer<TOutput> {\n  return async (context: EvalScorerContext<TOutput>): Promise<EvalScore> => {\n    const result = await fn(context);\n\n    return {\n      score: result ? 1 : 0,\n      passed: result,\n      reason: result ? \"predicate passed\" : \"predicate failed\",\n    };\n  };\n}\n"],"mappings":";;;;;;;AAqBA,SAAS,uBAAuB,OAAwB;CAEtD,QADa,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,KAAK,EAC1D,CAAC,KAAK,CAAC,CAAC,YAAY;AACjC;;;;;;;;;;;;;;;;;;AAmBA,SAAgB,QAAgD;CAC9D,QAAQ,YAAmD;EACzD,IAAI,QAAQ,KAAK,aAAa,QAC5B,OAAO;GACL,OAAO;GACP,QAAQ;GACR,QAAQ;EACV;EAGF,MAAM,SAAS,QAAQ,UAAU,QAAQ;EAEzC,IAAI,WAAW,QACb,OAAO;GAAE,OAAO;GAAG,QAAQ;GAAO,QAAQ;EAA2B;EAGvE,MAAM,UACJ,uBAAuB,MAAM,MAAM,uBAAuB,QAAQ,KAAK,QAAQ;EAEjF,OAAO;GACL,OAAO,UAAU,IAAI;GACrB,QAAQ;GACR,QAAQ,UAAU,gBAAgB;EACpC;CACF;AACF;;;;;;;;;AAUA,SAAgB,WAAmD;CACjE,QAAQ,YAAmD;EACzD,IAAI,QAAQ,KAAK,aAAa,QAC5B,OAAO;GACL,OAAO;GACP,QAAQ;GACR,QAAQ;EACV;EAGF,MAAM,SAAS,QAAQ,UAAU,QAAQ;EAEzC,IAAI,WAAW,QACb,OAAO;GAAE,OAAO;GAAG,QAAQ;GAAO,QAAQ;EAA2B;EAGvE,MAAM,QAAQ,uBAAuB,MAAM,CAAC,CAAC,SAC3C,uBAAuB,QAAQ,KAAK,QAAQ,CAC9C;EAEA,OAAO;GACL,OAAO,QAAQ,IAAI;GACnB,QAAQ;GACR,QAAQ,QAAQ,6BAA6B;EAC/C;CACF;AACF;;;;;;;;;;;AAYA,SAAgB,UACd,IACqB;CACrB,OAAO,OAAO,YAA4D;EACxE,MAAM,SAAS,MAAM,GAAG,OAAO;EAE/B,OAAO;GACL,OAAO,SAAS,IAAI;GACpB,QAAQ;GACR,QAAQ,SAAS,qBAAqB;EACxC;CACF;AACF"}

package/esm/ai/src/guard/detectors/index.mjs

@@ -0,0 +1,6 @@
+import { injection } from "./injection.mjs";
+import { pii } from "./pii.mjs";
+import { topic } from "./topic.mjs";
+import { moderation } from "./moderation.mjs";
+
+export {  };

package/esm/ai/src/guard/detectors/injection.mjs

@@ -0,0 +1,254 @@
+//#region ../@warlock.js/ai/src/guard/detectors/injection.ts
+const DETECTOR_NAME = "injection";
+/**
+* Built-in jailbreak / prompt-injection marker phrases. Each entry is a
+* case-insensitive substring (matched lowercased) paired with the rule
+* label surfaced on the {@link GuardrailMatch} (`injection.<label>`).
+*
+* The set targets the canonical override / role-reset / exfiltration
+* patterns rather than trying to be exhaustive — a curated, low-false-
+* positive seed that callers extend with their own `markers`. Phrases are
+* deliberately specific (`"ignore previous instructions"`, not the bare
+* word `"ignore"`) so ordinary prose does not trip the rule.
+*/
+const BUILT_IN_MARKERS = [
+	{
+		phrase: "ignore previous instructions",
+		label: "override"
+	},
+	{
+		phrase: "ignore all previous instructions",
+		label: "override"
+	},
+	{
+		phrase: "ignore the above instructions",
+		label: "override"
+	},
+	{
+		phrase: "disregard previous instructions",
+		label: "override"
+	},
+	{
+		phrase: "disregard all previous instructions",
+		label: "override"
+	},
+	{
+		phrase: "forget previous instructions",
+		label: "override"
+	},
+	{
+		phrase: "forget all previous instructions",
+		label: "override"
+	},
+	{
+		phrase: "ignore your instructions",
+		label: "override"
+	},
+	{
+		phrase: "override your instructions",
+		label: "override"
+	},
+	{
+		phrase: "do not follow your instructions",
+		label: "override"
+	},
+	{
+		phrase: "you are now",
+		label: "role-reset"
+	},
+	{
+		phrase: "act as",
+		label: "role-reset"
+	},
+	{
+		phrase: "pretend to be",
+		label: "role-reset"
+	},
+	{
+		phrase: "developer mode",
+		label: "jailbreak"
+	},
+	{
+		phrase: "jailbreak",
+		label: "jailbreak"
+	},
+	{
+		phrase: "dan mode",
+		label: "jailbreak"
+	},
+	{
+		phrase: "do anything now",
+		label: "jailbreak"
+	},
+	{
+		phrase: "bypass your",
+		label: "jailbreak"
+	},
+	{
+		phrase: "ignore your guidelines",
+		label: "jailbreak"
+	},
+	{
+		phrase: "ignore your safety",
+		label: "jailbreak"
+	},
+	{
+		phrase: "ignore the rules",
+		label: "jailbreak"
+	},
+	{
+		phrase: "without any restrictions",
+		label: "jailbreak"
+	},
+	{
+		phrase: "reveal your system prompt",
+		label: "exfiltration"
+	},
+	{
+		phrase: "print your system prompt",
+		label: "exfiltration"
+	},
+	{
+		phrase: "show your system prompt",
+		label: "exfiltration"
+	},
+	{
+		phrase: "repeat your instructions",
+		label: "exfiltration"
+	},
+	{
+		phrase: "what are your instructions",
+		label: "exfiltration"
+	},
+	{
+		phrase: "reveal your prompt",
+		label: "exfiltration"
+	}
+];
+/**
+* The zero-dependency built-in injection detector — the internal class
+* behind the {@link injection} factory. Scans for jailbreak / prompt-
+* injection marker phrases (built-in set + caller `markers`) and returns a
+* `block` or `flag` verdict (per `onMatch`) listing every match, or `allow`
+* when the text is clean.
+*
+* Detection only: a detector never throws or mutates the pipeline — the
+* `guard()` factory translates the verdict into the trip's throw / record
+* mechanics.
+*/
+var InjectionDetector = class {
+	constructor(options = {}) {
+		this.name = DETECTOR_NAME;
+		this.block = options.onMatch === "block";
+		this.markers = compileMarkers(options.markers ?? []);
+	}
+	/**
+	* Inspect `text` for any built-in or caller marker. Returns `allow` when
+	* none hit, otherwise the configured `block` / `flag` verdict carrying a
+	* {@link GuardrailMatch} per hit (with a `[start, end]` span for literal
+	* substrings; regex hits report a span only when the match is locatable).
+	*/
+	check(text) {
+		const matches = this.scan(text);
+		if (matches.length === 0) return { type: "allow" };
+		const reason = `Detected ${matches.length} prompt-injection marker(s).`;
+		if (this.block) return {
+			type: "block",
+			reason,
+			matches
+		};
+		return {
+			type: "flag",
+			reason,
+			matches
+		};
+	}
+	/** Collect every marker hit in `text`, in marker registration order. */
+	scan(text) {
+		const lowered = text.toLowerCase();
+		const matches = [];
+		for (const marker of this.markers) {
+			if (marker.phrase !== void 0) {
+				const start = lowered.indexOf(marker.phrase);
+				if (start !== -1) matches.push({
+					rule: `${DETECTOR_NAME}.${marker.label}`,
+					label: marker.label,
+					span: [start, start + marker.phrase.length - 1]
+				});
+				continue;
+			}
+			const pattern = marker.pattern;
+			if (pattern === void 0) continue;
+			const result = pattern.exec(text);
+			if (result !== null) {
+				const start = result.index;
+				matches.push({
+					rule: `${DETECTOR_NAME}.${marker.label}`,
+					label: marker.label,
+					span: [start, start + result[0].length - 1]
+				});
+			}
+		}
+		return matches;
+	}
+};
+/**
+* Compile the built-in phrase set plus any caller `markers` into a single
+* ordered list. A caller `string` becomes a lowercased substring matcher
+* (labelled `custom`); a caller `RegExp` is carried as-is (labelled
+* `custom`). Built-ins keep their threat-category label.
+*/
+function compileMarkers(extra) {
+	const compiled = BUILT_IN_MARKERS.map((entry) => ({
+		label: entry.label,
+		phrase: entry.phrase
+	}));
+	for (const marker of extra) {
+		if (typeof marker === "string") {
+			compiled.push({
+				label: "custom",
+				phrase: marker.toLowerCase()
+			});
+			continue;
+		}
+		compiled.push({
+			label: "custom",
+			pattern: marker
+		});
+	}
+	return compiled;
+}
+/**
+* Build the built-in `injection` detector (surfaced as
+* `ai.guardrail.injection(options?)`). Matches a curated set of jailbreak /
+* prompt-injection marker phrases — override (`"ignore previous
+* instructions"`), role-reset (`"you are now"`), jailbreak (`"developer
+* mode"`, `"do anything now"`), and exfiltration (`"reveal your system
+* prompt"`) — extensible with caller `markers` (case-insensitive substrings
+* or `RegExp`s).
+*
+* Zero runtime dependency: matching is pure string / regex. On a hit the
+* verdict is `flag` by default (record but allow); pass `onMatch: "block"`
+* to reject instead — commonly used on the `input` phase, where the core
+* `trip.before` seam supports `block` / `flag` only.
+*
+* @param options - Extra `markers` and the `onMatch` action (`"flag"` | `"block"`).
+* @returns A {@link SyncGuardrailDetector} for the guard's `input` / `output` / `tool` arrays.
+*
+* @example
+* const guard = ai.guardrail({
+*   input: [ai.guardrail.injection({ onMatch: "block" })],
+*   output: [ai.guardrail.injection()], // flag-only on the model's reply
+* });
+*
+* @example
+* // Extend the built-in set with a house rule.
+* ai.guardrail.injection({ markers: [/system\s*:\s*override/i, "sudo mode"] });
+*/
+function injection(options) {
+	return new InjectionDetector(options);
+}
+
+//#endregion
+export { injection };
+//# sourceMappingURL=injection.mjs.map