@warlock.js/ai 4.2.11 → 4.3.0

package/esm/errors/planner-plan-invalid-error.d.mts

@@ -0,0 +1,21 @@
+import { ErrorCategory } from "./error-category.type.mjs";
+import { AIErrorOptions } from "./ai-error.mjs";
+import { PlannerFailedError } from "./planner-failed-error.mjs";
+
+//#region ../@warlock.js/ai/src/errors/planner-plan-invalid-error.d.ts
+/**
+ * The planner asked the LLM for an execution plan but the response
+ * could not be parsed / validated into a usable {@link PlannerPlan},
+ * or it referenced a capability that was never registered.
+ *
+ * Surfaced on `result.error` with `report.status === "failed"` — the
+ * planner returns normally rather than throwing, so callers branch on
+ * the typed envelope like every other primitive.
+ */
+declare class PlannerPlanInvalidError extends PlannerFailedError {
+  static readonly defaultCategory: ErrorCategory;
+  constructor(message: string, options?: AIErrorOptions);
+}
+//#endregion
+export { PlannerPlanInvalidError };
+//# sourceMappingURL=planner-plan-invalid-error.d.mts.map

package/esm/errors/planner-plan-invalid-error.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"planner-plan-invalid-error.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/errors/planner-plan-invalid-error.ts"],"mappings":";;;;;;;AAaA;;;;;;;cAAa,uBAAA,SAAgC,kBAAA;EAAA,gBACpB,eAAA,EAAiB,aAAA;cAErB,OAAA,UAAiB,OAAA,GAAU,cAAA;AAAA"}

package/esm/errors/planner-plan-invalid-error.mjs

@@ -0,0 +1,25 @@
+import { PlannerFailedError } from "./planner-failed-error.mjs";
+
+//#region ../@warlock.js/ai/src/errors/planner-plan-invalid-error.ts
+/**
+* The planner asked the LLM for an execution plan but the response
+* could not be parsed / validated into a usable {@link PlannerPlan},
+* or it referenced a capability that was never registered.
+*
+* Surfaced on `result.error` with `report.status === "failed"` — the
+* planner returns normally rather than throwing, so callers branch on
+* the typed envelope like every other primitive.
+*/
+var PlannerPlanInvalidError = class extends PlannerFailedError {
+	static {
+		this.defaultCategory = "schema";
+	}
+	constructor(message, options) {
+		super(message, options, "PLANNER_PLAN_INVALID");
+		this.name = "PlannerPlanInvalidError";
+	}
+};
+
+//#endregion
+export { PlannerPlanInvalidError };
+//# sourceMappingURL=planner-plan-invalid-error.mjs.map

package/esm/errors/planner-plan-invalid-error.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"planner-plan-invalid-error.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/errors/planner-plan-invalid-error.ts"],"sourcesContent":["import type { AIErrorOptions } from \"./ai-error\";\nimport type { ErrorCategory } from \"./error-category.type\";\nimport { PlannerFailedError } from \"./planner-failed-error\";\n\n/**\n * The planner asked the LLM for an execution plan but the response\n * could not be parsed / validated into a usable {@link PlannerPlan},\n * or it referenced a capability that was never registered.\n *\n * Surfaced on `result.error` with `report.status === \"failed\"` — the\n * planner returns normally rather than throwing, so callers branch on\n * the typed envelope like every other primitive.\n */\nexport class PlannerPlanInvalidError extends PlannerFailedError {\n  public static readonly defaultCategory: ErrorCategory = \"schema\";\n\n  public constructor(message: string, options?: AIErrorOptions) {\n    super(message, options, \"PLANNER_PLAN_INVALID\");\n    this.name = \"PlannerPlanInvalidError\";\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAaA,IAAa,0BAAb,cAA6C,mBAAmB;;yBACN;;CAExD,AAAO,YAAY,SAAiB,SAA0B;EAC5D,MAAM,SAAS,SAAS,sBAAsB;EAC9C,KAAK,OAAO;CACd;AACF"}

package/esm/eval/eval-runner.d.mts

@@ -0,0 +1,17 @@
+import { EvalOptions, EvalReport } from "../contracts/agent/eval.type.mjs";
+import { AgentContract } from "../contracts/agent/agent.contract.mjs";
+
+//#region ../@warlock.js/ai/src/eval/eval-runner.d.ts
+/**
+ * Core implementation of `agent.eval`. Runs every case sequentially
+ * (cases share the agent and may carry side effects — ordering must be
+ * deterministic), scores each, fires `onFailure` for failed cases, and
+ * assembles the aggregate {@link EvalReport}.
+ *
+ * Never throws on a case-level failure; the only throw is the
+ * authoring-time "no scorer" guard from {@link resolveScorers}.
+ */
+declare function runEval<TOutput>(agent: AgentContract<TOutput>, options: EvalOptions<TOutput>): Promise<EvalReport<TOutput>>;
+//#endregion
+export { runEval };
+//# sourceMappingURL=eval-runner.d.mts.map

package/esm/eval/eval-runner.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-runner.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/eval-runner.ts"],"mappings":";;;;;;AAmIA;;;;;;;iBAAsB,OAAA,UACpB,KAAA,EAAO,aAAA,CAAc,OAAA,GACrB,OAAA,EAAS,WAAA,CAAY,OAAA,IACpB,OAAA,CAAQ,UAAA,CAAW,OAAA"}

package/esm/eval/eval-runner.mjs

@@ -0,0 +1,121 @@
+import { AgentExecutionError } from "../errors/agent-execution-error.mjs";
+import "../errors/index.mjs";
+import { judge } from "./judge-scorer.mjs";
+import { log } from "@warlock.js/logger";
+
+//#region ../@warlock.js/ai/src/eval/eval-runner.ts
+const LOG_MODULE = "ai.eval";
+const DEFAULT_PASS_THRESHOLD = .5;
+/**
+* Resolve the scorer list for a single case. Precedence: the case's
+* own `scorers` → the suite `scorers` → a synthesized judge scorer
+* when `judge` is configured. Throws an authoring-time
+* `AgentExecutionError` when a case can resolve none — an eval suite
+* with no way to score a case is a config bug worth surfacing at the
+* call site, not a silent pass.
+*/
+function resolveScorers(evalCase, options, passThreshold) {
+	if (evalCase.scorers && evalCase.scorers.length > 0) return evalCase.scorers;
+	if (options.scorers && options.scorers.length > 0) return options.scorers;
+	if (options.judge) return [judge(options.judge, passThreshold)];
+	throw new AgentExecutionError(`eval case "${evalCase.name}" has no scorer — supply per-case "scorers", suite "scorers", or a "judge"`, { context: {
+		authoring: true,
+		case: evalCase.name
+	} });
+}
+/**
+* Decide a single scorer verdict's pass/fail. Honors an explicit
+* `passed` from the scorer; otherwise derives it from
+* `score >= passThreshold`.
+*/
+function isScorePassing(score, passThreshold) {
+	if (typeof score.passed === "boolean") return score.passed;
+	return score.score >= passThreshold;
+}
+/**
+* Merge suite-level execute options with the case's own override.
+* Per-case wins on conflict (shallow merge).
+*/
+function mergeOptions(suite, perCase) {
+	if (!suite) return perCase;
+	if (!perCase) return suite;
+	return {
+		...suite,
+		...perCase
+	};
+}
+/**
+* Run one case end-to-end: execute the agent, run every resolved
+* scorer, aggregate into an {@link EvalCaseResult}. A case passes only
+* when the agent did not error AND every scorer passed.
+*/
+async function runCase(agent, evalCase, options, passThreshold) {
+	const scorers = resolveScorers(evalCase, options, passThreshold);
+	const executeOptions = mergeOptions(options.executeOptions, evalCase.options);
+	const start = performance.now();
+	const result = await agent.execute(evalCase.input, executeOptions);
+	const duration = performance.now() - start;
+	const context = {
+		case: evalCase,
+		result,
+		output: result.data,
+		text: result.text
+	};
+	const scores = [];
+	for (const scorer of scorers) scores.push(await scorer(context));
+	const meanScore = scores.length > 0 ? scores.reduce((sum, score) => sum + score.score, 0) / scores.length : 0;
+	const allScorersPassed = scores.every((score) => isScorePassing(score, passThreshold));
+	return {
+		case: evalCase,
+		result,
+		scores,
+		score: meanScore,
+		passed: result.error === void 0 && allScorersPassed,
+		duration
+	};
+}
+/**
+* Core implementation of `agent.eval`. Runs every case sequentially
+* (cases share the agent and may carry side effects — ordering must be
+* deterministic), scores each, fires `onFailure` for failed cases, and
+* assembles the aggregate {@link EvalReport}.
+*
+* Never throws on a case-level failure; the only throw is the
+* authoring-time "no scorer" guard from {@link resolveScorers}.
+*/
+async function runEval(agent, options) {
+	const passThreshold = options.passThreshold ?? DEFAULT_PASS_THRESHOLD;
+	const start = performance.now();
+	const cases = [];
+	for (const evalCase of options.cases) {
+		const caseResult = await runCase(agent, evalCase, options, passThreshold);
+		cases.push(caseResult);
+		if (!caseResult.passed && options.onFailure) try {
+			await options.onFailure(caseResult);
+		} catch (error) {
+			log.warn(LOG_MODULE, "onFailure.hook.error", "eval onFailure handler threw", {
+				agent: agent.name,
+				case: evalCase.name,
+				error: error instanceof Error ? error.message : String(error)
+			});
+		}
+	}
+	const passedCount = cases.filter((entry) => entry.passed).length;
+	const total = cases.length;
+	const meanScore = total > 0 ? cases.reduce((sum, entry) => sum + entry.score, 0) / total : 0;
+	return {
+		agentName: agent.name,
+		total,
+		passedCount,
+		failedCount: total - passedCount,
+		passRate: total > 0 ? passedCount / total : 0,
+		meanScore,
+		passed: total > 0 && passedCount === total,
+		cases,
+		duration: performance.now() - start
+	};
+}
+
+//#endregion
+export { runEval };
+//# sourceMappingURL=eval-runner.mjs.map

package/esm/eval/eval-runner.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-runner.mjs","names":["judgeScorer"],"sources":["../../../../../../../@warlock.js/ai/src/eval/eval-runner.ts"],"sourcesContent":["import type { AgentContract } from \"../contracts/agent/agent.contract\";\nimport type { AgentExecuteOptions } from \"../contracts/agent/agent-options.type\";\nimport type {\n  EvalCase,\n  EvalCaseResult,\n  EvalOptions,\n  EvalReport,\n  EvalScore,\n  EvalScorer,\n  EvalScorerContext,\n} from \"../contracts/agent/eval.type\";\nimport { AgentExecutionError } from \"../errors\";\nimport { log } from \"@warlock.js/logger\";\nimport { judge as judgeScorer } from \"./judge-scorer\";\n\nconst LOG_MODULE = \"ai.eval\";\nconst DEFAULT_PASS_THRESHOLD = 0.5;\n\n/**\n * Resolve the scorer list for a single case. Precedence: the case's\n * own `scorers` → the suite `scorers` → a synthesized judge scorer\n * when `judge` is configured. Throws an authoring-time\n * `AgentExecutionError` when a case can resolve none — an eval suite\n * with no way to score a case is a config bug worth surfacing at the\n * call site, not a silent pass.\n */\nfunction resolveScorers<TOutput>(\n  evalCase: EvalCase<TOutput>,\n  options: EvalOptions<TOutput>,\n  passThreshold: number,\n): EvalScorer<TOutput>[] {\n  if (evalCase.scorers && evalCase.scorers.length > 0) {\n    return evalCase.scorers;\n  }\n\n  if (options.scorers && options.scorers.length > 0) {\n    return options.scorers;\n  }\n\n  if (options.judge) {\n    return [judgeScorer<TOutput>(options.judge, passThreshold)];\n  }\n\n  throw new AgentExecutionError(\n    `eval case \"${evalCase.name}\" has no scorer — supply per-case \"scorers\", suite \"scorers\", or a \"judge\"`,\n    { context: { authoring: true, case: evalCase.name } },\n  );\n}\n\n/**\n * Decide a single scorer verdict's pass/fail. Honors an explicit\n * `passed` from the scorer; otherwise derives it from\n * `score >= passThreshold`.\n */\nfunction isScorePassing(score: EvalScore, passThreshold: number): boolean {\n  if (typeof score.passed === \"boolean\") {\n    return score.passed;\n  }\n\n  return score.score >= passThreshold;\n}\n\n/**\n * Merge suite-level execute options with the case's own override.\n * Per-case wins on conflict (shallow merge).\n */\nfunction mergeOptions<TOutput>(\n  suite: AgentExecuteOptions<TOutput> | undefined,\n  perCase: AgentExecuteOptions<TOutput> | undefined,\n): AgentExecuteOptions<TOutput> | undefined {\n  if (!suite) return perCase;\n  if (!perCase) return suite;\n  return { ...suite, ...perCase };\n}\n\n/**\n * Run one case end-to-end: execute the agent, run every resolved\n * scorer, aggregate into an {@link EvalCaseResult}. A case passes only\n * when the agent did not error AND every scorer passed.\n */\nasync function runCase<TOutput>(\n  agent: AgentContract<TOutput>,\n  evalCase: EvalCase<TOutput>,\n  options: EvalOptions<TOutput>,\n  passThreshold: number,\n): Promise<EvalCaseResult<TOutput>> {\n  const scorers = resolveScorers(evalCase, options, passThreshold);\n  const executeOptions = mergeOptions(options.executeOptions, evalCase.options);\n\n  const start = performance.now();\n  const result = await agent.execute(evalCase.input, executeOptions);\n  const duration = performance.now() - start;\n\n  const context: EvalScorerContext<TOutput> = {\n    case: evalCase,\n    result,\n    output: result.data,\n    text: result.text,\n  };\n\n  const scores: EvalScore[] = [];\n\n  for (const scorer of scorers) {\n    scores.push(await scorer(context));\n  }\n\n  const meanScore =\n    scores.length > 0 ? scores.reduce((sum, score) => sum + score.score, 0) / scores.length : 0;\n\n  const allScorersPassed = scores.every((score) => isScorePassing(score, passThreshold));\n  const passed = result.error === undefined && allScorersPassed;\n\n  return {\n    case: evalCase,\n    result,\n    scores,\n    score: meanScore,\n    passed,\n    duration,\n  };\n}\n\n/**\n * Core implementation of `agent.eval`. Runs every case sequentially\n * (cases share the agent and may carry side effects — ordering must be\n * deterministic), scores each, fires `onFailure` for failed cases, and\n * assembles the aggregate {@link EvalReport}.\n *\n * Never throws on a case-level failure; the only throw is the\n * authoring-time \"no scorer\" guard from {@link resolveScorers}.\n */\nexport async function runEval<TOutput>(\n  agent: AgentContract<TOutput>,\n  options: EvalOptions<TOutput>,\n): Promise<EvalReport<TOutput>> {\n  const passThreshold = options.passThreshold ?? DEFAULT_PASS_THRESHOLD;\n  const start = performance.now();\n\n  const cases: EvalCaseResult<TOutput>[] = [];\n\n  for (const evalCase of options.cases) {\n    const caseResult = await runCase(agent, evalCase, options, passThreshold);\n\n    cases.push(caseResult);\n\n    if (!caseResult.passed && options.onFailure) {\n      try {\n        await options.onFailure(caseResult);\n      } catch (error) {\n        log.warn(LOG_MODULE, \"onFailure.hook.error\", \"eval onFailure handler threw\", {\n          agent: agent.name,\n          case: evalCase.name,\n          error: error instanceof Error ? error.message : String(error),\n        });\n      }\n    }\n  }\n\n  const passedCount = cases.filter((entry) => entry.passed).length;\n  const total = cases.length;\n  const meanScore =\n    total > 0 ? cases.reduce((sum, entry) => sum + entry.score, 0) / total : 0;\n\n  return {\n    agentName: agent.name,\n    total,\n    passedCount,\n    failedCount: total - passedCount,\n    passRate: total > 0 ? passedCount / total : 0,\n    meanScore,\n    passed: total > 0 && passedCount === total,\n    cases,\n    duration: performance.now() - start,\n  };\n}\n"],"mappings":";;;;;;AAeA,MAAM,aAAa;AACnB,MAAM,yBAAyB;;;;;;;;;AAU/B,SAAS,eACP,UACA,SACA,eACuB;CACvB,IAAI,SAAS,WAAW,SAAS,QAAQ,SAAS,GAChD,OAAO,SAAS;CAGlB,IAAI,QAAQ,WAAW,QAAQ,QAAQ,SAAS,GAC9C,OAAO,QAAQ;CAGjB,IAAI,QAAQ,OACV,OAAO,CAACA,MAAqB,QAAQ,OAAO,aAAa,CAAC;CAG5D,MAAM,IAAI,oBACR,cAAc,SAAS,KAAK,6EAC5B,EAAE,SAAS;EAAE,WAAW;EAAM,MAAM,SAAS;CAAK,EAAE,CACtD;AACF;;;;;;AAOA,SAAS,eAAe,OAAkB,eAAgC;CACxE,IAAI,OAAO,MAAM,WAAW,WAC1B,OAAO,MAAM;CAGf,OAAO,MAAM,SAAS;AACxB;;;;;AAMA,SAAS,aACP,OACA,SAC0C;CAC1C,IAAI,CAAC,OAAO,OAAO;CACnB,IAAI,CAAC,SAAS,OAAO;CACrB,OAAO;EAAE,GAAG;EAAO,GAAG;CAAQ;AAChC;;;;;;AAOA,eAAe,QACb,OACA,UACA,SACA,eACkC;CAClC,MAAM,UAAU,eAAe,UAAU,SAAS,aAAa;CAC/D,MAAM,iBAAiB,aAAa,QAAQ,gBAAgB,SAAS,OAAO;CAE5E,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,SAAS,MAAM,MAAM,QAAQ,SAAS,OAAO,cAAc;CACjE,MAAM,WAAW,YAAY,IAAI,IAAI;CAErC,MAAM,UAAsC;EAC1C,MAAM;EACN;EACA,QAAQ,OAAO;EACf,MAAM,OAAO;CACf;CAEA,MAAM,SAAsB,CAAC;CAE7B,KAAK,MAAM,UAAU,SACnB,OAAO,KAAK,MAAM,OAAO,OAAO,CAAC;CAGnC,MAAM,YACJ,OAAO,SAAS,IAAI,OAAO,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,CAAC,IAAI,OAAO,SAAS;CAE5F,MAAM,mBAAmB,OAAO,OAAO,UAAU,eAAe,OAAO,aAAa,CAAC;CAGrF,OAAO;EACL,MAAM;EACN;EACA;EACA,OAAO;EACP,QAPa,OAAO,UAAU,UAAa;EAQ3C;CACF;AACF;;;;;;;;;;AAWA,eAAsB,QACpB,OACA,SAC8B;CAC9B,MAAM,gBAAgB,QAAQ,iBAAiB;CAC/C,MAAM,QAAQ,YAAY,IAAI;CAE9B,MAAM,QAAmC,CAAC;CAE1C,KAAK,MAAM,YAAY,QAAQ,OAAO;EACpC,MAAM,aAAa,MAAM,QAAQ,OAAO,UAAU,SAAS,aAAa;EAExE,MAAM,KAAK,UAAU;EAErB,IAAI,CAAC,WAAW,UAAU,QAAQ,WAChC,IAAI;GACF,MAAM,QAAQ,UAAU,UAAU;EACpC,SAAS,OAAO;GACd,IAAI,KAAK,YAAY,wBAAwB,gCAAgC;IAC3E,OAAO,MAAM;IACb,MAAM,SAAS;IACf,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D,CAAC;EACH;CAEJ;CAEA,MAAM,cAAc,MAAM,QAAQ,UAAU,MAAM,MAAM,CAAC,CAAC;CAC1D,MAAM,QAAQ,MAAM;CACpB,MAAM,YACJ,QAAQ,IAAI,MAAM,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,CAAC,IAAI,QAAQ;CAE3E,OAAO;EACL,WAAW,MAAM;EACjB;EACA;EACA,aAAa,QAAQ;EACrB,UAAU,QAAQ,IAAI,cAAc,QAAQ;EAC5C;EACA,QAAQ,QAAQ,KAAK,gBAAgB;EACrC;EACA,UAAU,YAAY,IAAI,IAAI;CAChC;AACF"}

package/esm/eval/index.d.mts

@@ -0,0 +1,29 @@
+import { judge } from "./judge-scorer.mjs";
+import { EvalPredicate, contains, exact, predicate } from "./scorers.mjs";
+import { runEval } from "./eval-runner.mjs";
+
+//#region ../@warlock.js/ai/src/eval/index.d.ts
+/**
+ * Built-in eval scorer factories, surfaced on `ai.eval.*`.
+ *
+ * - `exact()` — pass when output equals the case `expected` (trimmed,
+ *   case-insensitive; structured values compared by canonical JSON).
+ * - `contains()` — pass when `expected` appears as a substring.
+ * - `predicate(fn)` — wrap an arbitrary boolean assertion.
+ * - `judge(config)` — LLM-as-judge scoring against a rubric.
+ *
+ * @example
+ * await myAgent.eval({
+ *   cases: [{ name: "q", input: "Capital of Egypt?", expected: "Cairo" }],
+ *   scorers: [ai.eval.contains()],
+ * });
+ */
+declare const evalScorers: {
+  exact: typeof exact;
+  contains: typeof contains;
+  predicate: typeof predicate;
+  judge: typeof judge;
+};
+//#endregion
+export { evalScorers };
+//# sourceMappingURL=index.d.mts.map

package/esm/eval/index.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/index.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;cA0Ba,WAAA"}

package/esm/eval/index.mjs

@@ -0,0 +1,30 @@
+import { judge } from "./judge-scorer.mjs";
+import { runEval } from "./eval-runner.mjs";
+import { contains, exact, predicate } from "./scorers.mjs";
+
+//#region ../@warlock.js/ai/src/eval/index.ts
+/**
+* Built-in eval scorer factories, surfaced on `ai.eval.*`.
+*
+* - `exact()` — pass when output equals the case `expected` (trimmed,
+*   case-insensitive; structured values compared by canonical JSON).
+* - `contains()` — pass when `expected` appears as a substring.
+* - `predicate(fn)` — wrap an arbitrary boolean assertion.
+* - `judge(config)` — LLM-as-judge scoring against a rubric.
+*
+* @example
+* await myAgent.eval({
+*   cases: [{ name: "q", input: "Capital of Egypt?", expected: "Cairo" }],
+*   scorers: [ai.eval.contains()],
+* });
+*/
+const evalScorers = {
+	exact,
+	contains,
+	predicate,
+	judge
+};
+
+//#endregion
+export { evalScorers };
+//# sourceMappingURL=index.mjs.map

package/esm/eval/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/index.ts"],"sourcesContent":["import { judge } from \"./judge-scorer\";\nimport { contains, exact, predicate } from \"./scorers\";\n\n// Runner — wired into AgentContract.eval() by the agent factory.\nexport { runEval } from \"./eval-runner\";\n\n// Scorer factories — re-exported individually for direct import.\nexport { contains, exact, predicate } from \"./scorers\";\nexport type { EvalPredicate } from \"./scorers\";\nexport { judge } from \"./judge-scorer\";\n\n/**\n * Built-in eval scorer factories, surfaced on `ai.eval.*`.\n *\n * - `exact()` — pass when output equals the case `expected` (trimmed,\n *   case-insensitive; structured values compared by canonical JSON).\n * - `contains()` — pass when `expected` appears as a substring.\n * - `predicate(fn)` — wrap an arbitrary boolean assertion.\n * - `judge(config)` — LLM-as-judge scoring against a rubric.\n *\n * @example\n * await myAgent.eval({\n *   cases: [{ name: \"q\", input: \"Capital of Egypt?\", expected: \"Cairo\" }],\n *   scorers: [ai.eval.contains()],\n * });\n */\nexport const evalScorers = {\n  exact,\n  contains,\n  predicate,\n  judge,\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AA0BA,MAAa,cAAc;CACzB;CACA;CACA;CACA;AACF"}

package/esm/eval/judge-scorer.d.mts

@@ -0,0 +1,21 @@
+import { EvalJudge, EvalScorer } from "../contracts/agent/eval.type.mjs";
+
+//#region ../@warlock.js/ai/src/eval/judge-scorer.d.ts
+/**
+ * LLM-as-judge scorer. Runs the judge agent on a prompt built from the
+ * case (question + expected + actual answer + rubric) and parses its
+ * `{ score, passed?, reason? }` verdict.
+ *
+ * Verdict source order: `result.data` (when the judge declares an
+ * output schema), then `result.text` parsed as JSON. A judge that
+ * errors or returns unparseable text scores `0` with the failure
+ * reason attached — a broken judge fails the case rather than crashing
+ * the suite.
+ *
+ * @example
+ * scorers: [judge({ agent: judgeAgent, rubric: "Cite a source for full marks." })]
+ */
+declare function judge<TOutput = unknown>(config: EvalJudge, passThreshold?: number): EvalScorer<TOutput>;
+//#endregion
+export { judge };
+//# sourceMappingURL=judge-scorer.d.mts.map

package/esm/eval/judge-scorer.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge-scorer.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/judge-scorer.ts"],"mappings":";;;;;AA4FA;;;;;;;;;;;;iBAAgB,KAAA,oBACd,MAAA,EAAQ,SAAA,EACR,aAAA,YACC,UAAA,CAAW,OAAA"}

package/esm/eval/judge-scorer.mjs

@@ -0,0 +1,87 @@
+import { extractJsonPayload } from "../utils/extract-json-payload.mjs";
+import { safeJsonParse } from "../utils/safe-json-parse.mjs";
+import "../utils/index.mjs";
+
+//#region ../@warlock.js/ai/src/eval/judge-scorer.ts
+/** Clamp an arbitrary numeric value into the `[0, 1]` score range. */
+function clampScore(value) {
+	if (Number.isNaN(value)) return 0;
+	if (value < 0) return 0;
+	if (value > 1) return 1;
+	return value;
+}
+/**
+* Render the prompt the judge agent receives for one case. Includes
+* the rubric (if any), the original question, the expected reference
+* (when supplied), and the actual answer — then asks for a strict JSON
+* verdict so the response is machine-parseable even without an output
+* schema.
+*/
+function buildJudgePrompt(context, judge) {
+	const lines = [];
+	if (judge.rubric) lines.push(`Grading rubric:\n${judge.rubric}`, "");
+	lines.push(`Question:\n${context.case.input}`, "");
+	if (context.case.expected !== void 0) {
+		const expectedText = typeof context.case.expected === "string" ? context.case.expected : JSON.stringify(context.case.expected);
+		lines.push(`Reference answer:\n${expectedText}`, "");
+	}
+	const actual = context.text ?? JSON.stringify(context.output ?? null);
+	lines.push(`Answer to grade:\n${actual}`, "");
+	lines.push("Respond with JSON only: { \"score\": <0..1>, \"passed\": <true|false>, \"reason\": \"<short explanation>\" }.");
+	return lines.join("\n");
+}
+/**
+* Coerce the judge agent's parsed/extracted verdict into a normalized
+* {@link EvalScore}. Defends against a judge that returns a string
+* score or omits `passed` — the caller-supplied `passThreshold`
+* derives `passed` from `score` when the judge didn't decide.
+*/
+function toEvalScore(verdict, passThreshold) {
+	const rawScore = typeof verdict.score === "string" ? Number(verdict.score) : verdict.score;
+	const score = clampScore(typeof rawScore === "number" ? rawScore : 0);
+	return {
+		score,
+		passed: typeof verdict.passed === "boolean" ? verdict.passed : score >= passThreshold,
+		reason: typeof verdict.reason === "string" ? verdict.reason : void 0
+	};
+}
+/**
+* LLM-as-judge scorer. Runs the judge agent on a prompt built from the
+* case (question + expected + actual answer + rubric) and parses its
+* `{ score, passed?, reason? }` verdict.
+*
+* Verdict source order: `result.data` (when the judge declares an
+* output schema), then `result.text` parsed as JSON. A judge that
+* errors or returns unparseable text scores `0` with the failure
+* reason attached — a broken judge fails the case rather than crashing
+* the suite.
+*
+* @example
+* scorers: [judge({ agent: judgeAgent, rubric: "Cite a source for full marks." })]
+*/
+function judge(config, passThreshold = .5) {
+	const threshold = config.passThreshold ?? passThreshold;
+	return async (context) => {
+		const prompt = buildJudgePrompt(context, config);
+		const verdictResult = await config.agent.execute(prompt);
+		if (verdictResult.error) return {
+			score: 0,
+			passed: false,
+			reason: `judge failed: ${verdictResult.error.message}`
+		};
+		if (verdictResult.data && typeof verdictResult.data === "object") return toEvalScore(verdictResult.data, threshold);
+		const text = verdictResult.text ?? "";
+		const sentinel = Symbol("judge-parse-failed");
+		const parsed = safeJsonParse(extractJsonPayload(text), sentinel);
+		if (parsed === sentinel || parsed === null || typeof parsed !== "object") return {
+			score: 0,
+			passed: false,
+			reason: "judge returned no parseable verdict"
+		};
+		return toEvalScore(parsed, threshold);
+	};
+}
+
+//#endregion
+export { judge };
+//# sourceMappingURL=judge-scorer.mjs.map

package/esm/eval/judge-scorer.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge-scorer.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/judge-scorer.ts"],"sourcesContent":["import type {\n  EvalJudge,\n  EvalScore,\n  EvalScorer,\n  EvalScorerContext,\n} from \"../contracts/agent/eval.type\";\nimport { extractJsonPayload, safeJsonParse } from \"../utils\";\n\n/**\n * Raw shape the judge agent is expected to return — either parsed from\n * `result.data` (when the judge has an output schema) or extracted\n * from `result.text`.\n */\ntype JudgeVerdict = {\n  score?: unknown;\n  passed?: unknown;\n  reason?: unknown;\n};\n\n/** Clamp an arbitrary numeric value into the `[0, 1]` score range. */\nfunction clampScore(value: number): number {\n  if (Number.isNaN(value)) return 0;\n  if (value < 0) return 0;\n  if (value > 1) return 1;\n  return value;\n}\n\n/**\n * Render the prompt the judge agent receives for one case. Includes\n * the rubric (if any), the original question, the expected reference\n * (when supplied), and the actual answer — then asks for a strict JSON\n * verdict so the response is machine-parseable even without an output\n * schema.\n */\nfunction buildJudgePrompt(context: EvalScorerContext, judge: EvalJudge): string {\n  const lines: string[] = [];\n\n  if (judge.rubric) {\n    lines.push(`Grading rubric:\\n${judge.rubric}`, \"\");\n  }\n\n  lines.push(`Question:\\n${context.case.input}`, \"\");\n\n  if (context.case.expected !== undefined) {\n    const expectedText =\n      typeof context.case.expected === \"string\"\n        ? context.case.expected\n        : JSON.stringify(context.case.expected);\n    lines.push(`Reference answer:\\n${expectedText}`, \"\");\n  }\n\n  const actual = context.text ?? JSON.stringify(context.output ?? null);\n  lines.push(`Answer to grade:\\n${actual}`, \"\");\n\n  lines.push(\n    'Respond with JSON only: { \"score\": <0..1>, \"passed\": <true|false>, \"reason\": \"<short explanation>\" }.',\n  );\n\n  return lines.join(\"\\n\");\n}\n\n/**\n * Coerce the judge agent's parsed/extracted verdict into a normalized\n * {@link EvalScore}. Defends against a judge that returns a string\n * score or omits `passed` — the caller-supplied `passThreshold`\n * derives `passed` from `score` when the judge didn't decide.\n */\nfunction toEvalScore(verdict: JudgeVerdict, passThreshold: number): EvalScore {\n  const rawScore = typeof verdict.score === \"string\" ? Number(verdict.score) : verdict.score;\n  const score = clampScore(typeof rawScore === \"number\" ? rawScore : 0);\n\n  const passed = typeof verdict.passed === \"boolean\" ? verdict.passed : score >= passThreshold;\n\n  const reason = typeof verdict.reason === \"string\" ? verdict.reason : undefined;\n\n  return { score, passed, reason };\n}\n\n/**\n * LLM-as-judge scorer. Runs the judge agent on a prompt built from the\n * case (question + expected + actual answer + rubric) and parses its\n * `{ score, passed?, reason? }` verdict.\n *\n * Verdict source order: `result.data` (when the judge declares an\n * output schema), then `result.text` parsed as JSON. A judge that\n * errors or returns unparseable text scores `0` with the failure\n * reason attached — a broken judge fails the case rather than crashing\n * the suite.\n *\n * @example\n * scorers: [judge({ agent: judgeAgent, rubric: \"Cite a source for full marks.\" })]\n */\nexport function judge<TOutput = unknown>(\n  config: EvalJudge,\n  passThreshold = 0.5,\n): EvalScorer<TOutput> {\n  const threshold = config.passThreshold ?? passThreshold;\n\n  return async (context: EvalScorerContext<TOutput>): Promise<EvalScore> => {\n    const prompt = buildJudgePrompt(context as EvalScorerContext, config);\n\n    const verdictResult = await config.agent.execute(prompt);\n\n    if (verdictResult.error) {\n      return {\n        score: 0,\n        passed: false,\n        reason: `judge failed: ${verdictResult.error.message}`,\n      };\n    }\n\n    if (verdictResult.data && typeof verdictResult.data === \"object\") {\n      return toEvalScore(verdictResult.data as JudgeVerdict, threshold);\n    }\n\n    const text = verdictResult.text ?? \"\";\n    const sentinel = Symbol(\"judge-parse-failed\");\n    const parsed = safeJsonParse<unknown>(extractJsonPayload(text), sentinel);\n\n    if (parsed === sentinel || parsed === null || typeof parsed !== \"object\") {\n      return {\n        score: 0,\n        passed: false,\n        reason: \"judge returned no parseable verdict\",\n      };\n    }\n\n    return toEvalScore(parsed as JudgeVerdict, threshold);\n  };\n}\n"],"mappings":";;;;;;AAoBA,SAAS,WAAW,OAAuB;CACzC,IAAI,OAAO,MAAM,KAAK,GAAG,OAAO;CAChC,IAAI,QAAQ,GAAG,OAAO;CACtB,IAAI,QAAQ,GAAG,OAAO;CACtB,OAAO;AACT;;;;;;;;AASA,SAAS,iBAAiB,SAA4B,OAA0B;CAC9E,MAAM,QAAkB,CAAC;CAEzB,IAAI,MAAM,QACR,MAAM,KAAK,oBAAoB,MAAM,UAAU,EAAE;CAGnD,MAAM,KAAK,cAAc,QAAQ,KAAK,SAAS,EAAE;CAEjD,IAAI,QAAQ,KAAK,aAAa,QAAW;EACvC,MAAM,eACJ,OAAO,QAAQ,KAAK,aAAa,WAC7B,QAAQ,KAAK,WACb,KAAK,UAAU,QAAQ,KAAK,QAAQ;EAC1C,MAAM,KAAK,sBAAsB,gBAAgB,EAAE;CACrD;CAEA,MAAM,SAAS,QAAQ,QAAQ,KAAK,UAAU,QAAQ,UAAU,IAAI;CACpE,MAAM,KAAK,qBAAqB,UAAU,EAAE;CAE5C,MAAM,KACJ,+GACF;CAEA,OAAO,MAAM,KAAK,IAAI;AACxB;;;;;;;AAQA,SAAS,YAAY,SAAuB,eAAkC;CAC5E,MAAM,WAAW,OAAO,QAAQ,UAAU,WAAW,OAAO,QAAQ,KAAK,IAAI,QAAQ;CACrF,MAAM,QAAQ,WAAW,OAAO,aAAa,WAAW,WAAW,CAAC;CAMpE,OAAO;EAAE;EAAO,QAJD,OAAO,QAAQ,WAAW,YAAY,QAAQ,SAAS,SAAS;EAIvD,QAFT,OAAO,QAAQ,WAAW,WAAW,QAAQ,SAAS;CAEtC;AACjC;;;;;;;;;;;;;;;AAgBA,SAAgB,MACd,QACA,gBAAgB,IACK;CACrB,MAAM,YAAY,OAAO,iBAAiB;CAE1C,OAAO,OAAO,YAA4D;EACxE,MAAM,SAAS,iBAAiB,SAA8B,MAAM;EAEpE,MAAM,gBAAgB,MAAM,OAAO,MAAM,QAAQ,MAAM;EAEvD,IAAI,cAAc,OAChB,OAAO;GACL,OAAO;GACP,QAAQ;GACR,QAAQ,iBAAiB,cAAc,MAAM;EAC/C;EAGF,IAAI,cAAc,QAAQ,OAAO,cAAc,SAAS,UACtD,OAAO,YAAY,cAAc,MAAsB,SAAS;EAGlE,MAAM,OAAO,cAAc,QAAQ;EACnC,MAAM,WAAW,OAAO,oBAAoB;EAC5C,MAAM,SAAS,cAAuB,mBAAmB,IAAI,GAAG,QAAQ;EAExE,IAAI,WAAW,YAAY,WAAW,QAAQ,OAAO,WAAW,UAC9D,OAAO;GACL,OAAO;GACP,QAAQ;GACR,QAAQ;EACV;EAGF,OAAO,YAAY,QAAwB,SAAS;CACtD;AACF"}

package/esm/eval/scorers.d.mts

@@ -0,0 +1,50 @@
+import { EvalScorer, EvalScorerContext } from "../contracts/agent/eval.type.mjs";
+
+//#region ../@warlock.js/ai/src/eval/scorers.d.ts
+/**
+ * Predicate signature for {@link predicate}. Receives the same context
+ * a full scorer does and returns a boolean (sync or async). A `true`
+ * verdict scores `1`, `false` scores `0`.
+ */
+type EvalPredicate<TOutput = unknown> = (context: EvalScorerContext<TOutput>) => boolean | Promise<boolean>;
+/**
+ * Exact-match scorer. Compares the agent's output against the case's
+ * `expected` reference. Prefers `result.data` (parsed structured
+ * output) when present, falling back to `result.text`. Comparison is
+ * trimmed and case-insensitive; structured values are compared by
+ * canonical JSON.
+ *
+ * Scores `1` / `passed: true` on a match, `0` / `passed: false`
+ * otherwise. A case with no `expected` always scores `0` — exact
+ * matching is meaningless without a reference.
+ *
+ * @example
+ * const report = await agent.eval({
+ *   cases: [{ name: "q", input: "2+2?", expected: "4" }],
+ *   scorers: [exact()],
+ * });
+ */
+declare function exact<TOutput = unknown>(): EvalScorer<TOutput>;
+/**
+ * Substring / contains scorer. Passes when the normalized `expected`
+ * string appears anywhere in the agent's normalized output. Useful
+ * when the agent's phrasing varies but a key fact must be present.
+ *
+ * @example
+ * scorers: [contains()] // expected "Cairo" passes "The capital is Cairo."
+ */
+declare function contains<TOutput = unknown>(): EvalScorer<TOutput>;
+/**
+ * Predicate scorer. Wraps a boolean-returning callback into a scorer —
+ * `true` scores `1` / `passed`, `false` scores `0` / fails. The
+ * escape hatch for arbitrary assertions ("output is valid JSON", "no
+ * tool errored", "duration under budget") that don't fit exact or
+ * judge scoring.
+ *
+ * @example
+ * scorers: [predicate((ctx) => ctx.result.report.children.every(c => c.status === "completed"))]
+ */
+declare function predicate<TOutput = unknown>(fn: EvalPredicate<TOutput>): EvalScorer<TOutput>;
+//#endregion
+export { EvalPredicate, contains, exact, predicate };
+//# sourceMappingURL=scorers.d.mts.map

package/esm/eval/scorers.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorers.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/scorers.ts"],"mappings":";;;;;AAWA;;;KAAY,aAAA,uBACV,OAAA,EAAS,iBAAA,CAAkB,OAAA,gBACd,OAAA;;;;;;;;;;;AAAO;AA8BtB;;;;;;iBAAgB,KAAA,uBAA4B,UAAU,CAAC,OAAA;;AAAO;AAmC9D;;;;;;iBAAgB,QAAA,uBAA+B,UAAU,CAAC,OAAA;;AAAO;AAsCjE;;;;;;;;iBAAgB,SAAA,oBACd,EAAA,EAAI,aAAA,CAAc,OAAA,IACjB,UAAA,CAAW,OAAA"}

package/esm/eval/scorers.mjs

@@ -0,0 +1,101 @@
+//#region ../@warlock.js/ai/src/eval/scorers.ts
+/**
+* Normalize a value for case-insensitive, whitespace-trimmed string
+* comparison. Non-string values are JSON-serialized first so a
+* structured `expected` can still be matched against structured
+* `output`.
+*/
+function normalizeForComparison(value) {
+	return (typeof value === "string" ? value : JSON.stringify(value)).trim().toLowerCase();
+}
+/**
+* Exact-match scorer. Compares the agent's output against the case's
+* `expected` reference. Prefers `result.data` (parsed structured
+* output) when present, falling back to `result.text`. Comparison is
+* trimmed and case-insensitive; structured values are compared by
+* canonical JSON.
+*
+* Scores `1` / `passed: true` on a match, `0` / `passed: false`
+* otherwise. A case with no `expected` always scores `0` — exact
+* matching is meaningless without a reference.
+*
+* @example
+* const report = await agent.eval({
+*   cases: [{ name: "q", input: "2+2?", expected: "4" }],
+*   scorers: [exact()],
+* });
+*/
+function exact() {
+	return (context) => {
+		if (context.case.expected === void 0) return {
+			score: 0,
+			passed: false,
+			reason: "no expected value supplied for exact match"
+		};
+		const actual = context.output ?? context.text;
+		if (actual === void 0) return {
+			score: 0,
+			passed: false,
+			reason: "agent produced no output"
+		};
+		const matches = normalizeForComparison(actual) === normalizeForComparison(context.case.expected);
+		return {
+			score: matches ? 1 : 0,
+			passed: matches,
+			reason: matches ? "exact match" : "output did not match expected"
+		};
+	};
+}
+/**
+* Substring / contains scorer. Passes when the normalized `expected`
+* string appears anywhere in the agent's normalized output. Useful
+* when the agent's phrasing varies but a key fact must be present.
+*
+* @example
+* scorers: [contains()] // expected "Cairo" passes "The capital is Cairo."
+*/
+function contains() {
+	return (context) => {
+		if (context.case.expected === void 0) return {
+			score: 0,
+			passed: false,
+			reason: "no expected value supplied for contains match"
+		};
+		const actual = context.output ?? context.text;
+		if (actual === void 0) return {
+			score: 0,
+			passed: false,
+			reason: "agent produced no output"
+		};
+		const found = normalizeForComparison(actual).includes(normalizeForComparison(context.case.expected));
+		return {
+			score: found ? 1 : 0,
+			passed: found,
+			reason: found ? "expected substring found" : "expected substring not found"
+		};
+	};
+}
+/**
+* Predicate scorer. Wraps a boolean-returning callback into a scorer —
+* `true` scores `1` / `passed`, `false` scores `0` / fails. The
+* escape hatch for arbitrary assertions ("output is valid JSON", "no
+* tool errored", "duration under budget") that don't fit exact or
+* judge scoring.
+*
+* @example
+* scorers: [predicate((ctx) => ctx.result.report.children.every(c => c.status === "completed"))]
+*/
+function predicate(fn) {
+	return async (context) => {
+		const result = await fn(context);
+		return {
+			score: result ? 1 : 0,
+			passed: result,
+			reason: result ? "predicate passed" : "predicate failed"
+		};
+	};
+}
+
+//#endregion
+export { contains, exact, predicate };
+//# sourceMappingURL=scorers.mjs.map

package/esm/eval/scorers.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorers.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/scorers.ts"],"sourcesContent":["import type {\n  EvalScore,\n  EvalScorer,\n  EvalScorerContext,\n} from \"../contracts/agent/eval.type\";\n\n/**\n * Predicate signature for {@link predicate}. Receives the same context\n * a full scorer does and returns a boolean (sync or async). A `true`\n * verdict scores `1`, `false` scores `0`.\n */\nexport type EvalPredicate<TOutput = unknown> = (\n  context: EvalScorerContext<TOutput>,\n) => boolean | Promise<boolean>;\n\n/**\n * Normalize a value for case-insensitive, whitespace-trimmed string\n * comparison. Non-string values are JSON-serialized first so a\n * structured `expected` can still be matched against structured\n * `output`.\n */\nfunction normalizeForComparison(value: unknown): string {\n  const text = typeof value === \"string\" ? value : JSON.stringify(value);\n  return text.trim().toLowerCase();\n}\n\n/**\n * Exact-match scorer. Compares the agent's output against the case's\n * `expected` reference. Prefers `result.data` (parsed structured\n * output) when present, falling back to `result.text`. Comparison is\n * trimmed and case-insensitive; structured values are compared by\n * canonical JSON.\n *\n * Scores `1` / `passed: true` on a match, `0` / `passed: false`\n * otherwise. A case with no `expected` always scores `0` — exact\n * matching is meaningless without a reference.\n *\n * @example\n * const report = await agent.eval({\n *   cases: [{ name: \"q\", input: \"2+2?\", expected: \"4\" }],\n *   scorers: [exact()],\n * });\n */\nexport function exact<TOutput = unknown>(): EvalScorer<TOutput> {\n  return (context: EvalScorerContext<TOutput>): EvalScore => {\n    if (context.case.expected === undefined) {\n      return {\n        score: 0,\n        passed: false,\n        reason: \"no expected value supplied for exact match\",\n      };\n    }\n\n    const actual = context.output ?? context.text;\n\n    if (actual === undefined) {\n      return { score: 0, passed: false, reason: \"agent produced no output\" };\n    }\n\n    const matches =\n      normalizeForComparison(actual) === normalizeForComparison(context.case.expected);\n\n    return {\n      score: matches ? 1 : 0,\n      passed: matches,\n      reason: matches ? \"exact match\" : \"output did not match expected\",\n    };\n  };\n}\n\n/**\n * Substring / contains scorer. Passes when the normalized `expected`\n * string appears anywhere in the agent's normalized output. Useful\n * when the agent's phrasing varies but a key fact must be present.\n *\n * @example\n * scorers: [contains()] // expected \"Cairo\" passes \"The capital is Cairo.\"\n */\nexport function contains<TOutput = unknown>(): EvalScorer<TOutput> {\n  return (context: EvalScorerContext<TOutput>): EvalScore => {\n    if (context.case.expected === undefined) {\n      return {\n        score: 0,\n        passed: false,\n        reason: \"no expected value supplied for contains match\",\n      };\n    }\n\n    const actual = context.output ?? context.text;\n\n    if (actual === undefined) {\n      return { score: 0, passed: false, reason: \"agent produced no output\" };\n    }\n\n    const found = normalizeForComparison(actual).includes(\n      normalizeForComparison(context.case.expected),\n    );\n\n    return {\n      score: found ? 1 : 0,\n      passed: found,\n      reason: found ? \"expected substring found\" : \"expected substring not found\",\n    };\n  };\n}\n\n/**\n * Predicate scorer. Wraps a boolean-returning callback into a scorer —\n * `true` scores `1` / `passed`, `false` scores `0` / fails. The\n * escape hatch for arbitrary assertions (\"output is valid JSON\", \"no\n * tool errored\", \"duration under budget\") that don't fit exact or\n * judge scoring.\n *\n * @example\n * scorers: [predicate((ctx) => ctx.result.report.children.every(c => c.status === \"completed\"))]\n */\nexport function predicate<TOutput = unknown>(\n  fn: EvalPredicate<TOutput>,\n): EvalScorer<TOutput> {\n  return async (context: EvalScorerContext<TOutput>): Promise<EvalScore> => {\n    const result = await fn(context);\n\n    return {\n      score: result ? 1 : 0,\n      passed: result,\n      reason: result ? \"predicate passed\" : \"predicate failed\",\n    };\n  };\n}\n"],"mappings":";;;;;;;AAqBA,SAAS,uBAAuB,OAAwB;CAEtD,QADa,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,KAAK,EAC1D,CAAC,KAAK,CAAC,CAAC,YAAY;AACjC;;;;;;;;;;;;;;;;;;AAmBA,SAAgB,QAAgD;CAC9D,QAAQ,YAAmD;EACzD,IAAI,QAAQ,KAAK,aAAa,QAC5B,OAAO;GACL,OAAO;GACP,QAAQ;GACR,QAAQ;EACV;EAGF,MAAM,SAAS,QAAQ,UAAU,QAAQ;EAEzC,IAAI,WAAW,QACb,OAAO;GAAE,OAAO;GAAG,QAAQ;GAAO,QAAQ;EAA2B;EAGvE,MAAM,UACJ,uBAAuB,MAAM,MAAM,uBAAuB,QAAQ,KAAK,QAAQ;EAEjF,OAAO;GACL,OAAO,UAAU,IAAI;GACrB,QAAQ;GACR,QAAQ,UAAU,gBAAgB;EACpC;CACF;AACF;;;;;;;;;AAUA,SAAgB,WAAmD;CACjE,QAAQ,YAAmD;EACzD,IAAI,QAAQ,KAAK,aAAa,QAC5B,OAAO;GACL,OAAO;GACP,QAAQ;GACR,QAAQ;EACV;EAGF,MAAM,SAAS,QAAQ,UAAU,QAAQ;EAEzC,IAAI,WAAW,QACb,OAAO;GAAE,OAAO;GAAG,QAAQ;GAAO,QAAQ;EAA2B;EAGvE,MAAM,QAAQ,uBAAuB,MAAM,CAAC,CAAC,SAC3C,uBAAuB,QAAQ,KAAK,QAAQ,CAC9C;EAEA,OAAO;GACL,OAAO,QAAQ,IAAI;GACnB,QAAQ;GACR,QAAQ,QAAQ,6BAA6B;EAC/C;CACF;AACF;;;;;;;;;;;AAYA,SAAgB,UACd,IACqB;CACrB,OAAO,OAAO,YAA4D;EACxE,MAAM,SAAS,MAAM,GAAG,OAAO;EAE/B,OAAO;GACL,OAAO,SAAS,IAAI;GACpB,QAAQ;GACR,QAAQ,SAAS,qBAAqB;EACxC;CACF;AACF"}