@warlock.js/ai 4.2.11 → 4.4.0

package/esm/errors/planner-cancelled-error.d.mts

@@ -0,0 +1,33 @@
+import { ErrorCategory } from "./error-category.type.mjs";
+import { AIErrorOptions } from "./ai-error.mjs";
+import { PlannerFailedError } from "./planner-failed-error.mjs";
+
+//#region ../@warlock.js/ai/src/errors/planner-cancelled-error.d.ts
+/**
+ * Options for {@link PlannerCancelledError}. Carries the observation
+ * timestamp and the optional `controller.abort(reason)` payload.
+ */
+type PlannerCancelledErrorOptions = AIErrorOptions & {
+  /** ISO-8601 timestamp at which the abort was observed by the planner. */cancelledAt: string; /** `controller.abort(reason)` payload when the caller supplied one. */
+  reason?: string;
+};
+/**
+ * Planner run was cancelled via `AbortSignal` before it could finish.
+ * Between-step cancellation is guaranteed; mid-step cancellation is
+ * best-effort (the signal is threaded into every in-flight capability
+ * `execute()` call, but effectiveness depends on the child primitive
+ * respecting it).
+ *
+ * On cancellation the planner returns normally with
+ * `report.status === "cancelled"` and the partial step snapshots — the
+ * error is placed on `result.error` rather than thrown.
+ */
+declare class PlannerCancelledError extends PlannerFailedError {
+  static readonly defaultCategory: ErrorCategory;
+  readonly cancelledAt: string;
+  readonly reason?: string;
+  constructor(message: string, options: PlannerCancelledErrorOptions);
+}
+//#endregion
+export { PlannerCancelledError, PlannerCancelledErrorOptions };
+//# sourceMappingURL=planner-cancelled-error.d.mts.map

package/esm/errors/planner-cancelled-error.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"planner-cancelled-error.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/errors/planner-cancelled-error.ts"],"mappings":";;;;;;;AAQA;;KAAY,4BAAA,GAA+B,cAAc;EAAA,yEAEvD,WAAA;EAEA,MAAA;AAAA;AAAM;AAcR;;;;;;;;;;AAdQ,cAcK,qBAAA,SAA8B,kBAAA;EAAA,gBAClB,eAAA,EAAiB,aAAA;EAAA,SAExB,WAAA;EAAA,SACA,MAAA;cAEG,OAAA,UAAiB,OAAA,EAAS,4BAAA;AAAA"}

package/esm/errors/planner-cancelled-error.mjs

@@ -0,0 +1,29 @@
+import { PlannerFailedError } from "./planner-failed-error.mjs";
+
+//#region ../@warlock.js/ai/src/errors/planner-cancelled-error.ts
+/**
+* Planner run was cancelled via `AbortSignal` before it could finish.
+* Between-step cancellation is guaranteed; mid-step cancellation is
+* best-effort (the signal is threaded into every in-flight capability
+* `execute()` call, but effectiveness depends on the child primitive
+* respecting it).
+*
+* On cancellation the planner returns normally with
+* `report.status === "cancelled"` and the partial step snapshots — the
+* error is placed on `result.error` rather than thrown.
+*/
+var PlannerCancelledError = class extends PlannerFailedError {
+	static {
+		this.defaultCategory = "cancelled";
+	}
+	constructor(message, options) {
+		super(message, options, "PLANNER_CANCELLED");
+		this.name = "PlannerCancelledError";
+		this.cancelledAt = options.cancelledAt;
+		this.reason = options.reason;
+	}
+};
+
+//#endregion
+export { PlannerCancelledError };
+//# sourceMappingURL=planner-cancelled-error.mjs.map

package/esm/errors/planner-cancelled-error.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"planner-cancelled-error.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/errors/planner-cancelled-error.ts"],"sourcesContent":["import type { AIErrorOptions } from \"./ai-error\";\nimport type { ErrorCategory } from \"./error-category.type\";\nimport { PlannerFailedError } from \"./planner-failed-error\";\n\n/**\n * Options for {@link PlannerCancelledError}. Carries the observation\n * timestamp and the optional `controller.abort(reason)` payload.\n */\nexport type PlannerCancelledErrorOptions = AIErrorOptions & {\n  /** ISO-8601 timestamp at which the abort was observed by the planner. */\n  cancelledAt: string;\n  /** `controller.abort(reason)` payload when the caller supplied one. */\n  reason?: string;\n};\n\n/**\n * Planner run was cancelled via `AbortSignal` before it could finish.\n * Between-step cancellation is guaranteed; mid-step cancellation is\n * best-effort (the signal is threaded into every in-flight capability\n * `execute()` call, but effectiveness depends on the child primitive\n * respecting it).\n *\n * On cancellation the planner returns normally with\n * `report.status === \"cancelled\"` and the partial step snapshots — the\n * error is placed on `result.error` rather than thrown.\n */\nexport class PlannerCancelledError extends PlannerFailedError {\n  public static readonly defaultCategory: ErrorCategory = \"cancelled\";\n\n  public readonly cancelledAt: string;\n  public readonly reason?: string;\n\n  public constructor(message: string, options: PlannerCancelledErrorOptions) {\n    super(message, options, \"PLANNER_CANCELLED\");\n    this.name = \"PlannerCancelledError\";\n    this.cancelledAt = options.cancelledAt;\n    this.reason = options.reason;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AA0BA,IAAa,wBAAb,cAA2C,mBAAmB;;yBACJ;;CAKxD,AAAO,YAAY,SAAiB,SAAuC;EACzE,MAAM,SAAS,SAAS,mBAAmB;EAC3C,KAAK,OAAO;EACZ,KAAK,cAAc,QAAQ;EAC3B,KAAK,SAAS,QAAQ;CACxB;AACF"}

package/esm/errors/planner-failed-error.d.mts

@@ -0,0 +1,40 @@
+import { ErrorCategory } from "./error-category.type.mjs";
+import { AIErrorCode } from "./error-code.type.mjs";
+import { AIError, AIErrorOptions } from "./ai-error.mjs";
+
+//#region ../@warlock.js/ai/src/errors/planner-failed-error.d.ts
+/**
+ * Base class for every planner-specific failure surfaced from
+ * `planner.execute()` or thrown at authoring-time by `ai.planner()`
+ * validation.
+ *
+ * **Role.** Anchor for the `PLANNER_*` code family. Subclasses carry
+ * precise codes (`PLANNER_PLAN_INVALID`, `PLANNER_CANCELLED`); this base
+ * catches everything a planning run can produce that isn't already an
+ * agent / tool / workflow / supervisor error bubbling up from a
+ * dispatched capability.
+ *
+ * Child-execution errors (agent, tool, provider, workflow) flow through
+ * the planner unchanged — they are captured on the relevant step
+ * snapshot and surfaced on `result.error` directly, never re-wrapped.
+ *
+ * @example
+ * const result = await planner.execute("Research and summarize X");
+ * if (result.error instanceof PlannerFailedError) {
+ *   console.error(result.error.code, result.error.message);
+ * }
+ */
+declare class PlannerFailedError extends AIError {
+  /**
+   * Generic planner failures (authoring-time config violations, the
+   * `toAIError` catch-all for unexpected runtime crashes during a run)
+   * are orchestration-level provider failures. Subclasses with a more
+   * precise meaning redeclare their own — `PlannerPlanInvalidError` is
+   * `"schema"`, `PlannerCancelledError` is `"cancelled"`.
+   */
+  static readonly defaultCategory: ErrorCategory;
+  constructor(message: string, options?: AIErrorOptions, code?: AIErrorCode);
+}
+//#endregion
+export { PlannerFailedError };
+//# sourceMappingURL=planner-failed-error.d.mts.map

package/esm/errors/planner-failed-error.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"planner-failed-error.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/errors/planner-failed-error.ts"],"mappings":";;;;;;;AAyBA;;;;;;;;;;;;;;;;;;;cAAa,kBAAA,SAA2B,OAAA;;;;;;;;kBAQf,eAAA,EAAiB,aAAA;cAGtC,OAAA,UACA,OAAA,GAAU,cAAA,EACV,IAAA,GAAM,WAAA;AAAA"}

package/esm/errors/planner-failed-error.mjs

@@ -0,0 +1,37 @@
+import { AIError } from "./ai-error.mjs";
+
+//#region ../@warlock.js/ai/src/errors/planner-failed-error.ts
+/**
+* Base class for every planner-specific failure surfaced from
+* `planner.execute()` or thrown at authoring-time by `ai.planner()`
+* validation.
+*
+* **Role.** Anchor for the `PLANNER_*` code family. Subclasses carry
+* precise codes (`PLANNER_PLAN_INVALID`, `PLANNER_CANCELLED`); this base
+* catches everything a planning run can produce that isn't already an
+* agent / tool / workflow / supervisor error bubbling up from a
+* dispatched capability.
+*
+* Child-execution errors (agent, tool, provider, workflow) flow through
+* the planner unchanged — they are captured on the relevant step
+* snapshot and surfaced on `result.error` directly, never re-wrapped.
+*
+* @example
+* const result = await planner.execute("Research and summarize X");
+* if (result.error instanceof PlannerFailedError) {
+*   console.error(result.error.code, result.error.message);
+* }
+*/
+var PlannerFailedError = class extends AIError {
+	static {
+		this.defaultCategory = "provider";
+	}
+	constructor(message, options, code = "PLANNER_FAILED") {
+		super(code, message, options);
+		this.name = "PlannerFailedError";
+	}
+};
+
+//#endregion
+export { PlannerFailedError };
+//# sourceMappingURL=planner-failed-error.mjs.map

package/esm/errors/planner-failed-error.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"planner-failed-error.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/errors/planner-failed-error.ts"],"sourcesContent":["import { AIError, type AIErrorOptions } from \"./ai-error\";\nimport type { AIErrorCode } from \"./error-code.type\";\nimport type { ErrorCategory } from \"./error-category.type\";\n\n/**\n * Base class for every planner-specific failure surfaced from\n * `planner.execute()` or thrown at authoring-time by `ai.planner()`\n * validation.\n *\n * **Role.** Anchor for the `PLANNER_*` code family. Subclasses carry\n * precise codes (`PLANNER_PLAN_INVALID`, `PLANNER_CANCELLED`); this base\n * catches everything a planning run can produce that isn't already an\n * agent / tool / workflow / supervisor error bubbling up from a\n * dispatched capability.\n *\n * Child-execution errors (agent, tool, provider, workflow) flow through\n * the planner unchanged — they are captured on the relevant step\n * snapshot and surfaced on `result.error` directly, never re-wrapped.\n *\n * @example\n * const result = await planner.execute(\"Research and summarize X\");\n * if (result.error instanceof PlannerFailedError) {\n *   console.error(result.error.code, result.error.message);\n * }\n */\nexport class PlannerFailedError extends AIError {\n  /**\n   * Generic planner failures (authoring-time config violations, the\n   * `toAIError` catch-all for unexpected runtime crashes during a run)\n   * are orchestration-level provider failures. Subclasses with a more\n   * precise meaning redeclare their own — `PlannerPlanInvalidError` is\n   * `\"schema\"`, `PlannerCancelledError` is `\"cancelled\"`.\n   */\n  public static readonly defaultCategory: ErrorCategory = \"provider\";\n\n  public constructor(\n    message: string,\n    options?: AIErrorOptions,\n    code: AIErrorCode = \"PLANNER_FAILED\",\n  ) {\n    super(code, message, options);\n    this.name = \"PlannerFailedError\";\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAyBA,IAAa,qBAAb,cAAwC,QAAQ;;yBAQU;;CAExD,AAAO,YACL,SACA,SACA,OAAoB,kBACpB;EACA,MAAM,MAAM,SAAS,OAAO;EAC5B,KAAK,OAAO;CACd;AACF"}

package/esm/errors/planner-plan-invalid-error.d.mts

@@ -0,0 +1,21 @@
+import { ErrorCategory } from "./error-category.type.mjs";
+import { AIErrorOptions } from "./ai-error.mjs";
+import { PlannerFailedError } from "./planner-failed-error.mjs";
+
+//#region ../@warlock.js/ai/src/errors/planner-plan-invalid-error.d.ts
+/**
+ * The planner asked the LLM for an execution plan but the response
+ * could not be parsed / validated into a usable {@link PlannerPlan},
+ * or it referenced a capability that was never registered.
+ *
+ * Surfaced on `result.error` with `report.status === "failed"` — the
+ * planner returns normally rather than throwing, so callers branch on
+ * the typed envelope like every other primitive.
+ */
+declare class PlannerPlanInvalidError extends PlannerFailedError {
+  static readonly defaultCategory: ErrorCategory;
+  constructor(message: string, options?: AIErrorOptions);
+}
+//#endregion
+export { PlannerPlanInvalidError };
+//# sourceMappingURL=planner-plan-invalid-error.d.mts.map

package/esm/errors/planner-plan-invalid-error.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"planner-plan-invalid-error.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/errors/planner-plan-invalid-error.ts"],"mappings":";;;;;;;AAaA;;;;;;;cAAa,uBAAA,SAAgC,kBAAA;EAAA,gBACpB,eAAA,EAAiB,aAAA;cAErB,OAAA,UAAiB,OAAA,GAAU,cAAA;AAAA"}

package/esm/errors/planner-plan-invalid-error.mjs

@@ -0,0 +1,25 @@
+import { PlannerFailedError } from "./planner-failed-error.mjs";
+
+//#region ../@warlock.js/ai/src/errors/planner-plan-invalid-error.ts
+/**
+* The planner asked the LLM for an execution plan but the response
+* could not be parsed / validated into a usable {@link PlannerPlan},
+* or it referenced a capability that was never registered.
+*
+* Surfaced on `result.error` with `report.status === "failed"` — the
+* planner returns normally rather than throwing, so callers branch on
+* the typed envelope like every other primitive.
+*/
+var PlannerPlanInvalidError = class extends PlannerFailedError {
+	static {
+		this.defaultCategory = "schema";
+	}
+	constructor(message, options) {
+		super(message, options, "PLANNER_PLAN_INVALID");
+		this.name = "PlannerPlanInvalidError";
+	}
+};
+
+//#endregion
+export { PlannerPlanInvalidError };
+//# sourceMappingURL=planner-plan-invalid-error.mjs.map

package/esm/errors/planner-plan-invalid-error.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"planner-plan-invalid-error.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/errors/planner-plan-invalid-error.ts"],"sourcesContent":["import type { AIErrorOptions } from \"./ai-error\";\nimport type { ErrorCategory } from \"./error-category.type\";\nimport { PlannerFailedError } from \"./planner-failed-error\";\n\n/**\n * The planner asked the LLM for an execution plan but the response\n * could not be parsed / validated into a usable {@link PlannerPlan},\n * or it referenced a capability that was never registered.\n *\n * Surfaced on `result.error` with `report.status === \"failed\"` — the\n * planner returns normally rather than throwing, so callers branch on\n * the typed envelope like every other primitive.\n */\nexport class PlannerPlanInvalidError extends PlannerFailedError {\n  public static readonly defaultCategory: ErrorCategory = \"schema\";\n\n  public constructor(message: string, options?: AIErrorOptions) {\n    super(message, options, \"PLANNER_PLAN_INVALID\");\n    this.name = \"PlannerPlanInvalidError\";\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAaA,IAAa,0BAAb,cAA6C,mBAAmB;;yBACN;;CAExD,AAAO,YAAY,SAAiB,SAA0B;EAC5D,MAAM,SAAS,SAAS,sBAAsB;EAC9C,KAAK,OAAO;CACd;AACF"}

package/esm/eval/eval-runner.d.mts

@@ -0,0 +1,17 @@
+import { EvalOptions, EvalReport } from "../contracts/agent/eval.type.mjs";
+import { AgentContract } from "../contracts/agent/agent.contract.mjs";
+
+//#region ../@warlock.js/ai/src/eval/eval-runner.d.ts
+/**
+ * Core implementation of `agent.eval`. Runs every case sequentially
+ * (cases share the agent and may carry side effects — ordering must be
+ * deterministic), scores each, fires `onFailure` for failed cases, and
+ * assembles the aggregate {@link EvalReport}.
+ *
+ * Never throws on a case-level failure; the only throw is the
+ * authoring-time "no scorer" guard from {@link resolveScorers}.
+ */
+declare function runEval<TOutput>(agent: AgentContract<TOutput>, options: EvalOptions<TOutput>): Promise<EvalReport<TOutput>>;
+//#endregion
+export { runEval };
+//# sourceMappingURL=eval-runner.d.mts.map

package/esm/eval/eval-runner.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-runner.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/eval-runner.ts"],"mappings":";;;;;;AAmIA;;;;;;;iBAAsB,OAAA,UACpB,KAAA,EAAO,aAAA,CAAc,OAAA,GACrB,OAAA,EAAS,WAAA,CAAY,OAAA,IACpB,OAAA,CAAQ,UAAA,CAAW,OAAA"}

package/esm/eval/eval-runner.mjs

@@ -0,0 +1,121 @@
+import { AgentExecutionError } from "../errors/agent-execution-error.mjs";
+import "../errors/index.mjs";
+import { judge } from "./judge-scorer.mjs";
+import { log } from "@warlock.js/logger";
+
+//#region ../@warlock.js/ai/src/eval/eval-runner.ts
+const LOG_MODULE = "ai.eval";
+const DEFAULT_PASS_THRESHOLD = .5;
+/**
+* Resolve the scorer list for a single case. Precedence: the case's
+* own `scorers` → the suite `scorers` → a synthesized judge scorer
+* when `judge` is configured. Throws an authoring-time
+* `AgentExecutionError` when a case can resolve none — an eval suite
+* with no way to score a case is a config bug worth surfacing at the
+* call site, not a silent pass.
+*/
+function resolveScorers(evalCase, options, passThreshold) {
+	if (evalCase.scorers && evalCase.scorers.length > 0) return evalCase.scorers;
+	if (options.scorers && options.scorers.length > 0) return options.scorers;
+	if (options.judge) return [judge(options.judge, passThreshold)];
+	throw new AgentExecutionError(`eval case "${evalCase.name}" has no scorer — supply per-case "scorers", suite "scorers", or a "judge"`, { context: {
+		authoring: true,
+		case: evalCase.name
+	} });
+}
+/**
+* Decide a single scorer verdict's pass/fail. Honors an explicit
+* `passed` from the scorer; otherwise derives it from
+* `score >= passThreshold`.
+*/
+function isScorePassing(score, passThreshold) {
+	if (typeof score.passed === "boolean") return score.passed;
+	return score.score >= passThreshold;
+}
+/**
+* Merge suite-level execute options with the case's own override.
+* Per-case wins on conflict (shallow merge).
+*/
+function mergeOptions(suite, perCase) {
+	if (!suite) return perCase;
+	if (!perCase) return suite;
+	return {
+		...suite,
+		...perCase
+	};
+}
+/**
+* Run one case end-to-end: execute the agent, run every resolved
+* scorer, aggregate into an {@link EvalCaseResult}. A case passes only
+* when the agent did not error AND every scorer passed.
+*/
+async function runCase(agent, evalCase, options, passThreshold) {
+	const scorers = resolveScorers(evalCase, options, passThreshold);
+	const executeOptions = mergeOptions(options.executeOptions, evalCase.options);
+	const start = performance.now();
+	const result = await agent.execute(evalCase.input, executeOptions);
+	const duration = performance.now() - start;
+	const context = {
+		case: evalCase,
+		result,
+		output: result.data,
+		text: result.text
+	};
+	const scores = [];
+	for (const scorer of scorers) scores.push(await scorer(context));
+	const meanScore = scores.length > 0 ? scores.reduce((sum, score) => sum + score.score, 0) / scores.length : 0;
+	const allScorersPassed = scores.every((score) => isScorePassing(score, passThreshold));
+	return {
+		case: evalCase,
+		result,
+		scores,
+		score: meanScore,
+		passed: result.error === void 0 && allScorersPassed,
+		duration
+	};
+}
+/**
+* Core implementation of `agent.eval`. Runs every case sequentially
+* (cases share the agent and may carry side effects — ordering must be
+* deterministic), scores each, fires `onFailure` for failed cases, and
+* assembles the aggregate {@link EvalReport}.
+*
+* Never throws on a case-level failure; the only throw is the
+* authoring-time "no scorer" guard from {@link resolveScorers}.
+*/
+async function runEval(agent, options) {
+	const passThreshold = options.passThreshold ?? DEFAULT_PASS_THRESHOLD;
+	const start = performance.now();
+	const cases = [];
+	for (const evalCase of options.cases) {
+		const caseResult = await runCase(agent, evalCase, options, passThreshold);
+		cases.push(caseResult);
+		if (!caseResult.passed && options.onFailure) try {
+			await options.onFailure(caseResult);
+		} catch (error) {
+			log.warn(LOG_MODULE, "onFailure.hook.error", "eval onFailure handler threw", {
+				agent: agent.name,
+				case: evalCase.name,
+				error: error instanceof Error ? error.message : String(error)
+			});
+		}
+	}
+	const passedCount = cases.filter((entry) => entry.passed).length;
+	const total = cases.length;
+	const meanScore = total > 0 ? cases.reduce((sum, entry) => sum + entry.score, 0) / total : 0;
+	return {
+		agentName: agent.name,
+		total,
+		passedCount,
+		failedCount: total - passedCount,
+		passRate: total > 0 ? passedCount / total : 0,
+		meanScore,
+		passed: total > 0 && passedCount === total,
+		cases,
+		duration: performance.now() - start
+	};
+}
+
+//#endregion
+export { runEval };
+//# sourceMappingURL=eval-runner.mjs.map

package/esm/eval/eval-runner.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-runner.mjs","names":["judgeScorer"],"sources":["../../../../../../../@warlock.js/ai/src/eval/eval-runner.ts"],"sourcesContent":["import type { AgentContract } from \"../contracts/agent/agent.contract\";\nimport type { AgentExecuteOptions } from \"../contracts/agent/agent-options.type\";\nimport type {\n  EvalCase,\n  EvalCaseResult,\n  EvalOptions,\n  EvalReport,\n  EvalScore,\n  EvalScorer,\n  EvalScorerContext,\n} from \"../contracts/agent/eval.type\";\nimport { AgentExecutionError } from \"../errors\";\nimport { log } from \"@warlock.js/logger\";\nimport { judge as judgeScorer } from \"./judge-scorer\";\n\nconst LOG_MODULE = \"ai.eval\";\nconst DEFAULT_PASS_THRESHOLD = 0.5;\n\n/**\n * Resolve the scorer list for a single case. Precedence: the case's\n * own `scorers` → the suite `scorers` → a synthesized judge scorer\n * when `judge` is configured. Throws an authoring-time\n * `AgentExecutionError` when a case can resolve none — an eval suite\n * with no way to score a case is a config bug worth surfacing at the\n * call site, not a silent pass.\n */\nfunction resolveScorers<TOutput>(\n  evalCase: EvalCase<TOutput>,\n  options: EvalOptions<TOutput>,\n  passThreshold: number,\n): EvalScorer<TOutput>[] {\n  if (evalCase.scorers && evalCase.scorers.length > 0) {\n    return evalCase.scorers;\n  }\n\n  if (options.scorers && options.scorers.length > 0) {\n    return options.scorers;\n  }\n\n  if (options.judge) {\n    return [judgeScorer<TOutput>(options.judge, passThreshold)];\n  }\n\n  throw new AgentExecutionError(\n    `eval case \"${evalCase.name}\" has no scorer — supply per-case \"scorers\", suite \"scorers\", or a \"judge\"`,\n    { context: { authoring: true, case: evalCase.name } },\n  );\n}\n\n/**\n * Decide a single scorer verdict's pass/fail. Honors an explicit\n * `passed` from the scorer; otherwise derives it from\n * `score >= passThreshold`.\n */\nfunction isScorePassing(score: EvalScore, passThreshold: number): boolean {\n  if (typeof score.passed === \"boolean\") {\n    return score.passed;\n  }\n\n  return score.score >= passThreshold;\n}\n\n/**\n * Merge suite-level execute options with the case's own override.\n * Per-case wins on conflict (shallow merge).\n */\nfunction mergeOptions<TOutput>(\n  suite: AgentExecuteOptions<TOutput> | undefined,\n  perCase: AgentExecuteOptions<TOutput> | undefined,\n): AgentExecuteOptions<TOutput> | undefined {\n  if (!suite) return perCase;\n  if (!perCase) return suite;\n  return { ...suite, ...perCase };\n}\n\n/**\n * Run one case end-to-end: execute the agent, run every resolved\n * scorer, aggregate into an {@link EvalCaseResult}. A case passes only\n * when the agent did not error AND every scorer passed.\n */\nasync function runCase<TOutput>(\n  agent: AgentContract<TOutput>,\n  evalCase: EvalCase<TOutput>,\n  options: EvalOptions<TOutput>,\n  passThreshold: number,\n): Promise<EvalCaseResult<TOutput>> {\n  const scorers = resolveScorers(evalCase, options, passThreshold);\n  const executeOptions = mergeOptions(options.executeOptions, evalCase.options);\n\n  const start = performance.now();\n  const result = await agent.execute(evalCase.input, executeOptions);\n  const duration = performance.now() - start;\n\n  const context: EvalScorerContext<TOutput> = {\n    case: evalCase,\n    result,\n    output: result.data,\n    text: result.text,\n  };\n\n  const scores: EvalScore[] = [];\n\n  for (const scorer of scorers) {\n    scores.push(await scorer(context));\n  }\n\n  const meanScore =\n    scores.length > 0 ? scores.reduce((sum, score) => sum + score.score, 0) / scores.length : 0;\n\n  const allScorersPassed = scores.every((score) => isScorePassing(score, passThreshold));\n  const passed = result.error === undefined && allScorersPassed;\n\n  return {\n    case: evalCase,\n    result,\n    scores,\n    score: meanScore,\n    passed,\n    duration,\n  };\n}\n\n/**\n * Core implementation of `agent.eval`. Runs every case sequentially\n * (cases share the agent and may carry side effects — ordering must be\n * deterministic), scores each, fires `onFailure` for failed cases, and\n * assembles the aggregate {@link EvalReport}.\n *\n * Never throws on a case-level failure; the only throw is the\n * authoring-time \"no scorer\" guard from {@link resolveScorers}.\n */\nexport async function runEval<TOutput>(\n  agent: AgentContract<TOutput>,\n  options: EvalOptions<TOutput>,\n): Promise<EvalReport<TOutput>> {\n  const passThreshold = options.passThreshold ?? DEFAULT_PASS_THRESHOLD;\n  const start = performance.now();\n\n  const cases: EvalCaseResult<TOutput>[] = [];\n\n  for (const evalCase of options.cases) {\n    const caseResult = await runCase(agent, evalCase, options, passThreshold);\n\n    cases.push(caseResult);\n\n    if (!caseResult.passed && options.onFailure) {\n      try {\n        await options.onFailure(caseResult);\n      } catch (error) {\n        log.warn(LOG_MODULE, \"onFailure.hook.error\", \"eval onFailure handler threw\", {\n          agent: agent.name,\n          case: evalCase.name,\n          error: error instanceof Error ? error.message : String(error),\n        });\n      }\n    }\n  }\n\n  const passedCount = cases.filter((entry) => entry.passed).length;\n  const total = cases.length;\n  const meanScore =\n    total > 0 ? cases.reduce((sum, entry) => sum + entry.score, 0) / total : 0;\n\n  return {\n    agentName: agent.name,\n    total,\n    passedCount,\n    failedCount: total - passedCount,\n    passRate: total > 0 ? passedCount / total : 0,\n    meanScore,\n    passed: total > 0 && passedCount === total,\n    cases,\n    duration: performance.now() - start,\n  };\n}\n"],"mappings":";;;;;;AAeA,MAAM,aAAa;AACnB,MAAM,yBAAyB;;;;;;;;;AAU/B,SAAS,eACP,UACA,SACA,eACuB;CACvB,IAAI,SAAS,WAAW,SAAS,QAAQ,SAAS,GAChD,OAAO,SAAS;CAGlB,IAAI,QAAQ,WAAW,QAAQ,QAAQ,SAAS,GAC9C,OAAO,QAAQ;CAGjB,IAAI,QAAQ,OACV,OAAO,CAACA,MAAqB,QAAQ,OAAO,aAAa,CAAC;CAG5D,MAAM,IAAI,oBACR,cAAc,SAAS,KAAK,6EAC5B,EAAE,SAAS;EAAE,WAAW;EAAM,MAAM,SAAS;CAAK,EAAE,CACtD;AACF;;;;;;AAOA,SAAS,eAAe,OAAkB,eAAgC;CACxE,IAAI,OAAO,MAAM,WAAW,WAC1B,OAAO,MAAM;CAGf,OAAO,MAAM,SAAS;AACxB;;;;;AAMA,SAAS,aACP,OACA,SAC0C;CAC1C,IAAI,CAAC,OAAO,OAAO;CACnB,IAAI,CAAC,SAAS,OAAO;CACrB,OAAO;EAAE,GAAG;EAAO,GAAG;CAAQ;AAChC;;;;;;AAOA,eAAe,QACb,OACA,UACA,SACA,eACkC;CAClC,MAAM,UAAU,eAAe,UAAU,SAAS,aAAa;CAC/D,MAAM,iBAAiB,aAAa,QAAQ,gBAAgB,SAAS,OAAO;CAE5E,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,SAAS,MAAM,MAAM,QAAQ,SAAS,OAAO,cAAc;CACjE,MAAM,WAAW,YAAY,IAAI,IAAI;CAErC,MAAM,UAAsC;EAC1C,MAAM;EACN;EACA,QAAQ,OAAO;EACf,MAAM,OAAO;CACf;CAEA,MAAM,SAAsB,CAAC;CAE7B,KAAK,MAAM,UAAU,SACnB,OAAO,KAAK,MAAM,OAAO,OAAO,CAAC;CAGnC,MAAM,YACJ,OAAO,SAAS,IAAI,OAAO,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,CAAC,IAAI,OAAO,SAAS;CAE5F,MAAM,mBAAmB,OAAO,OAAO,UAAU,eAAe,OAAO,aAAa,CAAC;CAGrF,OAAO;EACL,MAAM;EACN;EACA;EACA,OAAO;EACP,QAPa,OAAO,UAAU,UAAa;EAQ3C;CACF;AACF;;;;;;;;;;AAWA,eAAsB,QACpB,OACA,SAC8B;CAC9B,MAAM,gBAAgB,QAAQ,iBAAiB;CAC/C,MAAM,QAAQ,YAAY,IAAI;CAE9B,MAAM,QAAmC,CAAC;CAE1C,KAAK,MAAM,YAAY,QAAQ,OAAO;EACpC,MAAM,aAAa,MAAM,QAAQ,OAAO,UAAU,SAAS,aAAa;EAExE,MAAM,KAAK,UAAU;EAErB,IAAI,CAAC,WAAW,UAAU,QAAQ,WAChC,IAAI;GACF,MAAM,QAAQ,UAAU,UAAU;EACpC,SAAS,OAAO;GACd,IAAI,KAAK,YAAY,wBAAwB,gCAAgC;IAC3E,OAAO,MAAM;IACb,MAAM,SAAS;IACf,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D,CAAC;EACH;CAEJ;CAEA,MAAM,cAAc,MAAM,QAAQ,UAAU,MAAM,MAAM,CAAC,CAAC;CAC1D,MAAM,QAAQ,MAAM;CACpB,MAAM,YACJ,QAAQ,IAAI,MAAM,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,CAAC,IAAI,QAAQ;CAE3E,OAAO;EACL,WAAW,MAAM;EACjB;EACA;EACA,aAAa,QAAQ;EACrB,UAAU,QAAQ,IAAI,cAAc,QAAQ;EAC5C;EACA,QAAQ,QAAQ,KAAK,gBAAgB;EACrC;EACA,UAAU,YAAY,IAAI,IAAI;CAChC;AACF"}

package/esm/eval/index.d.mts

@@ -0,0 +1,29 @@
+import { judge } from "./judge-scorer.mjs";
+import { EvalPredicate, contains, exact, predicate } from "./scorers.mjs";
+import { runEval } from "./eval-runner.mjs";
+
+//#region ../@warlock.js/ai/src/eval/index.d.ts
+/**
+ * Built-in eval scorer factories, surfaced on `ai.eval.*`.
+ *
+ * - `exact()` — pass when output equals the case `expected` (trimmed,
+ *   case-insensitive; structured values compared by canonical JSON).
+ * - `contains()` — pass when `expected` appears as a substring.
+ * - `predicate(fn)` — wrap an arbitrary boolean assertion.
+ * - `judge(config)` — LLM-as-judge scoring against a rubric.
+ *
+ * @example
+ * await myAgent.eval({
+ *   cases: [{ name: "q", input: "Capital of Egypt?", expected: "Cairo" }],
+ *   scorers: [ai.eval.contains()],
+ * });
+ */
+declare const evalScorers: {
+  exact: typeof exact;
+  contains: typeof contains;
+  predicate: typeof predicate;
+  judge: typeof judge;
+};
+//#endregion
+export { evalScorers };
+//# sourceMappingURL=index.d.mts.map

package/esm/eval/index.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/index.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;cA0Ba,WAAA"}

package/esm/eval/index.mjs

@@ -0,0 +1,30 @@
+import { judge } from "./judge-scorer.mjs";
+import { runEval } from "./eval-runner.mjs";
+import { contains, exact, predicate } from "./scorers.mjs";
+
+//#region ../@warlock.js/ai/src/eval/index.ts
+/**
+* Built-in eval scorer factories, surfaced on `ai.eval.*`.
+*
+* - `exact()` — pass when output equals the case `expected` (trimmed,
+*   case-insensitive; structured values compared by canonical JSON).
+* - `contains()` — pass when `expected` appears as a substring.
+* - `predicate(fn)` — wrap an arbitrary boolean assertion.
+* - `judge(config)` — LLM-as-judge scoring against a rubric.
+*
+* @example
+* await myAgent.eval({
+*   cases: [{ name: "q", input: "Capital of Egypt?", expected: "Cairo" }],
+*   scorers: [ai.eval.contains()],
+* });
+*/
+const evalScorers = {
+	exact,
+	contains,
+	predicate,
+	judge
+};
+
+//#endregion
+export { evalScorers };
+//# sourceMappingURL=index.mjs.map

package/esm/eval/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/index.ts"],"sourcesContent":["import { judge } from \"./judge-scorer\";\nimport { contains, exact, predicate } from \"./scorers\";\n\n// Runner — wired into AgentContract.eval() by the agent factory.\nexport { runEval } from \"./eval-runner\";\n\n// Scorer factories — re-exported individually for direct import.\nexport { contains, exact, predicate } from \"./scorers\";\nexport type { EvalPredicate } from \"./scorers\";\nexport { judge } from \"./judge-scorer\";\n\n/**\n * Built-in eval scorer factories, surfaced on `ai.eval.*`.\n *\n * - `exact()` — pass when output equals the case `expected` (trimmed,\n *   case-insensitive; structured values compared by canonical JSON).\n * - `contains()` — pass when `expected` appears as a substring.\n * - `predicate(fn)` — wrap an arbitrary boolean assertion.\n * - `judge(config)` — LLM-as-judge scoring against a rubric.\n *\n * @example\n * await myAgent.eval({\n *   cases: [{ name: \"q\", input: \"Capital of Egypt?\", expected: \"Cairo\" }],\n *   scorers: [ai.eval.contains()],\n * });\n */\nexport const evalScorers = {\n  exact,\n  contains,\n  predicate,\n  judge,\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AA0BA,MAAa,cAAc;CACzB;CACA;CACA;CACA;AACF"}

package/esm/eval/judge-scorer.d.mts

@@ -0,0 +1,21 @@
+import { EvalJudge, EvalScorer } from "../contracts/agent/eval.type.mjs";
+
+//#region ../@warlock.js/ai/src/eval/judge-scorer.d.ts
+/**
+ * LLM-as-judge scorer. Runs the judge agent on a prompt built from the
+ * case (question + expected + actual answer + rubric) and parses its
+ * `{ score, passed?, reason? }` verdict.
+ *
+ * Verdict source order: `result.data` (when the judge declares an
+ * output schema), then `result.text` parsed as JSON. A judge that
+ * errors or returns unparseable text scores `0` with the failure
+ * reason attached — a broken judge fails the case rather than crashing
+ * the suite.
+ *
+ * @example
+ * scorers: [judge({ agent: judgeAgent, rubric: "Cite a source for full marks." })]
+ */
+declare function judge<TOutput = unknown>(config: EvalJudge, passThreshold?: number): EvalScorer<TOutput>;
+//#endregion
+export { judge };
+//# sourceMappingURL=judge-scorer.d.mts.map

package/esm/eval/judge-scorer.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge-scorer.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/judge-scorer.ts"],"mappings":";;;;;AA4FA;;;;;;;;;;;;iBAAgB,KAAA,oBACd,MAAA,EAAQ,SAAA,EACR,aAAA,YACC,UAAA,CAAW,OAAA"}

package/esm/eval/judge-scorer.mjs

@@ -0,0 +1,87 @@
+import { extractJsonPayload } from "../utils/extract-json-payload.mjs";
+import { safeJsonParse } from "../utils/safe-json-parse.mjs";
+import "../utils/index.mjs";
+
+//#region ../@warlock.js/ai/src/eval/judge-scorer.ts
+/** Clamp an arbitrary numeric value into the `[0, 1]` score range. */
+function clampScore(value) {
+	if (Number.isNaN(value)) return 0;
+	if (value < 0) return 0;
+	if (value > 1) return 1;
+	return value;
+}
+/**
+* Render the prompt the judge agent receives for one case. Includes
+* the rubric (if any), the original question, the expected reference
+* (when supplied), and the actual answer — then asks for a strict JSON
+* verdict so the response is machine-parseable even without an output
+* schema.
+*/
+function buildJudgePrompt(context, judge) {
+	const lines = [];
+	if (judge.rubric) lines.push(`Grading rubric:\n${judge.rubric}`, "");
+	lines.push(`Question:\n${context.case.input}`, "");
+	if (context.case.expected !== void 0) {
+		const expectedText = typeof context.case.expected === "string" ? context.case.expected : JSON.stringify(context.case.expected);
+		lines.push(`Reference answer:\n${expectedText}`, "");
+	}
+	const actual = context.text ?? JSON.stringify(context.output ?? null);
+	lines.push(`Answer to grade:\n${actual}`, "");
+	lines.push("Respond with JSON only: { \"score\": <0..1>, \"passed\": <true|false>, \"reason\": \"<short explanation>\" }.");
+	return lines.join("\n");
+}
+/**
+* Coerce the judge agent's parsed/extracted verdict into a normalized
+* {@link EvalScore}. Defends against a judge that returns a string
+* score or omits `passed` — the caller-supplied `passThreshold`
+* derives `passed` from `score` when the judge didn't decide.
+*/
+function toEvalScore(verdict, passThreshold) {
+	const rawScore = typeof verdict.score === "string" ? Number(verdict.score) : verdict.score;
+	const score = clampScore(typeof rawScore === "number" ? rawScore : 0);
+	return {
+		score,
+		passed: typeof verdict.passed === "boolean" ? verdict.passed : score >= passThreshold,
+		reason: typeof verdict.reason === "string" ? verdict.reason : void 0
+	};
+}
+/**
+* LLM-as-judge scorer. Runs the judge agent on a prompt built from the
+* case (question + expected + actual answer + rubric) and parses its
+* `{ score, passed?, reason? }` verdict.
+*
+* Verdict source order: `result.data` (when the judge declares an
+* output schema), then `result.text` parsed as JSON. A judge that
+* errors or returns unparseable text scores `0` with the failure
+* reason attached — a broken judge fails the case rather than crashing
+* the suite.
+*
+* @example
+* scorers: [judge({ agent: judgeAgent, rubric: "Cite a source for full marks." })]
+*/
+function judge(config, passThreshold = .5) {
+	const threshold = config.passThreshold ?? passThreshold;
+	return async (context) => {
+		const prompt = buildJudgePrompt(context, config);
+		const verdictResult = await config.agent.execute(prompt);
+		if (verdictResult.error) return {
+			score: 0,
+			passed: false,
+			reason: `judge failed: ${verdictResult.error.message}`
+		};
+		if (verdictResult.data && typeof verdictResult.data === "object") return toEvalScore(verdictResult.data, threshold);
+		const text = verdictResult.text ?? "";
+		const sentinel = Symbol("judge-parse-failed");
+		const parsed = safeJsonParse(extractJsonPayload(text), sentinel);
+		if (parsed === sentinel || parsed === null || typeof parsed !== "object") return {
+			score: 0,
+			passed: false,
+			reason: "judge returned no parseable verdict"
+		};
+		return toEvalScore(parsed, threshold);
+	};
+}
+
+//#endregion
+export { judge };
+//# sourceMappingURL=judge-scorer.mjs.map

package/esm/eval/judge-scorer.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge-scorer.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/judge-scorer.ts"],"sourcesContent":["import type {\n  EvalJudge,\n  EvalScore,\n  EvalScorer,\n  EvalScorerContext,\n} from \"../contracts/agent/eval.type\";\nimport { extractJsonPayload, safeJsonParse } from \"../utils\";\n\n/**\n * Raw shape the judge agent is expected to return — either parsed from\n * `result.data` (when the judge has an output schema) or extracted\n * from `result.text`.\n */\ntype JudgeVerdict = {\n  score?: unknown;\n  passed?: unknown;\n  reason?: unknown;\n};\n\n/** Clamp an arbitrary numeric value into the `[0, 1]` score range. */\nfunction clampScore(value: number): number {\n  if (Number.isNaN(value)) return 0;\n  if (value < 0) return 0;\n  if (value > 1) return 1;\n  return value;\n}\n\n/**\n * Render the prompt the judge agent receives for one case. Includes\n * the rubric (if any), the original question, the expected reference\n * (when supplied), and the actual answer — then asks for a strict JSON\n * verdict so the response is machine-parseable even without an output\n * schema.\n */\nfunction buildJudgePrompt(context: EvalScorerContext, judge: EvalJudge): string {\n  const lines: string[] = [];\n\n  if (judge.rubric) {\n    lines.push(`Grading rubric:\\n${judge.rubric}`, \"\");\n  }\n\n  lines.push(`Question:\\n${context.case.input}`, \"\");\n\n  if (context.case.expected !== undefined) {\n    const expectedText =\n      typeof context.case.expected === \"string\"\n        ? context.case.expected\n        : JSON.stringify(context.case.expected);\n    lines.push(`Reference answer:\\n${expectedText}`, \"\");\n  }\n\n  const actual = context.text ?? JSON.stringify(context.output ?? null);\n  lines.push(`Answer to grade:\\n${actual}`, \"\");\n\n  lines.push(\n    'Respond with JSON only: { \"score\": <0..1>, \"passed\": <true|false>, \"reason\": \"<short explanation>\" }.',\n  );\n\n  return lines.join(\"\\n\");\n}\n\n/**\n * Coerce the judge agent's parsed/extracted verdict into a normalized\n * {@link EvalScore}. Defends against a judge that returns a string\n * score or omits `passed` — the caller-supplied `passThreshold`\n * derives `passed` from `score` when the judge didn't decide.\n */\nfunction toEvalScore(verdict: JudgeVerdict, passThreshold: number): EvalScore {\n  const rawScore = typeof verdict.score === \"string\" ? Number(verdict.score) : verdict.score;\n  const score = clampScore(typeof rawScore === \"number\" ? rawScore : 0);\n\n  const passed = typeof verdict.passed === \"boolean\" ? verdict.passed : score >= passThreshold;\n\n  const reason = typeof verdict.reason === \"string\" ? verdict.reason : undefined;\n\n  return { score, passed, reason };\n}\n\n/**\n * LLM-as-judge scorer. Runs the judge agent on a prompt built from the\n * case (question + expected + actual answer + rubric) and parses its\n * `{ score, passed?, reason? }` verdict.\n *\n * Verdict source order: `result.data` (when the judge declares an\n * output schema), then `result.text` parsed as JSON. A judge that\n * errors or returns unparseable text scores `0` with the failure\n * reason attached — a broken judge fails the case rather than crashing\n * the suite.\n *\n * @example\n * scorers: [judge({ agent: judgeAgent, rubric: \"Cite a source for full marks.\" })]\n */\nexport function judge<TOutput = unknown>(\n  config: EvalJudge,\n  passThreshold = 0.5,\n): EvalScorer<TOutput> {\n  const threshold = config.passThreshold ?? passThreshold;\n\n  return async (context: EvalScorerContext<TOutput>): Promise<EvalScore> => {\n    const prompt = buildJudgePrompt(context as EvalScorerContext, config);\n\n    const verdictResult = await config.agent.execute(prompt);\n\n    if (verdictResult.error) {\n      return {\n        score: 0,\n        passed: false,\n        reason: `judge failed: ${verdictResult.error.message}`,\n      };\n    }\n\n    if (verdictResult.data && typeof verdictResult.data === \"object\") {\n      return toEvalScore(verdictResult.data as JudgeVerdict, threshold);\n    }\n\n    const text = verdictResult.text ?? \"\";\n    const sentinel = Symbol(\"judge-parse-failed\");\n    const parsed = safeJsonParse<unknown>(extractJsonPayload(text), sentinel);\n\n    if (parsed === sentinel || parsed === null || typeof parsed !== \"object\") {\n      return {\n        score: 0,\n        passed: false,\n        reason: \"judge returned no parseable verdict\",\n      };\n    }\n\n    return toEvalScore(parsed as JudgeVerdict, threshold);\n  };\n}\n"],"mappings":";;;;;;AAoBA,SAAS,WAAW,OAAuB;CACzC,IAAI,OAAO,MAAM,KAAK,GAAG,OAAO;CAChC,IAAI,QAAQ,GAAG,OAAO;CACtB,IAAI,QAAQ,GAAG,OAAO;CACtB,OAAO;AACT;;;;;;;;AASA,SAAS,iBAAiB,SAA4B,OAA0B;CAC9E,MAAM,QAAkB,CAAC;CAEzB,IAAI,MAAM,QACR,MAAM,KAAK,oBAAoB,MAAM,UAAU,EAAE;CAGnD,MAAM,KAAK,cAAc,QAAQ,KAAK,SAAS,EAAE;CAEjD,IAAI,QAAQ,KAAK,aAAa,QAAW;EACvC,MAAM,eACJ,OAAO,QAAQ,KAAK,aAAa,WAC7B,QAAQ,KAAK,WACb,KAAK,UAAU,QAAQ,KAAK,QAAQ;EAC1C,MAAM,KAAK,sBAAsB,gBAAgB,EAAE;CACrD;CAEA,MAAM,SAAS,QAAQ,QAAQ,KAAK,UAAU,QAAQ,UAAU,IAAI;CACpE,MAAM,KAAK,qBAAqB,UAAU,EAAE;CAE5C,MAAM,KACJ,+GACF;CAEA,OAAO,MAAM,KAAK,IAAI;AACxB;;;;;;;AAQA,SAAS,YAAY,SAAuB,eAAkC;CAC5E,MAAM,WAAW,OAAO,QAAQ,UAAU,WAAW,OAAO,QAAQ,KAAK,IAAI,QAAQ;CACrF,MAAM,QAAQ,WAAW,OAAO,aAAa,WAAW,WAAW,CAAC;CAMpE,OAAO;EAAE;EAAO,QAJD,OAAO,QAAQ,WAAW,YAAY,QAAQ,SAAS,SAAS;EAIvD,QAFT,OAAO,QAAQ,WAAW,WAAW,QAAQ,SAAS;CAEtC;AACjC;;;;;;;;;;;;;;;AAgBA,SAAgB,MACd,QACA,gBAAgB,IACK;CACrB,MAAM,YAAY,OAAO,iBAAiB;CAE1C,OAAO,OAAO,YAA4D;EACxE,MAAM,SAAS,iBAAiB,SAA8B,MAAM;EAEpE,MAAM,gBAAgB,MAAM,OAAO,MAAM,QAAQ,MAAM;EAEvD,IAAI,cAAc,OAChB,OAAO;GACL,OAAO;GACP,QAAQ;GACR,QAAQ,iBAAiB,cAAc,MAAM;EAC/C;EAGF,IAAI,cAAc,QAAQ,OAAO,cAAc,SAAS,UACtD,OAAO,YAAY,cAAc,MAAsB,SAAS;EAGlE,MAAM,OAAO,cAAc,QAAQ;EACnC,MAAM,WAAW,OAAO,oBAAoB;EAC5C,MAAM,SAAS,cAAuB,mBAAmB,IAAI,GAAG,QAAQ;EAExE,IAAI,WAAW,YAAY,WAAW,QAAQ,OAAO,WAAW,UAC9D,OAAO;GACL,OAAO;GACP,QAAQ;GACR,QAAQ;EACV;EAGF,OAAO,YAAY,QAAwB,SAAS;CACtD;AACF"}

package/esm/eval/scorers.d.mts

@@ -0,0 +1,50 @@
+import { EvalScorer, EvalScorerContext } from "../contracts/agent/eval.type.mjs";
+
+//#region ../@warlock.js/ai/src/eval/scorers.d.ts
+/**
+ * Predicate signature for {@link predicate}. Receives the same context
+ * a full scorer does and returns a boolean (sync or async). A `true`
+ * verdict scores `1`, `false` scores `0`.
+ */
+type EvalPredicate<TOutput = unknown> = (context: EvalScorerContext<TOutput>) => boolean | Promise<boolean>;
+/**
+ * Exact-match scorer. Compares the agent's output against the case's
+ * `expected` reference. Prefers `result.data` (parsed structured
+ * output) when present, falling back to `result.text`. Comparison is
+ * trimmed and case-insensitive; structured values are compared by
+ * canonical JSON.
+ *
+ * Scores `1` / `passed: true` on a match, `0` / `passed: false`
+ * otherwise. A case with no `expected` always scores `0` — exact
+ * matching is meaningless without a reference.
+ *
+ * @example
+ * const report = await agent.eval({
+ *   cases: [{ name: "q", input: "2+2?", expected: "4" }],
+ *   scorers: [exact()],
+ * });
+ */
+declare function exact<TOutput = unknown>(): EvalScorer<TOutput>;
+/**
+ * Substring / contains scorer. Passes when the normalized `expected`
+ * string appears anywhere in the agent's normalized output. Useful
+ * when the agent's phrasing varies but a key fact must be present.
+ *
+ * @example
+ * scorers: [contains()] // expected "Cairo" passes "The capital is Cairo."
+ */
+declare function contains<TOutput = unknown>(): EvalScorer<TOutput>;
+/**
+ * Predicate scorer. Wraps a boolean-returning callback into a scorer —
+ * `true` scores `1` / `passed`, `false` scores `0` / fails. The
+ * escape hatch for arbitrary assertions ("output is valid JSON", "no
+ * tool errored", "duration under budget") that don't fit exact or
+ * judge scoring.
+ *
+ * @example
+ * scorers: [predicate((ctx) => ctx.result.report.children.every(c => c.status === "completed"))]
+ */
+declare function predicate<TOutput = unknown>(fn: EvalPredicate<TOutput>): EvalScorer<TOutput>;
+//#endregion
+export { EvalPredicate, contains, exact, predicate };
+//# sourceMappingURL=scorers.d.mts.map

package/esm/eval/scorers.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorers.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/scorers.ts"],"mappings":";;;;;AAWA;;;KAAY,aAAA,uBACV,OAAA,EAAS,iBAAA,CAAkB,OAAA,gBACd,OAAA;;;;;;;;;;;AAAO;AA8BtB;;;;;;iBAAgB,KAAA,uBAA4B,UAAU,CAAC,OAAA;;AAAO;AAmC9D;;;;;;iBAAgB,QAAA,uBAA+B,UAAU,CAAC,OAAA;;AAAO;AAsCjE;;;;;;;;iBAAgB,SAAA,oBACd,EAAA,EAAI,aAAA,CAAc,OAAA,IACjB,UAAA,CAAW,OAAA"}