@agentproto/eval 0.2.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jeremy (agentik.net)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,236 @@
+# @agentproto/eval
+
+Deterministic reference scorers for agentproto — shipped as AIP-14 TOOL
+contracts plus one builtin AIP-30 PROVIDER.
+
+## A scorer is a tool
+
+There is **no separate "scorer port"**. A scorer is an ordinary AIP-14 tool
+authored with `defineTool`, whose `outputSchema` is the shared `Score` shape,
+and implemented with `implementTool` bundled in a `defineDriver` — exactly like
+every other builtin. That reuse buys the registry, retries, and the
+`toMastraTool` / `toAiSdkTool` projections for free.
+
+```ts
+export const scoreSchema = z.object({
+  value: z.number().min(0).max(1), // normalized score
+  passed: z.boolean(),             // clears the scorer's threshold
+  label: z.string(),               // scorer id, e.g. "exact-match"
+  rationale: z.string().optional(),// human-readable why
+})
+```
+
+## The four deterministic scorers
+
+All are pure functions of their input — no LLM / model calls. For a
+model-backed scorer, see [LLM-as-judge scorer](#llm-as-judge-scorer) below.
+
+| Tool id                  | Input                                        | Behavior |
+| ------------------------ | -------------------------------------------- | -------- |
+| `eval.exact-match`       | `{ actual, expected, trim? }`                | `value` 1 when equal (after optional trim), else 0. |
+| `eval.regex-match`       | `{ actual, pattern, flags? }`                | 1 when the compiled RegExp matches, else 0. An **invalid pattern** returns a typed failure `Score` (`value 0`, `rationale: "invalid pattern: …"`) — it never throws. |
+| `eval.json-schema-valid` | `{ actual, schema }`                         | Minimal structural check of top-level `required` + `properties.type` only. 1 when valid, else 0 with the first violation in `rationale`. |
+| `eval.latency-budget`    | `{ durationMs, budgetMs }`                   | `passed = durationMs <= budgetMs`; `value` is 1 within budget, else `max(0, 1 - (durationMs - budgetMs) / budgetMs)`. |
+
+### `eval.json-schema-valid` limitation
+
+To keep this package light (no `ajv`), the schema check is intentionally
+minimal: it verifies the top-level `type`, that every name in `required` is
+present, and that any `properties.<name>.type` matches the value's primitive
+type. It does **not** recurse into nested schemas, and does not implement the
+rest of JSON Schema (`enum`, `format`, `oneOf`, `items`, …). A fuller
+validator is a later step.
+
+## Running a scorer
+
+Invoke any scorer through the driver's `runTool`, passing the provider as the
+sole candidate:
+
+```ts
+import { runTool } from "@agentproto/driver"
+import { exactMatchTool, evalScorersProvider } from "@agentproto/eval"
+
+const score = await runTool({
+  tool: exactMatchTool,
+  candidates: [evalScorersProvider],
+  input: { actual: "hello", expected: "hello" },
+})
+// → { value: 1, passed: true, label: "exact-match", rationale: "actual equals expected" }
+```
+
+## Running a suite
+
+An **eval is a workflow**: for each case it runs a `target`, then scores the
+target's output with one scorer `tool` step per binding, aggregates, and reports
+through the `@agentproto/telemetry` port. `runEval` composes a
+`@agentproto/workflow-runtime` `RuntimeWorkflow` per case — the target's
+structured output is the input selector for every scorer step — and rolls the
+per-case results up into an `EvalReport`.
+
+Each scorer is authored as a `ScorerBinding` (its `tool`, its `driver`, and a
+`mapInput` that shapes the target output into the scorer's input) and boxed with
+`bindScorer` so scorers with different input types coexist in one suite.
+
+```ts
+import { arrayTelemetry } from "@agentproto/telemetry"
+import {
+  runEval,
+  bindScorer,
+  exactMatchTool,
+  latencyBudgetTool,
+  evalScorersProvider,
+  type EvalEvent,
+  type TypedEvalSuite,
+} from "@agentproto/eval"
+
+interface Answer { text: string; latencyMs: number }
+
+const suite: TypedEvalSuite<{ prompt: string }, Answer> = {
+  id: "greeting-suite",
+  cases: [
+    { id: "hello", input: { prompt: "hi" }, expected: "hello" },
+    { id: "world", input: { prompt: "wo" }, expected: "world" },
+  ],
+  scorers: [
+    bindScorer<Answer, { actual: string; expected: string }>({
+      id: "exact",
+      tool: exactMatchTool,
+      driver: evalScorersProvider,
+      mapInput: ({ output, expected }) => ({
+        actual: output.text,
+        expected: typeof expected === "string" ? expected : "",
+      }),
+    }),
+    bindScorer<Answer, { durationMs: number; budgetMs: number }>({
+      id: "latency",
+      tool: latencyBudgetTool,
+      driver: evalScorersProvider,
+      mapInput: ({ output }) => ({ durationMs: output.latencyMs, budgetMs: 100 }),
+    }),
+  ],
+}
+
+const telemetry = arrayTelemetry<EvalEvent>()
+const report = await runEval(suite, {
+  target: async ({ prompt }) => ({ text: prompt === "hi" ? "hello" : "world", latencyMs: 10 }),
+  telemetry,
+})
+// report.passedCount / report.meanValue aggregate across cases;
+// telemetry.events runs eval.started … eval.finished in order.
+```
+
+The `target` is a plain async function (not required to be a tool); a
+tool/agent target is a documented follow-up. Events ride the shared telemetry
+port — `Telemetry<EvalEvent>` — under the `agentproto/eval/v1` schema
+(`eval.started`, `eval.case.started`, `eval.case.scored`, `eval.case.finished`,
+`eval.finished`). Sinks MUST tolerate unknown kinds (forward-compat).
+
+## LLM-as-judge scorer
+
+`eval.llm-judge` is a **model-backed** scorer — but it stays a TOOL, exactly
+like the four deterministic scorers above. What differs is where the model
+lives: the injected judge is closed over by the **driver**, not threaded
+through tool input/context, via `makeLlmJudgeDriver(judge)`. That means it
+composes with the existing `bindScorer` / `runEval` with zero changes to
+either.
+
+The judge itself is an injected function — `JudgeFn` — so this package stays
+vendor-neutral: no LLM SDK, no network dependency here.
+
+```ts
+export type JudgeFn = (args: {
+  output: JsonValue
+  criteria: string
+  expected?: JsonValue
+}) => Promise<JudgeVerdict>
+// JudgeVerdict = { value: number (0..1); passed?: boolean; rationale?: string }
+```
+
+Build a driver around a judge, then either call the tool directly or use the
+`llmJudge(...)` convenience to get a ready-to-use `ScorerBinding`:
+
+```ts
+import { runTool } from "@agentproto/driver"
+import {
+  llmJudgeTool,
+  makeLlmJudgeDriver,
+  llmJudge,
+  bindScorer,
+  type JudgeFn,
+} from "@agentproto/eval"
+
+// A real judge is injected by the caller — an LLM call, an agent session,
+// or the supervisor's judge-gate. Shown here as a stand-in.
+const judge: JudgeFn = async ({ output, criteria }) => {
+  // ... call out to a model, or whatever satisfies JudgeFn ...
+  return { value: 0.9, rationale: "meets the criteria" }
+}
+
+// Direct tool invocation:
+const score = await runTool({
+  tool: llmJudgeTool,
+  candidates: [makeLlmJudgeDriver(judge, { threshold: 0.7 })],
+  input: { output: "the produced answer", criteria: "Is the answer helpful?" },
+})
+
+// Or as a suite scorer:
+const binding = bindScorer(
+  llmJudge<Answer>({
+    id: "helpfulness",
+    judge,
+    criteria: "Is the answer helpful and correct?",
+    threshold: 0.7,
+    mapOutput: ({ output }) => output.text,
+  }),
+)
+```
+
+`passed` resolution: when the judge's verdict includes an explicit `passed`,
+it **wins** over the threshold comparison — a judge that says `passed: false`
+is never overridden by a high `value`, and vice versa. Otherwise `passed =
+value >= threshold` (`threshold` defaults to `0.5`).
+
+A real adapter wiring `JudgeFn` up to an agent session or the supervisor's
+judge-gate is a documented follow-up — not built in this package.
+
+## As a CI gate
+
+`toVitest` turns a suite into vitest test registrations — one `it(caseId)` per
+case that runs the case and asserts every scorer passed. Vitest's own
+`describe` / `it` / `expect` are **injected**, so this package carries no vitest
+runtime dependency:
+
+```ts
+import { describe, it, expect } from "vitest"
+import { toVitest } from "@agentproto/eval"
+
+toVitest(
+  suite,
+  { target: async ({ prompt }) => ({ text: prompt === "hi" ? "hello" : "world", latencyMs: 10 }) },
+  { describe, it, expect },
+)
+```
+
+## Exports
+
+- `runEval`, `bindScorer` — the eval harness (`EvalCase`, `ScorerBinding`,
+  `BoundScorer`, `EvalSuite`, `TypedEvalSuite`, `EvalReport`, `CaseReport`,
+  `CaseScore`, `RunEvalOptions`)
+- `EVAL_EVENT_SCHEMA`, `EvalEvent` — the telemetry-port event union
+- `toVitest` — the CI-gate bridge (`VitestHooks`, `ExpectApi`, `ToVitestOptions`)
+- `JsonValue`, `JsonObject`, `jsonValueSchema` — the canonical JSON model
+- `scoreSchema`, `Score`
+- `exactMatchTool` / `exactMatchImpl`
+- `regexMatchTool` / `regexMatchImpl`
+- `jsonSchemaValidTool` / `jsonSchemaValidImpl` (`MinimalJsonSchema`)
+- `latencyBudgetTool` / `latencyBudgetImpl`
+- `evalScorersProvider` — the builtin PROVIDER bundling all four
+- `llmJudgeTool` — the `eval.llm-judge` TOOL contract
+- `makeLlmJudgeDriver` — build a DRIVER that closes over an injected `JudgeFn`
+- `llmJudge` — convenience: build a ready-to-use `ScorerBinding` around a judge
+- `JudgeFn`, `JudgeVerdict`, `judgeVerdictSchema`, `LlmJudgeInput`,
+  `MakeLlmJudgeDriverOptions`, `LlmJudgeBinding`
+
+## License
+
+MIT

package/dist/index.d.ts

@@ -0,0 +1,449 @@
+import * as _agentproto_driver from '@agentproto/driver';
+import { DriverHandle } from '@agentproto/driver';
+import { z } from 'zod';
+import * as _agentproto_tool from '@agentproto/tool';
+import { ToolHandle } from '@agentproto/tool';
+import { RuntimeWorkflow } from '@agentproto/workflow-runtime';
+import { Telemetry } from '@agentproto/telemetry';
+
+/**
+ * The shared output shape every scorer produces.
+ *
+ * A scorer is an AIP-14 TOOL whose `outputSchema` is exactly this — there is
+ * no separate "scorer port". Reusing `Score` as the tool output means the
+ * registry, retries, and `toMastraTool` / `toAiSdkTool` projections all apply
+ * to scorers for free, identical to any other builtin tool.
+ */
+declare const scoreSchema: z.ZodObject<{
+    value: z.ZodNumber;
+    passed: z.ZodBoolean;
+    label: z.ZodString;
+    rationale: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+type Score = z.infer<typeof scoreSchema>;
+
+/**
+ * Recursive JSON value union. Modelling the JSON payload explicitly (rather
+ * than reaching for `any` / `unknown`) lets consumers inspect values with
+ * proper narrowing and no casts. This is the ONE canonical `JsonValue` for the
+ * package — scorers, the eval harness, and the vitest bridge all import it from
+ * here so there is no divergent redeclaration.
+ */
+type JsonValue = string | number | boolean | null | JsonValue[] | {
+    [key: string]: JsonValue;
+};
+/** A plain JSON object (not `null`, not an array). */
+type JsonObject = {
+    [key: string]: JsonValue;
+};
+/** Zod schema mirroring {@link JsonValue}. */
+declare const jsonValueSchema: z.ZodType<JsonValue>;
+
+declare const exactMatchTool: _agentproto_tool.ToolHandle<{
+    actual: string;
+    expected: string;
+    trim?: boolean | undefined;
+}, {
+    value: number;
+    passed: boolean;
+    label: string;
+    rationale?: string | undefined;
+}, _agentproto_tool.ToolContext>;
+declare const exactMatchImpl: _agentproto_driver.ToolImplementation<{
+    actual: string;
+    expected: string;
+    trim?: boolean | undefined;
+}, {
+    value: number;
+    passed: boolean;
+    label: string;
+    rationale?: string | undefined;
+}, _agentproto_tool.ToolContext>;
+declare const regexMatchTool: _agentproto_tool.ToolHandle<{
+    actual: string;
+    pattern: string;
+    flags?: string | undefined;
+}, {
+    value: number;
+    passed: boolean;
+    label: string;
+    rationale?: string | undefined;
+}, _agentproto_tool.ToolContext>;
+declare const regexMatchImpl: _agentproto_driver.ToolImplementation<{
+    actual: string;
+    pattern: string;
+    flags?: string | undefined;
+}, {
+    value: number;
+    passed: boolean;
+    label: string;
+    rationale?: string | undefined;
+}, _agentproto_tool.ToolContext>;
+/**
+ * A deliberately minimal, dependency-free JSON-schema shape.
+ *
+ * LIMITATION: this scorer checks ONLY the two most common structural
+ * constraints on a top-level object — that every name in `required` is
+ * present, and that any name in `properties` declaring a `type` has a value
+ * of that primitive type. It does NOT recurse into nested schemas, and does
+ * not implement the rest of JSON Schema (enum, format, oneOf, items, …). A
+ * full validator (e.g. ajv) is a later step; keeping this package light is
+ * the point.
+ */
+interface MinimalJsonSchema {
+    readonly type?: string;
+    readonly required?: readonly string[];
+    readonly properties?: {
+        readonly [key: string]: {
+            readonly type?: string;
+        };
+    };
+}
+declare const jsonSchemaValidTool: _agentproto_tool.ToolHandle<{
+    actual: JsonValue;
+    schema: MinimalJsonSchema;
+}, {
+    value: number;
+    passed: boolean;
+    label: string;
+    rationale?: string | undefined;
+}, _agentproto_tool.ToolContext>;
+declare const jsonSchemaValidImpl: _agentproto_driver.ToolImplementation<{
+    actual: JsonValue;
+    schema: MinimalJsonSchema;
+}, {
+    value: number;
+    passed: boolean;
+    label: string;
+    rationale?: string | undefined;
+}, _agentproto_tool.ToolContext>;
+declare const latencyBudgetTool: _agentproto_tool.ToolHandle<{
+    durationMs: number;
+    budgetMs: number;
+}, {
+    value: number;
+    passed: boolean;
+    label: string;
+    rationale?: string | undefined;
+}, _agentproto_tool.ToolContext>;
+declare const latencyBudgetImpl: _agentproto_driver.ToolImplementation<{
+    durationMs: number;
+    budgetMs: number;
+}, {
+    value: number;
+    passed: boolean;
+    label: string;
+    rationale?: string | undefined;
+}, _agentproto_tool.ToolContext>;
+
+/**
+ * Structured events emitted while an eval suite runs.
+ *
+ * The union mirrors the shape of `@agentproto/telemetry`'s `TelemetryEvent`:
+ * a discriminated union keyed on `kind`, every member carrying a correlation
+ * id (`runId`) and an ISO `at` timestamp so a sink can rebuild a per-run span
+ * tree. It rides the same {@link Telemetry} port — a `runEval` call takes a
+ * `Telemetry<EvalEvent>` sink.
+ *
+ * Sinks MUST tolerate unknown kinds — future versions may add event types
+ * under the same `agentproto/eval/v1` schema (identical forward-compat
+ * contract as the telemetry port).
+ */
+/** Schema identifier for this event family. */
+declare const EVAL_EVENT_SCHEMA: "agentproto/eval/v1";
+type EvalEvent = {
+    readonly kind: "eval.started";
+    readonly runId: string;
+    readonly at: string;
+    readonly suiteId: string;
+    readonly caseCount: number;
+    readonly scorerCount: number;
+} | {
+    readonly kind: "eval.case.started";
+    readonly runId: string;
+    readonly at: string;
+    readonly caseId: string;
+} | {
+    readonly kind: "eval.case.scored";
+    readonly runId: string;
+    readonly at: string;
+    readonly caseId: string;
+    readonly scorerId: string;
+    readonly value: number;
+    readonly passed: boolean;
+} | {
+    readonly kind: "eval.case.finished";
+    readonly runId: string;
+    readonly at: string;
+    readonly caseId: string;
+    /** True when EVERY scorer bound to the case passed. */
+    readonly passed: boolean;
+} | {
+    readonly kind: "eval.finished";
+    readonly runId: string;
+    readonly at: string;
+    readonly suiteId: string;
+    readonly total: number;
+    readonly passedCount: number;
+    readonly meanValue: number;
+    readonly durationMs: number;
+};
+
+/**
+ * `runEval` — the eval harness.
+ *
+ * The thesis this file proves: **an eval is a workflow that runs a target then
+ * scores its output with scorer-tools, reporting through the telemetry port.**
+ *
+ * For each case we build a {@link RuntimeWorkflow} that composes
+ *   target → one scorer `tool` step per binding → a per-case aggregate transform
+ * and execute it with `runWorkflow` (AIP-15). The target's structured output is
+ * the input selector for every scorer step — that is where composable tool I/O
+ * is demonstrated. `runEval` then aggregates across cases and emits
+ * {@link EvalEvent}s through an injected `Telemetry<EvalEvent>` sink.
+ *
+ * The `target` is a plain async function here (not required to be a tool); a
+ * tool/agent target is a documented follow-up.
+ */
+
+/** One case: an input for the target and an optional expected reference. */
+interface EvalCase<I> {
+    readonly id: string;
+    readonly input: I;
+    /** Optional reference value, modelled as JSON (never `unknown`/`any`). */
+    readonly expected?: JsonValue;
+}
+/** Context handed to a binding's `mapInput` when scoring one case's output. */
+interface ScorerInputContext<A> {
+    readonly output: A;
+    readonly expected?: JsonValue;
+}
+/**
+ * One scorer applied to the target output. Generic over the target output type
+ * `A` and the scorer's own input type `S` (captured when the binding is
+ * authored). {@link bindScorer} erases `S` so heterogeneous scorers coexist in
+ * one `EvalSuite.scorers` array without an `any`.
+ */
+interface ScorerBinding<A, S = JsonValue> {
+    /** Label for this binding, e.g. "exact". */
+    readonly id: string;
+    /** The scorer TOOL contract — its `outputSchema` is the shared {@link Score}. */
+    readonly tool: ToolHandle<S, Score>;
+    /** The DRIVER that implements the scorer tool. */
+    readonly driver: DriverHandle;
+    /** Derive the scorer's input from the target's output (+ optional expected). */
+    readonly mapInput: (ctx: ScorerInputContext<A>) => S;
+}
+/**
+ * The uniform, `S`-erased view of a binding stored in a suite. It exposes the
+ * authored `id` plus a closure that runs the scorer for a given case output —
+ * the existential box that lets a suite hold scorers with different input
+ * types with no `any` at the collection boundary.
+ */
+interface BoundScorer<A> {
+    readonly id: string;
+    /** Build the AIP-15 tool step that scores one case's output. */
+    toStep(stepId: string, output: A, expected?: JsonValue): RuntimeWorkflow["steps"][number];
+}
+/**
+ * Box a typed {@link ScorerBinding} into a {@link BoundScorer}, capturing the
+ * scorer input type `S` inside the closure so the outer type is `S`-free.
+ */
+declare function bindScorer<A, S>(binding: ScorerBinding<A, S>): BoundScorer<A>;
+interface EvalSuite<A> {
+    readonly id: string;
+    readonly cases: readonly EvalCase<unknown>[];
+    readonly scorers: readonly BoundScorer<A>[];
+}
+/** One scorer's outcome on one case. */
+interface CaseScore {
+    readonly scorerId: string;
+    readonly score: Score;
+}
+/** The per-case rollup carried in the final report. */
+interface CaseReport {
+    readonly caseId: string;
+    readonly passed: boolean;
+    readonly scores: readonly CaseScore[];
+}
+interface EvalReport {
+    readonly runId: string;
+    readonly suiteId: string;
+    readonly total: number;
+    readonly passedCount: number;
+    /** Mean of every scorer value across every case (0 when there are none). */
+    readonly meanValue: number;
+    readonly cases: readonly CaseReport[];
+}
+interface RunEvalOptions<I, A> {
+    /** Produce the target output for a case input. */
+    readonly target: (input: I) => Promise<A>;
+    /** Sink for {@link EvalEvent}s. Defaults to a no-op. */
+    readonly telemetry?: Telemetry<EvalEvent>;
+    /** Correlation id for this run. Defaults to a deterministic suite-based id. */
+    readonly runId?: string;
+}
+/** A suite whose cases share the target input type `I`. */
+interface TypedEvalSuite<I, A> extends EvalSuite<A> {
+    readonly cases: readonly EvalCase<I>[];
+}
+/**
+ * Run every case in the suite, scoring each with all bound scorers, and return
+ * the aggregated {@link EvalReport}. Emits `eval.started` … `eval.finished`.
+ */
+declare function runEval<I, A>(suite: TypedEvalSuite<I, A>, opts: RunEvalOptions<I, A>): Promise<EvalReport>;
+
+/**
+ * `llmJudge` — a model-backed scorer.
+ *
+ * Design (locked): a model-backed scorer stays a TOOL, exactly like the
+ * deterministic scorers in `scorers.ts`. What differs is where the model
+ * lives: the injected {@link JudgeFn} is closed over by the DRIVER (built by
+ * {@link makeLlmJudgeDriver}), never threaded through tool input/context. That
+ * keeps `eval.llm-judge`'s contract identical in shape to every other scorer
+ * tool, so it composes with the existing `bindScorer` / `runEval` in
+ * run-eval.ts with ZERO changes there.
+ *
+ * The judge itself is vendor-neutral: {@link JudgeFn} is just an injected
+ * async function. This file has no LLM SDK and no network dependency — a real
+ * adapter (an agent session, a supervisor judge-gate, …) is a documented
+ * follow-up, not built here.
+ */
+/** Zod schema for the raw verdict a {@link JudgeFn} produces. */
+declare const judgeVerdictSchema: z.ZodObject<{
+    value: z.ZodNumber;
+    passed: z.ZodOptional<z.ZodBoolean>;
+    rationale: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+type JudgeVerdict = z.infer<typeof judgeVerdictSchema>;
+/**
+ * The seam a real judge satisfies: given the produced `output`, free-form
+ * grading `criteria`, and an optional `expected` reference, return a
+ * {@link JudgeVerdict}. Callers supply this — a real LLM call, an agent
+ * session, or the supervisor's judge-gate all satisfy the same shape.
+ * Deliberately no LLM SDK / network types here: keeping this package
+ * vendor-neutral is the point.
+ */
+type JudgeFn = (args: {
+    readonly output: JsonValue;
+    readonly criteria: string;
+    readonly expected?: JsonValue;
+}) => Promise<JudgeVerdict>;
+/** Tool input for `eval.llm-judge`. */
+interface LlmJudgeInput {
+    readonly output: JsonValue;
+    readonly criteria: string;
+    readonly expected?: JsonValue;
+}
+declare const llmJudgeTool: _agentproto_tool.ToolHandle<{
+    output: JsonValue;
+    criteria: string;
+    expected?: JsonValue | undefined;
+}, {
+    value: number;
+    passed: boolean;
+    label: string;
+    rationale?: string | undefined;
+}, _agentproto_tool.ToolContext>;
+interface MakeLlmJudgeDriverOptions {
+    /** Minimum `value` to count as passed when the judge omits `passed`. Default 0.5. */
+    readonly threshold?: number;
+}
+/**
+ * Build a DRIVER that implements `eval.llm-judge` by delegating to `judge`.
+ * This is the one seam where the injected model-backed capability enters the
+ * system — everything downstream (`bindScorer`, `runEval`) stays unaware that
+ * this scorer is model-backed at all.
+ */
+declare function makeLlmJudgeDriver(judge: JudgeFn, opts?: MakeLlmJudgeDriverOptions): DriverHandle;
+interface LlmJudgeBinding<A> {
+    /** Label for this binding, e.g. "helpfulness". */
+    readonly id: string;
+    /** The injected judge capability. */
+    readonly judge: JudgeFn;
+    /** Free-form grading criteria/rubric handed to the judge on every call. */
+    readonly criteria: string;
+    /** Minimum value to count as passed when the judge omits `passed`. Default 0.5. */
+    readonly threshold?: number;
+    /** Derive the judge's `output` input from the target's output. */
+    readonly mapOutput: (ctx: {
+        output: A;
+        expected?: JsonValue;
+    }) => JsonValue;
+}
+/**
+ * Convenience: build a {@link ScorerBinding} ready to drop into a suite's
+ * `scorers` via `bindScorer` — wires up `eval.llm-judge`, a driver built with
+ * {@link makeLlmJudgeDriver}, and the input mapping in one call.
+ */
+declare function llmJudge<A>(binding: LlmJudgeBinding<A>): ScorerBinding<A, LlmJudgeInput>;
+
+/**
+ * `toVitest` — turn an {@link TypedEvalSuite} into vitest test registrations.
+ *
+ * The vitest primitives (`describe` / `it` / `expect`) are INJECTED by the
+ * host's own `import ... from "vitest"`, so THIS source carries no vitest
+ * runtime dependency (nothing to bundle, nothing to version-pin). It registers
+ * one `it(caseId)` per case that runs the case through {@link runEval} and
+ * asserts every scorer passed — the same workflow the harness runs, wired as a
+ * CI gate.
+ */
+
+/** The subset of vitest's `expect` this bridge needs. */
+interface ExpectApi {
+    (actual: unknown): {
+        toBe(expected: unknown): void;
+    };
+}
+/** The subset of vitest's `describe`/`it` this bridge needs. */
+interface VitestHooks {
+    describe(name: string, body: () => void): void;
+    it(name: string, body: () => Promise<void> | void): void;
+    expect: ExpectApi;
+}
+type ToVitestOptions<I, A> = RunEvalOptions<I, A>;
+/**
+ * Register `describe(suite.id)` with one `it` per case. Each `it` runs the
+ * single-case suite through {@link runEval} and asserts the case passed (all
+ * scorers green). Failing scorers surface as a failing vitest assertion.
+ */
+declare function toVitest<I, A>(suite: TypedEvalSuite<I, A>, opts: ToVitestOptions<I, A>, hooks: VitestHooks): void;
+
+/**
+ * @agentproto/eval — deterministic reference scorers.
+ *
+ * A scorer IS an AIP-14 TOOL whose `outputSchema` is the shared {@link Score}
+ * shape; there is no separate "scorer port". Each scorer is authored with
+ * `defineTool` + `implementTool` and bundled in a single builtin AIP-30
+ * PROVIDER, exactly like every other builtin. Invoke one through the driver:
+ *
+ * ```ts
+ * import { runTool } from "@agentproto/driver"
+ * import { exactMatchTool, evalScorersProvider } from "@agentproto/eval"
+ *
+ * const score = await runTool({
+ *   tool: exactMatchTool,
+ *   candidates: [evalScorersProvider],
+ *   input: { actual: "hello", expected: "hello" },
+ * })
+ * // → { value: 1, passed: true, label: "exact-match", rationale: "…" }
+ * ```
+ *
+ * Alongside the deterministic scorers, `eval.llm-judge` (see judge.ts) is a
+ * model-backed scorer whose judge is an INJECTED function — the driver
+ * closes over it via `makeLlmJudgeDriver(judge)`. This package still carries
+ * no LLM SDK or network dependency of its own; a real adapter (agent
+ * session, supervisor judge-gate, …) is a documented follow-up.
+ *
+ * Spec: https://agentproto.sh/docs/aip-14 (TOOL), /docs/aip-30 (PROVIDER)
+ */
+declare const SPEC_NAME: "agenteval/v1";
+declare const SPEC_VERSION: "1.0.0-alpha";
+
+/**
+ * Builtin AIP-30 PROVIDER bundling the four deterministic ref scorers.
+ * `kind: "builtin"` — pure in-process functions, no subprocess / network hop.
+ */
+declare const evalScorersProvider: _agentproto_driver.DriverHandle;
+
+export { type BoundScorer, type CaseReport, type CaseScore, EVAL_EVENT_SCHEMA, type EvalCase, type EvalEvent, type EvalReport, type EvalSuite, type ExpectApi, type JsonObject, type JsonValue, type JudgeFn, type JudgeVerdict, type LlmJudgeBinding, type LlmJudgeInput, type MakeLlmJudgeDriverOptions, type MinimalJsonSchema, type RunEvalOptions, SPEC_NAME, SPEC_VERSION, type Score, type ScorerBinding, type ScorerInputContext, type ToVitestOptions, type TypedEvalSuite, type VitestHooks, bindScorer, evalScorersProvider, exactMatchImpl, exactMatchTool, jsonSchemaValidImpl, jsonSchemaValidTool, jsonValueSchema, judgeVerdictSchema, latencyBudgetImpl, latencyBudgetTool, llmJudge, llmJudgeTool, makeLlmJudgeDriver, regexMatchImpl, regexMatchTool, runEval, scoreSchema, toVitest };

package/dist/index.mjs

@@ -0,0 +1,444 @@
+import { implementTool, defineDriver } from '@agentproto/driver';
+import { z } from 'zod';
+import { defineTool } from '@agentproto/tool';
+import { runWorkflow } from '@agentproto/workflow-runtime';
+
+/**
+ * @agentproto/eval v0.1.0
+ * Deterministic reference scorers as AIP-14 TOOL contracts + builtin PROVIDER.
+ */
+
+var scoreSchema = z.object({
+  /** Normalized score in [0, 1]. */
+  value: z.number().min(0).max(1),
+  /** Whether the score clears the scorer's own threshold. */
+  passed: z.boolean(),
+  /** Scorer id, e.g. "exact-match". */
+  label: z.string(),
+  /** Human-readable explanation of the outcome. */
+  rationale: z.string().optional()
+});
+var jsonValueSchema = z.lazy(
+  () => z.union([
+    z.string(),
+    z.number(),
+    z.boolean(),
+    z.null(),
+    z.array(jsonValueSchema),
+    z.record(z.string(), jsonValueSchema)
+  ])
+);
+
+// src/scorers.ts
+var exactMatchTool = defineTool({
+  id: "eval.exact-match",
+  description: "Deterministic scorer: value 1 when `actual` equals `expected` (optionally after trimming whitespace on both sides), else 0.",
+  version: "0.1.0",
+  inputSchema: z.object({
+    actual: z.string().describe("The produced string to score."),
+    expected: z.string().describe("The reference string to compare against."),
+    trim: z.boolean().optional().describe("Trim leading/trailing whitespace on both sides before comparing.")
+  }),
+  outputSchema: scoreSchema,
+  mutates: [],
+  approval: "auto",
+  riskLevel: 0
+});
+var exactMatchImpl = implementTool(exactMatchTool, ({ input }) => {
+  const a = input.trim ? input.actual.trim() : input.actual;
+  const b = input.trim ? input.expected.trim() : input.expected;
+  const equal = a === b;
+  return {
+    value: equal ? 1 : 0,
+    passed: equal,
+    label: "exact-match",
+    rationale: equal ? "actual equals expected" : "actual does not equal expected"
+  };
+});
+var regexMatchTool = defineTool({
+  id: "eval.regex-match",
+  description: "Deterministic scorer: value 1 when `actual` matches the RegExp built from `pattern`/`flags`, else 0. An invalid pattern yields a typed failure Score rather than a thrown error.",
+  version: "0.1.0",
+  inputSchema: z.object({
+    actual: z.string().describe("The produced string to test."),
+    pattern: z.string().describe("RegExp source, compiled inside the body."),
+    flags: z.string().optional().describe("RegExp flags, e.g. 'i', 'm', 'gimsuy'.")
+  }),
+  outputSchema: scoreSchema,
+  mutates: [],
+  approval: "auto",
+  riskLevel: 0
+});
+var regexMatchImpl = implementTool(regexMatchTool, ({ input }) => {
+  let regex;
+  try {
+    regex = new RegExp(input.pattern, input.flags);
+  } catch (err) {
+    const reason = err instanceof Error ? err.message : String(err);
+    return {
+      value: 0,
+      passed: false,
+      label: "regex-match",
+      rationale: `invalid pattern: ${reason}`
+    };
+  }
+  const matched = regex.test(input.actual);
+  return {
+    value: matched ? 1 : 0,
+    passed: matched,
+    label: "regex-match",
+    rationale: matched ? `actual matches /${input.pattern}/${input.flags ?? ""}` : `actual does not match /${input.pattern}/${input.flags ?? ""}`
+  };
+});
+var minimalJsonSchemaSchema = z.object({
+  type: z.string().optional(),
+  required: z.array(z.string()).optional(),
+  properties: z.record(z.string(), z.object({ type: z.string().optional() })).optional()
+});
+function isJsonObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function jsonTypeOf(value) {
+  if (value === null) return "null";
+  if (Array.isArray(value)) return "array";
+  if (typeof value === "number") return "number";
+  if (typeof value === "boolean") return "boolean";
+  if (typeof value === "string") return "string";
+  return "object";
+}
+var jsonSchemaValidTool = defineTool({
+  id: "eval.json-schema-valid",
+  description: "Deterministic scorer: minimal structural check of a JSON value against a schema's top-level `required` and `properties` type constraints only. Not a full JSON Schema validator (no recursion, enum, format, \u2026).",
+  version: "0.1.0",
+  inputSchema: z.object({
+    actual: jsonValueSchema.describe("The JSON value to validate."),
+    schema: minimalJsonSchemaSchema.describe(
+      "Minimal JSON schema: top-level `required` and `properties.type` only."
+    )
+  }),
+  outputSchema: scoreSchema,
+  mutates: [],
+  approval: "auto",
+  riskLevel: 0
+});
+var jsonSchemaValidImpl = implementTool(
+  jsonSchemaValidTool,
+  ({ input }) => {
+    const { actual, schema } = input;
+    const violation = firstSchemaViolation(actual, schema);
+    const valid = violation === void 0;
+    return {
+      value: valid ? 1 : 0,
+      passed: valid,
+      label: "json-schema-valid",
+      rationale: valid ? "value satisfies the schema" : violation
+    };
+  }
+);
+function firstSchemaViolation(actual, schema) {
+  const declaredType = schema.type ?? "object";
+  const actualType = jsonTypeOf(actual);
+  if (declaredType !== actualType) {
+    return `expected top-level type '${declaredType}' but got '${actualType}'`;
+  }
+  if (!isJsonObject(actual)) {
+    return void 0;
+  }
+  for (const key of schema.required ?? []) {
+    if (!(key in actual)) {
+      return `missing required property '${key}'`;
+    }
+  }
+  for (const [key, propSchema] of Object.entries(schema.properties ?? {})) {
+    if (propSchema.type === void 0) continue;
+    const propValue = actual[key];
+    if (propValue === void 0) continue;
+    const propType = jsonTypeOf(propValue);
+    if (propType !== propSchema.type) {
+      return `property '${key}' expected type '${propSchema.type}' but got '${propType}'`;
+    }
+  }
+  return void 0;
+}
+var latencyBudgetTool = defineTool({
+  id: "eval.latency-budget",
+  description: "Deterministic scorer: passes when `durationMs` is within `budgetMs`. value is 1 when within budget, else linearly decays toward 0 as the overrun approaches one full budget.",
+  version: "0.1.0",
+  inputSchema: z.object({
+    durationMs: z.number().min(0).describe("Observed duration in milliseconds."),
+    budgetMs: z.number().positive().describe("Allowed budget in milliseconds (must be > 0).")
+  }),
+  outputSchema: scoreSchema,
+  mutates: [],
+  approval: "auto",
+  riskLevel: 0
+});
+var latencyBudgetImpl = implementTool(latencyBudgetTool, ({ input }) => {
+  const { durationMs, budgetMs } = input;
+  const passed = durationMs <= budgetMs;
+  const value = passed ? 1 : Math.max(0, 1 - (durationMs - budgetMs) / budgetMs);
+  return {
+    value,
+    passed,
+    label: "latency-budget",
+    rationale: `durationMs=${durationMs}, budgetMs=${budgetMs}`
+  };
+});
+var judgeVerdictSchema = z.object({
+  /** Normalized judge score in [0, 1]. */
+  value: z.number().min(0).max(1),
+  /**
+   * Explicit pass/fail from the judge. When present it WINS over the
+   * threshold comparison — see {@link makeLlmJudgeDriver}.
+   */
+  passed: z.boolean().optional(),
+  /** Human-readable explanation of the verdict. */
+  rationale: z.string().optional()
+});
+var llmJudgeTool = defineTool({
+  id: "eval.llm-judge",
+  description: "Model-backed scorer: hands `output` (plus free-form `criteria` and an optional `expected` reference) to an injected judge and normalizes the judge's verdict into the shared Score shape. The judge itself lives in the driver (see makeLlmJudgeDriver) \u2014 this tool contract carries no model call of its own.",
+  version: "0.1.0",
+  inputSchema: z.object({
+    output: jsonValueSchema.describe("The produced value to judge."),
+    criteria: z.string().describe("Free-form grading criteria/rubric for the judge."),
+    expected: jsonValueSchema.optional().describe("Optional reference value the judge may compare against.")
+  }),
+  outputSchema: scoreSchema,
+  mutates: [],
+  approval: "auto",
+  riskLevel: 0
+});
+function clamp01(x) {
+  return Math.min(1, Math.max(0, x));
+}
+function makeLlmJudgeDriver(judge, opts) {
+  const threshold = opts?.threshold ?? 0.5;
+  return defineDriver({
+    id: "eval-llm-judge",
+    name: "Eval LLM Judge (model-backed)",
+    description: "Model-backed scorer driver: implements eval.llm-judge by awaiting an injected JudgeFn and mapping its verdict to the shared Score shape. The judge is supplied by the caller \u2014 no LLM SDK or network call here.",
+    version: "0.1.0",
+    kind: "builtin",
+    implements: [{ tool: "eval.llm-judge", version: "0.1.0" }],
+    implementations: [
+      implementTool(llmJudgeTool, async ({ input }) => {
+        const verdict = await judge({
+          output: input.output,
+          criteria: input.criteria,
+          expected: input.expected
+        });
+        const value = clamp01(verdict.value);
+        const passed = verdict.passed ?? value >= threshold;
+        return {
+          value,
+          passed,
+          label: "llm-judge",
+          ...verdict.rationale ? { rationale: verdict.rationale } : {}
+        };
+      })
+    ]
+  });
+}
+function llmJudge(binding) {
+  const driver = makeLlmJudgeDriver(binding.judge, { threshold: binding.threshold });
+  return {
+    id: binding.id,
+    tool: llmJudgeTool,
+    driver,
+    mapInput: (ctx) => ({
+      output: binding.mapOutput({ output: ctx.output, expected: ctx.expected }),
+      criteria: binding.criteria,
+      expected: ctx.expected
+    })
+  };
+}
+
+// src/events.ts
+var EVAL_EVENT_SCHEMA = "agentproto/eval/v1";
+function bindScorer(binding) {
+  return {
+    id: binding.id,
+    toStep(stepId, output, expected) {
+      return {
+        kind: "tool",
+        id: stepId,
+        tool: binding.tool,
+        candidates: [binding.driver],
+        input: () => binding.mapInput({ output, expected })
+      };
+    }
+  };
+}
+var runCounter = 0;
+function defaultRunId(suiteId) {
+  runCounter += 1;
+  return `${suiteId}#${runCounter}`;
+}
+function asScore(value) {
+  if (typeof value === "object" && value !== null && "value" in value && "passed" in value && "label" in value) {
+    const v = value.value;
+    const passed = value.passed;
+    const label = value.label;
+    const rationale = "rationale" in value ? value.rationale : void 0;
+    if (typeof v === "number" && typeof passed === "boolean" && typeof label === "string" && (rationale === void 0 || typeof rationale === "string")) {
+      return { value: v, passed, label, rationale };
+    }
+  }
+  throw new Error("scorer step did not produce a valid Score");
+}
+async function runEval(suite, opts) {
+  const telemetry = opts.telemetry;
+  const runId = opts.runId ?? defaultRunId(suite.id);
+  const start = Date.now();
+  telemetry?.emit({
+    kind: "eval.started",
+    runId,
+    at: (/* @__PURE__ */ new Date()).toISOString(),
+    suiteId: suite.id,
+    caseCount: suite.cases.length,
+    scorerCount: suite.scorers.length
+  });
+  const caseReports = [];
+  let valueSum = 0;
+  let valueCount = 0;
+  for (const evalCase of suite.cases) {
+    telemetry?.emit({
+      kind: "eval.case.started",
+      runId,
+      at: (/* @__PURE__ */ new Date()).toISOString(),
+      caseId: evalCase.id
+    });
+    const aggregate = await runCase(suite, evalCase, opts.target);
+    for (const cs of aggregate.scores) {
+      valueSum += cs.score.value;
+      valueCount += 1;
+      telemetry?.emit({
+        kind: "eval.case.scored",
+        runId,
+        at: (/* @__PURE__ */ new Date()).toISOString(),
+        caseId: evalCase.id,
+        scorerId: cs.scorerId,
+        value: cs.score.value,
+        passed: cs.score.passed
+      });
+    }
+    telemetry?.emit({
+      kind: "eval.case.finished",
+      runId,
+      at: (/* @__PURE__ */ new Date()).toISOString(),
+      caseId: evalCase.id,
+      passed: aggregate.passed
+    });
+    caseReports.push({
+      caseId: evalCase.id,
+      passed: aggregate.passed,
+      scores: aggregate.scores
+    });
+  }
+  const passedCount = caseReports.filter((c) => c.passed).length;
+  const meanValue = valueCount === 0 ? 0 : valueSum / valueCount;
+  const report = {
+    runId,
+    suiteId: suite.id,
+    total: suite.cases.length,
+    passedCount,
+    meanValue,
+    cases: caseReports
+  };
+  telemetry?.emit({
+    kind: "eval.finished",
+    runId,
+    at: (/* @__PURE__ */ new Date()).toISOString(),
+    suiteId: suite.id,
+    total: report.total,
+    passedCount,
+    meanValue,
+    durationMs: Date.now() - start
+  });
+  return report;
+}
+async function runCase(suite, evalCase, target) {
+  const output = await target(evalCase.input);
+  const scorerStepIds = suite.scorers.map((_, i) => `score_${i}`);
+  const workflow = {
+    id: `${suite.id}:${evalCase.id}`,
+    steps: [
+      ...suite.scorers.map(
+        (scorer, i) => scorer.toStep(scorerStepIds[i], output, evalCase.expected)
+      ),
+      {
+        kind: "transform",
+        id: "aggregate",
+        compute: (b) => {
+          const scores = suite.scorers.map((scorer, i) => ({
+            scorerId: scorer.id,
+            score: asScore(b.steps[scorerStepIds[i]])
+          }));
+          return { scores, passed: scores.every((s) => s.score.passed) };
+        }
+      }
+    ],
+    output: (b) => b.steps.aggregate
+  };
+  const { output: runOutput } = await runWorkflow({ workflow });
+  return toCaseAggregate(runOutput);
+}
+function toCaseAggregate(value) {
+  if (typeof value === "object" && value !== null && "scores" in value && "passed" in value && Array.isArray(value.scores) && typeof value.passed === "boolean") {
+    const scores = value.scores.map((entry) => {
+      if (typeof entry === "object" && entry !== null && "scorerId" in entry && "score" in entry && typeof entry.scorerId === "string") {
+        return { scorerId: entry.scorerId, score: asScore(entry.score) };
+      }
+      throw new Error("invalid CaseScore in aggregate");
+    });
+    return { scores, passed: value.passed };
+  }
+  throw new Error("workflow did not produce a CaseAggregate");
+}
+
+// src/to-vitest.ts
+function toVitest(suite, opts, hooks) {
+  const { describe, it, expect } = hooks;
+  describe(suite.id, () => {
+    for (const evalCase of suite.cases) {
+      it(evalCase.id, async () => {
+        const singleCaseSuite = {
+          id: `${suite.id}:${evalCase.id}`,
+          cases: [evalCase],
+          scorers: suite.scorers
+        };
+        const report = await runEval(singleCaseSuite, opts);
+        const caseReport = report.cases[0];
+        expect(caseReport?.passed).toBe(true);
+      });
+    }
+  });
+}
+
+// src/index.ts
+var SPEC_NAME = "agenteval/v1";
+var SPEC_VERSION = "1.0.0-alpha";
+var evalScorersProvider = defineDriver({
+  id: "eval-scorers",
+  name: "Eval Reference Scorers (built-in)",
+  description: "In-process deterministic scorers: exact-match, regex-match, json-schema-valid (minimal structural check), and latency-budget. Each is an AIP-14 TOOL whose output is the shared Score shape.",
+  version: "0.1.0",
+  kind: "builtin",
+  implements: [
+    { tool: "eval.exact-match", version: "0.1.0" },
+    { tool: "eval.regex-match", version: "0.1.0" },
+    { tool: "eval.json-schema-valid", version: "0.1.0" },
+    { tool: "eval.latency-budget", version: "0.1.0" }
+  ],
+  implementations: [
+    exactMatchImpl,
+    regexMatchImpl,
+    jsonSchemaValidImpl,
+    latencyBudgetImpl
+  ]
+});
+
+export { EVAL_EVENT_SCHEMA, SPEC_NAME, SPEC_VERSION, bindScorer, evalScorersProvider, exactMatchImpl, exactMatchTool, jsonSchemaValidImpl, jsonSchemaValidTool, jsonValueSchema, judgeVerdictSchema, latencyBudgetImpl, latencyBudgetTool, llmJudge, llmJudgeTool, makeLlmJudgeDriver, regexMatchImpl, regexMatchTool, runEval, scoreSchema, toVitest };
+//# sourceMappingURL=index.mjs.map
+//# sourceMappingURL=index.mjs.map

package/dist/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/score.ts","../src/json.ts","../src/scorers.ts","../src/judge.ts","../src/events.ts","../src/run-eval.ts","../src/to-vitest.ts","../src/index.ts"],"names":["z","defineTool","implementTool","defineDriver"],"mappings":";;;;;;;;;;AAUO,IAAM,WAAA,GAAc,EAAE,MAAA,CAAO;AAAA;AAAA,EAElC,KAAA,EAAO,EAAE,MAAA,EAAO,CAAE,IAAI,CAAC,CAAA,CAAE,IAAI,CAAC,CAAA;AAAA;AAAA,EAE9B,MAAA,EAAQ,EAAE,OAAA,EAAQ;AAAA;AAAA,EAElB,KAAA,EAAO,EAAE,MAAA,EAAO;AAAA;AAAA,EAEhB,SAAA,EAAW,CAAA,CAAE,MAAA,EAAO,CAAE,QAAA;AACxB,CAAC;ACEM,IAAM,kBAAwCA,CAAAA,CAAE,IAAA;AAAA,EAAK,MAC1DA,EAAE,KAAA,CAAM;AAAA,IACNA,EAAE,MAAA,EAAO;AAAA,IACTA,EAAE,MAAA,EAAO;AAAA,IACTA,EAAE,OAAA,EAAQ;AAAA,IACVA,EAAE,IAAA,EAAK;AAAA,IACPA,CAAAA,CAAE,MAAM,eAAe,CAAA;AAAA,IACvBA,CAAAA,CAAE,MAAA,CAAOA,CAAAA,CAAE,MAAA,IAAU,eAAe;AAAA,GACrC;AACH;;;AClBO,IAAM,iBAAiB,UAAA,CAAW;AAAA,EACvC,EAAA,EAAI,kBAAA;AAAA,EACJ,WAAA,EACE,6HAAA;AAAA,EAEF,OAAA,EAAS,OAAA;AAAA,EACT,WAAA,EAAaA,EAAE,MAAA,CAAO;AAAA,IACpB,MAAA,EAAQA,CAAAA,CAAE,MAAA,EAAO,CAAE,SAAS,+BAA+B,CAAA;AAAA,IAC3D,QAAA,EAAUA,CAAAA,CAAE,MAAA,EAAO,CAAE,SAAS,0CAA0C,CAAA;AAAA,IACxE,MAAMA,CAAAA,CACH,OAAA,GACA,QAAA,EAAS,CACT,SAAS,kEAAkE;AAAA,GAC/E,CAAA;AAAA,EACD,YAAA,EAAc,WAAA;AAAA,EACd,SAAS,EAAC;AAAA,EACV,QAAA,EAAU,MAAA;AAAA,EACV,SAAA,EAAW;AACb,CAAC;AAEM,IAAM,iBAAiB,aAAA,CAAc,cAAA,EAAgB,CAAC,EAAE,OAAM,KAAM;AACzE,EAAA,MAAM,IAAI,KAAA,CAAM,IAAA,GAAO,MAAM,MAAA,CAAO,IAAA,KAAS,KAAA,CAAM,MAAA;AACnD,EAAA,MAAM,IAAI,KAAA,CAAM,IAAA,GAAO,MAAM,QAAA,CAAS,IAAA,KAAS,KAAA,CAAM,QAAA;AACrD,EAAA,MAAM,QAAQ,CAAA,KAAM,CAAA;AACpB,EAAA,OAAO;AAAA,IACL,KAAA,EAAO,QAAQ,CAAA,GAAI,CAAA;AAAA,IACnB,MAAA,EAAQ,KAAA;AAAA,IACR,KAAA,EAAO,aAAA;AAAA,IACP,SAAA,EAAW,QACP,wBAAA,GACA;AAAA,GACN;AACF,CAAC;AAMM,IAAM,iBAAiB,UAAA,CAAW;AAAA,EACvC,EAAA,EAAI,kBAAA;AAAA,EACJ,WAAA,EACE,kLAAA;AAAA,EAGF,OAAA,EAAS,OAAA;AAAA,EACT,WAAA,EAAaA,EAAE,MAAA,CAAO;AAAA,IACpB,MAAA,EAAQA,CAAAA,CAAE,MAAA,EAAO,CAAE,SAAS,8BAA8B,CAAA;AAAA,IAC1D,OAAA,EAASA,CAAAA,CAAE,MAAA,EAAO,CAAE,SAAS,0CAA0C,CAAA;AAAA,IACvE,OAAOA,CAAAA,CACJ,MAAA,GACA,QAAA,EAAS,CACT,SAAS,wCAAwC;AAAA,GACrD,CAAA;AAAA,EACD,YAAA,EAAc,WAAA;AAAA,EACd,SAAS,EAAC;AAAA,EACV,QAAA,EAAU,MAAA;AAAA,EACV,SAAA,EAAW;AACb,CAAC;AAEM,IAAM,iBAAiB,aAAA,CAAc,cAAA,EAAgB,CAAC,EAAE,OAAM,KAAM;AACzE,EAAA,IAAI,KAAA;AACJ,EAAA,IAAI;AACF,IAAA,KAAA,GAAQ,IAAI,MAAA,CAAO,KAAA,CAAM,OAAA,EAAS,MAAM,KAAK,CAAA;AAAA,EAC/C,SAAS,GAAA,EAAK;AACZ,IAAA,MAAM,SAAS,GAAA,YAAe,KAAA,GAAQ,GAAA,CAAI,OAAA,GAAU,OAAO,GAAG,CAAA;AAC9D,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,CAAA;AAAA,MACP,MAAA,EAAQ,KAAA;AAAA,MACR,KAAA,EAAO,aAAA;AAAA,MACP,SAAA,EAAW,oBAAoB,MAAM,CAAA;AAAA,KACvC;AAAA,EACF;AACA,EAAA,MAAM,OAAA,GAAU,KAAA,CAAM,IAAA,CAAK,KAAA,CAAM,MAAM,CAAA;AACvC,EAAA,OAAO;AAAA,IACL,KAAA,EAAO,UAAU,CAAA,GAAI,CAAA;AAAA,IACrB,MAAA,EAAQ,OAAA;AAAA,IACR,KAAA,EAAO,aAAA;AAAA,IACP,WAAW,OAAA,GACP,CAAA,gBAAA,EAAmB,KAAA,CAAM,OAAO,IAAI,KAAA,CAAM,KAAA,IAAS,EAAE,CAAA,CAAA,GACrD,0BAA0B,KAAA,CAAM,OAAO,CAAA,CAAA,EAAI,KAAA,CAAM,SAAS,EAAE,CAAA;AAAA,GAClE;AACF,CAAC;AAyBD,IAAM,uBAAA,GAAwDA,EAAE,MAAA,CAAO;AAAA,EACrE,IAAA,EAAMA,CAAAA,CAAE,MAAA,EAAO,CAAE,QAAA,EAAS;AAAA,EAC1B,UAAUA,CAAAA,CAAE,KAAA,CAAMA,EAAE,MAAA,EAAQ,EAAE,QAAA,EAAS;AAAA,EACvC,YAAYA,CAAAA,CACT,MAAA,CAAOA,EAAE,MAAA,EAAO,EAAGA,EAAE,MAAA,CAAO,EAAE,IAAA,EAAMA,CAAAA,CAAE,QAAO,CAAE,QAAA,IAAY,CAAC,EAC5D,QAAA;AACL,CAAC,CAAA;AAGD,SAAS,aAAa,KAAA,EAAuC;AAC3D,EAAA,OAAO,OAAO,UAAU,QAAA,IAAY,KAAA,KAAU,QAAQ,CAAC,KAAA,CAAM,QAAQ,KAAK,CAAA;AAC5E;AAGA,SAAS,WAAW,KAAA,EAA0B;AAC5C,EAAA,IAAI,KAAA,KAAU,MAAM,OAAO,MAAA;AAC3B,EAAA,IAAI,KAAA,CAAM,OAAA,CAAQ,KAAK,CAAA,EAAG,OAAO,OAAA;AACjC,EAAA,IAAI,OAAO,KAAA,KAAU,QAAA,EAAU,OAAO,QAAA;AACtC,EAAA,IAAI,OAAO,KAAA,KAAU,SAAA,EAAW,OAAO,SAAA;AACvC,EAAA,IAAI,OAAO,KAAA,KAAU,QAAA,EAAU,OAAO,QAAA;AACtC,EAAA,OAAO,QAAA;AACT;AAEO,IAAM,sBAAsB,UAAA,CAAW;AAAA,EAC5C,EAAA,EAAI,wBAAA;AAAA,EACJ,WAAA,EACE,uNAAA;AAAA,EAGF,OAAA,EAAS,OAAA;AAAA,EACT,WAAA,EAAaA,EAAE,MAAA,CAAO;AAAA,IACpB,MAAA,EAAQ,eAAA,CAAgB,QAAA,CAAS,6BAA6B,CAAA;AAAA,IAC9D,QAAQ,uBAAA,CAAwB,QAAA;AAAA,MAC9B;AAAA;AACF,GACD,CAAA;AAAA,EACD,YAAA,EAAc,WAAA;AAAA,EACd,SAAS,EAAC;AAAA,EACV,QAAA,EAAU,MAAA;AAAA,EACV,SAAA,EAAW;AACb,CAAC;AAEM,IAAM,mBAAA,GAAsB,aAAA;AAAA,EACjC,mBAAA;AAAA,EACA,CAAC,EAAE,KAAA,EAAM,KAAM;AACb,IAAA,MAAM,EAAE,MAAA,EAAQ,MAAA,EAAO,GAAI,KAAA;AAC3B,IAAA,MAAM,SAAA,GAAY,oBAAA,CAAqB,MAAA,EAAQ,MAAM,CAAA;AACrD,IAAA,MAAM,QAAQ,SAAA,KAAc,MAAA;AAC5B,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,QAAQ,CAAA,GAAI,CAAA;AAAA,MACnB,MAAA,EAAQ,KAAA;AAAA,MACR,KAAA,EAAO,mBAAA;AAAA,MACP,SAAA,EAAW,QAAQ,4BAAA,GAA+B;AAAA,KACpD;AAAA,EACF;AACF;AAGA,SAAS,oBAAA,CACP,QACA,MAAA,EACoB;AACpB,EAAA,MAAM,YAAA,GAAe,OAAO,IAAA,IAAQ,QAAA;AACpC,EAAA,MAAM,UAAA,GAAa,WAAW,MAAM,CAAA;AACpC,EAAA,IAAI,iBAAiB,UAAA,EAAY;AAC/B,IAAA,OAAO,CAAA,yBAAA,EAA4B,YAAY,CAAA,WAAA,EAAc,UAAU,CAAA,CAAA,CAAA;AAAA,EACzE;AAGA,EAAA,IAAI,CAAC,YAAA,CAAa,MAAM,CAAA,EAAG;AACzB,IAAA,OAAO,MAAA;AAAA,EACT;AAEA,EAAA,KAAA,MAAW,GAAA,IAAO,MAAA,CAAO,QAAA,IAAY,EAAC,EAAG;AACvC,IAAA,IAAI,EAAE,OAAO,MAAA,CAAA,EAAS;AACpB,MAAA,OAAO,8BAA8B,GAAG,CAAA,CAAA,CAAA;AAAA,IAC1C;AAAA,EACF;AAEA,EAAA,KAAA,MAAW,CAAC,GAAA,EAAK,UAAU,CAAA,IAAK,MAAA,CAAO,QAAQ,MAAA,CAAO,UAAA,IAAc,EAAE,CAAA,EAAG;AACvE,IAAA,IAAI,UAAA,CAAW,SAAS,MAAA,EAAW;AACnC,IAAA,MAAM,SAAA,GAAY,OAAO,GAAG,CAAA;AAC5B,IAAA,IAAI,cAAc,MAAA,EAAW;AAC7B,IAAA,MAAM,QAAA,GAAW,WAAW,SAAS,CAAA;AACrC,IAAA,IAAI,QAAA,KAAa,WAAW,IAAA,EAAM;AAChC,MAAA,OAAO,aAAa,GAAG,CAAA,iBAAA,EAAoB,UAAA,CAAW,IAAI,cAAc,QAAQ,CAAA,CAAA,CAAA;AAAA,IAClF;AAAA,EACF;AAEA,EAAA,OAAO,MAAA;AACT;AAMO,IAAM,oBAAoB,UAAA,CAAW;AAAA,EAC1C,EAAA,EAAI,qBAAA;AAAA,EACJ,WAAA,EACE,8KAAA;AAAA,EAGF,OAAA,EAAS,OAAA;AAAA,EACT,WAAA,EAAaA,EAAE,MAAA,CAAO;AAAA,IACpB,UAAA,EAAYA,EACT,MAAA,EAAO,CACP,IAAI,CAAC,CAAA,CACL,SAAS,oCAAoC,CAAA;AAAA,IAChD,UAAUA,CAAAA,CACP,MAAA,GACA,QAAA,EAAS,CACT,SAAS,+CAA+C;AAAA,GAC5D,CAAA;AAAA,EACD,YAAA,EAAc,WAAA;AAAA,EACd,SAAS,EAAC;AAAA,EACV,QAAA,EAAU,MAAA;AAAA,EACV,SAAA,EAAW;AACb,CAAC;AAEM,IAAM,oBAAoB,aAAA,CAAc,iBAAA,EAAmB,CAAC,EAAE,OAAM,KAAM;AAC/E,EAAA,MAAM,EAAE,UAAA,EAAY,QAAA,EAAS,GAAI,KAAA;AACjC,EAAA,MAAM,SAAS,UAAA,IAAc,QAAA;AAC7B,EAAA,MAAM,KAAA,GAAQ,SACV,CAAA,GACA,IAAA,CAAK,IAAI,CAAA,EAAG,CAAA,GAAA,CAAK,UAAA,GAAa,QAAA,IAAY,QAAQ,CAAA;AACtD,EAAA,OAAO;AAAA,IACL,KAAA;AAAA,IACA,MAAA;AAAA,IACA,KAAA,EAAO,gBAAA;AAAA,IACP,SAAA,EAAW,CAAA,WAAA,EAAc,UAAU,CAAA,WAAA,EAAc,QAAQ,CAAA;AAAA,GAC3D;AACF,CAAC;AC5NM,IAAM,kBAAA,GAAqBA,EAAE,MAAA,CAAO;AAAA;AAAA,EAEzC,KAAA,EAAOA,EAAE,MAAA,EAAO,CAAE,IAAI,CAAC,CAAA,CAAE,IAAI,CAAC,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAK9B,MAAA,EAAQA,CAAAA,CAAE,OAAA,EAAQ,CAAE,QAAA,EAAS;AAAA;AAAA,EAE7B,SAAA,EAAWA,CAAAA,CAAE,MAAA,EAAO,CAAE,QAAA;AACxB,CAAC;AAiCM,IAAM,eAAeC,UAAAA,CAAW;AAAA,EACrC,EAAA,EAAI,gBAAA;AAAA,EACJ,WAAA,EACE,qTAAA;AAAA,EAKF,OAAA,EAAS,OAAA;AAAA,EACT,WAAA,EAAaD,EAAE,MAAA,CAAO;AAAA,IACpB,MAAA,EAAQ,eAAA,CAAgB,QAAA,CAAS,8BAA8B,CAAA;AAAA,IAC/D,QAAA,EAAUA,CAAAA,CAAE,MAAA,EAAO,CAAE,SAAS,kDAAkD,CAAA;AAAA,IAChF,QAAA,EAAU,eAAA,CACP,QAAA,EAAS,CACT,SAAS,yDAAyD;AAAA,GACtE,CAAA;AAAA,EACD,YAAA,EAAc,WAAA;AAAA,EACd,SAAS,EAAC;AAAA,EACV,QAAA,EAAU,MAAA;AAAA,EACV,SAAA,EAAW;AACb,CAAC;AAOD,SAAS,QAAQ,CAAA,EAAmB;AAClC,EAAA,OAAO,KAAK,GAAA,CAAI,CAAA,EAAG,KAAK,GAAA,CAAI,CAAA,EAAG,CAAC,CAAC,CAAA;AACnC;AAaO,SAAS,kBAAA,CACd,OACA,IAAA,EACc;AACd,EAAA,MAAM,SAAA,GAAY,MAAM,SAAA,IAAa,GAAA;AACrC,EAAA,OAAO,YAAA,CAAa;AAAA,IAClB,EAAA,EAAI,gBAAA;AAAA,IACJ,IAAA,EAAM,+BAAA;AAAA,IACN,WAAA,EACE,sNAAA;AAAA,IAGF,OAAA,EAAS,OAAA;AAAA,IACT,IAAA,EAAM,SAAA;AAAA,IACN,YAAY,CAAC,EAAE,MAAM,gBAAA,EAAkB,OAAA,EAAS,SAAS,CAAA;AAAA,IACzD,eAAA,EAAiB;AAAA,MACfE,aAAAA,CAAc,YAAA,EAAc,OAAO,EAAE,OAAM,KAAM;AAC/C,QAAA,MAAM,OAAA,GAAU,MAAM,KAAA,CAAM;AAAA,UAC1B,QAAQ,KAAA,CAAM,MAAA;AAAA,UACd,UAAU,KAAA,CAAM,QAAA;AAAA,UAChB,UAAU,KAAA,CAAM;AAAA,SACjB,CAAA;AACD,QAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,OAAA,CAAQ,KAAK,CAAA;AACnC,QAAA,MAAM,MAAA,GAAS,OAAA,CAAQ,MAAA,IAAU,KAAA,IAAS,SAAA;AAC1C,QAAA,OAAO;AAAA,UACL,KAAA;AAAA,UACA,MAAA;AAAA,UACA,KAAA,EAAO,WAAA;AAAA,UACP,GAAI,QAAQ,SAAA,GAAY,EAAE,WAAW,OAAA,CAAQ,SAAA,KAAc;AAAC,SAC9D;AAAA,MACF,CAAC;AAAA;AACH,GACD,CAAA;AACH;AAwBO,SAAS,SAAY,OAAA,EAA8D;AACxF,EAAA,MAAM,MAAA,GAAS,mBAAmB,OAAA,CAAQ,KAAA,EAAO,EAAE,SAAA,EAAW,OAAA,CAAQ,WAAW,CAAA;AACjF,EAAA,OAAO;AAAA,IACL,IAAI,OAAA,CAAQ,EAAA;AAAA,IACZ,IAAA,EAAM,YAAA;AAAA,IACN,MAAA;AAAA,IACA,QAAA,EAAU,CAAC,GAAA,MAAS;AAAA,MAClB,MAAA,EAAQ,OAAA,CAAQ,SAAA,CAAU,EAAE,MAAA,EAAQ,IAAI,MAAA,EAAQ,QAAA,EAAU,GAAA,CAAI,QAAA,EAAU,CAAA;AAAA,MACxE,UAAU,OAAA,CAAQ,QAAA;AAAA,MAClB,UAAU,GAAA,CAAI;AAAA,KAChB;AAAA,GACF;AACF;;;ACxKO,IAAM,iBAAA,GAAoB;ACyD1B,SAAS,WAAiB,OAAA,EAA8C;AAC7E,EAAA,OAAO;AAAA,IACL,IAAI,OAAA,CAAQ,EAAA;AAAA,IACZ,MAAA,CAAO,MAAA,EAAQ,MAAA,EAAQ,QAAA,EAAU;AAC/B,MAAA,OAAO;AAAA,QACL,IAAA,EAAM,MAAA;AAAA,QACN,EAAA,EAAI,MAAA;AAAA,QACJ,MAAM,OAAA,CAAQ,IAAA;AAAA,QACd,UAAA,EAAY,CAAC,OAAA,CAAQ,MAAM,CAAA;AAAA,QAC3B,OAAO,MAAM,OAAA,CAAQ,SAAS,EAAE,MAAA,EAAQ,UAAU;AAAA,OACpD;AAAA,IACF;AAAA,GACF;AACF;AA8CA,IAAI,UAAA,GAAa,CAAA;AAEjB,SAAS,aAAa,OAAA,EAAyB;AAC7C,EAAA,UAAA,IAAc,CAAA;AACd,EAAA,OAAO,CAAA,EAAG,OAAO,CAAA,CAAA,EAAI,UAAU,CAAA,CAAA;AACjC;AASA,SAAS,QAAQ,KAAA,EAAuB;AACtC,EAAA,IACE,OAAO,KAAA,KAAU,QAAA,IACjB,KAAA,KAAU,IAAA,IACV,WAAW,KAAA,IACX,QAAA,IAAY,KAAA,IACZ,OAAA,IAAW,KAAA,EACX;AACA,IAAA,MAAM,IAAa,KAAA,CAAM,KAAA;AACzB,IAAA,MAAM,SAAkB,KAAA,CAAM,MAAA;AAC9B,IAAA,MAAM,QAAiB,KAAA,CAAM,KAAA;AAC7B,IAAA,MAAM,SAAA,GAAqB,WAAA,IAAe,KAAA,GAAQ,KAAA,CAAM,SAAA,GAAY,MAAA;AACpE,IAAA,IACE,OAAO,CAAA,KAAM,QAAA,IACb,OAAO,MAAA,KAAW,SAAA,IAClB,OAAO,KAAA,KAAU,QAAA,KAChB,SAAA,KAAc,MAAA,IAAa,OAAO,cAAc,QAAA,CAAA,EACjD;AACA,MAAA,OAAO,EAAE,KAAA,EAAO,CAAA,EAAG,MAAA,EAAQ,OAAO,SAAA,EAAU;AAAA,IAC9C;AAAA,EACF;AACA,EAAA,MAAM,IAAI,MAAM,2CAA2C,CAAA;AAC7D;AAMA,eAAsB,OAAA,CACpB,OACA,IAAA,EACqB;AACrB,EAAA,MAAM,YAAY,IAAA,CAAK,SAAA;AACvB,EAAA,MAAM,KAAA,GAAQ,IAAA,CAAK,KAAA,IAAS,YAAA,CAAa,MAAM,EAAE,CAAA;AACjD,EAAA,MAAM,KAAA,GAAQ,KAAK,GAAA,EAAI;AAEvB,EAAA,SAAA,EAAW,IAAA,CAAK;AAAA,IACd,IAAA,EAAM,cAAA;AAAA,IACN,KAAA;AAAA,IACA,EAAA,EAAA,iBAAI,IAAI,IAAA,EAAK,EAAE,WAAA,EAAY;AAAA,IAC3B,SAAS,KAAA,CAAM,EAAA;AAAA,IACf,SAAA,EAAW,MAAM,KAAA,CAAM,MAAA;AAAA,IACvB,WAAA,EAAa,MAAM,OAAA,CAAQ;AAAA,GAC5B,CAAA;AAED,EAAA,MAAM,cAA4B,EAAC;AACnC,EAAA,IAAI,QAAA,GAAW,CAAA;AACf,EAAA,IAAI,UAAA,GAAa,CAAA;AAEjB,EAAA,KAAA,MAAW,QAAA,IAAY,MAAM,KAAA,EAAO;AAClC,IAAA,SAAA,EAAW,IAAA,CAAK;AAAA,MACd,IAAA,EAAM,mBAAA;AAAA,MACN,KAAA;AAAA,MACA,EAAA,EAAA,iBAAI,IAAI,IAAA,EAAK,EAAE,WAAA,EAAY;AAAA,MAC3B,QAAQ,QAAA,CAAS;AAAA,KAClB,CAAA;AAED,IAAA,MAAM,YAAY,MAAM,OAAA,CAAQ,KAAA,EAAO,QAAA,EAAU,KAAK,MAAM,CAAA;AAE5D,IAAA,KAAA,MAAW,EAAA,IAAM,UAAU,MAAA,EAAQ;AACjC,MAAA,QAAA,IAAY,GAAG,KAAA,CAAM,KAAA;AACrB,MAAA,UAAA,IAAc,CAAA;AACd,MAAA,SAAA,EAAW,IAAA,CAAK;AAAA,QACd,IAAA,EAAM,kBAAA;AAAA,QACN,KAAA;AAAA,QACA,EAAA,EAAA,iBAAI,IAAI,IAAA,EAAK,EAAE,WAAA,EAAY;AAAA,QAC3B,QAAQ,QAAA,CAAS,EAAA;AAAA,QACjB,UAAU,EAAA,CAAG,QAAA;AAAA,QACb,KAAA,EAAO,GAAG,KAAA,CAAM,KAAA;AAAA,QAChB,MAAA,EAAQ,GAAG,KAAA,CAAM;AAAA,OAClB,CAAA;AAAA,IACH;AAEA,IAAA,SAAA,EAAW,IAAA,CAAK;AAAA,MACd,IAAA,EAAM,oBAAA;AAAA,MACN,KAAA;AAAA,MACA,EAAA,EAAA,iBAAI,IAAI,IAAA,EAAK,EAAE,WAAA,EAAY;AAAA,MAC3B,QAAQ,QAAA,CAAS,EAAA;AAAA,MACjB,QAAQ,SAAA,CAAU;AAAA,KACnB,CAAA;AAED,IAAA,WAAA,CAAY,IAAA,CAAK;AAAA,MACf,QAAQ,QAAA,CAAS,EAAA;AAAA,MACjB,QAAQ,SAAA,CAAU,MAAA;AAAA,MAClB,QAAQ,SAAA,CAAU;AAAA,KACnB,CAAA;AAAA,EACH;AAEA,EAAA,MAAM,cAAc,WAAA,CAAY,MAAA,CAAO,CAAC,CAAA,KAAM,CAAA,CAAE,MAAM,CAAA,CAAE,MAAA;AACxD,EAAA,MAAM,SAAA,GAAY,UAAA,KAAe,CAAA,GAAI,CAAA,GAAI,QAAA,GAAW,UAAA;AAEpD,EAAA,MAAM,MAAA,GAAqB;AAAA,IACzB,KAAA;AAAA,IACA,SAAS,KAAA,CAAM,EAAA;AAAA,IACf,KAAA,EAAO,MAAM,KAAA,CAAM,MAAA;AAAA,IACnB,WAAA;AAAA,IACA,SAAA;AAAA,IACA,KAAA,EAAO;AAAA,GACT;AAEA,EAAA,SAAA,EAAW,IAAA,CAAK;AAAA,IACd,IAAA,EAAM,eAAA;AAAA,IACN,KAAA;AAAA,IACA,EAAA,EAAA,iBAAI,IAAI,IAAA,EAAK,EAAE,WAAA,EAAY;AAAA,IAC3B,SAAS,KAAA,CAAM,EAAA;AAAA,IACf,OAAO,MAAA,CAAO,KAAA;AAAA,IACd,WAAA;AAAA,IACA,SAAA;AAAA,IACA,UAAA,EAAY,IAAA,CAAK,GAAA,EAAI,GAAI;AAAA,GAC1B,CAAA;AAED,EAAA,OAAO,MAAA;AACT;AAMA,eAAe,OAAA,CACb,KAAA,EACA,QAAA,EACA,MAAA,EACwB;AACxB,EAAA,MAAM,MAAA,GAAS,MAAM,MAAA,CAAO,QAAA,CAAS,KAAK,CAAA;AAC1C,EAAA,MAAM,aAAA,GAAgB,MAAM,OAAA,CAAQ,GAAA,CAAI,CAAC,CAAA,EAAG,CAAA,KAAM,CAAA,MAAA,EAAS,CAAC,CAAA,CAAE,CAAA;AAE9D,EAAA,MAAM,QAAA,GAA4B;AAAA,IAChC,IAAI,CAAA,EAAG,KAAA,CAAM,EAAE,CAAA,CAAA,EAAI,SAAS,EAAE,CAAA,CAAA;AAAA,IAC9B,KAAA,EAAO;AAAA,MACL,GAAG,MAAM,OAAA,CAAQ,GAAA;AAAA,QAAI,CAAC,MAAA,EAAQ,CAAA,KAC5B,MAAA,CAAO,MAAA,CAAO,cAAc,CAAC,CAAA,EAAI,MAAA,EAAQ,QAAA,CAAS,QAAQ;AAAA,OAC5D;AAAA,MACA;AAAA,QACE,IAAA,EAAM,WAAA;AAAA,QACN,EAAA,EAAI,WAAA;AAAA,QACJ,OAAA,EAAS,CAAC,CAAA,KAAqB;AAC7B,UAAA,MAAM,SAAsB,KAAA,CAAM,OAAA,CAAQ,GAAA,CAAI,CAAC,QAAQ,CAAA,MAAO;AAAA,YAC5D,UAAU,MAAA,CAAO,EAAA;AAAA,YACjB,OAAO,OAAA,CAAQ,CAAA,CAAE,MAAM,aAAA,CAAc,CAAC,CAAE,CAAC;AAAA,WAC3C,CAAE,CAAA;AACF,UAAA,OAAO,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAA,CAAO,KAAA,CAAM,CAAC,CAAA,KAAM,CAAA,CAAE,KAAA,CAAM,MAAM,CAAA,EAAE;AAAA,QAC/D;AAAA;AACF,KACF;AAAA,IACA,MAAA,EAAQ,CAAC,CAAA,KAAM,CAAA,CAAE,KAAA,CAAM;AAAA,GACzB;AAEA,EAAA,MAAM,EAAE,QAAQ,SAAA,EAAU,GAAI,MAAM,WAAA,CAAY,EAAE,UAAU,CAAA;AAC5D,EAAA,OAAO,gBAAgB,SAAS,CAAA;AAClC;AAGA,SAAS,gBAAgB,KAAA,EAA+B;AACtD,EAAA,IACE,OAAO,KAAA,KAAU,QAAA,IACjB,KAAA,KAAU,IAAA,IACV,YAAY,KAAA,IACZ,QAAA,IAAY,KAAA,IACZ,KAAA,CAAM,QAAQ,KAAA,CAAM,MAAM,KAC1B,OAAO,KAAA,CAAM,WAAW,SAAA,EACxB;AACA,IAAA,MAAM,MAAA,GAAS,KAAA,CAAM,MAAA,CAAO,GAAA,CAAI,CAAC,KAAA,KAAmB;AAClD,MAAA,IACE,OAAO,KAAA,KAAU,QAAA,IACjB,KAAA,KAAU,IAAA,IACV,UAAA,IAAc,KAAA,IACd,OAAA,IAAW,KAAA,IACX,OAAO,KAAA,CAAM,QAAA,KAAa,QAAA,EAC1B;AACA,QAAA,OAAO,EAAE,UAAU,KAAA,CAAM,QAAA,EAAU,OAAO,OAAA,CAAQ,KAAA,CAAM,KAAK,CAAA,EAAE;AAAA,MACjE;AACA,MAAA,MAAM,IAAI,MAAM,gCAAgC,CAAA;AAAA,IAClD,CAAC,CAAA;AACD,IAAA,OAAO,EAAE,MAAA,EAAQ,MAAA,EAAQ,KAAA,CAAM,MAAA,EAAO;AAAA,EACxC;AACA,EAAA,MAAM,IAAI,MAAM,0CAA0C,CAAA;AAC5D;;;AC/RO,SAAS,QAAA,CACd,KAAA,EACA,IAAA,EACA,KAAA,EACM;AACN,EAAA,MAAM,EAAE,QAAA,EAAU,EAAA,EAAI,MAAA,EAAO,GAAI,KAAA;AACjC,EAAA,QAAA,CAAS,KAAA,CAAM,IAAI,MAAM;AACvB,IAAA,KAAA,MAAW,QAAA,IAAY,MAAM,KAAA,EAAO;AAClC,MAAA,EAAA,CAAG,QAAA,CAAS,IAAI,YAAY;AAC1B,QAAA,MAAM,eAAA,GAAwC;AAAA,UAC5C,IAAI,CAAA,EAAG,KAAA,CAAM,EAAE,CAAA,CAAA,EAAI,SAAS,EAAE,CAAA,CAAA;AAAA,UAC9B,KAAA,EAAO,CAAC,QAAQ,CAAA;AAAA,UAChB,SAAS,KAAA,CAAM;AAAA,SACjB;AACA,QAAA,MAAM,MAAA,GAAS,MAAM,OAAA,CAAQ,eAAA,EAAiB,IAAI,CAAA;AAClD,QAAA,MAAM,UAAA,GAAa,MAAA,CAAO,KAAA,CAAM,CAAC,CAAA;AACjC,QAAA,MAAA,CAAO,UAAA,EAAY,MAAM,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA;AAAA,MACtC,CAAC,CAAA;AAAA,IACH;AAAA,EACF,CAAC,CAAA;AACH;;;ACbO,IAAM,SAAA,GAAY;AAClB,IAAM,YAAA,GAAe;AA8DrB,IAAM,sBAAsBC,YAAAA,CAAa;AAAA,EAC9C,EAAA,EAAI,cAAA;AAAA,EACJ,IAAA,EAAM,mCAAA;AAAA,EACN,WAAA,EACE,8LAAA;AAAA,EAGF,OAAA,EAAS,OAAA;AAAA,EACT,IAAA,EAAM,SAAA;AAAA,EACN,UAAA,EAAY;AAAA,IACV,EAAE,IAAA,EAAM,kBAAA,EAAoB,OAAA,EAAS,OAAA,EAAQ;AAAA,IAC7C,EAAE,IAAA,EAAM,kBAAA,EAAoB,OAAA,EAAS,OAAA,EAAQ;AAAA,IAC7C,EAAE,IAAA,EAAM,wBAAA,EAA0B,OAAA,EAAS,OAAA,EAAQ;AAAA,IACnD,EAAE,IAAA,EAAM,qBAAA,EAAuB,OAAA,EAAS,OAAA;AAAQ,GAClD;AAAA,EACA,eAAA,EAAiB;AAAA,IACf,cAAA;AAAA,IACA,cAAA;AAAA,IACA,mBAAA;AAAA,IACA;AAAA;AAEJ,CAAC","file":"index.mjs","sourcesContent":["import { z } from \"zod\"\n\n/**\n * The shared output shape every scorer produces.\n *\n * A scorer is an AIP-14 TOOL whose `outputSchema` is exactly this — there is\n * no separate \"scorer port\". Reusing `Score` as the tool output means the\n * registry, retries, and `toMastraTool` / `toAiSdkTool` projections all apply\n * to scorers for free, identical to any other builtin tool.\n */\nexport const scoreSchema = z.object({\n  /** Normalized score in [0, 1]. */\n  value: z.number().min(0).max(1),\n  /** Whether the score clears the scorer's own threshold. */\n  passed: z.boolean(),\n  /** Scorer id, e.g. \"exact-match\". */\n  label: z.string(),\n  /** Human-readable explanation of the outcome. */\n  rationale: z.string().optional(),\n})\n\nexport type Score = z.infer<typeof scoreSchema>\n","import { z } from \"zod\"\n\n/**\n * Recursive JSON value union. Modelling the JSON payload explicitly (rather\n * than reaching for `any` / `unknown`) lets consumers inspect values with\n * proper narrowing and no casts. This is the ONE canonical `JsonValue` for the\n * package — scorers, the eval harness, and the vitest bridge all import it from\n * here so there is no divergent redeclaration.\n */\nexport type JsonValue =\n  | string\n  | number\n  | boolean\n  | null\n  | JsonValue[]\n  | { [key: string]: JsonValue }\n\n/** A plain JSON object (not `null`, not an array). */\nexport type JsonObject = { [key: string]: JsonValue }\n\n/** Zod schema mirroring {@link JsonValue}. */\nexport const jsonValueSchema: z.ZodType<JsonValue> = z.lazy(() =>\n  z.union([\n    z.string(),\n    z.number(),\n    z.boolean(),\n    z.null(),\n    z.array(jsonValueSchema),\n    z.record(z.string(), jsonValueSchema),\n  ]),\n)\n","import { z } from \"zod\"\nimport { defineTool } from \"@agentproto/tool\"\nimport { implementTool } from \"@agentproto/driver\"\nimport { scoreSchema } from \"./score.js\"\nimport { jsonValueSchema, type JsonObject, type JsonValue } from \"./json.js\"\n\nexport type { JsonValue } from \"./json.js\"\n\n// ---------------------------------------------------------------------------\n// eval.exact-match\n// ---------------------------------------------------------------------------\n\nexport const exactMatchTool = defineTool({\n  id: \"eval.exact-match\",\n  description:\n    \"Deterministic scorer: value 1 when `actual` equals `expected` \" +\n    \"(optionally after trimming whitespace on both sides), else 0.\",\n  version: \"0.1.0\",\n  inputSchema: z.object({\n    actual: z.string().describe(\"The produced string to score.\"),\n    expected: z.string().describe(\"The reference string to compare against.\"),\n    trim: z\n      .boolean()\n      .optional()\n      .describe(\"Trim leading/trailing whitespace on both sides before comparing.\"),\n  }),\n  outputSchema: scoreSchema,\n  mutates: [],\n  approval: \"auto\",\n  riskLevel: 0,\n})\n\nexport const exactMatchImpl = implementTool(exactMatchTool, ({ input }) => {\n  const a = input.trim ? input.actual.trim() : input.actual\n  const b = input.trim ? input.expected.trim() : input.expected\n  const equal = a === b\n  return {\n    value: equal ? 1 : 0,\n    passed: equal,\n    label: \"exact-match\",\n    rationale: equal\n      ? \"actual equals expected\"\n      : \"actual does not equal expected\",\n  }\n})\n\n// ---------------------------------------------------------------------------\n// eval.regex-match\n// ---------------------------------------------------------------------------\n\nexport const regexMatchTool = defineTool({\n  id: \"eval.regex-match\",\n  description:\n    \"Deterministic scorer: value 1 when `actual` matches the RegExp built \" +\n    \"from `pattern`/`flags`, else 0. An invalid pattern yields a typed \" +\n    \"failure Score rather than a thrown error.\",\n  version: \"0.1.0\",\n  inputSchema: z.object({\n    actual: z.string().describe(\"The produced string to test.\"),\n    pattern: z.string().describe(\"RegExp source, compiled inside the body.\"),\n    flags: z\n      .string()\n      .optional()\n      .describe(\"RegExp flags, e.g. 'i', 'm', 'gimsuy'.\"),\n  }),\n  outputSchema: scoreSchema,\n  mutates: [],\n  approval: \"auto\",\n  riskLevel: 0,\n})\n\nexport const regexMatchImpl = implementTool(regexMatchTool, ({ input }) => {\n  let regex: RegExp\n  try {\n    regex = new RegExp(input.pattern, input.flags)\n  } catch (err) {\n    const reason = err instanceof Error ? err.message : String(err)\n    return {\n      value: 0,\n      passed: false,\n      label: \"regex-match\",\n      rationale: `invalid pattern: ${reason}`,\n    }\n  }\n  const matched = regex.test(input.actual)\n  return {\n    value: matched ? 1 : 0,\n    passed: matched,\n    label: \"regex-match\",\n    rationale: matched\n      ? `actual matches /${input.pattern}/${input.flags ?? \"\"}`\n      : `actual does not match /${input.pattern}/${input.flags ?? \"\"}`,\n  }\n})\n\n// ---------------------------------------------------------------------------\n// eval.json-schema-valid\n// ---------------------------------------------------------------------------\n\n/**\n * A deliberately minimal, dependency-free JSON-schema shape.\n *\n * LIMITATION: this scorer checks ONLY the two most common structural\n * constraints on a top-level object — that every name in `required` is\n * present, and that any name in `properties` declaring a `type` has a value\n * of that primitive type. It does NOT recurse into nested schemas, and does\n * not implement the rest of JSON Schema (enum, format, oneOf, items, …). A\n * full validator (e.g. ajv) is a later step; keeping this package light is\n * the point.\n */\nexport interface MinimalJsonSchema {\n  readonly type?: string\n  readonly required?: readonly string[]\n  readonly properties?: {\n    readonly [key: string]: { readonly type?: string }\n  }\n}\n\nconst minimalJsonSchemaSchema: z.ZodType<MinimalJsonSchema> = z.object({\n  type: z.string().optional(),\n  required: z.array(z.string()).optional(),\n  properties: z\n    .record(z.string(), z.object({ type: z.string().optional() }))\n    .optional(),\n})\n\n/** Narrows a JSON value to a plain JSON object (not null, not an array). */\nfunction isJsonObject(value: JsonValue): value is JsonObject {\n  return typeof value === \"object\" && value !== null && !Array.isArray(value)\n}\n\n/** The JSON Schema primitive-type name for a runtime JSON value. */\nfunction jsonTypeOf(value: JsonValue): string {\n  if (value === null) return \"null\"\n  if (Array.isArray(value)) return \"array\"\n  if (typeof value === \"number\") return \"number\"\n  if (typeof value === \"boolean\") return \"boolean\"\n  if (typeof value === \"string\") return \"string\"\n  return \"object\"\n}\n\nexport const jsonSchemaValidTool = defineTool({\n  id: \"eval.json-schema-valid\",\n  description:\n    \"Deterministic scorer: minimal structural check of a JSON value against \" +\n    \"a schema's top-level `required` and `properties` type constraints only. \" +\n    \"Not a full JSON Schema validator (no recursion, enum, format, …).\",\n  version: \"0.1.0\",\n  inputSchema: z.object({\n    actual: jsonValueSchema.describe(\"The JSON value to validate.\"),\n    schema: minimalJsonSchemaSchema.describe(\n      \"Minimal JSON schema: top-level `required` and `properties.type` only.\",\n    ),\n  }),\n  outputSchema: scoreSchema,\n  mutates: [],\n  approval: \"auto\",\n  riskLevel: 0,\n})\n\nexport const jsonSchemaValidImpl = implementTool(\n  jsonSchemaValidTool,\n  ({ input }) => {\n    const { actual, schema } = input\n    const violation = firstSchemaViolation(actual, schema)\n    const valid = violation === undefined\n    return {\n      value: valid ? 1 : 0,\n      passed: valid,\n      label: \"json-schema-valid\",\n      rationale: valid ? \"value satisfies the schema\" : violation,\n    }\n  },\n)\n\n/** Returns the first violation message, or undefined when the value is valid. */\nfunction firstSchemaViolation(\n  actual: JsonValue,\n  schema: MinimalJsonSchema,\n): string | undefined {\n  const declaredType = schema.type ?? \"object\"\n  const actualType = jsonTypeOf(actual)\n  if (declaredType !== actualType) {\n    return `expected top-level type '${declaredType}' but got '${actualType}'`\n  }\n\n  // Only object payloads carry required/properties semantics here.\n  if (!isJsonObject(actual)) {\n    return undefined\n  }\n\n  for (const key of schema.required ?? []) {\n    if (!(key in actual)) {\n      return `missing required property '${key}'`\n    }\n  }\n\n  for (const [key, propSchema] of Object.entries(schema.properties ?? {})) {\n    if (propSchema.type === undefined) continue\n    const propValue = actual[key]\n    if (propValue === undefined) continue\n    const propType = jsonTypeOf(propValue)\n    if (propType !== propSchema.type) {\n      return `property '${key}' expected type '${propSchema.type}' but got '${propType}'`\n    }\n  }\n\n  return undefined\n}\n\n// ---------------------------------------------------------------------------\n// eval.latency-budget\n// ---------------------------------------------------------------------------\n\nexport const latencyBudgetTool = defineTool({\n  id: \"eval.latency-budget\",\n  description:\n    \"Deterministic scorer: passes when `durationMs` is within `budgetMs`. \" +\n    \"value is 1 when within budget, else linearly decays toward 0 as the \" +\n    \"overrun approaches one full budget.\",\n  version: \"0.1.0\",\n  inputSchema: z.object({\n    durationMs: z\n      .number()\n      .min(0)\n      .describe(\"Observed duration in milliseconds.\"),\n    budgetMs: z\n      .number()\n      .positive()\n      .describe(\"Allowed budget in milliseconds (must be > 0).\"),\n  }),\n  outputSchema: scoreSchema,\n  mutates: [],\n  approval: \"auto\",\n  riskLevel: 0,\n})\n\nexport const latencyBudgetImpl = implementTool(latencyBudgetTool, ({ input }) => {\n  const { durationMs, budgetMs } = input\n  const passed = durationMs <= budgetMs\n  const value = passed\n    ? 1\n    : Math.max(0, 1 - (durationMs - budgetMs) / budgetMs)\n  return {\n    value,\n    passed,\n    label: \"latency-budget\",\n    rationale: `durationMs=${durationMs}, budgetMs=${budgetMs}`,\n  }\n})\n","import { z } from \"zod\"\nimport { defineTool } from \"@agentproto/tool\"\nimport { defineDriver, implementTool, type DriverHandle } from \"@agentproto/driver\"\nimport { scoreSchema } from \"./score.js\"\nimport { jsonValueSchema, type JsonValue } from \"./json.js\"\nimport type { ScorerBinding } from \"./run-eval.js\"\n\n/**\n * `llmJudge` — a model-backed scorer.\n *\n * Design (locked): a model-backed scorer stays a TOOL, exactly like the\n * deterministic scorers in `scorers.ts`. What differs is where the model\n * lives: the injected {@link JudgeFn} is closed over by the DRIVER (built by\n * {@link makeLlmJudgeDriver}), never threaded through tool input/context. That\n * keeps `eval.llm-judge`'s contract identical in shape to every other scorer\n * tool, so it composes with the existing `bindScorer` / `runEval` in\n * run-eval.ts with ZERO changes there.\n *\n * The judge itself is vendor-neutral: {@link JudgeFn} is just an injected\n * async function. This file has no LLM SDK and no network dependency — a real\n * adapter (an agent session, a supervisor judge-gate, …) is a documented\n * follow-up, not built here.\n */\n\n// ---------------------------------------------------------------------------\n// JudgeVerdict — what an injected judge returns\n// ---------------------------------------------------------------------------\n\n/** Zod schema for the raw verdict a {@link JudgeFn} produces. */\nexport const judgeVerdictSchema = z.object({\n  /** Normalized judge score in [0, 1]. */\n  value: z.number().min(0).max(1),\n  /**\n   * Explicit pass/fail from the judge. When present it WINS over the\n   * threshold comparison — see {@link makeLlmJudgeDriver}.\n   */\n  passed: z.boolean().optional(),\n  /** Human-readable explanation of the verdict. */\n  rationale: z.string().optional(),\n})\n\nexport type JudgeVerdict = z.infer<typeof judgeVerdictSchema>\n\n// ---------------------------------------------------------------------------\n// JudgeFn — the injected capability\n// ---------------------------------------------------------------------------\n\n/**\n * The seam a real judge satisfies: given the produced `output`, free-form\n * grading `criteria`, and an optional `expected` reference, return a\n * {@link JudgeVerdict}. Callers supply this — a real LLM call, an agent\n * session, or the supervisor's judge-gate all satisfy the same shape.\n * Deliberately no LLM SDK / network types here: keeping this package\n * vendor-neutral is the point.\n */\nexport type JudgeFn = (args: {\n  readonly output: JsonValue\n  readonly criteria: string\n  readonly expected?: JsonValue\n}) => Promise<JudgeVerdict>\n\n// ---------------------------------------------------------------------------\n// eval.llm-judge — the TOOL contract\n// ---------------------------------------------------------------------------\n\n/** Tool input for `eval.llm-judge`. */\nexport interface LlmJudgeInput {\n  readonly output: JsonValue\n  readonly criteria: string\n  readonly expected?: JsonValue\n}\n\nexport const llmJudgeTool = defineTool({\n  id: \"eval.llm-judge\",\n  description:\n    \"Model-backed scorer: hands `output` (plus free-form `criteria` and an \" +\n    \"optional `expected` reference) to an injected judge and normalizes the \" +\n    \"judge's verdict into the shared Score shape. The judge itself lives in \" +\n    \"the driver (see makeLlmJudgeDriver) — this tool contract carries no \" +\n    \"model call of its own.\",\n  version: \"0.1.0\",\n  inputSchema: z.object({\n    output: jsonValueSchema.describe(\"The produced value to judge.\"),\n    criteria: z.string().describe(\"Free-form grading criteria/rubric for the judge.\"),\n    expected: jsonValueSchema\n      .optional()\n      .describe(\"Optional reference value the judge may compare against.\"),\n  }),\n  outputSchema: scoreSchema,\n  mutates: [],\n  approval: \"auto\",\n  riskLevel: 0,\n})\n\n// ---------------------------------------------------------------------------\n// makeLlmJudgeDriver — closes over the injected JudgeFn\n// ---------------------------------------------------------------------------\n\n/** Clamp a number into [0, 1]. */\nfunction clamp01(x: number): number {\n  return Math.min(1, Math.max(0, x))\n}\n\nexport interface MakeLlmJudgeDriverOptions {\n  /** Minimum `value` to count as passed when the judge omits `passed`. Default 0.5. */\n  readonly threshold?: number\n}\n\n/**\n * Build a DRIVER that implements `eval.llm-judge` by delegating to `judge`.\n * This is the one seam where the injected model-backed capability enters the\n * system — everything downstream (`bindScorer`, `runEval`) stays unaware that\n * this scorer is model-backed at all.\n */\nexport function makeLlmJudgeDriver(\n  judge: JudgeFn,\n  opts?: MakeLlmJudgeDriverOptions,\n): DriverHandle {\n  const threshold = opts?.threshold ?? 0.5\n  return defineDriver({\n    id: \"eval-llm-judge\",\n    name: \"Eval LLM Judge (model-backed)\",\n    description:\n      \"Model-backed scorer driver: implements eval.llm-judge by awaiting an \" +\n      \"injected JudgeFn and mapping its verdict to the shared Score shape. \" +\n      \"The judge is supplied by the caller — no LLM SDK or network call here.\",\n    version: \"0.1.0\",\n    kind: \"builtin\",\n    implements: [{ tool: \"eval.llm-judge\", version: \"0.1.0\" }],\n    implementations: [\n      implementTool(llmJudgeTool, async ({ input }) => {\n        const verdict = await judge({\n          output: input.output,\n          criteria: input.criteria,\n          expected: input.expected,\n        })\n        const value = clamp01(verdict.value)\n        const passed = verdict.passed ?? value >= threshold\n        return {\n          value,\n          passed,\n          label: \"llm-judge\",\n          ...(verdict.rationale ? { rationale: verdict.rationale } : {}),\n        }\n      }),\n    ],\n  })\n}\n\n// ---------------------------------------------------------------------------\n// llmJudge — convenience ScorerBinding factory\n// ---------------------------------------------------------------------------\n\nexport interface LlmJudgeBinding<A> {\n  /** Label for this binding, e.g. \"helpfulness\". */\n  readonly id: string\n  /** The injected judge capability. */\n  readonly judge: JudgeFn\n  /** Free-form grading criteria/rubric handed to the judge on every call. */\n  readonly criteria: string\n  /** Minimum value to count as passed when the judge omits `passed`. Default 0.5. */\n  readonly threshold?: number\n  /** Derive the judge's `output` input from the target's output. */\n  readonly mapOutput: (ctx: { output: A; expected?: JsonValue }) => JsonValue\n}\n\n/**\n * Convenience: build a {@link ScorerBinding} ready to drop into a suite's\n * `scorers` via `bindScorer` — wires up `eval.llm-judge`, a driver built with\n * {@link makeLlmJudgeDriver}, and the input mapping in one call.\n */\nexport function llmJudge<A>(binding: LlmJudgeBinding<A>): ScorerBinding<A, LlmJudgeInput> {\n  const driver = makeLlmJudgeDriver(binding.judge, { threshold: binding.threshold })\n  return {\n    id: binding.id,\n    tool: llmJudgeTool,\n    driver,\n    mapInput: (ctx) => ({\n      output: binding.mapOutput({ output: ctx.output, expected: ctx.expected }),\n      criteria: binding.criteria,\n      expected: ctx.expected,\n    }),\n  }\n}\n","/**\n * Structured events emitted while an eval suite runs.\n *\n * The union mirrors the shape of `@agentproto/telemetry`'s `TelemetryEvent`:\n * a discriminated union keyed on `kind`, every member carrying a correlation\n * id (`runId`) and an ISO `at` timestamp so a sink can rebuild a per-run span\n * tree. It rides the same {@link Telemetry} port — a `runEval` call takes a\n * `Telemetry<EvalEvent>` sink.\n *\n * Sinks MUST tolerate unknown kinds — future versions may add event types\n * under the same `agentproto/eval/v1` schema (identical forward-compat\n * contract as the telemetry port).\n */\n\n/** Schema identifier for this event family. */\nexport const EVAL_EVENT_SCHEMA = \"agentproto/eval/v1\" as const\n\nexport type EvalEvent =\n  | {\n      readonly kind: \"eval.started\"\n      readonly runId: string\n      readonly at: string\n      readonly suiteId: string\n      readonly caseCount: number\n      readonly scorerCount: number\n    }\n  | {\n      readonly kind: \"eval.case.started\"\n      readonly runId: string\n      readonly at: string\n      readonly caseId: string\n    }\n  | {\n      readonly kind: \"eval.case.scored\"\n      readonly runId: string\n      readonly at: string\n      readonly caseId: string\n      readonly scorerId: string\n      readonly value: number\n      readonly passed: boolean\n    }\n  | {\n      readonly kind: \"eval.case.finished\"\n      readonly runId: string\n      readonly at: string\n      readonly caseId: string\n      /** True when EVERY scorer bound to the case passed. */\n      readonly passed: boolean\n    }\n  | {\n      readonly kind: \"eval.finished\"\n      readonly runId: string\n      readonly at: string\n      readonly suiteId: string\n      readonly total: number\n      readonly passedCount: number\n      readonly meanValue: number\n      readonly durationMs: number\n    }\n","/**\n * `runEval` — the eval harness.\n *\n * The thesis this file proves: **an eval is a workflow that runs a target then\n * scores its output with scorer-tools, reporting through the telemetry port.**\n *\n * For each case we build a {@link RuntimeWorkflow} that composes\n *   target → one scorer `tool` step per binding → a per-case aggregate transform\n * and execute it with `runWorkflow` (AIP-15). The target's structured output is\n * the input selector for every scorer step — that is where composable tool I/O\n * is demonstrated. `runEval` then aggregates across cases and emits\n * {@link EvalEvent}s through an injected `Telemetry<EvalEvent>` sink.\n *\n * The `target` is a plain async function here (not required to be a tool); a\n * tool/agent target is a documented follow-up.\n */\n\nimport { runWorkflow, type RuntimeWorkflow } from \"@agentproto/workflow-runtime\"\nimport type { Telemetry } from \"@agentproto/telemetry\"\nimport type { DriverHandle } from \"@agentproto/driver\"\nimport type { ToolHandle } from \"@agentproto/tool\"\nimport type { Score } from \"./score.js\"\nimport type { JsonValue } from \"./json.js\"\nimport type { EvalEvent } from \"./events.js\"\n\n/** One case: an input for the target and an optional expected reference. */\nexport interface EvalCase<I> {\n  readonly id: string\n  readonly input: I\n  /** Optional reference value, modelled as JSON (never `unknown`/`any`). */\n  readonly expected?: JsonValue\n}\n\n/** Context handed to a binding's `mapInput` when scoring one case's output. */\nexport interface ScorerInputContext<A> {\n  readonly output: A\n  readonly expected?: JsonValue\n}\n\n/**\n * One scorer applied to the target output. Generic over the target output type\n * `A` and the scorer's own input type `S` (captured when the binding is\n * authored). {@link bindScorer} erases `S` so heterogeneous scorers coexist in\n * one `EvalSuite.scorers` array without an `any`.\n */\nexport interface ScorerBinding<A, S = JsonValue> {\n  /** Label for this binding, e.g. \"exact\". */\n  readonly id: string\n  /** The scorer TOOL contract — its `outputSchema` is the shared {@link Score}. */\n  readonly tool: ToolHandle<S, Score>\n  /** The DRIVER that implements the scorer tool. */\n  readonly driver: DriverHandle\n  /** Derive the scorer's input from the target's output (+ optional expected). */\n  readonly mapInput: (ctx: ScorerInputContext<A>) => S\n}\n\n/**\n * The uniform, `S`-erased view of a binding stored in a suite. It exposes the\n * authored `id` plus a closure that runs the scorer for a given case output —\n * the existential box that lets a suite hold scorers with different input\n * types with no `any` at the collection boundary.\n */\nexport interface BoundScorer<A> {\n  readonly id: string\n  /** Build the AIP-15 tool step that scores one case's output. */\n  toStep(stepId: string, output: A, expected?: JsonValue): RuntimeWorkflow[\"steps\"][number]\n}\n\n/**\n * Box a typed {@link ScorerBinding} into a {@link BoundScorer}, capturing the\n * scorer input type `S` inside the closure so the outer type is `S`-free.\n */\nexport function bindScorer<A, S>(binding: ScorerBinding<A, S>): BoundScorer<A> {\n  return {\n    id: binding.id,\n    toStep(stepId, output, expected) {\n      return {\n        kind: \"tool\",\n        id: stepId,\n        tool: binding.tool,\n        candidates: [binding.driver],\n        input: () => binding.mapInput({ output, expected }),\n      }\n    },\n  }\n}\n\nexport interface EvalSuite<A> {\n  readonly id: string\n  readonly cases: readonly EvalCase<unknown>[]\n  readonly scorers: readonly BoundScorer<A>[]\n}\n\n/** One scorer's outcome on one case. */\nexport interface CaseScore {\n  readonly scorerId: string\n  readonly score: Score\n}\n\n/** The per-case rollup carried in the final report. */\nexport interface CaseReport {\n  readonly caseId: string\n  readonly passed: boolean\n  readonly scores: readonly CaseScore[]\n}\n\nexport interface EvalReport {\n  readonly runId: string\n  readonly suiteId: string\n  readonly total: number\n  readonly passedCount: number\n  /** Mean of every scorer value across every case (0 when there are none). */\n  readonly meanValue: number\n  readonly cases: readonly CaseReport[]\n}\n\nexport interface RunEvalOptions<I, A> {\n  /** Produce the target output for a case input. */\n  readonly target: (input: I) => Promise<A>\n  /** Sink for {@link EvalEvent}s. Defaults to a no-op. */\n  readonly telemetry?: Telemetry<EvalEvent>\n  /** Correlation id for this run. Defaults to a deterministic suite-based id. */\n  readonly runId?: string\n}\n\n/** A suite whose cases share the target input type `I`. */\nexport interface TypedEvalSuite<I, A> extends EvalSuite<A> {\n  readonly cases: readonly EvalCase<I>[]\n}\n\n/** Per-process counter so a default runId is deterministic within a process. */\nlet runCounter = 0\n\nfunction defaultRunId(suiteId: string): string {\n  runCounter += 1\n  return `${suiteId}#${runCounter}`\n}\n\n/** The per-case aggregate: which scorers ran and whether all passed. */\ninterface CaseAggregate {\n  readonly scores: readonly CaseScore[]\n  readonly passed: boolean\n}\n\n/** Narrow a step-binding value to a {@link Score} without a cast. */\nfunction asScore(value: unknown): Score {\n  if (\n    typeof value === \"object\" &&\n    value !== null &&\n    \"value\" in value &&\n    \"passed\" in value &&\n    \"label\" in value\n  ) {\n    const v: unknown = value.value\n    const passed: unknown = value.passed\n    const label: unknown = value.label\n    const rationale: unknown = \"rationale\" in value ? value.rationale : undefined\n    if (\n      typeof v === \"number\" &&\n      typeof passed === \"boolean\" &&\n      typeof label === \"string\" &&\n      (rationale === undefined || typeof rationale === \"string\")\n    ) {\n      return { value: v, passed, label, rationale }\n    }\n  }\n  throw new Error(\"scorer step did not produce a valid Score\")\n}\n\n/**\n * Run every case in the suite, scoring each with all bound scorers, and return\n * the aggregated {@link EvalReport}. Emits `eval.started` … `eval.finished`.\n */\nexport async function runEval<I, A>(\n  suite: TypedEvalSuite<I, A>,\n  opts: RunEvalOptions<I, A>,\n): Promise<EvalReport> {\n  const telemetry = opts.telemetry\n  const runId = opts.runId ?? defaultRunId(suite.id)\n  const start = Date.now()\n\n  telemetry?.emit({\n    kind: \"eval.started\",\n    runId,\n    at: new Date().toISOString(),\n    suiteId: suite.id,\n    caseCount: suite.cases.length,\n    scorerCount: suite.scorers.length,\n  })\n\n  const caseReports: CaseReport[] = []\n  let valueSum = 0\n  let valueCount = 0\n\n  for (const evalCase of suite.cases) {\n    telemetry?.emit({\n      kind: \"eval.case.started\",\n      runId,\n      at: new Date().toISOString(),\n      caseId: evalCase.id,\n    })\n\n    const aggregate = await runCase(suite, evalCase, opts.target)\n\n    for (const cs of aggregate.scores) {\n      valueSum += cs.score.value\n      valueCount += 1\n      telemetry?.emit({\n        kind: \"eval.case.scored\",\n        runId,\n        at: new Date().toISOString(),\n        caseId: evalCase.id,\n        scorerId: cs.scorerId,\n        value: cs.score.value,\n        passed: cs.score.passed,\n      })\n    }\n\n    telemetry?.emit({\n      kind: \"eval.case.finished\",\n      runId,\n      at: new Date().toISOString(),\n      caseId: evalCase.id,\n      passed: aggregate.passed,\n    })\n\n    caseReports.push({\n      caseId: evalCase.id,\n      passed: aggregate.passed,\n      scores: aggregate.scores,\n    })\n  }\n\n  const passedCount = caseReports.filter((c) => c.passed).length\n  const meanValue = valueCount === 0 ? 0 : valueSum / valueCount\n\n  const report: EvalReport = {\n    runId,\n    suiteId: suite.id,\n    total: suite.cases.length,\n    passedCount,\n    meanValue,\n    cases: caseReports,\n  }\n\n  telemetry?.emit({\n    kind: \"eval.finished\",\n    runId,\n    at: new Date().toISOString(),\n    suiteId: suite.id,\n    total: report.total,\n    passedCount,\n    meanValue,\n    durationMs: Date.now() - start,\n  })\n\n  return report\n}\n\n/**\n * Build and run the per-case workflow: `target` → one scorer tool-step per\n * binding → an aggregate transform, then read the aggregate off the run output.\n */\nasync function runCase<I, A>(\n  suite: EvalSuite<A>,\n  evalCase: EvalCase<I>,\n  target: (input: I) => Promise<A>,\n): Promise<CaseAggregate> {\n  const output = await target(evalCase.input)\n  const scorerStepIds = suite.scorers.map((_, i) => `score_${i}`)\n\n  const workflow: RuntimeWorkflow = {\n    id: `${suite.id}:${evalCase.id}`,\n    steps: [\n      ...suite.scorers.map((scorer, i) =>\n        scorer.toStep(scorerStepIds[i]!, output, evalCase.expected),\n      ),\n      {\n        kind: \"transform\",\n        id: \"aggregate\",\n        compute: (b): CaseAggregate => {\n          const scores: CaseScore[] = suite.scorers.map((scorer, i) => ({\n            scorerId: scorer.id,\n            score: asScore(b.steps[scorerStepIds[i]!]),\n          }))\n          return { scores, passed: scores.every((s) => s.score.passed) }\n        },\n      },\n    ],\n    output: (b) => b.steps.aggregate,\n  }\n\n  const { output: runOutput } = await runWorkflow({ workflow })\n  return toCaseAggregate(runOutput)\n}\n\n/** Narrow the workflow output to a {@link CaseAggregate} without a cast. */\nfunction toCaseAggregate(value: unknown): CaseAggregate {\n  if (\n    typeof value === \"object\" &&\n    value !== null &&\n    \"scores\" in value &&\n    \"passed\" in value &&\n    Array.isArray(value.scores) &&\n    typeof value.passed === \"boolean\"\n  ) {\n    const scores = value.scores.map((entry: unknown) => {\n      if (\n        typeof entry === \"object\" &&\n        entry !== null &&\n        \"scorerId\" in entry &&\n        \"score\" in entry &&\n        typeof entry.scorerId === \"string\"\n      ) {\n        return { scorerId: entry.scorerId, score: asScore(entry.score) }\n      }\n      throw new Error(\"invalid CaseScore in aggregate\")\n    })\n    return { scores, passed: value.passed }\n  }\n  throw new Error(\"workflow did not produce a CaseAggregate\")\n}\n","/**\n * `toVitest` — turn an {@link TypedEvalSuite} into vitest test registrations.\n *\n * The vitest primitives (`describe` / `it` / `expect`) are INJECTED by the\n * host's own `import ... from \"vitest\"`, so THIS source carries no vitest\n * runtime dependency (nothing to bundle, nothing to version-pin). It registers\n * one `it(caseId)` per case that runs the case through {@link runEval} and\n * asserts every scorer passed — the same workflow the harness runs, wired as a\n * CI gate.\n */\n\nimport { runEval, type RunEvalOptions, type TypedEvalSuite } from \"./run-eval.js\"\n\n/** The subset of vitest's `expect` this bridge needs. */\nexport interface ExpectApi {\n  (actual: unknown): {\n    toBe(expected: unknown): void\n  }\n}\n\n/** The subset of vitest's `describe`/`it` this bridge needs. */\nexport interface VitestHooks {\n  describe(name: string, body: () => void): void\n  it(name: string, body: () => Promise<void> | void): void\n  expect: ExpectApi\n}\n\nexport type ToVitestOptions<I, A> = RunEvalOptions<I, A>\n\n/**\n * Register `describe(suite.id)` with one `it` per case. Each `it` runs the\n * single-case suite through {@link runEval} and asserts the case passed (all\n * scorers green). Failing scorers surface as a failing vitest assertion.\n */\nexport function toVitest<I, A>(\n  suite: TypedEvalSuite<I, A>,\n  opts: ToVitestOptions<I, A>,\n  hooks: VitestHooks,\n): void {\n  const { describe, it, expect } = hooks\n  describe(suite.id, () => {\n    for (const evalCase of suite.cases) {\n      it(evalCase.id, async () => {\n        const singleCaseSuite: TypedEvalSuite<I, A> = {\n          id: `${suite.id}:${evalCase.id}`,\n          cases: [evalCase],\n          scorers: suite.scorers,\n        }\n        const report = await runEval(singleCaseSuite, opts)\n        const caseReport = report.cases[0]\n        expect(caseReport?.passed).toBe(true)\n      })\n    }\n  })\n}\n","/**\n * @agentproto/eval — deterministic reference scorers.\n *\n * A scorer IS an AIP-14 TOOL whose `outputSchema` is the shared {@link Score}\n * shape; there is no separate \"scorer port\". Each scorer is authored with\n * `defineTool` + `implementTool` and bundled in a single builtin AIP-30\n * PROVIDER, exactly like every other builtin. Invoke one through the driver:\n *\n * ```ts\n * import { runTool } from \"@agentproto/driver\"\n * import { exactMatchTool, evalScorersProvider } from \"@agentproto/eval\"\n *\n * const score = await runTool({\n *   tool: exactMatchTool,\n *   candidates: [evalScorersProvider],\n *   input: { actual: \"hello\", expected: \"hello\" },\n * })\n * // → { value: 1, passed: true, label: \"exact-match\", rationale: \"…\" }\n * ```\n *\n * Alongside the deterministic scorers, `eval.llm-judge` (see judge.ts) is a\n * model-backed scorer whose judge is an INJECTED function — the driver\n * closes over it via `makeLlmJudgeDriver(judge)`. This package still carries\n * no LLM SDK or network dependency of its own; a real adapter (agent\n * session, supervisor judge-gate, …) is a documented follow-up.\n *\n * Spec: https://agentproto.sh/docs/aip-14 (TOOL), /docs/aip-30 (PROVIDER)\n */\n\nimport { defineDriver } from \"@agentproto/driver\"\nimport {\n  exactMatchTool,\n  exactMatchImpl,\n  regexMatchTool,\n  regexMatchImpl,\n  jsonSchemaValidTool,\n  jsonSchemaValidImpl,\n  latencyBudgetTool,\n  latencyBudgetImpl,\n} from \"./scorers.js\"\n\nexport const SPEC_NAME = \"agenteval/v1\" as const\nexport const SPEC_VERSION = \"1.0.0-alpha\" as const\n\nexport { scoreSchema, type Score } from \"./score.js\"\n\nexport {\n  type JsonValue,\n  type JsonObject,\n  jsonValueSchema,\n} from \"./json.js\"\n\nexport {\n  exactMatchTool,\n  exactMatchImpl,\n  regexMatchTool,\n  regexMatchImpl,\n  jsonSchemaValidTool,\n  jsonSchemaValidImpl,\n  latencyBudgetTool,\n  latencyBudgetImpl,\n  type MinimalJsonSchema,\n} from \"./scorers.js\"\n\nexport {\n  judgeVerdictSchema,\n  llmJudgeTool,\n  makeLlmJudgeDriver,\n  llmJudge,\n  type JudgeVerdict,\n  type JudgeFn,\n  type LlmJudgeInput,\n  type MakeLlmJudgeDriverOptions,\n  type LlmJudgeBinding,\n} from \"./judge.js\"\n\nexport { EVAL_EVENT_SCHEMA, type EvalEvent } from \"./events.js\"\n\nexport {\n  runEval,\n  bindScorer,\n  type EvalCase,\n  type ScorerBinding,\n  type ScorerInputContext,\n  type BoundScorer,\n  type EvalSuite,\n  type TypedEvalSuite,\n  type CaseScore,\n  type CaseReport,\n  type EvalReport,\n  type RunEvalOptions,\n} from \"./run-eval.js\"\n\nexport {\n  toVitest,\n  type VitestHooks,\n  type ExpectApi,\n  type ToVitestOptions,\n} from \"./to-vitest.js\"\n\n/**\n * Builtin AIP-30 PROVIDER bundling the four deterministic ref scorers.\n * `kind: \"builtin\"` — pure in-process functions, no subprocess / network hop.\n */\nexport const evalScorersProvider = defineDriver({\n  id: \"eval-scorers\",\n  name: \"Eval Reference Scorers (built-in)\",\n  description:\n    \"In-process deterministic scorers: exact-match, regex-match, \" +\n    \"json-schema-valid (minimal structural check), and latency-budget. \" +\n    \"Each is an AIP-14 TOOL whose output is the shared Score shape.\",\n  version: \"0.1.0\",\n  kind: \"builtin\",\n  implements: [\n    { tool: \"eval.exact-match\", version: \"0.1.0\" },\n    { tool: \"eval.regex-match\", version: \"0.1.0\" },\n    { tool: \"eval.json-schema-valid\", version: \"0.1.0\" },\n    { tool: \"eval.latency-budget\", version: \"0.1.0\" },\n  ],\n  implementations: [\n    exactMatchImpl,\n    regexMatchImpl,\n    jsonSchemaValidImpl,\n    latencyBudgetImpl,\n  ],\n})\n"]}

package/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "@agentproto/eval",
+  "version": "0.2.0",
+  "description": "Deterministic reference scorers as AIP-14 TOOL contracts + one builtin AIP-30 PROVIDER. A scorer IS a tool whose outputSchema is the shared `Score` shape — no new scorer port. Ships exact-match, regex-match, json-schema-valid, and latency-budget. No LLM/model calls.",
+  "keywords": [
+    "agentproto",
+    "aip-14",
+    "aip-30",
+    "eval",
+    "scorer",
+    "score",
+    "deterministic",
+    "open-standard",
+    "agentic"
+  ],
+  "homepage": "https://agentproto.sh",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/agentproto/ts",
+    "directory": "packages/eval"
+  },
+  "bugs": {
+    "url": "https://github.com/agentproto/ts/issues"
+  },
+  "license": "MIT",
+  "type": "module",
+  "main": "dist/index.mjs",
+  "module": "dist/index.mjs",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "default": "./dist/index.mjs"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "zod": "^4.4.3",
+    "@agentproto/driver": "0.1.2",
+    "@agentproto/tool": "0.2.0",
+    "@agentproto/workflow-runtime": "0.3.0",
+    "@agentproto/telemetry": "0.2.0"
+  },
+  "devDependencies": {
+    "@types/node": "^25.6.2",
+    "tsup": "^8.5.1",
+    "typescript": "^5.9.3",
+    "vitest": "^3.2.4",
+    "@agentproto/tooling": "0.1.0-alpha.0"
+  },
+  "scripts": {
+    "dev": "tsup --watch",
+    "build": "tsup",
+    "clean": "rm -rf dist",
+    "check-types": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  }
+}