@opensearch-project/agent-health 0.3.0 → 0.5.2

package/lib/dist/lib/testCases/evaluators.js

@@ -0,0 +1,105 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { recordVerdict } from '../matchers/session.js';
+const evaluatorRegistry = new Map();
+/**
+ * Register a custom evaluator under `id`.
+ *
+ * Registering the **same function** under an id again is a no-op (a watched
+ * file re-loading the identical module is fine). Registering a *different*
+ * function under an id that's already taken throws: the registry is
+ * process-global (shared like a helper library), so two `.eval.{js,ts}` files
+ * defining `defineEvaluator('len-check', …)` with different bodies would
+ * otherwise silently shadow each other based on load order. Fail loudly so the
+ * collision is visible at authoring time. Use distinct ids (or a shared helper
+ * module) when you need different implementations.
+ */
+export function defineEvaluator(id, fn) {
+    if (!id || typeof id !== 'string') {
+        throw new Error('defineEvaluator(id, fn): id must be a non-empty string');
+    }
+    if (typeof fn !== 'function') {
+        throw new Error(`defineEvaluator('${id}', fn): fn must be a function`);
+    }
+    const existing = evaluatorRegistry.get(id);
+    if (existing && existing !== fn) {
+        throw new Error(`defineEvaluator('${id}', ...): an evaluator with id '${id}' is already ` +
+            `registered with a different function. Evaluator ids are global — use a ` +
+            `unique id per evaluator (or share one function) to avoid silent shadowing.`);
+    }
+    evaluatorRegistry.set(id, fn);
+}
+/** Look up a registered evaluator by id. @internal */
+export function getEvaluator(id) {
+    return evaluatorRegistry.get(id);
+}
+/** Clear all registered evaluators (test isolation / per-load reset). @internal */
+export function clearEvaluators() {
+    evaluatorRegistry.clear();
+}
+/** Build a {@link Verdict} from a custom-evaluator result. */
+function evaluatorVerdict(r, role, errored, errorMessage) {
+    const verdict = {
+        passFailStatus: r.pass ? 'passed' : 'failed',
+        accuracy: typeof r.score === 'number' ? r.score * 100 : (r.pass ? 100 : 0),
+        score: typeof r.score === 'number' ? r.score : (r.pass ? 1 : 0),
+        reasoning: r.reasoning ?? '',
+        pass: r.pass,
+        role,
+        skipped: false,
+        errored,
+        errorMessage,
+        orThrow() {
+            if (this.pass || this.skipped)
+                return this;
+            const label = this.errored ? 'errored' : 'FAILED';
+            throw new Error(`Evaluator: ${label}\n${this.errorMessage ?? this.reasoning}`);
+        },
+    };
+    return verdict;
+}
+async function runEvaluator(result, evaluatorId, criteria, role) {
+    const fn = getEvaluator(evaluatorId);
+    const description = `evaluate: ${evaluatorId}${criteria ? ` — ${criteria}` : ''}`;
+    if (!fn) {
+        // Unknown evaluator id — this is an authoring error the run can't
+        // recover from. Record it as errored (excluded from pass-rate).
+        const msg = `No evaluator registered with id '${evaluatorId}'. Did you call defineEvaluator('${evaluatorId}', ...)?`;
+        recordVerdict({
+            description, pass: false, method: 'evaluator', role, errored: true,
+            durationMs: 0, errorMessage: msg, reasoning: '',
+        });
+        return evaluatorVerdict({ pass: false }, role, true, msg);
+    }
+    const startedAt = Date.now();
+    let out;
+    try {
+        out = await fn({ result, criteria, traces: result.traces });
+    }
+    catch (err) {
+        const msg = err?.message || String(err);
+        recordVerdict({
+            description, pass: false, method: 'evaluator', role, errored: true,
+            durationMs: Date.now() - startedAt, errorMessage: `Evaluator threw: ${msg}`, reasoning: '',
+        });
+        return evaluatorVerdict({ pass: false }, role, true, `Evaluator threw: ${msg}`);
+    }
+    const verdict = evaluatorVerdict(out, role, false);
+    recordVerdict({
+        description,
+        pass: verdict.pass,
+        method: 'evaluator',
+        role,
+        durationMs: Date.now() - startedAt,
+        score: verdict.score,
+        reasoning: verdict.reasoning,
+        errorMessage: verdict.pass ? undefined : (verdict.reasoning || 'evaluator failed'),
+    });
+    return verdict;
+}
+export const evaluate = Object.assign((result, evaluatorId, criteria) => runEvaluator(result, evaluatorId, criteria, 'gate'), {
+    observe: (result, evaluatorId, criteria) => runEvaluator(result, evaluatorId, criteria, 'observe'),
+});
+//# sourceMappingURL=evaluators.js.map

package/lib/dist/lib/testCases/evaluators.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluators.js","sourceRoot":"","sources":["../../../testCases/evaluators.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAqCH,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AA0BvD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAuB,CAAC;AAEzD;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,eAAe,CAAC,EAAU,EAAE,EAAe;IACzD,IAAI,CAAC,EAAE,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,OAAO,EAAE,KAAK,UAAU,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,oBAAoB,EAAE,+BAA+B,CAAC,CAAC;IACzE,CAAC;IACD,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC3C,IAAI,QAAQ,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,oBAAoB,EAAE,kCAAkC,EAAE,eAAe;YACvE,yEAAyE;YACzE,4EAA4E,CAC/E,CAAC;IACJ,CAAC;IACD,iBAAiB,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;AAChC,CAAC;AAED,sDAAsD;AACtD,MAAM,UAAU,YAAY,CAAC,EAAU;IACrC,OAAO,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AACnC,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,eAAe;IAC7B,iBAAiB,CAAC,KAAK,EAAE,CAAC;AAC5B,CAAC;AAED,8DAA8D;AAC9D,SAAS,gBAAgB,CACvB,CAAkB,EAClB,IAAe,EACf,OAAgB,EAChB,YAAqB;IAErB,MAAM,OAAO,GAAY;QACvB,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QAC5C,QAAQ,EAAE,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1E,KAAK,EAAE,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,EAAE;QAC5B,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,IAAI;QACJ,OAAO,EAAE,KAAK;QACd,OAAO;QACP,YAAY;QACZ,OAAO;YACL,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAO,IAAI,CAAC;YAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;YAClD,MAAM,IAAI,KAAK,CACb,cAAc,KAAK,KAAK,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,SAAS,EAAE,CAC9D,CAAC;QACJ,CAAC;KACF,CAAC;IACF,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,MAAkB,EAClB,WAAmB,EACnB,QAA4B,EAC5B,IAAe;IAEf,MAAM,EAAE,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IACrC,MAAM,WAAW,GAAG,aAAa,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAElF,IAAI,CAAC,EAAE,EAAE,CAAC;QACR,kEAAkE;QAClE,gEAAgE;QAChE,MAAM,GAAG,GAAG,oCAAoC,WAAW,oCAAoC,WAAW,UAAU,CAAC;QACrH,aAAa,CAAC;YACZ,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI;YAClE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;SAChD,CAAC,CAAC;QACH,OAAO,gBAAgB,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,GAAoB,CAAC;IACzB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9D,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;QACxC,aAAa,CAAC;YACZ,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI;YAClE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,YAAY,EAAE,oBAAoB,GAAG,EAAE,EAAE,SAAS,EAAE,EAAE;SAC3F,CAAC,CAAC;QACH,OAAO,gBAAgB,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,oBAAoB,GAAG,EAAE,CAAC,CAAC;IAClF,CAAC;IAED,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;IACnD,aAAa,CAAC;QACZ,WAAW;QACX,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM,EAAE,WAAW;QACnB,IAAI;QACJ,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;KACnF,CAAC,CAAC;IACH,OAAO,OAAO,CAAC;AACjB,CAAC;AAWD,MAAM,CAAC,MAAM,QAAQ,GAAe,MAAM,CAAC,MAAM,CAC/C,CAAC,MAAkB,EAAE,WAAmB,EAAE,QAAiB,EAAE,EAAE,CAC7D,YAAY,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,CAAC,EACrD;IACE,OAAO,EAAE,CAAC,MAAkB,EAAE,WAAmB,EAAE,QAAiB,EAAE,EAAE,CACtE,YAAY,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,CAAC;CACzD,CACF,CAAC"}

package/lib/dist/lib/testCases/index.d.ts

@@ -0,0 +1,14 @@
+export { test, describe, beforeAll, afterAll, beforeEach, afterEach, getRegisteredTests, getRegisteredHooks, clearRegistry, } from './define.js';
+export { loadTestCasesFromModule, isCodeFile, computeTestCaseHash } from './loader.js';
+export { migrateEvalSource } from './codemod.js';
+export type { CodemodResult } from './codemod.js';
+export { judge, wasJudgeCalled, resetJudgeFlag } from './judge.js';
+export { clearJudgeCache } from './judge.js';
+export { createAgentFixture } from './agentFixture.js';
+export { defineEvaluator, evaluate, getEvaluator, clearEvaluators, } from './evaluators.js';
+export type { EvaluatorContext, EvaluatorResult, EvaluatorFn, EvaluateFn, } from './evaluators.js';
+export type { AgentFixture, AgentRunOptions, AgentInvokeFn, CreateAgentFixtureOptions, } from './agentFixture.js';
+export type { CodeTestCase, EvalResult, TestOptions, TestFixtures, TestInfo, HookKind, HookFn, RegisteredHook, } from './types.js';
+export type { LoadResult, LoadedTestCase } from './loader.js';
+export type { JudgeVerdict, Verdict, JudgeRole, JudgeFn } from './judge.js';
+//# sourceMappingURL=index.d.ts.map

package/lib/dist/lib/testCases/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../testCases/index.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,IAAI,EACJ,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,UAAU,EACV,SAAS,EACT,kBAAkB,EAClB,kBAAkB,EAClB,aAAa,GACd,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,uBAAuB,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvF,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACjD,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EACL,eAAe,EACf,QAAQ,EACR,YAAY,EACZ,eAAe,GAChB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,gBAAgB,EAChB,eAAe,EACf,WAAW,EACX,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,YAAY,EACZ,eAAe,EACf,aAAa,EACb,yBAAyB,GAC1B,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACV,YAAY,EACZ,UAAU,EACV,WAAW,EACX,YAAY,EACZ,QAAQ,EACR,QAAQ,EACR,MAAM,EACN,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC9D,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC"}

package/lib/dist/lib/testCases/index.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export { test, describe, beforeAll, afterAll, beforeEach, afterEach, getRegisteredTests, getRegisteredHooks, clearRegistry, } from './define.js';
+export { loadTestCasesFromModule, isCodeFile, computeTestCaseHash } from './loader.js';
+export { migrateEvalSource } from './codemod.js';
+export { judge, wasJudgeCalled, resetJudgeFlag } from './judge.js';
+export { clearJudgeCache } from './judge.js';
+export { createAgentFixture } from './agentFixture.js';
+export { defineEvaluator, evaluate, getEvaluator, clearEvaluators, } from './evaluators.js';
+//# sourceMappingURL=index.js.map

package/lib/dist/lib/testCases/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../testCases/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,IAAI,EACJ,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,UAAU,EACV,SAAS,EACT,kBAAkB,EAClB,kBAAkB,EAClB,aAAa,GACd,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,uBAAuB,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvF,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EACL,eAAe,EACf,QAAQ,EACR,YAAY,EACZ,eAAe,GAChB,MAAM,iBAAiB,CAAC"}

package/lib/dist/lib/testCases/judge.d.ts

@@ -0,0 +1,165 @@
+/**
+ * `judge()` — LLM-judge matcher, callable from inside a test body.
+ *
+ * Two ergonomic forms:
+ *
+ *   await judge(result, 'identifies the failing dependency');         // single claim
+ *   await judge(result.trajectory, ['claim 1', 'claim 2']);            // legacy form
+ *
+ * Per-call options:
+ *
+ *   await judge(result, claim, { evaluatorId: 'system-rca-default' });
+ *   await judge(result, claim, { model: 'claude-sonnet' });
+ *
+ * On evaluation, `judge()` records a MatcherResult and returns a
+ * {@link Verdict} **without throwing** (RFC 004 §4.8). A failing
+ * `gate`-role verdict still fails the test (the runner inspects the
+ * recorded results), but the body keeps running so every signal is
+ * collected. To hard-stop the body on a bad verdict, call
+ * `(await judge(...)).orThrow()` or assert `expect(verdict).toPass()`.
+ *
+ * Two roles:
+ *   - `judge(result, claim)`         — gate: a failing verdict fails the test.
+ *   - `judge.observe(result, claim)` — observe: feeds score + insights only,
+ *                                       never fails the test.
+ *
+ * Calls the Agent Health server's /api/judge endpoint with the same
+ * payload shape (`{ trajectory, expectedOutcomes, modelId, evaluatorId }`)
+ * the UI "Run Test" path uses, so SDK and UI runs are scored by the same
+ * judge prompt and provider routing.
+ *
+ * Run-level evaluator default: the runner injects a bound version of
+ * `judge` into `TestFixtures` via `bindJudge(run.evaluatorId)`. Code
+ * that destructures `judge` from the fixture (`async ({ judge }) => ...`)
+ * automatically picks up the run's evaluator with no per-call argument.
+ * Code that imports `judge` from the package gets the unbound version
+ * and must pass `evaluatorId` explicitly.
+ */
+import type { TrajectoryStep } from '../../types/index.js';
+/** Whether a judge signal gates the test verdict or is observational only. */
+export type JudgeRole = 'gate' | 'observe';
+/** Test-only: reset the warn-once latch so unit tests can assert it fires. */
+export declare function __resetJudgePortWarning(): void;
+/**
+ * The result of a `judge()` call. Carries the verdict data plus a few
+ * convenience flags and an explicit hard-stop escape hatch. Returned
+ * (never thrown) so the body can collect multiple verdicts; the recorded
+ * MatcherResult is what actually gates the test.
+ *
+ * Backward-compatible with the old `JudgeVerdict` (same
+ * `passFailStatus`/`accuracy`/`reasoning` fields).
+ */
+export interface Verdict {
+    passFailStatus: 'passed' | 'failed';
+    /** Headline accuracy on the [0, 100] interval. */
+    accuracy: number;
+    /** Normalised score on the [0, 1] interval (`accuracy / 100`). */
+    score: number;
+    /** Free-form judge reasoning. */
+    reasoning: string;
+    /** Convenience: `passFailStatus === 'passed'`. */
+    pass: boolean;
+    /** Role this verdict was produced with (`gate` or `observe`). */
+    role: JudgeRole;
+    /** True when the judge was skipped (no LLM call was made). */
+    skipped: boolean;
+    /** True when the judge could not run at all (endpoint error). */
+    errored: boolean;
+    /** Underlying error message when `errored` is true. */
+    errorMessage?: string;
+    /**
+     * Hard-stop: throw if the verdict did not pass. A no-op for passing,
+     * skipped verdicts. Returns the verdict so it can be chained:
+     *   const v = (await judge(result, claim)).orThrow();
+     */
+    orThrow(): Verdict;
+}
+/** @deprecated Use {@link Verdict}. Retained as a structural alias. */
+export type JudgeVerdict = Verdict;
+/**
+ * Per-call options for the SDK `judge()` matcher. Mirrors the relevant
+ * fields of the `/api/judge` request body so SDK runs and UI runs use
+ * the same judge prompt + evaluator + provider routing.
+ */
+export interface JudgeOptions {
+    /**
+     * Override the agent-health server URL. Defaults to
+     * `http://localhost:${AGENT_HEALTH_PORT ?? 4001}`.
+     */
+    serverUrl?: string;
+    /**
+     * Override the judge model. Forwarded as `modelId` on the request body;
+     * the server resolves it through `config.models[modelId]` exactly as it
+     * does for UI runs.
+     */
+    model?: string;
+    /**
+     * Stored evaluator id (system or user). Forwarded as `evaluatorId` on
+     * the request body. The server resolves system evaluators via
+     * `getSystemEvaluatorById(...)` and user evaluators via
+     * `storage.evaluators.getById(...)` — identical to the UI path. Falls
+     * back to the default evaluator when omitted on both call and bind.
+     */
+    evaluatorId?: string;
+    /**
+     * Skip the LLM call entirely. The judge returns a non-gating `skipped`
+     * verdict (pass: true, skipped: true). Useful for fast/offline iteration.
+     * The global `AH_SKIP_JUDGE=1` env var forces this for every judge call.
+     */
+    skip?: boolean;
+}
+/** Clear the in-process judge verdict cache (test isolation / new run). */
+export declare function clearJudgeCache(): void;
+export declare function wasJudgeCalled(): boolean;
+export declare function resetJudgeFlag(): void;
+interface ResultLike {
+    trajectory?: TrajectoryStep[];
+    finalResponse?: () => string;
+    agentOutput?: string;
+    runId?: string;
+}
+/**
+ * The callable `judge` surface: a function (gate role) with an `.observe`
+ * method (observe role).
+ */
+export interface JudgeFn {
+    (resultOrTrajectory: ResultLike | TrajectoryStep[], claimOrClaims: string | string[], options?: JudgeOptions): Promise<Verdict>;
+    /**
+     * Observational judge: records the verdict for score + insights but does
+     * NOT gate the test (a failing observe verdict never fails the run).
+     */
+    observe(resultOrTrajectory: ResultLike | TrajectoryStep[], claimOrClaims: string | string[], options?: JudgeOptions): Promise<Verdict>;
+}
+/**
+ * Single-claim ergonomic form (gate role).
+ * @example
+ *   const v = await judge(result, 'identifies the failing dependency');
+ *   if (!v.pass) { ... }                       // non-throwing
+ *   (await judge(result, claim)).orThrow();      // explicit hard-stop
+ *   await judge.observe(result, claim);          // score-only, never gates
+ */
+export declare const judge: JudgeFn;
+/**
+ * Bind run-level defaults to `judge` and return a callable with the same
+ * signature. Used by the SDK runner to inject `run.evaluatorId` (and
+ * optionally a default judge model) into the `TestFixtures.judge` slot,
+ * so test bodies that destructure `({ judge })` automatically inherit
+ * the run's evaluator selection — matching the UI's behaviour where the
+ * evaluator picked on the run config applies to every judged test case.
+ *
+ * Per-call options always win over the bound defaults; pass an empty
+ * object (or omit the field) to fall through to the bound value.
+ *
+ *   const boundJudge = bindJudge({ evaluatorId: run.evaluatorId });
+ *   await boundJudge(result, claim);                                  // uses run.evaluatorId
+ *   await boundJudge(result, claim, { evaluatorId: 'other' });        // overrides
+ *   await boundJudge(result, claim, { evaluatorId: undefined });      // still uses bound default
+ */
+export declare function bindJudge(defaults?: {
+    evaluatorId?: string;
+    model?: string;
+    serverUrl?: string;
+    skip?: boolean;
+}): JudgeFn;
+export {};
+//# sourceMappingURL=judge.d.ts.map

package/lib/dist/lib/testCases/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../../testCases/judge.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAK9C,8EAA8E;AAC9E,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,SAAS,CAAC;AAuB3C,8EAA8E;AAC9E,wBAAgB,uBAAuB,IAAI,IAAI,CAE9C;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,OAAO;IACtB,cAAc,EAAE,QAAQ,GAAG,QAAQ,CAAC;IACpC,kDAAkD;IAClD,QAAQ,EAAE,MAAM,CAAC;IACjB,kEAAkE;IAClE,KAAK,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,IAAI,EAAE,OAAO,CAAC;IACd,iEAAiE;IACjE,IAAI,EAAE,SAAS,CAAC;IAChB,8DAA8D;IAC9D,OAAO,EAAE,OAAO,CAAC;IACjB,iEAAiE;IACjE,OAAO,EAAE,OAAO,CAAC;IACjB,uDAAuD;IACvD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,OAAO,IAAI,OAAO,CAAC;CACpB;AAED,uEAAuE;AACvE,MAAM,MAAM,YAAY,GAAG,OAAO,CAAC;AAEnC;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAeD,2EAA2E;AAC3E,wBAAgB,eAAe,IAAI,IAAI,CAEtC;AA0CD,wBAAgB,cAAc,IAAI,OAAO,CAExC;AAED,wBAAgB,cAAc,IAAI,IAAI,CAErC;AAED,UAAU,UAAU;IAClB,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAC9B,aAAa,CAAC,EAAE,MAAM,MAAM,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AA8BD;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,CACE,kBAAkB,EAAE,UAAU,GAAG,cAAc,EAAE,EACjD,aAAa,EAAE,MAAM,GAAG,MAAM,EAAE,EAChC,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB;;;OAGG;IACH,OAAO,CACL,kBAAkB,EAAE,UAAU,GAAG,cAAc,EAAE,EACjD,aAAa,EAAE,MAAM,GAAG,MAAM,EAAE,EAChC,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,OAAO,CAAC,CAAC;CACrB;AAyND;;;;;;;GAOG;AACH,eAAO,MAAM,KAAK,EAAE,OAanB,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,SAAS,CAAC,QAAQ,CAAC,EAAE;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB,GAAG,OAAO,CA+BV"}