@opensearch-project/agent-health 0.4.0 → 0.6.0

package/lib/dist/lib/testCases/judge.d.ts

@@ -0,0 +1,165 @@
+/**
+ * `judge()` — LLM-judge matcher, callable from inside a test body.
+ *
+ * Two ergonomic forms:
+ *
+ *   await judge(result, 'identifies the failing dependency');         // single claim
+ *   await judge(result.trajectory, ['claim 1', 'claim 2']);            // legacy form
+ *
+ * Per-call options:
+ *
+ *   await judge(result, claim, { evaluatorId: 'system-rca-default' });
+ *   await judge(result, claim, { model: 'claude-sonnet' });
+ *
+ * On evaluation, `judge()` records a MatcherResult and returns a
+ * {@link Verdict} **without throwing** (RFC 004 §4.8). A failing
+ * `gate`-role verdict still fails the test (the runner inspects the
+ * recorded results), but the body keeps running so every signal is
+ * collected. To hard-stop the body on a bad verdict, call
+ * `(await judge(...)).orThrow()` or assert `expect(verdict).toPass()`.
+ *
+ * Two roles:
+ *   - `judge(result, claim)`         — gate: a failing verdict fails the test.
+ *   - `judge.observe(result, claim)` — observe: feeds score + insights only,
+ *                                       never fails the test.
+ *
+ * Calls the Agent Health server's /api/judge endpoint with the same
+ * payload shape (`{ trajectory, expectedOutcomes, modelId, evaluatorId }`)
+ * the UI "Run Test" path uses, so SDK and UI runs are scored by the same
+ * judge prompt and provider routing.
+ *
+ * Run-level evaluator default: the runner injects a bound version of
+ * `judge` into `TestFixtures` via `bindJudge(run.evaluatorId)`. Code
+ * that destructures `judge` from the fixture (`async ({ judge }) => ...`)
+ * automatically picks up the run's evaluator with no per-call argument.
+ * Code that imports `judge` from the package gets the unbound version
+ * and must pass `evaluatorId` explicitly.
+ */
+import type { TrajectoryStep } from '../../types/index.js';
+/** Whether a judge signal gates the test verdict or is observational only. */
+export type JudgeRole = 'gate' | 'observe';
+/** Test-only: reset the warn-once latch so unit tests can assert it fires. */
+export declare function __resetJudgePortWarning(): void;
+/**
+ * The result of a `judge()` call. Carries the verdict data plus a few
+ * convenience flags and an explicit hard-stop escape hatch. Returned
+ * (never thrown) so the body can collect multiple verdicts; the recorded
+ * MatcherResult is what actually gates the test.
+ *
+ * Backward-compatible with the old `JudgeVerdict` (same
+ * `passFailStatus`/`accuracy`/`reasoning` fields).
+ */
+export interface Verdict {
+    passFailStatus: 'passed' | 'failed';
+    /** Headline accuracy on the [0, 100] interval. */
+    accuracy: number;
+    /** Normalised score on the [0, 1] interval (`accuracy / 100`). */
+    score: number;
+    /** Free-form judge reasoning. */
+    reasoning: string;
+    /** Convenience: `passFailStatus === 'passed'`. */
+    pass: boolean;
+    /** Role this verdict was produced with (`gate` or `observe`). */
+    role: JudgeRole;
+    /** True when the judge was skipped (no LLM call was made). */
+    skipped: boolean;
+    /** True when the judge could not run at all (endpoint error). */
+    errored: boolean;
+    /** Underlying error message when `errored` is true. */
+    errorMessage?: string;
+    /**
+     * Hard-stop: throw if the verdict did not pass. A no-op for passing,
+     * skipped verdicts. Returns the verdict so it can be chained:
+     *   const v = (await judge(result, claim)).orThrow();
+     */
+    orThrow(): Verdict;
+}
+/** @deprecated Use {@link Verdict}. Retained as a structural alias. */
+export type JudgeVerdict = Verdict;
+/**
+ * Per-call options for the SDK `judge()` matcher. Mirrors the relevant
+ * fields of the `/api/judge` request body so SDK runs and UI runs use
+ * the same judge prompt + evaluator + provider routing.
+ */
+export interface JudgeOptions {
+    /**
+     * Override the agent-health server URL. Defaults to
+     * `http://localhost:${AGENT_HEALTH_PORT ?? 4001}`.
+     */
+    serverUrl?: string;
+    /**
+     * Override the judge model. Forwarded as `modelId` on the request body;
+     * the server resolves it through `config.models[modelId]` exactly as it
+     * does for UI runs.
+     */
+    model?: string;
+    /**
+     * Stored evaluator id (system or user). Forwarded as `evaluatorId` on
+     * the request body. The server resolves system evaluators via
+     * `getSystemEvaluatorById(...)` and user evaluators via
+     * `storage.evaluators.getById(...)` — identical to the UI path. Falls
+     * back to the default evaluator when omitted on both call and bind.
+     */
+    evaluatorId?: string;
+    /**
+     * Skip the LLM call entirely. The judge returns a non-gating `skipped`
+     * verdict (pass: true, skipped: true). Useful for fast/offline iteration.
+     * The global `AH_SKIP_JUDGE=1` env var forces this for every judge call.
+     */
+    skip?: boolean;
+}
+/** Clear the in-process judge verdict cache (test isolation / new run). */
+export declare function clearJudgeCache(): void;
+export declare function wasJudgeCalled(): boolean;
+export declare function resetJudgeFlag(): void;
+interface ResultLike {
+    trajectory?: TrajectoryStep[];
+    finalResponse?: () => string;
+    agentOutput?: string;
+    runId?: string;
+}
+/**
+ * The callable `judge` surface: a function (gate role) with an `.observe`
+ * method (observe role).
+ */
+export interface JudgeFn {
+    (resultOrTrajectory: ResultLike | TrajectoryStep[], claimOrClaims: string | string[], options?: JudgeOptions): Promise<Verdict>;
+    /**
+     * Observational judge: records the verdict for score + insights but does
+     * NOT gate the test (a failing observe verdict never fails the run).
+     */
+    observe(resultOrTrajectory: ResultLike | TrajectoryStep[], claimOrClaims: string | string[], options?: JudgeOptions): Promise<Verdict>;
+}
+/**
+ * Single-claim ergonomic form (gate role).
+ * @example
+ *   const v = await judge(result, 'identifies the failing dependency');
+ *   if (!v.pass) { ... }                       // non-throwing
+ *   (await judge(result, claim)).orThrow();      // explicit hard-stop
+ *   await judge.observe(result, claim);          // score-only, never gates
+ */
+export declare const judge: JudgeFn;
+/**
+ * Bind run-level defaults to `judge` and return a callable with the same
+ * signature. Used by the SDK runner to inject `run.evaluatorId` (and
+ * optionally a default judge model) into the `TestFixtures.judge` slot,
+ * so test bodies that destructure `({ judge })` automatically inherit
+ * the run's evaluator selection — matching the UI's behaviour where the
+ * evaluator picked on the run config applies to every judged test case.
+ *
+ * Per-call options always win over the bound defaults; pass an empty
+ * object (or omit the field) to fall through to the bound value.
+ *
+ *   const boundJudge = bindJudge({ evaluatorId: run.evaluatorId });
+ *   await boundJudge(result, claim);                                  // uses run.evaluatorId
+ *   await boundJudge(result, claim, { evaluatorId: 'other' });        // overrides
+ *   await boundJudge(result, claim, { evaluatorId: undefined });      // still uses bound default
+ */
+export declare function bindJudge(defaults?: {
+    evaluatorId?: string;
+    model?: string;
+    serverUrl?: string;
+    skip?: boolean;
+}): JudgeFn;
+export {};
+//# sourceMappingURL=judge.d.ts.map

package/lib/dist/lib/testCases/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../../testCases/judge.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAK9C,8EAA8E;AAC9E,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,SAAS,CAAC;AAuB3C,8EAA8E;AAC9E,wBAAgB,uBAAuB,IAAI,IAAI,CAE9C;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,OAAO;IACtB,cAAc,EAAE,QAAQ,GAAG,QAAQ,CAAC;IACpC,kDAAkD;IAClD,QAAQ,EAAE,MAAM,CAAC;IACjB,kEAAkE;IAClE,KAAK,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,IAAI,EAAE,OAAO,CAAC;IACd,iEAAiE;IACjE,IAAI,EAAE,SAAS,CAAC;IAChB,8DAA8D;IAC9D,OAAO,EAAE,OAAO,CAAC;IACjB,iEAAiE;IACjE,OAAO,EAAE,OAAO,CAAC;IACjB,uDAAuD;IACvD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,OAAO,IAAI,OAAO,CAAC;CACpB;AAED,uEAAuE;AACvE,MAAM,MAAM,YAAY,GAAG,OAAO,CAAC;AAEnC;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAeD,2EAA2E;AAC3E,wBAAgB,eAAe,IAAI,IAAI,CAEtC;AA0CD,wBAAgB,cAAc,IAAI,OAAO,CAExC;AAED,wBAAgB,cAAc,IAAI,IAAI,CAErC;AAED,UAAU,UAAU;IAClB,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAC9B,aAAa,CAAC,EAAE,MAAM,MAAM,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AA8BD;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,CACE,kBAAkB,EAAE,UAAU,GAAG,cAAc,EAAE,EACjD,aAAa,EAAE,MAAM,GAAG,MAAM,EAAE,EAChC,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB;;;OAGG;IACH,OAAO,CACL,kBAAkB,EAAE,UAAU,GAAG,cAAc,EAAE,EACjD,aAAa,EAAE,MAAM,GAAG,MAAM,EAAE,EAChC,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,OAAO,CAAC,CAAC;CACrB;AAyND;;;;;;;GAOG;AACH,eAAO,MAAM,KAAK,EAAE,OAanB,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,SAAS,CAAC,QAAQ,CAAC,EAAE;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB,GAAG,OAAO,CA+BV"}

package/lib/dist/lib/testCases/judge.js

@@ -0,0 +1,359 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { recordVerdict } from '../matchers/session.js';
+import { readEnv } from '../envCompat.js';
+import { getBackendUrl, isBackendPortExplicit, DEFAULT_BACKEND_PORT } from '../portConfig.js';
+// Warn at most once per process when judge() falls back to the default
+// backend port. Defaulting to 4001 is dangerous when a *foreign* agent-health
+// server (a live demo, another checkout) is listening there — the judge call
+// would hit it instead of the run's own server. We warn rather than throw so
+// existing zero-config local usage (server actually on 4001) keeps working.
+let warnedDefaultJudgePort = false;
+function resolveJudgeServerUrl(explicit) {
+    if (explicit)
+        return explicit;
+    if (!isBackendPortExplicit() && !warnedDefaultJudgePort) {
+        warnedDefaultJudgePort = true;
+        // eslint-disable-next-line no-console
+        console.warn(`[agent-health] judge(): AH_PORT is not set and no options.serverUrl was ` +
+            `provided — defaulting to http://localhost:${DEFAULT_BACKEND_PORT}. If a ` +
+            `different agent-health server is on that port, judging will hit it. Set ` +
+            `AH_PORT or pass options.serverUrl to target the right server.`);
+    }
+    return getBackendUrl();
+}
+/** Test-only: reset the warn-once latch so unit tests can assert it fires. */
+export function __resetJudgePortWarning() {
+    warnedDefaultJudgePort = false;
+}
+const verdictCache = new Map();
+/** Clear the in-process judge verdict cache (test isolation / new run). */
+export function clearJudgeCache() {
+    verdictCache.clear();
+}
+/**
+ * Stable, collision-resistant cache key over the judging inputs. Uses
+ * `sha256` when Node's crypto is available, falling back to a fast JS hash
+ * so the SDK stays bundler-safe.
+ */
+function judgeCacheKey(input) {
+    const canonical = JSON.stringify({
+        evaluatorId: input.evaluatorId ?? '',
+        model: input.model ?? '',
+        claims: input.claims,
+        trajectory: input.trajectory,
+    });
+    try {
+        // eslint-disable-next-line @typescript-eslint/no-require-imports
+        const { createHash } = require('crypto');
+        return createHash('sha256').update(canonical).digest('hex');
+    }
+    catch {
+        // FNV-1a fallback — non-cryptographic but fine for cache addressing.
+        let h = 0x811c9dc5;
+        for (let i = 0; i < canonical.length; i++) {
+            h ^= canonical.charCodeAt(i);
+            h = Math.imul(h, 0x01000193);
+        }
+        return (h >>> 0).toString(16);
+    }
+}
+/** Whether judging is globally skipped via env (CLI `--no-judge`). */
+function judgeSkippedByEnv() {
+    const v = readEnv('AH_SKIP_JUDGE', 'AGENT_HEALTH_SKIP_JUDGE');
+    return v === '1' || v === 'true' || v === 'yes';
+}
+let judgeCalledInCurrentEval = false;
+export function wasJudgeCalled() {
+    return judgeCalledInCurrentEval;
+}
+export function resetJudgeFlag() {
+    judgeCalledInCurrentEval = false;
+}
+function isTrajectory(x) {
+    return Array.isArray(x);
+}
+function isResultLike(x) {
+    return typeof x === 'object' && x !== null && 'trajectory' in x;
+}
+/**
+ * Build a {@link Verdict} object with the `orThrow()` escape hatch attached.
+ */
+function makeVerdict(v) {
+    const verdict = {
+        ...v,
+        pass: v.passFailStatus === 'passed',
+        score: v.score ?? (typeof v.accuracy === 'number' ? v.accuracy / 100 : 0),
+        orThrow() {
+            // Passing or skipped verdicts never throw. Failed/errored ones do.
+            if (this.pass || this.skipped)
+                return this;
+            const label = this.errored ? 'errored' : 'FAILED';
+            throw new Error(`LLM Judge: ${label} (accuracy: ${this.accuracy})\n${this.errorMessage ?? this.reasoning}`);
+        },
+    };
+    return verdict;
+}
+/**
+ * Core judge implementation. Records exactly one MatcherResult (carrying
+ * `role`) and returns a non-throwing {@link Verdict}.
+ */
+async function runJudge(resultOrTrajectory, claimOrClaims, options, role) {
+    judgeCalledInCurrentEval = true;
+    const trajectory = isTrajectory(resultOrTrajectory)
+        ? resultOrTrajectory
+        : (isResultLike(resultOrTrajectory) ? resultOrTrajectory.trajectory ?? [] : []);
+    // Agent run id for trace/log correlation — lets the agentic trace judge
+    // scope its read-only tools to this single run. Only present when the
+    // caller passed a RunResult (not a bare trajectory array).
+    const runId = !isTrajectory(resultOrTrajectory) && isResultLike(resultOrTrajectory)
+        ? resultOrTrajectory.runId
+        : undefined;
+    // Strategy-C trace-correlation hints the runner attached to the result
+    // (service-name + window). Forwarding them as `agents` lets the agent
+    // (trace) judge find this run's spans in OpenSearch the same way the
+    // classic `waitForTracesAndJudge` path does (#264) — without them the SDK
+    // judge sends only `runId` and misses subprocess agents' spans. This is
+    // the SDK↔UI convergence fix.
+    const judgeAgents = !isTrajectory(resultOrTrajectory) && isResultLike(resultOrTrajectory)
+        ? resultOrTrajectory.judgeAgents
+        : undefined;
+    const claims = Array.isArray(claimOrClaims) ? claimOrClaims : [claimOrClaims];
+    const serverUrl = resolveJudgeServerUrl(options?.serverUrl);
+    const description = claims.length === 1 ? `judge: ${claims[0]}` : `judge: ${claims.length} claims`;
+    // Body shape matches the UI's /api/judge POST exactly: trajectory,
+    // expectedOutcomes, expectedTrajectory, optional modelId, optional
+    // evaluatorId. The server applies the same evaluator-resolution and
+    // provider-routing logic regardless of caller, so SDK and UI runs
+    // produce comparable verdicts.
+    const requestBody = {
+        trajectory,
+        expectedOutcomes: claims,
+        expectedTrajectory: [],
+    };
+    if (options?.model)
+        requestBody.modelId = options.model;
+    if (options?.evaluatorId)
+        requestBody.evaluatorId = options.evaluatorId;
+    if (runId)
+        requestBody.runId = runId;
+    if (Array.isArray(judgeAgents) && judgeAgents.length > 0)
+        requestBody.agents = judgeAgents;
+    // Record the success path identically whether the verdict came fresh from
+    // the endpoint or from the in-process cache.
+    const finalize = (raw, durationMs) => {
+        const accuracy = raw.metrics?.accuracy ?? 0;
+        const verdict = makeVerdict({
+            passFailStatus: raw.passFailStatus ?? 'failed',
+            accuracy,
+            reasoning: raw.llmJudgeReasoning ?? '',
+            role,
+            skipped: false,
+            errored: false,
+        });
+        recordVerdict({
+            description,
+            pass: verdict.pass,
+            method: 'llm-judge',
+            role,
+            durationMs,
+            score: verdict.score,
+            reasoning: verdict.reasoning,
+            model: options?.model,
+            errorMessage: verdict.passFailStatus === 'failed' ? verdict.reasoning : undefined,
+            // Preserve the rest of the judge payload — these were silently
+            // dropped before, which made SDK `judge()` calls strictly less
+            // informative than the legacy auto-judge path. See MatcherResult.
+            ...(Array.isArray(raw.improvementStrategies) && raw.improvementStrategies.length > 0
+                ? { improvementStrategies: raw.improvementStrategies }
+                : {}),
+            ...(raw.metrics && typeof raw.metrics === 'object'
+                ? { judgeMetrics: { ...raw.metrics } }
+                : {}),
+        });
+        return verdict;
+    };
+    // SKIP: no LLM call. `skip` is tri-state and takes precedence over the env:
+    //   skip === true   → always skip
+    //   skip === false  → never skip (force the judge to run, even if AH_SKIP_JUDGE is set)
+    //   skip undefined  → defer to AH_SKIP_JUDGE
+    // This makes a per-call/bound `skip: false` a meaningful override rather than
+    // having the env unconditionally win.
+    const shouldSkip = options?.skip === true || (options?.skip !== false && judgeSkippedByEnv());
+    if (shouldSkip) {
+        recordVerdict({
+            description: `${description} (skipped)`,
+            pass: true,
+            method: 'llm-judge',
+            role: 'observe',
+            durationMs: 0,
+            reasoning: 'Judge skipped (AH_SKIP_JUDGE / skip option).',
+        });
+        return makeVerdict({
+            passFailStatus: 'passed',
+            accuracy: 0,
+            reasoning: 'Judge skipped.',
+            role,
+            skipped: true,
+            errored: false,
+        });
+    }
+    // CACHE: identical (evaluator, model, claims, trajectory) → reuse verdict.
+    const cacheKey = judgeCacheKey({
+        evaluatorId: options?.evaluatorId,
+        model: options?.model,
+        claims,
+        trajectory,
+    });
+    const cached = verdictCache.get(cacheKey);
+    if (cached) {
+        return finalize(cached, 0);
+    }
+    const startedAt = Date.now();
+    // Resilience: the agentic (trace) judge can be slow and, under load (in-process
+    // pi sessions + query_spans/query_logs round-trips on the same server), the
+    // loopback fetch to /api/judge can transiently throw "fetch failed" (undici
+    // keep-alive socket reset / ~300s headersTimeout) or return a 5xx/429. Without
+    // a retry, ONE such transient blip records an `errored` matcher — which makes
+    // the runner clear `passFailStatus` to null and bucket the whole run as
+    // "errored" (#247) even when every other gate passed. Retry transient failures
+    // with exponential backoff before giving up. Deterministic 4xx (not 429) are
+    // not retried.
+    const MAX_JUDGE_ATTEMPTS = 3;
+    let response;
+    let lastErrMsg = '';
+    for (let attempt = 1; attempt <= MAX_JUDGE_ATTEMPTS; attempt++) {
+        try {
+            const r = await fetch(`${serverUrl}/api/judge`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify(requestBody),
+            });
+            if (r.ok) {
+                response = r;
+                break;
+            }
+            lastErrMsg = `Judge HTTP ${r.status}: ${await r.text()}`;
+            // 4xx (except 429) are deterministic — retrying won't help; surface as errored.
+            if (r.status < 500 && r.status !== 429) {
+                response = r;
+                break;
+            }
+        }
+        catch (err) {
+            lastErrMsg = err?.message || String(err); // e.g. "fetch failed"
+        }
+        if (attempt < MAX_JUDGE_ATTEMPTS) {
+            // Exponential backoff (1s, 2s). Overridable via AH_JUDGE_RETRY_BACKOFF_MS
+            // (set to 0 in tests to avoid real delays); 0 disables the wait.
+            const baseMs = process.env.AH_JUDGE_RETRY_BACKOFF_MS !== undefined
+                ? Number(process.env.AH_JUDGE_RETRY_BACKOFF_MS)
+                : 1000;
+            if (baseMs > 0)
+                await new Promise((res) => setTimeout(res, baseMs * 2 ** (attempt - 1)));
+        }
+    }
+    if (!response) {
+        // Every attempt failed transiently — the judge could not run. `errored`,
+        // not a clean `pass: false`.
+        recordVerdict({
+            description,
+            pass: false,
+            method: 'llm-judge',
+            role,
+            errored: true,
+            durationMs: Date.now() - startedAt,
+            errorMessage: `Judge request failed after ${MAX_JUDGE_ATTEMPTS} attempts: ${lastErrMsg}`,
+            reasoning: '',
+        });
+        return makeVerdict({
+            passFailStatus: 'failed',
+            accuracy: 0,
+            reasoning: '',
+            role,
+            skipped: false,
+            errored: true,
+            errorMessage: `Judge request failed after ${MAX_JUDGE_ATTEMPTS} attempts: ${lastErrMsg}`,
+        });
+    }
+    if (!response.ok) {
+        recordVerdict({
+            description,
+            pass: false,
+            method: 'llm-judge',
+            role,
+            errored: true,
+            durationMs: Date.now() - startedAt,
+            errorMessage: lastErrMsg,
+            reasoning: '',
+        });
+        return makeVerdict({
+            passFailStatus: 'failed',
+            accuracy: 0,
+            reasoning: '',
+            role,
+            skipped: false,
+            errored: true,
+            errorMessage: lastErrMsg,
+        });
+    }
+    const raw = (await response.json());
+    // Only successful, non-errored verdicts are cached.
+    verdictCache.set(cacheKey, raw);
+    return finalize(raw, Date.now() - startedAt);
+}
+/**
+ * Single-claim ergonomic form (gate role).
+ * @example
+ *   const v = await judge(result, 'identifies the failing dependency');
+ *   if (!v.pass) { ... }                       // non-throwing
+ *   (await judge(result, claim)).orThrow();      // explicit hard-stop
+ *   await judge.observe(result, claim);          // score-only, never gates
+ */
+export const judge = Object.assign((resultOrTrajectory, claimOrClaims, options) => runJudge(resultOrTrajectory, claimOrClaims, options, 'gate'), {
+    observe: (resultOrTrajectory, claimOrClaims, options) => runJudge(resultOrTrajectory, claimOrClaims, options, 'observe'),
+});
+/**
+ * Bind run-level defaults to `judge` and return a callable with the same
+ * signature. Used by the SDK runner to inject `run.evaluatorId` (and
+ * optionally a default judge model) into the `TestFixtures.judge` slot,
+ * so test bodies that destructure `({ judge })` automatically inherit
+ * the run's evaluator selection — matching the UI's behaviour where the
+ * evaluator picked on the run config applies to every judged test case.
+ *
+ * Per-call options always win over the bound defaults; pass an empty
+ * object (or omit the field) to fall through to the bound value.
+ *
+ *   const boundJudge = bindJudge({ evaluatorId: run.evaluatorId });
+ *   await boundJudge(result, claim);                                  // uses run.evaluatorId
+ *   await boundJudge(result, claim, { evaluatorId: 'other' });        // overrides
+ *   await boundJudge(result, claim, { evaluatorId: undefined });      // still uses bound default
+ */
+export function bindJudge(defaults) {
+    // No defaults set → return the unbound function unchanged. Keeps zero
+    // overhead for tests that don't use a run-level evaluator. Note `skip` is
+    // compared against `undefined` (not falsiness) so a binding of
+    // `{ skip: false }` — a meaningful "force the judge to run" — is preserved
+    // rather than short-circuited to the unbound judge.
+    if (!defaults ||
+        (!defaults.evaluatorId && !defaults.model && !defaults.serverUrl && defaults.skip === undefined)) {
+        return judge;
+    }
+    const mergeOptions = (options) => ({
+        // Per-call options win on every field that's actually set. We treat
+        // an explicit `undefined` the same as a missing field — callers who
+        // want to *clear* a bound default should pass an empty string or
+        // call the unbound `judge` directly.
+        serverUrl: options?.serverUrl ?? defaults.serverUrl,
+        model: options?.model ?? defaults.model,
+        evaluatorId: options?.evaluatorId ?? defaults.evaluatorId,
+        skip: options?.skip ?? defaults.skip,
+    });
+    const bound = Object.assign((resultOrTrajectory, claimOrClaims, options) => runJudge(resultOrTrajectory, claimOrClaims, mergeOptions(options), 'gate'), {
+        observe: (resultOrTrajectory, claimOrClaims, options) => runJudge(resultOrTrajectory, claimOrClaims, mergeOptions(options), 'observe'),
+    });
+    return bound;
+}
+//# sourceMappingURL=judge.js.map

package/lib/dist/lib/testCases/judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../../testCases/judge.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAyCH,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAK9F,uEAAuE;AACvE,8EAA8E;AAC9E,6EAA6E;AAC7E,6EAA6E;AAC7E,4EAA4E;AAC5E,IAAI,sBAAsB,GAAG,KAAK,CAAC;AACnC,SAAS,qBAAqB,CAAC,QAAiB;IAC9C,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9B,IAAI,CAAC,qBAAqB,EAAE,IAAI,CAAC,sBAAsB,EAAE,CAAC;QACxD,sBAAsB,GAAG,IAAI,CAAC;QAC9B,sCAAsC;QACtC,OAAO,CAAC,IAAI,CACV,0EAA0E;YACxE,6CAA6C,oBAAoB,SAAS;YAC1E,0EAA0E;YAC1E,+DAA+D,CAClE,CAAC;IACJ,CAAC;IACD,OAAO,aAAa,EAAE,CAAC;AACzB,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,uBAAuB;IACrC,sBAAsB,GAAG,KAAK,CAAC;AACjC,CAAC;AAoFD,MAAM,YAAY,GAAG,IAAI,GAAG,EAA4B,CAAC;AAEzD,2EAA2E;AAC3E,MAAM,UAAU,eAAe;IAC7B,YAAY,CAAC,KAAK,EAAE,CAAC;AACvB,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,KAKtB;IACC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAC/B,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,EAAE;QACpC,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,KAAK,CAAC,UAAU;KAC7B,CAAC,CAAC;IACH,IAAI,CAAC;QACH,iEAAiE;QACjE,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;IAAC,MAAM,CAAC;QACP,qEAAqE;QACrE,IAAI,CAAC,GAAG,UAAU,CAAC;QACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,CAAC,IAAI,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED,sEAAsE;AACtE,SAAS,iBAAiB;IACxB,MAAM,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,yBAAyB,CAAC,CAAC;IAC9D,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,CAAC;AAClD,CAAC;AAED,IAAI,wBAAwB,GAAG,KAAK,CAAC;AAErC,MAAM,UAAU,cAAc;IAC5B,OAAO,wBAAwB,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,wBAAwB,GAAG,KAAK,CAAC;AACnC,CAAC;AASD,SAAS,YAAY,CAAC,CAAU;IAC9B,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,YAAY,CAAC,CAAU;IAC9B,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,YAAY,IAAK,CAAY,CAAC;AAC9E,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,CAAmE;IACtF,MAAM,OAAO,GAAY;QACvB,GAAG,CAAC;QACJ,IAAI,EAAE,CAAC,CAAC,cAAc,KAAK,QAAQ;QACnC,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACzE,OAAO;YACL,mEAAmE;YACnE,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAO,IAAI,CAAC;YAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;YAClD,MAAM,IAAI,KAAK,CACb,cAAc,KAAK,eAAe,IAAI,CAAC,QAAQ,MAAM,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,SAAS,EAAE,CAC3F,CAAC;QACJ,CAAC;KACF,CAAC;IACF,OAAO,OAAO,CAAC;AACjB,CAAC;AAuBD;;;GAGG;AACH,KAAK,UAAU,QAAQ,CACrB,kBAAiD,EACjD,aAAgC,EAChC,OAAiC,EACjC,IAAe;IAEf,wBAAwB,GAAG,IAAI,CAAC;IAEhC,MAAM,UAAU,GAAG,YAAY,CAAC,kBAAkB,CAAC;QACjD,CAAC,CAAC,kBAAkB;QACpB,CAAC,CAAC,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAClF,wEAAwE;IACxE,sEAAsE;IACtE,2DAA2D;IAC3D,MAAM,KAAK,GAAG,CAAC,YAAY,CAAC,kBAAkB,CAAC,IAAI,YAAY,CAAC,kBAAkB,CAAC;QACjF,CAAC,CAAC,kBAAkB,CAAC,KAAK;QAC1B,CAAC,CAAC,SAAS,CAAC;IACd,uEAAuE;IACvE,sEAAsE;IACtE,qEAAqE;IACrE,0EAA0E;IAC1E,wEAAwE;IACxE,8BAA8B;IAC9B,MAAM,WAAW,GAAG,CAAC,YAAY,CAAC,kBAAkB,CAAC,IAAI,YAAY,CAAC,kBAAkB,CAAC;QACvF,CAAC,CAAE,kBAAgD,CAAC,WAAW;QAC/D,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;IAE9E,MAAM,SAAS,GAAG,qBAAqB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAE5D,MAAM,WAAW,GACf,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,MAAM,SAAS,CAAC;IAEjF,mEAAmE;IACnE,mEAAmE;IACnE,oEAAoE;IACpE,kEAAkE;IAClE,+BAA+B;IAC/B,MAAM,WAAW,GAA4B;QAC3C,UAAU;QACV,gBAAgB,EAAE,MAAM;QACxB,kBAAkB,EAAE,EAAE;KACvB,CAAC;IACF,IAAI,OAAO,EAAE,KAAK;QAAE,WAAW,CAAC,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC;IACxD,IAAI,OAAO,EAAE,WAAW;QAAE,WAAW,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IACxE,IAAI,KAAK;QAAE,WAAW,CAAC,KAAK,GAAG,KAAK,CAAC;IACrC,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;QAAE,WAAW,CAAC,MAAM,GAAG,WAAW,CAAC;IAE3F,0EAA0E;IAC1E,6CAA6C;IAC7C,MAAM,QAAQ,GAAG,CAAC,GAAqB,EAAE,UAAkB,EAAW,EAAE;QACtE,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,EAAE,QAAQ,IAAI,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAG,WAAW,CAAC;YAC1B,cAAc,EAAE,GAAG,CAAC,cAAc,IAAI,QAAQ;YAC9C,QAAQ;YACR,SAAS,EAAE,GAAG,CAAC,iBAAiB,IAAI,EAAE;YACtC,IAAI;YACJ,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;QACH,aAAa,CAAC;YACZ,WAAW;YACX,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,WAAW;YACnB,IAAI;YACJ,UAAU;YACV,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,KAAK,EAAE,OAAO,EAAE,KAAK;YACrB,YAAY,EAAE,OAAO,CAAC,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;YACjF,+DAA+D;YAC/D,+DAA+D;YAC/D,kEAAkE;YAClE,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,GAAG,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC;gBAClF,CAAC,CAAC,EAAE,qBAAqB,EAAE,GAAG,CAAC,qBAA4B,EAAE;gBAC7D,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,GAAG,CAAC,OAAO,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;gBAChD,CAAC,CAAC,EAAE,YAAY,EAAE,EAAE,GAAG,GAAG,CAAC,OAAO,EAAE,EAAE;gBACtC,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;IAEF,4EAA4E;IAC5E,kCAAkC;IAClC,wFAAwF;IACxF,6CAA6C;IAC7C,8EAA8E;IAC9E,sCAAsC;IACtC,MAAM,UAAU,GACd,OAAO,EAAE,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,KAAK,IAAI,iBAAiB,EAAE,CAAC,CAAC;IAC7E,IAAI,UAAU,EAAE,CAAC;QACf,aAAa,CAAC;YACZ,WAAW,EAAE,GAAG,WAAW,YAAY;YACvC,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,WAAW;YACnB,IAAI,EAAE,SAAS;YACf,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,8CAA8C;SAC1D,CAAC,CAAC;QACH,OAAO,WAAW,CAAC;YACjB,cAAc,EAAE,QAAQ;YACxB,QAAQ,EAAE,CAAC;YACX,SAAS,EAAE,gBAAgB;YAC3B,IAAI;YACJ,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;IACL,CAAC;IAED,2EAA2E;IAC3E,MAAM,QAAQ,GAAG,aAAa,CAAC;QAC7B,WAAW,EAAE,OAAO,EAAE,WAAW;QACjC,KAAK,EAAE,OAAO,EAAE,KAAK;QACrB,MAAM;QACN,UAAU;KACX,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC1C,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,gFAAgF;IAChF,4EAA4E;IAC5E,4EAA4E;IAC5E,+EAA+E;IAC/E,8EAA8E;IAC9E,wEAAwE;IACxE,+EAA+E;IAC/E,6EAA6E;IAC7E,eAAe;IACf,MAAM,kBAAkB,GAAG,CAAC,CAAC;IAC7B,IAAI,QAA8B,CAAC;IACnC,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,kBAAkB,EAAE,OAAO,EAAE,EAAE,CAAC;QAC/D,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,GAAG,SAAS,YAAY,EAAE;gBAC9C,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;aAClC,CAAC,CAAC;YACH,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;gBAAC,QAAQ,GAAG,CAAC,CAAC;gBAAC,MAAM;YAAC,CAAC;YAClC,UAAU,GAAG,cAAc,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YACzD,gFAAgF;YAChF,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAAC,QAAQ,GAAG,CAAC,CAAC;gBAAC,MAAM;YAAC,CAAC;QAClE,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,UAAU,GAAG,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,sBAAsB;QAClE,CAAC;QACD,IAAI,OAAO,GAAG,kBAAkB,EAAE,CAAC;YACjC,0EAA0E;YAC1E,iEAAiE;YACjE,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,yBAAyB,KAAK,SAAS;gBAChE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC;gBAC/C,CAAC,CAAC,IAAI,CAAC;YACT,IAAI,MAAM,GAAG,CAAC;gBAAE,MAAM,IAAI,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;IAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,yEAAyE;QACzE,6BAA6B;QAC7B,aAAa,CAAC;YACZ,WAAW;YACX,IAAI,EAAE,KAAK;YACX,MAAM,EAAE,WAAW;YACnB,IAAI;YACJ,OAAO,EAAE,IAAI;YACb,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,YAAY,EAAE,8BAA8B,kBAAkB,cAAc,UAAU,EAAE;YACxF,SAAS,EAAE,EAAE;SACd,CAAC,CAAC;QACH,OAAO,WAAW,CAAC;YACjB,cAAc,EAAE,QAAQ;YACxB,QAAQ,EAAE,CAAC;YACX,SAAS,EAAE,EAAE;YACb,IAAI;YACJ,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,IAAI;YACb,YAAY,EAAE,8BAA8B,kBAAkB,cAAc,UAAU,EAAE;SACzF,CAAC,CAAC;IACL,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,aAAa,CAAC;YACZ,WAAW;YACX,IAAI,EAAE,KAAK;YACX,MAAM,EAAE,WAAW;YACnB,IAAI;YACJ,OAAO,EAAE,IAAI;YACb,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,YAAY,EAAE,UAAU;YACxB,SAAS,EAAE,EAAE;SACd,CAAC,CAAC;QACH,OAAO,WAAW,CAAC;YACjB,cAAc,EAAE,QAAQ;YACxB,QAAQ,EAAE,CAAC;YACX,SAAS,EAAE,EAAE;YACb,IAAI;YACJ,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,IAAI;YACb,YAAY,EAAE,UAAU;SACzB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAqB,CAAC;IACxD,oDAAoD;IACpD,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAChC,OAAO,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;AAC/C,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,KAAK,GAAY,MAAM,CAAC,MAAM,CACzC,CACE,kBAAiD,EACjD,aAAgC,EAChC,OAAsB,EACJ,EAAE,CAAC,QAAQ,CAAC,kBAAkB,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,CAAC,EACnF;IACE,OAAO,EAAE,CACP,kBAAiD,EACjD,aAAgC,EAChC,OAAsB,EACJ,EAAE,CAAC,QAAQ,CAAC,kBAAkB,EAAE,aAAa,EAAE,OAAO,EAAE,SAAS,CAAC;CACvF,CACF,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,SAAS,CAAC,QAKzB;IACC,sEAAsE;IACtE,0EAA0E;IAC1E,+DAA+D;IAC/D,2EAA2E;IAC3E,oDAAoD;IACpD,IACE,CAAC,QAAQ;QACT,CAAC,CAAC,QAAQ,CAAC,WAAW,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,EAChG,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,YAAY,GAAG,CAAC,OAAsB,EAAgB,EAAE,CAAC,CAAC;QAC9D,oEAAoE;QACpE,oEAAoE;QACpE,iEAAiE;QACjE,qCAAqC;QACrC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,QAAQ,CAAC,SAAS;QACnD,KAAK,EAAE,OAAO,EAAE,KAAK,IAAI,QAAQ,CAAC,KAAK;QACvC,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,QAAQ,CAAC,WAAW;QACzD,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,QAAQ,CAAC,IAAI;KACrC,CAAC,CAAC;IACH,MAAM,KAAK,GAAY,MAAM,CAAC,MAAM,CAClC,CAAC,kBAAiD,EAAE,aAAgC,EAAE,OAAsB,EAAE,EAAE,CAC9G,QAAQ,CAAC,kBAAkB,EAAE,aAAa,EAAE,YAAY,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,EAC5E;QACE,OAAO,EAAE,CAAC,kBAAiD,EAAE,aAAgC,EAAE,OAAsB,EAAE,EAAE,CACvH,QAAQ,CAAC,kBAAkB,EAAE,aAAa,EAAE,YAAY,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC;KAChF,CACF,CAAC;IACF,OAAO,KAAK,CAAC;AACf,CAAC"}

package/lib/dist/lib/testCases/loader.d.ts

@@ -0,0 +1,36 @@
+import type { CodeTestCase, RegisteredHook } from './types.js';
+export declare function isCodeFile(filename: string): boolean;
+export { detectSourceLanguage } from '../../lib/utils.js';
+export declare function computeTestCaseHash(tc: CodeTestCase, fileSource?: string): string;
+export interface LoadedTestCase extends CodeTestCase {
+    hash: string;
+}
+export interface LoadResult {
+    testCases: LoadedTestCase[];
+    filePath: string;
+    /**
+     * Benchmark groups derived from `describe()` blocks in the file.
+     * Maps benchmark name (the joined describe path, e.g. 'RCA Suite' or
+     * 'A > B' for nested) to the test case names within. Tests outside any
+     * `describe()` are not in this map; they go to the file-default benchmark
+     * (CLI / server names it after the filename).
+     */
+    benchmarks: Map<string, string[]>;
+    /**
+     * Raw text of the eval file, read once up front (before execution) for
+     * BOTH .js and .ts/.mjs paths. Persisted verbatim as `sourceCode` on each
+     * imported TestCase (see cli/commands/benchmark.ts) so the Test Case
+     * detail page can render the whole file as an IDE-style code view --
+     * without this we'd only have the parsed `initialPrompt`/`expectedOutcomes`
+     * fields, not the `evaluate()` body that produced them.
+     */
+    fileSource: string;
+    /**
+     * All lifecycle hooks (`beforeEach`/`afterEach`/`beforeAll`/`afterAll`)
+     * registered while loading this file. Empty when the file declares none.
+     * The orchestrator filters by `(sourceFile, describePath)` at run time.
+     */
+    hooks: RegisteredHook[];
+}
+export declare function loadTestCasesFromModule(filePath: string): Promise<LoadResult>;
+//# sourceMappingURL=loader.d.ts.map

package/lib/dist/lib/testCases/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../../testCases/loader.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAW/D,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAGpD;AAQD,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAEtD,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,YAAY,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAyBjF;AAED,MAAM,WAAW,cAAe,SAAQ,YAAY;IAClD,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAClC;;;;;;;OAOG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,KAAK,EAAE,cAAc,EAAE,CAAC;CACzB;AAED,wBAAsB,uBAAuB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAiJnF"}