@opensearch-project/agent-health 0.4.0 → 0.6.0

package/lib/dist/lib/testCases/define.js

@@ -0,0 +1,253 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { readEnv } from '../envCompat.js';
+const registries = new Map();
+// Hooks live in a parallel registry keyed by file path, exactly matching
+// the test() registry. The HookOrchestrator filters by describePath at
+// run time. Cross-file hooks never leak — the loader resets per file.
+const hookRegistries = new Map();
+let activeFile = null;
+const DEFAULT_KEY = '__default__';
+// Active describe stack (synchronous push/pop). Each entry is a describe
+// name; nested describes accumulate. When test() runs, it captures the
+// current stack and joins with ' > ' to derive the test's benchmarkPath.
+const describeStack = [];
+// Emit a one-time experimental-status warning the first time the SDK is
+// touched, unless the user opts out. This is intentionally noisy enough to be
+// noticed but only fires once per process so it doesn't pollute test output.
+let experimentalWarningEmitted = false;
+function emitExperimentalWarningOnce() {
+    if (experimentalWarningEmitted)
+        return;
+    experimentalWarningEmitted = true;
+    if (readEnv('AH_SUPPRESS_EXPERIMENTAL', 'AGENT_HEALTH_SUPPRESS_EXPERIMENTAL') === '1')
+        return;
+    // eslint-disable-next-line no-console
+    console.warn('[agent-health] The code-based test SDK (test()/judge()/expect()) is ' +
+        'experimental. The API may change in a minor release without a ' +
+        'deprecation cycle. Pin your @opensearch-project/agent-health version, ' +
+        'or set AH_SUPPRESS_EXPERIMENTAL=1 to silence this notice.');
+}
+/** @internal */
+export function _resetExperimentalWarning() {
+    experimentalWarningEmitted = false;
+}
+export function setActiveFile(filePath) {
+    activeFile = filePath;
+    if (!registries.has(filePath)) {
+        registries.set(filePath, []);
+    }
+    if (!hookRegistries.has(filePath)) {
+        hookRegistries.set(filePath, []);
+    }
+}
+export function test(name, optionsOrBody, maybeBody) {
+    emitExperimentalWarningOnce();
+    // Resolve the two-arg / three-arg overload
+    let options;
+    let evaluate;
+    if (typeof optionsOrBody === 'function') {
+        options = {};
+        evaluate = optionsOrBody;
+    }
+    else {
+        options = optionsOrBody ?? {};
+        evaluate = maybeBody;
+    }
+    if (!name || typeof name !== 'string') {
+        throw new Error('test() requires a name (the first argument)');
+    }
+    if (typeof evaluate !== 'function') {
+        throw new Error(`test("${name}") requires a body function`);
+    }
+    const key = activeFile ?? DEFAULT_KEY;
+    if (!registries.has(key)) {
+        registries.set(key, []);
+    }
+    const registry = registries.get(key);
+    // Within-file uniqueness guard: a name+benchmarkPath pair must be unique.
+    // The same name in two different describe blocks is allowed because they
+    // map to different benchmarks.
+    const benchmarkPath = describeStack.length > 0 ? describeStack.join(' > ') : undefined;
+    if (registry.some(t => t.name === name && t.benchmarkPath === benchmarkPath)) {
+        const fileLabel = activeFile ? ` in ${activeFile}` : '';
+        const groupLabel = benchmarkPath ? ` (in describe "${benchmarkPath}")` : '';
+        throw new Error(`Duplicate test name "${name}"${groupLabel}${fileLabel}. ` +
+            `Test names must be unique within their describe block. ` +
+            `Move one of the tests to a different describe() or rename it.`);
+    }
+    registry.push({
+        name,
+        options,
+        evaluate,
+        sourceFile: activeFile ?? undefined,
+        benchmarkPath,
+    });
+}
+/**
+ * Group tests under a benchmark name. Equivalent to Playwright's
+ * `describe()` — the wrapped `test()` calls inherit the describe's name as
+ * their benchmark group. Nested describes flatten with ' > '.
+ *
+ * @example
+ *   describe('RCA Suite', () => {
+ *     test('payment-service is the root cause', { prompt: ... }, async ({ result, judge }) => {
+ *       expect(result.trajectory).to.haveCalledTool('search_logs');
+ *       await judge(result, 'identifies the failing dependency');
+ *     });
+ *   });
+ *
+ * The describe body MUST be synchronous (no `await`/dynamic content), like
+ * Playwright. The function is invoked once at registration time.
+ */
+export function describe(name, fn) {
+    emitExperimentalWarningOnce();
+    if (!name || typeof name !== 'string') {
+        throw new Error('describe() requires a name (the first argument)');
+    }
+    if (typeof fn !== 'function') {
+        throw new Error(`describe("${name}") requires a body function`);
+    }
+    describeStack.push(name);
+    try {
+        const result = fn();
+        if (result && typeof result.then === 'function') {
+            // Mirror Playwright — describe bodies must be synchronous because
+            // they run during registration, well before any test executes.
+            throw new Error(`describe("${name}") body returned a Promise. ` +
+                `describe blocks must be synchronous — use test() inside, not await.`);
+        }
+    }
+    finally {
+        describeStack.pop();
+    }
+}
+export function getRegisteredTests(filePath) {
+    if (filePath)
+        return [...(registries.get(filePath) ?? [])];
+    return [...registries.values()].flatMap(r => [...r]);
+}
+/**
+ * Return all hooks registered for the given file (or every file when no
+ * argument is given). Returns a snapshot — mutating the result has no
+ * effect on the registry.
+ */
+export function getRegisteredHooks(filePath) {
+    if (filePath)
+        return [...(hookRegistries.get(filePath) ?? [])];
+    return [...hookRegistries.values()].flatMap(r => [...r]);
+}
+export function clearRegistry(filePath) {
+    if (filePath) {
+        registries.delete(filePath);
+        hookRegistries.delete(filePath);
+    }
+    else {
+        registries.clear();
+        hookRegistries.clear();
+    }
+    activeFile = null;
+}
+/**
+ * Internal helper used by the four public hook registrars below.
+ *
+ * Captures the live `(activeFile, [...describeStack])` so the orchestrator
+ * can filter hooks by scope at run time. Multiple hooks of the same kind
+ * in the same scope are allowed and run in registration order (reversed
+ * for `afterEach`/`afterAll`, mirroring Playwright/Jest).
+ */
+function registerHook(kind, fn) {
+    emitExperimentalWarningOnce();
+    if (typeof fn !== 'function') {
+        throw new Error(`${kind}() requires a function as its first argument`);
+    }
+    const key = activeFile ?? DEFAULT_KEY;
+    if (!hookRegistries.has(key)) {
+        hookRegistries.set(key, []);
+    }
+    const describePath = describeStack.length > 0 ? describeStack.join(' > ') : undefined;
+    hookRegistries.get(key).push({
+        kind,
+        fn,
+        sourceFile: activeFile ?? undefined,
+        describePath,
+    });
+}
+/**
+ * Register a hook that runs **once** before the first test in its scope.
+ *
+ * Scope is the surrounding `describe(...)` block, or the whole file when
+ * called at the top level. With parallel test execution (the runner
+ * dispatches up to `concurrency` tests at once), the orchestrator uses a
+ * once-latch so all parallel arrivals await the same `beforeAll` promise.
+ *
+ * @example
+ *   beforeAll(async () => {
+ *     await fs.mkdir('/tmp/agent-health-fixtures', { recursive: true });
+ *   });
+ */
+export function beforeAll(fn) { registerHook('beforeAll', fn); }
+/**
+ * Register a hook that runs **once** after the last test in its scope.
+ *
+ * Always runs, even when every test in the scope failed. The orchestrator
+ * uses a remaining-test counter (decremented on each test completion,
+ * regardless of pass/fail) and triggers `afterAll` when it hits zero.
+ *
+ * @example
+ *   afterAll(async () => {
+ *     await fs.rm('/tmp/agent-health-fixtures', { recursive: true });
+ *   });
+ */
+export function afterAll(fn) { registerHook('afterAll', fn); }
+/**
+ * Register a hook that runs **before each test** in its scope.
+ *
+ * Receives the same fixtures object the body will see, plus a
+ * `provide(key, value)` function for stashing values that the test body
+ * (and `afterEach`) can read via `fixtures.provisioned[key]`. Each test
+ * gets its own provisioned bag, so concurrent tests are isolated.
+ *
+ * @example
+ *   beforeEach(async ({ provide, testInfo }) => {
+ *     const dir = await fs.mkdtemp(`/tmp/${testInfo.name}-`);
+ *     provide('workspaceDir', dir);
+ *   });
+ */
+export function beforeEach(fn) { registerHook('beforeEach', fn); }
+/**
+ * Register a hook that runs **after each test** in its scope.
+ *
+ * Always runs, even when the test body or a `beforeEach` threw. Reads
+ * provisioned values via `fixtures.provisioned[key]` for cleanup. Errors
+ * thrown from `afterEach` are captured as MatcherResult entries on the
+ * test — they don't crash the runner.
+ *
+ * @example
+ *   afterEach(async ({ provisioned }) => {
+ *     if (provisioned.workspaceDir) {
+ *       await fs.rm(provisioned.workspaceDir as string, { recursive: true, force: true });
+ *     }
+ *   });
+ */
+export function afterEach(fn) { registerHook('afterEach', fn); }
+// Playwright-style sugar: `test.beforeEach(fn)` etc.
+//
+// Declaration merging here gives TypeScript users typed access to the four
+// hook registrars *and* installs them as properties on the runtime `test`
+// function value. Both `import { beforeEach }` and `test.beforeEach` route
+// to the same internal registerHook() call.
+// eslint-disable-next-line @typescript-eslint/no-namespace
+(function (test) {
+    // eslint-disable-next-line @typescript-eslint/no-shadow
+    test.beforeAll = (fn) => registerHook('beforeAll', fn);
+    // eslint-disable-next-line @typescript-eslint/no-shadow
+    test.afterAll = (fn) => registerHook('afterAll', fn);
+    // eslint-disable-next-line @typescript-eslint/no-shadow
+    test.beforeEach = (fn) => registerHook('beforeEach', fn);
+    // eslint-disable-next-line @typescript-eslint/no-shadow
+    test.afterEach = (fn) => registerHook('afterEach', fn);
+})(test || (test = {}));
+//# sourceMappingURL=define.js.map

package/lib/dist/lib/testCases/define.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"define.js","sourceRoot":"","sources":["../../../testCases/define.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAUH,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE1C,MAAM,UAAU,GAAG,IAAI,GAAG,EAA0B,CAAC;AACrD,yEAAyE;AACzE,uEAAuE;AACvE,sEAAsE;AACtE,MAAM,cAAc,GAAG,IAAI,GAAG,EAA4B,CAAC;AAC3D,IAAI,UAAU,GAAkB,IAAI,CAAC;AACrC,MAAM,WAAW,GAAG,aAAa,CAAC;AAElC,yEAAyE;AACzE,uEAAuE;AACvE,yEAAyE;AACzE,MAAM,aAAa,GAAa,EAAE,CAAC;AAEnC,wEAAwE;AACxE,8EAA8E;AAC9E,6EAA6E;AAC7E,IAAI,0BAA0B,GAAG,KAAK,CAAC;AACvC,SAAS,2BAA2B;IAClC,IAAI,0BAA0B;QAAE,OAAO;IACvC,0BAA0B,GAAG,IAAI,CAAC;IAClC,IAAI,OAAO,CAAC,0BAA0B,EAAE,oCAAoC,CAAC,KAAK,GAAG;QAAE,OAAO;IAC9F,sCAAsC;IACtC,OAAO,CAAC,IAAI,CACV,sEAAsE;QACtE,gEAAgE;QAChE,wEAAwE;QACxE,2DAA2D,CAC5D,CAAC;AACJ,CAAC;AAED,gBAAgB;AAChB,MAAM,UAAU,yBAAyB;IACvC,0BAA0B,GAAG,KAAK,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,UAAU,GAAG,QAAQ,CAAC;IACtB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9B,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC/B,CAAC;IACD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAClC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AA6BD,MAAM,UAAU,IAAI,CAClB,IAAY,EACZ,aAA2E,EAC3E,SAAwD;IAExD,2BAA2B,EAAE,CAAC;IAE9B,2CAA2C;IAC3C,IAAI,OAAoB,CAAC;IACzB,IAAI,QAAsD,CAAC;IAC3D,IAAI,OAAO,aAAa,KAAK,UAAU,EAAE,CAAC;QACxC,OAAO,GAAG,EAAE,CAAC;QACb,QAAQ,GAAG,aAAa,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,aAAa,IAAI,EAAE,CAAC;QAC9B,QAAQ,GAAG,SAAyD,CAAC;IACvE,CAAC;IAED,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,SAAS,IAAI,6BAA6B,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,GAAG,GAAG,UAAU,IAAI,WAAW,CAAC;IACtC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAC1B,CAAC;IACD,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;IAEtC,0EAA0E;IAC1E,yEAAyE;IACzE,+BAA+B;IAC/B,MAAM,aAAa,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACvF,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,aAAa,KAAK,aAAa,CAAC,EAAE,CAAC;QAC7E,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,kBAAkB,aAAa,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,MAAM,IAAI,KAAK,CACb,wBAAwB,IAAI,IAAI,UAAU,GAAG,SAAS,IAAI;YAC1D,yDAAyD;YACzD,+DAA+D,CAChE,CAAC;IACJ,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC;QACZ,IAAI;QACJ,OAAO;QACP,QAAQ;QACR,UAAU,EAAE,UAAU,IAAI,SAAS;QACnC,aAAa;KACd,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAc;IACnD,2BAA2B,EAAE,CAAC;IAC9B,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,OAAO,EAAE,KAAK,UAAU,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,aAAa,IAAI,6BAA6B,CAAC,CAAC;IAClE,CAAC;IACD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,EAAE,EAAa,CAAC;QAC/B,IAAI,MAAM,IAAI,OAAQ,MAAc,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACzD,kEAAkE;YAClE,+DAA+D;YAC/D,MAAM,IAAI,KAAK,CACb,aAAa,IAAI,8BAA8B;gBAC/C,qEAAqE,CACtE,CAAC;QACJ,CAAC;IACH,CAAC;YAAS,CAAC;QACT,aAAa,CAAC,GAAG,EAAE,CAAC;IACtB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,QAAiB;IAClD,IAAI,QAAQ;QAAE,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACvD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAiB;IAClD,IAAI,QAAQ;QAAE,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAiB;IAC7C,IAAI,QAAQ,EAAE,CAAC;QACb,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;SAAM,CAAC;QACN,UAAU,CAAC,KAAK,EAAE,CAAC;QACnB,cAAc,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;IACD,UAAU,GAAG,IAAI,CAAC;AACpB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,YAAY,CAAC,IAAc,EAAE,EAAU;IAC9C,2BAA2B,EAAE,CAAC;IAC9B,IAAI,OAAO,EAAE,KAAK,UAAU,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,8CAA8C,CAAC,CAAC;IACzE,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,IAAI,WAAW,CAAC;IACtC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7B,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAC9B,CAAC;IACD,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACtF,cAAc,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC;QAC5B,IAAI;QACJ,EAAE;QACF,UAAU,EAAE,UAAU,IAAI,SAAS;QACnC,YAAY;KACb,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,SAAS,CAAC,EAAU,IAAU,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AAE9E;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,QAAQ,CAAC,EAAU,IAAU,YAAY,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AAE5E;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,UAAU,CAAC,EAAU,IAAU,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AAEhF;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,SAAS,CAAC,EAAU,IAAU,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AAE9E,qDAAqD;AACrD,EAAE;AACF,2EAA2E;AAC3E,0EAA0E;AAC1E,2EAA2E;AAC3E,4CAA4C;AAC5C,2DAA2D;AAC3D,WAAiB,IAAI;IACnB,wDAAwD;IAC3C,cAAS,GAAyB,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IACrF,wDAAwD;IAC3C,aAAQ,GAAyB,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IACnF,wDAAwD;IAC3C,eAAU,GAAyB,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IACvF,wDAAwD;IAC3C,cAAS,GAAyB,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;AACvF,CAAC,EATgB,IAAI,KAAJ,IAAI,QASpB"}

package/lib/dist/lib/testCases/evaluators.d.ts

@@ -0,0 +1,80 @@
+/**
+ * `defineEvaluator()` + `evaluate()` — custom programmatic evaluators
+ * (RFC 004 §4.4, closes #244).
+ *
+ * Not every check is an LLM judge or a chai assertion. Sometimes the
+ * ground truth lives in code: a SQL result must match a golden row set, a
+ * generated diff must apply cleanly, a JSON answer must validate against a
+ * schema. A custom evaluator is a plain function that inspects the run and
+ * returns a verdict — deterministic, free, and fully under the author's
+ * control.
+ *
+ * ```js
+ * const { defineEvaluator, test } = require('@opensearch-project/agent-health');
+ *
+ * defineEvaluator('sql-matches-golden', ({ result }) => {
+ *   const rows = JSON.parse(result.agentOutput);
+ *   return { pass: deepEqual(rows, GOLDEN), reasoning: 'row set comparison' };
+ * });
+ *
+ * test('answers the revenue query', { prompt: '...' }, async ({ agent, evaluate }) => {
+ *   const result = await agent.run();
+ *   await evaluate(result, 'sql-matches-golden');
+ * });
+ * ```
+ *
+ * Evaluators are registered into a process-global registry by id (like a
+ * shared helper library), and run **in-process** when the body calls
+ * `evaluate()`. They record a `MatcherResult` with `method: 'evaluator'`,
+ * gate the test by default, and support an `.observe()` variant that feeds
+ * score/insights only — mirroring the `judge` fixture.
+ */
+import type { EvalResult } from './types.js';
+import type { TracesAccessor } from '../matchers/traces.js';
+import type { Verdict } from './judge.js';
+/** Context handed to a custom evaluator. */
+export interface EvaluatorContext {
+    /** The agent run under evaluation. */
+    result: EvalResult;
+    /** Optional free-form criteria string passed at the call site. */
+    criteria?: string;
+    /** Run-scoped OTel traces, when available (same as `result.traces`). */
+    traces?: TracesAccessor;
+}
+/** What a custom evaluator returns. */
+export interface EvaluatorResult {
+    /** Whether the check passed. */
+    pass: boolean;
+    /** Optional normalised score on the [0, 1] interval. */
+    score?: number;
+    /** Human-readable explanation (shown in the per-matcher breakdown). */
+    reasoning?: string;
+}
+export type EvaluatorFn = (ctx: EvaluatorContext) => EvaluatorResult | Promise<EvaluatorResult>;
+/**
+ * Register a custom evaluator under `id`.
+ *
+ * Registering the **same function** under an id again is a no-op (a watched
+ * file re-loading the identical module is fine). Registering a *different*
+ * function under an id that's already taken throws: the registry is
+ * process-global (shared like a helper library), so two `.eval.{js,ts}` files
+ * defining `defineEvaluator('len-check', …)` with different bodies would
+ * otherwise silently shadow each other based on load order. Fail loudly so the
+ * collision is visible at authoring time. Use distinct ids (or a shared helper
+ * module) when you need different implementations.
+ */
+export declare function defineEvaluator(id: string, fn: EvaluatorFn): void;
+/** Look up a registered evaluator by id. @internal */
+export declare function getEvaluator(id: string): EvaluatorFn | undefined;
+/** Clear all registered evaluators (test isolation / per-load reset). @internal */
+export declare function clearEvaluators(): void;
+/**
+ * The `evaluate` fixture: run a registered custom evaluator against a
+ * result. Gate role by default; `.observe()` records score/insights only.
+ */
+export interface EvaluateFn {
+    (result: EvalResult, evaluatorId: string, criteria?: string): Promise<Verdict>;
+    observe(result: EvalResult, evaluatorId: string, criteria?: string): Promise<Verdict>;
+}
+export declare const evaluate: EvaluateFn;
+//# sourceMappingURL=evaluators.d.ts.map

package/lib/dist/lib/testCases/evaluators.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluators.d.ts","sourceRoot":"","sources":["../../../testCases/evaluators.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,KAAK,EAAa,OAAO,EAAE,MAAM,YAAY,CAAC;AAGrD,4CAA4C;AAC5C,MAAM,WAAW,gBAAgB;IAC/B,sCAAsC;IACtC,MAAM,EAAE,UAAU,CAAC;IACnB,kEAAkE;IAClE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wEAAwE;IACxE,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB;AAED,uCAAuC;AACvC,MAAM,WAAW,eAAe;IAC9B,gCAAgC;IAChC,IAAI,EAAE,OAAO,CAAC;IACd,wDAAwD;IACxD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uEAAuE;IACvE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,WAAW,GAAG,CACxB,GAAG,EAAE,gBAAgB,KAClB,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AAIhD;;;;;;;;;;;GAWG;AACH,wBAAgB,eAAe,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,GAAG,IAAI,CAgBjE;AAED,sDAAsD;AACtD,wBAAgB,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAEhE;AAED,mFAAmF;AACnF,wBAAgB,eAAe,IAAI,IAAI,CAEtC;AA6ED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/E,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACvF;AAED,eAAO,MAAM,QAAQ,EAAE,UAOtB,CAAC"}

package/lib/dist/lib/testCases/evaluators.js

@@ -0,0 +1,105 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { recordVerdict } from '../matchers/session.js';
+const evaluatorRegistry = new Map();
+/**
+ * Register a custom evaluator under `id`.
+ *
+ * Registering the **same function** under an id again is a no-op (a watched
+ * file re-loading the identical module is fine). Registering a *different*
+ * function under an id that's already taken throws: the registry is
+ * process-global (shared like a helper library), so two `.eval.{js,ts}` files
+ * defining `defineEvaluator('len-check', …)` with different bodies would
+ * otherwise silently shadow each other based on load order. Fail loudly so the
+ * collision is visible at authoring time. Use distinct ids (or a shared helper
+ * module) when you need different implementations.
+ */
+export function defineEvaluator(id, fn) {
+    if (!id || typeof id !== 'string') {
+        throw new Error('defineEvaluator(id, fn): id must be a non-empty string');
+    }
+    if (typeof fn !== 'function') {
+        throw new Error(`defineEvaluator('${id}', fn): fn must be a function`);
+    }
+    const existing = evaluatorRegistry.get(id);
+    if (existing && existing !== fn) {
+        throw new Error(`defineEvaluator('${id}', ...): an evaluator with id '${id}' is already ` +
+            `registered with a different function. Evaluator ids are global — use a ` +
+            `unique id per evaluator (or share one function) to avoid silent shadowing.`);
+    }
+    evaluatorRegistry.set(id, fn);
+}
+/** Look up a registered evaluator by id. @internal */
+export function getEvaluator(id) {
+    return evaluatorRegistry.get(id);
+}
+/** Clear all registered evaluators (test isolation / per-load reset). @internal */
+export function clearEvaluators() {
+    evaluatorRegistry.clear();
+}
+/** Build a {@link Verdict} from a custom-evaluator result. */
+function evaluatorVerdict(r, role, errored, errorMessage) {
+    const verdict = {
+        passFailStatus: r.pass ? 'passed' : 'failed',
+        accuracy: typeof r.score === 'number' ? r.score * 100 : (r.pass ? 100 : 0),
+        score: typeof r.score === 'number' ? r.score : (r.pass ? 1 : 0),
+        reasoning: r.reasoning ?? '',
+        pass: r.pass,
+        role,
+        skipped: false,
+        errored,
+        errorMessage,
+        orThrow() {
+            if (this.pass || this.skipped)
+                return this;
+            const label = this.errored ? 'errored' : 'FAILED';
+            throw new Error(`Evaluator: ${label}\n${this.errorMessage ?? this.reasoning}`);
+        },
+    };
+    return verdict;
+}
+async function runEvaluator(result, evaluatorId, criteria, role) {
+    const fn = getEvaluator(evaluatorId);
+    const description = `evaluate: ${evaluatorId}${criteria ? ` — ${criteria}` : ''}`;
+    if (!fn) {
+        // Unknown evaluator id — this is an authoring error the run can't
+        // recover from. Record it as errored (excluded from pass-rate).
+        const msg = `No evaluator registered with id '${evaluatorId}'. Did you call defineEvaluator('${evaluatorId}', ...)?`;
+        recordVerdict({
+            description, pass: false, method: 'evaluator', role, errored: true,
+            durationMs: 0, errorMessage: msg, reasoning: '',
+        });
+        return evaluatorVerdict({ pass: false }, role, true, msg);
+    }
+    const startedAt = Date.now();
+    let out;
+    try {
+        out = await fn({ result, criteria, traces: result.traces });
+    }
+    catch (err) {
+        const msg = err?.message || String(err);
+        recordVerdict({
+            description, pass: false, method: 'evaluator', role, errored: true,
+            durationMs: Date.now() - startedAt, errorMessage: `Evaluator threw: ${msg}`, reasoning: '',
+        });
+        return evaluatorVerdict({ pass: false }, role, true, `Evaluator threw: ${msg}`);
+    }
+    const verdict = evaluatorVerdict(out, role, false);
+    recordVerdict({
+        description,
+        pass: verdict.pass,
+        method: 'evaluator',
+        role,
+        durationMs: Date.now() - startedAt,
+        score: verdict.score,
+        reasoning: verdict.reasoning,
+        errorMessage: verdict.pass ? undefined : (verdict.reasoning || 'evaluator failed'),
+    });
+    return verdict;
+}
+export const evaluate = Object.assign((result, evaluatorId, criteria) => runEvaluator(result, evaluatorId, criteria, 'gate'), {
+    observe: (result, evaluatorId, criteria) => runEvaluator(result, evaluatorId, criteria, 'observe'),
+});
+//# sourceMappingURL=evaluators.js.map

package/lib/dist/lib/testCases/evaluators.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluators.js","sourceRoot":"","sources":["../../../testCases/evaluators.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAqCH,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AA0BvD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAuB,CAAC;AAEzD;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,eAAe,CAAC,EAAU,EAAE,EAAe;IACzD,IAAI,CAAC,EAAE,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,OAAO,EAAE,KAAK,UAAU,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,oBAAoB,EAAE,+BAA+B,CAAC,CAAC;IACzE,CAAC;IACD,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC3C,IAAI,QAAQ,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,oBAAoB,EAAE,kCAAkC,EAAE,eAAe;YACvE,yEAAyE;YACzE,4EAA4E,CAC/E,CAAC;IACJ,CAAC;IACD,iBAAiB,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;AAChC,CAAC;AAED,sDAAsD;AACtD,MAAM,UAAU,YAAY,CAAC,EAAU;IACrC,OAAO,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AACnC,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,eAAe;IAC7B,iBAAiB,CAAC,KAAK,EAAE,CAAC;AAC5B,CAAC;AAED,8DAA8D;AAC9D,SAAS,gBAAgB,CACvB,CAAkB,EAClB,IAAe,EACf,OAAgB,EAChB,YAAqB;IAErB,MAAM,OAAO,GAAY;QACvB,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QAC5C,QAAQ,EAAE,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1E,KAAK,EAAE,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,EAAE;QAC5B,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,IAAI;QACJ,OAAO,EAAE,KAAK;QACd,OAAO;QACP,YAAY;QACZ,OAAO;YACL,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAO,IAAI,CAAC;YAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;YAClD,MAAM,IAAI,KAAK,CACb,cAAc,KAAK,KAAK,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,SAAS,EAAE,CAC9D,CAAC;QACJ,CAAC;KACF,CAAC;IACF,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,MAAkB,EAClB,WAAmB,EACnB,QAA4B,EAC5B,IAAe;IAEf,MAAM,EAAE,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IACrC,MAAM,WAAW,GAAG,aAAa,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAElF,IAAI,CAAC,EAAE,EAAE,CAAC;QACR,kEAAkE;QAClE,gEAAgE;QAChE,MAAM,GAAG,GAAG,oCAAoC,WAAW,oCAAoC,WAAW,UAAU,CAAC;QACrH,aAAa,CAAC;YACZ,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI;YAClE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;SAChD,CAAC,CAAC;QACH,OAAO,gBAAgB,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,GAAoB,CAAC;IACzB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9D,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;QACxC,aAAa,CAAC;YACZ,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI;YAClE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,YAAY,EAAE,oBAAoB,GAAG,EAAE,EAAE,SAAS,EAAE,EAAE;SAC3F,CAAC,CAAC;QACH,OAAO,gBAAgB,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,oBAAoB,GAAG,EAAE,CAAC,CAAC;IAClF,CAAC;IAED,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;IACnD,aAAa,CAAC;QACZ,WAAW;QACX,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM,EAAE,WAAW;QACnB,IAAI;QACJ,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;KACnF,CAAC,CAAC;IACH,OAAO,OAAO,CAAC;AACjB,CAAC;AAWD,MAAM,CAAC,MAAM,QAAQ,GAAe,MAAM,CAAC,MAAM,CAC/C,CAAC,MAAkB,EAAE,WAAmB,EAAE,QAAiB,EAAE,EAAE,CAC7D,YAAY,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,CAAC,EACrD;IACE,OAAO,EAAE,CAAC,MAAkB,EAAE,WAAmB,EAAE,QAAiB,EAAE,EAAE,CACtE,YAAY,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,CAAC;CACzD,CACF,CAAC"}

package/lib/dist/lib/testCases/index.d.ts

@@ -0,0 +1,14 @@
+export { test, describe, beforeAll, afterAll, beforeEach, afterEach, getRegisteredTests, getRegisteredHooks, clearRegistry, } from './define.js';
+export { loadTestCasesFromModule, isCodeFile, computeTestCaseHash } from './loader.js';
+export { migrateEvalSource } from './codemod.js';
+export type { CodemodResult } from './codemod.js';
+export { judge, wasJudgeCalled, resetJudgeFlag } from './judge.js';
+export { clearJudgeCache } from './judge.js';
+export { createAgentFixture } from './agentFixture.js';
+export { defineEvaluator, evaluate, getEvaluator, clearEvaluators, } from './evaluators.js';
+export type { EvaluatorContext, EvaluatorResult, EvaluatorFn, EvaluateFn, } from './evaluators.js';
+export type { AgentFixture, AgentRunOptions, AgentInvokeFn, CreateAgentFixtureOptions, } from './agentFixture.js';
+export type { CodeTestCase, EvalResult, TestOptions, TestFixtures, TestInfo, HookKind, HookFn, RegisteredHook, } from './types.js';
+export type { LoadResult, LoadedTestCase } from './loader.js';
+export type { JudgeVerdict, Verdict, JudgeRole, JudgeFn } from './judge.js';
+//# sourceMappingURL=index.d.ts.map

package/lib/dist/lib/testCases/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../testCases/index.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,IAAI,EACJ,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,UAAU,EACV,SAAS,EACT,kBAAkB,EAClB,kBAAkB,EAClB,aAAa,GACd,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,uBAAuB,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvF,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACjD,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EACL,eAAe,EACf,QAAQ,EACR,YAAY,EACZ,eAAe,GAChB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,gBAAgB,EAChB,eAAe,EACf,WAAW,EACX,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,YAAY,EACZ,eAAe,EACf,aAAa,EACb,yBAAyB,GAC1B,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACV,YAAY,EACZ,UAAU,EACV,WAAW,EACX,YAAY,EACZ,QAAQ,EACR,QAAQ,EACR,MAAM,EACN,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC9D,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC"}

package/lib/dist/lib/testCases/index.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export { test, describe, beforeAll, afterAll, beforeEach, afterEach, getRegisteredTests, getRegisteredHooks, clearRegistry, } from './define.js';
+export { loadTestCasesFromModule, isCodeFile, computeTestCaseHash } from './loader.js';
+export { migrateEvalSource } from './codemod.js';
+export { judge, wasJudgeCalled, resetJudgeFlag } from './judge.js';
+export { clearJudgeCache } from './judge.js';
+export { createAgentFixture } from './agentFixture.js';
+export { defineEvaluator, evaluate, getEvaluator, clearEvaluators, } from './evaluators.js';
+//# sourceMappingURL=index.js.map

package/lib/dist/lib/testCases/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../testCases/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,IAAI,EACJ,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,UAAU,EACV,SAAS,EACT,kBAAkB,EAClB,kBAAkB,EAClB,aAAa,GACd,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,uBAAuB,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvF,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EACL,eAAe,EACf,QAAQ,EACR,YAAY,EACZ,eAAe,GAChB,MAAM,iBAAiB,CAAC"}