@opensearch-project/agent-health 0.3.0 → 0.5.2

package/lib/dist/lib/testCases/codemod.js

@@ -0,0 +1,169 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Codemod: migrate v1 code-SDK eval files to the RFC-004 control-inversion
+ * shape (#256, phase 10 tooling).
+ *
+ * v1 bodies received a pre-populated `result` fixture (the framework invoked
+ * the agent eagerly). v2 bodies own invocation: `const result = await
+ * agent.run()`. This transform rewrites each `test(name, options, body)`
+ * whose body destructures `result` AND whose options carry a `prompt`:
+ *
+ *   test('x', { prompt: 'p' }, function ({ result, judge }) {  ...result... })
+ *     ↓
+ *   test('x', { prompt: 'p' }, async ({ agent, judge }) => {
+ *     const result = await agent.run();
+ *     ...result...
+ *   })
+ *
+ * Tests with no `prompt` (data-only) are left untouched — their `result` is
+ * the empty placeholder, which is still valid. Bodies that already call
+ * `agent.run()` are skipped (idempotent).
+ *
+ * Uses the TypeScript compiler API for a precise, formatting-preserving
+ * transform via plain text-range edits (no full reprint), so it works on
+ * both `.eval.js` and `.eval.ts`.
+ */
+import * as ts from 'typescript';
+/** Is this call expression a `test(...)` / `test.only(...)` registration? */
+function isTestCall(node) {
+    const e = node.expression;
+    if (ts.isIdentifier(e))
+        return e.text === 'test';
+    if (ts.isPropertyAccessExpression(e) && ts.isIdentifier(e.expression)) {
+        return e.expression.text === 'test';
+    }
+    return false;
+}
+/** Extract the options-object literal argument (2nd arg) if present. */
+function optionsArg(call) {
+    const a = call.arguments[1];
+    return a && ts.isObjectLiteralExpression(a) ? a : undefined;
+}
+/** Does the options object declare a non-empty `prompt`? */
+function hasPrompt(opts) {
+    if (!opts)
+        return false;
+    return opts.properties.some((p) => ts.isPropertyAssignment(p) &&
+        ((ts.isIdentifier(p.name) && p.name.text === 'prompt') ||
+            (ts.isStringLiteral(p.name) && p.name.text === 'prompt')));
+}
+/** The body is the last argument when it's a function/arrow. */
+function bodyArg(call) {
+    const last = call.arguments[call.arguments.length - 1];
+    if (last && (ts.isFunctionExpression(last) || ts.isArrowFunction(last)))
+        return last;
+    return undefined;
+}
+/**
+ * Run the codemod over a single file's source text.
+ */
+export function migrateEvalSource(code, fileName = 'eval.ts') {
+    const sf = ts.createSourceFile(fileName, code, ts.ScriptTarget.Latest, 
+    /*setParentNodes*/ true, fileName.endsWith('.ts') ? ts.ScriptKind.TS : ts.ScriptKind.JS);
+    const edits = [];
+    const notes = [];
+    const visit = (node) => {
+        if (ts.isCallExpression(node) && isTestCall(node)) {
+            processTest(node);
+        }
+        ts.forEachChild(node, visit);
+    };
+    function testName(call) {
+        const a = call.arguments[0];
+        return a && ts.isStringLiteralLike(a) ? a.text : '<anonymous>';
+    }
+    function processTest(call) {
+        const name = testName(call);
+        const body = bodyArg(call);
+        if (!body)
+            return; // no function body (e.g. test(name) stub)
+        const param = body.parameters[0];
+        // Only the object-destructuring fixture form is migrated.
+        if (!param || !ts.isObjectBindingPattern(param.name)) {
+            return;
+        }
+        const binding = param.name;
+        const names = binding.elements
+            .map((el) => (ts.isIdentifier(el.name) ? el.name.text : ''))
+            .filter(Boolean);
+        const usesResult = names.includes('result');
+        const usesAgent = names.includes('agent');
+        // Idempotency / applicability guards.
+        const bodyText = body.body ? body.body.getText(sf) : '';
+        const hasAgentRun = /\bagent\s*\.\s*run\s*\(/.test(bodyText);
+        if (hasAgentRun && !usesResult) {
+            // Fully migrated: calls agent.run() and no longer destructures `result`.
+            notes.push(`skip  ${name}: already migrated (calls agent.run(), no result binding)`);
+            return;
+        }
+        if (hasAgentRun && usesResult) {
+            // Half-migrated: agent.run() is in, but `result` is still destructured
+            // (and almost certainly still referenced). The runner hands the body an
+            // empty placeholder `result` when `agent` is destructured separately, so
+            // those `result.*` reads silently read empties and assertions fail — the
+            // migration *looks* broken when it isn't. We can't safely rename those
+            // reads to the agent.run() return value, so surface for manual review
+            // instead of silently reporting "already migrated".
+            notes.push(`review ${name}: half-migrated — calls agent.run() but still destructures \`result\`. ` +
+                `Capture the agent.run() return value (\`const result = await agent.run()\`) and drop the \`result\` fixture.`);
+            return;
+        }
+        if (!hasPrompt(optionsArg(call))) {
+            notes.push(`skip  ${name}: no prompt (data-only test, result placeholder is valid)`);
+            return;
+        }
+        if (!usesResult) {
+            notes.push(`skip  ${name}: body does not use result`);
+            return;
+        }
+        // 1) Rewrite the destructuring: replace `result` with `agent` (or add
+        //    `agent` if result+agent both somehow absent). Keep other fixtures.
+        const newNames = names.map((n) => (n === 'result' ? 'agent' : n));
+        if (usesAgent) {
+            // agent already destructured — just drop `result`.
+            const filtered = names.filter((n) => n !== 'result');
+            newNames.length = 0;
+            newNames.push(...filtered);
+        }
+        const newBinding = `{ ${newNames.join(', ')} }`;
+        edits.push({ start: binding.getStart(sf), end: binding.getEnd(), text: newBinding });
+        // 2) Ensure the function is async.
+        const isAsync = body.modifiers?.some((m) => m.kind === ts.SyntaxKind.AsyncKeyword);
+        if (!isAsync) {
+            edits.push({ start: body.getStart(sf), end: body.getStart(sf), text: 'async ' });
+        }
+        // 3) Insert `const result = await agent.run();` at the top of the body.
+        //    Handle both block bodies `{ ... }` and expression arrow bodies.
+        if (body.body && ts.isBlock(body.body)) {
+            const openBrace = body.body.getStart(sf) + 1; // just after `{`
+            edits.push({ start: openBrace, end: openBrace, text: `\n  const result = await agent.run();` });
+        }
+        else if (body.body) {
+            // Expression-bodied arrow: wrap into a block.
+            const exprStart = body.body.getStart(sf);
+            const exprEnd = body.body.getEnd();
+            const exprText = body.body.getText(sf);
+            edits.push({
+                start: exprStart,
+                end: exprEnd,
+                text: `{\n  const result = await agent.run();\n  return ${exprText};\n}`,
+            });
+        }
+        notes.push(`migrate ${name}: result → agent.run()`);
+    }
+    visit(sf);
+    if (edits.length === 0) {
+        return { code, changed: false, notes };
+    }
+    // Apply edits right-to-left so offsets stay valid.
+    edits.sort((a, b) => b.start - a.start);
+    let out = code;
+    for (const e of edits) {
+        out = out.slice(0, e.start) + e.text + out.slice(e.end);
+    }
+    return { code: out, changed: true, notes };
+}
+//# sourceMappingURL=codemod.js.map

package/lib/dist/lib/testCases/codemod.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"codemod.js","sourceRoot":"","sources":["../../../testCases/codemod.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,MAAM,YAAY,CAAC;AAiBjC,6EAA6E;AAC7E,SAAS,UAAU,CAAC,IAAuB;IACzC,MAAM,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;IAC1B,IAAI,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;IACjD,IAAI,EAAE,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;QACtE,OAAO,CAAC,CAAC,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC;IACtC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,wEAAwE;AACxE,SAAS,UAAU,CAAC,IAAuB;IACzC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,IAAI,EAAE,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC9D,CAAC;AAED,4DAA4D;AAC5D,SAAS,SAAS,CAAC,IAA4C;IAC7D,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CACzB,CAAC,CAAC,EAAE,EAAE,CACJ,EAAE,CAAC,oBAAoB,CAAC,CAAC,CAAC;QAC1B,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC;YACpD,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAC9D,CAAC;AACJ,CAAC;AAED,gEAAgE;AAChE,SAAS,OAAO,CAAC,IAAuB;IACtC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACrF,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,QAAQ,GAAG,SAAS;IAClE,MAAM,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAC5B,QAAQ,EACR,IAAI,EACJ,EAAE,CAAC,YAAY,CAAC,MAAM;IACtB,kBAAkB,CAAC,IAAI,EACvB,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAC/D,CAAC;IAEF,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,KAAK,GAAG,CAAC,IAAa,EAAE,EAAE;QAC9B,IAAI,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAClD,WAAW,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;QACD,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC/B,CAAC,CAAC;IAEF,SAAS,QAAQ,CAAC,IAAuB;QACvC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,IAAI,EAAE,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC;IACjE,CAAC;IAED,SAAS,WAAW,CAAC,IAAuB;QAC1C,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC3B,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,0CAA0C;QAE7D,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACjC,0DAA0D;QAC1D,IAAI,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC,sBAAsB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACrD,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC;QAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ;aAC3B,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;aAC3D,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAE1C,sCAAsC;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,MAAM,WAAW,GAAG,yBAAyB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7D,IAAI,WAAW,IAAI,CAAC,UAAU,EAAE,CAAC;YAC/B,yEAAyE;YACzE,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,2DAA2D,CAAC,CAAC;YACrF,OAAO;QACT,CAAC;QACD,IAAI,WAAW,IAAI,UAAU,EAAE,CAAC;YAC9B,uEAAuE;YACvE,wEAAwE;YACxE,yEAAyE;YACzE,yEAAyE;YACzE,uEAAuE;YACvE,sEAAsE;YACtE,oDAAoD;YACpD,KAAK,CAAC,IAAI,CACR,UAAU,IAAI,yEAAyE;gBACrF,8GAA8G,CACjH,CAAC;YACF,OAAO;QACT,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,2DAA2D,CAAC,CAAC;YACrF,OAAO;QACT,CAAC;QACD,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,4BAA4B,CAAC,CAAC;YACtD,OAAO;QACT,CAAC;QAED,sEAAsE;QACtE,wEAAwE;QACxE,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,IAAI,SAAS,EAAE,CAAC;YACd,mDAAmD;YACnD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC;YACrD,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YACpB,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;QAC7B,CAAC;QACD,MAAM,UAAU,GAAG,KAAK,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QAErF,mCAAmC;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QACnF,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QACnF,CAAC;QAED,wEAAwE;QACxE,qEAAqE;QACrE,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACvC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,iBAAiB;YAC/D,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,uCAAuC,EAAE,CAAC,CAAC;QAClG,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACrB,8CAA8C;YAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC;gBACT,KAAK,EAAE,SAAS;gBAChB,GAAG,EAAE,OAAO;gBACZ,IAAI,EAAE,oDAAoD,QAAQ,MAAM;aACzE,CAAC,CAAC;QACL,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,WAAW,IAAI,wBAAwB,CAAC,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,EAAE,CAAC,CAAC;IAEV,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IACzC,CAAC;IAED,mDAAmD;IACnD,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACxC,IAAI,GAAG,GAAG,IAAI,CAAC;IACf,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AAC7C,CAAC"}

package/lib/dist/lib/testCases/define.d.ts

@@ -0,0 +1,114 @@
+import type { CodeTestCase, TestOptions, EvalResult, HookFn, RegisteredHook } from './types.js';
+/** @internal */
+export declare function _resetExperimentalWarning(): void;
+export declare function setActiveFile(filePath: string): void;
+/**
+ * Register a code-based test case.
+ *
+ * Two valid signatures (Playwright-style):
+ * - `test(name, body)` — no options at all
+ * - `test(name, options, body)` — with options
+ *
+ * Only `name` is required. All TestOptions fields are optional. When
+ * `options.prompt` is absent, the runner skips agent invocation and the
+ * body receives an empty EvalResult.
+ *
+ * Throws if a test with the same name is already registered in the same
+ * source file. Cross-file duplicates are allowed (storage identity is
+ * `name + sourceFile`).
+ *
+ * @experimental The SDK shape (signature, options, body fixtures) may change
+ * in a minor release without a deprecation cycle. See `lib/index.ts`.
+ */
+export declare function test(name: string, body: (result: EvalResult) => Promise<void> | void): void;
+export declare function test(name: string, options: TestOptions, body: (result: EvalResult) => Promise<void> | void): void;
+/**
+ * Group tests under a benchmark name. Equivalent to Playwright's
+ * `describe()` — the wrapped `test()` calls inherit the describe's name as
+ * their benchmark group. Nested describes flatten with ' > '.
+ *
+ * @example
+ *   describe('RCA Suite', () => {
+ *     test('payment-service is the root cause', { prompt: ... }, async ({ result, judge }) => {
+ *       expect(result.trajectory).to.haveCalledTool('search_logs');
+ *       await judge(result, 'identifies the failing dependency');
+ *     });
+ *   });
+ *
+ * The describe body MUST be synchronous (no `await`/dynamic content), like
+ * Playwright. The function is invoked once at registration time.
+ */
+export declare function describe(name: string, fn: () => void): void;
+export declare function getRegisteredTests(filePath?: string): CodeTestCase[];
+/**
+ * Return all hooks registered for the given file (or every file when no
+ * argument is given). Returns a snapshot — mutating the result has no
+ * effect on the registry.
+ */
+export declare function getRegisteredHooks(filePath?: string): RegisteredHook[];
+export declare function clearRegistry(filePath?: string): void;
+/**
+ * Register a hook that runs **once** before the first test in its scope.
+ *
+ * Scope is the surrounding `describe(...)` block, or the whole file when
+ * called at the top level. With parallel test execution (the runner
+ * dispatches up to `concurrency` tests at once), the orchestrator uses a
+ * once-latch so all parallel arrivals await the same `beforeAll` promise.
+ *
+ * @example
+ *   beforeAll(async () => {
+ *     await fs.mkdir('/tmp/agent-health-fixtures', { recursive: true });
+ *   });
+ */
+export declare function beforeAll(fn: HookFn): void;
+/**
+ * Register a hook that runs **once** after the last test in its scope.
+ *
+ * Always runs, even when every test in the scope failed. The orchestrator
+ * uses a remaining-test counter (decremented on each test completion,
+ * regardless of pass/fail) and triggers `afterAll` when it hits zero.
+ *
+ * @example
+ *   afterAll(async () => {
+ *     await fs.rm('/tmp/agent-health-fixtures', { recursive: true });
+ *   });
+ */
+export declare function afterAll(fn: HookFn): void;
+/**
+ * Register a hook that runs **before each test** in its scope.
+ *
+ * Receives the same fixtures object the body will see, plus a
+ * `provide(key, value)` function for stashing values that the test body
+ * (and `afterEach`) can read via `fixtures.provisioned[key]`. Each test
+ * gets its own provisioned bag, so concurrent tests are isolated.
+ *
+ * @example
+ *   beforeEach(async ({ provide, testInfo }) => {
+ *     const dir = await fs.mkdtemp(`/tmp/${testInfo.name}-`);
+ *     provide('workspaceDir', dir);
+ *   });
+ */
+export declare function beforeEach(fn: HookFn): void;
+/**
+ * Register a hook that runs **after each test** in its scope.
+ *
+ * Always runs, even when the test body or a `beforeEach` threw. Reads
+ * provisioned values via `fixtures.provisioned[key]` for cleanup. Errors
+ * thrown from `afterEach` are captured as MatcherResult entries on the
+ * test — they don't crash the runner.
+ *
+ * @example
+ *   afterEach(async ({ provisioned }) => {
+ *     if (provisioned.workspaceDir) {
+ *       await fs.rm(provisioned.workspaceDir as string, { recursive: true, force: true });
+ *     }
+ *   });
+ */
+export declare function afterEach(fn: HookFn): void;
+export declare namespace test {
+    const beforeAll: (fn: HookFn) => void;
+    const afterAll: (fn: HookFn) => void;
+    const beforeEach: (fn: HookFn) => void;
+    const afterEach: (fn: HookFn) => void;
+}
+//# sourceMappingURL=define.d.ts.map

package/lib/dist/lib/testCases/define.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"define.d.ts","sourceRoot":"","sources":["../../../testCases/define.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,YAAY,EACZ,WAAW,EACX,UAAU,EACV,MAAM,EAEN,cAAc,EACf,MAAM,YAAY,CAAC;AAiCpB,gBAAgB;AAChB,wBAAgB,yBAAyB,IAAI,IAAI,CAEhD;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAQpD;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,IAAI,CAClB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,GACjD,IAAI,CAAC;AACR,wBAAgB,IAAI,CAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,WAAW,EACpB,IAAI,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,GACjD,IAAI,CAAC;AAuDR;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,IAAI,GAAG,IAAI,CAsB3D;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,YAAY,EAAE,CAGpE;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,cAAc,EAAE,CAGtE;AAED,wBAAgB,aAAa,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CASrD;AA4BD;;;;;;;;;;;;GAYG;AACH,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAmC;AAE9E;;;;;;;;;;;GAWG;AACH,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAkC;AAE5E;;;;;;;;;;;;;GAaG;AACH,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAoC;AAEhF;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAmC;AAS9E,yBAAiB,IAAI,CAAC;IAEb,MAAM,SAAS,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAA4C,CAAC;IAE9E,MAAM,QAAQ,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAA2C,CAAC;IAE5E,MAAM,UAAU,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAA6C,CAAC;IAEhF,MAAM,SAAS,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAA4C,CAAC;CACtF"}

package/lib/dist/lib/testCases/define.js

@@ -0,0 +1,253 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { readEnv } from '../envCompat.js';
+const registries = new Map();
+// Hooks live in a parallel registry keyed by file path, exactly matching
+// the test() registry. The HookOrchestrator filters by describePath at
+// run time. Cross-file hooks never leak — the loader resets per file.
+const hookRegistries = new Map();
+let activeFile = null;
+const DEFAULT_KEY = '__default__';
+// Active describe stack (synchronous push/pop). Each entry is a describe
+// name; nested describes accumulate. When test() runs, it captures the
+// current stack and joins with ' > ' to derive the test's benchmarkPath.
+const describeStack = [];
+// Emit a one-time experimental-status warning the first time the SDK is
+// touched, unless the user opts out. This is intentionally noisy enough to be
+// noticed but only fires once per process so it doesn't pollute test output.
+let experimentalWarningEmitted = false;
+function emitExperimentalWarningOnce() {
+    if (experimentalWarningEmitted)
+        return;
+    experimentalWarningEmitted = true;
+    if (readEnv('AH_SUPPRESS_EXPERIMENTAL', 'AGENT_HEALTH_SUPPRESS_EXPERIMENTAL') === '1')
+        return;
+    // eslint-disable-next-line no-console
+    console.warn('[agent-health] The code-based test SDK (test()/judge()/expect()) is ' +
+        'experimental. The API may change in a minor release without a ' +
+        'deprecation cycle. Pin your @opensearch-project/agent-health version, ' +
+        'or set AH_SUPPRESS_EXPERIMENTAL=1 to silence this notice.');
+}
+/** @internal */
+export function _resetExperimentalWarning() {
+    experimentalWarningEmitted = false;
+}
+export function setActiveFile(filePath) {
+    activeFile = filePath;
+    if (!registries.has(filePath)) {
+        registries.set(filePath, []);
+    }
+    if (!hookRegistries.has(filePath)) {
+        hookRegistries.set(filePath, []);
+    }
+}
+export function test(name, optionsOrBody, maybeBody) {
+    emitExperimentalWarningOnce();
+    // Resolve the two-arg / three-arg overload
+    let options;
+    let evaluate;
+    if (typeof optionsOrBody === 'function') {
+        options = {};
+        evaluate = optionsOrBody;
+    }
+    else {
+        options = optionsOrBody ?? {};
+        evaluate = maybeBody;
+    }
+    if (!name || typeof name !== 'string') {
+        throw new Error('test() requires a name (the first argument)');
+    }
+    if (typeof evaluate !== 'function') {
+        throw new Error(`test("${name}") requires a body function`);
+    }
+    const key = activeFile ?? DEFAULT_KEY;
+    if (!registries.has(key)) {
+        registries.set(key, []);
+    }
+    const registry = registries.get(key);
+    // Within-file uniqueness guard: a name+benchmarkPath pair must be unique.
+    // The same name in two different describe blocks is allowed because they
+    // map to different benchmarks.
+    const benchmarkPath = describeStack.length > 0 ? describeStack.join(' > ') : undefined;
+    if (registry.some(t => t.name === name && t.benchmarkPath === benchmarkPath)) {
+        const fileLabel = activeFile ? ` in ${activeFile}` : '';
+        const groupLabel = benchmarkPath ? ` (in describe "${benchmarkPath}")` : '';
+        throw new Error(`Duplicate test name "${name}"${groupLabel}${fileLabel}. ` +
+            `Test names must be unique within their describe block. ` +
+            `Move one of the tests to a different describe() or rename it.`);
+    }
+    registry.push({
+        name,
+        options,
+        evaluate,
+        sourceFile: activeFile ?? undefined,
+        benchmarkPath,
+    });
+}
+/**
+ * Group tests under a benchmark name. Equivalent to Playwright's
+ * `describe()` — the wrapped `test()` calls inherit the describe's name as
+ * their benchmark group. Nested describes flatten with ' > '.
+ *
+ * @example
+ *   describe('RCA Suite', () => {
+ *     test('payment-service is the root cause', { prompt: ... }, async ({ result, judge }) => {
+ *       expect(result.trajectory).to.haveCalledTool('search_logs');
+ *       await judge(result, 'identifies the failing dependency');
+ *     });
+ *   });
+ *
+ * The describe body MUST be synchronous (no `await`/dynamic content), like
+ * Playwright. The function is invoked once at registration time.
+ */
+export function describe(name, fn) {
+    emitExperimentalWarningOnce();
+    if (!name || typeof name !== 'string') {
+        throw new Error('describe() requires a name (the first argument)');
+    }
+    if (typeof fn !== 'function') {
+        throw new Error(`describe("${name}") requires a body function`);
+    }
+    describeStack.push(name);
+    try {
+        const result = fn();
+        if (result && typeof result.then === 'function') {
+            // Mirror Playwright — describe bodies must be synchronous because
+            // they run during registration, well before any test executes.
+            throw new Error(`describe("${name}") body returned a Promise. ` +
+                `describe blocks must be synchronous — use test() inside, not await.`);
+        }
+    }
+    finally {
+        describeStack.pop();
+    }
+}
+export function getRegisteredTests(filePath) {
+    if (filePath)
+        return [...(registries.get(filePath) ?? [])];
+    return [...registries.values()].flatMap(r => [...r]);
+}
+/**
+ * Return all hooks registered for the given file (or every file when no
+ * argument is given). Returns a snapshot — mutating the result has no
+ * effect on the registry.
+ */
+export function getRegisteredHooks(filePath) {
+    if (filePath)
+        return [...(hookRegistries.get(filePath) ?? [])];
+    return [...hookRegistries.values()].flatMap(r => [...r]);
+}
+export function clearRegistry(filePath) {
+    if (filePath) {
+        registries.delete(filePath);
+        hookRegistries.delete(filePath);
+    }
+    else {
+        registries.clear();
+        hookRegistries.clear();
+    }
+    activeFile = null;
+}
+/**
+ * Internal helper used by the four public hook registrars below.
+ *
+ * Captures the live `(activeFile, [...describeStack])` so the orchestrator
+ * can filter hooks by scope at run time. Multiple hooks of the same kind
+ * in the same scope are allowed and run in registration order (reversed
+ * for `afterEach`/`afterAll`, mirroring Playwright/Jest).
+ */
+function registerHook(kind, fn) {
+    emitExperimentalWarningOnce();
+    if (typeof fn !== 'function') {
+        throw new Error(`${kind}() requires a function as its first argument`);
+    }
+    const key = activeFile ?? DEFAULT_KEY;
+    if (!hookRegistries.has(key)) {
+        hookRegistries.set(key, []);
+    }
+    const describePath = describeStack.length > 0 ? describeStack.join(' > ') : undefined;
+    hookRegistries.get(key).push({
+        kind,
+        fn,
+        sourceFile: activeFile ?? undefined,
+        describePath,
+    });
+}
+/**
+ * Register a hook that runs **once** before the first test in its scope.
+ *
+ * Scope is the surrounding `describe(...)` block, or the whole file when
+ * called at the top level. With parallel test execution (the runner
+ * dispatches up to `concurrency` tests at once), the orchestrator uses a
+ * once-latch so all parallel arrivals await the same `beforeAll` promise.
+ *
+ * @example
+ *   beforeAll(async () => {
+ *     await fs.mkdir('/tmp/agent-health-fixtures', { recursive: true });
+ *   });
+ */
+export function beforeAll(fn) { registerHook('beforeAll', fn); }
+/**
+ * Register a hook that runs **once** after the last test in its scope.
+ *
+ * Always runs, even when every test in the scope failed. The orchestrator
+ * uses a remaining-test counter (decremented on each test completion,
+ * regardless of pass/fail) and triggers `afterAll` when it hits zero.
+ *
+ * @example
+ *   afterAll(async () => {
+ *     await fs.rm('/tmp/agent-health-fixtures', { recursive: true });
+ *   });
+ */
+export function afterAll(fn) { registerHook('afterAll', fn); }
+/**
+ * Register a hook that runs **before each test** in its scope.
+ *
+ * Receives the same fixtures object the body will see, plus a
+ * `provide(key, value)` function for stashing values that the test body
+ * (and `afterEach`) can read via `fixtures.provisioned[key]`. Each test
+ * gets its own provisioned bag, so concurrent tests are isolated.
+ *
+ * @example
+ *   beforeEach(async ({ provide, testInfo }) => {
+ *     const dir = await fs.mkdtemp(`/tmp/${testInfo.name}-`);
+ *     provide('workspaceDir', dir);
+ *   });
+ */
+export function beforeEach(fn) { registerHook('beforeEach', fn); }
+/**
+ * Register a hook that runs **after each test** in its scope.
+ *
+ * Always runs, even when the test body or a `beforeEach` threw. Reads
+ * provisioned values via `fixtures.provisioned[key]` for cleanup. Errors
+ * thrown from `afterEach` are captured as MatcherResult entries on the
+ * test — they don't crash the runner.
+ *
+ * @example
+ *   afterEach(async ({ provisioned }) => {
+ *     if (provisioned.workspaceDir) {
+ *       await fs.rm(provisioned.workspaceDir as string, { recursive: true, force: true });
+ *     }
+ *   });
+ */
+export function afterEach(fn) { registerHook('afterEach', fn); }
+// Playwright-style sugar: `test.beforeEach(fn)` etc.
+//
+// Declaration merging here gives TypeScript users typed access to the four
+// hook registrars *and* installs them as properties on the runtime `test`
+// function value. Both `import { beforeEach }` and `test.beforeEach` route
+// to the same internal registerHook() call.
+// eslint-disable-next-line @typescript-eslint/no-namespace
+(function (test) {
+    // eslint-disable-next-line @typescript-eslint/no-shadow
+    test.beforeAll = (fn) => registerHook('beforeAll', fn);
+    // eslint-disable-next-line @typescript-eslint/no-shadow
+    test.afterAll = (fn) => registerHook('afterAll', fn);
+    // eslint-disable-next-line @typescript-eslint/no-shadow
+    test.beforeEach = (fn) => registerHook('beforeEach', fn);
+    // eslint-disable-next-line @typescript-eslint/no-shadow
+    test.afterEach = (fn) => registerHook('afterEach', fn);
+})(test || (test = {}));
+//# sourceMappingURL=define.js.map

package/lib/dist/lib/testCases/define.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"define.js","sourceRoot":"","sources":["../../../testCases/define.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAUH,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE1C,MAAM,UAAU,GAAG,IAAI,GAAG,EAA0B,CAAC;AACrD,yEAAyE;AACzE,uEAAuE;AACvE,sEAAsE;AACtE,MAAM,cAAc,GAAG,IAAI,GAAG,EAA4B,CAAC;AAC3D,IAAI,UAAU,GAAkB,IAAI,CAAC;AACrC,MAAM,WAAW,GAAG,aAAa,CAAC;AAElC,yEAAyE;AACzE,uEAAuE;AACvE,yEAAyE;AACzE,MAAM,aAAa,GAAa,EAAE,CAAC;AAEnC,wEAAwE;AACxE,8EAA8E;AAC9E,6EAA6E;AAC7E,IAAI,0BAA0B,GAAG,KAAK,CAAC;AACvC,SAAS,2BAA2B;IAClC,IAAI,0BAA0B;QAAE,OAAO;IACvC,0BAA0B,GAAG,IAAI,CAAC;IAClC,IAAI,OAAO,CAAC,0BAA0B,EAAE,oCAAoC,CAAC,KAAK,GAAG;QAAE,OAAO;IAC9F,sCAAsC;IACtC,OAAO,CAAC,IAAI,CACV,sEAAsE;QACtE,gEAAgE;QAChE,wEAAwE;QACxE,2DAA2D,CAC5D,CAAC;AACJ,CAAC;AAED,gBAAgB;AAChB,MAAM,UAAU,yBAAyB;IACvC,0BAA0B,GAAG,KAAK,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,UAAU,GAAG,QAAQ,CAAC;IACtB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9B,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC/B,CAAC;IACD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAClC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AA6BD,MAAM,UAAU,IAAI,CAClB,IAAY,EACZ,aAA2E,EAC3E,SAAwD;IAExD,2BAA2B,EAAE,CAAC;IAE9B,2CAA2C;IAC3C,IAAI,OAAoB,CAAC;IACzB,IAAI,QAAsD,CAAC;IAC3D,IAAI,OAAO,aAAa,KAAK,UAAU,EAAE,CAAC;QACxC,OAAO,GAAG,EAAE,CAAC;QACb,QAAQ,GAAG,aAAa,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,aAAa,IAAI,EAAE,CAAC;QAC9B,QAAQ,GAAG,SAAyD,CAAC;IACvE,CAAC;IAED,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,SAAS,IAAI,6BAA6B,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,GAAG,GAAG,UAAU,IAAI,WAAW,CAAC;IACtC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAC1B,CAAC;IACD,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;IAEtC,0EAA0E;IAC1E,yEAAyE;IACzE,+BAA+B;IAC/B,MAAM,aAAa,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACvF,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,aAAa,KAAK,aAAa,CAAC,EAAE,CAAC;QAC7E,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,kBAAkB,aAAa,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,MAAM,IAAI,KAAK,CACb,wBAAwB,IAAI,IAAI,UAAU,GAAG,SAAS,IAAI;YAC1D,yDAAyD;YACzD,+DAA+D,CAChE,CAAC;IACJ,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC;QACZ,IAAI;QACJ,OAAO;QACP,QAAQ;QACR,UAAU,EAAE,UAAU,IAAI,SAAS;QACnC,aAAa;KACd,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAc;IACnD,2BAA2B,EAAE,CAAC;IAC9B,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,OAAO,EAAE,KAAK,UAAU,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,aAAa,IAAI,6BAA6B,CAAC,CAAC;IAClE,CAAC;IACD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,EAAE,EAAa,CAAC;QAC/B,IAAI,MAAM,IAAI,OAAQ,MAAc,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACzD,kEAAkE;YAClE,+DAA+D;YAC/D,MAAM,IAAI,KAAK,CACb,aAAa,IAAI,8BAA8B;gBAC/C,qEAAqE,CACtE,CAAC;QACJ,CAAC;IACH,CAAC;YAAS,CAAC;QACT,aAAa,CAAC,GAAG,EAAE,CAAC;IACtB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,QAAiB;IAClD,IAAI,QAAQ;QAAE,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACvD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAiB;IAClD,IAAI,QAAQ;QAAE,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAiB;IAC7C,IAAI,QAAQ,EAAE,CAAC;QACb,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;SAAM,CAAC;QACN,UAAU,CAAC,KAAK,EAAE,CAAC;QACnB,cAAc,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;IACD,UAAU,GAAG,IAAI,CAAC;AACpB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,YAAY,CAAC,IAAc,EAAE,EAAU;IAC9C,2BAA2B,EAAE,CAAC;IAC9B,IAAI,OAAO,EAAE,KAAK,UAAU,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,8CAA8C,CAAC,CAAC;IACzE,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,IAAI,WAAW,CAAC;IACtC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7B,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAC9B,CAAC;IACD,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACtF,cAAc,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC;QAC5B,IAAI;QACJ,EAAE;QACF,UAAU,EAAE,UAAU,IAAI,SAAS;QACnC,YAAY;KACb,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,SAAS,CAAC,EAAU,IAAU,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AAE9E;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,QAAQ,CAAC,EAAU,IAAU,YAAY,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AAE5E;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,UAAU,CAAC,EAAU,IAAU,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AAEhF;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,SAAS,CAAC,EAAU,IAAU,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AAE9E,qDAAqD;AACrD,EAAE;AACF,2EAA2E;AAC3E,0EAA0E;AAC1E,2EAA2E;AAC3E,4CAA4C;AAC5C,2DAA2D;AAC3D,WAAiB,IAAI;IACnB,wDAAwD;IAC3C,cAAS,GAAyB,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IACrF,wDAAwD;IAC3C,aAAQ,GAAyB,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IACnF,wDAAwD;IAC3C,eAAU,GAAyB,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IACvF,wDAAwD;IAC3C,cAAS,GAAyB,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;AACvF,CAAC,EATgB,IAAI,KAAJ,IAAI,QASpB"}

package/lib/dist/lib/testCases/evaluators.d.ts

@@ -0,0 +1,80 @@
+/**
+ * `defineEvaluator()` + `evaluate()` — custom programmatic evaluators
+ * (RFC 004 §4.4, closes #244).
+ *
+ * Not every check is an LLM judge or a chai assertion. Sometimes the
+ * ground truth lives in code: a SQL result must match a golden row set, a
+ * generated diff must apply cleanly, a JSON answer must validate against a
+ * schema. A custom evaluator is a plain function that inspects the run and
+ * returns a verdict — deterministic, free, and fully under the author's
+ * control.
+ *
+ * ```js
+ * const { defineEvaluator, test } = require('@opensearch-project/agent-health');
+ *
+ * defineEvaluator('sql-matches-golden', ({ result }) => {
+ *   const rows = JSON.parse(result.agentOutput);
+ *   return { pass: deepEqual(rows, GOLDEN), reasoning: 'row set comparison' };
+ * });
+ *
+ * test('answers the revenue query', { prompt: '...' }, async ({ agent, evaluate }) => {
+ *   const result = await agent.run();
+ *   await evaluate(result, 'sql-matches-golden');
+ * });
+ * ```
+ *
+ * Evaluators are registered into a process-global registry by id (like a
+ * shared helper library), and run **in-process** when the body calls
+ * `evaluate()`. They record a `MatcherResult` with `method: 'evaluator'`,
+ * gate the test by default, and support an `.observe()` variant that feeds
+ * score/insights only — mirroring the `judge` fixture.
+ */
+import type { EvalResult } from './types.js';
+import type { TracesAccessor } from '../matchers/traces.js';
+import type { Verdict } from './judge.js';
+/** Context handed to a custom evaluator. */
+export interface EvaluatorContext {
+    /** The agent run under evaluation. */
+    result: EvalResult;
+    /** Optional free-form criteria string passed at the call site. */
+    criteria?: string;
+    /** Run-scoped OTel traces, when available (same as `result.traces`). */
+    traces?: TracesAccessor;
+}
+/** What a custom evaluator returns. */
+export interface EvaluatorResult {
+    /** Whether the check passed. */
+    pass: boolean;
+    /** Optional normalised score on the [0, 1] interval. */
+    score?: number;
+    /** Human-readable explanation (shown in the per-matcher breakdown). */
+    reasoning?: string;
+}
+export type EvaluatorFn = (ctx: EvaluatorContext) => EvaluatorResult | Promise<EvaluatorResult>;
+/**
+ * Register a custom evaluator under `id`.
+ *
+ * Registering the **same function** under an id again is a no-op (a watched
+ * file re-loading the identical module is fine). Registering a *different*
+ * function under an id that's already taken throws: the registry is
+ * process-global (shared like a helper library), so two `.eval.{js,ts}` files
+ * defining `defineEvaluator('len-check', …)` with different bodies would
+ * otherwise silently shadow each other based on load order. Fail loudly so the
+ * collision is visible at authoring time. Use distinct ids (or a shared helper
+ * module) when you need different implementations.
+ */
+export declare function defineEvaluator(id: string, fn: EvaluatorFn): void;
+/** Look up a registered evaluator by id. @internal */
+export declare function getEvaluator(id: string): EvaluatorFn | undefined;
+/** Clear all registered evaluators (test isolation / per-load reset). @internal */
+export declare function clearEvaluators(): void;
+/**
+ * The `evaluate` fixture: run a registered custom evaluator against a
+ * result. Gate role by default; `.observe()` records score/insights only.
+ */
+export interface EvaluateFn {
+    (result: EvalResult, evaluatorId: string, criteria?: string): Promise<Verdict>;
+    observe(result: EvalResult, evaluatorId: string, criteria?: string): Promise<Verdict>;
+}
+export declare const evaluate: EvaluateFn;
+//# sourceMappingURL=evaluators.d.ts.map

package/lib/dist/lib/testCases/evaluators.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluators.d.ts","sourceRoot":"","sources":["../../../testCases/evaluators.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,KAAK,EAAa,OAAO,EAAE,MAAM,YAAY,CAAC;AAGrD,4CAA4C;AAC5C,MAAM,WAAW,gBAAgB;IAC/B,sCAAsC;IACtC,MAAM,EAAE,UAAU,CAAC;IACnB,kEAAkE;IAClE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wEAAwE;IACxE,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB;AAED,uCAAuC;AACvC,MAAM,WAAW,eAAe;IAC9B,gCAAgC;IAChC,IAAI,EAAE,OAAO,CAAC;IACd,wDAAwD;IACxD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uEAAuE;IACvE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,WAAW,GAAG,CACxB,GAAG,EAAE,gBAAgB,KAClB,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AAIhD;;;;;;;;;;;GAWG;AACH,wBAAgB,eAAe,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,GAAG,IAAI,CAgBjE;AAED,sDAAsD;AACtD,wBAAgB,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAEhE;AAED,mFAAmF;AACnF,wBAAgB,eAAe,IAAI,IAAI,CAEtC;AA6ED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/E,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACvF;AAED,eAAO,MAAM,QAAQ,EAAE,UAOtB,CAAC"}