@opensearch-project/agent-health 0.4.0 → 0.6.0

package/lib/dist/lib/testCases/agentFixture.d.ts

@@ -0,0 +1,80 @@
+/**
+ * The `agent` fixture — RFC 004 control inversion (#256, §4.1).
+ *
+ * Instead of the framework eagerly invoking the agent before the test body
+ * runs, the body drives invocation itself, exactly the way a Playwright test
+ * calls `await page.goto(...)`:
+ *
+ * ```js
+ * test('payment RCA', async ({ agent, expect, judge }) => {
+ *   const id = await seedTicket();                 // setup BEFORE the agent
+ *   const result = await agent.run(`Triage ${id}`);
+ *   expect(result).toHaveCalledTool('search_logs');
+ *   await judge(result, 'identifies the DB outage');
+ * });
+ * ```
+ *
+ * **Exactly one invocation per test (enforced).** The goal is to *benchmark*
+ * an agent: one test ⇒ one invocation ⇒ one comparable trajectory. A second
+ * `agent.run()` throws. Multi-turn conversations, if a connector models them,
+ * happen inside that single run — they are not multiple harness invocations.
+ *
+ * This module owns only the **contract** and the one-run guard. The actual
+ * connector invocation + RunResult construction (trajectory accessor, lazy
+ * traces, runId correlation) is supplied by the runner via the `invoke`
+ * callback, since that requires server-side connector/trace machinery.
+ */
+import type { EvalResult } from './types.js';
+/** Options accepted by `agent.run(prompt, options)`. */
+export interface AgentRunOptions {
+    /** Context items passed to the agent alongside the prompt. */
+    context?: {
+        description: string;
+        value: string;
+    }[];
+    /**
+     * Structured values forwarded to the connector's payload builder (e.g. a
+     * provisioned workspace dir). Stringly-typed env is the lowest common
+     * denominator every subprocess connector inherits automatically.
+     */
+    env?: Record<string, string>;
+}
+/**
+ * The single capability the framework hands the test body. `run()` is the
+ * agent equivalent of Playwright's `page.goto()`.
+ */
+export interface AgentFixture {
+    /**
+     * Invoke the agent once and return a fully-captured result (trajectory,
+     * output, runId, traces). Throws if called more than once in a test, or if
+     * no prompt is available (none passed and no `defaultPrompt` configured).
+     */
+    run(prompt?: string, options?: AgentRunOptions): Promise<EvalResult>;
+    /** True once `run()` has been called (read-only introspection). */
+    readonly invoked: boolean;
+}
+/** The runner-supplied invocation implementation. */
+export type AgentInvokeFn = (prompt: string, options?: AgentRunOptions) => Promise<EvalResult>;
+export interface CreateAgentFixtureOptions {
+    /**
+     * Prompt used when `agent.run()` is called with no argument. Mirrors the
+     * test's `defaultPrompt` (discovery/UI display). When neither is present,
+     * `run()` throws rather than invoking with an empty prompt.
+     */
+    defaultPrompt?: string;
+    /** Default context merged under any per-call context. */
+    defaultContext?: {
+        description: string;
+        value: string;
+    }[];
+}
+/**
+ * Build an {@link AgentFixture} around a runner-supplied {@link AgentInvokeFn}.
+ *
+ * Enforces the one-invocation-per-test rule and resolves the effective
+ * prompt/context (per-call argument wins over the test's `defaultPrompt` /
+ * `defaultContext`). The `invoke` callback does the real work (connector +
+ * RunResult wrapping) and is only ever called at most once.
+ */
+export declare function createAgentFixture(invoke: AgentInvokeFn, options?: CreateAgentFixtureOptions): AgentFixture;
+//# sourceMappingURL=agentFixture.d.ts.map

package/lib/dist/lib/testCases/agentFixture.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agentFixture.d.ts","sourceRoot":"","sources":["../../../testCases/agentFixture.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7C,wDAAwD;AACxD,MAAM,WAAW,eAAe;IAC9B,8DAA8D;IAC9D,OAAO,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACnD;;;;OAIG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACrE,mEAAmE;IACnE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,qDAAqD;AACrD,MAAM,MAAM,aAAa,GAAG,CAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,eAAe,KACtB,OAAO,CAAC,UAAU,CAAC,CAAC;AAEzB,MAAM,WAAW,yBAAyB;IACxC;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,yDAAyD;IACzD,cAAc,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC3D;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,aAAa,EACrB,OAAO,GAAE,yBAA8B,GACtC,YAAY,CAoCd"}

package/lib/dist/lib/testCases/agentFixture.js

@@ -0,0 +1,43 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Build an {@link AgentFixture} around a runner-supplied {@link AgentInvokeFn}.
+ *
+ * Enforces the one-invocation-per-test rule and resolves the effective
+ * prompt/context (per-call argument wins over the test's `defaultPrompt` /
+ * `defaultContext`). The `invoke` callback does the real work (connector +
+ * RunResult wrapping) and is only ever called at most once.
+ */
+export function createAgentFixture(invoke, options = {}) {
+    let invoked = false;
+    return {
+        get invoked() {
+            return invoked;
+        },
+        async run(prompt, runOptions) {
+            if (invoked) {
+                throw new Error('agent.run() may be called at most once per test. Benchmarking ' +
+                    'requires exactly one agent invocation per test so trajectories ' +
+                    'stay comparable across runs. For multi-turn conversations, model ' +
+                    'the turns inside a single run via your connector.');
+            }
+            const effectivePrompt = prompt ?? options.defaultPrompt;
+            if (effectivePrompt == null || effectivePrompt === '') {
+                throw new Error('agent.run() requires a prompt — none was passed and the test has ' +
+                    'no `defaultPrompt`. Pass a prompt: `await agent.run("...")`.');
+            }
+            // Mark invoked BEFORE awaiting so a concurrent/re-entrant second call
+            // (e.g. forgetting to await) still trips the guard deterministically.
+            invoked = true;
+            const mergedContext = [
+                ...(options.defaultContext ?? []),
+                ...(runOptions?.context ?? []),
+            ];
+            const context = mergedContext.length > 0 ? mergedContext : undefined;
+            return invoke(effectivePrompt, { ...runOptions, context });
+        },
+    };
+}
+//# sourceMappingURL=agentFixture.js.map

package/lib/dist/lib/testCases/agentFixture.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agentFixture.js","sourceRoot":"","sources":["../../../testCases/agentFixture.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA2EH;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAChC,MAAqB,EACrB,UAAqC,EAAE;IAEvC,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,OAAO;QACL,IAAI,OAAO;YACT,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,MAAe,EAAE,UAA4B;YACrD,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CACb,gEAAgE;oBAChE,iEAAiE;oBACjE,mEAAmE;oBACnE,mDAAmD,CACpD,CAAC;YACJ,CAAC;YACD,MAAM,eAAe,GAAG,MAAM,IAAI,OAAO,CAAC,aAAa,CAAC;YACxD,IAAI,eAAe,IAAI,IAAI,IAAI,eAAe,KAAK,EAAE,EAAE,CAAC;gBACtD,MAAM,IAAI,KAAK,CACb,mEAAmE;oBACnE,8DAA8D,CAC/D,CAAC;YACJ,CAAC;YACD,sEAAsE;YACtE,sEAAsE;YACtE,OAAO,GAAG,IAAI,CAAC;YAEf,MAAM,aAAa,GAAG;gBACpB,GAAG,CAAC,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC;gBACjC,GAAG,CAAC,UAAU,EAAE,OAAO,IAAI,EAAE,CAAC;aAC/B,CAAC;YACF,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC;YAErE,OAAO,MAAM,CAAC,eAAe,EAAE,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC;QAC7D,CAAC;KACF,CAAC;AACJ,CAAC"}

package/lib/dist/lib/testCases/authoringSurface.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Build the authoring-surface object handed to code-imported eval files.
+ * Must mirror the public `@opensearch-project/agent-health` package exports
+ * that a test body uses (the runtime registrars + matchers), so `.js` and
+ * `.ts`/`.mjs` files behave identically.
+ */
+export declare function getAuthoringSurface(): Record<string, unknown>;
+/** The stable list of authoring-surface export names (for drift tests). */
+export declare const AUTHORING_SURFACE_NAMES: string[];
+//# sourceMappingURL=authoringSurface.d.ts.map

package/lib/dist/lib/testCases/authoringSurface.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"authoringSurface.d.ts","sourceRoot":"","sources":["../../../testCases/authoringSurface.ts"],"names":[],"mappings":"AAkCA;;;;;GAKG;AACH,wBAAgB,mBAAmB,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAoB7D;AAED,2EAA2E;AAC3E,eAAO,MAAM,uBAAuB,UAA4C,CAAC"}

package/lib/dist/lib/testCases/authoringSurface.js

@@ -0,0 +1,54 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * The single source of truth for the SDK's **test-authoring surface** — the
+ * exact set of names a `.eval.{js,ts,mjs}` file gets when it imports
+ * `@opensearch-project/agent-health` (RFC 004 §4.7, #232).
+ *
+ * Why this exists: the loader binds two ways. `.ts`/`.mjs` files are
+ * `import()`-ed and resolve the real package exports; `.js` (CJS) files are
+ * executed in a synthetic context where `require('@opensearch-project/...')`
+ * is intercepted and handed a hand-built object. Historically that object was
+ * a hardcoded subset that silently drifted from the real package exports — so
+ * a new export (e.g. `defineEvaluator`) worked in `.ts` files but was
+ * `undefined` in `.js` files. Centralizing the surface here, and having BOTH
+ * the loader's CJS injection and the package index reference it, kills the
+ * drift: add a name once and every loader path sees it.
+ */
+import { test, describe, beforeAll, afterAll, beforeEach, afterEach, } from './define.js';
+import { judge } from './judge.js';
+import { defineEvaluator, evaluate } from './evaluators.js';
+import { expect } from '../matchers/expect.js';
+import { source, workflow } from '../workflow/index.js';
+/**
+ * Build the authoring-surface object handed to code-imported eval files.
+ * Must mirror the public `@opensearch-project/agent-health` package exports
+ * that a test body uses (the runtime registrars + matchers), so `.js` and
+ * `.ts`/`.mjs` files behave identically.
+ */
+export function getAuthoringSurface() {
+    return {
+        // Registrars (file-scoped singletons — registrations land in the
+        // loader's per-file registry).
+        test,
+        describe,
+        beforeAll,
+        afterAll,
+        beforeEach,
+        afterEach,
+        // Judge + custom evaluators.
+        judge,
+        defineEvaluator,
+        evaluate,
+        // Matcher API.
+        expect,
+        // Workflow SDK (for .workflow.{js,ts,mjs} files).
+        source,
+        workflow,
+    };
+}
+/** The stable list of authoring-surface export names (for drift tests). */
+export const AUTHORING_SURFACE_NAMES = Object.keys(getAuthoringSurface()).sort();
+//# sourceMappingURL=authoringSurface.js.map

package/lib/dist/lib/testCases/authoringSurface.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"authoringSurface.js","sourceRoot":"","sources":["../../../testCases/authoringSurface.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EACL,IAAI,EACJ,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,UAAU,EACV,SAAS,GACV,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC5D,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAExD;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB;IACjC,OAAO;QACL,iEAAiE;QACjE,+BAA+B;QAC/B,IAAI;QACJ,QAAQ;QACR,SAAS;QACT,QAAQ;QACR,UAAU;QACV,SAAS;QACT,6BAA6B;QAC7B,KAAK;QACL,eAAe;QACf,QAAQ;QACR,eAAe;QACf,MAAM;QACN,kDAAkD;QAClD,MAAM;QACN,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,2EAA2E;AAC3E,MAAM,CAAC,MAAM,uBAAuB,GAAG,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC"}

package/lib/dist/lib/testCases/codemod.d.ts

@@ -0,0 +1,13 @@
+export interface CodemodResult {
+    /** The transformed source (equals input when nothing changed). */
+    code: string;
+    /** Whether any edit was applied. */
+    changed: boolean;
+    /** Human-readable notes per test (migrated / skipped + why). */
+    notes: string[];
+}
+/**
+ * Run the codemod over a single file's source text.
+ */
+export declare function migrateEvalSource(code: string, fileName?: string): CodemodResult;
+//# sourceMappingURL=codemod.d.ts.map

package/lib/dist/lib/testCases/codemod.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"codemod.d.ts","sourceRoot":"","sources":["../../../testCases/codemod.ts"],"names":[],"mappings":"AAgCA,MAAM,WAAW,aAAa;IAC5B,kEAAkE;IAClE,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,gEAAgE;IAChE,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AA0CD;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,SAAY,GAAG,aAAa,CA4HnF"}

package/lib/dist/lib/testCases/codemod.js

@@ -0,0 +1,169 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Codemod: migrate v1 code-SDK eval files to the RFC-004 control-inversion
+ * shape (#256, phase 10 tooling).
+ *
+ * v1 bodies received a pre-populated `result` fixture (the framework invoked
+ * the agent eagerly). v2 bodies own invocation: `const result = await
+ * agent.run()`. This transform rewrites each `test(name, options, body)`
+ * whose body destructures `result` AND whose options carry a `prompt`:
+ *
+ *   test('x', { prompt: 'p' }, function ({ result, judge }) {  ...result... })
+ *     ↓
+ *   test('x', { prompt: 'p' }, async ({ agent, judge }) => {
+ *     const result = await agent.run();
+ *     ...result...
+ *   })
+ *
+ * Tests with no `prompt` (data-only) are left untouched — their `result` is
+ * the empty placeholder, which is still valid. Bodies that already call
+ * `agent.run()` are skipped (idempotent).
+ *
+ * Uses the TypeScript compiler API for a precise, formatting-preserving
+ * transform via plain text-range edits (no full reprint), so it works on
+ * both `.eval.js` and `.eval.ts`.
+ */
+import * as ts from 'typescript';
+/** Is this call expression a `test(...)` / `test.only(...)` registration? */
+function isTestCall(node) {
+    const e = node.expression;
+    if (ts.isIdentifier(e))
+        return e.text === 'test';
+    if (ts.isPropertyAccessExpression(e) && ts.isIdentifier(e.expression)) {
+        return e.expression.text === 'test';
+    }
+    return false;
+}
+/** Extract the options-object literal argument (2nd arg) if present. */
+function optionsArg(call) {
+    const a = call.arguments[1];
+    return a && ts.isObjectLiteralExpression(a) ? a : undefined;
+}
+/** Does the options object declare a non-empty `prompt`? */
+function hasPrompt(opts) {
+    if (!opts)
+        return false;
+    return opts.properties.some((p) => ts.isPropertyAssignment(p) &&
+        ((ts.isIdentifier(p.name) && p.name.text === 'prompt') ||
+            (ts.isStringLiteral(p.name) && p.name.text === 'prompt')));
+}
+/** The body is the last argument when it's a function/arrow. */
+function bodyArg(call) {
+    const last = call.arguments[call.arguments.length - 1];
+    if (last && (ts.isFunctionExpression(last) || ts.isArrowFunction(last)))
+        return last;
+    return undefined;
+}
+/**
+ * Run the codemod over a single file's source text.
+ */
+export function migrateEvalSource(code, fileName = 'eval.ts') {
+    const sf = ts.createSourceFile(fileName, code, ts.ScriptTarget.Latest, 
+    /*setParentNodes*/ true, fileName.endsWith('.ts') ? ts.ScriptKind.TS : ts.ScriptKind.JS);
+    const edits = [];
+    const notes = [];
+    const visit = (node) => {
+        if (ts.isCallExpression(node) && isTestCall(node)) {
+            processTest(node);
+        }
+        ts.forEachChild(node, visit);
+    };
+    function testName(call) {
+        const a = call.arguments[0];
+        return a && ts.isStringLiteralLike(a) ? a.text : '<anonymous>';
+    }
+    function processTest(call) {
+        const name = testName(call);
+        const body = bodyArg(call);
+        if (!body)
+            return; // no function body (e.g. test(name) stub)
+        const param = body.parameters[0];
+        // Only the object-destructuring fixture form is migrated.
+        if (!param || !ts.isObjectBindingPattern(param.name)) {
+            return;
+        }
+        const binding = param.name;
+        const names = binding.elements
+            .map((el) => (ts.isIdentifier(el.name) ? el.name.text : ''))
+            .filter(Boolean);
+        const usesResult = names.includes('result');
+        const usesAgent = names.includes('agent');
+        // Idempotency / applicability guards.
+        const bodyText = body.body ? body.body.getText(sf) : '';
+        const hasAgentRun = /\bagent\s*\.\s*run\s*\(/.test(bodyText);
+        if (hasAgentRun && !usesResult) {
+            // Fully migrated: calls agent.run() and no longer destructures `result`.
+            notes.push(`skip  ${name}: already migrated (calls agent.run(), no result binding)`);
+            return;
+        }
+        if (hasAgentRun && usesResult) {
+            // Half-migrated: agent.run() is in, but `result` is still destructured
+            // (and almost certainly still referenced). The runner hands the body an
+            // empty placeholder `result` when `agent` is destructured separately, so
+            // those `result.*` reads silently read empties and assertions fail — the
+            // migration *looks* broken when it isn't. We can't safely rename those
+            // reads to the agent.run() return value, so surface for manual review
+            // instead of silently reporting "already migrated".
+            notes.push(`review ${name}: half-migrated — calls agent.run() but still destructures \`result\`. ` +
+                `Capture the agent.run() return value (\`const result = await agent.run()\`) and drop the \`result\` fixture.`);
+            return;
+        }
+        if (!hasPrompt(optionsArg(call))) {
+            notes.push(`skip  ${name}: no prompt (data-only test, result placeholder is valid)`);
+            return;
+        }
+        if (!usesResult) {
+            notes.push(`skip  ${name}: body does not use result`);
+            return;
+        }
+        // 1) Rewrite the destructuring: replace `result` with `agent` (or add
+        //    `agent` if result+agent both somehow absent). Keep other fixtures.
+        const newNames = names.map((n) => (n === 'result' ? 'agent' : n));
+        if (usesAgent) {
+            // agent already destructured — just drop `result`.
+            const filtered = names.filter((n) => n !== 'result');
+            newNames.length = 0;
+            newNames.push(...filtered);
+        }
+        const newBinding = `{ ${newNames.join(', ')} }`;
+        edits.push({ start: binding.getStart(sf), end: binding.getEnd(), text: newBinding });
+        // 2) Ensure the function is async.
+        const isAsync = body.modifiers?.some((m) => m.kind === ts.SyntaxKind.AsyncKeyword);
+        if (!isAsync) {
+            edits.push({ start: body.getStart(sf), end: body.getStart(sf), text: 'async ' });
+        }
+        // 3) Insert `const result = await agent.run();` at the top of the body.
+        //    Handle both block bodies `{ ... }` and expression arrow bodies.
+        if (body.body && ts.isBlock(body.body)) {
+            const openBrace = body.body.getStart(sf) + 1; // just after `{`
+            edits.push({ start: openBrace, end: openBrace, text: `\n  const result = await agent.run();` });
+        }
+        else if (body.body) {
+            // Expression-bodied arrow: wrap into a block.
+            const exprStart = body.body.getStart(sf);
+            const exprEnd = body.body.getEnd();
+            const exprText = body.body.getText(sf);
+            edits.push({
+                start: exprStart,
+                end: exprEnd,
+                text: `{\n  const result = await agent.run();\n  return ${exprText};\n}`,
+            });
+        }
+        notes.push(`migrate ${name}: result → agent.run()`);
+    }
+    visit(sf);
+    if (edits.length === 0) {
+        return { code, changed: false, notes };
+    }
+    // Apply edits right-to-left so offsets stay valid.
+    edits.sort((a, b) => b.start - a.start);
+    let out = code;
+    for (const e of edits) {
+        out = out.slice(0, e.start) + e.text + out.slice(e.end);
+    }
+    return { code: out, changed: true, notes };
+}
+//# sourceMappingURL=codemod.js.map

package/lib/dist/lib/testCases/codemod.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"codemod.js","sourceRoot":"","sources":["../../../testCases/codemod.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,MAAM,YAAY,CAAC;AAiBjC,6EAA6E;AAC7E,SAAS,UAAU,CAAC,IAAuB;IACzC,MAAM,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;IAC1B,IAAI,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;IACjD,IAAI,EAAE,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;QACtE,OAAO,CAAC,CAAC,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC;IACtC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,wEAAwE;AACxE,SAAS,UAAU,CAAC,IAAuB;IACzC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,IAAI,EAAE,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC9D,CAAC;AAED,4DAA4D;AAC5D,SAAS,SAAS,CAAC,IAA4C;IAC7D,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CACzB,CAAC,CAAC,EAAE,EAAE,CACJ,EAAE,CAAC,oBAAoB,CAAC,CAAC,CAAC;QAC1B,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC;YACpD,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAC9D,CAAC;AACJ,CAAC;AAED,gEAAgE;AAChE,SAAS,OAAO,CAAC,IAAuB;IACtC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACrF,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,QAAQ,GAAG,SAAS;IAClE,MAAM,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAC5B,QAAQ,EACR,IAAI,EACJ,EAAE,CAAC,YAAY,CAAC,MAAM;IACtB,kBAAkB,CAAC,IAAI,EACvB,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAC/D,CAAC;IAEF,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,KAAK,GAAG,CAAC,IAAa,EAAE,EAAE;QAC9B,IAAI,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAClD,WAAW,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;QACD,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC/B,CAAC,CAAC;IAEF,SAAS,QAAQ,CAAC,IAAuB;QACvC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,IAAI,EAAE,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC;IACjE,CAAC;IAED,SAAS,WAAW,CAAC,IAAuB;QAC1C,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC3B,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,0CAA0C;QAE7D,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACjC,0DAA0D;QAC1D,IAAI,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC,sBAAsB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACrD,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC;QAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ;aAC3B,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;aAC3D,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAE1C,sCAAsC;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,MAAM,WAAW,GAAG,yBAAyB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7D,IAAI,WAAW,IAAI,CAAC,UAAU,EAAE,CAAC;YAC/B,yEAAyE;YACzE,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,2DAA2D,CAAC,CAAC;YACrF,OAAO;QACT,CAAC;QACD,IAAI,WAAW,IAAI,UAAU,EAAE,CAAC;YAC9B,uEAAuE;YACvE,wEAAwE;YACxE,yEAAyE;YACzE,yEAAyE;YACzE,uEAAuE;YACvE,sEAAsE;YACtE,oDAAoD;YACpD,KAAK,CAAC,IAAI,CACR,UAAU,IAAI,yEAAyE;gBACrF,8GAA8G,CACjH,CAAC;YACF,OAAO;QACT,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,2DAA2D,CAAC,CAAC;YACrF,OAAO;QACT,CAAC;QACD,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,4BAA4B,CAAC,CAAC;YACtD,OAAO;QACT,CAAC;QAED,sEAAsE;QACtE,wEAAwE;QACxE,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,IAAI,SAAS,EAAE,CAAC;YACd,mDAAmD;YACnD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC;YACrD,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YACpB,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;QAC7B,CAAC;QACD,MAAM,UAAU,GAAG,KAAK,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QAErF,mCAAmC;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QACnF,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QACnF,CAAC;QAED,wEAAwE;QACxE,qEAAqE;QACrE,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACvC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,iBAAiB;YAC/D,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,uCAAuC,EAAE,CAAC,CAAC;QAClG,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACrB,8CAA8C;YAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC;gBACT,KAAK,EAAE,SAAS;gBAChB,GAAG,EAAE,OAAO;gBACZ,IAAI,EAAE,oDAAoD,QAAQ,MAAM;aACzE,CAAC,CAAC;QACL,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,WAAW,IAAI,wBAAwB,CAAC,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,EAAE,CAAC,CAAC;IAEV,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IACzC,CAAC;IAED,mDAAmD;IACnD,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACxC,IAAI,GAAG,GAAG,IAAI,CAAC;IACf,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AAC7C,CAAC"}

package/lib/dist/lib/testCases/define.d.ts

@@ -0,0 +1,114 @@
+import type { CodeTestCase, TestOptions, EvalResult, HookFn, RegisteredHook } from './types.js';
+/** @internal */
+export declare function _resetExperimentalWarning(): void;
+export declare function setActiveFile(filePath: string): void;
+/**
+ * Register a code-based test case.
+ *
+ * Two valid signatures (Playwright-style):
+ * - `test(name, body)` — no options at all
+ * - `test(name, options, body)` — with options
+ *
+ * Only `name` is required. All TestOptions fields are optional. When
+ * `options.prompt` is absent, the runner skips agent invocation and the
+ * body receives an empty EvalResult.
+ *
+ * Throws if a test with the same name is already registered in the same
+ * source file. Cross-file duplicates are allowed (storage identity is
+ * `name + sourceFile`).
+ *
+ * @experimental The SDK shape (signature, options, body fixtures) may change
+ * in a minor release without a deprecation cycle. See `lib/index.ts`.
+ */
+export declare function test(name: string, body: (result: EvalResult) => Promise<void> | void): void;
+export declare function test(name: string, options: TestOptions, body: (result: EvalResult) => Promise<void> | void): void;
+/**
+ * Group tests under a benchmark name. Equivalent to Playwright's
+ * `describe()` — the wrapped `test()` calls inherit the describe's name as
+ * their benchmark group. Nested describes flatten with ' > '.
+ *
+ * @example
+ *   describe('RCA Suite', () => {
+ *     test('payment-service is the root cause', { prompt: ... }, async ({ result, judge }) => {
+ *       expect(result.trajectory).to.haveCalledTool('search_logs');
+ *       await judge(result, 'identifies the failing dependency');
+ *     });
+ *   });
+ *
+ * The describe body MUST be synchronous (no `await`/dynamic content), like
+ * Playwright. The function is invoked once at registration time.
+ */
+export declare function describe(name: string, fn: () => void): void;
+export declare function getRegisteredTests(filePath?: string): CodeTestCase[];
+/**
+ * Return all hooks registered for the given file (or every file when no
+ * argument is given). Returns a snapshot — mutating the result has no
+ * effect on the registry.
+ */
+export declare function getRegisteredHooks(filePath?: string): RegisteredHook[];
+export declare function clearRegistry(filePath?: string): void;
+/**
+ * Register a hook that runs **once** before the first test in its scope.
+ *
+ * Scope is the surrounding `describe(...)` block, or the whole file when
+ * called at the top level. With parallel test execution (the runner
+ * dispatches up to `concurrency` tests at once), the orchestrator uses a
+ * once-latch so all parallel arrivals await the same `beforeAll` promise.
+ *
+ * @example
+ *   beforeAll(async () => {
+ *     await fs.mkdir('/tmp/agent-health-fixtures', { recursive: true });
+ *   });
+ */
+export declare function beforeAll(fn: HookFn): void;
+/**
+ * Register a hook that runs **once** after the last test in its scope.
+ *
+ * Always runs, even when every test in the scope failed. The orchestrator
+ * uses a remaining-test counter (decremented on each test completion,
+ * regardless of pass/fail) and triggers `afterAll` when it hits zero.
+ *
+ * @example
+ *   afterAll(async () => {
+ *     await fs.rm('/tmp/agent-health-fixtures', { recursive: true });
+ *   });
+ */
+export declare function afterAll(fn: HookFn): void;
+/**
+ * Register a hook that runs **before each test** in its scope.
+ *
+ * Receives the same fixtures object the body will see, plus a
+ * `provide(key, value)` function for stashing values that the test body
+ * (and `afterEach`) can read via `fixtures.provisioned[key]`. Each test
+ * gets its own provisioned bag, so concurrent tests are isolated.
+ *
+ * @example
+ *   beforeEach(async ({ provide, testInfo }) => {
+ *     const dir = await fs.mkdtemp(`/tmp/${testInfo.name}-`);
+ *     provide('workspaceDir', dir);
+ *   });
+ */
+export declare function beforeEach(fn: HookFn): void;
+/**
+ * Register a hook that runs **after each test** in its scope.
+ *
+ * Always runs, even when the test body or a `beforeEach` threw. Reads
+ * provisioned values via `fixtures.provisioned[key]` for cleanup. Errors
+ * thrown from `afterEach` are captured as MatcherResult entries on the
+ * test — they don't crash the runner.
+ *
+ * @example
+ *   afterEach(async ({ provisioned }) => {
+ *     if (provisioned.workspaceDir) {
+ *       await fs.rm(provisioned.workspaceDir as string, { recursive: true, force: true });
+ *     }
+ *   });
+ */
+export declare function afterEach(fn: HookFn): void;
+export declare namespace test {
+    const beforeAll: (fn: HookFn) => void;
+    const afterAll: (fn: HookFn) => void;
+    const beforeEach: (fn: HookFn) => void;
+    const afterEach: (fn: HookFn) => void;
+}
+//# sourceMappingURL=define.d.ts.map

package/lib/dist/lib/testCases/define.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"define.d.ts","sourceRoot":"","sources":["../../../testCases/define.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,YAAY,EACZ,WAAW,EACX,UAAU,EACV,MAAM,EAEN,cAAc,EACf,MAAM,YAAY,CAAC;AAiCpB,gBAAgB;AAChB,wBAAgB,yBAAyB,IAAI,IAAI,CAEhD;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAQpD;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,IAAI,CAClB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,GACjD,IAAI,CAAC;AACR,wBAAgB,IAAI,CAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,WAAW,EACpB,IAAI,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,GACjD,IAAI,CAAC;AAuDR;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,IAAI,GAAG,IAAI,CAsB3D;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,YAAY,EAAE,CAGpE;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,cAAc,EAAE,CAGtE;AAED,wBAAgB,aAAa,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CASrD;AA4BD;;;;;;;;;;;;GAYG;AACH,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAmC;AAE9E;;;;;;;;;;;GAWG;AACH,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAkC;AAE5E;;;;;;;;;;;;;GAaG;AACH,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAoC;AAEhF;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAmC;AAS9E,yBAAiB,IAAI,CAAC;IAEb,MAAM,SAAS,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAA4C,CAAC;IAE9E,MAAM,QAAQ,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAA2C,CAAC;IAE5E,MAAM,UAAU,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAA6C,CAAC;IAEhF,MAAM,SAAS,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAA4C,CAAC;CACtF"}