@opensearch-project/agent-health 0.3.0 → 0.5.2

package/lib/dist/lib/testCaseValidation.d.ts

@@ -0,0 +1,140 @@
+import { z } from 'zod';
+import type { AgentContextItem } from '../types/index.js';
+import type { CreateTestCaseInput } from '../services/storage/index.js';
+/**
+ * Zod schema for validating test case JSON input.
+ * This validates a subset of CreateTestCaseInput fields that are relevant for the JSON editor.
+ * The output is compatible with CreateTestCaseInput.
+ */
+export declare const testCaseSchema: z.ZodObject<{
+    name: z.ZodString;
+    description: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+    category: z.ZodString;
+    subcategory: z.ZodOptional<z.ZodString>;
+    difficulty: z.ZodEnum<["Easy", "Medium", "Hard"]>;
+    initialPrompt: z.ZodString;
+    context: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodObject<{
+        description: z.ZodString;
+        value: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        description?: string;
+        value?: string;
+    }, {
+        description?: string;
+        value?: string;
+    }>, "many">>>;
+    expectedOutcomes: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString, "many">>>;
+}, "strip", z.ZodTypeAny, {
+    name?: string;
+    description?: string;
+    initialPrompt?: string;
+    context?: {
+        description?: string;
+        value?: string;
+    }[];
+    expectedOutcomes?: string[];
+    category?: string;
+    subcategory?: string;
+    difficulty?: "Easy" | "Medium" | "Hard";
+}, {
+    name?: string;
+    description?: string;
+    initialPrompt?: string;
+    context?: {
+        description?: string;
+        value?: string;
+    }[];
+    expectedOutcomes?: string[];
+    category?: string;
+    subcategory?: string;
+    difficulty?: "Easy" | "Medium" | "Hard";
+}>;
+export declare const testCasesArraySchema: z.ZodArray<z.ZodObject<{
+    name: z.ZodString;
+    description: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+    category: z.ZodString;
+    subcategory: z.ZodOptional<z.ZodString>;
+    difficulty: z.ZodEnum<["Easy", "Medium", "Hard"]>;
+    initialPrompt: z.ZodString;
+    context: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodObject<{
+        description: z.ZodString;
+        value: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        description?: string;
+        value?: string;
+    }, {
+        description?: string;
+        value?: string;
+    }>, "many">>>;
+    expectedOutcomes: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString, "many">>>;
+}, "strip", z.ZodTypeAny, {
+    name?: string;
+    description?: string;
+    initialPrompt?: string;
+    context?: {
+        description?: string;
+        value?: string;
+    }[];
+    expectedOutcomes?: string[];
+    category?: string;
+    subcategory?: string;
+    difficulty?: "Easy" | "Medium" | "Hard";
+}, {
+    name?: string;
+    description?: string;
+    initialPrompt?: string;
+    context?: {
+        description?: string;
+        value?: string;
+    }[];
+    expectedOutcomes?: string[];
+    category?: string;
+    subcategory?: string;
+    difficulty?: "Easy" | "Medium" | "Hard";
+}>, "many">;
+/**
+ * The validated test case input from JSON.
+ * This is a subset of CreateTestCaseInput with only the fields the JSON editor handles.
+ * It's structurally compatible with CreateTestCaseInput for use with asyncTestCaseStorage.
+ */
+export type ValidatedTestCaseInput = Pick<CreateTestCaseInput, 'name' | 'description' | 'category' | 'subcategory' | 'difficulty' | 'initialPrompt' | 'context' | 'expectedOutcomes'>;
+export interface TestCaseFormState {
+    name: string;
+    description: string;
+    category: string;
+    subcategory: string;
+    difficulty: 'Easy' | 'Medium' | 'Hard';
+    initialPrompt: string;
+    context: AgentContextItem[];
+    expectedOutcomes: string[];
+}
+export interface ValidationError {
+    path: string;
+    message: string;
+    type: 'error' | 'warning';
+}
+export interface ValidationResult<T = unknown> {
+    valid: boolean;
+    errors: ValidationError[];
+    data?: T;
+}
+/**
+ * Validates a single test case JSON object.
+ * Returns data compatible with CreateTestCaseInput.
+ */
+export declare function validateTestCaseJson(json: unknown): ValidationResult<ValidatedTestCaseInput>;
+/**
+ * Validates an array of test cases for bulk import.
+ * Also handles single objects by wrapping them in an array.
+ * Returns data compatible with CreateTestCaseInput[].
+ */
+export declare function validateTestCasesArrayJson(json: unknown): ValidationResult<ValidatedTestCaseInput[]>;
+/**
+ * Serializes form state to JSON string for the editor.
+ */
+export declare function serializeFormToJson(formState: TestCaseFormState): string;
+/**
+ * Parses JSON string and returns form state if valid.
+ */
+export declare function parseJsonToFormState(jsonString: string): ValidationResult<TestCaseFormState>;
+//# sourceMappingURL=testCaseValidation.d.ts.map

package/lib/dist/lib/testCaseValidation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"testCaseValidation.d.ts","sourceRoot":"","sources":["../../testCaseValidation.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAa9D;;;;GAIG;AACH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAyBzB,CAAC;AAEH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;WAA0D,CAAC;AAI5F;;;;GAIG;AACH,MAAM,MAAM,sBAAsB,GAAG,IAAI,CACvC,mBAAmB,EACnB,MAAM,GAAG,aAAa,GAAG,UAAU,GAAG,aAAa,GAAG,YAAY,GAAG,eAAe,GAAG,SAAS,GAAG,kBAAkB,CACtH,CAAC;AAGF,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B;AAID,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,OAAO,GAAG,SAAS,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB,CAAC,CAAC,GAAG,OAAO;IAC3C,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,IAAI,CAAC,EAAE,CAAC,CAAC;CACV;AAcD;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,OAAO,GAAG,gBAAgB,CAAC,sBAAsB,CAAC,CAoB5F;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,OAAO,GAAG,gBAAgB,CAAC,sBAAsB,EAAE,CAAC,CAyBpG;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,iBAAiB,GAAG,MAAM,CAuBxE;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,MAAM,GAAG,gBAAgB,CAAC,iBAAiB,CAAC,CAkC5F"}

package/lib/dist/lib/testCaseValidation.js

@@ -0,0 +1,162 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { z } from 'zod';
+// ============ Zod Schemas ============
+const contextItemSchema = z
+    .object({
+    description: z.string(),
+    value: z.string(),
+})
+    .required();
+const difficultySchema = z.enum(['Easy', 'Medium', 'Hard']);
+/**
+ * Zod schema for validating test case JSON input.
+ * This validates a subset of CreateTestCaseInput fields that are relevant for the JSON editor.
+ * The output is compatible with CreateTestCaseInput.
+ */
+export const testCaseSchema = z.object({
+    name: z.string().min(1, 'Name is required'),
+    description: z.string().optional().default(''),
+    category: z.string().min(1, 'Category is required'),
+    subcategory: z.string().optional(),
+    difficulty: difficultySchema,
+    initialPrompt: z.string().min(1, 'Initial prompt is required'),
+    context: z.array(contextItemSchema).optional().default([]),
+    // expectedOutcomes is optional. SDK code-only tests (`test()` blocks
+    // that drive pass/fail via deterministic body or inline `judge()`
+    // calls) have no expectedOutcomes — so requiring "at least one
+    // non-empty" here would force UI-created cross-surface-equivalent test
+    // cases to invent placeholder outcomes that the runner ignores. The
+    // server-side judge contract does still require expectedOutcomes when
+    // it actually grades a run (see services/evaluation/evaluatorError.ts
+    // — issue #242 surfaces "Missing required field" as an evaluator
+    // error), but the test case schema itself shouldn't gate authoring.
+    //
+    // `expectedTrajectory` is intentionally NOT modelled here. It exists on
+    // the persisted `TestCase` shape (set by the SDK loader from `test()`
+    // options) but the JSON-paste / form authoring path doesn't surface it,
+    // and adding it here without plumbing it through `ValidatedTestCaseInput`
+    // / `TestCaseFormState` / the editor save path would silently drop it on
+    // round-trip (PR #258 review feedback).
+    expectedOutcomes: z.array(z.string()).optional().default([]),
+});
+export const testCasesArraySchema = z.array(testCaseSchema).min(1, 'Array cannot be empty');
+// ============ Helper Functions ============
+function zodErrorToValidationErrors(error) {
+    return error.errors.map((e) => ({
+        path: e.path.join('.'),
+        message: e.message,
+        type: 'error',
+    }));
+}
+// ============ Validation Functions ============
+/**
+ * Validates a single test case JSON object.
+ * Returns data compatible with CreateTestCaseInput.
+ */
+export function validateTestCaseJson(json) {
+    // Handle array case - should use bulk import
+    if (Array.isArray(json)) {
+        return {
+            valid: false,
+            errors: [{ path: '', message: 'Received an array. Use Bulk Import mode for multiple test cases', type: 'error' }],
+        };
+    }
+    const result = testCaseSchema.safeParse(json);
+    if (!result.success) {
+        return {
+            valid: false,
+            errors: zodErrorToValidationErrors(result.error),
+        };
+    }
+    // Cast is safe here - Zod has validated all required fields
+    return { valid: true, errors: [], data: result.data };
+}
+/**
+ * Validates an array of test cases for bulk import.
+ * Also handles single objects by wrapping them in an array.
+ * Returns data compatible with CreateTestCaseInput[].
+ */
+export function validateTestCasesArrayJson(json) {
+    // Handle single object - auto-wrap in array
+    if (json && typeof json === 'object' && !Array.isArray(json)) {
+        const singleResult = validateTestCaseJson(json);
+        if (singleResult.valid && singleResult.data) {
+            return {
+                valid: true,
+                errors: [],
+                data: [singleResult.data],
+            };
+        }
+        return { valid: false, errors: singleResult.errors };
+    }
+    const result = testCasesArraySchema.safeParse(json);
+    if (!result.success) {
+        return {
+            valid: false,
+            errors: zodErrorToValidationErrors(result.error),
+        };
+    }
+    // Cast is safe here - Zod has validated all required fields
+    return { valid: true, errors: [], data: result.data };
+}
+/**
+ * Serializes form state to JSON string for the editor.
+ */
+export function serializeFormToJson(formState) {
+    const json = {
+        name: formState.name,
+        category: formState.category,
+        difficulty: formState.difficulty,
+        initialPrompt: formState.initialPrompt,
+        expectedOutcomes: formState.expectedOutcomes.filter((o) => o.trim()),
+    };
+    // Only include optional fields if they have values
+    if (formState.description.trim()) {
+        json.description = formState.description;
+    }
+    if (formState.subcategory.trim()) {
+        json.subcategory = formState.subcategory;
+    }
+    if (formState.context.length > 0) {
+        json.context = formState.context;
+    }
+    return JSON.stringify(json, null, 2);
+}
+/**
+ * Parses JSON string and returns form state if valid.
+ */
+export function parseJsonToFormState(jsonString) {
+    let parsed;
+    try {
+        parsed = JSON.parse(jsonString);
+    }
+    catch (e) {
+        return {
+            valid: false,
+            errors: [{ path: '', message: `Invalid JSON: ${e.message}`, type: 'error' }],
+        };
+    }
+    const validation = validateTestCaseJson(parsed);
+    if (!validation.valid || !validation.data) {
+        return { valid: false, errors: validation.errors };
+    }
+    const data = validation.data;
+    return {
+        valid: true,
+        errors: [],
+        data: {
+            name: data.name,
+            description: data.description || '',
+            category: data.category,
+            subcategory: data.subcategory || '',
+            difficulty: data.difficulty,
+            initialPrompt: data.initialPrompt,
+            context: (data.context || []),
+            expectedOutcomes: data.expectedOutcomes.length > 0 ? data.expectedOutcomes : [''],
+        },
+    };
+}
+//# sourceMappingURL=testCaseValidation.js.map

package/lib/dist/lib/testCaseValidation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"testCaseValidation.js","sourceRoot":"","sources":["../../testCaseValidation.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,wCAAwC;AAExC,MAAM,iBAAiB,GAAG,CAAC;KACxB,MAAM,CAAC;IACN,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;CAClB,CAAC;KACD,QAAQ,EAAE,CAAC;AAEd,MAAM,gBAAgB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;AAE5D;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,kBAAkB,CAAC;IAC3C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC9C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,sBAAsB,CAAC;IACnD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,UAAU,EAAE,gBAAgB;IAC5B,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,4BAA4B,CAAC;IAC9D,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC1D,qEAAqE;IACrE,kEAAkE;IAClE,+DAA+D;IAC/D,uEAAuE;IACvE,oEAAoE;IACpE,sEAAsE;IACtE,sEAAsE;IACtE,iEAAiE;IACjE,oEAAoE;IACpE,EAAE;IACF,wEAAwE;IACxE,sEAAsE;IACtE,wEAAwE;IACxE,0EAA0E;IAC1E,yEAAyE;IACzE,wCAAwC;IACxC,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;CAC7D,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC;AAwC5F,6CAA6C;AAE7C,SAAS,0BAA0B,CAAC,KAAiB;IACnD,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9B,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QACtB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,IAAI,EAAE,OAAgB;KACvB,CAAC,CAAC,CAAC;AACN,CAAC;AAED,iDAAiD;AAEjD;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAa;IAChD,6CAA6C;IAC7C,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,iEAAiE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;SAClH,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAE9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,0BAA0B,CAAC,MAAM,CAAC,KAAK,CAAC;SACjD,CAAC;IACJ,CAAC;IAED,4DAA4D;IAC5D,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAA8B,EAAE,CAAC;AAClF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,0BAA0B,CAAC,IAAa;IACtD,4CAA4C;IAC5C,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7D,MAAM,YAAY,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,YAAY,CAAC,KAAK,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC;YAC5C,OAAO;gBACL,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,EAAE;gBACV,IAAI,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC;aAC1B,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC;IACvD,CAAC;IAED,MAAM,MAAM,GAAG,oBAAoB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAEpD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,0BAA0B,CAAC,MAAM,CAAC,KAAK,CAAC;SACjD,CAAC;IACJ,CAAC;IAED,4DAA4D;IAC5D,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAgC,EAAE,CAAC;AACpF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,SAA4B;IAC9D,MAAM,IAAI,GAA4B;QACpC,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,QAAQ,EAAE,SAAS,CAAC,QAAQ;QAC5B,UAAU,EAAE,SAAS,CAAC,UAAU;QAChC,aAAa,EAAE,SAAS,CAAC,aAAa;QACtC,gBAAgB,EAAE,SAAS,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;KACrE,CAAC;IAEF,mDAAmD;IACnD,IAAI,SAAS,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;QACjC,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC;IAC3C,CAAC;IAED,IAAI,SAAS,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;QACjC,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC;IAC3C,CAAC;IAED,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC;IACnC,CAAC;IAED,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,UAAkB;IACrD,IAAI,MAAe,CAAC;IAEpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,iBAAkB,CAAW,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;SACxF,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAEhD,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1C,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;IACrD,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC;IAE7B,OAAO;QACL,KAAK,EAAE,IAAI;QACX,MAAM,EAAE,EAAE;QACV,IAAI,EAAE;YACJ,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE;YACnC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE;YACnC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,OAAO,EAAE,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAuB;YACnD,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SAClF;KACF,CAAC;AACJ,CAAC"}

package/lib/dist/lib/testCases/agentFixture.d.ts

@@ -0,0 +1,80 @@
+/**
+ * The `agent` fixture — RFC 004 control inversion (#256, §4.1).
+ *
+ * Instead of the framework eagerly invoking the agent before the test body
+ * runs, the body drives invocation itself, exactly the way a Playwright test
+ * calls `await page.goto(...)`:
+ *
+ * ```js
+ * test('payment RCA', async ({ agent, expect, judge }) => {
+ *   const id = await seedTicket();                 // setup BEFORE the agent
+ *   const result = await agent.run(`Triage ${id}`);
+ *   expect(result).toHaveCalledTool('search_logs');
+ *   await judge(result, 'identifies the DB outage');
+ * });
+ * ```
+ *
+ * **Exactly one invocation per test (enforced).** The goal is to *benchmark*
+ * an agent: one test ⇒ one invocation ⇒ one comparable trajectory. A second
+ * `agent.run()` throws. Multi-turn conversations, if a connector models them,
+ * happen inside that single run — they are not multiple harness invocations.
+ *
+ * This module owns only the **contract** and the one-run guard. The actual
+ * connector invocation + RunResult construction (trajectory accessor, lazy
+ * traces, runId correlation) is supplied by the runner via the `invoke`
+ * callback, since that requires server-side connector/trace machinery.
+ */
+import type { EvalResult } from './types.js';
+/** Options accepted by `agent.run(prompt, options)`. */
+export interface AgentRunOptions {
+    /** Context items passed to the agent alongside the prompt. */
+    context?: {
+        description: string;
+        value: string;
+    }[];
+    /**
+     * Structured values forwarded to the connector's payload builder (e.g. a
+     * provisioned workspace dir). Stringly-typed env is the lowest common
+     * denominator every subprocess connector inherits automatically.
+     */
+    env?: Record<string, string>;
+}
+/**
+ * The single capability the framework hands the test body. `run()` is the
+ * agent equivalent of Playwright's `page.goto()`.
+ */
+export interface AgentFixture {
+    /**
+     * Invoke the agent once and return a fully-captured result (trajectory,
+     * output, runId, traces). Throws if called more than once in a test, or if
+     * no prompt is available (none passed and no `defaultPrompt` configured).
+     */
+    run(prompt?: string, options?: AgentRunOptions): Promise<EvalResult>;
+    /** True once `run()` has been called (read-only introspection). */
+    readonly invoked: boolean;
+}
+/** The runner-supplied invocation implementation. */
+export type AgentInvokeFn = (prompt: string, options?: AgentRunOptions) => Promise<EvalResult>;
+export interface CreateAgentFixtureOptions {
+    /**
+     * Prompt used when `agent.run()` is called with no argument. Mirrors the
+     * test's `defaultPrompt` (discovery/UI display). When neither is present,
+     * `run()` throws rather than invoking with an empty prompt.
+     */
+    defaultPrompt?: string;
+    /** Default context merged under any per-call context. */
+    defaultContext?: {
+        description: string;
+        value: string;
+    }[];
+}
+/**
+ * Build an {@link AgentFixture} around a runner-supplied {@link AgentInvokeFn}.
+ *
+ * Enforces the one-invocation-per-test rule and resolves the effective
+ * prompt/context (per-call argument wins over the test's `defaultPrompt` /
+ * `defaultContext`). The `invoke` callback does the real work (connector +
+ * RunResult wrapping) and is only ever called at most once.
+ */
+export declare function createAgentFixture(invoke: AgentInvokeFn, options?: CreateAgentFixtureOptions): AgentFixture;
+//# sourceMappingURL=agentFixture.d.ts.map

package/lib/dist/lib/testCases/agentFixture.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agentFixture.d.ts","sourceRoot":"","sources":["../../../testCases/agentFixture.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7C,wDAAwD;AACxD,MAAM,WAAW,eAAe;IAC9B,8DAA8D;IAC9D,OAAO,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACnD;;;;OAIG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACrE,mEAAmE;IACnE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,qDAAqD;AACrD,MAAM,MAAM,aAAa,GAAG,CAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,eAAe,KACtB,OAAO,CAAC,UAAU,CAAC,CAAC;AAEzB,MAAM,WAAW,yBAAyB;IACxC;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,yDAAyD;IACzD,cAAc,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC3D;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,aAAa,EACrB,OAAO,GAAE,yBAA8B,GACtC,YAAY,CAoCd"}

package/lib/dist/lib/testCases/agentFixture.js

@@ -0,0 +1,43 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Build an {@link AgentFixture} around a runner-supplied {@link AgentInvokeFn}.
+ *
+ * Enforces the one-invocation-per-test rule and resolves the effective
+ * prompt/context (per-call argument wins over the test's `defaultPrompt` /
+ * `defaultContext`). The `invoke` callback does the real work (connector +
+ * RunResult wrapping) and is only ever called at most once.
+ */
+export function createAgentFixture(invoke, options = {}) {
+    let invoked = false;
+    return {
+        get invoked() {
+            return invoked;
+        },
+        async run(prompt, runOptions) {
+            if (invoked) {
+                throw new Error('agent.run() may be called at most once per test. Benchmarking ' +
+                    'requires exactly one agent invocation per test so trajectories ' +
+                    'stay comparable across runs. For multi-turn conversations, model ' +
+                    'the turns inside a single run via your connector.');
+            }
+            const effectivePrompt = prompt ?? options.defaultPrompt;
+            if (effectivePrompt == null || effectivePrompt === '') {
+                throw new Error('agent.run() requires a prompt — none was passed and the test has ' +
+                    'no `defaultPrompt`. Pass a prompt: `await agent.run("...")`.');
+            }
+            // Mark invoked BEFORE awaiting so a concurrent/re-entrant second call
+            // (e.g. forgetting to await) still trips the guard deterministically.
+            invoked = true;
+            const mergedContext = [
+                ...(options.defaultContext ?? []),
+                ...(runOptions?.context ?? []),
+            ];
+            const context = mergedContext.length > 0 ? mergedContext : undefined;
+            return invoke(effectivePrompt, { ...runOptions, context });
+        },
+    };
+}
+//# sourceMappingURL=agentFixture.js.map

package/lib/dist/lib/testCases/agentFixture.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agentFixture.js","sourceRoot":"","sources":["../../../testCases/agentFixture.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA2EH;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAChC,MAAqB,EACrB,UAAqC,EAAE;IAEvC,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,OAAO;QACL,IAAI,OAAO;YACT,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,MAAe,EAAE,UAA4B;YACrD,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CACb,gEAAgE;oBAChE,iEAAiE;oBACjE,mEAAmE;oBACnE,mDAAmD,CACpD,CAAC;YACJ,CAAC;YACD,MAAM,eAAe,GAAG,MAAM,IAAI,OAAO,CAAC,aAAa,CAAC;YACxD,IAAI,eAAe,IAAI,IAAI,IAAI,eAAe,KAAK,EAAE,EAAE,CAAC;gBACtD,MAAM,IAAI,KAAK,CACb,mEAAmE;oBACnE,8DAA8D,CAC/D,CAAC;YACJ,CAAC;YACD,sEAAsE;YACtE,sEAAsE;YACtE,OAAO,GAAG,IAAI,CAAC;YAEf,MAAM,aAAa,GAAG;gBACpB,GAAG,CAAC,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC;gBACjC,GAAG,CAAC,UAAU,EAAE,OAAO,IAAI,EAAE,CAAC;aAC/B,CAAC;YACF,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC;YAErE,OAAO,MAAM,CAAC,eAAe,EAAE,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC;QAC7D,CAAC;KACF,CAAC;AACJ,CAAC"}

package/lib/dist/lib/testCases/authoringSurface.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Build the authoring-surface object handed to code-imported eval files.
+ * Must mirror the public `@opensearch-project/agent-health` package exports
+ * that a test body uses (the runtime registrars + matchers), so `.js` and
+ * `.ts`/`.mjs` files behave identically.
+ */
+export declare function getAuthoringSurface(): Record<string, unknown>;
+/** The stable list of authoring-surface export names (for drift tests). */
+export declare const AUTHORING_SURFACE_NAMES: string[];
+//# sourceMappingURL=authoringSurface.d.ts.map

package/lib/dist/lib/testCases/authoringSurface.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"authoringSurface.d.ts","sourceRoot":"","sources":["../../../testCases/authoringSurface.ts"],"names":[],"mappings":"AAkCA;;;;;GAKG;AACH,wBAAgB,mBAAmB,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAoB7D;AAED,2EAA2E;AAC3E,eAAO,MAAM,uBAAuB,UAA4C,CAAC"}

package/lib/dist/lib/testCases/authoringSurface.js

@@ -0,0 +1,54 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * The single source of truth for the SDK's **test-authoring surface** — the
+ * exact set of names a `.eval.{js,ts,mjs}` file gets when it imports
+ * `@opensearch-project/agent-health` (RFC 004 §4.7, #232).
+ *
+ * Why this exists: the loader binds two ways. `.ts`/`.mjs` files are
+ * `import()`-ed and resolve the real package exports; `.js` (CJS) files are
+ * executed in a synthetic context where `require('@opensearch-project/...')`
+ * is intercepted and handed a hand-built object. Historically that object was
+ * a hardcoded subset that silently drifted from the real package exports — so
+ * a new export (e.g. `defineEvaluator`) worked in `.ts` files but was
+ * `undefined` in `.js` files. Centralizing the surface here, and having BOTH
+ * the loader's CJS injection and the package index reference it, kills the
+ * drift: add a name once and every loader path sees it.
+ */
+import { test, describe, beforeAll, afterAll, beforeEach, afterEach, } from './define.js';
+import { judge } from './judge.js';
+import { defineEvaluator, evaluate } from './evaluators.js';
+import { expect } from '../matchers/expect.js';
+import { source, workflow } from '../workflow/index.js';
+/**
+ * Build the authoring-surface object handed to code-imported eval files.
+ * Must mirror the public `@opensearch-project/agent-health` package exports
+ * that a test body uses (the runtime registrars + matchers), so `.js` and
+ * `.ts`/`.mjs` files behave identically.
+ */
+export function getAuthoringSurface() {
+    return {
+        // Registrars (file-scoped singletons — registrations land in the
+        // loader's per-file registry).
+        test,
+        describe,
+        beforeAll,
+        afterAll,
+        beforeEach,
+        afterEach,
+        // Judge + custom evaluators.
+        judge,
+        defineEvaluator,
+        evaluate,
+        // Matcher API.
+        expect,
+        // Workflow SDK (for .workflow.{js,ts,mjs} files).
+        source,
+        workflow,
+    };
+}
+/** The stable list of authoring-surface export names (for drift tests). */
+export const AUTHORING_SURFACE_NAMES = Object.keys(getAuthoringSurface()).sort();
+//# sourceMappingURL=authoringSurface.js.map

package/lib/dist/lib/testCases/authoringSurface.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"authoringSurface.js","sourceRoot":"","sources":["../../../testCases/authoringSurface.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EACL,IAAI,EACJ,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,UAAU,EACV,SAAS,GACV,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC5D,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAExD;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB;IACjC,OAAO;QACL,iEAAiE;QACjE,+BAA+B;QAC/B,IAAI;QACJ,QAAQ;QACR,SAAS;QACT,QAAQ;QACR,UAAU;QACV,SAAS;QACT,6BAA6B;QAC7B,KAAK;QACL,eAAe;QACf,QAAQ;QACR,eAAe;QACf,MAAM;QACN,kDAAkD;QAClD,MAAM;QACN,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,2EAA2E;AAC3E,MAAM,CAAC,MAAM,uBAAuB,GAAG,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC"}

package/lib/dist/lib/testCases/codemod.d.ts

@@ -0,0 +1,13 @@
+export interface CodemodResult {
+    /** The transformed source (equals input when nothing changed). */
+    code: string;
+    /** Whether any edit was applied. */
+    changed: boolean;
+    /** Human-readable notes per test (migrated / skipped + why). */
+    notes: string[];
+}
+/**
+ * Run the codemod over a single file's source text.
+ */
+export declare function migrateEvalSource(code: string, fileName?: string): CodemodResult;
+//# sourceMappingURL=codemod.d.ts.map

package/lib/dist/lib/testCases/codemod.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"codemod.d.ts","sourceRoot":"","sources":["../../../testCases/codemod.ts"],"names":[],"mappings":"AAgCA,MAAM,WAAW,aAAa;IAC5B,kEAAkE;IAClE,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,gEAAgE;IAChE,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AA0CD;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,SAAY,GAAG,aAAa,CA4HnF"}