agentv 5.3.1-next.1 → 5.3.3

package/dist/sdk.d.ts

@@ -0,0 +1,802 @@
+export { A as AgentVConfig, d as defineConfig, z } from './contracts-DsZmZLl8.js';
+import { ScriptGraderInput, TokenUsage, ScriptGraderResult, ScriptGraderHandler } from '@agentv/core/script-grader';
+export { CodeGraderHandler, CodeGraderInput, CodeGraderInputSchema, CodeGraderResult, CodeGraderResultSchema, Content, ContentFile, ContentFileSchema, ContentImage, ContentImageSchema, ContentSchema, ContentText, ContentTextSchema, Message, MessageSchema, PromptTemplateInput, PromptTemplateInputSchema, ScriptGraderCheck, ScriptGraderCheckSchema, ScriptGraderHandler, ScriptGraderInput, ScriptGraderInputSchema, ScriptGraderResult, ScriptGraderResultSchema, TRACE_EVENT_TYPES, TRACE_REDACTION_LEVELS, TRACE_SOURCE_KINDS, TRACE_TOOL_STATUSES, TokenUsage, TokenUsageSchema, ToolCall, ToolCallSchema, Trace, TraceArtifact, TraceArtifactSchema, TraceBranch, TraceBranchSchema, TraceError, TraceErrorSchema, TraceEvent, TraceEventSchema, TraceMessage, TraceMessageSchema, TraceModel, TraceModelSchema, TraceRawEvidence, TraceRawEvidenceSchema, TraceRedactionState, TraceRedactionStateSchema, TraceSchema, TraceSession, TraceSessionSchema, TraceSource, TraceSourceRef, TraceSourceRefSchema, TraceSourceSchema, TraceSummary, TraceSummarySchema, TraceTool, TraceToolSchema, VitestWorkspaceGraderOptions, defineVitestWorkspaceGrader, runCodeGrader, runScriptGrader, runVitestWorkspaceGrader, vitestReportToCodeGraderResult, vitestReportToScriptGraderResult } from '@agentv/core/script-grader';
+export { a3 as AssertEntry, a4 as ConversationTurnInput, a8 as EvalAssertionInput, a6 as EvalRunArtifacts, a9 as EvalRunResult, aa as EvalSummary, a7 as EvalTestInput, a2 as evaluate } from './ts-eval-loader-kf6J7wSc.js';
+
+declare const EVAL_SUITE_SYMBOL: unique symbol;
+declare const TO_EVAL_YAML_OBJECT_SYMBOL: unique symbol;
+declare const KNOWN_SNAKE_CASE_KEYS: {
+    readonly afterAll: "after_all";
+    readonly afterEach: "after_each";
+    readonly argsMatch: "args_match";
+    readonly beforeAll: "before_all";
+    readonly beforeEach: "before_each";
+    readonly budgetUsd: "budget_usd";
+    readonly conversationId: "conversation_id";
+    readonly dependsOn: "depends_on";
+    readonly expectedOutput: "expected_output";
+    readonly explorationTolerance: "exploration_tolerance";
+    readonly failOnError: "fail_on_error";
+    readonly inputFiles: "input_files";
+    readonly keepWorkspaces: "keep_workspaces";
+    readonly maxCalls: "max_calls";
+    readonly maxCostUsd: "max_cost_usd";
+    readonly maxDurationMs: "max_duration_ms";
+    readonly maxInput: "max_input";
+    readonly maxLlmCalls: "max_llm_calls";
+    readonly maxOutput: "max_output";
+    readonly maxSteps: "max_steps";
+    readonly maxTokens: "max_tokens";
+    readonly maxToolCalls: "max_tool_calls";
+    readonly minScore: "min_score";
+    readonly onDependencyFailure: "on_dependency_failure";
+    readonly onTurnFailure: "on_turn_failure";
+    readonly outputPath: "output_path";
+    readonly readOnly: "read_only";
+    readonly reasoningEffort: "reasoning_effort";
+    readonly rubricPrompt: "rubric_prompt";
+    readonly scoreRange: "score_range";
+    readonly scoreRanges: "score_ranges";
+    readonly skipDefaults: "skip_defaults";
+    readonly targetExplorationRatio: "target_exploration_ratio";
+    readonly timeoutMs: "timeout_ms";
+    readonly timeoutSeconds: "timeout_seconds";
+    readonly useTarget: "use_target";
+    readonly defaultTest: "default_test";
+    readonly windowSize: "window_size";
+};
+type KnownSnakeCaseKeyMap = typeof KNOWN_SNAKE_CASE_KEYS;
+type LowerEvalKey<Key extends string> = Key extends keyof KnownSnakeCaseKeyMap ? KnownSnakeCaseKeyMap[Key] : Key;
+type LowerEvalYamlValue<Value> = Value extends readonly (infer Item)[] ? LowerEvalYamlValue<Item>[] : Value extends object ? {
+    [Key in keyof Value as Key extends string ? LowerEvalKey<Key> : never]: LowerEvalYamlValue<Value[Key]>;
+} : Value;
+type EvalMessageContent = string | Readonly<Record<string, unknown>> | readonly (string | Readonly<Record<string, unknown>>)[];
+interface EvalMessage {
+    readonly role: 'system' | 'user' | 'assistant' | 'tool';
+    readonly content: EvalMessageContent;
+    readonly [key: string]: unknown;
+}
+interface EvalAssertionConfig {
+    readonly type: string;
+    readonly provider?: string | true | object;
+    readonly [key: string]: unknown;
+}
+interface EvalLifecycleHook {
+    readonly command?: string | readonly string[];
+    readonly timeoutMs?: number;
+    readonly cwd?: string;
+    readonly reset?: 'none' | 'fast' | 'strict';
+    readonly [key: string]: unknown;
+}
+interface EvalLifecycleHooks {
+    readonly enabled?: boolean;
+    readonly beforeAll?: EvalLifecycleHook;
+    readonly beforeEach?: EvalLifecycleHook;
+    readonly afterEach?: EvalLifecycleHook;
+    readonly afterAll?: EvalLifecycleHook;
+}
+interface EvalEnvironmentSetup {
+    readonly command: readonly string[];
+    readonly cwd?: string;
+    readonly timeoutMs?: number;
+}
+interface EvalDockerEnvironmentMount {
+    readonly source: string;
+    readonly target: string;
+    readonly access?: 'ro' | 'rw';
+    readonly readOnly?: boolean;
+}
+interface EvalDockerEnvironmentResources {
+    readonly cpus?: number;
+    readonly memory?: string;
+    readonly disk?: string;
+    readonly gpu?: boolean | string;
+}
+interface EvalHostEnvironment {
+    readonly type: 'host';
+    readonly workdir: string;
+    readonly setup?: EvalEnvironmentSetup;
+    readonly env?: Readonly<Record<string, string>>;
+}
+interface EvalDockerEnvironment {
+    readonly type: 'docker';
+    readonly workdir: string;
+    readonly context?: string;
+    readonly dockerfile?: string;
+    readonly image?: string;
+    readonly setup?: EvalEnvironmentSetup;
+    readonly env?: Readonly<Record<string, string>>;
+    readonly resources?: EvalDockerEnvironmentResources;
+    readonly mounts?: readonly EvalDockerEnvironmentMount[];
+    readonly secrets?: Readonly<Record<string, string>>;
+}
+type EvalEnvironment = EvalHostEnvironment | EvalDockerEnvironment;
+interface EvalProviderRef {
+    readonly label: string;
+    readonly id?: string;
+    readonly useProvider?: string;
+    readonly hooks?: EvalLifecycleHooks;
+}
+interface EvalProviderConfig {
+    readonly extends?: string;
+    readonly id: string;
+    readonly label?: string;
+    readonly model?: string;
+    readonly config?: Readonly<Record<string, unknown>>;
+    readonly prompts?: unknown;
+    readonly transform?: unknown;
+    readonly delay?: number;
+    readonly inputs?: unknown;
+    readonly env?: Readonly<Record<string, string>>;
+    readonly reasoningEffort?: string;
+    readonly hooks?: EvalLifecycleHooks;
+    readonly [key: string]: unknown;
+}
+type EvalProviderMap = Readonly<Record<string, Omit<EvalProviderConfig, 'id'> | EvalProviderRef | Readonly<Record<string, unknown>>>>;
+type EvalProviderEntry = string | EvalProviderRef | EvalProviderConfig | EvalProviderMap;
+interface EvalDefaultsConfig {
+    readonly provider?: string;
+    readonly grader?: string;
+    readonly [key: string]: unknown;
+}
+interface EvalTestOptions {
+    readonly provider?: string;
+    readonly transform?: unknown;
+    readonly repeat?: EvalRepeat;
+    readonly rubricPrompt?: unknown;
+    readonly [key: string]: unknown;
+}
+interface EvalDefaultTest {
+    readonly vars?: Readonly<Record<string, unknown>>;
+    readonly assert?: readonly (string | EvalAssertionConfig)[];
+    readonly options?: EvalTestOptions;
+    readonly [key: string]: unknown;
+}
+type EvalRepeat = number;
+interface EvalExecution {
+    readonly provider?: string;
+    readonly providers?: readonly EvalProviderEntry[];
+    readonly assert?: readonly EvalAssertionConfig[];
+    readonly skipDefaults?: boolean;
+    readonly cache?: boolean;
+    readonly trials?: never;
+    readonly budgetUsd?: number;
+    readonly failOnError?: boolean;
+    readonly threshold?: number;
+    readonly [key: string]: unknown;
+}
+interface EvalTurn {
+    readonly input: EvalMessageContent;
+    readonly expectedOutput?: EvalMessageContent;
+    readonly assert?: readonly (string | EvalAssertionConfig)[];
+}
+interface EvalTest {
+    readonly id: string;
+    readonly vars?: Readonly<Record<string, unknown>>;
+    readonly criteria?: string;
+    readonly inputFiles?: readonly string[];
+    readonly expectedOutput?: string | Readonly<Record<string, unknown>> | readonly EvalMessage[];
+    readonly assert?: readonly EvalAssertionConfig[];
+    readonly options?: EvalTestOptions;
+    readonly execution?: EvalExecution;
+    readonly environment?: EvalEnvironment | string;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+    readonly conversationId?: string;
+    readonly suite?: string;
+    readonly dependsOn?: readonly string[];
+    readonly onDependencyFailure?: 'skip' | 'fail' | 'run';
+    readonly mode?: 'conversation';
+    readonly turns?: readonly EvalTurn[];
+    readonly aggregation?: 'mean' | 'min' | 'max';
+    readonly onTurnFailure?: 'continue' | 'stop';
+    readonly windowSize?: number;
+}
+interface EvalRequires {
+    readonly agentv?: string;
+    readonly [key: string]: unknown;
+}
+interface EvalConfig {
+    readonly $schema?: string;
+    readonly name?: string;
+    readonly description?: string;
+    readonly category?: string;
+    readonly version?: string;
+    readonly author?: string;
+    /**
+     * Suite tags. Either the selection list form (`string[]`, drives
+     * `select.tags` / `--tag name` filtering) or the promptfoo-shaped
+     * `Record<string,string>` map. In the map form the reserved `experiment` key
+     * labels the run/experiment (grouped by the Dashboard), matching
+     * `tags.experiment` in YAML evals.
+     */
+    readonly tags?: readonly string[] | Readonly<Record<string, string>>;
+    readonly license?: string;
+    readonly requires?: EvalRequires;
+    readonly inputFiles?: readonly string[];
+    readonly prompts?: unknown;
+    readonly providers?: readonly EvalProviderEntry[];
+    readonly defaults?: EvalDefaultsConfig;
+    readonly defaultTest?: EvalDefaultTest | string;
+    readonly tests: readonly EvalTest[] | string;
+    /** @deprecated Rejected. Use `tags: { experiment: '<name>' }` or CLI --experiment. */
+    readonly experiment?: never;
+    readonly repeat?: EvalRepeat;
+    readonly timeoutSeconds?: number;
+    readonly threshold?: number;
+    readonly budgetUsd?: number;
+    readonly assert?: readonly EvalAssertionConfig[];
+    readonly environment?: EvalEnvironment | string;
+}
+interface DefinedEvalSuite {
+    readonly [EVAL_SUITE_SYMBOL]: true;
+    readonly [TO_EVAL_YAML_OBJECT_SYMBOL]: () => Record<string, unknown>;
+}
+/**
+ * Define a YAML-aligned eval suite in TypeScript.
+ *
+ * The returned object preserves the TypeScript authoring shape and carries a
+ * non-enumerable lowering hook so AgentV can materialize the canonical
+ * snake_case eval contract when the suite is loaded from a `.eval.ts` file.
+ */
+declare function defineEval<T extends EvalConfig>(definition: T): T & DefinedEvalSuite;
+/**
+ * Lower a TypeScript-authored eval suite into the canonical snake_case object
+ * contract used by YAML files and the runtime loader.
+ *
+ * Only known AgentV wire keys are converted. Unknown keys are preserved as-is
+ * so opaque assertion, provider, and metadata payloads are not corrupted.
+ */
+declare function toEvalYamlObject<T extends EvalConfig | DefinedEvalSuite>(definition: T): LowerEvalYamlValue<T>;
+/**
+ * Serialize an eval suite to canonical YAML.
+ */
+declare function serializeEvalYaml<T extends EvalConfig | DefinedEvalSuite>(definition: T): string;
+
+type GraderCommand = string | readonly string[];
+interface GraderHelperOptions {
+    readonly metric?: string;
+    readonly weight?: number;
+    readonly required?: boolean;
+    readonly minScore?: number;
+    readonly negate?: boolean;
+    readonly transform?: string;
+}
+interface GraderCommonConfig {
+    readonly metric?: string;
+    readonly weight?: number;
+    readonly required?: boolean;
+    readonly minScore?: number;
+    readonly negate?: boolean;
+    readonly transform?: string;
+}
+interface ContainsGraderConfig extends EvalAssertionConfig, GraderCommonConfig {
+    readonly type: 'contains';
+    readonly value: string;
+}
+interface EqualsGraderConfig extends EvalAssertionConfig, GraderCommonConfig {
+    readonly type: 'equals';
+    readonly value: string;
+}
+interface RegexGraderOptions extends GraderHelperOptions {
+    readonly flags?: string;
+}
+interface RegexGraderConfig extends EvalAssertionConfig, GraderCommonConfig {
+    readonly type: 'regex';
+    readonly value: string;
+    readonly flags?: string;
+}
+interface IsJsonGraderConfig extends EvalAssertionConfig, GraderCommonConfig {
+    readonly type: 'is-json';
+}
+type GraderRubricOperator = 'correctness' | 'contradiction';
+interface GraderScoreRange {
+    readonly scoreRange: readonly [number, number];
+    readonly outcome: string;
+}
+interface GraderRubric {
+    readonly id?: string;
+    readonly outcome?: string;
+    readonly criteria?: string;
+    readonly operator?: GraderRubricOperator;
+    readonly weight?: number;
+    readonly required?: boolean;
+    readonly minScore?: number;
+    readonly scoreRanges?: readonly GraderScoreRange[];
+}
+type GraderRubricCriterion = string | GraderRubric;
+interface LlmRubricGraderConfig extends EvalAssertionConfig, GraderCommonConfig {
+    readonly type: 'llm-rubric';
+    readonly value?: unknown;
+    readonly prompt?: string | GraderPromptScriptConfig;
+    readonly provider?: string;
+    readonly config?: Readonly<Record<string, unknown>>;
+    readonly maxSteps?: number;
+    readonly temperature?: number;
+}
+interface GraderPromptScriptConfig {
+    readonly command: readonly string[];
+    readonly config?: Readonly<Record<string, unknown>>;
+}
+interface ScriptGraderProviderOptions {
+    readonly maxCalls?: number;
+}
+interface ScriptGraderOptions extends GraderHelperOptions {
+    readonly cwd?: string;
+    readonly provider?: true | ScriptGraderProviderOptions;
+    readonly config?: Readonly<Record<string, unknown>>;
+}
+interface ScriptGraderConfig extends EvalAssertionConfig, GraderCommonConfig {
+    readonly type: 'script';
+    readonly command: GraderCommand;
+    readonly cwd?: string;
+    readonly provider?: true | ScriptGraderProviderOptions;
+    readonly config?: Readonly<Record<string, unknown>>;
+}
+/** @deprecated Use ScriptGraderProviderOptions. */
+type ScriptGraderTargetOptions = ScriptGraderProviderOptions;
+/** @deprecated Use ScriptGraderProviderOptions. */
+type CodeGraderTargetOptions = ScriptGraderProviderOptions;
+/** @deprecated Use ScriptGraderOptions. */
+type CodeGraderOptions = ScriptGraderOptions;
+/** @deprecated Use ScriptGraderConfig with type: 'script'. */
+type CodeGraderConfig = ScriptGraderConfig;
+type GraderHelperConfig = ContainsGraderConfig | EqualsGraderConfig | RegexGraderConfig | IsJsonGraderConfig | LlmRubricGraderConfig | ScriptGraderConfig;
+declare function containsGrader(value: string, options?: GraderHelperOptions): ContainsGraderConfig;
+declare function equalsGrader(value: string, options?: GraderHelperOptions): EqualsGraderConfig;
+declare function exactGrader(value: string, options?: GraderHelperOptions): EqualsGraderConfig;
+declare function regexGrader(pattern: string | RegExp, options?: RegexGraderOptions): RegexGraderConfig;
+declare function isJsonGrader(options?: GraderHelperOptions): IsJsonGraderConfig;
+declare function jsonGrader(options?: GraderHelperOptions): IsJsonGraderConfig;
+declare function llmRubricGrader(valueOrCriteria?: string | readonly GraderRubricCriterion[] | Readonly<Record<string, unknown>>, options?: GraderHelperOptions & {
+    readonly prompt?: string | GraderPromptScriptConfig;
+    readonly provider?: string;
+    readonly config?: Readonly<Record<string, unknown>>;
+    readonly maxSteps?: number;
+    readonly temperature?: number;
+}): LlmRubricGraderConfig;
+/** @deprecated Use scriptGrader. */
+declare function codeGrader(command: GraderCommand, options?: ScriptGraderOptions): ScriptGraderConfig;
+declare function scriptGrader(command: GraderCommand, options?: ScriptGraderOptions): ScriptGraderConfig;
+declare const graders: Readonly<{
+    contains: typeof containsGrader;
+    equals: typeof equalsGrader;
+    exact: typeof exactGrader;
+    regex: typeof regexGrader;
+    isJson: typeof isJsonGrader;
+    json: typeof jsonGrader;
+    llmRubric: typeof llmRubricGrader;
+    codeGrader: typeof codeGrader;
+    script: typeof scriptGrader;
+    scriptGrader: typeof scriptGrader;
+}>;
+type GraderCatalog = typeof graders;
+
+/**
+ * Runtime-only client for invoking configured providers from script-grader scripts
+ * through AgentV's provider proxy.
+ *
+ * Environment variables (set automatically by AgentV when provider proxy access is enabled):
+ * - AGENTV_PROVIDER_PROXY_URL: The URL of the local proxy server
+ * - AGENTV_PROVIDER_PROXY_TOKEN: Bearer token for authentication
+ */
+
+/**
+ * Request to invoke the provider
+ */
+interface ProviderInvokeRequest {
+    readonly question: string;
+    readonly systemPrompt?: string;
+    readonly evalCaseId?: string;
+    readonly attempt?: number;
+    /** Optional provider override - use a different provider for this invocation */
+    readonly provider?: string;
+}
+/**
+ * Response from a provider invocation
+ */
+interface ProviderInvokeResponse {
+    readonly output: readonly unknown[];
+    readonly rawText?: string;
+    readonly tokenUsage?: TokenUsage;
+}
+/**
+ * Information about the provider proxy configuration
+ */
+interface ProviderInfo {
+    /** Label of the default provider being used */
+    readonly providerLabel: string;
+    /** Maximum number of calls allowed */
+    readonly maxCalls: number;
+    /** Current number of calls made */
+    readonly callCount: number;
+    /** Labels of all available providers */
+    readonly availableProviderLabels: readonly string[];
+}
+/**
+ * Provider client for making provider invocations
+ */
+interface ProviderClient {
+    /**
+     * Invoke the configured provider with a prompt.
+     * @param request - The question and optional system prompt
+     * @returns The provider's response with output messages and optional raw text
+     */
+    invoke(request: ProviderInvokeRequest): Promise<ProviderInvokeResponse>;
+    /**
+     * Invoke the provider with multiple requests in sequence.
+     * Each request counts toward the max_calls limit.
+     * @param requests - Array of provider requests
+     * @returns Array of provider responses
+     */
+    invokeBatch(requests: readonly ProviderInvokeRequest[]): Promise<readonly ProviderInvokeResponse[]>;
+    /**
+     * Get information about the provider proxy configuration.
+     * Returns the default provider label, max calls, current call count, and available providers.
+     */
+    getInfo(): Promise<ProviderInfo>;
+}
+/**
+ * Error thrown when provider proxy is not available
+ */
+declare class ProviderNotAvailableError extends Error {
+    constructor(message: string);
+}
+/**
+ * Error thrown when provider invocation fails
+ */
+declare class ProviderInvocationError extends Error {
+    readonly statusCode?: number;
+    constructor(message: string, statusCode?: number);
+}
+/**
+ * Create a provider client from environment variables.
+ *
+ * This function reads the proxy URL and token from environment variables
+ * that are automatically set by AgentV when provider access is enabled on a
+ * `script` evaluator.
+ *
+ * @returns A provider client if environment variables are set, otherwise undefined
+ * @throws ProviderNotAvailableError if token is missing when URL is present
+ *
+ * @example
+ * ```typescript
+ * import { createProviderClient, defineScriptGrader } from 'agentv';
+ *
+ * export default defineScriptGrader(async ({ input, criteria, output }) => {
+ *   const provider = createProviderClient();
+ *   const question = input
+ *     .filter((message) => message.role === 'user')
+ *     .map((message) => typeof message.content === 'string' ? message.content : '')
+ *     .join('\n');
+ *
+ *   if (!provider) {
+ *     return { pass: false, score: 0.5, reason: 'Provider proxy not available' };
+ *   }
+ *
+ *   const response = await provider.invoke({
+ *     question: `Is this answer correct? Question: ${question}, Expected: ${criteria}, Answer: ${output ?? ''}`,
+ *     systemPrompt: 'You are an expert grader. Respond with JSON: { "correct": true/false }'
+ *   });
+ *
+ *   const result = JSON.parse(response.rawText ?? '{}');
+ *   return {
+ *     pass: result.correct === true,
+ *     score: result.correct === true ? 1.0 : 0.0,
+ *     reason: result.correct === true ? 'Provider judged the answer correct' : 'Provider judged the answer incorrect',
+ *   };
+ * });
+ * ```
+ */
+declare function createProviderClient(): ProviderClient | undefined;
+
+interface WorkspaceCheck {
+    readonly text: string;
+    readonly pass: boolean;
+    readonly score?: number;
+    readonly reason: string;
+    readonly evidence?: string;
+}
+/** @deprecated Use WorkspaceCheck. */
+type WorkspaceAssertion = WorkspaceCheck;
+type Awaitable<T> = T | Promise<T>;
+type WorkspaceGraderReturn = ScriptGraderResult | WorkspaceCheck | readonly Awaitable<WorkspaceCheck>[];
+interface WorkspaceFileAssertionOptions {
+    readonly text?: string;
+}
+interface WorkspaceFile {
+    readonly path: string;
+    readonly absolutePath?: string;
+    readText(): Promise<string>;
+    exists(options?: WorkspaceFileAssertionOptions): Promise<WorkspaceCheck>;
+    contains(expected: string, options?: WorkspaceFileAssertionOptions): Promise<WorkspaceCheck>;
+    notContains(expected: string, options?: WorkspaceFileAssertionOptions): Promise<WorkspaceCheck>;
+    matches(pattern: RegExp, options?: WorkspaceFileAssertionOptions): Promise<WorkspaceCheck>;
+    notMatches(pattern: RegExp, options?: WorkspaceFileAssertionOptions): Promise<WorkspaceCheck>;
+}
+interface Workspace {
+    readonly path?: string;
+    file(relativePath: string): WorkspaceFile;
+    readText(relativePath: string): Promise<string>;
+}
+type WorkspaceGraderContext = ScriptGraderInput & {
+    readonly workspace: Workspace;
+};
+type WorkspaceGraderHandler = (context: WorkspaceGraderContext) => WorkspaceGraderReturn | Promise<WorkspaceGraderReturn>;
+declare function createWorkspace(input: ScriptGraderInput): Workspace;
+declare function normalizeWorkspaceGraderResult(result: WorkspaceGraderReturn): Promise<ScriptGraderResult>;
+declare function runWorkspaceGrader(handler: WorkspaceGraderHandler, input: ScriptGraderInput): Promise<ScriptGraderResult>;
+declare function defineWorkspaceGrader(handler: WorkspaceGraderHandler): void;
+
+/**
+ * Context provided to assertion handlers.
+ */
+type AssertionContext = ScriptGraderInput;
+/**
+ * Known built-in assertion types. Custom types are extensible via string.
+ *
+ * Use in EVAL.yaml `assert` blocks:
+ * ```yaml
+ * assert:
+ *   - type: contains
+ *     value: "Paris"
+ * ```
+ *
+ * Custom types registered via `.agentv/assertions/` or `defineAssertion()`
+ * are also valid — the `string & {}` escape hatch provides autocomplete
+ * for known types while accepting any string.
+ */
+type AssertionType = 'llm-rubric' | 'agent-rubric' | 'script' | 'assert-set'
+/** @deprecated Authored eval YAML rejects this compatibility-only runtime type. */
+ | 'tool-trajectory' | 'skill-used' | 'not-skill-used' | 'trajectory:tool-used' | 'trajectory:tool-args-match' | 'trajectory:tool-sequence' | 'trajectory:step-count' | 'trajectory:goal-success' | 'field-accuracy' | 'latency' | 'cost' | 'token-usage' | 'execution-metrics' | 'contains' | 'contains-any' | 'contains-all' | 'icontains' | 'icontains-any' | 'icontains-all' | 'starts-with' | 'ends-with' | 'equals' | 'regex' | 'is-json' | 'javascript' | 'python' | 'webhook' | 'similar' | (string & {});
+/**
+ * Check returned from an assertion handler.
+ */
+interface AssertionCheck {
+    readonly id?: string;
+    readonly text: string;
+    readonly pass: boolean;
+    readonly score?: number;
+    readonly reason: string;
+    readonly evidence?: string;
+}
+/**
+ * Result returned from an assertion handler.
+ *
+ * @example Pass with score
+ * ```ts
+ * { pass: true, score: 1, reason: 'Output contains expected keywords' }
+ * ```
+ *
+ * @example Fail with checks
+ * ```ts
+ * { pass: false, score: 0.3, reason: 'Missing required header', checks: [
+ *   { text: 'Header present', pass: false, reason: 'No header found' },
+ * ] }
+ * ```
+ *
+ * @example Granular score (0-1)
+ * ```ts
+ * { score: 0.75, reason: 'Two of three checks passed', checks: [
+ *   { text: 'Format correct', pass: true, reason: 'Matches expected format' },
+ *   { text: 'Content relevant', pass: true, reason: 'Addresses the request' },
+ *   { text: 'Citation present', pass: false, reason: 'Missing citation' },
+ * ] }
+ * ```
+ */
+interface AssertionScore {
+    /** Explicit pass/fail. If omitted, derived from score (>= 0.5 = pass). */
+    readonly pass?: boolean;
+    /** Numeric score between 0 and 1. Defaults to 1 if pass=true, 0 if pass=false. */
+    readonly score?: number;
+    /** Explanation for the aggregate pass/fail decision. */
+    readonly reason?: string;
+    /** Per-check verdicts with optional score and evidence. */
+    readonly checks?: readonly AssertionCheck[];
+    /** Optional structured details for domain-specific metrics. */
+    readonly details?: Record<string, unknown>;
+}
+/**
+ * Handler function type for assertions.
+ */
+type AssertionHandler = (ctx: AssertionContext) => AssertionScore | Promise<AssertionScore>;
+
+/**
+ * Handler function type for prompt templates.
+ * Returns the prompt string to use for evaluation.
+ */
+type PromptTemplateHandler = (input: ScriptGraderInput) => string | Promise<string>;
+
+/**
+ * AgentV Evaluation SDK
+ *
+ * Build custom assertions, script graders, and eval authoring helpers for AI agent outputs.
+ *
+ * @example Custom assertion (simplest way to add evaluation logic)
+ * ```typescript
+ * #!/usr/bin/env bun
+ * import { defineAssertion } from 'agentv';
+ *
+ * export default defineAssertion(({ output, criteria }) => {
+ *   const answer = output ?? '';
+ *   return {
+ *     pass: answer.includes('hello'),
+ *     score: answer.includes('hello') ? 1 : 0,
+ *     reason: answer.includes('hello') ? 'Greeting found' : 'Greeting missing',
+ *   };
+ * }));
+ * ```
+ *
+ * @example script grader (full control)
+ * ```typescript
+ * #!/usr/bin/env bun
+ * import { defineScriptGrader } from 'agentv';
+ *
+ * export default defineScriptGrader(({ output, traceSummary }) => {
+ *   return {
+ *     score: (output ?? '').length > 0 && (traceSummary?.eventCount ?? 0) <= 5 ? 1.0 : 0.5,
+ *     pass: (output ?? '').length > 0 && (traceSummary?.eventCount ?? 0) <= 5,
+ *     reason: 'Checks answer text and trace size',
+ *     checks: [
+ *       { text: 'Answer is not empty', pass: (output ?? '').length > 0, reason: 'Output text is present' },
+ *       { text: 'Efficient tool usage', pass: (traceSummary?.eventCount ?? 0) <= 5, reason: 'Trace event count is within limit' },
+ *     ],
+ *   };
+ * }));
+ * ```
+ *
+ * @example Vitest workspace verifier adapter (custom wrapper form)
+ * ```typescript
+ * #!/usr/bin/env bun
+ * import { defineVitestWorkspaceGrader } from 'agentv';
+ *
+ * export default defineVitestWorkspaceGrader({
+ *   testFile: 'graders/welcome-banner.test.ts',
+ *   copyTestFilesToWorkspace: true,
+ * });
+ * ```
+ *
+ * @example Workspace grader (small file checks)
+ * ```typescript
+ * #!/usr/bin/env bun
+ * import { defineWorkspaceGrader } from 'agentv';
+ *
+ * export default defineWorkspaceGrader(async ({ workspace }) => [
+ *   await workspace.file('app/page.tsx').contains('Status: All systems ready'),
+ *   await workspace.file('app/page.tsx').contains('Open dashboard'),
+ *   await workspace.file('app/page.tsx').matches(/href=["']\/dashboard["']/),
+ *   await workspace.file('app/page.tsx').notMatches(/TODO/i),
+ * ]);
+ * ```
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * Define a script grader with automatic stdin/stdout handling.
+ *
+ * This function:
+ * 1. Reads JSON from stdin (snake_case format)
+ * 2. Converts to camelCase and validates with Zod
+ * 3. Calls your handler with typed input
+ * 4. Validates the result and outputs JSON to stdout
+ * 5. Handles errors gracefully with proper exit codes
+ *
+ * @param handler - Function that evaluates the input and returns a result
+ *
+ * @example
+ * ```typescript
+ * import { defineScriptGrader } from 'agentv';
+ *
+ * export default defineScriptGrader(({ trace }) => {
+ *   if (!trace) {
+ *     return { pass: false, score: 0.5, reason: 'No trace available' };
+ *   }
+ *
+ *   const efficient = trace.eventCount <= 10;
+ *   return {
+ *     pass: efficient,
+ *     score: efficient ? 1.0 : 0.5,
+ *     reason: efficient ? 'Efficient execution' : 'Too many tool calls',
+ *     checks: [{ text: 'Trace event count within limit', pass: efficient, reason: `${trace.eventCount} events observed` }],
+ *   };
+ * });
+ * ```
+ *
+ * @example With typed config
+ * ```typescript
+ * import { defineScriptGrader, z } from 'agentv';
+ *
+ * const ConfigSchema = z.object({
+ *   maxToolCalls: z.number().default(10),
+ * });
+ *
+ * export default defineScriptGrader(({ trace, config }) => {
+ *   const { maxToolCalls } = ConfigSchema.parse(config ?? {});
+ *   // Use maxToolCalls...
+ * });
+ * ```
+ */
+declare function defineScriptGrader(handler: ScriptGraderHandler): void;
+/** @deprecated Use defineScriptGrader. */
+declare function defineCodeGrader(handler: ScriptGraderHandler): void;
+/**
+ * Define a prompt template with automatic stdin/stdout handling.
+ *
+ * This function:
+ * 1. Reads JSON from stdin (snake_case format)
+ * 2. Converts to camelCase and validates with Zod
+ * 3. Calls your handler with typed input
+ * 4. Outputs the generated prompt string to stdout
+ * 5. Handles errors gracefully with proper exit codes
+ *
+ * @param handler - Function that generates the prompt string from input
+ *
+ * @example
+ * ```typescript
+ * import { definePromptTemplate } from 'agentv';
+ *
+ * export default definePromptTemplate((ctx) => {
+ *   const question = ctx.input
+ *     .filter((message) => message.role === 'user')
+ *     .map((message) => typeof message.content === 'string' ? message.content : '')
+ *     .join('\n');
+ *   const answer = ctx.output ?? '';
+ *   return `Question: ${question}\nAnswer: ${answer}`;
+ * });
+ * ```
+ */
+declare function definePromptTemplate(handler: PromptTemplateHandler): void;
+/**
+ * Define a custom assertion with automatic stdin/stdout handling.
+ *
+ * Assertions are the simplest way to add reusable custom checks. They receive
+ * the full evaluation context and return a pass/fail result with optional
+ * granular scoring. Place these files in `.agentv/assertions/` and reference
+ * them by discovered assertion type name. Use defineScriptGrader for
+ * command-backed graders referenced with `type: script`.
+ *
+ * This function:
+ * 1. Reads JSON from stdin (snake_case format)
+ * 2. Converts to camelCase and validates with Zod
+ * 3. Calls your handler with typed context
+ * 4. Normalizes the result (pass→score, clamp, etc.)
+ * 5. Outputs JSON to stdout
+ * 6. Handles errors gracefully with proper exit codes
+ *
+ * @param handler - Function that evaluates the context and returns a result
+ *
+ * @example Simple pass/fail
+ * ```typescript
+ * import { defineAssertion } from 'agentv';
+ *
+ * export default defineAssertion(({ output }) => {
+ *   const text = output ?? '';
+ *   return {
+ *     pass: text.toLowerCase().includes('hello'),
+ *     reason: text.toLowerCase().includes('hello') ? 'Greeting found' : 'Greeting missing',
+ *   };
+ * }));
+ * ```
+ *
+ * @example Granular scoring
+ * ```typescript
+ * import { defineAssertion } from 'agentv';
+ *
+ * export default defineAssertion(({ output, traceSummary }) => {
+ *   const text = output ?? '';
+ *   const hasContent = text.length > 0 ? 0.5 : 0;
+ *   const isEfficient = (traceSummary?.eventCount ?? 0) <= 5 ? 0.5 : 0;
+ *   return {
+ *     score: hasContent + isEfficient,
+ *     reason: 'Checks content exists and trace size',
+ *     checks: [
+ *       { text: 'Has content', pass: !!hasContent, reason: hasContent ? 'Output is non-empty' : 'Output is empty' },
+ *       { text: 'Efficient', pass: !!isEfficient, reason: isEfficient ? 'Trace is within limit' : 'Trace exceeds limit' },
+ *     ],
+ *   };
+ * }));
+ * ```
+ */
+declare function defineAssertion(handler: AssertionHandler): void;
+
+export { type AssertionCheck, type AssertionContext, type AssertionHandler, type AssertionScore, type AssertionType, type CodeGraderConfig, type CodeGraderOptions, type CodeGraderTargetOptions, type ContainsGraderConfig, type DefinedEvalSuite, type EqualsGraderConfig, type EvalAssertionConfig, type EvalConfig, type EvalDefaultTest, type EvalDockerEnvironment, type EvalDockerEnvironmentMount, type EvalDockerEnvironmentResources, type EvalEnvironment, type EvalEnvironmentSetup, type EvalExecution, type EvalHostEnvironment, type EvalLifecycleHook, type EvalLifecycleHooks, type EvalMessage, type EvalMessageContent, type EvalProviderConfig, type EvalProviderEntry, type EvalProviderMap, type EvalProviderRef, type EvalRepeat, type EvalRequires, type EvalTest, type EvalTestOptions, type EvalTurn, type GraderCatalog, type GraderCommand, type GraderCommonConfig, type GraderHelperConfig, type GraderHelperOptions, type GraderPromptScriptConfig, type GraderRubric, type GraderRubricCriterion, type GraderRubricOperator, type GraderScoreRange, type IsJsonGraderConfig, type LlmRubricGraderConfig, type LowerEvalYamlValue, type PromptTemplateHandler, type ProviderClient, type ProviderInfo, ProviderInvocationError, type ProviderInvokeRequest, type ProviderInvokeResponse, ProviderNotAvailableError, type RegexGraderConfig, type RegexGraderOptions, type ScriptGraderConfig, type ScriptGraderOptions, type ScriptGraderProviderOptions, type ScriptGraderTargetOptions, type Workspace, type WorkspaceAssertion, type WorkspaceCheck, type WorkspaceFile, type WorkspaceFileAssertionOptions, type WorkspaceGraderContext, type WorkspaceGraderHandler, type WorkspaceGraderReturn, codeGrader, containsGrader, createProviderClient, createWorkspace, defineAssertion, defineCodeGrader, defineEval, definePromptTemplate, defineScriptGrader, defineWorkspaceGrader, equalsGrader, exactGrader, graders, isJsonGrader, jsonGrader, llmRubricGrader, normalizeWorkspaceGraderResult, regexGrader, runWorkspaceGrader, scriptGrader, serializeEvalYaml, toEvalYamlObject };