@agentv/core 3.14.6 → 4.0.0

package/dist/index.d.cts

@@ -1,6 +1,71 @@
 import * as ai from 'ai';
 import { z } from 'zod';
 
+/**
+ * Multimodal content types for the AgentV pipeline.
+ *
+ * Models structured content blocks (text, images, files) that flow end-to-end
+ * without lossy flattening. Modeled after Inspect AI's discriminated union approach.
+ *
+ * ## Content model
+ *
+ * `Message.content` accepts `string | Content[]`:
+ * - `string` — backward-compatible plain text (most common case)
+ * - `Content[]` — array of typed content blocks for multimodal messages
+ *
+ * Binary data (images, files) is referenced by URL/base64 string or filesystem
+ * path — never raw bytes. This keeps payloads serializable and lets code graders
+ * access files via path without decoding.
+ *
+ * ## How to extend
+ *
+ * To add a new content variant (e.g., `ContentAudio`):
+ * 1. Define the interface with a unique `type` discriminant
+ * 2. Add it to the `Content` union
+ * 3. Update `getTextContent()` if the new type has extractable text
+ * 4. Update `isContent()` type guard with the new type string
+ */
+/** A text content block. */
+interface ContentText {
+    readonly type: 'text';
+    readonly text: string;
+}
+/**
+ * An image content block.
+ * `source` is a URL, data URI (base64), or filesystem path.
+ */
+interface ContentImage {
+    readonly type: 'image';
+    readonly media_type: string;
+    readonly source: string;
+}
+/**
+ * A file content block.
+ * `path` is a filesystem path or URL referencing the file.
+ */
+interface ContentFile {
+    readonly type: 'file';
+    readonly media_type: string;
+    readonly path: string;
+}
+/** Discriminated union of all content block types. */
+type Content = ContentText | ContentImage | ContentFile;
+/** Check whether a value is a valid `Content` block. */
+declare function isContent(value: unknown): value is Content;
+/** Check whether a value is a `Content[]` array (at least one valid block). */
+declare function isContentArray(value: unknown): value is Content[];
+/**
+ * Extract plain text from `string | Content[]`.
+ *
+ * - If `content` is a string, returns it directly.
+ * - If `content` is a `Content[]`, concatenates all `ContentText.text` values
+ *   (separated by newlines) and returns the result.
+ * - Returns `''` for `undefined`/`null`/unrecognized shapes.
+ *
+ * This is a **non-destructive** accessor — the original `Content[]` is preserved.
+ */
+declare function getTextContent(content: string | Content[] | undefined | null): string;
+
 type ChatMessageRole = 'system' | 'user' | 'assistant' | 'tool' | 'function';
 interface ChatMessage {
     readonly role: ChatMessageRole;
@@ -74,8 +139,8 @@ interface Message {
     readonly role: string;
     /** Optional name for the message sender */
     readonly name?: string;
-    /** Message content */
-    readonly content?: unknown;
+    /** Message content — plain string or structured content blocks for multimodal data. */
+    readonly content?: string | Content[];
     /** Tool calls made in this message */
     readonly toolCalls?: readonly ToolCall[];
     /** ISO 8601 timestamp when the message started */
@@ -120,6 +185,13 @@ interface ProviderResponse {
     /** ISO 8601 timestamp when execution ended (optional) */
     readonly endTime?: string;
 }
+/**
+ * Extract the content from the last assistant message in an output message array.
+ * Returns empty string if no assistant message found.
+ *
+ * Handles both plain-string content and Content[] (extracts text blocks).
+ */
+declare function extractLastAssistantContent(messages: readonly Message[] | undefined): string;
 interface Provider {
     readonly id: string;
     readonly kind: ProviderKind;
@@ -1012,7 +1084,8 @@ type EvaluatorConfig = CodeEvaluatorConfig | LlmGraderEvaluatorConfig | Composit
  */
 interface EvalTest {
     readonly id: string;
-    readonly eval_set?: string;
+    readonly dataset?: string;
+    readonly category?: string;
     readonly conversation_id?: string;
     readonly question: string;
     readonly input: readonly TestMessage[];
@@ -1121,7 +1194,8 @@ type FailOnError = boolean;
 interface EvaluationResult {
     readonly timestamp: string;
     readonly testId: string;
-    readonly eval_set?: string;
+    readonly dataset?: string;
+    readonly category?: string;
     readonly conversationId?: string;
     readonly score: number;
     readonly assertions: readonly AssertionEntry[];
@@ -1347,6 +1421,8 @@ type LoadOptions = {
     readonly verbose?: boolean;
     /** Filter tests by ID pattern (glob supported, e.g., "summary-*") */
     readonly filter?: string;
+    /** Category derived from the eval file's directory path */
+    readonly category?: string;
 };
 /**
  * Read metadata from a test suite file (like target name).
@@ -1764,133 +1840,80 @@ interface AgentVResolvedConfig {
     readonly model: string;
     readonly temperature: number;
 }
-type ResolvedTarget = {
-    readonly kind: 'openai';
+/** Base fields shared by all resolved targets. */
+interface ResolvedTargetBase {
     readonly name: string;
     readonly graderTarget?: string;
     readonly workers?: number;
     readonly providerBatching?: boolean;
+    /**
+     * Whether this target can be executed via executor subagents in subagent mode.
+     * Defaults to `true` for all non-CLI providers. Set `false` in targets.yaml
+     * to force CLI invocation even in subagent mode.
+     */
+    readonly subagentModeAllowed?: boolean;
+}
+type ResolvedTarget = (ResolvedTargetBase & {
+    readonly kind: 'openai';
     readonly config: OpenAIResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'openrouter';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: OpenRouterResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'azure';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: AzureResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'anthropic';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: AnthropicResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'gemini';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: GeminiResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'codex';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: CodexResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'copilot-sdk';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: CopilotSdkResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'copilot-cli';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: CopilotCliResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'copilot-log';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: CopilotLogResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'pi-coding-agent';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: PiCodingAgentResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'pi-cli';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: PiCliResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'claude';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: ClaudeResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'claude-cli';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: ClaudeResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'claude-sdk';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: ClaudeResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'mock';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: MockResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'vscode' | 'vscode-insiders';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: VSCodeResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'agentv';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: AgentVResolvedConfig;
-} | {
+}) | (ResolvedTargetBase & {
     readonly kind: 'cli';
-    readonly name: string;
-    readonly graderTarget?: string;
-    readonly workers?: number;
-    readonly providerBatching?: boolean;
     readonly config: CliResolvedConfig;
-};
+});
+/**
+ * Optional settings accepted on ALL target definitions regardless of provider.
+ * Exported so the targets validator can reuse the same list — adding a field
+ * here automatically makes it valid in targets.yaml without a separate update.
+ */
+declare const COMMON_TARGET_SETTINGS: readonly ["provider_batching", "providerBatching", "subagent_mode_allowed", "subagentModeAllowed"];
 declare function resolveTargetDefinition(definition: TargetDefinition, env?: EnvLookup, evalFilePath?: string): ResolvedTarget;
 
 /**
@@ -2451,6 +2474,14 @@ declare function calculateRubricScore(result: z.infer<typeof rubricEvaluationSch
  * Build the output schema for score-range rubric evaluation.
  */
 declare function buildScoreRangeOutputSchema(): string;
+/**
+ * Extract all `ContentImage` blocks from assistant messages.
+ *
+ * Scans `messages` for assistant-role entries whose `content` is a `Content[]`
+ * array and collects every `ContentImage` block.  Non-assistant messages and
+ * plain-string content are skipped.
+ */
+declare function extractImageBlocks(messages: readonly Message[]): ContentImage[];
 
 /**
  * Built-in skill-trigger evaluator.
@@ -3507,6 +3538,17 @@ declare function getWorkspacePoolRoot(): string;
  */
 declare function trimBaselineResult(result: EvaluationResult): EvaluationResult;
 
+/** Default category for eval files without subdirectory structure. */
+declare const DEFAULT_CATEGORY = "Uncategorized";
+/**
+ * Derive a human-readable category from an eval file's relative path.
+ *
+ * Strips the filename and any `evals` directory segments, then joins
+ * remaining directories with `/`. Returns {@link DEFAULT_CATEGORY} for files
+ * at the root level.
+ */
+declare function deriveCategory(relativePath: string): string;
+
 /** Options for configuring the OTel trace exporter. */
 interface OtelExportOptions {
     /** OTLP endpoint URL */
@@ -3668,4 +3710,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AcquireWorkspaceOptions, type AgentKernel, type AgentVConfig$1 as AgentVConfig, type AgentVResolvedConfig, type AgentVConfig as AgentVTsConfig, type AnthropicResolvedConfig, type ArgsMatchMode, type AssertContext, type AssertEntry, type AssertFn, type AssertResult, type AssertionEntry, type AssertionResult, type AssistantTestMessage, type AzureResolvedConfig, type CacheConfig, type ChildEvaluatorResult, type ClaudeResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type ConfidenceIntervalAggregation, type ContainsAllEvaluatorConfig, type ContainsAnyEvaluatorConfig, type ContainsEvaluatorConfig, type CopilotCliResolvedConfig, type CopilotLogResolvedConfig, type CopilotSdkResolvedConfig, type CopilotSession, CostEvaluator, type CostEvaluatorConfig, type CostEvaluatorOptions, DEFAULT_EVALUATOR_TEMPLATE, DEFAULT_EVAL_PATTERNS, DEFAULT_EXPLORATION_TOOLS, DeterministicAssertionEvaluator, type EndsWithEvaluatorConfig, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EqualsEvaluatorConfig, type EvalAssertionInput, type EvalCase, type EvalConfig, type EvalMetadata, type EvalRunResult, type EvalSuiteResult, type EvalSummary, type EvalTest, type EvalTestInput, type EvalsJsonCase, type EvalsJsonFile, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorDispatchContext, type EvaluatorFactory, type EvaluatorFactoryFn, type EvaluatorKind, EvaluatorRegistry, type EvaluatorResult, type ExecutionDefaults, type ExecutionError, type ExecutionMetrics, ExecutionMetricsEvaluator, type ExecutionMetricsEvaluatorConfig, type ExecutionMetricsEvaluatorOptions, type ExecutionStatus, type FailOnError, type FailureStage, FieldAccuracyEvaluator, type FieldAccuracyEvaluatorConfig, type FieldAccuracyEvaluatorOptions, type FieldAggregationType, type FieldConfig, type FieldMatchType, type GeminiResolvedConfig, type GenerateRubricsOptions, type IcontainsAllEvaluatorConfig, type IcontainsAnyEvaluatorConfig, type IcontainsEvaluatorConfig, type InlineAssertEvaluatorConfig, type IsJsonEvaluatorConfig, type JsonObject, type JsonPrimitive, type JsonValue, LatencyEvaluator, type LatencyEvaluatorConfig, type LatencyEvaluatorOptions, LlmGraderEvaluator, type LlmGraderEvaluatorConfig, type LlmGraderEvaluatorOptions, type LlmGraderPromptAssembly, LlmGraderEvaluator as LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmGraderEvaluatorOptions as LlmJudgeEvaluatorOptions, type LlmGraderPromptAssembly as LlmJudgePromptAssembly, type LocalPathValidationError, type MeanAggregation, type Message, type MockResolvedConfig, OTEL_BACKEND_PRESETS, type OpenAIResolvedConfig, type OpenRouterResolvedConfig, type OtelBackendPreset, type OtelExportOptions, OtelStreamingObserver, OtelTraceExporter, OtlpJsonFileExporter, type OutputMessage, type PassAtKAggregation, type PiCliResolvedConfig, type PiCodingAgentResolvedConfig, type PoolSlot, type ProgressEvent, type PromptInputs, type PromptScriptConfig, type Provider, type ProviderFactoryFn, type ProviderKind, ProviderRegistry, type ProviderRequest, type ProviderResponse, type ProviderStreamCallbacks, type ProviderTokenUsage, type RegexEvaluatorConfig, type RepoCheckout, type RepoClone, type RepoConfig, RepoManager, type RepoSource, type ResolvedTarget, type ResolvedWorkspaceTemplate, ResponseCache, type RubricItem, type RubricsEvaluatorConfig, type RunEvalCaseOptions, type RunEvaluationOptions, type ScoreRange, type ScriptExecutionContext, SkillTriggerEvaluator, type SkillTriggerEvaluatorConfig, type StartsWithEvaluatorConfig, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetAccessConfig, type TargetDefinition, TemplateNotDirectoryError, TemplateNotFoundError, type TestMessage, type TestMessageContent, type TestMessageRole, type TokenUsage, TokenUsageEvaluator, type TokenUsageEvaluatorConfig, type TokenUsageEvaluatorOptions, type ToolCall, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceComputeResult, type TraceSummary, type TranspileResult, type TrialAggregation, type TrialResult, type TrialStrategy, type TrialsConfig, type UserTestMessage, type VSCodeResolvedConfig, type WorkspaceConfig, WorkspaceCreationError, type WorkspaceHookConfig, type WorkspaceHooksConfig, WorkspacePoolManager, type WorkspaceScriptConfig, assembleLlmGraderPrompt, assembleLlmGraderPrompt as assembleLlmJudgePrompt, avgToolDurationMs, buildDirectoryChain, buildOutputSchema, buildPromptInputs, buildRubricOutputSchema, buildScoreRangeOutputSchema, buildSearchRoots, calculateRubricScore, captureFileChanges, clampScore, cleanupEvalWorkspaces, cleanupWorkspace, computeTraceSummary, computeWorkspaceFingerprint, consumeClaudeLogEntries, consumeCodexLogEntries, consumeCopilotCliLogEntries, consumeCopilotSdkLogEntries, consumePiLogEntries, createAgentKernel, createBuiltinProviderRegistry, createBuiltinRegistry, createProvider, createTempWorkspace, deepEqual, defineConfig, detectFormat, discoverAssertions, discoverCopilotSessions, discoverGraders, discoverGraders as discoverJudges, discoverProviders, ensureVSCodeSubagents, evaluate, executeScript, executeWorkspaceScript, explorationRatio, extractCacheConfig, extractFailOnError, extractJsonBlob, extractTargetFromSuite, extractTargetsFromSuite, extractTargetsFromTestCase, extractThreshold, extractTrialsConfig, extractWorkersFromSuite, fileExists, findGitRoot, freeformEvaluationSchema, generateRubrics, getAgentvHome, getOutputFilenames, getSubagentsRoot, getTraceStateRoot, getWorkspacePath, getWorkspacePoolRoot, getWorkspacesRoot, initializeBaseline, isAgentSkillsFormat, isEvaluatorKind, isJsonObject, isJsonValue, isNonEmptyString, isTestMessage, isTestMessageRole, listTargetNames, loadConfig, loadEvalCaseById, loadEvalCases, loadEvalSuite, loadTestById, loadTestSuite, loadTests, loadTsConfig, mergeExecutionMetrics, negateScore, normalizeLineEndings, parseAgentSkillsEvals, parseJsonFromText, parseJsonSafe, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, resolveWorkspaceTemplate, rubricEvaluationSchema, runContainsAllAssertion, runContainsAnyAssertion, runContainsAssertion, runEndsWithAssertion, runEqualsAssertion, runEvalCase, runEvaluation, runIcontainsAllAssertion, runIcontainsAnyAssertion, runIcontainsAssertion, runIsJsonAssertion, runRegexAssertion, runStartsWithAssertion, scoreToVerdict, shouldEnableCache, shouldSkipCacheForTemperature, subscribeToClaudeLogEntries, subscribeToCodexLogEntries, subscribeToCopilotCliLogEntries, subscribeToCopilotSdkLogEntries, subscribeToPiLogEntries, substituteVariables, toCamelCaseDeep, toSnakeCaseDeep, tokensPerTool, transpileEvalYaml, transpileEvalYamlFile, trimBaselineResult };
+export { type AcquireWorkspaceOptions, type AgentKernel, type AgentVConfig$1 as AgentVConfig, type AgentVResolvedConfig, type AgentVConfig as AgentVTsConfig, type AnthropicResolvedConfig, type ArgsMatchMode, type AssertContext, type AssertEntry, type AssertFn, type AssertResult, type AssertionEntry, type AssertionResult, type AssistantTestMessage, type AzureResolvedConfig, COMMON_TARGET_SETTINGS, type CacheConfig, type ChildEvaluatorResult, type ClaudeResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type ConfidenceIntervalAggregation, type ContainsAllEvaluatorConfig, type ContainsAnyEvaluatorConfig, type ContainsEvaluatorConfig, type Content, type ContentFile, type ContentImage, type ContentText, type CopilotCliResolvedConfig, type CopilotLogResolvedConfig, type CopilotSdkResolvedConfig, type CopilotSession, CostEvaluator, type CostEvaluatorConfig, type CostEvaluatorOptions, DEFAULT_CATEGORY, DEFAULT_EVALUATOR_TEMPLATE, DEFAULT_EVAL_PATTERNS, DEFAULT_EXPLORATION_TOOLS, DeterministicAssertionEvaluator, type EndsWithEvaluatorConfig, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EqualsEvaluatorConfig, type EvalAssertionInput, type EvalCase, type EvalConfig, type EvalMetadata, type EvalRunResult, type EvalSuiteResult, type EvalSummary, type EvalTest, type EvalTestInput, type EvalsJsonCase, type EvalsJsonFile, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorDispatchContext, type EvaluatorFactory, type EvaluatorFactoryFn, type EvaluatorKind, EvaluatorRegistry, type EvaluatorResult, type ExecutionDefaults, type ExecutionError, type ExecutionMetrics, ExecutionMetricsEvaluator, type ExecutionMetricsEvaluatorConfig, type ExecutionMetricsEvaluatorOptions, type ExecutionStatus, type FailOnError, type FailureStage, FieldAccuracyEvaluator, type FieldAccuracyEvaluatorConfig, type FieldAccuracyEvaluatorOptions, type FieldAggregationType, type FieldConfig, type FieldMatchType, type GeminiResolvedConfig, type GenerateRubricsOptions, type IcontainsAllEvaluatorConfig, type IcontainsAnyEvaluatorConfig, type IcontainsEvaluatorConfig, type InlineAssertEvaluatorConfig, type IsJsonEvaluatorConfig, type JsonObject, type JsonPrimitive, type JsonValue, LatencyEvaluator, type LatencyEvaluatorConfig, type LatencyEvaluatorOptions, LlmGraderEvaluator, type LlmGraderEvaluatorConfig, type LlmGraderEvaluatorOptions, type LlmGraderPromptAssembly, LlmGraderEvaluator as LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmGraderEvaluatorOptions as LlmJudgeEvaluatorOptions, type LlmGraderPromptAssembly as LlmJudgePromptAssembly, type LocalPathValidationError, type MeanAggregation, type Message, type MockResolvedConfig, OTEL_BACKEND_PRESETS, type OpenAIResolvedConfig, type OpenRouterResolvedConfig, type OtelBackendPreset, type OtelExportOptions, OtelStreamingObserver, OtelTraceExporter, OtlpJsonFileExporter, type OutputMessage, type PassAtKAggregation, type PiCliResolvedConfig, type PiCodingAgentResolvedConfig, type PoolSlot, type ProgressEvent, type PromptInputs, type PromptScriptConfig, type Provider, type ProviderFactoryFn, type ProviderKind, ProviderRegistry, type ProviderRequest, type ProviderResponse, type ProviderStreamCallbacks, type ProviderTokenUsage, type RegexEvaluatorConfig, type RepoCheckout, type RepoClone, type RepoConfig, RepoManager, type RepoSource, type ResolvedTarget, type ResolvedWorkspaceTemplate, ResponseCache, type RubricItem, type RubricsEvaluatorConfig, type RunEvalCaseOptions, type RunEvaluationOptions, type ScoreRange, type ScriptExecutionContext, SkillTriggerEvaluator, type SkillTriggerEvaluatorConfig, type StartsWithEvaluatorConfig, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetAccessConfig, type TargetDefinition, TemplateNotDirectoryError, TemplateNotFoundError, type TestMessage, type TestMessageContent, type TestMessageRole, type TokenUsage, TokenUsageEvaluator, type TokenUsageEvaluatorConfig, type TokenUsageEvaluatorOptions, type ToolCall, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceComputeResult, type TraceSummary, type TranspileResult, type TrialAggregation, type TrialResult, type TrialStrategy, type TrialsConfig, type UserTestMessage, type VSCodeResolvedConfig, type WorkspaceConfig, WorkspaceCreationError, type WorkspaceHookConfig, type WorkspaceHooksConfig, WorkspacePoolManager, type WorkspaceScriptConfig, assembleLlmGraderPrompt, assembleLlmGraderPrompt as assembleLlmJudgePrompt, avgToolDurationMs, buildDirectoryChain, buildOutputSchema, buildPromptInputs, buildRubricOutputSchema, buildScoreRangeOutputSchema, buildSearchRoots, calculateRubricScore, captureFileChanges, clampScore, cleanupEvalWorkspaces, cleanupWorkspace, computeTraceSummary, computeWorkspaceFingerprint, consumeClaudeLogEntries, consumeCodexLogEntries, consumeCopilotCliLogEntries, consumeCopilotSdkLogEntries, consumePiLogEntries, createAgentKernel, createBuiltinProviderRegistry, createBuiltinRegistry, createProvider, createTempWorkspace, deepEqual, defineConfig, deriveCategory, detectFormat, discoverAssertions, discoverCopilotSessions, discoverGraders, discoverGraders as discoverJudges, discoverProviders, ensureVSCodeSubagents, evaluate, executeScript, executeWorkspaceScript, explorationRatio, extractCacheConfig, extractFailOnError, extractImageBlocks, extractJsonBlob, extractLastAssistantContent, extractTargetFromSuite, extractTargetsFromSuite, extractTargetsFromTestCase, extractThreshold, extractTrialsConfig, extractWorkersFromSuite, fileExists, findGitRoot, freeformEvaluationSchema, generateRubrics, getAgentvHome, getOutputFilenames, getSubagentsRoot, getTextContent, getTraceStateRoot, getWorkspacePath, getWorkspacePoolRoot, getWorkspacesRoot, initializeBaseline, isAgentSkillsFormat, isContent, isContentArray, isEvaluatorKind, isJsonObject, isJsonValue, isNonEmptyString, isTestMessage, isTestMessageRole, listTargetNames, loadConfig, loadEvalCaseById, loadEvalCases, loadEvalSuite, loadTestById, loadTestSuite, loadTests, loadTsConfig, mergeExecutionMetrics, negateScore, normalizeLineEndings, parseAgentSkillsEvals, parseJsonFromText, parseJsonSafe, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, resolveWorkspaceTemplate, rubricEvaluationSchema, runContainsAllAssertion, runContainsAnyAssertion, runContainsAssertion, runEndsWithAssertion, runEqualsAssertion, runEvalCase, runEvaluation, runIcontainsAllAssertion, runIcontainsAnyAssertion, runIcontainsAssertion, runIsJsonAssertion, runRegexAssertion, runStartsWithAssertion, scoreToVerdict, shouldEnableCache, shouldSkipCacheForTemperature, subscribeToClaudeLogEntries, subscribeToCodexLogEntries, subscribeToCopilotCliLogEntries, subscribeToCopilotSdkLogEntries, subscribeToPiLogEntries, substituteVariables, toCamelCaseDeep, toSnakeCaseDeep, tokensPerTool, transpileEvalYaml, transpileEvalYamlFile, trimBaselineResult };