@agentv/core 2.9.0-next.2 → 2.11.0

package/dist/index.d.cts

@@ -8,7 +8,7 @@ interface ChatMessage {
     readonly name?: string;
 }
 type ChatPrompt = readonly ChatMessage[];
-type ProviderKind = 'azure' | 'anthropic' | 'gemini' | 'codex' | 'copilot' | 'copilot-cli' | 'pi-coding-agent' | 'pi-agent-sdk' | 'claude' | 'cli' | 'mock' | 'vscode' | 'vscode-insiders';
+type ProviderKind = 'azure' | 'anthropic' | 'gemini' | 'codex' | 'copilot-sdk' | 'copilot-cli' | 'pi-coding-agent' | 'pi-agent-sdk' | 'claude' | 'cli' | 'mock' | 'vscode' | 'vscode-insiders';
 /** Callbacks for real-time observability during provider execution */
 interface ProviderStreamCallbacks {
     onToolCallStart?: (toolName: string, toolCallId?: string) => void;
@@ -453,7 +453,7 @@ declare function isJsonValue(value: unknown): value is JsonValue;
  * - Either content (string or array of objects) OR tool_calls (for assistant messages)
  */
 declare function isTestMessage(value: unknown): value is TestMessage;
-declare const EVALUATOR_KIND_VALUES: readonly ["code_judge", "llm_judge", "rubric", "composite", "tool_trajectory", "field_accuracy", "latency", "cost", "token_usage", "execution_metrics", "agent_judge", "contains", "regex", "is_json", "equals", "rubrics"];
+declare const EVALUATOR_KIND_VALUES: readonly ["code_judge", "llm_judge", "rubric", "composite", "tool_trajectory", "field_accuracy", "latency", "cost", "token_usage", "execution_metrics", "agent_judge", "contains", "contains_any", "contains_all", "icontains", "icontains_any", "icontains_all", "starts_with", "ends_with", "regex", "is_json", "equals", "rubrics"];
 type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
 declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
 /**
@@ -491,10 +491,43 @@ type WorkspaceScriptConfig = {
  * - before_each: runs before each test (optional)
  * - after_each: runs after each test (e.g., reset git state)
  */
+type RepoSource = {
+    readonly type: 'git';
+    readonly url: string;
+} | {
+    readonly type: 'local';
+    readonly path: string;
+};
+type RepoCheckout = {
+    readonly ref?: string;
+    readonly resolve?: 'remote' | 'local';
+    readonly ancestor?: number;
+};
+type RepoClone = {
+    readonly depth?: number;
+    readonly filter?: string;
+    readonly sparse?: readonly string[];
+};
+type RepoConfig = {
+    readonly path: string;
+    readonly source: RepoSource;
+    readonly checkout?: RepoCheckout;
+    readonly clone?: RepoClone;
+};
+type ResetConfig = {
+    readonly strategy?: 'none' | 'hard' | 'recreate';
+    readonly after_each?: boolean;
+};
 type WorkspaceConfig = {
     /** Template directory or .code-workspace file. Directories are copied to temp workspace.
      *  .code-workspace files are used by VS Code providers; CLI providers use the parent directory. */
     readonly template?: string;
+    /** Isolation strategy for workspace: shared (default) or per_test */
+    readonly isolation?: 'shared' | 'per_test';
+    /** Repository definitions to clone/checkout into workspace */
+    readonly repos?: readonly RepoConfig[];
+    /** Reset configuration for repos between test runs */
+    readonly reset?: ResetConfig;
     /** Command to run once before first test (after workspace creation, before git baseline) */
     readonly before_all?: WorkspaceScriptConfig;
     /** Command to run once after last test (before workspace cleanup) */
@@ -777,6 +810,97 @@ type ContainsEvaluatorConfig = {
     /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
     readonly negate?: boolean;
 };
+/**
+ * Configuration for the contains_any assertion evaluator.
+ * Checks whether the candidate output contains ANY of the specified substrings.
+ */
+type ContainsAnyEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'contains_any';
+    readonly value: readonly string[];
+    readonly weight?: number;
+    readonly required?: boolean | number;
+    /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
+    readonly negate?: boolean;
+};
+/**
+ * Configuration for the contains_all assertion evaluator.
+ * Checks whether the candidate output contains ALL of the specified substrings.
+ */
+type ContainsAllEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'contains_all';
+    readonly value: readonly string[];
+    readonly weight?: number;
+    readonly required?: boolean | number;
+    /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
+    readonly negate?: boolean;
+};
+/**
+ * Configuration for the icontains assertion evaluator.
+ * Case-insensitive check whether the candidate output contains a specified substring.
+ */
+type IcontainsEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'icontains';
+    readonly value: string;
+    readonly weight?: number;
+    readonly required?: boolean | number;
+    /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
+    readonly negate?: boolean;
+};
+/**
+ * Configuration for the icontains_any assertion evaluator.
+ * Case-insensitive check whether the candidate output contains ANY of the specified substrings.
+ */
+type IcontainsAnyEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'icontains_any';
+    readonly value: readonly string[];
+    readonly weight?: number;
+    readonly required?: boolean | number;
+    /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
+    readonly negate?: boolean;
+};
+/**
+ * Configuration for the icontains_all assertion evaluator.
+ * Case-insensitive check whether the candidate output contains ALL of the specified substrings.
+ */
+type IcontainsAllEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'icontains_all';
+    readonly value: readonly string[];
+    readonly weight?: number;
+    readonly required?: boolean | number;
+    /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
+    readonly negate?: boolean;
+};
+/**
+ * Configuration for the starts_with assertion evaluator.
+ * Checks whether the candidate output starts with a specified string (both trimmed).
+ */
+type StartsWithEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'starts_with';
+    readonly value: string;
+    readonly weight?: number;
+    readonly required?: boolean | number;
+    /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
+    readonly negate?: boolean;
+};
+/**
+ * Configuration for the ends_with assertion evaluator.
+ * Checks whether the candidate output ends with a specified string (both trimmed).
+ */
+type EndsWithEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'ends_with';
+    readonly value: string;
+    readonly weight?: number;
+    readonly required?: boolean | number;
+    /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
+    readonly negate?: boolean;
+};
 /**
  * Configuration for the regex assertion evaluator.
  * Checks whether the candidate output matches a regular expression pattern.
@@ -785,6 +909,8 @@ type RegexEvaluatorConfig = {
     readonly name: string;
     readonly type: 'regex';
     readonly value: string;
+    /** Optional regex flags (e.g., "i" for case-insensitive, "m" for multiline) */
+    readonly flags?: string;
     readonly weight?: number;
     readonly required?: boolean | number;
     /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
@@ -828,7 +954,7 @@ type RubricsEvaluatorConfig = {
     /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
     readonly negate?: boolean;
 };
-type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | CompositeEvaluatorConfig | ToolTrajectoryEvaluatorConfig | FieldAccuracyEvaluatorConfig | LatencyEvaluatorConfig | CostEvaluatorConfig | TokenUsageEvaluatorConfig | ExecutionMetricsEvaluatorConfig | AgentJudgeEvaluatorConfig | ContainsEvaluatorConfig | RegexEvaluatorConfig | IsJsonEvaluatorConfig | EqualsEvaluatorConfig | RubricsEvaluatorConfig;
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | CompositeEvaluatorConfig | ToolTrajectoryEvaluatorConfig | FieldAccuracyEvaluatorConfig | LatencyEvaluatorConfig | CostEvaluatorConfig | TokenUsageEvaluatorConfig | ExecutionMetricsEvaluatorConfig | AgentJudgeEvaluatorConfig | ContainsEvaluatorConfig | ContainsAnyEvaluatorConfig | ContainsAllEvaluatorConfig | IcontainsEvaluatorConfig | IcontainsAnyEvaluatorConfig | IcontainsAllEvaluatorConfig | StartsWithEvaluatorConfig | EndsWithEvaluatorConfig | RegexEvaluatorConfig | IsJsonEvaluatorConfig | EqualsEvaluatorConfig | RubricsEvaluatorConfig;
 /**
  * Eval test definition sourced from AgentV specs.
  */
@@ -968,7 +1094,7 @@ interface EvaluationResult {
     /** Whether the evaluation was skipped due to suite-level budget exhaustion */
     readonly budgetExceeded?: boolean;
 }
-type EvaluationVerdict = 'pass' | 'fail' | 'borderline';
+type EvaluationVerdict = 'pass' | 'fail' | 'borderline' | 'skip';
 interface EvaluatorResult {
     readonly name: string;
     readonly type: EvaluatorKind;
@@ -1427,7 +1553,7 @@ type ResolvedTarget = {
     readonly providerBatching?: boolean;
     readonly config: CodexResolvedConfig;
 } | {
-    readonly kind: 'copilot';
+    readonly kind: 'copilot-sdk';
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
@@ -1941,7 +2067,7 @@ declare function buildRubricOutputSchema(): string;
 declare function substituteVariables(template: string, variables: Record<string, string>): string;
 declare function calculateRubricScore(result: z.infer<typeof rubricEvaluationSchema>, rubrics: readonly RubricItem[]): {
     score: number;
-    verdict: 'pass' | 'fail' | 'borderline';
+    verdict: EvaluationVerdict;
     hits: string[];
     misses: string[];
 };
@@ -2072,8 +2198,22 @@ type AssertionResult = {
 };
 /** Checks if `output` contains the given `value` substring. */
 declare function runContainsAssertion(output: string, value: string): AssertionResult;
-/** Checks if `output` matches the given regex `pattern`. */
-declare function runRegexAssertion(output: string, pattern: string): AssertionResult;
+/** Checks if `output` contains ANY of the given `values`. */
+declare function runContainsAnyAssertion(output: string, values: readonly string[]): AssertionResult;
+/** Checks if `output` contains ALL of the given `values`. */
+declare function runContainsAllAssertion(output: string, values: readonly string[]): AssertionResult;
+/** Case-insensitive check if `output` contains `value`. */
+declare function runIcontainsAssertion(output: string, value: string): AssertionResult;
+/** Case-insensitive check if `output` contains ANY of the given `values`. */
+declare function runIcontainsAnyAssertion(output: string, values: readonly string[]): AssertionResult;
+/** Case-insensitive check if `output` contains ALL of the given `values`. */
+declare function runIcontainsAllAssertion(output: string, values: readonly string[]): AssertionResult;
+/** Checks if `output` starts with `value` (both trimmed). */
+declare function runStartsWithAssertion(output: string, value: string): AssertionResult;
+/** Checks if `output` ends with `value` (both trimmed). */
+declare function runEndsWithAssertion(output: string, value: string): AssertionResult;
+/** Checks if `output` matches the given regex `pattern` with optional `flags`. */
+declare function runRegexAssertion(output: string, pattern: string, flags?: string): AssertionResult;
 /** Checks if `output` is valid JSON. */
 declare function runIsJsonAssertion(output: string): AssertionResult;
 /** Checks if `output` exactly equals `value` (both trimmed). */
@@ -2150,6 +2290,28 @@ declare class DeterministicAssertionEvaluator implements Evaluator {
     evaluate(context: EvaluationContext): EvaluationScore;
 }
 
+declare class RepoManager {
+    private readonly cacheDir;
+    constructor(cacheDir?: string);
+    /**
+     * Ensure a bare mirror cache exists for the given source.
+     * Creates on first access, fetches updates on subsequent calls.
+     * Returns the absolute path to the cache directory.
+     */
+    ensureCache(source: RepoSource): Promise<string>;
+    /**
+     * Clone a repo from cache into the workspace at the configured path.
+     * Handles checkout, ref resolution, ancestor walking, shallow clone, sparse checkout.
+     */
+    materialize(repo: RepoConfig, workspacePath: string): Promise<void>;
+    /** Materialize all repos into the workspace. */
+    materializeAll(repos: readonly RepoConfig[], workspacePath: string): Promise<void>;
+    /** Reset repos in workspace to their checkout state. */
+    reset(repos: readonly RepoConfig[], workspacePath: string, strategy: 'hard' | 'recreate'): Promise<void>;
+    /** Remove the entire cache directory. */
+    cleanCache(): Promise<void>;
+}
+
 type MaybePromise<T> = T | Promise<T>;
 interface EvaluationCache {
     get(key: string): MaybePromise<ProviderResponse | undefined>;
@@ -2189,6 +2351,8 @@ interface RunEvalCaseOptions {
     readonly streamCallbacks?: ProviderStreamCallbacks;
     /** Evaluator type registry (with custom assertions discovered) */
     readonly typeRegistry?: EvaluatorRegistry;
+    /** RepoManager instance for repo lifecycle (shared workspace mode) */
+    readonly repoManager?: RepoManager;
 }
 interface ProgressEvent {
     readonly workerId: number;
@@ -2918,4 +3082,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { AgentJudgeEvaluator, type AgentJudgeEvaluatorConfig, type AgentJudgeEvaluatorOptions, type AgentKernel, type AgentVConfig$1 as AgentVConfig, type AgentVConfig as AgentVTsConfig, type AnthropicResolvedConfig, type ArgsMatchMode, type AssertionResult, type AssistantTestMessage, type AzureResolvedConfig, type CacheConfig, type ChildEvaluatorResult, type ClaudeResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type ConfidenceIntervalAggregation, type ContainsEvaluatorConfig, type CopilotCliResolvedConfig, type CopilotSdkResolvedConfig, CostEvaluator, type CostEvaluatorConfig, type CostEvaluatorOptions, DEFAULT_EVALUATOR_TEMPLATE, DEFAULT_EVAL_PATTERNS, DEFAULT_EXPLORATION_TOOLS, DeterministicAssertionEvaluator, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EqualsEvaluatorConfig, type EvalAssertionInput, type EvalCase, type EvalConfig, type EvalMetadata, type EvalRunResult, type EvalSuiteResult, type EvalSummary, type EvalTest, type EvalTestInput, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorDispatchContext, type EvaluatorFactory, type EvaluatorFactoryFn, type EvaluatorKind, EvaluatorRegistry, type EvaluatorResult, type ExecutionMetrics, ExecutionMetricsEvaluator, type ExecutionMetricsEvaluatorConfig, type ExecutionMetricsEvaluatorOptions, FieldAccuracyEvaluator, type FieldAccuracyEvaluatorConfig, type FieldAccuracyEvaluatorOptions, type FieldAggregationType, type FieldConfig, type FieldMatchType, type GeminiResolvedConfig, type GenerateRubricsOptions, type IsJsonEvaluatorConfig, type JsonObject, type JsonPrimitive, type JsonValue, LatencyEvaluator, type LatencyEvaluatorConfig, type LatencyEvaluatorOptions, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type LlmJudgePromptAssembly, type MeanAggregation, type Message, type MockResolvedConfig, OTEL_BACKEND_PRESETS, type OtelBackendPreset, type OtelExportOptions, OtelStreamingObserver, OtelTraceExporter, OtlpJsonFileExporter, type OutputMessage, type PassAtKAggregation, type PiAgentSdkResolvedConfig, type PiCodingAgentResolvedConfig, type ProgressEvent, type PromptInputs, type PromptScriptConfig, type Provider, type ProviderFactoryFn, type ProviderKind, ProviderRegistry, type ProviderRequest, type ProviderResponse, type ProviderStreamCallbacks, type ProviderTokenUsage, type RegexEvaluatorConfig, type ResolvedTarget, type ResolvedWorkspaceTemplate, ResponseCache, type RubricItem, type RubricsEvaluatorConfig, type RunEvalCaseOptions, type RunEvaluationOptions, type ScoreRange, type ScriptExecutionContext, SimpleTraceFileExporter, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetAccessConfig, type TargetDefinition, TemplateNotDirectoryError, TemplateNotFoundError, type TestMessage, type TestMessageContent, type TestMessageRole, type TokenUsage, TokenUsageEvaluator, type TokenUsageEvaluatorConfig, type TokenUsageEvaluatorOptions, type ToolCall, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceComputeResult, type TraceSummary, type TrialAggregation, type TrialResult, type TrialStrategy, type TrialsConfig, type UserTestMessage, type VSCodeResolvedConfig, type WorkspaceConfig, WorkspaceCreationError, type WorkspaceScriptConfig, assembleLlmJudgePrompt, avgToolDurationMs, buildDirectoryChain, buildOutputSchema, buildPromptInputs, buildRubricOutputSchema, buildScoreRangeOutputSchema, buildSearchRoots, calculateRubricScore, captureFileChanges, clampScore, cleanupEvalWorkspaces, cleanupWorkspace, computeTraceSummary, consumeClaudeLogEntries, consumeCodexLogEntries, consumeCopilotCliLogEntries, consumeCopilotSdkLogEntries, consumePiLogEntries, createAgentKernel, createBuiltinProviderRegistry, createBuiltinRegistry, createProvider, createTempWorkspace, deepEqual, defineConfig, detectFormat, discoverAssertions, discoverProviders, ensureVSCodeSubagents, evaluate, executeScript, executeWorkspaceScript, explorationRatio, extractCacheConfig, extractJsonBlob, extractTargetFromSuite, extractTargetsFromSuite, extractTargetsFromTestCase, extractTrialsConfig, fileExists, findGitRoot, freeformEvaluationSchema, generateRubrics, getHitCount, getWorkspacePath, initializeBaseline, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isNonEmptyString, isTestMessage, isTestMessageRole, listTargetNames, loadConfig, loadEvalCaseById, loadEvalCases, loadEvalSuite, loadTestById, loadTestSuite, loadTests, loadTsConfig, mergeExecutionMetrics, negateScore, normalizeLineEndings, parseJsonFromText, parseJsonSafe, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, resolveWorkspaceTemplate, rubricEvaluationSchema, runContainsAssertion, runEqualsAssertion, runEvalCase, runEvaluation, runIsJsonAssertion, runRegexAssertion, scoreToVerdict, shouldEnableCache, shouldSkipCacheForTemperature, subscribeToClaudeLogEntries, subscribeToCodexLogEntries, subscribeToCopilotCliLogEntries, subscribeToCopilotSdkLogEntries, subscribeToPiLogEntries, substituteVariables, toCamelCaseDeep, toSnakeCaseDeep, tokensPerTool, trimBaselineResult };
+export { AgentJudgeEvaluator, type AgentJudgeEvaluatorConfig, type AgentJudgeEvaluatorOptions, type AgentKernel, type AgentVConfig$1 as AgentVConfig, type AgentVConfig as AgentVTsConfig, type AnthropicResolvedConfig, type ArgsMatchMode, type AssertionResult, type AssistantTestMessage, type AzureResolvedConfig, type CacheConfig, type ChildEvaluatorResult, type ClaudeResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type ConfidenceIntervalAggregation, type ContainsAllEvaluatorConfig, type ContainsAnyEvaluatorConfig, type ContainsEvaluatorConfig, type CopilotCliResolvedConfig, type CopilotSdkResolvedConfig, CostEvaluator, type CostEvaluatorConfig, type CostEvaluatorOptions, DEFAULT_EVALUATOR_TEMPLATE, DEFAULT_EVAL_PATTERNS, DEFAULT_EXPLORATION_TOOLS, DeterministicAssertionEvaluator, type EndsWithEvaluatorConfig, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EqualsEvaluatorConfig, type EvalAssertionInput, type EvalCase, type EvalConfig, type EvalMetadata, type EvalRunResult, type EvalSuiteResult, type EvalSummary, type EvalTest, type EvalTestInput, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorDispatchContext, type EvaluatorFactory, type EvaluatorFactoryFn, type EvaluatorKind, EvaluatorRegistry, type EvaluatorResult, type ExecutionMetrics, ExecutionMetricsEvaluator, type ExecutionMetricsEvaluatorConfig, type ExecutionMetricsEvaluatorOptions, FieldAccuracyEvaluator, type FieldAccuracyEvaluatorConfig, type FieldAccuracyEvaluatorOptions, type FieldAggregationType, type FieldConfig, type FieldMatchType, type GeminiResolvedConfig, type GenerateRubricsOptions, type IcontainsAllEvaluatorConfig, type IcontainsAnyEvaluatorConfig, type IcontainsEvaluatorConfig, type IsJsonEvaluatorConfig, type JsonObject, type JsonPrimitive, type JsonValue, LatencyEvaluator, type LatencyEvaluatorConfig, type LatencyEvaluatorOptions, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type LlmJudgePromptAssembly, type MeanAggregation, type Message, type MockResolvedConfig, OTEL_BACKEND_PRESETS, type OtelBackendPreset, type OtelExportOptions, OtelStreamingObserver, OtelTraceExporter, OtlpJsonFileExporter, type OutputMessage, type PassAtKAggregation, type PiAgentSdkResolvedConfig, type PiCodingAgentResolvedConfig, type ProgressEvent, type PromptInputs, type PromptScriptConfig, type Provider, type ProviderFactoryFn, type ProviderKind, ProviderRegistry, type ProviderRequest, type ProviderResponse, type ProviderStreamCallbacks, type ProviderTokenUsage, type RegexEvaluatorConfig, type RepoCheckout, type RepoClone, type RepoConfig, RepoManager, type RepoSource, type ResetConfig, type ResolvedTarget, type ResolvedWorkspaceTemplate, ResponseCache, type RubricItem, type RubricsEvaluatorConfig, type RunEvalCaseOptions, type RunEvaluationOptions, type ScoreRange, type ScriptExecutionContext, SimpleTraceFileExporter, type StartsWithEvaluatorConfig, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetAccessConfig, type TargetDefinition, TemplateNotDirectoryError, TemplateNotFoundError, type TestMessage, type TestMessageContent, type TestMessageRole, type TokenUsage, TokenUsageEvaluator, type TokenUsageEvaluatorConfig, type TokenUsageEvaluatorOptions, type ToolCall, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceComputeResult, type TraceSummary, type TrialAggregation, type TrialResult, type TrialStrategy, type TrialsConfig, type UserTestMessage, type VSCodeResolvedConfig, type WorkspaceConfig, WorkspaceCreationError, type WorkspaceScriptConfig, assembleLlmJudgePrompt, avgToolDurationMs, buildDirectoryChain, buildOutputSchema, buildPromptInputs, buildRubricOutputSchema, buildScoreRangeOutputSchema, buildSearchRoots, calculateRubricScore, captureFileChanges, clampScore, cleanupEvalWorkspaces, cleanupWorkspace, computeTraceSummary, consumeClaudeLogEntries, consumeCodexLogEntries, consumeCopilotCliLogEntries, consumeCopilotSdkLogEntries, consumePiLogEntries, createAgentKernel, createBuiltinProviderRegistry, createBuiltinRegistry, createProvider, createTempWorkspace, deepEqual, defineConfig, detectFormat, discoverAssertions, discoverProviders, ensureVSCodeSubagents, evaluate, executeScript, executeWorkspaceScript, explorationRatio, extractCacheConfig, extractJsonBlob, extractTargetFromSuite, extractTargetsFromSuite, extractTargetsFromTestCase, extractTrialsConfig, fileExists, findGitRoot, freeformEvaluationSchema, generateRubrics, getHitCount, getWorkspacePath, initializeBaseline, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isNonEmptyString, isTestMessage, isTestMessageRole, listTargetNames, loadConfig, loadEvalCaseById, loadEvalCases, loadEvalSuite, loadTestById, loadTestSuite, loadTests, loadTsConfig, mergeExecutionMetrics, negateScore, normalizeLineEndings, parseJsonFromText, parseJsonSafe, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, resolveWorkspaceTemplate, rubricEvaluationSchema, runContainsAllAssertion, runContainsAnyAssertion, runContainsAssertion, runEndsWithAssertion, runEqualsAssertion, runEvalCase, runEvaluation, runIcontainsAllAssertion, runIcontainsAnyAssertion, runIcontainsAssertion, runIsJsonAssertion, runRegexAssertion, runStartsWithAssertion, scoreToVerdict, shouldEnableCache, shouldSkipCacheForTemperature, subscribeToClaudeLogEntries, subscribeToCodexLogEntries, subscribeToCopilotCliLogEntries, subscribeToCopilotSdkLogEntries, subscribeToPiLogEntries, substituteVariables, toCamelCaseDeep, toSnakeCaseDeep, tokensPerTool, trimBaselineResult };

package/dist/index.d.ts

@@ -8,7 +8,7 @@ interface ChatMessage {
     readonly name?: string;
 }
 type ChatPrompt = readonly ChatMessage[];
-type ProviderKind = 'azure' | 'anthropic' | 'gemini' | 'codex' | 'copilot' | 'copilot-cli' | 'pi-coding-agent' | 'pi-agent-sdk' | 'claude' | 'cli' | 'mock' | 'vscode' | 'vscode-insiders';
+type ProviderKind = 'azure' | 'anthropic' | 'gemini' | 'codex' | 'copilot-sdk' | 'copilot-cli' | 'pi-coding-agent' | 'pi-agent-sdk' | 'claude' | 'cli' | 'mock' | 'vscode' | 'vscode-insiders';
 /** Callbacks for real-time observability during provider execution */
 interface ProviderStreamCallbacks {
     onToolCallStart?: (toolName: string, toolCallId?: string) => void;
@@ -453,7 +453,7 @@ declare function isJsonValue(value: unknown): value is JsonValue;
  * - Either content (string or array of objects) OR tool_calls (for assistant messages)
  */
 declare function isTestMessage(value: unknown): value is TestMessage;
-declare const EVALUATOR_KIND_VALUES: readonly ["code_judge", "llm_judge", "rubric", "composite", "tool_trajectory", "field_accuracy", "latency", "cost", "token_usage", "execution_metrics", "agent_judge", "contains", "regex", "is_json", "equals", "rubrics"];
+declare const EVALUATOR_KIND_VALUES: readonly ["code_judge", "llm_judge", "rubric", "composite", "tool_trajectory", "field_accuracy", "latency", "cost", "token_usage", "execution_metrics", "agent_judge", "contains", "contains_any", "contains_all", "icontains", "icontains_any", "icontains_all", "starts_with", "ends_with", "regex", "is_json", "equals", "rubrics"];
 type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
 declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
 /**
@@ -491,10 +491,43 @@ type WorkspaceScriptConfig = {
  * - before_each: runs before each test (optional)
  * - after_each: runs after each test (e.g., reset git state)
  */
+type RepoSource = {
+    readonly type: 'git';
+    readonly url: string;
+} | {
+    readonly type: 'local';
+    readonly path: string;
+};
+type RepoCheckout = {
+    readonly ref?: string;
+    readonly resolve?: 'remote' | 'local';
+    readonly ancestor?: number;
+};
+type RepoClone = {
+    readonly depth?: number;
+    readonly filter?: string;
+    readonly sparse?: readonly string[];
+};
+type RepoConfig = {
+    readonly path: string;
+    readonly source: RepoSource;
+    readonly checkout?: RepoCheckout;
+    readonly clone?: RepoClone;
+};
+type ResetConfig = {
+    readonly strategy?: 'none' | 'hard' | 'recreate';
+    readonly after_each?: boolean;
+};
 type WorkspaceConfig = {
     /** Template directory or .code-workspace file. Directories are copied to temp workspace.
      *  .code-workspace files are used by VS Code providers; CLI providers use the parent directory. */
     readonly template?: string;
+    /** Isolation strategy for workspace: shared (default) or per_test */
+    readonly isolation?: 'shared' | 'per_test';
+    /** Repository definitions to clone/checkout into workspace */
+    readonly repos?: readonly RepoConfig[];
+    /** Reset configuration for repos between test runs */
+    readonly reset?: ResetConfig;
     /** Command to run once before first test (after workspace creation, before git baseline) */
     readonly before_all?: WorkspaceScriptConfig;
     /** Command to run once after last test (before workspace cleanup) */
@@ -777,6 +810,97 @@ type ContainsEvaluatorConfig = {
     /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
     readonly negate?: boolean;
 };
+/**
+ * Configuration for the contains_any assertion evaluator.
+ * Checks whether the candidate output contains ANY of the specified substrings.
+ */
+type ContainsAnyEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'contains_any';
+    readonly value: readonly string[];
+    readonly weight?: number;
+    readonly required?: boolean | number;
+    /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
+    readonly negate?: boolean;
+};
+/**
+ * Configuration for the contains_all assertion evaluator.
+ * Checks whether the candidate output contains ALL of the specified substrings.
+ */
+type ContainsAllEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'contains_all';
+    readonly value: readonly string[];
+    readonly weight?: number;
+    readonly required?: boolean | number;
+    /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
+    readonly negate?: boolean;
+};
+/**
+ * Configuration for the icontains assertion evaluator.
+ * Case-insensitive check whether the candidate output contains a specified substring.
+ */
+type IcontainsEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'icontains';
+    readonly value: string;
+    readonly weight?: number;
+    readonly required?: boolean | number;
+    /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
+    readonly negate?: boolean;
+};
+/**
+ * Configuration for the icontains_any assertion evaluator.
+ * Case-insensitive check whether the candidate output contains ANY of the specified substrings.
+ */
+type IcontainsAnyEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'icontains_any';
+    readonly value: readonly string[];
+    readonly weight?: number;
+    readonly required?: boolean | number;
+    /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
+    readonly negate?: boolean;
+};
+/**
+ * Configuration for the icontains_all assertion evaluator.
+ * Case-insensitive check whether the candidate output contains ALL of the specified substrings.
+ */
+type IcontainsAllEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'icontains_all';
+    readonly value: readonly string[];
+    readonly weight?: number;
+    readonly required?: boolean | number;
+    /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
+    readonly negate?: boolean;
+};
+/**
+ * Configuration for the starts_with assertion evaluator.
+ * Checks whether the candidate output starts with a specified string (both trimmed).
+ */
+type StartsWithEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'starts_with';
+    readonly value: string;
+    readonly weight?: number;
+    readonly required?: boolean | number;
+    /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
+    readonly negate?: boolean;
+};
+/**
+ * Configuration for the ends_with assertion evaluator.
+ * Checks whether the candidate output ends with a specified string (both trimmed).
+ */
+type EndsWithEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'ends_with';
+    readonly value: string;
+    readonly weight?: number;
+    readonly required?: boolean | number;
+    /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
+    readonly negate?: boolean;
+};
 /**
  * Configuration for the regex assertion evaluator.
  * Checks whether the candidate output matches a regular expression pattern.
@@ -785,6 +909,8 @@ type RegexEvaluatorConfig = {
     readonly name: string;
     readonly type: 'regex';
     readonly value: string;
+    /** Optional regex flags (e.g., "i" for case-insensitive, "m" for multiline) */
+    readonly flags?: string;
     readonly weight?: number;
     readonly required?: boolean | number;
     /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
@@ -828,7 +954,7 @@ type RubricsEvaluatorConfig = {
     /** When true, inverts the evaluator score (1 - score) and swaps pass/fail verdict */
     readonly negate?: boolean;
 };
-type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | CompositeEvaluatorConfig | ToolTrajectoryEvaluatorConfig | FieldAccuracyEvaluatorConfig | LatencyEvaluatorConfig | CostEvaluatorConfig | TokenUsageEvaluatorConfig | ExecutionMetricsEvaluatorConfig | AgentJudgeEvaluatorConfig | ContainsEvaluatorConfig | RegexEvaluatorConfig | IsJsonEvaluatorConfig | EqualsEvaluatorConfig | RubricsEvaluatorConfig;
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | CompositeEvaluatorConfig | ToolTrajectoryEvaluatorConfig | FieldAccuracyEvaluatorConfig | LatencyEvaluatorConfig | CostEvaluatorConfig | TokenUsageEvaluatorConfig | ExecutionMetricsEvaluatorConfig | AgentJudgeEvaluatorConfig | ContainsEvaluatorConfig | ContainsAnyEvaluatorConfig | ContainsAllEvaluatorConfig | IcontainsEvaluatorConfig | IcontainsAnyEvaluatorConfig | IcontainsAllEvaluatorConfig | StartsWithEvaluatorConfig | EndsWithEvaluatorConfig | RegexEvaluatorConfig | IsJsonEvaluatorConfig | EqualsEvaluatorConfig | RubricsEvaluatorConfig;
 /**
  * Eval test definition sourced from AgentV specs.
  */
@@ -968,7 +1094,7 @@ interface EvaluationResult {
     /** Whether the evaluation was skipped due to suite-level budget exhaustion */
     readonly budgetExceeded?: boolean;
 }
-type EvaluationVerdict = 'pass' | 'fail' | 'borderline';
+type EvaluationVerdict = 'pass' | 'fail' | 'borderline' | 'skip';
 interface EvaluatorResult {
     readonly name: string;
     readonly type: EvaluatorKind;
@@ -1427,7 +1553,7 @@ type ResolvedTarget = {
     readonly providerBatching?: boolean;
     readonly config: CodexResolvedConfig;
 } | {
-    readonly kind: 'copilot';
+    readonly kind: 'copilot-sdk';
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
@@ -1941,7 +2067,7 @@ declare function buildRubricOutputSchema(): string;
 declare function substituteVariables(template: string, variables: Record<string, string>): string;
 declare function calculateRubricScore(result: z.infer<typeof rubricEvaluationSchema>, rubrics: readonly RubricItem[]): {
     score: number;
-    verdict: 'pass' | 'fail' | 'borderline';
+    verdict: EvaluationVerdict;
     hits: string[];
     misses: string[];
 };
@@ -2072,8 +2198,22 @@ type AssertionResult = {
 };
 /** Checks if `output` contains the given `value` substring. */
 declare function runContainsAssertion(output: string, value: string): AssertionResult;
-/** Checks if `output` matches the given regex `pattern`. */
-declare function runRegexAssertion(output: string, pattern: string): AssertionResult;
+/** Checks if `output` contains ANY of the given `values`. */
+declare function runContainsAnyAssertion(output: string, values: readonly string[]): AssertionResult;
+/** Checks if `output` contains ALL of the given `values`. */
+declare function runContainsAllAssertion(output: string, values: readonly string[]): AssertionResult;
+/** Case-insensitive check if `output` contains `value`. */
+declare function runIcontainsAssertion(output: string, value: string): AssertionResult;
+/** Case-insensitive check if `output` contains ANY of the given `values`. */
+declare function runIcontainsAnyAssertion(output: string, values: readonly string[]): AssertionResult;
+/** Case-insensitive check if `output` contains ALL of the given `values`. */
+declare function runIcontainsAllAssertion(output: string, values: readonly string[]): AssertionResult;
+/** Checks if `output` starts with `value` (both trimmed). */
+declare function runStartsWithAssertion(output: string, value: string): AssertionResult;
+/** Checks if `output` ends with `value` (both trimmed). */
+declare function runEndsWithAssertion(output: string, value: string): AssertionResult;
+/** Checks if `output` matches the given regex `pattern` with optional `flags`. */
+declare function runRegexAssertion(output: string, pattern: string, flags?: string): AssertionResult;
 /** Checks if `output` is valid JSON. */
 declare function runIsJsonAssertion(output: string): AssertionResult;
 /** Checks if `output` exactly equals `value` (both trimmed). */
@@ -2150,6 +2290,28 @@ declare class DeterministicAssertionEvaluator implements Evaluator {
     evaluate(context: EvaluationContext): EvaluationScore;
 }
 
+declare class RepoManager {
+    private readonly cacheDir;
+    constructor(cacheDir?: string);
+    /**
+     * Ensure a bare mirror cache exists for the given source.
+     * Creates on first access, fetches updates on subsequent calls.
+     * Returns the absolute path to the cache directory.
+     */
+    ensureCache(source: RepoSource): Promise<string>;
+    /**
+     * Clone a repo from cache into the workspace at the configured path.
+     * Handles checkout, ref resolution, ancestor walking, shallow clone, sparse checkout.
+     */
+    materialize(repo: RepoConfig, workspacePath: string): Promise<void>;
+    /** Materialize all repos into the workspace. */
+    materializeAll(repos: readonly RepoConfig[], workspacePath: string): Promise<void>;
+    /** Reset repos in workspace to their checkout state. */
+    reset(repos: readonly RepoConfig[], workspacePath: string, strategy: 'hard' | 'recreate'): Promise<void>;
+    /** Remove the entire cache directory. */
+    cleanCache(): Promise<void>;
+}
+
 type MaybePromise<T> = T | Promise<T>;
 interface EvaluationCache {
     get(key: string): MaybePromise<ProviderResponse | undefined>;
@@ -2189,6 +2351,8 @@ interface RunEvalCaseOptions {
     readonly streamCallbacks?: ProviderStreamCallbacks;
     /** Evaluator type registry (with custom assertions discovered) */
     readonly typeRegistry?: EvaluatorRegistry;
+    /** RepoManager instance for repo lifecycle (shared workspace mode) */
+    readonly repoManager?: RepoManager;
 }
 interface ProgressEvent {
     readonly workerId: number;
@@ -2918,4 +3082,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { AgentJudgeEvaluator, type AgentJudgeEvaluatorConfig, type AgentJudgeEvaluatorOptions, type AgentKernel, type AgentVConfig$1 as AgentVConfig, type AgentVConfig as AgentVTsConfig, type AnthropicResolvedConfig, type ArgsMatchMode, type AssertionResult, type AssistantTestMessage, type AzureResolvedConfig, type CacheConfig, type ChildEvaluatorResult, type ClaudeResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type ConfidenceIntervalAggregation, type ContainsEvaluatorConfig, type CopilotCliResolvedConfig, type CopilotSdkResolvedConfig, CostEvaluator, type CostEvaluatorConfig, type CostEvaluatorOptions, DEFAULT_EVALUATOR_TEMPLATE, DEFAULT_EVAL_PATTERNS, DEFAULT_EXPLORATION_TOOLS, DeterministicAssertionEvaluator, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EqualsEvaluatorConfig, type EvalAssertionInput, type EvalCase, type EvalConfig, type EvalMetadata, type EvalRunResult, type EvalSuiteResult, type EvalSummary, type EvalTest, type EvalTestInput, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorDispatchContext, type EvaluatorFactory, type EvaluatorFactoryFn, type EvaluatorKind, EvaluatorRegistry, type EvaluatorResult, type ExecutionMetrics, ExecutionMetricsEvaluator, type ExecutionMetricsEvaluatorConfig, type ExecutionMetricsEvaluatorOptions, FieldAccuracyEvaluator, type FieldAccuracyEvaluatorConfig, type FieldAccuracyEvaluatorOptions, type FieldAggregationType, type FieldConfig, type FieldMatchType, type GeminiResolvedConfig, type GenerateRubricsOptions, type IsJsonEvaluatorConfig, type JsonObject, type JsonPrimitive, type JsonValue, LatencyEvaluator, type LatencyEvaluatorConfig, type LatencyEvaluatorOptions, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type LlmJudgePromptAssembly, type MeanAggregation, type Message, type MockResolvedConfig, OTEL_BACKEND_PRESETS, type OtelBackendPreset, type OtelExportOptions, OtelStreamingObserver, OtelTraceExporter, OtlpJsonFileExporter, type OutputMessage, type PassAtKAggregation, type PiAgentSdkResolvedConfig, type PiCodingAgentResolvedConfig, type ProgressEvent, type PromptInputs, type PromptScriptConfig, type Provider, type ProviderFactoryFn, type ProviderKind, ProviderRegistry, type ProviderRequest, type ProviderResponse, type ProviderStreamCallbacks, type ProviderTokenUsage, type RegexEvaluatorConfig, type ResolvedTarget, type ResolvedWorkspaceTemplate, ResponseCache, type RubricItem, type RubricsEvaluatorConfig, type RunEvalCaseOptions, type RunEvaluationOptions, type ScoreRange, type ScriptExecutionContext, SimpleTraceFileExporter, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetAccessConfig, type TargetDefinition, TemplateNotDirectoryError, TemplateNotFoundError, type TestMessage, type TestMessageContent, type TestMessageRole, type TokenUsage, TokenUsageEvaluator, type TokenUsageEvaluatorConfig, type TokenUsageEvaluatorOptions, type ToolCall, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceComputeResult, type TraceSummary, type TrialAggregation, type TrialResult, type TrialStrategy, type TrialsConfig, type UserTestMessage, type VSCodeResolvedConfig, type WorkspaceConfig, WorkspaceCreationError, type WorkspaceScriptConfig, assembleLlmJudgePrompt, avgToolDurationMs, buildDirectoryChain, buildOutputSchema, buildPromptInputs, buildRubricOutputSchema, buildScoreRangeOutputSchema, buildSearchRoots, calculateRubricScore, captureFileChanges, clampScore, cleanupEvalWorkspaces, cleanupWorkspace, computeTraceSummary, consumeClaudeLogEntries, consumeCodexLogEntries, consumeCopilotCliLogEntries, consumeCopilotSdkLogEntries, consumePiLogEntries, createAgentKernel, createBuiltinProviderRegistry, createBuiltinRegistry, createProvider, createTempWorkspace, deepEqual, defineConfig, detectFormat, discoverAssertions, discoverProviders, ensureVSCodeSubagents, evaluate, executeScript, executeWorkspaceScript, explorationRatio, extractCacheConfig, extractJsonBlob, extractTargetFromSuite, extractTargetsFromSuite, extractTargetsFromTestCase, extractTrialsConfig, fileExists, findGitRoot, freeformEvaluationSchema, generateRubrics, getHitCount, getWorkspacePath, initializeBaseline, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isNonEmptyString, isTestMessage, isTestMessageRole, listTargetNames, loadConfig, loadEvalCaseById, loadEvalCases, loadEvalSuite, loadTestById, loadTestSuite, loadTests, loadTsConfig, mergeExecutionMetrics, negateScore, normalizeLineEndings, parseJsonFromText, parseJsonSafe, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, resolveWorkspaceTemplate, rubricEvaluationSchema, runContainsAssertion, runEqualsAssertion, runEvalCase, runEvaluation, runIsJsonAssertion, runRegexAssertion, scoreToVerdict, shouldEnableCache, shouldSkipCacheForTemperature, subscribeToClaudeLogEntries, subscribeToCodexLogEntries, subscribeToCopilotCliLogEntries, subscribeToCopilotSdkLogEntries, subscribeToPiLogEntries, substituteVariables, toCamelCaseDeep, toSnakeCaseDeep, tokensPerTool, trimBaselineResult };
+export { AgentJudgeEvaluator, type AgentJudgeEvaluatorConfig, type AgentJudgeEvaluatorOptions, type AgentKernel, type AgentVConfig$1 as AgentVConfig, type AgentVConfig as AgentVTsConfig, type AnthropicResolvedConfig, type ArgsMatchMode, type AssertionResult, type AssistantTestMessage, type AzureResolvedConfig, type CacheConfig, type ChildEvaluatorResult, type ClaudeResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type ConfidenceIntervalAggregation, type ContainsAllEvaluatorConfig, type ContainsAnyEvaluatorConfig, type ContainsEvaluatorConfig, type CopilotCliResolvedConfig, type CopilotSdkResolvedConfig, CostEvaluator, type CostEvaluatorConfig, type CostEvaluatorOptions, DEFAULT_EVALUATOR_TEMPLATE, DEFAULT_EVAL_PATTERNS, DEFAULT_EXPLORATION_TOOLS, DeterministicAssertionEvaluator, type EndsWithEvaluatorConfig, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EqualsEvaluatorConfig, type EvalAssertionInput, type EvalCase, type EvalConfig, type EvalMetadata, type EvalRunResult, type EvalSuiteResult, type EvalSummary, type EvalTest, type EvalTestInput, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorDispatchContext, type EvaluatorFactory, type EvaluatorFactoryFn, type EvaluatorKind, EvaluatorRegistry, type EvaluatorResult, type ExecutionMetrics, ExecutionMetricsEvaluator, type ExecutionMetricsEvaluatorConfig, type ExecutionMetricsEvaluatorOptions, FieldAccuracyEvaluator, type FieldAccuracyEvaluatorConfig, type FieldAccuracyEvaluatorOptions, type FieldAggregationType, type FieldConfig, type FieldMatchType, type GeminiResolvedConfig, type GenerateRubricsOptions, type IcontainsAllEvaluatorConfig, type IcontainsAnyEvaluatorConfig, type IcontainsEvaluatorConfig, type IsJsonEvaluatorConfig, type JsonObject, type JsonPrimitive, type JsonValue, LatencyEvaluator, type LatencyEvaluatorConfig, type LatencyEvaluatorOptions, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type LlmJudgePromptAssembly, type MeanAggregation, type Message, type MockResolvedConfig, OTEL_BACKEND_PRESETS, type OtelBackendPreset, type OtelExportOptions, OtelStreamingObserver, OtelTraceExporter, OtlpJsonFileExporter, type OutputMessage, type PassAtKAggregation, type PiAgentSdkResolvedConfig, type PiCodingAgentResolvedConfig, type ProgressEvent, type PromptInputs, type PromptScriptConfig, type Provider, type ProviderFactoryFn, type ProviderKind, ProviderRegistry, type ProviderRequest, type ProviderResponse, type ProviderStreamCallbacks, type ProviderTokenUsage, type RegexEvaluatorConfig, type RepoCheckout, type RepoClone, type RepoConfig, RepoManager, type RepoSource, type ResetConfig, type ResolvedTarget, type ResolvedWorkspaceTemplate, ResponseCache, type RubricItem, type RubricsEvaluatorConfig, type RunEvalCaseOptions, type RunEvaluationOptions, type ScoreRange, type ScriptExecutionContext, SimpleTraceFileExporter, type StartsWithEvaluatorConfig, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetAccessConfig, type TargetDefinition, TemplateNotDirectoryError, TemplateNotFoundError, type TestMessage, type TestMessageContent, type TestMessageRole, type TokenUsage, TokenUsageEvaluator, type TokenUsageEvaluatorConfig, type TokenUsageEvaluatorOptions, type ToolCall, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceComputeResult, type TraceSummary, type TrialAggregation, type TrialResult, type TrialStrategy, type TrialsConfig, type UserTestMessage, type VSCodeResolvedConfig, type WorkspaceConfig, WorkspaceCreationError, type WorkspaceScriptConfig, assembleLlmJudgePrompt, avgToolDurationMs, buildDirectoryChain, buildOutputSchema, buildPromptInputs, buildRubricOutputSchema, buildScoreRangeOutputSchema, buildSearchRoots, calculateRubricScore, captureFileChanges, clampScore, cleanupEvalWorkspaces, cleanupWorkspace, computeTraceSummary, consumeClaudeLogEntries, consumeCodexLogEntries, consumeCopilotCliLogEntries, consumeCopilotSdkLogEntries, consumePiLogEntries, createAgentKernel, createBuiltinProviderRegistry, createBuiltinRegistry, createProvider, createTempWorkspace, deepEqual, defineConfig, detectFormat, discoverAssertions, discoverProviders, ensureVSCodeSubagents, evaluate, executeScript, executeWorkspaceScript, explorationRatio, extractCacheConfig, extractJsonBlob, extractTargetFromSuite, extractTargetsFromSuite, extractTargetsFromTestCase, extractTrialsConfig, fileExists, findGitRoot, freeformEvaluationSchema, generateRubrics, getHitCount, getWorkspacePath, initializeBaseline, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isNonEmptyString, isTestMessage, isTestMessageRole, listTargetNames, loadConfig, loadEvalCaseById, loadEvalCases, loadEvalSuite, loadTestById, loadTestSuite, loadTests, loadTsConfig, mergeExecutionMetrics, negateScore, normalizeLineEndings, parseJsonFromText, parseJsonSafe, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, resolveWorkspaceTemplate, rubricEvaluationSchema, runContainsAllAssertion, runContainsAnyAssertion, runContainsAssertion, runEndsWithAssertion, runEqualsAssertion, runEvalCase, runEvaluation, runIcontainsAllAssertion, runIcontainsAnyAssertion, runIcontainsAssertion, runIsJsonAssertion, runRegexAssertion, runStartsWithAssertion, scoreToVerdict, shouldEnableCache, shouldSkipCacheForTemperature, subscribeToClaudeLogEntries, subscribeToCodexLogEntries, subscribeToCopilotCliLogEntries, subscribeToCopilotSdkLogEntries, subscribeToPiLogEntries, substituteVariables, toCamelCaseDeep, toSnakeCaseDeep, tokensPerTool, trimBaselineResult };