@agentv/core 2.10.0 → 2.11.1

package/dist/index.js

@@ -17,7 +17,7 @@ import {
   readTextFile,
   resolveFileReference,
   resolveTargetDefinition
-} from "./chunk-7Q4PH265.js";
+} from "./chunk-REN5PS7B.js";
 import {
   OtlpJsonFileExporter
 } from "./chunk-HFSYZHGF.js";
@@ -414,9 +414,14 @@ async function loadConfig(evalFilePath, repoRoot) {
         logWarning(`Invalid eval_patterns in ${configPath}, all entries must be strings`);
         continue;
       }
+      const executionDefaults = parseExecutionDefaults(
+        parsed.execution,
+        configPath
+      );
       return {
         guideline_patterns: guidelinePatterns,
-        eval_patterns: evalPatterns
+        eval_patterns: evalPatterns,
+        execution: executionDefaults
       };
     } catch (error) {
       logWarning(
@@ -557,6 +562,36 @@ function extractTotalBudgetUsd(suite) {
   );
   return void 0;
 }
+function parseExecutionDefaults(raw, configPath) {
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
+    return void 0;
+  }
+  const obj = raw;
+  const result = {};
+  if (typeof obj.verbose === "boolean") {
+    result.verbose = obj.verbose;
+  } else if (obj.verbose !== void 0) {
+    logWarning(`Invalid execution.verbose in ${configPath}, expected boolean`);
+  }
+  const traceFile = obj.trace_file;
+  if (typeof traceFile === "string" && traceFile.trim().length > 0) {
+    result.trace_file = traceFile.trim();
+  } else if (traceFile !== void 0) {
+    logWarning(`Invalid execution.trace_file in ${configPath}, expected non-empty string`);
+  }
+  if (typeof obj.keep_workspaces === "boolean") {
+    result.keep_workspaces = obj.keep_workspaces;
+  } else if (obj.keep_workspaces !== void 0) {
+    logWarning(`Invalid execution.keep_workspaces in ${configPath}, expected boolean`);
+  }
+  const otelFile = obj.otel_file;
+  if (typeof otelFile === "string" && otelFile.trim().length > 0) {
+    result.otel_file = otelFile.trim();
+  } else if (otelFile !== void 0) {
+    logWarning(`Invalid execution.otel_file in ${configPath}, expected non-empty string`);
+  }
+  return Object.keys(result).length > 0 ? result : void 0;
+}
 function logWarning(message) {
   console.warn(`${ANSI_YELLOW2}Warning: ${message}${ANSI_RESET2}`);
 }
@@ -1285,18 +1320,96 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
+    if (typeValue === "contains_any" || typeValue === "contains_all") {
+      const value = asStringArrayStrict(rawEvaluator.value);
+      if (!value || value.length === 0) {
+        logWarning2(
+          `Skipping ${typeValue} evaluator '${name}' in '${evalId}': value must be a non-empty string array`
+        );
+        continue;
+      }
+      const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
+      const required2 = parseRequired(rawEvaluator.required);
+      evaluators.push({
+        name,
+        type: typeValue,
+        value,
+        ...weight2 !== void 0 ? { weight: weight2 } : {},
+        ...required2 !== void 0 ? { required: required2 } : {},
+        ...negate !== void 0 ? { negate } : {}
+      });
+      continue;
+    }
+    if (typeValue === "icontains") {
+      const value = asString(rawEvaluator.value);
+      if (!value) {
+        logWarning2(`Skipping icontains evaluator '${name}' in '${evalId}': missing value`);
+        continue;
+      }
+      const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
+      const required2 = parseRequired(rawEvaluator.required);
+      evaluators.push({
+        name,
+        type: "icontains",
+        value,
+        ...weight2 !== void 0 ? { weight: weight2 } : {},
+        ...required2 !== void 0 ? { required: required2 } : {},
+        ...negate !== void 0 ? { negate } : {}
+      });
+      continue;
+    }
+    if (typeValue === "icontains_any" || typeValue === "icontains_all") {
+      const value = asStringArrayStrict(rawEvaluator.value);
+      if (!value || value.length === 0) {
+        logWarning2(
+          `Skipping ${typeValue} evaluator '${name}' in '${evalId}': value must be a non-empty string array`
+        );
+        continue;
+      }
+      const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
+      const required2 = parseRequired(rawEvaluator.required);
+      evaluators.push({
+        name,
+        type: typeValue,
+        value,
+        ...weight2 !== void 0 ? { weight: weight2 } : {},
+        ...required2 !== void 0 ? { required: required2 } : {},
+        ...negate !== void 0 ? { negate } : {}
+      });
+      continue;
+    }
+    if (typeValue === "starts_with" || typeValue === "ends_with") {
+      const value = asString(rawEvaluator.value);
+      if (!value) {
+        logWarning2(`Skipping ${typeValue} evaluator '${name}' in '${evalId}': missing value`);
+        continue;
+      }
+      const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
+      const required2 = parseRequired(rawEvaluator.required);
+      evaluators.push({
+        name,
+        type: typeValue,
+        value,
+        ...weight2 !== void 0 ? { weight: weight2 } : {},
+        ...required2 !== void 0 ? { required: required2 } : {},
+        ...negate !== void 0 ? { negate } : {}
+      });
+      continue;
+    }
     if (typeValue === "regex") {
       const value = asString(rawEvaluator.value);
       if (!value) {
         logWarning2(`Skipping regex evaluator '${name}' in '${evalId}': missing value`);
         continue;
       }
+      const flags = asString(rawEvaluator.flags);
       const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
       const required2 = parseRequired(rawEvaluator.required);
       evaluators.push({
         name,
         type: "regex",
         value,
+        ...flags !== void 0 ? { flags } : {},
         ...weight2 !== void 0 ? { weight: weight2 } : {},
         ...required2 !== void 0 ? { required: required2 } : {},
         ...negate !== void 0 ? { negate } : {}
@@ -1469,15 +1582,43 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
   }
   return evaluators.length > 0 ? evaluators : void 0;
 }
-var ASSERTION_TYPES = /* @__PURE__ */ new Set(["contains", "regex", "is_json", "equals", "rubrics"]);
+var ASSERTION_TYPES = /* @__PURE__ */ new Set([
+  "contains",
+  "contains_any",
+  "contains_all",
+  "icontains",
+  "icontains_any",
+  "icontains_all",
+  "starts_with",
+  "ends_with",
+  "regex",
+  "is_json",
+  "equals",
+  "rubrics"
+]);
 function generateAssertionName(typeValue, rawEvaluator) {
   if (!ASSERTION_TYPES.has(typeValue)) {
     return void 0;
   }
   const value = asString(rawEvaluator.value);
+  const arrayValue = Array.isArray(rawEvaluator.value) ? rawEvaluator.value : void 0;
   switch (typeValue) {
     case "contains":
       return value ? `contains-${value}` : "contains";
+    case "contains_any":
+      return arrayValue ? `contains_any-${arrayValue.length}` : "contains_any";
+    case "contains_all":
+      return arrayValue ? `contains_all-${arrayValue.length}` : "contains_all";
+    case "icontains":
+      return value ? `icontains-${value}` : "icontains";
+    case "icontains_any":
+      return arrayValue ? `icontains_any-${arrayValue.length}` : "icontains_any";
+    case "icontains_all":
+      return arrayValue ? `icontains_all-${arrayValue.length}` : "icontains_all";
+    case "starts_with":
+      return value ? `starts_with-${value}` : "starts_with";
+    case "ends_with":
+      return value ? `ends_with-${value}` : "ends_with";
     case "regex":
       return value ? `regex-${value.length > 30 ? value.slice(0, 30) : value}` : "regex";
     case "is_json":
@@ -1503,6 +1644,13 @@ function coerceEvaluator(candidate, contextId) {
 function asString(value) {
   return typeof value === "string" ? value : void 0;
 }
+function asStringArrayStrict(value) {
+  if (!Array.isArray(value)) {
+    return void 0;
+  }
+  const result = value.filter((v) => typeof v === "string");
+  return result.length > 0 ? result : void 0;
+}
 function asStringArray(value, description) {
   if (value === void 0) {
     return void 0;
@@ -2820,6 +2968,69 @@ function parseWorkspaceScriptConfig(raw, evalFileDir) {
   }
   return cwd ? { ...config, cwd } : config;
 }
+function parseRepoSource(raw) {
+  if (!isJsonObject(raw)) return void 0;
+  const obj = raw;
+  if (obj.type === "git" && typeof obj.url === "string") {
+    return { type: "git", url: obj.url };
+  }
+  if (obj.type === "local" && typeof obj.path === "string") {
+    return { type: "local", path: obj.path };
+  }
+  return void 0;
+}
+function parseRepoCheckout(raw) {
+  if (!isJsonObject(raw)) return void 0;
+  const obj = raw;
+  const ref = typeof obj.ref === "string" ? obj.ref : void 0;
+  const resolve = obj.resolve === "remote" || obj.resolve === "local" ? obj.resolve : void 0;
+  const ancestor = typeof obj.ancestor === "number" ? obj.ancestor : void 0;
+  if (!ref && !resolve && ancestor === void 0) return void 0;
+  return {
+    ...ref !== void 0 && { ref },
+    ...resolve !== void 0 && { resolve },
+    ...ancestor !== void 0 && { ancestor }
+  };
+}
+function parseRepoClone(raw) {
+  if (!isJsonObject(raw)) return void 0;
+  const obj = raw;
+  const depth = typeof obj.depth === "number" ? obj.depth : void 0;
+  const filter = typeof obj.filter === "string" ? obj.filter : void 0;
+  const sparse = Array.isArray(obj.sparse) ? obj.sparse.filter((s) => typeof s === "string") : void 0;
+  if (depth === void 0 && !filter && !sparse) return void 0;
+  return {
+    ...depth !== void 0 && { depth },
+    ...filter !== void 0 && { filter },
+    ...sparse !== void 0 && { sparse }
+  };
+}
+function parseRepoConfig(raw) {
+  if (!isJsonObject(raw)) return void 0;
+  const obj = raw;
+  const repoPath = typeof obj.path === "string" ? obj.path : void 0;
+  const source = parseRepoSource(obj.source);
+  if (!repoPath || !source) return void 0;
+  const checkout = parseRepoCheckout(obj.checkout);
+  const clone = parseRepoClone(obj.clone);
+  return {
+    path: repoPath,
+    source,
+    ...checkout !== void 0 && { checkout },
+    ...clone !== void 0 && { clone }
+  };
+}
+function parseResetConfig(raw) {
+  if (!isJsonObject(raw)) return void 0;
+  const obj = raw;
+  const strategy = obj.strategy === "none" || obj.strategy === "hard" || obj.strategy === "recreate" ? obj.strategy : void 0;
+  const afterEach = typeof obj.after_each === "boolean" ? obj.after_each : void 0;
+  if (!strategy && afterEach === void 0) return void 0;
+  return {
+    ...strategy !== void 0 && { strategy },
+    ...afterEach !== void 0 && { after_each: afterEach }
+  };
+}
 function parseWorkspaceConfig(raw, evalFileDir) {
   if (!isJsonObject(raw)) return void 0;
   const obj = raw;
@@ -2827,13 +3038,20 @@ function parseWorkspaceConfig(raw, evalFileDir) {
   if (template && !path8.isAbsolute(template)) {
     template = path8.resolve(evalFileDir, template);
   }
+  const isolation = obj.isolation === "shared" || obj.isolation === "per_test" ? obj.isolation : void 0;
+  const repos = Array.isArray(obj.repos) ? obj.repos.map(parseRepoConfig).filter(Boolean) : void 0;
+  const reset = parseResetConfig(obj.reset);
   const beforeAll = parseWorkspaceScriptConfig(obj.before_all, evalFileDir);
   const afterAll = parseWorkspaceScriptConfig(obj.after_all, evalFileDir);
   const beforeEach = parseWorkspaceScriptConfig(obj.before_each, evalFileDir);
   const afterEach = parseWorkspaceScriptConfig(obj.after_each, evalFileDir);
-  if (!template && !beforeAll && !afterAll && !beforeEach && !afterEach) return void 0;
+  if (!template && !isolation && !repos && !reset && !beforeAll && !afterAll && !beforeEach && !afterEach)
+    return void 0;
   return {
     ...template !== void 0 && { template },
+    ...isolation !== void 0 && { isolation },
+    ...repos !== void 0 && { repos },
+    ...reset !== void 0 && { reset },
     ...beforeAll !== void 0 && { before_all: beforeAll },
     ...afterAll !== void 0 && { after_all: afterAll },
     ...beforeEach !== void 0 && { before_each: beforeEach },
@@ -2846,6 +3064,9 @@ function mergeWorkspaceConfigs(suiteLevel, caseLevel) {
   if (!caseLevel) return suiteLevel;
   return {
     template: caseLevel.template ?? suiteLevel.template,
+    isolation: caseLevel.isolation ?? suiteLevel.isolation,
+    repos: caseLevel.repos ?? suiteLevel.repos,
+    reset: caseLevel.reset ?? suiteLevel.reset,
     before_all: caseLevel.before_all ?? suiteLevel.before_all,
     after_all: caseLevel.after_all ?? suiteLevel.after_all,
     before_each: caseLevel.before_each ?? suiteLevel.before_each,
@@ -3385,11 +3606,6 @@ Original error: ${error instanceof Error ? error.message : String(error)}`
   }
   return claudeSdkModule;
 }
-var DEFAULT_SYSTEM_PROMPT2 = `**IMPORTANT**: Follow these instructions for your response:
-- Do NOT create any additional output files in the workspace.
-- All intended file outputs/changes MUST be written in your response.
-- For each intended file, include the relative path and unified git diff following the convention \`diff --git ...\`.
-This is required for evaluation scoring.`;
 var ClaudeProvider = class {
   id;
   kind = "claude";
@@ -3411,7 +3627,7 @@ var ClaudeProvider = class {
     const logger = await this.createStreamLogger(request).catch(() => void 0);
     const inputFiles = normalizeInputFiles(request.inputFiles);
     const prompt = buildPromptDocument(request, inputFiles);
-    const systemPrompt = this.config.systemPrompt ?? (request.captureFileChanges ? void 0 : DEFAULT_SYSTEM_PROMPT2);
+    const systemPrompt = this.config.systemPrompt;
     const queryOptions = {
       permissionMode: "bypassPermissions",
       allowDangerouslySkipPermissions: true,
@@ -4392,11 +4608,6 @@ Original error: ${error instanceof Error ? error.message : String(error)}`
   }
   return codexSdkModule;
 }
-var DEFAULT_SYSTEM_PROMPT3 = `**IMPORTANT**: Follow these instructions for your response:
-- Do NOT create any additional output files in the workspace.
-- All intended file outputs/changes MUST be written in your response.
-- For each intended file, include the relative path and unified git diff following the convention \`diff --git ...\`.
-This is required for evaluation scoring.`;
 var CodexProvider = class {
   id;
   kind = "codex";
@@ -4431,7 +4642,7 @@ var CodexProvider = class {
     const thread = codex.startThread(threadOptions);
     const inputFiles = normalizeInputFiles(request.inputFiles);
     const basePrompt = buildPromptDocument(request, inputFiles);
-    const systemPrompt = this.config.systemPrompt ?? (request.captureFileChanges ? void 0 : DEFAULT_SYSTEM_PROMPT3);
+    const systemPrompt = this.config.systemPrompt;
     const prompt = systemPrompt ? `${systemPrompt}
 
 ${basePrompt}` : basePrompt;
@@ -4797,7 +5008,7 @@ import { arch, platform } from "node:os";
 import path13 from "node:path";
 import { fileURLToPath as fileURLToPath2 } from "node:url";
 function resolvePlatformCliPath() {
-  const os4 = platform();
+  const os5 = platform();
   const cpu = arch();
   const platformMap = {
     linux: "linux",
@@ -4808,13 +5019,13 @@ function resolvePlatformCliPath() {
     x64: "x64",
     arm64: "arm64"
   };
-  const osPart = platformMap[os4];
+  const osPart = platformMap[os5];
   const archPart = archMap[cpu];
   if (!osPart || !archPart) {
     return void 0;
   }
   const packageName = `@github/copilot-${osPart}-${archPart}`;
-  const binaryName = os4 === "win32" ? "copilot.exe" : "copilot";
+  const binaryName = os5 === "win32" ? "copilot.exe" : "copilot";
   try {
     const resolved = import.meta.resolve(`${packageName}/package.json`);
     const packageJsonPath = resolved.startsWith("file:") ? fileURLToPath2(resolved) : resolved;
@@ -4956,11 +5167,6 @@ function isLogStreamingDisabled(envKey) {
 }
 
 // src/evaluation/providers/copilot-cli.ts
-var DEFAULT_SYSTEM_PROMPT4 = `**IMPORTANT**: Follow these instructions for your response:
-- Do NOT create any additional output files in the workspace.
-- All intended file outputs/changes MUST be written in your response.
-- For each intended file, include the relative path and unified git diff following the convention \`diff --git ...\`.
-This is required for evaluation scoring.`;
 var CopilotCliProvider = class {
   id;
   kind = "copilot-cli";
@@ -5122,6 +5328,16 @@ var CopilotCliProvider = class {
       }
       const endTime = (/* @__PURE__ */ new Date()).toISOString();
       const durationMs = Date.now() - startMs;
+      const rejectedCalls = completedToolCalls.filter((tc) => {
+        const out = tc.output;
+        return out && (out.code === "rejected" || out.code === "denied");
+      });
+      if (rejectedCalls.length > 0) {
+        const tools = rejectedCalls.map((tc) => tc.tool).join(", ");
+        throw new Error(
+          `Copilot rejected ${rejectedCalls.length} tool call(s): ${tools}. Add args: ["--yolo"] to your target config or re-run with --yolo to bypass permission checks.`
+        );
+      }
       const outputMessages = [];
       if (completedToolCalls.length > 0) {
         outputMessages.push({
@@ -5154,7 +5370,7 @@ var CopilotCliProvider = class {
     }
   }
   buildCliArgs() {
-    const args = ["--acp", "--stdio", "--allow-all-tools"];
+    const args = ["--acp", "--stdio", "--allow-all-tools", "--yolo"];
     if (this.config.model) {
       args.push("--model", this.config.model);
     }
@@ -5163,8 +5379,8 @@ var CopilotCliProvider = class {
     }
     return args;
   }
-  resolveSystemPrompt(request) {
-    return this.config.systemPrompt ?? (request.captureFileChanges ? void 0 : DEFAULT_SYSTEM_PROMPT4);
+  resolveSystemPrompt(_request) {
+    return this.config.systemPrompt;
   }
   async raceWithTimeout(sendPromise, agentProcess) {
     const timeoutMs = this.config.timeoutMs;
@@ -5352,21 +5568,16 @@ Original error: ${error instanceof Error ? error.message : String(error)}`
   }
   return copilotSdkModule;
 }
-var DEFAULT_SYSTEM_PROMPT5 = `**IMPORTANT**: Follow these instructions for your response:
-- Do NOT create any additional output files in the workspace.
-- All intended file outputs/changes MUST be written in your response.
-- For each intended file, include the relative path and unified git diff following the convention \`diff --git ...\`.
-This is required for evaluation scoring.`;
 var CopilotSdkProvider = class {
   id;
-  kind = "copilot";
+  kind = "copilot-sdk";
   targetName;
   supportsBatch = false;
   config;
   // biome-ignore lint/suspicious/noExplicitAny: SDK client type is dynamically loaded
   client = null;
   constructor(targetName, config) {
-    this.id = `copilot:${targetName}`;
+    this.id = `copilot-sdk:${targetName}`;
     this.targetName = targetName;
     this.config = config;
   }
@@ -5389,7 +5600,7 @@ var CopilotSdkProvider = class {
     if (cwd) {
       sessionOptions.workingDirectory = cwd;
     }
-    const systemPrompt = this.config.systemPrompt ?? (request.captureFileChanges ? void 0 : DEFAULT_SYSTEM_PROMPT5);
+    const systemPrompt = this.config.systemPrompt;
     if (systemPrompt) {
       sessionOptions.systemMessage = {
         mode: "append",
@@ -5905,11 +6116,6 @@ function subscribeToPiLogEntries(listener) {
 // src/evaluation/providers/pi-coding-agent.ts
 var WORKSPACE_PREFIX = "agentv-pi-";
 var PROMPT_FILENAME = "prompt.md";
-var DEFAULT_SYSTEM_PROMPT6 = `**IMPORTANT**: Follow these instructions for your response:
-- Do NOT create any additional output files in the workspace.
-- All intended file outputs/changes MUST be written in your response.
-- For each intended file, include the relative path and unified git diff following the convention \`diff --git ...\`.
-This is required for evaluation scoring.`;
 var PiCodingAgentProvider = class {
   id;
   kind = "pi-coding-agent";
@@ -5986,7 +6192,7 @@ var PiCodingAgentProvider = class {
     }
     return path16.resolve(this.config.cwd);
   }
-  buildPiArgs(prompt, inputFiles, captureFileChanges2) {
+  buildPiArgs(prompt, inputFiles, _captureFileChanges) {
     const args = [];
     if (this.config.provider) {
       args.push("--provider", this.config.provider);
@@ -6014,7 +6220,7 @@ var PiCodingAgentProvider = class {
         args.push(`@${file}`);
       }
     }
-    const systemPrompt = this.config.systemPrompt ?? (captureFileChanges2 ? void 0 : DEFAULT_SYSTEM_PROMPT6);
+    const systemPrompt = this.config.systemPrompt;
     const fullPrompt = systemPrompt ? `${systemPrompt}
 
 ${prompt}` : prompt;
@@ -7708,7 +7914,6 @@ var AGENTV_REQUEST_TEMPLATE = `[[ ## task ## ]]
 
 **IMPORTANT**: Follow these exact steps:
 1. Create and write your complete response to: {{responseFileTmp}}
-    - Do NOT create any additional output files in the workspace.
     - All intended file outputs/changes MUST be written in your response file.
     - For each intended file, include the repo name, relative path and unified git diff following the convention \`diff --git ...\`.
 2. When completely finished, run these PowerShell commands to signal completion:
@@ -7727,7 +7932,6 @@ var AGENTV_BATCH_REQUEST_TEMPLATE = `[[ ## task ## ]]
 
 **IMPORTANT**: Follow these exact steps:
 1. Create and write your complete response to: {{responseFileTmp}}
-    - Do NOT create any additional output files in the workspace.
     - All intended file outputs/changes MUST be written in your response file.
     - For each intended file, include the repo name, relative path and unified git diff following the convention \`diff --git ...\`.
 2. When completely finished and the response is stable, rename it to: {{responseFileFinal}}
@@ -8153,7 +8357,7 @@ async function discoverProviders(registry, baseDir) {
 // src/evaluation/providers/index.ts
 function createBuiltinProviderRegistry() {
   const registry = new ProviderRegistry();
-  registry.register("azure", (t) => new AzureProvider(t.name, t.config)).register("anthropic", (t) => new AnthropicProvider(t.name, t.config)).register("gemini", (t) => new GeminiProvider(t.name, t.config)).register("cli", (t) => new CliProvider(t.name, t.config)).register("codex", (t) => new CodexProvider(t.name, t.config)).register("copilot", (t) => new CopilotSdkProvider(t.name, t.config)).register("copilot-cli", (t) => new CopilotCliProvider(t.name, t.config)).register("pi-coding-agent", (t) => new PiCodingAgentProvider(t.name, t.config)).register("pi-agent-sdk", (t) => new PiAgentSdkProvider(t.name, t.config)).register("claude", (t) => new ClaudeProvider(t.name, t.config)).register("mock", (t) => new MockProvider(t.name, t.config)).register("vscode", (t) => new VSCodeProvider(t.name, t.config, "vscode")).register(
+  registry.register("azure", (t) => new AzureProvider(t.name, t.config)).register("anthropic", (t) => new AnthropicProvider(t.name, t.config)).register("gemini", (t) => new GeminiProvider(t.name, t.config)).register("cli", (t) => new CliProvider(t.name, t.config)).register("codex", (t) => new CodexProvider(t.name, t.config)).register("copilot-sdk", (t) => new CopilotSdkProvider(t.name, t.config)).register("copilot-cli", (t) => new CopilotCliProvider(t.name, t.config)).register("pi-coding-agent", (t) => new PiCodingAgentProvider(t.name, t.config)).register("pi-agent-sdk", (t) => new PiAgentSdkProvider(t.name, t.config)).register("claude", (t) => new ClaudeProvider(t.name, t.config)).register("mock", (t) => new MockProvider(t.name, t.config)).register("vscode", (t) => new VSCodeProvider(t.name, t.config, "vscode")).register(
     "vscode-insiders",
     (t) => new VSCodeProvider(t.name, t.config, "vscode-insiders")
   );
@@ -8342,16 +8546,16 @@ async function execFileWithStdinNode(argv, stdinPayload, options) {
   });
 }
 async function execShellWithStdin(command, stdinPayload, options = {}) {
-  const { mkdir: mkdir13, readFile: readFile12, rm: rm5, writeFile: writeFile8 } = await import("node:fs/promises");
+  const { mkdir: mkdir14, readFile: readFile12, rm: rm6, writeFile: writeFile9 } = await import("node:fs/promises");
   const { tmpdir: tmpdir3 } = await import("node:os");
-  const path39 = await import("node:path");
+  const path40 = await import("node:path");
   const { randomUUID: randomUUID8 } = await import("node:crypto");
-  const dir = path39.join(tmpdir3(), `agentv-exec-${randomUUID8()}`);
-  await mkdir13(dir, { recursive: true });
-  const stdinPath = path39.join(dir, "stdin.txt");
-  const stdoutPath = path39.join(dir, "stdout.txt");
-  const stderrPath = path39.join(dir, "stderr.txt");
-  await writeFile8(stdinPath, stdinPayload, "utf8");
+  const dir = path40.join(tmpdir3(), `agentv-exec-${randomUUID8()}`);
+  await mkdir14(dir, { recursive: true });
+  const stdinPath = path40.join(dir, "stdin.txt");
+  const stdoutPath = path40.join(dir, "stdout.txt");
+  const stderrPath = path40.join(dir, "stderr.txt");
+  await writeFile9(stdinPath, stdinPayload, "utf8");
   const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
   const { spawn: spawn4 } = await import("node:child_process");
   try {
@@ -8384,7 +8588,7 @@ async function execShellWithStdin(command, stdinPayload, options = {}) {
     const stderr = (await readFile12(stderrPath, "utf8")).replace(/\r\n/g, "\n");
     return { stdout, stderr, exitCode };
   } finally {
-    await rm5(dir, { recursive: true, force: true });
+    await rm6(dir, { recursive: true, force: true });
   }
 }
 
@@ -8702,7 +8906,7 @@ var CodeEvaluator = class {
       outputPath,
       guidelineFiles: context.evalCase.guideline_paths,
       inputFiles: context.evalCase.file_paths.filter(
-        (path39) => !context.evalCase.guideline_paths.includes(path39)
+        (path40) => !context.evalCase.guideline_paths.includes(path40)
       ),
       input: context.evalCase.input,
       trace: context.trace ?? null,
@@ -8950,13 +9154,15 @@ ${context.fileChanges}`;
         evaluatorRawRequest,
         tokenUsage
       };
-    } catch {
+    } catch (e) {
+      const message = e instanceof Error ? e.message : String(e);
       return {
         score: 0,
-        verdict: "fail",
+        verdict: "skip",
         hits: [],
-        misses: [],
+        misses: [`Judge parse failure after 3 attempts: ${message}`],
         expectedAspectCount: 1,
+        reasoning: `Judge parse failure after 3 attempts: ${message}`,
         evaluatorRawRequest
       };
     }
@@ -9898,115 +10104,115 @@ var FieldAccuracyEvaluator = class {
    * Evaluate a single field against the expected value.
    */
   evaluateField(fieldConfig, candidateData, expectedData) {
-    const { path: path39, match, required = true, weight = 1 } = fieldConfig;
-    const candidateValue = resolvePath(candidateData, path39);
-    const expectedValue = resolvePath(expectedData, path39);
+    const { path: path40, match, required = true, weight = 1 } = fieldConfig;
+    const candidateValue = resolvePath(candidateData, path40);
+    const expectedValue = resolvePath(expectedData, path40);
     if (expectedValue === void 0) {
       return {
-        path: path39,
+        path: path40,
         score: 1,
         // No expected value means no comparison needed
         weight,
         hit: true,
-        message: `${path39}: no expected value`
+        message: `${path40}: no expected value`
       };
     }
     if (candidateValue === void 0) {
       if (required) {
         return {
-          path: path39,
+          path: path40,
           score: 0,
           weight,
           hit: false,
-          message: `${path39} (required, missing)`
+          message: `${path40} (required, missing)`
         };
       }
       return {
-        path: path39,
+        path: path40,
         score: 1,
         // Don't penalize missing optional fields
         weight: 0,
         // Zero weight means it won't affect the score
         hit: true,
-        message: `${path39}: optional field missing`
+        message: `${path40}: optional field missing`
       };
     }
     switch (match) {
       case "exact":
-        return this.compareExact(path39, candidateValue, expectedValue, weight);
+        return this.compareExact(path40, candidateValue, expectedValue, weight);
       case "numeric_tolerance":
         return this.compareNumericTolerance(
-          path39,
+          path40,
           candidateValue,
           expectedValue,
           fieldConfig,
           weight
         );
       case "date":
-        return this.compareDate(path39, candidateValue, expectedValue, fieldConfig, weight);
+        return this.compareDate(path40, candidateValue, expectedValue, fieldConfig, weight);
       default:
         return {
-          path: path39,
+          path: path40,
           score: 0,
           weight,
           hit: false,
-          message: `${path39}: unknown match type "${match}"`
+          message: `${path40}: unknown match type "${match}"`
         };
     }
   }
   /**
    * Exact equality comparison.
    */
-  compareExact(path39, candidateValue, expectedValue, weight) {
+  compareExact(path40, candidateValue, expectedValue, weight) {
     if (deepEqual(candidateValue, expectedValue)) {
       return {
-        path: path39,
+        path: path40,
         score: 1,
         weight,
         hit: true,
-        message: path39
+        message: path40
       };
     }
     if (typeof candidateValue !== typeof expectedValue) {
       return {
-        path: path39,
+        path: path40,
         score: 0,
         weight,
         hit: false,
-        message: `${path39} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
+        message: `${path40} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
       };
     }
     return {
-      path: path39,
+      path: path40,
       score: 0,
       weight,
       hit: false,
-      message: `${path39} (value mismatch)`
+      message: `${path40} (value mismatch)`
     };
   }
   /**
    * Numeric comparison with absolute or relative tolerance.
    */
-  compareNumericTolerance(path39, candidateValue, expectedValue, fieldConfig, weight) {
+  compareNumericTolerance(path40, candidateValue, expectedValue, fieldConfig, weight) {
     const { tolerance = 0, relative = false } = fieldConfig;
     const candidateNum = toNumber2(candidateValue);
     const expectedNum = toNumber2(expectedValue);
     if (candidateNum === null || expectedNum === null) {
       return {
-        path: path39,
+        path: path40,
         score: 0,
         weight,
         hit: false,
-        message: `${path39} (non-numeric value)`
+        message: `${path40} (non-numeric value)`
       };
     }
     if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
       return {
-        path: path39,
+        path: path40,
         score: 0,
         weight,
         hit: false,
-        message: `${path39} (invalid numeric value)`
+        message: `${path40} (invalid numeric value)`
       };
     }
     const diff = Math.abs(candidateNum - expectedNum);
@@ -10019,61 +10225,61 @@ var FieldAccuracyEvaluator = class {
     }
     if (withinTolerance) {
       return {
-        path: path39,
+        path: path40,
         score: 1,
         weight,
         hit: true,
-        message: `${path39} (within tolerance: diff=${diff.toFixed(2)})`
+        message: `${path40} (within tolerance: diff=${diff.toFixed(2)})`
       };
     }
     return {
-      path: path39,
+      path: path40,
       score: 0,
       weight,
       hit: false,
-      message: `${path39} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
+      message: `${path40} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
     };
   }
   /**
    * Date comparison with format normalization.
    */
-  compareDate(path39, candidateValue, expectedValue, fieldConfig, weight) {
+  compareDate(path40, candidateValue, expectedValue, fieldConfig, weight) {
     const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
     const candidateDate = parseDate(String(candidateValue), formats);
     const expectedDate = parseDate(String(expectedValue), formats);
     if (candidateDate === null) {
       return {
-        path: path39,
+        path: path40,
         score: 0,
         weight,
         hit: false,
-        message: `${path39} (unparseable candidate date)`
+        message: `${path40} (unparseable candidate date)`
       };
     }
     if (expectedDate === null) {
       return {
-        path: path39,
+        path: path40,
         score: 0,
         weight,
         hit: false,
-        message: `${path39} (unparseable expected date)`
+        message: `${path40} (unparseable expected date)`
       };
     }
     if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
       return {
-        path: path39,
+        path: path40,
         score: 1,
         weight,
         hit: true,
-        message: path39
+        message: path40
       };
     }
     return {
-      path: path39,
+      path: path40,
       score: 0,
       weight,
       hit: false,
-      message: `${path39} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
+      message: `${path40} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
     };
   }
   /**
@@ -10114,11 +10320,11 @@ var FieldAccuracyEvaluator = class {
     };
   }
 };
-function resolvePath(obj, path39) {
-  if (!path39 || !obj) {
+function resolvePath(obj, path40) {
+  if (!path40 || !obj) {
     return void 0;
   }
-  const parts = path39.split(/\.|\[|\]/).filter((p) => p.length > 0);
+  const parts = path40.split(/\.|\[|\]/).filter((p) => p.length > 0);
   let current = obj;
   for (const part of parts) {
     if (current === null || current === void 0) {
@@ -10936,8 +11142,8 @@ var TokenUsageEvaluator = class {
 };
 
 // src/evaluation/evaluators/tool-trajectory.ts
-function getNestedValue(obj, path39) {
-  const parts = path39.split(".");
+function getNestedValue(obj, path40) {
+  const parts = path40.split(".");
   let current = obj;
   for (const part of parts) {
     if (current === null || current === void 0 || typeof current !== "object") {
@@ -11401,13 +11607,78 @@ function runContainsAssertion(output, value) {
     misses: passed ? [] : [`Output does not contain "${value}"`]
   };
 }
-function runRegexAssertion(output, pattern) {
-  const regex = new RegExp(pattern);
+function runContainsAnyAssertion(output, values) {
+  const matched = values.filter((v) => output.includes(v));
+  const passed = matched.length > 0;
+  return {
+    score: passed ? 1 : 0,
+    hits: passed ? [`Output contains "${matched[0]}"`] : [],
+    misses: passed ? [] : [`Output does not contain any of: ${values.map((v) => `"${v}"`).join(", ")}`]
+  };
+}
+function runContainsAllAssertion(output, values) {
+  const missing = values.filter((v) => !output.includes(v));
+  const passed = missing.length === 0;
+  return {
+    score: passed ? 1 : 0,
+    hits: passed ? [`Output contains all ${values.length} expected strings`] : [],
+    misses: passed ? [] : [`Output missing: ${missing.map((v) => `"${v}"`).join(", ")}`]
+  };
+}
+function runIcontainsAssertion(output, value) {
+  const passed = output.toLowerCase().includes(value.toLowerCase());
+  return {
+    score: passed ? 1 : 0,
+    hits: passed ? [`Output contains "${value}" (case-insensitive)`] : [],
+    misses: passed ? [] : [`Output does not contain "${value}" (case-insensitive)`]
+  };
+}
+function runIcontainsAnyAssertion(output, values) {
+  const lower = output.toLowerCase();
+  const matched = values.filter((v) => lower.includes(v.toLowerCase()));
+  const passed = matched.length > 0;
+  return {
+    score: passed ? 1 : 0,
+    hits: passed ? [`Output contains "${matched[0]}" (case-insensitive)`] : [],
+    misses: passed ? [] : [
+      `Output does not contain any of: ${values.map((v) => `"${v}"`).join(", ")} (case-insensitive)`
+    ]
+  };
+}
+function runIcontainsAllAssertion(output, values) {
+  const lower = output.toLowerCase();
+  const missing = values.filter((v) => !lower.includes(v.toLowerCase()));
+  const passed = missing.length === 0;
+  return {
+    score: passed ? 1 : 0,
+    hits: passed ? [`Output contains all ${values.length} expected strings (case-insensitive)`] : [],
+    misses: passed ? [] : [`Output missing (case-insensitive): ${missing.map((v) => `"${v}"`).join(", ")}`]
+  };
+}
+function runStartsWithAssertion(output, value) {
+  const passed = output.trim().startsWith(value.trim());
+  return {
+    score: passed ? 1 : 0,
+    hits: passed ? [`Output starts with "${value}"`] : [],
+    misses: passed ? [] : [`Output does not start with "${value}"`]
+  };
+}
+function runEndsWithAssertion(output, value) {
+  const passed = output.trim().endsWith(value.trim());
+  return {
+    score: passed ? 1 : 0,
+    hits: passed ? [`Output ends with "${value}"`] : [],
+    misses: passed ? [] : [`Output does not end with "${value}"`]
+  };
+}
+function runRegexAssertion(output, pattern, flags) {
+  const regex = new RegExp(pattern, flags);
   const passed = regex.test(output);
+  const flagsLabel = flags ? ` (flags: ${flags})` : "";
   return {
     score: passed ? 1 : 0,
-    hits: passed ? [`Output matches pattern /${pattern}/`] : [],
-    misses: passed ? [] : [`Output does not match pattern /${pattern}/`]
+    hits: passed ? [`Output matches pattern /${pattern}/${flags ?? ""}${flagsLabel}`] : [],
+    misses: passed ? [] : [`Output does not match pattern /${pattern}/${flags ?? ""}${flagsLabel}`]
   };
 }
 function runIsJsonAssertion(output) {
@@ -11433,9 +11704,9 @@ function runEqualsAssertion(output, value) {
 }
 
 // src/evaluation/orchestrator.ts
-import { createHash, randomUUID as randomUUID7 } from "node:crypto";
-import { mkdir as mkdir11 } from "node:fs/promises";
-import path36 from "node:path";
+import { createHash as createHash2, randomUUID as randomUUID7 } from "node:crypto";
+import { mkdir as mkdir12 } from "node:fs/promises";
+import path37 from "node:path";
 import micromatch4 from "micromatch";
 
 // ../../node_modules/.bun/yocto-queue@1.2.2/node_modules/yocto-queue/index.js
@@ -11825,13 +12096,13 @@ var containsFactory = (config) => {
 var regexFactory = (config) => {
   const c = config;
   return new DeterministicAssertionEvaluator("regex", (ctx) => {
-    const result = runRegexAssertion(ctx.candidate, c.value);
+    const result = runRegexAssertion(ctx.candidate, c.value, c.flags);
     return {
       score: result.score,
       verdict: result.score === 1 ? "pass" : "fail",
       hits: result.hits,
       misses: result.misses,
-      reasoning: result.score === 1 ? `Output matches pattern /${c.value}/` : `Output does not match pattern /${c.value}/`,
+      reasoning: result.score === 1 ? `Output matches pattern /${c.value}/${c.flags ?? ""}` : `Output does not match pattern /${c.value}/${c.flags ?? ""}`,
       expectedAspectCount: 1
     };
   });
@@ -11863,9 +12134,107 @@ var equalsFactory = (config) => {
     };
   });
 };
+var containsAnyFactory = (config) => {
+  const c = config;
+  return new DeterministicAssertionEvaluator("contains_any", (ctx) => {
+    const result = runContainsAnyAssertion(ctx.candidate, c.value);
+    return {
+      score: result.score,
+      verdict: result.score === 1 ? "pass" : "fail",
+      hits: result.hits,
+      misses: result.misses,
+      reasoning: result.score === 1 ? result.hits[0] : result.misses[0],
+      expectedAspectCount: 1
+    };
+  });
+};
+var containsAllFactory = (config) => {
+  const c = config;
+  return new DeterministicAssertionEvaluator("contains_all", (ctx) => {
+    const result = runContainsAllAssertion(ctx.candidate, c.value);
+    return {
+      score: result.score,
+      verdict: result.score === 1 ? "pass" : "fail",
+      hits: result.hits,
+      misses: result.misses,
+      reasoning: result.score === 1 ? result.hits[0] : result.misses[0],
+      expectedAspectCount: 1
+    };
+  });
+};
+var icontainsFactory = (config) => {
+  const c = config;
+  return new DeterministicAssertionEvaluator("icontains", (ctx) => {
+    const result = runIcontainsAssertion(ctx.candidate, c.value);
+    return {
+      score: result.score,
+      verdict: result.score === 1 ? "pass" : "fail",
+      hits: result.hits,
+      misses: result.misses,
+      reasoning: result.score === 1 ? result.hits[0] : result.misses[0],
+      expectedAspectCount: 1
+    };
+  });
+};
+var icontainsAnyFactory = (config) => {
+  const c = config;
+  return new DeterministicAssertionEvaluator("icontains_any", (ctx) => {
+    const result = runIcontainsAnyAssertion(ctx.candidate, c.value);
+    return {
+      score: result.score,
+      verdict: result.score === 1 ? "pass" : "fail",
+      hits: result.hits,
+      misses: result.misses,
+      reasoning: result.score === 1 ? result.hits[0] : result.misses[0],
+      expectedAspectCount: 1
+    };
+  });
+};
+var icontainsAllFactory = (config) => {
+  const c = config;
+  return new DeterministicAssertionEvaluator("icontains_all", (ctx) => {
+    const result = runIcontainsAllAssertion(ctx.candidate, c.value);
+    return {
+      score: result.score,
+      verdict: result.score === 1 ? "pass" : "fail",
+      hits: result.hits,
+      misses: result.misses,
+      reasoning: result.score === 1 ? result.hits[0] : result.misses[0],
+      expectedAspectCount: 1
+    };
+  });
+};
+var startsWithFactory = (config) => {
+  const c = config;
+  return new DeterministicAssertionEvaluator("starts_with", (ctx) => {
+    const result = runStartsWithAssertion(ctx.candidate, c.value);
+    return {
+      score: result.score,
+      verdict: result.score === 1 ? "pass" : "fail",
+      hits: result.hits,
+      misses: result.misses,
+      reasoning: result.score === 1 ? result.hits[0] : result.misses[0],
+      expectedAspectCount: 1
+    };
+  });
+};
+var endsWithFactory = (config) => {
+  const c = config;
+  return new DeterministicAssertionEvaluator("ends_with", (ctx) => {
+    const result = runEndsWithAssertion(ctx.candidate, c.value);
+    return {
+      score: result.score,
+      verdict: result.score === 1 ? "pass" : "fail",
+      hits: result.hits,
+      misses: result.misses,
+      reasoning: result.score === 1 ? result.hits[0] : result.misses[0],
+      expectedAspectCount: 1
+    };
+  });
+};
 function createBuiltinRegistry() {
   const registry = new EvaluatorRegistry();
-  registry.register("llm_judge", llmJudgeFactory).register("code", codeFactory).register("composite", compositeFactory).register("tool_trajectory", toolTrajectoryFactory).register("field_accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token_usage", tokenUsageFactory).register("execution_metrics", executionMetricsFactory).register("agent_judge", agentJudgeFactory).register("contains", containsFactory).register("regex", regexFactory).register("is_json", isJsonFactory).register("equals", equalsFactory);
+  registry.register("llm_judge", llmJudgeFactory).register("code", codeFactory).register("composite", compositeFactory).register("tool_trajectory", toolTrajectoryFactory).register("field_accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token_usage", tokenUsageFactory).register("execution_metrics", executionMetricsFactory).register("agent_judge", agentJudgeFactory).register("contains", containsFactory).register("contains_any", containsAnyFactory).register("contains_all", containsAllFactory).register("icontains", icontainsFactory).register("icontains_any", icontainsAnyFactory).register("icontains_all", icontainsAllFactory).register("starts_with", startsWithFactory).register("ends_with", endsWithFactory).register("regex", regexFactory).register("is_json", isJsonFactory).register("equals", equalsFactory);
   return registry;
 }
 
@@ -12209,18 +12578,236 @@ async function cleanupEvalWorkspaces(evalRunId, workspaceRoot) {
   }
 }
 
+// src/evaluation/workspace/repo-manager.ts
+import { execFile } from "node:child_process";
+import { createHash } from "node:crypto";
+import { existsSync as existsSync2 } from "node:fs";
+import { mkdir as mkdir11, rm as rm5, unlink, writeFile as writeFile7 } from "node:fs/promises";
+import os4 from "node:os";
+import path35 from "node:path";
+import { promisify as promisify5 } from "node:util";
+var execFileAsync = promisify5(execFile);
+var DEFAULT_CACHE_DIR = path35.join(os4.homedir(), ".agentv", "git-cache");
+var DEFAULT_TIMEOUT_MS2 = 3e5;
+var LOCK_TIMEOUT_MS = 6e4;
+function gitEnv() {
+  const env = { ...process.env };
+  for (const key of Object.keys(env)) {
+    if (key.startsWith("GIT_") && key !== "GIT_SSH_COMMAND") {
+      delete env[key];
+    }
+  }
+  return {
+    ...env,
+    GIT_TERMINAL_PROMPT: "0",
+    GIT_ASKPASS: "",
+    GIT_SSH_COMMAND: "ssh -o BatchMode=yes"
+  };
+}
+function cacheKey(source) {
+  const raw = source.type === "git" ? source.url.toLowerCase().replace(/\.git$/, "") : source.path;
+  return createHash("sha256").update(raw).digest("hex");
+}
+function getSourceUrl(source) {
+  return source.type === "git" ? source.url : source.path;
+}
+async function git(args, opts) {
+  const { stdout } = await execFileAsync("git", args, {
+    cwd: opts?.cwd,
+    timeout: opts?.timeout ?? DEFAULT_TIMEOUT_MS2,
+    env: gitEnv(),
+    maxBuffer: 50 * 1024 * 1024
+    // 50MB
+  });
+  return stdout.trim();
+}
+async function acquireLock(lockPath) {
+  const start = Date.now();
+  while (Date.now() - start < LOCK_TIMEOUT_MS) {
+    try {
+      await writeFile7(lockPath, String(process.pid), { flag: "wx" });
+      return;
+    } catch (err) {
+      if (err.code === "EEXIST") {
+        await new Promise((r) => setTimeout(r, 200));
+        continue;
+      }
+      throw err;
+    }
+  }
+  throw new Error(`Timed out waiting for lock: ${lockPath}`);
+}
+async function releaseLock(lockPath) {
+  try {
+    await unlink(lockPath);
+  } catch {
+  }
+}
+var RepoManager = class {
+  cacheDir;
+  constructor(cacheDir) {
+    this.cacheDir = cacheDir ?? DEFAULT_CACHE_DIR;
+  }
+  /**
+   * Ensure a bare mirror cache exists for the given source.
+   * Creates on first access, fetches updates on subsequent calls.
+   * Returns the absolute path to the cache directory.
+   */
+  async ensureCache(source, depth) {
+    const key = cacheKey(source);
+    const cachePath = path35.join(this.cacheDir, key);
+    const lockPath = `${cachePath}.lock`;
+    await mkdir11(this.cacheDir, { recursive: true });
+    await acquireLock(lockPath);
+    try {
+      if (existsSync2(path35.join(cachePath, "HEAD"))) {
+        const fetchArgs = ["fetch", "--prune"];
+        if (depth) {
+          fetchArgs.push("--depth", String(depth));
+        }
+        await git(fetchArgs, { cwd: cachePath });
+      } else {
+        const cloneArgs = ["clone", "--mirror", "--bare"];
+        if (depth) {
+          cloneArgs.push("--depth", String(depth));
+        }
+        const sourceUrl = getSourceUrl(source);
+        const cloneUrl = depth && source.type === "local" ? `file://${sourceUrl}` : sourceUrl;
+        cloneArgs.push(cloneUrl, cachePath);
+        await git(cloneArgs);
+      }
+    } finally {
+      await releaseLock(lockPath);
+    }
+    return cachePath;
+  }
+  /**
+   * Clone a repo from cache into the workspace at the configured path.
+   * Handles checkout, ref resolution, ancestor walking, shallow clone, sparse checkout.
+   */
+  async materialize(repo, workspacePath) {
+    const targetDir = path35.join(workspacePath, repo.path);
+    const cachePath = await this.ensureCache(repo.source, repo.clone?.depth);
+    const cloneArgs = ["clone"];
+    if (repo.clone?.depth) {
+      cloneArgs.push("--depth", String(repo.clone.depth));
+    }
+    if (repo.clone?.filter) {
+      cloneArgs.push("--filter", repo.clone.filter);
+    }
+    cloneArgs.push("--no-checkout");
+    const cloneUrl = repo.clone?.depth || repo.clone?.filter ? `file://${cachePath}` : cachePath;
+    cloneArgs.push(cloneUrl, targetDir);
+    await git(cloneArgs);
+    if (repo.clone?.sparse?.length) {
+      await git(["sparse-checkout", "init", "--cone"], { cwd: targetDir });
+      await git(["sparse-checkout", "set", ...repo.clone.sparse], { cwd: targetDir });
+    }
+    const ref = repo.checkout?.ref ?? "HEAD";
+    const resolve = repo.checkout?.resolve ?? "remote";
+    let resolvedSha;
+    if (resolve === "remote" && repo.source.type === "git") {
+      const url = getSourceUrl(repo.source);
+      try {
+        const lsOutput = await git(["ls-remote", url, ref]);
+        const match = lsOutput.split("	")[0];
+        if (!match) {
+          throw new Error(`Ref '${ref}' not found on remote ${url}`);
+        }
+        resolvedSha = match;
+      } catch (err) {
+        if (err instanceof Error && err.message.includes("not found")) throw err;
+        resolvedSha = ref;
+      }
+    } else {
+      resolvedSha = ref;
+    }
+    await git(["checkout", resolvedSha], { cwd: targetDir });
+    const ancestor = repo.checkout?.ancestor ?? 0;
+    if (ancestor > 0) {
+      try {
+        const ancestorSha = await git(["rev-parse", `HEAD~${ancestor}`], { cwd: targetDir });
+        await git(["checkout", ancestorSha], { cwd: targetDir });
+      } catch {
+        if (repo.clone?.depth) {
+          await git(["fetch", "--deepen", String(ancestor)], { cwd: targetDir });
+          const ancestorSha = await git(["rev-parse", `HEAD~${ancestor}`], { cwd: targetDir });
+          await git(["checkout", ancestorSha], { cwd: targetDir });
+        } else {
+          throw new Error(
+            `Cannot resolve ancestor ${ancestor} of ref '${ref}'. If using shallow clone, increase clone.depth to at least ${ancestor + 1}.`
+          );
+        }
+      }
+    }
+  }
+  /** Materialize all repos into the workspace. */
+  async materializeAll(repos, workspacePath) {
+    for (const repo of repos) {
+      await this.materialize(repo, workspacePath);
+    }
+  }
+  /** Reset repos in workspace to their checkout state. */
+  async reset(repos, workspacePath, strategy) {
+    if (strategy === "recreate") {
+      for (const repo of repos) {
+        const targetDir = path35.join(workspacePath, repo.path);
+        await rm5(targetDir, { recursive: true, force: true });
+      }
+      await this.materializeAll(repos, workspacePath);
+      return;
+    }
+    for (const repo of repos) {
+      const targetDir = path35.join(workspacePath, repo.path);
+      await git(["reset", "--hard", "HEAD"], { cwd: targetDir });
+      await git(["clean", "-fd"], { cwd: targetDir });
+    }
+  }
+  /**
+   * Seed the cache from a local repository, setting the remote to a given URL.
+   * Useful for avoiding slow network clones when a local clone already exists.
+   */
+  async seedCache(localPath, remoteUrl, opts) {
+    const source = { type: "git", url: remoteUrl };
+    const key = cacheKey(source);
+    const cachePath = path35.join(this.cacheDir, key);
+    const lockPath = `${cachePath}.lock`;
+    await mkdir11(this.cacheDir, { recursive: true });
+    await acquireLock(lockPath);
+    try {
+      if (existsSync2(path35.join(cachePath, "HEAD"))) {
+        if (!opts?.force) {
+          throw new Error(
+            `Cache already exists for ${remoteUrl} at ${cachePath}. Use force to overwrite.`
+          );
+        }
+        await rm5(cachePath, { recursive: true, force: true });
+      }
+      await git(["clone", "--mirror", "--bare", localPath, cachePath]);
+      await git(["remote", "set-url", "origin", remoteUrl], { cwd: cachePath });
+    } finally {
+      await releaseLock(lockPath);
+    }
+    return cachePath;
+  }
+  /** Remove the entire cache directory. */
+  async cleanCache() {
+    await rm5(this.cacheDir, { recursive: true, force: true });
+  }
+};
+
 // src/evaluation/workspace/resolve.ts
 import { readdir as readdir4, stat as stat6 } from "node:fs/promises";
-import path35 from "node:path";
+import path36 from "node:path";
 async function resolveWorkspaceTemplate(templatePath) {
   if (!templatePath) {
     return void 0;
   }
-  const resolved = path35.resolve(templatePath);
+  const resolved = path36.resolve(templatePath);
   const stats = await stat6(resolved);
   if (stats.isFile()) {
     return {
-      dir: path35.dirname(resolved),
+      dir: path36.dirname(resolved),
       workspaceFile: resolved
     };
   }
@@ -12232,14 +12819,14 @@ async function resolveWorkspaceTemplate(templatePath) {
   if (workspaceFiles.length === 1) {
     return {
       dir: resolved,
-      workspaceFile: path35.join(resolved, workspaceFiles[0])
+      workspaceFile: path36.join(resolved, workspaceFiles[0])
     };
   }
   if (workspaceFiles.length > 1) {
     const conventionFile = workspaceFiles.find((f) => f === "template.code-workspace");
     return {
       dir: resolved,
-      workspaceFile: conventionFile ? path35.join(resolved, conventionFile) : void 0
+      workspaceFile: conventionFile ? path36.join(resolved, conventionFile) : void 0
     };
   }
   return { dir: resolved };
@@ -12361,6 +12948,11 @@ async function runEvaluation(options) {
     }
     return getOrCreateProvider(resolvedJudge);
   };
+  if (isAgentProvider(getOrCreateProvider(target)) && !target.judgeTarget) {
+    throw new Error(
+      `Target "${target.name}" is an agent provider ("${target.kind}") with no judge_target \u2014 agent providers cannot return structured JSON for judging. Set judge_target to an LLM provider (e.g., azure_base).`
+    );
+  }
   const targetResolver = (name) => {
     const resolved = resolveTargetByName(name);
     if (!resolved) {
@@ -12374,7 +12966,7 @@ async function runEvaluation(options) {
   ];
   const evaluatorRegistry = buildEvaluatorRegistry(evaluators, resolveJudgeProvider);
   const typeRegistry = createBuiltinRegistry();
-  const discoveryBaseDir = evalFilePath ? path36.dirname(path36.resolve(evalFilePath)) : process.cwd();
+  const discoveryBaseDir = evalFilePath ? path37.dirname(path37.resolve(evalFilePath)) : process.cwd();
   await discoverAssertions(typeRegistry, discoveryBaseDir);
   const providerRegistry = createBuiltinProviderRegistry();
   await discoverProviders(providerRegistry, discoveryBaseDir);
@@ -12429,7 +13021,8 @@ async function runEvaluation(options) {
   const resolvedTemplate = await resolveWorkspaceTemplate(rawTemplate);
   const workspaceTemplate = resolvedTemplate?.dir;
   const suiteWorkspaceFile = resolvedTemplate?.workspaceFile;
-  const hasSharedWorkspace = !!(workspaceTemplate || suiteWorkspace?.before_all);
+  const isPerTestIsolation = suiteWorkspace?.isolation === "per_test";
+  const hasSharedWorkspace = !!(workspaceTemplate || suiteWorkspace?.before_all || suiteWorkspace?.repos?.length && !isPerTestIsolation);
   const requestedWorkers = options.maxConcurrency ?? target.workers ?? 1;
   const workers = hasSharedWorkspace ? 1 : requestedWorkers;
   if (hasSharedWorkspace && requestedWorkers > 1) {
@@ -12448,9 +13041,22 @@ async function runEvaluation(options) {
       const message = error instanceof Error ? error.message : String(error);
       throw new Error(`Failed to create shared workspace: ${message}`);
     }
-  } else if (suiteWorkspace?.before_all) {
+  } else if (suiteWorkspace?.before_all || suiteWorkspace?.repos?.length && !isPerTestIsolation) {
     sharedWorkspacePath = getWorkspacePath(evalRunId, "shared");
-    await mkdir11(sharedWorkspacePath, { recursive: true });
+    await mkdir12(sharedWorkspacePath, { recursive: true });
+  }
+  const repoManager = suiteWorkspace?.repos?.length ? new RepoManager() : void 0;
+  if (repoManager && sharedWorkspacePath && suiteWorkspace?.repos && !isPerTestIsolation) {
+    try {
+      await repoManager.materializeAll(suiteWorkspace.repos, sharedWorkspacePath);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      if (sharedWorkspacePath) {
+        await cleanupWorkspace(sharedWorkspacePath).catch(() => {
+        });
+      }
+      throw new Error(`Failed to materialize repos: ${message}`);
+    }
   }
   if (sharedWorkspacePath && suiteWorkspace?.before_all) {
     const scriptContext = {
@@ -12541,7 +13147,8 @@ async function runEvaluation(options) {
           sharedBaselineCommit,
           suiteWorkspaceFile,
           streamCallbacks,
-          typeRegistry
+          typeRegistry,
+          repoManager
         };
         let result = trials && trials.count > 1 ? await runEvalCaseWithTrials(runCaseOptions, trials) : await runEvalCase(runCaseOptions);
         if (totalBudgetUsd !== void 0) {
@@ -12816,15 +13423,16 @@ async function runEvalCase(options) {
     sharedWorkspacePath,
     sharedBaselineCommit,
     suiteWorkspaceFile,
-    typeRegistry: providedTypeRegistry
+    typeRegistry: providedTypeRegistry,
+    repoManager
   } = options;
   const formattingMode = usesFileReferencePrompt(provider) ? "agent" : "lm";
   const promptInputs = await buildPromptInputs(evalCase, formattingMode);
   const typeRegistry = providedTypeRegistry ?? createBuiltinRegistry();
-  const cacheKey = useCache ? createCacheKey(provider, target, evalCase, promptInputs) : void 0;
+  const cacheKey2 = useCache ? createCacheKey(provider, target, evalCase, promptInputs) : void 0;
   let cachedResponse;
-  if (cacheKey && cache) {
-    cachedResponse = await cache.get(cacheKey);
+  if (cacheKey2 && cache) {
+    cachedResponse = await cache.get(cacheKey2);
   }
   const nowFn = now ?? (() => /* @__PURE__ */ new Date());
   let workspacePath = sharedWorkspacePath;
@@ -12853,9 +13461,25 @@ async function runEvalCase(options) {
         );
       }
     }
-    if (!workspacePath && evalCase.workspace?.before_all && evalRunId) {
+    if (!workspacePath && (evalCase.workspace?.before_all || evalCase.workspace?.repos?.length) && evalRunId) {
       workspacePath = getWorkspacePath(evalRunId, evalCase.id);
-      await mkdir11(workspacePath, { recursive: true });
+      await mkdir12(workspacePath, { recursive: true });
+    }
+    if (evalCase.workspace?.repos?.length && workspacePath) {
+      const perCaseRepoManager = new RepoManager();
+      try {
+        await perCaseRepoManager.materializeAll(evalCase.workspace.repos, workspacePath);
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return buildErrorResult(
+          evalCase,
+          target.name,
+          nowFn(),
+          new Error(`Failed to materialize repos: ${message}`),
+          promptInputs,
+          provider
+        );
+      }
     }
     if (workspacePath && evalCase.workspace?.before_all) {
       const scriptContext = {
@@ -12979,8 +13603,8 @@ async function runEvalCase(options) {
     }
     return errorResult;
   }
-  if (cacheKey && cache && !cachedResponse) {
-    await cache.set(cacheKey, providerResponse);
+  if (cacheKey2 && cache && !cachedResponse) {
+    await cache.set(cacheKey2, providerResponse);
   }
   const output = providerResponse.output;
   const hasExecutionMetrics = providerResponse.tokenUsage !== void 0 || providerResponse.costUsd !== void 0 || providerResponse.durationMs !== void 0;
@@ -13008,6 +13632,16 @@ async function runEvalCase(options) {
     }
   }
   const providerError = extractProviderError(providerResponse);
+  if (repoManager && workspacePath && evalCase.workspace?.reset?.after_each && evalCase.workspace.reset.strategy && evalCase.workspace.reset.strategy !== "none" && evalCase.workspace.repos) {
+    try {
+      await repoManager.reset(
+        evalCase.workspace.repos,
+        workspacePath,
+        evalCase.workspace.reset.strategy
+      );
+    } catch {
+    }
+  }
   if (workspacePath && evalCase.workspace?.after_each) {
     const scriptContext = {
       workspacePath,
@@ -13372,7 +14006,7 @@ async function runEvaluatorList(options) {
     fileChanges,
     workspacePath
   };
-  const evalFileDir = evalCase.guideline_paths[0] ? path36.dirname(evalCase.guideline_paths[0]) : process.cwd();
+  const evalFileDir = evalCase.guideline_paths[0] ? path37.dirname(evalCase.guideline_paths[0]) : process.cwd();
   const dispatchContext = {
     judgeProvider,
     targetResolver,
@@ -13462,8 +14096,9 @@ async function runEvaluatorList(options) {
     const minScore = typeof entry.required === "number" ? entry.required : PASS_THRESHOLD;
     return entry.score.score < minScore;
   });
-  const aggregateScore = hasRequiredFailure ? 0 : scored.length > 0 ? computeWeightedMean(
-    scored.map((entry) => ({ score: entry.score.score, weight: entry.weight }))
+  const scorable = scored.filter((entry) => entry.score.verdict !== "skip");
+  const aggregateScore = hasRequiredFailure ? 0 : scorable.length > 0 ? computeWeightedMean(
+    scorable.map((entry) => ({ score: entry.score.score, weight: entry.weight }))
   ) : 0;
   const hits = scored.flatMap((entry) => entry.score.hits);
   const misses = scored.flatMap((entry) => entry.score.misses);
@@ -13603,7 +14238,7 @@ function extractProviderError(response) {
   return trimmed.length > 0 ? trimmed : void 0;
 }
 function createCacheKey(provider, target, evalCase, promptInputs) {
-  const hash = createHash("sha256");
+  const hash = createHash2("sha256");
   hash.update(provider.id);
   hash.update(target.name);
   hash.update(evalCase.id);
@@ -13671,8 +14306,8 @@ function computeWeightedMean(entries) {
 }
 
 // src/evaluation/evaluate.ts
-import { existsSync as existsSync2 } from "node:fs";
-import path37 from "node:path";
+import { existsSync as existsSync3 } from "node:fs";
+import path38 from "node:path";
 async function evaluate(config) {
   const startTime = Date.now();
   if (config.tests && config.specFile) {
@@ -13694,13 +14329,13 @@ async function evaluate(config) {
   let evalCases;
   let testFilePath;
   if (config.specFile) {
-    testFilePath = path37.resolve(config.specFile);
+    testFilePath = path38.resolve(config.specFile);
     evalCases = await loadTests(testFilePath, repoRoot, {
       verbose: config.verbose,
       filter: config.filter
     });
   } else {
-    testFilePath = path37.join(process.cwd(), "__programmatic__.yaml");
+    testFilePath = path38.join(process.cwd(), "__programmatic__.yaml");
     evalCases = (config.tests ?? []).map((test) => {
       const input = typeof test.input === "string" ? [{ role: "user", content: test.input }] : test.input;
       const question = typeof test.input === "string" ? test.input : test.input.find((m) => m.role === "user")?.content ?? "";
@@ -13791,11 +14426,11 @@ function computeSummary(results, durationMs) {
 var TARGET_FILE_CANDIDATES = [".agentv/targets.yaml", ".agentv/targets.yml"];
 async function discoverDefaultTarget(repoRoot) {
   const cwd = process.cwd();
-  const chain = buildDirectoryChain(path37.join(cwd, "_placeholder"), repoRoot);
+  const chain = buildDirectoryChain(path38.join(cwd, "_placeholder"), repoRoot);
   for (const dir of chain) {
     for (const candidate of TARGET_FILE_CANDIDATES) {
-      const targetsPath = path37.join(dir, candidate);
-      if (!existsSync2(targetsPath)) continue;
+      const targetsPath = path38.join(dir, candidate);
+      if (!existsSync3(targetsPath)) continue;
       try {
         const definitions = await readTargetDefinitions(targetsPath);
         const defaultTarget = definitions.find((d) => d.name === "default");
@@ -13809,11 +14444,11 @@ async function discoverDefaultTarget(repoRoot) {
 async function loadEnvHierarchy(repoRoot) {
   const { readFileSync: readFileSync2 } = await import("node:fs");
   const cwd = process.cwd();
-  const chain = buildDirectoryChain(path37.join(cwd, "_placeholder"), repoRoot);
+  const chain = buildDirectoryChain(path38.join(cwd, "_placeholder"), repoRoot);
   const envFiles = [];
   for (const dir of chain) {
-    const envPath = path37.join(dir, ".env");
-    if (existsSync2(envPath)) envFiles.push(envPath);
+    const envPath = path38.join(dir, ".env");
+    if (existsSync3(envPath)) envFiles.push(envPath);
   }
   for (let i = envFiles.length - 1; i >= 0; i--) {
     try {
@@ -13883,12 +14518,12 @@ var CONFIG_FILE_NAMES = [
   ".agentv/config.js"
 ];
 async function loadTsConfig(projectRoot) {
-  const { existsSync: existsSync3 } = await import("node:fs");
+  const { existsSync: existsSync4 } = await import("node:fs");
   const { pathToFileURL } = await import("node:url");
   const { join: join2 } = await import("node:path");
   for (const fileName of CONFIG_FILE_NAMES) {
     const filePath = join2(projectRoot, fileName);
-    if (!existsSync3(filePath)) {
+    if (!existsSync4(filePath)) {
       continue;
     }
     try {
@@ -13985,8 +14620,8 @@ function buildPrompt(criteria, question, referenceAnswer) {
 }
 
 // src/evaluation/cache/response-cache.ts
-import { mkdir as mkdir12, readFile as readFile11, writeFile as writeFile7 } from "node:fs/promises";
-import path38 from "node:path";
+import { mkdir as mkdir13, readFile as readFile11, writeFile as writeFile8 } from "node:fs/promises";
+import path39 from "node:path";
 var DEFAULT_CACHE_PATH = ".agentv/cache";
 var ResponseCache = class {
   cachePath;
@@ -14004,13 +14639,13 @@ var ResponseCache = class {
   }
   async set(key, value) {
     const filePath = this.keyToPath(key);
-    const dir = path38.dirname(filePath);
-    await mkdir12(dir, { recursive: true });
-    await writeFile7(filePath, JSON.stringify(value, null, 2), "utf8");
+    const dir = path39.dirname(filePath);
+    await mkdir13(dir, { recursive: true });
+    await writeFile8(filePath, JSON.stringify(value, null, 2), "utf8");
   }
   keyToPath(key) {
     const prefix = key.slice(0, 2);
-    return path38.join(this.cachePath, prefix, `${key}.json`);
+    return path39.join(this.cachePath, prefix, `${key}.json`);
   }
 };
 function shouldEnableCache(params) {
@@ -14483,6 +15118,7 @@ export {
   OtelTraceExporter,
   OtlpJsonFileExporter,
   ProviderRegistry,
+  RepoManager,
   ResponseCache,
   SimpleTraceFileExporter,
   TEST_MESSAGE_ROLES,
@@ -14568,12 +15204,19 @@ export {
   resolveTargetDefinition,
   resolveWorkspaceTemplate,
   rubricEvaluationSchema,
+  runContainsAllAssertion,
+  runContainsAnyAssertion,
   runContainsAssertion,
+  runEndsWithAssertion,
   runEqualsAssertion,
   runEvalCase,
   runEvaluation,
+  runIcontainsAllAssertion,
+  runIcontainsAnyAssertion,
+  runIcontainsAssertion,
   runIsJsonAssertion,
   runRegexAssertion,
+  runStartsWithAssertion,
   scoreToVerdict,
   shouldEnableCache,
   shouldSkipCacheForTemperature,