@agentv/core 2.10.0 → 2.11.0

package/dist/index.cjs

@@ -1244,12 +1244,12 @@ function serializeAttributeValue(value) {
   if (Array.isArray(value)) return { arrayValue: { values: value.map(serializeAttributeValue) } };
   return { stringValue: String(value) };
 }
-var import_promises30, import_node_path42, OtlpJsonFileExporter;
+var import_promises31, import_node_path43, OtlpJsonFileExporter;
 var init_otlp_json_file_exporter = __esm({
   "src/observability/otlp-json-file-exporter.ts"() {
     "use strict";
-    import_promises30 = require("fs/promises");
-    import_node_path42 = require("path");
+    import_promises31 = require("fs/promises");
+    import_node_path43 = require("path");
     OtlpJsonFileExporter = class {
       // biome-ignore lint/suspicious/noExplicitAny: serialized span data
       spans = [];
@@ -1288,7 +1288,7 @@ var init_otlp_json_file_exporter = __esm({
       }
       async flush() {
         if (this.spans.length === 0) return;
-        await (0, import_promises30.mkdir)((0, import_node_path42.dirname)(this.filePath), { recursive: true });
+        await (0, import_promises31.mkdir)((0, import_node_path43.dirname)(this.filePath), { recursive: true });
         const otlpJson = {
           resourceSpans: [
             {
@@ -1302,8 +1302,8 @@ var init_otlp_json_file_exporter = __esm({
             }
           ]
         };
-        const { writeFile: writeFile8 } = await import("fs/promises");
-        await writeFile8(this.filePath, JSON.stringify(otlpJson, null, 2));
+        const { writeFile: writeFile9 } = await import("fs/promises");
+        await writeFile9(this.filePath, JSON.stringify(otlpJson, null, 2));
       }
     };
   }
@@ -1319,13 +1319,13 @@ function hrTimeDiffMs(start, end) {
   const diffNano = end[1] - start[1];
   return Math.round(diffSec * 1e3 + diffNano / 1e6);
 }
-var import_node_fs12, import_promises31, import_node_path43, SimpleTraceFileExporter;
+var import_node_fs13, import_promises32, import_node_path44, SimpleTraceFileExporter;
 var init_simple_trace_file_exporter = __esm({
   "src/observability/simple-trace-file-exporter.ts"() {
     "use strict";
-    import_node_fs12 = require("fs");
-    import_promises31 = require("fs/promises");
-    import_node_path43 = require("path");
+    import_node_fs13 = require("fs");
+    import_promises32 = require("fs/promises");
+    import_node_path44 = require("path");
     SimpleTraceFileExporter = class {
       stream = null;
       filePath;
@@ -1338,8 +1338,8 @@ var init_simple_trace_file_exporter = __esm({
       async ensureStream() {
         if (!this.streamReady) {
           this.streamReady = (async () => {
-            await (0, import_promises31.mkdir)((0, import_node_path43.dirname)(this.filePath), { recursive: true });
-            this.stream = (0, import_node_fs12.createWriteStream)(this.filePath, { flags: "w" });
+            await (0, import_promises32.mkdir)((0, import_node_path44.dirname)(this.filePath), { recursive: true });
+            this.stream = (0, import_node_fs13.createWriteStream)(this.filePath, { flags: "w" });
             return this.stream;
           })();
         }
@@ -1448,6 +1448,7 @@ __export(index_exports, {
   OtelTraceExporter: () => OtelTraceExporter,
   OtlpJsonFileExporter: () => OtlpJsonFileExporter,
   ProviderRegistry: () => ProviderRegistry,
+  RepoManager: () => RepoManager,
   ResponseCache: () => ResponseCache,
   SimpleTraceFileExporter: () => SimpleTraceFileExporter,
   TEST_MESSAGE_ROLES: () => TEST_MESSAGE_ROLES,
@@ -1533,12 +1534,19 @@ __export(index_exports, {
   resolveTargetDefinition: () => resolveTargetDefinition,
   resolveWorkspaceTemplate: () => resolveWorkspaceTemplate,
   rubricEvaluationSchema: () => rubricEvaluationSchema,
+  runContainsAllAssertion: () => runContainsAllAssertion,
+  runContainsAnyAssertion: () => runContainsAnyAssertion,
   runContainsAssertion: () => runContainsAssertion,
+  runEndsWithAssertion: () => runEndsWithAssertion,
   runEqualsAssertion: () => runEqualsAssertion,
   runEvalCase: () => runEvalCase,
   runEvaluation: () => runEvaluation,
+  runIcontainsAllAssertion: () => runIcontainsAllAssertion,
+  runIcontainsAnyAssertion: () => runIcontainsAnyAssertion,
+  runIcontainsAssertion: () => runIcontainsAssertion,
   runIsJsonAssertion: () => runIsJsonAssertion,
   runRegexAssertion: () => runRegexAssertion,
+  runStartsWithAssertion: () => runStartsWithAssertion,
   scoreToVerdict: () => scoreToVerdict,
   shouldEnableCache: () => shouldEnableCache,
   shouldSkipCacheForTemperature: () => shouldSkipCacheForTemperature,
@@ -1615,6 +1623,13 @@ var EVALUATOR_KIND_VALUES = [
   "execution_metrics",
   "agent_judge",
   "contains",
+  "contains_any",
+  "contains_all",
+  "icontains",
+  "icontains_any",
+  "icontains_all",
+  "starts_with",
+  "ends_with",
   "regex",
   "is_json",
   "equals",
@@ -2888,18 +2903,96 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
+    if (typeValue === "contains_any" || typeValue === "contains_all") {
+      const value = asStringArrayStrict(rawEvaluator.value);
+      if (!value || value.length === 0) {
+        logWarning2(
+          `Skipping ${typeValue} evaluator '${name}' in '${evalId}': value must be a non-empty string array`
+        );
+        continue;
+      }
+      const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
+      const required2 = parseRequired(rawEvaluator.required);
+      evaluators.push({
+        name,
+        type: typeValue,
+        value,
+        ...weight2 !== void 0 ? { weight: weight2 } : {},
+        ...required2 !== void 0 ? { required: required2 } : {},
+        ...negate !== void 0 ? { negate } : {}
+      });
+      continue;
+    }
+    if (typeValue === "icontains") {
+      const value = asString(rawEvaluator.value);
+      if (!value) {
+        logWarning2(`Skipping icontains evaluator '${name}' in '${evalId}': missing value`);
+        continue;
+      }
+      const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
+      const required2 = parseRequired(rawEvaluator.required);
+      evaluators.push({
+        name,
+        type: "icontains",
+        value,
+        ...weight2 !== void 0 ? { weight: weight2 } : {},
+        ...required2 !== void 0 ? { required: required2 } : {},
+        ...negate !== void 0 ? { negate } : {}
+      });
+      continue;
+    }
+    if (typeValue === "icontains_any" || typeValue === "icontains_all") {
+      const value = asStringArrayStrict(rawEvaluator.value);
+      if (!value || value.length === 0) {
+        logWarning2(
+          `Skipping ${typeValue} evaluator '${name}' in '${evalId}': value must be a non-empty string array`
+        );
+        continue;
+      }
+      const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
+      const required2 = parseRequired(rawEvaluator.required);
+      evaluators.push({
+        name,
+        type: typeValue,
+        value,
+        ...weight2 !== void 0 ? { weight: weight2 } : {},
+        ...required2 !== void 0 ? { required: required2 } : {},
+        ...negate !== void 0 ? { negate } : {}
+      });
+      continue;
+    }
+    if (typeValue === "starts_with" || typeValue === "ends_with") {
+      const value = asString(rawEvaluator.value);
+      if (!value) {
+        logWarning2(`Skipping ${typeValue} evaluator '${name}' in '${evalId}': missing value`);
+        continue;
+      }
+      const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
+      const required2 = parseRequired(rawEvaluator.required);
+      evaluators.push({
+        name,
+        type: typeValue,
+        value,
+        ...weight2 !== void 0 ? { weight: weight2 } : {},
+        ...required2 !== void 0 ? { required: required2 } : {},
+        ...negate !== void 0 ? { negate } : {}
+      });
+      continue;
+    }
     if (typeValue === "regex") {
       const value = asString(rawEvaluator.value);
       if (!value) {
         logWarning2(`Skipping regex evaluator '${name}' in '${evalId}': missing value`);
         continue;
       }
+      const flags = asString(rawEvaluator.flags);
       const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
       const required2 = parseRequired(rawEvaluator.required);
       evaluators.push({
         name,
         type: "regex",
         value,
+        ...flags !== void 0 ? { flags } : {},
         ...weight2 !== void 0 ? { weight: weight2 } : {},
         ...required2 !== void 0 ? { required: required2 } : {},
         ...negate !== void 0 ? { negate } : {}
@@ -3072,15 +3165,43 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
   }
   return evaluators.length > 0 ? evaluators : void 0;
 }
-var ASSERTION_TYPES = /* @__PURE__ */ new Set(["contains", "regex", "is_json", "equals", "rubrics"]);
+var ASSERTION_TYPES = /* @__PURE__ */ new Set([
+  "contains",
+  "contains_any",
+  "contains_all",
+  "icontains",
+  "icontains_any",
+  "icontains_all",
+  "starts_with",
+  "ends_with",
+  "regex",
+  "is_json",
+  "equals",
+  "rubrics"
+]);
 function generateAssertionName(typeValue, rawEvaluator) {
   if (!ASSERTION_TYPES.has(typeValue)) {
     return void 0;
   }
   const value = asString(rawEvaluator.value);
+  const arrayValue = Array.isArray(rawEvaluator.value) ? rawEvaluator.value : void 0;
   switch (typeValue) {
     case "contains":
       return value ? `contains-${value}` : "contains";
+    case "contains_any":
+      return arrayValue ? `contains_any-${arrayValue.length}` : "contains_any";
+    case "contains_all":
+      return arrayValue ? `contains_all-${arrayValue.length}` : "contains_all";
+    case "icontains":
+      return value ? `icontains-${value}` : "icontains";
+    case "icontains_any":
+      return arrayValue ? `icontains_any-${arrayValue.length}` : "icontains_any";
+    case "icontains_all":
+      return arrayValue ? `icontains_all-${arrayValue.length}` : "icontains_all";
+    case "starts_with":
+      return value ? `starts_with-${value}` : "starts_with";
+    case "ends_with":
+      return value ? `ends_with-${value}` : "ends_with";
     case "regex":
       return value ? `regex-${value.length > 30 ? value.slice(0, 30) : value}` : "regex";
     case "is_json":
@@ -3106,6 +3227,13 @@ function coerceEvaluator(candidate, contextId) {
 function asString(value) {
   return typeof value === "string" ? value : void 0;
 }
+function asStringArrayStrict(value) {
+  if (!Array.isArray(value)) {
+    return void 0;
+  }
+  const result = value.filter((v) => typeof v === "string");
+  return result.length > 0 ? result : void 0;
+}
 function asStringArray(value, description) {
   if (value === void 0) {
     return void 0;
@@ -4423,6 +4551,69 @@ function parseWorkspaceScriptConfig(raw, evalFileDir) {
   }
   return cwd ? { ...config, cwd } : config;
 }
+function parseRepoSource(raw) {
+  if (!isJsonObject(raw)) return void 0;
+  const obj = raw;
+  if (obj.type === "git" && typeof obj.url === "string") {
+    return { type: "git", url: obj.url };
+  }
+  if (obj.type === "local" && typeof obj.path === "string") {
+    return { type: "local", path: obj.path };
+  }
+  return void 0;
+}
+function parseRepoCheckout(raw) {
+  if (!isJsonObject(raw)) return void 0;
+  const obj = raw;
+  const ref = typeof obj.ref === "string" ? obj.ref : void 0;
+  const resolve = obj.resolve === "remote" || obj.resolve === "local" ? obj.resolve : void 0;
+  const ancestor = typeof obj.ancestor === "number" ? obj.ancestor : void 0;
+  if (!ref && !resolve && ancestor === void 0) return void 0;
+  return {
+    ...ref !== void 0 && { ref },
+    ...resolve !== void 0 && { resolve },
+    ...ancestor !== void 0 && { ancestor }
+  };
+}
+function parseRepoClone(raw) {
+  if (!isJsonObject(raw)) return void 0;
+  const obj = raw;
+  const depth = typeof obj.depth === "number" ? obj.depth : void 0;
+  const filter = typeof obj.filter === "string" ? obj.filter : void 0;
+  const sparse = Array.isArray(obj.sparse) ? obj.sparse.filter((s) => typeof s === "string") : void 0;
+  if (depth === void 0 && !filter && !sparse) return void 0;
+  return {
+    ...depth !== void 0 && { depth },
+    ...filter !== void 0 && { filter },
+    ...sparse !== void 0 && { sparse }
+  };
+}
+function parseRepoConfig(raw) {
+  if (!isJsonObject(raw)) return void 0;
+  const obj = raw;
+  const repoPath = typeof obj.path === "string" ? obj.path : void 0;
+  const source = parseRepoSource(obj.source);
+  if (!repoPath || !source) return void 0;
+  const checkout = parseRepoCheckout(obj.checkout);
+  const clone = parseRepoClone(obj.clone);
+  return {
+    path: repoPath,
+    source,
+    ...checkout !== void 0 && { checkout },
+    ...clone !== void 0 && { clone }
+  };
+}
+function parseResetConfig(raw) {
+  if (!isJsonObject(raw)) return void 0;
+  const obj = raw;
+  const strategy = obj.strategy === "none" || obj.strategy === "hard" || obj.strategy === "recreate" ? obj.strategy : void 0;
+  const afterEach = typeof obj.after_each === "boolean" ? obj.after_each : void 0;
+  if (!strategy && afterEach === void 0) return void 0;
+  return {
+    ...strategy !== void 0 && { strategy },
+    ...afterEach !== void 0 && { after_each: afterEach }
+  };
+}
 function parseWorkspaceConfig(raw, evalFileDir) {
   if (!isJsonObject(raw)) return void 0;
   const obj = raw;
@@ -4430,13 +4621,20 @@ function parseWorkspaceConfig(raw, evalFileDir) {
   if (template && !import_node_path8.default.isAbsolute(template)) {
     template = import_node_path8.default.resolve(evalFileDir, template);
   }
+  const isolation = obj.isolation === "shared" || obj.isolation === "per_test" ? obj.isolation : void 0;
+  const repos = Array.isArray(obj.repos) ? obj.repos.map(parseRepoConfig).filter(Boolean) : void 0;
+  const reset = parseResetConfig(obj.reset);
   const beforeAll = parseWorkspaceScriptConfig(obj.before_all, evalFileDir);
   const afterAll = parseWorkspaceScriptConfig(obj.after_all, evalFileDir);
   const beforeEach = parseWorkspaceScriptConfig(obj.before_each, evalFileDir);
   const afterEach = parseWorkspaceScriptConfig(obj.after_each, evalFileDir);
-  if (!template && !beforeAll && !afterAll && !beforeEach && !afterEach) return void 0;
+  if (!template && !isolation && !repos && !reset && !beforeAll && !afterAll && !beforeEach && !afterEach)
+    return void 0;
   return {
     ...template !== void 0 && { template },
+    ...isolation !== void 0 && { isolation },
+    ...repos !== void 0 && { repos },
+    ...reset !== void 0 && { reset },
     ...beforeAll !== void 0 && { before_all: beforeAll },
     ...afterAll !== void 0 && { after_all: afterAll },
     ...beforeEach !== void 0 && { before_each: beforeEach },
@@ -4449,6 +4647,9 @@ function mergeWorkspaceConfigs(suiteLevel, caseLevel) {
   if (!caseLevel) return suiteLevel;
   return {
     template: caseLevel.template ?? suiteLevel.template,
+    isolation: caseLevel.isolation ?? suiteLevel.isolation,
+    repos: caseLevel.repos ?? suiteLevel.repos,
+    reset: caseLevel.reset ?? suiteLevel.reset,
     before_all: caseLevel.before_all ?? suiteLevel.before_all,
     after_all: caseLevel.after_all ?? suiteLevel.after_all,
     before_each: caseLevel.before_each ?? suiteLevel.before_each,
@@ -5103,11 +5304,6 @@ Original error: ${error instanceof Error ? error.message : String(error)}`
   }
   return claudeSdkModule;
 }
-var DEFAULT_SYSTEM_PROMPT2 = `**IMPORTANT**: Follow these instructions for your response:
-- Do NOT create any additional output files in the workspace.
-- All intended file outputs/changes MUST be written in your response.
-- For each intended file, include the relative path and unified git diff following the convention \`diff --git ...\`.
-This is required for evaluation scoring.`;
 var ClaudeProvider = class {
   id;
   kind = "claude";
@@ -5129,7 +5325,7 @@ var ClaudeProvider = class {
     const logger = await this.createStreamLogger(request).catch(() => void 0);
     const inputFiles = normalizeInputFiles(request.inputFiles);
     const prompt = buildPromptDocument(request, inputFiles);
-    const systemPrompt = this.config.systemPrompt ?? (request.captureFileChanges ? void 0 : DEFAULT_SYSTEM_PROMPT2);
+    const systemPrompt = this.config.systemPrompt;
     const queryOptions = {
       permissionMode: "bypassPermissions",
       allowDangerouslySkipPermissions: true,
@@ -6110,11 +6306,6 @@ Original error: ${error instanceof Error ? error.message : String(error)}`
   }
   return codexSdkModule;
 }
-var DEFAULT_SYSTEM_PROMPT3 = `**IMPORTANT**: Follow these instructions for your response:
-- Do NOT create any additional output files in the workspace.
-- All intended file outputs/changes MUST be written in your response.
-- For each intended file, include the relative path and unified git diff following the convention \`diff --git ...\`.
-This is required for evaluation scoring.`;
 var CodexProvider = class {
   id;
   kind = "codex";
@@ -6149,7 +6340,7 @@ var CodexProvider = class {
     const thread = codex.startThread(threadOptions);
     const inputFiles = normalizeInputFiles(request.inputFiles);
     const basePrompt = buildPromptDocument(request, inputFiles);
-    const systemPrompt = this.config.systemPrompt ?? (request.captureFileChanges ? void 0 : DEFAULT_SYSTEM_PROMPT3);
+    const systemPrompt = this.config.systemPrompt;
     const prompt = systemPrompt ? `${systemPrompt}
 
 ${basePrompt}` : basePrompt;
@@ -6516,7 +6707,7 @@ var import_node_path14 = __toESM(require("path"), 1);
 var import_node_url2 = require("url");
 var import_meta = {};
 function resolvePlatformCliPath() {
-  const os4 = (0, import_node_os2.platform)();
+  const os5 = (0, import_node_os2.platform)();
   const cpu = (0, import_node_os2.arch)();
   const platformMap = {
     linux: "linux",
@@ -6527,13 +6718,13 @@ function resolvePlatformCliPath() {
     x64: "x64",
     arm64: "arm64"
   };
-  const osPart = platformMap[os4];
+  const osPart = platformMap[os5];
   const archPart = archMap[cpu];
   if (!osPart || !archPart) {
     return void 0;
   }
   const packageName = `@github/copilot-${osPart}-${archPart}`;
-  const binaryName = os4 === "win32" ? "copilot.exe" : "copilot";
+  const binaryName = os5 === "win32" ? "copilot.exe" : "copilot";
   try {
     const resolved = import_meta.resolve(`${packageName}/package.json`);
     const packageJsonPath = resolved.startsWith("file:") ? (0, import_node_url2.fileURLToPath)(resolved) : resolved;
@@ -6675,11 +6866,6 @@ function isLogStreamingDisabled(envKey) {
 }
 
 // src/evaluation/providers/copilot-cli.ts
-var DEFAULT_SYSTEM_PROMPT4 = `**IMPORTANT**: Follow these instructions for your response:
-- Do NOT create any additional output files in the workspace.
-- All intended file outputs/changes MUST be written in your response.
-- For each intended file, include the relative path and unified git diff following the convention \`diff --git ...\`.
-This is required for evaluation scoring.`;
 var CopilotCliProvider = class {
   id;
   kind = "copilot-cli";
@@ -6882,8 +7068,8 @@ var CopilotCliProvider = class {
     }
     return args;
   }
-  resolveSystemPrompt(request) {
-    return this.config.systemPrompt ?? (request.captureFileChanges ? void 0 : DEFAULT_SYSTEM_PROMPT4);
+  resolveSystemPrompt(_request) {
+    return this.config.systemPrompt;
   }
   async raceWithTimeout(sendPromise, agentProcess) {
     const timeoutMs = this.config.timeoutMs;
@@ -7071,21 +7257,16 @@ Original error: ${error instanceof Error ? error.message : String(error)}`
   }
   return copilotSdkModule;
 }
-var DEFAULT_SYSTEM_PROMPT5 = `**IMPORTANT**: Follow these instructions for your response:
-- Do NOT create any additional output files in the workspace.
-- All intended file outputs/changes MUST be written in your response.
-- For each intended file, include the relative path and unified git diff following the convention \`diff --git ...\`.
-This is required for evaluation scoring.`;
 var CopilotSdkProvider = class {
   id;
-  kind = "copilot";
+  kind = "copilot-sdk";
   targetName;
   supportsBatch = false;
   config;
   // biome-ignore lint/suspicious/noExplicitAny: SDK client type is dynamically loaded
   client = null;
   constructor(targetName, config) {
-    this.id = `copilot:${targetName}`;
+    this.id = `copilot-sdk:${targetName}`;
     this.targetName = targetName;
     this.config = config;
   }
@@ -7108,7 +7289,7 @@ var CopilotSdkProvider = class {
     if (cwd) {
       sessionOptions.workingDirectory = cwd;
     }
-    const systemPrompt = this.config.systemPrompt ?? (request.captureFileChanges ? void 0 : DEFAULT_SYSTEM_PROMPT5);
+    const systemPrompt = this.config.systemPrompt;
     if (systemPrompt) {
       sessionOptions.systemMessage = {
         mode: "append",
@@ -7624,11 +7805,6 @@ function subscribeToPiLogEntries(listener) {
 // src/evaluation/providers/pi-coding-agent.ts
 var WORKSPACE_PREFIX = "agentv-pi-";
 var PROMPT_FILENAME = "prompt.md";
-var DEFAULT_SYSTEM_PROMPT6 = `**IMPORTANT**: Follow these instructions for your response:
-- Do NOT create any additional output files in the workspace.
-- All intended file outputs/changes MUST be written in your response.
-- For each intended file, include the relative path and unified git diff following the convention \`diff --git ...\`.
-This is required for evaluation scoring.`;
 var PiCodingAgentProvider = class {
   id;
   kind = "pi-coding-agent";
@@ -7705,7 +7881,7 @@ var PiCodingAgentProvider = class {
     }
     return import_node_path17.default.resolve(this.config.cwd);
   }
-  buildPiArgs(prompt, inputFiles, captureFileChanges2) {
+  buildPiArgs(prompt, inputFiles, _captureFileChanges) {
     const args = [];
     if (this.config.provider) {
       args.push("--provider", this.config.provider);
@@ -7733,7 +7909,7 @@ var PiCodingAgentProvider = class {
         args.push(`@${file}`);
       }
     }
-    const systemPrompt = this.config.systemPrompt ?? (captureFileChanges2 ? void 0 : DEFAULT_SYSTEM_PROMPT6);
+    const systemPrompt = this.config.systemPrompt;
     const fullPrompt = systemPrompt ? `${systemPrompt}
 
 ${prompt}` : prompt;
@@ -8604,17 +8780,17 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
         providerBatching,
         config: resolveCodexConfig(parsed, env, evalFilePath)
       };
-    case "copilot":
     case "copilot-sdk":
     case "copilot_sdk":
       return {
-        kind: "copilot",
+        kind: "copilot-sdk",
         name: parsed.name,
         judgeTarget: parsed.judge_target,
         workers: parsed.workers,
         providerBatching,
         config: resolveCopilotSdkConfig(parsed, env, evalFilePath)
       };
+    case "copilot":
     case "copilot-cli":
       return {
         kind: "copilot-cli",
@@ -9225,8 +9401,8 @@ function resolveCliConfig(target, env, evalFilePath) {
   const parseResult = CliTargetInputSchema.safeParse(target, { errorMap: cliErrorMap });
   if (!parseResult.success) {
     const firstError = parseResult.error.errors[0];
-    const path41 = firstError?.path.join(".") || "";
-    const prefix = path41 ? `${target.name} ${path41}: ` : `${target.name}: `;
+    const path42 = firstError?.path.join(".") || "";
+    const prefix = path42 ? `${target.name} ${path42}: ` : `${target.name}: `;
     throw new Error(`${prefix}${firstError?.message}`);
   }
   const normalized = normalizeCliTargetInput(parseResult.data, env, evalFilePath);
@@ -10523,7 +10699,6 @@ var AGENTV_REQUEST_TEMPLATE = `[[ ## task ## ]]
 
 **IMPORTANT**: Follow these exact steps:
 1. Create and write your complete response to: {{responseFileTmp}}
-    - Do NOT create any additional output files in the workspace.
     - All intended file outputs/changes MUST be written in your response file.
     - For each intended file, include the repo name, relative path and unified git diff following the convention \`diff --git ...\`.
 2. When completely finished, run these PowerShell commands to signal completion:
@@ -10542,7 +10717,6 @@ var AGENTV_BATCH_REQUEST_TEMPLATE = `[[ ## task ## ]]
 
 **IMPORTANT**: Follow these exact steps:
 1. Create and write your complete response to: {{responseFileTmp}}
-    - Do NOT create any additional output files in the workspace.
     - All intended file outputs/changes MUST be written in your response file.
     - For each intended file, include the repo name, relative path and unified git diff following the convention \`diff --git ...\`.
 2. When completely finished and the response is stable, rename it to: {{responseFileFinal}}
@@ -10968,7 +11142,7 @@ async function discoverProviders(registry, baseDir) {
 // src/evaluation/providers/index.ts
 function createBuiltinProviderRegistry() {
   const registry = new ProviderRegistry();
-  registry.register("azure", (t) => new AzureProvider(t.name, t.config)).register("anthropic", (t) => new AnthropicProvider(t.name, t.config)).register("gemini", (t) => new GeminiProvider(t.name, t.config)).register("cli", (t) => new CliProvider(t.name, t.config)).register("codex", (t) => new CodexProvider(t.name, t.config)).register("copilot", (t) => new CopilotSdkProvider(t.name, t.config)).register("copilot-cli", (t) => new CopilotCliProvider(t.name, t.config)).register("pi-coding-agent", (t) => new PiCodingAgentProvider(t.name, t.config)).register("pi-agent-sdk", (t) => new PiAgentSdkProvider(t.name, t.config)).register("claude", (t) => new ClaudeProvider(t.name, t.config)).register("mock", (t) => new MockProvider(t.name, t.config)).register("vscode", (t) => new VSCodeProvider(t.name, t.config, "vscode")).register(
+  registry.register("azure", (t) => new AzureProvider(t.name, t.config)).register("anthropic", (t) => new AnthropicProvider(t.name, t.config)).register("gemini", (t) => new GeminiProvider(t.name, t.config)).register("cli", (t) => new CliProvider(t.name, t.config)).register("codex", (t) => new CodexProvider(t.name, t.config)).register("copilot-sdk", (t) => new CopilotSdkProvider(t.name, t.config)).register("copilot-cli", (t) => new CopilotCliProvider(t.name, t.config)).register("pi-coding-agent", (t) => new PiCodingAgentProvider(t.name, t.config)).register("pi-agent-sdk", (t) => new PiAgentSdkProvider(t.name, t.config)).register("claude", (t) => new ClaudeProvider(t.name, t.config)).register("mock", (t) => new MockProvider(t.name, t.config)).register("vscode", (t) => new VSCodeProvider(t.name, t.config, "vscode")).register(
     "vscode-insiders",
     (t) => new VSCodeProvider(t.name, t.config, "vscode-insiders")
   );
@@ -11157,16 +11331,16 @@ async function execFileWithStdinNode(argv, stdinPayload, options) {
   });
 }
 async function execShellWithStdin(command, stdinPayload, options = {}) {
-  const { mkdir: mkdir15, readFile: readFile13, rm: rm5, writeFile: writeFile8 } = await import("fs/promises");
+  const { mkdir: mkdir16, readFile: readFile13, rm: rm6, writeFile: writeFile9 } = await import("fs/promises");
   const { tmpdir: tmpdir3 } = await import("os");
-  const path41 = await import("path");
+  const path42 = await import("path");
   const { randomUUID: randomUUID8 } = await import("crypto");
-  const dir = path41.join(tmpdir3(), `agentv-exec-${randomUUID8()}`);
-  await mkdir15(dir, { recursive: true });
-  const stdinPath = path41.join(dir, "stdin.txt");
-  const stdoutPath = path41.join(dir, "stdout.txt");
-  const stderrPath = path41.join(dir, "stderr.txt");
-  await writeFile8(stdinPath, stdinPayload, "utf8");
+  const dir = path42.join(tmpdir3(), `agentv-exec-${randomUUID8()}`);
+  await mkdir16(dir, { recursive: true });
+  const stdinPath = path42.join(dir, "stdin.txt");
+  const stdoutPath = path42.join(dir, "stdout.txt");
+  const stderrPath = path42.join(dir, "stderr.txt");
+  await writeFile9(stdinPath, stdinPayload, "utf8");
   const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
   const { spawn: spawn4 } = await import("child_process");
   try {
@@ -11199,7 +11373,7 @@ async function execShellWithStdin(command, stdinPayload, options = {}) {
     const stderr = (await readFile13(stderrPath, "utf8")).replace(/\r\n/g, "\n");
     return { stdout, stderr, exitCode };
   } finally {
-    await rm5(dir, { recursive: true, force: true });
+    await rm6(dir, { recursive: true, force: true });
   }
 }
 
@@ -11517,7 +11691,7 @@ var CodeEvaluator = class {
       outputPath,
       guidelineFiles: context2.evalCase.guideline_paths,
       inputFiles: context2.evalCase.file_paths.filter(
-        (path41) => !context2.evalCase.guideline_paths.includes(path41)
+        (path42) => !context2.evalCase.guideline_paths.includes(path42)
       ),
       input: context2.evalCase.input,
       trace: context2.trace ?? null,
@@ -11648,7 +11822,7 @@ var import_ai3 = require("ai");
 // src/evaluation/providers/types.ts
 var AGENT_PROVIDER_KINDS = [
   "codex",
-  "copilot",
+  "copilot-sdk",
   "copilot-cli",
   "pi-coding-agent",
   "claude",
@@ -11794,13 +11968,15 @@ ${context2.fileChanges}`;
         evaluatorRawRequest,
         tokenUsage
       };
-    } catch {
+    } catch (e) {
+      const message = e instanceof Error ? e.message : String(e);
       return {
         score: 0,
-        verdict: "fail",
+        verdict: "skip",
         hits: [],
-        misses: [],
+        misses: [`Judge parse failure after 3 attempts: ${message}`],
         expectedAspectCount: 1,
+        reasoning: `Judge parse failure after 3 attempts: ${message}`,
         evaluatorRawRequest
       };
     }
@@ -12742,115 +12918,115 @@ var FieldAccuracyEvaluator = class {
    * Evaluate a single field against the expected value.
    */
   evaluateField(fieldConfig, candidateData, expectedData) {
-    const { path: path41, match, required = true, weight = 1 } = fieldConfig;
-    const candidateValue = resolvePath(candidateData, path41);
-    const expectedValue = resolvePath(expectedData, path41);
+    const { path: path42, match, required = true, weight = 1 } = fieldConfig;
+    const candidateValue = resolvePath(candidateData, path42);
+    const expectedValue = resolvePath(expectedData, path42);
     if (expectedValue === void 0) {
       return {
-        path: path41,
+        path: path42,
         score: 1,
         // No expected value means no comparison needed
         weight,
         hit: true,
-        message: `${path41}: no expected value`
+        message: `${path42}: no expected value`
       };
     }
     if (candidateValue === void 0) {
       if (required) {
         return {
-          path: path41,
+          path: path42,
           score: 0,
           weight,
           hit: false,
-          message: `${path41} (required, missing)`
+          message: `${path42} (required, missing)`
         };
       }
       return {
-        path: path41,
+        path: path42,
         score: 1,
         // Don't penalize missing optional fields
         weight: 0,
         // Zero weight means it won't affect the score
         hit: true,
-        message: `${path41}: optional field missing`
+        message: `${path42}: optional field missing`
       };
     }
     switch (match) {
       case "exact":
-        return this.compareExact(path41, candidateValue, expectedValue, weight);
+        return this.compareExact(path42, candidateValue, expectedValue, weight);
       case "numeric_tolerance":
         return this.compareNumericTolerance(
-          path41,
+          path42,
           candidateValue,
           expectedValue,
           fieldConfig,
           weight
         );
       case "date":
-        return this.compareDate(path41, candidateValue, expectedValue, fieldConfig, weight);
+        return this.compareDate(path42, candidateValue, expectedValue, fieldConfig, weight);
       default:
         return {
-          path: path41,
+          path: path42,
           score: 0,
           weight,
           hit: false,
-          message: `${path41}: unknown match type "${match}"`
+          message: `${path42}: unknown match type "${match}"`
         };
     }
   }
   /**
    * Exact equality comparison.
    */
-  compareExact(path41, candidateValue, expectedValue, weight) {
+  compareExact(path42, candidateValue, expectedValue, weight) {
     if (deepEqual(candidateValue, expectedValue)) {
       return {
-        path: path41,
+        path: path42,
         score: 1,
         weight,
         hit: true,
-        message: path41
+        message: path42
       };
     }
     if (typeof candidateValue !== typeof expectedValue) {
       return {
-        path: path41,
+        path: path42,
         score: 0,
         weight,
         hit: false,
-        message: `${path41} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
+        message: `${path42} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
       };
     }
     return {
-      path: path41,
+      path: path42,
       score: 0,
       weight,
       hit: false,
-      message: `${path41} (value mismatch)`
+      message: `${path42} (value mismatch)`
     };
   }
   /**
    * Numeric comparison with absolute or relative tolerance.
    */
-  compareNumericTolerance(path41, candidateValue, expectedValue, fieldConfig, weight) {
+  compareNumericTolerance(path42, candidateValue, expectedValue, fieldConfig, weight) {
     const { tolerance = 0, relative = false } = fieldConfig;
     const candidateNum = toNumber2(candidateValue);
     const expectedNum = toNumber2(expectedValue);
     if (candidateNum === null || expectedNum === null) {
       return {
-        path: path41,
+        path: path42,
         score: 0,
         weight,
         hit: false,
-        message: `${path41} (non-numeric value)`
+        message: `${path42} (non-numeric value)`
       };
     }
     if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
       return {
-        path: path41,
+        path: path42,
         score: 0,
         weight,
         hit: false,
-        message: `${path41} (invalid numeric value)`
+        message: `${path42} (invalid numeric value)`
       };
     }
     const diff = Math.abs(candidateNum - expectedNum);
@@ -12863,61 +13039,61 @@ var FieldAccuracyEvaluator = class {
     }
     if (withinTolerance) {
       return {
-        path: path41,
+        path: path42,
         score: 1,
         weight,
         hit: true,
-        message: `${path41} (within tolerance: diff=${diff.toFixed(2)})`
+        message: `${path42} (within tolerance: diff=${diff.toFixed(2)})`
       };
     }
     return {
-      path: path41,
+      path: path42,
       score: 0,
       weight,
       hit: false,
-      message: `${path41} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
+      message: `${path42} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
     };
   }
   /**
    * Date comparison with format normalization.
    */
-  compareDate(path41, candidateValue, expectedValue, fieldConfig, weight) {
+  compareDate(path42, candidateValue, expectedValue, fieldConfig, weight) {
     const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
     const candidateDate = parseDate(String(candidateValue), formats);
     const expectedDate = parseDate(String(expectedValue), formats);
     if (candidateDate === null) {
       return {
-        path: path41,
+        path: path42,
         score: 0,
         weight,
         hit: false,
-        message: `${path41} (unparseable candidate date)`
+        message: `${path42} (unparseable candidate date)`
       };
     }
     if (expectedDate === null) {
       return {
-        path: path41,
+        path: path42,
         score: 0,
         weight,
         hit: false,
-        message: `${path41} (unparseable expected date)`
+        message: `${path42} (unparseable expected date)`
       };
     }
     if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
       return {
-        path: path41,
+        path: path42,
         score: 1,
         weight,
         hit: true,
-        message: path41
+        message: path42
       };
     }
     return {
-      path: path41,
+      path: path42,
       score: 0,
       weight,
       hit: false,
-      message: `${path41} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
+      message: `${path42} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
     };
   }
   /**
@@ -12958,11 +13134,11 @@ var FieldAccuracyEvaluator = class {
     };
   }
 };
-function resolvePath(obj, path41) {
-  if (!path41 || !obj) {
+function resolvePath(obj, path42) {
+  if (!path42 || !obj) {
     return void 0;
   }
-  const parts = path41.split(/\.|\[|\]/).filter((p) => p.length > 0);
+  const parts = path42.split(/\.|\[|\]/).filter((p) => p.length > 0);
   let current = obj;
   for (const part of parts) {
     if (current === null || current === void 0) {
@@ -13780,8 +13956,8 @@ var TokenUsageEvaluator = class {
 };
 
 // src/evaluation/evaluators/tool-trajectory.ts
-function getNestedValue(obj, path41) {
-  const parts = path41.split(".");
+function getNestedValue(obj, path42) {
+  const parts = path42.split(".");
   let current = obj;
   for (const part of parts) {
     if (current === null || current === void 0 || typeof current !== "object") {
@@ -14245,13 +14421,78 @@ function runContainsAssertion(output, value) {
     misses: passed ? [] : [`Output does not contain "${value}"`]
   };
 }
-function runRegexAssertion(output, pattern) {
-  const regex = new RegExp(pattern);
+function runContainsAnyAssertion(output, values) {
+  const matched = values.filter((v) => output.includes(v));
+  const passed = matched.length > 0;
+  return {
+    score: passed ? 1 : 0,
+    hits: passed ? [`Output contains "${matched[0]}"`] : [],
+    misses: passed ? [] : [`Output does not contain any of: ${values.map((v) => `"${v}"`).join(", ")}`]
+  };
+}
+function runContainsAllAssertion(output, values) {
+  const missing = values.filter((v) => !output.includes(v));
+  const passed = missing.length === 0;
+  return {
+    score: passed ? 1 : 0,
+    hits: passed ? [`Output contains all ${values.length} expected strings`] : [],
+    misses: passed ? [] : [`Output missing: ${missing.map((v) => `"${v}"`).join(", ")}`]
+  };
+}
+function runIcontainsAssertion(output, value) {
+  const passed = output.toLowerCase().includes(value.toLowerCase());
+  return {
+    score: passed ? 1 : 0,
+    hits: passed ? [`Output contains "${value}" (case-insensitive)`] : [],
+    misses: passed ? [] : [`Output does not contain "${value}" (case-insensitive)`]
+  };
+}
+function runIcontainsAnyAssertion(output, values) {
+  const lower = output.toLowerCase();
+  const matched = values.filter((v) => lower.includes(v.toLowerCase()));
+  const passed = matched.length > 0;
+  return {
+    score: passed ? 1 : 0,
+    hits: passed ? [`Output contains "${matched[0]}" (case-insensitive)`] : [],
+    misses: passed ? [] : [
+      `Output does not contain any of: ${values.map((v) => `"${v}"`).join(", ")} (case-insensitive)`
+    ]
+  };
+}
+function runIcontainsAllAssertion(output, values) {
+  const lower = output.toLowerCase();
+  const missing = values.filter((v) => !lower.includes(v.toLowerCase()));
+  const passed = missing.length === 0;
+  return {
+    score: passed ? 1 : 0,
+    hits: passed ? [`Output contains all ${values.length} expected strings (case-insensitive)`] : [],
+    misses: passed ? [] : [`Output missing (case-insensitive): ${missing.map((v) => `"${v}"`).join(", ")}`]
+  };
+}
+function runStartsWithAssertion(output, value) {
+  const passed = output.trim().startsWith(value.trim());
+  return {
+    score: passed ? 1 : 0,
+    hits: passed ? [`Output starts with "${value}"`] : [],
+    misses: passed ? [] : [`Output does not start with "${value}"`]
+  };
+}
+function runEndsWithAssertion(output, value) {
+  const passed = output.trim().endsWith(value.trim());
+  return {
+    score: passed ? 1 : 0,
+    hits: passed ? [`Output ends with "${value}"`] : [],
+    misses: passed ? [] : [`Output does not end with "${value}"`]
+  };
+}
+function runRegexAssertion(output, pattern, flags) {
+  const regex = new RegExp(pattern, flags);
   const passed = regex.test(output);
+  const flagsLabel = flags ? ` (flags: ${flags})` : "";
   return {
     score: passed ? 1 : 0,
-    hits: passed ? [`Output matches pattern /${pattern}/`] : [],
-    misses: passed ? [] : [`Output does not match pattern /${pattern}/`]
+    hits: passed ? [`Output matches pattern /${pattern}/${flags ?? ""}${flagsLabel}`] : [],
+    misses: passed ? [] : [`Output does not match pattern /${pattern}/${flags ?? ""}${flagsLabel}`]
   };
 }
 function runIsJsonAssertion(output) {
@@ -14277,9 +14518,9 @@ function runEqualsAssertion(output, value) {
 }
 
 // src/evaluation/orchestrator.ts
-var import_node_crypto8 = require("crypto");
-var import_promises28 = require("fs/promises");
-var import_node_path39 = __toESM(require("path"), 1);
+var import_node_crypto9 = require("crypto");
+var import_promises29 = require("fs/promises");
+var import_node_path40 = __toESM(require("path"), 1);
 var import_micromatch4 = __toESM(require("micromatch"), 1);
 
 // ../../node_modules/.bun/yocto-queue@1.2.2/node_modules/yocto-queue/index.js
@@ -14669,13 +14910,13 @@ var containsFactory = (config) => {
 var regexFactory = (config) => {
   const c = config;
   return new DeterministicAssertionEvaluator("regex", (ctx) => {
-    const result = runRegexAssertion(ctx.candidate, c.value);
+    const result = runRegexAssertion(ctx.candidate, c.value, c.flags);
     return {
       score: result.score,
       verdict: result.score === 1 ? "pass" : "fail",
       hits: result.hits,
       misses: result.misses,
-      reasoning: result.score === 1 ? `Output matches pattern /${c.value}/` : `Output does not match pattern /${c.value}/`,
+      reasoning: result.score === 1 ? `Output matches pattern /${c.value}/${c.flags ?? ""}` : `Output does not match pattern /${c.value}/${c.flags ?? ""}`,
       expectedAspectCount: 1
     };
   });
@@ -14707,9 +14948,107 @@ var equalsFactory = (config) => {
     };
   });
 };
+var containsAnyFactory = (config) => {
+  const c = config;
+  return new DeterministicAssertionEvaluator("contains_any", (ctx) => {
+    const result = runContainsAnyAssertion(ctx.candidate, c.value);
+    return {
+      score: result.score,
+      verdict: result.score === 1 ? "pass" : "fail",
+      hits: result.hits,
+      misses: result.misses,
+      reasoning: result.score === 1 ? result.hits[0] : result.misses[0],
+      expectedAspectCount: 1
+    };
+  });
+};
+var containsAllFactory = (config) => {
+  const c = config;
+  return new DeterministicAssertionEvaluator("contains_all", (ctx) => {
+    const result = runContainsAllAssertion(ctx.candidate, c.value);
+    return {
+      score: result.score,
+      verdict: result.score === 1 ? "pass" : "fail",
+      hits: result.hits,
+      misses: result.misses,
+      reasoning: result.score === 1 ? result.hits[0] : result.misses[0],
+      expectedAspectCount: 1
+    };
+  });
+};
+var icontainsFactory = (config) => {
+  const c = config;
+  return new DeterministicAssertionEvaluator("icontains", (ctx) => {
+    const result = runIcontainsAssertion(ctx.candidate, c.value);
+    return {
+      score: result.score,
+      verdict: result.score === 1 ? "pass" : "fail",
+      hits: result.hits,
+      misses: result.misses,
+      reasoning: result.score === 1 ? result.hits[0] : result.misses[0],
+      expectedAspectCount: 1
+    };
+  });
+};
+var icontainsAnyFactory = (config) => {
+  const c = config;
+  return new DeterministicAssertionEvaluator("icontains_any", (ctx) => {
+    const result = runIcontainsAnyAssertion(ctx.candidate, c.value);
+    return {
+      score: result.score,
+      verdict: result.score === 1 ? "pass" : "fail",
+      hits: result.hits,
+      misses: result.misses,
+      reasoning: result.score === 1 ? result.hits[0] : result.misses[0],
+      expectedAspectCount: 1
+    };
+  });
+};
+var icontainsAllFactory = (config) => {
+  const c = config;
+  return new DeterministicAssertionEvaluator("icontains_all", (ctx) => {
+    const result = runIcontainsAllAssertion(ctx.candidate, c.value);
+    return {
+      score: result.score,
+      verdict: result.score === 1 ? "pass" : "fail",
+      hits: result.hits,
+      misses: result.misses,
+      reasoning: result.score === 1 ? result.hits[0] : result.misses[0],
+      expectedAspectCount: 1
+    };
+  });
+};
+var startsWithFactory = (config) => {
+  const c = config;
+  return new DeterministicAssertionEvaluator("starts_with", (ctx) => {
+    const result = runStartsWithAssertion(ctx.candidate, c.value);
+    return {
+      score: result.score,
+      verdict: result.score === 1 ? "pass" : "fail",
+      hits: result.hits,
+      misses: result.misses,
+      reasoning: result.score === 1 ? result.hits[0] : result.misses[0],
+      expectedAspectCount: 1
+    };
+  });
+};
+var endsWithFactory = (config) => {
+  const c = config;
+  return new DeterministicAssertionEvaluator("ends_with", (ctx) => {
+    const result = runEndsWithAssertion(ctx.candidate, c.value);
+    return {
+      score: result.score,
+      verdict: result.score === 1 ? "pass" : "fail",
+      hits: result.hits,
+      misses: result.misses,
+      reasoning: result.score === 1 ? result.hits[0] : result.misses[0],
+      expectedAspectCount: 1
+    };
+  });
+};
 function createBuiltinRegistry() {
   const registry = new EvaluatorRegistry();
-  registry.register("llm_judge", llmJudgeFactory).register("code", codeFactory).register("composite", compositeFactory).register("tool_trajectory", toolTrajectoryFactory).register("field_accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token_usage", tokenUsageFactory).register("execution_metrics", executionMetricsFactory).register("agent_judge", agentJudgeFactory).register("contains", containsFactory).register("regex", regexFactory).register("is_json", isJsonFactory).register("equals", equalsFactory);
+  registry.register("llm_judge", llmJudgeFactory).register("code", codeFactory).register("composite", compositeFactory).register("tool_trajectory", toolTrajectoryFactory).register("field_accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token_usage", tokenUsageFactory).register("execution_metrics", executionMetricsFactory).register("agent_judge", agentJudgeFactory).register("contains", containsFactory).register("contains_any", containsAnyFactory).register("contains_all", containsAllFactory).register("icontains", icontainsFactory).register("icontains_any", icontainsAnyFactory).register("icontains_all", icontainsAllFactory).register("starts_with", startsWithFactory).register("ends_with", endsWithFactory).register("regex", regexFactory).register("is_json", isJsonFactory).register("equals", equalsFactory);
   return registry;
 }
 
@@ -15053,37 +15392,217 @@ async function cleanupEvalWorkspaces(evalRunId, workspaceRoot) {
   }
 }
 
-// src/evaluation/workspace/resolve.ts
+// src/evaluation/workspace/repo-manager.ts
+var import_node_child_process7 = require("child_process");
+var import_node_crypto8 = require("crypto");
+var import_node_fs11 = require("fs");
 var import_promises27 = require("fs/promises");
+var import_node_os7 = __toESM(require("os"), 1);
 var import_node_path38 = __toESM(require("path"), 1);
+var import_node_util5 = require("util");
+var execFileAsync = (0, import_node_util5.promisify)(import_node_child_process7.execFile);
+var DEFAULT_CACHE_DIR = import_node_path38.default.join(import_node_os7.default.homedir(), ".agentv", "git-cache");
+var DEFAULT_TIMEOUT_MS2 = 3e5;
+var LOCK_TIMEOUT_MS = 6e4;
+function gitEnv() {
+  const env = { ...process.env };
+  for (const key of Object.keys(env)) {
+    if (key.startsWith("GIT_") && key !== "GIT_SSH_COMMAND") {
+      delete env[key];
+    }
+  }
+  return {
+    ...env,
+    GIT_TERMINAL_PROMPT: "0",
+    GIT_ASKPASS: "",
+    GIT_SSH_COMMAND: "ssh -o BatchMode=yes"
+  };
+}
+function cacheKey(source) {
+  const raw = source.type === "git" ? source.url.toLowerCase().replace(/\.git$/, "") : source.path;
+  return (0, import_node_crypto8.createHash)("sha256").update(raw).digest("hex");
+}
+function getSourceUrl(source) {
+  return source.type === "git" ? source.url : source.path;
+}
+async function git(args, opts) {
+  const { stdout } = await execFileAsync("git", args, {
+    cwd: opts?.cwd,
+    timeout: opts?.timeout ?? DEFAULT_TIMEOUT_MS2,
+    env: gitEnv(),
+    maxBuffer: 50 * 1024 * 1024
+    // 50MB
+  });
+  return stdout.trim();
+}
+async function acquireLock(lockPath) {
+  const start = Date.now();
+  while (Date.now() - start < LOCK_TIMEOUT_MS) {
+    try {
+      await (0, import_promises27.writeFile)(lockPath, String(process.pid), { flag: "wx" });
+      return;
+    } catch (err) {
+      if (err.code === "EEXIST") {
+        await new Promise((r) => setTimeout(r, 200));
+        continue;
+      }
+      throw err;
+    }
+  }
+  throw new Error(`Timed out waiting for lock: ${lockPath}`);
+}
+async function releaseLock(lockPath) {
+  try {
+    await (0, import_promises27.unlink)(lockPath);
+  } catch {
+  }
+}
+var RepoManager = class {
+  cacheDir;
+  constructor(cacheDir) {
+    this.cacheDir = cacheDir ?? DEFAULT_CACHE_DIR;
+  }
+  /**
+   * Ensure a bare mirror cache exists for the given source.
+   * Creates on first access, fetches updates on subsequent calls.
+   * Returns the absolute path to the cache directory.
+   */
+  async ensureCache(source) {
+    const key = cacheKey(source);
+    const cachePath = import_node_path38.default.join(this.cacheDir, key);
+    const lockPath = `${cachePath}.lock`;
+    await (0, import_promises27.mkdir)(this.cacheDir, { recursive: true });
+    await acquireLock(lockPath);
+    try {
+      if ((0, import_node_fs11.existsSync)(import_node_path38.default.join(cachePath, "HEAD"))) {
+        await git(["fetch", "--prune"], { cwd: cachePath });
+      } else {
+        await git(["clone", "--mirror", "--bare", getSourceUrl(source), cachePath]);
+      }
+    } finally {
+      await releaseLock(lockPath);
+    }
+    return cachePath;
+  }
+  /**
+   * Clone a repo from cache into the workspace at the configured path.
+   * Handles checkout, ref resolution, ancestor walking, shallow clone, sparse checkout.
+   */
+  async materialize(repo, workspacePath) {
+    const targetDir = import_node_path38.default.join(workspacePath, repo.path);
+    const cachePath = await this.ensureCache(repo.source);
+    const cloneArgs = ["clone"];
+    if (repo.clone?.depth) {
+      cloneArgs.push("--depth", String(repo.clone.depth));
+    }
+    if (repo.clone?.filter) {
+      cloneArgs.push("--filter", repo.clone.filter);
+    }
+    cloneArgs.push("--no-checkout");
+    const cloneUrl = repo.clone?.depth || repo.clone?.filter ? `file://${cachePath}` : cachePath;
+    cloneArgs.push(cloneUrl, targetDir);
+    await git(cloneArgs);
+    if (repo.clone?.sparse?.length) {
+      await git(["sparse-checkout", "init", "--cone"], { cwd: targetDir });
+      await git(["sparse-checkout", "set", ...repo.clone.sparse], { cwd: targetDir });
+    }
+    const ref = repo.checkout?.ref ?? "HEAD";
+    const resolve = repo.checkout?.resolve ?? "remote";
+    let resolvedSha;
+    if (resolve === "remote" && repo.source.type === "git") {
+      const url = getSourceUrl(repo.source);
+      try {
+        const lsOutput = await git(["ls-remote", url, ref]);
+        const match = lsOutput.split("	")[0];
+        if (!match) {
+          throw new Error(`Ref '${ref}' not found on remote ${url}`);
+        }
+        resolvedSha = match;
+      } catch (err) {
+        if (err instanceof Error && err.message.includes("not found")) throw err;
+        resolvedSha = ref;
+      }
+    } else {
+      resolvedSha = ref;
+    }
+    await git(["checkout", resolvedSha], { cwd: targetDir });
+    const ancestor = repo.checkout?.ancestor ?? 0;
+    if (ancestor > 0) {
+      try {
+        const ancestorSha = await git(["rev-parse", `HEAD~${ancestor}`], { cwd: targetDir });
+        await git(["checkout", ancestorSha], { cwd: targetDir });
+      } catch {
+        if (repo.clone?.depth) {
+          await git(["fetch", "--deepen", String(ancestor)], { cwd: targetDir });
+          const ancestorSha = await git(["rev-parse", `HEAD~${ancestor}`], { cwd: targetDir });
+          await git(["checkout", ancestorSha], { cwd: targetDir });
+        } else {
+          throw new Error(
+            `Cannot resolve ancestor ${ancestor} of ref '${ref}'. If using shallow clone, increase clone.depth to at least ${ancestor + 1}.`
+          );
+        }
+      }
+    }
+  }
+  /** Materialize all repos into the workspace. */
+  async materializeAll(repos, workspacePath) {
+    for (const repo of repos) {
+      await this.materialize(repo, workspacePath);
+    }
+  }
+  /** Reset repos in workspace to their checkout state. */
+  async reset(repos, workspacePath, strategy) {
+    if (strategy === "recreate") {
+      for (const repo of repos) {
+        const targetDir = import_node_path38.default.join(workspacePath, repo.path);
+        await (0, import_promises27.rm)(targetDir, { recursive: true, force: true });
+      }
+      await this.materializeAll(repos, workspacePath);
+      return;
+    }
+    for (const repo of repos) {
+      const targetDir = import_node_path38.default.join(workspacePath, repo.path);
+      await git(["reset", "--hard", "HEAD"], { cwd: targetDir });
+      await git(["clean", "-fd"], { cwd: targetDir });
+    }
+  }
+  /** Remove the entire cache directory. */
+  async cleanCache() {
+    await (0, import_promises27.rm)(this.cacheDir, { recursive: true, force: true });
+  }
+};
+
+// src/evaluation/workspace/resolve.ts
+var import_promises28 = require("fs/promises");
+var import_node_path39 = __toESM(require("path"), 1);
 async function resolveWorkspaceTemplate(templatePath) {
   if (!templatePath) {
     return void 0;
   }
-  const resolved = import_node_path38.default.resolve(templatePath);
-  const stats = await (0, import_promises27.stat)(resolved);
+  const resolved = import_node_path39.default.resolve(templatePath);
+  const stats = await (0, import_promises28.stat)(resolved);
   if (stats.isFile()) {
     return {
-      dir: import_node_path38.default.dirname(resolved),
+      dir: import_node_path39.default.dirname(resolved),
       workspaceFile: resolved
     };
   }
   if (!stats.isDirectory()) {
     throw new Error(`workspace template is neither a file nor a directory: ${resolved}`);
   }
-  const entries = await (0, import_promises27.readdir)(resolved);
+  const entries = await (0, import_promises28.readdir)(resolved);
   const workspaceFiles = entries.filter((e) => e.endsWith(".code-workspace"));
   if (workspaceFiles.length === 1) {
     return {
       dir: resolved,
-      workspaceFile: import_node_path38.default.join(resolved, workspaceFiles[0])
+      workspaceFile: import_node_path39.default.join(resolved, workspaceFiles[0])
     };
   }
   if (workspaceFiles.length > 1) {
     const conventionFile = workspaceFiles.find((f) => f === "template.code-workspace");
     return {
       dir: resolved,
-      workspaceFile: conventionFile ? import_node_path38.default.join(resolved, conventionFile) : void 0
+      workspaceFile: conventionFile ? import_node_path39.default.join(resolved, conventionFile) : void 0
     };
   }
   return { dir: resolved };
@@ -15158,7 +15677,7 @@ async function runEvaluation(options) {
     );
     useCache = false;
   }
-  const evalRunId = (0, import_node_crypto8.randomUUID)();
+  const evalRunId = (0, import_node_crypto9.randomUUID)();
   const evalCases = preloadedEvalCases ?? await loadTests(evalFilePath, repoRoot, { verbose, filter });
   const filteredEvalCases = filterEvalCases(evalCases, filter);
   if (filteredEvalCases.length === 0) {
@@ -15205,6 +15724,11 @@ async function runEvaluation(options) {
     }
     return getOrCreateProvider(resolvedJudge);
   };
+  if (isAgentProvider(getOrCreateProvider(target)) && !target.judgeTarget) {
+    throw new Error(
+      `Target "${target.name}" is an agent provider ("${target.kind}") with no judge_target \u2014 agent providers cannot return structured JSON for judging. Set judge_target to an LLM provider (e.g., azure_base).`
+    );
+  }
   const targetResolver = (name) => {
     const resolved = resolveTargetByName(name);
     if (!resolved) {
@@ -15218,7 +15742,7 @@ async function runEvaluation(options) {
   ];
   const evaluatorRegistry = buildEvaluatorRegistry(evaluators, resolveJudgeProvider);
   const typeRegistry = createBuiltinRegistry();
-  const discoveryBaseDir = evalFilePath ? import_node_path39.default.dirname(import_node_path39.default.resolve(evalFilePath)) : process.cwd();
+  const discoveryBaseDir = evalFilePath ? import_node_path40.default.dirname(import_node_path40.default.resolve(evalFilePath)) : process.cwd();
   await discoverAssertions(typeRegistry, discoveryBaseDir);
   const providerRegistry = createBuiltinProviderRegistry();
   await discoverProviders(providerRegistry, discoveryBaseDir);
@@ -15273,7 +15797,8 @@ async function runEvaluation(options) {
   const resolvedTemplate = await resolveWorkspaceTemplate(rawTemplate);
   const workspaceTemplate = resolvedTemplate?.dir;
   const suiteWorkspaceFile = resolvedTemplate?.workspaceFile;
-  const hasSharedWorkspace = !!(workspaceTemplate || suiteWorkspace?.before_all);
+  const isPerTestIsolation = suiteWorkspace?.isolation === "per_test";
+  const hasSharedWorkspace = !!(workspaceTemplate || suiteWorkspace?.before_all || suiteWorkspace?.repos?.length && !isPerTestIsolation);
   const requestedWorkers = options.maxConcurrency ?? target.workers ?? 1;
   const workers = hasSharedWorkspace ? 1 : requestedWorkers;
   if (hasSharedWorkspace && requestedWorkers > 1) {
@@ -15292,9 +15817,22 @@ async function runEvaluation(options) {
       const message = error instanceof Error ? error.message : String(error);
       throw new Error(`Failed to create shared workspace: ${message}`);
     }
-  } else if (suiteWorkspace?.before_all) {
+  } else if (suiteWorkspace?.before_all || suiteWorkspace?.repos?.length && !isPerTestIsolation) {
     sharedWorkspacePath = getWorkspacePath(evalRunId, "shared");
-    await (0, import_promises28.mkdir)(sharedWorkspacePath, { recursive: true });
+    await (0, import_promises29.mkdir)(sharedWorkspacePath, { recursive: true });
+  }
+  const repoManager = suiteWorkspace?.repos?.length ? new RepoManager() : void 0;
+  if (repoManager && sharedWorkspacePath && suiteWorkspace?.repos && !isPerTestIsolation) {
+    try {
+      await repoManager.materializeAll(suiteWorkspace.repos, sharedWorkspacePath);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      if (sharedWorkspacePath) {
+        await cleanupWorkspace(sharedWorkspacePath).catch(() => {
+        });
+      }
+      throw new Error(`Failed to materialize repos: ${message}`);
+    }
   }
   if (sharedWorkspacePath && suiteWorkspace?.before_all) {
     const scriptContext = {
@@ -15385,7 +15923,8 @@ async function runEvaluation(options) {
           sharedBaselineCommit,
           suiteWorkspaceFile,
           streamCallbacks,
-          typeRegistry
+          typeRegistry,
+          repoManager
         };
         let result = trials && trials.count > 1 ? await runEvalCaseWithTrials(runCaseOptions, trials) : await runEvalCase(runCaseOptions);
         if (totalBudgetUsd !== void 0) {
@@ -15660,15 +16199,16 @@ async function runEvalCase(options) {
     sharedWorkspacePath,
     sharedBaselineCommit,
     suiteWorkspaceFile,
-    typeRegistry: providedTypeRegistry
+    typeRegistry: providedTypeRegistry,
+    repoManager
   } = options;
   const formattingMode = usesFileReferencePrompt(provider) ? "agent" : "lm";
   const promptInputs = await buildPromptInputs(evalCase, formattingMode);
   const typeRegistry = providedTypeRegistry ?? createBuiltinRegistry();
-  const cacheKey = useCache ? createCacheKey(provider, target, evalCase, promptInputs) : void 0;
+  const cacheKey2 = useCache ? createCacheKey(provider, target, evalCase, promptInputs) : void 0;
   let cachedResponse;
-  if (cacheKey && cache) {
-    cachedResponse = await cache.get(cacheKey);
+  if (cacheKey2 && cache) {
+    cachedResponse = await cache.get(cacheKey2);
   }
   const nowFn = now ?? (() => /* @__PURE__ */ new Date());
   let workspacePath = sharedWorkspacePath;
@@ -15697,9 +16237,25 @@ async function runEvalCase(options) {
         );
       }
     }
-    if (!workspacePath && evalCase.workspace?.before_all && evalRunId) {
+    if (!workspacePath && (evalCase.workspace?.before_all || evalCase.workspace?.repos?.length) && evalRunId) {
       workspacePath = getWorkspacePath(evalRunId, evalCase.id);
-      await (0, import_promises28.mkdir)(workspacePath, { recursive: true });
+      await (0, import_promises29.mkdir)(workspacePath, { recursive: true });
+    }
+    if (evalCase.workspace?.repos?.length && workspacePath) {
+      const perCaseRepoManager = new RepoManager();
+      try {
+        await perCaseRepoManager.materializeAll(evalCase.workspace.repos, workspacePath);
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return buildErrorResult(
+          evalCase,
+          target.name,
+          nowFn(),
+          new Error(`Failed to materialize repos: ${message}`),
+          promptInputs,
+          provider
+        );
+      }
     }
     if (workspacePath && evalCase.workspace?.before_all) {
       const scriptContext = {
@@ -15823,8 +16379,8 @@ async function runEvalCase(options) {
     }
     return errorResult;
   }
-  if (cacheKey && cache && !cachedResponse) {
-    await cache.set(cacheKey, providerResponse);
+  if (cacheKey2 && cache && !cachedResponse) {
+    await cache.set(cacheKey2, providerResponse);
   }
   const output = providerResponse.output;
   const hasExecutionMetrics = providerResponse.tokenUsage !== void 0 || providerResponse.costUsd !== void 0 || providerResponse.durationMs !== void 0;
@@ -15852,6 +16408,16 @@ async function runEvalCase(options) {
     }
   }
   const providerError = extractProviderError(providerResponse);
+  if (repoManager && workspacePath && evalCase.workspace?.reset?.after_each && evalCase.workspace.reset.strategy && evalCase.workspace.reset.strategy !== "none" && evalCase.workspace.repos) {
+    try {
+      await repoManager.reset(
+        evalCase.workspace.repos,
+        workspacePath,
+        evalCase.workspace.reset.strategy
+      );
+    } catch {
+    }
+  }
   if (workspacePath && evalCase.workspace?.after_each) {
     const scriptContext = {
       workspacePath,
@@ -16216,7 +16782,7 @@ async function runEvaluatorList(options) {
     fileChanges,
     workspacePath
   };
-  const evalFileDir = evalCase.guideline_paths[0] ? import_node_path39.default.dirname(evalCase.guideline_paths[0]) : process.cwd();
+  const evalFileDir = evalCase.guideline_paths[0] ? import_node_path40.default.dirname(evalCase.guideline_paths[0]) : process.cwd();
   const dispatchContext = {
     judgeProvider,
     targetResolver,
@@ -16306,8 +16872,9 @@ async function runEvaluatorList(options) {
     const minScore = typeof entry.required === "number" ? entry.required : PASS_THRESHOLD;
     return entry.score.score < minScore;
   });
-  const aggregateScore = hasRequiredFailure ? 0 : scored.length > 0 ? computeWeightedMean(
-    scored.map((entry) => ({ score: entry.score.score, weight: entry.weight }))
+  const scorable = scored.filter((entry) => entry.score.verdict !== "skip");
+  const aggregateScore = hasRequiredFailure ? 0 : scorable.length > 0 ? computeWeightedMean(
+    scorable.map((entry) => ({ score: entry.score.score, weight: entry.weight }))
   ) : 0;
   const hits = scored.flatMap((entry) => entry.score.hits);
   const misses = scored.flatMap((entry) => entry.score.misses);
@@ -16447,7 +17014,7 @@ function extractProviderError(response) {
   return trimmed.length > 0 ? trimmed : void 0;
 }
 function createCacheKey(provider, target, evalCase, promptInputs) {
-  const hash = (0, import_node_crypto8.createHash)("sha256");
+  const hash = (0, import_node_crypto9.createHash)("sha256");
   hash.update(provider.id);
   hash.update(target.name);
   hash.update(evalCase.id);
@@ -16515,8 +17082,8 @@ function computeWeightedMean(entries) {
 }
 
 // src/evaluation/evaluate.ts
-var import_node_fs11 = require("fs");
-var import_node_path40 = __toESM(require("path"), 1);
+var import_node_fs12 = require("fs");
+var import_node_path41 = __toESM(require("path"), 1);
 async function evaluate(config) {
   const startTime = Date.now();
   if (config.tests && config.specFile) {
@@ -16538,13 +17105,13 @@ async function evaluate(config) {
   let evalCases;
   let testFilePath;
   if (config.specFile) {
-    testFilePath = import_node_path40.default.resolve(config.specFile);
+    testFilePath = import_node_path41.default.resolve(config.specFile);
     evalCases = await loadTests(testFilePath, repoRoot, {
       verbose: config.verbose,
       filter: config.filter
     });
   } else {
-    testFilePath = import_node_path40.default.join(process.cwd(), "__programmatic__.yaml");
+    testFilePath = import_node_path41.default.join(process.cwd(), "__programmatic__.yaml");
     evalCases = (config.tests ?? []).map((test) => {
       const input = typeof test.input === "string" ? [{ role: "user", content: test.input }] : test.input;
       const question = typeof test.input === "string" ? test.input : test.input.find((m) => m.role === "user")?.content ?? "";
@@ -16635,11 +17202,11 @@ function computeSummary(results, durationMs) {
 var TARGET_FILE_CANDIDATES = [".agentv/targets.yaml", ".agentv/targets.yml"];
 async function discoverDefaultTarget(repoRoot) {
   const cwd = process.cwd();
-  const chain = buildDirectoryChain2(import_node_path40.default.join(cwd, "_placeholder"), repoRoot);
+  const chain = buildDirectoryChain2(import_node_path41.default.join(cwd, "_placeholder"), repoRoot);
   for (const dir of chain) {
     for (const candidate of TARGET_FILE_CANDIDATES) {
-      const targetsPath = import_node_path40.default.join(dir, candidate);
-      if (!(0, import_node_fs11.existsSync)(targetsPath)) continue;
+      const targetsPath = import_node_path41.default.join(dir, candidate);
+      if (!(0, import_node_fs12.existsSync)(targetsPath)) continue;
       try {
         const definitions = await readTargetDefinitions(targetsPath);
         const defaultTarget = definitions.find((d) => d.name === "default");
@@ -16653,11 +17220,11 @@ async function discoverDefaultTarget(repoRoot) {
 async function loadEnvHierarchy(repoRoot) {
   const { readFileSync: readFileSync2 } = await import("fs");
   const cwd = process.cwd();
-  const chain = buildDirectoryChain2(import_node_path40.default.join(cwd, "_placeholder"), repoRoot);
+  const chain = buildDirectoryChain2(import_node_path41.default.join(cwd, "_placeholder"), repoRoot);
   const envFiles = [];
   for (const dir of chain) {
-    const envPath = import_node_path40.default.join(dir, ".env");
-    if ((0, import_node_fs11.existsSync)(envPath)) envFiles.push(envPath);
+    const envPath = import_node_path41.default.join(dir, ".env");
+    if ((0, import_node_fs12.existsSync)(envPath)) envFiles.push(envPath);
   }
   for (let i = envFiles.length - 1; i >= 0; i--) {
     try {
@@ -16727,12 +17294,12 @@ var CONFIG_FILE_NAMES = [
   ".agentv/config.js"
 ];
 async function loadTsConfig(projectRoot) {
-  const { existsSync: existsSync3 } = await import("fs");
+  const { existsSync: existsSync4 } = await import("fs");
   const { pathToFileURL } = await import("url");
   const { join: join2 } = await import("path");
   for (const fileName of CONFIG_FILE_NAMES) {
     const filePath = join2(projectRoot, fileName);
-    if (!existsSync3(filePath)) {
+    if (!existsSync4(filePath)) {
       continue;
     }
     try {
@@ -16829,8 +17396,8 @@ function buildPrompt(criteria, question, referenceAnswer) {
 }
 
 // src/evaluation/cache/response-cache.ts
-var import_promises29 = require("fs/promises");
-var import_node_path41 = __toESM(require("path"), 1);
+var import_promises30 = require("fs/promises");
+var import_node_path42 = __toESM(require("path"), 1);
 var DEFAULT_CACHE_PATH = ".agentv/cache";
 var ResponseCache = class {
   cachePath;
@@ -16840,7 +17407,7 @@ var ResponseCache = class {
   async get(key) {
     const filePath = this.keyToPath(key);
     try {
-      const data = await (0, import_promises29.readFile)(filePath, "utf8");
+      const data = await (0, import_promises30.readFile)(filePath, "utf8");
       return JSON.parse(data);
     } catch {
       return void 0;
@@ -16848,13 +17415,13 @@ var ResponseCache = class {
   }
   async set(key, value) {
     const filePath = this.keyToPath(key);
-    const dir = import_node_path41.default.dirname(filePath);
-    await (0, import_promises29.mkdir)(dir, { recursive: true });
-    await (0, import_promises29.writeFile)(filePath, JSON.stringify(value, null, 2), "utf8");
+    const dir = import_node_path42.default.dirname(filePath);
+    await (0, import_promises30.mkdir)(dir, { recursive: true });
+    await (0, import_promises30.writeFile)(filePath, JSON.stringify(value, null, 2), "utf8");
   }
   keyToPath(key) {
     const prefix = key.slice(0, 2);
-    return import_node_path41.default.join(this.cachePath, prefix, `${key}.json`);
+    return import_node_path42.default.join(this.cachePath, prefix, `${key}.json`);
   }
 };
 function shouldEnableCache(params) {
@@ -17332,6 +17899,7 @@ function createAgentKernel() {
   OtelTraceExporter,
   OtlpJsonFileExporter,
   ProviderRegistry,
+  RepoManager,
   ResponseCache,
   SimpleTraceFileExporter,
   TEST_MESSAGE_ROLES,
@@ -17417,12 +17985,19 @@ function createAgentKernel() {
   resolveTargetDefinition,
   resolveWorkspaceTemplate,
   rubricEvaluationSchema,
+  runContainsAllAssertion,
+  runContainsAnyAssertion,
   runContainsAssertion,
+  runEndsWithAssertion,
   runEqualsAssertion,
   runEvalCase,
   runEvaluation,
+  runIcontainsAllAssertion,
+  runIcontainsAnyAssertion,
+  runIcontainsAssertion,
   runIsJsonAssertion,
   runRegexAssertion,
+  runStartsWithAssertion,
   scoreToVerdict,
   shouldEnableCache,
   shouldSkipCacheForTemperature,