@agentv/core 4.8.0 → 4.9.1

package/dist/index.cjs

@@ -1355,13 +1355,13 @@ function serializeAttributeValue(value) {
   if (Array.isArray(value)) return { arrayValue: { values: value.map(serializeAttributeValue) } };
   return { stringValue: String(value) };
 }
-var import_promises35, import_node_path52, OtlpJsonFileExporter;
+var import_promises36, import_node_path53, OtlpJsonFileExporter;
 var init_otlp_json_file_exporter = __esm({
   "src/observability/otlp-json-file-exporter.ts"() {
     "use strict";
     init_cjs_shims();
-    import_promises35 = require("fs/promises");
-    import_node_path52 = require("path");
+    import_promises36 = require("fs/promises");
+    import_node_path53 = require("path");
     OtlpJsonFileExporter = class {
       // biome-ignore lint/suspicious/noExplicitAny: serialized span data
       spans = [];
@@ -1400,7 +1400,7 @@ var init_otlp_json_file_exporter = __esm({
       }
       async flush() {
         if (this.spans.length === 0) return;
-        await (0, import_promises35.mkdir)((0, import_node_path52.dirname)(this.filePath), { recursive: true });
+        await (0, import_promises36.mkdir)((0, import_node_path53.dirname)(this.filePath), { recursive: true });
         const otlpJson = {
           resourceSpans: [
             {
@@ -1583,6 +1583,7 @@ __export(index_exports, {
   runRegexAssertion: () => runRegexAssertion,
   runStartsWithAssertion: () => runStartsWithAssertion,
   saveProjectRegistry: () => saveProjectRegistry,
+  scanRepoDeps: () => scanRepoDeps,
   scoreToVerdict: () => scoreToVerdict,
   shouldEnableCache: () => shouldEnableCache,
   shouldSkipCacheForTemperature: () => shouldSkipCacheForTemperature,
@@ -4741,6 +4742,61 @@ function parseMetadata(suite) {
   });
 }
 
+// src/evaluation/workspace/repo-config-parser.ts
+init_cjs_shims();
+function parseRepoSource(raw) {
+  if (!isJsonObject(raw)) return void 0;
+  const obj = raw;
+  if (obj.type === "git" && typeof obj.url === "string") {
+    return { type: "git", url: obj.url };
+  }
+  if (obj.type === "local" && typeof obj.path === "string") {
+    return { type: "local", path: obj.path };
+  }
+  return void 0;
+}
+function parseRepoCheckout(raw) {
+  if (!isJsonObject(raw)) return void 0;
+  const obj = raw;
+  const ref = typeof obj.ref === "string" ? obj.ref : void 0;
+  const resolve = obj.resolve === "remote" || obj.resolve === "local" ? obj.resolve : void 0;
+  const ancestor = typeof obj.ancestor === "number" ? obj.ancestor : void 0;
+  if (!ref && !resolve && ancestor === void 0) return void 0;
+  return {
+    ...ref !== void 0 && { ref },
+    ...resolve !== void 0 && { resolve },
+    ...ancestor !== void 0 && { ancestor }
+  };
+}
+function parseRepoClone(raw) {
+  if (!isJsonObject(raw)) return void 0;
+  const obj = raw;
+  const depth = typeof obj.depth === "number" ? obj.depth : void 0;
+  const filter = typeof obj.filter === "string" ? obj.filter : void 0;
+  const sparse = Array.isArray(obj.sparse) ? obj.sparse.filter((s) => typeof s === "string") : void 0;
+  if (depth === void 0 && !filter && !sparse) return void 0;
+  return {
+    ...depth !== void 0 && { depth },
+    ...filter !== void 0 && { filter },
+    ...sparse !== void 0 && { sparse }
+  };
+}
+function parseRepoConfig(raw) {
+  if (!isJsonObject(raw)) return void 0;
+  const obj = raw;
+  const repoPath = typeof obj.path === "string" ? obj.path : void 0;
+  const source = parseRepoSource(obj.source);
+  if (!repoPath || !source) return void 0;
+  const checkout = parseRepoCheckout(obj.checkout);
+  const clone = parseRepoClone(obj.clone);
+  return {
+    path: repoPath,
+    source,
+    ...checkout !== void 0 && { checkout },
+    ...clone !== void 0 && { clone }
+  };
+}
+
 // src/evaluation/formatting/prompt-builder.ts
 init_cjs_shims();
 async function buildPromptInputs(testCase, mode = "lm") {
@@ -5136,58 +5192,6 @@ function parseWorkspaceScriptConfig(raw, evalFileDir) {
   }
   return cwd ? { ...config, cwd } : config;
 }
-function parseRepoSource(raw) {
-  if (!isJsonObject(raw)) return void 0;
-  const obj = raw;
-  if (obj.type === "git" && typeof obj.url === "string") {
-    return { type: "git", url: obj.url };
-  }
-  if (obj.type === "local" && typeof obj.path === "string") {
-    return { type: "local", path: obj.path };
-  }
-  return void 0;
-}
-function parseRepoCheckout(raw) {
-  if (!isJsonObject(raw)) return void 0;
-  const obj = raw;
-  const ref = typeof obj.ref === "string" ? obj.ref : void 0;
-  const resolve = obj.resolve === "remote" || obj.resolve === "local" ? obj.resolve : void 0;
-  const ancestor = typeof obj.ancestor === "number" ? obj.ancestor : void 0;
-  if (!ref && !resolve && ancestor === void 0) return void 0;
-  return {
-    ...ref !== void 0 && { ref },
-    ...resolve !== void 0 && { resolve },
-    ...ancestor !== void 0 && { ancestor }
-  };
-}
-function parseRepoClone(raw) {
-  if (!isJsonObject(raw)) return void 0;
-  const obj = raw;
-  const depth = typeof obj.depth === "number" ? obj.depth : void 0;
-  const filter = typeof obj.filter === "string" ? obj.filter : void 0;
-  const sparse = Array.isArray(obj.sparse) ? obj.sparse.filter((s) => typeof s === "string") : void 0;
-  if (depth === void 0 && !filter && !sparse) return void 0;
-  return {
-    ...depth !== void 0 && { depth },
-    ...filter !== void 0 && { filter },
-    ...sparse !== void 0 && { sparse }
-  };
-}
-function parseRepoConfig(raw) {
-  if (!isJsonObject(raw)) return void 0;
-  const obj = raw;
-  const repoPath = typeof obj.path === "string" ? obj.path : void 0;
-  const source = parseRepoSource(obj.source);
-  if (!repoPath || !source) return void 0;
-  const checkout = parseRepoCheckout(obj.checkout);
-  const clone = parseRepoClone(obj.clone);
-  return {
-    path: repoPath,
-    source,
-    ...checkout !== void 0 && { checkout },
-    ...clone !== void 0 && { clone }
-  };
-}
 function parseWorkspaceHookConfig(raw, evalFileDir) {
   if (!isJsonObject(raw)) return void 0;
   const script = parseWorkspaceScriptConfig(raw, evalFileDir);
@@ -12291,8 +12295,8 @@ function resolveCliConfig(target, env, evalFilePath) {
   const parseResult = CliTargetInputSchema.safeParse(target, { errorMap: cliErrorMap });
   if (!parseResult.success) {
     const firstError = parseResult.error.errors[0];
-    const path53 = firstError?.path.join(".") || "";
-    const prefix = path53 ? `${target.name} ${path53}: ` : `${target.name}: `;
+    const path54 = firstError?.path.join(".") || "";
+    const prefix = path54 ? `${target.name} ${path54}: ` : `${target.name}: `;
     throw new Error(`${prefix}${firstError?.message}`);
   }
   const normalized = normalizeCliTargetInput(parseResult.data, env, evalFilePath);
@@ -14337,15 +14341,15 @@ async function execFileWithStdinNode(argv, stdinPayload, options) {
   });
 }
 async function execShellWithStdin(command, stdinPayload, options = {}) {
-  const { mkdir: mkdir17, readFile: readFile17, rm: rm6, writeFile: writeFile9 } = await import("fs/promises");
+  const { mkdir: mkdir17, readFile: readFile18, rm: rm6, writeFile: writeFile9 } = await import("fs/promises");
   const { tmpdir: tmpdir3 } = await import("os");
-  const path53 = await import("path");
+  const path54 = await import("path");
   const { randomUUID: randomUUID10 } = await import("crypto");
-  const dir = path53.join(tmpdir3(), `agentv-exec-${randomUUID10()}`);
+  const dir = path54.join(tmpdir3(), `agentv-exec-${randomUUID10()}`);
   await mkdir17(dir, { recursive: true });
-  const stdinPath = path53.join(dir, "stdin.txt");
-  const stdoutPath = path53.join(dir, "stdout.txt");
-  const stderrPath = path53.join(dir, "stderr.txt");
+  const stdinPath = path54.join(dir, "stdin.txt");
+  const stdoutPath = path54.join(dir, "stdout.txt");
+  const stderrPath = path54.join(dir, "stderr.txt");
   await writeFile9(stdinPath, stdinPayload, "utf8");
   const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
   const { spawn: spawn5 } = await import("child_process");
@@ -14375,8 +14379,8 @@ async function execShellWithStdin(command, stdinPayload, options = {}) {
         resolve(code ?? 0);
       });
     });
-    const stdout = (await readFile17(stdoutPath, "utf8")).replace(/\r\n/g, "\n");
-    const stderr = (await readFile17(stderrPath, "utf8")).replace(/\r\n/g, "\n");
+    const stdout = (await readFile18(stdoutPath, "utf8")).replace(/\r\n/g, "\n");
+    const stderr = (await readFile18(stderrPath, "utf8")).replace(/\r\n/g, "\n");
     return { stdout, stderr, exitCode };
   } finally {
     await rm6(dir, { recursive: true, force: true });
@@ -16642,115 +16646,115 @@ var FieldAccuracyEvaluator = class {
    * Evaluate a single field against the expected value.
    */
   evaluateField(fieldConfig, candidateData, expectedData) {
-    const { path: path53, match, required = true, weight = 1 } = fieldConfig;
-    const candidateValue = resolvePath(candidateData, path53);
-    const expectedValue = resolvePath(expectedData, path53);
+    const { path: path54, match, required = true, weight = 1 } = fieldConfig;
+    const candidateValue = resolvePath(candidateData, path54);
+    const expectedValue = resolvePath(expectedData, path54);
     if (expectedValue === void 0) {
       return {
-        path: path53,
+        path: path54,
         score: 1,
         // No expected value means no comparison needed
         weight,
         hit: true,
-        message: `${path53}: no expected value`
+        message: `${path54}: no expected value`
       };
     }
     if (candidateValue === void 0) {
       if (required) {
         return {
-          path: path53,
+          path: path54,
           score: 0,
           weight,
           hit: false,
-          message: `${path53} (required, missing)`
+          message: `${path54} (required, missing)`
         };
       }
       return {
-        path: path53,
+        path: path54,
         score: 1,
         // Don't penalize missing optional fields
         weight: 0,
         // Zero weight means it won't affect the score
         hit: true,
-        message: `${path53}: optional field missing`
+        message: `${path54}: optional field missing`
       };
     }
     switch (match) {
       case "exact":
-        return this.compareExact(path53, candidateValue, expectedValue, weight);
+        return this.compareExact(path54, candidateValue, expectedValue, weight);
       case "numeric_tolerance":
         return this.compareNumericTolerance(
-          path53,
+          path54,
           candidateValue,
           expectedValue,
           fieldConfig,
           weight
         );
       case "date":
-        return this.compareDate(path53, candidateValue, expectedValue, fieldConfig, weight);
+        return this.compareDate(path54, candidateValue, expectedValue, fieldConfig, weight);
       default:
         return {
-          path: path53,
+          path: path54,
           score: 0,
           weight,
           hit: false,
-          message: `${path53}: unknown match type "${match}"`
+          message: `${path54}: unknown match type "${match}"`
         };
     }
   }
   /**
    * Exact equality comparison.
    */
-  compareExact(path53, candidateValue, expectedValue, weight) {
+  compareExact(path54, candidateValue, expectedValue, weight) {
     if (deepEqual(candidateValue, expectedValue)) {
       return {
-        path: path53,
+        path: path54,
         score: 1,
         weight,
         hit: true,
-        message: path53
+        message: path54
       };
     }
     if (typeof candidateValue !== typeof expectedValue) {
       return {
-        path: path53,
+        path: path54,
         score: 0,
         weight,
         hit: false,
-        message: `${path53} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
+        message: `${path54} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
       };
     }
     return {
-      path: path53,
+      path: path54,
       score: 0,
       weight,
       hit: false,
-      message: `${path53} (value mismatch)`
+      message: `${path54} (value mismatch)`
     };
   }
   /**
    * Numeric comparison with absolute or relative tolerance.
    */
-  compareNumericTolerance(path53, candidateValue, expectedValue, fieldConfig, weight) {
+  compareNumericTolerance(path54, candidateValue, expectedValue, fieldConfig, weight) {
     const { tolerance = 0, relative = false } = fieldConfig;
     const candidateNum = toNumber(candidateValue);
     const expectedNum = toNumber(expectedValue);
     if (candidateNum === null || expectedNum === null) {
       return {
-        path: path53,
+        path: path54,
         score: 0,
         weight,
         hit: false,
-        message: `${path53} (non-numeric value)`
+        message: `${path54} (non-numeric value)`
       };
     }
     if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
       return {
-        path: path53,
+        path: path54,
         score: 0,
         weight,
         hit: false,
-        message: `${path53} (invalid numeric value)`
+        message: `${path54} (invalid numeric value)`
       };
     }
     const diff = Math.abs(candidateNum - expectedNum);
@@ -16763,61 +16767,61 @@ var FieldAccuracyEvaluator = class {
     }
     if (withinTolerance) {
       return {
-        path: path53,
+        path: path54,
         score: 1,
         weight,
         hit: true,
-        message: `${path53} (within tolerance: diff=${diff.toFixed(2)})`
+        message: `${path54} (within tolerance: diff=${diff.toFixed(2)})`
       };
     }
     return {
-      path: path53,
+      path: path54,
       score: 0,
       weight,
       hit: false,
-      message: `${path53} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
+      message: `${path54} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
     };
   }
   /**
    * Date comparison with format normalization.
    */
-  compareDate(path53, candidateValue, expectedValue, fieldConfig, weight) {
+  compareDate(path54, candidateValue, expectedValue, fieldConfig, weight) {
     const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
     const candidateDate = parseDate(String(candidateValue), formats);
     const expectedDate = parseDate(String(expectedValue), formats);
     if (candidateDate === null) {
       return {
-        path: path53,
+        path: path54,
         score: 0,
         weight,
         hit: false,
-        message: `${path53} (unparseable candidate date)`
+        message: `${path54} (unparseable candidate date)`
       };
     }
     if (expectedDate === null) {
       return {
-        path: path53,
+        path: path54,
         score: 0,
         weight,
         hit: false,
-        message: `${path53} (unparseable expected date)`
+        message: `${path54} (unparseable expected date)`
       };
     }
     if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
       return {
-        path: path53,
+        path: path54,
         score: 1,
         weight,
         hit: true,
-        message: path53
+        message: path54
       };
     }
     return {
-      path: path53,
+      path: path54,
       score: 0,
       weight,
       hit: false,
-      message: `${path53} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
+      message: `${path54} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
     };
   }
   /**
@@ -16850,11 +16854,11 @@ var FieldAccuracyEvaluator = class {
     };
   }
 };
-function resolvePath(obj, path53) {
-  if (!path53 || !obj) {
+function resolvePath(obj, path54) {
+  if (!path54 || !obj) {
     return void 0;
   }
-  const parts = path53.split(/\.|\[|\]/).filter((p) => p.length > 0);
+  const parts = path54.split(/\.|\[|\]/).filter((p) => p.length > 0);
   let current = obj;
   for (const part of parts) {
     if (current === null || current === void 0) {
@@ -17351,8 +17355,8 @@ var TokenUsageEvaluator = class {
 
 // src/evaluation/evaluators/tool-trajectory.ts
 init_cjs_shims();
-function getNestedValue(obj, path53) {
-  const parts = path53.split(".");
+function getNestedValue(obj, path54) {
+  const parts = path54.split(".");
   let current = obj;
   for (const part of parts) {
     if (current === null || current === void 0 || typeof current !== "object") {
@@ -17975,6 +17979,7 @@ function runEqualsAssertion(output, value) {
 // src/evaluation/orchestrator.ts
 init_cjs_shims();
 var import_node_crypto11 = require("crypto");
+var import_node_fs16 = require("fs");
 var import_promises33 = require("fs/promises");
 var import_node_path48 = __toESM(require("path"), 1);
 var import_micromatch3 = __toESM(require("micromatch"), 1);
@@ -19776,8 +19781,8 @@ async function runEvaluation(options) {
   const poolSlotBaselines = /* @__PURE__ */ new Map();
   const poolMaxSlots = Math.min(configPoolMaxSlots ?? 10, 50);
   let staticMaterialised = false;
+  const isYamlConfiguredPath = !cliWorkspacePath && !!yamlWorkspacePath;
   if (useStaticWorkspace && configuredStaticPath) {
-    const isYamlConfiguredPath = !cliWorkspacePath && !!yamlWorkspacePath;
     const dirExists = await (0, import_promises33.stat)(configuredStaticPath).then(
       (s) => s.isDirectory(),
       () => false
@@ -19842,14 +19847,28 @@ async function runEvaluation(options) {
       } catch {
       }
     }
-    const needsRepoMaterialisation = !!suiteWorkspace?.repos?.length && !usePool && (!useStaticWorkspace || staticMaterialised);
-    const repoManager = needsRepoMaterialisation ? new RepoManager(verbose) : void 0;
-    if (repoManager && sharedWorkspacePath && suiteWorkspace?.repos && !isPerTestIsolation) {
-      setupLog(
-        `materializing ${suiteWorkspace.repos.length} shared repo(s) into ${sharedWorkspacePath}`
-      );
+    const hasReposToMaterialize = !!suiteWorkspace?.repos?.length && !usePool && !isPerTestIsolation;
+    const needsRepoMaterialisation = hasReposToMaterialize && (!useStaticWorkspace || staticMaterialised);
+    const needsPerRepoCheck = hasReposToMaterialize && useStaticWorkspace && !staticMaterialised && isYamlConfiguredPath;
+    const repoManager = needsRepoMaterialisation || needsPerRepoCheck ? new RepoManager(verbose) : void 0;
+    if (repoManager && sharedWorkspacePath && suiteWorkspace?.repos) {
       try {
-        await repoManager.materializeAll(suiteWorkspace.repos, sharedWorkspacePath);
+        if (needsPerRepoCheck) {
+          for (const repo of suiteWorkspace.repos) {
+            const targetDir = import_node_path48.default.join(sharedWorkspacePath, repo.path);
+            if ((0, import_node_fs16.existsSync)(targetDir)) {
+              setupLog(`reusing existing repo at: ${targetDir}`);
+              continue;
+            }
+            setupLog(`materializing missing repo: ${repo.path}`);
+            await repoManager.materialize(repo, sharedWorkspacePath);
+          }
+        } else {
+          setupLog(
+            `materializing ${suiteWorkspace.repos.length} shared repo(s) into ${sharedWorkspacePath}`
+          );
+          await repoManager.materializeAll(suiteWorkspace.repos, sharedWorkspacePath);
+        }
         setupLog("shared repo materialization complete");
       } catch (error) {
         const message = error instanceof Error ? error.message : String(error);
@@ -21524,7 +21543,7 @@ function computeWeightedMean(entries) {
 
 // src/evaluation/evaluate.ts
 init_cjs_shims();
-var import_node_fs16 = require("fs");
+var import_node_fs17 = require("fs");
 var import_node_path49 = __toESM(require("path"), 1);
 
 // src/evaluation/providers/function-provider.ts
@@ -21682,7 +21701,7 @@ async function discoverDefaultTarget(repoRoot) {
   for (const dir of chain) {
     for (const candidate of TARGET_FILE_CANDIDATES) {
       const targetsPath = import_node_path49.default.join(dir, candidate);
-      if (!(0, import_node_fs16.existsSync)(targetsPath)) continue;
+      if (!(0, import_node_fs17.existsSync)(targetsPath)) continue;
       try {
         const definitions = await readTargetDefinitions(targetsPath);
         const defaultTarget = definitions.find((d) => d.name === "default");
@@ -21699,7 +21718,7 @@ async function loadEnvHierarchy(repoRoot, startPath) {
   const envFiles = [];
   for (const dir of chain) {
     const envPath = import_node_path49.default.join(dir, ".env");
-    if ((0, import_node_fs16.existsSync)(envPath)) envFiles.push(envPath);
+    if ((0, import_node_fs17.existsSync)(envPath)) envFiles.push(envPath);
   }
   for (let i = 0; i < envFiles.length; i++) {
     try {
@@ -21776,12 +21795,12 @@ var CONFIG_FILE_NAMES = [
   ".agentv/config.js"
 ];
 async function loadTsConfig(projectRoot) {
-  const { existsSync: existsSync7 } = await import("fs");
+  const { existsSync: existsSync8 } = await import("fs");
   const { pathToFileURL: pathToFileURL2 } = await import("url");
   const { join: join2 } = await import("path");
   for (const fileName of CONFIG_FILE_NAMES) {
     const filePath = join2(projectRoot, fileName);
-    if (!existsSync7(filePath)) {
+    if (!existsSync8(filePath)) {
       continue;
     }
     try {
@@ -21884,10 +21903,108 @@ function buildPrompt(criteria, question, referenceAnswer) {
 // src/evaluation/workspace/index.ts
 init_cjs_shims();
 
-// src/evaluation/cache/response-cache.ts
+// src/evaluation/workspace/deps-scanner.ts
 init_cjs_shims();
 var import_promises34 = require("fs/promises");
 var import_node_path50 = __toESM(require("path"), 1);
+var import_yaml8 = require("yaml");
+function normalizeGitUrl(url) {
+  let normalized = url.replace(/\.git$/, "");
+  try {
+    const parsed = new URL(normalized);
+    parsed.hostname = parsed.hostname.toLowerCase();
+    normalized = parsed.toString().replace(/\/$/, "");
+  } catch {
+  }
+  return normalized;
+}
+async function scanRepoDeps(evalFilePaths) {
+  const seen = /* @__PURE__ */ new Map();
+  const errors = [];
+  for (const filePath of evalFilePaths) {
+    try {
+      const repos = await extractReposFromEvalFile(filePath);
+      for (const repo of repos) {
+        if (repo.source.type !== "git") continue;
+        const ref = repo.checkout?.ref;
+        const key = `${normalizeGitUrl(repo.source.url)}\0${ref ?? ""}`;
+        const existing = seen.get(key);
+        if (existing) {
+          existing.usedBy.push(filePath);
+        } else {
+          const { ref: _ref, ...checkoutRest } = repo.checkout ?? {};
+          const hasCheckout = Object.keys(checkoutRest).length > 0;
+          seen.set(key, {
+            url: repo.source.url,
+            ref,
+            clone: repo.clone,
+            checkout: hasCheckout ? checkoutRest : void 0,
+            usedBy: [filePath]
+          });
+        }
+      }
+    } catch (err) {
+      errors.push({
+        file: filePath,
+        message: err instanceof Error ? err.message : String(err)
+      });
+    }
+  }
+  return { repos: [...seen.values()], errors };
+}
+async function extractReposFromEvalFile(filePath) {
+  const content = await (0, import_promises34.readFile)(filePath, "utf8");
+  const parsed = interpolateEnv((0, import_yaml8.parse)(content), process.env);
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
+  const obj = parsed;
+  const evalFileDir = import_node_path50.default.dirname(import_node_path50.default.resolve(filePath));
+  const repos = [];
+  const suiteRepos = await extractReposFromWorkspaceRaw(obj.workspace, evalFileDir);
+  repos.push(...suiteRepos);
+  const tests = Array.isArray(obj.tests) ? obj.tests : [];
+  for (const test of tests) {
+    if (test && typeof test === "object" && !Array.isArray(test)) {
+      const testObj = test;
+      const testRepos = await extractReposFromWorkspaceRaw(testObj.workspace, evalFileDir);
+      repos.push(...testRepos);
+    }
+  }
+  return repos;
+}
+async function extractReposFromWorkspaceRaw(raw, evalFileDir) {
+  if (typeof raw === "string") {
+    const workspaceFilePath = import_node_path50.default.resolve(evalFileDir, raw);
+    const content = await (0, import_promises34.readFile)(workspaceFilePath, "utf8");
+    const parsed = interpolateEnv((0, import_yaml8.parse)(content), process.env);
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
+    return extractReposFromObject(parsed);
+  }
+  if (raw && typeof raw === "object" && !Array.isArray(raw)) {
+    return extractReposFromObject(raw);
+  }
+  return [];
+}
+function extractReposFromObject(obj) {
+  const rawRepos = Array.isArray(obj.repos) ? obj.repos : [];
+  const result = [];
+  for (const r of rawRepos) {
+    if (!r || typeof r !== "object" || Array.isArray(r)) continue;
+    const repo = r;
+    const source = parseRepoSource(repo.source);
+    if (!source) continue;
+    result.push({
+      source,
+      checkout: parseRepoCheckout(repo.checkout),
+      clone: parseRepoClone(repo.clone)
+    });
+  }
+  return result;
+}
+
+// src/evaluation/cache/response-cache.ts
+init_cjs_shims();
+var import_promises35 = require("fs/promises");
+var import_node_path51 = __toESM(require("path"), 1);
 var DEFAULT_CACHE_PATH = ".agentv/cache";
 var ResponseCache = class {
   cachePath;
@@ -21897,7 +22014,7 @@ var ResponseCache = class {
   async get(key) {
     const filePath = this.keyToPath(key);
     try {
-      const data = await (0, import_promises34.readFile)(filePath, "utf8");
+      const data = await (0, import_promises35.readFile)(filePath, "utf8");
       return JSON.parse(data);
     } catch {
       return void 0;
@@ -21905,13 +22022,13 @@ var ResponseCache = class {
   }
   async set(key, value) {
     const filePath = this.keyToPath(key);
-    const dir = import_node_path50.default.dirname(filePath);
-    await (0, import_promises34.mkdir)(dir, { recursive: true });
-    await (0, import_promises34.writeFile)(filePath, JSON.stringify(value, null, 2), "utf8");
+    const dir = import_node_path51.default.dirname(filePath);
+    await (0, import_promises35.mkdir)(dir, { recursive: true });
+    await (0, import_promises35.writeFile)(filePath, JSON.stringify(value, null, 2), "utf8");
   }
   keyToPath(key) {
     const prefix = key.slice(0, 2);
-    return import_node_path50.default.join(this.cachePath, prefix, `${key}.json`);
+    return import_node_path51.default.join(this.cachePath, prefix, `${key}.json`);
   }
 };
 function shouldEnableCache(params) {
@@ -21928,20 +22045,20 @@ function shouldSkipCacheForTemperature(targetConfig) {
 
 // src/projects.ts
 init_cjs_shims();
-var import_node_fs17 = require("fs");
-var import_node_path51 = __toESM(require("path"), 1);
-var import_yaml8 = require("yaml");
+var import_node_fs18 = require("fs");
+var import_node_path52 = __toESM(require("path"), 1);
+var import_yaml9 = require("yaml");
 function getProjectsRegistryPath() {
-  return import_node_path51.default.join(getAgentvHome(), "projects.yaml");
+  return import_node_path52.default.join(getAgentvHome(), "projects.yaml");
 }
 function loadProjectRegistry() {
   const registryPath = getProjectsRegistryPath();
-  if (!(0, import_node_fs17.existsSync)(registryPath)) {
+  if (!(0, import_node_fs18.existsSync)(registryPath)) {
     return { projects: [] };
   }
   try {
-    const raw = (0, import_node_fs17.readFileSync)(registryPath, "utf-8");
-    const parsed = (0, import_yaml8.parse)(raw);
+    const raw = (0, import_node_fs18.readFileSync)(registryPath, "utf-8");
+    const parsed = (0, import_yaml9.parse)(raw);
     if (!parsed || !Array.isArray(parsed.projects)) {
       return { projects: [] };
     }
@@ -21952,14 +22069,14 @@ function loadProjectRegistry() {
 }
 function saveProjectRegistry(registry) {
   const registryPath = getProjectsRegistryPath();
-  const dir = import_node_path51.default.dirname(registryPath);
-  if (!(0, import_node_fs17.existsSync)(dir)) {
-    (0, import_node_fs17.mkdirSync)(dir, { recursive: true });
+  const dir = import_node_path52.default.dirname(registryPath);
+  if (!(0, import_node_fs18.existsSync)(dir)) {
+    (0, import_node_fs18.mkdirSync)(dir, { recursive: true });
   }
-  (0, import_node_fs17.writeFileSync)(registryPath, (0, import_yaml8.stringify)(registry), "utf-8");
+  (0, import_node_fs18.writeFileSync)(registryPath, (0, import_yaml9.stringify)(registry), "utf-8");
 }
 function deriveProjectId(dirPath, existingIds) {
-  const base = import_node_path51.default.basename(dirPath).toLowerCase().replace(/[^a-z0-9-]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
+  const base = import_node_path52.default.basename(dirPath).toLowerCase().replace(/[^a-z0-9-]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
   let candidate = base || "project";
   let suffix = 2;
   while (existingIds.includes(candidate)) {
@@ -21969,11 +22086,11 @@ function deriveProjectId(dirPath, existingIds) {
   return candidate;
 }
 function addProject(projectPath) {
-  const absPath = import_node_path51.default.resolve(projectPath);
-  if (!(0, import_node_fs17.existsSync)(absPath)) {
+  const absPath = import_node_path52.default.resolve(projectPath);
+  if (!(0, import_node_fs18.existsSync)(absPath)) {
     throw new Error(`Directory not found: ${absPath}`);
   }
-  if (!(0, import_node_fs17.existsSync)(import_node_path51.default.join(absPath, ".agentv"))) {
+  if (!(0, import_node_fs18.existsSync)(import_node_path52.default.join(absPath, ".agentv"))) {
     throw new Error(`No .agentv/ directory found in ${absPath}. Run an evaluation first.`);
   }
   const registry = loadProjectRegistry();
@@ -21987,7 +22104,7 @@ function addProject(projectPath) {
       absPath,
       registry.projects.map((p) => p.id)
     ),
-    name: import_node_path51.default.basename(absPath),
+    name: import_node_path52.default.basename(absPath),
     path: absPath,
     addedAt: now,
     lastOpenedAt: now
@@ -22016,24 +22133,24 @@ function touchProject(projectId) {
   }
 }
 function discoverProjects(rootDir, maxDepth = 2) {
-  const absRoot = import_node_path51.default.resolve(rootDir);
-  if (!(0, import_node_fs17.existsSync)(absRoot) || !(0, import_node_fs17.statSync)(absRoot).isDirectory()) {
+  const absRoot = import_node_path52.default.resolve(rootDir);
+  if (!(0, import_node_fs18.existsSync)(absRoot) || !(0, import_node_fs18.statSync)(absRoot).isDirectory()) {
     return [];
   }
   const results = [];
   function scan(dir, depth) {
     if (depth > maxDepth) return;
-    if ((0, import_node_fs17.existsSync)(import_node_path51.default.join(dir, ".agentv"))) {
+    if ((0, import_node_fs18.existsSync)(import_node_path52.default.join(dir, ".agentv"))) {
       results.push(dir);
       return;
     }
     if (depth === maxDepth) return;
     try {
-      const entries = (0, import_node_fs17.readdirSync)(dir, { withFileTypes: true });
+      const entries = (0, import_node_fs18.readdirSync)(dir, { withFileTypes: true });
       for (const entry of entries) {
         if (!entry.isDirectory()) continue;
         if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
-        scan(import_node_path51.default.join(dir, entry.name), depth + 1);
+        scan(import_node_path52.default.join(dir, entry.name), depth + 1);
       }
     } catch {
     }
@@ -22959,33 +23076,33 @@ function extractResponseItemContent(content) {
 
 // src/import/codex-session-discovery.ts
 init_cjs_shims();
-var import_promises36 = require("fs/promises");
+var import_promises37 = require("fs/promises");
 var import_node_os8 = require("os");
-var import_node_path53 = __toESM(require("path"), 1);
-var DEFAULT_SESSIONS_DIR = () => import_node_path53.default.join((0, import_node_os8.homedir)(), ".codex", "sessions");
+var import_node_path54 = __toESM(require("path"), 1);
+var DEFAULT_SESSIONS_DIR = () => import_node_path54.default.join((0, import_node_os8.homedir)(), ".codex", "sessions");
 async function discoverCodexSessions(opts) {
   const sessionsDir = opts?.sessionsDir ?? DEFAULT_SESSIONS_DIR();
   const limit = opts?.latest ? 1 : opts?.limit ?? 10;
   const sessions = [];
   let yearDirs;
   try {
-    yearDirs = await (0, import_promises36.readdir)(sessionsDir);
+    yearDirs = await (0, import_promises37.readdir)(sessionsDir);
   } catch {
     return [];
   }
   for (const year of yearDirs) {
-    const yearPath = import_node_path53.default.join(sessionsDir, year);
+    const yearPath = import_node_path54.default.join(sessionsDir, year);
     let monthDirs;
     try {
-      monthDirs = await (0, import_promises36.readdir)(yearPath);
+      monthDirs = await (0, import_promises37.readdir)(yearPath);
     } catch {
       continue;
     }
     for (const month of monthDirs) {
-      const monthPath = import_node_path53.default.join(yearPath, month);
+      const monthPath = import_node_path54.default.join(yearPath, month);
       let dayDirs;
       try {
-        dayDirs = await (0, import_promises36.readdir)(monthPath);
+        dayDirs = await (0, import_promises37.readdir)(monthPath);
       } catch {
         continue;
       }
@@ -22994,22 +23111,22 @@ async function discoverCodexSessions(opts) {
           const dirDate = `${year}-${month}-${day}`;
           if (dirDate !== opts.date) continue;
         }
-        const dayPath = import_node_path53.default.join(monthPath, day);
+        const dayPath = import_node_path54.default.join(monthPath, day);
         let files;
         try {
-          files = await (0, import_promises36.readdir)(dayPath);
+          files = await (0, import_promises37.readdir)(dayPath);
         } catch {
           continue;
         }
         for (const file of files) {
           if (!file.startsWith("rollout-") || !file.endsWith(".jsonl")) continue;
-          const filePath = import_node_path53.default.join(dayPath, file);
+          const filePath = import_node_path54.default.join(dayPath, file);
           const nameWithoutExt = file.replace(/\.jsonl$/, "");
           const parts = nameWithoutExt.split("-");
           const sessionId = parts.length >= 6 ? parts.slice(-5).join("-") : nameWithoutExt;
           let updatedAt;
           try {
-            const fileStat = await (0, import_promises36.stat)(filePath);
+            const fileStat = await (0, import_promises37.stat)(filePath);
             updatedAt = fileStat.mtime;
           } catch {
             updatedAt = /* @__PURE__ */ new Date(0);
@@ -23025,10 +23142,10 @@ async function discoverCodexSessions(opts) {
 
 // src/import/session-discovery.ts
 init_cjs_shims();
-var import_promises37 = require("fs/promises");
+var import_promises38 = require("fs/promises");
 var import_node_os9 = require("os");
-var import_node_path54 = __toESM(require("path"), 1);
-var DEFAULT_PROJECTS_DIR = () => import_node_path54.default.join((0, import_node_os9.homedir)(), ".claude", "projects");
+var import_node_path55 = __toESM(require("path"), 1);
+var DEFAULT_PROJECTS_DIR = () => import_node_path55.default.join((0, import_node_os9.homedir)(), ".claude", "projects");
 function encodeProjectPath(projectPath) {
   return projectPath.replace(/\//g, "-");
 }
@@ -23037,7 +23154,7 @@ async function discoverClaudeSessions(opts) {
   const limit = opts?.latest ? 1 : opts?.limit ?? 10;
   let projectDirs;
   try {
-    projectDirs = await (0, import_promises37.readdir)(projectsDir);
+    projectDirs = await (0, import_promises38.readdir)(projectsDir);
   } catch {
     return [];
   }
@@ -23047,10 +23164,10 @@ async function discoverClaudeSessions(opts) {
   }
   const sessions = [];
   for (const projectDir of projectDirs) {
-    const dirPath = import_node_path54.default.join(projectsDir, projectDir);
+    const dirPath = import_node_path55.default.join(projectsDir, projectDir);
     let entries;
     try {
-      entries = await (0, import_promises37.readdir)(dirPath);
+      entries = await (0, import_promises38.readdir)(dirPath);
     } catch {
       continue;
     }
@@ -23058,10 +23175,10 @@ async function discoverClaudeSessions(opts) {
       if (!entry.endsWith(".jsonl")) continue;
       const sessionId = entry.replace(/\.jsonl$/, "");
       if (opts?.sessionId && sessionId !== opts.sessionId) continue;
-      const filePath = import_node_path54.default.join(dirPath, entry);
+      const filePath = import_node_path55.default.join(dirPath, entry);
       let updatedAt;
       try {
-        const fileStat = await (0, import_promises37.stat)(filePath);
+        const fileStat = await (0, import_promises38.stat)(filePath);
         updatedAt = fileStat.mtime;
       } catch {
         updatedAt = /* @__PURE__ */ new Date(0);
@@ -23083,7 +23200,7 @@ init_cjs_shims();
 
 // src/import/types.ts
 init_cjs_shims();
-var import_promises38 = require("fs/promises");
+var import_promises39 = require("fs/promises");
 function toTranscriptJsonLine(entry) {
   const firstUserMessage = entry.messages.find((m) => m.role === "user");
   const input = typeof firstUserMessage?.content === "string" ? firstUserMessage.content : "";
@@ -23109,11 +23226,11 @@ function toTranscriptJsonLine(entry) {
   };
 }
 async function readTranscriptJsonl(filePath) {
-  const text = await (0, import_promises38.readFile)(filePath, "utf8");
+  const text = await (0, import_promises39.readFile)(filePath, "utf8");
   return text.split("\n").filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
 }
 async function readTranscriptFile(filePath) {
-  return (0, import_promises38.readFile)(filePath, "utf8");
+  return (0, import_promises39.readFile)(filePath, "utf8");
 }
 
 // src/import/transcript-provider.ts
@@ -23328,6 +23445,7 @@ function createAgentKernel() {
   runRegexAssertion,
   runStartsWithAssertion,
   saveProjectRegistry,
+  scanRepoDeps,
   scoreToVerdict,
   shouldEnableCache,
   shouldSkipCacheForTemperature,