@agentv/core 4.8.0 → 4.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -2914,6 +2914,60 @@ function parseMetadata(suite) {
2914
2914
  });
2915
2915
  }
2916
2916
 
2917
+ // src/evaluation/workspace/repo-config-parser.ts
2918
+ function parseRepoSource(raw) {
2919
+ if (!isJsonObject(raw)) return void 0;
2920
+ const obj = raw;
2921
+ if (obj.type === "git" && typeof obj.url === "string") {
2922
+ return { type: "git", url: obj.url };
2923
+ }
2924
+ if (obj.type === "local" && typeof obj.path === "string") {
2925
+ return { type: "local", path: obj.path };
2926
+ }
2927
+ return void 0;
2928
+ }
2929
+ function parseRepoCheckout(raw) {
2930
+ if (!isJsonObject(raw)) return void 0;
2931
+ const obj = raw;
2932
+ const ref = typeof obj.ref === "string" ? obj.ref : void 0;
2933
+ const resolve = obj.resolve === "remote" || obj.resolve === "local" ? obj.resolve : void 0;
2934
+ const ancestor = typeof obj.ancestor === "number" ? obj.ancestor : void 0;
2935
+ if (!ref && !resolve && ancestor === void 0) return void 0;
2936
+ return {
2937
+ ...ref !== void 0 && { ref },
2938
+ ...resolve !== void 0 && { resolve },
2939
+ ...ancestor !== void 0 && { ancestor }
2940
+ };
2941
+ }
2942
+ function parseRepoClone(raw) {
2943
+ if (!isJsonObject(raw)) return void 0;
2944
+ const obj = raw;
2945
+ const depth = typeof obj.depth === "number" ? obj.depth : void 0;
2946
+ const filter = typeof obj.filter === "string" ? obj.filter : void 0;
2947
+ const sparse = Array.isArray(obj.sparse) ? obj.sparse.filter((s) => typeof s === "string") : void 0;
2948
+ if (depth === void 0 && !filter && !sparse) return void 0;
2949
+ return {
2950
+ ...depth !== void 0 && { depth },
2951
+ ...filter !== void 0 && { filter },
2952
+ ...sparse !== void 0 && { sparse }
2953
+ };
2954
+ }
2955
+ function parseRepoConfig(raw) {
2956
+ if (!isJsonObject(raw)) return void 0;
2957
+ const obj = raw;
2958
+ const repoPath = typeof obj.path === "string" ? obj.path : void 0;
2959
+ const source = parseRepoSource(obj.source);
2960
+ if (!repoPath || !source) return void 0;
2961
+ const checkout = parseRepoCheckout(obj.checkout);
2962
+ const clone = parseRepoClone(obj.clone);
2963
+ return {
2964
+ path: repoPath,
2965
+ source,
2966
+ ...checkout !== void 0 && { checkout },
2967
+ ...clone !== void 0 && { clone }
2968
+ };
2969
+ }
2970
+
2917
2971
  // src/evaluation/formatting/prompt-builder.ts
2918
2972
  async function buildPromptInputs(testCase, mode = "lm") {
2919
2973
  const segmentsByMessage = testCase.input.map(
@@ -3308,58 +3362,6 @@ function parseWorkspaceScriptConfig(raw, evalFileDir) {
3308
3362
  }
3309
3363
  return cwd ? { ...config, cwd } : config;
3310
3364
  }
3311
- function parseRepoSource(raw) {
3312
- if (!isJsonObject(raw)) return void 0;
3313
- const obj = raw;
3314
- if (obj.type === "git" && typeof obj.url === "string") {
3315
- return { type: "git", url: obj.url };
3316
- }
3317
- if (obj.type === "local" && typeof obj.path === "string") {
3318
- return { type: "local", path: obj.path };
3319
- }
3320
- return void 0;
3321
- }
3322
- function parseRepoCheckout(raw) {
3323
- if (!isJsonObject(raw)) return void 0;
3324
- const obj = raw;
3325
- const ref = typeof obj.ref === "string" ? obj.ref : void 0;
3326
- const resolve = obj.resolve === "remote" || obj.resolve === "local" ? obj.resolve : void 0;
3327
- const ancestor = typeof obj.ancestor === "number" ? obj.ancestor : void 0;
3328
- if (!ref && !resolve && ancestor === void 0) return void 0;
3329
- return {
3330
- ...ref !== void 0 && { ref },
3331
- ...resolve !== void 0 && { resolve },
3332
- ...ancestor !== void 0 && { ancestor }
3333
- };
3334
- }
3335
- function parseRepoClone(raw) {
3336
- if (!isJsonObject(raw)) return void 0;
3337
- const obj = raw;
3338
- const depth = typeof obj.depth === "number" ? obj.depth : void 0;
3339
- const filter = typeof obj.filter === "string" ? obj.filter : void 0;
3340
- const sparse = Array.isArray(obj.sparse) ? obj.sparse.filter((s) => typeof s === "string") : void 0;
3341
- if (depth === void 0 && !filter && !sparse) return void 0;
3342
- return {
3343
- ...depth !== void 0 && { depth },
3344
- ...filter !== void 0 && { filter },
3345
- ...sparse !== void 0 && { sparse }
3346
- };
3347
- }
3348
- function parseRepoConfig(raw) {
3349
- if (!isJsonObject(raw)) return void 0;
3350
- const obj = raw;
3351
- const repoPath = typeof obj.path === "string" ? obj.path : void 0;
3352
- const source = parseRepoSource(obj.source);
3353
- if (!repoPath || !source) return void 0;
3354
- const checkout = parseRepoCheckout(obj.checkout);
3355
- const clone = parseRepoClone(obj.clone);
3356
- return {
3357
- path: repoPath,
3358
- source,
3359
- ...checkout !== void 0 && { checkout },
3360
- ...clone !== void 0 && { clone }
3361
- };
3362
- }
3363
3365
  function parseWorkspaceHookConfig(raw, evalFileDir) {
3364
3366
  if (!isJsonObject(raw)) return void 0;
3365
3367
  const script = parseWorkspaceScriptConfig(raw, evalFileDir);
@@ -10823,15 +10825,15 @@ async function execFileWithStdinNode(argv, stdinPayload, options) {
10823
10825
  });
10824
10826
  }
10825
10827
  async function execShellWithStdin(command, stdinPayload, options = {}) {
10826
- const { mkdir: mkdir16, readFile: readFile15, rm: rm6, writeFile: writeFile9 } = await import("node:fs/promises");
10828
+ const { mkdir: mkdir16, readFile: readFile16, rm: rm6, writeFile: writeFile9 } = await import("node:fs/promises");
10827
10829
  const { tmpdir: tmpdir3 } = await import("node:os");
10828
- const path50 = await import("node:path");
10830
+ const path51 = await import("node:path");
10829
10831
  const { randomUUID: randomUUID10 } = await import("node:crypto");
10830
- const dir = path50.join(tmpdir3(), `agentv-exec-${randomUUID10()}`);
10832
+ const dir = path51.join(tmpdir3(), `agentv-exec-${randomUUID10()}`);
10831
10833
  await mkdir16(dir, { recursive: true });
10832
- const stdinPath = path50.join(dir, "stdin.txt");
10833
- const stdoutPath = path50.join(dir, "stdout.txt");
10834
- const stderrPath = path50.join(dir, "stderr.txt");
10834
+ const stdinPath = path51.join(dir, "stdin.txt");
10835
+ const stdoutPath = path51.join(dir, "stdout.txt");
10836
+ const stderrPath = path51.join(dir, "stderr.txt");
10835
10837
  await writeFile9(stdinPath, stdinPayload, "utf8");
10836
10838
  const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
10837
10839
  const { spawn: spawn5 } = await import("node:child_process");
@@ -10861,8 +10863,8 @@ async function execShellWithStdin(command, stdinPayload, options = {}) {
10861
10863
  resolve(code ?? 0);
10862
10864
  });
10863
10865
  });
10864
- const stdout = (await readFile15(stdoutPath, "utf8")).replace(/\r\n/g, "\n");
10865
- const stderr = (await readFile15(stderrPath, "utf8")).replace(/\r\n/g, "\n");
10866
+ const stdout = (await readFile16(stdoutPath, "utf8")).replace(/\r\n/g, "\n");
10867
+ const stderr = (await readFile16(stderrPath, "utf8")).replace(/\r\n/g, "\n");
10866
10868
  return { stdout, stderr, exitCode };
10867
10869
  } finally {
10868
10870
  await rm6(dir, { recursive: true, force: true });
@@ -13121,115 +13123,115 @@ var FieldAccuracyEvaluator = class {
13121
13123
  * Evaluate a single field against the expected value.
13122
13124
  */
13123
13125
  evaluateField(fieldConfig, candidateData, expectedData) {
13124
- const { path: path50, match, required = true, weight = 1 } = fieldConfig;
13125
- const candidateValue = resolvePath(candidateData, path50);
13126
- const expectedValue = resolvePath(expectedData, path50);
13126
+ const { path: path51, match, required = true, weight = 1 } = fieldConfig;
13127
+ const candidateValue = resolvePath(candidateData, path51);
13128
+ const expectedValue = resolvePath(expectedData, path51);
13127
13129
  if (expectedValue === void 0) {
13128
13130
  return {
13129
- path: path50,
13131
+ path: path51,
13130
13132
  score: 1,
13131
13133
  // No expected value means no comparison needed
13132
13134
  weight,
13133
13135
  hit: true,
13134
- message: `${path50}: no expected value`
13136
+ message: `${path51}: no expected value`
13135
13137
  };
13136
13138
  }
13137
13139
  if (candidateValue === void 0) {
13138
13140
  if (required) {
13139
13141
  return {
13140
- path: path50,
13142
+ path: path51,
13141
13143
  score: 0,
13142
13144
  weight,
13143
13145
  hit: false,
13144
- message: `${path50} (required, missing)`
13146
+ message: `${path51} (required, missing)`
13145
13147
  };
13146
13148
  }
13147
13149
  return {
13148
- path: path50,
13150
+ path: path51,
13149
13151
  score: 1,
13150
13152
  // Don't penalize missing optional fields
13151
13153
  weight: 0,
13152
13154
  // Zero weight means it won't affect the score
13153
13155
  hit: true,
13154
- message: `${path50}: optional field missing`
13156
+ message: `${path51}: optional field missing`
13155
13157
  };
13156
13158
  }
13157
13159
  switch (match) {
13158
13160
  case "exact":
13159
- return this.compareExact(path50, candidateValue, expectedValue, weight);
13161
+ return this.compareExact(path51, candidateValue, expectedValue, weight);
13160
13162
  case "numeric_tolerance":
13161
13163
  return this.compareNumericTolerance(
13162
- path50,
13164
+ path51,
13163
13165
  candidateValue,
13164
13166
  expectedValue,
13165
13167
  fieldConfig,
13166
13168
  weight
13167
13169
  );
13168
13170
  case "date":
13169
- return this.compareDate(path50, candidateValue, expectedValue, fieldConfig, weight);
13171
+ return this.compareDate(path51, candidateValue, expectedValue, fieldConfig, weight);
13170
13172
  default:
13171
13173
  return {
13172
- path: path50,
13174
+ path: path51,
13173
13175
  score: 0,
13174
13176
  weight,
13175
13177
  hit: false,
13176
- message: `${path50}: unknown match type "${match}"`
13178
+ message: `${path51}: unknown match type "${match}"`
13177
13179
  };
13178
13180
  }
13179
13181
  }
13180
13182
  /**
13181
13183
  * Exact equality comparison.
13182
13184
  */
13183
- compareExact(path50, candidateValue, expectedValue, weight) {
13185
+ compareExact(path51, candidateValue, expectedValue, weight) {
13184
13186
  if (deepEqual(candidateValue, expectedValue)) {
13185
13187
  return {
13186
- path: path50,
13188
+ path: path51,
13187
13189
  score: 1,
13188
13190
  weight,
13189
13191
  hit: true,
13190
- message: path50
13192
+ message: path51
13191
13193
  };
13192
13194
  }
13193
13195
  if (typeof candidateValue !== typeof expectedValue) {
13194
13196
  return {
13195
- path: path50,
13197
+ path: path51,
13196
13198
  score: 0,
13197
13199
  weight,
13198
13200
  hit: false,
13199
- message: `${path50} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
13201
+ message: `${path51} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
13200
13202
  };
13201
13203
  }
13202
13204
  return {
13203
- path: path50,
13205
+ path: path51,
13204
13206
  score: 0,
13205
13207
  weight,
13206
13208
  hit: false,
13207
- message: `${path50} (value mismatch)`
13209
+ message: `${path51} (value mismatch)`
13208
13210
  };
13209
13211
  }
13210
13212
  /**
13211
13213
  * Numeric comparison with absolute or relative tolerance.
13212
13214
  */
13213
- compareNumericTolerance(path50, candidateValue, expectedValue, fieldConfig, weight) {
13215
+ compareNumericTolerance(path51, candidateValue, expectedValue, fieldConfig, weight) {
13214
13216
  const { tolerance = 0, relative = false } = fieldConfig;
13215
13217
  const candidateNum = toNumber(candidateValue);
13216
13218
  const expectedNum = toNumber(expectedValue);
13217
13219
  if (candidateNum === null || expectedNum === null) {
13218
13220
  return {
13219
- path: path50,
13221
+ path: path51,
13220
13222
  score: 0,
13221
13223
  weight,
13222
13224
  hit: false,
13223
- message: `${path50} (non-numeric value)`
13225
+ message: `${path51} (non-numeric value)`
13224
13226
  };
13225
13227
  }
13226
13228
  if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
13227
13229
  return {
13228
- path: path50,
13230
+ path: path51,
13229
13231
  score: 0,
13230
13232
  weight,
13231
13233
  hit: false,
13232
- message: `${path50} (invalid numeric value)`
13234
+ message: `${path51} (invalid numeric value)`
13233
13235
  };
13234
13236
  }
13235
13237
  const diff = Math.abs(candidateNum - expectedNum);
@@ -13242,61 +13244,61 @@ var FieldAccuracyEvaluator = class {
13242
13244
  }
13243
13245
  if (withinTolerance) {
13244
13246
  return {
13245
- path: path50,
13247
+ path: path51,
13246
13248
  score: 1,
13247
13249
  weight,
13248
13250
  hit: true,
13249
- message: `${path50} (within tolerance: diff=${diff.toFixed(2)})`
13251
+ message: `${path51} (within tolerance: diff=${diff.toFixed(2)})`
13250
13252
  };
13251
13253
  }
13252
13254
  return {
13253
- path: path50,
13255
+ path: path51,
13254
13256
  score: 0,
13255
13257
  weight,
13256
13258
  hit: false,
13257
- message: `${path50} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
13259
+ message: `${path51} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
13258
13260
  };
13259
13261
  }
13260
13262
  /**
13261
13263
  * Date comparison with format normalization.
13262
13264
  */
13263
- compareDate(path50, candidateValue, expectedValue, fieldConfig, weight) {
13265
+ compareDate(path51, candidateValue, expectedValue, fieldConfig, weight) {
13264
13266
  const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
13265
13267
  const candidateDate = parseDate(String(candidateValue), formats);
13266
13268
  const expectedDate = parseDate(String(expectedValue), formats);
13267
13269
  if (candidateDate === null) {
13268
13270
  return {
13269
- path: path50,
13271
+ path: path51,
13270
13272
  score: 0,
13271
13273
  weight,
13272
13274
  hit: false,
13273
- message: `${path50} (unparseable candidate date)`
13275
+ message: `${path51} (unparseable candidate date)`
13274
13276
  };
13275
13277
  }
13276
13278
  if (expectedDate === null) {
13277
13279
  return {
13278
- path: path50,
13280
+ path: path51,
13279
13281
  score: 0,
13280
13282
  weight,
13281
13283
  hit: false,
13282
- message: `${path50} (unparseable expected date)`
13284
+ message: `${path51} (unparseable expected date)`
13283
13285
  };
13284
13286
  }
13285
13287
  if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
13286
13288
  return {
13287
- path: path50,
13289
+ path: path51,
13288
13290
  score: 1,
13289
13291
  weight,
13290
13292
  hit: true,
13291
- message: path50
13293
+ message: path51
13292
13294
  };
13293
13295
  }
13294
13296
  return {
13295
- path: path50,
13297
+ path: path51,
13296
13298
  score: 0,
13297
13299
  weight,
13298
13300
  hit: false,
13299
- message: `${path50} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
13301
+ message: `${path51} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
13300
13302
  };
13301
13303
  }
13302
13304
  /**
@@ -13329,11 +13331,11 @@ var FieldAccuracyEvaluator = class {
13329
13331
  };
13330
13332
  }
13331
13333
  };
13332
- function resolvePath(obj, path50) {
13333
- if (!path50 || !obj) {
13334
+ function resolvePath(obj, path51) {
13335
+ if (!path51 || !obj) {
13334
13336
  return void 0;
13335
13337
  }
13336
- const parts = path50.split(/\.|\[|\]/).filter((p) => p.length > 0);
13338
+ const parts = path51.split(/\.|\[|\]/).filter((p) => p.length > 0);
13337
13339
  let current = obj;
13338
13340
  for (const part of parts) {
13339
13341
  if (current === null || current === void 0) {
@@ -13825,8 +13827,8 @@ var TokenUsageEvaluator = class {
13825
13827
  };
13826
13828
 
13827
13829
  // src/evaluation/evaluators/tool-trajectory.ts
13828
- function getNestedValue(obj, path50) {
13829
- const parts = path50.split(".");
13830
+ function getNestedValue(obj, path51) {
13831
+ const parts = path51.split(".");
13830
13832
  let current = obj;
13831
13833
  for (const part of parts) {
13832
13834
  if (current === null || current === void 0 || typeof current !== "object") {
@@ -14447,6 +14449,7 @@ function runEqualsAssertion(output, value) {
14447
14449
 
14448
14450
  // src/evaluation/orchestrator.ts
14449
14451
  import { createHash as createHash2, randomUUID as randomUUID9 } from "node:crypto";
14452
+ import { existsSync as existsSync5 } from "node:fs";
14450
14453
  import { copyFile as copyFile2, mkdir as mkdir14, readdir as readdir7, stat as stat8 } from "node:fs/promises";
14451
14454
  import path44 from "node:path";
14452
14455
  import micromatch3 from "micromatch";
@@ -16226,8 +16229,8 @@ async function runEvaluation(options) {
16226
16229
  const poolSlotBaselines = /* @__PURE__ */ new Map();
16227
16230
  const poolMaxSlots = Math.min(configPoolMaxSlots ?? 10, 50);
16228
16231
  let staticMaterialised = false;
16232
+ const isYamlConfiguredPath = !cliWorkspacePath && !!yamlWorkspacePath;
16229
16233
  if (useStaticWorkspace && configuredStaticPath) {
16230
- const isYamlConfiguredPath = !cliWorkspacePath && !!yamlWorkspacePath;
16231
16234
  const dirExists = await stat8(configuredStaticPath).then(
16232
16235
  (s) => s.isDirectory(),
16233
16236
  () => false
@@ -16292,14 +16295,28 @@ async function runEvaluation(options) {
16292
16295
  } catch {
16293
16296
  }
16294
16297
  }
16295
- const needsRepoMaterialisation = !!suiteWorkspace?.repos?.length && !usePool && (!useStaticWorkspace || staticMaterialised);
16296
- const repoManager = needsRepoMaterialisation ? new RepoManager(verbose) : void 0;
16297
- if (repoManager && sharedWorkspacePath && suiteWorkspace?.repos && !isPerTestIsolation) {
16298
- setupLog(
16299
- `materializing ${suiteWorkspace.repos.length} shared repo(s) into ${sharedWorkspacePath}`
16300
- );
16298
+ const hasReposToMaterialize = !!suiteWorkspace?.repos?.length && !usePool && !isPerTestIsolation;
16299
+ const needsRepoMaterialisation = hasReposToMaterialize && (!useStaticWorkspace || staticMaterialised);
16300
+ const needsPerRepoCheck = hasReposToMaterialize && useStaticWorkspace && !staticMaterialised && isYamlConfiguredPath;
16301
+ const repoManager = needsRepoMaterialisation || needsPerRepoCheck ? new RepoManager(verbose) : void 0;
16302
+ if (repoManager && sharedWorkspacePath && suiteWorkspace?.repos) {
16301
16303
  try {
16302
- await repoManager.materializeAll(suiteWorkspace.repos, sharedWorkspacePath);
16304
+ if (needsPerRepoCheck) {
16305
+ for (const repo of suiteWorkspace.repos) {
16306
+ const targetDir = path44.join(sharedWorkspacePath, repo.path);
16307
+ if (existsSync5(targetDir)) {
16308
+ setupLog(`reusing existing repo at: ${targetDir}`);
16309
+ continue;
16310
+ }
16311
+ setupLog(`materializing missing repo: ${repo.path}`);
16312
+ await repoManager.materialize(repo, sharedWorkspacePath);
16313
+ }
16314
+ } else {
16315
+ setupLog(
16316
+ `materializing ${suiteWorkspace.repos.length} shared repo(s) into ${sharedWorkspacePath}`
16317
+ );
16318
+ await repoManager.materializeAll(suiteWorkspace.repos, sharedWorkspacePath);
16319
+ }
16303
16320
  setupLog("shared repo materialization complete");
16304
16321
  } catch (error) {
16305
16322
  const message = error instanceof Error ? error.message : String(error);
@@ -17973,7 +17990,7 @@ function computeWeightedMean(entries) {
17973
17990
  }
17974
17991
 
17975
17992
  // src/evaluation/evaluate.ts
17976
- import { existsSync as existsSync5 } from "node:fs";
17993
+ import { existsSync as existsSync6 } from "node:fs";
17977
17994
  import path45 from "node:path";
17978
17995
 
17979
17996
  // src/evaluation/providers/function-provider.ts
@@ -18130,7 +18147,7 @@ async function discoverDefaultTarget(repoRoot) {
18130
18147
  for (const dir of chain) {
18131
18148
  for (const candidate of TARGET_FILE_CANDIDATES) {
18132
18149
  const targetsPath = path45.join(dir, candidate);
18133
- if (!existsSync5(targetsPath)) continue;
18150
+ if (!existsSync6(targetsPath)) continue;
18134
18151
  try {
18135
18152
  const definitions = await readTargetDefinitions(targetsPath);
18136
18153
  const defaultTarget = definitions.find((d) => d.name === "default");
@@ -18147,7 +18164,7 @@ async function loadEnvHierarchy(repoRoot, startPath) {
18147
18164
  const envFiles = [];
18148
18165
  for (const dir of chain) {
18149
18166
  const envPath = path45.join(dir, ".env");
18150
- if (existsSync5(envPath)) envFiles.push(envPath);
18167
+ if (existsSync6(envPath)) envFiles.push(envPath);
18151
18168
  }
18152
18169
  for (let i = 0; i < envFiles.length; i++) {
18153
18170
  try {
@@ -18223,12 +18240,12 @@ var CONFIG_FILE_NAMES = [
18223
18240
  ".agentv/config.js"
18224
18241
  ];
18225
18242
  async function loadTsConfig(projectRoot) {
18226
- const { existsSync: existsSync7 } = await import("node:fs");
18243
+ const { existsSync: existsSync8 } = await import("node:fs");
18227
18244
  const { pathToFileURL: pathToFileURL2 } = await import("node:url");
18228
18245
  const { join: join2 } = await import("node:path");
18229
18246
  for (const fileName of CONFIG_FILE_NAMES) {
18230
18247
  const filePath = join2(projectRoot, fileName);
18231
- if (!existsSync7(filePath)) {
18248
+ if (!existsSync8(filePath)) {
18232
18249
  continue;
18233
18250
  }
18234
18251
  try {
@@ -18324,9 +18341,106 @@ function buildPrompt(criteria, question, referenceAnswer) {
18324
18341
  return parts.join("\n");
18325
18342
  }
18326
18343
 
18327
- // src/evaluation/cache/response-cache.ts
18328
- import { mkdir as mkdir15, readFile as readFile13, writeFile as writeFile8 } from "node:fs/promises";
18344
+ // src/evaluation/workspace/deps-scanner.ts
18345
+ import { readFile as readFile13 } from "node:fs/promises";
18329
18346
  import path46 from "node:path";
18347
+ import { parse as parse5 } from "yaml";
18348
+ function normalizeGitUrl(url) {
18349
+ let normalized = url.replace(/\.git$/, "");
18350
+ try {
18351
+ const parsed = new URL(normalized);
18352
+ parsed.hostname = parsed.hostname.toLowerCase();
18353
+ normalized = parsed.toString().replace(/\/$/, "");
18354
+ } catch {
18355
+ }
18356
+ return normalized;
18357
+ }
18358
+ async function scanRepoDeps(evalFilePaths) {
18359
+ const seen = /* @__PURE__ */ new Map();
18360
+ const errors = [];
18361
+ for (const filePath of evalFilePaths) {
18362
+ try {
18363
+ const repos = await extractReposFromEvalFile(filePath);
18364
+ for (const repo of repos) {
18365
+ if (repo.source.type !== "git") continue;
18366
+ const ref = repo.checkout?.ref;
18367
+ const key = `${normalizeGitUrl(repo.source.url)}\0${ref ?? ""}`;
18368
+ const existing = seen.get(key);
18369
+ if (existing) {
18370
+ existing.usedBy.push(filePath);
18371
+ } else {
18372
+ const { ref: _ref, ...checkoutRest } = repo.checkout ?? {};
18373
+ const hasCheckout = Object.keys(checkoutRest).length > 0;
18374
+ seen.set(key, {
18375
+ url: repo.source.url,
18376
+ ref,
18377
+ clone: repo.clone,
18378
+ checkout: hasCheckout ? checkoutRest : void 0,
18379
+ usedBy: [filePath]
18380
+ });
18381
+ }
18382
+ }
18383
+ } catch (err) {
18384
+ errors.push({
18385
+ file: filePath,
18386
+ message: err instanceof Error ? err.message : String(err)
18387
+ });
18388
+ }
18389
+ }
18390
+ return { repos: [...seen.values()], errors };
18391
+ }
18392
+ async function extractReposFromEvalFile(filePath) {
18393
+ const content = await readFile13(filePath, "utf8");
18394
+ const parsed = interpolateEnv(parse5(content), process.env);
18395
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
18396
+ const obj = parsed;
18397
+ const evalFileDir = path46.dirname(path46.resolve(filePath));
18398
+ const repos = [];
18399
+ const suiteRepos = await extractReposFromWorkspaceRaw(obj.workspace, evalFileDir);
18400
+ repos.push(...suiteRepos);
18401
+ const tests = Array.isArray(obj.tests) ? obj.tests : [];
18402
+ for (const test of tests) {
18403
+ if (test && typeof test === "object" && !Array.isArray(test)) {
18404
+ const testObj = test;
18405
+ const testRepos = await extractReposFromWorkspaceRaw(testObj.workspace, evalFileDir);
18406
+ repos.push(...testRepos);
18407
+ }
18408
+ }
18409
+ return repos;
18410
+ }
18411
+ async function extractReposFromWorkspaceRaw(raw, evalFileDir) {
18412
+ if (typeof raw === "string") {
18413
+ const workspaceFilePath = path46.resolve(evalFileDir, raw);
18414
+ const content = await readFile13(workspaceFilePath, "utf8");
18415
+ const parsed = interpolateEnv(parse5(content), process.env);
18416
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
18417
+ return extractReposFromObject(parsed);
18418
+ }
18419
+ if (raw && typeof raw === "object" && !Array.isArray(raw)) {
18420
+ return extractReposFromObject(raw);
18421
+ }
18422
+ return [];
18423
+ }
18424
+ function extractReposFromObject(obj) {
18425
+ const rawRepos = Array.isArray(obj.repos) ? obj.repos : [];
18426
+ const result = [];
18427
+ for (const r of rawRepos) {
18428
+ if (!r || typeof r !== "object" || Array.isArray(r)) continue;
18429
+ const repo = r;
18430
+ const source = parseRepoSource(repo.source);
18431
+ if (!source) continue;
18432
+ result.push({
18433
+ source,
18434
+ checkout: parseRepoCheckout(repo.checkout),
18435
+ clone: parseRepoClone(repo.clone)
18436
+ });
18437
+ }
18438
+ return result;
18439
+ }
18440
+
18441
+ // src/evaluation/cache/response-cache.ts
18442
+ import { mkdir as mkdir15, readFile as readFile14, writeFile as writeFile8 } from "node:fs/promises";
18443
+ import path47 from "node:path";
18330
18444
  var DEFAULT_CACHE_PATH = ".agentv/cache";
18331
18445
  var ResponseCache = class {
18332
18446
  cachePath;
@@ -18336,7 +18450,7 @@ var ResponseCache = class {
18336
18450
  async get(key) {
18337
18451
  const filePath = this.keyToPath(key);
18338
18452
  try {
18339
- const data = await readFile13(filePath, "utf8");
18453
+ const data = await readFile14(filePath, "utf8");
18340
18454
  return JSON.parse(data);
18341
18455
  } catch {
18342
18456
  return void 0;
@@ -18344,13 +18458,13 @@ var ResponseCache = class {
18344
18458
  }
18345
18459
  async set(key, value) {
18346
18460
  const filePath = this.keyToPath(key);
18347
- const dir = path46.dirname(filePath);
18461
+ const dir = path47.dirname(filePath);
18348
18462
  await mkdir15(dir, { recursive: true });
18349
18463
  await writeFile8(filePath, JSON.stringify(value, null, 2), "utf8");
18350
18464
  }
18351
18465
  keyToPath(key) {
18352
18466
  const prefix = key.slice(0, 2);
18353
- return path46.join(this.cachePath, prefix, `${key}.json`);
18467
+ return path47.join(this.cachePath, prefix, `${key}.json`);
18354
18468
  }
18355
18469
  };
18356
18470
  function shouldEnableCache(params) {
@@ -18366,15 +18480,15 @@ function shouldSkipCacheForTemperature(targetConfig) {
18366
18480
  }
18367
18481
 
18368
18482
  // src/projects.ts
18369
- import { existsSync as existsSync6, mkdirSync as mkdirSync2, readFileSync as readFileSync3, readdirSync as readdirSync3, statSync as statSync2, writeFileSync } from "node:fs";
18370
- import path47 from "node:path";
18483
+ import { existsSync as existsSync7, mkdirSync as mkdirSync2, readFileSync as readFileSync3, readdirSync as readdirSync3, statSync as statSync2, writeFileSync } from "node:fs";
18484
+ import path48 from "node:path";
18371
18485
  import { parse as parseYaml3, stringify as stringifyYaml } from "yaml";
18372
18486
  function getProjectsRegistryPath() {
18373
- return path47.join(getAgentvHome(), "projects.yaml");
18487
+ return path48.join(getAgentvHome(), "projects.yaml");
18374
18488
  }
18375
18489
  function loadProjectRegistry() {
18376
18490
  const registryPath = getProjectsRegistryPath();
18377
- if (!existsSync6(registryPath)) {
18491
+ if (!existsSync7(registryPath)) {
18378
18492
  return { projects: [] };
18379
18493
  }
18380
18494
  try {
@@ -18390,14 +18504,14 @@ function loadProjectRegistry() {
18390
18504
  }
18391
18505
  function saveProjectRegistry(registry) {
18392
18506
  const registryPath = getProjectsRegistryPath();
18393
- const dir = path47.dirname(registryPath);
18394
- if (!existsSync6(dir)) {
18507
+ const dir = path48.dirname(registryPath);
18508
+ if (!existsSync7(dir)) {
18395
18509
  mkdirSync2(dir, { recursive: true });
18396
18510
  }
18397
18511
  writeFileSync(registryPath, stringifyYaml(registry), "utf-8");
18398
18512
  }
18399
18513
  function deriveProjectId(dirPath, existingIds) {
18400
- const base = path47.basename(dirPath).toLowerCase().replace(/[^a-z0-9-]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
18514
+ const base = path48.basename(dirPath).toLowerCase().replace(/[^a-z0-9-]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
18401
18515
  let candidate = base || "project";
18402
18516
  let suffix = 2;
18403
18517
  while (existingIds.includes(candidate)) {
@@ -18407,11 +18521,11 @@ function deriveProjectId(dirPath, existingIds) {
18407
18521
  return candidate;
18408
18522
  }
18409
18523
  function addProject(projectPath) {
18410
- const absPath = path47.resolve(projectPath);
18411
- if (!existsSync6(absPath)) {
18524
+ const absPath = path48.resolve(projectPath);
18525
+ if (!existsSync7(absPath)) {
18412
18526
  throw new Error(`Directory not found: ${absPath}`);
18413
18527
  }
18414
- if (!existsSync6(path47.join(absPath, ".agentv"))) {
18528
+ if (!existsSync7(path48.join(absPath, ".agentv"))) {
18415
18529
  throw new Error(`No .agentv/ directory found in ${absPath}. Run an evaluation first.`);
18416
18530
  }
18417
18531
  const registry = loadProjectRegistry();
@@ -18425,7 +18539,7 @@ function addProject(projectPath) {
18425
18539
  absPath,
18426
18540
  registry.projects.map((p) => p.id)
18427
18541
  ),
18428
- name: path47.basename(absPath),
18542
+ name: path48.basename(absPath),
18429
18543
  path: absPath,
18430
18544
  addedAt: now,
18431
18545
  lastOpenedAt: now
@@ -18454,14 +18568,14 @@ function touchProject(projectId) {
18454
18568
  }
18455
18569
  }
18456
18570
  function discoverProjects(rootDir, maxDepth = 2) {
18457
- const absRoot = path47.resolve(rootDir);
18458
- if (!existsSync6(absRoot) || !statSync2(absRoot).isDirectory()) {
18571
+ const absRoot = path48.resolve(rootDir);
18572
+ if (!existsSync7(absRoot) || !statSync2(absRoot).isDirectory()) {
18459
18573
  return [];
18460
18574
  }
18461
18575
  const results = [];
18462
18576
  function scan(dir, depth) {
18463
18577
  if (depth > maxDepth) return;
18464
- if (existsSync6(path47.join(dir, ".agentv"))) {
18578
+ if (existsSync7(path48.join(dir, ".agentv"))) {
18465
18579
  results.push(dir);
18466
18580
  return;
18467
18581
  }
@@ -18471,7 +18585,7 @@ function discoverProjects(rootDir, maxDepth = 2) {
18471
18585
  for (const entry of entries) {
18472
18586
  if (!entry.isDirectory()) continue;
18473
18587
  if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
18474
- scan(path47.join(dir, entry.name), depth + 1);
18588
+ scan(path48.join(dir, entry.name), depth + 1);
18475
18589
  }
18476
18590
  } catch {
18477
18591
  }
@@ -19384,8 +19498,8 @@ function extractResponseItemContent(content) {
19384
19498
  // src/import/codex-session-discovery.ts
19385
19499
  import { readdir as readdir8, stat as stat9 } from "node:fs/promises";
19386
19500
  import { homedir as homedir3 } from "node:os";
19387
- import path48 from "node:path";
19388
- var DEFAULT_SESSIONS_DIR = () => path48.join(homedir3(), ".codex", "sessions");
19501
+ import path49 from "node:path";
19502
+ var DEFAULT_SESSIONS_DIR = () => path49.join(homedir3(), ".codex", "sessions");
19389
19503
  async function discoverCodexSessions(opts) {
19390
19504
  const sessionsDir = opts?.sessionsDir ?? DEFAULT_SESSIONS_DIR();
19391
19505
  const limit = opts?.latest ? 1 : opts?.limit ?? 10;
@@ -19397,7 +19511,7 @@ async function discoverCodexSessions(opts) {
19397
19511
  return [];
19398
19512
  }
19399
19513
  for (const year of yearDirs) {
19400
- const yearPath = path48.join(sessionsDir, year);
19514
+ const yearPath = path49.join(sessionsDir, year);
19401
19515
  let monthDirs;
19402
19516
  try {
19403
19517
  monthDirs = await readdir8(yearPath);
@@ -19405,7 +19519,7 @@ async function discoverCodexSessions(opts) {
19405
19519
  continue;
19406
19520
  }
19407
19521
  for (const month of monthDirs) {
19408
- const monthPath = path48.join(yearPath, month);
19522
+ const monthPath = path49.join(yearPath, month);
19409
19523
  let dayDirs;
19410
19524
  try {
19411
19525
  dayDirs = await readdir8(monthPath);
@@ -19417,7 +19531,7 @@ async function discoverCodexSessions(opts) {
19417
19531
  const dirDate = `${year}-${month}-${day}`;
19418
19532
  if (dirDate !== opts.date) continue;
19419
19533
  }
19420
- const dayPath = path48.join(monthPath, day);
19534
+ const dayPath = path49.join(monthPath, day);
19421
19535
  let files;
19422
19536
  try {
19423
19537
  files = await readdir8(dayPath);
@@ -19426,7 +19540,7 @@ async function discoverCodexSessions(opts) {
19426
19540
  }
19427
19541
  for (const file of files) {
19428
19542
  if (!file.startsWith("rollout-") || !file.endsWith(".jsonl")) continue;
19429
- const filePath = path48.join(dayPath, file);
19543
+ const filePath = path49.join(dayPath, file);
19430
19544
  const nameWithoutExt = file.replace(/\.jsonl$/, "");
19431
19545
  const parts = nameWithoutExt.split("-");
19432
19546
  const sessionId = parts.length >= 6 ? parts.slice(-5).join("-") : nameWithoutExt;
@@ -19449,8 +19563,8 @@ async function discoverCodexSessions(opts) {
19449
19563
  // src/import/session-discovery.ts
19450
19564
  import { readdir as readdir9, stat as stat10 } from "node:fs/promises";
19451
19565
  import { homedir as homedir4 } from "node:os";
19452
- import path49 from "node:path";
19453
- var DEFAULT_PROJECTS_DIR = () => path49.join(homedir4(), ".claude", "projects");
19566
+ import path50 from "node:path";
19567
+ var DEFAULT_PROJECTS_DIR = () => path50.join(homedir4(), ".claude", "projects");
19454
19568
  function encodeProjectPath(projectPath) {
19455
19569
  return projectPath.replace(/\//g, "-");
19456
19570
  }
@@ -19469,7 +19583,7 @@ async function discoverClaudeSessions(opts) {
19469
19583
  }
19470
19584
  const sessions = [];
19471
19585
  for (const projectDir of projectDirs) {
19472
- const dirPath = path49.join(projectsDir, projectDir);
19586
+ const dirPath = path50.join(projectsDir, projectDir);
19473
19587
  let entries;
19474
19588
  try {
19475
19589
  entries = await readdir9(dirPath);
@@ -19480,7 +19594,7 @@ async function discoverClaudeSessions(opts) {
19480
19594
  if (!entry.endsWith(".jsonl")) continue;
19481
19595
  const sessionId = entry.replace(/\.jsonl$/, "");
19482
19596
  if (opts?.sessionId && sessionId !== opts.sessionId) continue;
19483
- const filePath = path49.join(dirPath, entry);
19597
+ const filePath = path50.join(dirPath, entry);
19484
19598
  let updatedAt;
19485
19599
  try {
19486
19600
  const fileStat = await stat10(filePath);
@@ -19501,7 +19615,7 @@ async function discoverClaudeSessions(opts) {
19501
19615
  }
19502
19616
 
19503
19617
  // src/import/types.ts
19504
- import { readFile as readFile14 } from "node:fs/promises";
19618
+ import { readFile as readFile15 } from "node:fs/promises";
19505
19619
  function toTranscriptJsonLine(entry) {
19506
19620
  const firstUserMessage = entry.messages.find((m) => m.role === "user");
19507
19621
  const input = typeof firstUserMessage?.content === "string" ? firstUserMessage.content : "";
@@ -19527,11 +19641,11 @@ function toTranscriptJsonLine(entry) {
19527
19641
  };
19528
19642
  }
19529
19643
  async function readTranscriptJsonl(filePath) {
19530
- const text = await readFile14(filePath, "utf8");
19644
+ const text = await readFile15(filePath, "utf8");
19531
19645
  return text.split("\n").filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
19532
19646
  }
19533
19647
  async function readTranscriptFile(filePath) {
19534
- return readFile14(filePath, "utf8");
19648
+ return readFile15(filePath, "utf8");
19535
19649
  }
19536
19650
 
19537
19651
  // src/import/transcript-provider.ts
@@ -19745,6 +19859,7 @@ export {
19745
19859
  runRegexAssertion,
19746
19860
  runStartsWithAssertion,
19747
19861
  saveProjectRegistry,
19862
+ scanRepoDeps,
19748
19863
  scoreToVerdict,
19749
19864
  shouldEnableCache,
19750
19865
  shouldSkipCacheForTemperature,