@agentv/core 4.8.0 → 4.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -1355,13 +1355,13 @@ function serializeAttributeValue(value) {
1355
1355
  if (Array.isArray(value)) return { arrayValue: { values: value.map(serializeAttributeValue) } };
1356
1356
  return { stringValue: String(value) };
1357
1357
  }
1358
- var import_promises35, import_node_path52, OtlpJsonFileExporter;
1358
+ var import_promises36, import_node_path53, OtlpJsonFileExporter;
1359
1359
  var init_otlp_json_file_exporter = __esm({
1360
1360
  "src/observability/otlp-json-file-exporter.ts"() {
1361
1361
  "use strict";
1362
1362
  init_cjs_shims();
1363
- import_promises35 = require("fs/promises");
1364
- import_node_path52 = require("path");
1363
+ import_promises36 = require("fs/promises");
1364
+ import_node_path53 = require("path");
1365
1365
  OtlpJsonFileExporter = class {
1366
1366
  // biome-ignore lint/suspicious/noExplicitAny: serialized span data
1367
1367
  spans = [];
@@ -1400,7 +1400,7 @@ var init_otlp_json_file_exporter = __esm({
1400
1400
  }
1401
1401
  async flush() {
1402
1402
  if (this.spans.length === 0) return;
1403
- await (0, import_promises35.mkdir)((0, import_node_path52.dirname)(this.filePath), { recursive: true });
1403
+ await (0, import_promises36.mkdir)((0, import_node_path53.dirname)(this.filePath), { recursive: true });
1404
1404
  const otlpJson = {
1405
1405
  resourceSpans: [
1406
1406
  {
@@ -1583,6 +1583,7 @@ __export(index_exports, {
1583
1583
  runRegexAssertion: () => runRegexAssertion,
1584
1584
  runStartsWithAssertion: () => runStartsWithAssertion,
1585
1585
  saveProjectRegistry: () => saveProjectRegistry,
1586
+ scanRepoDeps: () => scanRepoDeps,
1586
1587
  scoreToVerdict: () => scoreToVerdict,
1587
1588
  shouldEnableCache: () => shouldEnableCache,
1588
1589
  shouldSkipCacheForTemperature: () => shouldSkipCacheForTemperature,
@@ -4741,6 +4742,61 @@ function parseMetadata(suite) {
4741
4742
  });
4742
4743
  }
4743
4744
 
4745
+ // src/evaluation/workspace/repo-config-parser.ts
4746
+ init_cjs_shims();
4747
+ function parseRepoSource(raw) {
4748
+ if (!isJsonObject(raw)) return void 0;
4749
+ const obj = raw;
4750
+ if (obj.type === "git" && typeof obj.url === "string") {
4751
+ return { type: "git", url: obj.url };
4752
+ }
4753
+ if (obj.type === "local" && typeof obj.path === "string") {
4754
+ return { type: "local", path: obj.path };
4755
+ }
4756
+ return void 0;
4757
+ }
4758
+ function parseRepoCheckout(raw) {
4759
+ if (!isJsonObject(raw)) return void 0;
4760
+ const obj = raw;
4761
+ const ref = typeof obj.ref === "string" ? obj.ref : void 0;
4762
+ const resolve = obj.resolve === "remote" || obj.resolve === "local" ? obj.resolve : void 0;
4763
+ const ancestor = typeof obj.ancestor === "number" ? obj.ancestor : void 0;
4764
+ if (!ref && !resolve && ancestor === void 0) return void 0;
4765
+ return {
4766
+ ...ref !== void 0 && { ref },
4767
+ ...resolve !== void 0 && { resolve },
4768
+ ...ancestor !== void 0 && { ancestor }
4769
+ };
4770
+ }
4771
+ function parseRepoClone(raw) {
4772
+ if (!isJsonObject(raw)) return void 0;
4773
+ const obj = raw;
4774
+ const depth = typeof obj.depth === "number" ? obj.depth : void 0;
4775
+ const filter = typeof obj.filter === "string" ? obj.filter : void 0;
4776
+ const sparse = Array.isArray(obj.sparse) ? obj.sparse.filter((s) => typeof s === "string") : void 0;
4777
+ if (depth === void 0 && !filter && !sparse) return void 0;
4778
+ return {
4779
+ ...depth !== void 0 && { depth },
4780
+ ...filter !== void 0 && { filter },
4781
+ ...sparse !== void 0 && { sparse }
4782
+ };
4783
+ }
4784
+ function parseRepoConfig(raw) {
4785
+ if (!isJsonObject(raw)) return void 0;
4786
+ const obj = raw;
4787
+ const repoPath = typeof obj.path === "string" ? obj.path : void 0;
4788
+ const source = parseRepoSource(obj.source);
4789
+ if (!repoPath || !source) return void 0;
4790
+ const checkout = parseRepoCheckout(obj.checkout);
4791
+ const clone = parseRepoClone(obj.clone);
4792
+ return {
4793
+ path: repoPath,
4794
+ source,
4795
+ ...checkout !== void 0 && { checkout },
4796
+ ...clone !== void 0 && { clone }
4797
+ };
4798
+ }
4799
+
4744
4800
  // src/evaluation/formatting/prompt-builder.ts
4745
4801
  init_cjs_shims();
4746
4802
  async function buildPromptInputs(testCase, mode = "lm") {
@@ -5136,58 +5192,6 @@ function parseWorkspaceScriptConfig(raw, evalFileDir) {
5136
5192
  }
5137
5193
  return cwd ? { ...config, cwd } : config;
5138
5194
  }
5139
- function parseRepoSource(raw) {
5140
- if (!isJsonObject(raw)) return void 0;
5141
- const obj = raw;
5142
- if (obj.type === "git" && typeof obj.url === "string") {
5143
- return { type: "git", url: obj.url };
5144
- }
5145
- if (obj.type === "local" && typeof obj.path === "string") {
5146
- return { type: "local", path: obj.path };
5147
- }
5148
- return void 0;
5149
- }
5150
- function parseRepoCheckout(raw) {
5151
- if (!isJsonObject(raw)) return void 0;
5152
- const obj = raw;
5153
- const ref = typeof obj.ref === "string" ? obj.ref : void 0;
5154
- const resolve = obj.resolve === "remote" || obj.resolve === "local" ? obj.resolve : void 0;
5155
- const ancestor = typeof obj.ancestor === "number" ? obj.ancestor : void 0;
5156
- if (!ref && !resolve && ancestor === void 0) return void 0;
5157
- return {
5158
- ...ref !== void 0 && { ref },
5159
- ...resolve !== void 0 && { resolve },
5160
- ...ancestor !== void 0 && { ancestor }
5161
- };
5162
- }
5163
- function parseRepoClone(raw) {
5164
- if (!isJsonObject(raw)) return void 0;
5165
- const obj = raw;
5166
- const depth = typeof obj.depth === "number" ? obj.depth : void 0;
5167
- const filter = typeof obj.filter === "string" ? obj.filter : void 0;
5168
- const sparse = Array.isArray(obj.sparse) ? obj.sparse.filter((s) => typeof s === "string") : void 0;
5169
- if (depth === void 0 && !filter && !sparse) return void 0;
5170
- return {
5171
- ...depth !== void 0 && { depth },
5172
- ...filter !== void 0 && { filter },
5173
- ...sparse !== void 0 && { sparse }
5174
- };
5175
- }
5176
- function parseRepoConfig(raw) {
5177
- if (!isJsonObject(raw)) return void 0;
5178
- const obj = raw;
5179
- const repoPath = typeof obj.path === "string" ? obj.path : void 0;
5180
- const source = parseRepoSource(obj.source);
5181
- if (!repoPath || !source) return void 0;
5182
- const checkout = parseRepoCheckout(obj.checkout);
5183
- const clone = parseRepoClone(obj.clone);
5184
- return {
5185
- path: repoPath,
5186
- source,
5187
- ...checkout !== void 0 && { checkout },
5188
- ...clone !== void 0 && { clone }
5189
- };
5190
- }
5191
5195
  function parseWorkspaceHookConfig(raw, evalFileDir) {
5192
5196
  if (!isJsonObject(raw)) return void 0;
5193
5197
  const script = parseWorkspaceScriptConfig(raw, evalFileDir);
@@ -12291,8 +12295,8 @@ function resolveCliConfig(target, env, evalFilePath) {
12291
12295
  const parseResult = CliTargetInputSchema.safeParse(target, { errorMap: cliErrorMap });
12292
12296
  if (!parseResult.success) {
12293
12297
  const firstError = parseResult.error.errors[0];
12294
- const path53 = firstError?.path.join(".") || "";
12295
- const prefix = path53 ? `${target.name} ${path53}: ` : `${target.name}: `;
12298
+ const path54 = firstError?.path.join(".") || "";
12299
+ const prefix = path54 ? `${target.name} ${path54}: ` : `${target.name}: `;
12296
12300
  throw new Error(`${prefix}${firstError?.message}`);
12297
12301
  }
12298
12302
  const normalized = normalizeCliTargetInput(parseResult.data, env, evalFilePath);
@@ -14337,15 +14341,15 @@ async function execFileWithStdinNode(argv, stdinPayload, options) {
14337
14341
  });
14338
14342
  }
14339
14343
  async function execShellWithStdin(command, stdinPayload, options = {}) {
14340
- const { mkdir: mkdir17, readFile: readFile17, rm: rm6, writeFile: writeFile9 } = await import("fs/promises");
14344
+ const { mkdir: mkdir17, readFile: readFile18, rm: rm6, writeFile: writeFile9 } = await import("fs/promises");
14341
14345
  const { tmpdir: tmpdir3 } = await import("os");
14342
- const path53 = await import("path");
14346
+ const path54 = await import("path");
14343
14347
  const { randomUUID: randomUUID10 } = await import("crypto");
14344
- const dir = path53.join(tmpdir3(), `agentv-exec-${randomUUID10()}`);
14348
+ const dir = path54.join(tmpdir3(), `agentv-exec-${randomUUID10()}`);
14345
14349
  await mkdir17(dir, { recursive: true });
14346
- const stdinPath = path53.join(dir, "stdin.txt");
14347
- const stdoutPath = path53.join(dir, "stdout.txt");
14348
- const stderrPath = path53.join(dir, "stderr.txt");
14350
+ const stdinPath = path54.join(dir, "stdin.txt");
14351
+ const stdoutPath = path54.join(dir, "stdout.txt");
14352
+ const stderrPath = path54.join(dir, "stderr.txt");
14349
14353
  await writeFile9(stdinPath, stdinPayload, "utf8");
14350
14354
  const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
14351
14355
  const { spawn: spawn5 } = await import("child_process");
@@ -14375,8 +14379,8 @@ async function execShellWithStdin(command, stdinPayload, options = {}) {
14375
14379
  resolve(code ?? 0);
14376
14380
  });
14377
14381
  });
14378
- const stdout = (await readFile17(stdoutPath, "utf8")).replace(/\r\n/g, "\n");
14379
- const stderr = (await readFile17(stderrPath, "utf8")).replace(/\r\n/g, "\n");
14382
+ const stdout = (await readFile18(stdoutPath, "utf8")).replace(/\r\n/g, "\n");
14383
+ const stderr = (await readFile18(stderrPath, "utf8")).replace(/\r\n/g, "\n");
14380
14384
  return { stdout, stderr, exitCode };
14381
14385
  } finally {
14382
14386
  await rm6(dir, { recursive: true, force: true });
@@ -16642,115 +16646,115 @@ var FieldAccuracyEvaluator = class {
16642
16646
  * Evaluate a single field against the expected value.
16643
16647
  */
16644
16648
  evaluateField(fieldConfig, candidateData, expectedData) {
16645
- const { path: path53, match, required = true, weight = 1 } = fieldConfig;
16646
- const candidateValue = resolvePath(candidateData, path53);
16647
- const expectedValue = resolvePath(expectedData, path53);
16649
+ const { path: path54, match, required = true, weight = 1 } = fieldConfig;
16650
+ const candidateValue = resolvePath(candidateData, path54);
16651
+ const expectedValue = resolvePath(expectedData, path54);
16648
16652
  if (expectedValue === void 0) {
16649
16653
  return {
16650
- path: path53,
16654
+ path: path54,
16651
16655
  score: 1,
16652
16656
  // No expected value means no comparison needed
16653
16657
  weight,
16654
16658
  hit: true,
16655
- message: `${path53}: no expected value`
16659
+ message: `${path54}: no expected value`
16656
16660
  };
16657
16661
  }
16658
16662
  if (candidateValue === void 0) {
16659
16663
  if (required) {
16660
16664
  return {
16661
- path: path53,
16665
+ path: path54,
16662
16666
  score: 0,
16663
16667
  weight,
16664
16668
  hit: false,
16665
- message: `${path53} (required, missing)`
16669
+ message: `${path54} (required, missing)`
16666
16670
  };
16667
16671
  }
16668
16672
  return {
16669
- path: path53,
16673
+ path: path54,
16670
16674
  score: 1,
16671
16675
  // Don't penalize missing optional fields
16672
16676
  weight: 0,
16673
16677
  // Zero weight means it won't affect the score
16674
16678
  hit: true,
16675
- message: `${path53}: optional field missing`
16679
+ message: `${path54}: optional field missing`
16676
16680
  };
16677
16681
  }
16678
16682
  switch (match) {
16679
16683
  case "exact":
16680
- return this.compareExact(path53, candidateValue, expectedValue, weight);
16684
+ return this.compareExact(path54, candidateValue, expectedValue, weight);
16681
16685
  case "numeric_tolerance":
16682
16686
  return this.compareNumericTolerance(
16683
- path53,
16687
+ path54,
16684
16688
  candidateValue,
16685
16689
  expectedValue,
16686
16690
  fieldConfig,
16687
16691
  weight
16688
16692
  );
16689
16693
  case "date":
16690
- return this.compareDate(path53, candidateValue, expectedValue, fieldConfig, weight);
16694
+ return this.compareDate(path54, candidateValue, expectedValue, fieldConfig, weight);
16691
16695
  default:
16692
16696
  return {
16693
- path: path53,
16697
+ path: path54,
16694
16698
  score: 0,
16695
16699
  weight,
16696
16700
  hit: false,
16697
- message: `${path53}: unknown match type "${match}"`
16701
+ message: `${path54}: unknown match type "${match}"`
16698
16702
  };
16699
16703
  }
16700
16704
  }
16701
16705
  /**
16702
16706
  * Exact equality comparison.
16703
16707
  */
16704
- compareExact(path53, candidateValue, expectedValue, weight) {
16708
+ compareExact(path54, candidateValue, expectedValue, weight) {
16705
16709
  if (deepEqual(candidateValue, expectedValue)) {
16706
16710
  return {
16707
- path: path53,
16711
+ path: path54,
16708
16712
  score: 1,
16709
16713
  weight,
16710
16714
  hit: true,
16711
- message: path53
16715
+ message: path54
16712
16716
  };
16713
16717
  }
16714
16718
  if (typeof candidateValue !== typeof expectedValue) {
16715
16719
  return {
16716
- path: path53,
16720
+ path: path54,
16717
16721
  score: 0,
16718
16722
  weight,
16719
16723
  hit: false,
16720
- message: `${path53} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
16724
+ message: `${path54} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
16721
16725
  };
16722
16726
  }
16723
16727
  return {
16724
- path: path53,
16728
+ path: path54,
16725
16729
  score: 0,
16726
16730
  weight,
16727
16731
  hit: false,
16728
- message: `${path53} (value mismatch)`
16732
+ message: `${path54} (value mismatch)`
16729
16733
  };
16730
16734
  }
16731
16735
  /**
16732
16736
  * Numeric comparison with absolute or relative tolerance.
16733
16737
  */
16734
- compareNumericTolerance(path53, candidateValue, expectedValue, fieldConfig, weight) {
16738
+ compareNumericTolerance(path54, candidateValue, expectedValue, fieldConfig, weight) {
16735
16739
  const { tolerance = 0, relative = false } = fieldConfig;
16736
16740
  const candidateNum = toNumber(candidateValue);
16737
16741
  const expectedNum = toNumber(expectedValue);
16738
16742
  if (candidateNum === null || expectedNum === null) {
16739
16743
  return {
16740
- path: path53,
16744
+ path: path54,
16741
16745
  score: 0,
16742
16746
  weight,
16743
16747
  hit: false,
16744
- message: `${path53} (non-numeric value)`
16748
+ message: `${path54} (non-numeric value)`
16745
16749
  };
16746
16750
  }
16747
16751
  if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
16748
16752
  return {
16749
- path: path53,
16753
+ path: path54,
16750
16754
  score: 0,
16751
16755
  weight,
16752
16756
  hit: false,
16753
- message: `${path53} (invalid numeric value)`
16757
+ message: `${path54} (invalid numeric value)`
16754
16758
  };
16755
16759
  }
16756
16760
  const diff = Math.abs(candidateNum - expectedNum);
@@ -16763,61 +16767,61 @@ var FieldAccuracyEvaluator = class {
16763
16767
  }
16764
16768
  if (withinTolerance) {
16765
16769
  return {
16766
- path: path53,
16770
+ path: path54,
16767
16771
  score: 1,
16768
16772
  weight,
16769
16773
  hit: true,
16770
- message: `${path53} (within tolerance: diff=${diff.toFixed(2)})`
16774
+ message: `${path54} (within tolerance: diff=${diff.toFixed(2)})`
16771
16775
  };
16772
16776
  }
16773
16777
  return {
16774
- path: path53,
16778
+ path: path54,
16775
16779
  score: 0,
16776
16780
  weight,
16777
16781
  hit: false,
16778
- message: `${path53} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
16782
+ message: `${path54} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
16779
16783
  };
16780
16784
  }
16781
16785
  /**
16782
16786
  * Date comparison with format normalization.
16783
16787
  */
16784
- compareDate(path53, candidateValue, expectedValue, fieldConfig, weight) {
16788
+ compareDate(path54, candidateValue, expectedValue, fieldConfig, weight) {
16785
16789
  const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
16786
16790
  const candidateDate = parseDate(String(candidateValue), formats);
16787
16791
  const expectedDate = parseDate(String(expectedValue), formats);
16788
16792
  if (candidateDate === null) {
16789
16793
  return {
16790
- path: path53,
16794
+ path: path54,
16791
16795
  score: 0,
16792
16796
  weight,
16793
16797
  hit: false,
16794
- message: `${path53} (unparseable candidate date)`
16798
+ message: `${path54} (unparseable candidate date)`
16795
16799
  };
16796
16800
  }
16797
16801
  if (expectedDate === null) {
16798
16802
  return {
16799
- path: path53,
16803
+ path: path54,
16800
16804
  score: 0,
16801
16805
  weight,
16802
16806
  hit: false,
16803
- message: `${path53} (unparseable expected date)`
16807
+ message: `${path54} (unparseable expected date)`
16804
16808
  };
16805
16809
  }
16806
16810
  if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
16807
16811
  return {
16808
- path: path53,
16812
+ path: path54,
16809
16813
  score: 1,
16810
16814
  weight,
16811
16815
  hit: true,
16812
- message: path53
16816
+ message: path54
16813
16817
  };
16814
16818
  }
16815
16819
  return {
16816
- path: path53,
16820
+ path: path54,
16817
16821
  score: 0,
16818
16822
  weight,
16819
16823
  hit: false,
16820
- message: `${path53} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
16824
+ message: `${path54} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
16821
16825
  };
16822
16826
  }
16823
16827
  /**
@@ -16850,11 +16854,11 @@ var FieldAccuracyEvaluator = class {
16850
16854
  };
16851
16855
  }
16852
16856
  };
16853
- function resolvePath(obj, path53) {
16854
- if (!path53 || !obj) {
16857
+ function resolvePath(obj, path54) {
16858
+ if (!path54 || !obj) {
16855
16859
  return void 0;
16856
16860
  }
16857
- const parts = path53.split(/\.|\[|\]/).filter((p) => p.length > 0);
16861
+ const parts = path54.split(/\.|\[|\]/).filter((p) => p.length > 0);
16858
16862
  let current = obj;
16859
16863
  for (const part of parts) {
16860
16864
  if (current === null || current === void 0) {
@@ -17351,8 +17355,8 @@ var TokenUsageEvaluator = class {
17351
17355
 
17352
17356
  // src/evaluation/evaluators/tool-trajectory.ts
17353
17357
  init_cjs_shims();
17354
- function getNestedValue(obj, path53) {
17355
- const parts = path53.split(".");
17358
+ function getNestedValue(obj, path54) {
17359
+ const parts = path54.split(".");
17356
17360
  let current = obj;
17357
17361
  for (const part of parts) {
17358
17362
  if (current === null || current === void 0 || typeof current !== "object") {
@@ -21884,10 +21888,108 @@ function buildPrompt(criteria, question, referenceAnswer) {
21884
21888
  // src/evaluation/workspace/index.ts
21885
21889
  init_cjs_shims();
21886
21890
 
21887
- // src/evaluation/cache/response-cache.ts
21891
+ // src/evaluation/workspace/deps-scanner.ts
21888
21892
  init_cjs_shims();
21889
21893
  var import_promises34 = require("fs/promises");
21890
21894
  var import_node_path50 = __toESM(require("path"), 1);
21895
+ var import_yaml8 = require("yaml");
21896
+ function normalizeGitUrl(url) {
21897
+ let normalized = url.replace(/\.git$/, "");
21898
+ try {
21899
+ const parsed = new URL(normalized);
21900
+ parsed.hostname = parsed.hostname.toLowerCase();
21901
+ normalized = parsed.toString().replace(/\/$/, "");
21902
+ } catch {
21903
+ }
21904
+ return normalized;
21905
+ }
21906
+ async function scanRepoDeps(evalFilePaths) {
21907
+ const seen = /* @__PURE__ */ new Map();
21908
+ const errors = [];
21909
+ for (const filePath of evalFilePaths) {
21910
+ try {
21911
+ const repos = await extractReposFromEvalFile(filePath);
21912
+ for (const repo of repos) {
21913
+ if (repo.source.type !== "git") continue;
21914
+ const ref = repo.checkout?.ref;
21915
+ const key = `${normalizeGitUrl(repo.source.url)}\0${ref ?? ""}`;
21916
+ const existing = seen.get(key);
21917
+ if (existing) {
21918
+ existing.usedBy.push(filePath);
21919
+ } else {
21920
+ const { ref: _ref, ...checkoutRest } = repo.checkout ?? {};
21921
+ const hasCheckout = Object.keys(checkoutRest).length > 0;
21922
+ seen.set(key, {
21923
+ url: repo.source.url,
21924
+ ref,
21925
+ clone: repo.clone,
21926
+ checkout: hasCheckout ? checkoutRest : void 0,
21927
+ usedBy: [filePath]
21928
+ });
21929
+ }
21930
+ }
21931
+ } catch (err) {
21932
+ errors.push({
21933
+ file: filePath,
21934
+ message: err instanceof Error ? err.message : String(err)
21935
+ });
21936
+ }
21937
+ }
21938
+ return { repos: [...seen.values()], errors };
21939
+ }
21940
+ async function extractReposFromEvalFile(filePath) {
21941
+ const content = await (0, import_promises34.readFile)(filePath, "utf8");
21942
+ const parsed = interpolateEnv((0, import_yaml8.parse)(content), process.env);
21943
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
21944
+ const obj = parsed;
21945
+ const evalFileDir = import_node_path50.default.dirname(import_node_path50.default.resolve(filePath));
21946
+ const repos = [];
21947
+ const suiteRepos = await extractReposFromWorkspaceRaw(obj.workspace, evalFileDir);
21948
+ repos.push(...suiteRepos);
21949
+ const tests = Array.isArray(obj.tests) ? obj.tests : [];
21950
+ for (const test of tests) {
21951
+ if (test && typeof test === "object" && !Array.isArray(test)) {
21952
+ const testObj = test;
21953
+ const testRepos = await extractReposFromWorkspaceRaw(testObj.workspace, evalFileDir);
21954
+ repos.push(...testRepos);
21955
+ }
21956
+ }
21957
+ return repos;
21958
+ }
21959
+ async function extractReposFromWorkspaceRaw(raw, evalFileDir) {
21960
+ if (typeof raw === "string") {
21961
+ const workspaceFilePath = import_node_path50.default.resolve(evalFileDir, raw);
21962
+ const content = await (0, import_promises34.readFile)(workspaceFilePath, "utf8");
21963
+ const parsed = interpolateEnv((0, import_yaml8.parse)(content), process.env);
21964
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
21965
+ return extractReposFromObject(parsed);
21966
+ }
21967
+ if (raw && typeof raw === "object" && !Array.isArray(raw)) {
21968
+ return extractReposFromObject(raw);
21969
+ }
21970
+ return [];
21971
+ }
21972
+ function extractReposFromObject(obj) {
21973
+ const rawRepos = Array.isArray(obj.repos) ? obj.repos : [];
21974
+ const result = [];
21975
+ for (const r of rawRepos) {
21976
+ if (!r || typeof r !== "object" || Array.isArray(r)) continue;
21977
+ const repo = r;
21978
+ const source = parseRepoSource(repo.source);
21979
+ if (!source) continue;
21980
+ result.push({
21981
+ source,
21982
+ checkout: parseRepoCheckout(repo.checkout),
21983
+ clone: parseRepoClone(repo.clone)
21984
+ });
21985
+ }
21986
+ return result;
21987
+ }
21988
+
21989
+ // src/evaluation/cache/response-cache.ts
21990
+ init_cjs_shims();
21991
+ var import_promises35 = require("fs/promises");
21992
+ var import_node_path51 = __toESM(require("path"), 1);
21891
21993
  var DEFAULT_CACHE_PATH = ".agentv/cache";
21892
21994
  var ResponseCache = class {
21893
21995
  cachePath;
@@ -21897,7 +21999,7 @@ var ResponseCache = class {
21897
21999
  async get(key) {
21898
22000
  const filePath = this.keyToPath(key);
21899
22001
  try {
21900
- const data = await (0, import_promises34.readFile)(filePath, "utf8");
22002
+ const data = await (0, import_promises35.readFile)(filePath, "utf8");
21901
22003
  return JSON.parse(data);
21902
22004
  } catch {
21903
22005
  return void 0;
@@ -21905,13 +22007,13 @@ var ResponseCache = class {
21905
22007
  }
21906
22008
  async set(key, value) {
21907
22009
  const filePath = this.keyToPath(key);
21908
- const dir = import_node_path50.default.dirname(filePath);
21909
- await (0, import_promises34.mkdir)(dir, { recursive: true });
21910
- await (0, import_promises34.writeFile)(filePath, JSON.stringify(value, null, 2), "utf8");
22010
+ const dir = import_node_path51.default.dirname(filePath);
22011
+ await (0, import_promises35.mkdir)(dir, { recursive: true });
22012
+ await (0, import_promises35.writeFile)(filePath, JSON.stringify(value, null, 2), "utf8");
21911
22013
  }
21912
22014
  keyToPath(key) {
21913
22015
  const prefix = key.slice(0, 2);
21914
- return import_node_path50.default.join(this.cachePath, prefix, `${key}.json`);
22016
+ return import_node_path51.default.join(this.cachePath, prefix, `${key}.json`);
21915
22017
  }
21916
22018
  };
21917
22019
  function shouldEnableCache(params) {
@@ -21929,10 +22031,10 @@ function shouldSkipCacheForTemperature(targetConfig) {
21929
22031
  // src/projects.ts
21930
22032
  init_cjs_shims();
21931
22033
  var import_node_fs17 = require("fs");
21932
- var import_node_path51 = __toESM(require("path"), 1);
21933
- var import_yaml8 = require("yaml");
22034
+ var import_node_path52 = __toESM(require("path"), 1);
22035
+ var import_yaml9 = require("yaml");
21934
22036
  function getProjectsRegistryPath() {
21935
- return import_node_path51.default.join(getAgentvHome(), "projects.yaml");
22037
+ return import_node_path52.default.join(getAgentvHome(), "projects.yaml");
21936
22038
  }
21937
22039
  function loadProjectRegistry() {
21938
22040
  const registryPath = getProjectsRegistryPath();
@@ -21941,7 +22043,7 @@ function loadProjectRegistry() {
21941
22043
  }
21942
22044
  try {
21943
22045
  const raw = (0, import_node_fs17.readFileSync)(registryPath, "utf-8");
21944
- const parsed = (0, import_yaml8.parse)(raw);
22046
+ const parsed = (0, import_yaml9.parse)(raw);
21945
22047
  if (!parsed || !Array.isArray(parsed.projects)) {
21946
22048
  return { projects: [] };
21947
22049
  }
@@ -21952,14 +22054,14 @@ function loadProjectRegistry() {
21952
22054
  }
21953
22055
  function saveProjectRegistry(registry) {
21954
22056
  const registryPath = getProjectsRegistryPath();
21955
- const dir = import_node_path51.default.dirname(registryPath);
22057
+ const dir = import_node_path52.default.dirname(registryPath);
21956
22058
  if (!(0, import_node_fs17.existsSync)(dir)) {
21957
22059
  (0, import_node_fs17.mkdirSync)(dir, { recursive: true });
21958
22060
  }
21959
- (0, import_node_fs17.writeFileSync)(registryPath, (0, import_yaml8.stringify)(registry), "utf-8");
22061
+ (0, import_node_fs17.writeFileSync)(registryPath, (0, import_yaml9.stringify)(registry), "utf-8");
21960
22062
  }
21961
22063
  function deriveProjectId(dirPath, existingIds) {
21962
- const base = import_node_path51.default.basename(dirPath).toLowerCase().replace(/[^a-z0-9-]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
22064
+ const base = import_node_path52.default.basename(dirPath).toLowerCase().replace(/[^a-z0-9-]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
21963
22065
  let candidate = base || "project";
21964
22066
  let suffix = 2;
21965
22067
  while (existingIds.includes(candidate)) {
@@ -21969,11 +22071,11 @@ function deriveProjectId(dirPath, existingIds) {
21969
22071
  return candidate;
21970
22072
  }
21971
22073
  function addProject(projectPath) {
21972
- const absPath = import_node_path51.default.resolve(projectPath);
22074
+ const absPath = import_node_path52.default.resolve(projectPath);
21973
22075
  if (!(0, import_node_fs17.existsSync)(absPath)) {
21974
22076
  throw new Error(`Directory not found: ${absPath}`);
21975
22077
  }
21976
- if (!(0, import_node_fs17.existsSync)(import_node_path51.default.join(absPath, ".agentv"))) {
22078
+ if (!(0, import_node_fs17.existsSync)(import_node_path52.default.join(absPath, ".agentv"))) {
21977
22079
  throw new Error(`No .agentv/ directory found in ${absPath}. Run an evaluation first.`);
21978
22080
  }
21979
22081
  const registry = loadProjectRegistry();
@@ -21987,7 +22089,7 @@ function addProject(projectPath) {
21987
22089
  absPath,
21988
22090
  registry.projects.map((p) => p.id)
21989
22091
  ),
21990
- name: import_node_path51.default.basename(absPath),
22092
+ name: import_node_path52.default.basename(absPath),
21991
22093
  path: absPath,
21992
22094
  addedAt: now,
21993
22095
  lastOpenedAt: now
@@ -22016,14 +22118,14 @@ function touchProject(projectId) {
22016
22118
  }
22017
22119
  }
22018
22120
  function discoverProjects(rootDir, maxDepth = 2) {
22019
- const absRoot = import_node_path51.default.resolve(rootDir);
22121
+ const absRoot = import_node_path52.default.resolve(rootDir);
22020
22122
  if (!(0, import_node_fs17.existsSync)(absRoot) || !(0, import_node_fs17.statSync)(absRoot).isDirectory()) {
22021
22123
  return [];
22022
22124
  }
22023
22125
  const results = [];
22024
22126
  function scan(dir, depth) {
22025
22127
  if (depth > maxDepth) return;
22026
- if ((0, import_node_fs17.existsSync)(import_node_path51.default.join(dir, ".agentv"))) {
22128
+ if ((0, import_node_fs17.existsSync)(import_node_path52.default.join(dir, ".agentv"))) {
22027
22129
  results.push(dir);
22028
22130
  return;
22029
22131
  }
@@ -22033,7 +22135,7 @@ function discoverProjects(rootDir, maxDepth = 2) {
22033
22135
  for (const entry of entries) {
22034
22136
  if (!entry.isDirectory()) continue;
22035
22137
  if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
22036
- scan(import_node_path51.default.join(dir, entry.name), depth + 1);
22138
+ scan(import_node_path52.default.join(dir, entry.name), depth + 1);
22037
22139
  }
22038
22140
  } catch {
22039
22141
  }
@@ -22959,33 +23061,33 @@ function extractResponseItemContent(content) {
22959
23061
 
22960
23062
  // src/import/codex-session-discovery.ts
22961
23063
  init_cjs_shims();
22962
- var import_promises36 = require("fs/promises");
23064
+ var import_promises37 = require("fs/promises");
22963
23065
  var import_node_os8 = require("os");
22964
- var import_node_path53 = __toESM(require("path"), 1);
22965
- var DEFAULT_SESSIONS_DIR = () => import_node_path53.default.join((0, import_node_os8.homedir)(), ".codex", "sessions");
23066
+ var import_node_path54 = __toESM(require("path"), 1);
23067
+ var DEFAULT_SESSIONS_DIR = () => import_node_path54.default.join((0, import_node_os8.homedir)(), ".codex", "sessions");
22966
23068
  async function discoverCodexSessions(opts) {
22967
23069
  const sessionsDir = opts?.sessionsDir ?? DEFAULT_SESSIONS_DIR();
22968
23070
  const limit = opts?.latest ? 1 : opts?.limit ?? 10;
22969
23071
  const sessions = [];
22970
23072
  let yearDirs;
22971
23073
  try {
22972
- yearDirs = await (0, import_promises36.readdir)(sessionsDir);
23074
+ yearDirs = await (0, import_promises37.readdir)(sessionsDir);
22973
23075
  } catch {
22974
23076
  return [];
22975
23077
  }
22976
23078
  for (const year of yearDirs) {
22977
- const yearPath = import_node_path53.default.join(sessionsDir, year);
23079
+ const yearPath = import_node_path54.default.join(sessionsDir, year);
22978
23080
  let monthDirs;
22979
23081
  try {
22980
- monthDirs = await (0, import_promises36.readdir)(yearPath);
23082
+ monthDirs = await (0, import_promises37.readdir)(yearPath);
22981
23083
  } catch {
22982
23084
  continue;
22983
23085
  }
22984
23086
  for (const month of monthDirs) {
22985
- const monthPath = import_node_path53.default.join(yearPath, month);
23087
+ const monthPath = import_node_path54.default.join(yearPath, month);
22986
23088
  let dayDirs;
22987
23089
  try {
22988
- dayDirs = await (0, import_promises36.readdir)(monthPath);
23090
+ dayDirs = await (0, import_promises37.readdir)(monthPath);
22989
23091
  } catch {
22990
23092
  continue;
22991
23093
  }
@@ -22994,22 +23096,22 @@ async function discoverCodexSessions(opts) {
22994
23096
  const dirDate = `${year}-${month}-${day}`;
22995
23097
  if (dirDate !== opts.date) continue;
22996
23098
  }
22997
- const dayPath = import_node_path53.default.join(monthPath, day);
23099
+ const dayPath = import_node_path54.default.join(monthPath, day);
22998
23100
  let files;
22999
23101
  try {
23000
- files = await (0, import_promises36.readdir)(dayPath);
23102
+ files = await (0, import_promises37.readdir)(dayPath);
23001
23103
  } catch {
23002
23104
  continue;
23003
23105
  }
23004
23106
  for (const file of files) {
23005
23107
  if (!file.startsWith("rollout-") || !file.endsWith(".jsonl")) continue;
23006
- const filePath = import_node_path53.default.join(dayPath, file);
23108
+ const filePath = import_node_path54.default.join(dayPath, file);
23007
23109
  const nameWithoutExt = file.replace(/\.jsonl$/, "");
23008
23110
  const parts = nameWithoutExt.split("-");
23009
23111
  const sessionId = parts.length >= 6 ? parts.slice(-5).join("-") : nameWithoutExt;
23010
23112
  let updatedAt;
23011
23113
  try {
23012
- const fileStat = await (0, import_promises36.stat)(filePath);
23114
+ const fileStat = await (0, import_promises37.stat)(filePath);
23013
23115
  updatedAt = fileStat.mtime;
23014
23116
  } catch {
23015
23117
  updatedAt = /* @__PURE__ */ new Date(0);
@@ -23025,10 +23127,10 @@ async function discoverCodexSessions(opts) {
23025
23127
 
23026
23128
  // src/import/session-discovery.ts
23027
23129
  init_cjs_shims();
23028
- var import_promises37 = require("fs/promises");
23130
+ var import_promises38 = require("fs/promises");
23029
23131
  var import_node_os9 = require("os");
23030
- var import_node_path54 = __toESM(require("path"), 1);
23031
- var DEFAULT_PROJECTS_DIR = () => import_node_path54.default.join((0, import_node_os9.homedir)(), ".claude", "projects");
23132
+ var import_node_path55 = __toESM(require("path"), 1);
23133
+ var DEFAULT_PROJECTS_DIR = () => import_node_path55.default.join((0, import_node_os9.homedir)(), ".claude", "projects");
23032
23134
  function encodeProjectPath(projectPath) {
23033
23135
  return projectPath.replace(/\//g, "-");
23034
23136
  }
@@ -23037,7 +23139,7 @@ async function discoverClaudeSessions(opts) {
23037
23139
  const limit = opts?.latest ? 1 : opts?.limit ?? 10;
23038
23140
  let projectDirs;
23039
23141
  try {
23040
- projectDirs = await (0, import_promises37.readdir)(projectsDir);
23142
+ projectDirs = await (0, import_promises38.readdir)(projectsDir);
23041
23143
  } catch {
23042
23144
  return [];
23043
23145
  }
@@ -23047,10 +23149,10 @@ async function discoverClaudeSessions(opts) {
23047
23149
  }
23048
23150
  const sessions = [];
23049
23151
  for (const projectDir of projectDirs) {
23050
- const dirPath = import_node_path54.default.join(projectsDir, projectDir);
23152
+ const dirPath = import_node_path55.default.join(projectsDir, projectDir);
23051
23153
  let entries;
23052
23154
  try {
23053
- entries = await (0, import_promises37.readdir)(dirPath);
23155
+ entries = await (0, import_promises38.readdir)(dirPath);
23054
23156
  } catch {
23055
23157
  continue;
23056
23158
  }
@@ -23058,10 +23160,10 @@ async function discoverClaudeSessions(opts) {
23058
23160
  if (!entry.endsWith(".jsonl")) continue;
23059
23161
  const sessionId = entry.replace(/\.jsonl$/, "");
23060
23162
  if (opts?.sessionId && sessionId !== opts.sessionId) continue;
23061
- const filePath = import_node_path54.default.join(dirPath, entry);
23163
+ const filePath = import_node_path55.default.join(dirPath, entry);
23062
23164
  let updatedAt;
23063
23165
  try {
23064
- const fileStat = await (0, import_promises37.stat)(filePath);
23166
+ const fileStat = await (0, import_promises38.stat)(filePath);
23065
23167
  updatedAt = fileStat.mtime;
23066
23168
  } catch {
23067
23169
  updatedAt = /* @__PURE__ */ new Date(0);
@@ -23083,7 +23185,7 @@ init_cjs_shims();
23083
23185
 
23084
23186
  // src/import/types.ts
23085
23187
  init_cjs_shims();
23086
- var import_promises38 = require("fs/promises");
23188
+ var import_promises39 = require("fs/promises");
23087
23189
  function toTranscriptJsonLine(entry) {
23088
23190
  const firstUserMessage = entry.messages.find((m) => m.role === "user");
23089
23191
  const input = typeof firstUserMessage?.content === "string" ? firstUserMessage.content : "";
@@ -23109,11 +23211,11 @@ function toTranscriptJsonLine(entry) {
23109
23211
  };
23110
23212
  }
23111
23213
  async function readTranscriptJsonl(filePath) {
23112
- const text = await (0, import_promises38.readFile)(filePath, "utf8");
23214
+ const text = await (0, import_promises39.readFile)(filePath, "utf8");
23113
23215
  return text.split("\n").filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
23114
23216
  }
23115
23217
  async function readTranscriptFile(filePath) {
23116
- return (0, import_promises38.readFile)(filePath, "utf8");
23218
+ return (0, import_promises39.readFile)(filePath, "utf8");
23117
23219
  }
23118
23220
 
23119
23221
  // src/import/transcript-provider.ts
@@ -23328,6 +23430,7 @@ function createAgentKernel() {
23328
23430
  runRegexAssertion,
23329
23431
  runStartsWithAssertion,
23330
23432
  saveProjectRegistry,
23433
+ scanRepoDeps,
23331
23434
  scoreToVerdict,
23332
23435
  shouldEnableCache,
23333
23436
  shouldSkipCacheForTemperature,