agentv 5.0.0-next.1 → 5.2.0-next.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/README.md +239 -53
  2. package/dist/{agentv-provider-AYXH7WLW-NJRC6UQX.js → agentv-provider-AYXH7WLW-6YZKZ7PS.js} +2 -2
  3. package/dist/{artifact-writer-QD52FC56.js → artifact-writer-SN6UNL5G.js} +15 -8
  4. package/dist/{chunk-GZJMODLW.js → chunk-3S6A2RKR.js} +45880 -30542
  5. package/dist/chunk-3S6A2RKR.js.map +1 -0
  6. package/dist/{chunk-6Q56CEXV.js → chunk-3X357HS4.js} +7900 -8985
  7. package/dist/chunk-3X357HS4.js.map +1 -0
  8. package/dist/{chunk-6JVM4GJI.js → chunk-46K2OET3.js} +187 -80
  9. package/dist/chunk-46K2OET3.js.map +1 -0
  10. package/dist/{chunk-IJEZFZJR.js → chunk-7BGERE6L.js} +2 -2
  11. package/dist/chunk-7BGERE6L.js.map +1 -0
  12. package/dist/{chunk-UG4JHZYM.js → chunk-ALVZQUZP.js} +670 -1427
  13. package/dist/chunk-ALVZQUZP.js.map +1 -0
  14. package/dist/{chunk-RCIEHFQ2.js → chunk-PEUTJS7B.js} +3 -3
  15. package/dist/{chunk-5H446C7X.js → chunk-QMRVH5ZP.js} +2 -8
  16. package/dist/{chunk-IBI7A5RP.js → chunk-T4AD3H3Y.js} +3444 -2251
  17. package/dist/chunk-T4AD3H3Y.js.map +1 -0
  18. package/dist/cli.js +8 -9
  19. package/dist/cli.js.map +1 -1
  20. package/dist/dashboard/assets/{index-D5_8DE_B.js → index-DNgf3qJ2.js} +1 -1
  21. package/dist/dashboard/assets/index-D_bokML8.css +1 -0
  22. package/dist/dashboard/assets/index-r_jSJmlw.js +121 -0
  23. package/dist/dashboard/index.html +2 -2
  24. package/dist/{dist-KJ6PELQY.js → dist-DEPJOMCA.js} +74 -60
  25. package/dist/{docker-workspace-ZJ2ESVBI-6UDETQII.js → docker-workspace-NNK4Z6FR-QRTIG6TM.js} +3 -3
  26. package/dist/{exec-MYPAJJP7-C5HD5WG5.js → exec-4HFFRQ3B-AXDMJNLJ.js} +3 -3
  27. package/dist/index.js +8 -9
  28. package/dist/{interactive-2WMVAASS.js → interactive-36ZNMQB2.js} +13 -29
  29. package/dist/interactive-36ZNMQB2.js.map +1 -0
  30. package/dist/skills/agentv-bench/SKILL.md +38 -34
  31. package/dist/skills/agentv-bench/agents/analyzer.md +6 -6
  32. package/dist/skills/agentv-bench/agents/comparator.md +4 -5
  33. package/dist/skills/agentv-bench/agents/grader.md +4 -4
  34. package/dist/skills/agentv-bench/references/autoresearch.md +3 -3
  35. package/dist/skills/agentv-bench/references/description-optimization.md +2 -2
  36. package/dist/skills/agentv-bench/references/environment-adaptation.md +10 -10
  37. package/dist/skills/agentv-bench/references/eval-yaml-spec.md +51 -30
  38. package/dist/skills/agentv-bench/references/migrating-from-skill-creator.md +4 -4
  39. package/dist/skills/agentv-bench/references/subagent-pipeline.md +13 -12
  40. package/dist/skills/agentv-eval-migrations/SKILL.md +33 -0
  41. package/dist/skills/agentv-eval-migrations/references/breaking-changes.md +1012 -0
  42. package/dist/skills/agentv-eval-review/SKILL.md +6 -3
  43. package/dist/skills/agentv-eval-review/scripts/lint_eval.py +1 -1
  44. package/dist/skills/agentv-eval-writer/SKILL.md +380 -246
  45. package/dist/skills/agentv-eval-writer/references/{config-schema.json → config.schema.json} +6 -5
  46. package/dist/skills/agentv-eval-writer/references/custom-evaluators.md +23 -18
  47. package/dist/skills/agentv-eval-writer/references/eval.schema.json +9852 -0
  48. package/dist/skills/agentv-eval-writer/references/python-helpers.md +8 -8
  49. package/dist/skills/agentv-eval-writer/references/rubric-evaluator.md +21 -22
  50. package/dist/skills/agentv-trace-analyst/SKILL.md +4 -4
  51. package/dist/templates/.agentv/targets.yaml +41 -40
  52. package/dist/{ts-eval-loader-RE43OAGX-AMRCUKBZ.js → ts-eval-loader-OMG2JBHP-OP5RR7A3.js} +5 -4
  53. package/package.json +19 -4
  54. package/dist/chunk-6JVM4GJI.js.map +0 -1
  55. package/dist/chunk-6Q56CEXV.js.map +0 -1
  56. package/dist/chunk-76FOHROU.js +0 -96
  57. package/dist/chunk-76FOHROU.js.map +0 -1
  58. package/dist/chunk-GZJMODLW.js.map +0 -1
  59. package/dist/chunk-HSHTTN6C.js +0 -565
  60. package/dist/chunk-HSHTTN6C.js.map +0 -1
  61. package/dist/chunk-IBI7A5RP.js.map +0 -1
  62. package/dist/chunk-IJEZFZJR.js.map +0 -1
  63. package/dist/chunk-LRULMAAA.js +0 -1711
  64. package/dist/chunk-LRULMAAA.js.map +0 -1
  65. package/dist/chunk-R3CXYNTN.js +0 -1238
  66. package/dist/chunk-R3CXYNTN.js.map +0 -1
  67. package/dist/chunk-UG4JHZYM.js.map +0 -1
  68. package/dist/chunk-VQ2ZO7XJ.js +0 -2098
  69. package/dist/chunk-VQ2ZO7XJ.js.map +0 -1
  70. package/dist/chunk-XALGXSKB.js +0 -21
  71. package/dist/chunk-XALGXSKB.js.map +0 -1
  72. package/dist/dashboard/assets/index-BVMrdN8a.js +0 -119
  73. package/dist/dashboard/assets/index-DZWXfWsv.css +0 -1
  74. package/dist/esm-NNMQYIKT.js +0 -32
  75. package/dist/esm-R77SNOF5.js +0 -65
  76. package/dist/esm-RVQPUGWH.js +0 -1207
  77. package/dist/esm-RVQPUGWH.js.map +0 -1
  78. package/dist/esm-TTEMJEVV.js +0 -933
  79. package/dist/esm-TTEMJEVV.js.map +0 -1
  80. package/dist/esm-ZADQ4XQH-QUFGGZL4.js +0 -924
  81. package/dist/esm-ZADQ4XQH-QUFGGZL4.js.map +0 -1
  82. package/dist/exec-MYPAJJP7-C5HD5WG5.js.map +0 -1
  83. package/dist/getMachineId-bsd-HSK5LZMG.js +0 -41
  84. package/dist/getMachineId-bsd-HSK5LZMG.js.map +0 -1
  85. package/dist/getMachineId-darwin-4DP6CCJV.js +0 -41
  86. package/dist/getMachineId-darwin-4DP6CCJV.js.map +0 -1
  87. package/dist/getMachineId-linux-44LJ5UJB.js +0 -33
  88. package/dist/getMachineId-linux-44LJ5UJB.js.map +0 -1
  89. package/dist/getMachineId-unsupported-NVK6IATM.js +0 -24
  90. package/dist/getMachineId-unsupported-NVK6IATM.js.map +0 -1
  91. package/dist/getMachineId-win-YZ36S7VA.js +0 -43
  92. package/dist/getMachineId-win-YZ36S7VA.js.map +0 -1
  93. package/dist/interactive-2WMVAASS.js.map +0 -1
  94. package/dist/otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js +0 -9
  95. package/dist/otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js.map +0 -1
  96. package/dist/skills/agentv-eval-writer/references/eval-schema.json +0 -17295
  97. package/dist/skills/agentv-eval-writer/references/experiment-schema.json +0 -278
  98. package/dist/src-76ISS3LH.js +0 -1733
  99. package/dist/src-76ISS3LH.js.map +0 -1
  100. package/dist/ts-eval-loader-RE43OAGX-AMRCUKBZ.js.map +0 -1
  101. /package/dist/{agentv-provider-AYXH7WLW-NJRC6UQX.js.map → agentv-provider-AYXH7WLW-6YZKZ7PS.js.map} +0 -0
  102. /package/dist/{artifact-writer-QD52FC56.js.map → artifact-writer-SN6UNL5G.js.map} +0 -0
  103. /package/dist/{chunk-RCIEHFQ2.js.map → chunk-PEUTJS7B.js.map} +0 -0
  104. /package/dist/{chunk-5H446C7X.js.map → chunk-QMRVH5ZP.js.map} +0 -0
  105. /package/dist/{dist-KJ6PELQY.js.map → dist-DEPJOMCA.js.map} +0 -0
  106. /package/dist/{docker-workspace-ZJ2ESVBI-6UDETQII.js.map → docker-workspace-NNK4Z6FR-QRTIG6TM.js.map} +0 -0
  107. /package/dist/{esm-NNMQYIKT.js.map → exec-4HFFRQ3B-AXDMJNLJ.js.map} +0 -0
  108. /package/dist/{esm-R77SNOF5.js.map → ts-eval-loader-OMG2JBHP-OP5RR7A3.js.map} +0 -0
@@ -8,57 +8,47 @@ import {
8
8
  external_exports,
9
9
  extractLastAssistantContent,
10
10
  getAgentvDataDir,
11
- getRepoCheckoutRef,
12
11
  groupTranscriptJsonLines,
13
- interpolateEnv,
14
12
  loadTests,
15
- normalizeRepoIdentity,
16
13
  normalizeToolCall,
17
- parseRepoConfig,
18
14
  parseYamlValue,
19
15
  prepareEvalCaseWorkspace,
20
16
  prepareSharedWorkspaceSetup,
21
17
  readTranscriptJsonl,
22
- releaseSharedWorkspaceSetup,
23
- resolveRepoCloneUrl
24
- } from "./chunk-GZJMODLW.js";
18
+ releaseSharedWorkspaceSetup
19
+ } from "./chunk-3S6A2RKR.js";
25
20
 
26
21
  // ../../packages/core/dist/index.js
27
- import { createHash } from "node:crypto";
28
- import { readFile } from "node:fs/promises";
29
- import path from "node:path";
30
- import { pathToFileURL } from "node:url";
31
22
  import { readFileSync } from "node:fs";
32
- import path2 from "node:path";
23
+ import path from "node:path";
33
24
  import { randomUUID } from "node:crypto";
34
- import path3 from "node:path";
25
+ import path2 from "node:path";
35
26
  import micromatch from "micromatch";
36
- import { readFile as readFile2 } from "node:fs/promises";
37
- import path4 from "node:path";
38
27
  import { execFile, spawn } from "node:child_process";
39
- import { createHash as createHash2, randomBytes } from "node:crypto";
28
+ import { createHash, randomBytes } from "node:crypto";
40
29
  import {
41
30
  existsSync,
42
31
  mkdirSync,
43
32
  mkdtempSync,
44
33
  readFileSync as readFileSync2,
34
+ realpathSync,
45
35
  renameSync,
46
36
  rmSync,
47
37
  writeFileSync
48
38
  } from "node:fs";
49
- import { cp, lstat, mkdtemp, readdir, rm, stat } from "node:fs/promises";
39
+ import { cp, lstat, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
50
40
  import os from "node:os";
51
- import path5 from "node:path";
41
+ import path3 from "node:path";
52
42
  import { promisify } from "node:util";
53
43
  import * as childProcess from "node:child_process";
54
44
  import { existsSync as existsSync2 } from "node:fs";
55
45
  import { spawnSync } from "node:child_process";
56
46
  import { readdir as readdir2, stat as stat2 } from "node:fs/promises";
57
47
  import { homedir } from "node:os";
58
- import path6 from "node:path";
48
+ import path4 from "node:path";
59
49
  import { readdir as readdir3, stat as stat3 } from "node:fs/promises";
60
50
  import { homedir as homedir2 } from "node:os";
61
- import path7 from "node:path";
51
+ import path5 from "node:path";
62
52
  var TRACE_SESSION_SCHEMA_VERSION = "agentv.dashboard.trace_session.v1";
63
53
  function isRecord(value) {
64
54
  return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -1482,430 +1472,7 @@ function isPrimitive(value) {
1482
1472
  function dropUndefined2(value) {
1483
1473
  return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
1484
1474
  }
1485
- var EXPERIMENT_FILE_EXTENSIONS = /* @__PURE__ */ new Set([".yaml", ".yml", ".ts", ".js", ".mts", ".mjs"]);
1486
- var VALID_SANDBOXES = /* @__PURE__ */ new Set(["auto", "docker", "vercel"]);
1487
- var VALID_REPEAT_STRATEGIES = /* @__PURE__ */ new Set([
1488
- "pass_at_k",
1489
- "mean",
1490
- "confidence_interval"
1491
- ]);
1492
- function isExperimentFileReference(value) {
1493
- const trimmed = value.trim();
1494
- if (!trimmed) {
1495
- return false;
1496
- }
1497
- return trimmed.includes("/") || trimmed.includes("\\") || EXPERIMENT_FILE_EXTENSIONS.has(path.extname(trimmed).toLowerCase());
1498
- }
1499
- function deriveExperimentNameFromPath(filePath) {
1500
- return path.basename(filePath).replace(/\.experiment\.(ya?ml|[cm]?[jt]s)$/i, "").replace(/\.(ya?ml|[cm]?[jt]s)$/i, "");
1501
- }
1502
- async function loadExperimentConfig(filePath) {
1503
- const resolvedPath = path.resolve(filePath);
1504
- const ext = path.extname(resolvedPath).toLowerCase();
1505
- let rawConfig;
1506
- if (ext === ".yaml" || ext === ".yml") {
1507
- rawConfig = parseYamlValue(await readFile(resolvedPath, "utf8"));
1508
- } else if (EXPERIMENT_FILE_EXTENSIONS.has(ext)) {
1509
- const moduleUrl = pathToFileURL(resolvedPath).href;
1510
- const mod = await import(moduleUrl);
1511
- rawConfig = mod.default ?? mod.config ?? mod;
1512
- } else {
1513
- throw new Error(
1514
- `Unsupported experiment file extension '${ext}'. Use .yaml, .yml, .ts, .js, .mts, or .mjs.`
1515
- );
1516
- }
1517
- return normalizeExperimentConfig(rawConfig, { sourcePath: resolvedPath });
1518
- }
1519
- function normalizeExperimentConfig(rawConfig, options = {}) {
1520
- if (!isRecord2(rawConfig)) {
1521
- throw new Error("Experiment config must be an object.");
1522
- }
1523
- const name = readOptionalString(rawConfig.name, "name");
1524
- const agent = readOptionalString(rawConfig.agent, "agent");
1525
- const target = readOptionalString(rawConfig.target, "target");
1526
- const targets = readTargets(rawConfig.targets);
1527
- const model = readOptionalString(rawConfig.model, "model");
1528
- const agentOptions = readOptionalRecord(rawConfig.agent_options ?? rawConfig.agentOptions);
1529
- const suites = readSuites(rawConfig.suites);
1530
- const scripts = readScriptArray(rawConfig.scripts, "scripts");
1531
- const repeat = readRepeat(rawConfig.repeat);
1532
- const runs = readOptionalPositiveInteger(rawConfig.runs, "runs");
1533
- if (repeat !== void 0 && runs !== void 0) {
1534
- throw new Error("Experiment repeat and runs cannot both be set. Use repeat for AgentV config.");
1535
- }
1536
- const earlyExit = readOptionalBoolean(rawConfig.early_exit ?? rawConfig.earlyExit, "early_exit");
1537
- const timeoutSeconds = readOptionalPositiveNumber(
1538
- rawConfig.timeout_seconds ?? rawConfig.timeoutSeconds,
1539
- "timeout_seconds"
1540
- );
1541
- const workers = readOptionalPositiveInteger(rawConfig.workers, "workers");
1542
- const budgetUsd = readOptionalPositiveNumber(
1543
- rawConfig.budget_usd ?? rawConfig.budgetUsd,
1544
- "budget_usd"
1545
- );
1546
- const sandbox = readOptionalSandbox(rawConfig.sandbox);
1547
- const workspace = readOptionalRecord(rawConfig.workspace);
1548
- const setup = readSetup(rawConfig.setup);
1549
- const configWithoutFingerprint = {
1550
- ...name !== void 0 && { name },
1551
- ...agent !== void 0 && { agent },
1552
- ...target !== void 0 && { target },
1553
- ...targets !== void 0 && { targets },
1554
- ...model !== void 0 && { model },
1555
- ...agentOptions !== void 0 && { agentOptions },
1556
- ...suites !== void 0 && { suites },
1557
- ...scripts !== void 0 && { scripts },
1558
- ...repeat !== void 0 && { repeat },
1559
- ...runs !== void 0 && { runs },
1560
- ...earlyExit !== void 0 && { earlyExit },
1561
- ...timeoutSeconds !== void 0 && { timeoutSeconds },
1562
- ...workers !== void 0 && { workers },
1563
- ...budgetUsd !== void 0 && { budgetUsd },
1564
- ...sandbox !== void 0 && { sandbox },
1565
- ...workspace !== void 0 && { workspace },
1566
- ...setup !== void 0 && { setup },
1567
- ...options.sourcePath !== void 0 && { sourcePath: options.sourcePath }
1568
- };
1569
- return {
1570
- ...configWithoutFingerprint,
1571
- fingerprint: fingerprintExperimentConfig(configWithoutFingerprint)
1572
- };
1573
- }
1574
- function fingerprintExperimentConfig(config) {
1575
- const stablePayload = toStableJsonValue(config);
1576
- return createHash("sha256").update(JSON.stringify(stablePayload)).digest("hex");
1577
- }
1578
- function buildExperimentArtifactMetadata(config) {
1579
- if (!config) {
1580
- return void 0;
1581
- }
1582
- const targets = config.targets?.map((target) => typeof target === "string" ? target : target.name).filter((target) => target.trim().length > 0);
1583
- return {
1584
- ...config.name !== void 0 && { name: config.name },
1585
- ...config.sourcePath !== void 0 && { source_path: config.sourcePath },
1586
- ...config.fingerprint !== void 0 && { fingerprint: config.fingerprint },
1587
- ...config.agent !== void 0 && { agent: config.agent },
1588
- ...config.target !== void 0 && { target: config.target },
1589
- ...targets && targets.length > 0 && { targets },
1590
- ...config.model !== void 0 && { model: config.model },
1591
- ...config.suites !== void 0 && { suites: config.suites.map(toSuiteArtifactMetadata) },
1592
- ...config.repeat !== void 0 && {
1593
- repeat: {
1594
- count: config.repeat.count,
1595
- strategy: config.repeat.strategy,
1596
- ...config.repeat.costLimitUsd !== void 0 && {
1597
- cost_limit_usd: config.repeat.costLimitUsd
1598
- }
1599
- }
1600
- },
1601
- ...config.runs !== void 0 && { runs: config.runs },
1602
- ...config.earlyExit !== void 0 && { early_exit: config.earlyExit },
1603
- ...config.timeoutSeconds !== void 0 && { timeout_seconds: config.timeoutSeconds },
1604
- ...config.workers !== void 0 && { workers: config.workers },
1605
- ...config.budgetUsd !== void 0 && { budget_usd: config.budgetUsd },
1606
- ...config.sandbox !== void 0 && { sandbox: config.sandbox }
1607
- };
1608
- }
1609
- function toSuiteArtifactMetadata(suite) {
1610
- return {
1611
- ref: suite.ref,
1612
- ...suite.select !== void 0 && {
1613
- select: {
1614
- test_ids: suite.select.testIds
1615
- }
1616
- }
1617
- };
1618
- }
1619
- function readRepeat(raw) {
1620
- if (raw === void 0) {
1621
- return void 0;
1622
- }
1623
- if (!isRecord2(raw)) {
1624
- throw new Error("Experiment repeat must be an object.");
1625
- }
1626
- const count = readRequiredPositiveInteger(raw.count, "repeat.count");
1627
- const strategy = readOptionalRepeatStrategy(raw.strategy);
1628
- const costLimitUsd = readOptionalNonNegativeNumber(
1629
- raw.cost_limit_usd ?? raw.costLimitUsd,
1630
- "repeat.cost_limit_usd"
1631
- );
1632
- return {
1633
- count,
1634
- strategy: strategy ?? "pass_at_k",
1635
- ...costLimitUsd !== void 0 && { costLimitUsd }
1636
- };
1637
- }
1638
- function readTargets(raw) {
1639
- if (raw === void 0) {
1640
- return void 0;
1641
- }
1642
- if (!Array.isArray(raw)) {
1643
- throw new Error("Experiment targets must be an array.");
1644
- }
1645
- return raw.map((entry, index) => {
1646
- if (typeof entry === "string" && entry.trim().length > 0) {
1647
- return entry.trim();
1648
- }
1649
- if (!isRecord2(entry)) {
1650
- throw new Error(`Experiment targets[${index}] must be a string or object.`);
1651
- }
1652
- const name = readRequiredString(entry.name, `targets[${index}].name`);
1653
- const useTarget = readOptionalString(
1654
- entry.use_target ?? entry.useTarget,
1655
- `targets[${index}].use_target`
1656
- );
1657
- const hooks = readOptionalRecord(entry.hooks);
1658
- return {
1659
- name,
1660
- ...useTarget !== void 0 && { useTarget },
1661
- ...hooks !== void 0 && { hooks }
1662
- };
1663
- });
1664
- }
1665
- function readSuites(raw) {
1666
- if (raw === void 0) {
1667
- return void 0;
1668
- }
1669
- if (!Array.isArray(raw)) {
1670
- throw new Error("Experiment suites must be an array.");
1671
- }
1672
- if (raw.length === 0) {
1673
- throw new Error("Experiment suites must not be empty.");
1674
- }
1675
- return raw.map((entry, index) => {
1676
- if (!isRecord2(entry)) {
1677
- throw new Error(`Experiment suites[${index}] must be an object.`);
1678
- }
1679
- const ref = readRequiredString(entry.ref, `suites[${index}].ref`);
1680
- const select = readSuiteSelect(entry.select, `suites[${index}].select`);
1681
- return {
1682
- ref,
1683
- ...select !== void 0 && { select }
1684
- };
1685
- });
1686
- }
1687
- function readSuiteSelect(raw, location) {
1688
- if (raw === void 0) {
1689
- return void 0;
1690
- }
1691
- if (!isRecord2(raw)) {
1692
- throw new Error(`Experiment ${location} must be an object.`);
1693
- }
1694
- const testIds = readOptionalStringArray(raw.test_ids ?? raw.testIds, `${location}.test_ids`);
1695
- if (testIds === void 0) {
1696
- throw new Error(`Experiment ${location}.test_ids is required when select is set.`);
1697
- }
1698
- return { testIds };
1699
- }
1700
- function readScriptArray(raw, location) {
1701
- if (raw === void 0) {
1702
- return void 0;
1703
- }
1704
- if (!Array.isArray(raw)) {
1705
- throw new Error(`Experiment ${location} must be an array.`);
1706
- }
1707
- return raw.map((entry, index) => readScript(entry, `${location}[${index}]`));
1708
- }
1709
- function readSetup(raw) {
1710
- if (raw === void 0) {
1711
- return void 0;
1712
- }
1713
- if (typeof raw === "function") {
1714
- return raw;
1715
- }
1716
- return readScriptArray(raw, "setup");
1717
- }
1718
- function readScript(raw, location) {
1719
- if (typeof raw === "string") {
1720
- const script2 = raw.trim();
1721
- if (!script2) {
1722
- throw new Error(`Experiment ${location} must not be empty.`);
1723
- }
1724
- return { script: script2 };
1725
- }
1726
- if (!isRecord2(raw)) {
1727
- throw new Error(`Experiment ${location} must be a string or object.`);
1728
- }
1729
- const command = readOptionalCommand(raw.command, `${location}.command`);
1730
- const script = readOptionalStringOrStringArray(raw.script, `${location}.script`);
1731
- if (command === void 0 && script === void 0) {
1732
- throw new Error(`Experiment ${location} must define command or script.`);
1733
- }
1734
- const timeoutSeconds = readOptionalPositiveNumber(
1735
- raw.timeout_seconds ?? raw.timeoutSeconds,
1736
- `${location}.timeout_seconds`
1737
- );
1738
- const cwd = readOptionalString(raw.cwd, `${location}.cwd`);
1739
- const env = readOptionalStringRecord(raw.env, `${location}.env`);
1740
- return {
1741
- ...command !== void 0 && { command },
1742
- ...script !== void 0 && { script },
1743
- ...timeoutSeconds !== void 0 && { timeoutSeconds },
1744
- ...cwd !== void 0 && { cwd },
1745
- ...env !== void 0 && { env }
1746
- };
1747
- }
1748
- function readOptionalCommand(raw, location) {
1749
- if (raw === void 0) {
1750
- return void 0;
1751
- }
1752
- if (typeof raw === "string") {
1753
- const command = raw.trim();
1754
- if (!command) {
1755
- throw new Error(`Experiment ${location} must not be empty.`);
1756
- }
1757
- return ["sh", "-c", command];
1758
- }
1759
- if (Array.isArray(raw) && raw.length > 0 && raw.every((entry) => typeof entry === "string" && entry.trim())) {
1760
- return raw.map((entry) => entry.trim());
1761
- }
1762
- throw new Error(`Experiment ${location} must be a string or string array.`);
1763
- }
1764
- function readOptionalStringOrStringArray(raw, location) {
1765
- if (raw === void 0) {
1766
- return void 0;
1767
- }
1768
- if (typeof raw === "string") {
1769
- const trimmed = raw.trim();
1770
- if (!trimmed) {
1771
- throw new Error(`Experiment ${location} must not be empty.`);
1772
- }
1773
- return trimmed;
1774
- }
1775
- if (Array.isArray(raw) && raw.length > 0 && raw.every((entry) => typeof entry === "string" && entry.trim())) {
1776
- return raw.map((entry) => entry.trim());
1777
- }
1778
- throw new Error(`Experiment ${location} must be a string or string array.`);
1779
- }
1780
- function readOptionalStringArray(raw, location) {
1781
- if (raw === void 0) {
1782
- return void 0;
1783
- }
1784
- if (Array.isArray(raw) && raw.length > 0 && raw.every((entry) => typeof entry === "string" && entry.trim())) {
1785
- return raw.map((entry) => entry.trim());
1786
- }
1787
- throw new Error(`Experiment ${location} must be a non-empty string array.`);
1788
- }
1789
- function readOptionalString(raw, location) {
1790
- if (raw === void 0) {
1791
- return void 0;
1792
- }
1793
- if (typeof raw !== "string" || raw.trim().length === 0) {
1794
- throw new Error(`Experiment ${location} must be a non-empty string.`);
1795
- }
1796
- return raw.trim();
1797
- }
1798
- function readRequiredString(raw, location) {
1799
- const value = readOptionalString(raw, location);
1800
- if (value === void 0) {
1801
- throw new Error(`Experiment ${location} is required.`);
1802
- }
1803
- return value;
1804
- }
1805
- function readRequiredPositiveInteger(raw, location) {
1806
- const value = readOptionalPositiveInteger(raw, location);
1807
- if (value === void 0) {
1808
- throw new Error(`Experiment ${location} is required.`);
1809
- }
1810
- return value;
1811
- }
1812
- function readOptionalBoolean(raw, location) {
1813
- if (raw === void 0) {
1814
- return void 0;
1815
- }
1816
- if (typeof raw !== "boolean") {
1817
- throw new Error(`Experiment ${location} must be a boolean.`);
1818
- }
1819
- return raw;
1820
- }
1821
- function readOptionalRepeatStrategy(raw) {
1822
- if (raw === void 0) {
1823
- return void 0;
1824
- }
1825
- if (typeof raw !== "string" || !VALID_REPEAT_STRATEGIES.has(raw)) {
1826
- throw new Error(
1827
- "Experiment repeat.strategy must be one of 'pass_at_k', 'mean', or 'confidence_interval'."
1828
- );
1829
- }
1830
- return raw;
1831
- }
1832
- function readOptionalPositiveInteger(raw, location) {
1833
- if (raw === void 0) {
1834
- return void 0;
1835
- }
1836
- if (typeof raw !== "number" || !Number.isInteger(raw) || raw < 1) {
1837
- throw new Error(`Experiment ${location} must be a positive integer.`);
1838
- }
1839
- return raw;
1840
- }
1841
- function readOptionalNonNegativeNumber(raw, location) {
1842
- if (raw === void 0) {
1843
- return void 0;
1844
- }
1845
- if (typeof raw !== "number" || !Number.isFinite(raw) || raw < 0) {
1846
- throw new Error(`Experiment ${location} must be a non-negative number.`);
1847
- }
1848
- return raw;
1849
- }
1850
- function readOptionalPositiveNumber(raw, location) {
1851
- if (raw === void 0) {
1852
- return void 0;
1853
- }
1854
- if (typeof raw !== "number" || !Number.isFinite(raw) || raw <= 0) {
1855
- throw new Error(`Experiment ${location} must be a positive number.`);
1856
- }
1857
- return raw;
1858
- }
1859
- function readOptionalSandbox(raw) {
1860
- if (raw === void 0) {
1861
- return void 0;
1862
- }
1863
- if (typeof raw !== "string" || !VALID_SANDBOXES.has(raw)) {
1864
- throw new Error("Experiment sandbox must be one of 'auto', 'docker', or 'vercel'.");
1865
- }
1866
- return raw;
1867
- }
1868
- function readOptionalRecord(raw) {
1869
- if (raw === void 0) {
1870
- return void 0;
1871
- }
1872
- if (!isRecord2(raw)) {
1873
- throw new Error("Experiment object field must be an object.");
1874
- }
1875
- return raw;
1876
- }
1877
- function readOptionalStringRecord(raw, location) {
1878
- if (raw === void 0) {
1879
- return void 0;
1880
- }
1881
- if (!isRecord2(raw)) {
1882
- throw new Error(`Experiment ${location} must be an object.`);
1883
- }
1884
- const entries = Object.entries(raw);
1885
- if (!entries.every((entry) => typeof entry[1] === "string")) {
1886
- throw new Error(`Experiment ${location} values must be strings.`);
1887
- }
1888
- return Object.fromEntries(entries);
1889
- }
1890
- function isRecord2(value) {
1891
- return typeof value === "object" && value !== null && !Array.isArray(value);
1892
- }
1893
- function toStableJsonValue(value) {
1894
- if (typeof value === "function") {
1895
- return "[function]";
1896
- }
1897
- if (value === null || typeof value !== "object") {
1898
- return value;
1899
- }
1900
- if (Array.isArray(value)) {
1901
- return value.map(toStableJsonValue);
1902
- }
1903
- const record = value;
1904
- return Object.fromEntries(
1905
- Object.keys(record).sort().map((key) => [key, toStableJsonValue(record[key])])
1906
- );
1907
- }
1908
- function codeGraderInstruction(graderName, description) {
1475
+ function scriptGraderInstruction(graderName, description) {
1909
1476
  const desc = description ? ` This grader: ${description}.` : "";
1910
1477
  return `Run \`agentv eval assert ${graderName} --agent-output <agent_output> --agent-input <original_prompt>\` and check the result.${desc} The command accepts --agent-output (the agent's full response text) and --agent-input (the original user prompt). It returns JSON on stdout: {"score": 0-1, "reasoning": "..."}. A score >= 0.5 means pass (exit 0); below 0.5 means fail (exit 1).`;
1911
1478
  }
@@ -1923,21 +1490,15 @@ function assertionToNaturalLanguage(entry) {
1923
1490
  switch (type) {
1924
1491
  case "skill-trigger":
1925
1492
  return null;
1926
- case "rubrics": {
1927
- if (typeof entry.criteria === "string") {
1928
- return entry.criteria;
1929
- }
1930
- return null;
1931
- }
1493
+ case "llm-rubric":
1494
+ return typeof entry.value === "string" ? entry.value : null;
1932
1495
  case "contains":
1933
1496
  return `Output contains '${entry.value}'`;
1934
- case "contains-any":
1935
- case "contains_any": {
1497
+ case "contains-any": {
1936
1498
  const values = Array.isArray(entry.value) ? entry.value.join("', '") : entry.value;
1937
1499
  return `Output contains any of: '${values}'`;
1938
1500
  }
1939
- case "contains-all":
1940
- case "contains_all": {
1501
+ case "contains-all": {
1941
1502
  const values = Array.isArray(entry.value) ? entry.value.join("', '") : entry.value;
1942
1503
  return `Output contains all of: '${values}'`;
1943
1504
  }
@@ -1948,35 +1509,27 @@ function assertionToNaturalLanguage(entry) {
1948
1509
  case "equals":
1949
1510
  return `Output exactly equals: ${entry.value}`;
1950
1511
  case "is-json":
1951
- case "is_json":
1952
1512
  return "Output is valid JSON";
1953
1513
  case "starts-with":
1954
- case "starts_with":
1955
1514
  return `Output starts with '${entry.value}'`;
1956
1515
  case "ends-with":
1957
- case "ends_with":
1958
1516
  return `Output ends with '${entry.value}'`;
1959
1517
  case "llm-grader":
1960
- case "llm_grader": {
1961
1518
  if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
1962
1519
  return null;
1963
1520
  }
1964
1521
  return typeof entry.prompt === "string" ? entry.prompt : null;
1965
- }
1966
- case "tool-trajectory":
1967
- case "tool_trajectory": {
1522
+ case "tool-trajectory": {
1968
1523
  const expectedArr = Array.isArray(entry.expected) ? entry.expected : [];
1969
1524
  const tools = expectedArr.map((e) => e.tool).filter(Boolean).join(", ");
1970
1525
  return tools ? `Agent called tools in order: ${tools}` : "Agent followed expected tool trajectory";
1971
1526
  }
1972
- case "code-grader":
1973
- case "code_grader": {
1974
- const graderName = entry.name ?? deriveGraderNameFromCommand(entry.command) ?? "code-grader";
1527
+ case "script": {
1528
+ const graderName = entry.metric ?? deriveGraderNameFromCommand(entry.command) ?? "script";
1975
1529
  const desc = typeof entry.description === "string" ? entry.description : void 0;
1976
- return codeGraderInstruction(graderName, desc);
1530
+ return scriptGraderInstruction(graderName, desc);
1977
1531
  }
1978
- case "field-accuracy":
1979
- case "field_accuracy": {
1532
+ case "field-accuracy": {
1980
1533
  const fieldPaths = Array.isArray(entry.fields) ? entry.fields.map((f) => f.path).filter(Boolean).join(", ") : "";
1981
1534
  return fieldPaths ? `Fields ${fieldPaths} match expected values` : "Fields match expected values";
1982
1535
  }
@@ -1985,14 +1538,12 @@ function assertionToNaturalLanguage(entry) {
1985
1538
  case "cost":
1986
1539
  return typeof entry.budget === "number" ? `Cost under $${entry.budget}` : "Cost within budget";
1987
1540
  case "token-usage":
1988
- case "token_usage":
1989
1541
  return "Token usage within limits";
1990
1542
  case "execution-metrics":
1991
- case "execution_metrics":
1992
1543
  return "Execution within metric bounds";
1993
1544
  default: {
1994
1545
  if (entry.command !== void 0 && type) {
1995
- return codeGraderInstruction(deriveGraderNameFromCommand(entry.command) ?? type);
1546
+ return scriptGraderInstruction(deriveGraderNameFromCommand(entry.command) ?? type);
1996
1547
  }
1997
1548
  if (typeof entry.criteria === "string") return entry.criteria;
1998
1549
  if (typeof entry.prompt === "string") return entry.prompt;
@@ -2001,7 +1552,17 @@ function assertionToNaturalLanguage(entry) {
2001
1552
  }
2002
1553
  }
2003
1554
  function assertionToNaturalLanguageList(entry) {
2004
- if (entry.type === "llm-grader" || entry.type === "llm_grader") {
1555
+ if (entry.type === "llm-rubric") {
1556
+ if (Array.isArray(entry.value) && entry.value.length > 0) {
1557
+ return entry.value.map((rubric) => {
1558
+ if (typeof rubric === "string") return rubric;
1559
+ if (!rubric || typeof rubric !== "object") return void 0;
1560
+ const item = rubric;
1561
+ return item.outcome ?? item.criteria ?? item.id;
1562
+ }).filter((value) => typeof value === "string");
1563
+ }
1564
+ }
1565
+ if (entry.type === "llm-grader") {
2005
1566
  if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
2006
1567
  return entry.rubrics.map((r) => r.outcome ?? r.criteria ?? r.id).filter((s) => typeof s === "string");
2007
1568
  }
@@ -2012,16 +1573,6 @@ function assertionToNaturalLanguageList(entry) {
2012
1573
  function extractTriggerAssertions(assertions) {
2013
1574
  return assertions.filter((a) => a.type === "skill-trigger");
2014
1575
  }
2015
- function resolveAssertions(rawCase) {
2016
- if (Array.isArray(rawCase.assertions)) return rawCase.assertions;
2017
- if (Array.isArray(rawCase.assert)) return rawCase.assert;
2018
- return [];
2019
- }
2020
- function resolveSuiteAssertions(suite) {
2021
- if (Array.isArray(suite.assertions)) return suite.assertions;
2022
- if (Array.isArray(suite.assert)) return suite.assert;
2023
- return [];
2024
- }
2025
1576
  function extractInput(rawCase) {
2026
1577
  const files = Array.isArray(rawCase.input_files) ? rawCase.input_files.filter((f) => typeof f === "string") : [];
2027
1578
  const input = rawCase.input;
@@ -2059,7 +1610,6 @@ function extractExpectedOutput(raw) {
2059
1610
  return JSON.stringify(raw);
2060
1611
  }
2061
1612
  function transpileEvalYaml(suite, source = "EVAL.yaml") {
2062
- const warnings = [];
2063
1613
  const files = /* @__PURE__ */ new Map();
2064
1614
  if (typeof suite !== "object" || suite === null) {
2065
1615
  throw new Error(`Invalid EVAL.yaml: expected an object in '${source}'`);
@@ -2068,10 +1618,7 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
2068
1618
  if (!Array.isArray(rawSuite.tests)) {
2069
1619
  throw new Error(`Invalid EVAL.yaml: missing 'tests' array in '${source}'`);
2070
1620
  }
2071
- if (rawSuite.assert !== void 0 && rawSuite.assertions === void 0) {
2072
- warnings.push("'assert' is deprecated at the suite level. Use 'assertions' instead.");
2073
- }
2074
- const suiteAssertions = resolveSuiteAssertions(rawSuite);
1621
+ const suiteAssertions = rawSuite.assert ?? [];
2075
1622
  const suiteNlAssertions = suiteAssertions.filter((a) => a.type !== "skill-trigger").flatMap(assertionToNaturalLanguageList);
2076
1623
  function getSkillFile(skillName) {
2077
1624
  const existing = files.get(skillName);
@@ -2083,10 +1630,11 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
2083
1630
  const tests = rawSuite.tests;
2084
1631
  for (let idx = 0; idx < tests.length; idx++) {
2085
1632
  const rawCase = tests[idx];
2086
- const caseAssertions = resolveAssertions(rawCase);
2087
- if (rawCase.assert !== void 0 && rawCase.assertions === void 0) {
2088
- const caseId = rawCase.id ?? idx + 1;
2089
- warnings.push(`Test '${caseId}': 'assert' is deprecated. Use 'assertions' instead.`);
1633
+ const caseAssertions = rawCase.assert ?? [];
1634
+ if (typeof rawCase.criteria === "string" && rawCase.criteria.trim() && rawCase.assert !== void 0) {
1635
+ throw new Error(
1636
+ `Invalid EVAL.yaml test '${rawCase.id ?? idx + 1}' in '${source}': do not combine test-level 'criteria' with 'assert'. Put human-readable case descriptions in 'description', or express grading text as an explicit assertion such as { type: 'llm-rubric', value: ... }.`
1637
+ );
2090
1638
  }
2091
1639
  const nlAssertions = [];
2092
1640
  if (typeof rawCase.criteria === "string" && rawCase.criteria.trim()) {
@@ -2139,12 +1687,12 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
2139
1687
  files.delete("_no-skill");
2140
1688
  }
2141
1689
  }
2142
- return { files, warnings };
1690
+ return { files, warnings: [] };
2143
1691
  }
2144
1692
  function transpileEvalYamlFile(evalYamlPath) {
2145
1693
  const content = readFileSync(evalYamlPath, "utf8");
2146
1694
  const parsed = parseYamlValue(content);
2147
- return transpileEvalYaml(parsed, path2.basename(evalYamlPath));
1695
+ return transpileEvalYaml(parsed, path.basename(evalYamlPath));
2148
1696
  }
2149
1697
  function getOutputFilenames(result) {
2150
1698
  const names = /* @__PURE__ */ new Map();
@@ -2181,35 +1729,12 @@ function toRepoPins(repos) {
2181
1729
  ...repo.path !== void 0 && { path: repo.path },
2182
1730
  ...repo.repo !== void 0 && { repo: repo.repo },
2183
1731
  ...repo.commit !== void 0 && { commit: repo.commit },
2184
- ...repo.base_commit !== void 0 && { baseCommit: repo.base_commit },
2185
1732
  ...repo.ancestor !== void 0 && { ancestor: repo.ancestor },
2186
1733
  ...repo.sparse !== void 0 && { sparse: repo.sparse }
2187
1734
  }));
2188
1735
  }
2189
- function poolMetadata(setup, selectedSlotPath) {
2190
- const slot = setup.poolSlot ?? setup.poolSlots.find((candidate) => candidate.path === selectedSlotPath);
2191
- if (!slot) {
2192
- return void 0;
2193
- }
2194
- return {
2195
- fingerprint: slot.fingerprint,
2196
- slotIndex: slot.index,
2197
- lockPath: slot.lockPath
2198
- };
2199
- }
2200
- async function releaseUnselectedPoolSlots(setup, selectedSlotPath) {
2201
- if (!setup.poolManager) {
2202
- return;
2203
- }
2204
- for (const slot of setup.poolSlots) {
2205
- if (slot.path !== selectedSlotPath) {
2206
- await setup.poolManager.releaseSlot(slot).catch(() => {
2207
- });
2208
- }
2209
- }
2210
- }
2211
1736
  async function prepareEvalWorkspace(options) {
2212
- const evalPath = path3.resolve(options.testFilePath);
1737
+ const evalPath = path2.resolve(options.testFilePath);
2213
1738
  const evalRunId = randomUUID();
2214
1739
  const evalCases = options.evalCases ?? await loadTests(evalPath, options.repoRoot, {
2215
1740
  verbose: options.verbose,
@@ -2221,7 +1746,7 @@ async function prepareEvalWorkspace(options) {
2221
1746
  filter: options.filter,
2222
1747
  evalPath
2223
1748
  });
2224
- const evalDir = path3.dirname(evalPath);
1749
+ const evalDir = path2.dirname(evalPath);
2225
1750
  const workers = options.maxConcurrency ?? 1;
2226
1751
  const retainOnSuccess = options.retainOnSuccess ?? (options.keepWorkspaces ? "keep" : "cleanup");
2227
1752
  const retainOnFailure = options.retainOnFailure ?? (options.cleanupWorkspaces ? "cleanup" : "keep");
@@ -2236,15 +1761,11 @@ async function prepareEvalWorkspace(options) {
2236
1761
  evalDir,
2237
1762
  verbose: options.verbose,
2238
1763
  workers,
2239
- poolMaxSlots: options.poolMaxSlots,
2240
1764
  workspacePath: options.workspacePath,
2241
- legacyWorkspacePath: options.workspace,
2242
- workspaceMode: options.workspaceMode,
2243
- workspaceClean: options.workspaceClean
1765
+ legacyWorkspacePath: options.workspace
2244
1766
  });
2245
- const testPoolSlot = sharedSetup.availablePoolSlots.length > 0 ? sharedSetup.availablePoolSlots.pop() : void 0;
2246
- const selectedWorkspacePath = testPoolSlot?.path ?? sharedSetup.sharedWorkspacePath;
2247
- const selectedBaselineCommit = testPoolSlot ? sharedSetup.poolSlotBaselines.get(testPoolSlot.path) : sharedSetup.sharedBaselineCommit;
1767
+ const selectedWorkspacePath = sharedSetup.sharedWorkspacePath;
1768
+ const selectedBaselineCommit = sharedSetup.sharedBaselineCommit;
2248
1769
  const caseSetup = await prepareEvalCaseWorkspace({
2249
1770
  evalCase,
2250
1771
  targetName: options.target.name,
@@ -2256,15 +1777,14 @@ async function prepareEvalWorkspace(options) {
2256
1777
  evalDir,
2257
1778
  cleanupWorkspaces: options.cleanupWorkspaces,
2258
1779
  targetHooks: options.targetHooks,
1780
+ sharedExtensionState: sharedSetup.extensionState,
2259
1781
  setupDebug: options.verbose
2260
1782
  });
2261
1783
  if (!caseSetup.workspacePath) {
2262
1784
  throw new Error(
2263
- `No workspace was materialized for test "${evalCase.id}". Add workspace.template, workspace.repos, or workspace.hooks before preparing an external attempt.`
1785
+ `No workspace was materialized for test "${evalCase.id}". Add environment.workdir before preparing an external attempt, or use an internal workspace override for private runtime plumbing.`
2264
1786
  );
2265
1787
  }
2266
- await releaseUnselectedPoolSlots(sharedSetup, caseSetup.workspacePath);
2267
- const pool = poolMetadata(sharedSetup, caseSetup.workspacePath);
2268
1788
  return {
2269
1789
  evalPath,
2270
1790
  testId: evalCase.id,
@@ -2276,6 +1796,12 @@ async function prepareEvalWorkspace(options) {
2276
1796
  },
2277
1797
  createdAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
2278
1798
  hookExecutions: [...sharedSetup.hookExecutions, ...caseSetup.hookExecutions],
1799
+ ...caseSetup.extensionState?.providerContext !== void 0 && {
1800
+ providerContext: caseSetup.extensionState.providerContext
1801
+ },
1802
+ ...caseSetup.extensionState?.metadata !== void 0 && {
1803
+ metadata: caseSetup.extensionState.metadata
1804
+ },
2279
1805
  repoPins: toRepoPins(evalCase.workspace?.repos),
2280
1806
  baseline: caseSetup.baselineCommit ? { status: "initialized", commit: caseSetup.baselineCommit } : { status: "unavailable" },
2281
1807
  promptSource: {
@@ -2293,8 +1819,7 @@ async function prepareEvalWorkspace(options) {
2293
1819
  retainOnFailure,
2294
1820
  manualCleanup: true
2295
1821
  },
2296
- sharedWorkspace: caseSetup.isSharedWorkspace,
2297
- ...pool !== void 0 && { pool }
1822
+ sharedWorkspace: caseSetup.isSharedWorkspace
2298
1823
  };
2299
1824
  } catch (error) {
2300
1825
  if (sharedSetup) {
@@ -2304,22 +1829,51 @@ async function prepareEvalWorkspace(options) {
2304
1829
  throw error;
2305
1830
  }
2306
1831
  }
1832
+ var ExecutionConfigSchema = external_exports.object({
1833
+ /** General eval parallelism (default: 3) */
1834
+ maxConcurrency: external_exports.number().int().min(1).max(50).optional(),
1835
+ /** Maximum retries on failure (default: 2) */
1836
+ maxRetries: external_exports.number().int().min(0).optional(),
1837
+ /** Agent timeout in milliseconds. No timeout if not set. */
1838
+ agentTimeoutMs: external_exports.number().int().min(0).optional(),
1839
+ /** Enable verbose logging */
1840
+ verbose: external_exports.boolean().optional(),
1841
+ /** Always keep temp workspaces after eval */
1842
+ keepWorkspaces: external_exports.boolean().optional()
1843
+ }).passthrough().superRefine((value, ctx) => {
1844
+ const supportedFields = /* @__PURE__ */ new Set([
1845
+ "maxConcurrency",
1846
+ "maxRetries",
1847
+ "agentTimeoutMs",
1848
+ "verbose",
1849
+ "keepWorkspaces"
1850
+ ]);
1851
+ if ("otelFile" in value) {
1852
+ ctx.addIssue({
1853
+ code: external_exports.ZodIssueCode.custom,
1854
+ path: ["otelFile"],
1855
+ message: "execution.otelFile has been removed. Emit OpenTelemetry/OpenInference traces from the system under test or provider and correlate AgentV run artifacts with external_trace metadata."
1856
+ });
1857
+ }
1858
+ for (const key of Object.keys(value)) {
1859
+ if (!supportedFields.has(key) && key !== "otelFile") {
1860
+ ctx.addIssue({
1861
+ code: external_exports.ZodIssueCode.custom,
1862
+ path: [key],
1863
+ message: `Unsupported execution field '${key}'`
1864
+ });
1865
+ }
1866
+ }
1867
+ }).transform(({ maxConcurrency, maxRetries, agentTimeoutMs, verbose, keepWorkspaces }) => ({
1868
+ ...maxConcurrency !== void 0 && { maxConcurrency },
1869
+ ...maxRetries !== void 0 && { maxRetries },
1870
+ ...agentTimeoutMs !== void 0 && { agentTimeoutMs },
1871
+ ...verbose !== void 0 && { verbose },
1872
+ ...keepWorkspaces !== void 0 && { keepWorkspaces }
1873
+ }));
2307
1874
  var AgentVConfigSchema = external_exports.object({
2308
1875
  /** Default execution settings */
2309
- execution: external_exports.object({
2310
- /** Number of parallel workers (default: 3) */
2311
- workers: external_exports.number().int().min(1).max(50).optional(),
2312
- /** Maximum retries on failure (default: 2) */
2313
- maxRetries: external_exports.number().int().min(0).optional(),
2314
- /** Agent timeout in milliseconds. No timeout if not set. */
2315
- agentTimeoutMs: external_exports.number().int().min(0).optional(),
2316
- /** Enable verbose logging */
2317
- verbose: external_exports.boolean().optional(),
2318
- /** Always keep temp workspaces after eval */
2319
- keepWorkspaces: external_exports.boolean().optional(),
2320
- /** Write OTLP JSON trace to this path (supports {timestamp} placeholder) */
2321
- otelFile: external_exports.string().optional()
2322
- }).optional(),
1876
+ execution: ExecutionConfigSchema.optional(),
2323
1877
  /** Output settings */
2324
1878
  output: external_exports.object({
2325
1879
  /** Default eval run artifact directory */
@@ -2332,13 +1886,6 @@ var AgentVConfigSchema = external_exports.object({
2332
1886
  /** Response cache directory */
2333
1887
  path: external_exports.string().optional()
2334
1888
  }).optional(),
2335
- /** Compatibility shorthand for experiments.default */
2336
- defaultExperiment: external_exports.string().optional(),
2337
- /** Experiment defaults */
2338
- experiments: external_exports.object({
2339
- /** Default experiment label or path used when eval runs omit --experiment */
2340
- default: external_exports.string().optional()
2341
- }).strict().optional(),
2342
1889
  /** Cost and duration limits */
2343
1890
  limits: external_exports.object({
2344
1891
  /** Maximum cost per run in USD */
@@ -2356,7 +1903,9 @@ var AgentVConfigSchema = external_exports.object({
2356
1903
  * stderr is forwarded to the user. Non-zero exit aborts with an error.
2357
1904
  */
2358
1905
  beforeSession: external_exports.string().optional()
2359
- }).optional()
1906
+ }).optional(),
1907
+ /** Project-local reusable references for fields that support ref://name */
1908
+ refs: external_exports.record(external_exports.string().min(1)).optional()
2360
1909
  });
2361
1910
  function defineConfig(config) {
2362
1911
  return AgentVConfigSchema.parse(config);
@@ -2371,7 +1920,7 @@ var CONFIG_FILE_NAMES = [
2371
1920
  ];
2372
1921
  async function loadTsConfig(projectRoot) {
2373
1922
  const { existsSync: existsSync3 } = await import("node:fs");
2374
- const { pathToFileURL: pathToFileURL2 } = await import("node:url");
1923
+ const { pathToFileURL } = await import("node:url");
2375
1924
  const { join } = await import("node:path");
2376
1925
  for (const fileName of CONFIG_FILE_NAMES) {
2377
1926
  const filePath = join(projectRoot, fileName);
@@ -2379,7 +1928,7 @@ async function loadTsConfig(projectRoot) {
2379
1928
  continue;
2380
1929
  }
2381
1930
  try {
2382
- const fileUrl = pathToFileURL2(filePath).href;
1931
+ const fileUrl = pathToFileURL(filePath).href;
2383
1932
  const mod = await import(fileUrl);
2384
1933
  const config = mod.default ?? mod;
2385
1934
  return AgentVConfigSchema.parse(config);
@@ -2462,87 +2011,14 @@ function buildPrompt(criteria, question, referenceAnswer) {
2462
2011
  }
2463
2012
  return parts.join("\n");
2464
2013
  }
2465
- async function scanRepoDeps(evalFilePaths) {
2466
- const seen = /* @__PURE__ */ new Map();
2467
- const errors = [];
2468
- for (const filePath of evalFilePaths) {
2469
- try {
2470
- const repos = await extractReposFromEvalFile(filePath);
2471
- for (const repo of repos) {
2472
- if (!repo.repo) continue;
2473
- const checkoutRef = getRepoCheckoutRef(repo);
2474
- const ref = checkoutRef === "HEAD" ? void 0 : checkoutRef;
2475
- const key = `${normalizeRepoIdentity(repo.repo)}\0${ref ?? ""}`;
2476
- const existing = seen.get(key);
2477
- if (existing) {
2478
- existing.usedBy.push(filePath);
2479
- } else {
2480
- seen.set(key, {
2481
- url: resolveRepoCloneUrl(repo.repo),
2482
- ref,
2483
- sparse: repo.sparse,
2484
- ancestor: repo.ancestor,
2485
- usedBy: [filePath]
2486
- });
2487
- }
2488
- }
2489
- } catch (err) {
2490
- errors.push({
2491
- file: filePath,
2492
- message: err instanceof Error ? err.message : String(err)
2493
- });
2494
- }
2495
- }
2496
- return { repos: [...seen.values()], errors };
2497
- }
2498
- async function extractReposFromEvalFile(filePath) {
2499
- const content = await readFile2(filePath, "utf8");
2500
- const parsed = interpolateEnv(parseYamlValue(content), process.env);
2501
- if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
2502
- const obj = parsed;
2503
- const evalFileDir = path4.dirname(path4.resolve(filePath));
2504
- const repos = [];
2505
- const suiteRepos = await extractReposFromWorkspaceRaw(obj.workspace, evalFileDir);
2506
- repos.push(...suiteRepos);
2507
- const tests = Array.isArray(obj.tests) ? obj.tests : [];
2508
- for (const test of tests) {
2509
- if (test && typeof test === "object" && !Array.isArray(test)) {
2510
- const testObj = test;
2511
- const testRepos = await extractReposFromWorkspaceRaw(testObj.workspace, evalFileDir);
2512
- repos.push(...testRepos);
2513
- }
2514
- }
2515
- return repos;
2516
- }
2517
- async function extractReposFromWorkspaceRaw(raw, evalFileDir) {
2518
- if (typeof raw === "string") {
2519
- const workspaceFilePath = path4.resolve(evalFileDir, raw);
2520
- const content = await readFile2(workspaceFilePath, "utf8");
2521
- const parsed = interpolateEnv(parseYamlValue(content), process.env);
2522
- if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
2523
- return extractReposFromObject(parsed);
2524
- }
2525
- if (raw && typeof raw === "object" && !Array.isArray(raw)) {
2526
- return extractReposFromObject(raw);
2527
- }
2528
- return [];
2529
- }
2530
- function extractReposFromObject(obj) {
2531
- const rawRepos = Array.isArray(obj.repos) ? obj.repos : [];
2532
- const result = [];
2533
- for (const r of rawRepos) {
2534
- const parsed = parseRepoConfig(r);
2535
- if (parsed?.repo) {
2536
- result.push(parsed);
2537
- }
2538
- }
2539
- return result;
2540
- }
2541
2014
  var execFileAsync = promisify(execFile);
2542
2015
  var RESULTS_REPO_RESULTS_DIR = ".agentv/results";
2543
- var RESULTS_REPO_RUNS_DIR = "runs";
2544
- var RESULTS_REPO_METADATA_DIR = "metadata";
2545
- var RESULTS_REPO_TRACKED_DIRS = [RESULTS_REPO_RUNS_DIR, RESULTS_REPO_METADATA_DIR];
2016
+ var RESULTS_REPO_RUNS_DIR = ".";
2017
+ var RESULTS_REPO_INDEXES_DIR = ".indexes";
2018
+ var RESULTS_REPO_CACHE_DIR = ".cache";
2019
+ var RESULTS_REPO_TRACKED_DOT_DIRS = [RESULTS_REPO_INDEXES_DIR, RESULTS_REPO_CACHE_DIR];
2020
+ var GIT_RESULTS_INDEX_CACHE_SCHEMA_VERSION = "agentv.git_results_index_cache.v1";
2021
+ var GIT_RESULTS_INDEX_LAYOUT_VERSION = "agentv.results_repo_branch_root.v1";
2546
2022
  var FALLBACK_RESULTS_REPO_COMMIT_EMAIL = "agentv@results-repo";
2547
2023
  var FALLBACK_RESULTS_REPO_COMMIT_NAME = "AgentV Results";
2548
2024
  var GIT_COMMIT_IDENTITY_ENV_KEYS = [
@@ -2563,121 +2039,11 @@ var GIT_EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
2563
2039
  var RESULTS_REPO_GENESIS_MESSAGE = "chore(results): initialize AgentV results branch";
2564
2040
  var RESULTS_REPO_GENESIS_DATE = "@0 +0000";
2565
2041
  var RESULT_INDEX_FILENAME2 = "index.jsonl";
2042
+ var RUN_INTERNAL_DIRNAME = ".internal";
2566
2043
  var RESULTS_REPO_GITATTRIBUTES_FILE = ".gitattributes";
2567
2044
  var RESULTS_REPO_GITATTRIBUTES_CONTENT = `# Managed by AgentV. Artifact-aware merge so results sync never force-pushes.
2568
- # Append-only run index: union concurrent appends (lines are orthogonal).
2569
- index.jsonl merge=union
2570
- # Editable run overlay (tags/feedback): 3-way JSON set/field union via the
2571
- # agentv-json driver; a genuine scalar conflict falls through to a human merge.
2572
- metadata/runs/**/*.json merge=agentv-json
2573
- `;
2574
- var RESULTS_JSON_MERGE_DRIVER_NAME = "agentv-json";
2575
- var RESULTS_JSON_MERGE_DRIVER_SCRIPT = `#!/usr/bin/env node
2576
- // AgentV results overlay merge driver (merge=agentv-json).
2577
- //
2578
- // Performs a 3-way merge of AgentV's editable JSON overlay files (run tags /
2579
- // feedback). Tag lists merge as a 3-way set so concurrent add/remove are
2580
- // commutative; display-only bookkeeping scalars that always differ between
2581
- // writers (updated_at) are regenerated to the larger value instead of
2582
- // conflicting. Content-derived concurrency tokens (tag_revision) are dropped on
2583
- // merge so the reader recomputes a token matching the merged content rather than
2584
- // carrying a stale token that could bypass optimistic-concurrency checks. Any
2585
- // other genuinely diverging scalar exits non-zero, leaving the file conflicted
2586
- // so the caller can route it to a human GitHub merge.
2587
- //
2588
- // Invoked by git as: node json-merge-driver.mjs %O %A %B %P
2589
- // %O ancestor, %A current (overwritten with the result), %B other, %P path.
2590
- import { readFileSync, writeFileSync } from 'node:fs';
2591
-
2592
- const [, , ancestorFile, currentFile, otherFile] = process.argv;
2593
- const CONFLICT = Symbol('conflict');
2594
- // Scalars that legitimately differ on every write and carry no merge meaning.
2595
- const REGENERABLE_SCALARS = new Set(['updated_at']);
2596
- // Content-derived tokens (a hash of {tags, updated_at}) that guard optimistic
2597
- // concurrency. Carrying either side's pre-merge token forward would let a stale
2598
- // client whose token happens to match bypass the concurrency check and silently
2599
- // overwrite the merged set, so we drop them and let the reader recompute a token
2600
- // that matches the merged content.
2601
- const DROP_ON_MERGE = new Set(['tag_revision']);
2602
-
2603
- function readJson(file) {
2604
- try {
2605
- return JSON.parse(readFileSync(file, 'utf8'));
2606
- } catch {
2607
- return undefined;
2608
- }
2609
- }
2610
-
2611
- function isPrimitiveArray(value) {
2612
- return (
2613
- Array.isArray(value) &&
2614
- value.every((entry) => entry === null || ['string', 'number', 'boolean'].includes(typeof entry))
2615
- );
2616
- }
2617
-
2618
- function isPlainObject(value) {
2619
- return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
2620
- }
2621
-
2622
- function mergeSet(base, a, b) {
2623
- const baseSet = new Set((isPrimitiveArray(base) ? base : []).map(String));
2624
- const aSet = new Set(a.map(String));
2625
- const bSet = new Set(b.map(String));
2626
- const removed = new Set();
2627
- for (const value of baseSet) {
2628
- if (!aSet.has(value) || !bSet.has(value)) removed.add(value);
2629
- }
2630
- const merged = [];
2631
- const seen = new Set();
2632
- for (const value of [...a, ...b]) {
2633
- const key = String(value);
2634
- if (!removed.has(key) && !seen.has(key)) {
2635
- seen.add(key);
2636
- merged.push(value);
2637
- }
2638
- }
2639
- return merged;
2640
- }
2641
-
2642
- function merge3(base, a, b, key) {
2643
- if (JSON.stringify(a) === JSON.stringify(b)) return a;
2644
- if (JSON.stringify(a) === JSON.stringify(base)) return b;
2645
- if (JSON.stringify(b) === JSON.stringify(base)) return a;
2646
- if (isPrimitiveArray(a) && isPrimitiveArray(b)) return mergeSet(base, a, b);
2647
- if (isPlainObject(a) && isPlainObject(b)) {
2648
- const keys = new Set([
2649
- ...Object.keys(a),
2650
- ...Object.keys(b),
2651
- ...(isPlainObject(base) ? Object.keys(base) : []),
2652
- ]);
2653
- const out = {};
2654
- for (const childKey of keys) {
2655
- if (DROP_ON_MERGE.has(childKey)) continue;
2656
- const merged = merge3(
2657
- isPlainObject(base) ? base[childKey] : undefined,
2658
- a[childKey],
2659
- b[childKey],
2660
- childKey,
2661
- );
2662
- if (merged === CONFLICT) return CONFLICT;
2663
- if (merged !== undefined) out[childKey] = merged;
2664
- }
2665
- return out;
2666
- }
2667
- if (REGENERABLE_SCALARS.has(key)) {
2668
- return [a, b].filter((value) => value !== undefined).sort().pop();
2669
- }
2670
- return CONFLICT;
2671
- }
2672
-
2673
- const ancestor = readJson(ancestorFile) ?? {};
2674
- const current = readJson(currentFile);
2675
- const other = readJson(otherFile);
2676
- if (current === undefined || other === undefined) process.exit(1);
2677
- const result = merge3(ancestor, current, other, '');
2678
- if (result === CONFLICT) process.exit(1);
2679
- writeFileSync(currentFile, \`\${JSON.stringify(result, null, 2)}\\n\`);
2680
- process.exit(0);
2045
+ ${RESULTS_REPO_INDEXES_DIR}/*.jsonl merge=union
2046
+ */.internal/index.jsonl merge=union
2681
2047
  `;
2682
2048
  var activeResultsRepoSyncs = /* @__PURE__ */ new Set();
2683
2049
  function sanitizeRepoSlug(repo) {
@@ -2693,37 +2059,34 @@ function withFriendlyGitHubAuthError(error) {
2693
2059
  }
2694
2060
  function expandHome(p) {
2695
2061
  if (p === "~" || p.startsWith("~/") || p.startsWith("~\\")) {
2696
- return path5.join(os.homedir(), p.slice(1));
2062
+ return path3.join(os.homedir(), p.slice(1));
2697
2063
  }
2698
2064
  return p;
2699
2065
  }
2700
2066
  function resolveLocalPath(p, baseDir) {
2701
2067
  const expanded = expandHome(p);
2702
- return path5.isAbsolute(expanded) ? expanded : path5.resolve(baseDir, expanded);
2068
+ return path3.isAbsolute(expanded) ? expanded : path3.resolve(baseDir, expanded);
2069
+ }
2070
+ function isNormalizedResultsConfig(config) {
2071
+ return typeof config.storageBranchWorktree === "boolean";
2703
2072
  }
2704
2073
  function normalizeResultsConfig(config, options) {
2074
+ if (isNormalizedResultsConfig(config)) {
2075
+ return config;
2076
+ }
2705
2077
  const baseDir = options?.baseDir ?? process.cwd();
2706
2078
  const repoUrl = (config.repo_url ?? config.repo)?.trim();
2707
- const repoPath = config.repo_path?.trim();
2708
- const explicitClonePath = config.path?.trim();
2079
+ const explicitPath = config.path?.trim();
2080
+ const repoPath = config.repo_path?.trim() ?? (repoUrl ? void 0 : explicitPath);
2081
+ const explicitClonePath = repoUrl ? explicitPath : void 0;
2709
2082
  const repo = repoUrl ?? repoPath ?? "";
2710
2083
  const branch = config.branch?.trim() || (repoPath ? DEFAULT_RESULTS_BRANCH : void 0);
2711
2084
  const useStorageBranchWorktree = Boolean(repoPath || repoUrl && explicitClonePath && branch);
2712
2085
  const remote = config.remote?.trim() || "origin";
2713
- const autoPush = config.sync?.auto_push ?? config.auto_push === true;
2714
- const requirePush = config.sync?.require_push === true;
2715
- const configuredPushConflictPolicy = config.sync?.push_conflict_policy;
2716
- if (configuredPushConflictPolicy === "backup_and_force_push") {
2717
- throw new Error(
2718
- "results.sync.push_conflict_policy: 'backup_and_force_push' is no longer supported. Remove the field or set it to 'block'; AgentV never force-pushes result branches."
2719
- );
2720
- }
2721
- if (configuredPushConflictPolicy !== void 0 && configuredPushConflictPolicy !== "block") {
2722
- throw new Error("results.sync.push_conflict_policy must be 'block'");
2723
- }
2724
- const pushConflictPolicy = configuredPushConflictPolicy ?? "block";
2086
+ const autoPush = config.auto_push === true;
2087
+ const requirePush = config.require_push === true;
2725
2088
  const resolvedRepoPath = repoPath ? resolveLocalPath(repoPath, baseDir) : void 0;
2726
- const resolvedPath = explicitClonePath ? resolveLocalPath(explicitClonePath, baseDir) : repoUrl ? path5.join(getAgentvDataDir(), "results", sanitizeRepoSlug(repoUrl)) : resolvedRepoPath ?? path5.join(getAgentvDataDir(), "results", sanitizeRepoSlug(repo));
2089
+ const resolvedPath = explicitClonePath ? resolveLocalPath(explicitClonePath, baseDir) : repoUrl ? path3.join(getAgentvDataDir(), "results", sanitizeRepoSlug(repoUrl)) : resolvedRepoPath ?? path3.join(getAgentvDataDir(), "results", sanitizeRepoSlug(repo));
2727
2090
  return {
2728
2091
  mode: "github",
2729
2092
  repo,
@@ -2734,8 +2097,7 @@ function normalizeResultsConfig(config, options) {
2734
2097
  path: resolvedPath,
2735
2098
  auto_push: autoPush,
2736
2099
  require_push: requirePush,
2737
- push_conflict_policy: pushConflictPolicy,
2738
- branch_prefix: config.branch_prefix?.trim() || "eval-results",
2100
+ push_conflict_policy: "block",
2739
2101
  storageBranchWorktree: useStorageBranchWorktree
2740
2102
  };
2741
2103
  }
@@ -2754,11 +2116,11 @@ function resolveResultsRepoUrl(repo) {
2754
2116
  return trimmed;
2755
2117
  }
2756
2118
  function getResultsRepoLocalPaths(repo) {
2757
- const rootDir = path5.join(getAgentvDataDir(), "cache", "results-repo", sanitizeRepoSlug(repo));
2119
+ const rootDir = path3.join(getAgentvDataDir(), "cache", "results-repo", sanitizeRepoSlug(repo));
2758
2120
  return {
2759
2121
  rootDir,
2760
- repoDir: path5.join(rootDir, "repo"),
2761
- statusFile: path5.join(rootDir, "status.json")
2122
+ repoDir: path3.join(rootDir, "repo"),
2123
+ statusFile: path3.join(rootDir, "status.json")
2762
2124
  };
2763
2125
  }
2764
2126
  function usesStorageBranchWorktree(config) {
@@ -2775,7 +2137,7 @@ function readPersistedStatus(statusFile) {
2775
2137
  }
2776
2138
  }
2777
2139
  function writePersistedStatus(statusFile, status) {
2778
- mkdirSync(path5.dirname(statusFile), { recursive: true });
2140
+ mkdirSync(path3.dirname(statusFile), { recursive: true });
2779
2141
  writeFileSync(statusFile, `${JSON.stringify(status, null, 2)}
2780
2142
  `, "utf8");
2781
2143
  }
@@ -3018,37 +2380,17 @@ async function ensureResultsMergeConfig(repoDir) {
3018
2380
  try {
3019
2381
  const { stdout } = await runGit(["rev-parse", "--git-common-dir"], { cwd: repoDir });
3020
2382
  const resolved = stdout.trim();
3021
- gitDir = resolved ? path5.isAbsolute(resolved) ? resolved : path5.join(repoDir, resolved) : path5.join(repoDir, ".git");
2383
+ gitDir = resolved ? path3.isAbsolute(resolved) ? resolved : path3.join(repoDir, resolved) : path3.join(repoDir, ".git");
3022
2384
  } catch {
3023
2385
  return;
3024
2386
  }
3025
- const driverScriptPath = path5.join(gitDir, "agentv", "json-merge-driver.mjs");
3026
- let scriptCurrent = false;
3027
- try {
3028
- scriptCurrent = readFileSync2(driverScriptPath, "utf8") === RESULTS_JSON_MERGE_DRIVER_SCRIPT;
3029
- } catch {
3030
- scriptCurrent = false;
3031
- }
3032
- if (!scriptCurrent) {
3033
- mkdirSync(path5.dirname(driverScriptPath), { recursive: true });
3034
- writeFileSync(driverScriptPath, RESULTS_JSON_MERGE_DRIVER_SCRIPT, { mode: 493 });
3035
- }
3036
- const driverCommand = `node ${JSON.stringify(driverScriptPath)} %O %A %B %P`;
3037
- await runGit(
3038
- ["config", `merge.${RESULTS_JSON_MERGE_DRIVER_NAME}.name`, "AgentV results overlay JSON union"],
3039
- { cwd: repoDir, check: false }
3040
- );
3041
- await runGit(["config", `merge.${RESULTS_JSON_MERGE_DRIVER_NAME}.driver`, driverCommand], {
3042
- cwd: repoDir,
3043
- check: false
3044
- });
3045
2387
  writeResultsInfoAttributes(gitDir);
3046
2388
  }
3047
2389
  var RESULTS_INFO_ATTRIBUTES_BEGIN = "# >>> agentv results merge attributes >>>";
3048
2390
  var RESULTS_INFO_ATTRIBUTES_END = "# <<< agentv results merge attributes <<<";
3049
2391
  function writeResultsInfoAttributes(gitDir) {
3050
- const infoDir = path5.join(gitDir, "info");
3051
- const attributesPath = path5.join(infoDir, "attributes");
2392
+ const infoDir = path3.join(gitDir, "info");
2393
+ const attributesPath = path3.join(infoDir, "attributes");
3052
2394
  const managedBlock = `${RESULTS_INFO_ATTRIBUTES_BEGIN}
3053
2395
  ${RESULTS_REPO_GITATTRIBUTES_CONTENT.trimEnd()}
3054
2396
  ${RESULTS_INFO_ATTRIBUTES_END}
@@ -3084,9 +2426,9 @@ async function addResultsGitattributesToIndex(repoDir, indexEnv) {
3084
2426
  if (existing === RESULTS_REPO_GITATTRIBUTES_CONTENT) {
3085
2427
  return;
3086
2428
  }
3087
- const scratchDir = mkdtempSync(path5.join(os.tmpdir(), "agentv-results-attrs-"));
2429
+ const scratchDir = mkdtempSync(path3.join(os.tmpdir(), "agentv-results-attrs-"));
3088
2430
  try {
3089
- const scratchFile = path5.join(scratchDir, RESULTS_REPO_GITATTRIBUTES_FILE);
2431
+ const scratchFile = path3.join(scratchDir, RESULTS_REPO_GITATTRIBUTES_FILE);
3090
2432
  writeFileSync(scratchFile, RESULTS_REPO_GITATTRIBUTES_CONTENT);
3091
2433
  const { stdout: blob } = await runGit(["hash-object", "-w", "--no-filters", scratchFile], {
3092
2434
  cwd: repoDir
@@ -3105,7 +2447,7 @@ async function addResultsGitattributesToIndex(repoDir, indexEnv) {
3105
2447
  }
3106
2448
  }
3107
2449
  function updateStatusFile(config, patch) {
3108
- const repo = typeof config.repo === "string" ? config.repo : config.repo_url ?? config.repo_path ?? "";
2450
+ const repo = normalizeResultsConfig(config).repo;
3109
2451
  const cachePaths = getResultsRepoLocalPaths(repo);
3110
2452
  const current = readPersistedStatus(cachePaths.statusFile);
3111
2453
  writePersistedStatus(cachePaths.statusFile, {
@@ -3126,9 +2468,9 @@ async function ensureResultsRepoClone(config) {
3126
2468
  const cachePaths = getResultsRepoLocalPaths(normalized.repo);
3127
2469
  const cloneDir = normalized.path;
3128
2470
  mkdirSync(cachePaths.rootDir, { recursive: true });
3129
- mkdirSync(path5.dirname(cloneDir), { recursive: true });
2471
+ mkdirSync(path3.dirname(cloneDir), { recursive: true });
3130
2472
  const cloneMissing = !existsSync(cloneDir);
3131
- const gitDir = path5.join(cloneDir, ".git");
2473
+ const gitDir = path3.join(cloneDir, ".git");
3132
2474
  const cloneEmpty = !cloneMissing && !existsSync(gitDir) && (await readdir(cloneDir)).length === 0;
3133
2475
  if (cloneMissing || cloneEmpty) {
3134
2476
  try {
@@ -3174,7 +2516,6 @@ function getResultsRepoStatus(config) {
3174
2516
  auto_push: normalized.auto_push,
3175
2517
  require_push: normalized.require_push,
3176
2518
  push_conflict_policy: normalized.push_conflict_policy,
3177
- branch_prefix: normalized.branch_prefix,
3178
2519
  local_dir: normalized.path,
3179
2520
  last_synced_at: persisted.last_synced_at,
3180
2521
  last_error: persisted.last_error,
@@ -3295,7 +2636,7 @@ async function getStorageBranchWorktreeDirtyPaths(repoDir, config) {
3295
2636
  const comparisonRef = await gitRefExists(repoDir, localRef) ? localRef : await gitRefExists(repoDir, remoteRef) ? remoteRef : void 0;
3296
2637
  const dirtyPaths = [];
3297
2638
  for (const gitPath of safeDirtyPaths) {
3298
- const absolutePath = path5.join(repoDir, ...gitPath.split("/"));
2639
+ const absolutePath = path3.join(repoDir, ...gitPath.split("/"));
3299
2640
  if (!existsSync(absolutePath)) {
3300
2641
  if (await readGitText(repoDir, comparisonRef, gitPath) !== void 0) {
3301
2642
  dirtyPaths.push(gitPath);
@@ -3401,7 +2742,7 @@ async function hasInProgressGitConflict(repoDir) {
3401
2742
  check: false
3402
2743
  });
3403
2744
  const markerPath = stdout.trim();
3404
- const resolvedMarkerPath = path5.isAbsolute(markerPath) ? markerPath : path5.join(repoDir, markerPath);
2745
+ const resolvedMarkerPath = path3.isAbsolute(markerPath) ? markerPath : path3.join(repoDir, markerPath);
3405
2746
  if (markerPath && existsSync(resolvedMarkerPath)) {
3406
2747
  return true;
3407
2748
  }
@@ -3573,22 +2914,49 @@ function withActionFlags(status, flags) {
3573
2914
  };
3574
2915
  }
3575
2916
  function isSafeResultsRepoPath(p) {
3576
- return p === RESULTS_REPO_GITATTRIBUTES_FILE || RESULTS_REPO_TRACKED_DIRS.some((dir) => p === dir || p.startsWith(`${dir}/`));
3577
- }
3578
- async function existingTrackedResultsDirs(repoDir) {
3579
- const targets = [];
3580
- for (const dir of RESULTS_REPO_TRACKED_DIRS) {
3581
- if (existsSync(path5.join(repoDir, dir))) {
3582
- targets.push(dir);
2917
+ const normalized = p.split(path3.sep).join("/");
2918
+ const segments = normalized.split("/").filter(Boolean);
2919
+ const isRunBundlePath = segments.length >= 2 && !segments[0]?.startsWith(".") && segments[0] !== RESULTS_REPO_GITATTRIBUTES_FILE;
2920
+ return normalized === RESULTS_REPO_GITATTRIBUTES_FILE || RESULTS_REPO_TRACKED_DOT_DIRS.some(
2921
+ (dir) => normalized === dir || normalized.startsWith(`${dir}/`)
2922
+ ) || isRunBundlePath;
2923
+ }
2924
+ async function existingTrackedResultsPathspecs(repoDir) {
2925
+ const targets = /* @__PURE__ */ new Set();
2926
+ if (existsSync(path3.join(repoDir, RESULTS_REPO_GITATTRIBUTES_FILE))) {
2927
+ targets.add(RESULTS_REPO_GITATTRIBUTES_FILE);
2928
+ }
2929
+ for (const dir of RESULTS_REPO_TRACKED_DOT_DIRS) {
2930
+ if (existsSync(path3.join(repoDir, dir))) {
2931
+ targets.add(dir);
3583
2932
  continue;
3584
2933
  }
3585
- const { stdout } = await runGit(["ls-files", "--", dir], { cwd: repoDir, check: false });
3586
- if (stdout.trim().length > 0) {
3587
- targets.push(dir);
2934
+ const { stdout: stdout2 } = await runGit(["ls-files", "--", dir], { cwd: repoDir, check: false });
2935
+ if (stdout2.trim().length > 0) {
2936
+ targets.add(dir);
3588
2937
  }
3589
2938
  }
3590
- return targets;
3591
- }
2939
+ for (const entry of await readdir(repoDir, { withFileTypes: true }).catch(() => [])) {
2940
+ if (entry.name.startsWith(".") || !entry.isDirectory()) {
2941
+ continue;
2942
+ }
2943
+ if (existsSync(path3.join(repoDir, entry.name, "summary.json"))) {
2944
+ targets.add(entry.name);
2945
+ }
2946
+ }
2947
+ const { stdout } = await runGit(["ls-files"], { cwd: repoDir, check: false });
2948
+ for (const file of stdout.split(/\r?\n/)) {
2949
+ const normalized = file.trim();
2950
+ if (!normalized) {
2951
+ continue;
2952
+ }
2953
+ const segments = normalized.split("/");
2954
+ if (segments.length >= 2 && segments[0] && !segments[0].startsWith(".")) {
2955
+ targets.add(segments[0]);
2956
+ }
2957
+ }
2958
+ return [...targets].sort();
2959
+ }
3592
2960
  function areSafeResultsRepoPaths(paths) {
3593
2961
  return paths.length > 0 && paths.every(isSafeResultsRepoPath);
3594
2962
  }
@@ -3638,7 +3006,7 @@ async function getMergeBaseCommit(repoDir, a, b) {
3638
3006
  return exitCode === 0 && sha.length > 0 ? sha : void 0;
3639
3007
  }
3640
3008
  function buildNeedsHumanMergeReason(targetBranch, remoteCommit, localCommit) {
3641
- return `Results branch ${targetBranch} diverged from the remote and could not be auto-merged: a genuine content conflict in the editable overlay remains (remote ${formatShortSha(
3009
+ return `Results branch ${targetBranch} diverged from the remote and could not be auto-merged: a genuine results content conflict remains (remote ${formatShortSha(
3642
3010
  remoteCommit
3643
3011
  )}, local ${formatShortSha(
3644
3012
  localCommit
@@ -3692,13 +3060,7 @@ async function countContributedRunDirs(repoDir, remoteTargetCommit, sourceCommit
3692
3060
  return void 0;
3693
3061
  }
3694
3062
  const { stdout, exitCode } = await runGit(
3695
- [
3696
- "diff",
3697
- "--name-only",
3698
- `${remoteTargetCommit}...${sourceCommit}`,
3699
- "--",
3700
- `${RESULTS_REPO_RUNS_DIR}/`
3701
- ],
3063
+ ["diff", "--name-only", `${remoteTargetCommit}...${sourceCommit}`, "--", RESULTS_REPO_RUNS_DIR],
3702
3064
  { cwd: repoDir, check: false }
3703
3065
  );
3704
3066
  if (exitCode !== 0) {
@@ -3711,8 +3073,8 @@ async function countContributedRunDirs(repoDir, remoteTargetCommit, sourceCommit
3711
3073
  continue;
3712
3074
  }
3713
3075
  const segments = file.split("/");
3714
- if (segments[0] === RESULTS_REPO_RUNS_DIR && segments.length >= 3) {
3715
- runDirs.add(`${segments[0]}/${segments[1]}/${segments[2]}`);
3076
+ if (segments.length >= 2 && segments[0] && !segments[0].startsWith(".")) {
3077
+ runDirs.add(segments[0]);
3716
3078
  }
3717
3079
  }
3718
3080
  return runDirs.size;
@@ -3954,9 +3316,6 @@ async function getResultsRepoSyncStatus(config) {
3954
3316
  }
3955
3317
  try {
3956
3318
  if (usesStorageBranchWorktree(normalized)) {
3957
- await fetchResultsRepo(normalized.path, normalized.remote, normalized.branch).catch(
3958
- () => void 0
3959
- );
3960
3319
  return withGitInspection(
3961
3320
  baseStatus,
3962
3321
  await inspectResultsStorageBranchGit(normalized.path, normalized)
@@ -4331,10 +3690,16 @@ async function syncResultsRepoForProject(config) {
4331
3690
  }
4332
3691
  }
4333
3692
  if (inspection.syncStatus === "dirty") {
4334
- const trackedDirs = await existingTrackedResultsDirs(repoDir);
4335
- await runGit(["add", "--all", "--", ...trackedDirs], { cwd: repoDir });
3693
+ const trackedPathspecs = await existingTrackedResultsPathspecs(repoDir);
3694
+ await runGit(["add", "--all", "--", ...trackedPathspecs], { cwd: repoDir });
4336
3695
  await runGitWithFallbackCommitIdentity(
4337
- ["commit", "-m", "chore(results): sync local result metadata", "--", ...trackedDirs],
3696
+ [
3697
+ "commit",
3698
+ "-m",
3699
+ "chore(results): sync local results branch changes",
3700
+ "--",
3701
+ ...trackedPathspecs
3702
+ ],
4338
3703
  {
4339
3704
  cwd: repoDir
4340
3705
  }
@@ -4552,8 +3917,8 @@ async function prepareResultsRepoBranch(config, branchName) {
4552
3917
  const cloneDir = await ensureResultsRepoClone(normalized);
4553
3918
  const baseBranch = await resolveDefaultBranch(cloneDir);
4554
3919
  await fetchResultsRepo(cloneDir, normalized.remote);
4555
- const worktreeRoot = await mkdtemp(path5.join(os.tmpdir(), "agentv-results-repo-"));
4556
- const worktreeDir = path5.join(worktreeRoot, "repo");
3920
+ const worktreeRoot = await mkdtemp(path3.join(os.tmpdir(), "agentv-results-repo-"));
3921
+ const worktreeDir = path3.join(worktreeRoot, "repo");
4557
3922
  await runGit(
4558
3923
  ["worktree", "add", "-B", branchName, worktreeDir, `${normalized.remote}/${baseBranch}`],
4559
3924
  {
@@ -4575,12 +3940,12 @@ async function prepareResultsRepoBranch(config, branchName) {
4575
3940
  }
4576
3941
  async function stageResultsArtifacts(params) {
4577
3942
  rmSync(params.destinationDir, { recursive: true, force: true });
4578
- mkdirSync(path5.dirname(params.destinationDir), { recursive: true });
3943
+ mkdirSync(path3.dirname(params.destinationDir), { recursive: true });
4579
3944
  await cp(params.sourceDir, params.destinationDir, { recursive: true });
4580
3945
  }
4581
3946
  function resolveResultsRepoRunsDir(config) {
4582
3947
  const normalized = normalizeResultsConfig(config);
4583
- return path5.join(normalized.path, RESULTS_REPO_RESULTS_DIR);
3948
+ return path3.join(normalized.path, RESULTS_REPO_RESULTS_DIR);
4584
3949
  }
4585
3950
  async function directorySizeBytes(targetPath) {
4586
3951
  const entry = await stat(targetPath);
@@ -4589,7 +3954,7 @@ async function directorySizeBytes(targetPath) {
4589
3954
  }
4590
3955
  let total = 0;
4591
3956
  for (const child of await readdir(targetPath, { withFileTypes: true })) {
4592
- total += await directorySizeBytes(path5.join(targetPath, child.name));
3957
+ total += await directorySizeBytes(path3.join(targetPath, child.name));
4593
3958
  }
4594
3959
  return total;
4595
3960
  }
@@ -4656,7 +4021,7 @@ async function assertValidResultsBranchName(repoDir, branch) {
4656
4021
  await runGit(["check-ref-format", "--branch", branch], { cwd: repoDir });
4657
4022
  }
4658
4023
  function normalizeDestinationPath(destinationPath) {
4659
- const normalized = destinationPath.split(path5.sep).join("/");
4024
+ const normalized = destinationPath.split(path3.sep).join("/");
4660
4025
  const segments = normalized.split("/").filter(Boolean);
4661
4026
  if (segments.length === 0 || normalized.startsWith("/") || segments.some((segment) => segment === "..") || normalized.split("").some((char) => {
4662
4027
  const code = char.charCodeAt(0);
@@ -4670,7 +4035,7 @@ async function listSourceFiles(sourceDir) {
4670
4035
  const entries = [];
4671
4036
  async function visit(dir) {
4672
4037
  for (const entry of await readdir(dir, { withFileTypes: true })) {
4673
- const absolutePath = path5.join(dir, entry.name);
4038
+ const absolutePath = path3.join(dir, entry.name);
4674
4039
  if (entry.isDirectory()) {
4675
4040
  await visit(absolutePath);
4676
4041
  } else if (entry.isFile() || entry.isSymbolicLink()) {
@@ -4682,20 +4047,18 @@ async function listSourceFiles(sourceDir) {
4682
4047
  entries.sort();
4683
4048
  return entries;
4684
4049
  }
4685
- function isRecord3(value) {
4050
+ function isRecord2(value) {
4686
4051
  return !!value && typeof value === "object" && !Array.isArray(value);
4687
4052
  }
4688
4053
  function artifactSidecarPointers(record) {
4689
- if (!isRecord3(record) || !isRecord3(record.artifact_pointers)) {
4054
+ if (!isRecord2(record) || !isRecord2(record.artifact_pointers)) {
4690
4055
  return [];
4691
4056
  }
4692
4057
  const pointers = [];
4693
- for (const pointer of Object.values(record.artifact_pointers)) {
4694
- if (!isRecord3(pointer)) {
4695
- continue;
4696
- }
4058
+ const pointer = record.artifact_pointers.transcript;
4059
+ if (isRecord2(pointer)) {
4697
4060
  if (pointer.ref !== AGENTV_RESULTS_ARTIFACTS_REF || typeof pointer.path !== "string") {
4698
- continue;
4061
+ return pointers;
4699
4062
  }
4700
4063
  pointers.push({
4701
4064
  path: pointer.path,
@@ -4706,15 +4069,56 @@ function artifactSidecarPointers(record) {
4706
4069
  return pointers;
4707
4070
  }
4708
4071
  function artifactSidecarKey(destinationPath, pointerPath) {
4709
- return path5.posix.join(
4072
+ return path3.posix.join(
4710
4073
  RESULTS_REPO_RUNS_DIR,
4711
4074
  normalizeDestinationPath(destinationPath),
4712
4075
  normalizeDestinationPath(pointerPath)
4713
4076
  );
4714
4077
  }
4078
+ function isDeprecatedTraceArtifactPath(relativePath) {
4079
+ return relativePath === "trace.json" || relativePath.endsWith("/trace.json");
4080
+ }
4081
+ function isResultManifestFilename(filename) {
4082
+ return filename === RESULT_INDEX_FILENAME2;
4083
+ }
4084
+ function safeLocalSummaryManifestPath(sourceDir, manifestPath) {
4085
+ if (typeof manifestPath !== "string" || manifestPath.trim().length === 0) {
4086
+ return void 0;
4087
+ }
4088
+ if (path3.isAbsolute(manifestPath)) {
4089
+ return void 0;
4090
+ }
4091
+ const normalized = path3.normalize(manifestPath);
4092
+ if (normalized.startsWith("..") || path3.isAbsolute(normalized)) {
4093
+ return void 0;
4094
+ }
4095
+ return path3.join(sourceDir, normalized);
4096
+ }
4097
+ function resolveLocalResultManifestPath(sourceDir) {
4098
+ try {
4099
+ const summary = JSON.parse(readFileSync2(path3.join(sourceDir, "summary.json"), "utf8"));
4100
+ const manifestPath2 = safeLocalSummaryManifestPath(
4101
+ sourceDir,
4102
+ summary.index_path ?? summary.manifest_path
4103
+ );
4104
+ if (manifestPath2 && existsSync(manifestPath2)) {
4105
+ return manifestPath2;
4106
+ }
4107
+ } catch {
4108
+ }
4109
+ const manifestPath = path3.join(sourceDir, RUN_INTERNAL_DIRNAME, RESULT_INDEX_FILENAME2);
4110
+ if (existsSync(manifestPath)) {
4111
+ return manifestPath;
4112
+ }
4113
+ const legacyManifestPath = path3.join(sourceDir, RESULT_INDEX_FILENAME2);
4114
+ if (existsSync(legacyManifestPath)) {
4115
+ return legacyManifestPath;
4116
+ }
4117
+ return void 0;
4118
+ }
4715
4119
  function collectArtifactSidecarPointers(sourceDir) {
4716
- const indexPath = path5.join(sourceDir, RESULT_INDEX_FILENAME2);
4717
- if (!existsSync(indexPath)) {
4120
+ const indexPath = resolveLocalResultManifestPath(sourceDir);
4121
+ if (!indexPath) {
4718
4122
  return [];
4719
4123
  }
4720
4124
  const pointers = [];
@@ -4732,16 +4136,16 @@ function collectArtifactSidecarPointers(sourceDir) {
4732
4136
  }
4733
4137
  function resolveArtifactPointerSourcePath(sourceDir, pointerPath) {
4734
4138
  const normalizedPointerPath = normalizeDestinationPath(pointerPath);
4735
- const sourceRoot = path5.resolve(sourceDir);
4736
- const sourcePath = path5.resolve(sourceRoot, ...normalizedPointerPath.split("/"));
4737
- const relative = path5.relative(sourceRoot, sourcePath);
4738
- if (relative.startsWith("..") || path5.isAbsolute(relative)) {
4139
+ const sourceRoot = path3.resolve(sourceDir);
4140
+ const sourcePath = path3.resolve(sourceRoot, ...normalizedPointerPath.split("/"));
4141
+ const relative = path3.relative(sourceRoot, sourcePath);
4142
+ if (relative.startsWith("..") || path3.isAbsolute(relative)) {
4739
4143
  throw new Error(`Artifact pointer path escapes run directory: ${pointerPath}`);
4740
4144
  }
4741
4145
  return sourcePath;
4742
4146
  }
4743
4147
  function verifyArtifactPointerChecksum(pointer, content) {
4744
- const sha256 = createHash2("sha256").update(content).digest("hex");
4148
+ const sha256 = createHash("sha256").update(content).digest("hex");
4745
4149
  if (pointer.sha256 && pointer.sha256 !== sha256) {
4746
4150
  throw new Error(`Artifact pointer checksum mismatch for ${pointer.path}`);
4747
4151
  }
@@ -4753,7 +4157,7 @@ async function prepareArtifactSidecar(params) {
4753
4157
  if (params.pointers.length === 0) {
4754
4158
  return void 0;
4755
4159
  }
4756
- const sidecarRoot = await mkdtemp(path5.join(os.tmpdir(), "agentv-artifact-sidecar-"));
4160
+ const sidecarRoot = await mkdtemp(path3.join(os.tmpdir(), "agentv-artifact-sidecar-"));
4757
4161
  const copied = /* @__PURE__ */ new Set();
4758
4162
  const contentByPath = /* @__PURE__ */ new Map();
4759
4163
  try {
@@ -4769,8 +4173,8 @@ async function prepareArtifactSidecar(params) {
4769
4173
  if (copied.has(relativePointerPath)) {
4770
4174
  continue;
4771
4175
  }
4772
- const destinationPath = path5.join(sidecarRoot, ...relativePointerPath.split("/"));
4773
- mkdirSync(path5.dirname(destinationPath), { recursive: true });
4176
+ const destinationPath = path3.join(sidecarRoot, ...relativePointerPath.split("/"));
4177
+ mkdirSync(path3.dirname(destinationPath), { recursive: true });
4774
4178
  writeFileSync(destinationPath, content);
4775
4179
  copied.add(relativePointerPath);
4776
4180
  }
@@ -4797,22 +4201,24 @@ function rewritePublishedIndexLine(line, destinationPath) {
4797
4201
  } catch {
4798
4202
  return line;
4799
4203
  }
4800
- if (!isRecord3(record) || !isRecord3(record.artifact_pointers)) {
4204
+ if (!isRecord2(record) || !isRecord2(record.artifact_pointers)) {
4801
4205
  return line;
4802
4206
  }
4803
4207
  let changed = false;
4804
- for (const pointer of Object.values(record.artifact_pointers)) {
4805
- if (!isRecord3(pointer)) {
4806
- continue;
4807
- }
4808
- if (pointer.ref !== AGENTV_RESULTS_ARTIFACTS_REF || typeof pointer.path !== "string") {
4809
- continue;
4810
- }
4208
+ const pointer = record.artifact_pointers.transcript;
4209
+ if (isRecord2(pointer) && pointer.ref === AGENTV_RESULTS_ARTIFACTS_REF && typeof pointer.path === "string") {
4811
4210
  const key = artifactSidecarKey(destinationPath, pointer.path);
4812
4211
  if (pointer.key !== key) {
4813
4212
  pointer.key = key;
4814
4213
  changed = true;
4815
4214
  }
4215
+ if (Object.keys(record.artifact_pointers).length !== 1 || record.artifact_pointers.transcript !== pointer) {
4216
+ changed = true;
4217
+ }
4218
+ record.artifact_pointers = { transcript: pointer };
4219
+ } else {
4220
+ record.artifact_pointers = void 0;
4221
+ changed = true;
4816
4222
  }
4817
4223
  return changed ? JSON.stringify(record) : line;
4818
4224
  }
@@ -4820,26 +4226,29 @@ async function preparePublishedResultsSource(params) {
4820
4226
  if (params.pointers.length === 0) {
4821
4227
  return void 0;
4822
4228
  }
4823
- const publishedRoot = await mkdtemp(path5.join(os.tmpdir(), "agentv-results-published-"));
4229
+ const publishedRoot = await mkdtemp(path3.join(os.tmpdir(), "agentv-results-published-"));
4824
4230
  const omittedPaths = new Set(
4825
4231
  params.pointers.map((pointer) => normalizeDestinationPath(pointer.path))
4826
4232
  );
4827
4233
  try {
4828
4234
  const sourceFiles = await listSourceFiles(params.sourceDir);
4829
4235
  for (const sourceFile of sourceFiles) {
4830
- const relativeFile = path5.relative(params.sourceDir, sourceFile).split(path5.sep).join("/");
4831
- const destinationFile = path5.join(publishedRoot, ...relativeFile.split("/"));
4832
- if (relativeFile === RESULT_INDEX_FILENAME2) {
4236
+ const relativeFile = path3.relative(params.sourceDir, sourceFile).split(path3.sep).join("/");
4237
+ const destinationFile = path3.join(publishedRoot, ...relativeFile.split("/"));
4238
+ if (isResultManifestFilename(path3.posix.basename(relativeFile))) {
4833
4239
  const original = readFileSync2(sourceFile, "utf8");
4834
4240
  const rewritten = original.split(/\r?\n/).map((line) => rewritePublishedIndexLine(line, params.destinationPath)).join("\n");
4835
- mkdirSync(path5.dirname(destinationFile), { recursive: true });
4241
+ mkdirSync(path3.dirname(destinationFile), { recursive: true });
4836
4242
  writeFileSync(destinationFile, rewritten);
4837
4243
  continue;
4838
4244
  }
4245
+ if (isDeprecatedTraceArtifactPath(relativeFile)) {
4246
+ continue;
4247
+ }
4839
4248
  if (omittedPaths.has(relativeFile)) {
4840
4249
  continue;
4841
4250
  }
4842
- mkdirSync(path5.dirname(destinationFile), { recursive: true });
4251
+ mkdirSync(path3.dirname(destinationFile), { recursive: true });
4843
4252
  await cp(sourceFile, destinationFile, { dereference: false });
4844
4253
  }
4845
4254
  } catch (error) {
@@ -4906,8 +4315,8 @@ async function commitStorageBranchWorktreePaths(params) {
4906
4315
  normalized
4907
4316
  });
4908
4317
  }
4909
- const indexRoot = await mkdtemp(path5.join(os.tmpdir(), "agentv-results-index-"));
4910
- const indexFile = path5.join(indexRoot, "index");
4318
+ const indexRoot = await mkdtemp(path3.join(os.tmpdir(), "agentv-results-index-"));
4319
+ const indexFile = path3.join(indexRoot, "index");
4911
4320
  const indexEnv = { GIT_INDEX_FILE: indexFile };
4912
4321
  try {
4913
4322
  if (base.baseRef) {
@@ -4917,7 +4326,7 @@ async function commitStorageBranchWorktreePaths(params) {
4917
4326
  }
4918
4327
  await addResultsGitattributesToIndex(params.repoDir, indexEnv);
4919
4328
  for (const gitPath of paths) {
4920
- const sourcePath = path5.join(params.repoDir, ...gitPath.split("/"));
4329
+ const sourcePath = path3.join(params.repoDir, ...gitPath.split("/"));
4921
4330
  if (!existsSync(sourcePath)) {
4922
4331
  await runGit(["update-index", "--force-remove", "--", gitPath], {
4923
4332
  cwd: params.repoDir,
@@ -4983,7 +4392,7 @@ async function commitResultsRunWithTemporaryIndex(params) {
4983
4392
  await assertValidResultsBranchName(params.repoDir, normalized.branch);
4984
4393
  await ensureResultsBranchNotCheckedOut(params.repoDir, normalized);
4985
4394
  const destinationRunPath = normalizeDestinationPath(params.destinationPath);
4986
- const destinationTreePath = path5.posix.join(RESULTS_REPO_RUNS_DIR, destinationRunPath);
4395
+ const destinationTreePath = path3.posix.join(RESULTS_REPO_RUNS_DIR, destinationRunPath);
4987
4396
  let base = await resolveStorageBranchBase({
4988
4397
  repoDir: params.repoDir,
4989
4398
  normalized,
@@ -4997,8 +4406,8 @@ async function commitResultsRunWithTemporaryIndex(params) {
4997
4406
  preferRemote: params.preferRemoteBase
4998
4407
  });
4999
4408
  }
5000
- const indexRoot = await mkdtemp(path5.join(os.tmpdir(), "agentv-results-index-"));
5001
- const indexFile = path5.join(indexRoot, "index");
4409
+ const indexRoot = await mkdtemp(path3.join(os.tmpdir(), "agentv-results-index-"));
4410
+ const indexFile = path3.join(indexRoot, "index");
5002
4411
  const indexEnv = { GIT_INDEX_FILE: indexFile };
5003
4412
  try {
5004
4413
  if (base.baseRef) {
@@ -5021,8 +4430,8 @@ async function commitResultsRunWithTemporaryIndex(params) {
5021
4430
  }
5022
4431
  const sourceFiles = await listSourceFiles(params.sourceDir);
5023
4432
  for (const sourceFile of sourceFiles) {
5024
- const relativeFile = path5.relative(params.sourceDir, sourceFile).split(path5.sep).join("/");
5025
- const destinationFile = path5.posix.join(destinationTreePath, relativeFile);
4433
+ const relativeFile = path3.relative(params.sourceDir, sourceFile).split(path3.sep).join("/");
4434
+ const destinationFile = path3.posix.join(destinationTreePath, relativeFile);
5026
4435
  const fileStat = await lstat(sourceFile);
5027
4436
  let mode = fileStat.mode & 73 ? "100755" : "100644";
5028
4437
  let hashInputPath = sourceFile;
@@ -5295,17 +4704,70 @@ async function directPushResultsWithDetails(params) {
5295
4704
  await sidecar?.cleanup().catch(() => void 0);
5296
4705
  }
5297
4706
  }
5298
- function buildGitRunId(relativeRunPath) {
5299
- const normalized = relativeRunPath.split(path5.sep).join("/");
5300
- const segments = normalized.split("/").filter(Boolean);
5301
- if (segments.length >= 2) {
5302
- const experiment = segments.slice(0, -1).join("/");
5303
- const timestamp = segments.at(-1);
5304
- if (experiment === "default") {
5305
- return timestamp ?? normalized;
4707
+ function safeGitSummaryManifestPath(runDir, manifestPath) {
4708
+ if (typeof manifestPath !== "string" || manifestPath.trim().length === 0) {
4709
+ return void 0;
4710
+ }
4711
+ if (manifestPath.startsWith("/")) {
4712
+ return void 0;
4713
+ }
4714
+ const normalized = path3.posix.normalize(manifestPath);
4715
+ if (normalized === ".." || normalized.startsWith("../")) {
4716
+ return void 0;
4717
+ }
4718
+ return path3.posix.join(runDir, normalized);
4719
+ }
4720
+ function buildGitManifestPaths(treePaths, summaryByPath) {
4721
+ const treePathSet = new Set(treePaths);
4722
+ const manifestByRunDir = /* @__PURE__ */ new Map();
4723
+ for (const [summaryPath, summary] of summaryByPath) {
4724
+ const runDir = path3.posix.dirname(summaryPath);
4725
+ const relativeRunPath = path3.posix.relative(RESULTS_REPO_RUNS_DIR, runDir);
4726
+ if (!isV2ResultsRepoRunPath(relativeRunPath)) {
4727
+ continue;
4728
+ }
4729
+ const manifestPath = safeGitSummaryManifestPath(runDir, summary.index_path ?? summary.manifest_path) ?? fallbackGitManifestPath(runDir, treePathSet);
4730
+ if (manifestPath && treePathSet.has(manifestPath)) {
4731
+ manifestByRunDir.set(runDir, manifestPath);
5306
4732
  }
5307
- return `${experiment}::${timestamp}`;
5308
4733
  }
4734
+ for (const treePath of treePaths) {
4735
+ if (!treePath.endsWith(`/${RESULT_INDEX_FILENAME2}`)) {
4736
+ continue;
4737
+ }
4738
+ const runDir = gitRunDirForManifestPath(treePath);
4739
+ const relativeRunPath = path3.posix.relative(RESULTS_REPO_RUNS_DIR, runDir);
4740
+ if (!isV2ResultsRepoRunPath(relativeRunPath)) {
4741
+ continue;
4742
+ }
4743
+ if (!manifestByRunDir.has(runDir)) {
4744
+ manifestByRunDir.set(runDir, treePath);
4745
+ }
4746
+ }
4747
+ return [...manifestByRunDir.values()].sort();
4748
+ }
4749
+ function fallbackGitManifestPath(runDir, treePathSet) {
4750
+ for (const candidate of [
4751
+ path3.posix.join(runDir, ".internal", RESULT_INDEX_FILENAME2),
4752
+ path3.posix.join(runDir, RESULT_INDEX_FILENAME2)
4753
+ ]) {
4754
+ if (treePathSet.has(candidate)) {
4755
+ return candidate;
4756
+ }
4757
+ }
4758
+ return void 0;
4759
+ }
4760
+ function gitRunDirForManifestPath(manifestPath) {
4761
+ const manifestDir = path3.posix.dirname(manifestPath);
4762
+ return path3.posix.basename(manifestDir) === ".internal" ? path3.posix.dirname(manifestDir) : manifestDir;
4763
+ }
4764
+ function isV2ResultsRepoRunPath(relativeRunPath) {
4765
+ const segments = relativeRunPath.split("/").filter(Boolean);
4766
+ return segments.length === 1 && !segments[0].startsWith(".");
4767
+ }
4768
+ function buildGitRunId(relativeRunPath) {
4769
+ const normalized = relativeRunPath.split(path3.sep).join("/");
4770
+ const segments = normalized.split("/").filter(Boolean);
5309
4771
  return segments[0] ?? relativeRunPath;
5310
4772
  }
5311
4773
  function getRunExperiment(runId, summary) {
@@ -5433,7 +4895,7 @@ async function readGitResultArtifact(params) {
5433
4895
  }
5434
4896
  function buildWipBranchName(runDir) {
5435
4897
  const hostname = os.hostname().replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 40);
5436
- const runBasename = path5.basename(runDir).replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 60);
4898
+ const runBasename = path3.basename(runDir).replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 60);
5437
4899
  return `agentv/wip/${hostname}/${runBasename}`;
5438
4900
  }
5439
4901
  async function setupWipWorktree(params) {
@@ -5452,8 +4914,8 @@ async function setupWipWorktree(params) {
5452
4914
  if (!baseRef) {
5453
4915
  throw new Error("Could not resolve a base ref for the WIP results branch");
5454
4916
  }
5455
- const worktreeRoot = await mkdtemp(path5.join(os.tmpdir(), "agentv-wip-"));
5456
- const worktreeDir = path5.join(worktreeRoot, "repo");
4917
+ const worktreeRoot = await mkdtemp(path3.join(os.tmpdir(), "agentv-wip-"));
4918
+ const worktreeDir = path3.join(worktreeRoot, "repo");
5457
4919
  await runGit(["worktree", "add", "-B", params.wipBranch, worktreeDir, baseRef], {
5458
4920
  cwd: cloneDir
5459
4921
  });
@@ -5472,7 +4934,7 @@ async function setupWipWorktree(params) {
5472
4934
  };
5473
4935
  }
5474
4936
  async function pushWipCheckpoint(params) {
5475
- const destinationDir = path5.join(
4937
+ const destinationDir = path3.join(
5476
4938
  params.handle.worktreeDir,
5477
4939
  RESULTS_REPO_RUNS_DIR,
5478
4940
  params.destinationPath
@@ -5482,8 +4944,11 @@ async function pushWipCheckpoint(params) {
5482
4944
  sourceDir: params.sourceDir,
5483
4945
  destinationDir
5484
4946
  });
5485
- const trackedDirs = await existingTrackedResultsDirs(params.handle.worktreeDir);
5486
- await runGit(["add", "--all", "--", ...trackedDirs], {
4947
+ writeFileSync(
4948
+ path3.join(params.handle.worktreeDir, RESULTS_REPO_GITATTRIBUTES_FILE),
4949
+ RESULTS_REPO_GITATTRIBUTES_CONTENT
4950
+ );
4951
+ await runGit(["add", "--all", "--", RESULTS_REPO_GITATTRIBUTES_FILE, params.destinationPath], {
5487
4952
  cwd: params.handle.worktreeDir
5488
4953
  });
5489
4954
  const { stdout: status } = await runGit(["status", "--porcelain"], {
@@ -5506,7 +4971,19 @@ async function pushWipCheckpoint(params) {
5506
4971
  async function deleteWipBranch(params) {
5507
4972
  const normalized = normalizeResultsConfig(params.config);
5508
4973
  const cloneDir = await ensureResultsRepoClone(normalized);
5509
- await runGit(["push", normalized.remote, "--delete", params.wipBranch], { cwd: cloneDir });
4974
+ const result = await runGit(["push", normalized.remote, "--delete", params.wipBranch], {
4975
+ cwd: cloneDir,
4976
+ check: false
4977
+ });
4978
+ if (result.exitCode === 0 || isMissingRemoteRefDelete(result)) {
4979
+ return;
4980
+ }
4981
+ throw new Error(result.stderr.trim() || result.stdout.trim() || "Failed to delete WIP branch");
4982
+ }
4983
+ function isMissingRemoteRefDelete(result) {
4984
+ const text = `${result.stderr}
4985
+ ${result.stdout}`.toLowerCase();
4986
+ return text.includes("remote ref does not exist");
5510
4987
  }
5511
4988
  function isMissingGitRefError(error) {
5512
4989
  const parts = [];
@@ -5518,7 +4995,107 @@ function isMissingGitRefError(error) {
5518
4995
  parts.push(error);
5519
4996
  }
5520
4997
  const haystack = parts.join("\n").toLowerCase();
5521
- return haystack.includes("not a valid object name") || haystack.includes("unknown revision or path") || haystack.includes("bad revision") || haystack.includes("does not exist");
4998
+ return haystack.includes("not a valid object name") || haystack.includes("unknown revision or path") || haystack.includes("bad revision") || haystack.includes("does not exist") || haystack.includes("needed a single revision");
4999
+ }
5000
+ function gitResultsIndexHash(value) {
5001
+ return createHash("sha256").update(value).digest("hex");
5002
+ }
5003
+ function resolveGitResultsIndexRepoDir(repoDir) {
5004
+ const resolved = path3.resolve(repoDir);
5005
+ try {
5006
+ return realpathSync(resolved);
5007
+ } catch {
5008
+ return resolved;
5009
+ }
5010
+ }
5011
+ function gitResultsIndexRepoKey(repoDir) {
5012
+ return `sha256-${gitResultsIndexHash(resolveGitResultsIndexRepoDir(repoDir)).slice(0, 32)}`;
5013
+ }
5014
+ function gitResultsIndexRefKey(ref) {
5015
+ const slug = sanitizeRepoSlug(ref).slice(0, 80) || "default";
5016
+ return `${slug}-${gitResultsIndexHash(ref).slice(0, 16)}`;
5017
+ }
5018
+ function resolveGitResultsIndexCacheFile(params) {
5019
+ const ref = params.ref ?? "origin/main";
5020
+ return path3.join(
5021
+ getAgentvDataDir(),
5022
+ "cache",
5023
+ "results-index",
5024
+ gitResultsIndexRepoKey(params.repoDir),
5025
+ gitResultsIndexRefKey(ref),
5026
+ `${params.commitSha}.json`
5027
+ );
5028
+ }
5029
+ function isGitListedRun(value) {
5030
+ if (!value || typeof value !== "object") return false;
5031
+ const record = value;
5032
+ return typeof record.run_id === "string" && typeof record.experiment === "string" && typeof record.timestamp === "string" && (typeof record.index_path === "string" || typeof record.manifest_path === "string") && typeof record.display_name === "string" && typeof record.test_count === "number" && typeof record.avg_score === "number" && typeof record.size_bytes === "number" && (record.execution_error_count === void 0 || typeof record.execution_error_count === "number") && (record.pass_rate === void 0 || typeof record.pass_rate === "number") && (record.target === void 0 || typeof record.target === "string") && (record.summary_path === void 0 || typeof record.summary_path === "string");
5033
+ }
5034
+ function validateGitResultsIndexCacheFile(value, params) {
5035
+ if (!value || typeof value !== "object") return void 0;
5036
+ const record = value;
5037
+ const repoDir = resolveGitResultsIndexRepoDir(params.repoDir);
5038
+ const repoKey = gitResultsIndexRepoKey(params.repoDir);
5039
+ if (record.schema_version !== GIT_RESULTS_INDEX_CACHE_SCHEMA_VERSION || record.result_layout_version !== GIT_RESULTS_INDEX_LAYOUT_VERSION || record.repo_key !== repoKey || record.repo_dir !== repoDir || record.ref !== params.ref || record.commit_sha !== params.commitSha || !Array.isArray(record.runs) || !record.runs.every(isGitListedRun)) {
5040
+ return void 0;
5041
+ }
5042
+ return value;
5043
+ }
5044
+ async function readGitResultsIndexCache(params) {
5045
+ const cacheFile = resolveGitResultsIndexCacheFile(params);
5046
+ try {
5047
+ const parsed = JSON.parse(await readFile(cacheFile, "utf8"));
5048
+ return validateGitResultsIndexCacheFile(parsed, params)?.runs;
5049
+ } catch {
5050
+ return void 0;
5051
+ }
5052
+ }
5053
+ async function writeGitResultsIndexCache(params) {
5054
+ const cacheFile = resolveGitResultsIndexCacheFile(params);
5055
+ const envelope = {
5056
+ schema_version: GIT_RESULTS_INDEX_CACHE_SCHEMA_VERSION,
5057
+ result_layout_version: GIT_RESULTS_INDEX_LAYOUT_VERSION,
5058
+ repo_key: gitResultsIndexRepoKey(params.repoDir),
5059
+ repo_dir: resolveGitResultsIndexRepoDir(params.repoDir),
5060
+ ref: params.ref,
5061
+ commit_sha: params.commitSha,
5062
+ generated_at: (/* @__PURE__ */ new Date()).toISOString(),
5063
+ runs: params.runs
5064
+ };
5065
+ try {
5066
+ mkdirSync(path3.dirname(cacheFile), { recursive: true });
5067
+ const tempFile = `${cacheFile}.${process.pid}.${Date.now()}.tmp`;
5068
+ await writeFile(tempFile, `${JSON.stringify(envelope, null, 2)}
5069
+ `, "utf8");
5070
+ renameSync(tempFile, cacheFile);
5071
+ } catch {
5072
+ }
5073
+ }
5074
+ async function resolveGitRunsRefCommit(repoDir, ref = "origin/main") {
5075
+ try {
5076
+ const { stdout } = await runGit(["rev-parse", "--verify", `${ref}^{commit}`], {
5077
+ cwd: repoDir
5078
+ });
5079
+ return stdout.trim() || void 0;
5080
+ } catch (error) {
5081
+ if (isMissingGitRefError(error)) {
5082
+ return void 0;
5083
+ }
5084
+ throw error;
5085
+ }
5086
+ }
5087
+ async function listGitRunsCached(repoDir, ref = "origin/main") {
5088
+ const commitSha = await resolveGitRunsRefCommit(repoDir, ref);
5089
+ if (!commitSha) {
5090
+ return [];
5091
+ }
5092
+ const cached = await readGitResultsIndexCache({ repoDir, ref, commitSha });
5093
+ if (cached) {
5094
+ return [...cached];
5095
+ }
5096
+ const runs = await listGitRuns(repoDir, ref);
5097
+ await writeGitResultsIndexCache({ repoDir, ref, commitSha, runs });
5098
+ return runs;
5522
5099
  }
5523
5100
  async function listGitRuns(repoDir, ref = "origin/main") {
5524
5101
  let treeOut;
@@ -5536,19 +5113,7 @@ async function listGitRuns(repoDir, ref = "origin/main") {
5536
5113
  throw error;
5537
5114
  }
5538
5115
  const treePaths = treeOut.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
5539
- const indexPaths = treePaths.filter((line) => line.endsWith("/index.jsonl"));
5540
- if (indexPaths.length === 0) {
5541
- return [];
5542
- }
5543
- const batchInput = `${indexPaths.map((indexPath) => `${ref}:${indexPath}`).join("\n")}
5544
- `;
5545
- const blobs = parseGitBatchBlobs(await runGitBatch(repoDir, batchInput));
5546
- if (blobs.length !== indexPaths.length) {
5547
- throw new Error(
5548
- `Expected ${indexPaths.length} git blobs but received ${blobs.length} while listing results runs`
5549
- );
5550
- }
5551
- const summaryPaths = indexPaths.map((indexPath) => path5.posix.join(path5.posix.dirname(indexPath), "summary.json")).filter((summaryPath) => treePaths.includes(summaryPath));
5116
+ const summaryPaths = treePaths.filter((line) => line.endsWith("/summary.json"));
5552
5117
  const summaryByPath = /* @__PURE__ */ new Map();
5553
5118
  if (summaryPaths.length > 0) {
5554
5119
  const summaryBatchInput = `${summaryPaths.map((summaryPath) => `${ref}:${summaryPath}`).join("\n")}
@@ -5561,12 +5126,24 @@ async function listGitRuns(repoDir, ref = "origin/main") {
5561
5126
  summaryByPath.set(summaryPath, JSON.parse(blob.content.toString("utf8")));
5562
5127
  }
5563
5128
  }
5129
+ const indexPaths = buildGitManifestPaths(treePaths, summaryByPath);
5130
+ if (indexPaths.length === 0) {
5131
+ return [];
5132
+ }
5133
+ const batchInput = `${indexPaths.map((indexPath) => `${ref}:${indexPath}`).join("\n")}
5134
+ `;
5135
+ const blobs = parseGitBatchBlobs(await runGitBatch(repoDir, batchInput));
5136
+ if (blobs.length !== indexPaths.length) {
5137
+ throw new Error(
5138
+ `Expected ${indexPaths.length} git blobs but received ${blobs.length} while listing results runs`
5139
+ );
5140
+ }
5564
5141
  const runs = blobs.flatMap((blob, index) => {
5565
5142
  const manifestPath = indexPaths[index];
5566
- const runDir = path5.posix.dirname(manifestPath);
5567
- const summaryPath = path5.posix.join(runDir, "summary.json");
5143
+ const runDir = gitRunDirForManifestPath(manifestPath);
5144
+ const summaryPath = path3.posix.join(runDir, "summary.json");
5568
5145
  const summary = summaryByPath.get(summaryPath);
5569
- const relativeRunPath = path5.posix.relative(RESULTS_REPO_RUNS_DIR, runDir);
5146
+ const relativeRunPath = path3.posix.relative(RESULTS_REPO_RUNS_DIR, runDir);
5570
5147
  const runId = buildGitRunId(relativeRunPath);
5571
5148
  const rows = blob.content.toString("utf8").split(/\r?\n/).map((line) => line.trim()).filter(Boolean).flatMap(
5572
5149
  (line) => {
@@ -5587,10 +5164,13 @@ async function listGitRuns(repoDir, ref = "origin/main") {
5587
5164
  ];
5588
5165
  const rowScores = rows.map((row) => row.score).filter((score) => typeof score === "number" && Number.isFinite(score));
5589
5166
  const avgScore = rowScores.length > 0 ? rowScores.reduce((sum, score) => sum + score, 0) / rowScores.length : 0;
5590
- const timestamp = summary?.metadata?.timestamp?.trim() || rows[0]?.timestamp || path5.posix.basename(runDir);
5591
- const displayName = summary?.metadata?.display_name?.trim() || path5.posix.basename(runDir);
5167
+ const timestamp = summary?.metadata?.timestamp?.trim() || rows[0]?.timestamp || path3.posix.basename(runDir);
5168
+ const displayName = summary?.metadata?.display_name?.trim() || path3.posix.basename(runDir);
5592
5169
  const targets = summary?.metadata?.targets ?? rowTargets;
5593
5170
  const passRate = computeAveragePassRate(summary?.run_summary) ?? avgScore;
5171
+ const executionErrorCount = rows.filter(
5172
+ (row) => row.execution_status === "execution_error"
5173
+ ).length;
5594
5174
  return [
5595
5175
  {
5596
5176
  run_id: runId,
@@ -5598,10 +5178,11 @@ async function listGitRuns(repoDir, ref = "origin/main") {
5598
5178
  timestamp,
5599
5179
  ...passRate !== void 0 && { pass_rate: passRate },
5600
5180
  ...targets.length === 1 && targets[0] ? { target: targets[0] } : {},
5601
- manifest_path: manifestPath,
5181
+ index_path: manifestPath,
5602
5182
  ...summaryByPath.has(summaryPath) && { summary_path: summaryPath },
5603
5183
  display_name: displayName,
5604
5184
  test_count: summary?.metadata?.tests_run?.length ?? rowTestIds.length,
5185
+ ...executionErrorCount > 0 && { execution_error_count: executionErrorCount },
5605
5186
  avg_score: avgScore,
5606
5187
  size_bytes: blob.size
5607
5188
  }
@@ -5611,9 +5192,9 @@ async function listGitRuns(repoDir, ref = "origin/main") {
5611
5192
  return runs;
5612
5193
  }
5613
5194
  async function materializeGitRun(repoDir, relativeRunPath, ref = "origin/main") {
5614
- const normalizedRunPath = relativeRunPath.split(path5.sep).join("/");
5615
- const runTreePath = path5.posix.join(RESULTS_REPO_RUNS_DIR, normalizedRunPath);
5616
- const targetRunDir = path5.join(repoDir, ...runTreePath.split("/"));
5195
+ const normalizedRunPath = relativeRunPath.split(path3.sep).join("/");
5196
+ const runTreePath = path3.posix.join(RESULTS_REPO_RUNS_DIR, normalizedRunPath);
5197
+ const targetRunDir = path3.join(repoDir, ...runTreePath.split("/"));
5617
5198
  const { stdout: treeOut } = await runGit(["ls-tree", "-r", "--name-only", ref, runTreePath], {
5618
5199
  cwd: repoDir
5619
5200
  });
@@ -5629,16 +5210,16 @@ async function materializeGitRun(repoDir, relativeRunPath, ref = "origin/main")
5629
5210
  `Expected ${filePaths.length} git blobs but received ${blobs.length} while materializing results run`
5630
5211
  );
5631
5212
  }
5632
- const tempRoot = mkdtempSync(path5.join(repoDir, ".agentv-run-"));
5633
- const tempRunDir = path5.join(tempRoot, "run");
5213
+ const tempRoot = mkdtempSync(path3.join(repoDir, ".agentv-run-"));
5214
+ const tempRunDir = path3.join(tempRoot, "run");
5634
5215
  try {
5635
5216
  for (const [index, filePath] of filePaths.entries()) {
5636
- const relativeFilePath = path5.posix.relative(runTreePath, filePath);
5637
- const absolutePath = path5.join(tempRunDir, ...relativeFilePath.split("/"));
5638
- mkdirSync(path5.dirname(absolutePath), { recursive: true });
5217
+ const relativeFilePath = path3.posix.relative(runTreePath, filePath);
5218
+ const absolutePath = path3.join(tempRunDir, ...relativeFilePath.split("/"));
5219
+ mkdirSync(path3.dirname(absolutePath), { recursive: true });
5639
5220
  writeFileSync(absolutePath, blobs[index].content);
5640
5221
  }
5641
- mkdirSync(path5.dirname(targetRunDir), { recursive: true });
5222
+ mkdirSync(path3.dirname(targetRunDir), { recursive: true });
5642
5223
  try {
5643
5224
  renameSync(tempRunDir, targetRunDir);
5644
5225
  } catch (error) {
@@ -5720,483 +5301,6 @@ function trimBaselineResult(result) {
5720
5301
  }
5721
5302
  return trimmed;
5722
5303
  }
5723
- var DEFAULT_CATEGORY = "Uncategorized";
5724
- function deriveCategory(relativePath) {
5725
- const parts = relativePath.split(/[/\\]/);
5726
- if (parts.length <= 1) {
5727
- return DEFAULT_CATEGORY;
5728
- }
5729
- const dirs = parts.slice(0, -1).filter((d) => d !== "evals");
5730
- return dirs.length > 0 ? dirs.join("/") : DEFAULT_CATEGORY;
5731
- }
5732
- var OtelTraceExporter = class {
5733
- constructor(options) {
5734
- this.options = options;
5735
- }
5736
- provider = null;
5737
- tracer = null;
5738
- api = null;
5739
- // biome-ignore lint/suspicious/noExplicitAny: OTel types loaded dynamically
5740
- W3CPropagator = null;
5741
- /** Initialize the OTel SDK. Returns false if OTel packages are not available. */
5742
- async init() {
5743
- try {
5744
- const [sdkTraceNode, resourcesMod, semconvMod, api, coreMod] = await Promise.all([
5745
- import("./src-76ISS3LH.js"),
5746
- import("./esm-NNMQYIKT.js"),
5747
- import("./esm-RVQPUGWH.js"),
5748
- import("./esm-R77SNOF5.js"),
5749
- import("./esm-ZADQ4XQH-QUFGGZL4.js").catch(() => null)
5750
- ]);
5751
- const { NodeTracerProvider: Provider, SimpleSpanProcessor } = sdkTraceNode;
5752
- const { resourceFromAttributes } = resourcesMod;
5753
- const { ATTR_SERVICE_NAME } = semconvMod;
5754
- const resourceAttributes = {
5755
- [ATTR_SERVICE_NAME]: this.options.serviceName ?? "agentv",
5756
- ...this.options.resourceAttributes
5757
- };
5758
- const resource = resourceFromAttributes(resourceAttributes);
5759
- const processors = [];
5760
- if (this.options.endpoint) {
5761
- const otlpHttp = await import("./esm-TTEMJEVV.js");
5762
- const { OTLPTraceExporter } = otlpHttp;
5763
- const exporter = new OTLPTraceExporter({
5764
- url: this.options.endpoint,
5765
- headers: this.options.headers
5766
- });
5767
- processors.push(new SimpleSpanProcessor(exporter));
5768
- }
5769
- if (this.options.otlpFilePath) {
5770
- const { OtlpJsonFileExporter: OtlpJsonFileExporter2 } = await import("./otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js");
5771
- processors.push(
5772
- new SimpleSpanProcessor(
5773
- new OtlpJsonFileExporter2(this.options.otlpFilePath, resourceAttributes)
5774
- )
5775
- );
5776
- }
5777
- if (processors.length === 0) {
5778
- return false;
5779
- }
5780
- this.provider = new Provider({
5781
- resource,
5782
- spanProcessors: processors
5783
- });
5784
- this.provider.register();
5785
- this.api = api;
5786
- this.tracer = api.trace.getTracer("agentv", "1.0.0");
5787
- this.W3CPropagator = coreMod?.W3CTraceContextPropagator ?? null;
5788
- return true;
5789
- } catch {
5790
- return false;
5791
- }
5792
- }
5793
- /** Export a single evaluation result as an OTel trace. */
5794
- async exportResult(result) {
5795
- if (!this.tracer || !this.api) return;
5796
- const api = this.api;
5797
- const tracer = this.tracer;
5798
- const captureContent = this.options.captureContent ?? false;
5799
- const startHr = toHrTime(result.startTime ?? result.timestamp);
5800
- const endHr = toHrTime(result.endTime ?? result.timestamp);
5801
- let parentCtx = api.ROOT_CONTEXT;
5802
- const traceparent = process.env.TRACEPARENT;
5803
- if (traceparent && this.W3CPropagator) {
5804
- try {
5805
- const propagator = new this.W3CPropagator();
5806
- parentCtx = propagator.extract(
5807
- api.ROOT_CONTEXT,
5808
- { traceparent, tracestate: process.env.TRACESTATE ?? "" },
5809
- {
5810
- get: (carrier, key) => carrier[key],
5811
- keys: (carrier) => Object.keys(carrier)
5812
- }
5813
- );
5814
- } catch {
5815
- }
5816
- }
5817
- tracer.startActiveSpan(
5818
- "agentv.eval",
5819
- { startTime: startHr },
5820
- parentCtx,
5821
- (rootSpan) => {
5822
- rootSpan.setAttribute("gen_ai.operation.name", "evaluate");
5823
- rootSpan.setAttribute("gen_ai.system", "agentv");
5824
- rootSpan.setAttribute("agentv.test_id", result.testId);
5825
- rootSpan.setAttribute("agentv.target", result.target);
5826
- if (result.suite) rootSpan.setAttribute("agentv.suite", result.suite);
5827
- rootSpan.setAttribute("agentv.score", result.score);
5828
- if (captureContent && result.output.length > 0) {
5829
- rootSpan.setAttribute("agentv.output_text", result.output);
5830
- }
5831
- if (result.durationMs != null)
5832
- rootSpan.setAttribute("agentv.trace.duration_ms", result.durationMs);
5833
- if (result.costUsd != null) rootSpan.setAttribute("agentv.trace.cost_usd", result.costUsd);
5834
- if (result.tokenUsage) {
5835
- if (result.tokenUsage.input != null) {
5836
- rootSpan.setAttribute("agentv.trace.token_input", result.tokenUsage.input);
5837
- }
5838
- if (result.tokenUsage.output != null) {
5839
- rootSpan.setAttribute("agentv.trace.token_output", result.tokenUsage.output);
5840
- }
5841
- if (result.tokenUsage.cached != null) {
5842
- rootSpan.setAttribute("agentv.trace.token_cached", result.tokenUsage.cached);
5843
- }
5844
- }
5845
- if (result.trace) {
5846
- const t = result.trace;
5847
- rootSpan.setAttribute("agentv.trace.event_count", t.eventCount);
5848
- rootSpan.setAttribute(
5849
- "agentv.trace.tool_names",
5850
- Object.keys(t.toolCalls).sort().join(",")
5851
- );
5852
- if (t.llmCallCount != null)
5853
- rootSpan.setAttribute("agentv.trace.llm_call_count", t.llmCallCount);
5854
- }
5855
- const traceMessages = result.trace?.messages ?? [];
5856
- if (traceMessages.length > 0) {
5857
- const parentCtx2 = api.trace.setSpan(api.context.active(), rootSpan);
5858
- if (this.options.groupTurns) {
5859
- const turns = groupMessagesIntoTurns(traceMessages);
5860
- if (turns.length > 1) {
5861
- for (const [i, turn] of turns.entries()) {
5862
- api.context.with(parentCtx2, () => {
5863
- tracer.startActiveSpan(
5864
- `agentv.turn.${i + 1}`,
5865
- {},
5866
- (turnSpan) => {
5867
- const turnCtx = api.trace.setSpan(api.context.active(), turnSpan);
5868
- for (const msg of turn.messages) {
5869
- this.exportMessage(tracer, api, turnCtx, msg, captureContent);
5870
- }
5871
- turnSpan.end();
5872
- }
5873
- );
5874
- });
5875
- }
5876
- } else {
5877
- for (const msg of traceMessages) {
5878
- this.exportMessage(tracer, api, parentCtx2, msg, captureContent);
5879
- }
5880
- }
5881
- } else {
5882
- for (const msg of traceMessages) {
5883
- this.exportMessage(tracer, api, parentCtx2, msg, captureContent);
5884
- }
5885
- }
5886
- }
5887
- if (result.scores) {
5888
- for (const score of result.scores) {
5889
- rootSpan.addEvent(`agentv.grader.${score.name}`, {
5890
- "agentv.grader.score": score.score,
5891
- "agentv.grader.type": score.type,
5892
- ...score.verdict ? { "agentv.grader.verdict": score.verdict } : {}
5893
- });
5894
- }
5895
- }
5896
- if (result.error) {
5897
- rootSpan.setStatus({ code: api.SpanStatusCode.ERROR, message: result.error });
5898
- } else {
5899
- rootSpan.setStatus({ code: api.SpanStatusCode.OK });
5900
- }
5901
- rootSpan.end(endHr);
5902
- }
5903
- );
5904
- }
5905
- /** Flush pending spans and shut down. */
5906
- async shutdown() {
5907
- await this.provider?.shutdown();
5908
- }
5909
- /** Create a streaming observer for real-time span export */
5910
- createStreamingObserver() {
5911
- if (!this.tracer || !this.api) return null;
5912
- let parentCtx;
5913
- const traceparent = process.env.TRACEPARENT;
5914
- if (traceparent && this.W3CPropagator) {
5915
- try {
5916
- const propagator = new this.W3CPropagator();
5917
- parentCtx = propagator.extract(
5918
- this.api.ROOT_CONTEXT,
5919
- { traceparent, tracestate: process.env.TRACESTATE ?? "" },
5920
- {
5921
- get: (carrier, key) => carrier[key],
5922
- keys: (carrier) => Object.keys(carrier)
5923
- }
5924
- );
5925
- } catch {
5926
- }
5927
- }
5928
- return new OtelStreamingObserver(
5929
- this.tracer,
5930
- this.api,
5931
- this.options.captureContent ?? false,
5932
- parentCtx
5933
- );
5934
- }
5935
- // -----------------------------------------------------------------------
5936
- // Private helpers
5937
- // -----------------------------------------------------------------------
5938
- exportMessage(tracer, api, parentCtx, msg, captureContent) {
5939
- const isAssistant = msg.role === "assistant";
5940
- const model = msg.metadata?.model ? String(msg.metadata.model) : void 0;
5941
- const spanName = isAssistant ? `chat ${model ?? "unknown"}` : `gen_ai.message.${msg.role}`;
5942
- const startHr = toHrTime(msg.startTime);
5943
- const endHr = toHrTime(msg.endTime);
5944
- api.context.with(parentCtx, () => {
5945
- tracer.startActiveSpan(
5946
- spanName,
5947
- { startTime: startHr },
5948
- parentCtx,
5949
- (span) => {
5950
- if (isAssistant) {
5951
- span.setAttribute("gen_ai.operation.name", "chat");
5952
- }
5953
- if (model) {
5954
- span.setAttribute("gen_ai.request.model", model);
5955
- span.setAttribute("gen_ai.response.model", model);
5956
- }
5957
- if (msg.tokenUsage) {
5958
- if (msg.tokenUsage.input != null) {
5959
- span.setAttribute("gen_ai.usage.input_tokens", msg.tokenUsage.input);
5960
- }
5961
- if (msg.tokenUsage.output != null) {
5962
- span.setAttribute("gen_ai.usage.output_tokens", msg.tokenUsage.output);
5963
- }
5964
- if (msg.tokenUsage.cached != null) {
5965
- span.setAttribute("gen_ai.usage.cache_read.input_tokens", msg.tokenUsage.cached);
5966
- }
5967
- }
5968
- if (captureContent && msg.content != null) {
5969
- span.setAttribute(
5970
- "gen_ai.output.messages",
5971
- typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
5972
- );
5973
- }
5974
- if (msg.toolCalls) {
5975
- const msgCtx = api.trace.setSpan(api.context.active(), span);
5976
- for (const tc of msg.toolCalls) {
5977
- api.context.with(msgCtx, () => {
5978
- tracer.startActiveSpan(
5979
- `execute_tool ${tc.tool}`,
5980
- {},
5981
- msgCtx,
5982
- (toolSpan) => {
5983
- toolSpan.setAttribute("gen_ai.tool.name", tc.tool);
5984
- if (tc.id) toolSpan.setAttribute("gen_ai.tool.call.id", tc.id);
5985
- if (captureContent) {
5986
- if (tc.input != null) {
5987
- toolSpan.setAttribute(
5988
- "gen_ai.tool.call.arguments",
5989
- typeof tc.input === "string" ? tc.input : JSON.stringify(tc.input)
5990
- );
5991
- }
5992
- if (tc.output != null) {
5993
- toolSpan.setAttribute(
5994
- "gen_ai.tool.call.result",
5995
- typeof tc.output === "string" ? tc.output : JSON.stringify(tc.output)
5996
- );
5997
- }
5998
- }
5999
- toolSpan.end();
6000
- }
6001
- );
6002
- });
6003
- }
6004
- }
6005
- span.end(endHr);
6006
- }
6007
- );
6008
- });
6009
- }
6010
- };
6011
- var OtelStreamingObserver = class {
6012
- constructor(tracer, api, captureContent, parentCtx) {
6013
- this.tracer = tracer;
6014
- this.api = api;
6015
- this.captureContent = captureContent;
6016
- this.parentCtx = parentCtx;
6017
- }
6018
- // biome-ignore lint/suspicious/noExplicitAny: OTel span type loaded dynamically
6019
- rootSpan = null;
6020
- // biome-ignore lint/suspicious/noExplicitAny: OTel context loaded dynamically
6021
- rootCtx = null;
6022
- observedChildSpans = false;
6023
- pendingMetrics = null;
6024
- /** Create root eval span immediately (visible in backend right away) */
6025
- startEvalCase(testId, target, evalSet) {
6026
- this.pendingMetrics = null;
6027
- this.observedChildSpans = false;
6028
- const ctx = this.parentCtx ?? this.api.context.active();
6029
- this.rootSpan = this.tracer.startSpan("agentv.eval", void 0, ctx);
6030
- this.rootSpan.setAttribute("gen_ai.operation.name", "evaluate");
6031
- this.rootSpan.setAttribute("gen_ai.system", "agentv");
6032
- this.rootSpan.setAttribute("agentv.test_id", testId);
6033
- this.rootSpan.setAttribute("agentv.target", target);
6034
- if (evalSet) this.rootSpan.setAttribute("agentv.suite", evalSet);
6035
- this.rootCtx = this.api.trace.setSpan(this.api.context.active(), this.rootSpan);
6036
- }
6037
- /** Create and immediately export a tool span */
6038
- onToolCall(name, input, output, _durationMs, toolCallId) {
6039
- if (!this.rootCtx) return;
6040
- this.observedChildSpans = true;
6041
- this.api.context.with(this.rootCtx, () => {
6042
- const span = this.tracer.startSpan(`execute_tool ${name}`, void 0, this.rootCtx);
6043
- span.setAttribute("gen_ai.tool.name", name);
6044
- if (toolCallId) span.setAttribute("gen_ai.tool.call.id", toolCallId);
6045
- if (this.captureContent) {
6046
- if (input != null)
6047
- span.setAttribute(
6048
- "gen_ai.tool.call.arguments",
6049
- typeof input === "string" ? input : JSON.stringify(input)
6050
- );
6051
- if (output != null)
6052
- span.setAttribute(
6053
- "gen_ai.tool.call.result",
6054
- typeof output === "string" ? output : JSON.stringify(output)
6055
- );
6056
- }
6057
- span.end();
6058
- });
6059
- }
6060
- /** Create and immediately export an LLM span */
6061
- onLlmCall(model, tokenUsage) {
6062
- if (!this.rootCtx) return;
6063
- this.observedChildSpans = true;
6064
- this.api.context.with(this.rootCtx, () => {
6065
- const span = this.tracer.startSpan(`chat ${model}`, void 0, this.rootCtx);
6066
- span.setAttribute("gen_ai.operation.name", "chat");
6067
- span.setAttribute("gen_ai.request.model", model);
6068
- span.setAttribute("gen_ai.response.model", model);
6069
- if (tokenUsage) {
6070
- if (tokenUsage.input != null)
6071
- span.setAttribute("gen_ai.usage.input_tokens", tokenUsage.input);
6072
- if (tokenUsage.output != null)
6073
- span.setAttribute("gen_ai.usage.output_tokens", tokenUsage.output);
6074
- if (tokenUsage.cached != null)
6075
- span.setAttribute("gen_ai.usage.cache_read.input_tokens", tokenUsage.cached);
6076
- }
6077
- span.end();
6078
- });
6079
- }
6080
- /** Record final execution metrics before the root span is finalized. */
6081
- recordEvalMetrics(result) {
6082
- this.pendingMetrics = result;
6083
- }
6084
- /** Finalize root span with score/verdict after evaluation completes */
6085
- finalizeEvalCase(score, error) {
6086
- if (!this.rootSpan) return;
6087
- this.rootSpan.setAttribute("agentv.score", score);
6088
- if (this.pendingMetrics?.durationMs != null) {
6089
- this.rootSpan.setAttribute("agentv.trace.duration_ms", this.pendingMetrics.durationMs);
6090
- }
6091
- if (this.pendingMetrics?.costUsd != null) {
6092
- this.rootSpan.setAttribute("agentv.trace.cost_usd", this.pendingMetrics.costUsd);
6093
- }
6094
- if (this.pendingMetrics?.tokenUsage) {
6095
- if (this.pendingMetrics.tokenUsage.input != null) {
6096
- this.rootSpan.setAttribute(
6097
- "agentv.trace.token_input",
6098
- this.pendingMetrics.tokenUsage.input
6099
- );
6100
- }
6101
- if (this.pendingMetrics.tokenUsage.output != null) {
6102
- this.rootSpan.setAttribute(
6103
- "agentv.trace.token_output",
6104
- this.pendingMetrics.tokenUsage.output
6105
- );
6106
- }
6107
- if (this.pendingMetrics.tokenUsage.cached != null) {
6108
- this.rootSpan.setAttribute(
6109
- "agentv.trace.token_cached",
6110
- this.pendingMetrics.tokenUsage.cached
6111
- );
6112
- }
6113
- }
6114
- if (this.pendingMetrics?.trace) {
6115
- this.rootSpan.setAttribute("agentv.trace.event_count", this.pendingMetrics.trace.eventCount);
6116
- this.rootSpan.setAttribute(
6117
- "agentv.trace.tool_names",
6118
- Object.keys(this.pendingMetrics.trace.toolCalls).sort().join(",")
6119
- );
6120
- if (this.pendingMetrics.trace.llmCallCount != null) {
6121
- this.rootSpan.setAttribute(
6122
- "agentv.trace.llm_call_count",
6123
- this.pendingMetrics.trace.llmCallCount
6124
- );
6125
- }
6126
- }
6127
- if (error) {
6128
- this.rootSpan.setStatus({ code: this.api.SpanStatusCode.ERROR, message: error });
6129
- } else {
6130
- this.rootSpan.setStatus({ code: this.api.SpanStatusCode.OK });
6131
- }
6132
- this.rootSpan.end();
6133
- this.rootSpan = null;
6134
- this.rootCtx = null;
6135
- this.observedChildSpans = false;
6136
- this.pendingMetrics = null;
6137
- }
6138
- /** Backfill child spans from the completed result when the provider emitted no live callbacks. */
6139
- completeFromResult(result) {
6140
- this.recordEvalMetrics({
6141
- durationMs: result.durationMs,
6142
- costUsd: result.costUsd,
6143
- tokenUsage: result.tokenUsage,
6144
- trace: result.trace
6145
- });
6146
- if (this.observedChildSpans || !this.rootCtx) {
6147
- return;
6148
- }
6149
- const model = result.trace.messages.find((msg) => msg.role === "assistant")?.metadata?.model ?? result.target ?? "unknown";
6150
- this.onLlmCall(String(model), result.tokenUsage);
6151
- for (const message of result.trace.messages) {
6152
- for (const toolCall of message.toolCalls ?? []) {
6153
- this.onToolCall(
6154
- toolCall.tool,
6155
- toolCall.input,
6156
- toolCall.output,
6157
- toolCall.durationMs ?? 0,
6158
- toolCall.id
6159
- );
6160
- }
6161
- }
6162
- }
6163
- /** Return the active eval span's trace ID and span ID for Braintrust trace bridging */
6164
- getActiveSpanIds() {
6165
- if (!this.rootSpan) return null;
6166
- try {
6167
- const spanCtx = this.rootSpan.spanContext?.() ?? this.rootSpan._spanContext;
6168
- if (!spanCtx?.traceId || !spanCtx?.spanId) return null;
6169
- return { parentSpanId: spanCtx.spanId, rootSpanId: spanCtx.traceId };
6170
- } catch {
6171
- return null;
6172
- }
6173
- }
6174
- /** Get ProviderStreamCallbacks for passing to providers */
6175
- getStreamCallbacks() {
6176
- return {
6177
- onToolCallEnd: (name, input, output, durationMs, toolCallId) => this.onToolCall(name, input, output, durationMs, toolCallId),
6178
- onLlmCallEnd: (model, tokenUsage) => this.onLlmCall(model, tokenUsage),
6179
- getActiveSpanIds: () => this.getActiveSpanIds()
6180
- };
6181
- }
6182
- };
6183
- function groupMessagesIntoTurns(messages) {
6184
- const turns = [];
6185
- let current = [];
6186
- for (const msg of messages) {
6187
- if (msg.role === "user" && current.length > 0) {
6188
- turns.push({ messages: current });
6189
- current = [];
6190
- }
6191
- current.push(msg);
6192
- }
6193
- if (current.length > 0) turns.push({ messages: current });
6194
- return turns;
6195
- }
6196
- function toHrTime(iso) {
6197
- if (!iso) return void 0;
6198
- return new Date(iso).getTime();
6199
- }
6200
5304
  var RunBudgetTracker = class {
6201
5305
  constructor(capUsd) {
6202
5306
  this.capUsd = capUsd;
@@ -6376,7 +5480,7 @@ function parseClaudeSession(jsonl) {
6376
5480
  durationMs = new Date(endTimestamp).getTime() - new Date(startTimestamp).getTime();
6377
5481
  }
6378
5482
  const source = {
6379
- provider: "claude",
5483
+ provider: "claude-sdk",
6380
5484
  sessionId,
6381
5485
  projectPath,
6382
5486
  startedAt: startTimestamp,
@@ -6426,7 +5530,7 @@ function extractAssistantContent(content) {
6426
5530
  case "tool_use":
6427
5531
  if (block.name) {
6428
5532
  toolCalls.push(
6429
- normalizeToolCall("claude", {
5533
+ normalizeToolCall("claude-sdk", {
6430
5534
  tool: block.name,
6431
5535
  input: block.input,
6432
5536
  id: block.id
@@ -6521,7 +5625,7 @@ function parseCodexSession(jsonl) {
6521
5625
  } else {
6522
5626
  input = payload.arguments;
6523
5627
  }
6524
- const toolCall = normalizeToolCall("codex", {
5628
+ const toolCall = normalizeToolCall("codex-cli", {
6525
5629
  tool: toolName,
6526
5630
  input,
6527
5631
  id: callId
@@ -6549,7 +5653,7 @@ function parseCodexSession(jsonl) {
6549
5653
  } else {
6550
5654
  input = payload.arguments;
6551
5655
  }
6552
- const toolCall = normalizeToolCall("codex", {
5656
+ const toolCall = normalizeToolCall("codex-cli", {
6553
5657
  tool: toolName,
6554
5658
  input,
6555
5659
  id: callId
@@ -6623,7 +5727,7 @@ function extractResponseItemContent(content) {
6623
5727
  }
6624
5728
  return parts.length > 0 ? parts.join("") : void 0;
6625
5729
  }
6626
- var DEFAULT_SESSIONS_DIR = () => path6.join(homedir(), ".codex", "sessions");
5730
+ var DEFAULT_SESSIONS_DIR = () => path4.join(homedir(), ".codex", "sessions");
6627
5731
  async function discoverCodexSessions(opts) {
6628
5732
  const sessionsDir = opts?.sessionsDir ?? DEFAULT_SESSIONS_DIR();
6629
5733
  const limit = opts?.latest ? 1 : opts?.limit ?? 10;
@@ -6635,7 +5739,7 @@ async function discoverCodexSessions(opts) {
6635
5739
  return [];
6636
5740
  }
6637
5741
  for (const year of yearDirs) {
6638
- const yearPath = path6.join(sessionsDir, year);
5742
+ const yearPath = path4.join(sessionsDir, year);
6639
5743
  let monthDirs;
6640
5744
  try {
6641
5745
  monthDirs = await readdir2(yearPath);
@@ -6643,7 +5747,7 @@ async function discoverCodexSessions(opts) {
6643
5747
  continue;
6644
5748
  }
6645
5749
  for (const month of monthDirs) {
6646
- const monthPath = path6.join(yearPath, month);
5750
+ const monthPath = path4.join(yearPath, month);
6647
5751
  let dayDirs;
6648
5752
  try {
6649
5753
  dayDirs = await readdir2(monthPath);
@@ -6655,7 +5759,7 @@ async function discoverCodexSessions(opts) {
6655
5759
  const dirDate = `${year}-${month}-${day}`;
6656
5760
  if (dirDate !== opts.date) continue;
6657
5761
  }
6658
- const dayPath = path6.join(monthPath, day);
5762
+ const dayPath = path4.join(monthPath, day);
6659
5763
  let files;
6660
5764
  try {
6661
5765
  files = await readdir2(dayPath);
@@ -6664,7 +5768,7 @@ async function discoverCodexSessions(opts) {
6664
5768
  }
6665
5769
  for (const file of files) {
6666
5770
  if (!file.startsWith("rollout-") || !file.endsWith(".jsonl")) continue;
6667
- const filePath = path6.join(dayPath, file);
5771
+ const filePath = path4.join(dayPath, file);
6668
5772
  const nameWithoutExt = file.replace(/\.jsonl$/, "");
6669
5773
  const parts = nameWithoutExt.split("-");
6670
5774
  const sessionId = parts.length >= 6 ? parts.slice(-5).join("-") : nameWithoutExt;
@@ -6683,7 +5787,7 @@ async function discoverCodexSessions(opts) {
6683
5787
  sessions.sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime());
6684
5788
  return sessions.slice(0, limit);
6685
5789
  }
6686
- var DEFAULT_PROJECTS_DIR = () => path7.join(homedir2(), ".claude", "projects");
5790
+ var DEFAULT_PROJECTS_DIR = () => path5.join(homedir2(), ".claude", "projects");
6687
5791
  function encodeProjectPath(projectPath) {
6688
5792
  return projectPath.replace(/\//g, "-");
6689
5793
  }
@@ -6702,7 +5806,7 @@ async function discoverClaudeSessions(opts) {
6702
5806
  }
6703
5807
  const sessions = [];
6704
5808
  for (const projectDir of projectDirs) {
6705
- const dirPath = path7.join(projectsDir, projectDir);
5809
+ const dirPath = path5.join(projectsDir, projectDir);
6706
5810
  let entries;
6707
5811
  try {
6708
5812
  entries = await readdir3(dirPath);
@@ -6713,7 +5817,7 @@ async function discoverClaudeSessions(opts) {
6713
5817
  if (!entry.endsWith(".jsonl")) continue;
6714
5818
  const sessionId = entry.replace(/\.jsonl$/, "");
6715
5819
  if (opts?.sessionId && sessionId !== opts.sessionId) continue;
6716
- const filePath = path7.join(dirPath, entry);
5820
+ const filePath = path5.join(dirPath, entry);
6717
5821
  let updatedAt;
6718
5822
  try {
6719
5823
  const fileStat = await stat3(filePath);
@@ -6737,11 +5841,13 @@ var TranscriptProvider = class _TranscriptProvider {
6737
5841
  kind = "transcript";
6738
5842
  targetName;
6739
5843
  entries;
5844
+ entriesByTestId;
6740
5845
  cursor = 0;
6741
5846
  constructor(targetName, entries) {
6742
5847
  this.targetName = targetName;
6743
5848
  this.id = `transcript:${targetName}`;
6744
5849
  this.entries = entries;
5850
+ this.entriesByTestId = new Map(entries.map((entry) => [entry.testId, entry]));
6745
5851
  }
6746
5852
  /**
6747
5853
  * Create a TranscriptProvider from a JSONL file path.
@@ -6758,13 +5864,11 @@ var TranscriptProvider = class _TranscriptProvider {
6758
5864
  get lineCount() {
6759
5865
  return this.entries.length;
6760
5866
  }
6761
- async invoke(_request) {
6762
- if (this.cursor >= this.entries.length) {
6763
- throw new Error(
6764
- `Transcript exhausted: ${this.entries.length} entr${this.entries.length === 1 ? "y" : "ies"} available but ${this.cursor + 1} invocations attempted. Each transcript entry maps to one test case.`
6765
- );
6766
- }
6767
- const entry = this.entries[this.cursor++];
5867
+ get testIds() {
5868
+ return this.entries.map((entry) => entry.testId);
5869
+ }
5870
+ async invoke(request) {
5871
+ const entry = request.evalCaseId ? this.entryForTestId(request.evalCaseId) : this.nextPositionalEntry();
6768
5872
  return {
6769
5873
  output: entry.messages,
6770
5874
  tokenUsage: entry.tokenUsage ? {
@@ -6778,7 +5882,153 @@ var TranscriptProvider = class _TranscriptProvider {
6778
5882
  startTime: entry.source.startedAt
6779
5883
  };
6780
5884
  }
5885
+ entryForTestId(testId) {
5886
+ const entry = this.entriesByTestId.get(testId);
5887
+ if (entry) {
5888
+ return entry;
5889
+ }
5890
+ throw new Error(
5891
+ `Transcript replay found no entry for test_id=${testId}. Available test_id values: ${this.testIds.join(", ") || "<none>"}`
5892
+ );
5893
+ }
5894
+ nextPositionalEntry() {
5895
+ if (this.cursor >= this.entries.length) {
5896
+ throw new Error(
5897
+ `Transcript exhausted: ${this.entries.length} entr${this.entries.length === 1 ? "y" : "ies"} available but ${this.cursor + 1} invocations attempted. Each transcript entry maps to one test case.`
5898
+ );
5899
+ }
5900
+ return this.entries[this.cursor++];
5901
+ }
6781
5902
  };
5903
+ function parseCopilotEvents(eventsJsonl) {
5904
+ const messages = [];
5905
+ const meta = { sessionId: "", model: "", cwd: "" };
5906
+ let totalInputTokens = 0;
5907
+ let totalOutputTokens = 0;
5908
+ let hasUsage = false;
5909
+ let startTimestamp;
5910
+ let endTimestamp;
5911
+ const toolCallsInProgress = /* @__PURE__ */ new Map();
5912
+ const lines = eventsJsonl.split("\n").filter((l) => l.trim().length > 0);
5913
+ for (const line of lines) {
5914
+ let event;
5915
+ try {
5916
+ event = JSON.parse(line);
5917
+ } catch {
5918
+ continue;
5919
+ }
5920
+ const eventType = event.type;
5921
+ if (!eventType) continue;
5922
+ const data = event.data ?? {};
5923
+ switch (eventType) {
5924
+ case "session.start": {
5925
+ meta.sessionId = String(data.sessionId ?? "");
5926
+ const ctx = data.context;
5927
+ meta.cwd = String(ctx?.cwd ?? "");
5928
+ meta.repository = ctx?.repository ? String(ctx.repository) : void 0;
5929
+ meta.branch = ctx?.branch ? String(ctx.branch) : void 0;
5930
+ const ts = event.timestamp ?? data.startTime;
5931
+ meta.startedAt = ts ? String(ts) : void 0;
5932
+ startTimestamp = ts ? String(ts) : void 0;
5933
+ break;
5934
+ }
5935
+ case "user.message": {
5936
+ messages.push({
5937
+ role: "user",
5938
+ content: data.content != null ? String(data.content) : ""
5939
+ });
5940
+ break;
5941
+ }
5942
+ case "assistant.message": {
5943
+ const toolRequests = data.toolRequests;
5944
+ const toolCalls = (toolRequests ?? []).map(
5945
+ (req) => normalizeToolCall("copilot-events", {
5946
+ tool: String(req.name ?? req.toolName ?? ""),
5947
+ input: req.arguments,
5948
+ id: req.toolCallId ? String(req.toolCallId) : void 0
5949
+ })
5950
+ );
5951
+ messages.push({
5952
+ role: "assistant",
5953
+ content: data.content != null ? String(data.content) : void 0,
5954
+ toolCalls: toolCalls.length > 0 ? toolCalls : void 0
5955
+ });
5956
+ break;
5957
+ }
5958
+ case "skill.invoked": {
5959
+ const skillName = String(data.name ?? "");
5960
+ messages.push({
5961
+ role: "assistant",
5962
+ toolCalls: [
5963
+ {
5964
+ tool: "Skill",
5965
+ input: { skill: skillName }
5966
+ }
5967
+ ]
5968
+ });
5969
+ break;
5970
+ }
5971
+ case "tool.execution_start": {
5972
+ const toolCallId = String(data.toolCallId ?? "");
5973
+ if (toolCallId) {
5974
+ toolCallsInProgress.set(toolCallId, {
5975
+ toolName: String(data.toolName ?? ""),
5976
+ input: data.arguments,
5977
+ toolCallId
5978
+ });
5979
+ }
5980
+ break;
5981
+ }
5982
+ case "tool.execution_complete": {
5983
+ const toolCallId = String(data.toolCallId ?? "");
5984
+ const started = toolCallsInProgress.get(toolCallId);
5985
+ if (started) {
5986
+ toolCallsInProgress.delete(toolCallId);
5987
+ messages.push({
5988
+ role: "assistant",
5989
+ toolCalls: [
5990
+ normalizeToolCall("copilot-events", {
5991
+ tool: started.toolName,
5992
+ input: started.input,
5993
+ output: data.result,
5994
+ id: toolCallId
5995
+ })
5996
+ ]
5997
+ });
5998
+ }
5999
+ break;
6000
+ }
6001
+ case "session.shutdown": {
6002
+ endTimestamp = event.timestamp ? String(event.timestamp) : void 0;
6003
+ const modelMetrics = data.modelMetrics;
6004
+ if (modelMetrics) {
6005
+ for (const metrics of Object.values(modelMetrics)) {
6006
+ if (metrics.usage) {
6007
+ hasUsage = true;
6008
+ totalInputTokens += Number(metrics.usage.inputTokens ?? 0);
6009
+ totalOutputTokens += Number(metrics.usage.outputTokens ?? 0);
6010
+ }
6011
+ }
6012
+ }
6013
+ const currentModel = data.currentModel;
6014
+ if (currentModel && !meta.model) {
6015
+ meta.model = String(currentModel);
6016
+ }
6017
+ break;
6018
+ }
6019
+ }
6020
+ }
6021
+ let durationMs;
6022
+ if (startTimestamp && endTimestamp) {
6023
+ durationMs = new Date(endTimestamp).getTime() - new Date(startTimestamp).getTime();
6024
+ }
6025
+ return {
6026
+ messages,
6027
+ meta,
6028
+ tokenUsage: hasUsage ? { input: totalInputTokens, output: totalOutputTokens } : void 0,
6029
+ durationMs
6030
+ };
6031
+ }
6782
6032
  function createAgentKernel() {
6783
6033
  return { status: "stub" };
6784
6034
  }
@@ -6788,12 +6038,6 @@ export {
6788
6038
  traceEnvelopeToTraceSessionResponse,
6789
6039
  buildTraceSpanTree,
6790
6040
  normalizeTraceArtifactToTraceSessionResponse,
6791
- isExperimentFileReference,
6792
- deriveExperimentNameFromPath,
6793
- loadExperimentConfig,
6794
- normalizeExperimentConfig,
6795
- fingerprintExperimentConfig,
6796
- buildExperimentArtifactMetadata,
6797
6041
  transpileEvalYaml,
6798
6042
  transpileEvalYamlFile,
6799
6043
  getOutputFilenames,
@@ -6801,7 +6045,6 @@ export {
6801
6045
  defineConfig,
6802
6046
  loadTsConfig,
6803
6047
  generateRubrics,
6804
- scanRepoDeps,
6805
6048
  normalizeResultsConfig,
6806
6049
  resolveResultsRepoUrl,
6807
6050
  getResultsRepoLocalPaths,
@@ -6827,15 +6070,14 @@ export {
6827
6070
  setupWipWorktree,
6828
6071
  pushWipCheckpoint,
6829
6072
  deleteWipBranch,
6073
+ resolveGitResultsIndexCacheFile,
6074
+ resolveGitRunsRefCommit,
6075
+ listGitRunsCached,
6830
6076
  listGitRuns,
6831
6077
  materializeGitRun,
6832
6078
  syncProject,
6833
6079
  syncProjects,
6834
6080
  trimBaselineResult,
6835
- DEFAULT_CATEGORY,
6836
- deriveCategory,
6837
- OtelTraceExporter,
6838
- OtelStreamingObserver,
6839
6081
  RunBudgetTracker,
6840
6082
  parseEnvOutput,
6841
6083
  runBeforeSessionHook,
@@ -6844,6 +6086,7 @@ export {
6844
6086
  discoverCodexSessions,
6845
6087
  discoverClaudeSessions,
6846
6088
  TranscriptProvider,
6089
+ parseCopilotEvents,
6847
6090
  createAgentKernel
6848
6091
  };
6849
- //# sourceMappingURL=chunk-UG4JHZYM.js.map
6092
+ //# sourceMappingURL=chunk-ALVZQUZP.js.map