agentv 4.44.0-next.1 → 5.1.0-next.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/README.md +236 -54
  2. package/dist/{agentv-provider-AYXH7WLW-NJRC6UQX.js → agentv-provider-AYXH7WLW-6YZKZ7PS.js} +2 -2
  3. package/dist/{artifact-writer-72MQQELS.js → artifact-writer-IORG7CW5.js} +21 -12
  4. package/dist/{chunk-NPVGBFF6.js → chunk-7BGERE6L.js} +2 -43
  5. package/dist/chunk-7BGERE6L.js.map +1 -0
  6. package/dist/{chunk-CXHYTYHU.js → chunk-AM2IPHKS.js} +3496 -1955
  7. package/dist/chunk-AM2IPHKS.js.map +1 -0
  8. package/dist/{chunk-KAGZRUO6.js → chunk-CBBBRTQ3.js} +127 -80
  9. package/dist/chunk-CBBBRTQ3.js.map +1 -0
  10. package/dist/{chunk-2N3YSEMI.js → chunk-GIRUNQGP.js} +8323 -9226
  11. package/dist/chunk-GIRUNQGP.js.map +1 -0
  12. package/dist/{chunk-NCNDA3F6.js → chunk-GJWPQXHN.js} +1070 -791
  13. package/dist/chunk-GJWPQXHN.js.map +1 -0
  14. package/dist/{chunk-BPGJ4HBU.js → chunk-PEUTJS7B.js} +3 -3
  15. package/dist/{chunk-5H446C7X.js → chunk-QMRVH5ZP.js} +2 -8
  16. package/dist/{chunk-5DYGN547.js → chunk-ZGPCX334.js} +44263 -30424
  17. package/dist/chunk-ZGPCX334.js.map +1 -0
  18. package/dist/cli.js +8 -9
  19. package/dist/cli.js.map +1 -1
  20. package/dist/dashboard/assets/{index-DitffAxS.js → index-BSHoWrdr.js} +1 -1
  21. package/dist/dashboard/assets/index-BaLLxmVO.js +121 -0
  22. package/dist/dashboard/assets/index-zvUKAkKn.css +1 -0
  23. package/dist/dashboard/index.html +2 -2
  24. package/dist/{dist-L4YCQOQ6.js → dist-2D2GBHZV.js} +101 -51
  25. package/dist/{docker-workspace-RPPXBT27-B4AQHVWA.js → docker-workspace-NNK4Z6FR-QRTIG6TM.js} +3 -3
  26. package/dist/{exec-AR6JUUN5-6MBPURPR.js → exec-4HFFRQ3B-AXDMJNLJ.js} +3 -3
  27. package/dist/index.js +8 -9
  28. package/dist/{interactive-DRNRKLEQ.js → interactive-V4SEHLNY.js} +13 -29
  29. package/dist/interactive-V4SEHLNY.js.map +1 -0
  30. package/dist/skills/agentv-bench/SKILL.md +26 -26
  31. package/dist/skills/agentv-bench/agents/analyzer.md +7 -7
  32. package/dist/skills/agentv-bench/agents/comparator.md +4 -5
  33. package/dist/skills/agentv-bench/agents/grader.md +4 -4
  34. package/dist/skills/agentv-bench/references/autoresearch.md +4 -4
  35. package/dist/skills/agentv-bench/references/description-optimization.md +2 -2
  36. package/dist/skills/agentv-bench/references/environment-adaptation.md +10 -10
  37. package/dist/skills/agentv-bench/references/eval-yaml-spec.md +52 -26
  38. package/dist/skills/agentv-bench/references/migrating-from-skill-creator.md +6 -6
  39. package/dist/skills/agentv-bench/references/schemas.md +6 -6
  40. package/dist/skills/agentv-bench/references/subagent-pipeline.md +14 -13
  41. package/dist/skills/agentv-eval-migrations/SKILL.md +33 -0
  42. package/dist/skills/agentv-eval-migrations/references/breaking-changes.md +1013 -0
  43. package/dist/skills/agentv-eval-review/SKILL.md +6 -3
  44. package/dist/skills/agentv-eval-review/scripts/lint_eval.py +1 -1
  45. package/dist/skills/agentv-eval-writer/SKILL.md +329 -200
  46. package/dist/skills/agentv-eval-writer/references/{config-schema.json → config.schema.json} +6 -5
  47. package/dist/skills/agentv-eval-writer/references/custom-evaluators.md +23 -18
  48. package/dist/skills/agentv-eval-writer/references/eval.schema.json +10447 -0
  49. package/dist/skills/agentv-eval-writer/references/python-helpers.md +8 -8
  50. package/dist/skills/agentv-eval-writer/references/rubric-evaluator.md +21 -22
  51. package/dist/skills/agentv-trace-analyst/SKILL.md +4 -4
  52. package/dist/templates/.agentv/targets.yaml +41 -40
  53. package/dist/{ts-eval-loader-S7ESROX2-6CB3HB6H.js → ts-eval-loader-SSPJVEHD-AL2A3V5Z.js} +4 -4
  54. package/package.json +13 -4
  55. package/dist/chunk-2N3YSEMI.js.map +0 -1
  56. package/dist/chunk-5DYGN547.js.map +0 -1
  57. package/dist/chunk-76FOHROU.js +0 -96
  58. package/dist/chunk-76FOHROU.js.map +0 -1
  59. package/dist/chunk-CXHYTYHU.js.map +0 -1
  60. package/dist/chunk-HSHTTN6C.js +0 -565
  61. package/dist/chunk-HSHTTN6C.js.map +0 -1
  62. package/dist/chunk-KAGZRUO6.js.map +0 -1
  63. package/dist/chunk-LRULMAAA.js +0 -1711
  64. package/dist/chunk-LRULMAAA.js.map +0 -1
  65. package/dist/chunk-NCNDA3F6.js.map +0 -1
  66. package/dist/chunk-NPVGBFF6.js.map +0 -1
  67. package/dist/chunk-R3CXYNTN.js +0 -1238
  68. package/dist/chunk-R3CXYNTN.js.map +0 -1
  69. package/dist/chunk-VQ2ZO7XJ.js +0 -2098
  70. package/dist/chunk-VQ2ZO7XJ.js.map +0 -1
  71. package/dist/chunk-XALGXSKB.js +0 -21
  72. package/dist/chunk-XALGXSKB.js.map +0 -1
  73. package/dist/dashboard/assets/index-1Hvy8sBy.js +0 -119
  74. package/dist/dashboard/assets/index-J5sPcQO3.css +0 -1
  75. package/dist/esm-NNMQYIKT.js +0 -32
  76. package/dist/esm-R77SNOF5.js +0 -65
  77. package/dist/esm-RVQPUGWH.js +0 -1207
  78. package/dist/esm-RVQPUGWH.js.map +0 -1
  79. package/dist/esm-TTEMJEVV.js +0 -933
  80. package/dist/esm-TTEMJEVV.js.map +0 -1
  81. package/dist/esm-ZADQ4XQH-QUFGGZL4.js +0 -924
  82. package/dist/esm-ZADQ4XQH-QUFGGZL4.js.map +0 -1
  83. package/dist/exec-AR6JUUN5-6MBPURPR.js.map +0 -1
  84. package/dist/getMachineId-bsd-HSK5LZMG.js +0 -41
  85. package/dist/getMachineId-bsd-HSK5LZMG.js.map +0 -1
  86. package/dist/getMachineId-darwin-4DP6CCJV.js +0 -41
  87. package/dist/getMachineId-darwin-4DP6CCJV.js.map +0 -1
  88. package/dist/getMachineId-linux-44LJ5UJB.js +0 -33
  89. package/dist/getMachineId-linux-44LJ5UJB.js.map +0 -1
  90. package/dist/getMachineId-unsupported-NVK6IATM.js +0 -24
  91. package/dist/getMachineId-unsupported-NVK6IATM.js.map +0 -1
  92. package/dist/getMachineId-win-YZ36S7VA.js +0 -43
  93. package/dist/getMachineId-win-YZ36S7VA.js.map +0 -1
  94. package/dist/interactive-DRNRKLEQ.js.map +0 -1
  95. package/dist/otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js +0 -9
  96. package/dist/otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js.map +0 -1
  97. package/dist/skills/agentv-eval-writer/references/eval-schema.json +0 -17355
  98. package/dist/src-76ISS3LH.js +0 -1733
  99. package/dist/src-76ISS3LH.js.map +0 -1
  100. package/dist/ts-eval-loader-S7ESROX2-6CB3HB6H.js.map +0 -1
  101. /package/dist/{agentv-provider-AYXH7WLW-NJRC6UQX.js.map → agentv-provider-AYXH7WLW-6YZKZ7PS.js.map} +0 -0
  102. /package/dist/{artifact-writer-72MQQELS.js.map → artifact-writer-IORG7CW5.js.map} +0 -0
  103. /package/dist/{chunk-BPGJ4HBU.js.map → chunk-PEUTJS7B.js.map} +0 -0
  104. /package/dist/{chunk-5H446C7X.js.map → chunk-QMRVH5ZP.js.map} +0 -0
  105. /package/dist/{dist-L4YCQOQ6.js.map → dist-2D2GBHZV.js.map} +0 -0
  106. /package/dist/{docker-workspace-RPPXBT27-B4AQHVWA.js.map → docker-workspace-NNK4Z6FR-QRTIG6TM.js.map} +0 -0
  107. /package/dist/{esm-NNMQYIKT.js.map → exec-4HFFRQ3B-AXDMJNLJ.js.map} +0 -0
  108. /package/dist/{esm-R77SNOF5.js.map → ts-eval-loader-SSPJVEHD-AL2A3V5Z.js.map} +0 -0
@@ -21,7 +21,7 @@ import {
21
21
  readTranscriptJsonl,
22
22
  releaseSharedWorkspaceSetup,
23
23
  resolveRepoCloneUrl
24
- } from "./chunk-5DYGN547.js";
24
+ } from "./chunk-ZGPCX334.js";
25
25
 
26
26
  // ../../packages/core/dist/index.js
27
27
  import { readFileSync } from "node:fs";
@@ -32,17 +32,18 @@ import micromatch from "micromatch";
32
32
  import { readFile } from "node:fs/promises";
33
33
  import path3 from "node:path";
34
34
  import { execFile, spawn } from "node:child_process";
35
- import { createHash } from "node:crypto";
35
+ import { createHash, randomBytes } from "node:crypto";
36
36
  import {
37
37
  existsSync,
38
38
  mkdirSync,
39
39
  mkdtempSync,
40
40
  readFileSync as readFileSync2,
41
+ realpathSync,
41
42
  renameSync,
42
43
  rmSync,
43
44
  writeFileSync
44
45
  } from "node:fs";
45
- import { cp, lstat, mkdtemp, readdir, rm, stat } from "node:fs/promises";
46
+ import { cp, lstat, mkdtemp, readFile as readFile2, readdir, rm, stat, writeFile } from "node:fs/promises";
46
47
  import os from "node:os";
47
48
  import path4 from "node:path";
48
49
  import { promisify } from "node:util";
@@ -1478,7 +1479,7 @@ function isPrimitive(value) {
1478
1479
  function dropUndefined2(value) {
1479
1480
  return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
1480
1481
  }
1481
- function codeGraderInstruction(graderName, description) {
1482
+ function scriptGraderInstruction(graderName, description) {
1482
1483
  const desc = description ? ` This grader: ${description}.` : "";
1483
1484
  return `Run \`agentv eval assert ${graderName} --agent-output <agent_output> --agent-input <original_prompt>\` and check the result.${desc} The command accepts --agent-output (the agent's full response text) and --agent-input (the original user prompt). It returns JSON on stdout: {"score": 0-1, "reasoning": "..."}. A score >= 0.5 means pass (exit 0); below 0.5 means fail (exit 1).`;
1484
1485
  }
@@ -1496,21 +1497,15 @@ function assertionToNaturalLanguage(entry) {
1496
1497
  switch (type) {
1497
1498
  case "skill-trigger":
1498
1499
  return null;
1499
- case "rubrics": {
1500
- if (typeof entry.criteria === "string") {
1501
- return entry.criteria;
1502
- }
1503
- return null;
1504
- }
1500
+ case "llm-rubric":
1501
+ return typeof entry.value === "string" ? entry.value : null;
1505
1502
  case "contains":
1506
1503
  return `Output contains '${entry.value}'`;
1507
- case "contains-any":
1508
- case "contains_any": {
1504
+ case "contains-any": {
1509
1505
  const values = Array.isArray(entry.value) ? entry.value.join("', '") : entry.value;
1510
1506
  return `Output contains any of: '${values}'`;
1511
1507
  }
1512
- case "contains-all":
1513
- case "contains_all": {
1508
+ case "contains-all": {
1514
1509
  const values = Array.isArray(entry.value) ? entry.value.join("', '") : entry.value;
1515
1510
  return `Output contains all of: '${values}'`;
1516
1511
  }
@@ -1521,35 +1516,27 @@ function assertionToNaturalLanguage(entry) {
1521
1516
  case "equals":
1522
1517
  return `Output exactly equals: ${entry.value}`;
1523
1518
  case "is-json":
1524
- case "is_json":
1525
1519
  return "Output is valid JSON";
1526
1520
  case "starts-with":
1527
- case "starts_with":
1528
1521
  return `Output starts with '${entry.value}'`;
1529
1522
  case "ends-with":
1530
- case "ends_with":
1531
1523
  return `Output ends with '${entry.value}'`;
1532
1524
  case "llm-grader":
1533
- case "llm_grader": {
1534
1525
  if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
1535
1526
  return null;
1536
1527
  }
1537
1528
  return typeof entry.prompt === "string" ? entry.prompt : null;
1538
- }
1539
- case "tool-trajectory":
1540
- case "tool_trajectory": {
1529
+ case "tool-trajectory": {
1541
1530
  const expectedArr = Array.isArray(entry.expected) ? entry.expected : [];
1542
1531
  const tools = expectedArr.map((e) => e.tool).filter(Boolean).join(", ");
1543
1532
  return tools ? `Agent called tools in order: ${tools}` : "Agent followed expected tool trajectory";
1544
1533
  }
1545
- case "code-grader":
1546
- case "code_grader": {
1547
- const graderName = entry.name ?? deriveGraderNameFromCommand(entry.command) ?? "code-grader";
1534
+ case "script": {
1535
+ const graderName = entry.metric ?? deriveGraderNameFromCommand(entry.command) ?? "script";
1548
1536
  const desc = typeof entry.description === "string" ? entry.description : void 0;
1549
- return codeGraderInstruction(graderName, desc);
1537
+ return scriptGraderInstruction(graderName, desc);
1550
1538
  }
1551
- case "field-accuracy":
1552
- case "field_accuracy": {
1539
+ case "field-accuracy": {
1553
1540
  const fieldPaths = Array.isArray(entry.fields) ? entry.fields.map((f) => f.path).filter(Boolean).join(", ") : "";
1554
1541
  return fieldPaths ? `Fields ${fieldPaths} match expected values` : "Fields match expected values";
1555
1542
  }
@@ -1558,14 +1545,12 @@ function assertionToNaturalLanguage(entry) {
1558
1545
  case "cost":
1559
1546
  return typeof entry.budget === "number" ? `Cost under $${entry.budget}` : "Cost within budget";
1560
1547
  case "token-usage":
1561
- case "token_usage":
1562
1548
  return "Token usage within limits";
1563
1549
  case "execution-metrics":
1564
- case "execution_metrics":
1565
1550
  return "Execution within metric bounds";
1566
1551
  default: {
1567
1552
  if (entry.command !== void 0 && type) {
1568
- return codeGraderInstruction(deriveGraderNameFromCommand(entry.command) ?? type);
1553
+ return scriptGraderInstruction(deriveGraderNameFromCommand(entry.command) ?? type);
1569
1554
  }
1570
1555
  if (typeof entry.criteria === "string") return entry.criteria;
1571
1556
  if (typeof entry.prompt === "string") return entry.prompt;
@@ -1574,7 +1559,17 @@ function assertionToNaturalLanguage(entry) {
1574
1559
  }
1575
1560
  }
1576
1561
  function assertionToNaturalLanguageList(entry) {
1577
- if (entry.type === "llm-grader" || entry.type === "llm_grader") {
1562
+ if (entry.type === "llm-rubric") {
1563
+ if (Array.isArray(entry.value) && entry.value.length > 0) {
1564
+ return entry.value.map((rubric) => {
1565
+ if (typeof rubric === "string") return rubric;
1566
+ if (!rubric || typeof rubric !== "object") return void 0;
1567
+ const item = rubric;
1568
+ return item.outcome ?? item.criteria ?? item.id;
1569
+ }).filter((value) => typeof value === "string");
1570
+ }
1571
+ }
1572
+ if (entry.type === "llm-grader") {
1578
1573
  if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
1579
1574
  return entry.rubrics.map((r) => r.outcome ?? r.criteria ?? r.id).filter((s) => typeof s === "string");
1580
1575
  }
@@ -1585,16 +1580,6 @@ function assertionToNaturalLanguageList(entry) {
1585
1580
  function extractTriggerAssertions(assertions) {
1586
1581
  return assertions.filter((a) => a.type === "skill-trigger");
1587
1582
  }
1588
- function resolveAssertions(rawCase) {
1589
- if (Array.isArray(rawCase.assertions)) return rawCase.assertions;
1590
- if (Array.isArray(rawCase.assert)) return rawCase.assert;
1591
- return [];
1592
- }
1593
- function resolveSuiteAssertions(suite) {
1594
- if (Array.isArray(suite.assertions)) return suite.assertions;
1595
- if (Array.isArray(suite.assert)) return suite.assert;
1596
- return [];
1597
- }
1598
1583
  function extractInput(rawCase) {
1599
1584
  const files = Array.isArray(rawCase.input_files) ? rawCase.input_files.filter((f) => typeof f === "string") : [];
1600
1585
  const input = rawCase.input;
@@ -1632,7 +1617,6 @@ function extractExpectedOutput(raw) {
1632
1617
  return JSON.stringify(raw);
1633
1618
  }
1634
1619
  function transpileEvalYaml(suite, source = "EVAL.yaml") {
1635
- const warnings = [];
1636
1620
  const files = /* @__PURE__ */ new Map();
1637
1621
  if (typeof suite !== "object" || suite === null) {
1638
1622
  throw new Error(`Invalid EVAL.yaml: expected an object in '${source}'`);
@@ -1641,10 +1625,7 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
1641
1625
  if (!Array.isArray(rawSuite.tests)) {
1642
1626
  throw new Error(`Invalid EVAL.yaml: missing 'tests' array in '${source}'`);
1643
1627
  }
1644
- if (rawSuite.assert !== void 0 && rawSuite.assertions === void 0) {
1645
- warnings.push("'assert' is deprecated at the suite level. Use 'assertions' instead.");
1646
- }
1647
- const suiteAssertions = resolveSuiteAssertions(rawSuite);
1628
+ const suiteAssertions = rawSuite.assert ?? [];
1648
1629
  const suiteNlAssertions = suiteAssertions.filter((a) => a.type !== "skill-trigger").flatMap(assertionToNaturalLanguageList);
1649
1630
  function getSkillFile(skillName) {
1650
1631
  const existing = files.get(skillName);
@@ -1656,10 +1637,11 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
1656
1637
  const tests = rawSuite.tests;
1657
1638
  for (let idx = 0; idx < tests.length; idx++) {
1658
1639
  const rawCase = tests[idx];
1659
- const caseAssertions = resolveAssertions(rawCase);
1660
- if (rawCase.assert !== void 0 && rawCase.assertions === void 0) {
1661
- const caseId = rawCase.id ?? idx + 1;
1662
- warnings.push(`Test '${caseId}': 'assert' is deprecated. Use 'assertions' instead.`);
1640
+ const caseAssertions = rawCase.assert ?? [];
1641
+ if (typeof rawCase.criteria === "string" && rawCase.criteria.trim() && rawCase.assert !== void 0) {
1642
+ throw new Error(
1643
+ `Invalid EVAL.yaml test '${rawCase.id ?? idx + 1}' in '${source}': do not combine test-level 'criteria' with 'assert'. Put human-readable case descriptions in 'description', or express grading text as an explicit assertion such as { type: 'llm-rubric', value: ... }.`
1644
+ );
1663
1645
  }
1664
1646
  const nlAssertions = [];
1665
1647
  if (typeof rawCase.criteria === "string" && rawCase.criteria.trim()) {
@@ -1712,7 +1694,7 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
1712
1694
  files.delete("_no-skill");
1713
1695
  }
1714
1696
  }
1715
- return { files, warnings };
1697
+ return { files, warnings: [] };
1716
1698
  }
1717
1699
  function transpileEvalYamlFile(evalYamlPath) {
1718
1700
  const content = readFileSync(evalYamlPath, "utf8");
@@ -1754,33 +1736,10 @@ function toRepoPins(repos) {
1754
1736
  ...repo.path !== void 0 && { path: repo.path },
1755
1737
  ...repo.repo !== void 0 && { repo: repo.repo },
1756
1738
  ...repo.commit !== void 0 && { commit: repo.commit },
1757
- ...repo.base_commit !== void 0 && { baseCommit: repo.base_commit },
1758
1739
  ...repo.ancestor !== void 0 && { ancestor: repo.ancestor },
1759
1740
  ...repo.sparse !== void 0 && { sparse: repo.sparse }
1760
1741
  }));
1761
1742
  }
1762
- function poolMetadata(setup, selectedSlotPath) {
1763
- const slot = setup.poolSlot ?? setup.poolSlots.find((candidate) => candidate.path === selectedSlotPath);
1764
- if (!slot) {
1765
- return void 0;
1766
- }
1767
- return {
1768
- fingerprint: slot.fingerprint,
1769
- slotIndex: slot.index,
1770
- lockPath: slot.lockPath
1771
- };
1772
- }
1773
- async function releaseUnselectedPoolSlots(setup, selectedSlotPath) {
1774
- if (!setup.poolManager) {
1775
- return;
1776
- }
1777
- for (const slot of setup.poolSlots) {
1778
- if (slot.path !== selectedSlotPath) {
1779
- await setup.poolManager.releaseSlot(slot).catch(() => {
1780
- });
1781
- }
1782
- }
1783
- }
1784
1743
  async function prepareEvalWorkspace(options) {
1785
1744
  const evalPath = path2.resolve(options.testFilePath);
1786
1745
  const evalRunId = randomUUID();
@@ -1809,15 +1768,11 @@ async function prepareEvalWorkspace(options) {
1809
1768
  evalDir,
1810
1769
  verbose: options.verbose,
1811
1770
  workers,
1812
- poolMaxSlots: options.poolMaxSlots,
1813
1771
  workspacePath: options.workspacePath,
1814
- legacyWorkspacePath: options.workspace,
1815
- workspaceMode: options.workspaceMode,
1816
- workspaceClean: options.workspaceClean
1772
+ legacyWorkspacePath: options.workspace
1817
1773
  });
1818
- const testPoolSlot = sharedSetup.availablePoolSlots.length > 0 ? sharedSetup.availablePoolSlots.pop() : void 0;
1819
- const selectedWorkspacePath = testPoolSlot?.path ?? sharedSetup.sharedWorkspacePath;
1820
- const selectedBaselineCommit = testPoolSlot ? sharedSetup.poolSlotBaselines.get(testPoolSlot.path) : sharedSetup.sharedBaselineCommit;
1774
+ const selectedWorkspacePath = sharedSetup.sharedWorkspacePath;
1775
+ const selectedBaselineCommit = sharedSetup.sharedBaselineCommit;
1821
1776
  const caseSetup = await prepareEvalCaseWorkspace({
1822
1777
  evalCase,
1823
1778
  targetName: options.target.name,
@@ -1829,6 +1784,7 @@ async function prepareEvalWorkspace(options) {
1829
1784
  evalDir,
1830
1785
  cleanupWorkspaces: options.cleanupWorkspaces,
1831
1786
  targetHooks: options.targetHooks,
1787
+ sharedExtensionState: sharedSetup.extensionState,
1832
1788
  setupDebug: options.verbose
1833
1789
  });
1834
1790
  if (!caseSetup.workspacePath) {
@@ -1836,8 +1792,6 @@ async function prepareEvalWorkspace(options) {
1836
1792
  `No workspace was materialized for test "${evalCase.id}". Add workspace.template, workspace.repos, or workspace.hooks before preparing an external attempt.`
1837
1793
  );
1838
1794
  }
1839
- await releaseUnselectedPoolSlots(sharedSetup, caseSetup.workspacePath);
1840
- const pool = poolMetadata(sharedSetup, caseSetup.workspacePath);
1841
1795
  return {
1842
1796
  evalPath,
1843
1797
  testId: evalCase.id,
@@ -1849,6 +1803,12 @@ async function prepareEvalWorkspace(options) {
1849
1803
  },
1850
1804
  createdAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
1851
1805
  hookExecutions: [...sharedSetup.hookExecutions, ...caseSetup.hookExecutions],
1806
+ ...caseSetup.extensionState?.providerContext !== void 0 && {
1807
+ providerContext: caseSetup.extensionState.providerContext
1808
+ },
1809
+ ...caseSetup.extensionState?.metadata !== void 0 && {
1810
+ metadata: caseSetup.extensionState.metadata
1811
+ },
1852
1812
  repoPins: toRepoPins(evalCase.workspace?.repos),
1853
1813
  baseline: caseSetup.baselineCommit ? { status: "initialized", commit: caseSetup.baselineCommit } : { status: "unavailable" },
1854
1814
  promptSource: {
@@ -1866,8 +1826,7 @@ async function prepareEvalWorkspace(options) {
1866
1826
  retainOnFailure,
1867
1827
  manualCleanup: true
1868
1828
  },
1869
- sharedWorkspace: caseSetup.isSharedWorkspace,
1870
- ...pool !== void 0 && { pool }
1829
+ sharedWorkspace: caseSetup.isSharedWorkspace
1871
1830
  };
1872
1831
  } catch (error) {
1873
1832
  if (sharedSetup) {
@@ -1877,22 +1836,51 @@ async function prepareEvalWorkspace(options) {
1877
1836
  throw error;
1878
1837
  }
1879
1838
  }
1839
+ var ExecutionConfigSchema = external_exports.object({
1840
+ /** General eval parallelism (default: 3) */
1841
+ maxConcurrency: external_exports.number().int().min(1).max(50).optional(),
1842
+ /** Maximum retries on failure (default: 2) */
1843
+ maxRetries: external_exports.number().int().min(0).optional(),
1844
+ /** Agent timeout in milliseconds. No timeout if not set. */
1845
+ agentTimeoutMs: external_exports.number().int().min(0).optional(),
1846
+ /** Enable verbose logging */
1847
+ verbose: external_exports.boolean().optional(),
1848
+ /** Always keep temp workspaces after eval */
1849
+ keepWorkspaces: external_exports.boolean().optional()
1850
+ }).passthrough().superRefine((value, ctx) => {
1851
+ const supportedFields = /* @__PURE__ */ new Set([
1852
+ "maxConcurrency",
1853
+ "maxRetries",
1854
+ "agentTimeoutMs",
1855
+ "verbose",
1856
+ "keepWorkspaces"
1857
+ ]);
1858
+ if ("otelFile" in value) {
1859
+ ctx.addIssue({
1860
+ code: external_exports.ZodIssueCode.custom,
1861
+ path: ["otelFile"],
1862
+ message: "execution.otelFile has been removed. Emit OpenTelemetry/OpenInference traces from the system under test or provider and correlate AgentV run artifacts with external_trace metadata."
1863
+ });
1864
+ }
1865
+ for (const key of Object.keys(value)) {
1866
+ if (!supportedFields.has(key) && key !== "otelFile") {
1867
+ ctx.addIssue({
1868
+ code: external_exports.ZodIssueCode.custom,
1869
+ path: [key],
1870
+ message: `Unsupported execution field '${key}'`
1871
+ });
1872
+ }
1873
+ }
1874
+ }).transform(({ maxConcurrency, maxRetries, agentTimeoutMs, verbose, keepWorkspaces }) => ({
1875
+ ...maxConcurrency !== void 0 && { maxConcurrency },
1876
+ ...maxRetries !== void 0 && { maxRetries },
1877
+ ...agentTimeoutMs !== void 0 && { agentTimeoutMs },
1878
+ ...verbose !== void 0 && { verbose },
1879
+ ...keepWorkspaces !== void 0 && { keepWorkspaces }
1880
+ }));
1880
1881
  var AgentVConfigSchema = external_exports.object({
1881
1882
  /** Default execution settings */
1882
- execution: external_exports.object({
1883
- /** Number of parallel workers (default: 3) */
1884
- workers: external_exports.number().int().min(1).max(50).optional(),
1885
- /** Maximum retries on failure (default: 2) */
1886
- maxRetries: external_exports.number().int().min(0).optional(),
1887
- /** Agent timeout in milliseconds. No timeout if not set. */
1888
- agentTimeoutMs: external_exports.number().int().min(0).optional(),
1889
- /** Enable verbose logging */
1890
- verbose: external_exports.boolean().optional(),
1891
- /** Always keep temp workspaces after eval */
1892
- keepWorkspaces: external_exports.boolean().optional(),
1893
- /** Write OTLP JSON trace to this path (supports {timestamp} placeholder) */
1894
- otelFile: external_exports.string().optional()
1895
- }).optional(),
1883
+ execution: ExecutionConfigSchema.optional(),
1896
1884
  /** Output settings */
1897
1885
  output: external_exports.object({
1898
1886
  /** Default eval run artifact directory */
@@ -1922,7 +1910,9 @@ var AgentVConfigSchema = external_exports.object({
1922
1910
  * stderr is forwarded to the user. Non-zero exit aborts with an error.
1923
1911
  */
1924
1912
  beforeSession: external_exports.string().optional()
1925
- }).optional()
1913
+ }).optional(),
1914
+ /** Project-local reusable references for fields that support ref://name */
1915
+ refs: external_exports.record(external_exports.string().min(1)).optional()
1926
1916
  });
1927
1917
  function defineConfig(config) {
1928
1918
  return AgentVConfigSchema.parse(config);
@@ -2106,9 +2096,12 @@ function extractReposFromObject(obj) {
2106
2096
  }
2107
2097
  var execFileAsync = promisify(execFile);
2108
2098
  var RESULTS_REPO_RESULTS_DIR = ".agentv/results";
2109
- var RESULTS_REPO_RUNS_DIR = "runs";
2110
- var RESULTS_REPO_METADATA_DIR = "metadata";
2111
- var RESULTS_REPO_TRACKED_DIRS = [RESULTS_REPO_RUNS_DIR, RESULTS_REPO_METADATA_DIR];
2099
+ var RESULTS_REPO_RUNS_DIR = ".";
2100
+ var RESULTS_REPO_INDEXES_DIR = ".indexes";
2101
+ var RESULTS_REPO_CACHE_DIR = ".cache";
2102
+ var RESULTS_REPO_TRACKED_DOT_DIRS = [RESULTS_REPO_INDEXES_DIR, RESULTS_REPO_CACHE_DIR];
2103
+ var GIT_RESULTS_INDEX_CACHE_SCHEMA_VERSION = "agentv.git_results_index_cache.v1";
2104
+ var GIT_RESULTS_INDEX_LAYOUT_VERSION = "agentv.results_repo_branch_root.v1";
2112
2105
  var FALLBACK_RESULTS_REPO_COMMIT_EMAIL = "agentv@results-repo";
2113
2106
  var FALLBACK_RESULTS_REPO_COMMIT_NAME = "AgentV Results";
2114
2107
  var GIT_COMMIT_IDENTITY_ENV_KEYS = [
@@ -2125,11 +2118,16 @@ var GIT_ENV_INHERIT_ALLOWLIST = /* @__PURE__ */ new Set([
2125
2118
  "GIT_USERNAME"
2126
2119
  ]);
2127
2120
  var DEFAULT_RESULTS_BRANCH = AGENTV_RESULTS_PRIMARY_REF;
2128
- var MANAGED_RESULTS_REMOTE = "agentv-results";
2129
2121
  var GIT_EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
2130
2122
  var RESULTS_REPO_GENESIS_MESSAGE = "chore(results): initialize AgentV results branch";
2131
2123
  var RESULTS_REPO_GENESIS_DATE = "@0 +0000";
2132
2124
  var RESULT_INDEX_FILENAME2 = "index.jsonl";
2125
+ var RUN_INTERNAL_DIRNAME = ".internal";
2126
+ var RESULTS_REPO_GITATTRIBUTES_FILE = ".gitattributes";
2127
+ var RESULTS_REPO_GITATTRIBUTES_CONTENT = `# Managed by AgentV. Artifact-aware merge so results sync never force-pushes.
2128
+ ${RESULTS_REPO_INDEXES_DIR}/*.jsonl merge=union
2129
+ */.internal/index.jsonl merge=union
2130
+ `;
2133
2131
  var activeResultsRepoSyncs = /* @__PURE__ */ new Set();
2134
2132
  function sanitizeRepoSlug(repo) {
2135
2133
  return repo.trim().replace(/[^A-Za-z0-9._-]+/g, "-");
@@ -2152,18 +2150,24 @@ function resolveLocalPath(p, baseDir) {
2152
2150
  const expanded = expandHome(p);
2153
2151
  return path4.isAbsolute(expanded) ? expanded : path4.resolve(baseDir, expanded);
2154
2152
  }
2153
+ function isNormalizedResultsConfig(config) {
2154
+ return typeof config.storageBranchWorktree === "boolean";
2155
+ }
2155
2156
  function normalizeResultsConfig(config, options) {
2157
+ if (isNormalizedResultsConfig(config)) {
2158
+ return config;
2159
+ }
2156
2160
  const baseDir = options?.baseDir ?? process.cwd();
2157
2161
  const repoUrl = (config.repo_url ?? config.repo)?.trim();
2158
- const repoPath = config.repo_path?.trim();
2159
- const explicitClonePath = config.path?.trim();
2162
+ const explicitPath = config.path?.trim();
2163
+ const repoPath = config.repo_path?.trim() ?? (repoUrl ? void 0 : explicitPath);
2164
+ const explicitClonePath = repoUrl ? explicitPath : void 0;
2160
2165
  const repo = repoUrl ?? repoPath ?? "";
2161
2166
  const branch = config.branch?.trim() || (repoPath ? DEFAULT_RESULTS_BRANCH : void 0);
2162
2167
  const useStorageBranchWorktree = Boolean(repoPath || repoUrl && explicitClonePath && branch);
2163
- const remote = config.remote?.trim() || (repoUrl && useStorageBranchWorktree ? MANAGED_RESULTS_REMOTE : "origin");
2164
- const autoPush = config.sync?.auto_push ?? config.auto_push === true;
2165
- const requirePush = config.sync?.require_push === true;
2166
- const pushConflictPolicy = config.sync?.push_conflict_policy ?? "block";
2168
+ const remote = config.remote?.trim() || "origin";
2169
+ const autoPush = config.auto_push === true;
2170
+ const requirePush = config.require_push === true;
2167
2171
  const resolvedRepoPath = repoPath ? resolveLocalPath(repoPath, baseDir) : void 0;
2168
2172
  const resolvedPath = explicitClonePath ? resolveLocalPath(explicitClonePath, baseDir) : repoUrl ? path4.join(getAgentvDataDir(), "results", sanitizeRepoSlug(repoUrl)) : resolvedRepoPath ?? path4.join(getAgentvDataDir(), "results", sanitizeRepoSlug(repo));
2169
2173
  return {
@@ -2176,16 +2180,23 @@ function normalizeResultsConfig(config, options) {
2176
2180
  path: resolvedPath,
2177
2181
  auto_push: autoPush,
2178
2182
  require_push: requirePush,
2179
- push_conflict_policy: pushConflictPolicy,
2180
- branch_prefix: config.branch_prefix?.trim() || "eval-results",
2183
+ push_conflict_policy: "block",
2181
2184
  storageBranchWorktree: useStorageBranchWorktree
2182
2185
  };
2183
2186
  }
2187
+ var GITHUB_OWNER_REPO_SHORTHAND = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/;
2188
+ function isExplicitRemoteUrl(value) {
2189
+ return value.includes("://") || value.startsWith("git@");
2190
+ }
2184
2191
  function resolveResultsRepoUrl(repo) {
2185
- if (repo.includes("://") || repo.startsWith("git@")) {
2186
- return repo;
2192
+ const trimmed = repo.trim();
2193
+ if (isExplicitRemoteUrl(trimmed)) {
2194
+ return trimmed;
2187
2195
  }
2188
- return `https://github.com/${repo}.git`;
2196
+ if (GITHUB_OWNER_REPO_SHORTHAND.test(trimmed)) {
2197
+ return `https://github.com/${trimmed}.git`;
2198
+ }
2199
+ return trimmed;
2189
2200
  }
2190
2201
  function getResultsRepoLocalPaths(repo) {
2191
2202
  const rootDir = path4.join(getAgentvDataDir(), "cache", "results-repo", sanitizeRepoSlug(repo));
@@ -2447,26 +2458,79 @@ async function resolveGitTopLevel(repoDir) {
2447
2458
  const { stdout } = await runGit(["rev-parse", "--show-toplevel"], { cwd: repoDir });
2448
2459
  return stdout.trim() || repoDir;
2449
2460
  }
2450
- async function ensureResultsRepoRemote(repoDir, config) {
2451
- if (!config.repo_url) {
2461
+ async function ensureResultsMergeConfig(repoDir) {
2462
+ let gitDir;
2463
+ try {
2464
+ const { stdout } = await runGit(["rev-parse", "--git-common-dir"], { cwd: repoDir });
2465
+ const resolved = stdout.trim();
2466
+ gitDir = resolved ? path4.isAbsolute(resolved) ? resolved : path4.join(repoDir, resolved) : path4.join(repoDir, ".git");
2467
+ } catch {
2452
2468
  return;
2453
2469
  }
2454
- const remoteUrl = resolveResultsRepoUrl(config.repo_url);
2455
- const { stdout } = await runGit(["remote", "get-url", config.remote], {
2456
- cwd: repoDir,
2457
- check: false
2458
- });
2459
- const existingUrl = stdout.trim();
2460
- if (!existingUrl) {
2461
- await runGit(["remote", "add", config.remote, remoteUrl], { cwd: repoDir });
2470
+ writeResultsInfoAttributes(gitDir);
2471
+ }
2472
+ var RESULTS_INFO_ATTRIBUTES_BEGIN = "# >>> agentv results merge attributes >>>";
2473
+ var RESULTS_INFO_ATTRIBUTES_END = "# <<< agentv results merge attributes <<<";
2474
+ function writeResultsInfoAttributes(gitDir) {
2475
+ const infoDir = path4.join(gitDir, "info");
2476
+ const attributesPath = path4.join(infoDir, "attributes");
2477
+ const managedBlock = `${RESULTS_INFO_ATTRIBUTES_BEGIN}
2478
+ ${RESULTS_REPO_GITATTRIBUTES_CONTENT.trimEnd()}
2479
+ ${RESULTS_INFO_ATTRIBUTES_END}
2480
+ `;
2481
+ let existing = "";
2482
+ try {
2483
+ existing = readFileSync2(attributesPath, "utf8");
2484
+ } catch {
2485
+ existing = "";
2486
+ }
2487
+ const blockPattern = new RegExp(
2488
+ `${escapeRegExp(RESULTS_INFO_ATTRIBUTES_BEGIN)}[\\s\\S]*?${escapeRegExp(
2489
+ RESULTS_INFO_ATTRIBUTES_END
2490
+ )}\\n?`
2491
+ );
2492
+ const withoutManaged = existing.replace(blockPattern, "");
2493
+ const next = withoutManaged.length > 0 && !withoutManaged.endsWith("\n") ? `${withoutManaged}
2494
+ ${managedBlock}` : `${withoutManaged}${managedBlock}`;
2495
+ if (next === existing) {
2496
+ return;
2497
+ }
2498
+ mkdirSync(infoDir, { recursive: true });
2499
+ writeFileSync(attributesPath, next);
2500
+ }
2501
+ function escapeRegExp(value) {
2502
+ return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
2503
+ }
2504
+ async function addResultsGitattributesToIndex(repoDir, indexEnv) {
2505
+ const { stdout: existing } = await runGit(
2506
+ ["cat-file", "-p", `:0:${RESULTS_REPO_GITATTRIBUTES_FILE}`],
2507
+ { cwd: repoDir, env: indexEnv, check: false }
2508
+ );
2509
+ if (existing === RESULTS_REPO_GITATTRIBUTES_CONTENT) {
2462
2510
  return;
2463
2511
  }
2464
- if (existingUrl !== remoteUrl) {
2465
- await runGit(["remote", "set-url", config.remote, remoteUrl], { cwd: repoDir });
2512
+ const scratchDir = mkdtempSync(path4.join(os.tmpdir(), "agentv-results-attrs-"));
2513
+ try {
2514
+ const scratchFile = path4.join(scratchDir, RESULTS_REPO_GITATTRIBUTES_FILE);
2515
+ writeFileSync(scratchFile, RESULTS_REPO_GITATTRIBUTES_CONTENT);
2516
+ const { stdout: blob } = await runGit(["hash-object", "-w", "--no-filters", scratchFile], {
2517
+ cwd: repoDir
2518
+ });
2519
+ await runGit(
2520
+ [
2521
+ "update-index",
2522
+ "--add",
2523
+ "--cacheinfo",
2524
+ `100644,${blob.trim()},${RESULTS_REPO_GITATTRIBUTES_FILE}`
2525
+ ],
2526
+ { cwd: repoDir, env: indexEnv }
2527
+ );
2528
+ } finally {
2529
+ rmSync(scratchDir, { recursive: true, force: true });
2466
2530
  }
2467
2531
  }
2468
2532
  function updateStatusFile(config, patch) {
2469
- const repo = typeof config.repo === "string" ? config.repo : config.repo_url ?? config.repo_path ?? "";
2533
+ const repo = normalizeResultsConfig(config).repo;
2470
2534
  const cachePaths = getResultsRepoLocalPaths(repo);
2471
2535
  const current = readPersistedStatus(cachePaths.statusFile);
2472
2536
  writePersistedStatus(cachePaths.statusFile, {
@@ -2480,7 +2544,9 @@ async function ensureResultsRepoClone(config) {
2480
2544
  if (!await isGitRepository(normalized.repo_path)) {
2481
2545
  throw new Error(`Results repo_path is not a git repository: ${normalized.repo_path}`);
2482
2546
  }
2483
- return resolveGitTopLevel(normalized.repo_path);
2547
+ const topLevel = await resolveGitTopLevel(normalized.repo_path);
2548
+ await ensureResultsMergeConfig(topLevel);
2549
+ return topLevel;
2484
2550
  }
2485
2551
  const cachePaths = getResultsRepoLocalPaths(normalized.repo);
2486
2552
  const cloneDir = normalized.path;
@@ -2494,10 +2560,11 @@ async function ensureResultsRepoClone(config) {
2494
2560
  await runGit([
2495
2561
  "clone",
2496
2562
  "--filter=blob:none",
2563
+ ...normalized.remote === "origin" ? [] : ["--origin", normalized.remote],
2497
2564
  resolveResultsRepoUrl(normalized.repo_url ?? normalized.repo),
2498
2565
  cloneDir
2499
2566
  ]);
2500
- await ensureResultsRepoRemote(cloneDir, normalized);
2567
+ await ensureResultsMergeConfig(cloneDir);
2501
2568
  return cloneDir;
2502
2569
  } catch (error) {
2503
2570
  updateStatusFile(normalized, { last_error: withFriendlyGitHubAuthError(error).message });
@@ -2507,7 +2574,7 @@ async function ensureResultsRepoClone(config) {
2507
2574
  if (!existsSync(gitDir)) {
2508
2575
  throw new Error(`Results repo clone path is not a git repository: ${cloneDir}`);
2509
2576
  }
2510
- await ensureResultsRepoRemote(cloneDir, normalized);
2577
+ await ensureResultsMergeConfig(cloneDir);
2511
2578
  return cloneDir;
2512
2579
  }
2513
2580
  function getResultsRepoStatus(config) {
@@ -2532,7 +2599,6 @@ function getResultsRepoStatus(config) {
2532
2599
  auto_push: normalized.auto_push,
2533
2600
  require_push: normalized.require_push,
2534
2601
  push_conflict_policy: normalized.push_conflict_policy,
2535
- branch_prefix: normalized.branch_prefix,
2536
2602
  local_dir: normalized.path,
2537
2603
  last_synced_at: persisted.last_synced_at,
2538
2604
  last_error: persisted.last_error,
@@ -2859,6 +2925,17 @@ var ResultsBranchPushConflictError = class extends Error {
2859
2925
  this.name = "ResultsBranchPushConflictError";
2860
2926
  }
2861
2927
  };
2928
+ function pendingMergeToWire(details) {
2929
+ return {
2930
+ temp_branch: details.tempBranch,
2931
+ target_branch: details.targetBranch,
2932
+ ...details.compareUrl !== void 0 && { compare_url: details.compareUrl },
2933
+ ...details.contributedRunCount !== void 0 && {
2934
+ contributed_run_count: details.contributedRunCount
2935
+ },
2936
+ created_at: details.createdAt
2937
+ };
2938
+ }
2862
2939
  function pushDetailsToWire(details) {
2863
2940
  if (!details) {
2864
2941
  return {};
@@ -2876,7 +2953,11 @@ function pushDetailsToWire(details) {
2876
2953
  ...details.forcePushedCommit !== void 0 && {
2877
2954
  force_pushed_commit: details.forcePushedCommit
2878
2955
  },
2879
- ...details.leaseCommit !== void 0 && { lease_commit: details.leaseCommit }
2956
+ ...details.leaseCommit !== void 0 && { lease_commit: details.leaseCommit },
2957
+ ...details.autoMergedRemote === true && { auto_merged_remote: true },
2958
+ ...details.pendingMerge !== void 0 && {
2959
+ pending_merge: pendingMergeToWire(details.pendingMerge)
2960
+ }
2880
2961
  };
2881
2962
  }
2882
2963
  function withBlockedStatus(status, blockReason, flags) {
@@ -2894,11 +2975,13 @@ function withPushConflictStatus(status, blockReason, details, flags) {
2894
2975
  return withBlockedStatus(
2895
2976
  {
2896
2977
  ...status,
2897
- sync_status: "push_conflict"
2978
+ sync_status: flags.syncStatus ?? "push_conflict"
2898
2979
  },
2899
2980
  blockReason,
2900
2981
  {
2901
- ...flags,
2982
+ pullPerformed: flags.pullPerformed,
2983
+ pushPerformed: flags.pushPerformed,
2984
+ commitCreated: flags.commitCreated,
2902
2985
  pushDetails: details
2903
2986
  }
2904
2987
  );
@@ -2914,21 +2997,48 @@ function withActionFlags(status, flags) {
2914
2997
  };
2915
2998
  }
2916
2999
  function isSafeResultsRepoPath(p) {
2917
- return RESULTS_REPO_TRACKED_DIRS.some((dir) => p === dir || p.startsWith(`${dir}/`));
3000
+ const normalized = p.split(path4.sep).join("/");
3001
+ const segments = normalized.split("/").filter(Boolean);
3002
+ const isRunBundlePath = segments.length >= 2 && !segments[0]?.startsWith(".") && segments[0] !== RESULTS_REPO_GITATTRIBUTES_FILE;
3003
+ return normalized === RESULTS_REPO_GITATTRIBUTES_FILE || RESULTS_REPO_TRACKED_DOT_DIRS.some(
3004
+ (dir) => normalized === dir || normalized.startsWith(`${dir}/`)
3005
+ ) || isRunBundlePath;
2918
3006
  }
2919
- async function existingTrackedResultsDirs(repoDir) {
2920
- const targets = [];
2921
- for (const dir of RESULTS_REPO_TRACKED_DIRS) {
3007
+ async function existingTrackedResultsPathspecs(repoDir) {
3008
+ const targets = /* @__PURE__ */ new Set();
3009
+ if (existsSync(path4.join(repoDir, RESULTS_REPO_GITATTRIBUTES_FILE))) {
3010
+ targets.add(RESULTS_REPO_GITATTRIBUTES_FILE);
3011
+ }
3012
+ for (const dir of RESULTS_REPO_TRACKED_DOT_DIRS) {
2922
3013
  if (existsSync(path4.join(repoDir, dir))) {
2923
- targets.push(dir);
3014
+ targets.add(dir);
3015
+ continue;
3016
+ }
3017
+ const { stdout: stdout2 } = await runGit(["ls-files", "--", dir], { cwd: repoDir, check: false });
3018
+ if (stdout2.trim().length > 0) {
3019
+ targets.add(dir);
3020
+ }
3021
+ }
3022
+ for (const entry of await readdir(repoDir, { withFileTypes: true }).catch(() => [])) {
3023
+ if (entry.name.startsWith(".") || !entry.isDirectory()) {
3024
+ continue;
3025
+ }
3026
+ if (existsSync(path4.join(repoDir, entry.name, "summary.json"))) {
3027
+ targets.add(entry.name);
3028
+ }
3029
+ }
3030
+ const { stdout } = await runGit(["ls-files"], { cwd: repoDir, check: false });
3031
+ for (const file of stdout.split(/\r?\n/)) {
3032
+ const normalized = file.trim();
3033
+ if (!normalized) {
2924
3034
  continue;
2925
3035
  }
2926
- const { stdout } = await runGit(["ls-files", "--", dir], { cwd: repoDir, check: false });
2927
- if (stdout.trim().length > 0) {
2928
- targets.push(dir);
3036
+ const segments = normalized.split("/");
3037
+ if (segments.length >= 2 && segments[0] && !segments[0].startsWith(".")) {
3038
+ targets.add(segments[0]);
2929
3039
  }
2930
3040
  }
2931
- return targets;
3041
+ return [...targets].sort();
2932
3042
  }
2933
3043
  function areSafeResultsRepoPaths(paths) {
2934
3044
  return paths.length > 0 && paths.every(isSafeResultsRepoPath);
@@ -2947,19 +3057,8 @@ function getPushTargetBranch(upstream, baseBranch, remote = "origin") {
2947
3057
  const prefix = `${remote}/`;
2948
3058
  return upstream?.startsWith(prefix) ? upstream.slice(prefix.length) : baseBranch;
2949
3059
  }
2950
- function timestampForBackupRef(date = /* @__PURE__ */ new Date()) {
2951
- const pad = (value) => String(value).padStart(2, "0");
2952
- return `${date.getUTCFullYear()}${pad(date.getUTCMonth() + 1)}${pad(date.getUTCDate())}T${pad(
2953
- date.getUTCHours()
2954
- )}${pad(date.getUTCMinutes())}${pad(date.getUTCSeconds())}Z`;
2955
- }
2956
- function slugifyBackupTargetBranch(branch) {
2957
- return branch.trim().replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "results";
2958
- }
2959
- function buildResultsBackupRef(targetBranch, remoteCommit) {
2960
- return `agentv/backups/${timestampForBackupRef()}-${slugifyBackupTargetBranch(
2961
- targetBranch
2962
- )}-${remoteCommit.slice(0, 7)}`;
3060
+ function formatShortSha(sha) {
3061
+ return sha ? sha.slice(0, 12) : "unknown";
2963
3062
  }
2964
3063
  async function getCommitSha(repoDir, ref) {
2965
3064
  if (!ref) {
@@ -2976,91 +3075,303 @@ function isNonFastForwardPushError(error) {
2976
3075
  const text = gitErrorText(error);
2977
3076
  return text.includes("non-fast-forward") || text.includes("fetch first") || text.includes("tip is behind its remote") || text.includes("note about fast-forwards") || text.includes("stale info");
2978
3077
  }
2979
- function formatShortSha(sha) {
2980
- return sha ? sha.slice(0, 12) : "unknown";
3078
+ var RESULTS_PUSH_MERGE_MAX_ATTEMPTS = 5;
3079
+ async function isAncestorCommit(repoDir, ancestor, descendant) {
3080
+ const { exitCode } = await runGit(["merge-base", "--is-ancestor", ancestor, descendant], {
3081
+ cwd: repoDir,
3082
+ check: false
3083
+ });
3084
+ return exitCode === 0;
2981
3085
  }
2982
- function buildBlockedPushConflictReason(details) {
2983
- return `Results branch push conflict on ${details.targetBranch}: remote ${formatShortSha(
2984
- details.remoteCommit
2985
- )}, local ${formatShortSha(details.localCommit)}. Configure results.sync.push_conflict_policy: backup_and_force_push to back up the remote ref before replacing it.`;
3086
+ async function getMergeBaseCommit(repoDir, a, b) {
3087
+ const { stdout, exitCode } = await runGit(["merge-base", a, b], { cwd: repoDir, check: false });
3088
+ const sha = stdout.trim();
3089
+ return exitCode === 0 && sha.length > 0 ? sha : void 0;
2986
3090
  }
2987
- async function resolveResultBranchPushConflict(params) {
2988
- await fetchResultsRepo(params.repoDir, params.normalized.remote, params.targetBranch);
2989
- const remoteRef = remoteBranchRef(params.targetBranch, params.normalized.remote);
2990
- const remoteCommit = await getCommitSha(params.repoDir, remoteRef);
2991
- const localCommit = await getCommitSha(params.repoDir, params.sourceRef);
2992
- const baseDetails = {
2993
- pushConflictPolicy: params.normalized.push_conflict_policy,
2994
- targetBranch: params.targetBranch,
2995
- ...remoteCommit !== void 0 && {
2996
- remoteCommit,
2997
- previousRemoteCommit: remoteCommit,
2998
- leaseCommit: remoteCommit
2999
- },
3000
- ...localCommit !== void 0 && { localCommit }
3001
- };
3002
- if (!remoteCommit) {
3003
- return {
3004
- blocked: true,
3005
- blockReason: `Results branch push conflict on ${params.targetBranch}: remote commit could not be resolved after fetch`,
3006
- details: baseDetails
3007
- };
3091
+ function buildNeedsHumanMergeReason(targetBranch, remoteCommit, localCommit) {
3092
+ return `Results branch ${targetBranch} diverged from the remote and could not be auto-merged: a genuine results content conflict remains (remote ${formatShortSha(
3093
+ remoteCommit
3094
+ )}, local ${formatShortSha(
3095
+ localCommit
3096
+ )}). The remote branch is unchanged and no history was rewritten; resolve it with a GitHub pull request.`;
3097
+ }
3098
+ function timestampForResultsRef(date = /* @__PURE__ */ new Date()) {
3099
+ const pad = (value) => String(value).padStart(2, "0");
3100
+ return `${date.getUTCFullYear()}${pad(date.getUTCMonth() + 1)}${pad(date.getUTCDate())}T${pad(
3101
+ date.getUTCHours()
3102
+ )}${pad(date.getUTCMinutes())}${pad(date.getUTCSeconds())}Z`;
3103
+ }
3104
+ function slugifyResultsTargetBranch(branch) {
3105
+ return branch.trim().replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "results";
3106
+ }
3107
+ function randomResultsRefToken() {
3108
+ return randomBytes(4).toString("hex").slice(0, 6);
3109
+ }
3110
+ function buildResultsSyncBranchName(targetBranch, token = randomResultsRefToken()) {
3111
+ return `agentv/results-sync/${timestampForResultsRef()}-${slugifyResultsTargetBranch(
3112
+ targetBranch
3113
+ )}-${token}`;
3114
+ }
3115
+ function buildResultsCompareUrl(remoteUrl, targetBranch, syncBranch) {
3116
+ if (!remoteUrl) {
3117
+ return void 0;
3008
3118
  }
3009
- if (params.normalized.push_conflict_policy === "block") {
3010
- return {
3011
- blocked: true,
3012
- blockReason: buildBlockedPushConflictReason(baseDetails),
3013
- details: baseDetails
3014
- };
3119
+ const match = remoteUrl.trim().match(/github\.com[:/]([^/\s]+)\/([^/\s]+?)(?:\.git)?\/?$/i);
3120
+ if (!match) {
3121
+ return void 0;
3015
3122
  }
3016
- const backupRef = buildResultsBackupRef(params.targetBranch, remoteCommit);
3017
- const backupDetails = {
3018
- ...baseDetails,
3019
- backupRef,
3020
- backupCommit: remoteCommit
3021
- };
3022
- try {
3023
- await assertValidResultsBranchName(params.repoDir, backupRef);
3024
- await runGit(
3025
- ["push", "--porcelain", params.normalized.remote, `${remoteCommit}:refs/heads/${backupRef}`],
3026
- { cwd: params.repoDir }
3027
- );
3028
- } catch (error) {
3029
- return {
3030
- blocked: true,
3031
- blockReason: `Results branch backup creation failed for ${params.targetBranch} at ${formatShortSha(
3032
- remoteCommit
3033
- )}: ${getStatusMessage(error)}`,
3034
- details: backupDetails
3035
- };
3123
+ const [, owner, repo] = match;
3124
+ return `https://github.com/${owner}/${repo}/compare/${encodeURIComponent(
3125
+ targetBranch
3126
+ )}...${encodeURIComponent(syncBranch)}?expand=1`;
3127
+ }
3128
+ async function resolveRemotePushUrl(repoDir, remote) {
3129
+ for (const args of [
3130
+ ["remote", "get-url", "--push", remote],
3131
+ ["remote", "get-url", remote]
3132
+ ]) {
3133
+ const { stdout, exitCode } = await runGit(args, { cwd: repoDir, check: false });
3134
+ const url = stdout.trim();
3135
+ if (exitCode === 0 && url.length > 0) {
3136
+ return url;
3137
+ }
3138
+ }
3139
+ return void 0;
3140
+ }
3141
+ async function countContributedRunDirs(repoDir, remoteTargetCommit, sourceCommit) {
3142
+ if (!remoteTargetCommit) {
3143
+ return void 0;
3144
+ }
3145
+ const { stdout, exitCode } = await runGit(
3146
+ ["diff", "--name-only", `${remoteTargetCommit}...${sourceCommit}`, "--", RESULTS_REPO_RUNS_DIR],
3147
+ { cwd: repoDir, check: false }
3148
+ );
3149
+ if (exitCode !== 0) {
3150
+ return void 0;
3151
+ }
3152
+ const runDirs = /* @__PURE__ */ new Set();
3153
+ for (const line of stdout.split(/\r?\n/)) {
3154
+ const file = line.trim();
3155
+ if (!file) {
3156
+ continue;
3157
+ }
3158
+ const segments = file.split("/");
3159
+ if (segments.length >= 2 && segments[0] && !segments[0].startsWith(".")) {
3160
+ runDirs.add(segments[0]);
3161
+ }
3036
3162
  }
3163
+ return runDirs.size;
3164
+ }
3165
+ async function pushResultsSyncBranch(params) {
3166
+ const { repoDir, remote, sourceCommit, targetBranch } = params;
3167
+ const remoteTargetCommit = await getCommitSha(repoDir, remoteBranchRef(targetBranch, remote));
3168
+ let syncBranch = buildResultsSyncBranchName(targetBranch);
3169
+ await assertValidResultsBranchName(repoDir, syncBranch);
3037
3170
  try {
3038
- await runGit(
3039
- [
3040
- "push",
3041
- "--porcelain",
3042
- `--force-with-lease=refs/heads/${params.targetBranch}:${remoteCommit}`,
3043
- params.normalized.remote,
3044
- `${params.sourceRef}:refs/heads/${params.targetBranch}`
3045
- ],
3046
- { cwd: params.repoDir }
3047
- );
3171
+ await runGit(["push", "--porcelain", remote, `${sourceCommit}:refs/heads/${syncBranch}`], {
3172
+ cwd: repoDir
3173
+ });
3048
3174
  } catch (error) {
3049
- return {
3050
- blocked: true,
3051
- blockReason: `Results branch force push lease failed for ${params.targetBranch}; remote changed after backup ${backupRef} was created with lease ${formatShortSha(
3052
- remoteCommit
3053
- )}: ${getStatusMessage(error)}`,
3054
- details: backupDetails
3055
- };
3175
+ if (!isRefAlreadyExistsPushError(error)) {
3176
+ throw error;
3177
+ }
3178
+ syncBranch = buildResultsSyncBranchName(targetBranch);
3179
+ await assertValidResultsBranchName(repoDir, syncBranch);
3180
+ await runGit(["push", "--porcelain", remote, `${sourceCommit}:refs/heads/${syncBranch}`], {
3181
+ cwd: repoDir
3182
+ });
3056
3183
  }
3184
+ const compareUrl = buildResultsCompareUrl(
3185
+ await resolveRemotePushUrl(repoDir, remote),
3186
+ targetBranch,
3187
+ syncBranch
3188
+ );
3189
+ const contributedRunCount = await countContributedRunDirs(
3190
+ repoDir,
3191
+ remoteTargetCommit,
3192
+ sourceCommit
3193
+ ).catch(() => void 0);
3057
3194
  return {
3058
- blocked: false,
3059
- details: {
3060
- ...backupDetails,
3061
- ...localCommit !== void 0 && { forcePushedCommit: localCommit }
3195
+ tempBranch: syncBranch,
3196
+ targetBranch,
3197
+ ...compareUrl !== void 0 && { compareUrl },
3198
+ ...contributedRunCount !== void 0 && { contributedRunCount },
3199
+ createdAt: (/* @__PURE__ */ new Date()).toISOString()
3200
+ };
3201
+ }
3202
+ function isRefAlreadyExistsPushError(error) {
3203
+ const text = gitErrorText(error);
3204
+ return text.includes("already exists") || text.includes("cannot lock ref") || text.includes("reference already exists");
3205
+ }
3206
+ async function mergeRemoteIntoCheckedOutBranch(repoDir, remoteCommit) {
3207
+ const args = ["merge", "--no-edit", "-m", "chore(results): merge remote results", remoteCommit];
3208
+ let result = await runGit(args, {
3209
+ cwd: repoDir,
3210
+ check: false,
3211
+ env: configuredGitCommitIdentityEnv()
3212
+ });
3213
+ if (result.exitCode !== 0 && isMissingGitIdentityError({ stderr: result.stderr })) {
3214
+ await runGit(["merge", "--abort"], { cwd: repoDir, check: false });
3215
+ result = await runGit(args, {
3216
+ cwd: repoDir,
3217
+ check: false,
3218
+ env: { ...configuredGitCommitIdentityEnv(), ...fallbackResultsRepoCommitEnv() }
3219
+ });
3220
+ }
3221
+ return result.exitCode;
3222
+ }
3223
+ async function resolveResultBranchPushConflict(params) {
3224
+ await ensureResultsMergeConfig(params.repoDir);
3225
+ const { repoDir, targetBranch } = params;
3226
+ const remote = params.normalized.remote;
3227
+ const localRef = params.sourceRef;
3228
+ const remoteRef = remoteBranchRef(targetBranch, remote);
3229
+ const branchCheckedOut = params.sourceRef === "HEAD";
3230
+ let lastDetails = {
3231
+ pushConflictPolicy: params.normalized.push_conflict_policy,
3232
+ targetBranch
3233
+ };
3234
+ const buildHumanMergeOutcome = async (reason, details) => {
3235
+ if (!details.localCommit) {
3236
+ return { blocked: true, blockReason: reason, details, syncStatus: "needs_human_merge" };
3237
+ }
3238
+ try {
3239
+ const pendingMerge = await pushResultsSyncBranch({
3240
+ repoDir,
3241
+ remote,
3242
+ sourceCommit: details.localCommit,
3243
+ targetBranch: details.targetBranch
3244
+ });
3245
+ return {
3246
+ blocked: true,
3247
+ blockReason: reason,
3248
+ details: { ...details, pendingMerge },
3249
+ syncStatus: "needs_human_merge"
3250
+ };
3251
+ } catch (error) {
3252
+ return {
3253
+ blocked: true,
3254
+ blockReason: `${reason} (could not create the temp branch for the GitHub merge: ${getStatusMessage(
3255
+ error
3256
+ )})`,
3257
+ details,
3258
+ syncStatus: "needs_human_merge"
3259
+ };
3062
3260
  }
3063
3261
  };
3262
+ for (let attempt = 1; attempt <= RESULTS_PUSH_MERGE_MAX_ATTEMPTS; attempt += 1) {
3263
+ await fetchResultsRepo(repoDir, remote, targetBranch);
3264
+ const remoteCommit = await getCommitSha(repoDir, remoteRef);
3265
+ const localCommit = await getCommitSha(repoDir, localRef);
3266
+ const details = {
3267
+ pushConflictPolicy: params.normalized.push_conflict_policy,
3268
+ targetBranch,
3269
+ ...remoteCommit !== void 0 && { remoteCommit, previousRemoteCommit: remoteCommit },
3270
+ ...localCommit !== void 0 && { localCommit }
3271
+ };
3272
+ lastDetails = details;
3273
+ if (!remoteCommit) {
3274
+ return {
3275
+ blocked: true,
3276
+ blockReason: `Results branch push conflict on ${targetBranch}: remote commit could not be resolved after fetch`,
3277
+ details,
3278
+ syncStatus: "push_conflict"
3279
+ };
3280
+ }
3281
+ if (!localCommit) {
3282
+ return {
3283
+ blocked: true,
3284
+ blockReason: `Results branch push conflict on ${targetBranch}: local commit could not be resolved`,
3285
+ details,
3286
+ syncStatus: "push_conflict"
3287
+ };
3288
+ }
3289
+ if (await isAncestorCommit(repoDir, localCommit, remoteCommit)) {
3290
+ if (branchCheckedOut) {
3291
+ await runGit(["merge", "--ff-only", remoteCommit], { cwd: repoDir, check: false });
3292
+ } else {
3293
+ await runGit(["update-ref", localRef, remoteCommit, localCommit], {
3294
+ cwd: repoDir,
3295
+ check: false
3296
+ });
3297
+ }
3298
+ return { blocked: false, details };
3299
+ }
3300
+ let pushSpec = branchCheckedOut ? "HEAD" : localCommit;
3301
+ let autoMergedRemote = false;
3302
+ if (!await isAncestorCommit(repoDir, remoteCommit, localCommit)) {
3303
+ if (branchCheckedOut) {
3304
+ const exitCode = await mergeRemoteIntoCheckedOutBranch(repoDir, remoteCommit);
3305
+ if (exitCode !== 0) {
3306
+ await runGit(["merge", "--abort"], { cwd: repoDir, check: false });
3307
+ return await buildHumanMergeOutcome(
3308
+ buildNeedsHumanMergeReason(targetBranch, remoteCommit, localCommit),
3309
+ details
3310
+ );
3311
+ }
3312
+ pushSpec = "HEAD";
3313
+ autoMergedRemote = true;
3314
+ } else {
3315
+ const base = await getMergeBaseCommit(repoDir, localCommit, remoteCommit);
3316
+ if (!base) {
3317
+ return await buildHumanMergeOutcome(
3318
+ buildNeedsHumanMergeReason(targetBranch, remoteCommit, localCommit),
3319
+ details
3320
+ );
3321
+ }
3322
+ const merge = await runGit(
3323
+ ["merge-tree", "--write-tree", `--merge-base=${base}`, localCommit, remoteCommit],
3324
+ { cwd: repoDir, check: false }
3325
+ );
3326
+ const mergedTree = merge.stdout.trim().split(/\r?\n/)[0]?.trim();
3327
+ if (merge.exitCode !== 0 || !mergedTree) {
3328
+ return await buildHumanMergeOutcome(
3329
+ buildNeedsHumanMergeReason(targetBranch, remoteCommit, localCommit),
3330
+ details
3331
+ );
3332
+ }
3333
+ const { stdout: mergeCommitOut } = await runGitWithFallbackCommitIdentity(
3334
+ [
3335
+ "commit-tree",
3336
+ mergedTree,
3337
+ "-p",
3338
+ localCommit,
3339
+ "-p",
3340
+ remoteCommit,
3341
+ "-m",
3342
+ "chore(results): merge remote results"
3343
+ ],
3344
+ { cwd: repoDir }
3345
+ );
3346
+ pushSpec = mergeCommitOut.trim();
3347
+ await runGit(["update-ref", localRef, pushSpec, localCommit], { cwd: repoDir });
3348
+ autoMergedRemote = true;
3349
+ }
3350
+ }
3351
+ try {
3352
+ await runGit(["push", "--porcelain", remote, `${pushSpec}:refs/heads/${targetBranch}`], {
3353
+ cwd: repoDir
3354
+ });
3355
+ } catch (error) {
3356
+ if (isNonFastForwardPushError(error)) {
3357
+ continue;
3358
+ }
3359
+ throw error;
3360
+ }
3361
+ const pushedCommit = await getCommitSha(repoDir, pushSpec) ?? localCommit;
3362
+ return {
3363
+ blocked: false,
3364
+ details: {
3365
+ ...details,
3366
+ localCommit: pushedCommit,
3367
+ ...autoMergedRemote && { autoMergedRemote }
3368
+ }
3369
+ };
3370
+ }
3371
+ return await buildHumanMergeOutcome(
3372
+ `Results branch ${targetBranch} could not be reconciled after ${RESULTS_PUSH_MERGE_MAX_ATTEMPTS} attempts because the remote kept advancing; retry sync. The remote branch is unchanged.`,
3373
+ lastDetails
3374
+ );
3064
3375
  }
3065
3376
  async function statusFromInspection(normalized, repoDir) {
3066
3377
  return withGitInspection(
@@ -3087,11 +3398,7 @@ async function getResultsRepoSyncStatus(config) {
3087
3398
  };
3088
3399
  }
3089
3400
  try {
3090
- await ensureResultsRepoRemote(normalized.path, normalized);
3091
3401
  if (usesStorageBranchWorktree(normalized)) {
3092
- await fetchResultsRepo(normalized.path, normalized.remote, normalized.branch).catch(
3093
- () => void 0
3094
- );
3095
3402
  return withGitInspection(
3096
3403
  baseStatus,
3097
3404
  await inspectResultsStorageBranchGit(normalized.path, normalized)
@@ -3252,7 +3559,8 @@ async function syncResultsRepoForProject(config) {
3252
3559
  return withPushConflictStatus(status2, outcome.blockReason, outcome.details, {
3253
3560
  pullPerformed,
3254
3561
  pushPerformed,
3255
- commitCreated
3562
+ commitCreated,
3563
+ syncStatus: outcome.syncStatus
3256
3564
  });
3257
3565
  }
3258
3566
  pushPerformed = true;
@@ -3369,7 +3677,8 @@ async function syncResultsRepoForProject(config) {
3369
3677
  return withPushConflictStatus(status2, outcome.blockReason, outcome.details, {
3370
3678
  pullPerformed,
3371
3679
  pushPerformed,
3372
- commitCreated
3680
+ commitCreated,
3681
+ syncStatus: outcome.syncStatus
3373
3682
  });
3374
3683
  }
3375
3684
  } else {
@@ -3464,10 +3773,16 @@ async function syncResultsRepoForProject(config) {
3464
3773
  }
3465
3774
  }
3466
3775
  if (inspection.syncStatus === "dirty") {
3467
- const trackedDirs = await existingTrackedResultsDirs(repoDir);
3468
- await runGit(["add", "--all", "--", ...trackedDirs], { cwd: repoDir });
3776
+ const trackedPathspecs = await existingTrackedResultsPathspecs(repoDir);
3777
+ await runGit(["add", "--all", "--", ...trackedPathspecs], { cwd: repoDir });
3469
3778
  await runGitWithFallbackCommitIdentity(
3470
- ["commit", "-m", "chore(results): sync local result metadata", "--", ...trackedDirs],
3779
+ [
3780
+ "commit",
3781
+ "-m",
3782
+ "chore(results): sync local results branch changes",
3783
+ "--",
3784
+ ...trackedPathspecs
3785
+ ],
3471
3786
  {
3472
3787
  cwd: repoDir
3473
3788
  }
@@ -3502,7 +3817,8 @@ async function syncResultsRepoForProject(config) {
3502
3817
  return withPushConflictStatus(status, outcome.blockReason, outcome.details, {
3503
3818
  pullPerformed,
3504
3819
  pushPerformed,
3505
- commitCreated
3820
+ commitCreated,
3821
+ syncStatus: outcome.syncStatus
3506
3822
  });
3507
3823
  }
3508
3824
  pushPerformed = true;
@@ -3587,7 +3903,8 @@ async function syncResultsRepoForProject(config) {
3587
3903
  return withPushConflictStatus(status, outcome.blockReason, outcome.details, {
3588
3904
  pullPerformed,
3589
3905
  pushPerformed,
3590
- commitCreated
3906
+ commitCreated,
3907
+ syncStatus: outcome.syncStatus
3591
3908
  });
3592
3909
  }
3593
3910
  } else {
@@ -3625,6 +3942,44 @@ async function syncResultsRepoForProject(config) {
3625
3942
  activeResultsRepoSyncs.delete(syncKey);
3626
3943
  }
3627
3944
  }
3945
+ async function fastForwardResultsTowardTarget(repoDir, normalized, targetBranch) {
3946
+ const upstreamRef = remoteBranchRef(targetBranch, normalized.remote);
3947
+ if (!await gitRefExists(repoDir, upstreamRef)) {
3948
+ return;
3949
+ }
3950
+ const upstreamCommit = await getCommitSha(repoDir, upstreamRef);
3951
+ if (!upstreamCommit) {
3952
+ return;
3953
+ }
3954
+ if (usesStorageBranchWorktree(normalized) && normalized.branch) {
3955
+ const localRef = `refs/heads/${normalized.branch}`;
3956
+ if (!await gitRefExists(repoDir, localRef)) {
3957
+ await runGit(["update-ref", localRef, upstreamCommit], { cwd: repoDir, check: false });
3958
+ await runGit(["branch", "--set-upstream-to", upstreamRef, normalized.branch], {
3959
+ cwd: repoDir,
3960
+ check: false
3961
+ });
3962
+ return;
3963
+ }
3964
+ const localCommit = await getCommitSha(repoDir, localRef);
3965
+ if (localCommit && await isAncestorCommit(repoDir, localCommit, upstreamCommit)) {
3966
+ await runGit(["update-ref", localRef, upstreamCommit, localCommit], {
3967
+ cwd: repoDir,
3968
+ check: false
3969
+ });
3970
+ }
3971
+ return;
3972
+ }
3973
+ await runGit(["merge", "--ff-only", upstreamRef], { cwd: repoDir, check: false });
3974
+ }
3975
+ async function confirmResultsMergeAndPull(config) {
3976
+ const normalized = normalizeResultsConfig(config);
3977
+ const repoDir = await ensureResultsRepoClone(normalized);
3978
+ const targetBranch = normalized.branch ?? await resolveDefaultBranch(repoDir);
3979
+ await fetchResultsRepo(repoDir, normalized.remote, normalized.branch).catch(() => void 0);
3980
+ await fastForwardResultsTowardTarget(repoDir, normalized, targetBranch);
3981
+ return syncResultsRepoForProject(config);
3982
+ }
3628
3983
  async function checkoutResultsRepoBranch(config, branchName) {
3629
3984
  const normalized = normalizeResultsConfig(config);
3630
3985
  const repoDir = await ensureResultsRepoClone(normalized);
@@ -3783,12 +4138,10 @@ function artifactSidecarPointers(record) {
3783
4138
  return [];
3784
4139
  }
3785
4140
  const pointers = [];
3786
- for (const pointer of Object.values(record.artifact_pointers)) {
3787
- if (!isRecord2(pointer)) {
3788
- continue;
3789
- }
4141
+ const pointer = record.artifact_pointers.transcript;
4142
+ if (isRecord2(pointer)) {
3790
4143
  if (pointer.ref !== AGENTV_RESULTS_ARTIFACTS_REF || typeof pointer.path !== "string") {
3791
- continue;
4144
+ return pointers;
3792
4145
  }
3793
4146
  pointers.push({
3794
4147
  path: pointer.path,
@@ -3805,17 +4158,58 @@ function artifactSidecarKey(destinationPath, pointerPath) {
3805
4158
  normalizeDestinationPath(pointerPath)
3806
4159
  );
3807
4160
  }
3808
- function collectArtifactSidecarPointers(sourceDir) {
3809
- const indexPath = path4.join(sourceDir, RESULT_INDEX_FILENAME2);
3810
- if (!existsSync(indexPath)) {
3811
- return [];
4161
+ function isDeprecatedTraceArtifactPath(relativePath) {
4162
+ return relativePath === "trace.json" || relativePath.endsWith("/trace.json");
4163
+ }
4164
+ function isResultManifestFilename(filename) {
4165
+ return filename === RESULT_INDEX_FILENAME2;
4166
+ }
4167
+ function safeLocalSummaryManifestPath(sourceDir, manifestPath) {
4168
+ if (typeof manifestPath !== "string" || manifestPath.trim().length === 0) {
4169
+ return void 0;
3812
4170
  }
3813
- const pointers = [];
3814
- for (const line of readFileSync2(indexPath, "utf8").split(/\r?\n/)) {
3815
- const trimmed = line.trim();
3816
- if (!trimmed) {
3817
- continue;
3818
- }
4171
+ if (path4.isAbsolute(manifestPath)) {
4172
+ return void 0;
4173
+ }
4174
+ const normalized = path4.normalize(manifestPath);
4175
+ if (normalized.startsWith("..") || path4.isAbsolute(normalized)) {
4176
+ return void 0;
4177
+ }
4178
+ return path4.join(sourceDir, normalized);
4179
+ }
4180
+ function resolveLocalResultManifestPath(sourceDir) {
4181
+ try {
4182
+ const summary = JSON.parse(readFileSync2(path4.join(sourceDir, "summary.json"), "utf8"));
4183
+ const manifestPath2 = safeLocalSummaryManifestPath(
4184
+ sourceDir,
4185
+ summary.index_path ?? summary.manifest_path
4186
+ );
4187
+ if (manifestPath2 && existsSync(manifestPath2)) {
4188
+ return manifestPath2;
4189
+ }
4190
+ } catch {
4191
+ }
4192
+ const manifestPath = path4.join(sourceDir, RUN_INTERNAL_DIRNAME, RESULT_INDEX_FILENAME2);
4193
+ if (existsSync(manifestPath)) {
4194
+ return manifestPath;
4195
+ }
4196
+ const legacyManifestPath = path4.join(sourceDir, RESULT_INDEX_FILENAME2);
4197
+ if (existsSync(legacyManifestPath)) {
4198
+ return legacyManifestPath;
4199
+ }
4200
+ return void 0;
4201
+ }
4202
+ function collectArtifactSidecarPointers(sourceDir) {
4203
+ const indexPath = resolveLocalResultManifestPath(sourceDir);
4204
+ if (!indexPath) {
4205
+ return [];
4206
+ }
4207
+ const pointers = [];
4208
+ for (const line of readFileSync2(indexPath, "utf8").split(/\r?\n/)) {
4209
+ const trimmed = line.trim();
4210
+ if (!trimmed) {
4211
+ continue;
4212
+ }
3819
4213
  try {
3820
4214
  pointers.push(...artifactSidecarPointers(JSON.parse(trimmed)));
3821
4215
  } catch {
@@ -3894,18 +4288,20 @@ function rewritePublishedIndexLine(line, destinationPath) {
3894
4288
  return line;
3895
4289
  }
3896
4290
  let changed = false;
3897
- for (const pointer of Object.values(record.artifact_pointers)) {
3898
- if (!isRecord2(pointer)) {
3899
- continue;
3900
- }
3901
- if (pointer.ref !== AGENTV_RESULTS_ARTIFACTS_REF || typeof pointer.path !== "string") {
3902
- continue;
3903
- }
4291
+ const pointer = record.artifact_pointers.transcript;
4292
+ if (isRecord2(pointer) && pointer.ref === AGENTV_RESULTS_ARTIFACTS_REF && typeof pointer.path === "string") {
3904
4293
  const key = artifactSidecarKey(destinationPath, pointer.path);
3905
4294
  if (pointer.key !== key) {
3906
4295
  pointer.key = key;
3907
4296
  changed = true;
3908
4297
  }
4298
+ if (Object.keys(record.artifact_pointers).length !== 1 || record.artifact_pointers.transcript !== pointer) {
4299
+ changed = true;
4300
+ }
4301
+ record.artifact_pointers = { transcript: pointer };
4302
+ } else {
4303
+ record.artifact_pointers = void 0;
4304
+ changed = true;
3909
4305
  }
3910
4306
  return changed ? JSON.stringify(record) : line;
3911
4307
  }
@@ -3922,13 +4318,16 @@ async function preparePublishedResultsSource(params) {
3922
4318
  for (const sourceFile of sourceFiles) {
3923
4319
  const relativeFile = path4.relative(params.sourceDir, sourceFile).split(path4.sep).join("/");
3924
4320
  const destinationFile = path4.join(publishedRoot, ...relativeFile.split("/"));
3925
- if (relativeFile === RESULT_INDEX_FILENAME2) {
4321
+ if (isResultManifestFilename(path4.posix.basename(relativeFile))) {
3926
4322
  const original = readFileSync2(sourceFile, "utf8");
3927
4323
  const rewritten = original.split(/\r?\n/).map((line) => rewritePublishedIndexLine(line, params.destinationPath)).join("\n");
3928
4324
  mkdirSync(path4.dirname(destinationFile), { recursive: true });
3929
4325
  writeFileSync(destinationFile, rewritten);
3930
4326
  continue;
3931
4327
  }
4328
+ if (isDeprecatedTraceArtifactPath(relativeFile)) {
4329
+ continue;
4330
+ }
3932
4331
  if (omittedPaths.has(relativeFile)) {
3933
4332
  continue;
3934
4333
  }
@@ -4008,6 +4407,7 @@ async function commitStorageBranchWorktreePaths(params) {
4008
4407
  } else {
4009
4408
  await runGit(["read-tree", "--empty"], { cwd: params.repoDir, env: indexEnv });
4010
4409
  }
4410
+ await addResultsGitattributesToIndex(params.repoDir, indexEnv);
4011
4411
  for (const gitPath of paths) {
4012
4412
  const sourcePath = path4.join(params.repoDir, ...gitPath.split("/"));
4013
4413
  if (!existsSync(sourcePath)) {
@@ -4098,6 +4498,7 @@ async function commitResultsRunWithTemporaryIndex(params) {
4098
4498
  } else {
4099
4499
  await runGit(["read-tree", "--empty"], { cwd: params.repoDir, env: indexEnv });
4100
4500
  }
4501
+ await addResultsGitattributesToIndex(params.repoDir, indexEnv);
4101
4502
  const existingPaths = await getExistingRunTreePaths(
4102
4503
  params.repoDir,
4103
4504
  base.baseRef,
@@ -4189,7 +4590,7 @@ function buildDirectPushResult(normalized, changed, outcome) {
4189
4590
  ...outcome?.blocked === true && {
4190
4591
  blocked: true,
4191
4592
  block_reason: outcome.blockReason,
4192
- sync_status: "push_conflict"
4593
+ sync_status: outcome.syncStatus ?? "push_conflict"
4193
4594
  },
4194
4595
  ...pushDetailsToWire(details)
4195
4596
  };
@@ -4199,10 +4600,11 @@ function mergeDirectPushResults(normalized, results) {
4199
4600
  if (blocked) {
4200
4601
  return blocked;
4201
4602
  }
4202
- const detailed = [...results].reverse().find((result) => result.backup_ref !== void 0);
4603
+ const detailed = [...results].reverse().find((result) => result.backup_ref !== void 0 || result.auto_merged_remote === true);
4203
4604
  return {
4204
4605
  changed: results.some((result) => result.changed),
4205
4606
  push_conflict_policy: normalized.push_conflict_policy,
4607
+ ...detailed?.auto_merged_remote === true && { auto_merged_remote: true },
4206
4608
  ...detailed?.backup_ref !== void 0 && {
4207
4609
  backup_ref: detailed.backup_ref
4208
4610
  },
@@ -4385,21 +4787,74 @@ async function directPushResultsWithDetails(params) {
4385
4787
  await sidecar?.cleanup().catch(() => void 0);
4386
4788
  }
4387
4789
  }
4790
+ function safeGitSummaryManifestPath(runDir, manifestPath) {
4791
+ if (typeof manifestPath !== "string" || manifestPath.trim().length === 0) {
4792
+ return void 0;
4793
+ }
4794
+ if (manifestPath.startsWith("/")) {
4795
+ return void 0;
4796
+ }
4797
+ const normalized = path4.posix.normalize(manifestPath);
4798
+ if (normalized === ".." || normalized.startsWith("../")) {
4799
+ return void 0;
4800
+ }
4801
+ return path4.posix.join(runDir, normalized);
4802
+ }
4803
+ function buildGitManifestPaths(treePaths, summaryByPath) {
4804
+ const treePathSet = new Set(treePaths);
4805
+ const manifestByRunDir = /* @__PURE__ */ new Map();
4806
+ for (const [summaryPath, summary] of summaryByPath) {
4807
+ const runDir = path4.posix.dirname(summaryPath);
4808
+ const relativeRunPath = path4.posix.relative(RESULTS_REPO_RUNS_DIR, runDir);
4809
+ if (!isV2ResultsRepoRunPath(relativeRunPath)) {
4810
+ continue;
4811
+ }
4812
+ const manifestPath = safeGitSummaryManifestPath(runDir, summary.index_path ?? summary.manifest_path) ?? fallbackGitManifestPath(runDir, treePathSet);
4813
+ if (manifestPath && treePathSet.has(manifestPath)) {
4814
+ manifestByRunDir.set(runDir, manifestPath);
4815
+ }
4816
+ }
4817
+ for (const treePath of treePaths) {
4818
+ if (!treePath.endsWith(`/${RESULT_INDEX_FILENAME2}`)) {
4819
+ continue;
4820
+ }
4821
+ const runDir = gitRunDirForManifestPath(treePath);
4822
+ const relativeRunPath = path4.posix.relative(RESULTS_REPO_RUNS_DIR, runDir);
4823
+ if (!isV2ResultsRepoRunPath(relativeRunPath)) {
4824
+ continue;
4825
+ }
4826
+ if (!manifestByRunDir.has(runDir)) {
4827
+ manifestByRunDir.set(runDir, treePath);
4828
+ }
4829
+ }
4830
+ return [...manifestByRunDir.values()].sort();
4831
+ }
4832
+ function fallbackGitManifestPath(runDir, treePathSet) {
4833
+ for (const candidate of [
4834
+ path4.posix.join(runDir, ".internal", RESULT_INDEX_FILENAME2),
4835
+ path4.posix.join(runDir, RESULT_INDEX_FILENAME2)
4836
+ ]) {
4837
+ if (treePathSet.has(candidate)) {
4838
+ return candidate;
4839
+ }
4840
+ }
4841
+ return void 0;
4842
+ }
4843
+ function gitRunDirForManifestPath(manifestPath) {
4844
+ const manifestDir = path4.posix.dirname(manifestPath);
4845
+ return path4.posix.basename(manifestDir) === ".internal" ? path4.posix.dirname(manifestDir) : manifestDir;
4846
+ }
4847
+ function isV2ResultsRepoRunPath(relativeRunPath) {
4848
+ const segments = relativeRunPath.split("/").filter(Boolean);
4849
+ return segments.length === 1 && !segments[0].startsWith(".");
4850
+ }
4388
4851
  function buildGitRunId(relativeRunPath) {
4389
4852
  const normalized = relativeRunPath.split(path4.sep).join("/");
4390
4853
  const segments = normalized.split("/").filter(Boolean);
4391
- if (segments.length >= 2) {
4392
- const experiment = segments.slice(0, -1).join("/");
4393
- const timestamp = segments.at(-1);
4394
- if (experiment === "default") {
4395
- return timestamp ?? normalized;
4396
- }
4397
- return `${experiment}::${timestamp}`;
4398
- }
4399
4854
  return segments[0] ?? relativeRunPath;
4400
4855
  }
4401
- function getRunExperiment(runId, benchmark) {
4402
- const experiment = benchmark.metadata?.experiment?.trim();
4856
+ function getRunExperiment(runId, summary) {
4857
+ const experiment = summary?.metadata?.experiment?.trim();
4403
4858
  if (experiment) {
4404
4859
  return experiment;
4405
4860
  }
@@ -4572,8 +5027,11 @@ async function pushWipCheckpoint(params) {
4572
5027
  sourceDir: params.sourceDir,
4573
5028
  destinationDir
4574
5029
  });
4575
- const trackedDirs = await existingTrackedResultsDirs(params.handle.worktreeDir);
4576
- await runGit(["add", "--all", "--", ...trackedDirs], {
5030
+ writeFileSync(
5031
+ path4.join(params.handle.worktreeDir, RESULTS_REPO_GITATTRIBUTES_FILE),
5032
+ RESULTS_REPO_GITATTRIBUTES_CONTENT
5033
+ );
5034
+ await runGit(["add", "--all", "--", RESULTS_REPO_GITATTRIBUTES_FILE, params.destinationPath], {
4577
5035
  cwd: params.handle.worktreeDir
4578
5036
  });
4579
5037
  const { stdout: status } = await runGit(["status", "--porcelain"], {
@@ -4596,7 +5054,19 @@ async function pushWipCheckpoint(params) {
4596
5054
  async function deleteWipBranch(params) {
4597
5055
  const normalized = normalizeResultsConfig(params.config);
4598
5056
  const cloneDir = await ensureResultsRepoClone(normalized);
4599
- await runGit(["push", normalized.remote, "--delete", params.wipBranch], { cwd: cloneDir });
5057
+ const result = await runGit(["push", normalized.remote, "--delete", params.wipBranch], {
5058
+ cwd: cloneDir,
5059
+ check: false
5060
+ });
5061
+ if (result.exitCode === 0 || isMissingRemoteRefDelete(result)) {
5062
+ return;
5063
+ }
5064
+ throw new Error(result.stderr.trim() || result.stdout.trim() || "Failed to delete WIP branch");
5065
+ }
5066
+ function isMissingRemoteRefDelete(result) {
5067
+ const text = `${result.stderr}
5068
+ ${result.stdout}`.toLowerCase();
5069
+ return text.includes("remote ref does not exist");
4600
5070
  }
4601
5071
  function isMissingGitRefError(error) {
4602
5072
  const parts = [];
@@ -4608,7 +5078,107 @@ function isMissingGitRefError(error) {
4608
5078
  parts.push(error);
4609
5079
  }
4610
5080
  const haystack = parts.join("\n").toLowerCase();
4611
- return haystack.includes("not a valid object name") || haystack.includes("unknown revision or path") || haystack.includes("bad revision") || haystack.includes("does not exist");
5081
+ return haystack.includes("not a valid object name") || haystack.includes("unknown revision or path") || haystack.includes("bad revision") || haystack.includes("does not exist") || haystack.includes("needed a single revision");
5082
+ }
5083
+ function gitResultsIndexHash(value) {
5084
+ return createHash("sha256").update(value).digest("hex");
5085
+ }
5086
+ function resolveGitResultsIndexRepoDir(repoDir) {
5087
+ const resolved = path4.resolve(repoDir);
5088
+ try {
5089
+ return realpathSync(resolved);
5090
+ } catch {
5091
+ return resolved;
5092
+ }
5093
+ }
5094
+ function gitResultsIndexRepoKey(repoDir) {
5095
+ return `sha256-${gitResultsIndexHash(resolveGitResultsIndexRepoDir(repoDir)).slice(0, 32)}`;
5096
+ }
5097
+ function gitResultsIndexRefKey(ref) {
5098
+ const slug = sanitizeRepoSlug(ref).slice(0, 80) || "default";
5099
+ return `${slug}-${gitResultsIndexHash(ref).slice(0, 16)}`;
5100
+ }
5101
+ function resolveGitResultsIndexCacheFile(params) {
5102
+ const ref = params.ref ?? "origin/main";
5103
+ return path4.join(
5104
+ getAgentvDataDir(),
5105
+ "cache",
5106
+ "results-index",
5107
+ gitResultsIndexRepoKey(params.repoDir),
5108
+ gitResultsIndexRefKey(ref),
5109
+ `${params.commitSha}.json`
5110
+ );
5111
+ }
5112
+ function isGitListedRun(value) {
5113
+ if (!value || typeof value !== "object") return false;
5114
+ const record = value;
5115
+ return typeof record.run_id === "string" && typeof record.experiment === "string" && typeof record.timestamp === "string" && (typeof record.index_path === "string" || typeof record.manifest_path === "string") && typeof record.display_name === "string" && typeof record.test_count === "number" && typeof record.avg_score === "number" && typeof record.size_bytes === "number" && (record.execution_error_count === void 0 || typeof record.execution_error_count === "number") && (record.pass_rate === void 0 || typeof record.pass_rate === "number") && (record.target === void 0 || typeof record.target === "string") && (record.summary_path === void 0 || typeof record.summary_path === "string");
5116
+ }
5117
+ function validateGitResultsIndexCacheFile(value, params) {
5118
+ if (!value || typeof value !== "object") return void 0;
5119
+ const record = value;
5120
+ const repoDir = resolveGitResultsIndexRepoDir(params.repoDir);
5121
+ const repoKey = gitResultsIndexRepoKey(params.repoDir);
5122
+ if (record.schema_version !== GIT_RESULTS_INDEX_CACHE_SCHEMA_VERSION || record.result_layout_version !== GIT_RESULTS_INDEX_LAYOUT_VERSION || record.repo_key !== repoKey || record.repo_dir !== repoDir || record.ref !== params.ref || record.commit_sha !== params.commitSha || !Array.isArray(record.runs) || !record.runs.every(isGitListedRun)) {
5123
+ return void 0;
5124
+ }
5125
+ return value;
5126
+ }
5127
+ async function readGitResultsIndexCache(params) {
5128
+ const cacheFile = resolveGitResultsIndexCacheFile(params);
5129
+ try {
5130
+ const parsed = JSON.parse(await readFile2(cacheFile, "utf8"));
5131
+ return validateGitResultsIndexCacheFile(parsed, params)?.runs;
5132
+ } catch {
5133
+ return void 0;
5134
+ }
5135
+ }
5136
+ async function writeGitResultsIndexCache(params) {
5137
+ const cacheFile = resolveGitResultsIndexCacheFile(params);
5138
+ const envelope = {
5139
+ schema_version: GIT_RESULTS_INDEX_CACHE_SCHEMA_VERSION,
5140
+ result_layout_version: GIT_RESULTS_INDEX_LAYOUT_VERSION,
5141
+ repo_key: gitResultsIndexRepoKey(params.repoDir),
5142
+ repo_dir: resolveGitResultsIndexRepoDir(params.repoDir),
5143
+ ref: params.ref,
5144
+ commit_sha: params.commitSha,
5145
+ generated_at: (/* @__PURE__ */ new Date()).toISOString(),
5146
+ runs: params.runs
5147
+ };
5148
+ try {
5149
+ mkdirSync(path4.dirname(cacheFile), { recursive: true });
5150
+ const tempFile = `${cacheFile}.${process.pid}.${Date.now()}.tmp`;
5151
+ await writeFile(tempFile, `${JSON.stringify(envelope, null, 2)}
5152
+ `, "utf8");
5153
+ renameSync(tempFile, cacheFile);
5154
+ } catch {
5155
+ }
5156
+ }
5157
+ async function resolveGitRunsRefCommit(repoDir, ref = "origin/main") {
5158
+ try {
5159
+ const { stdout } = await runGit(["rev-parse", "--verify", `${ref}^{commit}`], {
5160
+ cwd: repoDir
5161
+ });
5162
+ return stdout.trim() || void 0;
5163
+ } catch (error) {
5164
+ if (isMissingGitRefError(error)) {
5165
+ return void 0;
5166
+ }
5167
+ throw error;
5168
+ }
5169
+ }
5170
+ async function listGitRunsCached(repoDir, ref = "origin/main") {
5171
+ const commitSha = await resolveGitRunsRefCommit(repoDir, ref);
5172
+ if (!commitSha) {
5173
+ return [];
5174
+ }
5175
+ const cached = await readGitResultsIndexCache({ repoDir, ref, commitSha });
5176
+ if (cached) {
5177
+ return [...cached];
5178
+ }
5179
+ const runs = await listGitRuns(repoDir, ref);
5180
+ await writeGitResultsIndexCache({ repoDir, ref, commitSha, runs });
5181
+ return runs;
4612
5182
  }
4613
5183
  async function listGitRuns(repoDir, ref = "origin/main") {
4614
5184
  let treeOut;
@@ -4625,40 +5195,78 @@ async function listGitRuns(repoDir, ref = "origin/main") {
4625
5195
  }
4626
5196
  throw error;
4627
5197
  }
4628
- const benchmarkPaths = treeOut.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.endsWith("/benchmark.json"));
4629
- if (benchmarkPaths.length === 0) {
5198
+ const treePaths = treeOut.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
5199
+ const summaryPaths = treePaths.filter((line) => line.endsWith("/summary.json"));
5200
+ const summaryByPath = /* @__PURE__ */ new Map();
5201
+ if (summaryPaths.length > 0) {
5202
+ const summaryBatchInput = `${summaryPaths.map((summaryPath) => `${ref}:${summaryPath}`).join("\n")}
5203
+ `;
5204
+ const summaryBlobs = parseGitBatchBlobs(await runGitBatch(repoDir, summaryBatchInput));
5205
+ for (let i = 0; i < summaryBlobs.length; i++) {
5206
+ const summaryPath = summaryPaths[i];
5207
+ const blob = summaryBlobs[i];
5208
+ if (!summaryPath || !blob) continue;
5209
+ summaryByPath.set(summaryPath, JSON.parse(blob.content.toString("utf8")));
5210
+ }
5211
+ }
5212
+ const indexPaths = buildGitManifestPaths(treePaths, summaryByPath);
5213
+ if (indexPaths.length === 0) {
4630
5214
  return [];
4631
5215
  }
4632
- const batchInput = `${benchmarkPaths.map((benchmarkPath) => `${ref}:${benchmarkPath}`).join("\n")}
5216
+ const batchInput = `${indexPaths.map((indexPath) => `${ref}:${indexPath}`).join("\n")}
4633
5217
  `;
4634
5218
  const blobs = parseGitBatchBlobs(await runGitBatch(repoDir, batchInput));
4635
- if (blobs.length !== benchmarkPaths.length) {
5219
+ if (blobs.length !== indexPaths.length) {
4636
5220
  throw new Error(
4637
- `Expected ${benchmarkPaths.length} git blobs but received ${blobs.length} while listing results runs`
5221
+ `Expected ${indexPaths.length} git blobs but received ${blobs.length} while listing results runs`
4638
5222
  );
4639
5223
  }
4640
5224
  const runs = blobs.flatMap((blob, index) => {
4641
- const benchmarkPath = benchmarkPaths[index];
4642
- const benchmark = JSON.parse(blob.content.toString("utf8"));
4643
- const runDir = path4.posix.dirname(benchmarkPath);
5225
+ const manifestPath = indexPaths[index];
5226
+ const runDir = gitRunDirForManifestPath(manifestPath);
5227
+ const summaryPath = path4.posix.join(runDir, "summary.json");
5228
+ const summary = summaryByPath.get(summaryPath);
4644
5229
  const relativeRunPath = path4.posix.relative(RESULTS_REPO_RUNS_DIR, runDir);
4645
5230
  const runId = buildGitRunId(relativeRunPath);
4646
- const timestamp = benchmark.metadata?.timestamp?.trim() || path4.posix.basename(runDir);
4647
- const displayName = benchmark.metadata?.display_name?.trim() || path4.posix.basename(runDir);
4648
- const targets = benchmark.metadata?.targets ?? [];
4649
- const passRate = computeAveragePassRate(benchmark.run_summary);
5231
+ const rows = blob.content.toString("utf8").split(/\r?\n/).map((line) => line.trim()).filter(Boolean).flatMap(
5232
+ (line) => {
5233
+ try {
5234
+ return [
5235
+ JSON.parse(line)
5236
+ ];
5237
+ } catch {
5238
+ return [];
5239
+ }
5240
+ }
5241
+ );
5242
+ const rowTargets = [
5243
+ ...new Set(rows.map((row) => row.target).filter((target) => !!target))
5244
+ ];
5245
+ const rowTestIds = [
5246
+ ...new Set(rows.map((row) => row.test_id).filter((testId) => !!testId))
5247
+ ];
5248
+ const rowScores = rows.map((row) => row.score).filter((score) => typeof score === "number" && Number.isFinite(score));
5249
+ const avgScore = rowScores.length > 0 ? rowScores.reduce((sum, score) => sum + score, 0) / rowScores.length : 0;
5250
+ const timestamp = summary?.metadata?.timestamp?.trim() || rows[0]?.timestamp || path4.posix.basename(runDir);
5251
+ const displayName = summary?.metadata?.display_name?.trim() || path4.posix.basename(runDir);
5252
+ const targets = summary?.metadata?.targets ?? rowTargets;
5253
+ const passRate = computeAveragePassRate(summary?.run_summary) ?? avgScore;
5254
+ const executionErrorCount = rows.filter(
5255
+ (row) => row.execution_status === "execution_error"
5256
+ ).length;
4650
5257
  return [
4651
5258
  {
4652
5259
  run_id: runId,
4653
- experiment: getRunExperiment(runId, benchmark),
5260
+ experiment: getRunExperiment(runId, summary),
4654
5261
  timestamp,
4655
5262
  ...passRate !== void 0 && { pass_rate: passRate },
4656
5263
  ...targets.length === 1 && targets[0] ? { target: targets[0] } : {},
4657
- manifest_path: path4.posix.join(runDir, "index.jsonl"),
4658
- benchmark_path: benchmarkPath,
5264
+ index_path: manifestPath,
5265
+ ...summaryByPath.has(summaryPath) && { summary_path: summaryPath },
4659
5266
  display_name: displayName,
4660
- test_count: benchmark.metadata?.tests_run?.length ?? 0,
4661
- avg_score: 0,
5267
+ test_count: summary?.metadata?.tests_run?.length ?? rowTestIds.length,
5268
+ ...executionErrorCount > 0 && { execution_error_count: executionErrorCount },
5269
+ avg_score: avgScore,
4662
5270
  size_bytes: blob.size
4663
5271
  }
4664
5272
  ];
@@ -4776,483 +5384,6 @@ function trimBaselineResult(result) {
4776
5384
  }
4777
5385
  return trimmed;
4778
5386
  }
4779
- var DEFAULT_CATEGORY = "Uncategorized";
4780
- function deriveCategory(relativePath) {
4781
- const parts = relativePath.split(/[/\\]/);
4782
- if (parts.length <= 1) {
4783
- return DEFAULT_CATEGORY;
4784
- }
4785
- const dirs = parts.slice(0, -1).filter((d) => d !== "evals");
4786
- return dirs.length > 0 ? dirs.join("/") : DEFAULT_CATEGORY;
4787
- }
4788
- var OtelTraceExporter = class {
4789
- constructor(options) {
4790
- this.options = options;
4791
- }
4792
- provider = null;
4793
- tracer = null;
4794
- api = null;
4795
- // biome-ignore lint/suspicious/noExplicitAny: OTel types loaded dynamically
4796
- W3CPropagator = null;
4797
- /** Initialize the OTel SDK. Returns false if OTel packages are not available. */
4798
- async init() {
4799
- try {
4800
- const [sdkTraceNode, resourcesMod, semconvMod, api, coreMod] = await Promise.all([
4801
- import("./src-76ISS3LH.js"),
4802
- import("./esm-NNMQYIKT.js"),
4803
- import("./esm-RVQPUGWH.js"),
4804
- import("./esm-R77SNOF5.js"),
4805
- import("./esm-ZADQ4XQH-QUFGGZL4.js").catch(() => null)
4806
- ]);
4807
- const { NodeTracerProvider: Provider, SimpleSpanProcessor } = sdkTraceNode;
4808
- const { resourceFromAttributes } = resourcesMod;
4809
- const { ATTR_SERVICE_NAME } = semconvMod;
4810
- const resourceAttributes = {
4811
- [ATTR_SERVICE_NAME]: this.options.serviceName ?? "agentv",
4812
- ...this.options.resourceAttributes
4813
- };
4814
- const resource = resourceFromAttributes(resourceAttributes);
4815
- const processors = [];
4816
- if (this.options.endpoint) {
4817
- const otlpHttp = await import("./esm-TTEMJEVV.js");
4818
- const { OTLPTraceExporter } = otlpHttp;
4819
- const exporter = new OTLPTraceExporter({
4820
- url: this.options.endpoint,
4821
- headers: this.options.headers
4822
- });
4823
- processors.push(new SimpleSpanProcessor(exporter));
4824
- }
4825
- if (this.options.otlpFilePath) {
4826
- const { OtlpJsonFileExporter: OtlpJsonFileExporter2 } = await import("./otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js");
4827
- processors.push(
4828
- new SimpleSpanProcessor(
4829
- new OtlpJsonFileExporter2(this.options.otlpFilePath, resourceAttributes)
4830
- )
4831
- );
4832
- }
4833
- if (processors.length === 0) {
4834
- return false;
4835
- }
4836
- this.provider = new Provider({
4837
- resource,
4838
- spanProcessors: processors
4839
- });
4840
- this.provider.register();
4841
- this.api = api;
4842
- this.tracer = api.trace.getTracer("agentv", "1.0.0");
4843
- this.W3CPropagator = coreMod?.W3CTraceContextPropagator ?? null;
4844
- return true;
4845
- } catch {
4846
- return false;
4847
- }
4848
- }
4849
- /** Export a single evaluation result as an OTel trace. */
4850
- async exportResult(result) {
4851
- if (!this.tracer || !this.api) return;
4852
- const api = this.api;
4853
- const tracer = this.tracer;
4854
- const captureContent = this.options.captureContent ?? false;
4855
- const startHr = toHrTime(result.startTime ?? result.timestamp);
4856
- const endHr = toHrTime(result.endTime ?? result.timestamp);
4857
- let parentCtx = api.ROOT_CONTEXT;
4858
- const traceparent = process.env.TRACEPARENT;
4859
- if (traceparent && this.W3CPropagator) {
4860
- try {
4861
- const propagator = new this.W3CPropagator();
4862
- parentCtx = propagator.extract(
4863
- api.ROOT_CONTEXT,
4864
- { traceparent, tracestate: process.env.TRACESTATE ?? "" },
4865
- {
4866
- get: (carrier, key) => carrier[key],
4867
- keys: (carrier) => Object.keys(carrier)
4868
- }
4869
- );
4870
- } catch {
4871
- }
4872
- }
4873
- tracer.startActiveSpan(
4874
- "agentv.eval",
4875
- { startTime: startHr },
4876
- parentCtx,
4877
- (rootSpan) => {
4878
- rootSpan.setAttribute("gen_ai.operation.name", "evaluate");
4879
- rootSpan.setAttribute("gen_ai.system", "agentv");
4880
- rootSpan.setAttribute("agentv.test_id", result.testId);
4881
- rootSpan.setAttribute("agentv.target", result.target);
4882
- if (result.suite) rootSpan.setAttribute("agentv.suite", result.suite);
4883
- rootSpan.setAttribute("agentv.score", result.score);
4884
- if (captureContent && result.output.length > 0) {
4885
- rootSpan.setAttribute("agentv.output_text", result.output);
4886
- }
4887
- if (result.durationMs != null)
4888
- rootSpan.setAttribute("agentv.trace.duration_ms", result.durationMs);
4889
- if (result.costUsd != null) rootSpan.setAttribute("agentv.trace.cost_usd", result.costUsd);
4890
- if (result.tokenUsage) {
4891
- if (result.tokenUsage.input != null) {
4892
- rootSpan.setAttribute("agentv.trace.token_input", result.tokenUsage.input);
4893
- }
4894
- if (result.tokenUsage.output != null) {
4895
- rootSpan.setAttribute("agentv.trace.token_output", result.tokenUsage.output);
4896
- }
4897
- if (result.tokenUsage.cached != null) {
4898
- rootSpan.setAttribute("agentv.trace.token_cached", result.tokenUsage.cached);
4899
- }
4900
- }
4901
- if (result.trace) {
4902
- const t = result.trace;
4903
- rootSpan.setAttribute("agentv.trace.event_count", t.eventCount);
4904
- rootSpan.setAttribute(
4905
- "agentv.trace.tool_names",
4906
- Object.keys(t.toolCalls).sort().join(",")
4907
- );
4908
- if (t.llmCallCount != null)
4909
- rootSpan.setAttribute("agentv.trace.llm_call_count", t.llmCallCount);
4910
- }
4911
- const traceMessages = result.trace?.messages ?? [];
4912
- if (traceMessages.length > 0) {
4913
- const parentCtx2 = api.trace.setSpan(api.context.active(), rootSpan);
4914
- if (this.options.groupTurns) {
4915
- const turns = groupMessagesIntoTurns(traceMessages);
4916
- if (turns.length > 1) {
4917
- for (const [i, turn] of turns.entries()) {
4918
- api.context.with(parentCtx2, () => {
4919
- tracer.startActiveSpan(
4920
- `agentv.turn.${i + 1}`,
4921
- {},
4922
- (turnSpan) => {
4923
- const turnCtx = api.trace.setSpan(api.context.active(), turnSpan);
4924
- for (const msg of turn.messages) {
4925
- this.exportMessage(tracer, api, turnCtx, msg, captureContent);
4926
- }
4927
- turnSpan.end();
4928
- }
4929
- );
4930
- });
4931
- }
4932
- } else {
4933
- for (const msg of traceMessages) {
4934
- this.exportMessage(tracer, api, parentCtx2, msg, captureContent);
4935
- }
4936
- }
4937
- } else {
4938
- for (const msg of traceMessages) {
4939
- this.exportMessage(tracer, api, parentCtx2, msg, captureContent);
4940
- }
4941
- }
4942
- }
4943
- if (result.scores) {
4944
- for (const score of result.scores) {
4945
- rootSpan.addEvent(`agentv.grader.${score.name}`, {
4946
- "agentv.grader.score": score.score,
4947
- "agentv.grader.type": score.type,
4948
- ...score.verdict ? { "agentv.grader.verdict": score.verdict } : {}
4949
- });
4950
- }
4951
- }
4952
- if (result.error) {
4953
- rootSpan.setStatus({ code: api.SpanStatusCode.ERROR, message: result.error });
4954
- } else {
4955
- rootSpan.setStatus({ code: api.SpanStatusCode.OK });
4956
- }
4957
- rootSpan.end(endHr);
4958
- }
4959
- );
4960
- }
4961
- /** Flush pending spans and shut down. */
4962
- async shutdown() {
4963
- await this.provider?.shutdown();
4964
- }
4965
- /** Create a streaming observer for real-time span export */
4966
- createStreamingObserver() {
4967
- if (!this.tracer || !this.api) return null;
4968
- let parentCtx;
4969
- const traceparent = process.env.TRACEPARENT;
4970
- if (traceparent && this.W3CPropagator) {
4971
- try {
4972
- const propagator = new this.W3CPropagator();
4973
- parentCtx = propagator.extract(
4974
- this.api.ROOT_CONTEXT,
4975
- { traceparent, tracestate: process.env.TRACESTATE ?? "" },
4976
- {
4977
- get: (carrier, key) => carrier[key],
4978
- keys: (carrier) => Object.keys(carrier)
4979
- }
4980
- );
4981
- } catch {
4982
- }
4983
- }
4984
- return new OtelStreamingObserver(
4985
- this.tracer,
4986
- this.api,
4987
- this.options.captureContent ?? false,
4988
- parentCtx
4989
- );
4990
- }
4991
- // -----------------------------------------------------------------------
4992
- // Private helpers
4993
- // -----------------------------------------------------------------------
4994
- exportMessage(tracer, api, parentCtx, msg, captureContent) {
4995
- const isAssistant = msg.role === "assistant";
4996
- const model = msg.metadata?.model ? String(msg.metadata.model) : void 0;
4997
- const spanName = isAssistant ? `chat ${model ?? "unknown"}` : `gen_ai.message.${msg.role}`;
4998
- const startHr = toHrTime(msg.startTime);
4999
- const endHr = toHrTime(msg.endTime);
5000
- api.context.with(parentCtx, () => {
5001
- tracer.startActiveSpan(
5002
- spanName,
5003
- { startTime: startHr },
5004
- parentCtx,
5005
- (span) => {
5006
- if (isAssistant) {
5007
- span.setAttribute("gen_ai.operation.name", "chat");
5008
- }
5009
- if (model) {
5010
- span.setAttribute("gen_ai.request.model", model);
5011
- span.setAttribute("gen_ai.response.model", model);
5012
- }
5013
- if (msg.tokenUsage) {
5014
- if (msg.tokenUsage.input != null) {
5015
- span.setAttribute("gen_ai.usage.input_tokens", msg.tokenUsage.input);
5016
- }
5017
- if (msg.tokenUsage.output != null) {
5018
- span.setAttribute("gen_ai.usage.output_tokens", msg.tokenUsage.output);
5019
- }
5020
- if (msg.tokenUsage.cached != null) {
5021
- span.setAttribute("gen_ai.usage.cache_read.input_tokens", msg.tokenUsage.cached);
5022
- }
5023
- }
5024
- if (captureContent && msg.content != null) {
5025
- span.setAttribute(
5026
- "gen_ai.output.messages",
5027
- typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
5028
- );
5029
- }
5030
- if (msg.toolCalls) {
5031
- const msgCtx = api.trace.setSpan(api.context.active(), span);
5032
- for (const tc of msg.toolCalls) {
5033
- api.context.with(msgCtx, () => {
5034
- tracer.startActiveSpan(
5035
- `execute_tool ${tc.tool}`,
5036
- {},
5037
- msgCtx,
5038
- (toolSpan) => {
5039
- toolSpan.setAttribute("gen_ai.tool.name", tc.tool);
5040
- if (tc.id) toolSpan.setAttribute("gen_ai.tool.call.id", tc.id);
5041
- if (captureContent) {
5042
- if (tc.input != null) {
5043
- toolSpan.setAttribute(
5044
- "gen_ai.tool.call.arguments",
5045
- typeof tc.input === "string" ? tc.input : JSON.stringify(tc.input)
5046
- );
5047
- }
5048
- if (tc.output != null) {
5049
- toolSpan.setAttribute(
5050
- "gen_ai.tool.call.result",
5051
- typeof tc.output === "string" ? tc.output : JSON.stringify(tc.output)
5052
- );
5053
- }
5054
- }
5055
- toolSpan.end();
5056
- }
5057
- );
5058
- });
5059
- }
5060
- }
5061
- span.end(endHr);
5062
- }
5063
- );
5064
- });
5065
- }
5066
- };
5067
- var OtelStreamingObserver = class {
5068
- constructor(tracer, api, captureContent, parentCtx) {
5069
- this.tracer = tracer;
5070
- this.api = api;
5071
- this.captureContent = captureContent;
5072
- this.parentCtx = parentCtx;
5073
- }
5074
- // biome-ignore lint/suspicious/noExplicitAny: OTel span type loaded dynamically
5075
- rootSpan = null;
5076
- // biome-ignore lint/suspicious/noExplicitAny: OTel context loaded dynamically
5077
- rootCtx = null;
5078
- observedChildSpans = false;
5079
- pendingMetrics = null;
5080
- /** Create root eval span immediately (visible in backend right away) */
5081
- startEvalCase(testId, target, evalSet) {
5082
- this.pendingMetrics = null;
5083
- this.observedChildSpans = false;
5084
- const ctx = this.parentCtx ?? this.api.context.active();
5085
- this.rootSpan = this.tracer.startSpan("agentv.eval", void 0, ctx);
5086
- this.rootSpan.setAttribute("gen_ai.operation.name", "evaluate");
5087
- this.rootSpan.setAttribute("gen_ai.system", "agentv");
5088
- this.rootSpan.setAttribute("agentv.test_id", testId);
5089
- this.rootSpan.setAttribute("agentv.target", target);
5090
- if (evalSet) this.rootSpan.setAttribute("agentv.suite", evalSet);
5091
- this.rootCtx = this.api.trace.setSpan(this.api.context.active(), this.rootSpan);
5092
- }
5093
- /** Create and immediately export a tool span */
5094
- onToolCall(name, input, output, _durationMs, toolCallId) {
5095
- if (!this.rootCtx) return;
5096
- this.observedChildSpans = true;
5097
- this.api.context.with(this.rootCtx, () => {
5098
- const span = this.tracer.startSpan(`execute_tool ${name}`, void 0, this.rootCtx);
5099
- span.setAttribute("gen_ai.tool.name", name);
5100
- if (toolCallId) span.setAttribute("gen_ai.tool.call.id", toolCallId);
5101
- if (this.captureContent) {
5102
- if (input != null)
5103
- span.setAttribute(
5104
- "gen_ai.tool.call.arguments",
5105
- typeof input === "string" ? input : JSON.stringify(input)
5106
- );
5107
- if (output != null)
5108
- span.setAttribute(
5109
- "gen_ai.tool.call.result",
5110
- typeof output === "string" ? output : JSON.stringify(output)
5111
- );
5112
- }
5113
- span.end();
5114
- });
5115
- }
5116
- /** Create and immediately export an LLM span */
5117
- onLlmCall(model, tokenUsage) {
5118
- if (!this.rootCtx) return;
5119
- this.observedChildSpans = true;
5120
- this.api.context.with(this.rootCtx, () => {
5121
- const span = this.tracer.startSpan(`chat ${model}`, void 0, this.rootCtx);
5122
- span.setAttribute("gen_ai.operation.name", "chat");
5123
- span.setAttribute("gen_ai.request.model", model);
5124
- span.setAttribute("gen_ai.response.model", model);
5125
- if (tokenUsage) {
5126
- if (tokenUsage.input != null)
5127
- span.setAttribute("gen_ai.usage.input_tokens", tokenUsage.input);
5128
- if (tokenUsage.output != null)
5129
- span.setAttribute("gen_ai.usage.output_tokens", tokenUsage.output);
5130
- if (tokenUsage.cached != null)
5131
- span.setAttribute("gen_ai.usage.cache_read.input_tokens", tokenUsage.cached);
5132
- }
5133
- span.end();
5134
- });
5135
- }
5136
- /** Record final execution metrics before the root span is finalized. */
5137
- recordEvalMetrics(result) {
5138
- this.pendingMetrics = result;
5139
- }
5140
- /** Finalize root span with score/verdict after evaluation completes */
5141
- finalizeEvalCase(score, error) {
5142
- if (!this.rootSpan) return;
5143
- this.rootSpan.setAttribute("agentv.score", score);
5144
- if (this.pendingMetrics?.durationMs != null) {
5145
- this.rootSpan.setAttribute("agentv.trace.duration_ms", this.pendingMetrics.durationMs);
5146
- }
5147
- if (this.pendingMetrics?.costUsd != null) {
5148
- this.rootSpan.setAttribute("agentv.trace.cost_usd", this.pendingMetrics.costUsd);
5149
- }
5150
- if (this.pendingMetrics?.tokenUsage) {
5151
- if (this.pendingMetrics.tokenUsage.input != null) {
5152
- this.rootSpan.setAttribute(
5153
- "agentv.trace.token_input",
5154
- this.pendingMetrics.tokenUsage.input
5155
- );
5156
- }
5157
- if (this.pendingMetrics.tokenUsage.output != null) {
5158
- this.rootSpan.setAttribute(
5159
- "agentv.trace.token_output",
5160
- this.pendingMetrics.tokenUsage.output
5161
- );
5162
- }
5163
- if (this.pendingMetrics.tokenUsage.cached != null) {
5164
- this.rootSpan.setAttribute(
5165
- "agentv.trace.token_cached",
5166
- this.pendingMetrics.tokenUsage.cached
5167
- );
5168
- }
5169
- }
5170
- if (this.pendingMetrics?.trace) {
5171
- this.rootSpan.setAttribute("agentv.trace.event_count", this.pendingMetrics.trace.eventCount);
5172
- this.rootSpan.setAttribute(
5173
- "agentv.trace.tool_names",
5174
- Object.keys(this.pendingMetrics.trace.toolCalls).sort().join(",")
5175
- );
5176
- if (this.pendingMetrics.trace.llmCallCount != null) {
5177
- this.rootSpan.setAttribute(
5178
- "agentv.trace.llm_call_count",
5179
- this.pendingMetrics.trace.llmCallCount
5180
- );
5181
- }
5182
- }
5183
- if (error) {
5184
- this.rootSpan.setStatus({ code: this.api.SpanStatusCode.ERROR, message: error });
5185
- } else {
5186
- this.rootSpan.setStatus({ code: this.api.SpanStatusCode.OK });
5187
- }
5188
- this.rootSpan.end();
5189
- this.rootSpan = null;
5190
- this.rootCtx = null;
5191
- this.observedChildSpans = false;
5192
- this.pendingMetrics = null;
5193
- }
5194
- /** Backfill child spans from the completed result when the provider emitted no live callbacks. */
5195
- completeFromResult(result) {
5196
- this.recordEvalMetrics({
5197
- durationMs: result.durationMs,
5198
- costUsd: result.costUsd,
5199
- tokenUsage: result.tokenUsage,
5200
- trace: result.trace
5201
- });
5202
- if (this.observedChildSpans || !this.rootCtx) {
5203
- return;
5204
- }
5205
- const model = result.trace.messages.find((msg) => msg.role === "assistant")?.metadata?.model ?? result.target ?? "unknown";
5206
- this.onLlmCall(String(model), result.tokenUsage);
5207
- for (const message of result.trace.messages) {
5208
- for (const toolCall of message.toolCalls ?? []) {
5209
- this.onToolCall(
5210
- toolCall.tool,
5211
- toolCall.input,
5212
- toolCall.output,
5213
- toolCall.durationMs ?? 0,
5214
- toolCall.id
5215
- );
5216
- }
5217
- }
5218
- }
5219
- /** Return the active eval span's trace ID and span ID for Braintrust trace bridging */
5220
- getActiveSpanIds() {
5221
- if (!this.rootSpan) return null;
5222
- try {
5223
- const spanCtx = this.rootSpan.spanContext?.() ?? this.rootSpan._spanContext;
5224
- if (!spanCtx?.traceId || !spanCtx?.spanId) return null;
5225
- return { parentSpanId: spanCtx.spanId, rootSpanId: spanCtx.traceId };
5226
- } catch {
5227
- return null;
5228
- }
5229
- }
5230
- /** Get ProviderStreamCallbacks for passing to providers */
5231
- getStreamCallbacks() {
5232
- return {
5233
- onToolCallEnd: (name, input, output, durationMs, toolCallId) => this.onToolCall(name, input, output, durationMs, toolCallId),
5234
- onLlmCallEnd: (model, tokenUsage) => this.onLlmCall(model, tokenUsage),
5235
- getActiveSpanIds: () => this.getActiveSpanIds()
5236
- };
5237
- }
5238
- };
5239
- function groupMessagesIntoTurns(messages) {
5240
- const turns = [];
5241
- let current = [];
5242
- for (const msg of messages) {
5243
- if (msg.role === "user" && current.length > 0) {
5244
- turns.push({ messages: current });
5245
- current = [];
5246
- }
5247
- current.push(msg);
5248
- }
5249
- if (current.length > 0) turns.push({ messages: current });
5250
- return turns;
5251
- }
5252
- function toHrTime(iso) {
5253
- if (!iso) return void 0;
5254
- return new Date(iso).getTime();
5255
- }
5256
5387
  var RunBudgetTracker = class {
5257
5388
  constructor(capUsd) {
5258
5389
  this.capUsd = capUsd;
@@ -5432,7 +5563,7 @@ function parseClaudeSession(jsonl) {
5432
5563
  durationMs = new Date(endTimestamp).getTime() - new Date(startTimestamp).getTime();
5433
5564
  }
5434
5565
  const source = {
5435
- provider: "claude",
5566
+ provider: "claude-sdk",
5436
5567
  sessionId,
5437
5568
  projectPath,
5438
5569
  startedAt: startTimestamp,
@@ -5482,7 +5613,7 @@ function extractAssistantContent(content) {
5482
5613
  case "tool_use":
5483
5614
  if (block.name) {
5484
5615
  toolCalls.push(
5485
- normalizeToolCall("claude", {
5616
+ normalizeToolCall("claude-sdk", {
5486
5617
  tool: block.name,
5487
5618
  input: block.input,
5488
5619
  id: block.id
@@ -5577,7 +5708,7 @@ function parseCodexSession(jsonl) {
5577
5708
  } else {
5578
5709
  input = payload.arguments;
5579
5710
  }
5580
- const toolCall = normalizeToolCall("codex", {
5711
+ const toolCall = normalizeToolCall("codex-cli", {
5581
5712
  tool: toolName,
5582
5713
  input,
5583
5714
  id: callId
@@ -5605,7 +5736,7 @@ function parseCodexSession(jsonl) {
5605
5736
  } else {
5606
5737
  input = payload.arguments;
5607
5738
  }
5608
- const toolCall = normalizeToolCall("codex", {
5739
+ const toolCall = normalizeToolCall("codex-cli", {
5609
5740
  tool: toolName,
5610
5741
  input,
5611
5742
  id: callId
@@ -5793,11 +5924,13 @@ var TranscriptProvider = class _TranscriptProvider {
5793
5924
  kind = "transcript";
5794
5925
  targetName;
5795
5926
  entries;
5927
+ entriesByTestId;
5796
5928
  cursor = 0;
5797
5929
  constructor(targetName, entries) {
5798
5930
  this.targetName = targetName;
5799
5931
  this.id = `transcript:${targetName}`;
5800
5932
  this.entries = entries;
5933
+ this.entriesByTestId = new Map(entries.map((entry) => [entry.testId, entry]));
5801
5934
  }
5802
5935
  /**
5803
5936
  * Create a TranscriptProvider from a JSONL file path.
@@ -5814,13 +5947,11 @@ var TranscriptProvider = class _TranscriptProvider {
5814
5947
  get lineCount() {
5815
5948
  return this.entries.length;
5816
5949
  }
5817
- async invoke(_request) {
5818
- if (this.cursor >= this.entries.length) {
5819
- throw new Error(
5820
- `Transcript exhausted: ${this.entries.length} entr${this.entries.length === 1 ? "y" : "ies"} available but ${this.cursor + 1} invocations attempted. Each transcript entry maps to one test case.`
5821
- );
5822
- }
5823
- const entry = this.entries[this.cursor++];
5950
+ get testIds() {
5951
+ return this.entries.map((entry) => entry.testId);
5952
+ }
5953
+ async invoke(request) {
5954
+ const entry = request.evalCaseId ? this.entryForTestId(request.evalCaseId) : this.nextPositionalEntry();
5824
5955
  return {
5825
5956
  output: entry.messages,
5826
5957
  tokenUsage: entry.tokenUsage ? {
@@ -5834,7 +5965,153 @@ var TranscriptProvider = class _TranscriptProvider {
5834
5965
  startTime: entry.source.startedAt
5835
5966
  };
5836
5967
  }
5968
+ entryForTestId(testId) {
5969
+ const entry = this.entriesByTestId.get(testId);
5970
+ if (entry) {
5971
+ return entry;
5972
+ }
5973
+ throw new Error(
5974
+ `Transcript replay found no entry for test_id=${testId}. Available test_id values: ${this.testIds.join(", ") || "<none>"}`
5975
+ );
5976
+ }
5977
+ nextPositionalEntry() {
5978
+ if (this.cursor >= this.entries.length) {
5979
+ throw new Error(
5980
+ `Transcript exhausted: ${this.entries.length} entr${this.entries.length === 1 ? "y" : "ies"} available but ${this.cursor + 1} invocations attempted. Each transcript entry maps to one test case.`
5981
+ );
5982
+ }
5983
+ return this.entries[this.cursor++];
5984
+ }
5837
5985
  };
5986
+ function parseCopilotEvents(eventsJsonl) {
5987
+ const messages = [];
5988
+ const meta = { sessionId: "", model: "", cwd: "" };
5989
+ let totalInputTokens = 0;
5990
+ let totalOutputTokens = 0;
5991
+ let hasUsage = false;
5992
+ let startTimestamp;
5993
+ let endTimestamp;
5994
+ const toolCallsInProgress = /* @__PURE__ */ new Map();
5995
+ const lines = eventsJsonl.split("\n").filter((l) => l.trim().length > 0);
5996
+ for (const line of lines) {
5997
+ let event;
5998
+ try {
5999
+ event = JSON.parse(line);
6000
+ } catch {
6001
+ continue;
6002
+ }
6003
+ const eventType = event.type;
6004
+ if (!eventType) continue;
6005
+ const data = event.data ?? {};
6006
+ switch (eventType) {
6007
+ case "session.start": {
6008
+ meta.sessionId = String(data.sessionId ?? "");
6009
+ const ctx = data.context;
6010
+ meta.cwd = String(ctx?.cwd ?? "");
6011
+ meta.repository = ctx?.repository ? String(ctx.repository) : void 0;
6012
+ meta.branch = ctx?.branch ? String(ctx.branch) : void 0;
6013
+ const ts = event.timestamp ?? data.startTime;
6014
+ meta.startedAt = ts ? String(ts) : void 0;
6015
+ startTimestamp = ts ? String(ts) : void 0;
6016
+ break;
6017
+ }
6018
+ case "user.message": {
6019
+ messages.push({
6020
+ role: "user",
6021
+ content: data.content != null ? String(data.content) : ""
6022
+ });
6023
+ break;
6024
+ }
6025
+ case "assistant.message": {
6026
+ const toolRequests = data.toolRequests;
6027
+ const toolCalls = (toolRequests ?? []).map(
6028
+ (req) => normalizeToolCall("copilot-events", {
6029
+ tool: String(req.name ?? req.toolName ?? ""),
6030
+ input: req.arguments,
6031
+ id: req.toolCallId ? String(req.toolCallId) : void 0
6032
+ })
6033
+ );
6034
+ messages.push({
6035
+ role: "assistant",
6036
+ content: data.content != null ? String(data.content) : void 0,
6037
+ toolCalls: toolCalls.length > 0 ? toolCalls : void 0
6038
+ });
6039
+ break;
6040
+ }
6041
+ case "skill.invoked": {
6042
+ const skillName = String(data.name ?? "");
6043
+ messages.push({
6044
+ role: "assistant",
6045
+ toolCalls: [
6046
+ {
6047
+ tool: "Skill",
6048
+ input: { skill: skillName }
6049
+ }
6050
+ ]
6051
+ });
6052
+ break;
6053
+ }
6054
+ case "tool.execution_start": {
6055
+ const toolCallId = String(data.toolCallId ?? "");
6056
+ if (toolCallId) {
6057
+ toolCallsInProgress.set(toolCallId, {
6058
+ toolName: String(data.toolName ?? ""),
6059
+ input: data.arguments,
6060
+ toolCallId
6061
+ });
6062
+ }
6063
+ break;
6064
+ }
6065
+ case "tool.execution_complete": {
6066
+ const toolCallId = String(data.toolCallId ?? "");
6067
+ const started = toolCallsInProgress.get(toolCallId);
6068
+ if (started) {
6069
+ toolCallsInProgress.delete(toolCallId);
6070
+ messages.push({
6071
+ role: "assistant",
6072
+ toolCalls: [
6073
+ normalizeToolCall("copilot-events", {
6074
+ tool: started.toolName,
6075
+ input: started.input,
6076
+ output: data.result,
6077
+ id: toolCallId
6078
+ })
6079
+ ]
6080
+ });
6081
+ }
6082
+ break;
6083
+ }
6084
+ case "session.shutdown": {
6085
+ endTimestamp = event.timestamp ? String(event.timestamp) : void 0;
6086
+ const modelMetrics = data.modelMetrics;
6087
+ if (modelMetrics) {
6088
+ for (const metrics of Object.values(modelMetrics)) {
6089
+ if (metrics.usage) {
6090
+ hasUsage = true;
6091
+ totalInputTokens += Number(metrics.usage.inputTokens ?? 0);
6092
+ totalOutputTokens += Number(metrics.usage.outputTokens ?? 0);
6093
+ }
6094
+ }
6095
+ }
6096
+ const currentModel = data.currentModel;
6097
+ if (currentModel && !meta.model) {
6098
+ meta.model = String(currentModel);
6099
+ }
6100
+ break;
6101
+ }
6102
+ }
6103
+ }
6104
+ let durationMs;
6105
+ if (startTimestamp && endTimestamp) {
6106
+ durationMs = new Date(endTimestamp).getTime() - new Date(startTimestamp).getTime();
6107
+ }
6108
+ return {
6109
+ messages,
6110
+ meta,
6111
+ tokenUsage: hasUsage ? { input: totalInputTokens, output: totalOutputTokens } : void 0,
6112
+ durationMs
6113
+ };
6114
+ }
5838
6115
  function createAgentKernel() {
5839
6116
  return { status: "stub" };
5840
6117
  }
@@ -5857,9 +6134,11 @@ export {
5857
6134
  getResultsRepoLocalPaths,
5858
6135
  ensureResultsRepoClone,
5859
6136
  getResultsRepoStatus,
6137
+ buildResultsCompareUrl,
5860
6138
  getResultsRepoSyncStatus,
5861
6139
  syncResultsRepo,
5862
6140
  syncResultsRepoForProject,
6141
+ confirmResultsMergeAndPull,
5863
6142
  checkoutResultsRepoBranch,
5864
6143
  prepareResultsRepoBranch,
5865
6144
  stageResultsArtifacts,
@@ -5875,15 +6154,14 @@ export {
5875
6154
  setupWipWorktree,
5876
6155
  pushWipCheckpoint,
5877
6156
  deleteWipBranch,
6157
+ resolveGitResultsIndexCacheFile,
6158
+ resolveGitRunsRefCommit,
6159
+ listGitRunsCached,
5878
6160
  listGitRuns,
5879
6161
  materializeGitRun,
5880
6162
  syncProject,
5881
6163
  syncProjects,
5882
6164
  trimBaselineResult,
5883
- DEFAULT_CATEGORY,
5884
- deriveCategory,
5885
- OtelTraceExporter,
5886
- OtelStreamingObserver,
5887
6165
  RunBudgetTracker,
5888
6166
  parseEnvOutput,
5889
6167
  runBeforeSessionHook,
@@ -5892,6 +6170,7 @@ export {
5892
6170
  discoverCodexSessions,
5893
6171
  discoverClaudeSessions,
5894
6172
  TranscriptProvider,
6173
+ parseCopilotEvents,
5895
6174
  createAgentKernel
5896
6175
  };
5897
- //# sourceMappingURL=chunk-NCNDA3F6.js.map
6176
+ //# sourceMappingURL=chunk-GJWPQXHN.js.map