agentv 4.44.0-next.1 → 5.1.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +236 -54
- package/dist/{agentv-provider-AYXH7WLW-NJRC6UQX.js → agentv-provider-AYXH7WLW-6YZKZ7PS.js} +2 -2
- package/dist/{artifact-writer-72MQQELS.js → artifact-writer-IORG7CW5.js} +21 -12
- package/dist/{chunk-NPVGBFF6.js → chunk-7BGERE6L.js} +2 -43
- package/dist/chunk-7BGERE6L.js.map +1 -0
- package/dist/{chunk-CXHYTYHU.js → chunk-AM2IPHKS.js} +3496 -1955
- package/dist/chunk-AM2IPHKS.js.map +1 -0
- package/dist/{chunk-KAGZRUO6.js → chunk-CBBBRTQ3.js} +127 -80
- package/dist/chunk-CBBBRTQ3.js.map +1 -0
- package/dist/{chunk-2N3YSEMI.js → chunk-GIRUNQGP.js} +8323 -9226
- package/dist/chunk-GIRUNQGP.js.map +1 -0
- package/dist/{chunk-NCNDA3F6.js → chunk-GJWPQXHN.js} +1070 -791
- package/dist/chunk-GJWPQXHN.js.map +1 -0
- package/dist/{chunk-BPGJ4HBU.js → chunk-PEUTJS7B.js} +3 -3
- package/dist/{chunk-5H446C7X.js → chunk-QMRVH5ZP.js} +2 -8
- package/dist/{chunk-5DYGN547.js → chunk-ZGPCX334.js} +44263 -30424
- package/dist/chunk-ZGPCX334.js.map +1 -0
- package/dist/cli.js +8 -9
- package/dist/cli.js.map +1 -1
- package/dist/dashboard/assets/{index-DitffAxS.js → index-BSHoWrdr.js} +1 -1
- package/dist/dashboard/assets/index-BaLLxmVO.js +121 -0
- package/dist/dashboard/assets/index-zvUKAkKn.css +1 -0
- package/dist/dashboard/index.html +2 -2
- package/dist/{dist-L4YCQOQ6.js → dist-2D2GBHZV.js} +101 -51
- package/dist/{docker-workspace-RPPXBT27-B4AQHVWA.js → docker-workspace-NNK4Z6FR-QRTIG6TM.js} +3 -3
- package/dist/{exec-AR6JUUN5-6MBPURPR.js → exec-4HFFRQ3B-AXDMJNLJ.js} +3 -3
- package/dist/index.js +8 -9
- package/dist/{interactive-DRNRKLEQ.js → interactive-V4SEHLNY.js} +13 -29
- package/dist/interactive-V4SEHLNY.js.map +1 -0
- package/dist/skills/agentv-bench/SKILL.md +26 -26
- package/dist/skills/agentv-bench/agents/analyzer.md +7 -7
- package/dist/skills/agentv-bench/agents/comparator.md +4 -5
- package/dist/skills/agentv-bench/agents/grader.md +4 -4
- package/dist/skills/agentv-bench/references/autoresearch.md +4 -4
- package/dist/skills/agentv-bench/references/description-optimization.md +2 -2
- package/dist/skills/agentv-bench/references/environment-adaptation.md +10 -10
- package/dist/skills/agentv-bench/references/eval-yaml-spec.md +52 -26
- package/dist/skills/agentv-bench/references/migrating-from-skill-creator.md +6 -6
- package/dist/skills/agentv-bench/references/schemas.md +6 -6
- package/dist/skills/agentv-bench/references/subagent-pipeline.md +14 -13
- package/dist/skills/agentv-eval-migrations/SKILL.md +33 -0
- package/dist/skills/agentv-eval-migrations/references/breaking-changes.md +1013 -0
- package/dist/skills/agentv-eval-review/SKILL.md +6 -3
- package/dist/skills/agentv-eval-review/scripts/lint_eval.py +1 -1
- package/dist/skills/agentv-eval-writer/SKILL.md +329 -200
- package/dist/skills/agentv-eval-writer/references/{config-schema.json → config.schema.json} +6 -5
- package/dist/skills/agentv-eval-writer/references/custom-evaluators.md +23 -18
- package/dist/skills/agentv-eval-writer/references/eval.schema.json +10447 -0
- package/dist/skills/agentv-eval-writer/references/python-helpers.md +8 -8
- package/dist/skills/agentv-eval-writer/references/rubric-evaluator.md +21 -22
- package/dist/skills/agentv-trace-analyst/SKILL.md +4 -4
- package/dist/templates/.agentv/targets.yaml +41 -40
- package/dist/{ts-eval-loader-S7ESROX2-6CB3HB6H.js → ts-eval-loader-SSPJVEHD-AL2A3V5Z.js} +4 -4
- package/package.json +13 -4
- package/dist/chunk-2N3YSEMI.js.map +0 -1
- package/dist/chunk-5DYGN547.js.map +0 -1
- package/dist/chunk-76FOHROU.js +0 -96
- package/dist/chunk-76FOHROU.js.map +0 -1
- package/dist/chunk-CXHYTYHU.js.map +0 -1
- package/dist/chunk-HSHTTN6C.js +0 -565
- package/dist/chunk-HSHTTN6C.js.map +0 -1
- package/dist/chunk-KAGZRUO6.js.map +0 -1
- package/dist/chunk-LRULMAAA.js +0 -1711
- package/dist/chunk-LRULMAAA.js.map +0 -1
- package/dist/chunk-NCNDA3F6.js.map +0 -1
- package/dist/chunk-NPVGBFF6.js.map +0 -1
- package/dist/chunk-R3CXYNTN.js +0 -1238
- package/dist/chunk-R3CXYNTN.js.map +0 -1
- package/dist/chunk-VQ2ZO7XJ.js +0 -2098
- package/dist/chunk-VQ2ZO7XJ.js.map +0 -1
- package/dist/chunk-XALGXSKB.js +0 -21
- package/dist/chunk-XALGXSKB.js.map +0 -1
- package/dist/dashboard/assets/index-1Hvy8sBy.js +0 -119
- package/dist/dashboard/assets/index-J5sPcQO3.css +0 -1
- package/dist/esm-NNMQYIKT.js +0 -32
- package/dist/esm-R77SNOF5.js +0 -65
- package/dist/esm-RVQPUGWH.js +0 -1207
- package/dist/esm-RVQPUGWH.js.map +0 -1
- package/dist/esm-TTEMJEVV.js +0 -933
- package/dist/esm-TTEMJEVV.js.map +0 -1
- package/dist/esm-ZADQ4XQH-QUFGGZL4.js +0 -924
- package/dist/esm-ZADQ4XQH-QUFGGZL4.js.map +0 -1
- package/dist/exec-AR6JUUN5-6MBPURPR.js.map +0 -1
- package/dist/getMachineId-bsd-HSK5LZMG.js +0 -41
- package/dist/getMachineId-bsd-HSK5LZMG.js.map +0 -1
- package/dist/getMachineId-darwin-4DP6CCJV.js +0 -41
- package/dist/getMachineId-darwin-4DP6CCJV.js.map +0 -1
- package/dist/getMachineId-linux-44LJ5UJB.js +0 -33
- package/dist/getMachineId-linux-44LJ5UJB.js.map +0 -1
- package/dist/getMachineId-unsupported-NVK6IATM.js +0 -24
- package/dist/getMachineId-unsupported-NVK6IATM.js.map +0 -1
- package/dist/getMachineId-win-YZ36S7VA.js +0 -43
- package/dist/getMachineId-win-YZ36S7VA.js.map +0 -1
- package/dist/interactive-DRNRKLEQ.js.map +0 -1
- package/dist/otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js +0 -9
- package/dist/otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js.map +0 -1
- package/dist/skills/agentv-eval-writer/references/eval-schema.json +0 -17355
- package/dist/src-76ISS3LH.js +0 -1733
- package/dist/src-76ISS3LH.js.map +0 -1
- package/dist/ts-eval-loader-S7ESROX2-6CB3HB6H.js.map +0 -1
- /package/dist/{agentv-provider-AYXH7WLW-NJRC6UQX.js.map → agentv-provider-AYXH7WLW-6YZKZ7PS.js.map} +0 -0
- /package/dist/{artifact-writer-72MQQELS.js.map → artifact-writer-IORG7CW5.js.map} +0 -0
- /package/dist/{chunk-BPGJ4HBU.js.map → chunk-PEUTJS7B.js.map} +0 -0
- /package/dist/{chunk-5H446C7X.js.map → chunk-QMRVH5ZP.js.map} +0 -0
- /package/dist/{dist-L4YCQOQ6.js.map → dist-2D2GBHZV.js.map} +0 -0
- /package/dist/{docker-workspace-RPPXBT27-B4AQHVWA.js.map → docker-workspace-NNK4Z6FR-QRTIG6TM.js.map} +0 -0
- /package/dist/{esm-NNMQYIKT.js.map → exec-4HFFRQ3B-AXDMJNLJ.js.map} +0 -0
- /package/dist/{esm-R77SNOF5.js.map → ts-eval-loader-SSPJVEHD-AL2A3V5Z.js.map} +0 -0
|
@@ -21,7 +21,7 @@ import {
|
|
|
21
21
|
readTranscriptJsonl,
|
|
22
22
|
releaseSharedWorkspaceSetup,
|
|
23
23
|
resolveRepoCloneUrl
|
|
24
|
-
} from "./chunk-
|
|
24
|
+
} from "./chunk-ZGPCX334.js";
|
|
25
25
|
|
|
26
26
|
// ../../packages/core/dist/index.js
|
|
27
27
|
import { readFileSync } from "node:fs";
|
|
@@ -32,17 +32,18 @@ import micromatch from "micromatch";
|
|
|
32
32
|
import { readFile } from "node:fs/promises";
|
|
33
33
|
import path3 from "node:path";
|
|
34
34
|
import { execFile, spawn } from "node:child_process";
|
|
35
|
-
import { createHash } from "node:crypto";
|
|
35
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
36
36
|
import {
|
|
37
37
|
existsSync,
|
|
38
38
|
mkdirSync,
|
|
39
39
|
mkdtempSync,
|
|
40
40
|
readFileSync as readFileSync2,
|
|
41
|
+
realpathSync,
|
|
41
42
|
renameSync,
|
|
42
43
|
rmSync,
|
|
43
44
|
writeFileSync
|
|
44
45
|
} from "node:fs";
|
|
45
|
-
import { cp, lstat, mkdtemp, readdir, rm, stat } from "node:fs/promises";
|
|
46
|
+
import { cp, lstat, mkdtemp, readFile as readFile2, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
46
47
|
import os from "node:os";
|
|
47
48
|
import path4 from "node:path";
|
|
48
49
|
import { promisify } from "node:util";
|
|
@@ -1478,7 +1479,7 @@ function isPrimitive(value) {
|
|
|
1478
1479
|
function dropUndefined2(value) {
|
|
1479
1480
|
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
|
|
1480
1481
|
}
|
|
1481
|
-
function
|
|
1482
|
+
function scriptGraderInstruction(graderName, description) {
|
|
1482
1483
|
const desc = description ? ` This grader: ${description}.` : "";
|
|
1483
1484
|
return `Run \`agentv eval assert ${graderName} --agent-output <agent_output> --agent-input <original_prompt>\` and check the result.${desc} The command accepts --agent-output (the agent's full response text) and --agent-input (the original user prompt). It returns JSON on stdout: {"score": 0-1, "reasoning": "..."}. A score >= 0.5 means pass (exit 0); below 0.5 means fail (exit 1).`;
|
|
1484
1485
|
}
|
|
@@ -1496,21 +1497,15 @@ function assertionToNaturalLanguage(entry) {
|
|
|
1496
1497
|
switch (type) {
|
|
1497
1498
|
case "skill-trigger":
|
|
1498
1499
|
return null;
|
|
1499
|
-
case "
|
|
1500
|
-
|
|
1501
|
-
return entry.criteria;
|
|
1502
|
-
}
|
|
1503
|
-
return null;
|
|
1504
|
-
}
|
|
1500
|
+
case "llm-rubric":
|
|
1501
|
+
return typeof entry.value === "string" ? entry.value : null;
|
|
1505
1502
|
case "contains":
|
|
1506
1503
|
return `Output contains '${entry.value}'`;
|
|
1507
|
-
case "contains-any":
|
|
1508
|
-
case "contains_any": {
|
|
1504
|
+
case "contains-any": {
|
|
1509
1505
|
const values = Array.isArray(entry.value) ? entry.value.join("', '") : entry.value;
|
|
1510
1506
|
return `Output contains any of: '${values}'`;
|
|
1511
1507
|
}
|
|
1512
|
-
case "contains-all":
|
|
1513
|
-
case "contains_all": {
|
|
1508
|
+
case "contains-all": {
|
|
1514
1509
|
const values = Array.isArray(entry.value) ? entry.value.join("', '") : entry.value;
|
|
1515
1510
|
return `Output contains all of: '${values}'`;
|
|
1516
1511
|
}
|
|
@@ -1521,35 +1516,27 @@ function assertionToNaturalLanguage(entry) {
|
|
|
1521
1516
|
case "equals":
|
|
1522
1517
|
return `Output exactly equals: ${entry.value}`;
|
|
1523
1518
|
case "is-json":
|
|
1524
|
-
case "is_json":
|
|
1525
1519
|
return "Output is valid JSON";
|
|
1526
1520
|
case "starts-with":
|
|
1527
|
-
case "starts_with":
|
|
1528
1521
|
return `Output starts with '${entry.value}'`;
|
|
1529
1522
|
case "ends-with":
|
|
1530
|
-
case "ends_with":
|
|
1531
1523
|
return `Output ends with '${entry.value}'`;
|
|
1532
1524
|
case "llm-grader":
|
|
1533
|
-
case "llm_grader": {
|
|
1534
1525
|
if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
|
|
1535
1526
|
return null;
|
|
1536
1527
|
}
|
|
1537
1528
|
return typeof entry.prompt === "string" ? entry.prompt : null;
|
|
1538
|
-
|
|
1539
|
-
case "tool-trajectory":
|
|
1540
|
-
case "tool_trajectory": {
|
|
1529
|
+
case "tool-trajectory": {
|
|
1541
1530
|
const expectedArr = Array.isArray(entry.expected) ? entry.expected : [];
|
|
1542
1531
|
const tools = expectedArr.map((e) => e.tool).filter(Boolean).join(", ");
|
|
1543
1532
|
return tools ? `Agent called tools in order: ${tools}` : "Agent followed expected tool trajectory";
|
|
1544
1533
|
}
|
|
1545
|
-
case "
|
|
1546
|
-
|
|
1547
|
-
const graderName = entry.name ?? deriveGraderNameFromCommand(entry.command) ?? "code-grader";
|
|
1534
|
+
case "script": {
|
|
1535
|
+
const graderName = entry.metric ?? deriveGraderNameFromCommand(entry.command) ?? "script";
|
|
1548
1536
|
const desc = typeof entry.description === "string" ? entry.description : void 0;
|
|
1549
|
-
return
|
|
1537
|
+
return scriptGraderInstruction(graderName, desc);
|
|
1550
1538
|
}
|
|
1551
|
-
case "field-accuracy":
|
|
1552
|
-
case "field_accuracy": {
|
|
1539
|
+
case "field-accuracy": {
|
|
1553
1540
|
const fieldPaths = Array.isArray(entry.fields) ? entry.fields.map((f) => f.path).filter(Boolean).join(", ") : "";
|
|
1554
1541
|
return fieldPaths ? `Fields ${fieldPaths} match expected values` : "Fields match expected values";
|
|
1555
1542
|
}
|
|
@@ -1558,14 +1545,12 @@ function assertionToNaturalLanguage(entry) {
|
|
|
1558
1545
|
case "cost":
|
|
1559
1546
|
return typeof entry.budget === "number" ? `Cost under $${entry.budget}` : "Cost within budget";
|
|
1560
1547
|
case "token-usage":
|
|
1561
|
-
case "token_usage":
|
|
1562
1548
|
return "Token usage within limits";
|
|
1563
1549
|
case "execution-metrics":
|
|
1564
|
-
case "execution_metrics":
|
|
1565
1550
|
return "Execution within metric bounds";
|
|
1566
1551
|
default: {
|
|
1567
1552
|
if (entry.command !== void 0 && type) {
|
|
1568
|
-
return
|
|
1553
|
+
return scriptGraderInstruction(deriveGraderNameFromCommand(entry.command) ?? type);
|
|
1569
1554
|
}
|
|
1570
1555
|
if (typeof entry.criteria === "string") return entry.criteria;
|
|
1571
1556
|
if (typeof entry.prompt === "string") return entry.prompt;
|
|
@@ -1574,7 +1559,17 @@ function assertionToNaturalLanguage(entry) {
|
|
|
1574
1559
|
}
|
|
1575
1560
|
}
|
|
1576
1561
|
function assertionToNaturalLanguageList(entry) {
|
|
1577
|
-
if (entry.type === "llm-
|
|
1562
|
+
if (entry.type === "llm-rubric") {
|
|
1563
|
+
if (Array.isArray(entry.value) && entry.value.length > 0) {
|
|
1564
|
+
return entry.value.map((rubric) => {
|
|
1565
|
+
if (typeof rubric === "string") return rubric;
|
|
1566
|
+
if (!rubric || typeof rubric !== "object") return void 0;
|
|
1567
|
+
const item = rubric;
|
|
1568
|
+
return item.outcome ?? item.criteria ?? item.id;
|
|
1569
|
+
}).filter((value) => typeof value === "string");
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
if (entry.type === "llm-grader") {
|
|
1578
1573
|
if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
|
|
1579
1574
|
return entry.rubrics.map((r) => r.outcome ?? r.criteria ?? r.id).filter((s) => typeof s === "string");
|
|
1580
1575
|
}
|
|
@@ -1585,16 +1580,6 @@ function assertionToNaturalLanguageList(entry) {
|
|
|
1585
1580
|
function extractTriggerAssertions(assertions) {
|
|
1586
1581
|
return assertions.filter((a) => a.type === "skill-trigger");
|
|
1587
1582
|
}
|
|
1588
|
-
function resolveAssertions(rawCase) {
|
|
1589
|
-
if (Array.isArray(rawCase.assertions)) return rawCase.assertions;
|
|
1590
|
-
if (Array.isArray(rawCase.assert)) return rawCase.assert;
|
|
1591
|
-
return [];
|
|
1592
|
-
}
|
|
1593
|
-
function resolveSuiteAssertions(suite) {
|
|
1594
|
-
if (Array.isArray(suite.assertions)) return suite.assertions;
|
|
1595
|
-
if (Array.isArray(suite.assert)) return suite.assert;
|
|
1596
|
-
return [];
|
|
1597
|
-
}
|
|
1598
1583
|
function extractInput(rawCase) {
|
|
1599
1584
|
const files = Array.isArray(rawCase.input_files) ? rawCase.input_files.filter((f) => typeof f === "string") : [];
|
|
1600
1585
|
const input = rawCase.input;
|
|
@@ -1632,7 +1617,6 @@ function extractExpectedOutput(raw) {
|
|
|
1632
1617
|
return JSON.stringify(raw);
|
|
1633
1618
|
}
|
|
1634
1619
|
function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
1635
|
-
const warnings = [];
|
|
1636
1620
|
const files = /* @__PURE__ */ new Map();
|
|
1637
1621
|
if (typeof suite !== "object" || suite === null) {
|
|
1638
1622
|
throw new Error(`Invalid EVAL.yaml: expected an object in '${source}'`);
|
|
@@ -1641,10 +1625,7 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
|
1641
1625
|
if (!Array.isArray(rawSuite.tests)) {
|
|
1642
1626
|
throw new Error(`Invalid EVAL.yaml: missing 'tests' array in '${source}'`);
|
|
1643
1627
|
}
|
|
1644
|
-
|
|
1645
|
-
warnings.push("'assert' is deprecated at the suite level. Use 'assertions' instead.");
|
|
1646
|
-
}
|
|
1647
|
-
const suiteAssertions = resolveSuiteAssertions(rawSuite);
|
|
1628
|
+
const suiteAssertions = rawSuite.assert ?? [];
|
|
1648
1629
|
const suiteNlAssertions = suiteAssertions.filter((a) => a.type !== "skill-trigger").flatMap(assertionToNaturalLanguageList);
|
|
1649
1630
|
function getSkillFile(skillName) {
|
|
1650
1631
|
const existing = files.get(skillName);
|
|
@@ -1656,10 +1637,11 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
|
1656
1637
|
const tests = rawSuite.tests;
|
|
1657
1638
|
for (let idx = 0; idx < tests.length; idx++) {
|
|
1658
1639
|
const rawCase = tests[idx];
|
|
1659
|
-
const caseAssertions =
|
|
1660
|
-
if (rawCase.
|
|
1661
|
-
|
|
1662
|
-
|
|
1640
|
+
const caseAssertions = rawCase.assert ?? [];
|
|
1641
|
+
if (typeof rawCase.criteria === "string" && rawCase.criteria.trim() && rawCase.assert !== void 0) {
|
|
1642
|
+
throw new Error(
|
|
1643
|
+
`Invalid EVAL.yaml test '${rawCase.id ?? idx + 1}' in '${source}': do not combine test-level 'criteria' with 'assert'. Put human-readable case descriptions in 'description', or express grading text as an explicit assertion such as { type: 'llm-rubric', value: ... }.`
|
|
1644
|
+
);
|
|
1663
1645
|
}
|
|
1664
1646
|
const nlAssertions = [];
|
|
1665
1647
|
if (typeof rawCase.criteria === "string" && rawCase.criteria.trim()) {
|
|
@@ -1712,7 +1694,7 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
|
1712
1694
|
files.delete("_no-skill");
|
|
1713
1695
|
}
|
|
1714
1696
|
}
|
|
1715
|
-
return { files, warnings };
|
|
1697
|
+
return { files, warnings: [] };
|
|
1716
1698
|
}
|
|
1717
1699
|
function transpileEvalYamlFile(evalYamlPath) {
|
|
1718
1700
|
const content = readFileSync(evalYamlPath, "utf8");
|
|
@@ -1754,33 +1736,10 @@ function toRepoPins(repos) {
|
|
|
1754
1736
|
...repo.path !== void 0 && { path: repo.path },
|
|
1755
1737
|
...repo.repo !== void 0 && { repo: repo.repo },
|
|
1756
1738
|
...repo.commit !== void 0 && { commit: repo.commit },
|
|
1757
|
-
...repo.base_commit !== void 0 && { baseCommit: repo.base_commit },
|
|
1758
1739
|
...repo.ancestor !== void 0 && { ancestor: repo.ancestor },
|
|
1759
1740
|
...repo.sparse !== void 0 && { sparse: repo.sparse }
|
|
1760
1741
|
}));
|
|
1761
1742
|
}
|
|
1762
|
-
function poolMetadata(setup, selectedSlotPath) {
|
|
1763
|
-
const slot = setup.poolSlot ?? setup.poolSlots.find((candidate) => candidate.path === selectedSlotPath);
|
|
1764
|
-
if (!slot) {
|
|
1765
|
-
return void 0;
|
|
1766
|
-
}
|
|
1767
|
-
return {
|
|
1768
|
-
fingerprint: slot.fingerprint,
|
|
1769
|
-
slotIndex: slot.index,
|
|
1770
|
-
lockPath: slot.lockPath
|
|
1771
|
-
};
|
|
1772
|
-
}
|
|
1773
|
-
async function releaseUnselectedPoolSlots(setup, selectedSlotPath) {
|
|
1774
|
-
if (!setup.poolManager) {
|
|
1775
|
-
return;
|
|
1776
|
-
}
|
|
1777
|
-
for (const slot of setup.poolSlots) {
|
|
1778
|
-
if (slot.path !== selectedSlotPath) {
|
|
1779
|
-
await setup.poolManager.releaseSlot(slot).catch(() => {
|
|
1780
|
-
});
|
|
1781
|
-
}
|
|
1782
|
-
}
|
|
1783
|
-
}
|
|
1784
1743
|
async function prepareEvalWorkspace(options) {
|
|
1785
1744
|
const evalPath = path2.resolve(options.testFilePath);
|
|
1786
1745
|
const evalRunId = randomUUID();
|
|
@@ -1809,15 +1768,11 @@ async function prepareEvalWorkspace(options) {
|
|
|
1809
1768
|
evalDir,
|
|
1810
1769
|
verbose: options.verbose,
|
|
1811
1770
|
workers,
|
|
1812
|
-
poolMaxSlots: options.poolMaxSlots,
|
|
1813
1771
|
workspacePath: options.workspacePath,
|
|
1814
|
-
legacyWorkspacePath: options.workspace
|
|
1815
|
-
workspaceMode: options.workspaceMode,
|
|
1816
|
-
workspaceClean: options.workspaceClean
|
|
1772
|
+
legacyWorkspacePath: options.workspace
|
|
1817
1773
|
});
|
|
1818
|
-
const
|
|
1819
|
-
const
|
|
1820
|
-
const selectedBaselineCommit = testPoolSlot ? sharedSetup.poolSlotBaselines.get(testPoolSlot.path) : sharedSetup.sharedBaselineCommit;
|
|
1774
|
+
const selectedWorkspacePath = sharedSetup.sharedWorkspacePath;
|
|
1775
|
+
const selectedBaselineCommit = sharedSetup.sharedBaselineCommit;
|
|
1821
1776
|
const caseSetup = await prepareEvalCaseWorkspace({
|
|
1822
1777
|
evalCase,
|
|
1823
1778
|
targetName: options.target.name,
|
|
@@ -1829,6 +1784,7 @@ async function prepareEvalWorkspace(options) {
|
|
|
1829
1784
|
evalDir,
|
|
1830
1785
|
cleanupWorkspaces: options.cleanupWorkspaces,
|
|
1831
1786
|
targetHooks: options.targetHooks,
|
|
1787
|
+
sharedExtensionState: sharedSetup.extensionState,
|
|
1832
1788
|
setupDebug: options.verbose
|
|
1833
1789
|
});
|
|
1834
1790
|
if (!caseSetup.workspacePath) {
|
|
@@ -1836,8 +1792,6 @@ async function prepareEvalWorkspace(options) {
|
|
|
1836
1792
|
`No workspace was materialized for test "${evalCase.id}". Add workspace.template, workspace.repos, or workspace.hooks before preparing an external attempt.`
|
|
1837
1793
|
);
|
|
1838
1794
|
}
|
|
1839
|
-
await releaseUnselectedPoolSlots(sharedSetup, caseSetup.workspacePath);
|
|
1840
|
-
const pool = poolMetadata(sharedSetup, caseSetup.workspacePath);
|
|
1841
1795
|
return {
|
|
1842
1796
|
evalPath,
|
|
1843
1797
|
testId: evalCase.id,
|
|
@@ -1849,6 +1803,12 @@ async function prepareEvalWorkspace(options) {
|
|
|
1849
1803
|
},
|
|
1850
1804
|
createdAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
|
|
1851
1805
|
hookExecutions: [...sharedSetup.hookExecutions, ...caseSetup.hookExecutions],
|
|
1806
|
+
...caseSetup.extensionState?.providerContext !== void 0 && {
|
|
1807
|
+
providerContext: caseSetup.extensionState.providerContext
|
|
1808
|
+
},
|
|
1809
|
+
...caseSetup.extensionState?.metadata !== void 0 && {
|
|
1810
|
+
metadata: caseSetup.extensionState.metadata
|
|
1811
|
+
},
|
|
1852
1812
|
repoPins: toRepoPins(evalCase.workspace?.repos),
|
|
1853
1813
|
baseline: caseSetup.baselineCommit ? { status: "initialized", commit: caseSetup.baselineCommit } : { status: "unavailable" },
|
|
1854
1814
|
promptSource: {
|
|
@@ -1866,8 +1826,7 @@ async function prepareEvalWorkspace(options) {
|
|
|
1866
1826
|
retainOnFailure,
|
|
1867
1827
|
manualCleanup: true
|
|
1868
1828
|
},
|
|
1869
|
-
sharedWorkspace: caseSetup.isSharedWorkspace
|
|
1870
|
-
...pool !== void 0 && { pool }
|
|
1829
|
+
sharedWorkspace: caseSetup.isSharedWorkspace
|
|
1871
1830
|
};
|
|
1872
1831
|
} catch (error) {
|
|
1873
1832
|
if (sharedSetup) {
|
|
@@ -1877,22 +1836,51 @@ async function prepareEvalWorkspace(options) {
|
|
|
1877
1836
|
throw error;
|
|
1878
1837
|
}
|
|
1879
1838
|
}
|
|
1839
|
+
var ExecutionConfigSchema = external_exports.object({
|
|
1840
|
+
/** General eval parallelism (default: 3) */
|
|
1841
|
+
maxConcurrency: external_exports.number().int().min(1).max(50).optional(),
|
|
1842
|
+
/** Maximum retries on failure (default: 2) */
|
|
1843
|
+
maxRetries: external_exports.number().int().min(0).optional(),
|
|
1844
|
+
/** Agent timeout in milliseconds. No timeout if not set. */
|
|
1845
|
+
agentTimeoutMs: external_exports.number().int().min(0).optional(),
|
|
1846
|
+
/** Enable verbose logging */
|
|
1847
|
+
verbose: external_exports.boolean().optional(),
|
|
1848
|
+
/** Always keep temp workspaces after eval */
|
|
1849
|
+
keepWorkspaces: external_exports.boolean().optional()
|
|
1850
|
+
}).passthrough().superRefine((value, ctx) => {
|
|
1851
|
+
const supportedFields = /* @__PURE__ */ new Set([
|
|
1852
|
+
"maxConcurrency",
|
|
1853
|
+
"maxRetries",
|
|
1854
|
+
"agentTimeoutMs",
|
|
1855
|
+
"verbose",
|
|
1856
|
+
"keepWorkspaces"
|
|
1857
|
+
]);
|
|
1858
|
+
if ("otelFile" in value) {
|
|
1859
|
+
ctx.addIssue({
|
|
1860
|
+
code: external_exports.ZodIssueCode.custom,
|
|
1861
|
+
path: ["otelFile"],
|
|
1862
|
+
message: "execution.otelFile has been removed. Emit OpenTelemetry/OpenInference traces from the system under test or provider and correlate AgentV run artifacts with external_trace metadata."
|
|
1863
|
+
});
|
|
1864
|
+
}
|
|
1865
|
+
for (const key of Object.keys(value)) {
|
|
1866
|
+
if (!supportedFields.has(key) && key !== "otelFile") {
|
|
1867
|
+
ctx.addIssue({
|
|
1868
|
+
code: external_exports.ZodIssueCode.custom,
|
|
1869
|
+
path: [key],
|
|
1870
|
+
message: `Unsupported execution field '${key}'`
|
|
1871
|
+
});
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
}).transform(({ maxConcurrency, maxRetries, agentTimeoutMs, verbose, keepWorkspaces }) => ({
|
|
1875
|
+
...maxConcurrency !== void 0 && { maxConcurrency },
|
|
1876
|
+
...maxRetries !== void 0 && { maxRetries },
|
|
1877
|
+
...agentTimeoutMs !== void 0 && { agentTimeoutMs },
|
|
1878
|
+
...verbose !== void 0 && { verbose },
|
|
1879
|
+
...keepWorkspaces !== void 0 && { keepWorkspaces }
|
|
1880
|
+
}));
|
|
1880
1881
|
var AgentVConfigSchema = external_exports.object({
|
|
1881
1882
|
/** Default execution settings */
|
|
1882
|
-
execution:
|
|
1883
|
-
/** Number of parallel workers (default: 3) */
|
|
1884
|
-
workers: external_exports.number().int().min(1).max(50).optional(),
|
|
1885
|
-
/** Maximum retries on failure (default: 2) */
|
|
1886
|
-
maxRetries: external_exports.number().int().min(0).optional(),
|
|
1887
|
-
/** Agent timeout in milliseconds. No timeout if not set. */
|
|
1888
|
-
agentTimeoutMs: external_exports.number().int().min(0).optional(),
|
|
1889
|
-
/** Enable verbose logging */
|
|
1890
|
-
verbose: external_exports.boolean().optional(),
|
|
1891
|
-
/** Always keep temp workspaces after eval */
|
|
1892
|
-
keepWorkspaces: external_exports.boolean().optional(),
|
|
1893
|
-
/** Write OTLP JSON trace to this path (supports {timestamp} placeholder) */
|
|
1894
|
-
otelFile: external_exports.string().optional()
|
|
1895
|
-
}).optional(),
|
|
1883
|
+
execution: ExecutionConfigSchema.optional(),
|
|
1896
1884
|
/** Output settings */
|
|
1897
1885
|
output: external_exports.object({
|
|
1898
1886
|
/** Default eval run artifact directory */
|
|
@@ -1922,7 +1910,9 @@ var AgentVConfigSchema = external_exports.object({
|
|
|
1922
1910
|
* stderr is forwarded to the user. Non-zero exit aborts with an error.
|
|
1923
1911
|
*/
|
|
1924
1912
|
beforeSession: external_exports.string().optional()
|
|
1925
|
-
}).optional()
|
|
1913
|
+
}).optional(),
|
|
1914
|
+
/** Project-local reusable references for fields that support ref://name */
|
|
1915
|
+
refs: external_exports.record(external_exports.string().min(1)).optional()
|
|
1926
1916
|
});
|
|
1927
1917
|
function defineConfig(config) {
|
|
1928
1918
|
return AgentVConfigSchema.parse(config);
|
|
@@ -2106,9 +2096,12 @@ function extractReposFromObject(obj) {
|
|
|
2106
2096
|
}
|
|
2107
2097
|
var execFileAsync = promisify(execFile);
|
|
2108
2098
|
var RESULTS_REPO_RESULTS_DIR = ".agentv/results";
|
|
2109
|
-
var RESULTS_REPO_RUNS_DIR = "
|
|
2110
|
-
var
|
|
2111
|
-
var
|
|
2099
|
+
var RESULTS_REPO_RUNS_DIR = ".";
|
|
2100
|
+
var RESULTS_REPO_INDEXES_DIR = ".indexes";
|
|
2101
|
+
var RESULTS_REPO_CACHE_DIR = ".cache";
|
|
2102
|
+
var RESULTS_REPO_TRACKED_DOT_DIRS = [RESULTS_REPO_INDEXES_DIR, RESULTS_REPO_CACHE_DIR];
|
|
2103
|
+
var GIT_RESULTS_INDEX_CACHE_SCHEMA_VERSION = "agentv.git_results_index_cache.v1";
|
|
2104
|
+
var GIT_RESULTS_INDEX_LAYOUT_VERSION = "agentv.results_repo_branch_root.v1";
|
|
2112
2105
|
var FALLBACK_RESULTS_REPO_COMMIT_EMAIL = "agentv@results-repo";
|
|
2113
2106
|
var FALLBACK_RESULTS_REPO_COMMIT_NAME = "AgentV Results";
|
|
2114
2107
|
var GIT_COMMIT_IDENTITY_ENV_KEYS = [
|
|
@@ -2125,11 +2118,16 @@ var GIT_ENV_INHERIT_ALLOWLIST = /* @__PURE__ */ new Set([
|
|
|
2125
2118
|
"GIT_USERNAME"
|
|
2126
2119
|
]);
|
|
2127
2120
|
var DEFAULT_RESULTS_BRANCH = AGENTV_RESULTS_PRIMARY_REF;
|
|
2128
|
-
var MANAGED_RESULTS_REMOTE = "agentv-results";
|
|
2129
2121
|
var GIT_EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
|
|
2130
2122
|
var RESULTS_REPO_GENESIS_MESSAGE = "chore(results): initialize AgentV results branch";
|
|
2131
2123
|
var RESULTS_REPO_GENESIS_DATE = "@0 +0000";
|
|
2132
2124
|
var RESULT_INDEX_FILENAME2 = "index.jsonl";
|
|
2125
|
+
var RUN_INTERNAL_DIRNAME = ".internal";
|
|
2126
|
+
var RESULTS_REPO_GITATTRIBUTES_FILE = ".gitattributes";
|
|
2127
|
+
var RESULTS_REPO_GITATTRIBUTES_CONTENT = `# Managed by AgentV. Artifact-aware merge so results sync never force-pushes.
|
|
2128
|
+
${RESULTS_REPO_INDEXES_DIR}/*.jsonl merge=union
|
|
2129
|
+
*/.internal/index.jsonl merge=union
|
|
2130
|
+
`;
|
|
2133
2131
|
var activeResultsRepoSyncs = /* @__PURE__ */ new Set();
|
|
2134
2132
|
function sanitizeRepoSlug(repo) {
|
|
2135
2133
|
return repo.trim().replace(/[^A-Za-z0-9._-]+/g, "-");
|
|
@@ -2152,18 +2150,24 @@ function resolveLocalPath(p, baseDir) {
|
|
|
2152
2150
|
const expanded = expandHome(p);
|
|
2153
2151
|
return path4.isAbsolute(expanded) ? expanded : path4.resolve(baseDir, expanded);
|
|
2154
2152
|
}
|
|
2153
|
+
function isNormalizedResultsConfig(config) {
|
|
2154
|
+
return typeof config.storageBranchWorktree === "boolean";
|
|
2155
|
+
}
|
|
2155
2156
|
function normalizeResultsConfig(config, options) {
|
|
2157
|
+
if (isNormalizedResultsConfig(config)) {
|
|
2158
|
+
return config;
|
|
2159
|
+
}
|
|
2156
2160
|
const baseDir = options?.baseDir ?? process.cwd();
|
|
2157
2161
|
const repoUrl = (config.repo_url ?? config.repo)?.trim();
|
|
2158
|
-
const
|
|
2159
|
-
const
|
|
2162
|
+
const explicitPath = config.path?.trim();
|
|
2163
|
+
const repoPath = config.repo_path?.trim() ?? (repoUrl ? void 0 : explicitPath);
|
|
2164
|
+
const explicitClonePath = repoUrl ? explicitPath : void 0;
|
|
2160
2165
|
const repo = repoUrl ?? repoPath ?? "";
|
|
2161
2166
|
const branch = config.branch?.trim() || (repoPath ? DEFAULT_RESULTS_BRANCH : void 0);
|
|
2162
2167
|
const useStorageBranchWorktree = Boolean(repoPath || repoUrl && explicitClonePath && branch);
|
|
2163
|
-
const remote = config.remote?.trim() ||
|
|
2164
|
-
const autoPush = config.
|
|
2165
|
-
const requirePush = config.
|
|
2166
|
-
const pushConflictPolicy = config.sync?.push_conflict_policy ?? "block";
|
|
2168
|
+
const remote = config.remote?.trim() || "origin";
|
|
2169
|
+
const autoPush = config.auto_push === true;
|
|
2170
|
+
const requirePush = config.require_push === true;
|
|
2167
2171
|
const resolvedRepoPath = repoPath ? resolveLocalPath(repoPath, baseDir) : void 0;
|
|
2168
2172
|
const resolvedPath = explicitClonePath ? resolveLocalPath(explicitClonePath, baseDir) : repoUrl ? path4.join(getAgentvDataDir(), "results", sanitizeRepoSlug(repoUrl)) : resolvedRepoPath ?? path4.join(getAgentvDataDir(), "results", sanitizeRepoSlug(repo));
|
|
2169
2173
|
return {
|
|
@@ -2176,16 +2180,23 @@ function normalizeResultsConfig(config, options) {
|
|
|
2176
2180
|
path: resolvedPath,
|
|
2177
2181
|
auto_push: autoPush,
|
|
2178
2182
|
require_push: requirePush,
|
|
2179
|
-
push_conflict_policy:
|
|
2180
|
-
branch_prefix: config.branch_prefix?.trim() || "eval-results",
|
|
2183
|
+
push_conflict_policy: "block",
|
|
2181
2184
|
storageBranchWorktree: useStorageBranchWorktree
|
|
2182
2185
|
};
|
|
2183
2186
|
}
|
|
2187
|
+
var GITHUB_OWNER_REPO_SHORTHAND = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/;
|
|
2188
|
+
function isExplicitRemoteUrl(value) {
|
|
2189
|
+
return value.includes("://") || value.startsWith("git@");
|
|
2190
|
+
}
|
|
2184
2191
|
function resolveResultsRepoUrl(repo) {
|
|
2185
|
-
|
|
2186
|
-
|
|
2192
|
+
const trimmed = repo.trim();
|
|
2193
|
+
if (isExplicitRemoteUrl(trimmed)) {
|
|
2194
|
+
return trimmed;
|
|
2187
2195
|
}
|
|
2188
|
-
|
|
2196
|
+
if (GITHUB_OWNER_REPO_SHORTHAND.test(trimmed)) {
|
|
2197
|
+
return `https://github.com/${trimmed}.git`;
|
|
2198
|
+
}
|
|
2199
|
+
return trimmed;
|
|
2189
2200
|
}
|
|
2190
2201
|
function getResultsRepoLocalPaths(repo) {
|
|
2191
2202
|
const rootDir = path4.join(getAgentvDataDir(), "cache", "results-repo", sanitizeRepoSlug(repo));
|
|
@@ -2447,26 +2458,79 @@ async function resolveGitTopLevel(repoDir) {
|
|
|
2447
2458
|
const { stdout } = await runGit(["rev-parse", "--show-toplevel"], { cwd: repoDir });
|
|
2448
2459
|
return stdout.trim() || repoDir;
|
|
2449
2460
|
}
|
|
2450
|
-
async function
|
|
2451
|
-
|
|
2461
|
+
async function ensureResultsMergeConfig(repoDir) {
|
|
2462
|
+
let gitDir;
|
|
2463
|
+
try {
|
|
2464
|
+
const { stdout } = await runGit(["rev-parse", "--git-common-dir"], { cwd: repoDir });
|
|
2465
|
+
const resolved = stdout.trim();
|
|
2466
|
+
gitDir = resolved ? path4.isAbsolute(resolved) ? resolved : path4.join(repoDir, resolved) : path4.join(repoDir, ".git");
|
|
2467
|
+
} catch {
|
|
2452
2468
|
return;
|
|
2453
2469
|
}
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
const
|
|
2460
|
-
|
|
2461
|
-
|
|
2470
|
+
writeResultsInfoAttributes(gitDir);
|
|
2471
|
+
}
|
|
2472
|
+
var RESULTS_INFO_ATTRIBUTES_BEGIN = "# >>> agentv results merge attributes >>>";
|
|
2473
|
+
var RESULTS_INFO_ATTRIBUTES_END = "# <<< agentv results merge attributes <<<";
|
|
2474
|
+
function writeResultsInfoAttributes(gitDir) {
|
|
2475
|
+
const infoDir = path4.join(gitDir, "info");
|
|
2476
|
+
const attributesPath = path4.join(infoDir, "attributes");
|
|
2477
|
+
const managedBlock = `${RESULTS_INFO_ATTRIBUTES_BEGIN}
|
|
2478
|
+
${RESULTS_REPO_GITATTRIBUTES_CONTENT.trimEnd()}
|
|
2479
|
+
${RESULTS_INFO_ATTRIBUTES_END}
|
|
2480
|
+
`;
|
|
2481
|
+
let existing = "";
|
|
2482
|
+
try {
|
|
2483
|
+
existing = readFileSync2(attributesPath, "utf8");
|
|
2484
|
+
} catch {
|
|
2485
|
+
existing = "";
|
|
2486
|
+
}
|
|
2487
|
+
const blockPattern = new RegExp(
|
|
2488
|
+
`${escapeRegExp(RESULTS_INFO_ATTRIBUTES_BEGIN)}[\\s\\S]*?${escapeRegExp(
|
|
2489
|
+
RESULTS_INFO_ATTRIBUTES_END
|
|
2490
|
+
)}\\n?`
|
|
2491
|
+
);
|
|
2492
|
+
const withoutManaged = existing.replace(blockPattern, "");
|
|
2493
|
+
const next = withoutManaged.length > 0 && !withoutManaged.endsWith("\n") ? `${withoutManaged}
|
|
2494
|
+
${managedBlock}` : `${withoutManaged}${managedBlock}`;
|
|
2495
|
+
if (next === existing) {
|
|
2496
|
+
return;
|
|
2497
|
+
}
|
|
2498
|
+
mkdirSync(infoDir, { recursive: true });
|
|
2499
|
+
writeFileSync(attributesPath, next);
|
|
2500
|
+
}
|
|
2501
|
+
function escapeRegExp(value) {
|
|
2502
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
2503
|
+
}
|
|
2504
|
+
async function addResultsGitattributesToIndex(repoDir, indexEnv) {
|
|
2505
|
+
const { stdout: existing } = await runGit(
|
|
2506
|
+
["cat-file", "-p", `:0:${RESULTS_REPO_GITATTRIBUTES_FILE}`],
|
|
2507
|
+
{ cwd: repoDir, env: indexEnv, check: false }
|
|
2508
|
+
);
|
|
2509
|
+
if (existing === RESULTS_REPO_GITATTRIBUTES_CONTENT) {
|
|
2462
2510
|
return;
|
|
2463
2511
|
}
|
|
2464
|
-
|
|
2465
|
-
|
|
2512
|
+
const scratchDir = mkdtempSync(path4.join(os.tmpdir(), "agentv-results-attrs-"));
|
|
2513
|
+
try {
|
|
2514
|
+
const scratchFile = path4.join(scratchDir, RESULTS_REPO_GITATTRIBUTES_FILE);
|
|
2515
|
+
writeFileSync(scratchFile, RESULTS_REPO_GITATTRIBUTES_CONTENT);
|
|
2516
|
+
const { stdout: blob } = await runGit(["hash-object", "-w", "--no-filters", scratchFile], {
|
|
2517
|
+
cwd: repoDir
|
|
2518
|
+
});
|
|
2519
|
+
await runGit(
|
|
2520
|
+
[
|
|
2521
|
+
"update-index",
|
|
2522
|
+
"--add",
|
|
2523
|
+
"--cacheinfo",
|
|
2524
|
+
`100644,${blob.trim()},${RESULTS_REPO_GITATTRIBUTES_FILE}`
|
|
2525
|
+
],
|
|
2526
|
+
{ cwd: repoDir, env: indexEnv }
|
|
2527
|
+
);
|
|
2528
|
+
} finally {
|
|
2529
|
+
rmSync(scratchDir, { recursive: true, force: true });
|
|
2466
2530
|
}
|
|
2467
2531
|
}
|
|
2468
2532
|
function updateStatusFile(config, patch) {
|
|
2469
|
-
const repo =
|
|
2533
|
+
const repo = normalizeResultsConfig(config).repo;
|
|
2470
2534
|
const cachePaths = getResultsRepoLocalPaths(repo);
|
|
2471
2535
|
const current = readPersistedStatus(cachePaths.statusFile);
|
|
2472
2536
|
writePersistedStatus(cachePaths.statusFile, {
|
|
@@ -2480,7 +2544,9 @@ async function ensureResultsRepoClone(config) {
|
|
|
2480
2544
|
if (!await isGitRepository(normalized.repo_path)) {
|
|
2481
2545
|
throw new Error(`Results repo_path is not a git repository: ${normalized.repo_path}`);
|
|
2482
2546
|
}
|
|
2483
|
-
|
|
2547
|
+
const topLevel = await resolveGitTopLevel(normalized.repo_path);
|
|
2548
|
+
await ensureResultsMergeConfig(topLevel);
|
|
2549
|
+
return topLevel;
|
|
2484
2550
|
}
|
|
2485
2551
|
const cachePaths = getResultsRepoLocalPaths(normalized.repo);
|
|
2486
2552
|
const cloneDir = normalized.path;
|
|
@@ -2494,10 +2560,11 @@ async function ensureResultsRepoClone(config) {
|
|
|
2494
2560
|
await runGit([
|
|
2495
2561
|
"clone",
|
|
2496
2562
|
"--filter=blob:none",
|
|
2563
|
+
...normalized.remote === "origin" ? [] : ["--origin", normalized.remote],
|
|
2497
2564
|
resolveResultsRepoUrl(normalized.repo_url ?? normalized.repo),
|
|
2498
2565
|
cloneDir
|
|
2499
2566
|
]);
|
|
2500
|
-
await
|
|
2567
|
+
await ensureResultsMergeConfig(cloneDir);
|
|
2501
2568
|
return cloneDir;
|
|
2502
2569
|
} catch (error) {
|
|
2503
2570
|
updateStatusFile(normalized, { last_error: withFriendlyGitHubAuthError(error).message });
|
|
@@ -2507,7 +2574,7 @@ async function ensureResultsRepoClone(config) {
|
|
|
2507
2574
|
if (!existsSync(gitDir)) {
|
|
2508
2575
|
throw new Error(`Results repo clone path is not a git repository: ${cloneDir}`);
|
|
2509
2576
|
}
|
|
2510
|
-
await
|
|
2577
|
+
await ensureResultsMergeConfig(cloneDir);
|
|
2511
2578
|
return cloneDir;
|
|
2512
2579
|
}
|
|
2513
2580
|
function getResultsRepoStatus(config) {
|
|
@@ -2532,7 +2599,6 @@ function getResultsRepoStatus(config) {
|
|
|
2532
2599
|
auto_push: normalized.auto_push,
|
|
2533
2600
|
require_push: normalized.require_push,
|
|
2534
2601
|
push_conflict_policy: normalized.push_conflict_policy,
|
|
2535
|
-
branch_prefix: normalized.branch_prefix,
|
|
2536
2602
|
local_dir: normalized.path,
|
|
2537
2603
|
last_synced_at: persisted.last_synced_at,
|
|
2538
2604
|
last_error: persisted.last_error,
|
|
@@ -2859,6 +2925,17 @@ var ResultsBranchPushConflictError = class extends Error {
|
|
|
2859
2925
|
this.name = "ResultsBranchPushConflictError";
|
|
2860
2926
|
}
|
|
2861
2927
|
};
|
|
2928
|
+
function pendingMergeToWire(details) {
|
|
2929
|
+
return {
|
|
2930
|
+
temp_branch: details.tempBranch,
|
|
2931
|
+
target_branch: details.targetBranch,
|
|
2932
|
+
...details.compareUrl !== void 0 && { compare_url: details.compareUrl },
|
|
2933
|
+
...details.contributedRunCount !== void 0 && {
|
|
2934
|
+
contributed_run_count: details.contributedRunCount
|
|
2935
|
+
},
|
|
2936
|
+
created_at: details.createdAt
|
|
2937
|
+
};
|
|
2938
|
+
}
|
|
2862
2939
|
function pushDetailsToWire(details) {
|
|
2863
2940
|
if (!details) {
|
|
2864
2941
|
return {};
|
|
@@ -2876,7 +2953,11 @@ function pushDetailsToWire(details) {
|
|
|
2876
2953
|
...details.forcePushedCommit !== void 0 && {
|
|
2877
2954
|
force_pushed_commit: details.forcePushedCommit
|
|
2878
2955
|
},
|
|
2879
|
-
...details.leaseCommit !== void 0 && { lease_commit: details.leaseCommit }
|
|
2956
|
+
...details.leaseCommit !== void 0 && { lease_commit: details.leaseCommit },
|
|
2957
|
+
...details.autoMergedRemote === true && { auto_merged_remote: true },
|
|
2958
|
+
...details.pendingMerge !== void 0 && {
|
|
2959
|
+
pending_merge: pendingMergeToWire(details.pendingMerge)
|
|
2960
|
+
}
|
|
2880
2961
|
};
|
|
2881
2962
|
}
|
|
2882
2963
|
function withBlockedStatus(status, blockReason, flags) {
|
|
@@ -2894,11 +2975,13 @@ function withPushConflictStatus(status, blockReason, details, flags) {
|
|
|
2894
2975
|
return withBlockedStatus(
|
|
2895
2976
|
{
|
|
2896
2977
|
...status,
|
|
2897
|
-
sync_status: "push_conflict"
|
|
2978
|
+
sync_status: flags.syncStatus ?? "push_conflict"
|
|
2898
2979
|
},
|
|
2899
2980
|
blockReason,
|
|
2900
2981
|
{
|
|
2901
|
-
|
|
2982
|
+
pullPerformed: flags.pullPerformed,
|
|
2983
|
+
pushPerformed: flags.pushPerformed,
|
|
2984
|
+
commitCreated: flags.commitCreated,
|
|
2902
2985
|
pushDetails: details
|
|
2903
2986
|
}
|
|
2904
2987
|
);
|
|
@@ -2914,21 +2997,48 @@ function withActionFlags(status, flags) {
|
|
|
2914
2997
|
};
|
|
2915
2998
|
}
|
|
2916
2999
|
function isSafeResultsRepoPath(p) {
|
|
2917
|
-
|
|
3000
|
+
const normalized = p.split(path4.sep).join("/");
|
|
3001
|
+
const segments = normalized.split("/").filter(Boolean);
|
|
3002
|
+
const isRunBundlePath = segments.length >= 2 && !segments[0]?.startsWith(".") && segments[0] !== RESULTS_REPO_GITATTRIBUTES_FILE;
|
|
3003
|
+
return normalized === RESULTS_REPO_GITATTRIBUTES_FILE || RESULTS_REPO_TRACKED_DOT_DIRS.some(
|
|
3004
|
+
(dir) => normalized === dir || normalized.startsWith(`${dir}/`)
|
|
3005
|
+
) || isRunBundlePath;
|
|
2918
3006
|
}
|
|
2919
|
-
async function
|
|
2920
|
-
const targets =
|
|
2921
|
-
|
|
3007
|
+
async function existingTrackedResultsPathspecs(repoDir) {
|
|
3008
|
+
const targets = /* @__PURE__ */ new Set();
|
|
3009
|
+
if (existsSync(path4.join(repoDir, RESULTS_REPO_GITATTRIBUTES_FILE))) {
|
|
3010
|
+
targets.add(RESULTS_REPO_GITATTRIBUTES_FILE);
|
|
3011
|
+
}
|
|
3012
|
+
for (const dir of RESULTS_REPO_TRACKED_DOT_DIRS) {
|
|
2922
3013
|
if (existsSync(path4.join(repoDir, dir))) {
|
|
2923
|
-
targets.
|
|
3014
|
+
targets.add(dir);
|
|
3015
|
+
continue;
|
|
3016
|
+
}
|
|
3017
|
+
const { stdout: stdout2 } = await runGit(["ls-files", "--", dir], { cwd: repoDir, check: false });
|
|
3018
|
+
if (stdout2.trim().length > 0) {
|
|
3019
|
+
targets.add(dir);
|
|
3020
|
+
}
|
|
3021
|
+
}
|
|
3022
|
+
for (const entry of await readdir(repoDir, { withFileTypes: true }).catch(() => [])) {
|
|
3023
|
+
if (entry.name.startsWith(".") || !entry.isDirectory()) {
|
|
3024
|
+
continue;
|
|
3025
|
+
}
|
|
3026
|
+
if (existsSync(path4.join(repoDir, entry.name, "summary.json"))) {
|
|
3027
|
+
targets.add(entry.name);
|
|
3028
|
+
}
|
|
3029
|
+
}
|
|
3030
|
+
const { stdout } = await runGit(["ls-files"], { cwd: repoDir, check: false });
|
|
3031
|
+
for (const file of stdout.split(/\r?\n/)) {
|
|
3032
|
+
const normalized = file.trim();
|
|
3033
|
+
if (!normalized) {
|
|
2924
3034
|
continue;
|
|
2925
3035
|
}
|
|
2926
|
-
const
|
|
2927
|
-
if (
|
|
2928
|
-
targets.
|
|
3036
|
+
const segments = normalized.split("/");
|
|
3037
|
+
if (segments.length >= 2 && segments[0] && !segments[0].startsWith(".")) {
|
|
3038
|
+
targets.add(segments[0]);
|
|
2929
3039
|
}
|
|
2930
3040
|
}
|
|
2931
|
-
return targets;
|
|
3041
|
+
return [...targets].sort();
|
|
2932
3042
|
}
|
|
2933
3043
|
function areSafeResultsRepoPaths(paths) {
|
|
2934
3044
|
return paths.length > 0 && paths.every(isSafeResultsRepoPath);
|
|
@@ -2947,19 +3057,8 @@ function getPushTargetBranch(upstream, baseBranch, remote = "origin") {
|
|
|
2947
3057
|
const prefix = `${remote}/`;
|
|
2948
3058
|
return upstream?.startsWith(prefix) ? upstream.slice(prefix.length) : baseBranch;
|
|
2949
3059
|
}
|
|
2950
|
-
function
|
|
2951
|
-
|
|
2952
|
-
return `${date.getUTCFullYear()}${pad(date.getUTCMonth() + 1)}${pad(date.getUTCDate())}T${pad(
|
|
2953
|
-
date.getUTCHours()
|
|
2954
|
-
)}${pad(date.getUTCMinutes())}${pad(date.getUTCSeconds())}Z`;
|
|
2955
|
-
}
|
|
2956
|
-
function slugifyBackupTargetBranch(branch) {
|
|
2957
|
-
return branch.trim().replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "results";
|
|
2958
|
-
}
|
|
2959
|
-
function buildResultsBackupRef(targetBranch, remoteCommit) {
|
|
2960
|
-
return `agentv/backups/${timestampForBackupRef()}-${slugifyBackupTargetBranch(
|
|
2961
|
-
targetBranch
|
|
2962
|
-
)}-${remoteCommit.slice(0, 7)}`;
|
|
3060
|
+
function formatShortSha(sha) {
|
|
3061
|
+
return sha ? sha.slice(0, 12) : "unknown";
|
|
2963
3062
|
}
|
|
2964
3063
|
async function getCommitSha(repoDir, ref) {
|
|
2965
3064
|
if (!ref) {
|
|
@@ -2976,91 +3075,303 @@ function isNonFastForwardPushError(error) {
|
|
|
2976
3075
|
const text = gitErrorText(error);
|
|
2977
3076
|
return text.includes("non-fast-forward") || text.includes("fetch first") || text.includes("tip is behind its remote") || text.includes("note about fast-forwards") || text.includes("stale info");
|
|
2978
3077
|
}
|
|
2979
|
-
|
|
2980
|
-
|
|
3078
|
+
var RESULTS_PUSH_MERGE_MAX_ATTEMPTS = 5;
|
|
3079
|
+
async function isAncestorCommit(repoDir, ancestor, descendant) {
|
|
3080
|
+
const { exitCode } = await runGit(["merge-base", "--is-ancestor", ancestor, descendant], {
|
|
3081
|
+
cwd: repoDir,
|
|
3082
|
+
check: false
|
|
3083
|
+
});
|
|
3084
|
+
return exitCode === 0;
|
|
2981
3085
|
}
|
|
2982
|
-
function
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
3086
|
+
async function getMergeBaseCommit(repoDir, a, b) {
|
|
3087
|
+
const { stdout, exitCode } = await runGit(["merge-base", a, b], { cwd: repoDir, check: false });
|
|
3088
|
+
const sha = stdout.trim();
|
|
3089
|
+
return exitCode === 0 && sha.length > 0 ? sha : void 0;
|
|
2986
3090
|
}
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3091
|
+
function buildNeedsHumanMergeReason(targetBranch, remoteCommit, localCommit) {
|
|
3092
|
+
return `Results branch ${targetBranch} diverged from the remote and could not be auto-merged: a genuine results content conflict remains (remote ${formatShortSha(
|
|
3093
|
+
remoteCommit
|
|
3094
|
+
)}, local ${formatShortSha(
|
|
3095
|
+
localCommit
|
|
3096
|
+
)}). The remote branch is unchanged and no history was rewritten; resolve it with a GitHub pull request.`;
|
|
3097
|
+
}
|
|
3098
|
+
function timestampForResultsRef(date = /* @__PURE__ */ new Date()) {
|
|
3099
|
+
const pad = (value) => String(value).padStart(2, "0");
|
|
3100
|
+
return `${date.getUTCFullYear()}${pad(date.getUTCMonth() + 1)}${pad(date.getUTCDate())}T${pad(
|
|
3101
|
+
date.getUTCHours()
|
|
3102
|
+
)}${pad(date.getUTCMinutes())}${pad(date.getUTCSeconds())}Z`;
|
|
3103
|
+
}
|
|
3104
|
+
function slugifyResultsTargetBranch(branch) {
|
|
3105
|
+
return branch.trim().replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "results";
|
|
3106
|
+
}
|
|
3107
|
+
function randomResultsRefToken() {
|
|
3108
|
+
return randomBytes(4).toString("hex").slice(0, 6);
|
|
3109
|
+
}
|
|
3110
|
+
function buildResultsSyncBranchName(targetBranch, token = randomResultsRefToken()) {
|
|
3111
|
+
return `agentv/results-sync/${timestampForResultsRef()}-${slugifyResultsTargetBranch(
|
|
3112
|
+
targetBranch
|
|
3113
|
+
)}-${token}`;
|
|
3114
|
+
}
|
|
3115
|
+
function buildResultsCompareUrl(remoteUrl, targetBranch, syncBranch) {
|
|
3116
|
+
if (!remoteUrl) {
|
|
3117
|
+
return void 0;
|
|
3008
3118
|
}
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
blockReason: buildBlockedPushConflictReason(baseDetails),
|
|
3013
|
-
details: baseDetails
|
|
3014
|
-
};
|
|
3119
|
+
const match = remoteUrl.trim().match(/github\.com[:/]([^/\s]+)\/([^/\s]+?)(?:\.git)?\/?$/i);
|
|
3120
|
+
if (!match) {
|
|
3121
|
+
return void 0;
|
|
3015
3122
|
}
|
|
3016
|
-
const
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
);
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3123
|
+
const [, owner, repo] = match;
|
|
3124
|
+
return `https://github.com/${owner}/${repo}/compare/${encodeURIComponent(
|
|
3125
|
+
targetBranch
|
|
3126
|
+
)}...${encodeURIComponent(syncBranch)}?expand=1`;
|
|
3127
|
+
}
|
|
3128
|
+
async function resolveRemotePushUrl(repoDir, remote) {
|
|
3129
|
+
for (const args of [
|
|
3130
|
+
["remote", "get-url", "--push", remote],
|
|
3131
|
+
["remote", "get-url", remote]
|
|
3132
|
+
]) {
|
|
3133
|
+
const { stdout, exitCode } = await runGit(args, { cwd: repoDir, check: false });
|
|
3134
|
+
const url = stdout.trim();
|
|
3135
|
+
if (exitCode === 0 && url.length > 0) {
|
|
3136
|
+
return url;
|
|
3137
|
+
}
|
|
3138
|
+
}
|
|
3139
|
+
return void 0;
|
|
3140
|
+
}
|
|
3141
|
+
async function countContributedRunDirs(repoDir, remoteTargetCommit, sourceCommit) {
|
|
3142
|
+
if (!remoteTargetCommit) {
|
|
3143
|
+
return void 0;
|
|
3144
|
+
}
|
|
3145
|
+
const { stdout, exitCode } = await runGit(
|
|
3146
|
+
["diff", "--name-only", `${remoteTargetCommit}...${sourceCommit}`, "--", RESULTS_REPO_RUNS_DIR],
|
|
3147
|
+
{ cwd: repoDir, check: false }
|
|
3148
|
+
);
|
|
3149
|
+
if (exitCode !== 0) {
|
|
3150
|
+
return void 0;
|
|
3151
|
+
}
|
|
3152
|
+
const runDirs = /* @__PURE__ */ new Set();
|
|
3153
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
3154
|
+
const file = line.trim();
|
|
3155
|
+
if (!file) {
|
|
3156
|
+
continue;
|
|
3157
|
+
}
|
|
3158
|
+
const segments = file.split("/");
|
|
3159
|
+
if (segments.length >= 2 && segments[0] && !segments[0].startsWith(".")) {
|
|
3160
|
+
runDirs.add(segments[0]);
|
|
3161
|
+
}
|
|
3036
3162
|
}
|
|
3163
|
+
return runDirs.size;
|
|
3164
|
+
}
|
|
3165
|
+
async function pushResultsSyncBranch(params) {
|
|
3166
|
+
const { repoDir, remote, sourceCommit, targetBranch } = params;
|
|
3167
|
+
const remoteTargetCommit = await getCommitSha(repoDir, remoteBranchRef(targetBranch, remote));
|
|
3168
|
+
let syncBranch = buildResultsSyncBranchName(targetBranch);
|
|
3169
|
+
await assertValidResultsBranchName(repoDir, syncBranch);
|
|
3037
3170
|
try {
|
|
3038
|
-
await runGit(
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
"--porcelain",
|
|
3042
|
-
`--force-with-lease=refs/heads/${params.targetBranch}:${remoteCommit}`,
|
|
3043
|
-
params.normalized.remote,
|
|
3044
|
-
`${params.sourceRef}:refs/heads/${params.targetBranch}`
|
|
3045
|
-
],
|
|
3046
|
-
{ cwd: params.repoDir }
|
|
3047
|
-
);
|
|
3171
|
+
await runGit(["push", "--porcelain", remote, `${sourceCommit}:refs/heads/${syncBranch}`], {
|
|
3172
|
+
cwd: repoDir
|
|
3173
|
+
});
|
|
3048
3174
|
} catch (error) {
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
|
|
3055
|
-
|
|
3175
|
+
if (!isRefAlreadyExistsPushError(error)) {
|
|
3176
|
+
throw error;
|
|
3177
|
+
}
|
|
3178
|
+
syncBranch = buildResultsSyncBranchName(targetBranch);
|
|
3179
|
+
await assertValidResultsBranchName(repoDir, syncBranch);
|
|
3180
|
+
await runGit(["push", "--porcelain", remote, `${sourceCommit}:refs/heads/${syncBranch}`], {
|
|
3181
|
+
cwd: repoDir
|
|
3182
|
+
});
|
|
3056
3183
|
}
|
|
3184
|
+
const compareUrl = buildResultsCompareUrl(
|
|
3185
|
+
await resolveRemotePushUrl(repoDir, remote),
|
|
3186
|
+
targetBranch,
|
|
3187
|
+
syncBranch
|
|
3188
|
+
);
|
|
3189
|
+
const contributedRunCount = await countContributedRunDirs(
|
|
3190
|
+
repoDir,
|
|
3191
|
+
remoteTargetCommit,
|
|
3192
|
+
sourceCommit
|
|
3193
|
+
).catch(() => void 0);
|
|
3057
3194
|
return {
|
|
3058
|
-
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
|
|
3195
|
+
tempBranch: syncBranch,
|
|
3196
|
+
targetBranch,
|
|
3197
|
+
...compareUrl !== void 0 && { compareUrl },
|
|
3198
|
+
...contributedRunCount !== void 0 && { contributedRunCount },
|
|
3199
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
3200
|
+
};
|
|
3201
|
+
}
|
|
3202
|
+
function isRefAlreadyExistsPushError(error) {
|
|
3203
|
+
const text = gitErrorText(error);
|
|
3204
|
+
return text.includes("already exists") || text.includes("cannot lock ref") || text.includes("reference already exists");
|
|
3205
|
+
}
|
|
3206
|
+
async function mergeRemoteIntoCheckedOutBranch(repoDir, remoteCommit) {
|
|
3207
|
+
const args = ["merge", "--no-edit", "-m", "chore(results): merge remote results", remoteCommit];
|
|
3208
|
+
let result = await runGit(args, {
|
|
3209
|
+
cwd: repoDir,
|
|
3210
|
+
check: false,
|
|
3211
|
+
env: configuredGitCommitIdentityEnv()
|
|
3212
|
+
});
|
|
3213
|
+
if (result.exitCode !== 0 && isMissingGitIdentityError({ stderr: result.stderr })) {
|
|
3214
|
+
await runGit(["merge", "--abort"], { cwd: repoDir, check: false });
|
|
3215
|
+
result = await runGit(args, {
|
|
3216
|
+
cwd: repoDir,
|
|
3217
|
+
check: false,
|
|
3218
|
+
env: { ...configuredGitCommitIdentityEnv(), ...fallbackResultsRepoCommitEnv() }
|
|
3219
|
+
});
|
|
3220
|
+
}
|
|
3221
|
+
return result.exitCode;
|
|
3222
|
+
}
|
|
3223
|
+
async function resolveResultBranchPushConflict(params) {
|
|
3224
|
+
await ensureResultsMergeConfig(params.repoDir);
|
|
3225
|
+
const { repoDir, targetBranch } = params;
|
|
3226
|
+
const remote = params.normalized.remote;
|
|
3227
|
+
const localRef = params.sourceRef;
|
|
3228
|
+
const remoteRef = remoteBranchRef(targetBranch, remote);
|
|
3229
|
+
const branchCheckedOut = params.sourceRef === "HEAD";
|
|
3230
|
+
let lastDetails = {
|
|
3231
|
+
pushConflictPolicy: params.normalized.push_conflict_policy,
|
|
3232
|
+
targetBranch
|
|
3233
|
+
};
|
|
3234
|
+
const buildHumanMergeOutcome = async (reason, details) => {
|
|
3235
|
+
if (!details.localCommit) {
|
|
3236
|
+
return { blocked: true, blockReason: reason, details, syncStatus: "needs_human_merge" };
|
|
3237
|
+
}
|
|
3238
|
+
try {
|
|
3239
|
+
const pendingMerge = await pushResultsSyncBranch({
|
|
3240
|
+
repoDir,
|
|
3241
|
+
remote,
|
|
3242
|
+
sourceCommit: details.localCommit,
|
|
3243
|
+
targetBranch: details.targetBranch
|
|
3244
|
+
});
|
|
3245
|
+
return {
|
|
3246
|
+
blocked: true,
|
|
3247
|
+
blockReason: reason,
|
|
3248
|
+
details: { ...details, pendingMerge },
|
|
3249
|
+
syncStatus: "needs_human_merge"
|
|
3250
|
+
};
|
|
3251
|
+
} catch (error) {
|
|
3252
|
+
return {
|
|
3253
|
+
blocked: true,
|
|
3254
|
+
blockReason: `${reason} (could not create the temp branch for the GitHub merge: ${getStatusMessage(
|
|
3255
|
+
error
|
|
3256
|
+
)})`,
|
|
3257
|
+
details,
|
|
3258
|
+
syncStatus: "needs_human_merge"
|
|
3259
|
+
};
|
|
3062
3260
|
}
|
|
3063
3261
|
};
|
|
3262
|
+
for (let attempt = 1; attempt <= RESULTS_PUSH_MERGE_MAX_ATTEMPTS; attempt += 1) {
|
|
3263
|
+
await fetchResultsRepo(repoDir, remote, targetBranch);
|
|
3264
|
+
const remoteCommit = await getCommitSha(repoDir, remoteRef);
|
|
3265
|
+
const localCommit = await getCommitSha(repoDir, localRef);
|
|
3266
|
+
const details = {
|
|
3267
|
+
pushConflictPolicy: params.normalized.push_conflict_policy,
|
|
3268
|
+
targetBranch,
|
|
3269
|
+
...remoteCommit !== void 0 && { remoteCommit, previousRemoteCommit: remoteCommit },
|
|
3270
|
+
...localCommit !== void 0 && { localCommit }
|
|
3271
|
+
};
|
|
3272
|
+
lastDetails = details;
|
|
3273
|
+
if (!remoteCommit) {
|
|
3274
|
+
return {
|
|
3275
|
+
blocked: true,
|
|
3276
|
+
blockReason: `Results branch push conflict on ${targetBranch}: remote commit could not be resolved after fetch`,
|
|
3277
|
+
details,
|
|
3278
|
+
syncStatus: "push_conflict"
|
|
3279
|
+
};
|
|
3280
|
+
}
|
|
3281
|
+
if (!localCommit) {
|
|
3282
|
+
return {
|
|
3283
|
+
blocked: true,
|
|
3284
|
+
blockReason: `Results branch push conflict on ${targetBranch}: local commit could not be resolved`,
|
|
3285
|
+
details,
|
|
3286
|
+
syncStatus: "push_conflict"
|
|
3287
|
+
};
|
|
3288
|
+
}
|
|
3289
|
+
if (await isAncestorCommit(repoDir, localCommit, remoteCommit)) {
|
|
3290
|
+
if (branchCheckedOut) {
|
|
3291
|
+
await runGit(["merge", "--ff-only", remoteCommit], { cwd: repoDir, check: false });
|
|
3292
|
+
} else {
|
|
3293
|
+
await runGit(["update-ref", localRef, remoteCommit, localCommit], {
|
|
3294
|
+
cwd: repoDir,
|
|
3295
|
+
check: false
|
|
3296
|
+
});
|
|
3297
|
+
}
|
|
3298
|
+
return { blocked: false, details };
|
|
3299
|
+
}
|
|
3300
|
+
let pushSpec = branchCheckedOut ? "HEAD" : localCommit;
|
|
3301
|
+
let autoMergedRemote = false;
|
|
3302
|
+
if (!await isAncestorCommit(repoDir, remoteCommit, localCommit)) {
|
|
3303
|
+
if (branchCheckedOut) {
|
|
3304
|
+
const exitCode = await mergeRemoteIntoCheckedOutBranch(repoDir, remoteCommit);
|
|
3305
|
+
if (exitCode !== 0) {
|
|
3306
|
+
await runGit(["merge", "--abort"], { cwd: repoDir, check: false });
|
|
3307
|
+
return await buildHumanMergeOutcome(
|
|
3308
|
+
buildNeedsHumanMergeReason(targetBranch, remoteCommit, localCommit),
|
|
3309
|
+
details
|
|
3310
|
+
);
|
|
3311
|
+
}
|
|
3312
|
+
pushSpec = "HEAD";
|
|
3313
|
+
autoMergedRemote = true;
|
|
3314
|
+
} else {
|
|
3315
|
+
const base = await getMergeBaseCommit(repoDir, localCommit, remoteCommit);
|
|
3316
|
+
if (!base) {
|
|
3317
|
+
return await buildHumanMergeOutcome(
|
|
3318
|
+
buildNeedsHumanMergeReason(targetBranch, remoteCommit, localCommit),
|
|
3319
|
+
details
|
|
3320
|
+
);
|
|
3321
|
+
}
|
|
3322
|
+
const merge = await runGit(
|
|
3323
|
+
["merge-tree", "--write-tree", `--merge-base=${base}`, localCommit, remoteCommit],
|
|
3324
|
+
{ cwd: repoDir, check: false }
|
|
3325
|
+
);
|
|
3326
|
+
const mergedTree = merge.stdout.trim().split(/\r?\n/)[0]?.trim();
|
|
3327
|
+
if (merge.exitCode !== 0 || !mergedTree) {
|
|
3328
|
+
return await buildHumanMergeOutcome(
|
|
3329
|
+
buildNeedsHumanMergeReason(targetBranch, remoteCommit, localCommit),
|
|
3330
|
+
details
|
|
3331
|
+
);
|
|
3332
|
+
}
|
|
3333
|
+
const { stdout: mergeCommitOut } = await runGitWithFallbackCommitIdentity(
|
|
3334
|
+
[
|
|
3335
|
+
"commit-tree",
|
|
3336
|
+
mergedTree,
|
|
3337
|
+
"-p",
|
|
3338
|
+
localCommit,
|
|
3339
|
+
"-p",
|
|
3340
|
+
remoteCommit,
|
|
3341
|
+
"-m",
|
|
3342
|
+
"chore(results): merge remote results"
|
|
3343
|
+
],
|
|
3344
|
+
{ cwd: repoDir }
|
|
3345
|
+
);
|
|
3346
|
+
pushSpec = mergeCommitOut.trim();
|
|
3347
|
+
await runGit(["update-ref", localRef, pushSpec, localCommit], { cwd: repoDir });
|
|
3348
|
+
autoMergedRemote = true;
|
|
3349
|
+
}
|
|
3350
|
+
}
|
|
3351
|
+
try {
|
|
3352
|
+
await runGit(["push", "--porcelain", remote, `${pushSpec}:refs/heads/${targetBranch}`], {
|
|
3353
|
+
cwd: repoDir
|
|
3354
|
+
});
|
|
3355
|
+
} catch (error) {
|
|
3356
|
+
if (isNonFastForwardPushError(error)) {
|
|
3357
|
+
continue;
|
|
3358
|
+
}
|
|
3359
|
+
throw error;
|
|
3360
|
+
}
|
|
3361
|
+
const pushedCommit = await getCommitSha(repoDir, pushSpec) ?? localCommit;
|
|
3362
|
+
return {
|
|
3363
|
+
blocked: false,
|
|
3364
|
+
details: {
|
|
3365
|
+
...details,
|
|
3366
|
+
localCommit: pushedCommit,
|
|
3367
|
+
...autoMergedRemote && { autoMergedRemote }
|
|
3368
|
+
}
|
|
3369
|
+
};
|
|
3370
|
+
}
|
|
3371
|
+
return await buildHumanMergeOutcome(
|
|
3372
|
+
`Results branch ${targetBranch} could not be reconciled after ${RESULTS_PUSH_MERGE_MAX_ATTEMPTS} attempts because the remote kept advancing; retry sync. The remote branch is unchanged.`,
|
|
3373
|
+
lastDetails
|
|
3374
|
+
);
|
|
3064
3375
|
}
|
|
3065
3376
|
async function statusFromInspection(normalized, repoDir) {
|
|
3066
3377
|
return withGitInspection(
|
|
@@ -3087,11 +3398,7 @@ async function getResultsRepoSyncStatus(config) {
|
|
|
3087
3398
|
};
|
|
3088
3399
|
}
|
|
3089
3400
|
try {
|
|
3090
|
-
await ensureResultsRepoRemote(normalized.path, normalized);
|
|
3091
3401
|
if (usesStorageBranchWorktree(normalized)) {
|
|
3092
|
-
await fetchResultsRepo(normalized.path, normalized.remote, normalized.branch).catch(
|
|
3093
|
-
() => void 0
|
|
3094
|
-
);
|
|
3095
3402
|
return withGitInspection(
|
|
3096
3403
|
baseStatus,
|
|
3097
3404
|
await inspectResultsStorageBranchGit(normalized.path, normalized)
|
|
@@ -3252,7 +3559,8 @@ async function syncResultsRepoForProject(config) {
|
|
|
3252
3559
|
return withPushConflictStatus(status2, outcome.blockReason, outcome.details, {
|
|
3253
3560
|
pullPerformed,
|
|
3254
3561
|
pushPerformed,
|
|
3255
|
-
commitCreated
|
|
3562
|
+
commitCreated,
|
|
3563
|
+
syncStatus: outcome.syncStatus
|
|
3256
3564
|
});
|
|
3257
3565
|
}
|
|
3258
3566
|
pushPerformed = true;
|
|
@@ -3369,7 +3677,8 @@ async function syncResultsRepoForProject(config) {
|
|
|
3369
3677
|
return withPushConflictStatus(status2, outcome.blockReason, outcome.details, {
|
|
3370
3678
|
pullPerformed,
|
|
3371
3679
|
pushPerformed,
|
|
3372
|
-
commitCreated
|
|
3680
|
+
commitCreated,
|
|
3681
|
+
syncStatus: outcome.syncStatus
|
|
3373
3682
|
});
|
|
3374
3683
|
}
|
|
3375
3684
|
} else {
|
|
@@ -3464,10 +3773,16 @@ async function syncResultsRepoForProject(config) {
|
|
|
3464
3773
|
}
|
|
3465
3774
|
}
|
|
3466
3775
|
if (inspection.syncStatus === "dirty") {
|
|
3467
|
-
const
|
|
3468
|
-
await runGit(["add", "--all", "--", ...
|
|
3776
|
+
const trackedPathspecs = await existingTrackedResultsPathspecs(repoDir);
|
|
3777
|
+
await runGit(["add", "--all", "--", ...trackedPathspecs], { cwd: repoDir });
|
|
3469
3778
|
await runGitWithFallbackCommitIdentity(
|
|
3470
|
-
[
|
|
3779
|
+
[
|
|
3780
|
+
"commit",
|
|
3781
|
+
"-m",
|
|
3782
|
+
"chore(results): sync local results branch changes",
|
|
3783
|
+
"--",
|
|
3784
|
+
...trackedPathspecs
|
|
3785
|
+
],
|
|
3471
3786
|
{
|
|
3472
3787
|
cwd: repoDir
|
|
3473
3788
|
}
|
|
@@ -3502,7 +3817,8 @@ async function syncResultsRepoForProject(config) {
|
|
|
3502
3817
|
return withPushConflictStatus(status, outcome.blockReason, outcome.details, {
|
|
3503
3818
|
pullPerformed,
|
|
3504
3819
|
pushPerformed,
|
|
3505
|
-
commitCreated
|
|
3820
|
+
commitCreated,
|
|
3821
|
+
syncStatus: outcome.syncStatus
|
|
3506
3822
|
});
|
|
3507
3823
|
}
|
|
3508
3824
|
pushPerformed = true;
|
|
@@ -3587,7 +3903,8 @@ async function syncResultsRepoForProject(config) {
|
|
|
3587
3903
|
return withPushConflictStatus(status, outcome.blockReason, outcome.details, {
|
|
3588
3904
|
pullPerformed,
|
|
3589
3905
|
pushPerformed,
|
|
3590
|
-
commitCreated
|
|
3906
|
+
commitCreated,
|
|
3907
|
+
syncStatus: outcome.syncStatus
|
|
3591
3908
|
});
|
|
3592
3909
|
}
|
|
3593
3910
|
} else {
|
|
@@ -3625,6 +3942,44 @@ async function syncResultsRepoForProject(config) {
|
|
|
3625
3942
|
activeResultsRepoSyncs.delete(syncKey);
|
|
3626
3943
|
}
|
|
3627
3944
|
}
|
|
3945
|
+
async function fastForwardResultsTowardTarget(repoDir, normalized, targetBranch) {
|
|
3946
|
+
const upstreamRef = remoteBranchRef(targetBranch, normalized.remote);
|
|
3947
|
+
if (!await gitRefExists(repoDir, upstreamRef)) {
|
|
3948
|
+
return;
|
|
3949
|
+
}
|
|
3950
|
+
const upstreamCommit = await getCommitSha(repoDir, upstreamRef);
|
|
3951
|
+
if (!upstreamCommit) {
|
|
3952
|
+
return;
|
|
3953
|
+
}
|
|
3954
|
+
if (usesStorageBranchWorktree(normalized) && normalized.branch) {
|
|
3955
|
+
const localRef = `refs/heads/${normalized.branch}`;
|
|
3956
|
+
if (!await gitRefExists(repoDir, localRef)) {
|
|
3957
|
+
await runGit(["update-ref", localRef, upstreamCommit], { cwd: repoDir, check: false });
|
|
3958
|
+
await runGit(["branch", "--set-upstream-to", upstreamRef, normalized.branch], {
|
|
3959
|
+
cwd: repoDir,
|
|
3960
|
+
check: false
|
|
3961
|
+
});
|
|
3962
|
+
return;
|
|
3963
|
+
}
|
|
3964
|
+
const localCommit = await getCommitSha(repoDir, localRef);
|
|
3965
|
+
if (localCommit && await isAncestorCommit(repoDir, localCommit, upstreamCommit)) {
|
|
3966
|
+
await runGit(["update-ref", localRef, upstreamCommit, localCommit], {
|
|
3967
|
+
cwd: repoDir,
|
|
3968
|
+
check: false
|
|
3969
|
+
});
|
|
3970
|
+
}
|
|
3971
|
+
return;
|
|
3972
|
+
}
|
|
3973
|
+
await runGit(["merge", "--ff-only", upstreamRef], { cwd: repoDir, check: false });
|
|
3974
|
+
}
|
|
3975
|
+
async function confirmResultsMergeAndPull(config) {
|
|
3976
|
+
const normalized = normalizeResultsConfig(config);
|
|
3977
|
+
const repoDir = await ensureResultsRepoClone(normalized);
|
|
3978
|
+
const targetBranch = normalized.branch ?? await resolveDefaultBranch(repoDir);
|
|
3979
|
+
await fetchResultsRepo(repoDir, normalized.remote, normalized.branch).catch(() => void 0);
|
|
3980
|
+
await fastForwardResultsTowardTarget(repoDir, normalized, targetBranch);
|
|
3981
|
+
return syncResultsRepoForProject(config);
|
|
3982
|
+
}
|
|
3628
3983
|
async function checkoutResultsRepoBranch(config, branchName) {
|
|
3629
3984
|
const normalized = normalizeResultsConfig(config);
|
|
3630
3985
|
const repoDir = await ensureResultsRepoClone(normalized);
|
|
@@ -3783,12 +4138,10 @@ function artifactSidecarPointers(record) {
|
|
|
3783
4138
|
return [];
|
|
3784
4139
|
}
|
|
3785
4140
|
const pointers = [];
|
|
3786
|
-
|
|
3787
|
-
|
|
3788
|
-
continue;
|
|
3789
|
-
}
|
|
4141
|
+
const pointer = record.artifact_pointers.transcript;
|
|
4142
|
+
if (isRecord2(pointer)) {
|
|
3790
4143
|
if (pointer.ref !== AGENTV_RESULTS_ARTIFACTS_REF || typeof pointer.path !== "string") {
|
|
3791
|
-
|
|
4144
|
+
return pointers;
|
|
3792
4145
|
}
|
|
3793
4146
|
pointers.push({
|
|
3794
4147
|
path: pointer.path,
|
|
@@ -3805,17 +4158,58 @@ function artifactSidecarKey(destinationPath, pointerPath) {
|
|
|
3805
4158
|
normalizeDestinationPath(pointerPath)
|
|
3806
4159
|
);
|
|
3807
4160
|
}
|
|
3808
|
-
function
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
4161
|
+
function isDeprecatedTraceArtifactPath(relativePath) {
|
|
4162
|
+
return relativePath === "trace.json" || relativePath.endsWith("/trace.json");
|
|
4163
|
+
}
|
|
4164
|
+
function isResultManifestFilename(filename) {
|
|
4165
|
+
return filename === RESULT_INDEX_FILENAME2;
|
|
4166
|
+
}
|
|
4167
|
+
function safeLocalSummaryManifestPath(sourceDir, manifestPath) {
|
|
4168
|
+
if (typeof manifestPath !== "string" || manifestPath.trim().length === 0) {
|
|
4169
|
+
return void 0;
|
|
3812
4170
|
}
|
|
3813
|
-
|
|
3814
|
-
|
|
3815
|
-
|
|
3816
|
-
|
|
3817
|
-
|
|
3818
|
-
|
|
4171
|
+
if (path4.isAbsolute(manifestPath)) {
|
|
4172
|
+
return void 0;
|
|
4173
|
+
}
|
|
4174
|
+
const normalized = path4.normalize(manifestPath);
|
|
4175
|
+
if (normalized.startsWith("..") || path4.isAbsolute(normalized)) {
|
|
4176
|
+
return void 0;
|
|
4177
|
+
}
|
|
4178
|
+
return path4.join(sourceDir, normalized);
|
|
4179
|
+
}
|
|
4180
|
+
function resolveLocalResultManifestPath(sourceDir) {
|
|
4181
|
+
try {
|
|
4182
|
+
const summary = JSON.parse(readFileSync2(path4.join(sourceDir, "summary.json"), "utf8"));
|
|
4183
|
+
const manifestPath2 = safeLocalSummaryManifestPath(
|
|
4184
|
+
sourceDir,
|
|
4185
|
+
summary.index_path ?? summary.manifest_path
|
|
4186
|
+
);
|
|
4187
|
+
if (manifestPath2 && existsSync(manifestPath2)) {
|
|
4188
|
+
return manifestPath2;
|
|
4189
|
+
}
|
|
4190
|
+
} catch {
|
|
4191
|
+
}
|
|
4192
|
+
const manifestPath = path4.join(sourceDir, RUN_INTERNAL_DIRNAME, RESULT_INDEX_FILENAME2);
|
|
4193
|
+
if (existsSync(manifestPath)) {
|
|
4194
|
+
return manifestPath;
|
|
4195
|
+
}
|
|
4196
|
+
const legacyManifestPath = path4.join(sourceDir, RESULT_INDEX_FILENAME2);
|
|
4197
|
+
if (existsSync(legacyManifestPath)) {
|
|
4198
|
+
return legacyManifestPath;
|
|
4199
|
+
}
|
|
4200
|
+
return void 0;
|
|
4201
|
+
}
|
|
4202
|
+
function collectArtifactSidecarPointers(sourceDir) {
|
|
4203
|
+
const indexPath = resolveLocalResultManifestPath(sourceDir);
|
|
4204
|
+
if (!indexPath) {
|
|
4205
|
+
return [];
|
|
4206
|
+
}
|
|
4207
|
+
const pointers = [];
|
|
4208
|
+
for (const line of readFileSync2(indexPath, "utf8").split(/\r?\n/)) {
|
|
4209
|
+
const trimmed = line.trim();
|
|
4210
|
+
if (!trimmed) {
|
|
4211
|
+
continue;
|
|
4212
|
+
}
|
|
3819
4213
|
try {
|
|
3820
4214
|
pointers.push(...artifactSidecarPointers(JSON.parse(trimmed)));
|
|
3821
4215
|
} catch {
|
|
@@ -3894,18 +4288,20 @@ function rewritePublishedIndexLine(line, destinationPath) {
|
|
|
3894
4288
|
return line;
|
|
3895
4289
|
}
|
|
3896
4290
|
let changed = false;
|
|
3897
|
-
|
|
3898
|
-
|
|
3899
|
-
continue;
|
|
3900
|
-
}
|
|
3901
|
-
if (pointer.ref !== AGENTV_RESULTS_ARTIFACTS_REF || typeof pointer.path !== "string") {
|
|
3902
|
-
continue;
|
|
3903
|
-
}
|
|
4291
|
+
const pointer = record.artifact_pointers.transcript;
|
|
4292
|
+
if (isRecord2(pointer) && pointer.ref === AGENTV_RESULTS_ARTIFACTS_REF && typeof pointer.path === "string") {
|
|
3904
4293
|
const key = artifactSidecarKey(destinationPath, pointer.path);
|
|
3905
4294
|
if (pointer.key !== key) {
|
|
3906
4295
|
pointer.key = key;
|
|
3907
4296
|
changed = true;
|
|
3908
4297
|
}
|
|
4298
|
+
if (Object.keys(record.artifact_pointers).length !== 1 || record.artifact_pointers.transcript !== pointer) {
|
|
4299
|
+
changed = true;
|
|
4300
|
+
}
|
|
4301
|
+
record.artifact_pointers = { transcript: pointer };
|
|
4302
|
+
} else {
|
|
4303
|
+
record.artifact_pointers = void 0;
|
|
4304
|
+
changed = true;
|
|
3909
4305
|
}
|
|
3910
4306
|
return changed ? JSON.stringify(record) : line;
|
|
3911
4307
|
}
|
|
@@ -3922,13 +4318,16 @@ async function preparePublishedResultsSource(params) {
|
|
|
3922
4318
|
for (const sourceFile of sourceFiles) {
|
|
3923
4319
|
const relativeFile = path4.relative(params.sourceDir, sourceFile).split(path4.sep).join("/");
|
|
3924
4320
|
const destinationFile = path4.join(publishedRoot, ...relativeFile.split("/"));
|
|
3925
|
-
if (relativeFile
|
|
4321
|
+
if (isResultManifestFilename(path4.posix.basename(relativeFile))) {
|
|
3926
4322
|
const original = readFileSync2(sourceFile, "utf8");
|
|
3927
4323
|
const rewritten = original.split(/\r?\n/).map((line) => rewritePublishedIndexLine(line, params.destinationPath)).join("\n");
|
|
3928
4324
|
mkdirSync(path4.dirname(destinationFile), { recursive: true });
|
|
3929
4325
|
writeFileSync(destinationFile, rewritten);
|
|
3930
4326
|
continue;
|
|
3931
4327
|
}
|
|
4328
|
+
if (isDeprecatedTraceArtifactPath(relativeFile)) {
|
|
4329
|
+
continue;
|
|
4330
|
+
}
|
|
3932
4331
|
if (omittedPaths.has(relativeFile)) {
|
|
3933
4332
|
continue;
|
|
3934
4333
|
}
|
|
@@ -4008,6 +4407,7 @@ async function commitStorageBranchWorktreePaths(params) {
|
|
|
4008
4407
|
} else {
|
|
4009
4408
|
await runGit(["read-tree", "--empty"], { cwd: params.repoDir, env: indexEnv });
|
|
4010
4409
|
}
|
|
4410
|
+
await addResultsGitattributesToIndex(params.repoDir, indexEnv);
|
|
4011
4411
|
for (const gitPath of paths) {
|
|
4012
4412
|
const sourcePath = path4.join(params.repoDir, ...gitPath.split("/"));
|
|
4013
4413
|
if (!existsSync(sourcePath)) {
|
|
@@ -4098,6 +4498,7 @@ async function commitResultsRunWithTemporaryIndex(params) {
|
|
|
4098
4498
|
} else {
|
|
4099
4499
|
await runGit(["read-tree", "--empty"], { cwd: params.repoDir, env: indexEnv });
|
|
4100
4500
|
}
|
|
4501
|
+
await addResultsGitattributesToIndex(params.repoDir, indexEnv);
|
|
4101
4502
|
const existingPaths = await getExistingRunTreePaths(
|
|
4102
4503
|
params.repoDir,
|
|
4103
4504
|
base.baseRef,
|
|
@@ -4189,7 +4590,7 @@ function buildDirectPushResult(normalized, changed, outcome) {
|
|
|
4189
4590
|
...outcome?.blocked === true && {
|
|
4190
4591
|
blocked: true,
|
|
4191
4592
|
block_reason: outcome.blockReason,
|
|
4192
|
-
sync_status: "push_conflict"
|
|
4593
|
+
sync_status: outcome.syncStatus ?? "push_conflict"
|
|
4193
4594
|
},
|
|
4194
4595
|
...pushDetailsToWire(details)
|
|
4195
4596
|
};
|
|
@@ -4199,10 +4600,11 @@ function mergeDirectPushResults(normalized, results) {
|
|
|
4199
4600
|
if (blocked) {
|
|
4200
4601
|
return blocked;
|
|
4201
4602
|
}
|
|
4202
|
-
const detailed = [...results].reverse().find((result) => result.backup_ref !== void 0);
|
|
4603
|
+
const detailed = [...results].reverse().find((result) => result.backup_ref !== void 0 || result.auto_merged_remote === true);
|
|
4203
4604
|
return {
|
|
4204
4605
|
changed: results.some((result) => result.changed),
|
|
4205
4606
|
push_conflict_policy: normalized.push_conflict_policy,
|
|
4607
|
+
...detailed?.auto_merged_remote === true && { auto_merged_remote: true },
|
|
4206
4608
|
...detailed?.backup_ref !== void 0 && {
|
|
4207
4609
|
backup_ref: detailed.backup_ref
|
|
4208
4610
|
},
|
|
@@ -4385,21 +4787,74 @@ async function directPushResultsWithDetails(params) {
|
|
|
4385
4787
|
await sidecar?.cleanup().catch(() => void 0);
|
|
4386
4788
|
}
|
|
4387
4789
|
}
|
|
4790
|
+
function safeGitSummaryManifestPath(runDir, manifestPath) {
|
|
4791
|
+
if (typeof manifestPath !== "string" || manifestPath.trim().length === 0) {
|
|
4792
|
+
return void 0;
|
|
4793
|
+
}
|
|
4794
|
+
if (manifestPath.startsWith("/")) {
|
|
4795
|
+
return void 0;
|
|
4796
|
+
}
|
|
4797
|
+
const normalized = path4.posix.normalize(manifestPath);
|
|
4798
|
+
if (normalized === ".." || normalized.startsWith("../")) {
|
|
4799
|
+
return void 0;
|
|
4800
|
+
}
|
|
4801
|
+
return path4.posix.join(runDir, normalized);
|
|
4802
|
+
}
|
|
4803
|
+
function buildGitManifestPaths(treePaths, summaryByPath) {
|
|
4804
|
+
const treePathSet = new Set(treePaths);
|
|
4805
|
+
const manifestByRunDir = /* @__PURE__ */ new Map();
|
|
4806
|
+
for (const [summaryPath, summary] of summaryByPath) {
|
|
4807
|
+
const runDir = path4.posix.dirname(summaryPath);
|
|
4808
|
+
const relativeRunPath = path4.posix.relative(RESULTS_REPO_RUNS_DIR, runDir);
|
|
4809
|
+
if (!isV2ResultsRepoRunPath(relativeRunPath)) {
|
|
4810
|
+
continue;
|
|
4811
|
+
}
|
|
4812
|
+
const manifestPath = safeGitSummaryManifestPath(runDir, summary.index_path ?? summary.manifest_path) ?? fallbackGitManifestPath(runDir, treePathSet);
|
|
4813
|
+
if (manifestPath && treePathSet.has(manifestPath)) {
|
|
4814
|
+
manifestByRunDir.set(runDir, manifestPath);
|
|
4815
|
+
}
|
|
4816
|
+
}
|
|
4817
|
+
for (const treePath of treePaths) {
|
|
4818
|
+
if (!treePath.endsWith(`/${RESULT_INDEX_FILENAME2}`)) {
|
|
4819
|
+
continue;
|
|
4820
|
+
}
|
|
4821
|
+
const runDir = gitRunDirForManifestPath(treePath);
|
|
4822
|
+
const relativeRunPath = path4.posix.relative(RESULTS_REPO_RUNS_DIR, runDir);
|
|
4823
|
+
if (!isV2ResultsRepoRunPath(relativeRunPath)) {
|
|
4824
|
+
continue;
|
|
4825
|
+
}
|
|
4826
|
+
if (!manifestByRunDir.has(runDir)) {
|
|
4827
|
+
manifestByRunDir.set(runDir, treePath);
|
|
4828
|
+
}
|
|
4829
|
+
}
|
|
4830
|
+
return [...manifestByRunDir.values()].sort();
|
|
4831
|
+
}
|
|
4832
|
+
function fallbackGitManifestPath(runDir, treePathSet) {
|
|
4833
|
+
for (const candidate of [
|
|
4834
|
+
path4.posix.join(runDir, ".internal", RESULT_INDEX_FILENAME2),
|
|
4835
|
+
path4.posix.join(runDir, RESULT_INDEX_FILENAME2)
|
|
4836
|
+
]) {
|
|
4837
|
+
if (treePathSet.has(candidate)) {
|
|
4838
|
+
return candidate;
|
|
4839
|
+
}
|
|
4840
|
+
}
|
|
4841
|
+
return void 0;
|
|
4842
|
+
}
|
|
4843
|
+
function gitRunDirForManifestPath(manifestPath) {
|
|
4844
|
+
const manifestDir = path4.posix.dirname(manifestPath);
|
|
4845
|
+
return path4.posix.basename(manifestDir) === ".internal" ? path4.posix.dirname(manifestDir) : manifestDir;
|
|
4846
|
+
}
|
|
4847
|
+
function isV2ResultsRepoRunPath(relativeRunPath) {
|
|
4848
|
+
const segments = relativeRunPath.split("/").filter(Boolean);
|
|
4849
|
+
return segments.length === 1 && !segments[0].startsWith(".");
|
|
4850
|
+
}
|
|
4388
4851
|
function buildGitRunId(relativeRunPath) {
|
|
4389
4852
|
const normalized = relativeRunPath.split(path4.sep).join("/");
|
|
4390
4853
|
const segments = normalized.split("/").filter(Boolean);
|
|
4391
|
-
if (segments.length >= 2) {
|
|
4392
|
-
const experiment = segments.slice(0, -1).join("/");
|
|
4393
|
-
const timestamp = segments.at(-1);
|
|
4394
|
-
if (experiment === "default") {
|
|
4395
|
-
return timestamp ?? normalized;
|
|
4396
|
-
}
|
|
4397
|
-
return `${experiment}::${timestamp}`;
|
|
4398
|
-
}
|
|
4399
4854
|
return segments[0] ?? relativeRunPath;
|
|
4400
4855
|
}
|
|
4401
|
-
function getRunExperiment(runId,
|
|
4402
|
-
const experiment =
|
|
4856
|
+
function getRunExperiment(runId, summary) {
|
|
4857
|
+
const experiment = summary?.metadata?.experiment?.trim();
|
|
4403
4858
|
if (experiment) {
|
|
4404
4859
|
return experiment;
|
|
4405
4860
|
}
|
|
@@ -4572,8 +5027,11 @@ async function pushWipCheckpoint(params) {
|
|
|
4572
5027
|
sourceDir: params.sourceDir,
|
|
4573
5028
|
destinationDir
|
|
4574
5029
|
});
|
|
4575
|
-
|
|
4576
|
-
|
|
5030
|
+
writeFileSync(
|
|
5031
|
+
path4.join(params.handle.worktreeDir, RESULTS_REPO_GITATTRIBUTES_FILE),
|
|
5032
|
+
RESULTS_REPO_GITATTRIBUTES_CONTENT
|
|
5033
|
+
);
|
|
5034
|
+
await runGit(["add", "--all", "--", RESULTS_REPO_GITATTRIBUTES_FILE, params.destinationPath], {
|
|
4577
5035
|
cwd: params.handle.worktreeDir
|
|
4578
5036
|
});
|
|
4579
5037
|
const { stdout: status } = await runGit(["status", "--porcelain"], {
|
|
@@ -4596,7 +5054,19 @@ async function pushWipCheckpoint(params) {
|
|
|
4596
5054
|
async function deleteWipBranch(params) {
|
|
4597
5055
|
const normalized = normalizeResultsConfig(params.config);
|
|
4598
5056
|
const cloneDir = await ensureResultsRepoClone(normalized);
|
|
4599
|
-
await runGit(["push", normalized.remote, "--delete", params.wipBranch], {
|
|
5057
|
+
const result = await runGit(["push", normalized.remote, "--delete", params.wipBranch], {
|
|
5058
|
+
cwd: cloneDir,
|
|
5059
|
+
check: false
|
|
5060
|
+
});
|
|
5061
|
+
if (result.exitCode === 0 || isMissingRemoteRefDelete(result)) {
|
|
5062
|
+
return;
|
|
5063
|
+
}
|
|
5064
|
+
throw new Error(result.stderr.trim() || result.stdout.trim() || "Failed to delete WIP branch");
|
|
5065
|
+
}
|
|
5066
|
+
function isMissingRemoteRefDelete(result) {
|
|
5067
|
+
const text = `${result.stderr}
|
|
5068
|
+
${result.stdout}`.toLowerCase();
|
|
5069
|
+
return text.includes("remote ref does not exist");
|
|
4600
5070
|
}
|
|
4601
5071
|
function isMissingGitRefError(error) {
|
|
4602
5072
|
const parts = [];
|
|
@@ -4608,7 +5078,107 @@ function isMissingGitRefError(error) {
|
|
|
4608
5078
|
parts.push(error);
|
|
4609
5079
|
}
|
|
4610
5080
|
const haystack = parts.join("\n").toLowerCase();
|
|
4611
|
-
return haystack.includes("not a valid object name") || haystack.includes("unknown revision or path") || haystack.includes("bad revision") || haystack.includes("does not exist");
|
|
5081
|
+
return haystack.includes("not a valid object name") || haystack.includes("unknown revision or path") || haystack.includes("bad revision") || haystack.includes("does not exist") || haystack.includes("needed a single revision");
|
|
5082
|
+
}
|
|
5083
|
+
function gitResultsIndexHash(value) {
|
|
5084
|
+
return createHash("sha256").update(value).digest("hex");
|
|
5085
|
+
}
|
|
5086
|
+
function resolveGitResultsIndexRepoDir(repoDir) {
|
|
5087
|
+
const resolved = path4.resolve(repoDir);
|
|
5088
|
+
try {
|
|
5089
|
+
return realpathSync(resolved);
|
|
5090
|
+
} catch {
|
|
5091
|
+
return resolved;
|
|
5092
|
+
}
|
|
5093
|
+
}
|
|
5094
|
+
function gitResultsIndexRepoKey(repoDir) {
|
|
5095
|
+
return `sha256-${gitResultsIndexHash(resolveGitResultsIndexRepoDir(repoDir)).slice(0, 32)}`;
|
|
5096
|
+
}
|
|
5097
|
+
function gitResultsIndexRefKey(ref) {
|
|
5098
|
+
const slug = sanitizeRepoSlug(ref).slice(0, 80) || "default";
|
|
5099
|
+
return `${slug}-${gitResultsIndexHash(ref).slice(0, 16)}`;
|
|
5100
|
+
}
|
|
5101
|
+
function resolveGitResultsIndexCacheFile(params) {
|
|
5102
|
+
const ref = params.ref ?? "origin/main";
|
|
5103
|
+
return path4.join(
|
|
5104
|
+
getAgentvDataDir(),
|
|
5105
|
+
"cache",
|
|
5106
|
+
"results-index",
|
|
5107
|
+
gitResultsIndexRepoKey(params.repoDir),
|
|
5108
|
+
gitResultsIndexRefKey(ref),
|
|
5109
|
+
`${params.commitSha}.json`
|
|
5110
|
+
);
|
|
5111
|
+
}
|
|
5112
|
+
function isGitListedRun(value) {
|
|
5113
|
+
if (!value || typeof value !== "object") return false;
|
|
5114
|
+
const record = value;
|
|
5115
|
+
return typeof record.run_id === "string" && typeof record.experiment === "string" && typeof record.timestamp === "string" && (typeof record.index_path === "string" || typeof record.manifest_path === "string") && typeof record.display_name === "string" && typeof record.test_count === "number" && typeof record.avg_score === "number" && typeof record.size_bytes === "number" && (record.execution_error_count === void 0 || typeof record.execution_error_count === "number") && (record.pass_rate === void 0 || typeof record.pass_rate === "number") && (record.target === void 0 || typeof record.target === "string") && (record.summary_path === void 0 || typeof record.summary_path === "string");
|
|
5116
|
+
}
|
|
5117
|
+
function validateGitResultsIndexCacheFile(value, params) {
|
|
5118
|
+
if (!value || typeof value !== "object") return void 0;
|
|
5119
|
+
const record = value;
|
|
5120
|
+
const repoDir = resolveGitResultsIndexRepoDir(params.repoDir);
|
|
5121
|
+
const repoKey = gitResultsIndexRepoKey(params.repoDir);
|
|
5122
|
+
if (record.schema_version !== GIT_RESULTS_INDEX_CACHE_SCHEMA_VERSION || record.result_layout_version !== GIT_RESULTS_INDEX_LAYOUT_VERSION || record.repo_key !== repoKey || record.repo_dir !== repoDir || record.ref !== params.ref || record.commit_sha !== params.commitSha || !Array.isArray(record.runs) || !record.runs.every(isGitListedRun)) {
|
|
5123
|
+
return void 0;
|
|
5124
|
+
}
|
|
5125
|
+
return value;
|
|
5126
|
+
}
|
|
5127
|
+
async function readGitResultsIndexCache(params) {
|
|
5128
|
+
const cacheFile = resolveGitResultsIndexCacheFile(params);
|
|
5129
|
+
try {
|
|
5130
|
+
const parsed = JSON.parse(await readFile2(cacheFile, "utf8"));
|
|
5131
|
+
return validateGitResultsIndexCacheFile(parsed, params)?.runs;
|
|
5132
|
+
} catch {
|
|
5133
|
+
return void 0;
|
|
5134
|
+
}
|
|
5135
|
+
}
|
|
5136
|
+
async function writeGitResultsIndexCache(params) {
|
|
5137
|
+
const cacheFile = resolveGitResultsIndexCacheFile(params);
|
|
5138
|
+
const envelope = {
|
|
5139
|
+
schema_version: GIT_RESULTS_INDEX_CACHE_SCHEMA_VERSION,
|
|
5140
|
+
result_layout_version: GIT_RESULTS_INDEX_LAYOUT_VERSION,
|
|
5141
|
+
repo_key: gitResultsIndexRepoKey(params.repoDir),
|
|
5142
|
+
repo_dir: resolveGitResultsIndexRepoDir(params.repoDir),
|
|
5143
|
+
ref: params.ref,
|
|
5144
|
+
commit_sha: params.commitSha,
|
|
5145
|
+
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5146
|
+
runs: params.runs
|
|
5147
|
+
};
|
|
5148
|
+
try {
|
|
5149
|
+
mkdirSync(path4.dirname(cacheFile), { recursive: true });
|
|
5150
|
+
const tempFile = `${cacheFile}.${process.pid}.${Date.now()}.tmp`;
|
|
5151
|
+
await writeFile(tempFile, `${JSON.stringify(envelope, null, 2)}
|
|
5152
|
+
`, "utf8");
|
|
5153
|
+
renameSync(tempFile, cacheFile);
|
|
5154
|
+
} catch {
|
|
5155
|
+
}
|
|
5156
|
+
}
|
|
5157
|
+
async function resolveGitRunsRefCommit(repoDir, ref = "origin/main") {
|
|
5158
|
+
try {
|
|
5159
|
+
const { stdout } = await runGit(["rev-parse", "--verify", `${ref}^{commit}`], {
|
|
5160
|
+
cwd: repoDir
|
|
5161
|
+
});
|
|
5162
|
+
return stdout.trim() || void 0;
|
|
5163
|
+
} catch (error) {
|
|
5164
|
+
if (isMissingGitRefError(error)) {
|
|
5165
|
+
return void 0;
|
|
5166
|
+
}
|
|
5167
|
+
throw error;
|
|
5168
|
+
}
|
|
5169
|
+
}
|
|
5170
|
+
async function listGitRunsCached(repoDir, ref = "origin/main") {
|
|
5171
|
+
const commitSha = await resolveGitRunsRefCommit(repoDir, ref);
|
|
5172
|
+
if (!commitSha) {
|
|
5173
|
+
return [];
|
|
5174
|
+
}
|
|
5175
|
+
const cached = await readGitResultsIndexCache({ repoDir, ref, commitSha });
|
|
5176
|
+
if (cached) {
|
|
5177
|
+
return [...cached];
|
|
5178
|
+
}
|
|
5179
|
+
const runs = await listGitRuns(repoDir, ref);
|
|
5180
|
+
await writeGitResultsIndexCache({ repoDir, ref, commitSha, runs });
|
|
5181
|
+
return runs;
|
|
4612
5182
|
}
|
|
4613
5183
|
async function listGitRuns(repoDir, ref = "origin/main") {
|
|
4614
5184
|
let treeOut;
|
|
@@ -4625,40 +5195,78 @@ async function listGitRuns(repoDir, ref = "origin/main") {
|
|
|
4625
5195
|
}
|
|
4626
5196
|
throw error;
|
|
4627
5197
|
}
|
|
4628
|
-
const
|
|
4629
|
-
|
|
5198
|
+
const treePaths = treeOut.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
|
|
5199
|
+
const summaryPaths = treePaths.filter((line) => line.endsWith("/summary.json"));
|
|
5200
|
+
const summaryByPath = /* @__PURE__ */ new Map();
|
|
5201
|
+
if (summaryPaths.length > 0) {
|
|
5202
|
+
const summaryBatchInput = `${summaryPaths.map((summaryPath) => `${ref}:${summaryPath}`).join("\n")}
|
|
5203
|
+
`;
|
|
5204
|
+
const summaryBlobs = parseGitBatchBlobs(await runGitBatch(repoDir, summaryBatchInput));
|
|
5205
|
+
for (let i = 0; i < summaryBlobs.length; i++) {
|
|
5206
|
+
const summaryPath = summaryPaths[i];
|
|
5207
|
+
const blob = summaryBlobs[i];
|
|
5208
|
+
if (!summaryPath || !blob) continue;
|
|
5209
|
+
summaryByPath.set(summaryPath, JSON.parse(blob.content.toString("utf8")));
|
|
5210
|
+
}
|
|
5211
|
+
}
|
|
5212
|
+
const indexPaths = buildGitManifestPaths(treePaths, summaryByPath);
|
|
5213
|
+
if (indexPaths.length === 0) {
|
|
4630
5214
|
return [];
|
|
4631
5215
|
}
|
|
4632
|
-
const batchInput = `${
|
|
5216
|
+
const batchInput = `${indexPaths.map((indexPath) => `${ref}:${indexPath}`).join("\n")}
|
|
4633
5217
|
`;
|
|
4634
5218
|
const blobs = parseGitBatchBlobs(await runGitBatch(repoDir, batchInput));
|
|
4635
|
-
if (blobs.length !==
|
|
5219
|
+
if (blobs.length !== indexPaths.length) {
|
|
4636
5220
|
throw new Error(
|
|
4637
|
-
`Expected ${
|
|
5221
|
+
`Expected ${indexPaths.length} git blobs but received ${blobs.length} while listing results runs`
|
|
4638
5222
|
);
|
|
4639
5223
|
}
|
|
4640
5224
|
const runs = blobs.flatMap((blob, index) => {
|
|
4641
|
-
const
|
|
4642
|
-
const
|
|
4643
|
-
const
|
|
5225
|
+
const manifestPath = indexPaths[index];
|
|
5226
|
+
const runDir = gitRunDirForManifestPath(manifestPath);
|
|
5227
|
+
const summaryPath = path4.posix.join(runDir, "summary.json");
|
|
5228
|
+
const summary = summaryByPath.get(summaryPath);
|
|
4644
5229
|
const relativeRunPath = path4.posix.relative(RESULTS_REPO_RUNS_DIR, runDir);
|
|
4645
5230
|
const runId = buildGitRunId(relativeRunPath);
|
|
4646
|
-
const
|
|
4647
|
-
|
|
4648
|
-
|
|
4649
|
-
|
|
5231
|
+
const rows = blob.content.toString("utf8").split(/\r?\n/).map((line) => line.trim()).filter(Boolean).flatMap(
|
|
5232
|
+
(line) => {
|
|
5233
|
+
try {
|
|
5234
|
+
return [
|
|
5235
|
+
JSON.parse(line)
|
|
5236
|
+
];
|
|
5237
|
+
} catch {
|
|
5238
|
+
return [];
|
|
5239
|
+
}
|
|
5240
|
+
}
|
|
5241
|
+
);
|
|
5242
|
+
const rowTargets = [
|
|
5243
|
+
...new Set(rows.map((row) => row.target).filter((target) => !!target))
|
|
5244
|
+
];
|
|
5245
|
+
const rowTestIds = [
|
|
5246
|
+
...new Set(rows.map((row) => row.test_id).filter((testId) => !!testId))
|
|
5247
|
+
];
|
|
5248
|
+
const rowScores = rows.map((row) => row.score).filter((score) => typeof score === "number" && Number.isFinite(score));
|
|
5249
|
+
const avgScore = rowScores.length > 0 ? rowScores.reduce((sum, score) => sum + score, 0) / rowScores.length : 0;
|
|
5250
|
+
const timestamp = summary?.metadata?.timestamp?.trim() || rows[0]?.timestamp || path4.posix.basename(runDir);
|
|
5251
|
+
const displayName = summary?.metadata?.display_name?.trim() || path4.posix.basename(runDir);
|
|
5252
|
+
const targets = summary?.metadata?.targets ?? rowTargets;
|
|
5253
|
+
const passRate = computeAveragePassRate(summary?.run_summary) ?? avgScore;
|
|
5254
|
+
const executionErrorCount = rows.filter(
|
|
5255
|
+
(row) => row.execution_status === "execution_error"
|
|
5256
|
+
).length;
|
|
4650
5257
|
return [
|
|
4651
5258
|
{
|
|
4652
5259
|
run_id: runId,
|
|
4653
|
-
experiment: getRunExperiment(runId,
|
|
5260
|
+
experiment: getRunExperiment(runId, summary),
|
|
4654
5261
|
timestamp,
|
|
4655
5262
|
...passRate !== void 0 && { pass_rate: passRate },
|
|
4656
5263
|
...targets.length === 1 && targets[0] ? { target: targets[0] } : {},
|
|
4657
|
-
|
|
4658
|
-
|
|
5264
|
+
index_path: manifestPath,
|
|
5265
|
+
...summaryByPath.has(summaryPath) && { summary_path: summaryPath },
|
|
4659
5266
|
display_name: displayName,
|
|
4660
|
-
test_count:
|
|
4661
|
-
|
|
5267
|
+
test_count: summary?.metadata?.tests_run?.length ?? rowTestIds.length,
|
|
5268
|
+
...executionErrorCount > 0 && { execution_error_count: executionErrorCount },
|
|
5269
|
+
avg_score: avgScore,
|
|
4662
5270
|
size_bytes: blob.size
|
|
4663
5271
|
}
|
|
4664
5272
|
];
|
|
@@ -4776,483 +5384,6 @@ function trimBaselineResult(result) {
|
|
|
4776
5384
|
}
|
|
4777
5385
|
return trimmed;
|
|
4778
5386
|
}
|
|
4779
|
-
var DEFAULT_CATEGORY = "Uncategorized";
|
|
4780
|
-
function deriveCategory(relativePath) {
|
|
4781
|
-
const parts = relativePath.split(/[/\\]/);
|
|
4782
|
-
if (parts.length <= 1) {
|
|
4783
|
-
return DEFAULT_CATEGORY;
|
|
4784
|
-
}
|
|
4785
|
-
const dirs = parts.slice(0, -1).filter((d) => d !== "evals");
|
|
4786
|
-
return dirs.length > 0 ? dirs.join("/") : DEFAULT_CATEGORY;
|
|
4787
|
-
}
|
|
4788
|
-
var OtelTraceExporter = class {
|
|
4789
|
-
constructor(options) {
|
|
4790
|
-
this.options = options;
|
|
4791
|
-
}
|
|
4792
|
-
provider = null;
|
|
4793
|
-
tracer = null;
|
|
4794
|
-
api = null;
|
|
4795
|
-
// biome-ignore lint/suspicious/noExplicitAny: OTel types loaded dynamically
|
|
4796
|
-
W3CPropagator = null;
|
|
4797
|
-
/** Initialize the OTel SDK. Returns false if OTel packages are not available. */
|
|
4798
|
-
async init() {
|
|
4799
|
-
try {
|
|
4800
|
-
const [sdkTraceNode, resourcesMod, semconvMod, api, coreMod] = await Promise.all([
|
|
4801
|
-
import("./src-76ISS3LH.js"),
|
|
4802
|
-
import("./esm-NNMQYIKT.js"),
|
|
4803
|
-
import("./esm-RVQPUGWH.js"),
|
|
4804
|
-
import("./esm-R77SNOF5.js"),
|
|
4805
|
-
import("./esm-ZADQ4XQH-QUFGGZL4.js").catch(() => null)
|
|
4806
|
-
]);
|
|
4807
|
-
const { NodeTracerProvider: Provider, SimpleSpanProcessor } = sdkTraceNode;
|
|
4808
|
-
const { resourceFromAttributes } = resourcesMod;
|
|
4809
|
-
const { ATTR_SERVICE_NAME } = semconvMod;
|
|
4810
|
-
const resourceAttributes = {
|
|
4811
|
-
[ATTR_SERVICE_NAME]: this.options.serviceName ?? "agentv",
|
|
4812
|
-
...this.options.resourceAttributes
|
|
4813
|
-
};
|
|
4814
|
-
const resource = resourceFromAttributes(resourceAttributes);
|
|
4815
|
-
const processors = [];
|
|
4816
|
-
if (this.options.endpoint) {
|
|
4817
|
-
const otlpHttp = await import("./esm-TTEMJEVV.js");
|
|
4818
|
-
const { OTLPTraceExporter } = otlpHttp;
|
|
4819
|
-
const exporter = new OTLPTraceExporter({
|
|
4820
|
-
url: this.options.endpoint,
|
|
4821
|
-
headers: this.options.headers
|
|
4822
|
-
});
|
|
4823
|
-
processors.push(new SimpleSpanProcessor(exporter));
|
|
4824
|
-
}
|
|
4825
|
-
if (this.options.otlpFilePath) {
|
|
4826
|
-
const { OtlpJsonFileExporter: OtlpJsonFileExporter2 } = await import("./otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js");
|
|
4827
|
-
processors.push(
|
|
4828
|
-
new SimpleSpanProcessor(
|
|
4829
|
-
new OtlpJsonFileExporter2(this.options.otlpFilePath, resourceAttributes)
|
|
4830
|
-
)
|
|
4831
|
-
);
|
|
4832
|
-
}
|
|
4833
|
-
if (processors.length === 0) {
|
|
4834
|
-
return false;
|
|
4835
|
-
}
|
|
4836
|
-
this.provider = new Provider({
|
|
4837
|
-
resource,
|
|
4838
|
-
spanProcessors: processors
|
|
4839
|
-
});
|
|
4840
|
-
this.provider.register();
|
|
4841
|
-
this.api = api;
|
|
4842
|
-
this.tracer = api.trace.getTracer("agentv", "1.0.0");
|
|
4843
|
-
this.W3CPropagator = coreMod?.W3CTraceContextPropagator ?? null;
|
|
4844
|
-
return true;
|
|
4845
|
-
} catch {
|
|
4846
|
-
return false;
|
|
4847
|
-
}
|
|
4848
|
-
}
|
|
4849
|
-
/** Export a single evaluation result as an OTel trace. */
|
|
4850
|
-
async exportResult(result) {
|
|
4851
|
-
if (!this.tracer || !this.api) return;
|
|
4852
|
-
const api = this.api;
|
|
4853
|
-
const tracer = this.tracer;
|
|
4854
|
-
const captureContent = this.options.captureContent ?? false;
|
|
4855
|
-
const startHr = toHrTime(result.startTime ?? result.timestamp);
|
|
4856
|
-
const endHr = toHrTime(result.endTime ?? result.timestamp);
|
|
4857
|
-
let parentCtx = api.ROOT_CONTEXT;
|
|
4858
|
-
const traceparent = process.env.TRACEPARENT;
|
|
4859
|
-
if (traceparent && this.W3CPropagator) {
|
|
4860
|
-
try {
|
|
4861
|
-
const propagator = new this.W3CPropagator();
|
|
4862
|
-
parentCtx = propagator.extract(
|
|
4863
|
-
api.ROOT_CONTEXT,
|
|
4864
|
-
{ traceparent, tracestate: process.env.TRACESTATE ?? "" },
|
|
4865
|
-
{
|
|
4866
|
-
get: (carrier, key) => carrier[key],
|
|
4867
|
-
keys: (carrier) => Object.keys(carrier)
|
|
4868
|
-
}
|
|
4869
|
-
);
|
|
4870
|
-
} catch {
|
|
4871
|
-
}
|
|
4872
|
-
}
|
|
4873
|
-
tracer.startActiveSpan(
|
|
4874
|
-
"agentv.eval",
|
|
4875
|
-
{ startTime: startHr },
|
|
4876
|
-
parentCtx,
|
|
4877
|
-
(rootSpan) => {
|
|
4878
|
-
rootSpan.setAttribute("gen_ai.operation.name", "evaluate");
|
|
4879
|
-
rootSpan.setAttribute("gen_ai.system", "agentv");
|
|
4880
|
-
rootSpan.setAttribute("agentv.test_id", result.testId);
|
|
4881
|
-
rootSpan.setAttribute("agentv.target", result.target);
|
|
4882
|
-
if (result.suite) rootSpan.setAttribute("agentv.suite", result.suite);
|
|
4883
|
-
rootSpan.setAttribute("agentv.score", result.score);
|
|
4884
|
-
if (captureContent && result.output.length > 0) {
|
|
4885
|
-
rootSpan.setAttribute("agentv.output_text", result.output);
|
|
4886
|
-
}
|
|
4887
|
-
if (result.durationMs != null)
|
|
4888
|
-
rootSpan.setAttribute("agentv.trace.duration_ms", result.durationMs);
|
|
4889
|
-
if (result.costUsd != null) rootSpan.setAttribute("agentv.trace.cost_usd", result.costUsd);
|
|
4890
|
-
if (result.tokenUsage) {
|
|
4891
|
-
if (result.tokenUsage.input != null) {
|
|
4892
|
-
rootSpan.setAttribute("agentv.trace.token_input", result.tokenUsage.input);
|
|
4893
|
-
}
|
|
4894
|
-
if (result.tokenUsage.output != null) {
|
|
4895
|
-
rootSpan.setAttribute("agentv.trace.token_output", result.tokenUsage.output);
|
|
4896
|
-
}
|
|
4897
|
-
if (result.tokenUsage.cached != null) {
|
|
4898
|
-
rootSpan.setAttribute("agentv.trace.token_cached", result.tokenUsage.cached);
|
|
4899
|
-
}
|
|
4900
|
-
}
|
|
4901
|
-
if (result.trace) {
|
|
4902
|
-
const t = result.trace;
|
|
4903
|
-
rootSpan.setAttribute("agentv.trace.event_count", t.eventCount);
|
|
4904
|
-
rootSpan.setAttribute(
|
|
4905
|
-
"agentv.trace.tool_names",
|
|
4906
|
-
Object.keys(t.toolCalls).sort().join(",")
|
|
4907
|
-
);
|
|
4908
|
-
if (t.llmCallCount != null)
|
|
4909
|
-
rootSpan.setAttribute("agentv.trace.llm_call_count", t.llmCallCount);
|
|
4910
|
-
}
|
|
4911
|
-
const traceMessages = result.trace?.messages ?? [];
|
|
4912
|
-
if (traceMessages.length > 0) {
|
|
4913
|
-
const parentCtx2 = api.trace.setSpan(api.context.active(), rootSpan);
|
|
4914
|
-
if (this.options.groupTurns) {
|
|
4915
|
-
const turns = groupMessagesIntoTurns(traceMessages);
|
|
4916
|
-
if (turns.length > 1) {
|
|
4917
|
-
for (const [i, turn] of turns.entries()) {
|
|
4918
|
-
api.context.with(parentCtx2, () => {
|
|
4919
|
-
tracer.startActiveSpan(
|
|
4920
|
-
`agentv.turn.${i + 1}`,
|
|
4921
|
-
{},
|
|
4922
|
-
(turnSpan) => {
|
|
4923
|
-
const turnCtx = api.trace.setSpan(api.context.active(), turnSpan);
|
|
4924
|
-
for (const msg of turn.messages) {
|
|
4925
|
-
this.exportMessage(tracer, api, turnCtx, msg, captureContent);
|
|
4926
|
-
}
|
|
4927
|
-
turnSpan.end();
|
|
4928
|
-
}
|
|
4929
|
-
);
|
|
4930
|
-
});
|
|
4931
|
-
}
|
|
4932
|
-
} else {
|
|
4933
|
-
for (const msg of traceMessages) {
|
|
4934
|
-
this.exportMessage(tracer, api, parentCtx2, msg, captureContent);
|
|
4935
|
-
}
|
|
4936
|
-
}
|
|
4937
|
-
} else {
|
|
4938
|
-
for (const msg of traceMessages) {
|
|
4939
|
-
this.exportMessage(tracer, api, parentCtx2, msg, captureContent);
|
|
4940
|
-
}
|
|
4941
|
-
}
|
|
4942
|
-
}
|
|
4943
|
-
if (result.scores) {
|
|
4944
|
-
for (const score of result.scores) {
|
|
4945
|
-
rootSpan.addEvent(`agentv.grader.${score.name}`, {
|
|
4946
|
-
"agentv.grader.score": score.score,
|
|
4947
|
-
"agentv.grader.type": score.type,
|
|
4948
|
-
...score.verdict ? { "agentv.grader.verdict": score.verdict } : {}
|
|
4949
|
-
});
|
|
4950
|
-
}
|
|
4951
|
-
}
|
|
4952
|
-
if (result.error) {
|
|
4953
|
-
rootSpan.setStatus({ code: api.SpanStatusCode.ERROR, message: result.error });
|
|
4954
|
-
} else {
|
|
4955
|
-
rootSpan.setStatus({ code: api.SpanStatusCode.OK });
|
|
4956
|
-
}
|
|
4957
|
-
rootSpan.end(endHr);
|
|
4958
|
-
}
|
|
4959
|
-
);
|
|
4960
|
-
}
|
|
4961
|
-
/** Flush pending spans and shut down. */
|
|
4962
|
-
async shutdown() {
|
|
4963
|
-
await this.provider?.shutdown();
|
|
4964
|
-
}
|
|
4965
|
-
/** Create a streaming observer for real-time span export */
|
|
4966
|
-
createStreamingObserver() {
|
|
4967
|
-
if (!this.tracer || !this.api) return null;
|
|
4968
|
-
let parentCtx;
|
|
4969
|
-
const traceparent = process.env.TRACEPARENT;
|
|
4970
|
-
if (traceparent && this.W3CPropagator) {
|
|
4971
|
-
try {
|
|
4972
|
-
const propagator = new this.W3CPropagator();
|
|
4973
|
-
parentCtx = propagator.extract(
|
|
4974
|
-
this.api.ROOT_CONTEXT,
|
|
4975
|
-
{ traceparent, tracestate: process.env.TRACESTATE ?? "" },
|
|
4976
|
-
{
|
|
4977
|
-
get: (carrier, key) => carrier[key],
|
|
4978
|
-
keys: (carrier) => Object.keys(carrier)
|
|
4979
|
-
}
|
|
4980
|
-
);
|
|
4981
|
-
} catch {
|
|
4982
|
-
}
|
|
4983
|
-
}
|
|
4984
|
-
return new OtelStreamingObserver(
|
|
4985
|
-
this.tracer,
|
|
4986
|
-
this.api,
|
|
4987
|
-
this.options.captureContent ?? false,
|
|
4988
|
-
parentCtx
|
|
4989
|
-
);
|
|
4990
|
-
}
|
|
4991
|
-
// -----------------------------------------------------------------------
|
|
4992
|
-
// Private helpers
|
|
4993
|
-
// -----------------------------------------------------------------------
|
|
4994
|
-
exportMessage(tracer, api, parentCtx, msg, captureContent) {
|
|
4995
|
-
const isAssistant = msg.role === "assistant";
|
|
4996
|
-
const model = msg.metadata?.model ? String(msg.metadata.model) : void 0;
|
|
4997
|
-
const spanName = isAssistant ? `chat ${model ?? "unknown"}` : `gen_ai.message.${msg.role}`;
|
|
4998
|
-
const startHr = toHrTime(msg.startTime);
|
|
4999
|
-
const endHr = toHrTime(msg.endTime);
|
|
5000
|
-
api.context.with(parentCtx, () => {
|
|
5001
|
-
tracer.startActiveSpan(
|
|
5002
|
-
spanName,
|
|
5003
|
-
{ startTime: startHr },
|
|
5004
|
-
parentCtx,
|
|
5005
|
-
(span) => {
|
|
5006
|
-
if (isAssistant) {
|
|
5007
|
-
span.setAttribute("gen_ai.operation.name", "chat");
|
|
5008
|
-
}
|
|
5009
|
-
if (model) {
|
|
5010
|
-
span.setAttribute("gen_ai.request.model", model);
|
|
5011
|
-
span.setAttribute("gen_ai.response.model", model);
|
|
5012
|
-
}
|
|
5013
|
-
if (msg.tokenUsage) {
|
|
5014
|
-
if (msg.tokenUsage.input != null) {
|
|
5015
|
-
span.setAttribute("gen_ai.usage.input_tokens", msg.tokenUsage.input);
|
|
5016
|
-
}
|
|
5017
|
-
if (msg.tokenUsage.output != null) {
|
|
5018
|
-
span.setAttribute("gen_ai.usage.output_tokens", msg.tokenUsage.output);
|
|
5019
|
-
}
|
|
5020
|
-
if (msg.tokenUsage.cached != null) {
|
|
5021
|
-
span.setAttribute("gen_ai.usage.cache_read.input_tokens", msg.tokenUsage.cached);
|
|
5022
|
-
}
|
|
5023
|
-
}
|
|
5024
|
-
if (captureContent && msg.content != null) {
|
|
5025
|
-
span.setAttribute(
|
|
5026
|
-
"gen_ai.output.messages",
|
|
5027
|
-
typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
|
|
5028
|
-
);
|
|
5029
|
-
}
|
|
5030
|
-
if (msg.toolCalls) {
|
|
5031
|
-
const msgCtx = api.trace.setSpan(api.context.active(), span);
|
|
5032
|
-
for (const tc of msg.toolCalls) {
|
|
5033
|
-
api.context.with(msgCtx, () => {
|
|
5034
|
-
tracer.startActiveSpan(
|
|
5035
|
-
`execute_tool ${tc.tool}`,
|
|
5036
|
-
{},
|
|
5037
|
-
msgCtx,
|
|
5038
|
-
(toolSpan) => {
|
|
5039
|
-
toolSpan.setAttribute("gen_ai.tool.name", tc.tool);
|
|
5040
|
-
if (tc.id) toolSpan.setAttribute("gen_ai.tool.call.id", tc.id);
|
|
5041
|
-
if (captureContent) {
|
|
5042
|
-
if (tc.input != null) {
|
|
5043
|
-
toolSpan.setAttribute(
|
|
5044
|
-
"gen_ai.tool.call.arguments",
|
|
5045
|
-
typeof tc.input === "string" ? tc.input : JSON.stringify(tc.input)
|
|
5046
|
-
);
|
|
5047
|
-
}
|
|
5048
|
-
if (tc.output != null) {
|
|
5049
|
-
toolSpan.setAttribute(
|
|
5050
|
-
"gen_ai.tool.call.result",
|
|
5051
|
-
typeof tc.output === "string" ? tc.output : JSON.stringify(tc.output)
|
|
5052
|
-
);
|
|
5053
|
-
}
|
|
5054
|
-
}
|
|
5055
|
-
toolSpan.end();
|
|
5056
|
-
}
|
|
5057
|
-
);
|
|
5058
|
-
});
|
|
5059
|
-
}
|
|
5060
|
-
}
|
|
5061
|
-
span.end(endHr);
|
|
5062
|
-
}
|
|
5063
|
-
);
|
|
5064
|
-
});
|
|
5065
|
-
}
|
|
5066
|
-
};
|
|
5067
|
-
var OtelStreamingObserver = class {
|
|
5068
|
-
constructor(tracer, api, captureContent, parentCtx) {
|
|
5069
|
-
this.tracer = tracer;
|
|
5070
|
-
this.api = api;
|
|
5071
|
-
this.captureContent = captureContent;
|
|
5072
|
-
this.parentCtx = parentCtx;
|
|
5073
|
-
}
|
|
5074
|
-
// biome-ignore lint/suspicious/noExplicitAny: OTel span type loaded dynamically
|
|
5075
|
-
rootSpan = null;
|
|
5076
|
-
// biome-ignore lint/suspicious/noExplicitAny: OTel context loaded dynamically
|
|
5077
|
-
rootCtx = null;
|
|
5078
|
-
observedChildSpans = false;
|
|
5079
|
-
pendingMetrics = null;
|
|
5080
|
-
/** Create root eval span immediately (visible in backend right away) */
|
|
5081
|
-
startEvalCase(testId, target, evalSet) {
|
|
5082
|
-
this.pendingMetrics = null;
|
|
5083
|
-
this.observedChildSpans = false;
|
|
5084
|
-
const ctx = this.parentCtx ?? this.api.context.active();
|
|
5085
|
-
this.rootSpan = this.tracer.startSpan("agentv.eval", void 0, ctx);
|
|
5086
|
-
this.rootSpan.setAttribute("gen_ai.operation.name", "evaluate");
|
|
5087
|
-
this.rootSpan.setAttribute("gen_ai.system", "agentv");
|
|
5088
|
-
this.rootSpan.setAttribute("agentv.test_id", testId);
|
|
5089
|
-
this.rootSpan.setAttribute("agentv.target", target);
|
|
5090
|
-
if (evalSet) this.rootSpan.setAttribute("agentv.suite", evalSet);
|
|
5091
|
-
this.rootCtx = this.api.trace.setSpan(this.api.context.active(), this.rootSpan);
|
|
5092
|
-
}
|
|
5093
|
-
/** Create and immediately export a tool span */
|
|
5094
|
-
onToolCall(name, input, output, _durationMs, toolCallId) {
|
|
5095
|
-
if (!this.rootCtx) return;
|
|
5096
|
-
this.observedChildSpans = true;
|
|
5097
|
-
this.api.context.with(this.rootCtx, () => {
|
|
5098
|
-
const span = this.tracer.startSpan(`execute_tool ${name}`, void 0, this.rootCtx);
|
|
5099
|
-
span.setAttribute("gen_ai.tool.name", name);
|
|
5100
|
-
if (toolCallId) span.setAttribute("gen_ai.tool.call.id", toolCallId);
|
|
5101
|
-
if (this.captureContent) {
|
|
5102
|
-
if (input != null)
|
|
5103
|
-
span.setAttribute(
|
|
5104
|
-
"gen_ai.tool.call.arguments",
|
|
5105
|
-
typeof input === "string" ? input : JSON.stringify(input)
|
|
5106
|
-
);
|
|
5107
|
-
if (output != null)
|
|
5108
|
-
span.setAttribute(
|
|
5109
|
-
"gen_ai.tool.call.result",
|
|
5110
|
-
typeof output === "string" ? output : JSON.stringify(output)
|
|
5111
|
-
);
|
|
5112
|
-
}
|
|
5113
|
-
span.end();
|
|
5114
|
-
});
|
|
5115
|
-
}
|
|
5116
|
-
/** Create and immediately export an LLM span */
|
|
5117
|
-
onLlmCall(model, tokenUsage) {
|
|
5118
|
-
if (!this.rootCtx) return;
|
|
5119
|
-
this.observedChildSpans = true;
|
|
5120
|
-
this.api.context.with(this.rootCtx, () => {
|
|
5121
|
-
const span = this.tracer.startSpan(`chat ${model}`, void 0, this.rootCtx);
|
|
5122
|
-
span.setAttribute("gen_ai.operation.name", "chat");
|
|
5123
|
-
span.setAttribute("gen_ai.request.model", model);
|
|
5124
|
-
span.setAttribute("gen_ai.response.model", model);
|
|
5125
|
-
if (tokenUsage) {
|
|
5126
|
-
if (tokenUsage.input != null)
|
|
5127
|
-
span.setAttribute("gen_ai.usage.input_tokens", tokenUsage.input);
|
|
5128
|
-
if (tokenUsage.output != null)
|
|
5129
|
-
span.setAttribute("gen_ai.usage.output_tokens", tokenUsage.output);
|
|
5130
|
-
if (tokenUsage.cached != null)
|
|
5131
|
-
span.setAttribute("gen_ai.usage.cache_read.input_tokens", tokenUsage.cached);
|
|
5132
|
-
}
|
|
5133
|
-
span.end();
|
|
5134
|
-
});
|
|
5135
|
-
}
|
|
5136
|
-
/** Record final execution metrics before the root span is finalized. */
|
|
5137
|
-
recordEvalMetrics(result) {
|
|
5138
|
-
this.pendingMetrics = result;
|
|
5139
|
-
}
|
|
5140
|
-
/** Finalize root span with score/verdict after evaluation completes */
|
|
5141
|
-
finalizeEvalCase(score, error) {
|
|
5142
|
-
if (!this.rootSpan) return;
|
|
5143
|
-
this.rootSpan.setAttribute("agentv.score", score);
|
|
5144
|
-
if (this.pendingMetrics?.durationMs != null) {
|
|
5145
|
-
this.rootSpan.setAttribute("agentv.trace.duration_ms", this.pendingMetrics.durationMs);
|
|
5146
|
-
}
|
|
5147
|
-
if (this.pendingMetrics?.costUsd != null) {
|
|
5148
|
-
this.rootSpan.setAttribute("agentv.trace.cost_usd", this.pendingMetrics.costUsd);
|
|
5149
|
-
}
|
|
5150
|
-
if (this.pendingMetrics?.tokenUsage) {
|
|
5151
|
-
if (this.pendingMetrics.tokenUsage.input != null) {
|
|
5152
|
-
this.rootSpan.setAttribute(
|
|
5153
|
-
"agentv.trace.token_input",
|
|
5154
|
-
this.pendingMetrics.tokenUsage.input
|
|
5155
|
-
);
|
|
5156
|
-
}
|
|
5157
|
-
if (this.pendingMetrics.tokenUsage.output != null) {
|
|
5158
|
-
this.rootSpan.setAttribute(
|
|
5159
|
-
"agentv.trace.token_output",
|
|
5160
|
-
this.pendingMetrics.tokenUsage.output
|
|
5161
|
-
);
|
|
5162
|
-
}
|
|
5163
|
-
if (this.pendingMetrics.tokenUsage.cached != null) {
|
|
5164
|
-
this.rootSpan.setAttribute(
|
|
5165
|
-
"agentv.trace.token_cached",
|
|
5166
|
-
this.pendingMetrics.tokenUsage.cached
|
|
5167
|
-
);
|
|
5168
|
-
}
|
|
5169
|
-
}
|
|
5170
|
-
if (this.pendingMetrics?.trace) {
|
|
5171
|
-
this.rootSpan.setAttribute("agentv.trace.event_count", this.pendingMetrics.trace.eventCount);
|
|
5172
|
-
this.rootSpan.setAttribute(
|
|
5173
|
-
"agentv.trace.tool_names",
|
|
5174
|
-
Object.keys(this.pendingMetrics.trace.toolCalls).sort().join(",")
|
|
5175
|
-
);
|
|
5176
|
-
if (this.pendingMetrics.trace.llmCallCount != null) {
|
|
5177
|
-
this.rootSpan.setAttribute(
|
|
5178
|
-
"agentv.trace.llm_call_count",
|
|
5179
|
-
this.pendingMetrics.trace.llmCallCount
|
|
5180
|
-
);
|
|
5181
|
-
}
|
|
5182
|
-
}
|
|
5183
|
-
if (error) {
|
|
5184
|
-
this.rootSpan.setStatus({ code: this.api.SpanStatusCode.ERROR, message: error });
|
|
5185
|
-
} else {
|
|
5186
|
-
this.rootSpan.setStatus({ code: this.api.SpanStatusCode.OK });
|
|
5187
|
-
}
|
|
5188
|
-
this.rootSpan.end();
|
|
5189
|
-
this.rootSpan = null;
|
|
5190
|
-
this.rootCtx = null;
|
|
5191
|
-
this.observedChildSpans = false;
|
|
5192
|
-
this.pendingMetrics = null;
|
|
5193
|
-
}
|
|
5194
|
-
/** Backfill child spans from the completed result when the provider emitted no live callbacks. */
|
|
5195
|
-
completeFromResult(result) {
|
|
5196
|
-
this.recordEvalMetrics({
|
|
5197
|
-
durationMs: result.durationMs,
|
|
5198
|
-
costUsd: result.costUsd,
|
|
5199
|
-
tokenUsage: result.tokenUsage,
|
|
5200
|
-
trace: result.trace
|
|
5201
|
-
});
|
|
5202
|
-
if (this.observedChildSpans || !this.rootCtx) {
|
|
5203
|
-
return;
|
|
5204
|
-
}
|
|
5205
|
-
const model = result.trace.messages.find((msg) => msg.role === "assistant")?.metadata?.model ?? result.target ?? "unknown";
|
|
5206
|
-
this.onLlmCall(String(model), result.tokenUsage);
|
|
5207
|
-
for (const message of result.trace.messages) {
|
|
5208
|
-
for (const toolCall of message.toolCalls ?? []) {
|
|
5209
|
-
this.onToolCall(
|
|
5210
|
-
toolCall.tool,
|
|
5211
|
-
toolCall.input,
|
|
5212
|
-
toolCall.output,
|
|
5213
|
-
toolCall.durationMs ?? 0,
|
|
5214
|
-
toolCall.id
|
|
5215
|
-
);
|
|
5216
|
-
}
|
|
5217
|
-
}
|
|
5218
|
-
}
|
|
5219
|
-
/** Return the active eval span's trace ID and span ID for Braintrust trace bridging */
|
|
5220
|
-
getActiveSpanIds() {
|
|
5221
|
-
if (!this.rootSpan) return null;
|
|
5222
|
-
try {
|
|
5223
|
-
const spanCtx = this.rootSpan.spanContext?.() ?? this.rootSpan._spanContext;
|
|
5224
|
-
if (!spanCtx?.traceId || !spanCtx?.spanId) return null;
|
|
5225
|
-
return { parentSpanId: spanCtx.spanId, rootSpanId: spanCtx.traceId };
|
|
5226
|
-
} catch {
|
|
5227
|
-
return null;
|
|
5228
|
-
}
|
|
5229
|
-
}
|
|
5230
|
-
/** Get ProviderStreamCallbacks for passing to providers */
|
|
5231
|
-
getStreamCallbacks() {
|
|
5232
|
-
return {
|
|
5233
|
-
onToolCallEnd: (name, input, output, durationMs, toolCallId) => this.onToolCall(name, input, output, durationMs, toolCallId),
|
|
5234
|
-
onLlmCallEnd: (model, tokenUsage) => this.onLlmCall(model, tokenUsage),
|
|
5235
|
-
getActiveSpanIds: () => this.getActiveSpanIds()
|
|
5236
|
-
};
|
|
5237
|
-
}
|
|
5238
|
-
};
|
|
5239
|
-
function groupMessagesIntoTurns(messages) {
|
|
5240
|
-
const turns = [];
|
|
5241
|
-
let current = [];
|
|
5242
|
-
for (const msg of messages) {
|
|
5243
|
-
if (msg.role === "user" && current.length > 0) {
|
|
5244
|
-
turns.push({ messages: current });
|
|
5245
|
-
current = [];
|
|
5246
|
-
}
|
|
5247
|
-
current.push(msg);
|
|
5248
|
-
}
|
|
5249
|
-
if (current.length > 0) turns.push({ messages: current });
|
|
5250
|
-
return turns;
|
|
5251
|
-
}
|
|
5252
|
-
function toHrTime(iso) {
|
|
5253
|
-
if (!iso) return void 0;
|
|
5254
|
-
return new Date(iso).getTime();
|
|
5255
|
-
}
|
|
5256
5387
|
var RunBudgetTracker = class {
|
|
5257
5388
|
constructor(capUsd) {
|
|
5258
5389
|
this.capUsd = capUsd;
|
|
@@ -5432,7 +5563,7 @@ function parseClaudeSession(jsonl) {
|
|
|
5432
5563
|
durationMs = new Date(endTimestamp).getTime() - new Date(startTimestamp).getTime();
|
|
5433
5564
|
}
|
|
5434
5565
|
const source = {
|
|
5435
|
-
provider: "claude",
|
|
5566
|
+
provider: "claude-sdk",
|
|
5436
5567
|
sessionId,
|
|
5437
5568
|
projectPath,
|
|
5438
5569
|
startedAt: startTimestamp,
|
|
@@ -5482,7 +5613,7 @@ function extractAssistantContent(content) {
|
|
|
5482
5613
|
case "tool_use":
|
|
5483
5614
|
if (block.name) {
|
|
5484
5615
|
toolCalls.push(
|
|
5485
|
-
normalizeToolCall("claude", {
|
|
5616
|
+
normalizeToolCall("claude-sdk", {
|
|
5486
5617
|
tool: block.name,
|
|
5487
5618
|
input: block.input,
|
|
5488
5619
|
id: block.id
|
|
@@ -5577,7 +5708,7 @@ function parseCodexSession(jsonl) {
|
|
|
5577
5708
|
} else {
|
|
5578
5709
|
input = payload.arguments;
|
|
5579
5710
|
}
|
|
5580
|
-
const toolCall = normalizeToolCall("codex", {
|
|
5711
|
+
const toolCall = normalizeToolCall("codex-cli", {
|
|
5581
5712
|
tool: toolName,
|
|
5582
5713
|
input,
|
|
5583
5714
|
id: callId
|
|
@@ -5605,7 +5736,7 @@ function parseCodexSession(jsonl) {
|
|
|
5605
5736
|
} else {
|
|
5606
5737
|
input = payload.arguments;
|
|
5607
5738
|
}
|
|
5608
|
-
const toolCall = normalizeToolCall("codex", {
|
|
5739
|
+
const toolCall = normalizeToolCall("codex-cli", {
|
|
5609
5740
|
tool: toolName,
|
|
5610
5741
|
input,
|
|
5611
5742
|
id: callId
|
|
@@ -5793,11 +5924,13 @@ var TranscriptProvider = class _TranscriptProvider {
|
|
|
5793
5924
|
kind = "transcript";
|
|
5794
5925
|
targetName;
|
|
5795
5926
|
entries;
|
|
5927
|
+
entriesByTestId;
|
|
5796
5928
|
cursor = 0;
|
|
5797
5929
|
constructor(targetName, entries) {
|
|
5798
5930
|
this.targetName = targetName;
|
|
5799
5931
|
this.id = `transcript:${targetName}`;
|
|
5800
5932
|
this.entries = entries;
|
|
5933
|
+
this.entriesByTestId = new Map(entries.map((entry) => [entry.testId, entry]));
|
|
5801
5934
|
}
|
|
5802
5935
|
/**
|
|
5803
5936
|
* Create a TranscriptProvider from a JSONL file path.
|
|
@@ -5814,13 +5947,11 @@ var TranscriptProvider = class _TranscriptProvider {
|
|
|
5814
5947
|
get lineCount() {
|
|
5815
5948
|
return this.entries.length;
|
|
5816
5949
|
}
|
|
5817
|
-
|
|
5818
|
-
|
|
5819
|
-
|
|
5820
|
-
|
|
5821
|
-
|
|
5822
|
-
}
|
|
5823
|
-
const entry = this.entries[this.cursor++];
|
|
5950
|
+
get testIds() {
|
|
5951
|
+
return this.entries.map((entry) => entry.testId);
|
|
5952
|
+
}
|
|
5953
|
+
async invoke(request) {
|
|
5954
|
+
const entry = request.evalCaseId ? this.entryForTestId(request.evalCaseId) : this.nextPositionalEntry();
|
|
5824
5955
|
return {
|
|
5825
5956
|
output: entry.messages,
|
|
5826
5957
|
tokenUsage: entry.tokenUsage ? {
|
|
@@ -5834,7 +5965,153 @@ var TranscriptProvider = class _TranscriptProvider {
|
|
|
5834
5965
|
startTime: entry.source.startedAt
|
|
5835
5966
|
};
|
|
5836
5967
|
}
|
|
5968
|
+
entryForTestId(testId) {
|
|
5969
|
+
const entry = this.entriesByTestId.get(testId);
|
|
5970
|
+
if (entry) {
|
|
5971
|
+
return entry;
|
|
5972
|
+
}
|
|
5973
|
+
throw new Error(
|
|
5974
|
+
`Transcript replay found no entry for test_id=${testId}. Available test_id values: ${this.testIds.join(", ") || "<none>"}`
|
|
5975
|
+
);
|
|
5976
|
+
}
|
|
5977
|
+
nextPositionalEntry() {
|
|
5978
|
+
if (this.cursor >= this.entries.length) {
|
|
5979
|
+
throw new Error(
|
|
5980
|
+
`Transcript exhausted: ${this.entries.length} entr${this.entries.length === 1 ? "y" : "ies"} available but ${this.cursor + 1} invocations attempted. Each transcript entry maps to one test case.`
|
|
5981
|
+
);
|
|
5982
|
+
}
|
|
5983
|
+
return this.entries[this.cursor++];
|
|
5984
|
+
}
|
|
5837
5985
|
};
|
|
5986
|
+
function parseCopilotEvents(eventsJsonl) {
|
|
5987
|
+
const messages = [];
|
|
5988
|
+
const meta = { sessionId: "", model: "", cwd: "" };
|
|
5989
|
+
let totalInputTokens = 0;
|
|
5990
|
+
let totalOutputTokens = 0;
|
|
5991
|
+
let hasUsage = false;
|
|
5992
|
+
let startTimestamp;
|
|
5993
|
+
let endTimestamp;
|
|
5994
|
+
const toolCallsInProgress = /* @__PURE__ */ new Map();
|
|
5995
|
+
const lines = eventsJsonl.split("\n").filter((l) => l.trim().length > 0);
|
|
5996
|
+
for (const line of lines) {
|
|
5997
|
+
let event;
|
|
5998
|
+
try {
|
|
5999
|
+
event = JSON.parse(line);
|
|
6000
|
+
} catch {
|
|
6001
|
+
continue;
|
|
6002
|
+
}
|
|
6003
|
+
const eventType = event.type;
|
|
6004
|
+
if (!eventType) continue;
|
|
6005
|
+
const data = event.data ?? {};
|
|
6006
|
+
switch (eventType) {
|
|
6007
|
+
case "session.start": {
|
|
6008
|
+
meta.sessionId = String(data.sessionId ?? "");
|
|
6009
|
+
const ctx = data.context;
|
|
6010
|
+
meta.cwd = String(ctx?.cwd ?? "");
|
|
6011
|
+
meta.repository = ctx?.repository ? String(ctx.repository) : void 0;
|
|
6012
|
+
meta.branch = ctx?.branch ? String(ctx.branch) : void 0;
|
|
6013
|
+
const ts = event.timestamp ?? data.startTime;
|
|
6014
|
+
meta.startedAt = ts ? String(ts) : void 0;
|
|
6015
|
+
startTimestamp = ts ? String(ts) : void 0;
|
|
6016
|
+
break;
|
|
6017
|
+
}
|
|
6018
|
+
case "user.message": {
|
|
6019
|
+
messages.push({
|
|
6020
|
+
role: "user",
|
|
6021
|
+
content: data.content != null ? String(data.content) : ""
|
|
6022
|
+
});
|
|
6023
|
+
break;
|
|
6024
|
+
}
|
|
6025
|
+
case "assistant.message": {
|
|
6026
|
+
const toolRequests = data.toolRequests;
|
|
6027
|
+
const toolCalls = (toolRequests ?? []).map(
|
|
6028
|
+
(req) => normalizeToolCall("copilot-events", {
|
|
6029
|
+
tool: String(req.name ?? req.toolName ?? ""),
|
|
6030
|
+
input: req.arguments,
|
|
6031
|
+
id: req.toolCallId ? String(req.toolCallId) : void 0
|
|
6032
|
+
})
|
|
6033
|
+
);
|
|
6034
|
+
messages.push({
|
|
6035
|
+
role: "assistant",
|
|
6036
|
+
content: data.content != null ? String(data.content) : void 0,
|
|
6037
|
+
toolCalls: toolCalls.length > 0 ? toolCalls : void 0
|
|
6038
|
+
});
|
|
6039
|
+
break;
|
|
6040
|
+
}
|
|
6041
|
+
case "skill.invoked": {
|
|
6042
|
+
const skillName = String(data.name ?? "");
|
|
6043
|
+
messages.push({
|
|
6044
|
+
role: "assistant",
|
|
6045
|
+
toolCalls: [
|
|
6046
|
+
{
|
|
6047
|
+
tool: "Skill",
|
|
6048
|
+
input: { skill: skillName }
|
|
6049
|
+
}
|
|
6050
|
+
]
|
|
6051
|
+
});
|
|
6052
|
+
break;
|
|
6053
|
+
}
|
|
6054
|
+
case "tool.execution_start": {
|
|
6055
|
+
const toolCallId = String(data.toolCallId ?? "");
|
|
6056
|
+
if (toolCallId) {
|
|
6057
|
+
toolCallsInProgress.set(toolCallId, {
|
|
6058
|
+
toolName: String(data.toolName ?? ""),
|
|
6059
|
+
input: data.arguments,
|
|
6060
|
+
toolCallId
|
|
6061
|
+
});
|
|
6062
|
+
}
|
|
6063
|
+
break;
|
|
6064
|
+
}
|
|
6065
|
+
case "tool.execution_complete": {
|
|
6066
|
+
const toolCallId = String(data.toolCallId ?? "");
|
|
6067
|
+
const started = toolCallsInProgress.get(toolCallId);
|
|
6068
|
+
if (started) {
|
|
6069
|
+
toolCallsInProgress.delete(toolCallId);
|
|
6070
|
+
messages.push({
|
|
6071
|
+
role: "assistant",
|
|
6072
|
+
toolCalls: [
|
|
6073
|
+
normalizeToolCall("copilot-events", {
|
|
6074
|
+
tool: started.toolName,
|
|
6075
|
+
input: started.input,
|
|
6076
|
+
output: data.result,
|
|
6077
|
+
id: toolCallId
|
|
6078
|
+
})
|
|
6079
|
+
]
|
|
6080
|
+
});
|
|
6081
|
+
}
|
|
6082
|
+
break;
|
|
6083
|
+
}
|
|
6084
|
+
case "session.shutdown": {
|
|
6085
|
+
endTimestamp = event.timestamp ? String(event.timestamp) : void 0;
|
|
6086
|
+
const modelMetrics = data.modelMetrics;
|
|
6087
|
+
if (modelMetrics) {
|
|
6088
|
+
for (const metrics of Object.values(modelMetrics)) {
|
|
6089
|
+
if (metrics.usage) {
|
|
6090
|
+
hasUsage = true;
|
|
6091
|
+
totalInputTokens += Number(metrics.usage.inputTokens ?? 0);
|
|
6092
|
+
totalOutputTokens += Number(metrics.usage.outputTokens ?? 0);
|
|
6093
|
+
}
|
|
6094
|
+
}
|
|
6095
|
+
}
|
|
6096
|
+
const currentModel = data.currentModel;
|
|
6097
|
+
if (currentModel && !meta.model) {
|
|
6098
|
+
meta.model = String(currentModel);
|
|
6099
|
+
}
|
|
6100
|
+
break;
|
|
6101
|
+
}
|
|
6102
|
+
}
|
|
6103
|
+
}
|
|
6104
|
+
let durationMs;
|
|
6105
|
+
if (startTimestamp && endTimestamp) {
|
|
6106
|
+
durationMs = new Date(endTimestamp).getTime() - new Date(startTimestamp).getTime();
|
|
6107
|
+
}
|
|
6108
|
+
return {
|
|
6109
|
+
messages,
|
|
6110
|
+
meta,
|
|
6111
|
+
tokenUsage: hasUsage ? { input: totalInputTokens, output: totalOutputTokens } : void 0,
|
|
6112
|
+
durationMs
|
|
6113
|
+
};
|
|
6114
|
+
}
|
|
5838
6115
|
function createAgentKernel() {
|
|
5839
6116
|
return { status: "stub" };
|
|
5840
6117
|
}
|
|
@@ -5857,9 +6134,11 @@ export {
|
|
|
5857
6134
|
getResultsRepoLocalPaths,
|
|
5858
6135
|
ensureResultsRepoClone,
|
|
5859
6136
|
getResultsRepoStatus,
|
|
6137
|
+
buildResultsCompareUrl,
|
|
5860
6138
|
getResultsRepoSyncStatus,
|
|
5861
6139
|
syncResultsRepo,
|
|
5862
6140
|
syncResultsRepoForProject,
|
|
6141
|
+
confirmResultsMergeAndPull,
|
|
5863
6142
|
checkoutResultsRepoBranch,
|
|
5864
6143
|
prepareResultsRepoBranch,
|
|
5865
6144
|
stageResultsArtifacts,
|
|
@@ -5875,15 +6154,14 @@ export {
|
|
|
5875
6154
|
setupWipWorktree,
|
|
5876
6155
|
pushWipCheckpoint,
|
|
5877
6156
|
deleteWipBranch,
|
|
6157
|
+
resolveGitResultsIndexCacheFile,
|
|
6158
|
+
resolveGitRunsRefCommit,
|
|
6159
|
+
listGitRunsCached,
|
|
5878
6160
|
listGitRuns,
|
|
5879
6161
|
materializeGitRun,
|
|
5880
6162
|
syncProject,
|
|
5881
6163
|
syncProjects,
|
|
5882
6164
|
trimBaselineResult,
|
|
5883
|
-
DEFAULT_CATEGORY,
|
|
5884
|
-
deriveCategory,
|
|
5885
|
-
OtelTraceExporter,
|
|
5886
|
-
OtelStreamingObserver,
|
|
5887
6165
|
RunBudgetTracker,
|
|
5888
6166
|
parseEnvOutput,
|
|
5889
6167
|
runBeforeSessionHook,
|
|
@@ -5892,6 +6170,7 @@ export {
|
|
|
5892
6170
|
discoverCodexSessions,
|
|
5893
6171
|
discoverClaudeSessions,
|
|
5894
6172
|
TranscriptProvider,
|
|
6173
|
+
parseCopilotEvents,
|
|
5895
6174
|
createAgentKernel
|
|
5896
6175
|
};
|
|
5897
|
-
//# sourceMappingURL=chunk-
|
|
6176
|
+
//# sourceMappingURL=chunk-GJWPQXHN.js.map
|