agentv 5.0.0-next.1 → 5.1.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +234 -52
- package/dist/{agentv-provider-AYXH7WLW-NJRC6UQX.js → agentv-provider-AYXH7WLW-6YZKZ7PS.js} +2 -2
- package/dist/{artifact-writer-QD52FC56.js → artifact-writer-IORG7CW5.js} +15 -8
- package/dist/{chunk-IJEZFZJR.js → chunk-7BGERE6L.js} +2 -2
- package/dist/chunk-7BGERE6L.js.map +1 -0
- package/dist/{chunk-IBI7A5RP.js → chunk-AM2IPHKS.js} +3355 -2244
- package/dist/chunk-AM2IPHKS.js.map +1 -0
- package/dist/{chunk-6JVM4GJI.js → chunk-CBBBRTQ3.js} +126 -80
- package/dist/chunk-CBBBRTQ3.js.map +1 -0
- package/dist/{chunk-6Q56CEXV.js → chunk-GIRUNQGP.js} +8171 -9308
- package/dist/chunk-GIRUNQGP.js.map +1 -0
- package/dist/{chunk-UG4JHZYM.js → chunk-GJWPQXHN.js} +673 -1346
- package/dist/chunk-GJWPQXHN.js.map +1 -0
- package/dist/{chunk-RCIEHFQ2.js → chunk-PEUTJS7B.js} +3 -3
- package/dist/{chunk-5H446C7X.js → chunk-QMRVH5ZP.js} +2 -8
- package/dist/{chunk-GZJMODLW.js → chunk-ZGPCX334.js} +45022 -31528
- package/dist/chunk-ZGPCX334.js.map +1 -0
- package/dist/cli.js +8 -9
- package/dist/cli.js.map +1 -1
- package/dist/dashboard/assets/{index-D5_8DE_B.js → index-BSHoWrdr.js} +1 -1
- package/dist/dashboard/assets/index-BaLLxmVO.js +121 -0
- package/dist/dashboard/assets/index-zvUKAkKn.css +1 -0
- package/dist/dashboard/index.html +2 -2
- package/dist/{dist-KJ6PELQY.js → dist-2D2GBHZV.js} +72 -56
- package/dist/{docker-workspace-ZJ2ESVBI-6UDETQII.js → docker-workspace-NNK4Z6FR-QRTIG6TM.js} +3 -3
- package/dist/{exec-MYPAJJP7-C5HD5WG5.js → exec-4HFFRQ3B-AXDMJNLJ.js} +3 -3
- package/dist/index.js +8 -9
- package/dist/{interactive-2WMVAASS.js → interactive-V4SEHLNY.js} +13 -29
- package/dist/interactive-V4SEHLNY.js.map +1 -0
- package/dist/skills/agentv-bench/SKILL.md +24 -24
- package/dist/skills/agentv-bench/agents/analyzer.md +6 -6
- package/dist/skills/agentv-bench/agents/comparator.md +4 -5
- package/dist/skills/agentv-bench/agents/grader.md +4 -4
- package/dist/skills/agentv-bench/references/autoresearch.md +3 -3
- package/dist/skills/agentv-bench/references/description-optimization.md +2 -2
- package/dist/skills/agentv-bench/references/environment-adaptation.md +10 -10
- package/dist/skills/agentv-bench/references/eval-yaml-spec.md +42 -26
- package/dist/skills/agentv-bench/references/migrating-from-skill-creator.md +4 -4
- package/dist/skills/agentv-bench/references/subagent-pipeline.md +13 -12
- package/dist/skills/agentv-eval-migrations/SKILL.md +33 -0
- package/dist/skills/agentv-eval-migrations/references/breaking-changes.md +1013 -0
- package/dist/skills/agentv-eval-review/SKILL.md +6 -3
- package/dist/skills/agentv-eval-review/scripts/lint_eval.py +1 -1
- package/dist/skills/agentv-eval-writer/SKILL.md +328 -209
- package/dist/skills/agentv-eval-writer/references/{config-schema.json → config.schema.json} +6 -5
- package/dist/skills/agentv-eval-writer/references/custom-evaluators.md +23 -18
- package/dist/skills/agentv-eval-writer/references/eval.schema.json +10447 -0
- package/dist/skills/agentv-eval-writer/references/python-helpers.md +8 -8
- package/dist/skills/agentv-eval-writer/references/rubric-evaluator.md +21 -22
- package/dist/skills/agentv-trace-analyst/SKILL.md +4 -4
- package/dist/templates/.agentv/targets.yaml +41 -40
- package/dist/{ts-eval-loader-RE43OAGX-AMRCUKBZ.js → ts-eval-loader-SSPJVEHD-AL2A3V5Z.js} +4 -4
- package/package.json +13 -4
- package/dist/chunk-6JVM4GJI.js.map +0 -1
- package/dist/chunk-6Q56CEXV.js.map +0 -1
- package/dist/chunk-76FOHROU.js +0 -96
- package/dist/chunk-76FOHROU.js.map +0 -1
- package/dist/chunk-GZJMODLW.js.map +0 -1
- package/dist/chunk-HSHTTN6C.js +0 -565
- package/dist/chunk-HSHTTN6C.js.map +0 -1
- package/dist/chunk-IBI7A5RP.js.map +0 -1
- package/dist/chunk-IJEZFZJR.js.map +0 -1
- package/dist/chunk-LRULMAAA.js +0 -1711
- package/dist/chunk-LRULMAAA.js.map +0 -1
- package/dist/chunk-R3CXYNTN.js +0 -1238
- package/dist/chunk-R3CXYNTN.js.map +0 -1
- package/dist/chunk-UG4JHZYM.js.map +0 -1
- package/dist/chunk-VQ2ZO7XJ.js +0 -2098
- package/dist/chunk-VQ2ZO7XJ.js.map +0 -1
- package/dist/chunk-XALGXSKB.js +0 -21
- package/dist/chunk-XALGXSKB.js.map +0 -1
- package/dist/dashboard/assets/index-BVMrdN8a.js +0 -119
- package/dist/dashboard/assets/index-DZWXfWsv.css +0 -1
- package/dist/esm-NNMQYIKT.js +0 -32
- package/dist/esm-R77SNOF5.js +0 -65
- package/dist/esm-RVQPUGWH.js +0 -1207
- package/dist/esm-RVQPUGWH.js.map +0 -1
- package/dist/esm-TTEMJEVV.js +0 -933
- package/dist/esm-TTEMJEVV.js.map +0 -1
- package/dist/esm-ZADQ4XQH-QUFGGZL4.js +0 -924
- package/dist/esm-ZADQ4XQH-QUFGGZL4.js.map +0 -1
- package/dist/exec-MYPAJJP7-C5HD5WG5.js.map +0 -1
- package/dist/getMachineId-bsd-HSK5LZMG.js +0 -41
- package/dist/getMachineId-bsd-HSK5LZMG.js.map +0 -1
- package/dist/getMachineId-darwin-4DP6CCJV.js +0 -41
- package/dist/getMachineId-darwin-4DP6CCJV.js.map +0 -1
- package/dist/getMachineId-linux-44LJ5UJB.js +0 -33
- package/dist/getMachineId-linux-44LJ5UJB.js.map +0 -1
- package/dist/getMachineId-unsupported-NVK6IATM.js +0 -24
- package/dist/getMachineId-unsupported-NVK6IATM.js.map +0 -1
- package/dist/getMachineId-win-YZ36S7VA.js +0 -43
- package/dist/getMachineId-win-YZ36S7VA.js.map +0 -1
- package/dist/interactive-2WMVAASS.js.map +0 -1
- package/dist/otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js +0 -9
- package/dist/otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js.map +0 -1
- package/dist/skills/agentv-eval-writer/references/eval-schema.json +0 -17295
- package/dist/skills/agentv-eval-writer/references/experiment-schema.json +0 -278
- package/dist/src-76ISS3LH.js +0 -1733
- package/dist/src-76ISS3LH.js.map +0 -1
- package/dist/ts-eval-loader-RE43OAGX-AMRCUKBZ.js.map +0 -1
- /package/dist/{agentv-provider-AYXH7WLW-NJRC6UQX.js.map → agentv-provider-AYXH7WLW-6YZKZ7PS.js.map} +0 -0
- /package/dist/{artifact-writer-QD52FC56.js.map → artifact-writer-IORG7CW5.js.map} +0 -0
- /package/dist/{chunk-RCIEHFQ2.js.map → chunk-PEUTJS7B.js.map} +0 -0
- /package/dist/{chunk-5H446C7X.js.map → chunk-QMRVH5ZP.js.map} +0 -0
- /package/dist/{dist-KJ6PELQY.js.map → dist-2D2GBHZV.js.map} +0 -0
- /package/dist/{docker-workspace-ZJ2ESVBI-6UDETQII.js.map → docker-workspace-NNK4Z6FR-QRTIG6TM.js.map} +0 -0
- /package/dist/{esm-NNMQYIKT.js.map → exec-4HFFRQ3B-AXDMJNLJ.js.map} +0 -0
- /package/dist/{esm-R77SNOF5.js.map → ts-eval-loader-SSPJVEHD-AL2A3V5Z.js.map} +0 -0
|
@@ -21,44 +21,41 @@ import {
|
|
|
21
21
|
readTranscriptJsonl,
|
|
22
22
|
releaseSharedWorkspaceSetup,
|
|
23
23
|
resolveRepoCloneUrl
|
|
24
|
-
} from "./chunk-
|
|
24
|
+
} from "./chunk-ZGPCX334.js";
|
|
25
25
|
|
|
26
26
|
// ../../packages/core/dist/index.js
|
|
27
|
-
import { createHash } from "node:crypto";
|
|
28
|
-
import { readFile } from "node:fs/promises";
|
|
29
|
-
import path from "node:path";
|
|
30
|
-
import { pathToFileURL } from "node:url";
|
|
31
27
|
import { readFileSync } from "node:fs";
|
|
32
|
-
import
|
|
28
|
+
import path from "node:path";
|
|
33
29
|
import { randomUUID } from "node:crypto";
|
|
34
|
-
import
|
|
30
|
+
import path2 from "node:path";
|
|
35
31
|
import micromatch from "micromatch";
|
|
36
|
-
import { readFile
|
|
37
|
-
import
|
|
32
|
+
import { readFile } from "node:fs/promises";
|
|
33
|
+
import path3 from "node:path";
|
|
38
34
|
import { execFile, spawn } from "node:child_process";
|
|
39
|
-
import { createHash
|
|
35
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
40
36
|
import {
|
|
41
37
|
existsSync,
|
|
42
38
|
mkdirSync,
|
|
43
39
|
mkdtempSync,
|
|
44
40
|
readFileSync as readFileSync2,
|
|
41
|
+
realpathSync,
|
|
45
42
|
renameSync,
|
|
46
43
|
rmSync,
|
|
47
44
|
writeFileSync
|
|
48
45
|
} from "node:fs";
|
|
49
|
-
import { cp, lstat, mkdtemp, readdir, rm, stat } from "node:fs/promises";
|
|
46
|
+
import { cp, lstat, mkdtemp, readFile as readFile2, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
50
47
|
import os from "node:os";
|
|
51
|
-
import
|
|
48
|
+
import path4 from "node:path";
|
|
52
49
|
import { promisify } from "node:util";
|
|
53
50
|
import * as childProcess from "node:child_process";
|
|
54
51
|
import { existsSync as existsSync2 } from "node:fs";
|
|
55
52
|
import { spawnSync } from "node:child_process";
|
|
56
53
|
import { readdir as readdir2, stat as stat2 } from "node:fs/promises";
|
|
57
54
|
import { homedir } from "node:os";
|
|
58
|
-
import
|
|
55
|
+
import path5 from "node:path";
|
|
59
56
|
import { readdir as readdir3, stat as stat3 } from "node:fs/promises";
|
|
60
57
|
import { homedir as homedir2 } from "node:os";
|
|
61
|
-
import
|
|
58
|
+
import path6 from "node:path";
|
|
62
59
|
var TRACE_SESSION_SCHEMA_VERSION = "agentv.dashboard.trace_session.v1";
|
|
63
60
|
function isRecord(value) {
|
|
64
61
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -1482,430 +1479,7 @@ function isPrimitive(value) {
|
|
|
1482
1479
|
function dropUndefined2(value) {
|
|
1483
1480
|
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
|
|
1484
1481
|
}
|
|
1485
|
-
|
|
1486
|
-
var VALID_SANDBOXES = /* @__PURE__ */ new Set(["auto", "docker", "vercel"]);
|
|
1487
|
-
var VALID_REPEAT_STRATEGIES = /* @__PURE__ */ new Set([
|
|
1488
|
-
"pass_at_k",
|
|
1489
|
-
"mean",
|
|
1490
|
-
"confidence_interval"
|
|
1491
|
-
]);
|
|
1492
|
-
function isExperimentFileReference(value) {
|
|
1493
|
-
const trimmed = value.trim();
|
|
1494
|
-
if (!trimmed) {
|
|
1495
|
-
return false;
|
|
1496
|
-
}
|
|
1497
|
-
return trimmed.includes("/") || trimmed.includes("\\") || EXPERIMENT_FILE_EXTENSIONS.has(path.extname(trimmed).toLowerCase());
|
|
1498
|
-
}
|
|
1499
|
-
function deriveExperimentNameFromPath(filePath) {
|
|
1500
|
-
return path.basename(filePath).replace(/\.experiment\.(ya?ml|[cm]?[jt]s)$/i, "").replace(/\.(ya?ml|[cm]?[jt]s)$/i, "");
|
|
1501
|
-
}
|
|
1502
|
-
async function loadExperimentConfig(filePath) {
|
|
1503
|
-
const resolvedPath = path.resolve(filePath);
|
|
1504
|
-
const ext = path.extname(resolvedPath).toLowerCase();
|
|
1505
|
-
let rawConfig;
|
|
1506
|
-
if (ext === ".yaml" || ext === ".yml") {
|
|
1507
|
-
rawConfig = parseYamlValue(await readFile(resolvedPath, "utf8"));
|
|
1508
|
-
} else if (EXPERIMENT_FILE_EXTENSIONS.has(ext)) {
|
|
1509
|
-
const moduleUrl = pathToFileURL(resolvedPath).href;
|
|
1510
|
-
const mod = await import(moduleUrl);
|
|
1511
|
-
rawConfig = mod.default ?? mod.config ?? mod;
|
|
1512
|
-
} else {
|
|
1513
|
-
throw new Error(
|
|
1514
|
-
`Unsupported experiment file extension '${ext}'. Use .yaml, .yml, .ts, .js, .mts, or .mjs.`
|
|
1515
|
-
);
|
|
1516
|
-
}
|
|
1517
|
-
return normalizeExperimentConfig(rawConfig, { sourcePath: resolvedPath });
|
|
1518
|
-
}
|
|
1519
|
-
function normalizeExperimentConfig(rawConfig, options = {}) {
|
|
1520
|
-
if (!isRecord2(rawConfig)) {
|
|
1521
|
-
throw new Error("Experiment config must be an object.");
|
|
1522
|
-
}
|
|
1523
|
-
const name = readOptionalString(rawConfig.name, "name");
|
|
1524
|
-
const agent = readOptionalString(rawConfig.agent, "agent");
|
|
1525
|
-
const target = readOptionalString(rawConfig.target, "target");
|
|
1526
|
-
const targets = readTargets(rawConfig.targets);
|
|
1527
|
-
const model = readOptionalString(rawConfig.model, "model");
|
|
1528
|
-
const agentOptions = readOptionalRecord(rawConfig.agent_options ?? rawConfig.agentOptions);
|
|
1529
|
-
const suites = readSuites(rawConfig.suites);
|
|
1530
|
-
const scripts = readScriptArray(rawConfig.scripts, "scripts");
|
|
1531
|
-
const repeat = readRepeat(rawConfig.repeat);
|
|
1532
|
-
const runs = readOptionalPositiveInteger(rawConfig.runs, "runs");
|
|
1533
|
-
if (repeat !== void 0 && runs !== void 0) {
|
|
1534
|
-
throw new Error("Experiment repeat and runs cannot both be set. Use repeat for AgentV config.");
|
|
1535
|
-
}
|
|
1536
|
-
const earlyExit = readOptionalBoolean(rawConfig.early_exit ?? rawConfig.earlyExit, "early_exit");
|
|
1537
|
-
const timeoutSeconds = readOptionalPositiveNumber(
|
|
1538
|
-
rawConfig.timeout_seconds ?? rawConfig.timeoutSeconds,
|
|
1539
|
-
"timeout_seconds"
|
|
1540
|
-
);
|
|
1541
|
-
const workers = readOptionalPositiveInteger(rawConfig.workers, "workers");
|
|
1542
|
-
const budgetUsd = readOptionalPositiveNumber(
|
|
1543
|
-
rawConfig.budget_usd ?? rawConfig.budgetUsd,
|
|
1544
|
-
"budget_usd"
|
|
1545
|
-
);
|
|
1546
|
-
const sandbox = readOptionalSandbox(rawConfig.sandbox);
|
|
1547
|
-
const workspace = readOptionalRecord(rawConfig.workspace);
|
|
1548
|
-
const setup = readSetup(rawConfig.setup);
|
|
1549
|
-
const configWithoutFingerprint = {
|
|
1550
|
-
...name !== void 0 && { name },
|
|
1551
|
-
...agent !== void 0 && { agent },
|
|
1552
|
-
...target !== void 0 && { target },
|
|
1553
|
-
...targets !== void 0 && { targets },
|
|
1554
|
-
...model !== void 0 && { model },
|
|
1555
|
-
...agentOptions !== void 0 && { agentOptions },
|
|
1556
|
-
...suites !== void 0 && { suites },
|
|
1557
|
-
...scripts !== void 0 && { scripts },
|
|
1558
|
-
...repeat !== void 0 && { repeat },
|
|
1559
|
-
...runs !== void 0 && { runs },
|
|
1560
|
-
...earlyExit !== void 0 && { earlyExit },
|
|
1561
|
-
...timeoutSeconds !== void 0 && { timeoutSeconds },
|
|
1562
|
-
...workers !== void 0 && { workers },
|
|
1563
|
-
...budgetUsd !== void 0 && { budgetUsd },
|
|
1564
|
-
...sandbox !== void 0 && { sandbox },
|
|
1565
|
-
...workspace !== void 0 && { workspace },
|
|
1566
|
-
...setup !== void 0 && { setup },
|
|
1567
|
-
...options.sourcePath !== void 0 && { sourcePath: options.sourcePath }
|
|
1568
|
-
};
|
|
1569
|
-
return {
|
|
1570
|
-
...configWithoutFingerprint,
|
|
1571
|
-
fingerprint: fingerprintExperimentConfig(configWithoutFingerprint)
|
|
1572
|
-
};
|
|
1573
|
-
}
|
|
1574
|
-
function fingerprintExperimentConfig(config) {
|
|
1575
|
-
const stablePayload = toStableJsonValue(config);
|
|
1576
|
-
return createHash("sha256").update(JSON.stringify(stablePayload)).digest("hex");
|
|
1577
|
-
}
|
|
1578
|
-
function buildExperimentArtifactMetadata(config) {
|
|
1579
|
-
if (!config) {
|
|
1580
|
-
return void 0;
|
|
1581
|
-
}
|
|
1582
|
-
const targets = config.targets?.map((target) => typeof target === "string" ? target : target.name).filter((target) => target.trim().length > 0);
|
|
1583
|
-
return {
|
|
1584
|
-
...config.name !== void 0 && { name: config.name },
|
|
1585
|
-
...config.sourcePath !== void 0 && { source_path: config.sourcePath },
|
|
1586
|
-
...config.fingerprint !== void 0 && { fingerprint: config.fingerprint },
|
|
1587
|
-
...config.agent !== void 0 && { agent: config.agent },
|
|
1588
|
-
...config.target !== void 0 && { target: config.target },
|
|
1589
|
-
...targets && targets.length > 0 && { targets },
|
|
1590
|
-
...config.model !== void 0 && { model: config.model },
|
|
1591
|
-
...config.suites !== void 0 && { suites: config.suites.map(toSuiteArtifactMetadata) },
|
|
1592
|
-
...config.repeat !== void 0 && {
|
|
1593
|
-
repeat: {
|
|
1594
|
-
count: config.repeat.count,
|
|
1595
|
-
strategy: config.repeat.strategy,
|
|
1596
|
-
...config.repeat.costLimitUsd !== void 0 && {
|
|
1597
|
-
cost_limit_usd: config.repeat.costLimitUsd
|
|
1598
|
-
}
|
|
1599
|
-
}
|
|
1600
|
-
},
|
|
1601
|
-
...config.runs !== void 0 && { runs: config.runs },
|
|
1602
|
-
...config.earlyExit !== void 0 && { early_exit: config.earlyExit },
|
|
1603
|
-
...config.timeoutSeconds !== void 0 && { timeout_seconds: config.timeoutSeconds },
|
|
1604
|
-
...config.workers !== void 0 && { workers: config.workers },
|
|
1605
|
-
...config.budgetUsd !== void 0 && { budget_usd: config.budgetUsd },
|
|
1606
|
-
...config.sandbox !== void 0 && { sandbox: config.sandbox }
|
|
1607
|
-
};
|
|
1608
|
-
}
|
|
1609
|
-
function toSuiteArtifactMetadata(suite) {
|
|
1610
|
-
return {
|
|
1611
|
-
ref: suite.ref,
|
|
1612
|
-
...suite.select !== void 0 && {
|
|
1613
|
-
select: {
|
|
1614
|
-
test_ids: suite.select.testIds
|
|
1615
|
-
}
|
|
1616
|
-
}
|
|
1617
|
-
};
|
|
1618
|
-
}
|
|
1619
|
-
function readRepeat(raw) {
|
|
1620
|
-
if (raw === void 0) {
|
|
1621
|
-
return void 0;
|
|
1622
|
-
}
|
|
1623
|
-
if (!isRecord2(raw)) {
|
|
1624
|
-
throw new Error("Experiment repeat must be an object.");
|
|
1625
|
-
}
|
|
1626
|
-
const count = readRequiredPositiveInteger(raw.count, "repeat.count");
|
|
1627
|
-
const strategy = readOptionalRepeatStrategy(raw.strategy);
|
|
1628
|
-
const costLimitUsd = readOptionalNonNegativeNumber(
|
|
1629
|
-
raw.cost_limit_usd ?? raw.costLimitUsd,
|
|
1630
|
-
"repeat.cost_limit_usd"
|
|
1631
|
-
);
|
|
1632
|
-
return {
|
|
1633
|
-
count,
|
|
1634
|
-
strategy: strategy ?? "pass_at_k",
|
|
1635
|
-
...costLimitUsd !== void 0 && { costLimitUsd }
|
|
1636
|
-
};
|
|
1637
|
-
}
|
|
1638
|
-
function readTargets(raw) {
|
|
1639
|
-
if (raw === void 0) {
|
|
1640
|
-
return void 0;
|
|
1641
|
-
}
|
|
1642
|
-
if (!Array.isArray(raw)) {
|
|
1643
|
-
throw new Error("Experiment targets must be an array.");
|
|
1644
|
-
}
|
|
1645
|
-
return raw.map((entry, index) => {
|
|
1646
|
-
if (typeof entry === "string" && entry.trim().length > 0) {
|
|
1647
|
-
return entry.trim();
|
|
1648
|
-
}
|
|
1649
|
-
if (!isRecord2(entry)) {
|
|
1650
|
-
throw new Error(`Experiment targets[${index}] must be a string or object.`);
|
|
1651
|
-
}
|
|
1652
|
-
const name = readRequiredString(entry.name, `targets[${index}].name`);
|
|
1653
|
-
const useTarget = readOptionalString(
|
|
1654
|
-
entry.use_target ?? entry.useTarget,
|
|
1655
|
-
`targets[${index}].use_target`
|
|
1656
|
-
);
|
|
1657
|
-
const hooks = readOptionalRecord(entry.hooks);
|
|
1658
|
-
return {
|
|
1659
|
-
name,
|
|
1660
|
-
...useTarget !== void 0 && { useTarget },
|
|
1661
|
-
...hooks !== void 0 && { hooks }
|
|
1662
|
-
};
|
|
1663
|
-
});
|
|
1664
|
-
}
|
|
1665
|
-
function readSuites(raw) {
|
|
1666
|
-
if (raw === void 0) {
|
|
1667
|
-
return void 0;
|
|
1668
|
-
}
|
|
1669
|
-
if (!Array.isArray(raw)) {
|
|
1670
|
-
throw new Error("Experiment suites must be an array.");
|
|
1671
|
-
}
|
|
1672
|
-
if (raw.length === 0) {
|
|
1673
|
-
throw new Error("Experiment suites must not be empty.");
|
|
1674
|
-
}
|
|
1675
|
-
return raw.map((entry, index) => {
|
|
1676
|
-
if (!isRecord2(entry)) {
|
|
1677
|
-
throw new Error(`Experiment suites[${index}] must be an object.`);
|
|
1678
|
-
}
|
|
1679
|
-
const ref = readRequiredString(entry.ref, `suites[${index}].ref`);
|
|
1680
|
-
const select = readSuiteSelect(entry.select, `suites[${index}].select`);
|
|
1681
|
-
return {
|
|
1682
|
-
ref,
|
|
1683
|
-
...select !== void 0 && { select }
|
|
1684
|
-
};
|
|
1685
|
-
});
|
|
1686
|
-
}
|
|
1687
|
-
function readSuiteSelect(raw, location) {
|
|
1688
|
-
if (raw === void 0) {
|
|
1689
|
-
return void 0;
|
|
1690
|
-
}
|
|
1691
|
-
if (!isRecord2(raw)) {
|
|
1692
|
-
throw new Error(`Experiment ${location} must be an object.`);
|
|
1693
|
-
}
|
|
1694
|
-
const testIds = readOptionalStringArray(raw.test_ids ?? raw.testIds, `${location}.test_ids`);
|
|
1695
|
-
if (testIds === void 0) {
|
|
1696
|
-
throw new Error(`Experiment ${location}.test_ids is required when select is set.`);
|
|
1697
|
-
}
|
|
1698
|
-
return { testIds };
|
|
1699
|
-
}
|
|
1700
|
-
function readScriptArray(raw, location) {
|
|
1701
|
-
if (raw === void 0) {
|
|
1702
|
-
return void 0;
|
|
1703
|
-
}
|
|
1704
|
-
if (!Array.isArray(raw)) {
|
|
1705
|
-
throw new Error(`Experiment ${location} must be an array.`);
|
|
1706
|
-
}
|
|
1707
|
-
return raw.map((entry, index) => readScript(entry, `${location}[${index}]`));
|
|
1708
|
-
}
|
|
1709
|
-
function readSetup(raw) {
|
|
1710
|
-
if (raw === void 0) {
|
|
1711
|
-
return void 0;
|
|
1712
|
-
}
|
|
1713
|
-
if (typeof raw === "function") {
|
|
1714
|
-
return raw;
|
|
1715
|
-
}
|
|
1716
|
-
return readScriptArray(raw, "setup");
|
|
1717
|
-
}
|
|
1718
|
-
function readScript(raw, location) {
|
|
1719
|
-
if (typeof raw === "string") {
|
|
1720
|
-
const script2 = raw.trim();
|
|
1721
|
-
if (!script2) {
|
|
1722
|
-
throw new Error(`Experiment ${location} must not be empty.`);
|
|
1723
|
-
}
|
|
1724
|
-
return { script: script2 };
|
|
1725
|
-
}
|
|
1726
|
-
if (!isRecord2(raw)) {
|
|
1727
|
-
throw new Error(`Experiment ${location} must be a string or object.`);
|
|
1728
|
-
}
|
|
1729
|
-
const command = readOptionalCommand(raw.command, `${location}.command`);
|
|
1730
|
-
const script = readOptionalStringOrStringArray(raw.script, `${location}.script`);
|
|
1731
|
-
if (command === void 0 && script === void 0) {
|
|
1732
|
-
throw new Error(`Experiment ${location} must define command or script.`);
|
|
1733
|
-
}
|
|
1734
|
-
const timeoutSeconds = readOptionalPositiveNumber(
|
|
1735
|
-
raw.timeout_seconds ?? raw.timeoutSeconds,
|
|
1736
|
-
`${location}.timeout_seconds`
|
|
1737
|
-
);
|
|
1738
|
-
const cwd = readOptionalString(raw.cwd, `${location}.cwd`);
|
|
1739
|
-
const env = readOptionalStringRecord(raw.env, `${location}.env`);
|
|
1740
|
-
return {
|
|
1741
|
-
...command !== void 0 && { command },
|
|
1742
|
-
...script !== void 0 && { script },
|
|
1743
|
-
...timeoutSeconds !== void 0 && { timeoutSeconds },
|
|
1744
|
-
...cwd !== void 0 && { cwd },
|
|
1745
|
-
...env !== void 0 && { env }
|
|
1746
|
-
};
|
|
1747
|
-
}
|
|
1748
|
-
function readOptionalCommand(raw, location) {
|
|
1749
|
-
if (raw === void 0) {
|
|
1750
|
-
return void 0;
|
|
1751
|
-
}
|
|
1752
|
-
if (typeof raw === "string") {
|
|
1753
|
-
const command = raw.trim();
|
|
1754
|
-
if (!command) {
|
|
1755
|
-
throw new Error(`Experiment ${location} must not be empty.`);
|
|
1756
|
-
}
|
|
1757
|
-
return ["sh", "-c", command];
|
|
1758
|
-
}
|
|
1759
|
-
if (Array.isArray(raw) && raw.length > 0 && raw.every((entry) => typeof entry === "string" && entry.trim())) {
|
|
1760
|
-
return raw.map((entry) => entry.trim());
|
|
1761
|
-
}
|
|
1762
|
-
throw new Error(`Experiment ${location} must be a string or string array.`);
|
|
1763
|
-
}
|
|
1764
|
-
function readOptionalStringOrStringArray(raw, location) {
|
|
1765
|
-
if (raw === void 0) {
|
|
1766
|
-
return void 0;
|
|
1767
|
-
}
|
|
1768
|
-
if (typeof raw === "string") {
|
|
1769
|
-
const trimmed = raw.trim();
|
|
1770
|
-
if (!trimmed) {
|
|
1771
|
-
throw new Error(`Experiment ${location} must not be empty.`);
|
|
1772
|
-
}
|
|
1773
|
-
return trimmed;
|
|
1774
|
-
}
|
|
1775
|
-
if (Array.isArray(raw) && raw.length > 0 && raw.every((entry) => typeof entry === "string" && entry.trim())) {
|
|
1776
|
-
return raw.map((entry) => entry.trim());
|
|
1777
|
-
}
|
|
1778
|
-
throw new Error(`Experiment ${location} must be a string or string array.`);
|
|
1779
|
-
}
|
|
1780
|
-
function readOptionalStringArray(raw, location) {
|
|
1781
|
-
if (raw === void 0) {
|
|
1782
|
-
return void 0;
|
|
1783
|
-
}
|
|
1784
|
-
if (Array.isArray(raw) && raw.length > 0 && raw.every((entry) => typeof entry === "string" && entry.trim())) {
|
|
1785
|
-
return raw.map((entry) => entry.trim());
|
|
1786
|
-
}
|
|
1787
|
-
throw new Error(`Experiment ${location} must be a non-empty string array.`);
|
|
1788
|
-
}
|
|
1789
|
-
function readOptionalString(raw, location) {
|
|
1790
|
-
if (raw === void 0) {
|
|
1791
|
-
return void 0;
|
|
1792
|
-
}
|
|
1793
|
-
if (typeof raw !== "string" || raw.trim().length === 0) {
|
|
1794
|
-
throw new Error(`Experiment ${location} must be a non-empty string.`);
|
|
1795
|
-
}
|
|
1796
|
-
return raw.trim();
|
|
1797
|
-
}
|
|
1798
|
-
function readRequiredString(raw, location) {
|
|
1799
|
-
const value = readOptionalString(raw, location);
|
|
1800
|
-
if (value === void 0) {
|
|
1801
|
-
throw new Error(`Experiment ${location} is required.`);
|
|
1802
|
-
}
|
|
1803
|
-
return value;
|
|
1804
|
-
}
|
|
1805
|
-
function readRequiredPositiveInteger(raw, location) {
|
|
1806
|
-
const value = readOptionalPositiveInteger(raw, location);
|
|
1807
|
-
if (value === void 0) {
|
|
1808
|
-
throw new Error(`Experiment ${location} is required.`);
|
|
1809
|
-
}
|
|
1810
|
-
return value;
|
|
1811
|
-
}
|
|
1812
|
-
function readOptionalBoolean(raw, location) {
|
|
1813
|
-
if (raw === void 0) {
|
|
1814
|
-
return void 0;
|
|
1815
|
-
}
|
|
1816
|
-
if (typeof raw !== "boolean") {
|
|
1817
|
-
throw new Error(`Experiment ${location} must be a boolean.`);
|
|
1818
|
-
}
|
|
1819
|
-
return raw;
|
|
1820
|
-
}
|
|
1821
|
-
function readOptionalRepeatStrategy(raw) {
|
|
1822
|
-
if (raw === void 0) {
|
|
1823
|
-
return void 0;
|
|
1824
|
-
}
|
|
1825
|
-
if (typeof raw !== "string" || !VALID_REPEAT_STRATEGIES.has(raw)) {
|
|
1826
|
-
throw new Error(
|
|
1827
|
-
"Experiment repeat.strategy must be one of 'pass_at_k', 'mean', or 'confidence_interval'."
|
|
1828
|
-
);
|
|
1829
|
-
}
|
|
1830
|
-
return raw;
|
|
1831
|
-
}
|
|
1832
|
-
function readOptionalPositiveInteger(raw, location) {
|
|
1833
|
-
if (raw === void 0) {
|
|
1834
|
-
return void 0;
|
|
1835
|
-
}
|
|
1836
|
-
if (typeof raw !== "number" || !Number.isInteger(raw) || raw < 1) {
|
|
1837
|
-
throw new Error(`Experiment ${location} must be a positive integer.`);
|
|
1838
|
-
}
|
|
1839
|
-
return raw;
|
|
1840
|
-
}
|
|
1841
|
-
function readOptionalNonNegativeNumber(raw, location) {
|
|
1842
|
-
if (raw === void 0) {
|
|
1843
|
-
return void 0;
|
|
1844
|
-
}
|
|
1845
|
-
if (typeof raw !== "number" || !Number.isFinite(raw) || raw < 0) {
|
|
1846
|
-
throw new Error(`Experiment ${location} must be a non-negative number.`);
|
|
1847
|
-
}
|
|
1848
|
-
return raw;
|
|
1849
|
-
}
|
|
1850
|
-
function readOptionalPositiveNumber(raw, location) {
|
|
1851
|
-
if (raw === void 0) {
|
|
1852
|
-
return void 0;
|
|
1853
|
-
}
|
|
1854
|
-
if (typeof raw !== "number" || !Number.isFinite(raw) || raw <= 0) {
|
|
1855
|
-
throw new Error(`Experiment ${location} must be a positive number.`);
|
|
1856
|
-
}
|
|
1857
|
-
return raw;
|
|
1858
|
-
}
|
|
1859
|
-
function readOptionalSandbox(raw) {
|
|
1860
|
-
if (raw === void 0) {
|
|
1861
|
-
return void 0;
|
|
1862
|
-
}
|
|
1863
|
-
if (typeof raw !== "string" || !VALID_SANDBOXES.has(raw)) {
|
|
1864
|
-
throw new Error("Experiment sandbox must be one of 'auto', 'docker', or 'vercel'.");
|
|
1865
|
-
}
|
|
1866
|
-
return raw;
|
|
1867
|
-
}
|
|
1868
|
-
function readOptionalRecord(raw) {
|
|
1869
|
-
if (raw === void 0) {
|
|
1870
|
-
return void 0;
|
|
1871
|
-
}
|
|
1872
|
-
if (!isRecord2(raw)) {
|
|
1873
|
-
throw new Error("Experiment object field must be an object.");
|
|
1874
|
-
}
|
|
1875
|
-
return raw;
|
|
1876
|
-
}
|
|
1877
|
-
function readOptionalStringRecord(raw, location) {
|
|
1878
|
-
if (raw === void 0) {
|
|
1879
|
-
return void 0;
|
|
1880
|
-
}
|
|
1881
|
-
if (!isRecord2(raw)) {
|
|
1882
|
-
throw new Error(`Experiment ${location} must be an object.`);
|
|
1883
|
-
}
|
|
1884
|
-
const entries = Object.entries(raw);
|
|
1885
|
-
if (!entries.every((entry) => typeof entry[1] === "string")) {
|
|
1886
|
-
throw new Error(`Experiment ${location} values must be strings.`);
|
|
1887
|
-
}
|
|
1888
|
-
return Object.fromEntries(entries);
|
|
1889
|
-
}
|
|
1890
|
-
function isRecord2(value) {
|
|
1891
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1892
|
-
}
|
|
1893
|
-
function toStableJsonValue(value) {
|
|
1894
|
-
if (typeof value === "function") {
|
|
1895
|
-
return "[function]";
|
|
1896
|
-
}
|
|
1897
|
-
if (value === null || typeof value !== "object") {
|
|
1898
|
-
return value;
|
|
1899
|
-
}
|
|
1900
|
-
if (Array.isArray(value)) {
|
|
1901
|
-
return value.map(toStableJsonValue);
|
|
1902
|
-
}
|
|
1903
|
-
const record = value;
|
|
1904
|
-
return Object.fromEntries(
|
|
1905
|
-
Object.keys(record).sort().map((key) => [key, toStableJsonValue(record[key])])
|
|
1906
|
-
);
|
|
1907
|
-
}
|
|
1908
|
-
function codeGraderInstruction(graderName, description) {
|
|
1482
|
+
function scriptGraderInstruction(graderName, description) {
|
|
1909
1483
|
const desc = description ? ` This grader: ${description}.` : "";
|
|
1910
1484
|
return `Run \`agentv eval assert ${graderName} --agent-output <agent_output> --agent-input <original_prompt>\` and check the result.${desc} The command accepts --agent-output (the agent's full response text) and --agent-input (the original user prompt). It returns JSON on stdout: {"score": 0-1, "reasoning": "..."}. A score >= 0.5 means pass (exit 0); below 0.5 means fail (exit 1).`;
|
|
1911
1485
|
}
|
|
@@ -1923,21 +1497,15 @@ function assertionToNaturalLanguage(entry) {
|
|
|
1923
1497
|
switch (type) {
|
|
1924
1498
|
case "skill-trigger":
|
|
1925
1499
|
return null;
|
|
1926
|
-
case "
|
|
1927
|
-
|
|
1928
|
-
return entry.criteria;
|
|
1929
|
-
}
|
|
1930
|
-
return null;
|
|
1931
|
-
}
|
|
1500
|
+
case "llm-rubric":
|
|
1501
|
+
return typeof entry.value === "string" ? entry.value : null;
|
|
1932
1502
|
case "contains":
|
|
1933
1503
|
return `Output contains '${entry.value}'`;
|
|
1934
|
-
case "contains-any":
|
|
1935
|
-
case "contains_any": {
|
|
1504
|
+
case "contains-any": {
|
|
1936
1505
|
const values = Array.isArray(entry.value) ? entry.value.join("', '") : entry.value;
|
|
1937
1506
|
return `Output contains any of: '${values}'`;
|
|
1938
1507
|
}
|
|
1939
|
-
case "contains-all":
|
|
1940
|
-
case "contains_all": {
|
|
1508
|
+
case "contains-all": {
|
|
1941
1509
|
const values = Array.isArray(entry.value) ? entry.value.join("', '") : entry.value;
|
|
1942
1510
|
return `Output contains all of: '${values}'`;
|
|
1943
1511
|
}
|
|
@@ -1948,35 +1516,27 @@ function assertionToNaturalLanguage(entry) {
|
|
|
1948
1516
|
case "equals":
|
|
1949
1517
|
return `Output exactly equals: ${entry.value}`;
|
|
1950
1518
|
case "is-json":
|
|
1951
|
-
case "is_json":
|
|
1952
1519
|
return "Output is valid JSON";
|
|
1953
1520
|
case "starts-with":
|
|
1954
|
-
case "starts_with":
|
|
1955
1521
|
return `Output starts with '${entry.value}'`;
|
|
1956
1522
|
case "ends-with":
|
|
1957
|
-
case "ends_with":
|
|
1958
1523
|
return `Output ends with '${entry.value}'`;
|
|
1959
1524
|
case "llm-grader":
|
|
1960
|
-
case "llm_grader": {
|
|
1961
1525
|
if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
|
|
1962
1526
|
return null;
|
|
1963
1527
|
}
|
|
1964
1528
|
return typeof entry.prompt === "string" ? entry.prompt : null;
|
|
1965
|
-
|
|
1966
|
-
case "tool-trajectory":
|
|
1967
|
-
case "tool_trajectory": {
|
|
1529
|
+
case "tool-trajectory": {
|
|
1968
1530
|
const expectedArr = Array.isArray(entry.expected) ? entry.expected : [];
|
|
1969
1531
|
const tools = expectedArr.map((e) => e.tool).filter(Boolean).join(", ");
|
|
1970
1532
|
return tools ? `Agent called tools in order: ${tools}` : "Agent followed expected tool trajectory";
|
|
1971
1533
|
}
|
|
1972
|
-
case "
|
|
1973
|
-
|
|
1974
|
-
const graderName = entry.name ?? deriveGraderNameFromCommand(entry.command) ?? "code-grader";
|
|
1534
|
+
case "script": {
|
|
1535
|
+
const graderName = entry.metric ?? deriveGraderNameFromCommand(entry.command) ?? "script";
|
|
1975
1536
|
const desc = typeof entry.description === "string" ? entry.description : void 0;
|
|
1976
|
-
return
|
|
1537
|
+
return scriptGraderInstruction(graderName, desc);
|
|
1977
1538
|
}
|
|
1978
|
-
case "field-accuracy":
|
|
1979
|
-
case "field_accuracy": {
|
|
1539
|
+
case "field-accuracy": {
|
|
1980
1540
|
const fieldPaths = Array.isArray(entry.fields) ? entry.fields.map((f) => f.path).filter(Boolean).join(", ") : "";
|
|
1981
1541
|
return fieldPaths ? `Fields ${fieldPaths} match expected values` : "Fields match expected values";
|
|
1982
1542
|
}
|
|
@@ -1985,14 +1545,12 @@ function assertionToNaturalLanguage(entry) {
|
|
|
1985
1545
|
case "cost":
|
|
1986
1546
|
return typeof entry.budget === "number" ? `Cost under $${entry.budget}` : "Cost within budget";
|
|
1987
1547
|
case "token-usage":
|
|
1988
|
-
case "token_usage":
|
|
1989
1548
|
return "Token usage within limits";
|
|
1990
1549
|
case "execution-metrics":
|
|
1991
|
-
case "execution_metrics":
|
|
1992
1550
|
return "Execution within metric bounds";
|
|
1993
1551
|
default: {
|
|
1994
1552
|
if (entry.command !== void 0 && type) {
|
|
1995
|
-
return
|
|
1553
|
+
return scriptGraderInstruction(deriveGraderNameFromCommand(entry.command) ?? type);
|
|
1996
1554
|
}
|
|
1997
1555
|
if (typeof entry.criteria === "string") return entry.criteria;
|
|
1998
1556
|
if (typeof entry.prompt === "string") return entry.prompt;
|
|
@@ -2001,7 +1559,17 @@ function assertionToNaturalLanguage(entry) {
|
|
|
2001
1559
|
}
|
|
2002
1560
|
}
|
|
2003
1561
|
function assertionToNaturalLanguageList(entry) {
|
|
2004
|
-
if (entry.type === "llm-
|
|
1562
|
+
if (entry.type === "llm-rubric") {
|
|
1563
|
+
if (Array.isArray(entry.value) && entry.value.length > 0) {
|
|
1564
|
+
return entry.value.map((rubric) => {
|
|
1565
|
+
if (typeof rubric === "string") return rubric;
|
|
1566
|
+
if (!rubric || typeof rubric !== "object") return void 0;
|
|
1567
|
+
const item = rubric;
|
|
1568
|
+
return item.outcome ?? item.criteria ?? item.id;
|
|
1569
|
+
}).filter((value) => typeof value === "string");
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
if (entry.type === "llm-grader") {
|
|
2005
1573
|
if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
|
|
2006
1574
|
return entry.rubrics.map((r) => r.outcome ?? r.criteria ?? r.id).filter((s) => typeof s === "string");
|
|
2007
1575
|
}
|
|
@@ -2012,16 +1580,6 @@ function assertionToNaturalLanguageList(entry) {
|
|
|
2012
1580
|
function extractTriggerAssertions(assertions) {
|
|
2013
1581
|
return assertions.filter((a) => a.type === "skill-trigger");
|
|
2014
1582
|
}
|
|
2015
|
-
function resolveAssertions(rawCase) {
|
|
2016
|
-
if (Array.isArray(rawCase.assertions)) return rawCase.assertions;
|
|
2017
|
-
if (Array.isArray(rawCase.assert)) return rawCase.assert;
|
|
2018
|
-
return [];
|
|
2019
|
-
}
|
|
2020
|
-
function resolveSuiteAssertions(suite) {
|
|
2021
|
-
if (Array.isArray(suite.assertions)) return suite.assertions;
|
|
2022
|
-
if (Array.isArray(suite.assert)) return suite.assert;
|
|
2023
|
-
return [];
|
|
2024
|
-
}
|
|
2025
1583
|
function extractInput(rawCase) {
|
|
2026
1584
|
const files = Array.isArray(rawCase.input_files) ? rawCase.input_files.filter((f) => typeof f === "string") : [];
|
|
2027
1585
|
const input = rawCase.input;
|
|
@@ -2059,7 +1617,6 @@ function extractExpectedOutput(raw) {
|
|
|
2059
1617
|
return JSON.stringify(raw);
|
|
2060
1618
|
}
|
|
2061
1619
|
function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
2062
|
-
const warnings = [];
|
|
2063
1620
|
const files = /* @__PURE__ */ new Map();
|
|
2064
1621
|
if (typeof suite !== "object" || suite === null) {
|
|
2065
1622
|
throw new Error(`Invalid EVAL.yaml: expected an object in '${source}'`);
|
|
@@ -2068,10 +1625,7 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
|
2068
1625
|
if (!Array.isArray(rawSuite.tests)) {
|
|
2069
1626
|
throw new Error(`Invalid EVAL.yaml: missing 'tests' array in '${source}'`);
|
|
2070
1627
|
}
|
|
2071
|
-
|
|
2072
|
-
warnings.push("'assert' is deprecated at the suite level. Use 'assertions' instead.");
|
|
2073
|
-
}
|
|
2074
|
-
const suiteAssertions = resolveSuiteAssertions(rawSuite);
|
|
1628
|
+
const suiteAssertions = rawSuite.assert ?? [];
|
|
2075
1629
|
const suiteNlAssertions = suiteAssertions.filter((a) => a.type !== "skill-trigger").flatMap(assertionToNaturalLanguageList);
|
|
2076
1630
|
function getSkillFile(skillName) {
|
|
2077
1631
|
const existing = files.get(skillName);
|
|
@@ -2083,10 +1637,11 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
|
2083
1637
|
const tests = rawSuite.tests;
|
|
2084
1638
|
for (let idx = 0; idx < tests.length; idx++) {
|
|
2085
1639
|
const rawCase = tests[idx];
|
|
2086
|
-
const caseAssertions =
|
|
2087
|
-
if (rawCase.
|
|
2088
|
-
|
|
2089
|
-
|
|
1640
|
+
const caseAssertions = rawCase.assert ?? [];
|
|
1641
|
+
if (typeof rawCase.criteria === "string" && rawCase.criteria.trim() && rawCase.assert !== void 0) {
|
|
1642
|
+
throw new Error(
|
|
1643
|
+
`Invalid EVAL.yaml test '${rawCase.id ?? idx + 1}' in '${source}': do not combine test-level 'criteria' with 'assert'. Put human-readable case descriptions in 'description', or express grading text as an explicit assertion such as { type: 'llm-rubric', value: ... }.`
|
|
1644
|
+
);
|
|
2090
1645
|
}
|
|
2091
1646
|
const nlAssertions = [];
|
|
2092
1647
|
if (typeof rawCase.criteria === "string" && rawCase.criteria.trim()) {
|
|
@@ -2139,12 +1694,12 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
|
2139
1694
|
files.delete("_no-skill");
|
|
2140
1695
|
}
|
|
2141
1696
|
}
|
|
2142
|
-
return { files, warnings };
|
|
1697
|
+
return { files, warnings: [] };
|
|
2143
1698
|
}
|
|
2144
1699
|
function transpileEvalYamlFile(evalYamlPath) {
|
|
2145
1700
|
const content = readFileSync(evalYamlPath, "utf8");
|
|
2146
1701
|
const parsed = parseYamlValue(content);
|
|
2147
|
-
return transpileEvalYaml(parsed,
|
|
1702
|
+
return transpileEvalYaml(parsed, path.basename(evalYamlPath));
|
|
2148
1703
|
}
|
|
2149
1704
|
function getOutputFilenames(result) {
|
|
2150
1705
|
const names = /* @__PURE__ */ new Map();
|
|
@@ -2181,35 +1736,12 @@ function toRepoPins(repos) {
|
|
|
2181
1736
|
...repo.path !== void 0 && { path: repo.path },
|
|
2182
1737
|
...repo.repo !== void 0 && { repo: repo.repo },
|
|
2183
1738
|
...repo.commit !== void 0 && { commit: repo.commit },
|
|
2184
|
-
...repo.base_commit !== void 0 && { baseCommit: repo.base_commit },
|
|
2185
1739
|
...repo.ancestor !== void 0 && { ancestor: repo.ancestor },
|
|
2186
1740
|
...repo.sparse !== void 0 && { sparse: repo.sparse }
|
|
2187
1741
|
}));
|
|
2188
1742
|
}
|
|
2189
|
-
function poolMetadata(setup, selectedSlotPath) {
|
|
2190
|
-
const slot = setup.poolSlot ?? setup.poolSlots.find((candidate) => candidate.path === selectedSlotPath);
|
|
2191
|
-
if (!slot) {
|
|
2192
|
-
return void 0;
|
|
2193
|
-
}
|
|
2194
|
-
return {
|
|
2195
|
-
fingerprint: slot.fingerprint,
|
|
2196
|
-
slotIndex: slot.index,
|
|
2197
|
-
lockPath: slot.lockPath
|
|
2198
|
-
};
|
|
2199
|
-
}
|
|
2200
|
-
async function releaseUnselectedPoolSlots(setup, selectedSlotPath) {
|
|
2201
|
-
if (!setup.poolManager) {
|
|
2202
|
-
return;
|
|
2203
|
-
}
|
|
2204
|
-
for (const slot of setup.poolSlots) {
|
|
2205
|
-
if (slot.path !== selectedSlotPath) {
|
|
2206
|
-
await setup.poolManager.releaseSlot(slot).catch(() => {
|
|
2207
|
-
});
|
|
2208
|
-
}
|
|
2209
|
-
}
|
|
2210
|
-
}
|
|
2211
1743
|
async function prepareEvalWorkspace(options) {
|
|
2212
|
-
const evalPath =
|
|
1744
|
+
const evalPath = path2.resolve(options.testFilePath);
|
|
2213
1745
|
const evalRunId = randomUUID();
|
|
2214
1746
|
const evalCases = options.evalCases ?? await loadTests(evalPath, options.repoRoot, {
|
|
2215
1747
|
verbose: options.verbose,
|
|
@@ -2221,7 +1753,7 @@ async function prepareEvalWorkspace(options) {
|
|
|
2221
1753
|
filter: options.filter,
|
|
2222
1754
|
evalPath
|
|
2223
1755
|
});
|
|
2224
|
-
const evalDir =
|
|
1756
|
+
const evalDir = path2.dirname(evalPath);
|
|
2225
1757
|
const workers = options.maxConcurrency ?? 1;
|
|
2226
1758
|
const retainOnSuccess = options.retainOnSuccess ?? (options.keepWorkspaces ? "keep" : "cleanup");
|
|
2227
1759
|
const retainOnFailure = options.retainOnFailure ?? (options.cleanupWorkspaces ? "cleanup" : "keep");
|
|
@@ -2236,15 +1768,11 @@ async function prepareEvalWorkspace(options) {
|
|
|
2236
1768
|
evalDir,
|
|
2237
1769
|
verbose: options.verbose,
|
|
2238
1770
|
workers,
|
|
2239
|
-
poolMaxSlots: options.poolMaxSlots,
|
|
2240
1771
|
workspacePath: options.workspacePath,
|
|
2241
|
-
legacyWorkspacePath: options.workspace
|
|
2242
|
-
workspaceMode: options.workspaceMode,
|
|
2243
|
-
workspaceClean: options.workspaceClean
|
|
1772
|
+
legacyWorkspacePath: options.workspace
|
|
2244
1773
|
});
|
|
2245
|
-
const
|
|
2246
|
-
const
|
|
2247
|
-
const selectedBaselineCommit = testPoolSlot ? sharedSetup.poolSlotBaselines.get(testPoolSlot.path) : sharedSetup.sharedBaselineCommit;
|
|
1774
|
+
const selectedWorkspacePath = sharedSetup.sharedWorkspacePath;
|
|
1775
|
+
const selectedBaselineCommit = sharedSetup.sharedBaselineCommit;
|
|
2248
1776
|
const caseSetup = await prepareEvalCaseWorkspace({
|
|
2249
1777
|
evalCase,
|
|
2250
1778
|
targetName: options.target.name,
|
|
@@ -2256,6 +1784,7 @@ async function prepareEvalWorkspace(options) {
|
|
|
2256
1784
|
evalDir,
|
|
2257
1785
|
cleanupWorkspaces: options.cleanupWorkspaces,
|
|
2258
1786
|
targetHooks: options.targetHooks,
|
|
1787
|
+
sharedExtensionState: sharedSetup.extensionState,
|
|
2259
1788
|
setupDebug: options.verbose
|
|
2260
1789
|
});
|
|
2261
1790
|
if (!caseSetup.workspacePath) {
|
|
@@ -2263,8 +1792,6 @@ async function prepareEvalWorkspace(options) {
|
|
|
2263
1792
|
`No workspace was materialized for test "${evalCase.id}". Add workspace.template, workspace.repos, or workspace.hooks before preparing an external attempt.`
|
|
2264
1793
|
);
|
|
2265
1794
|
}
|
|
2266
|
-
await releaseUnselectedPoolSlots(sharedSetup, caseSetup.workspacePath);
|
|
2267
|
-
const pool = poolMetadata(sharedSetup, caseSetup.workspacePath);
|
|
2268
1795
|
return {
|
|
2269
1796
|
evalPath,
|
|
2270
1797
|
testId: evalCase.id,
|
|
@@ -2276,6 +1803,12 @@ async function prepareEvalWorkspace(options) {
|
|
|
2276
1803
|
},
|
|
2277
1804
|
createdAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
|
|
2278
1805
|
hookExecutions: [...sharedSetup.hookExecutions, ...caseSetup.hookExecutions],
|
|
1806
|
+
...caseSetup.extensionState?.providerContext !== void 0 && {
|
|
1807
|
+
providerContext: caseSetup.extensionState.providerContext
|
|
1808
|
+
},
|
|
1809
|
+
...caseSetup.extensionState?.metadata !== void 0 && {
|
|
1810
|
+
metadata: caseSetup.extensionState.metadata
|
|
1811
|
+
},
|
|
2279
1812
|
repoPins: toRepoPins(evalCase.workspace?.repos),
|
|
2280
1813
|
baseline: caseSetup.baselineCommit ? { status: "initialized", commit: caseSetup.baselineCommit } : { status: "unavailable" },
|
|
2281
1814
|
promptSource: {
|
|
@@ -2293,8 +1826,7 @@ async function prepareEvalWorkspace(options) {
|
|
|
2293
1826
|
retainOnFailure,
|
|
2294
1827
|
manualCleanup: true
|
|
2295
1828
|
},
|
|
2296
|
-
sharedWorkspace: caseSetup.isSharedWorkspace
|
|
2297
|
-
...pool !== void 0 && { pool }
|
|
1829
|
+
sharedWorkspace: caseSetup.isSharedWorkspace
|
|
2298
1830
|
};
|
|
2299
1831
|
} catch (error) {
|
|
2300
1832
|
if (sharedSetup) {
|
|
@@ -2304,22 +1836,51 @@ async function prepareEvalWorkspace(options) {
|
|
|
2304
1836
|
throw error;
|
|
2305
1837
|
}
|
|
2306
1838
|
}
|
|
1839
|
+
var ExecutionConfigSchema = external_exports.object({
|
|
1840
|
+
/** General eval parallelism (default: 3) */
|
|
1841
|
+
maxConcurrency: external_exports.number().int().min(1).max(50).optional(),
|
|
1842
|
+
/** Maximum retries on failure (default: 2) */
|
|
1843
|
+
maxRetries: external_exports.number().int().min(0).optional(),
|
|
1844
|
+
/** Agent timeout in milliseconds. No timeout if not set. */
|
|
1845
|
+
agentTimeoutMs: external_exports.number().int().min(0).optional(),
|
|
1846
|
+
/** Enable verbose logging */
|
|
1847
|
+
verbose: external_exports.boolean().optional(),
|
|
1848
|
+
/** Always keep temp workspaces after eval */
|
|
1849
|
+
keepWorkspaces: external_exports.boolean().optional()
|
|
1850
|
+
}).passthrough().superRefine((value, ctx) => {
|
|
1851
|
+
const supportedFields = /* @__PURE__ */ new Set([
|
|
1852
|
+
"maxConcurrency",
|
|
1853
|
+
"maxRetries",
|
|
1854
|
+
"agentTimeoutMs",
|
|
1855
|
+
"verbose",
|
|
1856
|
+
"keepWorkspaces"
|
|
1857
|
+
]);
|
|
1858
|
+
if ("otelFile" in value) {
|
|
1859
|
+
ctx.addIssue({
|
|
1860
|
+
code: external_exports.ZodIssueCode.custom,
|
|
1861
|
+
path: ["otelFile"],
|
|
1862
|
+
message: "execution.otelFile has been removed. Emit OpenTelemetry/OpenInference traces from the system under test or provider and correlate AgentV run artifacts with external_trace metadata."
|
|
1863
|
+
});
|
|
1864
|
+
}
|
|
1865
|
+
for (const key of Object.keys(value)) {
|
|
1866
|
+
if (!supportedFields.has(key) && key !== "otelFile") {
|
|
1867
|
+
ctx.addIssue({
|
|
1868
|
+
code: external_exports.ZodIssueCode.custom,
|
|
1869
|
+
path: [key],
|
|
1870
|
+
message: `Unsupported execution field '${key}'`
|
|
1871
|
+
});
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
}).transform(({ maxConcurrency, maxRetries, agentTimeoutMs, verbose, keepWorkspaces }) => ({
|
|
1875
|
+
...maxConcurrency !== void 0 && { maxConcurrency },
|
|
1876
|
+
...maxRetries !== void 0 && { maxRetries },
|
|
1877
|
+
...agentTimeoutMs !== void 0 && { agentTimeoutMs },
|
|
1878
|
+
...verbose !== void 0 && { verbose },
|
|
1879
|
+
...keepWorkspaces !== void 0 && { keepWorkspaces }
|
|
1880
|
+
}));
|
|
2307
1881
|
var AgentVConfigSchema = external_exports.object({
|
|
2308
1882
|
/** Default execution settings */
|
|
2309
|
-
execution:
|
|
2310
|
-
/** Number of parallel workers (default: 3) */
|
|
2311
|
-
workers: external_exports.number().int().min(1).max(50).optional(),
|
|
2312
|
-
/** Maximum retries on failure (default: 2) */
|
|
2313
|
-
maxRetries: external_exports.number().int().min(0).optional(),
|
|
2314
|
-
/** Agent timeout in milliseconds. No timeout if not set. */
|
|
2315
|
-
agentTimeoutMs: external_exports.number().int().min(0).optional(),
|
|
2316
|
-
/** Enable verbose logging */
|
|
2317
|
-
verbose: external_exports.boolean().optional(),
|
|
2318
|
-
/** Always keep temp workspaces after eval */
|
|
2319
|
-
keepWorkspaces: external_exports.boolean().optional(),
|
|
2320
|
-
/** Write OTLP JSON trace to this path (supports {timestamp} placeholder) */
|
|
2321
|
-
otelFile: external_exports.string().optional()
|
|
2322
|
-
}).optional(),
|
|
1883
|
+
execution: ExecutionConfigSchema.optional(),
|
|
2323
1884
|
/** Output settings */
|
|
2324
1885
|
output: external_exports.object({
|
|
2325
1886
|
/** Default eval run artifact directory */
|
|
@@ -2332,13 +1893,6 @@ var AgentVConfigSchema = external_exports.object({
|
|
|
2332
1893
|
/** Response cache directory */
|
|
2333
1894
|
path: external_exports.string().optional()
|
|
2334
1895
|
}).optional(),
|
|
2335
|
-
/** Compatibility shorthand for experiments.default */
|
|
2336
|
-
defaultExperiment: external_exports.string().optional(),
|
|
2337
|
-
/** Experiment defaults */
|
|
2338
|
-
experiments: external_exports.object({
|
|
2339
|
-
/** Default experiment label or path used when eval runs omit --experiment */
|
|
2340
|
-
default: external_exports.string().optional()
|
|
2341
|
-
}).strict().optional(),
|
|
2342
1896
|
/** Cost and duration limits */
|
|
2343
1897
|
limits: external_exports.object({
|
|
2344
1898
|
/** Maximum cost per run in USD */
|
|
@@ -2356,7 +1910,9 @@ var AgentVConfigSchema = external_exports.object({
|
|
|
2356
1910
|
* stderr is forwarded to the user. Non-zero exit aborts with an error.
|
|
2357
1911
|
*/
|
|
2358
1912
|
beforeSession: external_exports.string().optional()
|
|
2359
|
-
}).optional()
|
|
1913
|
+
}).optional(),
|
|
1914
|
+
/** Project-local reusable references for fields that support ref://name */
|
|
1915
|
+
refs: external_exports.record(external_exports.string().min(1)).optional()
|
|
2360
1916
|
});
|
|
2361
1917
|
function defineConfig(config) {
|
|
2362
1918
|
return AgentVConfigSchema.parse(config);
|
|
@@ -2371,7 +1927,7 @@ var CONFIG_FILE_NAMES = [
|
|
|
2371
1927
|
];
|
|
2372
1928
|
async function loadTsConfig(projectRoot) {
|
|
2373
1929
|
const { existsSync: existsSync3 } = await import("node:fs");
|
|
2374
|
-
const { pathToFileURL
|
|
1930
|
+
const { pathToFileURL } = await import("node:url");
|
|
2375
1931
|
const { join } = await import("node:path");
|
|
2376
1932
|
for (const fileName of CONFIG_FILE_NAMES) {
|
|
2377
1933
|
const filePath = join(projectRoot, fileName);
|
|
@@ -2379,7 +1935,7 @@ async function loadTsConfig(projectRoot) {
|
|
|
2379
1935
|
continue;
|
|
2380
1936
|
}
|
|
2381
1937
|
try {
|
|
2382
|
-
const fileUrl =
|
|
1938
|
+
const fileUrl = pathToFileURL(filePath).href;
|
|
2383
1939
|
const mod = await import(fileUrl);
|
|
2384
1940
|
const config = mod.default ?? mod;
|
|
2385
1941
|
return AgentVConfigSchema.parse(config);
|
|
@@ -2496,11 +2052,11 @@ async function scanRepoDeps(evalFilePaths) {
|
|
|
2496
2052
|
return { repos: [...seen.values()], errors };
|
|
2497
2053
|
}
|
|
2498
2054
|
async function extractReposFromEvalFile(filePath) {
|
|
2499
|
-
const content = await
|
|
2055
|
+
const content = await readFile(filePath, "utf8");
|
|
2500
2056
|
const parsed = interpolateEnv(parseYamlValue(content), process.env);
|
|
2501
2057
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
|
|
2502
2058
|
const obj = parsed;
|
|
2503
|
-
const evalFileDir =
|
|
2059
|
+
const evalFileDir = path3.dirname(path3.resolve(filePath));
|
|
2504
2060
|
const repos = [];
|
|
2505
2061
|
const suiteRepos = await extractReposFromWorkspaceRaw(obj.workspace, evalFileDir);
|
|
2506
2062
|
repos.push(...suiteRepos);
|
|
@@ -2516,8 +2072,8 @@ async function extractReposFromEvalFile(filePath) {
|
|
|
2516
2072
|
}
|
|
2517
2073
|
async function extractReposFromWorkspaceRaw(raw, evalFileDir) {
|
|
2518
2074
|
if (typeof raw === "string") {
|
|
2519
|
-
const workspaceFilePath =
|
|
2520
|
-
const content = await
|
|
2075
|
+
const workspaceFilePath = path3.resolve(evalFileDir, raw);
|
|
2076
|
+
const content = await readFile(workspaceFilePath, "utf8");
|
|
2521
2077
|
const parsed = interpolateEnv(parseYamlValue(content), process.env);
|
|
2522
2078
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
|
|
2523
2079
|
return extractReposFromObject(parsed);
|
|
@@ -2540,9 +2096,12 @@ function extractReposFromObject(obj) {
|
|
|
2540
2096
|
}
|
|
2541
2097
|
var execFileAsync = promisify(execFile);
|
|
2542
2098
|
var RESULTS_REPO_RESULTS_DIR = ".agentv/results";
|
|
2543
|
-
var RESULTS_REPO_RUNS_DIR = "
|
|
2544
|
-
var
|
|
2545
|
-
var
|
|
2099
|
+
var RESULTS_REPO_RUNS_DIR = ".";
|
|
2100
|
+
var RESULTS_REPO_INDEXES_DIR = ".indexes";
|
|
2101
|
+
var RESULTS_REPO_CACHE_DIR = ".cache";
|
|
2102
|
+
var RESULTS_REPO_TRACKED_DOT_DIRS = [RESULTS_REPO_INDEXES_DIR, RESULTS_REPO_CACHE_DIR];
|
|
2103
|
+
var GIT_RESULTS_INDEX_CACHE_SCHEMA_VERSION = "agentv.git_results_index_cache.v1";
|
|
2104
|
+
var GIT_RESULTS_INDEX_LAYOUT_VERSION = "agentv.results_repo_branch_root.v1";
|
|
2546
2105
|
var FALLBACK_RESULTS_REPO_COMMIT_EMAIL = "agentv@results-repo";
|
|
2547
2106
|
var FALLBACK_RESULTS_REPO_COMMIT_NAME = "AgentV Results";
|
|
2548
2107
|
var GIT_COMMIT_IDENTITY_ENV_KEYS = [
|
|
@@ -2563,121 +2122,11 @@ var GIT_EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
|
|
|
2563
2122
|
var RESULTS_REPO_GENESIS_MESSAGE = "chore(results): initialize AgentV results branch";
|
|
2564
2123
|
var RESULTS_REPO_GENESIS_DATE = "@0 +0000";
|
|
2565
2124
|
var RESULT_INDEX_FILENAME2 = "index.jsonl";
|
|
2125
|
+
var RUN_INTERNAL_DIRNAME = ".internal";
|
|
2566
2126
|
var RESULTS_REPO_GITATTRIBUTES_FILE = ".gitattributes";
|
|
2567
2127
|
var RESULTS_REPO_GITATTRIBUTES_CONTENT = `# Managed by AgentV. Artifact-aware merge so results sync never force-pushes.
|
|
2568
|
-
|
|
2569
|
-
index.jsonl merge=union
|
|
2570
|
-
# Editable run overlay (tags/feedback): 3-way JSON set/field union via the
|
|
2571
|
-
# agentv-json driver; a genuine scalar conflict falls through to a human merge.
|
|
2572
|
-
metadata/runs/**/*.json merge=agentv-json
|
|
2573
|
-
`;
|
|
2574
|
-
var RESULTS_JSON_MERGE_DRIVER_NAME = "agentv-json";
|
|
2575
|
-
var RESULTS_JSON_MERGE_DRIVER_SCRIPT = `#!/usr/bin/env node
|
|
2576
|
-
// AgentV results overlay merge driver (merge=agentv-json).
|
|
2577
|
-
//
|
|
2578
|
-
// Performs a 3-way merge of AgentV's editable JSON overlay files (run tags /
|
|
2579
|
-
// feedback). Tag lists merge as a 3-way set so concurrent add/remove are
|
|
2580
|
-
// commutative; display-only bookkeeping scalars that always differ between
|
|
2581
|
-
// writers (updated_at) are regenerated to the larger value instead of
|
|
2582
|
-
// conflicting. Content-derived concurrency tokens (tag_revision) are dropped on
|
|
2583
|
-
// merge so the reader recomputes a token matching the merged content rather than
|
|
2584
|
-
// carrying a stale token that could bypass optimistic-concurrency checks. Any
|
|
2585
|
-
// other genuinely diverging scalar exits non-zero, leaving the file conflicted
|
|
2586
|
-
// so the caller can route it to a human GitHub merge.
|
|
2587
|
-
//
|
|
2588
|
-
// Invoked by git as: node json-merge-driver.mjs %O %A %B %P
|
|
2589
|
-
// %O ancestor, %A current (overwritten with the result), %B other, %P path.
|
|
2590
|
-
import { readFileSync, writeFileSync } from 'node:fs';
|
|
2591
|
-
|
|
2592
|
-
const [, , ancestorFile, currentFile, otherFile] = process.argv;
|
|
2593
|
-
const CONFLICT = Symbol('conflict');
|
|
2594
|
-
// Scalars that legitimately differ on every write and carry no merge meaning.
|
|
2595
|
-
const REGENERABLE_SCALARS = new Set(['updated_at']);
|
|
2596
|
-
// Content-derived tokens (a hash of {tags, updated_at}) that guard optimistic
|
|
2597
|
-
// concurrency. Carrying either side's pre-merge token forward would let a stale
|
|
2598
|
-
// client whose token happens to match bypass the concurrency check and silently
|
|
2599
|
-
// overwrite the merged set, so we drop them and let the reader recompute a token
|
|
2600
|
-
// that matches the merged content.
|
|
2601
|
-
const DROP_ON_MERGE = new Set(['tag_revision']);
|
|
2602
|
-
|
|
2603
|
-
function readJson(file) {
|
|
2604
|
-
try {
|
|
2605
|
-
return JSON.parse(readFileSync(file, 'utf8'));
|
|
2606
|
-
} catch {
|
|
2607
|
-
return undefined;
|
|
2608
|
-
}
|
|
2609
|
-
}
|
|
2610
|
-
|
|
2611
|
-
function isPrimitiveArray(value) {
|
|
2612
|
-
return (
|
|
2613
|
-
Array.isArray(value) &&
|
|
2614
|
-
value.every((entry) => entry === null || ['string', 'number', 'boolean'].includes(typeof entry))
|
|
2615
|
-
);
|
|
2616
|
-
}
|
|
2617
|
-
|
|
2618
|
-
function isPlainObject(value) {
|
|
2619
|
-
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
|
2620
|
-
}
|
|
2621
|
-
|
|
2622
|
-
function mergeSet(base, a, b) {
|
|
2623
|
-
const baseSet = new Set((isPrimitiveArray(base) ? base : []).map(String));
|
|
2624
|
-
const aSet = new Set(a.map(String));
|
|
2625
|
-
const bSet = new Set(b.map(String));
|
|
2626
|
-
const removed = new Set();
|
|
2627
|
-
for (const value of baseSet) {
|
|
2628
|
-
if (!aSet.has(value) || !bSet.has(value)) removed.add(value);
|
|
2629
|
-
}
|
|
2630
|
-
const merged = [];
|
|
2631
|
-
const seen = new Set();
|
|
2632
|
-
for (const value of [...a, ...b]) {
|
|
2633
|
-
const key = String(value);
|
|
2634
|
-
if (!removed.has(key) && !seen.has(key)) {
|
|
2635
|
-
seen.add(key);
|
|
2636
|
-
merged.push(value);
|
|
2637
|
-
}
|
|
2638
|
-
}
|
|
2639
|
-
return merged;
|
|
2640
|
-
}
|
|
2641
|
-
|
|
2642
|
-
function merge3(base, a, b, key) {
|
|
2643
|
-
if (JSON.stringify(a) === JSON.stringify(b)) return a;
|
|
2644
|
-
if (JSON.stringify(a) === JSON.stringify(base)) return b;
|
|
2645
|
-
if (JSON.stringify(b) === JSON.stringify(base)) return a;
|
|
2646
|
-
if (isPrimitiveArray(a) && isPrimitiveArray(b)) return mergeSet(base, a, b);
|
|
2647
|
-
if (isPlainObject(a) && isPlainObject(b)) {
|
|
2648
|
-
const keys = new Set([
|
|
2649
|
-
...Object.keys(a),
|
|
2650
|
-
...Object.keys(b),
|
|
2651
|
-
...(isPlainObject(base) ? Object.keys(base) : []),
|
|
2652
|
-
]);
|
|
2653
|
-
const out = {};
|
|
2654
|
-
for (const childKey of keys) {
|
|
2655
|
-
if (DROP_ON_MERGE.has(childKey)) continue;
|
|
2656
|
-
const merged = merge3(
|
|
2657
|
-
isPlainObject(base) ? base[childKey] : undefined,
|
|
2658
|
-
a[childKey],
|
|
2659
|
-
b[childKey],
|
|
2660
|
-
childKey,
|
|
2661
|
-
);
|
|
2662
|
-
if (merged === CONFLICT) return CONFLICT;
|
|
2663
|
-
if (merged !== undefined) out[childKey] = merged;
|
|
2664
|
-
}
|
|
2665
|
-
return out;
|
|
2666
|
-
}
|
|
2667
|
-
if (REGENERABLE_SCALARS.has(key)) {
|
|
2668
|
-
return [a, b].filter((value) => value !== undefined).sort().pop();
|
|
2669
|
-
}
|
|
2670
|
-
return CONFLICT;
|
|
2671
|
-
}
|
|
2672
|
-
|
|
2673
|
-
const ancestor = readJson(ancestorFile) ?? {};
|
|
2674
|
-
const current = readJson(currentFile);
|
|
2675
|
-
const other = readJson(otherFile);
|
|
2676
|
-
if (current === undefined || other === undefined) process.exit(1);
|
|
2677
|
-
const result = merge3(ancestor, current, other, '');
|
|
2678
|
-
if (result === CONFLICT) process.exit(1);
|
|
2679
|
-
writeFileSync(currentFile, \`\${JSON.stringify(result, null, 2)}\\n\`);
|
|
2680
|
-
process.exit(0);
|
|
2128
|
+
${RESULTS_REPO_INDEXES_DIR}/*.jsonl merge=union
|
|
2129
|
+
*/.internal/index.jsonl merge=union
|
|
2681
2130
|
`;
|
|
2682
2131
|
var activeResultsRepoSyncs = /* @__PURE__ */ new Set();
|
|
2683
2132
|
function sanitizeRepoSlug(repo) {
|
|
@@ -2693,37 +2142,34 @@ function withFriendlyGitHubAuthError(error) {
|
|
|
2693
2142
|
}
|
|
2694
2143
|
function expandHome(p) {
|
|
2695
2144
|
if (p === "~" || p.startsWith("~/") || p.startsWith("~\\")) {
|
|
2696
|
-
return
|
|
2145
|
+
return path4.join(os.homedir(), p.slice(1));
|
|
2697
2146
|
}
|
|
2698
2147
|
return p;
|
|
2699
2148
|
}
|
|
2700
2149
|
function resolveLocalPath(p, baseDir) {
|
|
2701
2150
|
const expanded = expandHome(p);
|
|
2702
|
-
return
|
|
2151
|
+
return path4.isAbsolute(expanded) ? expanded : path4.resolve(baseDir, expanded);
|
|
2152
|
+
}
|
|
2153
|
+
function isNormalizedResultsConfig(config) {
|
|
2154
|
+
return typeof config.storageBranchWorktree === "boolean";
|
|
2703
2155
|
}
|
|
2704
2156
|
function normalizeResultsConfig(config, options) {
|
|
2157
|
+
if (isNormalizedResultsConfig(config)) {
|
|
2158
|
+
return config;
|
|
2159
|
+
}
|
|
2705
2160
|
const baseDir = options?.baseDir ?? process.cwd();
|
|
2706
2161
|
const repoUrl = (config.repo_url ?? config.repo)?.trim();
|
|
2707
|
-
const
|
|
2708
|
-
const
|
|
2162
|
+
const explicitPath = config.path?.trim();
|
|
2163
|
+
const repoPath = config.repo_path?.trim() ?? (repoUrl ? void 0 : explicitPath);
|
|
2164
|
+
const explicitClonePath = repoUrl ? explicitPath : void 0;
|
|
2709
2165
|
const repo = repoUrl ?? repoPath ?? "";
|
|
2710
2166
|
const branch = config.branch?.trim() || (repoPath ? DEFAULT_RESULTS_BRANCH : void 0);
|
|
2711
2167
|
const useStorageBranchWorktree = Boolean(repoPath || repoUrl && explicitClonePath && branch);
|
|
2712
2168
|
const remote = config.remote?.trim() || "origin";
|
|
2713
|
-
const autoPush = config.
|
|
2714
|
-
const requirePush = config.
|
|
2715
|
-
const configuredPushConflictPolicy = config.sync?.push_conflict_policy;
|
|
2716
|
-
if (configuredPushConflictPolicy === "backup_and_force_push") {
|
|
2717
|
-
throw new Error(
|
|
2718
|
-
"results.sync.push_conflict_policy: 'backup_and_force_push' is no longer supported. Remove the field or set it to 'block'; AgentV never force-pushes result branches."
|
|
2719
|
-
);
|
|
2720
|
-
}
|
|
2721
|
-
if (configuredPushConflictPolicy !== void 0 && configuredPushConflictPolicy !== "block") {
|
|
2722
|
-
throw new Error("results.sync.push_conflict_policy must be 'block'");
|
|
2723
|
-
}
|
|
2724
|
-
const pushConflictPolicy = configuredPushConflictPolicy ?? "block";
|
|
2169
|
+
const autoPush = config.auto_push === true;
|
|
2170
|
+
const requirePush = config.require_push === true;
|
|
2725
2171
|
const resolvedRepoPath = repoPath ? resolveLocalPath(repoPath, baseDir) : void 0;
|
|
2726
|
-
const resolvedPath = explicitClonePath ? resolveLocalPath(explicitClonePath, baseDir) : repoUrl ?
|
|
2172
|
+
const resolvedPath = explicitClonePath ? resolveLocalPath(explicitClonePath, baseDir) : repoUrl ? path4.join(getAgentvDataDir(), "results", sanitizeRepoSlug(repoUrl)) : resolvedRepoPath ?? path4.join(getAgentvDataDir(), "results", sanitizeRepoSlug(repo));
|
|
2727
2173
|
return {
|
|
2728
2174
|
mode: "github",
|
|
2729
2175
|
repo,
|
|
@@ -2734,8 +2180,7 @@ function normalizeResultsConfig(config, options) {
|
|
|
2734
2180
|
path: resolvedPath,
|
|
2735
2181
|
auto_push: autoPush,
|
|
2736
2182
|
require_push: requirePush,
|
|
2737
|
-
push_conflict_policy:
|
|
2738
|
-
branch_prefix: config.branch_prefix?.trim() || "eval-results",
|
|
2183
|
+
push_conflict_policy: "block",
|
|
2739
2184
|
storageBranchWorktree: useStorageBranchWorktree
|
|
2740
2185
|
};
|
|
2741
2186
|
}
|
|
@@ -2754,11 +2199,11 @@ function resolveResultsRepoUrl(repo) {
|
|
|
2754
2199
|
return trimmed;
|
|
2755
2200
|
}
|
|
2756
2201
|
function getResultsRepoLocalPaths(repo) {
|
|
2757
|
-
const rootDir =
|
|
2202
|
+
const rootDir = path4.join(getAgentvDataDir(), "cache", "results-repo", sanitizeRepoSlug(repo));
|
|
2758
2203
|
return {
|
|
2759
2204
|
rootDir,
|
|
2760
|
-
repoDir:
|
|
2761
|
-
statusFile:
|
|
2205
|
+
repoDir: path4.join(rootDir, "repo"),
|
|
2206
|
+
statusFile: path4.join(rootDir, "status.json")
|
|
2762
2207
|
};
|
|
2763
2208
|
}
|
|
2764
2209
|
function usesStorageBranchWorktree(config) {
|
|
@@ -2775,7 +2220,7 @@ function readPersistedStatus(statusFile) {
|
|
|
2775
2220
|
}
|
|
2776
2221
|
}
|
|
2777
2222
|
function writePersistedStatus(statusFile, status) {
|
|
2778
|
-
mkdirSync(
|
|
2223
|
+
mkdirSync(path4.dirname(statusFile), { recursive: true });
|
|
2779
2224
|
writeFileSync(statusFile, `${JSON.stringify(status, null, 2)}
|
|
2780
2225
|
`, "utf8");
|
|
2781
2226
|
}
|
|
@@ -3018,37 +2463,17 @@ async function ensureResultsMergeConfig(repoDir) {
|
|
|
3018
2463
|
try {
|
|
3019
2464
|
const { stdout } = await runGit(["rev-parse", "--git-common-dir"], { cwd: repoDir });
|
|
3020
2465
|
const resolved = stdout.trim();
|
|
3021
|
-
gitDir = resolved ?
|
|
2466
|
+
gitDir = resolved ? path4.isAbsolute(resolved) ? resolved : path4.join(repoDir, resolved) : path4.join(repoDir, ".git");
|
|
3022
2467
|
} catch {
|
|
3023
2468
|
return;
|
|
3024
2469
|
}
|
|
3025
|
-
const driverScriptPath = path5.join(gitDir, "agentv", "json-merge-driver.mjs");
|
|
3026
|
-
let scriptCurrent = false;
|
|
3027
|
-
try {
|
|
3028
|
-
scriptCurrent = readFileSync2(driverScriptPath, "utf8") === RESULTS_JSON_MERGE_DRIVER_SCRIPT;
|
|
3029
|
-
} catch {
|
|
3030
|
-
scriptCurrent = false;
|
|
3031
|
-
}
|
|
3032
|
-
if (!scriptCurrent) {
|
|
3033
|
-
mkdirSync(path5.dirname(driverScriptPath), { recursive: true });
|
|
3034
|
-
writeFileSync(driverScriptPath, RESULTS_JSON_MERGE_DRIVER_SCRIPT, { mode: 493 });
|
|
3035
|
-
}
|
|
3036
|
-
const driverCommand = `node ${JSON.stringify(driverScriptPath)} %O %A %B %P`;
|
|
3037
|
-
await runGit(
|
|
3038
|
-
["config", `merge.${RESULTS_JSON_MERGE_DRIVER_NAME}.name`, "AgentV results overlay JSON union"],
|
|
3039
|
-
{ cwd: repoDir, check: false }
|
|
3040
|
-
);
|
|
3041
|
-
await runGit(["config", `merge.${RESULTS_JSON_MERGE_DRIVER_NAME}.driver`, driverCommand], {
|
|
3042
|
-
cwd: repoDir,
|
|
3043
|
-
check: false
|
|
3044
|
-
});
|
|
3045
2470
|
writeResultsInfoAttributes(gitDir);
|
|
3046
2471
|
}
|
|
3047
2472
|
var RESULTS_INFO_ATTRIBUTES_BEGIN = "# >>> agentv results merge attributes >>>";
|
|
3048
2473
|
var RESULTS_INFO_ATTRIBUTES_END = "# <<< agentv results merge attributes <<<";
|
|
3049
2474
|
function writeResultsInfoAttributes(gitDir) {
|
|
3050
|
-
const infoDir =
|
|
3051
|
-
const attributesPath =
|
|
2475
|
+
const infoDir = path4.join(gitDir, "info");
|
|
2476
|
+
const attributesPath = path4.join(infoDir, "attributes");
|
|
3052
2477
|
const managedBlock = `${RESULTS_INFO_ATTRIBUTES_BEGIN}
|
|
3053
2478
|
${RESULTS_REPO_GITATTRIBUTES_CONTENT.trimEnd()}
|
|
3054
2479
|
${RESULTS_INFO_ATTRIBUTES_END}
|
|
@@ -3084,9 +2509,9 @@ async function addResultsGitattributesToIndex(repoDir, indexEnv) {
|
|
|
3084
2509
|
if (existing === RESULTS_REPO_GITATTRIBUTES_CONTENT) {
|
|
3085
2510
|
return;
|
|
3086
2511
|
}
|
|
3087
|
-
const scratchDir = mkdtempSync(
|
|
2512
|
+
const scratchDir = mkdtempSync(path4.join(os.tmpdir(), "agentv-results-attrs-"));
|
|
3088
2513
|
try {
|
|
3089
|
-
const scratchFile =
|
|
2514
|
+
const scratchFile = path4.join(scratchDir, RESULTS_REPO_GITATTRIBUTES_FILE);
|
|
3090
2515
|
writeFileSync(scratchFile, RESULTS_REPO_GITATTRIBUTES_CONTENT);
|
|
3091
2516
|
const { stdout: blob } = await runGit(["hash-object", "-w", "--no-filters", scratchFile], {
|
|
3092
2517
|
cwd: repoDir
|
|
@@ -3105,7 +2530,7 @@ async function addResultsGitattributesToIndex(repoDir, indexEnv) {
|
|
|
3105
2530
|
}
|
|
3106
2531
|
}
|
|
3107
2532
|
function updateStatusFile(config, patch) {
|
|
3108
|
-
const repo =
|
|
2533
|
+
const repo = normalizeResultsConfig(config).repo;
|
|
3109
2534
|
const cachePaths = getResultsRepoLocalPaths(repo);
|
|
3110
2535
|
const current = readPersistedStatus(cachePaths.statusFile);
|
|
3111
2536
|
writePersistedStatus(cachePaths.statusFile, {
|
|
@@ -3126,9 +2551,9 @@ async function ensureResultsRepoClone(config) {
|
|
|
3126
2551
|
const cachePaths = getResultsRepoLocalPaths(normalized.repo);
|
|
3127
2552
|
const cloneDir = normalized.path;
|
|
3128
2553
|
mkdirSync(cachePaths.rootDir, { recursive: true });
|
|
3129
|
-
mkdirSync(
|
|
2554
|
+
mkdirSync(path4.dirname(cloneDir), { recursive: true });
|
|
3130
2555
|
const cloneMissing = !existsSync(cloneDir);
|
|
3131
|
-
const gitDir =
|
|
2556
|
+
const gitDir = path4.join(cloneDir, ".git");
|
|
3132
2557
|
const cloneEmpty = !cloneMissing && !existsSync(gitDir) && (await readdir(cloneDir)).length === 0;
|
|
3133
2558
|
if (cloneMissing || cloneEmpty) {
|
|
3134
2559
|
try {
|
|
@@ -3174,7 +2599,6 @@ function getResultsRepoStatus(config) {
|
|
|
3174
2599
|
auto_push: normalized.auto_push,
|
|
3175
2600
|
require_push: normalized.require_push,
|
|
3176
2601
|
push_conflict_policy: normalized.push_conflict_policy,
|
|
3177
|
-
branch_prefix: normalized.branch_prefix,
|
|
3178
2602
|
local_dir: normalized.path,
|
|
3179
2603
|
last_synced_at: persisted.last_synced_at,
|
|
3180
2604
|
last_error: persisted.last_error,
|
|
@@ -3295,7 +2719,7 @@ async function getStorageBranchWorktreeDirtyPaths(repoDir, config) {
|
|
|
3295
2719
|
const comparisonRef = await gitRefExists(repoDir, localRef) ? localRef : await gitRefExists(repoDir, remoteRef) ? remoteRef : void 0;
|
|
3296
2720
|
const dirtyPaths = [];
|
|
3297
2721
|
for (const gitPath of safeDirtyPaths) {
|
|
3298
|
-
const absolutePath =
|
|
2722
|
+
const absolutePath = path4.join(repoDir, ...gitPath.split("/"));
|
|
3299
2723
|
if (!existsSync(absolutePath)) {
|
|
3300
2724
|
if (await readGitText(repoDir, comparisonRef, gitPath) !== void 0) {
|
|
3301
2725
|
dirtyPaths.push(gitPath);
|
|
@@ -3401,7 +2825,7 @@ async function hasInProgressGitConflict(repoDir) {
|
|
|
3401
2825
|
check: false
|
|
3402
2826
|
});
|
|
3403
2827
|
const markerPath = stdout.trim();
|
|
3404
|
-
const resolvedMarkerPath =
|
|
2828
|
+
const resolvedMarkerPath = path4.isAbsolute(markerPath) ? markerPath : path4.join(repoDir, markerPath);
|
|
3405
2829
|
if (markerPath && existsSync(resolvedMarkerPath)) {
|
|
3406
2830
|
return true;
|
|
3407
2831
|
}
|
|
@@ -3573,21 +2997,48 @@ function withActionFlags(status, flags) {
|
|
|
3573
2997
|
};
|
|
3574
2998
|
}
|
|
3575
2999
|
function isSafeResultsRepoPath(p) {
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
|
|
3000
|
+
const normalized = p.split(path4.sep).join("/");
|
|
3001
|
+
const segments = normalized.split("/").filter(Boolean);
|
|
3002
|
+
const isRunBundlePath = segments.length >= 2 && !segments[0]?.startsWith(".") && segments[0] !== RESULTS_REPO_GITATTRIBUTES_FILE;
|
|
3003
|
+
return normalized === RESULTS_REPO_GITATTRIBUTES_FILE || RESULTS_REPO_TRACKED_DOT_DIRS.some(
|
|
3004
|
+
(dir) => normalized === dir || normalized.startsWith(`${dir}/`)
|
|
3005
|
+
) || isRunBundlePath;
|
|
3006
|
+
}
|
|
3007
|
+
async function existingTrackedResultsPathspecs(repoDir) {
|
|
3008
|
+
const targets = /* @__PURE__ */ new Set();
|
|
3009
|
+
if (existsSync(path4.join(repoDir, RESULTS_REPO_GITATTRIBUTES_FILE))) {
|
|
3010
|
+
targets.add(RESULTS_REPO_GITATTRIBUTES_FILE);
|
|
3011
|
+
}
|
|
3012
|
+
for (const dir of RESULTS_REPO_TRACKED_DOT_DIRS) {
|
|
3013
|
+
if (existsSync(path4.join(repoDir, dir))) {
|
|
3014
|
+
targets.add(dir);
|
|
3015
|
+
continue;
|
|
3016
|
+
}
|
|
3017
|
+
const { stdout: stdout2 } = await runGit(["ls-files", "--", dir], { cwd: repoDir, check: false });
|
|
3018
|
+
if (stdout2.trim().length > 0) {
|
|
3019
|
+
targets.add(dir);
|
|
3020
|
+
}
|
|
3021
|
+
}
|
|
3022
|
+
for (const entry of await readdir(repoDir, { withFileTypes: true }).catch(() => [])) {
|
|
3023
|
+
if (entry.name.startsWith(".") || !entry.isDirectory()) {
|
|
3024
|
+
continue;
|
|
3025
|
+
}
|
|
3026
|
+
if (existsSync(path4.join(repoDir, entry.name, "summary.json"))) {
|
|
3027
|
+
targets.add(entry.name);
|
|
3028
|
+
}
|
|
3029
|
+
}
|
|
3030
|
+
const { stdout } = await runGit(["ls-files"], { cwd: repoDir, check: false });
|
|
3031
|
+
for (const file of stdout.split(/\r?\n/)) {
|
|
3032
|
+
const normalized = file.trim();
|
|
3033
|
+
if (!normalized) {
|
|
3583
3034
|
continue;
|
|
3584
3035
|
}
|
|
3585
|
-
const
|
|
3586
|
-
if (
|
|
3587
|
-
targets.
|
|
3036
|
+
const segments = normalized.split("/");
|
|
3037
|
+
if (segments.length >= 2 && segments[0] && !segments[0].startsWith(".")) {
|
|
3038
|
+
targets.add(segments[0]);
|
|
3588
3039
|
}
|
|
3589
3040
|
}
|
|
3590
|
-
return targets;
|
|
3041
|
+
return [...targets].sort();
|
|
3591
3042
|
}
|
|
3592
3043
|
function areSafeResultsRepoPaths(paths) {
|
|
3593
3044
|
return paths.length > 0 && paths.every(isSafeResultsRepoPath);
|
|
@@ -3638,7 +3089,7 @@ async function getMergeBaseCommit(repoDir, a, b) {
|
|
|
3638
3089
|
return exitCode === 0 && sha.length > 0 ? sha : void 0;
|
|
3639
3090
|
}
|
|
3640
3091
|
function buildNeedsHumanMergeReason(targetBranch, remoteCommit, localCommit) {
|
|
3641
|
-
return `Results branch ${targetBranch} diverged from the remote and could not be auto-merged: a genuine content conflict
|
|
3092
|
+
return `Results branch ${targetBranch} diverged from the remote and could not be auto-merged: a genuine results content conflict remains (remote ${formatShortSha(
|
|
3642
3093
|
remoteCommit
|
|
3643
3094
|
)}, local ${formatShortSha(
|
|
3644
3095
|
localCommit
|
|
@@ -3692,13 +3143,7 @@ async function countContributedRunDirs(repoDir, remoteTargetCommit, sourceCommit
|
|
|
3692
3143
|
return void 0;
|
|
3693
3144
|
}
|
|
3694
3145
|
const { stdout, exitCode } = await runGit(
|
|
3695
|
-
[
|
|
3696
|
-
"diff",
|
|
3697
|
-
"--name-only",
|
|
3698
|
-
`${remoteTargetCommit}...${sourceCommit}`,
|
|
3699
|
-
"--",
|
|
3700
|
-
`${RESULTS_REPO_RUNS_DIR}/`
|
|
3701
|
-
],
|
|
3146
|
+
["diff", "--name-only", `${remoteTargetCommit}...${sourceCommit}`, "--", RESULTS_REPO_RUNS_DIR],
|
|
3702
3147
|
{ cwd: repoDir, check: false }
|
|
3703
3148
|
);
|
|
3704
3149
|
if (exitCode !== 0) {
|
|
@@ -3711,8 +3156,8 @@ async function countContributedRunDirs(repoDir, remoteTargetCommit, sourceCommit
|
|
|
3711
3156
|
continue;
|
|
3712
3157
|
}
|
|
3713
3158
|
const segments = file.split("/");
|
|
3714
|
-
if (segments[0]
|
|
3715
|
-
runDirs.add(
|
|
3159
|
+
if (segments.length >= 2 && segments[0] && !segments[0].startsWith(".")) {
|
|
3160
|
+
runDirs.add(segments[0]);
|
|
3716
3161
|
}
|
|
3717
3162
|
}
|
|
3718
3163
|
return runDirs.size;
|
|
@@ -3954,9 +3399,6 @@ async function getResultsRepoSyncStatus(config) {
|
|
|
3954
3399
|
}
|
|
3955
3400
|
try {
|
|
3956
3401
|
if (usesStorageBranchWorktree(normalized)) {
|
|
3957
|
-
await fetchResultsRepo(normalized.path, normalized.remote, normalized.branch).catch(
|
|
3958
|
-
() => void 0
|
|
3959
|
-
);
|
|
3960
3402
|
return withGitInspection(
|
|
3961
3403
|
baseStatus,
|
|
3962
3404
|
await inspectResultsStorageBranchGit(normalized.path, normalized)
|
|
@@ -4331,10 +3773,16 @@ async function syncResultsRepoForProject(config) {
|
|
|
4331
3773
|
}
|
|
4332
3774
|
}
|
|
4333
3775
|
if (inspection.syncStatus === "dirty") {
|
|
4334
|
-
const
|
|
4335
|
-
await runGit(["add", "--all", "--", ...
|
|
3776
|
+
const trackedPathspecs = await existingTrackedResultsPathspecs(repoDir);
|
|
3777
|
+
await runGit(["add", "--all", "--", ...trackedPathspecs], { cwd: repoDir });
|
|
4336
3778
|
await runGitWithFallbackCommitIdentity(
|
|
4337
|
-
[
|
|
3779
|
+
[
|
|
3780
|
+
"commit",
|
|
3781
|
+
"-m",
|
|
3782
|
+
"chore(results): sync local results branch changes",
|
|
3783
|
+
"--",
|
|
3784
|
+
...trackedPathspecs
|
|
3785
|
+
],
|
|
4338
3786
|
{
|
|
4339
3787
|
cwd: repoDir
|
|
4340
3788
|
}
|
|
@@ -4552,8 +4000,8 @@ async function prepareResultsRepoBranch(config, branchName) {
|
|
|
4552
4000
|
const cloneDir = await ensureResultsRepoClone(normalized);
|
|
4553
4001
|
const baseBranch = await resolveDefaultBranch(cloneDir);
|
|
4554
4002
|
await fetchResultsRepo(cloneDir, normalized.remote);
|
|
4555
|
-
const worktreeRoot = await mkdtemp(
|
|
4556
|
-
const worktreeDir =
|
|
4003
|
+
const worktreeRoot = await mkdtemp(path4.join(os.tmpdir(), "agentv-results-repo-"));
|
|
4004
|
+
const worktreeDir = path4.join(worktreeRoot, "repo");
|
|
4557
4005
|
await runGit(
|
|
4558
4006
|
["worktree", "add", "-B", branchName, worktreeDir, `${normalized.remote}/${baseBranch}`],
|
|
4559
4007
|
{
|
|
@@ -4575,12 +4023,12 @@ async function prepareResultsRepoBranch(config, branchName) {
|
|
|
4575
4023
|
}
|
|
4576
4024
|
async function stageResultsArtifacts(params) {
|
|
4577
4025
|
rmSync(params.destinationDir, { recursive: true, force: true });
|
|
4578
|
-
mkdirSync(
|
|
4026
|
+
mkdirSync(path4.dirname(params.destinationDir), { recursive: true });
|
|
4579
4027
|
await cp(params.sourceDir, params.destinationDir, { recursive: true });
|
|
4580
4028
|
}
|
|
4581
4029
|
function resolveResultsRepoRunsDir(config) {
|
|
4582
4030
|
const normalized = normalizeResultsConfig(config);
|
|
4583
|
-
return
|
|
4031
|
+
return path4.join(normalized.path, RESULTS_REPO_RESULTS_DIR);
|
|
4584
4032
|
}
|
|
4585
4033
|
async function directorySizeBytes(targetPath) {
|
|
4586
4034
|
const entry = await stat(targetPath);
|
|
@@ -4589,7 +4037,7 @@ async function directorySizeBytes(targetPath) {
|
|
|
4589
4037
|
}
|
|
4590
4038
|
let total = 0;
|
|
4591
4039
|
for (const child of await readdir(targetPath, { withFileTypes: true })) {
|
|
4592
|
-
total += await directorySizeBytes(
|
|
4040
|
+
total += await directorySizeBytes(path4.join(targetPath, child.name));
|
|
4593
4041
|
}
|
|
4594
4042
|
return total;
|
|
4595
4043
|
}
|
|
@@ -4656,7 +4104,7 @@ async function assertValidResultsBranchName(repoDir, branch) {
|
|
|
4656
4104
|
await runGit(["check-ref-format", "--branch", branch], { cwd: repoDir });
|
|
4657
4105
|
}
|
|
4658
4106
|
function normalizeDestinationPath(destinationPath) {
|
|
4659
|
-
const normalized = destinationPath.split(
|
|
4107
|
+
const normalized = destinationPath.split(path4.sep).join("/");
|
|
4660
4108
|
const segments = normalized.split("/").filter(Boolean);
|
|
4661
4109
|
if (segments.length === 0 || normalized.startsWith("/") || segments.some((segment) => segment === "..") || normalized.split("").some((char) => {
|
|
4662
4110
|
const code = char.charCodeAt(0);
|
|
@@ -4670,7 +4118,7 @@ async function listSourceFiles(sourceDir) {
|
|
|
4670
4118
|
const entries = [];
|
|
4671
4119
|
async function visit(dir) {
|
|
4672
4120
|
for (const entry of await readdir(dir, { withFileTypes: true })) {
|
|
4673
|
-
const absolutePath =
|
|
4121
|
+
const absolutePath = path4.join(dir, entry.name);
|
|
4674
4122
|
if (entry.isDirectory()) {
|
|
4675
4123
|
await visit(absolutePath);
|
|
4676
4124
|
} else if (entry.isFile() || entry.isSymbolicLink()) {
|
|
@@ -4682,20 +4130,18 @@ async function listSourceFiles(sourceDir) {
|
|
|
4682
4130
|
entries.sort();
|
|
4683
4131
|
return entries;
|
|
4684
4132
|
}
|
|
4685
|
-
function
|
|
4133
|
+
function isRecord2(value) {
|
|
4686
4134
|
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
4687
4135
|
}
|
|
4688
4136
|
function artifactSidecarPointers(record) {
|
|
4689
|
-
if (!
|
|
4137
|
+
if (!isRecord2(record) || !isRecord2(record.artifact_pointers)) {
|
|
4690
4138
|
return [];
|
|
4691
4139
|
}
|
|
4692
4140
|
const pointers = [];
|
|
4693
|
-
|
|
4694
|
-
|
|
4695
|
-
continue;
|
|
4696
|
-
}
|
|
4141
|
+
const pointer = record.artifact_pointers.transcript;
|
|
4142
|
+
if (isRecord2(pointer)) {
|
|
4697
4143
|
if (pointer.ref !== AGENTV_RESULTS_ARTIFACTS_REF || typeof pointer.path !== "string") {
|
|
4698
|
-
|
|
4144
|
+
return pointers;
|
|
4699
4145
|
}
|
|
4700
4146
|
pointers.push({
|
|
4701
4147
|
path: pointer.path,
|
|
@@ -4706,15 +4152,56 @@ function artifactSidecarPointers(record) {
|
|
|
4706
4152
|
return pointers;
|
|
4707
4153
|
}
|
|
4708
4154
|
function artifactSidecarKey(destinationPath, pointerPath) {
|
|
4709
|
-
return
|
|
4155
|
+
return path4.posix.join(
|
|
4710
4156
|
RESULTS_REPO_RUNS_DIR,
|
|
4711
4157
|
normalizeDestinationPath(destinationPath),
|
|
4712
4158
|
normalizeDestinationPath(pointerPath)
|
|
4713
4159
|
);
|
|
4714
4160
|
}
|
|
4161
|
+
function isDeprecatedTraceArtifactPath(relativePath) {
|
|
4162
|
+
return relativePath === "trace.json" || relativePath.endsWith("/trace.json");
|
|
4163
|
+
}
|
|
4164
|
+
function isResultManifestFilename(filename) {
|
|
4165
|
+
return filename === RESULT_INDEX_FILENAME2;
|
|
4166
|
+
}
|
|
4167
|
+
function safeLocalSummaryManifestPath(sourceDir, manifestPath) {
|
|
4168
|
+
if (typeof manifestPath !== "string" || manifestPath.trim().length === 0) {
|
|
4169
|
+
return void 0;
|
|
4170
|
+
}
|
|
4171
|
+
if (path4.isAbsolute(manifestPath)) {
|
|
4172
|
+
return void 0;
|
|
4173
|
+
}
|
|
4174
|
+
const normalized = path4.normalize(manifestPath);
|
|
4175
|
+
if (normalized.startsWith("..") || path4.isAbsolute(normalized)) {
|
|
4176
|
+
return void 0;
|
|
4177
|
+
}
|
|
4178
|
+
return path4.join(sourceDir, normalized);
|
|
4179
|
+
}
|
|
4180
|
+
function resolveLocalResultManifestPath(sourceDir) {
|
|
4181
|
+
try {
|
|
4182
|
+
const summary = JSON.parse(readFileSync2(path4.join(sourceDir, "summary.json"), "utf8"));
|
|
4183
|
+
const manifestPath2 = safeLocalSummaryManifestPath(
|
|
4184
|
+
sourceDir,
|
|
4185
|
+
summary.index_path ?? summary.manifest_path
|
|
4186
|
+
);
|
|
4187
|
+
if (manifestPath2 && existsSync(manifestPath2)) {
|
|
4188
|
+
return manifestPath2;
|
|
4189
|
+
}
|
|
4190
|
+
} catch {
|
|
4191
|
+
}
|
|
4192
|
+
const manifestPath = path4.join(sourceDir, RUN_INTERNAL_DIRNAME, RESULT_INDEX_FILENAME2);
|
|
4193
|
+
if (existsSync(manifestPath)) {
|
|
4194
|
+
return manifestPath;
|
|
4195
|
+
}
|
|
4196
|
+
const legacyManifestPath = path4.join(sourceDir, RESULT_INDEX_FILENAME2);
|
|
4197
|
+
if (existsSync(legacyManifestPath)) {
|
|
4198
|
+
return legacyManifestPath;
|
|
4199
|
+
}
|
|
4200
|
+
return void 0;
|
|
4201
|
+
}
|
|
4715
4202
|
function collectArtifactSidecarPointers(sourceDir) {
|
|
4716
|
-
const indexPath =
|
|
4717
|
-
if (!
|
|
4203
|
+
const indexPath = resolveLocalResultManifestPath(sourceDir);
|
|
4204
|
+
if (!indexPath) {
|
|
4718
4205
|
return [];
|
|
4719
4206
|
}
|
|
4720
4207
|
const pointers = [];
|
|
@@ -4732,16 +4219,16 @@ function collectArtifactSidecarPointers(sourceDir) {
|
|
|
4732
4219
|
}
|
|
4733
4220
|
function resolveArtifactPointerSourcePath(sourceDir, pointerPath) {
|
|
4734
4221
|
const normalizedPointerPath = normalizeDestinationPath(pointerPath);
|
|
4735
|
-
const sourceRoot =
|
|
4736
|
-
const sourcePath =
|
|
4737
|
-
const relative =
|
|
4738
|
-
if (relative.startsWith("..") ||
|
|
4222
|
+
const sourceRoot = path4.resolve(sourceDir);
|
|
4223
|
+
const sourcePath = path4.resolve(sourceRoot, ...normalizedPointerPath.split("/"));
|
|
4224
|
+
const relative = path4.relative(sourceRoot, sourcePath);
|
|
4225
|
+
if (relative.startsWith("..") || path4.isAbsolute(relative)) {
|
|
4739
4226
|
throw new Error(`Artifact pointer path escapes run directory: ${pointerPath}`);
|
|
4740
4227
|
}
|
|
4741
4228
|
return sourcePath;
|
|
4742
4229
|
}
|
|
4743
4230
|
function verifyArtifactPointerChecksum(pointer, content) {
|
|
4744
|
-
const sha256 =
|
|
4231
|
+
const sha256 = createHash("sha256").update(content).digest("hex");
|
|
4745
4232
|
if (pointer.sha256 && pointer.sha256 !== sha256) {
|
|
4746
4233
|
throw new Error(`Artifact pointer checksum mismatch for ${pointer.path}`);
|
|
4747
4234
|
}
|
|
@@ -4753,7 +4240,7 @@ async function prepareArtifactSidecar(params) {
|
|
|
4753
4240
|
if (params.pointers.length === 0) {
|
|
4754
4241
|
return void 0;
|
|
4755
4242
|
}
|
|
4756
|
-
const sidecarRoot = await mkdtemp(
|
|
4243
|
+
const sidecarRoot = await mkdtemp(path4.join(os.tmpdir(), "agentv-artifact-sidecar-"));
|
|
4757
4244
|
const copied = /* @__PURE__ */ new Set();
|
|
4758
4245
|
const contentByPath = /* @__PURE__ */ new Map();
|
|
4759
4246
|
try {
|
|
@@ -4769,8 +4256,8 @@ async function prepareArtifactSidecar(params) {
|
|
|
4769
4256
|
if (copied.has(relativePointerPath)) {
|
|
4770
4257
|
continue;
|
|
4771
4258
|
}
|
|
4772
|
-
const destinationPath =
|
|
4773
|
-
mkdirSync(
|
|
4259
|
+
const destinationPath = path4.join(sidecarRoot, ...relativePointerPath.split("/"));
|
|
4260
|
+
mkdirSync(path4.dirname(destinationPath), { recursive: true });
|
|
4774
4261
|
writeFileSync(destinationPath, content);
|
|
4775
4262
|
copied.add(relativePointerPath);
|
|
4776
4263
|
}
|
|
@@ -4797,22 +4284,24 @@ function rewritePublishedIndexLine(line, destinationPath) {
|
|
|
4797
4284
|
} catch {
|
|
4798
4285
|
return line;
|
|
4799
4286
|
}
|
|
4800
|
-
if (!
|
|
4287
|
+
if (!isRecord2(record) || !isRecord2(record.artifact_pointers)) {
|
|
4801
4288
|
return line;
|
|
4802
4289
|
}
|
|
4803
4290
|
let changed = false;
|
|
4804
|
-
|
|
4805
|
-
|
|
4806
|
-
continue;
|
|
4807
|
-
}
|
|
4808
|
-
if (pointer.ref !== AGENTV_RESULTS_ARTIFACTS_REF || typeof pointer.path !== "string") {
|
|
4809
|
-
continue;
|
|
4810
|
-
}
|
|
4291
|
+
const pointer = record.artifact_pointers.transcript;
|
|
4292
|
+
if (isRecord2(pointer) && pointer.ref === AGENTV_RESULTS_ARTIFACTS_REF && typeof pointer.path === "string") {
|
|
4811
4293
|
const key = artifactSidecarKey(destinationPath, pointer.path);
|
|
4812
4294
|
if (pointer.key !== key) {
|
|
4813
4295
|
pointer.key = key;
|
|
4814
4296
|
changed = true;
|
|
4815
4297
|
}
|
|
4298
|
+
if (Object.keys(record.artifact_pointers).length !== 1 || record.artifact_pointers.transcript !== pointer) {
|
|
4299
|
+
changed = true;
|
|
4300
|
+
}
|
|
4301
|
+
record.artifact_pointers = { transcript: pointer };
|
|
4302
|
+
} else {
|
|
4303
|
+
record.artifact_pointers = void 0;
|
|
4304
|
+
changed = true;
|
|
4816
4305
|
}
|
|
4817
4306
|
return changed ? JSON.stringify(record) : line;
|
|
4818
4307
|
}
|
|
@@ -4820,26 +4309,29 @@ async function preparePublishedResultsSource(params) {
|
|
|
4820
4309
|
if (params.pointers.length === 0) {
|
|
4821
4310
|
return void 0;
|
|
4822
4311
|
}
|
|
4823
|
-
const publishedRoot = await mkdtemp(
|
|
4312
|
+
const publishedRoot = await mkdtemp(path4.join(os.tmpdir(), "agentv-results-published-"));
|
|
4824
4313
|
const omittedPaths = new Set(
|
|
4825
4314
|
params.pointers.map((pointer) => normalizeDestinationPath(pointer.path))
|
|
4826
4315
|
);
|
|
4827
4316
|
try {
|
|
4828
4317
|
const sourceFiles = await listSourceFiles(params.sourceDir);
|
|
4829
4318
|
for (const sourceFile of sourceFiles) {
|
|
4830
|
-
const relativeFile =
|
|
4831
|
-
const destinationFile =
|
|
4832
|
-
if (relativeFile
|
|
4319
|
+
const relativeFile = path4.relative(params.sourceDir, sourceFile).split(path4.sep).join("/");
|
|
4320
|
+
const destinationFile = path4.join(publishedRoot, ...relativeFile.split("/"));
|
|
4321
|
+
if (isResultManifestFilename(path4.posix.basename(relativeFile))) {
|
|
4833
4322
|
const original = readFileSync2(sourceFile, "utf8");
|
|
4834
4323
|
const rewritten = original.split(/\r?\n/).map((line) => rewritePublishedIndexLine(line, params.destinationPath)).join("\n");
|
|
4835
|
-
mkdirSync(
|
|
4324
|
+
mkdirSync(path4.dirname(destinationFile), { recursive: true });
|
|
4836
4325
|
writeFileSync(destinationFile, rewritten);
|
|
4837
4326
|
continue;
|
|
4838
4327
|
}
|
|
4328
|
+
if (isDeprecatedTraceArtifactPath(relativeFile)) {
|
|
4329
|
+
continue;
|
|
4330
|
+
}
|
|
4839
4331
|
if (omittedPaths.has(relativeFile)) {
|
|
4840
4332
|
continue;
|
|
4841
4333
|
}
|
|
4842
|
-
mkdirSync(
|
|
4334
|
+
mkdirSync(path4.dirname(destinationFile), { recursive: true });
|
|
4843
4335
|
await cp(sourceFile, destinationFile, { dereference: false });
|
|
4844
4336
|
}
|
|
4845
4337
|
} catch (error) {
|
|
@@ -4906,8 +4398,8 @@ async function commitStorageBranchWorktreePaths(params) {
|
|
|
4906
4398
|
normalized
|
|
4907
4399
|
});
|
|
4908
4400
|
}
|
|
4909
|
-
const indexRoot = await mkdtemp(
|
|
4910
|
-
const indexFile =
|
|
4401
|
+
const indexRoot = await mkdtemp(path4.join(os.tmpdir(), "agentv-results-index-"));
|
|
4402
|
+
const indexFile = path4.join(indexRoot, "index");
|
|
4911
4403
|
const indexEnv = { GIT_INDEX_FILE: indexFile };
|
|
4912
4404
|
try {
|
|
4913
4405
|
if (base.baseRef) {
|
|
@@ -4917,7 +4409,7 @@ async function commitStorageBranchWorktreePaths(params) {
|
|
|
4917
4409
|
}
|
|
4918
4410
|
await addResultsGitattributesToIndex(params.repoDir, indexEnv);
|
|
4919
4411
|
for (const gitPath of paths) {
|
|
4920
|
-
const sourcePath =
|
|
4412
|
+
const sourcePath = path4.join(params.repoDir, ...gitPath.split("/"));
|
|
4921
4413
|
if (!existsSync(sourcePath)) {
|
|
4922
4414
|
await runGit(["update-index", "--force-remove", "--", gitPath], {
|
|
4923
4415
|
cwd: params.repoDir,
|
|
@@ -4983,7 +4475,7 @@ async function commitResultsRunWithTemporaryIndex(params) {
|
|
|
4983
4475
|
await assertValidResultsBranchName(params.repoDir, normalized.branch);
|
|
4984
4476
|
await ensureResultsBranchNotCheckedOut(params.repoDir, normalized);
|
|
4985
4477
|
const destinationRunPath = normalizeDestinationPath(params.destinationPath);
|
|
4986
|
-
const destinationTreePath =
|
|
4478
|
+
const destinationTreePath = path4.posix.join(RESULTS_REPO_RUNS_DIR, destinationRunPath);
|
|
4987
4479
|
let base = await resolveStorageBranchBase({
|
|
4988
4480
|
repoDir: params.repoDir,
|
|
4989
4481
|
normalized,
|
|
@@ -4997,8 +4489,8 @@ async function commitResultsRunWithTemporaryIndex(params) {
|
|
|
4997
4489
|
preferRemote: params.preferRemoteBase
|
|
4998
4490
|
});
|
|
4999
4491
|
}
|
|
5000
|
-
const indexRoot = await mkdtemp(
|
|
5001
|
-
const indexFile =
|
|
4492
|
+
const indexRoot = await mkdtemp(path4.join(os.tmpdir(), "agentv-results-index-"));
|
|
4493
|
+
const indexFile = path4.join(indexRoot, "index");
|
|
5002
4494
|
const indexEnv = { GIT_INDEX_FILE: indexFile };
|
|
5003
4495
|
try {
|
|
5004
4496
|
if (base.baseRef) {
|
|
@@ -5021,8 +4513,8 @@ async function commitResultsRunWithTemporaryIndex(params) {
|
|
|
5021
4513
|
}
|
|
5022
4514
|
const sourceFiles = await listSourceFiles(params.sourceDir);
|
|
5023
4515
|
for (const sourceFile of sourceFiles) {
|
|
5024
|
-
const relativeFile =
|
|
5025
|
-
const destinationFile =
|
|
4516
|
+
const relativeFile = path4.relative(params.sourceDir, sourceFile).split(path4.sep).join("/");
|
|
4517
|
+
const destinationFile = path4.posix.join(destinationTreePath, relativeFile);
|
|
5026
4518
|
const fileStat = await lstat(sourceFile);
|
|
5027
4519
|
let mode = fileStat.mode & 73 ? "100755" : "100644";
|
|
5028
4520
|
let hashInputPath = sourceFile;
|
|
@@ -5295,17 +4787,70 @@ async function directPushResultsWithDetails(params) {
|
|
|
5295
4787
|
await sidecar?.cleanup().catch(() => void 0);
|
|
5296
4788
|
}
|
|
5297
4789
|
}
|
|
5298
|
-
function
|
|
5299
|
-
|
|
5300
|
-
|
|
5301
|
-
|
|
5302
|
-
|
|
5303
|
-
|
|
5304
|
-
|
|
5305
|
-
|
|
4790
|
+
function safeGitSummaryManifestPath(runDir, manifestPath) {
|
|
4791
|
+
if (typeof manifestPath !== "string" || manifestPath.trim().length === 0) {
|
|
4792
|
+
return void 0;
|
|
4793
|
+
}
|
|
4794
|
+
if (manifestPath.startsWith("/")) {
|
|
4795
|
+
return void 0;
|
|
4796
|
+
}
|
|
4797
|
+
const normalized = path4.posix.normalize(manifestPath);
|
|
4798
|
+
if (normalized === ".." || normalized.startsWith("../")) {
|
|
4799
|
+
return void 0;
|
|
4800
|
+
}
|
|
4801
|
+
return path4.posix.join(runDir, normalized);
|
|
4802
|
+
}
|
|
4803
|
+
function buildGitManifestPaths(treePaths, summaryByPath) {
|
|
4804
|
+
const treePathSet = new Set(treePaths);
|
|
4805
|
+
const manifestByRunDir = /* @__PURE__ */ new Map();
|
|
4806
|
+
for (const [summaryPath, summary] of summaryByPath) {
|
|
4807
|
+
const runDir = path4.posix.dirname(summaryPath);
|
|
4808
|
+
const relativeRunPath = path4.posix.relative(RESULTS_REPO_RUNS_DIR, runDir);
|
|
4809
|
+
if (!isV2ResultsRepoRunPath(relativeRunPath)) {
|
|
4810
|
+
continue;
|
|
4811
|
+
}
|
|
4812
|
+
const manifestPath = safeGitSummaryManifestPath(runDir, summary.index_path ?? summary.manifest_path) ?? fallbackGitManifestPath(runDir, treePathSet);
|
|
4813
|
+
if (manifestPath && treePathSet.has(manifestPath)) {
|
|
4814
|
+
manifestByRunDir.set(runDir, manifestPath);
|
|
4815
|
+
}
|
|
4816
|
+
}
|
|
4817
|
+
for (const treePath of treePaths) {
|
|
4818
|
+
if (!treePath.endsWith(`/${RESULT_INDEX_FILENAME2}`)) {
|
|
4819
|
+
continue;
|
|
4820
|
+
}
|
|
4821
|
+
const runDir = gitRunDirForManifestPath(treePath);
|
|
4822
|
+
const relativeRunPath = path4.posix.relative(RESULTS_REPO_RUNS_DIR, runDir);
|
|
4823
|
+
if (!isV2ResultsRepoRunPath(relativeRunPath)) {
|
|
4824
|
+
continue;
|
|
4825
|
+
}
|
|
4826
|
+
if (!manifestByRunDir.has(runDir)) {
|
|
4827
|
+
manifestByRunDir.set(runDir, treePath);
|
|
4828
|
+
}
|
|
4829
|
+
}
|
|
4830
|
+
return [...manifestByRunDir.values()].sort();
|
|
4831
|
+
}
|
|
4832
|
+
function fallbackGitManifestPath(runDir, treePathSet) {
|
|
4833
|
+
for (const candidate of [
|
|
4834
|
+
path4.posix.join(runDir, ".internal", RESULT_INDEX_FILENAME2),
|
|
4835
|
+
path4.posix.join(runDir, RESULT_INDEX_FILENAME2)
|
|
4836
|
+
]) {
|
|
4837
|
+
if (treePathSet.has(candidate)) {
|
|
4838
|
+
return candidate;
|
|
5306
4839
|
}
|
|
5307
|
-
return `${experiment}::${timestamp}`;
|
|
5308
4840
|
}
|
|
4841
|
+
return void 0;
|
|
4842
|
+
}
|
|
4843
|
+
function gitRunDirForManifestPath(manifestPath) {
|
|
4844
|
+
const manifestDir = path4.posix.dirname(manifestPath);
|
|
4845
|
+
return path4.posix.basename(manifestDir) === ".internal" ? path4.posix.dirname(manifestDir) : manifestDir;
|
|
4846
|
+
}
|
|
4847
|
+
function isV2ResultsRepoRunPath(relativeRunPath) {
|
|
4848
|
+
const segments = relativeRunPath.split("/").filter(Boolean);
|
|
4849
|
+
return segments.length === 1 && !segments[0].startsWith(".");
|
|
4850
|
+
}
|
|
4851
|
+
function buildGitRunId(relativeRunPath) {
|
|
4852
|
+
const normalized = relativeRunPath.split(path4.sep).join("/");
|
|
4853
|
+
const segments = normalized.split("/").filter(Boolean);
|
|
5309
4854
|
return segments[0] ?? relativeRunPath;
|
|
5310
4855
|
}
|
|
5311
4856
|
function getRunExperiment(runId, summary) {
|
|
@@ -5433,7 +4978,7 @@ async function readGitResultArtifact(params) {
|
|
|
5433
4978
|
}
|
|
5434
4979
|
function buildWipBranchName(runDir) {
|
|
5435
4980
|
const hostname = os.hostname().replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 40);
|
|
5436
|
-
const runBasename =
|
|
4981
|
+
const runBasename = path4.basename(runDir).replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 60);
|
|
5437
4982
|
return `agentv/wip/${hostname}/${runBasename}`;
|
|
5438
4983
|
}
|
|
5439
4984
|
async function setupWipWorktree(params) {
|
|
@@ -5452,8 +4997,8 @@ async function setupWipWorktree(params) {
|
|
|
5452
4997
|
if (!baseRef) {
|
|
5453
4998
|
throw new Error("Could not resolve a base ref for the WIP results branch");
|
|
5454
4999
|
}
|
|
5455
|
-
const worktreeRoot = await mkdtemp(
|
|
5456
|
-
const worktreeDir =
|
|
5000
|
+
const worktreeRoot = await mkdtemp(path4.join(os.tmpdir(), "agentv-wip-"));
|
|
5001
|
+
const worktreeDir = path4.join(worktreeRoot, "repo");
|
|
5457
5002
|
await runGit(["worktree", "add", "-B", params.wipBranch, worktreeDir, baseRef], {
|
|
5458
5003
|
cwd: cloneDir
|
|
5459
5004
|
});
|
|
@@ -5472,7 +5017,7 @@ async function setupWipWorktree(params) {
|
|
|
5472
5017
|
};
|
|
5473
5018
|
}
|
|
5474
5019
|
async function pushWipCheckpoint(params) {
|
|
5475
|
-
const destinationDir =
|
|
5020
|
+
const destinationDir = path4.join(
|
|
5476
5021
|
params.handle.worktreeDir,
|
|
5477
5022
|
RESULTS_REPO_RUNS_DIR,
|
|
5478
5023
|
params.destinationPath
|
|
@@ -5482,8 +5027,11 @@ async function pushWipCheckpoint(params) {
|
|
|
5482
5027
|
sourceDir: params.sourceDir,
|
|
5483
5028
|
destinationDir
|
|
5484
5029
|
});
|
|
5485
|
-
|
|
5486
|
-
|
|
5030
|
+
writeFileSync(
|
|
5031
|
+
path4.join(params.handle.worktreeDir, RESULTS_REPO_GITATTRIBUTES_FILE),
|
|
5032
|
+
RESULTS_REPO_GITATTRIBUTES_CONTENT
|
|
5033
|
+
);
|
|
5034
|
+
await runGit(["add", "--all", "--", RESULTS_REPO_GITATTRIBUTES_FILE, params.destinationPath], {
|
|
5487
5035
|
cwd: params.handle.worktreeDir
|
|
5488
5036
|
});
|
|
5489
5037
|
const { stdout: status } = await runGit(["status", "--porcelain"], {
|
|
@@ -5506,7 +5054,19 @@ async function pushWipCheckpoint(params) {
|
|
|
5506
5054
|
async function deleteWipBranch(params) {
|
|
5507
5055
|
const normalized = normalizeResultsConfig(params.config);
|
|
5508
5056
|
const cloneDir = await ensureResultsRepoClone(normalized);
|
|
5509
|
-
await runGit(["push", normalized.remote, "--delete", params.wipBranch], {
|
|
5057
|
+
const result = await runGit(["push", normalized.remote, "--delete", params.wipBranch], {
|
|
5058
|
+
cwd: cloneDir,
|
|
5059
|
+
check: false
|
|
5060
|
+
});
|
|
5061
|
+
if (result.exitCode === 0 || isMissingRemoteRefDelete(result)) {
|
|
5062
|
+
return;
|
|
5063
|
+
}
|
|
5064
|
+
throw new Error(result.stderr.trim() || result.stdout.trim() || "Failed to delete WIP branch");
|
|
5065
|
+
}
|
|
5066
|
+
function isMissingRemoteRefDelete(result) {
|
|
5067
|
+
const text = `${result.stderr}
|
|
5068
|
+
${result.stdout}`.toLowerCase();
|
|
5069
|
+
return text.includes("remote ref does not exist");
|
|
5510
5070
|
}
|
|
5511
5071
|
function isMissingGitRefError(error) {
|
|
5512
5072
|
const parts = [];
|
|
@@ -5518,7 +5078,107 @@ function isMissingGitRefError(error) {
|
|
|
5518
5078
|
parts.push(error);
|
|
5519
5079
|
}
|
|
5520
5080
|
const haystack = parts.join("\n").toLowerCase();
|
|
5521
|
-
return haystack.includes("not a valid object name") || haystack.includes("unknown revision or path") || haystack.includes("bad revision") || haystack.includes("does not exist");
|
|
5081
|
+
return haystack.includes("not a valid object name") || haystack.includes("unknown revision or path") || haystack.includes("bad revision") || haystack.includes("does not exist") || haystack.includes("needed a single revision");
|
|
5082
|
+
}
|
|
5083
|
+
function gitResultsIndexHash(value) {
|
|
5084
|
+
return createHash("sha256").update(value).digest("hex");
|
|
5085
|
+
}
|
|
5086
|
+
function resolveGitResultsIndexRepoDir(repoDir) {
|
|
5087
|
+
const resolved = path4.resolve(repoDir);
|
|
5088
|
+
try {
|
|
5089
|
+
return realpathSync(resolved);
|
|
5090
|
+
} catch {
|
|
5091
|
+
return resolved;
|
|
5092
|
+
}
|
|
5093
|
+
}
|
|
5094
|
+
function gitResultsIndexRepoKey(repoDir) {
|
|
5095
|
+
return `sha256-${gitResultsIndexHash(resolveGitResultsIndexRepoDir(repoDir)).slice(0, 32)}`;
|
|
5096
|
+
}
|
|
5097
|
+
function gitResultsIndexRefKey(ref) {
|
|
5098
|
+
const slug = sanitizeRepoSlug(ref).slice(0, 80) || "default";
|
|
5099
|
+
return `${slug}-${gitResultsIndexHash(ref).slice(0, 16)}`;
|
|
5100
|
+
}
|
|
5101
|
+
function resolveGitResultsIndexCacheFile(params) {
|
|
5102
|
+
const ref = params.ref ?? "origin/main";
|
|
5103
|
+
return path4.join(
|
|
5104
|
+
getAgentvDataDir(),
|
|
5105
|
+
"cache",
|
|
5106
|
+
"results-index",
|
|
5107
|
+
gitResultsIndexRepoKey(params.repoDir),
|
|
5108
|
+
gitResultsIndexRefKey(ref),
|
|
5109
|
+
`${params.commitSha}.json`
|
|
5110
|
+
);
|
|
5111
|
+
}
|
|
5112
|
+
function isGitListedRun(value) {
|
|
5113
|
+
if (!value || typeof value !== "object") return false;
|
|
5114
|
+
const record = value;
|
|
5115
|
+
return typeof record.run_id === "string" && typeof record.experiment === "string" && typeof record.timestamp === "string" && (typeof record.index_path === "string" || typeof record.manifest_path === "string") && typeof record.display_name === "string" && typeof record.test_count === "number" && typeof record.avg_score === "number" && typeof record.size_bytes === "number" && (record.execution_error_count === void 0 || typeof record.execution_error_count === "number") && (record.pass_rate === void 0 || typeof record.pass_rate === "number") && (record.target === void 0 || typeof record.target === "string") && (record.summary_path === void 0 || typeof record.summary_path === "string");
|
|
5116
|
+
}
|
|
5117
|
+
function validateGitResultsIndexCacheFile(value, params) {
|
|
5118
|
+
if (!value || typeof value !== "object") return void 0;
|
|
5119
|
+
const record = value;
|
|
5120
|
+
const repoDir = resolveGitResultsIndexRepoDir(params.repoDir);
|
|
5121
|
+
const repoKey = gitResultsIndexRepoKey(params.repoDir);
|
|
5122
|
+
if (record.schema_version !== GIT_RESULTS_INDEX_CACHE_SCHEMA_VERSION || record.result_layout_version !== GIT_RESULTS_INDEX_LAYOUT_VERSION || record.repo_key !== repoKey || record.repo_dir !== repoDir || record.ref !== params.ref || record.commit_sha !== params.commitSha || !Array.isArray(record.runs) || !record.runs.every(isGitListedRun)) {
|
|
5123
|
+
return void 0;
|
|
5124
|
+
}
|
|
5125
|
+
return value;
|
|
5126
|
+
}
|
|
5127
|
+
async function readGitResultsIndexCache(params) {
|
|
5128
|
+
const cacheFile = resolveGitResultsIndexCacheFile(params);
|
|
5129
|
+
try {
|
|
5130
|
+
const parsed = JSON.parse(await readFile2(cacheFile, "utf8"));
|
|
5131
|
+
return validateGitResultsIndexCacheFile(parsed, params)?.runs;
|
|
5132
|
+
} catch {
|
|
5133
|
+
return void 0;
|
|
5134
|
+
}
|
|
5135
|
+
}
|
|
5136
|
+
async function writeGitResultsIndexCache(params) {
|
|
5137
|
+
const cacheFile = resolveGitResultsIndexCacheFile(params);
|
|
5138
|
+
const envelope = {
|
|
5139
|
+
schema_version: GIT_RESULTS_INDEX_CACHE_SCHEMA_VERSION,
|
|
5140
|
+
result_layout_version: GIT_RESULTS_INDEX_LAYOUT_VERSION,
|
|
5141
|
+
repo_key: gitResultsIndexRepoKey(params.repoDir),
|
|
5142
|
+
repo_dir: resolveGitResultsIndexRepoDir(params.repoDir),
|
|
5143
|
+
ref: params.ref,
|
|
5144
|
+
commit_sha: params.commitSha,
|
|
5145
|
+
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5146
|
+
runs: params.runs
|
|
5147
|
+
};
|
|
5148
|
+
try {
|
|
5149
|
+
mkdirSync(path4.dirname(cacheFile), { recursive: true });
|
|
5150
|
+
const tempFile = `${cacheFile}.${process.pid}.${Date.now()}.tmp`;
|
|
5151
|
+
await writeFile(tempFile, `${JSON.stringify(envelope, null, 2)}
|
|
5152
|
+
`, "utf8");
|
|
5153
|
+
renameSync(tempFile, cacheFile);
|
|
5154
|
+
} catch {
|
|
5155
|
+
}
|
|
5156
|
+
}
|
|
5157
|
+
async function resolveGitRunsRefCommit(repoDir, ref = "origin/main") {
|
|
5158
|
+
try {
|
|
5159
|
+
const { stdout } = await runGit(["rev-parse", "--verify", `${ref}^{commit}`], {
|
|
5160
|
+
cwd: repoDir
|
|
5161
|
+
});
|
|
5162
|
+
return stdout.trim() || void 0;
|
|
5163
|
+
} catch (error) {
|
|
5164
|
+
if (isMissingGitRefError(error)) {
|
|
5165
|
+
return void 0;
|
|
5166
|
+
}
|
|
5167
|
+
throw error;
|
|
5168
|
+
}
|
|
5169
|
+
}
|
|
5170
|
+
async function listGitRunsCached(repoDir, ref = "origin/main") {
|
|
5171
|
+
const commitSha = await resolveGitRunsRefCommit(repoDir, ref);
|
|
5172
|
+
if (!commitSha) {
|
|
5173
|
+
return [];
|
|
5174
|
+
}
|
|
5175
|
+
const cached = await readGitResultsIndexCache({ repoDir, ref, commitSha });
|
|
5176
|
+
if (cached) {
|
|
5177
|
+
return [...cached];
|
|
5178
|
+
}
|
|
5179
|
+
const runs = await listGitRuns(repoDir, ref);
|
|
5180
|
+
await writeGitResultsIndexCache({ repoDir, ref, commitSha, runs });
|
|
5181
|
+
return runs;
|
|
5522
5182
|
}
|
|
5523
5183
|
async function listGitRuns(repoDir, ref = "origin/main") {
|
|
5524
5184
|
let treeOut;
|
|
@@ -5536,19 +5196,7 @@ async function listGitRuns(repoDir, ref = "origin/main") {
|
|
|
5536
5196
|
throw error;
|
|
5537
5197
|
}
|
|
5538
5198
|
const treePaths = treeOut.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
|
|
5539
|
-
const
|
|
5540
|
-
if (indexPaths.length === 0) {
|
|
5541
|
-
return [];
|
|
5542
|
-
}
|
|
5543
|
-
const batchInput = `${indexPaths.map((indexPath) => `${ref}:${indexPath}`).join("\n")}
|
|
5544
|
-
`;
|
|
5545
|
-
const blobs = parseGitBatchBlobs(await runGitBatch(repoDir, batchInput));
|
|
5546
|
-
if (blobs.length !== indexPaths.length) {
|
|
5547
|
-
throw new Error(
|
|
5548
|
-
`Expected ${indexPaths.length} git blobs but received ${blobs.length} while listing results runs`
|
|
5549
|
-
);
|
|
5550
|
-
}
|
|
5551
|
-
const summaryPaths = indexPaths.map((indexPath) => path5.posix.join(path5.posix.dirname(indexPath), "summary.json")).filter((summaryPath) => treePaths.includes(summaryPath));
|
|
5199
|
+
const summaryPaths = treePaths.filter((line) => line.endsWith("/summary.json"));
|
|
5552
5200
|
const summaryByPath = /* @__PURE__ */ new Map();
|
|
5553
5201
|
if (summaryPaths.length > 0) {
|
|
5554
5202
|
const summaryBatchInput = `${summaryPaths.map((summaryPath) => `${ref}:${summaryPath}`).join("\n")}
|
|
@@ -5561,12 +5209,24 @@ async function listGitRuns(repoDir, ref = "origin/main") {
|
|
|
5561
5209
|
summaryByPath.set(summaryPath, JSON.parse(blob.content.toString("utf8")));
|
|
5562
5210
|
}
|
|
5563
5211
|
}
|
|
5212
|
+
const indexPaths = buildGitManifestPaths(treePaths, summaryByPath);
|
|
5213
|
+
if (indexPaths.length === 0) {
|
|
5214
|
+
return [];
|
|
5215
|
+
}
|
|
5216
|
+
const batchInput = `${indexPaths.map((indexPath) => `${ref}:${indexPath}`).join("\n")}
|
|
5217
|
+
`;
|
|
5218
|
+
const blobs = parseGitBatchBlobs(await runGitBatch(repoDir, batchInput));
|
|
5219
|
+
if (blobs.length !== indexPaths.length) {
|
|
5220
|
+
throw new Error(
|
|
5221
|
+
`Expected ${indexPaths.length} git blobs but received ${blobs.length} while listing results runs`
|
|
5222
|
+
);
|
|
5223
|
+
}
|
|
5564
5224
|
const runs = blobs.flatMap((blob, index) => {
|
|
5565
5225
|
const manifestPath = indexPaths[index];
|
|
5566
|
-
const runDir =
|
|
5567
|
-
const summaryPath =
|
|
5226
|
+
const runDir = gitRunDirForManifestPath(manifestPath);
|
|
5227
|
+
const summaryPath = path4.posix.join(runDir, "summary.json");
|
|
5568
5228
|
const summary = summaryByPath.get(summaryPath);
|
|
5569
|
-
const relativeRunPath =
|
|
5229
|
+
const relativeRunPath = path4.posix.relative(RESULTS_REPO_RUNS_DIR, runDir);
|
|
5570
5230
|
const runId = buildGitRunId(relativeRunPath);
|
|
5571
5231
|
const rows = blob.content.toString("utf8").split(/\r?\n/).map((line) => line.trim()).filter(Boolean).flatMap(
|
|
5572
5232
|
(line) => {
|
|
@@ -5587,10 +5247,13 @@ async function listGitRuns(repoDir, ref = "origin/main") {
|
|
|
5587
5247
|
];
|
|
5588
5248
|
const rowScores = rows.map((row) => row.score).filter((score) => typeof score === "number" && Number.isFinite(score));
|
|
5589
5249
|
const avgScore = rowScores.length > 0 ? rowScores.reduce((sum, score) => sum + score, 0) / rowScores.length : 0;
|
|
5590
|
-
const timestamp = summary?.metadata?.timestamp?.trim() || rows[0]?.timestamp ||
|
|
5591
|
-
const displayName = summary?.metadata?.display_name?.trim() ||
|
|
5250
|
+
const timestamp = summary?.metadata?.timestamp?.trim() || rows[0]?.timestamp || path4.posix.basename(runDir);
|
|
5251
|
+
const displayName = summary?.metadata?.display_name?.trim() || path4.posix.basename(runDir);
|
|
5592
5252
|
const targets = summary?.metadata?.targets ?? rowTargets;
|
|
5593
5253
|
const passRate = computeAveragePassRate(summary?.run_summary) ?? avgScore;
|
|
5254
|
+
const executionErrorCount = rows.filter(
|
|
5255
|
+
(row) => row.execution_status === "execution_error"
|
|
5256
|
+
).length;
|
|
5594
5257
|
return [
|
|
5595
5258
|
{
|
|
5596
5259
|
run_id: runId,
|
|
@@ -5598,10 +5261,11 @@ async function listGitRuns(repoDir, ref = "origin/main") {
|
|
|
5598
5261
|
timestamp,
|
|
5599
5262
|
...passRate !== void 0 && { pass_rate: passRate },
|
|
5600
5263
|
...targets.length === 1 && targets[0] ? { target: targets[0] } : {},
|
|
5601
|
-
|
|
5264
|
+
index_path: manifestPath,
|
|
5602
5265
|
...summaryByPath.has(summaryPath) && { summary_path: summaryPath },
|
|
5603
5266
|
display_name: displayName,
|
|
5604
5267
|
test_count: summary?.metadata?.tests_run?.length ?? rowTestIds.length,
|
|
5268
|
+
...executionErrorCount > 0 && { execution_error_count: executionErrorCount },
|
|
5605
5269
|
avg_score: avgScore,
|
|
5606
5270
|
size_bytes: blob.size
|
|
5607
5271
|
}
|
|
@@ -5611,9 +5275,9 @@ async function listGitRuns(repoDir, ref = "origin/main") {
|
|
|
5611
5275
|
return runs;
|
|
5612
5276
|
}
|
|
5613
5277
|
async function materializeGitRun(repoDir, relativeRunPath, ref = "origin/main") {
|
|
5614
|
-
const normalizedRunPath = relativeRunPath.split(
|
|
5615
|
-
const runTreePath =
|
|
5616
|
-
const targetRunDir =
|
|
5278
|
+
const normalizedRunPath = relativeRunPath.split(path4.sep).join("/");
|
|
5279
|
+
const runTreePath = path4.posix.join(RESULTS_REPO_RUNS_DIR, normalizedRunPath);
|
|
5280
|
+
const targetRunDir = path4.join(repoDir, ...runTreePath.split("/"));
|
|
5617
5281
|
const { stdout: treeOut } = await runGit(["ls-tree", "-r", "--name-only", ref, runTreePath], {
|
|
5618
5282
|
cwd: repoDir
|
|
5619
5283
|
});
|
|
@@ -5629,16 +5293,16 @@ async function materializeGitRun(repoDir, relativeRunPath, ref = "origin/main")
|
|
|
5629
5293
|
`Expected ${filePaths.length} git blobs but received ${blobs.length} while materializing results run`
|
|
5630
5294
|
);
|
|
5631
5295
|
}
|
|
5632
|
-
const tempRoot = mkdtempSync(
|
|
5633
|
-
const tempRunDir =
|
|
5296
|
+
const tempRoot = mkdtempSync(path4.join(repoDir, ".agentv-run-"));
|
|
5297
|
+
const tempRunDir = path4.join(tempRoot, "run");
|
|
5634
5298
|
try {
|
|
5635
5299
|
for (const [index, filePath] of filePaths.entries()) {
|
|
5636
|
-
const relativeFilePath =
|
|
5637
|
-
const absolutePath =
|
|
5638
|
-
mkdirSync(
|
|
5300
|
+
const relativeFilePath = path4.posix.relative(runTreePath, filePath);
|
|
5301
|
+
const absolutePath = path4.join(tempRunDir, ...relativeFilePath.split("/"));
|
|
5302
|
+
mkdirSync(path4.dirname(absolutePath), { recursive: true });
|
|
5639
5303
|
writeFileSync(absolutePath, blobs[index].content);
|
|
5640
5304
|
}
|
|
5641
|
-
mkdirSync(
|
|
5305
|
+
mkdirSync(path4.dirname(targetRunDir), { recursive: true });
|
|
5642
5306
|
try {
|
|
5643
5307
|
renameSync(tempRunDir, targetRunDir);
|
|
5644
5308
|
} catch (error) {
|
|
@@ -5720,483 +5384,6 @@ function trimBaselineResult(result) {
|
|
|
5720
5384
|
}
|
|
5721
5385
|
return trimmed;
|
|
5722
5386
|
}
|
|
5723
|
-
var DEFAULT_CATEGORY = "Uncategorized";
|
|
5724
|
-
function deriveCategory(relativePath) {
|
|
5725
|
-
const parts = relativePath.split(/[/\\]/);
|
|
5726
|
-
if (parts.length <= 1) {
|
|
5727
|
-
return DEFAULT_CATEGORY;
|
|
5728
|
-
}
|
|
5729
|
-
const dirs = parts.slice(0, -1).filter((d) => d !== "evals");
|
|
5730
|
-
return dirs.length > 0 ? dirs.join("/") : DEFAULT_CATEGORY;
|
|
5731
|
-
}
|
|
5732
|
-
var OtelTraceExporter = class {
|
|
5733
|
-
constructor(options) {
|
|
5734
|
-
this.options = options;
|
|
5735
|
-
}
|
|
5736
|
-
provider = null;
|
|
5737
|
-
tracer = null;
|
|
5738
|
-
api = null;
|
|
5739
|
-
// biome-ignore lint/suspicious/noExplicitAny: OTel types loaded dynamically
|
|
5740
|
-
W3CPropagator = null;
|
|
5741
|
-
/** Initialize the OTel SDK. Returns false if OTel packages are not available. */
|
|
5742
|
-
async init() {
|
|
5743
|
-
try {
|
|
5744
|
-
const [sdkTraceNode, resourcesMod, semconvMod, api, coreMod] = await Promise.all([
|
|
5745
|
-
import("./src-76ISS3LH.js"),
|
|
5746
|
-
import("./esm-NNMQYIKT.js"),
|
|
5747
|
-
import("./esm-RVQPUGWH.js"),
|
|
5748
|
-
import("./esm-R77SNOF5.js"),
|
|
5749
|
-
import("./esm-ZADQ4XQH-QUFGGZL4.js").catch(() => null)
|
|
5750
|
-
]);
|
|
5751
|
-
const { NodeTracerProvider: Provider, SimpleSpanProcessor } = sdkTraceNode;
|
|
5752
|
-
const { resourceFromAttributes } = resourcesMod;
|
|
5753
|
-
const { ATTR_SERVICE_NAME } = semconvMod;
|
|
5754
|
-
const resourceAttributes = {
|
|
5755
|
-
[ATTR_SERVICE_NAME]: this.options.serviceName ?? "agentv",
|
|
5756
|
-
...this.options.resourceAttributes
|
|
5757
|
-
};
|
|
5758
|
-
const resource = resourceFromAttributes(resourceAttributes);
|
|
5759
|
-
const processors = [];
|
|
5760
|
-
if (this.options.endpoint) {
|
|
5761
|
-
const otlpHttp = await import("./esm-TTEMJEVV.js");
|
|
5762
|
-
const { OTLPTraceExporter } = otlpHttp;
|
|
5763
|
-
const exporter = new OTLPTraceExporter({
|
|
5764
|
-
url: this.options.endpoint,
|
|
5765
|
-
headers: this.options.headers
|
|
5766
|
-
});
|
|
5767
|
-
processors.push(new SimpleSpanProcessor(exporter));
|
|
5768
|
-
}
|
|
5769
|
-
if (this.options.otlpFilePath) {
|
|
5770
|
-
const { OtlpJsonFileExporter: OtlpJsonFileExporter2 } = await import("./otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js");
|
|
5771
|
-
processors.push(
|
|
5772
|
-
new SimpleSpanProcessor(
|
|
5773
|
-
new OtlpJsonFileExporter2(this.options.otlpFilePath, resourceAttributes)
|
|
5774
|
-
)
|
|
5775
|
-
);
|
|
5776
|
-
}
|
|
5777
|
-
if (processors.length === 0) {
|
|
5778
|
-
return false;
|
|
5779
|
-
}
|
|
5780
|
-
this.provider = new Provider({
|
|
5781
|
-
resource,
|
|
5782
|
-
spanProcessors: processors
|
|
5783
|
-
});
|
|
5784
|
-
this.provider.register();
|
|
5785
|
-
this.api = api;
|
|
5786
|
-
this.tracer = api.trace.getTracer("agentv", "1.0.0");
|
|
5787
|
-
this.W3CPropagator = coreMod?.W3CTraceContextPropagator ?? null;
|
|
5788
|
-
return true;
|
|
5789
|
-
} catch {
|
|
5790
|
-
return false;
|
|
5791
|
-
}
|
|
5792
|
-
}
|
|
5793
|
-
/** Export a single evaluation result as an OTel trace. */
|
|
5794
|
-
async exportResult(result) {
|
|
5795
|
-
if (!this.tracer || !this.api) return;
|
|
5796
|
-
const api = this.api;
|
|
5797
|
-
const tracer = this.tracer;
|
|
5798
|
-
const captureContent = this.options.captureContent ?? false;
|
|
5799
|
-
const startHr = toHrTime(result.startTime ?? result.timestamp);
|
|
5800
|
-
const endHr = toHrTime(result.endTime ?? result.timestamp);
|
|
5801
|
-
let parentCtx = api.ROOT_CONTEXT;
|
|
5802
|
-
const traceparent = process.env.TRACEPARENT;
|
|
5803
|
-
if (traceparent && this.W3CPropagator) {
|
|
5804
|
-
try {
|
|
5805
|
-
const propagator = new this.W3CPropagator();
|
|
5806
|
-
parentCtx = propagator.extract(
|
|
5807
|
-
api.ROOT_CONTEXT,
|
|
5808
|
-
{ traceparent, tracestate: process.env.TRACESTATE ?? "" },
|
|
5809
|
-
{
|
|
5810
|
-
get: (carrier, key) => carrier[key],
|
|
5811
|
-
keys: (carrier) => Object.keys(carrier)
|
|
5812
|
-
}
|
|
5813
|
-
);
|
|
5814
|
-
} catch {
|
|
5815
|
-
}
|
|
5816
|
-
}
|
|
5817
|
-
tracer.startActiveSpan(
|
|
5818
|
-
"agentv.eval",
|
|
5819
|
-
{ startTime: startHr },
|
|
5820
|
-
parentCtx,
|
|
5821
|
-
(rootSpan) => {
|
|
5822
|
-
rootSpan.setAttribute("gen_ai.operation.name", "evaluate");
|
|
5823
|
-
rootSpan.setAttribute("gen_ai.system", "agentv");
|
|
5824
|
-
rootSpan.setAttribute("agentv.test_id", result.testId);
|
|
5825
|
-
rootSpan.setAttribute("agentv.target", result.target);
|
|
5826
|
-
if (result.suite) rootSpan.setAttribute("agentv.suite", result.suite);
|
|
5827
|
-
rootSpan.setAttribute("agentv.score", result.score);
|
|
5828
|
-
if (captureContent && result.output.length > 0) {
|
|
5829
|
-
rootSpan.setAttribute("agentv.output_text", result.output);
|
|
5830
|
-
}
|
|
5831
|
-
if (result.durationMs != null)
|
|
5832
|
-
rootSpan.setAttribute("agentv.trace.duration_ms", result.durationMs);
|
|
5833
|
-
if (result.costUsd != null) rootSpan.setAttribute("agentv.trace.cost_usd", result.costUsd);
|
|
5834
|
-
if (result.tokenUsage) {
|
|
5835
|
-
if (result.tokenUsage.input != null) {
|
|
5836
|
-
rootSpan.setAttribute("agentv.trace.token_input", result.tokenUsage.input);
|
|
5837
|
-
}
|
|
5838
|
-
if (result.tokenUsage.output != null) {
|
|
5839
|
-
rootSpan.setAttribute("agentv.trace.token_output", result.tokenUsage.output);
|
|
5840
|
-
}
|
|
5841
|
-
if (result.tokenUsage.cached != null) {
|
|
5842
|
-
rootSpan.setAttribute("agentv.trace.token_cached", result.tokenUsage.cached);
|
|
5843
|
-
}
|
|
5844
|
-
}
|
|
5845
|
-
if (result.trace) {
|
|
5846
|
-
const t = result.trace;
|
|
5847
|
-
rootSpan.setAttribute("agentv.trace.event_count", t.eventCount);
|
|
5848
|
-
rootSpan.setAttribute(
|
|
5849
|
-
"agentv.trace.tool_names",
|
|
5850
|
-
Object.keys(t.toolCalls).sort().join(",")
|
|
5851
|
-
);
|
|
5852
|
-
if (t.llmCallCount != null)
|
|
5853
|
-
rootSpan.setAttribute("agentv.trace.llm_call_count", t.llmCallCount);
|
|
5854
|
-
}
|
|
5855
|
-
const traceMessages = result.trace?.messages ?? [];
|
|
5856
|
-
if (traceMessages.length > 0) {
|
|
5857
|
-
const parentCtx2 = api.trace.setSpan(api.context.active(), rootSpan);
|
|
5858
|
-
if (this.options.groupTurns) {
|
|
5859
|
-
const turns = groupMessagesIntoTurns(traceMessages);
|
|
5860
|
-
if (turns.length > 1) {
|
|
5861
|
-
for (const [i, turn] of turns.entries()) {
|
|
5862
|
-
api.context.with(parentCtx2, () => {
|
|
5863
|
-
tracer.startActiveSpan(
|
|
5864
|
-
`agentv.turn.${i + 1}`,
|
|
5865
|
-
{},
|
|
5866
|
-
(turnSpan) => {
|
|
5867
|
-
const turnCtx = api.trace.setSpan(api.context.active(), turnSpan);
|
|
5868
|
-
for (const msg of turn.messages) {
|
|
5869
|
-
this.exportMessage(tracer, api, turnCtx, msg, captureContent);
|
|
5870
|
-
}
|
|
5871
|
-
turnSpan.end();
|
|
5872
|
-
}
|
|
5873
|
-
);
|
|
5874
|
-
});
|
|
5875
|
-
}
|
|
5876
|
-
} else {
|
|
5877
|
-
for (const msg of traceMessages) {
|
|
5878
|
-
this.exportMessage(tracer, api, parentCtx2, msg, captureContent);
|
|
5879
|
-
}
|
|
5880
|
-
}
|
|
5881
|
-
} else {
|
|
5882
|
-
for (const msg of traceMessages) {
|
|
5883
|
-
this.exportMessage(tracer, api, parentCtx2, msg, captureContent);
|
|
5884
|
-
}
|
|
5885
|
-
}
|
|
5886
|
-
}
|
|
5887
|
-
if (result.scores) {
|
|
5888
|
-
for (const score of result.scores) {
|
|
5889
|
-
rootSpan.addEvent(`agentv.grader.${score.name}`, {
|
|
5890
|
-
"agentv.grader.score": score.score,
|
|
5891
|
-
"agentv.grader.type": score.type,
|
|
5892
|
-
...score.verdict ? { "agentv.grader.verdict": score.verdict } : {}
|
|
5893
|
-
});
|
|
5894
|
-
}
|
|
5895
|
-
}
|
|
5896
|
-
if (result.error) {
|
|
5897
|
-
rootSpan.setStatus({ code: api.SpanStatusCode.ERROR, message: result.error });
|
|
5898
|
-
} else {
|
|
5899
|
-
rootSpan.setStatus({ code: api.SpanStatusCode.OK });
|
|
5900
|
-
}
|
|
5901
|
-
rootSpan.end(endHr);
|
|
5902
|
-
}
|
|
5903
|
-
);
|
|
5904
|
-
}
|
|
5905
|
-
/** Flush pending spans and shut down. */
|
|
5906
|
-
async shutdown() {
|
|
5907
|
-
await this.provider?.shutdown();
|
|
5908
|
-
}
|
|
5909
|
-
/** Create a streaming observer for real-time span export */
|
|
5910
|
-
createStreamingObserver() {
|
|
5911
|
-
if (!this.tracer || !this.api) return null;
|
|
5912
|
-
let parentCtx;
|
|
5913
|
-
const traceparent = process.env.TRACEPARENT;
|
|
5914
|
-
if (traceparent && this.W3CPropagator) {
|
|
5915
|
-
try {
|
|
5916
|
-
const propagator = new this.W3CPropagator();
|
|
5917
|
-
parentCtx = propagator.extract(
|
|
5918
|
-
this.api.ROOT_CONTEXT,
|
|
5919
|
-
{ traceparent, tracestate: process.env.TRACESTATE ?? "" },
|
|
5920
|
-
{
|
|
5921
|
-
get: (carrier, key) => carrier[key],
|
|
5922
|
-
keys: (carrier) => Object.keys(carrier)
|
|
5923
|
-
}
|
|
5924
|
-
);
|
|
5925
|
-
} catch {
|
|
5926
|
-
}
|
|
5927
|
-
}
|
|
5928
|
-
return new OtelStreamingObserver(
|
|
5929
|
-
this.tracer,
|
|
5930
|
-
this.api,
|
|
5931
|
-
this.options.captureContent ?? false,
|
|
5932
|
-
parentCtx
|
|
5933
|
-
);
|
|
5934
|
-
}
|
|
5935
|
-
// -----------------------------------------------------------------------
|
|
5936
|
-
// Private helpers
|
|
5937
|
-
// -----------------------------------------------------------------------
|
|
5938
|
-
exportMessage(tracer, api, parentCtx, msg, captureContent) {
|
|
5939
|
-
const isAssistant = msg.role === "assistant";
|
|
5940
|
-
const model = msg.metadata?.model ? String(msg.metadata.model) : void 0;
|
|
5941
|
-
const spanName = isAssistant ? `chat ${model ?? "unknown"}` : `gen_ai.message.${msg.role}`;
|
|
5942
|
-
const startHr = toHrTime(msg.startTime);
|
|
5943
|
-
const endHr = toHrTime(msg.endTime);
|
|
5944
|
-
api.context.with(parentCtx, () => {
|
|
5945
|
-
tracer.startActiveSpan(
|
|
5946
|
-
spanName,
|
|
5947
|
-
{ startTime: startHr },
|
|
5948
|
-
parentCtx,
|
|
5949
|
-
(span) => {
|
|
5950
|
-
if (isAssistant) {
|
|
5951
|
-
span.setAttribute("gen_ai.operation.name", "chat");
|
|
5952
|
-
}
|
|
5953
|
-
if (model) {
|
|
5954
|
-
span.setAttribute("gen_ai.request.model", model);
|
|
5955
|
-
span.setAttribute("gen_ai.response.model", model);
|
|
5956
|
-
}
|
|
5957
|
-
if (msg.tokenUsage) {
|
|
5958
|
-
if (msg.tokenUsage.input != null) {
|
|
5959
|
-
span.setAttribute("gen_ai.usage.input_tokens", msg.tokenUsage.input);
|
|
5960
|
-
}
|
|
5961
|
-
if (msg.tokenUsage.output != null) {
|
|
5962
|
-
span.setAttribute("gen_ai.usage.output_tokens", msg.tokenUsage.output);
|
|
5963
|
-
}
|
|
5964
|
-
if (msg.tokenUsage.cached != null) {
|
|
5965
|
-
span.setAttribute("gen_ai.usage.cache_read.input_tokens", msg.tokenUsage.cached);
|
|
5966
|
-
}
|
|
5967
|
-
}
|
|
5968
|
-
if (captureContent && msg.content != null) {
|
|
5969
|
-
span.setAttribute(
|
|
5970
|
-
"gen_ai.output.messages",
|
|
5971
|
-
typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
|
|
5972
|
-
);
|
|
5973
|
-
}
|
|
5974
|
-
if (msg.toolCalls) {
|
|
5975
|
-
const msgCtx = api.trace.setSpan(api.context.active(), span);
|
|
5976
|
-
for (const tc of msg.toolCalls) {
|
|
5977
|
-
api.context.with(msgCtx, () => {
|
|
5978
|
-
tracer.startActiveSpan(
|
|
5979
|
-
`execute_tool ${tc.tool}`,
|
|
5980
|
-
{},
|
|
5981
|
-
msgCtx,
|
|
5982
|
-
(toolSpan) => {
|
|
5983
|
-
toolSpan.setAttribute("gen_ai.tool.name", tc.tool);
|
|
5984
|
-
if (tc.id) toolSpan.setAttribute("gen_ai.tool.call.id", tc.id);
|
|
5985
|
-
if (captureContent) {
|
|
5986
|
-
if (tc.input != null) {
|
|
5987
|
-
toolSpan.setAttribute(
|
|
5988
|
-
"gen_ai.tool.call.arguments",
|
|
5989
|
-
typeof tc.input === "string" ? tc.input : JSON.stringify(tc.input)
|
|
5990
|
-
);
|
|
5991
|
-
}
|
|
5992
|
-
if (tc.output != null) {
|
|
5993
|
-
toolSpan.setAttribute(
|
|
5994
|
-
"gen_ai.tool.call.result",
|
|
5995
|
-
typeof tc.output === "string" ? tc.output : JSON.stringify(tc.output)
|
|
5996
|
-
);
|
|
5997
|
-
}
|
|
5998
|
-
}
|
|
5999
|
-
toolSpan.end();
|
|
6000
|
-
}
|
|
6001
|
-
);
|
|
6002
|
-
});
|
|
6003
|
-
}
|
|
6004
|
-
}
|
|
6005
|
-
span.end(endHr);
|
|
6006
|
-
}
|
|
6007
|
-
);
|
|
6008
|
-
});
|
|
6009
|
-
}
|
|
6010
|
-
};
|
|
6011
|
-
var OtelStreamingObserver = class {
|
|
6012
|
-
constructor(tracer, api, captureContent, parentCtx) {
|
|
6013
|
-
this.tracer = tracer;
|
|
6014
|
-
this.api = api;
|
|
6015
|
-
this.captureContent = captureContent;
|
|
6016
|
-
this.parentCtx = parentCtx;
|
|
6017
|
-
}
|
|
6018
|
-
// biome-ignore lint/suspicious/noExplicitAny: OTel span type loaded dynamically
|
|
6019
|
-
rootSpan = null;
|
|
6020
|
-
// biome-ignore lint/suspicious/noExplicitAny: OTel context loaded dynamically
|
|
6021
|
-
rootCtx = null;
|
|
6022
|
-
observedChildSpans = false;
|
|
6023
|
-
pendingMetrics = null;
|
|
6024
|
-
/** Create root eval span immediately (visible in backend right away) */
|
|
6025
|
-
startEvalCase(testId, target, evalSet) {
|
|
6026
|
-
this.pendingMetrics = null;
|
|
6027
|
-
this.observedChildSpans = false;
|
|
6028
|
-
const ctx = this.parentCtx ?? this.api.context.active();
|
|
6029
|
-
this.rootSpan = this.tracer.startSpan("agentv.eval", void 0, ctx);
|
|
6030
|
-
this.rootSpan.setAttribute("gen_ai.operation.name", "evaluate");
|
|
6031
|
-
this.rootSpan.setAttribute("gen_ai.system", "agentv");
|
|
6032
|
-
this.rootSpan.setAttribute("agentv.test_id", testId);
|
|
6033
|
-
this.rootSpan.setAttribute("agentv.target", target);
|
|
6034
|
-
if (evalSet) this.rootSpan.setAttribute("agentv.suite", evalSet);
|
|
6035
|
-
this.rootCtx = this.api.trace.setSpan(this.api.context.active(), this.rootSpan);
|
|
6036
|
-
}
|
|
6037
|
-
/** Create and immediately export a tool span */
|
|
6038
|
-
onToolCall(name, input, output, _durationMs, toolCallId) {
|
|
6039
|
-
if (!this.rootCtx) return;
|
|
6040
|
-
this.observedChildSpans = true;
|
|
6041
|
-
this.api.context.with(this.rootCtx, () => {
|
|
6042
|
-
const span = this.tracer.startSpan(`execute_tool ${name}`, void 0, this.rootCtx);
|
|
6043
|
-
span.setAttribute("gen_ai.tool.name", name);
|
|
6044
|
-
if (toolCallId) span.setAttribute("gen_ai.tool.call.id", toolCallId);
|
|
6045
|
-
if (this.captureContent) {
|
|
6046
|
-
if (input != null)
|
|
6047
|
-
span.setAttribute(
|
|
6048
|
-
"gen_ai.tool.call.arguments",
|
|
6049
|
-
typeof input === "string" ? input : JSON.stringify(input)
|
|
6050
|
-
);
|
|
6051
|
-
if (output != null)
|
|
6052
|
-
span.setAttribute(
|
|
6053
|
-
"gen_ai.tool.call.result",
|
|
6054
|
-
typeof output === "string" ? output : JSON.stringify(output)
|
|
6055
|
-
);
|
|
6056
|
-
}
|
|
6057
|
-
span.end();
|
|
6058
|
-
});
|
|
6059
|
-
}
|
|
6060
|
-
/** Create and immediately export an LLM span */
|
|
6061
|
-
onLlmCall(model, tokenUsage) {
|
|
6062
|
-
if (!this.rootCtx) return;
|
|
6063
|
-
this.observedChildSpans = true;
|
|
6064
|
-
this.api.context.with(this.rootCtx, () => {
|
|
6065
|
-
const span = this.tracer.startSpan(`chat ${model}`, void 0, this.rootCtx);
|
|
6066
|
-
span.setAttribute("gen_ai.operation.name", "chat");
|
|
6067
|
-
span.setAttribute("gen_ai.request.model", model);
|
|
6068
|
-
span.setAttribute("gen_ai.response.model", model);
|
|
6069
|
-
if (tokenUsage) {
|
|
6070
|
-
if (tokenUsage.input != null)
|
|
6071
|
-
span.setAttribute("gen_ai.usage.input_tokens", tokenUsage.input);
|
|
6072
|
-
if (tokenUsage.output != null)
|
|
6073
|
-
span.setAttribute("gen_ai.usage.output_tokens", tokenUsage.output);
|
|
6074
|
-
if (tokenUsage.cached != null)
|
|
6075
|
-
span.setAttribute("gen_ai.usage.cache_read.input_tokens", tokenUsage.cached);
|
|
6076
|
-
}
|
|
6077
|
-
span.end();
|
|
6078
|
-
});
|
|
6079
|
-
}
|
|
6080
|
-
/** Record final execution metrics before the root span is finalized. */
|
|
6081
|
-
recordEvalMetrics(result) {
|
|
6082
|
-
this.pendingMetrics = result;
|
|
6083
|
-
}
|
|
6084
|
-
/** Finalize root span with score/verdict after evaluation completes */
|
|
6085
|
-
finalizeEvalCase(score, error) {
|
|
6086
|
-
if (!this.rootSpan) return;
|
|
6087
|
-
this.rootSpan.setAttribute("agentv.score", score);
|
|
6088
|
-
if (this.pendingMetrics?.durationMs != null) {
|
|
6089
|
-
this.rootSpan.setAttribute("agentv.trace.duration_ms", this.pendingMetrics.durationMs);
|
|
6090
|
-
}
|
|
6091
|
-
if (this.pendingMetrics?.costUsd != null) {
|
|
6092
|
-
this.rootSpan.setAttribute("agentv.trace.cost_usd", this.pendingMetrics.costUsd);
|
|
6093
|
-
}
|
|
6094
|
-
if (this.pendingMetrics?.tokenUsage) {
|
|
6095
|
-
if (this.pendingMetrics.tokenUsage.input != null) {
|
|
6096
|
-
this.rootSpan.setAttribute(
|
|
6097
|
-
"agentv.trace.token_input",
|
|
6098
|
-
this.pendingMetrics.tokenUsage.input
|
|
6099
|
-
);
|
|
6100
|
-
}
|
|
6101
|
-
if (this.pendingMetrics.tokenUsage.output != null) {
|
|
6102
|
-
this.rootSpan.setAttribute(
|
|
6103
|
-
"agentv.trace.token_output",
|
|
6104
|
-
this.pendingMetrics.tokenUsage.output
|
|
6105
|
-
);
|
|
6106
|
-
}
|
|
6107
|
-
if (this.pendingMetrics.tokenUsage.cached != null) {
|
|
6108
|
-
this.rootSpan.setAttribute(
|
|
6109
|
-
"agentv.trace.token_cached",
|
|
6110
|
-
this.pendingMetrics.tokenUsage.cached
|
|
6111
|
-
);
|
|
6112
|
-
}
|
|
6113
|
-
}
|
|
6114
|
-
if (this.pendingMetrics?.trace) {
|
|
6115
|
-
this.rootSpan.setAttribute("agentv.trace.event_count", this.pendingMetrics.trace.eventCount);
|
|
6116
|
-
this.rootSpan.setAttribute(
|
|
6117
|
-
"agentv.trace.tool_names",
|
|
6118
|
-
Object.keys(this.pendingMetrics.trace.toolCalls).sort().join(",")
|
|
6119
|
-
);
|
|
6120
|
-
if (this.pendingMetrics.trace.llmCallCount != null) {
|
|
6121
|
-
this.rootSpan.setAttribute(
|
|
6122
|
-
"agentv.trace.llm_call_count",
|
|
6123
|
-
this.pendingMetrics.trace.llmCallCount
|
|
6124
|
-
);
|
|
6125
|
-
}
|
|
6126
|
-
}
|
|
6127
|
-
if (error) {
|
|
6128
|
-
this.rootSpan.setStatus({ code: this.api.SpanStatusCode.ERROR, message: error });
|
|
6129
|
-
} else {
|
|
6130
|
-
this.rootSpan.setStatus({ code: this.api.SpanStatusCode.OK });
|
|
6131
|
-
}
|
|
6132
|
-
this.rootSpan.end();
|
|
6133
|
-
this.rootSpan = null;
|
|
6134
|
-
this.rootCtx = null;
|
|
6135
|
-
this.observedChildSpans = false;
|
|
6136
|
-
this.pendingMetrics = null;
|
|
6137
|
-
}
|
|
6138
|
-
/** Backfill child spans from the completed result when the provider emitted no live callbacks. */
|
|
6139
|
-
completeFromResult(result) {
|
|
6140
|
-
this.recordEvalMetrics({
|
|
6141
|
-
durationMs: result.durationMs,
|
|
6142
|
-
costUsd: result.costUsd,
|
|
6143
|
-
tokenUsage: result.tokenUsage,
|
|
6144
|
-
trace: result.trace
|
|
6145
|
-
});
|
|
6146
|
-
if (this.observedChildSpans || !this.rootCtx) {
|
|
6147
|
-
return;
|
|
6148
|
-
}
|
|
6149
|
-
const model = result.trace.messages.find((msg) => msg.role === "assistant")?.metadata?.model ?? result.target ?? "unknown";
|
|
6150
|
-
this.onLlmCall(String(model), result.tokenUsage);
|
|
6151
|
-
for (const message of result.trace.messages) {
|
|
6152
|
-
for (const toolCall of message.toolCalls ?? []) {
|
|
6153
|
-
this.onToolCall(
|
|
6154
|
-
toolCall.tool,
|
|
6155
|
-
toolCall.input,
|
|
6156
|
-
toolCall.output,
|
|
6157
|
-
toolCall.durationMs ?? 0,
|
|
6158
|
-
toolCall.id
|
|
6159
|
-
);
|
|
6160
|
-
}
|
|
6161
|
-
}
|
|
6162
|
-
}
|
|
6163
|
-
/** Return the active eval span's trace ID and span ID for Braintrust trace bridging */
|
|
6164
|
-
getActiveSpanIds() {
|
|
6165
|
-
if (!this.rootSpan) return null;
|
|
6166
|
-
try {
|
|
6167
|
-
const spanCtx = this.rootSpan.spanContext?.() ?? this.rootSpan._spanContext;
|
|
6168
|
-
if (!spanCtx?.traceId || !spanCtx?.spanId) return null;
|
|
6169
|
-
return { parentSpanId: spanCtx.spanId, rootSpanId: spanCtx.traceId };
|
|
6170
|
-
} catch {
|
|
6171
|
-
return null;
|
|
6172
|
-
}
|
|
6173
|
-
}
|
|
6174
|
-
/** Get ProviderStreamCallbacks for passing to providers */
|
|
6175
|
-
getStreamCallbacks() {
|
|
6176
|
-
return {
|
|
6177
|
-
onToolCallEnd: (name, input, output, durationMs, toolCallId) => this.onToolCall(name, input, output, durationMs, toolCallId),
|
|
6178
|
-
onLlmCallEnd: (model, tokenUsage) => this.onLlmCall(model, tokenUsage),
|
|
6179
|
-
getActiveSpanIds: () => this.getActiveSpanIds()
|
|
6180
|
-
};
|
|
6181
|
-
}
|
|
6182
|
-
};
|
|
6183
|
-
function groupMessagesIntoTurns(messages) {
|
|
6184
|
-
const turns = [];
|
|
6185
|
-
let current = [];
|
|
6186
|
-
for (const msg of messages) {
|
|
6187
|
-
if (msg.role === "user" && current.length > 0) {
|
|
6188
|
-
turns.push({ messages: current });
|
|
6189
|
-
current = [];
|
|
6190
|
-
}
|
|
6191
|
-
current.push(msg);
|
|
6192
|
-
}
|
|
6193
|
-
if (current.length > 0) turns.push({ messages: current });
|
|
6194
|
-
return turns;
|
|
6195
|
-
}
|
|
6196
|
-
function toHrTime(iso) {
|
|
6197
|
-
if (!iso) return void 0;
|
|
6198
|
-
return new Date(iso).getTime();
|
|
6199
|
-
}
|
|
6200
5387
|
var RunBudgetTracker = class {
|
|
6201
5388
|
constructor(capUsd) {
|
|
6202
5389
|
this.capUsd = capUsd;
|
|
@@ -6376,7 +5563,7 @@ function parseClaudeSession(jsonl) {
|
|
|
6376
5563
|
durationMs = new Date(endTimestamp).getTime() - new Date(startTimestamp).getTime();
|
|
6377
5564
|
}
|
|
6378
5565
|
const source = {
|
|
6379
|
-
provider: "claude",
|
|
5566
|
+
provider: "claude-sdk",
|
|
6380
5567
|
sessionId,
|
|
6381
5568
|
projectPath,
|
|
6382
5569
|
startedAt: startTimestamp,
|
|
@@ -6426,7 +5613,7 @@ function extractAssistantContent(content) {
|
|
|
6426
5613
|
case "tool_use":
|
|
6427
5614
|
if (block.name) {
|
|
6428
5615
|
toolCalls.push(
|
|
6429
|
-
normalizeToolCall("claude", {
|
|
5616
|
+
normalizeToolCall("claude-sdk", {
|
|
6430
5617
|
tool: block.name,
|
|
6431
5618
|
input: block.input,
|
|
6432
5619
|
id: block.id
|
|
@@ -6521,7 +5708,7 @@ function parseCodexSession(jsonl) {
|
|
|
6521
5708
|
} else {
|
|
6522
5709
|
input = payload.arguments;
|
|
6523
5710
|
}
|
|
6524
|
-
const toolCall = normalizeToolCall("codex", {
|
|
5711
|
+
const toolCall = normalizeToolCall("codex-cli", {
|
|
6525
5712
|
tool: toolName,
|
|
6526
5713
|
input,
|
|
6527
5714
|
id: callId
|
|
@@ -6549,7 +5736,7 @@ function parseCodexSession(jsonl) {
|
|
|
6549
5736
|
} else {
|
|
6550
5737
|
input = payload.arguments;
|
|
6551
5738
|
}
|
|
6552
|
-
const toolCall = normalizeToolCall("codex", {
|
|
5739
|
+
const toolCall = normalizeToolCall("codex-cli", {
|
|
6553
5740
|
tool: toolName,
|
|
6554
5741
|
input,
|
|
6555
5742
|
id: callId
|
|
@@ -6623,7 +5810,7 @@ function extractResponseItemContent(content) {
|
|
|
6623
5810
|
}
|
|
6624
5811
|
return parts.length > 0 ? parts.join("") : void 0;
|
|
6625
5812
|
}
|
|
6626
|
-
var DEFAULT_SESSIONS_DIR = () =>
|
|
5813
|
+
var DEFAULT_SESSIONS_DIR = () => path5.join(homedir(), ".codex", "sessions");
|
|
6627
5814
|
async function discoverCodexSessions(opts) {
|
|
6628
5815
|
const sessionsDir = opts?.sessionsDir ?? DEFAULT_SESSIONS_DIR();
|
|
6629
5816
|
const limit = opts?.latest ? 1 : opts?.limit ?? 10;
|
|
@@ -6635,7 +5822,7 @@ async function discoverCodexSessions(opts) {
|
|
|
6635
5822
|
return [];
|
|
6636
5823
|
}
|
|
6637
5824
|
for (const year of yearDirs) {
|
|
6638
|
-
const yearPath =
|
|
5825
|
+
const yearPath = path5.join(sessionsDir, year);
|
|
6639
5826
|
let monthDirs;
|
|
6640
5827
|
try {
|
|
6641
5828
|
monthDirs = await readdir2(yearPath);
|
|
@@ -6643,7 +5830,7 @@ async function discoverCodexSessions(opts) {
|
|
|
6643
5830
|
continue;
|
|
6644
5831
|
}
|
|
6645
5832
|
for (const month of monthDirs) {
|
|
6646
|
-
const monthPath =
|
|
5833
|
+
const monthPath = path5.join(yearPath, month);
|
|
6647
5834
|
let dayDirs;
|
|
6648
5835
|
try {
|
|
6649
5836
|
dayDirs = await readdir2(monthPath);
|
|
@@ -6655,7 +5842,7 @@ async function discoverCodexSessions(opts) {
|
|
|
6655
5842
|
const dirDate = `${year}-${month}-${day}`;
|
|
6656
5843
|
if (dirDate !== opts.date) continue;
|
|
6657
5844
|
}
|
|
6658
|
-
const dayPath =
|
|
5845
|
+
const dayPath = path5.join(monthPath, day);
|
|
6659
5846
|
let files;
|
|
6660
5847
|
try {
|
|
6661
5848
|
files = await readdir2(dayPath);
|
|
@@ -6664,7 +5851,7 @@ async function discoverCodexSessions(opts) {
|
|
|
6664
5851
|
}
|
|
6665
5852
|
for (const file of files) {
|
|
6666
5853
|
if (!file.startsWith("rollout-") || !file.endsWith(".jsonl")) continue;
|
|
6667
|
-
const filePath =
|
|
5854
|
+
const filePath = path5.join(dayPath, file);
|
|
6668
5855
|
const nameWithoutExt = file.replace(/\.jsonl$/, "");
|
|
6669
5856
|
const parts = nameWithoutExt.split("-");
|
|
6670
5857
|
const sessionId = parts.length >= 6 ? parts.slice(-5).join("-") : nameWithoutExt;
|
|
@@ -6683,7 +5870,7 @@ async function discoverCodexSessions(opts) {
|
|
|
6683
5870
|
sessions.sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime());
|
|
6684
5871
|
return sessions.slice(0, limit);
|
|
6685
5872
|
}
|
|
6686
|
-
var DEFAULT_PROJECTS_DIR = () =>
|
|
5873
|
+
var DEFAULT_PROJECTS_DIR = () => path6.join(homedir2(), ".claude", "projects");
|
|
6687
5874
|
function encodeProjectPath(projectPath) {
|
|
6688
5875
|
return projectPath.replace(/\//g, "-");
|
|
6689
5876
|
}
|
|
@@ -6702,7 +5889,7 @@ async function discoverClaudeSessions(opts) {
|
|
|
6702
5889
|
}
|
|
6703
5890
|
const sessions = [];
|
|
6704
5891
|
for (const projectDir of projectDirs) {
|
|
6705
|
-
const dirPath =
|
|
5892
|
+
const dirPath = path6.join(projectsDir, projectDir);
|
|
6706
5893
|
let entries;
|
|
6707
5894
|
try {
|
|
6708
5895
|
entries = await readdir3(dirPath);
|
|
@@ -6713,7 +5900,7 @@ async function discoverClaudeSessions(opts) {
|
|
|
6713
5900
|
if (!entry.endsWith(".jsonl")) continue;
|
|
6714
5901
|
const sessionId = entry.replace(/\.jsonl$/, "");
|
|
6715
5902
|
if (opts?.sessionId && sessionId !== opts.sessionId) continue;
|
|
6716
|
-
const filePath =
|
|
5903
|
+
const filePath = path6.join(dirPath, entry);
|
|
6717
5904
|
let updatedAt;
|
|
6718
5905
|
try {
|
|
6719
5906
|
const fileStat = await stat3(filePath);
|
|
@@ -6737,11 +5924,13 @@ var TranscriptProvider = class _TranscriptProvider {
|
|
|
6737
5924
|
kind = "transcript";
|
|
6738
5925
|
targetName;
|
|
6739
5926
|
entries;
|
|
5927
|
+
entriesByTestId;
|
|
6740
5928
|
cursor = 0;
|
|
6741
5929
|
constructor(targetName, entries) {
|
|
6742
5930
|
this.targetName = targetName;
|
|
6743
5931
|
this.id = `transcript:${targetName}`;
|
|
6744
5932
|
this.entries = entries;
|
|
5933
|
+
this.entriesByTestId = new Map(entries.map((entry) => [entry.testId, entry]));
|
|
6745
5934
|
}
|
|
6746
5935
|
/**
|
|
6747
5936
|
* Create a TranscriptProvider from a JSONL file path.
|
|
@@ -6758,13 +5947,11 @@ var TranscriptProvider = class _TranscriptProvider {
|
|
|
6758
5947
|
get lineCount() {
|
|
6759
5948
|
return this.entries.length;
|
|
6760
5949
|
}
|
|
6761
|
-
|
|
6762
|
-
|
|
6763
|
-
|
|
6764
|
-
|
|
6765
|
-
|
|
6766
|
-
}
|
|
6767
|
-
const entry = this.entries[this.cursor++];
|
|
5950
|
+
get testIds() {
|
|
5951
|
+
return this.entries.map((entry) => entry.testId);
|
|
5952
|
+
}
|
|
5953
|
+
async invoke(request) {
|
|
5954
|
+
const entry = request.evalCaseId ? this.entryForTestId(request.evalCaseId) : this.nextPositionalEntry();
|
|
6768
5955
|
return {
|
|
6769
5956
|
output: entry.messages,
|
|
6770
5957
|
tokenUsage: entry.tokenUsage ? {
|
|
@@ -6778,7 +5965,153 @@ var TranscriptProvider = class _TranscriptProvider {
|
|
|
6778
5965
|
startTime: entry.source.startedAt
|
|
6779
5966
|
};
|
|
6780
5967
|
}
|
|
5968
|
+
entryForTestId(testId) {
|
|
5969
|
+
const entry = this.entriesByTestId.get(testId);
|
|
5970
|
+
if (entry) {
|
|
5971
|
+
return entry;
|
|
5972
|
+
}
|
|
5973
|
+
throw new Error(
|
|
5974
|
+
`Transcript replay found no entry for test_id=${testId}. Available test_id values: ${this.testIds.join(", ") || "<none>"}`
|
|
5975
|
+
);
|
|
5976
|
+
}
|
|
5977
|
+
nextPositionalEntry() {
|
|
5978
|
+
if (this.cursor >= this.entries.length) {
|
|
5979
|
+
throw new Error(
|
|
5980
|
+
`Transcript exhausted: ${this.entries.length} entr${this.entries.length === 1 ? "y" : "ies"} available but ${this.cursor + 1} invocations attempted. Each transcript entry maps to one test case.`
|
|
5981
|
+
);
|
|
5982
|
+
}
|
|
5983
|
+
return this.entries[this.cursor++];
|
|
5984
|
+
}
|
|
6781
5985
|
};
|
|
5986
|
+
function parseCopilotEvents(eventsJsonl) {
|
|
5987
|
+
const messages = [];
|
|
5988
|
+
const meta = { sessionId: "", model: "", cwd: "" };
|
|
5989
|
+
let totalInputTokens = 0;
|
|
5990
|
+
let totalOutputTokens = 0;
|
|
5991
|
+
let hasUsage = false;
|
|
5992
|
+
let startTimestamp;
|
|
5993
|
+
let endTimestamp;
|
|
5994
|
+
const toolCallsInProgress = /* @__PURE__ */ new Map();
|
|
5995
|
+
const lines = eventsJsonl.split("\n").filter((l) => l.trim().length > 0);
|
|
5996
|
+
for (const line of lines) {
|
|
5997
|
+
let event;
|
|
5998
|
+
try {
|
|
5999
|
+
event = JSON.parse(line);
|
|
6000
|
+
} catch {
|
|
6001
|
+
continue;
|
|
6002
|
+
}
|
|
6003
|
+
const eventType = event.type;
|
|
6004
|
+
if (!eventType) continue;
|
|
6005
|
+
const data = event.data ?? {};
|
|
6006
|
+
switch (eventType) {
|
|
6007
|
+
case "session.start": {
|
|
6008
|
+
meta.sessionId = String(data.sessionId ?? "");
|
|
6009
|
+
const ctx = data.context;
|
|
6010
|
+
meta.cwd = String(ctx?.cwd ?? "");
|
|
6011
|
+
meta.repository = ctx?.repository ? String(ctx.repository) : void 0;
|
|
6012
|
+
meta.branch = ctx?.branch ? String(ctx.branch) : void 0;
|
|
6013
|
+
const ts = event.timestamp ?? data.startTime;
|
|
6014
|
+
meta.startedAt = ts ? String(ts) : void 0;
|
|
6015
|
+
startTimestamp = ts ? String(ts) : void 0;
|
|
6016
|
+
break;
|
|
6017
|
+
}
|
|
6018
|
+
case "user.message": {
|
|
6019
|
+
messages.push({
|
|
6020
|
+
role: "user",
|
|
6021
|
+
content: data.content != null ? String(data.content) : ""
|
|
6022
|
+
});
|
|
6023
|
+
break;
|
|
6024
|
+
}
|
|
6025
|
+
case "assistant.message": {
|
|
6026
|
+
const toolRequests = data.toolRequests;
|
|
6027
|
+
const toolCalls = (toolRequests ?? []).map(
|
|
6028
|
+
(req) => normalizeToolCall("copilot-events", {
|
|
6029
|
+
tool: String(req.name ?? req.toolName ?? ""),
|
|
6030
|
+
input: req.arguments,
|
|
6031
|
+
id: req.toolCallId ? String(req.toolCallId) : void 0
|
|
6032
|
+
})
|
|
6033
|
+
);
|
|
6034
|
+
messages.push({
|
|
6035
|
+
role: "assistant",
|
|
6036
|
+
content: data.content != null ? String(data.content) : void 0,
|
|
6037
|
+
toolCalls: toolCalls.length > 0 ? toolCalls : void 0
|
|
6038
|
+
});
|
|
6039
|
+
break;
|
|
6040
|
+
}
|
|
6041
|
+
case "skill.invoked": {
|
|
6042
|
+
const skillName = String(data.name ?? "");
|
|
6043
|
+
messages.push({
|
|
6044
|
+
role: "assistant",
|
|
6045
|
+
toolCalls: [
|
|
6046
|
+
{
|
|
6047
|
+
tool: "Skill",
|
|
6048
|
+
input: { skill: skillName }
|
|
6049
|
+
}
|
|
6050
|
+
]
|
|
6051
|
+
});
|
|
6052
|
+
break;
|
|
6053
|
+
}
|
|
6054
|
+
case "tool.execution_start": {
|
|
6055
|
+
const toolCallId = String(data.toolCallId ?? "");
|
|
6056
|
+
if (toolCallId) {
|
|
6057
|
+
toolCallsInProgress.set(toolCallId, {
|
|
6058
|
+
toolName: String(data.toolName ?? ""),
|
|
6059
|
+
input: data.arguments,
|
|
6060
|
+
toolCallId
|
|
6061
|
+
});
|
|
6062
|
+
}
|
|
6063
|
+
break;
|
|
6064
|
+
}
|
|
6065
|
+
case "tool.execution_complete": {
|
|
6066
|
+
const toolCallId = String(data.toolCallId ?? "");
|
|
6067
|
+
const started = toolCallsInProgress.get(toolCallId);
|
|
6068
|
+
if (started) {
|
|
6069
|
+
toolCallsInProgress.delete(toolCallId);
|
|
6070
|
+
messages.push({
|
|
6071
|
+
role: "assistant",
|
|
6072
|
+
toolCalls: [
|
|
6073
|
+
normalizeToolCall("copilot-events", {
|
|
6074
|
+
tool: started.toolName,
|
|
6075
|
+
input: started.input,
|
|
6076
|
+
output: data.result,
|
|
6077
|
+
id: toolCallId
|
|
6078
|
+
})
|
|
6079
|
+
]
|
|
6080
|
+
});
|
|
6081
|
+
}
|
|
6082
|
+
break;
|
|
6083
|
+
}
|
|
6084
|
+
case "session.shutdown": {
|
|
6085
|
+
endTimestamp = event.timestamp ? String(event.timestamp) : void 0;
|
|
6086
|
+
const modelMetrics = data.modelMetrics;
|
|
6087
|
+
if (modelMetrics) {
|
|
6088
|
+
for (const metrics of Object.values(modelMetrics)) {
|
|
6089
|
+
if (metrics.usage) {
|
|
6090
|
+
hasUsage = true;
|
|
6091
|
+
totalInputTokens += Number(metrics.usage.inputTokens ?? 0);
|
|
6092
|
+
totalOutputTokens += Number(metrics.usage.outputTokens ?? 0);
|
|
6093
|
+
}
|
|
6094
|
+
}
|
|
6095
|
+
}
|
|
6096
|
+
const currentModel = data.currentModel;
|
|
6097
|
+
if (currentModel && !meta.model) {
|
|
6098
|
+
meta.model = String(currentModel);
|
|
6099
|
+
}
|
|
6100
|
+
break;
|
|
6101
|
+
}
|
|
6102
|
+
}
|
|
6103
|
+
}
|
|
6104
|
+
let durationMs;
|
|
6105
|
+
if (startTimestamp && endTimestamp) {
|
|
6106
|
+
durationMs = new Date(endTimestamp).getTime() - new Date(startTimestamp).getTime();
|
|
6107
|
+
}
|
|
6108
|
+
return {
|
|
6109
|
+
messages,
|
|
6110
|
+
meta,
|
|
6111
|
+
tokenUsage: hasUsage ? { input: totalInputTokens, output: totalOutputTokens } : void 0,
|
|
6112
|
+
durationMs
|
|
6113
|
+
};
|
|
6114
|
+
}
|
|
6782
6115
|
function createAgentKernel() {
|
|
6783
6116
|
return { status: "stub" };
|
|
6784
6117
|
}
|
|
@@ -6788,12 +6121,6 @@ export {
|
|
|
6788
6121
|
traceEnvelopeToTraceSessionResponse,
|
|
6789
6122
|
buildTraceSpanTree,
|
|
6790
6123
|
normalizeTraceArtifactToTraceSessionResponse,
|
|
6791
|
-
isExperimentFileReference,
|
|
6792
|
-
deriveExperimentNameFromPath,
|
|
6793
|
-
loadExperimentConfig,
|
|
6794
|
-
normalizeExperimentConfig,
|
|
6795
|
-
fingerprintExperimentConfig,
|
|
6796
|
-
buildExperimentArtifactMetadata,
|
|
6797
6124
|
transpileEvalYaml,
|
|
6798
6125
|
transpileEvalYamlFile,
|
|
6799
6126
|
getOutputFilenames,
|
|
@@ -6827,15 +6154,14 @@ export {
|
|
|
6827
6154
|
setupWipWorktree,
|
|
6828
6155
|
pushWipCheckpoint,
|
|
6829
6156
|
deleteWipBranch,
|
|
6157
|
+
resolveGitResultsIndexCacheFile,
|
|
6158
|
+
resolveGitRunsRefCommit,
|
|
6159
|
+
listGitRunsCached,
|
|
6830
6160
|
listGitRuns,
|
|
6831
6161
|
materializeGitRun,
|
|
6832
6162
|
syncProject,
|
|
6833
6163
|
syncProjects,
|
|
6834
6164
|
trimBaselineResult,
|
|
6835
|
-
DEFAULT_CATEGORY,
|
|
6836
|
-
deriveCategory,
|
|
6837
|
-
OtelTraceExporter,
|
|
6838
|
-
OtelStreamingObserver,
|
|
6839
6165
|
RunBudgetTracker,
|
|
6840
6166
|
parseEnvOutput,
|
|
6841
6167
|
runBeforeSessionHook,
|
|
@@ -6844,6 +6170,7 @@ export {
|
|
|
6844
6170
|
discoverCodexSessions,
|
|
6845
6171
|
discoverClaudeSessions,
|
|
6846
6172
|
TranscriptProvider,
|
|
6173
|
+
parseCopilotEvents,
|
|
6847
6174
|
createAgentKernel
|
|
6848
6175
|
};
|
|
6849
|
-
//# sourceMappingURL=chunk-
|
|
6176
|
+
//# sourceMappingURL=chunk-GJWPQXHN.js.map
|