agentv 5.0.0-next.1 → 5.2.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +239 -53
- package/dist/{agentv-provider-AYXH7WLW-NJRC6UQX.js → agentv-provider-AYXH7WLW-6YZKZ7PS.js} +2 -2
- package/dist/{artifact-writer-QD52FC56.js → artifact-writer-SN6UNL5G.js} +15 -8
- package/dist/{chunk-GZJMODLW.js → chunk-3S6A2RKR.js} +45880 -30542
- package/dist/chunk-3S6A2RKR.js.map +1 -0
- package/dist/{chunk-6Q56CEXV.js → chunk-3X357HS4.js} +7900 -8985
- package/dist/chunk-3X357HS4.js.map +1 -0
- package/dist/{chunk-6JVM4GJI.js → chunk-46K2OET3.js} +187 -80
- package/dist/chunk-46K2OET3.js.map +1 -0
- package/dist/{chunk-IJEZFZJR.js → chunk-7BGERE6L.js} +2 -2
- package/dist/chunk-7BGERE6L.js.map +1 -0
- package/dist/{chunk-UG4JHZYM.js → chunk-ALVZQUZP.js} +670 -1427
- package/dist/chunk-ALVZQUZP.js.map +1 -0
- package/dist/{chunk-RCIEHFQ2.js → chunk-PEUTJS7B.js} +3 -3
- package/dist/{chunk-5H446C7X.js → chunk-QMRVH5ZP.js} +2 -8
- package/dist/{chunk-IBI7A5RP.js → chunk-T4AD3H3Y.js} +3444 -2251
- package/dist/chunk-T4AD3H3Y.js.map +1 -0
- package/dist/cli.js +8 -9
- package/dist/cli.js.map +1 -1
- package/dist/dashboard/assets/{index-D5_8DE_B.js → index-DNgf3qJ2.js} +1 -1
- package/dist/dashboard/assets/index-D_bokML8.css +1 -0
- package/dist/dashboard/assets/index-r_jSJmlw.js +121 -0
- package/dist/dashboard/index.html +2 -2
- package/dist/{dist-KJ6PELQY.js → dist-DEPJOMCA.js} +74 -60
- package/dist/{docker-workspace-ZJ2ESVBI-6UDETQII.js → docker-workspace-NNK4Z6FR-QRTIG6TM.js} +3 -3
- package/dist/{exec-MYPAJJP7-C5HD5WG5.js → exec-4HFFRQ3B-AXDMJNLJ.js} +3 -3
- package/dist/index.js +8 -9
- package/dist/{interactive-2WMVAASS.js → interactive-36ZNMQB2.js} +13 -29
- package/dist/interactive-36ZNMQB2.js.map +1 -0
- package/dist/skills/agentv-bench/SKILL.md +38 -34
- package/dist/skills/agentv-bench/agents/analyzer.md +6 -6
- package/dist/skills/agentv-bench/agents/comparator.md +4 -5
- package/dist/skills/agentv-bench/agents/grader.md +4 -4
- package/dist/skills/agentv-bench/references/autoresearch.md +3 -3
- package/dist/skills/agentv-bench/references/description-optimization.md +2 -2
- package/dist/skills/agentv-bench/references/environment-adaptation.md +10 -10
- package/dist/skills/agentv-bench/references/eval-yaml-spec.md +51 -30
- package/dist/skills/agentv-bench/references/migrating-from-skill-creator.md +4 -4
- package/dist/skills/agentv-bench/references/subagent-pipeline.md +13 -12
- package/dist/skills/agentv-eval-migrations/SKILL.md +33 -0
- package/dist/skills/agentv-eval-migrations/references/breaking-changes.md +1012 -0
- package/dist/skills/agentv-eval-review/SKILL.md +6 -3
- package/dist/skills/agentv-eval-review/scripts/lint_eval.py +1 -1
- package/dist/skills/agentv-eval-writer/SKILL.md +380 -246
- package/dist/skills/agentv-eval-writer/references/{config-schema.json → config.schema.json} +6 -5
- package/dist/skills/agentv-eval-writer/references/custom-evaluators.md +23 -18
- package/dist/skills/agentv-eval-writer/references/eval.schema.json +9852 -0
- package/dist/skills/agentv-eval-writer/references/python-helpers.md +8 -8
- package/dist/skills/agentv-eval-writer/references/rubric-evaluator.md +21 -22
- package/dist/skills/agentv-trace-analyst/SKILL.md +4 -4
- package/dist/templates/.agentv/targets.yaml +41 -40
- package/dist/{ts-eval-loader-RE43OAGX-AMRCUKBZ.js → ts-eval-loader-OMG2JBHP-OP5RR7A3.js} +5 -4
- package/package.json +19 -4
- package/dist/chunk-6JVM4GJI.js.map +0 -1
- package/dist/chunk-6Q56CEXV.js.map +0 -1
- package/dist/chunk-76FOHROU.js +0 -96
- package/dist/chunk-76FOHROU.js.map +0 -1
- package/dist/chunk-GZJMODLW.js.map +0 -1
- package/dist/chunk-HSHTTN6C.js +0 -565
- package/dist/chunk-HSHTTN6C.js.map +0 -1
- package/dist/chunk-IBI7A5RP.js.map +0 -1
- package/dist/chunk-IJEZFZJR.js.map +0 -1
- package/dist/chunk-LRULMAAA.js +0 -1711
- package/dist/chunk-LRULMAAA.js.map +0 -1
- package/dist/chunk-R3CXYNTN.js +0 -1238
- package/dist/chunk-R3CXYNTN.js.map +0 -1
- package/dist/chunk-UG4JHZYM.js.map +0 -1
- package/dist/chunk-VQ2ZO7XJ.js +0 -2098
- package/dist/chunk-VQ2ZO7XJ.js.map +0 -1
- package/dist/chunk-XALGXSKB.js +0 -21
- package/dist/chunk-XALGXSKB.js.map +0 -1
- package/dist/dashboard/assets/index-BVMrdN8a.js +0 -119
- package/dist/dashboard/assets/index-DZWXfWsv.css +0 -1
- package/dist/esm-NNMQYIKT.js +0 -32
- package/dist/esm-R77SNOF5.js +0 -65
- package/dist/esm-RVQPUGWH.js +0 -1207
- package/dist/esm-RVQPUGWH.js.map +0 -1
- package/dist/esm-TTEMJEVV.js +0 -933
- package/dist/esm-TTEMJEVV.js.map +0 -1
- package/dist/esm-ZADQ4XQH-QUFGGZL4.js +0 -924
- package/dist/esm-ZADQ4XQH-QUFGGZL4.js.map +0 -1
- package/dist/exec-MYPAJJP7-C5HD5WG5.js.map +0 -1
- package/dist/getMachineId-bsd-HSK5LZMG.js +0 -41
- package/dist/getMachineId-bsd-HSK5LZMG.js.map +0 -1
- package/dist/getMachineId-darwin-4DP6CCJV.js +0 -41
- package/dist/getMachineId-darwin-4DP6CCJV.js.map +0 -1
- package/dist/getMachineId-linux-44LJ5UJB.js +0 -33
- package/dist/getMachineId-linux-44LJ5UJB.js.map +0 -1
- package/dist/getMachineId-unsupported-NVK6IATM.js +0 -24
- package/dist/getMachineId-unsupported-NVK6IATM.js.map +0 -1
- package/dist/getMachineId-win-YZ36S7VA.js +0 -43
- package/dist/getMachineId-win-YZ36S7VA.js.map +0 -1
- package/dist/interactive-2WMVAASS.js.map +0 -1
- package/dist/otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js +0 -9
- package/dist/otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js.map +0 -1
- package/dist/skills/agentv-eval-writer/references/eval-schema.json +0 -17295
- package/dist/skills/agentv-eval-writer/references/experiment-schema.json +0 -278
- package/dist/src-76ISS3LH.js +0 -1733
- package/dist/src-76ISS3LH.js.map +0 -1
- package/dist/ts-eval-loader-RE43OAGX-AMRCUKBZ.js.map +0 -1
- /package/dist/{agentv-provider-AYXH7WLW-NJRC6UQX.js.map → agentv-provider-AYXH7WLW-6YZKZ7PS.js.map} +0 -0
- /package/dist/{artifact-writer-QD52FC56.js.map → artifact-writer-SN6UNL5G.js.map} +0 -0
- /package/dist/{chunk-RCIEHFQ2.js.map → chunk-PEUTJS7B.js.map} +0 -0
- /package/dist/{chunk-5H446C7X.js.map → chunk-QMRVH5ZP.js.map} +0 -0
- /package/dist/{dist-KJ6PELQY.js.map → dist-DEPJOMCA.js.map} +0 -0
- /package/dist/{docker-workspace-ZJ2ESVBI-6UDETQII.js.map → docker-workspace-NNK4Z6FR-QRTIG6TM.js.map} +0 -0
- /package/dist/{esm-NNMQYIKT.js.map → exec-4HFFRQ3B-AXDMJNLJ.js.map} +0 -0
- /package/dist/{esm-R77SNOF5.js.map → ts-eval-loader-OMG2JBHP-OP5RR7A3.js.map} +0 -0
|
@@ -8,57 +8,47 @@ import {
|
|
|
8
8
|
external_exports,
|
|
9
9
|
extractLastAssistantContent,
|
|
10
10
|
getAgentvDataDir,
|
|
11
|
-
getRepoCheckoutRef,
|
|
12
11
|
groupTranscriptJsonLines,
|
|
13
|
-
interpolateEnv,
|
|
14
12
|
loadTests,
|
|
15
|
-
normalizeRepoIdentity,
|
|
16
13
|
normalizeToolCall,
|
|
17
|
-
parseRepoConfig,
|
|
18
14
|
parseYamlValue,
|
|
19
15
|
prepareEvalCaseWorkspace,
|
|
20
16
|
prepareSharedWorkspaceSetup,
|
|
21
17
|
readTranscriptJsonl,
|
|
22
|
-
releaseSharedWorkspaceSetup
|
|
23
|
-
|
|
24
|
-
} from "./chunk-GZJMODLW.js";
|
|
18
|
+
releaseSharedWorkspaceSetup
|
|
19
|
+
} from "./chunk-3S6A2RKR.js";
|
|
25
20
|
|
|
26
21
|
// ../../packages/core/dist/index.js
|
|
27
|
-
import { createHash } from "node:crypto";
|
|
28
|
-
import { readFile } from "node:fs/promises";
|
|
29
|
-
import path from "node:path";
|
|
30
|
-
import { pathToFileURL } from "node:url";
|
|
31
22
|
import { readFileSync } from "node:fs";
|
|
32
|
-
import
|
|
23
|
+
import path from "node:path";
|
|
33
24
|
import { randomUUID } from "node:crypto";
|
|
34
|
-
import
|
|
25
|
+
import path2 from "node:path";
|
|
35
26
|
import micromatch from "micromatch";
|
|
36
|
-
import { readFile as readFile2 } from "node:fs/promises";
|
|
37
|
-
import path4 from "node:path";
|
|
38
27
|
import { execFile, spawn } from "node:child_process";
|
|
39
|
-
import { createHash
|
|
28
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
40
29
|
import {
|
|
41
30
|
existsSync,
|
|
42
31
|
mkdirSync,
|
|
43
32
|
mkdtempSync,
|
|
44
33
|
readFileSync as readFileSync2,
|
|
34
|
+
realpathSync,
|
|
45
35
|
renameSync,
|
|
46
36
|
rmSync,
|
|
47
37
|
writeFileSync
|
|
48
38
|
} from "node:fs";
|
|
49
|
-
import { cp, lstat, mkdtemp, readdir, rm, stat } from "node:fs/promises";
|
|
39
|
+
import { cp, lstat, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
50
40
|
import os from "node:os";
|
|
51
|
-
import
|
|
41
|
+
import path3 from "node:path";
|
|
52
42
|
import { promisify } from "node:util";
|
|
53
43
|
import * as childProcess from "node:child_process";
|
|
54
44
|
import { existsSync as existsSync2 } from "node:fs";
|
|
55
45
|
import { spawnSync } from "node:child_process";
|
|
56
46
|
import { readdir as readdir2, stat as stat2 } from "node:fs/promises";
|
|
57
47
|
import { homedir } from "node:os";
|
|
58
|
-
import
|
|
48
|
+
import path4 from "node:path";
|
|
59
49
|
import { readdir as readdir3, stat as stat3 } from "node:fs/promises";
|
|
60
50
|
import { homedir as homedir2 } from "node:os";
|
|
61
|
-
import
|
|
51
|
+
import path5 from "node:path";
|
|
62
52
|
var TRACE_SESSION_SCHEMA_VERSION = "agentv.dashboard.trace_session.v1";
|
|
63
53
|
function isRecord(value) {
|
|
64
54
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -1482,430 +1472,7 @@ function isPrimitive(value) {
|
|
|
1482
1472
|
function dropUndefined2(value) {
|
|
1483
1473
|
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
|
|
1484
1474
|
}
|
|
1485
|
-
|
|
1486
|
-
var VALID_SANDBOXES = /* @__PURE__ */ new Set(["auto", "docker", "vercel"]);
|
|
1487
|
-
var VALID_REPEAT_STRATEGIES = /* @__PURE__ */ new Set([
|
|
1488
|
-
"pass_at_k",
|
|
1489
|
-
"mean",
|
|
1490
|
-
"confidence_interval"
|
|
1491
|
-
]);
|
|
1492
|
-
function isExperimentFileReference(value) {
|
|
1493
|
-
const trimmed = value.trim();
|
|
1494
|
-
if (!trimmed) {
|
|
1495
|
-
return false;
|
|
1496
|
-
}
|
|
1497
|
-
return trimmed.includes("/") || trimmed.includes("\\") || EXPERIMENT_FILE_EXTENSIONS.has(path.extname(trimmed).toLowerCase());
|
|
1498
|
-
}
|
|
1499
|
-
function deriveExperimentNameFromPath(filePath) {
|
|
1500
|
-
return path.basename(filePath).replace(/\.experiment\.(ya?ml|[cm]?[jt]s)$/i, "").replace(/\.(ya?ml|[cm]?[jt]s)$/i, "");
|
|
1501
|
-
}
|
|
1502
|
-
async function loadExperimentConfig(filePath) {
|
|
1503
|
-
const resolvedPath = path.resolve(filePath);
|
|
1504
|
-
const ext = path.extname(resolvedPath).toLowerCase();
|
|
1505
|
-
let rawConfig;
|
|
1506
|
-
if (ext === ".yaml" || ext === ".yml") {
|
|
1507
|
-
rawConfig = parseYamlValue(await readFile(resolvedPath, "utf8"));
|
|
1508
|
-
} else if (EXPERIMENT_FILE_EXTENSIONS.has(ext)) {
|
|
1509
|
-
const moduleUrl = pathToFileURL(resolvedPath).href;
|
|
1510
|
-
const mod = await import(moduleUrl);
|
|
1511
|
-
rawConfig = mod.default ?? mod.config ?? mod;
|
|
1512
|
-
} else {
|
|
1513
|
-
throw new Error(
|
|
1514
|
-
`Unsupported experiment file extension '${ext}'. Use .yaml, .yml, .ts, .js, .mts, or .mjs.`
|
|
1515
|
-
);
|
|
1516
|
-
}
|
|
1517
|
-
return normalizeExperimentConfig(rawConfig, { sourcePath: resolvedPath });
|
|
1518
|
-
}
|
|
1519
|
-
function normalizeExperimentConfig(rawConfig, options = {}) {
|
|
1520
|
-
if (!isRecord2(rawConfig)) {
|
|
1521
|
-
throw new Error("Experiment config must be an object.");
|
|
1522
|
-
}
|
|
1523
|
-
const name = readOptionalString(rawConfig.name, "name");
|
|
1524
|
-
const agent = readOptionalString(rawConfig.agent, "agent");
|
|
1525
|
-
const target = readOptionalString(rawConfig.target, "target");
|
|
1526
|
-
const targets = readTargets(rawConfig.targets);
|
|
1527
|
-
const model = readOptionalString(rawConfig.model, "model");
|
|
1528
|
-
const agentOptions = readOptionalRecord(rawConfig.agent_options ?? rawConfig.agentOptions);
|
|
1529
|
-
const suites = readSuites(rawConfig.suites);
|
|
1530
|
-
const scripts = readScriptArray(rawConfig.scripts, "scripts");
|
|
1531
|
-
const repeat = readRepeat(rawConfig.repeat);
|
|
1532
|
-
const runs = readOptionalPositiveInteger(rawConfig.runs, "runs");
|
|
1533
|
-
if (repeat !== void 0 && runs !== void 0) {
|
|
1534
|
-
throw new Error("Experiment repeat and runs cannot both be set. Use repeat for AgentV config.");
|
|
1535
|
-
}
|
|
1536
|
-
const earlyExit = readOptionalBoolean(rawConfig.early_exit ?? rawConfig.earlyExit, "early_exit");
|
|
1537
|
-
const timeoutSeconds = readOptionalPositiveNumber(
|
|
1538
|
-
rawConfig.timeout_seconds ?? rawConfig.timeoutSeconds,
|
|
1539
|
-
"timeout_seconds"
|
|
1540
|
-
);
|
|
1541
|
-
const workers = readOptionalPositiveInteger(rawConfig.workers, "workers");
|
|
1542
|
-
const budgetUsd = readOptionalPositiveNumber(
|
|
1543
|
-
rawConfig.budget_usd ?? rawConfig.budgetUsd,
|
|
1544
|
-
"budget_usd"
|
|
1545
|
-
);
|
|
1546
|
-
const sandbox = readOptionalSandbox(rawConfig.sandbox);
|
|
1547
|
-
const workspace = readOptionalRecord(rawConfig.workspace);
|
|
1548
|
-
const setup = readSetup(rawConfig.setup);
|
|
1549
|
-
const configWithoutFingerprint = {
|
|
1550
|
-
...name !== void 0 && { name },
|
|
1551
|
-
...agent !== void 0 && { agent },
|
|
1552
|
-
...target !== void 0 && { target },
|
|
1553
|
-
...targets !== void 0 && { targets },
|
|
1554
|
-
...model !== void 0 && { model },
|
|
1555
|
-
...agentOptions !== void 0 && { agentOptions },
|
|
1556
|
-
...suites !== void 0 && { suites },
|
|
1557
|
-
...scripts !== void 0 && { scripts },
|
|
1558
|
-
...repeat !== void 0 && { repeat },
|
|
1559
|
-
...runs !== void 0 && { runs },
|
|
1560
|
-
...earlyExit !== void 0 && { earlyExit },
|
|
1561
|
-
...timeoutSeconds !== void 0 && { timeoutSeconds },
|
|
1562
|
-
...workers !== void 0 && { workers },
|
|
1563
|
-
...budgetUsd !== void 0 && { budgetUsd },
|
|
1564
|
-
...sandbox !== void 0 && { sandbox },
|
|
1565
|
-
...workspace !== void 0 && { workspace },
|
|
1566
|
-
...setup !== void 0 && { setup },
|
|
1567
|
-
...options.sourcePath !== void 0 && { sourcePath: options.sourcePath }
|
|
1568
|
-
};
|
|
1569
|
-
return {
|
|
1570
|
-
...configWithoutFingerprint,
|
|
1571
|
-
fingerprint: fingerprintExperimentConfig(configWithoutFingerprint)
|
|
1572
|
-
};
|
|
1573
|
-
}
|
|
1574
|
-
function fingerprintExperimentConfig(config) {
|
|
1575
|
-
const stablePayload = toStableJsonValue(config);
|
|
1576
|
-
return createHash("sha256").update(JSON.stringify(stablePayload)).digest("hex");
|
|
1577
|
-
}
|
|
1578
|
-
function buildExperimentArtifactMetadata(config) {
|
|
1579
|
-
if (!config) {
|
|
1580
|
-
return void 0;
|
|
1581
|
-
}
|
|
1582
|
-
const targets = config.targets?.map((target) => typeof target === "string" ? target : target.name).filter((target) => target.trim().length > 0);
|
|
1583
|
-
return {
|
|
1584
|
-
...config.name !== void 0 && { name: config.name },
|
|
1585
|
-
...config.sourcePath !== void 0 && { source_path: config.sourcePath },
|
|
1586
|
-
...config.fingerprint !== void 0 && { fingerprint: config.fingerprint },
|
|
1587
|
-
...config.agent !== void 0 && { agent: config.agent },
|
|
1588
|
-
...config.target !== void 0 && { target: config.target },
|
|
1589
|
-
...targets && targets.length > 0 && { targets },
|
|
1590
|
-
...config.model !== void 0 && { model: config.model },
|
|
1591
|
-
...config.suites !== void 0 && { suites: config.suites.map(toSuiteArtifactMetadata) },
|
|
1592
|
-
...config.repeat !== void 0 && {
|
|
1593
|
-
repeat: {
|
|
1594
|
-
count: config.repeat.count,
|
|
1595
|
-
strategy: config.repeat.strategy,
|
|
1596
|
-
...config.repeat.costLimitUsd !== void 0 && {
|
|
1597
|
-
cost_limit_usd: config.repeat.costLimitUsd
|
|
1598
|
-
}
|
|
1599
|
-
}
|
|
1600
|
-
},
|
|
1601
|
-
...config.runs !== void 0 && { runs: config.runs },
|
|
1602
|
-
...config.earlyExit !== void 0 && { early_exit: config.earlyExit },
|
|
1603
|
-
...config.timeoutSeconds !== void 0 && { timeout_seconds: config.timeoutSeconds },
|
|
1604
|
-
...config.workers !== void 0 && { workers: config.workers },
|
|
1605
|
-
...config.budgetUsd !== void 0 && { budget_usd: config.budgetUsd },
|
|
1606
|
-
...config.sandbox !== void 0 && { sandbox: config.sandbox }
|
|
1607
|
-
};
|
|
1608
|
-
}
|
|
1609
|
-
function toSuiteArtifactMetadata(suite) {
|
|
1610
|
-
return {
|
|
1611
|
-
ref: suite.ref,
|
|
1612
|
-
...suite.select !== void 0 && {
|
|
1613
|
-
select: {
|
|
1614
|
-
test_ids: suite.select.testIds
|
|
1615
|
-
}
|
|
1616
|
-
}
|
|
1617
|
-
};
|
|
1618
|
-
}
|
|
1619
|
-
function readRepeat(raw) {
|
|
1620
|
-
if (raw === void 0) {
|
|
1621
|
-
return void 0;
|
|
1622
|
-
}
|
|
1623
|
-
if (!isRecord2(raw)) {
|
|
1624
|
-
throw new Error("Experiment repeat must be an object.");
|
|
1625
|
-
}
|
|
1626
|
-
const count = readRequiredPositiveInteger(raw.count, "repeat.count");
|
|
1627
|
-
const strategy = readOptionalRepeatStrategy(raw.strategy);
|
|
1628
|
-
const costLimitUsd = readOptionalNonNegativeNumber(
|
|
1629
|
-
raw.cost_limit_usd ?? raw.costLimitUsd,
|
|
1630
|
-
"repeat.cost_limit_usd"
|
|
1631
|
-
);
|
|
1632
|
-
return {
|
|
1633
|
-
count,
|
|
1634
|
-
strategy: strategy ?? "pass_at_k",
|
|
1635
|
-
...costLimitUsd !== void 0 && { costLimitUsd }
|
|
1636
|
-
};
|
|
1637
|
-
}
|
|
1638
|
-
function readTargets(raw) {
|
|
1639
|
-
if (raw === void 0) {
|
|
1640
|
-
return void 0;
|
|
1641
|
-
}
|
|
1642
|
-
if (!Array.isArray(raw)) {
|
|
1643
|
-
throw new Error("Experiment targets must be an array.");
|
|
1644
|
-
}
|
|
1645
|
-
return raw.map((entry, index) => {
|
|
1646
|
-
if (typeof entry === "string" && entry.trim().length > 0) {
|
|
1647
|
-
return entry.trim();
|
|
1648
|
-
}
|
|
1649
|
-
if (!isRecord2(entry)) {
|
|
1650
|
-
throw new Error(`Experiment targets[${index}] must be a string or object.`);
|
|
1651
|
-
}
|
|
1652
|
-
const name = readRequiredString(entry.name, `targets[${index}].name`);
|
|
1653
|
-
const useTarget = readOptionalString(
|
|
1654
|
-
entry.use_target ?? entry.useTarget,
|
|
1655
|
-
`targets[${index}].use_target`
|
|
1656
|
-
);
|
|
1657
|
-
const hooks = readOptionalRecord(entry.hooks);
|
|
1658
|
-
return {
|
|
1659
|
-
name,
|
|
1660
|
-
...useTarget !== void 0 && { useTarget },
|
|
1661
|
-
...hooks !== void 0 && { hooks }
|
|
1662
|
-
};
|
|
1663
|
-
});
|
|
1664
|
-
}
|
|
1665
|
-
function readSuites(raw) {
|
|
1666
|
-
if (raw === void 0) {
|
|
1667
|
-
return void 0;
|
|
1668
|
-
}
|
|
1669
|
-
if (!Array.isArray(raw)) {
|
|
1670
|
-
throw new Error("Experiment suites must be an array.");
|
|
1671
|
-
}
|
|
1672
|
-
if (raw.length === 0) {
|
|
1673
|
-
throw new Error("Experiment suites must not be empty.");
|
|
1674
|
-
}
|
|
1675
|
-
return raw.map((entry, index) => {
|
|
1676
|
-
if (!isRecord2(entry)) {
|
|
1677
|
-
throw new Error(`Experiment suites[${index}] must be an object.`);
|
|
1678
|
-
}
|
|
1679
|
-
const ref = readRequiredString(entry.ref, `suites[${index}].ref`);
|
|
1680
|
-
const select = readSuiteSelect(entry.select, `suites[${index}].select`);
|
|
1681
|
-
return {
|
|
1682
|
-
ref,
|
|
1683
|
-
...select !== void 0 && { select }
|
|
1684
|
-
};
|
|
1685
|
-
});
|
|
1686
|
-
}
|
|
1687
|
-
function readSuiteSelect(raw, location) {
|
|
1688
|
-
if (raw === void 0) {
|
|
1689
|
-
return void 0;
|
|
1690
|
-
}
|
|
1691
|
-
if (!isRecord2(raw)) {
|
|
1692
|
-
throw new Error(`Experiment ${location} must be an object.`);
|
|
1693
|
-
}
|
|
1694
|
-
const testIds = readOptionalStringArray(raw.test_ids ?? raw.testIds, `${location}.test_ids`);
|
|
1695
|
-
if (testIds === void 0) {
|
|
1696
|
-
throw new Error(`Experiment ${location}.test_ids is required when select is set.`);
|
|
1697
|
-
}
|
|
1698
|
-
return { testIds };
|
|
1699
|
-
}
|
|
1700
|
-
function readScriptArray(raw, location) {
|
|
1701
|
-
if (raw === void 0) {
|
|
1702
|
-
return void 0;
|
|
1703
|
-
}
|
|
1704
|
-
if (!Array.isArray(raw)) {
|
|
1705
|
-
throw new Error(`Experiment ${location} must be an array.`);
|
|
1706
|
-
}
|
|
1707
|
-
return raw.map((entry, index) => readScript(entry, `${location}[${index}]`));
|
|
1708
|
-
}
|
|
1709
|
-
function readSetup(raw) {
|
|
1710
|
-
if (raw === void 0) {
|
|
1711
|
-
return void 0;
|
|
1712
|
-
}
|
|
1713
|
-
if (typeof raw === "function") {
|
|
1714
|
-
return raw;
|
|
1715
|
-
}
|
|
1716
|
-
return readScriptArray(raw, "setup");
|
|
1717
|
-
}
|
|
1718
|
-
function readScript(raw, location) {
|
|
1719
|
-
if (typeof raw === "string") {
|
|
1720
|
-
const script2 = raw.trim();
|
|
1721
|
-
if (!script2) {
|
|
1722
|
-
throw new Error(`Experiment ${location} must not be empty.`);
|
|
1723
|
-
}
|
|
1724
|
-
return { script: script2 };
|
|
1725
|
-
}
|
|
1726
|
-
if (!isRecord2(raw)) {
|
|
1727
|
-
throw new Error(`Experiment ${location} must be a string or object.`);
|
|
1728
|
-
}
|
|
1729
|
-
const command = readOptionalCommand(raw.command, `${location}.command`);
|
|
1730
|
-
const script = readOptionalStringOrStringArray(raw.script, `${location}.script`);
|
|
1731
|
-
if (command === void 0 && script === void 0) {
|
|
1732
|
-
throw new Error(`Experiment ${location} must define command or script.`);
|
|
1733
|
-
}
|
|
1734
|
-
const timeoutSeconds = readOptionalPositiveNumber(
|
|
1735
|
-
raw.timeout_seconds ?? raw.timeoutSeconds,
|
|
1736
|
-
`${location}.timeout_seconds`
|
|
1737
|
-
);
|
|
1738
|
-
const cwd = readOptionalString(raw.cwd, `${location}.cwd`);
|
|
1739
|
-
const env = readOptionalStringRecord(raw.env, `${location}.env`);
|
|
1740
|
-
return {
|
|
1741
|
-
...command !== void 0 && { command },
|
|
1742
|
-
...script !== void 0 && { script },
|
|
1743
|
-
...timeoutSeconds !== void 0 && { timeoutSeconds },
|
|
1744
|
-
...cwd !== void 0 && { cwd },
|
|
1745
|
-
...env !== void 0 && { env }
|
|
1746
|
-
};
|
|
1747
|
-
}
|
|
1748
|
-
function readOptionalCommand(raw, location) {
|
|
1749
|
-
if (raw === void 0) {
|
|
1750
|
-
return void 0;
|
|
1751
|
-
}
|
|
1752
|
-
if (typeof raw === "string") {
|
|
1753
|
-
const command = raw.trim();
|
|
1754
|
-
if (!command) {
|
|
1755
|
-
throw new Error(`Experiment ${location} must not be empty.`);
|
|
1756
|
-
}
|
|
1757
|
-
return ["sh", "-c", command];
|
|
1758
|
-
}
|
|
1759
|
-
if (Array.isArray(raw) && raw.length > 0 && raw.every((entry) => typeof entry === "string" && entry.trim())) {
|
|
1760
|
-
return raw.map((entry) => entry.trim());
|
|
1761
|
-
}
|
|
1762
|
-
throw new Error(`Experiment ${location} must be a string or string array.`);
|
|
1763
|
-
}
|
|
1764
|
-
function readOptionalStringOrStringArray(raw, location) {
|
|
1765
|
-
if (raw === void 0) {
|
|
1766
|
-
return void 0;
|
|
1767
|
-
}
|
|
1768
|
-
if (typeof raw === "string") {
|
|
1769
|
-
const trimmed = raw.trim();
|
|
1770
|
-
if (!trimmed) {
|
|
1771
|
-
throw new Error(`Experiment ${location} must not be empty.`);
|
|
1772
|
-
}
|
|
1773
|
-
return trimmed;
|
|
1774
|
-
}
|
|
1775
|
-
if (Array.isArray(raw) && raw.length > 0 && raw.every((entry) => typeof entry === "string" && entry.trim())) {
|
|
1776
|
-
return raw.map((entry) => entry.trim());
|
|
1777
|
-
}
|
|
1778
|
-
throw new Error(`Experiment ${location} must be a string or string array.`);
|
|
1779
|
-
}
|
|
1780
|
-
function readOptionalStringArray(raw, location) {
|
|
1781
|
-
if (raw === void 0) {
|
|
1782
|
-
return void 0;
|
|
1783
|
-
}
|
|
1784
|
-
if (Array.isArray(raw) && raw.length > 0 && raw.every((entry) => typeof entry === "string" && entry.trim())) {
|
|
1785
|
-
return raw.map((entry) => entry.trim());
|
|
1786
|
-
}
|
|
1787
|
-
throw new Error(`Experiment ${location} must be a non-empty string array.`);
|
|
1788
|
-
}
|
|
1789
|
-
function readOptionalString(raw, location) {
|
|
1790
|
-
if (raw === void 0) {
|
|
1791
|
-
return void 0;
|
|
1792
|
-
}
|
|
1793
|
-
if (typeof raw !== "string" || raw.trim().length === 0) {
|
|
1794
|
-
throw new Error(`Experiment ${location} must be a non-empty string.`);
|
|
1795
|
-
}
|
|
1796
|
-
return raw.trim();
|
|
1797
|
-
}
|
|
1798
|
-
function readRequiredString(raw, location) {
|
|
1799
|
-
const value = readOptionalString(raw, location);
|
|
1800
|
-
if (value === void 0) {
|
|
1801
|
-
throw new Error(`Experiment ${location} is required.`);
|
|
1802
|
-
}
|
|
1803
|
-
return value;
|
|
1804
|
-
}
|
|
1805
|
-
function readRequiredPositiveInteger(raw, location) {
|
|
1806
|
-
const value = readOptionalPositiveInteger(raw, location);
|
|
1807
|
-
if (value === void 0) {
|
|
1808
|
-
throw new Error(`Experiment ${location} is required.`);
|
|
1809
|
-
}
|
|
1810
|
-
return value;
|
|
1811
|
-
}
|
|
1812
|
-
function readOptionalBoolean(raw, location) {
|
|
1813
|
-
if (raw === void 0) {
|
|
1814
|
-
return void 0;
|
|
1815
|
-
}
|
|
1816
|
-
if (typeof raw !== "boolean") {
|
|
1817
|
-
throw new Error(`Experiment ${location} must be a boolean.`);
|
|
1818
|
-
}
|
|
1819
|
-
return raw;
|
|
1820
|
-
}
|
|
1821
|
-
function readOptionalRepeatStrategy(raw) {
|
|
1822
|
-
if (raw === void 0) {
|
|
1823
|
-
return void 0;
|
|
1824
|
-
}
|
|
1825
|
-
if (typeof raw !== "string" || !VALID_REPEAT_STRATEGIES.has(raw)) {
|
|
1826
|
-
throw new Error(
|
|
1827
|
-
"Experiment repeat.strategy must be one of 'pass_at_k', 'mean', or 'confidence_interval'."
|
|
1828
|
-
);
|
|
1829
|
-
}
|
|
1830
|
-
return raw;
|
|
1831
|
-
}
|
|
1832
|
-
function readOptionalPositiveInteger(raw, location) {
|
|
1833
|
-
if (raw === void 0) {
|
|
1834
|
-
return void 0;
|
|
1835
|
-
}
|
|
1836
|
-
if (typeof raw !== "number" || !Number.isInteger(raw) || raw < 1) {
|
|
1837
|
-
throw new Error(`Experiment ${location} must be a positive integer.`);
|
|
1838
|
-
}
|
|
1839
|
-
return raw;
|
|
1840
|
-
}
|
|
1841
|
-
function readOptionalNonNegativeNumber(raw, location) {
|
|
1842
|
-
if (raw === void 0) {
|
|
1843
|
-
return void 0;
|
|
1844
|
-
}
|
|
1845
|
-
if (typeof raw !== "number" || !Number.isFinite(raw) || raw < 0) {
|
|
1846
|
-
throw new Error(`Experiment ${location} must be a non-negative number.`);
|
|
1847
|
-
}
|
|
1848
|
-
return raw;
|
|
1849
|
-
}
|
|
1850
|
-
function readOptionalPositiveNumber(raw, location) {
|
|
1851
|
-
if (raw === void 0) {
|
|
1852
|
-
return void 0;
|
|
1853
|
-
}
|
|
1854
|
-
if (typeof raw !== "number" || !Number.isFinite(raw) || raw <= 0) {
|
|
1855
|
-
throw new Error(`Experiment ${location} must be a positive number.`);
|
|
1856
|
-
}
|
|
1857
|
-
return raw;
|
|
1858
|
-
}
|
|
1859
|
-
function readOptionalSandbox(raw) {
|
|
1860
|
-
if (raw === void 0) {
|
|
1861
|
-
return void 0;
|
|
1862
|
-
}
|
|
1863
|
-
if (typeof raw !== "string" || !VALID_SANDBOXES.has(raw)) {
|
|
1864
|
-
throw new Error("Experiment sandbox must be one of 'auto', 'docker', or 'vercel'.");
|
|
1865
|
-
}
|
|
1866
|
-
return raw;
|
|
1867
|
-
}
|
|
1868
|
-
function readOptionalRecord(raw) {
|
|
1869
|
-
if (raw === void 0) {
|
|
1870
|
-
return void 0;
|
|
1871
|
-
}
|
|
1872
|
-
if (!isRecord2(raw)) {
|
|
1873
|
-
throw new Error("Experiment object field must be an object.");
|
|
1874
|
-
}
|
|
1875
|
-
return raw;
|
|
1876
|
-
}
|
|
1877
|
-
function readOptionalStringRecord(raw, location) {
|
|
1878
|
-
if (raw === void 0) {
|
|
1879
|
-
return void 0;
|
|
1880
|
-
}
|
|
1881
|
-
if (!isRecord2(raw)) {
|
|
1882
|
-
throw new Error(`Experiment ${location} must be an object.`);
|
|
1883
|
-
}
|
|
1884
|
-
const entries = Object.entries(raw);
|
|
1885
|
-
if (!entries.every((entry) => typeof entry[1] === "string")) {
|
|
1886
|
-
throw new Error(`Experiment ${location} values must be strings.`);
|
|
1887
|
-
}
|
|
1888
|
-
return Object.fromEntries(entries);
|
|
1889
|
-
}
|
|
1890
|
-
function isRecord2(value) {
|
|
1891
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1892
|
-
}
|
|
1893
|
-
function toStableJsonValue(value) {
|
|
1894
|
-
if (typeof value === "function") {
|
|
1895
|
-
return "[function]";
|
|
1896
|
-
}
|
|
1897
|
-
if (value === null || typeof value !== "object") {
|
|
1898
|
-
return value;
|
|
1899
|
-
}
|
|
1900
|
-
if (Array.isArray(value)) {
|
|
1901
|
-
return value.map(toStableJsonValue);
|
|
1902
|
-
}
|
|
1903
|
-
const record = value;
|
|
1904
|
-
return Object.fromEntries(
|
|
1905
|
-
Object.keys(record).sort().map((key) => [key, toStableJsonValue(record[key])])
|
|
1906
|
-
);
|
|
1907
|
-
}
|
|
1908
|
-
function codeGraderInstruction(graderName, description) {
|
|
1475
|
+
function scriptGraderInstruction(graderName, description) {
|
|
1909
1476
|
const desc = description ? ` This grader: ${description}.` : "";
|
|
1910
1477
|
return `Run \`agentv eval assert ${graderName} --agent-output <agent_output> --agent-input <original_prompt>\` and check the result.${desc} The command accepts --agent-output (the agent's full response text) and --agent-input (the original user prompt). It returns JSON on stdout: {"score": 0-1, "reasoning": "..."}. A score >= 0.5 means pass (exit 0); below 0.5 means fail (exit 1).`;
|
|
1911
1478
|
}
|
|
@@ -1923,21 +1490,15 @@ function assertionToNaturalLanguage(entry) {
|
|
|
1923
1490
|
switch (type) {
|
|
1924
1491
|
case "skill-trigger":
|
|
1925
1492
|
return null;
|
|
1926
|
-
case "
|
|
1927
|
-
|
|
1928
|
-
return entry.criteria;
|
|
1929
|
-
}
|
|
1930
|
-
return null;
|
|
1931
|
-
}
|
|
1493
|
+
case "llm-rubric":
|
|
1494
|
+
return typeof entry.value === "string" ? entry.value : null;
|
|
1932
1495
|
case "contains":
|
|
1933
1496
|
return `Output contains '${entry.value}'`;
|
|
1934
|
-
case "contains-any":
|
|
1935
|
-
case "contains_any": {
|
|
1497
|
+
case "contains-any": {
|
|
1936
1498
|
const values = Array.isArray(entry.value) ? entry.value.join("', '") : entry.value;
|
|
1937
1499
|
return `Output contains any of: '${values}'`;
|
|
1938
1500
|
}
|
|
1939
|
-
case "contains-all":
|
|
1940
|
-
case "contains_all": {
|
|
1501
|
+
case "contains-all": {
|
|
1941
1502
|
const values = Array.isArray(entry.value) ? entry.value.join("', '") : entry.value;
|
|
1942
1503
|
return `Output contains all of: '${values}'`;
|
|
1943
1504
|
}
|
|
@@ -1948,35 +1509,27 @@ function assertionToNaturalLanguage(entry) {
|
|
|
1948
1509
|
case "equals":
|
|
1949
1510
|
return `Output exactly equals: ${entry.value}`;
|
|
1950
1511
|
case "is-json":
|
|
1951
|
-
case "is_json":
|
|
1952
1512
|
return "Output is valid JSON";
|
|
1953
1513
|
case "starts-with":
|
|
1954
|
-
case "starts_with":
|
|
1955
1514
|
return `Output starts with '${entry.value}'`;
|
|
1956
1515
|
case "ends-with":
|
|
1957
|
-
case "ends_with":
|
|
1958
1516
|
return `Output ends with '${entry.value}'`;
|
|
1959
1517
|
case "llm-grader":
|
|
1960
|
-
case "llm_grader": {
|
|
1961
1518
|
if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
|
|
1962
1519
|
return null;
|
|
1963
1520
|
}
|
|
1964
1521
|
return typeof entry.prompt === "string" ? entry.prompt : null;
|
|
1965
|
-
|
|
1966
|
-
case "tool-trajectory":
|
|
1967
|
-
case "tool_trajectory": {
|
|
1522
|
+
case "tool-trajectory": {
|
|
1968
1523
|
const expectedArr = Array.isArray(entry.expected) ? entry.expected : [];
|
|
1969
1524
|
const tools = expectedArr.map((e) => e.tool).filter(Boolean).join(", ");
|
|
1970
1525
|
return tools ? `Agent called tools in order: ${tools}` : "Agent followed expected tool trajectory";
|
|
1971
1526
|
}
|
|
1972
|
-
case "
|
|
1973
|
-
|
|
1974
|
-
const graderName = entry.name ?? deriveGraderNameFromCommand(entry.command) ?? "code-grader";
|
|
1527
|
+
case "script": {
|
|
1528
|
+
const graderName = entry.metric ?? deriveGraderNameFromCommand(entry.command) ?? "script";
|
|
1975
1529
|
const desc = typeof entry.description === "string" ? entry.description : void 0;
|
|
1976
|
-
return
|
|
1530
|
+
return scriptGraderInstruction(graderName, desc);
|
|
1977
1531
|
}
|
|
1978
|
-
case "field-accuracy":
|
|
1979
|
-
case "field_accuracy": {
|
|
1532
|
+
case "field-accuracy": {
|
|
1980
1533
|
const fieldPaths = Array.isArray(entry.fields) ? entry.fields.map((f) => f.path).filter(Boolean).join(", ") : "";
|
|
1981
1534
|
return fieldPaths ? `Fields ${fieldPaths} match expected values` : "Fields match expected values";
|
|
1982
1535
|
}
|
|
@@ -1985,14 +1538,12 @@ function assertionToNaturalLanguage(entry) {
|
|
|
1985
1538
|
case "cost":
|
|
1986
1539
|
return typeof entry.budget === "number" ? `Cost under $${entry.budget}` : "Cost within budget";
|
|
1987
1540
|
case "token-usage":
|
|
1988
|
-
case "token_usage":
|
|
1989
1541
|
return "Token usage within limits";
|
|
1990
1542
|
case "execution-metrics":
|
|
1991
|
-
case "execution_metrics":
|
|
1992
1543
|
return "Execution within metric bounds";
|
|
1993
1544
|
default: {
|
|
1994
1545
|
if (entry.command !== void 0 && type) {
|
|
1995
|
-
return
|
|
1546
|
+
return scriptGraderInstruction(deriveGraderNameFromCommand(entry.command) ?? type);
|
|
1996
1547
|
}
|
|
1997
1548
|
if (typeof entry.criteria === "string") return entry.criteria;
|
|
1998
1549
|
if (typeof entry.prompt === "string") return entry.prompt;
|
|
@@ -2001,7 +1552,17 @@ function assertionToNaturalLanguage(entry) {
|
|
|
2001
1552
|
}
|
|
2002
1553
|
}
|
|
2003
1554
|
function assertionToNaturalLanguageList(entry) {
|
|
2004
|
-
if (entry.type === "llm-
|
|
1555
|
+
if (entry.type === "llm-rubric") {
|
|
1556
|
+
if (Array.isArray(entry.value) && entry.value.length > 0) {
|
|
1557
|
+
return entry.value.map((rubric) => {
|
|
1558
|
+
if (typeof rubric === "string") return rubric;
|
|
1559
|
+
if (!rubric || typeof rubric !== "object") return void 0;
|
|
1560
|
+
const item = rubric;
|
|
1561
|
+
return item.outcome ?? item.criteria ?? item.id;
|
|
1562
|
+
}).filter((value) => typeof value === "string");
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
if (entry.type === "llm-grader") {
|
|
2005
1566
|
if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
|
|
2006
1567
|
return entry.rubrics.map((r) => r.outcome ?? r.criteria ?? r.id).filter((s) => typeof s === "string");
|
|
2007
1568
|
}
|
|
@@ -2012,16 +1573,6 @@ function assertionToNaturalLanguageList(entry) {
|
|
|
2012
1573
|
function extractTriggerAssertions(assertions) {
|
|
2013
1574
|
return assertions.filter((a) => a.type === "skill-trigger");
|
|
2014
1575
|
}
|
|
2015
|
-
function resolveAssertions(rawCase) {
|
|
2016
|
-
if (Array.isArray(rawCase.assertions)) return rawCase.assertions;
|
|
2017
|
-
if (Array.isArray(rawCase.assert)) return rawCase.assert;
|
|
2018
|
-
return [];
|
|
2019
|
-
}
|
|
2020
|
-
function resolveSuiteAssertions(suite) {
|
|
2021
|
-
if (Array.isArray(suite.assertions)) return suite.assertions;
|
|
2022
|
-
if (Array.isArray(suite.assert)) return suite.assert;
|
|
2023
|
-
return [];
|
|
2024
|
-
}
|
|
2025
1576
|
function extractInput(rawCase) {
|
|
2026
1577
|
const files = Array.isArray(rawCase.input_files) ? rawCase.input_files.filter((f) => typeof f === "string") : [];
|
|
2027
1578
|
const input = rawCase.input;
|
|
@@ -2059,7 +1610,6 @@ function extractExpectedOutput(raw) {
|
|
|
2059
1610
|
return JSON.stringify(raw);
|
|
2060
1611
|
}
|
|
2061
1612
|
function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
2062
|
-
const warnings = [];
|
|
2063
1613
|
const files = /* @__PURE__ */ new Map();
|
|
2064
1614
|
if (typeof suite !== "object" || suite === null) {
|
|
2065
1615
|
throw new Error(`Invalid EVAL.yaml: expected an object in '${source}'`);
|
|
@@ -2068,10 +1618,7 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
|
2068
1618
|
if (!Array.isArray(rawSuite.tests)) {
|
|
2069
1619
|
throw new Error(`Invalid EVAL.yaml: missing 'tests' array in '${source}'`);
|
|
2070
1620
|
}
|
|
2071
|
-
|
|
2072
|
-
warnings.push("'assert' is deprecated at the suite level. Use 'assertions' instead.");
|
|
2073
|
-
}
|
|
2074
|
-
const suiteAssertions = resolveSuiteAssertions(rawSuite);
|
|
1621
|
+
const suiteAssertions = rawSuite.assert ?? [];
|
|
2075
1622
|
const suiteNlAssertions = suiteAssertions.filter((a) => a.type !== "skill-trigger").flatMap(assertionToNaturalLanguageList);
|
|
2076
1623
|
function getSkillFile(skillName) {
|
|
2077
1624
|
const existing = files.get(skillName);
|
|
@@ -2083,10 +1630,11 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
|
2083
1630
|
const tests = rawSuite.tests;
|
|
2084
1631
|
for (let idx = 0; idx < tests.length; idx++) {
|
|
2085
1632
|
const rawCase = tests[idx];
|
|
2086
|
-
const caseAssertions =
|
|
2087
|
-
if (rawCase.
|
|
2088
|
-
|
|
2089
|
-
|
|
1633
|
+
const caseAssertions = rawCase.assert ?? [];
|
|
1634
|
+
if (typeof rawCase.criteria === "string" && rawCase.criteria.trim() && rawCase.assert !== void 0) {
|
|
1635
|
+
throw new Error(
|
|
1636
|
+
`Invalid EVAL.yaml test '${rawCase.id ?? idx + 1}' in '${source}': do not combine test-level 'criteria' with 'assert'. Put human-readable case descriptions in 'description', or express grading text as an explicit assertion such as { type: 'llm-rubric', value: ... }.`
|
|
1637
|
+
);
|
|
2090
1638
|
}
|
|
2091
1639
|
const nlAssertions = [];
|
|
2092
1640
|
if (typeof rawCase.criteria === "string" && rawCase.criteria.trim()) {
|
|
@@ -2139,12 +1687,12 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
|
2139
1687
|
files.delete("_no-skill");
|
|
2140
1688
|
}
|
|
2141
1689
|
}
|
|
2142
|
-
return { files, warnings };
|
|
1690
|
+
return { files, warnings: [] };
|
|
2143
1691
|
}
|
|
2144
1692
|
function transpileEvalYamlFile(evalYamlPath) {
|
|
2145
1693
|
const content = readFileSync(evalYamlPath, "utf8");
|
|
2146
1694
|
const parsed = parseYamlValue(content);
|
|
2147
|
-
return transpileEvalYaml(parsed,
|
|
1695
|
+
return transpileEvalYaml(parsed, path.basename(evalYamlPath));
|
|
2148
1696
|
}
|
|
2149
1697
|
function getOutputFilenames(result) {
|
|
2150
1698
|
const names = /* @__PURE__ */ new Map();
|
|
@@ -2181,35 +1729,12 @@ function toRepoPins(repos) {
|
|
|
2181
1729
|
...repo.path !== void 0 && { path: repo.path },
|
|
2182
1730
|
...repo.repo !== void 0 && { repo: repo.repo },
|
|
2183
1731
|
...repo.commit !== void 0 && { commit: repo.commit },
|
|
2184
|
-
...repo.base_commit !== void 0 && { baseCommit: repo.base_commit },
|
|
2185
1732
|
...repo.ancestor !== void 0 && { ancestor: repo.ancestor },
|
|
2186
1733
|
...repo.sparse !== void 0 && { sparse: repo.sparse }
|
|
2187
1734
|
}));
|
|
2188
1735
|
}
|
|
2189
|
-
function poolMetadata(setup, selectedSlotPath) {
|
|
2190
|
-
const slot = setup.poolSlot ?? setup.poolSlots.find((candidate) => candidate.path === selectedSlotPath);
|
|
2191
|
-
if (!slot) {
|
|
2192
|
-
return void 0;
|
|
2193
|
-
}
|
|
2194
|
-
return {
|
|
2195
|
-
fingerprint: slot.fingerprint,
|
|
2196
|
-
slotIndex: slot.index,
|
|
2197
|
-
lockPath: slot.lockPath
|
|
2198
|
-
};
|
|
2199
|
-
}
|
|
2200
|
-
async function releaseUnselectedPoolSlots(setup, selectedSlotPath) {
|
|
2201
|
-
if (!setup.poolManager) {
|
|
2202
|
-
return;
|
|
2203
|
-
}
|
|
2204
|
-
for (const slot of setup.poolSlots) {
|
|
2205
|
-
if (slot.path !== selectedSlotPath) {
|
|
2206
|
-
await setup.poolManager.releaseSlot(slot).catch(() => {
|
|
2207
|
-
});
|
|
2208
|
-
}
|
|
2209
|
-
}
|
|
2210
|
-
}
|
|
2211
1736
|
async function prepareEvalWorkspace(options) {
|
|
2212
|
-
const evalPath =
|
|
1737
|
+
const evalPath = path2.resolve(options.testFilePath);
|
|
2213
1738
|
const evalRunId = randomUUID();
|
|
2214
1739
|
const evalCases = options.evalCases ?? await loadTests(evalPath, options.repoRoot, {
|
|
2215
1740
|
verbose: options.verbose,
|
|
@@ -2221,7 +1746,7 @@ async function prepareEvalWorkspace(options) {
|
|
|
2221
1746
|
filter: options.filter,
|
|
2222
1747
|
evalPath
|
|
2223
1748
|
});
|
|
2224
|
-
const evalDir =
|
|
1749
|
+
const evalDir = path2.dirname(evalPath);
|
|
2225
1750
|
const workers = options.maxConcurrency ?? 1;
|
|
2226
1751
|
const retainOnSuccess = options.retainOnSuccess ?? (options.keepWorkspaces ? "keep" : "cleanup");
|
|
2227
1752
|
const retainOnFailure = options.retainOnFailure ?? (options.cleanupWorkspaces ? "cleanup" : "keep");
|
|
@@ -2236,15 +1761,11 @@ async function prepareEvalWorkspace(options) {
|
|
|
2236
1761
|
evalDir,
|
|
2237
1762
|
verbose: options.verbose,
|
|
2238
1763
|
workers,
|
|
2239
|
-
poolMaxSlots: options.poolMaxSlots,
|
|
2240
1764
|
workspacePath: options.workspacePath,
|
|
2241
|
-
legacyWorkspacePath: options.workspace
|
|
2242
|
-
workspaceMode: options.workspaceMode,
|
|
2243
|
-
workspaceClean: options.workspaceClean
|
|
1765
|
+
legacyWorkspacePath: options.workspace
|
|
2244
1766
|
});
|
|
2245
|
-
const
|
|
2246
|
-
const
|
|
2247
|
-
const selectedBaselineCommit = testPoolSlot ? sharedSetup.poolSlotBaselines.get(testPoolSlot.path) : sharedSetup.sharedBaselineCommit;
|
|
1767
|
+
const selectedWorkspacePath = sharedSetup.sharedWorkspacePath;
|
|
1768
|
+
const selectedBaselineCommit = sharedSetup.sharedBaselineCommit;
|
|
2248
1769
|
const caseSetup = await prepareEvalCaseWorkspace({
|
|
2249
1770
|
evalCase,
|
|
2250
1771
|
targetName: options.target.name,
|
|
@@ -2256,15 +1777,14 @@ async function prepareEvalWorkspace(options) {
|
|
|
2256
1777
|
evalDir,
|
|
2257
1778
|
cleanupWorkspaces: options.cleanupWorkspaces,
|
|
2258
1779
|
targetHooks: options.targetHooks,
|
|
1780
|
+
sharedExtensionState: sharedSetup.extensionState,
|
|
2259
1781
|
setupDebug: options.verbose
|
|
2260
1782
|
});
|
|
2261
1783
|
if (!caseSetup.workspacePath) {
|
|
2262
1784
|
throw new Error(
|
|
2263
|
-
`No workspace was materialized for test "${evalCase.id}". Add
|
|
1785
|
+
`No workspace was materialized for test "${evalCase.id}". Add environment.workdir before preparing an external attempt, or use an internal workspace override for private runtime plumbing.`
|
|
2264
1786
|
);
|
|
2265
1787
|
}
|
|
2266
|
-
await releaseUnselectedPoolSlots(sharedSetup, caseSetup.workspacePath);
|
|
2267
|
-
const pool = poolMetadata(sharedSetup, caseSetup.workspacePath);
|
|
2268
1788
|
return {
|
|
2269
1789
|
evalPath,
|
|
2270
1790
|
testId: evalCase.id,
|
|
@@ -2276,6 +1796,12 @@ async function prepareEvalWorkspace(options) {
|
|
|
2276
1796
|
},
|
|
2277
1797
|
createdAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
|
|
2278
1798
|
hookExecutions: [...sharedSetup.hookExecutions, ...caseSetup.hookExecutions],
|
|
1799
|
+
...caseSetup.extensionState?.providerContext !== void 0 && {
|
|
1800
|
+
providerContext: caseSetup.extensionState.providerContext
|
|
1801
|
+
},
|
|
1802
|
+
...caseSetup.extensionState?.metadata !== void 0 && {
|
|
1803
|
+
metadata: caseSetup.extensionState.metadata
|
|
1804
|
+
},
|
|
2279
1805
|
repoPins: toRepoPins(evalCase.workspace?.repos),
|
|
2280
1806
|
baseline: caseSetup.baselineCommit ? { status: "initialized", commit: caseSetup.baselineCommit } : { status: "unavailable" },
|
|
2281
1807
|
promptSource: {
|
|
@@ -2293,8 +1819,7 @@ async function prepareEvalWorkspace(options) {
|
|
|
2293
1819
|
retainOnFailure,
|
|
2294
1820
|
manualCleanup: true
|
|
2295
1821
|
},
|
|
2296
|
-
sharedWorkspace: caseSetup.isSharedWorkspace
|
|
2297
|
-
...pool !== void 0 && { pool }
|
|
1822
|
+
sharedWorkspace: caseSetup.isSharedWorkspace
|
|
2298
1823
|
};
|
|
2299
1824
|
} catch (error) {
|
|
2300
1825
|
if (sharedSetup) {
|
|
@@ -2304,22 +1829,51 @@ async function prepareEvalWorkspace(options) {
|
|
|
2304
1829
|
throw error;
|
|
2305
1830
|
}
|
|
2306
1831
|
}
|
|
1832
|
+
var ExecutionConfigSchema = external_exports.object({
|
|
1833
|
+
/** General eval parallelism (default: 3) */
|
|
1834
|
+
maxConcurrency: external_exports.number().int().min(1).max(50).optional(),
|
|
1835
|
+
/** Maximum retries on failure (default: 2) */
|
|
1836
|
+
maxRetries: external_exports.number().int().min(0).optional(),
|
|
1837
|
+
/** Agent timeout in milliseconds. No timeout if not set. */
|
|
1838
|
+
agentTimeoutMs: external_exports.number().int().min(0).optional(),
|
|
1839
|
+
/** Enable verbose logging */
|
|
1840
|
+
verbose: external_exports.boolean().optional(),
|
|
1841
|
+
/** Always keep temp workspaces after eval */
|
|
1842
|
+
keepWorkspaces: external_exports.boolean().optional()
|
|
1843
|
+
}).passthrough().superRefine((value, ctx) => {
|
|
1844
|
+
const supportedFields = /* @__PURE__ */ new Set([
|
|
1845
|
+
"maxConcurrency",
|
|
1846
|
+
"maxRetries",
|
|
1847
|
+
"agentTimeoutMs",
|
|
1848
|
+
"verbose",
|
|
1849
|
+
"keepWorkspaces"
|
|
1850
|
+
]);
|
|
1851
|
+
if ("otelFile" in value) {
|
|
1852
|
+
ctx.addIssue({
|
|
1853
|
+
code: external_exports.ZodIssueCode.custom,
|
|
1854
|
+
path: ["otelFile"],
|
|
1855
|
+
message: "execution.otelFile has been removed. Emit OpenTelemetry/OpenInference traces from the system under test or provider and correlate AgentV run artifacts with external_trace metadata."
|
|
1856
|
+
});
|
|
1857
|
+
}
|
|
1858
|
+
for (const key of Object.keys(value)) {
|
|
1859
|
+
if (!supportedFields.has(key) && key !== "otelFile") {
|
|
1860
|
+
ctx.addIssue({
|
|
1861
|
+
code: external_exports.ZodIssueCode.custom,
|
|
1862
|
+
path: [key],
|
|
1863
|
+
message: `Unsupported execution field '${key}'`
|
|
1864
|
+
});
|
|
1865
|
+
}
|
|
1866
|
+
}
|
|
1867
|
+
}).transform(({ maxConcurrency, maxRetries, agentTimeoutMs, verbose, keepWorkspaces }) => ({
|
|
1868
|
+
...maxConcurrency !== void 0 && { maxConcurrency },
|
|
1869
|
+
...maxRetries !== void 0 && { maxRetries },
|
|
1870
|
+
...agentTimeoutMs !== void 0 && { agentTimeoutMs },
|
|
1871
|
+
...verbose !== void 0 && { verbose },
|
|
1872
|
+
...keepWorkspaces !== void 0 && { keepWorkspaces }
|
|
1873
|
+
}));
|
|
2307
1874
|
var AgentVConfigSchema = external_exports.object({
|
|
2308
1875
|
/** Default execution settings */
|
|
2309
|
-
execution:
|
|
2310
|
-
/** Number of parallel workers (default: 3) */
|
|
2311
|
-
workers: external_exports.number().int().min(1).max(50).optional(),
|
|
2312
|
-
/** Maximum retries on failure (default: 2) */
|
|
2313
|
-
maxRetries: external_exports.number().int().min(0).optional(),
|
|
2314
|
-
/** Agent timeout in milliseconds. No timeout if not set. */
|
|
2315
|
-
agentTimeoutMs: external_exports.number().int().min(0).optional(),
|
|
2316
|
-
/** Enable verbose logging */
|
|
2317
|
-
verbose: external_exports.boolean().optional(),
|
|
2318
|
-
/** Always keep temp workspaces after eval */
|
|
2319
|
-
keepWorkspaces: external_exports.boolean().optional(),
|
|
2320
|
-
/** Write OTLP JSON trace to this path (supports {timestamp} placeholder) */
|
|
2321
|
-
otelFile: external_exports.string().optional()
|
|
2322
|
-
}).optional(),
|
|
1876
|
+
execution: ExecutionConfigSchema.optional(),
|
|
2323
1877
|
/** Output settings */
|
|
2324
1878
|
output: external_exports.object({
|
|
2325
1879
|
/** Default eval run artifact directory */
|
|
@@ -2332,13 +1886,6 @@ var AgentVConfigSchema = external_exports.object({
|
|
|
2332
1886
|
/** Response cache directory */
|
|
2333
1887
|
path: external_exports.string().optional()
|
|
2334
1888
|
}).optional(),
|
|
2335
|
-
/** Compatibility shorthand for experiments.default */
|
|
2336
|
-
defaultExperiment: external_exports.string().optional(),
|
|
2337
|
-
/** Experiment defaults */
|
|
2338
|
-
experiments: external_exports.object({
|
|
2339
|
-
/** Default experiment label or path used when eval runs omit --experiment */
|
|
2340
|
-
default: external_exports.string().optional()
|
|
2341
|
-
}).strict().optional(),
|
|
2342
1889
|
/** Cost and duration limits */
|
|
2343
1890
|
limits: external_exports.object({
|
|
2344
1891
|
/** Maximum cost per run in USD */
|
|
@@ -2356,7 +1903,9 @@ var AgentVConfigSchema = external_exports.object({
|
|
|
2356
1903
|
* stderr is forwarded to the user. Non-zero exit aborts with an error.
|
|
2357
1904
|
*/
|
|
2358
1905
|
beforeSession: external_exports.string().optional()
|
|
2359
|
-
}).optional()
|
|
1906
|
+
}).optional(),
|
|
1907
|
+
/** Project-local reusable references for fields that support ref://name */
|
|
1908
|
+
refs: external_exports.record(external_exports.string().min(1)).optional()
|
|
2360
1909
|
});
|
|
2361
1910
|
function defineConfig(config) {
|
|
2362
1911
|
return AgentVConfigSchema.parse(config);
|
|
@@ -2371,7 +1920,7 @@ var CONFIG_FILE_NAMES = [
|
|
|
2371
1920
|
];
|
|
2372
1921
|
async function loadTsConfig(projectRoot) {
|
|
2373
1922
|
const { existsSync: existsSync3 } = await import("node:fs");
|
|
2374
|
-
const { pathToFileURL
|
|
1923
|
+
const { pathToFileURL } = await import("node:url");
|
|
2375
1924
|
const { join } = await import("node:path");
|
|
2376
1925
|
for (const fileName of CONFIG_FILE_NAMES) {
|
|
2377
1926
|
const filePath = join(projectRoot, fileName);
|
|
@@ -2379,7 +1928,7 @@ async function loadTsConfig(projectRoot) {
|
|
|
2379
1928
|
continue;
|
|
2380
1929
|
}
|
|
2381
1930
|
try {
|
|
2382
|
-
const fileUrl =
|
|
1931
|
+
const fileUrl = pathToFileURL(filePath).href;
|
|
2383
1932
|
const mod = await import(fileUrl);
|
|
2384
1933
|
const config = mod.default ?? mod;
|
|
2385
1934
|
return AgentVConfigSchema.parse(config);
|
|
@@ -2462,87 +2011,14 @@ function buildPrompt(criteria, question, referenceAnswer) {
|
|
|
2462
2011
|
}
|
|
2463
2012
|
return parts.join("\n");
|
|
2464
2013
|
}
|
|
2465
|
-
async function scanRepoDeps(evalFilePaths) {
|
|
2466
|
-
const seen = /* @__PURE__ */ new Map();
|
|
2467
|
-
const errors = [];
|
|
2468
|
-
for (const filePath of evalFilePaths) {
|
|
2469
|
-
try {
|
|
2470
|
-
const repos = await extractReposFromEvalFile(filePath);
|
|
2471
|
-
for (const repo of repos) {
|
|
2472
|
-
if (!repo.repo) continue;
|
|
2473
|
-
const checkoutRef = getRepoCheckoutRef(repo);
|
|
2474
|
-
const ref = checkoutRef === "HEAD" ? void 0 : checkoutRef;
|
|
2475
|
-
const key = `${normalizeRepoIdentity(repo.repo)}\0${ref ?? ""}`;
|
|
2476
|
-
const existing = seen.get(key);
|
|
2477
|
-
if (existing) {
|
|
2478
|
-
existing.usedBy.push(filePath);
|
|
2479
|
-
} else {
|
|
2480
|
-
seen.set(key, {
|
|
2481
|
-
url: resolveRepoCloneUrl(repo.repo),
|
|
2482
|
-
ref,
|
|
2483
|
-
sparse: repo.sparse,
|
|
2484
|
-
ancestor: repo.ancestor,
|
|
2485
|
-
usedBy: [filePath]
|
|
2486
|
-
});
|
|
2487
|
-
}
|
|
2488
|
-
}
|
|
2489
|
-
} catch (err) {
|
|
2490
|
-
errors.push({
|
|
2491
|
-
file: filePath,
|
|
2492
|
-
message: err instanceof Error ? err.message : String(err)
|
|
2493
|
-
});
|
|
2494
|
-
}
|
|
2495
|
-
}
|
|
2496
|
-
return { repos: [...seen.values()], errors };
|
|
2497
|
-
}
|
|
2498
|
-
async function extractReposFromEvalFile(filePath) {
|
|
2499
|
-
const content = await readFile2(filePath, "utf8");
|
|
2500
|
-
const parsed = interpolateEnv(parseYamlValue(content), process.env);
|
|
2501
|
-
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
|
|
2502
|
-
const obj = parsed;
|
|
2503
|
-
const evalFileDir = path4.dirname(path4.resolve(filePath));
|
|
2504
|
-
const repos = [];
|
|
2505
|
-
const suiteRepos = await extractReposFromWorkspaceRaw(obj.workspace, evalFileDir);
|
|
2506
|
-
repos.push(...suiteRepos);
|
|
2507
|
-
const tests = Array.isArray(obj.tests) ? obj.tests : [];
|
|
2508
|
-
for (const test of tests) {
|
|
2509
|
-
if (test && typeof test === "object" && !Array.isArray(test)) {
|
|
2510
|
-
const testObj = test;
|
|
2511
|
-
const testRepos = await extractReposFromWorkspaceRaw(testObj.workspace, evalFileDir);
|
|
2512
|
-
repos.push(...testRepos);
|
|
2513
|
-
}
|
|
2514
|
-
}
|
|
2515
|
-
return repos;
|
|
2516
|
-
}
|
|
2517
|
-
async function extractReposFromWorkspaceRaw(raw, evalFileDir) {
|
|
2518
|
-
if (typeof raw === "string") {
|
|
2519
|
-
const workspaceFilePath = path4.resolve(evalFileDir, raw);
|
|
2520
|
-
const content = await readFile2(workspaceFilePath, "utf8");
|
|
2521
|
-
const parsed = interpolateEnv(parseYamlValue(content), process.env);
|
|
2522
|
-
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
|
|
2523
|
-
return extractReposFromObject(parsed);
|
|
2524
|
-
}
|
|
2525
|
-
if (raw && typeof raw === "object" && !Array.isArray(raw)) {
|
|
2526
|
-
return extractReposFromObject(raw);
|
|
2527
|
-
}
|
|
2528
|
-
return [];
|
|
2529
|
-
}
|
|
2530
|
-
function extractReposFromObject(obj) {
|
|
2531
|
-
const rawRepos = Array.isArray(obj.repos) ? obj.repos : [];
|
|
2532
|
-
const result = [];
|
|
2533
|
-
for (const r of rawRepos) {
|
|
2534
|
-
const parsed = parseRepoConfig(r);
|
|
2535
|
-
if (parsed?.repo) {
|
|
2536
|
-
result.push(parsed);
|
|
2537
|
-
}
|
|
2538
|
-
}
|
|
2539
|
-
return result;
|
|
2540
|
-
}
|
|
2541
2014
|
var execFileAsync = promisify(execFile);
|
|
2542
2015
|
var RESULTS_REPO_RESULTS_DIR = ".agentv/results";
|
|
2543
|
-
var RESULTS_REPO_RUNS_DIR = "
|
|
2544
|
-
var
|
|
2545
|
-
var
|
|
2016
|
+
var RESULTS_REPO_RUNS_DIR = ".";
|
|
2017
|
+
var RESULTS_REPO_INDEXES_DIR = ".indexes";
|
|
2018
|
+
var RESULTS_REPO_CACHE_DIR = ".cache";
|
|
2019
|
+
var RESULTS_REPO_TRACKED_DOT_DIRS = [RESULTS_REPO_INDEXES_DIR, RESULTS_REPO_CACHE_DIR];
|
|
2020
|
+
var GIT_RESULTS_INDEX_CACHE_SCHEMA_VERSION = "agentv.git_results_index_cache.v1";
|
|
2021
|
+
var GIT_RESULTS_INDEX_LAYOUT_VERSION = "agentv.results_repo_branch_root.v1";
|
|
2546
2022
|
var FALLBACK_RESULTS_REPO_COMMIT_EMAIL = "agentv@results-repo";
|
|
2547
2023
|
var FALLBACK_RESULTS_REPO_COMMIT_NAME = "AgentV Results";
|
|
2548
2024
|
var GIT_COMMIT_IDENTITY_ENV_KEYS = [
|
|
@@ -2563,121 +2039,11 @@ var GIT_EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
|
|
|
2563
2039
|
var RESULTS_REPO_GENESIS_MESSAGE = "chore(results): initialize AgentV results branch";
|
|
2564
2040
|
var RESULTS_REPO_GENESIS_DATE = "@0 +0000";
|
|
2565
2041
|
var RESULT_INDEX_FILENAME2 = "index.jsonl";
|
|
2042
|
+
var RUN_INTERNAL_DIRNAME = ".internal";
|
|
2566
2043
|
var RESULTS_REPO_GITATTRIBUTES_FILE = ".gitattributes";
|
|
2567
2044
|
var RESULTS_REPO_GITATTRIBUTES_CONTENT = `# Managed by AgentV. Artifact-aware merge so results sync never force-pushes.
|
|
2568
|
-
|
|
2569
|
-
index.jsonl merge=union
|
|
2570
|
-
# Editable run overlay (tags/feedback): 3-way JSON set/field union via the
|
|
2571
|
-
# agentv-json driver; a genuine scalar conflict falls through to a human merge.
|
|
2572
|
-
metadata/runs/**/*.json merge=agentv-json
|
|
2573
|
-
`;
|
|
2574
|
-
var RESULTS_JSON_MERGE_DRIVER_NAME = "agentv-json";
|
|
2575
|
-
var RESULTS_JSON_MERGE_DRIVER_SCRIPT = `#!/usr/bin/env node
|
|
2576
|
-
// AgentV results overlay merge driver (merge=agentv-json).
|
|
2577
|
-
//
|
|
2578
|
-
// Performs a 3-way merge of AgentV's editable JSON overlay files (run tags /
|
|
2579
|
-
// feedback). Tag lists merge as a 3-way set so concurrent add/remove are
|
|
2580
|
-
// commutative; display-only bookkeeping scalars that always differ between
|
|
2581
|
-
// writers (updated_at) are regenerated to the larger value instead of
|
|
2582
|
-
// conflicting. Content-derived concurrency tokens (tag_revision) are dropped on
|
|
2583
|
-
// merge so the reader recomputes a token matching the merged content rather than
|
|
2584
|
-
// carrying a stale token that could bypass optimistic-concurrency checks. Any
|
|
2585
|
-
// other genuinely diverging scalar exits non-zero, leaving the file conflicted
|
|
2586
|
-
// so the caller can route it to a human GitHub merge.
|
|
2587
|
-
//
|
|
2588
|
-
// Invoked by git as: node json-merge-driver.mjs %O %A %B %P
|
|
2589
|
-
// %O ancestor, %A current (overwritten with the result), %B other, %P path.
|
|
2590
|
-
import { readFileSync, writeFileSync } from 'node:fs';
|
|
2591
|
-
|
|
2592
|
-
const [, , ancestorFile, currentFile, otherFile] = process.argv;
|
|
2593
|
-
const CONFLICT = Symbol('conflict');
|
|
2594
|
-
// Scalars that legitimately differ on every write and carry no merge meaning.
|
|
2595
|
-
const REGENERABLE_SCALARS = new Set(['updated_at']);
|
|
2596
|
-
// Content-derived tokens (a hash of {tags, updated_at}) that guard optimistic
|
|
2597
|
-
// concurrency. Carrying either side's pre-merge token forward would let a stale
|
|
2598
|
-
// client whose token happens to match bypass the concurrency check and silently
|
|
2599
|
-
// overwrite the merged set, so we drop them and let the reader recompute a token
|
|
2600
|
-
// that matches the merged content.
|
|
2601
|
-
const DROP_ON_MERGE = new Set(['tag_revision']);
|
|
2602
|
-
|
|
2603
|
-
function readJson(file) {
|
|
2604
|
-
try {
|
|
2605
|
-
return JSON.parse(readFileSync(file, 'utf8'));
|
|
2606
|
-
} catch {
|
|
2607
|
-
return undefined;
|
|
2608
|
-
}
|
|
2609
|
-
}
|
|
2610
|
-
|
|
2611
|
-
function isPrimitiveArray(value) {
|
|
2612
|
-
return (
|
|
2613
|
-
Array.isArray(value) &&
|
|
2614
|
-
value.every((entry) => entry === null || ['string', 'number', 'boolean'].includes(typeof entry))
|
|
2615
|
-
);
|
|
2616
|
-
}
|
|
2617
|
-
|
|
2618
|
-
function isPlainObject(value) {
|
|
2619
|
-
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
|
2620
|
-
}
|
|
2621
|
-
|
|
2622
|
-
function mergeSet(base, a, b) {
|
|
2623
|
-
const baseSet = new Set((isPrimitiveArray(base) ? base : []).map(String));
|
|
2624
|
-
const aSet = new Set(a.map(String));
|
|
2625
|
-
const bSet = new Set(b.map(String));
|
|
2626
|
-
const removed = new Set();
|
|
2627
|
-
for (const value of baseSet) {
|
|
2628
|
-
if (!aSet.has(value) || !bSet.has(value)) removed.add(value);
|
|
2629
|
-
}
|
|
2630
|
-
const merged = [];
|
|
2631
|
-
const seen = new Set();
|
|
2632
|
-
for (const value of [...a, ...b]) {
|
|
2633
|
-
const key = String(value);
|
|
2634
|
-
if (!removed.has(key) && !seen.has(key)) {
|
|
2635
|
-
seen.add(key);
|
|
2636
|
-
merged.push(value);
|
|
2637
|
-
}
|
|
2638
|
-
}
|
|
2639
|
-
return merged;
|
|
2640
|
-
}
|
|
2641
|
-
|
|
2642
|
-
function merge3(base, a, b, key) {
|
|
2643
|
-
if (JSON.stringify(a) === JSON.stringify(b)) return a;
|
|
2644
|
-
if (JSON.stringify(a) === JSON.stringify(base)) return b;
|
|
2645
|
-
if (JSON.stringify(b) === JSON.stringify(base)) return a;
|
|
2646
|
-
if (isPrimitiveArray(a) && isPrimitiveArray(b)) return mergeSet(base, a, b);
|
|
2647
|
-
if (isPlainObject(a) && isPlainObject(b)) {
|
|
2648
|
-
const keys = new Set([
|
|
2649
|
-
...Object.keys(a),
|
|
2650
|
-
...Object.keys(b),
|
|
2651
|
-
...(isPlainObject(base) ? Object.keys(base) : []),
|
|
2652
|
-
]);
|
|
2653
|
-
const out = {};
|
|
2654
|
-
for (const childKey of keys) {
|
|
2655
|
-
if (DROP_ON_MERGE.has(childKey)) continue;
|
|
2656
|
-
const merged = merge3(
|
|
2657
|
-
isPlainObject(base) ? base[childKey] : undefined,
|
|
2658
|
-
a[childKey],
|
|
2659
|
-
b[childKey],
|
|
2660
|
-
childKey,
|
|
2661
|
-
);
|
|
2662
|
-
if (merged === CONFLICT) return CONFLICT;
|
|
2663
|
-
if (merged !== undefined) out[childKey] = merged;
|
|
2664
|
-
}
|
|
2665
|
-
return out;
|
|
2666
|
-
}
|
|
2667
|
-
if (REGENERABLE_SCALARS.has(key)) {
|
|
2668
|
-
return [a, b].filter((value) => value !== undefined).sort().pop();
|
|
2669
|
-
}
|
|
2670
|
-
return CONFLICT;
|
|
2671
|
-
}
|
|
2672
|
-
|
|
2673
|
-
const ancestor = readJson(ancestorFile) ?? {};
|
|
2674
|
-
const current = readJson(currentFile);
|
|
2675
|
-
const other = readJson(otherFile);
|
|
2676
|
-
if (current === undefined || other === undefined) process.exit(1);
|
|
2677
|
-
const result = merge3(ancestor, current, other, '');
|
|
2678
|
-
if (result === CONFLICT) process.exit(1);
|
|
2679
|
-
writeFileSync(currentFile, \`\${JSON.stringify(result, null, 2)}\\n\`);
|
|
2680
|
-
process.exit(0);
|
|
2045
|
+
${RESULTS_REPO_INDEXES_DIR}/*.jsonl merge=union
|
|
2046
|
+
*/.internal/index.jsonl merge=union
|
|
2681
2047
|
`;
|
|
2682
2048
|
var activeResultsRepoSyncs = /* @__PURE__ */ new Set();
|
|
2683
2049
|
function sanitizeRepoSlug(repo) {
|
|
@@ -2693,37 +2059,34 @@ function withFriendlyGitHubAuthError(error) {
|
|
|
2693
2059
|
}
|
|
2694
2060
|
function expandHome(p) {
|
|
2695
2061
|
if (p === "~" || p.startsWith("~/") || p.startsWith("~\\")) {
|
|
2696
|
-
return
|
|
2062
|
+
return path3.join(os.homedir(), p.slice(1));
|
|
2697
2063
|
}
|
|
2698
2064
|
return p;
|
|
2699
2065
|
}
|
|
2700
2066
|
function resolveLocalPath(p, baseDir) {
|
|
2701
2067
|
const expanded = expandHome(p);
|
|
2702
|
-
return
|
|
2068
|
+
return path3.isAbsolute(expanded) ? expanded : path3.resolve(baseDir, expanded);
|
|
2069
|
+
}
|
|
2070
|
+
function isNormalizedResultsConfig(config) {
|
|
2071
|
+
return typeof config.storageBranchWorktree === "boolean";
|
|
2703
2072
|
}
|
|
2704
2073
|
function normalizeResultsConfig(config, options) {
|
|
2074
|
+
if (isNormalizedResultsConfig(config)) {
|
|
2075
|
+
return config;
|
|
2076
|
+
}
|
|
2705
2077
|
const baseDir = options?.baseDir ?? process.cwd();
|
|
2706
2078
|
const repoUrl = (config.repo_url ?? config.repo)?.trim();
|
|
2707
|
-
const
|
|
2708
|
-
const
|
|
2079
|
+
const explicitPath = config.path?.trim();
|
|
2080
|
+
const repoPath = config.repo_path?.trim() ?? (repoUrl ? void 0 : explicitPath);
|
|
2081
|
+
const explicitClonePath = repoUrl ? explicitPath : void 0;
|
|
2709
2082
|
const repo = repoUrl ?? repoPath ?? "";
|
|
2710
2083
|
const branch = config.branch?.trim() || (repoPath ? DEFAULT_RESULTS_BRANCH : void 0);
|
|
2711
2084
|
const useStorageBranchWorktree = Boolean(repoPath || repoUrl && explicitClonePath && branch);
|
|
2712
2085
|
const remote = config.remote?.trim() || "origin";
|
|
2713
|
-
const autoPush = config.
|
|
2714
|
-
const requirePush = config.
|
|
2715
|
-
const configuredPushConflictPolicy = config.sync?.push_conflict_policy;
|
|
2716
|
-
if (configuredPushConflictPolicy === "backup_and_force_push") {
|
|
2717
|
-
throw new Error(
|
|
2718
|
-
"results.sync.push_conflict_policy: 'backup_and_force_push' is no longer supported. Remove the field or set it to 'block'; AgentV never force-pushes result branches."
|
|
2719
|
-
);
|
|
2720
|
-
}
|
|
2721
|
-
if (configuredPushConflictPolicy !== void 0 && configuredPushConflictPolicy !== "block") {
|
|
2722
|
-
throw new Error("results.sync.push_conflict_policy must be 'block'");
|
|
2723
|
-
}
|
|
2724
|
-
const pushConflictPolicy = configuredPushConflictPolicy ?? "block";
|
|
2086
|
+
const autoPush = config.auto_push === true;
|
|
2087
|
+
const requirePush = config.require_push === true;
|
|
2725
2088
|
const resolvedRepoPath = repoPath ? resolveLocalPath(repoPath, baseDir) : void 0;
|
|
2726
|
-
const resolvedPath = explicitClonePath ? resolveLocalPath(explicitClonePath, baseDir) : repoUrl ?
|
|
2089
|
+
const resolvedPath = explicitClonePath ? resolveLocalPath(explicitClonePath, baseDir) : repoUrl ? path3.join(getAgentvDataDir(), "results", sanitizeRepoSlug(repoUrl)) : resolvedRepoPath ?? path3.join(getAgentvDataDir(), "results", sanitizeRepoSlug(repo));
|
|
2727
2090
|
return {
|
|
2728
2091
|
mode: "github",
|
|
2729
2092
|
repo,
|
|
@@ -2734,8 +2097,7 @@ function normalizeResultsConfig(config, options) {
|
|
|
2734
2097
|
path: resolvedPath,
|
|
2735
2098
|
auto_push: autoPush,
|
|
2736
2099
|
require_push: requirePush,
|
|
2737
|
-
push_conflict_policy:
|
|
2738
|
-
branch_prefix: config.branch_prefix?.trim() || "eval-results",
|
|
2100
|
+
push_conflict_policy: "block",
|
|
2739
2101
|
storageBranchWorktree: useStorageBranchWorktree
|
|
2740
2102
|
};
|
|
2741
2103
|
}
|
|
@@ -2754,11 +2116,11 @@ function resolveResultsRepoUrl(repo) {
|
|
|
2754
2116
|
return trimmed;
|
|
2755
2117
|
}
|
|
2756
2118
|
function getResultsRepoLocalPaths(repo) {
|
|
2757
|
-
const rootDir =
|
|
2119
|
+
const rootDir = path3.join(getAgentvDataDir(), "cache", "results-repo", sanitizeRepoSlug(repo));
|
|
2758
2120
|
return {
|
|
2759
2121
|
rootDir,
|
|
2760
|
-
repoDir:
|
|
2761
|
-
statusFile:
|
|
2122
|
+
repoDir: path3.join(rootDir, "repo"),
|
|
2123
|
+
statusFile: path3.join(rootDir, "status.json")
|
|
2762
2124
|
};
|
|
2763
2125
|
}
|
|
2764
2126
|
function usesStorageBranchWorktree(config) {
|
|
@@ -2775,7 +2137,7 @@ function readPersistedStatus(statusFile) {
|
|
|
2775
2137
|
}
|
|
2776
2138
|
}
|
|
2777
2139
|
function writePersistedStatus(statusFile, status) {
|
|
2778
|
-
mkdirSync(
|
|
2140
|
+
mkdirSync(path3.dirname(statusFile), { recursive: true });
|
|
2779
2141
|
writeFileSync(statusFile, `${JSON.stringify(status, null, 2)}
|
|
2780
2142
|
`, "utf8");
|
|
2781
2143
|
}
|
|
@@ -3018,37 +2380,17 @@ async function ensureResultsMergeConfig(repoDir) {
|
|
|
3018
2380
|
try {
|
|
3019
2381
|
const { stdout } = await runGit(["rev-parse", "--git-common-dir"], { cwd: repoDir });
|
|
3020
2382
|
const resolved = stdout.trim();
|
|
3021
|
-
gitDir = resolved ?
|
|
2383
|
+
gitDir = resolved ? path3.isAbsolute(resolved) ? resolved : path3.join(repoDir, resolved) : path3.join(repoDir, ".git");
|
|
3022
2384
|
} catch {
|
|
3023
2385
|
return;
|
|
3024
2386
|
}
|
|
3025
|
-
const driverScriptPath = path5.join(gitDir, "agentv", "json-merge-driver.mjs");
|
|
3026
|
-
let scriptCurrent = false;
|
|
3027
|
-
try {
|
|
3028
|
-
scriptCurrent = readFileSync2(driverScriptPath, "utf8") === RESULTS_JSON_MERGE_DRIVER_SCRIPT;
|
|
3029
|
-
} catch {
|
|
3030
|
-
scriptCurrent = false;
|
|
3031
|
-
}
|
|
3032
|
-
if (!scriptCurrent) {
|
|
3033
|
-
mkdirSync(path5.dirname(driverScriptPath), { recursive: true });
|
|
3034
|
-
writeFileSync(driverScriptPath, RESULTS_JSON_MERGE_DRIVER_SCRIPT, { mode: 493 });
|
|
3035
|
-
}
|
|
3036
|
-
const driverCommand = `node ${JSON.stringify(driverScriptPath)} %O %A %B %P`;
|
|
3037
|
-
await runGit(
|
|
3038
|
-
["config", `merge.${RESULTS_JSON_MERGE_DRIVER_NAME}.name`, "AgentV results overlay JSON union"],
|
|
3039
|
-
{ cwd: repoDir, check: false }
|
|
3040
|
-
);
|
|
3041
|
-
await runGit(["config", `merge.${RESULTS_JSON_MERGE_DRIVER_NAME}.driver`, driverCommand], {
|
|
3042
|
-
cwd: repoDir,
|
|
3043
|
-
check: false
|
|
3044
|
-
});
|
|
3045
2387
|
writeResultsInfoAttributes(gitDir);
|
|
3046
2388
|
}
|
|
3047
2389
|
var RESULTS_INFO_ATTRIBUTES_BEGIN = "# >>> agentv results merge attributes >>>";
|
|
3048
2390
|
var RESULTS_INFO_ATTRIBUTES_END = "# <<< agentv results merge attributes <<<";
|
|
3049
2391
|
function writeResultsInfoAttributes(gitDir) {
|
|
3050
|
-
const infoDir =
|
|
3051
|
-
const attributesPath =
|
|
2392
|
+
const infoDir = path3.join(gitDir, "info");
|
|
2393
|
+
const attributesPath = path3.join(infoDir, "attributes");
|
|
3052
2394
|
const managedBlock = `${RESULTS_INFO_ATTRIBUTES_BEGIN}
|
|
3053
2395
|
${RESULTS_REPO_GITATTRIBUTES_CONTENT.trimEnd()}
|
|
3054
2396
|
${RESULTS_INFO_ATTRIBUTES_END}
|
|
@@ -3084,9 +2426,9 @@ async function addResultsGitattributesToIndex(repoDir, indexEnv) {
|
|
|
3084
2426
|
if (existing === RESULTS_REPO_GITATTRIBUTES_CONTENT) {
|
|
3085
2427
|
return;
|
|
3086
2428
|
}
|
|
3087
|
-
const scratchDir = mkdtempSync(
|
|
2429
|
+
const scratchDir = mkdtempSync(path3.join(os.tmpdir(), "agentv-results-attrs-"));
|
|
3088
2430
|
try {
|
|
3089
|
-
const scratchFile =
|
|
2431
|
+
const scratchFile = path3.join(scratchDir, RESULTS_REPO_GITATTRIBUTES_FILE);
|
|
3090
2432
|
writeFileSync(scratchFile, RESULTS_REPO_GITATTRIBUTES_CONTENT);
|
|
3091
2433
|
const { stdout: blob } = await runGit(["hash-object", "-w", "--no-filters", scratchFile], {
|
|
3092
2434
|
cwd: repoDir
|
|
@@ -3105,7 +2447,7 @@ async function addResultsGitattributesToIndex(repoDir, indexEnv) {
|
|
|
3105
2447
|
}
|
|
3106
2448
|
}
|
|
3107
2449
|
function updateStatusFile(config, patch) {
|
|
3108
|
-
const repo =
|
|
2450
|
+
const repo = normalizeResultsConfig(config).repo;
|
|
3109
2451
|
const cachePaths = getResultsRepoLocalPaths(repo);
|
|
3110
2452
|
const current = readPersistedStatus(cachePaths.statusFile);
|
|
3111
2453
|
writePersistedStatus(cachePaths.statusFile, {
|
|
@@ -3126,9 +2468,9 @@ async function ensureResultsRepoClone(config) {
|
|
|
3126
2468
|
const cachePaths = getResultsRepoLocalPaths(normalized.repo);
|
|
3127
2469
|
const cloneDir = normalized.path;
|
|
3128
2470
|
mkdirSync(cachePaths.rootDir, { recursive: true });
|
|
3129
|
-
mkdirSync(
|
|
2471
|
+
mkdirSync(path3.dirname(cloneDir), { recursive: true });
|
|
3130
2472
|
const cloneMissing = !existsSync(cloneDir);
|
|
3131
|
-
const gitDir =
|
|
2473
|
+
const gitDir = path3.join(cloneDir, ".git");
|
|
3132
2474
|
const cloneEmpty = !cloneMissing && !existsSync(gitDir) && (await readdir(cloneDir)).length === 0;
|
|
3133
2475
|
if (cloneMissing || cloneEmpty) {
|
|
3134
2476
|
try {
|
|
@@ -3174,7 +2516,6 @@ function getResultsRepoStatus(config) {
|
|
|
3174
2516
|
auto_push: normalized.auto_push,
|
|
3175
2517
|
require_push: normalized.require_push,
|
|
3176
2518
|
push_conflict_policy: normalized.push_conflict_policy,
|
|
3177
|
-
branch_prefix: normalized.branch_prefix,
|
|
3178
2519
|
local_dir: normalized.path,
|
|
3179
2520
|
last_synced_at: persisted.last_synced_at,
|
|
3180
2521
|
last_error: persisted.last_error,
|
|
@@ -3295,7 +2636,7 @@ async function getStorageBranchWorktreeDirtyPaths(repoDir, config) {
|
|
|
3295
2636
|
const comparisonRef = await gitRefExists(repoDir, localRef) ? localRef : await gitRefExists(repoDir, remoteRef) ? remoteRef : void 0;
|
|
3296
2637
|
const dirtyPaths = [];
|
|
3297
2638
|
for (const gitPath of safeDirtyPaths) {
|
|
3298
|
-
const absolutePath =
|
|
2639
|
+
const absolutePath = path3.join(repoDir, ...gitPath.split("/"));
|
|
3299
2640
|
if (!existsSync(absolutePath)) {
|
|
3300
2641
|
if (await readGitText(repoDir, comparisonRef, gitPath) !== void 0) {
|
|
3301
2642
|
dirtyPaths.push(gitPath);
|
|
@@ -3401,7 +2742,7 @@ async function hasInProgressGitConflict(repoDir) {
|
|
|
3401
2742
|
check: false
|
|
3402
2743
|
});
|
|
3403
2744
|
const markerPath = stdout.trim();
|
|
3404
|
-
const resolvedMarkerPath =
|
|
2745
|
+
const resolvedMarkerPath = path3.isAbsolute(markerPath) ? markerPath : path3.join(repoDir, markerPath);
|
|
3405
2746
|
if (markerPath && existsSync(resolvedMarkerPath)) {
|
|
3406
2747
|
return true;
|
|
3407
2748
|
}
|
|
@@ -3573,22 +2914,49 @@ function withActionFlags(status, flags) {
|
|
|
3573
2914
|
};
|
|
3574
2915
|
}
|
|
3575
2916
|
function isSafeResultsRepoPath(p) {
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
|
|
2917
|
+
const normalized = p.split(path3.sep).join("/");
|
|
2918
|
+
const segments = normalized.split("/").filter(Boolean);
|
|
2919
|
+
const isRunBundlePath = segments.length >= 2 && !segments[0]?.startsWith(".") && segments[0] !== RESULTS_REPO_GITATTRIBUTES_FILE;
|
|
2920
|
+
return normalized === RESULTS_REPO_GITATTRIBUTES_FILE || RESULTS_REPO_TRACKED_DOT_DIRS.some(
|
|
2921
|
+
(dir) => normalized === dir || normalized.startsWith(`${dir}/`)
|
|
2922
|
+
) || isRunBundlePath;
|
|
2923
|
+
}
|
|
2924
|
+
async function existingTrackedResultsPathspecs(repoDir) {
|
|
2925
|
+
const targets = /* @__PURE__ */ new Set();
|
|
2926
|
+
if (existsSync(path3.join(repoDir, RESULTS_REPO_GITATTRIBUTES_FILE))) {
|
|
2927
|
+
targets.add(RESULTS_REPO_GITATTRIBUTES_FILE);
|
|
2928
|
+
}
|
|
2929
|
+
for (const dir of RESULTS_REPO_TRACKED_DOT_DIRS) {
|
|
2930
|
+
if (existsSync(path3.join(repoDir, dir))) {
|
|
2931
|
+
targets.add(dir);
|
|
3583
2932
|
continue;
|
|
3584
2933
|
}
|
|
3585
|
-
const { stdout } = await runGit(["ls-files", "--", dir], { cwd: repoDir, check: false });
|
|
3586
|
-
if (
|
|
3587
|
-
targets.
|
|
2934
|
+
const { stdout: stdout2 } = await runGit(["ls-files", "--", dir], { cwd: repoDir, check: false });
|
|
2935
|
+
if (stdout2.trim().length > 0) {
|
|
2936
|
+
targets.add(dir);
|
|
3588
2937
|
}
|
|
3589
2938
|
}
|
|
3590
|
-
|
|
3591
|
-
|
|
2939
|
+
for (const entry of await readdir(repoDir, { withFileTypes: true }).catch(() => [])) {
|
|
2940
|
+
if (entry.name.startsWith(".") || !entry.isDirectory()) {
|
|
2941
|
+
continue;
|
|
2942
|
+
}
|
|
2943
|
+
if (existsSync(path3.join(repoDir, entry.name, "summary.json"))) {
|
|
2944
|
+
targets.add(entry.name);
|
|
2945
|
+
}
|
|
2946
|
+
}
|
|
2947
|
+
const { stdout } = await runGit(["ls-files"], { cwd: repoDir, check: false });
|
|
2948
|
+
for (const file of stdout.split(/\r?\n/)) {
|
|
2949
|
+
const normalized = file.trim();
|
|
2950
|
+
if (!normalized) {
|
|
2951
|
+
continue;
|
|
2952
|
+
}
|
|
2953
|
+
const segments = normalized.split("/");
|
|
2954
|
+
if (segments.length >= 2 && segments[0] && !segments[0].startsWith(".")) {
|
|
2955
|
+
targets.add(segments[0]);
|
|
2956
|
+
}
|
|
2957
|
+
}
|
|
2958
|
+
return [...targets].sort();
|
|
2959
|
+
}
|
|
3592
2960
|
function areSafeResultsRepoPaths(paths) {
|
|
3593
2961
|
return paths.length > 0 && paths.every(isSafeResultsRepoPath);
|
|
3594
2962
|
}
|
|
@@ -3638,7 +3006,7 @@ async function getMergeBaseCommit(repoDir, a, b) {
|
|
|
3638
3006
|
return exitCode === 0 && sha.length > 0 ? sha : void 0;
|
|
3639
3007
|
}
|
|
3640
3008
|
function buildNeedsHumanMergeReason(targetBranch, remoteCommit, localCommit) {
|
|
3641
|
-
return `Results branch ${targetBranch} diverged from the remote and could not be auto-merged: a genuine content conflict
|
|
3009
|
+
return `Results branch ${targetBranch} diverged from the remote and could not be auto-merged: a genuine results content conflict remains (remote ${formatShortSha(
|
|
3642
3010
|
remoteCommit
|
|
3643
3011
|
)}, local ${formatShortSha(
|
|
3644
3012
|
localCommit
|
|
@@ -3692,13 +3060,7 @@ async function countContributedRunDirs(repoDir, remoteTargetCommit, sourceCommit
|
|
|
3692
3060
|
return void 0;
|
|
3693
3061
|
}
|
|
3694
3062
|
const { stdout, exitCode } = await runGit(
|
|
3695
|
-
[
|
|
3696
|
-
"diff",
|
|
3697
|
-
"--name-only",
|
|
3698
|
-
`${remoteTargetCommit}...${sourceCommit}`,
|
|
3699
|
-
"--",
|
|
3700
|
-
`${RESULTS_REPO_RUNS_DIR}/`
|
|
3701
|
-
],
|
|
3063
|
+
["diff", "--name-only", `${remoteTargetCommit}...${sourceCommit}`, "--", RESULTS_REPO_RUNS_DIR],
|
|
3702
3064
|
{ cwd: repoDir, check: false }
|
|
3703
3065
|
);
|
|
3704
3066
|
if (exitCode !== 0) {
|
|
@@ -3711,8 +3073,8 @@ async function countContributedRunDirs(repoDir, remoteTargetCommit, sourceCommit
|
|
|
3711
3073
|
continue;
|
|
3712
3074
|
}
|
|
3713
3075
|
const segments = file.split("/");
|
|
3714
|
-
if (segments[0]
|
|
3715
|
-
runDirs.add(
|
|
3076
|
+
if (segments.length >= 2 && segments[0] && !segments[0].startsWith(".")) {
|
|
3077
|
+
runDirs.add(segments[0]);
|
|
3716
3078
|
}
|
|
3717
3079
|
}
|
|
3718
3080
|
return runDirs.size;
|
|
@@ -3954,9 +3316,6 @@ async function getResultsRepoSyncStatus(config) {
|
|
|
3954
3316
|
}
|
|
3955
3317
|
try {
|
|
3956
3318
|
if (usesStorageBranchWorktree(normalized)) {
|
|
3957
|
-
await fetchResultsRepo(normalized.path, normalized.remote, normalized.branch).catch(
|
|
3958
|
-
() => void 0
|
|
3959
|
-
);
|
|
3960
3319
|
return withGitInspection(
|
|
3961
3320
|
baseStatus,
|
|
3962
3321
|
await inspectResultsStorageBranchGit(normalized.path, normalized)
|
|
@@ -4331,10 +3690,16 @@ async function syncResultsRepoForProject(config) {
|
|
|
4331
3690
|
}
|
|
4332
3691
|
}
|
|
4333
3692
|
if (inspection.syncStatus === "dirty") {
|
|
4334
|
-
const
|
|
4335
|
-
await runGit(["add", "--all", "--", ...
|
|
3693
|
+
const trackedPathspecs = await existingTrackedResultsPathspecs(repoDir);
|
|
3694
|
+
await runGit(["add", "--all", "--", ...trackedPathspecs], { cwd: repoDir });
|
|
4336
3695
|
await runGitWithFallbackCommitIdentity(
|
|
4337
|
-
[
|
|
3696
|
+
[
|
|
3697
|
+
"commit",
|
|
3698
|
+
"-m",
|
|
3699
|
+
"chore(results): sync local results branch changes",
|
|
3700
|
+
"--",
|
|
3701
|
+
...trackedPathspecs
|
|
3702
|
+
],
|
|
4338
3703
|
{
|
|
4339
3704
|
cwd: repoDir
|
|
4340
3705
|
}
|
|
@@ -4552,8 +3917,8 @@ async function prepareResultsRepoBranch(config, branchName) {
|
|
|
4552
3917
|
const cloneDir = await ensureResultsRepoClone(normalized);
|
|
4553
3918
|
const baseBranch = await resolveDefaultBranch(cloneDir);
|
|
4554
3919
|
await fetchResultsRepo(cloneDir, normalized.remote);
|
|
4555
|
-
const worktreeRoot = await mkdtemp(
|
|
4556
|
-
const worktreeDir =
|
|
3920
|
+
const worktreeRoot = await mkdtemp(path3.join(os.tmpdir(), "agentv-results-repo-"));
|
|
3921
|
+
const worktreeDir = path3.join(worktreeRoot, "repo");
|
|
4557
3922
|
await runGit(
|
|
4558
3923
|
["worktree", "add", "-B", branchName, worktreeDir, `${normalized.remote}/${baseBranch}`],
|
|
4559
3924
|
{
|
|
@@ -4575,12 +3940,12 @@ async function prepareResultsRepoBranch(config, branchName) {
|
|
|
4575
3940
|
}
|
|
4576
3941
|
async function stageResultsArtifacts(params) {
|
|
4577
3942
|
rmSync(params.destinationDir, { recursive: true, force: true });
|
|
4578
|
-
mkdirSync(
|
|
3943
|
+
mkdirSync(path3.dirname(params.destinationDir), { recursive: true });
|
|
4579
3944
|
await cp(params.sourceDir, params.destinationDir, { recursive: true });
|
|
4580
3945
|
}
|
|
4581
3946
|
function resolveResultsRepoRunsDir(config) {
|
|
4582
3947
|
const normalized = normalizeResultsConfig(config);
|
|
4583
|
-
return
|
|
3948
|
+
return path3.join(normalized.path, RESULTS_REPO_RESULTS_DIR);
|
|
4584
3949
|
}
|
|
4585
3950
|
async function directorySizeBytes(targetPath) {
|
|
4586
3951
|
const entry = await stat(targetPath);
|
|
@@ -4589,7 +3954,7 @@ async function directorySizeBytes(targetPath) {
|
|
|
4589
3954
|
}
|
|
4590
3955
|
let total = 0;
|
|
4591
3956
|
for (const child of await readdir(targetPath, { withFileTypes: true })) {
|
|
4592
|
-
total += await directorySizeBytes(
|
|
3957
|
+
total += await directorySizeBytes(path3.join(targetPath, child.name));
|
|
4593
3958
|
}
|
|
4594
3959
|
return total;
|
|
4595
3960
|
}
|
|
@@ -4656,7 +4021,7 @@ async function assertValidResultsBranchName(repoDir, branch) {
|
|
|
4656
4021
|
await runGit(["check-ref-format", "--branch", branch], { cwd: repoDir });
|
|
4657
4022
|
}
|
|
4658
4023
|
function normalizeDestinationPath(destinationPath) {
|
|
4659
|
-
const normalized = destinationPath.split(
|
|
4024
|
+
const normalized = destinationPath.split(path3.sep).join("/");
|
|
4660
4025
|
const segments = normalized.split("/").filter(Boolean);
|
|
4661
4026
|
if (segments.length === 0 || normalized.startsWith("/") || segments.some((segment) => segment === "..") || normalized.split("").some((char) => {
|
|
4662
4027
|
const code = char.charCodeAt(0);
|
|
@@ -4670,7 +4035,7 @@ async function listSourceFiles(sourceDir) {
|
|
|
4670
4035
|
const entries = [];
|
|
4671
4036
|
async function visit(dir) {
|
|
4672
4037
|
for (const entry of await readdir(dir, { withFileTypes: true })) {
|
|
4673
|
-
const absolutePath =
|
|
4038
|
+
const absolutePath = path3.join(dir, entry.name);
|
|
4674
4039
|
if (entry.isDirectory()) {
|
|
4675
4040
|
await visit(absolutePath);
|
|
4676
4041
|
} else if (entry.isFile() || entry.isSymbolicLink()) {
|
|
@@ -4682,20 +4047,18 @@ async function listSourceFiles(sourceDir) {
|
|
|
4682
4047
|
entries.sort();
|
|
4683
4048
|
return entries;
|
|
4684
4049
|
}
|
|
4685
|
-
function
|
|
4050
|
+
function isRecord2(value) {
|
|
4686
4051
|
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
4687
4052
|
}
|
|
4688
4053
|
function artifactSidecarPointers(record) {
|
|
4689
|
-
if (!
|
|
4054
|
+
if (!isRecord2(record) || !isRecord2(record.artifact_pointers)) {
|
|
4690
4055
|
return [];
|
|
4691
4056
|
}
|
|
4692
4057
|
const pointers = [];
|
|
4693
|
-
|
|
4694
|
-
|
|
4695
|
-
continue;
|
|
4696
|
-
}
|
|
4058
|
+
const pointer = record.artifact_pointers.transcript;
|
|
4059
|
+
if (isRecord2(pointer)) {
|
|
4697
4060
|
if (pointer.ref !== AGENTV_RESULTS_ARTIFACTS_REF || typeof pointer.path !== "string") {
|
|
4698
|
-
|
|
4061
|
+
return pointers;
|
|
4699
4062
|
}
|
|
4700
4063
|
pointers.push({
|
|
4701
4064
|
path: pointer.path,
|
|
@@ -4706,15 +4069,56 @@ function artifactSidecarPointers(record) {
|
|
|
4706
4069
|
return pointers;
|
|
4707
4070
|
}
|
|
4708
4071
|
function artifactSidecarKey(destinationPath, pointerPath) {
|
|
4709
|
-
return
|
|
4072
|
+
return path3.posix.join(
|
|
4710
4073
|
RESULTS_REPO_RUNS_DIR,
|
|
4711
4074
|
normalizeDestinationPath(destinationPath),
|
|
4712
4075
|
normalizeDestinationPath(pointerPath)
|
|
4713
4076
|
);
|
|
4714
4077
|
}
|
|
4078
|
+
function isDeprecatedTraceArtifactPath(relativePath) {
|
|
4079
|
+
return relativePath === "trace.json" || relativePath.endsWith("/trace.json");
|
|
4080
|
+
}
|
|
4081
|
+
function isResultManifestFilename(filename) {
|
|
4082
|
+
return filename === RESULT_INDEX_FILENAME2;
|
|
4083
|
+
}
|
|
4084
|
+
function safeLocalSummaryManifestPath(sourceDir, manifestPath) {
|
|
4085
|
+
if (typeof manifestPath !== "string" || manifestPath.trim().length === 0) {
|
|
4086
|
+
return void 0;
|
|
4087
|
+
}
|
|
4088
|
+
if (path3.isAbsolute(manifestPath)) {
|
|
4089
|
+
return void 0;
|
|
4090
|
+
}
|
|
4091
|
+
const normalized = path3.normalize(manifestPath);
|
|
4092
|
+
if (normalized.startsWith("..") || path3.isAbsolute(normalized)) {
|
|
4093
|
+
return void 0;
|
|
4094
|
+
}
|
|
4095
|
+
return path3.join(sourceDir, normalized);
|
|
4096
|
+
}
|
|
4097
|
+
function resolveLocalResultManifestPath(sourceDir) {
|
|
4098
|
+
try {
|
|
4099
|
+
const summary = JSON.parse(readFileSync2(path3.join(sourceDir, "summary.json"), "utf8"));
|
|
4100
|
+
const manifestPath2 = safeLocalSummaryManifestPath(
|
|
4101
|
+
sourceDir,
|
|
4102
|
+
summary.index_path ?? summary.manifest_path
|
|
4103
|
+
);
|
|
4104
|
+
if (manifestPath2 && existsSync(manifestPath2)) {
|
|
4105
|
+
return manifestPath2;
|
|
4106
|
+
}
|
|
4107
|
+
} catch {
|
|
4108
|
+
}
|
|
4109
|
+
const manifestPath = path3.join(sourceDir, RUN_INTERNAL_DIRNAME, RESULT_INDEX_FILENAME2);
|
|
4110
|
+
if (existsSync(manifestPath)) {
|
|
4111
|
+
return manifestPath;
|
|
4112
|
+
}
|
|
4113
|
+
const legacyManifestPath = path3.join(sourceDir, RESULT_INDEX_FILENAME2);
|
|
4114
|
+
if (existsSync(legacyManifestPath)) {
|
|
4115
|
+
return legacyManifestPath;
|
|
4116
|
+
}
|
|
4117
|
+
return void 0;
|
|
4118
|
+
}
|
|
4715
4119
|
function collectArtifactSidecarPointers(sourceDir) {
|
|
4716
|
-
const indexPath =
|
|
4717
|
-
if (!
|
|
4120
|
+
const indexPath = resolveLocalResultManifestPath(sourceDir);
|
|
4121
|
+
if (!indexPath) {
|
|
4718
4122
|
return [];
|
|
4719
4123
|
}
|
|
4720
4124
|
const pointers = [];
|
|
@@ -4732,16 +4136,16 @@ function collectArtifactSidecarPointers(sourceDir) {
|
|
|
4732
4136
|
}
|
|
4733
4137
|
function resolveArtifactPointerSourcePath(sourceDir, pointerPath) {
|
|
4734
4138
|
const normalizedPointerPath = normalizeDestinationPath(pointerPath);
|
|
4735
|
-
const sourceRoot =
|
|
4736
|
-
const sourcePath =
|
|
4737
|
-
const relative =
|
|
4738
|
-
if (relative.startsWith("..") ||
|
|
4139
|
+
const sourceRoot = path3.resolve(sourceDir);
|
|
4140
|
+
const sourcePath = path3.resolve(sourceRoot, ...normalizedPointerPath.split("/"));
|
|
4141
|
+
const relative = path3.relative(sourceRoot, sourcePath);
|
|
4142
|
+
if (relative.startsWith("..") || path3.isAbsolute(relative)) {
|
|
4739
4143
|
throw new Error(`Artifact pointer path escapes run directory: ${pointerPath}`);
|
|
4740
4144
|
}
|
|
4741
4145
|
return sourcePath;
|
|
4742
4146
|
}
|
|
4743
4147
|
function verifyArtifactPointerChecksum(pointer, content) {
|
|
4744
|
-
const sha256 =
|
|
4148
|
+
const sha256 = createHash("sha256").update(content).digest("hex");
|
|
4745
4149
|
if (pointer.sha256 && pointer.sha256 !== sha256) {
|
|
4746
4150
|
throw new Error(`Artifact pointer checksum mismatch for ${pointer.path}`);
|
|
4747
4151
|
}
|
|
@@ -4753,7 +4157,7 @@ async function prepareArtifactSidecar(params) {
|
|
|
4753
4157
|
if (params.pointers.length === 0) {
|
|
4754
4158
|
return void 0;
|
|
4755
4159
|
}
|
|
4756
|
-
const sidecarRoot = await mkdtemp(
|
|
4160
|
+
const sidecarRoot = await mkdtemp(path3.join(os.tmpdir(), "agentv-artifact-sidecar-"));
|
|
4757
4161
|
const copied = /* @__PURE__ */ new Set();
|
|
4758
4162
|
const contentByPath = /* @__PURE__ */ new Map();
|
|
4759
4163
|
try {
|
|
@@ -4769,8 +4173,8 @@ async function prepareArtifactSidecar(params) {
|
|
|
4769
4173
|
if (copied.has(relativePointerPath)) {
|
|
4770
4174
|
continue;
|
|
4771
4175
|
}
|
|
4772
|
-
const destinationPath =
|
|
4773
|
-
mkdirSync(
|
|
4176
|
+
const destinationPath = path3.join(sidecarRoot, ...relativePointerPath.split("/"));
|
|
4177
|
+
mkdirSync(path3.dirname(destinationPath), { recursive: true });
|
|
4774
4178
|
writeFileSync(destinationPath, content);
|
|
4775
4179
|
copied.add(relativePointerPath);
|
|
4776
4180
|
}
|
|
@@ -4797,22 +4201,24 @@ function rewritePublishedIndexLine(line, destinationPath) {
|
|
|
4797
4201
|
} catch {
|
|
4798
4202
|
return line;
|
|
4799
4203
|
}
|
|
4800
|
-
if (!
|
|
4204
|
+
if (!isRecord2(record) || !isRecord2(record.artifact_pointers)) {
|
|
4801
4205
|
return line;
|
|
4802
4206
|
}
|
|
4803
4207
|
let changed = false;
|
|
4804
|
-
|
|
4805
|
-
|
|
4806
|
-
continue;
|
|
4807
|
-
}
|
|
4808
|
-
if (pointer.ref !== AGENTV_RESULTS_ARTIFACTS_REF || typeof pointer.path !== "string") {
|
|
4809
|
-
continue;
|
|
4810
|
-
}
|
|
4208
|
+
const pointer = record.artifact_pointers.transcript;
|
|
4209
|
+
if (isRecord2(pointer) && pointer.ref === AGENTV_RESULTS_ARTIFACTS_REF && typeof pointer.path === "string") {
|
|
4811
4210
|
const key = artifactSidecarKey(destinationPath, pointer.path);
|
|
4812
4211
|
if (pointer.key !== key) {
|
|
4813
4212
|
pointer.key = key;
|
|
4814
4213
|
changed = true;
|
|
4815
4214
|
}
|
|
4215
|
+
if (Object.keys(record.artifact_pointers).length !== 1 || record.artifact_pointers.transcript !== pointer) {
|
|
4216
|
+
changed = true;
|
|
4217
|
+
}
|
|
4218
|
+
record.artifact_pointers = { transcript: pointer };
|
|
4219
|
+
} else {
|
|
4220
|
+
record.artifact_pointers = void 0;
|
|
4221
|
+
changed = true;
|
|
4816
4222
|
}
|
|
4817
4223
|
return changed ? JSON.stringify(record) : line;
|
|
4818
4224
|
}
|
|
@@ -4820,26 +4226,29 @@ async function preparePublishedResultsSource(params) {
|
|
|
4820
4226
|
if (params.pointers.length === 0) {
|
|
4821
4227
|
return void 0;
|
|
4822
4228
|
}
|
|
4823
|
-
const publishedRoot = await mkdtemp(
|
|
4229
|
+
const publishedRoot = await mkdtemp(path3.join(os.tmpdir(), "agentv-results-published-"));
|
|
4824
4230
|
const omittedPaths = new Set(
|
|
4825
4231
|
params.pointers.map((pointer) => normalizeDestinationPath(pointer.path))
|
|
4826
4232
|
);
|
|
4827
4233
|
try {
|
|
4828
4234
|
const sourceFiles = await listSourceFiles(params.sourceDir);
|
|
4829
4235
|
for (const sourceFile of sourceFiles) {
|
|
4830
|
-
const relativeFile =
|
|
4831
|
-
const destinationFile =
|
|
4832
|
-
if (relativeFile
|
|
4236
|
+
const relativeFile = path3.relative(params.sourceDir, sourceFile).split(path3.sep).join("/");
|
|
4237
|
+
const destinationFile = path3.join(publishedRoot, ...relativeFile.split("/"));
|
|
4238
|
+
if (isResultManifestFilename(path3.posix.basename(relativeFile))) {
|
|
4833
4239
|
const original = readFileSync2(sourceFile, "utf8");
|
|
4834
4240
|
const rewritten = original.split(/\r?\n/).map((line) => rewritePublishedIndexLine(line, params.destinationPath)).join("\n");
|
|
4835
|
-
mkdirSync(
|
|
4241
|
+
mkdirSync(path3.dirname(destinationFile), { recursive: true });
|
|
4836
4242
|
writeFileSync(destinationFile, rewritten);
|
|
4837
4243
|
continue;
|
|
4838
4244
|
}
|
|
4245
|
+
if (isDeprecatedTraceArtifactPath(relativeFile)) {
|
|
4246
|
+
continue;
|
|
4247
|
+
}
|
|
4839
4248
|
if (omittedPaths.has(relativeFile)) {
|
|
4840
4249
|
continue;
|
|
4841
4250
|
}
|
|
4842
|
-
mkdirSync(
|
|
4251
|
+
mkdirSync(path3.dirname(destinationFile), { recursive: true });
|
|
4843
4252
|
await cp(sourceFile, destinationFile, { dereference: false });
|
|
4844
4253
|
}
|
|
4845
4254
|
} catch (error) {
|
|
@@ -4906,8 +4315,8 @@ async function commitStorageBranchWorktreePaths(params) {
|
|
|
4906
4315
|
normalized
|
|
4907
4316
|
});
|
|
4908
4317
|
}
|
|
4909
|
-
const indexRoot = await mkdtemp(
|
|
4910
|
-
const indexFile =
|
|
4318
|
+
const indexRoot = await mkdtemp(path3.join(os.tmpdir(), "agentv-results-index-"));
|
|
4319
|
+
const indexFile = path3.join(indexRoot, "index");
|
|
4911
4320
|
const indexEnv = { GIT_INDEX_FILE: indexFile };
|
|
4912
4321
|
try {
|
|
4913
4322
|
if (base.baseRef) {
|
|
@@ -4917,7 +4326,7 @@ async function commitStorageBranchWorktreePaths(params) {
|
|
|
4917
4326
|
}
|
|
4918
4327
|
await addResultsGitattributesToIndex(params.repoDir, indexEnv);
|
|
4919
4328
|
for (const gitPath of paths) {
|
|
4920
|
-
const sourcePath =
|
|
4329
|
+
const sourcePath = path3.join(params.repoDir, ...gitPath.split("/"));
|
|
4921
4330
|
if (!existsSync(sourcePath)) {
|
|
4922
4331
|
await runGit(["update-index", "--force-remove", "--", gitPath], {
|
|
4923
4332
|
cwd: params.repoDir,
|
|
@@ -4983,7 +4392,7 @@ async function commitResultsRunWithTemporaryIndex(params) {
|
|
|
4983
4392
|
await assertValidResultsBranchName(params.repoDir, normalized.branch);
|
|
4984
4393
|
await ensureResultsBranchNotCheckedOut(params.repoDir, normalized);
|
|
4985
4394
|
const destinationRunPath = normalizeDestinationPath(params.destinationPath);
|
|
4986
|
-
const destinationTreePath =
|
|
4395
|
+
const destinationTreePath = path3.posix.join(RESULTS_REPO_RUNS_DIR, destinationRunPath);
|
|
4987
4396
|
let base = await resolveStorageBranchBase({
|
|
4988
4397
|
repoDir: params.repoDir,
|
|
4989
4398
|
normalized,
|
|
@@ -4997,8 +4406,8 @@ async function commitResultsRunWithTemporaryIndex(params) {
|
|
|
4997
4406
|
preferRemote: params.preferRemoteBase
|
|
4998
4407
|
});
|
|
4999
4408
|
}
|
|
5000
|
-
const indexRoot = await mkdtemp(
|
|
5001
|
-
const indexFile =
|
|
4409
|
+
const indexRoot = await mkdtemp(path3.join(os.tmpdir(), "agentv-results-index-"));
|
|
4410
|
+
const indexFile = path3.join(indexRoot, "index");
|
|
5002
4411
|
const indexEnv = { GIT_INDEX_FILE: indexFile };
|
|
5003
4412
|
try {
|
|
5004
4413
|
if (base.baseRef) {
|
|
@@ -5021,8 +4430,8 @@ async function commitResultsRunWithTemporaryIndex(params) {
|
|
|
5021
4430
|
}
|
|
5022
4431
|
const sourceFiles = await listSourceFiles(params.sourceDir);
|
|
5023
4432
|
for (const sourceFile of sourceFiles) {
|
|
5024
|
-
const relativeFile =
|
|
5025
|
-
const destinationFile =
|
|
4433
|
+
const relativeFile = path3.relative(params.sourceDir, sourceFile).split(path3.sep).join("/");
|
|
4434
|
+
const destinationFile = path3.posix.join(destinationTreePath, relativeFile);
|
|
5026
4435
|
const fileStat = await lstat(sourceFile);
|
|
5027
4436
|
let mode = fileStat.mode & 73 ? "100755" : "100644";
|
|
5028
4437
|
let hashInputPath = sourceFile;
|
|
@@ -5295,17 +4704,70 @@ async function directPushResultsWithDetails(params) {
|
|
|
5295
4704
|
await sidecar?.cleanup().catch(() => void 0);
|
|
5296
4705
|
}
|
|
5297
4706
|
}
|
|
5298
|
-
function
|
|
5299
|
-
|
|
5300
|
-
|
|
5301
|
-
|
|
5302
|
-
|
|
5303
|
-
|
|
5304
|
-
|
|
5305
|
-
|
|
4707
|
+
function safeGitSummaryManifestPath(runDir, manifestPath) {
|
|
4708
|
+
if (typeof manifestPath !== "string" || manifestPath.trim().length === 0) {
|
|
4709
|
+
return void 0;
|
|
4710
|
+
}
|
|
4711
|
+
if (manifestPath.startsWith("/")) {
|
|
4712
|
+
return void 0;
|
|
4713
|
+
}
|
|
4714
|
+
const normalized = path3.posix.normalize(manifestPath);
|
|
4715
|
+
if (normalized === ".." || normalized.startsWith("../")) {
|
|
4716
|
+
return void 0;
|
|
4717
|
+
}
|
|
4718
|
+
return path3.posix.join(runDir, normalized);
|
|
4719
|
+
}
|
|
4720
|
+
function buildGitManifestPaths(treePaths, summaryByPath) {
|
|
4721
|
+
const treePathSet = new Set(treePaths);
|
|
4722
|
+
const manifestByRunDir = /* @__PURE__ */ new Map();
|
|
4723
|
+
for (const [summaryPath, summary] of summaryByPath) {
|
|
4724
|
+
const runDir = path3.posix.dirname(summaryPath);
|
|
4725
|
+
const relativeRunPath = path3.posix.relative(RESULTS_REPO_RUNS_DIR, runDir);
|
|
4726
|
+
if (!isV2ResultsRepoRunPath(relativeRunPath)) {
|
|
4727
|
+
continue;
|
|
4728
|
+
}
|
|
4729
|
+
const manifestPath = safeGitSummaryManifestPath(runDir, summary.index_path ?? summary.manifest_path) ?? fallbackGitManifestPath(runDir, treePathSet);
|
|
4730
|
+
if (manifestPath && treePathSet.has(manifestPath)) {
|
|
4731
|
+
manifestByRunDir.set(runDir, manifestPath);
|
|
5306
4732
|
}
|
|
5307
|
-
return `${experiment}::${timestamp}`;
|
|
5308
4733
|
}
|
|
4734
|
+
for (const treePath of treePaths) {
|
|
4735
|
+
if (!treePath.endsWith(`/${RESULT_INDEX_FILENAME2}`)) {
|
|
4736
|
+
continue;
|
|
4737
|
+
}
|
|
4738
|
+
const runDir = gitRunDirForManifestPath(treePath);
|
|
4739
|
+
const relativeRunPath = path3.posix.relative(RESULTS_REPO_RUNS_DIR, runDir);
|
|
4740
|
+
if (!isV2ResultsRepoRunPath(relativeRunPath)) {
|
|
4741
|
+
continue;
|
|
4742
|
+
}
|
|
4743
|
+
if (!manifestByRunDir.has(runDir)) {
|
|
4744
|
+
manifestByRunDir.set(runDir, treePath);
|
|
4745
|
+
}
|
|
4746
|
+
}
|
|
4747
|
+
return [...manifestByRunDir.values()].sort();
|
|
4748
|
+
}
|
|
4749
|
+
function fallbackGitManifestPath(runDir, treePathSet) {
|
|
4750
|
+
for (const candidate of [
|
|
4751
|
+
path3.posix.join(runDir, ".internal", RESULT_INDEX_FILENAME2),
|
|
4752
|
+
path3.posix.join(runDir, RESULT_INDEX_FILENAME2)
|
|
4753
|
+
]) {
|
|
4754
|
+
if (treePathSet.has(candidate)) {
|
|
4755
|
+
return candidate;
|
|
4756
|
+
}
|
|
4757
|
+
}
|
|
4758
|
+
return void 0;
|
|
4759
|
+
}
|
|
4760
|
+
function gitRunDirForManifestPath(manifestPath) {
|
|
4761
|
+
const manifestDir = path3.posix.dirname(manifestPath);
|
|
4762
|
+
return path3.posix.basename(manifestDir) === ".internal" ? path3.posix.dirname(manifestDir) : manifestDir;
|
|
4763
|
+
}
|
|
4764
|
+
function isV2ResultsRepoRunPath(relativeRunPath) {
|
|
4765
|
+
const segments = relativeRunPath.split("/").filter(Boolean);
|
|
4766
|
+
return segments.length === 1 && !segments[0].startsWith(".");
|
|
4767
|
+
}
|
|
4768
|
+
function buildGitRunId(relativeRunPath) {
|
|
4769
|
+
const normalized = relativeRunPath.split(path3.sep).join("/");
|
|
4770
|
+
const segments = normalized.split("/").filter(Boolean);
|
|
5309
4771
|
return segments[0] ?? relativeRunPath;
|
|
5310
4772
|
}
|
|
5311
4773
|
function getRunExperiment(runId, summary) {
|
|
@@ -5433,7 +4895,7 @@ async function readGitResultArtifact(params) {
|
|
|
5433
4895
|
}
|
|
5434
4896
|
function buildWipBranchName(runDir) {
|
|
5435
4897
|
const hostname = os.hostname().replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 40);
|
|
5436
|
-
const runBasename =
|
|
4898
|
+
const runBasename = path3.basename(runDir).replace(/[^A-Za-z0-9._-]+/g, "-").slice(0, 60);
|
|
5437
4899
|
return `agentv/wip/${hostname}/${runBasename}`;
|
|
5438
4900
|
}
|
|
5439
4901
|
async function setupWipWorktree(params) {
|
|
@@ -5452,8 +4914,8 @@ async function setupWipWorktree(params) {
|
|
|
5452
4914
|
if (!baseRef) {
|
|
5453
4915
|
throw new Error("Could not resolve a base ref for the WIP results branch");
|
|
5454
4916
|
}
|
|
5455
|
-
const worktreeRoot = await mkdtemp(
|
|
5456
|
-
const worktreeDir =
|
|
4917
|
+
const worktreeRoot = await mkdtemp(path3.join(os.tmpdir(), "agentv-wip-"));
|
|
4918
|
+
const worktreeDir = path3.join(worktreeRoot, "repo");
|
|
5457
4919
|
await runGit(["worktree", "add", "-B", params.wipBranch, worktreeDir, baseRef], {
|
|
5458
4920
|
cwd: cloneDir
|
|
5459
4921
|
});
|
|
@@ -5472,7 +4934,7 @@ async function setupWipWorktree(params) {
|
|
|
5472
4934
|
};
|
|
5473
4935
|
}
|
|
5474
4936
|
async function pushWipCheckpoint(params) {
|
|
5475
|
-
const destinationDir =
|
|
4937
|
+
const destinationDir = path3.join(
|
|
5476
4938
|
params.handle.worktreeDir,
|
|
5477
4939
|
RESULTS_REPO_RUNS_DIR,
|
|
5478
4940
|
params.destinationPath
|
|
@@ -5482,8 +4944,11 @@ async function pushWipCheckpoint(params) {
|
|
|
5482
4944
|
sourceDir: params.sourceDir,
|
|
5483
4945
|
destinationDir
|
|
5484
4946
|
});
|
|
5485
|
-
|
|
5486
|
-
|
|
4947
|
+
writeFileSync(
|
|
4948
|
+
path3.join(params.handle.worktreeDir, RESULTS_REPO_GITATTRIBUTES_FILE),
|
|
4949
|
+
RESULTS_REPO_GITATTRIBUTES_CONTENT
|
|
4950
|
+
);
|
|
4951
|
+
await runGit(["add", "--all", "--", RESULTS_REPO_GITATTRIBUTES_FILE, params.destinationPath], {
|
|
5487
4952
|
cwd: params.handle.worktreeDir
|
|
5488
4953
|
});
|
|
5489
4954
|
const { stdout: status } = await runGit(["status", "--porcelain"], {
|
|
@@ -5506,7 +4971,19 @@ async function pushWipCheckpoint(params) {
|
|
|
5506
4971
|
async function deleteWipBranch(params) {
|
|
5507
4972
|
const normalized = normalizeResultsConfig(params.config);
|
|
5508
4973
|
const cloneDir = await ensureResultsRepoClone(normalized);
|
|
5509
|
-
await runGit(["push", normalized.remote, "--delete", params.wipBranch], {
|
|
4974
|
+
const result = await runGit(["push", normalized.remote, "--delete", params.wipBranch], {
|
|
4975
|
+
cwd: cloneDir,
|
|
4976
|
+
check: false
|
|
4977
|
+
});
|
|
4978
|
+
if (result.exitCode === 0 || isMissingRemoteRefDelete(result)) {
|
|
4979
|
+
return;
|
|
4980
|
+
}
|
|
4981
|
+
throw new Error(result.stderr.trim() || result.stdout.trim() || "Failed to delete WIP branch");
|
|
4982
|
+
}
|
|
4983
|
+
function isMissingRemoteRefDelete(result) {
|
|
4984
|
+
const text = `${result.stderr}
|
|
4985
|
+
${result.stdout}`.toLowerCase();
|
|
4986
|
+
return text.includes("remote ref does not exist");
|
|
5510
4987
|
}
|
|
5511
4988
|
function isMissingGitRefError(error) {
|
|
5512
4989
|
const parts = [];
|
|
@@ -5518,7 +4995,107 @@ function isMissingGitRefError(error) {
|
|
|
5518
4995
|
parts.push(error);
|
|
5519
4996
|
}
|
|
5520
4997
|
const haystack = parts.join("\n").toLowerCase();
|
|
5521
|
-
return haystack.includes("not a valid object name") || haystack.includes("unknown revision or path") || haystack.includes("bad revision") || haystack.includes("does not exist");
|
|
4998
|
+
return haystack.includes("not a valid object name") || haystack.includes("unknown revision or path") || haystack.includes("bad revision") || haystack.includes("does not exist") || haystack.includes("needed a single revision");
|
|
4999
|
+
}
|
|
5000
|
+
function gitResultsIndexHash(value) {
|
|
5001
|
+
return createHash("sha256").update(value).digest("hex");
|
|
5002
|
+
}
|
|
5003
|
+
function resolveGitResultsIndexRepoDir(repoDir) {
|
|
5004
|
+
const resolved = path3.resolve(repoDir);
|
|
5005
|
+
try {
|
|
5006
|
+
return realpathSync(resolved);
|
|
5007
|
+
} catch {
|
|
5008
|
+
return resolved;
|
|
5009
|
+
}
|
|
5010
|
+
}
|
|
5011
|
+
function gitResultsIndexRepoKey(repoDir) {
|
|
5012
|
+
return `sha256-${gitResultsIndexHash(resolveGitResultsIndexRepoDir(repoDir)).slice(0, 32)}`;
|
|
5013
|
+
}
|
|
5014
|
+
function gitResultsIndexRefKey(ref) {
|
|
5015
|
+
const slug = sanitizeRepoSlug(ref).slice(0, 80) || "default";
|
|
5016
|
+
return `${slug}-${gitResultsIndexHash(ref).slice(0, 16)}`;
|
|
5017
|
+
}
|
|
5018
|
+
function resolveGitResultsIndexCacheFile(params) {
|
|
5019
|
+
const ref = params.ref ?? "origin/main";
|
|
5020
|
+
return path3.join(
|
|
5021
|
+
getAgentvDataDir(),
|
|
5022
|
+
"cache",
|
|
5023
|
+
"results-index",
|
|
5024
|
+
gitResultsIndexRepoKey(params.repoDir),
|
|
5025
|
+
gitResultsIndexRefKey(ref),
|
|
5026
|
+
`${params.commitSha}.json`
|
|
5027
|
+
);
|
|
5028
|
+
}
|
|
5029
|
+
function isGitListedRun(value) {
|
|
5030
|
+
if (!value || typeof value !== "object") return false;
|
|
5031
|
+
const record = value;
|
|
5032
|
+
return typeof record.run_id === "string" && typeof record.experiment === "string" && typeof record.timestamp === "string" && (typeof record.index_path === "string" || typeof record.manifest_path === "string") && typeof record.display_name === "string" && typeof record.test_count === "number" && typeof record.avg_score === "number" && typeof record.size_bytes === "number" && (record.execution_error_count === void 0 || typeof record.execution_error_count === "number") && (record.pass_rate === void 0 || typeof record.pass_rate === "number") && (record.target === void 0 || typeof record.target === "string") && (record.summary_path === void 0 || typeof record.summary_path === "string");
|
|
5033
|
+
}
|
|
5034
|
+
function validateGitResultsIndexCacheFile(value, params) {
|
|
5035
|
+
if (!value || typeof value !== "object") return void 0;
|
|
5036
|
+
const record = value;
|
|
5037
|
+
const repoDir = resolveGitResultsIndexRepoDir(params.repoDir);
|
|
5038
|
+
const repoKey = gitResultsIndexRepoKey(params.repoDir);
|
|
5039
|
+
if (record.schema_version !== GIT_RESULTS_INDEX_CACHE_SCHEMA_VERSION || record.result_layout_version !== GIT_RESULTS_INDEX_LAYOUT_VERSION || record.repo_key !== repoKey || record.repo_dir !== repoDir || record.ref !== params.ref || record.commit_sha !== params.commitSha || !Array.isArray(record.runs) || !record.runs.every(isGitListedRun)) {
|
|
5040
|
+
return void 0;
|
|
5041
|
+
}
|
|
5042
|
+
return value;
|
|
5043
|
+
}
|
|
5044
|
+
async function readGitResultsIndexCache(params) {
|
|
5045
|
+
const cacheFile = resolveGitResultsIndexCacheFile(params);
|
|
5046
|
+
try {
|
|
5047
|
+
const parsed = JSON.parse(await readFile(cacheFile, "utf8"));
|
|
5048
|
+
return validateGitResultsIndexCacheFile(parsed, params)?.runs;
|
|
5049
|
+
} catch {
|
|
5050
|
+
return void 0;
|
|
5051
|
+
}
|
|
5052
|
+
}
|
|
5053
|
+
async function writeGitResultsIndexCache(params) {
|
|
5054
|
+
const cacheFile = resolveGitResultsIndexCacheFile(params);
|
|
5055
|
+
const envelope = {
|
|
5056
|
+
schema_version: GIT_RESULTS_INDEX_CACHE_SCHEMA_VERSION,
|
|
5057
|
+
result_layout_version: GIT_RESULTS_INDEX_LAYOUT_VERSION,
|
|
5058
|
+
repo_key: gitResultsIndexRepoKey(params.repoDir),
|
|
5059
|
+
repo_dir: resolveGitResultsIndexRepoDir(params.repoDir),
|
|
5060
|
+
ref: params.ref,
|
|
5061
|
+
commit_sha: params.commitSha,
|
|
5062
|
+
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5063
|
+
runs: params.runs
|
|
5064
|
+
};
|
|
5065
|
+
try {
|
|
5066
|
+
mkdirSync(path3.dirname(cacheFile), { recursive: true });
|
|
5067
|
+
const tempFile = `${cacheFile}.${process.pid}.${Date.now()}.tmp`;
|
|
5068
|
+
await writeFile(tempFile, `${JSON.stringify(envelope, null, 2)}
|
|
5069
|
+
`, "utf8");
|
|
5070
|
+
renameSync(tempFile, cacheFile);
|
|
5071
|
+
} catch {
|
|
5072
|
+
}
|
|
5073
|
+
}
|
|
5074
|
+
async function resolveGitRunsRefCommit(repoDir, ref = "origin/main") {
|
|
5075
|
+
try {
|
|
5076
|
+
const { stdout } = await runGit(["rev-parse", "--verify", `${ref}^{commit}`], {
|
|
5077
|
+
cwd: repoDir
|
|
5078
|
+
});
|
|
5079
|
+
return stdout.trim() || void 0;
|
|
5080
|
+
} catch (error) {
|
|
5081
|
+
if (isMissingGitRefError(error)) {
|
|
5082
|
+
return void 0;
|
|
5083
|
+
}
|
|
5084
|
+
throw error;
|
|
5085
|
+
}
|
|
5086
|
+
}
|
|
5087
|
+
async function listGitRunsCached(repoDir, ref = "origin/main") {
|
|
5088
|
+
const commitSha = await resolveGitRunsRefCommit(repoDir, ref);
|
|
5089
|
+
if (!commitSha) {
|
|
5090
|
+
return [];
|
|
5091
|
+
}
|
|
5092
|
+
const cached = await readGitResultsIndexCache({ repoDir, ref, commitSha });
|
|
5093
|
+
if (cached) {
|
|
5094
|
+
return [...cached];
|
|
5095
|
+
}
|
|
5096
|
+
const runs = await listGitRuns(repoDir, ref);
|
|
5097
|
+
await writeGitResultsIndexCache({ repoDir, ref, commitSha, runs });
|
|
5098
|
+
return runs;
|
|
5522
5099
|
}
|
|
5523
5100
|
async function listGitRuns(repoDir, ref = "origin/main") {
|
|
5524
5101
|
let treeOut;
|
|
@@ -5536,19 +5113,7 @@ async function listGitRuns(repoDir, ref = "origin/main") {
|
|
|
5536
5113
|
throw error;
|
|
5537
5114
|
}
|
|
5538
5115
|
const treePaths = treeOut.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
|
|
5539
|
-
const
|
|
5540
|
-
if (indexPaths.length === 0) {
|
|
5541
|
-
return [];
|
|
5542
|
-
}
|
|
5543
|
-
const batchInput = `${indexPaths.map((indexPath) => `${ref}:${indexPath}`).join("\n")}
|
|
5544
|
-
`;
|
|
5545
|
-
const blobs = parseGitBatchBlobs(await runGitBatch(repoDir, batchInput));
|
|
5546
|
-
if (blobs.length !== indexPaths.length) {
|
|
5547
|
-
throw new Error(
|
|
5548
|
-
`Expected ${indexPaths.length} git blobs but received ${blobs.length} while listing results runs`
|
|
5549
|
-
);
|
|
5550
|
-
}
|
|
5551
|
-
const summaryPaths = indexPaths.map((indexPath) => path5.posix.join(path5.posix.dirname(indexPath), "summary.json")).filter((summaryPath) => treePaths.includes(summaryPath));
|
|
5116
|
+
const summaryPaths = treePaths.filter((line) => line.endsWith("/summary.json"));
|
|
5552
5117
|
const summaryByPath = /* @__PURE__ */ new Map();
|
|
5553
5118
|
if (summaryPaths.length > 0) {
|
|
5554
5119
|
const summaryBatchInput = `${summaryPaths.map((summaryPath) => `${ref}:${summaryPath}`).join("\n")}
|
|
@@ -5561,12 +5126,24 @@ async function listGitRuns(repoDir, ref = "origin/main") {
|
|
|
5561
5126
|
summaryByPath.set(summaryPath, JSON.parse(blob.content.toString("utf8")));
|
|
5562
5127
|
}
|
|
5563
5128
|
}
|
|
5129
|
+
const indexPaths = buildGitManifestPaths(treePaths, summaryByPath);
|
|
5130
|
+
if (indexPaths.length === 0) {
|
|
5131
|
+
return [];
|
|
5132
|
+
}
|
|
5133
|
+
const batchInput = `${indexPaths.map((indexPath) => `${ref}:${indexPath}`).join("\n")}
|
|
5134
|
+
`;
|
|
5135
|
+
const blobs = parseGitBatchBlobs(await runGitBatch(repoDir, batchInput));
|
|
5136
|
+
if (blobs.length !== indexPaths.length) {
|
|
5137
|
+
throw new Error(
|
|
5138
|
+
`Expected ${indexPaths.length} git blobs but received ${blobs.length} while listing results runs`
|
|
5139
|
+
);
|
|
5140
|
+
}
|
|
5564
5141
|
const runs = blobs.flatMap((blob, index) => {
|
|
5565
5142
|
const manifestPath = indexPaths[index];
|
|
5566
|
-
const runDir =
|
|
5567
|
-
const summaryPath =
|
|
5143
|
+
const runDir = gitRunDirForManifestPath(manifestPath);
|
|
5144
|
+
const summaryPath = path3.posix.join(runDir, "summary.json");
|
|
5568
5145
|
const summary = summaryByPath.get(summaryPath);
|
|
5569
|
-
const relativeRunPath =
|
|
5146
|
+
const relativeRunPath = path3.posix.relative(RESULTS_REPO_RUNS_DIR, runDir);
|
|
5570
5147
|
const runId = buildGitRunId(relativeRunPath);
|
|
5571
5148
|
const rows = blob.content.toString("utf8").split(/\r?\n/).map((line) => line.trim()).filter(Boolean).flatMap(
|
|
5572
5149
|
(line) => {
|
|
@@ -5587,10 +5164,13 @@ async function listGitRuns(repoDir, ref = "origin/main") {
|
|
|
5587
5164
|
];
|
|
5588
5165
|
const rowScores = rows.map((row) => row.score).filter((score) => typeof score === "number" && Number.isFinite(score));
|
|
5589
5166
|
const avgScore = rowScores.length > 0 ? rowScores.reduce((sum, score) => sum + score, 0) / rowScores.length : 0;
|
|
5590
|
-
const timestamp = summary?.metadata?.timestamp?.trim() || rows[0]?.timestamp ||
|
|
5591
|
-
const displayName = summary?.metadata?.display_name?.trim() ||
|
|
5167
|
+
const timestamp = summary?.metadata?.timestamp?.trim() || rows[0]?.timestamp || path3.posix.basename(runDir);
|
|
5168
|
+
const displayName = summary?.metadata?.display_name?.trim() || path3.posix.basename(runDir);
|
|
5592
5169
|
const targets = summary?.metadata?.targets ?? rowTargets;
|
|
5593
5170
|
const passRate = computeAveragePassRate(summary?.run_summary) ?? avgScore;
|
|
5171
|
+
const executionErrorCount = rows.filter(
|
|
5172
|
+
(row) => row.execution_status === "execution_error"
|
|
5173
|
+
).length;
|
|
5594
5174
|
return [
|
|
5595
5175
|
{
|
|
5596
5176
|
run_id: runId,
|
|
@@ -5598,10 +5178,11 @@ async function listGitRuns(repoDir, ref = "origin/main") {
|
|
|
5598
5178
|
timestamp,
|
|
5599
5179
|
...passRate !== void 0 && { pass_rate: passRate },
|
|
5600
5180
|
...targets.length === 1 && targets[0] ? { target: targets[0] } : {},
|
|
5601
|
-
|
|
5181
|
+
index_path: manifestPath,
|
|
5602
5182
|
...summaryByPath.has(summaryPath) && { summary_path: summaryPath },
|
|
5603
5183
|
display_name: displayName,
|
|
5604
5184
|
test_count: summary?.metadata?.tests_run?.length ?? rowTestIds.length,
|
|
5185
|
+
...executionErrorCount > 0 && { execution_error_count: executionErrorCount },
|
|
5605
5186
|
avg_score: avgScore,
|
|
5606
5187
|
size_bytes: blob.size
|
|
5607
5188
|
}
|
|
@@ -5611,9 +5192,9 @@ async function listGitRuns(repoDir, ref = "origin/main") {
|
|
|
5611
5192
|
return runs;
|
|
5612
5193
|
}
|
|
5613
5194
|
async function materializeGitRun(repoDir, relativeRunPath, ref = "origin/main") {
|
|
5614
|
-
const normalizedRunPath = relativeRunPath.split(
|
|
5615
|
-
const runTreePath =
|
|
5616
|
-
const targetRunDir =
|
|
5195
|
+
const normalizedRunPath = relativeRunPath.split(path3.sep).join("/");
|
|
5196
|
+
const runTreePath = path3.posix.join(RESULTS_REPO_RUNS_DIR, normalizedRunPath);
|
|
5197
|
+
const targetRunDir = path3.join(repoDir, ...runTreePath.split("/"));
|
|
5617
5198
|
const { stdout: treeOut } = await runGit(["ls-tree", "-r", "--name-only", ref, runTreePath], {
|
|
5618
5199
|
cwd: repoDir
|
|
5619
5200
|
});
|
|
@@ -5629,16 +5210,16 @@ async function materializeGitRun(repoDir, relativeRunPath, ref = "origin/main")
|
|
|
5629
5210
|
`Expected ${filePaths.length} git blobs but received ${blobs.length} while materializing results run`
|
|
5630
5211
|
);
|
|
5631
5212
|
}
|
|
5632
|
-
const tempRoot = mkdtempSync(
|
|
5633
|
-
const tempRunDir =
|
|
5213
|
+
const tempRoot = mkdtempSync(path3.join(repoDir, ".agentv-run-"));
|
|
5214
|
+
const tempRunDir = path3.join(tempRoot, "run");
|
|
5634
5215
|
try {
|
|
5635
5216
|
for (const [index, filePath] of filePaths.entries()) {
|
|
5636
|
-
const relativeFilePath =
|
|
5637
|
-
const absolutePath =
|
|
5638
|
-
mkdirSync(
|
|
5217
|
+
const relativeFilePath = path3.posix.relative(runTreePath, filePath);
|
|
5218
|
+
const absolutePath = path3.join(tempRunDir, ...relativeFilePath.split("/"));
|
|
5219
|
+
mkdirSync(path3.dirname(absolutePath), { recursive: true });
|
|
5639
5220
|
writeFileSync(absolutePath, blobs[index].content);
|
|
5640
5221
|
}
|
|
5641
|
-
mkdirSync(
|
|
5222
|
+
mkdirSync(path3.dirname(targetRunDir), { recursive: true });
|
|
5642
5223
|
try {
|
|
5643
5224
|
renameSync(tempRunDir, targetRunDir);
|
|
5644
5225
|
} catch (error) {
|
|
@@ -5720,483 +5301,6 @@ function trimBaselineResult(result) {
|
|
|
5720
5301
|
}
|
|
5721
5302
|
return trimmed;
|
|
5722
5303
|
}
|
|
5723
|
-
var DEFAULT_CATEGORY = "Uncategorized";
|
|
5724
|
-
function deriveCategory(relativePath) {
|
|
5725
|
-
const parts = relativePath.split(/[/\\]/);
|
|
5726
|
-
if (parts.length <= 1) {
|
|
5727
|
-
return DEFAULT_CATEGORY;
|
|
5728
|
-
}
|
|
5729
|
-
const dirs = parts.slice(0, -1).filter((d) => d !== "evals");
|
|
5730
|
-
return dirs.length > 0 ? dirs.join("/") : DEFAULT_CATEGORY;
|
|
5731
|
-
}
|
|
5732
|
-
var OtelTraceExporter = class {
|
|
5733
|
-
constructor(options) {
|
|
5734
|
-
this.options = options;
|
|
5735
|
-
}
|
|
5736
|
-
provider = null;
|
|
5737
|
-
tracer = null;
|
|
5738
|
-
api = null;
|
|
5739
|
-
// biome-ignore lint/suspicious/noExplicitAny: OTel types loaded dynamically
|
|
5740
|
-
W3CPropagator = null;
|
|
5741
|
-
/** Initialize the OTel SDK. Returns false if OTel packages are not available. */
|
|
5742
|
-
async init() {
|
|
5743
|
-
try {
|
|
5744
|
-
const [sdkTraceNode, resourcesMod, semconvMod, api, coreMod] = await Promise.all([
|
|
5745
|
-
import("./src-76ISS3LH.js"),
|
|
5746
|
-
import("./esm-NNMQYIKT.js"),
|
|
5747
|
-
import("./esm-RVQPUGWH.js"),
|
|
5748
|
-
import("./esm-R77SNOF5.js"),
|
|
5749
|
-
import("./esm-ZADQ4XQH-QUFGGZL4.js").catch(() => null)
|
|
5750
|
-
]);
|
|
5751
|
-
const { NodeTracerProvider: Provider, SimpleSpanProcessor } = sdkTraceNode;
|
|
5752
|
-
const { resourceFromAttributes } = resourcesMod;
|
|
5753
|
-
const { ATTR_SERVICE_NAME } = semconvMod;
|
|
5754
|
-
const resourceAttributes = {
|
|
5755
|
-
[ATTR_SERVICE_NAME]: this.options.serviceName ?? "agentv",
|
|
5756
|
-
...this.options.resourceAttributes
|
|
5757
|
-
};
|
|
5758
|
-
const resource = resourceFromAttributes(resourceAttributes);
|
|
5759
|
-
const processors = [];
|
|
5760
|
-
if (this.options.endpoint) {
|
|
5761
|
-
const otlpHttp = await import("./esm-TTEMJEVV.js");
|
|
5762
|
-
const { OTLPTraceExporter } = otlpHttp;
|
|
5763
|
-
const exporter = new OTLPTraceExporter({
|
|
5764
|
-
url: this.options.endpoint,
|
|
5765
|
-
headers: this.options.headers
|
|
5766
|
-
});
|
|
5767
|
-
processors.push(new SimpleSpanProcessor(exporter));
|
|
5768
|
-
}
|
|
5769
|
-
if (this.options.otlpFilePath) {
|
|
5770
|
-
const { OtlpJsonFileExporter: OtlpJsonFileExporter2 } = await import("./otlp-json-file-exporter-RY63S3IG-PZBQPVYY.js");
|
|
5771
|
-
processors.push(
|
|
5772
|
-
new SimpleSpanProcessor(
|
|
5773
|
-
new OtlpJsonFileExporter2(this.options.otlpFilePath, resourceAttributes)
|
|
5774
|
-
)
|
|
5775
|
-
);
|
|
5776
|
-
}
|
|
5777
|
-
if (processors.length === 0) {
|
|
5778
|
-
return false;
|
|
5779
|
-
}
|
|
5780
|
-
this.provider = new Provider({
|
|
5781
|
-
resource,
|
|
5782
|
-
spanProcessors: processors
|
|
5783
|
-
});
|
|
5784
|
-
this.provider.register();
|
|
5785
|
-
this.api = api;
|
|
5786
|
-
this.tracer = api.trace.getTracer("agentv", "1.0.0");
|
|
5787
|
-
this.W3CPropagator = coreMod?.W3CTraceContextPropagator ?? null;
|
|
5788
|
-
return true;
|
|
5789
|
-
} catch {
|
|
5790
|
-
return false;
|
|
5791
|
-
}
|
|
5792
|
-
}
|
|
5793
|
-
/** Export a single evaluation result as an OTel trace. */
|
|
5794
|
-
async exportResult(result) {
|
|
5795
|
-
if (!this.tracer || !this.api) return;
|
|
5796
|
-
const api = this.api;
|
|
5797
|
-
const tracer = this.tracer;
|
|
5798
|
-
const captureContent = this.options.captureContent ?? false;
|
|
5799
|
-
const startHr = toHrTime(result.startTime ?? result.timestamp);
|
|
5800
|
-
const endHr = toHrTime(result.endTime ?? result.timestamp);
|
|
5801
|
-
let parentCtx = api.ROOT_CONTEXT;
|
|
5802
|
-
const traceparent = process.env.TRACEPARENT;
|
|
5803
|
-
if (traceparent && this.W3CPropagator) {
|
|
5804
|
-
try {
|
|
5805
|
-
const propagator = new this.W3CPropagator();
|
|
5806
|
-
parentCtx = propagator.extract(
|
|
5807
|
-
api.ROOT_CONTEXT,
|
|
5808
|
-
{ traceparent, tracestate: process.env.TRACESTATE ?? "" },
|
|
5809
|
-
{
|
|
5810
|
-
get: (carrier, key) => carrier[key],
|
|
5811
|
-
keys: (carrier) => Object.keys(carrier)
|
|
5812
|
-
}
|
|
5813
|
-
);
|
|
5814
|
-
} catch {
|
|
5815
|
-
}
|
|
5816
|
-
}
|
|
5817
|
-
tracer.startActiveSpan(
|
|
5818
|
-
"agentv.eval",
|
|
5819
|
-
{ startTime: startHr },
|
|
5820
|
-
parentCtx,
|
|
5821
|
-
(rootSpan) => {
|
|
5822
|
-
rootSpan.setAttribute("gen_ai.operation.name", "evaluate");
|
|
5823
|
-
rootSpan.setAttribute("gen_ai.system", "agentv");
|
|
5824
|
-
rootSpan.setAttribute("agentv.test_id", result.testId);
|
|
5825
|
-
rootSpan.setAttribute("agentv.target", result.target);
|
|
5826
|
-
if (result.suite) rootSpan.setAttribute("agentv.suite", result.suite);
|
|
5827
|
-
rootSpan.setAttribute("agentv.score", result.score);
|
|
5828
|
-
if (captureContent && result.output.length > 0) {
|
|
5829
|
-
rootSpan.setAttribute("agentv.output_text", result.output);
|
|
5830
|
-
}
|
|
5831
|
-
if (result.durationMs != null)
|
|
5832
|
-
rootSpan.setAttribute("agentv.trace.duration_ms", result.durationMs);
|
|
5833
|
-
if (result.costUsd != null) rootSpan.setAttribute("agentv.trace.cost_usd", result.costUsd);
|
|
5834
|
-
if (result.tokenUsage) {
|
|
5835
|
-
if (result.tokenUsage.input != null) {
|
|
5836
|
-
rootSpan.setAttribute("agentv.trace.token_input", result.tokenUsage.input);
|
|
5837
|
-
}
|
|
5838
|
-
if (result.tokenUsage.output != null) {
|
|
5839
|
-
rootSpan.setAttribute("agentv.trace.token_output", result.tokenUsage.output);
|
|
5840
|
-
}
|
|
5841
|
-
if (result.tokenUsage.cached != null) {
|
|
5842
|
-
rootSpan.setAttribute("agentv.trace.token_cached", result.tokenUsage.cached);
|
|
5843
|
-
}
|
|
5844
|
-
}
|
|
5845
|
-
if (result.trace) {
|
|
5846
|
-
const t = result.trace;
|
|
5847
|
-
rootSpan.setAttribute("agentv.trace.event_count", t.eventCount);
|
|
5848
|
-
rootSpan.setAttribute(
|
|
5849
|
-
"agentv.trace.tool_names",
|
|
5850
|
-
Object.keys(t.toolCalls).sort().join(",")
|
|
5851
|
-
);
|
|
5852
|
-
if (t.llmCallCount != null)
|
|
5853
|
-
rootSpan.setAttribute("agentv.trace.llm_call_count", t.llmCallCount);
|
|
5854
|
-
}
|
|
5855
|
-
const traceMessages = result.trace?.messages ?? [];
|
|
5856
|
-
if (traceMessages.length > 0) {
|
|
5857
|
-
const parentCtx2 = api.trace.setSpan(api.context.active(), rootSpan);
|
|
5858
|
-
if (this.options.groupTurns) {
|
|
5859
|
-
const turns = groupMessagesIntoTurns(traceMessages);
|
|
5860
|
-
if (turns.length > 1) {
|
|
5861
|
-
for (const [i, turn] of turns.entries()) {
|
|
5862
|
-
api.context.with(parentCtx2, () => {
|
|
5863
|
-
tracer.startActiveSpan(
|
|
5864
|
-
`agentv.turn.${i + 1}`,
|
|
5865
|
-
{},
|
|
5866
|
-
(turnSpan) => {
|
|
5867
|
-
const turnCtx = api.trace.setSpan(api.context.active(), turnSpan);
|
|
5868
|
-
for (const msg of turn.messages) {
|
|
5869
|
-
this.exportMessage(tracer, api, turnCtx, msg, captureContent);
|
|
5870
|
-
}
|
|
5871
|
-
turnSpan.end();
|
|
5872
|
-
}
|
|
5873
|
-
);
|
|
5874
|
-
});
|
|
5875
|
-
}
|
|
5876
|
-
} else {
|
|
5877
|
-
for (const msg of traceMessages) {
|
|
5878
|
-
this.exportMessage(tracer, api, parentCtx2, msg, captureContent);
|
|
5879
|
-
}
|
|
5880
|
-
}
|
|
5881
|
-
} else {
|
|
5882
|
-
for (const msg of traceMessages) {
|
|
5883
|
-
this.exportMessage(tracer, api, parentCtx2, msg, captureContent);
|
|
5884
|
-
}
|
|
5885
|
-
}
|
|
5886
|
-
}
|
|
5887
|
-
if (result.scores) {
|
|
5888
|
-
for (const score of result.scores) {
|
|
5889
|
-
rootSpan.addEvent(`agentv.grader.${score.name}`, {
|
|
5890
|
-
"agentv.grader.score": score.score,
|
|
5891
|
-
"agentv.grader.type": score.type,
|
|
5892
|
-
...score.verdict ? { "agentv.grader.verdict": score.verdict } : {}
|
|
5893
|
-
});
|
|
5894
|
-
}
|
|
5895
|
-
}
|
|
5896
|
-
if (result.error) {
|
|
5897
|
-
rootSpan.setStatus({ code: api.SpanStatusCode.ERROR, message: result.error });
|
|
5898
|
-
} else {
|
|
5899
|
-
rootSpan.setStatus({ code: api.SpanStatusCode.OK });
|
|
5900
|
-
}
|
|
5901
|
-
rootSpan.end(endHr);
|
|
5902
|
-
}
|
|
5903
|
-
);
|
|
5904
|
-
}
|
|
5905
|
-
/** Flush pending spans and shut down. */
|
|
5906
|
-
async shutdown() {
|
|
5907
|
-
await this.provider?.shutdown();
|
|
5908
|
-
}
|
|
5909
|
-
/** Create a streaming observer for real-time span export */
|
|
5910
|
-
createStreamingObserver() {
|
|
5911
|
-
if (!this.tracer || !this.api) return null;
|
|
5912
|
-
let parentCtx;
|
|
5913
|
-
const traceparent = process.env.TRACEPARENT;
|
|
5914
|
-
if (traceparent && this.W3CPropagator) {
|
|
5915
|
-
try {
|
|
5916
|
-
const propagator = new this.W3CPropagator();
|
|
5917
|
-
parentCtx = propagator.extract(
|
|
5918
|
-
this.api.ROOT_CONTEXT,
|
|
5919
|
-
{ traceparent, tracestate: process.env.TRACESTATE ?? "" },
|
|
5920
|
-
{
|
|
5921
|
-
get: (carrier, key) => carrier[key],
|
|
5922
|
-
keys: (carrier) => Object.keys(carrier)
|
|
5923
|
-
}
|
|
5924
|
-
);
|
|
5925
|
-
} catch {
|
|
5926
|
-
}
|
|
5927
|
-
}
|
|
5928
|
-
return new OtelStreamingObserver(
|
|
5929
|
-
this.tracer,
|
|
5930
|
-
this.api,
|
|
5931
|
-
this.options.captureContent ?? false,
|
|
5932
|
-
parentCtx
|
|
5933
|
-
);
|
|
5934
|
-
}
|
|
5935
|
-
// -----------------------------------------------------------------------
|
|
5936
|
-
// Private helpers
|
|
5937
|
-
// -----------------------------------------------------------------------
|
|
5938
|
-
exportMessage(tracer, api, parentCtx, msg, captureContent) {
|
|
5939
|
-
const isAssistant = msg.role === "assistant";
|
|
5940
|
-
const model = msg.metadata?.model ? String(msg.metadata.model) : void 0;
|
|
5941
|
-
const spanName = isAssistant ? `chat ${model ?? "unknown"}` : `gen_ai.message.${msg.role}`;
|
|
5942
|
-
const startHr = toHrTime(msg.startTime);
|
|
5943
|
-
const endHr = toHrTime(msg.endTime);
|
|
5944
|
-
api.context.with(parentCtx, () => {
|
|
5945
|
-
tracer.startActiveSpan(
|
|
5946
|
-
spanName,
|
|
5947
|
-
{ startTime: startHr },
|
|
5948
|
-
parentCtx,
|
|
5949
|
-
(span) => {
|
|
5950
|
-
if (isAssistant) {
|
|
5951
|
-
span.setAttribute("gen_ai.operation.name", "chat");
|
|
5952
|
-
}
|
|
5953
|
-
if (model) {
|
|
5954
|
-
span.setAttribute("gen_ai.request.model", model);
|
|
5955
|
-
span.setAttribute("gen_ai.response.model", model);
|
|
5956
|
-
}
|
|
5957
|
-
if (msg.tokenUsage) {
|
|
5958
|
-
if (msg.tokenUsage.input != null) {
|
|
5959
|
-
span.setAttribute("gen_ai.usage.input_tokens", msg.tokenUsage.input);
|
|
5960
|
-
}
|
|
5961
|
-
if (msg.tokenUsage.output != null) {
|
|
5962
|
-
span.setAttribute("gen_ai.usage.output_tokens", msg.tokenUsage.output);
|
|
5963
|
-
}
|
|
5964
|
-
if (msg.tokenUsage.cached != null) {
|
|
5965
|
-
span.setAttribute("gen_ai.usage.cache_read.input_tokens", msg.tokenUsage.cached);
|
|
5966
|
-
}
|
|
5967
|
-
}
|
|
5968
|
-
if (captureContent && msg.content != null) {
|
|
5969
|
-
span.setAttribute(
|
|
5970
|
-
"gen_ai.output.messages",
|
|
5971
|
-
typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
|
|
5972
|
-
);
|
|
5973
|
-
}
|
|
5974
|
-
if (msg.toolCalls) {
|
|
5975
|
-
const msgCtx = api.trace.setSpan(api.context.active(), span);
|
|
5976
|
-
for (const tc of msg.toolCalls) {
|
|
5977
|
-
api.context.with(msgCtx, () => {
|
|
5978
|
-
tracer.startActiveSpan(
|
|
5979
|
-
`execute_tool ${tc.tool}`,
|
|
5980
|
-
{},
|
|
5981
|
-
msgCtx,
|
|
5982
|
-
(toolSpan) => {
|
|
5983
|
-
toolSpan.setAttribute("gen_ai.tool.name", tc.tool);
|
|
5984
|
-
if (tc.id) toolSpan.setAttribute("gen_ai.tool.call.id", tc.id);
|
|
5985
|
-
if (captureContent) {
|
|
5986
|
-
if (tc.input != null) {
|
|
5987
|
-
toolSpan.setAttribute(
|
|
5988
|
-
"gen_ai.tool.call.arguments",
|
|
5989
|
-
typeof tc.input === "string" ? tc.input : JSON.stringify(tc.input)
|
|
5990
|
-
);
|
|
5991
|
-
}
|
|
5992
|
-
if (tc.output != null) {
|
|
5993
|
-
toolSpan.setAttribute(
|
|
5994
|
-
"gen_ai.tool.call.result",
|
|
5995
|
-
typeof tc.output === "string" ? tc.output : JSON.stringify(tc.output)
|
|
5996
|
-
);
|
|
5997
|
-
}
|
|
5998
|
-
}
|
|
5999
|
-
toolSpan.end();
|
|
6000
|
-
}
|
|
6001
|
-
);
|
|
6002
|
-
});
|
|
6003
|
-
}
|
|
6004
|
-
}
|
|
6005
|
-
span.end(endHr);
|
|
6006
|
-
}
|
|
6007
|
-
);
|
|
6008
|
-
});
|
|
6009
|
-
}
|
|
6010
|
-
};
|
|
6011
|
-
var OtelStreamingObserver = class {
|
|
6012
|
-
constructor(tracer, api, captureContent, parentCtx) {
|
|
6013
|
-
this.tracer = tracer;
|
|
6014
|
-
this.api = api;
|
|
6015
|
-
this.captureContent = captureContent;
|
|
6016
|
-
this.parentCtx = parentCtx;
|
|
6017
|
-
}
|
|
6018
|
-
// biome-ignore lint/suspicious/noExplicitAny: OTel span type loaded dynamically
|
|
6019
|
-
rootSpan = null;
|
|
6020
|
-
// biome-ignore lint/suspicious/noExplicitAny: OTel context loaded dynamically
|
|
6021
|
-
rootCtx = null;
|
|
6022
|
-
observedChildSpans = false;
|
|
6023
|
-
pendingMetrics = null;
|
|
6024
|
-
/** Create root eval span immediately (visible in backend right away) */
|
|
6025
|
-
startEvalCase(testId, target, evalSet) {
|
|
6026
|
-
this.pendingMetrics = null;
|
|
6027
|
-
this.observedChildSpans = false;
|
|
6028
|
-
const ctx = this.parentCtx ?? this.api.context.active();
|
|
6029
|
-
this.rootSpan = this.tracer.startSpan("agentv.eval", void 0, ctx);
|
|
6030
|
-
this.rootSpan.setAttribute("gen_ai.operation.name", "evaluate");
|
|
6031
|
-
this.rootSpan.setAttribute("gen_ai.system", "agentv");
|
|
6032
|
-
this.rootSpan.setAttribute("agentv.test_id", testId);
|
|
6033
|
-
this.rootSpan.setAttribute("agentv.target", target);
|
|
6034
|
-
if (evalSet) this.rootSpan.setAttribute("agentv.suite", evalSet);
|
|
6035
|
-
this.rootCtx = this.api.trace.setSpan(this.api.context.active(), this.rootSpan);
|
|
6036
|
-
}
|
|
6037
|
-
/** Create and immediately export a tool span */
|
|
6038
|
-
onToolCall(name, input, output, _durationMs, toolCallId) {
|
|
6039
|
-
if (!this.rootCtx) return;
|
|
6040
|
-
this.observedChildSpans = true;
|
|
6041
|
-
this.api.context.with(this.rootCtx, () => {
|
|
6042
|
-
const span = this.tracer.startSpan(`execute_tool ${name}`, void 0, this.rootCtx);
|
|
6043
|
-
span.setAttribute("gen_ai.tool.name", name);
|
|
6044
|
-
if (toolCallId) span.setAttribute("gen_ai.tool.call.id", toolCallId);
|
|
6045
|
-
if (this.captureContent) {
|
|
6046
|
-
if (input != null)
|
|
6047
|
-
span.setAttribute(
|
|
6048
|
-
"gen_ai.tool.call.arguments",
|
|
6049
|
-
typeof input === "string" ? input : JSON.stringify(input)
|
|
6050
|
-
);
|
|
6051
|
-
if (output != null)
|
|
6052
|
-
span.setAttribute(
|
|
6053
|
-
"gen_ai.tool.call.result",
|
|
6054
|
-
typeof output === "string" ? output : JSON.stringify(output)
|
|
6055
|
-
);
|
|
6056
|
-
}
|
|
6057
|
-
span.end();
|
|
6058
|
-
});
|
|
6059
|
-
}
|
|
6060
|
-
/** Create and immediately export an LLM span */
|
|
6061
|
-
onLlmCall(model, tokenUsage) {
|
|
6062
|
-
if (!this.rootCtx) return;
|
|
6063
|
-
this.observedChildSpans = true;
|
|
6064
|
-
this.api.context.with(this.rootCtx, () => {
|
|
6065
|
-
const span = this.tracer.startSpan(`chat ${model}`, void 0, this.rootCtx);
|
|
6066
|
-
span.setAttribute("gen_ai.operation.name", "chat");
|
|
6067
|
-
span.setAttribute("gen_ai.request.model", model);
|
|
6068
|
-
span.setAttribute("gen_ai.response.model", model);
|
|
6069
|
-
if (tokenUsage) {
|
|
6070
|
-
if (tokenUsage.input != null)
|
|
6071
|
-
span.setAttribute("gen_ai.usage.input_tokens", tokenUsage.input);
|
|
6072
|
-
if (tokenUsage.output != null)
|
|
6073
|
-
span.setAttribute("gen_ai.usage.output_tokens", tokenUsage.output);
|
|
6074
|
-
if (tokenUsage.cached != null)
|
|
6075
|
-
span.setAttribute("gen_ai.usage.cache_read.input_tokens", tokenUsage.cached);
|
|
6076
|
-
}
|
|
6077
|
-
span.end();
|
|
6078
|
-
});
|
|
6079
|
-
}
|
|
6080
|
-
/** Record final execution metrics before the root span is finalized. */
|
|
6081
|
-
recordEvalMetrics(result) {
|
|
6082
|
-
this.pendingMetrics = result;
|
|
6083
|
-
}
|
|
6084
|
-
/** Finalize root span with score/verdict after evaluation completes */
|
|
6085
|
-
finalizeEvalCase(score, error) {
|
|
6086
|
-
if (!this.rootSpan) return;
|
|
6087
|
-
this.rootSpan.setAttribute("agentv.score", score);
|
|
6088
|
-
if (this.pendingMetrics?.durationMs != null) {
|
|
6089
|
-
this.rootSpan.setAttribute("agentv.trace.duration_ms", this.pendingMetrics.durationMs);
|
|
6090
|
-
}
|
|
6091
|
-
if (this.pendingMetrics?.costUsd != null) {
|
|
6092
|
-
this.rootSpan.setAttribute("agentv.trace.cost_usd", this.pendingMetrics.costUsd);
|
|
6093
|
-
}
|
|
6094
|
-
if (this.pendingMetrics?.tokenUsage) {
|
|
6095
|
-
if (this.pendingMetrics.tokenUsage.input != null) {
|
|
6096
|
-
this.rootSpan.setAttribute(
|
|
6097
|
-
"agentv.trace.token_input",
|
|
6098
|
-
this.pendingMetrics.tokenUsage.input
|
|
6099
|
-
);
|
|
6100
|
-
}
|
|
6101
|
-
if (this.pendingMetrics.tokenUsage.output != null) {
|
|
6102
|
-
this.rootSpan.setAttribute(
|
|
6103
|
-
"agentv.trace.token_output",
|
|
6104
|
-
this.pendingMetrics.tokenUsage.output
|
|
6105
|
-
);
|
|
6106
|
-
}
|
|
6107
|
-
if (this.pendingMetrics.tokenUsage.cached != null) {
|
|
6108
|
-
this.rootSpan.setAttribute(
|
|
6109
|
-
"agentv.trace.token_cached",
|
|
6110
|
-
this.pendingMetrics.tokenUsage.cached
|
|
6111
|
-
);
|
|
6112
|
-
}
|
|
6113
|
-
}
|
|
6114
|
-
if (this.pendingMetrics?.trace) {
|
|
6115
|
-
this.rootSpan.setAttribute("agentv.trace.event_count", this.pendingMetrics.trace.eventCount);
|
|
6116
|
-
this.rootSpan.setAttribute(
|
|
6117
|
-
"agentv.trace.tool_names",
|
|
6118
|
-
Object.keys(this.pendingMetrics.trace.toolCalls).sort().join(",")
|
|
6119
|
-
);
|
|
6120
|
-
if (this.pendingMetrics.trace.llmCallCount != null) {
|
|
6121
|
-
this.rootSpan.setAttribute(
|
|
6122
|
-
"agentv.trace.llm_call_count",
|
|
6123
|
-
this.pendingMetrics.trace.llmCallCount
|
|
6124
|
-
);
|
|
6125
|
-
}
|
|
6126
|
-
}
|
|
6127
|
-
if (error) {
|
|
6128
|
-
this.rootSpan.setStatus({ code: this.api.SpanStatusCode.ERROR, message: error });
|
|
6129
|
-
} else {
|
|
6130
|
-
this.rootSpan.setStatus({ code: this.api.SpanStatusCode.OK });
|
|
6131
|
-
}
|
|
6132
|
-
this.rootSpan.end();
|
|
6133
|
-
this.rootSpan = null;
|
|
6134
|
-
this.rootCtx = null;
|
|
6135
|
-
this.observedChildSpans = false;
|
|
6136
|
-
this.pendingMetrics = null;
|
|
6137
|
-
}
|
|
6138
|
-
/** Backfill child spans from the completed result when the provider emitted no live callbacks. */
|
|
6139
|
-
completeFromResult(result) {
|
|
6140
|
-
this.recordEvalMetrics({
|
|
6141
|
-
durationMs: result.durationMs,
|
|
6142
|
-
costUsd: result.costUsd,
|
|
6143
|
-
tokenUsage: result.tokenUsage,
|
|
6144
|
-
trace: result.trace
|
|
6145
|
-
});
|
|
6146
|
-
if (this.observedChildSpans || !this.rootCtx) {
|
|
6147
|
-
return;
|
|
6148
|
-
}
|
|
6149
|
-
const model = result.trace.messages.find((msg) => msg.role === "assistant")?.metadata?.model ?? result.target ?? "unknown";
|
|
6150
|
-
this.onLlmCall(String(model), result.tokenUsage);
|
|
6151
|
-
for (const message of result.trace.messages) {
|
|
6152
|
-
for (const toolCall of message.toolCalls ?? []) {
|
|
6153
|
-
this.onToolCall(
|
|
6154
|
-
toolCall.tool,
|
|
6155
|
-
toolCall.input,
|
|
6156
|
-
toolCall.output,
|
|
6157
|
-
toolCall.durationMs ?? 0,
|
|
6158
|
-
toolCall.id
|
|
6159
|
-
);
|
|
6160
|
-
}
|
|
6161
|
-
}
|
|
6162
|
-
}
|
|
6163
|
-
/** Return the active eval span's trace ID and span ID for Braintrust trace bridging */
|
|
6164
|
-
getActiveSpanIds() {
|
|
6165
|
-
if (!this.rootSpan) return null;
|
|
6166
|
-
try {
|
|
6167
|
-
const spanCtx = this.rootSpan.spanContext?.() ?? this.rootSpan._spanContext;
|
|
6168
|
-
if (!spanCtx?.traceId || !spanCtx?.spanId) return null;
|
|
6169
|
-
return { parentSpanId: spanCtx.spanId, rootSpanId: spanCtx.traceId };
|
|
6170
|
-
} catch {
|
|
6171
|
-
return null;
|
|
6172
|
-
}
|
|
6173
|
-
}
|
|
6174
|
-
/** Get ProviderStreamCallbacks for passing to providers */
|
|
6175
|
-
getStreamCallbacks() {
|
|
6176
|
-
return {
|
|
6177
|
-
onToolCallEnd: (name, input, output, durationMs, toolCallId) => this.onToolCall(name, input, output, durationMs, toolCallId),
|
|
6178
|
-
onLlmCallEnd: (model, tokenUsage) => this.onLlmCall(model, tokenUsage),
|
|
6179
|
-
getActiveSpanIds: () => this.getActiveSpanIds()
|
|
6180
|
-
};
|
|
6181
|
-
}
|
|
6182
|
-
};
|
|
6183
|
-
function groupMessagesIntoTurns(messages) {
|
|
6184
|
-
const turns = [];
|
|
6185
|
-
let current = [];
|
|
6186
|
-
for (const msg of messages) {
|
|
6187
|
-
if (msg.role === "user" && current.length > 0) {
|
|
6188
|
-
turns.push({ messages: current });
|
|
6189
|
-
current = [];
|
|
6190
|
-
}
|
|
6191
|
-
current.push(msg);
|
|
6192
|
-
}
|
|
6193
|
-
if (current.length > 0) turns.push({ messages: current });
|
|
6194
|
-
return turns;
|
|
6195
|
-
}
|
|
6196
|
-
function toHrTime(iso) {
|
|
6197
|
-
if (!iso) return void 0;
|
|
6198
|
-
return new Date(iso).getTime();
|
|
6199
|
-
}
|
|
6200
5304
|
var RunBudgetTracker = class {
|
|
6201
5305
|
constructor(capUsd) {
|
|
6202
5306
|
this.capUsd = capUsd;
|
|
@@ -6376,7 +5480,7 @@ function parseClaudeSession(jsonl) {
|
|
|
6376
5480
|
durationMs = new Date(endTimestamp).getTime() - new Date(startTimestamp).getTime();
|
|
6377
5481
|
}
|
|
6378
5482
|
const source = {
|
|
6379
|
-
provider: "claude",
|
|
5483
|
+
provider: "claude-sdk",
|
|
6380
5484
|
sessionId,
|
|
6381
5485
|
projectPath,
|
|
6382
5486
|
startedAt: startTimestamp,
|
|
@@ -6426,7 +5530,7 @@ function extractAssistantContent(content) {
|
|
|
6426
5530
|
case "tool_use":
|
|
6427
5531
|
if (block.name) {
|
|
6428
5532
|
toolCalls.push(
|
|
6429
|
-
normalizeToolCall("claude", {
|
|
5533
|
+
normalizeToolCall("claude-sdk", {
|
|
6430
5534
|
tool: block.name,
|
|
6431
5535
|
input: block.input,
|
|
6432
5536
|
id: block.id
|
|
@@ -6521,7 +5625,7 @@ function parseCodexSession(jsonl) {
|
|
|
6521
5625
|
} else {
|
|
6522
5626
|
input = payload.arguments;
|
|
6523
5627
|
}
|
|
6524
|
-
const toolCall = normalizeToolCall("codex", {
|
|
5628
|
+
const toolCall = normalizeToolCall("codex-cli", {
|
|
6525
5629
|
tool: toolName,
|
|
6526
5630
|
input,
|
|
6527
5631
|
id: callId
|
|
@@ -6549,7 +5653,7 @@ function parseCodexSession(jsonl) {
|
|
|
6549
5653
|
} else {
|
|
6550
5654
|
input = payload.arguments;
|
|
6551
5655
|
}
|
|
6552
|
-
const toolCall = normalizeToolCall("codex", {
|
|
5656
|
+
const toolCall = normalizeToolCall("codex-cli", {
|
|
6553
5657
|
tool: toolName,
|
|
6554
5658
|
input,
|
|
6555
5659
|
id: callId
|
|
@@ -6623,7 +5727,7 @@ function extractResponseItemContent(content) {
|
|
|
6623
5727
|
}
|
|
6624
5728
|
return parts.length > 0 ? parts.join("") : void 0;
|
|
6625
5729
|
}
|
|
6626
|
-
var DEFAULT_SESSIONS_DIR = () =>
|
|
5730
|
+
var DEFAULT_SESSIONS_DIR = () => path4.join(homedir(), ".codex", "sessions");
|
|
6627
5731
|
async function discoverCodexSessions(opts) {
|
|
6628
5732
|
const sessionsDir = opts?.sessionsDir ?? DEFAULT_SESSIONS_DIR();
|
|
6629
5733
|
const limit = opts?.latest ? 1 : opts?.limit ?? 10;
|
|
@@ -6635,7 +5739,7 @@ async function discoverCodexSessions(opts) {
|
|
|
6635
5739
|
return [];
|
|
6636
5740
|
}
|
|
6637
5741
|
for (const year of yearDirs) {
|
|
6638
|
-
const yearPath =
|
|
5742
|
+
const yearPath = path4.join(sessionsDir, year);
|
|
6639
5743
|
let monthDirs;
|
|
6640
5744
|
try {
|
|
6641
5745
|
monthDirs = await readdir2(yearPath);
|
|
@@ -6643,7 +5747,7 @@ async function discoverCodexSessions(opts) {
|
|
|
6643
5747
|
continue;
|
|
6644
5748
|
}
|
|
6645
5749
|
for (const month of monthDirs) {
|
|
6646
|
-
const monthPath =
|
|
5750
|
+
const monthPath = path4.join(yearPath, month);
|
|
6647
5751
|
let dayDirs;
|
|
6648
5752
|
try {
|
|
6649
5753
|
dayDirs = await readdir2(monthPath);
|
|
@@ -6655,7 +5759,7 @@ async function discoverCodexSessions(opts) {
|
|
|
6655
5759
|
const dirDate = `${year}-${month}-${day}`;
|
|
6656
5760
|
if (dirDate !== opts.date) continue;
|
|
6657
5761
|
}
|
|
6658
|
-
const dayPath =
|
|
5762
|
+
const dayPath = path4.join(monthPath, day);
|
|
6659
5763
|
let files;
|
|
6660
5764
|
try {
|
|
6661
5765
|
files = await readdir2(dayPath);
|
|
@@ -6664,7 +5768,7 @@ async function discoverCodexSessions(opts) {
|
|
|
6664
5768
|
}
|
|
6665
5769
|
for (const file of files) {
|
|
6666
5770
|
if (!file.startsWith("rollout-") || !file.endsWith(".jsonl")) continue;
|
|
6667
|
-
const filePath =
|
|
5771
|
+
const filePath = path4.join(dayPath, file);
|
|
6668
5772
|
const nameWithoutExt = file.replace(/\.jsonl$/, "");
|
|
6669
5773
|
const parts = nameWithoutExt.split("-");
|
|
6670
5774
|
const sessionId = parts.length >= 6 ? parts.slice(-5).join("-") : nameWithoutExt;
|
|
@@ -6683,7 +5787,7 @@ async function discoverCodexSessions(opts) {
|
|
|
6683
5787
|
sessions.sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime());
|
|
6684
5788
|
return sessions.slice(0, limit);
|
|
6685
5789
|
}
|
|
6686
|
-
var DEFAULT_PROJECTS_DIR = () =>
|
|
5790
|
+
var DEFAULT_PROJECTS_DIR = () => path5.join(homedir2(), ".claude", "projects");
|
|
6687
5791
|
function encodeProjectPath(projectPath) {
|
|
6688
5792
|
return projectPath.replace(/\//g, "-");
|
|
6689
5793
|
}
|
|
@@ -6702,7 +5806,7 @@ async function discoverClaudeSessions(opts) {
|
|
|
6702
5806
|
}
|
|
6703
5807
|
const sessions = [];
|
|
6704
5808
|
for (const projectDir of projectDirs) {
|
|
6705
|
-
const dirPath =
|
|
5809
|
+
const dirPath = path5.join(projectsDir, projectDir);
|
|
6706
5810
|
let entries;
|
|
6707
5811
|
try {
|
|
6708
5812
|
entries = await readdir3(dirPath);
|
|
@@ -6713,7 +5817,7 @@ async function discoverClaudeSessions(opts) {
|
|
|
6713
5817
|
if (!entry.endsWith(".jsonl")) continue;
|
|
6714
5818
|
const sessionId = entry.replace(/\.jsonl$/, "");
|
|
6715
5819
|
if (opts?.sessionId && sessionId !== opts.sessionId) continue;
|
|
6716
|
-
const filePath =
|
|
5820
|
+
const filePath = path5.join(dirPath, entry);
|
|
6717
5821
|
let updatedAt;
|
|
6718
5822
|
try {
|
|
6719
5823
|
const fileStat = await stat3(filePath);
|
|
@@ -6737,11 +5841,13 @@ var TranscriptProvider = class _TranscriptProvider {
|
|
|
6737
5841
|
kind = "transcript";
|
|
6738
5842
|
targetName;
|
|
6739
5843
|
entries;
|
|
5844
|
+
entriesByTestId;
|
|
6740
5845
|
cursor = 0;
|
|
6741
5846
|
constructor(targetName, entries) {
|
|
6742
5847
|
this.targetName = targetName;
|
|
6743
5848
|
this.id = `transcript:${targetName}`;
|
|
6744
5849
|
this.entries = entries;
|
|
5850
|
+
this.entriesByTestId = new Map(entries.map((entry) => [entry.testId, entry]));
|
|
6745
5851
|
}
|
|
6746
5852
|
/**
|
|
6747
5853
|
* Create a TranscriptProvider from a JSONL file path.
|
|
@@ -6758,13 +5864,11 @@ var TranscriptProvider = class _TranscriptProvider {
|
|
|
6758
5864
|
get lineCount() {
|
|
6759
5865
|
return this.entries.length;
|
|
6760
5866
|
}
|
|
6761
|
-
|
|
6762
|
-
|
|
6763
|
-
|
|
6764
|
-
|
|
6765
|
-
|
|
6766
|
-
}
|
|
6767
|
-
const entry = this.entries[this.cursor++];
|
|
5867
|
+
get testIds() {
|
|
5868
|
+
return this.entries.map((entry) => entry.testId);
|
|
5869
|
+
}
|
|
5870
|
+
async invoke(request) {
|
|
5871
|
+
const entry = request.evalCaseId ? this.entryForTestId(request.evalCaseId) : this.nextPositionalEntry();
|
|
6768
5872
|
return {
|
|
6769
5873
|
output: entry.messages,
|
|
6770
5874
|
tokenUsage: entry.tokenUsage ? {
|
|
@@ -6778,7 +5882,153 @@ var TranscriptProvider = class _TranscriptProvider {
|
|
|
6778
5882
|
startTime: entry.source.startedAt
|
|
6779
5883
|
};
|
|
6780
5884
|
}
|
|
5885
|
+
entryForTestId(testId) {
|
|
5886
|
+
const entry = this.entriesByTestId.get(testId);
|
|
5887
|
+
if (entry) {
|
|
5888
|
+
return entry;
|
|
5889
|
+
}
|
|
5890
|
+
throw new Error(
|
|
5891
|
+
`Transcript replay found no entry for test_id=${testId}. Available test_id values: ${this.testIds.join(", ") || "<none>"}`
|
|
5892
|
+
);
|
|
5893
|
+
}
|
|
5894
|
+
nextPositionalEntry() {
|
|
5895
|
+
if (this.cursor >= this.entries.length) {
|
|
5896
|
+
throw new Error(
|
|
5897
|
+
`Transcript exhausted: ${this.entries.length} entr${this.entries.length === 1 ? "y" : "ies"} available but ${this.cursor + 1} invocations attempted. Each transcript entry maps to one test case.`
|
|
5898
|
+
);
|
|
5899
|
+
}
|
|
5900
|
+
return this.entries[this.cursor++];
|
|
5901
|
+
}
|
|
6781
5902
|
};
|
|
5903
|
+
function parseCopilotEvents(eventsJsonl) {
|
|
5904
|
+
const messages = [];
|
|
5905
|
+
const meta = { sessionId: "", model: "", cwd: "" };
|
|
5906
|
+
let totalInputTokens = 0;
|
|
5907
|
+
let totalOutputTokens = 0;
|
|
5908
|
+
let hasUsage = false;
|
|
5909
|
+
let startTimestamp;
|
|
5910
|
+
let endTimestamp;
|
|
5911
|
+
const toolCallsInProgress = /* @__PURE__ */ new Map();
|
|
5912
|
+
const lines = eventsJsonl.split("\n").filter((l) => l.trim().length > 0);
|
|
5913
|
+
for (const line of lines) {
|
|
5914
|
+
let event;
|
|
5915
|
+
try {
|
|
5916
|
+
event = JSON.parse(line);
|
|
5917
|
+
} catch {
|
|
5918
|
+
continue;
|
|
5919
|
+
}
|
|
5920
|
+
const eventType = event.type;
|
|
5921
|
+
if (!eventType) continue;
|
|
5922
|
+
const data = event.data ?? {};
|
|
5923
|
+
switch (eventType) {
|
|
5924
|
+
case "session.start": {
|
|
5925
|
+
meta.sessionId = String(data.sessionId ?? "");
|
|
5926
|
+
const ctx = data.context;
|
|
5927
|
+
meta.cwd = String(ctx?.cwd ?? "");
|
|
5928
|
+
meta.repository = ctx?.repository ? String(ctx.repository) : void 0;
|
|
5929
|
+
meta.branch = ctx?.branch ? String(ctx.branch) : void 0;
|
|
5930
|
+
const ts = event.timestamp ?? data.startTime;
|
|
5931
|
+
meta.startedAt = ts ? String(ts) : void 0;
|
|
5932
|
+
startTimestamp = ts ? String(ts) : void 0;
|
|
5933
|
+
break;
|
|
5934
|
+
}
|
|
5935
|
+
case "user.message": {
|
|
5936
|
+
messages.push({
|
|
5937
|
+
role: "user",
|
|
5938
|
+
content: data.content != null ? String(data.content) : ""
|
|
5939
|
+
});
|
|
5940
|
+
break;
|
|
5941
|
+
}
|
|
5942
|
+
case "assistant.message": {
|
|
5943
|
+
const toolRequests = data.toolRequests;
|
|
5944
|
+
const toolCalls = (toolRequests ?? []).map(
|
|
5945
|
+
(req) => normalizeToolCall("copilot-events", {
|
|
5946
|
+
tool: String(req.name ?? req.toolName ?? ""),
|
|
5947
|
+
input: req.arguments,
|
|
5948
|
+
id: req.toolCallId ? String(req.toolCallId) : void 0
|
|
5949
|
+
})
|
|
5950
|
+
);
|
|
5951
|
+
messages.push({
|
|
5952
|
+
role: "assistant",
|
|
5953
|
+
content: data.content != null ? String(data.content) : void 0,
|
|
5954
|
+
toolCalls: toolCalls.length > 0 ? toolCalls : void 0
|
|
5955
|
+
});
|
|
5956
|
+
break;
|
|
5957
|
+
}
|
|
5958
|
+
case "skill.invoked": {
|
|
5959
|
+
const skillName = String(data.name ?? "");
|
|
5960
|
+
messages.push({
|
|
5961
|
+
role: "assistant",
|
|
5962
|
+
toolCalls: [
|
|
5963
|
+
{
|
|
5964
|
+
tool: "Skill",
|
|
5965
|
+
input: { skill: skillName }
|
|
5966
|
+
}
|
|
5967
|
+
]
|
|
5968
|
+
});
|
|
5969
|
+
break;
|
|
5970
|
+
}
|
|
5971
|
+
case "tool.execution_start": {
|
|
5972
|
+
const toolCallId = String(data.toolCallId ?? "");
|
|
5973
|
+
if (toolCallId) {
|
|
5974
|
+
toolCallsInProgress.set(toolCallId, {
|
|
5975
|
+
toolName: String(data.toolName ?? ""),
|
|
5976
|
+
input: data.arguments,
|
|
5977
|
+
toolCallId
|
|
5978
|
+
});
|
|
5979
|
+
}
|
|
5980
|
+
break;
|
|
5981
|
+
}
|
|
5982
|
+
case "tool.execution_complete": {
|
|
5983
|
+
const toolCallId = String(data.toolCallId ?? "");
|
|
5984
|
+
const started = toolCallsInProgress.get(toolCallId);
|
|
5985
|
+
if (started) {
|
|
5986
|
+
toolCallsInProgress.delete(toolCallId);
|
|
5987
|
+
messages.push({
|
|
5988
|
+
role: "assistant",
|
|
5989
|
+
toolCalls: [
|
|
5990
|
+
normalizeToolCall("copilot-events", {
|
|
5991
|
+
tool: started.toolName,
|
|
5992
|
+
input: started.input,
|
|
5993
|
+
output: data.result,
|
|
5994
|
+
id: toolCallId
|
|
5995
|
+
})
|
|
5996
|
+
]
|
|
5997
|
+
});
|
|
5998
|
+
}
|
|
5999
|
+
break;
|
|
6000
|
+
}
|
|
6001
|
+
case "session.shutdown": {
|
|
6002
|
+
endTimestamp = event.timestamp ? String(event.timestamp) : void 0;
|
|
6003
|
+
const modelMetrics = data.modelMetrics;
|
|
6004
|
+
if (modelMetrics) {
|
|
6005
|
+
for (const metrics of Object.values(modelMetrics)) {
|
|
6006
|
+
if (metrics.usage) {
|
|
6007
|
+
hasUsage = true;
|
|
6008
|
+
totalInputTokens += Number(metrics.usage.inputTokens ?? 0);
|
|
6009
|
+
totalOutputTokens += Number(metrics.usage.outputTokens ?? 0);
|
|
6010
|
+
}
|
|
6011
|
+
}
|
|
6012
|
+
}
|
|
6013
|
+
const currentModel = data.currentModel;
|
|
6014
|
+
if (currentModel && !meta.model) {
|
|
6015
|
+
meta.model = String(currentModel);
|
|
6016
|
+
}
|
|
6017
|
+
break;
|
|
6018
|
+
}
|
|
6019
|
+
}
|
|
6020
|
+
}
|
|
6021
|
+
let durationMs;
|
|
6022
|
+
if (startTimestamp && endTimestamp) {
|
|
6023
|
+
durationMs = new Date(endTimestamp).getTime() - new Date(startTimestamp).getTime();
|
|
6024
|
+
}
|
|
6025
|
+
return {
|
|
6026
|
+
messages,
|
|
6027
|
+
meta,
|
|
6028
|
+
tokenUsage: hasUsage ? { input: totalInputTokens, output: totalOutputTokens } : void 0,
|
|
6029
|
+
durationMs
|
|
6030
|
+
};
|
|
6031
|
+
}
|
|
6782
6032
|
function createAgentKernel() {
|
|
6783
6033
|
return { status: "stub" };
|
|
6784
6034
|
}
|
|
@@ -6788,12 +6038,6 @@ export {
|
|
|
6788
6038
|
traceEnvelopeToTraceSessionResponse,
|
|
6789
6039
|
buildTraceSpanTree,
|
|
6790
6040
|
normalizeTraceArtifactToTraceSessionResponse,
|
|
6791
|
-
isExperimentFileReference,
|
|
6792
|
-
deriveExperimentNameFromPath,
|
|
6793
|
-
loadExperimentConfig,
|
|
6794
|
-
normalizeExperimentConfig,
|
|
6795
|
-
fingerprintExperimentConfig,
|
|
6796
|
-
buildExperimentArtifactMetadata,
|
|
6797
6041
|
transpileEvalYaml,
|
|
6798
6042
|
transpileEvalYamlFile,
|
|
6799
6043
|
getOutputFilenames,
|
|
@@ -6801,7 +6045,6 @@ export {
|
|
|
6801
6045
|
defineConfig,
|
|
6802
6046
|
loadTsConfig,
|
|
6803
6047
|
generateRubrics,
|
|
6804
|
-
scanRepoDeps,
|
|
6805
6048
|
normalizeResultsConfig,
|
|
6806
6049
|
resolveResultsRepoUrl,
|
|
6807
6050
|
getResultsRepoLocalPaths,
|
|
@@ -6827,15 +6070,14 @@ export {
|
|
|
6827
6070
|
setupWipWorktree,
|
|
6828
6071
|
pushWipCheckpoint,
|
|
6829
6072
|
deleteWipBranch,
|
|
6073
|
+
resolveGitResultsIndexCacheFile,
|
|
6074
|
+
resolveGitRunsRefCommit,
|
|
6075
|
+
listGitRunsCached,
|
|
6830
6076
|
listGitRuns,
|
|
6831
6077
|
materializeGitRun,
|
|
6832
6078
|
syncProject,
|
|
6833
6079
|
syncProjects,
|
|
6834
6080
|
trimBaselineResult,
|
|
6835
|
-
DEFAULT_CATEGORY,
|
|
6836
|
-
deriveCategory,
|
|
6837
|
-
OtelTraceExporter,
|
|
6838
|
-
OtelStreamingObserver,
|
|
6839
6081
|
RunBudgetTracker,
|
|
6840
6082
|
parseEnvOutput,
|
|
6841
6083
|
runBeforeSessionHook,
|
|
@@ -6844,6 +6086,7 @@ export {
|
|
|
6844
6086
|
discoverCodexSessions,
|
|
6845
6087
|
discoverClaudeSessions,
|
|
6846
6088
|
TranscriptProvider,
|
|
6089
|
+
parseCopilotEvents,
|
|
6847
6090
|
createAgentKernel
|
|
6848
6091
|
};
|
|
6849
|
-
//# sourceMappingURL=chunk-
|
|
6092
|
+
//# sourceMappingURL=chunk-ALVZQUZP.js.map
|