agentv 4.42.4 → 4.44.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{artifact-writer-7PXSB6LM.js → artifact-writer-72MQQELS.js} +4 -4
- package/dist/{chunk-ISPJY2AX.js → chunk-2N3YSEMI.js} +2454 -415
- package/dist/chunk-2N3YSEMI.js.map +1 -0
- package/dist/{chunk-ZPFM52HV.js → chunk-5DYGN547.js} +2379 -635
- package/dist/chunk-5DYGN547.js.map +1 -0
- package/dist/{chunk-N6U3GXMF.js → chunk-CXHYTYHU.js} +620 -342
- package/dist/chunk-CXHYTYHU.js.map +1 -0
- package/dist/{chunk-6SQDBXT7.js → chunk-KAGZRUO6.js} +2 -2
- package/dist/chunk-KAGZRUO6.js.map +1 -0
- package/dist/{chunk-RBOP5X36.js → chunk-NCNDA3F6.js} +2605 -171
- package/dist/chunk-NCNDA3F6.js.map +1 -0
- package/dist/cli.js +5 -5
- package/dist/dashboard/assets/index-1Hvy8sBy.js +119 -0
- package/dist/dashboard/assets/{index-uXJsSG83.js → index-DitffAxS.js} +1 -1
- package/dist/dashboard/assets/index-J5sPcQO3.css +1 -0
- package/dist/dashboard/index.html +2 -2
- package/dist/{dist-VQWTETNY.js → dist-L4YCQOQ6.js} +67 -3
- package/dist/index.js +9 -5
- package/dist/{interactive-TNRJEFA2.js → interactive-DRNRKLEQ.js} +5 -5
- package/dist/skills/agentv-bench/SKILL.md +2 -2
- package/dist/skills/agentv-bench/agents/executor.md +1 -1
- package/dist/skills/agentv-bench/agents/grader.md +1 -1
- package/dist/skills/agentv-bench/references/autoresearch.md +4 -4
- package/dist/skills/agentv-bench/references/eval-yaml-spec.md +2 -1
- package/dist/skills/agentv-bench/references/schemas.md +1 -1
- package/dist/skills/agentv-bench/references/subagent-pipeline.md +4 -4
- package/dist/skills/agentv-eval-writer/SKILL.md +24 -17
- package/dist/skills/agentv-eval-writer/references/config-schema.json +130 -0
- package/dist/skills/agentv-eval-writer/references/eval-schema.json +192 -0
- package/dist/{ts-eval-loader-PYEU2PMP-YLXIQTEQ.js → ts-eval-loader-S7ESROX2-6CB3HB6H.js} +2 -2
- package/package.json +6 -5
- package/dist/chunk-6SQDBXT7.js.map +0 -1
- package/dist/chunk-ISPJY2AX.js.map +0 -1
- package/dist/chunk-N6U3GXMF.js.map +0 -1
- package/dist/chunk-RBOP5X36.js.map +0 -1
- package/dist/chunk-ZPFM52HV.js.map +0 -1
- package/dist/dashboard/assets/index-BwBJwHp4.js +0 -118
- package/dist/dashboard/assets/index-C5v6_zhR.css +0 -1
- /package/dist/{artifact-writer-7PXSB6LM.js.map → artifact-writer-72MQQELS.js.map} +0 -0
- /package/dist/{dist-VQWTETNY.js.map → dist-L4YCQOQ6.js.map} +0 -0
- /package/dist/{interactive-TNRJEFA2.js.map → interactive-DRNRKLEQ.js.map} +0 -0
- /package/dist/{ts-eval-loader-PYEU2PMP-YLXIQTEQ.js.map → ts-eval-loader-S7ESROX2-6CB3HB6H.js.map} +0 -0
|
@@ -2,13 +2,13 @@ import { createRequire } from 'node:module'; const require = createRequire(impor
|
|
|
2
2
|
import {
|
|
3
3
|
toSnakeCaseDeep as toSnakeCaseDeep2,
|
|
4
4
|
writeArtifactsFromResults
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-KAGZRUO6.js";
|
|
6
6
|
import {
|
|
7
7
|
RunBudgetTracker,
|
|
8
8
|
buildWipBranchName,
|
|
9
9
|
deleteWipBranch,
|
|
10
10
|
deriveCategory,
|
|
11
|
-
|
|
11
|
+
directPushResultsWithDetails,
|
|
12
12
|
directorySizeBytes,
|
|
13
13
|
getResultsRepoSyncStatus,
|
|
14
14
|
listGitRuns,
|
|
@@ -19,7 +19,7 @@ import {
|
|
|
19
19
|
resolveResultsRepoRunsDir,
|
|
20
20
|
setupWipWorktree,
|
|
21
21
|
syncResultsRepoForProject
|
|
22
|
-
} from "./chunk-
|
|
22
|
+
} from "./chunk-NCNDA3F6.js";
|
|
23
23
|
import {
|
|
24
24
|
CLI_PLACEHOLDERS,
|
|
25
25
|
COMMON_TARGET_SETTINGS,
|
|
@@ -67,7 +67,7 @@ import {
|
|
|
67
67
|
toSnakeCaseDeep,
|
|
68
68
|
traceFromTranscriptJsonLines,
|
|
69
69
|
writeInitialBenchmarkArtifact
|
|
70
|
-
} from "./chunk-
|
|
70
|
+
} from "./chunk-5DYGN547.js";
|
|
71
71
|
|
|
72
72
|
// src/commands/eval/shared.ts
|
|
73
73
|
import { constants } from "node:fs";
|
|
@@ -183,7 +183,7 @@ async function findRepoRoot(start) {
|
|
|
183
183
|
// package.json
|
|
184
184
|
var package_default = {
|
|
185
185
|
name: "agentv",
|
|
186
|
-
version: "4.
|
|
186
|
+
version: "4.44.0-next.1",
|
|
187
187
|
description: "CLI entry point for AgentV",
|
|
188
188
|
type: "module",
|
|
189
189
|
repository: {
|
|
@@ -200,19 +200,19 @@ var package_default = {
|
|
|
200
200
|
files: ["dist", "README.md"],
|
|
201
201
|
scripts: {
|
|
202
202
|
dev: "bun src/cli.ts",
|
|
203
|
-
build: "tsup && bun run copy-readme",
|
|
203
|
+
build: "(cd ../../packages/sdk && bun run build) && tsup && bun run copy-readme",
|
|
204
204
|
"copy-readme": `bun -e "import { cpSync } from 'fs'; cpSync('../../README.md', 'README.md')"`,
|
|
205
205
|
prepublishOnly: `node -e "if(process.env.ALLOW_PUBLISH!=='1'){console.error('ERROR: Use bun run publish:next, then bun run promote:latest');process.exit(1)}"`,
|
|
206
|
-
typecheck: "tsc --noEmit",
|
|
206
|
+
typecheck: "(cd ../../packages/sdk && bun run build) && tsc --noEmit",
|
|
207
207
|
lint: "biome check .",
|
|
208
208
|
format: "biome format --write .",
|
|
209
209
|
fix: "biome check --write .",
|
|
210
|
-
test: "bun test",
|
|
210
|
+
test: "(cd ../../packages/sdk && bun run build) && bun test",
|
|
211
211
|
"test:watch": "bun test --watch"
|
|
212
212
|
},
|
|
213
213
|
dependencies: {
|
|
214
214
|
"@anthropic-ai/claude-agent-sdk": "^0.2.49",
|
|
215
|
-
"@github/copilot-sdk": "^0.
|
|
215
|
+
"@github/copilot-sdk": "^1.0.3",
|
|
216
216
|
"@hono/node-server": "^1.19.11",
|
|
217
217
|
"@inquirer/prompts": "^8.2.1",
|
|
218
218
|
"@earendil-works/pi-ai": "^0.74.0",
|
|
@@ -236,6 +236,7 @@ var package_default = {
|
|
|
236
236
|
},
|
|
237
237
|
devDependencies: {
|
|
238
238
|
"@agentv/core": "workspace:*",
|
|
239
|
+
"@agentv/sdk": "workspace:*",
|
|
239
240
|
"@types/semver": "^7.7.1",
|
|
240
241
|
execa: "^9.3.0"
|
|
241
242
|
}
|
|
@@ -351,16 +352,16 @@ function enforceRequiredVersion(requiredVersion, options) {
|
|
|
351
352
|
import { existsSync as existsSync5 } from "node:fs";
|
|
352
353
|
import path8 from "node:path";
|
|
353
354
|
|
|
354
|
-
// src/commands/inspect/utils.ts
|
|
355
|
-
import { readFileSync as readFileSync2, readdirSync, statSync as statSync2 } from "node:fs";
|
|
356
|
-
import path5 from "node:path";
|
|
357
|
-
|
|
358
355
|
// src/commands/eval/result-layout.ts
|
|
359
356
|
import { existsSync, statSync } from "node:fs";
|
|
360
357
|
import path3 from "node:path";
|
|
361
358
|
var RESULT_INDEX_FILENAME = "index.jsonl";
|
|
362
|
-
var
|
|
359
|
+
var RESULTS_DIRNAME = "results";
|
|
363
360
|
var DEFAULT_EXPERIMENT_NAME = "default";
|
|
361
|
+
var RESERVED_RESULTS_NAMESPACES = /* @__PURE__ */ new Set(["export", "metadata", "runs"]);
|
|
362
|
+
function isReservedResultsNamespace(value) {
|
|
363
|
+
return value !== void 0 && RESERVED_RESULTS_NAMESPACES.has(value);
|
|
364
|
+
}
|
|
364
365
|
function normalizeExperimentName(experiment) {
|
|
365
366
|
const trimmed = experiment?.trim();
|
|
366
367
|
if (!trimmed) {
|
|
@@ -371,20 +372,26 @@ function normalizeExperimentName(experiment) {
|
|
|
371
372
|
`Invalid experiment name "${trimmed}". Use only letters, numbers, ".", "_" and "-".`
|
|
372
373
|
);
|
|
373
374
|
}
|
|
375
|
+
if (isReservedResultsNamespace(trimmed)) {
|
|
376
|
+
throw new Error(`Invalid experiment name "${trimmed}". This results namespace is reserved.`);
|
|
377
|
+
}
|
|
374
378
|
return trimmed;
|
|
375
379
|
}
|
|
376
380
|
function createRunDirName(timestamp = /* @__PURE__ */ new Date()) {
|
|
377
381
|
return timestamp.toISOString().replace(/[:.]/g, "-");
|
|
378
382
|
}
|
|
383
|
+
function defaultRunPathSegments(experiment, runDirName) {
|
|
384
|
+
const normalizedExperiment = normalizeExperimentName(experiment);
|
|
385
|
+
return [normalizedExperiment, runDirName];
|
|
386
|
+
}
|
|
387
|
+
function buildResultsRootDir(cwd) {
|
|
388
|
+
return path3.join(cwd, ".agentv", RESULTS_DIRNAME);
|
|
389
|
+
}
|
|
390
|
+
function buildDefaultRunDirFromName(cwd, experiment, runDirName) {
|
|
391
|
+
return path3.join(buildResultsRootDir(cwd), ...defaultRunPathSegments(experiment, runDirName));
|
|
392
|
+
}
|
|
379
393
|
function buildDefaultRunDir(cwd, experiment, timestamp = /* @__PURE__ */ new Date()) {
|
|
380
|
-
return
|
|
381
|
-
cwd,
|
|
382
|
-
".agentv",
|
|
383
|
-
"results",
|
|
384
|
-
RESULT_RUNS_DIRNAME,
|
|
385
|
-
normalizeExperimentName(experiment),
|
|
386
|
-
createRunDirName(timestamp)
|
|
387
|
-
);
|
|
394
|
+
return buildDefaultRunDirFromName(cwd, experiment, createRunDirName(timestamp));
|
|
388
395
|
}
|
|
389
396
|
function resolveRunIndexPath(runDir) {
|
|
390
397
|
return path3.join(runDir, RESULT_INDEX_FILENAME);
|
|
@@ -427,6 +434,21 @@ function resolveRunManifestPath(filePath) {
|
|
|
427
434
|
}
|
|
428
435
|
return filePath;
|
|
429
436
|
}
|
|
437
|
+
function relativeRunPathFromCwd(cwd, runDir) {
|
|
438
|
+
const relative = path3.relative(path3.resolve(buildResultsRootDir(cwd)), path3.resolve(runDir));
|
|
439
|
+
if (!relative || relative.startsWith("..") || path3.isAbsolute(relative)) {
|
|
440
|
+
return void 0;
|
|
441
|
+
}
|
|
442
|
+
const parts = relative.split(path3.sep).filter(Boolean);
|
|
443
|
+
if (parts.length < 2 || isReservedResultsNamespace(parts[0])) {
|
|
444
|
+
return void 0;
|
|
445
|
+
}
|
|
446
|
+
return parts.join(path3.posix.sep);
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
// src/commands/inspect/utils.ts
|
|
450
|
+
import { readFileSync as readFileSync2, readdirSync, statSync as statSync2 } from "node:fs";
|
|
451
|
+
import path5 from "node:path";
|
|
430
452
|
|
|
431
453
|
// src/commands/results/manifest.ts
|
|
432
454
|
import { existsSync as existsSync2, readFileSync } from "node:fs";
|
|
@@ -474,6 +496,21 @@ function readOptionalJson(baseDir, relativePath) {
|
|
|
474
496
|
return void 0;
|
|
475
497
|
}
|
|
476
498
|
}
|
|
499
|
+
function nonEmptyString(value) {
|
|
500
|
+
return typeof value === "string" && value.trim().length > 0 ? value : void 0;
|
|
501
|
+
}
|
|
502
|
+
function artifactPointerPath(pointer) {
|
|
503
|
+
if (typeof pointer === "string") {
|
|
504
|
+
return nonEmptyString(pointer);
|
|
505
|
+
}
|
|
506
|
+
if (!pointer) {
|
|
507
|
+
return void 0;
|
|
508
|
+
}
|
|
509
|
+
return nonEmptyString(pointer.path) ?? nonEmptyString(pointer.artifact_path) ?? nonEmptyString(pointer.relative_path);
|
|
510
|
+
}
|
|
511
|
+
function resolveTranscriptPath(record) {
|
|
512
|
+
return record.transcript_path ?? record.artifact_pointers?.transcript?.path ?? record.artifacts?.transcript_path ?? artifactPointerPath(record.transcript ?? record.artifacts?.transcript);
|
|
513
|
+
}
|
|
477
514
|
function hydrateInput(baseDir, record) {
|
|
478
515
|
const inputText = readOptionalText(baseDir, record.input_path);
|
|
479
516
|
if (!inputText) {
|
|
@@ -492,12 +529,14 @@ function hydrateOutput(baseDir, record) {
|
|
|
492
529
|
}
|
|
493
530
|
return responseText.trimEnd();
|
|
494
531
|
}
|
|
495
|
-
function hydrateTrace(baseDir, record) {
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
532
|
+
function hydrateTrace(baseDir, record, options) {
|
|
533
|
+
if (options.hydrateTranscriptTrace !== false) {
|
|
534
|
+
const transcriptText = readOptionalText(baseDir, resolveTranscriptPath(record));
|
|
535
|
+
if (transcriptText) {
|
|
536
|
+
try {
|
|
537
|
+
return traceFromTranscriptJsonLines(parseJsonlLines(transcriptText));
|
|
538
|
+
} catch {
|
|
539
|
+
}
|
|
501
540
|
}
|
|
502
541
|
}
|
|
503
542
|
const output = hydrateOutput(baseDir, record) ?? "";
|
|
@@ -510,7 +549,7 @@ function hydrateTrace(baseDir, record) {
|
|
|
510
549
|
testId: record.test_id
|
|
511
550
|
});
|
|
512
551
|
}
|
|
513
|
-
function hydrateManifestRecord(baseDir, record) {
|
|
552
|
+
function hydrateManifestRecord(baseDir, record, options) {
|
|
514
553
|
const grading = readOptionalJson(baseDir, record.grading_path);
|
|
515
554
|
const timing = readOptionalJson(baseDir, record.timing_path);
|
|
516
555
|
const testId = record.test_id ?? "unknown";
|
|
@@ -554,7 +593,7 @@ function hydrateManifestRecord(baseDir, record) {
|
|
|
554
593
|
costUsd: record.cost_usd,
|
|
555
594
|
input: hydrateInput(baseDir, record),
|
|
556
595
|
output: hydrateOutput(baseDir, record) ?? "",
|
|
557
|
-
trace: hydrateTrace(baseDir, record),
|
|
596
|
+
trace: hydrateTrace(baseDir, record, options),
|
|
558
597
|
metadata: record.metadata
|
|
559
598
|
};
|
|
560
599
|
}
|
|
@@ -568,12 +607,12 @@ function resolveResultSourcePath(source, cwd) {
|
|
|
568
607
|
}
|
|
569
608
|
return resolved;
|
|
570
609
|
}
|
|
571
|
-
function loadManifestResults(sourceFile) {
|
|
610
|
+
function loadManifestResults(sourceFile, options = {}) {
|
|
572
611
|
const resolvedSourceFile = resolveRunManifestPath(sourceFile);
|
|
573
612
|
const content = readFileSync(resolvedSourceFile, "utf8");
|
|
574
613
|
const records = parseResultRows(content, resolvedSourceFile);
|
|
575
614
|
const baseDir = path4.dirname(resolvedSourceFile);
|
|
576
|
-
return records.map((record) => hydrateManifestRecord(baseDir, record));
|
|
615
|
+
return records.map((record) => hydrateManifestRecord(baseDir, record, options));
|
|
577
616
|
}
|
|
578
617
|
function loadLightweightResults(sourceFile) {
|
|
579
618
|
const resolvedSourceFile = resolveRunManifestPath(sourceFile);
|
|
@@ -588,6 +627,7 @@ function loadLightweightResults(sourceFile) {
|
|
|
588
627
|
scores: record.scores,
|
|
589
628
|
executionStatus: record.execution_status,
|
|
590
629
|
error: record.error,
|
|
630
|
+
costUsd: record.cost_usd,
|
|
591
631
|
timestamp: record.timestamp
|
|
592
632
|
}));
|
|
593
633
|
}
|
|
@@ -961,24 +1001,27 @@ function collectRunManifestPaths(runsDir, currentDir, files) {
|
|
|
961
1001
|
}
|
|
962
1002
|
}
|
|
963
1003
|
}
|
|
964
|
-
function
|
|
1004
|
+
function listResultFilesFromRoot(runsDir, options) {
|
|
965
1005
|
const files = [];
|
|
966
1006
|
try {
|
|
967
1007
|
const entries = readdirSync(runsDir, { withFileTypes: true });
|
|
968
1008
|
for (const entry of entries) {
|
|
969
1009
|
if (entry.isDirectory()) {
|
|
1010
|
+
if (options?.skipTopLevelDirs?.has(entry.name) || isReservedResultsNamespace(entry.name)) {
|
|
1011
|
+
continue;
|
|
1012
|
+
}
|
|
970
1013
|
collectRunManifestPaths(runsDir, path5.join(runsDir, entry.name), files);
|
|
971
1014
|
}
|
|
972
1015
|
}
|
|
973
1016
|
} catch {
|
|
974
1017
|
}
|
|
975
1018
|
files.sort((a, b) => b.sortName.localeCompare(a.sortName));
|
|
976
|
-
const limited = limit !== void 0 && limit > 0 ? files.slice(0, limit) : files;
|
|
1019
|
+
const limited = options?.limit !== void 0 && options.limit > 0 ? files.slice(0, options.limit) : files;
|
|
977
1020
|
const metas = [];
|
|
978
1021
|
for (const { filePath, displayName, runId } of limited) {
|
|
979
1022
|
try {
|
|
980
1023
|
const fileStat = statSync2(filePath);
|
|
981
|
-
const results =
|
|
1024
|
+
const results = loadLightweightResults(filePath);
|
|
982
1025
|
const testCount = results.length;
|
|
983
1026
|
const passCount = results.filter((r) => r.score >= DEFAULT_THRESHOLD).length;
|
|
984
1027
|
const passRate = testCount > 0 ? passCount / testCount : 0;
|
|
@@ -1001,10 +1044,19 @@ function listResultFilesFromRunsDir(runsDir, limit) {
|
|
|
1001
1044
|
return metas;
|
|
1002
1045
|
}
|
|
1003
1046
|
function listResultFiles(cwd, limit) {
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1047
|
+
const metas = listResultFilesFromRoot(buildResultsRootDir(cwd), {
|
|
1048
|
+
skipTopLevelDirs: /* @__PURE__ */ new Set(["export", "metadata", "runs"])
|
|
1049
|
+
}).sort((a, b) => {
|
|
1050
|
+
const byTimestamp = b.timestamp.localeCompare(a.timestamp);
|
|
1051
|
+
return byTimestamp !== 0 ? byTimestamp : b.displayName.localeCompare(a.displayName);
|
|
1052
|
+
});
|
|
1053
|
+
return limit !== void 0 && limit > 0 ? metas.slice(0, limit) : metas;
|
|
1054
|
+
}
|
|
1055
|
+
function listResultFilesFromRunsDir(runsDir, limit) {
|
|
1056
|
+
return listResultFilesFromRoot(runsDir, {
|
|
1057
|
+
limit,
|
|
1058
|
+
skipTopLevelDirs: /* @__PURE__ */ new Set(["export", "metadata", "runs"])
|
|
1059
|
+
});
|
|
1008
1060
|
}
|
|
1009
1061
|
function extractTimestampFromFilename(filename) {
|
|
1010
1062
|
const match = filename.match(
|
|
@@ -1041,6 +1093,43 @@ import { execFileSync } from "node:child_process";
|
|
|
1041
1093
|
import { existsSync as existsSync4, mkdirSync, readFileSync as readFileSync4, rmSync, writeFileSync as writeFileSync2 } from "node:fs";
|
|
1042
1094
|
import path7 from "node:path";
|
|
1043
1095
|
|
|
1096
|
+
// src/commands/results/run-state.ts
|
|
1097
|
+
import { createHash } from "node:crypto";
|
|
1098
|
+
var TagRevisionConflictError = class extends Error {
|
|
1099
|
+
expectedRevision;
|
|
1100
|
+
currentRevision;
|
|
1101
|
+
constructor(expectedRevision, currentRevision) {
|
|
1102
|
+
super("Run tags changed. Refresh the run and try again.");
|
|
1103
|
+
this.name = "TagRevisionConflictError";
|
|
1104
|
+
this.expectedRevision = expectedRevision;
|
|
1105
|
+
this.currentRevision = currentRevision;
|
|
1106
|
+
}
|
|
1107
|
+
};
|
|
1108
|
+
function createTagRevision(tags, updatedAt) {
|
|
1109
|
+
const hash = createHash("sha256");
|
|
1110
|
+
hash.update("agentv.run_tags.v1\0");
|
|
1111
|
+
hash.update(JSON.stringify({ tags: [...tags], updated_at: updatedAt ?? "" }));
|
|
1112
|
+
return `sha256:${hash.digest("hex")}`;
|
|
1113
|
+
}
|
|
1114
|
+
function normalizeTagRevision(input, tags, updatedAt) {
|
|
1115
|
+
return typeof input === "string" && input.trim().length > 0 ? input : createTagRevision(tags, updatedAt);
|
|
1116
|
+
}
|
|
1117
|
+
function assertExpectedTagRevision(expectedRevision, currentRevision) {
|
|
1118
|
+
if (expectedRevision !== void 0 && expectedRevision !== currentRevision) {
|
|
1119
|
+
throw new TagRevisionConflictError(expectedRevision, currentRevision);
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
function materializeRunState(input) {
|
|
1123
|
+
const tags = [...input?.tags ?? []];
|
|
1124
|
+
return {
|
|
1125
|
+
final_state: {
|
|
1126
|
+
lifecycle: input?.lifecycle ?? "active",
|
|
1127
|
+
tags
|
|
1128
|
+
},
|
|
1129
|
+
tag_revision: input?.tagRevision ?? createTagRevision(tags, input?.updatedAt)
|
|
1130
|
+
};
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1044
1133
|
// src/commands/results/run-tags.ts
|
|
1045
1134
|
import { existsSync as existsSync3, readFileSync as readFileSync3, unlinkSync, writeFileSync } from "node:fs";
|
|
1046
1135
|
import path6 from "node:path";
|
|
@@ -1061,10 +1150,11 @@ function readRunTags(manifestPath) {
|
|
|
1061
1150
|
const tags = record.tags.filter(
|
|
1062
1151
|
(t) => typeof t === "string" && t.trim().length > 0
|
|
1063
1152
|
);
|
|
1064
|
-
|
|
1153
|
+
const updatedAt = typeof record.updated_at === "string" ? record.updated_at : "";
|
|
1065
1154
|
return {
|
|
1066
1155
|
tags,
|
|
1067
|
-
updated_at:
|
|
1156
|
+
updated_at: updatedAt,
|
|
1157
|
+
tag_revision: normalizeTagRevision(record.tag_revision, tags, updatedAt || void 0)
|
|
1068
1158
|
};
|
|
1069
1159
|
} catch {
|
|
1070
1160
|
return void 0;
|
|
@@ -1072,24 +1162,16 @@ function readRunTags(manifestPath) {
|
|
|
1072
1162
|
}
|
|
1073
1163
|
function writeRunTags(manifestPath, tags) {
|
|
1074
1164
|
const cleaned = normalizeTags(tags);
|
|
1075
|
-
|
|
1076
|
-
deleteRunTags(manifestPath);
|
|
1077
|
-
return null;
|
|
1078
|
-
}
|
|
1165
|
+
const updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1079
1166
|
const entry = {
|
|
1080
1167
|
tags: cleaned,
|
|
1081
|
-
updated_at:
|
|
1168
|
+
updated_at: updatedAt,
|
|
1169
|
+
tag_revision: createTagRevision(cleaned, updatedAt)
|
|
1082
1170
|
};
|
|
1083
1171
|
writeFileSync(runTagsPath(manifestPath), `${JSON.stringify(entry, null, 2)}
|
|
1084
1172
|
`, "utf8");
|
|
1085
1173
|
return entry;
|
|
1086
1174
|
}
|
|
1087
|
-
function deleteRunTags(manifestPath) {
|
|
1088
|
-
const fp = runTagsPath(manifestPath);
|
|
1089
|
-
if (existsSync3(fp)) {
|
|
1090
|
-
unlinkSync(fp);
|
|
1091
|
-
}
|
|
1092
|
-
}
|
|
1093
1175
|
function normalizeTags(tags) {
|
|
1094
1176
|
const seen = /* @__PURE__ */ new Set();
|
|
1095
1177
|
const out = [];
|
|
@@ -1172,15 +1254,23 @@ function parseTagsFile(content) {
|
|
|
1172
1254
|
const record = parsed;
|
|
1173
1255
|
if (!Array.isArray(record.tags)) return void 0;
|
|
1174
1256
|
const tags = record.tags.filter((tag) => typeof tag === "string");
|
|
1257
|
+
const updatedAt = typeof record.updated_at === "string" ? record.updated_at : void 0;
|
|
1175
1258
|
return {
|
|
1176
1259
|
tags,
|
|
1177
|
-
updatedAt
|
|
1260
|
+
updatedAt,
|
|
1261
|
+
tagRevision: normalizeTagRevision(record.tag_revision, tags, updatedAt)
|
|
1178
1262
|
};
|
|
1179
1263
|
}
|
|
1180
1264
|
function equalTags(a, b) {
|
|
1181
1265
|
if (a.length !== b.length) return false;
|
|
1182
1266
|
return a.every((tag, index) => tag === b[index]);
|
|
1183
1267
|
}
|
|
1268
|
+
function equalTagFiles(a, b) {
|
|
1269
|
+
if (a === void 0 || b === void 0) {
|
|
1270
|
+
return a === b;
|
|
1271
|
+
}
|
|
1272
|
+
return equalTags(a.tags, b.tags) && a.updatedAt === b.updatedAt && a.tagRevision === b.tagRevision;
|
|
1273
|
+
}
|
|
1184
1274
|
function resolveComparisonRef(repoDir) {
|
|
1185
1275
|
const upstream = tryRunGit(repoDir, [
|
|
1186
1276
|
"rev-parse",
|
|
@@ -1215,9 +1305,8 @@ function resolveRemoteRunMetadataPaths(repoDir, manifestPath) {
|
|
|
1215
1305
|
overlayTagsGitPath: toGitPath(path7.relative(repoDir, overlayTagsPath))
|
|
1216
1306
|
};
|
|
1217
1307
|
}
|
|
1218
|
-
function readRemoteRunTagsContext(repoDir, manifestPath) {
|
|
1308
|
+
function readRemoteRunTagsContext(repoDir, manifestPath, comparisonRef = resolveComparisonRef(repoDir)) {
|
|
1219
1309
|
const paths = resolveRemoteRunMetadataPaths(repoDir, manifestPath);
|
|
1220
|
-
const comparisonRef = resolveComparisonRef(repoDir);
|
|
1221
1310
|
const artifactTags = readTagsFile(paths.artifactTagsPath) ?? readTagsFromGit(repoDir, comparisonRef, paths.artifactTagsGitPath);
|
|
1222
1311
|
const baseOverlayTags = readTagsFromGit(repoDir, comparisonRef, paths.overlayTagsGitPath);
|
|
1223
1312
|
const localOverlayTags = readTagsFile(paths.overlayTagsPath);
|
|
@@ -1231,13 +1320,16 @@ function readRemoteRunTagsContext(repoDir, manifestPath) {
|
|
|
1231
1320
|
function toRemoteRunTagState(context) {
|
|
1232
1321
|
const remoteTags = context.baseOverlayTags?.tags ?? context.artifactTags?.tags ?? [];
|
|
1233
1322
|
const effectiveTags = context.localOverlayTags?.tags ?? remoteTags;
|
|
1234
|
-
const dirty = !equalTags(effectiveTags, remoteTags);
|
|
1323
|
+
const dirty = context.localOverlayTags ? !equalTagFiles(context.localOverlayTags, context.baseOverlayTags) : !equalTags(effectiveTags, remoteTags);
|
|
1324
|
+
const updatedAt = context.localOverlayTags?.updatedAt ?? context.baseOverlayTags?.updatedAt ?? context.artifactTags?.updatedAt;
|
|
1325
|
+
const tagRevision = context.localOverlayTags?.tagRevision ?? context.baseOverlayTags?.tagRevision ?? context.artifactTags?.tagRevision ?? createTagRevision(effectiveTags, updatedAt);
|
|
1235
1326
|
return {
|
|
1236
1327
|
tags: effectiveTags,
|
|
1237
1328
|
remoteTags,
|
|
1238
1329
|
...dirty && { pendingTags: effectiveTags },
|
|
1239
1330
|
dirty,
|
|
1240
|
-
updatedAt
|
|
1331
|
+
updatedAt,
|
|
1332
|
+
tagRevision,
|
|
1241
1333
|
metadataPath: context.paths.overlayTagsPath
|
|
1242
1334
|
};
|
|
1243
1335
|
}
|
|
@@ -1250,37 +1342,44 @@ function assertWritableResultsRepo(repoDir) {
|
|
|
1250
1342
|
throw new Error(`Configured results repo is not a writable git checkout: ${repoDir}`);
|
|
1251
1343
|
}
|
|
1252
1344
|
}
|
|
1253
|
-
function readRemoteRunTags(repoDir, manifestPath) {
|
|
1254
|
-
const context = readRemoteRunTagsContext(repoDir, manifestPath);
|
|
1345
|
+
function readRemoteRunTags(repoDir, manifestPath, comparisonRef) {
|
|
1346
|
+
const context = readRemoteRunTagsContext(repoDir, manifestPath, comparisonRef);
|
|
1255
1347
|
return toRemoteRunTagState(context);
|
|
1256
1348
|
}
|
|
1257
|
-
function writeRemoteRunTags(repoDir, manifestPath, tags) {
|
|
1349
|
+
function writeRemoteRunTags(repoDir, manifestPath, tags, comparisonRef, expectedTagRevision) {
|
|
1258
1350
|
assertWritableResultsRepo(repoDir);
|
|
1259
1351
|
const cleaned = normalizeTags(tags);
|
|
1260
|
-
const context = readRemoteRunTagsContext(repoDir, manifestPath);
|
|
1352
|
+
const context = readRemoteRunTagsContext(repoDir, manifestPath, comparisonRef);
|
|
1353
|
+
const currentState = toRemoteRunTagState(context);
|
|
1354
|
+
assertExpectedTagRevision(expectedTagRevision, currentState.tagRevision);
|
|
1261
1355
|
const remoteTags = context.baseOverlayTags?.tags ?? context.artifactTags?.tags ?? [];
|
|
1262
|
-
if (equalTags(cleaned, remoteTags) && context.baseOverlayTags === void 0) {
|
|
1356
|
+
if (cleaned.length > 0 && equalTags(cleaned, remoteTags) && context.baseOverlayTags === void 0) {
|
|
1263
1357
|
rmSync(context.paths.overlayTagsPath, { force: true });
|
|
1264
|
-
return readRemoteRunTags(repoDir, manifestPath);
|
|
1358
|
+
return readRemoteRunTags(repoDir, manifestPath, comparisonRef);
|
|
1265
1359
|
}
|
|
1360
|
+
const updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1266
1361
|
const entry = {
|
|
1267
1362
|
tags: cleaned,
|
|
1268
|
-
updated_at:
|
|
1363
|
+
updated_at: updatedAt,
|
|
1364
|
+
tag_revision: createTagRevision(cleaned, updatedAt)
|
|
1269
1365
|
};
|
|
1270
1366
|
mkdirSync(path7.dirname(context.paths.overlayTagsPath), { recursive: true });
|
|
1271
1367
|
writeFileSync2(context.paths.overlayTagsPath, `${JSON.stringify(entry, null, 2)}
|
|
1272
1368
|
`, "utf8");
|
|
1273
|
-
return readRemoteRunTags(repoDir, manifestPath);
|
|
1369
|
+
return readRemoteRunTags(repoDir, manifestPath, comparisonRef);
|
|
1274
1370
|
}
|
|
1275
|
-
function deleteRemoteRunTags(repoDir, manifestPath) {
|
|
1276
|
-
return writeRemoteRunTags(repoDir, manifestPath, []);
|
|
1371
|
+
function deleteRemoteRunTags(repoDir, manifestPath, comparisonRef, expectedTagRevision) {
|
|
1372
|
+
return writeRemoteRunTags(repoDir, manifestPath, [], comparisonRef, expectedTagRevision);
|
|
1277
1373
|
}
|
|
1278
1374
|
|
|
1279
1375
|
// src/commands/results/remote.ts
|
|
1280
1376
|
var gitRunsCache = /* @__PURE__ */ new Map();
|
|
1281
1377
|
var GIT_RUNS_CACHE_TTL_MS = 6e4;
|
|
1282
1378
|
function getResultsStorageRef(config) {
|
|
1283
|
-
|
|
1379
|
+
if (!config.branch) {
|
|
1380
|
+
return void 0;
|
|
1381
|
+
}
|
|
1382
|
+
return config.storageBranchWorktree ? `refs/remotes/${config.remote}/${config.branch}` : config.branch;
|
|
1284
1383
|
}
|
|
1285
1384
|
function cachedListGitRuns(repoDir, ref) {
|
|
1286
1385
|
const now = Date.now();
|
|
@@ -1307,19 +1406,36 @@ function invalidateGitRunsCache(repoDir) {
|
|
|
1307
1406
|
}
|
|
1308
1407
|
}
|
|
1309
1408
|
}
|
|
1409
|
+
function relativeLocalRunPath(cwd, manifestPath) {
|
|
1410
|
+
const manifestDir = path8.resolve(path8.dirname(manifestPath));
|
|
1411
|
+
return relativeRunPathFromCwd(cwd, manifestDir);
|
|
1412
|
+
}
|
|
1413
|
+
function remoteMetadataManifestPath(cwd, config, meta) {
|
|
1414
|
+
if (meta.source === "remote") {
|
|
1415
|
+
return meta.path;
|
|
1416
|
+
}
|
|
1417
|
+
if (!meta.on_remote) {
|
|
1418
|
+
return void 0;
|
|
1419
|
+
}
|
|
1420
|
+
const relativeRunPath = relativeLocalRunPath(cwd, meta.path);
|
|
1421
|
+
if (!relativeRunPath) {
|
|
1422
|
+
return void 0;
|
|
1423
|
+
}
|
|
1424
|
+
return path8.join(config.path, "runs", ...relativeRunPath.split("/"), "index.jsonl");
|
|
1425
|
+
}
|
|
1310
1426
|
var REMOTE_RUN_PREFIX = "remote::";
|
|
1311
1427
|
var SIZE_WARNING_BYTES = 10 * 1024 * 1024;
|
|
1312
1428
|
function getStatusMessage(error) {
|
|
1313
1429
|
return error instanceof Error ? error.message : String(error);
|
|
1314
1430
|
}
|
|
1315
1431
|
function getRelativeRunPath(cwd, runDir) {
|
|
1316
|
-
const relative =
|
|
1317
|
-
if (
|
|
1432
|
+
const relative = relativeRunPathFromCwd(cwd, runDir);
|
|
1433
|
+
if (relative) {
|
|
1318
1434
|
return relative;
|
|
1319
1435
|
}
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1436
|
+
throw new Error(
|
|
1437
|
+
`Run workspace must use .agentv/results/<experiment>/<timestamp>: ${path8.resolve(runDir)}`
|
|
1438
|
+
);
|
|
1323
1439
|
}
|
|
1324
1440
|
function buildCommitTitle(payload) {
|
|
1325
1441
|
const passed = payload.results.filter((result) => result.score >= DEFAULT_THRESHOLD).length;
|
|
@@ -1349,13 +1465,16 @@ async function loadNormalizedResultsConfig(cwd, projectId, overrides) {
|
|
|
1349
1465
|
...project.results.branch !== void 0 && { branch: project.results.branch },
|
|
1350
1466
|
...project.results.remote !== void 0 && { remote: project.results.remote },
|
|
1351
1467
|
...project.results.path !== void 0 && { path: project.results.path },
|
|
1352
|
-
...(project.results.sync?.autoPush !== void 0 || project.results.sync?.requirePush !== void 0) && {
|
|
1468
|
+
...(project.results.sync?.autoPush !== void 0 || project.results.sync?.requirePush !== void 0 || project.results.sync?.pushConflictPolicy !== void 0) && {
|
|
1353
1469
|
sync: {
|
|
1354
1470
|
...project.results.sync?.autoPush !== void 0 && {
|
|
1355
1471
|
auto_push: project.results.sync.autoPush
|
|
1356
1472
|
},
|
|
1357
1473
|
...project.results.sync?.requirePush !== void 0 && {
|
|
1358
1474
|
require_push: project.results.sync.requirePush
|
|
1475
|
+
},
|
|
1476
|
+
...project.results.sync?.pushConflictPolicy !== void 0 && {
|
|
1477
|
+
push_conflict_policy: project.results.sync.pushConflictPolicy
|
|
1359
1478
|
}
|
|
1360
1479
|
}
|
|
1361
1480
|
},
|
|
@@ -1381,13 +1500,16 @@ async function loadNormalizedResultsConfig(cwd, projectId, overrides) {
|
|
|
1381
1500
|
...overrides.branch !== void 0 ? { branch: overrides.branch } : baseConfig?.branch ? { branch: baseConfig.branch } : {},
|
|
1382
1501
|
...overrides.remote !== void 0 ? { remote: overrides.remote } : baseConfig?.remote ? { remote: baseConfig.remote } : {},
|
|
1383
1502
|
...repoOverride === void 0 && baseConfig?.repo_path === void 0 && baseConfig?.path ? { path: baseConfig.path } : {},
|
|
1384
|
-
...(overrides.auto_push !== void 0 || overrides.require_push !== void 0 || baseConfig?.auto_push !== void 0 || baseConfig?.require_push !== void 0) && {
|
|
1503
|
+
...(overrides.auto_push !== void 0 || overrides.require_push !== void 0 || overrides.push_conflict_policy !== void 0 || baseConfig?.auto_push !== void 0 || baseConfig?.require_push !== void 0 || baseConfig?.push_conflict_policy !== void 0) && {
|
|
1385
1504
|
sync: {
|
|
1386
1505
|
...(overrides.auto_push ?? baseConfig?.auto_push) !== void 0 && {
|
|
1387
1506
|
auto_push: overrides.auto_push ?? baseConfig?.auto_push
|
|
1388
1507
|
},
|
|
1389
1508
|
...(overrides.require_push ?? baseConfig?.require_push) !== void 0 && {
|
|
1390
1509
|
require_push: overrides.require_push ?? baseConfig?.require_push
|
|
1510
|
+
},
|
|
1511
|
+
...(overrides.push_conflict_policy ?? baseConfig?.push_conflict_policy) !== void 0 && {
|
|
1512
|
+
push_conflict_policy: overrides.push_conflict_policy ?? baseConfig?.push_conflict_policy
|
|
1391
1513
|
}
|
|
1392
1514
|
}
|
|
1393
1515
|
},
|
|
@@ -1554,36 +1676,57 @@ async function ensureRemoteRunAvailable(cwd, meta, projectId) {
|
|
|
1554
1676
|
await materializeGitRun(config.path, relativeRunPath, getResultsStorageRef(config));
|
|
1555
1677
|
}
|
|
1556
1678
|
async function readRemoteRunTagState(cwd, meta, projectId) {
|
|
1557
|
-
if (meta.source !== "remote") return void 0;
|
|
1679
|
+
if (meta.source !== "remote" && !meta.on_remote) return void 0;
|
|
1558
1680
|
const config = await loadNormalizedResultsConfig(cwd, projectId);
|
|
1559
1681
|
if (!config) return void 0;
|
|
1682
|
+
const manifestPath = remoteMetadataManifestPath(cwd, config, meta);
|
|
1683
|
+
if (!manifestPath) return void 0;
|
|
1560
1684
|
try {
|
|
1561
|
-
return readRemoteRunTags(config.path,
|
|
1685
|
+
return readRemoteRunTags(config.path, manifestPath, getResultsStorageRef(config));
|
|
1562
1686
|
} catch {
|
|
1563
1687
|
return void 0;
|
|
1564
1688
|
}
|
|
1565
1689
|
}
|
|
1566
|
-
async function setRemoteRunTags(cwd, meta, tags, projectId) {
|
|
1567
|
-
if (meta.source !== "remote") {
|
|
1690
|
+
async function setRemoteRunTags(cwd, meta, tags, projectId, expectedTagRevision) {
|
|
1691
|
+
if (meta.source !== "remote" && !meta.on_remote) {
|
|
1568
1692
|
throw new Error("Remote metadata can only be set on remote runs");
|
|
1569
1693
|
}
|
|
1570
1694
|
const config = await loadNormalizedResultsConfig(cwd, projectId);
|
|
1571
1695
|
if (!config) {
|
|
1572
1696
|
throw new Error("Writable results repo is not configured for remote metadata");
|
|
1573
1697
|
}
|
|
1698
|
+
const manifestPath = remoteMetadataManifestPath(cwd, config, meta);
|
|
1699
|
+
if (!manifestPath) {
|
|
1700
|
+
throw new Error("Remote metadata can only be set on remote runs");
|
|
1701
|
+
}
|
|
1574
1702
|
assertWritableResultsRepo(config.path);
|
|
1575
|
-
return writeRemoteRunTags(
|
|
1703
|
+
return writeRemoteRunTags(
|
|
1704
|
+
config.path,
|
|
1705
|
+
manifestPath,
|
|
1706
|
+
tags,
|
|
1707
|
+
getResultsStorageRef(config),
|
|
1708
|
+
expectedTagRevision
|
|
1709
|
+
);
|
|
1576
1710
|
}
|
|
1577
|
-
async function clearRemoteRunTags(cwd, meta, projectId) {
|
|
1578
|
-
if (meta.source !== "remote") {
|
|
1711
|
+
async function clearRemoteRunTags(cwd, meta, projectId, expectedTagRevision) {
|
|
1712
|
+
if (meta.source !== "remote" && !meta.on_remote) {
|
|
1579
1713
|
throw new Error("Remote metadata can only be removed from remote runs");
|
|
1580
1714
|
}
|
|
1581
1715
|
const config = await loadNormalizedResultsConfig(cwd, projectId);
|
|
1582
1716
|
if (!config) {
|
|
1583
1717
|
throw new Error("Writable results repo is not configured for remote metadata");
|
|
1584
1718
|
}
|
|
1719
|
+
const manifestPath = remoteMetadataManifestPath(cwd, config, meta);
|
|
1720
|
+
if (!manifestPath) {
|
|
1721
|
+
throw new Error("Remote metadata can only be removed from remote runs");
|
|
1722
|
+
}
|
|
1585
1723
|
assertWritableResultsRepo(config.path);
|
|
1586
|
-
return deleteRemoteRunTags(
|
|
1724
|
+
return deleteRemoteRunTags(
|
|
1725
|
+
config.path,
|
|
1726
|
+
manifestPath,
|
|
1727
|
+
getResultsStorageRef(config),
|
|
1728
|
+
expectedTagRevision
|
|
1729
|
+
);
|
|
1587
1730
|
}
|
|
1588
1731
|
async function maybeAutoExportRunArtifacts(payload) {
|
|
1589
1732
|
const config = await loadNormalizedResultsConfig(
|
|
@@ -1598,13 +1741,20 @@ async function maybeAutoExportRunArtifacts(payload) {
|
|
|
1598
1741
|
await maybeWarnLargeArtifact(payload.run_dir);
|
|
1599
1742
|
const relativeRunPath = getRelativeRunPath(payload.cwd, payload.run_dir);
|
|
1600
1743
|
const commitTitle = buildCommitTitle(payload);
|
|
1601
|
-
const
|
|
1744
|
+
const pushResult = await directPushResultsWithDetails({
|
|
1602
1745
|
config,
|
|
1603
1746
|
sourceDir: payload.run_dir,
|
|
1604
1747
|
destinationPath: relativeRunPath,
|
|
1605
1748
|
commitMessage: commitTitle
|
|
1606
1749
|
});
|
|
1607
|
-
if (
|
|
1750
|
+
if (pushResult.blocked) {
|
|
1751
|
+
if (config.require_push) {
|
|
1752
|
+
throw new Error(pushResult.block_reason ?? "Results branch push conflict");
|
|
1753
|
+
}
|
|
1754
|
+
console.warn(`Warning: skipping results export: ${pushResult.block_reason}`);
|
|
1755
|
+
return "failed";
|
|
1756
|
+
}
|
|
1757
|
+
if (!pushResult.changed) {
|
|
1608
1758
|
console.warn("Warning: results export produced no git changes.");
|
|
1609
1759
|
return "already_published";
|
|
1610
1760
|
}
|
|
@@ -1612,6 +1762,11 @@ async function maybeAutoExportRunArtifacts(payload) {
|
|
|
1612
1762
|
console.log(
|
|
1613
1763
|
`Results ${pushLabel} to ${config.repo} (${config.branch ?? "default branch"}:${relativeRunPath})`
|
|
1614
1764
|
);
|
|
1765
|
+
if (pushResult.backup_ref) {
|
|
1766
|
+
console.log(
|
|
1767
|
+
`Backed up previous remote ${pushResult.target_branch ?? config.branch ?? "results branch"} at ${pushResult.backup_ref}`
|
|
1768
|
+
);
|
|
1769
|
+
}
|
|
1615
1770
|
return "published";
|
|
1616
1771
|
} catch (error) {
|
|
1617
1772
|
if (config.require_push) {
|
|
@@ -2728,20 +2883,7 @@ async function validateEvalFile(filePath) {
|
|
|
2728
2883
|
});
|
|
2729
2884
|
}
|
|
2730
2885
|
}
|
|
2731
|
-
|
|
2732
|
-
if (suiteInput !== void 0) {
|
|
2733
|
-
if (typeof suiteInput === "string") {
|
|
2734
|
-
} else if (Array.isArray(suiteInput)) {
|
|
2735
|
-
validateMessages(suiteInput, "input", absolutePath, errors);
|
|
2736
|
-
} else {
|
|
2737
|
-
errors.push({
|
|
2738
|
-
severity: "error",
|
|
2739
|
-
filePath: absolutePath,
|
|
2740
|
-
location: "input",
|
|
2741
|
-
message: "Invalid suite-level 'input' field (must be a string or array of messages)"
|
|
2742
|
-
});
|
|
2743
|
-
}
|
|
2744
|
-
}
|
|
2886
|
+
validateInputField(parsed.input, "input", absolutePath, errors);
|
|
2745
2887
|
const cases = parsed.tests;
|
|
2746
2888
|
if (typeof cases === "string") {
|
|
2747
2889
|
await validateWorkspaceConfig(parsed.workspace, absolutePath, errors, "workspace");
|
|
@@ -2881,27 +3023,9 @@ async function validateEvalFile(filePath) {
|
|
|
2881
3023
|
message: "Invalid 'criteria' field (must be a non-empty string if provided)"
|
|
2882
3024
|
});
|
|
2883
3025
|
}
|
|
2884
|
-
|
|
2885
|
-
|
|
2886
|
-
|
|
2887
|
-
} else if (Array.isArray(inputField)) {
|
|
2888
|
-
validateMessages(inputField, `${location}.input`, absolutePath, errors);
|
|
2889
|
-
} else {
|
|
2890
|
-
errors.push({
|
|
2891
|
-
severity: "error",
|
|
2892
|
-
filePath: absolutePath,
|
|
2893
|
-
location: `${location}.input`,
|
|
2894
|
-
message: "Invalid 'input' field (must be a string or array of messages)"
|
|
2895
|
-
});
|
|
2896
|
-
}
|
|
2897
|
-
} else {
|
|
2898
|
-
errors.push({
|
|
2899
|
-
severity: "error",
|
|
2900
|
-
filePath: absolutePath,
|
|
2901
|
-
location: `${location}.input`,
|
|
2902
|
-
message: "Missing 'input' field (must be a string or array of messages)"
|
|
2903
|
-
});
|
|
2904
|
-
}
|
|
3026
|
+
validateInputField(evalCase.input, `${location}.input`, absolutePath, errors, {
|
|
3027
|
+
required: true
|
|
3028
|
+
});
|
|
2905
3029
|
const expectedOutputField = evalCase.expected_output;
|
|
2906
3030
|
if (expectedOutputField !== void 0) {
|
|
2907
3031
|
if (typeof expectedOutputField === "string") {
|
|
@@ -3053,7 +3177,7 @@ function validateMessages(messages, location, filePath, errors) {
|
|
|
3053
3177
|
continue;
|
|
3054
3178
|
}
|
|
3055
3179
|
const role = message.role;
|
|
3056
|
-
const validRoles = ["system", "user", "assistant"];
|
|
3180
|
+
const validRoles = ["system", "user", "assistant", "tool"];
|
|
3057
3181
|
if (!validRoles.includes(role)) {
|
|
3058
3182
|
errors.push({
|
|
3059
3183
|
severity: "error",
|
|
@@ -3116,6 +3240,39 @@ function validateMessages(messages, location, filePath, errors) {
|
|
|
3116
3240
|
}
|
|
3117
3241
|
}
|
|
3118
3242
|
}
|
|
3243
|
+
function validateInputField(inputField, location, filePath, errors, options = {}) {
|
|
3244
|
+
if (inputField === void 0) {
|
|
3245
|
+
if (options.required) {
|
|
3246
|
+
errors.push({
|
|
3247
|
+
severity: "error",
|
|
3248
|
+
filePath,
|
|
3249
|
+
location,
|
|
3250
|
+
message: "Missing 'input' field (must be a string, object, or array of messages)"
|
|
3251
|
+
});
|
|
3252
|
+
}
|
|
3253
|
+
return;
|
|
3254
|
+
}
|
|
3255
|
+
if (typeof inputField === "string") {
|
|
3256
|
+
return;
|
|
3257
|
+
}
|
|
3258
|
+
if (Array.isArray(inputField)) {
|
|
3259
|
+
validateMessages(inputField, location, filePath, errors);
|
|
3260
|
+
return;
|
|
3261
|
+
}
|
|
3262
|
+
if (isObject(inputField)) {
|
|
3263
|
+
if ("role" in inputField) {
|
|
3264
|
+
validateMessages([inputField], location, filePath, errors);
|
|
3265
|
+
}
|
|
3266
|
+
return;
|
|
3267
|
+
}
|
|
3268
|
+
const label = location === "input" ? "suite-level 'input'" : "'input'";
|
|
3269
|
+
errors.push({
|
|
3270
|
+
severity: "error",
|
|
3271
|
+
filePath,
|
|
3272
|
+
location,
|
|
3273
|
+
message: `Invalid ${label} field (must be a string, object, or array of messages)`
|
|
3274
|
+
});
|
|
3275
|
+
}
|
|
3119
3276
|
function validateMetadata(parsed, filePath, errors) {
|
|
3120
3277
|
const name = parsed.name;
|
|
3121
3278
|
if (name !== void 0) {
|
|
@@ -3500,8 +3657,15 @@ var GEMINI_SETTINGS = /* @__PURE__ */ new Set([
|
|
|
3500
3657
|
]);
|
|
3501
3658
|
var CODEX_SETTINGS = /* @__PURE__ */ new Set([
|
|
3502
3659
|
...COMMON_SETTINGS,
|
|
3660
|
+
"endpoint",
|
|
3661
|
+
"base_url",
|
|
3662
|
+
"api_key",
|
|
3663
|
+
"api_format",
|
|
3503
3664
|
"model",
|
|
3504
3665
|
"model_reasoning_effort",
|
|
3666
|
+
"model_verbosity",
|
|
3667
|
+
"sandbox_mode",
|
|
3668
|
+
"approval_policy",
|
|
3505
3669
|
"executable",
|
|
3506
3670
|
"command",
|
|
3507
3671
|
"binary",
|
|
@@ -3533,7 +3697,9 @@ var COPILOT_SDK_SETTINGS = /* @__PURE__ */ new Set([
|
|
|
3533
3697
|
"api_key",
|
|
3534
3698
|
"bearer_token",
|
|
3535
3699
|
"api_version",
|
|
3536
|
-
"api_format"
|
|
3700
|
+
"api_format",
|
|
3701
|
+
"model_id",
|
|
3702
|
+
"wire_model"
|
|
3537
3703
|
]);
|
|
3538
3704
|
var COPILOT_CLI_SETTINGS = /* @__PURE__ */ new Set([
|
|
3539
3705
|
...COMMON_SETTINGS,
|
|
@@ -3554,7 +3720,9 @@ var COPILOT_CLI_SETTINGS = /* @__PURE__ */ new Set([
|
|
|
3554
3720
|
"api_key",
|
|
3555
3721
|
"bearer_token",
|
|
3556
3722
|
"api_version",
|
|
3557
|
-
"api_format"
|
|
3723
|
+
"api_format",
|
|
3724
|
+
"model_id",
|
|
3725
|
+
"wire_model"
|
|
3558
3726
|
]);
|
|
3559
3727
|
var VSCODE_SETTINGS = /* @__PURE__ */ new Set([
|
|
3560
3728
|
...COMMON_SETTINGS,
|
|
@@ -4093,31 +4261,114 @@ function validateProjects(errors, filePath, projects) {
|
|
|
4093
4261
|
const projectRecord = project;
|
|
4094
4262
|
validateRequiredString(errors, filePath, projectRecord.id, `${location}.id`);
|
|
4095
4263
|
validateRequiredString(errors, filePath, projectRecord.name, `${location}.name`);
|
|
4096
|
-
|
|
4097
|
-
|
|
4098
|
-
|
|
4099
|
-
|
|
4264
|
+
validateProjectRepoConfig(errors, filePath, projectRecord, location);
|
|
4265
|
+
validateProjectResultsConfig(errors, filePath, projectRecord.results, `${location}.results`);
|
|
4266
|
+
});
|
|
4267
|
+
}
|
|
4268
|
+
function addWarning(errors, filePath, location, message) {
|
|
4269
|
+
errors.push({ severity: "warning", filePath, location, message });
|
|
4270
|
+
}
|
|
4271
|
+
function addError(errors, filePath, location, message) {
|
|
4272
|
+
errors.push({ severity: "error", filePath, location, message });
|
|
4273
|
+
}
|
|
4274
|
+
function isPlainObject(value) {
|
|
4275
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4276
|
+
}
|
|
4277
|
+
function isGitRemoteUrlValue(value) {
|
|
4278
|
+
return /^(https?:\/\/|ssh:\/\/|git@|file:\/\/).+/.test(value.trim());
|
|
4279
|
+
}
|
|
4280
|
+
function validateProjectRepoConfig(errors, filePath, projectRecord, location) {
|
|
4281
|
+
if (projectRecord.source !== void 0) {
|
|
4282
|
+
addError(
|
|
4283
|
+
errors,
|
|
4284
|
+
filePath,
|
|
4285
|
+
`${location}.source`,
|
|
4286
|
+
`Field '${location}.source' was removed. Move 'source.url' to '${location}.repo.url', move 'source.ref' to '${location}.repo.branch', and set '${location}.repo.path' to the local checkout path.`
|
|
4287
|
+
);
|
|
4288
|
+
}
|
|
4289
|
+
if (projectRecord.repository !== void 0) {
|
|
4290
|
+
addError(
|
|
4291
|
+
errors,
|
|
4292
|
+
filePath,
|
|
4293
|
+
`${location}.repository`,
|
|
4294
|
+
`Field '${location}.repository' was removed. Use '${location}.repo.url' with a Git remote URL instead.`
|
|
4295
|
+
);
|
|
4296
|
+
}
|
|
4297
|
+
if (projectRecord.repo !== void 0) {
|
|
4298
|
+
if (!isPlainObject(projectRecord.repo)) {
|
|
4299
|
+
addError(
|
|
4300
|
+
errors,
|
|
4100
4301
|
filePath,
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4302
|
+
`${location}.repo`,
|
|
4303
|
+
`Field '${location}.repo' must be an object with path, optional url, and optional branch.`
|
|
4304
|
+
);
|
|
4305
|
+
return;
|
|
4104
4306
|
}
|
|
4105
|
-
|
|
4106
|
-
|
|
4107
|
-
|
|
4307
|
+
for (const flatField of ["path", "repo_url", "ref"]) {
|
|
4308
|
+
if (projectRecord[flatField] !== void 0) {
|
|
4309
|
+
addError(
|
|
4310
|
+
errors,
|
|
4311
|
+
filePath,
|
|
4312
|
+
`${location}.${flatField}`,
|
|
4313
|
+
`Do not mix '${location}.${flatField}' with '${location}.repo'. Move source repo fields under '${location}.repo'.`
|
|
4314
|
+
);
|
|
4315
|
+
}
|
|
4316
|
+
}
|
|
4317
|
+
validateRequiredString(errors, filePath, projectRecord.repo.path, `${location}.repo.path`);
|
|
4318
|
+
if (projectRecord.repo.url !== void 0) {
|
|
4319
|
+
validateGitRemoteUrl(errors, filePath, projectRecord.repo.url, `${location}.repo.url`);
|
|
4320
|
+
}
|
|
4321
|
+
if (projectRecord.repo.branch !== void 0) {
|
|
4322
|
+
validateRequiredString(
|
|
4323
|
+
errors,
|
|
4108
4324
|
filePath,
|
|
4109
|
-
|
|
4110
|
-
|
|
4111
|
-
|
|
4325
|
+
projectRecord.repo.branch,
|
|
4326
|
+
`${location}.repo.branch`
|
|
4327
|
+
);
|
|
4112
4328
|
}
|
|
4113
|
-
if (projectRecord.
|
|
4114
|
-
|
|
4329
|
+
if (projectRecord.repo.ref !== void 0) {
|
|
4330
|
+
addError(
|
|
4331
|
+
errors,
|
|
4332
|
+
filePath,
|
|
4333
|
+
`${location}.repo.ref`,
|
|
4334
|
+
`Field '${location}.repo.ref' is not supported. Use '${location}.repo.branch'.`
|
|
4335
|
+
);
|
|
4115
4336
|
}
|
|
4116
|
-
if (projectRecord.
|
|
4117
|
-
|
|
4337
|
+
if (projectRecord.repo.remote !== void 0) {
|
|
4338
|
+
addError(
|
|
4339
|
+
errors,
|
|
4340
|
+
filePath,
|
|
4341
|
+
`${location}.repo.remote`,
|
|
4342
|
+
`Use '${location}.repo.url' for the source Git URL. '${location}.repo.remote' is only valid inside results repo config.`
|
|
4343
|
+
);
|
|
4118
4344
|
}
|
|
4119
|
-
|
|
4120
|
-
}
|
|
4345
|
+
return;
|
|
4346
|
+
}
|
|
4347
|
+
validateRequiredString(errors, filePath, projectRecord.path, `${location}.path`);
|
|
4348
|
+
addWarning(
|
|
4349
|
+
errors,
|
|
4350
|
+
filePath,
|
|
4351
|
+
`${location}.path`,
|
|
4352
|
+
`Field '${location}.path' is deprecated. Use '${location}.repo.path'. Existing flat project entries still load and are written back in nested form.`
|
|
4353
|
+
);
|
|
4354
|
+
if (projectRecord.repo_url !== void 0) {
|
|
4355
|
+
validateGitRemoteUrl(errors, filePath, projectRecord.repo_url, `${location}.repo_url`);
|
|
4356
|
+
addWarning(
|
|
4357
|
+
errors,
|
|
4358
|
+
filePath,
|
|
4359
|
+
`${location}.repo_url`,
|
|
4360
|
+
`Field '${location}.repo_url' is deprecated. Use '${location}.repo.url'.`
|
|
4361
|
+
);
|
|
4362
|
+
}
|
|
4363
|
+
if (projectRecord.ref !== void 0) {
|
|
4364
|
+
validateRequiredString(errors, filePath, projectRecord.ref, `${location}.ref`);
|
|
4365
|
+
addWarning(
|
|
4366
|
+
errors,
|
|
4367
|
+
filePath,
|
|
4368
|
+
`${location}.ref`,
|
|
4369
|
+
`Field '${location}.ref' is deprecated. Use '${location}.repo.branch'.`
|
|
4370
|
+
);
|
|
4371
|
+
}
|
|
4121
4372
|
}
|
|
4122
4373
|
function validateRequiredString(errors, filePath, value, location) {
|
|
4123
4374
|
if (typeof value !== "string" || value.trim().length === 0) {
|
|
@@ -4140,7 +4391,7 @@ function validateGitRemoteUrl(errors, filePath, value, location) {
|
|
|
4140
4391
|
return;
|
|
4141
4392
|
}
|
|
4142
4393
|
const repoUrl = value.trim();
|
|
4143
|
-
if (
|
|
4394
|
+
if (!isGitRemoteUrlValue(repoUrl)) {
|
|
4144
4395
|
errors.push({
|
|
4145
4396
|
severity: "error",
|
|
4146
4397
|
filePath,
|
|
@@ -4149,168 +4400,117 @@ function validateGitRemoteUrl(errors, filePath, value, location) {
|
|
|
4149
4400
|
});
|
|
4150
4401
|
}
|
|
4151
4402
|
}
|
|
4152
|
-
function
|
|
4153
|
-
if (
|
|
4403
|
+
function validateResultsRepoBlock(errors, filePath, rawRepo, location) {
|
|
4404
|
+
if (!isPlainObject(rawRepo)) {
|
|
4405
|
+
addError(errors, filePath, location, `Field '${location}' must be an object`);
|
|
4154
4406
|
return;
|
|
4155
4407
|
}
|
|
4156
|
-
|
|
4157
|
-
|
|
4158
|
-
|
|
4408
|
+
const repoRecord = rawRepo;
|
|
4409
|
+
const hasUrl = repoRecord.url !== void 0;
|
|
4410
|
+
const hasRemote = repoRecord.remote !== void 0;
|
|
4411
|
+
const hasPath = repoRecord.path !== void 0;
|
|
4412
|
+
if (!hasRemote && !hasUrl && !hasPath) {
|
|
4413
|
+
addError(errors, filePath, location, `Field '${location}' must set remote or path`);
|
|
4414
|
+
}
|
|
4415
|
+
if (hasRemote && hasUrl) {
|
|
4416
|
+
addError(
|
|
4417
|
+
errors,
|
|
4159
4418
|
filePath,
|
|
4160
4419
|
location,
|
|
4161
|
-
|
|
4162
|
-
|
|
4163
|
-
return;
|
|
4420
|
+
`Field '${location}' must set only one remote endpoint. Use '${location}.remote'.`
|
|
4421
|
+
);
|
|
4164
4422
|
}
|
|
4165
|
-
|
|
4166
|
-
|
|
4167
|
-
mode: `Remove '${location}.mode'; project results use '${location}.repo_url' as the Git remote URL.`,
|
|
4168
|
-
repo: `Field '${location}.repo' was removed. Use '${location}.repo_url' with a Git remote URL instead.`,
|
|
4169
|
-
repository: `Field '${location}.repository' was removed. Use '${location}.repo_url' with a Git remote URL instead.`,
|
|
4170
|
-
local_path: `Field '${location}.local_path' was removed. Use '${location}.path' for the local clone path instead.`,
|
|
4171
|
-
auto_push: `Field '${location}.auto_push' was removed. Use '${location}.sync.auto_push' instead.`
|
|
4172
|
-
};
|
|
4173
|
-
for (const [field, message] of Object.entries(removedFields)) {
|
|
4174
|
-
if (resultsRecord[field] !== void 0) {
|
|
4175
|
-
errors.push({
|
|
4176
|
-
severity: "error",
|
|
4177
|
-
filePath,
|
|
4178
|
-
location: `${location}.${field}`,
|
|
4179
|
-
message
|
|
4180
|
-
});
|
|
4181
|
-
}
|
|
4423
|
+
if (hasRemote) {
|
|
4424
|
+
validateGitRemoteUrl(errors, filePath, repoRecord.remote, `${location}.remote`);
|
|
4182
4425
|
}
|
|
4183
|
-
|
|
4184
|
-
|
|
4185
|
-
|
|
4186
|
-
|
|
4187
|
-
severity: "error",
|
|
4188
|
-
filePath,
|
|
4189
|
-
location,
|
|
4190
|
-
message: `Field '${location}' must set only one of repo_url or repo_path`
|
|
4191
|
-
});
|
|
4192
|
-
} else if (hasRepoUrl) {
|
|
4193
|
-
validateGitRemoteUrl(errors, filePath, resultsRecord.repo_url, `${location}.repo_url`);
|
|
4194
|
-
} else if (hasRepoPath) {
|
|
4195
|
-
validateRequiredString(errors, filePath, resultsRecord.repo_path, `${location}.repo_path`);
|
|
4196
|
-
} else {
|
|
4197
|
-
errors.push({
|
|
4198
|
-
severity: "error",
|
|
4426
|
+
if (hasUrl) {
|
|
4427
|
+
validateGitRemoteUrl(errors, filePath, repoRecord.url, `${location}.url`);
|
|
4428
|
+
addWarning(
|
|
4429
|
+
errors,
|
|
4199
4430
|
filePath,
|
|
4200
|
-
location
|
|
4201
|
-
|
|
4202
|
-
|
|
4431
|
+
`${location}.url`,
|
|
4432
|
+
`Field '${location}.url' is accepted for compatibility. Use '${location}.remote' for the Git remote URL.`
|
|
4433
|
+
);
|
|
4203
4434
|
}
|
|
4204
|
-
if (
|
|
4205
|
-
errors.
|
|
4206
|
-
severity: "error",
|
|
4207
|
-
filePath,
|
|
4208
|
-
location: `${location}.branch`,
|
|
4209
|
-
message: `Field '${location}.branch' must be a non-empty string`
|
|
4210
|
-
});
|
|
4435
|
+
if (hasPath) {
|
|
4436
|
+
validateRequiredString(errors, filePath, repoRecord.path, `${location}.path`);
|
|
4211
4437
|
}
|
|
4212
|
-
if (
|
|
4213
|
-
|
|
4214
|
-
|
|
4438
|
+
if (repoRecord.branch !== void 0 && (typeof repoRecord.branch !== "string" || repoRecord.branch.trim().length === 0)) {
|
|
4439
|
+
addError(
|
|
4440
|
+
errors,
|
|
4215
4441
|
filePath,
|
|
4216
|
-
|
|
4217
|
-
|
|
4218
|
-
|
|
4442
|
+
`${location}.branch`,
|
|
4443
|
+
`Field '${location}.branch' must be a non-empty string`
|
|
4444
|
+
);
|
|
4219
4445
|
}
|
|
4220
|
-
|
|
4221
|
-
|
|
4222
|
-
|
|
4223
|
-
|
|
4224
|
-
|
|
4225
|
-
|
|
4226
|
-
|
|
4227
|
-
}
|
|
4228
|
-
|
|
4229
|
-
errors.push({
|
|
4230
|
-
severity: "error",
|
|
4231
|
-
filePath,
|
|
4232
|
-
location: `${location}.path`,
|
|
4233
|
-
message: `'${location}.path' must be an absolute or home-relative filesystem path (e.g., ~/data/agentv-results).`
|
|
4234
|
-
});
|
|
4235
|
-
}
|
|
4446
|
+
}
|
|
4447
|
+
function validateResultsSyncAndBranchPrefix(errors, filePath, resultsRecord, location) {
|
|
4448
|
+
if (resultsRecord.auto_push !== void 0 && typeof resultsRecord.auto_push !== "boolean") {
|
|
4449
|
+
addError(
|
|
4450
|
+
errors,
|
|
4451
|
+
filePath,
|
|
4452
|
+
`${location}.auto_push`,
|
|
4453
|
+
`Field '${location}.auto_push' must be a boolean`
|
|
4454
|
+
);
|
|
4236
4455
|
}
|
|
4237
4456
|
if (resultsRecord.sync !== void 0) {
|
|
4238
4457
|
if (typeof resultsRecord.sync !== "object" || resultsRecord.sync === null || Array.isArray(resultsRecord.sync)) {
|
|
4239
|
-
errors.
|
|
4240
|
-
severity: "error",
|
|
4241
|
-
filePath,
|
|
4242
|
-
location: `${location}.sync`,
|
|
4243
|
-
message: `Field '${location}.sync' must be an object`
|
|
4244
|
-
});
|
|
4458
|
+
addError(errors, filePath, `${location}.sync`, `Field '${location}.sync' must be an object`);
|
|
4245
4459
|
} else {
|
|
4246
4460
|
const syncRecord = resultsRecord.sync;
|
|
4247
4461
|
if (syncRecord.auto_push !== void 0 && typeof syncRecord.auto_push !== "boolean") {
|
|
4248
|
-
|
|
4249
|
-
|
|
4462
|
+
addError(
|
|
4463
|
+
errors,
|
|
4250
4464
|
filePath,
|
|
4251
|
-
|
|
4252
|
-
|
|
4253
|
-
|
|
4465
|
+
`${location}.sync.auto_push`,
|
|
4466
|
+
`Field '${location}.sync.auto_push' must be a boolean`
|
|
4467
|
+
);
|
|
4254
4468
|
}
|
|
4255
4469
|
if (syncRecord.require_push !== void 0 && typeof syncRecord.require_push !== "boolean") {
|
|
4256
|
-
|
|
4257
|
-
|
|
4470
|
+
addError(
|
|
4471
|
+
errors,
|
|
4258
4472
|
filePath,
|
|
4259
|
-
|
|
4260
|
-
|
|
4261
|
-
|
|
4473
|
+
`${location}.sync.require_push`,
|
|
4474
|
+
`Field '${location}.sync.require_push' must be a boolean`
|
|
4475
|
+
);
|
|
4476
|
+
}
|
|
4477
|
+
if (syncRecord.push_conflict_policy !== void 0 && syncRecord.push_conflict_policy !== "block" && syncRecord.push_conflict_policy !== "backup_and_force_push") {
|
|
4478
|
+
addError(
|
|
4479
|
+
errors,
|
|
4480
|
+
filePath,
|
|
4481
|
+
`${location}.sync.push_conflict_policy`,
|
|
4482
|
+
`Field '${location}.sync.push_conflict_policy' must be 'block' or 'backup_and_force_push'`
|
|
4483
|
+
);
|
|
4262
4484
|
}
|
|
4263
4485
|
}
|
|
4264
4486
|
}
|
|
4265
4487
|
if (resultsRecord.branch_prefix !== void 0 && (typeof resultsRecord.branch_prefix !== "string" || resultsRecord.branch_prefix.trim().length === 0)) {
|
|
4266
|
-
|
|
4267
|
-
|
|
4488
|
+
addError(
|
|
4489
|
+
errors,
|
|
4268
4490
|
filePath,
|
|
4269
|
-
|
|
4270
|
-
|
|
4271
|
-
|
|
4491
|
+
`${location}.branch_prefix`,
|
|
4492
|
+
`Field '${location}.branch_prefix' must be a non-empty string`
|
|
4493
|
+
);
|
|
4272
4494
|
}
|
|
4273
4495
|
}
|
|
4274
|
-
function
|
|
4275
|
-
|
|
4276
|
-
return;
|
|
4277
|
-
}
|
|
4278
|
-
if (typeof rawResults !== "object" || rawResults === null || Array.isArray(rawResults)) {
|
|
4279
|
-
errors.push({
|
|
4280
|
-
severity: "error",
|
|
4281
|
-
filePath,
|
|
4282
|
-
location,
|
|
4283
|
-
message: `Field '${location}' must be an object`
|
|
4284
|
-
});
|
|
4285
|
-
return;
|
|
4286
|
-
}
|
|
4287
|
-
const resultsRecord = rawResults;
|
|
4288
|
-
if (resultsRecord.mode !== void 0 && resultsRecord.mode !== "github") {
|
|
4289
|
-
errors.push({
|
|
4290
|
-
severity: "error",
|
|
4291
|
-
filePath,
|
|
4292
|
-
location: `${location}.mode`,
|
|
4293
|
-
message: `Field '${location}.mode' must be 'github'`
|
|
4294
|
-
});
|
|
4295
|
-
}
|
|
4296
|
-
const hasLegacyRepo = resultsRecord.repo !== void 0;
|
|
4496
|
+
function validateFlatResultsRepoConfig(errors, filePath, resultsRecord, location, options) {
|
|
4497
|
+
const hasLegacyRepo = typeof resultsRecord.repo === "string";
|
|
4297
4498
|
const hasRepoUrl = resultsRecord.repo_url !== void 0;
|
|
4298
4499
|
const hasRepoPath = resultsRecord.repo_path !== void 0;
|
|
4299
|
-
const sourceCount = [
|
|
4500
|
+
const sourceCount = [
|
|
4501
|
+
options.allowLegacyRepoString && hasLegacyRepo,
|
|
4502
|
+
hasRepoUrl,
|
|
4503
|
+
hasRepoPath
|
|
4504
|
+
].filter(Boolean).length;
|
|
4300
4505
|
if (sourceCount === 0) {
|
|
4301
|
-
errors.
|
|
4302
|
-
severity: "error",
|
|
4303
|
-
filePath,
|
|
4304
|
-
location,
|
|
4305
|
-
message: `Field '${location}' must set repo_url/repo or repo_path`
|
|
4306
|
-
});
|
|
4506
|
+
addError(errors, filePath, location, `Field '${location}' must set repo.remote or repo.path`);
|
|
4307
4507
|
} else if (sourceCount > 1) {
|
|
4308
|
-
|
|
4309
|
-
|
|
4508
|
+
addError(
|
|
4509
|
+
errors,
|
|
4310
4510
|
filePath,
|
|
4311
4511
|
location,
|
|
4312
|
-
|
|
4313
|
-
|
|
4512
|
+
`Field '${location}' must set only one results repo source. Use '${location}.repo.remote' for a managed clone or '${location}.repo.path' for an existing local checkout.`
|
|
4513
|
+
);
|
|
4314
4514
|
} else if (hasLegacyRepo) {
|
|
4315
4515
|
validateRequiredString(errors, filePath, resultsRecord.repo, `${location}.repo`);
|
|
4316
4516
|
} else if (hasRepoUrl) {
|
|
@@ -4319,85 +4519,151 @@ function validateResultsConfig(errors, filePath, rawResults, location) {
|
|
|
4319
4519
|
validateRequiredString(errors, filePath, resultsRecord.repo_path, `${location}.repo_path`);
|
|
4320
4520
|
}
|
|
4321
4521
|
if (resultsRecord.branch !== void 0 && (typeof resultsRecord.branch !== "string" || resultsRecord.branch.trim().length === 0)) {
|
|
4322
|
-
|
|
4323
|
-
|
|
4522
|
+
addError(
|
|
4523
|
+
errors,
|
|
4324
4524
|
filePath,
|
|
4325
|
-
|
|
4326
|
-
|
|
4327
|
-
|
|
4525
|
+
`${location}.branch`,
|
|
4526
|
+
`Field '${location}.branch' must be a non-empty string`
|
|
4527
|
+
);
|
|
4328
4528
|
}
|
|
4329
4529
|
if (resultsRecord.remote !== void 0 && (typeof resultsRecord.remote !== "string" || resultsRecord.remote.trim().length === 0)) {
|
|
4330
|
-
|
|
4331
|
-
|
|
4530
|
+
addError(
|
|
4531
|
+
errors,
|
|
4332
4532
|
filePath,
|
|
4333
|
-
|
|
4334
|
-
|
|
4335
|
-
|
|
4533
|
+
`${location}.remote`,
|
|
4534
|
+
`Field '${location}.remote' must be a non-empty string`
|
|
4535
|
+
);
|
|
4336
4536
|
}
|
|
4337
4537
|
if (resultsRecord.path !== void 0) {
|
|
4338
4538
|
if (typeof resultsRecord.path !== "string" || resultsRecord.path.trim().length === 0) {
|
|
4339
|
-
|
|
4340
|
-
|
|
4539
|
+
addError(
|
|
4540
|
+
errors,
|
|
4341
4541
|
filePath,
|
|
4342
|
-
|
|
4343
|
-
|
|
4344
|
-
|
|
4542
|
+
`${location}.path`,
|
|
4543
|
+
`Field '${location}.path' must be a non-empty string`
|
|
4544
|
+
);
|
|
4345
4545
|
} else {
|
|
4346
4546
|
const p = resultsRecord.path.trim();
|
|
4347
4547
|
if (!isFilesystemPath(p)) {
|
|
4348
|
-
|
|
4349
|
-
|
|
4548
|
+
addError(
|
|
4549
|
+
errors,
|
|
4350
4550
|
filePath,
|
|
4351
|
-
|
|
4352
|
-
|
|
4353
|
-
|
|
4551
|
+
`${location}.path`,
|
|
4552
|
+
`'${location}.path' must be an absolute or home-relative filesystem path (e.g., ~/data/agentv-results). Found: '${p}'. Remove 'path' to use the default.`
|
|
4553
|
+
);
|
|
4354
4554
|
}
|
|
4355
4555
|
}
|
|
4356
4556
|
}
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4557
|
+
}
|
|
4558
|
+
function warnFlatResultsMigration(errors, filePath, resultsRecord, location) {
|
|
4559
|
+
const migrations = {
|
|
4560
|
+
repo: `${location}.repo.remote`,
|
|
4561
|
+
repo_url: `${location}.repo.remote`,
|
|
4562
|
+
repo_path: `${location}.repo.path`,
|
|
4563
|
+
branch: `${location}.repo.branch`,
|
|
4564
|
+
path: `${location}.repo.path`,
|
|
4565
|
+
auto_push: `${location}.sync.auto_push`,
|
|
4566
|
+
mode: "(remove this field)"
|
|
4567
|
+
};
|
|
4568
|
+
for (const [field, replacement] of Object.entries(migrations)) {
|
|
4569
|
+
if (resultsRecord[field] !== void 0) {
|
|
4570
|
+
addWarning(
|
|
4571
|
+
errors,
|
|
4572
|
+
filePath,
|
|
4573
|
+
`${location}.${field}`,
|
|
4574
|
+
`Field '${location}.${field}' is deprecated. Use '${replacement}' in the nested results repo schema.`
|
|
4575
|
+
);
|
|
4576
|
+
}
|
|
4577
|
+
}
|
|
4578
|
+
if (resultsRecord.remote !== void 0) {
|
|
4579
|
+
addWarning(
|
|
4580
|
+
errors,
|
|
4360
4581
|
filePath,
|
|
4361
|
-
|
|
4362
|
-
|
|
4363
|
-
|
|
4582
|
+
`${location}.remote`,
|
|
4583
|
+
`Field '${location}.remote' is a legacy local Git remote-name override. Prefer omitting it; nested '${location}.repo.remote' is the portable Git remote URL and AgentV manages local aliases automatically.`
|
|
4584
|
+
);
|
|
4364
4585
|
}
|
|
4365
|
-
|
|
4366
|
-
|
|
4367
|
-
|
|
4368
|
-
|
|
4586
|
+
}
|
|
4587
|
+
function validateResultsConfigBody(errors, filePath, rawResults, location, options) {
|
|
4588
|
+
if (rawResults === void 0) {
|
|
4589
|
+
return;
|
|
4590
|
+
}
|
|
4591
|
+
if (!isPlainObject(rawResults)) {
|
|
4592
|
+
addError(errors, filePath, location, `Field '${location}' must be an object`);
|
|
4593
|
+
return;
|
|
4594
|
+
}
|
|
4595
|
+
const resultsRecord = rawResults;
|
|
4596
|
+
if (resultsRecord.mode !== void 0) {
|
|
4597
|
+
if (options.projectScoped) {
|
|
4598
|
+
addError(
|
|
4599
|
+
errors,
|
|
4369
4600
|
filePath,
|
|
4370
|
-
|
|
4371
|
-
|
|
4372
|
-
|
|
4601
|
+
`${location}.mode`,
|
|
4602
|
+
`Remove '${location}.mode'; project results use '${location}.repo.remote' or '${location}.repo.path'.`
|
|
4603
|
+
);
|
|
4604
|
+
} else if (resultsRecord.mode !== "github") {
|
|
4605
|
+
addError(errors, filePath, `${location}.mode`, `Field '${location}.mode' must be 'github'`);
|
|
4606
|
+
}
|
|
4607
|
+
}
|
|
4608
|
+
for (const [field, message] of Object.entries({
|
|
4609
|
+
repository: `Field '${location}.repository' was removed. Use '${location}.repo.remote' with a Git remote URL instead.`,
|
|
4610
|
+
local_path: `Field '${location}.local_path' was removed. Use '${location}.repo.path' for the local clone path instead.`
|
|
4611
|
+
})) {
|
|
4612
|
+
if (resultsRecord[field] !== void 0) {
|
|
4613
|
+
addError(errors, filePath, `${location}.${field}`, message);
|
|
4614
|
+
}
|
|
4615
|
+
}
|
|
4616
|
+
if (options.projectScoped && resultsRecord.auto_push !== void 0) {
|
|
4617
|
+
addError(
|
|
4618
|
+
errors,
|
|
4619
|
+
filePath,
|
|
4620
|
+
`${location}.auto_push`,
|
|
4621
|
+
`Field '${location}.auto_push' was removed. Use '${location}.sync.auto_push' instead.`
|
|
4622
|
+
);
|
|
4623
|
+
}
|
|
4624
|
+
const hasNestedRepo = isPlainObject(resultsRecord.repo);
|
|
4625
|
+
if (resultsRecord.repo !== void 0 && !hasNestedRepo) {
|
|
4626
|
+
if (typeof resultsRecord.repo === "string" && options.allowLegacyRepoString) {
|
|
4373
4627
|
} else {
|
|
4374
|
-
|
|
4375
|
-
|
|
4376
|
-
|
|
4377
|
-
|
|
4378
|
-
|
|
4379
|
-
|
|
4380
|
-
|
|
4381
|
-
|
|
4382
|
-
|
|
4383
|
-
|
|
4384
|
-
|
|
4385
|
-
|
|
4628
|
+
addError(
|
|
4629
|
+
errors,
|
|
4630
|
+
filePath,
|
|
4631
|
+
`${location}.repo`,
|
|
4632
|
+
`Field '${location}.repo' must be an object. Use '${location}.repo.remote' for a Git remote URL or '${location}.repo.path' for an existing local checkout.`
|
|
4633
|
+
);
|
|
4634
|
+
}
|
|
4635
|
+
}
|
|
4636
|
+
if (hasNestedRepo) {
|
|
4637
|
+
for (const flatField of ["repo_url", "repo_path", "branch", "remote", "path"]) {
|
|
4638
|
+
if (resultsRecord[flatField] !== void 0) {
|
|
4639
|
+
addError(
|
|
4640
|
+
errors,
|
|
4386
4641
|
filePath,
|
|
4387
|
-
|
|
4388
|
-
|
|
4389
|
-
|
|
4642
|
+
`${location}.${flatField}`,
|
|
4643
|
+
`Do not mix '${location}.${flatField}' with '${location}.repo'. Move results repo fields under '${location}.repo'.`
|
|
4644
|
+
);
|
|
4390
4645
|
}
|
|
4391
4646
|
}
|
|
4392
|
-
|
|
4393
|
-
|
|
4394
|
-
errors
|
|
4395
|
-
|
|
4396
|
-
filePath,
|
|
4397
|
-
location: `${location}.branch_prefix`,
|
|
4398
|
-
message: `Field '${location}.branch_prefix' must be a non-empty string`
|
|
4647
|
+
validateResultsRepoBlock(errors, filePath, resultsRecord.repo, `${location}.repo`);
|
|
4648
|
+
} else {
|
|
4649
|
+
validateFlatResultsRepoConfig(errors, filePath, resultsRecord, location, {
|
|
4650
|
+
allowLegacyRepoString: options.allowLegacyRepoString
|
|
4399
4651
|
});
|
|
4652
|
+
warnFlatResultsMigration(errors, filePath, resultsRecord, location);
|
|
4400
4653
|
}
|
|
4654
|
+
validateResultsSyncAndBranchPrefix(errors, filePath, resultsRecord, location);
|
|
4655
|
+
}
|
|
4656
|
+
function validateProjectResultsConfig(errors, filePath, rawResults, location) {
|
|
4657
|
+
validateResultsConfigBody(errors, filePath, rawResults, location, {
|
|
4658
|
+
allowLegacyRepoString: false,
|
|
4659
|
+
projectScoped: true
|
|
4660
|
+
});
|
|
4661
|
+
}
|
|
4662
|
+
function validateResultsConfig(errors, filePath, rawResults, location) {
|
|
4663
|
+
validateResultsConfigBody(errors, filePath, rawResults, location, {
|
|
4664
|
+
allowLegacyRepoString: true,
|
|
4665
|
+
projectScoped: false
|
|
4666
|
+
});
|
|
4401
4667
|
}
|
|
4402
4668
|
function isFilesystemPath(p) {
|
|
4403
4669
|
return p.startsWith("/") || p.startsWith("~/") || p.startsWith("~\\") || p === "~" || /^[A-Za-z]:[/\\]/.test(p);
|
|
@@ -5110,6 +5376,12 @@ function normalizeOutputMessages(cliValue) {
|
|
|
5110
5376
|
function trimOutputMessages(output, _outputMessages) {
|
|
5111
5377
|
return output;
|
|
5112
5378
|
}
|
|
5379
|
+
function prepareResultForJsonl(result, options) {
|
|
5380
|
+
return {
|
|
5381
|
+
...result,
|
|
5382
|
+
output: trimOutputMessages(result.output, options.outputMessages)
|
|
5383
|
+
};
|
|
5384
|
+
}
|
|
5113
5385
|
function normalizeOptions(rawOptions, config, yamlExecution) {
|
|
5114
5386
|
const cliWorkers = normalizeOptionalNumber(rawOptions.workers);
|
|
5115
5387
|
const configWorkers = config?.execution?.workers;
|
|
@@ -5240,8 +5512,8 @@ async function ensureFileExists(filePath, description) {
|
|
|
5240
5512
|
throw new Error(`${description} not found: ${filePath}`);
|
|
5241
5513
|
}
|
|
5242
5514
|
}
|
|
5243
|
-
function buildDefaultOutputPathForExperiment(cwd, experiment) {
|
|
5244
|
-
const runDir =
|
|
5515
|
+
function buildDefaultOutputPathForExperiment(cwd, experiment, runDirName) {
|
|
5516
|
+
const runDir = buildDefaultRunDirFromName(cwd, experiment, runDirName);
|
|
5245
5517
|
mkdirSync2(runDir, { recursive: true });
|
|
5246
5518
|
return path14.join(runDir, "index.jsonl");
|
|
5247
5519
|
}
|
|
@@ -5597,11 +5869,7 @@ async function runSingleEvalFile(params) {
|
|
|
5597
5869
|
streamingObserver?.completeFromResult?.(result);
|
|
5598
5870
|
streamingObserver?.finalizeEvalCase(result.score, result.error);
|
|
5599
5871
|
const resultWithMetadata = withSourceMetadata(result, testFilePath, options);
|
|
5600
|
-
const
|
|
5601
|
-
const trimmedResult = {
|
|
5602
|
-
...resultWithMetadata,
|
|
5603
|
-
output: trimmedOutput
|
|
5604
|
-
};
|
|
5872
|
+
const trimmedResult = prepareResultForJsonl(resultWithMetadata, options);
|
|
5605
5873
|
await outputWriter.append(trimmedResult);
|
|
5606
5874
|
if (otelExporter && !streamingObserver) {
|
|
5607
5875
|
try {
|
|
@@ -5649,9 +5917,6 @@ async function runSingleEvalFile(params) {
|
|
|
5649
5917
|
}
|
|
5650
5918
|
async function runEvalCommand(input) {
|
|
5651
5919
|
const cwd = process.cwd();
|
|
5652
|
-
if (!process.env.AGENTV_RUN_TIMESTAMP) {
|
|
5653
|
-
process.env.AGENTV_RUN_TIMESTAMP = (/* @__PURE__ */ new Date()).toISOString().replace(/:/g, "-").replace(/\./g, "-");
|
|
5654
|
-
}
|
|
5655
5920
|
let config = null;
|
|
5656
5921
|
try {
|
|
5657
5922
|
config = await loadTsConfig(cwd);
|
|
@@ -5770,19 +6035,24 @@ async function runEvalCommand(input) {
|
|
|
5770
6035
|
const explicitDir = options.outputDir;
|
|
5771
6036
|
let runDir;
|
|
5772
6037
|
let outputPath;
|
|
6038
|
+
const runDirName = process.env.AGENTV_RUN_TIMESTAMP?.trim() || createRunDirName();
|
|
5773
6039
|
if (explicitDir) {
|
|
5774
6040
|
runDir = path14.resolve(explicitDir);
|
|
5775
6041
|
mkdirSync2(runDir, { recursive: true });
|
|
5776
6042
|
outputPath = path14.join(runDir, "index.jsonl");
|
|
5777
6043
|
} else {
|
|
5778
|
-
outputPath = buildDefaultOutputPathForExperiment(cwd, options.experiment);
|
|
6044
|
+
outputPath = buildDefaultOutputPathForExperiment(cwd, options.experiment, runDirName);
|
|
5779
6045
|
runDir = path14.dirname(outputPath);
|
|
5780
6046
|
}
|
|
6047
|
+
if (!process.env.AGENTV_RUN_TIMESTAMP) {
|
|
6048
|
+
process.env.AGENTV_RUN_TIMESTAMP = path14.basename(runDir);
|
|
6049
|
+
}
|
|
6050
|
+
process.env.AGENTV_RUN_DIR = runDir;
|
|
5781
6051
|
let otelExporter = null;
|
|
5782
6052
|
const useFileExport = !!options.otelFile;
|
|
5783
6053
|
if (options.exportOtel || useFileExport) {
|
|
5784
6054
|
try {
|
|
5785
|
-
const { OtelTraceExporter } = await import("./dist-
|
|
6055
|
+
const { OtelTraceExporter } = await import("./dist-L4YCQOQ6.js");
|
|
5786
6056
|
let endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
|
|
5787
6057
|
let headers = {};
|
|
5788
6058
|
let resourceAttributes = {};
|
|
@@ -5987,7 +6257,7 @@ async function runEvalCommand(input) {
|
|
|
5987
6257
|
const activeTestFiles = resolvedTestFiles.filter((f) => fileMetadata.has(f));
|
|
5988
6258
|
let transcriptProviderFactory;
|
|
5989
6259
|
if (options.transcript) {
|
|
5990
|
-
const { TranscriptProvider } = await import("./dist-
|
|
6260
|
+
const { TranscriptProvider } = await import("./dist-L4YCQOQ6.js");
|
|
5991
6261
|
const transcriptProvider = await TranscriptProvider.fromFile(options.transcript);
|
|
5992
6262
|
const totalTests = [...fileMetadata.values()].reduce(
|
|
5993
6263
|
(sum, meta) => sum + meta.testCases.length,
|
|
@@ -6211,7 +6481,7 @@ ${formatRequiredVersionFailureNote(requiredVersionCheck)}`);
|
|
|
6211
6481
|
);
|
|
6212
6482
|
const taskBundleTargets = buildTaskBundleTargetSelections(activeTestFiles, fileMetadata);
|
|
6213
6483
|
if (isResumeAppend) {
|
|
6214
|
-
const { writePerTestArtifacts } = await import("./artifact-writer-
|
|
6484
|
+
const { writePerTestArtifacts } = await import("./artifact-writer-72MQQELS.js");
|
|
6215
6485
|
await writePerTestArtifacts(allResults, runDir, {
|
|
6216
6486
|
experiment: normalizeExperimentName(options.experiment),
|
|
6217
6487
|
cwd,
|
|
@@ -6394,9 +6664,13 @@ export {
|
|
|
6394
6664
|
findRepoRoot,
|
|
6395
6665
|
package_default,
|
|
6396
6666
|
RESULT_INDEX_FILENAME,
|
|
6667
|
+
isReservedResultsNamespace,
|
|
6668
|
+
normalizeExperimentName,
|
|
6397
6669
|
createRunDirName,
|
|
6670
|
+
buildDefaultRunDirFromName,
|
|
6398
6671
|
buildDefaultRunDir,
|
|
6399
6672
|
resolveRunManifestPath,
|
|
6673
|
+
relativeRunPathFromCwd,
|
|
6400
6674
|
parseResultManifest,
|
|
6401
6675
|
resolveResultSourcePath,
|
|
6402
6676
|
loadManifestResults,
|
|
@@ -6430,9 +6704,13 @@ export {
|
|
|
6430
6704
|
formatCost,
|
|
6431
6705
|
formatSize,
|
|
6432
6706
|
formatScore,
|
|
6707
|
+
TagRevisionConflictError,
|
|
6708
|
+
assertExpectedTagRevision,
|
|
6709
|
+
materializeRunState,
|
|
6433
6710
|
readRunTags,
|
|
6434
6711
|
writeRunTags,
|
|
6435
|
-
|
|
6712
|
+
normalizeTags,
|
|
6713
|
+
loadNormalizedResultsConfig,
|
|
6436
6714
|
isRemoteRunId,
|
|
6437
6715
|
getRemoteResultsStatus,
|
|
6438
6716
|
syncRemoteResults,
|
|
@@ -6450,4 +6728,4 @@ export {
|
|
|
6450
6728
|
getCategories,
|
|
6451
6729
|
filterByCategory
|
|
6452
6730
|
};
|
|
6453
|
-
//# sourceMappingURL=chunk-
|
|
6731
|
+
//# sourceMappingURL=chunk-CXHYTYHU.js.map
|