agentv 4.41.1 → 4.41.4-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/{artifact-writer-AMV64TWV.js → artifact-writer-GKRPI5WH.js} +4 -4
- package/dist/{chunk-LXXMQSAR.js → chunk-AHQWZGJE.js} +659 -286
- package/dist/chunk-AHQWZGJE.js.map +1 -0
- package/dist/{chunk-JQ6XRG7X.js → chunk-PC33TGIV.js} +22 -20
- package/dist/chunk-PC33TGIV.js.map +1 -0
- package/dist/{chunk-ENHX2CCS.js → chunk-QCF3LUI7.js} +269 -157
- package/dist/chunk-QCF3LUI7.js.map +1 -0
- package/dist/{chunk-A4J456KS.js → chunk-VBEWRK44.js} +429 -12
- package/dist/chunk-VBEWRK44.js.map +1 -0
- package/dist/{chunk-Z45FKRMJ.js → chunk-XDE2ZRII.js} +2 -2
- package/dist/cli.js +5 -5
- package/dist/{dist-X5P5IR65.js → dist-FDVFAGRG.js} +9 -47
- package/dist/index.js +5 -5
- package/dist/{interactive-OPSG4MPD.js → interactive-VNGVNNRF.js} +5 -5
- package/dist/skills/agentv-bench/references/environment-adaptation.md +1 -1
- package/dist/skills/agentv-bench/references/eval-yaml-spec.md +2 -2
- package/dist/skills/agentv-eval-writer/SKILL.md +4 -4
- package/dist/skills/agentv-eval-writer/references/custom-evaluators.md +2 -2
- package/dist/{ts-eval-loader-ZVL6CGTE-TZYZX3QS.js → ts-eval-loader-UPFWPMQO-SKZNUXPB.js} +2 -2
- package/package.json +1 -1
- package/dist/chunk-A4J456KS.js.map +0 -1
- package/dist/chunk-ENHX2CCS.js.map +0 -1
- package/dist/chunk-JQ6XRG7X.js.map +0 -1
- package/dist/chunk-LXXMQSAR.js.map +0 -1
- /package/dist/{artifact-writer-AMV64TWV.js.map → artifact-writer-GKRPI5WH.js.map} +0 -0
- /package/dist/{chunk-Z45FKRMJ.js.map → chunk-XDE2ZRII.js.map} +0 -0
- /package/dist/{dist-X5P5IR65.js.map → dist-FDVFAGRG.js.map} +0 -0
- /package/dist/{interactive-OPSG4MPD.js.map → interactive-VNGVNNRF.js.map} +0 -0
- /package/dist/{ts-eval-loader-ZVL6CGTE-TZYZX3QS.js.map → ts-eval-loader-UPFWPMQO-SKZNUXPB.js.map} +0 -0
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
deleteRunTags,
|
|
11
11
|
detectFileType,
|
|
12
12
|
discoverEvalFiles,
|
|
13
|
+
discoverTargetsFile,
|
|
13
14
|
enforceRequiredVersion,
|
|
14
15
|
ensureRemoteRunAvailable,
|
|
15
16
|
findRepoRoot,
|
|
@@ -37,6 +38,7 @@ import {
|
|
|
37
38
|
parseResultManifest,
|
|
38
39
|
readRemoteRunTagState,
|
|
39
40
|
readRunTags,
|
|
41
|
+
readTestSuiteTarget,
|
|
40
42
|
resolveEvalPaths,
|
|
41
43
|
resolveResultSourcePath,
|
|
42
44
|
resolveRunCacheFile,
|
|
@@ -54,11 +56,12 @@ import {
|
|
|
54
56
|
validateTargetsFile,
|
|
55
57
|
validateWorkspacePaths,
|
|
56
58
|
writeRunTags
|
|
57
|
-
} from "./chunk-
|
|
59
|
+
} from "./chunk-PC33TGIV.js";
|
|
58
60
|
import {
|
|
61
|
+
materializeEvalBundle,
|
|
59
62
|
toSnakeCaseDeep as toSnakeCaseDeep2,
|
|
60
63
|
writeArtifactsFromResults as writeArtifactsFromResults2
|
|
61
|
-
} from "./chunk-
|
|
64
|
+
} from "./chunk-VBEWRK44.js";
|
|
62
65
|
import {
|
|
63
66
|
DEFAULT_CATEGORY,
|
|
64
67
|
deriveCategory,
|
|
@@ -73,9 +76,10 @@ import {
|
|
|
73
76
|
syncProjects,
|
|
74
77
|
transpileEvalYamlFile,
|
|
75
78
|
trimBaselineResult
|
|
76
|
-
} from "./chunk-
|
|
79
|
+
} from "./chunk-XDE2ZRII.js";
|
|
77
80
|
import {
|
|
78
81
|
DEFAULT_THRESHOLD,
|
|
82
|
+
EXECUTION_TRACE_SCHEMA_VERSION,
|
|
79
83
|
addProject,
|
|
80
84
|
aggregateRunDir,
|
|
81
85
|
buildBenchmarkArtifact,
|
|
@@ -100,7 +104,9 @@ import {
|
|
|
100
104
|
parseCopilotEvents,
|
|
101
105
|
parseYamlValue,
|
|
102
106
|
readTargetDefinitions,
|
|
107
|
+
readTraceEnvelopeReplayRecords,
|
|
103
108
|
readTranscriptFile,
|
|
109
|
+
readTranscriptJsonl,
|
|
104
110
|
removeProject,
|
|
105
111
|
runContainsAllAssertion,
|
|
106
112
|
runContainsAnyAssertion,
|
|
@@ -117,8 +123,11 @@ import {
|
|
|
117
123
|
toSnakeCaseDeep,
|
|
118
124
|
toTranscriptJsonLines,
|
|
119
125
|
touchProject,
|
|
126
|
+
traceEnvelopeToTraceSummary,
|
|
127
|
+
traceEnvelopeToTranscriptMessages,
|
|
128
|
+
traceFromTranscriptJsonLines,
|
|
120
129
|
writeArtifactsFromResults
|
|
121
|
-
} from "./chunk-
|
|
130
|
+
} from "./chunk-QCF3LUI7.js";
|
|
122
131
|
import {
|
|
123
132
|
__commonJS,
|
|
124
133
|
__require,
|
|
@@ -879,7 +888,7 @@ var require_src = __commonJS({
|
|
|
879
888
|
});
|
|
880
889
|
|
|
881
890
|
// src/index.ts
|
|
882
|
-
import
|
|
891
|
+
import path35 from "node:path";
|
|
883
892
|
|
|
884
893
|
// ../../node_modules/.bun/chalk@5.6.2/node_modules/chalk/source/vendor/ansi-styles/index.js
|
|
885
894
|
var ANSI_BACKGROUND_OFFSET = 10;
|
|
@@ -3959,7 +3968,7 @@ import { mkdir as mkdir2, writeFile as writeFile2 } from "node:fs/promises";
|
|
|
3959
3968
|
import path3 from "node:path";
|
|
3960
3969
|
var ASSERTION_TEMPLATES = {
|
|
3961
3970
|
default: `#!/usr/bin/env bun
|
|
3962
|
-
import { defineAssertion } from '@agentv/
|
|
3971
|
+
import { defineAssertion } from '@agentv/sdk';
|
|
3963
3972
|
|
|
3964
3973
|
export default defineAssertion(({ output }) => {
|
|
3965
3974
|
// TODO: Implement your assertion logic
|
|
@@ -3972,7 +3981,7 @@ export default defineAssertion(({ output }) => {
|
|
|
3972
3981
|
});
|
|
3973
3982
|
`,
|
|
3974
3983
|
score: `#!/usr/bin/env bun
|
|
3975
|
-
import { defineAssertion } from '@agentv/
|
|
3984
|
+
import { defineAssertion } from '@agentv/sdk';
|
|
3976
3985
|
|
|
3977
3986
|
export default defineAssertion(({ output }) => {
|
|
3978
3987
|
// TODO: Implement your scoring logic (0.0 to 1.0)
|
|
@@ -4364,6 +4373,201 @@ async function findGraderScript(graderName, startDir) {
|
|
|
4364
4373
|
return null;
|
|
4365
4374
|
}
|
|
4366
4375
|
|
|
4376
|
+
// src/commands/eval/commands/bundle.ts
|
|
4377
|
+
import path7 from "node:path";
|
|
4378
|
+
function unique(values) {
|
|
4379
|
+
const result = [];
|
|
4380
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4381
|
+
for (const value of values) {
|
|
4382
|
+
const trimmed = value.trim();
|
|
4383
|
+
if (!trimmed || seen.has(trimmed)) {
|
|
4384
|
+
continue;
|
|
4385
|
+
}
|
|
4386
|
+
seen.add(trimmed);
|
|
4387
|
+
result.push(trimmed);
|
|
4388
|
+
}
|
|
4389
|
+
return result;
|
|
4390
|
+
}
|
|
4391
|
+
function targetReferenceNames(target) {
|
|
4392
|
+
const references = [];
|
|
4393
|
+
for (const key of ["use_target", "grader_target", "judge_target"]) {
|
|
4394
|
+
const value = target[key];
|
|
4395
|
+
if (typeof value === "string" && value.trim().length > 0 && !value.includes("${{")) {
|
|
4396
|
+
references.push(value.trim());
|
|
4397
|
+
}
|
|
4398
|
+
}
|
|
4399
|
+
const fallbackTargets = target.fallback_targets;
|
|
4400
|
+
if (Array.isArray(fallbackTargets)) {
|
|
4401
|
+
for (const value of fallbackTargets) {
|
|
4402
|
+
if (typeof value === "string" && value.trim().length > 0 && !value.includes("${{")) {
|
|
4403
|
+
references.push(value.trim());
|
|
4404
|
+
}
|
|
4405
|
+
}
|
|
4406
|
+
}
|
|
4407
|
+
return references;
|
|
4408
|
+
}
|
|
4409
|
+
function ensureTargetGraph(targetName2, definitions, targetsFilePath) {
|
|
4410
|
+
const byName = new Map(definitions.map((definition) => [definition.name, definition]));
|
|
4411
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4412
|
+
function visit(name, requestedBy) {
|
|
4413
|
+
if (seen.has(name)) {
|
|
4414
|
+
return;
|
|
4415
|
+
}
|
|
4416
|
+
const definition = byName.get(name);
|
|
4417
|
+
if (!definition) {
|
|
4418
|
+
const available = definitions.map((entry) => entry.name).sort().join(", ");
|
|
4419
|
+
const owner = requestedBy ? ` referenced by target '${requestedBy}'` : "";
|
|
4420
|
+
throw new Error(
|
|
4421
|
+
`Target '${name}'${owner} not found in ${targetsFilePath}. Available targets: ${available}`
|
|
4422
|
+
);
|
|
4423
|
+
}
|
|
4424
|
+
seen.add(name);
|
|
4425
|
+
for (const referenceName of targetReferenceNames(definition)) {
|
|
4426
|
+
visit(referenceName, name);
|
|
4427
|
+
}
|
|
4428
|
+
}
|
|
4429
|
+
visit(targetName2);
|
|
4430
|
+
}
|
|
4431
|
+
function definitionsWithEvalTargetRefs(definitions, targetRefs) {
|
|
4432
|
+
if (!targetRefs) {
|
|
4433
|
+
return definitions;
|
|
4434
|
+
}
|
|
4435
|
+
const result = [...definitions];
|
|
4436
|
+
for (const ref of targetRefs) {
|
|
4437
|
+
if (ref.use_target && !result.some((definition) => definition.name === ref.name)) {
|
|
4438
|
+
result.push({ name: ref.name, use_target: ref.use_target });
|
|
4439
|
+
}
|
|
4440
|
+
}
|
|
4441
|
+
return result;
|
|
4442
|
+
}
|
|
4443
|
+
function buildBundleExecution(options) {
|
|
4444
|
+
const targetRefsByName = new Map((options.targetRefs ?? []).map((ref) => [ref.name, ref]));
|
|
4445
|
+
const serializeTargetRef = (name) => {
|
|
4446
|
+
const ref = targetRefsByName.get(name);
|
|
4447
|
+
return ref?.hooks || ref?.use_target ? ref : name;
|
|
4448
|
+
};
|
|
4449
|
+
const singleTargetRef = options.targetNames[0] ? targetRefsByName.get(options.targetNames[0]) : void 0;
|
|
4450
|
+
const execution = options.targetNames.length === 1 && !singleTargetRef?.hooks && !singleTargetRef?.use_target ? { target: options.targetNames[0] } : {
|
|
4451
|
+
targets: options.targetNames.map((name) => serializeTargetRef(name))
|
|
4452
|
+
};
|
|
4453
|
+
if (options.workers !== void 0) {
|
|
4454
|
+
execution.workers = options.workers;
|
|
4455
|
+
}
|
|
4456
|
+
if (options.cache !== void 0) {
|
|
4457
|
+
execution.cache = options.cache;
|
|
4458
|
+
}
|
|
4459
|
+
if (options.cachePath !== void 0) {
|
|
4460
|
+
execution.cache_path = options.cachePath;
|
|
4461
|
+
}
|
|
4462
|
+
if (options.budgetUsd !== void 0) {
|
|
4463
|
+
execution.budget_usd = options.budgetUsd;
|
|
4464
|
+
}
|
|
4465
|
+
if (options.threshold !== void 0) {
|
|
4466
|
+
execution.threshold = options.threshold;
|
|
4467
|
+
}
|
|
4468
|
+
return execution;
|
|
4469
|
+
}
|
|
4470
|
+
var evalBundleCommand = command({
|
|
4471
|
+
name: "bundle",
|
|
4472
|
+
description: "Create a portable self-contained directory for an eval suite",
|
|
4473
|
+
args: {
|
|
4474
|
+
evalPath: positional({
|
|
4475
|
+
type: string,
|
|
4476
|
+
displayName: "eval",
|
|
4477
|
+
description: "Path or glob resolving to one eval file"
|
|
4478
|
+
}),
|
|
4479
|
+
out: option({
|
|
4480
|
+
type: string,
|
|
4481
|
+
long: "out",
|
|
4482
|
+
description: "Portable bundle output directory"
|
|
4483
|
+
}),
|
|
4484
|
+
target: multioption({
|
|
4485
|
+
type: array(string),
|
|
4486
|
+
long: "target",
|
|
4487
|
+
description: "Target name to bundle (repeatable). Defaults to eval target(s) or default."
|
|
4488
|
+
}),
|
|
4489
|
+
targets: option({
|
|
4490
|
+
type: optional(string),
|
|
4491
|
+
long: "targets",
|
|
4492
|
+
description: "Path to targets.yaml (overrides discovery)"
|
|
4493
|
+
})
|
|
4494
|
+
},
|
|
4495
|
+
handler: async (args) => {
|
|
4496
|
+
const cwd = process.cwd();
|
|
4497
|
+
const repoRoot = await findRepoRoot(cwd);
|
|
4498
|
+
const resolvedPaths = await resolveEvalPaths([args.evalPath], cwd);
|
|
4499
|
+
if (resolvedPaths.length !== 1) {
|
|
4500
|
+
throw new Error(
|
|
4501
|
+
`agentv eval bundle requires exactly one eval file, but matched ${resolvedPaths.length}: ${resolvedPaths.join(", ")}`
|
|
4502
|
+
);
|
|
4503
|
+
}
|
|
4504
|
+
const evalFilePath = resolvedPaths[0];
|
|
4505
|
+
await loadEnvFromHierarchy({ testFilePath: evalFilePath, repoRoot, verbose: false });
|
|
4506
|
+
const suite = await loadTestSuite(evalFilePath, repoRoot);
|
|
4507
|
+
if (suite.providerFactory) {
|
|
4508
|
+
throw new Error(
|
|
4509
|
+
"TypeScript evals with task() provider functions cannot be bundled into portable YAML yet."
|
|
4510
|
+
);
|
|
4511
|
+
}
|
|
4512
|
+
let definitions;
|
|
4513
|
+
let targetNames;
|
|
4514
|
+
if (suite.inlineTarget) {
|
|
4515
|
+
definitions = [suite.inlineTarget];
|
|
4516
|
+
targetNames = unique(args.target.length > 0 ? args.target : [suite.inlineTarget.name]);
|
|
4517
|
+
} else {
|
|
4518
|
+
const targetsFilePath = await discoverTargetsFile({
|
|
4519
|
+
explicitPath: args.targets,
|
|
4520
|
+
testFilePath: evalFilePath,
|
|
4521
|
+
repoRoot,
|
|
4522
|
+
cwd
|
|
4523
|
+
});
|
|
4524
|
+
definitions = definitionsWithEvalTargetRefs(
|
|
4525
|
+
await readTargetDefinitions(targetsFilePath),
|
|
4526
|
+
suite.targetRefs
|
|
4527
|
+
);
|
|
4528
|
+
const suiteTarget = await readTestSuiteTarget(evalFilePath);
|
|
4529
|
+
targetNames = unique(
|
|
4530
|
+
args.target.length > 0 ? args.target : suite.targets ?? [suiteTarget ?? "default"]
|
|
4531
|
+
);
|
|
4532
|
+
for (const targetName2 of targetNames) {
|
|
4533
|
+
ensureTargetGraph(targetName2, definitions, targetsFilePath);
|
|
4534
|
+
}
|
|
4535
|
+
}
|
|
4536
|
+
const targetSelections = targetNames.map((targetName2) => ({
|
|
4537
|
+
evalFileAbsolutePath: evalFilePath,
|
|
4538
|
+
targetName: targetName2,
|
|
4539
|
+
definitions
|
|
4540
|
+
}));
|
|
4541
|
+
const paths = await materializeEvalBundle({
|
|
4542
|
+
evalFilePath,
|
|
4543
|
+
tests: suite.tests,
|
|
4544
|
+
targetSelections,
|
|
4545
|
+
outputDir: args.out,
|
|
4546
|
+
cwd,
|
|
4547
|
+
repoRoot,
|
|
4548
|
+
execution: buildBundleExecution({
|
|
4549
|
+
targetNames,
|
|
4550
|
+
targetRefs: suite.targetRefs,
|
|
4551
|
+
workers: suite.workers,
|
|
4552
|
+
cache: suite.cacheConfig?.enabled,
|
|
4553
|
+
cachePath: suite.cacheConfig?.cachePath,
|
|
4554
|
+
budgetUsd: suite.budgetUsd,
|
|
4555
|
+
threshold: suite.threshold
|
|
4556
|
+
})
|
|
4557
|
+
});
|
|
4558
|
+
console.log(`Bundle written to: ${paths.bundleDir}`);
|
|
4559
|
+
console.log(
|
|
4560
|
+
` Eval: ${path7.relative(paths.bundleDir, paths.evalPath).split(path7.sep).join("/")}`
|
|
4561
|
+
);
|
|
4562
|
+
console.log(
|
|
4563
|
+
` Targets: ${path7.relative(paths.bundleDir, paths.targetsPath).split(path7.sep).join("/")}`
|
|
4564
|
+
);
|
|
4565
|
+
console.log(
|
|
4566
|
+
` Manifest: ${path7.relative(paths.bundleDir, paths.manifestPath).split(path7.sep).join("/")}`
|
|
4567
|
+
);
|
|
4568
|
+
}
|
|
4569
|
+
});
|
|
4570
|
+
|
|
4367
4571
|
// src/commands/eval/commands/run.ts
|
|
4368
4572
|
var evalRunCommand = command({
|
|
4369
4573
|
name: "eval",
|
|
@@ -4602,7 +4806,7 @@ var evalRunCommand = command({
|
|
|
4602
4806
|
},
|
|
4603
4807
|
handler: async (args) => {
|
|
4604
4808
|
if (args.evalPaths.length === 0 && process.stdin.isTTY) {
|
|
4605
|
-
const { launchInteractiveWizard } = await import("./interactive-
|
|
4809
|
+
const { launchInteractiveWizard } = await import("./interactive-VNGVNNRF.js");
|
|
4606
4810
|
await launchInteractiveWizard();
|
|
4607
4811
|
return;
|
|
4608
4812
|
}
|
|
@@ -4685,13 +4889,14 @@ var evalCommand = subcommands({
|
|
|
4685
4889
|
cmds: {
|
|
4686
4890
|
run: evalRunCommand,
|
|
4687
4891
|
assert: evalAssertCommand,
|
|
4688
|
-
aggregate: evalAggregateCommand
|
|
4892
|
+
aggregate: evalAggregateCommand,
|
|
4893
|
+
bundle: evalBundleCommand
|
|
4689
4894
|
}
|
|
4690
4895
|
});
|
|
4691
4896
|
|
|
4692
4897
|
// src/commands/grade/index.ts
|
|
4693
4898
|
import { readFile, stat } from "node:fs/promises";
|
|
4694
|
-
import
|
|
4899
|
+
import path8 from "node:path";
|
|
4695
4900
|
function isRecord(value) {
|
|
4696
4901
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4697
4902
|
}
|
|
@@ -4749,8 +4954,10 @@ function fromManifestWire(value, manifestPath) {
|
|
|
4749
4954
|
if (setupStatus !== "ok") {
|
|
4750
4955
|
throw invalidManifest(manifestPath, "field 'setup_status' must be 'ok'");
|
|
4751
4956
|
}
|
|
4752
|
-
const preparedDir =
|
|
4753
|
-
const resolveManifestPath = (rawPath) =>
|
|
4957
|
+
const preparedDir = path8.dirname(manifestPath);
|
|
4958
|
+
const resolveManifestPath = (rawPath) => path8.isAbsolute(rawPath) ? rawPath : path8.resolve(preparedDir, rawPath);
|
|
4959
|
+
expectArray(value, "setup_steps", manifestPath);
|
|
4960
|
+
expectArray(value, "repo_pins", manifestPath);
|
|
4754
4961
|
return {
|
|
4755
4962
|
schemaVersion: 1,
|
|
4756
4963
|
evalPath: resolveManifestPath(expectString(value, "eval_path", manifestPath)),
|
|
@@ -4759,8 +4966,6 @@ function fromManifestWire(value, manifestPath) {
|
|
|
4759
4966
|
workspacePath: resolveManifestPath(expectString(value, "workspace_path", manifestPath)),
|
|
4760
4967
|
promptPath: resolveManifestPath(expectString(value, "prompt_path", manifestPath)),
|
|
4761
4968
|
setupStatus,
|
|
4762
|
-
setupSteps: expectArray(value, "setup_steps", manifestPath),
|
|
4763
|
-
repoPins: expectArray(value, "repo_pins", manifestPath),
|
|
4764
4969
|
baseline: expectBaseline(value.baseline, manifestPath),
|
|
4765
4970
|
createdAt: expectString(value, "created_at", manifestPath),
|
|
4766
4971
|
manifestPath,
|
|
@@ -4768,12 +4973,12 @@ function fromManifestWire(value, manifestPath) {
|
|
|
4768
4973
|
};
|
|
4769
4974
|
}
|
|
4770
4975
|
async function resolvePreparedManifestPath(preparedPath) {
|
|
4771
|
-
const resolved =
|
|
4976
|
+
const resolved = path8.resolve(preparedPath);
|
|
4772
4977
|
try {
|
|
4773
4978
|
const stats = await stat(resolved);
|
|
4774
|
-
return stats.isDirectory() ?
|
|
4979
|
+
return stats.isDirectory() ? path8.join(resolved, "agentv_prepare.json") : resolved;
|
|
4775
4980
|
} catch {
|
|
4776
|
-
return
|
|
4981
|
+
return path8.basename(resolved) === "agentv_prepare.json" ? resolved : path8.join(resolved, "agentv_prepare.json");
|
|
4777
4982
|
}
|
|
4778
4983
|
}
|
|
4779
4984
|
async function readPreparedManifest(preparedPath) {
|
|
@@ -4825,8 +5030,8 @@ async function ensureFileExists(filePath, description) {
|
|
|
4825
5030
|
}
|
|
4826
5031
|
}
|
|
4827
5032
|
function assertMatchesManifest(options) {
|
|
4828
|
-
const commandEvalPath =
|
|
4829
|
-
if (
|
|
5033
|
+
const commandEvalPath = path8.resolve(options.evalPath);
|
|
5034
|
+
if (path8.resolve(options.manifest.evalPath) !== commandEvalPath) {
|
|
4830
5035
|
throw new Error(
|
|
4831
5036
|
`Prepared manifest eval_path does not match command eval path: ${options.manifest.evalPath} !== ${commandEvalPath}`
|
|
4832
5037
|
);
|
|
@@ -4838,13 +5043,14 @@ function assertMatchesManifest(options) {
|
|
|
4838
5043
|
}
|
|
4839
5044
|
return options.testId ?? options.manifest.testId;
|
|
4840
5045
|
}
|
|
4841
|
-
function toPreparedAttemptMetadata(manifest) {
|
|
5046
|
+
function toPreparedAttemptMetadata(manifest, tracePath) {
|
|
4842
5047
|
return {
|
|
4843
5048
|
source: "manual",
|
|
4844
5049
|
manifestPath: manifest.manifestPath,
|
|
4845
5050
|
preparedDir: manifest.preparedDir,
|
|
4846
5051
|
workspacePath: manifest.workspacePath,
|
|
4847
5052
|
promptPath: manifest.promptPath,
|
|
5053
|
+
...tracePath !== void 0 && { tracePath },
|
|
4848
5054
|
target: manifest.target,
|
|
4849
5055
|
preparedAt: manifest.createdAt,
|
|
4850
5056
|
setupStatus: manifest.setupStatus,
|
|
@@ -4860,6 +5066,7 @@ function toCommandOutputWire(result) {
|
|
|
4860
5066
|
execution_status: result.executionStatus,
|
|
4861
5067
|
workspace_path: result.workspacePath,
|
|
4862
5068
|
manifest_path: result.manifestPath,
|
|
5069
|
+
...result.tracePath !== void 0 && { trace_path: result.tracePath },
|
|
4863
5070
|
output_dir: result.outputDir,
|
|
4864
5071
|
index_path: result.indexPath
|
|
4865
5072
|
};
|
|
@@ -4869,19 +5076,167 @@ function printHumanOutput(result) {
|
|
|
4869
5076
|
console.log(`Score: ${result.score.toFixed(3)} (${result.executionStatus})`);
|
|
4870
5077
|
console.log(`Workspace: ${result.workspacePath}`);
|
|
4871
5078
|
console.log(`Manifest: ${result.manifestPath}`);
|
|
5079
|
+
if (result.tracePath) {
|
|
5080
|
+
console.log(`Trace: ${result.tracePath}`);
|
|
5081
|
+
}
|
|
4872
5082
|
console.log(`Artifact workspace: ${result.outputDir}`);
|
|
4873
5083
|
console.log(`Index: ${result.indexPath}`);
|
|
4874
5084
|
}
|
|
5085
|
+
function isTraceEnvelopeDocument(value) {
|
|
5086
|
+
return isRecord(value) && value.schema_version === EXECUTION_TRACE_SCHEMA_VERSION;
|
|
5087
|
+
}
|
|
5088
|
+
function isTranscriptJsonLine(value) {
|
|
5089
|
+
return isRecord(value) && typeof value.test_id === "string" && typeof value.target === "string" && typeof value.message_index === "number" && isRecord(value.source);
|
|
5090
|
+
}
|
|
5091
|
+
function parseFirstJsonLine(raw) {
|
|
5092
|
+
const firstLine = raw.split(/\r?\n/).map((line) => line.trim()).find((line) => line.length > 0);
|
|
5093
|
+
if (!firstLine) {
|
|
5094
|
+
return void 0;
|
|
5095
|
+
}
|
|
5096
|
+
try {
|
|
5097
|
+
return JSON.parse(firstLine);
|
|
5098
|
+
} catch {
|
|
5099
|
+
return void 0;
|
|
5100
|
+
}
|
|
5101
|
+
}
|
|
5102
|
+
function looksLikeTraceEnvelopeJsonText(trimmed) {
|
|
5103
|
+
if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) {
|
|
5104
|
+
return false;
|
|
5105
|
+
}
|
|
5106
|
+
return trimmed.includes('"schema_version"') && trimmed.includes(JSON.stringify(EXECUTION_TRACE_SCHEMA_VERSION));
|
|
5107
|
+
}
|
|
5108
|
+
function traceEnvelopeRecordMatchesPreparedAttempt(record, options) {
|
|
5109
|
+
if (record.envelope.eval.testId !== options.testId) {
|
|
5110
|
+
return false;
|
|
5111
|
+
}
|
|
5112
|
+
const sourceTarget = record.envelope.eval.sourceTarget ?? record.envelope.eval.target;
|
|
5113
|
+
return sourceTarget === options.target || record.envelope.eval.target === options.target;
|
|
5114
|
+
}
|
|
5115
|
+
function selectTraceEnvelopeRecord(records, options) {
|
|
5116
|
+
if (records.length === 0) {
|
|
5117
|
+
throw new Error(`Trace file has no execution trace records: ${options.sourcePath}`);
|
|
5118
|
+
}
|
|
5119
|
+
const matchingTest = records.filter((record) => record.envelope.eval.testId === options.testId);
|
|
5120
|
+
const candidates = matchingTest.filter(
|
|
5121
|
+
(record) => traceEnvelopeRecordMatchesPreparedAttempt(record, options)
|
|
5122
|
+
);
|
|
5123
|
+
if (candidates.length === 1) {
|
|
5124
|
+
return candidates[0];
|
|
5125
|
+
}
|
|
5126
|
+
const detail = candidates.length === 0 ? matchingTest.length === 0 ? `no record matches test_id '${options.testId}'` : `${matchingTest.length} record(s) match test_id '${options.testId}' but none match target '${options.target}'` : `${candidates.length} records match test_id '${options.testId}' and target '${options.target}'`;
|
|
5127
|
+
throw new Error(
|
|
5128
|
+
`Trace file ${options.sourcePath} is ambiguous for prepared grading: ${detail}. Pass a single-record trace file or a file with one matching test/target record.`
|
|
5129
|
+
);
|
|
5130
|
+
}
|
|
5131
|
+
function traceFromEnvelopeRecord(record) {
|
|
5132
|
+
const summary = traceEnvelopeToTraceSummary(record.envelope);
|
|
5133
|
+
return buildTraceFromMessages({
|
|
5134
|
+
output: traceEnvelopeToTranscriptMessages(record.envelope),
|
|
5135
|
+
summary: summary.trace,
|
|
5136
|
+
tokenUsage: summary.tokenUsage,
|
|
5137
|
+
costUsd: summary.costUsd,
|
|
5138
|
+
durationMs: summary.durationMs,
|
|
5139
|
+
startTime: summary.startTime,
|
|
5140
|
+
endTime: summary.endTime,
|
|
5141
|
+
provider: record.envelope.source.provider,
|
|
5142
|
+
target: record.envelope.eval.target,
|
|
5143
|
+
testId: record.envelope.eval.testId,
|
|
5144
|
+
conversationId: record.envelope.eval.runId,
|
|
5145
|
+
metadata: {
|
|
5146
|
+
trace_source_path: record.sourcePath,
|
|
5147
|
+
trace_source_format: EXECUTION_TRACE_SCHEMA_VERSION,
|
|
5148
|
+
trace_artifact_id: record.envelope.artifactId,
|
|
5149
|
+
...record.lineNumber !== void 0 && { trace_source_line: record.lineNumber }
|
|
5150
|
+
}
|
|
5151
|
+
});
|
|
5152
|
+
}
|
|
5153
|
+
function groupTranscriptLinesByTestTarget(lines) {
|
|
5154
|
+
const groups = /* @__PURE__ */ new Map();
|
|
5155
|
+
for (const line of lines) {
|
|
5156
|
+
const key = `${line.test_id}\0${line.target}`;
|
|
5157
|
+
const group = groups.get(key) ?? (() => {
|
|
5158
|
+
const created = { testId: line.test_id, target: line.target, lines: [] };
|
|
5159
|
+
groups.set(key, created);
|
|
5160
|
+
return created;
|
|
5161
|
+
})();
|
|
5162
|
+
group.lines.push(line);
|
|
5163
|
+
}
|
|
5164
|
+
return [...groups.values()];
|
|
5165
|
+
}
|
|
5166
|
+
function selectTranscriptLines(lines, options) {
|
|
5167
|
+
const groups = groupTranscriptLinesByTestTarget(lines);
|
|
5168
|
+
if (groups.length === 0) {
|
|
5169
|
+
throw new Error(`Trace file has no transcript rows: ${options.sourcePath}`);
|
|
5170
|
+
}
|
|
5171
|
+
const matchingTest = groups.filter((group) => group.testId === options.testId);
|
|
5172
|
+
const candidates = matchingTest.filter((group) => group.target === options.target);
|
|
5173
|
+
if (candidates.length === 1) {
|
|
5174
|
+
return candidates[0].lines;
|
|
5175
|
+
}
|
|
5176
|
+
const detail = candidates.length === 0 ? matchingTest.length === 0 ? `no transcript group matches test_id '${options.testId}'` : `${matchingTest.length} transcript group(s) match test_id '${options.testId}' but none match target '${options.target}'` : `${candidates.length} transcript groups match test_id '${options.testId}' and target '${options.target}'`;
|
|
5177
|
+
throw new Error(
|
|
5178
|
+
`Trace file ${options.sourcePath} is ambiguous for prepared grading: ${detail}. Pass a single-session transcript or a transcript with one matching test/target group.`
|
|
5179
|
+
);
|
|
5180
|
+
}
|
|
5181
|
+
async function readPreparedTrace(tracePath, options) {
|
|
5182
|
+
const sourcePath = path8.resolve(tracePath);
|
|
5183
|
+
const raw = await readFile(sourcePath, "utf8");
|
|
5184
|
+
const trimmed = raw.trim();
|
|
5185
|
+
if (trimmed.length === 0) {
|
|
5186
|
+
throw new Error(`Trace file is empty: ${sourcePath}`);
|
|
5187
|
+
}
|
|
5188
|
+
const firstLine = parseFirstJsonLine(raw);
|
|
5189
|
+
if (isTraceEnvelopeDocument(firstLine)) {
|
|
5190
|
+
const records = await readTraceEnvelopeReplayRecords(sourcePath);
|
|
5191
|
+
return {
|
|
5192
|
+
sourcePath,
|
|
5193
|
+
trace: traceFromEnvelopeRecord(
|
|
5194
|
+
selectTraceEnvelopeRecord(records, { sourcePath, ...options })
|
|
5195
|
+
)
|
|
5196
|
+
};
|
|
5197
|
+
}
|
|
5198
|
+
if (isTranscriptJsonLine(firstLine)) {
|
|
5199
|
+
const lines = selectTranscriptLines(await readTranscriptJsonl(sourcePath), {
|
|
5200
|
+
sourcePath,
|
|
5201
|
+
...options
|
|
5202
|
+
});
|
|
5203
|
+
const trace = traceFromTranscriptJsonLines(lines);
|
|
5204
|
+
return {
|
|
5205
|
+
sourcePath,
|
|
5206
|
+
trace: {
|
|
5207
|
+
...trace,
|
|
5208
|
+
metadata: {
|
|
5209
|
+
...trace.metadata,
|
|
5210
|
+
trace_source_path: sourcePath,
|
|
5211
|
+
trace_source_format: "agentv.transcript.jsonl"
|
|
5212
|
+
}
|
|
5213
|
+
}
|
|
5214
|
+
};
|
|
5215
|
+
}
|
|
5216
|
+
if (looksLikeTraceEnvelopeJsonText(trimmed)) {
|
|
5217
|
+
const records = await readTraceEnvelopeReplayRecords(sourcePath);
|
|
5218
|
+
return {
|
|
5219
|
+
sourcePath,
|
|
5220
|
+
trace: traceFromEnvelopeRecord(
|
|
5221
|
+
selectTraceEnvelopeRecord(records, { sourcePath, ...options })
|
|
5222
|
+
)
|
|
5223
|
+
};
|
|
5224
|
+
}
|
|
5225
|
+
throw new Error(
|
|
5226
|
+
`Unsupported trace format at ${sourcePath}. Expected agentv.trace.v1 JSON/JSONL or AgentV transcript JSONL.`
|
|
5227
|
+
);
|
|
5228
|
+
}
|
|
4875
5229
|
async function gradePreparedAttempt(options) {
|
|
4876
5230
|
const manifest = await readPreparedManifest(options.preparedPath);
|
|
4877
|
-
const evalPath =
|
|
5231
|
+
const evalPath = path8.resolve(options.evalPath);
|
|
4878
5232
|
const testId = assertMatchesManifest({ manifest, evalPath, testId: options.testId });
|
|
4879
5233
|
await ensureDirectoryExists(manifest.workspacePath, "Prepared workspace");
|
|
4880
5234
|
await ensureFileExists(manifest.promptPath, "Prepared prompt");
|
|
4881
|
-
const
|
|
5235
|
+
const preparedTrace = options.tracePath !== void 0 ? await readPreparedTrace(options.tracePath, { testId, target: manifest.target }) : void 0;
|
|
5236
|
+
const evalDir = path8.dirname(evalPath);
|
|
4882
5237
|
const repoRoot = await findRepoRoot(evalDir);
|
|
4883
5238
|
await loadEnvFromHierarchy({ testFilePath: evalPath, repoRoot, verbose: !!options.verbose });
|
|
4884
|
-
const category = deriveCategory(
|
|
5239
|
+
const category = deriveCategory(path8.relative(process.cwd(), evalPath));
|
|
4885
5240
|
const suite = await loadTestSuite(evalPath, repoRoot, { category });
|
|
4886
5241
|
const test = suite.tests.find((candidate) => candidate.id === testId);
|
|
4887
5242
|
if (!test) {
|
|
@@ -4907,8 +5262,8 @@ async function gradePreparedAttempt(options) {
|
|
|
4907
5262
|
...selection.resolvedTarget,
|
|
4908
5263
|
name: manifest.target
|
|
4909
5264
|
};
|
|
4910
|
-
const response = options.responsePath !== void 0 ? await readFile(
|
|
4911
|
-
const runDir =
|
|
5265
|
+
const response = options.responsePath !== void 0 ? await readFile(path8.resolve(options.responsePath), "utf8") : void 0;
|
|
5266
|
+
const runDir = path8.resolve(
|
|
4912
5267
|
options.outputDir ?? buildDefaultRunDir(process.cwd(), options.experiment)
|
|
4913
5268
|
);
|
|
4914
5269
|
const result = await gradePreparedEvalCase({
|
|
@@ -4920,11 +5275,12 @@ async function gradePreparedAttempt(options) {
|
|
|
4920
5275
|
workspacePath: manifest.workspacePath,
|
|
4921
5276
|
baselineCommit: manifest.baseline.commit,
|
|
4922
5277
|
response,
|
|
5278
|
+
trace: preparedTrace?.trace,
|
|
4923
5279
|
verbose: options.verbose,
|
|
4924
5280
|
graderTarget: options.graderTarget,
|
|
4925
5281
|
model: options.model,
|
|
4926
5282
|
threshold: options.threshold ?? suite.threshold,
|
|
4927
|
-
preparedAttempt: toPreparedAttemptMetadata(manifest)
|
|
5283
|
+
preparedAttempt: toPreparedAttemptMetadata(manifest, preparedTrace?.sourcePath)
|
|
4928
5284
|
});
|
|
4929
5285
|
const artifacts = await writeArtifactsFromResults([result], runDir, {
|
|
4930
5286
|
evalFile: evalPath,
|
|
@@ -4939,6 +5295,7 @@ async function gradePreparedAttempt(options) {
|
|
|
4939
5295
|
executionStatus: result.executionStatus,
|
|
4940
5296
|
workspacePath: manifest.workspacePath,
|
|
4941
5297
|
manifestPath: manifest.manifestPath,
|
|
5298
|
+
tracePath: preparedTrace?.sourcePath,
|
|
4942
5299
|
outputDir: runDir,
|
|
4943
5300
|
indexPath: artifacts.indexPath
|
|
4944
5301
|
};
|
|
@@ -4973,6 +5330,11 @@ var gradeCommand = command({
|
|
|
4973
5330
|
long: "response",
|
|
4974
5331
|
description: "Optional final response text file from the human or external agent"
|
|
4975
5332
|
}),
|
|
5333
|
+
trace: option({
|
|
5334
|
+
type: optional(string),
|
|
5335
|
+
long: "trace",
|
|
5336
|
+
description: "Optional AgentV trace/session file for trace-aware graders (execution trace JSON/JSONL or transcript JSONL)"
|
|
5337
|
+
}),
|
|
4976
5338
|
experiment: option({
|
|
4977
5339
|
type: optional(string),
|
|
4978
5340
|
long: "experiment",
|
|
@@ -5005,6 +5367,7 @@ var gradeCommand = command({
|
|
|
5005
5367
|
prepared,
|
|
5006
5368
|
output,
|
|
5007
5369
|
response,
|
|
5370
|
+
trace,
|
|
5008
5371
|
experiment,
|
|
5009
5372
|
graderTarget,
|
|
5010
5373
|
model,
|
|
@@ -5017,6 +5380,7 @@ var gradeCommand = command({
|
|
|
5017
5380
|
preparedPath: prepared,
|
|
5018
5381
|
outputDir: output,
|
|
5019
5382
|
responsePath: response,
|
|
5383
|
+
tracePath: trace,
|
|
5020
5384
|
experiment,
|
|
5021
5385
|
graderTarget,
|
|
5022
5386
|
model,
|
|
@@ -5033,7 +5397,7 @@ var gradeCommand = command({
|
|
|
5033
5397
|
|
|
5034
5398
|
// src/commands/import/claude.ts
|
|
5035
5399
|
import { mkdir as mkdir3, writeFile as writeFile3 } from "node:fs/promises";
|
|
5036
|
-
import
|
|
5400
|
+
import path9 from "node:path";
|
|
5037
5401
|
var importClaudeCommand = command({
|
|
5038
5402
|
name: "claude",
|
|
5039
5403
|
description: "Import a Claude Code session transcript for offline grading",
|
|
@@ -5105,8 +5469,8 @@ var importClaudeCommand = command({
|
|
|
5105
5469
|
const rawJsonl = await readTranscriptFile(sessionFilePath);
|
|
5106
5470
|
const transcript = parseClaudeSession(rawJsonl);
|
|
5107
5471
|
const shortId = (sessionId ?? transcript.source.sessionId).slice(0, 8);
|
|
5108
|
-
const outputPath = output ??
|
|
5109
|
-
await mkdir3(
|
|
5472
|
+
const outputPath = output ?? path9.join(".agentv", "transcripts", `claude-${shortId}.jsonl`);
|
|
5473
|
+
await mkdir3(path9.dirname(outputPath), { recursive: true });
|
|
5110
5474
|
const jsonLines = toTranscriptJsonLines(transcript);
|
|
5111
5475
|
await writeFile3(
|
|
5112
5476
|
outputPath,
|
|
@@ -5150,7 +5514,7 @@ function formatDurationMs(ms) {
|
|
|
5150
5514
|
|
|
5151
5515
|
// src/commands/import/codex.ts
|
|
5152
5516
|
import { mkdir as mkdir4, writeFile as writeFile4 } from "node:fs/promises";
|
|
5153
|
-
import
|
|
5517
|
+
import path10 from "node:path";
|
|
5154
5518
|
var importCodexCommand = command({
|
|
5155
5519
|
name: "codex",
|
|
5156
5520
|
description: "Import a Codex CLI session transcript for offline grading",
|
|
@@ -5222,8 +5586,8 @@ var importCodexCommand = command({
|
|
|
5222
5586
|
const rawJsonl = await readTranscriptFile(session.filePath);
|
|
5223
5587
|
const transcript = parseCodexSession(rawJsonl);
|
|
5224
5588
|
const shortId = session.sessionId.slice(0, 8);
|
|
5225
|
-
const outputPath = output ??
|
|
5226
|
-
await mkdir4(
|
|
5589
|
+
const outputPath = output ?? path10.join(".agentv", "transcripts", `codex-${shortId}.jsonl`);
|
|
5590
|
+
await mkdir4(path10.dirname(outputPath), { recursive: true });
|
|
5227
5591
|
const jsonLines = toTranscriptJsonLines(transcript);
|
|
5228
5592
|
await writeFile4(
|
|
5229
5593
|
outputPath,
|
|
@@ -5262,7 +5626,7 @@ function formatDurationMs2(ms) {
|
|
|
5262
5626
|
|
|
5263
5627
|
// src/commands/import/copilot.ts
|
|
5264
5628
|
import { mkdir as mkdir5, readFile as readFile2, writeFile as writeFile5 } from "node:fs/promises";
|
|
5265
|
-
import
|
|
5629
|
+
import path11 from "node:path";
|
|
5266
5630
|
var importCopilotCommand = command({
|
|
5267
5631
|
name: "copilot",
|
|
5268
5632
|
description: "Import a Copilot CLI session transcript for offline grading",
|
|
@@ -5327,7 +5691,7 @@ var importCopilotCommand = command({
|
|
|
5327
5691
|
);
|
|
5328
5692
|
process.exit(1);
|
|
5329
5693
|
}
|
|
5330
|
-
const eventsPath =
|
|
5694
|
+
const eventsPath = path11.join(sessionDir, "events.jsonl");
|
|
5331
5695
|
const rawJsonl = await readFile2(eventsPath, "utf8");
|
|
5332
5696
|
const parsed = parseCopilotEvents(rawJsonl);
|
|
5333
5697
|
const transcript = {
|
|
@@ -5344,8 +5708,8 @@ var importCopilotCommand = command({
|
|
|
5344
5708
|
costUsd: null
|
|
5345
5709
|
};
|
|
5346
5710
|
const shortId = resolvedSessionId.slice(0, 8);
|
|
5347
|
-
const outputPath = output ??
|
|
5348
|
-
await mkdir5(
|
|
5711
|
+
const outputPath = output ?? path11.join(".agentv", "transcripts", `copilot-${shortId}.jsonl`);
|
|
5712
|
+
await mkdir5(path11.dirname(outputPath), { recursive: true });
|
|
5349
5713
|
const jsonLines = toTranscriptJsonLines(transcript);
|
|
5350
5714
|
await writeFile5(
|
|
5351
5715
|
outputPath,
|
|
@@ -5390,12 +5754,12 @@ function formatDurationMs3(ms) {
|
|
|
5390
5754
|
// src/commands/import/huggingface.ts
|
|
5391
5755
|
import { execFile } from "node:child_process";
|
|
5392
5756
|
import { existsSync as existsSync2 } from "node:fs";
|
|
5393
|
-
import
|
|
5757
|
+
import path12 from "node:path";
|
|
5394
5758
|
function findScript() {
|
|
5395
5759
|
const candidates = [
|
|
5396
|
-
|
|
5397
|
-
|
|
5398
|
-
|
|
5760
|
+
path12.resolve(__dirname, "..", "..", "..", "..", "..", "scripts", "import-huggingface.py"),
|
|
5761
|
+
path12.resolve(__dirname, "..", "..", "..", "..", "scripts", "import-huggingface.py"),
|
|
5762
|
+
path12.resolve(process.cwd(), "scripts", "import-huggingface.py")
|
|
5399
5763
|
];
|
|
5400
5764
|
for (const candidate of candidates) {
|
|
5401
5765
|
if (existsSync2(candidate)) return candidate;
|
|
@@ -5495,17 +5859,17 @@ Imported ${summary.files_created} eval(s) from ${summary.dataset} \u2192 ${summa
|
|
|
5495
5859
|
|
|
5496
5860
|
// src/commands/import/promptfoo.ts
|
|
5497
5861
|
import { mkdir as mkdir6, readFile as readFile3, writeFile as writeFile6 } from "node:fs/promises";
|
|
5498
|
-
import
|
|
5862
|
+
import path13 from "node:path";
|
|
5499
5863
|
import fg2 from "fast-glob";
|
|
5500
5864
|
import JSON5 from "json5";
|
|
5501
5865
|
import { parse as parseYaml, stringify as stringifyYaml2 } from "yaml";
|
|
5502
5866
|
var FILE_PREFIX = "file://";
|
|
5503
5867
|
var PROMPTFOO_COMMENT_PREFIX = "# Converted from promptfoo config: ";
|
|
5504
5868
|
async function convertPromptfooToAgentvSuite(options) {
|
|
5505
|
-
const absoluteInputPath =
|
|
5506
|
-
const configDir =
|
|
5869
|
+
const absoluteInputPath = path13.resolve(options.inputPath);
|
|
5870
|
+
const configDir = path13.dirname(absoluteInputPath);
|
|
5507
5871
|
const rawConfig = await loadPromptfooConfig(absoluteInputPath);
|
|
5508
|
-
const suiteName = sanitizeName(
|
|
5872
|
+
const suiteName = sanitizeName(path13.basename(absoluteInputPath, path13.extname(absoluteInputPath)));
|
|
5509
5873
|
const prompts = await loadPromptfooPrompts(rawConfig.prompts, configDir);
|
|
5510
5874
|
if (prompts.length === 0) {
|
|
5511
5875
|
throw new Error(`promptfoo import requires at least one prompt in ${absoluteInputPath}`);
|
|
@@ -5546,7 +5910,7 @@ async function convertPromptfooToAgentvYaml(inputPath) {
|
|
|
5546
5910
|
indent: 2,
|
|
5547
5911
|
lineWidth: 0
|
|
5548
5912
|
});
|
|
5549
|
-
return `${PROMPTFOO_COMMENT_PREFIX}${
|
|
5913
|
+
return `${PROMPTFOO_COMMENT_PREFIX}${path13.resolve(inputPath)}
|
|
5550
5914
|
${yaml}`;
|
|
5551
5915
|
}
|
|
5552
5916
|
var importPromptfooCommand = command({
|
|
@@ -5570,14 +5934,14 @@ var importPromptfooCommand = command({
|
|
|
5570
5934
|
})
|
|
5571
5935
|
},
|
|
5572
5936
|
handler: async ({ input, output, dryRun: dryRun2 }) => {
|
|
5573
|
-
const absoluteInput =
|
|
5937
|
+
const absoluteInput = path13.resolve(input);
|
|
5574
5938
|
const yaml = await convertPromptfooToAgentvYaml(absoluteInput);
|
|
5575
5939
|
if (dryRun2) {
|
|
5576
5940
|
process.stdout.write(yaml);
|
|
5577
5941
|
return;
|
|
5578
5942
|
}
|
|
5579
|
-
const outputPath =
|
|
5580
|
-
await mkdir6(
|
|
5943
|
+
const outputPath = path13.resolve(output ?? path13.join(path13.dirname(absoluteInput), "EVAL.yaml"));
|
|
5944
|
+
await mkdir6(path13.dirname(outputPath), { recursive: true });
|
|
5581
5945
|
await writeFile6(outputPath, yaml, "utf8");
|
|
5582
5946
|
console.log(`Imported promptfoo config \u2192 ${outputPath}`);
|
|
5583
5947
|
}
|
|
@@ -5591,7 +5955,7 @@ async function loadPromptfooConfig(filePath) {
|
|
|
5591
5955
|
return parsed;
|
|
5592
5956
|
}
|
|
5593
5957
|
function parseStructuredText(content, filePath) {
|
|
5594
|
-
const ext =
|
|
5958
|
+
const ext = path13.extname(filePath).toLowerCase();
|
|
5595
5959
|
if (ext === ".json" || ext === ".json5" || ext === ".jsonc") {
|
|
5596
5960
|
return JSON5.parse(content);
|
|
5597
5961
|
}
|
|
@@ -5665,7 +6029,7 @@ async function loadPromptFromReference(reference, baseDir, identity2) {
|
|
|
5665
6029
|
const prompts = [];
|
|
5666
6030
|
for (let index = 0; index < files.length; index++) {
|
|
5667
6031
|
const filePath = files[index];
|
|
5668
|
-
const ext =
|
|
6032
|
+
const ext = path13.extname(filePath).toLowerCase();
|
|
5669
6033
|
const raw = await readFile3(filePath, "utf8");
|
|
5670
6034
|
if (ext === ".json" || ext === ".json5" || ext === ".jsonc") {
|
|
5671
6035
|
const parsed = parseStructuredText(raw, filePath);
|
|
@@ -5808,10 +6172,10 @@ async function loadPromptfooTestsFromReference(reference, baseDir) {
|
|
|
5808
6172
|
const files = await resolvePromptfooFileReference(reference, baseDir);
|
|
5809
6173
|
const tests = [];
|
|
5810
6174
|
for (const filePath of files) {
|
|
5811
|
-
const ext =
|
|
6175
|
+
const ext = path13.extname(stripSheetSuffix(filePath)).toLowerCase();
|
|
5812
6176
|
if (ext === ".xlsx" || ext === ".xls") {
|
|
5813
6177
|
throw new Error(
|
|
5814
|
-
`Unsupported test dataset '${
|
|
6178
|
+
`Unsupported test dataset '${path13.basename(filePath)}': XLSX promptfoo datasets are not imported yet`
|
|
5815
6179
|
);
|
|
5816
6180
|
}
|
|
5817
6181
|
const content = await readFile3(filePath, "utf8");
|
|
@@ -6409,7 +6773,7 @@ function normalizeAssertionType(rawType) {
|
|
|
6409
6773
|
async function resolvePromptfooFileReference(reference, baseDir) {
|
|
6410
6774
|
const rawPath = reference.slice(FILE_PREFIX.length);
|
|
6411
6775
|
const [pathWithoutSheet] = rawPath.split("#");
|
|
6412
|
-
const absolutePath =
|
|
6776
|
+
const absolutePath = path13.resolve(baseDir, pathWithoutSheet);
|
|
6413
6777
|
const normalizedPattern = absolutePath.replaceAll("\\", "/");
|
|
6414
6778
|
const matches = await fg2(normalizedPattern, {
|
|
6415
6779
|
onlyFiles: true,
|
|
@@ -6502,29 +6866,29 @@ var importCommand = subcommands({
|
|
|
6502
6866
|
|
|
6503
6867
|
// src/commands/init/index.ts
|
|
6504
6868
|
import { existsSync as existsSync3, mkdirSync, writeFileSync as writeFileSync2 } from "node:fs";
|
|
6505
|
-
import
|
|
6869
|
+
import path15 from "node:path";
|
|
6506
6870
|
import * as readline from "node:readline/promises";
|
|
6507
6871
|
|
|
6508
6872
|
// src/templates/index.ts
|
|
6509
6873
|
import { readFileSync as readFileSync3, readdirSync, statSync } from "node:fs";
|
|
6510
|
-
import
|
|
6874
|
+
import path14 from "node:path";
|
|
6511
6875
|
import { fileURLToPath } from "node:url";
|
|
6512
6876
|
function getAgentvTemplates() {
|
|
6513
6877
|
return getTemplatesFromDir(".agentv");
|
|
6514
6878
|
}
|
|
6515
6879
|
function getEnvExampleTemplate() {
|
|
6516
|
-
const currentDir =
|
|
6517
|
-
const templatesBase = currentDir.includes(`${
|
|
6518
|
-
const content = readFileSync3(
|
|
6880
|
+
const currentDir = path14.dirname(fileURLToPath(import.meta.url));
|
|
6881
|
+
const templatesBase = currentDir.includes(`${path14.sep}dist`) ? path14.join(currentDir, "templates") : currentDir;
|
|
6882
|
+
const content = readFileSync3(path14.join(templatesBase, ".env.example"), "utf-8");
|
|
6519
6883
|
return { path: ".env.example", content };
|
|
6520
6884
|
}
|
|
6521
6885
|
function getTemplatesFromDir(subdir) {
|
|
6522
|
-
const currentDir =
|
|
6886
|
+
const currentDir = path14.dirname(fileURLToPath(import.meta.url));
|
|
6523
6887
|
let templatesDir;
|
|
6524
|
-
if (currentDir.includes(`${
|
|
6525
|
-
templatesDir =
|
|
6888
|
+
if (currentDir.includes(`${path14.sep}dist`)) {
|
|
6889
|
+
templatesDir = path14.join(currentDir, "templates", subdir);
|
|
6526
6890
|
} else {
|
|
6527
|
-
templatesDir =
|
|
6891
|
+
templatesDir = path14.join(currentDir, subdir);
|
|
6528
6892
|
}
|
|
6529
6893
|
return readTemplatesRecursively(templatesDir, "");
|
|
6530
6894
|
}
|
|
@@ -6532,15 +6896,15 @@ function readTemplatesRecursively(dir, relativePath) {
|
|
|
6532
6896
|
const templates = [];
|
|
6533
6897
|
const entries2 = readdirSync(dir);
|
|
6534
6898
|
for (const entry of entries2) {
|
|
6535
|
-
const fullPath =
|
|
6899
|
+
const fullPath = path14.join(dir, entry);
|
|
6536
6900
|
const stat4 = statSync(fullPath);
|
|
6537
|
-
const entryRelativePath = relativePath ?
|
|
6901
|
+
const entryRelativePath = relativePath ? path14.join(relativePath, entry) : entry;
|
|
6538
6902
|
if (stat4.isDirectory()) {
|
|
6539
6903
|
templates.push(...readTemplatesRecursively(fullPath, entryRelativePath));
|
|
6540
6904
|
} else {
|
|
6541
6905
|
const content = readFileSync3(fullPath, "utf-8");
|
|
6542
6906
|
templates.push({
|
|
6543
|
-
path: entryRelativePath.split(
|
|
6907
|
+
path: entryRelativePath.split(path14.sep).join("/"),
|
|
6544
6908
|
// Normalize to forward slashes
|
|
6545
6909
|
content
|
|
6546
6910
|
});
|
|
@@ -6571,22 +6935,22 @@ async function promptYesNo(message) {
|
|
|
6571
6935
|
}
|
|
6572
6936
|
}
|
|
6573
6937
|
async function initCommand(options = {}) {
|
|
6574
|
-
const targetPath =
|
|
6575
|
-
const agentvDir =
|
|
6938
|
+
const targetPath = path15.resolve(options.targetPath ?? ".");
|
|
6939
|
+
const agentvDir = path15.join(targetPath, ".agentv");
|
|
6576
6940
|
const otherAgentvTemplates = getAgentvTemplates();
|
|
6577
6941
|
const envTemplate = getEnvExampleTemplate();
|
|
6578
6942
|
const existingFiles = [];
|
|
6579
6943
|
if (envTemplate) {
|
|
6580
|
-
const envFilePath =
|
|
6944
|
+
const envFilePath = path15.join(targetPath, ".env.example");
|
|
6581
6945
|
if (existsSync3(envFilePath)) {
|
|
6582
6946
|
existingFiles.push(".env.example");
|
|
6583
6947
|
}
|
|
6584
6948
|
}
|
|
6585
6949
|
if (existsSync3(agentvDir)) {
|
|
6586
6950
|
for (const template of otherAgentvTemplates) {
|
|
6587
|
-
const targetFilePath =
|
|
6951
|
+
const targetFilePath = path15.join(agentvDir, template.path);
|
|
6588
6952
|
if (existsSync3(targetFilePath)) {
|
|
6589
|
-
existingFiles.push(
|
|
6953
|
+
existingFiles.push(path15.relative(targetPath, targetFilePath));
|
|
6590
6954
|
}
|
|
6591
6955
|
}
|
|
6592
6956
|
}
|
|
@@ -6608,18 +6972,18 @@ async function initCommand(options = {}) {
|
|
|
6608
6972
|
mkdirSync(agentvDir, { recursive: true });
|
|
6609
6973
|
}
|
|
6610
6974
|
if (envTemplate) {
|
|
6611
|
-
const envFilePath =
|
|
6975
|
+
const envFilePath = path15.join(targetPath, ".env.example");
|
|
6612
6976
|
writeFileSync2(envFilePath, envTemplate.content, "utf-8");
|
|
6613
6977
|
console.log("Created .env.example");
|
|
6614
6978
|
}
|
|
6615
6979
|
for (const template of otherAgentvTemplates) {
|
|
6616
|
-
const targetFilePath =
|
|
6617
|
-
const targetDirPath =
|
|
6980
|
+
const targetFilePath = path15.join(agentvDir, template.path);
|
|
6981
|
+
const targetDirPath = path15.dirname(targetFilePath);
|
|
6618
6982
|
if (!existsSync3(targetDirPath)) {
|
|
6619
6983
|
mkdirSync(targetDirPath, { recursive: true });
|
|
6620
6984
|
}
|
|
6621
6985
|
writeFileSync2(targetFilePath, template.content, "utf-8");
|
|
6622
|
-
console.log(`Created ${
|
|
6986
|
+
console.log(`Created ${path15.relative(targetPath, targetFilePath)}`);
|
|
6623
6987
|
}
|
|
6624
6988
|
console.log("\nAgentV initialized successfully!");
|
|
6625
6989
|
console.log("\nFiles installed to root:");
|
|
@@ -6627,7 +6991,7 @@ async function initCommand(options = {}) {
|
|
|
6627
6991
|
console.log(" - .env.example");
|
|
6628
6992
|
}
|
|
6629
6993
|
console.log(`
|
|
6630
|
-
Files installed to ${
|
|
6994
|
+
Files installed to ${path15.relative(targetPath, agentvDir)}:`);
|
|
6631
6995
|
for (const t of otherAgentvTemplates) {
|
|
6632
6996
|
console.log(` - ${t.path}`);
|
|
6633
6997
|
}
|
|
@@ -6659,13 +7023,13 @@ var initCmdTsCommand = command({
|
|
|
6659
7023
|
|
|
6660
7024
|
// src/commands/inspect/filter.ts
|
|
6661
7025
|
import { existsSync as existsSync4, readFileSync as readFileSync4, readdirSync as readdirSync2, statSync as statSync2 } from "node:fs";
|
|
6662
|
-
import
|
|
7026
|
+
import path16 from "node:path";
|
|
6663
7027
|
function collectIndexFiles(dir) {
|
|
6664
7028
|
const files = [];
|
|
6665
7029
|
try {
|
|
6666
7030
|
const entries2 = readdirSync2(dir, { withFileTypes: true });
|
|
6667
7031
|
for (const entry of entries2) {
|
|
6668
|
-
const fullPath =
|
|
7032
|
+
const fullPath = path16.join(dir, entry.name);
|
|
6669
7033
|
if (entry.isDirectory()) {
|
|
6670
7034
|
files.push(...collectIndexFiles(fullPath));
|
|
6671
7035
|
} else if (entry.name === "index.jsonl") {
|
|
@@ -6718,7 +7082,7 @@ function parseFilterableRecords(filePath) {
|
|
|
6718
7082
|
raw = normalizeResultRow(raw, { lineNumber: i + 1, sourceLabel: filePath });
|
|
6719
7083
|
let experiment = typeof raw.experiment === "string" ? raw.experiment : void 0;
|
|
6720
7084
|
if (!experiment) {
|
|
6721
|
-
const parts = filePath.split(
|
|
7085
|
+
const parts = filePath.split(path16.sep);
|
|
6722
7086
|
const runsIdx = parts.indexOf("runs");
|
|
6723
7087
|
if (runsIdx !== -1 && parts.length - runsIdx >= 3) {
|
|
6724
7088
|
const candidate = parts[runsIdx + 1];
|
|
@@ -6773,7 +7137,7 @@ function buildFilterPredicate(opts) {
|
|
|
6773
7137
|
}
|
|
6774
7138
|
function discoverFilterSources(searchPath, cwd) {
|
|
6775
7139
|
if (searchPath) {
|
|
6776
|
-
const resolved =
|
|
7140
|
+
const resolved = path16.isAbsolute(searchPath) ? searchPath : path16.resolve(cwd, searchPath);
|
|
6777
7141
|
if (!existsSync4(resolved)) {
|
|
6778
7142
|
console.error(`${c.red}Error:${c.reset} Path does not exist: ${resolved}`);
|
|
6779
7143
|
process.exit(1);
|
|
@@ -6786,7 +7150,7 @@ function discoverFilterSources(searchPath, cwd) {
|
|
|
6786
7150
|
}
|
|
6787
7151
|
return [resolved];
|
|
6788
7152
|
}
|
|
6789
|
-
return collectIndexFiles(
|
|
7153
|
+
return collectIndexFiles(path16.join(cwd, ".agentv", "results", "runs"));
|
|
6790
7154
|
}
|
|
6791
7155
|
function formatFilterTable(records) {
|
|
6792
7156
|
const lines = [];
|
|
@@ -7266,13 +7630,13 @@ var traceScoreCommand = command({
|
|
|
7266
7630
|
|
|
7267
7631
|
// src/commands/inspect/search.ts
|
|
7268
7632
|
import { existsSync as existsSync5, readFileSync as readFileSync5, readdirSync as readdirSync3, statSync as statSync3 } from "node:fs";
|
|
7269
|
-
import
|
|
7633
|
+
import path17 from "node:path";
|
|
7270
7634
|
function collectJsonlFiles(dir) {
|
|
7271
7635
|
const files = [];
|
|
7272
7636
|
try {
|
|
7273
7637
|
const entries2 = readdirSync3(dir, { withFileTypes: true });
|
|
7274
7638
|
for (const entry of entries2) {
|
|
7275
|
-
const fullPath =
|
|
7639
|
+
const fullPath = path17.join(dir, entry.name);
|
|
7276
7640
|
if (entry.isDirectory()) {
|
|
7277
7641
|
files.push(...collectJsonlFiles(fullPath));
|
|
7278
7642
|
} else if (entry.name.endsWith(".jsonl")) {
|
|
@@ -7331,7 +7695,7 @@ function searchJsonlFile(filePath, regex2, targetFilter, experimentFilter) {
|
|
|
7331
7695
|
}
|
|
7332
7696
|
function discoverSources(basePath, cwd) {
|
|
7333
7697
|
if (basePath) {
|
|
7334
|
-
const resolved =
|
|
7698
|
+
const resolved = path17.isAbsolute(basePath) ? basePath : path17.resolve(cwd, basePath);
|
|
7335
7699
|
if (!existsSync5(resolved)) {
|
|
7336
7700
|
console.error(`${c.red}Error:${c.reset} Path does not exist: ${resolved}`);
|
|
7337
7701
|
process.exit(1);
|
|
@@ -7345,8 +7709,8 @@ function discoverSources(basePath, cwd) {
|
|
|
7345
7709
|
return [resolved];
|
|
7346
7710
|
}
|
|
7347
7711
|
const sources = [];
|
|
7348
|
-
sources.push(...collectJsonlFiles(
|
|
7349
|
-
sources.push(...collectJsonlFiles(
|
|
7712
|
+
sources.push(...collectJsonlFiles(path17.join(cwd, ".agentv", "results", "runs")));
|
|
7713
|
+
sources.push(...collectJsonlFiles(path17.join(cwd, ".agentv", "transcripts")));
|
|
7350
7714
|
return sources;
|
|
7351
7715
|
}
|
|
7352
7716
|
function formatSearchResults(matches, pattern) {
|
|
@@ -8972,7 +9336,7 @@ var pipelineCommand = subcommands({
|
|
|
8972
9336
|
|
|
8973
9337
|
// src/commands/prepare/index.ts
|
|
8974
9338
|
import { cp, mkdir as mkdir10, rename, rm, writeFile as writeFile11 } from "node:fs/promises";
|
|
8975
|
-
import
|
|
9339
|
+
import path18 from "node:path";
|
|
8976
9340
|
function setupStatusFromHook(status) {
|
|
8977
9341
|
if (status === "success") {
|
|
8978
9342
|
return "ok";
|
|
@@ -9019,10 +9383,10 @@ async function moveDirectory(sourcePath, destinationPath) {
|
|
|
9019
9383
|
}
|
|
9020
9384
|
async function placePreparedWorkspace(prepared, destinationPath) {
|
|
9021
9385
|
const sourcePath = prepared.workspacePath;
|
|
9022
|
-
if (
|
|
9386
|
+
if (path18.resolve(sourcePath) === path18.resolve(destinationPath)) {
|
|
9023
9387
|
return destinationPath;
|
|
9024
9388
|
}
|
|
9025
|
-
await mkdir10(
|
|
9389
|
+
await mkdir10(path18.dirname(destinationPath), { recursive: true });
|
|
9026
9390
|
await rm(destinationPath, { recursive: true, force: true });
|
|
9027
9391
|
if (prepared.cleanupPolicy.mode !== "static" && prepared.pool === void 0) {
|
|
9028
9392
|
await moveDirectory(sourcePath, destinationPath);
|
|
@@ -9110,12 +9474,12 @@ async function selectPrepareTarget(options) {
|
|
|
9110
9474
|
};
|
|
9111
9475
|
}
|
|
9112
9476
|
async function prepareAttempt(options) {
|
|
9113
|
-
const evalPath =
|
|
9114
|
-
const outDir =
|
|
9115
|
-
const evalDir =
|
|
9477
|
+
const evalPath = path18.resolve(options.evalPath);
|
|
9478
|
+
const outDir = path18.resolve(options.outDir);
|
|
9479
|
+
const evalDir = path18.dirname(evalPath);
|
|
9116
9480
|
const repoRoot = await findRepoRoot(evalDir);
|
|
9117
9481
|
await loadEnvFromHierarchy({ testFilePath: evalPath, repoRoot, verbose: false });
|
|
9118
|
-
const category = deriveCategory(
|
|
9482
|
+
const category = deriveCategory(path18.relative(process.cwd(), evalPath));
|
|
9119
9483
|
const suite = await loadTestSuite(evalPath, repoRoot, { category });
|
|
9120
9484
|
const test = suite.tests.find((candidate) => candidate.id === options.testId);
|
|
9121
9485
|
if (!test) {
|
|
@@ -9140,9 +9504,9 @@ async function prepareAttempt(options) {
|
|
|
9140
9504
|
retainOnFailure: "keep"
|
|
9141
9505
|
});
|
|
9142
9506
|
await mkdir10(outDir, { recursive: true });
|
|
9143
|
-
const workspacePath = await placePreparedWorkspace(prepared,
|
|
9144
|
-
const promptPath =
|
|
9145
|
-
const manifestPath =
|
|
9507
|
+
const workspacePath = await placePreparedWorkspace(prepared, path18.join(outDir, "workspace"));
|
|
9508
|
+
const promptPath = path18.join(outDir, "prompt.md");
|
|
9509
|
+
const manifestPath = path18.join(outDir, "agentv_prepare.json");
|
|
9146
9510
|
const prompt = renderPrompt({
|
|
9147
9511
|
workspacePath,
|
|
9148
9512
|
target: options.target,
|
|
@@ -9231,7 +9595,7 @@ import {
|
|
|
9231
9595
|
statSync as statSync4,
|
|
9232
9596
|
writeFileSync as writeFileSync3
|
|
9233
9597
|
} from "node:fs";
|
|
9234
|
-
import
|
|
9598
|
+
import path19 from "node:path";
|
|
9235
9599
|
var CombineDuplicateError = class extends Error {
|
|
9236
9600
|
constructor(conflicts) {
|
|
9237
9601
|
super(`Duplicate result rows found for ${conflicts.length} (test_id, target) pair(s)`);
|
|
@@ -9247,7 +9611,7 @@ function readManifestRecords(manifestPath) {
|
|
|
9247
9611
|
}
|
|
9248
9612
|
function readBenchmarkMetadata(manifestPath) {
|
|
9249
9613
|
try {
|
|
9250
|
-
const benchmarkPath =
|
|
9614
|
+
const benchmarkPath = path19.join(path19.dirname(manifestPath), "benchmark.json");
|
|
9251
9615
|
const parsed = JSON.parse(readFileSync7(benchmarkPath, "utf8"));
|
|
9252
9616
|
return {
|
|
9253
9617
|
timestamp: parsed.metadata?.timestamp,
|
|
@@ -9366,7 +9730,7 @@ function defaultDisplayName(sources) {
|
|
|
9366
9730
|
function defaultCombinedRunDir(cwd, startedAt) {
|
|
9367
9731
|
const parsed = startedAt ? new Date(startedAt) : void 0;
|
|
9368
9732
|
const timestamp = parsed && !Number.isNaN(parsed.getTime()) ? createRunDirName(parsed) : sanitizePathSegment(startedAt ?? "unknown-time");
|
|
9369
|
-
return
|
|
9733
|
+
return path19.join(cwd, ".agentv", "results", "runs", "combined", timestamp);
|
|
9370
9734
|
}
|
|
9371
9735
|
function uniqueRunDir(baseDir) {
|
|
9372
9736
|
if (!existsSync8(baseDir)) return baseDir;
|
|
@@ -9377,13 +9741,13 @@ function uniqueRunDir(baseDir) {
|
|
|
9377
9741
|
return `${baseDir}-${suffix}`;
|
|
9378
9742
|
}
|
|
9379
9743
|
function toRunId(cwd, runDir) {
|
|
9380
|
-
const runsRoot =
|
|
9381
|
-
const relative3 =
|
|
9382
|
-
if (relative3 === "" || relative3.startsWith("..") ||
|
|
9383
|
-
return
|
|
9744
|
+
const runsRoot = path19.join(cwd, ".agentv", "results", "runs");
|
|
9745
|
+
const relative3 = path19.relative(runsRoot, runDir);
|
|
9746
|
+
if (relative3 === "" || relative3.startsWith("..") || path19.isAbsolute(relative3)) {
|
|
9747
|
+
return path19.basename(runDir);
|
|
9384
9748
|
}
|
|
9385
|
-
const parts = relative3.split(
|
|
9386
|
-
return parts.length > 1 ? `${parts[0]}::${parts.slice(1).join(
|
|
9749
|
+
const parts = relative3.split(path19.sep);
|
|
9750
|
+
return parts.length > 1 ? `${parts[0]}::${parts.slice(1).join(path19.sep)}` : relative3;
|
|
9387
9751
|
}
|
|
9388
9752
|
var MANIFEST_PATH_FIELDS = [
|
|
9389
9753
|
"grading_path",
|
|
@@ -9399,17 +9763,17 @@ var MANIFEST_PATH_FIELDS = [
|
|
|
9399
9763
|
];
|
|
9400
9764
|
function copyReferencedArtifact(sourceBaseDir, outputDir, sourceIndex, relativePath) {
|
|
9401
9765
|
if (!relativePath) return void 0;
|
|
9402
|
-
if (
|
|
9766
|
+
if (path19.isAbsolute(relativePath) || relativePath.split(/[\\/]+/).includes("..")) {
|
|
9403
9767
|
throw new Error(`Unsafe artifact path in source manifest: ${relativePath}`);
|
|
9404
9768
|
}
|
|
9405
|
-
const sourcePath =
|
|
9769
|
+
const sourcePath = path19.join(sourceBaseDir, relativePath);
|
|
9406
9770
|
if (!existsSync8(sourcePath)) {
|
|
9407
9771
|
return relativePath;
|
|
9408
9772
|
}
|
|
9409
|
-
const rewritten =
|
|
9410
|
-
const destPath =
|
|
9773
|
+
const rewritten = path19.posix.join(`sources/source-${sourceIndex + 1}`, relativePath);
|
|
9774
|
+
const destPath = path19.join(outputDir, rewritten);
|
|
9411
9775
|
const sourceStat = statSync4(sourcePath);
|
|
9412
|
-
mkdirSync2(
|
|
9776
|
+
mkdirSync2(path19.dirname(destPath), { recursive: true });
|
|
9413
9777
|
if (sourceStat.isDirectory()) {
|
|
9414
9778
|
cpSync(sourcePath, destPath, { recursive: true });
|
|
9415
9779
|
} else if (sourceStat.isFile()) {
|
|
@@ -9418,7 +9782,7 @@ function copyReferencedArtifact(sourceBaseDir, outputDir, sourceIndex, relativeP
|
|
|
9418
9782
|
return rewritten;
|
|
9419
9783
|
}
|
|
9420
9784
|
function rewriteAndCopyRecord(row, outputDir) {
|
|
9421
|
-
const sourceBaseDir =
|
|
9785
|
+
const sourceBaseDir = path19.dirname(row.source.manifestPath);
|
|
9422
9786
|
const rewritten = { ...row.record };
|
|
9423
9787
|
for (const field of MANIFEST_PATH_FIELDS) {
|
|
9424
9788
|
rewritten[field] = copyReferencedArtifact(
|
|
@@ -9446,8 +9810,8 @@ function buildCombineRunSources(sourcePaths, cwd, options) {
|
|
|
9446
9810
|
return sourcePaths.map((sourcePath, index) => {
|
|
9447
9811
|
const manifestPath = resolveResultSourcePath(sourcePath, cwd);
|
|
9448
9812
|
return {
|
|
9449
|
-
id: options?.ids?.[index] ??
|
|
9450
|
-
displayName: options?.displayNames?.[index] ??
|
|
9813
|
+
id: options?.ids?.[index] ?? path19.basename(path19.dirname(manifestPath)),
|
|
9814
|
+
displayName: options?.displayNames?.[index] ?? path19.basename(path19.dirname(manifestPath)),
|
|
9451
9815
|
manifestPath,
|
|
9452
9816
|
tags: options?.tags?.[index]
|
|
9453
9817
|
};
|
|
@@ -9461,7 +9825,7 @@ function combineRunSources(options) {
|
|
|
9461
9825
|
const startedAt = earliestTimestamp(loadedSources.map((source) => source.startedAt));
|
|
9462
9826
|
const displayName = options.displayName?.trim() || defaultDisplayName(loadedSources);
|
|
9463
9827
|
const runDir = uniqueRunDir(
|
|
9464
|
-
options.outputDir ?
|
|
9828
|
+
options.outputDir ? path19.resolve(options.cwd, options.outputDir) : defaultCombinedRunDir(options.cwd, startedAt)
|
|
9465
9829
|
);
|
|
9466
9830
|
const { rows, conflicts } = selectRows(
|
|
9467
9831
|
loadedSources,
|
|
@@ -9475,10 +9839,10 @@ function combineRunSources(options) {
|
|
|
9475
9839
|
});
|
|
9476
9840
|
mkdirSync2(runDir, { recursive: true });
|
|
9477
9841
|
const records = rows.map((row) => rewriteAndCopyRecord(row, runDir));
|
|
9478
|
-
const manifestPath =
|
|
9842
|
+
const manifestPath = path19.join(runDir, "index.jsonl");
|
|
9479
9843
|
writeJsonl(manifestPath, records);
|
|
9480
9844
|
const timing = buildTimingArtifact(results);
|
|
9481
|
-
const timingPath =
|
|
9845
|
+
const timingPath = path19.join(runDir, "timing.json");
|
|
9482
9846
|
writeJson3(timingPath, timing);
|
|
9483
9847
|
const benchmark = buildBenchmarkArtifact(results, "", "combined", results.length);
|
|
9484
9848
|
const benchmarkWithMetadata = {
|
|
@@ -9492,7 +9856,7 @@ function combineRunSources(options) {
|
|
|
9492
9856
|
duplicate_policy: options.duplicatePolicy
|
|
9493
9857
|
}
|
|
9494
9858
|
};
|
|
9495
|
-
const benchmarkPath =
|
|
9859
|
+
const benchmarkPath = path19.join(runDir, "benchmark.json");
|
|
9496
9860
|
writeJson3(benchmarkPath, benchmarkWithMetadata);
|
|
9497
9861
|
const tags = [...new Set(loadedSources.flatMap((source) => source.tags ?? []))].sort();
|
|
9498
9862
|
return {
|
|
@@ -9628,19 +9992,19 @@ import * as readline3 from "node:readline/promises";
|
|
|
9628
9992
|
|
|
9629
9993
|
// src/commands/results/delete-run.ts
|
|
9630
9994
|
import { existsSync as existsSync9, rmSync } from "node:fs";
|
|
9631
|
-
import
|
|
9995
|
+
import path20 from "node:path";
|
|
9632
9996
|
function localRunsRoot(cwd) {
|
|
9633
|
-
return
|
|
9997
|
+
return path20.resolve(cwd, ".agentv", "results", "runs");
|
|
9634
9998
|
}
|
|
9635
9999
|
function assertLocalRunManifest(cwd, manifestPath, runId) {
|
|
9636
|
-
const resolvedManifestPath =
|
|
9637
|
-
if (
|
|
10000
|
+
const resolvedManifestPath = path20.resolve(manifestPath);
|
|
10001
|
+
if (path20.basename(resolvedManifestPath) !== RESULT_INDEX_FILENAME) {
|
|
9638
10002
|
throw new Error("Expected a run workspace directory or index.jsonl manifest");
|
|
9639
10003
|
}
|
|
9640
|
-
const runDir =
|
|
10004
|
+
const runDir = path20.dirname(resolvedManifestPath);
|
|
9641
10005
|
const runsRoot = localRunsRoot(cwd);
|
|
9642
|
-
const relativeRunDir =
|
|
9643
|
-
if (relativeRunDir === "" || relativeRunDir.startsWith("..") ||
|
|
10006
|
+
const relativeRunDir = path20.relative(runsRoot, runDir);
|
|
10007
|
+
if (relativeRunDir === "" || relativeRunDir.startsWith("..") || path20.isAbsolute(relativeRunDir)) {
|
|
9644
10008
|
throw new Error("Run workspace is outside the local results directory");
|
|
9645
10009
|
}
|
|
9646
10010
|
if (!existsSync9(resolvedManifestPath)) {
|
|
@@ -9726,7 +10090,7 @@ var resultsDeleteCommand = command({
|
|
|
9726
10090
|
});
|
|
9727
10091
|
|
|
9728
10092
|
// src/commands/results/export.ts
|
|
9729
|
-
import
|
|
10093
|
+
import path21 from "node:path";
|
|
9730
10094
|
|
|
9731
10095
|
// src/commands/results/shared.ts
|
|
9732
10096
|
import { existsSync as existsSync10 } from "node:fs";
|
|
@@ -9773,20 +10137,20 @@ async function loadResults(source, cwd) {
|
|
|
9773
10137
|
|
|
9774
10138
|
// src/commands/results/export.ts
|
|
9775
10139
|
function deriveOutputDir(cwd, sourceFile) {
|
|
9776
|
-
if (
|
|
10140
|
+
if (path21.basename(sourceFile) !== RESULT_INDEX_FILENAME) {
|
|
9777
10141
|
throw new Error(`Expected a run manifest named ${RESULT_INDEX_FILENAME}: ${sourceFile}`);
|
|
9778
10142
|
}
|
|
9779
|
-
const runDir =
|
|
9780
|
-
const segments =
|
|
10143
|
+
const runDir = path21.dirname(sourceFile);
|
|
10144
|
+
const segments = path21.normalize(runDir).split(path21.sep).filter(Boolean);
|
|
9781
10145
|
const runsIndex = segments.lastIndexOf("runs");
|
|
9782
10146
|
if (runsIndex >= 0 && runsIndex < segments.length - 1) {
|
|
9783
|
-
return
|
|
10147
|
+
return path21.join(cwd, ".agentv", "results", "export", ...segments.slice(runsIndex + 1));
|
|
9784
10148
|
}
|
|
9785
|
-
const parentDir =
|
|
10149
|
+
const parentDir = path21.basename(runDir);
|
|
9786
10150
|
if (parentDir.startsWith("eval_")) {
|
|
9787
|
-
return
|
|
10151
|
+
return path21.join(cwd, ".agentv", "results", "export", parentDir.slice(5));
|
|
9788
10152
|
}
|
|
9789
|
-
return
|
|
10153
|
+
return path21.join(cwd, ".agentv", "results", "export", parentDir);
|
|
9790
10154
|
}
|
|
9791
10155
|
async function loadExportSource(source, cwd) {
|
|
9792
10156
|
const { sourceFile } = await resolveSourceFile(source, cwd);
|
|
@@ -9819,7 +10183,7 @@ var resultsExportCommand = command({
|
|
|
9819
10183
|
const cwd = dir ?? process.cwd();
|
|
9820
10184
|
try {
|
|
9821
10185
|
const { sourceFile, results } = await loadExportSource(source, cwd);
|
|
9822
|
-
const outputDir = out ?
|
|
10186
|
+
const outputDir = out ? path21.isAbsolute(out) ? out : path21.resolve(cwd, out) : deriveOutputDir(cwd, sourceFile);
|
|
9823
10187
|
await writeArtifactsFromResults2(results, outputDir, {
|
|
9824
10188
|
evalFile: sourceFile
|
|
9825
10189
|
});
|
|
@@ -9880,7 +10244,7 @@ var resultsFailuresCommand = command({
|
|
|
9880
10244
|
|
|
9881
10245
|
// src/commands/results/report.ts
|
|
9882
10246
|
import { existsSync as existsSync11, mkdirSync as mkdirSync3, readFileSync as readFileSync8, writeFileSync as writeFileSync4 } from "node:fs";
|
|
9883
|
-
import
|
|
10247
|
+
import path22 from "node:path";
|
|
9884
10248
|
|
|
9885
10249
|
// src/commands/results/report-template.ts
|
|
9886
10250
|
var RESULTS_REPORT_TEMPLATE = `<!DOCTYPE html>
|
|
@@ -11504,10 +11868,10 @@ function normalizeEvalFileLabel(value) {
|
|
|
11504
11868
|
if (!trimmed) {
|
|
11505
11869
|
return void 0;
|
|
11506
11870
|
}
|
|
11507
|
-
return
|
|
11871
|
+
return path22.basename(trimmed).replace(/\.results\.jsonl$/i, "").replace(/\.eval\.ya?ml$/i, "").replace(/\.ya?ml$/i, "").replace(/\.jsonl$/i, "");
|
|
11508
11872
|
}
|
|
11509
11873
|
function readBenchmarkEvalFile(sourceFile) {
|
|
11510
|
-
const benchmarkPath =
|
|
11874
|
+
const benchmarkPath = path22.join(path22.dirname(sourceFile), "benchmark.json");
|
|
11511
11875
|
if (!existsSync11(benchmarkPath)) {
|
|
11512
11876
|
return void 0;
|
|
11513
11877
|
}
|
|
@@ -11519,10 +11883,10 @@ function readBenchmarkEvalFile(sourceFile) {
|
|
|
11519
11883
|
}
|
|
11520
11884
|
}
|
|
11521
11885
|
function deriveReportPath(sourceFile) {
|
|
11522
|
-
return
|
|
11886
|
+
return path22.join(path22.dirname(sourceFile), "report.html");
|
|
11523
11887
|
}
|
|
11524
11888
|
function serializeReportResult(result, sourceFile, manifestRecord, benchmarkEvalFile) {
|
|
11525
|
-
const fallbackEvalFile = normalizeEvalFileLabel(manifestRecord?.eval_file) ?? benchmarkEvalFile ?? normalizeEvalFileLabel(result.suite) ??
|
|
11889
|
+
const fallbackEvalFile = normalizeEvalFileLabel(manifestRecord?.eval_file) ?? benchmarkEvalFile ?? normalizeEvalFileLabel(result.suite) ?? path22.basename(path22.dirname(sourceFile));
|
|
11526
11890
|
return {
|
|
11527
11891
|
timestamp: result.timestamp,
|
|
11528
11892
|
test_id: result.testId,
|
|
@@ -11570,9 +11934,9 @@ function renderResultsReport(results, sourceFile, records, benchmarkEvalFile) {
|
|
|
11570
11934
|
}
|
|
11571
11935
|
async function writeResultsReport(source, outputPath, cwd) {
|
|
11572
11936
|
const { sourceFile, results, records, benchmarkEvalFile } = await loadReportSource(source, cwd);
|
|
11573
|
-
const resolvedOutputPath = outputPath ?
|
|
11937
|
+
const resolvedOutputPath = outputPath ? path22.isAbsolute(outputPath) ? outputPath : path22.resolve(cwd, outputPath) : deriveReportPath(sourceFile);
|
|
11574
11938
|
const html = renderResultsReport(results, sourceFile, records, benchmarkEvalFile);
|
|
11575
|
-
mkdirSync3(
|
|
11939
|
+
mkdirSync3(path22.dirname(resolvedOutputPath), { recursive: true });
|
|
11576
11940
|
writeFileSync4(resolvedOutputPath, html, "utf8");
|
|
11577
11941
|
const written = readFileSync8(resolvedOutputPath, "utf8");
|
|
11578
11942
|
if (written.includes("__DATA_PLACEHOLDER__")) {
|
|
@@ -11760,10 +12124,10 @@ var resultsSummaryCommand = command({
|
|
|
11760
12124
|
|
|
11761
12125
|
// src/commands/results/validate.ts
|
|
11762
12126
|
import { existsSync as existsSync13, readFileSync as readFileSync10, statSync as statSync5 } from "node:fs";
|
|
11763
|
-
import
|
|
12127
|
+
import path23 from "node:path";
|
|
11764
12128
|
function checkDirectoryNaming(runDir) {
|
|
11765
|
-
const dirName =
|
|
11766
|
-
const pathSegments =
|
|
12129
|
+
const dirName = path23.basename(runDir);
|
|
12130
|
+
const pathSegments = path23.normalize(runDir).split(path23.sep).filter(Boolean);
|
|
11767
12131
|
const runsIndex = pathSegments.lastIndexOf("runs");
|
|
11768
12132
|
const diagnostics = [];
|
|
11769
12133
|
if (runsIndex < 0 || runsIndex >= pathSegments.length - 1) {
|
|
@@ -11793,7 +12157,7 @@ function validateRunDirectory(runDir) {
|
|
|
11793
12157
|
return { diagnostics, entries: entries2 };
|
|
11794
12158
|
}
|
|
11795
12159
|
function checkIndexJsonl(runDir) {
|
|
11796
|
-
const indexPath =
|
|
12160
|
+
const indexPath = path23.join(runDir, "index.jsonl");
|
|
11797
12161
|
const diagnostics = [];
|
|
11798
12162
|
const entries2 = [];
|
|
11799
12163
|
if (!existsSync13(indexPath)) {
|
|
@@ -11892,7 +12256,7 @@ function checkArtifactFiles(runDir, entries2) {
|
|
|
11892
12256
|
for (const entry of entries2) {
|
|
11893
12257
|
const testId = entry.test_id ?? "?";
|
|
11894
12258
|
if (entry.grading_path) {
|
|
11895
|
-
const gradingPath =
|
|
12259
|
+
const gradingPath = path23.join(runDir, entry.grading_path);
|
|
11896
12260
|
if (!existsSync13(gradingPath)) {
|
|
11897
12261
|
diagnostics.push({
|
|
11898
12262
|
severity: "error",
|
|
@@ -11922,7 +12286,7 @@ function checkArtifactFiles(runDir, entries2) {
|
|
|
11922
12286
|
}
|
|
11923
12287
|
}
|
|
11924
12288
|
if (entry.timing_path) {
|
|
11925
|
-
const timingPath =
|
|
12289
|
+
const timingPath = path23.join(runDir, entry.timing_path);
|
|
11926
12290
|
if (!existsSync13(timingPath)) {
|
|
11927
12291
|
diagnostics.push({
|
|
11928
12292
|
severity: "warning",
|
|
@@ -11931,7 +12295,7 @@ function checkArtifactFiles(runDir, entries2) {
|
|
|
11931
12295
|
}
|
|
11932
12296
|
}
|
|
11933
12297
|
}
|
|
11934
|
-
const benchmarkPath =
|
|
12298
|
+
const benchmarkPath = path23.join(runDir, "benchmark.json");
|
|
11935
12299
|
if (!existsSync13(benchmarkPath)) {
|
|
11936
12300
|
diagnostics.push({ severity: "warning", message: "benchmark.json is missing" });
|
|
11937
12301
|
}
|
|
@@ -11948,7 +12312,7 @@ var resultsValidateCommand = command({
|
|
|
11948
12312
|
})
|
|
11949
12313
|
},
|
|
11950
12314
|
handler: async ({ runDir }) => {
|
|
11951
|
-
const resolvedDir =
|
|
12315
|
+
const resolvedDir = path23.resolve(runDir);
|
|
11952
12316
|
if (!existsSync13(resolvedDir) || !statSync5(resolvedDir).isDirectory()) {
|
|
11953
12317
|
console.error(`Error: '${runDir}' is not a directory`);
|
|
11954
12318
|
process.exit(1);
|
|
@@ -11994,14 +12358,14 @@ var resultsCommand = subcommands({
|
|
|
11994
12358
|
|
|
11995
12359
|
// src/commands/results/serve.ts
|
|
11996
12360
|
import { existsSync as existsSync16, readFileSync as readFileSync12, readdirSync as readdirSync4, statSync as statSync6, writeFileSync as writeFileSync6 } from "node:fs";
|
|
11997
|
-
import
|
|
12361
|
+
import path26 from "node:path";
|
|
11998
12362
|
import { fileURLToPath as fileURLToPath3 } from "node:url";
|
|
11999
12363
|
import { Hono } from "hono";
|
|
12000
12364
|
|
|
12001
12365
|
// src/commands/results/eval-runner.ts
|
|
12002
12366
|
import { execFileSync as execFileSync2, spawn } from "node:child_process";
|
|
12003
12367
|
import { createWriteStream, existsSync as existsSync14, mkdirSync as mkdirSync4 } from "node:fs";
|
|
12004
|
-
import
|
|
12368
|
+
import path24 from "node:path";
|
|
12005
12369
|
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
12006
12370
|
var activeRuns = /* @__PURE__ */ new Map();
|
|
12007
12371
|
function generateRunId() {
|
|
@@ -12021,7 +12385,7 @@ function pruneFinishedRuns() {
|
|
|
12021
12385
|
}
|
|
12022
12386
|
function getActiveRunTarget(indexJsonlPath) {
|
|
12023
12387
|
for (const run2 of activeRuns.values()) {
|
|
12024
|
-
if (run2.outputDir &&
|
|
12388
|
+
if (run2.outputDir && path24.join(run2.outputDir, "index.jsonl") === indexJsonlPath) {
|
|
12025
12389
|
return run2.target;
|
|
12026
12390
|
}
|
|
12027
12391
|
}
|
|
@@ -12029,7 +12393,7 @@ function getActiveRunTarget(indexJsonlPath) {
|
|
|
12029
12393
|
}
|
|
12030
12394
|
function getActiveRunStatus(indexJsonlPath) {
|
|
12031
12395
|
for (const run2 of activeRuns.values()) {
|
|
12032
|
-
if (run2.outputDir &&
|
|
12396
|
+
if (run2.outputDir && path24.join(run2.outputDir, "index.jsonl") === indexJsonlPath) {
|
|
12033
12397
|
return run2.status;
|
|
12034
12398
|
}
|
|
12035
12399
|
}
|
|
@@ -12039,7 +12403,7 @@ async function discoverTargetsInProject(cwd) {
|
|
|
12039
12403
|
const repoRoot = await findRepoRoot(cwd) ?? cwd;
|
|
12040
12404
|
let targetsFilePath;
|
|
12041
12405
|
for (const candidate of TARGET_FILE_CANDIDATES) {
|
|
12042
|
-
const fullPath =
|
|
12406
|
+
const fullPath = path24.join(cwd, candidate);
|
|
12043
12407
|
if (existsSync14(fullPath)) {
|
|
12044
12408
|
targetsFilePath = fullPath;
|
|
12045
12409
|
break;
|
|
@@ -12047,7 +12411,7 @@ async function discoverTargetsInProject(cwd) {
|
|
|
12047
12411
|
}
|
|
12048
12412
|
if (!targetsFilePath) {
|
|
12049
12413
|
for (const candidate of TARGET_FILE_CANDIDATES) {
|
|
12050
|
-
const fullPath =
|
|
12414
|
+
const fullPath = path24.join(repoRoot, candidate);
|
|
12051
12415
|
if (existsSync14(fullPath)) {
|
|
12052
12416
|
targetsFilePath = fullPath;
|
|
12053
12417
|
break;
|
|
@@ -12121,17 +12485,17 @@ function buildCliPreview(args) {
|
|
|
12121
12485
|
}
|
|
12122
12486
|
function resolveCliPath(cwd) {
|
|
12123
12487
|
const candidates = [
|
|
12124
|
-
|
|
12125
|
-
|
|
12488
|
+
path24.join(cwd, "apps/cli/src/cli.ts"),
|
|
12489
|
+
path24.join(cwd, "apps/cli/dist/cli.js")
|
|
12126
12490
|
];
|
|
12127
12491
|
for (const c4 of candidates) {
|
|
12128
12492
|
if (existsSync14(c4)) {
|
|
12129
12493
|
return { binPath: "bun", args: [c4] };
|
|
12130
12494
|
}
|
|
12131
12495
|
}
|
|
12132
|
-
const currentDir = typeof __dirname !== "undefined" ? __dirname :
|
|
12133
|
-
const fromSrc =
|
|
12134
|
-
const fromDist =
|
|
12496
|
+
const currentDir = typeof __dirname !== "undefined" ? __dirname : path24.dirname(fileURLToPath2(import.meta.url));
|
|
12497
|
+
const fromSrc = path24.resolve(currentDir, "../../cli.ts");
|
|
12498
|
+
const fromDist = path24.resolve(currentDir, "cli.js");
|
|
12135
12499
|
if (existsSync14(fromSrc)) return { binPath: "bun", args: [fromSrc] };
|
|
12136
12500
|
if (existsSync14(fromDist)) return { binPath: "bun", args: [fromDist] };
|
|
12137
12501
|
if (isCommandAvailable("agentv")) {
|
|
@@ -12150,7 +12514,7 @@ function isCommandAvailable(cmd) {
|
|
|
12150
12514
|
function openConsoleLogStream(outputDir) {
|
|
12151
12515
|
try {
|
|
12152
12516
|
mkdirSync4(outputDir, { recursive: true });
|
|
12153
|
-
const stream = createWriteStream(
|
|
12517
|
+
const stream = createWriteStream(path24.join(outputDir, "console.log"), { flags: "w" });
|
|
12154
12518
|
stream.on("error", () => {
|
|
12155
12519
|
});
|
|
12156
12520
|
return stream;
|
|
@@ -12207,7 +12571,7 @@ function registerEvalRoutes(app2, getCwd, options) {
|
|
|
12207
12571
|
return c4.json({ error: "Cannot locate agentv CLI entry point" }, 500);
|
|
12208
12572
|
}
|
|
12209
12573
|
const args = buildCliArgs(body);
|
|
12210
|
-
const outputDir = body.output?.trim() ?
|
|
12574
|
+
const outputDir = body.output?.trim() ? path24.resolve(cwd, body.output.trim()) : buildDefaultRunDir(cwd);
|
|
12211
12575
|
if (!body.output?.trim()) {
|
|
12212
12576
|
args.push("--output", outputDir);
|
|
12213
12577
|
}
|
|
@@ -12385,7 +12749,7 @@ Process error: ${err2.message}
|
|
|
12385
12749
|
return c4.json({ error: "Cannot locate agentv CLI entry point" }, 500);
|
|
12386
12750
|
}
|
|
12387
12751
|
const args = buildCliArgs(body);
|
|
12388
|
-
const outputDir = body.output?.trim() ?
|
|
12752
|
+
const outputDir = body.output?.trim() ? path24.resolve(cwd, body.output.trim()) : buildDefaultRunDir(cwd);
|
|
12389
12753
|
if (!body.output?.trim()) {
|
|
12390
12754
|
args.push("--output", outputDir);
|
|
12391
12755
|
}
|
|
@@ -12508,17 +12872,17 @@ Process error: ${err2.message}
|
|
|
12508
12872
|
|
|
12509
12873
|
// src/commands/results/studio-config.ts
|
|
12510
12874
|
import { existsSync as existsSync15, mkdirSync as mkdirSync5, readFileSync as readFileSync11, writeFileSync as writeFileSync5 } from "node:fs";
|
|
12511
|
-
import
|
|
12875
|
+
import path25 from "node:path";
|
|
12512
12876
|
import { stringify as stringifyYaml3 } from "yaml";
|
|
12513
12877
|
var DEFAULTS = {
|
|
12514
12878
|
threshold: DEFAULT_THRESHOLD,
|
|
12515
12879
|
appName: "agentv"
|
|
12516
12880
|
};
|
|
12517
12881
|
function loadStudioConfig(agentvDir) {
|
|
12518
|
-
const localConfigPath =
|
|
12519
|
-
const globalConfigPath =
|
|
12882
|
+
const localConfigPath = path25.join(agentvDir, "config.yaml");
|
|
12883
|
+
const globalConfigPath = path25.join(getAgentvConfigDir(), "config.yaml");
|
|
12520
12884
|
const localConfig = loadParsedConfig(localConfigPath);
|
|
12521
|
-
const globalConfig =
|
|
12885
|
+
const globalConfig = path25.resolve(globalConfigPath) === path25.resolve(localConfigPath) ? void 0 : loadParsedConfig(globalConfigPath);
|
|
12522
12886
|
const threshold = [
|
|
12523
12887
|
readThreshold(localConfig?.dashboard),
|
|
12524
12888
|
readThreshold(localConfig?.studio),
|
|
@@ -12558,7 +12922,7 @@ function saveStudioConfig(agentvDir, config) {
|
|
|
12558
12922
|
if (!existsSync15(agentvDir)) {
|
|
12559
12923
|
mkdirSync5(agentvDir, { recursive: true });
|
|
12560
12924
|
}
|
|
12561
|
-
const configPath =
|
|
12925
|
+
const configPath = path25.join(agentvDir, "config.yaml");
|
|
12562
12926
|
let existing = {};
|
|
12563
12927
|
if (existsSync15(configPath)) {
|
|
12564
12928
|
const raw = readFileSync11(configPath, "utf-8");
|
|
@@ -12590,14 +12954,23 @@ function saveStudioConfig(agentvDir, config) {
|
|
|
12590
12954
|
}
|
|
12591
12955
|
|
|
12592
12956
|
// src/commands/results/serve.ts
|
|
12957
|
+
var DIRECT_DASHBOARD_SOURCE_GUIDANCE = [
|
|
12958
|
+
"Dashboard reads configured project run sources only.",
|
|
12959
|
+
"Run it from a project root, or pass --dir so Dashboard uses <project>/.agentv/results/runs/:",
|
|
12960
|
+
" agentv dashboard --dir <project-dir>",
|
|
12961
|
+
"To browse external results, configure results.repo_url or results.repo_path in config YAML.",
|
|
12962
|
+
"For a one-off run bundle, use: agentv results report <run-workspace-or-index.jsonl>"
|
|
12963
|
+
].join("\n");
|
|
12964
|
+
function unsupportedDashboardSourceError(source, cwd) {
|
|
12965
|
+
const resolved = path26.isAbsolute(source) ? source : path26.resolve(cwd, source);
|
|
12966
|
+
return new Error(
|
|
12967
|
+
`Unsupported Dashboard source: ${resolved}
|
|
12968
|
+
${DIRECT_DASHBOARD_SOURCE_GUIDANCE}`
|
|
12969
|
+
);
|
|
12970
|
+
}
|
|
12593
12971
|
async function resolveSourceFile2(source, cwd) {
|
|
12594
12972
|
if (source) {
|
|
12595
|
-
|
|
12596
|
-
if (!existsSync16(resolved)) {
|
|
12597
|
-
throw new Error(`Source file not found: ${resolved}`);
|
|
12598
|
-
}
|
|
12599
|
-
resolved = resolveRunManifestPath(resolved);
|
|
12600
|
-
return resolved;
|
|
12973
|
+
throw unsupportedDashboardSourceError(source, cwd);
|
|
12601
12974
|
}
|
|
12602
12975
|
const cache = await loadRunCache(cwd);
|
|
12603
12976
|
const cachedFile = cache ? resolveRunCacheFile(cache) : "";
|
|
@@ -12629,13 +13002,13 @@ function resolveDashboardMode(_projectCount, options) {
|
|
|
12629
13002
|
}
|
|
12630
13003
|
function bootstrapCurrentProject(cwd, options) {
|
|
12631
13004
|
if (options.single === true) return {};
|
|
12632
|
-
if (!existsSync16(
|
|
13005
|
+
if (!existsSync16(path26.join(cwd, ".agentv"))) return {};
|
|
12633
13006
|
const entry = addProject(cwd);
|
|
12634
13007
|
touchProject(entry.id);
|
|
12635
13008
|
return { currentProjectId: entry.id };
|
|
12636
13009
|
}
|
|
12637
13010
|
function feedbackPath(resultDir) {
|
|
12638
|
-
return
|
|
13011
|
+
return path26.join(resultDir, "feedback.json");
|
|
12639
13012
|
}
|
|
12640
13013
|
function readFeedback(cwd) {
|
|
12641
13014
|
const fp = feedbackPath(cwd);
|
|
@@ -12662,8 +13035,8 @@ function buildFileTree(dirPath, relativeTo) {
|
|
|
12662
13035
|
if (a.isDirectory() !== b.isDirectory()) return a.isDirectory() ? -1 : 1;
|
|
12663
13036
|
return a.name.localeCompare(b.name);
|
|
12664
13037
|
}).map((entry) => {
|
|
12665
|
-
const fullPath =
|
|
12666
|
-
const relPath =
|
|
13038
|
+
const fullPath = path26.join(dirPath, entry.name);
|
|
13039
|
+
const relPath = path26.relative(relativeTo, fullPath);
|
|
12667
13040
|
if (entry.isDirectory()) {
|
|
12668
13041
|
return {
|
|
12669
13042
|
name: entry.name,
|
|
@@ -12676,7 +13049,7 @@ function buildFileTree(dirPath, relativeTo) {
|
|
|
12676
13049
|
});
|
|
12677
13050
|
}
|
|
12678
13051
|
function inferLanguage(filePath) {
|
|
12679
|
-
const ext =
|
|
13052
|
+
const ext = path26.extname(filePath).toLowerCase();
|
|
12680
13053
|
const langMap = {
|
|
12681
13054
|
".json": "json",
|
|
12682
13055
|
".jsonl": "json",
|
|
@@ -12703,14 +13076,14 @@ function inferLanguage(filePath) {
|
|
|
12703
13076
|
return langMap[ext] ?? "plaintext";
|
|
12704
13077
|
}
|
|
12705
13078
|
function inferRawContentType(filePath) {
|
|
12706
|
-
const ext =
|
|
13079
|
+
const ext = path26.extname(filePath).toLowerCase();
|
|
12707
13080
|
if (ext === ".json") return "application/json; charset=utf-8";
|
|
12708
13081
|
if (ext === ".jsonl") return "text/plain; charset=utf-8";
|
|
12709
13082
|
if (ext === ".md") return "text/markdown; charset=utf-8";
|
|
12710
13083
|
return "text/plain; charset=utf-8";
|
|
12711
13084
|
}
|
|
12712
13085
|
function contentDispositionFilename(filePath) {
|
|
12713
|
-
return
|
|
13086
|
+
return path26.basename(filePath).replace(/["\\\r\n]/g, "_");
|
|
12714
13087
|
}
|
|
12715
13088
|
function stripHeavyFields(results) {
|
|
12716
13089
|
return results.map((r) => {
|
|
@@ -12930,7 +13303,7 @@ async function handleRunLog(c4, { searchDir, projectId }) {
|
|
|
12930
13303
|
if (meta.source === "remote") {
|
|
12931
13304
|
return c4.json({ error: "Run log is not available for remote runs" }, 404);
|
|
12932
13305
|
}
|
|
12933
|
-
const logPath =
|
|
13306
|
+
const logPath = path26.join(path26.dirname(meta.path), "console.log");
|
|
12934
13307
|
if (!existsSync16(logPath)) {
|
|
12935
13308
|
return c4.json({ error: "Run log not found for this run" }, 404);
|
|
12936
13309
|
}
|
|
@@ -12964,11 +13337,11 @@ async function handleRunDetail(c4, { searchDir, projectId }) {
|
|
|
12964
13337
|
}
|
|
12965
13338
|
function deriveResumeMeta(cwd, manifestPath) {
|
|
12966
13339
|
const out = {};
|
|
12967
|
-
const runDir =
|
|
12968
|
-
const relative3 =
|
|
13340
|
+
const runDir = path26.dirname(manifestPath);
|
|
13341
|
+
const relative3 = path26.relative(cwd, runDir);
|
|
12969
13342
|
out.run_dir = relative3 !== "" && !relative3.startsWith("..") ? relative3 : runDir;
|
|
12970
13343
|
try {
|
|
12971
|
-
const benchmarkPath =
|
|
13344
|
+
const benchmarkPath = path26.join(runDir, "benchmark.json");
|
|
12972
13345
|
if (existsSync16(benchmarkPath)) {
|
|
12973
13346
|
const parsed = JSON.parse(readFileSync12(benchmarkPath, "utf8"));
|
|
12974
13347
|
const evalFile = parsed.metadata?.eval_file;
|
|
@@ -13105,7 +13478,7 @@ async function handleEvalFiles(c4, { searchDir, projectId }) {
|
|
|
13105
13478
|
const records = await parseManifestForMeta(searchDir, meta, projectId);
|
|
13106
13479
|
const record = records.find((r) => r.test_id === evalId);
|
|
13107
13480
|
if (!record) return c4.json({ error: "Eval not found" }, 404);
|
|
13108
|
-
const baseDir =
|
|
13481
|
+
const baseDir = path26.dirname(meta.path);
|
|
13109
13482
|
const knownPaths = [
|
|
13110
13483
|
record.grading_path,
|
|
13111
13484
|
record.timing_path,
|
|
@@ -13121,14 +13494,14 @@ async function handleEvalFiles(c4, { searchDir, projectId }) {
|
|
|
13121
13494
|
record.graders_path
|
|
13122
13495
|
].filter((p) => !!p);
|
|
13123
13496
|
if (knownPaths.length === 0) return c4.json({ files: [] });
|
|
13124
|
-
const artifactDirs = knownPaths.map((p) =>
|
|
13497
|
+
const artifactDirs = knownPaths.map((p) => path26.dirname(p));
|
|
13125
13498
|
let commonDir = artifactDirs[0];
|
|
13126
13499
|
for (const dir of artifactDirs) {
|
|
13127
13500
|
while (!dir.startsWith(commonDir)) {
|
|
13128
|
-
commonDir =
|
|
13501
|
+
commonDir = path26.dirname(commonDir);
|
|
13129
13502
|
}
|
|
13130
13503
|
}
|
|
13131
|
-
const artifactAbsDir =
|
|
13504
|
+
const artifactAbsDir = path26.join(baseDir, commonDir);
|
|
13132
13505
|
const files = buildFileTree(artifactAbsDir, baseDir);
|
|
13133
13506
|
return c4.json({ files });
|
|
13134
13507
|
} catch {
|
|
@@ -13150,9 +13523,9 @@ async function handleEvalFileContent(c4, { searchDir, projectId }) {
|
|
|
13150
13523
|
}
|
|
13151
13524
|
if (!filePath) return c4.json({ error: "No file path specified" }, 400);
|
|
13152
13525
|
await ensureRunReadable(searchDir, meta, projectId);
|
|
13153
|
-
const baseDir =
|
|
13154
|
-
const absolutePath =
|
|
13155
|
-
if (!absolutePath.startsWith(
|
|
13526
|
+
const baseDir = path26.dirname(meta.path);
|
|
13527
|
+
const absolutePath = path26.resolve(baseDir, filePath);
|
|
13528
|
+
if (!absolutePath.startsWith(path26.resolve(baseDir) + path26.sep) && absolutePath !== path26.resolve(baseDir)) {
|
|
13156
13529
|
return c4.json({ error: "Path traversal not allowed" }, 403);
|
|
13157
13530
|
}
|
|
13158
13531
|
if (!existsSync16(absolutePath) || !statSync6(absolutePath).isFile()) {
|
|
@@ -13423,13 +13796,13 @@ function handleConfig(c4, { agentvDir, searchDir }, options) {
|
|
|
13423
13796
|
threshold: config.threshold,
|
|
13424
13797
|
app_name: config.appName,
|
|
13425
13798
|
read_only: options?.readOnly === true,
|
|
13426
|
-
project_name:
|
|
13799
|
+
project_name: path26.basename(searchDir),
|
|
13427
13800
|
project_dashboard: options?.projectDashboard === true,
|
|
13428
13801
|
...options?.currentProjectId && { current_project_id: options.currentProjectId }
|
|
13429
13802
|
});
|
|
13430
13803
|
}
|
|
13431
13804
|
function handleFeedbackRead(c4, { searchDir }) {
|
|
13432
|
-
const resultsDir =
|
|
13805
|
+
const resultsDir = path26.join(searchDir, ".agentv", "results");
|
|
13433
13806
|
return c4.json(readFeedback(existsSync16(resultsDir) ? resultsDir : searchDir));
|
|
13434
13807
|
}
|
|
13435
13808
|
async function handleRunTagsPut(c4, { searchDir, projectId }) {
|
|
@@ -13499,7 +13872,7 @@ async function handleRunDelete(c4, { searchDir, projectId }) {
|
|
|
13499
13872
|
}
|
|
13500
13873
|
}
|
|
13501
13874
|
function getLocalRunsRoot(searchDir) {
|
|
13502
|
-
return
|
|
13875
|
+
return path26.join(searchDir, ".agentv", "results", "runs");
|
|
13503
13876
|
}
|
|
13504
13877
|
function validateLocalCompletedRun(searchDir, meta, actionName = "Run combine") {
|
|
13505
13878
|
if (meta.source === "remote") {
|
|
@@ -13508,13 +13881,13 @@ function validateLocalCompletedRun(searchDir, meta, actionName = "Run combine")
|
|
|
13508
13881
|
if (getActiveRunStatus(meta.path) === "starting" || getActiveRunStatus(meta.path) === "running") {
|
|
13509
13882
|
return { error: "Run is still active", status: 409 };
|
|
13510
13883
|
}
|
|
13511
|
-
const manifestPath =
|
|
13512
|
-
if (
|
|
13884
|
+
const manifestPath = path26.resolve(meta.path);
|
|
13885
|
+
if (path26.basename(manifestPath) !== "index.jsonl") {
|
|
13513
13886
|
return { error: "Run workspace is invalid", status: 400 };
|
|
13514
13887
|
}
|
|
13515
|
-
const runDir =
|
|
13516
|
-
const runsRoot =
|
|
13517
|
-
if (runDir !== runsRoot && runDir.startsWith(`${runsRoot}${
|
|
13888
|
+
const runDir = path26.dirname(manifestPath);
|
|
13889
|
+
const runsRoot = path26.resolve(getLocalRunsRoot(searchDir));
|
|
13890
|
+
if (runDir !== runsRoot && runDir.startsWith(`${runsRoot}${path26.sep}`) && existsSync16(runDir)) {
|
|
13518
13891
|
return { ok: true };
|
|
13519
13892
|
}
|
|
13520
13893
|
return { error: "Run workspace is outside the local results directory", status: 400 };
|
|
@@ -13596,7 +13969,7 @@ async function handleRunsCombine(c4, { searchDir, projectId }) {
|
|
|
13596
13969
|
}
|
|
13597
13970
|
function createApp(results, resultDir, cwd, sourceFile, options) {
|
|
13598
13971
|
const searchDir = cwd ?? resultDir;
|
|
13599
|
-
const agentvDir =
|
|
13972
|
+
const agentvDir = path26.join(searchDir, ".agentv");
|
|
13600
13973
|
const defaultCtx = { searchDir, agentvDir, projectId: options?.currentProjectId };
|
|
13601
13974
|
const readOnly = options?.readOnly === true;
|
|
13602
13975
|
const app2 = new Hono();
|
|
@@ -13607,7 +13980,7 @@ function createApp(results, resultDir, cwd, sourceFile, options) {
|
|
|
13607
13980
|
}
|
|
13608
13981
|
return handler(c4, {
|
|
13609
13982
|
searchDir: project.path,
|
|
13610
|
-
agentvDir:
|
|
13983
|
+
agentvDir: path26.join(project.path, ".agentv"),
|
|
13611
13984
|
projectId: project.id
|
|
13612
13985
|
});
|
|
13613
13986
|
}
|
|
@@ -13639,7 +14012,7 @@ function createApp(results, resultDir, cwd, sourceFile, options) {
|
|
|
13639
14012
|
}
|
|
13640
14013
|
async function summarizeProjectRunMetas(project) {
|
|
13641
14014
|
const { runs: metas } = await listMergedResultFiles(project.path, void 0, project.id);
|
|
13642
|
-
const threshold = loadStudioConfig(
|
|
14015
|
+
const threshold = loadStudioConfig(path26.join(project.path, ".agentv")).threshold;
|
|
13643
14016
|
let passRateSum = 0;
|
|
13644
14017
|
let executionErrorCount = 0;
|
|
13645
14018
|
for (const meta of metas) {
|
|
@@ -13735,7 +14108,7 @@ function createApp(results, resultDir, cwd, sourceFile, options) {
|
|
|
13735
14108
|
if (records.length > 0) {
|
|
13736
14109
|
const qualitySummary = summarizeQualityResults(
|
|
13737
14110
|
records,
|
|
13738
|
-
loadStudioConfig(
|
|
14111
|
+
loadStudioConfig(path26.join(p.path, ".agentv")).threshold
|
|
13739
14112
|
);
|
|
13740
14113
|
target = records[0].target;
|
|
13741
14114
|
experiment = records[0].experiment ?? experiment;
|
|
@@ -14005,22 +14378,22 @@ function createApp(results, resultDir, cwd, sourceFile, options) {
|
|
|
14005
14378
|
{ readOnly }
|
|
14006
14379
|
);
|
|
14007
14380
|
const studioDistPath = options?.studioDir ?? resolveStudioDistDir();
|
|
14008
|
-
if (!studioDistPath || !existsSync16(
|
|
14381
|
+
if (!studioDistPath || !existsSync16(path26.join(studioDistPath, "index.html"))) {
|
|
14009
14382
|
throw new Error(
|
|
14010
14383
|
'Dashboard dist not found. Run "bun run build" in apps/dashboard/ to build the SPA.'
|
|
14011
14384
|
);
|
|
14012
14385
|
}
|
|
14013
14386
|
app2.get("/", (c4) => {
|
|
14014
|
-
const indexPath =
|
|
14387
|
+
const indexPath = path26.join(studioDistPath, "index.html");
|
|
14015
14388
|
if (existsSync16(indexPath)) return c4.html(readFileSync12(indexPath, "utf8"));
|
|
14016
14389
|
return c4.notFound();
|
|
14017
14390
|
});
|
|
14018
14391
|
app2.get("/assets/*", (c4) => {
|
|
14019
14392
|
const assetPath = c4.req.path;
|
|
14020
|
-
const filePath =
|
|
14393
|
+
const filePath = path26.join(studioDistPath, assetPath);
|
|
14021
14394
|
if (!existsSync16(filePath)) return c4.notFound();
|
|
14022
14395
|
const content = readFileSync12(filePath);
|
|
14023
|
-
const ext =
|
|
14396
|
+
const ext = path26.extname(filePath);
|
|
14024
14397
|
const mimeTypes = {
|
|
14025
14398
|
".js": "application/javascript",
|
|
14026
14399
|
".css": "text/css",
|
|
@@ -14041,26 +14414,26 @@ function createApp(results, resultDir, cwd, sourceFile, options) {
|
|
|
14041
14414
|
});
|
|
14042
14415
|
app2.get("*", (c4) => {
|
|
14043
14416
|
if (c4.req.path.startsWith("/api/")) return c4.json({ error: "Not found" }, 404);
|
|
14044
|
-
const indexPath =
|
|
14417
|
+
const indexPath = path26.join(studioDistPath, "index.html");
|
|
14045
14418
|
if (existsSync16(indexPath)) return c4.html(readFileSync12(indexPath, "utf8"));
|
|
14046
14419
|
return c4.notFound();
|
|
14047
14420
|
});
|
|
14048
14421
|
return app2;
|
|
14049
14422
|
}
|
|
14050
14423
|
function resolveStudioDistDir() {
|
|
14051
|
-
const currentDir = typeof __dirname !== "undefined" ? __dirname :
|
|
14424
|
+
const currentDir = typeof __dirname !== "undefined" ? __dirname : path26.dirname(fileURLToPath3(import.meta.url));
|
|
14052
14425
|
const candidates = [
|
|
14053
14426
|
// From src/commands/results/ → sibling apps/dashboard/dist
|
|
14054
|
-
|
|
14427
|
+
path26.resolve(currentDir, "../../../../dashboard/dist"),
|
|
14055
14428
|
// From dist/ → sibling apps/dashboard/dist (monorepo dev)
|
|
14056
|
-
|
|
14429
|
+
path26.resolve(currentDir, "../../dashboard/dist"),
|
|
14057
14430
|
// Bundled inside CLI dist (published package: dist/dashboard/)
|
|
14058
|
-
|
|
14431
|
+
path26.resolve(currentDir, "dashboard"),
|
|
14059
14432
|
// From dist/ in monorepo root context
|
|
14060
|
-
|
|
14433
|
+
path26.resolve(currentDir, "../../../apps/dashboard/dist")
|
|
14061
14434
|
];
|
|
14062
14435
|
for (const candidate of candidates) {
|
|
14063
|
-
if (existsSync16(candidate) && existsSync16(
|
|
14436
|
+
if (existsSync16(candidate) && existsSync16(path26.join(candidate, "index.html"))) {
|
|
14064
14437
|
return candidate;
|
|
14065
14438
|
}
|
|
14066
14439
|
}
|
|
@@ -14073,7 +14446,7 @@ var resultsServeCommand = command({
|
|
|
14073
14446
|
source: positional({
|
|
14074
14447
|
type: optional(string),
|
|
14075
14448
|
displayName: "source",
|
|
14076
|
-
description: "
|
|
14449
|
+
description: "Legacy direct run source (unsupported); use --dir <project-dir> or results: config"
|
|
14077
14450
|
}),
|
|
14078
14451
|
port: option({
|
|
14079
14452
|
type: optional(number),
|
|
@@ -14130,7 +14503,7 @@ var resultsServeCommand = command({
|
|
|
14130
14503
|
return;
|
|
14131
14504
|
}
|
|
14132
14505
|
const repoRoot = await findRepoRoot(cwd);
|
|
14133
|
-
const yamlConfig = await loadConfig(
|
|
14506
|
+
const yamlConfig = await loadConfig(path26.join(cwd, "_"), repoRoot);
|
|
14134
14507
|
if (yamlConfig?.required_version) {
|
|
14135
14508
|
await enforceRequiredVersion(yamlConfig.required_version);
|
|
14136
14509
|
}
|
|
@@ -14160,7 +14533,7 @@ var resultsServeCommand = command({
|
|
|
14160
14533
|
}
|
|
14161
14534
|
}
|
|
14162
14535
|
}
|
|
14163
|
-
const resultDir = sourceFile ?
|
|
14536
|
+
const resultDir = sourceFile ? path26.dirname(path26.resolve(sourceFile)) : cwd;
|
|
14164
14537
|
const app2 = createApp(results, resultDir, cwd, sourceFile, {
|
|
14165
14538
|
readOnly,
|
|
14166
14539
|
projectDashboard,
|
|
@@ -14197,7 +14570,7 @@ var resultsServeCommand = command({
|
|
|
14197
14570
|
// src/commands/runs/rerun.ts
|
|
14198
14571
|
import { constants } from "node:fs";
|
|
14199
14572
|
import { access, readFile as readFile8 } from "node:fs/promises";
|
|
14200
|
-
import
|
|
14573
|
+
import path27 from "node:path";
|
|
14201
14574
|
import { config as loadDotenv } from "dotenv";
|
|
14202
14575
|
var TASK_EVAL_FILENAME = "EVAL.yaml";
|
|
14203
14576
|
var TASK_TARGETS_FILENAME = "targets.yaml";
|
|
@@ -14209,13 +14582,13 @@ function displayRecord(record) {
|
|
|
14209
14582
|
return `${record.test_id ?? "unknown"}@${record.target ?? "unknown"}`;
|
|
14210
14583
|
}
|
|
14211
14584
|
function resolveSourcePath(cwd, source) {
|
|
14212
|
-
return
|
|
14585
|
+
return path27.isAbsolute(source) ? source : path27.resolve(cwd, source);
|
|
14213
14586
|
}
|
|
14214
14587
|
function resolveRelativeRunPath(runDir, relativePath) {
|
|
14215
14588
|
if (!relativePath || relativePath.trim().length === 0) {
|
|
14216
14589
|
return void 0;
|
|
14217
14590
|
}
|
|
14218
|
-
return
|
|
14591
|
+
return path27.resolve(runDir, relativePath);
|
|
14219
14592
|
}
|
|
14220
14593
|
async function ensureFile(filePath, label) {
|
|
14221
14594
|
try {
|
|
@@ -14373,15 +14746,15 @@ async function validateTargetFile(targetsPath, targetNames, label) {
|
|
|
14373
14746
|
}
|
|
14374
14747
|
}
|
|
14375
14748
|
function isInsideOrSame(root, candidate) {
|
|
14376
|
-
const relative3 =
|
|
14377
|
-
return relative3 === "" || !relative3.startsWith("..") && !
|
|
14749
|
+
const relative3 = path27.relative(path27.resolve(root), path27.resolve(candidate));
|
|
14750
|
+
return relative3 === "" || !relative3.startsWith("..") && !path27.isAbsolute(relative3);
|
|
14378
14751
|
}
|
|
14379
14752
|
function forbiddenOutputRoots(sourceRunDir, selected) {
|
|
14380
14753
|
return [
|
|
14381
|
-
|
|
14754
|
+
path27.resolve(sourceRunDir),
|
|
14382
14755
|
...selected.flatMap((bundle) => [
|
|
14383
|
-
|
|
14384
|
-
|
|
14756
|
+
path27.resolve(bundle.artifactDir),
|
|
14757
|
+
path27.resolve(bundle.taskDir)
|
|
14385
14758
|
])
|
|
14386
14759
|
];
|
|
14387
14760
|
}
|
|
@@ -14400,13 +14773,13 @@ function defaultOutputDir(cwd, sourceRunDir, selected, experiment) {
|
|
|
14400
14773
|
if (!roots.some((root) => isInsideOrSame(root, candidate))) {
|
|
14401
14774
|
return candidate;
|
|
14402
14775
|
}
|
|
14403
|
-
return
|
|
14776
|
+
return path27.join(path27.dirname(path27.resolve(sourceRunDir)), `rerun-${createRunDirName()}`);
|
|
14404
14777
|
}
|
|
14405
14778
|
async function loadEnvFile2(envFile, cwd, verbose) {
|
|
14406
14779
|
if (!envFile) {
|
|
14407
14780
|
return;
|
|
14408
14781
|
}
|
|
14409
|
-
const resolved =
|
|
14782
|
+
const resolved = path27.isAbsolute(envFile) ? envFile : path27.resolve(cwd, envFile);
|
|
14410
14783
|
await ensureFile(resolved, "Environment file");
|
|
14411
14784
|
const loaded = loadDotenv({ path: resolved, override: false });
|
|
14412
14785
|
if (loaded.error) {
|
|
@@ -14438,8 +14811,8 @@ async function loadSelectedTaskBundles(options) {
|
|
|
14438
14811
|
options.sourceRunDir,
|
|
14439
14812
|
record.task_dir && `${record.task_dir}/${TASK_TARGETS_FILENAME}`
|
|
14440
14813
|
);
|
|
14441
|
-
const taskDir = resolveRelativeRunPath(options.sourceRunDir, record.task_dir) ?? (evalPath ?
|
|
14442
|
-
const artifactDir = resolveRelativeRunPath(options.sourceRunDir, record.artifact_dir) ?? (taskDir ?
|
|
14814
|
+
const taskDir = resolveRelativeRunPath(options.sourceRunDir, record.task_dir) ?? (evalPath ? path27.dirname(evalPath) : void 0);
|
|
14815
|
+
const artifactDir = resolveRelativeRunPath(options.sourceRunDir, record.artifact_dir) ?? (taskDir ? path27.dirname(taskDir) : void 0);
|
|
14443
14816
|
if (!evalPath || !targetsPath || !taskDir || !artifactDir) {
|
|
14444
14817
|
throw new Error(
|
|
14445
14818
|
`Selected result ${recordLabel} is missing task bundle paths. Re-run requires task/EVAL.yaml and task/targets.yaml.`
|
|
@@ -14469,14 +14842,14 @@ async function loadSelectedTaskBundles(options) {
|
|
|
14469
14842
|
function buildSourceMetadataByEvalFile(sourceRunDir, indexPath, selected) {
|
|
14470
14843
|
return new Map(
|
|
14471
14844
|
selected.map((bundle) => [
|
|
14472
|
-
|
|
14845
|
+
path27.resolve(bundle.evalPath),
|
|
14473
14846
|
{
|
|
14474
14847
|
rerunSource: {
|
|
14475
14848
|
mode: "rerun",
|
|
14476
|
-
sourceRunDir:
|
|
14477
|
-
sourceIndexPath:
|
|
14478
|
-
sourceArtifactDir:
|
|
14479
|
-
sourceTaskDir:
|
|
14849
|
+
sourceRunDir: path27.resolve(sourceRunDir),
|
|
14850
|
+
sourceIndexPath: path27.resolve(indexPath),
|
|
14851
|
+
sourceArtifactDir: path27.resolve(bundle.artifactDir),
|
|
14852
|
+
sourceTaskDir: path27.resolve(bundle.taskDir),
|
|
14480
14853
|
sourceTestId: bundle.testId,
|
|
14481
14854
|
sourceTarget: bundle.sourceTarget,
|
|
14482
14855
|
sourceTimestamp: bundle.record.timestamp
|
|
@@ -14547,7 +14920,7 @@ var runsRerunCommand = command({
|
|
|
14547
14920
|
handler: async (args) => {
|
|
14548
14921
|
const cwd = process.cwd();
|
|
14549
14922
|
const indexPath = resolveRunManifestPath(resolveSourcePath(cwd, args.runDir));
|
|
14550
|
-
const sourceRunDir =
|
|
14923
|
+
const sourceRunDir = path27.dirname(indexPath);
|
|
14551
14924
|
await loadEnvFile2(args.envFile, cwd, args.verbose);
|
|
14552
14925
|
const selected = await loadSelectedTaskBundles({
|
|
14553
14926
|
indexPath,
|
|
@@ -14556,10 +14929,10 @@ var runsRerunCommand = command({
|
|
|
14556
14929
|
sourceTargets: args.sourceTarget
|
|
14557
14930
|
});
|
|
14558
14931
|
const targetOverrides = args.target;
|
|
14559
|
-
const outputDir = args.output ?
|
|
14932
|
+
const outputDir = args.output ? path27.resolve(cwd, args.output) : defaultOutputDir(cwd, sourceRunDir, selected, args.experiment);
|
|
14560
14933
|
assertOutputIsSeparate(outputDir, forbiddenOutputRoots(sourceRunDir, selected));
|
|
14561
14934
|
if (args.targets) {
|
|
14562
|
-
const overrideTargetsPath =
|
|
14935
|
+
const overrideTargetsPath = path27.resolve(cwd, args.targets);
|
|
14563
14936
|
await ensureFile(overrideTargetsPath, "Target override");
|
|
14564
14937
|
const targetNames = targetOverrides.length > 0 ? targetOverrides : selected.map((bundle) => bundle.taskTarget);
|
|
14565
14938
|
await validateTargetFile(overrideTargetsPath, targetNames, "Target override");
|
|
@@ -14583,7 +14956,7 @@ var runsRerunCommand = command({
|
|
|
14583
14956
|
testFiles: selected.map((bundle) => bundle.evalPath),
|
|
14584
14957
|
rawOptions: {
|
|
14585
14958
|
target: targetOverrides,
|
|
14586
|
-
targets: args.targets ?
|
|
14959
|
+
targets: args.targets ? path27.resolve(cwd, args.targets) : void 0,
|
|
14587
14960
|
output: outputDir,
|
|
14588
14961
|
experiment: args.experiment ?? "rerun",
|
|
14589
14962
|
workers: args.workers,
|
|
@@ -14814,13 +15187,13 @@ var selfCommand = subcommands({
|
|
|
14814
15187
|
|
|
14815
15188
|
// src/commands/skills/index.ts
|
|
14816
15189
|
import { existsSync as existsSync18, readFileSync as readFileSync13, readdirSync as readdirSync5 } from "node:fs";
|
|
14817
|
-
import
|
|
15190
|
+
import path28 from "node:path";
|
|
14818
15191
|
import { fileURLToPath as fileURLToPath4 } from "node:url";
|
|
14819
15192
|
function isValidSkillsDir(dir) {
|
|
14820
15193
|
if (!existsSync18(dir)) return false;
|
|
14821
15194
|
try {
|
|
14822
15195
|
return readdirSync5(dir, { withFileTypes: true }).some(
|
|
14823
|
-
(e) => e.isDirectory() && existsSync18(
|
|
15196
|
+
(e) => e.isDirectory() && existsSync18(path28.join(dir, e.name, "SKILL.md"))
|
|
14824
15197
|
);
|
|
14825
15198
|
} catch {
|
|
14826
15199
|
return false;
|
|
@@ -14828,15 +15201,15 @@ function isValidSkillsDir(dir) {
|
|
|
14828
15201
|
}
|
|
14829
15202
|
function findSkillsDir() {
|
|
14830
15203
|
const selfFile = fileURLToPath4(import.meta.url);
|
|
14831
|
-
let dir =
|
|
15204
|
+
let dir = path28.dirname(selfFile);
|
|
14832
15205
|
for (let i = 0; i < 6; i++) {
|
|
14833
|
-
const distCandidate =
|
|
15206
|
+
const distCandidate = path28.join(dir, "dist", "skills");
|
|
14834
15207
|
if (isValidSkillsDir(distCandidate)) return distCandidate;
|
|
14835
|
-
const repoRootCandidate =
|
|
15208
|
+
const repoRootCandidate = path28.join(dir, "skills-data");
|
|
14836
15209
|
if (isValidSkillsDir(repoRootCandidate)) return repoRootCandidate;
|
|
14837
|
-
const legacyCandidate =
|
|
15210
|
+
const legacyCandidate = path28.join(dir, "skills");
|
|
14838
15211
|
if (isValidSkillsDir(legacyCandidate)) return legacyCandidate;
|
|
14839
|
-
dir =
|
|
15212
|
+
dir = path28.dirname(dir);
|
|
14840
15213
|
}
|
|
14841
15214
|
return null;
|
|
14842
15215
|
}
|
|
@@ -14855,7 +15228,7 @@ function listSkillNames(skillsDir) {
|
|
|
14855
15228
|
return readdirSync5(skillsDir, { withFileTypes: true }).filter((e) => e.isDirectory()).map((e) => e.name).sort();
|
|
14856
15229
|
}
|
|
14857
15230
|
function readSkillFile(skillDir, relPath) {
|
|
14858
|
-
const full =
|
|
15231
|
+
const full = path28.join(skillDir, relPath);
|
|
14859
15232
|
if (!existsSync18(full)) return null;
|
|
14860
15233
|
return readFileSync13(full, "utf-8");
|
|
14861
15234
|
}
|
|
@@ -14865,9 +15238,9 @@ function collectDir(dir, prefix = "") {
|
|
|
14865
15238
|
for (const entry of readdirSync5(dir, { withFileTypes: true })) {
|
|
14866
15239
|
const relPath = prefix ? `${prefix}/${entry.name}` : entry.name;
|
|
14867
15240
|
if (entry.isDirectory()) {
|
|
14868
|
-
Object.assign(result, collectDir(
|
|
15241
|
+
Object.assign(result, collectDir(path28.join(dir, entry.name), relPath));
|
|
14869
15242
|
} else {
|
|
14870
|
-
result[relPath] = readFileSync13(
|
|
15243
|
+
result[relPath] = readFileSync13(path28.join(dir, entry.name), "utf-8");
|
|
14871
15244
|
}
|
|
14872
15245
|
}
|
|
14873
15246
|
return result;
|
|
@@ -14877,14 +15250,14 @@ function listSkillSubdirs(skillDir) {
|
|
|
14877
15250
|
return readdirSync5(skillDir, { withFileTypes: true }).filter((e) => e.isDirectory() && !e.name.startsWith(".")).map((e) => e.name).sort();
|
|
14878
15251
|
}
|
|
14879
15252
|
function readSkill(skillsDir, name, full) {
|
|
14880
|
-
const skillDir =
|
|
15253
|
+
const skillDir = path28.join(skillsDir, name);
|
|
14881
15254
|
if (!existsSync18(skillDir)) return null;
|
|
14882
15255
|
const content = readSkillFile(skillDir, "SKILL.md");
|
|
14883
15256
|
if (content === null) return null;
|
|
14884
15257
|
if (!full) return { name, content };
|
|
14885
15258
|
const files = {};
|
|
14886
15259
|
for (const sub of listSkillSubdirs(skillDir)) {
|
|
14887
|
-
const subDir =
|
|
15260
|
+
const subDir = path28.join(skillDir, sub);
|
|
14888
15261
|
const collected = collectDir(subDir, sub);
|
|
14889
15262
|
Object.assign(files, collected);
|
|
14890
15263
|
}
|
|
@@ -14894,14 +15267,14 @@ function findRefFile(skillDir, refName) {
|
|
|
14894
15267
|
const candidates = refName.endsWith(".md") ? [refName] : [refName, `${refName}.md`];
|
|
14895
15268
|
for (const sub of listSkillSubdirs(skillDir)) {
|
|
14896
15269
|
for (const candidate of candidates) {
|
|
14897
|
-
const filePath =
|
|
15270
|
+
const filePath = path28.join(skillDir, sub, candidate);
|
|
14898
15271
|
if (existsSync18(filePath)) {
|
|
14899
15272
|
return { relPath: `${sub}/${candidate}`, content: readFileSync13(filePath, "utf-8") };
|
|
14900
15273
|
}
|
|
14901
15274
|
}
|
|
14902
15275
|
}
|
|
14903
15276
|
for (const candidate of candidates) {
|
|
14904
|
-
const filePath =
|
|
15277
|
+
const filePath = path28.join(skillDir, candidate);
|
|
14905
15278
|
if (existsSync18(filePath)) {
|
|
14906
15279
|
return { relPath: candidate, content: readFileSync13(filePath, "utf-8") };
|
|
14907
15280
|
}
|
|
@@ -14911,7 +15284,7 @@ function findRefFile(skillDir, refName) {
|
|
|
14911
15284
|
function listRefFiles(skillDir) {
|
|
14912
15285
|
const out = [];
|
|
14913
15286
|
for (const sub of listSkillSubdirs(skillDir)) {
|
|
14914
|
-
const subDir =
|
|
15287
|
+
const subDir = path28.join(skillDir, sub);
|
|
14915
15288
|
for (const entry of readdirSync5(subDir, { withFileTypes: true })) {
|
|
14916
15289
|
if (entry.isFile()) out.push(`${sub}/${entry.name}`);
|
|
14917
15290
|
}
|
|
@@ -14995,7 +15368,7 @@ var skillsGetCommand = command({
|
|
|
14995
15368
|
}
|
|
14996
15369
|
process.exit(1);
|
|
14997
15370
|
}
|
|
14998
|
-
const skillDir =
|
|
15371
|
+
const skillDir = path28.join(skillsDir, name);
|
|
14999
15372
|
if (!existsSync18(skillDir)) {
|
|
15000
15373
|
const msg = `skill '${name}' not found`;
|
|
15001
15374
|
if (json) {
|
|
@@ -15084,7 +15457,7 @@ var skillsPathCommand = command({
|
|
|
15084
15457
|
handler: ({ name }) => {
|
|
15085
15458
|
const skillsDir = requireSkillsDir();
|
|
15086
15459
|
if (name) {
|
|
15087
|
-
const skillDir =
|
|
15460
|
+
const skillDir = path28.join(skillsDir, name);
|
|
15088
15461
|
if (!existsSync18(skillDir)) {
|
|
15089
15462
|
console.error(`Error: skill '${name}' not found`);
|
|
15090
15463
|
process.exit(1);
|
|
@@ -15107,7 +15480,7 @@ var skillsCommand = subcommands({
|
|
|
15107
15480
|
|
|
15108
15481
|
// src/commands/transpile/index.ts
|
|
15109
15482
|
import { writeFileSync as writeFileSync7 } from "node:fs";
|
|
15110
|
-
import
|
|
15483
|
+
import path29 from "node:path";
|
|
15111
15484
|
var transpileCommand = command({
|
|
15112
15485
|
name: "transpile",
|
|
15113
15486
|
description: "Convert an EVAL.yaml file to Agent Skills evals.json format",
|
|
@@ -15131,7 +15504,7 @@ var transpileCommand = command({
|
|
|
15131
15504
|
handler: async ({ input, outDir, stdout }) => {
|
|
15132
15505
|
let result;
|
|
15133
15506
|
try {
|
|
15134
|
-
result = transpileEvalYamlFile(
|
|
15507
|
+
result = transpileEvalYamlFile(path29.resolve(input));
|
|
15135
15508
|
} catch (error) {
|
|
15136
15509
|
console.error(`Error: ${error.message}`);
|
|
15137
15510
|
process.exit(1);
|
|
@@ -15155,11 +15528,11 @@ var transpileCommand = command({
|
|
|
15155
15528
|
process.stdout.write("\n");
|
|
15156
15529
|
return;
|
|
15157
15530
|
}
|
|
15158
|
-
const outputDir = outDir ?
|
|
15531
|
+
const outputDir = outDir ? path29.resolve(outDir) : path29.dirname(path29.resolve(input));
|
|
15159
15532
|
const fileNames = getOutputFilenames(result);
|
|
15160
15533
|
for (const [skill, evalsJson] of result.files) {
|
|
15161
15534
|
const fileName = fileNames.get(skill) ?? "evals.json";
|
|
15162
|
-
const outputPath =
|
|
15535
|
+
const outputPath = path29.join(outputDir, fileName);
|
|
15163
15536
|
writeFileSync7(outputPath, `${JSON.stringify(evalsJson, null, 2)}
|
|
15164
15537
|
`);
|
|
15165
15538
|
console.log(`Transpiled to ${outputPath}`);
|
|
@@ -15168,7 +15541,7 @@ var transpileCommand = command({
|
|
|
15168
15541
|
});
|
|
15169
15542
|
|
|
15170
15543
|
// src/commands/trend/index.ts
|
|
15171
|
-
import
|
|
15544
|
+
import path30 from "node:path";
|
|
15172
15545
|
var colors2 = {
|
|
15173
15546
|
reset: "\x1B[0m",
|
|
15174
15547
|
bold: "\x1B[1m",
|
|
@@ -15218,7 +15591,7 @@ function colorizeSlope(value) {
|
|
|
15218
15591
|
}
|
|
15219
15592
|
function ensureTrendIndexPath(source, cwd) {
|
|
15220
15593
|
const resolved = resolveResultSourcePath(source, cwd);
|
|
15221
|
-
if (
|
|
15594
|
+
if (path30.basename(resolved) !== RESULT_INDEX_FILENAME) {
|
|
15222
15595
|
throw new Error(
|
|
15223
15596
|
`Unsupported result source for trend: ${source}. Use a run workspace directory or ${RESULT_INDEX_FILENAME} manifest.`
|
|
15224
15597
|
);
|
|
@@ -15238,7 +15611,7 @@ function resolveTrendSources(cwd, sources, last) {
|
|
|
15238
15611
|
if (last < 2) {
|
|
15239
15612
|
throw new Error("--last must be at least 2");
|
|
15240
15613
|
}
|
|
15241
|
-
const metas = listResultFiles(cwd).filter((meta) =>
|
|
15614
|
+
const metas = listResultFiles(cwd).filter((meta) => path30.basename(meta.path) === RESULT_INDEX_FILENAME).slice(0, last);
|
|
15242
15615
|
if (metas.length < 2) {
|
|
15243
15616
|
throw new Error(
|
|
15244
15617
|
"Trend analysis requires at least 2 canonical run workspaces in .agentv/results/runs/"
|
|
@@ -15253,10 +15626,10 @@ function getRunLabel(sourcePath, timestamp) {
|
|
|
15253
15626
|
if (timestamp) {
|
|
15254
15627
|
return timestamp;
|
|
15255
15628
|
}
|
|
15256
|
-
return
|
|
15629
|
+
return path30.basename(path30.dirname(sourcePath));
|
|
15257
15630
|
}
|
|
15258
15631
|
function getRunSortKey(sourcePath, timestamp) {
|
|
15259
|
-
return timestamp ??
|
|
15632
|
+
return timestamp ?? path30.basename(path30.dirname(sourcePath));
|
|
15260
15633
|
}
|
|
15261
15634
|
function mean2(values) {
|
|
15262
15635
|
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
@@ -15637,7 +16010,7 @@ function isTTY() {
|
|
|
15637
16010
|
// src/commands/validate/validate-files.ts
|
|
15638
16011
|
import { constants as constants2 } from "node:fs";
|
|
15639
16012
|
import { access as access2, readdir as readdir4, stat as stat2 } from "node:fs/promises";
|
|
15640
|
-
import
|
|
16013
|
+
import path31 from "node:path";
|
|
15641
16014
|
import fg3 from "fast-glob";
|
|
15642
16015
|
async function validateFiles(paths) {
|
|
15643
16016
|
const filePaths = await expandPaths(paths);
|
|
@@ -15652,7 +16025,7 @@ async function validateFiles(paths) {
|
|
|
15652
16025
|
};
|
|
15653
16026
|
}
|
|
15654
16027
|
async function validateSingleFile(filePath) {
|
|
15655
|
-
const absolutePath =
|
|
16028
|
+
const absolutePath = path31.resolve(filePath);
|
|
15656
16029
|
const fileType = await detectFileType(absolutePath);
|
|
15657
16030
|
let result;
|
|
15658
16031
|
if (fileType === "eval") {
|
|
@@ -15696,7 +16069,7 @@ async function validateSingleFile(filePath) {
|
|
|
15696
16069
|
async function expandPaths(paths) {
|
|
15697
16070
|
const expanded = /* @__PURE__ */ new Set();
|
|
15698
16071
|
for (const inputPath of paths) {
|
|
15699
|
-
const absolutePath =
|
|
16072
|
+
const absolutePath = path31.resolve(inputPath);
|
|
15700
16073
|
try {
|
|
15701
16074
|
await access2(absolutePath, constants2.F_OK);
|
|
15702
16075
|
const stats = await stat2(absolutePath);
|
|
@@ -15724,7 +16097,7 @@ async function expandPaths(paths) {
|
|
|
15724
16097
|
if (yamlMatches.length === 0) {
|
|
15725
16098
|
console.warn(`Warning: No YAML files matched pattern: ${inputPath}`);
|
|
15726
16099
|
}
|
|
15727
|
-
for (const f of yamlMatches) expanded.add(
|
|
16100
|
+
for (const f of yamlMatches) expanded.add(path31.normalize(f));
|
|
15728
16101
|
}
|
|
15729
16102
|
const sorted = Array.from(expanded);
|
|
15730
16103
|
sorted.sort();
|
|
@@ -15735,7 +16108,7 @@ async function findYamlFiles(dirPath) {
|
|
|
15735
16108
|
try {
|
|
15736
16109
|
const entries2 = await readdir4(dirPath, { withFileTypes: true });
|
|
15737
16110
|
for (const entry of entries2) {
|
|
15738
|
-
const fullPath =
|
|
16111
|
+
const fullPath = path31.join(dirPath, entry.name);
|
|
15739
16112
|
if (entry.isDirectory()) {
|
|
15740
16113
|
if (entry.name === "node_modules" || entry.name.startsWith(".")) {
|
|
15741
16114
|
continue;
|
|
@@ -15752,11 +16125,11 @@ async function findYamlFiles(dirPath) {
|
|
|
15752
16125
|
return results;
|
|
15753
16126
|
}
|
|
15754
16127
|
function isYamlFile(filePath) {
|
|
15755
|
-
const ext =
|
|
16128
|
+
const ext = path31.extname(filePath).toLowerCase();
|
|
15756
16129
|
return ext === ".yaml" || ext === ".yml";
|
|
15757
16130
|
}
|
|
15758
16131
|
function isEvalYamlFile(filePath) {
|
|
15759
|
-
const lower =
|
|
16132
|
+
const lower = path31.basename(filePath).toLowerCase();
|
|
15760
16133
|
return lower.endsWith(".eval.yaml") || lower.endsWith(".eval.yml");
|
|
15761
16134
|
}
|
|
15762
16135
|
|
|
@@ -15813,7 +16186,7 @@ var validateCommand = command({
|
|
|
15813
16186
|
// src/commands/workspace/clean.ts
|
|
15814
16187
|
import { existsSync as existsSync19 } from "node:fs";
|
|
15815
16188
|
import { readFile as readFile9, readdir as readdir5, rm as rm2 } from "node:fs/promises";
|
|
15816
|
-
import
|
|
16189
|
+
import path32 from "node:path";
|
|
15817
16190
|
async function confirm2(message) {
|
|
15818
16191
|
const readline4 = await import("node:readline");
|
|
15819
16192
|
const rl = readline4.createInterface({ input: process.stdin, output: process.stdout });
|
|
@@ -15849,8 +16222,8 @@ var cleanCommand = command({
|
|
|
15849
16222
|
const poolDirs = entries2.filter((e) => e.isDirectory());
|
|
15850
16223
|
const matchingDirs = [];
|
|
15851
16224
|
for (const dir of poolDirs) {
|
|
15852
|
-
const poolDir =
|
|
15853
|
-
const metadataPath =
|
|
16225
|
+
const poolDir = path32.join(poolRoot, dir.name);
|
|
16226
|
+
const metadataPath = path32.join(poolDir, "metadata.json");
|
|
15854
16227
|
try {
|
|
15855
16228
|
const raw = await readFile9(metadataPath, "utf-8");
|
|
15856
16229
|
const metadata = JSON.parse(raw);
|
|
@@ -15879,7 +16252,7 @@ var cleanCommand = command({
|
|
|
15879
16252
|
}
|
|
15880
16253
|
for (const dir of matchingDirs) {
|
|
15881
16254
|
await rm2(dir, { recursive: true, force: true });
|
|
15882
|
-
console.log(`Removed: ${
|
|
16255
|
+
console.log(`Removed: ${path32.basename(dir).slice(0, 12)}...`);
|
|
15883
16256
|
}
|
|
15884
16257
|
console.log("Done.");
|
|
15885
16258
|
} else {
|
|
@@ -15897,7 +16270,7 @@ var cleanCommand = command({
|
|
|
15897
16270
|
});
|
|
15898
16271
|
|
|
15899
16272
|
// src/commands/workspace/deps.ts
|
|
15900
|
-
import
|
|
16273
|
+
import path33 from "node:path";
|
|
15901
16274
|
var depsCommand = command({
|
|
15902
16275
|
name: "deps",
|
|
15903
16276
|
description: "Scan eval files and list git repo dependencies needed by workspaces",
|
|
@@ -15921,7 +16294,7 @@ var depsCommand = command({
|
|
|
15921
16294
|
const resolvedPaths = await resolveEvalPaths(evalPaths, cwd);
|
|
15922
16295
|
const result = await scanRepoDeps(resolvedPaths);
|
|
15923
16296
|
for (const err2 of result.errors) {
|
|
15924
|
-
console.error(`error: ${
|
|
16297
|
+
console.error(`error: ${path33.relative(cwd, err2.file)}: ${err2.message}`);
|
|
15925
16298
|
}
|
|
15926
16299
|
if (result.errors.length > 0) {
|
|
15927
16300
|
process.exit(1);
|
|
@@ -15932,7 +16305,7 @@ var depsCommand = command({
|
|
|
15932
16305
|
...r.ref !== void 0 && { ref: r.ref },
|
|
15933
16306
|
...r.sparse !== void 0 && { sparse: r.sparse },
|
|
15934
16307
|
...r.ancestor !== void 0 && { ancestor: r.ancestor },
|
|
15935
|
-
...usedBy && { used_by: r.usedBy.map((p) =>
|
|
16308
|
+
...usedBy && { used_by: r.usedBy.map((p) => path33.relative(cwd, p)) }
|
|
15936
16309
|
}))
|
|
15937
16310
|
};
|
|
15938
16311
|
console.log(JSON.stringify(output, null, 2));
|
|
@@ -15942,13 +16315,13 @@ var depsCommand = command({
|
|
|
15942
16315
|
// src/commands/workspace/list.ts
|
|
15943
16316
|
import { existsSync as existsSync20 } from "node:fs";
|
|
15944
16317
|
import { readFile as readFile10, readdir as readdir6, stat as stat3 } from "node:fs/promises";
|
|
15945
|
-
import
|
|
16318
|
+
import path34 from "node:path";
|
|
15946
16319
|
async function getDirectorySize(dirPath) {
|
|
15947
16320
|
let totalSize = 0;
|
|
15948
16321
|
try {
|
|
15949
16322
|
const entries2 = await readdir6(dirPath, { withFileTypes: true });
|
|
15950
16323
|
for (const entry of entries2) {
|
|
15951
|
-
const fullPath =
|
|
16324
|
+
const fullPath = path34.join(dirPath, entry.name);
|
|
15952
16325
|
if (entry.isDirectory()) {
|
|
15953
16326
|
totalSize += await getDirectorySize(fullPath);
|
|
15954
16327
|
} else {
|
|
@@ -15983,11 +16356,11 @@ var listCommand = command({
|
|
|
15983
16356
|
return;
|
|
15984
16357
|
}
|
|
15985
16358
|
for (const dir of poolDirs) {
|
|
15986
|
-
const poolDir =
|
|
16359
|
+
const poolDir = path34.join(poolRoot, dir.name);
|
|
15987
16360
|
const fingerprint = dir.name;
|
|
15988
16361
|
const poolEntries = await readdir6(poolDir, { withFileTypes: true });
|
|
15989
16362
|
const slots = poolEntries.filter((e) => e.isDirectory() && e.name.startsWith("slot-"));
|
|
15990
|
-
const metadataPath =
|
|
16363
|
+
const metadataPath = path34.join(poolDir, "metadata.json");
|
|
15991
16364
|
let metadata = null;
|
|
15992
16365
|
try {
|
|
15993
16366
|
const raw = await readFile10(metadataPath, "utf-8");
|
|
@@ -16034,8 +16407,8 @@ var CHECK_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
|
16034
16407
|
var CONFIG_DIR = getAgentvConfigDir();
|
|
16035
16408
|
var CACHE_FILE = "version-check.json";
|
|
16036
16409
|
var NPM_REGISTRY_BASE2 = "https://registry.npmjs.org/agentv/";
|
|
16037
|
-
async function getCachedUpdateInfo(
|
|
16038
|
-
const filePath =
|
|
16410
|
+
async function getCachedUpdateInfo(path36) {
|
|
16411
|
+
const filePath = path36 ?? join6(CONFIG_DIR, CACHE_FILE);
|
|
16039
16412
|
try {
|
|
16040
16413
|
const raw = await readFile11(filePath, "utf-8");
|
|
16041
16414
|
const data = JSON.parse(raw);
|
|
@@ -16144,7 +16517,7 @@ var app = subcommands({
|
|
|
16144
16517
|
workspace: workspaceCommand
|
|
16145
16518
|
}
|
|
16146
16519
|
});
|
|
16147
|
-
var EVAL_SUBCOMMANDS = /* @__PURE__ */ new Set(["run", "assert", "aggregate"]);
|
|
16520
|
+
var EVAL_SUBCOMMANDS = /* @__PURE__ */ new Set(["run", "assert", "aggregate", "bundle"]);
|
|
16148
16521
|
var TOP_LEVEL_COMMANDS = /* @__PURE__ */ new Set([
|
|
16149
16522
|
"import",
|
|
16150
16523
|
"inspect",
|
|
@@ -16216,7 +16589,7 @@ ${updateNotice}
|
|
|
16216
16589
|
}
|
|
16217
16590
|
const cwd = process.cwd();
|
|
16218
16591
|
const repoRoot = await findRepoRoot(cwd);
|
|
16219
|
-
const sessionConfig = await loadConfig(
|
|
16592
|
+
const sessionConfig = await loadConfig(path35.join(cwd, "_"), repoRoot);
|
|
16220
16593
|
const beforeSessionCommand = sessionConfig?.hooks?.before_session;
|
|
16221
16594
|
if (beforeSessionCommand) {
|
|
16222
16595
|
runBeforeSessionHook(beforeSessionCommand);
|
|
@@ -16230,4 +16603,4 @@ export {
|
|
|
16230
16603
|
preprocessArgv,
|
|
16231
16604
|
runCli
|
|
16232
16605
|
};
|
|
16233
|
-
//# sourceMappingURL=chunk-
|
|
16606
|
+
//# sourceMappingURL=chunk-AHQWZGJE.js.map
|