agentv 5.1.0-next.1 → 5.2.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -12
- package/dist/{artifact-writer-IORG7CW5.js → artifact-writer-SN6UNL5G.js} +5 -5
- package/dist/{chunk-ZGPCX334.js → chunk-3S6A2RKR.js} +2803 -959
- package/dist/chunk-3S6A2RKR.js.map +1 -0
- package/dist/{chunk-GIRUNQGP.js → chunk-3X357HS4.js} +205 -153
- package/dist/chunk-3X357HS4.js.map +1 -0
- package/dist/{chunk-CBBBRTQ3.js → chunk-46K2OET3.js} +64 -3
- package/dist/chunk-46K2OET3.js.map +1 -0
- package/dist/{chunk-GJWPQXHN.js → chunk-ALVZQUZP.js} +104 -188
- package/dist/chunk-ALVZQUZP.js.map +1 -0
- package/dist/{chunk-AM2IPHKS.js → chunk-T4AD3H3Y.js} +236 -154
- package/dist/chunk-T4AD3H3Y.js.map +1 -0
- package/dist/cli.js +6 -6
- package/dist/dashboard/assets/{index-BSHoWrdr.js → index-DNgf3qJ2.js} +1 -1
- package/dist/dashboard/assets/{index-zvUKAkKn.css → index-D_bokML8.css} +1 -1
- package/dist/dashboard/assets/index-r_jSJmlw.js +121 -0
- package/dist/dashboard/index.html +2 -2
- package/dist/{dist-2D2GBHZV.js → dist-DEPJOMCA.js} +6 -8
- package/dist/index.js +6 -6
- package/dist/{interactive-V4SEHLNY.js → interactive-36ZNMQB2.js} +6 -6
- package/dist/skills/agentv-bench/SKILL.md +16 -12
- package/dist/skills/agentv-bench/references/eval-yaml-spec.md +15 -10
- package/dist/skills/agentv-bench/references/migrating-from-skill-creator.md +1 -1
- package/dist/skills/agentv-eval-migrations/references/breaking-changes.md +26 -27
- package/dist/skills/agentv-eval-writer/SKILL.md +86 -71
- package/dist/skills/agentv-eval-writer/references/eval.schema.json +772 -1367
- package/dist/{ts-eval-loader-SSPJVEHD-AL2A3V5Z.js → ts-eval-loader-OMG2JBHP-OP5RR7A3.js} +3 -2
- package/package.json +8 -2
- package/dist/chunk-AM2IPHKS.js.map +0 -1
- package/dist/chunk-CBBBRTQ3.js.map +0 -1
- package/dist/chunk-GIRUNQGP.js.map +0 -1
- package/dist/chunk-GJWPQXHN.js.map +0 -1
- package/dist/chunk-ZGPCX334.js.map +0 -1
- package/dist/dashboard/assets/index-BaLLxmVO.js +0 -121
- /package/dist/{artifact-writer-IORG7CW5.js.map → artifact-writer-SN6UNL5G.js.map} +0 -0
- /package/dist/{dist-2D2GBHZV.js.map → dist-DEPJOMCA.js.map} +0 -0
- /package/dist/{interactive-V4SEHLNY.js.map → interactive-36ZNMQB2.js.map} +0 -0
- /package/dist/{ts-eval-loader-SSPJVEHD-AL2A3V5Z.js.map → ts-eval-loader-OMG2JBHP-OP5RR7A3.js.map} +0 -0
|
@@ -61,11 +61,11 @@ import {
|
|
|
61
61
|
validateFileReferences,
|
|
62
62
|
validateTargetsFile,
|
|
63
63
|
validateWorkspacePaths
|
|
64
|
-
} from "./chunk-
|
|
64
|
+
} from "./chunk-T4AD3H3Y.js";
|
|
65
65
|
import {
|
|
66
66
|
materializeEvalBundle,
|
|
67
67
|
writeArtifactsFromResults as writeArtifactsFromResults2
|
|
68
|
-
} from "./chunk-
|
|
68
|
+
} from "./chunk-46K2OET3.js";
|
|
69
69
|
import {
|
|
70
70
|
discoverClaudeSessions,
|
|
71
71
|
discoverCodexSessions,
|
|
@@ -77,11 +77,10 @@ import {
|
|
|
77
77
|
prepareEvalWorkspace,
|
|
78
78
|
readGitResultArtifact,
|
|
79
79
|
runBeforeSessionHook,
|
|
80
|
-
scanRepoDeps,
|
|
81
80
|
syncProjects,
|
|
82
81
|
transpileEvalYamlFile,
|
|
83
82
|
trimBaselineResult
|
|
84
|
-
} from "./chunk-
|
|
83
|
+
} from "./chunk-ALVZQUZP.js";
|
|
85
84
|
import {
|
|
86
85
|
AGENTV_RESULTS_ARTIFACTS_REF,
|
|
87
86
|
DEFAULT_CATEGORY,
|
|
@@ -140,7 +139,7 @@ import {
|
|
|
140
139
|
traceEnvelopeToTranscriptMessages,
|
|
141
140
|
traceFromTranscriptJsonLines,
|
|
142
141
|
writeArtifactsFromResults
|
|
143
|
-
} from "./chunk-
|
|
142
|
+
} from "./chunk-3S6A2RKR.js";
|
|
144
143
|
import {
|
|
145
144
|
__commonJS,
|
|
146
145
|
__require,
|
|
@@ -901,7 +900,7 @@ var require_src = __commonJS({
|
|
|
901
900
|
});
|
|
902
901
|
|
|
903
902
|
// src/index.ts
|
|
904
|
-
import
|
|
903
|
+
import path32 from "node:path";
|
|
905
904
|
|
|
906
905
|
// ../../node_modules/.bun/chalk@5.6.2/node_modules/chalk/source/vendor/ansi-styles/index.js
|
|
907
906
|
var ANSI_BACKGROUND_OFFSET = 10;
|
|
@@ -3975,7 +3974,8 @@ export default defineAssertion(({ output }) => {
|
|
|
3975
3974
|
const pass = text.length > 0;
|
|
3976
3975
|
return {
|
|
3977
3976
|
pass,
|
|
3978
|
-
|
|
3977
|
+
score: pass ? 1 : 0,
|
|
3978
|
+
reason: pass ? 'Output has content' : 'Output is empty',
|
|
3979
3979
|
};
|
|
3980
3980
|
});
|
|
3981
3981
|
`,
|
|
@@ -3989,7 +3989,14 @@ export default defineAssertion(({ output }) => {
|
|
|
3989
3989
|
return {
|
|
3990
3990
|
pass: score >= 0.5,
|
|
3991
3991
|
score,
|
|
3992
|
-
|
|
3992
|
+
reason: score === 1.0 ? 'Output has content' : 'Output is empty',
|
|
3993
|
+
checks: [
|
|
3994
|
+
{
|
|
3995
|
+
text: 'Output has content',
|
|
3996
|
+
pass: score === 1.0,
|
|
3997
|
+
reason: score === 1.0 ? 'Output is non-empty' : 'Output is empty',
|
|
3998
|
+
},
|
|
3999
|
+
],
|
|
3993
4000
|
};
|
|
3994
4001
|
});
|
|
3995
4002
|
`
|
|
@@ -5042,15 +5049,19 @@ var ScriptGraderInputSchema = external_exports.object({
|
|
|
5042
5049
|
workspacePath: external_exports.string().nullable().optional(),
|
|
5043
5050
|
config: external_exports.record(external_exports.unknown()).nullable().optional()
|
|
5044
5051
|
});
|
|
5052
|
+
var ScriptGraderCheckSchema = external_exports.object({
|
|
5053
|
+
id: external_exports.string().optional(),
|
|
5054
|
+
text: external_exports.string(),
|
|
5055
|
+
pass: external_exports.boolean(),
|
|
5056
|
+
score: external_exports.number().min(0).max(1).optional(),
|
|
5057
|
+
reason: external_exports.string(),
|
|
5058
|
+
evidence: external_exports.string().optional()
|
|
5059
|
+
});
|
|
5045
5060
|
var ScriptGraderResultSchema = external_exports.object({
|
|
5061
|
+
pass: external_exports.boolean(),
|
|
5046
5062
|
score: external_exports.number().min(0).max(1),
|
|
5047
|
-
|
|
5048
|
-
|
|
5049
|
-
text: external_exports.string(),
|
|
5050
|
-
passed: external_exports.boolean(),
|
|
5051
|
-
evidence: external_exports.string().optional()
|
|
5052
|
-
})
|
|
5053
|
-
).optional().default([]),
|
|
5063
|
+
reason: external_exports.string(),
|
|
5064
|
+
checks: external_exports.array(ScriptGraderCheckSchema).optional().default([]),
|
|
5054
5065
|
/** Optional structured details for domain-specific metrics (e.g., TP/TN/FP/FN counts, alignments). */
|
|
5055
5066
|
details: external_exports.record(external_exports.unknown()).optional()
|
|
5056
5067
|
});
|
|
@@ -5192,14 +5203,26 @@ async function runScriptGrader(handler) {
|
|
|
5192
5203
|
const rawResult = await handler(input);
|
|
5193
5204
|
const result = ScriptGraderResultSchema.parse({
|
|
5194
5205
|
...rawResult,
|
|
5195
|
-
score: clampScore(rawResult.score)
|
|
5206
|
+
score: clampScore(rawResult.score),
|
|
5207
|
+
checks: rawResult.checks?.map((check2) => ({
|
|
5208
|
+
...check2,
|
|
5209
|
+
...check2.score !== void 0 ? { score: clampScore(check2.score) } : {}
|
|
5210
|
+
}))
|
|
5196
5211
|
});
|
|
5197
5212
|
console.log(JSON.stringify(result, null, 2));
|
|
5198
5213
|
} catch (error) {
|
|
5199
5214
|
const errorMessage = formatError(error);
|
|
5200
5215
|
const errorResult = {
|
|
5216
|
+
pass: false,
|
|
5201
5217
|
score: 0,
|
|
5202
|
-
|
|
5218
|
+
reason: `Evaluation failed: ${errorMessage}`,
|
|
5219
|
+
checks: [
|
|
5220
|
+
{
|
|
5221
|
+
text: "Script grader execution",
|
|
5222
|
+
pass: false,
|
|
5223
|
+
reason: errorMessage
|
|
5224
|
+
}
|
|
5225
|
+
]
|
|
5203
5226
|
};
|
|
5204
5227
|
console.log(JSON.stringify(errorResult, null, 2));
|
|
5205
5228
|
process.exit(1);
|
|
@@ -5267,27 +5290,36 @@ function parseJsonObjectFromText(text) {
|
|
|
5267
5290
|
function isVitestJsonReport(value) {
|
|
5268
5291
|
return typeof value === "object" && value !== null && Array.isArray(value.testResults);
|
|
5269
5292
|
}
|
|
5270
|
-
function assertionText(file,
|
|
5271
|
-
const title =
|
|
5293
|
+
function assertionText(file, assertion) {
|
|
5294
|
+
const title = assertion.fullName ?? [...assertion.ancestorTitles ?? [], assertion.title].filter(Boolean).join(" ");
|
|
5272
5295
|
return title || file.name || "Vitest assertion";
|
|
5273
5296
|
}
|
|
5274
5297
|
function vitestReportToScriptGraderResult(report, options = {}) {
|
|
5275
|
-
const
|
|
5298
|
+
const checks = (report.testResults ?? []).flatMap(
|
|
5276
5299
|
(file) => (file.assertionResults ?? []).map((item) => {
|
|
5277
|
-
const
|
|
5300
|
+
const pass2 = item.status === "passed";
|
|
5278
5301
|
const evidence = item.failureMessages && item.failureMessages.length > 0 ? truncate(item.failureMessages.join("\n\n")) : void 0;
|
|
5279
5302
|
return {
|
|
5280
5303
|
text: assertionText(file, item),
|
|
5281
|
-
|
|
5304
|
+
pass: pass2,
|
|
5305
|
+
reason: pass2 ? "Vitest test passed." : "Vitest test failed.",
|
|
5282
5306
|
...evidence !== void 0 ? { evidence } : {}
|
|
5283
5307
|
};
|
|
5284
5308
|
})
|
|
5285
5309
|
);
|
|
5286
|
-
if (
|
|
5287
|
-
const
|
|
5310
|
+
if (checks.length === 0) {
|
|
5311
|
+
const pass2 = options.passWithNoTests === true;
|
|
5288
5312
|
return ScriptGraderResultSchema.parse({
|
|
5289
|
-
|
|
5290
|
-
|
|
5313
|
+
pass: pass2,
|
|
5314
|
+
score: pass2 ? 1 : 0,
|
|
5315
|
+
reason: pass2 ? "Vitest reported no tests; configured to pass." : "Vitest reported no tests.",
|
|
5316
|
+
checks: [
|
|
5317
|
+
{
|
|
5318
|
+
text: "Vitest reported no tests",
|
|
5319
|
+
pass: pass2,
|
|
5320
|
+
reason: pass2 ? "passWithNoTests is enabled." : "No Vitest tests were discovered."
|
|
5321
|
+
}
|
|
5322
|
+
],
|
|
5291
5323
|
details: {
|
|
5292
5324
|
vitest_success: report.success ?? false,
|
|
5293
5325
|
num_total_tests: report.numTotalTests ?? 0,
|
|
@@ -5298,15 +5330,18 @@ function vitestReportToScriptGraderResult(report, options = {}) {
|
|
|
5298
5330
|
}
|
|
5299
5331
|
});
|
|
5300
5332
|
}
|
|
5301
|
-
const passedCount =
|
|
5333
|
+
const passedCount = checks.filter((item) => item.pass).length;
|
|
5334
|
+
const pass = passedCount === checks.length;
|
|
5302
5335
|
return ScriptGraderResultSchema.parse({
|
|
5303
|
-
|
|
5304
|
-
|
|
5336
|
+
pass,
|
|
5337
|
+
score: passedCount / checks.length,
|
|
5338
|
+
reason: `${passedCount}/${checks.length} Vitest tests passed.`,
|
|
5339
|
+
checks,
|
|
5305
5340
|
details: {
|
|
5306
|
-
vitest_success: report.success ??
|
|
5307
|
-
num_total_tests: report.numTotalTests ??
|
|
5341
|
+
vitest_success: report.success ?? pass,
|
|
5342
|
+
num_total_tests: report.numTotalTests ?? checks.length,
|
|
5308
5343
|
num_passed_tests: report.numPassedTests ?? passedCount,
|
|
5309
|
-
num_failed_tests: report.numFailedTests ??
|
|
5344
|
+
num_failed_tests: report.numFailedTests ?? checks.length - passedCount,
|
|
5310
5345
|
num_pending_tests: report.numPendingTests ?? 0,
|
|
5311
5346
|
num_todo_tests: report.numTodoTests ?? 0
|
|
5312
5347
|
}
|
|
@@ -5384,12 +5419,14 @@ async function runVitestWorkspaceGrader(options, input) {
|
|
|
5384
5419
|
const workspacePath = workspacePathFrom2(input);
|
|
5385
5420
|
if (!workspacePath) {
|
|
5386
5421
|
return {
|
|
5422
|
+
pass: false,
|
|
5387
5423
|
score: 0,
|
|
5388
|
-
|
|
5424
|
+
reason: "Vitest workspace verifier requires workspace_path.",
|
|
5425
|
+
checks: [
|
|
5389
5426
|
{
|
|
5390
5427
|
text: "Vitest workspace verifier requires workspace_path",
|
|
5391
|
-
|
|
5392
|
-
|
|
5428
|
+
pass: false,
|
|
5429
|
+
reason: "Configure an environment recipe in the eval YAML so AgentV can pass workspace_path."
|
|
5393
5430
|
}
|
|
5394
5431
|
]
|
|
5395
5432
|
};
|
|
@@ -5423,13 +5460,16 @@ async function runVitestWorkspaceGrader(options, input) {
|
|
|
5423
5460
|
const report = await readVitestReport(result, outputFile);
|
|
5424
5461
|
return vitestReportToScriptGraderResult(report, options);
|
|
5425
5462
|
} catch (error) {
|
|
5463
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
5426
5464
|
return {
|
|
5465
|
+
pass: false,
|
|
5427
5466
|
score: 0,
|
|
5428
|
-
|
|
5467
|
+
reason: "Vitest workspace verifier failed to run.",
|
|
5468
|
+
checks: [
|
|
5429
5469
|
{
|
|
5430
5470
|
text: "Vitest workspace verifier failed to run",
|
|
5431
|
-
|
|
5432
|
-
|
|
5471
|
+
pass: false,
|
|
5472
|
+
reason
|
|
5433
5473
|
}
|
|
5434
5474
|
]
|
|
5435
5475
|
};
|
|
@@ -7970,13 +8010,31 @@ var inspectCommand = subcommands({
|
|
|
7970
8010
|
import { existsSync as existsSync6 } from "node:fs";
|
|
7971
8011
|
import { mkdir as mkdir6, readFile as readFile4, readdir, writeFile as writeFile6 } from "node:fs/promises";
|
|
7972
8012
|
import { join } from "node:path";
|
|
7973
|
-
function
|
|
8013
|
+
function toComponentResult(assertion, evaluator) {
|
|
7974
8014
|
return {
|
|
7975
|
-
|
|
7976
|
-
passed: assertion.passed,
|
|
7977
|
-
evidence: assertion.evidence ?? "",
|
|
8015
|
+
pass: assertion.passed,
|
|
7978
8016
|
score: assertion.passed ? 1 : 0,
|
|
7979
|
-
|
|
8017
|
+
reason: assertion.evidence ?? assertion.text,
|
|
8018
|
+
assertion: {
|
|
8019
|
+
...evaluator ? { name: evaluator.name, type: evaluator.type } : {},
|
|
8020
|
+
value: assertion.text
|
|
8021
|
+
}
|
|
8022
|
+
};
|
|
8023
|
+
}
|
|
8024
|
+
function evaluatorComponent(evaluator) {
|
|
8025
|
+
const pass = evaluator.score >= DEFAULT_THRESHOLD;
|
|
8026
|
+
return {
|
|
8027
|
+
pass,
|
|
8028
|
+
score: evaluator.score,
|
|
8029
|
+
reason: pass ? "Grader passed." : "Grader failed.",
|
|
8030
|
+
assertion: {
|
|
8031
|
+
name: evaluator.name,
|
|
8032
|
+
type: evaluator.type,
|
|
8033
|
+
weight: evaluator.weight
|
|
8034
|
+
},
|
|
8035
|
+
component_results: evaluator.assertions.map(
|
|
8036
|
+
(assertion) => toComponentResult(assertion, evaluator)
|
|
8037
|
+
)
|
|
7980
8038
|
};
|
|
7981
8039
|
}
|
|
7982
8040
|
var evalBenchCommand = command({
|
|
@@ -8055,20 +8113,18 @@ var evalBenchCommand = command({
|
|
|
8055
8113
|
const failed = allAssertions.filter((a) => !a.passed).length;
|
|
8056
8114
|
const passRate = allAssertions.length > 0 ? Math.round(passed / allAssertions.length * 1e3) / 1e3 : weightedScore >= 0.5 ? 1 : 0;
|
|
8057
8115
|
allPassRates.push(passRate);
|
|
8116
|
+
const pass = weightedScore >= DEFAULT_THRESHOLD;
|
|
8058
8117
|
const grading = {
|
|
8118
|
+
pass,
|
|
8059
8119
|
score: Math.round(weightedScore * 1e3) / 1e3,
|
|
8060
|
-
|
|
8061
|
-
|
|
8062
|
-
|
|
8063
|
-
|
|
8064
|
-
|
|
8065
|
-
|
|
8066
|
-
|
|
8067
|
-
|
|
8068
|
-
reasoning: "",
|
|
8069
|
-
weight: e.weight,
|
|
8070
|
-
assertion_results: e.assertions.map(toAssertionResult)
|
|
8071
|
-
}))
|
|
8120
|
+
reason: pass ? "All grading components passed." : "One or more grading components failed.",
|
|
8121
|
+
component_results: evaluators.length > 0 ? evaluators.map(evaluatorComponent) : allAssertions.map((assertion) => toComponentResult(assertion)),
|
|
8122
|
+
metadata: {
|
|
8123
|
+
pass_count: passed,
|
|
8124
|
+
fail_count: failed,
|
|
8125
|
+
sample_count: allAssertions.length,
|
|
8126
|
+
pass_rate: passRate
|
|
8127
|
+
}
|
|
8072
8128
|
};
|
|
8073
8129
|
await writeFile6(
|
|
8074
8130
|
join(testDir, "grading.json"),
|
|
@@ -8079,14 +8135,10 @@ var evalBenchCommand = command({
|
|
|
8079
8135
|
const scores = evaluators.map((e) => ({
|
|
8080
8136
|
name: e.name,
|
|
8081
8137
|
type: e.type,
|
|
8138
|
+
pass: e.score >= DEFAULT_THRESHOLD,
|
|
8082
8139
|
score: e.score,
|
|
8083
8140
|
weight: e.weight,
|
|
8084
|
-
|
|
8085
|
-
assertions: e.assertions.map((a) => ({
|
|
8086
|
-
text: a.text,
|
|
8087
|
-
passed: a.passed,
|
|
8088
|
-
evidence: a.evidence ?? ""
|
|
8089
|
-
}))
|
|
8141
|
+
reason: e.score >= DEFAULT_THRESHOLD ? "Grader passed." : "Grader failed."
|
|
8090
8142
|
}));
|
|
8091
8143
|
let executionStatus = "ok";
|
|
8092
8144
|
const metricsPath = join(testDir, "metrics.json");
|
|
@@ -12783,9 +12835,10 @@ function formatSummary(results, grading) {
|
|
|
12783
12835
|
let failed;
|
|
12784
12836
|
let passRate;
|
|
12785
12837
|
if (grading) {
|
|
12786
|
-
|
|
12787
|
-
|
|
12788
|
-
|
|
12838
|
+
const metadata = grading.metadata ?? {};
|
|
12839
|
+
passed = typeof metadata.pass_count === "number" ? metadata.pass_count : grading.pass ? total : 0;
|
|
12840
|
+
failed = typeof metadata.sample_count === "number" && typeof metadata.pass_count === "number" ? metadata.sample_count - metadata.pass_count : total - passed;
|
|
12841
|
+
passRate = typeof metadata.pass_rate === "number" ? metadata.pass_rate : total > 0 ? Math.round(passed / total * 1e3) / 1e3 : 0;
|
|
12789
12842
|
} else {
|
|
12790
12843
|
passed = results.filter((r) => r.score >= 1).length;
|
|
12791
12844
|
failed = total - passed;
|
|
@@ -12844,6 +12897,56 @@ var resultsSummaryCommand = command({
|
|
|
12844
12897
|
// src/commands/results/validate.ts
|
|
12845
12898
|
import { existsSync as existsSync14, readFileSync as readFileSync12, statSync as statSync5 } from "node:fs";
|
|
12846
12899
|
import path24 from "node:path";
|
|
12900
|
+
var LEGACY_GRADING_FIELDS = [
|
|
12901
|
+
"assertion_results",
|
|
12902
|
+
"assertions",
|
|
12903
|
+
"passed",
|
|
12904
|
+
"evidence",
|
|
12905
|
+
"verdict",
|
|
12906
|
+
"graders",
|
|
12907
|
+
"checks"
|
|
12908
|
+
];
|
|
12909
|
+
function isRecord2(value) {
|
|
12910
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
12911
|
+
}
|
|
12912
|
+
function validateGradingNode(value, pathLabel) {
|
|
12913
|
+
if (!isRecord2(value)) {
|
|
12914
|
+
return [`${pathLabel} must be an object`];
|
|
12915
|
+
}
|
|
12916
|
+
const errors = [];
|
|
12917
|
+
if (typeof value.pass !== "boolean" || typeof value.score !== "number" || typeof value.reason !== "string") {
|
|
12918
|
+
errors.push(`${pathLabel} must include pass, score, and reason`);
|
|
12919
|
+
}
|
|
12920
|
+
if (value.component_results !== void 0) {
|
|
12921
|
+
if (!Array.isArray(value.component_results)) {
|
|
12922
|
+
errors.push(`${pathLabel}.component_results must be an array when present`);
|
|
12923
|
+
} else {
|
|
12924
|
+
value.component_results.forEach((component, index) => {
|
|
12925
|
+
errors.push(...validateGradingNode(component, `${pathLabel}.component_results[${index}]`));
|
|
12926
|
+
});
|
|
12927
|
+
}
|
|
12928
|
+
}
|
|
12929
|
+
return errors;
|
|
12930
|
+
}
|
|
12931
|
+
function validateNoLegacyGradingFields(value, pathLabel) {
|
|
12932
|
+
if (Array.isArray(value)) {
|
|
12933
|
+
return value.flatMap(
|
|
12934
|
+
(entry, index) => validateNoLegacyGradingFields(entry, `${pathLabel}[${index}]`)
|
|
12935
|
+
);
|
|
12936
|
+
}
|
|
12937
|
+
if (!isRecord2(value)) {
|
|
12938
|
+
return [];
|
|
12939
|
+
}
|
|
12940
|
+
const errors = [];
|
|
12941
|
+
const legacyFields = LEGACY_GRADING_FIELDS.filter((field) => Object.hasOwn(value, field));
|
|
12942
|
+
if (legacyFields.length > 0) {
|
|
12943
|
+
errors.push(`${pathLabel} uses legacy field(s): ${legacyFields.join(", ")}`);
|
|
12944
|
+
}
|
|
12945
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
12946
|
+
errors.push(...validateNoLegacyGradingFields(entry, `${pathLabel}.${key}`));
|
|
12947
|
+
}
|
|
12948
|
+
return errors;
|
|
12949
|
+
}
|
|
12847
12950
|
function checkDirectoryNaming(runDir) {
|
|
12848
12951
|
const dirName = path24.basename(runDir);
|
|
12849
12952
|
const pathSegments = path24.normalize(runDir).split(path24.sep).filter(Boolean);
|
|
@@ -12962,7 +13065,8 @@ function checkIndexJsonl(runDir) {
|
|
|
12962
13065
|
if (typeof s.name !== "string") missing.push("name");
|
|
12963
13066
|
if (typeof s.type !== "string") missing.push("type");
|
|
12964
13067
|
if (typeof s.score !== "number") missing.push("score");
|
|
12965
|
-
if (typeof s.
|
|
13068
|
+
if (typeof s.pass !== "boolean") missing.push("pass");
|
|
13069
|
+
if (typeof s.reason !== "string") missing.push("reason");
|
|
12966
13070
|
if (missing.length > 0) {
|
|
12967
13071
|
diagnostics.push({
|
|
12968
13072
|
severity: "warning",
|
|
@@ -13035,22 +13139,13 @@ function checkArtifactFiles(runDir, entries2) {
|
|
|
13035
13139
|
} else {
|
|
13036
13140
|
try {
|
|
13037
13141
|
const grading = JSON.parse(readFileSync12(gradingPath, "utf8"));
|
|
13038
|
-
|
|
13039
|
-
|
|
13040
|
-
|
|
13041
|
-
|
|
13042
|
-
message: `${testId}: grading.json uses legacy 'assertions' array; rewrite the run to emit 'assertion_results'`
|
|
13043
|
-
});
|
|
13044
|
-
} else {
|
|
13142
|
+
for (const error of [
|
|
13143
|
+
...validateGradingNode(grading, "grading.json"),
|
|
13144
|
+
...validateNoLegacyGradingFields(grading, "grading.json")
|
|
13145
|
+
]) {
|
|
13045
13146
|
diagnostics.push({
|
|
13046
13147
|
severity: "error",
|
|
13047
|
-
message: `${testId}:
|
|
13048
|
-
});
|
|
13049
|
-
}
|
|
13050
|
-
if (!grading.summary) {
|
|
13051
|
-
diagnostics.push({
|
|
13052
|
-
severity: "warning",
|
|
13053
|
-
message: `${testId}: grading.json missing 'summary' object`
|
|
13148
|
+
message: `${testId}: ${error}`
|
|
13054
13149
|
});
|
|
13055
13150
|
}
|
|
13056
13151
|
} catch {
|
|
@@ -13964,7 +14059,7 @@ function inferRawContentType(filePath) {
|
|
|
13964
14059
|
function contentDispositionFilename(filePath) {
|
|
13965
14060
|
return path27.basename(filePath).replace(/["\\\r\n]/g, "_");
|
|
13966
14061
|
}
|
|
13967
|
-
function
|
|
14062
|
+
function isRecord3(value) {
|
|
13968
14063
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
13969
14064
|
}
|
|
13970
14065
|
function nonEmptyString(value) {
|
|
@@ -13972,12 +14067,12 @@ function nonEmptyString(value) {
|
|
|
13972
14067
|
}
|
|
13973
14068
|
function artifactPointerPath(pointer) {
|
|
13974
14069
|
if (typeof pointer === "string") return nonEmptyString(pointer);
|
|
13975
|
-
if (!
|
|
14070
|
+
if (!isRecord3(pointer)) return void 0;
|
|
13976
14071
|
return nonEmptyString(pointer.path) ?? nonEmptyString(pointer.artifact_path) ?? nonEmptyString(pointer.relative_path);
|
|
13977
14072
|
}
|
|
13978
14073
|
function artifactPointerDescription(pointer) {
|
|
13979
14074
|
if (typeof pointer === "string") return pointer;
|
|
13980
|
-
if (!
|
|
14075
|
+
if (!isRecord3(pointer)) return void 0;
|
|
13981
14076
|
const ref = nonEmptyString(pointer.ref);
|
|
13982
14077
|
const storage = nonEmptyString(pointer.storage);
|
|
13983
14078
|
const uri = nonEmptyString(pointer.uri) ?? nonEmptyString(pointer.href);
|
|
@@ -13991,19 +14086,19 @@ function artifactPointerDescription(pointer) {
|
|
|
13991
14086
|
return parts.length > 0 ? parts.join(", ") : void 0;
|
|
13992
14087
|
}
|
|
13993
14088
|
function artifactPointerRef(pointer) {
|
|
13994
|
-
return
|
|
14089
|
+
return isRecord3(pointer) ? nonEmptyString(pointer.ref) : void 0;
|
|
13995
14090
|
}
|
|
13996
14091
|
function artifactPointerKey(pointer) {
|
|
13997
|
-
return
|
|
14092
|
+
return isRecord3(pointer) ? nonEmptyString(pointer.key) : void 0;
|
|
13998
14093
|
}
|
|
13999
14094
|
function artifactPointerSha256(pointer) {
|
|
14000
|
-
return
|
|
14095
|
+
return isRecord3(pointer) ? nonEmptyString(pointer.sha256) : void 0;
|
|
14001
14096
|
}
|
|
14002
14097
|
function artifactPointerMediaType(pointer) {
|
|
14003
|
-
return
|
|
14098
|
+
return isRecord3(pointer) ? nonEmptyString(pointer.media_type) : void 0;
|
|
14004
14099
|
}
|
|
14005
14100
|
function artifactPointerObjectVersion(pointer) {
|
|
14006
|
-
return
|
|
14101
|
+
return isRecord3(pointer) ? nonEmptyString(pointer.object_version) : void 0;
|
|
14007
14102
|
}
|
|
14008
14103
|
function resolveRecordArtifactPointer(record, kind) {
|
|
14009
14104
|
const legacyArtifactPointers = record.artifact_pointers;
|
|
@@ -14350,6 +14445,12 @@ async function readArtifactCatalogEntryText(searchDir, projectId, meta, entry) {
|
|
|
14350
14445
|
});
|
|
14351
14446
|
return { content };
|
|
14352
14447
|
}
|
|
14448
|
+
function readableLocalCatalogEntry(meta, entry) {
|
|
14449
|
+
if (!entry || entry.storage !== "local") return entry;
|
|
14450
|
+
const baseDir = runWorkspaceDirFromManifestPath(meta.path);
|
|
14451
|
+
const resolved = resolveReadableRunArtifactFile(baseDir, entry.displayPath);
|
|
14452
|
+
return resolved.absolutePath ? entry : void 0;
|
|
14453
|
+
}
|
|
14353
14454
|
function artifactFileContentResponse(c4, filePath, fileContent) {
|
|
14354
14455
|
if (c4.req.query("raw") === "1" || c4.req.query("download") === "1") {
|
|
14355
14456
|
c4.header("Content-Type", inferRawContentType(filePath));
|
|
@@ -15199,6 +15300,10 @@ async function handleEvalTranscript(c4, { searchDir, projectId }) {
|
|
|
15199
15300
|
runPath
|
|
15200
15301
|
);
|
|
15201
15302
|
const answerEntry = findPointerArtifactCatalogEntry(catalog, answer, "answer", runPath);
|
|
15303
|
+
const transcriptRawEntry = readableLocalCatalogEntry(
|
|
15304
|
+
meta,
|
|
15305
|
+
record.transcript_raw_path ? findArtifactCatalogEntry(catalog, record.transcript_raw_path) : void 0
|
|
15306
|
+
);
|
|
15202
15307
|
if (!transcriptEntry && !transcript.path) {
|
|
15203
15308
|
return c4.json({
|
|
15204
15309
|
status: transcript.unsupportedReason ? "unsupported" : "missing",
|
|
@@ -15242,6 +15347,7 @@ async function handleEvalTranscript(c4, { searchDir, projectId }) {
|
|
|
15242
15347
|
return c4.json({
|
|
15243
15348
|
status: "ok",
|
|
15244
15349
|
transcript_path: transcriptEntry.displayPath,
|
|
15350
|
+
...transcriptRawEntry && { transcript_raw_path: transcriptRawEntry.displayPath },
|
|
15245
15351
|
content,
|
|
15246
15352
|
language: inferLanguage(transcriptEntry.displayPath),
|
|
15247
15353
|
...answerEntry && { answer_path: answerEntry.displayPath },
|
|
@@ -16326,7 +16432,7 @@ import { config as loadDotenv } from "dotenv";
|
|
|
16326
16432
|
var TASK_EVAL_FILENAME = "EVAL.yaml";
|
|
16327
16433
|
var TASK_TARGETS_FILENAME = "targets.yaml";
|
|
16328
16434
|
var ENV_REF_PATTERN = /\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
|
|
16329
|
-
function
|
|
16435
|
+
function isRecord4(value) {
|
|
16330
16436
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
16331
16437
|
}
|
|
16332
16438
|
function displayRecord(record) {
|
|
@@ -16368,11 +16474,11 @@ function stringArray(value) {
|
|
|
16368
16474
|
return Array.isArray(value) ? value.filter((entry) => typeof entry === "string" && entry.length > 0) : [];
|
|
16369
16475
|
}
|
|
16370
16476
|
function readExecutionTarget(parsedEval) {
|
|
16371
|
-
if (!
|
|
16477
|
+
if (!isRecord4(parsedEval)) {
|
|
16372
16478
|
return void 0;
|
|
16373
16479
|
}
|
|
16374
16480
|
const execution = parsedEval.execution;
|
|
16375
|
-
if (
|
|
16481
|
+
if (isRecord4(execution) && typeof execution.target === "string" && execution.target.length > 0) {
|
|
16376
16482
|
return execution.target;
|
|
16377
16483
|
}
|
|
16378
16484
|
return typeof parsedEval.target === "string" && parsedEval.target.length > 0 ? parsedEval.target : void 0;
|
|
@@ -16383,10 +16489,10 @@ async function readTaskTarget(evalPath, fallback) {
|
|
|
16383
16489
|
}
|
|
16384
16490
|
async function readTargetDefinitions2(targetsPath) {
|
|
16385
16491
|
const parsed = parseYamlValue(await readFile8(targetsPath, "utf8"));
|
|
16386
|
-
if (!
|
|
16492
|
+
if (!isRecord4(parsed) || !Array.isArray(parsed.targets)) {
|
|
16387
16493
|
throw new Error(`Targets file is missing a top-level targets array: ${targetsPath}`);
|
|
16388
16494
|
}
|
|
16389
|
-
return parsed.targets.filter(
|
|
16495
|
+
return parsed.targets.filter(isRecord4);
|
|
16390
16496
|
}
|
|
16391
16497
|
function targetName(definition) {
|
|
16392
16498
|
if (typeof definition.id === "string" && definition.id.trim().length > 0) {
|
|
@@ -16442,7 +16548,7 @@ function collectEnvRefs(value, names = /* @__PURE__ */ new Set()) {
|
|
|
16442
16548
|
}
|
|
16443
16549
|
return names;
|
|
16444
16550
|
}
|
|
16445
|
-
if (
|
|
16551
|
+
if (isRecord4(value)) {
|
|
16446
16552
|
for (const [key, entry] of Object.entries(value)) {
|
|
16447
16553
|
if (key === "required_env") {
|
|
16448
16554
|
for (const required of stringArray(entry)) {
|
|
@@ -17595,58 +17701,6 @@ var validateCommand = command({
|
|
|
17595
17701
|
}
|
|
17596
17702
|
});
|
|
17597
17703
|
|
|
17598
|
-
// src/commands/workspace/deps.ts
|
|
17599
|
-
import path32 from "node:path";
|
|
17600
|
-
var depsCommand = command({
|
|
17601
|
-
name: "deps",
|
|
17602
|
-
description: "Scan eval files and list git repo dependencies needed by workspaces",
|
|
17603
|
-
args: {
|
|
17604
|
-
evalPaths: restPositionals({
|
|
17605
|
-
type: string,
|
|
17606
|
-
displayName: "eval-paths",
|
|
17607
|
-
description: "Path(s) or glob(s) to evaluation .yaml file(s)"
|
|
17608
|
-
}),
|
|
17609
|
-
usedBy: flag({
|
|
17610
|
-
long: "used-by",
|
|
17611
|
-
description: "Include list of eval files that reference each repo"
|
|
17612
|
-
})
|
|
17613
|
-
},
|
|
17614
|
-
handler: async ({ evalPaths, usedBy }) => {
|
|
17615
|
-
if (evalPaths.length === 0) {
|
|
17616
|
-
console.error("Usage: agentv workspace deps <eval-paths...>");
|
|
17617
|
-
process.exit(1);
|
|
17618
|
-
}
|
|
17619
|
-
const cwd = process.cwd();
|
|
17620
|
-
const resolvedPaths = await resolveEvalPaths(evalPaths, cwd);
|
|
17621
|
-
const result = await scanRepoDeps(resolvedPaths);
|
|
17622
|
-
for (const err2 of result.errors) {
|
|
17623
|
-
console.error(`error: ${path32.relative(cwd, err2.file)}: ${err2.message}`);
|
|
17624
|
-
}
|
|
17625
|
-
if (result.errors.length > 0) {
|
|
17626
|
-
process.exit(1);
|
|
17627
|
-
}
|
|
17628
|
-
const output = {
|
|
17629
|
-
repos: result.repos.map((r) => ({
|
|
17630
|
-
url: r.url,
|
|
17631
|
-
...r.ref !== void 0 && { ref: r.ref },
|
|
17632
|
-
...r.sparse !== void 0 && { sparse: r.sparse },
|
|
17633
|
-
...r.ancestor !== void 0 && { ancestor: r.ancestor },
|
|
17634
|
-
...usedBy && { used_by: r.usedBy.map((p) => path32.relative(cwd, p)) }
|
|
17635
|
-
}))
|
|
17636
|
-
};
|
|
17637
|
-
console.log(JSON.stringify(output, null, 2));
|
|
17638
|
-
}
|
|
17639
|
-
});
|
|
17640
|
-
|
|
17641
|
-
// src/commands/workspace/index.ts
|
|
17642
|
-
var workspaceCommand = subcommands({
|
|
17643
|
-
name: "workspace",
|
|
17644
|
-
description: "Inspect workspace dependencies",
|
|
17645
|
-
cmds: {
|
|
17646
|
-
deps: depsCommand
|
|
17647
|
-
}
|
|
17648
|
-
});
|
|
17649
|
-
|
|
17650
17704
|
// src/update-check.ts
|
|
17651
17705
|
import { spawn as spawn4 } from "node:child_process";
|
|
17652
17706
|
import { readFile as readFile9 } from "node:fs/promises";
|
|
@@ -17655,8 +17709,8 @@ var CHECK_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
|
17655
17709
|
var CONFIG_DIR = getAgentvConfigDir();
|
|
17656
17710
|
var CACHE_FILE = "version-check.json";
|
|
17657
17711
|
var NPM_REGISTRY_BASE2 = "https://registry.npmjs.org/agentv/";
|
|
17658
|
-
async function getCachedUpdateInfo(
|
|
17659
|
-
const filePath =
|
|
17712
|
+
async function getCachedUpdateInfo(path33) {
|
|
17713
|
+
const filePath = path33 ?? join6(CONFIG_DIR, CACHE_FILE);
|
|
17660
17714
|
try {
|
|
17661
17715
|
const raw = await readFile9(filePath, "utf-8");
|
|
17662
17716
|
const data = JSON.parse(raw);
|
|
@@ -17761,8 +17815,7 @@ var app = subcommands({
|
|
|
17761
17815
|
trend: trendCommand,
|
|
17762
17816
|
transpile: transpileCommand,
|
|
17763
17817
|
trim: trimCommand,
|
|
17764
|
-
validate: validateCommand
|
|
17765
|
-
workspace: workspaceCommand
|
|
17818
|
+
validate: validateCommand
|
|
17766
17819
|
}
|
|
17767
17820
|
});
|
|
17768
17821
|
var EVAL_SUBCOMMANDS = /* @__PURE__ */ new Set(["run", "assert", "aggregate", "bundle", "vitest"]);
|
|
@@ -17788,8 +17841,7 @@ var TOP_LEVEL_COMMANDS = /* @__PURE__ */ new Set([
|
|
|
17788
17841
|
"trend",
|
|
17789
17842
|
"transpile",
|
|
17790
17843
|
"trim",
|
|
17791
|
-
"validate"
|
|
17792
|
-
"workspace"
|
|
17844
|
+
"validate"
|
|
17793
17845
|
]);
|
|
17794
17846
|
function usesDeprecatedStudioAlias(argv) {
|
|
17795
17847
|
return argv[2] === "studio";
|
|
@@ -17845,7 +17897,7 @@ ${updateNotice}
|
|
|
17845
17897
|
if (shouldRunBeforeSessionHook(processedArgv)) {
|
|
17846
17898
|
const cwd = process.cwd();
|
|
17847
17899
|
const repoRoot = await findRepoRoot(cwd);
|
|
17848
|
-
const sessionConfig = await loadConfig(
|
|
17900
|
+
const sessionConfig = await loadConfig(path32.join(cwd, "_"), repoRoot);
|
|
17849
17901
|
const beforeSessionCommand = sessionConfig?.hooks?.before_session;
|
|
17850
17902
|
if (beforeSessionCommand) {
|
|
17851
17903
|
runBeforeSessionHook(beforeSessionCommand);
|
|
@@ -17862,4 +17914,4 @@ export {
|
|
|
17862
17914
|
preprocessArgv,
|
|
17863
17915
|
runCli
|
|
17864
17916
|
};
|
|
17865
|
-
//# sourceMappingURL=chunk-
|
|
17917
|
+
//# sourceMappingURL=chunk-3X357HS4.js.map
|