@mappies/miniwat-eval 1.3.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -5
- package/dist/index.cjs +234 -100
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +40 -12
- package/dist/index.d.ts +40 -12
- package/dist/index.js +235 -101
- package/dist/index.js.map +1 -1
- package/dist/main.js +269 -115
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -680,15 +680,17 @@ const CONFIG_FILE_NAME = "eval-config.yml";
|
|
|
680
680
|
const PROJECT_ROOT_MARKERS = [".git", "package.json"];
|
|
681
681
|
var ConfigLocator = class ConfigLocator {
|
|
682
682
|
cwd;
|
|
683
|
-
|
|
683
|
+
displayBase;
|
|
684
|
+
constructor(cwd = process.cwd(), displayBase = cwd) {
|
|
684
685
|
this.cwd = cwd;
|
|
686
|
+
this.displayBase = displayBase;
|
|
685
687
|
}
|
|
686
688
|
locate(target, explicitConfig) {
|
|
687
689
|
if (explicitConfig !== "") {
|
|
688
690
|
const path = resolve(this.cwd, explicitConfig);
|
|
689
691
|
if (!ConfigLocator.isFile(path)) throw new EvalError({
|
|
690
692
|
category: ErrorCategory.Configuration,
|
|
691
|
-
message: `configuration file not found: ${explicitConfig}`
|
|
693
|
+
message: `configuration file not found: ${relative(this.displayBase, path) || explicitConfig}`
|
|
692
694
|
});
|
|
693
695
|
return path;
|
|
694
696
|
}
|
|
@@ -697,7 +699,7 @@ var ConfigLocator = class ConfigLocator {
|
|
|
697
699
|
const found = ConfigLocator.searchUpward(start) ?? ConfigLocator.candidate(this.cwd);
|
|
698
700
|
if (found === void 0) throw new EvalError({
|
|
699
701
|
category: ErrorCategory.Configuration,
|
|
700
|
-
message: `no ${CONFIG_FILE_NAME} found above ${relative(this.
|
|
702
|
+
message: `no ${CONFIG_FILE_NAME} found above ${relative(this.displayBase, targetPath) || "."} or in the working directory; pass --config <path>`
|
|
701
703
|
});
|
|
702
704
|
return found;
|
|
703
705
|
}
|
|
@@ -1292,17 +1294,19 @@ var WebSocketResolver = class WebSocketResolver {
|
|
|
1292
1294
|
// src/config/evalConfigLoader.ts
|
|
1293
1295
|
var EvalConfigLoader = class EvalConfigLoader {
|
|
1294
1296
|
cwd;
|
|
1297
|
+
displayBase;
|
|
1295
1298
|
documentLoader = new YamlDocumentLoader();
|
|
1296
1299
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
1297
1300
|
variablesReader = new VariablesReader();
|
|
1298
1301
|
webSocketResolver = new WebSocketResolver();
|
|
1299
1302
|
environmentResolver;
|
|
1300
|
-
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1303
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
|
|
1301
1304
|
this.cwd = cwd;
|
|
1305
|
+
this.displayBase = displayBase;
|
|
1302
1306
|
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
1303
1307
|
}
|
|
1304
1308
|
load(configPath, override, report = new ReportOverride()) {
|
|
1305
|
-
const document = this.documentLoader.load(configPath, relative(this.
|
|
1309
|
+
const document = this.documentLoader.load(configPath, relative(this.displayBase, configPath));
|
|
1306
1310
|
const issues = [...document.errors];
|
|
1307
1311
|
if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
|
|
1308
1312
|
const named = [];
|
|
@@ -1458,12 +1462,14 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1458
1462
|
// src/config/parametersLoader.ts
|
|
1459
1463
|
var ParametersLoader = class ParametersLoader {
|
|
1460
1464
|
cwd;
|
|
1461
|
-
|
|
1465
|
+
displayBase;
|
|
1466
|
+
constructor(cwd = process.cwd(), displayBase = cwd) {
|
|
1462
1467
|
this.cwd = cwd;
|
|
1468
|
+
this.displayBase = displayBase;
|
|
1463
1469
|
}
|
|
1464
1470
|
load(path) {
|
|
1465
1471
|
const absolute = resolve(this.cwd, path);
|
|
1466
|
-
const shown = relative(this.
|
|
1472
|
+
const shown = relative(this.displayBase, absolute) || path;
|
|
1467
1473
|
let text;
|
|
1468
1474
|
let value;
|
|
1469
1475
|
try {
|
|
@@ -1922,6 +1928,7 @@ var RunWithReader = class RunWithReader {
|
|
|
1922
1928
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
1923
1929
|
var ScenarioLoader = class ScenarioLoader {
|
|
1924
1930
|
cwd;
|
|
1931
|
+
displayBase;
|
|
1925
1932
|
documentLoader = new YamlDocumentLoader();
|
|
1926
1933
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
1927
1934
|
conversationValidator = new ConversationValidator();
|
|
@@ -1929,18 +1936,19 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1929
1936
|
runWithReader = new RunWithReader();
|
|
1930
1937
|
webSocketResolver = new WebSocketResolver();
|
|
1931
1938
|
environmentResolver;
|
|
1932
|
-
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1939
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
|
|
1933
1940
|
this.cwd = cwd;
|
|
1941
|
+
this.displayBase = displayBase;
|
|
1934
1942
|
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
1935
1943
|
}
|
|
1936
1944
|
load(target, config) {
|
|
1937
1945
|
const files = this.discover(resolve(this.cwd, target), config);
|
|
1938
|
-
const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.
|
|
1946
|
+
const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.displayBase, config.configPath)) : void 0;
|
|
1939
1947
|
const issues = [];
|
|
1940
1948
|
const connectionIssues = [];
|
|
1941
1949
|
const specs = [];
|
|
1942
1950
|
for (const file of files) {
|
|
1943
|
-
const document = this.documentLoader.load(file, relative(this.
|
|
1951
|
+
const document = this.documentLoader.load(file, relative(this.displayBase, file));
|
|
1944
1952
|
const fileIssues = [...document.errors];
|
|
1945
1953
|
const fileSecrets = [];
|
|
1946
1954
|
const used = /* @__PURE__ */ new Set();
|
|
@@ -2070,13 +2078,13 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
2070
2078
|
discover(target, config) {
|
|
2071
2079
|
if (!existsSync(target)) throw new EvalError({
|
|
2072
2080
|
category: ErrorCategory.Specification,
|
|
2073
|
-
message: `target not found: ${relative(this.
|
|
2081
|
+
message: `target not found: ${relative(this.displayBase, target) || "."}`
|
|
2074
2082
|
});
|
|
2075
2083
|
const excluded = (path) => path === config.configPath || config.report.directory !== "" && path === config.report.directory;
|
|
2076
2084
|
const files = statSync(target).isDirectory() ? this.walk(target, excluded) : [target].filter((file) => !excluded(file));
|
|
2077
2085
|
if (files.length === 0) throw new EvalError({
|
|
2078
2086
|
category: ErrorCategory.Specification,
|
|
2079
|
-
message: `no scenario files found in ${relative(this.
|
|
2087
|
+
message: `no scenario files found in ${relative(this.displayBase, target) || "."}`
|
|
2080
2088
|
});
|
|
2081
2089
|
return files;
|
|
2082
2090
|
}
|
|
@@ -3534,64 +3542,94 @@ var ExitCodeCalculator = class {
|
|
|
3534
3542
|
}
|
|
3535
3543
|
};
|
|
3536
3544
|
// src/reporting/consoleReporter.ts
|
|
3537
|
-
const
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
3541
|
-
|
|
3542
|
-
[ResultStatus.
|
|
3543
|
-
|
|
3544
|
-
|
|
3545
|
+
const MARKS = {
|
|
3546
|
+
[ResultStatus.Pass]: {
|
|
3547
|
+
symbol: "✔",
|
|
3548
|
+
colour: "green"
|
|
3549
|
+
},
|
|
3550
|
+
[ResultStatus.Flaky]: {
|
|
3551
|
+
symbol: "⚠",
|
|
3552
|
+
colour: "yellow"
|
|
3553
|
+
},
|
|
3554
|
+
[ResultStatus.Fail]: {
|
|
3555
|
+
symbol: "✖",
|
|
3556
|
+
colour: "red"
|
|
3557
|
+
},
|
|
3558
|
+
[ResultStatus.Error]: {
|
|
3559
|
+
symbol: "●",
|
|
3560
|
+
colour: "red"
|
|
3561
|
+
},
|
|
3562
|
+
[ResultStatus.Skipped]: {
|
|
3563
|
+
symbol: "○",
|
|
3564
|
+
colour: "dim"
|
|
3565
|
+
},
|
|
3566
|
+
[ResultStatus.Interrupted]: {
|
|
3567
|
+
symbol: "◌",
|
|
3568
|
+
colour: "dim"
|
|
3569
|
+
}
|
|
3545
3570
|
};
|
|
3571
|
+
const MIN_COLUMNS = 40;
|
|
3572
|
+
const RULE_WIDTH = 72;
|
|
3546
3573
|
var ConsoleReporter = class ConsoleReporter {
|
|
3547
3574
|
verbose;
|
|
3548
3575
|
colour;
|
|
3549
|
-
|
|
3576
|
+
nameWidth;
|
|
3577
|
+
columns;
|
|
3578
|
+
thresholds;
|
|
3579
|
+
constructor(verbose, colour, options = {}) {
|
|
3550
3580
|
this.verbose = verbose;
|
|
3551
3581
|
this.colour = colour;
|
|
3582
|
+
this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
|
|
3583
|
+
this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
|
|
3584
|
+
this.thresholds = options.thresholds;
|
|
3585
|
+
}
|
|
3586
|
+
static nameWidth(specs) {
|
|
3587
|
+
return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
|
|
3552
3588
|
}
|
|
3553
3589
|
header(run) {
|
|
3554
3590
|
return [
|
|
3555
|
-
"Miniwat Eval",
|
|
3591
|
+
" Miniwat Eval",
|
|
3556
3592
|
"",
|
|
3557
|
-
...run.quick ? ["Execution
|
|
3558
|
-
`Repeat
|
|
3559
|
-
`Minimum
|
|
3593
|
+
...run.quick ? [" Execution mode Quick"] : [],
|
|
3594
|
+
` Repeat ${run.repeat}`,
|
|
3595
|
+
` Minimum pass ${run.minPass}`,
|
|
3560
3596
|
"",
|
|
3561
|
-
`Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
3597
|
+
` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
3562
3598
|
"",
|
|
3563
3599
|
""
|
|
3564
3600
|
].join("\n");
|
|
3565
3601
|
}
|
|
3566
3602
|
scenario(result, repeat, tokenUsageCapability) {
|
|
3567
3603
|
const spec = result.spec;
|
|
3568
|
-
const
|
|
3569
|
-
const
|
|
3604
|
+
const title = ConsoleReporter.title(spec);
|
|
3605
|
+
const started = result.attempts.length > 0;
|
|
3606
|
+
const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
|
|
3607
|
+
const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
|
|
3608
|
+
const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
|
|
3570
3609
|
if (!this.verbose) return line;
|
|
3571
|
-
if (
|
|
3610
|
+
if (!started) return `${line}\n`;
|
|
3572
3611
|
const websocket = spec.websocket;
|
|
3573
3612
|
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
3574
|
-
const lines = [
|
|
3575
|
-
|
|
3576
|
-
lines.push(`
|
|
3577
|
-
lines.push(
|
|
3578
|
-
lines.push(`
|
|
3579
|
-
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3613
|
+
const lines = [];
|
|
3614
|
+
if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
|
|
3615
|
+
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
3616
|
+
lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
|
|
3617
|
+
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3580
3618
|
return line + lines.join("\n");
|
|
3581
3619
|
}
|
|
3582
3620
|
static title(spec) {
|
|
3583
3621
|
return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
|
|
3584
3622
|
}
|
|
3585
3623
|
summary(run, exitCode, reportPaths = []) {
|
|
3586
|
-
const
|
|
3624
|
+
const started = run.scenarios.filter((result) => result.attempts.length > 0);
|
|
3625
|
+
const failures = started.flatMap((result) => this.blocks(result, false));
|
|
3626
|
+
const flaky = started.flatMap((result) => this.blocks(result, true));
|
|
3587
3627
|
const lines = this.verbose ? [] : [""];
|
|
3588
|
-
if (
|
|
3589
|
-
|
|
3590
|
-
for (const result of problems) lines.push(...this.problems(result), "");
|
|
3591
|
-
}
|
|
3628
|
+
if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
|
|
3629
|
+
if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
|
|
3592
3630
|
const tests = run.scenarios.flatMap((result) => result.tests);
|
|
3593
|
-
lines.push(...run.status === RunStatus.Interrupted ? [`Run
|
|
3594
|
-
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
|
|
3631
|
+
lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
|
|
3632
|
+
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
|
|
3595
3633
|
}
|
|
3596
3634
|
static footer(run, exitCode, reportPaths) {
|
|
3597
3635
|
return [
|
|
@@ -3605,57 +3643,148 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3605
3643
|
return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
|
|
3606
3644
|
}
|
|
3607
3645
|
static duration(ms) {
|
|
3646
|
+
if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
|
|
3608
3647
|
const seconds = Math.floor(ms / 1e3);
|
|
3609
3648
|
const pad = (value) => String(value).padStart(2, "0");
|
|
3610
3649
|
const hours = Math.floor(seconds / 3600);
|
|
3611
|
-
const
|
|
3612
|
-
return hours > 0 ? `${hours}:${
|
|
3650
|
+
const minutes = Math.floor(seconds / 60) % 60;
|
|
3651
|
+
return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
|
|
3613
3652
|
}
|
|
3614
|
-
|
|
3615
|
-
const
|
|
3616
|
-
const
|
|
3617
|
-
|
|
3618
|
-
const
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
|
|
3623
|
-
for (const test of result.tests.filter((entry) => entry.status !== ResultStatus.Pass)) {
|
|
3624
|
-
lines.push(` ${this.status(test.status, 6)} ${ConsoleReporter.testTitle(test)} ${test.location.toString()} ${ConsoleReporter.counts(test)}`);
|
|
3625
|
-
test.history.forEach((entry, index) => {
|
|
3626
|
-
if (entry.status === ResultStatus.Pass) return;
|
|
3627
|
-
const prefix = ` Attempt ${result.attempts[index].attempt} ${this.status(entry.status)}`;
|
|
3628
|
-
const width = ` Attempt ${result.attempts[index].attempt} ${entry.status}: `.length;
|
|
3629
|
-
if (entry.details.length === 0) {
|
|
3630
|
-
lines.push(prefix);
|
|
3631
|
-
return;
|
|
3632
|
-
}
|
|
3633
|
-
lines.push(`${prefix}: ${ConsoleReporter.indent(entry.details[0], width)}`, ...entry.details.slice(1).map((detail) => `${" ".repeat(width)}${ConsoleReporter.indent(detail, width)}`));
|
|
3634
|
-
});
|
|
3653
|
+
static wrap(text, width) {
|
|
3654
|
+
const lines = [];
|
|
3655
|
+
for (const paragraph of text.split("\n")) {
|
|
3656
|
+
let line = "";
|
|
3657
|
+
for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
|
|
3658
|
+
lines.push(line);
|
|
3659
|
+
line = word;
|
|
3660
|
+
} else line = line === "" ? word : `${line} ${word}`;
|
|
3661
|
+
lines.push(line);
|
|
3635
3662
|
}
|
|
3636
3663
|
return lines;
|
|
3637
3664
|
}
|
|
3665
|
+
section(name, blocks) {
|
|
3666
|
+
return [
|
|
3667
|
+
...this.rule(name),
|
|
3668
|
+
"",
|
|
3669
|
+
...blocks.flatMap((block) => [...block, ""])
|
|
3670
|
+
];
|
|
3671
|
+
}
|
|
3672
|
+
rule(name) {
|
|
3673
|
+
const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
|
|
3674
|
+
return [
|
|
3675
|
+
rule,
|
|
3676
|
+
` ${name}`,
|
|
3677
|
+
rule
|
|
3678
|
+
];
|
|
3679
|
+
}
|
|
3680
|
+
blocks(result, flaky) {
|
|
3681
|
+
const spec = result.spec;
|
|
3682
|
+
const title = ConsoleReporter.title(spec);
|
|
3683
|
+
const blocks = [];
|
|
3684
|
+
if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
|
|
3685
|
+
const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
|
|
3686
|
+
for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
|
|
3687
|
+
return blocks;
|
|
3688
|
+
}
|
|
3689
|
+
attemptBlock(spec, title, attempt) {
|
|
3690
|
+
const execution = attempt.execution;
|
|
3691
|
+
const problems = [];
|
|
3692
|
+
if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
|
|
3693
|
+
for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
|
|
3694
|
+
if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
|
|
3695
|
+
if (problems.length === 0) return [];
|
|
3696
|
+
const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
|
|
3697
|
+
const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
|
|
3698
|
+
return [[
|
|
3699
|
+
` scenario at ${spec.location.file}`,
|
|
3700
|
+
` ${this.mark(status)} ${title} › ${what}`,
|
|
3701
|
+
...problems.flatMap((problem) => this.paragraph(problem, 3))
|
|
3702
|
+
]];
|
|
3703
|
+
}
|
|
3704
|
+
testBlock(spec, title, test, result) {
|
|
3705
|
+
const location = test.location;
|
|
3706
|
+
const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
|
|
3707
|
+
const failing = test.history.map((entry, index) => ({
|
|
3708
|
+
entry,
|
|
3709
|
+
run: result.attempts[index],
|
|
3710
|
+
attempt: result.attempts[index]?.attempt ?? index + 1
|
|
3711
|
+
})).filter(({ entry }) => entry.status !== ResultStatus.Pass);
|
|
3712
|
+
const several = test.history.length > 1;
|
|
3713
|
+
for (const { entry, run, attempt } of failing) {
|
|
3714
|
+
if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
|
|
3715
|
+
lines.push(...this.entryBody(spec, entry, run, several ? 5 : 3));
|
|
3716
|
+
}
|
|
3717
|
+
lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
|
|
3718
|
+
return lines;
|
|
3719
|
+
}
|
|
3720
|
+
entryBody(spec, entry, attempt, indent) {
|
|
3721
|
+
const pad = " ".repeat(indent);
|
|
3722
|
+
const evaluation = entry.evaluation;
|
|
3723
|
+
if (evaluation === void 0) {
|
|
3724
|
+
if (entry.details.length === 0) return [];
|
|
3725
|
+
return [
|
|
3726
|
+
"",
|
|
3727
|
+
`${pad}Details`,
|
|
3728
|
+
...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
|
|
3729
|
+
];
|
|
3730
|
+
}
|
|
3731
|
+
const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
|
|
3732
|
+
const expected = ConsoleReporter.expected(spec, entry);
|
|
3733
|
+
const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
|
|
3734
|
+
const actual = ConsoleReporter.actual(entry, attempt);
|
|
3735
|
+
if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
|
|
3736
|
+
if (actual !== void 0) lines.push("", `${pad}Actual`, ...this.capped(actual, indent + 2));
|
|
3737
|
+
if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
|
|
3738
|
+
return lines;
|
|
3739
|
+
}
|
|
3740
|
+
static actual(entry, attempt) {
|
|
3741
|
+
const turns = attempt?.execution.turns ?? [];
|
|
3742
|
+
if (entry.kind === TestKind.Conversation) return turns.length === 0 ? void 0 : turns.map((turn, index) => `Turn ${index + 1}: ${turn.response}`).join("\n");
|
|
3743
|
+
return turns[entry.turn - 1]?.response;
|
|
3744
|
+
}
|
|
3745
|
+
capped(text, indent) {
|
|
3746
|
+
const lines = text.trim() === "" ? [`${" ".repeat(indent)}(empty)`] : this.paragraph(text, indent);
|
|
3747
|
+
if (lines.length <= 12) return lines;
|
|
3748
|
+
const more = lines.length - 12;
|
|
3749
|
+
return [...lines.slice(0, 12), `${" ".repeat(indent)}… ${more} more ${more === 1 ? "line" : "lines"} in the JSON report`];
|
|
3750
|
+
}
|
|
3751
|
+
static expected(spec, entry) {
|
|
3752
|
+
if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
|
|
3753
|
+
return spec.turns[entry.turn - 1]?.expect.response?.criteria;
|
|
3754
|
+
}
|
|
3755
|
+
paragraph(text, indent) {
|
|
3756
|
+
const pad = " ".repeat(indent);
|
|
3757
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
|
|
3758
|
+
}
|
|
3759
|
+
bullet(text, indent) {
|
|
3760
|
+
const pad = " ".repeat(indent);
|
|
3761
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
|
|
3762
|
+
}
|
|
3638
3763
|
static testTitle(test) {
|
|
3639
3764
|
return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
|
|
3640
3765
|
}
|
|
3641
3766
|
static tally(statuses) {
|
|
3642
3767
|
const count = (status) => statuses.filter((entry) => entry === status).length;
|
|
3643
|
-
const
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3768
|
+
const parts = [
|
|
3769
|
+
[ResultStatus.Pass, "passed"],
|
|
3770
|
+
[ResultStatus.Flaky, "flaky"],
|
|
3771
|
+
[ResultStatus.Fail, "failed"],
|
|
3772
|
+
[ResultStatus.Error, "errors"],
|
|
3773
|
+
[ResultStatus.Interrupted, "interrupted"]
|
|
3774
|
+
].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
|
|
3775
|
+
return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
|
|
3776
|
+
}
|
|
3777
|
+
mark(status) {
|
|
3778
|
+
const { symbol, colour } = MARKS[status];
|
|
3779
|
+
return this.colour ? styleText(colour, symbol, { validateStream: false }) : symbol;
|
|
3652
3780
|
}
|
|
3653
3781
|
static usage(label, usage, capability) {
|
|
3654
|
-
|
|
3655
|
-
if (
|
|
3656
|
-
if (usage.
|
|
3782
|
+
const name = label.padEnd(16);
|
|
3783
|
+
if (!capability) return `${name} n/a (not supported by host)`;
|
|
3784
|
+
if (usage.expected === 0) return `${name} none`;
|
|
3785
|
+
if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
|
|
3657
3786
|
const number = (value) => value.toLocaleString("en-US");
|
|
3658
|
-
const total = `${
|
|
3787
|
+
const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
|
|
3659
3788
|
return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
|
|
3660
3789
|
}
|
|
3661
3790
|
formatAttempt(spec, attempt) {
|
|
@@ -3667,28 +3796,28 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3667
3796
|
lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
|
|
3668
3797
|
lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
|
|
3669
3798
|
});
|
|
3670
|
-
if (execution.failure !== void 0) lines.push(`
|
|
3799
|
+
if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
|
|
3671
3800
|
const notRun = attempt.tests.filter((test) => test.turn > ran);
|
|
3672
|
-
if (notRun.length > 0) lines.push("
|
|
3801
|
+
if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
|
|
3673
3802
|
lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
|
|
3674
|
-
for (const error of execution.attemptErrors) lines.push(`
|
|
3675
|
-
if (execution.closeError !== void 0) lines.push(`
|
|
3676
|
-
lines.push(`
|
|
3803
|
+
for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
|
|
3804
|
+
if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
|
|
3805
|
+
lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
|
|
3677
3806
|
return lines;
|
|
3678
3807
|
}
|
|
3679
3808
|
formatTest(test) {
|
|
3680
|
-
return [`
|
|
3809
|
+
return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
|
|
3681
3810
|
}
|
|
3682
3811
|
static formatTurn(number, input, turn) {
|
|
3683
3812
|
const lines = [
|
|
3684
|
-
`
|
|
3685
|
-
`
|
|
3686
|
-
`
|
|
3813
|
+
` Turn ${number} (${turn.durationMs} ms)`,
|
|
3814
|
+
` Input: ${ConsoleReporter.indent(input, 22)}`,
|
|
3815
|
+
` Response: ${ConsoleReporter.indent(turn.response, 22)}`
|
|
3687
3816
|
];
|
|
3688
|
-
for (const use of turn.toolUses ?? []) lines.push(`
|
|
3689
|
-
for (const result of turn.toolResults ?? []) lines.push(`
|
|
3690
|
-
for (const status of turn.statuses) lines.push(`
|
|
3691
|
-
if (turn.tokenUsage !== void 0) lines.push(`
|
|
3817
|
+
for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
|
|
3818
|
+
for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
|
|
3819
|
+
for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
|
|
3820
|
+
if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
|
|
3692
3821
|
return lines;
|
|
3693
3822
|
}
|
|
3694
3823
|
static indent(text, width) {
|
|
@@ -3718,10 +3847,12 @@ const JSON_REPORT_VERSION = 1;
|
|
|
3718
3847
|
var JsonReporter = class JsonReporter {
|
|
3719
3848
|
redactor;
|
|
3720
3849
|
cwd;
|
|
3850
|
+
displayBase;
|
|
3721
3851
|
tokens = new TokenUsageAggregator();
|
|
3722
|
-
constructor(redactor, cwd) {
|
|
3852
|
+
constructor(redactor, cwd, displayBase = cwd) {
|
|
3723
3853
|
this.redactor = redactor;
|
|
3724
3854
|
this.cwd = cwd;
|
|
3855
|
+
this.displayBase = displayBase;
|
|
3725
3856
|
}
|
|
3726
3857
|
render(run, config, unsuccessfulCount, exitCode) {
|
|
3727
3858
|
const report = {
|
|
@@ -3792,7 +3923,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3792
3923
|
return {
|
|
3793
3924
|
name: result.spec.name,
|
|
3794
3925
|
runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
|
|
3795
|
-
file: result.spec.location.file,
|
|
3926
|
+
file: this.file(result.spec.location.file),
|
|
3796
3927
|
status: result.status,
|
|
3797
3928
|
connection: {
|
|
3798
3929
|
endpoint: websocket.endpoint,
|
|
@@ -3802,7 +3933,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3802
3933
|
agentUsage: JsonReporter.usage(result.agentUsage),
|
|
3803
3934
|
evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
|
|
3804
3935
|
tests: result.tests.map((test) => ({
|
|
3805
|
-
...
|
|
3936
|
+
...this.identity(test),
|
|
3806
3937
|
status: test.status,
|
|
3807
3938
|
...JsonReporter.counts(test),
|
|
3808
3939
|
history: test.history.map((entry, index) => ({
|
|
@@ -3827,7 +3958,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3827
3958
|
agentUsage: JsonReporter.usage(this.tokens.agent([attempt], capability)),
|
|
3828
3959
|
evaluatorUsage: JsonReporter.usage(this.tokens.evaluator([attempt])),
|
|
3829
3960
|
turns: execution.turns.map((turn, index) => JsonReporter.turn(index + 1, spec.turns[index].input, turn)),
|
|
3830
|
-
tests: attempt.tests.map((test) =>
|
|
3961
|
+
tests: attempt.tests.map((test) => this.test(test, config))
|
|
3831
3962
|
};
|
|
3832
3963
|
}
|
|
3833
3964
|
static turn(number, input, turn) {
|
|
@@ -3852,11 +3983,11 @@ var JsonReporter = class JsonReporter {
|
|
|
3852
3983
|
tokenUsage: turn.tokenUsage === void 0 ? void 0 : JsonReporter.tokenUsage(turn.tokenUsage)
|
|
3853
3984
|
};
|
|
3854
3985
|
}
|
|
3855
|
-
|
|
3986
|
+
test(test, config) {
|
|
3856
3987
|
const thresholds = config.evaluator.thresholds;
|
|
3857
3988
|
const evaluation = test.evaluation;
|
|
3858
3989
|
return {
|
|
3859
|
-
...
|
|
3990
|
+
...this.identity(test),
|
|
3860
3991
|
status: test.status,
|
|
3861
3992
|
details: [...test.details],
|
|
3862
3993
|
evaluation: evaluation === void 0 ? void 0 : {
|
|
@@ -3869,17 +4000,17 @@ var JsonReporter = class JsonReporter {
|
|
|
3869
4000
|
evaluatorUsage: JsonReporter.usage(test.evaluatorUsage)
|
|
3870
4001
|
};
|
|
3871
4002
|
}
|
|
3872
|
-
|
|
4003
|
+
identity(test) {
|
|
3873
4004
|
return {
|
|
3874
4005
|
kind: test.kind,
|
|
3875
4006
|
turn: test.turn,
|
|
3876
4007
|
title: test.title,
|
|
3877
|
-
location:
|
|
4008
|
+
location: this.location(test.location)
|
|
3878
4009
|
};
|
|
3879
4010
|
}
|
|
3880
|
-
|
|
4011
|
+
location(location) {
|
|
3881
4012
|
return {
|
|
3882
|
-
file: location.file,
|
|
4013
|
+
file: this.file(location.file),
|
|
3883
4014
|
line: location.line,
|
|
3884
4015
|
column: location.column,
|
|
3885
4016
|
path: location.path
|
|
@@ -3926,6 +4057,9 @@ var JsonReporter = class JsonReporter {
|
|
|
3926
4057
|
path(path) {
|
|
3927
4058
|
return relative(this.cwd, path) || ".";
|
|
3928
4059
|
}
|
|
4060
|
+
file(shown) {
|
|
4061
|
+
return shown === "" ? shown : this.path(resolve(this.displayBase, shown));
|
|
4062
|
+
}
|
|
3929
4063
|
};
|
|
3930
4064
|
// src/reporting/reportFileWriter.ts
|
|
3931
4065
|
var ReportFileWriter = class {
|