@mappies/miniwat-eval 1.3.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -5
- package/dist/index.cjs +234 -100
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +40 -12
- package/dist/index.d.ts +40 -12
- package/dist/index.js +235 -101
- package/dist/index.js.map +1 -1
- package/dist/main.js +269 -115
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -299,13 +299,53 @@ A turn stops its attempt when the host reports an error (`statusCode >= 400` or
|
|
|
299
299
|
|
|
300
300
|
### Reports
|
|
301
301
|
|
|
302
|
-
The console prints one line per scenario as it finishes, in file order
|
|
302
|
+
The console prints one line per scenario as it finishes, in file order: a status symbol, the name, the attempts that passed out of `repeat`, and the time it took.
|
|
303
303
|
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
304
|
+
```text
|
|
305
|
+
✔ Greeting [user-token] 1/1 2.6s
|
|
306
|
+
✖ Password reset 0/1 7.9s
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
| Symbol | Status |
|
|
310
|
+
|---|---|
|
|
311
|
+
| `✔` | `PASS` |
|
|
312
|
+
| `✖` | `FAIL` |
|
|
313
|
+
| `⚠` | `FLAKY` |
|
|
314
|
+
| `●` | `ERROR` |
|
|
315
|
+
| `○` | `SKIPPED` (a test that did not run) |
|
|
316
|
+
| `◌` | `INTERRUPTED`, or a scenario that never started |
|
|
317
|
+
|
|
318
|
+
A `FAILURES` section follows, with one block per test that did not pass and per attempt that stopped:
|
|
319
|
+
|
|
320
|
+
```text
|
|
321
|
+
test at scenarios/authenticated/knowledge-base/password-reset.yml:11:9
|
|
322
|
+
✖ Password reset › turn 1: response
|
|
323
|
+
Score 0.00, threshold 0.85
|
|
324
|
+
|
|
325
|
+
Expected
|
|
326
|
+
Explains how to reset the password using the steps from the knowledge
|
|
327
|
+
base article it found, without adding steps the article does not support.
|
|
328
|
+
|
|
329
|
+
Actual
|
|
330
|
+
I'm afraid the available documentation does not contain a specific
|
|
331
|
+
procedure for resetting a Virginia Cyber Range password. ...
|
|
332
|
+
|
|
333
|
+
Explanation
|
|
334
|
+
The response does not explain any password-reset steps from a
|
|
335
|
+
knowledge-base article.
|
|
336
|
+
|
|
337
|
+
Attempts 1: 0 passed, 1 failed, 0 errors (1 required)
|
|
338
|
+
```
|
|
339
|
+
|
|
340
|
+
- **Semantic checks** show the score and threshold, the `should` text as Expected, the agent's response as Actual (every turn's response for `expect.conversation`, up to 12 lines) and the evaluator's explanation. The evaluator's evidence and the full response are in the JSON report.
|
|
341
|
+
- **Other checks** list their details, such as the closest tool use and how its arguments differ.
|
|
342
|
+
- **Repeated runs:** with `repeat` above 1, each failing attempt has its own entry.
|
|
343
|
+
- **Flaky tests** are listed in a `FLAKY` section after it.
|
|
344
|
+
- **Summary:** the report ends with a `SUMMARY` of scenario and test counts, agent and evaluator token usage with coverage (such as `coverage 8/9, partial`, when not every call reported usage), the duration and the exit code.
|
|
345
|
+
- **Wrapping:** text wraps at the terminal's width, or at 100 columns when output is redirected.
|
|
346
|
+
- **Paths** are relative to the directory `npm run` was started from (npm's `INIT_CWD`) when a script changes directory first, so `eval/scenarios/a.yml:11:17` is a link in the terminal. Without npm they are relative to the working directory. Paths you pass are always resolved against the working directory, and the JSON report's paths are relative to it.
|
|
307
347
|
|
|
308
|
-
`--verbose` adds every attempt's transcript.
|
|
348
|
+
`--verbose` adds every attempt's transcript under its scenario line. Symbols are coloured on a terminal unless `NO_COLOR` is set; the symbol is shown either way. The console format changed in 1.4.0; scripts should read the JSON report, not the console.
|
|
309
349
|
|
|
310
350
|
`json` writes `miniwat-eval-report.json` to `report.directory`, replacing the previous one, and prints `Report: <path>` above the final line. It follows `schemas/report.schema.json` and holds:
|
|
311
351
|
|
package/dist/index.cjs
CHANGED
|
@@ -703,15 +703,17 @@ const CONFIG_FILE_NAME = "eval-config.yml";
|
|
|
703
703
|
const PROJECT_ROOT_MARKERS = [".git", "package.json"];
|
|
704
704
|
var ConfigLocator = class ConfigLocator {
|
|
705
705
|
cwd;
|
|
706
|
-
|
|
706
|
+
displayBase;
|
|
707
|
+
constructor(cwd = process.cwd(), displayBase = cwd) {
|
|
707
708
|
this.cwd = cwd;
|
|
709
|
+
this.displayBase = displayBase;
|
|
708
710
|
}
|
|
709
711
|
locate(target, explicitConfig) {
|
|
710
712
|
if (explicitConfig !== "") {
|
|
711
713
|
const path = (0, node_path.resolve)(this.cwd, explicitConfig);
|
|
712
714
|
if (!ConfigLocator.isFile(path)) throw new EvalError({
|
|
713
715
|
category: ErrorCategory.Configuration,
|
|
714
|
-
message: `configuration file not found: ${explicitConfig}`
|
|
716
|
+
message: `configuration file not found: ${(0, node_path.relative)(this.displayBase, path) || explicitConfig}`
|
|
715
717
|
});
|
|
716
718
|
return path;
|
|
717
719
|
}
|
|
@@ -720,7 +722,7 @@ var ConfigLocator = class ConfigLocator {
|
|
|
720
722
|
const found = ConfigLocator.searchUpward(start) ?? ConfigLocator.candidate(this.cwd);
|
|
721
723
|
if (found === void 0) throw new EvalError({
|
|
722
724
|
category: ErrorCategory.Configuration,
|
|
723
|
-
message: `no ${CONFIG_FILE_NAME} found above ${(0, node_path.relative)(this.
|
|
725
|
+
message: `no ${CONFIG_FILE_NAME} found above ${(0, node_path.relative)(this.displayBase, targetPath) || "."} or in the working directory; pass --config <path>`
|
|
724
726
|
});
|
|
725
727
|
return found;
|
|
726
728
|
}
|
|
@@ -1315,17 +1317,19 @@ var WebSocketResolver = class WebSocketResolver {
|
|
|
1315
1317
|
// src/config/evalConfigLoader.ts
|
|
1316
1318
|
var EvalConfigLoader = class EvalConfigLoader {
|
|
1317
1319
|
cwd;
|
|
1320
|
+
displayBase;
|
|
1318
1321
|
documentLoader = new YamlDocumentLoader();
|
|
1319
1322
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
1320
1323
|
variablesReader = new VariablesReader();
|
|
1321
1324
|
webSocketResolver = new WebSocketResolver();
|
|
1322
1325
|
environmentResolver;
|
|
1323
|
-
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1326
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
|
|
1324
1327
|
this.cwd = cwd;
|
|
1328
|
+
this.displayBase = displayBase;
|
|
1325
1329
|
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
1326
1330
|
}
|
|
1327
1331
|
load(configPath, override, report = new ReportOverride()) {
|
|
1328
|
-
const document = this.documentLoader.load(configPath, (0, node_path.relative)(this.
|
|
1332
|
+
const document = this.documentLoader.load(configPath, (0, node_path.relative)(this.displayBase, configPath));
|
|
1329
1333
|
const issues = [...document.errors];
|
|
1330
1334
|
if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
|
|
1331
1335
|
const named = [];
|
|
@@ -1481,12 +1485,14 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1481
1485
|
// src/config/parametersLoader.ts
|
|
1482
1486
|
var ParametersLoader = class ParametersLoader {
|
|
1483
1487
|
cwd;
|
|
1484
|
-
|
|
1488
|
+
displayBase;
|
|
1489
|
+
constructor(cwd = process.cwd(), displayBase = cwd) {
|
|
1485
1490
|
this.cwd = cwd;
|
|
1491
|
+
this.displayBase = displayBase;
|
|
1486
1492
|
}
|
|
1487
1493
|
load(path) {
|
|
1488
1494
|
const absolute = (0, node_path.resolve)(this.cwd, path);
|
|
1489
|
-
const shown = (0, node_path.relative)(this.
|
|
1495
|
+
const shown = (0, node_path.relative)(this.displayBase, absolute) || path;
|
|
1490
1496
|
let text;
|
|
1491
1497
|
let value;
|
|
1492
1498
|
try {
|
|
@@ -1945,6 +1951,7 @@ var RunWithReader = class RunWithReader {
|
|
|
1945
1951
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
1946
1952
|
var ScenarioLoader = class ScenarioLoader {
|
|
1947
1953
|
cwd;
|
|
1954
|
+
displayBase;
|
|
1948
1955
|
documentLoader = new YamlDocumentLoader();
|
|
1949
1956
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
1950
1957
|
conversationValidator = new ConversationValidator();
|
|
@@ -1952,18 +1959,19 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1952
1959
|
runWithReader = new RunWithReader();
|
|
1953
1960
|
webSocketResolver = new WebSocketResolver();
|
|
1954
1961
|
environmentResolver;
|
|
1955
|
-
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1962
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
|
|
1956
1963
|
this.cwd = cwd;
|
|
1964
|
+
this.displayBase = displayBase;
|
|
1957
1965
|
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
1958
1966
|
}
|
|
1959
1967
|
load(target, config) {
|
|
1960
1968
|
const files = this.discover((0, node_path.resolve)(this.cwd, target), config);
|
|
1961
|
-
const configDocument = (0, node_fs.existsSync)(config.configPath) ? this.documentLoader.load(config.configPath, (0, node_path.relative)(this.
|
|
1969
|
+
const configDocument = (0, node_fs.existsSync)(config.configPath) ? this.documentLoader.load(config.configPath, (0, node_path.relative)(this.displayBase, config.configPath)) : void 0;
|
|
1962
1970
|
const issues = [];
|
|
1963
1971
|
const connectionIssues = [];
|
|
1964
1972
|
const specs = [];
|
|
1965
1973
|
for (const file of files) {
|
|
1966
|
-
const document = this.documentLoader.load(file, (0, node_path.relative)(this.
|
|
1974
|
+
const document = this.documentLoader.load(file, (0, node_path.relative)(this.displayBase, file));
|
|
1967
1975
|
const fileIssues = [...document.errors];
|
|
1968
1976
|
const fileSecrets = [];
|
|
1969
1977
|
const used = /* @__PURE__ */ new Set();
|
|
@@ -2093,13 +2101,13 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
2093
2101
|
discover(target, config) {
|
|
2094
2102
|
if (!(0, node_fs.existsSync)(target)) throw new EvalError({
|
|
2095
2103
|
category: ErrorCategory.Specification,
|
|
2096
|
-
message: `target not found: ${(0, node_path.relative)(this.
|
|
2104
|
+
message: `target not found: ${(0, node_path.relative)(this.displayBase, target) || "."}`
|
|
2097
2105
|
});
|
|
2098
2106
|
const excluded = (path) => path === config.configPath || config.report.directory !== "" && path === config.report.directory;
|
|
2099
2107
|
const files = (0, node_fs.statSync)(target).isDirectory() ? this.walk(target, excluded) : [target].filter((file) => !excluded(file));
|
|
2100
2108
|
if (files.length === 0) throw new EvalError({
|
|
2101
2109
|
category: ErrorCategory.Specification,
|
|
2102
|
-
message: `no scenario files found in ${(0, node_path.relative)(this.
|
|
2110
|
+
message: `no scenario files found in ${(0, node_path.relative)(this.displayBase, target) || "."}`
|
|
2103
2111
|
});
|
|
2104
2112
|
return files;
|
|
2105
2113
|
}
|
|
@@ -3557,64 +3565,94 @@ var ExitCodeCalculator = class {
|
|
|
3557
3565
|
}
|
|
3558
3566
|
};
|
|
3559
3567
|
// src/reporting/consoleReporter.ts
|
|
3560
|
-
const
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
[ResultStatus.
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
+
const MARKS = {
|
|
3569
|
+
[ResultStatus.Pass]: {
|
|
3570
|
+
symbol: "✔",
|
|
3571
|
+
colour: "green"
|
|
3572
|
+
},
|
|
3573
|
+
[ResultStatus.Flaky]: {
|
|
3574
|
+
symbol: "⚠",
|
|
3575
|
+
colour: "yellow"
|
|
3576
|
+
},
|
|
3577
|
+
[ResultStatus.Fail]: {
|
|
3578
|
+
symbol: "✖",
|
|
3579
|
+
colour: "red"
|
|
3580
|
+
},
|
|
3581
|
+
[ResultStatus.Error]: {
|
|
3582
|
+
symbol: "●",
|
|
3583
|
+
colour: "red"
|
|
3584
|
+
},
|
|
3585
|
+
[ResultStatus.Skipped]: {
|
|
3586
|
+
symbol: "○",
|
|
3587
|
+
colour: "dim"
|
|
3588
|
+
},
|
|
3589
|
+
[ResultStatus.Interrupted]: {
|
|
3590
|
+
symbol: "◌",
|
|
3591
|
+
colour: "dim"
|
|
3592
|
+
}
|
|
3568
3593
|
};
|
|
3594
|
+
const MIN_COLUMNS = 40;
|
|
3595
|
+
const RULE_WIDTH = 72;
|
|
3569
3596
|
var ConsoleReporter = class ConsoleReporter {
|
|
3570
3597
|
verbose;
|
|
3571
3598
|
colour;
|
|
3572
|
-
|
|
3599
|
+
nameWidth;
|
|
3600
|
+
columns;
|
|
3601
|
+
thresholds;
|
|
3602
|
+
constructor(verbose, colour, options = {}) {
|
|
3573
3603
|
this.verbose = verbose;
|
|
3574
3604
|
this.colour = colour;
|
|
3605
|
+
this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
|
|
3606
|
+
this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
|
|
3607
|
+
this.thresholds = options.thresholds;
|
|
3608
|
+
}
|
|
3609
|
+
static nameWidth(specs) {
|
|
3610
|
+
return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
|
|
3575
3611
|
}
|
|
3576
3612
|
header(run) {
|
|
3577
3613
|
return [
|
|
3578
|
-
"Miniwat Eval",
|
|
3614
|
+
" Miniwat Eval",
|
|
3579
3615
|
"",
|
|
3580
|
-
...run.quick ? ["Execution
|
|
3581
|
-
`Repeat
|
|
3582
|
-
`Minimum
|
|
3616
|
+
...run.quick ? [" Execution mode Quick"] : [],
|
|
3617
|
+
` Repeat ${run.repeat}`,
|
|
3618
|
+
` Minimum pass ${run.minPass}`,
|
|
3583
3619
|
"",
|
|
3584
|
-
`Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
3620
|
+
` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
3585
3621
|
"",
|
|
3586
3622
|
""
|
|
3587
3623
|
].join("\n");
|
|
3588
3624
|
}
|
|
3589
3625
|
scenario(result, repeat, tokenUsageCapability) {
|
|
3590
3626
|
const spec = result.spec;
|
|
3591
|
-
const
|
|
3592
|
-
const
|
|
3627
|
+
const title = ConsoleReporter.title(spec);
|
|
3628
|
+
const started = result.attempts.length > 0;
|
|
3629
|
+
const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
|
|
3630
|
+
const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
|
|
3631
|
+
const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
|
|
3593
3632
|
if (!this.verbose) return line;
|
|
3594
|
-
if (
|
|
3633
|
+
if (!started) return `${line}\n`;
|
|
3595
3634
|
const websocket = spec.websocket;
|
|
3596
3635
|
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
3597
|
-
const lines = [
|
|
3598
|
-
|
|
3599
|
-
lines.push(`
|
|
3600
|
-
lines.push(
|
|
3601
|
-
lines.push(`
|
|
3602
|
-
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3636
|
+
const lines = [];
|
|
3637
|
+
if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
|
|
3638
|
+
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
3639
|
+
lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
|
|
3640
|
+
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3603
3641
|
return line + lines.join("\n");
|
|
3604
3642
|
}
|
|
3605
3643
|
static title(spec) {
|
|
3606
3644
|
return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
|
|
3607
3645
|
}
|
|
3608
3646
|
summary(run, exitCode, reportPaths = []) {
|
|
3609
|
-
const
|
|
3647
|
+
const started = run.scenarios.filter((result) => result.attempts.length > 0);
|
|
3648
|
+
const failures = started.flatMap((result) => this.blocks(result, false));
|
|
3649
|
+
const flaky = started.flatMap((result) => this.blocks(result, true));
|
|
3610
3650
|
const lines = this.verbose ? [] : [""];
|
|
3611
|
-
if (
|
|
3612
|
-
|
|
3613
|
-
for (const result of problems) lines.push(...this.problems(result), "");
|
|
3614
|
-
}
|
|
3651
|
+
if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
|
|
3652
|
+
if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
|
|
3615
3653
|
const tests = run.scenarios.flatMap((result) => result.tests);
|
|
3616
|
-
lines.push(...run.status === RunStatus.Interrupted ? [`Run
|
|
3617
|
-
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
|
|
3654
|
+
lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
|
|
3655
|
+
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
|
|
3618
3656
|
}
|
|
3619
3657
|
static footer(run, exitCode, reportPaths) {
|
|
3620
3658
|
return [
|
|
@@ -3628,57 +3666,148 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3628
3666
|
return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
|
|
3629
3667
|
}
|
|
3630
3668
|
static duration(ms) {
|
|
3669
|
+
if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
|
|
3631
3670
|
const seconds = Math.floor(ms / 1e3);
|
|
3632
3671
|
const pad = (value) => String(value).padStart(2, "0");
|
|
3633
3672
|
const hours = Math.floor(seconds / 3600);
|
|
3634
|
-
const
|
|
3635
|
-
return hours > 0 ? `${hours}:${
|
|
3673
|
+
const minutes = Math.floor(seconds / 60) % 60;
|
|
3674
|
+
return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
|
|
3636
3675
|
}
|
|
3637
|
-
|
|
3676
|
+
static wrap(text, width) {
|
|
3677
|
+
const lines = [];
|
|
3678
|
+
for (const paragraph of text.split("\n")) {
|
|
3679
|
+
let line = "";
|
|
3680
|
+
for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
|
|
3681
|
+
lines.push(line);
|
|
3682
|
+
line = word;
|
|
3683
|
+
} else line = line === "" ? word : `${line} ${word}`;
|
|
3684
|
+
lines.push(line);
|
|
3685
|
+
}
|
|
3686
|
+
return lines;
|
|
3687
|
+
}
|
|
3688
|
+
section(name, blocks) {
|
|
3689
|
+
return [
|
|
3690
|
+
...this.rule(name),
|
|
3691
|
+
"",
|
|
3692
|
+
...blocks.flatMap((block) => [...block, ""])
|
|
3693
|
+
];
|
|
3694
|
+
}
|
|
3695
|
+
rule(name) {
|
|
3696
|
+
const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
|
|
3697
|
+
return [
|
|
3698
|
+
rule,
|
|
3699
|
+
` ${name}`,
|
|
3700
|
+
rule
|
|
3701
|
+
];
|
|
3702
|
+
}
|
|
3703
|
+
blocks(result, flaky) {
|
|
3638
3704
|
const spec = result.spec;
|
|
3639
|
-
const
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
|
|
3653
|
-
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3657
|
-
}
|
|
3705
|
+
const title = ConsoleReporter.title(spec);
|
|
3706
|
+
const blocks = [];
|
|
3707
|
+
if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
|
|
3708
|
+
const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
|
|
3709
|
+
for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
|
|
3710
|
+
return blocks;
|
|
3711
|
+
}
|
|
3712
|
+
attemptBlock(spec, title, attempt) {
|
|
3713
|
+
const execution = attempt.execution;
|
|
3714
|
+
const problems = [];
|
|
3715
|
+
if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
|
|
3716
|
+
for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
|
|
3717
|
+
if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
|
|
3718
|
+
if (problems.length === 0) return [];
|
|
3719
|
+
const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
|
|
3720
|
+
const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
|
|
3721
|
+
return [[
|
|
3722
|
+
` scenario at ${spec.location.file}`,
|
|
3723
|
+
` ${this.mark(status)} ${title} › ${what}`,
|
|
3724
|
+
...problems.flatMap((problem) => this.paragraph(problem, 3))
|
|
3725
|
+
]];
|
|
3726
|
+
}
|
|
3727
|
+
testBlock(spec, title, test, result) {
|
|
3728
|
+
const location = test.location;
|
|
3729
|
+
const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
|
|
3730
|
+
const failing = test.history.map((entry, index) => ({
|
|
3731
|
+
entry,
|
|
3732
|
+
run: result.attempts[index],
|
|
3733
|
+
attempt: result.attempts[index]?.attempt ?? index + 1
|
|
3734
|
+
})).filter(({ entry }) => entry.status !== ResultStatus.Pass);
|
|
3735
|
+
const several = test.history.length > 1;
|
|
3736
|
+
for (const { entry, run, attempt } of failing) {
|
|
3737
|
+
if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
|
|
3738
|
+
lines.push(...this.entryBody(spec, entry, run, several ? 5 : 3));
|
|
3739
|
+
}
|
|
3740
|
+
lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
|
|
3741
|
+
return lines;
|
|
3742
|
+
}
|
|
3743
|
+
entryBody(spec, entry, attempt, indent) {
|
|
3744
|
+
const pad = " ".repeat(indent);
|
|
3745
|
+
const evaluation = entry.evaluation;
|
|
3746
|
+
if (evaluation === void 0) {
|
|
3747
|
+
if (entry.details.length === 0) return [];
|
|
3748
|
+
return [
|
|
3749
|
+
"",
|
|
3750
|
+
`${pad}Details`,
|
|
3751
|
+
...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
|
|
3752
|
+
];
|
|
3658
3753
|
}
|
|
3754
|
+
const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
|
|
3755
|
+
const expected = ConsoleReporter.expected(spec, entry);
|
|
3756
|
+
const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
|
|
3757
|
+
const actual = ConsoleReporter.actual(entry, attempt);
|
|
3758
|
+
if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
|
|
3759
|
+
if (actual !== void 0) lines.push("", `${pad}Actual`, ...this.capped(actual, indent + 2));
|
|
3760
|
+
if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
|
|
3659
3761
|
return lines;
|
|
3660
3762
|
}
|
|
3763
|
+
static actual(entry, attempt) {
|
|
3764
|
+
const turns = attempt?.execution.turns ?? [];
|
|
3765
|
+
if (entry.kind === TestKind.Conversation) return turns.length === 0 ? void 0 : turns.map((turn, index) => `Turn ${index + 1}: ${turn.response}`).join("\n");
|
|
3766
|
+
return turns[entry.turn - 1]?.response;
|
|
3767
|
+
}
|
|
3768
|
+
capped(text, indent) {
|
|
3769
|
+
const lines = text.trim() === "" ? [`${" ".repeat(indent)}(empty)`] : this.paragraph(text, indent);
|
|
3770
|
+
if (lines.length <= 12) return lines;
|
|
3771
|
+
const more = lines.length - 12;
|
|
3772
|
+
return [...lines.slice(0, 12), `${" ".repeat(indent)}… ${more} more ${more === 1 ? "line" : "lines"} in the JSON report`];
|
|
3773
|
+
}
|
|
3774
|
+
static expected(spec, entry) {
|
|
3775
|
+
if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
|
|
3776
|
+
return spec.turns[entry.turn - 1]?.expect.response?.criteria;
|
|
3777
|
+
}
|
|
3778
|
+
paragraph(text, indent) {
|
|
3779
|
+
const pad = " ".repeat(indent);
|
|
3780
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
|
|
3781
|
+
}
|
|
3782
|
+
bullet(text, indent) {
|
|
3783
|
+
const pad = " ".repeat(indent);
|
|
3784
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
|
|
3785
|
+
}
|
|
3661
3786
|
static testTitle(test) {
|
|
3662
3787
|
return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
|
|
3663
3788
|
}
|
|
3664
3789
|
static tally(statuses) {
|
|
3665
3790
|
const count = (status) => statuses.filter((entry) => entry === status).length;
|
|
3666
|
-
const
|
|
3667
|
-
|
|
3668
|
-
|
|
3669
|
-
|
|
3670
|
-
|
|
3671
|
-
|
|
3672
|
-
|
|
3673
|
-
|
|
3674
|
-
|
|
3791
|
+
const parts = [
|
|
3792
|
+
[ResultStatus.Pass, "passed"],
|
|
3793
|
+
[ResultStatus.Flaky, "flaky"],
|
|
3794
|
+
[ResultStatus.Fail, "failed"],
|
|
3795
|
+
[ResultStatus.Error, "errors"],
|
|
3796
|
+
[ResultStatus.Interrupted, "interrupted"]
|
|
3797
|
+
].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
|
|
3798
|
+
return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
|
|
3799
|
+
}
|
|
3800
|
+
mark(status) {
|
|
3801
|
+
const { symbol, colour } = MARKS[status];
|
|
3802
|
+
return this.colour ? (0, node_util.styleText)(colour, symbol, { validateStream: false }) : symbol;
|
|
3675
3803
|
}
|
|
3676
3804
|
static usage(label, usage, capability) {
|
|
3677
|
-
|
|
3678
|
-
if (
|
|
3679
|
-
if (usage.
|
|
3805
|
+
const name = label.padEnd(16);
|
|
3806
|
+
if (!capability) return `${name} n/a (not supported by host)`;
|
|
3807
|
+
if (usage.expected === 0) return `${name} none`;
|
|
3808
|
+
if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
|
|
3680
3809
|
const number = (value) => value.toLocaleString("en-US");
|
|
3681
|
-
const total = `${
|
|
3810
|
+
const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
|
|
3682
3811
|
return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
|
|
3683
3812
|
}
|
|
3684
3813
|
formatAttempt(spec, attempt) {
|
|
@@ -3690,28 +3819,28 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3690
3819
|
lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
|
|
3691
3820
|
lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
|
|
3692
3821
|
});
|
|
3693
|
-
if (execution.failure !== void 0) lines.push(`
|
|
3822
|
+
if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
|
|
3694
3823
|
const notRun = attempt.tests.filter((test) => test.turn > ran);
|
|
3695
|
-
if (notRun.length > 0) lines.push("
|
|
3824
|
+
if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
|
|
3696
3825
|
lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
|
|
3697
|
-
for (const error of execution.attemptErrors) lines.push(`
|
|
3698
|
-
if (execution.closeError !== void 0) lines.push(`
|
|
3699
|
-
lines.push(`
|
|
3826
|
+
for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
|
|
3827
|
+
if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
|
|
3828
|
+
lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
|
|
3700
3829
|
return lines;
|
|
3701
3830
|
}
|
|
3702
3831
|
formatTest(test) {
|
|
3703
|
-
return [`
|
|
3832
|
+
return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
|
|
3704
3833
|
}
|
|
3705
3834
|
static formatTurn(number, input, turn) {
|
|
3706
3835
|
const lines = [
|
|
3707
|
-
`
|
|
3708
|
-
`
|
|
3709
|
-
`
|
|
3836
|
+
` Turn ${number} (${turn.durationMs} ms)`,
|
|
3837
|
+
` Input: ${ConsoleReporter.indent(input, 22)}`,
|
|
3838
|
+
` Response: ${ConsoleReporter.indent(turn.response, 22)}`
|
|
3710
3839
|
];
|
|
3711
|
-
for (const use of turn.toolUses ?? []) lines.push(`
|
|
3712
|
-
for (const result of turn.toolResults ?? []) lines.push(`
|
|
3713
|
-
for (const status of turn.statuses) lines.push(`
|
|
3714
|
-
if (turn.tokenUsage !== void 0) lines.push(`
|
|
3840
|
+
for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
|
|
3841
|
+
for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
|
|
3842
|
+
for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
|
|
3843
|
+
if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
|
|
3715
3844
|
return lines;
|
|
3716
3845
|
}
|
|
3717
3846
|
static indent(text, width) {
|
|
@@ -3741,10 +3870,12 @@ const JSON_REPORT_VERSION = 1;
|
|
|
3741
3870
|
var JsonReporter = class JsonReporter {
|
|
3742
3871
|
redactor;
|
|
3743
3872
|
cwd;
|
|
3873
|
+
displayBase;
|
|
3744
3874
|
tokens = new TokenUsageAggregator();
|
|
3745
|
-
constructor(redactor, cwd) {
|
|
3875
|
+
constructor(redactor, cwd, displayBase = cwd) {
|
|
3746
3876
|
this.redactor = redactor;
|
|
3747
3877
|
this.cwd = cwd;
|
|
3878
|
+
this.displayBase = displayBase;
|
|
3748
3879
|
}
|
|
3749
3880
|
render(run, config, unsuccessfulCount, exitCode) {
|
|
3750
3881
|
const report = {
|
|
@@ -3815,7 +3946,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3815
3946
|
return {
|
|
3816
3947
|
name: result.spec.name,
|
|
3817
3948
|
runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
|
|
3818
|
-
file: result.spec.location.file,
|
|
3949
|
+
file: this.file(result.spec.location.file),
|
|
3819
3950
|
status: result.status,
|
|
3820
3951
|
connection: {
|
|
3821
3952
|
endpoint: websocket.endpoint,
|
|
@@ -3825,7 +3956,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3825
3956
|
agentUsage: JsonReporter.usage(result.agentUsage),
|
|
3826
3957
|
evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
|
|
3827
3958
|
tests: result.tests.map((test) => ({
|
|
3828
|
-
...
|
|
3959
|
+
...this.identity(test),
|
|
3829
3960
|
status: test.status,
|
|
3830
3961
|
...JsonReporter.counts(test),
|
|
3831
3962
|
history: test.history.map((entry, index) => ({
|
|
@@ -3850,7 +3981,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3850
3981
|
agentUsage: JsonReporter.usage(this.tokens.agent([attempt], capability)),
|
|
3851
3982
|
evaluatorUsage: JsonReporter.usage(this.tokens.evaluator([attempt])),
|
|
3852
3983
|
turns: execution.turns.map((turn, index) => JsonReporter.turn(index + 1, spec.turns[index].input, turn)),
|
|
3853
|
-
tests: attempt.tests.map((test) =>
|
|
3984
|
+
tests: attempt.tests.map((test) => this.test(test, config))
|
|
3854
3985
|
};
|
|
3855
3986
|
}
|
|
3856
3987
|
static turn(number, input, turn) {
|
|
@@ -3875,11 +4006,11 @@ var JsonReporter = class JsonReporter {
|
|
|
3875
4006
|
tokenUsage: turn.tokenUsage === void 0 ? void 0 : JsonReporter.tokenUsage(turn.tokenUsage)
|
|
3876
4007
|
};
|
|
3877
4008
|
}
|
|
3878
|
-
|
|
4009
|
+
test(test, config) {
|
|
3879
4010
|
const thresholds = config.evaluator.thresholds;
|
|
3880
4011
|
const evaluation = test.evaluation;
|
|
3881
4012
|
return {
|
|
3882
|
-
...
|
|
4013
|
+
...this.identity(test),
|
|
3883
4014
|
status: test.status,
|
|
3884
4015
|
details: [...test.details],
|
|
3885
4016
|
evaluation: evaluation === void 0 ? void 0 : {
|
|
@@ -3892,17 +4023,17 @@ var JsonReporter = class JsonReporter {
|
|
|
3892
4023
|
evaluatorUsage: JsonReporter.usage(test.evaluatorUsage)
|
|
3893
4024
|
};
|
|
3894
4025
|
}
|
|
3895
|
-
|
|
4026
|
+
identity(test) {
|
|
3896
4027
|
return {
|
|
3897
4028
|
kind: test.kind,
|
|
3898
4029
|
turn: test.turn,
|
|
3899
4030
|
title: test.title,
|
|
3900
|
-
location:
|
|
4031
|
+
location: this.location(test.location)
|
|
3901
4032
|
};
|
|
3902
4033
|
}
|
|
3903
|
-
|
|
4034
|
+
location(location) {
|
|
3904
4035
|
return {
|
|
3905
|
-
file: location.file,
|
|
4036
|
+
file: this.file(location.file),
|
|
3906
4037
|
line: location.line,
|
|
3907
4038
|
column: location.column,
|
|
3908
4039
|
path: location.path
|
|
@@ -3949,6 +4080,9 @@ var JsonReporter = class JsonReporter {
|
|
|
3949
4080
|
path(path) {
|
|
3950
4081
|
return (0, node_path.relative)(this.cwd, path) || ".";
|
|
3951
4082
|
}
|
|
4083
|
+
file(shown) {
|
|
4084
|
+
return shown === "" ? shown : this.path((0, node_path.resolve)(this.displayBase, shown));
|
|
4085
|
+
}
|
|
3952
4086
|
};
|
|
3953
4087
|
// src/reporting/reportFileWriter.ts
|
|
3954
4088
|
var ReportFileWriter = class {
|