@mappies/miniwat-eval 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -5
- package/dist/index.cjs +186 -78
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +24 -4
- package/dist/index.d.ts +24 -4
- package/dist/index.js +187 -79
- package/dist/index.js.map +1 -1
- package/dist/main.js +196 -82
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -299,13 +299,51 @@ A turn stops its attempt when the host reports an error (`statusCode >= 400` or
|
|
|
299
299
|
|
|
300
300
|
### Reports
|
|
301
301
|
|
|
302
|
-
The console prints one line per scenario as it finishes, in file order
|
|
302
|
+
The console prints one line per scenario as it finishes, in file order: a status symbol, the name, the attempts that passed out of `repeat`, and the time it took.
|
|
303
303
|
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
304
|
+
```text
|
|
305
|
+
✔ Greeting [user-token] 1/1 2.6s
|
|
306
|
+
✖ Password reset 0/1 7.9s
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
| Symbol | Status |
|
|
310
|
+
|---|---|
|
|
311
|
+
| `✔` | `PASS` |
|
|
312
|
+
| `✖` | `FAIL` |
|
|
313
|
+
| `⚠` | `FLAKY` |
|
|
314
|
+
| `●` | `ERROR` |
|
|
315
|
+
| `○` | `SKIPPED` (a test that did not run) |
|
|
316
|
+
| `◌` | `INTERRUPTED`, or a scenario that never started |
|
|
317
|
+
|
|
318
|
+
A `FAILURES` section follows, with one block per test that did not pass and per attempt that stopped:
|
|
319
|
+
|
|
320
|
+
```text
|
|
321
|
+
test at scenarios/authenticated/knowledge-base/password-reset.yml:11:9
|
|
322
|
+
✖ Password reset › turn 1: response
|
|
323
|
+
Score 0.00, threshold 0.85
|
|
324
|
+
|
|
325
|
+
Expected
|
|
326
|
+
Explains how to reset the password using the steps from the knowledge
|
|
327
|
+
base article it found, without adding steps the article does not support.
|
|
328
|
+
|
|
329
|
+
Explanation
|
|
330
|
+
The response does not explain any password-reset steps from a
|
|
331
|
+
knowledge-base article.
|
|
332
|
+
|
|
333
|
+
Evidence
|
|
334
|
+
• Actual response: "I'm afraid the available documentation does not..."
|
|
335
|
+
|
|
336
|
+
Attempts 1: 0 passed, 1 failed, 0 errors (1 required)
|
|
337
|
+
```
|
|
338
|
+
|
|
339
|
+
- **Semantic checks** show the score and threshold, the `should` text, the evaluator's explanation and its evidence.
|
|
340
|
+
- **Other checks** list their details, such as the closest tool use and how its arguments differ.
|
|
341
|
+
- **Repeated runs:** with `repeat` above 1, each failing attempt has its own entry.
|
|
342
|
+
- **Flaky tests** are listed in a `FLAKY` section after it.
|
|
343
|
+
- **Summary:** the report ends with a `SUMMARY` of scenario and test counts, agent and evaluator token usage with coverage (such as `coverage 8/9, partial`, when not every call reported usage), the duration and the exit code.
|
|
344
|
+
- **Wrapping:** text wraps at the terminal's width, or at 100 columns when output is redirected.
|
|
307
345
|
|
|
308
|
-
`--verbose` adds every attempt's transcript.
|
|
346
|
+
`--verbose` adds every attempt's transcript under its scenario line. Symbols are coloured on a terminal unless `NO_COLOR` is set; the symbol is shown either way. The console format changed in 1.4.0; scripts should read the JSON report, not the console.
|
|
309
347
|
|
|
310
348
|
`json` writes `miniwat-eval-report.json` to `report.directory`, replacing the previous one, and prints `Report: <path>` above the final line. It follows `schemas/report.schema.json` and holds:
|
|
311
349
|
|
package/dist/index.cjs
CHANGED
|
@@ -3557,64 +3557,94 @@ var ExitCodeCalculator = class {
|
|
|
3557
3557
|
}
|
|
3558
3558
|
};
|
|
3559
3559
|
// src/reporting/consoleReporter.ts
|
|
3560
|
-
const
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
[ResultStatus.
|
|
3566
|
-
|
|
3567
|
-
|
|
3560
|
+
const MARKS = {
|
|
3561
|
+
[ResultStatus.Pass]: {
|
|
3562
|
+
symbol: "✔",
|
|
3563
|
+
colour: "green"
|
|
3564
|
+
},
|
|
3565
|
+
[ResultStatus.Flaky]: {
|
|
3566
|
+
symbol: "⚠",
|
|
3567
|
+
colour: "yellow"
|
|
3568
|
+
},
|
|
3569
|
+
[ResultStatus.Fail]: {
|
|
3570
|
+
symbol: "✖",
|
|
3571
|
+
colour: "red"
|
|
3572
|
+
},
|
|
3573
|
+
[ResultStatus.Error]: {
|
|
3574
|
+
symbol: "●",
|
|
3575
|
+
colour: "red"
|
|
3576
|
+
},
|
|
3577
|
+
[ResultStatus.Skipped]: {
|
|
3578
|
+
symbol: "○",
|
|
3579
|
+
colour: "dim"
|
|
3580
|
+
},
|
|
3581
|
+
[ResultStatus.Interrupted]: {
|
|
3582
|
+
symbol: "◌",
|
|
3583
|
+
colour: "dim"
|
|
3584
|
+
}
|
|
3568
3585
|
};
|
|
3586
|
+
const MIN_COLUMNS = 40;
|
|
3587
|
+
const RULE_WIDTH = 72;
|
|
3569
3588
|
var ConsoleReporter = class ConsoleReporter {
|
|
3570
3589
|
verbose;
|
|
3571
3590
|
colour;
|
|
3572
|
-
|
|
3591
|
+
nameWidth;
|
|
3592
|
+
columns;
|
|
3593
|
+
thresholds;
|
|
3594
|
+
constructor(verbose, colour, options = {}) {
|
|
3573
3595
|
this.verbose = verbose;
|
|
3574
3596
|
this.colour = colour;
|
|
3597
|
+
this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
|
|
3598
|
+
this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
|
|
3599
|
+
this.thresholds = options.thresholds;
|
|
3600
|
+
}
|
|
3601
|
+
static nameWidth(specs) {
|
|
3602
|
+
return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
|
|
3575
3603
|
}
|
|
3576
3604
|
header(run) {
|
|
3577
3605
|
return [
|
|
3578
|
-
"Miniwat Eval",
|
|
3606
|
+
" Miniwat Eval",
|
|
3579
3607
|
"",
|
|
3580
|
-
...run.quick ? ["Execution
|
|
3581
|
-
`Repeat
|
|
3582
|
-
`Minimum
|
|
3608
|
+
...run.quick ? [" Execution mode Quick"] : [],
|
|
3609
|
+
` Repeat ${run.repeat}`,
|
|
3610
|
+
` Minimum pass ${run.minPass}`,
|
|
3583
3611
|
"",
|
|
3584
|
-
`Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
3612
|
+
` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
3585
3613
|
"",
|
|
3586
3614
|
""
|
|
3587
3615
|
].join("\n");
|
|
3588
3616
|
}
|
|
3589
3617
|
scenario(result, repeat, tokenUsageCapability) {
|
|
3590
3618
|
const spec = result.spec;
|
|
3591
|
-
const
|
|
3592
|
-
const
|
|
3619
|
+
const title = ConsoleReporter.title(spec);
|
|
3620
|
+
const started = result.attempts.length > 0;
|
|
3621
|
+
const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
|
|
3622
|
+
const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
|
|
3623
|
+
const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
|
|
3593
3624
|
if (!this.verbose) return line;
|
|
3594
|
-
if (
|
|
3625
|
+
if (!started) return `${line}\n`;
|
|
3595
3626
|
const websocket = spec.websocket;
|
|
3596
3627
|
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
3597
|
-
const lines = [
|
|
3598
|
-
|
|
3599
|
-
lines.push(`
|
|
3600
|
-
lines.push(
|
|
3601
|
-
lines.push(`
|
|
3602
|
-
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3628
|
+
const lines = [];
|
|
3629
|
+
if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
|
|
3630
|
+
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
3631
|
+
lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
|
|
3632
|
+
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3603
3633
|
return line + lines.join("\n");
|
|
3604
3634
|
}
|
|
3605
3635
|
static title(spec) {
|
|
3606
3636
|
return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
|
|
3607
3637
|
}
|
|
3608
3638
|
summary(run, exitCode, reportPaths = []) {
|
|
3609
|
-
const
|
|
3639
|
+
const started = run.scenarios.filter((result) => result.attempts.length > 0);
|
|
3640
|
+
const failures = started.flatMap((result) => this.blocks(result, false));
|
|
3641
|
+
const flaky = started.flatMap((result) => this.blocks(result, true));
|
|
3610
3642
|
const lines = this.verbose ? [] : [""];
|
|
3611
|
-
if (
|
|
3612
|
-
|
|
3613
|
-
for (const result of problems) lines.push(...this.problems(result), "");
|
|
3614
|
-
}
|
|
3643
|
+
if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
|
|
3644
|
+
if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
|
|
3615
3645
|
const tests = run.scenarios.flatMap((result) => result.tests);
|
|
3616
|
-
lines.push(...run.status === RunStatus.Interrupted ? [`Run
|
|
3617
|
-
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
|
|
3646
|
+
lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
|
|
3647
|
+
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
|
|
3618
3648
|
}
|
|
3619
3649
|
static footer(run, exitCode, reportPaths) {
|
|
3620
3650
|
return [
|
|
@@ -3628,57 +3658,135 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3628
3658
|
return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
|
|
3629
3659
|
}
|
|
3630
3660
|
static duration(ms) {
|
|
3661
|
+
if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
|
|
3631
3662
|
const seconds = Math.floor(ms / 1e3);
|
|
3632
3663
|
const pad = (value) => String(value).padStart(2, "0");
|
|
3633
3664
|
const hours = Math.floor(seconds / 3600);
|
|
3634
|
-
const
|
|
3635
|
-
return hours > 0 ? `${hours}:${
|
|
3665
|
+
const minutes = Math.floor(seconds / 60) % 60;
|
|
3666
|
+
return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
|
|
3667
|
+
}
|
|
3668
|
+
static wrap(text, width) {
|
|
3669
|
+
const lines = [];
|
|
3670
|
+
for (const paragraph of text.split("\n")) {
|
|
3671
|
+
let line = "";
|
|
3672
|
+
for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
|
|
3673
|
+
lines.push(line);
|
|
3674
|
+
line = word;
|
|
3675
|
+
} else line = line === "" ? word : `${line} ${word}`;
|
|
3676
|
+
lines.push(line);
|
|
3677
|
+
}
|
|
3678
|
+
return lines;
|
|
3679
|
+
}
|
|
3680
|
+
section(name, blocks) {
|
|
3681
|
+
return [
|
|
3682
|
+
...this.rule(name),
|
|
3683
|
+
"",
|
|
3684
|
+
...blocks.flatMap((block) => [...block, ""])
|
|
3685
|
+
];
|
|
3636
3686
|
}
|
|
3637
|
-
|
|
3687
|
+
rule(name) {
|
|
3688
|
+
const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
|
|
3689
|
+
return [
|
|
3690
|
+
rule,
|
|
3691
|
+
` ${name}`,
|
|
3692
|
+
rule
|
|
3693
|
+
];
|
|
3694
|
+
}
|
|
3695
|
+
blocks(result, flaky) {
|
|
3638
3696
|
const spec = result.spec;
|
|
3639
|
-
const
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
|
|
3653
|
-
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3657
|
-
}
|
|
3697
|
+
const title = ConsoleReporter.title(spec);
|
|
3698
|
+
const blocks = [];
|
|
3699
|
+
if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
|
|
3700
|
+
const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
|
|
3701
|
+
for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
|
|
3702
|
+
return blocks;
|
|
3703
|
+
}
|
|
3704
|
+
attemptBlock(spec, title, attempt) {
|
|
3705
|
+
const execution = attempt.execution;
|
|
3706
|
+
const problems = [];
|
|
3707
|
+
if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
|
|
3708
|
+
for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
|
|
3709
|
+
if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
|
|
3710
|
+
if (problems.length === 0) return [];
|
|
3711
|
+
const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
|
|
3712
|
+
const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
|
|
3713
|
+
return [[
|
|
3714
|
+
` scenario at ${spec.location.file}`,
|
|
3715
|
+
` ${this.mark(status)} ${title} › ${what}`,
|
|
3716
|
+
...problems.flatMap((problem) => this.paragraph(problem, 3))
|
|
3717
|
+
]];
|
|
3718
|
+
}
|
|
3719
|
+
testBlock(spec, title, test, result) {
|
|
3720
|
+
const location = test.location;
|
|
3721
|
+
const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
|
|
3722
|
+
const failing = test.history.map((entry, index) => ({
|
|
3723
|
+
entry,
|
|
3724
|
+
attempt: result.attempts[index]?.attempt ?? index + 1
|
|
3725
|
+
})).filter(({ entry }) => entry.status !== ResultStatus.Pass);
|
|
3726
|
+
const several = test.history.length > 1;
|
|
3727
|
+
for (const { entry, attempt } of failing) {
|
|
3728
|
+
if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
|
|
3729
|
+
lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
|
|
3658
3730
|
}
|
|
3731
|
+
lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
|
|
3659
3732
|
return lines;
|
|
3660
3733
|
}
|
|
3734
|
+
entryBody(spec, entry, indent) {
|
|
3735
|
+
const pad = " ".repeat(indent);
|
|
3736
|
+
const evaluation = entry.evaluation;
|
|
3737
|
+
if (evaluation === void 0) {
|
|
3738
|
+
if (entry.details.length === 0) return [];
|
|
3739
|
+
return [
|
|
3740
|
+
"",
|
|
3741
|
+
`${pad}Details`,
|
|
3742
|
+
...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
|
|
3743
|
+
];
|
|
3744
|
+
}
|
|
3745
|
+
const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
|
|
3746
|
+
const expected = ConsoleReporter.expected(spec, entry);
|
|
3747
|
+
const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
|
|
3748
|
+
if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
|
|
3749
|
+
if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
|
|
3750
|
+
if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
|
|
3751
|
+
return lines;
|
|
3752
|
+
}
|
|
3753
|
+
static expected(spec, entry) {
|
|
3754
|
+
if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
|
|
3755
|
+
return spec.turns[entry.turn - 1]?.expect.response?.criteria;
|
|
3756
|
+
}
|
|
3757
|
+
paragraph(text, indent) {
|
|
3758
|
+
const pad = " ".repeat(indent);
|
|
3759
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
|
|
3760
|
+
}
|
|
3761
|
+
bullet(text, indent) {
|
|
3762
|
+
const pad = " ".repeat(indent);
|
|
3763
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
|
|
3764
|
+
}
|
|
3661
3765
|
static testTitle(test) {
|
|
3662
3766
|
return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
|
|
3663
3767
|
}
|
|
3664
3768
|
static tally(statuses) {
|
|
3665
3769
|
const count = (status) => statuses.filter((entry) => entry === status).length;
|
|
3666
|
-
const
|
|
3667
|
-
|
|
3668
|
-
|
|
3669
|
-
|
|
3670
|
-
|
|
3671
|
-
|
|
3672
|
-
|
|
3673
|
-
|
|
3674
|
-
|
|
3770
|
+
const parts = [
|
|
3771
|
+
[ResultStatus.Pass, "passed"],
|
|
3772
|
+
[ResultStatus.Flaky, "flaky"],
|
|
3773
|
+
[ResultStatus.Fail, "failed"],
|
|
3774
|
+
[ResultStatus.Error, "errors"],
|
|
3775
|
+
[ResultStatus.Interrupted, "interrupted"]
|
|
3776
|
+
].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
|
|
3777
|
+
return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
|
|
3778
|
+
}
|
|
3779
|
+
mark(status) {
|
|
3780
|
+
const { symbol, colour } = MARKS[status];
|
|
3781
|
+
return this.colour ? (0, node_util.styleText)(colour, symbol, { validateStream: false }) : symbol;
|
|
3675
3782
|
}
|
|
3676
3783
|
static usage(label, usage, capability) {
|
|
3677
|
-
|
|
3678
|
-
if (
|
|
3679
|
-
if (usage.
|
|
3784
|
+
const name = label.padEnd(16);
|
|
3785
|
+
if (!capability) return `${name} n/a (not supported by host)`;
|
|
3786
|
+
if (usage.expected === 0) return `${name} none`;
|
|
3787
|
+
if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
|
|
3680
3788
|
const number = (value) => value.toLocaleString("en-US");
|
|
3681
|
-
const total = `${
|
|
3789
|
+
const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
|
|
3682
3790
|
return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
|
|
3683
3791
|
}
|
|
3684
3792
|
formatAttempt(spec, attempt) {
|
|
@@ -3690,28 +3798,28 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3690
3798
|
lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
|
|
3691
3799
|
lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
|
|
3692
3800
|
});
|
|
3693
|
-
if (execution.failure !== void 0) lines.push(`
|
|
3801
|
+
if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
|
|
3694
3802
|
const notRun = attempt.tests.filter((test) => test.turn > ran);
|
|
3695
|
-
if (notRun.length > 0) lines.push("
|
|
3803
|
+
if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
|
|
3696
3804
|
lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
|
|
3697
|
-
for (const error of execution.attemptErrors) lines.push(`
|
|
3698
|
-
if (execution.closeError !== void 0) lines.push(`
|
|
3699
|
-
lines.push(`
|
|
3805
|
+
for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
|
|
3806
|
+
if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
|
|
3807
|
+
lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
|
|
3700
3808
|
return lines;
|
|
3701
3809
|
}
|
|
3702
3810
|
formatTest(test) {
|
|
3703
|
-
return [`
|
|
3811
|
+
return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
|
|
3704
3812
|
}
|
|
3705
3813
|
static formatTurn(number, input, turn) {
|
|
3706
3814
|
const lines = [
|
|
3707
|
-
`
|
|
3708
|
-
`
|
|
3709
|
-
`
|
|
3815
|
+
` Turn ${number} (${turn.durationMs} ms)`,
|
|
3816
|
+
` Input: ${ConsoleReporter.indent(input, 22)}`,
|
|
3817
|
+
` Response: ${ConsoleReporter.indent(turn.response, 22)}`
|
|
3710
3818
|
];
|
|
3711
|
-
for (const use of turn.toolUses ?? []) lines.push(`
|
|
3712
|
-
for (const result of turn.toolResults ?? []) lines.push(`
|
|
3713
|
-
for (const status of turn.statuses) lines.push(`
|
|
3714
|
-
if (turn.tokenUsage !== void 0) lines.push(`
|
|
3819
|
+
for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
|
|
3820
|
+
for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
|
|
3821
|
+
for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
|
|
3822
|
+
if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
|
|
3715
3823
|
return lines;
|
|
3716
3824
|
}
|
|
3717
3825
|
static indent(text, width) {
|