@mappies/miniwat-eval 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -299,13 +299,51 @@ A turn stops its attempt when the host reports an error (`statusCode >= 400` or
299
299
 
300
300
  ### Reports
301
301
 
302
- The console prints one line per scenario as it finishes, in file order. A Problems section follows for each scenario that did not pass. It lists attempts that stopped and why, then each test that did not pass with its location, its counts across attempts and the details of each failing attempt, including evaluator scores and explanations. The report ends with:
302
+ The console prints one line per scenario as it finishes, in file order: a status symbol, the name, the attempts that passed out of `repeat`, and the time it took.
303
303
 
304
- - scenario and test counts;
305
- - agent and evaluator token usage with coverage, such as `coverage 8/9, partial`, when not every call reported usage;
306
- - the duration and the exit code.
304
+ ```text
305
+ ✔ Greeting [user-token] 1/1 2.6s
306
+ ✖ Password reset 0/1 7.9s
307
+ ```
308
+
309
+ | Symbol | Status |
310
+ |---|---|
311
+ | `✔` | `PASS` |
312
+ | `✖` | `FAIL` |
313
+ | `⚠` | `FLAKY` |
314
+ | `●` | `ERROR` |
315
+ | `○` | `SKIPPED` (a test that did not run) |
316
+ | `◌` | `INTERRUPTED`, or a scenario that never started |
317
+
318
+ A `FAILURES` section follows, with one block per test that did not pass and per attempt that stopped:
319
+
320
+ ```text
321
+ test at scenarios/authenticated/knowledge-base/password-reset.yml:11:9
322
+ ✖ Password reset › turn 1: response
323
+ Score 0.00, threshold 0.85
324
+
325
+ Expected
326
+ Explains how to reset the password using the steps from the knowledge
327
+ base article it found, without adding steps the article does not support.
328
+
329
+ Explanation
330
+ The response does not explain any password-reset steps from a
331
+ knowledge-base article.
332
+
333
+ Evidence
334
+ • Actual response: "I'm afraid the available documentation does not..."
335
+
336
+ Attempts 1: 0 passed, 1 failed, 0 errors (1 required)
337
+ ```
338
+
339
+ - **Semantic checks** show the score and threshold, the `should` text, the evaluator's explanation and its evidence.
340
+ - **Other checks** list their details, such as the closest tool use and how its arguments differ.
341
+ - **Repeated runs:** with `repeat` above 1, each failing attempt has its own entry.
342
+ - **Flaky tests** are listed in a `FLAKY` section after it.
343
+ - **Summary:** the report ends with a `SUMMARY` of scenario and test counts, agent and evaluator token usage with coverage (such as `coverage 8/9, partial`, when not every call reported usage), the duration and the exit code.
344
+ - **Wrapping:** text wraps at the terminal's width, or at 100 columns when output is redirected.
307
345
 
308
- `--verbose` adds every attempt's transcript. Status words are coloured on a terminal unless `NO_COLOR` is set.
346
+ `--verbose` adds every attempt's transcript under its scenario line. Symbols are coloured on a terminal unless `NO_COLOR` is set; the symbol is shown either way. The console format changed in 1.4.0; scripts should read the JSON report, not the console.
309
347
 
310
348
  `json` writes `miniwat-eval-report.json` to `report.directory`, replacing the previous one, and prints `Report: <path>` above the final line. It follows `schemas/report.schema.json` and holds:
311
349
 
package/dist/index.cjs CHANGED
@@ -3557,64 +3557,94 @@ var ExitCodeCalculator = class {
3557
3557
  }
3558
3558
  };
3559
3559
  // src/reporting/consoleReporter.ts
3560
- const STATUS_WIDTH = 11;
3561
- const COLOURS = {
3562
- [ResultStatus.Pass]: "green",
3563
- [ResultStatus.Flaky]: "yellow",
3564
- [ResultStatus.Fail]: "red",
3565
- [ResultStatus.Error]: "red",
3566
- [ResultStatus.Skipped]: "dim",
3567
- [ResultStatus.Interrupted]: "dim"
3560
+ const MARKS = {
3561
+ [ResultStatus.Pass]: {
3562
+ symbol: "✔",
3563
+ colour: "green"
3564
+ },
3565
+ [ResultStatus.Flaky]: {
3566
+ symbol: "⚠",
3567
+ colour: "yellow"
3568
+ },
3569
+ [ResultStatus.Fail]: {
3570
+ symbol: "✖",
3571
+ colour: "red"
3572
+ },
3573
+ [ResultStatus.Error]: {
3574
+ symbol: "●",
3575
+ colour: "red"
3576
+ },
3577
+ [ResultStatus.Skipped]: {
3578
+ symbol: "○",
3579
+ colour: "dim"
3580
+ },
3581
+ [ResultStatus.Interrupted]: {
3582
+ symbol: "◌",
3583
+ colour: "dim"
3584
+ }
3568
3585
  };
3586
+ const MIN_COLUMNS = 40;
3587
+ const RULE_WIDTH = 72;
3569
3588
  var ConsoleReporter = class ConsoleReporter {
3570
3589
  verbose;
3571
3590
  colour;
3572
- constructor(verbose, colour) {
3591
+ nameWidth;
3592
+ columns;
3593
+ thresholds;
3594
+ constructor(verbose, colour, options = {}) {
3573
3595
  this.verbose = verbose;
3574
3596
  this.colour = colour;
3597
+ this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
3598
+ this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
3599
+ this.thresholds = options.thresholds;
3600
+ }
3601
+ static nameWidth(specs) {
3602
+ return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
3575
3603
  }
3576
3604
  header(run) {
3577
3605
  return [
3578
- "Miniwat Eval",
3606
+ " Miniwat Eval",
3579
3607
  "",
3580
- ...run.quick ? ["Execution Mode: Quick"] : [],
3581
- `Repeat: ${run.repeat}`,
3582
- `Minimum Pass: ${run.minPass}`,
3608
+ ...run.quick ? [" Execution mode Quick"] : [],
3609
+ ` Repeat ${run.repeat}`,
3610
+ ` Minimum pass ${run.minPass}`,
3583
3611
  "",
3584
- `Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
3612
+ ` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
3585
3613
  "",
3586
3614
  ""
3587
3615
  ].join("\n");
3588
3616
  }
3589
3617
  scenario(result, repeat, tokenUsageCapability) {
3590
3618
  const spec = result.spec;
3591
- const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
3592
- const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
3619
+ const title = ConsoleReporter.title(spec);
3620
+ const started = result.attempts.length > 0;
3621
+ const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
3622
+ const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
3623
+ const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
3593
3624
  if (!this.verbose) return line;
3594
- if (result.attempts.length === 0) return `${line}\n`;
3625
+ if (!started) return `${line}\n`;
3595
3626
  const websocket = spec.websocket;
3596
3627
  const subprotocols = websocket?.connection.subprotocols.length ?? 0;
3597
- const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
3598
- for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3599
- lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
3600
- lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
3601
- lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
3602
- lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3628
+ const lines = [];
3629
+ if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
3630
+ for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3631
+ lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
3632
+ lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3603
3633
  return line + lines.join("\n");
3604
3634
  }
3605
3635
  static title(spec) {
3606
3636
  return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
3607
3637
  }
3608
3638
  summary(run, exitCode, reportPaths = []) {
3609
- const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
3639
+ const started = run.scenarios.filter((result) => result.attempts.length > 0);
3640
+ const failures = started.flatMap((result) => this.blocks(result, false));
3641
+ const flaky = started.flatMap((result) => this.blocks(result, true));
3610
3642
  const lines = this.verbose ? [] : [""];
3611
- if (problems.length > 0) {
3612
- lines.push("Problems", "");
3613
- for (const result of problems) lines.push(...this.problems(result), "");
3614
- }
3643
+ if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
3644
+ if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
3615
3645
  const tests = run.scenarios.flatMap((result) => result.tests);
3616
- lines.push(...run.status === RunStatus.Interrupted ? [`Run: ${run.status}`] : [], `Scenarios: ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, `Tests: ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability), ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true), `Duration: ${ConsoleReporter.duration(run.durationMs)}`, "", "");
3617
- return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
3646
+ lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
3647
+ return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
3618
3648
  }
3619
3649
  static footer(run, exitCode, reportPaths) {
3620
3650
  return [
@@ -3628,57 +3658,135 @@ var ConsoleReporter = class ConsoleReporter {
3628
3658
  return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
3629
3659
  }
3630
3660
  static duration(ms) {
3661
+ if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
3631
3662
  const seconds = Math.floor(ms / 1e3);
3632
3663
  const pad = (value) => String(value).padStart(2, "0");
3633
3664
  const hours = Math.floor(seconds / 3600);
3634
- const rest = `${pad(Math.floor(seconds / 60) % 60)}:${pad(seconds % 60)}`;
3635
- return hours > 0 ? `${hours}:${rest}` : rest;
3665
+ const minutes = Math.floor(seconds / 60) % 60;
3666
+ return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
3667
+ }
3668
+ static wrap(text, width) {
3669
+ const lines = [];
3670
+ for (const paragraph of text.split("\n")) {
3671
+ let line = "";
3672
+ for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
3673
+ lines.push(line);
3674
+ line = word;
3675
+ } else line = line === "" ? word : `${line} ${word}`;
3676
+ lines.push(line);
3677
+ }
3678
+ return lines;
3679
+ }
3680
+ section(name, blocks) {
3681
+ return [
3682
+ ...this.rule(name),
3683
+ "",
3684
+ ...blocks.flatMap((block) => [...block, ""])
3685
+ ];
3636
3686
  }
3637
- problems(result) {
3687
+ rule(name) {
3688
+ const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
3689
+ return [
3690
+ rule,
3691
+ ` ${name}`,
3692
+ rule
3693
+ ];
3694
+ }
3695
+ blocks(result, flaky) {
3638
3696
  const spec = result.spec;
3639
- const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
3640
- for (const attempt of result.attempts) {
3641
- const execution = attempt.execution;
3642
- if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3643
- for (const error of execution.attemptErrors) lines.push(` Attempt ${attempt.attempt} error: ${error.category} error: ${error.message}`);
3644
- if (execution.closeError !== void 0) lines.push(` Attempt ${attempt.attempt} close error: ${execution.closeError.message}`);
3645
- }
3646
- for (const test of result.tests.filter((entry) => entry.status !== ResultStatus.Pass)) {
3647
- lines.push(` ${this.status(test.status, 6)} ${ConsoleReporter.testTitle(test)} ${test.location.toString()} ${ConsoleReporter.counts(test)}`);
3648
- test.history.forEach((entry, index) => {
3649
- if (entry.status === ResultStatus.Pass) return;
3650
- const prefix = ` Attempt ${result.attempts[index].attempt} ${this.status(entry.status)}`;
3651
- const width = ` Attempt ${result.attempts[index].attempt} ${entry.status}: `.length;
3652
- if (entry.details.length === 0) {
3653
- lines.push(prefix);
3654
- return;
3655
- }
3656
- lines.push(`${prefix}: ${ConsoleReporter.indent(entry.details[0], width)}`, ...entry.details.slice(1).map((detail) => `${" ".repeat(width)}${ConsoleReporter.indent(detail, width)}`));
3657
- });
3697
+ const title = ConsoleReporter.title(spec);
3698
+ const blocks = [];
3699
+ if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
3700
+ const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
3701
+ for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
3702
+ return blocks;
3703
+ }
3704
+ attemptBlock(spec, title, attempt) {
3705
+ const execution = attempt.execution;
3706
+ const problems = [];
3707
+ if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
3708
+ for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
3709
+ if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
3710
+ if (problems.length === 0) return [];
3711
+ const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
3712
+ const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
3713
+ return [[
3714
+ ` scenario at ${spec.location.file}`,
3715
+ ` ${this.mark(status)} ${title} › ${what}`,
3716
+ ...problems.flatMap((problem) => this.paragraph(problem, 3))
3717
+ ]];
3718
+ }
3719
+ testBlock(spec, title, test, result) {
3720
+ const location = test.location;
3721
+ const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
3722
+ const failing = test.history.map((entry, index) => ({
3723
+ entry,
3724
+ attempt: result.attempts[index]?.attempt ?? index + 1
3725
+ })).filter(({ entry }) => entry.status !== ResultStatus.Pass);
3726
+ const several = test.history.length > 1;
3727
+ for (const { entry, attempt } of failing) {
3728
+ if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
3729
+ lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
3658
3730
  }
3731
+ lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
3659
3732
  return lines;
3660
3733
  }
3734
+ entryBody(spec, entry, indent) {
3735
+ const pad = " ".repeat(indent);
3736
+ const evaluation = entry.evaluation;
3737
+ if (evaluation === void 0) {
3738
+ if (entry.details.length === 0) return [];
3739
+ return [
3740
+ "",
3741
+ `${pad}Details`,
3742
+ ...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
3743
+ ];
3744
+ }
3745
+ const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
3746
+ const expected = ConsoleReporter.expected(spec, entry);
3747
+ const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
3748
+ if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
3749
+ if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
3750
+ if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
3751
+ return lines;
3752
+ }
3753
+ static expected(spec, entry) {
3754
+ if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
3755
+ return spec.turns[entry.turn - 1]?.expect.response?.criteria;
3756
+ }
3757
+ paragraph(text, indent) {
3758
+ const pad = " ".repeat(indent);
3759
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
3760
+ }
3761
+ bullet(text, indent) {
3762
+ const pad = " ".repeat(indent);
3763
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
3764
+ }
3661
3765
  static testTitle(test) {
3662
3766
  return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
3663
3767
  }
3664
3768
  static tally(statuses) {
3665
3769
  const count = (status) => statuses.filter((entry) => entry === status).length;
3666
- const interrupted = count(ResultStatus.Interrupted);
3667
- return `${count(ResultStatus.Pass)} passed, ${count(ResultStatus.Flaky)} flaky, ${count(ResultStatus.Fail)} failed, ${count(ResultStatus.Error)} errors${interrupted > 0 ? `, ${interrupted} interrupted` : ""}`;
3668
- }
3669
- status(status, width = 0) {
3670
- const padding = " ".repeat(Math.max(0, width - status.length));
3671
- return (this.colour ? (0, node_util.styleText)(COLOURS[status], status, { validateStream: false }) : status) + padding;
3672
- }
3673
- static counts(result) {
3674
- return `(${result.passed} passed, ${result.failed} failed, ${result.errors} errors; required ${result.required})`;
3770
+ const parts = [
3771
+ [ResultStatus.Pass, "passed"],
3772
+ [ResultStatus.Flaky, "flaky"],
3773
+ [ResultStatus.Fail, "failed"],
3774
+ [ResultStatus.Error, "errors"],
3775
+ [ResultStatus.Interrupted, "interrupted"]
3776
+ ].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
3777
+ return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
3778
+ }
3779
+ mark(status) {
3780
+ const { symbol, colour } = MARKS[status];
3781
+ return this.colour ? (0, node_util.styleText)(colour, symbol, { validateStream: false }) : symbol;
3675
3782
  }
3676
3783
  static usage(label, usage, capability) {
3677
- if (!capability) return `${label}: n/a (not supported by host)`;
3678
- if (usage.expected === 0) return `${label}: none`;
3679
- if (usage.total === void 0) return `${label}: n/a (coverage 0/${usage.expected})`;
3784
+ const name = label.padEnd(16);
3785
+ if (!capability) return `${name} n/a (not supported by host)`;
3786
+ if (usage.expected === 0) return `${name} none`;
3787
+ if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
3680
3788
  const number = (value) => value.toLocaleString("en-US");
3681
- const total = `${label}: total ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3789
+ const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3682
3790
  return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
3683
3791
  }
3684
3792
  formatAttempt(spec, attempt) {
@@ -3690,28 +3798,28 @@ var ConsoleReporter = class ConsoleReporter {
3690
3798
  lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
3691
3799
  lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
3692
3800
  });
3693
- if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3801
+ if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3694
3802
  const notRun = attempt.tests.filter((test) => test.turn > ran);
3695
- if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3803
+ if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3696
3804
  lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
3697
- for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3698
- if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3699
- lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3805
+ for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3806
+ if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3807
+ lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3700
3808
  return lines;
3701
3809
  }
3702
3810
  formatTest(test) {
3703
- return [` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${detail}`)];
3811
+ return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
3704
3812
  }
3705
3813
  static formatTurn(number, input, turn) {
3706
3814
  const lines = [
3707
- ` Turn ${number} (${turn.durationMs} ms)`,
3708
- ` Input: ${ConsoleReporter.indent(input, 17)}`,
3709
- ` Response: ${ConsoleReporter.indent(turn.response, 17)}`
3815
+ ` Turn ${number} (${turn.durationMs} ms)`,
3816
+ ` Input: ${ConsoleReporter.indent(input, 22)}`,
3817
+ ` Response: ${ConsoleReporter.indent(turn.response, 22)}`
3710
3818
  ];
3711
- for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3712
- for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3713
- for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 17)}`);
3714
- if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3819
+ for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3820
+ for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3821
+ for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
3822
+ if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3715
3823
  return lines;
3716
3824
  }
3717
3825
  static indent(text, width) {