@mappies/miniwat-eval 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -909,10 +909,22 @@ type RunHeader = {
909
909
  scenarios: number;
910
910
  concurrency: number;
911
911
  };
912
+ type ConsoleOptions = {
913
+ nameWidth?: number;
914
+ columns?: number;
915
+ thresholds?: {
916
+ response: number;
917
+ conversation: number;
918
+ };
919
+ };
912
920
  declare class ConsoleReporter {
913
921
  private readonly verbose;
914
922
  private readonly colour;
915
- constructor(verbose: boolean, colour: boolean);
923
+ private readonly nameWidth;
924
+ private readonly columns;
925
+ private readonly thresholds;
926
+ constructor(verbose: boolean, colour: boolean, options?: ConsoleOptions);
927
+ static nameWidth(specs: ConversationSpec[]): number;
916
928
  header(run: RunHeader): string;
917
929
  scenario(result: ScenarioResult, repeat: number, tokenUsageCapability: boolean): string;
918
930
  static title(spec: ConversationSpec): string;
@@ -920,11 +932,19 @@ declare class ConsoleReporter {
920
932
  static footer(run: EvalRunResult, exitCode: number, reportPaths: string[]): string;
921
933
  static finalLine(run: EvalRunResult, exitCode: number): string;
922
934
  static duration(ms: number): string;
923
- private problems;
935
+ static wrap(text: string, width: number): string[];
936
+ private section;
937
+ private rule;
938
+ private blocks;
939
+ private attemptBlock;
940
+ private testBlock;
941
+ private entryBody;
942
+ private static expected;
943
+ private paragraph;
944
+ private bullet;
924
945
  private static testTitle;
925
946
  private static tally;
926
- private status;
927
- private static counts;
947
+ private mark;
928
948
  private static usage;
929
949
  private formatAttempt;
930
950
  private formatTest;
package/dist/index.d.ts CHANGED
@@ -910,10 +910,22 @@ type RunHeader = {
910
910
  scenarios: number;
911
911
  concurrency: number;
912
912
  };
913
+ type ConsoleOptions = {
914
+ nameWidth?: number;
915
+ columns?: number;
916
+ thresholds?: {
917
+ response: number;
918
+ conversation: number;
919
+ };
920
+ };
913
921
  declare class ConsoleReporter {
914
922
  private readonly verbose;
915
923
  private readonly colour;
916
- constructor(verbose: boolean, colour: boolean);
924
+ private readonly nameWidth;
925
+ private readonly columns;
926
+ private readonly thresholds;
927
+ constructor(verbose: boolean, colour: boolean, options?: ConsoleOptions);
928
+ static nameWidth(specs: ConversationSpec[]): number;
917
929
  header(run: RunHeader): string;
918
930
  scenario(result: ScenarioResult, repeat: number, tokenUsageCapability: boolean): string;
919
931
  static title(spec: ConversationSpec): string;
@@ -921,11 +933,19 @@ declare class ConsoleReporter {
921
933
  static footer(run: EvalRunResult, exitCode: number, reportPaths: string[]): string;
922
934
  static finalLine(run: EvalRunResult, exitCode: number): string;
923
935
  static duration(ms: number): string;
924
- private problems;
936
+ static wrap(text: string, width: number): string[];
937
+ private section;
938
+ private rule;
939
+ private blocks;
940
+ private attemptBlock;
941
+ private testBlock;
942
+ private entryBody;
943
+ private static expected;
944
+ private paragraph;
945
+ private bullet;
925
946
  private static testTitle;
926
947
  private static tally;
927
- private status;
928
- private static counts;
948
+ private mark;
929
949
  private static usage;
930
950
  private formatAttempt;
931
951
  private formatTest;
package/dist/index.js CHANGED
@@ -3534,64 +3534,94 @@ var ExitCodeCalculator = class {
3534
3534
  }
3535
3535
  };
3536
3536
  // src/reporting/consoleReporter.ts
3537
- const STATUS_WIDTH = 11;
3538
- const COLOURS = {
3539
- [ResultStatus.Pass]: "green",
3540
- [ResultStatus.Flaky]: "yellow",
3541
- [ResultStatus.Fail]: "red",
3542
- [ResultStatus.Error]: "red",
3543
- [ResultStatus.Skipped]: "dim",
3544
- [ResultStatus.Interrupted]: "dim"
3537
+ const MARKS = {
3538
+ [ResultStatus.Pass]: {
3539
+ symbol: "✔",
3540
+ colour: "green"
3541
+ },
3542
+ [ResultStatus.Flaky]: {
3543
+ symbol: "⚠",
3544
+ colour: "yellow"
3545
+ },
3546
+ [ResultStatus.Fail]: {
3547
+ symbol: "✖",
3548
+ colour: "red"
3549
+ },
3550
+ [ResultStatus.Error]: {
3551
+ symbol: "●",
3552
+ colour: "red"
3553
+ },
3554
+ [ResultStatus.Skipped]: {
3555
+ symbol: "○",
3556
+ colour: "dim"
3557
+ },
3558
+ [ResultStatus.Interrupted]: {
3559
+ symbol: "◌",
3560
+ colour: "dim"
3561
+ }
3545
3562
  };
3563
+ const MIN_COLUMNS = 40;
3564
+ const RULE_WIDTH = 72;
3546
3565
  var ConsoleReporter = class ConsoleReporter {
3547
3566
  verbose;
3548
3567
  colour;
3549
- constructor(verbose, colour) {
3568
+ nameWidth;
3569
+ columns;
3570
+ thresholds;
3571
+ constructor(verbose, colour, options = {}) {
3550
3572
  this.verbose = verbose;
3551
3573
  this.colour = colour;
3574
+ this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
3575
+ this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
3576
+ this.thresholds = options.thresholds;
3577
+ }
3578
+ static nameWidth(specs) {
3579
+ return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
3552
3580
  }
3553
3581
  header(run) {
3554
3582
  return [
3555
- "Miniwat Eval",
3583
+ " Miniwat Eval",
3556
3584
  "",
3557
- ...run.quick ? ["Execution Mode: Quick"] : [],
3558
- `Repeat: ${run.repeat}`,
3559
- `Minimum Pass: ${run.minPass}`,
3585
+ ...run.quick ? [" Execution mode Quick"] : [],
3586
+ ` Repeat ${run.repeat}`,
3587
+ ` Minimum pass ${run.minPass}`,
3560
3588
  "",
3561
- `Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
3589
+ ` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
3562
3590
  "",
3563
3591
  ""
3564
3592
  ].join("\n");
3565
3593
  }
3566
3594
  scenario(result, repeat, tokenUsageCapability) {
3567
3595
  const spec = result.spec;
3568
- const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
3569
- const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
3596
+ const title = ConsoleReporter.title(spec);
3597
+ const started = result.attempts.length > 0;
3598
+ const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
3599
+ const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
3600
+ const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
3570
3601
  if (!this.verbose) return line;
3571
- if (result.attempts.length === 0) return `${line}\n`;
3602
+ if (!started) return `${line}\n`;
3572
3603
  const websocket = spec.websocket;
3573
3604
  const subprotocols = websocket?.connection.subprotocols.length ?? 0;
3574
- const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
3575
- for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3576
- lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
3577
- lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
3578
- lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
3579
- lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3605
+ const lines = [];
3606
+ if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
3607
+ for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3608
+ lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
3609
+ lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3580
3610
  return line + lines.join("\n");
3581
3611
  }
3582
3612
  static title(spec) {
3583
3613
  return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
3584
3614
  }
3585
3615
  summary(run, exitCode, reportPaths = []) {
3586
- const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
3616
+ const started = run.scenarios.filter((result) => result.attempts.length > 0);
3617
+ const failures = started.flatMap((result) => this.blocks(result, false));
3618
+ const flaky = started.flatMap((result) => this.blocks(result, true));
3587
3619
  const lines = this.verbose ? [] : [""];
3588
- if (problems.length > 0) {
3589
- lines.push("Problems", "");
3590
- for (const result of problems) lines.push(...this.problems(result), "");
3591
- }
3620
+ if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
3621
+ if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
3592
3622
  const tests = run.scenarios.flatMap((result) => result.tests);
3593
- lines.push(...run.status === RunStatus.Interrupted ? [`Run: ${run.status}`] : [], `Scenarios: ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, `Tests: ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability), ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true), `Duration: ${ConsoleReporter.duration(run.durationMs)}`, "", "");
3594
- return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
3623
+ lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
3624
+ return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
3595
3625
  }
3596
3626
  static footer(run, exitCode, reportPaths) {
3597
3627
  return [
@@ -3605,57 +3635,135 @@ var ConsoleReporter = class ConsoleReporter {
3605
3635
  return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
3606
3636
  }
3607
3637
  static duration(ms) {
3638
+ if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
3608
3639
  const seconds = Math.floor(ms / 1e3);
3609
3640
  const pad = (value) => String(value).padStart(2, "0");
3610
3641
  const hours = Math.floor(seconds / 3600);
3611
- const rest = `${pad(Math.floor(seconds / 60) % 60)}:${pad(seconds % 60)}`;
3612
- return hours > 0 ? `${hours}:${rest}` : rest;
3642
+ const minutes = Math.floor(seconds / 60) % 60;
3643
+ return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
3613
3644
  }
3614
- problems(result) {
3615
- const spec = result.spec;
3616
- const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
3617
- for (const attempt of result.attempts) {
3618
- const execution = attempt.execution;
3619
- if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3620
- for (const error of execution.attemptErrors) lines.push(` Attempt ${attempt.attempt} error: ${error.category} error: ${error.message}`);
3621
- if (execution.closeError !== void 0) lines.push(` Attempt ${attempt.attempt} close error: ${execution.closeError.message}`);
3622
- }
3623
- for (const test of result.tests.filter((entry) => entry.status !== ResultStatus.Pass)) {
3624
- lines.push(` ${this.status(test.status, 6)} ${ConsoleReporter.testTitle(test)} ${test.location.toString()} ${ConsoleReporter.counts(test)}`);
3625
- test.history.forEach((entry, index) => {
3626
- if (entry.status === ResultStatus.Pass) return;
3627
- const prefix = ` Attempt ${result.attempts[index].attempt} ${this.status(entry.status)}`;
3628
- const width = ` Attempt ${result.attempts[index].attempt} ${entry.status}: `.length;
3629
- if (entry.details.length === 0) {
3630
- lines.push(prefix);
3631
- return;
3632
- }
3633
- lines.push(`${prefix}: ${ConsoleReporter.indent(entry.details[0], width)}`, ...entry.details.slice(1).map((detail) => `${" ".repeat(width)}${ConsoleReporter.indent(detail, width)}`));
3634
- });
3645
+ static wrap(text, width) {
3646
+ const lines = [];
3647
+ for (const paragraph of text.split("\n")) {
3648
+ let line = "";
3649
+ for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
3650
+ lines.push(line);
3651
+ line = word;
3652
+ } else line = line === "" ? word : `${line} ${word}`;
3653
+ lines.push(line);
3635
3654
  }
3636
3655
  return lines;
3637
3656
  }
3657
+ section(name, blocks) {
3658
+ return [
3659
+ ...this.rule(name),
3660
+ "",
3661
+ ...blocks.flatMap((block) => [...block, ""])
3662
+ ];
3663
+ }
3664
+ rule(name) {
3665
+ const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
3666
+ return [
3667
+ rule,
3668
+ ` ${name}`,
3669
+ rule
3670
+ ];
3671
+ }
3672
+ blocks(result, flaky) {
3673
+ const spec = result.spec;
3674
+ const title = ConsoleReporter.title(spec);
3675
+ const blocks = [];
3676
+ if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
3677
+ const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
3678
+ for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
3679
+ return blocks;
3680
+ }
3681
+ attemptBlock(spec, title, attempt) {
3682
+ const execution = attempt.execution;
3683
+ const problems = [];
3684
+ if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
3685
+ for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
3686
+ if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
3687
+ if (problems.length === 0) return [];
3688
+ const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
3689
+ const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
3690
+ return [[
3691
+ ` scenario at ${spec.location.file}`,
3692
+ ` ${this.mark(status)} ${title} › ${what}`,
3693
+ ...problems.flatMap((problem) => this.paragraph(problem, 3))
3694
+ ]];
3695
+ }
3696
+ testBlock(spec, title, test, result) {
3697
+ const location = test.location;
3698
+ const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
3699
+ const failing = test.history.map((entry, index) => ({
3700
+ entry,
3701
+ attempt: result.attempts[index]?.attempt ?? index + 1
3702
+ })).filter(({ entry }) => entry.status !== ResultStatus.Pass);
3703
+ const several = test.history.length > 1;
3704
+ for (const { entry, attempt } of failing) {
3705
+ if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
3706
+ lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
3707
+ }
3708
+ lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
3709
+ return lines;
3710
+ }
3711
+ entryBody(spec, entry, indent) {
3712
+ const pad = " ".repeat(indent);
3713
+ const evaluation = entry.evaluation;
3714
+ if (evaluation === void 0) {
3715
+ if (entry.details.length === 0) return [];
3716
+ return [
3717
+ "",
3718
+ `${pad}Details`,
3719
+ ...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
3720
+ ];
3721
+ }
3722
+ const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
3723
+ const expected = ConsoleReporter.expected(spec, entry);
3724
+ const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
3725
+ if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
3726
+ if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
3727
+ if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
3728
+ return lines;
3729
+ }
3730
+ static expected(spec, entry) {
3731
+ if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
3732
+ return spec.turns[entry.turn - 1]?.expect.response?.criteria;
3733
+ }
3734
+ paragraph(text, indent) {
3735
+ const pad = " ".repeat(indent);
3736
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
3737
+ }
3738
+ bullet(text, indent) {
3739
+ const pad = " ".repeat(indent);
3740
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
3741
+ }
3638
3742
  static testTitle(test) {
3639
3743
  return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
3640
3744
  }
3641
3745
  static tally(statuses) {
3642
3746
  const count = (status) => statuses.filter((entry) => entry === status).length;
3643
- const interrupted = count(ResultStatus.Interrupted);
3644
- return `${count(ResultStatus.Pass)} passed, ${count(ResultStatus.Flaky)} flaky, ${count(ResultStatus.Fail)} failed, ${count(ResultStatus.Error)} errors${interrupted > 0 ? `, ${interrupted} interrupted` : ""}`;
3645
- }
3646
- status(status, width = 0) {
3647
- const padding = " ".repeat(Math.max(0, width - status.length));
3648
- return (this.colour ? styleText(COLOURS[status], status, { validateStream: false }) : status) + padding;
3649
- }
3650
- static counts(result) {
3651
- return `(${result.passed} passed, ${result.failed} failed, ${result.errors} errors; required ${result.required})`;
3747
+ const parts = [
3748
+ [ResultStatus.Pass, "passed"],
3749
+ [ResultStatus.Flaky, "flaky"],
3750
+ [ResultStatus.Fail, "failed"],
3751
+ [ResultStatus.Error, "errors"],
3752
+ [ResultStatus.Interrupted, "interrupted"]
3753
+ ].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
3754
+ return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
3755
+ }
3756
+ mark(status) {
3757
+ const { symbol, colour } = MARKS[status];
3758
+ return this.colour ? styleText(colour, symbol, { validateStream: false }) : symbol;
3652
3759
  }
3653
3760
  static usage(label, usage, capability) {
3654
- if (!capability) return `${label}: n/a (not supported by host)`;
3655
- if (usage.expected === 0) return `${label}: none`;
3656
- if (usage.total === void 0) return `${label}: n/a (coverage 0/${usage.expected})`;
3761
+ const name = label.padEnd(16);
3762
+ if (!capability) return `${name} n/a (not supported by host)`;
3763
+ if (usage.expected === 0) return `${name} none`;
3764
+ if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
3657
3765
  const number = (value) => value.toLocaleString("en-US");
3658
- const total = `${label}: total ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3766
+ const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3659
3767
  return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
3660
3768
  }
3661
3769
  formatAttempt(spec, attempt) {
@@ -3667,28 +3775,28 @@ var ConsoleReporter = class ConsoleReporter {
3667
3775
  lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
3668
3776
  lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
3669
3777
  });
3670
- if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3778
+ if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3671
3779
  const notRun = attempt.tests.filter((test) => test.turn > ran);
3672
- if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3780
+ if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3673
3781
  lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
3674
- for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3675
- if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3676
- lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3782
+ for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3783
+ if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3784
+ lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3677
3785
  return lines;
3678
3786
  }
3679
3787
  formatTest(test) {
3680
- return [` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${detail}`)];
3788
+ return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
3681
3789
  }
3682
3790
  static formatTurn(number, input, turn) {
3683
3791
  const lines = [
3684
- ` Turn ${number} (${turn.durationMs} ms)`,
3685
- ` Input: ${ConsoleReporter.indent(input, 17)}`,
3686
- ` Response: ${ConsoleReporter.indent(turn.response, 17)}`
3792
+ ` Turn ${number} (${turn.durationMs} ms)`,
3793
+ ` Input: ${ConsoleReporter.indent(input, 22)}`,
3794
+ ` Response: ${ConsoleReporter.indent(turn.response, 22)}`
3687
3795
  ];
3688
- for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3689
- for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3690
- for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 17)}`);
3691
- if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3796
+ for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3797
+ for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3798
+ for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
3799
+ if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3692
3800
  return lines;
3693
3801
  }
3694
3802
  static indent(text, width) {