@mappies/miniwat-eval 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -5
- package/dist/index.cjs +186 -78
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +24 -4
- package/dist/index.d.ts +24 -4
- package/dist/index.js +187 -79
- package/dist/index.js.map +1 -1
- package/dist/main.js +196 -82
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -909,10 +909,22 @@ type RunHeader = {
|
|
|
909
909
|
scenarios: number;
|
|
910
910
|
concurrency: number;
|
|
911
911
|
};
|
|
912
|
+
type ConsoleOptions = {
|
|
913
|
+
nameWidth?: number;
|
|
914
|
+
columns?: number;
|
|
915
|
+
thresholds?: {
|
|
916
|
+
response: number;
|
|
917
|
+
conversation: number;
|
|
918
|
+
};
|
|
919
|
+
};
|
|
912
920
|
declare class ConsoleReporter {
|
|
913
921
|
private readonly verbose;
|
|
914
922
|
private readonly colour;
|
|
915
|
-
|
|
923
|
+
private readonly nameWidth;
|
|
924
|
+
private readonly columns;
|
|
925
|
+
private readonly thresholds;
|
|
926
|
+
constructor(verbose: boolean, colour: boolean, options?: ConsoleOptions);
|
|
927
|
+
static nameWidth(specs: ConversationSpec[]): number;
|
|
916
928
|
header(run: RunHeader): string;
|
|
917
929
|
scenario(result: ScenarioResult, repeat: number, tokenUsageCapability: boolean): string;
|
|
918
930
|
static title(spec: ConversationSpec): string;
|
|
@@ -920,11 +932,19 @@ declare class ConsoleReporter {
|
|
|
920
932
|
static footer(run: EvalRunResult, exitCode: number, reportPaths: string[]): string;
|
|
921
933
|
static finalLine(run: EvalRunResult, exitCode: number): string;
|
|
922
934
|
static duration(ms: number): string;
|
|
923
|
-
|
|
935
|
+
static wrap(text: string, width: number): string[];
|
|
936
|
+
private section;
|
|
937
|
+
private rule;
|
|
938
|
+
private blocks;
|
|
939
|
+
private attemptBlock;
|
|
940
|
+
private testBlock;
|
|
941
|
+
private entryBody;
|
|
942
|
+
private static expected;
|
|
943
|
+
private paragraph;
|
|
944
|
+
private bullet;
|
|
924
945
|
private static testTitle;
|
|
925
946
|
private static tally;
|
|
926
|
-
private
|
|
927
|
-
private static counts;
|
|
947
|
+
private mark;
|
|
928
948
|
private static usage;
|
|
929
949
|
private formatAttempt;
|
|
930
950
|
private formatTest;
|
package/dist/index.d.ts
CHANGED
|
@@ -910,10 +910,22 @@ type RunHeader = {
|
|
|
910
910
|
scenarios: number;
|
|
911
911
|
concurrency: number;
|
|
912
912
|
};
|
|
913
|
+
type ConsoleOptions = {
|
|
914
|
+
nameWidth?: number;
|
|
915
|
+
columns?: number;
|
|
916
|
+
thresholds?: {
|
|
917
|
+
response: number;
|
|
918
|
+
conversation: number;
|
|
919
|
+
};
|
|
920
|
+
};
|
|
913
921
|
declare class ConsoleReporter {
|
|
914
922
|
private readonly verbose;
|
|
915
923
|
private readonly colour;
|
|
916
|
-
|
|
924
|
+
private readonly nameWidth;
|
|
925
|
+
private readonly columns;
|
|
926
|
+
private readonly thresholds;
|
|
927
|
+
constructor(verbose: boolean, colour: boolean, options?: ConsoleOptions);
|
|
928
|
+
static nameWidth(specs: ConversationSpec[]): number;
|
|
917
929
|
header(run: RunHeader): string;
|
|
918
930
|
scenario(result: ScenarioResult, repeat: number, tokenUsageCapability: boolean): string;
|
|
919
931
|
static title(spec: ConversationSpec): string;
|
|
@@ -921,11 +933,19 @@ declare class ConsoleReporter {
|
|
|
921
933
|
static footer(run: EvalRunResult, exitCode: number, reportPaths: string[]): string;
|
|
922
934
|
static finalLine(run: EvalRunResult, exitCode: number): string;
|
|
923
935
|
static duration(ms: number): string;
|
|
924
|
-
|
|
936
|
+
static wrap(text: string, width: number): string[];
|
|
937
|
+
private section;
|
|
938
|
+
private rule;
|
|
939
|
+
private blocks;
|
|
940
|
+
private attemptBlock;
|
|
941
|
+
private testBlock;
|
|
942
|
+
private entryBody;
|
|
943
|
+
private static expected;
|
|
944
|
+
private paragraph;
|
|
945
|
+
private bullet;
|
|
925
946
|
private static testTitle;
|
|
926
947
|
private static tally;
|
|
927
|
-
private
|
|
928
|
-
private static counts;
|
|
948
|
+
private mark;
|
|
929
949
|
private static usage;
|
|
930
950
|
private formatAttempt;
|
|
931
951
|
private formatTest;
|
package/dist/index.js
CHANGED
|
@@ -3534,64 +3534,94 @@ var ExitCodeCalculator = class {
|
|
|
3534
3534
|
}
|
|
3535
3535
|
};
|
|
3536
3536
|
// src/reporting/consoleReporter.ts
|
|
3537
|
-
const
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
3541
|
-
|
|
3542
|
-
[ResultStatus.
|
|
3543
|
-
|
|
3544
|
-
|
|
3537
|
+
const MARKS = {
|
|
3538
|
+
[ResultStatus.Pass]: {
|
|
3539
|
+
symbol: "✔",
|
|
3540
|
+
colour: "green"
|
|
3541
|
+
},
|
|
3542
|
+
[ResultStatus.Flaky]: {
|
|
3543
|
+
symbol: "⚠",
|
|
3544
|
+
colour: "yellow"
|
|
3545
|
+
},
|
|
3546
|
+
[ResultStatus.Fail]: {
|
|
3547
|
+
symbol: "✖",
|
|
3548
|
+
colour: "red"
|
|
3549
|
+
},
|
|
3550
|
+
[ResultStatus.Error]: {
|
|
3551
|
+
symbol: "●",
|
|
3552
|
+
colour: "red"
|
|
3553
|
+
},
|
|
3554
|
+
[ResultStatus.Skipped]: {
|
|
3555
|
+
symbol: "○",
|
|
3556
|
+
colour: "dim"
|
|
3557
|
+
},
|
|
3558
|
+
[ResultStatus.Interrupted]: {
|
|
3559
|
+
symbol: "◌",
|
|
3560
|
+
colour: "dim"
|
|
3561
|
+
}
|
|
3545
3562
|
};
|
|
3563
|
+
const MIN_COLUMNS = 40;
|
|
3564
|
+
const RULE_WIDTH = 72;
|
|
3546
3565
|
var ConsoleReporter = class ConsoleReporter {
|
|
3547
3566
|
verbose;
|
|
3548
3567
|
colour;
|
|
3549
|
-
|
|
3568
|
+
nameWidth;
|
|
3569
|
+
columns;
|
|
3570
|
+
thresholds;
|
|
3571
|
+
constructor(verbose, colour, options = {}) {
|
|
3550
3572
|
this.verbose = verbose;
|
|
3551
3573
|
this.colour = colour;
|
|
3574
|
+
this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
|
|
3575
|
+
this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
|
|
3576
|
+
this.thresholds = options.thresholds;
|
|
3577
|
+
}
|
|
3578
|
+
static nameWidth(specs) {
|
|
3579
|
+
return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
|
|
3552
3580
|
}
|
|
3553
3581
|
header(run) {
|
|
3554
3582
|
return [
|
|
3555
|
-
"Miniwat Eval",
|
|
3583
|
+
" Miniwat Eval",
|
|
3556
3584
|
"",
|
|
3557
|
-
...run.quick ? ["Execution
|
|
3558
|
-
`Repeat
|
|
3559
|
-
`Minimum
|
|
3585
|
+
...run.quick ? [" Execution mode Quick"] : [],
|
|
3586
|
+
` Repeat ${run.repeat}`,
|
|
3587
|
+
` Minimum pass ${run.minPass}`,
|
|
3560
3588
|
"",
|
|
3561
|
-
`Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
3589
|
+
` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
3562
3590
|
"",
|
|
3563
3591
|
""
|
|
3564
3592
|
].join("\n");
|
|
3565
3593
|
}
|
|
3566
3594
|
scenario(result, repeat, tokenUsageCapability) {
|
|
3567
3595
|
const spec = result.spec;
|
|
3568
|
-
const
|
|
3569
|
-
const
|
|
3596
|
+
const title = ConsoleReporter.title(spec);
|
|
3597
|
+
const started = result.attempts.length > 0;
|
|
3598
|
+
const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
|
|
3599
|
+
const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
|
|
3600
|
+
const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
|
|
3570
3601
|
if (!this.verbose) return line;
|
|
3571
|
-
if (
|
|
3602
|
+
if (!started) return `${line}\n`;
|
|
3572
3603
|
const websocket = spec.websocket;
|
|
3573
3604
|
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
3574
|
-
const lines = [
|
|
3575
|
-
|
|
3576
|
-
lines.push(`
|
|
3577
|
-
lines.push(
|
|
3578
|
-
lines.push(`
|
|
3579
|
-
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3605
|
+
const lines = [];
|
|
3606
|
+
if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
|
|
3607
|
+
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
3608
|
+
lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
|
|
3609
|
+
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3580
3610
|
return line + lines.join("\n");
|
|
3581
3611
|
}
|
|
3582
3612
|
static title(spec) {
|
|
3583
3613
|
return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
|
|
3584
3614
|
}
|
|
3585
3615
|
summary(run, exitCode, reportPaths = []) {
|
|
3586
|
-
const
|
|
3616
|
+
const started = run.scenarios.filter((result) => result.attempts.length > 0);
|
|
3617
|
+
const failures = started.flatMap((result) => this.blocks(result, false));
|
|
3618
|
+
const flaky = started.flatMap((result) => this.blocks(result, true));
|
|
3587
3619
|
const lines = this.verbose ? [] : [""];
|
|
3588
|
-
if (
|
|
3589
|
-
|
|
3590
|
-
for (const result of problems) lines.push(...this.problems(result), "");
|
|
3591
|
-
}
|
|
3620
|
+
if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
|
|
3621
|
+
if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
|
|
3592
3622
|
const tests = run.scenarios.flatMap((result) => result.tests);
|
|
3593
|
-
lines.push(...run.status === RunStatus.Interrupted ? [`Run
|
|
3594
|
-
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
|
|
3623
|
+
lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
|
|
3624
|
+
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
|
|
3595
3625
|
}
|
|
3596
3626
|
static footer(run, exitCode, reportPaths) {
|
|
3597
3627
|
return [
|
|
@@ -3605,57 +3635,135 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3605
3635
|
return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
|
|
3606
3636
|
}
|
|
3607
3637
|
static duration(ms) {
|
|
3638
|
+
if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
|
|
3608
3639
|
const seconds = Math.floor(ms / 1e3);
|
|
3609
3640
|
const pad = (value) => String(value).padStart(2, "0");
|
|
3610
3641
|
const hours = Math.floor(seconds / 3600);
|
|
3611
|
-
const
|
|
3612
|
-
return hours > 0 ? `${hours}:${
|
|
3642
|
+
const minutes = Math.floor(seconds / 60) % 60;
|
|
3643
|
+
return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
|
|
3613
3644
|
}
|
|
3614
|
-
|
|
3615
|
-
const
|
|
3616
|
-
const
|
|
3617
|
-
|
|
3618
|
-
const
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
|
|
3623
|
-
for (const test of result.tests.filter((entry) => entry.status !== ResultStatus.Pass)) {
|
|
3624
|
-
lines.push(` ${this.status(test.status, 6)} ${ConsoleReporter.testTitle(test)} ${test.location.toString()} ${ConsoleReporter.counts(test)}`);
|
|
3625
|
-
test.history.forEach((entry, index) => {
|
|
3626
|
-
if (entry.status === ResultStatus.Pass) return;
|
|
3627
|
-
const prefix = ` Attempt ${result.attempts[index].attempt} ${this.status(entry.status)}`;
|
|
3628
|
-
const width = ` Attempt ${result.attempts[index].attempt} ${entry.status}: `.length;
|
|
3629
|
-
if (entry.details.length === 0) {
|
|
3630
|
-
lines.push(prefix);
|
|
3631
|
-
return;
|
|
3632
|
-
}
|
|
3633
|
-
lines.push(`${prefix}: ${ConsoleReporter.indent(entry.details[0], width)}`, ...entry.details.slice(1).map((detail) => `${" ".repeat(width)}${ConsoleReporter.indent(detail, width)}`));
|
|
3634
|
-
});
|
|
3645
|
+
static wrap(text, width) {
|
|
3646
|
+
const lines = [];
|
|
3647
|
+
for (const paragraph of text.split("\n")) {
|
|
3648
|
+
let line = "";
|
|
3649
|
+
for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
|
|
3650
|
+
lines.push(line);
|
|
3651
|
+
line = word;
|
|
3652
|
+
} else line = line === "" ? word : `${line} ${word}`;
|
|
3653
|
+
lines.push(line);
|
|
3635
3654
|
}
|
|
3636
3655
|
return lines;
|
|
3637
3656
|
}
|
|
3657
|
+
section(name, blocks) {
|
|
3658
|
+
return [
|
|
3659
|
+
...this.rule(name),
|
|
3660
|
+
"",
|
|
3661
|
+
...blocks.flatMap((block) => [...block, ""])
|
|
3662
|
+
];
|
|
3663
|
+
}
|
|
3664
|
+
rule(name) {
|
|
3665
|
+
const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
|
|
3666
|
+
return [
|
|
3667
|
+
rule,
|
|
3668
|
+
` ${name}`,
|
|
3669
|
+
rule
|
|
3670
|
+
];
|
|
3671
|
+
}
|
|
3672
|
+
blocks(result, flaky) {
|
|
3673
|
+
const spec = result.spec;
|
|
3674
|
+
const title = ConsoleReporter.title(spec);
|
|
3675
|
+
const blocks = [];
|
|
3676
|
+
if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
|
|
3677
|
+
const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
|
|
3678
|
+
for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
|
|
3679
|
+
return blocks;
|
|
3680
|
+
}
|
|
3681
|
+
attemptBlock(spec, title, attempt) {
|
|
3682
|
+
const execution = attempt.execution;
|
|
3683
|
+
const problems = [];
|
|
3684
|
+
if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
|
|
3685
|
+
for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
|
|
3686
|
+
if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
|
|
3687
|
+
if (problems.length === 0) return [];
|
|
3688
|
+
const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
|
|
3689
|
+
const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
|
|
3690
|
+
return [[
|
|
3691
|
+
` scenario at ${spec.location.file}`,
|
|
3692
|
+
` ${this.mark(status)} ${title} › ${what}`,
|
|
3693
|
+
...problems.flatMap((problem) => this.paragraph(problem, 3))
|
|
3694
|
+
]];
|
|
3695
|
+
}
|
|
3696
|
+
testBlock(spec, title, test, result) {
|
|
3697
|
+
const location = test.location;
|
|
3698
|
+
const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
|
|
3699
|
+
const failing = test.history.map((entry, index) => ({
|
|
3700
|
+
entry,
|
|
3701
|
+
attempt: result.attempts[index]?.attempt ?? index + 1
|
|
3702
|
+
})).filter(({ entry }) => entry.status !== ResultStatus.Pass);
|
|
3703
|
+
const several = test.history.length > 1;
|
|
3704
|
+
for (const { entry, attempt } of failing) {
|
|
3705
|
+
if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
|
|
3706
|
+
lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
|
|
3707
|
+
}
|
|
3708
|
+
lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
|
|
3709
|
+
return lines;
|
|
3710
|
+
}
|
|
3711
|
+
entryBody(spec, entry, indent) {
|
|
3712
|
+
const pad = " ".repeat(indent);
|
|
3713
|
+
const evaluation = entry.evaluation;
|
|
3714
|
+
if (evaluation === void 0) {
|
|
3715
|
+
if (entry.details.length === 0) return [];
|
|
3716
|
+
return [
|
|
3717
|
+
"",
|
|
3718
|
+
`${pad}Details`,
|
|
3719
|
+
...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
|
|
3720
|
+
];
|
|
3721
|
+
}
|
|
3722
|
+
const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
|
|
3723
|
+
const expected = ConsoleReporter.expected(spec, entry);
|
|
3724
|
+
const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
|
|
3725
|
+
if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
|
|
3726
|
+
if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
|
|
3727
|
+
if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
|
|
3728
|
+
return lines;
|
|
3729
|
+
}
|
|
3730
|
+
static expected(spec, entry) {
|
|
3731
|
+
if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
|
|
3732
|
+
return spec.turns[entry.turn - 1]?.expect.response?.criteria;
|
|
3733
|
+
}
|
|
3734
|
+
paragraph(text, indent) {
|
|
3735
|
+
const pad = " ".repeat(indent);
|
|
3736
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
|
|
3737
|
+
}
|
|
3738
|
+
bullet(text, indent) {
|
|
3739
|
+
const pad = " ".repeat(indent);
|
|
3740
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
|
|
3741
|
+
}
|
|
3638
3742
|
static testTitle(test) {
|
|
3639
3743
|
return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
|
|
3640
3744
|
}
|
|
3641
3745
|
static tally(statuses) {
|
|
3642
3746
|
const count = (status) => statuses.filter((entry) => entry === status).length;
|
|
3643
|
-
const
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3747
|
+
const parts = [
|
|
3748
|
+
[ResultStatus.Pass, "passed"],
|
|
3749
|
+
[ResultStatus.Flaky, "flaky"],
|
|
3750
|
+
[ResultStatus.Fail, "failed"],
|
|
3751
|
+
[ResultStatus.Error, "errors"],
|
|
3752
|
+
[ResultStatus.Interrupted, "interrupted"]
|
|
3753
|
+
].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
|
|
3754
|
+
return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
|
|
3755
|
+
}
|
|
3756
|
+
mark(status) {
|
|
3757
|
+
const { symbol, colour } = MARKS[status];
|
|
3758
|
+
return this.colour ? styleText(colour, symbol, { validateStream: false }) : symbol;
|
|
3652
3759
|
}
|
|
3653
3760
|
static usage(label, usage, capability) {
|
|
3654
|
-
|
|
3655
|
-
if (
|
|
3656
|
-
if (usage.
|
|
3761
|
+
const name = label.padEnd(16);
|
|
3762
|
+
if (!capability) return `${name} n/a (not supported by host)`;
|
|
3763
|
+
if (usage.expected === 0) return `${name} none`;
|
|
3764
|
+
if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
|
|
3657
3765
|
const number = (value) => value.toLocaleString("en-US");
|
|
3658
|
-
const total = `${
|
|
3766
|
+
const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
|
|
3659
3767
|
return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
|
|
3660
3768
|
}
|
|
3661
3769
|
formatAttempt(spec, attempt) {
|
|
@@ -3667,28 +3775,28 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3667
3775
|
lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
|
|
3668
3776
|
lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
|
|
3669
3777
|
});
|
|
3670
|
-
if (execution.failure !== void 0) lines.push(`
|
|
3778
|
+
if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
|
|
3671
3779
|
const notRun = attempt.tests.filter((test) => test.turn > ran);
|
|
3672
|
-
if (notRun.length > 0) lines.push("
|
|
3780
|
+
if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
|
|
3673
3781
|
lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
|
|
3674
|
-
for (const error of execution.attemptErrors) lines.push(`
|
|
3675
|
-
if (execution.closeError !== void 0) lines.push(`
|
|
3676
|
-
lines.push(`
|
|
3782
|
+
for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
|
|
3783
|
+
if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
|
|
3784
|
+
lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
|
|
3677
3785
|
return lines;
|
|
3678
3786
|
}
|
|
3679
3787
|
formatTest(test) {
|
|
3680
|
-
return [`
|
|
3788
|
+
return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
|
|
3681
3789
|
}
|
|
3682
3790
|
static formatTurn(number, input, turn) {
|
|
3683
3791
|
const lines = [
|
|
3684
|
-
`
|
|
3685
|
-
`
|
|
3686
|
-
`
|
|
3792
|
+
` Turn ${number} (${turn.durationMs} ms)`,
|
|
3793
|
+
` Input: ${ConsoleReporter.indent(input, 22)}`,
|
|
3794
|
+
` Response: ${ConsoleReporter.indent(turn.response, 22)}`
|
|
3687
3795
|
];
|
|
3688
|
-
for (const use of turn.toolUses ?? []) lines.push(`
|
|
3689
|
-
for (const result of turn.toolResults ?? []) lines.push(`
|
|
3690
|
-
for (const status of turn.statuses) lines.push(`
|
|
3691
|
-
if (turn.tokenUsage !== void 0) lines.push(`
|
|
3796
|
+
for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
|
|
3797
|
+
for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
|
|
3798
|
+
for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
|
|
3799
|
+
if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
|
|
3692
3800
|
return lines;
|
|
3693
3801
|
}
|
|
3694
3802
|
static indent(text, width) {
|