@mappies/miniwat-eval 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -5
- package/dist/index.cjs +186 -78
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +24 -4
- package/dist/index.d.ts +24 -4
- package/dist/index.js +187 -79
- package/dist/index.js.map +1 -1
- package/dist/main.js +196 -82
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
package/dist/main.js
CHANGED
|
@@ -2932,64 +2932,94 @@ var ExitCodeCalculator = class {
|
|
|
2932
2932
|
}
|
|
2933
2933
|
};
|
|
2934
2934
|
// src/reporting/consoleReporter.ts
|
|
2935
|
-
const
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
[ResultStatus.
|
|
2941
|
-
|
|
2942
|
-
|
|
2935
|
+
const MARKS = {
|
|
2936
|
+
[ResultStatus.Pass]: {
|
|
2937
|
+
symbol: "✔",
|
|
2938
|
+
colour: "green"
|
|
2939
|
+
},
|
|
2940
|
+
[ResultStatus.Flaky]: {
|
|
2941
|
+
symbol: "⚠",
|
|
2942
|
+
colour: "yellow"
|
|
2943
|
+
},
|
|
2944
|
+
[ResultStatus.Fail]: {
|
|
2945
|
+
symbol: "✖",
|
|
2946
|
+
colour: "red"
|
|
2947
|
+
},
|
|
2948
|
+
[ResultStatus.Error]: {
|
|
2949
|
+
symbol: "●",
|
|
2950
|
+
colour: "red"
|
|
2951
|
+
},
|
|
2952
|
+
[ResultStatus.Skipped]: {
|
|
2953
|
+
symbol: "○",
|
|
2954
|
+
colour: "dim"
|
|
2955
|
+
},
|
|
2956
|
+
[ResultStatus.Interrupted]: {
|
|
2957
|
+
symbol: "◌",
|
|
2958
|
+
colour: "dim"
|
|
2959
|
+
}
|
|
2943
2960
|
};
|
|
2961
|
+
const MIN_COLUMNS = 40;
|
|
2962
|
+
const RULE_WIDTH = 72;
|
|
2944
2963
|
var ConsoleReporter = class ConsoleReporter {
|
|
2945
2964
|
verbose;
|
|
2946
2965
|
colour;
|
|
2947
|
-
|
|
2966
|
+
nameWidth;
|
|
2967
|
+
columns;
|
|
2968
|
+
thresholds;
|
|
2969
|
+
constructor(verbose, colour, options = {}) {
|
|
2948
2970
|
this.verbose = verbose;
|
|
2949
2971
|
this.colour = colour;
|
|
2972
|
+
this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
|
|
2973
|
+
this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
|
|
2974
|
+
this.thresholds = options.thresholds;
|
|
2975
|
+
}
|
|
2976
|
+
static nameWidth(specs) {
|
|
2977
|
+
return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
|
|
2950
2978
|
}
|
|
2951
2979
|
header(run) {
|
|
2952
2980
|
return [
|
|
2953
|
-
"Miniwat Eval",
|
|
2981
|
+
" Miniwat Eval",
|
|
2954
2982
|
"",
|
|
2955
|
-
...run.quick ? ["Execution
|
|
2956
|
-
`Repeat
|
|
2957
|
-
`Minimum
|
|
2983
|
+
...run.quick ? [" Execution mode Quick"] : [],
|
|
2984
|
+
` Repeat ${run.repeat}`,
|
|
2985
|
+
` Minimum pass ${run.minPass}`,
|
|
2958
2986
|
"",
|
|
2959
|
-
`Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
2987
|
+
` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
2960
2988
|
"",
|
|
2961
2989
|
""
|
|
2962
2990
|
].join("\n");
|
|
2963
2991
|
}
|
|
2964
2992
|
scenario(result, repeat, tokenUsageCapability) {
|
|
2965
2993
|
const spec = result.spec;
|
|
2966
|
-
const
|
|
2967
|
-
const
|
|
2994
|
+
const title = ConsoleReporter.title(spec);
|
|
2995
|
+
const started = result.attempts.length > 0;
|
|
2996
|
+
const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
|
|
2997
|
+
const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
|
|
2998
|
+
const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
|
|
2968
2999
|
if (!this.verbose) return line;
|
|
2969
|
-
if (
|
|
3000
|
+
if (!started) return `${line}\n`;
|
|
2970
3001
|
const websocket = spec.websocket;
|
|
2971
3002
|
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
2972
|
-
const lines = [
|
|
2973
|
-
|
|
2974
|
-
lines.push(`
|
|
2975
|
-
lines.push(
|
|
2976
|
-
lines.push(`
|
|
2977
|
-
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3003
|
+
const lines = [];
|
|
3004
|
+
if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
|
|
3005
|
+
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
3006
|
+
lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
|
|
3007
|
+
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
2978
3008
|
return line + lines.join("\n");
|
|
2979
3009
|
}
|
|
2980
3010
|
static title(spec) {
|
|
2981
3011
|
return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
|
|
2982
3012
|
}
|
|
2983
3013
|
summary(run, exitCode, reportPaths = []) {
|
|
2984
|
-
const
|
|
3014
|
+
const started = run.scenarios.filter((result) => result.attempts.length > 0);
|
|
3015
|
+
const failures = started.flatMap((result) => this.blocks(result, false));
|
|
3016
|
+
const flaky = started.flatMap((result) => this.blocks(result, true));
|
|
2985
3017
|
const lines = this.verbose ? [] : [""];
|
|
2986
|
-
if (
|
|
2987
|
-
|
|
2988
|
-
for (const result of problems) lines.push(...this.problems(result), "");
|
|
2989
|
-
}
|
|
3018
|
+
if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
|
|
3019
|
+
if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
|
|
2990
3020
|
const tests = run.scenarios.flatMap((result) => result.tests);
|
|
2991
|
-
lines.push(...run.status === RunStatus.Interrupted ? [`Run
|
|
2992
|
-
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
|
|
3021
|
+
lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
|
|
3022
|
+
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
|
|
2993
3023
|
}
|
|
2994
3024
|
static footer(run, exitCode, reportPaths) {
|
|
2995
3025
|
return [
|
|
@@ -3003,57 +3033,135 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3003
3033
|
return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
|
|
3004
3034
|
}
|
|
3005
3035
|
static duration(ms) {
|
|
3036
|
+
if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
|
|
3006
3037
|
const seconds = Math.floor(ms / 1e3);
|
|
3007
3038
|
const pad = (value) => String(value).padStart(2, "0");
|
|
3008
3039
|
const hours = Math.floor(seconds / 3600);
|
|
3009
|
-
const
|
|
3010
|
-
return hours > 0 ? `${hours}:${
|
|
3040
|
+
const minutes = Math.floor(seconds / 60) % 60;
|
|
3041
|
+
return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
|
|
3042
|
+
}
|
|
3043
|
+
static wrap(text, width) {
|
|
3044
|
+
const lines = [];
|
|
3045
|
+
for (const paragraph of text.split("\n")) {
|
|
3046
|
+
let line = "";
|
|
3047
|
+
for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
|
|
3048
|
+
lines.push(line);
|
|
3049
|
+
line = word;
|
|
3050
|
+
} else line = line === "" ? word : `${line} ${word}`;
|
|
3051
|
+
lines.push(line);
|
|
3052
|
+
}
|
|
3053
|
+
return lines;
|
|
3054
|
+
}
|
|
3055
|
+
section(name, blocks) {
|
|
3056
|
+
return [
|
|
3057
|
+
...this.rule(name),
|
|
3058
|
+
"",
|
|
3059
|
+
...blocks.flatMap((block) => [...block, ""])
|
|
3060
|
+
];
|
|
3011
3061
|
}
|
|
3012
|
-
|
|
3062
|
+
rule(name) {
|
|
3063
|
+
const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
|
|
3064
|
+
return [
|
|
3065
|
+
rule,
|
|
3066
|
+
` ${name}`,
|
|
3067
|
+
rule
|
|
3068
|
+
];
|
|
3069
|
+
}
|
|
3070
|
+
blocks(result, flaky) {
|
|
3013
3071
|
const spec = result.spec;
|
|
3014
|
-
const
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3072
|
+
const title = ConsoleReporter.title(spec);
|
|
3073
|
+
const blocks = [];
|
|
3074
|
+
if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
|
|
3075
|
+
const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
|
|
3076
|
+
for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
|
|
3077
|
+
return blocks;
|
|
3078
|
+
}
|
|
3079
|
+
attemptBlock(spec, title, attempt) {
|
|
3080
|
+
const execution = attempt.execution;
|
|
3081
|
+
const problems = [];
|
|
3082
|
+
if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
|
|
3083
|
+
for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
|
|
3084
|
+
if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
|
|
3085
|
+
if (problems.length === 0) return [];
|
|
3086
|
+
const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
|
|
3087
|
+
const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
|
|
3088
|
+
return [[
|
|
3089
|
+
` scenario at ${spec.location.file}`,
|
|
3090
|
+
` ${this.mark(status)} ${title} › ${what}`,
|
|
3091
|
+
...problems.flatMap((problem) => this.paragraph(problem, 3))
|
|
3092
|
+
]];
|
|
3093
|
+
}
|
|
3094
|
+
testBlock(spec, title, test, result) {
|
|
3095
|
+
const location = test.location;
|
|
3096
|
+
const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
|
|
3097
|
+
const failing = test.history.map((entry, index) => ({
|
|
3098
|
+
entry,
|
|
3099
|
+
attempt: result.attempts[index]?.attempt ?? index + 1
|
|
3100
|
+
})).filter(({ entry }) => entry.status !== ResultStatus.Pass);
|
|
3101
|
+
const several = test.history.length > 1;
|
|
3102
|
+
for (const { entry, attempt } of failing) {
|
|
3103
|
+
if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
|
|
3104
|
+
lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
|
|
3020
3105
|
}
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3106
|
+
lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
|
|
3107
|
+
return lines;
|
|
3108
|
+
}
|
|
3109
|
+
entryBody(spec, entry, indent) {
|
|
3110
|
+
const pad = " ".repeat(indent);
|
|
3111
|
+
const evaluation = entry.evaluation;
|
|
3112
|
+
if (evaluation === void 0) {
|
|
3113
|
+
if (entry.details.length === 0) return [];
|
|
3114
|
+
return [
|
|
3115
|
+
"",
|
|
3116
|
+
`${pad}Details`,
|
|
3117
|
+
...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
|
|
3118
|
+
];
|
|
3033
3119
|
}
|
|
3120
|
+
const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
|
|
3121
|
+
const expected = ConsoleReporter.expected(spec, entry);
|
|
3122
|
+
const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
|
|
3123
|
+
if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
|
|
3124
|
+
if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
|
|
3125
|
+
if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
|
|
3034
3126
|
return lines;
|
|
3035
3127
|
}
|
|
3128
|
+
static expected(spec, entry) {
|
|
3129
|
+
if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
|
|
3130
|
+
return spec.turns[entry.turn - 1]?.expect.response?.criteria;
|
|
3131
|
+
}
|
|
3132
|
+
paragraph(text, indent) {
|
|
3133
|
+
const pad = " ".repeat(indent);
|
|
3134
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
|
|
3135
|
+
}
|
|
3136
|
+
bullet(text, indent) {
|
|
3137
|
+
const pad = " ".repeat(indent);
|
|
3138
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
|
|
3139
|
+
}
|
|
3036
3140
|
static testTitle(test) {
|
|
3037
3141
|
return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
|
|
3038
3142
|
}
|
|
3039
3143
|
static tally(statuses) {
|
|
3040
3144
|
const count = (status) => statuses.filter((entry) => entry === status).length;
|
|
3041
|
-
const
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3145
|
+
const parts = [
|
|
3146
|
+
[ResultStatus.Pass, "passed"],
|
|
3147
|
+
[ResultStatus.Flaky, "flaky"],
|
|
3148
|
+
[ResultStatus.Fail, "failed"],
|
|
3149
|
+
[ResultStatus.Error, "errors"],
|
|
3150
|
+
[ResultStatus.Interrupted, "interrupted"]
|
|
3151
|
+
].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
|
|
3152
|
+
return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
|
|
3153
|
+
}
|
|
3154
|
+
mark(status) {
|
|
3155
|
+
const { symbol, colour } = MARKS[status];
|
|
3156
|
+
return this.colour ? styleText(colour, symbol, { validateStream: false }) : symbol;
|
|
3050
3157
|
}
|
|
3051
3158
|
static usage(label, usage, capability) {
|
|
3052
|
-
|
|
3053
|
-
if (
|
|
3054
|
-
if (usage.
|
|
3159
|
+
const name = label.padEnd(16);
|
|
3160
|
+
if (!capability) return `${name} n/a (not supported by host)`;
|
|
3161
|
+
if (usage.expected === 0) return `${name} none`;
|
|
3162
|
+
if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
|
|
3055
3163
|
const number = (value) => value.toLocaleString("en-US");
|
|
3056
|
-
const total = `${
|
|
3164
|
+
const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
|
|
3057
3165
|
return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
|
|
3058
3166
|
}
|
|
3059
3167
|
formatAttempt(spec, attempt) {
|
|
@@ -3065,28 +3173,28 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3065
3173
|
lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
|
|
3066
3174
|
lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
|
|
3067
3175
|
});
|
|
3068
|
-
if (execution.failure !== void 0) lines.push(`
|
|
3176
|
+
if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
|
|
3069
3177
|
const notRun = attempt.tests.filter((test) => test.turn > ran);
|
|
3070
|
-
if (notRun.length > 0) lines.push("
|
|
3178
|
+
if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
|
|
3071
3179
|
lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
|
|
3072
|
-
for (const error of execution.attemptErrors) lines.push(`
|
|
3073
|
-
if (execution.closeError !== void 0) lines.push(`
|
|
3074
|
-
lines.push(`
|
|
3180
|
+
for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
|
|
3181
|
+
if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
|
|
3182
|
+
lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
|
|
3075
3183
|
return lines;
|
|
3076
3184
|
}
|
|
3077
3185
|
formatTest(test) {
|
|
3078
|
-
return [`
|
|
3186
|
+
return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
|
|
3079
3187
|
}
|
|
3080
3188
|
static formatTurn(number, input, turn) {
|
|
3081
3189
|
const lines = [
|
|
3082
|
-
`
|
|
3083
|
-
`
|
|
3084
|
-
`
|
|
3190
|
+
` Turn ${number} (${turn.durationMs} ms)`,
|
|
3191
|
+
` Input: ${ConsoleReporter.indent(input, 22)}`,
|
|
3192
|
+
` Response: ${ConsoleReporter.indent(turn.response, 22)}`
|
|
3085
3193
|
];
|
|
3086
|
-
for (const use of turn.toolUses ?? []) lines.push(`
|
|
3087
|
-
for (const result of turn.toolResults ?? []) lines.push(`
|
|
3088
|
-
for (const status of turn.statuses) lines.push(`
|
|
3089
|
-
if (turn.tokenUsage !== void 0) lines.push(`
|
|
3194
|
+
for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
|
|
3195
|
+
for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
|
|
3196
|
+
for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
|
|
3197
|
+
if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
|
|
3090
3198
|
return lines;
|
|
3091
3199
|
}
|
|
3092
3200
|
static indent(text, width) {
|
|
@@ -4022,7 +4130,8 @@ var RunCommand = class RunCommand {
|
|
|
4022
4130
|
createEvaluator;
|
|
4023
4131
|
signals;
|
|
4024
4132
|
colour;
|
|
4025
|
-
|
|
4133
|
+
columns;
|
|
4134
|
+
constructor(out, err, environment = process.env, cwd = process.cwd(), createRunner = (config, evaluator) => new ConversationRunner(config, evaluator), createEvaluator = createLlmEvaluator, signals = process, colour = false, columns = void 0) {
|
|
4026
4135
|
this.out = out;
|
|
4027
4136
|
this.err = err;
|
|
4028
4137
|
this.environment = environment;
|
|
@@ -4031,6 +4140,7 @@ var RunCommand = class RunCommand {
|
|
|
4031
4140
|
this.createEvaluator = createEvaluator;
|
|
4032
4141
|
this.signals = signals;
|
|
4033
4142
|
this.colour = colour;
|
|
4143
|
+
this.columns = columns;
|
|
4034
4144
|
}
|
|
4035
4145
|
async execute(target, config, execution = new ExecutionOverride(), report = new ReportOverride(), parameters = "") {
|
|
4036
4146
|
let evalConfig;
|
|
@@ -4057,7 +4167,11 @@ var RunCommand = class RunCommand {
|
|
|
4057
4167
|
const { repeat, minPass, flakyPolicy, scenarioConcurrency } = evalConfig.execution;
|
|
4058
4168
|
const tokenUsageCapability = evalConfig.websocket.protocol.capabilities.tokenUsage;
|
|
4059
4169
|
const consoleReport = evalConfig.report.formats.includes(ReportFormat.Console);
|
|
4060
|
-
const reporter = new ConsoleReporter(report.verbose, this.colour
|
|
4170
|
+
const reporter = new ConsoleReporter(report.verbose, this.colour, {
|
|
4171
|
+
nameWidth: ConsoleReporter.nameWidth(specs),
|
|
4172
|
+
columns: this.columns,
|
|
4173
|
+
thresholds: evalConfig.evaluator.thresholds
|
|
4174
|
+
});
|
|
4061
4175
|
const write = (text) => {
|
|
4062
4176
|
if (consoleReport) this.out(redactor.redact(text));
|
|
4063
4177
|
};
|
|
@@ -4151,7 +4265,7 @@ var ValidateCommand = class {
|
|
|
4151
4265
|
}
|
|
4152
4266
|
};
|
|
4153
4267
|
// src/cli/cli.ts
|
|
4154
|
-
function createProgram(setExitCode, out, err, colour = false) {
|
|
4268
|
+
function createProgram(setExitCode, out, err, colour = false, columns = void 0) {
|
|
4155
4269
|
const program = new Command().name("miniwat-eval").description("Black-box, YAML-driven evaluation for agents built on miniwat-agent").configureOutput({
|
|
4156
4270
|
writeOut: out,
|
|
4157
4271
|
writeErr: err
|
|
@@ -4171,14 +4285,14 @@ function createProgram(setExitCode, out, err, colour = false) {
|
|
|
4171
4285
|
setExitCode(1);
|
|
4172
4286
|
return;
|
|
4173
4287
|
}
|
|
4174
|
-
setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour).execute(target, options.config ?? "", execution, report, options.parameters ?? ""));
|
|
4288
|
+
setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour, columns).execute(target, options.config ?? "", execution, report, options.parameters ?? ""));
|
|
4175
4289
|
});
|
|
4176
4290
|
return program;
|
|
4177
4291
|
}
|
|
4178
4292
|
// src/main.ts
|
|
4179
4293
|
const program = createProgram((code) => {
|
|
4180
4294
|
process.exitCode = code;
|
|
4181
|
-
}, (text) => process.stdout.write(text), (text) => process.stderr.write(text), process.stdout.isTTY === true && !process.env.NO_COLOR);
|
|
4295
|
+
}, (text) => process.stdout.write(text), (text) => process.stderr.write(text), process.stdout.isTTY === true && !process.env.NO_COLOR, process.stdout.isTTY === true ? process.stdout.columns : void 0);
|
|
4182
4296
|
try {
|
|
4183
4297
|
await program.parseAsync(process.argv);
|
|
4184
4298
|
} catch (error) {
|