@mappies/miniwat-eval 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/main.js CHANGED
@@ -2932,64 +2932,94 @@ var ExitCodeCalculator = class {
2932
2932
  }
2933
2933
  };
2934
2934
  // src/reporting/consoleReporter.ts
2935
- const STATUS_WIDTH = 11;
2936
- const COLOURS = {
2937
- [ResultStatus.Pass]: "green",
2938
- [ResultStatus.Flaky]: "yellow",
2939
- [ResultStatus.Fail]: "red",
2940
- [ResultStatus.Error]: "red",
2941
- [ResultStatus.Skipped]: "dim",
2942
- [ResultStatus.Interrupted]: "dim"
2935
+ const MARKS = {
2936
+ [ResultStatus.Pass]: {
2937
+ symbol: "✔",
2938
+ colour: "green"
2939
+ },
2940
+ [ResultStatus.Flaky]: {
2941
+ symbol: "⚠",
2942
+ colour: "yellow"
2943
+ },
2944
+ [ResultStatus.Fail]: {
2945
+ symbol: "✖",
2946
+ colour: "red"
2947
+ },
2948
+ [ResultStatus.Error]: {
2949
+ symbol: "●",
2950
+ colour: "red"
2951
+ },
2952
+ [ResultStatus.Skipped]: {
2953
+ symbol: "○",
2954
+ colour: "dim"
2955
+ },
2956
+ [ResultStatus.Interrupted]: {
2957
+ symbol: "◌",
2958
+ colour: "dim"
2959
+ }
2943
2960
  };
2961
+ const MIN_COLUMNS = 40;
2962
+ const RULE_WIDTH = 72;
2944
2963
  var ConsoleReporter = class ConsoleReporter {
2945
2964
  verbose;
2946
2965
  colour;
2947
- constructor(verbose, colour) {
2966
+ nameWidth;
2967
+ columns;
2968
+ thresholds;
2969
+ constructor(verbose, colour, options = {}) {
2948
2970
  this.verbose = verbose;
2949
2971
  this.colour = colour;
2972
+ this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
2973
+ this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
2974
+ this.thresholds = options.thresholds;
2975
+ }
2976
+ static nameWidth(specs) {
2977
+ return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
2950
2978
  }
2951
2979
  header(run) {
2952
2980
  return [
2953
- "Miniwat Eval",
2981
+ " Miniwat Eval",
2954
2982
  "",
2955
- ...run.quick ? ["Execution Mode: Quick"] : [],
2956
- `Repeat: ${run.repeat}`,
2957
- `Minimum Pass: ${run.minPass}`,
2983
+ ...run.quick ? [" Execution mode Quick"] : [],
2984
+ ` Repeat ${run.repeat}`,
2985
+ ` Minimum pass ${run.minPass}`,
2958
2986
  "",
2959
- `Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
2987
+ ` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
2960
2988
  "",
2961
2989
  ""
2962
2990
  ].join("\n");
2963
2991
  }
2964
2992
  scenario(result, repeat, tokenUsageCapability) {
2965
2993
  const spec = result.spec;
2966
- const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
2967
- const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
2994
+ const title = ConsoleReporter.title(spec);
2995
+ const started = result.attempts.length > 0;
2996
+ const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
2997
+ const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
2998
+ const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
2968
2999
  if (!this.verbose) return line;
2969
- if (result.attempts.length === 0) return `${line}\n`;
3000
+ if (!started) return `${line}\n`;
2970
3001
  const websocket = spec.websocket;
2971
3002
  const subprotocols = websocket?.connection.subprotocols.length ?? 0;
2972
- const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
2973
- for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
2974
- lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
2975
- lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
2976
- lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
2977
- lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3003
+ const lines = [];
3004
+ if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
3005
+ for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3006
+ lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
3007
+ lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
2978
3008
  return line + lines.join("\n");
2979
3009
  }
2980
3010
  static title(spec) {
2981
3011
  return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
2982
3012
  }
2983
3013
  summary(run, exitCode, reportPaths = []) {
2984
- const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
3014
+ const started = run.scenarios.filter((result) => result.attempts.length > 0);
3015
+ const failures = started.flatMap((result) => this.blocks(result, false));
3016
+ const flaky = started.flatMap((result) => this.blocks(result, true));
2985
3017
  const lines = this.verbose ? [] : [""];
2986
- if (problems.length > 0) {
2987
- lines.push("Problems", "");
2988
- for (const result of problems) lines.push(...this.problems(result), "");
2989
- }
3018
+ if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
3019
+ if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
2990
3020
  const tests = run.scenarios.flatMap((result) => result.tests);
2991
- lines.push(...run.status === RunStatus.Interrupted ? [`Run: ${run.status}`] : [], `Scenarios: ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, `Tests: ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability), ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true), `Duration: ${ConsoleReporter.duration(run.durationMs)}`, "", "");
2992
- return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
3021
+ lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
3022
+ return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
2993
3023
  }
2994
3024
  static footer(run, exitCode, reportPaths) {
2995
3025
  return [
@@ -3003,57 +3033,135 @@ var ConsoleReporter = class ConsoleReporter {
3003
3033
  return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
3004
3034
  }
3005
3035
  static duration(ms) {
3036
+ if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
3006
3037
  const seconds = Math.floor(ms / 1e3);
3007
3038
  const pad = (value) => String(value).padStart(2, "0");
3008
3039
  const hours = Math.floor(seconds / 3600);
3009
- const rest = `${pad(Math.floor(seconds / 60) % 60)}:${pad(seconds % 60)}`;
3010
- return hours > 0 ? `${hours}:${rest}` : rest;
3040
+ const minutes = Math.floor(seconds / 60) % 60;
3041
+ return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
3042
+ }
3043
+ static wrap(text, width) {
3044
+ const lines = [];
3045
+ for (const paragraph of text.split("\n")) {
3046
+ let line = "";
3047
+ for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
3048
+ lines.push(line);
3049
+ line = word;
3050
+ } else line = line === "" ? word : `${line} ${word}`;
3051
+ lines.push(line);
3052
+ }
3053
+ return lines;
3054
+ }
3055
+ section(name, blocks) {
3056
+ return [
3057
+ ...this.rule(name),
3058
+ "",
3059
+ ...blocks.flatMap((block) => [...block, ""])
3060
+ ];
3011
3061
  }
3012
- problems(result) {
3062
+ rule(name) {
3063
+ const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
3064
+ return [
3065
+ rule,
3066
+ ` ${name}`,
3067
+ rule
3068
+ ];
3069
+ }
3070
+ blocks(result, flaky) {
3013
3071
  const spec = result.spec;
3014
- const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
3015
- for (const attempt of result.attempts) {
3016
- const execution = attempt.execution;
3017
- if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3018
- for (const error of execution.attemptErrors) lines.push(` Attempt ${attempt.attempt} error: ${error.category} error: ${error.message}`);
3019
- if (execution.closeError !== void 0) lines.push(` Attempt ${attempt.attempt} close error: ${execution.closeError.message}`);
3072
+ const title = ConsoleReporter.title(spec);
3073
+ const blocks = [];
3074
+ if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
3075
+ const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
3076
+ for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
3077
+ return blocks;
3078
+ }
3079
+ attemptBlock(spec, title, attempt) {
3080
+ const execution = attempt.execution;
3081
+ const problems = [];
3082
+ if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
3083
+ for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
3084
+ if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
3085
+ if (problems.length === 0) return [];
3086
+ const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
3087
+ const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
3088
+ return [[
3089
+ ` scenario at ${spec.location.file}`,
3090
+ ` ${this.mark(status)} ${title} › ${what}`,
3091
+ ...problems.flatMap((problem) => this.paragraph(problem, 3))
3092
+ ]];
3093
+ }
3094
+ testBlock(spec, title, test, result) {
3095
+ const location = test.location;
3096
+ const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
3097
+ const failing = test.history.map((entry, index) => ({
3098
+ entry,
3099
+ attempt: result.attempts[index]?.attempt ?? index + 1
3100
+ })).filter(({ entry }) => entry.status !== ResultStatus.Pass);
3101
+ const several = test.history.length > 1;
3102
+ for (const { entry, attempt } of failing) {
3103
+ if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
3104
+ lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
3020
3105
  }
3021
- for (const test of result.tests.filter((entry) => entry.status !== ResultStatus.Pass)) {
3022
- lines.push(` ${this.status(test.status, 6)} ${ConsoleReporter.testTitle(test)} ${test.location.toString()} ${ConsoleReporter.counts(test)}`);
3023
- test.history.forEach((entry, index) => {
3024
- if (entry.status === ResultStatus.Pass) return;
3025
- const prefix = ` Attempt ${result.attempts[index].attempt} ${this.status(entry.status)}`;
3026
- const width = ` Attempt ${result.attempts[index].attempt} ${entry.status}: `.length;
3027
- if (entry.details.length === 0) {
3028
- lines.push(prefix);
3029
- return;
3030
- }
3031
- lines.push(`${prefix}: ${ConsoleReporter.indent(entry.details[0], width)}`, ...entry.details.slice(1).map((detail) => `${" ".repeat(width)}${ConsoleReporter.indent(detail, width)}`));
3032
- });
3106
+ lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
3107
+ return lines;
3108
+ }
3109
+ entryBody(spec, entry, indent) {
3110
+ const pad = " ".repeat(indent);
3111
+ const evaluation = entry.evaluation;
3112
+ if (evaluation === void 0) {
3113
+ if (entry.details.length === 0) return [];
3114
+ return [
3115
+ "",
3116
+ `${pad}Details`,
3117
+ ...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
3118
+ ];
3033
3119
  }
3120
+ const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
3121
+ const expected = ConsoleReporter.expected(spec, entry);
3122
+ const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
3123
+ if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
3124
+ if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
3125
+ if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
3034
3126
  return lines;
3035
3127
  }
3128
+ static expected(spec, entry) {
3129
+ if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
3130
+ return spec.turns[entry.turn - 1]?.expect.response?.criteria;
3131
+ }
3132
+ paragraph(text, indent) {
3133
+ const pad = " ".repeat(indent);
3134
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
3135
+ }
3136
+ bullet(text, indent) {
3137
+ const pad = " ".repeat(indent);
3138
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
3139
+ }
3036
3140
  static testTitle(test) {
3037
3141
  return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
3038
3142
  }
3039
3143
  static tally(statuses) {
3040
3144
  const count = (status) => statuses.filter((entry) => entry === status).length;
3041
- const interrupted = count(ResultStatus.Interrupted);
3042
- return `${count(ResultStatus.Pass)} passed, ${count(ResultStatus.Flaky)} flaky, ${count(ResultStatus.Fail)} failed, ${count(ResultStatus.Error)} errors${interrupted > 0 ? `, ${interrupted} interrupted` : ""}`;
3043
- }
3044
- status(status, width = 0) {
3045
- const padding = " ".repeat(Math.max(0, width - status.length));
3046
- return (this.colour ? styleText(COLOURS[status], status, { validateStream: false }) : status) + padding;
3047
- }
3048
- static counts(result) {
3049
- return `(${result.passed} passed, ${result.failed} failed, ${result.errors} errors; required ${result.required})`;
3145
+ const parts = [
3146
+ [ResultStatus.Pass, "passed"],
3147
+ [ResultStatus.Flaky, "flaky"],
3148
+ [ResultStatus.Fail, "failed"],
3149
+ [ResultStatus.Error, "errors"],
3150
+ [ResultStatus.Interrupted, "interrupted"]
3151
+ ].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
3152
+ return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
3153
+ }
3154
+ mark(status) {
3155
+ const { symbol, colour } = MARKS[status];
3156
+ return this.colour ? styleText(colour, symbol, { validateStream: false }) : symbol;
3050
3157
  }
3051
3158
  static usage(label, usage, capability) {
3052
- if (!capability) return `${label}: n/a (not supported by host)`;
3053
- if (usage.expected === 0) return `${label}: none`;
3054
- if (usage.total === void 0) return `${label}: n/a (coverage 0/${usage.expected})`;
3159
+ const name = label.padEnd(16);
3160
+ if (!capability) return `${name} n/a (not supported by host)`;
3161
+ if (usage.expected === 0) return `${name} none`;
3162
+ if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
3055
3163
  const number = (value) => value.toLocaleString("en-US");
3056
- const total = `${label}: total ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3164
+ const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3057
3165
  return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
3058
3166
  }
3059
3167
  formatAttempt(spec, attempt) {
@@ -3065,28 +3173,28 @@ var ConsoleReporter = class ConsoleReporter {
3065
3173
  lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
3066
3174
  lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
3067
3175
  });
3068
- if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3176
+ if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3069
3177
  const notRun = attempt.tests.filter((test) => test.turn > ran);
3070
- if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3178
+ if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3071
3179
  lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
3072
- for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3073
- if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3074
- lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3180
+ for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3181
+ if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3182
+ lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3075
3183
  return lines;
3076
3184
  }
3077
3185
  formatTest(test) {
3078
- return [` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${detail}`)];
3186
+ return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
3079
3187
  }
3080
3188
  static formatTurn(number, input, turn) {
3081
3189
  const lines = [
3082
- ` Turn ${number} (${turn.durationMs} ms)`,
3083
- ` Input: ${ConsoleReporter.indent(input, 17)}`,
3084
- ` Response: ${ConsoleReporter.indent(turn.response, 17)}`
3190
+ ` Turn ${number} (${turn.durationMs} ms)`,
3191
+ ` Input: ${ConsoleReporter.indent(input, 22)}`,
3192
+ ` Response: ${ConsoleReporter.indent(turn.response, 22)}`
3085
3193
  ];
3086
- for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3087
- for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3088
- for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 17)}`);
3089
- if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3194
+ for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3195
+ for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3196
+ for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
3197
+ if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3090
3198
  return lines;
3091
3199
  }
3092
3200
  static indent(text, width) {
@@ -4022,7 +4130,8 @@ var RunCommand = class RunCommand {
4022
4130
  createEvaluator;
4023
4131
  signals;
4024
4132
  colour;
4025
- constructor(out, err, environment = process.env, cwd = process.cwd(), createRunner = (config, evaluator) => new ConversationRunner(config, evaluator), createEvaluator = createLlmEvaluator, signals = process, colour = false) {
4133
+ columns;
4134
+ constructor(out, err, environment = process.env, cwd = process.cwd(), createRunner = (config, evaluator) => new ConversationRunner(config, evaluator), createEvaluator = createLlmEvaluator, signals = process, colour = false, columns = void 0) {
4026
4135
  this.out = out;
4027
4136
  this.err = err;
4028
4137
  this.environment = environment;
@@ -4031,6 +4140,7 @@ var RunCommand = class RunCommand {
4031
4140
  this.createEvaluator = createEvaluator;
4032
4141
  this.signals = signals;
4033
4142
  this.colour = colour;
4143
+ this.columns = columns;
4034
4144
  }
4035
4145
  async execute(target, config, execution = new ExecutionOverride(), report = new ReportOverride(), parameters = "") {
4036
4146
  let evalConfig;
@@ -4057,7 +4167,11 @@ var RunCommand = class RunCommand {
4057
4167
  const { repeat, minPass, flakyPolicy, scenarioConcurrency } = evalConfig.execution;
4058
4168
  const tokenUsageCapability = evalConfig.websocket.protocol.capabilities.tokenUsage;
4059
4169
  const consoleReport = evalConfig.report.formats.includes(ReportFormat.Console);
4060
- const reporter = new ConsoleReporter(report.verbose, this.colour);
4170
+ const reporter = new ConsoleReporter(report.verbose, this.colour, {
4171
+ nameWidth: ConsoleReporter.nameWidth(specs),
4172
+ columns: this.columns,
4173
+ thresholds: evalConfig.evaluator.thresholds
4174
+ });
4061
4175
  const write = (text) => {
4062
4176
  if (consoleReport) this.out(redactor.redact(text));
4063
4177
  };
@@ -4151,7 +4265,7 @@ var ValidateCommand = class {
4151
4265
  }
4152
4266
  };
4153
4267
  // src/cli/cli.ts
4154
- function createProgram(setExitCode, out, err, colour = false) {
4268
+ function createProgram(setExitCode, out, err, colour = false, columns = void 0) {
4155
4269
  const program = new Command().name("miniwat-eval").description("Black-box, YAML-driven evaluation for agents built on miniwat-agent").configureOutput({
4156
4270
  writeOut: out,
4157
4271
  writeErr: err
@@ -4171,14 +4285,14 @@ function createProgram(setExitCode, out, err, colour = false) {
4171
4285
  setExitCode(1);
4172
4286
  return;
4173
4287
  }
4174
- setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour).execute(target, options.config ?? "", execution, report, options.parameters ?? ""));
4288
+ setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour, columns).execute(target, options.config ?? "", execution, report, options.parameters ?? ""));
4175
4289
  });
4176
4290
  return program;
4177
4291
  }
4178
4292
  // src/main.ts
4179
4293
  const program = createProgram((code) => {
4180
4294
  process.exitCode = code;
4181
- }, (text) => process.stdout.write(text), (text) => process.stderr.write(text), process.stdout.isTTY === true && !process.env.NO_COLOR);
4295
+ }, (text) => process.stdout.write(text), (text) => process.stderr.write(text), process.stdout.isTTY === true && !process.env.NO_COLOR, process.stdout.isTTY === true ? process.stdout.columns : void 0);
4182
4296
  try {
4183
4297
  await program.parseAsync(process.argv);
4184
4298
  } catch (error) {