@mappies/miniwat-eval 1.3.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/main.js CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env node
2
2
  import { Command, CommanderError } from "commander";
3
- import { dirname, extname, join, relative, resolve } from "node:path";
3
+ import { dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
4
4
  import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
5
5
  import { LineCounter, isMap, isPair, isScalar, isSeq, parseDocument } from "yaml";
6
6
  import { Ajv } from "ajv";
@@ -122,15 +122,17 @@ const CONFIG_FILE_NAME = "eval-config.yml";
122
122
  const PROJECT_ROOT_MARKERS = [".git", "package.json"];
123
123
  var ConfigLocator = class ConfigLocator {
124
124
  cwd;
125
- constructor(cwd = process.cwd()) {
125
+ displayBase;
126
+ constructor(cwd = process.cwd(), displayBase = cwd) {
126
127
  this.cwd = cwd;
128
+ this.displayBase = displayBase;
127
129
  }
128
130
  locate(target, explicitConfig) {
129
131
  if (explicitConfig !== "") {
130
132
  const path = resolve(this.cwd, explicitConfig);
131
133
  if (!ConfigLocator.isFile(path)) throw new EvalError({
132
134
  category: ErrorCategory.Configuration,
133
- message: `configuration file not found: ${explicitConfig}`
135
+ message: `configuration file not found: ${relative(this.displayBase, path) || explicitConfig}`
134
136
  });
135
137
  return path;
136
138
  }
@@ -139,7 +141,7 @@ var ConfigLocator = class ConfigLocator {
139
141
  const found = ConfigLocator.searchUpward(start) ?? ConfigLocator.candidate(this.cwd);
140
142
  if (found === void 0) throw new EvalError({
141
143
  category: ErrorCategory.Configuration,
142
- message: `no ${CONFIG_FILE_NAME} found above ${relative(this.cwd, targetPath) || "."} or in the working directory; pass --config <path>`
144
+ message: `no ${CONFIG_FILE_NAME} found above ${relative(this.displayBase, targetPath) || "."} or in the working directory; pass --config <path>`
143
145
  });
144
146
  return found;
145
147
  }
@@ -171,12 +173,14 @@ function isRecord(value) {
171
173
  // src/config/parametersLoader.ts
172
174
  var ParametersLoader = class ParametersLoader {
173
175
  cwd;
174
- constructor(cwd = process.cwd()) {
176
+ displayBase;
177
+ constructor(cwd = process.cwd(), displayBase = cwd) {
175
178
  this.cwd = cwd;
179
+ this.displayBase = displayBase;
176
180
  }
177
181
  load(path) {
178
182
  const absolute = resolve(this.cwd, path);
179
- const shown = relative(this.cwd, absolute) || path;
183
+ const shown = relative(this.displayBase, absolute) || path;
180
184
  let text;
181
185
  let value;
182
186
  try {
@@ -928,17 +932,19 @@ const DEFAULT_CONFIG = new EvalConfig({
928
932
  // src/config/evalConfigLoader.ts
929
933
  var EvalConfigLoader = class EvalConfigLoader {
930
934
  cwd;
935
+ displayBase;
931
936
  documentLoader = new YamlDocumentLoader();
932
937
  schemaValidator = new SchemaValidator(eval_config_schema_default);
933
938
  variablesReader = new VariablesReader();
934
939
  webSocketResolver = new WebSocketResolver();
935
940
  environmentResolver;
936
- constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
941
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
937
942
  this.cwd = cwd;
943
+ this.displayBase = displayBase;
938
944
  this.environmentResolver = new EnvironmentResolver(environment, parameters);
939
945
  }
940
946
  load(configPath, override, report = new ReportOverride()) {
941
- const document = this.documentLoader.load(configPath, relative(this.cwd, configPath));
947
+ const document = this.documentLoader.load(configPath, relative(this.displayBase, configPath));
942
948
  const issues = [...document.errors];
943
949
  if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
944
950
  const named = [];
@@ -2932,64 +2938,94 @@ var ExitCodeCalculator = class {
2932
2938
  }
2933
2939
  };
2934
2940
  // src/reporting/consoleReporter.ts
2935
- const STATUS_WIDTH = 11;
2936
- const COLOURS = {
2937
- [ResultStatus.Pass]: "green",
2938
- [ResultStatus.Flaky]: "yellow",
2939
- [ResultStatus.Fail]: "red",
2940
- [ResultStatus.Error]: "red",
2941
- [ResultStatus.Skipped]: "dim",
2942
- [ResultStatus.Interrupted]: "dim"
2941
+ const MARKS = {
2942
+ [ResultStatus.Pass]: {
2943
+ symbol: "✔",
2944
+ colour: "green"
2945
+ },
2946
+ [ResultStatus.Flaky]: {
2947
+ symbol: "⚠",
2948
+ colour: "yellow"
2949
+ },
2950
+ [ResultStatus.Fail]: {
2951
+ symbol: "✖",
2952
+ colour: "red"
2953
+ },
2954
+ [ResultStatus.Error]: {
2955
+ symbol: "●",
2956
+ colour: "red"
2957
+ },
2958
+ [ResultStatus.Skipped]: {
2959
+ symbol: "○",
2960
+ colour: "dim"
2961
+ },
2962
+ [ResultStatus.Interrupted]: {
2963
+ symbol: "◌",
2964
+ colour: "dim"
2965
+ }
2943
2966
  };
2967
+ const MIN_COLUMNS = 40;
2968
+ const RULE_WIDTH = 72;
2944
2969
  var ConsoleReporter = class ConsoleReporter {
2945
2970
  verbose;
2946
2971
  colour;
2947
- constructor(verbose, colour) {
2972
+ nameWidth;
2973
+ columns;
2974
+ thresholds;
2975
+ constructor(verbose, colour, options = {}) {
2948
2976
  this.verbose = verbose;
2949
2977
  this.colour = colour;
2978
+ this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
2979
+ this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
2980
+ this.thresholds = options.thresholds;
2981
+ }
2982
+ static nameWidth(specs) {
2983
+ return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
2950
2984
  }
2951
2985
  header(run) {
2952
2986
  return [
2953
- "Miniwat Eval",
2987
+ " Miniwat Eval",
2954
2988
  "",
2955
- ...run.quick ? ["Execution Mode: Quick"] : [],
2956
- `Repeat: ${run.repeat}`,
2957
- `Minimum Pass: ${run.minPass}`,
2989
+ ...run.quick ? [" Execution mode Quick"] : [],
2990
+ ` Repeat ${run.repeat}`,
2991
+ ` Minimum pass ${run.minPass}`,
2958
2992
  "",
2959
- `Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
2993
+ ` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
2960
2994
  "",
2961
2995
  ""
2962
2996
  ].join("\n");
2963
2997
  }
2964
2998
  scenario(result, repeat, tokenUsageCapability) {
2965
2999
  const spec = result.spec;
2966
- const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
2967
- const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
3000
+ const title = ConsoleReporter.title(spec);
3001
+ const started = result.attempts.length > 0;
3002
+ const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
3003
+ const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
3004
+ const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
2968
3005
  if (!this.verbose) return line;
2969
- if (result.attempts.length === 0) return `${line}\n`;
3006
+ if (!started) return `${line}\n`;
2970
3007
  const websocket = spec.websocket;
2971
3008
  const subprotocols = websocket?.connection.subprotocols.length ?? 0;
2972
- const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
2973
- for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
2974
- lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
2975
- lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
2976
- lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
2977
- lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3009
+ const lines = [];
3010
+ if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
3011
+ for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3012
+ lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
3013
+ lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
2978
3014
  return line + lines.join("\n");
2979
3015
  }
2980
3016
  static title(spec) {
2981
3017
  return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
2982
3018
  }
2983
3019
  summary(run, exitCode, reportPaths = []) {
2984
- const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
3020
+ const started = run.scenarios.filter((result) => result.attempts.length > 0);
3021
+ const failures = started.flatMap((result) => this.blocks(result, false));
3022
+ const flaky = started.flatMap((result) => this.blocks(result, true));
2985
3023
  const lines = this.verbose ? [] : [""];
2986
- if (problems.length > 0) {
2987
- lines.push("Problems", "");
2988
- for (const result of problems) lines.push(...this.problems(result), "");
2989
- }
3024
+ if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
3025
+ if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
2990
3026
  const tests = run.scenarios.flatMap((result) => result.tests);
2991
- lines.push(...run.status === RunStatus.Interrupted ? [`Run: ${run.status}`] : [], `Scenarios: ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, `Tests: ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability), ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true), `Duration: ${ConsoleReporter.duration(run.durationMs)}`, "", "");
2992
- return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
3027
+ lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
3028
+ return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
2993
3029
  }
2994
3030
  static footer(run, exitCode, reportPaths) {
2995
3031
  return [
@@ -3003,57 +3039,148 @@ var ConsoleReporter = class ConsoleReporter {
3003
3039
  return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
3004
3040
  }
3005
3041
  static duration(ms) {
3042
+ if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
3006
3043
  const seconds = Math.floor(ms / 1e3);
3007
3044
  const pad = (value) => String(value).padStart(2, "0");
3008
3045
  const hours = Math.floor(seconds / 3600);
3009
- const rest = `${pad(Math.floor(seconds / 60) % 60)}:${pad(seconds % 60)}`;
3010
- return hours > 0 ? `${hours}:${rest}` : rest;
3046
+ const minutes = Math.floor(seconds / 60) % 60;
3047
+ return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
3048
+ }
3049
+ static wrap(text, width) {
3050
+ const lines = [];
3051
+ for (const paragraph of text.split("\n")) {
3052
+ let line = "";
3053
+ for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
3054
+ lines.push(line);
3055
+ line = word;
3056
+ } else line = line === "" ? word : `${line} ${word}`;
3057
+ lines.push(line);
3058
+ }
3059
+ return lines;
3060
+ }
3061
+ section(name, blocks) {
3062
+ return [
3063
+ ...this.rule(name),
3064
+ "",
3065
+ ...blocks.flatMap((block) => [...block, ""])
3066
+ ];
3067
+ }
3068
+ rule(name) {
3069
+ const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
3070
+ return [
3071
+ rule,
3072
+ ` ${name}`,
3073
+ rule
3074
+ ];
3011
3075
  }
3012
- problems(result) {
3076
+ blocks(result, flaky) {
3013
3077
  const spec = result.spec;
3014
- const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
3015
- for (const attempt of result.attempts) {
3016
- const execution = attempt.execution;
3017
- if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3018
- for (const error of execution.attemptErrors) lines.push(` Attempt ${attempt.attempt} error: ${error.category} error: ${error.message}`);
3019
- if (execution.closeError !== void 0) lines.push(` Attempt ${attempt.attempt} close error: ${execution.closeError.message}`);
3078
+ const title = ConsoleReporter.title(spec);
3079
+ const blocks = [];
3080
+ if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
3081
+ const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
3082
+ for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
3083
+ return blocks;
3084
+ }
3085
+ attemptBlock(spec, title, attempt) {
3086
+ const execution = attempt.execution;
3087
+ const problems = [];
3088
+ if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
3089
+ for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
3090
+ if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
3091
+ if (problems.length === 0) return [];
3092
+ const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
3093
+ const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
3094
+ return [[
3095
+ ` scenario at ${spec.location.file}`,
3096
+ ` ${this.mark(status)} ${title} › ${what}`,
3097
+ ...problems.flatMap((problem) => this.paragraph(problem, 3))
3098
+ ]];
3099
+ }
3100
+ testBlock(spec, title, test, result) {
3101
+ const location = test.location;
3102
+ const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
3103
+ const failing = test.history.map((entry, index) => ({
3104
+ entry,
3105
+ run: result.attempts[index],
3106
+ attempt: result.attempts[index]?.attempt ?? index + 1
3107
+ })).filter(({ entry }) => entry.status !== ResultStatus.Pass);
3108
+ const several = test.history.length > 1;
3109
+ for (const { entry, run, attempt } of failing) {
3110
+ if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
3111
+ lines.push(...this.entryBody(spec, entry, run, several ? 5 : 3));
3020
3112
  }
3021
- for (const test of result.tests.filter((entry) => entry.status !== ResultStatus.Pass)) {
3022
- lines.push(` ${this.status(test.status, 6)} ${ConsoleReporter.testTitle(test)} ${test.location.toString()} ${ConsoleReporter.counts(test)}`);
3023
- test.history.forEach((entry, index) => {
3024
- if (entry.status === ResultStatus.Pass) return;
3025
- const prefix = ` Attempt ${result.attempts[index].attempt} ${this.status(entry.status)}`;
3026
- const width = ` Attempt ${result.attempts[index].attempt} ${entry.status}: `.length;
3027
- if (entry.details.length === 0) {
3028
- lines.push(prefix);
3029
- return;
3030
- }
3031
- lines.push(`${prefix}: ${ConsoleReporter.indent(entry.details[0], width)}`, ...entry.details.slice(1).map((detail) => `${" ".repeat(width)}${ConsoleReporter.indent(detail, width)}`));
3032
- });
3113
+ lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
3114
+ return lines;
3115
+ }
3116
+ entryBody(spec, entry, attempt, indent) {
3117
+ const pad = " ".repeat(indent);
3118
+ const evaluation = entry.evaluation;
3119
+ if (evaluation === void 0) {
3120
+ if (entry.details.length === 0) return [];
3121
+ return [
3122
+ "",
3123
+ `${pad}Details`,
3124
+ ...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
3125
+ ];
3033
3126
  }
3127
+ const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
3128
+ const expected = ConsoleReporter.expected(spec, entry);
3129
+ const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
3130
+ const actual = ConsoleReporter.actual(entry, attempt);
3131
+ if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
3132
+ if (actual !== void 0) lines.push("", `${pad}Actual`, ...this.capped(actual, indent + 2));
3133
+ if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
3034
3134
  return lines;
3035
3135
  }
3136
+ static actual(entry, attempt) {
3137
+ const turns = attempt?.execution.turns ?? [];
3138
+ if (entry.kind === TestKind.Conversation) return turns.length === 0 ? void 0 : turns.map((turn, index) => `Turn ${index + 1}: ${turn.response}`).join("\n");
3139
+ return turns[entry.turn - 1]?.response;
3140
+ }
3141
+ capped(text, indent) {
3142
+ const lines = text.trim() === "" ? [`${" ".repeat(indent)}(empty)`] : this.paragraph(text, indent);
3143
+ if (lines.length <= 12) return lines;
3144
+ const more = lines.length - 12;
3145
+ return [...lines.slice(0, 12), `${" ".repeat(indent)}… ${more} more ${more === 1 ? "line" : "lines"} in the JSON report`];
3146
+ }
3147
+ static expected(spec, entry) {
3148
+ if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
3149
+ return spec.turns[entry.turn - 1]?.expect.response?.criteria;
3150
+ }
3151
+ paragraph(text, indent) {
3152
+ const pad = " ".repeat(indent);
3153
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
3154
+ }
3155
+ bullet(text, indent) {
3156
+ const pad = " ".repeat(indent);
3157
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
3158
+ }
3036
3159
  static testTitle(test) {
3037
3160
  return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
3038
3161
  }
3039
3162
  static tally(statuses) {
3040
3163
  const count = (status) => statuses.filter((entry) => entry === status).length;
3041
- const interrupted = count(ResultStatus.Interrupted);
3042
- return `${count(ResultStatus.Pass)} passed, ${count(ResultStatus.Flaky)} flaky, ${count(ResultStatus.Fail)} failed, ${count(ResultStatus.Error)} errors${interrupted > 0 ? `, ${interrupted} interrupted` : ""}`;
3043
- }
3044
- status(status, width = 0) {
3045
- const padding = " ".repeat(Math.max(0, width - status.length));
3046
- return (this.colour ? styleText(COLOURS[status], status, { validateStream: false }) : status) + padding;
3047
- }
3048
- static counts(result) {
3049
- return `(${result.passed} passed, ${result.failed} failed, ${result.errors} errors; required ${result.required})`;
3164
+ const parts = [
3165
+ [ResultStatus.Pass, "passed"],
3166
+ [ResultStatus.Flaky, "flaky"],
3167
+ [ResultStatus.Fail, "failed"],
3168
+ [ResultStatus.Error, "errors"],
3169
+ [ResultStatus.Interrupted, "interrupted"]
3170
+ ].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
3171
+ return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
3172
+ }
3173
+ mark(status) {
3174
+ const { symbol, colour } = MARKS[status];
3175
+ return this.colour ? styleText(colour, symbol, { validateStream: false }) : symbol;
3050
3176
  }
3051
3177
  static usage(label, usage, capability) {
3052
- if (!capability) return `${label}: n/a (not supported by host)`;
3053
- if (usage.expected === 0) return `${label}: none`;
3054
- if (usage.total === void 0) return `${label}: n/a (coverage 0/${usage.expected})`;
3178
+ const name = label.padEnd(16);
3179
+ if (!capability) return `${name} n/a (not supported by host)`;
3180
+ if (usage.expected === 0) return `${name} none`;
3181
+ if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
3055
3182
  const number = (value) => value.toLocaleString("en-US");
3056
- const total = `${label}: total ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3183
+ const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3057
3184
  return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
3058
3185
  }
3059
3186
  formatAttempt(spec, attempt) {
@@ -3065,28 +3192,28 @@ var ConsoleReporter = class ConsoleReporter {
3065
3192
  lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
3066
3193
  lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
3067
3194
  });
3068
- if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3195
+ if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3069
3196
  const notRun = attempt.tests.filter((test) => test.turn > ran);
3070
- if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3197
+ if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3071
3198
  lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
3072
- for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3073
- if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3074
- lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3199
+ for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3200
+ if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3201
+ lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3075
3202
  return lines;
3076
3203
  }
3077
3204
  formatTest(test) {
3078
- return [` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${detail}`)];
3205
+ return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
3079
3206
  }
3080
3207
  static formatTurn(number, input, turn) {
3081
3208
  const lines = [
3082
- ` Turn ${number} (${turn.durationMs} ms)`,
3083
- ` Input: ${ConsoleReporter.indent(input, 17)}`,
3084
- ` Response: ${ConsoleReporter.indent(turn.response, 17)}`
3209
+ ` Turn ${number} (${turn.durationMs} ms)`,
3210
+ ` Input: ${ConsoleReporter.indent(input, 22)}`,
3211
+ ` Response: ${ConsoleReporter.indent(turn.response, 22)}`
3085
3212
  ];
3086
- for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3087
- for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3088
- for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 17)}`);
3089
- if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3213
+ for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3214
+ for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3215
+ for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
3216
+ if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3090
3217
  return lines;
3091
3218
  }
3092
3219
  static indent(text, width) {
@@ -3115,10 +3242,12 @@ const JSON_REPORT_FILE = "miniwat-eval-report.json";
3115
3242
  var JsonReporter = class JsonReporter {
3116
3243
  redactor;
3117
3244
  cwd;
3245
+ displayBase;
3118
3246
  tokens = new TokenUsageAggregator();
3119
- constructor(redactor, cwd) {
3247
+ constructor(redactor, cwd, displayBase = cwd) {
3120
3248
  this.redactor = redactor;
3121
3249
  this.cwd = cwd;
3250
+ this.displayBase = displayBase;
3122
3251
  }
3123
3252
  render(run, config, unsuccessfulCount, exitCode) {
3124
3253
  const report = {
@@ -3189,7 +3318,7 @@ var JsonReporter = class JsonReporter {
3189
3318
  return {
3190
3319
  name: result.spec.name,
3191
3320
  runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
3192
- file: result.spec.location.file,
3321
+ file: this.file(result.spec.location.file),
3193
3322
  status: result.status,
3194
3323
  connection: {
3195
3324
  endpoint: websocket.endpoint,
@@ -3199,7 +3328,7 @@ var JsonReporter = class JsonReporter {
3199
3328
  agentUsage: JsonReporter.usage(result.agentUsage),
3200
3329
  evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
3201
3330
  tests: result.tests.map((test) => ({
3202
- ...JsonReporter.identity(test),
3331
+ ...this.identity(test),
3203
3332
  status: test.status,
3204
3333
  ...JsonReporter.counts(test),
3205
3334
  history: test.history.map((entry, index) => ({
@@ -3224,7 +3353,7 @@ var JsonReporter = class JsonReporter {
3224
3353
  agentUsage: JsonReporter.usage(this.tokens.agent([attempt], capability)),
3225
3354
  evaluatorUsage: JsonReporter.usage(this.tokens.evaluator([attempt])),
3226
3355
  turns: execution.turns.map((turn, index) => JsonReporter.turn(index + 1, spec.turns[index].input, turn)),
3227
- tests: attempt.tests.map((test) => JsonReporter.test(test, config))
3356
+ tests: attempt.tests.map((test) => this.test(test, config))
3228
3357
  };
3229
3358
  }
3230
3359
  static turn(number, input, turn) {
@@ -3249,11 +3378,11 @@ var JsonReporter = class JsonReporter {
3249
3378
  tokenUsage: turn.tokenUsage === void 0 ? void 0 : JsonReporter.tokenUsage(turn.tokenUsage)
3250
3379
  };
3251
3380
  }
3252
- static test(test, config) {
3381
+ test(test, config) {
3253
3382
  const thresholds = config.evaluator.thresholds;
3254
3383
  const evaluation = test.evaluation;
3255
3384
  return {
3256
- ...JsonReporter.identity(test),
3385
+ ...this.identity(test),
3257
3386
  status: test.status,
3258
3387
  details: [...test.details],
3259
3388
  evaluation: evaluation === void 0 ? void 0 : {
@@ -3266,17 +3395,17 @@ var JsonReporter = class JsonReporter {
3266
3395
  evaluatorUsage: JsonReporter.usage(test.evaluatorUsage)
3267
3396
  };
3268
3397
  }
3269
- static identity(test) {
3398
+ identity(test) {
3270
3399
  return {
3271
3400
  kind: test.kind,
3272
3401
  turn: test.turn,
3273
3402
  title: test.title,
3274
- location: JsonReporter.location(test.location)
3403
+ location: this.location(test.location)
3275
3404
  };
3276
3405
  }
3277
- static location(location) {
3406
+ location(location) {
3278
3407
  return {
3279
- file: location.file,
3408
+ file: this.file(location.file),
3280
3409
  line: location.line,
3281
3410
  column: location.column,
3282
3411
  path: location.path
@@ -3323,6 +3452,9 @@ var JsonReporter = class JsonReporter {
3323
3452
  path(path) {
3324
3453
  return relative(this.cwd, path) || ".";
3325
3454
  }
3455
+ file(shown) {
3456
+ return shown === "" ? shown : this.path(resolve(this.displayBase, shown));
3457
+ }
3326
3458
  };
3327
3459
  // src/reporting/reportFileWriter.ts
3328
3460
  var ReportFileWriter = class {
@@ -3773,6 +3905,7 @@ var RunWithReader = class RunWithReader {
3773
3905
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
3774
3906
  var ScenarioLoader = class ScenarioLoader {
3775
3907
  cwd;
3908
+ displayBase;
3776
3909
  documentLoader = new YamlDocumentLoader();
3777
3910
  schemaValidator = new SchemaValidator(conversation_schema_default);
3778
3911
  conversationValidator = new ConversationValidator();
@@ -3780,18 +3913,19 @@ var ScenarioLoader = class ScenarioLoader {
3780
3913
  runWithReader = new RunWithReader();
3781
3914
  webSocketResolver = new WebSocketResolver();
3782
3915
  environmentResolver;
3783
- constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
3916
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
3784
3917
  this.cwd = cwd;
3918
+ this.displayBase = displayBase;
3785
3919
  this.environmentResolver = new EnvironmentResolver(environment, parameters);
3786
3920
  }
3787
3921
  load(target, config) {
3788
3922
  const files = this.discover(resolve(this.cwd, target), config);
3789
- const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.cwd, config.configPath)) : void 0;
3923
+ const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.displayBase, config.configPath)) : void 0;
3790
3924
  const issues = [];
3791
3925
  const connectionIssues = [];
3792
3926
  const specs = [];
3793
3927
  for (const file of files) {
3794
- const document = this.documentLoader.load(file, relative(this.cwd, file));
3928
+ const document = this.documentLoader.load(file, relative(this.displayBase, file));
3795
3929
  const fileIssues = [...document.errors];
3796
3930
  const fileSecrets = [];
3797
3931
  const used = /* @__PURE__ */ new Set();
@@ -3921,13 +4055,13 @@ var ScenarioLoader = class ScenarioLoader {
3921
4055
  discover(target, config) {
3922
4056
  if (!existsSync(target)) throw new EvalError({
3923
4057
  category: ErrorCategory.Specification,
3924
- message: `target not found: ${relative(this.cwd, target) || "."}`
4058
+ message: `target not found: ${relative(this.displayBase, target) || "."}`
3925
4059
  });
3926
4060
  const excluded = (path) => path === config.configPath || config.report.directory !== "" && path === config.report.directory;
3927
4061
  const files = statSync(target).isDirectory() ? this.walk(target, excluded) : [target].filter((file) => !excluded(file));
3928
4062
  if (files.length === 0) throw new EvalError({
3929
4063
  category: ErrorCategory.Specification,
3930
- message: `no scenario files found in ${relative(this.cwd, target) || "."}`
4064
+ message: `no scenario files found in ${relative(this.displayBase, target) || "."}`
3931
4065
  });
3932
4066
  return files;
3933
4067
  }
@@ -4007,6 +4141,16 @@ var ScenarioLoader = class ScenarioLoader {
4007
4141
  });
4008
4142
  }
4009
4143
  };
4144
+ // src/utils/displayPath.ts
4145
+ function displayBase(environment, cwd) {
4146
+ const start = environment.INIT_CWD;
4147
+ if (start === void 0 || start === "" || !isAbsolute(start)) return cwd;
4148
+ try {
4149
+ return existsSync(start) && statSync(start).isDirectory() ? start : cwd;
4150
+ } catch {
4151
+ return cwd;
4152
+ }
4153
+ }
4010
4154
  // src/cli/runCommand.ts
4011
4155
  const createLlmEvaluator = async (config) => {
4012
4156
  const factory = new EvaluatorAgentFactory(config.evaluator);
@@ -4022,7 +4166,8 @@ var RunCommand = class RunCommand {
4022
4166
  createEvaluator;
4023
4167
  signals;
4024
4168
  colour;
4025
- constructor(out, err, environment = process.env, cwd = process.cwd(), createRunner = (config, evaluator) => new ConversationRunner(config, evaluator), createEvaluator = createLlmEvaluator, signals = process, colour = false) {
4169
+ columns;
4170
+ constructor(out, err, environment = process.env, cwd = process.cwd(), createRunner = (config, evaluator) => new ConversationRunner(config, evaluator), createEvaluator = createLlmEvaluator, signals = process, colour = false, columns = void 0) {
4026
4171
  this.out = out;
4027
4172
  this.err = err;
4028
4173
  this.environment = environment;
@@ -4031,21 +4176,24 @@ var RunCommand = class RunCommand {
4031
4176
  this.createEvaluator = createEvaluator;
4032
4177
  this.signals = signals;
4033
4178
  this.colour = colour;
4179
+ this.columns = columns;
4180
+ this.displayBase = displayBase(environment, cwd);
4034
4181
  }
4182
+ displayBase;
4035
4183
  async execute(target, config, execution = new ExecutionOverride(), report = new ReportOverride(), parameters = "") {
4036
4184
  let evalConfig;
4037
4185
  let specs;
4038
4186
  let evaluator;
4039
4187
  let redactor = new SecretRedactor([]);
4040
4188
  try {
4041
- const configPath = new ConfigLocator(this.cwd).locate(target, config);
4189
+ const configPath = new ConfigLocator(this.cwd, this.displayBase).locate(target, config);
4042
4190
  const parameterFile = parameters === "" ? {
4043
4191
  path: "",
4044
4192
  parameters: {}
4045
- } : new ParametersLoader(this.cwd).load(parameters);
4046
- evalConfig = new EvalConfigLoader(this.environment, this.cwd, parameterFile.parameters).load(configPath, execution, report);
4193
+ } : new ParametersLoader(this.cwd, this.displayBase).load(parameters);
4194
+ evalConfig = new EvalConfigLoader(this.environment, this.cwd, parameterFile.parameters, this.displayBase).load(configPath, execution, report);
4047
4195
  evalConfig.parametersPath = parameterFile.path;
4048
- specs = new ScenarioLoader(this.environment, this.cwd, parameterFile.parameters).load(target, evalConfig);
4196
+ specs = new ScenarioLoader(this.environment, this.cwd, parameterFile.parameters, this.displayBase).load(target, evalConfig);
4049
4197
  redactor = new SecretRedactor([...evalConfig.secrets, ...specs.flatMap((spec) => spec.secrets)]);
4050
4198
  evaluator = RunCommand.needsEvaluator(specs) ? await this.createEvaluator(evalConfig) : new UnavailableEvaluator();
4051
4199
  } catch (error) {
@@ -4057,7 +4205,11 @@ var RunCommand = class RunCommand {
4057
4205
  const { repeat, minPass, flakyPolicy, scenarioConcurrency } = evalConfig.execution;
4058
4206
  const tokenUsageCapability = evalConfig.websocket.protocol.capabilities.tokenUsage;
4059
4207
  const consoleReport = evalConfig.report.formats.includes(ReportFormat.Console);
4060
- const reporter = new ConsoleReporter(report.verbose, this.colour);
4208
+ const reporter = new ConsoleReporter(report.verbose, this.colour, {
4209
+ nameWidth: ConsoleReporter.nameWidth(specs),
4210
+ columns: this.columns,
4211
+ thresholds: evalConfig.evaluator.thresholds
4212
+ });
4061
4213
  const write = (text) => {
4062
4214
  if (consoleReport) this.out(redactor.redact(text));
4063
4215
  };
@@ -4103,9 +4255,9 @@ var RunCommand = class RunCommand {
4103
4255
  let exitCode = calculator.exitCode(run);
4104
4256
  const reportPaths = [];
4105
4257
  if (evalConfig.report.formats.includes(ReportFormat.Json)) {
4106
- const content = new JsonReporter(redactor, this.cwd).render(run, evalConfig, calculator.count(run), exitCode);
4258
+ const content = new JsonReporter(redactor, this.cwd, this.displayBase).render(run, evalConfig, calculator.count(run), exitCode);
4107
4259
  const writer = new ReportFileWriter(evalConfig.report.directory);
4108
- const path = relative(this.cwd, writer.path(JSON_REPORT_FILE));
4260
+ const path = relative(this.displayBase, writer.path(JSON_REPORT_FILE));
4109
4261
  try {
4110
4262
  writer.write(JSON_REPORT_FILE, content);
4111
4263
  reportPaths.push(path);
@@ -4133,13 +4285,15 @@ var ValidateCommand = class {
4133
4285
  this.err = err;
4134
4286
  this.environment = environment;
4135
4287
  this.cwd = cwd;
4288
+ this.displayBase = displayBase(environment, cwd);
4136
4289
  }
4290
+ displayBase;
4137
4291
  execute(target, config, parameters = "") {
4138
4292
  try {
4139
- const configPath = new ConfigLocator(this.cwd).locate(target, config);
4140
- const values = parameters === "" ? {} : new ParametersLoader(this.cwd).load(parameters).parameters;
4141
- const evalConfig = new EvalConfigLoader(this.environment, this.cwd, values).load(configPath, new ExecutionOverride());
4142
- const specs = new ScenarioLoader(this.environment, this.cwd, values).load(target, evalConfig);
4293
+ const configPath = new ConfigLocator(this.cwd, this.displayBase).locate(target, config);
4294
+ const values = parameters === "" ? {} : new ParametersLoader(this.cwd, this.displayBase).load(parameters).parameters;
4295
+ const evalConfig = new EvalConfigLoader(this.environment, this.cwd, values, this.displayBase).load(configPath, new ExecutionOverride());
4296
+ const specs = new ScenarioLoader(this.environment, this.cwd, values, this.displayBase).load(target, evalConfig);
4143
4297
  this.out(`Valid: ${specs.length} scenario${specs.length === 1 ? "" : "s"}\n`);
4144
4298
  return 0;
4145
4299
  } catch (error) {
@@ -4151,7 +4305,7 @@ var ValidateCommand = class {
4151
4305
  }
4152
4306
  };
4153
4307
  // src/cli/cli.ts
4154
- function createProgram(setExitCode, out, err, colour = false) {
4308
+ function createProgram(setExitCode, out, err, colour = false, columns = void 0) {
4155
4309
  const program = new Command().name("miniwat-eval").description("Black-box, YAML-driven evaluation for agents built on miniwat-agent").configureOutput({
4156
4310
  writeOut: out,
4157
4311
  writeErr: err
@@ -4171,14 +4325,14 @@ function createProgram(setExitCode, out, err, colour = false) {
4171
4325
  setExitCode(1);
4172
4326
  return;
4173
4327
  }
4174
- setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour).execute(target, options.config ?? "", execution, report, options.parameters ?? ""));
4328
+ setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour, columns).execute(target, options.config ?? "", execution, report, options.parameters ?? ""));
4175
4329
  });
4176
4330
  return program;
4177
4331
  }
4178
4332
  // src/main.ts
4179
4333
  const program = createProgram((code) => {
4180
4334
  process.exitCode = code;
4181
- }, (text) => process.stdout.write(text), (text) => process.stderr.write(text), process.stdout.isTTY === true && !process.env.NO_COLOR);
4335
+ }, (text) => process.stdout.write(text), (text) => process.stderr.write(text), process.stdout.isTTY === true && !process.env.NO_COLOR, process.stdout.isTTY === true ? process.stdout.columns : void 0);
4182
4336
  try {
4183
4337
  await program.parseAsync(process.argv);
4184
4338
  } catch (error) {