@mappies/miniwat-eval 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/main.js CHANGED
@@ -218,7 +218,10 @@ var eval_config_schema_default = {
218
218
  "variables": {
219
219
  "type": "object",
220
220
  "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
221
- "additionalProperties": { "type": "string" }
221
+ "additionalProperties": { "anyOf": [{ "type": "string" }, {
222
+ "type": "array",
223
+ "items": { "type": "string" }
224
+ }] }
222
225
  },
223
226
  "websocket": {
224
227
  "type": "object",
@@ -525,7 +528,7 @@ var EvalConfig = class {
525
528
  this.version = data?.version ?? 1;
526
529
  this.configPath = data?.configPath ?? "";
527
530
  this.parametersPath = data?.parametersPath ?? "";
528
- this.variables = { ...data?.variables ?? {} };
531
+ this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
529
532
  this.websocket = new WebSocketConfig(data?.websocket);
530
533
  this.execution = new ExecutionConfig(data?.execution);
531
534
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -611,7 +614,7 @@ var EnvironmentResolver = class EnvironmentResolver {
611
614
  return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
612
615
  }
613
616
  resolve(value, document, issues, secrets = [], options = {}) {
614
- return this.resolveValue(value, [], document, issues, secrets, options);
617
+ return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
615
618
  }
616
619
  static hasReference(value) {
617
620
  return new RegExp(REFERENCE.source).test(value);
@@ -619,7 +622,7 @@ var EnvironmentResolver = class EnvironmentResolver {
619
622
  static isSecretName(name) {
620
623
  return SECRET_NAME.test(name);
621
624
  }
622
- lookup(name) {
625
+ find(name) {
623
626
  for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
624
627
  return this.environment[name];
625
628
  }
@@ -634,7 +637,7 @@ var EnvironmentResolver = class EnvironmentResolver {
634
637
  resolveValue(value, path, document, issues, secrets, options) {
635
638
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
636
639
  options.used?.add(name);
637
- const resolved = this.lookup(name);
640
+ const resolved = this.find(name);
638
641
  if (resolved === void 0) {
639
642
  if (options.optional?.(path)) {
640
643
  options.unresolved?.add(YamlDocument.formatPath(path));
@@ -646,6 +649,13 @@ var EnvironmentResolver = class EnvironmentResolver {
646
649
  }));
647
650
  return reference;
648
651
  }
652
+ if (Array.isArray(resolved)) {
653
+ issues.push(new ValidationIssue({
654
+ message: `variable '${name}' is a list, which can only be used as runWith.from`,
655
+ location: document.locate(path)
656
+ }));
657
+ return reference;
658
+ }
649
659
  if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
650
660
  return resolved;
651
661
  });
@@ -770,9 +780,25 @@ var VariablesReader = class {
770
780
  }));
771
781
  continue;
772
782
  }
783
+ if (Array.isArray(value)) {
784
+ const bad = value.findIndex((item) => typeof item !== "string");
785
+ if (bad >= 0) {
786
+ issues.push(new ValidationIssue({
787
+ message: "must be string",
788
+ location: document.locate([
789
+ "variables",
790
+ name,
791
+ bad
792
+ ])
793
+ }));
794
+ continue;
795
+ }
796
+ variables[name] = [...value];
797
+ continue;
798
+ }
773
799
  if (typeof value !== "string") {
774
800
  issues.push(new ValidationIssue({
775
- message: "must be string",
801
+ message: "must be string or a list of strings",
776
802
  location: document.locate(["variables", name])
777
803
  }));
778
804
  continue;
@@ -780,8 +806,9 @@ var VariablesReader = class {
780
806
  const before = issues.length;
781
807
  const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
782
808
  if (issues.length > before) continue;
783
- variables[name] = resolved.variables[name];
784
- if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
809
+ const text = resolved.variables[name];
810
+ variables[name] = text;
811
+ if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
785
812
  }
786
813
  return variables;
787
814
  }
@@ -1253,6 +1280,7 @@ var TurnSpec = class {
1253
1280
  var ConversationSpec = class {
1254
1281
  version;
1255
1282
  name;
1283
+ runName;
1256
1284
  description;
1257
1285
  turns;
1258
1286
  conversation;
@@ -1262,6 +1290,7 @@ var ConversationSpec = class {
1262
1290
  constructor(data) {
1263
1291
  this.version = data?.version ?? 1;
1264
1292
  this.name = data?.name ?? "";
1293
+ this.runName = data?.runName ?? "";
1265
1294
  this.description = data?.description ?? "";
1266
1295
  this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
1267
1296
  this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
@@ -2903,61 +2932,94 @@ var ExitCodeCalculator = class {
2903
2932
  }
2904
2933
  };
2905
2934
  // src/reporting/consoleReporter.ts
2906
- const STATUS_WIDTH = 11;
2907
- const COLOURS = {
2908
- [ResultStatus.Pass]: "green",
2909
- [ResultStatus.Flaky]: "yellow",
2910
- [ResultStatus.Fail]: "red",
2911
- [ResultStatus.Error]: "red",
2912
- [ResultStatus.Skipped]: "dim",
2913
- [ResultStatus.Interrupted]: "dim"
2935
+ const MARKS = {
2936
+ [ResultStatus.Pass]: {
2937
+ symbol: "✔",
2938
+ colour: "green"
2939
+ },
2940
+ [ResultStatus.Flaky]: {
2941
+ symbol: "⚠",
2942
+ colour: "yellow"
2943
+ },
2944
+ [ResultStatus.Fail]: {
2945
+ symbol: "✖",
2946
+ colour: "red"
2947
+ },
2948
+ [ResultStatus.Error]: {
2949
+ symbol: "●",
2950
+ colour: "red"
2951
+ },
2952
+ [ResultStatus.Skipped]: {
2953
+ symbol: "○",
2954
+ colour: "dim"
2955
+ },
2956
+ [ResultStatus.Interrupted]: {
2957
+ symbol: "◌",
2958
+ colour: "dim"
2959
+ }
2914
2960
  };
2961
+ const MIN_COLUMNS = 40;
2962
+ const RULE_WIDTH = 72;
2915
2963
  var ConsoleReporter = class ConsoleReporter {
2916
2964
  verbose;
2917
2965
  colour;
2918
- constructor(verbose, colour) {
2966
+ nameWidth;
2967
+ columns;
2968
+ thresholds;
2969
+ constructor(verbose, colour, options = {}) {
2919
2970
  this.verbose = verbose;
2920
2971
  this.colour = colour;
2972
+ this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
2973
+ this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
2974
+ this.thresholds = options.thresholds;
2975
+ }
2976
+ static nameWidth(specs) {
2977
+ return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
2921
2978
  }
2922
2979
  header(run) {
2923
2980
  return [
2924
- "Miniwat Eval",
2981
+ " Miniwat Eval",
2925
2982
  "",
2926
- ...run.quick ? ["Execution Mode: Quick"] : [],
2927
- `Repeat: ${run.repeat}`,
2928
- `Minimum Pass: ${run.minPass}`,
2983
+ ...run.quick ? [" Execution mode Quick"] : [],
2984
+ ` Repeat ${run.repeat}`,
2985
+ ` Minimum pass ${run.minPass}`,
2929
2986
  "",
2930
- `Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
2987
+ ` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
2931
2988
  "",
2932
2989
  ""
2933
2990
  ].join("\n");
2934
2991
  }
2935
2992
  scenario(result, repeat, tokenUsageCapability) {
2936
2993
  const spec = result.spec;
2937
- const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
2938
- const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
2994
+ const title = ConsoleReporter.title(spec);
2995
+ const started = result.attempts.length > 0;
2996
+ const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
2997
+ const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
2998
+ const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
2939
2999
  if (!this.verbose) return line;
2940
- if (result.attempts.length === 0) return `${line}\n`;
3000
+ if (!started) return `${line}\n`;
2941
3001
  const websocket = spec.websocket;
2942
3002
  const subprotocols = websocket?.connection.subprotocols.length ?? 0;
2943
- const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
2944
- for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
2945
- lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
2946
- lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
2947
- lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
2948
- lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3003
+ const lines = [];
3004
+ if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
3005
+ for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3006
+ lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
3007
+ lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
2949
3008
  return line + lines.join("\n");
2950
3009
  }
3010
+ static title(spec) {
3011
+ return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
3012
+ }
2951
3013
  summary(run, exitCode, reportPaths = []) {
2952
- const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
3014
+ const started = run.scenarios.filter((result) => result.attempts.length > 0);
3015
+ const failures = started.flatMap((result) => this.blocks(result, false));
3016
+ const flaky = started.flatMap((result) => this.blocks(result, true));
2953
3017
  const lines = this.verbose ? [] : [""];
2954
- if (problems.length > 0) {
2955
- lines.push("Problems", "");
2956
- for (const result of problems) lines.push(...this.problems(result), "");
2957
- }
3018
+ if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
3019
+ if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
2958
3020
  const tests = run.scenarios.flatMap((result) => result.tests);
2959
- lines.push(...run.status === RunStatus.Interrupted ? [`Run: ${run.status}`] : [], `Scenarios: ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, `Tests: ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability), ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true), `Duration: ${ConsoleReporter.duration(run.durationMs)}`, "", "");
2960
- return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
3021
+ lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
3022
+ return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
2961
3023
  }
2962
3024
  static footer(run, exitCode, reportPaths) {
2963
3025
  return [
@@ -2971,57 +3033,135 @@ var ConsoleReporter = class ConsoleReporter {
2971
3033
  return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
2972
3034
  }
2973
3035
  static duration(ms) {
3036
+ if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
2974
3037
  const seconds = Math.floor(ms / 1e3);
2975
3038
  const pad = (value) => String(value).padStart(2, "0");
2976
3039
  const hours = Math.floor(seconds / 3600);
2977
- const rest = `${pad(Math.floor(seconds / 60) % 60)}:${pad(seconds % 60)}`;
2978
- return hours > 0 ? `${hours}:${rest}` : rest;
3040
+ const minutes = Math.floor(seconds / 60) % 60;
3041
+ return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
3042
+ }
3043
+ static wrap(text, width) {
3044
+ const lines = [];
3045
+ for (const paragraph of text.split("\n")) {
3046
+ let line = "";
3047
+ for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
3048
+ lines.push(line);
3049
+ line = word;
3050
+ } else line = line === "" ? word : `${line} ${word}`;
3051
+ lines.push(line);
3052
+ }
3053
+ return lines;
3054
+ }
3055
+ section(name, blocks) {
3056
+ return [
3057
+ ...this.rule(name),
3058
+ "",
3059
+ ...blocks.flatMap((block) => [...block, ""])
3060
+ ];
2979
3061
  }
2980
- problems(result) {
3062
+ rule(name) {
3063
+ const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
3064
+ return [
3065
+ rule,
3066
+ ` ${name}`,
3067
+ rule
3068
+ ];
3069
+ }
3070
+ blocks(result, flaky) {
2981
3071
  const spec = result.spec;
2982
- const lines = [`${this.status(result.status, 6)} ${spec.name} (${spec.location.file})`];
2983
- for (const attempt of result.attempts) {
2984
- const execution = attempt.execution;
2985
- if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
2986
- for (const error of execution.attemptErrors) lines.push(` Attempt ${attempt.attempt} error: ${error.category} error: ${error.message}`);
2987
- if (execution.closeError !== void 0) lines.push(` Attempt ${attempt.attempt} close error: ${execution.closeError.message}`);
2988
- }
2989
- for (const test of result.tests.filter((entry) => entry.status !== ResultStatus.Pass)) {
2990
- lines.push(` ${this.status(test.status, 6)} ${ConsoleReporter.testTitle(test)} ${test.location.toString()} ${ConsoleReporter.counts(test)}`);
2991
- test.history.forEach((entry, index) => {
2992
- if (entry.status === ResultStatus.Pass) return;
2993
- const prefix = ` Attempt ${result.attempts[index].attempt} ${this.status(entry.status)}`;
2994
- const width = ` Attempt ${result.attempts[index].attempt} ${entry.status}: `.length;
2995
- if (entry.details.length === 0) {
2996
- lines.push(prefix);
2997
- return;
2998
- }
2999
- lines.push(`${prefix}: ${ConsoleReporter.indent(entry.details[0], width)}`, ...entry.details.slice(1).map((detail) => `${" ".repeat(width)}${ConsoleReporter.indent(detail, width)}`));
3000
- });
3072
+ const title = ConsoleReporter.title(spec);
3073
+ const blocks = [];
3074
+ if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
3075
+ const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
3076
+ for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
3077
+ return blocks;
3078
+ }
3079
+ attemptBlock(spec, title, attempt) {
3080
+ const execution = attempt.execution;
3081
+ const problems = [];
3082
+ if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
3083
+ for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
3084
+ if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
3085
+ if (problems.length === 0) return [];
3086
+ const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
3087
+ const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
3088
+ return [[
3089
+ ` scenario at ${spec.location.file}`,
3090
+ ` ${this.mark(status)} ${title} › ${what}`,
3091
+ ...problems.flatMap((problem) => this.paragraph(problem, 3))
3092
+ ]];
3093
+ }
3094
+ testBlock(spec, title, test, result) {
3095
+ const location = test.location;
3096
+ const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
3097
+ const failing = test.history.map((entry, index) => ({
3098
+ entry,
3099
+ attempt: result.attempts[index]?.attempt ?? index + 1
3100
+ })).filter(({ entry }) => entry.status !== ResultStatus.Pass);
3101
+ const several = test.history.length > 1;
3102
+ for (const { entry, attempt } of failing) {
3103
+ if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
3104
+ lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
3105
+ }
3106
+ lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
3107
+ return lines;
3108
+ }
3109
+ entryBody(spec, entry, indent) {
3110
+ const pad = " ".repeat(indent);
3111
+ const evaluation = entry.evaluation;
3112
+ if (evaluation === void 0) {
3113
+ if (entry.details.length === 0) return [];
3114
+ return [
3115
+ "",
3116
+ `${pad}Details`,
3117
+ ...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
3118
+ ];
3001
3119
  }
3120
+ const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
3121
+ const expected = ConsoleReporter.expected(spec, entry);
3122
+ const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
3123
+ if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
3124
+ if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
3125
+ if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
3002
3126
  return lines;
3003
3127
  }
3128
+ static expected(spec, entry) {
3129
+ if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
3130
+ return spec.turns[entry.turn - 1]?.expect.response?.criteria;
3131
+ }
3132
+ paragraph(text, indent) {
3133
+ const pad = " ".repeat(indent);
3134
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
3135
+ }
3136
+ bullet(text, indent) {
3137
+ const pad = " ".repeat(indent);
3138
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
3139
+ }
3004
3140
  static testTitle(test) {
3005
3141
  return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
3006
3142
  }
3007
3143
  static tally(statuses) {
3008
3144
  const count = (status) => statuses.filter((entry) => entry === status).length;
3009
- const interrupted = count(ResultStatus.Interrupted);
3010
- return `${count(ResultStatus.Pass)} passed, ${count(ResultStatus.Flaky)} flaky, ${count(ResultStatus.Fail)} failed, ${count(ResultStatus.Error)} errors${interrupted > 0 ? `, ${interrupted} interrupted` : ""}`;
3011
- }
3012
- status(status, width = 0) {
3013
- const padding = " ".repeat(Math.max(0, width - status.length));
3014
- return (this.colour ? styleText(COLOURS[status], status, { validateStream: false }) : status) + padding;
3015
- }
3016
- static counts(result) {
3017
- return `(${result.passed} passed, ${result.failed} failed, ${result.errors} errors; required ${result.required})`;
3145
+ const parts = [
3146
+ [ResultStatus.Pass, "passed"],
3147
+ [ResultStatus.Flaky, "flaky"],
3148
+ [ResultStatus.Fail, "failed"],
3149
+ [ResultStatus.Error, "errors"],
3150
+ [ResultStatus.Interrupted, "interrupted"]
3151
+ ].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
3152
+ return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
3153
+ }
3154
+ mark(status) {
3155
+ const { symbol, colour } = MARKS[status];
3156
+ return this.colour ? styleText(colour, symbol, { validateStream: false }) : symbol;
3018
3157
  }
3019
3158
  static usage(label, usage, capability) {
3020
- if (!capability) return `${label}: n/a (not supported by host)`;
3021
- if (usage.expected === 0) return `${label}: none`;
3022
- if (usage.total === void 0) return `${label}: n/a (coverage 0/${usage.expected})`;
3159
+ const name = label.padEnd(16);
3160
+ if (!capability) return `${name} n/a (not supported by host)`;
3161
+ if (usage.expected === 0) return `${name} none`;
3162
+ if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
3023
3163
  const number = (value) => value.toLocaleString("en-US");
3024
- const total = `${label}: total ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3164
+ const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3025
3165
  return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
3026
3166
  }
3027
3167
  formatAttempt(spec, attempt) {
@@ -3033,28 +3173,28 @@ var ConsoleReporter = class ConsoleReporter {
3033
3173
  lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
3034
3174
  lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
3035
3175
  });
3036
- if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3176
+ if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3037
3177
  const notRun = attempt.tests.filter((test) => test.turn > ran);
3038
- if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3178
+ if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3039
3179
  lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
3040
- for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3041
- if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3042
- lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3180
+ for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3181
+ if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3182
+ lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3043
3183
  return lines;
3044
3184
  }
3045
3185
  formatTest(test) {
3046
- return [` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${detail}`)];
3186
+ return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
3047
3187
  }
3048
3188
  static formatTurn(number, input, turn) {
3049
3189
  const lines = [
3050
- ` Turn ${number} (${turn.durationMs} ms)`,
3051
- ` Input: ${ConsoleReporter.indent(input, 17)}`,
3052
- ` Response: ${ConsoleReporter.indent(turn.response, 17)}`
3190
+ ` Turn ${number} (${turn.durationMs} ms)`,
3191
+ ` Input: ${ConsoleReporter.indent(input, 22)}`,
3192
+ ` Response: ${ConsoleReporter.indent(turn.response, 22)}`
3053
3193
  ];
3054
- for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3055
- for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3056
- for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 17)}`);
3057
- if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3194
+ for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3195
+ for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3196
+ for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
3197
+ if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3058
3198
  return lines;
3059
3199
  }
3060
3200
  static indent(text, width) {
@@ -3156,6 +3296,7 @@ var JsonReporter = class JsonReporter {
3156
3296
  const capability = websocket.protocol.capabilities.tokenUsage;
3157
3297
  return {
3158
3298
  name: result.spec.name,
3299
+ runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
3159
3300
  file: result.spec.location.file,
3160
3301
  status: result.status,
3161
3302
  connection: {
@@ -3325,7 +3466,44 @@ var conversation_schema_default = {
3325
3466
  "variables": {
3326
3467
  "type": "object",
3327
3468
  "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
3328
- "additionalProperties": { "type": "string" }
3469
+ "additionalProperties": { "anyOf": [{ "type": "string" }, {
3470
+ "type": "array",
3471
+ "items": { "type": "string" }
3472
+ }] }
3473
+ },
3474
+ "runWith": {
3475
+ "description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
3476
+ "anyOf": [{
3477
+ "type": "array",
3478
+ "minItems": 1,
3479
+ "items": {
3480
+ "type": "object",
3481
+ "required": ["name"],
3482
+ "properties": { "name": {
3483
+ "type": "string",
3484
+ "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
3485
+ } },
3486
+ "additionalProperties": { "type": "string" }
3487
+ }
3488
+ }, {
3489
+ "type": "object",
3490
+ "additionalProperties": false,
3491
+ "required": ["variable", "from"],
3492
+ "properties": {
3493
+ "variable": {
3494
+ "type": "string",
3495
+ "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
3496
+ },
3497
+ "from": { "anyOf": [{
3498
+ "type": "array",
3499
+ "minItems": 1,
3500
+ "items": { "type": "string" }
3501
+ }, {
3502
+ "type": "string",
3503
+ "pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
3504
+ }] }
3505
+ }
3506
+ }]
3329
3507
  },
3330
3508
  "turns": {
3331
3509
  "type": "array",
@@ -3484,6 +3662,221 @@ var ConversationValidator = class ConversationValidator {
3484
3662
  return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
3485
3663
  }
3486
3664
  };
3665
+ // src/scenarios/runWithReader.ts
3666
+ const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
3667
+ const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
3668
+ const FORMS = "must be a list of rows, or an object with variable and from";
3669
+ const SINGLE_RUN = {
3670
+ name: "",
3671
+ variables: {},
3672
+ keys: [],
3673
+ location: [],
3674
+ secrets: []
3675
+ };
3676
+ var RunWithReader = class RunWithReader {
3677
+ read(document, resolver, issues, used) {
3678
+ const raw = isRecord(document.value) ? document.value.runWith : void 0;
3679
+ if (raw === void 0) return void 0;
3680
+ if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
3681
+ if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
3682
+ issues.push(new ValidationIssue({
3683
+ message: FORMS,
3684
+ location: document.locate(["runWith"])
3685
+ }));
3686
+ return [];
3687
+ }
3688
+ rows(rows, document, resolver, issues, used) {
3689
+ if (rows.length === 0) {
3690
+ issues.push(new ValidationIssue({
3691
+ message: "must not be empty",
3692
+ location: document.locate(["runWith"])
3693
+ }));
3694
+ return [];
3695
+ }
3696
+ const names = /* @__PURE__ */ new Set();
3697
+ const runs = [];
3698
+ rows.forEach((row, index) => {
3699
+ const path = ["runWith", index];
3700
+ if (!isRecord(row)) {
3701
+ issues.push(new ValidationIssue({
3702
+ message: "must be object",
3703
+ location: document.locate(path)
3704
+ }));
3705
+ return;
3706
+ }
3707
+ const name = row.name;
3708
+ if (name === void 0) issues.push(new ValidationIssue({
3709
+ message: "missing required key 'name'",
3710
+ location: document.locate(path)
3711
+ }));
3712
+ else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
3713
+ message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
3714
+ location: document.locate([...path, "name"])
3715
+ }));
3716
+ else if (names.has(name)) issues.push(new ValidationIssue({
3717
+ message: `duplicate run name '${name}'`,
3718
+ location: document.locate([...path, "name"])
3719
+ }));
3720
+ else names.add(name);
3721
+ const run = {
3722
+ name: typeof name === "string" ? name : "",
3723
+ variables: {},
3724
+ keys: [],
3725
+ location: path,
3726
+ secrets: []
3727
+ };
3728
+ for (const [key, value] of Object.entries(row)) {
3729
+ if (key === "name") continue;
3730
+ if (!VARIABLE_NAME.test(key)) {
3731
+ issues.push(new ValidationIssue({
3732
+ message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
3733
+ location: document.locateKey(path, key)
3734
+ }));
3735
+ continue;
3736
+ }
3737
+ if (typeof value !== "string") {
3738
+ issues.push(new ValidationIssue({
3739
+ message: "must be string",
3740
+ location: document.locate([...path, key])
3741
+ }));
3742
+ continue;
3743
+ }
3744
+ const resolved = resolver.resolve(value, document, issues, run.secrets, {
3745
+ used,
3746
+ at: [...path, key]
3747
+ });
3748
+ RunWithReader.set(run, key, resolved, document.locateKey(path, key));
3749
+ }
3750
+ runs.push(run);
3751
+ });
3752
+ return runs;
3753
+ }
3754
+ list(form, document, resolver, issues, used) {
3755
+ const before = issues.length;
3756
+ for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
3757
+ message: `unknown key '${key}'`,
3758
+ location: document.locateKey(["runWith"], key)
3759
+ }));
3760
+ const variable = form.variable;
3761
+ if (variable === void 0) issues.push(new ValidationIssue({
3762
+ message: "missing required key 'variable'",
3763
+ location: document.locate(["runWith"])
3764
+ }));
3765
+ else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
3766
+ message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
3767
+ location: document.locate(["runWith", "variable"])
3768
+ }));
3769
+ const entries = this.entries(form.from, document, resolver, issues, used);
3770
+ if (issues.length > before || entries === void 0) return [];
3771
+ const name = variable;
3772
+ const keyLocation = document.locate(["runWith", "variable"]);
3773
+ const names = /* @__PURE__ */ new Set();
3774
+ const runs = [];
3775
+ for (const { entry, path } of entries) {
3776
+ if (!VARIABLE_NAME.test(entry)) {
3777
+ issues.push(new ValidationIssue({
3778
+ message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
3779
+ location: document.locate(path)
3780
+ }));
3781
+ continue;
3782
+ }
3783
+ if (names.has(entry)) {
3784
+ issues.push(new ValidationIssue({
3785
+ message: `duplicate run name '${entry}'`,
3786
+ location: document.locate(path)
3787
+ }));
3788
+ continue;
3789
+ }
3790
+ names.add(entry);
3791
+ const run = {
3792
+ name: entry,
3793
+ variables: {},
3794
+ keys: [],
3795
+ location: path,
3796
+ secrets: []
3797
+ };
3798
+ const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
3799
+ used,
3800
+ at: path
3801
+ });
3802
+ RunWithReader.set(run, name, value, keyLocation);
3803
+ runs.push(run);
3804
+ }
3805
+ return runs;
3806
+ }
3807
+ entries(from, document, resolver, issues, used) {
3808
+ const path = ["runWith", "from"];
3809
+ const location = document.locate(path);
3810
+ if (from === void 0) {
3811
+ issues.push(new ValidationIssue({
3812
+ message: "missing required key 'from'",
3813
+ location: document.locate(["runWith"])
3814
+ }));
3815
+ return;
3816
+ }
3817
+ let list;
3818
+ let at;
3819
+ if (Array.isArray(from)) {
3820
+ list = from;
3821
+ at = (index) => [...path, index];
3822
+ } else {
3823
+ const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
3824
+ if (reference === null) {
3825
+ issues.push(new ValidationIssue({
3826
+ message: "must be a list, or a list variable such as ${tokens}",
3827
+ location
3828
+ }));
3829
+ return;
3830
+ }
3831
+ const name = reference[1];
3832
+ const found = resolver.find(name);
3833
+ used.add(name);
3834
+ if (found === void 0) {
3835
+ issues.push(new ValidationIssue({
3836
+ message: `list variable '${name}' is not set`,
3837
+ location
3838
+ }));
3839
+ return;
3840
+ }
3841
+ if (!Array.isArray(found)) {
3842
+ issues.push(new ValidationIssue({
3843
+ message: `variable '${name}' is not a list`,
3844
+ location
3845
+ }));
3846
+ return;
3847
+ }
3848
+ list = found;
3849
+ at = () => path;
3850
+ }
3851
+ if (list.length === 0) {
3852
+ issues.push(new ValidationIssue({
3853
+ message: "must not be empty",
3854
+ location
3855
+ }));
3856
+ return;
3857
+ }
3858
+ const bad = list.findIndex((item) => typeof item !== "string");
3859
+ if (bad >= 0) {
3860
+ issues.push(new ValidationIssue({
3861
+ message: "must be string",
3862
+ location: document.locate(at(bad))
3863
+ }));
3864
+ return;
3865
+ }
3866
+ return list.map((entry, index) => ({
3867
+ entry,
3868
+ path: at(index)
3869
+ }));
3870
+ }
3871
+ static set(run, name, value, location) {
3872
+ run.variables[name] = value;
3873
+ run.keys.push({
3874
+ name,
3875
+ location
3876
+ });
3877
+ if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
3878
+ }
3879
+ };
3487
3880
  // src/scenarios/scenarioLoader.ts
3488
3881
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
3489
3882
  var ScenarioLoader = class ScenarioLoader {
@@ -3492,6 +3885,7 @@ var ScenarioLoader = class ScenarioLoader {
3492
3885
  schemaValidator = new SchemaValidator(conversation_schema_default);
3493
3886
  conversationValidator = new ConversationValidator();
3494
3887
  variablesReader = new VariablesReader();
3888
+ runWithReader = new RunWithReader();
3495
3889
  webSocketResolver = new WebSocketResolver();
3496
3890
  environmentResolver;
3497
3891
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
@@ -3507,45 +3901,59 @@ var ScenarioLoader = class ScenarioLoader {
3507
3901
  for (const file of files) {
3508
3902
  const document = this.documentLoader.load(file, relative(this.cwd, file));
3509
3903
  const fileIssues = [...document.errors];
3510
- const secrets = [];
3904
+ const fileSecrets = [];
3511
3905
  const used = /* @__PURE__ */ new Set();
3512
- const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
3513
- const resolver = this.environmentResolver.withLevels([variables, config.variables]);
3514
- let value;
3515
- if (fileIssues.length === 0) {
3516
- const { variables: raw, ...rest } = document.value;
3517
- value = {
3518
- ...resolver.resolve(rest, document, fileIssues, secrets, { used }),
3519
- ...raw === void 0 ? {} : { variables }
3520
- };
3906
+ const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
3907
+ const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
3908
+ const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
3909
+ if (fileIssues.length > 0) {
3910
+ issues.push(...fileIssues);
3911
+ continue;
3521
3912
  }
3522
- let websocket;
3523
- const websocketIssues = [];
3524
- if (fileIssues.length === 0 && configDocument !== void 0) {
3525
- const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
3526
- if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
3527
- connectionIssues.push({
3528
- file: document,
3529
- issues: websocketIssues
3530
- });
3531
- if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
3913
+ const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
3914
+ const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
3915
+ for (const result of results) for (const name of result.used) used.add(name);
3916
+ fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
3917
+ run: result.run,
3918
+ issues: result.issues
3919
+ })), (issue, run) => run === void 0 ? issue : new ValidationIssue({
3920
+ message: `run '${run.name}': ${issue.message}`,
3921
+ location: issue.location
3922
+ })));
3923
+ const attempted = results.filter((result) => result.connectionChecked);
3924
+ if (attempted.length > 0) connectionIssues.push({
3925
+ file: document,
3926
+ runs: attempted.map((result) => ({
3927
+ run: result.run,
3928
+ issues: result.websocketIssues
3929
+ }))
3930
+ });
3931
+ if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
3932
+ if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
3933
+ const websocket = result.websocket;
3934
+ specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
3532
3935
  }
3533
- if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
3534
- if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
3535
- if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
3536
- if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
3537
3936
  else issues.push(...fileIssues);
3538
3937
  }
3539
- const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
3938
+ const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
3540
3939
  if (shared.length > 0) throw new EvalError({
3541
3940
  category: ErrorCategory.Configuration,
3542
3941
  message: "invalid configuration",
3543
3942
  issues: shared
3544
3943
  });
3545
- for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
3546
- message: issue.toString(),
3547
- location: ScenarioLoader.variablesLocation(entry.file)
3548
- }));
3944
+ for (const entry of connectionIssues) {
3945
+ const fileLocation = ScenarioLoader.variablesLocation(entry.file);
3946
+ issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
3947
+ message: issue.toString(),
3948
+ location: fileLocation
3949
+ }) : new ValidationIssue({
3950
+ message: `run '${run.name}': ${issue.toString()}`,
3951
+ location: new SourceLocation({
3952
+ ...entry.file.locate(run.location),
3953
+ path: ""
3954
+ })
3955
+ })));
3956
+ }
3549
3957
  if (issues.length > 0) throw new EvalError({
3550
3958
  category: ErrorCategory.Specification,
3551
3959
  message: "invalid scenario specification",
@@ -3553,6 +3961,43 @@ var ScenarioLoader = class ScenarioLoader {
3553
3961
  });
3554
3962
  return specs;
3555
3963
  }
3964
+ resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
3965
+ const issues = [];
3966
+ const used = /* @__PURE__ */ new Set();
3967
+ const resolver = scenarioResolver.withLevels([run.variables]);
3968
+ const value = {
3969
+ ...resolver.resolve(rest, document, issues, secrets, { used }),
3970
+ ...variables === void 0 ? {} : { variables }
3971
+ };
3972
+ const websocketIssues = [];
3973
+ let websocket;
3974
+ let connectionChecked = false;
3975
+ if (issues.length === 0 && configDocument !== void 0) {
3976
+ const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
3977
+ if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
3978
+ if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
3979
+ connectionChecked = true;
3980
+ }
3981
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
3982
+ if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
3983
+ return {
3984
+ run,
3985
+ value,
3986
+ used,
3987
+ secrets,
3988
+ issues,
3989
+ websocket,
3990
+ websocketIssues,
3991
+ connectionChecked
3992
+ };
3993
+ }
3994
+ static byRun(runs, place) {
3995
+ const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
3996
+ const sharedTexts = new Set(shared.map((issue) => issue.toString()));
3997
+ const result = shared.map((issue) => place(issue, void 0));
3998
+ for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
3999
+ return result;
4000
+ }
3556
4001
  static shared(lists) {
3557
4002
  if (lists.length === 0) return [];
3558
4003
  const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
@@ -3564,11 +4009,22 @@ var ScenarioLoader = class ScenarioLoader {
3564
4009
  path: ""
3565
4010
  });
3566
4011
  }
3567
- static unusedVariables(variables, used, document) {
3568
- return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
3569
- message: `variable '${name}' is not used by the scenario or the websocket configuration`,
4012
+ static unusedVariables(variables, used, results, document) {
4013
+ const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
4014
+ const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
4015
+ message: message(name),
3570
4016
  location: document.locateKey(["variables"], name)
3571
4017
  }));
4018
+ const seen = /* @__PURE__ */ new Set();
4019
+ for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
4020
+ const issue = new ValidationIssue({
4021
+ message: message(key.name),
4022
+ location: key.location
4023
+ });
4024
+ if (!seen.has(issue.toString())) issues.push(issue);
4025
+ seen.add(issue.toString());
4026
+ }
4027
+ return issues;
3572
4028
  }
3573
4029
  discover(target, config) {
3574
4030
  if (!existsSync(target)) throw new EvalError({
@@ -3594,13 +4050,14 @@ var ScenarioLoader = class ScenarioLoader {
3594
4050
  }
3595
4051
  return files;
3596
4052
  }
3597
- build(value, document, websocket, secrets) {
4053
+ build(value, document, runName, websocket, secrets) {
3598
4054
  const scenario = value;
3599
4055
  const turns = scenario.turns;
3600
4056
  const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
3601
4057
  return new ConversationSpec({
3602
4058
  version: scenario.version,
3603
4059
  name: scenario.name,
4060
+ runName,
3604
4061
  description: typeof scenario.description === "string" ? scenario.description : "",
3605
4062
  turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
3606
4063
  conversation: conversation ? new ConversationExpectation({
@@ -3673,7 +4130,8 @@ var RunCommand = class RunCommand {
3673
4130
  createEvaluator;
3674
4131
  signals;
3675
4132
  colour;
3676
- constructor(out, err, environment = process.env, cwd = process.cwd(), createRunner = (config, evaluator) => new ConversationRunner(config, evaluator), createEvaluator = createLlmEvaluator, signals = process, colour = false) {
4133
+ columns;
4134
+ constructor(out, err, environment = process.env, cwd = process.cwd(), createRunner = (config, evaluator) => new ConversationRunner(config, evaluator), createEvaluator = createLlmEvaluator, signals = process, colour = false, columns = void 0) {
3677
4135
  this.out = out;
3678
4136
  this.err = err;
3679
4137
  this.environment = environment;
@@ -3682,6 +4140,7 @@ var RunCommand = class RunCommand {
3682
4140
  this.createEvaluator = createEvaluator;
3683
4141
  this.signals = signals;
3684
4142
  this.colour = colour;
4143
+ this.columns = columns;
3685
4144
  }
3686
4145
  async execute(target, config, execution = new ExecutionOverride(), report = new ReportOverride(), parameters = "") {
3687
4146
  let evalConfig;
@@ -3708,7 +4167,11 @@ var RunCommand = class RunCommand {
3708
4167
  const { repeat, minPass, flakyPolicy, scenarioConcurrency } = evalConfig.execution;
3709
4168
  const tokenUsageCapability = evalConfig.websocket.protocol.capabilities.tokenUsage;
3710
4169
  const consoleReport = evalConfig.report.formats.includes(ReportFormat.Console);
3711
- const reporter = new ConsoleReporter(report.verbose, this.colour);
4170
+ const reporter = new ConsoleReporter(report.verbose, this.colour, {
4171
+ nameWidth: ConsoleReporter.nameWidth(specs),
4172
+ columns: this.columns,
4173
+ thresholds: evalConfig.evaluator.thresholds
4174
+ });
3712
4175
  const write = (text) => {
3713
4176
  if (consoleReport) this.out(redactor.redact(text));
3714
4177
  };
@@ -3802,7 +4265,7 @@ var ValidateCommand = class {
3802
4265
  }
3803
4266
  };
3804
4267
  // src/cli/cli.ts
3805
- function createProgram(setExitCode, out, err, colour = false) {
4268
+ function createProgram(setExitCode, out, err, colour = false, columns = void 0) {
3806
4269
  const program = new Command().name("miniwat-eval").description("Black-box, YAML-driven evaluation for agents built on miniwat-agent").configureOutput({
3807
4270
  writeOut: out,
3808
4271
  writeErr: err
@@ -3822,14 +4285,14 @@ function createProgram(setExitCode, out, err, colour = false) {
3822
4285
  setExitCode(1);
3823
4286
  return;
3824
4287
  }
3825
- setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour).execute(target, options.config ?? "", execution, report, options.parameters ?? ""));
4288
+ setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour, columns).execute(target, options.config ?? "", execution, report, options.parameters ?? ""));
3826
4289
  });
3827
4290
  return program;
3828
4291
  }
3829
4292
  // src/main.ts
3830
4293
  const program = createProgram((code) => {
3831
4294
  process.exitCode = code;
3832
- }, (text) => process.stdout.write(text), (text) => process.stderr.write(text), process.stdout.isTTY === true && !process.env.NO_COLOR);
4295
+ }, (text) => process.stdout.write(text), (text) => process.stderr.write(text), process.stdout.isTTY === true && !process.env.NO_COLOR, process.stdout.isTTY === true ? process.stdout.columns : void 0);
3833
4296
  try {
3834
4297
  await program.parseAsync(process.argv);
3835
4298
  } catch (error) {