@mappies/miniwat-eval 1.3.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -5
- package/dist/index.cjs +234 -100
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +40 -12
- package/dist/index.d.ts +40 -12
- package/dist/index.js +235 -101
- package/dist/index.js.map +1 -1
- package/dist/main.js +269 -115
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
package/dist/main.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { Command, CommanderError } from "commander";
|
|
3
|
-
import { dirname, extname, join, relative, resolve } from "node:path";
|
|
3
|
+
import { dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
|
|
4
4
|
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
5
5
|
import { LineCounter, isMap, isPair, isScalar, isSeq, parseDocument } from "yaml";
|
|
6
6
|
import { Ajv } from "ajv";
|
|
@@ -122,15 +122,17 @@ const CONFIG_FILE_NAME = "eval-config.yml";
|
|
|
122
122
|
const PROJECT_ROOT_MARKERS = [".git", "package.json"];
|
|
123
123
|
var ConfigLocator = class ConfigLocator {
|
|
124
124
|
cwd;
|
|
125
|
-
|
|
125
|
+
displayBase;
|
|
126
|
+
constructor(cwd = process.cwd(), displayBase = cwd) {
|
|
126
127
|
this.cwd = cwd;
|
|
128
|
+
this.displayBase = displayBase;
|
|
127
129
|
}
|
|
128
130
|
locate(target, explicitConfig) {
|
|
129
131
|
if (explicitConfig !== "") {
|
|
130
132
|
const path = resolve(this.cwd, explicitConfig);
|
|
131
133
|
if (!ConfigLocator.isFile(path)) throw new EvalError({
|
|
132
134
|
category: ErrorCategory.Configuration,
|
|
133
|
-
message: `configuration file not found: ${explicitConfig}`
|
|
135
|
+
message: `configuration file not found: ${relative(this.displayBase, path) || explicitConfig}`
|
|
134
136
|
});
|
|
135
137
|
return path;
|
|
136
138
|
}
|
|
@@ -139,7 +141,7 @@ var ConfigLocator = class ConfigLocator {
|
|
|
139
141
|
const found = ConfigLocator.searchUpward(start) ?? ConfigLocator.candidate(this.cwd);
|
|
140
142
|
if (found === void 0) throw new EvalError({
|
|
141
143
|
category: ErrorCategory.Configuration,
|
|
142
|
-
message: `no ${CONFIG_FILE_NAME} found above ${relative(this.
|
|
144
|
+
message: `no ${CONFIG_FILE_NAME} found above ${relative(this.displayBase, targetPath) || "."} or in the working directory; pass --config <path>`
|
|
143
145
|
});
|
|
144
146
|
return found;
|
|
145
147
|
}
|
|
@@ -171,12 +173,14 @@ function isRecord(value) {
|
|
|
171
173
|
// src/config/parametersLoader.ts
|
|
172
174
|
var ParametersLoader = class ParametersLoader {
|
|
173
175
|
cwd;
|
|
174
|
-
|
|
176
|
+
displayBase;
|
|
177
|
+
constructor(cwd = process.cwd(), displayBase = cwd) {
|
|
175
178
|
this.cwd = cwd;
|
|
179
|
+
this.displayBase = displayBase;
|
|
176
180
|
}
|
|
177
181
|
load(path) {
|
|
178
182
|
const absolute = resolve(this.cwd, path);
|
|
179
|
-
const shown = relative(this.
|
|
183
|
+
const shown = relative(this.displayBase, absolute) || path;
|
|
180
184
|
let text;
|
|
181
185
|
let value;
|
|
182
186
|
try {
|
|
@@ -928,17 +932,19 @@ const DEFAULT_CONFIG = new EvalConfig({
|
|
|
928
932
|
// src/config/evalConfigLoader.ts
|
|
929
933
|
var EvalConfigLoader = class EvalConfigLoader {
|
|
930
934
|
cwd;
|
|
935
|
+
displayBase;
|
|
931
936
|
documentLoader = new YamlDocumentLoader();
|
|
932
937
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
933
938
|
variablesReader = new VariablesReader();
|
|
934
939
|
webSocketResolver = new WebSocketResolver();
|
|
935
940
|
environmentResolver;
|
|
936
|
-
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
941
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
|
|
937
942
|
this.cwd = cwd;
|
|
943
|
+
this.displayBase = displayBase;
|
|
938
944
|
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
939
945
|
}
|
|
940
946
|
load(configPath, override, report = new ReportOverride()) {
|
|
941
|
-
const document = this.documentLoader.load(configPath, relative(this.
|
|
947
|
+
const document = this.documentLoader.load(configPath, relative(this.displayBase, configPath));
|
|
942
948
|
const issues = [...document.errors];
|
|
943
949
|
if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
|
|
944
950
|
const named = [];
|
|
@@ -2932,64 +2938,94 @@ var ExitCodeCalculator = class {
|
|
|
2932
2938
|
}
|
|
2933
2939
|
};
|
|
2934
2940
|
// src/reporting/consoleReporter.ts
|
|
2935
|
-
const
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
[ResultStatus.
|
|
2941
|
-
|
|
2942
|
-
|
|
2941
|
+
const MARKS = {
|
|
2942
|
+
[ResultStatus.Pass]: {
|
|
2943
|
+
symbol: "✔",
|
|
2944
|
+
colour: "green"
|
|
2945
|
+
},
|
|
2946
|
+
[ResultStatus.Flaky]: {
|
|
2947
|
+
symbol: "⚠",
|
|
2948
|
+
colour: "yellow"
|
|
2949
|
+
},
|
|
2950
|
+
[ResultStatus.Fail]: {
|
|
2951
|
+
symbol: "✖",
|
|
2952
|
+
colour: "red"
|
|
2953
|
+
},
|
|
2954
|
+
[ResultStatus.Error]: {
|
|
2955
|
+
symbol: "●",
|
|
2956
|
+
colour: "red"
|
|
2957
|
+
},
|
|
2958
|
+
[ResultStatus.Skipped]: {
|
|
2959
|
+
symbol: "○",
|
|
2960
|
+
colour: "dim"
|
|
2961
|
+
},
|
|
2962
|
+
[ResultStatus.Interrupted]: {
|
|
2963
|
+
symbol: "◌",
|
|
2964
|
+
colour: "dim"
|
|
2965
|
+
}
|
|
2943
2966
|
};
|
|
2967
|
+
const MIN_COLUMNS = 40;
|
|
2968
|
+
const RULE_WIDTH = 72;
|
|
2944
2969
|
var ConsoleReporter = class ConsoleReporter {
|
|
2945
2970
|
verbose;
|
|
2946
2971
|
colour;
|
|
2947
|
-
|
|
2972
|
+
nameWidth;
|
|
2973
|
+
columns;
|
|
2974
|
+
thresholds;
|
|
2975
|
+
constructor(verbose, colour, options = {}) {
|
|
2948
2976
|
this.verbose = verbose;
|
|
2949
2977
|
this.colour = colour;
|
|
2978
|
+
this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
|
|
2979
|
+
this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
|
|
2980
|
+
this.thresholds = options.thresholds;
|
|
2981
|
+
}
|
|
2982
|
+
static nameWidth(specs) {
|
|
2983
|
+
return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
|
|
2950
2984
|
}
|
|
2951
2985
|
header(run) {
|
|
2952
2986
|
return [
|
|
2953
|
-
"Miniwat Eval",
|
|
2987
|
+
" Miniwat Eval",
|
|
2954
2988
|
"",
|
|
2955
|
-
...run.quick ? ["Execution
|
|
2956
|
-
`Repeat
|
|
2957
|
-
`Minimum
|
|
2989
|
+
...run.quick ? [" Execution mode Quick"] : [],
|
|
2990
|
+
` Repeat ${run.repeat}`,
|
|
2991
|
+
` Minimum pass ${run.minPass}`,
|
|
2958
2992
|
"",
|
|
2959
|
-
`Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
2993
|
+
` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
2960
2994
|
"",
|
|
2961
2995
|
""
|
|
2962
2996
|
].join("\n");
|
|
2963
2997
|
}
|
|
2964
2998
|
scenario(result, repeat, tokenUsageCapability) {
|
|
2965
2999
|
const spec = result.spec;
|
|
2966
|
-
const
|
|
2967
|
-
const
|
|
3000
|
+
const title = ConsoleReporter.title(spec);
|
|
3001
|
+
const started = result.attempts.length > 0;
|
|
3002
|
+
const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
|
|
3003
|
+
const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
|
|
3004
|
+
const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
|
|
2968
3005
|
if (!this.verbose) return line;
|
|
2969
|
-
if (
|
|
3006
|
+
if (!started) return `${line}\n`;
|
|
2970
3007
|
const websocket = spec.websocket;
|
|
2971
3008
|
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
2972
|
-
const lines = [
|
|
2973
|
-
|
|
2974
|
-
lines.push(`
|
|
2975
|
-
lines.push(
|
|
2976
|
-
lines.push(`
|
|
2977
|
-
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3009
|
+
const lines = [];
|
|
3010
|
+
if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
|
|
3011
|
+
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
3012
|
+
lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
|
|
3013
|
+
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
2978
3014
|
return line + lines.join("\n");
|
|
2979
3015
|
}
|
|
2980
3016
|
static title(spec) {
|
|
2981
3017
|
return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
|
|
2982
3018
|
}
|
|
2983
3019
|
summary(run, exitCode, reportPaths = []) {
|
|
2984
|
-
const
|
|
3020
|
+
const started = run.scenarios.filter((result) => result.attempts.length > 0);
|
|
3021
|
+
const failures = started.flatMap((result) => this.blocks(result, false));
|
|
3022
|
+
const flaky = started.flatMap((result) => this.blocks(result, true));
|
|
2985
3023
|
const lines = this.verbose ? [] : [""];
|
|
2986
|
-
if (
|
|
2987
|
-
|
|
2988
|
-
for (const result of problems) lines.push(...this.problems(result), "");
|
|
2989
|
-
}
|
|
3024
|
+
if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
|
|
3025
|
+
if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
|
|
2990
3026
|
const tests = run.scenarios.flatMap((result) => result.tests);
|
|
2991
|
-
lines.push(...run.status === RunStatus.Interrupted ? [`Run
|
|
2992
|
-
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
|
|
3027
|
+
lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
|
|
3028
|
+
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
|
|
2993
3029
|
}
|
|
2994
3030
|
static footer(run, exitCode, reportPaths) {
|
|
2995
3031
|
return [
|
|
@@ -3003,57 +3039,148 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3003
3039
|
return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
|
|
3004
3040
|
}
|
|
3005
3041
|
static duration(ms) {
|
|
3042
|
+
if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
|
|
3006
3043
|
const seconds = Math.floor(ms / 1e3);
|
|
3007
3044
|
const pad = (value) => String(value).padStart(2, "0");
|
|
3008
3045
|
const hours = Math.floor(seconds / 3600);
|
|
3009
|
-
const
|
|
3010
|
-
return hours > 0 ? `${hours}:${
|
|
3046
|
+
const minutes = Math.floor(seconds / 60) % 60;
|
|
3047
|
+
return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
|
|
3048
|
+
}
|
|
3049
|
+
static wrap(text, width) {
|
|
3050
|
+
const lines = [];
|
|
3051
|
+
for (const paragraph of text.split("\n")) {
|
|
3052
|
+
let line = "";
|
|
3053
|
+
for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
|
|
3054
|
+
lines.push(line);
|
|
3055
|
+
line = word;
|
|
3056
|
+
} else line = line === "" ? word : `${line} ${word}`;
|
|
3057
|
+
lines.push(line);
|
|
3058
|
+
}
|
|
3059
|
+
return lines;
|
|
3060
|
+
}
|
|
3061
|
+
section(name, blocks) {
|
|
3062
|
+
return [
|
|
3063
|
+
...this.rule(name),
|
|
3064
|
+
"",
|
|
3065
|
+
...blocks.flatMap((block) => [...block, ""])
|
|
3066
|
+
];
|
|
3067
|
+
}
|
|
3068
|
+
rule(name) {
|
|
3069
|
+
const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
|
|
3070
|
+
return [
|
|
3071
|
+
rule,
|
|
3072
|
+
` ${name}`,
|
|
3073
|
+
rule
|
|
3074
|
+
];
|
|
3011
3075
|
}
|
|
3012
|
-
|
|
3076
|
+
blocks(result, flaky) {
|
|
3013
3077
|
const spec = result.spec;
|
|
3014
|
-
const
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3078
|
+
const title = ConsoleReporter.title(spec);
|
|
3079
|
+
const blocks = [];
|
|
3080
|
+
if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
|
|
3081
|
+
const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
|
|
3082
|
+
for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
|
|
3083
|
+
return blocks;
|
|
3084
|
+
}
|
|
3085
|
+
attemptBlock(spec, title, attempt) {
|
|
3086
|
+
const execution = attempt.execution;
|
|
3087
|
+
const problems = [];
|
|
3088
|
+
if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
|
|
3089
|
+
for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
|
|
3090
|
+
if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
|
|
3091
|
+
if (problems.length === 0) return [];
|
|
3092
|
+
const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
|
|
3093
|
+
const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
|
|
3094
|
+
return [[
|
|
3095
|
+
` scenario at ${spec.location.file}`,
|
|
3096
|
+
` ${this.mark(status)} ${title} › ${what}`,
|
|
3097
|
+
...problems.flatMap((problem) => this.paragraph(problem, 3))
|
|
3098
|
+
]];
|
|
3099
|
+
}
|
|
3100
|
+
testBlock(spec, title, test, result) {
|
|
3101
|
+
const location = test.location;
|
|
3102
|
+
const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
|
|
3103
|
+
const failing = test.history.map((entry, index) => ({
|
|
3104
|
+
entry,
|
|
3105
|
+
run: result.attempts[index],
|
|
3106
|
+
attempt: result.attempts[index]?.attempt ?? index + 1
|
|
3107
|
+
})).filter(({ entry }) => entry.status !== ResultStatus.Pass);
|
|
3108
|
+
const several = test.history.length > 1;
|
|
3109
|
+
for (const { entry, run, attempt } of failing) {
|
|
3110
|
+
if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
|
|
3111
|
+
lines.push(...this.entryBody(spec, entry, run, several ? 5 : 3));
|
|
3020
3112
|
}
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3113
|
+
lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
|
|
3114
|
+
return lines;
|
|
3115
|
+
}
|
|
3116
|
+
entryBody(spec, entry, attempt, indent) {
|
|
3117
|
+
const pad = " ".repeat(indent);
|
|
3118
|
+
const evaluation = entry.evaluation;
|
|
3119
|
+
if (evaluation === void 0) {
|
|
3120
|
+
if (entry.details.length === 0) return [];
|
|
3121
|
+
return [
|
|
3122
|
+
"",
|
|
3123
|
+
`${pad}Details`,
|
|
3124
|
+
...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
|
|
3125
|
+
];
|
|
3033
3126
|
}
|
|
3127
|
+
const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
|
|
3128
|
+
const expected = ConsoleReporter.expected(spec, entry);
|
|
3129
|
+
const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
|
|
3130
|
+
const actual = ConsoleReporter.actual(entry, attempt);
|
|
3131
|
+
if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
|
|
3132
|
+
if (actual !== void 0) lines.push("", `${pad}Actual`, ...this.capped(actual, indent + 2));
|
|
3133
|
+
if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
|
|
3034
3134
|
return lines;
|
|
3035
3135
|
}
|
|
3136
|
+
static actual(entry, attempt) {
|
|
3137
|
+
const turns = attempt?.execution.turns ?? [];
|
|
3138
|
+
if (entry.kind === TestKind.Conversation) return turns.length === 0 ? void 0 : turns.map((turn, index) => `Turn ${index + 1}: ${turn.response}`).join("\n");
|
|
3139
|
+
return turns[entry.turn - 1]?.response;
|
|
3140
|
+
}
|
|
3141
|
+
capped(text, indent) {
|
|
3142
|
+
const lines = text.trim() === "" ? [`${" ".repeat(indent)}(empty)`] : this.paragraph(text, indent);
|
|
3143
|
+
if (lines.length <= 12) return lines;
|
|
3144
|
+
const more = lines.length - 12;
|
|
3145
|
+
return [...lines.slice(0, 12), `${" ".repeat(indent)}… ${more} more ${more === 1 ? "line" : "lines"} in the JSON report`];
|
|
3146
|
+
}
|
|
3147
|
+
static expected(spec, entry) {
|
|
3148
|
+
if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
|
|
3149
|
+
return spec.turns[entry.turn - 1]?.expect.response?.criteria;
|
|
3150
|
+
}
|
|
3151
|
+
paragraph(text, indent) {
|
|
3152
|
+
const pad = " ".repeat(indent);
|
|
3153
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
|
|
3154
|
+
}
|
|
3155
|
+
bullet(text, indent) {
|
|
3156
|
+
const pad = " ".repeat(indent);
|
|
3157
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
|
|
3158
|
+
}
|
|
3036
3159
|
static testTitle(test) {
|
|
3037
3160
|
return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
|
|
3038
3161
|
}
|
|
3039
3162
|
static tally(statuses) {
|
|
3040
3163
|
const count = (status) => statuses.filter((entry) => entry === status).length;
|
|
3041
|
-
const
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3164
|
+
const parts = [
|
|
3165
|
+
[ResultStatus.Pass, "passed"],
|
|
3166
|
+
[ResultStatus.Flaky, "flaky"],
|
|
3167
|
+
[ResultStatus.Fail, "failed"],
|
|
3168
|
+
[ResultStatus.Error, "errors"],
|
|
3169
|
+
[ResultStatus.Interrupted, "interrupted"]
|
|
3170
|
+
].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
|
|
3171
|
+
return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
|
|
3172
|
+
}
|
|
3173
|
+
mark(status) {
|
|
3174
|
+
const { symbol, colour } = MARKS[status];
|
|
3175
|
+
return this.colour ? styleText(colour, symbol, { validateStream: false }) : symbol;
|
|
3050
3176
|
}
|
|
3051
3177
|
static usage(label, usage, capability) {
|
|
3052
|
-
|
|
3053
|
-
if (
|
|
3054
|
-
if (usage.
|
|
3178
|
+
const name = label.padEnd(16);
|
|
3179
|
+
if (!capability) return `${name} n/a (not supported by host)`;
|
|
3180
|
+
if (usage.expected === 0) return `${name} none`;
|
|
3181
|
+
if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
|
|
3055
3182
|
const number = (value) => value.toLocaleString("en-US");
|
|
3056
|
-
const total = `${
|
|
3183
|
+
const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
|
|
3057
3184
|
return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
|
|
3058
3185
|
}
|
|
3059
3186
|
formatAttempt(spec, attempt) {
|
|
@@ -3065,28 +3192,28 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3065
3192
|
lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
|
|
3066
3193
|
lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
|
|
3067
3194
|
});
|
|
3068
|
-
if (execution.failure !== void 0) lines.push(`
|
|
3195
|
+
if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
|
|
3069
3196
|
const notRun = attempt.tests.filter((test) => test.turn > ran);
|
|
3070
|
-
if (notRun.length > 0) lines.push("
|
|
3197
|
+
if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
|
|
3071
3198
|
lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
|
|
3072
|
-
for (const error of execution.attemptErrors) lines.push(`
|
|
3073
|
-
if (execution.closeError !== void 0) lines.push(`
|
|
3074
|
-
lines.push(`
|
|
3199
|
+
for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
|
|
3200
|
+
if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
|
|
3201
|
+
lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
|
|
3075
3202
|
return lines;
|
|
3076
3203
|
}
|
|
3077
3204
|
formatTest(test) {
|
|
3078
|
-
return [`
|
|
3205
|
+
return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
|
|
3079
3206
|
}
|
|
3080
3207
|
static formatTurn(number, input, turn) {
|
|
3081
3208
|
const lines = [
|
|
3082
|
-
`
|
|
3083
|
-
`
|
|
3084
|
-
`
|
|
3209
|
+
` Turn ${number} (${turn.durationMs} ms)`,
|
|
3210
|
+
` Input: ${ConsoleReporter.indent(input, 22)}`,
|
|
3211
|
+
` Response: ${ConsoleReporter.indent(turn.response, 22)}`
|
|
3085
3212
|
];
|
|
3086
|
-
for (const use of turn.toolUses ?? []) lines.push(`
|
|
3087
|
-
for (const result of turn.toolResults ?? []) lines.push(`
|
|
3088
|
-
for (const status of turn.statuses) lines.push(`
|
|
3089
|
-
if (turn.tokenUsage !== void 0) lines.push(`
|
|
3213
|
+
for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
|
|
3214
|
+
for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
|
|
3215
|
+
for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
|
|
3216
|
+
if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
|
|
3090
3217
|
return lines;
|
|
3091
3218
|
}
|
|
3092
3219
|
static indent(text, width) {
|
|
@@ -3115,10 +3242,12 @@ const JSON_REPORT_FILE = "miniwat-eval-report.json";
|
|
|
3115
3242
|
var JsonReporter = class JsonReporter {
|
|
3116
3243
|
redactor;
|
|
3117
3244
|
cwd;
|
|
3245
|
+
displayBase;
|
|
3118
3246
|
tokens = new TokenUsageAggregator();
|
|
3119
|
-
constructor(redactor, cwd) {
|
|
3247
|
+
constructor(redactor, cwd, displayBase = cwd) {
|
|
3120
3248
|
this.redactor = redactor;
|
|
3121
3249
|
this.cwd = cwd;
|
|
3250
|
+
this.displayBase = displayBase;
|
|
3122
3251
|
}
|
|
3123
3252
|
render(run, config, unsuccessfulCount, exitCode) {
|
|
3124
3253
|
const report = {
|
|
@@ -3189,7 +3318,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3189
3318
|
return {
|
|
3190
3319
|
name: result.spec.name,
|
|
3191
3320
|
runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
|
|
3192
|
-
file: result.spec.location.file,
|
|
3321
|
+
file: this.file(result.spec.location.file),
|
|
3193
3322
|
status: result.status,
|
|
3194
3323
|
connection: {
|
|
3195
3324
|
endpoint: websocket.endpoint,
|
|
@@ -3199,7 +3328,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3199
3328
|
agentUsage: JsonReporter.usage(result.agentUsage),
|
|
3200
3329
|
evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
|
|
3201
3330
|
tests: result.tests.map((test) => ({
|
|
3202
|
-
...
|
|
3331
|
+
...this.identity(test),
|
|
3203
3332
|
status: test.status,
|
|
3204
3333
|
...JsonReporter.counts(test),
|
|
3205
3334
|
history: test.history.map((entry, index) => ({
|
|
@@ -3224,7 +3353,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3224
3353
|
agentUsage: JsonReporter.usage(this.tokens.agent([attempt], capability)),
|
|
3225
3354
|
evaluatorUsage: JsonReporter.usage(this.tokens.evaluator([attempt])),
|
|
3226
3355
|
turns: execution.turns.map((turn, index) => JsonReporter.turn(index + 1, spec.turns[index].input, turn)),
|
|
3227
|
-
tests: attempt.tests.map((test) =>
|
|
3356
|
+
tests: attempt.tests.map((test) => this.test(test, config))
|
|
3228
3357
|
};
|
|
3229
3358
|
}
|
|
3230
3359
|
static turn(number, input, turn) {
|
|
@@ -3249,11 +3378,11 @@ var JsonReporter = class JsonReporter {
|
|
|
3249
3378
|
tokenUsage: turn.tokenUsage === void 0 ? void 0 : JsonReporter.tokenUsage(turn.tokenUsage)
|
|
3250
3379
|
};
|
|
3251
3380
|
}
|
|
3252
|
-
|
|
3381
|
+
test(test, config) {
|
|
3253
3382
|
const thresholds = config.evaluator.thresholds;
|
|
3254
3383
|
const evaluation = test.evaluation;
|
|
3255
3384
|
return {
|
|
3256
|
-
...
|
|
3385
|
+
...this.identity(test),
|
|
3257
3386
|
status: test.status,
|
|
3258
3387
|
details: [...test.details],
|
|
3259
3388
|
evaluation: evaluation === void 0 ? void 0 : {
|
|
@@ -3266,17 +3395,17 @@ var JsonReporter = class JsonReporter {
|
|
|
3266
3395
|
evaluatorUsage: JsonReporter.usage(test.evaluatorUsage)
|
|
3267
3396
|
};
|
|
3268
3397
|
}
|
|
3269
|
-
|
|
3398
|
+
identity(test) {
|
|
3270
3399
|
return {
|
|
3271
3400
|
kind: test.kind,
|
|
3272
3401
|
turn: test.turn,
|
|
3273
3402
|
title: test.title,
|
|
3274
|
-
location:
|
|
3403
|
+
location: this.location(test.location)
|
|
3275
3404
|
};
|
|
3276
3405
|
}
|
|
3277
|
-
|
|
3406
|
+
location(location) {
|
|
3278
3407
|
return {
|
|
3279
|
-
file: location.file,
|
|
3408
|
+
file: this.file(location.file),
|
|
3280
3409
|
line: location.line,
|
|
3281
3410
|
column: location.column,
|
|
3282
3411
|
path: location.path
|
|
@@ -3323,6 +3452,9 @@ var JsonReporter = class JsonReporter {
|
|
|
3323
3452
|
path(path) {
|
|
3324
3453
|
return relative(this.cwd, path) || ".";
|
|
3325
3454
|
}
|
|
3455
|
+
file(shown) {
|
|
3456
|
+
return shown === "" ? shown : this.path(resolve(this.displayBase, shown));
|
|
3457
|
+
}
|
|
3326
3458
|
};
|
|
3327
3459
|
// src/reporting/reportFileWriter.ts
|
|
3328
3460
|
var ReportFileWriter = class {
|
|
@@ -3773,6 +3905,7 @@ var RunWithReader = class RunWithReader {
|
|
|
3773
3905
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
3774
3906
|
var ScenarioLoader = class ScenarioLoader {
|
|
3775
3907
|
cwd;
|
|
3908
|
+
displayBase;
|
|
3776
3909
|
documentLoader = new YamlDocumentLoader();
|
|
3777
3910
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
3778
3911
|
conversationValidator = new ConversationValidator();
|
|
@@ -3780,18 +3913,19 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
3780
3913
|
runWithReader = new RunWithReader();
|
|
3781
3914
|
webSocketResolver = new WebSocketResolver();
|
|
3782
3915
|
environmentResolver;
|
|
3783
|
-
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
3916
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
|
|
3784
3917
|
this.cwd = cwd;
|
|
3918
|
+
this.displayBase = displayBase;
|
|
3785
3919
|
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
3786
3920
|
}
|
|
3787
3921
|
load(target, config) {
|
|
3788
3922
|
const files = this.discover(resolve(this.cwd, target), config);
|
|
3789
|
-
const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.
|
|
3923
|
+
const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.displayBase, config.configPath)) : void 0;
|
|
3790
3924
|
const issues = [];
|
|
3791
3925
|
const connectionIssues = [];
|
|
3792
3926
|
const specs = [];
|
|
3793
3927
|
for (const file of files) {
|
|
3794
|
-
const document = this.documentLoader.load(file, relative(this.
|
|
3928
|
+
const document = this.documentLoader.load(file, relative(this.displayBase, file));
|
|
3795
3929
|
const fileIssues = [...document.errors];
|
|
3796
3930
|
const fileSecrets = [];
|
|
3797
3931
|
const used = /* @__PURE__ */ new Set();
|
|
@@ -3921,13 +4055,13 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
3921
4055
|
discover(target, config) {
|
|
3922
4056
|
if (!existsSync(target)) throw new EvalError({
|
|
3923
4057
|
category: ErrorCategory.Specification,
|
|
3924
|
-
message: `target not found: ${relative(this.
|
|
4058
|
+
message: `target not found: ${relative(this.displayBase, target) || "."}`
|
|
3925
4059
|
});
|
|
3926
4060
|
const excluded = (path) => path === config.configPath || config.report.directory !== "" && path === config.report.directory;
|
|
3927
4061
|
const files = statSync(target).isDirectory() ? this.walk(target, excluded) : [target].filter((file) => !excluded(file));
|
|
3928
4062
|
if (files.length === 0) throw new EvalError({
|
|
3929
4063
|
category: ErrorCategory.Specification,
|
|
3930
|
-
message: `no scenario files found in ${relative(this.
|
|
4064
|
+
message: `no scenario files found in ${relative(this.displayBase, target) || "."}`
|
|
3931
4065
|
});
|
|
3932
4066
|
return files;
|
|
3933
4067
|
}
|
|
@@ -4007,6 +4141,16 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
4007
4141
|
});
|
|
4008
4142
|
}
|
|
4009
4143
|
};
|
|
4144
|
+
// src/utils/displayPath.ts
|
|
4145
|
+
function displayBase(environment, cwd) {
|
|
4146
|
+
const start = environment.INIT_CWD;
|
|
4147
|
+
if (start === void 0 || start === "" || !isAbsolute(start)) return cwd;
|
|
4148
|
+
try {
|
|
4149
|
+
return existsSync(start) && statSync(start).isDirectory() ? start : cwd;
|
|
4150
|
+
} catch {
|
|
4151
|
+
return cwd;
|
|
4152
|
+
}
|
|
4153
|
+
}
|
|
4010
4154
|
// src/cli/runCommand.ts
|
|
4011
4155
|
const createLlmEvaluator = async (config) => {
|
|
4012
4156
|
const factory = new EvaluatorAgentFactory(config.evaluator);
|
|
@@ -4022,7 +4166,8 @@ var RunCommand = class RunCommand {
|
|
|
4022
4166
|
createEvaluator;
|
|
4023
4167
|
signals;
|
|
4024
4168
|
colour;
|
|
4025
|
-
|
|
4169
|
+
columns;
|
|
4170
|
+
constructor(out, err, environment = process.env, cwd = process.cwd(), createRunner = (config, evaluator) => new ConversationRunner(config, evaluator), createEvaluator = createLlmEvaluator, signals = process, colour = false, columns = void 0) {
|
|
4026
4171
|
this.out = out;
|
|
4027
4172
|
this.err = err;
|
|
4028
4173
|
this.environment = environment;
|
|
@@ -4031,21 +4176,24 @@ var RunCommand = class RunCommand {
|
|
|
4031
4176
|
this.createEvaluator = createEvaluator;
|
|
4032
4177
|
this.signals = signals;
|
|
4033
4178
|
this.colour = colour;
|
|
4179
|
+
this.columns = columns;
|
|
4180
|
+
this.displayBase = displayBase(environment, cwd);
|
|
4034
4181
|
}
|
|
4182
|
+
displayBase;
|
|
4035
4183
|
async execute(target, config, execution = new ExecutionOverride(), report = new ReportOverride(), parameters = "") {
|
|
4036
4184
|
let evalConfig;
|
|
4037
4185
|
let specs;
|
|
4038
4186
|
let evaluator;
|
|
4039
4187
|
let redactor = new SecretRedactor([]);
|
|
4040
4188
|
try {
|
|
4041
|
-
const configPath = new ConfigLocator(this.cwd).locate(target, config);
|
|
4189
|
+
const configPath = new ConfigLocator(this.cwd, this.displayBase).locate(target, config);
|
|
4042
4190
|
const parameterFile = parameters === "" ? {
|
|
4043
4191
|
path: "",
|
|
4044
4192
|
parameters: {}
|
|
4045
|
-
} : new ParametersLoader(this.cwd).load(parameters);
|
|
4046
|
-
evalConfig = new EvalConfigLoader(this.environment, this.cwd, parameterFile.parameters).load(configPath, execution, report);
|
|
4193
|
+
} : new ParametersLoader(this.cwd, this.displayBase).load(parameters);
|
|
4194
|
+
evalConfig = new EvalConfigLoader(this.environment, this.cwd, parameterFile.parameters, this.displayBase).load(configPath, execution, report);
|
|
4047
4195
|
evalConfig.parametersPath = parameterFile.path;
|
|
4048
|
-
specs = new ScenarioLoader(this.environment, this.cwd, parameterFile.parameters).load(target, evalConfig);
|
|
4196
|
+
specs = new ScenarioLoader(this.environment, this.cwd, parameterFile.parameters, this.displayBase).load(target, evalConfig);
|
|
4049
4197
|
redactor = new SecretRedactor([...evalConfig.secrets, ...specs.flatMap((spec) => spec.secrets)]);
|
|
4050
4198
|
evaluator = RunCommand.needsEvaluator(specs) ? await this.createEvaluator(evalConfig) : new UnavailableEvaluator();
|
|
4051
4199
|
} catch (error) {
|
|
@@ -4057,7 +4205,11 @@ var RunCommand = class RunCommand {
|
|
|
4057
4205
|
const { repeat, minPass, flakyPolicy, scenarioConcurrency } = evalConfig.execution;
|
|
4058
4206
|
const tokenUsageCapability = evalConfig.websocket.protocol.capabilities.tokenUsage;
|
|
4059
4207
|
const consoleReport = evalConfig.report.formats.includes(ReportFormat.Console);
|
|
4060
|
-
const reporter = new ConsoleReporter(report.verbose, this.colour
|
|
4208
|
+
const reporter = new ConsoleReporter(report.verbose, this.colour, {
|
|
4209
|
+
nameWidth: ConsoleReporter.nameWidth(specs),
|
|
4210
|
+
columns: this.columns,
|
|
4211
|
+
thresholds: evalConfig.evaluator.thresholds
|
|
4212
|
+
});
|
|
4061
4213
|
const write = (text) => {
|
|
4062
4214
|
if (consoleReport) this.out(redactor.redact(text));
|
|
4063
4215
|
};
|
|
@@ -4103,9 +4255,9 @@ var RunCommand = class RunCommand {
|
|
|
4103
4255
|
let exitCode = calculator.exitCode(run);
|
|
4104
4256
|
const reportPaths = [];
|
|
4105
4257
|
if (evalConfig.report.formats.includes(ReportFormat.Json)) {
|
|
4106
|
-
const content = new JsonReporter(redactor, this.cwd).render(run, evalConfig, calculator.count(run), exitCode);
|
|
4258
|
+
const content = new JsonReporter(redactor, this.cwd, this.displayBase).render(run, evalConfig, calculator.count(run), exitCode);
|
|
4107
4259
|
const writer = new ReportFileWriter(evalConfig.report.directory);
|
|
4108
|
-
const path = relative(this.
|
|
4260
|
+
const path = relative(this.displayBase, writer.path(JSON_REPORT_FILE));
|
|
4109
4261
|
try {
|
|
4110
4262
|
writer.write(JSON_REPORT_FILE, content);
|
|
4111
4263
|
reportPaths.push(path);
|
|
@@ -4133,13 +4285,15 @@ var ValidateCommand = class {
|
|
|
4133
4285
|
this.err = err;
|
|
4134
4286
|
this.environment = environment;
|
|
4135
4287
|
this.cwd = cwd;
|
|
4288
|
+
this.displayBase = displayBase(environment, cwd);
|
|
4136
4289
|
}
|
|
4290
|
+
displayBase;
|
|
4137
4291
|
execute(target, config, parameters = "") {
|
|
4138
4292
|
try {
|
|
4139
|
-
const configPath = new ConfigLocator(this.cwd).locate(target, config);
|
|
4140
|
-
const values = parameters === "" ? {} : new ParametersLoader(this.cwd).load(parameters).parameters;
|
|
4141
|
-
const evalConfig = new EvalConfigLoader(this.environment, this.cwd, values).load(configPath, new ExecutionOverride());
|
|
4142
|
-
const specs = new ScenarioLoader(this.environment, this.cwd, values).load(target, evalConfig);
|
|
4293
|
+
const configPath = new ConfigLocator(this.cwd, this.displayBase).locate(target, config);
|
|
4294
|
+
const values = parameters === "" ? {} : new ParametersLoader(this.cwd, this.displayBase).load(parameters).parameters;
|
|
4295
|
+
const evalConfig = new EvalConfigLoader(this.environment, this.cwd, values, this.displayBase).load(configPath, new ExecutionOverride());
|
|
4296
|
+
const specs = new ScenarioLoader(this.environment, this.cwd, values, this.displayBase).load(target, evalConfig);
|
|
4143
4297
|
this.out(`Valid: ${specs.length} scenario${specs.length === 1 ? "" : "s"}\n`);
|
|
4144
4298
|
return 0;
|
|
4145
4299
|
} catch (error) {
|
|
@@ -4151,7 +4305,7 @@ var ValidateCommand = class {
|
|
|
4151
4305
|
}
|
|
4152
4306
|
};
|
|
4153
4307
|
// src/cli/cli.ts
|
|
4154
|
-
function createProgram(setExitCode, out, err, colour = false) {
|
|
4308
|
+
function createProgram(setExitCode, out, err, colour = false, columns = void 0) {
|
|
4155
4309
|
const program = new Command().name("miniwat-eval").description("Black-box, YAML-driven evaluation for agents built on miniwat-agent").configureOutput({
|
|
4156
4310
|
writeOut: out,
|
|
4157
4311
|
writeErr: err
|
|
@@ -4171,14 +4325,14 @@ function createProgram(setExitCode, out, err, colour = false) {
|
|
|
4171
4325
|
setExitCode(1);
|
|
4172
4326
|
return;
|
|
4173
4327
|
}
|
|
4174
|
-
setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour).execute(target, options.config ?? "", execution, report, options.parameters ?? ""));
|
|
4328
|
+
setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour, columns).execute(target, options.config ?? "", execution, report, options.parameters ?? ""));
|
|
4175
4329
|
});
|
|
4176
4330
|
return program;
|
|
4177
4331
|
}
|
|
4178
4332
|
// src/main.ts
|
|
4179
4333
|
const program = createProgram((code) => {
|
|
4180
4334
|
process.exitCode = code;
|
|
4181
|
-
}, (text) => process.stdout.write(text), (text) => process.stderr.write(text), process.stdout.isTTY === true && !process.env.NO_COLOR);
|
|
4335
|
+
}, (text) => process.stdout.write(text), (text) => process.stderr.write(text), process.stdout.isTTY === true && !process.env.NO_COLOR, process.stdout.isTTY === true ? process.stdout.columns : void 0);
|
|
4182
4336
|
try {
|
|
4183
4337
|
await program.parseAsync(process.argv);
|
|
4184
4338
|
} catch (error) {
|