@mappies/miniwat-eval 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +91 -10
- package/dist/index.cjs +577 -120
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +33 -6
- package/dist/index.d.ts +34 -6
- package/dist/index.js +577 -120
- package/dist/index.js.map +1 -1
- package/dist/main.js +588 -125
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +35 -1
- package/schemas/eval-config.schema.json +6 -1
- package/schemas/report.schema.json +1 -0
package/dist/main.js
CHANGED
|
@@ -218,7 +218,10 @@ var eval_config_schema_default = {
|
|
|
218
218
|
"variables": {
|
|
219
219
|
"type": "object",
|
|
220
220
|
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
221
|
-
"additionalProperties": { "type": "string" }
|
|
221
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
222
|
+
"type": "array",
|
|
223
|
+
"items": { "type": "string" }
|
|
224
|
+
}] }
|
|
222
225
|
},
|
|
223
226
|
"websocket": {
|
|
224
227
|
"type": "object",
|
|
@@ -525,7 +528,7 @@ var EvalConfig = class {
|
|
|
525
528
|
this.version = data?.version ?? 1;
|
|
526
529
|
this.configPath = data?.configPath ?? "";
|
|
527
530
|
this.parametersPath = data?.parametersPath ?? "";
|
|
528
|
-
this.variables =
|
|
531
|
+
this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
|
|
529
532
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
530
533
|
this.execution = new ExecutionConfig(data?.execution);
|
|
531
534
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -611,7 +614,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
611
614
|
return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
|
|
612
615
|
}
|
|
613
616
|
resolve(value, document, issues, secrets = [], options = {}) {
|
|
614
|
-
return this.resolveValue(value, [], document, issues, secrets, options);
|
|
617
|
+
return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
|
|
615
618
|
}
|
|
616
619
|
static hasReference(value) {
|
|
617
620
|
return new RegExp(REFERENCE.source).test(value);
|
|
@@ -619,7 +622,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
619
622
|
static isSecretName(name) {
|
|
620
623
|
return SECRET_NAME.test(name);
|
|
621
624
|
}
|
|
622
|
-
|
|
625
|
+
find(name) {
|
|
623
626
|
for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
|
|
624
627
|
return this.environment[name];
|
|
625
628
|
}
|
|
@@ -634,7 +637,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
634
637
|
resolveValue(value, path, document, issues, secrets, options) {
|
|
635
638
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
636
639
|
options.used?.add(name);
|
|
637
|
-
const resolved = this.
|
|
640
|
+
const resolved = this.find(name);
|
|
638
641
|
if (resolved === void 0) {
|
|
639
642
|
if (options.optional?.(path)) {
|
|
640
643
|
options.unresolved?.add(YamlDocument.formatPath(path));
|
|
@@ -646,6 +649,13 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
646
649
|
}));
|
|
647
650
|
return reference;
|
|
648
651
|
}
|
|
652
|
+
if (Array.isArray(resolved)) {
|
|
653
|
+
issues.push(new ValidationIssue({
|
|
654
|
+
message: `variable '${name}' is a list, which can only be used as runWith.from`,
|
|
655
|
+
location: document.locate(path)
|
|
656
|
+
}));
|
|
657
|
+
return reference;
|
|
658
|
+
}
|
|
649
659
|
if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
|
|
650
660
|
return resolved;
|
|
651
661
|
});
|
|
@@ -770,9 +780,25 @@ var VariablesReader = class {
|
|
|
770
780
|
}));
|
|
771
781
|
continue;
|
|
772
782
|
}
|
|
783
|
+
if (Array.isArray(value)) {
|
|
784
|
+
const bad = value.findIndex((item) => typeof item !== "string");
|
|
785
|
+
if (bad >= 0) {
|
|
786
|
+
issues.push(new ValidationIssue({
|
|
787
|
+
message: "must be string",
|
|
788
|
+
location: document.locate([
|
|
789
|
+
"variables",
|
|
790
|
+
name,
|
|
791
|
+
bad
|
|
792
|
+
])
|
|
793
|
+
}));
|
|
794
|
+
continue;
|
|
795
|
+
}
|
|
796
|
+
variables[name] = [...value];
|
|
797
|
+
continue;
|
|
798
|
+
}
|
|
773
799
|
if (typeof value !== "string") {
|
|
774
800
|
issues.push(new ValidationIssue({
|
|
775
|
-
message: "must be string",
|
|
801
|
+
message: "must be string or a list of strings",
|
|
776
802
|
location: document.locate(["variables", name])
|
|
777
803
|
}));
|
|
778
804
|
continue;
|
|
@@ -780,8 +806,9 @@ var VariablesReader = class {
|
|
|
780
806
|
const before = issues.length;
|
|
781
807
|
const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
|
|
782
808
|
if (issues.length > before) continue;
|
|
783
|
-
|
|
784
|
-
|
|
809
|
+
const text = resolved.variables[name];
|
|
810
|
+
variables[name] = text;
|
|
811
|
+
if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
|
|
785
812
|
}
|
|
786
813
|
return variables;
|
|
787
814
|
}
|
|
@@ -1253,6 +1280,7 @@ var TurnSpec = class {
|
|
|
1253
1280
|
var ConversationSpec = class {
|
|
1254
1281
|
version;
|
|
1255
1282
|
name;
|
|
1283
|
+
runName;
|
|
1256
1284
|
description;
|
|
1257
1285
|
turns;
|
|
1258
1286
|
conversation;
|
|
@@ -1262,6 +1290,7 @@ var ConversationSpec = class {
|
|
|
1262
1290
|
constructor(data) {
|
|
1263
1291
|
this.version = data?.version ?? 1;
|
|
1264
1292
|
this.name = data?.name ?? "";
|
|
1293
|
+
this.runName = data?.runName ?? "";
|
|
1265
1294
|
this.description = data?.description ?? "";
|
|
1266
1295
|
this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
|
|
1267
1296
|
this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
|
|
@@ -2903,61 +2932,94 @@ var ExitCodeCalculator = class {
|
|
|
2903
2932
|
}
|
|
2904
2933
|
};
|
|
2905
2934
|
// src/reporting/consoleReporter.ts
|
|
2906
|
-
const
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
|
|
2911
|
-
[ResultStatus.
|
|
2912
|
-
|
|
2913
|
-
|
|
2935
|
+
const MARKS = {
|
|
2936
|
+
[ResultStatus.Pass]: {
|
|
2937
|
+
symbol: "✔",
|
|
2938
|
+
colour: "green"
|
|
2939
|
+
},
|
|
2940
|
+
[ResultStatus.Flaky]: {
|
|
2941
|
+
symbol: "⚠",
|
|
2942
|
+
colour: "yellow"
|
|
2943
|
+
},
|
|
2944
|
+
[ResultStatus.Fail]: {
|
|
2945
|
+
symbol: "✖",
|
|
2946
|
+
colour: "red"
|
|
2947
|
+
},
|
|
2948
|
+
[ResultStatus.Error]: {
|
|
2949
|
+
symbol: "●",
|
|
2950
|
+
colour: "red"
|
|
2951
|
+
},
|
|
2952
|
+
[ResultStatus.Skipped]: {
|
|
2953
|
+
symbol: "○",
|
|
2954
|
+
colour: "dim"
|
|
2955
|
+
},
|
|
2956
|
+
[ResultStatus.Interrupted]: {
|
|
2957
|
+
symbol: "◌",
|
|
2958
|
+
colour: "dim"
|
|
2959
|
+
}
|
|
2914
2960
|
};
|
|
2961
|
+
const MIN_COLUMNS = 40;
|
|
2962
|
+
const RULE_WIDTH = 72;
|
|
2915
2963
|
var ConsoleReporter = class ConsoleReporter {
|
|
2916
2964
|
verbose;
|
|
2917
2965
|
colour;
|
|
2918
|
-
|
|
2966
|
+
nameWidth;
|
|
2967
|
+
columns;
|
|
2968
|
+
thresholds;
|
|
2969
|
+
constructor(verbose, colour, options = {}) {
|
|
2919
2970
|
this.verbose = verbose;
|
|
2920
2971
|
this.colour = colour;
|
|
2972
|
+
this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
|
|
2973
|
+
this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
|
|
2974
|
+
this.thresholds = options.thresholds;
|
|
2975
|
+
}
|
|
2976
|
+
static nameWidth(specs) {
|
|
2977
|
+
return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
|
|
2921
2978
|
}
|
|
2922
2979
|
header(run) {
|
|
2923
2980
|
return [
|
|
2924
|
-
"Miniwat Eval",
|
|
2981
|
+
" Miniwat Eval",
|
|
2925
2982
|
"",
|
|
2926
|
-
...run.quick ? ["Execution
|
|
2927
|
-
`Repeat
|
|
2928
|
-
`Minimum
|
|
2983
|
+
...run.quick ? [" Execution mode Quick"] : [],
|
|
2984
|
+
` Repeat ${run.repeat}`,
|
|
2985
|
+
` Minimum pass ${run.minPass}`,
|
|
2929
2986
|
"",
|
|
2930
|
-
`Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
2987
|
+
` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
2931
2988
|
"",
|
|
2932
2989
|
""
|
|
2933
2990
|
].join("\n");
|
|
2934
2991
|
}
|
|
2935
2992
|
scenario(result, repeat, tokenUsageCapability) {
|
|
2936
2993
|
const spec = result.spec;
|
|
2937
|
-
const
|
|
2938
|
-
const
|
|
2994
|
+
const title = ConsoleReporter.title(spec);
|
|
2995
|
+
const started = result.attempts.length > 0;
|
|
2996
|
+
const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
|
|
2997
|
+
const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
|
|
2998
|
+
const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
|
|
2939
2999
|
if (!this.verbose) return line;
|
|
2940
|
-
if (
|
|
3000
|
+
if (!started) return `${line}\n`;
|
|
2941
3001
|
const websocket = spec.websocket;
|
|
2942
3002
|
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
2943
|
-
const lines = [
|
|
2944
|
-
|
|
2945
|
-
lines.push(`
|
|
2946
|
-
lines.push(
|
|
2947
|
-
lines.push(`
|
|
2948
|
-
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3003
|
+
const lines = [];
|
|
3004
|
+
if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
|
|
3005
|
+
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
3006
|
+
lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
|
|
3007
|
+
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
2949
3008
|
return line + lines.join("\n");
|
|
2950
3009
|
}
|
|
3010
|
+
static title(spec) {
|
|
3011
|
+
return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
|
|
3012
|
+
}
|
|
2951
3013
|
summary(run, exitCode, reportPaths = []) {
|
|
2952
|
-
const
|
|
3014
|
+
const started = run.scenarios.filter((result) => result.attempts.length > 0);
|
|
3015
|
+
const failures = started.flatMap((result) => this.blocks(result, false));
|
|
3016
|
+
const flaky = started.flatMap((result) => this.blocks(result, true));
|
|
2953
3017
|
const lines = this.verbose ? [] : [""];
|
|
2954
|
-
if (
|
|
2955
|
-
|
|
2956
|
-
for (const result of problems) lines.push(...this.problems(result), "");
|
|
2957
|
-
}
|
|
3018
|
+
if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
|
|
3019
|
+
if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
|
|
2958
3020
|
const tests = run.scenarios.flatMap((result) => result.tests);
|
|
2959
|
-
lines.push(...run.status === RunStatus.Interrupted ? [`Run
|
|
2960
|
-
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
|
|
3021
|
+
lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
|
|
3022
|
+
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
|
|
2961
3023
|
}
|
|
2962
3024
|
static footer(run, exitCode, reportPaths) {
|
|
2963
3025
|
return [
|
|
@@ -2971,57 +3033,135 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
2971
3033
|
return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
|
|
2972
3034
|
}
|
|
2973
3035
|
static duration(ms) {
|
|
3036
|
+
if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
|
|
2974
3037
|
const seconds = Math.floor(ms / 1e3);
|
|
2975
3038
|
const pad = (value) => String(value).padStart(2, "0");
|
|
2976
3039
|
const hours = Math.floor(seconds / 3600);
|
|
2977
|
-
const
|
|
2978
|
-
return hours > 0 ? `${hours}:${
|
|
3040
|
+
const minutes = Math.floor(seconds / 60) % 60;
|
|
3041
|
+
return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
|
|
3042
|
+
}
|
|
3043
|
+
static wrap(text, width) {
|
|
3044
|
+
const lines = [];
|
|
3045
|
+
for (const paragraph of text.split("\n")) {
|
|
3046
|
+
let line = "";
|
|
3047
|
+
for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
|
|
3048
|
+
lines.push(line);
|
|
3049
|
+
line = word;
|
|
3050
|
+
} else line = line === "" ? word : `${line} ${word}`;
|
|
3051
|
+
lines.push(line);
|
|
3052
|
+
}
|
|
3053
|
+
return lines;
|
|
3054
|
+
}
|
|
3055
|
+
section(name, blocks) {
|
|
3056
|
+
return [
|
|
3057
|
+
...this.rule(name),
|
|
3058
|
+
"",
|
|
3059
|
+
...blocks.flatMap((block) => [...block, ""])
|
|
3060
|
+
];
|
|
2979
3061
|
}
|
|
2980
|
-
|
|
3062
|
+
rule(name) {
|
|
3063
|
+
const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
|
|
3064
|
+
return [
|
|
3065
|
+
rule,
|
|
3066
|
+
` ${name}`,
|
|
3067
|
+
rule
|
|
3068
|
+
];
|
|
3069
|
+
}
|
|
3070
|
+
blocks(result, flaky) {
|
|
2981
3071
|
const spec = result.spec;
|
|
2982
|
-
const
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
}
|
|
3072
|
+
const title = ConsoleReporter.title(spec);
|
|
3073
|
+
const blocks = [];
|
|
3074
|
+
if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
|
|
3075
|
+
const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
|
|
3076
|
+
for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
|
|
3077
|
+
return blocks;
|
|
3078
|
+
}
|
|
3079
|
+
attemptBlock(spec, title, attempt) {
|
|
3080
|
+
const execution = attempt.execution;
|
|
3081
|
+
const problems = [];
|
|
3082
|
+
if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
|
|
3083
|
+
for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
|
|
3084
|
+
if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
|
|
3085
|
+
if (problems.length === 0) return [];
|
|
3086
|
+
const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
|
|
3087
|
+
const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
|
|
3088
|
+
return [[
|
|
3089
|
+
` scenario at ${spec.location.file}`,
|
|
3090
|
+
` ${this.mark(status)} ${title} › ${what}`,
|
|
3091
|
+
...problems.flatMap((problem) => this.paragraph(problem, 3))
|
|
3092
|
+
]];
|
|
3093
|
+
}
|
|
3094
|
+
testBlock(spec, title, test, result) {
|
|
3095
|
+
const location = test.location;
|
|
3096
|
+
const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
|
|
3097
|
+
const failing = test.history.map((entry, index) => ({
|
|
3098
|
+
entry,
|
|
3099
|
+
attempt: result.attempts[index]?.attempt ?? index + 1
|
|
3100
|
+
})).filter(({ entry }) => entry.status !== ResultStatus.Pass);
|
|
3101
|
+
const several = test.history.length > 1;
|
|
3102
|
+
for (const { entry, attempt } of failing) {
|
|
3103
|
+
if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
|
|
3104
|
+
lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
|
|
3105
|
+
}
|
|
3106
|
+
lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
|
|
3107
|
+
return lines;
|
|
3108
|
+
}
|
|
3109
|
+
entryBody(spec, entry, indent) {
|
|
3110
|
+
const pad = " ".repeat(indent);
|
|
3111
|
+
const evaluation = entry.evaluation;
|
|
3112
|
+
if (evaluation === void 0) {
|
|
3113
|
+
if (entry.details.length === 0) return [];
|
|
3114
|
+
return [
|
|
3115
|
+
"",
|
|
3116
|
+
`${pad}Details`,
|
|
3117
|
+
...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
|
|
3118
|
+
];
|
|
3001
3119
|
}
|
|
3120
|
+
const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
|
|
3121
|
+
const expected = ConsoleReporter.expected(spec, entry);
|
|
3122
|
+
const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
|
|
3123
|
+
if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
|
|
3124
|
+
if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
|
|
3125
|
+
if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
|
|
3002
3126
|
return lines;
|
|
3003
3127
|
}
|
|
3128
|
+
static expected(spec, entry) {
|
|
3129
|
+
if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
|
|
3130
|
+
return spec.turns[entry.turn - 1]?.expect.response?.criteria;
|
|
3131
|
+
}
|
|
3132
|
+
paragraph(text, indent) {
|
|
3133
|
+
const pad = " ".repeat(indent);
|
|
3134
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
|
|
3135
|
+
}
|
|
3136
|
+
bullet(text, indent) {
|
|
3137
|
+
const pad = " ".repeat(indent);
|
|
3138
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
|
|
3139
|
+
}
|
|
3004
3140
|
static testTitle(test) {
|
|
3005
3141
|
return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
|
|
3006
3142
|
}
|
|
3007
3143
|
static tally(statuses) {
|
|
3008
3144
|
const count = (status) => statuses.filter((entry) => entry === status).length;
|
|
3009
|
-
const
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3145
|
+
const parts = [
|
|
3146
|
+
[ResultStatus.Pass, "passed"],
|
|
3147
|
+
[ResultStatus.Flaky, "flaky"],
|
|
3148
|
+
[ResultStatus.Fail, "failed"],
|
|
3149
|
+
[ResultStatus.Error, "errors"],
|
|
3150
|
+
[ResultStatus.Interrupted, "interrupted"]
|
|
3151
|
+
].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
|
|
3152
|
+
return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
|
|
3153
|
+
}
|
|
3154
|
+
mark(status) {
|
|
3155
|
+
const { symbol, colour } = MARKS[status];
|
|
3156
|
+
return this.colour ? styleText(colour, symbol, { validateStream: false }) : symbol;
|
|
3018
3157
|
}
|
|
3019
3158
|
static usage(label, usage, capability) {
|
|
3020
|
-
|
|
3021
|
-
if (
|
|
3022
|
-
if (usage.
|
|
3159
|
+
const name = label.padEnd(16);
|
|
3160
|
+
if (!capability) return `${name} n/a (not supported by host)`;
|
|
3161
|
+
if (usage.expected === 0) return `${name} none`;
|
|
3162
|
+
if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
|
|
3023
3163
|
const number = (value) => value.toLocaleString("en-US");
|
|
3024
|
-
const total = `${
|
|
3164
|
+
const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
|
|
3025
3165
|
return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
|
|
3026
3166
|
}
|
|
3027
3167
|
formatAttempt(spec, attempt) {
|
|
@@ -3033,28 +3173,28 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3033
3173
|
lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
|
|
3034
3174
|
lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
|
|
3035
3175
|
});
|
|
3036
|
-
if (execution.failure !== void 0) lines.push(`
|
|
3176
|
+
if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
|
|
3037
3177
|
const notRun = attempt.tests.filter((test) => test.turn > ran);
|
|
3038
|
-
if (notRun.length > 0) lines.push("
|
|
3178
|
+
if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
|
|
3039
3179
|
lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
|
|
3040
|
-
for (const error of execution.attemptErrors) lines.push(`
|
|
3041
|
-
if (execution.closeError !== void 0) lines.push(`
|
|
3042
|
-
lines.push(`
|
|
3180
|
+
for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
|
|
3181
|
+
if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
|
|
3182
|
+
lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
|
|
3043
3183
|
return lines;
|
|
3044
3184
|
}
|
|
3045
3185
|
formatTest(test) {
|
|
3046
|
-
return [`
|
|
3186
|
+
return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
|
|
3047
3187
|
}
|
|
3048
3188
|
static formatTurn(number, input, turn) {
|
|
3049
3189
|
const lines = [
|
|
3050
|
-
`
|
|
3051
|
-
`
|
|
3052
|
-
`
|
|
3190
|
+
` Turn ${number} (${turn.durationMs} ms)`,
|
|
3191
|
+
` Input: ${ConsoleReporter.indent(input, 22)}`,
|
|
3192
|
+
` Response: ${ConsoleReporter.indent(turn.response, 22)}`
|
|
3053
3193
|
];
|
|
3054
|
-
for (const use of turn.toolUses ?? []) lines.push(`
|
|
3055
|
-
for (const result of turn.toolResults ?? []) lines.push(`
|
|
3056
|
-
for (const status of turn.statuses) lines.push(`
|
|
3057
|
-
if (turn.tokenUsage !== void 0) lines.push(`
|
|
3194
|
+
for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
|
|
3195
|
+
for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
|
|
3196
|
+
for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
|
|
3197
|
+
if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
|
|
3058
3198
|
return lines;
|
|
3059
3199
|
}
|
|
3060
3200
|
static indent(text, width) {
|
|
@@ -3156,6 +3296,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3156
3296
|
const capability = websocket.protocol.capabilities.tokenUsage;
|
|
3157
3297
|
return {
|
|
3158
3298
|
name: result.spec.name,
|
|
3299
|
+
runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
|
|
3159
3300
|
file: result.spec.location.file,
|
|
3160
3301
|
status: result.status,
|
|
3161
3302
|
connection: {
|
|
@@ -3325,7 +3466,44 @@ var conversation_schema_default = {
|
|
|
3325
3466
|
"variables": {
|
|
3326
3467
|
"type": "object",
|
|
3327
3468
|
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
3328
|
-
"additionalProperties": { "type": "string" }
|
|
3469
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
3470
|
+
"type": "array",
|
|
3471
|
+
"items": { "type": "string" }
|
|
3472
|
+
}] }
|
|
3473
|
+
},
|
|
3474
|
+
"runWith": {
|
|
3475
|
+
"description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
|
|
3476
|
+
"anyOf": [{
|
|
3477
|
+
"type": "array",
|
|
3478
|
+
"minItems": 1,
|
|
3479
|
+
"items": {
|
|
3480
|
+
"type": "object",
|
|
3481
|
+
"required": ["name"],
|
|
3482
|
+
"properties": { "name": {
|
|
3483
|
+
"type": "string",
|
|
3484
|
+
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
|
|
3485
|
+
} },
|
|
3486
|
+
"additionalProperties": { "type": "string" }
|
|
3487
|
+
}
|
|
3488
|
+
}, {
|
|
3489
|
+
"type": "object",
|
|
3490
|
+
"additionalProperties": false,
|
|
3491
|
+
"required": ["variable", "from"],
|
|
3492
|
+
"properties": {
|
|
3493
|
+
"variable": {
|
|
3494
|
+
"type": "string",
|
|
3495
|
+
"pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
|
|
3496
|
+
},
|
|
3497
|
+
"from": { "anyOf": [{
|
|
3498
|
+
"type": "array",
|
|
3499
|
+
"minItems": 1,
|
|
3500
|
+
"items": { "type": "string" }
|
|
3501
|
+
}, {
|
|
3502
|
+
"type": "string",
|
|
3503
|
+
"pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
|
|
3504
|
+
}] }
|
|
3505
|
+
}
|
|
3506
|
+
}]
|
|
3329
3507
|
},
|
|
3330
3508
|
"turns": {
|
|
3331
3509
|
"type": "array",
|
|
@@ -3484,6 +3662,221 @@ var ConversationValidator = class ConversationValidator {
|
|
|
3484
3662
|
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
3485
3663
|
}
|
|
3486
3664
|
};
|
|
3665
|
+
// src/scenarios/runWithReader.ts
|
|
3666
|
+
const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
|
|
3667
|
+
const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
|
|
3668
|
+
const FORMS = "must be a list of rows, or an object with variable and from";
|
|
3669
|
+
const SINGLE_RUN = {
|
|
3670
|
+
name: "",
|
|
3671
|
+
variables: {},
|
|
3672
|
+
keys: [],
|
|
3673
|
+
location: [],
|
|
3674
|
+
secrets: []
|
|
3675
|
+
};
|
|
3676
|
+
var RunWithReader = class RunWithReader {
|
|
3677
|
+
read(document, resolver, issues, used) {
|
|
3678
|
+
const raw = isRecord(document.value) ? document.value.runWith : void 0;
|
|
3679
|
+
if (raw === void 0) return void 0;
|
|
3680
|
+
if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
|
|
3681
|
+
if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
|
|
3682
|
+
issues.push(new ValidationIssue({
|
|
3683
|
+
message: FORMS,
|
|
3684
|
+
location: document.locate(["runWith"])
|
|
3685
|
+
}));
|
|
3686
|
+
return [];
|
|
3687
|
+
}
|
|
3688
|
+
rows(rows, document, resolver, issues, used) {
|
|
3689
|
+
if (rows.length === 0) {
|
|
3690
|
+
issues.push(new ValidationIssue({
|
|
3691
|
+
message: "must not be empty",
|
|
3692
|
+
location: document.locate(["runWith"])
|
|
3693
|
+
}));
|
|
3694
|
+
return [];
|
|
3695
|
+
}
|
|
3696
|
+
const names = /* @__PURE__ */ new Set();
|
|
3697
|
+
const runs = [];
|
|
3698
|
+
rows.forEach((row, index) => {
|
|
3699
|
+
const path = ["runWith", index];
|
|
3700
|
+
if (!isRecord(row)) {
|
|
3701
|
+
issues.push(new ValidationIssue({
|
|
3702
|
+
message: "must be object",
|
|
3703
|
+
location: document.locate(path)
|
|
3704
|
+
}));
|
|
3705
|
+
return;
|
|
3706
|
+
}
|
|
3707
|
+
const name = row.name;
|
|
3708
|
+
if (name === void 0) issues.push(new ValidationIssue({
|
|
3709
|
+
message: "missing required key 'name'",
|
|
3710
|
+
location: document.locate(path)
|
|
3711
|
+
}));
|
|
3712
|
+
else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
|
|
3713
|
+
message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
|
|
3714
|
+
location: document.locate([...path, "name"])
|
|
3715
|
+
}));
|
|
3716
|
+
else if (names.has(name)) issues.push(new ValidationIssue({
|
|
3717
|
+
message: `duplicate run name '${name}'`,
|
|
3718
|
+
location: document.locate([...path, "name"])
|
|
3719
|
+
}));
|
|
3720
|
+
else names.add(name);
|
|
3721
|
+
const run = {
|
|
3722
|
+
name: typeof name === "string" ? name : "",
|
|
3723
|
+
variables: {},
|
|
3724
|
+
keys: [],
|
|
3725
|
+
location: path,
|
|
3726
|
+
secrets: []
|
|
3727
|
+
};
|
|
3728
|
+
for (const [key, value] of Object.entries(row)) {
|
|
3729
|
+
if (key === "name") continue;
|
|
3730
|
+
if (!VARIABLE_NAME.test(key)) {
|
|
3731
|
+
issues.push(new ValidationIssue({
|
|
3732
|
+
message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
3733
|
+
location: document.locateKey(path, key)
|
|
3734
|
+
}));
|
|
3735
|
+
continue;
|
|
3736
|
+
}
|
|
3737
|
+
if (typeof value !== "string") {
|
|
3738
|
+
issues.push(new ValidationIssue({
|
|
3739
|
+
message: "must be string",
|
|
3740
|
+
location: document.locate([...path, key])
|
|
3741
|
+
}));
|
|
3742
|
+
continue;
|
|
3743
|
+
}
|
|
3744
|
+
const resolved = resolver.resolve(value, document, issues, run.secrets, {
|
|
3745
|
+
used,
|
|
3746
|
+
at: [...path, key]
|
|
3747
|
+
});
|
|
3748
|
+
RunWithReader.set(run, key, resolved, document.locateKey(path, key));
|
|
3749
|
+
}
|
|
3750
|
+
runs.push(run);
|
|
3751
|
+
});
|
|
3752
|
+
return runs;
|
|
3753
|
+
}
|
|
3754
|
+
list(form, document, resolver, issues, used) {
|
|
3755
|
+
const before = issues.length;
|
|
3756
|
+
for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
|
|
3757
|
+
message: `unknown key '${key}'`,
|
|
3758
|
+
location: document.locateKey(["runWith"], key)
|
|
3759
|
+
}));
|
|
3760
|
+
const variable = form.variable;
|
|
3761
|
+
if (variable === void 0) issues.push(new ValidationIssue({
|
|
3762
|
+
message: "missing required key 'variable'",
|
|
3763
|
+
location: document.locate(["runWith"])
|
|
3764
|
+
}));
|
|
3765
|
+
else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
|
|
3766
|
+
message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
|
|
3767
|
+
location: document.locate(["runWith", "variable"])
|
|
3768
|
+
}));
|
|
3769
|
+
const entries = this.entries(form.from, document, resolver, issues, used);
|
|
3770
|
+
if (issues.length > before || entries === void 0) return [];
|
|
3771
|
+
const name = variable;
|
|
3772
|
+
const keyLocation = document.locate(["runWith", "variable"]);
|
|
3773
|
+
const names = /* @__PURE__ */ new Set();
|
|
3774
|
+
const runs = [];
|
|
3775
|
+
for (const { entry, path } of entries) {
|
|
3776
|
+
if (!VARIABLE_NAME.test(entry)) {
|
|
3777
|
+
issues.push(new ValidationIssue({
|
|
3778
|
+
message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
|
|
3779
|
+
location: document.locate(path)
|
|
3780
|
+
}));
|
|
3781
|
+
continue;
|
|
3782
|
+
}
|
|
3783
|
+
if (names.has(entry)) {
|
|
3784
|
+
issues.push(new ValidationIssue({
|
|
3785
|
+
message: `duplicate run name '${entry}'`,
|
|
3786
|
+
location: document.locate(path)
|
|
3787
|
+
}));
|
|
3788
|
+
continue;
|
|
3789
|
+
}
|
|
3790
|
+
names.add(entry);
|
|
3791
|
+
const run = {
|
|
3792
|
+
name: entry,
|
|
3793
|
+
variables: {},
|
|
3794
|
+
keys: [],
|
|
3795
|
+
location: path,
|
|
3796
|
+
secrets: []
|
|
3797
|
+
};
|
|
3798
|
+
const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
|
|
3799
|
+
used,
|
|
3800
|
+
at: path
|
|
3801
|
+
});
|
|
3802
|
+
RunWithReader.set(run, name, value, keyLocation);
|
|
3803
|
+
runs.push(run);
|
|
3804
|
+
}
|
|
3805
|
+
return runs;
|
|
3806
|
+
}
|
|
3807
|
+
entries(from, document, resolver, issues, used) {
|
|
3808
|
+
const path = ["runWith", "from"];
|
|
3809
|
+
const location = document.locate(path);
|
|
3810
|
+
if (from === void 0) {
|
|
3811
|
+
issues.push(new ValidationIssue({
|
|
3812
|
+
message: "missing required key 'from'",
|
|
3813
|
+
location: document.locate(["runWith"])
|
|
3814
|
+
}));
|
|
3815
|
+
return;
|
|
3816
|
+
}
|
|
3817
|
+
let list;
|
|
3818
|
+
let at;
|
|
3819
|
+
if (Array.isArray(from)) {
|
|
3820
|
+
list = from;
|
|
3821
|
+
at = (index) => [...path, index];
|
|
3822
|
+
} else {
|
|
3823
|
+
const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
|
|
3824
|
+
if (reference === null) {
|
|
3825
|
+
issues.push(new ValidationIssue({
|
|
3826
|
+
message: "must be a list, or a list variable such as ${tokens}",
|
|
3827
|
+
location
|
|
3828
|
+
}));
|
|
3829
|
+
return;
|
|
3830
|
+
}
|
|
3831
|
+
const name = reference[1];
|
|
3832
|
+
const found = resolver.find(name);
|
|
3833
|
+
used.add(name);
|
|
3834
|
+
if (found === void 0) {
|
|
3835
|
+
issues.push(new ValidationIssue({
|
|
3836
|
+
message: `list variable '${name}' is not set`,
|
|
3837
|
+
location
|
|
3838
|
+
}));
|
|
3839
|
+
return;
|
|
3840
|
+
}
|
|
3841
|
+
if (!Array.isArray(found)) {
|
|
3842
|
+
issues.push(new ValidationIssue({
|
|
3843
|
+
message: `variable '${name}' is not a list`,
|
|
3844
|
+
location
|
|
3845
|
+
}));
|
|
3846
|
+
return;
|
|
3847
|
+
}
|
|
3848
|
+
list = found;
|
|
3849
|
+
at = () => path;
|
|
3850
|
+
}
|
|
3851
|
+
if (list.length === 0) {
|
|
3852
|
+
issues.push(new ValidationIssue({
|
|
3853
|
+
message: "must not be empty",
|
|
3854
|
+
location
|
|
3855
|
+
}));
|
|
3856
|
+
return;
|
|
3857
|
+
}
|
|
3858
|
+
const bad = list.findIndex((item) => typeof item !== "string");
|
|
3859
|
+
if (bad >= 0) {
|
|
3860
|
+
issues.push(new ValidationIssue({
|
|
3861
|
+
message: "must be string",
|
|
3862
|
+
location: document.locate(at(bad))
|
|
3863
|
+
}));
|
|
3864
|
+
return;
|
|
3865
|
+
}
|
|
3866
|
+
return list.map((entry, index) => ({
|
|
3867
|
+
entry,
|
|
3868
|
+
path: at(index)
|
|
3869
|
+
}));
|
|
3870
|
+
}
|
|
3871
|
+
static set(run, name, value, location) {
|
|
3872
|
+
run.variables[name] = value;
|
|
3873
|
+
run.keys.push({
|
|
3874
|
+
name,
|
|
3875
|
+
location
|
|
3876
|
+
});
|
|
3877
|
+
if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
|
|
3878
|
+
}
|
|
3879
|
+
};
|
|
3487
3880
|
// src/scenarios/scenarioLoader.ts
|
|
3488
3881
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
3489
3882
|
var ScenarioLoader = class ScenarioLoader {
|
|
@@ -3492,6 +3885,7 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
3492
3885
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
3493
3886
|
conversationValidator = new ConversationValidator();
|
|
3494
3887
|
variablesReader = new VariablesReader();
|
|
3888
|
+
runWithReader = new RunWithReader();
|
|
3495
3889
|
webSocketResolver = new WebSocketResolver();
|
|
3496
3890
|
environmentResolver;
|
|
3497
3891
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
@@ -3507,45 +3901,59 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
3507
3901
|
for (const file of files) {
|
|
3508
3902
|
const document = this.documentLoader.load(file, relative(this.cwd, file));
|
|
3509
3903
|
const fileIssues = [...document.errors];
|
|
3510
|
-
const
|
|
3904
|
+
const fileSecrets = [];
|
|
3511
3905
|
const used = /* @__PURE__ */ new Set();
|
|
3512
|
-
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues,
|
|
3513
|
-
const
|
|
3514
|
-
|
|
3515
|
-
if (fileIssues.length
|
|
3516
|
-
|
|
3517
|
-
|
|
3518
|
-
...resolver.resolve(rest, document, fileIssues, secrets, { used }),
|
|
3519
|
-
...raw === void 0 ? {} : { variables }
|
|
3520
|
-
};
|
|
3906
|
+
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
|
|
3907
|
+
const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
|
|
3908
|
+
const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
|
|
3909
|
+
if (fileIssues.length > 0) {
|
|
3910
|
+
issues.push(...fileIssues);
|
|
3911
|
+
continue;
|
|
3521
3912
|
}
|
|
3522
|
-
|
|
3523
|
-
const
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
|
|
3527
|
-
|
|
3528
|
-
|
|
3529
|
-
|
|
3530
|
-
|
|
3531
|
-
|
|
3913
|
+
const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
|
|
3914
|
+
const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
|
|
3915
|
+
for (const result of results) for (const name of result.used) used.add(name);
|
|
3916
|
+
fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
|
|
3917
|
+
run: result.run,
|
|
3918
|
+
issues: result.issues
|
|
3919
|
+
})), (issue, run) => run === void 0 ? issue : new ValidationIssue({
|
|
3920
|
+
message: `run '${run.name}': ${issue.message}`,
|
|
3921
|
+
location: issue.location
|
|
3922
|
+
})));
|
|
3923
|
+
const attempted = results.filter((result) => result.connectionChecked);
|
|
3924
|
+
if (attempted.length > 0) connectionIssues.push({
|
|
3925
|
+
file: document,
|
|
3926
|
+
runs: attempted.map((result) => ({
|
|
3927
|
+
run: result.run,
|
|
3928
|
+
issues: result.websocketIssues
|
|
3929
|
+
}))
|
|
3930
|
+
});
|
|
3931
|
+
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
|
|
3932
|
+
if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
|
|
3933
|
+
const websocket = result.websocket;
|
|
3934
|
+
specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
3532
3935
|
}
|
|
3533
|
-
if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
|
|
3534
|
-
if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
|
|
3535
|
-
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
|
|
3536
|
-
if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
3537
3936
|
else issues.push(...fileIssues);
|
|
3538
3937
|
}
|
|
3539
|
-
const shared = ScenarioLoader.shared(connectionIssues.
|
|
3938
|
+
const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
|
|
3540
3939
|
if (shared.length > 0) throw new EvalError({
|
|
3541
3940
|
category: ErrorCategory.Configuration,
|
|
3542
3941
|
message: "invalid configuration",
|
|
3543
3942
|
issues: shared
|
|
3544
3943
|
});
|
|
3545
|
-
for (const entry of connectionIssues)
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
|
|
3944
|
+
for (const entry of connectionIssues) {
|
|
3945
|
+
const fileLocation = ScenarioLoader.variablesLocation(entry.file);
|
|
3946
|
+
issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
|
|
3947
|
+
message: issue.toString(),
|
|
3948
|
+
location: fileLocation
|
|
3949
|
+
}) : new ValidationIssue({
|
|
3950
|
+
message: `run '${run.name}': ${issue.toString()}`,
|
|
3951
|
+
location: new SourceLocation({
|
|
3952
|
+
...entry.file.locate(run.location),
|
|
3953
|
+
path: ""
|
|
3954
|
+
})
|
|
3955
|
+
})));
|
|
3956
|
+
}
|
|
3549
3957
|
if (issues.length > 0) throw new EvalError({
|
|
3550
3958
|
category: ErrorCategory.Specification,
|
|
3551
3959
|
message: "invalid scenario specification",
|
|
@@ -3553,6 +3961,43 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
3553
3961
|
});
|
|
3554
3962
|
return specs;
|
|
3555
3963
|
}
|
|
3964
|
+
resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
|
|
3965
|
+
const issues = [];
|
|
3966
|
+
const used = /* @__PURE__ */ new Set();
|
|
3967
|
+
const resolver = scenarioResolver.withLevels([run.variables]);
|
|
3968
|
+
const value = {
|
|
3969
|
+
...resolver.resolve(rest, document, issues, secrets, { used }),
|
|
3970
|
+
...variables === void 0 ? {} : { variables }
|
|
3971
|
+
};
|
|
3972
|
+
const websocketIssues = [];
|
|
3973
|
+
let websocket;
|
|
3974
|
+
let connectionChecked = false;
|
|
3975
|
+
if (issues.length === 0 && configDocument !== void 0) {
|
|
3976
|
+
const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
|
|
3977
|
+
if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
|
|
3978
|
+
if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
|
|
3979
|
+
connectionChecked = true;
|
|
3980
|
+
}
|
|
3981
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
|
|
3982
|
+
if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
|
|
3983
|
+
return {
|
|
3984
|
+
run,
|
|
3985
|
+
value,
|
|
3986
|
+
used,
|
|
3987
|
+
secrets,
|
|
3988
|
+
issues,
|
|
3989
|
+
websocket,
|
|
3990
|
+
websocketIssues,
|
|
3991
|
+
connectionChecked
|
|
3992
|
+
};
|
|
3993
|
+
}
|
|
3994
|
+
static byRun(runs, place) {
|
|
3995
|
+
const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
|
|
3996
|
+
const sharedTexts = new Set(shared.map((issue) => issue.toString()));
|
|
3997
|
+
const result = shared.map((issue) => place(issue, void 0));
|
|
3998
|
+
for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
|
|
3999
|
+
return result;
|
|
4000
|
+
}
|
|
3556
4001
|
static shared(lists) {
|
|
3557
4002
|
if (lists.length === 0) return [];
|
|
3558
4003
|
const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
|
|
@@ -3564,11 +4009,22 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
3564
4009
|
path: ""
|
|
3565
4010
|
});
|
|
3566
4011
|
}
|
|
3567
|
-
static unusedVariables(variables, used, document) {
|
|
3568
|
-
|
|
3569
|
-
|
|
4012
|
+
static unusedVariables(variables, used, results, document) {
|
|
4013
|
+
const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
|
|
4014
|
+
const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
|
|
4015
|
+
message: message(name),
|
|
3570
4016
|
location: document.locateKey(["variables"], name)
|
|
3571
4017
|
}));
|
|
4018
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4019
|
+
for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
|
|
4020
|
+
const issue = new ValidationIssue({
|
|
4021
|
+
message: message(key.name),
|
|
4022
|
+
location: key.location
|
|
4023
|
+
});
|
|
4024
|
+
if (!seen.has(issue.toString())) issues.push(issue);
|
|
4025
|
+
seen.add(issue.toString());
|
|
4026
|
+
}
|
|
4027
|
+
return issues;
|
|
3572
4028
|
}
|
|
3573
4029
|
discover(target, config) {
|
|
3574
4030
|
if (!existsSync(target)) throw new EvalError({
|
|
@@ -3594,13 +4050,14 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
3594
4050
|
}
|
|
3595
4051
|
return files;
|
|
3596
4052
|
}
|
|
3597
|
-
build(value, document, websocket, secrets) {
|
|
4053
|
+
build(value, document, runName, websocket, secrets) {
|
|
3598
4054
|
const scenario = value;
|
|
3599
4055
|
const turns = scenario.turns;
|
|
3600
4056
|
const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
|
|
3601
4057
|
return new ConversationSpec({
|
|
3602
4058
|
version: scenario.version,
|
|
3603
4059
|
name: scenario.name,
|
|
4060
|
+
runName,
|
|
3604
4061
|
description: typeof scenario.description === "string" ? scenario.description : "",
|
|
3605
4062
|
turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
|
|
3606
4063
|
conversation: conversation ? new ConversationExpectation({
|
|
@@ -3673,7 +4130,8 @@ var RunCommand = class RunCommand {
|
|
|
3673
4130
|
createEvaluator;
|
|
3674
4131
|
signals;
|
|
3675
4132
|
colour;
|
|
3676
|
-
|
|
4133
|
+
columns;
|
|
4134
|
+
constructor(out, err, environment = process.env, cwd = process.cwd(), createRunner = (config, evaluator) => new ConversationRunner(config, evaluator), createEvaluator = createLlmEvaluator, signals = process, colour = false, columns = void 0) {
|
|
3677
4135
|
this.out = out;
|
|
3678
4136
|
this.err = err;
|
|
3679
4137
|
this.environment = environment;
|
|
@@ -3682,6 +4140,7 @@ var RunCommand = class RunCommand {
|
|
|
3682
4140
|
this.createEvaluator = createEvaluator;
|
|
3683
4141
|
this.signals = signals;
|
|
3684
4142
|
this.colour = colour;
|
|
4143
|
+
this.columns = columns;
|
|
3685
4144
|
}
|
|
3686
4145
|
async execute(target, config, execution = new ExecutionOverride(), report = new ReportOverride(), parameters = "") {
|
|
3687
4146
|
let evalConfig;
|
|
@@ -3708,7 +4167,11 @@ var RunCommand = class RunCommand {
|
|
|
3708
4167
|
const { repeat, minPass, flakyPolicy, scenarioConcurrency } = evalConfig.execution;
|
|
3709
4168
|
const tokenUsageCapability = evalConfig.websocket.protocol.capabilities.tokenUsage;
|
|
3710
4169
|
const consoleReport = evalConfig.report.formats.includes(ReportFormat.Console);
|
|
3711
|
-
const reporter = new ConsoleReporter(report.verbose, this.colour
|
|
4170
|
+
const reporter = new ConsoleReporter(report.verbose, this.colour, {
|
|
4171
|
+
nameWidth: ConsoleReporter.nameWidth(specs),
|
|
4172
|
+
columns: this.columns,
|
|
4173
|
+
thresholds: evalConfig.evaluator.thresholds
|
|
4174
|
+
});
|
|
3712
4175
|
const write = (text) => {
|
|
3713
4176
|
if (consoleReport) this.out(redactor.redact(text));
|
|
3714
4177
|
};
|
|
@@ -3802,7 +4265,7 @@ var ValidateCommand = class {
|
|
|
3802
4265
|
}
|
|
3803
4266
|
};
|
|
3804
4267
|
// src/cli/cli.ts
|
|
3805
|
-
function createProgram(setExitCode, out, err, colour = false) {
|
|
4268
|
+
function createProgram(setExitCode, out, err, colour = false, columns = void 0) {
|
|
3806
4269
|
const program = new Command().name("miniwat-eval").description("Black-box, YAML-driven evaluation for agents built on miniwat-agent").configureOutput({
|
|
3807
4270
|
writeOut: out,
|
|
3808
4271
|
writeErr: err
|
|
@@ -3822,14 +4285,14 @@ function createProgram(setExitCode, out, err, colour = false) {
|
|
|
3822
4285
|
setExitCode(1);
|
|
3823
4286
|
return;
|
|
3824
4287
|
}
|
|
3825
|
-
setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour).execute(target, options.config ?? "", execution, report, options.parameters ?? ""));
|
|
4288
|
+
setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour, columns).execute(target, options.config ?? "", execution, report, options.parameters ?? ""));
|
|
3826
4289
|
});
|
|
3827
4290
|
return program;
|
|
3828
4291
|
}
|
|
3829
4292
|
// src/main.ts
|
|
3830
4293
|
const program = createProgram((code) => {
|
|
3831
4294
|
process.exitCode = code;
|
|
3832
|
-
}, (text) => process.stdout.write(text), (text) => process.stderr.write(text), process.stdout.isTTY === true && !process.env.NO_COLOR);
|
|
4295
|
+
}, (text) => process.stdout.write(text), (text) => process.stderr.write(text), process.stdout.isTTY === true && !process.env.NO_COLOR, process.stdout.isTTY === true ? process.stdout.columns : void 0);
|
|
3833
4296
|
try {
|
|
3834
4297
|
await program.parseAsync(process.argv);
|
|
3835
4298
|
} catch (error) {
|