@mappies/miniwat-eval 1.4.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/dist/index.cjs +52 -26
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +16 -8
- package/dist/index.d.ts +16 -8
- package/dist/index.js +52 -26
- package/dist/index.js.map +1 -1
- package/dist/main.js +77 -37
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -326,22 +326,24 @@ A `FAILURES` section follows, with one block per test that did not pass and per
|
|
|
326
326
|
Explains how to reset the password using the steps from the knowledge
|
|
327
327
|
base article it found, without adding steps the article does not support.
|
|
328
328
|
|
|
329
|
+
Actual
|
|
330
|
+
I'm afraid the available documentation does not contain a specific
|
|
331
|
+
procedure for resetting a Virginia Cyber Range password. ...
|
|
332
|
+
|
|
329
333
|
Explanation
|
|
330
334
|
The response does not explain any password-reset steps from a
|
|
331
335
|
knowledge-base article.
|
|
332
336
|
|
|
333
|
-
Evidence
|
|
334
|
-
• Actual response: "I'm afraid the available documentation does not..."
|
|
335
|
-
|
|
336
337
|
Attempts 1: 0 passed, 1 failed, 0 errors (1 required)
|
|
337
338
|
```
|
|
338
339
|
|
|
339
|
-
- **Semantic checks** show the score and threshold, the `should` text, the evaluator's explanation and
|
|
340
|
+
- **Semantic checks** show the score and threshold, the `should` text as Expected, the agent's response as Actual (every turn's response for `expect.conversation`, up to 12 lines) and the evaluator's explanation. The evaluator's evidence and the full response are in the JSON report.
|
|
340
341
|
- **Other checks** list their details, such as the closest tool use and how its arguments differ.
|
|
341
342
|
- **Repeated runs:** with `repeat` above 1, each failing attempt has its own entry.
|
|
342
343
|
- **Flaky tests** are listed in a `FLAKY` section after it.
|
|
343
344
|
- **Summary:** the report ends with a `SUMMARY` of scenario and test counts, agent and evaluator token usage with coverage (such as `coverage 8/9, partial`, when not every call reported usage), the duration and the exit code.
|
|
344
345
|
- **Wrapping:** text wraps at the terminal's width, or at 100 columns when output is redirected.
|
|
346
|
+
- **Paths** are relative to the directory `npm run` was started from (npm's `INIT_CWD`) when a script changes directory first, so `eval/scenarios/a.yml:11:17` is a link in the terminal. Without npm they are relative to the working directory. Paths you pass are always resolved against the working directory, and the JSON report's paths are relative to it.
|
|
345
347
|
|
|
346
348
|
`--verbose` adds every attempt's transcript under its scenario line. Symbols are coloured on a terminal unless `NO_COLOR` is set; the symbol is shown either way. The console format changed in 1.4.0; scripts should read the JSON report, not the console.
|
|
347
349
|
|
package/dist/index.cjs
CHANGED
|
@@ -703,15 +703,17 @@ const CONFIG_FILE_NAME = "eval-config.yml";
|
|
|
703
703
|
const PROJECT_ROOT_MARKERS = [".git", "package.json"];
|
|
704
704
|
var ConfigLocator = class ConfigLocator {
|
|
705
705
|
cwd;
|
|
706
|
-
|
|
706
|
+
displayBase;
|
|
707
|
+
constructor(cwd = process.cwd(), displayBase = cwd) {
|
|
707
708
|
this.cwd = cwd;
|
|
709
|
+
this.displayBase = displayBase;
|
|
708
710
|
}
|
|
709
711
|
locate(target, explicitConfig) {
|
|
710
712
|
if (explicitConfig !== "") {
|
|
711
713
|
const path = (0, node_path.resolve)(this.cwd, explicitConfig);
|
|
712
714
|
if (!ConfigLocator.isFile(path)) throw new EvalError({
|
|
713
715
|
category: ErrorCategory.Configuration,
|
|
714
|
-
message: `configuration file not found: ${explicitConfig}`
|
|
716
|
+
message: `configuration file not found: ${(0, node_path.relative)(this.displayBase, path) || explicitConfig}`
|
|
715
717
|
});
|
|
716
718
|
return path;
|
|
717
719
|
}
|
|
@@ -720,7 +722,7 @@ var ConfigLocator = class ConfigLocator {
|
|
|
720
722
|
const found = ConfigLocator.searchUpward(start) ?? ConfigLocator.candidate(this.cwd);
|
|
721
723
|
if (found === void 0) throw new EvalError({
|
|
722
724
|
category: ErrorCategory.Configuration,
|
|
723
|
-
message: `no ${CONFIG_FILE_NAME} found above ${(0, node_path.relative)(this.
|
|
725
|
+
message: `no ${CONFIG_FILE_NAME} found above ${(0, node_path.relative)(this.displayBase, targetPath) || "."} or in the working directory; pass --config <path>`
|
|
724
726
|
});
|
|
725
727
|
return found;
|
|
726
728
|
}
|
|
@@ -1315,17 +1317,19 @@ var WebSocketResolver = class WebSocketResolver {
|
|
|
1315
1317
|
// src/config/evalConfigLoader.ts
|
|
1316
1318
|
var EvalConfigLoader = class EvalConfigLoader {
|
|
1317
1319
|
cwd;
|
|
1320
|
+
displayBase;
|
|
1318
1321
|
documentLoader = new YamlDocumentLoader();
|
|
1319
1322
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
1320
1323
|
variablesReader = new VariablesReader();
|
|
1321
1324
|
webSocketResolver = new WebSocketResolver();
|
|
1322
1325
|
environmentResolver;
|
|
1323
|
-
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1326
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
|
|
1324
1327
|
this.cwd = cwd;
|
|
1328
|
+
this.displayBase = displayBase;
|
|
1325
1329
|
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
1326
1330
|
}
|
|
1327
1331
|
load(configPath, override, report = new ReportOverride()) {
|
|
1328
|
-
const document = this.documentLoader.load(configPath, (0, node_path.relative)(this.
|
|
1332
|
+
const document = this.documentLoader.load(configPath, (0, node_path.relative)(this.displayBase, configPath));
|
|
1329
1333
|
const issues = [...document.errors];
|
|
1330
1334
|
if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
|
|
1331
1335
|
const named = [];
|
|
@@ -1481,12 +1485,14 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1481
1485
|
// src/config/parametersLoader.ts
|
|
1482
1486
|
var ParametersLoader = class ParametersLoader {
|
|
1483
1487
|
cwd;
|
|
1484
|
-
|
|
1488
|
+
displayBase;
|
|
1489
|
+
constructor(cwd = process.cwd(), displayBase = cwd) {
|
|
1485
1490
|
this.cwd = cwd;
|
|
1491
|
+
this.displayBase = displayBase;
|
|
1486
1492
|
}
|
|
1487
1493
|
load(path) {
|
|
1488
1494
|
const absolute = (0, node_path.resolve)(this.cwd, path);
|
|
1489
|
-
const shown = (0, node_path.relative)(this.
|
|
1495
|
+
const shown = (0, node_path.relative)(this.displayBase, absolute) || path;
|
|
1490
1496
|
let text;
|
|
1491
1497
|
let value;
|
|
1492
1498
|
try {
|
|
@@ -1945,6 +1951,7 @@ var RunWithReader = class RunWithReader {
|
|
|
1945
1951
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
1946
1952
|
var ScenarioLoader = class ScenarioLoader {
|
|
1947
1953
|
cwd;
|
|
1954
|
+
displayBase;
|
|
1948
1955
|
documentLoader = new YamlDocumentLoader();
|
|
1949
1956
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
1950
1957
|
conversationValidator = new ConversationValidator();
|
|
@@ -1952,18 +1959,19 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1952
1959
|
runWithReader = new RunWithReader();
|
|
1953
1960
|
webSocketResolver = new WebSocketResolver();
|
|
1954
1961
|
environmentResolver;
|
|
1955
|
-
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1962
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
|
|
1956
1963
|
this.cwd = cwd;
|
|
1964
|
+
this.displayBase = displayBase;
|
|
1957
1965
|
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
1958
1966
|
}
|
|
1959
1967
|
load(target, config) {
|
|
1960
1968
|
const files = this.discover((0, node_path.resolve)(this.cwd, target), config);
|
|
1961
|
-
const configDocument = (0, node_fs.existsSync)(config.configPath) ? this.documentLoader.load(config.configPath, (0, node_path.relative)(this.
|
|
1969
|
+
const configDocument = (0, node_fs.existsSync)(config.configPath) ? this.documentLoader.load(config.configPath, (0, node_path.relative)(this.displayBase, config.configPath)) : void 0;
|
|
1962
1970
|
const issues = [];
|
|
1963
1971
|
const connectionIssues = [];
|
|
1964
1972
|
const specs = [];
|
|
1965
1973
|
for (const file of files) {
|
|
1966
|
-
const document = this.documentLoader.load(file, (0, node_path.relative)(this.
|
|
1974
|
+
const document = this.documentLoader.load(file, (0, node_path.relative)(this.displayBase, file));
|
|
1967
1975
|
const fileIssues = [...document.errors];
|
|
1968
1976
|
const fileSecrets = [];
|
|
1969
1977
|
const used = /* @__PURE__ */ new Set();
|
|
@@ -2093,13 +2101,13 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
2093
2101
|
discover(target, config) {
|
|
2094
2102
|
if (!(0, node_fs.existsSync)(target)) throw new EvalError({
|
|
2095
2103
|
category: ErrorCategory.Specification,
|
|
2096
|
-
message: `target not found: ${(0, node_path.relative)(this.
|
|
2104
|
+
message: `target not found: ${(0, node_path.relative)(this.displayBase, target) || "."}`
|
|
2097
2105
|
});
|
|
2098
2106
|
const excluded = (path) => path === config.configPath || config.report.directory !== "" && path === config.report.directory;
|
|
2099
2107
|
const files = (0, node_fs.statSync)(target).isDirectory() ? this.walk(target, excluded) : [target].filter((file) => !excluded(file));
|
|
2100
2108
|
if (files.length === 0) throw new EvalError({
|
|
2101
2109
|
category: ErrorCategory.Specification,
|
|
2102
|
-
message: `no scenario files found in ${(0, node_path.relative)(this.
|
|
2110
|
+
message: `no scenario files found in ${(0, node_path.relative)(this.displayBase, target) || "."}`
|
|
2103
2111
|
});
|
|
2104
2112
|
return files;
|
|
2105
2113
|
}
|
|
@@ -3721,17 +3729,18 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3721
3729
|
const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
|
|
3722
3730
|
const failing = test.history.map((entry, index) => ({
|
|
3723
3731
|
entry,
|
|
3732
|
+
run: result.attempts[index],
|
|
3724
3733
|
attempt: result.attempts[index]?.attempt ?? index + 1
|
|
3725
3734
|
})).filter(({ entry }) => entry.status !== ResultStatus.Pass);
|
|
3726
3735
|
const several = test.history.length > 1;
|
|
3727
|
-
for (const { entry, attempt } of failing) {
|
|
3736
|
+
for (const { entry, run, attempt } of failing) {
|
|
3728
3737
|
if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
|
|
3729
|
-
lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
|
|
3738
|
+
lines.push(...this.entryBody(spec, entry, run, several ? 5 : 3));
|
|
3730
3739
|
}
|
|
3731
3740
|
lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
|
|
3732
3741
|
return lines;
|
|
3733
3742
|
}
|
|
3734
|
-
entryBody(spec, entry, indent) {
|
|
3743
|
+
entryBody(spec, entry, attempt, indent) {
|
|
3735
3744
|
const pad = " ".repeat(indent);
|
|
3736
3745
|
const evaluation = entry.evaluation;
|
|
3737
3746
|
if (evaluation === void 0) {
|
|
@@ -3745,11 +3754,23 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3745
3754
|
const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
|
|
3746
3755
|
const expected = ConsoleReporter.expected(spec, entry);
|
|
3747
3756
|
const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
|
|
3757
|
+
const actual = ConsoleReporter.actual(entry, attempt);
|
|
3748
3758
|
if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
|
|
3759
|
+
if (actual !== void 0) lines.push("", `${pad}Actual`, ...this.capped(actual, indent + 2));
|
|
3749
3760
|
if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
|
|
3750
|
-
if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
|
|
3751
3761
|
return lines;
|
|
3752
3762
|
}
|
|
3763
|
+
static actual(entry, attempt) {
|
|
3764
|
+
const turns = attempt?.execution.turns ?? [];
|
|
3765
|
+
if (entry.kind === TestKind.Conversation) return turns.length === 0 ? void 0 : turns.map((turn, index) => `Turn ${index + 1}: ${turn.response}`).join("\n");
|
|
3766
|
+
return turns[entry.turn - 1]?.response;
|
|
3767
|
+
}
|
|
3768
|
+
capped(text, indent) {
|
|
3769
|
+
const lines = text.trim() === "" ? [`${" ".repeat(indent)}(empty)`] : this.paragraph(text, indent);
|
|
3770
|
+
if (lines.length <= 12) return lines;
|
|
3771
|
+
const more = lines.length - 12;
|
|
3772
|
+
return [...lines.slice(0, 12), `${" ".repeat(indent)}… ${more} more ${more === 1 ? "line" : "lines"} in the JSON report`];
|
|
3773
|
+
}
|
|
3753
3774
|
static expected(spec, entry) {
|
|
3754
3775
|
if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
|
|
3755
3776
|
return spec.turns[entry.turn - 1]?.expect.response?.criteria;
|
|
@@ -3849,10 +3870,12 @@ const JSON_REPORT_VERSION = 1;
|
|
|
3849
3870
|
var JsonReporter = class JsonReporter {
|
|
3850
3871
|
redactor;
|
|
3851
3872
|
cwd;
|
|
3873
|
+
displayBase;
|
|
3852
3874
|
tokens = new TokenUsageAggregator();
|
|
3853
|
-
constructor(redactor, cwd) {
|
|
3875
|
+
constructor(redactor, cwd, displayBase = cwd) {
|
|
3854
3876
|
this.redactor = redactor;
|
|
3855
3877
|
this.cwd = cwd;
|
|
3878
|
+
this.displayBase = displayBase;
|
|
3856
3879
|
}
|
|
3857
3880
|
render(run, config, unsuccessfulCount, exitCode) {
|
|
3858
3881
|
const report = {
|
|
@@ -3923,7 +3946,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3923
3946
|
return {
|
|
3924
3947
|
name: result.spec.name,
|
|
3925
3948
|
runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
|
|
3926
|
-
file: result.spec.location.file,
|
|
3949
|
+
file: this.file(result.spec.location.file),
|
|
3927
3950
|
status: result.status,
|
|
3928
3951
|
connection: {
|
|
3929
3952
|
endpoint: websocket.endpoint,
|
|
@@ -3933,7 +3956,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3933
3956
|
agentUsage: JsonReporter.usage(result.agentUsage),
|
|
3934
3957
|
evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
|
|
3935
3958
|
tests: result.tests.map((test) => ({
|
|
3936
|
-
...
|
|
3959
|
+
...this.identity(test),
|
|
3937
3960
|
status: test.status,
|
|
3938
3961
|
...JsonReporter.counts(test),
|
|
3939
3962
|
history: test.history.map((entry, index) => ({
|
|
@@ -3958,7 +3981,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3958
3981
|
agentUsage: JsonReporter.usage(this.tokens.agent([attempt], capability)),
|
|
3959
3982
|
evaluatorUsage: JsonReporter.usage(this.tokens.evaluator([attempt])),
|
|
3960
3983
|
turns: execution.turns.map((turn, index) => JsonReporter.turn(index + 1, spec.turns[index].input, turn)),
|
|
3961
|
-
tests: attempt.tests.map((test) =>
|
|
3984
|
+
tests: attempt.tests.map((test) => this.test(test, config))
|
|
3962
3985
|
};
|
|
3963
3986
|
}
|
|
3964
3987
|
static turn(number, input, turn) {
|
|
@@ -3983,11 +4006,11 @@ var JsonReporter = class JsonReporter {
|
|
|
3983
4006
|
tokenUsage: turn.tokenUsage === void 0 ? void 0 : JsonReporter.tokenUsage(turn.tokenUsage)
|
|
3984
4007
|
};
|
|
3985
4008
|
}
|
|
3986
|
-
|
|
4009
|
+
test(test, config) {
|
|
3987
4010
|
const thresholds = config.evaluator.thresholds;
|
|
3988
4011
|
const evaluation = test.evaluation;
|
|
3989
4012
|
return {
|
|
3990
|
-
...
|
|
4013
|
+
...this.identity(test),
|
|
3991
4014
|
status: test.status,
|
|
3992
4015
|
details: [...test.details],
|
|
3993
4016
|
evaluation: evaluation === void 0 ? void 0 : {
|
|
@@ -4000,17 +4023,17 @@ var JsonReporter = class JsonReporter {
|
|
|
4000
4023
|
evaluatorUsage: JsonReporter.usage(test.evaluatorUsage)
|
|
4001
4024
|
};
|
|
4002
4025
|
}
|
|
4003
|
-
|
|
4026
|
+
identity(test) {
|
|
4004
4027
|
return {
|
|
4005
4028
|
kind: test.kind,
|
|
4006
4029
|
turn: test.turn,
|
|
4007
4030
|
title: test.title,
|
|
4008
|
-
location:
|
|
4031
|
+
location: this.location(test.location)
|
|
4009
4032
|
};
|
|
4010
4033
|
}
|
|
4011
|
-
|
|
4034
|
+
location(location) {
|
|
4012
4035
|
return {
|
|
4013
|
-
file: location.file,
|
|
4036
|
+
file: this.file(location.file),
|
|
4014
4037
|
line: location.line,
|
|
4015
4038
|
column: location.column,
|
|
4016
4039
|
path: location.path
|
|
@@ -4057,6 +4080,9 @@ var JsonReporter = class JsonReporter {
|
|
|
4057
4080
|
path(path) {
|
|
4058
4081
|
return (0, node_path.relative)(this.cwd, path) || ".";
|
|
4059
4082
|
}
|
|
4083
|
+
file(shown) {
|
|
4084
|
+
return shown === "" ? shown : this.path((0, node_path.resolve)(this.displayBase, shown));
|
|
4085
|
+
}
|
|
4060
4086
|
};
|
|
4061
4087
|
// src/reporting/reportFileWriter.ts
|
|
4062
4088
|
var ReportFileWriter = class {
|