@mappies/miniwat-eval 1.4.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -326,22 +326,24 @@ A `FAILURES` section follows, with one block per test that did not pass and per
326
326
  Explains how to reset the password using the steps from the knowledge
327
327
  base article it found, without adding steps the article does not support.
328
328
 
329
+ Actual
330
+ I'm afraid the available documentation does not contain a specific
331
+ procedure for resetting a Virginia Cyber Range password. ...
332
+
329
333
  Explanation
330
334
  The response does not explain any password-reset steps from a
331
335
  knowledge-base article.
332
336
 
333
- Evidence
334
- • Actual response: "I'm afraid the available documentation does not..."
335
-
336
337
  Attempts 1: 0 passed, 1 failed, 0 errors (1 required)
337
338
  ```
338
339
 
339
- - **Semantic checks** show the score and threshold, the `should` text, the evaluator's explanation and its evidence.
340
+ - **Semantic checks** show the score and threshold, the `should` text as Expected, the agent's response as Actual (every turn's response for `expect.conversation`, up to 12 lines) and the evaluator's explanation. The evaluator's evidence and the full response are in the JSON report.
340
341
  - **Other checks** list their details, such as the closest tool use and how its arguments differ.
341
342
  - **Repeated runs:** with `repeat` above 1, each failing attempt has its own entry.
342
343
  - **Flaky tests** are listed in a `FLAKY` section after it.
343
344
  - **Summary:** the report ends with a `SUMMARY` of scenario and test counts, agent and evaluator token usage with coverage (such as `coverage 8/9, partial`, when not every call reported usage), the duration and the exit code.
344
345
  - **Wrapping:** text wraps at the terminal's width, or at 100 columns when output is redirected.
346
+ - **Paths** are relative to the directory `npm run` was started from (npm's `INIT_CWD`) when a script changes directory first, so `eval/scenarios/a.yml:11:17` is a link in the terminal. Without npm they are relative to the working directory. Paths you pass are always resolved against the working directory, and the JSON report's paths are relative to it.
345
347
 
346
348
  `--verbose` adds every attempt's transcript under its scenario line. Symbols are coloured on a terminal unless `NO_COLOR` is set; the symbol is shown either way. The console format changed in 1.4.0; scripts should read the JSON report, not the console.
347
349
 
package/dist/index.cjs CHANGED
@@ -703,15 +703,17 @@ const CONFIG_FILE_NAME = "eval-config.yml";
703
703
  const PROJECT_ROOT_MARKERS = [".git", "package.json"];
704
704
  var ConfigLocator = class ConfigLocator {
705
705
  cwd;
706
- constructor(cwd = process.cwd()) {
706
+ displayBase;
707
+ constructor(cwd = process.cwd(), displayBase = cwd) {
707
708
  this.cwd = cwd;
709
+ this.displayBase = displayBase;
708
710
  }
709
711
  locate(target, explicitConfig) {
710
712
  if (explicitConfig !== "") {
711
713
  const path = (0, node_path.resolve)(this.cwd, explicitConfig);
712
714
  if (!ConfigLocator.isFile(path)) throw new EvalError({
713
715
  category: ErrorCategory.Configuration,
714
- message: `configuration file not found: ${explicitConfig}`
716
+ message: `configuration file not found: ${(0, node_path.relative)(this.displayBase, path) || explicitConfig}`
715
717
  });
716
718
  return path;
717
719
  }
@@ -720,7 +722,7 @@ var ConfigLocator = class ConfigLocator {
720
722
  const found = ConfigLocator.searchUpward(start) ?? ConfigLocator.candidate(this.cwd);
721
723
  if (found === void 0) throw new EvalError({
722
724
  category: ErrorCategory.Configuration,
723
- message: `no ${CONFIG_FILE_NAME} found above ${(0, node_path.relative)(this.cwd, targetPath) || "."} or in the working directory; pass --config <path>`
725
+ message: `no ${CONFIG_FILE_NAME} found above ${(0, node_path.relative)(this.displayBase, targetPath) || "."} or in the working directory; pass --config <path>`
724
726
  });
725
727
  return found;
726
728
  }
@@ -1315,17 +1317,19 @@ var WebSocketResolver = class WebSocketResolver {
1315
1317
  // src/config/evalConfigLoader.ts
1316
1318
  var EvalConfigLoader = class EvalConfigLoader {
1317
1319
  cwd;
1320
+ displayBase;
1318
1321
  documentLoader = new YamlDocumentLoader();
1319
1322
  schemaValidator = new SchemaValidator(eval_config_schema_default);
1320
1323
  variablesReader = new VariablesReader();
1321
1324
  webSocketResolver = new WebSocketResolver();
1322
1325
  environmentResolver;
1323
- constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1326
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
1324
1327
  this.cwd = cwd;
1328
+ this.displayBase = displayBase;
1325
1329
  this.environmentResolver = new EnvironmentResolver(environment, parameters);
1326
1330
  }
1327
1331
  load(configPath, override, report = new ReportOverride()) {
1328
- const document = this.documentLoader.load(configPath, (0, node_path.relative)(this.cwd, configPath));
1332
+ const document = this.documentLoader.load(configPath, (0, node_path.relative)(this.displayBase, configPath));
1329
1333
  const issues = [...document.errors];
1330
1334
  if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
1331
1335
  const named = [];
@@ -1481,12 +1485,14 @@ var EvalConfigLoader = class EvalConfigLoader {
1481
1485
  // src/config/parametersLoader.ts
1482
1486
  var ParametersLoader = class ParametersLoader {
1483
1487
  cwd;
1484
- constructor(cwd = process.cwd()) {
1488
+ displayBase;
1489
+ constructor(cwd = process.cwd(), displayBase = cwd) {
1485
1490
  this.cwd = cwd;
1491
+ this.displayBase = displayBase;
1486
1492
  }
1487
1493
  load(path) {
1488
1494
  const absolute = (0, node_path.resolve)(this.cwd, path);
1489
- const shown = (0, node_path.relative)(this.cwd, absolute) || path;
1495
+ const shown = (0, node_path.relative)(this.displayBase, absolute) || path;
1490
1496
  let text;
1491
1497
  let value;
1492
1498
  try {
@@ -1945,6 +1951,7 @@ var RunWithReader = class RunWithReader {
1945
1951
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
1946
1952
  var ScenarioLoader = class ScenarioLoader {
1947
1953
  cwd;
1954
+ displayBase;
1948
1955
  documentLoader = new YamlDocumentLoader();
1949
1956
  schemaValidator = new SchemaValidator(conversation_schema_default);
1950
1957
  conversationValidator = new ConversationValidator();
@@ -1952,18 +1959,19 @@ var ScenarioLoader = class ScenarioLoader {
1952
1959
  runWithReader = new RunWithReader();
1953
1960
  webSocketResolver = new WebSocketResolver();
1954
1961
  environmentResolver;
1955
- constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1962
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
1956
1963
  this.cwd = cwd;
1964
+ this.displayBase = displayBase;
1957
1965
  this.environmentResolver = new EnvironmentResolver(environment, parameters);
1958
1966
  }
1959
1967
  load(target, config) {
1960
1968
  const files = this.discover((0, node_path.resolve)(this.cwd, target), config);
1961
- const configDocument = (0, node_fs.existsSync)(config.configPath) ? this.documentLoader.load(config.configPath, (0, node_path.relative)(this.cwd, config.configPath)) : void 0;
1969
+ const configDocument = (0, node_fs.existsSync)(config.configPath) ? this.documentLoader.load(config.configPath, (0, node_path.relative)(this.displayBase, config.configPath)) : void 0;
1962
1970
  const issues = [];
1963
1971
  const connectionIssues = [];
1964
1972
  const specs = [];
1965
1973
  for (const file of files) {
1966
- const document = this.documentLoader.load(file, (0, node_path.relative)(this.cwd, file));
1974
+ const document = this.documentLoader.load(file, (0, node_path.relative)(this.displayBase, file));
1967
1975
  const fileIssues = [...document.errors];
1968
1976
  const fileSecrets = [];
1969
1977
  const used = /* @__PURE__ */ new Set();
@@ -2093,13 +2101,13 @@ var ScenarioLoader = class ScenarioLoader {
2093
2101
  discover(target, config) {
2094
2102
  if (!(0, node_fs.existsSync)(target)) throw new EvalError({
2095
2103
  category: ErrorCategory.Specification,
2096
- message: `target not found: ${(0, node_path.relative)(this.cwd, target) || "."}`
2104
+ message: `target not found: ${(0, node_path.relative)(this.displayBase, target) || "."}`
2097
2105
  });
2098
2106
  const excluded = (path) => path === config.configPath || config.report.directory !== "" && path === config.report.directory;
2099
2107
  const files = (0, node_fs.statSync)(target).isDirectory() ? this.walk(target, excluded) : [target].filter((file) => !excluded(file));
2100
2108
  if (files.length === 0) throw new EvalError({
2101
2109
  category: ErrorCategory.Specification,
2102
- message: `no scenario files found in ${(0, node_path.relative)(this.cwd, target) || "."}`
2110
+ message: `no scenario files found in ${(0, node_path.relative)(this.displayBase, target) || "."}`
2103
2111
  });
2104
2112
  return files;
2105
2113
  }
@@ -3721,17 +3729,18 @@ var ConsoleReporter = class ConsoleReporter {
3721
3729
  const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
3722
3730
  const failing = test.history.map((entry, index) => ({
3723
3731
  entry,
3732
+ run: result.attempts[index],
3724
3733
  attempt: result.attempts[index]?.attempt ?? index + 1
3725
3734
  })).filter(({ entry }) => entry.status !== ResultStatus.Pass);
3726
3735
  const several = test.history.length > 1;
3727
- for (const { entry, attempt } of failing) {
3736
+ for (const { entry, run, attempt } of failing) {
3728
3737
  if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
3729
- lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
3738
+ lines.push(...this.entryBody(spec, entry, run, several ? 5 : 3));
3730
3739
  }
3731
3740
  lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
3732
3741
  return lines;
3733
3742
  }
3734
- entryBody(spec, entry, indent) {
3743
+ entryBody(spec, entry, attempt, indent) {
3735
3744
  const pad = " ".repeat(indent);
3736
3745
  const evaluation = entry.evaluation;
3737
3746
  if (evaluation === void 0) {
@@ -3745,11 +3754,23 @@ var ConsoleReporter = class ConsoleReporter {
3745
3754
  const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
3746
3755
  const expected = ConsoleReporter.expected(spec, entry);
3747
3756
  const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
3757
+ const actual = ConsoleReporter.actual(entry, attempt);
3748
3758
  if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
3759
+ if (actual !== void 0) lines.push("", `${pad}Actual`, ...this.capped(actual, indent + 2));
3749
3760
  if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
3750
- if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
3751
3761
  return lines;
3752
3762
  }
3763
+ static actual(entry, attempt) {
3764
+ const turns = attempt?.execution.turns ?? [];
3765
+ if (entry.kind === TestKind.Conversation) return turns.length === 0 ? void 0 : turns.map((turn, index) => `Turn ${index + 1}: ${turn.response}`).join("\n");
3766
+ return turns[entry.turn - 1]?.response;
3767
+ }
3768
+ capped(text, indent) {
3769
+ const lines = text.trim() === "" ? [`${" ".repeat(indent)}(empty)`] : this.paragraph(text, indent);
3770
+ if (lines.length <= 12) return lines;
3771
+ const more = lines.length - 12;
3772
+ return [...lines.slice(0, 12), `${" ".repeat(indent)}… ${more} more ${more === 1 ? "line" : "lines"} in the JSON report`];
3773
+ }
3753
3774
  static expected(spec, entry) {
3754
3775
  if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
3755
3776
  return spec.turns[entry.turn - 1]?.expect.response?.criteria;
@@ -3849,10 +3870,12 @@ const JSON_REPORT_VERSION = 1;
3849
3870
  var JsonReporter = class JsonReporter {
3850
3871
  redactor;
3851
3872
  cwd;
3873
+ displayBase;
3852
3874
  tokens = new TokenUsageAggregator();
3853
- constructor(redactor, cwd) {
3875
+ constructor(redactor, cwd, displayBase = cwd) {
3854
3876
  this.redactor = redactor;
3855
3877
  this.cwd = cwd;
3878
+ this.displayBase = displayBase;
3856
3879
  }
3857
3880
  render(run, config, unsuccessfulCount, exitCode) {
3858
3881
  const report = {
@@ -3923,7 +3946,7 @@ var JsonReporter = class JsonReporter {
3923
3946
  return {
3924
3947
  name: result.spec.name,
3925
3948
  runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
3926
- file: result.spec.location.file,
3949
+ file: this.file(result.spec.location.file),
3927
3950
  status: result.status,
3928
3951
  connection: {
3929
3952
  endpoint: websocket.endpoint,
@@ -3933,7 +3956,7 @@ var JsonReporter = class JsonReporter {
3933
3956
  agentUsage: JsonReporter.usage(result.agentUsage),
3934
3957
  evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
3935
3958
  tests: result.tests.map((test) => ({
3936
- ...JsonReporter.identity(test),
3959
+ ...this.identity(test),
3937
3960
  status: test.status,
3938
3961
  ...JsonReporter.counts(test),
3939
3962
  history: test.history.map((entry, index) => ({
@@ -3958,7 +3981,7 @@ var JsonReporter = class JsonReporter {
3958
3981
  agentUsage: JsonReporter.usage(this.tokens.agent([attempt], capability)),
3959
3982
  evaluatorUsage: JsonReporter.usage(this.tokens.evaluator([attempt])),
3960
3983
  turns: execution.turns.map((turn, index) => JsonReporter.turn(index + 1, spec.turns[index].input, turn)),
3961
- tests: attempt.tests.map((test) => JsonReporter.test(test, config))
3984
+ tests: attempt.tests.map((test) => this.test(test, config))
3962
3985
  };
3963
3986
  }
3964
3987
  static turn(number, input, turn) {
@@ -3983,11 +4006,11 @@ var JsonReporter = class JsonReporter {
3983
4006
  tokenUsage: turn.tokenUsage === void 0 ? void 0 : JsonReporter.tokenUsage(turn.tokenUsage)
3984
4007
  };
3985
4008
  }
3986
- static test(test, config) {
4009
+ test(test, config) {
3987
4010
  const thresholds = config.evaluator.thresholds;
3988
4011
  const evaluation = test.evaluation;
3989
4012
  return {
3990
- ...JsonReporter.identity(test),
4013
+ ...this.identity(test),
3991
4014
  status: test.status,
3992
4015
  details: [...test.details],
3993
4016
  evaluation: evaluation === void 0 ? void 0 : {
@@ -4000,17 +4023,17 @@ var JsonReporter = class JsonReporter {
4000
4023
  evaluatorUsage: JsonReporter.usage(test.evaluatorUsage)
4001
4024
  };
4002
4025
  }
4003
- static identity(test) {
4026
+ identity(test) {
4004
4027
  return {
4005
4028
  kind: test.kind,
4006
4029
  turn: test.turn,
4007
4030
  title: test.title,
4008
- location: JsonReporter.location(test.location)
4031
+ location: this.location(test.location)
4009
4032
  };
4010
4033
  }
4011
- static location(location) {
4034
+ location(location) {
4012
4035
  return {
4013
- file: location.file,
4036
+ file: this.file(location.file),
4014
4037
  line: location.line,
4015
4038
  column: location.column,
4016
4039
  path: location.path
@@ -4057,6 +4080,9 @@ var JsonReporter = class JsonReporter {
4057
4080
  path(path) {
4058
4081
  return (0, node_path.relative)(this.cwd, path) || ".";
4059
4082
  }
4083
+ file(shown) {
4084
+ return shown === "" ? shown : this.path((0, node_path.resolve)(this.displayBase, shown));
4085
+ }
4060
4086
  };
4061
4087
  // src/reporting/reportFileWriter.ts
4062
4088
  var ReportFileWriter = class {