@mappies/miniwat-eval 1.3.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -299,13 +299,53 @@ A turn stops its attempt when the host reports an error (`statusCode >= 400` or
299
299
 
300
300
  ### Reports
301
301
 
302
- The console prints one line per scenario as it finishes, in file order. A Problems section follows for each scenario that did not pass. It lists attempts that stopped and why, then each test that did not pass with its location, its counts across attempts and the details of each failing attempt, including evaluator scores and explanations. The report ends with:
302
+ The console prints one line per scenario as it finishes, in file order: a status symbol, the name, the attempts that passed out of `repeat`, and the time it took.
303
303
 
304
- - scenario and test counts;
305
- - agent and evaluator token usage with coverage, such as `coverage 8/9, partial`, when not every call reported usage;
306
- - the duration and the exit code.
304
+ ```text
305
+ ✔ Greeting [user-token] 1/1 2.6s
306
+ ✖ Password reset 0/1 7.9s
307
+ ```
308
+
309
+ | Symbol | Status |
310
+ |---|---|
311
+ | `✔` | `PASS` |
312
+ | `✖` | `FAIL` |
313
+ | `⚠` | `FLAKY` |
314
+ | `●` | `ERROR` |
315
+ | `○` | `SKIPPED` (a test that did not run) |
316
+ | `◌` | `INTERRUPTED`, or a scenario that never started |
317
+
318
+ A `FAILURES` section follows, with one block per test that did not pass and per attempt that stopped:
319
+
320
+ ```text
321
+ test at scenarios/authenticated/knowledge-base/password-reset.yml:11:9
322
+ ✖ Password reset › turn 1: response
323
+ Score 0.00, threshold 0.85
324
+
325
+ Expected
326
+ Explains how to reset the password using the steps from the knowledge
327
+ base article it found, without adding steps the article does not support.
328
+
329
+ Actual
330
+ I'm afraid the available documentation does not contain a specific
331
+ procedure for resetting a Virginia Cyber Range password. ...
332
+
333
+ Explanation
334
+ The response does not explain any password-reset steps from a
335
+ knowledge-base article.
336
+
337
+ Attempts 1: 0 passed, 1 failed, 0 errors (1 required)
338
+ ```
339
+
340
+ - **Semantic checks** show the score and threshold, the `should` text as Expected, the agent's response as Actual (every turn's response for `expect.conversation`, up to 12 lines) and the evaluator's explanation. The evaluator's evidence and the full response are in the JSON report.
341
+ - **Other checks** list their details, such as the closest tool use and how its arguments differ.
342
+ - **Repeated runs:** with `repeat` above 1, each failing attempt has its own entry.
343
+ - **Flaky tests** are listed in a `FLAKY` section after it.
344
+ - **Summary:** the report ends with a `SUMMARY` of scenario and test counts, agent and evaluator token usage with coverage (such as `coverage 8/9, partial`, when not every call reported usage), the duration and the exit code.
345
+ - **Wrapping:** text wraps at the terminal's width, or at 100 columns when output is redirected.
346
+ - **Paths** are relative to the directory `npm run` was started from (npm's `INIT_CWD`) when a script changes directory first, so `eval/scenarios/a.yml:11:17` is a link in the terminal. Without npm they are relative to the working directory. Paths you pass are always resolved against the working directory, and the JSON report's paths are relative to it.
307
347
 
308
- `--verbose` adds every attempt's transcript. Status words are coloured on a terminal unless `NO_COLOR` is set.
348
+ `--verbose` adds every attempt's transcript under its scenario line. Symbols are coloured on a terminal unless `NO_COLOR` is set; the symbol is shown either way. The console format changed in 1.4.0; scripts should read the JSON report, not the console.
309
349
 
310
350
  `json` writes `miniwat-eval-report.json` to `report.directory`, replacing the previous one, and prints `Report: <path>` above the final line. It follows `schemas/report.schema.json` and holds:
311
351
 
package/dist/index.cjs CHANGED
@@ -703,15 +703,17 @@ const CONFIG_FILE_NAME = "eval-config.yml";
703
703
  const PROJECT_ROOT_MARKERS = [".git", "package.json"];
704
704
  var ConfigLocator = class ConfigLocator {
705
705
  cwd;
706
- constructor(cwd = process.cwd()) {
706
+ displayBase;
707
+ constructor(cwd = process.cwd(), displayBase = cwd) {
707
708
  this.cwd = cwd;
709
+ this.displayBase = displayBase;
708
710
  }
709
711
  locate(target, explicitConfig) {
710
712
  if (explicitConfig !== "") {
711
713
  const path = (0, node_path.resolve)(this.cwd, explicitConfig);
712
714
  if (!ConfigLocator.isFile(path)) throw new EvalError({
713
715
  category: ErrorCategory.Configuration,
714
- message: `configuration file not found: ${explicitConfig}`
716
+ message: `configuration file not found: ${(0, node_path.relative)(this.displayBase, path) || explicitConfig}`
715
717
  });
716
718
  return path;
717
719
  }
@@ -720,7 +722,7 @@ var ConfigLocator = class ConfigLocator {
720
722
  const found = ConfigLocator.searchUpward(start) ?? ConfigLocator.candidate(this.cwd);
721
723
  if (found === void 0) throw new EvalError({
722
724
  category: ErrorCategory.Configuration,
723
- message: `no ${CONFIG_FILE_NAME} found above ${(0, node_path.relative)(this.cwd, targetPath) || "."} or in the working directory; pass --config <path>`
725
+ message: `no ${CONFIG_FILE_NAME} found above ${(0, node_path.relative)(this.displayBase, targetPath) || "."} or in the working directory; pass --config <path>`
724
726
  });
725
727
  return found;
726
728
  }
@@ -1315,17 +1317,19 @@ var WebSocketResolver = class WebSocketResolver {
1315
1317
  // src/config/evalConfigLoader.ts
1316
1318
  var EvalConfigLoader = class EvalConfigLoader {
1317
1319
  cwd;
1320
+ displayBase;
1318
1321
  documentLoader = new YamlDocumentLoader();
1319
1322
  schemaValidator = new SchemaValidator(eval_config_schema_default);
1320
1323
  variablesReader = new VariablesReader();
1321
1324
  webSocketResolver = new WebSocketResolver();
1322
1325
  environmentResolver;
1323
- constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1326
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
1324
1327
  this.cwd = cwd;
1328
+ this.displayBase = displayBase;
1325
1329
  this.environmentResolver = new EnvironmentResolver(environment, parameters);
1326
1330
  }
1327
1331
  load(configPath, override, report = new ReportOverride()) {
1328
- const document = this.documentLoader.load(configPath, (0, node_path.relative)(this.cwd, configPath));
1332
+ const document = this.documentLoader.load(configPath, (0, node_path.relative)(this.displayBase, configPath));
1329
1333
  const issues = [...document.errors];
1330
1334
  if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
1331
1335
  const named = [];
@@ -1481,12 +1485,14 @@ var EvalConfigLoader = class EvalConfigLoader {
1481
1485
  // src/config/parametersLoader.ts
1482
1486
  var ParametersLoader = class ParametersLoader {
1483
1487
  cwd;
1484
- constructor(cwd = process.cwd()) {
1488
+ displayBase;
1489
+ constructor(cwd = process.cwd(), displayBase = cwd) {
1485
1490
  this.cwd = cwd;
1491
+ this.displayBase = displayBase;
1486
1492
  }
1487
1493
  load(path) {
1488
1494
  const absolute = (0, node_path.resolve)(this.cwd, path);
1489
- const shown = (0, node_path.relative)(this.cwd, absolute) || path;
1495
+ const shown = (0, node_path.relative)(this.displayBase, absolute) || path;
1490
1496
  let text;
1491
1497
  let value;
1492
1498
  try {
@@ -1945,6 +1951,7 @@ var RunWithReader = class RunWithReader {
1945
1951
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
1946
1952
  var ScenarioLoader = class ScenarioLoader {
1947
1953
  cwd;
1954
+ displayBase;
1948
1955
  documentLoader = new YamlDocumentLoader();
1949
1956
  schemaValidator = new SchemaValidator(conversation_schema_default);
1950
1957
  conversationValidator = new ConversationValidator();
@@ -1952,18 +1959,19 @@ var ScenarioLoader = class ScenarioLoader {
1952
1959
  runWithReader = new RunWithReader();
1953
1960
  webSocketResolver = new WebSocketResolver();
1954
1961
  environmentResolver;
1955
- constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1962
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
1956
1963
  this.cwd = cwd;
1964
+ this.displayBase = displayBase;
1957
1965
  this.environmentResolver = new EnvironmentResolver(environment, parameters);
1958
1966
  }
1959
1967
  load(target, config) {
1960
1968
  const files = this.discover((0, node_path.resolve)(this.cwd, target), config);
1961
- const configDocument = (0, node_fs.existsSync)(config.configPath) ? this.documentLoader.load(config.configPath, (0, node_path.relative)(this.cwd, config.configPath)) : void 0;
1969
+ const configDocument = (0, node_fs.existsSync)(config.configPath) ? this.documentLoader.load(config.configPath, (0, node_path.relative)(this.displayBase, config.configPath)) : void 0;
1962
1970
  const issues = [];
1963
1971
  const connectionIssues = [];
1964
1972
  const specs = [];
1965
1973
  for (const file of files) {
1966
- const document = this.documentLoader.load(file, (0, node_path.relative)(this.cwd, file));
1974
+ const document = this.documentLoader.load(file, (0, node_path.relative)(this.displayBase, file));
1967
1975
  const fileIssues = [...document.errors];
1968
1976
  const fileSecrets = [];
1969
1977
  const used = /* @__PURE__ */ new Set();
@@ -2093,13 +2101,13 @@ var ScenarioLoader = class ScenarioLoader {
2093
2101
  discover(target, config) {
2094
2102
  if (!(0, node_fs.existsSync)(target)) throw new EvalError({
2095
2103
  category: ErrorCategory.Specification,
2096
- message: `target not found: ${(0, node_path.relative)(this.cwd, target) || "."}`
2104
+ message: `target not found: ${(0, node_path.relative)(this.displayBase, target) || "."}`
2097
2105
  });
2098
2106
  const excluded = (path) => path === config.configPath || config.report.directory !== "" && path === config.report.directory;
2099
2107
  const files = (0, node_fs.statSync)(target).isDirectory() ? this.walk(target, excluded) : [target].filter((file) => !excluded(file));
2100
2108
  if (files.length === 0) throw new EvalError({
2101
2109
  category: ErrorCategory.Specification,
2102
- message: `no scenario files found in ${(0, node_path.relative)(this.cwd, target) || "."}`
2110
+ message: `no scenario files found in ${(0, node_path.relative)(this.displayBase, target) || "."}`
2103
2111
  });
2104
2112
  return files;
2105
2113
  }
@@ -3557,64 +3565,94 @@ var ExitCodeCalculator = class {
3557
3565
  }
3558
3566
  };
3559
3567
  // src/reporting/consoleReporter.ts
3560
- const STATUS_WIDTH = 11;
3561
- const COLOURS = {
3562
- [ResultStatus.Pass]: "green",
3563
- [ResultStatus.Flaky]: "yellow",
3564
- [ResultStatus.Fail]: "red",
3565
- [ResultStatus.Error]: "red",
3566
- [ResultStatus.Skipped]: "dim",
3567
- [ResultStatus.Interrupted]: "dim"
3568
+ const MARKS = {
3569
+ [ResultStatus.Pass]: {
3570
+ symbol: "✔",
3571
+ colour: "green"
3572
+ },
3573
+ [ResultStatus.Flaky]: {
3574
+ symbol: "⚠",
3575
+ colour: "yellow"
3576
+ },
3577
+ [ResultStatus.Fail]: {
3578
+ symbol: "✖",
3579
+ colour: "red"
3580
+ },
3581
+ [ResultStatus.Error]: {
3582
+ symbol: "●",
3583
+ colour: "red"
3584
+ },
3585
+ [ResultStatus.Skipped]: {
3586
+ symbol: "○",
3587
+ colour: "dim"
3588
+ },
3589
+ [ResultStatus.Interrupted]: {
3590
+ symbol: "◌",
3591
+ colour: "dim"
3592
+ }
3568
3593
  };
3594
+ const MIN_COLUMNS = 40;
3595
+ const RULE_WIDTH = 72;
3569
3596
  var ConsoleReporter = class ConsoleReporter {
3570
3597
  verbose;
3571
3598
  colour;
3572
- constructor(verbose, colour) {
3599
+ nameWidth;
3600
+ columns;
3601
+ thresholds;
3602
+ constructor(verbose, colour, options = {}) {
3573
3603
  this.verbose = verbose;
3574
3604
  this.colour = colour;
3605
+ this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
3606
+ this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
3607
+ this.thresholds = options.thresholds;
3608
+ }
3609
+ static nameWidth(specs) {
3610
+ return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
3575
3611
  }
3576
3612
  header(run) {
3577
3613
  return [
3578
- "Miniwat Eval",
3614
+ " Miniwat Eval",
3579
3615
  "",
3580
- ...run.quick ? ["Execution Mode: Quick"] : [],
3581
- `Repeat: ${run.repeat}`,
3582
- `Minimum Pass: ${run.minPass}`,
3616
+ ...run.quick ? [" Execution mode Quick"] : [],
3617
+ ` Repeat ${run.repeat}`,
3618
+ ` Minimum pass ${run.minPass}`,
3583
3619
  "",
3584
- `Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
3620
+ ` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
3585
3621
  "",
3586
3622
  ""
3587
3623
  ].join("\n");
3588
3624
  }
3589
3625
  scenario(result, repeat, tokenUsageCapability) {
3590
3626
  const spec = result.spec;
3591
- const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
3592
- const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
3627
+ const title = ConsoleReporter.title(spec);
3628
+ const started = result.attempts.length > 0;
3629
+ const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
3630
+ const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
3631
+ const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
3593
3632
  if (!this.verbose) return line;
3594
- if (result.attempts.length === 0) return `${line}\n`;
3633
+ if (!started) return `${line}\n`;
3595
3634
  const websocket = spec.websocket;
3596
3635
  const subprotocols = websocket?.connection.subprotocols.length ?? 0;
3597
- const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
3598
- for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3599
- lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
3600
- lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
3601
- lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
3602
- lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3636
+ const lines = [];
3637
+ if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
3638
+ for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3639
+ lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
3640
+ lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3603
3641
  return line + lines.join("\n");
3604
3642
  }
3605
3643
  static title(spec) {
3606
3644
  return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
3607
3645
  }
3608
3646
  summary(run, exitCode, reportPaths = []) {
3609
- const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
3647
+ const started = run.scenarios.filter((result) => result.attempts.length > 0);
3648
+ const failures = started.flatMap((result) => this.blocks(result, false));
3649
+ const flaky = started.flatMap((result) => this.blocks(result, true));
3610
3650
  const lines = this.verbose ? [] : [""];
3611
- if (problems.length > 0) {
3612
- lines.push("Problems", "");
3613
- for (const result of problems) lines.push(...this.problems(result), "");
3614
- }
3651
+ if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
3652
+ if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
3615
3653
  const tests = run.scenarios.flatMap((result) => result.tests);
3616
- lines.push(...run.status === RunStatus.Interrupted ? [`Run: ${run.status}`] : [], `Scenarios: ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, `Tests: ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability), ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true), `Duration: ${ConsoleReporter.duration(run.durationMs)}`, "", "");
3617
- return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
3654
+ lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
3655
+ return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
3618
3656
  }
3619
3657
  static footer(run, exitCode, reportPaths) {
3620
3658
  return [
@@ -3628,57 +3666,148 @@ var ConsoleReporter = class ConsoleReporter {
3628
3666
  return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
3629
3667
  }
3630
3668
  static duration(ms) {
3669
+ if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
3631
3670
  const seconds = Math.floor(ms / 1e3);
3632
3671
  const pad = (value) => String(value).padStart(2, "0");
3633
3672
  const hours = Math.floor(seconds / 3600);
3634
- const rest = `${pad(Math.floor(seconds / 60) % 60)}:${pad(seconds % 60)}`;
3635
- return hours > 0 ? `${hours}:${rest}` : rest;
3673
+ const minutes = Math.floor(seconds / 60) % 60;
3674
+ return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
3636
3675
  }
3637
- problems(result) {
3676
+ static wrap(text, width) {
3677
+ const lines = [];
3678
+ for (const paragraph of text.split("\n")) {
3679
+ let line = "";
3680
+ for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
3681
+ lines.push(line);
3682
+ line = word;
3683
+ } else line = line === "" ? word : `${line} ${word}`;
3684
+ lines.push(line);
3685
+ }
3686
+ return lines;
3687
+ }
3688
+ section(name, blocks) {
3689
+ return [
3690
+ ...this.rule(name),
3691
+ "",
3692
+ ...blocks.flatMap((block) => [...block, ""])
3693
+ ];
3694
+ }
3695
+ rule(name) {
3696
+ const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
3697
+ return [
3698
+ rule,
3699
+ ` ${name}`,
3700
+ rule
3701
+ ];
3702
+ }
3703
+ blocks(result, flaky) {
3638
3704
  const spec = result.spec;
3639
- const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
3640
- for (const attempt of result.attempts) {
3641
- const execution = attempt.execution;
3642
- if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3643
- for (const error of execution.attemptErrors) lines.push(` Attempt ${attempt.attempt} error: ${error.category} error: ${error.message}`);
3644
- if (execution.closeError !== void 0) lines.push(` Attempt ${attempt.attempt} close error: ${execution.closeError.message}`);
3645
- }
3646
- for (const test of result.tests.filter((entry) => entry.status !== ResultStatus.Pass)) {
3647
- lines.push(` ${this.status(test.status, 6)} ${ConsoleReporter.testTitle(test)} ${test.location.toString()} ${ConsoleReporter.counts(test)}`);
3648
- test.history.forEach((entry, index) => {
3649
- if (entry.status === ResultStatus.Pass) return;
3650
- const prefix = ` Attempt ${result.attempts[index].attempt} ${this.status(entry.status)}`;
3651
- const width = ` Attempt ${result.attempts[index].attempt} ${entry.status}: `.length;
3652
- if (entry.details.length === 0) {
3653
- lines.push(prefix);
3654
- return;
3655
- }
3656
- lines.push(`${prefix}: ${ConsoleReporter.indent(entry.details[0], width)}`, ...entry.details.slice(1).map((detail) => `${" ".repeat(width)}${ConsoleReporter.indent(detail, width)}`));
3657
- });
3705
+ const title = ConsoleReporter.title(spec);
3706
+ const blocks = [];
3707
+ if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
3708
+ const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
3709
+ for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
3710
+ return blocks;
3711
+ }
3712
+ attemptBlock(spec, title, attempt) {
3713
+ const execution = attempt.execution;
3714
+ const problems = [];
3715
+ if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
3716
+ for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
3717
+ if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
3718
+ if (problems.length === 0) return [];
3719
+ const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
3720
+ const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
3721
+ return [[
3722
+ ` scenario at ${spec.location.file}`,
3723
+ ` ${this.mark(status)} ${title} › ${what}`,
3724
+ ...problems.flatMap((problem) => this.paragraph(problem, 3))
3725
+ ]];
3726
+ }
3727
+ testBlock(spec, title, test, result) {
3728
+ const location = test.location;
3729
+ const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
3730
+ const failing = test.history.map((entry, index) => ({
3731
+ entry,
3732
+ run: result.attempts[index],
3733
+ attempt: result.attempts[index]?.attempt ?? index + 1
3734
+ })).filter(({ entry }) => entry.status !== ResultStatus.Pass);
3735
+ const several = test.history.length > 1;
3736
+ for (const { entry, run, attempt } of failing) {
3737
+ if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
3738
+ lines.push(...this.entryBody(spec, entry, run, several ? 5 : 3));
3739
+ }
3740
+ lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
3741
+ return lines;
3742
+ }
3743
+ entryBody(spec, entry, attempt, indent) {
3744
+ const pad = " ".repeat(indent);
3745
+ const evaluation = entry.evaluation;
3746
+ if (evaluation === void 0) {
3747
+ if (entry.details.length === 0) return [];
3748
+ return [
3749
+ "",
3750
+ `${pad}Details`,
3751
+ ...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
3752
+ ];
3658
3753
  }
3754
+ const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
3755
+ const expected = ConsoleReporter.expected(spec, entry);
3756
+ const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
3757
+ const actual = ConsoleReporter.actual(entry, attempt);
3758
+ if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
3759
+ if (actual !== void 0) lines.push("", `${pad}Actual`, ...this.capped(actual, indent + 2));
3760
+ if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
3659
3761
  return lines;
3660
3762
  }
3763
+ static actual(entry, attempt) {
3764
+ const turns = attempt?.execution.turns ?? [];
3765
+ if (entry.kind === TestKind.Conversation) return turns.length === 0 ? void 0 : turns.map((turn, index) => `Turn ${index + 1}: ${turn.response}`).join("\n");
3766
+ return turns[entry.turn - 1]?.response;
3767
+ }
3768
+ capped(text, indent) {
3769
+ const lines = text.trim() === "" ? [`${" ".repeat(indent)}(empty)`] : this.paragraph(text, indent);
3770
+ if (lines.length <= 12) return lines;
3771
+ const more = lines.length - 12;
3772
+ return [...lines.slice(0, 12), `${" ".repeat(indent)}… ${more} more ${more === 1 ? "line" : "lines"} in the JSON report`];
3773
+ }
3774
+ static expected(spec, entry) {
3775
+ if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
3776
+ return spec.turns[entry.turn - 1]?.expect.response?.criteria;
3777
+ }
3778
+ paragraph(text, indent) {
3779
+ const pad = " ".repeat(indent);
3780
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
3781
+ }
3782
+ bullet(text, indent) {
3783
+ const pad = " ".repeat(indent);
3784
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
3785
+ }
3661
3786
  static testTitle(test) {
3662
3787
  return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
3663
3788
  }
3664
3789
  static tally(statuses) {
3665
3790
  const count = (status) => statuses.filter((entry) => entry === status).length;
3666
- const interrupted = count(ResultStatus.Interrupted);
3667
- return `${count(ResultStatus.Pass)} passed, ${count(ResultStatus.Flaky)} flaky, ${count(ResultStatus.Fail)} failed, ${count(ResultStatus.Error)} errors${interrupted > 0 ? `, ${interrupted} interrupted` : ""}`;
3668
- }
3669
- status(status, width = 0) {
3670
- const padding = " ".repeat(Math.max(0, width - status.length));
3671
- return (this.colour ? (0, node_util.styleText)(COLOURS[status], status, { validateStream: false }) : status) + padding;
3672
- }
3673
- static counts(result) {
3674
- return `(${result.passed} passed, ${result.failed} failed, ${result.errors} errors; required ${result.required})`;
3791
+ const parts = [
3792
+ [ResultStatus.Pass, "passed"],
3793
+ [ResultStatus.Flaky, "flaky"],
3794
+ [ResultStatus.Fail, "failed"],
3795
+ [ResultStatus.Error, "errors"],
3796
+ [ResultStatus.Interrupted, "interrupted"]
3797
+ ].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
3798
+ return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
3799
+ }
3800
+ mark(status) {
3801
+ const { symbol, colour } = MARKS[status];
3802
+ return this.colour ? (0, node_util.styleText)(colour, symbol, { validateStream: false }) : symbol;
3675
3803
  }
3676
3804
  static usage(label, usage, capability) {
3677
- if (!capability) return `${label}: n/a (not supported by host)`;
3678
- if (usage.expected === 0) return `${label}: none`;
3679
- if (usage.total === void 0) return `${label}: n/a (coverage 0/${usage.expected})`;
3805
+ const name = label.padEnd(16);
3806
+ if (!capability) return `${name} n/a (not supported by host)`;
3807
+ if (usage.expected === 0) return `${name} none`;
3808
+ if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
3680
3809
  const number = (value) => value.toLocaleString("en-US");
3681
- const total = `${label}: total ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3810
+ const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3682
3811
  return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
3683
3812
  }
3684
3813
  formatAttempt(spec, attempt) {
@@ -3690,28 +3819,28 @@ var ConsoleReporter = class ConsoleReporter {
3690
3819
  lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
3691
3820
  lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
3692
3821
  });
3693
- if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3822
+ if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3694
3823
  const notRun = attempt.tests.filter((test) => test.turn > ran);
3695
- if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3824
+ if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3696
3825
  lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
3697
- for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3698
- if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3699
- lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3826
+ for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3827
+ if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3828
+ lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3700
3829
  return lines;
3701
3830
  }
3702
3831
  formatTest(test) {
3703
- return [` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${detail}`)];
3832
+ return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
3704
3833
  }
3705
3834
  static formatTurn(number, input, turn) {
3706
3835
  const lines = [
3707
- ` Turn ${number} (${turn.durationMs} ms)`,
3708
- ` Input: ${ConsoleReporter.indent(input, 17)}`,
3709
- ` Response: ${ConsoleReporter.indent(turn.response, 17)}`
3836
+ ` Turn ${number} (${turn.durationMs} ms)`,
3837
+ ` Input: ${ConsoleReporter.indent(input, 22)}`,
3838
+ ` Response: ${ConsoleReporter.indent(turn.response, 22)}`
3710
3839
  ];
3711
- for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3712
- for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3713
- for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 17)}`);
3714
- if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3840
+ for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3841
+ for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3842
+ for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
3843
+ if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3715
3844
  return lines;
3716
3845
  }
3717
3846
  static indent(text, width) {
@@ -3741,10 +3870,12 @@ const JSON_REPORT_VERSION = 1;
3741
3870
  var JsonReporter = class JsonReporter {
3742
3871
  redactor;
3743
3872
  cwd;
3873
+ displayBase;
3744
3874
  tokens = new TokenUsageAggregator();
3745
- constructor(redactor, cwd) {
3875
+ constructor(redactor, cwd, displayBase = cwd) {
3746
3876
  this.redactor = redactor;
3747
3877
  this.cwd = cwd;
3878
+ this.displayBase = displayBase;
3748
3879
  }
3749
3880
  render(run, config, unsuccessfulCount, exitCode) {
3750
3881
  const report = {
@@ -3815,7 +3946,7 @@ var JsonReporter = class JsonReporter {
3815
3946
  return {
3816
3947
  name: result.spec.name,
3817
3948
  runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
3818
- file: result.spec.location.file,
3949
+ file: this.file(result.spec.location.file),
3819
3950
  status: result.status,
3820
3951
  connection: {
3821
3952
  endpoint: websocket.endpoint,
@@ -3825,7 +3956,7 @@ var JsonReporter = class JsonReporter {
3825
3956
  agentUsage: JsonReporter.usage(result.agentUsage),
3826
3957
  evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
3827
3958
  tests: result.tests.map((test) => ({
3828
- ...JsonReporter.identity(test),
3959
+ ...this.identity(test),
3829
3960
  status: test.status,
3830
3961
  ...JsonReporter.counts(test),
3831
3962
  history: test.history.map((entry, index) => ({
@@ -3850,7 +3981,7 @@ var JsonReporter = class JsonReporter {
3850
3981
  agentUsage: JsonReporter.usage(this.tokens.agent([attempt], capability)),
3851
3982
  evaluatorUsage: JsonReporter.usage(this.tokens.evaluator([attempt])),
3852
3983
  turns: execution.turns.map((turn, index) => JsonReporter.turn(index + 1, spec.turns[index].input, turn)),
3853
- tests: attempt.tests.map((test) => JsonReporter.test(test, config))
3984
+ tests: attempt.tests.map((test) => this.test(test, config))
3854
3985
  };
3855
3986
  }
3856
3987
  static turn(number, input, turn) {
@@ -3875,11 +4006,11 @@ var JsonReporter = class JsonReporter {
3875
4006
  tokenUsage: turn.tokenUsage === void 0 ? void 0 : JsonReporter.tokenUsage(turn.tokenUsage)
3876
4007
  };
3877
4008
  }
3878
- static test(test, config) {
4009
+ test(test, config) {
3879
4010
  const thresholds = config.evaluator.thresholds;
3880
4011
  const evaluation = test.evaluation;
3881
4012
  return {
3882
- ...JsonReporter.identity(test),
4013
+ ...this.identity(test),
3883
4014
  status: test.status,
3884
4015
  details: [...test.details],
3885
4016
  evaluation: evaluation === void 0 ? void 0 : {
@@ -3892,17 +4023,17 @@ var JsonReporter = class JsonReporter {
3892
4023
  evaluatorUsage: JsonReporter.usage(test.evaluatorUsage)
3893
4024
  };
3894
4025
  }
3895
- static identity(test) {
4026
+ identity(test) {
3896
4027
  return {
3897
4028
  kind: test.kind,
3898
4029
  turn: test.turn,
3899
4030
  title: test.title,
3900
- location: JsonReporter.location(test.location)
4031
+ location: this.location(test.location)
3901
4032
  };
3902
4033
  }
3903
- static location(location) {
4034
+ location(location) {
3904
4035
  return {
3905
- file: location.file,
4036
+ file: this.file(location.file),
3906
4037
  line: location.line,
3907
4038
  column: location.column,
3908
4039
  path: location.path
@@ -3949,6 +4080,9 @@ var JsonReporter = class JsonReporter {
3949
4080
  path(path) {
3950
4081
  return (0, node_path.relative)(this.cwd, path) || ".";
3951
4082
  }
4083
+ file(shown) {
4084
+ return shown === "" ? shown : this.path((0, node_path.resolve)(this.displayBase, shown));
4085
+ }
3952
4086
  };
3953
4087
  // src/reporting/reportFileWriter.ts
3954
4088
  var ReportFileWriter = class {