@mappies/miniwat-eval 1.3.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -680,15 +680,17 @@ const CONFIG_FILE_NAME = "eval-config.yml";
680
680
  const PROJECT_ROOT_MARKERS = [".git", "package.json"];
681
681
  var ConfigLocator = class ConfigLocator {
682
682
  cwd;
683
- constructor(cwd = process.cwd()) {
683
+ displayBase;
684
+ constructor(cwd = process.cwd(), displayBase = cwd) {
684
685
  this.cwd = cwd;
686
+ this.displayBase = displayBase;
685
687
  }
686
688
  locate(target, explicitConfig) {
687
689
  if (explicitConfig !== "") {
688
690
  const path = resolve(this.cwd, explicitConfig);
689
691
  if (!ConfigLocator.isFile(path)) throw new EvalError({
690
692
  category: ErrorCategory.Configuration,
691
- message: `configuration file not found: ${explicitConfig}`
693
+ message: `configuration file not found: ${relative(this.displayBase, path) || explicitConfig}`
692
694
  });
693
695
  return path;
694
696
  }
@@ -697,7 +699,7 @@ var ConfigLocator = class ConfigLocator {
697
699
  const found = ConfigLocator.searchUpward(start) ?? ConfigLocator.candidate(this.cwd);
698
700
  if (found === void 0) throw new EvalError({
699
701
  category: ErrorCategory.Configuration,
700
- message: `no ${CONFIG_FILE_NAME} found above ${relative(this.cwd, targetPath) || "."} or in the working directory; pass --config <path>`
702
+ message: `no ${CONFIG_FILE_NAME} found above ${relative(this.displayBase, targetPath) || "."} or in the working directory; pass --config <path>`
701
703
  });
702
704
  return found;
703
705
  }
@@ -1292,17 +1294,19 @@ var WebSocketResolver = class WebSocketResolver {
1292
1294
  // src/config/evalConfigLoader.ts
1293
1295
  var EvalConfigLoader = class EvalConfigLoader {
1294
1296
  cwd;
1297
+ displayBase;
1295
1298
  documentLoader = new YamlDocumentLoader();
1296
1299
  schemaValidator = new SchemaValidator(eval_config_schema_default);
1297
1300
  variablesReader = new VariablesReader();
1298
1301
  webSocketResolver = new WebSocketResolver();
1299
1302
  environmentResolver;
1300
- constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1303
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
1301
1304
  this.cwd = cwd;
1305
+ this.displayBase = displayBase;
1302
1306
  this.environmentResolver = new EnvironmentResolver(environment, parameters);
1303
1307
  }
1304
1308
  load(configPath, override, report = new ReportOverride()) {
1305
- const document = this.documentLoader.load(configPath, relative(this.cwd, configPath));
1309
+ const document = this.documentLoader.load(configPath, relative(this.displayBase, configPath));
1306
1310
  const issues = [...document.errors];
1307
1311
  if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
1308
1312
  const named = [];
@@ -1458,12 +1462,14 @@ var EvalConfigLoader = class EvalConfigLoader {
1458
1462
  // src/config/parametersLoader.ts
1459
1463
  var ParametersLoader = class ParametersLoader {
1460
1464
  cwd;
1461
- constructor(cwd = process.cwd()) {
1465
+ displayBase;
1466
+ constructor(cwd = process.cwd(), displayBase = cwd) {
1462
1467
  this.cwd = cwd;
1468
+ this.displayBase = displayBase;
1463
1469
  }
1464
1470
  load(path) {
1465
1471
  const absolute = resolve(this.cwd, path);
1466
- const shown = relative(this.cwd, absolute) || path;
1472
+ const shown = relative(this.displayBase, absolute) || path;
1467
1473
  let text;
1468
1474
  let value;
1469
1475
  try {
@@ -1922,6 +1928,7 @@ var RunWithReader = class RunWithReader {
1922
1928
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
1923
1929
  var ScenarioLoader = class ScenarioLoader {
1924
1930
  cwd;
1931
+ displayBase;
1925
1932
  documentLoader = new YamlDocumentLoader();
1926
1933
  schemaValidator = new SchemaValidator(conversation_schema_default);
1927
1934
  conversationValidator = new ConversationValidator();
@@ -1929,18 +1936,19 @@ var ScenarioLoader = class ScenarioLoader {
1929
1936
  runWithReader = new RunWithReader();
1930
1937
  webSocketResolver = new WebSocketResolver();
1931
1938
  environmentResolver;
1932
- constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1939
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
1933
1940
  this.cwd = cwd;
1941
+ this.displayBase = displayBase;
1934
1942
  this.environmentResolver = new EnvironmentResolver(environment, parameters);
1935
1943
  }
1936
1944
  load(target, config) {
1937
1945
  const files = this.discover(resolve(this.cwd, target), config);
1938
- const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.cwd, config.configPath)) : void 0;
1946
+ const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.displayBase, config.configPath)) : void 0;
1939
1947
  const issues = [];
1940
1948
  const connectionIssues = [];
1941
1949
  const specs = [];
1942
1950
  for (const file of files) {
1943
- const document = this.documentLoader.load(file, relative(this.cwd, file));
1951
+ const document = this.documentLoader.load(file, relative(this.displayBase, file));
1944
1952
  const fileIssues = [...document.errors];
1945
1953
  const fileSecrets = [];
1946
1954
  const used = /* @__PURE__ */ new Set();
@@ -2070,13 +2078,13 @@ var ScenarioLoader = class ScenarioLoader {
2070
2078
  discover(target, config) {
2071
2079
  if (!existsSync(target)) throw new EvalError({
2072
2080
  category: ErrorCategory.Specification,
2073
- message: `target not found: ${relative(this.cwd, target) || "."}`
2081
+ message: `target not found: ${relative(this.displayBase, target) || "."}`
2074
2082
  });
2075
2083
  const excluded = (path) => path === config.configPath || config.report.directory !== "" && path === config.report.directory;
2076
2084
  const files = statSync(target).isDirectory() ? this.walk(target, excluded) : [target].filter((file) => !excluded(file));
2077
2085
  if (files.length === 0) throw new EvalError({
2078
2086
  category: ErrorCategory.Specification,
2079
- message: `no scenario files found in ${relative(this.cwd, target) || "."}`
2087
+ message: `no scenario files found in ${relative(this.displayBase, target) || "."}`
2080
2088
  });
2081
2089
  return files;
2082
2090
  }
@@ -3534,64 +3542,94 @@ var ExitCodeCalculator = class {
3534
3542
  }
3535
3543
  };
3536
3544
  // src/reporting/consoleReporter.ts
3537
- const STATUS_WIDTH = 11;
3538
- const COLOURS = {
3539
- [ResultStatus.Pass]: "green",
3540
- [ResultStatus.Flaky]: "yellow",
3541
- [ResultStatus.Fail]: "red",
3542
- [ResultStatus.Error]: "red",
3543
- [ResultStatus.Skipped]: "dim",
3544
- [ResultStatus.Interrupted]: "dim"
3545
+ const MARKS = {
3546
+ [ResultStatus.Pass]: {
3547
+ symbol: "✔",
3548
+ colour: "green"
3549
+ },
3550
+ [ResultStatus.Flaky]: {
3551
+ symbol: "⚠",
3552
+ colour: "yellow"
3553
+ },
3554
+ [ResultStatus.Fail]: {
3555
+ symbol: "✖",
3556
+ colour: "red"
3557
+ },
3558
+ [ResultStatus.Error]: {
3559
+ symbol: "●",
3560
+ colour: "red"
3561
+ },
3562
+ [ResultStatus.Skipped]: {
3563
+ symbol: "○",
3564
+ colour: "dim"
3565
+ },
3566
+ [ResultStatus.Interrupted]: {
3567
+ symbol: "◌",
3568
+ colour: "dim"
3569
+ }
3545
3570
  };
3571
+ const MIN_COLUMNS = 40;
3572
+ const RULE_WIDTH = 72;
3546
3573
  var ConsoleReporter = class ConsoleReporter {
3547
3574
  verbose;
3548
3575
  colour;
3549
- constructor(verbose, colour) {
3576
+ nameWidth;
3577
+ columns;
3578
+ thresholds;
3579
+ constructor(verbose, colour, options = {}) {
3550
3580
  this.verbose = verbose;
3551
3581
  this.colour = colour;
3582
+ this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
3583
+ this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
3584
+ this.thresholds = options.thresholds;
3585
+ }
3586
+ static nameWidth(specs) {
3587
+ return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
3552
3588
  }
3553
3589
  header(run) {
3554
3590
  return [
3555
- "Miniwat Eval",
3591
+ " Miniwat Eval",
3556
3592
  "",
3557
- ...run.quick ? ["Execution Mode: Quick"] : [],
3558
- `Repeat: ${run.repeat}`,
3559
- `Minimum Pass: ${run.minPass}`,
3593
+ ...run.quick ? [" Execution mode Quick"] : [],
3594
+ ` Repeat ${run.repeat}`,
3595
+ ` Minimum pass ${run.minPass}`,
3560
3596
  "",
3561
- `Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
3597
+ ` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
3562
3598
  "",
3563
3599
  ""
3564
3600
  ].join("\n");
3565
3601
  }
3566
3602
  scenario(result, repeat, tokenUsageCapability) {
3567
3603
  const spec = result.spec;
3568
- const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
3569
- const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
3604
+ const title = ConsoleReporter.title(spec);
3605
+ const started = result.attempts.length > 0;
3606
+ const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
3607
+ const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
3608
+ const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
3570
3609
  if (!this.verbose) return line;
3571
- if (result.attempts.length === 0) return `${line}\n`;
3610
+ if (!started) return `${line}\n`;
3572
3611
  const websocket = spec.websocket;
3573
3612
  const subprotocols = websocket?.connection.subprotocols.length ?? 0;
3574
- const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
3575
- for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3576
- lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
3577
- lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
3578
- lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
3579
- lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3613
+ const lines = [];
3614
+ if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
3615
+ for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3616
+ lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
3617
+ lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3580
3618
  return line + lines.join("\n");
3581
3619
  }
3582
3620
  static title(spec) {
3583
3621
  return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
3584
3622
  }
3585
3623
  summary(run, exitCode, reportPaths = []) {
3586
- const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
3624
+ const started = run.scenarios.filter((result) => result.attempts.length > 0);
3625
+ const failures = started.flatMap((result) => this.blocks(result, false));
3626
+ const flaky = started.flatMap((result) => this.blocks(result, true));
3587
3627
  const lines = this.verbose ? [] : [""];
3588
- if (problems.length > 0) {
3589
- lines.push("Problems", "");
3590
- for (const result of problems) lines.push(...this.problems(result), "");
3591
- }
3628
+ if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
3629
+ if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
3592
3630
  const tests = run.scenarios.flatMap((result) => result.tests);
3593
- lines.push(...run.status === RunStatus.Interrupted ? [`Run: ${run.status}`] : [], `Scenarios: ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, `Tests: ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability), ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true), `Duration: ${ConsoleReporter.duration(run.durationMs)}`, "", "");
3594
- return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
3631
+ lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
3632
+ return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
3595
3633
  }
3596
3634
  static footer(run, exitCode, reportPaths) {
3597
3635
  return [
@@ -3605,57 +3643,148 @@ var ConsoleReporter = class ConsoleReporter {
3605
3643
  return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
3606
3644
  }
3607
3645
  static duration(ms) {
3646
+ if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
3608
3647
  const seconds = Math.floor(ms / 1e3);
3609
3648
  const pad = (value) => String(value).padStart(2, "0");
3610
3649
  const hours = Math.floor(seconds / 3600);
3611
- const rest = `${pad(Math.floor(seconds / 60) % 60)}:${pad(seconds % 60)}`;
3612
- return hours > 0 ? `${hours}:${rest}` : rest;
3650
+ const minutes = Math.floor(seconds / 60) % 60;
3651
+ return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
3613
3652
  }
3614
- problems(result) {
3615
- const spec = result.spec;
3616
- const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
3617
- for (const attempt of result.attempts) {
3618
- const execution = attempt.execution;
3619
- if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3620
- for (const error of execution.attemptErrors) lines.push(` Attempt ${attempt.attempt} error: ${error.category} error: ${error.message}`);
3621
- if (execution.closeError !== void 0) lines.push(` Attempt ${attempt.attempt} close error: ${execution.closeError.message}`);
3622
- }
3623
- for (const test of result.tests.filter((entry) => entry.status !== ResultStatus.Pass)) {
3624
- lines.push(` ${this.status(test.status, 6)} ${ConsoleReporter.testTitle(test)} ${test.location.toString()} ${ConsoleReporter.counts(test)}`);
3625
- test.history.forEach((entry, index) => {
3626
- if (entry.status === ResultStatus.Pass) return;
3627
- const prefix = ` Attempt ${result.attempts[index].attempt} ${this.status(entry.status)}`;
3628
- const width = ` Attempt ${result.attempts[index].attempt} ${entry.status}: `.length;
3629
- if (entry.details.length === 0) {
3630
- lines.push(prefix);
3631
- return;
3632
- }
3633
- lines.push(`${prefix}: ${ConsoleReporter.indent(entry.details[0], width)}`, ...entry.details.slice(1).map((detail) => `${" ".repeat(width)}${ConsoleReporter.indent(detail, width)}`));
3634
- });
3653
+ static wrap(text, width) {
3654
+ const lines = [];
3655
+ for (const paragraph of text.split("\n")) {
3656
+ let line = "";
3657
+ for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
3658
+ lines.push(line);
3659
+ line = word;
3660
+ } else line = line === "" ? word : `${line} ${word}`;
3661
+ lines.push(line);
3635
3662
  }
3636
3663
  return lines;
3637
3664
  }
3665
+ section(name, blocks) {
3666
+ return [
3667
+ ...this.rule(name),
3668
+ "",
3669
+ ...blocks.flatMap((block) => [...block, ""])
3670
+ ];
3671
+ }
3672
+ rule(name) {
3673
+ const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
3674
+ return [
3675
+ rule,
3676
+ ` ${name}`,
3677
+ rule
3678
+ ];
3679
+ }
3680
+ blocks(result, flaky) {
3681
+ const spec = result.spec;
3682
+ const title = ConsoleReporter.title(spec);
3683
+ const blocks = [];
3684
+ if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
3685
+ const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
3686
+ for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
3687
+ return blocks;
3688
+ }
3689
+ attemptBlock(spec, title, attempt) {
3690
+ const execution = attempt.execution;
3691
+ const problems = [];
3692
+ if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
3693
+ for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
3694
+ if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
3695
+ if (problems.length === 0) return [];
3696
+ const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
3697
+ const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
3698
+ return [[
3699
+ ` scenario at ${spec.location.file}`,
3700
+ ` ${this.mark(status)} ${title} › ${what}`,
3701
+ ...problems.flatMap((problem) => this.paragraph(problem, 3))
3702
+ ]];
3703
+ }
3704
+ testBlock(spec, title, test, result) {
3705
+ const location = test.location;
3706
+ const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
3707
+ const failing = test.history.map((entry, index) => ({
3708
+ entry,
3709
+ run: result.attempts[index],
3710
+ attempt: result.attempts[index]?.attempt ?? index + 1
3711
+ })).filter(({ entry }) => entry.status !== ResultStatus.Pass);
3712
+ const several = test.history.length > 1;
3713
+ for (const { entry, run, attempt } of failing) {
3714
+ if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
3715
+ lines.push(...this.entryBody(spec, entry, run, several ? 5 : 3));
3716
+ }
3717
+ lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
3718
+ return lines;
3719
+ }
3720
+ entryBody(spec, entry, attempt, indent) {
3721
+ const pad = " ".repeat(indent);
3722
+ const evaluation = entry.evaluation;
3723
+ if (evaluation === void 0) {
3724
+ if (entry.details.length === 0) return [];
3725
+ return [
3726
+ "",
3727
+ `${pad}Details`,
3728
+ ...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
3729
+ ];
3730
+ }
3731
+ const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
3732
+ const expected = ConsoleReporter.expected(spec, entry);
3733
+ const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
3734
+ const actual = ConsoleReporter.actual(entry, attempt);
3735
+ if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
3736
+ if (actual !== void 0) lines.push("", `${pad}Actual`, ...this.capped(actual, indent + 2));
3737
+ if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
3738
+ return lines;
3739
+ }
3740
+ static actual(entry, attempt) {
3741
+ const turns = attempt?.execution.turns ?? [];
3742
+ if (entry.kind === TestKind.Conversation) return turns.length === 0 ? void 0 : turns.map((turn, index) => `Turn ${index + 1}: ${turn.response}`).join("\n");
3743
+ return turns[entry.turn - 1]?.response;
3744
+ }
3745
+ capped(text, indent) {
3746
+ const lines = text.trim() === "" ? [`${" ".repeat(indent)}(empty)`] : this.paragraph(text, indent);
3747
+ if (lines.length <= 12) return lines;
3748
+ const more = lines.length - 12;
3749
+ return [...lines.slice(0, 12), `${" ".repeat(indent)}… ${more} more ${more === 1 ? "line" : "lines"} in the JSON report`];
3750
+ }
3751
+ static expected(spec, entry) {
3752
+ if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
3753
+ return spec.turns[entry.turn - 1]?.expect.response?.criteria;
3754
+ }
3755
+ paragraph(text, indent) {
3756
+ const pad = " ".repeat(indent);
3757
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
3758
+ }
3759
+ bullet(text, indent) {
3760
+ const pad = " ".repeat(indent);
3761
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
3762
+ }
3638
3763
  static testTitle(test) {
3639
3764
  return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
3640
3765
  }
3641
3766
  static tally(statuses) {
3642
3767
  const count = (status) => statuses.filter((entry) => entry === status).length;
3643
- const interrupted = count(ResultStatus.Interrupted);
3644
- return `${count(ResultStatus.Pass)} passed, ${count(ResultStatus.Flaky)} flaky, ${count(ResultStatus.Fail)} failed, ${count(ResultStatus.Error)} errors${interrupted > 0 ? `, ${interrupted} interrupted` : ""}`;
3645
- }
3646
- status(status, width = 0) {
3647
- const padding = " ".repeat(Math.max(0, width - status.length));
3648
- return (this.colour ? styleText(COLOURS[status], status, { validateStream: false }) : status) + padding;
3649
- }
3650
- static counts(result) {
3651
- return `(${result.passed} passed, ${result.failed} failed, ${result.errors} errors; required ${result.required})`;
3768
+ const parts = [
3769
+ [ResultStatus.Pass, "passed"],
3770
+ [ResultStatus.Flaky, "flaky"],
3771
+ [ResultStatus.Fail, "failed"],
3772
+ [ResultStatus.Error, "errors"],
3773
+ [ResultStatus.Interrupted, "interrupted"]
3774
+ ].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
3775
+ return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
3776
+ }
3777
+ mark(status) {
3778
+ const { symbol, colour } = MARKS[status];
3779
+ return this.colour ? styleText(colour, symbol, { validateStream: false }) : symbol;
3652
3780
  }
3653
3781
  static usage(label, usage, capability) {
3654
- if (!capability) return `${label}: n/a (not supported by host)`;
3655
- if (usage.expected === 0) return `${label}: none`;
3656
- if (usage.total === void 0) return `${label}: n/a (coverage 0/${usage.expected})`;
3782
+ const name = label.padEnd(16);
3783
+ if (!capability) return `${name} n/a (not supported by host)`;
3784
+ if (usage.expected === 0) return `${name} none`;
3785
+ if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
3657
3786
  const number = (value) => value.toLocaleString("en-US");
3658
- const total = `${label}: total ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3787
+ const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3659
3788
  return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
3660
3789
  }
3661
3790
  formatAttempt(spec, attempt) {
@@ -3667,28 +3796,28 @@ var ConsoleReporter = class ConsoleReporter {
3667
3796
  lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
3668
3797
  lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
3669
3798
  });
3670
- if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3799
+ if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3671
3800
  const notRun = attempt.tests.filter((test) => test.turn > ran);
3672
- if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3801
+ if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3673
3802
  lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
3674
- for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3675
- if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3676
- lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3803
+ for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3804
+ if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3805
+ lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3677
3806
  return lines;
3678
3807
  }
3679
3808
  formatTest(test) {
3680
- return [` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${detail}`)];
3809
+ return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
3681
3810
  }
3682
3811
  static formatTurn(number, input, turn) {
3683
3812
  const lines = [
3684
- ` Turn ${number} (${turn.durationMs} ms)`,
3685
- ` Input: ${ConsoleReporter.indent(input, 17)}`,
3686
- ` Response: ${ConsoleReporter.indent(turn.response, 17)}`
3813
+ ` Turn ${number} (${turn.durationMs} ms)`,
3814
+ ` Input: ${ConsoleReporter.indent(input, 22)}`,
3815
+ ` Response: ${ConsoleReporter.indent(turn.response, 22)}`
3687
3816
  ];
3688
- for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3689
- for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3690
- for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 17)}`);
3691
- if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3817
+ for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3818
+ for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3819
+ for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
3820
+ if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3692
3821
  return lines;
3693
3822
  }
3694
3823
  static indent(text, width) {
@@ -3718,10 +3847,12 @@ const JSON_REPORT_VERSION = 1;
3718
3847
  var JsonReporter = class JsonReporter {
3719
3848
  redactor;
3720
3849
  cwd;
3850
+ displayBase;
3721
3851
  tokens = new TokenUsageAggregator();
3722
- constructor(redactor, cwd) {
3852
+ constructor(redactor, cwd, displayBase = cwd) {
3723
3853
  this.redactor = redactor;
3724
3854
  this.cwd = cwd;
3855
+ this.displayBase = displayBase;
3725
3856
  }
3726
3857
  render(run, config, unsuccessfulCount, exitCode) {
3727
3858
  const report = {
@@ -3792,7 +3923,7 @@ var JsonReporter = class JsonReporter {
3792
3923
  return {
3793
3924
  name: result.spec.name,
3794
3925
  runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
3795
- file: result.spec.location.file,
3926
+ file: this.file(result.spec.location.file),
3796
3927
  status: result.status,
3797
3928
  connection: {
3798
3929
  endpoint: websocket.endpoint,
@@ -3802,7 +3933,7 @@ var JsonReporter = class JsonReporter {
3802
3933
  agentUsage: JsonReporter.usage(result.agentUsage),
3803
3934
  evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
3804
3935
  tests: result.tests.map((test) => ({
3805
- ...JsonReporter.identity(test),
3936
+ ...this.identity(test),
3806
3937
  status: test.status,
3807
3938
  ...JsonReporter.counts(test),
3808
3939
  history: test.history.map((entry, index) => ({
@@ -3827,7 +3958,7 @@ var JsonReporter = class JsonReporter {
3827
3958
  agentUsage: JsonReporter.usage(this.tokens.agent([attempt], capability)),
3828
3959
  evaluatorUsage: JsonReporter.usage(this.tokens.evaluator([attempt])),
3829
3960
  turns: execution.turns.map((turn, index) => JsonReporter.turn(index + 1, spec.turns[index].input, turn)),
3830
- tests: attempt.tests.map((test) => JsonReporter.test(test, config))
3961
+ tests: attempt.tests.map((test) => this.test(test, config))
3831
3962
  };
3832
3963
  }
3833
3964
  static turn(number, input, turn) {
@@ -3852,11 +3983,11 @@ var JsonReporter = class JsonReporter {
3852
3983
  tokenUsage: turn.tokenUsage === void 0 ? void 0 : JsonReporter.tokenUsage(turn.tokenUsage)
3853
3984
  };
3854
3985
  }
3855
- static test(test, config) {
3986
+ test(test, config) {
3856
3987
  const thresholds = config.evaluator.thresholds;
3857
3988
  const evaluation = test.evaluation;
3858
3989
  return {
3859
- ...JsonReporter.identity(test),
3990
+ ...this.identity(test),
3860
3991
  status: test.status,
3861
3992
  details: [...test.details],
3862
3993
  evaluation: evaluation === void 0 ? void 0 : {
@@ -3869,17 +4000,17 @@ var JsonReporter = class JsonReporter {
3869
4000
  evaluatorUsage: JsonReporter.usage(test.evaluatorUsage)
3870
4001
  };
3871
4002
  }
3872
- static identity(test) {
4003
+ identity(test) {
3873
4004
  return {
3874
4005
  kind: test.kind,
3875
4006
  turn: test.turn,
3876
4007
  title: test.title,
3877
- location: JsonReporter.location(test.location)
4008
+ location: this.location(test.location)
3878
4009
  };
3879
4010
  }
3880
- static location(location) {
4011
+ location(location) {
3881
4012
  return {
3882
- file: location.file,
4013
+ file: this.file(location.file),
3883
4014
  line: location.line,
3884
4015
  column: location.column,
3885
4016
  path: location.path
@@ -3926,6 +4057,9 @@ var JsonReporter = class JsonReporter {
3926
4057
  path(path) {
3927
4058
  return relative(this.cwd, path) || ".";
3928
4059
  }
4060
+ file(shown) {
4061
+ return shown === "" ? shown : this.path(resolve(this.displayBase, shown));
4062
+ }
3929
4063
  };
3930
4064
  // src/reporting/reportFileWriter.ts
3931
4065
  var ReportFileWriter = class {