@mappies/miniwat-eval 1.4.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -670,7 +670,8 @@ interface IEvaluator {
670
670
  export declare const CONFIG_FILE_NAME = "eval-config.yml";
671
671
  declare class ConfigLocator {
672
672
  private readonly cwd;
673
- constructor(cwd?: string);
673
+ private readonly displayBase;
674
+ constructor(cwd?: string, displayBase?: string);
674
675
  locate(target: string, explicitConfig: string): string;
675
676
  private static searchUpward;
676
677
  private static isProjectRoot;
@@ -680,12 +681,13 @@ declare class ConfigLocator {
680
681
  export declare const DEFAULT_CONFIG: EvalConfig;
681
682
  declare class EvalConfigLoader {
682
683
  private readonly cwd;
684
+ private readonly displayBase;
683
685
  private readonly documentLoader;
684
686
  private readonly schemaValidator;
685
687
  private readonly variablesReader;
686
688
  private readonly webSocketResolver;
687
689
  private readonly environmentResolver;
688
- constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
690
+ constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>, displayBase?: string);
689
691
  load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
690
692
  static buildWebSocket(value: unknown): WebSocketConfig;
691
693
  private static applyReport;
@@ -697,7 +699,8 @@ declare class EvalConfigLoader {
697
699
  }
698
700
  declare class ParametersLoader {
699
701
  private readonly cwd;
700
- constructor(cwd?: string);
702
+ private readonly displayBase;
703
+ constructor(cwd?: string, displayBase?: string);
701
704
  load(path: string): {
702
705
  path: string;
703
706
  parameters: Record<string, string>;
@@ -706,6 +709,7 @@ declare class ParametersLoader {
706
709
  }
707
710
  declare class ScenarioLoader {
708
711
  private readonly cwd;
712
+ private readonly displayBase;
709
713
  private readonly documentLoader;
710
714
  private readonly schemaValidator;
711
715
  private readonly conversationValidator;
@@ -713,7 +717,7 @@ declare class ScenarioLoader {
713
717
  private readonly runWithReader;
714
718
  private readonly webSocketResolver;
715
719
  private readonly environmentResolver;
716
- constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
720
+ constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>, displayBase?: string);
717
721
  load(target: string, config: EvalConfig): ConversationSpec[];
718
722
  private resolveRun;
719
723
  private static byRun;
@@ -939,6 +943,8 @@ declare class ConsoleReporter {
939
943
  private attemptBlock;
940
944
  private testBlock;
941
945
  private entryBody;
946
+ private static actual;
947
+ private capped;
942
948
  private static expected;
943
949
  private paragraph;
944
950
  private bullet;
@@ -963,22 +969,24 @@ export declare const JSON_REPORT_VERSION = 1;
963
969
  declare class JsonReporter {
964
970
  private readonly redactor;
965
971
  private readonly cwd;
972
+ private readonly displayBase;
966
973
  private readonly tokens;
967
- constructor(redactor: SecretRedactor, cwd: string);
974
+ constructor(redactor: SecretRedactor, cwd: string, displayBase?: string);
968
975
  render(run: EvalRunResult, config: EvalConfig, unsuccessfulCount: number, exitCode: number): string;
969
976
  private configuration;
970
977
  private scenario;
971
978
  private attempt;
972
979
  private static turn;
973
- private static test;
974
- private static identity;
975
- private static location;
980
+ private test;
981
+ private identity;
982
+ private location;
976
983
  private static counts;
977
984
  private static tally;
978
985
  private static usage;
979
986
  private static tokenUsage;
980
987
  private static error;
981
988
  private path;
989
+ private file;
982
990
  }
983
991
  declare class ReportFileWriter {
984
992
  private readonly directory;
package/dist/index.d.ts CHANGED
@@ -671,7 +671,8 @@ interface IEvaluator {
671
671
  export declare const CONFIG_FILE_NAME = "eval-config.yml";
672
672
  declare class ConfigLocator {
673
673
  private readonly cwd;
674
- constructor(cwd?: string);
674
+ private readonly displayBase;
675
+ constructor(cwd?: string, displayBase?: string);
675
676
  locate(target: string, explicitConfig: string): string;
676
677
  private static searchUpward;
677
678
  private static isProjectRoot;
@@ -681,12 +682,13 @@ declare class ConfigLocator {
681
682
  export declare const DEFAULT_CONFIG: EvalConfig;
682
683
  declare class EvalConfigLoader {
683
684
  private readonly cwd;
685
+ private readonly displayBase;
684
686
  private readonly documentLoader;
685
687
  private readonly schemaValidator;
686
688
  private readonly variablesReader;
687
689
  private readonly webSocketResolver;
688
690
  private readonly environmentResolver;
689
- constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
691
+ constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>, displayBase?: string);
690
692
  load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
691
693
  static buildWebSocket(value: unknown): WebSocketConfig;
692
694
  private static applyReport;
@@ -698,7 +700,8 @@ declare class EvalConfigLoader {
698
700
  }
699
701
  declare class ParametersLoader {
700
702
  private readonly cwd;
701
- constructor(cwd?: string);
703
+ private readonly displayBase;
704
+ constructor(cwd?: string, displayBase?: string);
702
705
  load(path: string): {
703
706
  path: string;
704
707
  parameters: Record<string, string>;
@@ -707,6 +710,7 @@ declare class ParametersLoader {
707
710
  }
708
711
  declare class ScenarioLoader {
709
712
  private readonly cwd;
713
+ private readonly displayBase;
710
714
  private readonly documentLoader;
711
715
  private readonly schemaValidator;
712
716
  private readonly conversationValidator;
@@ -714,7 +718,7 @@ declare class ScenarioLoader {
714
718
  private readonly runWithReader;
715
719
  private readonly webSocketResolver;
716
720
  private readonly environmentResolver;
717
- constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
721
+ constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>, displayBase?: string);
718
722
  load(target: string, config: EvalConfig): ConversationSpec[];
719
723
  private resolveRun;
720
724
  private static byRun;
@@ -940,6 +944,8 @@ declare class ConsoleReporter {
940
944
  private attemptBlock;
941
945
  private testBlock;
942
946
  private entryBody;
947
+ private static actual;
948
+ private capped;
943
949
  private static expected;
944
950
  private paragraph;
945
951
  private bullet;
@@ -964,22 +970,24 @@ export declare const JSON_REPORT_VERSION = 1;
964
970
  declare class JsonReporter {
965
971
  private readonly redactor;
966
972
  private readonly cwd;
973
+ private readonly displayBase;
967
974
  private readonly tokens;
968
- constructor(redactor: SecretRedactor, cwd: string);
975
+ constructor(redactor: SecretRedactor, cwd: string, displayBase?: string);
969
976
  render(run: EvalRunResult, config: EvalConfig, unsuccessfulCount: number, exitCode: number): string;
970
977
  private configuration;
971
978
  private scenario;
972
979
  private attempt;
973
980
  private static turn;
974
- private static test;
975
- private static identity;
976
- private static location;
981
+ private test;
982
+ private identity;
983
+ private location;
977
984
  private static counts;
978
985
  private static tally;
979
986
  private static usage;
980
987
  private static tokenUsage;
981
988
  private static error;
982
989
  private path;
990
+ private file;
983
991
  }
984
992
  declare class ReportFileWriter {
985
993
  private readonly directory;
package/dist/index.js CHANGED
@@ -680,15 +680,17 @@ const CONFIG_FILE_NAME = "eval-config.yml";
680
680
  const PROJECT_ROOT_MARKERS = [".git", "package.json"];
681
681
  var ConfigLocator = class ConfigLocator {
682
682
  cwd;
683
- constructor(cwd = process.cwd()) {
683
+ displayBase;
684
+ constructor(cwd = process.cwd(), displayBase = cwd) {
684
685
  this.cwd = cwd;
686
+ this.displayBase = displayBase;
685
687
  }
686
688
  locate(target, explicitConfig) {
687
689
  if (explicitConfig !== "") {
688
690
  const path = resolve(this.cwd, explicitConfig);
689
691
  if (!ConfigLocator.isFile(path)) throw new EvalError({
690
692
  category: ErrorCategory.Configuration,
691
- message: `configuration file not found: ${explicitConfig}`
693
+ message: `configuration file not found: ${relative(this.displayBase, path) || explicitConfig}`
692
694
  });
693
695
  return path;
694
696
  }
@@ -697,7 +699,7 @@ var ConfigLocator = class ConfigLocator {
697
699
  const found = ConfigLocator.searchUpward(start) ?? ConfigLocator.candidate(this.cwd);
698
700
  if (found === void 0) throw new EvalError({
699
701
  category: ErrorCategory.Configuration,
700
- message: `no ${CONFIG_FILE_NAME} found above ${relative(this.cwd, targetPath) || "."} or in the working directory; pass --config <path>`
702
+ message: `no ${CONFIG_FILE_NAME} found above ${relative(this.displayBase, targetPath) || "."} or in the working directory; pass --config <path>`
701
703
  });
702
704
  return found;
703
705
  }
@@ -1292,17 +1294,19 @@ var WebSocketResolver = class WebSocketResolver {
1292
1294
  // src/config/evalConfigLoader.ts
1293
1295
  var EvalConfigLoader = class EvalConfigLoader {
1294
1296
  cwd;
1297
+ displayBase;
1295
1298
  documentLoader = new YamlDocumentLoader();
1296
1299
  schemaValidator = new SchemaValidator(eval_config_schema_default);
1297
1300
  variablesReader = new VariablesReader();
1298
1301
  webSocketResolver = new WebSocketResolver();
1299
1302
  environmentResolver;
1300
- constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1303
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
1301
1304
  this.cwd = cwd;
1305
+ this.displayBase = displayBase;
1302
1306
  this.environmentResolver = new EnvironmentResolver(environment, parameters);
1303
1307
  }
1304
1308
  load(configPath, override, report = new ReportOverride()) {
1305
- const document = this.documentLoader.load(configPath, relative(this.cwd, configPath));
1309
+ const document = this.documentLoader.load(configPath, relative(this.displayBase, configPath));
1306
1310
  const issues = [...document.errors];
1307
1311
  if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
1308
1312
  const named = [];
@@ -1458,12 +1462,14 @@ var EvalConfigLoader = class EvalConfigLoader {
1458
1462
  // src/config/parametersLoader.ts
1459
1463
  var ParametersLoader = class ParametersLoader {
1460
1464
  cwd;
1461
- constructor(cwd = process.cwd()) {
1465
+ displayBase;
1466
+ constructor(cwd = process.cwd(), displayBase = cwd) {
1462
1467
  this.cwd = cwd;
1468
+ this.displayBase = displayBase;
1463
1469
  }
1464
1470
  load(path) {
1465
1471
  const absolute = resolve(this.cwd, path);
1466
- const shown = relative(this.cwd, absolute) || path;
1472
+ const shown = relative(this.displayBase, absolute) || path;
1467
1473
  let text;
1468
1474
  let value;
1469
1475
  try {
@@ -1922,6 +1928,7 @@ var RunWithReader = class RunWithReader {
1922
1928
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
1923
1929
  var ScenarioLoader = class ScenarioLoader {
1924
1930
  cwd;
1931
+ displayBase;
1925
1932
  documentLoader = new YamlDocumentLoader();
1926
1933
  schemaValidator = new SchemaValidator(conversation_schema_default);
1927
1934
  conversationValidator = new ConversationValidator();
@@ -1929,18 +1936,19 @@ var ScenarioLoader = class ScenarioLoader {
1929
1936
  runWithReader = new RunWithReader();
1930
1937
  webSocketResolver = new WebSocketResolver();
1931
1938
  environmentResolver;
1932
- constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1939
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
1933
1940
  this.cwd = cwd;
1941
+ this.displayBase = displayBase;
1934
1942
  this.environmentResolver = new EnvironmentResolver(environment, parameters);
1935
1943
  }
1936
1944
  load(target, config) {
1937
1945
  const files = this.discover(resolve(this.cwd, target), config);
1938
- const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.cwd, config.configPath)) : void 0;
1946
+ const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.displayBase, config.configPath)) : void 0;
1939
1947
  const issues = [];
1940
1948
  const connectionIssues = [];
1941
1949
  const specs = [];
1942
1950
  for (const file of files) {
1943
- const document = this.documentLoader.load(file, relative(this.cwd, file));
1951
+ const document = this.documentLoader.load(file, relative(this.displayBase, file));
1944
1952
  const fileIssues = [...document.errors];
1945
1953
  const fileSecrets = [];
1946
1954
  const used = /* @__PURE__ */ new Set();
@@ -2070,13 +2078,13 @@ var ScenarioLoader = class ScenarioLoader {
2070
2078
  discover(target, config) {
2071
2079
  if (!existsSync(target)) throw new EvalError({
2072
2080
  category: ErrorCategory.Specification,
2073
- message: `target not found: ${relative(this.cwd, target) || "."}`
2081
+ message: `target not found: ${relative(this.displayBase, target) || "."}`
2074
2082
  });
2075
2083
  const excluded = (path) => path === config.configPath || config.report.directory !== "" && path === config.report.directory;
2076
2084
  const files = statSync(target).isDirectory() ? this.walk(target, excluded) : [target].filter((file) => !excluded(file));
2077
2085
  if (files.length === 0) throw new EvalError({
2078
2086
  category: ErrorCategory.Specification,
2079
- message: `no scenario files found in ${relative(this.cwd, target) || "."}`
2087
+ message: `no scenario files found in ${relative(this.displayBase, target) || "."}`
2080
2088
  });
2081
2089
  return files;
2082
2090
  }
@@ -3698,17 +3706,18 @@ var ConsoleReporter = class ConsoleReporter {
3698
3706
  const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
3699
3707
  const failing = test.history.map((entry, index) => ({
3700
3708
  entry,
3709
+ run: result.attempts[index],
3701
3710
  attempt: result.attempts[index]?.attempt ?? index + 1
3702
3711
  })).filter(({ entry }) => entry.status !== ResultStatus.Pass);
3703
3712
  const several = test.history.length > 1;
3704
- for (const { entry, attempt } of failing) {
3713
+ for (const { entry, run, attempt } of failing) {
3705
3714
  if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
3706
- lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
3715
+ lines.push(...this.entryBody(spec, entry, run, several ? 5 : 3));
3707
3716
  }
3708
3717
  lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
3709
3718
  return lines;
3710
3719
  }
3711
- entryBody(spec, entry, indent) {
3720
+ entryBody(spec, entry, attempt, indent) {
3712
3721
  const pad = " ".repeat(indent);
3713
3722
  const evaluation = entry.evaluation;
3714
3723
  if (evaluation === void 0) {
@@ -3722,11 +3731,23 @@ var ConsoleReporter = class ConsoleReporter {
3722
3731
  const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
3723
3732
  const expected = ConsoleReporter.expected(spec, entry);
3724
3733
  const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
3734
+ const actual = ConsoleReporter.actual(entry, attempt);
3725
3735
  if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
3736
+ if (actual !== void 0) lines.push("", `${pad}Actual`, ...this.capped(actual, indent + 2));
3726
3737
  if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
3727
- if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
3728
3738
  return lines;
3729
3739
  }
3740
+ static actual(entry, attempt) {
3741
+ const turns = attempt?.execution.turns ?? [];
3742
+ if (entry.kind === TestKind.Conversation) return turns.length === 0 ? void 0 : turns.map((turn, index) => `Turn ${index + 1}: ${turn.response}`).join("\n");
3743
+ return turns[entry.turn - 1]?.response;
3744
+ }
3745
+ capped(text, indent) {
3746
+ const lines = text.trim() === "" ? [`${" ".repeat(indent)}(empty)`] : this.paragraph(text, indent);
3747
+ if (lines.length <= 12) return lines;
3748
+ const more = lines.length - 12;
3749
+ return [...lines.slice(0, 12), `${" ".repeat(indent)}… ${more} more ${more === 1 ? "line" : "lines"} in the JSON report`];
3750
+ }
3730
3751
  static expected(spec, entry) {
3731
3752
  if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
3732
3753
  return spec.turns[entry.turn - 1]?.expect.response?.criteria;
@@ -3826,10 +3847,12 @@ const JSON_REPORT_VERSION = 1;
3826
3847
  var JsonReporter = class JsonReporter {
3827
3848
  redactor;
3828
3849
  cwd;
3850
+ displayBase;
3829
3851
  tokens = new TokenUsageAggregator();
3830
- constructor(redactor, cwd) {
3852
+ constructor(redactor, cwd, displayBase = cwd) {
3831
3853
  this.redactor = redactor;
3832
3854
  this.cwd = cwd;
3855
+ this.displayBase = displayBase;
3833
3856
  }
3834
3857
  render(run, config, unsuccessfulCount, exitCode) {
3835
3858
  const report = {
@@ -3900,7 +3923,7 @@ var JsonReporter = class JsonReporter {
3900
3923
  return {
3901
3924
  name: result.spec.name,
3902
3925
  runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
3903
- file: result.spec.location.file,
3926
+ file: this.file(result.spec.location.file),
3904
3927
  status: result.status,
3905
3928
  connection: {
3906
3929
  endpoint: websocket.endpoint,
@@ -3910,7 +3933,7 @@ var JsonReporter = class JsonReporter {
3910
3933
  agentUsage: JsonReporter.usage(result.agentUsage),
3911
3934
  evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
3912
3935
  tests: result.tests.map((test) => ({
3913
- ...JsonReporter.identity(test),
3936
+ ...this.identity(test),
3914
3937
  status: test.status,
3915
3938
  ...JsonReporter.counts(test),
3916
3939
  history: test.history.map((entry, index) => ({
@@ -3935,7 +3958,7 @@ var JsonReporter = class JsonReporter {
3935
3958
  agentUsage: JsonReporter.usage(this.tokens.agent([attempt], capability)),
3936
3959
  evaluatorUsage: JsonReporter.usage(this.tokens.evaluator([attempt])),
3937
3960
  turns: execution.turns.map((turn, index) => JsonReporter.turn(index + 1, spec.turns[index].input, turn)),
3938
- tests: attempt.tests.map((test) => JsonReporter.test(test, config))
3961
+ tests: attempt.tests.map((test) => this.test(test, config))
3939
3962
  };
3940
3963
  }
3941
3964
  static turn(number, input, turn) {
@@ -3960,11 +3983,11 @@ var JsonReporter = class JsonReporter {
3960
3983
  tokenUsage: turn.tokenUsage === void 0 ? void 0 : JsonReporter.tokenUsage(turn.tokenUsage)
3961
3984
  };
3962
3985
  }
3963
- static test(test, config) {
3986
+ test(test, config) {
3964
3987
  const thresholds = config.evaluator.thresholds;
3965
3988
  const evaluation = test.evaluation;
3966
3989
  return {
3967
- ...JsonReporter.identity(test),
3990
+ ...this.identity(test),
3968
3991
  status: test.status,
3969
3992
  details: [...test.details],
3970
3993
  evaluation: evaluation === void 0 ? void 0 : {
@@ -3977,17 +4000,17 @@ var JsonReporter = class JsonReporter {
3977
4000
  evaluatorUsage: JsonReporter.usage(test.evaluatorUsage)
3978
4001
  };
3979
4002
  }
3980
- static identity(test) {
4003
+ identity(test) {
3981
4004
  return {
3982
4005
  kind: test.kind,
3983
4006
  turn: test.turn,
3984
4007
  title: test.title,
3985
- location: JsonReporter.location(test.location)
4008
+ location: this.location(test.location)
3986
4009
  };
3987
4010
  }
3988
- static location(location) {
4011
+ location(location) {
3989
4012
  return {
3990
- file: location.file,
4013
+ file: this.file(location.file),
3991
4014
  line: location.line,
3992
4015
  column: location.column,
3993
4016
  path: location.path
@@ -4034,6 +4057,9 @@ var JsonReporter = class JsonReporter {
4034
4057
  path(path) {
4035
4058
  return relative(this.cwd, path) || ".";
4036
4059
  }
4060
+ file(shown) {
4061
+ return shown === "" ? shown : this.path(resolve(this.displayBase, shown));
4062
+ }
4037
4063
  };
4038
4064
  // src/reporting/reportFileWriter.ts
4039
4065
  var ReportFileWriter = class {