@mappies/miniwat-eval 1.4.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/main.js CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env node
2
2
  import { Command, CommanderError } from "commander";
3
- import { dirname, extname, join, relative, resolve } from "node:path";
3
+ import { dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
4
4
  import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
5
5
  import { LineCounter, isMap, isPair, isScalar, isSeq, parseDocument } from "yaml";
6
6
  import { Ajv } from "ajv";
@@ -122,15 +122,17 @@ const CONFIG_FILE_NAME = "eval-config.yml";
122
122
  const PROJECT_ROOT_MARKERS = [".git", "package.json"];
123
123
  var ConfigLocator = class ConfigLocator {
124
124
  cwd;
125
- constructor(cwd = process.cwd()) {
125
+ displayBase;
126
+ constructor(cwd = process.cwd(), displayBase = cwd) {
126
127
  this.cwd = cwd;
128
+ this.displayBase = displayBase;
127
129
  }
128
130
  locate(target, explicitConfig) {
129
131
  if (explicitConfig !== "") {
130
132
  const path = resolve(this.cwd, explicitConfig);
131
133
  if (!ConfigLocator.isFile(path)) throw new EvalError({
132
134
  category: ErrorCategory.Configuration,
133
- message: `configuration file not found: ${explicitConfig}`
135
+ message: `configuration file not found: ${relative(this.displayBase, path) || explicitConfig}`
134
136
  });
135
137
  return path;
136
138
  }
@@ -139,7 +141,7 @@ var ConfigLocator = class ConfigLocator {
139
141
  const found = ConfigLocator.searchUpward(start) ?? ConfigLocator.candidate(this.cwd);
140
142
  if (found === void 0) throw new EvalError({
141
143
  category: ErrorCategory.Configuration,
142
- message: `no ${CONFIG_FILE_NAME} found above ${relative(this.cwd, targetPath) || "."} or in the working directory; pass --config <path>`
144
+ message: `no ${CONFIG_FILE_NAME} found above ${relative(this.displayBase, targetPath) || "."} or in the working directory; pass --config <path>`
143
145
  });
144
146
  return found;
145
147
  }
@@ -171,12 +173,14 @@ function isRecord(value) {
171
173
  // src/config/parametersLoader.ts
172
174
  var ParametersLoader = class ParametersLoader {
173
175
  cwd;
174
- constructor(cwd = process.cwd()) {
176
+ displayBase;
177
+ constructor(cwd = process.cwd(), displayBase = cwd) {
175
178
  this.cwd = cwd;
179
+ this.displayBase = displayBase;
176
180
  }
177
181
  load(path) {
178
182
  const absolute = resolve(this.cwd, path);
179
- const shown = relative(this.cwd, absolute) || path;
183
+ const shown = relative(this.displayBase, absolute) || path;
180
184
  let text;
181
185
  let value;
182
186
  try {
@@ -928,17 +932,19 @@ const DEFAULT_CONFIG = new EvalConfig({
928
932
  // src/config/evalConfigLoader.ts
929
933
  var EvalConfigLoader = class EvalConfigLoader {
930
934
  cwd;
935
+ displayBase;
931
936
  documentLoader = new YamlDocumentLoader();
932
937
  schemaValidator = new SchemaValidator(eval_config_schema_default);
933
938
  variablesReader = new VariablesReader();
934
939
  webSocketResolver = new WebSocketResolver();
935
940
  environmentResolver;
936
- constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
941
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
937
942
  this.cwd = cwd;
943
+ this.displayBase = displayBase;
938
944
  this.environmentResolver = new EnvironmentResolver(environment, parameters);
939
945
  }
940
946
  load(configPath, override, report = new ReportOverride()) {
941
- const document = this.documentLoader.load(configPath, relative(this.cwd, configPath));
947
+ const document = this.documentLoader.load(configPath, relative(this.displayBase, configPath));
942
948
  const issues = [...document.errors];
943
949
  if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
944
950
  const named = [];
@@ -3096,17 +3102,18 @@ var ConsoleReporter = class ConsoleReporter {
3096
3102
  const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
3097
3103
  const failing = test.history.map((entry, index) => ({
3098
3104
  entry,
3105
+ run: result.attempts[index],
3099
3106
  attempt: result.attempts[index]?.attempt ?? index + 1
3100
3107
  })).filter(({ entry }) => entry.status !== ResultStatus.Pass);
3101
3108
  const several = test.history.length > 1;
3102
- for (const { entry, attempt } of failing) {
3109
+ for (const { entry, run, attempt } of failing) {
3103
3110
  if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
3104
- lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
3111
+ lines.push(...this.entryBody(spec, entry, run, several ? 5 : 3));
3105
3112
  }
3106
3113
  lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
3107
3114
  return lines;
3108
3115
  }
3109
- entryBody(spec, entry, indent) {
3116
+ entryBody(spec, entry, attempt, indent) {
3110
3117
  const pad = " ".repeat(indent);
3111
3118
  const evaluation = entry.evaluation;
3112
3119
  if (evaluation === void 0) {
@@ -3120,11 +3127,23 @@ var ConsoleReporter = class ConsoleReporter {
3120
3127
  const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
3121
3128
  const expected = ConsoleReporter.expected(spec, entry);
3122
3129
  const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
3130
+ const actual = ConsoleReporter.actual(entry, attempt);
3123
3131
  if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
3132
+ if (actual !== void 0) lines.push("", `${pad}Actual`, ...this.capped(actual, indent + 2));
3124
3133
  if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
3125
- if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
3126
3134
  return lines;
3127
3135
  }
3136
+ static actual(entry, attempt) {
3137
+ const turns = attempt?.execution.turns ?? [];
3138
+ if (entry.kind === TestKind.Conversation) return turns.length === 0 ? void 0 : turns.map((turn, index) => `Turn ${index + 1}: ${turn.response}`).join("\n");
3139
+ return turns[entry.turn - 1]?.response;
3140
+ }
3141
+ capped(text, indent) {
3142
+ const lines = text.trim() === "" ? [`${" ".repeat(indent)}(empty)`] : this.paragraph(text, indent);
3143
+ if (lines.length <= 12) return lines;
3144
+ const more = lines.length - 12;
3145
+ return [...lines.slice(0, 12), `${" ".repeat(indent)}… ${more} more ${more === 1 ? "line" : "lines"} in the JSON report`];
3146
+ }
3128
3147
  static expected(spec, entry) {
3129
3148
  if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
3130
3149
  return spec.turns[entry.turn - 1]?.expect.response?.criteria;
@@ -3223,10 +3242,12 @@ const JSON_REPORT_FILE = "miniwat-eval-report.json";
3223
3242
  var JsonReporter = class JsonReporter {
3224
3243
  redactor;
3225
3244
  cwd;
3245
+ displayBase;
3226
3246
  tokens = new TokenUsageAggregator();
3227
- constructor(redactor, cwd) {
3247
+ constructor(redactor, cwd, displayBase = cwd) {
3228
3248
  this.redactor = redactor;
3229
3249
  this.cwd = cwd;
3250
+ this.displayBase = displayBase;
3230
3251
  }
3231
3252
  render(run, config, unsuccessfulCount, exitCode) {
3232
3253
  const report = {
@@ -3297,7 +3318,7 @@ var JsonReporter = class JsonReporter {
3297
3318
  return {
3298
3319
  name: result.spec.name,
3299
3320
  runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
3300
- file: result.spec.location.file,
3321
+ file: this.file(result.spec.location.file),
3301
3322
  status: result.status,
3302
3323
  connection: {
3303
3324
  endpoint: websocket.endpoint,
@@ -3307,7 +3328,7 @@ var JsonReporter = class JsonReporter {
3307
3328
  agentUsage: JsonReporter.usage(result.agentUsage),
3308
3329
  evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
3309
3330
  tests: result.tests.map((test) => ({
3310
- ...JsonReporter.identity(test),
3331
+ ...this.identity(test),
3311
3332
  status: test.status,
3312
3333
  ...JsonReporter.counts(test),
3313
3334
  history: test.history.map((entry, index) => ({
@@ -3332,7 +3353,7 @@ var JsonReporter = class JsonReporter {
3332
3353
  agentUsage: JsonReporter.usage(this.tokens.agent([attempt], capability)),
3333
3354
  evaluatorUsage: JsonReporter.usage(this.tokens.evaluator([attempt])),
3334
3355
  turns: execution.turns.map((turn, index) => JsonReporter.turn(index + 1, spec.turns[index].input, turn)),
3335
- tests: attempt.tests.map((test) => JsonReporter.test(test, config))
3356
+ tests: attempt.tests.map((test) => this.test(test, config))
3336
3357
  };
3337
3358
  }
3338
3359
  static turn(number, input, turn) {
@@ -3357,11 +3378,11 @@ var JsonReporter = class JsonReporter {
3357
3378
  tokenUsage: turn.tokenUsage === void 0 ? void 0 : JsonReporter.tokenUsage(turn.tokenUsage)
3358
3379
  };
3359
3380
  }
3360
- static test(test, config) {
3381
+ test(test, config) {
3361
3382
  const thresholds = config.evaluator.thresholds;
3362
3383
  const evaluation = test.evaluation;
3363
3384
  return {
3364
- ...JsonReporter.identity(test),
3385
+ ...this.identity(test),
3365
3386
  status: test.status,
3366
3387
  details: [...test.details],
3367
3388
  evaluation: evaluation === void 0 ? void 0 : {
@@ -3374,17 +3395,17 @@ var JsonReporter = class JsonReporter {
3374
3395
  evaluatorUsage: JsonReporter.usage(test.evaluatorUsage)
3375
3396
  };
3376
3397
  }
3377
- static identity(test) {
3398
+ identity(test) {
3378
3399
  return {
3379
3400
  kind: test.kind,
3380
3401
  turn: test.turn,
3381
3402
  title: test.title,
3382
- location: JsonReporter.location(test.location)
3403
+ location: this.location(test.location)
3383
3404
  };
3384
3405
  }
3385
- static location(location) {
3406
+ location(location) {
3386
3407
  return {
3387
- file: location.file,
3408
+ file: this.file(location.file),
3388
3409
  line: location.line,
3389
3410
  column: location.column,
3390
3411
  path: location.path
@@ -3431,6 +3452,9 @@ var JsonReporter = class JsonReporter {
3431
3452
  path(path) {
3432
3453
  return relative(this.cwd, path) || ".";
3433
3454
  }
3455
+ file(shown) {
3456
+ return shown === "" ? shown : this.path(resolve(this.displayBase, shown));
3457
+ }
3434
3458
  };
3435
3459
  // src/reporting/reportFileWriter.ts
3436
3460
  var ReportFileWriter = class {
@@ -3881,6 +3905,7 @@ var RunWithReader = class RunWithReader {
3881
3905
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
3882
3906
  var ScenarioLoader = class ScenarioLoader {
3883
3907
  cwd;
3908
+ displayBase;
3884
3909
  documentLoader = new YamlDocumentLoader();
3885
3910
  schemaValidator = new SchemaValidator(conversation_schema_default);
3886
3911
  conversationValidator = new ConversationValidator();
@@ -3888,18 +3913,19 @@ var ScenarioLoader = class ScenarioLoader {
3888
3913
  runWithReader = new RunWithReader();
3889
3914
  webSocketResolver = new WebSocketResolver();
3890
3915
  environmentResolver;
3891
- constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
3916
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
3892
3917
  this.cwd = cwd;
3918
+ this.displayBase = displayBase;
3893
3919
  this.environmentResolver = new EnvironmentResolver(environment, parameters);
3894
3920
  }
3895
3921
  load(target, config) {
3896
3922
  const files = this.discover(resolve(this.cwd, target), config);
3897
- const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.cwd, config.configPath)) : void 0;
3923
+ const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.displayBase, config.configPath)) : void 0;
3898
3924
  const issues = [];
3899
3925
  const connectionIssues = [];
3900
3926
  const specs = [];
3901
3927
  for (const file of files) {
3902
- const document = this.documentLoader.load(file, relative(this.cwd, file));
3928
+ const document = this.documentLoader.load(file, relative(this.displayBase, file));
3903
3929
  const fileIssues = [...document.errors];
3904
3930
  const fileSecrets = [];
3905
3931
  const used = /* @__PURE__ */ new Set();
@@ -4029,13 +4055,13 @@ var ScenarioLoader = class ScenarioLoader {
4029
4055
  discover(target, config) {
4030
4056
  if (!existsSync(target)) throw new EvalError({
4031
4057
  category: ErrorCategory.Specification,
4032
- message: `target not found: ${relative(this.cwd, target) || "."}`
4058
+ message: `target not found: ${relative(this.displayBase, target) || "."}`
4033
4059
  });
4034
4060
  const excluded = (path) => path === config.configPath || config.report.directory !== "" && path === config.report.directory;
4035
4061
  const files = statSync(target).isDirectory() ? this.walk(target, excluded) : [target].filter((file) => !excluded(file));
4036
4062
  if (files.length === 0) throw new EvalError({
4037
4063
  category: ErrorCategory.Specification,
4038
- message: `no scenario files found in ${relative(this.cwd, target) || "."}`
4064
+ message: `no scenario files found in ${relative(this.displayBase, target) || "."}`
4039
4065
  });
4040
4066
  return files;
4041
4067
  }
@@ -4115,6 +4141,16 @@ var ScenarioLoader = class ScenarioLoader {
4115
4141
  });
4116
4142
  }
4117
4143
  };
4144
+ // src/utils/displayPath.ts
4145
+ function displayBase(environment, cwd) {
4146
+ const start = environment.INIT_CWD;
4147
+ if (start === void 0 || start === "" || !isAbsolute(start)) return cwd;
4148
+ try {
4149
+ return existsSync(start) && statSync(start).isDirectory() ? start : cwd;
4150
+ } catch {
4151
+ return cwd;
4152
+ }
4153
+ }
4118
4154
  // src/cli/runCommand.ts
4119
4155
  const createLlmEvaluator = async (config) => {
4120
4156
  const factory = new EvaluatorAgentFactory(config.evaluator);
@@ -4141,21 +4177,23 @@ var RunCommand = class RunCommand {
4141
4177
  this.signals = signals;
4142
4178
  this.colour = colour;
4143
4179
  this.columns = columns;
4180
+ this.displayBase = displayBase(environment, cwd);
4144
4181
  }
4182
+ displayBase;
4145
4183
  async execute(target, config, execution = new ExecutionOverride(), report = new ReportOverride(), parameters = "") {
4146
4184
  let evalConfig;
4147
4185
  let specs;
4148
4186
  let evaluator;
4149
4187
  let redactor = new SecretRedactor([]);
4150
4188
  try {
4151
- const configPath = new ConfigLocator(this.cwd).locate(target, config);
4189
+ const configPath = new ConfigLocator(this.cwd, this.displayBase).locate(target, config);
4152
4190
  const parameterFile = parameters === "" ? {
4153
4191
  path: "",
4154
4192
  parameters: {}
4155
- } : new ParametersLoader(this.cwd).load(parameters);
4156
- evalConfig = new EvalConfigLoader(this.environment, this.cwd, parameterFile.parameters).load(configPath, execution, report);
4193
+ } : new ParametersLoader(this.cwd, this.displayBase).load(parameters);
4194
+ evalConfig = new EvalConfigLoader(this.environment, this.cwd, parameterFile.parameters, this.displayBase).load(configPath, execution, report);
4157
4195
  evalConfig.parametersPath = parameterFile.path;
4158
- specs = new ScenarioLoader(this.environment, this.cwd, parameterFile.parameters).load(target, evalConfig);
4196
+ specs = new ScenarioLoader(this.environment, this.cwd, parameterFile.parameters, this.displayBase).load(target, evalConfig);
4159
4197
  redactor = new SecretRedactor([...evalConfig.secrets, ...specs.flatMap((spec) => spec.secrets)]);
4160
4198
  evaluator = RunCommand.needsEvaluator(specs) ? await this.createEvaluator(evalConfig) : new UnavailableEvaluator();
4161
4199
  } catch (error) {
@@ -4217,9 +4255,9 @@ var RunCommand = class RunCommand {
4217
4255
  let exitCode = calculator.exitCode(run);
4218
4256
  const reportPaths = [];
4219
4257
  if (evalConfig.report.formats.includes(ReportFormat.Json)) {
4220
- const content = new JsonReporter(redactor, this.cwd).render(run, evalConfig, calculator.count(run), exitCode);
4258
+ const content = new JsonReporter(redactor, this.cwd, this.displayBase).render(run, evalConfig, calculator.count(run), exitCode);
4221
4259
  const writer = new ReportFileWriter(evalConfig.report.directory);
4222
- const path = relative(this.cwd, writer.path(JSON_REPORT_FILE));
4260
+ const path = relative(this.displayBase, writer.path(JSON_REPORT_FILE));
4223
4261
  try {
4224
4262
  writer.write(JSON_REPORT_FILE, content);
4225
4263
  reportPaths.push(path);
@@ -4247,13 +4285,15 @@ var ValidateCommand = class {
4247
4285
  this.err = err;
4248
4286
  this.environment = environment;
4249
4287
  this.cwd = cwd;
4288
+ this.displayBase = displayBase(environment, cwd);
4250
4289
  }
4290
+ displayBase;
4251
4291
  execute(target, config, parameters = "") {
4252
4292
  try {
4253
- const configPath = new ConfigLocator(this.cwd).locate(target, config);
4254
- const values = parameters === "" ? {} : new ParametersLoader(this.cwd).load(parameters).parameters;
4255
- const evalConfig = new EvalConfigLoader(this.environment, this.cwd, values).load(configPath, new ExecutionOverride());
4256
- const specs = new ScenarioLoader(this.environment, this.cwd, values).load(target, evalConfig);
4293
+ const configPath = new ConfigLocator(this.cwd, this.displayBase).locate(target, config);
4294
+ const values = parameters === "" ? {} : new ParametersLoader(this.cwd, this.displayBase).load(parameters).parameters;
4295
+ const evalConfig = new EvalConfigLoader(this.environment, this.cwd, values, this.displayBase).load(configPath, new ExecutionOverride());
4296
+ const specs = new ScenarioLoader(this.environment, this.cwd, values, this.displayBase).load(target, evalConfig);
4257
4297
  this.out(`Valid: ${specs.length} scenario${specs.length === 1 ? "" : "s"}\n`);
4258
4298
  return 0;
4259
4299
  } catch (error) {