@mappies/miniwat-eval 0.1.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/main.js CHANGED
@@ -164,6 +164,44 @@ var ConfigLocator = class ConfigLocator {
164
164
  return existsSync(path) && statSync(path).isFile();
165
165
  }
166
166
  };
167
+ // src/utils/isRecord.ts
168
+ function isRecord(value) {
169
+ return value !== null && typeof value === "object" && !Array.isArray(value);
170
+ }
171
+ // src/config/parametersLoader.ts
172
+ var ParametersLoader = class ParametersLoader {
173
+ cwd;
174
+ constructor(cwd = process.cwd()) {
175
+ this.cwd = cwd;
176
+ }
177
+ load(path) {
178
+ const absolute = resolve(this.cwd, path);
179
+ const shown = relative(this.cwd, absolute) || path;
180
+ let text;
181
+ let value;
182
+ try {
183
+ text = readFileSync(absolute, "utf8");
184
+ } catch {
185
+ throw ParametersLoader.error(`parameters file not found: ${shown}`);
186
+ }
187
+ try {
188
+ value = JSON.parse(text);
189
+ } catch {
190
+ throw ParametersLoader.error(`parameters file ${shown} is not valid JSON`);
191
+ }
192
+ if (!isRecord(value)) throw ParametersLoader.error(`parameters file ${shown} must contain a JSON object`);
193
+ return {
194
+ path: absolute,
195
+ parameters: Object.fromEntries(Object.entries(value).filter((entry) => typeof entry[1] === "string"))
196
+ };
197
+ }
198
+ static error(message) {
199
+ return new EvalError({
200
+ category: ErrorCategory.Configuration,
201
+ message
202
+ });
203
+ }
204
+ };
167
205
  // schemas/eval-config.schema.json
168
206
  var eval_config_schema_default = {
169
207
  $schema: "http://json-schema.org/draft-07/schema#",
@@ -471,6 +509,7 @@ var WebSocketConfig = class {
471
509
  var EvalConfig = class {
472
510
  version;
473
511
  configPath;
512
+ parametersPath;
474
513
  websocket;
475
514
  execution;
476
515
  evaluator;
@@ -479,6 +518,7 @@ var EvalConfig = class {
479
518
  constructor(data) {
480
519
  this.version = data?.version ?? 1;
481
520
  this.configPath = data?.configPath ?? "";
521
+ this.parametersPath = data?.parametersPath ?? "";
482
522
  this.websocket = new WebSocketConfig(data?.websocket);
483
523
  this.execution = new ExecutionConfig(data?.execution);
484
524
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -492,12 +532,14 @@ const ReportFormat = {
492
532
  Json: "json"
493
533
  };
494
534
  // src/yaml/environmentResolver.ts
495
- const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g;
535
+ const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
496
536
  const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
497
537
  var EnvironmentResolver = class EnvironmentResolver {
498
538
  environment;
499
- constructor(environment = process.env) {
539
+ parameters;
540
+ constructor(environment = process.env, parameters = {}) {
500
541
  this.environment = environment;
542
+ this.parameters = parameters;
501
543
  }
502
544
  resolve(value, document, issues, secrets = []) {
503
545
  return this.resolveValue(value, [], document, issues, secrets);
@@ -510,10 +552,11 @@ var EnvironmentResolver = class EnvironmentResolver {
510
552
  }
511
553
  resolveValue(value, path, document, issues, secrets) {
512
554
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
513
- const resolved = this.environment[name];
555
+ const resolved = Object.hasOwn(this.parameters, name) ? this.parameters[name] : this.environment[name];
514
556
  if (resolved === void 0) {
557
+ const source = Object.keys(this.parameters).length > 0 ? "parameter or environment variable" : "environment variable";
515
558
  issues.push(new ValidationIssue({
516
- message: `environment variable '${name}' is not set`,
559
+ message: `${source} '${name}' is not set`,
517
560
  location: document.locate(path)
518
561
  }));
519
562
  return reference;
@@ -677,10 +720,6 @@ var YamlDocumentLoader = class {
677
720
  }), lineCounter);
678
721
  }
679
722
  };
680
- // src/utils/isRecord.ts
681
- function isRecord(value) {
682
- return value !== null && typeof value === "object" && !Array.isArray(value);
683
- }
684
723
  // src/config/defaultConfig.ts
685
724
  const DEFAULT_CONFIG = new EvalConfig({
686
725
  execution: new ExecutionConfig({
@@ -717,9 +756,9 @@ var EvalConfigLoader = class EvalConfigLoader {
717
756
  documentLoader = new YamlDocumentLoader();
718
757
  schemaValidator = new SchemaValidator(eval_config_schema_default);
719
758
  environmentResolver;
720
- constructor(environment = process.env, cwd = process.cwd()) {
759
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
721
760
  this.cwd = cwd;
722
- this.environmentResolver = new EnvironmentResolver(environment);
761
+ this.environmentResolver = new EnvironmentResolver(environment, parameters);
723
762
  }
724
763
  load(configPath, override, report = new ReportOverride()) {
725
764
  const document = this.documentLoader.load(configPath, relative(this.cwd, configPath));
@@ -756,7 +795,7 @@ var EvalConfigLoader = class EvalConfigLoader {
756
795
  const apiKey = EvalConfigLoader.section(document.value, "evaluator").apiKey;
757
796
  if (typeof apiKey !== "string" || EnvironmentResolver.hasReference(apiKey)) return [];
758
797
  return [new ValidationIssue({
759
- message: "must be an environment reference such as ${OPENAI_API_KEY}",
798
+ message: "must be a reference such as ${OPENAI_API_KEY}",
760
799
  location: document.locate(["evaluator", "apiKey"])
761
800
  })];
762
801
  }
@@ -2928,6 +2967,7 @@ var JsonReporter = class JsonReporter {
2928
2967
  const { websocket, execution, evaluator, report } = config;
2929
2968
  return {
2930
2969
  path: this.path(config.configPath),
2970
+ parametersPath: config.parametersPath === "" ? void 0 : this.path(config.parametersPath),
2931
2971
  websocket: {
2932
2972
  endpoint: websocket.endpoint,
2933
2973
  subprotocols: [...websocket.connection.subprotocols],
@@ -3290,9 +3330,9 @@ var ScenarioLoader = class {
3290
3330
  schemaValidator = new SchemaValidator(conversation_schema_default);
3291
3331
  conversationValidator = new ConversationValidator();
3292
3332
  environmentResolver;
3293
- constructor(environment = process.env, cwd = process.cwd()) {
3333
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
3294
3334
  this.cwd = cwd;
3295
- this.environmentResolver = new EnvironmentResolver(environment);
3335
+ this.environmentResolver = new EnvironmentResolver(environment, parameters);
3296
3336
  }
3297
3337
  load(target, config) {
3298
3338
  const files = this.discover(resolve(this.cwd, target), config);
@@ -3427,15 +3467,20 @@ var RunCommand = class RunCommand {
3427
3467
  this.signals = signals;
3428
3468
  this.colour = colour;
3429
3469
  }
3430
- async execute(target, config, execution = new ExecutionOverride(), report = new ReportOverride()) {
3470
+ async execute(target, config, execution = new ExecutionOverride(), report = new ReportOverride(), parameters = "") {
3431
3471
  let evalConfig;
3432
3472
  let specs;
3433
3473
  let evaluator;
3434
3474
  let redactor = new SecretRedactor([]);
3435
3475
  try {
3436
3476
  const configPath = new ConfigLocator(this.cwd).locate(target, config);
3437
- evalConfig = new EvalConfigLoader(this.environment, this.cwd).load(configPath, execution, report);
3438
- specs = new ScenarioLoader(this.environment, this.cwd).load(target, evalConfig);
3477
+ const parameterFile = parameters === "" ? {
3478
+ path: "",
3479
+ parameters: {}
3480
+ } : new ParametersLoader(this.cwd).load(parameters);
3481
+ evalConfig = new EvalConfigLoader(this.environment, this.cwd, parameterFile.parameters).load(configPath, execution, report);
3482
+ evalConfig.parametersPath = parameterFile.path;
3483
+ specs = new ScenarioLoader(this.environment, this.cwd, parameterFile.parameters).load(target, evalConfig);
3439
3484
  redactor = new SecretRedactor([...evalConfig.secrets, ...specs.flatMap((spec) => spec.secrets)]);
3440
3485
  evaluator = RunCommand.needsEvaluator(specs) ? await this.createEvaluator(evalConfig) : new UnavailableEvaluator();
3441
3486
  } catch (error) {
@@ -3524,11 +3569,12 @@ var ValidateCommand = class {
3524
3569
  this.environment = environment;
3525
3570
  this.cwd = cwd;
3526
3571
  }
3527
- execute(target, config) {
3572
+ execute(target, config, parameters = "") {
3528
3573
  try {
3529
3574
  const configPath = new ConfigLocator(this.cwd).locate(target, config);
3530
- const evalConfig = new EvalConfigLoader(this.environment, this.cwd).load(configPath, new ExecutionOverride());
3531
- const specs = new ScenarioLoader(this.environment, this.cwd).load(target, evalConfig);
3575
+ const values = parameters === "" ? {} : new ParametersLoader(this.cwd).load(parameters).parameters;
3576
+ const evalConfig = new EvalConfigLoader(this.environment, this.cwd, values).load(configPath, new ExecutionOverride());
3577
+ const specs = new ScenarioLoader(this.environment, this.cwd, values).load(target, evalConfig);
3532
3578
  this.out(`Valid: ${specs.length} scenario${specs.length === 1 ? "" : "s"}\n`);
3533
3579
  return 0;
3534
3580
  } catch (error) {
@@ -3545,10 +3591,10 @@ function createProgram(setExitCode, out, err, colour = false) {
3545
3591
  writeOut: out,
3546
3592
  writeErr: err
3547
3593
  }).exitOverride();
3548
- program.command("validate").description("Parse, interpolate and validate configuration and scenarios with no network calls").argument("<target>", "scenario file or directory").option("--config <path>", "configuration file (default: search upward for eval-config.yml)").action((target, options) => {
3549
- setExitCode(new ValidateCommand(out, err).execute(target, options.config ?? ""));
3594
+ program.command("validate").description("Parse, interpolate and validate configuration and scenarios with no network calls").argument("<target>", "scenario file or directory").option("--config <path>", "configuration file (default: search upward for eval-config.yml)").option("--parameters <path>", "JSON file whose string values resolve ${name} references before the environment").action((target, options) => {
3595
+ setExitCode(new ValidateCommand(out, err).execute(target, options.config ?? "", options.parameters ?? ""));
3550
3596
  });
3551
- program.command("run").description("Run every scenario against the agent, repeat times each, and report the results").argument("<target>", "scenario file or directory").option("--config <path>", "configuration file (default: search upward for eval-config.yml)").option("--repeat <n>", "attempts per scenario (overrides execution.repeat)").option("--min-pass <n>", "passing attempts a scenario needs (overrides execution.minPass)").option("--quick", "shorthand for --repeat 1 --min-pass 1; wins over both").option("--concurrency <n>", "scenarios run at once (overrides execution.scenarioConcurrency)").option("--report <formats>", "comma-separated console, json (overrides report.formats)").option("--verbose", "include every attempt's transcript in the console report").action(async (target, options) => {
3597
+ program.command("run").description("Run every scenario against the agent, repeat times each, and report the results").argument("<target>", "scenario file or directory").option("--config <path>", "configuration file (default: search upward for eval-config.yml)").option("--parameters <path>", "JSON file whose string values resolve ${name} references before the environment").option("--repeat <n>", "attempts per scenario (overrides execution.repeat)").option("--min-pass <n>", "passing attempts a scenario needs (overrides execution.minPass)").option("--quick", "shorthand for --repeat 1 --min-pass 1; wins over both").option("--concurrency <n>", "scenarios run at once (overrides execution.scenarioConcurrency)").option("--report <formats>", "comma-separated console, json (overrides report.formats)").option("--verbose", "include every attempt's transcript in the console report").action(async (target, options) => {
3552
3598
  let execution;
3553
3599
  let report;
3554
3600
  try {
@@ -3560,7 +3606,7 @@ function createProgram(setExitCode, out, err, colour = false) {
3560
3606
  setExitCode(1);
3561
3607
  return;
3562
3608
  }
3563
- setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour).execute(target, options.config ?? "", execution, report));
3609
+ setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour).execute(target, options.config ?? "", execution, report, options.parameters ?? ""));
3564
3610
  });
3565
3611
  return program;
3566
3612
  }