@mappies/miniwat-eval 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -30,8 +30,8 @@ npx miniwat-eval run examples/scenarios --quick
30
30
  ## Commands
31
31
 
32
32
  ```bash
33
- miniwat-eval validate <target> [--config <path>]
34
- miniwat-eval run <target> [--config <path>] [--repeat <n>] [--min-pass <n>] [--quick] [--concurrency <n>] [--report <formats>] [--verbose]
33
+ miniwat-eval validate <target> [--config <path>] [--parameters <path>]
34
+ miniwat-eval run <target> [--config <path>] [--parameters <path>] [--repeat <n>] [--min-pass <n>] [--quick] [--concurrency <n>] [--report <formats>] [--verbose]
35
35
  ```
36
36
 
37
37
  `<target>` is a scenario file, or a directory searched recursively for `*.yml` and `*.yaml`. The configuration file and the report directory are skipped.
@@ -39,6 +39,7 @@ miniwat-eval run <target> [--config <path>] [--repeat <n>] [--min-pass <n>] [--q
39
39
  | Option | Effect |
40
40
  |---|---|
41
41
  | `--config <path>` | The configuration file. Without it, `eval-config.yml` is searched for upward from the target, stopping at the nearest directory with `.git` or `package.json`, then in the working directory. |
42
+ | `--parameters <path>` | A JSON file of values for `${name}` references, looked up before environment variables. See [References](#references). |
42
43
  | `--repeat <n>` | Overrides `execution.repeat`. |
43
44
  | `--min-pass <n>` | Overrides `execution.minPass`. |
44
45
  | `--quick` | One attempt that must pass: `repeat 1`, `minPass 1`. Wins over `--repeat` and `--min-pass`. |
@@ -109,7 +110,7 @@ evaluator:
109
110
  | `execution.timeouts.conversation` | 180000 | Milliseconds for one attempt's connection, from connect to close. Semantic checks after close are not counted. |
110
111
  | `evaluator.provider` | required | `openai` or `bedrock`. |
111
112
  | `evaluator.model` | required | The provider's model ID. |
112
- | `evaluator.apiKey` | OpenAI only | Must be an environment reference. |
113
+ | `evaluator.apiKey` | OpenAI only | Must be a reference, such as `${OPENAI_API_KEY}`. |
113
114
  | `evaluator.region` | Bedrock only | Credentials come from the AWS credential chain, for example `AWS_PROFILE`. |
114
115
  | `evaluator.concurrency` | 3 | Evaluator calls at once, across all scenarios. |
115
116
  | `evaluator.timeout` | 60000 | Milliseconds per call. |
@@ -122,16 +123,41 @@ evaluator:
122
123
 
123
124
  Unknown keys are errors. The `evaluator` section is required, but its provider package is only loaded when a scenario has semantic criteria.
124
125
 
125
- ### Environment references
126
+ ### References
126
127
 
127
- `${NAME}` in any string value, in the configuration or a scenario, is replaced with the environment variable `NAME` after the YAML is parsed, so a value containing YAML syntax stays one string. A missing variable is an error at its location; its value never appears in messages. `.env` files are not loaded.
128
+ `${name}` in any string value, in the configuration or a scenario, is replaced after the YAML is parsed, so a value containing YAML syntax stays one string. The value comes from:
129
+
130
+ 1. the `--parameters` file, when one is given;
131
+ 2. otherwise, the environment variable of that name.
132
+
133
+ A missing value is an error at its location; values never appear in messages. `.env` files are not loaded.
134
+
135
+ The parameters file is a JSON object. Each top-level string value is a parameter, and anything else is ignored, so an apiunit `parameters.json`, with its `settings` object, can be used as it is. Names may contain hyphens, as in apiunit:
136
+
137
+ ```json
138
+ { "knowledge-ws": "ws://localhost:8122", "user-token": "eyJ..." }
139
+ ```
140
+
141
+ ```yaml
142
+ websocket:
143
+ endpoint: ${knowledge-ws}
144
+ connection:
145
+ subprotocols:
146
+ - ${user-token}
147
+ ```
148
+
149
+ ```bash
150
+ miniwat-eval run eval/scenarios --parameters apiunit/parameters.json
151
+ ```
152
+
153
+ Keeping one file per environment, such as `parameters.json` and `parameters_cicd.json`, switches endpoints and tokens without changing the configuration.
128
154
 
129
155
  ### Redaction
130
156
 
131
157
  Secrets are replaced with `[REDACTED]` in everything `run` prints and in report files. These are:
132
158
 
133
159
  - the values of `evaluator.apiKey` and `websocket.connection.subprotocols`;
134
- - the value of every `${NAME}` reference, in the configuration or a scenario, whose name contains `TOKEN`, `SECRET`, `PASSWORD` or `KEY`, in any case.
160
+ - the value of every `${name}` reference, in the configuration or a scenario, whose name contains `TOKEN`, `SECRET`, `PASSWORD` or `KEY` in any case, whether it comes from `--parameters` or the environment. `${user-token}` is one.
135
161
 
136
162
  Only exact values are replaced, so text that does not contain a secret is never changed. Every occurrence is replaced, so a very short value, such as a one-letter placeholder, also changes ordinary words that contain it. The evaluator model still receives the agent's real responses.
137
163
 
package/dist/index.cjs CHANGED
@@ -26,10 +26,10 @@ let node_path = require("node:path");
26
26
  let ajv = require("ajv");
27
27
  let yaml = require("yaml");
28
28
  let configurapi_handler_ws = require("configurapi-handler-ws");
29
- configurapi_handler_ws = __toESM(configurapi_handler_ws, 1);
30
29
  let node_util = require("node:util");
31
30
  let _mappies_miniwat_agent = require("@mappies/miniwat-agent");
32
31
  let node_crypto = require("node:crypto");
32
+ configurapi_handler_ws = __toESM(configurapi_handler_ws, 1);
33
33
  // src/entities/turnExecution.ts
34
34
  var TurnExecution = class {
35
35
  response;
@@ -461,6 +461,7 @@ var WebSocketConfig = class {
461
461
  var EvalConfig = class {
462
462
  version;
463
463
  configPath;
464
+ parametersPath;
464
465
  websocket;
465
466
  execution;
466
467
  evaluator;
@@ -469,6 +470,7 @@ var EvalConfig = class {
469
470
  constructor(data) {
470
471
  this.version = data?.version ?? 1;
471
472
  this.configPath = data?.configPath ?? "";
473
+ this.parametersPath = data?.parametersPath ?? "";
472
474
  this.websocket = new WebSocketConfig(data?.websocket);
473
475
  this.execution = new ExecutionConfig(data?.execution);
474
476
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -939,12 +941,14 @@ var eval_config_schema_default = {
939
941
  }
940
942
  };
941
943
  // src/yaml/environmentResolver.ts
942
- const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g;
944
+ const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
943
945
  const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
944
946
  var EnvironmentResolver = class EnvironmentResolver {
945
947
  environment;
946
- constructor(environment = process.env) {
948
+ parameters;
949
+ constructor(environment = process.env, parameters = {}) {
947
950
  this.environment = environment;
951
+ this.parameters = parameters;
948
952
  }
949
953
  resolve(value, document, issues, secrets = []) {
950
954
  return this.resolveValue(value, [], document, issues, secrets);
@@ -957,10 +961,11 @@ var EnvironmentResolver = class EnvironmentResolver {
957
961
  }
958
962
  resolveValue(value, path, document, issues, secrets) {
959
963
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
960
- const resolved = this.environment[name];
964
+ const resolved = Object.hasOwn(this.parameters, name) ? this.parameters[name] : this.environment[name];
961
965
  if (resolved === void 0) {
966
+ const source = Object.keys(this.parameters).length > 0 ? "parameter or environment variable" : "environment variable";
962
967
  issues.push(new ValidationIssue({
963
- message: `environment variable '${name}' is not set`,
968
+ message: `${source} '${name}' is not set`,
964
969
  location: document.locate(path)
965
970
  }));
966
971
  return reference;
@@ -1134,9 +1139,9 @@ var EvalConfigLoader = class EvalConfigLoader {
1134
1139
  documentLoader = new YamlDocumentLoader();
1135
1140
  schemaValidator = new SchemaValidator(eval_config_schema_default);
1136
1141
  environmentResolver;
1137
- constructor(environment = process.env, cwd = process.cwd()) {
1142
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1138
1143
  this.cwd = cwd;
1139
- this.environmentResolver = new EnvironmentResolver(environment);
1144
+ this.environmentResolver = new EnvironmentResolver(environment, parameters);
1140
1145
  }
1141
1146
  load(configPath, override, report = new ReportOverride()) {
1142
1147
  const document = this.documentLoader.load(configPath, (0, node_path.relative)(this.cwd, configPath));
@@ -1173,7 +1178,7 @@ var EvalConfigLoader = class EvalConfigLoader {
1173
1178
  const apiKey = EvalConfigLoader.section(document.value, "evaluator").apiKey;
1174
1179
  if (typeof apiKey !== "string" || EnvironmentResolver.hasReference(apiKey)) return [];
1175
1180
  return [new ValidationIssue({
1176
- message: "must be an environment reference such as ${OPENAI_API_KEY}",
1181
+ message: "must be a reference such as ${OPENAI_API_KEY}",
1177
1182
  location: document.locate(["evaluator", "apiKey"])
1178
1183
  })];
1179
1184
  }
@@ -1298,6 +1303,40 @@ var EvalConfigLoader = class EvalConfigLoader {
1298
1303
  return isRecord(value) ? value[key] : void 0;
1299
1304
  }
1300
1305
  };
1306
+ // src/config/parametersLoader.ts
1307
+ var ParametersLoader = class ParametersLoader {
1308
+ cwd;
1309
+ constructor(cwd = process.cwd()) {
1310
+ this.cwd = cwd;
1311
+ }
1312
+ load(path) {
1313
+ const absolute = (0, node_path.resolve)(this.cwd, path);
1314
+ const shown = (0, node_path.relative)(this.cwd, absolute) || path;
1315
+ let text;
1316
+ let value;
1317
+ try {
1318
+ text = (0, node_fs.readFileSync)(absolute, "utf8");
1319
+ } catch {
1320
+ throw ParametersLoader.error(`parameters file not found: ${shown}`);
1321
+ }
1322
+ try {
1323
+ value = JSON.parse(text);
1324
+ } catch {
1325
+ throw ParametersLoader.error(`parameters file ${shown} is not valid JSON`);
1326
+ }
1327
+ if (!isRecord(value)) throw ParametersLoader.error(`parameters file ${shown} must contain a JSON object`);
1328
+ return {
1329
+ path: absolute,
1330
+ parameters: Object.fromEntries(Object.entries(value).filter((entry) => typeof entry[1] === "string"))
1331
+ };
1332
+ }
1333
+ static error(message) {
1334
+ return new EvalError({
1335
+ category: ErrorCategory.Configuration,
1336
+ message
1337
+ });
1338
+ }
1339
+ };
1301
1340
  // schemas/conversation.schema.json
1302
1341
  var conversation_schema_default = {
1303
1342
  $schema: "http://json-schema.org/draft-07/schema#",
@@ -1478,9 +1517,9 @@ var ScenarioLoader = class {
1478
1517
  schemaValidator = new SchemaValidator(conversation_schema_default);
1479
1518
  conversationValidator = new ConversationValidator();
1480
1519
  environmentResolver;
1481
- constructor(environment = process.env, cwd = process.cwd()) {
1520
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1482
1521
  this.cwd = cwd;
1483
- this.environmentResolver = new EnvironmentResolver(environment);
1522
+ this.environmentResolver = new EnvironmentResolver(environment, parameters);
1484
1523
  }
1485
1524
  load(target, config) {
1486
1525
  const files = this.discover((0, node_path.resolve)(this.cwd, target), config);
@@ -3179,6 +3218,7 @@ var JsonReporter = class JsonReporter {
3179
3218
  const { websocket, execution, evaluator, report } = config;
3180
3219
  return {
3181
3220
  path: this.path(config.configPath),
3221
+ parametersPath: config.parametersPath === "" ? void 0 : this.path(config.parametersPath),
3182
3222
  websocket: {
3183
3223
  endpoint: websocket.endpoint,
3184
3224
  subprotocols: [...websocket.connection.subprotocols],
@@ -3406,6 +3446,7 @@ exports.JSON_REPORT_FILE = JSON_REPORT_FILE;
3406
3446
  exports.JSON_REPORT_VERSION = JSON_REPORT_VERSION;
3407
3447
  exports.JsonReporter = JsonReporter;
3408
3448
  exports.LlmEvaluator = LlmEvaluator;
3449
+ exports.ParametersLoader = ParametersLoader;
3409
3450
  exports.ProtocolCapabilities = ProtocolCapabilities;
3410
3451
  exports.ProtocolConfig = ProtocolConfig;
3411
3452
  exports.REDACTED = REDACTED;