@mappies/miniwat-eval 0.1.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -6
- package/dist/index.cjs +51 -10
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +14 -3
- package/dist/index.d.ts +14 -3
- package/dist/index.js +50 -10
- package/dist/index.js.map +1 -1
- package/dist/main.js +69 -23
- package/dist/main.js.map +1 -1
- package/package.json +2 -5
- package/schemas/report.schema.json +1 -0
package/README.md
CHANGED
|
@@ -30,8 +30,8 @@ npx miniwat-eval run examples/scenarios --quick
|
|
|
30
30
|
## Commands
|
|
31
31
|
|
|
32
32
|
```bash
|
|
33
|
-
miniwat-eval validate <target> [--config <path>]
|
|
34
|
-
miniwat-eval run <target> [--config <path>] [--repeat <n>] [--min-pass <n>] [--quick] [--concurrency <n>] [--report <formats>] [--verbose]
|
|
33
|
+
miniwat-eval validate <target> [--config <path>] [--parameters <path>]
|
|
34
|
+
miniwat-eval run <target> [--config <path>] [--parameters <path>] [--repeat <n>] [--min-pass <n>] [--quick] [--concurrency <n>] [--report <formats>] [--verbose]
|
|
35
35
|
```
|
|
36
36
|
|
|
37
37
|
`<target>` is a scenario file, or a directory searched recursively for `*.yml` and `*.yaml`. The configuration file and the report directory are skipped.
|
|
@@ -39,6 +39,7 @@ miniwat-eval run <target> [--config <path>] [--repeat <n>] [--min-pass <n>] [--q
|
|
|
39
39
|
| Option | Effect |
|
|
40
40
|
|---|---|
|
|
41
41
|
| `--config <path>` | The configuration file. Without it, `eval-config.yml` is searched for upward from the target, stopping at the nearest directory with `.git` or `package.json`, then in the working directory. |
|
|
42
|
+
| `--parameters <path>` | A JSON file of values for `${name}` references, looked up before environment variables. See [References](#references). |
|
|
42
43
|
| `--repeat <n>` | Overrides `execution.repeat`. |
|
|
43
44
|
| `--min-pass <n>` | Overrides `execution.minPass`. |
|
|
44
45
|
| `--quick` | One attempt that must pass: `repeat 1`, `minPass 1`. Wins over `--repeat` and `--min-pass`. |
|
|
@@ -109,7 +110,7 @@ evaluator:
|
|
|
109
110
|
| `execution.timeouts.conversation` | 180000 | Milliseconds for one attempt's connection, from connect to close. Semantic checks after close are not counted. |
|
|
110
111
|
| `evaluator.provider` | required | `openai` or `bedrock`. |
|
|
111
112
|
| `evaluator.model` | required | The provider's model ID. |
|
|
112
|
-
| `evaluator.apiKey` | OpenAI only | Must be
|
|
113
|
+
| `evaluator.apiKey` | OpenAI only | Must be a reference, such as `${OPENAI_API_KEY}`. |
|
|
113
114
|
| `evaluator.region` | Bedrock only | Credentials come from the AWS credential chain, for example `AWS_PROFILE`. |
|
|
114
115
|
| `evaluator.concurrency` | 3 | Evaluator calls at once, across all scenarios. |
|
|
115
116
|
| `evaluator.timeout` | 60000 | Milliseconds per call. |
|
|
@@ -122,16 +123,41 @@ evaluator:
|
|
|
122
123
|
|
|
123
124
|
Unknown keys are errors. The `evaluator` section is required, but its provider package is only loaded when a scenario has semantic criteria.
|
|
124
125
|
|
|
125
|
-
###
|
|
126
|
+
### References
|
|
126
127
|
|
|
127
|
-
`${
|
|
128
|
+
`${name}` in any string value, in the configuration or a scenario, is replaced after the YAML is parsed, so a value containing YAML syntax stays one string. The value comes from:
|
|
129
|
+
|
|
130
|
+
1. the `--parameters` file, when one is given;
|
|
131
|
+
2. otherwise, the environment variable of that name.
|
|
132
|
+
|
|
133
|
+
A missing value is an error at its location; values never appear in messages. `.env` files are not loaded.
|
|
134
|
+
|
|
135
|
+
The parameters file is a JSON object. Each top-level string value is a parameter, and anything else is ignored, so an apiunit `parameters.json`, with its `settings` object, can be used as it is. Names may contain hyphens, as in apiunit:
|
|
136
|
+
|
|
137
|
+
```json
|
|
138
|
+
{ "knowledge-ws": "ws://localhost:8122", "user-token": "eyJ..." }
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
```yaml
|
|
142
|
+
websocket:
|
|
143
|
+
endpoint: ${knowledge-ws}
|
|
144
|
+
connection:
|
|
145
|
+
subprotocols:
|
|
146
|
+
- ${user-token}
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
miniwat-eval run eval/scenarios --parameters apiunit/parameters.json
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Keeping one file per environment, such as `parameters.json` and `parameters_cicd.json`, switches endpoints and tokens without changing the configuration.
|
|
128
154
|
|
|
129
155
|
### Redaction
|
|
130
156
|
|
|
131
157
|
Secrets are replaced with `[REDACTED]` in everything `run` prints and in report files. These are:
|
|
132
158
|
|
|
133
159
|
- the values of `evaluator.apiKey` and `websocket.connection.subprotocols`;
|
|
134
|
-
- the value of every `${
|
|
160
|
+
- the value of every `${name}` reference, in the configuration or a scenario, whose name contains `TOKEN`, `SECRET`, `PASSWORD` or `KEY` in any case, whether it comes from `--parameters` or the environment. `${user-token}` is one.
|
|
135
161
|
|
|
136
162
|
Only exact values are replaced, so text that does not contain a secret is never changed. Every occurrence is replaced, so a very short value, such as a one-letter placeholder, also changes ordinary words that contain it. The evaluator model still receives the agent's real responses.
|
|
137
163
|
|
package/dist/index.cjs
CHANGED
|
@@ -26,10 +26,10 @@ let node_path = require("node:path");
|
|
|
26
26
|
let ajv = require("ajv");
|
|
27
27
|
let yaml = require("yaml");
|
|
28
28
|
let configurapi_handler_ws = require("configurapi-handler-ws");
|
|
29
|
-
configurapi_handler_ws = __toESM(configurapi_handler_ws, 1);
|
|
30
29
|
let node_util = require("node:util");
|
|
31
30
|
let _mappies_miniwat_agent = require("@mappies/miniwat-agent");
|
|
32
31
|
let node_crypto = require("node:crypto");
|
|
32
|
+
configurapi_handler_ws = __toESM(configurapi_handler_ws, 1);
|
|
33
33
|
// src/entities/turnExecution.ts
|
|
34
34
|
var TurnExecution = class {
|
|
35
35
|
response;
|
|
@@ -461,6 +461,7 @@ var WebSocketConfig = class {
|
|
|
461
461
|
var EvalConfig = class {
|
|
462
462
|
version;
|
|
463
463
|
configPath;
|
|
464
|
+
parametersPath;
|
|
464
465
|
websocket;
|
|
465
466
|
execution;
|
|
466
467
|
evaluator;
|
|
@@ -469,6 +470,7 @@ var EvalConfig = class {
|
|
|
469
470
|
constructor(data) {
|
|
470
471
|
this.version = data?.version ?? 1;
|
|
471
472
|
this.configPath = data?.configPath ?? "";
|
|
473
|
+
this.parametersPath = data?.parametersPath ?? "";
|
|
472
474
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
473
475
|
this.execution = new ExecutionConfig(data?.execution);
|
|
474
476
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -939,12 +941,14 @@ var eval_config_schema_default = {
|
|
|
939
941
|
}
|
|
940
942
|
};
|
|
941
943
|
// src/yaml/environmentResolver.ts
|
|
942
|
-
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g;
|
|
944
|
+
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
|
|
943
945
|
const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
|
|
944
946
|
var EnvironmentResolver = class EnvironmentResolver {
|
|
945
947
|
environment;
|
|
946
|
-
|
|
948
|
+
parameters;
|
|
949
|
+
constructor(environment = process.env, parameters = {}) {
|
|
947
950
|
this.environment = environment;
|
|
951
|
+
this.parameters = parameters;
|
|
948
952
|
}
|
|
949
953
|
resolve(value, document, issues, secrets = []) {
|
|
950
954
|
return this.resolveValue(value, [], document, issues, secrets);
|
|
@@ -957,10 +961,11 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
957
961
|
}
|
|
958
962
|
resolveValue(value, path, document, issues, secrets) {
|
|
959
963
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
960
|
-
const resolved = this.environment[name];
|
|
964
|
+
const resolved = Object.hasOwn(this.parameters, name) ? this.parameters[name] : this.environment[name];
|
|
961
965
|
if (resolved === void 0) {
|
|
966
|
+
const source = Object.keys(this.parameters).length > 0 ? "parameter or environment variable" : "environment variable";
|
|
962
967
|
issues.push(new ValidationIssue({
|
|
963
|
-
message:
|
|
968
|
+
message: `${source} '${name}' is not set`,
|
|
964
969
|
location: document.locate(path)
|
|
965
970
|
}));
|
|
966
971
|
return reference;
|
|
@@ -1134,9 +1139,9 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1134
1139
|
documentLoader = new YamlDocumentLoader();
|
|
1135
1140
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
1136
1141
|
environmentResolver;
|
|
1137
|
-
constructor(environment = process.env, cwd = process.cwd()) {
|
|
1142
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1138
1143
|
this.cwd = cwd;
|
|
1139
|
-
this.environmentResolver = new EnvironmentResolver(environment);
|
|
1144
|
+
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
1140
1145
|
}
|
|
1141
1146
|
load(configPath, override, report = new ReportOverride()) {
|
|
1142
1147
|
const document = this.documentLoader.load(configPath, (0, node_path.relative)(this.cwd, configPath));
|
|
@@ -1173,7 +1178,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1173
1178
|
const apiKey = EvalConfigLoader.section(document.value, "evaluator").apiKey;
|
|
1174
1179
|
if (typeof apiKey !== "string" || EnvironmentResolver.hasReference(apiKey)) return [];
|
|
1175
1180
|
return [new ValidationIssue({
|
|
1176
|
-
message: "must be
|
|
1181
|
+
message: "must be a reference such as ${OPENAI_API_KEY}",
|
|
1177
1182
|
location: document.locate(["evaluator", "apiKey"])
|
|
1178
1183
|
})];
|
|
1179
1184
|
}
|
|
@@ -1298,6 +1303,40 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1298
1303
|
return isRecord(value) ? value[key] : void 0;
|
|
1299
1304
|
}
|
|
1300
1305
|
};
|
|
1306
|
+
// src/config/parametersLoader.ts
|
|
1307
|
+
var ParametersLoader = class ParametersLoader {
|
|
1308
|
+
cwd;
|
|
1309
|
+
constructor(cwd = process.cwd()) {
|
|
1310
|
+
this.cwd = cwd;
|
|
1311
|
+
}
|
|
1312
|
+
load(path) {
|
|
1313
|
+
const absolute = (0, node_path.resolve)(this.cwd, path);
|
|
1314
|
+
const shown = (0, node_path.relative)(this.cwd, absolute) || path;
|
|
1315
|
+
let text;
|
|
1316
|
+
let value;
|
|
1317
|
+
try {
|
|
1318
|
+
text = (0, node_fs.readFileSync)(absolute, "utf8");
|
|
1319
|
+
} catch {
|
|
1320
|
+
throw ParametersLoader.error(`parameters file not found: ${shown}`);
|
|
1321
|
+
}
|
|
1322
|
+
try {
|
|
1323
|
+
value = JSON.parse(text);
|
|
1324
|
+
} catch {
|
|
1325
|
+
throw ParametersLoader.error(`parameters file ${shown} is not valid JSON`);
|
|
1326
|
+
}
|
|
1327
|
+
if (!isRecord(value)) throw ParametersLoader.error(`parameters file ${shown} must contain a JSON object`);
|
|
1328
|
+
return {
|
|
1329
|
+
path: absolute,
|
|
1330
|
+
parameters: Object.fromEntries(Object.entries(value).filter((entry) => typeof entry[1] === "string"))
|
|
1331
|
+
};
|
|
1332
|
+
}
|
|
1333
|
+
static error(message) {
|
|
1334
|
+
return new EvalError({
|
|
1335
|
+
category: ErrorCategory.Configuration,
|
|
1336
|
+
message
|
|
1337
|
+
});
|
|
1338
|
+
}
|
|
1339
|
+
};
|
|
1301
1340
|
// schemas/conversation.schema.json
|
|
1302
1341
|
var conversation_schema_default = {
|
|
1303
1342
|
$schema: "http://json-schema.org/draft-07/schema#",
|
|
@@ -1478,9 +1517,9 @@ var ScenarioLoader = class {
|
|
|
1478
1517
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
1479
1518
|
conversationValidator = new ConversationValidator();
|
|
1480
1519
|
environmentResolver;
|
|
1481
|
-
constructor(environment = process.env, cwd = process.cwd()) {
|
|
1520
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1482
1521
|
this.cwd = cwd;
|
|
1483
|
-
this.environmentResolver = new EnvironmentResolver(environment);
|
|
1522
|
+
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
1484
1523
|
}
|
|
1485
1524
|
load(target, config) {
|
|
1486
1525
|
const files = this.discover((0, node_path.resolve)(this.cwd, target), config);
|
|
@@ -3179,6 +3218,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3179
3218
|
const { websocket, execution, evaluator, report } = config;
|
|
3180
3219
|
return {
|
|
3181
3220
|
path: this.path(config.configPath),
|
|
3221
|
+
parametersPath: config.parametersPath === "" ? void 0 : this.path(config.parametersPath),
|
|
3182
3222
|
websocket: {
|
|
3183
3223
|
endpoint: websocket.endpoint,
|
|
3184
3224
|
subprotocols: [...websocket.connection.subprotocols],
|
|
@@ -3406,6 +3446,7 @@ exports.JSON_REPORT_FILE = JSON_REPORT_FILE;
|
|
|
3406
3446
|
exports.JSON_REPORT_VERSION = JSON_REPORT_VERSION;
|
|
3407
3447
|
exports.JsonReporter = JsonReporter;
|
|
3408
3448
|
exports.LlmEvaluator = LlmEvaluator;
|
|
3449
|
+
exports.ParametersLoader = ParametersLoader;
|
|
3409
3450
|
exports.ProtocolCapabilities = ProtocolCapabilities;
|
|
3410
3451
|
exports.ProtocolConfig = ProtocolConfig;
|
|
3411
3452
|
exports.REDACTED = REDACTED;
|