@mappies/miniwat-eval 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -5
- package/dist/index.cjs +393 -44
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +9 -2
- package/dist/index.d.ts +10 -2
- package/dist/index.js +393 -44
- package/dist/index.js.map +1 -1
- package/dist/main.js +393 -44
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +35 -1
- package/schemas/eval-config.schema.json +6 -1
- package/schemas/report.schema.json +1 -0
package/README.md
CHANGED
|
@@ -127,10 +127,11 @@ Unknown keys are errors. The `evaluator` section is required, but its provider p
|
|
|
127
127
|
|
|
128
128
|
`${name}` in any string value, in the configuration or a scenario, is replaced after the YAML is parsed, so a value containing YAML syntax stays one string. The value comes from the first of these that has it:
|
|
129
129
|
|
|
130
|
-
1. the
|
|
131
|
-
2. the
|
|
132
|
-
3. the
|
|
133
|
-
4. the
|
|
130
|
+
1. the run's variables, for a scenario with `runWith:` (see [Running a scenario several times](#running-a-scenario-several-times));
|
|
131
|
+
2. the scenario's `variables:` (see [Variables](#variables));
|
|
132
|
+
3. the configuration's `variables:`;
|
|
133
|
+
4. the `--parameters` file, when one is given;
|
|
134
|
+
5. the environment variable of that name.
|
|
134
135
|
|
|
135
136
|
A missing value is an error at its location; values never appear in messages. `.env` files are not loaded.
|
|
136
137
|
|
|
@@ -156,7 +157,7 @@ Keeping one file per environment, such as `parameters.json` and `parameters_cicd
|
|
|
156
157
|
|
|
157
158
|
### Variables
|
|
158
159
|
|
|
159
|
-
`variables:` is a map of names to strings, at the top of a scenario file or of the configuration. Names follow the same rule as references. One configuration can then serve scenarios that connect differently, for example as different users:
|
|
160
|
+
`variables:` is a map of names to strings, or to lists of strings (see [Running a scenario several times](#running-a-scenario-several-times)), at the top of a scenario file or of the configuration. Names follow the same rule as references. One configuration can then serve scenarios that connect differently, for example as different users:
|
|
160
161
|
|
|
161
162
|
```yaml
|
|
162
163
|
# eval-config.yml
|
|
@@ -244,6 +245,48 @@ An expectation whose capability is off is `ERROR`, not `PASS`. Tool expectations
|
|
|
244
245
|
|
|
245
246
|
The JSON Schemas in `schemas/` describe the configuration, scenarios and the JSON report, and can be used for editor completion.
|
|
246
247
|
|
|
248
|
+
### Running a scenario several times
|
|
249
|
+
|
|
250
|
+
`runWith:` runs one scenario once per entry, each with its own variables. Each run is a scenario of its own in every count, in the exit code and in the reports, named `Greeting [student]`. It has its own connection, so a run can change the token.
|
|
251
|
+
|
|
252
|
+
Rows set several variables per run. `name` is required and labels the run; every other key is a variable:
|
|
253
|
+
|
|
254
|
+
```yaml
|
|
255
|
+
version: 1
|
|
256
|
+
name: Greeting
|
|
257
|
+
runWith:
|
|
258
|
+
- name: instructor
|
|
259
|
+
token: ${instructor-token}
|
|
260
|
+
org: Acme University
|
|
261
|
+
- name: visitor # sets no token, so it connects without one
|
|
262
|
+
org: Acme University
|
|
263
|
+
turns:
|
|
264
|
+
- input: Hello, I am from ${org}.
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
`{ variable, from }` sets one variable from a list of variable or parameter names. Each run is named after its entry, never after the value, so a token never appears in a run's name:
|
|
268
|
+
|
|
269
|
+
```yaml
|
|
270
|
+
runWith:
|
|
271
|
+
variable: token
|
|
272
|
+
from: [instructor-token, student-token] # or a list variable: ${authenticated-tokens}
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
A list defined once in the configuration can be shared by many scenarios:
|
|
276
|
+
|
|
277
|
+
```yaml
|
|
278
|
+
# eval-config.yml
|
|
279
|
+
variables:
|
|
280
|
+
authenticated-tokens: [instructor-token, student-token, user-token]
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
- **Values:** a run's variables come first in [references](#references). A row value may use the scenario's variables, but a scenario variable never sees a run's value.
|
|
284
|
+
- **Defaults:** a scenario variable is the default for runs that do not set it, so one that every run overrides is allowed.
|
|
285
|
+
- **Lists:** a list variable can only be used as the whole value of `from`. Its items are names, and are not resolved themselves. Lists come only from `variables:` sections, because parameters and environment variables are strings.
|
|
286
|
+
- **Checks:** run names are unique within a scenario and match `[A-Za-z0-9][A-Za-z0-9_.-]*`. A run variable that neither the scenario nor the `websocket` section references is an error, as for scenario variables.
|
|
287
|
+
- **Errors:** a problem only some runs have names the run, for example `scenarios/order.yml:9:14 turns[0].input: run 'lost': ...`. One that every run has is reported once.
|
|
288
|
+
- **Reports:** the JSON report keeps each scenario's `name` and gives the run's name as `runWith`.
|
|
289
|
+
|
|
247
290
|
## Results
|
|
248
291
|
|
|
249
292
|
| Level | Statuses |
|
package/dist/index.cjs
CHANGED
|
@@ -344,6 +344,7 @@ var WebSocketConfig = class {
|
|
|
344
344
|
var ConversationSpec = class {
|
|
345
345
|
version;
|
|
346
346
|
name;
|
|
347
|
+
runName;
|
|
347
348
|
description;
|
|
348
349
|
turns;
|
|
349
350
|
conversation;
|
|
@@ -353,6 +354,7 @@ var ConversationSpec = class {
|
|
|
353
354
|
constructor(data) {
|
|
354
355
|
this.version = data?.version ?? 1;
|
|
355
356
|
this.name = data?.name ?? "";
|
|
357
|
+
this.runName = data?.runName ?? "";
|
|
356
358
|
this.description = data?.description ?? "";
|
|
357
359
|
this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
|
|
358
360
|
this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
|
|
@@ -474,7 +476,7 @@ var EvalConfig = class {
|
|
|
474
476
|
this.version = data?.version ?? 1;
|
|
475
477
|
this.configPath = data?.configPath ?? "";
|
|
476
478
|
this.parametersPath = data?.parametersPath ?? "";
|
|
477
|
-
this.variables =
|
|
479
|
+
this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
|
|
478
480
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
479
481
|
this.execution = new ExecutionConfig(data?.execution);
|
|
480
482
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -789,7 +791,10 @@ var eval_config_schema_default = {
|
|
|
789
791
|
"variables": {
|
|
790
792
|
"type": "object",
|
|
791
793
|
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
792
|
-
"additionalProperties": { "type": "string" }
|
|
794
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
795
|
+
"type": "array",
|
|
796
|
+
"items": { "type": "string" }
|
|
797
|
+
}] }
|
|
793
798
|
},
|
|
794
799
|
"websocket": {
|
|
795
800
|
"type": "object",
|
|
@@ -1022,7 +1027,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
1022
1027
|
return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
|
|
1023
1028
|
}
|
|
1024
1029
|
resolve(value, document, issues, secrets = [], options = {}) {
|
|
1025
|
-
return this.resolveValue(value, [], document, issues, secrets, options);
|
|
1030
|
+
return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
|
|
1026
1031
|
}
|
|
1027
1032
|
static hasReference(value) {
|
|
1028
1033
|
return new RegExp(REFERENCE.source).test(value);
|
|
@@ -1030,7 +1035,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
1030
1035
|
static isSecretName(name) {
|
|
1031
1036
|
return SECRET_NAME.test(name);
|
|
1032
1037
|
}
|
|
1033
|
-
|
|
1038
|
+
find(name) {
|
|
1034
1039
|
for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
|
|
1035
1040
|
return this.environment[name];
|
|
1036
1041
|
}
|
|
@@ -1045,7 +1050,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
1045
1050
|
resolveValue(value, path, document, issues, secrets, options) {
|
|
1046
1051
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
1047
1052
|
options.used?.add(name);
|
|
1048
|
-
const resolved = this.
|
|
1053
|
+
const resolved = this.find(name);
|
|
1049
1054
|
if (resolved === void 0) {
|
|
1050
1055
|
if (options.optional?.(path)) {
|
|
1051
1056
|
options.unresolved?.add(YamlDocument.formatPath(path));
|
|
@@ -1057,6 +1062,13 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
1057
1062
|
}));
|
|
1058
1063
|
return reference;
|
|
1059
1064
|
}
|
|
1065
|
+
if (Array.isArray(resolved)) {
|
|
1066
|
+
issues.push(new ValidationIssue({
|
|
1067
|
+
message: `variable '${name}' is a list, which can only be used as runWith.from`,
|
|
1068
|
+
location: document.locate(path)
|
|
1069
|
+
}));
|
|
1070
|
+
return reference;
|
|
1071
|
+
}
|
|
1060
1072
|
if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
|
|
1061
1073
|
return resolved;
|
|
1062
1074
|
});
|
|
@@ -1185,9 +1197,25 @@ var VariablesReader = class {
|
|
|
1185
1197
|
}));
|
|
1186
1198
|
continue;
|
|
1187
1199
|
}
|
|
1200
|
+
if (Array.isArray(value)) {
|
|
1201
|
+
const bad = value.findIndex((item) => typeof item !== "string");
|
|
1202
|
+
if (bad >= 0) {
|
|
1203
|
+
issues.push(new ValidationIssue({
|
|
1204
|
+
message: "must be string",
|
|
1205
|
+
location: document.locate([
|
|
1206
|
+
"variables",
|
|
1207
|
+
name,
|
|
1208
|
+
bad
|
|
1209
|
+
])
|
|
1210
|
+
}));
|
|
1211
|
+
continue;
|
|
1212
|
+
}
|
|
1213
|
+
variables[name] = [...value];
|
|
1214
|
+
continue;
|
|
1215
|
+
}
|
|
1188
1216
|
if (typeof value !== "string") {
|
|
1189
1217
|
issues.push(new ValidationIssue({
|
|
1190
|
-
message: "must be string",
|
|
1218
|
+
message: "must be string or a list of strings",
|
|
1191
1219
|
location: document.locate(["variables", name])
|
|
1192
1220
|
}));
|
|
1193
1221
|
continue;
|
|
@@ -1195,8 +1223,9 @@ var VariablesReader = class {
|
|
|
1195
1223
|
const before = issues.length;
|
|
1196
1224
|
const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
|
|
1197
1225
|
if (issues.length > before) continue;
|
|
1198
|
-
|
|
1199
|
-
|
|
1226
|
+
const text = resolved.variables[name];
|
|
1227
|
+
variables[name] = text;
|
|
1228
|
+
if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
|
|
1200
1229
|
}
|
|
1201
1230
|
return variables;
|
|
1202
1231
|
}
|
|
@@ -1501,7 +1530,44 @@ var conversation_schema_default = {
|
|
|
1501
1530
|
"variables": {
|
|
1502
1531
|
"type": "object",
|
|
1503
1532
|
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
1504
|
-
"additionalProperties": { "type": "string" }
|
|
1533
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
1534
|
+
"type": "array",
|
|
1535
|
+
"items": { "type": "string" }
|
|
1536
|
+
}] }
|
|
1537
|
+
},
|
|
1538
|
+
"runWith": {
|
|
1539
|
+
"description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
|
|
1540
|
+
"anyOf": [{
|
|
1541
|
+
"type": "array",
|
|
1542
|
+
"minItems": 1,
|
|
1543
|
+
"items": {
|
|
1544
|
+
"type": "object",
|
|
1545
|
+
"required": ["name"],
|
|
1546
|
+
"properties": { "name": {
|
|
1547
|
+
"type": "string",
|
|
1548
|
+
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
|
|
1549
|
+
} },
|
|
1550
|
+
"additionalProperties": { "type": "string" }
|
|
1551
|
+
}
|
|
1552
|
+
}, {
|
|
1553
|
+
"type": "object",
|
|
1554
|
+
"additionalProperties": false,
|
|
1555
|
+
"required": ["variable", "from"],
|
|
1556
|
+
"properties": {
|
|
1557
|
+
"variable": {
|
|
1558
|
+
"type": "string",
|
|
1559
|
+
"pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
|
|
1560
|
+
},
|
|
1561
|
+
"from": { "anyOf": [{
|
|
1562
|
+
"type": "array",
|
|
1563
|
+
"minItems": 1,
|
|
1564
|
+
"items": { "type": "string" }
|
|
1565
|
+
}, {
|
|
1566
|
+
"type": "string",
|
|
1567
|
+
"pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
|
|
1568
|
+
}] }
|
|
1569
|
+
}
|
|
1570
|
+
}]
|
|
1505
1571
|
},
|
|
1506
1572
|
"turns": {
|
|
1507
1573
|
"type": "array",
|
|
@@ -1660,6 +1726,221 @@ var ConversationValidator = class ConversationValidator {
|
|
|
1660
1726
|
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
1661
1727
|
}
|
|
1662
1728
|
};
|
|
1729
|
+
// src/scenarios/runWithReader.ts
|
|
1730
|
+
const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
|
|
1731
|
+
const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
|
|
1732
|
+
const FORMS = "must be a list of rows, or an object with variable and from";
|
|
1733
|
+
const SINGLE_RUN = {
|
|
1734
|
+
name: "",
|
|
1735
|
+
variables: {},
|
|
1736
|
+
keys: [],
|
|
1737
|
+
location: [],
|
|
1738
|
+
secrets: []
|
|
1739
|
+
};
|
|
1740
|
+
var RunWithReader = class RunWithReader {
|
|
1741
|
+
read(document, resolver, issues, used) {
|
|
1742
|
+
const raw = isRecord(document.value) ? document.value.runWith : void 0;
|
|
1743
|
+
if (raw === void 0) return void 0;
|
|
1744
|
+
if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
|
|
1745
|
+
if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
|
|
1746
|
+
issues.push(new ValidationIssue({
|
|
1747
|
+
message: FORMS,
|
|
1748
|
+
location: document.locate(["runWith"])
|
|
1749
|
+
}));
|
|
1750
|
+
return [];
|
|
1751
|
+
}
|
|
1752
|
+
rows(rows, document, resolver, issues, used) {
|
|
1753
|
+
if (rows.length === 0) {
|
|
1754
|
+
issues.push(new ValidationIssue({
|
|
1755
|
+
message: "must not be empty",
|
|
1756
|
+
location: document.locate(["runWith"])
|
|
1757
|
+
}));
|
|
1758
|
+
return [];
|
|
1759
|
+
}
|
|
1760
|
+
const names = /* @__PURE__ */ new Set();
|
|
1761
|
+
const runs = [];
|
|
1762
|
+
rows.forEach((row, index) => {
|
|
1763
|
+
const path = ["runWith", index];
|
|
1764
|
+
if (!isRecord(row)) {
|
|
1765
|
+
issues.push(new ValidationIssue({
|
|
1766
|
+
message: "must be object",
|
|
1767
|
+
location: document.locate(path)
|
|
1768
|
+
}));
|
|
1769
|
+
return;
|
|
1770
|
+
}
|
|
1771
|
+
const name = row.name;
|
|
1772
|
+
if (name === void 0) issues.push(new ValidationIssue({
|
|
1773
|
+
message: "missing required key 'name'",
|
|
1774
|
+
location: document.locate(path)
|
|
1775
|
+
}));
|
|
1776
|
+
else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
|
|
1777
|
+
message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
|
|
1778
|
+
location: document.locate([...path, "name"])
|
|
1779
|
+
}));
|
|
1780
|
+
else if (names.has(name)) issues.push(new ValidationIssue({
|
|
1781
|
+
message: `duplicate run name '${name}'`,
|
|
1782
|
+
location: document.locate([...path, "name"])
|
|
1783
|
+
}));
|
|
1784
|
+
else names.add(name);
|
|
1785
|
+
const run = {
|
|
1786
|
+
name: typeof name === "string" ? name : "",
|
|
1787
|
+
variables: {},
|
|
1788
|
+
keys: [],
|
|
1789
|
+
location: path,
|
|
1790
|
+
secrets: []
|
|
1791
|
+
};
|
|
1792
|
+
for (const [key, value] of Object.entries(row)) {
|
|
1793
|
+
if (key === "name") continue;
|
|
1794
|
+
if (!VARIABLE_NAME.test(key)) {
|
|
1795
|
+
issues.push(new ValidationIssue({
|
|
1796
|
+
message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1797
|
+
location: document.locateKey(path, key)
|
|
1798
|
+
}));
|
|
1799
|
+
continue;
|
|
1800
|
+
}
|
|
1801
|
+
if (typeof value !== "string") {
|
|
1802
|
+
issues.push(new ValidationIssue({
|
|
1803
|
+
message: "must be string",
|
|
1804
|
+
location: document.locate([...path, key])
|
|
1805
|
+
}));
|
|
1806
|
+
continue;
|
|
1807
|
+
}
|
|
1808
|
+
const resolved = resolver.resolve(value, document, issues, run.secrets, {
|
|
1809
|
+
used,
|
|
1810
|
+
at: [...path, key]
|
|
1811
|
+
});
|
|
1812
|
+
RunWithReader.set(run, key, resolved, document.locateKey(path, key));
|
|
1813
|
+
}
|
|
1814
|
+
runs.push(run);
|
|
1815
|
+
});
|
|
1816
|
+
return runs;
|
|
1817
|
+
}
|
|
1818
|
+
list(form, document, resolver, issues, used) {
|
|
1819
|
+
const before = issues.length;
|
|
1820
|
+
for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
|
|
1821
|
+
message: `unknown key '${key}'`,
|
|
1822
|
+
location: document.locateKey(["runWith"], key)
|
|
1823
|
+
}));
|
|
1824
|
+
const variable = form.variable;
|
|
1825
|
+
if (variable === void 0) issues.push(new ValidationIssue({
|
|
1826
|
+
message: "missing required key 'variable'",
|
|
1827
|
+
location: document.locate(["runWith"])
|
|
1828
|
+
}));
|
|
1829
|
+
else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
|
|
1830
|
+
message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
|
|
1831
|
+
location: document.locate(["runWith", "variable"])
|
|
1832
|
+
}));
|
|
1833
|
+
const entries = this.entries(form.from, document, resolver, issues, used);
|
|
1834
|
+
if (issues.length > before || entries === void 0) return [];
|
|
1835
|
+
const name = variable;
|
|
1836
|
+
const keyLocation = document.locate(["runWith", "variable"]);
|
|
1837
|
+
const names = /* @__PURE__ */ new Set();
|
|
1838
|
+
const runs = [];
|
|
1839
|
+
for (const { entry, path } of entries) {
|
|
1840
|
+
if (!VARIABLE_NAME.test(entry)) {
|
|
1841
|
+
issues.push(new ValidationIssue({
|
|
1842
|
+
message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1843
|
+
location: document.locate(path)
|
|
1844
|
+
}));
|
|
1845
|
+
continue;
|
|
1846
|
+
}
|
|
1847
|
+
if (names.has(entry)) {
|
|
1848
|
+
issues.push(new ValidationIssue({
|
|
1849
|
+
message: `duplicate run name '${entry}'`,
|
|
1850
|
+
location: document.locate(path)
|
|
1851
|
+
}));
|
|
1852
|
+
continue;
|
|
1853
|
+
}
|
|
1854
|
+
names.add(entry);
|
|
1855
|
+
const run = {
|
|
1856
|
+
name: entry,
|
|
1857
|
+
variables: {},
|
|
1858
|
+
keys: [],
|
|
1859
|
+
location: path,
|
|
1860
|
+
secrets: []
|
|
1861
|
+
};
|
|
1862
|
+
const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
|
|
1863
|
+
used,
|
|
1864
|
+
at: path
|
|
1865
|
+
});
|
|
1866
|
+
RunWithReader.set(run, name, value, keyLocation);
|
|
1867
|
+
runs.push(run);
|
|
1868
|
+
}
|
|
1869
|
+
return runs;
|
|
1870
|
+
}
|
|
1871
|
+
entries(from, document, resolver, issues, used) {
|
|
1872
|
+
const path = ["runWith", "from"];
|
|
1873
|
+
const location = document.locate(path);
|
|
1874
|
+
if (from === void 0) {
|
|
1875
|
+
issues.push(new ValidationIssue({
|
|
1876
|
+
message: "missing required key 'from'",
|
|
1877
|
+
location: document.locate(["runWith"])
|
|
1878
|
+
}));
|
|
1879
|
+
return;
|
|
1880
|
+
}
|
|
1881
|
+
let list;
|
|
1882
|
+
let at;
|
|
1883
|
+
if (Array.isArray(from)) {
|
|
1884
|
+
list = from;
|
|
1885
|
+
at = (index) => [...path, index];
|
|
1886
|
+
} else {
|
|
1887
|
+
const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
|
|
1888
|
+
if (reference === null) {
|
|
1889
|
+
issues.push(new ValidationIssue({
|
|
1890
|
+
message: "must be a list, or a list variable such as ${tokens}",
|
|
1891
|
+
location
|
|
1892
|
+
}));
|
|
1893
|
+
return;
|
|
1894
|
+
}
|
|
1895
|
+
const name = reference[1];
|
|
1896
|
+
const found = resolver.find(name);
|
|
1897
|
+
used.add(name);
|
|
1898
|
+
if (found === void 0) {
|
|
1899
|
+
issues.push(new ValidationIssue({
|
|
1900
|
+
message: `list variable '${name}' is not set`,
|
|
1901
|
+
location
|
|
1902
|
+
}));
|
|
1903
|
+
return;
|
|
1904
|
+
}
|
|
1905
|
+
if (!Array.isArray(found)) {
|
|
1906
|
+
issues.push(new ValidationIssue({
|
|
1907
|
+
message: `variable '${name}' is not a list`,
|
|
1908
|
+
location
|
|
1909
|
+
}));
|
|
1910
|
+
return;
|
|
1911
|
+
}
|
|
1912
|
+
list = found;
|
|
1913
|
+
at = () => path;
|
|
1914
|
+
}
|
|
1915
|
+
if (list.length === 0) {
|
|
1916
|
+
issues.push(new ValidationIssue({
|
|
1917
|
+
message: "must not be empty",
|
|
1918
|
+
location
|
|
1919
|
+
}));
|
|
1920
|
+
return;
|
|
1921
|
+
}
|
|
1922
|
+
const bad = list.findIndex((item) => typeof item !== "string");
|
|
1923
|
+
if (bad >= 0) {
|
|
1924
|
+
issues.push(new ValidationIssue({
|
|
1925
|
+
message: "must be string",
|
|
1926
|
+
location: document.locate(at(bad))
|
|
1927
|
+
}));
|
|
1928
|
+
return;
|
|
1929
|
+
}
|
|
1930
|
+
return list.map((entry, index) => ({
|
|
1931
|
+
entry,
|
|
1932
|
+
path: at(index)
|
|
1933
|
+
}));
|
|
1934
|
+
}
|
|
1935
|
+
static set(run, name, value, location) {
|
|
1936
|
+
run.variables[name] = value;
|
|
1937
|
+
run.keys.push({
|
|
1938
|
+
name,
|
|
1939
|
+
location
|
|
1940
|
+
});
|
|
1941
|
+
if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
|
|
1942
|
+
}
|
|
1943
|
+
};
|
|
1663
1944
|
// src/scenarios/scenarioLoader.ts
|
|
1664
1945
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
1665
1946
|
var ScenarioLoader = class ScenarioLoader {
|
|
@@ -1668,6 +1949,7 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1668
1949
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
1669
1950
|
conversationValidator = new ConversationValidator();
|
|
1670
1951
|
variablesReader = new VariablesReader();
|
|
1952
|
+
runWithReader = new RunWithReader();
|
|
1671
1953
|
webSocketResolver = new WebSocketResolver();
|
|
1672
1954
|
environmentResolver;
|
|
1673
1955
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
@@ -1683,45 +1965,59 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1683
1965
|
for (const file of files) {
|
|
1684
1966
|
const document = this.documentLoader.load(file, (0, node_path.relative)(this.cwd, file));
|
|
1685
1967
|
const fileIssues = [...document.errors];
|
|
1686
|
-
const
|
|
1968
|
+
const fileSecrets = [];
|
|
1687
1969
|
const used = /* @__PURE__ */ new Set();
|
|
1688
|
-
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues,
|
|
1689
|
-
const
|
|
1690
|
-
|
|
1691
|
-
if (fileIssues.length
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
...resolver.resolve(rest, document, fileIssues, secrets, { used }),
|
|
1695
|
-
...raw === void 0 ? {} : { variables }
|
|
1696
|
-
};
|
|
1970
|
+
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
|
|
1971
|
+
const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
|
|
1972
|
+
const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
|
|
1973
|
+
if (fileIssues.length > 0) {
|
|
1974
|
+
issues.push(...fileIssues);
|
|
1975
|
+
continue;
|
|
1697
1976
|
}
|
|
1698
|
-
|
|
1699
|
-
const
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1977
|
+
const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
|
|
1978
|
+
const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
|
|
1979
|
+
for (const result of results) for (const name of result.used) used.add(name);
|
|
1980
|
+
fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
|
|
1981
|
+
run: result.run,
|
|
1982
|
+
issues: result.issues
|
|
1983
|
+
})), (issue, run) => run === void 0 ? issue : new ValidationIssue({
|
|
1984
|
+
message: `run '${run.name}': ${issue.message}`,
|
|
1985
|
+
location: issue.location
|
|
1986
|
+
})));
|
|
1987
|
+
const attempted = results.filter((result) => result.connectionChecked);
|
|
1988
|
+
if (attempted.length > 0) connectionIssues.push({
|
|
1989
|
+
file: document,
|
|
1990
|
+
runs: attempted.map((result) => ({
|
|
1991
|
+
run: result.run,
|
|
1992
|
+
issues: result.websocketIssues
|
|
1993
|
+
}))
|
|
1994
|
+
});
|
|
1995
|
+
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
|
|
1996
|
+
if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
|
|
1997
|
+
const websocket = result.websocket;
|
|
1998
|
+
specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
1708
1999
|
}
|
|
1709
|
-
if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
|
|
1710
|
-
if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
|
|
1711
|
-
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
|
|
1712
|
-
if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
1713
2000
|
else issues.push(...fileIssues);
|
|
1714
2001
|
}
|
|
1715
|
-
const shared = ScenarioLoader.shared(connectionIssues.
|
|
2002
|
+
const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
|
|
1716
2003
|
if (shared.length > 0) throw new EvalError({
|
|
1717
2004
|
category: ErrorCategory.Configuration,
|
|
1718
2005
|
message: "invalid configuration",
|
|
1719
2006
|
issues: shared
|
|
1720
2007
|
});
|
|
1721
|
-
for (const entry of connectionIssues)
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
2008
|
+
for (const entry of connectionIssues) {
|
|
2009
|
+
const fileLocation = ScenarioLoader.variablesLocation(entry.file);
|
|
2010
|
+
issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
|
|
2011
|
+
message: issue.toString(),
|
|
2012
|
+
location: fileLocation
|
|
2013
|
+
}) : new ValidationIssue({
|
|
2014
|
+
message: `run '${run.name}': ${issue.toString()}`,
|
|
2015
|
+
location: new SourceLocation({
|
|
2016
|
+
...entry.file.locate(run.location),
|
|
2017
|
+
path: ""
|
|
2018
|
+
})
|
|
2019
|
+
})));
|
|
2020
|
+
}
|
|
1725
2021
|
if (issues.length > 0) throw new EvalError({
|
|
1726
2022
|
category: ErrorCategory.Specification,
|
|
1727
2023
|
message: "invalid scenario specification",
|
|
@@ -1729,6 +2025,43 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1729
2025
|
});
|
|
1730
2026
|
return specs;
|
|
1731
2027
|
}
|
|
2028
|
+
resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
|
|
2029
|
+
const issues = [];
|
|
2030
|
+
const used = /* @__PURE__ */ new Set();
|
|
2031
|
+
const resolver = scenarioResolver.withLevels([run.variables]);
|
|
2032
|
+
const value = {
|
|
2033
|
+
...resolver.resolve(rest, document, issues, secrets, { used }),
|
|
2034
|
+
...variables === void 0 ? {} : { variables }
|
|
2035
|
+
};
|
|
2036
|
+
const websocketIssues = [];
|
|
2037
|
+
let websocket;
|
|
2038
|
+
let connectionChecked = false;
|
|
2039
|
+
if (issues.length === 0 && configDocument !== void 0) {
|
|
2040
|
+
const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
|
|
2041
|
+
if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
|
|
2042
|
+
if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
|
|
2043
|
+
connectionChecked = true;
|
|
2044
|
+
}
|
|
2045
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
|
|
2046
|
+
if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
|
|
2047
|
+
return {
|
|
2048
|
+
run,
|
|
2049
|
+
value,
|
|
2050
|
+
used,
|
|
2051
|
+
secrets,
|
|
2052
|
+
issues,
|
|
2053
|
+
websocket,
|
|
2054
|
+
websocketIssues,
|
|
2055
|
+
connectionChecked
|
|
2056
|
+
};
|
|
2057
|
+
}
|
|
2058
|
+
static byRun(runs, place) {
|
|
2059
|
+
const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
|
|
2060
|
+
const sharedTexts = new Set(shared.map((issue) => issue.toString()));
|
|
2061
|
+
const result = shared.map((issue) => place(issue, void 0));
|
|
2062
|
+
for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
|
|
2063
|
+
return result;
|
|
2064
|
+
}
|
|
1732
2065
|
static shared(lists) {
|
|
1733
2066
|
if (lists.length === 0) return [];
|
|
1734
2067
|
const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
|
|
@@ -1740,11 +2073,22 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1740
2073
|
path: ""
|
|
1741
2074
|
});
|
|
1742
2075
|
}
|
|
1743
|
-
static unusedVariables(variables, used, document) {
|
|
1744
|
-
|
|
1745
|
-
|
|
2076
|
+
static unusedVariables(variables, used, results, document) {
|
|
2077
|
+
const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
|
|
2078
|
+
const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
|
|
2079
|
+
message: message(name),
|
|
1746
2080
|
location: document.locateKey(["variables"], name)
|
|
1747
2081
|
}));
|
|
2082
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2083
|
+
for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
|
|
2084
|
+
const issue = new ValidationIssue({
|
|
2085
|
+
message: message(key.name),
|
|
2086
|
+
location: key.location
|
|
2087
|
+
});
|
|
2088
|
+
if (!seen.has(issue.toString())) issues.push(issue);
|
|
2089
|
+
seen.add(issue.toString());
|
|
2090
|
+
}
|
|
2091
|
+
return issues;
|
|
1748
2092
|
}
|
|
1749
2093
|
discover(target, config) {
|
|
1750
2094
|
if (!(0, node_fs.existsSync)(target)) throw new EvalError({
|
|
@@ -1770,13 +2114,14 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1770
2114
|
}
|
|
1771
2115
|
return files;
|
|
1772
2116
|
}
|
|
1773
|
-
build(value, document, websocket, secrets) {
|
|
2117
|
+
build(value, document, runName, websocket, secrets) {
|
|
1774
2118
|
const scenario = value;
|
|
1775
2119
|
const turns = scenario.turns;
|
|
1776
2120
|
const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
|
|
1777
2121
|
return new ConversationSpec({
|
|
1778
2122
|
version: scenario.version,
|
|
1779
2123
|
name: scenario.name,
|
|
2124
|
+
runName,
|
|
1780
2125
|
description: typeof scenario.description === "string" ? scenario.description : "",
|
|
1781
2126
|
turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
|
|
1782
2127
|
conversation: conversation ? new ConversationExpectation({
|
|
@@ -3244,7 +3589,7 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3244
3589
|
scenario(result, repeat, tokenUsageCapability) {
|
|
3245
3590
|
const spec = result.spec;
|
|
3246
3591
|
const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
|
|
3247
|
-
const line = `${this.status(result.status, STATUS_WIDTH)} ${spec
|
|
3592
|
+
const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
|
|
3248
3593
|
if (!this.verbose) return line;
|
|
3249
3594
|
if (result.attempts.length === 0) return `${line}\n`;
|
|
3250
3595
|
const websocket = spec.websocket;
|
|
@@ -3257,6 +3602,9 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3257
3602
|
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3258
3603
|
return line + lines.join("\n");
|
|
3259
3604
|
}
|
|
3605
|
+
static title(spec) {
|
|
3606
|
+
return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
|
|
3607
|
+
}
|
|
3260
3608
|
summary(run, exitCode, reportPaths = []) {
|
|
3261
3609
|
const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
|
|
3262
3610
|
const lines = this.verbose ? [] : [""];
|
|
@@ -3288,7 +3636,7 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3288
3636
|
}
|
|
3289
3637
|
problems(result) {
|
|
3290
3638
|
const spec = result.spec;
|
|
3291
|
-
const lines = [`${this.status(result.status, 6)} ${spec
|
|
3639
|
+
const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
|
|
3292
3640
|
for (const attempt of result.attempts) {
|
|
3293
3641
|
const execution = attempt.execution;
|
|
3294
3642
|
if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
|
|
@@ -3466,6 +3814,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3466
3814
|
const capability = websocket.protocol.capabilities.tokenUsage;
|
|
3467
3815
|
return {
|
|
3468
3816
|
name: result.spec.name,
|
|
3817
|
+
runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
|
|
3469
3818
|
file: result.spec.location.file,
|
|
3470
3819
|
status: result.status,
|
|
3471
3820
|
connection: {
|