@mappies/miniwat-eval 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -127,10 +127,11 @@ Unknown keys are errors. The `evaluator` section is required, but its provider p
127
127
 
128
128
  `${name}` in any string value, in the configuration or a scenario, is replaced after the YAML is parsed, so a value containing YAML syntax stays one string. The value comes from the first of these that has it:
129
129
 
130
- 1. the scenario's `variables:` (see [Variables](#variables));
131
- 2. the configuration's `variables:`;
132
- 3. the `--parameters` file, when one is given;
133
- 4. the environment variable of that name.
130
+ 1. the run's variables, for a scenario with `runWith:` (see [Running a scenario several times](#running-a-scenario-several-times));
131
+ 2. the scenario's `variables:` (see [Variables](#variables));
132
+ 3. the configuration's `variables:`;
133
+ 4. the `--parameters` file, when one is given;
134
+ 5. the environment variable of that name.
134
135
 
135
136
  A missing value is an error at its location; values never appear in messages. `.env` files are not loaded.
136
137
 
@@ -156,7 +157,7 @@ Keeping one file per environment, such as `parameters.json` and `parameters_cicd
156
157
 
157
158
  ### Variables
158
159
 
159
- `variables:` is a map of names to strings, at the top of a scenario file or of the configuration. Names follow the same rule as references. One configuration can then serve scenarios that connect differently, for example as different users:
160
+ `variables:` is a map of names to strings, or to lists of strings (see [Running a scenario several times](#running-a-scenario-several-times)), at the top of a scenario file or of the configuration. Names follow the same rule as references. One configuration can then serve scenarios that connect differently, for example as different users:
160
161
 
161
162
  ```yaml
162
163
  # eval-config.yml
@@ -244,6 +245,48 @@ An expectation whose capability is off is `ERROR`, not `PASS`. Tool expectations
244
245
 
245
246
  The JSON Schemas in `schemas/` describe the configuration, scenarios and the JSON report, and can be used for editor completion.
246
247
 
248
+ ### Running a scenario several times
249
+
250
+ `runWith:` runs one scenario once per entry, each with its own variables. Each run is a scenario of its own in every count, in the exit code and in the reports, named `Greeting [student]`. It has its own connection, so a run can change the token.
251
+
252
+ Rows set several variables per run. `name` is required and labels the run; every other key is a variable:
253
+
254
+ ```yaml
255
+ version: 1
256
+ name: Greeting
257
+ runWith:
258
+ - name: instructor
259
+ token: ${instructor-token}
260
+ org: Acme University
261
+ - name: visitor # sets no token, so it connects without one
262
+ org: Acme University
263
+ turns:
264
+ - input: Hello, I am from ${org}.
265
+ ```
266
+
267
+ `{ variable, from }` sets one variable from a list of variable or parameter names. Each run is named after its entry, never after the value, so a token never appears in a run's name:
268
+
269
+ ```yaml
270
+ runWith:
271
+ variable: token
272
+ from: [instructor-token, student-token] # or a list variable: ${authenticated-tokens}
273
+ ```
274
+
275
+ A list defined once in the configuration can be shared by many scenarios:
276
+
277
+ ```yaml
278
+ # eval-config.yml
279
+ variables:
280
+ authenticated-tokens: [instructor-token, student-token, user-token]
281
+ ```
282
+
283
+ - **Values:** a run's variables come first in [references](#references). A row value may use the scenario's variables, but a scenario variable never sees a run's value.
284
+ - **Defaults:** a scenario variable is the default for runs that do not set it, so one that every run overrides is allowed.
285
+ - **Lists:** a list variable can only be used as the whole value of `from`. Its items are names, and are not resolved themselves. Lists come only from `variables:` sections, because parameters and environment variables are strings.
286
+ - **Checks:** run names are unique within a scenario and match `[A-Za-z0-9][A-Za-z0-9_.-]*`. A run variable that neither the scenario nor the `websocket` section references is an error, as for scenario variables.
287
+ - **Errors:** a problem only some runs have names the run, for example `scenarios/order.yml:9:14 turns[0].input: run 'lost': ...`. One that every run has is reported once.
288
+ - **Reports:** the JSON report keeps each scenario's `name` and gives the run's name as `runWith`.
289
+
247
290
  ## Results
248
291
 
249
292
  | Level | Statuses |
package/dist/index.cjs CHANGED
@@ -344,6 +344,7 @@ var WebSocketConfig = class {
344
344
  var ConversationSpec = class {
345
345
  version;
346
346
  name;
347
+ runName;
347
348
  description;
348
349
  turns;
349
350
  conversation;
@@ -353,6 +354,7 @@ var ConversationSpec = class {
353
354
  constructor(data) {
354
355
  this.version = data?.version ?? 1;
355
356
  this.name = data?.name ?? "";
357
+ this.runName = data?.runName ?? "";
356
358
  this.description = data?.description ?? "";
357
359
  this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
358
360
  this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
@@ -474,7 +476,7 @@ var EvalConfig = class {
474
476
  this.version = data?.version ?? 1;
475
477
  this.configPath = data?.configPath ?? "";
476
478
  this.parametersPath = data?.parametersPath ?? "";
477
- this.variables = { ...data?.variables ?? {} };
479
+ this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
478
480
  this.websocket = new WebSocketConfig(data?.websocket);
479
481
  this.execution = new ExecutionConfig(data?.execution);
480
482
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -789,7 +791,10 @@ var eval_config_schema_default = {
789
791
  "variables": {
790
792
  "type": "object",
791
793
  "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
792
- "additionalProperties": { "type": "string" }
794
+ "additionalProperties": { "anyOf": [{ "type": "string" }, {
795
+ "type": "array",
796
+ "items": { "type": "string" }
797
+ }] }
793
798
  },
794
799
  "websocket": {
795
800
  "type": "object",
@@ -1022,7 +1027,7 @@ var EnvironmentResolver = class EnvironmentResolver {
1022
1027
  return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
1023
1028
  }
1024
1029
  resolve(value, document, issues, secrets = [], options = {}) {
1025
- return this.resolveValue(value, [], document, issues, secrets, options);
1030
+ return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
1026
1031
  }
1027
1032
  static hasReference(value) {
1028
1033
  return new RegExp(REFERENCE.source).test(value);
@@ -1030,7 +1035,7 @@ var EnvironmentResolver = class EnvironmentResolver {
1030
1035
  static isSecretName(name) {
1031
1036
  return SECRET_NAME.test(name);
1032
1037
  }
1033
- lookup(name) {
1038
+ find(name) {
1034
1039
  for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
1035
1040
  return this.environment[name];
1036
1041
  }
@@ -1045,7 +1050,7 @@ var EnvironmentResolver = class EnvironmentResolver {
1045
1050
  resolveValue(value, path, document, issues, secrets, options) {
1046
1051
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
1047
1052
  options.used?.add(name);
1048
- const resolved = this.lookup(name);
1053
+ const resolved = this.find(name);
1049
1054
  if (resolved === void 0) {
1050
1055
  if (options.optional?.(path)) {
1051
1056
  options.unresolved?.add(YamlDocument.formatPath(path));
@@ -1057,6 +1062,13 @@ var EnvironmentResolver = class EnvironmentResolver {
1057
1062
  }));
1058
1063
  return reference;
1059
1064
  }
1065
+ if (Array.isArray(resolved)) {
1066
+ issues.push(new ValidationIssue({
1067
+ message: `variable '${name}' is a list, which can only be used as runWith.from`,
1068
+ location: document.locate(path)
1069
+ }));
1070
+ return reference;
1071
+ }
1060
1072
  if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
1061
1073
  return resolved;
1062
1074
  });
@@ -1185,9 +1197,25 @@ var VariablesReader = class {
1185
1197
  }));
1186
1198
  continue;
1187
1199
  }
1200
+ if (Array.isArray(value)) {
1201
+ const bad = value.findIndex((item) => typeof item !== "string");
1202
+ if (bad >= 0) {
1203
+ issues.push(new ValidationIssue({
1204
+ message: "must be string",
1205
+ location: document.locate([
1206
+ "variables",
1207
+ name,
1208
+ bad
1209
+ ])
1210
+ }));
1211
+ continue;
1212
+ }
1213
+ variables[name] = [...value];
1214
+ continue;
1215
+ }
1188
1216
  if (typeof value !== "string") {
1189
1217
  issues.push(new ValidationIssue({
1190
- message: "must be string",
1218
+ message: "must be string or a list of strings",
1191
1219
  location: document.locate(["variables", name])
1192
1220
  }));
1193
1221
  continue;
@@ -1195,8 +1223,9 @@ var VariablesReader = class {
1195
1223
  const before = issues.length;
1196
1224
  const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
1197
1225
  if (issues.length > before) continue;
1198
- variables[name] = resolved.variables[name];
1199
- if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
1226
+ const text = resolved.variables[name];
1227
+ variables[name] = text;
1228
+ if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
1200
1229
  }
1201
1230
  return variables;
1202
1231
  }
@@ -1501,7 +1530,44 @@ var conversation_schema_default = {
1501
1530
  "variables": {
1502
1531
  "type": "object",
1503
1532
  "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
1504
- "additionalProperties": { "type": "string" }
1533
+ "additionalProperties": { "anyOf": [{ "type": "string" }, {
1534
+ "type": "array",
1535
+ "items": { "type": "string" }
1536
+ }] }
1537
+ },
1538
+ "runWith": {
1539
+ "description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
1540
+ "anyOf": [{
1541
+ "type": "array",
1542
+ "minItems": 1,
1543
+ "items": {
1544
+ "type": "object",
1545
+ "required": ["name"],
1546
+ "properties": { "name": {
1547
+ "type": "string",
1548
+ "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
1549
+ } },
1550
+ "additionalProperties": { "type": "string" }
1551
+ }
1552
+ }, {
1553
+ "type": "object",
1554
+ "additionalProperties": false,
1555
+ "required": ["variable", "from"],
1556
+ "properties": {
1557
+ "variable": {
1558
+ "type": "string",
1559
+ "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
1560
+ },
1561
+ "from": { "anyOf": [{
1562
+ "type": "array",
1563
+ "minItems": 1,
1564
+ "items": { "type": "string" }
1565
+ }, {
1566
+ "type": "string",
1567
+ "pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
1568
+ }] }
1569
+ }
1570
+ }]
1505
1571
  },
1506
1572
  "turns": {
1507
1573
  "type": "array",
@@ -1660,6 +1726,221 @@ var ConversationValidator = class ConversationValidator {
1660
1726
  return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
1661
1727
  }
1662
1728
  };
1729
+ // src/scenarios/runWithReader.ts
1730
+ const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
1731
+ const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
1732
+ const FORMS = "must be a list of rows, or an object with variable and from";
1733
+ const SINGLE_RUN = {
1734
+ name: "",
1735
+ variables: {},
1736
+ keys: [],
1737
+ location: [],
1738
+ secrets: []
1739
+ };
1740
+ var RunWithReader = class RunWithReader {
1741
+ read(document, resolver, issues, used) {
1742
+ const raw = isRecord(document.value) ? document.value.runWith : void 0;
1743
+ if (raw === void 0) return void 0;
1744
+ if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
1745
+ if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
1746
+ issues.push(new ValidationIssue({
1747
+ message: FORMS,
1748
+ location: document.locate(["runWith"])
1749
+ }));
1750
+ return [];
1751
+ }
1752
+ rows(rows, document, resolver, issues, used) {
1753
+ if (rows.length === 0) {
1754
+ issues.push(new ValidationIssue({
1755
+ message: "must not be empty",
1756
+ location: document.locate(["runWith"])
1757
+ }));
1758
+ return [];
1759
+ }
1760
+ const names = /* @__PURE__ */ new Set();
1761
+ const runs = [];
1762
+ rows.forEach((row, index) => {
1763
+ const path = ["runWith", index];
1764
+ if (!isRecord(row)) {
1765
+ issues.push(new ValidationIssue({
1766
+ message: "must be object",
1767
+ location: document.locate(path)
1768
+ }));
1769
+ return;
1770
+ }
1771
+ const name = row.name;
1772
+ if (name === void 0) issues.push(new ValidationIssue({
1773
+ message: "missing required key 'name'",
1774
+ location: document.locate(path)
1775
+ }));
1776
+ else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
1777
+ message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
1778
+ location: document.locate([...path, "name"])
1779
+ }));
1780
+ else if (names.has(name)) issues.push(new ValidationIssue({
1781
+ message: `duplicate run name '${name}'`,
1782
+ location: document.locate([...path, "name"])
1783
+ }));
1784
+ else names.add(name);
1785
+ const run = {
1786
+ name: typeof name === "string" ? name : "",
1787
+ variables: {},
1788
+ keys: [],
1789
+ location: path,
1790
+ secrets: []
1791
+ };
1792
+ for (const [key, value] of Object.entries(row)) {
1793
+ if (key === "name") continue;
1794
+ if (!VARIABLE_NAME.test(key)) {
1795
+ issues.push(new ValidationIssue({
1796
+ message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
1797
+ location: document.locateKey(path, key)
1798
+ }));
1799
+ continue;
1800
+ }
1801
+ if (typeof value !== "string") {
1802
+ issues.push(new ValidationIssue({
1803
+ message: "must be string",
1804
+ location: document.locate([...path, key])
1805
+ }));
1806
+ continue;
1807
+ }
1808
+ const resolved = resolver.resolve(value, document, issues, run.secrets, {
1809
+ used,
1810
+ at: [...path, key]
1811
+ });
1812
+ RunWithReader.set(run, key, resolved, document.locateKey(path, key));
1813
+ }
1814
+ runs.push(run);
1815
+ });
1816
+ return runs;
1817
+ }
1818
+ list(form, document, resolver, issues, used) {
1819
+ const before = issues.length;
1820
+ for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
1821
+ message: `unknown key '${key}'`,
1822
+ location: document.locateKey(["runWith"], key)
1823
+ }));
1824
+ const variable = form.variable;
1825
+ if (variable === void 0) issues.push(new ValidationIssue({
1826
+ message: "missing required key 'variable'",
1827
+ location: document.locate(["runWith"])
1828
+ }));
1829
+ else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
1830
+ message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
1831
+ location: document.locate(["runWith", "variable"])
1832
+ }));
1833
+ const entries = this.entries(form.from, document, resolver, issues, used);
1834
+ if (issues.length > before || entries === void 0) return [];
1835
+ const name = variable;
1836
+ const keyLocation = document.locate(["runWith", "variable"]);
1837
+ const names = /* @__PURE__ */ new Set();
1838
+ const runs = [];
1839
+ for (const { entry, path } of entries) {
1840
+ if (!VARIABLE_NAME.test(entry)) {
1841
+ issues.push(new ValidationIssue({
1842
+ message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
1843
+ location: document.locate(path)
1844
+ }));
1845
+ continue;
1846
+ }
1847
+ if (names.has(entry)) {
1848
+ issues.push(new ValidationIssue({
1849
+ message: `duplicate run name '${entry}'`,
1850
+ location: document.locate(path)
1851
+ }));
1852
+ continue;
1853
+ }
1854
+ names.add(entry);
1855
+ const run = {
1856
+ name: entry,
1857
+ variables: {},
1858
+ keys: [],
1859
+ location: path,
1860
+ secrets: []
1861
+ };
1862
+ const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
1863
+ used,
1864
+ at: path
1865
+ });
1866
+ RunWithReader.set(run, name, value, keyLocation);
1867
+ runs.push(run);
1868
+ }
1869
+ return runs;
1870
+ }
1871
+ entries(from, document, resolver, issues, used) {
1872
+ const path = ["runWith", "from"];
1873
+ const location = document.locate(path);
1874
+ if (from === void 0) {
1875
+ issues.push(new ValidationIssue({
1876
+ message: "missing required key 'from'",
1877
+ location: document.locate(["runWith"])
1878
+ }));
1879
+ return;
1880
+ }
1881
+ let list;
1882
+ let at;
1883
+ if (Array.isArray(from)) {
1884
+ list = from;
1885
+ at = (index) => [...path, index];
1886
+ } else {
1887
+ const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
1888
+ if (reference === null) {
1889
+ issues.push(new ValidationIssue({
1890
+ message: "must be a list, or a list variable such as ${tokens}",
1891
+ location
1892
+ }));
1893
+ return;
1894
+ }
1895
+ const name = reference[1];
1896
+ const found = resolver.find(name);
1897
+ used.add(name);
1898
+ if (found === void 0) {
1899
+ issues.push(new ValidationIssue({
1900
+ message: `list variable '${name}' is not set`,
1901
+ location
1902
+ }));
1903
+ return;
1904
+ }
1905
+ if (!Array.isArray(found)) {
1906
+ issues.push(new ValidationIssue({
1907
+ message: `variable '${name}' is not a list`,
1908
+ location
1909
+ }));
1910
+ return;
1911
+ }
1912
+ list = found;
1913
+ at = () => path;
1914
+ }
1915
+ if (list.length === 0) {
1916
+ issues.push(new ValidationIssue({
1917
+ message: "must not be empty",
1918
+ location
1919
+ }));
1920
+ return;
1921
+ }
1922
+ const bad = list.findIndex((item) => typeof item !== "string");
1923
+ if (bad >= 0) {
1924
+ issues.push(new ValidationIssue({
1925
+ message: "must be string",
1926
+ location: document.locate(at(bad))
1927
+ }));
1928
+ return;
1929
+ }
1930
+ return list.map((entry, index) => ({
1931
+ entry,
1932
+ path: at(index)
1933
+ }));
1934
+ }
1935
+ static set(run, name, value, location) {
1936
+ run.variables[name] = value;
1937
+ run.keys.push({
1938
+ name,
1939
+ location
1940
+ });
1941
+ if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
1942
+ }
1943
+ };
1663
1944
  // src/scenarios/scenarioLoader.ts
1664
1945
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
1665
1946
  var ScenarioLoader = class ScenarioLoader {
@@ -1668,6 +1949,7 @@ var ScenarioLoader = class ScenarioLoader {
1668
1949
  schemaValidator = new SchemaValidator(conversation_schema_default);
1669
1950
  conversationValidator = new ConversationValidator();
1670
1951
  variablesReader = new VariablesReader();
1952
+ runWithReader = new RunWithReader();
1671
1953
  webSocketResolver = new WebSocketResolver();
1672
1954
  environmentResolver;
1673
1955
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
@@ -1683,45 +1965,59 @@ var ScenarioLoader = class ScenarioLoader {
1683
1965
  for (const file of files) {
1684
1966
  const document = this.documentLoader.load(file, (0, node_path.relative)(this.cwd, file));
1685
1967
  const fileIssues = [...document.errors];
1686
- const secrets = [];
1968
+ const fileSecrets = [];
1687
1969
  const used = /* @__PURE__ */ new Set();
1688
- const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
1689
- const resolver = this.environmentResolver.withLevels([variables, config.variables]);
1690
- let value;
1691
- if (fileIssues.length === 0) {
1692
- const { variables: raw, ...rest } = document.value;
1693
- value = {
1694
- ...resolver.resolve(rest, document, fileIssues, secrets, { used }),
1695
- ...raw === void 0 ? {} : { variables }
1696
- };
1970
+ const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
1971
+ const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
1972
+ const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
1973
+ if (fileIssues.length > 0) {
1974
+ issues.push(...fileIssues);
1975
+ continue;
1697
1976
  }
1698
- let websocket;
1699
- const websocketIssues = [];
1700
- if (fileIssues.length === 0 && configDocument !== void 0) {
1701
- const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
1702
- if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
1703
- connectionIssues.push({
1704
- file: document,
1705
- issues: websocketIssues
1706
- });
1707
- if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
1977
+ const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
1978
+ const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
1979
+ for (const result of results) for (const name of result.used) used.add(name);
1980
+ fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
1981
+ run: result.run,
1982
+ issues: result.issues
1983
+ })), (issue, run) => run === void 0 ? issue : new ValidationIssue({
1984
+ message: `run '${run.name}': ${issue.message}`,
1985
+ location: issue.location
1986
+ })));
1987
+ const attempted = results.filter((result) => result.connectionChecked);
1988
+ if (attempted.length > 0) connectionIssues.push({
1989
+ file: document,
1990
+ runs: attempted.map((result) => ({
1991
+ run: result.run,
1992
+ issues: result.websocketIssues
1993
+ }))
1994
+ });
1995
+ if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
1996
+ if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
1997
+ const websocket = result.websocket;
1998
+ specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
1708
1999
  }
1709
- if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
1710
- if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
1711
- if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
1712
- if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
1713
2000
  else issues.push(...fileIssues);
1714
2001
  }
1715
- const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
2002
+ const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
1716
2003
  if (shared.length > 0) throw new EvalError({
1717
2004
  category: ErrorCategory.Configuration,
1718
2005
  message: "invalid configuration",
1719
2006
  issues: shared
1720
2007
  });
1721
- for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
1722
- message: issue.toString(),
1723
- location: ScenarioLoader.variablesLocation(entry.file)
1724
- }));
2008
+ for (const entry of connectionIssues) {
2009
+ const fileLocation = ScenarioLoader.variablesLocation(entry.file);
2010
+ issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
2011
+ message: issue.toString(),
2012
+ location: fileLocation
2013
+ }) : new ValidationIssue({
2014
+ message: `run '${run.name}': ${issue.toString()}`,
2015
+ location: new SourceLocation({
2016
+ ...entry.file.locate(run.location),
2017
+ path: ""
2018
+ })
2019
+ })));
2020
+ }
1725
2021
  if (issues.length > 0) throw new EvalError({
1726
2022
  category: ErrorCategory.Specification,
1727
2023
  message: "invalid scenario specification",
@@ -1729,6 +2025,43 @@ var ScenarioLoader = class ScenarioLoader {
1729
2025
  });
1730
2026
  return specs;
1731
2027
  }
2028
+ resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
2029
+ const issues = [];
2030
+ const used = /* @__PURE__ */ new Set();
2031
+ const resolver = scenarioResolver.withLevels([run.variables]);
2032
+ const value = {
2033
+ ...resolver.resolve(rest, document, issues, secrets, { used }),
2034
+ ...variables === void 0 ? {} : { variables }
2035
+ };
2036
+ const websocketIssues = [];
2037
+ let websocket;
2038
+ let connectionChecked = false;
2039
+ if (issues.length === 0 && configDocument !== void 0) {
2040
+ const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
2041
+ if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
2042
+ if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
2043
+ connectionChecked = true;
2044
+ }
2045
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
2046
+ if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
2047
+ return {
2048
+ run,
2049
+ value,
2050
+ used,
2051
+ secrets,
2052
+ issues,
2053
+ websocket,
2054
+ websocketIssues,
2055
+ connectionChecked
2056
+ };
2057
+ }
2058
+ static byRun(runs, place) {
2059
+ const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
2060
+ const sharedTexts = new Set(shared.map((issue) => issue.toString()));
2061
+ const result = shared.map((issue) => place(issue, void 0));
2062
+ for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
2063
+ return result;
2064
+ }
1732
2065
  static shared(lists) {
1733
2066
  if (lists.length === 0) return [];
1734
2067
  const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
@@ -1740,11 +2073,22 @@ var ScenarioLoader = class ScenarioLoader {
1740
2073
  path: ""
1741
2074
  });
1742
2075
  }
1743
- static unusedVariables(variables, used, document) {
1744
- return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
1745
- message: `variable '${name}' is not used by the scenario or the websocket configuration`,
2076
+ static unusedVariables(variables, used, results, document) {
2077
+ const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
2078
+ const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
2079
+ message: message(name),
1746
2080
  location: document.locateKey(["variables"], name)
1747
2081
  }));
2082
+ const seen = /* @__PURE__ */ new Set();
2083
+ for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
2084
+ const issue = new ValidationIssue({
2085
+ message: message(key.name),
2086
+ location: key.location
2087
+ });
2088
+ if (!seen.has(issue.toString())) issues.push(issue);
2089
+ seen.add(issue.toString());
2090
+ }
2091
+ return issues;
1748
2092
  }
1749
2093
  discover(target, config) {
1750
2094
  if (!(0, node_fs.existsSync)(target)) throw new EvalError({
@@ -1770,13 +2114,14 @@ var ScenarioLoader = class ScenarioLoader {
1770
2114
  }
1771
2115
  return files;
1772
2116
  }
1773
- build(value, document, websocket, secrets) {
2117
+ build(value, document, runName, websocket, secrets) {
1774
2118
  const scenario = value;
1775
2119
  const turns = scenario.turns;
1776
2120
  const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
1777
2121
  return new ConversationSpec({
1778
2122
  version: scenario.version,
1779
2123
  name: scenario.name,
2124
+ runName,
1780
2125
  description: typeof scenario.description === "string" ? scenario.description : "",
1781
2126
  turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
1782
2127
  conversation: conversation ? new ConversationExpectation({
@@ -3244,7 +3589,7 @@ var ConsoleReporter = class ConsoleReporter {
3244
3589
  scenario(result, repeat, tokenUsageCapability) {
3245
3590
  const spec = result.spec;
3246
3591
  const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
3247
- const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
3592
+ const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
3248
3593
  if (!this.verbose) return line;
3249
3594
  if (result.attempts.length === 0) return `${line}\n`;
3250
3595
  const websocket = spec.websocket;
@@ -3257,6 +3602,9 @@ var ConsoleReporter = class ConsoleReporter {
3257
3602
  lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3258
3603
  return line + lines.join("\n");
3259
3604
  }
3605
+ static title(spec) {
3606
+ return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
3607
+ }
3260
3608
  summary(run, exitCode, reportPaths = []) {
3261
3609
  const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
3262
3610
  const lines = this.verbose ? [] : [""];
@@ -3288,7 +3636,7 @@ var ConsoleReporter = class ConsoleReporter {
3288
3636
  }
3289
3637
  problems(result) {
3290
3638
  const spec = result.spec;
3291
- const lines = [`${this.status(result.status, 6)} ${spec.name} (${spec.location.file})`];
3639
+ const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
3292
3640
  for (const attempt of result.attempts) {
3293
3641
  const execution = attempt.execution;
3294
3642
  if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
@@ -3466,6 +3814,7 @@ var JsonReporter = class JsonReporter {
3466
3814
  const capability = websocket.protocol.capabilities.tokenUsage;
3467
3815
  return {
3468
3816
  name: result.spec.name,
3817
+ runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
3469
3818
  file: result.spec.location.file,
3470
3819
  status: result.status,
3471
3820
  connection: {