@mappies/miniwat-eval 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -321,6 +321,7 @@ var WebSocketConfig = class {
321
321
  var ConversationSpec = class {
322
322
  version;
323
323
  name;
324
+ runName;
324
325
  description;
325
326
  turns;
326
327
  conversation;
@@ -330,6 +331,7 @@ var ConversationSpec = class {
330
331
  constructor(data) {
331
332
  this.version = data?.version ?? 1;
332
333
  this.name = data?.name ?? "";
334
+ this.runName = data?.runName ?? "";
333
335
  this.description = data?.description ?? "";
334
336
  this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
335
337
  this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
@@ -451,7 +453,7 @@ var EvalConfig = class {
451
453
  this.version = data?.version ?? 1;
452
454
  this.configPath = data?.configPath ?? "";
453
455
  this.parametersPath = data?.parametersPath ?? "";
454
- this.variables = { ...data?.variables ?? {} };
456
+ this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
455
457
  this.websocket = new WebSocketConfig(data?.websocket);
456
458
  this.execution = new ExecutionConfig(data?.execution);
457
459
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -766,7 +768,10 @@ var eval_config_schema_default = {
766
768
  "variables": {
767
769
  "type": "object",
768
770
  "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
769
- "additionalProperties": { "type": "string" }
771
+ "additionalProperties": { "anyOf": [{ "type": "string" }, {
772
+ "type": "array",
773
+ "items": { "type": "string" }
774
+ }] }
770
775
  },
771
776
  "websocket": {
772
777
  "type": "object",
@@ -999,7 +1004,7 @@ var EnvironmentResolver = class EnvironmentResolver {
999
1004
  return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
1000
1005
  }
1001
1006
  resolve(value, document, issues, secrets = [], options = {}) {
1002
- return this.resolveValue(value, [], document, issues, secrets, options);
1007
+ return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
1003
1008
  }
1004
1009
  static hasReference(value) {
1005
1010
  return new RegExp(REFERENCE.source).test(value);
@@ -1007,7 +1012,7 @@ var EnvironmentResolver = class EnvironmentResolver {
1007
1012
  static isSecretName(name) {
1008
1013
  return SECRET_NAME.test(name);
1009
1014
  }
1010
- lookup(name) {
1015
+ find(name) {
1011
1016
  for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
1012
1017
  return this.environment[name];
1013
1018
  }
@@ -1022,7 +1027,7 @@ var EnvironmentResolver = class EnvironmentResolver {
1022
1027
  resolveValue(value, path, document, issues, secrets, options) {
1023
1028
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
1024
1029
  options.used?.add(name);
1025
- const resolved = this.lookup(name);
1030
+ const resolved = this.find(name);
1026
1031
  if (resolved === void 0) {
1027
1032
  if (options.optional?.(path)) {
1028
1033
  options.unresolved?.add(YamlDocument.formatPath(path));
@@ -1034,6 +1039,13 @@ var EnvironmentResolver = class EnvironmentResolver {
1034
1039
  }));
1035
1040
  return reference;
1036
1041
  }
1042
+ if (Array.isArray(resolved)) {
1043
+ issues.push(new ValidationIssue({
1044
+ message: `variable '${name}' is a list, which can only be used as runWith.from`,
1045
+ location: document.locate(path)
1046
+ }));
1047
+ return reference;
1048
+ }
1037
1049
  if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
1038
1050
  return resolved;
1039
1051
  });
@@ -1162,9 +1174,25 @@ var VariablesReader = class {
1162
1174
  }));
1163
1175
  continue;
1164
1176
  }
1177
+ if (Array.isArray(value)) {
1178
+ const bad = value.findIndex((item) => typeof item !== "string");
1179
+ if (bad >= 0) {
1180
+ issues.push(new ValidationIssue({
1181
+ message: "must be string",
1182
+ location: document.locate([
1183
+ "variables",
1184
+ name,
1185
+ bad
1186
+ ])
1187
+ }));
1188
+ continue;
1189
+ }
1190
+ variables[name] = [...value];
1191
+ continue;
1192
+ }
1165
1193
  if (typeof value !== "string") {
1166
1194
  issues.push(new ValidationIssue({
1167
- message: "must be string",
1195
+ message: "must be string or a list of strings",
1168
1196
  location: document.locate(["variables", name])
1169
1197
  }));
1170
1198
  continue;
@@ -1172,8 +1200,9 @@ var VariablesReader = class {
1172
1200
  const before = issues.length;
1173
1201
  const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
1174
1202
  if (issues.length > before) continue;
1175
- variables[name] = resolved.variables[name];
1176
- if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
1203
+ const text = resolved.variables[name];
1204
+ variables[name] = text;
1205
+ if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
1177
1206
  }
1178
1207
  return variables;
1179
1208
  }
@@ -1478,7 +1507,44 @@ var conversation_schema_default = {
1478
1507
  "variables": {
1479
1508
  "type": "object",
1480
1509
  "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
1481
- "additionalProperties": { "type": "string" }
1510
+ "additionalProperties": { "anyOf": [{ "type": "string" }, {
1511
+ "type": "array",
1512
+ "items": { "type": "string" }
1513
+ }] }
1514
+ },
1515
+ "runWith": {
1516
+ "description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
1517
+ "anyOf": [{
1518
+ "type": "array",
1519
+ "minItems": 1,
1520
+ "items": {
1521
+ "type": "object",
1522
+ "required": ["name"],
1523
+ "properties": { "name": {
1524
+ "type": "string",
1525
+ "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
1526
+ } },
1527
+ "additionalProperties": { "type": "string" }
1528
+ }
1529
+ }, {
1530
+ "type": "object",
1531
+ "additionalProperties": false,
1532
+ "required": ["variable", "from"],
1533
+ "properties": {
1534
+ "variable": {
1535
+ "type": "string",
1536
+ "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
1537
+ },
1538
+ "from": { "anyOf": [{
1539
+ "type": "array",
1540
+ "minItems": 1,
1541
+ "items": { "type": "string" }
1542
+ }, {
1543
+ "type": "string",
1544
+ "pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
1545
+ }] }
1546
+ }
1547
+ }]
1482
1548
  },
1483
1549
  "turns": {
1484
1550
  "type": "array",
@@ -1637,6 +1703,221 @@ var ConversationValidator = class ConversationValidator {
1637
1703
  return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
1638
1704
  }
1639
1705
  };
1706
+ // src/scenarios/runWithReader.ts
1707
+ const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
1708
+ const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
1709
+ const FORMS = "must be a list of rows, or an object with variable and from";
1710
+ const SINGLE_RUN = {
1711
+ name: "",
1712
+ variables: {},
1713
+ keys: [],
1714
+ location: [],
1715
+ secrets: []
1716
+ };
1717
+ var RunWithReader = class RunWithReader {
1718
+ read(document, resolver, issues, used) {
1719
+ const raw = isRecord(document.value) ? document.value.runWith : void 0;
1720
+ if (raw === void 0) return void 0;
1721
+ if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
1722
+ if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
1723
+ issues.push(new ValidationIssue({
1724
+ message: FORMS,
1725
+ location: document.locate(["runWith"])
1726
+ }));
1727
+ return [];
1728
+ }
1729
+ rows(rows, document, resolver, issues, used) {
1730
+ if (rows.length === 0) {
1731
+ issues.push(new ValidationIssue({
1732
+ message: "must not be empty",
1733
+ location: document.locate(["runWith"])
1734
+ }));
1735
+ return [];
1736
+ }
1737
+ const names = /* @__PURE__ */ new Set();
1738
+ const runs = [];
1739
+ rows.forEach((row, index) => {
1740
+ const path = ["runWith", index];
1741
+ if (!isRecord(row)) {
1742
+ issues.push(new ValidationIssue({
1743
+ message: "must be object",
1744
+ location: document.locate(path)
1745
+ }));
1746
+ return;
1747
+ }
1748
+ const name = row.name;
1749
+ if (name === void 0) issues.push(new ValidationIssue({
1750
+ message: "missing required key 'name'",
1751
+ location: document.locate(path)
1752
+ }));
1753
+ else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
1754
+ message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
1755
+ location: document.locate([...path, "name"])
1756
+ }));
1757
+ else if (names.has(name)) issues.push(new ValidationIssue({
1758
+ message: `duplicate run name '${name}'`,
1759
+ location: document.locate([...path, "name"])
1760
+ }));
1761
+ else names.add(name);
1762
+ const run = {
1763
+ name: typeof name === "string" ? name : "",
1764
+ variables: {},
1765
+ keys: [],
1766
+ location: path,
1767
+ secrets: []
1768
+ };
1769
+ for (const [key, value] of Object.entries(row)) {
1770
+ if (key === "name") continue;
1771
+ if (!VARIABLE_NAME.test(key)) {
1772
+ issues.push(new ValidationIssue({
1773
+ message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
1774
+ location: document.locateKey(path, key)
1775
+ }));
1776
+ continue;
1777
+ }
1778
+ if (typeof value !== "string") {
1779
+ issues.push(new ValidationIssue({
1780
+ message: "must be string",
1781
+ location: document.locate([...path, key])
1782
+ }));
1783
+ continue;
1784
+ }
1785
+ const resolved = resolver.resolve(value, document, issues, run.secrets, {
1786
+ used,
1787
+ at: [...path, key]
1788
+ });
1789
+ RunWithReader.set(run, key, resolved, document.locateKey(path, key));
1790
+ }
1791
+ runs.push(run);
1792
+ });
1793
+ return runs;
1794
+ }
1795
+ list(form, document, resolver, issues, used) {
1796
+ const before = issues.length;
1797
+ for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
1798
+ message: `unknown key '${key}'`,
1799
+ location: document.locateKey(["runWith"], key)
1800
+ }));
1801
+ const variable = form.variable;
1802
+ if (variable === void 0) issues.push(new ValidationIssue({
1803
+ message: "missing required key 'variable'",
1804
+ location: document.locate(["runWith"])
1805
+ }));
1806
+ else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
1807
+ message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
1808
+ location: document.locate(["runWith", "variable"])
1809
+ }));
1810
+ const entries = this.entries(form.from, document, resolver, issues, used);
1811
+ if (issues.length > before || entries === void 0) return [];
1812
+ const name = variable;
1813
+ const keyLocation = document.locate(["runWith", "variable"]);
1814
+ const names = /* @__PURE__ */ new Set();
1815
+ const runs = [];
1816
+ for (const { entry, path } of entries) {
1817
+ if (!VARIABLE_NAME.test(entry)) {
1818
+ issues.push(new ValidationIssue({
1819
+ message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
1820
+ location: document.locate(path)
1821
+ }));
1822
+ continue;
1823
+ }
1824
+ if (names.has(entry)) {
1825
+ issues.push(new ValidationIssue({
1826
+ message: `duplicate run name '${entry}'`,
1827
+ location: document.locate(path)
1828
+ }));
1829
+ continue;
1830
+ }
1831
+ names.add(entry);
1832
+ const run = {
1833
+ name: entry,
1834
+ variables: {},
1835
+ keys: [],
1836
+ location: path,
1837
+ secrets: []
1838
+ };
1839
+ const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
1840
+ used,
1841
+ at: path
1842
+ });
1843
+ RunWithReader.set(run, name, value, keyLocation);
1844
+ runs.push(run);
1845
+ }
1846
+ return runs;
1847
+ }
1848
+ entries(from, document, resolver, issues, used) {
1849
+ const path = ["runWith", "from"];
1850
+ const location = document.locate(path);
1851
+ if (from === void 0) {
1852
+ issues.push(new ValidationIssue({
1853
+ message: "missing required key 'from'",
1854
+ location: document.locate(["runWith"])
1855
+ }));
1856
+ return;
1857
+ }
1858
+ let list;
1859
+ let at;
1860
+ if (Array.isArray(from)) {
1861
+ list = from;
1862
+ at = (index) => [...path, index];
1863
+ } else {
1864
+ const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
1865
+ if (reference === null) {
1866
+ issues.push(new ValidationIssue({
1867
+ message: "must be a list, or a list variable such as ${tokens}",
1868
+ location
1869
+ }));
1870
+ return;
1871
+ }
1872
+ const name = reference[1];
1873
+ const found = resolver.find(name);
1874
+ used.add(name);
1875
+ if (found === void 0) {
1876
+ issues.push(new ValidationIssue({
1877
+ message: `list variable '${name}' is not set`,
1878
+ location
1879
+ }));
1880
+ return;
1881
+ }
1882
+ if (!Array.isArray(found)) {
1883
+ issues.push(new ValidationIssue({
1884
+ message: `variable '${name}' is not a list`,
1885
+ location
1886
+ }));
1887
+ return;
1888
+ }
1889
+ list = found;
1890
+ at = () => path;
1891
+ }
1892
+ if (list.length === 0) {
1893
+ issues.push(new ValidationIssue({
1894
+ message: "must not be empty",
1895
+ location
1896
+ }));
1897
+ return;
1898
+ }
1899
+ const bad = list.findIndex((item) => typeof item !== "string");
1900
+ if (bad >= 0) {
1901
+ issues.push(new ValidationIssue({
1902
+ message: "must be string",
1903
+ location: document.locate(at(bad))
1904
+ }));
1905
+ return;
1906
+ }
1907
+ return list.map((entry, index) => ({
1908
+ entry,
1909
+ path: at(index)
1910
+ }));
1911
+ }
1912
+ static set(run, name, value, location) {
1913
+ run.variables[name] = value;
1914
+ run.keys.push({
1915
+ name,
1916
+ location
1917
+ });
1918
+ if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
1919
+ }
1920
+ };
1640
1921
  // src/scenarios/scenarioLoader.ts
1641
1922
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
1642
1923
  var ScenarioLoader = class ScenarioLoader {
@@ -1645,6 +1926,7 @@ var ScenarioLoader = class ScenarioLoader {
1645
1926
  schemaValidator = new SchemaValidator(conversation_schema_default);
1646
1927
  conversationValidator = new ConversationValidator();
1647
1928
  variablesReader = new VariablesReader();
1929
+ runWithReader = new RunWithReader();
1648
1930
  webSocketResolver = new WebSocketResolver();
1649
1931
  environmentResolver;
1650
1932
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
@@ -1660,45 +1942,59 @@ var ScenarioLoader = class ScenarioLoader {
1660
1942
  for (const file of files) {
1661
1943
  const document = this.documentLoader.load(file, relative(this.cwd, file));
1662
1944
  const fileIssues = [...document.errors];
1663
- const secrets = [];
1945
+ const fileSecrets = [];
1664
1946
  const used = /* @__PURE__ */ new Set();
1665
- const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
1666
- const resolver = this.environmentResolver.withLevels([variables, config.variables]);
1667
- let value;
1668
- if (fileIssues.length === 0) {
1669
- const { variables: raw, ...rest } = document.value;
1670
- value = {
1671
- ...resolver.resolve(rest, document, fileIssues, secrets, { used }),
1672
- ...raw === void 0 ? {} : { variables }
1673
- };
1947
+ const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
1948
+ const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
1949
+ const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
1950
+ if (fileIssues.length > 0) {
1951
+ issues.push(...fileIssues);
1952
+ continue;
1674
1953
  }
1675
- let websocket;
1676
- const websocketIssues = [];
1677
- if (fileIssues.length === 0 && configDocument !== void 0) {
1678
- const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
1679
- if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
1680
- connectionIssues.push({
1681
- file: document,
1682
- issues: websocketIssues
1683
- });
1684
- if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
1954
+ const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
1955
+ const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
1956
+ for (const result of results) for (const name of result.used) used.add(name);
1957
+ fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
1958
+ run: result.run,
1959
+ issues: result.issues
1960
+ })), (issue, run) => run === void 0 ? issue : new ValidationIssue({
1961
+ message: `run '${run.name}': ${issue.message}`,
1962
+ location: issue.location
1963
+ })));
1964
+ const attempted = results.filter((result) => result.connectionChecked);
1965
+ if (attempted.length > 0) connectionIssues.push({
1966
+ file: document,
1967
+ runs: attempted.map((result) => ({
1968
+ run: result.run,
1969
+ issues: result.websocketIssues
1970
+ }))
1971
+ });
1972
+ if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
1973
+ if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
1974
+ const websocket = result.websocket;
1975
+ specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
1685
1976
  }
1686
- if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
1687
- if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
1688
- if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
1689
- if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
1690
1977
  else issues.push(...fileIssues);
1691
1978
  }
1692
- const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
1979
+ const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
1693
1980
  if (shared.length > 0) throw new EvalError({
1694
1981
  category: ErrorCategory.Configuration,
1695
1982
  message: "invalid configuration",
1696
1983
  issues: shared
1697
1984
  });
1698
- for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
1699
- message: issue.toString(),
1700
- location: ScenarioLoader.variablesLocation(entry.file)
1701
- }));
1985
+ for (const entry of connectionIssues) {
1986
+ const fileLocation = ScenarioLoader.variablesLocation(entry.file);
1987
+ issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
1988
+ message: issue.toString(),
1989
+ location: fileLocation
1990
+ }) : new ValidationIssue({
1991
+ message: `run '${run.name}': ${issue.toString()}`,
1992
+ location: new SourceLocation({
1993
+ ...entry.file.locate(run.location),
1994
+ path: ""
1995
+ })
1996
+ })));
1997
+ }
1702
1998
  if (issues.length > 0) throw new EvalError({
1703
1999
  category: ErrorCategory.Specification,
1704
2000
  message: "invalid scenario specification",
@@ -1706,6 +2002,43 @@ var ScenarioLoader = class ScenarioLoader {
1706
2002
  });
1707
2003
  return specs;
1708
2004
  }
2005
+ resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
2006
+ const issues = [];
2007
+ const used = /* @__PURE__ */ new Set();
2008
+ const resolver = scenarioResolver.withLevels([run.variables]);
2009
+ const value = {
2010
+ ...resolver.resolve(rest, document, issues, secrets, { used }),
2011
+ ...variables === void 0 ? {} : { variables }
2012
+ };
2013
+ const websocketIssues = [];
2014
+ let websocket;
2015
+ let connectionChecked = false;
2016
+ if (issues.length === 0 && configDocument !== void 0) {
2017
+ const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
2018
+ if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
2019
+ if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
2020
+ connectionChecked = true;
2021
+ }
2022
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
2023
+ if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
2024
+ return {
2025
+ run,
2026
+ value,
2027
+ used,
2028
+ secrets,
2029
+ issues,
2030
+ websocket,
2031
+ websocketIssues,
2032
+ connectionChecked
2033
+ };
2034
+ }
2035
+ static byRun(runs, place) {
2036
+ const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
2037
+ const sharedTexts = new Set(shared.map((issue) => issue.toString()));
2038
+ const result = shared.map((issue) => place(issue, void 0));
2039
+ for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
2040
+ return result;
2041
+ }
1709
2042
  static shared(lists) {
1710
2043
  if (lists.length === 0) return [];
1711
2044
  const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
@@ -1717,11 +2050,22 @@ var ScenarioLoader = class ScenarioLoader {
1717
2050
  path: ""
1718
2051
  });
1719
2052
  }
1720
- static unusedVariables(variables, used, document) {
1721
- return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
1722
- message: `variable '${name}' is not used by the scenario or the websocket configuration`,
2053
+ static unusedVariables(variables, used, results, document) {
2054
+ const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
2055
+ const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
2056
+ message: message(name),
1723
2057
  location: document.locateKey(["variables"], name)
1724
2058
  }));
2059
+ const seen = /* @__PURE__ */ new Set();
2060
+ for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
2061
+ const issue = new ValidationIssue({
2062
+ message: message(key.name),
2063
+ location: key.location
2064
+ });
2065
+ if (!seen.has(issue.toString())) issues.push(issue);
2066
+ seen.add(issue.toString());
2067
+ }
2068
+ return issues;
1725
2069
  }
1726
2070
  discover(target, config) {
1727
2071
  if (!existsSync(target)) throw new EvalError({
@@ -1747,13 +2091,14 @@ var ScenarioLoader = class ScenarioLoader {
1747
2091
  }
1748
2092
  return files;
1749
2093
  }
1750
- build(value, document, websocket, secrets) {
2094
+ build(value, document, runName, websocket, secrets) {
1751
2095
  const scenario = value;
1752
2096
  const turns = scenario.turns;
1753
2097
  const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
1754
2098
  return new ConversationSpec({
1755
2099
  version: scenario.version,
1756
2100
  name: scenario.name,
2101
+ runName,
1757
2102
  description: typeof scenario.description === "string" ? scenario.description : "",
1758
2103
  turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
1759
2104
  conversation: conversation ? new ConversationExpectation({
@@ -3189,61 +3534,94 @@ var ExitCodeCalculator = class {
3189
3534
  }
3190
3535
  };
3191
3536
  // src/reporting/consoleReporter.ts
3192
- const STATUS_WIDTH = 11;
3193
- const COLOURS = {
3194
- [ResultStatus.Pass]: "green",
3195
- [ResultStatus.Flaky]: "yellow",
3196
- [ResultStatus.Fail]: "red",
3197
- [ResultStatus.Error]: "red",
3198
- [ResultStatus.Skipped]: "dim",
3199
- [ResultStatus.Interrupted]: "dim"
3537
+ const MARKS = {
3538
+ [ResultStatus.Pass]: {
3539
+ symbol: "✔",
3540
+ colour: "green"
3541
+ },
3542
+ [ResultStatus.Flaky]: {
3543
+ symbol: "⚠",
3544
+ colour: "yellow"
3545
+ },
3546
+ [ResultStatus.Fail]: {
3547
+ symbol: "✖",
3548
+ colour: "red"
3549
+ },
3550
+ [ResultStatus.Error]: {
3551
+ symbol: "●",
3552
+ colour: "red"
3553
+ },
3554
+ [ResultStatus.Skipped]: {
3555
+ symbol: "○",
3556
+ colour: "dim"
3557
+ },
3558
+ [ResultStatus.Interrupted]: {
3559
+ symbol: "◌",
3560
+ colour: "dim"
3561
+ }
3200
3562
  };
3563
+ const MIN_COLUMNS = 40;
3564
+ const RULE_WIDTH = 72;
3201
3565
  var ConsoleReporter = class ConsoleReporter {
3202
3566
  verbose;
3203
3567
  colour;
3204
- constructor(verbose, colour) {
3568
+ nameWidth;
3569
+ columns;
3570
+ thresholds;
3571
+ constructor(verbose, colour, options = {}) {
3205
3572
  this.verbose = verbose;
3206
3573
  this.colour = colour;
3574
+ this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
3575
+ this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
3576
+ this.thresholds = options.thresholds;
3577
+ }
3578
+ static nameWidth(specs) {
3579
+ return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
3207
3580
  }
3208
3581
  header(run) {
3209
3582
  return [
3210
- "Miniwat Eval",
3583
+ " Miniwat Eval",
3211
3584
  "",
3212
- ...run.quick ? ["Execution Mode: Quick"] : [],
3213
- `Repeat: ${run.repeat}`,
3214
- `Minimum Pass: ${run.minPass}`,
3585
+ ...run.quick ? [" Execution mode Quick"] : [],
3586
+ ` Repeat ${run.repeat}`,
3587
+ ` Minimum pass ${run.minPass}`,
3215
3588
  "",
3216
- `Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
3589
+ ` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
3217
3590
  "",
3218
3591
  ""
3219
3592
  ].join("\n");
3220
3593
  }
3221
3594
  scenario(result, repeat, tokenUsageCapability) {
3222
3595
  const spec = result.spec;
3223
- const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
3224
- const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
3596
+ const title = ConsoleReporter.title(spec);
3597
+ const started = result.attempts.length > 0;
3598
+ const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
3599
+ const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
3600
+ const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
3225
3601
  if (!this.verbose) return line;
3226
- if (result.attempts.length === 0) return `${line}\n`;
3602
+ if (!started) return `${line}\n`;
3227
3603
  const websocket = spec.websocket;
3228
3604
  const subprotocols = websocket?.connection.subprotocols.length ?? 0;
3229
- const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
3230
- for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3231
- lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
3232
- lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
3233
- lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
3234
- lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3605
+ const lines = [];
3606
+ if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
3607
+ for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3608
+ lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
3609
+ lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3235
3610
  return line + lines.join("\n");
3236
3611
  }
3612
+ static title(spec) {
3613
+ return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
3614
+ }
3237
3615
  summary(run, exitCode, reportPaths = []) {
3238
- const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
3616
+ const started = run.scenarios.filter((result) => result.attempts.length > 0);
3617
+ const failures = started.flatMap((result) => this.blocks(result, false));
3618
+ const flaky = started.flatMap((result) => this.blocks(result, true));
3239
3619
  const lines = this.verbose ? [] : [""];
3240
- if (problems.length > 0) {
3241
- lines.push("Problems", "");
3242
- for (const result of problems) lines.push(...this.problems(result), "");
3243
- }
3620
+ if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
3621
+ if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
3244
3622
  const tests = run.scenarios.flatMap((result) => result.tests);
3245
- lines.push(...run.status === RunStatus.Interrupted ? [`Run: ${run.status}`] : [], `Scenarios: ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, `Tests: ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability), ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true), `Duration: ${ConsoleReporter.duration(run.durationMs)}`, "", "");
3246
- return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
3623
+ lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
3624
+ return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
3247
3625
  }
3248
3626
  static footer(run, exitCode, reportPaths) {
3249
3627
  return [
@@ -3257,57 +3635,135 @@ var ConsoleReporter = class ConsoleReporter {
3257
3635
  return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
3258
3636
  }
3259
3637
  static duration(ms) {
3638
+ if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
3260
3639
  const seconds = Math.floor(ms / 1e3);
3261
3640
  const pad = (value) => String(value).padStart(2, "0");
3262
3641
  const hours = Math.floor(seconds / 3600);
3263
- const rest = `${pad(Math.floor(seconds / 60) % 60)}:${pad(seconds % 60)}`;
3264
- return hours > 0 ? `${hours}:${rest}` : rest;
3642
+ const minutes = Math.floor(seconds / 60) % 60;
3643
+ return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
3644
+ }
3645
+ static wrap(text, width) {
3646
+ const lines = [];
3647
+ for (const paragraph of text.split("\n")) {
3648
+ let line = "";
3649
+ for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
3650
+ lines.push(line);
3651
+ line = word;
3652
+ } else line = line === "" ? word : `${line} ${word}`;
3653
+ lines.push(line);
3654
+ }
3655
+ return lines;
3656
+ }
3657
+ section(name, blocks) {
3658
+ return [
3659
+ ...this.rule(name),
3660
+ "",
3661
+ ...blocks.flatMap((block) => [...block, ""])
3662
+ ];
3663
+ }
3664
+ rule(name) {
3665
+ const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
3666
+ return [
3667
+ rule,
3668
+ ` ${name}`,
3669
+ rule
3670
+ ];
3265
3671
  }
3266
- problems(result) {
3672
+ blocks(result, flaky) {
3267
3673
  const spec = result.spec;
3268
- const lines = [`${this.status(result.status, 6)} ${spec.name} (${spec.location.file})`];
3269
- for (const attempt of result.attempts) {
3270
- const execution = attempt.execution;
3271
- if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3272
- for (const error of execution.attemptErrors) lines.push(` Attempt ${attempt.attempt} error: ${error.category} error: ${error.message}`);
3273
- if (execution.closeError !== void 0) lines.push(` Attempt ${attempt.attempt} close error: ${execution.closeError.message}`);
3274
- }
3275
- for (const test of result.tests.filter((entry) => entry.status !== ResultStatus.Pass)) {
3276
- lines.push(` ${this.status(test.status, 6)} ${ConsoleReporter.testTitle(test)} ${test.location.toString()} ${ConsoleReporter.counts(test)}`);
3277
- test.history.forEach((entry, index) => {
3278
- if (entry.status === ResultStatus.Pass) return;
3279
- const prefix = ` Attempt ${result.attempts[index].attempt} ${this.status(entry.status)}`;
3280
- const width = ` Attempt ${result.attempts[index].attempt} ${entry.status}: `.length;
3281
- if (entry.details.length === 0) {
3282
- lines.push(prefix);
3283
- return;
3284
- }
3285
- lines.push(`${prefix}: ${ConsoleReporter.indent(entry.details[0], width)}`, ...entry.details.slice(1).map((detail) => `${" ".repeat(width)}${ConsoleReporter.indent(detail, width)}`));
3286
- });
3674
+ const title = ConsoleReporter.title(spec);
3675
+ const blocks = [];
3676
+ if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
3677
+ const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
3678
+ for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
3679
+ return blocks;
3680
+ }
3681
+ attemptBlock(spec, title, attempt) {
3682
+ const execution = attempt.execution;
3683
+ const problems = [];
3684
+ if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
3685
+ for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
3686
+ if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
3687
+ if (problems.length === 0) return [];
3688
+ const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
3689
+ const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
3690
+ return [[
3691
+ ` scenario at ${spec.location.file}`,
3692
+ ` ${this.mark(status)} ${title} › ${what}`,
3693
+ ...problems.flatMap((problem) => this.paragraph(problem, 3))
3694
+ ]];
3695
+ }
3696
+ testBlock(spec, title, test, result) {
3697
+ const location = test.location;
3698
+ const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
3699
+ const failing = test.history.map((entry, index) => ({
3700
+ entry,
3701
+ attempt: result.attempts[index]?.attempt ?? index + 1
3702
+ })).filter(({ entry }) => entry.status !== ResultStatus.Pass);
3703
+ const several = test.history.length > 1;
3704
+ for (const { entry, attempt } of failing) {
3705
+ if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
3706
+ lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
3287
3707
  }
3708
+ lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
3288
3709
  return lines;
3289
3710
  }
3711
+ entryBody(spec, entry, indent) {
3712
+ const pad = " ".repeat(indent);
3713
+ const evaluation = entry.evaluation;
3714
+ if (evaluation === void 0) {
3715
+ if (entry.details.length === 0) return [];
3716
+ return [
3717
+ "",
3718
+ `${pad}Details`,
3719
+ ...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
3720
+ ];
3721
+ }
3722
+ const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
3723
+ const expected = ConsoleReporter.expected(spec, entry);
3724
+ const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
3725
+ if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
3726
+ if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
3727
+ if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
3728
+ return lines;
3729
+ }
3730
+ static expected(spec, entry) {
3731
+ if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
3732
+ return spec.turns[entry.turn - 1]?.expect.response?.criteria;
3733
+ }
3734
+ paragraph(text, indent) {
3735
+ const pad = " ".repeat(indent);
3736
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
3737
+ }
3738
+ bullet(text, indent) {
3739
+ const pad = " ".repeat(indent);
3740
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
3741
+ }
3290
3742
  static testTitle(test) {
3291
3743
  return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
3292
3744
  }
3293
3745
  static tally(statuses) {
3294
3746
  const count = (status) => statuses.filter((entry) => entry === status).length;
3295
- const interrupted = count(ResultStatus.Interrupted);
3296
- return `${count(ResultStatus.Pass)} passed, ${count(ResultStatus.Flaky)} flaky, ${count(ResultStatus.Fail)} failed, ${count(ResultStatus.Error)} errors${interrupted > 0 ? `, ${interrupted} interrupted` : ""}`;
3297
- }
3298
- status(status, width = 0) {
3299
- const padding = " ".repeat(Math.max(0, width - status.length));
3300
- return (this.colour ? styleText(COLOURS[status], status, { validateStream: false }) : status) + padding;
3301
- }
3302
- static counts(result) {
3303
- return `(${result.passed} passed, ${result.failed} failed, ${result.errors} errors; required ${result.required})`;
3747
+ const parts = [
3748
+ [ResultStatus.Pass, "passed"],
3749
+ [ResultStatus.Flaky, "flaky"],
3750
+ [ResultStatus.Fail, "failed"],
3751
+ [ResultStatus.Error, "errors"],
3752
+ [ResultStatus.Interrupted, "interrupted"]
3753
+ ].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
3754
+ return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
3755
+ }
3756
+ mark(status) {
3757
+ const { symbol, colour } = MARKS[status];
3758
+ return this.colour ? styleText(colour, symbol, { validateStream: false }) : symbol;
3304
3759
  }
3305
3760
  static usage(label, usage, capability) {
3306
- if (!capability) return `${label}: n/a (not supported by host)`;
3307
- if (usage.expected === 0) return `${label}: none`;
3308
- if (usage.total === void 0) return `${label}: n/a (coverage 0/${usage.expected})`;
3761
+ const name = label.padEnd(16);
3762
+ if (!capability) return `${name} n/a (not supported by host)`;
3763
+ if (usage.expected === 0) return `${name} none`;
3764
+ if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
3309
3765
  const number = (value) => value.toLocaleString("en-US");
3310
- const total = `${label}: total ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3766
+ const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3311
3767
  return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
3312
3768
  }
3313
3769
  formatAttempt(spec, attempt) {
@@ -3319,28 +3775,28 @@ var ConsoleReporter = class ConsoleReporter {
3319
3775
  lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
3320
3776
  lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
3321
3777
  });
3322
- if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3778
+ if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3323
3779
  const notRun = attempt.tests.filter((test) => test.turn > ran);
3324
- if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3780
+ if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3325
3781
  lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
3326
- for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3327
- if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3328
- lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3782
+ for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3783
+ if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3784
+ lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3329
3785
  return lines;
3330
3786
  }
3331
3787
  formatTest(test) {
3332
- return [` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${detail}`)];
3788
+ return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
3333
3789
  }
3334
3790
  static formatTurn(number, input, turn) {
3335
3791
  const lines = [
3336
- ` Turn ${number} (${turn.durationMs} ms)`,
3337
- ` Input: ${ConsoleReporter.indent(input, 17)}`,
3338
- ` Response: ${ConsoleReporter.indent(turn.response, 17)}`
3792
+ ` Turn ${number} (${turn.durationMs} ms)`,
3793
+ ` Input: ${ConsoleReporter.indent(input, 22)}`,
3794
+ ` Response: ${ConsoleReporter.indent(turn.response, 22)}`
3339
3795
  ];
3340
- for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3341
- for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3342
- for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 17)}`);
3343
- if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3796
+ for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3797
+ for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3798
+ for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
3799
+ if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3344
3800
  return lines;
3345
3801
  }
3346
3802
  static indent(text, width) {
@@ -3443,6 +3899,7 @@ var JsonReporter = class JsonReporter {
3443
3899
  const capability = websocket.protocol.capabilities.tokenUsage;
3444
3900
  return {
3445
3901
  name: result.spec.name,
3902
+ runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
3446
3903
  file: result.spec.location.file,
3447
3904
  status: result.status,
3448
3905
  connection: {