@mappies/miniwat-eval 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -344,6 +344,7 @@ var WebSocketConfig = class {
344
344
  var ConversationSpec = class {
345
345
  version;
346
346
  name;
347
+ runName;
347
348
  description;
348
349
  turns;
349
350
  conversation;
@@ -353,6 +354,7 @@ var ConversationSpec = class {
353
354
  constructor(data) {
354
355
  this.version = data?.version ?? 1;
355
356
  this.name = data?.name ?? "";
357
+ this.runName = data?.runName ?? "";
356
358
  this.description = data?.description ?? "";
357
359
  this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
358
360
  this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
@@ -474,7 +476,7 @@ var EvalConfig = class {
474
476
  this.version = data?.version ?? 1;
475
477
  this.configPath = data?.configPath ?? "";
476
478
  this.parametersPath = data?.parametersPath ?? "";
477
- this.variables = { ...data?.variables ?? {} };
479
+ this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
478
480
  this.websocket = new WebSocketConfig(data?.websocket);
479
481
  this.execution = new ExecutionConfig(data?.execution);
480
482
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -789,7 +791,10 @@ var eval_config_schema_default = {
789
791
  "variables": {
790
792
  "type": "object",
791
793
  "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
792
- "additionalProperties": { "type": "string" }
794
+ "additionalProperties": { "anyOf": [{ "type": "string" }, {
795
+ "type": "array",
796
+ "items": { "type": "string" }
797
+ }] }
793
798
  },
794
799
  "websocket": {
795
800
  "type": "object",
@@ -1022,7 +1027,7 @@ var EnvironmentResolver = class EnvironmentResolver {
1022
1027
  return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
1023
1028
  }
1024
1029
  resolve(value, document, issues, secrets = [], options = {}) {
1025
- return this.resolveValue(value, [], document, issues, secrets, options);
1030
+ return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
1026
1031
  }
1027
1032
  static hasReference(value) {
1028
1033
  return new RegExp(REFERENCE.source).test(value);
@@ -1030,7 +1035,7 @@ var EnvironmentResolver = class EnvironmentResolver {
1030
1035
  static isSecretName(name) {
1031
1036
  return SECRET_NAME.test(name);
1032
1037
  }
1033
- lookup(name) {
1038
+ find(name) {
1034
1039
  for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
1035
1040
  return this.environment[name];
1036
1041
  }
@@ -1045,7 +1050,7 @@ var EnvironmentResolver = class EnvironmentResolver {
1045
1050
  resolveValue(value, path, document, issues, secrets, options) {
1046
1051
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
1047
1052
  options.used?.add(name);
1048
- const resolved = this.lookup(name);
1053
+ const resolved = this.find(name);
1049
1054
  if (resolved === void 0) {
1050
1055
  if (options.optional?.(path)) {
1051
1056
  options.unresolved?.add(YamlDocument.formatPath(path));
@@ -1057,6 +1062,13 @@ var EnvironmentResolver = class EnvironmentResolver {
1057
1062
  }));
1058
1063
  return reference;
1059
1064
  }
1065
+ if (Array.isArray(resolved)) {
1066
+ issues.push(new ValidationIssue({
1067
+ message: `variable '${name}' is a list, which can only be used as runWith.from`,
1068
+ location: document.locate(path)
1069
+ }));
1070
+ return reference;
1071
+ }
1060
1072
  if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
1061
1073
  return resolved;
1062
1074
  });
@@ -1185,9 +1197,25 @@ var VariablesReader = class {
1185
1197
  }));
1186
1198
  continue;
1187
1199
  }
1200
+ if (Array.isArray(value)) {
1201
+ const bad = value.findIndex((item) => typeof item !== "string");
1202
+ if (bad >= 0) {
1203
+ issues.push(new ValidationIssue({
1204
+ message: "must be string",
1205
+ location: document.locate([
1206
+ "variables",
1207
+ name,
1208
+ bad
1209
+ ])
1210
+ }));
1211
+ continue;
1212
+ }
1213
+ variables[name] = [...value];
1214
+ continue;
1215
+ }
1188
1216
  if (typeof value !== "string") {
1189
1217
  issues.push(new ValidationIssue({
1190
- message: "must be string",
1218
+ message: "must be string or a list of strings",
1191
1219
  location: document.locate(["variables", name])
1192
1220
  }));
1193
1221
  continue;
@@ -1195,8 +1223,9 @@ var VariablesReader = class {
1195
1223
  const before = issues.length;
1196
1224
  const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
1197
1225
  if (issues.length > before) continue;
1198
- variables[name] = resolved.variables[name];
1199
- if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
1226
+ const text = resolved.variables[name];
1227
+ variables[name] = text;
1228
+ if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
1200
1229
  }
1201
1230
  return variables;
1202
1231
  }
@@ -1501,7 +1530,44 @@ var conversation_schema_default = {
1501
1530
  "variables": {
1502
1531
  "type": "object",
1503
1532
  "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
1504
- "additionalProperties": { "type": "string" }
1533
+ "additionalProperties": { "anyOf": [{ "type": "string" }, {
1534
+ "type": "array",
1535
+ "items": { "type": "string" }
1536
+ }] }
1537
+ },
1538
+ "runWith": {
1539
+ "description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
1540
+ "anyOf": [{
1541
+ "type": "array",
1542
+ "minItems": 1,
1543
+ "items": {
1544
+ "type": "object",
1545
+ "required": ["name"],
1546
+ "properties": { "name": {
1547
+ "type": "string",
1548
+ "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
1549
+ } },
1550
+ "additionalProperties": { "type": "string" }
1551
+ }
1552
+ }, {
1553
+ "type": "object",
1554
+ "additionalProperties": false,
1555
+ "required": ["variable", "from"],
1556
+ "properties": {
1557
+ "variable": {
1558
+ "type": "string",
1559
+ "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
1560
+ },
1561
+ "from": { "anyOf": [{
1562
+ "type": "array",
1563
+ "minItems": 1,
1564
+ "items": { "type": "string" }
1565
+ }, {
1566
+ "type": "string",
1567
+ "pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
1568
+ }] }
1569
+ }
1570
+ }]
1505
1571
  },
1506
1572
  "turns": {
1507
1573
  "type": "array",
@@ -1660,6 +1726,221 @@ var ConversationValidator = class ConversationValidator {
1660
1726
  return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
1661
1727
  }
1662
1728
  };
1729
+ // src/scenarios/runWithReader.ts
1730
+ const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
1731
+ const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
1732
+ const FORMS = "must be a list of rows, or an object with variable and from";
1733
+ const SINGLE_RUN = {
1734
+ name: "",
1735
+ variables: {},
1736
+ keys: [],
1737
+ location: [],
1738
+ secrets: []
1739
+ };
1740
+ var RunWithReader = class RunWithReader {
1741
+ read(document, resolver, issues, used) {
1742
+ const raw = isRecord(document.value) ? document.value.runWith : void 0;
1743
+ if (raw === void 0) return void 0;
1744
+ if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
1745
+ if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
1746
+ issues.push(new ValidationIssue({
1747
+ message: FORMS,
1748
+ location: document.locate(["runWith"])
1749
+ }));
1750
+ return [];
1751
+ }
1752
+ rows(rows, document, resolver, issues, used) {
1753
+ if (rows.length === 0) {
1754
+ issues.push(new ValidationIssue({
1755
+ message: "must not be empty",
1756
+ location: document.locate(["runWith"])
1757
+ }));
1758
+ return [];
1759
+ }
1760
+ const names = /* @__PURE__ */ new Set();
1761
+ const runs = [];
1762
+ rows.forEach((row, index) => {
1763
+ const path = ["runWith", index];
1764
+ if (!isRecord(row)) {
1765
+ issues.push(new ValidationIssue({
1766
+ message: "must be object",
1767
+ location: document.locate(path)
1768
+ }));
1769
+ return;
1770
+ }
1771
+ const name = row.name;
1772
+ if (name === void 0) issues.push(new ValidationIssue({
1773
+ message: "missing required key 'name'",
1774
+ location: document.locate(path)
1775
+ }));
1776
+ else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
1777
+ message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
1778
+ location: document.locate([...path, "name"])
1779
+ }));
1780
+ else if (names.has(name)) issues.push(new ValidationIssue({
1781
+ message: `duplicate run name '${name}'`,
1782
+ location: document.locate([...path, "name"])
1783
+ }));
1784
+ else names.add(name);
1785
+ const run = {
1786
+ name: typeof name === "string" ? name : "",
1787
+ variables: {},
1788
+ keys: [],
1789
+ location: path,
1790
+ secrets: []
1791
+ };
1792
+ for (const [key, value] of Object.entries(row)) {
1793
+ if (key === "name") continue;
1794
+ if (!VARIABLE_NAME.test(key)) {
1795
+ issues.push(new ValidationIssue({
1796
+ message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
1797
+ location: document.locateKey(path, key)
1798
+ }));
1799
+ continue;
1800
+ }
1801
+ if (typeof value !== "string") {
1802
+ issues.push(new ValidationIssue({
1803
+ message: "must be string",
1804
+ location: document.locate([...path, key])
1805
+ }));
1806
+ continue;
1807
+ }
1808
+ const resolved = resolver.resolve(value, document, issues, run.secrets, {
1809
+ used,
1810
+ at: [...path, key]
1811
+ });
1812
+ RunWithReader.set(run, key, resolved, document.locateKey(path, key));
1813
+ }
1814
+ runs.push(run);
1815
+ });
1816
+ return runs;
1817
+ }
1818
+ list(form, document, resolver, issues, used) {
1819
+ const before = issues.length;
1820
+ for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
1821
+ message: `unknown key '${key}'`,
1822
+ location: document.locateKey(["runWith"], key)
1823
+ }));
1824
+ const variable = form.variable;
1825
+ if (variable === void 0) issues.push(new ValidationIssue({
1826
+ message: "missing required key 'variable'",
1827
+ location: document.locate(["runWith"])
1828
+ }));
1829
+ else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
1830
+ message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
1831
+ location: document.locate(["runWith", "variable"])
1832
+ }));
1833
+ const entries = this.entries(form.from, document, resolver, issues, used);
1834
+ if (issues.length > before || entries === void 0) return [];
1835
+ const name = variable;
1836
+ const keyLocation = document.locate(["runWith", "variable"]);
1837
+ const names = /* @__PURE__ */ new Set();
1838
+ const runs = [];
1839
+ for (const { entry, path } of entries) {
1840
+ if (!VARIABLE_NAME.test(entry)) {
1841
+ issues.push(new ValidationIssue({
1842
+ message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
1843
+ location: document.locate(path)
1844
+ }));
1845
+ continue;
1846
+ }
1847
+ if (names.has(entry)) {
1848
+ issues.push(new ValidationIssue({
1849
+ message: `duplicate run name '${entry}'`,
1850
+ location: document.locate(path)
1851
+ }));
1852
+ continue;
1853
+ }
1854
+ names.add(entry);
1855
+ const run = {
1856
+ name: entry,
1857
+ variables: {},
1858
+ keys: [],
1859
+ location: path,
1860
+ secrets: []
1861
+ };
1862
+ const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
1863
+ used,
1864
+ at: path
1865
+ });
1866
+ RunWithReader.set(run, name, value, keyLocation);
1867
+ runs.push(run);
1868
+ }
1869
+ return runs;
1870
+ }
1871
+ entries(from, document, resolver, issues, used) {
1872
+ const path = ["runWith", "from"];
1873
+ const location = document.locate(path);
1874
+ if (from === void 0) {
1875
+ issues.push(new ValidationIssue({
1876
+ message: "missing required key 'from'",
1877
+ location: document.locate(["runWith"])
1878
+ }));
1879
+ return;
1880
+ }
1881
+ let list;
1882
+ let at;
1883
+ if (Array.isArray(from)) {
1884
+ list = from;
1885
+ at = (index) => [...path, index];
1886
+ } else {
1887
+ const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
1888
+ if (reference === null) {
1889
+ issues.push(new ValidationIssue({
1890
+ message: "must be a list, or a list variable such as ${tokens}",
1891
+ location
1892
+ }));
1893
+ return;
1894
+ }
1895
+ const name = reference[1];
1896
+ const found = resolver.find(name);
1897
+ used.add(name);
1898
+ if (found === void 0) {
1899
+ issues.push(new ValidationIssue({
1900
+ message: `list variable '${name}' is not set`,
1901
+ location
1902
+ }));
1903
+ return;
1904
+ }
1905
+ if (!Array.isArray(found)) {
1906
+ issues.push(new ValidationIssue({
1907
+ message: `variable '${name}' is not a list`,
1908
+ location
1909
+ }));
1910
+ return;
1911
+ }
1912
+ list = found;
1913
+ at = () => path;
1914
+ }
1915
+ if (list.length === 0) {
1916
+ issues.push(new ValidationIssue({
1917
+ message: "must not be empty",
1918
+ location
1919
+ }));
1920
+ return;
1921
+ }
1922
+ const bad = list.findIndex((item) => typeof item !== "string");
1923
+ if (bad >= 0) {
1924
+ issues.push(new ValidationIssue({
1925
+ message: "must be string",
1926
+ location: document.locate(at(bad))
1927
+ }));
1928
+ return;
1929
+ }
1930
+ return list.map((entry, index) => ({
1931
+ entry,
1932
+ path: at(index)
1933
+ }));
1934
+ }
1935
+ static set(run, name, value, location) {
1936
+ run.variables[name] = value;
1937
+ run.keys.push({
1938
+ name,
1939
+ location
1940
+ });
1941
+ if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
1942
+ }
1943
+ };
1663
1944
  // src/scenarios/scenarioLoader.ts
1664
1945
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
1665
1946
  var ScenarioLoader = class ScenarioLoader {
@@ -1668,6 +1949,7 @@ var ScenarioLoader = class ScenarioLoader {
1668
1949
  schemaValidator = new SchemaValidator(conversation_schema_default);
1669
1950
  conversationValidator = new ConversationValidator();
1670
1951
  variablesReader = new VariablesReader();
1952
+ runWithReader = new RunWithReader();
1671
1953
  webSocketResolver = new WebSocketResolver();
1672
1954
  environmentResolver;
1673
1955
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
@@ -1683,45 +1965,59 @@ var ScenarioLoader = class ScenarioLoader {
1683
1965
  for (const file of files) {
1684
1966
  const document = this.documentLoader.load(file, (0, node_path.relative)(this.cwd, file));
1685
1967
  const fileIssues = [...document.errors];
1686
- const secrets = [];
1968
+ const fileSecrets = [];
1687
1969
  const used = /* @__PURE__ */ new Set();
1688
- const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
1689
- const resolver = this.environmentResolver.withLevels([variables, config.variables]);
1690
- let value;
1691
- if (fileIssues.length === 0) {
1692
- const { variables: raw, ...rest } = document.value;
1693
- value = {
1694
- ...resolver.resolve(rest, document, fileIssues, secrets, { used }),
1695
- ...raw === void 0 ? {} : { variables }
1696
- };
1970
+ const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
1971
+ const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
1972
+ const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
1973
+ if (fileIssues.length > 0) {
1974
+ issues.push(...fileIssues);
1975
+ continue;
1697
1976
  }
1698
- let websocket;
1699
- const websocketIssues = [];
1700
- if (fileIssues.length === 0 && configDocument !== void 0) {
1701
- const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
1702
- if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
1703
- connectionIssues.push({
1704
- file: document,
1705
- issues: websocketIssues
1706
- });
1707
- if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
1977
+ const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
1978
+ const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
1979
+ for (const result of results) for (const name of result.used) used.add(name);
1980
+ fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
1981
+ run: result.run,
1982
+ issues: result.issues
1983
+ })), (issue, run) => run === void 0 ? issue : new ValidationIssue({
1984
+ message: `run '${run.name}': ${issue.message}`,
1985
+ location: issue.location
1986
+ })));
1987
+ const attempted = results.filter((result) => result.connectionChecked);
1988
+ if (attempted.length > 0) connectionIssues.push({
1989
+ file: document,
1990
+ runs: attempted.map((result) => ({
1991
+ run: result.run,
1992
+ issues: result.websocketIssues
1993
+ }))
1994
+ });
1995
+ if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
1996
+ if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
1997
+ const websocket = result.websocket;
1998
+ specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
1708
1999
  }
1709
- if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
1710
- if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
1711
- if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
1712
- if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
1713
2000
  else issues.push(...fileIssues);
1714
2001
  }
1715
- const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
2002
+ const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
1716
2003
  if (shared.length > 0) throw new EvalError({
1717
2004
  category: ErrorCategory.Configuration,
1718
2005
  message: "invalid configuration",
1719
2006
  issues: shared
1720
2007
  });
1721
- for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
1722
- message: issue.toString(),
1723
- location: ScenarioLoader.variablesLocation(entry.file)
1724
- }));
2008
+ for (const entry of connectionIssues) {
2009
+ const fileLocation = ScenarioLoader.variablesLocation(entry.file);
2010
+ issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
2011
+ message: issue.toString(),
2012
+ location: fileLocation
2013
+ }) : new ValidationIssue({
2014
+ message: `run '${run.name}': ${issue.toString()}`,
2015
+ location: new SourceLocation({
2016
+ ...entry.file.locate(run.location),
2017
+ path: ""
2018
+ })
2019
+ })));
2020
+ }
1725
2021
  if (issues.length > 0) throw new EvalError({
1726
2022
  category: ErrorCategory.Specification,
1727
2023
  message: "invalid scenario specification",
@@ -1729,6 +2025,43 @@ var ScenarioLoader = class ScenarioLoader {
1729
2025
  });
1730
2026
  return specs;
1731
2027
  }
2028
+ resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
2029
+ const issues = [];
2030
+ const used = /* @__PURE__ */ new Set();
2031
+ const resolver = scenarioResolver.withLevels([run.variables]);
2032
+ const value = {
2033
+ ...resolver.resolve(rest, document, issues, secrets, { used }),
2034
+ ...variables === void 0 ? {} : { variables }
2035
+ };
2036
+ const websocketIssues = [];
2037
+ let websocket;
2038
+ let connectionChecked = false;
2039
+ if (issues.length === 0 && configDocument !== void 0) {
2040
+ const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
2041
+ if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
2042
+ if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
2043
+ connectionChecked = true;
2044
+ }
2045
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
2046
+ if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
2047
+ return {
2048
+ run,
2049
+ value,
2050
+ used,
2051
+ secrets,
2052
+ issues,
2053
+ websocket,
2054
+ websocketIssues,
2055
+ connectionChecked
2056
+ };
2057
+ }
2058
+ static byRun(runs, place) {
2059
+ const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
2060
+ const sharedTexts = new Set(shared.map((issue) => issue.toString()));
2061
+ const result = shared.map((issue) => place(issue, void 0));
2062
+ for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
2063
+ return result;
2064
+ }
1732
2065
  static shared(lists) {
1733
2066
  if (lists.length === 0) return [];
1734
2067
  const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
@@ -1740,11 +2073,22 @@ var ScenarioLoader = class ScenarioLoader {
1740
2073
  path: ""
1741
2074
  });
1742
2075
  }
1743
- static unusedVariables(variables, used, document) {
1744
- return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
1745
- message: `variable '${name}' is not used by the scenario or the websocket configuration`,
2076
+ static unusedVariables(variables, used, results, document) {
2077
+ const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
2078
+ const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
2079
+ message: message(name),
1746
2080
  location: document.locateKey(["variables"], name)
1747
2081
  }));
2082
+ const seen = /* @__PURE__ */ new Set();
2083
+ for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
2084
+ const issue = new ValidationIssue({
2085
+ message: message(key.name),
2086
+ location: key.location
2087
+ });
2088
+ if (!seen.has(issue.toString())) issues.push(issue);
2089
+ seen.add(issue.toString());
2090
+ }
2091
+ return issues;
1748
2092
  }
1749
2093
  discover(target, config) {
1750
2094
  if (!(0, node_fs.existsSync)(target)) throw new EvalError({
@@ -1770,13 +2114,14 @@ var ScenarioLoader = class ScenarioLoader {
1770
2114
  }
1771
2115
  return files;
1772
2116
  }
1773
- build(value, document, websocket, secrets) {
2117
+ build(value, document, runName, websocket, secrets) {
1774
2118
  const scenario = value;
1775
2119
  const turns = scenario.turns;
1776
2120
  const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
1777
2121
  return new ConversationSpec({
1778
2122
  version: scenario.version,
1779
2123
  name: scenario.name,
2124
+ runName,
1780
2125
  description: typeof scenario.description === "string" ? scenario.description : "",
1781
2126
  turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
1782
2127
  conversation: conversation ? new ConversationExpectation({
@@ -3212,61 +3557,94 @@ var ExitCodeCalculator = class {
3212
3557
  }
3213
3558
  };
3214
3559
  // src/reporting/consoleReporter.ts
3215
- const STATUS_WIDTH = 11;
3216
- const COLOURS = {
3217
- [ResultStatus.Pass]: "green",
3218
- [ResultStatus.Flaky]: "yellow",
3219
- [ResultStatus.Fail]: "red",
3220
- [ResultStatus.Error]: "red",
3221
- [ResultStatus.Skipped]: "dim",
3222
- [ResultStatus.Interrupted]: "dim"
3560
+ const MARKS = {
3561
+ [ResultStatus.Pass]: {
3562
+ symbol: "✔",
3563
+ colour: "green"
3564
+ },
3565
+ [ResultStatus.Flaky]: {
3566
+ symbol: "⚠",
3567
+ colour: "yellow"
3568
+ },
3569
+ [ResultStatus.Fail]: {
3570
+ symbol: "✖",
3571
+ colour: "red"
3572
+ },
3573
+ [ResultStatus.Error]: {
3574
+ symbol: "●",
3575
+ colour: "red"
3576
+ },
3577
+ [ResultStatus.Skipped]: {
3578
+ symbol: "○",
3579
+ colour: "dim"
3580
+ },
3581
+ [ResultStatus.Interrupted]: {
3582
+ symbol: "◌",
3583
+ colour: "dim"
3584
+ }
3223
3585
  };
3586
+ const MIN_COLUMNS = 40;
3587
+ const RULE_WIDTH = 72;
3224
3588
  var ConsoleReporter = class ConsoleReporter {
3225
3589
  verbose;
3226
3590
  colour;
3227
- constructor(verbose, colour) {
3591
+ nameWidth;
3592
+ columns;
3593
+ thresholds;
3594
+ constructor(verbose, colour, options = {}) {
3228
3595
  this.verbose = verbose;
3229
3596
  this.colour = colour;
3597
+ this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
3598
+ this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
3599
+ this.thresholds = options.thresholds;
3600
+ }
3601
+ static nameWidth(specs) {
3602
+ return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
3230
3603
  }
3231
3604
  header(run) {
3232
3605
  return [
3233
- "Miniwat Eval",
3606
+ " Miniwat Eval",
3234
3607
  "",
3235
- ...run.quick ? ["Execution Mode: Quick"] : [],
3236
- `Repeat: ${run.repeat}`,
3237
- `Minimum Pass: ${run.minPass}`,
3608
+ ...run.quick ? [" Execution mode Quick"] : [],
3609
+ ` Repeat ${run.repeat}`,
3610
+ ` Minimum pass ${run.minPass}`,
3238
3611
  "",
3239
- `Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
3612
+ ` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
3240
3613
  "",
3241
3614
  ""
3242
3615
  ].join("\n");
3243
3616
  }
3244
3617
  scenario(result, repeat, tokenUsageCapability) {
3245
3618
  const spec = result.spec;
3246
- const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
3247
- const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
3619
+ const title = ConsoleReporter.title(spec);
3620
+ const started = result.attempts.length > 0;
3621
+ const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
3622
+ const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
3623
+ const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
3248
3624
  if (!this.verbose) return line;
3249
- if (result.attempts.length === 0) return `${line}\n`;
3625
+ if (!started) return `${line}\n`;
3250
3626
  const websocket = spec.websocket;
3251
3627
  const subprotocols = websocket?.connection.subprotocols.length ?? 0;
3252
- const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
3253
- for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3254
- lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
3255
- lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
3256
- lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
3257
- lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3628
+ const lines = [];
3629
+ if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
3630
+ for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3631
+ lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
3632
+ lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3258
3633
  return line + lines.join("\n");
3259
3634
  }
3635
+ static title(spec) {
3636
+ return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
3637
+ }
3260
3638
  summary(run, exitCode, reportPaths = []) {
3261
- const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
3639
+ const started = run.scenarios.filter((result) => result.attempts.length > 0);
3640
+ const failures = started.flatMap((result) => this.blocks(result, false));
3641
+ const flaky = started.flatMap((result) => this.blocks(result, true));
3262
3642
  const lines = this.verbose ? [] : [""];
3263
- if (problems.length > 0) {
3264
- lines.push("Problems", "");
3265
- for (const result of problems) lines.push(...this.problems(result), "");
3266
- }
3643
+ if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
3644
+ if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
3267
3645
  const tests = run.scenarios.flatMap((result) => result.tests);
3268
- lines.push(...run.status === RunStatus.Interrupted ? [`Run: ${run.status}`] : [], `Scenarios: ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, `Tests: ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability), ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true), `Duration: ${ConsoleReporter.duration(run.durationMs)}`, "", "");
3269
- return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
3646
+ lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
3647
+ return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
3270
3648
  }
3271
3649
  static footer(run, exitCode, reportPaths) {
3272
3650
  return [
@@ -3280,57 +3658,135 @@ var ConsoleReporter = class ConsoleReporter {
3280
3658
  return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
3281
3659
  }
3282
3660
  static duration(ms) {
3661
+ if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
3283
3662
  const seconds = Math.floor(ms / 1e3);
3284
3663
  const pad = (value) => String(value).padStart(2, "0");
3285
3664
  const hours = Math.floor(seconds / 3600);
3286
- const rest = `${pad(Math.floor(seconds / 60) % 60)}:${pad(seconds % 60)}`;
3287
- return hours > 0 ? `${hours}:${rest}` : rest;
3665
+ const minutes = Math.floor(seconds / 60) % 60;
3666
+ return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
3288
3667
  }
3289
- problems(result) {
3668
+ static wrap(text, width) {
3669
+ const lines = [];
3670
+ for (const paragraph of text.split("\n")) {
3671
+ let line = "";
3672
+ for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
3673
+ lines.push(line);
3674
+ line = word;
3675
+ } else line = line === "" ? word : `${line} ${word}`;
3676
+ lines.push(line);
3677
+ }
3678
+ return lines;
3679
+ }
3680
+ section(name, blocks) {
3681
+ return [
3682
+ ...this.rule(name),
3683
+ "",
3684
+ ...blocks.flatMap((block) => [...block, ""])
3685
+ ];
3686
+ }
3687
+ rule(name) {
3688
+ const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
3689
+ return [
3690
+ rule,
3691
+ ` ${name}`,
3692
+ rule
3693
+ ];
3694
+ }
3695
+ blocks(result, flaky) {
3290
3696
  const spec = result.spec;
3291
- const lines = [`${this.status(result.status, 6)} ${spec.name} (${spec.location.file})`];
3292
- for (const attempt of result.attempts) {
3293
- const execution = attempt.execution;
3294
- if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3295
- for (const error of execution.attemptErrors) lines.push(` Attempt ${attempt.attempt} error: ${error.category} error: ${error.message}`);
3296
- if (execution.closeError !== void 0) lines.push(` Attempt ${attempt.attempt} close error: ${execution.closeError.message}`);
3297
- }
3298
- for (const test of result.tests.filter((entry) => entry.status !== ResultStatus.Pass)) {
3299
- lines.push(` ${this.status(test.status, 6)} ${ConsoleReporter.testTitle(test)} ${test.location.toString()} ${ConsoleReporter.counts(test)}`);
3300
- test.history.forEach((entry, index) => {
3301
- if (entry.status === ResultStatus.Pass) return;
3302
- const prefix = ` Attempt ${result.attempts[index].attempt} ${this.status(entry.status)}`;
3303
- const width = ` Attempt ${result.attempts[index].attempt} ${entry.status}: `.length;
3304
- if (entry.details.length === 0) {
3305
- lines.push(prefix);
3306
- return;
3307
- }
3308
- lines.push(`${prefix}: ${ConsoleReporter.indent(entry.details[0], width)}`, ...entry.details.slice(1).map((detail) => `${" ".repeat(width)}${ConsoleReporter.indent(detail, width)}`));
3309
- });
3697
+ const title = ConsoleReporter.title(spec);
3698
+ const blocks = [];
3699
+ if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
3700
+ const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
3701
+ for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
3702
+ return blocks;
3703
+ }
3704
+ attemptBlock(spec, title, attempt) {
3705
+ const execution = attempt.execution;
3706
+ const problems = [];
3707
+ if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
3708
+ for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
3709
+ if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
3710
+ if (problems.length === 0) return [];
3711
+ const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
3712
+ const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
3713
+ return [[
3714
+ ` scenario at ${spec.location.file}`,
3715
+ ` ${this.mark(status)} ${title} › ${what}`,
3716
+ ...problems.flatMap((problem) => this.paragraph(problem, 3))
3717
+ ]];
3718
+ }
3719
+ testBlock(spec, title, test, result) {
3720
+ const location = test.location;
3721
+ const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
3722
+ const failing = test.history.map((entry, index) => ({
3723
+ entry,
3724
+ attempt: result.attempts[index]?.attempt ?? index + 1
3725
+ })).filter(({ entry }) => entry.status !== ResultStatus.Pass);
3726
+ const several = test.history.length > 1;
3727
+ for (const { entry, attempt } of failing) {
3728
+ if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
3729
+ lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
3310
3730
  }
3731
+ lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
3311
3732
  return lines;
3312
3733
  }
3734
+ entryBody(spec, entry, indent) {
3735
+ const pad = " ".repeat(indent);
3736
+ const evaluation = entry.evaluation;
3737
+ if (evaluation === void 0) {
3738
+ if (entry.details.length === 0) return [];
3739
+ return [
3740
+ "",
3741
+ `${pad}Details`,
3742
+ ...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
3743
+ ];
3744
+ }
3745
+ const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
3746
+ const expected = ConsoleReporter.expected(spec, entry);
3747
+ const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
3748
+ if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
3749
+ if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
3750
+ if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
3751
+ return lines;
3752
+ }
3753
+ static expected(spec, entry) {
3754
+ if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
3755
+ return spec.turns[entry.turn - 1]?.expect.response?.criteria;
3756
+ }
3757
+ paragraph(text, indent) {
3758
+ const pad = " ".repeat(indent);
3759
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
3760
+ }
3761
+ bullet(text, indent) {
3762
+ const pad = " ".repeat(indent);
3763
+ return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
3764
+ }
3313
3765
  static testTitle(test) {
3314
3766
  return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
3315
3767
  }
3316
3768
  static tally(statuses) {
3317
3769
  const count = (status) => statuses.filter((entry) => entry === status).length;
3318
- const interrupted = count(ResultStatus.Interrupted);
3319
- return `${count(ResultStatus.Pass)} passed, ${count(ResultStatus.Flaky)} flaky, ${count(ResultStatus.Fail)} failed, ${count(ResultStatus.Error)} errors${interrupted > 0 ? `, ${interrupted} interrupted` : ""}`;
3320
- }
3321
- status(status, width = 0) {
3322
- const padding = " ".repeat(Math.max(0, width - status.length));
3323
- return (this.colour ? (0, node_util.styleText)(COLOURS[status], status, { validateStream: false }) : status) + padding;
3324
- }
3325
- static counts(result) {
3326
- return `(${result.passed} passed, ${result.failed} failed, ${result.errors} errors; required ${result.required})`;
3770
+ const parts = [
3771
+ [ResultStatus.Pass, "passed"],
3772
+ [ResultStatus.Flaky, "flaky"],
3773
+ [ResultStatus.Fail, "failed"],
3774
+ [ResultStatus.Error, "errors"],
3775
+ [ResultStatus.Interrupted, "interrupted"]
3776
+ ].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
3777
+ return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
3778
+ }
3779
+ mark(status) {
3780
+ const { symbol, colour } = MARKS[status];
3781
+ return this.colour ? (0, node_util.styleText)(colour, symbol, { validateStream: false }) : symbol;
3327
3782
  }
3328
3783
  static usage(label, usage, capability) {
3329
- if (!capability) return `${label}: n/a (not supported by host)`;
3330
- if (usage.expected === 0) return `${label}: none`;
3331
- if (usage.total === void 0) return `${label}: n/a (coverage 0/${usage.expected})`;
3784
+ const name = label.padEnd(16);
3785
+ if (!capability) return `${name} n/a (not supported by host)`;
3786
+ if (usage.expected === 0) return `${name} none`;
3787
+ if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
3332
3788
  const number = (value) => value.toLocaleString("en-US");
3333
- const total = `${label}: total ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3789
+ const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
3334
3790
  return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
3335
3791
  }
3336
3792
  formatAttempt(spec, attempt) {
@@ -3342,28 +3798,28 @@ var ConsoleReporter = class ConsoleReporter {
3342
3798
  lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
3343
3799
  lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
3344
3800
  });
3345
- if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3801
+ if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
3346
3802
  const notRun = attempt.tests.filter((test) => test.turn > ran);
3347
- if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3803
+ if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
3348
3804
  lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
3349
- for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3350
- if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3351
- lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3805
+ for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
3806
+ if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
3807
+ lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
3352
3808
  return lines;
3353
3809
  }
3354
3810
  formatTest(test) {
3355
- return [` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${detail}`)];
3811
+ return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
3356
3812
  }
3357
3813
  static formatTurn(number, input, turn) {
3358
3814
  const lines = [
3359
- ` Turn ${number} (${turn.durationMs} ms)`,
3360
- ` Input: ${ConsoleReporter.indent(input, 17)}`,
3361
- ` Response: ${ConsoleReporter.indent(turn.response, 17)}`
3815
+ ` Turn ${number} (${turn.durationMs} ms)`,
3816
+ ` Input: ${ConsoleReporter.indent(input, 22)}`,
3817
+ ` Response: ${ConsoleReporter.indent(turn.response, 22)}`
3362
3818
  ];
3363
- for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3364
- for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3365
- for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 17)}`);
3366
- if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3819
+ for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
3820
+ for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
3821
+ for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
3822
+ if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
3367
3823
  return lines;
3368
3824
  }
3369
3825
  static indent(text, width) {
@@ -3466,6 +3922,7 @@ var JsonReporter = class JsonReporter {
3466
3922
  const capability = websocket.protocol.capabilities.tokenUsage;
3467
3923
  return {
3468
3924
  name: result.spec.name,
3925
+ runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
3469
3926
  file: result.spec.location.file,
3470
3927
  status: result.status,
3471
3928
  connection: {