@mappies/miniwat-eval 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +91 -10
- package/dist/index.cjs +577 -120
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +33 -6
- package/dist/index.d.ts +34 -6
- package/dist/index.js +577 -120
- package/dist/index.js.map +1 -1
- package/dist/main.js +588 -125
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +35 -1
- package/schemas/eval-config.schema.json +6 -1
- package/schemas/report.schema.json +1 -0
package/dist/index.js
CHANGED
|
@@ -321,6 +321,7 @@ var WebSocketConfig = class {
|
|
|
321
321
|
var ConversationSpec = class {
|
|
322
322
|
version;
|
|
323
323
|
name;
|
|
324
|
+
runName;
|
|
324
325
|
description;
|
|
325
326
|
turns;
|
|
326
327
|
conversation;
|
|
@@ -330,6 +331,7 @@ var ConversationSpec = class {
|
|
|
330
331
|
constructor(data) {
|
|
331
332
|
this.version = data?.version ?? 1;
|
|
332
333
|
this.name = data?.name ?? "";
|
|
334
|
+
this.runName = data?.runName ?? "";
|
|
333
335
|
this.description = data?.description ?? "";
|
|
334
336
|
this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
|
|
335
337
|
this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
|
|
@@ -451,7 +453,7 @@ var EvalConfig = class {
|
|
|
451
453
|
this.version = data?.version ?? 1;
|
|
452
454
|
this.configPath = data?.configPath ?? "";
|
|
453
455
|
this.parametersPath = data?.parametersPath ?? "";
|
|
454
|
-
this.variables =
|
|
456
|
+
this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
|
|
455
457
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
456
458
|
this.execution = new ExecutionConfig(data?.execution);
|
|
457
459
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -766,7 +768,10 @@ var eval_config_schema_default = {
|
|
|
766
768
|
"variables": {
|
|
767
769
|
"type": "object",
|
|
768
770
|
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
769
|
-
"additionalProperties": { "type": "string" }
|
|
771
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
772
|
+
"type": "array",
|
|
773
|
+
"items": { "type": "string" }
|
|
774
|
+
}] }
|
|
770
775
|
},
|
|
771
776
|
"websocket": {
|
|
772
777
|
"type": "object",
|
|
@@ -999,7 +1004,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
999
1004
|
return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
|
|
1000
1005
|
}
|
|
1001
1006
|
resolve(value, document, issues, secrets = [], options = {}) {
|
|
1002
|
-
return this.resolveValue(value, [], document, issues, secrets, options);
|
|
1007
|
+
return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
|
|
1003
1008
|
}
|
|
1004
1009
|
static hasReference(value) {
|
|
1005
1010
|
return new RegExp(REFERENCE.source).test(value);
|
|
@@ -1007,7 +1012,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
1007
1012
|
static isSecretName(name) {
|
|
1008
1013
|
return SECRET_NAME.test(name);
|
|
1009
1014
|
}
|
|
1010
|
-
|
|
1015
|
+
find(name) {
|
|
1011
1016
|
for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
|
|
1012
1017
|
return this.environment[name];
|
|
1013
1018
|
}
|
|
@@ -1022,7 +1027,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
1022
1027
|
resolveValue(value, path, document, issues, secrets, options) {
|
|
1023
1028
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
1024
1029
|
options.used?.add(name);
|
|
1025
|
-
const resolved = this.
|
|
1030
|
+
const resolved = this.find(name);
|
|
1026
1031
|
if (resolved === void 0) {
|
|
1027
1032
|
if (options.optional?.(path)) {
|
|
1028
1033
|
options.unresolved?.add(YamlDocument.formatPath(path));
|
|
@@ -1034,6 +1039,13 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
1034
1039
|
}));
|
|
1035
1040
|
return reference;
|
|
1036
1041
|
}
|
|
1042
|
+
if (Array.isArray(resolved)) {
|
|
1043
|
+
issues.push(new ValidationIssue({
|
|
1044
|
+
message: `variable '${name}' is a list, which can only be used as runWith.from`,
|
|
1045
|
+
location: document.locate(path)
|
|
1046
|
+
}));
|
|
1047
|
+
return reference;
|
|
1048
|
+
}
|
|
1037
1049
|
if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
|
|
1038
1050
|
return resolved;
|
|
1039
1051
|
});
|
|
@@ -1162,9 +1174,25 @@ var VariablesReader = class {
|
|
|
1162
1174
|
}));
|
|
1163
1175
|
continue;
|
|
1164
1176
|
}
|
|
1177
|
+
if (Array.isArray(value)) {
|
|
1178
|
+
const bad = value.findIndex((item) => typeof item !== "string");
|
|
1179
|
+
if (bad >= 0) {
|
|
1180
|
+
issues.push(new ValidationIssue({
|
|
1181
|
+
message: "must be string",
|
|
1182
|
+
location: document.locate([
|
|
1183
|
+
"variables",
|
|
1184
|
+
name,
|
|
1185
|
+
bad
|
|
1186
|
+
])
|
|
1187
|
+
}));
|
|
1188
|
+
continue;
|
|
1189
|
+
}
|
|
1190
|
+
variables[name] = [...value];
|
|
1191
|
+
continue;
|
|
1192
|
+
}
|
|
1165
1193
|
if (typeof value !== "string") {
|
|
1166
1194
|
issues.push(new ValidationIssue({
|
|
1167
|
-
message: "must be string",
|
|
1195
|
+
message: "must be string or a list of strings",
|
|
1168
1196
|
location: document.locate(["variables", name])
|
|
1169
1197
|
}));
|
|
1170
1198
|
continue;
|
|
@@ -1172,8 +1200,9 @@ var VariablesReader = class {
|
|
|
1172
1200
|
const before = issues.length;
|
|
1173
1201
|
const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
|
|
1174
1202
|
if (issues.length > before) continue;
|
|
1175
|
-
|
|
1176
|
-
|
|
1203
|
+
const text = resolved.variables[name];
|
|
1204
|
+
variables[name] = text;
|
|
1205
|
+
if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
|
|
1177
1206
|
}
|
|
1178
1207
|
return variables;
|
|
1179
1208
|
}
|
|
@@ -1478,7 +1507,44 @@ var conversation_schema_default = {
|
|
|
1478
1507
|
"variables": {
|
|
1479
1508
|
"type": "object",
|
|
1480
1509
|
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
1481
|
-
"additionalProperties": { "type": "string" }
|
|
1510
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
1511
|
+
"type": "array",
|
|
1512
|
+
"items": { "type": "string" }
|
|
1513
|
+
}] }
|
|
1514
|
+
},
|
|
1515
|
+
"runWith": {
|
|
1516
|
+
"description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
|
|
1517
|
+
"anyOf": [{
|
|
1518
|
+
"type": "array",
|
|
1519
|
+
"minItems": 1,
|
|
1520
|
+
"items": {
|
|
1521
|
+
"type": "object",
|
|
1522
|
+
"required": ["name"],
|
|
1523
|
+
"properties": { "name": {
|
|
1524
|
+
"type": "string",
|
|
1525
|
+
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
|
|
1526
|
+
} },
|
|
1527
|
+
"additionalProperties": { "type": "string" }
|
|
1528
|
+
}
|
|
1529
|
+
}, {
|
|
1530
|
+
"type": "object",
|
|
1531
|
+
"additionalProperties": false,
|
|
1532
|
+
"required": ["variable", "from"],
|
|
1533
|
+
"properties": {
|
|
1534
|
+
"variable": {
|
|
1535
|
+
"type": "string",
|
|
1536
|
+
"pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
|
|
1537
|
+
},
|
|
1538
|
+
"from": { "anyOf": [{
|
|
1539
|
+
"type": "array",
|
|
1540
|
+
"minItems": 1,
|
|
1541
|
+
"items": { "type": "string" }
|
|
1542
|
+
}, {
|
|
1543
|
+
"type": "string",
|
|
1544
|
+
"pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
|
|
1545
|
+
}] }
|
|
1546
|
+
}
|
|
1547
|
+
}]
|
|
1482
1548
|
},
|
|
1483
1549
|
"turns": {
|
|
1484
1550
|
"type": "array",
|
|
@@ -1637,6 +1703,221 @@ var ConversationValidator = class ConversationValidator {
|
|
|
1637
1703
|
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
1638
1704
|
}
|
|
1639
1705
|
};
|
|
1706
|
+
// src/scenarios/runWithReader.ts
|
|
1707
|
+
const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
|
|
1708
|
+
const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
|
|
1709
|
+
const FORMS = "must be a list of rows, or an object with variable and from";
|
|
1710
|
+
const SINGLE_RUN = {
|
|
1711
|
+
name: "",
|
|
1712
|
+
variables: {},
|
|
1713
|
+
keys: [],
|
|
1714
|
+
location: [],
|
|
1715
|
+
secrets: []
|
|
1716
|
+
};
|
|
1717
|
+
var RunWithReader = class RunWithReader {
|
|
1718
|
+
read(document, resolver, issues, used) {
|
|
1719
|
+
const raw = isRecord(document.value) ? document.value.runWith : void 0;
|
|
1720
|
+
if (raw === void 0) return void 0;
|
|
1721
|
+
if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
|
|
1722
|
+
if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
|
|
1723
|
+
issues.push(new ValidationIssue({
|
|
1724
|
+
message: FORMS,
|
|
1725
|
+
location: document.locate(["runWith"])
|
|
1726
|
+
}));
|
|
1727
|
+
return [];
|
|
1728
|
+
}
|
|
1729
|
+
rows(rows, document, resolver, issues, used) {
|
|
1730
|
+
if (rows.length === 0) {
|
|
1731
|
+
issues.push(new ValidationIssue({
|
|
1732
|
+
message: "must not be empty",
|
|
1733
|
+
location: document.locate(["runWith"])
|
|
1734
|
+
}));
|
|
1735
|
+
return [];
|
|
1736
|
+
}
|
|
1737
|
+
const names = /* @__PURE__ */ new Set();
|
|
1738
|
+
const runs = [];
|
|
1739
|
+
rows.forEach((row, index) => {
|
|
1740
|
+
const path = ["runWith", index];
|
|
1741
|
+
if (!isRecord(row)) {
|
|
1742
|
+
issues.push(new ValidationIssue({
|
|
1743
|
+
message: "must be object",
|
|
1744
|
+
location: document.locate(path)
|
|
1745
|
+
}));
|
|
1746
|
+
return;
|
|
1747
|
+
}
|
|
1748
|
+
const name = row.name;
|
|
1749
|
+
if (name === void 0) issues.push(new ValidationIssue({
|
|
1750
|
+
message: "missing required key 'name'",
|
|
1751
|
+
location: document.locate(path)
|
|
1752
|
+
}));
|
|
1753
|
+
else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
|
|
1754
|
+
message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
|
|
1755
|
+
location: document.locate([...path, "name"])
|
|
1756
|
+
}));
|
|
1757
|
+
else if (names.has(name)) issues.push(new ValidationIssue({
|
|
1758
|
+
message: `duplicate run name '${name}'`,
|
|
1759
|
+
location: document.locate([...path, "name"])
|
|
1760
|
+
}));
|
|
1761
|
+
else names.add(name);
|
|
1762
|
+
const run = {
|
|
1763
|
+
name: typeof name === "string" ? name : "",
|
|
1764
|
+
variables: {},
|
|
1765
|
+
keys: [],
|
|
1766
|
+
location: path,
|
|
1767
|
+
secrets: []
|
|
1768
|
+
};
|
|
1769
|
+
for (const [key, value] of Object.entries(row)) {
|
|
1770
|
+
if (key === "name") continue;
|
|
1771
|
+
if (!VARIABLE_NAME.test(key)) {
|
|
1772
|
+
issues.push(new ValidationIssue({
|
|
1773
|
+
message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1774
|
+
location: document.locateKey(path, key)
|
|
1775
|
+
}));
|
|
1776
|
+
continue;
|
|
1777
|
+
}
|
|
1778
|
+
if (typeof value !== "string") {
|
|
1779
|
+
issues.push(new ValidationIssue({
|
|
1780
|
+
message: "must be string",
|
|
1781
|
+
location: document.locate([...path, key])
|
|
1782
|
+
}));
|
|
1783
|
+
continue;
|
|
1784
|
+
}
|
|
1785
|
+
const resolved = resolver.resolve(value, document, issues, run.secrets, {
|
|
1786
|
+
used,
|
|
1787
|
+
at: [...path, key]
|
|
1788
|
+
});
|
|
1789
|
+
RunWithReader.set(run, key, resolved, document.locateKey(path, key));
|
|
1790
|
+
}
|
|
1791
|
+
runs.push(run);
|
|
1792
|
+
});
|
|
1793
|
+
return runs;
|
|
1794
|
+
}
|
|
1795
|
+
list(form, document, resolver, issues, used) {
|
|
1796
|
+
const before = issues.length;
|
|
1797
|
+
for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
|
|
1798
|
+
message: `unknown key '${key}'`,
|
|
1799
|
+
location: document.locateKey(["runWith"], key)
|
|
1800
|
+
}));
|
|
1801
|
+
const variable = form.variable;
|
|
1802
|
+
if (variable === void 0) issues.push(new ValidationIssue({
|
|
1803
|
+
message: "missing required key 'variable'",
|
|
1804
|
+
location: document.locate(["runWith"])
|
|
1805
|
+
}));
|
|
1806
|
+
else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
|
|
1807
|
+
message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
|
|
1808
|
+
location: document.locate(["runWith", "variable"])
|
|
1809
|
+
}));
|
|
1810
|
+
const entries = this.entries(form.from, document, resolver, issues, used);
|
|
1811
|
+
if (issues.length > before || entries === void 0) return [];
|
|
1812
|
+
const name = variable;
|
|
1813
|
+
const keyLocation = document.locate(["runWith", "variable"]);
|
|
1814
|
+
const names = /* @__PURE__ */ new Set();
|
|
1815
|
+
const runs = [];
|
|
1816
|
+
for (const { entry, path } of entries) {
|
|
1817
|
+
if (!VARIABLE_NAME.test(entry)) {
|
|
1818
|
+
issues.push(new ValidationIssue({
|
|
1819
|
+
message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1820
|
+
location: document.locate(path)
|
|
1821
|
+
}));
|
|
1822
|
+
continue;
|
|
1823
|
+
}
|
|
1824
|
+
if (names.has(entry)) {
|
|
1825
|
+
issues.push(new ValidationIssue({
|
|
1826
|
+
message: `duplicate run name '${entry}'`,
|
|
1827
|
+
location: document.locate(path)
|
|
1828
|
+
}));
|
|
1829
|
+
continue;
|
|
1830
|
+
}
|
|
1831
|
+
names.add(entry);
|
|
1832
|
+
const run = {
|
|
1833
|
+
name: entry,
|
|
1834
|
+
variables: {},
|
|
1835
|
+
keys: [],
|
|
1836
|
+
location: path,
|
|
1837
|
+
secrets: []
|
|
1838
|
+
};
|
|
1839
|
+
const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
|
|
1840
|
+
used,
|
|
1841
|
+
at: path
|
|
1842
|
+
});
|
|
1843
|
+
RunWithReader.set(run, name, value, keyLocation);
|
|
1844
|
+
runs.push(run);
|
|
1845
|
+
}
|
|
1846
|
+
return runs;
|
|
1847
|
+
}
|
|
1848
|
+
entries(from, document, resolver, issues, used) {
|
|
1849
|
+
const path = ["runWith", "from"];
|
|
1850
|
+
const location = document.locate(path);
|
|
1851
|
+
if (from === void 0) {
|
|
1852
|
+
issues.push(new ValidationIssue({
|
|
1853
|
+
message: "missing required key 'from'",
|
|
1854
|
+
location: document.locate(["runWith"])
|
|
1855
|
+
}));
|
|
1856
|
+
return;
|
|
1857
|
+
}
|
|
1858
|
+
let list;
|
|
1859
|
+
let at;
|
|
1860
|
+
if (Array.isArray(from)) {
|
|
1861
|
+
list = from;
|
|
1862
|
+
at = (index) => [...path, index];
|
|
1863
|
+
} else {
|
|
1864
|
+
const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
|
|
1865
|
+
if (reference === null) {
|
|
1866
|
+
issues.push(new ValidationIssue({
|
|
1867
|
+
message: "must be a list, or a list variable such as ${tokens}",
|
|
1868
|
+
location
|
|
1869
|
+
}));
|
|
1870
|
+
return;
|
|
1871
|
+
}
|
|
1872
|
+
const name = reference[1];
|
|
1873
|
+
const found = resolver.find(name);
|
|
1874
|
+
used.add(name);
|
|
1875
|
+
if (found === void 0) {
|
|
1876
|
+
issues.push(new ValidationIssue({
|
|
1877
|
+
message: `list variable '${name}' is not set`,
|
|
1878
|
+
location
|
|
1879
|
+
}));
|
|
1880
|
+
return;
|
|
1881
|
+
}
|
|
1882
|
+
if (!Array.isArray(found)) {
|
|
1883
|
+
issues.push(new ValidationIssue({
|
|
1884
|
+
message: `variable '${name}' is not a list`,
|
|
1885
|
+
location
|
|
1886
|
+
}));
|
|
1887
|
+
return;
|
|
1888
|
+
}
|
|
1889
|
+
list = found;
|
|
1890
|
+
at = () => path;
|
|
1891
|
+
}
|
|
1892
|
+
if (list.length === 0) {
|
|
1893
|
+
issues.push(new ValidationIssue({
|
|
1894
|
+
message: "must not be empty",
|
|
1895
|
+
location
|
|
1896
|
+
}));
|
|
1897
|
+
return;
|
|
1898
|
+
}
|
|
1899
|
+
const bad = list.findIndex((item) => typeof item !== "string");
|
|
1900
|
+
if (bad >= 0) {
|
|
1901
|
+
issues.push(new ValidationIssue({
|
|
1902
|
+
message: "must be string",
|
|
1903
|
+
location: document.locate(at(bad))
|
|
1904
|
+
}));
|
|
1905
|
+
return;
|
|
1906
|
+
}
|
|
1907
|
+
return list.map((entry, index) => ({
|
|
1908
|
+
entry,
|
|
1909
|
+
path: at(index)
|
|
1910
|
+
}));
|
|
1911
|
+
}
|
|
1912
|
+
static set(run, name, value, location) {
|
|
1913
|
+
run.variables[name] = value;
|
|
1914
|
+
run.keys.push({
|
|
1915
|
+
name,
|
|
1916
|
+
location
|
|
1917
|
+
});
|
|
1918
|
+
if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
|
|
1919
|
+
}
|
|
1920
|
+
};
|
|
1640
1921
|
// src/scenarios/scenarioLoader.ts
|
|
1641
1922
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
1642
1923
|
var ScenarioLoader = class ScenarioLoader {
|
|
@@ -1645,6 +1926,7 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1645
1926
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
1646
1927
|
conversationValidator = new ConversationValidator();
|
|
1647
1928
|
variablesReader = new VariablesReader();
|
|
1929
|
+
runWithReader = new RunWithReader();
|
|
1648
1930
|
webSocketResolver = new WebSocketResolver();
|
|
1649
1931
|
environmentResolver;
|
|
1650
1932
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
@@ -1660,45 +1942,59 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1660
1942
|
for (const file of files) {
|
|
1661
1943
|
const document = this.documentLoader.load(file, relative(this.cwd, file));
|
|
1662
1944
|
const fileIssues = [...document.errors];
|
|
1663
|
-
const
|
|
1945
|
+
const fileSecrets = [];
|
|
1664
1946
|
const used = /* @__PURE__ */ new Set();
|
|
1665
|
-
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues,
|
|
1666
|
-
const
|
|
1667
|
-
|
|
1668
|
-
if (fileIssues.length
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
...resolver.resolve(rest, document, fileIssues, secrets, { used }),
|
|
1672
|
-
...raw === void 0 ? {} : { variables }
|
|
1673
|
-
};
|
|
1947
|
+
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
|
|
1948
|
+
const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
|
|
1949
|
+
const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
|
|
1950
|
+
if (fileIssues.length > 0) {
|
|
1951
|
+
issues.push(...fileIssues);
|
|
1952
|
+
continue;
|
|
1674
1953
|
}
|
|
1675
|
-
|
|
1676
|
-
const
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1954
|
+
const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
|
|
1955
|
+
const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
|
|
1956
|
+
for (const result of results) for (const name of result.used) used.add(name);
|
|
1957
|
+
fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
|
|
1958
|
+
run: result.run,
|
|
1959
|
+
issues: result.issues
|
|
1960
|
+
})), (issue, run) => run === void 0 ? issue : new ValidationIssue({
|
|
1961
|
+
message: `run '${run.name}': ${issue.message}`,
|
|
1962
|
+
location: issue.location
|
|
1963
|
+
})));
|
|
1964
|
+
const attempted = results.filter((result) => result.connectionChecked);
|
|
1965
|
+
if (attempted.length > 0) connectionIssues.push({
|
|
1966
|
+
file: document,
|
|
1967
|
+
runs: attempted.map((result) => ({
|
|
1968
|
+
run: result.run,
|
|
1969
|
+
issues: result.websocketIssues
|
|
1970
|
+
}))
|
|
1971
|
+
});
|
|
1972
|
+
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
|
|
1973
|
+
if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
|
|
1974
|
+
const websocket = result.websocket;
|
|
1975
|
+
specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
1685
1976
|
}
|
|
1686
|
-
if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
|
|
1687
|
-
if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
|
|
1688
|
-
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
|
|
1689
|
-
if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
1690
1977
|
else issues.push(...fileIssues);
|
|
1691
1978
|
}
|
|
1692
|
-
const shared = ScenarioLoader.shared(connectionIssues.
|
|
1979
|
+
const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
|
|
1693
1980
|
if (shared.length > 0) throw new EvalError({
|
|
1694
1981
|
category: ErrorCategory.Configuration,
|
|
1695
1982
|
message: "invalid configuration",
|
|
1696
1983
|
issues: shared
|
|
1697
1984
|
});
|
|
1698
|
-
for (const entry of connectionIssues)
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1985
|
+
for (const entry of connectionIssues) {
|
|
1986
|
+
const fileLocation = ScenarioLoader.variablesLocation(entry.file);
|
|
1987
|
+
issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
|
|
1988
|
+
message: issue.toString(),
|
|
1989
|
+
location: fileLocation
|
|
1990
|
+
}) : new ValidationIssue({
|
|
1991
|
+
message: `run '${run.name}': ${issue.toString()}`,
|
|
1992
|
+
location: new SourceLocation({
|
|
1993
|
+
...entry.file.locate(run.location),
|
|
1994
|
+
path: ""
|
|
1995
|
+
})
|
|
1996
|
+
})));
|
|
1997
|
+
}
|
|
1702
1998
|
if (issues.length > 0) throw new EvalError({
|
|
1703
1999
|
category: ErrorCategory.Specification,
|
|
1704
2000
|
message: "invalid scenario specification",
|
|
@@ -1706,6 +2002,43 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1706
2002
|
});
|
|
1707
2003
|
return specs;
|
|
1708
2004
|
}
|
|
2005
|
+
resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
|
|
2006
|
+
const issues = [];
|
|
2007
|
+
const used = /* @__PURE__ */ new Set();
|
|
2008
|
+
const resolver = scenarioResolver.withLevels([run.variables]);
|
|
2009
|
+
const value = {
|
|
2010
|
+
...resolver.resolve(rest, document, issues, secrets, { used }),
|
|
2011
|
+
...variables === void 0 ? {} : { variables }
|
|
2012
|
+
};
|
|
2013
|
+
const websocketIssues = [];
|
|
2014
|
+
let websocket;
|
|
2015
|
+
let connectionChecked = false;
|
|
2016
|
+
if (issues.length === 0 && configDocument !== void 0) {
|
|
2017
|
+
const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
|
|
2018
|
+
if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
|
|
2019
|
+
if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
|
|
2020
|
+
connectionChecked = true;
|
|
2021
|
+
}
|
|
2022
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
|
|
2023
|
+
if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
|
|
2024
|
+
return {
|
|
2025
|
+
run,
|
|
2026
|
+
value,
|
|
2027
|
+
used,
|
|
2028
|
+
secrets,
|
|
2029
|
+
issues,
|
|
2030
|
+
websocket,
|
|
2031
|
+
websocketIssues,
|
|
2032
|
+
connectionChecked
|
|
2033
|
+
};
|
|
2034
|
+
}
|
|
2035
|
+
static byRun(runs, place) {
|
|
2036
|
+
const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
|
|
2037
|
+
const sharedTexts = new Set(shared.map((issue) => issue.toString()));
|
|
2038
|
+
const result = shared.map((issue) => place(issue, void 0));
|
|
2039
|
+
for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
|
|
2040
|
+
return result;
|
|
2041
|
+
}
|
|
1709
2042
|
static shared(lists) {
|
|
1710
2043
|
if (lists.length === 0) return [];
|
|
1711
2044
|
const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
|
|
@@ -1717,11 +2050,22 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1717
2050
|
path: ""
|
|
1718
2051
|
});
|
|
1719
2052
|
}
|
|
1720
|
-
static unusedVariables(variables, used, document) {
|
|
1721
|
-
|
|
1722
|
-
|
|
2053
|
+
static unusedVariables(variables, used, results, document) {
|
|
2054
|
+
const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
|
|
2055
|
+
const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
|
|
2056
|
+
message: message(name),
|
|
1723
2057
|
location: document.locateKey(["variables"], name)
|
|
1724
2058
|
}));
|
|
2059
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2060
|
+
for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
|
|
2061
|
+
const issue = new ValidationIssue({
|
|
2062
|
+
message: message(key.name),
|
|
2063
|
+
location: key.location
|
|
2064
|
+
});
|
|
2065
|
+
if (!seen.has(issue.toString())) issues.push(issue);
|
|
2066
|
+
seen.add(issue.toString());
|
|
2067
|
+
}
|
|
2068
|
+
return issues;
|
|
1725
2069
|
}
|
|
1726
2070
|
discover(target, config) {
|
|
1727
2071
|
if (!existsSync(target)) throw new EvalError({
|
|
@@ -1747,13 +2091,14 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1747
2091
|
}
|
|
1748
2092
|
return files;
|
|
1749
2093
|
}
|
|
1750
|
-
build(value, document, websocket, secrets) {
|
|
2094
|
+
build(value, document, runName, websocket, secrets) {
|
|
1751
2095
|
const scenario = value;
|
|
1752
2096
|
const turns = scenario.turns;
|
|
1753
2097
|
const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
|
|
1754
2098
|
return new ConversationSpec({
|
|
1755
2099
|
version: scenario.version,
|
|
1756
2100
|
name: scenario.name,
|
|
2101
|
+
runName,
|
|
1757
2102
|
description: typeof scenario.description === "string" ? scenario.description : "",
|
|
1758
2103
|
turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
|
|
1759
2104
|
conversation: conversation ? new ConversationExpectation({
|
|
@@ -3189,61 +3534,94 @@ var ExitCodeCalculator = class {
|
|
|
3189
3534
|
}
|
|
3190
3535
|
};
|
|
3191
3536
|
// src/reporting/consoleReporter.ts
|
|
3192
|
-
const
|
|
3193
|
-
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
[ResultStatus.
|
|
3198
|
-
|
|
3199
|
-
|
|
3537
|
+
const MARKS = {
|
|
3538
|
+
[ResultStatus.Pass]: {
|
|
3539
|
+
symbol: "✔",
|
|
3540
|
+
colour: "green"
|
|
3541
|
+
},
|
|
3542
|
+
[ResultStatus.Flaky]: {
|
|
3543
|
+
symbol: "⚠",
|
|
3544
|
+
colour: "yellow"
|
|
3545
|
+
},
|
|
3546
|
+
[ResultStatus.Fail]: {
|
|
3547
|
+
symbol: "✖",
|
|
3548
|
+
colour: "red"
|
|
3549
|
+
},
|
|
3550
|
+
[ResultStatus.Error]: {
|
|
3551
|
+
symbol: "●",
|
|
3552
|
+
colour: "red"
|
|
3553
|
+
},
|
|
3554
|
+
[ResultStatus.Skipped]: {
|
|
3555
|
+
symbol: "○",
|
|
3556
|
+
colour: "dim"
|
|
3557
|
+
},
|
|
3558
|
+
[ResultStatus.Interrupted]: {
|
|
3559
|
+
symbol: "◌",
|
|
3560
|
+
colour: "dim"
|
|
3561
|
+
}
|
|
3200
3562
|
};
|
|
3563
|
+
const MIN_COLUMNS = 40;
|
|
3564
|
+
const RULE_WIDTH = 72;
|
|
3201
3565
|
var ConsoleReporter = class ConsoleReporter {
|
|
3202
3566
|
verbose;
|
|
3203
3567
|
colour;
|
|
3204
|
-
|
|
3568
|
+
nameWidth;
|
|
3569
|
+
columns;
|
|
3570
|
+
thresholds;
|
|
3571
|
+
constructor(verbose, colour, options = {}) {
|
|
3205
3572
|
this.verbose = verbose;
|
|
3206
3573
|
this.colour = colour;
|
|
3574
|
+
this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
|
|
3575
|
+
this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
|
|
3576
|
+
this.thresholds = options.thresholds;
|
|
3577
|
+
}
|
|
3578
|
+
static nameWidth(specs) {
|
|
3579
|
+
return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
|
|
3207
3580
|
}
|
|
3208
3581
|
header(run) {
|
|
3209
3582
|
return [
|
|
3210
|
-
"Miniwat Eval",
|
|
3583
|
+
" Miniwat Eval",
|
|
3211
3584
|
"",
|
|
3212
|
-
...run.quick ? ["Execution
|
|
3213
|
-
`Repeat
|
|
3214
|
-
`Minimum
|
|
3585
|
+
...run.quick ? [" Execution mode Quick"] : [],
|
|
3586
|
+
` Repeat ${run.repeat}`,
|
|
3587
|
+
` Minimum pass ${run.minPass}`,
|
|
3215
3588
|
"",
|
|
3216
|
-
`Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
3589
|
+
` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
3217
3590
|
"",
|
|
3218
3591
|
""
|
|
3219
3592
|
].join("\n");
|
|
3220
3593
|
}
|
|
3221
3594
|
scenario(result, repeat, tokenUsageCapability) {
|
|
3222
3595
|
const spec = result.spec;
|
|
3223
|
-
const
|
|
3224
|
-
const
|
|
3596
|
+
const title = ConsoleReporter.title(spec);
|
|
3597
|
+
const started = result.attempts.length > 0;
|
|
3598
|
+
const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
|
|
3599
|
+
const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
|
|
3600
|
+
const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
|
|
3225
3601
|
if (!this.verbose) return line;
|
|
3226
|
-
if (
|
|
3602
|
+
if (!started) return `${line}\n`;
|
|
3227
3603
|
const websocket = spec.websocket;
|
|
3228
3604
|
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
3229
|
-
const lines = [
|
|
3230
|
-
|
|
3231
|
-
lines.push(`
|
|
3232
|
-
lines.push(
|
|
3233
|
-
lines.push(`
|
|
3234
|
-
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3605
|
+
const lines = [];
|
|
3606
|
+
if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
|
|
3607
|
+
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
3608
|
+
lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
|
|
3609
|
+
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3235
3610
|
return line + lines.join("\n");
|
|
3236
3611
|
}
|
|
3612
|
+
static title(spec) {
|
|
3613
|
+
return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
|
|
3614
|
+
}
|
|
3237
3615
|
summary(run, exitCode, reportPaths = []) {
|
|
3238
|
-
const
|
|
3616
|
+
const started = run.scenarios.filter((result) => result.attempts.length > 0);
|
|
3617
|
+
const failures = started.flatMap((result) => this.blocks(result, false));
|
|
3618
|
+
const flaky = started.flatMap((result) => this.blocks(result, true));
|
|
3239
3619
|
const lines = this.verbose ? [] : [""];
|
|
3240
|
-
if (
|
|
3241
|
-
|
|
3242
|
-
for (const result of problems) lines.push(...this.problems(result), "");
|
|
3243
|
-
}
|
|
3620
|
+
if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
|
|
3621
|
+
if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
|
|
3244
3622
|
const tests = run.scenarios.flatMap((result) => result.tests);
|
|
3245
|
-
lines.push(...run.status === RunStatus.Interrupted ? [`Run
|
|
3246
|
-
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
|
|
3623
|
+
lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
|
|
3624
|
+
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
|
|
3247
3625
|
}
|
|
3248
3626
|
static footer(run, exitCode, reportPaths) {
|
|
3249
3627
|
return [
|
|
@@ -3257,57 +3635,135 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3257
3635
|
return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
|
|
3258
3636
|
}
|
|
3259
3637
|
static duration(ms) {
|
|
3638
|
+
if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
|
|
3260
3639
|
const seconds = Math.floor(ms / 1e3);
|
|
3261
3640
|
const pad = (value) => String(value).padStart(2, "0");
|
|
3262
3641
|
const hours = Math.floor(seconds / 3600);
|
|
3263
|
-
const
|
|
3264
|
-
return hours > 0 ? `${hours}:${
|
|
3642
|
+
const minutes = Math.floor(seconds / 60) % 60;
|
|
3643
|
+
return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
|
|
3644
|
+
}
|
|
3645
|
+
static wrap(text, width) {
|
|
3646
|
+
const lines = [];
|
|
3647
|
+
for (const paragraph of text.split("\n")) {
|
|
3648
|
+
let line = "";
|
|
3649
|
+
for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
|
|
3650
|
+
lines.push(line);
|
|
3651
|
+
line = word;
|
|
3652
|
+
} else line = line === "" ? word : `${line} ${word}`;
|
|
3653
|
+
lines.push(line);
|
|
3654
|
+
}
|
|
3655
|
+
return lines;
|
|
3656
|
+
}
|
|
3657
|
+
section(name, blocks) {
|
|
3658
|
+
return [
|
|
3659
|
+
...this.rule(name),
|
|
3660
|
+
"",
|
|
3661
|
+
...blocks.flatMap((block) => [...block, ""])
|
|
3662
|
+
];
|
|
3663
|
+
}
|
|
3664
|
+
rule(name) {
|
|
3665
|
+
const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
|
|
3666
|
+
return [
|
|
3667
|
+
rule,
|
|
3668
|
+
` ${name}`,
|
|
3669
|
+
rule
|
|
3670
|
+
];
|
|
3265
3671
|
}
|
|
3266
|
-
|
|
3672
|
+
blocks(result, flaky) {
|
|
3267
3673
|
const spec = result.spec;
|
|
3268
|
-
const
|
|
3269
|
-
|
|
3270
|
-
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
|
|
3274
|
-
|
|
3275
|
-
|
|
3276
|
-
|
|
3277
|
-
|
|
3278
|
-
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
}
|
|
3674
|
+
const title = ConsoleReporter.title(spec);
|
|
3675
|
+
const blocks = [];
|
|
3676
|
+
if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
|
|
3677
|
+
const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
|
|
3678
|
+
for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
|
|
3679
|
+
return blocks;
|
|
3680
|
+
}
|
|
3681
|
+
attemptBlock(spec, title, attempt) {
|
|
3682
|
+
const execution = attempt.execution;
|
|
3683
|
+
const problems = [];
|
|
3684
|
+
if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
|
|
3685
|
+
for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
|
|
3686
|
+
if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
|
|
3687
|
+
if (problems.length === 0) return [];
|
|
3688
|
+
const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
|
|
3689
|
+
const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
|
|
3690
|
+
return [[
|
|
3691
|
+
` scenario at ${spec.location.file}`,
|
|
3692
|
+
` ${this.mark(status)} ${title} › ${what}`,
|
|
3693
|
+
...problems.flatMap((problem) => this.paragraph(problem, 3))
|
|
3694
|
+
]];
|
|
3695
|
+
}
|
|
3696
|
+
testBlock(spec, title, test, result) {
|
|
3697
|
+
const location = test.location;
|
|
3698
|
+
const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
|
|
3699
|
+
const failing = test.history.map((entry, index) => ({
|
|
3700
|
+
entry,
|
|
3701
|
+
attempt: result.attempts[index]?.attempt ?? index + 1
|
|
3702
|
+
})).filter(({ entry }) => entry.status !== ResultStatus.Pass);
|
|
3703
|
+
const several = test.history.length > 1;
|
|
3704
|
+
for (const { entry, attempt } of failing) {
|
|
3705
|
+
if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
|
|
3706
|
+
lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
|
|
3287
3707
|
}
|
|
3708
|
+
lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
|
|
3288
3709
|
return lines;
|
|
3289
3710
|
}
|
|
3711
|
+
entryBody(spec, entry, indent) {
|
|
3712
|
+
const pad = " ".repeat(indent);
|
|
3713
|
+
const evaluation = entry.evaluation;
|
|
3714
|
+
if (evaluation === void 0) {
|
|
3715
|
+
if (entry.details.length === 0) return [];
|
|
3716
|
+
return [
|
|
3717
|
+
"",
|
|
3718
|
+
`${pad}Details`,
|
|
3719
|
+
...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
|
|
3720
|
+
];
|
|
3721
|
+
}
|
|
3722
|
+
const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
|
|
3723
|
+
const expected = ConsoleReporter.expected(spec, entry);
|
|
3724
|
+
const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
|
|
3725
|
+
if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
|
|
3726
|
+
if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
|
|
3727
|
+
if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
|
|
3728
|
+
return lines;
|
|
3729
|
+
}
|
|
3730
|
+
static expected(spec, entry) {
|
|
3731
|
+
if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
|
|
3732
|
+
return spec.turns[entry.turn - 1]?.expect.response?.criteria;
|
|
3733
|
+
}
|
|
3734
|
+
paragraph(text, indent) {
|
|
3735
|
+
const pad = " ".repeat(indent);
|
|
3736
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
|
|
3737
|
+
}
|
|
3738
|
+
bullet(text, indent) {
|
|
3739
|
+
const pad = " ".repeat(indent);
|
|
3740
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
|
|
3741
|
+
}
|
|
3290
3742
|
static testTitle(test) {
|
|
3291
3743
|
return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
|
|
3292
3744
|
}
|
|
3293
3745
|
static tally(statuses) {
|
|
3294
3746
|
const count = (status) => statuses.filter((entry) => entry === status).length;
|
|
3295
|
-
const
|
|
3296
|
-
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
|
|
3747
|
+
const parts = [
|
|
3748
|
+
[ResultStatus.Pass, "passed"],
|
|
3749
|
+
[ResultStatus.Flaky, "flaky"],
|
|
3750
|
+
[ResultStatus.Fail, "failed"],
|
|
3751
|
+
[ResultStatus.Error, "errors"],
|
|
3752
|
+
[ResultStatus.Interrupted, "interrupted"]
|
|
3753
|
+
].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
|
|
3754
|
+
return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
|
|
3755
|
+
}
|
|
3756
|
+
mark(status) {
|
|
3757
|
+
const { symbol, colour } = MARKS[status];
|
|
3758
|
+
return this.colour ? styleText(colour, symbol, { validateStream: false }) : symbol;
|
|
3304
3759
|
}
|
|
3305
3760
|
static usage(label, usage, capability) {
|
|
3306
|
-
|
|
3307
|
-
if (
|
|
3308
|
-
if (usage.
|
|
3761
|
+
const name = label.padEnd(16);
|
|
3762
|
+
if (!capability) return `${name} n/a (not supported by host)`;
|
|
3763
|
+
if (usage.expected === 0) return `${name} none`;
|
|
3764
|
+
if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
|
|
3309
3765
|
const number = (value) => value.toLocaleString("en-US");
|
|
3310
|
-
const total = `${
|
|
3766
|
+
const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
|
|
3311
3767
|
return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
|
|
3312
3768
|
}
|
|
3313
3769
|
formatAttempt(spec, attempt) {
|
|
@@ -3319,28 +3775,28 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3319
3775
|
lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
|
|
3320
3776
|
lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
|
|
3321
3777
|
});
|
|
3322
|
-
if (execution.failure !== void 0) lines.push(`
|
|
3778
|
+
if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
|
|
3323
3779
|
const notRun = attempt.tests.filter((test) => test.turn > ran);
|
|
3324
|
-
if (notRun.length > 0) lines.push("
|
|
3780
|
+
if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
|
|
3325
3781
|
lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
|
|
3326
|
-
for (const error of execution.attemptErrors) lines.push(`
|
|
3327
|
-
if (execution.closeError !== void 0) lines.push(`
|
|
3328
|
-
lines.push(`
|
|
3782
|
+
for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
|
|
3783
|
+
if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
|
|
3784
|
+
lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
|
|
3329
3785
|
return lines;
|
|
3330
3786
|
}
|
|
3331
3787
|
formatTest(test) {
|
|
3332
|
-
return [`
|
|
3788
|
+
return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
|
|
3333
3789
|
}
|
|
3334
3790
|
static formatTurn(number, input, turn) {
|
|
3335
3791
|
const lines = [
|
|
3336
|
-
`
|
|
3337
|
-
`
|
|
3338
|
-
`
|
|
3792
|
+
` Turn ${number} (${turn.durationMs} ms)`,
|
|
3793
|
+
` Input: ${ConsoleReporter.indent(input, 22)}`,
|
|
3794
|
+
` Response: ${ConsoleReporter.indent(turn.response, 22)}`
|
|
3339
3795
|
];
|
|
3340
|
-
for (const use of turn.toolUses ?? []) lines.push(`
|
|
3341
|
-
for (const result of turn.toolResults ?? []) lines.push(`
|
|
3342
|
-
for (const status of turn.statuses) lines.push(`
|
|
3343
|
-
if (turn.tokenUsage !== void 0) lines.push(`
|
|
3796
|
+
for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
|
|
3797
|
+
for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
|
|
3798
|
+
for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
|
|
3799
|
+
if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
|
|
3344
3800
|
return lines;
|
|
3345
3801
|
}
|
|
3346
3802
|
static indent(text, width) {
|
|
@@ -3443,6 +3899,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3443
3899
|
const capability = websocket.protocol.capabilities.tokenUsage;
|
|
3444
3900
|
return {
|
|
3445
3901
|
name: result.spec.name,
|
|
3902
|
+
runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
|
|
3446
3903
|
file: result.spec.location.file,
|
|
3447
3904
|
status: result.status,
|
|
3448
3905
|
connection: {
|