@mappies/miniwat-eval 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +91 -10
- package/dist/index.cjs +577 -120
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +33 -6
- package/dist/index.d.ts +34 -6
- package/dist/index.js +577 -120
- package/dist/index.js.map +1 -1
- package/dist/main.js +588 -125
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +35 -1
- package/schemas/eval-config.schema.json +6 -1
- package/schemas/report.schema.json +1 -0
package/dist/index.cjs
CHANGED
|
@@ -344,6 +344,7 @@ var WebSocketConfig = class {
|
|
|
344
344
|
var ConversationSpec = class {
|
|
345
345
|
version;
|
|
346
346
|
name;
|
|
347
|
+
runName;
|
|
347
348
|
description;
|
|
348
349
|
turns;
|
|
349
350
|
conversation;
|
|
@@ -353,6 +354,7 @@ var ConversationSpec = class {
|
|
|
353
354
|
constructor(data) {
|
|
354
355
|
this.version = data?.version ?? 1;
|
|
355
356
|
this.name = data?.name ?? "";
|
|
357
|
+
this.runName = data?.runName ?? "";
|
|
356
358
|
this.description = data?.description ?? "";
|
|
357
359
|
this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
|
|
358
360
|
this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
|
|
@@ -474,7 +476,7 @@ var EvalConfig = class {
|
|
|
474
476
|
this.version = data?.version ?? 1;
|
|
475
477
|
this.configPath = data?.configPath ?? "";
|
|
476
478
|
this.parametersPath = data?.parametersPath ?? "";
|
|
477
|
-
this.variables =
|
|
479
|
+
this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
|
|
478
480
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
479
481
|
this.execution = new ExecutionConfig(data?.execution);
|
|
480
482
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -789,7 +791,10 @@ var eval_config_schema_default = {
|
|
|
789
791
|
"variables": {
|
|
790
792
|
"type": "object",
|
|
791
793
|
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
792
|
-
"additionalProperties": { "type": "string" }
|
|
794
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
795
|
+
"type": "array",
|
|
796
|
+
"items": { "type": "string" }
|
|
797
|
+
}] }
|
|
793
798
|
},
|
|
794
799
|
"websocket": {
|
|
795
800
|
"type": "object",
|
|
@@ -1022,7 +1027,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
1022
1027
|
return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
|
|
1023
1028
|
}
|
|
1024
1029
|
resolve(value, document, issues, secrets = [], options = {}) {
|
|
1025
|
-
return this.resolveValue(value, [], document, issues, secrets, options);
|
|
1030
|
+
return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
|
|
1026
1031
|
}
|
|
1027
1032
|
static hasReference(value) {
|
|
1028
1033
|
return new RegExp(REFERENCE.source).test(value);
|
|
@@ -1030,7 +1035,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
1030
1035
|
static isSecretName(name) {
|
|
1031
1036
|
return SECRET_NAME.test(name);
|
|
1032
1037
|
}
|
|
1033
|
-
|
|
1038
|
+
find(name) {
|
|
1034
1039
|
for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
|
|
1035
1040
|
return this.environment[name];
|
|
1036
1041
|
}
|
|
@@ -1045,7 +1050,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
1045
1050
|
resolveValue(value, path, document, issues, secrets, options) {
|
|
1046
1051
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
1047
1052
|
options.used?.add(name);
|
|
1048
|
-
const resolved = this.
|
|
1053
|
+
const resolved = this.find(name);
|
|
1049
1054
|
if (resolved === void 0) {
|
|
1050
1055
|
if (options.optional?.(path)) {
|
|
1051
1056
|
options.unresolved?.add(YamlDocument.formatPath(path));
|
|
@@ -1057,6 +1062,13 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
1057
1062
|
}));
|
|
1058
1063
|
return reference;
|
|
1059
1064
|
}
|
|
1065
|
+
if (Array.isArray(resolved)) {
|
|
1066
|
+
issues.push(new ValidationIssue({
|
|
1067
|
+
message: `variable '${name}' is a list, which can only be used as runWith.from`,
|
|
1068
|
+
location: document.locate(path)
|
|
1069
|
+
}));
|
|
1070
|
+
return reference;
|
|
1071
|
+
}
|
|
1060
1072
|
if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
|
|
1061
1073
|
return resolved;
|
|
1062
1074
|
});
|
|
@@ -1185,9 +1197,25 @@ var VariablesReader = class {
|
|
|
1185
1197
|
}));
|
|
1186
1198
|
continue;
|
|
1187
1199
|
}
|
|
1200
|
+
if (Array.isArray(value)) {
|
|
1201
|
+
const bad = value.findIndex((item) => typeof item !== "string");
|
|
1202
|
+
if (bad >= 0) {
|
|
1203
|
+
issues.push(new ValidationIssue({
|
|
1204
|
+
message: "must be string",
|
|
1205
|
+
location: document.locate([
|
|
1206
|
+
"variables",
|
|
1207
|
+
name,
|
|
1208
|
+
bad
|
|
1209
|
+
])
|
|
1210
|
+
}));
|
|
1211
|
+
continue;
|
|
1212
|
+
}
|
|
1213
|
+
variables[name] = [...value];
|
|
1214
|
+
continue;
|
|
1215
|
+
}
|
|
1188
1216
|
if (typeof value !== "string") {
|
|
1189
1217
|
issues.push(new ValidationIssue({
|
|
1190
|
-
message: "must be string",
|
|
1218
|
+
message: "must be string or a list of strings",
|
|
1191
1219
|
location: document.locate(["variables", name])
|
|
1192
1220
|
}));
|
|
1193
1221
|
continue;
|
|
@@ -1195,8 +1223,9 @@ var VariablesReader = class {
|
|
|
1195
1223
|
const before = issues.length;
|
|
1196
1224
|
const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
|
|
1197
1225
|
if (issues.length > before) continue;
|
|
1198
|
-
|
|
1199
|
-
|
|
1226
|
+
const text = resolved.variables[name];
|
|
1227
|
+
variables[name] = text;
|
|
1228
|
+
if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
|
|
1200
1229
|
}
|
|
1201
1230
|
return variables;
|
|
1202
1231
|
}
|
|
@@ -1501,7 +1530,44 @@ var conversation_schema_default = {
|
|
|
1501
1530
|
"variables": {
|
|
1502
1531
|
"type": "object",
|
|
1503
1532
|
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
1504
|
-
"additionalProperties": { "type": "string" }
|
|
1533
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
1534
|
+
"type": "array",
|
|
1535
|
+
"items": { "type": "string" }
|
|
1536
|
+
}] }
|
|
1537
|
+
},
|
|
1538
|
+
"runWith": {
|
|
1539
|
+
"description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
|
|
1540
|
+
"anyOf": [{
|
|
1541
|
+
"type": "array",
|
|
1542
|
+
"minItems": 1,
|
|
1543
|
+
"items": {
|
|
1544
|
+
"type": "object",
|
|
1545
|
+
"required": ["name"],
|
|
1546
|
+
"properties": { "name": {
|
|
1547
|
+
"type": "string",
|
|
1548
|
+
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
|
|
1549
|
+
} },
|
|
1550
|
+
"additionalProperties": { "type": "string" }
|
|
1551
|
+
}
|
|
1552
|
+
}, {
|
|
1553
|
+
"type": "object",
|
|
1554
|
+
"additionalProperties": false,
|
|
1555
|
+
"required": ["variable", "from"],
|
|
1556
|
+
"properties": {
|
|
1557
|
+
"variable": {
|
|
1558
|
+
"type": "string",
|
|
1559
|
+
"pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
|
|
1560
|
+
},
|
|
1561
|
+
"from": { "anyOf": [{
|
|
1562
|
+
"type": "array",
|
|
1563
|
+
"minItems": 1,
|
|
1564
|
+
"items": { "type": "string" }
|
|
1565
|
+
}, {
|
|
1566
|
+
"type": "string",
|
|
1567
|
+
"pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
|
|
1568
|
+
}] }
|
|
1569
|
+
}
|
|
1570
|
+
}]
|
|
1505
1571
|
},
|
|
1506
1572
|
"turns": {
|
|
1507
1573
|
"type": "array",
|
|
@@ -1660,6 +1726,221 @@ var ConversationValidator = class ConversationValidator {
|
|
|
1660
1726
|
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
1661
1727
|
}
|
|
1662
1728
|
};
|
|
1729
|
+
// src/scenarios/runWithReader.ts
|
|
1730
|
+
const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
|
|
1731
|
+
const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
|
|
1732
|
+
const FORMS = "must be a list of rows, or an object with variable and from";
|
|
1733
|
+
const SINGLE_RUN = {
|
|
1734
|
+
name: "",
|
|
1735
|
+
variables: {},
|
|
1736
|
+
keys: [],
|
|
1737
|
+
location: [],
|
|
1738
|
+
secrets: []
|
|
1739
|
+
};
|
|
1740
|
+
var RunWithReader = class RunWithReader {
|
|
1741
|
+
read(document, resolver, issues, used) {
|
|
1742
|
+
const raw = isRecord(document.value) ? document.value.runWith : void 0;
|
|
1743
|
+
if (raw === void 0) return void 0;
|
|
1744
|
+
if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
|
|
1745
|
+
if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
|
|
1746
|
+
issues.push(new ValidationIssue({
|
|
1747
|
+
message: FORMS,
|
|
1748
|
+
location: document.locate(["runWith"])
|
|
1749
|
+
}));
|
|
1750
|
+
return [];
|
|
1751
|
+
}
|
|
1752
|
+
rows(rows, document, resolver, issues, used) {
|
|
1753
|
+
if (rows.length === 0) {
|
|
1754
|
+
issues.push(new ValidationIssue({
|
|
1755
|
+
message: "must not be empty",
|
|
1756
|
+
location: document.locate(["runWith"])
|
|
1757
|
+
}));
|
|
1758
|
+
return [];
|
|
1759
|
+
}
|
|
1760
|
+
const names = /* @__PURE__ */ new Set();
|
|
1761
|
+
const runs = [];
|
|
1762
|
+
rows.forEach((row, index) => {
|
|
1763
|
+
const path = ["runWith", index];
|
|
1764
|
+
if (!isRecord(row)) {
|
|
1765
|
+
issues.push(new ValidationIssue({
|
|
1766
|
+
message: "must be object",
|
|
1767
|
+
location: document.locate(path)
|
|
1768
|
+
}));
|
|
1769
|
+
return;
|
|
1770
|
+
}
|
|
1771
|
+
const name = row.name;
|
|
1772
|
+
if (name === void 0) issues.push(new ValidationIssue({
|
|
1773
|
+
message: "missing required key 'name'",
|
|
1774
|
+
location: document.locate(path)
|
|
1775
|
+
}));
|
|
1776
|
+
else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
|
|
1777
|
+
message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
|
|
1778
|
+
location: document.locate([...path, "name"])
|
|
1779
|
+
}));
|
|
1780
|
+
else if (names.has(name)) issues.push(new ValidationIssue({
|
|
1781
|
+
message: `duplicate run name '${name}'`,
|
|
1782
|
+
location: document.locate([...path, "name"])
|
|
1783
|
+
}));
|
|
1784
|
+
else names.add(name);
|
|
1785
|
+
const run = {
|
|
1786
|
+
name: typeof name === "string" ? name : "",
|
|
1787
|
+
variables: {},
|
|
1788
|
+
keys: [],
|
|
1789
|
+
location: path,
|
|
1790
|
+
secrets: []
|
|
1791
|
+
};
|
|
1792
|
+
for (const [key, value] of Object.entries(row)) {
|
|
1793
|
+
if (key === "name") continue;
|
|
1794
|
+
if (!VARIABLE_NAME.test(key)) {
|
|
1795
|
+
issues.push(new ValidationIssue({
|
|
1796
|
+
message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1797
|
+
location: document.locateKey(path, key)
|
|
1798
|
+
}));
|
|
1799
|
+
continue;
|
|
1800
|
+
}
|
|
1801
|
+
if (typeof value !== "string") {
|
|
1802
|
+
issues.push(new ValidationIssue({
|
|
1803
|
+
message: "must be string",
|
|
1804
|
+
location: document.locate([...path, key])
|
|
1805
|
+
}));
|
|
1806
|
+
continue;
|
|
1807
|
+
}
|
|
1808
|
+
const resolved = resolver.resolve(value, document, issues, run.secrets, {
|
|
1809
|
+
used,
|
|
1810
|
+
at: [...path, key]
|
|
1811
|
+
});
|
|
1812
|
+
RunWithReader.set(run, key, resolved, document.locateKey(path, key));
|
|
1813
|
+
}
|
|
1814
|
+
runs.push(run);
|
|
1815
|
+
});
|
|
1816
|
+
return runs;
|
|
1817
|
+
}
|
|
1818
|
+
list(form, document, resolver, issues, used) {
|
|
1819
|
+
const before = issues.length;
|
|
1820
|
+
for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
|
|
1821
|
+
message: `unknown key '${key}'`,
|
|
1822
|
+
location: document.locateKey(["runWith"], key)
|
|
1823
|
+
}));
|
|
1824
|
+
const variable = form.variable;
|
|
1825
|
+
if (variable === void 0) issues.push(new ValidationIssue({
|
|
1826
|
+
message: "missing required key 'variable'",
|
|
1827
|
+
location: document.locate(["runWith"])
|
|
1828
|
+
}));
|
|
1829
|
+
else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
|
|
1830
|
+
message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
|
|
1831
|
+
location: document.locate(["runWith", "variable"])
|
|
1832
|
+
}));
|
|
1833
|
+
const entries = this.entries(form.from, document, resolver, issues, used);
|
|
1834
|
+
if (issues.length > before || entries === void 0) return [];
|
|
1835
|
+
const name = variable;
|
|
1836
|
+
const keyLocation = document.locate(["runWith", "variable"]);
|
|
1837
|
+
const names = /* @__PURE__ */ new Set();
|
|
1838
|
+
const runs = [];
|
|
1839
|
+
for (const { entry, path } of entries) {
|
|
1840
|
+
if (!VARIABLE_NAME.test(entry)) {
|
|
1841
|
+
issues.push(new ValidationIssue({
|
|
1842
|
+
message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1843
|
+
location: document.locate(path)
|
|
1844
|
+
}));
|
|
1845
|
+
continue;
|
|
1846
|
+
}
|
|
1847
|
+
if (names.has(entry)) {
|
|
1848
|
+
issues.push(new ValidationIssue({
|
|
1849
|
+
message: `duplicate run name '${entry}'`,
|
|
1850
|
+
location: document.locate(path)
|
|
1851
|
+
}));
|
|
1852
|
+
continue;
|
|
1853
|
+
}
|
|
1854
|
+
names.add(entry);
|
|
1855
|
+
const run = {
|
|
1856
|
+
name: entry,
|
|
1857
|
+
variables: {},
|
|
1858
|
+
keys: [],
|
|
1859
|
+
location: path,
|
|
1860
|
+
secrets: []
|
|
1861
|
+
};
|
|
1862
|
+
const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
|
|
1863
|
+
used,
|
|
1864
|
+
at: path
|
|
1865
|
+
});
|
|
1866
|
+
RunWithReader.set(run, name, value, keyLocation);
|
|
1867
|
+
runs.push(run);
|
|
1868
|
+
}
|
|
1869
|
+
return runs;
|
|
1870
|
+
}
|
|
1871
|
+
entries(from, document, resolver, issues, used) {
|
|
1872
|
+
const path = ["runWith", "from"];
|
|
1873
|
+
const location = document.locate(path);
|
|
1874
|
+
if (from === void 0) {
|
|
1875
|
+
issues.push(new ValidationIssue({
|
|
1876
|
+
message: "missing required key 'from'",
|
|
1877
|
+
location: document.locate(["runWith"])
|
|
1878
|
+
}));
|
|
1879
|
+
return;
|
|
1880
|
+
}
|
|
1881
|
+
let list;
|
|
1882
|
+
let at;
|
|
1883
|
+
if (Array.isArray(from)) {
|
|
1884
|
+
list = from;
|
|
1885
|
+
at = (index) => [...path, index];
|
|
1886
|
+
} else {
|
|
1887
|
+
const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
|
|
1888
|
+
if (reference === null) {
|
|
1889
|
+
issues.push(new ValidationIssue({
|
|
1890
|
+
message: "must be a list, or a list variable such as ${tokens}",
|
|
1891
|
+
location
|
|
1892
|
+
}));
|
|
1893
|
+
return;
|
|
1894
|
+
}
|
|
1895
|
+
const name = reference[1];
|
|
1896
|
+
const found = resolver.find(name);
|
|
1897
|
+
used.add(name);
|
|
1898
|
+
if (found === void 0) {
|
|
1899
|
+
issues.push(new ValidationIssue({
|
|
1900
|
+
message: `list variable '${name}' is not set`,
|
|
1901
|
+
location
|
|
1902
|
+
}));
|
|
1903
|
+
return;
|
|
1904
|
+
}
|
|
1905
|
+
if (!Array.isArray(found)) {
|
|
1906
|
+
issues.push(new ValidationIssue({
|
|
1907
|
+
message: `variable '${name}' is not a list`,
|
|
1908
|
+
location
|
|
1909
|
+
}));
|
|
1910
|
+
return;
|
|
1911
|
+
}
|
|
1912
|
+
list = found;
|
|
1913
|
+
at = () => path;
|
|
1914
|
+
}
|
|
1915
|
+
if (list.length === 0) {
|
|
1916
|
+
issues.push(new ValidationIssue({
|
|
1917
|
+
message: "must not be empty",
|
|
1918
|
+
location
|
|
1919
|
+
}));
|
|
1920
|
+
return;
|
|
1921
|
+
}
|
|
1922
|
+
const bad = list.findIndex((item) => typeof item !== "string");
|
|
1923
|
+
if (bad >= 0) {
|
|
1924
|
+
issues.push(new ValidationIssue({
|
|
1925
|
+
message: "must be string",
|
|
1926
|
+
location: document.locate(at(bad))
|
|
1927
|
+
}));
|
|
1928
|
+
return;
|
|
1929
|
+
}
|
|
1930
|
+
return list.map((entry, index) => ({
|
|
1931
|
+
entry,
|
|
1932
|
+
path: at(index)
|
|
1933
|
+
}));
|
|
1934
|
+
}
|
|
1935
|
+
static set(run, name, value, location) {
|
|
1936
|
+
run.variables[name] = value;
|
|
1937
|
+
run.keys.push({
|
|
1938
|
+
name,
|
|
1939
|
+
location
|
|
1940
|
+
});
|
|
1941
|
+
if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
|
|
1942
|
+
}
|
|
1943
|
+
};
|
|
1663
1944
|
// src/scenarios/scenarioLoader.ts
|
|
1664
1945
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
1665
1946
|
var ScenarioLoader = class ScenarioLoader {
|
|
@@ -1668,6 +1949,7 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1668
1949
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
1669
1950
|
conversationValidator = new ConversationValidator();
|
|
1670
1951
|
variablesReader = new VariablesReader();
|
|
1952
|
+
runWithReader = new RunWithReader();
|
|
1671
1953
|
webSocketResolver = new WebSocketResolver();
|
|
1672
1954
|
environmentResolver;
|
|
1673
1955
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
@@ -1683,45 +1965,59 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1683
1965
|
for (const file of files) {
|
|
1684
1966
|
const document = this.documentLoader.load(file, (0, node_path.relative)(this.cwd, file));
|
|
1685
1967
|
const fileIssues = [...document.errors];
|
|
1686
|
-
const
|
|
1968
|
+
const fileSecrets = [];
|
|
1687
1969
|
const used = /* @__PURE__ */ new Set();
|
|
1688
|
-
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues,
|
|
1689
|
-
const
|
|
1690
|
-
|
|
1691
|
-
if (fileIssues.length
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
...resolver.resolve(rest, document, fileIssues, secrets, { used }),
|
|
1695
|
-
...raw === void 0 ? {} : { variables }
|
|
1696
|
-
};
|
|
1970
|
+
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
|
|
1971
|
+
const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
|
|
1972
|
+
const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
|
|
1973
|
+
if (fileIssues.length > 0) {
|
|
1974
|
+
issues.push(...fileIssues);
|
|
1975
|
+
continue;
|
|
1697
1976
|
}
|
|
1698
|
-
|
|
1699
|
-
const
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1977
|
+
const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
|
|
1978
|
+
const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
|
|
1979
|
+
for (const result of results) for (const name of result.used) used.add(name);
|
|
1980
|
+
fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
|
|
1981
|
+
run: result.run,
|
|
1982
|
+
issues: result.issues
|
|
1983
|
+
})), (issue, run) => run === void 0 ? issue : new ValidationIssue({
|
|
1984
|
+
message: `run '${run.name}': ${issue.message}`,
|
|
1985
|
+
location: issue.location
|
|
1986
|
+
})));
|
|
1987
|
+
const attempted = results.filter((result) => result.connectionChecked);
|
|
1988
|
+
if (attempted.length > 0) connectionIssues.push({
|
|
1989
|
+
file: document,
|
|
1990
|
+
runs: attempted.map((result) => ({
|
|
1991
|
+
run: result.run,
|
|
1992
|
+
issues: result.websocketIssues
|
|
1993
|
+
}))
|
|
1994
|
+
});
|
|
1995
|
+
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
|
|
1996
|
+
if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
|
|
1997
|
+
const websocket = result.websocket;
|
|
1998
|
+
specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
1708
1999
|
}
|
|
1709
|
-
if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
|
|
1710
|
-
if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
|
|
1711
|
-
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
|
|
1712
|
-
if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
1713
2000
|
else issues.push(...fileIssues);
|
|
1714
2001
|
}
|
|
1715
|
-
const shared = ScenarioLoader.shared(connectionIssues.
|
|
2002
|
+
const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
|
|
1716
2003
|
if (shared.length > 0) throw new EvalError({
|
|
1717
2004
|
category: ErrorCategory.Configuration,
|
|
1718
2005
|
message: "invalid configuration",
|
|
1719
2006
|
issues: shared
|
|
1720
2007
|
});
|
|
1721
|
-
for (const entry of connectionIssues)
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
2008
|
+
for (const entry of connectionIssues) {
|
|
2009
|
+
const fileLocation = ScenarioLoader.variablesLocation(entry.file);
|
|
2010
|
+
issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
|
|
2011
|
+
message: issue.toString(),
|
|
2012
|
+
location: fileLocation
|
|
2013
|
+
}) : new ValidationIssue({
|
|
2014
|
+
message: `run '${run.name}': ${issue.toString()}`,
|
|
2015
|
+
location: new SourceLocation({
|
|
2016
|
+
...entry.file.locate(run.location),
|
|
2017
|
+
path: ""
|
|
2018
|
+
})
|
|
2019
|
+
})));
|
|
2020
|
+
}
|
|
1725
2021
|
if (issues.length > 0) throw new EvalError({
|
|
1726
2022
|
category: ErrorCategory.Specification,
|
|
1727
2023
|
message: "invalid scenario specification",
|
|
@@ -1729,6 +2025,43 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1729
2025
|
});
|
|
1730
2026
|
return specs;
|
|
1731
2027
|
}
|
|
2028
|
+
resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
|
|
2029
|
+
const issues = [];
|
|
2030
|
+
const used = /* @__PURE__ */ new Set();
|
|
2031
|
+
const resolver = scenarioResolver.withLevels([run.variables]);
|
|
2032
|
+
const value = {
|
|
2033
|
+
...resolver.resolve(rest, document, issues, secrets, { used }),
|
|
2034
|
+
...variables === void 0 ? {} : { variables }
|
|
2035
|
+
};
|
|
2036
|
+
const websocketIssues = [];
|
|
2037
|
+
let websocket;
|
|
2038
|
+
let connectionChecked = false;
|
|
2039
|
+
if (issues.length === 0 && configDocument !== void 0) {
|
|
2040
|
+
const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
|
|
2041
|
+
if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
|
|
2042
|
+
if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
|
|
2043
|
+
connectionChecked = true;
|
|
2044
|
+
}
|
|
2045
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
|
|
2046
|
+
if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
|
|
2047
|
+
return {
|
|
2048
|
+
run,
|
|
2049
|
+
value,
|
|
2050
|
+
used,
|
|
2051
|
+
secrets,
|
|
2052
|
+
issues,
|
|
2053
|
+
websocket,
|
|
2054
|
+
websocketIssues,
|
|
2055
|
+
connectionChecked
|
|
2056
|
+
};
|
|
2057
|
+
}
|
|
2058
|
+
static byRun(runs, place) {
|
|
2059
|
+
const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
|
|
2060
|
+
const sharedTexts = new Set(shared.map((issue) => issue.toString()));
|
|
2061
|
+
const result = shared.map((issue) => place(issue, void 0));
|
|
2062
|
+
for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
|
|
2063
|
+
return result;
|
|
2064
|
+
}
|
|
1732
2065
|
static shared(lists) {
|
|
1733
2066
|
if (lists.length === 0) return [];
|
|
1734
2067
|
const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
|
|
@@ -1740,11 +2073,22 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1740
2073
|
path: ""
|
|
1741
2074
|
});
|
|
1742
2075
|
}
|
|
1743
|
-
static unusedVariables(variables, used, document) {
|
|
1744
|
-
|
|
1745
|
-
|
|
2076
|
+
static unusedVariables(variables, used, results, document) {
|
|
2077
|
+
const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
|
|
2078
|
+
const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
|
|
2079
|
+
message: message(name),
|
|
1746
2080
|
location: document.locateKey(["variables"], name)
|
|
1747
2081
|
}));
|
|
2082
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2083
|
+
for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
|
|
2084
|
+
const issue = new ValidationIssue({
|
|
2085
|
+
message: message(key.name),
|
|
2086
|
+
location: key.location
|
|
2087
|
+
});
|
|
2088
|
+
if (!seen.has(issue.toString())) issues.push(issue);
|
|
2089
|
+
seen.add(issue.toString());
|
|
2090
|
+
}
|
|
2091
|
+
return issues;
|
|
1748
2092
|
}
|
|
1749
2093
|
discover(target, config) {
|
|
1750
2094
|
if (!(0, node_fs.existsSync)(target)) throw new EvalError({
|
|
@@ -1770,13 +2114,14 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1770
2114
|
}
|
|
1771
2115
|
return files;
|
|
1772
2116
|
}
|
|
1773
|
-
build(value, document, websocket, secrets) {
|
|
2117
|
+
build(value, document, runName, websocket, secrets) {
|
|
1774
2118
|
const scenario = value;
|
|
1775
2119
|
const turns = scenario.turns;
|
|
1776
2120
|
const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
|
|
1777
2121
|
return new ConversationSpec({
|
|
1778
2122
|
version: scenario.version,
|
|
1779
2123
|
name: scenario.name,
|
|
2124
|
+
runName,
|
|
1780
2125
|
description: typeof scenario.description === "string" ? scenario.description : "",
|
|
1781
2126
|
turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
|
|
1782
2127
|
conversation: conversation ? new ConversationExpectation({
|
|
@@ -3212,61 +3557,94 @@ var ExitCodeCalculator = class {
|
|
|
3212
3557
|
}
|
|
3213
3558
|
};
|
|
3214
3559
|
// src/reporting/consoleReporter.ts
|
|
3215
|
-
const
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
[ResultStatus.
|
|
3221
|
-
|
|
3222
|
-
|
|
3560
|
+
const MARKS = {
|
|
3561
|
+
[ResultStatus.Pass]: {
|
|
3562
|
+
symbol: "✔",
|
|
3563
|
+
colour: "green"
|
|
3564
|
+
},
|
|
3565
|
+
[ResultStatus.Flaky]: {
|
|
3566
|
+
symbol: "⚠",
|
|
3567
|
+
colour: "yellow"
|
|
3568
|
+
},
|
|
3569
|
+
[ResultStatus.Fail]: {
|
|
3570
|
+
symbol: "✖",
|
|
3571
|
+
colour: "red"
|
|
3572
|
+
},
|
|
3573
|
+
[ResultStatus.Error]: {
|
|
3574
|
+
symbol: "●",
|
|
3575
|
+
colour: "red"
|
|
3576
|
+
},
|
|
3577
|
+
[ResultStatus.Skipped]: {
|
|
3578
|
+
symbol: "○",
|
|
3579
|
+
colour: "dim"
|
|
3580
|
+
},
|
|
3581
|
+
[ResultStatus.Interrupted]: {
|
|
3582
|
+
symbol: "◌",
|
|
3583
|
+
colour: "dim"
|
|
3584
|
+
}
|
|
3223
3585
|
};
|
|
3586
|
+
const MIN_COLUMNS = 40;
|
|
3587
|
+
const RULE_WIDTH = 72;
|
|
3224
3588
|
var ConsoleReporter = class ConsoleReporter {
|
|
3225
3589
|
verbose;
|
|
3226
3590
|
colour;
|
|
3227
|
-
|
|
3591
|
+
nameWidth;
|
|
3592
|
+
columns;
|
|
3593
|
+
thresholds;
|
|
3594
|
+
constructor(verbose, colour, options = {}) {
|
|
3228
3595
|
this.verbose = verbose;
|
|
3229
3596
|
this.colour = colour;
|
|
3597
|
+
this.nameWidth = Math.min(options.nameWidth ?? 0, 50);
|
|
3598
|
+
this.columns = Math.max(options.columns ?? 100, MIN_COLUMNS);
|
|
3599
|
+
this.thresholds = options.thresholds;
|
|
3600
|
+
}
|
|
3601
|
+
static nameWidth(specs) {
|
|
3602
|
+
return Math.min(Math.max(0, ...specs.map((spec) => ConsoleReporter.title(spec).length)), 50);
|
|
3230
3603
|
}
|
|
3231
3604
|
header(run) {
|
|
3232
3605
|
return [
|
|
3233
|
-
"Miniwat Eval",
|
|
3606
|
+
" Miniwat Eval",
|
|
3234
3607
|
"",
|
|
3235
|
-
...run.quick ? ["Execution
|
|
3236
|
-
`Repeat
|
|
3237
|
-
`Minimum
|
|
3608
|
+
...run.quick ? [" Execution mode Quick"] : [],
|
|
3609
|
+
` Repeat ${run.repeat}`,
|
|
3610
|
+
` Minimum pass ${run.minPass}`,
|
|
3238
3611
|
"",
|
|
3239
|
-
`Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
3612
|
+
` Running ${run.scenarios} ${run.scenarios === 1 ? "scenario" : "scenarios"} (concurrency ${run.concurrency})`,
|
|
3240
3613
|
"",
|
|
3241
3614
|
""
|
|
3242
3615
|
].join("\n");
|
|
3243
3616
|
}
|
|
3244
3617
|
scenario(result, repeat, tokenUsageCapability) {
|
|
3245
3618
|
const spec = result.spec;
|
|
3246
|
-
const
|
|
3247
|
-
const
|
|
3619
|
+
const title = ConsoleReporter.title(spec);
|
|
3620
|
+
const started = result.attempts.length > 0;
|
|
3621
|
+
const counts = started ? `${result.passed}/${repeat}`.padStart(`${repeat}/${repeat}`.length) : "not started";
|
|
3622
|
+
const duration = started ? ` ${ConsoleReporter.duration(result.attempts.reduce((total, attempt) => total + attempt.execution.durationMs, 0)).padStart(6)}` : "";
|
|
3623
|
+
const line = ` ${this.mark(result.status)} ${title.padEnd(this.nameWidth)} ${counts}${duration}\n`;
|
|
3248
3624
|
if (!this.verbose) return line;
|
|
3249
|
-
if (
|
|
3625
|
+
if (!started) return `${line}\n`;
|
|
3250
3626
|
const websocket = spec.websocket;
|
|
3251
3627
|
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
3252
|
-
const lines = [
|
|
3253
|
-
|
|
3254
|
-
lines.push(`
|
|
3255
|
-
lines.push(
|
|
3256
|
-
lines.push(`
|
|
3257
|
-
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3628
|
+
const lines = [];
|
|
3629
|
+
if (websocket !== void 0) lines.push(` Connection ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, "");
|
|
3630
|
+
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat} ${this.mark(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
3631
|
+
lines.push(` ${ConsoleReporter.usage("Agent tokens", result.agentUsage, tokenUsageCapability)}`);
|
|
3632
|
+
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3258
3633
|
return line + lines.join("\n");
|
|
3259
3634
|
}
|
|
3635
|
+
static title(spec) {
|
|
3636
|
+
return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
|
|
3637
|
+
}
|
|
3260
3638
|
summary(run, exitCode, reportPaths = []) {
|
|
3261
|
-
const
|
|
3639
|
+
const started = run.scenarios.filter((result) => result.attempts.length > 0);
|
|
3640
|
+
const failures = started.flatMap((result) => this.blocks(result, false));
|
|
3641
|
+
const flaky = started.flatMap((result) => this.blocks(result, true));
|
|
3262
3642
|
const lines = this.verbose ? [] : [""];
|
|
3263
|
-
if (
|
|
3264
|
-
|
|
3265
|
-
for (const result of problems) lines.push(...this.problems(result), "");
|
|
3266
|
-
}
|
|
3643
|
+
if (failures.length > 0) lines.push(...this.section("FAILURES", failures));
|
|
3644
|
+
if (flaky.length > 0) lines.push(...this.section("FLAKY", flaky));
|
|
3267
3645
|
const tests = run.scenarios.flatMap((result) => result.tests);
|
|
3268
|
-
lines.push(...run.status === RunStatus.Interrupted ? [`Run
|
|
3269
|
-
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths);
|
|
3646
|
+
lines.push(...this.rule("SUMMARY"), "", ...run.status === RunStatus.Interrupted ? [` Run ${run.status}`] : [], ` Scenarios ${ConsoleReporter.tally(run.scenarios.map((result) => result.status))}`, ` Tests ${ConsoleReporter.tally(tests.map((test) => test.status))}`, ` ${ConsoleReporter.usage("Agent tokens", run.agentUsage, run.tokenUsageCapability)}`, ` ${ConsoleReporter.usage("Evaluator tokens", run.evaluatorUsage, true)}`, ` Duration ${ConsoleReporter.duration(run.durationMs)}`, "", "");
|
|
3647
|
+
return lines.join("\n") + ConsoleReporter.footer(run, exitCode, reportPaths).split("\n").map((line) => line === "" ? line : ` ${line}`).join("\n");
|
|
3270
3648
|
}
|
|
3271
3649
|
static footer(run, exitCode, reportPaths) {
|
|
3272
3650
|
return [
|
|
@@ -3280,57 +3658,135 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3280
3658
|
return exitCode === 0 ? `Evaluation passed. Exit code ${exitCode}.` : `Evaluation completed with failures. Exit code ${exitCode}.`;
|
|
3281
3659
|
}
|
|
3282
3660
|
static duration(ms) {
|
|
3661
|
+
if (ms < 6e4) return `${(Math.floor(ms / 100) / 10).toFixed(1)}s`;
|
|
3283
3662
|
const seconds = Math.floor(ms / 1e3);
|
|
3284
3663
|
const pad = (value) => String(value).padStart(2, "0");
|
|
3285
3664
|
const hours = Math.floor(seconds / 3600);
|
|
3286
|
-
const
|
|
3287
|
-
return hours > 0 ? `${hours}:${
|
|
3665
|
+
const minutes = Math.floor(seconds / 60) % 60;
|
|
3666
|
+
return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
|
|
3288
3667
|
}
|
|
3289
|
-
|
|
3668
|
+
static wrap(text, width) {
|
|
3669
|
+
const lines = [];
|
|
3670
|
+
for (const paragraph of text.split("\n")) {
|
|
3671
|
+
let line = "";
|
|
3672
|
+
for (const word of paragraph.split(/ +/)) if (line !== "" && line.length + 1 + word.length > width) {
|
|
3673
|
+
lines.push(line);
|
|
3674
|
+
line = word;
|
|
3675
|
+
} else line = line === "" ? word : `${line} ${word}`;
|
|
3676
|
+
lines.push(line);
|
|
3677
|
+
}
|
|
3678
|
+
return lines;
|
|
3679
|
+
}
|
|
3680
|
+
section(name, blocks) {
|
|
3681
|
+
return [
|
|
3682
|
+
...this.rule(name),
|
|
3683
|
+
"",
|
|
3684
|
+
...blocks.flatMap((block) => [...block, ""])
|
|
3685
|
+
];
|
|
3686
|
+
}
|
|
3687
|
+
rule(name) {
|
|
3688
|
+
const rule = "─".repeat(Math.min(this.columns, RULE_WIDTH));
|
|
3689
|
+
return [
|
|
3690
|
+
rule,
|
|
3691
|
+
` ${name}`,
|
|
3692
|
+
rule
|
|
3693
|
+
];
|
|
3694
|
+
}
|
|
3695
|
+
blocks(result, flaky) {
|
|
3290
3696
|
const spec = result.spec;
|
|
3291
|
-
const
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
|
|
3304
|
-
|
|
3305
|
-
|
|
3306
|
-
|
|
3307
|
-
|
|
3308
|
-
|
|
3309
|
-
}
|
|
3697
|
+
const title = ConsoleReporter.title(spec);
|
|
3698
|
+
const blocks = [];
|
|
3699
|
+
if (result.status === ResultStatus.Flaky === flaky) for (const attempt of result.attempts) blocks.push(...this.attemptBlock(spec, title, attempt));
|
|
3700
|
+
const tests = result.tests.filter((test) => test.status !== ResultStatus.Pass && test.status === ResultStatus.Flaky === flaky);
|
|
3701
|
+
for (const test of tests) blocks.push(this.testBlock(spec, title, test, result));
|
|
3702
|
+
return blocks;
|
|
3703
|
+
}
|
|
3704
|
+
attemptBlock(spec, title, attempt) {
|
|
3705
|
+
const execution = attempt.execution;
|
|
3706
|
+
const problems = [];
|
|
3707
|
+
if (execution.failure !== void 0) problems.push(`${execution.failure.category} error: ${execution.failure.message}`);
|
|
3708
|
+
for (const error of execution.attemptErrors) problems.push(`${error.category} error: ${error.message}`);
|
|
3709
|
+
if (execution.closeError !== void 0) problems.push(`close error: ${execution.closeError.message}`);
|
|
3710
|
+
if (problems.length === 0) return [];
|
|
3711
|
+
const what = execution.failure !== void 0 ? `attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}` : `attempt ${attempt.attempt}`;
|
|
3712
|
+
const status = execution.failure !== void 0 ? attempt.status : ResultStatus.Error;
|
|
3713
|
+
return [[
|
|
3714
|
+
` scenario at ${spec.location.file}`,
|
|
3715
|
+
` ${this.mark(status)} ${title} › ${what}`,
|
|
3716
|
+
...problems.flatMap((problem) => this.paragraph(problem, 3))
|
|
3717
|
+
]];
|
|
3718
|
+
}
|
|
3719
|
+
testBlock(spec, title, test, result) {
|
|
3720
|
+
const location = test.location;
|
|
3721
|
+
const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
|
|
3722
|
+
const failing = test.history.map((entry, index) => ({
|
|
3723
|
+
entry,
|
|
3724
|
+
attempt: result.attempts[index]?.attempt ?? index + 1
|
|
3725
|
+
})).filter(({ entry }) => entry.status !== ResultStatus.Pass);
|
|
3726
|
+
const several = test.history.length > 1;
|
|
3727
|
+
for (const { entry, attempt } of failing) {
|
|
3728
|
+
if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
|
|
3729
|
+
lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
|
|
3310
3730
|
}
|
|
3731
|
+
lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
|
|
3311
3732
|
return lines;
|
|
3312
3733
|
}
|
|
3734
|
+
entryBody(spec, entry, indent) {
|
|
3735
|
+
const pad = " ".repeat(indent);
|
|
3736
|
+
const evaluation = entry.evaluation;
|
|
3737
|
+
if (evaluation === void 0) {
|
|
3738
|
+
if (entry.details.length === 0) return [];
|
|
3739
|
+
return [
|
|
3740
|
+
"",
|
|
3741
|
+
`${pad}Details`,
|
|
3742
|
+
...entry.details.flatMap((detail) => this.bullet(detail, indent + 2))
|
|
3743
|
+
];
|
|
3744
|
+
}
|
|
3745
|
+
const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
|
|
3746
|
+
const expected = ConsoleReporter.expected(spec, entry);
|
|
3747
|
+
const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
|
|
3748
|
+
if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
|
|
3749
|
+
if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
|
|
3750
|
+
if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
|
|
3751
|
+
return lines;
|
|
3752
|
+
}
|
|
3753
|
+
static expected(spec, entry) {
|
|
3754
|
+
if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
|
|
3755
|
+
return spec.turns[entry.turn - 1]?.expect.response?.criteria;
|
|
3756
|
+
}
|
|
3757
|
+
paragraph(text, indent) {
|
|
3758
|
+
const pad = " ".repeat(indent);
|
|
3759
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent).map((line) => line === "" ? "" : `${pad}${line}`);
|
|
3760
|
+
}
|
|
3761
|
+
bullet(text, indent) {
|
|
3762
|
+
const pad = " ".repeat(indent);
|
|
3763
|
+
return ConsoleReporter.wrap(text.trim(), this.columns - indent - 2).map((line, index) => index === 0 ? `${pad}• ${line}` : line === "" ? "" : `${pad} ${line}`);
|
|
3764
|
+
}
|
|
3313
3765
|
static testTitle(test) {
|
|
3314
3766
|
return test.turn > 0 ? `turn ${test.turn}: ${test.title}` : test.title;
|
|
3315
3767
|
}
|
|
3316
3768
|
static tally(statuses) {
|
|
3317
3769
|
const count = (status) => statuses.filter((entry) => entry === status).length;
|
|
3318
|
-
const
|
|
3319
|
-
|
|
3320
|
-
|
|
3321
|
-
|
|
3322
|
-
|
|
3323
|
-
|
|
3324
|
-
|
|
3325
|
-
|
|
3326
|
-
|
|
3770
|
+
const parts = [
|
|
3771
|
+
[ResultStatus.Pass, "passed"],
|
|
3772
|
+
[ResultStatus.Flaky, "flaky"],
|
|
3773
|
+
[ResultStatus.Fail, "failed"],
|
|
3774
|
+
[ResultStatus.Error, "errors"],
|
|
3775
|
+
[ResultStatus.Interrupted, "interrupted"]
|
|
3776
|
+
].filter(([status]) => count(status) > 0).map(([status, word]) => `${count(status)} ${word}`);
|
|
3777
|
+
return `${parts.length === 0 ? "none" : parts.join(" | ")} (${statuses.length})`;
|
|
3778
|
+
}
|
|
3779
|
+
mark(status) {
|
|
3780
|
+
const { symbol, colour } = MARKS[status];
|
|
3781
|
+
return this.colour ? (0, node_util.styleText)(colour, symbol, { validateStream: false }) : symbol;
|
|
3327
3782
|
}
|
|
3328
3783
|
static usage(label, usage, capability) {
|
|
3329
|
-
|
|
3330
|
-
if (
|
|
3331
|
-
if (usage.
|
|
3784
|
+
const name = label.padEnd(16);
|
|
3785
|
+
if (!capability) return `${name} n/a (not supported by host)`;
|
|
3786
|
+
if (usage.expected === 0) return `${name} none`;
|
|
3787
|
+
if (usage.total === void 0) return `${name} n/a (coverage 0/${usage.expected})`;
|
|
3332
3788
|
const number = (value) => value.toLocaleString("en-US");
|
|
3333
|
-
const total = `${
|
|
3789
|
+
const total = `${name} ${number(usage.total.totalTokens)} (input ${number(usage.total.inputTokens)}, output ${number(usage.total.outputTokens)})`;
|
|
3334
3790
|
return usage.reported < usage.expected ? `${total}, coverage ${usage.reported}/${usage.expected}, partial` : total;
|
|
3335
3791
|
}
|
|
3336
3792
|
formatAttempt(spec, attempt) {
|
|
@@ -3342,28 +3798,28 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3342
3798
|
lines.push(...ConsoleReporter.formatTurn(index + 1, spec.turns[index].input, turn));
|
|
3343
3799
|
lines.push(...attempt.tests.filter((test) => test.turn === index + 1).flatMap(formatTest));
|
|
3344
3800
|
});
|
|
3345
|
-
if (execution.failure !== void 0) lines.push(`
|
|
3801
|
+
if (execution.failure !== void 0) lines.push(` Stopped at turn ${ran + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
|
|
3346
3802
|
const notRun = attempt.tests.filter((test) => test.turn > ran);
|
|
3347
|
-
if (notRun.length > 0) lines.push("
|
|
3803
|
+
if (notRun.length > 0) lines.push(" Not run:", ...notRun.flatMap(formatTest));
|
|
3348
3804
|
lines.push(...attempt.tests.filter((test) => test.turn === 0).flatMap(formatTest));
|
|
3349
|
-
for (const error of execution.attemptErrors) lines.push(`
|
|
3350
|
-
if (execution.closeError !== void 0) lines.push(`
|
|
3351
|
-
lines.push(`
|
|
3805
|
+
for (const error of execution.attemptErrors) lines.push(` Attempt error: ${error.category} error: ${error.message}`);
|
|
3806
|
+
if (execution.closeError !== void 0) lines.push(` Close error: ${execution.closeError.message}`);
|
|
3807
|
+
lines.push(` Completed ${ran} of ${spec.turns.length} turns in ${execution.durationMs} ms`);
|
|
3352
3808
|
return lines;
|
|
3353
3809
|
}
|
|
3354
3810
|
formatTest(test) {
|
|
3355
|
-
return [`
|
|
3811
|
+
return [` ${this.mark(test.status)} ${test.title} ${test.location.toString()}`, ...test.details.map((detail) => ` ${ConsoleReporter.indent(detail, 11)}`)];
|
|
3356
3812
|
}
|
|
3357
3813
|
static formatTurn(number, input, turn) {
|
|
3358
3814
|
const lines = [
|
|
3359
|
-
`
|
|
3360
|
-
`
|
|
3361
|
-
`
|
|
3815
|
+
` Turn ${number} (${turn.durationMs} ms)`,
|
|
3816
|
+
` Input: ${ConsoleReporter.indent(input, 22)}`,
|
|
3817
|
+
` Response: ${ConsoleReporter.indent(turn.response, 22)}`
|
|
3362
3818
|
];
|
|
3363
|
-
for (const use of turn.toolUses ?? []) lines.push(`
|
|
3364
|
-
for (const result of turn.toolResults ?? []) lines.push(`
|
|
3365
|
-
for (const status of turn.statuses) lines.push(`
|
|
3366
|
-
if (turn.tokenUsage !== void 0) lines.push(`
|
|
3819
|
+
for (const use of turn.toolUses ?? []) lines.push(` Tool use: ${use.name} ${JSON.stringify(use.input ?? use.inputBuffer ?? {})} (${use.toolUseId})`);
|
|
3820
|
+
for (const result of turn.toolResults ?? []) lines.push(` Tool result: ${result.toolUseId} status ${result.status}`);
|
|
3821
|
+
for (const status of turn.statuses) lines.push(` Status: ${ConsoleReporter.indent(status, 22)}`);
|
|
3822
|
+
if (turn.tokenUsage !== void 0) lines.push(` Tokens: input ${turn.tokenUsage.inputTokens}, output ${turn.tokenUsage.outputTokens}, total ${turn.tokenUsage.totalTokens}`);
|
|
3367
3823
|
return lines;
|
|
3368
3824
|
}
|
|
3369
3825
|
static indent(text, width) {
|
|
@@ -3466,6 +3922,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3466
3922
|
const capability = websocket.protocol.capabilities.tokenUsage;
|
|
3467
3923
|
return {
|
|
3468
3924
|
name: result.spec.name,
|
|
3925
|
+
runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
|
|
3469
3926
|
file: result.spec.location.file,
|
|
3470
3927
|
status: result.status,
|
|
3471
3928
|
connection: {
|