@mappies/miniwat-eval 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -5
- package/dist/index.cjs +393 -44
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +9 -2
- package/dist/index.d.ts +10 -2
- package/dist/index.js +393 -44
- package/dist/index.js.map +1 -1
- package/dist/main.js +393 -44
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +35 -1
- package/schemas/eval-config.schema.json +6 -1
- package/schemas/report.schema.json +1 -0
package/dist/index.js
CHANGED
|
@@ -321,6 +321,7 @@ var WebSocketConfig = class {
|
|
|
321
321
|
var ConversationSpec = class {
|
|
322
322
|
version;
|
|
323
323
|
name;
|
|
324
|
+
runName;
|
|
324
325
|
description;
|
|
325
326
|
turns;
|
|
326
327
|
conversation;
|
|
@@ -330,6 +331,7 @@ var ConversationSpec = class {
|
|
|
330
331
|
constructor(data) {
|
|
331
332
|
this.version = data?.version ?? 1;
|
|
332
333
|
this.name = data?.name ?? "";
|
|
334
|
+
this.runName = data?.runName ?? "";
|
|
333
335
|
this.description = data?.description ?? "";
|
|
334
336
|
this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
|
|
335
337
|
this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
|
|
@@ -451,7 +453,7 @@ var EvalConfig = class {
|
|
|
451
453
|
this.version = data?.version ?? 1;
|
|
452
454
|
this.configPath = data?.configPath ?? "";
|
|
453
455
|
this.parametersPath = data?.parametersPath ?? "";
|
|
454
|
-
this.variables =
|
|
456
|
+
this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
|
|
455
457
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
456
458
|
this.execution = new ExecutionConfig(data?.execution);
|
|
457
459
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -766,7 +768,10 @@ var eval_config_schema_default = {
|
|
|
766
768
|
"variables": {
|
|
767
769
|
"type": "object",
|
|
768
770
|
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
769
|
-
"additionalProperties": { "type": "string" }
|
|
771
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
772
|
+
"type": "array",
|
|
773
|
+
"items": { "type": "string" }
|
|
774
|
+
}] }
|
|
770
775
|
},
|
|
771
776
|
"websocket": {
|
|
772
777
|
"type": "object",
|
|
@@ -999,7 +1004,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
999
1004
|
return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
|
|
1000
1005
|
}
|
|
1001
1006
|
resolve(value, document, issues, secrets = [], options = {}) {
|
|
1002
|
-
return this.resolveValue(value, [], document, issues, secrets, options);
|
|
1007
|
+
return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
|
|
1003
1008
|
}
|
|
1004
1009
|
static hasReference(value) {
|
|
1005
1010
|
return new RegExp(REFERENCE.source).test(value);
|
|
@@ -1007,7 +1012,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
1007
1012
|
static isSecretName(name) {
|
|
1008
1013
|
return SECRET_NAME.test(name);
|
|
1009
1014
|
}
|
|
1010
|
-
|
|
1015
|
+
find(name) {
|
|
1011
1016
|
for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
|
|
1012
1017
|
return this.environment[name];
|
|
1013
1018
|
}
|
|
@@ -1022,7 +1027,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
1022
1027
|
resolveValue(value, path, document, issues, secrets, options) {
|
|
1023
1028
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
1024
1029
|
options.used?.add(name);
|
|
1025
|
-
const resolved = this.
|
|
1030
|
+
const resolved = this.find(name);
|
|
1026
1031
|
if (resolved === void 0) {
|
|
1027
1032
|
if (options.optional?.(path)) {
|
|
1028
1033
|
options.unresolved?.add(YamlDocument.formatPath(path));
|
|
@@ -1034,6 +1039,13 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
1034
1039
|
}));
|
|
1035
1040
|
return reference;
|
|
1036
1041
|
}
|
|
1042
|
+
if (Array.isArray(resolved)) {
|
|
1043
|
+
issues.push(new ValidationIssue({
|
|
1044
|
+
message: `variable '${name}' is a list, which can only be used as runWith.from`,
|
|
1045
|
+
location: document.locate(path)
|
|
1046
|
+
}));
|
|
1047
|
+
return reference;
|
|
1048
|
+
}
|
|
1037
1049
|
if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
|
|
1038
1050
|
return resolved;
|
|
1039
1051
|
});
|
|
@@ -1162,9 +1174,25 @@ var VariablesReader = class {
|
|
|
1162
1174
|
}));
|
|
1163
1175
|
continue;
|
|
1164
1176
|
}
|
|
1177
|
+
if (Array.isArray(value)) {
|
|
1178
|
+
const bad = value.findIndex((item) => typeof item !== "string");
|
|
1179
|
+
if (bad >= 0) {
|
|
1180
|
+
issues.push(new ValidationIssue({
|
|
1181
|
+
message: "must be string",
|
|
1182
|
+
location: document.locate([
|
|
1183
|
+
"variables",
|
|
1184
|
+
name,
|
|
1185
|
+
bad
|
|
1186
|
+
])
|
|
1187
|
+
}));
|
|
1188
|
+
continue;
|
|
1189
|
+
}
|
|
1190
|
+
variables[name] = [...value];
|
|
1191
|
+
continue;
|
|
1192
|
+
}
|
|
1165
1193
|
if (typeof value !== "string") {
|
|
1166
1194
|
issues.push(new ValidationIssue({
|
|
1167
|
-
message: "must be string",
|
|
1195
|
+
message: "must be string or a list of strings",
|
|
1168
1196
|
location: document.locate(["variables", name])
|
|
1169
1197
|
}));
|
|
1170
1198
|
continue;
|
|
@@ -1172,8 +1200,9 @@ var VariablesReader = class {
|
|
|
1172
1200
|
const before = issues.length;
|
|
1173
1201
|
const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
|
|
1174
1202
|
if (issues.length > before) continue;
|
|
1175
|
-
|
|
1176
|
-
|
|
1203
|
+
const text = resolved.variables[name];
|
|
1204
|
+
variables[name] = text;
|
|
1205
|
+
if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
|
|
1177
1206
|
}
|
|
1178
1207
|
return variables;
|
|
1179
1208
|
}
|
|
@@ -1478,7 +1507,44 @@ var conversation_schema_default = {
|
|
|
1478
1507
|
"variables": {
|
|
1479
1508
|
"type": "object",
|
|
1480
1509
|
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
1481
|
-
"additionalProperties": { "type": "string" }
|
|
1510
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
1511
|
+
"type": "array",
|
|
1512
|
+
"items": { "type": "string" }
|
|
1513
|
+
}] }
|
|
1514
|
+
},
|
|
1515
|
+
"runWith": {
|
|
1516
|
+
"description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
|
|
1517
|
+
"anyOf": [{
|
|
1518
|
+
"type": "array",
|
|
1519
|
+
"minItems": 1,
|
|
1520
|
+
"items": {
|
|
1521
|
+
"type": "object",
|
|
1522
|
+
"required": ["name"],
|
|
1523
|
+
"properties": { "name": {
|
|
1524
|
+
"type": "string",
|
|
1525
|
+
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
|
|
1526
|
+
} },
|
|
1527
|
+
"additionalProperties": { "type": "string" }
|
|
1528
|
+
}
|
|
1529
|
+
}, {
|
|
1530
|
+
"type": "object",
|
|
1531
|
+
"additionalProperties": false,
|
|
1532
|
+
"required": ["variable", "from"],
|
|
1533
|
+
"properties": {
|
|
1534
|
+
"variable": {
|
|
1535
|
+
"type": "string",
|
|
1536
|
+
"pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
|
|
1537
|
+
},
|
|
1538
|
+
"from": { "anyOf": [{
|
|
1539
|
+
"type": "array",
|
|
1540
|
+
"minItems": 1,
|
|
1541
|
+
"items": { "type": "string" }
|
|
1542
|
+
}, {
|
|
1543
|
+
"type": "string",
|
|
1544
|
+
"pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
|
|
1545
|
+
}] }
|
|
1546
|
+
}
|
|
1547
|
+
}]
|
|
1482
1548
|
},
|
|
1483
1549
|
"turns": {
|
|
1484
1550
|
"type": "array",
|
|
@@ -1637,6 +1703,221 @@ var ConversationValidator = class ConversationValidator {
|
|
|
1637
1703
|
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
1638
1704
|
}
|
|
1639
1705
|
};
|
|
1706
|
+
// src/scenarios/runWithReader.ts
|
|
1707
|
+
const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
|
|
1708
|
+
const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
|
|
1709
|
+
const FORMS = "must be a list of rows, or an object with variable and from";
|
|
1710
|
+
const SINGLE_RUN = {
|
|
1711
|
+
name: "",
|
|
1712
|
+
variables: {},
|
|
1713
|
+
keys: [],
|
|
1714
|
+
location: [],
|
|
1715
|
+
secrets: []
|
|
1716
|
+
};
|
|
1717
|
+
var RunWithReader = class RunWithReader {
|
|
1718
|
+
read(document, resolver, issues, used) {
|
|
1719
|
+
const raw = isRecord(document.value) ? document.value.runWith : void 0;
|
|
1720
|
+
if (raw === void 0) return void 0;
|
|
1721
|
+
if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
|
|
1722
|
+
if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
|
|
1723
|
+
issues.push(new ValidationIssue({
|
|
1724
|
+
message: FORMS,
|
|
1725
|
+
location: document.locate(["runWith"])
|
|
1726
|
+
}));
|
|
1727
|
+
return [];
|
|
1728
|
+
}
|
|
1729
|
+
rows(rows, document, resolver, issues, used) {
|
|
1730
|
+
if (rows.length === 0) {
|
|
1731
|
+
issues.push(new ValidationIssue({
|
|
1732
|
+
message: "must not be empty",
|
|
1733
|
+
location: document.locate(["runWith"])
|
|
1734
|
+
}));
|
|
1735
|
+
return [];
|
|
1736
|
+
}
|
|
1737
|
+
const names = /* @__PURE__ */ new Set();
|
|
1738
|
+
const runs = [];
|
|
1739
|
+
rows.forEach((row, index) => {
|
|
1740
|
+
const path = ["runWith", index];
|
|
1741
|
+
if (!isRecord(row)) {
|
|
1742
|
+
issues.push(new ValidationIssue({
|
|
1743
|
+
message: "must be object",
|
|
1744
|
+
location: document.locate(path)
|
|
1745
|
+
}));
|
|
1746
|
+
return;
|
|
1747
|
+
}
|
|
1748
|
+
const name = row.name;
|
|
1749
|
+
if (name === void 0) issues.push(new ValidationIssue({
|
|
1750
|
+
message: "missing required key 'name'",
|
|
1751
|
+
location: document.locate(path)
|
|
1752
|
+
}));
|
|
1753
|
+
else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
|
|
1754
|
+
message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
|
|
1755
|
+
location: document.locate([...path, "name"])
|
|
1756
|
+
}));
|
|
1757
|
+
else if (names.has(name)) issues.push(new ValidationIssue({
|
|
1758
|
+
message: `duplicate run name '${name}'`,
|
|
1759
|
+
location: document.locate([...path, "name"])
|
|
1760
|
+
}));
|
|
1761
|
+
else names.add(name);
|
|
1762
|
+
const run = {
|
|
1763
|
+
name: typeof name === "string" ? name : "",
|
|
1764
|
+
variables: {},
|
|
1765
|
+
keys: [],
|
|
1766
|
+
location: path,
|
|
1767
|
+
secrets: []
|
|
1768
|
+
};
|
|
1769
|
+
for (const [key, value] of Object.entries(row)) {
|
|
1770
|
+
if (key === "name") continue;
|
|
1771
|
+
if (!VARIABLE_NAME.test(key)) {
|
|
1772
|
+
issues.push(new ValidationIssue({
|
|
1773
|
+
message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1774
|
+
location: document.locateKey(path, key)
|
|
1775
|
+
}));
|
|
1776
|
+
continue;
|
|
1777
|
+
}
|
|
1778
|
+
if (typeof value !== "string") {
|
|
1779
|
+
issues.push(new ValidationIssue({
|
|
1780
|
+
message: "must be string",
|
|
1781
|
+
location: document.locate([...path, key])
|
|
1782
|
+
}));
|
|
1783
|
+
continue;
|
|
1784
|
+
}
|
|
1785
|
+
const resolved = resolver.resolve(value, document, issues, run.secrets, {
|
|
1786
|
+
used,
|
|
1787
|
+
at: [...path, key]
|
|
1788
|
+
});
|
|
1789
|
+
RunWithReader.set(run, key, resolved, document.locateKey(path, key));
|
|
1790
|
+
}
|
|
1791
|
+
runs.push(run);
|
|
1792
|
+
});
|
|
1793
|
+
return runs;
|
|
1794
|
+
}
|
|
1795
|
+
list(form, document, resolver, issues, used) {
|
|
1796
|
+
const before = issues.length;
|
|
1797
|
+
for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
|
|
1798
|
+
message: `unknown key '${key}'`,
|
|
1799
|
+
location: document.locateKey(["runWith"], key)
|
|
1800
|
+
}));
|
|
1801
|
+
const variable = form.variable;
|
|
1802
|
+
if (variable === void 0) issues.push(new ValidationIssue({
|
|
1803
|
+
message: "missing required key 'variable'",
|
|
1804
|
+
location: document.locate(["runWith"])
|
|
1805
|
+
}));
|
|
1806
|
+
else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
|
|
1807
|
+
message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
|
|
1808
|
+
location: document.locate(["runWith", "variable"])
|
|
1809
|
+
}));
|
|
1810
|
+
const entries = this.entries(form.from, document, resolver, issues, used);
|
|
1811
|
+
if (issues.length > before || entries === void 0) return [];
|
|
1812
|
+
const name = variable;
|
|
1813
|
+
const keyLocation = document.locate(["runWith", "variable"]);
|
|
1814
|
+
const names = /* @__PURE__ */ new Set();
|
|
1815
|
+
const runs = [];
|
|
1816
|
+
for (const { entry, path } of entries) {
|
|
1817
|
+
if (!VARIABLE_NAME.test(entry)) {
|
|
1818
|
+
issues.push(new ValidationIssue({
|
|
1819
|
+
message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1820
|
+
location: document.locate(path)
|
|
1821
|
+
}));
|
|
1822
|
+
continue;
|
|
1823
|
+
}
|
|
1824
|
+
if (names.has(entry)) {
|
|
1825
|
+
issues.push(new ValidationIssue({
|
|
1826
|
+
message: `duplicate run name '${entry}'`,
|
|
1827
|
+
location: document.locate(path)
|
|
1828
|
+
}));
|
|
1829
|
+
continue;
|
|
1830
|
+
}
|
|
1831
|
+
names.add(entry);
|
|
1832
|
+
const run = {
|
|
1833
|
+
name: entry,
|
|
1834
|
+
variables: {},
|
|
1835
|
+
keys: [],
|
|
1836
|
+
location: path,
|
|
1837
|
+
secrets: []
|
|
1838
|
+
};
|
|
1839
|
+
const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
|
|
1840
|
+
used,
|
|
1841
|
+
at: path
|
|
1842
|
+
});
|
|
1843
|
+
RunWithReader.set(run, name, value, keyLocation);
|
|
1844
|
+
runs.push(run);
|
|
1845
|
+
}
|
|
1846
|
+
return runs;
|
|
1847
|
+
}
|
|
1848
|
+
entries(from, document, resolver, issues, used) {
|
|
1849
|
+
const path = ["runWith", "from"];
|
|
1850
|
+
const location = document.locate(path);
|
|
1851
|
+
if (from === void 0) {
|
|
1852
|
+
issues.push(new ValidationIssue({
|
|
1853
|
+
message: "missing required key 'from'",
|
|
1854
|
+
location: document.locate(["runWith"])
|
|
1855
|
+
}));
|
|
1856
|
+
return;
|
|
1857
|
+
}
|
|
1858
|
+
let list;
|
|
1859
|
+
let at;
|
|
1860
|
+
if (Array.isArray(from)) {
|
|
1861
|
+
list = from;
|
|
1862
|
+
at = (index) => [...path, index];
|
|
1863
|
+
} else {
|
|
1864
|
+
const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
|
|
1865
|
+
if (reference === null) {
|
|
1866
|
+
issues.push(new ValidationIssue({
|
|
1867
|
+
message: "must be a list, or a list variable such as ${tokens}",
|
|
1868
|
+
location
|
|
1869
|
+
}));
|
|
1870
|
+
return;
|
|
1871
|
+
}
|
|
1872
|
+
const name = reference[1];
|
|
1873
|
+
const found = resolver.find(name);
|
|
1874
|
+
used.add(name);
|
|
1875
|
+
if (found === void 0) {
|
|
1876
|
+
issues.push(new ValidationIssue({
|
|
1877
|
+
message: `list variable '${name}' is not set`,
|
|
1878
|
+
location
|
|
1879
|
+
}));
|
|
1880
|
+
return;
|
|
1881
|
+
}
|
|
1882
|
+
if (!Array.isArray(found)) {
|
|
1883
|
+
issues.push(new ValidationIssue({
|
|
1884
|
+
message: `variable '${name}' is not a list`,
|
|
1885
|
+
location
|
|
1886
|
+
}));
|
|
1887
|
+
return;
|
|
1888
|
+
}
|
|
1889
|
+
list = found;
|
|
1890
|
+
at = () => path;
|
|
1891
|
+
}
|
|
1892
|
+
if (list.length === 0) {
|
|
1893
|
+
issues.push(new ValidationIssue({
|
|
1894
|
+
message: "must not be empty",
|
|
1895
|
+
location
|
|
1896
|
+
}));
|
|
1897
|
+
return;
|
|
1898
|
+
}
|
|
1899
|
+
const bad = list.findIndex((item) => typeof item !== "string");
|
|
1900
|
+
if (bad >= 0) {
|
|
1901
|
+
issues.push(new ValidationIssue({
|
|
1902
|
+
message: "must be string",
|
|
1903
|
+
location: document.locate(at(bad))
|
|
1904
|
+
}));
|
|
1905
|
+
return;
|
|
1906
|
+
}
|
|
1907
|
+
return list.map((entry, index) => ({
|
|
1908
|
+
entry,
|
|
1909
|
+
path: at(index)
|
|
1910
|
+
}));
|
|
1911
|
+
}
|
|
1912
|
+
static set(run, name, value, location) {
|
|
1913
|
+
run.variables[name] = value;
|
|
1914
|
+
run.keys.push({
|
|
1915
|
+
name,
|
|
1916
|
+
location
|
|
1917
|
+
});
|
|
1918
|
+
if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
|
|
1919
|
+
}
|
|
1920
|
+
};
|
|
1640
1921
|
// src/scenarios/scenarioLoader.ts
|
|
1641
1922
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
1642
1923
|
var ScenarioLoader = class ScenarioLoader {
|
|
@@ -1645,6 +1926,7 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1645
1926
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
1646
1927
|
conversationValidator = new ConversationValidator();
|
|
1647
1928
|
variablesReader = new VariablesReader();
|
|
1929
|
+
runWithReader = new RunWithReader();
|
|
1648
1930
|
webSocketResolver = new WebSocketResolver();
|
|
1649
1931
|
environmentResolver;
|
|
1650
1932
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
@@ -1660,45 +1942,59 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1660
1942
|
for (const file of files) {
|
|
1661
1943
|
const document = this.documentLoader.load(file, relative(this.cwd, file));
|
|
1662
1944
|
const fileIssues = [...document.errors];
|
|
1663
|
-
const
|
|
1945
|
+
const fileSecrets = [];
|
|
1664
1946
|
const used = /* @__PURE__ */ new Set();
|
|
1665
|
-
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues,
|
|
1666
|
-
const
|
|
1667
|
-
|
|
1668
|
-
if (fileIssues.length
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
...resolver.resolve(rest, document, fileIssues, secrets, { used }),
|
|
1672
|
-
...raw === void 0 ? {} : { variables }
|
|
1673
|
-
};
|
|
1947
|
+
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
|
|
1948
|
+
const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
|
|
1949
|
+
const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
|
|
1950
|
+
if (fileIssues.length > 0) {
|
|
1951
|
+
issues.push(...fileIssues);
|
|
1952
|
+
continue;
|
|
1674
1953
|
}
|
|
1675
|
-
|
|
1676
|
-
const
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1954
|
+
const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
|
|
1955
|
+
const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
|
|
1956
|
+
for (const result of results) for (const name of result.used) used.add(name);
|
|
1957
|
+
fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
|
|
1958
|
+
run: result.run,
|
|
1959
|
+
issues: result.issues
|
|
1960
|
+
})), (issue, run) => run === void 0 ? issue : new ValidationIssue({
|
|
1961
|
+
message: `run '${run.name}': ${issue.message}`,
|
|
1962
|
+
location: issue.location
|
|
1963
|
+
})));
|
|
1964
|
+
const attempted = results.filter((result) => result.connectionChecked);
|
|
1965
|
+
if (attempted.length > 0) connectionIssues.push({
|
|
1966
|
+
file: document,
|
|
1967
|
+
runs: attempted.map((result) => ({
|
|
1968
|
+
run: result.run,
|
|
1969
|
+
issues: result.websocketIssues
|
|
1970
|
+
}))
|
|
1971
|
+
});
|
|
1972
|
+
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
|
|
1973
|
+
if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
|
|
1974
|
+
const websocket = result.websocket;
|
|
1975
|
+
specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
1685
1976
|
}
|
|
1686
|
-
if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
|
|
1687
|
-
if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
|
|
1688
|
-
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
|
|
1689
|
-
if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
1690
1977
|
else issues.push(...fileIssues);
|
|
1691
1978
|
}
|
|
1692
|
-
const shared = ScenarioLoader.shared(connectionIssues.
|
|
1979
|
+
const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
|
|
1693
1980
|
if (shared.length > 0) throw new EvalError({
|
|
1694
1981
|
category: ErrorCategory.Configuration,
|
|
1695
1982
|
message: "invalid configuration",
|
|
1696
1983
|
issues: shared
|
|
1697
1984
|
});
|
|
1698
|
-
for (const entry of connectionIssues)
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1985
|
+
for (const entry of connectionIssues) {
|
|
1986
|
+
const fileLocation = ScenarioLoader.variablesLocation(entry.file);
|
|
1987
|
+
issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
|
|
1988
|
+
message: issue.toString(),
|
|
1989
|
+
location: fileLocation
|
|
1990
|
+
}) : new ValidationIssue({
|
|
1991
|
+
message: `run '${run.name}': ${issue.toString()}`,
|
|
1992
|
+
location: new SourceLocation({
|
|
1993
|
+
...entry.file.locate(run.location),
|
|
1994
|
+
path: ""
|
|
1995
|
+
})
|
|
1996
|
+
})));
|
|
1997
|
+
}
|
|
1702
1998
|
if (issues.length > 0) throw new EvalError({
|
|
1703
1999
|
category: ErrorCategory.Specification,
|
|
1704
2000
|
message: "invalid scenario specification",
|
|
@@ -1706,6 +2002,43 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1706
2002
|
});
|
|
1707
2003
|
return specs;
|
|
1708
2004
|
}
|
|
2005
|
+
resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
|
|
2006
|
+
const issues = [];
|
|
2007
|
+
const used = /* @__PURE__ */ new Set();
|
|
2008
|
+
const resolver = scenarioResolver.withLevels([run.variables]);
|
|
2009
|
+
const value = {
|
|
2010
|
+
...resolver.resolve(rest, document, issues, secrets, { used }),
|
|
2011
|
+
...variables === void 0 ? {} : { variables }
|
|
2012
|
+
};
|
|
2013
|
+
const websocketIssues = [];
|
|
2014
|
+
let websocket;
|
|
2015
|
+
let connectionChecked = false;
|
|
2016
|
+
if (issues.length === 0 && configDocument !== void 0) {
|
|
2017
|
+
const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
|
|
2018
|
+
if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
|
|
2019
|
+
if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
|
|
2020
|
+
connectionChecked = true;
|
|
2021
|
+
}
|
|
2022
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
|
|
2023
|
+
if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
|
|
2024
|
+
return {
|
|
2025
|
+
run,
|
|
2026
|
+
value,
|
|
2027
|
+
used,
|
|
2028
|
+
secrets,
|
|
2029
|
+
issues,
|
|
2030
|
+
websocket,
|
|
2031
|
+
websocketIssues,
|
|
2032
|
+
connectionChecked
|
|
2033
|
+
};
|
|
2034
|
+
}
|
|
2035
|
+
static byRun(runs, place) {
|
|
2036
|
+
const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
|
|
2037
|
+
const sharedTexts = new Set(shared.map((issue) => issue.toString()));
|
|
2038
|
+
const result = shared.map((issue) => place(issue, void 0));
|
|
2039
|
+
for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
|
|
2040
|
+
return result;
|
|
2041
|
+
}
|
|
1709
2042
|
static shared(lists) {
|
|
1710
2043
|
if (lists.length === 0) return [];
|
|
1711
2044
|
const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
|
|
@@ -1717,11 +2050,22 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1717
2050
|
path: ""
|
|
1718
2051
|
});
|
|
1719
2052
|
}
|
|
1720
|
-
static unusedVariables(variables, used, document) {
|
|
1721
|
-
|
|
1722
|
-
|
|
2053
|
+
static unusedVariables(variables, used, results, document) {
|
|
2054
|
+
const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
|
|
2055
|
+
const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
|
|
2056
|
+
message: message(name),
|
|
1723
2057
|
location: document.locateKey(["variables"], name)
|
|
1724
2058
|
}));
|
|
2059
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2060
|
+
for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
|
|
2061
|
+
const issue = new ValidationIssue({
|
|
2062
|
+
message: message(key.name),
|
|
2063
|
+
location: key.location
|
|
2064
|
+
});
|
|
2065
|
+
if (!seen.has(issue.toString())) issues.push(issue);
|
|
2066
|
+
seen.add(issue.toString());
|
|
2067
|
+
}
|
|
2068
|
+
return issues;
|
|
1725
2069
|
}
|
|
1726
2070
|
discover(target, config) {
|
|
1727
2071
|
if (!existsSync(target)) throw new EvalError({
|
|
@@ -1747,13 +2091,14 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1747
2091
|
}
|
|
1748
2092
|
return files;
|
|
1749
2093
|
}
|
|
1750
|
-
build(value, document, websocket, secrets) {
|
|
2094
|
+
build(value, document, runName, websocket, secrets) {
|
|
1751
2095
|
const scenario = value;
|
|
1752
2096
|
const turns = scenario.turns;
|
|
1753
2097
|
const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
|
|
1754
2098
|
return new ConversationSpec({
|
|
1755
2099
|
version: scenario.version,
|
|
1756
2100
|
name: scenario.name,
|
|
2101
|
+
runName,
|
|
1757
2102
|
description: typeof scenario.description === "string" ? scenario.description : "",
|
|
1758
2103
|
turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
|
|
1759
2104
|
conversation: conversation ? new ConversationExpectation({
|
|
@@ -3221,7 +3566,7 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3221
3566
|
scenario(result, repeat, tokenUsageCapability) {
|
|
3222
3567
|
const spec = result.spec;
|
|
3223
3568
|
const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
|
|
3224
|
-
const line = `${this.status(result.status, STATUS_WIDTH)} ${spec
|
|
3569
|
+
const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
|
|
3225
3570
|
if (!this.verbose) return line;
|
|
3226
3571
|
if (result.attempts.length === 0) return `${line}\n`;
|
|
3227
3572
|
const websocket = spec.websocket;
|
|
@@ -3234,6 +3579,9 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3234
3579
|
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3235
3580
|
return line + lines.join("\n");
|
|
3236
3581
|
}
|
|
3582
|
+
static title(spec) {
|
|
3583
|
+
return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
|
|
3584
|
+
}
|
|
3237
3585
|
summary(run, exitCode, reportPaths = []) {
|
|
3238
3586
|
const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
|
|
3239
3587
|
const lines = this.verbose ? [] : [""];
|
|
@@ -3265,7 +3613,7 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3265
3613
|
}
|
|
3266
3614
|
problems(result) {
|
|
3267
3615
|
const spec = result.spec;
|
|
3268
|
-
const lines = [`${this.status(result.status, 6)} ${spec
|
|
3616
|
+
const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
|
|
3269
3617
|
for (const attempt of result.attempts) {
|
|
3270
3618
|
const execution = attempt.execution;
|
|
3271
3619
|
if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
|
|
@@ -3443,6 +3791,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3443
3791
|
const capability = websocket.protocol.capabilities.tokenUsage;
|
|
3444
3792
|
return {
|
|
3445
3793
|
name: result.spec.name,
|
|
3794
|
+
runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
|
|
3446
3795
|
file: result.spec.location.file,
|
|
3447
3796
|
status: result.status,
|
|
3448
3797
|
connection: {
|