@mappies/miniwat-eval 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -11
- package/dist/index.cjs +389 -173
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +58 -48
- package/dist/index.d.ts +58 -48
- package/dist/index.js +389 -173
- package/dist/index.js.map +1 -1
- package/dist/main.js +345 -129
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +9 -4
- package/schemas/eval-config.schema.json +5 -0
- package/schemas/report.schema.json +11 -1
package/dist/index.cjs
CHANGED
|
@@ -23,8 +23,8 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
23
23
|
let _mappies_miniwat_agent_core = require("@mappies/miniwat-agent-core");
|
|
24
24
|
let node_fs = require("node:fs");
|
|
25
25
|
let node_path = require("node:path");
|
|
26
|
-
let ajv = require("ajv");
|
|
27
26
|
let yaml = require("yaml");
|
|
27
|
+
let ajv = require("ajv");
|
|
28
28
|
let configurapi_handler_ws = require("configurapi-handler-ws");
|
|
29
29
|
let node_util = require("node:util");
|
|
30
30
|
let _mappies_miniwat_agent = require("@mappies/miniwat-agent");
|
|
@@ -296,6 +296,50 @@ var TurnSpec = class {
|
|
|
296
296
|
this.location = new SourceLocation(data?.location);
|
|
297
297
|
}
|
|
298
298
|
};
|
|
299
|
+
// src/entities/protocolCapabilities.ts
|
|
300
|
+
var ProtocolCapabilities = class {
|
|
301
|
+
toolUses;
|
|
302
|
+
toolResults;
|
|
303
|
+
tokenUsage;
|
|
304
|
+
constructor(data) {
|
|
305
|
+
this.toolUses = data?.toolUses ?? false;
|
|
306
|
+
this.toolResults = data?.toolResults ?? false;
|
|
307
|
+
this.tokenUsage = data?.tokenUsage ?? false;
|
|
308
|
+
}
|
|
309
|
+
};
|
|
310
|
+
// src/interfaces/requestFormat.ts
|
|
311
|
+
const RequestFormat = { Json: "json" };
|
|
312
|
+
// src/entities/requestConfig.ts
|
|
313
|
+
var RequestConfig = class {
|
|
314
|
+
format;
|
|
315
|
+
value;
|
|
316
|
+
inputPath;
|
|
317
|
+
constructor(data) {
|
|
318
|
+
this.format = data?.format ?? RequestFormat.Json;
|
|
319
|
+
this.value = data?.value;
|
|
320
|
+
this.inputPath = data?.inputPath;
|
|
321
|
+
}
|
|
322
|
+
};
|
|
323
|
+
// src/entities/protocolConfig.ts
|
|
324
|
+
var ProtocolConfig = class {
|
|
325
|
+
request;
|
|
326
|
+
capabilities;
|
|
327
|
+
constructor(data) {
|
|
328
|
+
this.request = new RequestConfig(data?.request);
|
|
329
|
+
this.capabilities = new ProtocolCapabilities(data?.capabilities);
|
|
330
|
+
}
|
|
331
|
+
};
|
|
332
|
+
// src/entities/webSocketConfig.ts
|
|
333
|
+
var WebSocketConfig = class {
|
|
334
|
+
endpoint;
|
|
335
|
+
connection;
|
|
336
|
+
protocol;
|
|
337
|
+
constructor(data) {
|
|
338
|
+
this.endpoint = data?.endpoint ?? "";
|
|
339
|
+
this.connection = new ConnectionConfig(data?.connection);
|
|
340
|
+
this.protocol = new ProtocolConfig(data?.protocol);
|
|
341
|
+
}
|
|
342
|
+
};
|
|
299
343
|
// src/entities/conversationSpec.ts
|
|
300
344
|
var ConversationSpec = class {
|
|
301
345
|
version;
|
|
@@ -304,6 +348,7 @@ var ConversationSpec = class {
|
|
|
304
348
|
turns;
|
|
305
349
|
conversation;
|
|
306
350
|
location;
|
|
351
|
+
websocket;
|
|
307
352
|
secrets;
|
|
308
353
|
constructor(data) {
|
|
309
354
|
this.version = data?.version ?? 1;
|
|
@@ -312,6 +357,7 @@ var ConversationSpec = class {
|
|
|
312
357
|
this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
|
|
313
358
|
this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
|
|
314
359
|
this.location = new SourceLocation(data?.location);
|
|
360
|
+
this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
|
|
315
361
|
this.secrets = [...data?.secrets ?? []];
|
|
316
362
|
}
|
|
317
363
|
};
|
|
@@ -413,55 +459,12 @@ var ReportConfig = class {
|
|
|
413
459
|
this.formats = data?.formats ?? [];
|
|
414
460
|
}
|
|
415
461
|
};
|
|
416
|
-
// src/entities/protocolCapabilities.ts
|
|
417
|
-
var ProtocolCapabilities = class {
|
|
418
|
-
toolUses;
|
|
419
|
-
toolResults;
|
|
420
|
-
tokenUsage;
|
|
421
|
-
constructor(data) {
|
|
422
|
-
this.toolUses = data?.toolUses ?? false;
|
|
423
|
-
this.toolResults = data?.toolResults ?? false;
|
|
424
|
-
this.tokenUsage = data?.tokenUsage ?? false;
|
|
425
|
-
}
|
|
426
|
-
};
|
|
427
|
-
// src/interfaces/requestFormat.ts
|
|
428
|
-
const RequestFormat = { Json: "json" };
|
|
429
|
-
// src/entities/requestConfig.ts
|
|
430
|
-
var RequestConfig = class {
|
|
431
|
-
format;
|
|
432
|
-
value;
|
|
433
|
-
inputPath;
|
|
434
|
-
constructor(data) {
|
|
435
|
-
this.format = data?.format ?? RequestFormat.Json;
|
|
436
|
-
this.value = data?.value;
|
|
437
|
-
this.inputPath = data?.inputPath;
|
|
438
|
-
}
|
|
439
|
-
};
|
|
440
|
-
// src/entities/protocolConfig.ts
|
|
441
|
-
var ProtocolConfig = class {
|
|
442
|
-
request;
|
|
443
|
-
capabilities;
|
|
444
|
-
constructor(data) {
|
|
445
|
-
this.request = new RequestConfig(data?.request);
|
|
446
|
-
this.capabilities = new ProtocolCapabilities(data?.capabilities);
|
|
447
|
-
}
|
|
448
|
-
};
|
|
449
|
-
// src/entities/webSocketConfig.ts
|
|
450
|
-
var WebSocketConfig = class {
|
|
451
|
-
endpoint;
|
|
452
|
-
connection;
|
|
453
|
-
protocol;
|
|
454
|
-
constructor(data) {
|
|
455
|
-
this.endpoint = data?.endpoint ?? "";
|
|
456
|
-
this.connection = new ConnectionConfig(data?.connection);
|
|
457
|
-
this.protocol = new ProtocolConfig(data?.protocol);
|
|
458
|
-
}
|
|
459
|
-
};
|
|
460
462
|
// src/entities/evalConfig.ts
|
|
461
463
|
var EvalConfig = class {
|
|
462
464
|
version;
|
|
463
465
|
configPath;
|
|
464
466
|
parametersPath;
|
|
467
|
+
variables;
|
|
465
468
|
websocket;
|
|
466
469
|
execution;
|
|
467
470
|
evaluator;
|
|
@@ -471,6 +474,7 @@ var EvalConfig = class {
|
|
|
471
474
|
this.version = data?.version ?? 1;
|
|
472
475
|
this.configPath = data?.configPath ?? "";
|
|
473
476
|
this.parametersPath = data?.parametersPath ?? "";
|
|
477
|
+
this.variables = { ...data?.variables ?? {} };
|
|
474
478
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
475
479
|
this.execution = new ExecutionConfig(data?.execution);
|
|
476
480
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -782,6 +786,11 @@ var eval_config_schema_default = {
|
|
|
782
786
|
],
|
|
783
787
|
properties: {
|
|
784
788
|
"version": { "const": 1 },
|
|
789
|
+
"variables": {
|
|
790
|
+
"type": "object",
|
|
791
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
792
|
+
"additionalProperties": { "type": "string" }
|
|
793
|
+
},
|
|
785
794
|
"websocket": {
|
|
786
795
|
"type": "object",
|
|
787
796
|
"additionalProperties": false,
|
|
@@ -940,18 +949,80 @@ var eval_config_schema_default = {
|
|
|
940
949
|
}
|
|
941
950
|
}
|
|
942
951
|
};
|
|
952
|
+
// src/yaml/yamlDocument.ts
|
|
953
|
+
var YamlDocument = class YamlDocument {
|
|
954
|
+
file;
|
|
955
|
+
document;
|
|
956
|
+
lineCounter;
|
|
957
|
+
value;
|
|
958
|
+
errors;
|
|
959
|
+
constructor(file, document, lineCounter) {
|
|
960
|
+
this.file = file;
|
|
961
|
+
this.document = document;
|
|
962
|
+
this.lineCounter = lineCounter;
|
|
963
|
+
this.value = document.toJS();
|
|
964
|
+
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
965
|
+
message: error.message.split("\n")[0],
|
|
966
|
+
location: this.toLocation(error.pos[0], [])
|
|
967
|
+
}));
|
|
968
|
+
}
|
|
969
|
+
locate(path) {
|
|
970
|
+
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
971
|
+
}
|
|
972
|
+
locateKey(path, key) {
|
|
973
|
+
const parent = this.findNode(path);
|
|
974
|
+
if ((0, yaml.isMap)(parent)) {
|
|
975
|
+
const pair = parent.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === key);
|
|
976
|
+
if (pair && (0, yaml.isScalar)(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
977
|
+
}
|
|
978
|
+
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
979
|
+
}
|
|
980
|
+
static formatPath(path) {
|
|
981
|
+
return path.map((segment, index) => {
|
|
982
|
+
if (typeof segment === "number") return `[${segment}]`;
|
|
983
|
+
return index === 0 ? segment : `.${segment}`;
|
|
984
|
+
}).join("");
|
|
985
|
+
}
|
|
986
|
+
findNode(path) {
|
|
987
|
+
let node = this.document.contents ?? void 0;
|
|
988
|
+
for (const segment of path) {
|
|
989
|
+
let next;
|
|
990
|
+
if ((0, yaml.isMap)(node)) {
|
|
991
|
+
const pair = node.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === String(segment));
|
|
992
|
+
next = pair?.value ?? pair?.key;
|
|
993
|
+
} else if ((0, yaml.isSeq)(node) && typeof segment === "number") next = node.items[segment];
|
|
994
|
+
if (!next) break;
|
|
995
|
+
node = next;
|
|
996
|
+
}
|
|
997
|
+
return node;
|
|
998
|
+
}
|
|
999
|
+
toLocation(offset, path) {
|
|
1000
|
+
const position = this.lineCounter.linePos(offset);
|
|
1001
|
+
return new SourceLocation({
|
|
1002
|
+
file: this.file,
|
|
1003
|
+
line: position.line,
|
|
1004
|
+
column: position.col,
|
|
1005
|
+
path: YamlDocument.formatPath(path)
|
|
1006
|
+
});
|
|
1007
|
+
}
|
|
1008
|
+
};
|
|
943
1009
|
// src/yaml/environmentResolver.ts
|
|
944
1010
|
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
|
|
945
1011
|
const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
|
|
946
1012
|
var EnvironmentResolver = class EnvironmentResolver {
|
|
947
1013
|
environment;
|
|
948
1014
|
parameters;
|
|
949
|
-
|
|
1015
|
+
levels;
|
|
1016
|
+
constructor(environment = process.env, parameters = {}, levels = []) {
|
|
950
1017
|
this.environment = environment;
|
|
951
1018
|
this.parameters = parameters;
|
|
1019
|
+
this.levels = levels;
|
|
1020
|
+
}
|
|
1021
|
+
withLevels(levels) {
|
|
1022
|
+
return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
|
|
952
1023
|
}
|
|
953
|
-
resolve(value, document, issues, secrets = []) {
|
|
954
|
-
return this.resolveValue(value, [], document, issues, secrets);
|
|
1024
|
+
resolve(value, document, issues, secrets = [], options = {}) {
|
|
1025
|
+
return this.resolveValue(value, [], document, issues, secrets, options);
|
|
955
1026
|
}
|
|
956
1027
|
static hasReference(value) {
|
|
957
1028
|
return new RegExp(REFERENCE.source).test(value);
|
|
@@ -959,22 +1030,38 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
959
1030
|
static isSecretName(name) {
|
|
960
1031
|
return SECRET_NAME.test(name);
|
|
961
1032
|
}
|
|
962
|
-
|
|
1033
|
+
lookup(name) {
|
|
1034
|
+
for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
|
|
1035
|
+
return this.environment[name];
|
|
1036
|
+
}
|
|
1037
|
+
sources() {
|
|
1038
|
+
const sources = [
|
|
1039
|
+
...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
|
|
1040
|
+
...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
|
|
1041
|
+
"environment variable"
|
|
1042
|
+
];
|
|
1043
|
+
return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
|
|
1044
|
+
}
|
|
1045
|
+
resolveValue(value, path, document, issues, secrets, options) {
|
|
963
1046
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
964
|
-
|
|
1047
|
+
options.used?.add(name);
|
|
1048
|
+
const resolved = this.lookup(name);
|
|
965
1049
|
if (resolved === void 0) {
|
|
966
|
-
|
|
1050
|
+
if (options.optional?.(path)) {
|
|
1051
|
+
options.unresolved?.add(YamlDocument.formatPath(path));
|
|
1052
|
+
return reference;
|
|
1053
|
+
}
|
|
967
1054
|
issues.push(new ValidationIssue({
|
|
968
|
-
message: `${
|
|
1055
|
+
message: `${this.sources()} '${name}' is not set`,
|
|
969
1056
|
location: document.locate(path)
|
|
970
1057
|
}));
|
|
971
1058
|
return reference;
|
|
972
1059
|
}
|
|
973
|
-
if (EnvironmentResolver.isSecretName(name) && !secrets.includes(resolved)) secrets.push(resolved);
|
|
1060
|
+
if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
|
|
974
1061
|
return resolved;
|
|
975
1062
|
});
|
|
976
|
-
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
|
|
977
|
-
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
|
|
1063
|
+
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
|
|
1064
|
+
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
|
|
978
1065
|
return value;
|
|
979
1066
|
}
|
|
980
1067
|
};
|
|
@@ -1058,63 +1145,6 @@ var SchemaValidator = class SchemaValidator {
|
|
|
1058
1145
|
return path;
|
|
1059
1146
|
}
|
|
1060
1147
|
};
|
|
1061
|
-
// src/yaml/yamlDocument.ts
|
|
1062
|
-
var YamlDocument = class YamlDocument {
|
|
1063
|
-
file;
|
|
1064
|
-
document;
|
|
1065
|
-
lineCounter;
|
|
1066
|
-
value;
|
|
1067
|
-
errors;
|
|
1068
|
-
constructor(file, document, lineCounter) {
|
|
1069
|
-
this.file = file;
|
|
1070
|
-
this.document = document;
|
|
1071
|
-
this.lineCounter = lineCounter;
|
|
1072
|
-
this.value = document.toJS();
|
|
1073
|
-
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
1074
|
-
message: error.message.split("\n")[0],
|
|
1075
|
-
location: this.toLocation(error.pos[0], [])
|
|
1076
|
-
}));
|
|
1077
|
-
}
|
|
1078
|
-
locate(path) {
|
|
1079
|
-
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
1080
|
-
}
|
|
1081
|
-
locateKey(path, key) {
|
|
1082
|
-
const parent = this.findNode(path);
|
|
1083
|
-
if ((0, yaml.isMap)(parent)) {
|
|
1084
|
-
const pair = parent.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === key);
|
|
1085
|
-
if (pair && (0, yaml.isScalar)(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
1086
|
-
}
|
|
1087
|
-
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
1088
|
-
}
|
|
1089
|
-
static formatPath(path) {
|
|
1090
|
-
return path.map((segment, index) => {
|
|
1091
|
-
if (typeof segment === "number") return `[${segment}]`;
|
|
1092
|
-
return index === 0 ? segment : `.${segment}`;
|
|
1093
|
-
}).join("");
|
|
1094
|
-
}
|
|
1095
|
-
findNode(path) {
|
|
1096
|
-
let node = this.document.contents ?? void 0;
|
|
1097
|
-
for (const segment of path) {
|
|
1098
|
-
let next;
|
|
1099
|
-
if ((0, yaml.isMap)(node)) {
|
|
1100
|
-
const pair = node.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === String(segment));
|
|
1101
|
-
next = pair?.value ?? pair?.key;
|
|
1102
|
-
} else if ((0, yaml.isSeq)(node) && typeof segment === "number") next = node.items[segment];
|
|
1103
|
-
if (!next) break;
|
|
1104
|
-
node = next;
|
|
1105
|
-
}
|
|
1106
|
-
return node;
|
|
1107
|
-
}
|
|
1108
|
-
toLocation(offset, path) {
|
|
1109
|
-
const position = this.lineCounter.linePos(offset);
|
|
1110
|
-
return new SourceLocation({
|
|
1111
|
-
file: this.file,
|
|
1112
|
-
line: position.line,
|
|
1113
|
-
column: position.col,
|
|
1114
|
-
path: YamlDocument.formatPath(path)
|
|
1115
|
-
});
|
|
1116
|
-
}
|
|
1117
|
-
};
|
|
1118
1148
|
// src/yaml/yamlDocumentLoader.ts
|
|
1119
1149
|
var YamlDocumentLoader = class {
|
|
1120
1150
|
load(filePath, displayPath) {
|
|
@@ -1133,11 +1163,133 @@ var YamlDocumentLoader = class {
|
|
|
1133
1163
|
function isRecord(value) {
|
|
1134
1164
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
1135
1165
|
}
|
|
1166
|
+
// src/yaml/variablesReader.ts
|
|
1167
|
+
const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
1168
|
+
var VariablesReader = class {
|
|
1169
|
+
read(document, resolver, issues, secrets) {
|
|
1170
|
+
const raw = isRecord(document.value) ? document.value.variables : void 0;
|
|
1171
|
+
if (raw === void 0) return {};
|
|
1172
|
+
if (!isRecord(raw)) {
|
|
1173
|
+
issues.push(new ValidationIssue({
|
|
1174
|
+
message: "must be object",
|
|
1175
|
+
location: document.locate(["variables"])
|
|
1176
|
+
}));
|
|
1177
|
+
return {};
|
|
1178
|
+
}
|
|
1179
|
+
const variables = {};
|
|
1180
|
+
for (const [name, value] of Object.entries(raw)) {
|
|
1181
|
+
if (!VARIABLE_NAME.test(name)) {
|
|
1182
|
+
issues.push(new ValidationIssue({
|
|
1183
|
+
message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1184
|
+
location: document.locateKey(["variables"], name)
|
|
1185
|
+
}));
|
|
1186
|
+
continue;
|
|
1187
|
+
}
|
|
1188
|
+
if (typeof value !== "string") {
|
|
1189
|
+
issues.push(new ValidationIssue({
|
|
1190
|
+
message: "must be string",
|
|
1191
|
+
location: document.locate(["variables", name])
|
|
1192
|
+
}));
|
|
1193
|
+
continue;
|
|
1194
|
+
}
|
|
1195
|
+
const before = issues.length;
|
|
1196
|
+
const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
|
|
1197
|
+
if (issues.length > before) continue;
|
|
1198
|
+
variables[name] = resolved.variables[name];
|
|
1199
|
+
if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
|
|
1200
|
+
}
|
|
1201
|
+
return variables;
|
|
1202
|
+
}
|
|
1203
|
+
};
|
|
1204
|
+
// src/config/webSocketResolver.ts
|
|
1205
|
+
const WEBSOCKET_SCHEMA = {
|
|
1206
|
+
...eval_config_schema_default,
|
|
1207
|
+
required: ["websocket"],
|
|
1208
|
+
additionalProperties: true,
|
|
1209
|
+
properties: { websocket: eval_config_schema_default.properties.websocket }
|
|
1210
|
+
};
|
|
1211
|
+
var WebSocketResolver = class WebSocketResolver {
|
|
1212
|
+
schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
|
|
1213
|
+
resolve(document, resolver, issues, secrets, used, defer) {
|
|
1214
|
+
const raw = isRecord(document.value) ? document.value.websocket : void 0;
|
|
1215
|
+
const unresolved = /* @__PURE__ */ new Set();
|
|
1216
|
+
const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
|
|
1217
|
+
const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
|
|
1218
|
+
used,
|
|
1219
|
+
optional,
|
|
1220
|
+
unresolved
|
|
1221
|
+
});
|
|
1222
|
+
const subprotocols = WebSocketResolver.subprotocols(raw);
|
|
1223
|
+
if (subprotocols !== void 0) {
|
|
1224
|
+
const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
|
|
1225
|
+
const set = !unresolved.has(YamlDocument.formatPath([
|
|
1226
|
+
"websocket",
|
|
1227
|
+
"connection",
|
|
1228
|
+
"subprotocols",
|
|
1229
|
+
index
|
|
1230
|
+
]));
|
|
1231
|
+
const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
|
|
1232
|
+
return set && !empty;
|
|
1233
|
+
});
|
|
1234
|
+
value.websocket.connection.subprotocols = kept;
|
|
1235
|
+
}
|
|
1236
|
+
const prefix = YamlDocument.formatPath([
|
|
1237
|
+
"websocket",
|
|
1238
|
+
"connection",
|
|
1239
|
+
"subprotocols"
|
|
1240
|
+
]);
|
|
1241
|
+
return {
|
|
1242
|
+
value,
|
|
1243
|
+
deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
|
|
1244
|
+
};
|
|
1245
|
+
}
|
|
1246
|
+
validate(value, document) {
|
|
1247
|
+
const issues = this.schemaValidator.validate(value, document);
|
|
1248
|
+
if (issues.length > 0) return issues;
|
|
1249
|
+
return WebSocketResolver.validateInputPath(value, document);
|
|
1250
|
+
}
|
|
1251
|
+
static validateInputPath(value, document) {
|
|
1252
|
+
const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
|
|
1253
|
+
if (typeof request.inputPath !== "string") return [];
|
|
1254
|
+
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
1255
|
+
const last = tokens.pop();
|
|
1256
|
+
let parent = request.value;
|
|
1257
|
+
for (const token of tokens) parent = WebSocketResolver.child(parent, token);
|
|
1258
|
+
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
1259
|
+
return [new ValidationIssue({
|
|
1260
|
+
message: "must point to a missing or string value whose parent exists in request.value",
|
|
1261
|
+
location: document.locate([
|
|
1262
|
+
"websocket",
|
|
1263
|
+
"protocol",
|
|
1264
|
+
"request",
|
|
1265
|
+
"inputPath"
|
|
1266
|
+
])
|
|
1267
|
+
})];
|
|
1268
|
+
}
|
|
1269
|
+
static section(value, ...path) {
|
|
1270
|
+
let current = value;
|
|
1271
|
+
for (const key of path) current = WebSocketResolver.child(current, key);
|
|
1272
|
+
return isRecord(current) ? current : {};
|
|
1273
|
+
}
|
|
1274
|
+
static child(value, key) {
|
|
1275
|
+
if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
|
|
1276
|
+
return isRecord(value) ? value[key] : void 0;
|
|
1277
|
+
}
|
|
1278
|
+
static isSubprotocol(path) {
|
|
1279
|
+
return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
|
|
1280
|
+
}
|
|
1281
|
+
static subprotocols(websocket) {
|
|
1282
|
+
const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
|
|
1283
|
+
return Array.isArray(subprotocols) ? subprotocols : void 0;
|
|
1284
|
+
}
|
|
1285
|
+
};
|
|
1136
1286
|
// src/config/evalConfigLoader.ts
|
|
1137
1287
|
var EvalConfigLoader = class EvalConfigLoader {
|
|
1138
1288
|
cwd;
|
|
1139
1289
|
documentLoader = new YamlDocumentLoader();
|
|
1140
1290
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
1291
|
+
variablesReader = new VariablesReader();
|
|
1292
|
+
webSocketResolver = new WebSocketResolver();
|
|
1141
1293
|
environmentResolver;
|
|
1142
1294
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1143
1295
|
this.cwd = cwd;
|
|
@@ -1148,19 +1300,47 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1148
1300
|
const issues = [...document.errors];
|
|
1149
1301
|
if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
|
|
1150
1302
|
const named = [];
|
|
1151
|
-
const
|
|
1152
|
-
|
|
1153
|
-
if (issues.length === 0)
|
|
1303
|
+
const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
|
|
1304
|
+
let value = {};
|
|
1305
|
+
if (issues.length === 0) {
|
|
1306
|
+
const resolver = this.environmentResolver.withLevels([variables]);
|
|
1307
|
+
const { websocket: _websocket, variables: _variables, ...rest } = document.value;
|
|
1308
|
+
const resolved = resolver.resolve(rest, document, issues, named);
|
|
1309
|
+
const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
|
|
1310
|
+
value = {
|
|
1311
|
+
...resolved,
|
|
1312
|
+
..._variables === void 0 ? {} : { variables },
|
|
1313
|
+
..._websocket === void 0 ? {} : websocket.value
|
|
1314
|
+
};
|
|
1315
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
|
|
1316
|
+
if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
|
|
1317
|
+
}
|
|
1154
1318
|
if (issues.length > 0) throw new EvalError({
|
|
1155
1319
|
category: ErrorCategory.Configuration,
|
|
1156
1320
|
message: "invalid configuration",
|
|
1157
1321
|
issues
|
|
1158
1322
|
});
|
|
1159
|
-
const config = this.build(value, configPath, named);
|
|
1323
|
+
const config = this.build(value, configPath, variables, named);
|
|
1160
1324
|
this.applyRepetition(config, override, value, document);
|
|
1161
1325
|
EvalConfigLoader.applyReport(config, report);
|
|
1162
1326
|
return config;
|
|
1163
1327
|
}
|
|
1328
|
+
static buildWebSocket(value) {
|
|
1329
|
+
const section = (...path) => WebSocketResolver.section(value, ...path);
|
|
1330
|
+
const connection = section("websocket", "connection");
|
|
1331
|
+
const request = section("websocket", "protocol", "request");
|
|
1332
|
+
return new WebSocketConfig({
|
|
1333
|
+
endpoint: WebSocketResolver.section(value, "websocket").endpoint,
|
|
1334
|
+
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
1335
|
+
protocol: new ProtocolConfig({
|
|
1336
|
+
request: new RequestConfig({
|
|
1337
|
+
...request,
|
|
1338
|
+
format: request.format ?? RequestFormat.Json
|
|
1339
|
+
}),
|
|
1340
|
+
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
1341
|
+
})
|
|
1342
|
+
});
|
|
1343
|
+
}
|
|
1164
1344
|
static applyReport(config, report) {
|
|
1165
1345
|
if (report.formats === void 0) return;
|
|
1166
1346
|
const known = Object.values(ReportFormat);
|
|
@@ -1182,45 +1362,17 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1182
1362
|
location: document.locate(["evaluator", "apiKey"])
|
|
1183
1363
|
})];
|
|
1184
1364
|
}
|
|
1185
|
-
|
|
1186
|
-
const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
|
|
1187
|
-
if (typeof request.inputPath !== "string") return [];
|
|
1188
|
-
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
1189
|
-
const last = tokens.pop();
|
|
1190
|
-
let parent = request.value;
|
|
1191
|
-
for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
|
|
1192
|
-
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
1193
|
-
return [new ValidationIssue({
|
|
1194
|
-
message: "must point to a missing or string value whose parent exists in request.value",
|
|
1195
|
-
location: document.locate([
|
|
1196
|
-
"websocket",
|
|
1197
|
-
"protocol",
|
|
1198
|
-
"request",
|
|
1199
|
-
"inputPath"
|
|
1200
|
-
])
|
|
1201
|
-
})];
|
|
1202
|
-
}
|
|
1203
|
-
build(value, configPath, named) {
|
|
1365
|
+
build(value, configPath, variables, named) {
|
|
1204
1366
|
const section = (...path) => EvalConfigLoader.section(value, ...path);
|
|
1205
1367
|
const configDirectory = (0, node_path.dirname)(configPath);
|
|
1206
|
-
const
|
|
1207
|
-
const request = section("websocket", "protocol", "request");
|
|
1368
|
+
const websocket = EvalConfigLoader.buildWebSocket(value);
|
|
1208
1369
|
const evaluator = section("evaluator");
|
|
1209
1370
|
const report = section("report");
|
|
1210
1371
|
return new EvalConfig({
|
|
1211
1372
|
version: value.version,
|
|
1212
1373
|
configPath,
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
1216
|
-
protocol: new ProtocolConfig({
|
|
1217
|
-
request: new RequestConfig({
|
|
1218
|
-
...request,
|
|
1219
|
-
format: request.format ?? RequestFormat.Json
|
|
1220
|
-
}),
|
|
1221
|
-
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
1222
|
-
})
|
|
1223
|
-
}),
|
|
1374
|
+
variables,
|
|
1375
|
+
websocket,
|
|
1224
1376
|
execution: new ExecutionConfig({
|
|
1225
1377
|
...DEFAULT_CONFIG.execution,
|
|
1226
1378
|
...section("execution"),
|
|
@@ -1241,7 +1393,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1241
1393
|
directory: (0, node_path.resolve)(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
|
|
1242
1394
|
formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
|
|
1243
1395
|
}),
|
|
1244
|
-
secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols
|
|
1396
|
+
secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
|
|
1245
1397
|
});
|
|
1246
1398
|
}
|
|
1247
1399
|
applyRepetition(config, override, value, document) {
|
|
@@ -1294,13 +1446,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1294
1446
|
])];
|
|
1295
1447
|
}
|
|
1296
1448
|
static section(value, ...path) {
|
|
1297
|
-
|
|
1298
|
-
for (const key of path) current = EvalConfigLoader.child(current, key);
|
|
1299
|
-
return isRecord(current) ? current : {};
|
|
1300
|
-
}
|
|
1301
|
-
static child(value, key) {
|
|
1302
|
-
if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
|
|
1303
|
-
return isRecord(value) ? value[key] : void 0;
|
|
1449
|
+
return WebSocketResolver.section(value, ...path);
|
|
1304
1450
|
}
|
|
1305
1451
|
};
|
|
1306
1452
|
// src/config/parametersLoader.ts
|
|
@@ -1352,6 +1498,11 @@ var conversation_schema_default = {
|
|
|
1352
1498
|
"version": { "const": 1 },
|
|
1353
1499
|
"name": { "$ref": "#/definitions/text" },
|
|
1354
1500
|
"description": { "type": "string" },
|
|
1501
|
+
"variables": {
|
|
1502
|
+
"type": "object",
|
|
1503
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
1504
|
+
"additionalProperties": { "type": "string" }
|
|
1505
|
+
},
|
|
1355
1506
|
"turns": {
|
|
1356
1507
|
"type": "array",
|
|
1357
1508
|
"minItems": 1,
|
|
@@ -1364,8 +1515,8 @@ var conversation_schema_default = {
|
|
|
1364
1515
|
"properties": { "conversation": {
|
|
1365
1516
|
"type": "object",
|
|
1366
1517
|
"additionalProperties": false,
|
|
1367
|
-
"required": ["
|
|
1368
|
-
"properties": { "
|
|
1518
|
+
"required": ["should"],
|
|
1519
|
+
"properties": { "should": { "$ref": "#/definitions/text" } }
|
|
1369
1520
|
} }
|
|
1370
1521
|
}
|
|
1371
1522
|
},
|
|
@@ -1408,12 +1559,12 @@ var conversation_schema_default = {
|
|
|
1408
1559
|
}
|
|
1409
1560
|
},
|
|
1410
1561
|
"response": {
|
|
1411
|
-
"description": "A string is shorthand for
|
|
1562
|
+
"description": "A string is shorthand for should.",
|
|
1412
1563
|
"type": ["string", "object"],
|
|
1413
1564
|
"pattern": "\\S",
|
|
1414
1565
|
"additionalProperties": false,
|
|
1415
1566
|
"properties": {
|
|
1416
|
-
"
|
|
1567
|
+
"should": { "$ref": "#/definitions/text" },
|
|
1417
1568
|
"contains": { "$ref": "#/definitions/textList" },
|
|
1418
1569
|
"notContains": { "$ref": "#/definitions/textList" }
|
|
1419
1570
|
}
|
|
@@ -1445,7 +1596,7 @@ var conversation_schema_default = {
|
|
|
1445
1596
|
};
|
|
1446
1597
|
// src/scenarios/conversationValidator.ts
|
|
1447
1598
|
const RESPONSE_KEYS = [
|
|
1448
|
-
"
|
|
1599
|
+
"should",
|
|
1449
1600
|
"contains",
|
|
1450
1601
|
"notContains"
|
|
1451
1602
|
];
|
|
@@ -1468,7 +1619,7 @@ var ConversationValidator = class ConversationValidator {
|
|
|
1468
1619
|
validateResponse(response, path, document, issues) {
|
|
1469
1620
|
if (!RESPONSE_KEYS.some((key) => key in response)) {
|
|
1470
1621
|
issues.push(new ValidationIssue({
|
|
1471
|
-
message: "response must have at least one of:
|
|
1622
|
+
message: "response must have at least one of: should, contains, notContains",
|
|
1472
1623
|
location: document.locate(path)
|
|
1473
1624
|
}));
|
|
1474
1625
|
return;
|
|
@@ -1511,11 +1662,13 @@ var ConversationValidator = class ConversationValidator {
|
|
|
1511
1662
|
};
|
|
1512
1663
|
// src/scenarios/scenarioLoader.ts
|
|
1513
1664
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
1514
|
-
var ScenarioLoader = class {
|
|
1665
|
+
var ScenarioLoader = class ScenarioLoader {
|
|
1515
1666
|
cwd;
|
|
1516
1667
|
documentLoader = new YamlDocumentLoader();
|
|
1517
1668
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
1518
1669
|
conversationValidator = new ConversationValidator();
|
|
1670
|
+
variablesReader = new VariablesReader();
|
|
1671
|
+
webSocketResolver = new WebSocketResolver();
|
|
1519
1672
|
environmentResolver;
|
|
1520
1673
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1521
1674
|
this.cwd = cwd;
|
|
@@ -1523,18 +1676,52 @@ var ScenarioLoader = class {
|
|
|
1523
1676
|
}
|
|
1524
1677
|
load(target, config) {
|
|
1525
1678
|
const files = this.discover((0, node_path.resolve)(this.cwd, target), config);
|
|
1679
|
+
const configDocument = (0, node_fs.existsSync)(config.configPath) ? this.documentLoader.load(config.configPath, (0, node_path.relative)(this.cwd, config.configPath)) : void 0;
|
|
1526
1680
|
const issues = [];
|
|
1681
|
+
const connectionIssues = [];
|
|
1527
1682
|
const specs = [];
|
|
1528
1683
|
for (const file of files) {
|
|
1529
1684
|
const document = this.documentLoader.load(file, (0, node_path.relative)(this.cwd, file));
|
|
1530
1685
|
const fileIssues = [...document.errors];
|
|
1531
1686
|
const secrets = [];
|
|
1532
|
-
const
|
|
1687
|
+
const used = /* @__PURE__ */ new Set();
|
|
1688
|
+
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
|
|
1689
|
+
const resolver = this.environmentResolver.withLevels([variables, config.variables]);
|
|
1690
|
+
let value;
|
|
1691
|
+
if (fileIssues.length === 0) {
|
|
1692
|
+
const { variables: raw, ...rest } = document.value;
|
|
1693
|
+
value = {
|
|
1694
|
+
...resolver.resolve(rest, document, fileIssues, secrets, { used }),
|
|
1695
|
+
...raw === void 0 ? {} : { variables }
|
|
1696
|
+
};
|
|
1697
|
+
}
|
|
1698
|
+
let websocket;
|
|
1699
|
+
const websocketIssues = [];
|
|
1700
|
+
if (fileIssues.length === 0 && configDocument !== void 0) {
|
|
1701
|
+
const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
|
|
1702
|
+
if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
|
|
1703
|
+
connectionIssues.push({
|
|
1704
|
+
file: document,
|
|
1705
|
+
issues: websocketIssues
|
|
1706
|
+
});
|
|
1707
|
+
if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
|
|
1708
|
+
}
|
|
1533
1709
|
if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
|
|
1534
1710
|
if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
|
|
1535
|
-
if (fileIssues.length === 0)
|
|
1711
|
+
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
|
|
1712
|
+
if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
1536
1713
|
else issues.push(...fileIssues);
|
|
1537
1714
|
}
|
|
1715
|
+
const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
|
|
1716
|
+
if (shared.length > 0) throw new EvalError({
|
|
1717
|
+
category: ErrorCategory.Configuration,
|
|
1718
|
+
message: "invalid configuration",
|
|
1719
|
+
issues: shared
|
|
1720
|
+
});
|
|
1721
|
+
for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
|
|
1722
|
+
message: issue.toString(),
|
|
1723
|
+
location: ScenarioLoader.variablesLocation(entry.file)
|
|
1724
|
+
}));
|
|
1538
1725
|
if (issues.length > 0) throw new EvalError({
|
|
1539
1726
|
category: ErrorCategory.Specification,
|
|
1540
1727
|
message: "invalid scenario specification",
|
|
@@ -1542,6 +1729,23 @@ var ScenarioLoader = class {
|
|
|
1542
1729
|
});
|
|
1543
1730
|
return specs;
|
|
1544
1731
|
}
|
|
1732
|
+
static shared(lists) {
|
|
1733
|
+
if (lists.length === 0) return [];
|
|
1734
|
+
const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
|
|
1735
|
+
return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
|
|
1736
|
+
}
|
|
1737
|
+
static variablesLocation(document) {
|
|
1738
|
+
return new SourceLocation({
|
|
1739
|
+
...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
|
|
1740
|
+
path: ""
|
|
1741
|
+
});
|
|
1742
|
+
}
|
|
1743
|
+
static unusedVariables(variables, used, document) {
|
|
1744
|
+
return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
|
|
1745
|
+
message: `variable '${name}' is not used by the scenario or the websocket configuration`,
|
|
1746
|
+
location: document.locateKey(["variables"], name)
|
|
1747
|
+
}));
|
|
1748
|
+
}
|
|
1545
1749
|
discover(target, config) {
|
|
1546
1750
|
if (!(0, node_fs.existsSync)(target)) throw new EvalError({
|
|
1547
1751
|
category: ErrorCategory.Specification,
|
|
@@ -1566,7 +1770,7 @@ var ScenarioLoader = class {
|
|
|
1566
1770
|
}
|
|
1567
1771
|
return files;
|
|
1568
1772
|
}
|
|
1569
|
-
build(value, document, secrets) {
|
|
1773
|
+
build(value, document, websocket, secrets) {
|
|
1570
1774
|
const scenario = value;
|
|
1571
1775
|
const turns = scenario.turns;
|
|
1572
1776
|
const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
|
|
@@ -1576,10 +1780,11 @@ var ScenarioLoader = class {
|
|
|
1576
1780
|
description: typeof scenario.description === "string" ? scenario.description : "",
|
|
1577
1781
|
turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
|
|
1578
1782
|
conversation: conversation ? new ConversationExpectation({
|
|
1579
|
-
criteria: conversation.
|
|
1783
|
+
criteria: conversation.should,
|
|
1580
1784
|
location: document.locate(["expect", "conversation"])
|
|
1581
1785
|
}) : void 0,
|
|
1582
1786
|
location: document.locate([]),
|
|
1787
|
+
websocket,
|
|
1583
1788
|
secrets
|
|
1584
1789
|
});
|
|
1585
1790
|
}
|
|
@@ -1605,7 +1810,7 @@ var ScenarioLoader = class {
|
|
|
1605
1810
|
});
|
|
1606
1811
|
const fields = response;
|
|
1607
1812
|
return new ResponseExpectation({
|
|
1608
|
-
criteria: fields.
|
|
1813
|
+
criteria: fields.should,
|
|
1609
1814
|
contains: fields.contains ?? [],
|
|
1610
1815
|
notContains: fields.notContains ?? [],
|
|
1611
1816
|
location
|
|
@@ -2428,7 +2633,11 @@ var ConversationRunner = class {
|
|
|
2428
2633
|
async run(spec, signal = NEVER_ABORTED) {
|
|
2429
2634
|
const startedAt = Date.now();
|
|
2430
2635
|
const outcomes = [];
|
|
2431
|
-
const
|
|
2636
|
+
const config = new EvalConfig({
|
|
2637
|
+
...this.config,
|
|
2638
|
+
websocket: spec.websocket ?? this.config.websocket
|
|
2639
|
+
});
|
|
2640
|
+
const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
|
|
2432
2641
|
const history = [];
|
|
2433
2642
|
const evaluate = this.semanticEvaluator(signal);
|
|
2434
2643
|
const inline = !this.config.execution.continueOnFailure;
|
|
@@ -2453,7 +2662,7 @@ var ConversationRunner = class {
|
|
|
2453
2662
|
try {
|
|
2454
2663
|
const steps = (async () => {
|
|
2455
2664
|
for (let attempt = 0;; attempt++) {
|
|
2456
|
-
connection = this.createConnection(
|
|
2665
|
+
connection = this.createConnection(config);
|
|
2457
2666
|
connectionAttempts++;
|
|
2458
2667
|
try {
|
|
2459
2668
|
await connection.connect();
|
|
@@ -2947,7 +3156,7 @@ var ScenarioRunner = class {
|
|
|
2947
3156
|
required: minPass,
|
|
2948
3157
|
attempts,
|
|
2949
3158
|
tests: this.results.tests(attempts, minPass),
|
|
2950
|
-
agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
|
|
3159
|
+
agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
|
|
2951
3160
|
evaluatorUsage: this.tokens.evaluator(attempts)
|
|
2952
3161
|
});
|
|
2953
3162
|
}
|
|
@@ -3038,7 +3247,9 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3038
3247
|
const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
|
|
3039
3248
|
if (!this.verbose) return line;
|
|
3040
3249
|
if (result.attempts.length === 0) return `${line}\n`;
|
|
3041
|
-
const
|
|
3250
|
+
const websocket = spec.websocket;
|
|
3251
|
+
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
3252
|
+
const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
|
|
3042
3253
|
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
3043
3254
|
lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
|
|
3044
3255
|
lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
|
|
@@ -3251,11 +3462,16 @@ var JsonReporter = class JsonReporter {
|
|
|
3251
3462
|
};
|
|
3252
3463
|
}
|
|
3253
3464
|
scenario(result, config) {
|
|
3254
|
-
const
|
|
3465
|
+
const websocket = result.spec.websocket ?? config.websocket;
|
|
3466
|
+
const capability = websocket.protocol.capabilities.tokenUsage;
|
|
3255
3467
|
return {
|
|
3256
3468
|
name: result.spec.name,
|
|
3257
3469
|
file: result.spec.location.file,
|
|
3258
3470
|
status: result.status,
|
|
3471
|
+
connection: {
|
|
3472
|
+
endpoint: websocket.endpoint,
|
|
3473
|
+
subprotocols: websocket.connection.subprotocols.length
|
|
3474
|
+
},
|
|
3259
3475
|
...JsonReporter.counts(result),
|
|
3260
3476
|
agentUsage: JsonReporter.usage(result.agentUsage),
|
|
3261
3477
|
evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
|