@mappies/miniwat-eval 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -11
- package/dist/index.cjs +389 -173
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +58 -48
- package/dist/index.d.ts +58 -48
- package/dist/index.js +389 -173
- package/dist/index.js.map +1 -1
- package/dist/main.js +345 -129
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +9 -4
- package/schemas/eval-config.schema.json +5 -0
- package/schemas/report.schema.json +11 -1
package/dist/index.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { Message, ResponseMessage, TokenUsage } from "@mappies/miniwat-agent-core";
|
|
2
2
|
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
3
3
|
import { dirname, extname, join, relative, resolve } from "node:path";
|
|
4
|
-
import { Ajv } from "ajv";
|
|
5
4
|
import { LineCounter, isMap, isPair, isScalar, isSeq, parseDocument } from "yaml";
|
|
5
|
+
import { Ajv } from "ajv";
|
|
6
6
|
import handlerWs from "configurapi-handler-ws";
|
|
7
7
|
import { isDeepStrictEqual, styleText } from "node:util";
|
|
8
8
|
import { Agent, InMemoryAgentStateStore } from "@mappies/miniwat-agent";
|
|
@@ -273,6 +273,50 @@ var TurnSpec = class {
|
|
|
273
273
|
this.location = new SourceLocation(data?.location);
|
|
274
274
|
}
|
|
275
275
|
};
|
|
276
|
+
// src/entities/protocolCapabilities.ts
|
|
277
|
+
var ProtocolCapabilities = class {
|
|
278
|
+
toolUses;
|
|
279
|
+
toolResults;
|
|
280
|
+
tokenUsage;
|
|
281
|
+
constructor(data) {
|
|
282
|
+
this.toolUses = data?.toolUses ?? false;
|
|
283
|
+
this.toolResults = data?.toolResults ?? false;
|
|
284
|
+
this.tokenUsage = data?.tokenUsage ?? false;
|
|
285
|
+
}
|
|
286
|
+
};
|
|
287
|
+
// src/interfaces/requestFormat.ts
|
|
288
|
+
const RequestFormat = { Json: "json" };
|
|
289
|
+
// src/entities/requestConfig.ts
|
|
290
|
+
var RequestConfig = class {
|
|
291
|
+
format;
|
|
292
|
+
value;
|
|
293
|
+
inputPath;
|
|
294
|
+
constructor(data) {
|
|
295
|
+
this.format = data?.format ?? RequestFormat.Json;
|
|
296
|
+
this.value = data?.value;
|
|
297
|
+
this.inputPath = data?.inputPath;
|
|
298
|
+
}
|
|
299
|
+
};
|
|
300
|
+
// src/entities/protocolConfig.ts
|
|
301
|
+
var ProtocolConfig = class {
|
|
302
|
+
request;
|
|
303
|
+
capabilities;
|
|
304
|
+
constructor(data) {
|
|
305
|
+
this.request = new RequestConfig(data?.request);
|
|
306
|
+
this.capabilities = new ProtocolCapabilities(data?.capabilities);
|
|
307
|
+
}
|
|
308
|
+
};
|
|
309
|
+
// src/entities/webSocketConfig.ts
|
|
310
|
+
var WebSocketConfig = class {
|
|
311
|
+
endpoint;
|
|
312
|
+
connection;
|
|
313
|
+
protocol;
|
|
314
|
+
constructor(data) {
|
|
315
|
+
this.endpoint = data?.endpoint ?? "";
|
|
316
|
+
this.connection = new ConnectionConfig(data?.connection);
|
|
317
|
+
this.protocol = new ProtocolConfig(data?.protocol);
|
|
318
|
+
}
|
|
319
|
+
};
|
|
276
320
|
// src/entities/conversationSpec.ts
|
|
277
321
|
var ConversationSpec = class {
|
|
278
322
|
version;
|
|
@@ -281,6 +325,7 @@ var ConversationSpec = class {
|
|
|
281
325
|
turns;
|
|
282
326
|
conversation;
|
|
283
327
|
location;
|
|
328
|
+
websocket;
|
|
284
329
|
secrets;
|
|
285
330
|
constructor(data) {
|
|
286
331
|
this.version = data?.version ?? 1;
|
|
@@ -289,6 +334,7 @@ var ConversationSpec = class {
|
|
|
289
334
|
this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
|
|
290
335
|
this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
|
|
291
336
|
this.location = new SourceLocation(data?.location);
|
|
337
|
+
this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
|
|
292
338
|
this.secrets = [...data?.secrets ?? []];
|
|
293
339
|
}
|
|
294
340
|
};
|
|
@@ -390,55 +436,12 @@ var ReportConfig = class {
|
|
|
390
436
|
this.formats = data?.formats ?? [];
|
|
391
437
|
}
|
|
392
438
|
};
|
|
393
|
-
// src/entities/protocolCapabilities.ts
|
|
394
|
-
var ProtocolCapabilities = class {
|
|
395
|
-
toolUses;
|
|
396
|
-
toolResults;
|
|
397
|
-
tokenUsage;
|
|
398
|
-
constructor(data) {
|
|
399
|
-
this.toolUses = data?.toolUses ?? false;
|
|
400
|
-
this.toolResults = data?.toolResults ?? false;
|
|
401
|
-
this.tokenUsage = data?.tokenUsage ?? false;
|
|
402
|
-
}
|
|
403
|
-
};
|
|
404
|
-
// src/interfaces/requestFormat.ts
|
|
405
|
-
const RequestFormat = { Json: "json" };
|
|
406
|
-
// src/entities/requestConfig.ts
|
|
407
|
-
var RequestConfig = class {
|
|
408
|
-
format;
|
|
409
|
-
value;
|
|
410
|
-
inputPath;
|
|
411
|
-
constructor(data) {
|
|
412
|
-
this.format = data?.format ?? RequestFormat.Json;
|
|
413
|
-
this.value = data?.value;
|
|
414
|
-
this.inputPath = data?.inputPath;
|
|
415
|
-
}
|
|
416
|
-
};
|
|
417
|
-
// src/entities/protocolConfig.ts
|
|
418
|
-
var ProtocolConfig = class {
|
|
419
|
-
request;
|
|
420
|
-
capabilities;
|
|
421
|
-
constructor(data) {
|
|
422
|
-
this.request = new RequestConfig(data?.request);
|
|
423
|
-
this.capabilities = new ProtocolCapabilities(data?.capabilities);
|
|
424
|
-
}
|
|
425
|
-
};
|
|
426
|
-
// src/entities/webSocketConfig.ts
|
|
427
|
-
var WebSocketConfig = class {
|
|
428
|
-
endpoint;
|
|
429
|
-
connection;
|
|
430
|
-
protocol;
|
|
431
|
-
constructor(data) {
|
|
432
|
-
this.endpoint = data?.endpoint ?? "";
|
|
433
|
-
this.connection = new ConnectionConfig(data?.connection);
|
|
434
|
-
this.protocol = new ProtocolConfig(data?.protocol);
|
|
435
|
-
}
|
|
436
|
-
};
|
|
437
439
|
// src/entities/evalConfig.ts
|
|
438
440
|
var EvalConfig = class {
|
|
439
441
|
version;
|
|
440
442
|
configPath;
|
|
441
443
|
parametersPath;
|
|
444
|
+
variables;
|
|
442
445
|
websocket;
|
|
443
446
|
execution;
|
|
444
447
|
evaluator;
|
|
@@ -448,6 +451,7 @@ var EvalConfig = class {
|
|
|
448
451
|
this.version = data?.version ?? 1;
|
|
449
452
|
this.configPath = data?.configPath ?? "";
|
|
450
453
|
this.parametersPath = data?.parametersPath ?? "";
|
|
454
|
+
this.variables = { ...data?.variables ?? {} };
|
|
451
455
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
452
456
|
this.execution = new ExecutionConfig(data?.execution);
|
|
453
457
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -759,6 +763,11 @@ var eval_config_schema_default = {
|
|
|
759
763
|
],
|
|
760
764
|
properties: {
|
|
761
765
|
"version": { "const": 1 },
|
|
766
|
+
"variables": {
|
|
767
|
+
"type": "object",
|
|
768
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
769
|
+
"additionalProperties": { "type": "string" }
|
|
770
|
+
},
|
|
762
771
|
"websocket": {
|
|
763
772
|
"type": "object",
|
|
764
773
|
"additionalProperties": false,
|
|
@@ -917,18 +926,80 @@ var eval_config_schema_default = {
|
|
|
917
926
|
}
|
|
918
927
|
}
|
|
919
928
|
};
|
|
929
|
+
// src/yaml/yamlDocument.ts
|
|
930
|
+
var YamlDocument = class YamlDocument {
|
|
931
|
+
file;
|
|
932
|
+
document;
|
|
933
|
+
lineCounter;
|
|
934
|
+
value;
|
|
935
|
+
errors;
|
|
936
|
+
constructor(file, document, lineCounter) {
|
|
937
|
+
this.file = file;
|
|
938
|
+
this.document = document;
|
|
939
|
+
this.lineCounter = lineCounter;
|
|
940
|
+
this.value = document.toJS();
|
|
941
|
+
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
942
|
+
message: error.message.split("\n")[0],
|
|
943
|
+
location: this.toLocation(error.pos[0], [])
|
|
944
|
+
}));
|
|
945
|
+
}
|
|
946
|
+
locate(path) {
|
|
947
|
+
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
948
|
+
}
|
|
949
|
+
locateKey(path, key) {
|
|
950
|
+
const parent = this.findNode(path);
|
|
951
|
+
if (isMap(parent)) {
|
|
952
|
+
const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
|
|
953
|
+
if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
954
|
+
}
|
|
955
|
+
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
956
|
+
}
|
|
957
|
+
static formatPath(path) {
|
|
958
|
+
return path.map((segment, index) => {
|
|
959
|
+
if (typeof segment === "number") return `[${segment}]`;
|
|
960
|
+
return index === 0 ? segment : `.${segment}`;
|
|
961
|
+
}).join("");
|
|
962
|
+
}
|
|
963
|
+
findNode(path) {
|
|
964
|
+
let node = this.document.contents ?? void 0;
|
|
965
|
+
for (const segment of path) {
|
|
966
|
+
let next;
|
|
967
|
+
if (isMap(node)) {
|
|
968
|
+
const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
|
|
969
|
+
next = pair?.value ?? pair?.key;
|
|
970
|
+
} else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
|
|
971
|
+
if (!next) break;
|
|
972
|
+
node = next;
|
|
973
|
+
}
|
|
974
|
+
return node;
|
|
975
|
+
}
|
|
976
|
+
toLocation(offset, path) {
|
|
977
|
+
const position = this.lineCounter.linePos(offset);
|
|
978
|
+
return new SourceLocation({
|
|
979
|
+
file: this.file,
|
|
980
|
+
line: position.line,
|
|
981
|
+
column: position.col,
|
|
982
|
+
path: YamlDocument.formatPath(path)
|
|
983
|
+
});
|
|
984
|
+
}
|
|
985
|
+
};
|
|
920
986
|
// src/yaml/environmentResolver.ts
|
|
921
987
|
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
|
|
922
988
|
const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
|
|
923
989
|
var EnvironmentResolver = class EnvironmentResolver {
|
|
924
990
|
environment;
|
|
925
991
|
parameters;
|
|
926
|
-
|
|
992
|
+
levels;
|
|
993
|
+
constructor(environment = process.env, parameters = {}, levels = []) {
|
|
927
994
|
this.environment = environment;
|
|
928
995
|
this.parameters = parameters;
|
|
996
|
+
this.levels = levels;
|
|
997
|
+
}
|
|
998
|
+
withLevels(levels) {
|
|
999
|
+
return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
|
|
929
1000
|
}
|
|
930
|
-
resolve(value, document, issues, secrets = []) {
|
|
931
|
-
return this.resolveValue(value, [], document, issues, secrets);
|
|
1001
|
+
resolve(value, document, issues, secrets = [], options = {}) {
|
|
1002
|
+
return this.resolveValue(value, [], document, issues, secrets, options);
|
|
932
1003
|
}
|
|
933
1004
|
static hasReference(value) {
|
|
934
1005
|
return new RegExp(REFERENCE.source).test(value);
|
|
@@ -936,22 +1007,38 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
936
1007
|
static isSecretName(name) {
|
|
937
1008
|
return SECRET_NAME.test(name);
|
|
938
1009
|
}
|
|
939
|
-
|
|
1010
|
+
lookup(name) {
|
|
1011
|
+
for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
|
|
1012
|
+
return this.environment[name];
|
|
1013
|
+
}
|
|
1014
|
+
sources() {
|
|
1015
|
+
const sources = [
|
|
1016
|
+
...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
|
|
1017
|
+
...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
|
|
1018
|
+
"environment variable"
|
|
1019
|
+
];
|
|
1020
|
+
return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
|
|
1021
|
+
}
|
|
1022
|
+
resolveValue(value, path, document, issues, secrets, options) {
|
|
940
1023
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
941
|
-
|
|
1024
|
+
options.used?.add(name);
|
|
1025
|
+
const resolved = this.lookup(name);
|
|
942
1026
|
if (resolved === void 0) {
|
|
943
|
-
|
|
1027
|
+
if (options.optional?.(path)) {
|
|
1028
|
+
options.unresolved?.add(YamlDocument.formatPath(path));
|
|
1029
|
+
return reference;
|
|
1030
|
+
}
|
|
944
1031
|
issues.push(new ValidationIssue({
|
|
945
|
-
message: `${
|
|
1032
|
+
message: `${this.sources()} '${name}' is not set`,
|
|
946
1033
|
location: document.locate(path)
|
|
947
1034
|
}));
|
|
948
1035
|
return reference;
|
|
949
1036
|
}
|
|
950
|
-
if (EnvironmentResolver.isSecretName(name) && !secrets.includes(resolved)) secrets.push(resolved);
|
|
1037
|
+
if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
|
|
951
1038
|
return resolved;
|
|
952
1039
|
});
|
|
953
|
-
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
|
|
954
|
-
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
|
|
1040
|
+
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
|
|
1041
|
+
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
|
|
955
1042
|
return value;
|
|
956
1043
|
}
|
|
957
1044
|
};
|
|
@@ -1035,63 +1122,6 @@ var SchemaValidator = class SchemaValidator {
|
|
|
1035
1122
|
return path;
|
|
1036
1123
|
}
|
|
1037
1124
|
};
|
|
1038
|
-
// src/yaml/yamlDocument.ts
|
|
1039
|
-
var YamlDocument = class YamlDocument {
|
|
1040
|
-
file;
|
|
1041
|
-
document;
|
|
1042
|
-
lineCounter;
|
|
1043
|
-
value;
|
|
1044
|
-
errors;
|
|
1045
|
-
constructor(file, document, lineCounter) {
|
|
1046
|
-
this.file = file;
|
|
1047
|
-
this.document = document;
|
|
1048
|
-
this.lineCounter = lineCounter;
|
|
1049
|
-
this.value = document.toJS();
|
|
1050
|
-
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
1051
|
-
message: error.message.split("\n")[0],
|
|
1052
|
-
location: this.toLocation(error.pos[0], [])
|
|
1053
|
-
}));
|
|
1054
|
-
}
|
|
1055
|
-
locate(path) {
|
|
1056
|
-
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
1057
|
-
}
|
|
1058
|
-
locateKey(path, key) {
|
|
1059
|
-
const parent = this.findNode(path);
|
|
1060
|
-
if (isMap(parent)) {
|
|
1061
|
-
const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
|
|
1062
|
-
if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
1063
|
-
}
|
|
1064
|
-
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
1065
|
-
}
|
|
1066
|
-
static formatPath(path) {
|
|
1067
|
-
return path.map((segment, index) => {
|
|
1068
|
-
if (typeof segment === "number") return `[${segment}]`;
|
|
1069
|
-
return index === 0 ? segment : `.${segment}`;
|
|
1070
|
-
}).join("");
|
|
1071
|
-
}
|
|
1072
|
-
findNode(path) {
|
|
1073
|
-
let node = this.document.contents ?? void 0;
|
|
1074
|
-
for (const segment of path) {
|
|
1075
|
-
let next;
|
|
1076
|
-
if (isMap(node)) {
|
|
1077
|
-
const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
|
|
1078
|
-
next = pair?.value ?? pair?.key;
|
|
1079
|
-
} else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
|
|
1080
|
-
if (!next) break;
|
|
1081
|
-
node = next;
|
|
1082
|
-
}
|
|
1083
|
-
return node;
|
|
1084
|
-
}
|
|
1085
|
-
toLocation(offset, path) {
|
|
1086
|
-
const position = this.lineCounter.linePos(offset);
|
|
1087
|
-
return new SourceLocation({
|
|
1088
|
-
file: this.file,
|
|
1089
|
-
line: position.line,
|
|
1090
|
-
column: position.col,
|
|
1091
|
-
path: YamlDocument.formatPath(path)
|
|
1092
|
-
});
|
|
1093
|
-
}
|
|
1094
|
-
};
|
|
1095
1125
|
// src/yaml/yamlDocumentLoader.ts
|
|
1096
1126
|
var YamlDocumentLoader = class {
|
|
1097
1127
|
load(filePath, displayPath) {
|
|
@@ -1110,11 +1140,133 @@ var YamlDocumentLoader = class {
|
|
|
1110
1140
|
function isRecord(value) {
|
|
1111
1141
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
1112
1142
|
}
|
|
1143
|
+
// src/yaml/variablesReader.ts
|
|
1144
|
+
const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
1145
|
+
var VariablesReader = class {
|
|
1146
|
+
read(document, resolver, issues, secrets) {
|
|
1147
|
+
const raw = isRecord(document.value) ? document.value.variables : void 0;
|
|
1148
|
+
if (raw === void 0) return {};
|
|
1149
|
+
if (!isRecord(raw)) {
|
|
1150
|
+
issues.push(new ValidationIssue({
|
|
1151
|
+
message: "must be object",
|
|
1152
|
+
location: document.locate(["variables"])
|
|
1153
|
+
}));
|
|
1154
|
+
return {};
|
|
1155
|
+
}
|
|
1156
|
+
const variables = {};
|
|
1157
|
+
for (const [name, value] of Object.entries(raw)) {
|
|
1158
|
+
if (!VARIABLE_NAME.test(name)) {
|
|
1159
|
+
issues.push(new ValidationIssue({
|
|
1160
|
+
message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1161
|
+
location: document.locateKey(["variables"], name)
|
|
1162
|
+
}));
|
|
1163
|
+
continue;
|
|
1164
|
+
}
|
|
1165
|
+
if (typeof value !== "string") {
|
|
1166
|
+
issues.push(new ValidationIssue({
|
|
1167
|
+
message: "must be string",
|
|
1168
|
+
location: document.locate(["variables", name])
|
|
1169
|
+
}));
|
|
1170
|
+
continue;
|
|
1171
|
+
}
|
|
1172
|
+
const before = issues.length;
|
|
1173
|
+
const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
|
|
1174
|
+
if (issues.length > before) continue;
|
|
1175
|
+
variables[name] = resolved.variables[name];
|
|
1176
|
+
if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
|
|
1177
|
+
}
|
|
1178
|
+
return variables;
|
|
1179
|
+
}
|
|
1180
|
+
};
|
|
1181
|
+
// src/config/webSocketResolver.ts
|
|
1182
|
+
const WEBSOCKET_SCHEMA = {
|
|
1183
|
+
...eval_config_schema_default,
|
|
1184
|
+
required: ["websocket"],
|
|
1185
|
+
additionalProperties: true,
|
|
1186
|
+
properties: { websocket: eval_config_schema_default.properties.websocket }
|
|
1187
|
+
};
|
|
1188
|
+
var WebSocketResolver = class WebSocketResolver {
|
|
1189
|
+
schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
|
|
1190
|
+
resolve(document, resolver, issues, secrets, used, defer) {
|
|
1191
|
+
const raw = isRecord(document.value) ? document.value.websocket : void 0;
|
|
1192
|
+
const unresolved = /* @__PURE__ */ new Set();
|
|
1193
|
+
const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
|
|
1194
|
+
const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
|
|
1195
|
+
used,
|
|
1196
|
+
optional,
|
|
1197
|
+
unresolved
|
|
1198
|
+
});
|
|
1199
|
+
const subprotocols = WebSocketResolver.subprotocols(raw);
|
|
1200
|
+
if (subprotocols !== void 0) {
|
|
1201
|
+
const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
|
|
1202
|
+
const set = !unresolved.has(YamlDocument.formatPath([
|
|
1203
|
+
"websocket",
|
|
1204
|
+
"connection",
|
|
1205
|
+
"subprotocols",
|
|
1206
|
+
index
|
|
1207
|
+
]));
|
|
1208
|
+
const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
|
|
1209
|
+
return set && !empty;
|
|
1210
|
+
});
|
|
1211
|
+
value.websocket.connection.subprotocols = kept;
|
|
1212
|
+
}
|
|
1213
|
+
const prefix = YamlDocument.formatPath([
|
|
1214
|
+
"websocket",
|
|
1215
|
+
"connection",
|
|
1216
|
+
"subprotocols"
|
|
1217
|
+
]);
|
|
1218
|
+
return {
|
|
1219
|
+
value,
|
|
1220
|
+
deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
|
|
1221
|
+
};
|
|
1222
|
+
}
|
|
1223
|
+
validate(value, document) {
|
|
1224
|
+
const issues = this.schemaValidator.validate(value, document);
|
|
1225
|
+
if (issues.length > 0) return issues;
|
|
1226
|
+
return WebSocketResolver.validateInputPath(value, document);
|
|
1227
|
+
}
|
|
1228
|
+
static validateInputPath(value, document) {
|
|
1229
|
+
const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
|
|
1230
|
+
if (typeof request.inputPath !== "string") return [];
|
|
1231
|
+
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
1232
|
+
const last = tokens.pop();
|
|
1233
|
+
let parent = request.value;
|
|
1234
|
+
for (const token of tokens) parent = WebSocketResolver.child(parent, token);
|
|
1235
|
+
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
1236
|
+
return [new ValidationIssue({
|
|
1237
|
+
message: "must point to a missing or string value whose parent exists in request.value",
|
|
1238
|
+
location: document.locate([
|
|
1239
|
+
"websocket",
|
|
1240
|
+
"protocol",
|
|
1241
|
+
"request",
|
|
1242
|
+
"inputPath"
|
|
1243
|
+
])
|
|
1244
|
+
})];
|
|
1245
|
+
}
|
|
1246
|
+
static section(value, ...path) {
|
|
1247
|
+
let current = value;
|
|
1248
|
+
for (const key of path) current = WebSocketResolver.child(current, key);
|
|
1249
|
+
return isRecord(current) ? current : {};
|
|
1250
|
+
}
|
|
1251
|
+
static child(value, key) {
|
|
1252
|
+
if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
|
|
1253
|
+
return isRecord(value) ? value[key] : void 0;
|
|
1254
|
+
}
|
|
1255
|
+
static isSubprotocol(path) {
|
|
1256
|
+
return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
|
|
1257
|
+
}
|
|
1258
|
+
static subprotocols(websocket) {
|
|
1259
|
+
const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
|
|
1260
|
+
return Array.isArray(subprotocols) ? subprotocols : void 0;
|
|
1261
|
+
}
|
|
1262
|
+
};
|
|
1113
1263
|
// src/config/evalConfigLoader.ts
|
|
1114
1264
|
var EvalConfigLoader = class EvalConfigLoader {
|
|
1115
1265
|
cwd;
|
|
1116
1266
|
documentLoader = new YamlDocumentLoader();
|
|
1117
1267
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
1268
|
+
variablesReader = new VariablesReader();
|
|
1269
|
+
webSocketResolver = new WebSocketResolver();
|
|
1118
1270
|
environmentResolver;
|
|
1119
1271
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1120
1272
|
this.cwd = cwd;
|
|
@@ -1125,19 +1277,47 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1125
1277
|
const issues = [...document.errors];
|
|
1126
1278
|
if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
|
|
1127
1279
|
const named = [];
|
|
1128
|
-
const
|
|
1129
|
-
|
|
1130
|
-
if (issues.length === 0)
|
|
1280
|
+
const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
|
|
1281
|
+
let value = {};
|
|
1282
|
+
if (issues.length === 0) {
|
|
1283
|
+
const resolver = this.environmentResolver.withLevels([variables]);
|
|
1284
|
+
const { websocket: _websocket, variables: _variables, ...rest } = document.value;
|
|
1285
|
+
const resolved = resolver.resolve(rest, document, issues, named);
|
|
1286
|
+
const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
|
|
1287
|
+
value = {
|
|
1288
|
+
...resolved,
|
|
1289
|
+
..._variables === void 0 ? {} : { variables },
|
|
1290
|
+
..._websocket === void 0 ? {} : websocket.value
|
|
1291
|
+
};
|
|
1292
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
|
|
1293
|
+
if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
|
|
1294
|
+
}
|
|
1131
1295
|
if (issues.length > 0) throw new EvalError({
|
|
1132
1296
|
category: ErrorCategory.Configuration,
|
|
1133
1297
|
message: "invalid configuration",
|
|
1134
1298
|
issues
|
|
1135
1299
|
});
|
|
1136
|
-
const config = this.build(value, configPath, named);
|
|
1300
|
+
const config = this.build(value, configPath, variables, named);
|
|
1137
1301
|
this.applyRepetition(config, override, value, document);
|
|
1138
1302
|
EvalConfigLoader.applyReport(config, report);
|
|
1139
1303
|
return config;
|
|
1140
1304
|
}
|
|
1305
|
+
static buildWebSocket(value) {
|
|
1306
|
+
const section = (...path) => WebSocketResolver.section(value, ...path);
|
|
1307
|
+
const connection = section("websocket", "connection");
|
|
1308
|
+
const request = section("websocket", "protocol", "request");
|
|
1309
|
+
return new WebSocketConfig({
|
|
1310
|
+
endpoint: WebSocketResolver.section(value, "websocket").endpoint,
|
|
1311
|
+
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
1312
|
+
protocol: new ProtocolConfig({
|
|
1313
|
+
request: new RequestConfig({
|
|
1314
|
+
...request,
|
|
1315
|
+
format: request.format ?? RequestFormat.Json
|
|
1316
|
+
}),
|
|
1317
|
+
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
1318
|
+
})
|
|
1319
|
+
});
|
|
1320
|
+
}
|
|
1141
1321
|
static applyReport(config, report) {
|
|
1142
1322
|
if (report.formats === void 0) return;
|
|
1143
1323
|
const known = Object.values(ReportFormat);
|
|
@@ -1159,45 +1339,17 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1159
1339
|
location: document.locate(["evaluator", "apiKey"])
|
|
1160
1340
|
})];
|
|
1161
1341
|
}
|
|
1162
|
-
|
|
1163
|
-
const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
|
|
1164
|
-
if (typeof request.inputPath !== "string") return [];
|
|
1165
|
-
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
1166
|
-
const last = tokens.pop();
|
|
1167
|
-
let parent = request.value;
|
|
1168
|
-
for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
|
|
1169
|
-
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
1170
|
-
return [new ValidationIssue({
|
|
1171
|
-
message: "must point to a missing or string value whose parent exists in request.value",
|
|
1172
|
-
location: document.locate([
|
|
1173
|
-
"websocket",
|
|
1174
|
-
"protocol",
|
|
1175
|
-
"request",
|
|
1176
|
-
"inputPath"
|
|
1177
|
-
])
|
|
1178
|
-
})];
|
|
1179
|
-
}
|
|
1180
|
-
build(value, configPath, named) {
|
|
1342
|
+
build(value, configPath, variables, named) {
|
|
1181
1343
|
const section = (...path) => EvalConfigLoader.section(value, ...path);
|
|
1182
1344
|
const configDirectory = dirname(configPath);
|
|
1183
|
-
const
|
|
1184
|
-
const request = section("websocket", "protocol", "request");
|
|
1345
|
+
const websocket = EvalConfigLoader.buildWebSocket(value);
|
|
1185
1346
|
const evaluator = section("evaluator");
|
|
1186
1347
|
const report = section("report");
|
|
1187
1348
|
return new EvalConfig({
|
|
1188
1349
|
version: value.version,
|
|
1189
1350
|
configPath,
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
1193
|
-
protocol: new ProtocolConfig({
|
|
1194
|
-
request: new RequestConfig({
|
|
1195
|
-
...request,
|
|
1196
|
-
format: request.format ?? RequestFormat.Json
|
|
1197
|
-
}),
|
|
1198
|
-
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
1199
|
-
})
|
|
1200
|
-
}),
|
|
1351
|
+
variables,
|
|
1352
|
+
websocket,
|
|
1201
1353
|
execution: new ExecutionConfig({
|
|
1202
1354
|
...DEFAULT_CONFIG.execution,
|
|
1203
1355
|
...section("execution"),
|
|
@@ -1218,7 +1370,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1218
1370
|
directory: resolve(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
|
|
1219
1371
|
formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
|
|
1220
1372
|
}),
|
|
1221
|
-
secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols
|
|
1373
|
+
secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
|
|
1222
1374
|
});
|
|
1223
1375
|
}
|
|
1224
1376
|
applyRepetition(config, override, value, document) {
|
|
@@ -1271,13 +1423,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1271
1423
|
])];
|
|
1272
1424
|
}
|
|
1273
1425
|
static section(value, ...path) {
|
|
1274
|
-
|
|
1275
|
-
for (const key of path) current = EvalConfigLoader.child(current, key);
|
|
1276
|
-
return isRecord(current) ? current : {};
|
|
1277
|
-
}
|
|
1278
|
-
static child(value, key) {
|
|
1279
|
-
if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
|
|
1280
|
-
return isRecord(value) ? value[key] : void 0;
|
|
1426
|
+
return WebSocketResolver.section(value, ...path);
|
|
1281
1427
|
}
|
|
1282
1428
|
};
|
|
1283
1429
|
// src/config/parametersLoader.ts
|
|
@@ -1329,6 +1475,11 @@ var conversation_schema_default = {
|
|
|
1329
1475
|
"version": { "const": 1 },
|
|
1330
1476
|
"name": { "$ref": "#/definitions/text" },
|
|
1331
1477
|
"description": { "type": "string" },
|
|
1478
|
+
"variables": {
|
|
1479
|
+
"type": "object",
|
|
1480
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
1481
|
+
"additionalProperties": { "type": "string" }
|
|
1482
|
+
},
|
|
1332
1483
|
"turns": {
|
|
1333
1484
|
"type": "array",
|
|
1334
1485
|
"minItems": 1,
|
|
@@ -1341,8 +1492,8 @@ var conversation_schema_default = {
|
|
|
1341
1492
|
"properties": { "conversation": {
|
|
1342
1493
|
"type": "object",
|
|
1343
1494
|
"additionalProperties": false,
|
|
1344
|
-
"required": ["
|
|
1345
|
-
"properties": { "
|
|
1495
|
+
"required": ["should"],
|
|
1496
|
+
"properties": { "should": { "$ref": "#/definitions/text" } }
|
|
1346
1497
|
} }
|
|
1347
1498
|
}
|
|
1348
1499
|
},
|
|
@@ -1385,12 +1536,12 @@ var conversation_schema_default = {
|
|
|
1385
1536
|
}
|
|
1386
1537
|
},
|
|
1387
1538
|
"response": {
|
|
1388
|
-
"description": "A string is shorthand for
|
|
1539
|
+
"description": "A string is shorthand for should.",
|
|
1389
1540
|
"type": ["string", "object"],
|
|
1390
1541
|
"pattern": "\\S",
|
|
1391
1542
|
"additionalProperties": false,
|
|
1392
1543
|
"properties": {
|
|
1393
|
-
"
|
|
1544
|
+
"should": { "$ref": "#/definitions/text" },
|
|
1394
1545
|
"contains": { "$ref": "#/definitions/textList" },
|
|
1395
1546
|
"notContains": { "$ref": "#/definitions/textList" }
|
|
1396
1547
|
}
|
|
@@ -1422,7 +1573,7 @@ var conversation_schema_default = {
|
|
|
1422
1573
|
};
|
|
1423
1574
|
// src/scenarios/conversationValidator.ts
|
|
1424
1575
|
const RESPONSE_KEYS = [
|
|
1425
|
-
"
|
|
1576
|
+
"should",
|
|
1426
1577
|
"contains",
|
|
1427
1578
|
"notContains"
|
|
1428
1579
|
];
|
|
@@ -1445,7 +1596,7 @@ var ConversationValidator = class ConversationValidator {
|
|
|
1445
1596
|
validateResponse(response, path, document, issues) {
|
|
1446
1597
|
if (!RESPONSE_KEYS.some((key) => key in response)) {
|
|
1447
1598
|
issues.push(new ValidationIssue({
|
|
1448
|
-
message: "response must have at least one of:
|
|
1599
|
+
message: "response must have at least one of: should, contains, notContains",
|
|
1449
1600
|
location: document.locate(path)
|
|
1450
1601
|
}));
|
|
1451
1602
|
return;
|
|
@@ -1488,11 +1639,13 @@ var ConversationValidator = class ConversationValidator {
|
|
|
1488
1639
|
};
|
|
1489
1640
|
// src/scenarios/scenarioLoader.ts
|
|
1490
1641
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
1491
|
-
var ScenarioLoader = class {
|
|
1642
|
+
var ScenarioLoader = class ScenarioLoader {
|
|
1492
1643
|
cwd;
|
|
1493
1644
|
documentLoader = new YamlDocumentLoader();
|
|
1494
1645
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
1495
1646
|
conversationValidator = new ConversationValidator();
|
|
1647
|
+
variablesReader = new VariablesReader();
|
|
1648
|
+
webSocketResolver = new WebSocketResolver();
|
|
1496
1649
|
environmentResolver;
|
|
1497
1650
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1498
1651
|
this.cwd = cwd;
|
|
@@ -1500,18 +1653,52 @@ var ScenarioLoader = class {
|
|
|
1500
1653
|
}
|
|
1501
1654
|
load(target, config) {
|
|
1502
1655
|
const files = this.discover(resolve(this.cwd, target), config);
|
|
1656
|
+
const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.cwd, config.configPath)) : void 0;
|
|
1503
1657
|
const issues = [];
|
|
1658
|
+
const connectionIssues = [];
|
|
1504
1659
|
const specs = [];
|
|
1505
1660
|
for (const file of files) {
|
|
1506
1661
|
const document = this.documentLoader.load(file, relative(this.cwd, file));
|
|
1507
1662
|
const fileIssues = [...document.errors];
|
|
1508
1663
|
const secrets = [];
|
|
1509
|
-
const
|
|
1664
|
+
const used = /* @__PURE__ */ new Set();
|
|
1665
|
+
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
|
|
1666
|
+
const resolver = this.environmentResolver.withLevels([variables, config.variables]);
|
|
1667
|
+
let value;
|
|
1668
|
+
if (fileIssues.length === 0) {
|
|
1669
|
+
const { variables: raw, ...rest } = document.value;
|
|
1670
|
+
value = {
|
|
1671
|
+
...resolver.resolve(rest, document, fileIssues, secrets, { used }),
|
|
1672
|
+
...raw === void 0 ? {} : { variables }
|
|
1673
|
+
};
|
|
1674
|
+
}
|
|
1675
|
+
let websocket;
|
|
1676
|
+
const websocketIssues = [];
|
|
1677
|
+
if (fileIssues.length === 0 && configDocument !== void 0) {
|
|
1678
|
+
const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
|
|
1679
|
+
if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
|
|
1680
|
+
connectionIssues.push({
|
|
1681
|
+
file: document,
|
|
1682
|
+
issues: websocketIssues
|
|
1683
|
+
});
|
|
1684
|
+
if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
|
|
1685
|
+
}
|
|
1510
1686
|
if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
|
|
1511
1687
|
if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
|
|
1512
|
-
if (fileIssues.length === 0)
|
|
1688
|
+
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
|
|
1689
|
+
if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
1513
1690
|
else issues.push(...fileIssues);
|
|
1514
1691
|
}
|
|
1692
|
+
const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
|
|
1693
|
+
if (shared.length > 0) throw new EvalError({
|
|
1694
|
+
category: ErrorCategory.Configuration,
|
|
1695
|
+
message: "invalid configuration",
|
|
1696
|
+
issues: shared
|
|
1697
|
+
});
|
|
1698
|
+
for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
|
|
1699
|
+
message: issue.toString(),
|
|
1700
|
+
location: ScenarioLoader.variablesLocation(entry.file)
|
|
1701
|
+
}));
|
|
1515
1702
|
if (issues.length > 0) throw new EvalError({
|
|
1516
1703
|
category: ErrorCategory.Specification,
|
|
1517
1704
|
message: "invalid scenario specification",
|
|
@@ -1519,6 +1706,23 @@ var ScenarioLoader = class {
|
|
|
1519
1706
|
});
|
|
1520
1707
|
return specs;
|
|
1521
1708
|
}
|
|
1709
|
+
static shared(lists) {
|
|
1710
|
+
if (lists.length === 0) return [];
|
|
1711
|
+
const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
|
|
1712
|
+
return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
|
|
1713
|
+
}
|
|
1714
|
+
static variablesLocation(document) {
|
|
1715
|
+
return new SourceLocation({
|
|
1716
|
+
...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
|
|
1717
|
+
path: ""
|
|
1718
|
+
});
|
|
1719
|
+
}
|
|
1720
|
+
static unusedVariables(variables, used, document) {
|
|
1721
|
+
return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
|
|
1722
|
+
message: `variable '${name}' is not used by the scenario or the websocket configuration`,
|
|
1723
|
+
location: document.locateKey(["variables"], name)
|
|
1724
|
+
}));
|
|
1725
|
+
}
|
|
1522
1726
|
discover(target, config) {
|
|
1523
1727
|
if (!existsSync(target)) throw new EvalError({
|
|
1524
1728
|
category: ErrorCategory.Specification,
|
|
@@ -1543,7 +1747,7 @@ var ScenarioLoader = class {
|
|
|
1543
1747
|
}
|
|
1544
1748
|
return files;
|
|
1545
1749
|
}
|
|
1546
|
-
build(value, document, secrets) {
|
|
1750
|
+
build(value, document, websocket, secrets) {
|
|
1547
1751
|
const scenario = value;
|
|
1548
1752
|
const turns = scenario.turns;
|
|
1549
1753
|
const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
|
|
@@ -1553,10 +1757,11 @@ var ScenarioLoader = class {
|
|
|
1553
1757
|
description: typeof scenario.description === "string" ? scenario.description : "",
|
|
1554
1758
|
turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
|
|
1555
1759
|
conversation: conversation ? new ConversationExpectation({
|
|
1556
|
-
criteria: conversation.
|
|
1760
|
+
criteria: conversation.should,
|
|
1557
1761
|
location: document.locate(["expect", "conversation"])
|
|
1558
1762
|
}) : void 0,
|
|
1559
1763
|
location: document.locate([]),
|
|
1764
|
+
websocket,
|
|
1560
1765
|
secrets
|
|
1561
1766
|
});
|
|
1562
1767
|
}
|
|
@@ -1582,7 +1787,7 @@ var ScenarioLoader = class {
|
|
|
1582
1787
|
});
|
|
1583
1788
|
const fields = response;
|
|
1584
1789
|
return new ResponseExpectation({
|
|
1585
|
-
criteria: fields.
|
|
1790
|
+
criteria: fields.should,
|
|
1586
1791
|
contains: fields.contains ?? [],
|
|
1587
1792
|
notContains: fields.notContains ?? [],
|
|
1588
1793
|
location
|
|
@@ -2405,7 +2610,11 @@ var ConversationRunner = class {
|
|
|
2405
2610
|
async run(spec, signal = NEVER_ABORTED) {
|
|
2406
2611
|
const startedAt = Date.now();
|
|
2407
2612
|
const outcomes = [];
|
|
2408
|
-
const
|
|
2613
|
+
const config = new EvalConfig({
|
|
2614
|
+
...this.config,
|
|
2615
|
+
websocket: spec.websocket ?? this.config.websocket
|
|
2616
|
+
});
|
|
2617
|
+
const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
|
|
2409
2618
|
const history = [];
|
|
2410
2619
|
const evaluate = this.semanticEvaluator(signal);
|
|
2411
2620
|
const inline = !this.config.execution.continueOnFailure;
|
|
@@ -2430,7 +2639,7 @@ var ConversationRunner = class {
|
|
|
2430
2639
|
try {
|
|
2431
2640
|
const steps = (async () => {
|
|
2432
2641
|
for (let attempt = 0;; attempt++) {
|
|
2433
|
-
connection = this.createConnection(
|
|
2642
|
+
connection = this.createConnection(config);
|
|
2434
2643
|
connectionAttempts++;
|
|
2435
2644
|
try {
|
|
2436
2645
|
await connection.connect();
|
|
@@ -2924,7 +3133,7 @@ var ScenarioRunner = class {
|
|
|
2924
3133
|
required: minPass,
|
|
2925
3134
|
attempts,
|
|
2926
3135
|
tests: this.results.tests(attempts, minPass),
|
|
2927
|
-
agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
|
|
3136
|
+
agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
|
|
2928
3137
|
evaluatorUsage: this.tokens.evaluator(attempts)
|
|
2929
3138
|
});
|
|
2930
3139
|
}
|
|
@@ -3015,7 +3224,9 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3015
3224
|
const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
|
|
3016
3225
|
if (!this.verbose) return line;
|
|
3017
3226
|
if (result.attempts.length === 0) return `${line}\n`;
|
|
3018
|
-
const
|
|
3227
|
+
const websocket = spec.websocket;
|
|
3228
|
+
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
3229
|
+
const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
|
|
3019
3230
|
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
3020
3231
|
lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
|
|
3021
3232
|
lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
|
|
@@ -3228,11 +3439,16 @@ var JsonReporter = class JsonReporter {
|
|
|
3228
3439
|
};
|
|
3229
3440
|
}
|
|
3230
3441
|
scenario(result, config) {
|
|
3231
|
-
const
|
|
3442
|
+
const websocket = result.spec.websocket ?? config.websocket;
|
|
3443
|
+
const capability = websocket.protocol.capabilities.tokenUsage;
|
|
3232
3444
|
return {
|
|
3233
3445
|
name: result.spec.name,
|
|
3234
3446
|
file: result.spec.location.file,
|
|
3235
3447
|
status: result.status,
|
|
3448
|
+
connection: {
|
|
3449
|
+
endpoint: websocket.endpoint,
|
|
3450
|
+
subprotocols: websocket.connection.subprotocols.length
|
|
3451
|
+
},
|
|
3236
3452
|
...JsonReporter.counts(result),
|
|
3237
3453
|
agentUsage: JsonReporter.usage(result.agentUsage),
|
|
3238
3454
|
evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
|