@mappies/miniwat-eval 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -11
- package/dist/index.cjs +743 -178
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +65 -48
- package/dist/index.d.ts +66 -48
- package/dist/index.js +743 -178
- package/dist/index.js.map +1 -1
- package/dist/main.js +699 -134
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +43 -4
- package/schemas/eval-config.schema.json +10 -0
- package/schemas/report.schema.json +12 -1
package/dist/index.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { Message, ResponseMessage, TokenUsage } from "@mappies/miniwat-agent-core";
|
|
2
2
|
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
3
3
|
import { dirname, extname, join, relative, resolve } from "node:path";
|
|
4
|
-
import { Ajv } from "ajv";
|
|
5
4
|
import { LineCounter, isMap, isPair, isScalar, isSeq, parseDocument } from "yaml";
|
|
5
|
+
import { Ajv } from "ajv";
|
|
6
6
|
import handlerWs from "configurapi-handler-ws";
|
|
7
7
|
import { isDeepStrictEqual, styleText } from "node:util";
|
|
8
8
|
import { Agent, InMemoryAgentStateStore } from "@mappies/miniwat-agent";
|
|
@@ -273,22 +273,70 @@ var TurnSpec = class {
|
|
|
273
273
|
this.location = new SourceLocation(data?.location);
|
|
274
274
|
}
|
|
275
275
|
};
|
|
276
|
+
// src/entities/protocolCapabilities.ts
|
|
277
|
+
var ProtocolCapabilities = class {
|
|
278
|
+
toolUses;
|
|
279
|
+
toolResults;
|
|
280
|
+
tokenUsage;
|
|
281
|
+
constructor(data) {
|
|
282
|
+
this.toolUses = data?.toolUses ?? false;
|
|
283
|
+
this.toolResults = data?.toolResults ?? false;
|
|
284
|
+
this.tokenUsage = data?.tokenUsage ?? false;
|
|
285
|
+
}
|
|
286
|
+
};
|
|
287
|
+
// src/interfaces/requestFormat.ts
|
|
288
|
+
const RequestFormat = { Json: "json" };
|
|
289
|
+
// src/entities/requestConfig.ts
|
|
290
|
+
var RequestConfig = class {
|
|
291
|
+
format;
|
|
292
|
+
value;
|
|
293
|
+
inputPath;
|
|
294
|
+
constructor(data) {
|
|
295
|
+
this.format = data?.format ?? RequestFormat.Json;
|
|
296
|
+
this.value = data?.value;
|
|
297
|
+
this.inputPath = data?.inputPath;
|
|
298
|
+
}
|
|
299
|
+
};
|
|
300
|
+
// src/entities/protocolConfig.ts
|
|
301
|
+
var ProtocolConfig = class {
|
|
302
|
+
request;
|
|
303
|
+
capabilities;
|
|
304
|
+
constructor(data) {
|
|
305
|
+
this.request = new RequestConfig(data?.request);
|
|
306
|
+
this.capabilities = new ProtocolCapabilities(data?.capabilities);
|
|
307
|
+
}
|
|
308
|
+
};
|
|
309
|
+
// src/entities/webSocketConfig.ts
|
|
310
|
+
var WebSocketConfig = class {
|
|
311
|
+
endpoint;
|
|
312
|
+
connection;
|
|
313
|
+
protocol;
|
|
314
|
+
constructor(data) {
|
|
315
|
+
this.endpoint = data?.endpoint ?? "";
|
|
316
|
+
this.connection = new ConnectionConfig(data?.connection);
|
|
317
|
+
this.protocol = new ProtocolConfig(data?.protocol);
|
|
318
|
+
}
|
|
319
|
+
};
|
|
276
320
|
// src/entities/conversationSpec.ts
|
|
277
321
|
var ConversationSpec = class {
|
|
278
322
|
version;
|
|
279
323
|
name;
|
|
324
|
+
runName;
|
|
280
325
|
description;
|
|
281
326
|
turns;
|
|
282
327
|
conversation;
|
|
283
328
|
location;
|
|
329
|
+
websocket;
|
|
284
330
|
secrets;
|
|
285
331
|
constructor(data) {
|
|
286
332
|
this.version = data?.version ?? 1;
|
|
287
333
|
this.name = data?.name ?? "";
|
|
334
|
+
this.runName = data?.runName ?? "";
|
|
288
335
|
this.description = data?.description ?? "";
|
|
289
336
|
this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
|
|
290
337
|
this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
|
|
291
338
|
this.location = new SourceLocation(data?.location);
|
|
339
|
+
this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
|
|
292
340
|
this.secrets = [...data?.secrets ?? []];
|
|
293
341
|
}
|
|
294
342
|
};
|
|
@@ -390,55 +438,12 @@ var ReportConfig = class {
|
|
|
390
438
|
this.formats = data?.formats ?? [];
|
|
391
439
|
}
|
|
392
440
|
};
|
|
393
|
-
// src/entities/protocolCapabilities.ts
|
|
394
|
-
var ProtocolCapabilities = class {
|
|
395
|
-
toolUses;
|
|
396
|
-
toolResults;
|
|
397
|
-
tokenUsage;
|
|
398
|
-
constructor(data) {
|
|
399
|
-
this.toolUses = data?.toolUses ?? false;
|
|
400
|
-
this.toolResults = data?.toolResults ?? false;
|
|
401
|
-
this.tokenUsage = data?.tokenUsage ?? false;
|
|
402
|
-
}
|
|
403
|
-
};
|
|
404
|
-
// src/interfaces/requestFormat.ts
|
|
405
|
-
const RequestFormat = { Json: "json" };
|
|
406
|
-
// src/entities/requestConfig.ts
|
|
407
|
-
var RequestConfig = class {
|
|
408
|
-
format;
|
|
409
|
-
value;
|
|
410
|
-
inputPath;
|
|
411
|
-
constructor(data) {
|
|
412
|
-
this.format = data?.format ?? RequestFormat.Json;
|
|
413
|
-
this.value = data?.value;
|
|
414
|
-
this.inputPath = data?.inputPath;
|
|
415
|
-
}
|
|
416
|
-
};
|
|
417
|
-
// src/entities/protocolConfig.ts
|
|
418
|
-
var ProtocolConfig = class {
|
|
419
|
-
request;
|
|
420
|
-
capabilities;
|
|
421
|
-
constructor(data) {
|
|
422
|
-
this.request = new RequestConfig(data?.request);
|
|
423
|
-
this.capabilities = new ProtocolCapabilities(data?.capabilities);
|
|
424
|
-
}
|
|
425
|
-
};
|
|
426
|
-
// src/entities/webSocketConfig.ts
|
|
427
|
-
var WebSocketConfig = class {
|
|
428
|
-
endpoint;
|
|
429
|
-
connection;
|
|
430
|
-
protocol;
|
|
431
|
-
constructor(data) {
|
|
432
|
-
this.endpoint = data?.endpoint ?? "";
|
|
433
|
-
this.connection = new ConnectionConfig(data?.connection);
|
|
434
|
-
this.protocol = new ProtocolConfig(data?.protocol);
|
|
435
|
-
}
|
|
436
|
-
};
|
|
437
441
|
// src/entities/evalConfig.ts
|
|
438
442
|
var EvalConfig = class {
|
|
439
443
|
version;
|
|
440
444
|
configPath;
|
|
441
445
|
parametersPath;
|
|
446
|
+
variables;
|
|
442
447
|
websocket;
|
|
443
448
|
execution;
|
|
444
449
|
evaluator;
|
|
@@ -448,6 +453,7 @@ var EvalConfig = class {
|
|
|
448
453
|
this.version = data?.version ?? 1;
|
|
449
454
|
this.configPath = data?.configPath ?? "";
|
|
450
455
|
this.parametersPath = data?.parametersPath ?? "";
|
|
456
|
+
this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
|
|
451
457
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
452
458
|
this.execution = new ExecutionConfig(data?.execution);
|
|
453
459
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -759,6 +765,14 @@ var eval_config_schema_default = {
|
|
|
759
765
|
],
|
|
760
766
|
properties: {
|
|
761
767
|
"version": { "const": 1 },
|
|
768
|
+
"variables": {
|
|
769
|
+
"type": "object",
|
|
770
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
771
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
772
|
+
"type": "array",
|
|
773
|
+
"items": { "type": "string" }
|
|
774
|
+
}] }
|
|
775
|
+
},
|
|
762
776
|
"websocket": {
|
|
763
777
|
"type": "object",
|
|
764
778
|
"additionalProperties": false,
|
|
@@ -917,18 +931,80 @@ var eval_config_schema_default = {
|
|
|
917
931
|
}
|
|
918
932
|
}
|
|
919
933
|
};
|
|
934
|
+
// src/yaml/yamlDocument.ts
|
|
935
|
+
var YamlDocument = class YamlDocument {
|
|
936
|
+
file;
|
|
937
|
+
document;
|
|
938
|
+
lineCounter;
|
|
939
|
+
value;
|
|
940
|
+
errors;
|
|
941
|
+
constructor(file, document, lineCounter) {
|
|
942
|
+
this.file = file;
|
|
943
|
+
this.document = document;
|
|
944
|
+
this.lineCounter = lineCounter;
|
|
945
|
+
this.value = document.toJS();
|
|
946
|
+
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
947
|
+
message: error.message.split("\n")[0],
|
|
948
|
+
location: this.toLocation(error.pos[0], [])
|
|
949
|
+
}));
|
|
950
|
+
}
|
|
951
|
+
locate(path) {
|
|
952
|
+
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
953
|
+
}
|
|
954
|
+
locateKey(path, key) {
|
|
955
|
+
const parent = this.findNode(path);
|
|
956
|
+
if (isMap(parent)) {
|
|
957
|
+
const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
|
|
958
|
+
if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
959
|
+
}
|
|
960
|
+
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
961
|
+
}
|
|
962
|
+
static formatPath(path) {
|
|
963
|
+
return path.map((segment, index) => {
|
|
964
|
+
if (typeof segment === "number") return `[${segment}]`;
|
|
965
|
+
return index === 0 ? segment : `.${segment}`;
|
|
966
|
+
}).join("");
|
|
967
|
+
}
|
|
968
|
+
findNode(path) {
|
|
969
|
+
let node = this.document.contents ?? void 0;
|
|
970
|
+
for (const segment of path) {
|
|
971
|
+
let next;
|
|
972
|
+
if (isMap(node)) {
|
|
973
|
+
const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
|
|
974
|
+
next = pair?.value ?? pair?.key;
|
|
975
|
+
} else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
|
|
976
|
+
if (!next) break;
|
|
977
|
+
node = next;
|
|
978
|
+
}
|
|
979
|
+
return node;
|
|
980
|
+
}
|
|
981
|
+
toLocation(offset, path) {
|
|
982
|
+
const position = this.lineCounter.linePos(offset);
|
|
983
|
+
return new SourceLocation({
|
|
984
|
+
file: this.file,
|
|
985
|
+
line: position.line,
|
|
986
|
+
column: position.col,
|
|
987
|
+
path: YamlDocument.formatPath(path)
|
|
988
|
+
});
|
|
989
|
+
}
|
|
990
|
+
};
|
|
920
991
|
// src/yaml/environmentResolver.ts
|
|
921
992
|
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
|
|
922
993
|
const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
|
|
923
994
|
var EnvironmentResolver = class EnvironmentResolver {
|
|
924
995
|
environment;
|
|
925
996
|
parameters;
|
|
926
|
-
|
|
997
|
+
levels;
|
|
998
|
+
constructor(environment = process.env, parameters = {}, levels = []) {
|
|
927
999
|
this.environment = environment;
|
|
928
1000
|
this.parameters = parameters;
|
|
1001
|
+
this.levels = levels;
|
|
1002
|
+
}
|
|
1003
|
+
withLevels(levels) {
|
|
1004
|
+
return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
|
|
929
1005
|
}
|
|
930
|
-
resolve(value, document, issues, secrets = []) {
|
|
931
|
-
return this.resolveValue(value, [], document, issues, secrets);
|
|
1006
|
+
resolve(value, document, issues, secrets = [], options = {}) {
|
|
1007
|
+
return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
|
|
932
1008
|
}
|
|
933
1009
|
static hasReference(value) {
|
|
934
1010
|
return new RegExp(REFERENCE.source).test(value);
|
|
@@ -936,22 +1012,45 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
936
1012
|
static isSecretName(name) {
|
|
937
1013
|
return SECRET_NAME.test(name);
|
|
938
1014
|
}
|
|
939
|
-
|
|
1015
|
+
find(name) {
|
|
1016
|
+
for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
|
|
1017
|
+
return this.environment[name];
|
|
1018
|
+
}
|
|
1019
|
+
sources() {
|
|
1020
|
+
const sources = [
|
|
1021
|
+
...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
|
|
1022
|
+
...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
|
|
1023
|
+
"environment variable"
|
|
1024
|
+
];
|
|
1025
|
+
return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
|
|
1026
|
+
}
|
|
1027
|
+
resolveValue(value, path, document, issues, secrets, options) {
|
|
940
1028
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
941
|
-
|
|
1029
|
+
options.used?.add(name);
|
|
1030
|
+
const resolved = this.find(name);
|
|
942
1031
|
if (resolved === void 0) {
|
|
943
|
-
|
|
1032
|
+
if (options.optional?.(path)) {
|
|
1033
|
+
options.unresolved?.add(YamlDocument.formatPath(path));
|
|
1034
|
+
return reference;
|
|
1035
|
+
}
|
|
944
1036
|
issues.push(new ValidationIssue({
|
|
945
|
-
message: `${
|
|
1037
|
+
message: `${this.sources()} '${name}' is not set`,
|
|
946
1038
|
location: document.locate(path)
|
|
947
1039
|
}));
|
|
948
1040
|
return reference;
|
|
949
1041
|
}
|
|
950
|
-
if (
|
|
1042
|
+
if (Array.isArray(resolved)) {
|
|
1043
|
+
issues.push(new ValidationIssue({
|
|
1044
|
+
message: `variable '${name}' is a list, which can only be used as runWith.from`,
|
|
1045
|
+
location: document.locate(path)
|
|
1046
|
+
}));
|
|
1047
|
+
return reference;
|
|
1048
|
+
}
|
|
1049
|
+
if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
|
|
951
1050
|
return resolved;
|
|
952
1051
|
});
|
|
953
|
-
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
|
|
954
|
-
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
|
|
1052
|
+
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
|
|
1053
|
+
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
|
|
955
1054
|
return value;
|
|
956
1055
|
}
|
|
957
1056
|
};
|
|
@@ -1035,63 +1134,6 @@ var SchemaValidator = class SchemaValidator {
|
|
|
1035
1134
|
return path;
|
|
1036
1135
|
}
|
|
1037
1136
|
};
|
|
1038
|
-
// src/yaml/yamlDocument.ts
|
|
1039
|
-
var YamlDocument = class YamlDocument {
|
|
1040
|
-
file;
|
|
1041
|
-
document;
|
|
1042
|
-
lineCounter;
|
|
1043
|
-
value;
|
|
1044
|
-
errors;
|
|
1045
|
-
constructor(file, document, lineCounter) {
|
|
1046
|
-
this.file = file;
|
|
1047
|
-
this.document = document;
|
|
1048
|
-
this.lineCounter = lineCounter;
|
|
1049
|
-
this.value = document.toJS();
|
|
1050
|
-
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
1051
|
-
message: error.message.split("\n")[0],
|
|
1052
|
-
location: this.toLocation(error.pos[0], [])
|
|
1053
|
-
}));
|
|
1054
|
-
}
|
|
1055
|
-
locate(path) {
|
|
1056
|
-
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
1057
|
-
}
|
|
1058
|
-
locateKey(path, key) {
|
|
1059
|
-
const parent = this.findNode(path);
|
|
1060
|
-
if (isMap(parent)) {
|
|
1061
|
-
const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
|
|
1062
|
-
if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
1063
|
-
}
|
|
1064
|
-
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
1065
|
-
}
|
|
1066
|
-
static formatPath(path) {
|
|
1067
|
-
return path.map((segment, index) => {
|
|
1068
|
-
if (typeof segment === "number") return `[${segment}]`;
|
|
1069
|
-
return index === 0 ? segment : `.${segment}`;
|
|
1070
|
-
}).join("");
|
|
1071
|
-
}
|
|
1072
|
-
findNode(path) {
|
|
1073
|
-
let node = this.document.contents ?? void 0;
|
|
1074
|
-
for (const segment of path) {
|
|
1075
|
-
let next;
|
|
1076
|
-
if (isMap(node)) {
|
|
1077
|
-
const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
|
|
1078
|
-
next = pair?.value ?? pair?.key;
|
|
1079
|
-
} else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
|
|
1080
|
-
if (!next) break;
|
|
1081
|
-
node = next;
|
|
1082
|
-
}
|
|
1083
|
-
return node;
|
|
1084
|
-
}
|
|
1085
|
-
toLocation(offset, path) {
|
|
1086
|
-
const position = this.lineCounter.linePos(offset);
|
|
1087
|
-
return new SourceLocation({
|
|
1088
|
-
file: this.file,
|
|
1089
|
-
line: position.line,
|
|
1090
|
-
column: position.col,
|
|
1091
|
-
path: YamlDocument.formatPath(path)
|
|
1092
|
-
});
|
|
1093
|
-
}
|
|
1094
|
-
};
|
|
1095
1137
|
// src/yaml/yamlDocumentLoader.ts
|
|
1096
1138
|
var YamlDocumentLoader = class {
|
|
1097
1139
|
load(filePath, displayPath) {
|
|
@@ -1110,11 +1152,150 @@ var YamlDocumentLoader = class {
|
|
|
1110
1152
|
function isRecord(value) {
|
|
1111
1153
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
1112
1154
|
}
|
|
1155
|
+
// src/yaml/variablesReader.ts
|
|
1156
|
+
const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
1157
|
+
var VariablesReader = class {
|
|
1158
|
+
read(document, resolver, issues, secrets) {
|
|
1159
|
+
const raw = isRecord(document.value) ? document.value.variables : void 0;
|
|
1160
|
+
if (raw === void 0) return {};
|
|
1161
|
+
if (!isRecord(raw)) {
|
|
1162
|
+
issues.push(new ValidationIssue({
|
|
1163
|
+
message: "must be object",
|
|
1164
|
+
location: document.locate(["variables"])
|
|
1165
|
+
}));
|
|
1166
|
+
return {};
|
|
1167
|
+
}
|
|
1168
|
+
const variables = {};
|
|
1169
|
+
for (const [name, value] of Object.entries(raw)) {
|
|
1170
|
+
if (!VARIABLE_NAME.test(name)) {
|
|
1171
|
+
issues.push(new ValidationIssue({
|
|
1172
|
+
message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1173
|
+
location: document.locateKey(["variables"], name)
|
|
1174
|
+
}));
|
|
1175
|
+
continue;
|
|
1176
|
+
}
|
|
1177
|
+
if (Array.isArray(value)) {
|
|
1178
|
+
const bad = value.findIndex((item) => typeof item !== "string");
|
|
1179
|
+
if (bad >= 0) {
|
|
1180
|
+
issues.push(new ValidationIssue({
|
|
1181
|
+
message: "must be string",
|
|
1182
|
+
location: document.locate([
|
|
1183
|
+
"variables",
|
|
1184
|
+
name,
|
|
1185
|
+
bad
|
|
1186
|
+
])
|
|
1187
|
+
}));
|
|
1188
|
+
continue;
|
|
1189
|
+
}
|
|
1190
|
+
variables[name] = [...value];
|
|
1191
|
+
continue;
|
|
1192
|
+
}
|
|
1193
|
+
if (typeof value !== "string") {
|
|
1194
|
+
issues.push(new ValidationIssue({
|
|
1195
|
+
message: "must be string or a list of strings",
|
|
1196
|
+
location: document.locate(["variables", name])
|
|
1197
|
+
}));
|
|
1198
|
+
continue;
|
|
1199
|
+
}
|
|
1200
|
+
const before = issues.length;
|
|
1201
|
+
const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
|
|
1202
|
+
if (issues.length > before) continue;
|
|
1203
|
+
const text = resolved.variables[name];
|
|
1204
|
+
variables[name] = text;
|
|
1205
|
+
if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
|
|
1206
|
+
}
|
|
1207
|
+
return variables;
|
|
1208
|
+
}
|
|
1209
|
+
};
|
|
1210
|
+
// src/config/webSocketResolver.ts
|
|
1211
|
+
const WEBSOCKET_SCHEMA = {
|
|
1212
|
+
...eval_config_schema_default,
|
|
1213
|
+
required: ["websocket"],
|
|
1214
|
+
additionalProperties: true,
|
|
1215
|
+
properties: { websocket: eval_config_schema_default.properties.websocket }
|
|
1216
|
+
};
|
|
1217
|
+
var WebSocketResolver = class WebSocketResolver {
|
|
1218
|
+
schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
|
|
1219
|
+
resolve(document, resolver, issues, secrets, used, defer) {
|
|
1220
|
+
const raw = isRecord(document.value) ? document.value.websocket : void 0;
|
|
1221
|
+
const unresolved = /* @__PURE__ */ new Set();
|
|
1222
|
+
const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
|
|
1223
|
+
const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
|
|
1224
|
+
used,
|
|
1225
|
+
optional,
|
|
1226
|
+
unresolved
|
|
1227
|
+
});
|
|
1228
|
+
const subprotocols = WebSocketResolver.subprotocols(raw);
|
|
1229
|
+
if (subprotocols !== void 0) {
|
|
1230
|
+
const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
|
|
1231
|
+
const set = !unresolved.has(YamlDocument.formatPath([
|
|
1232
|
+
"websocket",
|
|
1233
|
+
"connection",
|
|
1234
|
+
"subprotocols",
|
|
1235
|
+
index
|
|
1236
|
+
]));
|
|
1237
|
+
const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
|
|
1238
|
+
return set && !empty;
|
|
1239
|
+
});
|
|
1240
|
+
value.websocket.connection.subprotocols = kept;
|
|
1241
|
+
}
|
|
1242
|
+
const prefix = YamlDocument.formatPath([
|
|
1243
|
+
"websocket",
|
|
1244
|
+
"connection",
|
|
1245
|
+
"subprotocols"
|
|
1246
|
+
]);
|
|
1247
|
+
return {
|
|
1248
|
+
value,
|
|
1249
|
+
deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
|
|
1250
|
+
};
|
|
1251
|
+
}
|
|
1252
|
+
validate(value, document) {
|
|
1253
|
+
const issues = this.schemaValidator.validate(value, document);
|
|
1254
|
+
if (issues.length > 0) return issues;
|
|
1255
|
+
return WebSocketResolver.validateInputPath(value, document);
|
|
1256
|
+
}
|
|
1257
|
+
static validateInputPath(value, document) {
|
|
1258
|
+
const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
|
|
1259
|
+
if (typeof request.inputPath !== "string") return [];
|
|
1260
|
+
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
1261
|
+
const last = tokens.pop();
|
|
1262
|
+
let parent = request.value;
|
|
1263
|
+
for (const token of tokens) parent = WebSocketResolver.child(parent, token);
|
|
1264
|
+
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
1265
|
+
return [new ValidationIssue({
|
|
1266
|
+
message: "must point to a missing or string value whose parent exists in request.value",
|
|
1267
|
+
location: document.locate([
|
|
1268
|
+
"websocket",
|
|
1269
|
+
"protocol",
|
|
1270
|
+
"request",
|
|
1271
|
+
"inputPath"
|
|
1272
|
+
])
|
|
1273
|
+
})];
|
|
1274
|
+
}
|
|
1275
|
+
static section(value, ...path) {
|
|
1276
|
+
let current = value;
|
|
1277
|
+
for (const key of path) current = WebSocketResolver.child(current, key);
|
|
1278
|
+
return isRecord(current) ? current : {};
|
|
1279
|
+
}
|
|
1280
|
+
static child(value, key) {
|
|
1281
|
+
if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
|
|
1282
|
+
return isRecord(value) ? value[key] : void 0;
|
|
1283
|
+
}
|
|
1284
|
+
static isSubprotocol(path) {
|
|
1285
|
+
return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
|
|
1286
|
+
}
|
|
1287
|
+
static subprotocols(websocket) {
|
|
1288
|
+
const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
|
|
1289
|
+
return Array.isArray(subprotocols) ? subprotocols : void 0;
|
|
1290
|
+
}
|
|
1291
|
+
};
|
|
1113
1292
|
// src/config/evalConfigLoader.ts
|
|
1114
1293
|
var EvalConfigLoader = class EvalConfigLoader {
|
|
1115
1294
|
cwd;
|
|
1116
1295
|
documentLoader = new YamlDocumentLoader();
|
|
1117
1296
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
1297
|
+
variablesReader = new VariablesReader();
|
|
1298
|
+
webSocketResolver = new WebSocketResolver();
|
|
1118
1299
|
environmentResolver;
|
|
1119
1300
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1120
1301
|
this.cwd = cwd;
|
|
@@ -1125,19 +1306,47 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1125
1306
|
const issues = [...document.errors];
|
|
1126
1307
|
if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
|
|
1127
1308
|
const named = [];
|
|
1128
|
-
const
|
|
1129
|
-
|
|
1130
|
-
if (issues.length === 0)
|
|
1309
|
+
const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
|
|
1310
|
+
let value = {};
|
|
1311
|
+
if (issues.length === 0) {
|
|
1312
|
+
const resolver = this.environmentResolver.withLevels([variables]);
|
|
1313
|
+
const { websocket: _websocket, variables: _variables, ...rest } = document.value;
|
|
1314
|
+
const resolved = resolver.resolve(rest, document, issues, named);
|
|
1315
|
+
const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
|
|
1316
|
+
value = {
|
|
1317
|
+
...resolved,
|
|
1318
|
+
..._variables === void 0 ? {} : { variables },
|
|
1319
|
+
..._websocket === void 0 ? {} : websocket.value
|
|
1320
|
+
};
|
|
1321
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
|
|
1322
|
+
if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
|
|
1323
|
+
}
|
|
1131
1324
|
if (issues.length > 0) throw new EvalError({
|
|
1132
1325
|
category: ErrorCategory.Configuration,
|
|
1133
1326
|
message: "invalid configuration",
|
|
1134
1327
|
issues
|
|
1135
1328
|
});
|
|
1136
|
-
const config = this.build(value, configPath, named);
|
|
1329
|
+
const config = this.build(value, configPath, variables, named);
|
|
1137
1330
|
this.applyRepetition(config, override, value, document);
|
|
1138
1331
|
EvalConfigLoader.applyReport(config, report);
|
|
1139
1332
|
return config;
|
|
1140
1333
|
}
|
|
1334
|
+
static buildWebSocket(value) {
|
|
1335
|
+
const section = (...path) => WebSocketResolver.section(value, ...path);
|
|
1336
|
+
const connection = section("websocket", "connection");
|
|
1337
|
+
const request = section("websocket", "protocol", "request");
|
|
1338
|
+
return new WebSocketConfig({
|
|
1339
|
+
endpoint: WebSocketResolver.section(value, "websocket").endpoint,
|
|
1340
|
+
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
1341
|
+
protocol: new ProtocolConfig({
|
|
1342
|
+
request: new RequestConfig({
|
|
1343
|
+
...request,
|
|
1344
|
+
format: request.format ?? RequestFormat.Json
|
|
1345
|
+
}),
|
|
1346
|
+
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
1347
|
+
})
|
|
1348
|
+
});
|
|
1349
|
+
}
|
|
1141
1350
|
static applyReport(config, report) {
|
|
1142
1351
|
if (report.formats === void 0) return;
|
|
1143
1352
|
const known = Object.values(ReportFormat);
|
|
@@ -1159,45 +1368,17 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1159
1368
|
location: document.locate(["evaluator", "apiKey"])
|
|
1160
1369
|
})];
|
|
1161
1370
|
}
|
|
1162
|
-
|
|
1163
|
-
const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
|
|
1164
|
-
if (typeof request.inputPath !== "string") return [];
|
|
1165
|
-
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
1166
|
-
const last = tokens.pop();
|
|
1167
|
-
let parent = request.value;
|
|
1168
|
-
for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
|
|
1169
|
-
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
1170
|
-
return [new ValidationIssue({
|
|
1171
|
-
message: "must point to a missing or string value whose parent exists in request.value",
|
|
1172
|
-
location: document.locate([
|
|
1173
|
-
"websocket",
|
|
1174
|
-
"protocol",
|
|
1175
|
-
"request",
|
|
1176
|
-
"inputPath"
|
|
1177
|
-
])
|
|
1178
|
-
})];
|
|
1179
|
-
}
|
|
1180
|
-
build(value, configPath, named) {
|
|
1371
|
+
build(value, configPath, variables, named) {
|
|
1181
1372
|
const section = (...path) => EvalConfigLoader.section(value, ...path);
|
|
1182
1373
|
const configDirectory = dirname(configPath);
|
|
1183
|
-
const
|
|
1184
|
-
const request = section("websocket", "protocol", "request");
|
|
1374
|
+
const websocket = EvalConfigLoader.buildWebSocket(value);
|
|
1185
1375
|
const evaluator = section("evaluator");
|
|
1186
1376
|
const report = section("report");
|
|
1187
1377
|
return new EvalConfig({
|
|
1188
1378
|
version: value.version,
|
|
1189
1379
|
configPath,
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
1193
|
-
protocol: new ProtocolConfig({
|
|
1194
|
-
request: new RequestConfig({
|
|
1195
|
-
...request,
|
|
1196
|
-
format: request.format ?? RequestFormat.Json
|
|
1197
|
-
}),
|
|
1198
|
-
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
1199
|
-
})
|
|
1200
|
-
}),
|
|
1380
|
+
variables,
|
|
1381
|
+
websocket,
|
|
1201
1382
|
execution: new ExecutionConfig({
|
|
1202
1383
|
...DEFAULT_CONFIG.execution,
|
|
1203
1384
|
...section("execution"),
|
|
@@ -1218,7 +1399,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1218
1399
|
directory: resolve(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
|
|
1219
1400
|
formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
|
|
1220
1401
|
}),
|
|
1221
|
-
secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols
|
|
1402
|
+
secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
|
|
1222
1403
|
});
|
|
1223
1404
|
}
|
|
1224
1405
|
applyRepetition(config, override, value, document) {
|
|
@@ -1271,13 +1452,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1271
1452
|
])];
|
|
1272
1453
|
}
|
|
1273
1454
|
static section(value, ...path) {
|
|
1274
|
-
|
|
1275
|
-
for (const key of path) current = EvalConfigLoader.child(current, key);
|
|
1276
|
-
return isRecord(current) ? current : {};
|
|
1277
|
-
}
|
|
1278
|
-
static child(value, key) {
|
|
1279
|
-
if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
|
|
1280
|
-
return isRecord(value) ? value[key] : void 0;
|
|
1455
|
+
return WebSocketResolver.section(value, ...path);
|
|
1281
1456
|
}
|
|
1282
1457
|
};
|
|
1283
1458
|
// src/config/parametersLoader.ts
|
|
@@ -1329,6 +1504,48 @@ var conversation_schema_default = {
|
|
|
1329
1504
|
"version": { "const": 1 },
|
|
1330
1505
|
"name": { "$ref": "#/definitions/text" },
|
|
1331
1506
|
"description": { "type": "string" },
|
|
1507
|
+
"variables": {
|
|
1508
|
+
"type": "object",
|
|
1509
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
1510
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
1511
|
+
"type": "array",
|
|
1512
|
+
"items": { "type": "string" }
|
|
1513
|
+
}] }
|
|
1514
|
+
},
|
|
1515
|
+
"runWith": {
|
|
1516
|
+
"description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
|
|
1517
|
+
"anyOf": [{
|
|
1518
|
+
"type": "array",
|
|
1519
|
+
"minItems": 1,
|
|
1520
|
+
"items": {
|
|
1521
|
+
"type": "object",
|
|
1522
|
+
"required": ["name"],
|
|
1523
|
+
"properties": { "name": {
|
|
1524
|
+
"type": "string",
|
|
1525
|
+
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
|
|
1526
|
+
} },
|
|
1527
|
+
"additionalProperties": { "type": "string" }
|
|
1528
|
+
}
|
|
1529
|
+
}, {
|
|
1530
|
+
"type": "object",
|
|
1531
|
+
"additionalProperties": false,
|
|
1532
|
+
"required": ["variable", "from"],
|
|
1533
|
+
"properties": {
|
|
1534
|
+
"variable": {
|
|
1535
|
+
"type": "string",
|
|
1536
|
+
"pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
|
|
1537
|
+
},
|
|
1538
|
+
"from": { "anyOf": [{
|
|
1539
|
+
"type": "array",
|
|
1540
|
+
"minItems": 1,
|
|
1541
|
+
"items": { "type": "string" }
|
|
1542
|
+
}, {
|
|
1543
|
+
"type": "string",
|
|
1544
|
+
"pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
|
|
1545
|
+
}] }
|
|
1546
|
+
}
|
|
1547
|
+
}]
|
|
1548
|
+
},
|
|
1332
1549
|
"turns": {
|
|
1333
1550
|
"type": "array",
|
|
1334
1551
|
"minItems": 1,
|
|
@@ -1341,8 +1558,8 @@ var conversation_schema_default = {
|
|
|
1341
1558
|
"properties": { "conversation": {
|
|
1342
1559
|
"type": "object",
|
|
1343
1560
|
"additionalProperties": false,
|
|
1344
|
-
"required": ["
|
|
1345
|
-
"properties": { "
|
|
1561
|
+
"required": ["should"],
|
|
1562
|
+
"properties": { "should": { "$ref": "#/definitions/text" } }
|
|
1346
1563
|
} }
|
|
1347
1564
|
}
|
|
1348
1565
|
},
|
|
@@ -1385,12 +1602,12 @@ var conversation_schema_default = {
|
|
|
1385
1602
|
}
|
|
1386
1603
|
},
|
|
1387
1604
|
"response": {
|
|
1388
|
-
"description": "A string is shorthand for
|
|
1605
|
+
"description": "A string is shorthand for should.",
|
|
1389
1606
|
"type": ["string", "object"],
|
|
1390
1607
|
"pattern": "\\S",
|
|
1391
1608
|
"additionalProperties": false,
|
|
1392
1609
|
"properties": {
|
|
1393
|
-
"
|
|
1610
|
+
"should": { "$ref": "#/definitions/text" },
|
|
1394
1611
|
"contains": { "$ref": "#/definitions/textList" },
|
|
1395
1612
|
"notContains": { "$ref": "#/definitions/textList" }
|
|
1396
1613
|
}
|
|
@@ -1422,7 +1639,7 @@ var conversation_schema_default = {
|
|
|
1422
1639
|
};
|
|
1423
1640
|
// src/scenarios/conversationValidator.ts
|
|
1424
1641
|
const RESPONSE_KEYS = [
|
|
1425
|
-
"
|
|
1642
|
+
"should",
|
|
1426
1643
|
"contains",
|
|
1427
1644
|
"notContains"
|
|
1428
1645
|
];
|
|
@@ -1445,7 +1662,7 @@ var ConversationValidator = class ConversationValidator {
|
|
|
1445
1662
|
validateResponse(response, path, document, issues) {
|
|
1446
1663
|
if (!RESPONSE_KEYS.some((key) => key in response)) {
|
|
1447
1664
|
issues.push(new ValidationIssue({
|
|
1448
|
-
message: "response must have at least one of:
|
|
1665
|
+
message: "response must have at least one of: should, contains, notContains",
|
|
1449
1666
|
location: document.locate(path)
|
|
1450
1667
|
}));
|
|
1451
1668
|
return;
|
|
@@ -1486,13 +1703,231 @@ var ConversationValidator = class ConversationValidator {
|
|
|
1486
1703
|
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
1487
1704
|
}
|
|
1488
1705
|
};
|
|
1706
|
+
// src/scenarios/runWithReader.ts
|
|
1707
|
+
const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
|
|
1708
|
+
const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
|
|
1709
|
+
const FORMS = "must be a list of rows, or an object with variable and from";
|
|
1710
|
+
const SINGLE_RUN = {
|
|
1711
|
+
name: "",
|
|
1712
|
+
variables: {},
|
|
1713
|
+
keys: [],
|
|
1714
|
+
location: [],
|
|
1715
|
+
secrets: []
|
|
1716
|
+
};
|
|
1717
|
+
var RunWithReader = class RunWithReader {
|
|
1718
|
+
read(document, resolver, issues, used) {
|
|
1719
|
+
const raw = isRecord(document.value) ? document.value.runWith : void 0;
|
|
1720
|
+
if (raw === void 0) return void 0;
|
|
1721
|
+
if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
|
|
1722
|
+
if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
|
|
1723
|
+
issues.push(new ValidationIssue({
|
|
1724
|
+
message: FORMS,
|
|
1725
|
+
location: document.locate(["runWith"])
|
|
1726
|
+
}));
|
|
1727
|
+
return [];
|
|
1728
|
+
}
|
|
1729
|
+
rows(rows, document, resolver, issues, used) {
|
|
1730
|
+
if (rows.length === 0) {
|
|
1731
|
+
issues.push(new ValidationIssue({
|
|
1732
|
+
message: "must not be empty",
|
|
1733
|
+
location: document.locate(["runWith"])
|
|
1734
|
+
}));
|
|
1735
|
+
return [];
|
|
1736
|
+
}
|
|
1737
|
+
const names = /* @__PURE__ */ new Set();
|
|
1738
|
+
const runs = [];
|
|
1739
|
+
rows.forEach((row, index) => {
|
|
1740
|
+
const path = ["runWith", index];
|
|
1741
|
+
if (!isRecord(row)) {
|
|
1742
|
+
issues.push(new ValidationIssue({
|
|
1743
|
+
message: "must be object",
|
|
1744
|
+
location: document.locate(path)
|
|
1745
|
+
}));
|
|
1746
|
+
return;
|
|
1747
|
+
}
|
|
1748
|
+
const name = row.name;
|
|
1749
|
+
if (name === void 0) issues.push(new ValidationIssue({
|
|
1750
|
+
message: "missing required key 'name'",
|
|
1751
|
+
location: document.locate(path)
|
|
1752
|
+
}));
|
|
1753
|
+
else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
|
|
1754
|
+
message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
|
|
1755
|
+
location: document.locate([...path, "name"])
|
|
1756
|
+
}));
|
|
1757
|
+
else if (names.has(name)) issues.push(new ValidationIssue({
|
|
1758
|
+
message: `duplicate run name '${name}'`,
|
|
1759
|
+
location: document.locate([...path, "name"])
|
|
1760
|
+
}));
|
|
1761
|
+
else names.add(name);
|
|
1762
|
+
const run = {
|
|
1763
|
+
name: typeof name === "string" ? name : "",
|
|
1764
|
+
variables: {},
|
|
1765
|
+
keys: [],
|
|
1766
|
+
location: path,
|
|
1767
|
+
secrets: []
|
|
1768
|
+
};
|
|
1769
|
+
for (const [key, value] of Object.entries(row)) {
|
|
1770
|
+
if (key === "name") continue;
|
|
1771
|
+
if (!VARIABLE_NAME.test(key)) {
|
|
1772
|
+
issues.push(new ValidationIssue({
|
|
1773
|
+
message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1774
|
+
location: document.locateKey(path, key)
|
|
1775
|
+
}));
|
|
1776
|
+
continue;
|
|
1777
|
+
}
|
|
1778
|
+
if (typeof value !== "string") {
|
|
1779
|
+
issues.push(new ValidationIssue({
|
|
1780
|
+
message: "must be string",
|
|
1781
|
+
location: document.locate([...path, key])
|
|
1782
|
+
}));
|
|
1783
|
+
continue;
|
|
1784
|
+
}
|
|
1785
|
+
const resolved = resolver.resolve(value, document, issues, run.secrets, {
|
|
1786
|
+
used,
|
|
1787
|
+
at: [...path, key]
|
|
1788
|
+
});
|
|
1789
|
+
RunWithReader.set(run, key, resolved, document.locateKey(path, key));
|
|
1790
|
+
}
|
|
1791
|
+
runs.push(run);
|
|
1792
|
+
});
|
|
1793
|
+
return runs;
|
|
1794
|
+
}
|
|
1795
|
+
list(form, document, resolver, issues, used) {
|
|
1796
|
+
const before = issues.length;
|
|
1797
|
+
for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
|
|
1798
|
+
message: `unknown key '${key}'`,
|
|
1799
|
+
location: document.locateKey(["runWith"], key)
|
|
1800
|
+
}));
|
|
1801
|
+
const variable = form.variable;
|
|
1802
|
+
if (variable === void 0) issues.push(new ValidationIssue({
|
|
1803
|
+
message: "missing required key 'variable'",
|
|
1804
|
+
location: document.locate(["runWith"])
|
|
1805
|
+
}));
|
|
1806
|
+
else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
|
|
1807
|
+
message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
|
|
1808
|
+
location: document.locate(["runWith", "variable"])
|
|
1809
|
+
}));
|
|
1810
|
+
const entries = this.entries(form.from, document, resolver, issues, used);
|
|
1811
|
+
if (issues.length > before || entries === void 0) return [];
|
|
1812
|
+
const name = variable;
|
|
1813
|
+
const keyLocation = document.locate(["runWith", "variable"]);
|
|
1814
|
+
const names = /* @__PURE__ */ new Set();
|
|
1815
|
+
const runs = [];
|
|
1816
|
+
for (const { entry, path } of entries) {
|
|
1817
|
+
if (!VARIABLE_NAME.test(entry)) {
|
|
1818
|
+
issues.push(new ValidationIssue({
|
|
1819
|
+
message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1820
|
+
location: document.locate(path)
|
|
1821
|
+
}));
|
|
1822
|
+
continue;
|
|
1823
|
+
}
|
|
1824
|
+
if (names.has(entry)) {
|
|
1825
|
+
issues.push(new ValidationIssue({
|
|
1826
|
+
message: `duplicate run name '${entry}'`,
|
|
1827
|
+
location: document.locate(path)
|
|
1828
|
+
}));
|
|
1829
|
+
continue;
|
|
1830
|
+
}
|
|
1831
|
+
names.add(entry);
|
|
1832
|
+
const run = {
|
|
1833
|
+
name: entry,
|
|
1834
|
+
variables: {},
|
|
1835
|
+
keys: [],
|
|
1836
|
+
location: path,
|
|
1837
|
+
secrets: []
|
|
1838
|
+
};
|
|
1839
|
+
const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
|
|
1840
|
+
used,
|
|
1841
|
+
at: path
|
|
1842
|
+
});
|
|
1843
|
+
RunWithReader.set(run, name, value, keyLocation);
|
|
1844
|
+
runs.push(run);
|
|
1845
|
+
}
|
|
1846
|
+
return runs;
|
|
1847
|
+
}
|
|
1848
|
+
entries(from, document, resolver, issues, used) {
|
|
1849
|
+
const path = ["runWith", "from"];
|
|
1850
|
+
const location = document.locate(path);
|
|
1851
|
+
if (from === void 0) {
|
|
1852
|
+
issues.push(new ValidationIssue({
|
|
1853
|
+
message: "missing required key 'from'",
|
|
1854
|
+
location: document.locate(["runWith"])
|
|
1855
|
+
}));
|
|
1856
|
+
return;
|
|
1857
|
+
}
|
|
1858
|
+
let list;
|
|
1859
|
+
let at;
|
|
1860
|
+
if (Array.isArray(from)) {
|
|
1861
|
+
list = from;
|
|
1862
|
+
at = (index) => [...path, index];
|
|
1863
|
+
} else {
|
|
1864
|
+
const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
|
|
1865
|
+
if (reference === null) {
|
|
1866
|
+
issues.push(new ValidationIssue({
|
|
1867
|
+
message: "must be a list, or a list variable such as ${tokens}",
|
|
1868
|
+
location
|
|
1869
|
+
}));
|
|
1870
|
+
return;
|
|
1871
|
+
}
|
|
1872
|
+
const name = reference[1];
|
|
1873
|
+
const found = resolver.find(name);
|
|
1874
|
+
used.add(name);
|
|
1875
|
+
if (found === void 0) {
|
|
1876
|
+
issues.push(new ValidationIssue({
|
|
1877
|
+
message: `list variable '${name}' is not set`,
|
|
1878
|
+
location
|
|
1879
|
+
}));
|
|
1880
|
+
return;
|
|
1881
|
+
}
|
|
1882
|
+
if (!Array.isArray(found)) {
|
|
1883
|
+
issues.push(new ValidationIssue({
|
|
1884
|
+
message: `variable '${name}' is not a list`,
|
|
1885
|
+
location
|
|
1886
|
+
}));
|
|
1887
|
+
return;
|
|
1888
|
+
}
|
|
1889
|
+
list = found;
|
|
1890
|
+
at = () => path;
|
|
1891
|
+
}
|
|
1892
|
+
if (list.length === 0) {
|
|
1893
|
+
issues.push(new ValidationIssue({
|
|
1894
|
+
message: "must not be empty",
|
|
1895
|
+
location
|
|
1896
|
+
}));
|
|
1897
|
+
return;
|
|
1898
|
+
}
|
|
1899
|
+
const bad = list.findIndex((item) => typeof item !== "string");
|
|
1900
|
+
if (bad >= 0) {
|
|
1901
|
+
issues.push(new ValidationIssue({
|
|
1902
|
+
message: "must be string",
|
|
1903
|
+
location: document.locate(at(bad))
|
|
1904
|
+
}));
|
|
1905
|
+
return;
|
|
1906
|
+
}
|
|
1907
|
+
return list.map((entry, index) => ({
|
|
1908
|
+
entry,
|
|
1909
|
+
path: at(index)
|
|
1910
|
+
}));
|
|
1911
|
+
}
|
|
1912
|
+
static set(run, name, value, location) {
|
|
1913
|
+
run.variables[name] = value;
|
|
1914
|
+
run.keys.push({
|
|
1915
|
+
name,
|
|
1916
|
+
location
|
|
1917
|
+
});
|
|
1918
|
+
if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
|
|
1919
|
+
}
|
|
1920
|
+
};
|
|
1489
1921
|
// src/scenarios/scenarioLoader.ts
|
|
1490
1922
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
1491
|
-
var ScenarioLoader = class {
|
|
1923
|
+
var ScenarioLoader = class ScenarioLoader {
|
|
1492
1924
|
cwd;
|
|
1493
1925
|
documentLoader = new YamlDocumentLoader();
|
|
1494
1926
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
1495
1927
|
conversationValidator = new ConversationValidator();
|
|
1928
|
+
variablesReader = new VariablesReader();
|
|
1929
|
+
runWithReader = new RunWithReader();
|
|
1930
|
+
webSocketResolver = new WebSocketResolver();
|
|
1496
1931
|
environmentResolver;
|
|
1497
1932
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1498
1933
|
this.cwd = cwd;
|
|
@@ -1500,18 +1935,66 @@ var ScenarioLoader = class {
|
|
|
1500
1935
|
}
|
|
1501
1936
|
load(target, config) {
|
|
1502
1937
|
const files = this.discover(resolve(this.cwd, target), config);
|
|
1938
|
+
const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.cwd, config.configPath)) : void 0;
|
|
1503
1939
|
const issues = [];
|
|
1940
|
+
const connectionIssues = [];
|
|
1504
1941
|
const specs = [];
|
|
1505
1942
|
for (const file of files) {
|
|
1506
1943
|
const document = this.documentLoader.load(file, relative(this.cwd, file));
|
|
1507
1944
|
const fileIssues = [...document.errors];
|
|
1508
|
-
const
|
|
1509
|
-
const
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1945
|
+
const fileSecrets = [];
|
|
1946
|
+
const used = /* @__PURE__ */ new Set();
|
|
1947
|
+
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
|
|
1948
|
+
const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
|
|
1949
|
+
const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
|
|
1950
|
+
if (fileIssues.length > 0) {
|
|
1951
|
+
issues.push(...fileIssues);
|
|
1952
|
+
continue;
|
|
1953
|
+
}
|
|
1954
|
+
const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
|
|
1955
|
+
const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
|
|
1956
|
+
for (const result of results) for (const name of result.used) used.add(name);
|
|
1957
|
+
fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
|
|
1958
|
+
run: result.run,
|
|
1959
|
+
issues: result.issues
|
|
1960
|
+
})), (issue, run) => run === void 0 ? issue : new ValidationIssue({
|
|
1961
|
+
message: `run '${run.name}': ${issue.message}`,
|
|
1962
|
+
location: issue.location
|
|
1963
|
+
})));
|
|
1964
|
+
const attempted = results.filter((result) => result.connectionChecked);
|
|
1965
|
+
if (attempted.length > 0) connectionIssues.push({
|
|
1966
|
+
file: document,
|
|
1967
|
+
runs: attempted.map((result) => ({
|
|
1968
|
+
run: result.run,
|
|
1969
|
+
issues: result.websocketIssues
|
|
1970
|
+
}))
|
|
1971
|
+
});
|
|
1972
|
+
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
|
|
1973
|
+
if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
|
|
1974
|
+
const websocket = result.websocket;
|
|
1975
|
+
specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
1976
|
+
}
|
|
1513
1977
|
else issues.push(...fileIssues);
|
|
1514
1978
|
}
|
|
1979
|
+
const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
|
|
1980
|
+
if (shared.length > 0) throw new EvalError({
|
|
1981
|
+
category: ErrorCategory.Configuration,
|
|
1982
|
+
message: "invalid configuration",
|
|
1983
|
+
issues: shared
|
|
1984
|
+
});
|
|
1985
|
+
for (const entry of connectionIssues) {
|
|
1986
|
+
const fileLocation = ScenarioLoader.variablesLocation(entry.file);
|
|
1987
|
+
issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
|
|
1988
|
+
message: issue.toString(),
|
|
1989
|
+
location: fileLocation
|
|
1990
|
+
}) : new ValidationIssue({
|
|
1991
|
+
message: `run '${run.name}': ${issue.toString()}`,
|
|
1992
|
+
location: new SourceLocation({
|
|
1993
|
+
...entry.file.locate(run.location),
|
|
1994
|
+
path: ""
|
|
1995
|
+
})
|
|
1996
|
+
})));
|
|
1997
|
+
}
|
|
1515
1998
|
if (issues.length > 0) throw new EvalError({
|
|
1516
1999
|
category: ErrorCategory.Specification,
|
|
1517
2000
|
message: "invalid scenario specification",
|
|
@@ -1519,6 +2002,71 @@ var ScenarioLoader = class {
|
|
|
1519
2002
|
});
|
|
1520
2003
|
return specs;
|
|
1521
2004
|
}
|
|
2005
|
+
resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
|
|
2006
|
+
const issues = [];
|
|
2007
|
+
const used = /* @__PURE__ */ new Set();
|
|
2008
|
+
const resolver = scenarioResolver.withLevels([run.variables]);
|
|
2009
|
+
const value = {
|
|
2010
|
+
...resolver.resolve(rest, document, issues, secrets, { used }),
|
|
2011
|
+
...variables === void 0 ? {} : { variables }
|
|
2012
|
+
};
|
|
2013
|
+
const websocketIssues = [];
|
|
2014
|
+
let websocket;
|
|
2015
|
+
let connectionChecked = false;
|
|
2016
|
+
if (issues.length === 0 && configDocument !== void 0) {
|
|
2017
|
+
const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
|
|
2018
|
+
if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
|
|
2019
|
+
if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
|
|
2020
|
+
connectionChecked = true;
|
|
2021
|
+
}
|
|
2022
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
|
|
2023
|
+
if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
|
|
2024
|
+
return {
|
|
2025
|
+
run,
|
|
2026
|
+
value,
|
|
2027
|
+
used,
|
|
2028
|
+
secrets,
|
|
2029
|
+
issues,
|
|
2030
|
+
websocket,
|
|
2031
|
+
websocketIssues,
|
|
2032
|
+
connectionChecked
|
|
2033
|
+
};
|
|
2034
|
+
}
|
|
2035
|
+
static byRun(runs, place) {
|
|
2036
|
+
const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
|
|
2037
|
+
const sharedTexts = new Set(shared.map((issue) => issue.toString()));
|
|
2038
|
+
const result = shared.map((issue) => place(issue, void 0));
|
|
2039
|
+
for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
|
|
2040
|
+
return result;
|
|
2041
|
+
}
|
|
2042
|
+
static shared(lists) {
|
|
2043
|
+
if (lists.length === 0) return [];
|
|
2044
|
+
const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
|
|
2045
|
+
return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
|
|
2046
|
+
}
|
|
2047
|
+
static variablesLocation(document) {
|
|
2048
|
+
return new SourceLocation({
|
|
2049
|
+
...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
|
|
2050
|
+
path: ""
|
|
2051
|
+
});
|
|
2052
|
+
}
|
|
2053
|
+
static unusedVariables(variables, used, results, document) {
|
|
2054
|
+
const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
|
|
2055
|
+
const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
|
|
2056
|
+
message: message(name),
|
|
2057
|
+
location: document.locateKey(["variables"], name)
|
|
2058
|
+
}));
|
|
2059
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2060
|
+
for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
|
|
2061
|
+
const issue = new ValidationIssue({
|
|
2062
|
+
message: message(key.name),
|
|
2063
|
+
location: key.location
|
|
2064
|
+
});
|
|
2065
|
+
if (!seen.has(issue.toString())) issues.push(issue);
|
|
2066
|
+
seen.add(issue.toString());
|
|
2067
|
+
}
|
|
2068
|
+
return issues;
|
|
2069
|
+
}
|
|
1522
2070
|
discover(target, config) {
|
|
1523
2071
|
if (!existsSync(target)) throw new EvalError({
|
|
1524
2072
|
category: ErrorCategory.Specification,
|
|
@@ -1543,20 +2091,22 @@ var ScenarioLoader = class {
|
|
|
1543
2091
|
}
|
|
1544
2092
|
return files;
|
|
1545
2093
|
}
|
|
1546
|
-
build(value, document, secrets) {
|
|
2094
|
+
build(value, document, runName, websocket, secrets) {
|
|
1547
2095
|
const scenario = value;
|
|
1548
2096
|
const turns = scenario.turns;
|
|
1549
2097
|
const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
|
|
1550
2098
|
return new ConversationSpec({
|
|
1551
2099
|
version: scenario.version,
|
|
1552
2100
|
name: scenario.name,
|
|
2101
|
+
runName,
|
|
1553
2102
|
description: typeof scenario.description === "string" ? scenario.description : "",
|
|
1554
2103
|
turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
|
|
1555
2104
|
conversation: conversation ? new ConversationExpectation({
|
|
1556
|
-
criteria: conversation.
|
|
2105
|
+
criteria: conversation.should,
|
|
1557
2106
|
location: document.locate(["expect", "conversation"])
|
|
1558
2107
|
}) : void 0,
|
|
1559
2108
|
location: document.locate([]),
|
|
2109
|
+
websocket,
|
|
1560
2110
|
secrets
|
|
1561
2111
|
});
|
|
1562
2112
|
}
|
|
@@ -1582,7 +2132,7 @@ var ScenarioLoader = class {
|
|
|
1582
2132
|
});
|
|
1583
2133
|
const fields = response;
|
|
1584
2134
|
return new ResponseExpectation({
|
|
1585
|
-
criteria: fields.
|
|
2135
|
+
criteria: fields.should,
|
|
1586
2136
|
contains: fields.contains ?? [],
|
|
1587
2137
|
notContains: fields.notContains ?? [],
|
|
1588
2138
|
location
|
|
@@ -2405,7 +2955,11 @@ var ConversationRunner = class {
|
|
|
2405
2955
|
async run(spec, signal = NEVER_ABORTED) {
|
|
2406
2956
|
const startedAt = Date.now();
|
|
2407
2957
|
const outcomes = [];
|
|
2408
|
-
const
|
|
2958
|
+
const config = new EvalConfig({
|
|
2959
|
+
...this.config,
|
|
2960
|
+
websocket: spec.websocket ?? this.config.websocket
|
|
2961
|
+
});
|
|
2962
|
+
const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
|
|
2409
2963
|
const history = [];
|
|
2410
2964
|
const evaluate = this.semanticEvaluator(signal);
|
|
2411
2965
|
const inline = !this.config.execution.continueOnFailure;
|
|
@@ -2430,7 +2984,7 @@ var ConversationRunner = class {
|
|
|
2430
2984
|
try {
|
|
2431
2985
|
const steps = (async () => {
|
|
2432
2986
|
for (let attempt = 0;; attempt++) {
|
|
2433
|
-
connection = this.createConnection(
|
|
2987
|
+
connection = this.createConnection(config);
|
|
2434
2988
|
connectionAttempts++;
|
|
2435
2989
|
try {
|
|
2436
2990
|
await connection.connect();
|
|
@@ -2924,7 +3478,7 @@ var ScenarioRunner = class {
|
|
|
2924
3478
|
required: minPass,
|
|
2925
3479
|
attempts,
|
|
2926
3480
|
tests: this.results.tests(attempts, minPass),
|
|
2927
|
-
agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
|
|
3481
|
+
agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
|
|
2928
3482
|
evaluatorUsage: this.tokens.evaluator(attempts)
|
|
2929
3483
|
});
|
|
2930
3484
|
}
|
|
@@ -3012,10 +3566,12 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3012
3566
|
scenario(result, repeat, tokenUsageCapability) {
|
|
3013
3567
|
const spec = result.spec;
|
|
3014
3568
|
const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
|
|
3015
|
-
const line = `${this.status(result.status, STATUS_WIDTH)} ${spec
|
|
3569
|
+
const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
|
|
3016
3570
|
if (!this.verbose) return line;
|
|
3017
3571
|
if (result.attempts.length === 0) return `${line}\n`;
|
|
3018
|
-
const
|
|
3572
|
+
const websocket = spec.websocket;
|
|
3573
|
+
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
3574
|
+
const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
|
|
3019
3575
|
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
3020
3576
|
lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
|
|
3021
3577
|
lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
|
|
@@ -3023,6 +3579,9 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3023
3579
|
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3024
3580
|
return line + lines.join("\n");
|
|
3025
3581
|
}
|
|
3582
|
+
static title(spec) {
|
|
3583
|
+
return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
|
|
3584
|
+
}
|
|
3026
3585
|
summary(run, exitCode, reportPaths = []) {
|
|
3027
3586
|
const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
|
|
3028
3587
|
const lines = this.verbose ? [] : [""];
|
|
@@ -3054,7 +3613,7 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3054
3613
|
}
|
|
3055
3614
|
problems(result) {
|
|
3056
3615
|
const spec = result.spec;
|
|
3057
|
-
const lines = [`${this.status(result.status, 6)} ${spec
|
|
3616
|
+
const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
|
|
3058
3617
|
for (const attempt of result.attempts) {
|
|
3059
3618
|
const execution = attempt.execution;
|
|
3060
3619
|
if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
|
|
@@ -3228,11 +3787,17 @@ var JsonReporter = class JsonReporter {
|
|
|
3228
3787
|
};
|
|
3229
3788
|
}
|
|
3230
3789
|
scenario(result, config) {
|
|
3231
|
-
const
|
|
3790
|
+
const websocket = result.spec.websocket ?? config.websocket;
|
|
3791
|
+
const capability = websocket.protocol.capabilities.tokenUsage;
|
|
3232
3792
|
return {
|
|
3233
3793
|
name: result.spec.name,
|
|
3794
|
+
runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
|
|
3234
3795
|
file: result.spec.location.file,
|
|
3235
3796
|
status: result.status,
|
|
3797
|
+
connection: {
|
|
3798
|
+
endpoint: websocket.endpoint,
|
|
3799
|
+
subprotocols: websocket.connection.subprotocols.length
|
|
3800
|
+
},
|
|
3236
3801
|
...JsonReporter.counts(result),
|
|
3237
3802
|
agentUsage: JsonReporter.usage(result.agentUsage),
|
|
3238
3803
|
evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
|