@mappies/miniwat-eval 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -11
- package/dist/index.cjs +743 -178
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +65 -48
- package/dist/index.d.ts +66 -48
- package/dist/index.js +743 -178
- package/dist/index.js.map +1 -1
- package/dist/main.js +699 -134
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +43 -4
- package/schemas/eval-config.schema.json +10 -0
- package/schemas/report.schema.json +12 -1
package/dist/index.cjs
CHANGED
|
@@ -23,8 +23,8 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
23
23
|
let _mappies_miniwat_agent_core = require("@mappies/miniwat-agent-core");
|
|
24
24
|
let node_fs = require("node:fs");
|
|
25
25
|
let node_path = require("node:path");
|
|
26
|
-
let ajv = require("ajv");
|
|
27
26
|
let yaml = require("yaml");
|
|
27
|
+
let ajv = require("ajv");
|
|
28
28
|
let configurapi_handler_ws = require("configurapi-handler-ws");
|
|
29
29
|
let node_util = require("node:util");
|
|
30
30
|
let _mappies_miniwat_agent = require("@mappies/miniwat-agent");
|
|
@@ -296,22 +296,70 @@ var TurnSpec = class {
|
|
|
296
296
|
this.location = new SourceLocation(data?.location);
|
|
297
297
|
}
|
|
298
298
|
};
|
|
299
|
+
// src/entities/protocolCapabilities.ts
|
|
300
|
+
var ProtocolCapabilities = class {
|
|
301
|
+
toolUses;
|
|
302
|
+
toolResults;
|
|
303
|
+
tokenUsage;
|
|
304
|
+
constructor(data) {
|
|
305
|
+
this.toolUses = data?.toolUses ?? false;
|
|
306
|
+
this.toolResults = data?.toolResults ?? false;
|
|
307
|
+
this.tokenUsage = data?.tokenUsage ?? false;
|
|
308
|
+
}
|
|
309
|
+
};
|
|
310
|
+
// src/interfaces/requestFormat.ts
|
|
311
|
+
const RequestFormat = { Json: "json" };
|
|
312
|
+
// src/entities/requestConfig.ts
|
|
313
|
+
var RequestConfig = class {
|
|
314
|
+
format;
|
|
315
|
+
value;
|
|
316
|
+
inputPath;
|
|
317
|
+
constructor(data) {
|
|
318
|
+
this.format = data?.format ?? RequestFormat.Json;
|
|
319
|
+
this.value = data?.value;
|
|
320
|
+
this.inputPath = data?.inputPath;
|
|
321
|
+
}
|
|
322
|
+
};
|
|
323
|
+
// src/entities/protocolConfig.ts
|
|
324
|
+
var ProtocolConfig = class {
|
|
325
|
+
request;
|
|
326
|
+
capabilities;
|
|
327
|
+
constructor(data) {
|
|
328
|
+
this.request = new RequestConfig(data?.request);
|
|
329
|
+
this.capabilities = new ProtocolCapabilities(data?.capabilities);
|
|
330
|
+
}
|
|
331
|
+
};
|
|
332
|
+
// src/entities/webSocketConfig.ts
|
|
333
|
+
var WebSocketConfig = class {
|
|
334
|
+
endpoint;
|
|
335
|
+
connection;
|
|
336
|
+
protocol;
|
|
337
|
+
constructor(data) {
|
|
338
|
+
this.endpoint = data?.endpoint ?? "";
|
|
339
|
+
this.connection = new ConnectionConfig(data?.connection);
|
|
340
|
+
this.protocol = new ProtocolConfig(data?.protocol);
|
|
341
|
+
}
|
|
342
|
+
};
|
|
299
343
|
// src/entities/conversationSpec.ts
|
|
300
344
|
var ConversationSpec = class {
|
|
301
345
|
version;
|
|
302
346
|
name;
|
|
347
|
+
runName;
|
|
303
348
|
description;
|
|
304
349
|
turns;
|
|
305
350
|
conversation;
|
|
306
351
|
location;
|
|
352
|
+
websocket;
|
|
307
353
|
secrets;
|
|
308
354
|
constructor(data) {
|
|
309
355
|
this.version = data?.version ?? 1;
|
|
310
356
|
this.name = data?.name ?? "";
|
|
357
|
+
this.runName = data?.runName ?? "";
|
|
311
358
|
this.description = data?.description ?? "";
|
|
312
359
|
this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
|
|
313
360
|
this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
|
|
314
361
|
this.location = new SourceLocation(data?.location);
|
|
362
|
+
this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
|
|
315
363
|
this.secrets = [...data?.secrets ?? []];
|
|
316
364
|
}
|
|
317
365
|
};
|
|
@@ -413,55 +461,12 @@ var ReportConfig = class {
|
|
|
413
461
|
this.formats = data?.formats ?? [];
|
|
414
462
|
}
|
|
415
463
|
};
|
|
416
|
-
// src/entities/protocolCapabilities.ts
|
|
417
|
-
var ProtocolCapabilities = class {
|
|
418
|
-
toolUses;
|
|
419
|
-
toolResults;
|
|
420
|
-
tokenUsage;
|
|
421
|
-
constructor(data) {
|
|
422
|
-
this.toolUses = data?.toolUses ?? false;
|
|
423
|
-
this.toolResults = data?.toolResults ?? false;
|
|
424
|
-
this.tokenUsage = data?.tokenUsage ?? false;
|
|
425
|
-
}
|
|
426
|
-
};
|
|
427
|
-
// src/interfaces/requestFormat.ts
|
|
428
|
-
const RequestFormat = { Json: "json" };
|
|
429
|
-
// src/entities/requestConfig.ts
|
|
430
|
-
var RequestConfig = class {
|
|
431
|
-
format;
|
|
432
|
-
value;
|
|
433
|
-
inputPath;
|
|
434
|
-
constructor(data) {
|
|
435
|
-
this.format = data?.format ?? RequestFormat.Json;
|
|
436
|
-
this.value = data?.value;
|
|
437
|
-
this.inputPath = data?.inputPath;
|
|
438
|
-
}
|
|
439
|
-
};
|
|
440
|
-
// src/entities/protocolConfig.ts
|
|
441
|
-
var ProtocolConfig = class {
|
|
442
|
-
request;
|
|
443
|
-
capabilities;
|
|
444
|
-
constructor(data) {
|
|
445
|
-
this.request = new RequestConfig(data?.request);
|
|
446
|
-
this.capabilities = new ProtocolCapabilities(data?.capabilities);
|
|
447
|
-
}
|
|
448
|
-
};
|
|
449
|
-
// src/entities/webSocketConfig.ts
|
|
450
|
-
var WebSocketConfig = class {
|
|
451
|
-
endpoint;
|
|
452
|
-
connection;
|
|
453
|
-
protocol;
|
|
454
|
-
constructor(data) {
|
|
455
|
-
this.endpoint = data?.endpoint ?? "";
|
|
456
|
-
this.connection = new ConnectionConfig(data?.connection);
|
|
457
|
-
this.protocol = new ProtocolConfig(data?.protocol);
|
|
458
|
-
}
|
|
459
|
-
};
|
|
460
464
|
// src/entities/evalConfig.ts
|
|
461
465
|
var EvalConfig = class {
|
|
462
466
|
version;
|
|
463
467
|
configPath;
|
|
464
468
|
parametersPath;
|
|
469
|
+
variables;
|
|
465
470
|
websocket;
|
|
466
471
|
execution;
|
|
467
472
|
evaluator;
|
|
@@ -471,6 +476,7 @@ var EvalConfig = class {
|
|
|
471
476
|
this.version = data?.version ?? 1;
|
|
472
477
|
this.configPath = data?.configPath ?? "";
|
|
473
478
|
this.parametersPath = data?.parametersPath ?? "";
|
|
479
|
+
this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
|
|
474
480
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
475
481
|
this.execution = new ExecutionConfig(data?.execution);
|
|
476
482
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -782,6 +788,14 @@ var eval_config_schema_default = {
|
|
|
782
788
|
],
|
|
783
789
|
properties: {
|
|
784
790
|
"version": { "const": 1 },
|
|
791
|
+
"variables": {
|
|
792
|
+
"type": "object",
|
|
793
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
794
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
795
|
+
"type": "array",
|
|
796
|
+
"items": { "type": "string" }
|
|
797
|
+
}] }
|
|
798
|
+
},
|
|
785
799
|
"websocket": {
|
|
786
800
|
"type": "object",
|
|
787
801
|
"additionalProperties": false,
|
|
@@ -940,18 +954,80 @@ var eval_config_schema_default = {
|
|
|
940
954
|
}
|
|
941
955
|
}
|
|
942
956
|
};
|
|
957
|
+
// src/yaml/yamlDocument.ts
|
|
958
|
+
var YamlDocument = class YamlDocument {
|
|
959
|
+
file;
|
|
960
|
+
document;
|
|
961
|
+
lineCounter;
|
|
962
|
+
value;
|
|
963
|
+
errors;
|
|
964
|
+
constructor(file, document, lineCounter) {
|
|
965
|
+
this.file = file;
|
|
966
|
+
this.document = document;
|
|
967
|
+
this.lineCounter = lineCounter;
|
|
968
|
+
this.value = document.toJS();
|
|
969
|
+
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
970
|
+
message: error.message.split("\n")[0],
|
|
971
|
+
location: this.toLocation(error.pos[0], [])
|
|
972
|
+
}));
|
|
973
|
+
}
|
|
974
|
+
locate(path) {
|
|
975
|
+
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
976
|
+
}
|
|
977
|
+
locateKey(path, key) {
|
|
978
|
+
const parent = this.findNode(path);
|
|
979
|
+
if ((0, yaml.isMap)(parent)) {
|
|
980
|
+
const pair = parent.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === key);
|
|
981
|
+
if (pair && (0, yaml.isScalar)(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
982
|
+
}
|
|
983
|
+
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
984
|
+
}
|
|
985
|
+
static formatPath(path) {
|
|
986
|
+
return path.map((segment, index) => {
|
|
987
|
+
if (typeof segment === "number") return `[${segment}]`;
|
|
988
|
+
return index === 0 ? segment : `.${segment}`;
|
|
989
|
+
}).join("");
|
|
990
|
+
}
|
|
991
|
+
findNode(path) {
|
|
992
|
+
let node = this.document.contents ?? void 0;
|
|
993
|
+
for (const segment of path) {
|
|
994
|
+
let next;
|
|
995
|
+
if ((0, yaml.isMap)(node)) {
|
|
996
|
+
const pair = node.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === String(segment));
|
|
997
|
+
next = pair?.value ?? pair?.key;
|
|
998
|
+
} else if ((0, yaml.isSeq)(node) && typeof segment === "number") next = node.items[segment];
|
|
999
|
+
if (!next) break;
|
|
1000
|
+
node = next;
|
|
1001
|
+
}
|
|
1002
|
+
return node;
|
|
1003
|
+
}
|
|
1004
|
+
toLocation(offset, path) {
|
|
1005
|
+
const position = this.lineCounter.linePos(offset);
|
|
1006
|
+
return new SourceLocation({
|
|
1007
|
+
file: this.file,
|
|
1008
|
+
line: position.line,
|
|
1009
|
+
column: position.col,
|
|
1010
|
+
path: YamlDocument.formatPath(path)
|
|
1011
|
+
});
|
|
1012
|
+
}
|
|
1013
|
+
};
|
|
943
1014
|
// src/yaml/environmentResolver.ts
|
|
944
1015
|
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
|
|
945
1016
|
const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
|
|
946
1017
|
var EnvironmentResolver = class EnvironmentResolver {
|
|
947
1018
|
environment;
|
|
948
1019
|
parameters;
|
|
949
|
-
|
|
1020
|
+
levels;
|
|
1021
|
+
constructor(environment = process.env, parameters = {}, levels = []) {
|
|
950
1022
|
this.environment = environment;
|
|
951
1023
|
this.parameters = parameters;
|
|
1024
|
+
this.levels = levels;
|
|
1025
|
+
}
|
|
1026
|
+
withLevels(levels) {
|
|
1027
|
+
return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
|
|
952
1028
|
}
|
|
953
|
-
resolve(value, document, issues, secrets = []) {
|
|
954
|
-
return this.resolveValue(value, [], document, issues, secrets);
|
|
1029
|
+
resolve(value, document, issues, secrets = [], options = {}) {
|
|
1030
|
+
return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
|
|
955
1031
|
}
|
|
956
1032
|
static hasReference(value) {
|
|
957
1033
|
return new RegExp(REFERENCE.source).test(value);
|
|
@@ -959,22 +1035,45 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
959
1035
|
static isSecretName(name) {
|
|
960
1036
|
return SECRET_NAME.test(name);
|
|
961
1037
|
}
|
|
962
|
-
|
|
1038
|
+
find(name) {
|
|
1039
|
+
for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
|
|
1040
|
+
return this.environment[name];
|
|
1041
|
+
}
|
|
1042
|
+
sources() {
|
|
1043
|
+
const sources = [
|
|
1044
|
+
...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
|
|
1045
|
+
...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
|
|
1046
|
+
"environment variable"
|
|
1047
|
+
];
|
|
1048
|
+
return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
|
|
1049
|
+
}
|
|
1050
|
+
resolveValue(value, path, document, issues, secrets, options) {
|
|
963
1051
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
964
|
-
|
|
1052
|
+
options.used?.add(name);
|
|
1053
|
+
const resolved = this.find(name);
|
|
965
1054
|
if (resolved === void 0) {
|
|
966
|
-
|
|
1055
|
+
if (options.optional?.(path)) {
|
|
1056
|
+
options.unresolved?.add(YamlDocument.formatPath(path));
|
|
1057
|
+
return reference;
|
|
1058
|
+
}
|
|
967
1059
|
issues.push(new ValidationIssue({
|
|
968
|
-
message: `${
|
|
1060
|
+
message: `${this.sources()} '${name}' is not set`,
|
|
969
1061
|
location: document.locate(path)
|
|
970
1062
|
}));
|
|
971
1063
|
return reference;
|
|
972
1064
|
}
|
|
973
|
-
if (
|
|
1065
|
+
if (Array.isArray(resolved)) {
|
|
1066
|
+
issues.push(new ValidationIssue({
|
|
1067
|
+
message: `variable '${name}' is a list, which can only be used as runWith.from`,
|
|
1068
|
+
location: document.locate(path)
|
|
1069
|
+
}));
|
|
1070
|
+
return reference;
|
|
1071
|
+
}
|
|
1072
|
+
if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
|
|
974
1073
|
return resolved;
|
|
975
1074
|
});
|
|
976
|
-
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
|
|
977
|
-
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
|
|
1075
|
+
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
|
|
1076
|
+
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
|
|
978
1077
|
return value;
|
|
979
1078
|
}
|
|
980
1079
|
};
|
|
@@ -1058,63 +1157,6 @@ var SchemaValidator = class SchemaValidator {
|
|
|
1058
1157
|
return path;
|
|
1059
1158
|
}
|
|
1060
1159
|
};
|
|
1061
|
-
// src/yaml/yamlDocument.ts
|
|
1062
|
-
var YamlDocument = class YamlDocument {
|
|
1063
|
-
file;
|
|
1064
|
-
document;
|
|
1065
|
-
lineCounter;
|
|
1066
|
-
value;
|
|
1067
|
-
errors;
|
|
1068
|
-
constructor(file, document, lineCounter) {
|
|
1069
|
-
this.file = file;
|
|
1070
|
-
this.document = document;
|
|
1071
|
-
this.lineCounter = lineCounter;
|
|
1072
|
-
this.value = document.toJS();
|
|
1073
|
-
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
1074
|
-
message: error.message.split("\n")[0],
|
|
1075
|
-
location: this.toLocation(error.pos[0], [])
|
|
1076
|
-
}));
|
|
1077
|
-
}
|
|
1078
|
-
locate(path) {
|
|
1079
|
-
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
1080
|
-
}
|
|
1081
|
-
locateKey(path, key) {
|
|
1082
|
-
const parent = this.findNode(path);
|
|
1083
|
-
if ((0, yaml.isMap)(parent)) {
|
|
1084
|
-
const pair = parent.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === key);
|
|
1085
|
-
if (pair && (0, yaml.isScalar)(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
1086
|
-
}
|
|
1087
|
-
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
1088
|
-
}
|
|
1089
|
-
static formatPath(path) {
|
|
1090
|
-
return path.map((segment, index) => {
|
|
1091
|
-
if (typeof segment === "number") return `[${segment}]`;
|
|
1092
|
-
return index === 0 ? segment : `.${segment}`;
|
|
1093
|
-
}).join("");
|
|
1094
|
-
}
|
|
1095
|
-
findNode(path) {
|
|
1096
|
-
let node = this.document.contents ?? void 0;
|
|
1097
|
-
for (const segment of path) {
|
|
1098
|
-
let next;
|
|
1099
|
-
if ((0, yaml.isMap)(node)) {
|
|
1100
|
-
const pair = node.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === String(segment));
|
|
1101
|
-
next = pair?.value ?? pair?.key;
|
|
1102
|
-
} else if ((0, yaml.isSeq)(node) && typeof segment === "number") next = node.items[segment];
|
|
1103
|
-
if (!next) break;
|
|
1104
|
-
node = next;
|
|
1105
|
-
}
|
|
1106
|
-
return node;
|
|
1107
|
-
}
|
|
1108
|
-
toLocation(offset, path) {
|
|
1109
|
-
const position = this.lineCounter.linePos(offset);
|
|
1110
|
-
return new SourceLocation({
|
|
1111
|
-
file: this.file,
|
|
1112
|
-
line: position.line,
|
|
1113
|
-
column: position.col,
|
|
1114
|
-
path: YamlDocument.formatPath(path)
|
|
1115
|
-
});
|
|
1116
|
-
}
|
|
1117
|
-
};
|
|
1118
1160
|
// src/yaml/yamlDocumentLoader.ts
|
|
1119
1161
|
var YamlDocumentLoader = class {
|
|
1120
1162
|
load(filePath, displayPath) {
|
|
@@ -1133,11 +1175,150 @@ var YamlDocumentLoader = class {
|
|
|
1133
1175
|
function isRecord(value) {
|
|
1134
1176
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
1135
1177
|
}
|
|
1178
|
+
// src/yaml/variablesReader.ts
|
|
1179
|
+
const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
1180
|
+
var VariablesReader = class {
|
|
1181
|
+
read(document, resolver, issues, secrets) {
|
|
1182
|
+
const raw = isRecord(document.value) ? document.value.variables : void 0;
|
|
1183
|
+
if (raw === void 0) return {};
|
|
1184
|
+
if (!isRecord(raw)) {
|
|
1185
|
+
issues.push(new ValidationIssue({
|
|
1186
|
+
message: "must be object",
|
|
1187
|
+
location: document.locate(["variables"])
|
|
1188
|
+
}));
|
|
1189
|
+
return {};
|
|
1190
|
+
}
|
|
1191
|
+
const variables = {};
|
|
1192
|
+
for (const [name, value] of Object.entries(raw)) {
|
|
1193
|
+
if (!VARIABLE_NAME.test(name)) {
|
|
1194
|
+
issues.push(new ValidationIssue({
|
|
1195
|
+
message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1196
|
+
location: document.locateKey(["variables"], name)
|
|
1197
|
+
}));
|
|
1198
|
+
continue;
|
|
1199
|
+
}
|
|
1200
|
+
if (Array.isArray(value)) {
|
|
1201
|
+
const bad = value.findIndex((item) => typeof item !== "string");
|
|
1202
|
+
if (bad >= 0) {
|
|
1203
|
+
issues.push(new ValidationIssue({
|
|
1204
|
+
message: "must be string",
|
|
1205
|
+
location: document.locate([
|
|
1206
|
+
"variables",
|
|
1207
|
+
name,
|
|
1208
|
+
bad
|
|
1209
|
+
])
|
|
1210
|
+
}));
|
|
1211
|
+
continue;
|
|
1212
|
+
}
|
|
1213
|
+
variables[name] = [...value];
|
|
1214
|
+
continue;
|
|
1215
|
+
}
|
|
1216
|
+
if (typeof value !== "string") {
|
|
1217
|
+
issues.push(new ValidationIssue({
|
|
1218
|
+
message: "must be string or a list of strings",
|
|
1219
|
+
location: document.locate(["variables", name])
|
|
1220
|
+
}));
|
|
1221
|
+
continue;
|
|
1222
|
+
}
|
|
1223
|
+
const before = issues.length;
|
|
1224
|
+
const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
|
|
1225
|
+
if (issues.length > before) continue;
|
|
1226
|
+
const text = resolved.variables[name];
|
|
1227
|
+
variables[name] = text;
|
|
1228
|
+
if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
|
|
1229
|
+
}
|
|
1230
|
+
return variables;
|
|
1231
|
+
}
|
|
1232
|
+
};
|
|
1233
|
+
// src/config/webSocketResolver.ts
|
|
1234
|
+
const WEBSOCKET_SCHEMA = {
|
|
1235
|
+
...eval_config_schema_default,
|
|
1236
|
+
required: ["websocket"],
|
|
1237
|
+
additionalProperties: true,
|
|
1238
|
+
properties: { websocket: eval_config_schema_default.properties.websocket }
|
|
1239
|
+
};
|
|
1240
|
+
var WebSocketResolver = class WebSocketResolver {
|
|
1241
|
+
schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
|
|
1242
|
+
resolve(document, resolver, issues, secrets, used, defer) {
|
|
1243
|
+
const raw = isRecord(document.value) ? document.value.websocket : void 0;
|
|
1244
|
+
const unresolved = /* @__PURE__ */ new Set();
|
|
1245
|
+
const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
|
|
1246
|
+
const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
|
|
1247
|
+
used,
|
|
1248
|
+
optional,
|
|
1249
|
+
unresolved
|
|
1250
|
+
});
|
|
1251
|
+
const subprotocols = WebSocketResolver.subprotocols(raw);
|
|
1252
|
+
if (subprotocols !== void 0) {
|
|
1253
|
+
const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
|
|
1254
|
+
const set = !unresolved.has(YamlDocument.formatPath([
|
|
1255
|
+
"websocket",
|
|
1256
|
+
"connection",
|
|
1257
|
+
"subprotocols",
|
|
1258
|
+
index
|
|
1259
|
+
]));
|
|
1260
|
+
const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
|
|
1261
|
+
return set && !empty;
|
|
1262
|
+
});
|
|
1263
|
+
value.websocket.connection.subprotocols = kept;
|
|
1264
|
+
}
|
|
1265
|
+
const prefix = YamlDocument.formatPath([
|
|
1266
|
+
"websocket",
|
|
1267
|
+
"connection",
|
|
1268
|
+
"subprotocols"
|
|
1269
|
+
]);
|
|
1270
|
+
return {
|
|
1271
|
+
value,
|
|
1272
|
+
deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
|
|
1273
|
+
};
|
|
1274
|
+
}
|
|
1275
|
+
validate(value, document) {
|
|
1276
|
+
const issues = this.schemaValidator.validate(value, document);
|
|
1277
|
+
if (issues.length > 0) return issues;
|
|
1278
|
+
return WebSocketResolver.validateInputPath(value, document);
|
|
1279
|
+
}
|
|
1280
|
+
static validateInputPath(value, document) {
|
|
1281
|
+
const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
|
|
1282
|
+
if (typeof request.inputPath !== "string") return [];
|
|
1283
|
+
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
1284
|
+
const last = tokens.pop();
|
|
1285
|
+
let parent = request.value;
|
|
1286
|
+
for (const token of tokens) parent = WebSocketResolver.child(parent, token);
|
|
1287
|
+
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
1288
|
+
return [new ValidationIssue({
|
|
1289
|
+
message: "must point to a missing or string value whose parent exists in request.value",
|
|
1290
|
+
location: document.locate([
|
|
1291
|
+
"websocket",
|
|
1292
|
+
"protocol",
|
|
1293
|
+
"request",
|
|
1294
|
+
"inputPath"
|
|
1295
|
+
])
|
|
1296
|
+
})];
|
|
1297
|
+
}
|
|
1298
|
+
static section(value, ...path) {
|
|
1299
|
+
let current = value;
|
|
1300
|
+
for (const key of path) current = WebSocketResolver.child(current, key);
|
|
1301
|
+
return isRecord(current) ? current : {};
|
|
1302
|
+
}
|
|
1303
|
+
static child(value, key) {
|
|
1304
|
+
if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
|
|
1305
|
+
return isRecord(value) ? value[key] : void 0;
|
|
1306
|
+
}
|
|
1307
|
+
static isSubprotocol(path) {
|
|
1308
|
+
return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
|
|
1309
|
+
}
|
|
1310
|
+
static subprotocols(websocket) {
|
|
1311
|
+
const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
|
|
1312
|
+
return Array.isArray(subprotocols) ? subprotocols : void 0;
|
|
1313
|
+
}
|
|
1314
|
+
};
|
|
1136
1315
|
// src/config/evalConfigLoader.ts
|
|
1137
1316
|
var EvalConfigLoader = class EvalConfigLoader {
|
|
1138
1317
|
cwd;
|
|
1139
1318
|
documentLoader = new YamlDocumentLoader();
|
|
1140
1319
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
1320
|
+
variablesReader = new VariablesReader();
|
|
1321
|
+
webSocketResolver = new WebSocketResolver();
|
|
1141
1322
|
environmentResolver;
|
|
1142
1323
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1143
1324
|
this.cwd = cwd;
|
|
@@ -1148,19 +1329,47 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1148
1329
|
const issues = [...document.errors];
|
|
1149
1330
|
if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
|
|
1150
1331
|
const named = [];
|
|
1151
|
-
const
|
|
1152
|
-
|
|
1153
|
-
if (issues.length === 0)
|
|
1332
|
+
const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
|
|
1333
|
+
let value = {};
|
|
1334
|
+
if (issues.length === 0) {
|
|
1335
|
+
const resolver = this.environmentResolver.withLevels([variables]);
|
|
1336
|
+
const { websocket: _websocket, variables: _variables, ...rest } = document.value;
|
|
1337
|
+
const resolved = resolver.resolve(rest, document, issues, named);
|
|
1338
|
+
const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
|
|
1339
|
+
value = {
|
|
1340
|
+
...resolved,
|
|
1341
|
+
..._variables === void 0 ? {} : { variables },
|
|
1342
|
+
..._websocket === void 0 ? {} : websocket.value
|
|
1343
|
+
};
|
|
1344
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
|
|
1345
|
+
if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
|
|
1346
|
+
}
|
|
1154
1347
|
if (issues.length > 0) throw new EvalError({
|
|
1155
1348
|
category: ErrorCategory.Configuration,
|
|
1156
1349
|
message: "invalid configuration",
|
|
1157
1350
|
issues
|
|
1158
1351
|
});
|
|
1159
|
-
const config = this.build(value, configPath, named);
|
|
1352
|
+
const config = this.build(value, configPath, variables, named);
|
|
1160
1353
|
this.applyRepetition(config, override, value, document);
|
|
1161
1354
|
EvalConfigLoader.applyReport(config, report);
|
|
1162
1355
|
return config;
|
|
1163
1356
|
}
|
|
1357
|
+
static buildWebSocket(value) {
|
|
1358
|
+
const section = (...path) => WebSocketResolver.section(value, ...path);
|
|
1359
|
+
const connection = section("websocket", "connection");
|
|
1360
|
+
const request = section("websocket", "protocol", "request");
|
|
1361
|
+
return new WebSocketConfig({
|
|
1362
|
+
endpoint: WebSocketResolver.section(value, "websocket").endpoint,
|
|
1363
|
+
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
1364
|
+
protocol: new ProtocolConfig({
|
|
1365
|
+
request: new RequestConfig({
|
|
1366
|
+
...request,
|
|
1367
|
+
format: request.format ?? RequestFormat.Json
|
|
1368
|
+
}),
|
|
1369
|
+
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
1370
|
+
})
|
|
1371
|
+
});
|
|
1372
|
+
}
|
|
1164
1373
|
static applyReport(config, report) {
|
|
1165
1374
|
if (report.formats === void 0) return;
|
|
1166
1375
|
const known = Object.values(ReportFormat);
|
|
@@ -1182,45 +1391,17 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1182
1391
|
location: document.locate(["evaluator", "apiKey"])
|
|
1183
1392
|
})];
|
|
1184
1393
|
}
|
|
1185
|
-
|
|
1186
|
-
const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
|
|
1187
|
-
if (typeof request.inputPath !== "string") return [];
|
|
1188
|
-
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
1189
|
-
const last = tokens.pop();
|
|
1190
|
-
let parent = request.value;
|
|
1191
|
-
for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
|
|
1192
|
-
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
1193
|
-
return [new ValidationIssue({
|
|
1194
|
-
message: "must point to a missing or string value whose parent exists in request.value",
|
|
1195
|
-
location: document.locate([
|
|
1196
|
-
"websocket",
|
|
1197
|
-
"protocol",
|
|
1198
|
-
"request",
|
|
1199
|
-
"inputPath"
|
|
1200
|
-
])
|
|
1201
|
-
})];
|
|
1202
|
-
}
|
|
1203
|
-
build(value, configPath, named) {
|
|
1394
|
+
build(value, configPath, variables, named) {
|
|
1204
1395
|
const section = (...path) => EvalConfigLoader.section(value, ...path);
|
|
1205
1396
|
const configDirectory = (0, node_path.dirname)(configPath);
|
|
1206
|
-
const
|
|
1207
|
-
const request = section("websocket", "protocol", "request");
|
|
1397
|
+
const websocket = EvalConfigLoader.buildWebSocket(value);
|
|
1208
1398
|
const evaluator = section("evaluator");
|
|
1209
1399
|
const report = section("report");
|
|
1210
1400
|
return new EvalConfig({
|
|
1211
1401
|
version: value.version,
|
|
1212
1402
|
configPath,
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
1216
|
-
protocol: new ProtocolConfig({
|
|
1217
|
-
request: new RequestConfig({
|
|
1218
|
-
...request,
|
|
1219
|
-
format: request.format ?? RequestFormat.Json
|
|
1220
|
-
}),
|
|
1221
|
-
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
1222
|
-
})
|
|
1223
|
-
}),
|
|
1403
|
+
variables,
|
|
1404
|
+
websocket,
|
|
1224
1405
|
execution: new ExecutionConfig({
|
|
1225
1406
|
...DEFAULT_CONFIG.execution,
|
|
1226
1407
|
...section("execution"),
|
|
@@ -1241,7 +1422,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1241
1422
|
directory: (0, node_path.resolve)(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
|
|
1242
1423
|
formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
|
|
1243
1424
|
}),
|
|
1244
|
-
secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols
|
|
1425
|
+
secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
|
|
1245
1426
|
});
|
|
1246
1427
|
}
|
|
1247
1428
|
applyRepetition(config, override, value, document) {
|
|
@@ -1294,13 +1475,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1294
1475
|
])];
|
|
1295
1476
|
}
|
|
1296
1477
|
static section(value, ...path) {
|
|
1297
|
-
|
|
1298
|
-
for (const key of path) current = EvalConfigLoader.child(current, key);
|
|
1299
|
-
return isRecord(current) ? current : {};
|
|
1300
|
-
}
|
|
1301
|
-
static child(value, key) {
|
|
1302
|
-
if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
|
|
1303
|
-
return isRecord(value) ? value[key] : void 0;
|
|
1478
|
+
return WebSocketResolver.section(value, ...path);
|
|
1304
1479
|
}
|
|
1305
1480
|
};
|
|
1306
1481
|
// src/config/parametersLoader.ts
|
|
@@ -1352,6 +1527,48 @@ var conversation_schema_default = {
|
|
|
1352
1527
|
"version": { "const": 1 },
|
|
1353
1528
|
"name": { "$ref": "#/definitions/text" },
|
|
1354
1529
|
"description": { "type": "string" },
|
|
1530
|
+
"variables": {
|
|
1531
|
+
"type": "object",
|
|
1532
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
1533
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
1534
|
+
"type": "array",
|
|
1535
|
+
"items": { "type": "string" }
|
|
1536
|
+
}] }
|
|
1537
|
+
},
|
|
1538
|
+
"runWith": {
|
|
1539
|
+
"description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
|
|
1540
|
+
"anyOf": [{
|
|
1541
|
+
"type": "array",
|
|
1542
|
+
"minItems": 1,
|
|
1543
|
+
"items": {
|
|
1544
|
+
"type": "object",
|
|
1545
|
+
"required": ["name"],
|
|
1546
|
+
"properties": { "name": {
|
|
1547
|
+
"type": "string",
|
|
1548
|
+
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
|
|
1549
|
+
} },
|
|
1550
|
+
"additionalProperties": { "type": "string" }
|
|
1551
|
+
}
|
|
1552
|
+
}, {
|
|
1553
|
+
"type": "object",
|
|
1554
|
+
"additionalProperties": false,
|
|
1555
|
+
"required": ["variable", "from"],
|
|
1556
|
+
"properties": {
|
|
1557
|
+
"variable": {
|
|
1558
|
+
"type": "string",
|
|
1559
|
+
"pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
|
|
1560
|
+
},
|
|
1561
|
+
"from": { "anyOf": [{
|
|
1562
|
+
"type": "array",
|
|
1563
|
+
"minItems": 1,
|
|
1564
|
+
"items": { "type": "string" }
|
|
1565
|
+
}, {
|
|
1566
|
+
"type": "string",
|
|
1567
|
+
"pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
|
|
1568
|
+
}] }
|
|
1569
|
+
}
|
|
1570
|
+
}]
|
|
1571
|
+
},
|
|
1355
1572
|
"turns": {
|
|
1356
1573
|
"type": "array",
|
|
1357
1574
|
"minItems": 1,
|
|
@@ -1364,8 +1581,8 @@ var conversation_schema_default = {
|
|
|
1364
1581
|
"properties": { "conversation": {
|
|
1365
1582
|
"type": "object",
|
|
1366
1583
|
"additionalProperties": false,
|
|
1367
|
-
"required": ["
|
|
1368
|
-
"properties": { "
|
|
1584
|
+
"required": ["should"],
|
|
1585
|
+
"properties": { "should": { "$ref": "#/definitions/text" } }
|
|
1369
1586
|
} }
|
|
1370
1587
|
}
|
|
1371
1588
|
},
|
|
@@ -1408,12 +1625,12 @@ var conversation_schema_default = {
|
|
|
1408
1625
|
}
|
|
1409
1626
|
},
|
|
1410
1627
|
"response": {
|
|
1411
|
-
"description": "A string is shorthand for
|
|
1628
|
+
"description": "A string is shorthand for should.",
|
|
1412
1629
|
"type": ["string", "object"],
|
|
1413
1630
|
"pattern": "\\S",
|
|
1414
1631
|
"additionalProperties": false,
|
|
1415
1632
|
"properties": {
|
|
1416
|
-
"
|
|
1633
|
+
"should": { "$ref": "#/definitions/text" },
|
|
1417
1634
|
"contains": { "$ref": "#/definitions/textList" },
|
|
1418
1635
|
"notContains": { "$ref": "#/definitions/textList" }
|
|
1419
1636
|
}
|
|
@@ -1445,7 +1662,7 @@ var conversation_schema_default = {
|
|
|
1445
1662
|
};
|
|
1446
1663
|
// src/scenarios/conversationValidator.ts
|
|
1447
1664
|
const RESPONSE_KEYS = [
|
|
1448
|
-
"
|
|
1665
|
+
"should",
|
|
1449
1666
|
"contains",
|
|
1450
1667
|
"notContains"
|
|
1451
1668
|
];
|
|
@@ -1468,7 +1685,7 @@ var ConversationValidator = class ConversationValidator {
|
|
|
1468
1685
|
validateResponse(response, path, document, issues) {
|
|
1469
1686
|
if (!RESPONSE_KEYS.some((key) => key in response)) {
|
|
1470
1687
|
issues.push(new ValidationIssue({
|
|
1471
|
-
message: "response must have at least one of:
|
|
1688
|
+
message: "response must have at least one of: should, contains, notContains",
|
|
1472
1689
|
location: document.locate(path)
|
|
1473
1690
|
}));
|
|
1474
1691
|
return;
|
|
@@ -1509,13 +1726,231 @@ var ConversationValidator = class ConversationValidator {
|
|
|
1509
1726
|
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
1510
1727
|
}
|
|
1511
1728
|
};
|
|
1729
|
+
// src/scenarios/runWithReader.ts
|
|
1730
|
+
const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
|
|
1731
|
+
const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
|
|
1732
|
+
const FORMS = "must be a list of rows, or an object with variable and from";
|
|
1733
|
+
const SINGLE_RUN = {
|
|
1734
|
+
name: "",
|
|
1735
|
+
variables: {},
|
|
1736
|
+
keys: [],
|
|
1737
|
+
location: [],
|
|
1738
|
+
secrets: []
|
|
1739
|
+
};
|
|
1740
|
+
var RunWithReader = class RunWithReader {
|
|
1741
|
+
read(document, resolver, issues, used) {
|
|
1742
|
+
const raw = isRecord(document.value) ? document.value.runWith : void 0;
|
|
1743
|
+
if (raw === void 0) return void 0;
|
|
1744
|
+
if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
|
|
1745
|
+
if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
|
|
1746
|
+
issues.push(new ValidationIssue({
|
|
1747
|
+
message: FORMS,
|
|
1748
|
+
location: document.locate(["runWith"])
|
|
1749
|
+
}));
|
|
1750
|
+
return [];
|
|
1751
|
+
}
|
|
1752
|
+
rows(rows, document, resolver, issues, used) {
|
|
1753
|
+
if (rows.length === 0) {
|
|
1754
|
+
issues.push(new ValidationIssue({
|
|
1755
|
+
message: "must not be empty",
|
|
1756
|
+
location: document.locate(["runWith"])
|
|
1757
|
+
}));
|
|
1758
|
+
return [];
|
|
1759
|
+
}
|
|
1760
|
+
const names = /* @__PURE__ */ new Set();
|
|
1761
|
+
const runs = [];
|
|
1762
|
+
rows.forEach((row, index) => {
|
|
1763
|
+
const path = ["runWith", index];
|
|
1764
|
+
if (!isRecord(row)) {
|
|
1765
|
+
issues.push(new ValidationIssue({
|
|
1766
|
+
message: "must be object",
|
|
1767
|
+
location: document.locate(path)
|
|
1768
|
+
}));
|
|
1769
|
+
return;
|
|
1770
|
+
}
|
|
1771
|
+
const name = row.name;
|
|
1772
|
+
if (name === void 0) issues.push(new ValidationIssue({
|
|
1773
|
+
message: "missing required key 'name'",
|
|
1774
|
+
location: document.locate(path)
|
|
1775
|
+
}));
|
|
1776
|
+
else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
|
|
1777
|
+
message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
|
|
1778
|
+
location: document.locate([...path, "name"])
|
|
1779
|
+
}));
|
|
1780
|
+
else if (names.has(name)) issues.push(new ValidationIssue({
|
|
1781
|
+
message: `duplicate run name '${name}'`,
|
|
1782
|
+
location: document.locate([...path, "name"])
|
|
1783
|
+
}));
|
|
1784
|
+
else names.add(name);
|
|
1785
|
+
const run = {
|
|
1786
|
+
name: typeof name === "string" ? name : "",
|
|
1787
|
+
variables: {},
|
|
1788
|
+
keys: [],
|
|
1789
|
+
location: path,
|
|
1790
|
+
secrets: []
|
|
1791
|
+
};
|
|
1792
|
+
for (const [key, value] of Object.entries(row)) {
|
|
1793
|
+
if (key === "name") continue;
|
|
1794
|
+
if (!VARIABLE_NAME.test(key)) {
|
|
1795
|
+
issues.push(new ValidationIssue({
|
|
1796
|
+
message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1797
|
+
location: document.locateKey(path, key)
|
|
1798
|
+
}));
|
|
1799
|
+
continue;
|
|
1800
|
+
}
|
|
1801
|
+
if (typeof value !== "string") {
|
|
1802
|
+
issues.push(new ValidationIssue({
|
|
1803
|
+
message: "must be string",
|
|
1804
|
+
location: document.locate([...path, key])
|
|
1805
|
+
}));
|
|
1806
|
+
continue;
|
|
1807
|
+
}
|
|
1808
|
+
const resolved = resolver.resolve(value, document, issues, run.secrets, {
|
|
1809
|
+
used,
|
|
1810
|
+
at: [...path, key]
|
|
1811
|
+
});
|
|
1812
|
+
RunWithReader.set(run, key, resolved, document.locateKey(path, key));
|
|
1813
|
+
}
|
|
1814
|
+
runs.push(run);
|
|
1815
|
+
});
|
|
1816
|
+
return runs;
|
|
1817
|
+
}
|
|
1818
|
+
list(form, document, resolver, issues, used) {
|
|
1819
|
+
const before = issues.length;
|
|
1820
|
+
for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
|
|
1821
|
+
message: `unknown key '${key}'`,
|
|
1822
|
+
location: document.locateKey(["runWith"], key)
|
|
1823
|
+
}));
|
|
1824
|
+
const variable = form.variable;
|
|
1825
|
+
if (variable === void 0) issues.push(new ValidationIssue({
|
|
1826
|
+
message: "missing required key 'variable'",
|
|
1827
|
+
location: document.locate(["runWith"])
|
|
1828
|
+
}));
|
|
1829
|
+
else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
|
|
1830
|
+
message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
|
|
1831
|
+
location: document.locate(["runWith", "variable"])
|
|
1832
|
+
}));
|
|
1833
|
+
const entries = this.entries(form.from, document, resolver, issues, used);
|
|
1834
|
+
if (issues.length > before || entries === void 0) return [];
|
|
1835
|
+
const name = variable;
|
|
1836
|
+
const keyLocation = document.locate(["runWith", "variable"]);
|
|
1837
|
+
const names = /* @__PURE__ */ new Set();
|
|
1838
|
+
const runs = [];
|
|
1839
|
+
for (const { entry, path } of entries) {
|
|
1840
|
+
if (!VARIABLE_NAME.test(entry)) {
|
|
1841
|
+
issues.push(new ValidationIssue({
|
|
1842
|
+
message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1843
|
+
location: document.locate(path)
|
|
1844
|
+
}));
|
|
1845
|
+
continue;
|
|
1846
|
+
}
|
|
1847
|
+
if (names.has(entry)) {
|
|
1848
|
+
issues.push(new ValidationIssue({
|
|
1849
|
+
message: `duplicate run name '${entry}'`,
|
|
1850
|
+
location: document.locate(path)
|
|
1851
|
+
}));
|
|
1852
|
+
continue;
|
|
1853
|
+
}
|
|
1854
|
+
names.add(entry);
|
|
1855
|
+
const run = {
|
|
1856
|
+
name: entry,
|
|
1857
|
+
variables: {},
|
|
1858
|
+
keys: [],
|
|
1859
|
+
location: path,
|
|
1860
|
+
secrets: []
|
|
1861
|
+
};
|
|
1862
|
+
const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
|
|
1863
|
+
used,
|
|
1864
|
+
at: path
|
|
1865
|
+
});
|
|
1866
|
+
RunWithReader.set(run, name, value, keyLocation);
|
|
1867
|
+
runs.push(run);
|
|
1868
|
+
}
|
|
1869
|
+
return runs;
|
|
1870
|
+
}
|
|
1871
|
+
entries(from, document, resolver, issues, used) {
|
|
1872
|
+
const path = ["runWith", "from"];
|
|
1873
|
+
const location = document.locate(path);
|
|
1874
|
+
if (from === void 0) {
|
|
1875
|
+
issues.push(new ValidationIssue({
|
|
1876
|
+
message: "missing required key 'from'",
|
|
1877
|
+
location: document.locate(["runWith"])
|
|
1878
|
+
}));
|
|
1879
|
+
return;
|
|
1880
|
+
}
|
|
1881
|
+
let list;
|
|
1882
|
+
let at;
|
|
1883
|
+
if (Array.isArray(from)) {
|
|
1884
|
+
list = from;
|
|
1885
|
+
at = (index) => [...path, index];
|
|
1886
|
+
} else {
|
|
1887
|
+
const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
|
|
1888
|
+
if (reference === null) {
|
|
1889
|
+
issues.push(new ValidationIssue({
|
|
1890
|
+
message: "must be a list, or a list variable such as ${tokens}",
|
|
1891
|
+
location
|
|
1892
|
+
}));
|
|
1893
|
+
return;
|
|
1894
|
+
}
|
|
1895
|
+
const name = reference[1];
|
|
1896
|
+
const found = resolver.find(name);
|
|
1897
|
+
used.add(name);
|
|
1898
|
+
if (found === void 0) {
|
|
1899
|
+
issues.push(new ValidationIssue({
|
|
1900
|
+
message: `list variable '${name}' is not set`,
|
|
1901
|
+
location
|
|
1902
|
+
}));
|
|
1903
|
+
return;
|
|
1904
|
+
}
|
|
1905
|
+
if (!Array.isArray(found)) {
|
|
1906
|
+
issues.push(new ValidationIssue({
|
|
1907
|
+
message: `variable '${name}' is not a list`,
|
|
1908
|
+
location
|
|
1909
|
+
}));
|
|
1910
|
+
return;
|
|
1911
|
+
}
|
|
1912
|
+
list = found;
|
|
1913
|
+
at = () => path;
|
|
1914
|
+
}
|
|
1915
|
+
if (list.length === 0) {
|
|
1916
|
+
issues.push(new ValidationIssue({
|
|
1917
|
+
message: "must not be empty",
|
|
1918
|
+
location
|
|
1919
|
+
}));
|
|
1920
|
+
return;
|
|
1921
|
+
}
|
|
1922
|
+
const bad = list.findIndex((item) => typeof item !== "string");
|
|
1923
|
+
if (bad >= 0) {
|
|
1924
|
+
issues.push(new ValidationIssue({
|
|
1925
|
+
message: "must be string",
|
|
1926
|
+
location: document.locate(at(bad))
|
|
1927
|
+
}));
|
|
1928
|
+
return;
|
|
1929
|
+
}
|
|
1930
|
+
return list.map((entry, index) => ({
|
|
1931
|
+
entry,
|
|
1932
|
+
path: at(index)
|
|
1933
|
+
}));
|
|
1934
|
+
}
|
|
1935
|
+
static set(run, name, value, location) {
|
|
1936
|
+
run.variables[name] = value;
|
|
1937
|
+
run.keys.push({
|
|
1938
|
+
name,
|
|
1939
|
+
location
|
|
1940
|
+
});
|
|
1941
|
+
if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
|
|
1942
|
+
}
|
|
1943
|
+
};
|
|
1512
1944
|
// src/scenarios/scenarioLoader.ts
|
|
1513
1945
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
1514
|
-
var ScenarioLoader = class {
|
|
1946
|
+
var ScenarioLoader = class ScenarioLoader {
|
|
1515
1947
|
cwd;
|
|
1516
1948
|
documentLoader = new YamlDocumentLoader();
|
|
1517
1949
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
1518
1950
|
conversationValidator = new ConversationValidator();
|
|
1951
|
+
variablesReader = new VariablesReader();
|
|
1952
|
+
runWithReader = new RunWithReader();
|
|
1953
|
+
webSocketResolver = new WebSocketResolver();
|
|
1519
1954
|
environmentResolver;
|
|
1520
1955
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1521
1956
|
this.cwd = cwd;
|
|
@@ -1523,18 +1958,66 @@ var ScenarioLoader = class {
|
|
|
1523
1958
|
}
|
|
1524
1959
|
load(target, config) {
|
|
1525
1960
|
const files = this.discover((0, node_path.resolve)(this.cwd, target), config);
|
|
1961
|
+
const configDocument = (0, node_fs.existsSync)(config.configPath) ? this.documentLoader.load(config.configPath, (0, node_path.relative)(this.cwd, config.configPath)) : void 0;
|
|
1526
1962
|
const issues = [];
|
|
1963
|
+
const connectionIssues = [];
|
|
1527
1964
|
const specs = [];
|
|
1528
1965
|
for (const file of files) {
|
|
1529
1966
|
const document = this.documentLoader.load(file, (0, node_path.relative)(this.cwd, file));
|
|
1530
1967
|
const fileIssues = [...document.errors];
|
|
1531
|
-
const
|
|
1532
|
-
const
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1968
|
+
const fileSecrets = [];
|
|
1969
|
+
const used = /* @__PURE__ */ new Set();
|
|
1970
|
+
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
|
|
1971
|
+
const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
|
|
1972
|
+
const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
|
|
1973
|
+
if (fileIssues.length > 0) {
|
|
1974
|
+
issues.push(...fileIssues);
|
|
1975
|
+
continue;
|
|
1976
|
+
}
|
|
1977
|
+
const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
|
|
1978
|
+
const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
|
|
1979
|
+
for (const result of results) for (const name of result.used) used.add(name);
|
|
1980
|
+
fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
|
|
1981
|
+
run: result.run,
|
|
1982
|
+
issues: result.issues
|
|
1983
|
+
})), (issue, run) => run === void 0 ? issue : new ValidationIssue({
|
|
1984
|
+
message: `run '${run.name}': ${issue.message}`,
|
|
1985
|
+
location: issue.location
|
|
1986
|
+
})));
|
|
1987
|
+
const attempted = results.filter((result) => result.connectionChecked);
|
|
1988
|
+
if (attempted.length > 0) connectionIssues.push({
|
|
1989
|
+
file: document,
|
|
1990
|
+
runs: attempted.map((result) => ({
|
|
1991
|
+
run: result.run,
|
|
1992
|
+
issues: result.websocketIssues
|
|
1993
|
+
}))
|
|
1994
|
+
});
|
|
1995
|
+
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
|
|
1996
|
+
if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
|
|
1997
|
+
const websocket = result.websocket;
|
|
1998
|
+
specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
1999
|
+
}
|
|
1536
2000
|
else issues.push(...fileIssues);
|
|
1537
2001
|
}
|
|
2002
|
+
const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
|
|
2003
|
+
if (shared.length > 0) throw new EvalError({
|
|
2004
|
+
category: ErrorCategory.Configuration,
|
|
2005
|
+
message: "invalid configuration",
|
|
2006
|
+
issues: shared
|
|
2007
|
+
});
|
|
2008
|
+
for (const entry of connectionIssues) {
|
|
2009
|
+
const fileLocation = ScenarioLoader.variablesLocation(entry.file);
|
|
2010
|
+
issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
|
|
2011
|
+
message: issue.toString(),
|
|
2012
|
+
location: fileLocation
|
|
2013
|
+
}) : new ValidationIssue({
|
|
2014
|
+
message: `run '${run.name}': ${issue.toString()}`,
|
|
2015
|
+
location: new SourceLocation({
|
|
2016
|
+
...entry.file.locate(run.location),
|
|
2017
|
+
path: ""
|
|
2018
|
+
})
|
|
2019
|
+
})));
|
|
2020
|
+
}
|
|
1538
2021
|
if (issues.length > 0) throw new EvalError({
|
|
1539
2022
|
category: ErrorCategory.Specification,
|
|
1540
2023
|
message: "invalid scenario specification",
|
|
@@ -1542,6 +2025,71 @@ var ScenarioLoader = class {
|
|
|
1542
2025
|
});
|
|
1543
2026
|
return specs;
|
|
1544
2027
|
}
|
|
2028
|
+
resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
|
|
2029
|
+
const issues = [];
|
|
2030
|
+
const used = /* @__PURE__ */ new Set();
|
|
2031
|
+
const resolver = scenarioResolver.withLevels([run.variables]);
|
|
2032
|
+
const value = {
|
|
2033
|
+
...resolver.resolve(rest, document, issues, secrets, { used }),
|
|
2034
|
+
...variables === void 0 ? {} : { variables }
|
|
2035
|
+
};
|
|
2036
|
+
const websocketIssues = [];
|
|
2037
|
+
let websocket;
|
|
2038
|
+
let connectionChecked = false;
|
|
2039
|
+
if (issues.length === 0 && configDocument !== void 0) {
|
|
2040
|
+
const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
|
|
2041
|
+
if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
|
|
2042
|
+
if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
|
|
2043
|
+
connectionChecked = true;
|
|
2044
|
+
}
|
|
2045
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
|
|
2046
|
+
if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
|
|
2047
|
+
return {
|
|
2048
|
+
run,
|
|
2049
|
+
value,
|
|
2050
|
+
used,
|
|
2051
|
+
secrets,
|
|
2052
|
+
issues,
|
|
2053
|
+
websocket,
|
|
2054
|
+
websocketIssues,
|
|
2055
|
+
connectionChecked
|
|
2056
|
+
};
|
|
2057
|
+
}
|
|
2058
|
+
static byRun(runs, place) {
|
|
2059
|
+
const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
|
|
2060
|
+
const sharedTexts = new Set(shared.map((issue) => issue.toString()));
|
|
2061
|
+
const result = shared.map((issue) => place(issue, void 0));
|
|
2062
|
+
for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
|
|
2063
|
+
return result;
|
|
2064
|
+
}
|
|
2065
|
+
static shared(lists) {
|
|
2066
|
+
if (lists.length === 0) return [];
|
|
2067
|
+
const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
|
|
2068
|
+
return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
|
|
2069
|
+
}
|
|
2070
|
+
static variablesLocation(document) {
|
|
2071
|
+
return new SourceLocation({
|
|
2072
|
+
...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
|
|
2073
|
+
path: ""
|
|
2074
|
+
});
|
|
2075
|
+
}
|
|
2076
|
+
static unusedVariables(variables, used, results, document) {
|
|
2077
|
+
const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
|
|
2078
|
+
const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
|
|
2079
|
+
message: message(name),
|
|
2080
|
+
location: document.locateKey(["variables"], name)
|
|
2081
|
+
}));
|
|
2082
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2083
|
+
for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
|
|
2084
|
+
const issue = new ValidationIssue({
|
|
2085
|
+
message: message(key.name),
|
|
2086
|
+
location: key.location
|
|
2087
|
+
});
|
|
2088
|
+
if (!seen.has(issue.toString())) issues.push(issue);
|
|
2089
|
+
seen.add(issue.toString());
|
|
2090
|
+
}
|
|
2091
|
+
return issues;
|
|
2092
|
+
}
|
|
1545
2093
|
discover(target, config) {
|
|
1546
2094
|
if (!(0, node_fs.existsSync)(target)) throw new EvalError({
|
|
1547
2095
|
category: ErrorCategory.Specification,
|
|
@@ -1566,20 +2114,22 @@ var ScenarioLoader = class {
|
|
|
1566
2114
|
}
|
|
1567
2115
|
return files;
|
|
1568
2116
|
}
|
|
1569
|
-
build(value, document, secrets) {
|
|
2117
|
+
build(value, document, runName, websocket, secrets) {
|
|
1570
2118
|
const scenario = value;
|
|
1571
2119
|
const turns = scenario.turns;
|
|
1572
2120
|
const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
|
|
1573
2121
|
return new ConversationSpec({
|
|
1574
2122
|
version: scenario.version,
|
|
1575
2123
|
name: scenario.name,
|
|
2124
|
+
runName,
|
|
1576
2125
|
description: typeof scenario.description === "string" ? scenario.description : "",
|
|
1577
2126
|
turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
|
|
1578
2127
|
conversation: conversation ? new ConversationExpectation({
|
|
1579
|
-
criteria: conversation.
|
|
2128
|
+
criteria: conversation.should,
|
|
1580
2129
|
location: document.locate(["expect", "conversation"])
|
|
1581
2130
|
}) : void 0,
|
|
1582
2131
|
location: document.locate([]),
|
|
2132
|
+
websocket,
|
|
1583
2133
|
secrets
|
|
1584
2134
|
});
|
|
1585
2135
|
}
|
|
@@ -1605,7 +2155,7 @@ var ScenarioLoader = class {
|
|
|
1605
2155
|
});
|
|
1606
2156
|
const fields = response;
|
|
1607
2157
|
return new ResponseExpectation({
|
|
1608
|
-
criteria: fields.
|
|
2158
|
+
criteria: fields.should,
|
|
1609
2159
|
contains: fields.contains ?? [],
|
|
1610
2160
|
notContains: fields.notContains ?? [],
|
|
1611
2161
|
location
|
|
@@ -2428,7 +2978,11 @@ var ConversationRunner = class {
|
|
|
2428
2978
|
async run(spec, signal = NEVER_ABORTED) {
|
|
2429
2979
|
const startedAt = Date.now();
|
|
2430
2980
|
const outcomes = [];
|
|
2431
|
-
const
|
|
2981
|
+
const config = new EvalConfig({
|
|
2982
|
+
...this.config,
|
|
2983
|
+
websocket: spec.websocket ?? this.config.websocket
|
|
2984
|
+
});
|
|
2985
|
+
const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
|
|
2432
2986
|
const history = [];
|
|
2433
2987
|
const evaluate = this.semanticEvaluator(signal);
|
|
2434
2988
|
const inline = !this.config.execution.continueOnFailure;
|
|
@@ -2453,7 +3007,7 @@ var ConversationRunner = class {
|
|
|
2453
3007
|
try {
|
|
2454
3008
|
const steps = (async () => {
|
|
2455
3009
|
for (let attempt = 0;; attempt++) {
|
|
2456
|
-
connection = this.createConnection(
|
|
3010
|
+
connection = this.createConnection(config);
|
|
2457
3011
|
connectionAttempts++;
|
|
2458
3012
|
try {
|
|
2459
3013
|
await connection.connect();
|
|
@@ -2947,7 +3501,7 @@ var ScenarioRunner = class {
|
|
|
2947
3501
|
required: minPass,
|
|
2948
3502
|
attempts,
|
|
2949
3503
|
tests: this.results.tests(attempts, minPass),
|
|
2950
|
-
agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
|
|
3504
|
+
agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
|
|
2951
3505
|
evaluatorUsage: this.tokens.evaluator(attempts)
|
|
2952
3506
|
});
|
|
2953
3507
|
}
|
|
@@ -3035,10 +3589,12 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3035
3589
|
scenario(result, repeat, tokenUsageCapability) {
|
|
3036
3590
|
const spec = result.spec;
|
|
3037
3591
|
const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
|
|
3038
|
-
const line = `${this.status(result.status, STATUS_WIDTH)} ${spec
|
|
3592
|
+
const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
|
|
3039
3593
|
if (!this.verbose) return line;
|
|
3040
3594
|
if (result.attempts.length === 0) return `${line}\n`;
|
|
3041
|
-
const
|
|
3595
|
+
const websocket = spec.websocket;
|
|
3596
|
+
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
3597
|
+
const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
|
|
3042
3598
|
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
3043
3599
|
lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
|
|
3044
3600
|
lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
|
|
@@ -3046,6 +3602,9 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3046
3602
|
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
3047
3603
|
return line + lines.join("\n");
|
|
3048
3604
|
}
|
|
3605
|
+
static title(spec) {
|
|
3606
|
+
return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
|
|
3607
|
+
}
|
|
3049
3608
|
summary(run, exitCode, reportPaths = []) {
|
|
3050
3609
|
const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
|
|
3051
3610
|
const lines = this.verbose ? [] : [""];
|
|
@@ -3077,7 +3636,7 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3077
3636
|
}
|
|
3078
3637
|
problems(result) {
|
|
3079
3638
|
const spec = result.spec;
|
|
3080
|
-
const lines = [`${this.status(result.status, 6)} ${spec
|
|
3639
|
+
const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
|
|
3081
3640
|
for (const attempt of result.attempts) {
|
|
3082
3641
|
const execution = attempt.execution;
|
|
3083
3642
|
if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
|
|
@@ -3251,11 +3810,17 @@ var JsonReporter = class JsonReporter {
|
|
|
3251
3810
|
};
|
|
3252
3811
|
}
|
|
3253
3812
|
scenario(result, config) {
|
|
3254
|
-
const
|
|
3813
|
+
const websocket = result.spec.websocket ?? config.websocket;
|
|
3814
|
+
const capability = websocket.protocol.capabilities.tokenUsage;
|
|
3255
3815
|
return {
|
|
3256
3816
|
name: result.spec.name,
|
|
3817
|
+
runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
|
|
3257
3818
|
file: result.spec.location.file,
|
|
3258
3819
|
status: result.status,
|
|
3820
|
+
connection: {
|
|
3821
|
+
endpoint: websocket.endpoint,
|
|
3822
|
+
subprotocols: websocket.connection.subprotocols.length
|
|
3823
|
+
},
|
|
3259
3824
|
...JsonReporter.counts(result),
|
|
3260
3825
|
agentUsage: JsonReporter.usage(result.agentUsage),
|
|
3261
3826
|
evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
|