@mappies/miniwat-eval 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -13
- package/dist/index.cjs +435 -178
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +72 -51
- package/dist/index.d.ts +72 -51
- package/dist/index.js +434 -178
- package/dist/index.js.map +1 -1
- package/dist/main.js +410 -148
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +9 -4
- package/schemas/eval-config.schema.json +5 -0
- package/schemas/report.schema.json +12 -1
package/dist/index.cjs
CHANGED
|
@@ -23,13 +23,13 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
23
23
|
let _mappies_miniwat_agent_core = require("@mappies/miniwat-agent-core");
|
|
24
24
|
let node_fs = require("node:fs");
|
|
25
25
|
let node_path = require("node:path");
|
|
26
|
-
let ajv = require("ajv");
|
|
27
26
|
let yaml = require("yaml");
|
|
27
|
+
let ajv = require("ajv");
|
|
28
28
|
let configurapi_handler_ws = require("configurapi-handler-ws");
|
|
29
|
-
configurapi_handler_ws = __toESM(configurapi_handler_ws, 1);
|
|
30
29
|
let node_util = require("node:util");
|
|
31
30
|
let _mappies_miniwat_agent = require("@mappies/miniwat-agent");
|
|
32
31
|
let node_crypto = require("node:crypto");
|
|
32
|
+
configurapi_handler_ws = __toESM(configurapi_handler_ws, 1);
|
|
33
33
|
// src/entities/turnExecution.ts
|
|
34
34
|
var TurnExecution = class {
|
|
35
35
|
response;
|
|
@@ -296,6 +296,50 @@ var TurnSpec = class {
|
|
|
296
296
|
this.location = new SourceLocation(data?.location);
|
|
297
297
|
}
|
|
298
298
|
};
|
|
299
|
+
// src/entities/protocolCapabilities.ts
|
|
300
|
+
var ProtocolCapabilities = class {
|
|
301
|
+
toolUses;
|
|
302
|
+
toolResults;
|
|
303
|
+
tokenUsage;
|
|
304
|
+
constructor(data) {
|
|
305
|
+
this.toolUses = data?.toolUses ?? false;
|
|
306
|
+
this.toolResults = data?.toolResults ?? false;
|
|
307
|
+
this.tokenUsage = data?.tokenUsage ?? false;
|
|
308
|
+
}
|
|
309
|
+
};
|
|
310
|
+
// src/interfaces/requestFormat.ts
|
|
311
|
+
const RequestFormat = { Json: "json" };
|
|
312
|
+
// src/entities/requestConfig.ts
|
|
313
|
+
var RequestConfig = class {
|
|
314
|
+
format;
|
|
315
|
+
value;
|
|
316
|
+
inputPath;
|
|
317
|
+
constructor(data) {
|
|
318
|
+
this.format = data?.format ?? RequestFormat.Json;
|
|
319
|
+
this.value = data?.value;
|
|
320
|
+
this.inputPath = data?.inputPath;
|
|
321
|
+
}
|
|
322
|
+
};
|
|
323
|
+
// src/entities/protocolConfig.ts
|
|
324
|
+
var ProtocolConfig = class {
|
|
325
|
+
request;
|
|
326
|
+
capabilities;
|
|
327
|
+
constructor(data) {
|
|
328
|
+
this.request = new RequestConfig(data?.request);
|
|
329
|
+
this.capabilities = new ProtocolCapabilities(data?.capabilities);
|
|
330
|
+
}
|
|
331
|
+
};
|
|
332
|
+
// src/entities/webSocketConfig.ts
|
|
333
|
+
var WebSocketConfig = class {
|
|
334
|
+
endpoint;
|
|
335
|
+
connection;
|
|
336
|
+
protocol;
|
|
337
|
+
constructor(data) {
|
|
338
|
+
this.endpoint = data?.endpoint ?? "";
|
|
339
|
+
this.connection = new ConnectionConfig(data?.connection);
|
|
340
|
+
this.protocol = new ProtocolConfig(data?.protocol);
|
|
341
|
+
}
|
|
342
|
+
};
|
|
299
343
|
// src/entities/conversationSpec.ts
|
|
300
344
|
var ConversationSpec = class {
|
|
301
345
|
version;
|
|
@@ -304,6 +348,7 @@ var ConversationSpec = class {
|
|
|
304
348
|
turns;
|
|
305
349
|
conversation;
|
|
306
350
|
location;
|
|
351
|
+
websocket;
|
|
307
352
|
secrets;
|
|
308
353
|
constructor(data) {
|
|
309
354
|
this.version = data?.version ?? 1;
|
|
@@ -312,6 +357,7 @@ var ConversationSpec = class {
|
|
|
312
357
|
this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
|
|
313
358
|
this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
|
|
314
359
|
this.location = new SourceLocation(data?.location);
|
|
360
|
+
this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
|
|
315
361
|
this.secrets = [...data?.secrets ?? []];
|
|
316
362
|
}
|
|
317
363
|
};
|
|
@@ -413,54 +459,12 @@ var ReportConfig = class {
|
|
|
413
459
|
this.formats = data?.formats ?? [];
|
|
414
460
|
}
|
|
415
461
|
};
|
|
416
|
-
// src/entities/protocolCapabilities.ts
|
|
417
|
-
var ProtocolCapabilities = class {
|
|
418
|
-
toolUses;
|
|
419
|
-
toolResults;
|
|
420
|
-
tokenUsage;
|
|
421
|
-
constructor(data) {
|
|
422
|
-
this.toolUses = data?.toolUses ?? false;
|
|
423
|
-
this.toolResults = data?.toolResults ?? false;
|
|
424
|
-
this.tokenUsage = data?.tokenUsage ?? false;
|
|
425
|
-
}
|
|
426
|
-
};
|
|
427
|
-
// src/interfaces/requestFormat.ts
|
|
428
|
-
const RequestFormat = { Json: "json" };
|
|
429
|
-
// src/entities/requestConfig.ts
|
|
430
|
-
var RequestConfig = class {
|
|
431
|
-
format;
|
|
432
|
-
value;
|
|
433
|
-
inputPath;
|
|
434
|
-
constructor(data) {
|
|
435
|
-
this.format = data?.format ?? RequestFormat.Json;
|
|
436
|
-
this.value = data?.value;
|
|
437
|
-
this.inputPath = data?.inputPath;
|
|
438
|
-
}
|
|
439
|
-
};
|
|
440
|
-
// src/entities/protocolConfig.ts
|
|
441
|
-
var ProtocolConfig = class {
|
|
442
|
-
request;
|
|
443
|
-
capabilities;
|
|
444
|
-
constructor(data) {
|
|
445
|
-
this.request = new RequestConfig(data?.request);
|
|
446
|
-
this.capabilities = new ProtocolCapabilities(data?.capabilities);
|
|
447
|
-
}
|
|
448
|
-
};
|
|
449
|
-
// src/entities/webSocketConfig.ts
|
|
450
|
-
var WebSocketConfig = class {
|
|
451
|
-
endpoint;
|
|
452
|
-
connection;
|
|
453
|
-
protocol;
|
|
454
|
-
constructor(data) {
|
|
455
|
-
this.endpoint = data?.endpoint ?? "";
|
|
456
|
-
this.connection = new ConnectionConfig(data?.connection);
|
|
457
|
-
this.protocol = new ProtocolConfig(data?.protocol);
|
|
458
|
-
}
|
|
459
|
-
};
|
|
460
462
|
// src/entities/evalConfig.ts
|
|
461
463
|
var EvalConfig = class {
|
|
462
464
|
version;
|
|
463
465
|
configPath;
|
|
466
|
+
parametersPath;
|
|
467
|
+
variables;
|
|
464
468
|
websocket;
|
|
465
469
|
execution;
|
|
466
470
|
evaluator;
|
|
@@ -469,6 +473,8 @@ var EvalConfig = class {
|
|
|
469
473
|
constructor(data) {
|
|
470
474
|
this.version = data?.version ?? 1;
|
|
471
475
|
this.configPath = data?.configPath ?? "";
|
|
476
|
+
this.parametersPath = data?.parametersPath ?? "";
|
|
477
|
+
this.variables = { ...data?.variables ?? {} };
|
|
472
478
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
473
479
|
this.execution = new ExecutionConfig(data?.execution);
|
|
474
480
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -780,6 +786,11 @@ var eval_config_schema_default = {
|
|
|
780
786
|
],
|
|
781
787
|
properties: {
|
|
782
788
|
"version": { "const": 1 },
|
|
789
|
+
"variables": {
|
|
790
|
+
"type": "object",
|
|
791
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
792
|
+
"additionalProperties": { "type": "string" }
|
|
793
|
+
},
|
|
783
794
|
"websocket": {
|
|
784
795
|
"type": "object",
|
|
785
796
|
"additionalProperties": false,
|
|
@@ -938,16 +949,80 @@ var eval_config_schema_default = {
|
|
|
938
949
|
}
|
|
939
950
|
}
|
|
940
951
|
};
|
|
952
|
+
// src/yaml/yamlDocument.ts
|
|
953
|
+
var YamlDocument = class YamlDocument {
|
|
954
|
+
file;
|
|
955
|
+
document;
|
|
956
|
+
lineCounter;
|
|
957
|
+
value;
|
|
958
|
+
errors;
|
|
959
|
+
constructor(file, document, lineCounter) {
|
|
960
|
+
this.file = file;
|
|
961
|
+
this.document = document;
|
|
962
|
+
this.lineCounter = lineCounter;
|
|
963
|
+
this.value = document.toJS();
|
|
964
|
+
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
965
|
+
message: error.message.split("\n")[0],
|
|
966
|
+
location: this.toLocation(error.pos[0], [])
|
|
967
|
+
}));
|
|
968
|
+
}
|
|
969
|
+
locate(path) {
|
|
970
|
+
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
971
|
+
}
|
|
972
|
+
locateKey(path, key) {
|
|
973
|
+
const parent = this.findNode(path);
|
|
974
|
+
if ((0, yaml.isMap)(parent)) {
|
|
975
|
+
const pair = parent.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === key);
|
|
976
|
+
if (pair && (0, yaml.isScalar)(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
977
|
+
}
|
|
978
|
+
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
979
|
+
}
|
|
980
|
+
static formatPath(path) {
|
|
981
|
+
return path.map((segment, index) => {
|
|
982
|
+
if (typeof segment === "number") return `[${segment}]`;
|
|
983
|
+
return index === 0 ? segment : `.${segment}`;
|
|
984
|
+
}).join("");
|
|
985
|
+
}
|
|
986
|
+
findNode(path) {
|
|
987
|
+
let node = this.document.contents ?? void 0;
|
|
988
|
+
for (const segment of path) {
|
|
989
|
+
let next;
|
|
990
|
+
if ((0, yaml.isMap)(node)) {
|
|
991
|
+
const pair = node.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === String(segment));
|
|
992
|
+
next = pair?.value ?? pair?.key;
|
|
993
|
+
} else if ((0, yaml.isSeq)(node) && typeof segment === "number") next = node.items[segment];
|
|
994
|
+
if (!next) break;
|
|
995
|
+
node = next;
|
|
996
|
+
}
|
|
997
|
+
return node;
|
|
998
|
+
}
|
|
999
|
+
toLocation(offset, path) {
|
|
1000
|
+
const position = this.lineCounter.linePos(offset);
|
|
1001
|
+
return new SourceLocation({
|
|
1002
|
+
file: this.file,
|
|
1003
|
+
line: position.line,
|
|
1004
|
+
column: position.col,
|
|
1005
|
+
path: YamlDocument.formatPath(path)
|
|
1006
|
+
});
|
|
1007
|
+
}
|
|
1008
|
+
};
|
|
941
1009
|
// src/yaml/environmentResolver.ts
|
|
942
|
-
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g;
|
|
1010
|
+
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
|
|
943
1011
|
const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
|
|
944
1012
|
var EnvironmentResolver = class EnvironmentResolver {
|
|
945
1013
|
environment;
|
|
946
|
-
|
|
1014
|
+
parameters;
|
|
1015
|
+
levels;
|
|
1016
|
+
constructor(environment = process.env, parameters = {}, levels = []) {
|
|
947
1017
|
this.environment = environment;
|
|
1018
|
+
this.parameters = parameters;
|
|
1019
|
+
this.levels = levels;
|
|
1020
|
+
}
|
|
1021
|
+
withLevels(levels) {
|
|
1022
|
+
return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
|
|
948
1023
|
}
|
|
949
|
-
resolve(value, document, issues, secrets = []) {
|
|
950
|
-
return this.resolveValue(value, [], document, issues, secrets);
|
|
1024
|
+
resolve(value, document, issues, secrets = [], options = {}) {
|
|
1025
|
+
return this.resolveValue(value, [], document, issues, secrets, options);
|
|
951
1026
|
}
|
|
952
1027
|
static hasReference(value) {
|
|
953
1028
|
return new RegExp(REFERENCE.source).test(value);
|
|
@@ -955,21 +1030,38 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
955
1030
|
static isSecretName(name) {
|
|
956
1031
|
return SECRET_NAME.test(name);
|
|
957
1032
|
}
|
|
958
|
-
|
|
1033
|
+
lookup(name) {
|
|
1034
|
+
for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
|
|
1035
|
+
return this.environment[name];
|
|
1036
|
+
}
|
|
1037
|
+
sources() {
|
|
1038
|
+
const sources = [
|
|
1039
|
+
...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
|
|
1040
|
+
...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
|
|
1041
|
+
"environment variable"
|
|
1042
|
+
];
|
|
1043
|
+
return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
|
|
1044
|
+
}
|
|
1045
|
+
resolveValue(value, path, document, issues, secrets, options) {
|
|
959
1046
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
960
|
-
|
|
1047
|
+
options.used?.add(name);
|
|
1048
|
+
const resolved = this.lookup(name);
|
|
961
1049
|
if (resolved === void 0) {
|
|
1050
|
+
if (options.optional?.(path)) {
|
|
1051
|
+
options.unresolved?.add(YamlDocument.formatPath(path));
|
|
1052
|
+
return reference;
|
|
1053
|
+
}
|
|
962
1054
|
issues.push(new ValidationIssue({
|
|
963
|
-
message:
|
|
1055
|
+
message: `${this.sources()} '${name}' is not set`,
|
|
964
1056
|
location: document.locate(path)
|
|
965
1057
|
}));
|
|
966
1058
|
return reference;
|
|
967
1059
|
}
|
|
968
|
-
if (EnvironmentResolver.isSecretName(name) && !secrets.includes(resolved)) secrets.push(resolved);
|
|
1060
|
+
if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
|
|
969
1061
|
return resolved;
|
|
970
1062
|
});
|
|
971
|
-
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
|
|
972
|
-
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
|
|
1063
|
+
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
|
|
1064
|
+
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
|
|
973
1065
|
return value;
|
|
974
1066
|
}
|
|
975
1067
|
};
|
|
@@ -1053,63 +1145,6 @@ var SchemaValidator = class SchemaValidator {
|
|
|
1053
1145
|
return path;
|
|
1054
1146
|
}
|
|
1055
1147
|
};
|
|
1056
|
-
// src/yaml/yamlDocument.ts
|
|
1057
|
-
var YamlDocument = class YamlDocument {
|
|
1058
|
-
file;
|
|
1059
|
-
document;
|
|
1060
|
-
lineCounter;
|
|
1061
|
-
value;
|
|
1062
|
-
errors;
|
|
1063
|
-
constructor(file, document, lineCounter) {
|
|
1064
|
-
this.file = file;
|
|
1065
|
-
this.document = document;
|
|
1066
|
-
this.lineCounter = lineCounter;
|
|
1067
|
-
this.value = document.toJS();
|
|
1068
|
-
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
1069
|
-
message: error.message.split("\n")[0],
|
|
1070
|
-
location: this.toLocation(error.pos[0], [])
|
|
1071
|
-
}));
|
|
1072
|
-
}
|
|
1073
|
-
locate(path) {
|
|
1074
|
-
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
1075
|
-
}
|
|
1076
|
-
locateKey(path, key) {
|
|
1077
|
-
const parent = this.findNode(path);
|
|
1078
|
-
if ((0, yaml.isMap)(parent)) {
|
|
1079
|
-
const pair = parent.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === key);
|
|
1080
|
-
if (pair && (0, yaml.isScalar)(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
1081
|
-
}
|
|
1082
|
-
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
1083
|
-
}
|
|
1084
|
-
static formatPath(path) {
|
|
1085
|
-
return path.map((segment, index) => {
|
|
1086
|
-
if (typeof segment === "number") return `[${segment}]`;
|
|
1087
|
-
return index === 0 ? segment : `.${segment}`;
|
|
1088
|
-
}).join("");
|
|
1089
|
-
}
|
|
1090
|
-
findNode(path) {
|
|
1091
|
-
let node = this.document.contents ?? void 0;
|
|
1092
|
-
for (const segment of path) {
|
|
1093
|
-
let next;
|
|
1094
|
-
if ((0, yaml.isMap)(node)) {
|
|
1095
|
-
const pair = node.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === String(segment));
|
|
1096
|
-
next = pair?.value ?? pair?.key;
|
|
1097
|
-
} else if ((0, yaml.isSeq)(node) && typeof segment === "number") next = node.items[segment];
|
|
1098
|
-
if (!next) break;
|
|
1099
|
-
node = next;
|
|
1100
|
-
}
|
|
1101
|
-
return node;
|
|
1102
|
-
}
|
|
1103
|
-
toLocation(offset, path) {
|
|
1104
|
-
const position = this.lineCounter.linePos(offset);
|
|
1105
|
-
return new SourceLocation({
|
|
1106
|
-
file: this.file,
|
|
1107
|
-
line: position.line,
|
|
1108
|
-
column: position.col,
|
|
1109
|
-
path: YamlDocument.formatPath(path)
|
|
1110
|
-
});
|
|
1111
|
-
}
|
|
1112
|
-
};
|
|
1113
1148
|
// src/yaml/yamlDocumentLoader.ts
|
|
1114
1149
|
var YamlDocumentLoader = class {
|
|
1115
1150
|
load(filePath, displayPath) {
|
|
@@ -1128,34 +1163,184 @@ var YamlDocumentLoader = class {
|
|
|
1128
1163
|
function isRecord(value) {
|
|
1129
1164
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
1130
1165
|
}
|
|
1166
|
+
// src/yaml/variablesReader.ts
|
|
1167
|
+
const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
1168
|
+
var VariablesReader = class {
|
|
1169
|
+
read(document, resolver, issues, secrets) {
|
|
1170
|
+
const raw = isRecord(document.value) ? document.value.variables : void 0;
|
|
1171
|
+
if (raw === void 0) return {};
|
|
1172
|
+
if (!isRecord(raw)) {
|
|
1173
|
+
issues.push(new ValidationIssue({
|
|
1174
|
+
message: "must be object",
|
|
1175
|
+
location: document.locate(["variables"])
|
|
1176
|
+
}));
|
|
1177
|
+
return {};
|
|
1178
|
+
}
|
|
1179
|
+
const variables = {};
|
|
1180
|
+
for (const [name, value] of Object.entries(raw)) {
|
|
1181
|
+
if (!VARIABLE_NAME.test(name)) {
|
|
1182
|
+
issues.push(new ValidationIssue({
|
|
1183
|
+
message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1184
|
+
location: document.locateKey(["variables"], name)
|
|
1185
|
+
}));
|
|
1186
|
+
continue;
|
|
1187
|
+
}
|
|
1188
|
+
if (typeof value !== "string") {
|
|
1189
|
+
issues.push(new ValidationIssue({
|
|
1190
|
+
message: "must be string",
|
|
1191
|
+
location: document.locate(["variables", name])
|
|
1192
|
+
}));
|
|
1193
|
+
continue;
|
|
1194
|
+
}
|
|
1195
|
+
const before = issues.length;
|
|
1196
|
+
const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
|
|
1197
|
+
if (issues.length > before) continue;
|
|
1198
|
+
variables[name] = resolved.variables[name];
|
|
1199
|
+
if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
|
|
1200
|
+
}
|
|
1201
|
+
return variables;
|
|
1202
|
+
}
|
|
1203
|
+
};
|
|
1204
|
+
// src/config/webSocketResolver.ts
|
|
1205
|
+
const WEBSOCKET_SCHEMA = {
|
|
1206
|
+
...eval_config_schema_default,
|
|
1207
|
+
required: ["websocket"],
|
|
1208
|
+
additionalProperties: true,
|
|
1209
|
+
properties: { websocket: eval_config_schema_default.properties.websocket }
|
|
1210
|
+
};
|
|
1211
|
+
var WebSocketResolver = class WebSocketResolver {
|
|
1212
|
+
schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
|
|
1213
|
+
resolve(document, resolver, issues, secrets, used, defer) {
|
|
1214
|
+
const raw = isRecord(document.value) ? document.value.websocket : void 0;
|
|
1215
|
+
const unresolved = /* @__PURE__ */ new Set();
|
|
1216
|
+
const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
|
|
1217
|
+
const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
|
|
1218
|
+
used,
|
|
1219
|
+
optional,
|
|
1220
|
+
unresolved
|
|
1221
|
+
});
|
|
1222
|
+
const subprotocols = WebSocketResolver.subprotocols(raw);
|
|
1223
|
+
if (subprotocols !== void 0) {
|
|
1224
|
+
const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
|
|
1225
|
+
const set = !unresolved.has(YamlDocument.formatPath([
|
|
1226
|
+
"websocket",
|
|
1227
|
+
"connection",
|
|
1228
|
+
"subprotocols",
|
|
1229
|
+
index
|
|
1230
|
+
]));
|
|
1231
|
+
const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
|
|
1232
|
+
return set && !empty;
|
|
1233
|
+
});
|
|
1234
|
+
value.websocket.connection.subprotocols = kept;
|
|
1235
|
+
}
|
|
1236
|
+
const prefix = YamlDocument.formatPath([
|
|
1237
|
+
"websocket",
|
|
1238
|
+
"connection",
|
|
1239
|
+
"subprotocols"
|
|
1240
|
+
]);
|
|
1241
|
+
return {
|
|
1242
|
+
value,
|
|
1243
|
+
deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
|
|
1244
|
+
};
|
|
1245
|
+
}
|
|
1246
|
+
validate(value, document) {
|
|
1247
|
+
const issues = this.schemaValidator.validate(value, document);
|
|
1248
|
+
if (issues.length > 0) return issues;
|
|
1249
|
+
return WebSocketResolver.validateInputPath(value, document);
|
|
1250
|
+
}
|
|
1251
|
+
static validateInputPath(value, document) {
|
|
1252
|
+
const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
|
|
1253
|
+
if (typeof request.inputPath !== "string") return [];
|
|
1254
|
+
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
1255
|
+
const last = tokens.pop();
|
|
1256
|
+
let parent = request.value;
|
|
1257
|
+
for (const token of tokens) parent = WebSocketResolver.child(parent, token);
|
|
1258
|
+
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
1259
|
+
return [new ValidationIssue({
|
|
1260
|
+
message: "must point to a missing or string value whose parent exists in request.value",
|
|
1261
|
+
location: document.locate([
|
|
1262
|
+
"websocket",
|
|
1263
|
+
"protocol",
|
|
1264
|
+
"request",
|
|
1265
|
+
"inputPath"
|
|
1266
|
+
])
|
|
1267
|
+
})];
|
|
1268
|
+
}
|
|
1269
|
+
static section(value, ...path) {
|
|
1270
|
+
let current = value;
|
|
1271
|
+
for (const key of path) current = WebSocketResolver.child(current, key);
|
|
1272
|
+
return isRecord(current) ? current : {};
|
|
1273
|
+
}
|
|
1274
|
+
static child(value, key) {
|
|
1275
|
+
if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
|
|
1276
|
+
return isRecord(value) ? value[key] : void 0;
|
|
1277
|
+
}
|
|
1278
|
+
static isSubprotocol(path) {
|
|
1279
|
+
return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
|
|
1280
|
+
}
|
|
1281
|
+
static subprotocols(websocket) {
|
|
1282
|
+
const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
|
|
1283
|
+
return Array.isArray(subprotocols) ? subprotocols : void 0;
|
|
1284
|
+
}
|
|
1285
|
+
};
|
|
1131
1286
|
// src/config/evalConfigLoader.ts
|
|
1132
1287
|
var EvalConfigLoader = class EvalConfigLoader {
|
|
1133
1288
|
cwd;
|
|
1134
1289
|
documentLoader = new YamlDocumentLoader();
|
|
1135
1290
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
1291
|
+
variablesReader = new VariablesReader();
|
|
1292
|
+
webSocketResolver = new WebSocketResolver();
|
|
1136
1293
|
environmentResolver;
|
|
1137
|
-
constructor(environment = process.env, cwd = process.cwd()) {
|
|
1294
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1138
1295
|
this.cwd = cwd;
|
|
1139
|
-
this.environmentResolver = new EnvironmentResolver(environment);
|
|
1296
|
+
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
1140
1297
|
}
|
|
1141
1298
|
load(configPath, override, report = new ReportOverride()) {
|
|
1142
1299
|
const document = this.documentLoader.load(configPath, (0, node_path.relative)(this.cwd, configPath));
|
|
1143
1300
|
const issues = [...document.errors];
|
|
1144
1301
|
if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
|
|
1145
1302
|
const named = [];
|
|
1146
|
-
const
|
|
1147
|
-
|
|
1148
|
-
if (issues.length === 0)
|
|
1303
|
+
const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
|
|
1304
|
+
let value = {};
|
|
1305
|
+
if (issues.length === 0) {
|
|
1306
|
+
const resolver = this.environmentResolver.withLevels([variables]);
|
|
1307
|
+
const { websocket: _websocket, variables: _variables, ...rest } = document.value;
|
|
1308
|
+
const resolved = resolver.resolve(rest, document, issues, named);
|
|
1309
|
+
const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
|
|
1310
|
+
value = {
|
|
1311
|
+
...resolved,
|
|
1312
|
+
..._variables === void 0 ? {} : { variables },
|
|
1313
|
+
..._websocket === void 0 ? {} : websocket.value
|
|
1314
|
+
};
|
|
1315
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
|
|
1316
|
+
if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
|
|
1317
|
+
}
|
|
1149
1318
|
if (issues.length > 0) throw new EvalError({
|
|
1150
1319
|
category: ErrorCategory.Configuration,
|
|
1151
1320
|
message: "invalid configuration",
|
|
1152
1321
|
issues
|
|
1153
1322
|
});
|
|
1154
|
-
const config = this.build(value, configPath, named);
|
|
1323
|
+
const config = this.build(value, configPath, variables, named);
|
|
1155
1324
|
this.applyRepetition(config, override, value, document);
|
|
1156
1325
|
EvalConfigLoader.applyReport(config, report);
|
|
1157
1326
|
return config;
|
|
1158
1327
|
}
|
|
1328
|
+
static buildWebSocket(value) {
|
|
1329
|
+
const section = (...path) => WebSocketResolver.section(value, ...path);
|
|
1330
|
+
const connection = section("websocket", "connection");
|
|
1331
|
+
const request = section("websocket", "protocol", "request");
|
|
1332
|
+
return new WebSocketConfig({
|
|
1333
|
+
endpoint: WebSocketResolver.section(value, "websocket").endpoint,
|
|
1334
|
+
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
1335
|
+
protocol: new ProtocolConfig({
|
|
1336
|
+
request: new RequestConfig({
|
|
1337
|
+
...request,
|
|
1338
|
+
format: request.format ?? RequestFormat.Json
|
|
1339
|
+
}),
|
|
1340
|
+
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
1341
|
+
})
|
|
1342
|
+
});
|
|
1343
|
+
}
|
|
1159
1344
|
static applyReport(config, report) {
|
|
1160
1345
|
if (report.formats === void 0) return;
|
|
1161
1346
|
const known = Object.values(ReportFormat);
|
|
@@ -1173,49 +1358,21 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1173
1358
|
const apiKey = EvalConfigLoader.section(document.value, "evaluator").apiKey;
|
|
1174
1359
|
if (typeof apiKey !== "string" || EnvironmentResolver.hasReference(apiKey)) return [];
|
|
1175
1360
|
return [new ValidationIssue({
|
|
1176
|
-
message: "must be
|
|
1361
|
+
message: "must be a reference such as ${OPENAI_API_KEY}",
|
|
1177
1362
|
location: document.locate(["evaluator", "apiKey"])
|
|
1178
1363
|
})];
|
|
1179
1364
|
}
|
|
1180
|
-
|
|
1181
|
-
const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
|
|
1182
|
-
if (typeof request.inputPath !== "string") return [];
|
|
1183
|
-
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
1184
|
-
const last = tokens.pop();
|
|
1185
|
-
let parent = request.value;
|
|
1186
|
-
for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
|
|
1187
|
-
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
1188
|
-
return [new ValidationIssue({
|
|
1189
|
-
message: "must point to a missing or string value whose parent exists in request.value",
|
|
1190
|
-
location: document.locate([
|
|
1191
|
-
"websocket",
|
|
1192
|
-
"protocol",
|
|
1193
|
-
"request",
|
|
1194
|
-
"inputPath"
|
|
1195
|
-
])
|
|
1196
|
-
})];
|
|
1197
|
-
}
|
|
1198
|
-
build(value, configPath, named) {
|
|
1365
|
+
build(value, configPath, variables, named) {
|
|
1199
1366
|
const section = (...path) => EvalConfigLoader.section(value, ...path);
|
|
1200
1367
|
const configDirectory = (0, node_path.dirname)(configPath);
|
|
1201
|
-
const
|
|
1202
|
-
const request = section("websocket", "protocol", "request");
|
|
1368
|
+
const websocket = EvalConfigLoader.buildWebSocket(value);
|
|
1203
1369
|
const evaluator = section("evaluator");
|
|
1204
1370
|
const report = section("report");
|
|
1205
1371
|
return new EvalConfig({
|
|
1206
1372
|
version: value.version,
|
|
1207
1373
|
configPath,
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
1211
|
-
protocol: new ProtocolConfig({
|
|
1212
|
-
request: new RequestConfig({
|
|
1213
|
-
...request,
|
|
1214
|
-
format: request.format ?? RequestFormat.Json
|
|
1215
|
-
}),
|
|
1216
|
-
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
1217
|
-
})
|
|
1218
|
-
}),
|
|
1374
|
+
variables,
|
|
1375
|
+
websocket,
|
|
1219
1376
|
execution: new ExecutionConfig({
|
|
1220
1377
|
...DEFAULT_CONFIG.execution,
|
|
1221
1378
|
...section("execution"),
|
|
@@ -1236,7 +1393,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1236
1393
|
directory: (0, node_path.resolve)(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
|
|
1237
1394
|
formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
|
|
1238
1395
|
}),
|
|
1239
|
-
secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols
|
|
1396
|
+
secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
|
|
1240
1397
|
});
|
|
1241
1398
|
}
|
|
1242
1399
|
applyRepetition(config, override, value, document) {
|
|
@@ -1289,13 +1446,41 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1289
1446
|
])];
|
|
1290
1447
|
}
|
|
1291
1448
|
static section(value, ...path) {
|
|
1292
|
-
|
|
1293
|
-
for (const key of path) current = EvalConfigLoader.child(current, key);
|
|
1294
|
-
return isRecord(current) ? current : {};
|
|
1449
|
+
return WebSocketResolver.section(value, ...path);
|
|
1295
1450
|
}
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1451
|
+
};
|
|
1452
|
+
// src/config/parametersLoader.ts
|
|
1453
|
+
var ParametersLoader = class ParametersLoader {
|
|
1454
|
+
cwd;
|
|
1455
|
+
constructor(cwd = process.cwd()) {
|
|
1456
|
+
this.cwd = cwd;
|
|
1457
|
+
}
|
|
1458
|
+
load(path) {
|
|
1459
|
+
const absolute = (0, node_path.resolve)(this.cwd, path);
|
|
1460
|
+
const shown = (0, node_path.relative)(this.cwd, absolute) || path;
|
|
1461
|
+
let text;
|
|
1462
|
+
let value;
|
|
1463
|
+
try {
|
|
1464
|
+
text = (0, node_fs.readFileSync)(absolute, "utf8");
|
|
1465
|
+
} catch {
|
|
1466
|
+
throw ParametersLoader.error(`parameters file not found: ${shown}`);
|
|
1467
|
+
}
|
|
1468
|
+
try {
|
|
1469
|
+
value = JSON.parse(text);
|
|
1470
|
+
} catch {
|
|
1471
|
+
throw ParametersLoader.error(`parameters file ${shown} is not valid JSON`);
|
|
1472
|
+
}
|
|
1473
|
+
if (!isRecord(value)) throw ParametersLoader.error(`parameters file ${shown} must contain a JSON object`);
|
|
1474
|
+
return {
|
|
1475
|
+
path: absolute,
|
|
1476
|
+
parameters: Object.fromEntries(Object.entries(value).filter((entry) => typeof entry[1] === "string"))
|
|
1477
|
+
};
|
|
1478
|
+
}
|
|
1479
|
+
static error(message) {
|
|
1480
|
+
return new EvalError({
|
|
1481
|
+
category: ErrorCategory.Configuration,
|
|
1482
|
+
message
|
|
1483
|
+
});
|
|
1299
1484
|
}
|
|
1300
1485
|
};
|
|
1301
1486
|
// schemas/conversation.schema.json
|
|
@@ -1313,6 +1498,11 @@ var conversation_schema_default = {
|
|
|
1313
1498
|
"version": { "const": 1 },
|
|
1314
1499
|
"name": { "$ref": "#/definitions/text" },
|
|
1315
1500
|
"description": { "type": "string" },
|
|
1501
|
+
"variables": {
|
|
1502
|
+
"type": "object",
|
|
1503
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
1504
|
+
"additionalProperties": { "type": "string" }
|
|
1505
|
+
},
|
|
1316
1506
|
"turns": {
|
|
1317
1507
|
"type": "array",
|
|
1318
1508
|
"minItems": 1,
|
|
@@ -1325,8 +1515,8 @@ var conversation_schema_default = {
|
|
|
1325
1515
|
"properties": { "conversation": {
|
|
1326
1516
|
"type": "object",
|
|
1327
1517
|
"additionalProperties": false,
|
|
1328
|
-
"required": ["
|
|
1329
|
-
"properties": { "
|
|
1518
|
+
"required": ["should"],
|
|
1519
|
+
"properties": { "should": { "$ref": "#/definitions/text" } }
|
|
1330
1520
|
} }
|
|
1331
1521
|
}
|
|
1332
1522
|
},
|
|
@@ -1369,12 +1559,12 @@ var conversation_schema_default = {
|
|
|
1369
1559
|
}
|
|
1370
1560
|
},
|
|
1371
1561
|
"response": {
|
|
1372
|
-
"description": "A string is shorthand for
|
|
1562
|
+
"description": "A string is shorthand for should.",
|
|
1373
1563
|
"type": ["string", "object"],
|
|
1374
1564
|
"pattern": "\\S",
|
|
1375
1565
|
"additionalProperties": false,
|
|
1376
1566
|
"properties": {
|
|
1377
|
-
"
|
|
1567
|
+
"should": { "$ref": "#/definitions/text" },
|
|
1378
1568
|
"contains": { "$ref": "#/definitions/textList" },
|
|
1379
1569
|
"notContains": { "$ref": "#/definitions/textList" }
|
|
1380
1570
|
}
|
|
@@ -1406,7 +1596,7 @@ var conversation_schema_default = {
|
|
|
1406
1596
|
};
|
|
1407
1597
|
// src/scenarios/conversationValidator.ts
|
|
1408
1598
|
const RESPONSE_KEYS = [
|
|
1409
|
-
"
|
|
1599
|
+
"should",
|
|
1410
1600
|
"contains",
|
|
1411
1601
|
"notContains"
|
|
1412
1602
|
];
|
|
@@ -1429,7 +1619,7 @@ var ConversationValidator = class ConversationValidator {
|
|
|
1429
1619
|
validateResponse(response, path, document, issues) {
|
|
1430
1620
|
if (!RESPONSE_KEYS.some((key) => key in response)) {
|
|
1431
1621
|
issues.push(new ValidationIssue({
|
|
1432
|
-
message: "response must have at least one of:
|
|
1622
|
+
message: "response must have at least one of: should, contains, notContains",
|
|
1433
1623
|
location: document.locate(path)
|
|
1434
1624
|
}));
|
|
1435
1625
|
return;
|
|
@@ -1472,30 +1662,66 @@ var ConversationValidator = class ConversationValidator {
|
|
|
1472
1662
|
};
|
|
1473
1663
|
// src/scenarios/scenarioLoader.ts
|
|
1474
1664
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
1475
|
-
var ScenarioLoader = class {
|
|
1665
|
+
var ScenarioLoader = class ScenarioLoader {
|
|
1476
1666
|
cwd;
|
|
1477
1667
|
documentLoader = new YamlDocumentLoader();
|
|
1478
1668
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
1479
1669
|
conversationValidator = new ConversationValidator();
|
|
1670
|
+
variablesReader = new VariablesReader();
|
|
1671
|
+
webSocketResolver = new WebSocketResolver();
|
|
1480
1672
|
environmentResolver;
|
|
1481
|
-
constructor(environment = process.env, cwd = process.cwd()) {
|
|
1673
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1482
1674
|
this.cwd = cwd;
|
|
1483
|
-
this.environmentResolver = new EnvironmentResolver(environment);
|
|
1675
|
+
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
1484
1676
|
}
|
|
1485
1677
|
load(target, config) {
|
|
1486
1678
|
const files = this.discover((0, node_path.resolve)(this.cwd, target), config);
|
|
1679
|
+
const configDocument = (0, node_fs.existsSync)(config.configPath) ? this.documentLoader.load(config.configPath, (0, node_path.relative)(this.cwd, config.configPath)) : void 0;
|
|
1487
1680
|
const issues = [];
|
|
1681
|
+
const connectionIssues = [];
|
|
1488
1682
|
const specs = [];
|
|
1489
1683
|
for (const file of files) {
|
|
1490
1684
|
const document = this.documentLoader.load(file, (0, node_path.relative)(this.cwd, file));
|
|
1491
1685
|
const fileIssues = [...document.errors];
|
|
1492
1686
|
const secrets = [];
|
|
1493
|
-
const
|
|
1687
|
+
const used = /* @__PURE__ */ new Set();
|
|
1688
|
+
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
|
|
1689
|
+
const resolver = this.environmentResolver.withLevels([variables, config.variables]);
|
|
1690
|
+
let value;
|
|
1691
|
+
if (fileIssues.length === 0) {
|
|
1692
|
+
const { variables: raw, ...rest } = document.value;
|
|
1693
|
+
value = {
|
|
1694
|
+
...resolver.resolve(rest, document, fileIssues, secrets, { used }),
|
|
1695
|
+
...raw === void 0 ? {} : { variables }
|
|
1696
|
+
};
|
|
1697
|
+
}
|
|
1698
|
+
let websocket;
|
|
1699
|
+
const websocketIssues = [];
|
|
1700
|
+
if (fileIssues.length === 0 && configDocument !== void 0) {
|
|
1701
|
+
const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
|
|
1702
|
+
if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
|
|
1703
|
+
connectionIssues.push({
|
|
1704
|
+
file: document,
|
|
1705
|
+
issues: websocketIssues
|
|
1706
|
+
});
|
|
1707
|
+
if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
|
|
1708
|
+
}
|
|
1494
1709
|
if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
|
|
1495
1710
|
if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
|
|
1496
|
-
if (fileIssues.length === 0)
|
|
1711
|
+
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
|
|
1712
|
+
if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
1497
1713
|
else issues.push(...fileIssues);
|
|
1498
1714
|
}
|
|
1715
|
+
const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
|
|
1716
|
+
if (shared.length > 0) throw new EvalError({
|
|
1717
|
+
category: ErrorCategory.Configuration,
|
|
1718
|
+
message: "invalid configuration",
|
|
1719
|
+
issues: shared
|
|
1720
|
+
});
|
|
1721
|
+
for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
|
|
1722
|
+
message: issue.toString(),
|
|
1723
|
+
location: ScenarioLoader.variablesLocation(entry.file)
|
|
1724
|
+
}));
|
|
1499
1725
|
if (issues.length > 0) throw new EvalError({
|
|
1500
1726
|
category: ErrorCategory.Specification,
|
|
1501
1727
|
message: "invalid scenario specification",
|
|
@@ -1503,6 +1729,23 @@ var ScenarioLoader = class {
|
|
|
1503
1729
|
});
|
|
1504
1730
|
return specs;
|
|
1505
1731
|
}
|
|
1732
|
+
static shared(lists) {
|
|
1733
|
+
if (lists.length === 0) return [];
|
|
1734
|
+
const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
|
|
1735
|
+
return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
|
|
1736
|
+
}
|
|
1737
|
+
static variablesLocation(document) {
|
|
1738
|
+
return new SourceLocation({
|
|
1739
|
+
...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
|
|
1740
|
+
path: ""
|
|
1741
|
+
});
|
|
1742
|
+
}
|
|
1743
|
+
static unusedVariables(variables, used, document) {
|
|
1744
|
+
return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
|
|
1745
|
+
message: `variable '${name}' is not used by the scenario or the websocket configuration`,
|
|
1746
|
+
location: document.locateKey(["variables"], name)
|
|
1747
|
+
}));
|
|
1748
|
+
}
|
|
1506
1749
|
discover(target, config) {
|
|
1507
1750
|
if (!(0, node_fs.existsSync)(target)) throw new EvalError({
|
|
1508
1751
|
category: ErrorCategory.Specification,
|
|
@@ -1527,7 +1770,7 @@ var ScenarioLoader = class {
|
|
|
1527
1770
|
}
|
|
1528
1771
|
return files;
|
|
1529
1772
|
}
|
|
1530
|
-
build(value, document, secrets) {
|
|
1773
|
+
build(value, document, websocket, secrets) {
|
|
1531
1774
|
const scenario = value;
|
|
1532
1775
|
const turns = scenario.turns;
|
|
1533
1776
|
const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
|
|
@@ -1537,10 +1780,11 @@ var ScenarioLoader = class {
|
|
|
1537
1780
|
description: typeof scenario.description === "string" ? scenario.description : "",
|
|
1538
1781
|
turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
|
|
1539
1782
|
conversation: conversation ? new ConversationExpectation({
|
|
1540
|
-
criteria: conversation.
|
|
1783
|
+
criteria: conversation.should,
|
|
1541
1784
|
location: document.locate(["expect", "conversation"])
|
|
1542
1785
|
}) : void 0,
|
|
1543
1786
|
location: document.locate([]),
|
|
1787
|
+
websocket,
|
|
1544
1788
|
secrets
|
|
1545
1789
|
});
|
|
1546
1790
|
}
|
|
@@ -1566,7 +1810,7 @@ var ScenarioLoader = class {
|
|
|
1566
1810
|
});
|
|
1567
1811
|
const fields = response;
|
|
1568
1812
|
return new ResponseExpectation({
|
|
1569
|
-
criteria: fields.
|
|
1813
|
+
criteria: fields.should,
|
|
1570
1814
|
contains: fields.contains ?? [],
|
|
1571
1815
|
notContains: fields.notContains ?? [],
|
|
1572
1816
|
location
|
|
@@ -2389,7 +2633,11 @@ var ConversationRunner = class {
|
|
|
2389
2633
|
async run(spec, signal = NEVER_ABORTED) {
|
|
2390
2634
|
const startedAt = Date.now();
|
|
2391
2635
|
const outcomes = [];
|
|
2392
|
-
const
|
|
2636
|
+
const config = new EvalConfig({
|
|
2637
|
+
...this.config,
|
|
2638
|
+
websocket: spec.websocket ?? this.config.websocket
|
|
2639
|
+
});
|
|
2640
|
+
const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
|
|
2393
2641
|
const history = [];
|
|
2394
2642
|
const evaluate = this.semanticEvaluator(signal);
|
|
2395
2643
|
const inline = !this.config.execution.continueOnFailure;
|
|
@@ -2414,7 +2662,7 @@ var ConversationRunner = class {
|
|
|
2414
2662
|
try {
|
|
2415
2663
|
const steps = (async () => {
|
|
2416
2664
|
for (let attempt = 0;; attempt++) {
|
|
2417
|
-
connection = this.createConnection(
|
|
2665
|
+
connection = this.createConnection(config);
|
|
2418
2666
|
connectionAttempts++;
|
|
2419
2667
|
try {
|
|
2420
2668
|
await connection.connect();
|
|
@@ -2908,7 +3156,7 @@ var ScenarioRunner = class {
|
|
|
2908
3156
|
required: minPass,
|
|
2909
3157
|
attempts,
|
|
2910
3158
|
tests: this.results.tests(attempts, minPass),
|
|
2911
|
-
agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
|
|
3159
|
+
agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
|
|
2912
3160
|
evaluatorUsage: this.tokens.evaluator(attempts)
|
|
2913
3161
|
});
|
|
2914
3162
|
}
|
|
@@ -2999,7 +3247,9 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
2999
3247
|
const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
|
|
3000
3248
|
if (!this.verbose) return line;
|
|
3001
3249
|
if (result.attempts.length === 0) return `${line}\n`;
|
|
3002
|
-
const
|
|
3250
|
+
const websocket = spec.websocket;
|
|
3251
|
+
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
3252
|
+
const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
|
|
3003
3253
|
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
3004
3254
|
lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
|
|
3005
3255
|
lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
|
|
@@ -3179,6 +3429,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3179
3429
|
const { websocket, execution, evaluator, report } = config;
|
|
3180
3430
|
return {
|
|
3181
3431
|
path: this.path(config.configPath),
|
|
3432
|
+
parametersPath: config.parametersPath === "" ? void 0 : this.path(config.parametersPath),
|
|
3182
3433
|
websocket: {
|
|
3183
3434
|
endpoint: websocket.endpoint,
|
|
3184
3435
|
subprotocols: [...websocket.connection.subprotocols],
|
|
@@ -3211,11 +3462,16 @@ var JsonReporter = class JsonReporter {
|
|
|
3211
3462
|
};
|
|
3212
3463
|
}
|
|
3213
3464
|
scenario(result, config) {
|
|
3214
|
-
const
|
|
3465
|
+
const websocket = result.spec.websocket ?? config.websocket;
|
|
3466
|
+
const capability = websocket.protocol.capabilities.tokenUsage;
|
|
3215
3467
|
return {
|
|
3216
3468
|
name: result.spec.name,
|
|
3217
3469
|
file: result.spec.location.file,
|
|
3218
3470
|
status: result.status,
|
|
3471
|
+
connection: {
|
|
3472
|
+
endpoint: websocket.endpoint,
|
|
3473
|
+
subprotocols: websocket.connection.subprotocols.length
|
|
3474
|
+
},
|
|
3219
3475
|
...JsonReporter.counts(result),
|
|
3220
3476
|
agentUsage: JsonReporter.usage(result.agentUsage),
|
|
3221
3477
|
evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
|
|
@@ -3406,6 +3662,7 @@ exports.JSON_REPORT_FILE = JSON_REPORT_FILE;
|
|
|
3406
3662
|
exports.JSON_REPORT_VERSION = JSON_REPORT_VERSION;
|
|
3407
3663
|
exports.JsonReporter = JsonReporter;
|
|
3408
3664
|
exports.LlmEvaluator = LlmEvaluator;
|
|
3665
|
+
exports.ParametersLoader = ParametersLoader;
|
|
3409
3666
|
exports.ProtocolCapabilities = ProtocolCapabilities;
|
|
3410
3667
|
exports.ProtocolConfig = ProtocolConfig;
|
|
3411
3668
|
exports.REDACTED = REDACTED;
|