@mappies/miniwat-eval 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -13
- package/dist/index.cjs +435 -178
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +72 -51
- package/dist/index.d.ts +72 -51
- package/dist/index.js +434 -178
- package/dist/index.js.map +1 -1
- package/dist/main.js +410 -148
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +9 -4
- package/schemas/eval-config.schema.json +5 -0
- package/schemas/report.schema.json +12 -1
package/dist/index.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { Message, ResponseMessage, TokenUsage } from "@mappies/miniwat-agent-core";
|
|
2
2
|
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
3
3
|
import { dirname, extname, join, relative, resolve } from "node:path";
|
|
4
|
-
import { Ajv } from "ajv";
|
|
5
4
|
import { LineCounter, isMap, isPair, isScalar, isSeq, parseDocument } from "yaml";
|
|
5
|
+
import { Ajv } from "ajv";
|
|
6
6
|
import handlerWs from "configurapi-handler-ws";
|
|
7
7
|
import { isDeepStrictEqual, styleText } from "node:util";
|
|
8
8
|
import { Agent, InMemoryAgentStateStore } from "@mappies/miniwat-agent";
|
|
@@ -273,6 +273,50 @@ var TurnSpec = class {
|
|
|
273
273
|
this.location = new SourceLocation(data?.location);
|
|
274
274
|
}
|
|
275
275
|
};
|
|
276
|
+
// src/entities/protocolCapabilities.ts
|
|
277
|
+
var ProtocolCapabilities = class {
|
|
278
|
+
toolUses;
|
|
279
|
+
toolResults;
|
|
280
|
+
tokenUsage;
|
|
281
|
+
constructor(data) {
|
|
282
|
+
this.toolUses = data?.toolUses ?? false;
|
|
283
|
+
this.toolResults = data?.toolResults ?? false;
|
|
284
|
+
this.tokenUsage = data?.tokenUsage ?? false;
|
|
285
|
+
}
|
|
286
|
+
};
|
|
287
|
+
// src/interfaces/requestFormat.ts
|
|
288
|
+
const RequestFormat = { Json: "json" };
|
|
289
|
+
// src/entities/requestConfig.ts
|
|
290
|
+
var RequestConfig = class {
|
|
291
|
+
format;
|
|
292
|
+
value;
|
|
293
|
+
inputPath;
|
|
294
|
+
constructor(data) {
|
|
295
|
+
this.format = data?.format ?? RequestFormat.Json;
|
|
296
|
+
this.value = data?.value;
|
|
297
|
+
this.inputPath = data?.inputPath;
|
|
298
|
+
}
|
|
299
|
+
};
|
|
300
|
+
// src/entities/protocolConfig.ts
|
|
301
|
+
var ProtocolConfig = class {
|
|
302
|
+
request;
|
|
303
|
+
capabilities;
|
|
304
|
+
constructor(data) {
|
|
305
|
+
this.request = new RequestConfig(data?.request);
|
|
306
|
+
this.capabilities = new ProtocolCapabilities(data?.capabilities);
|
|
307
|
+
}
|
|
308
|
+
};
|
|
309
|
+
// src/entities/webSocketConfig.ts
|
|
310
|
+
var WebSocketConfig = class {
|
|
311
|
+
endpoint;
|
|
312
|
+
connection;
|
|
313
|
+
protocol;
|
|
314
|
+
constructor(data) {
|
|
315
|
+
this.endpoint = data?.endpoint ?? "";
|
|
316
|
+
this.connection = new ConnectionConfig(data?.connection);
|
|
317
|
+
this.protocol = new ProtocolConfig(data?.protocol);
|
|
318
|
+
}
|
|
319
|
+
};
|
|
276
320
|
// src/entities/conversationSpec.ts
|
|
277
321
|
var ConversationSpec = class {
|
|
278
322
|
version;
|
|
@@ -281,6 +325,7 @@ var ConversationSpec = class {
|
|
|
281
325
|
turns;
|
|
282
326
|
conversation;
|
|
283
327
|
location;
|
|
328
|
+
websocket;
|
|
284
329
|
secrets;
|
|
285
330
|
constructor(data) {
|
|
286
331
|
this.version = data?.version ?? 1;
|
|
@@ -289,6 +334,7 @@ var ConversationSpec = class {
|
|
|
289
334
|
this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
|
|
290
335
|
this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
|
|
291
336
|
this.location = new SourceLocation(data?.location);
|
|
337
|
+
this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
|
|
292
338
|
this.secrets = [...data?.secrets ?? []];
|
|
293
339
|
}
|
|
294
340
|
};
|
|
@@ -390,54 +436,12 @@ var ReportConfig = class {
|
|
|
390
436
|
this.formats = data?.formats ?? [];
|
|
391
437
|
}
|
|
392
438
|
};
|
|
393
|
-
// src/entities/protocolCapabilities.ts
|
|
394
|
-
var ProtocolCapabilities = class {
|
|
395
|
-
toolUses;
|
|
396
|
-
toolResults;
|
|
397
|
-
tokenUsage;
|
|
398
|
-
constructor(data) {
|
|
399
|
-
this.toolUses = data?.toolUses ?? false;
|
|
400
|
-
this.toolResults = data?.toolResults ?? false;
|
|
401
|
-
this.tokenUsage = data?.tokenUsage ?? false;
|
|
402
|
-
}
|
|
403
|
-
};
|
|
404
|
-
// src/interfaces/requestFormat.ts
|
|
405
|
-
const RequestFormat = { Json: "json" };
|
|
406
|
-
// src/entities/requestConfig.ts
|
|
407
|
-
var RequestConfig = class {
|
|
408
|
-
format;
|
|
409
|
-
value;
|
|
410
|
-
inputPath;
|
|
411
|
-
constructor(data) {
|
|
412
|
-
this.format = data?.format ?? RequestFormat.Json;
|
|
413
|
-
this.value = data?.value;
|
|
414
|
-
this.inputPath = data?.inputPath;
|
|
415
|
-
}
|
|
416
|
-
};
|
|
417
|
-
// src/entities/protocolConfig.ts
|
|
418
|
-
var ProtocolConfig = class {
|
|
419
|
-
request;
|
|
420
|
-
capabilities;
|
|
421
|
-
constructor(data) {
|
|
422
|
-
this.request = new RequestConfig(data?.request);
|
|
423
|
-
this.capabilities = new ProtocolCapabilities(data?.capabilities);
|
|
424
|
-
}
|
|
425
|
-
};
|
|
426
|
-
// src/entities/webSocketConfig.ts
|
|
427
|
-
var WebSocketConfig = class {
|
|
428
|
-
endpoint;
|
|
429
|
-
connection;
|
|
430
|
-
protocol;
|
|
431
|
-
constructor(data) {
|
|
432
|
-
this.endpoint = data?.endpoint ?? "";
|
|
433
|
-
this.connection = new ConnectionConfig(data?.connection);
|
|
434
|
-
this.protocol = new ProtocolConfig(data?.protocol);
|
|
435
|
-
}
|
|
436
|
-
};
|
|
437
439
|
// src/entities/evalConfig.ts
|
|
438
440
|
var EvalConfig = class {
|
|
439
441
|
version;
|
|
440
442
|
configPath;
|
|
443
|
+
parametersPath;
|
|
444
|
+
variables;
|
|
441
445
|
websocket;
|
|
442
446
|
execution;
|
|
443
447
|
evaluator;
|
|
@@ -446,6 +450,8 @@ var EvalConfig = class {
|
|
|
446
450
|
constructor(data) {
|
|
447
451
|
this.version = data?.version ?? 1;
|
|
448
452
|
this.configPath = data?.configPath ?? "";
|
|
453
|
+
this.parametersPath = data?.parametersPath ?? "";
|
|
454
|
+
this.variables = { ...data?.variables ?? {} };
|
|
449
455
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
450
456
|
this.execution = new ExecutionConfig(data?.execution);
|
|
451
457
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -757,6 +763,11 @@ var eval_config_schema_default = {
|
|
|
757
763
|
],
|
|
758
764
|
properties: {
|
|
759
765
|
"version": { "const": 1 },
|
|
766
|
+
"variables": {
|
|
767
|
+
"type": "object",
|
|
768
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
769
|
+
"additionalProperties": { "type": "string" }
|
|
770
|
+
},
|
|
760
771
|
"websocket": {
|
|
761
772
|
"type": "object",
|
|
762
773
|
"additionalProperties": false,
|
|
@@ -915,16 +926,80 @@ var eval_config_schema_default = {
|
|
|
915
926
|
}
|
|
916
927
|
}
|
|
917
928
|
};
|
|
929
|
+
// src/yaml/yamlDocument.ts
|
|
930
|
+
var YamlDocument = class YamlDocument {
|
|
931
|
+
file;
|
|
932
|
+
document;
|
|
933
|
+
lineCounter;
|
|
934
|
+
value;
|
|
935
|
+
errors;
|
|
936
|
+
constructor(file, document, lineCounter) {
|
|
937
|
+
this.file = file;
|
|
938
|
+
this.document = document;
|
|
939
|
+
this.lineCounter = lineCounter;
|
|
940
|
+
this.value = document.toJS();
|
|
941
|
+
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
942
|
+
message: error.message.split("\n")[0],
|
|
943
|
+
location: this.toLocation(error.pos[0], [])
|
|
944
|
+
}));
|
|
945
|
+
}
|
|
946
|
+
locate(path) {
|
|
947
|
+
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
948
|
+
}
|
|
949
|
+
locateKey(path, key) {
|
|
950
|
+
const parent = this.findNode(path);
|
|
951
|
+
if (isMap(parent)) {
|
|
952
|
+
const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
|
|
953
|
+
if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
954
|
+
}
|
|
955
|
+
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
956
|
+
}
|
|
957
|
+
static formatPath(path) {
|
|
958
|
+
return path.map((segment, index) => {
|
|
959
|
+
if (typeof segment === "number") return `[${segment}]`;
|
|
960
|
+
return index === 0 ? segment : `.${segment}`;
|
|
961
|
+
}).join("");
|
|
962
|
+
}
|
|
963
|
+
findNode(path) {
|
|
964
|
+
let node = this.document.contents ?? void 0;
|
|
965
|
+
for (const segment of path) {
|
|
966
|
+
let next;
|
|
967
|
+
if (isMap(node)) {
|
|
968
|
+
const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
|
|
969
|
+
next = pair?.value ?? pair?.key;
|
|
970
|
+
} else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
|
|
971
|
+
if (!next) break;
|
|
972
|
+
node = next;
|
|
973
|
+
}
|
|
974
|
+
return node;
|
|
975
|
+
}
|
|
976
|
+
toLocation(offset, path) {
|
|
977
|
+
const position = this.lineCounter.linePos(offset);
|
|
978
|
+
return new SourceLocation({
|
|
979
|
+
file: this.file,
|
|
980
|
+
line: position.line,
|
|
981
|
+
column: position.col,
|
|
982
|
+
path: YamlDocument.formatPath(path)
|
|
983
|
+
});
|
|
984
|
+
}
|
|
985
|
+
};
|
|
918
986
|
// src/yaml/environmentResolver.ts
|
|
919
|
-
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g;
|
|
987
|
+
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
|
|
920
988
|
const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
|
|
921
989
|
var EnvironmentResolver = class EnvironmentResolver {
|
|
922
990
|
environment;
|
|
923
|
-
|
|
991
|
+
parameters;
|
|
992
|
+
levels;
|
|
993
|
+
constructor(environment = process.env, parameters = {}, levels = []) {
|
|
924
994
|
this.environment = environment;
|
|
995
|
+
this.parameters = parameters;
|
|
996
|
+
this.levels = levels;
|
|
925
997
|
}
|
|
926
|
-
|
|
927
|
-
return this.
|
|
998
|
+
withLevels(levels) {
|
|
999
|
+
return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
|
|
1000
|
+
}
|
|
1001
|
+
resolve(value, document, issues, secrets = [], options = {}) {
|
|
1002
|
+
return this.resolveValue(value, [], document, issues, secrets, options);
|
|
928
1003
|
}
|
|
929
1004
|
static hasReference(value) {
|
|
930
1005
|
return new RegExp(REFERENCE.source).test(value);
|
|
@@ -932,21 +1007,38 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
932
1007
|
static isSecretName(name) {
|
|
933
1008
|
return SECRET_NAME.test(name);
|
|
934
1009
|
}
|
|
935
|
-
|
|
1010
|
+
lookup(name) {
|
|
1011
|
+
for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
|
|
1012
|
+
return this.environment[name];
|
|
1013
|
+
}
|
|
1014
|
+
sources() {
|
|
1015
|
+
const sources = [
|
|
1016
|
+
...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
|
|
1017
|
+
...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
|
|
1018
|
+
"environment variable"
|
|
1019
|
+
];
|
|
1020
|
+
return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
|
|
1021
|
+
}
|
|
1022
|
+
resolveValue(value, path, document, issues, secrets, options) {
|
|
936
1023
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
937
|
-
|
|
1024
|
+
options.used?.add(name);
|
|
1025
|
+
const resolved = this.lookup(name);
|
|
938
1026
|
if (resolved === void 0) {
|
|
1027
|
+
if (options.optional?.(path)) {
|
|
1028
|
+
options.unresolved?.add(YamlDocument.formatPath(path));
|
|
1029
|
+
return reference;
|
|
1030
|
+
}
|
|
939
1031
|
issues.push(new ValidationIssue({
|
|
940
|
-
message:
|
|
1032
|
+
message: `${this.sources()} '${name}' is not set`,
|
|
941
1033
|
location: document.locate(path)
|
|
942
1034
|
}));
|
|
943
1035
|
return reference;
|
|
944
1036
|
}
|
|
945
|
-
if (EnvironmentResolver.isSecretName(name) && !secrets.includes(resolved)) secrets.push(resolved);
|
|
1037
|
+
if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
|
|
946
1038
|
return resolved;
|
|
947
1039
|
});
|
|
948
|
-
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
|
|
949
|
-
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
|
|
1040
|
+
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
|
|
1041
|
+
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
|
|
950
1042
|
return value;
|
|
951
1043
|
}
|
|
952
1044
|
};
|
|
@@ -1030,63 +1122,6 @@ var SchemaValidator = class SchemaValidator {
|
|
|
1030
1122
|
return path;
|
|
1031
1123
|
}
|
|
1032
1124
|
};
|
|
1033
|
-
// src/yaml/yamlDocument.ts
|
|
1034
|
-
var YamlDocument = class YamlDocument {
|
|
1035
|
-
file;
|
|
1036
|
-
document;
|
|
1037
|
-
lineCounter;
|
|
1038
|
-
value;
|
|
1039
|
-
errors;
|
|
1040
|
-
constructor(file, document, lineCounter) {
|
|
1041
|
-
this.file = file;
|
|
1042
|
-
this.document = document;
|
|
1043
|
-
this.lineCounter = lineCounter;
|
|
1044
|
-
this.value = document.toJS();
|
|
1045
|
-
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
1046
|
-
message: error.message.split("\n")[0],
|
|
1047
|
-
location: this.toLocation(error.pos[0], [])
|
|
1048
|
-
}));
|
|
1049
|
-
}
|
|
1050
|
-
locate(path) {
|
|
1051
|
-
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
1052
|
-
}
|
|
1053
|
-
locateKey(path, key) {
|
|
1054
|
-
const parent = this.findNode(path);
|
|
1055
|
-
if (isMap(parent)) {
|
|
1056
|
-
const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
|
|
1057
|
-
if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
1058
|
-
}
|
|
1059
|
-
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
1060
|
-
}
|
|
1061
|
-
static formatPath(path) {
|
|
1062
|
-
return path.map((segment, index) => {
|
|
1063
|
-
if (typeof segment === "number") return `[${segment}]`;
|
|
1064
|
-
return index === 0 ? segment : `.${segment}`;
|
|
1065
|
-
}).join("");
|
|
1066
|
-
}
|
|
1067
|
-
findNode(path) {
|
|
1068
|
-
let node = this.document.contents ?? void 0;
|
|
1069
|
-
for (const segment of path) {
|
|
1070
|
-
let next;
|
|
1071
|
-
if (isMap(node)) {
|
|
1072
|
-
const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
|
|
1073
|
-
next = pair?.value ?? pair?.key;
|
|
1074
|
-
} else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
|
|
1075
|
-
if (!next) break;
|
|
1076
|
-
node = next;
|
|
1077
|
-
}
|
|
1078
|
-
return node;
|
|
1079
|
-
}
|
|
1080
|
-
toLocation(offset, path) {
|
|
1081
|
-
const position = this.lineCounter.linePos(offset);
|
|
1082
|
-
return new SourceLocation({
|
|
1083
|
-
file: this.file,
|
|
1084
|
-
line: position.line,
|
|
1085
|
-
column: position.col,
|
|
1086
|
-
path: YamlDocument.formatPath(path)
|
|
1087
|
-
});
|
|
1088
|
-
}
|
|
1089
|
-
};
|
|
1090
1125
|
// src/yaml/yamlDocumentLoader.ts
|
|
1091
1126
|
var YamlDocumentLoader = class {
|
|
1092
1127
|
load(filePath, displayPath) {
|
|
@@ -1105,34 +1140,184 @@ var YamlDocumentLoader = class {
|
|
|
1105
1140
|
function isRecord(value) {
|
|
1106
1141
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
1107
1142
|
}
|
|
1143
|
+
// src/yaml/variablesReader.ts
|
|
1144
|
+
const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
1145
|
+
var VariablesReader = class {
|
|
1146
|
+
read(document, resolver, issues, secrets) {
|
|
1147
|
+
const raw = isRecord(document.value) ? document.value.variables : void 0;
|
|
1148
|
+
if (raw === void 0) return {};
|
|
1149
|
+
if (!isRecord(raw)) {
|
|
1150
|
+
issues.push(new ValidationIssue({
|
|
1151
|
+
message: "must be object",
|
|
1152
|
+
location: document.locate(["variables"])
|
|
1153
|
+
}));
|
|
1154
|
+
return {};
|
|
1155
|
+
}
|
|
1156
|
+
const variables = {};
|
|
1157
|
+
for (const [name, value] of Object.entries(raw)) {
|
|
1158
|
+
if (!VARIABLE_NAME.test(name)) {
|
|
1159
|
+
issues.push(new ValidationIssue({
|
|
1160
|
+
message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
1161
|
+
location: document.locateKey(["variables"], name)
|
|
1162
|
+
}));
|
|
1163
|
+
continue;
|
|
1164
|
+
}
|
|
1165
|
+
if (typeof value !== "string") {
|
|
1166
|
+
issues.push(new ValidationIssue({
|
|
1167
|
+
message: "must be string",
|
|
1168
|
+
location: document.locate(["variables", name])
|
|
1169
|
+
}));
|
|
1170
|
+
continue;
|
|
1171
|
+
}
|
|
1172
|
+
const before = issues.length;
|
|
1173
|
+
const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
|
|
1174
|
+
if (issues.length > before) continue;
|
|
1175
|
+
variables[name] = resolved.variables[name];
|
|
1176
|
+
if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
|
|
1177
|
+
}
|
|
1178
|
+
return variables;
|
|
1179
|
+
}
|
|
1180
|
+
};
|
|
1181
|
+
// src/config/webSocketResolver.ts
|
|
1182
|
+
const WEBSOCKET_SCHEMA = {
|
|
1183
|
+
...eval_config_schema_default,
|
|
1184
|
+
required: ["websocket"],
|
|
1185
|
+
additionalProperties: true,
|
|
1186
|
+
properties: { websocket: eval_config_schema_default.properties.websocket }
|
|
1187
|
+
};
|
|
1188
|
+
var WebSocketResolver = class WebSocketResolver {
|
|
1189
|
+
schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
|
|
1190
|
+
resolve(document, resolver, issues, secrets, used, defer) {
|
|
1191
|
+
const raw = isRecord(document.value) ? document.value.websocket : void 0;
|
|
1192
|
+
const unresolved = /* @__PURE__ */ new Set();
|
|
1193
|
+
const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
|
|
1194
|
+
const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
|
|
1195
|
+
used,
|
|
1196
|
+
optional,
|
|
1197
|
+
unresolved
|
|
1198
|
+
});
|
|
1199
|
+
const subprotocols = WebSocketResolver.subprotocols(raw);
|
|
1200
|
+
if (subprotocols !== void 0) {
|
|
1201
|
+
const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
|
|
1202
|
+
const set = !unresolved.has(YamlDocument.formatPath([
|
|
1203
|
+
"websocket",
|
|
1204
|
+
"connection",
|
|
1205
|
+
"subprotocols",
|
|
1206
|
+
index
|
|
1207
|
+
]));
|
|
1208
|
+
const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
|
|
1209
|
+
return set && !empty;
|
|
1210
|
+
});
|
|
1211
|
+
value.websocket.connection.subprotocols = kept;
|
|
1212
|
+
}
|
|
1213
|
+
const prefix = YamlDocument.formatPath([
|
|
1214
|
+
"websocket",
|
|
1215
|
+
"connection",
|
|
1216
|
+
"subprotocols"
|
|
1217
|
+
]);
|
|
1218
|
+
return {
|
|
1219
|
+
value,
|
|
1220
|
+
deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
|
|
1221
|
+
};
|
|
1222
|
+
}
|
|
1223
|
+
validate(value, document) {
|
|
1224
|
+
const issues = this.schemaValidator.validate(value, document);
|
|
1225
|
+
if (issues.length > 0) return issues;
|
|
1226
|
+
return WebSocketResolver.validateInputPath(value, document);
|
|
1227
|
+
}
|
|
1228
|
+
static validateInputPath(value, document) {
|
|
1229
|
+
const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
|
|
1230
|
+
if (typeof request.inputPath !== "string") return [];
|
|
1231
|
+
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
1232
|
+
const last = tokens.pop();
|
|
1233
|
+
let parent = request.value;
|
|
1234
|
+
for (const token of tokens) parent = WebSocketResolver.child(parent, token);
|
|
1235
|
+
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
1236
|
+
return [new ValidationIssue({
|
|
1237
|
+
message: "must point to a missing or string value whose parent exists in request.value",
|
|
1238
|
+
location: document.locate([
|
|
1239
|
+
"websocket",
|
|
1240
|
+
"protocol",
|
|
1241
|
+
"request",
|
|
1242
|
+
"inputPath"
|
|
1243
|
+
])
|
|
1244
|
+
})];
|
|
1245
|
+
}
|
|
1246
|
+
static section(value, ...path) {
|
|
1247
|
+
let current = value;
|
|
1248
|
+
for (const key of path) current = WebSocketResolver.child(current, key);
|
|
1249
|
+
return isRecord(current) ? current : {};
|
|
1250
|
+
}
|
|
1251
|
+
static child(value, key) {
|
|
1252
|
+
if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
|
|
1253
|
+
return isRecord(value) ? value[key] : void 0;
|
|
1254
|
+
}
|
|
1255
|
+
static isSubprotocol(path) {
|
|
1256
|
+
return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
|
|
1257
|
+
}
|
|
1258
|
+
static subprotocols(websocket) {
|
|
1259
|
+
const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
|
|
1260
|
+
return Array.isArray(subprotocols) ? subprotocols : void 0;
|
|
1261
|
+
}
|
|
1262
|
+
};
|
|
1108
1263
|
// src/config/evalConfigLoader.ts
|
|
1109
1264
|
var EvalConfigLoader = class EvalConfigLoader {
|
|
1110
1265
|
cwd;
|
|
1111
1266
|
documentLoader = new YamlDocumentLoader();
|
|
1112
1267
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
1268
|
+
variablesReader = new VariablesReader();
|
|
1269
|
+
webSocketResolver = new WebSocketResolver();
|
|
1113
1270
|
environmentResolver;
|
|
1114
|
-
constructor(environment = process.env, cwd = process.cwd()) {
|
|
1271
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1115
1272
|
this.cwd = cwd;
|
|
1116
|
-
this.environmentResolver = new EnvironmentResolver(environment);
|
|
1273
|
+
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
1117
1274
|
}
|
|
1118
1275
|
load(configPath, override, report = new ReportOverride()) {
|
|
1119
1276
|
const document = this.documentLoader.load(configPath, relative(this.cwd, configPath));
|
|
1120
1277
|
const issues = [...document.errors];
|
|
1121
1278
|
if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
|
|
1122
1279
|
const named = [];
|
|
1123
|
-
const
|
|
1124
|
-
|
|
1125
|
-
if (issues.length === 0)
|
|
1280
|
+
const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
|
|
1281
|
+
let value = {};
|
|
1282
|
+
if (issues.length === 0) {
|
|
1283
|
+
const resolver = this.environmentResolver.withLevels([variables]);
|
|
1284
|
+
const { websocket: _websocket, variables: _variables, ...rest } = document.value;
|
|
1285
|
+
const resolved = resolver.resolve(rest, document, issues, named);
|
|
1286
|
+
const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
|
|
1287
|
+
value = {
|
|
1288
|
+
...resolved,
|
|
1289
|
+
..._variables === void 0 ? {} : { variables },
|
|
1290
|
+
..._websocket === void 0 ? {} : websocket.value
|
|
1291
|
+
};
|
|
1292
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
|
|
1293
|
+
if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
|
|
1294
|
+
}
|
|
1126
1295
|
if (issues.length > 0) throw new EvalError({
|
|
1127
1296
|
category: ErrorCategory.Configuration,
|
|
1128
1297
|
message: "invalid configuration",
|
|
1129
1298
|
issues
|
|
1130
1299
|
});
|
|
1131
|
-
const config = this.build(value, configPath, named);
|
|
1300
|
+
const config = this.build(value, configPath, variables, named);
|
|
1132
1301
|
this.applyRepetition(config, override, value, document);
|
|
1133
1302
|
EvalConfigLoader.applyReport(config, report);
|
|
1134
1303
|
return config;
|
|
1135
1304
|
}
|
|
1305
|
+
static buildWebSocket(value) {
|
|
1306
|
+
const section = (...path) => WebSocketResolver.section(value, ...path);
|
|
1307
|
+
const connection = section("websocket", "connection");
|
|
1308
|
+
const request = section("websocket", "protocol", "request");
|
|
1309
|
+
return new WebSocketConfig({
|
|
1310
|
+
endpoint: WebSocketResolver.section(value, "websocket").endpoint,
|
|
1311
|
+
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
1312
|
+
protocol: new ProtocolConfig({
|
|
1313
|
+
request: new RequestConfig({
|
|
1314
|
+
...request,
|
|
1315
|
+
format: request.format ?? RequestFormat.Json
|
|
1316
|
+
}),
|
|
1317
|
+
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
1318
|
+
})
|
|
1319
|
+
});
|
|
1320
|
+
}
|
|
1136
1321
|
static applyReport(config, report) {
|
|
1137
1322
|
if (report.formats === void 0) return;
|
|
1138
1323
|
const known = Object.values(ReportFormat);
|
|
@@ -1150,49 +1335,21 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1150
1335
|
const apiKey = EvalConfigLoader.section(document.value, "evaluator").apiKey;
|
|
1151
1336
|
if (typeof apiKey !== "string" || EnvironmentResolver.hasReference(apiKey)) return [];
|
|
1152
1337
|
return [new ValidationIssue({
|
|
1153
|
-
message: "must be
|
|
1338
|
+
message: "must be a reference such as ${OPENAI_API_KEY}",
|
|
1154
1339
|
location: document.locate(["evaluator", "apiKey"])
|
|
1155
1340
|
})];
|
|
1156
1341
|
}
|
|
1157
|
-
|
|
1158
|
-
const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
|
|
1159
|
-
if (typeof request.inputPath !== "string") return [];
|
|
1160
|
-
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
1161
|
-
const last = tokens.pop();
|
|
1162
|
-
let parent = request.value;
|
|
1163
|
-
for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
|
|
1164
|
-
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
1165
|
-
return [new ValidationIssue({
|
|
1166
|
-
message: "must point to a missing or string value whose parent exists in request.value",
|
|
1167
|
-
location: document.locate([
|
|
1168
|
-
"websocket",
|
|
1169
|
-
"protocol",
|
|
1170
|
-
"request",
|
|
1171
|
-
"inputPath"
|
|
1172
|
-
])
|
|
1173
|
-
})];
|
|
1174
|
-
}
|
|
1175
|
-
build(value, configPath, named) {
|
|
1342
|
+
build(value, configPath, variables, named) {
|
|
1176
1343
|
const section = (...path) => EvalConfigLoader.section(value, ...path);
|
|
1177
1344
|
const configDirectory = dirname(configPath);
|
|
1178
|
-
const
|
|
1179
|
-
const request = section("websocket", "protocol", "request");
|
|
1345
|
+
const websocket = EvalConfigLoader.buildWebSocket(value);
|
|
1180
1346
|
const evaluator = section("evaluator");
|
|
1181
1347
|
const report = section("report");
|
|
1182
1348
|
return new EvalConfig({
|
|
1183
1349
|
version: value.version,
|
|
1184
1350
|
configPath,
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
1188
|
-
protocol: new ProtocolConfig({
|
|
1189
|
-
request: new RequestConfig({
|
|
1190
|
-
...request,
|
|
1191
|
-
format: request.format ?? RequestFormat.Json
|
|
1192
|
-
}),
|
|
1193
|
-
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
1194
|
-
})
|
|
1195
|
-
}),
|
|
1351
|
+
variables,
|
|
1352
|
+
websocket,
|
|
1196
1353
|
execution: new ExecutionConfig({
|
|
1197
1354
|
...DEFAULT_CONFIG.execution,
|
|
1198
1355
|
...section("execution"),
|
|
@@ -1213,7 +1370,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1213
1370
|
directory: resolve(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
|
|
1214
1371
|
formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
|
|
1215
1372
|
}),
|
|
1216
|
-
secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols
|
|
1373
|
+
secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
|
|
1217
1374
|
});
|
|
1218
1375
|
}
|
|
1219
1376
|
applyRepetition(config, override, value, document) {
|
|
@@ -1266,13 +1423,41 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1266
1423
|
])];
|
|
1267
1424
|
}
|
|
1268
1425
|
static section(value, ...path) {
|
|
1269
|
-
|
|
1270
|
-
for (const key of path) current = EvalConfigLoader.child(current, key);
|
|
1271
|
-
return isRecord(current) ? current : {};
|
|
1426
|
+
return WebSocketResolver.section(value, ...path);
|
|
1272
1427
|
}
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1428
|
+
};
|
|
1429
|
+
// src/config/parametersLoader.ts
|
|
1430
|
+
var ParametersLoader = class ParametersLoader {
|
|
1431
|
+
cwd;
|
|
1432
|
+
constructor(cwd = process.cwd()) {
|
|
1433
|
+
this.cwd = cwd;
|
|
1434
|
+
}
|
|
1435
|
+
load(path) {
|
|
1436
|
+
const absolute = resolve(this.cwd, path);
|
|
1437
|
+
const shown = relative(this.cwd, absolute) || path;
|
|
1438
|
+
let text;
|
|
1439
|
+
let value;
|
|
1440
|
+
try {
|
|
1441
|
+
text = readFileSync(absolute, "utf8");
|
|
1442
|
+
} catch {
|
|
1443
|
+
throw ParametersLoader.error(`parameters file not found: ${shown}`);
|
|
1444
|
+
}
|
|
1445
|
+
try {
|
|
1446
|
+
value = JSON.parse(text);
|
|
1447
|
+
} catch {
|
|
1448
|
+
throw ParametersLoader.error(`parameters file ${shown} is not valid JSON`);
|
|
1449
|
+
}
|
|
1450
|
+
if (!isRecord(value)) throw ParametersLoader.error(`parameters file ${shown} must contain a JSON object`);
|
|
1451
|
+
return {
|
|
1452
|
+
path: absolute,
|
|
1453
|
+
parameters: Object.fromEntries(Object.entries(value).filter((entry) => typeof entry[1] === "string"))
|
|
1454
|
+
};
|
|
1455
|
+
}
|
|
1456
|
+
static error(message) {
|
|
1457
|
+
return new EvalError({
|
|
1458
|
+
category: ErrorCategory.Configuration,
|
|
1459
|
+
message
|
|
1460
|
+
});
|
|
1276
1461
|
}
|
|
1277
1462
|
};
|
|
1278
1463
|
// schemas/conversation.schema.json
|
|
@@ -1290,6 +1475,11 @@ var conversation_schema_default = {
|
|
|
1290
1475
|
"version": { "const": 1 },
|
|
1291
1476
|
"name": { "$ref": "#/definitions/text" },
|
|
1292
1477
|
"description": { "type": "string" },
|
|
1478
|
+
"variables": {
|
|
1479
|
+
"type": "object",
|
|
1480
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
1481
|
+
"additionalProperties": { "type": "string" }
|
|
1482
|
+
},
|
|
1293
1483
|
"turns": {
|
|
1294
1484
|
"type": "array",
|
|
1295
1485
|
"minItems": 1,
|
|
@@ -1302,8 +1492,8 @@ var conversation_schema_default = {
|
|
|
1302
1492
|
"properties": { "conversation": {
|
|
1303
1493
|
"type": "object",
|
|
1304
1494
|
"additionalProperties": false,
|
|
1305
|
-
"required": ["
|
|
1306
|
-
"properties": { "
|
|
1495
|
+
"required": ["should"],
|
|
1496
|
+
"properties": { "should": { "$ref": "#/definitions/text" } }
|
|
1307
1497
|
} }
|
|
1308
1498
|
}
|
|
1309
1499
|
},
|
|
@@ -1346,12 +1536,12 @@ var conversation_schema_default = {
|
|
|
1346
1536
|
}
|
|
1347
1537
|
},
|
|
1348
1538
|
"response": {
|
|
1349
|
-
"description": "A string is shorthand for
|
|
1539
|
+
"description": "A string is shorthand for should.",
|
|
1350
1540
|
"type": ["string", "object"],
|
|
1351
1541
|
"pattern": "\\S",
|
|
1352
1542
|
"additionalProperties": false,
|
|
1353
1543
|
"properties": {
|
|
1354
|
-
"
|
|
1544
|
+
"should": { "$ref": "#/definitions/text" },
|
|
1355
1545
|
"contains": { "$ref": "#/definitions/textList" },
|
|
1356
1546
|
"notContains": { "$ref": "#/definitions/textList" }
|
|
1357
1547
|
}
|
|
@@ -1383,7 +1573,7 @@ var conversation_schema_default = {
|
|
|
1383
1573
|
};
|
|
1384
1574
|
// src/scenarios/conversationValidator.ts
|
|
1385
1575
|
const RESPONSE_KEYS = [
|
|
1386
|
-
"
|
|
1576
|
+
"should",
|
|
1387
1577
|
"contains",
|
|
1388
1578
|
"notContains"
|
|
1389
1579
|
];
|
|
@@ -1406,7 +1596,7 @@ var ConversationValidator = class ConversationValidator {
|
|
|
1406
1596
|
validateResponse(response, path, document, issues) {
|
|
1407
1597
|
if (!RESPONSE_KEYS.some((key) => key in response)) {
|
|
1408
1598
|
issues.push(new ValidationIssue({
|
|
1409
|
-
message: "response must have at least one of:
|
|
1599
|
+
message: "response must have at least one of: should, contains, notContains",
|
|
1410
1600
|
location: document.locate(path)
|
|
1411
1601
|
}));
|
|
1412
1602
|
return;
|
|
@@ -1449,30 +1639,66 @@ var ConversationValidator = class ConversationValidator {
|
|
|
1449
1639
|
};
|
|
1450
1640
|
// src/scenarios/scenarioLoader.ts
|
|
1451
1641
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
1452
|
-
var ScenarioLoader = class {
|
|
1642
|
+
var ScenarioLoader = class ScenarioLoader {
|
|
1453
1643
|
cwd;
|
|
1454
1644
|
documentLoader = new YamlDocumentLoader();
|
|
1455
1645
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
1456
1646
|
conversationValidator = new ConversationValidator();
|
|
1647
|
+
variablesReader = new VariablesReader();
|
|
1648
|
+
webSocketResolver = new WebSocketResolver();
|
|
1457
1649
|
environmentResolver;
|
|
1458
|
-
constructor(environment = process.env, cwd = process.cwd()) {
|
|
1650
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1459
1651
|
this.cwd = cwd;
|
|
1460
|
-
this.environmentResolver = new EnvironmentResolver(environment);
|
|
1652
|
+
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
1461
1653
|
}
|
|
1462
1654
|
load(target, config) {
|
|
1463
1655
|
const files = this.discover(resolve(this.cwd, target), config);
|
|
1656
|
+
const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.cwd, config.configPath)) : void 0;
|
|
1464
1657
|
const issues = [];
|
|
1658
|
+
const connectionIssues = [];
|
|
1465
1659
|
const specs = [];
|
|
1466
1660
|
for (const file of files) {
|
|
1467
1661
|
const document = this.documentLoader.load(file, relative(this.cwd, file));
|
|
1468
1662
|
const fileIssues = [...document.errors];
|
|
1469
1663
|
const secrets = [];
|
|
1470
|
-
const
|
|
1664
|
+
const used = /* @__PURE__ */ new Set();
|
|
1665
|
+
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
|
|
1666
|
+
const resolver = this.environmentResolver.withLevels([variables, config.variables]);
|
|
1667
|
+
let value;
|
|
1668
|
+
if (fileIssues.length === 0) {
|
|
1669
|
+
const { variables: raw, ...rest } = document.value;
|
|
1670
|
+
value = {
|
|
1671
|
+
...resolver.resolve(rest, document, fileIssues, secrets, { used }),
|
|
1672
|
+
...raw === void 0 ? {} : { variables }
|
|
1673
|
+
};
|
|
1674
|
+
}
|
|
1675
|
+
let websocket;
|
|
1676
|
+
const websocketIssues = [];
|
|
1677
|
+
if (fileIssues.length === 0 && configDocument !== void 0) {
|
|
1678
|
+
const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
|
|
1679
|
+
if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
|
|
1680
|
+
connectionIssues.push({
|
|
1681
|
+
file: document,
|
|
1682
|
+
issues: websocketIssues
|
|
1683
|
+
});
|
|
1684
|
+
if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
|
|
1685
|
+
}
|
|
1471
1686
|
if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
|
|
1472
1687
|
if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
|
|
1473
|
-
if (fileIssues.length === 0)
|
|
1688
|
+
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
|
|
1689
|
+
if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
1474
1690
|
else issues.push(...fileIssues);
|
|
1475
1691
|
}
|
|
1692
|
+
const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
|
|
1693
|
+
if (shared.length > 0) throw new EvalError({
|
|
1694
|
+
category: ErrorCategory.Configuration,
|
|
1695
|
+
message: "invalid configuration",
|
|
1696
|
+
issues: shared
|
|
1697
|
+
});
|
|
1698
|
+
for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
|
|
1699
|
+
message: issue.toString(),
|
|
1700
|
+
location: ScenarioLoader.variablesLocation(entry.file)
|
|
1701
|
+
}));
|
|
1476
1702
|
if (issues.length > 0) throw new EvalError({
|
|
1477
1703
|
category: ErrorCategory.Specification,
|
|
1478
1704
|
message: "invalid scenario specification",
|
|
@@ -1480,6 +1706,23 @@ var ScenarioLoader = class {
|
|
|
1480
1706
|
});
|
|
1481
1707
|
return specs;
|
|
1482
1708
|
}
|
|
1709
|
+
static shared(lists) {
|
|
1710
|
+
if (lists.length === 0) return [];
|
|
1711
|
+
const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
|
|
1712
|
+
return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
|
|
1713
|
+
}
|
|
1714
|
+
static variablesLocation(document) {
|
|
1715
|
+
return new SourceLocation({
|
|
1716
|
+
...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
|
|
1717
|
+
path: ""
|
|
1718
|
+
});
|
|
1719
|
+
}
|
|
1720
|
+
static unusedVariables(variables, used, document) {
|
|
1721
|
+
return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
|
|
1722
|
+
message: `variable '${name}' is not used by the scenario or the websocket configuration`,
|
|
1723
|
+
location: document.locateKey(["variables"], name)
|
|
1724
|
+
}));
|
|
1725
|
+
}
|
|
1483
1726
|
discover(target, config) {
|
|
1484
1727
|
if (!existsSync(target)) throw new EvalError({
|
|
1485
1728
|
category: ErrorCategory.Specification,
|
|
@@ -1504,7 +1747,7 @@ var ScenarioLoader = class {
|
|
|
1504
1747
|
}
|
|
1505
1748
|
return files;
|
|
1506
1749
|
}
|
|
1507
|
-
build(value, document, secrets) {
|
|
1750
|
+
build(value, document, websocket, secrets) {
|
|
1508
1751
|
const scenario = value;
|
|
1509
1752
|
const turns = scenario.turns;
|
|
1510
1753
|
const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
|
|
@@ -1514,10 +1757,11 @@ var ScenarioLoader = class {
|
|
|
1514
1757
|
description: typeof scenario.description === "string" ? scenario.description : "",
|
|
1515
1758
|
turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
|
|
1516
1759
|
conversation: conversation ? new ConversationExpectation({
|
|
1517
|
-
criteria: conversation.
|
|
1760
|
+
criteria: conversation.should,
|
|
1518
1761
|
location: document.locate(["expect", "conversation"])
|
|
1519
1762
|
}) : void 0,
|
|
1520
1763
|
location: document.locate([]),
|
|
1764
|
+
websocket,
|
|
1521
1765
|
secrets
|
|
1522
1766
|
});
|
|
1523
1767
|
}
|
|
@@ -1543,7 +1787,7 @@ var ScenarioLoader = class {
|
|
|
1543
1787
|
});
|
|
1544
1788
|
const fields = response;
|
|
1545
1789
|
return new ResponseExpectation({
|
|
1546
|
-
criteria: fields.
|
|
1790
|
+
criteria: fields.should,
|
|
1547
1791
|
contains: fields.contains ?? [],
|
|
1548
1792
|
notContains: fields.notContains ?? [],
|
|
1549
1793
|
location
|
|
@@ -2366,7 +2610,11 @@ var ConversationRunner = class {
|
|
|
2366
2610
|
async run(spec, signal = NEVER_ABORTED) {
|
|
2367
2611
|
const startedAt = Date.now();
|
|
2368
2612
|
const outcomes = [];
|
|
2369
|
-
const
|
|
2613
|
+
const config = new EvalConfig({
|
|
2614
|
+
...this.config,
|
|
2615
|
+
websocket: spec.websocket ?? this.config.websocket
|
|
2616
|
+
});
|
|
2617
|
+
const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
|
|
2370
2618
|
const history = [];
|
|
2371
2619
|
const evaluate = this.semanticEvaluator(signal);
|
|
2372
2620
|
const inline = !this.config.execution.continueOnFailure;
|
|
@@ -2391,7 +2639,7 @@ var ConversationRunner = class {
|
|
|
2391
2639
|
try {
|
|
2392
2640
|
const steps = (async () => {
|
|
2393
2641
|
for (let attempt = 0;; attempt++) {
|
|
2394
|
-
connection = this.createConnection(
|
|
2642
|
+
connection = this.createConnection(config);
|
|
2395
2643
|
connectionAttempts++;
|
|
2396
2644
|
try {
|
|
2397
2645
|
await connection.connect();
|
|
@@ -2885,7 +3133,7 @@ var ScenarioRunner = class {
|
|
|
2885
3133
|
required: minPass,
|
|
2886
3134
|
attempts,
|
|
2887
3135
|
tests: this.results.tests(attempts, minPass),
|
|
2888
|
-
agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
|
|
3136
|
+
agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
|
|
2889
3137
|
evaluatorUsage: this.tokens.evaluator(attempts)
|
|
2890
3138
|
});
|
|
2891
3139
|
}
|
|
@@ -2976,7 +3224,9 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
2976
3224
|
const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
|
|
2977
3225
|
if (!this.verbose) return line;
|
|
2978
3226
|
if (result.attempts.length === 0) return `${line}\n`;
|
|
2979
|
-
const
|
|
3227
|
+
const websocket = spec.websocket;
|
|
3228
|
+
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
3229
|
+
const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
|
|
2980
3230
|
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
2981
3231
|
lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
|
|
2982
3232
|
lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
|
|
@@ -3156,6 +3406,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3156
3406
|
const { websocket, execution, evaluator, report } = config;
|
|
3157
3407
|
return {
|
|
3158
3408
|
path: this.path(config.configPath),
|
|
3409
|
+
parametersPath: config.parametersPath === "" ? void 0 : this.path(config.parametersPath),
|
|
3159
3410
|
websocket: {
|
|
3160
3411
|
endpoint: websocket.endpoint,
|
|
3161
3412
|
subprotocols: [...websocket.connection.subprotocols],
|
|
@@ -3188,11 +3439,16 @@ var JsonReporter = class JsonReporter {
|
|
|
3188
3439
|
};
|
|
3189
3440
|
}
|
|
3190
3441
|
scenario(result, config) {
|
|
3191
|
-
const
|
|
3442
|
+
const websocket = result.spec.websocket ?? config.websocket;
|
|
3443
|
+
const capability = websocket.protocol.capabilities.tokenUsage;
|
|
3192
3444
|
return {
|
|
3193
3445
|
name: result.spec.name,
|
|
3194
3446
|
file: result.spec.location.file,
|
|
3195
3447
|
status: result.status,
|
|
3448
|
+
connection: {
|
|
3449
|
+
endpoint: websocket.endpoint,
|
|
3450
|
+
subprotocols: websocket.connection.subprotocols.length
|
|
3451
|
+
},
|
|
3196
3452
|
...JsonReporter.counts(result),
|
|
3197
3453
|
agentUsage: JsonReporter.usage(result.agentUsage),
|
|
3198
3454
|
evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
|
|
@@ -3338,6 +3594,6 @@ var ReportFileWriter = class {
|
|
|
3338
3594
|
return path;
|
|
3339
3595
|
}
|
|
3340
3596
|
};
|
|
3341
|
-
export { ActiveTurn, AgentEventType, ArgumentDifference, ArgumentDifferenceKind, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, CONFIG_FILE_NAME, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, DEFAULT_CONFIG, ErrorCategory, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationScope, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorProvider, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, FlakyPolicy, INTERRUPTED_EXIT_CODE, JSON_REPORT_FILE, JSON_REPORT_VERSION, JsonReporter, LlmEvaluator, ProtocolCapabilities, ProtocolConfig, REDACTED, ReportConfig, ReportFileWriter, ReportFormat, ReportOverride, RequestConfig, RequestFormat, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ResultStatus, RunStatus, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestKind, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, WebSocketConfig, applyScore };
|
|
3597
|
+
export { ActiveTurn, AgentEventType, ArgumentDifference, ArgumentDifferenceKind, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, CONFIG_FILE_NAME, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, DEFAULT_CONFIG, ErrorCategory, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationScope, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorProvider, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, FlakyPolicy, INTERRUPTED_EXIT_CODE, JSON_REPORT_FILE, JSON_REPORT_VERSION, JsonReporter, LlmEvaluator, ParametersLoader, ProtocolCapabilities, ProtocolConfig, REDACTED, ReportConfig, ReportFileWriter, ReportFormat, ReportOverride, RequestConfig, RequestFormat, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ResultStatus, RunStatus, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestKind, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, WebSocketConfig, applyScore };
|
|
3342
3598
|
|
|
3343
3599
|
//# sourceMappingURL=index.js.map
|