@mappies/miniwat-eval 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,8 +1,8 @@
1
1
  import { Message, ResponseMessage, TokenUsage } from "@mappies/miniwat-agent-core";
2
2
  import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
3
3
  import { dirname, extname, join, relative, resolve } from "node:path";
4
- import { Ajv } from "ajv";
5
4
  import { LineCounter, isMap, isPair, isScalar, isSeq, parseDocument } from "yaml";
5
+ import { Ajv } from "ajv";
6
6
  import handlerWs from "configurapi-handler-ws";
7
7
  import { isDeepStrictEqual, styleText } from "node:util";
8
8
  import { Agent, InMemoryAgentStateStore } from "@mappies/miniwat-agent";
@@ -273,6 +273,50 @@ var TurnSpec = class {
273
273
  this.location = new SourceLocation(data?.location);
274
274
  }
275
275
  };
276
+ // src/entities/protocolCapabilities.ts
277
+ var ProtocolCapabilities = class {
278
+ toolUses;
279
+ toolResults;
280
+ tokenUsage;
281
+ constructor(data) {
282
+ this.toolUses = data?.toolUses ?? false;
283
+ this.toolResults = data?.toolResults ?? false;
284
+ this.tokenUsage = data?.tokenUsage ?? false;
285
+ }
286
+ };
287
+ // src/interfaces/requestFormat.ts
288
+ const RequestFormat = { Json: "json" };
289
+ // src/entities/requestConfig.ts
290
+ var RequestConfig = class {
291
+ format;
292
+ value;
293
+ inputPath;
294
+ constructor(data) {
295
+ this.format = data?.format ?? RequestFormat.Json;
296
+ this.value = data?.value;
297
+ this.inputPath = data?.inputPath;
298
+ }
299
+ };
300
+ // src/entities/protocolConfig.ts
301
+ var ProtocolConfig = class {
302
+ request;
303
+ capabilities;
304
+ constructor(data) {
305
+ this.request = new RequestConfig(data?.request);
306
+ this.capabilities = new ProtocolCapabilities(data?.capabilities);
307
+ }
308
+ };
309
+ // src/entities/webSocketConfig.ts
310
+ var WebSocketConfig = class {
311
+ endpoint;
312
+ connection;
313
+ protocol;
314
+ constructor(data) {
315
+ this.endpoint = data?.endpoint ?? "";
316
+ this.connection = new ConnectionConfig(data?.connection);
317
+ this.protocol = new ProtocolConfig(data?.protocol);
318
+ }
319
+ };
276
320
  // src/entities/conversationSpec.ts
277
321
  var ConversationSpec = class {
278
322
  version;
@@ -281,6 +325,7 @@ var ConversationSpec = class {
281
325
  turns;
282
326
  conversation;
283
327
  location;
328
+ websocket;
284
329
  secrets;
285
330
  constructor(data) {
286
331
  this.version = data?.version ?? 1;
@@ -289,6 +334,7 @@ var ConversationSpec = class {
289
334
  this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
290
335
  this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
291
336
  this.location = new SourceLocation(data?.location);
337
+ this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
292
338
  this.secrets = [...data?.secrets ?? []];
293
339
  }
294
340
  };
@@ -390,54 +436,12 @@ var ReportConfig = class {
390
436
  this.formats = data?.formats ?? [];
391
437
  }
392
438
  };
393
- // src/entities/protocolCapabilities.ts
394
- var ProtocolCapabilities = class {
395
- toolUses;
396
- toolResults;
397
- tokenUsage;
398
- constructor(data) {
399
- this.toolUses = data?.toolUses ?? false;
400
- this.toolResults = data?.toolResults ?? false;
401
- this.tokenUsage = data?.tokenUsage ?? false;
402
- }
403
- };
404
- // src/interfaces/requestFormat.ts
405
- const RequestFormat = { Json: "json" };
406
- // src/entities/requestConfig.ts
407
- var RequestConfig = class {
408
- format;
409
- value;
410
- inputPath;
411
- constructor(data) {
412
- this.format = data?.format ?? RequestFormat.Json;
413
- this.value = data?.value;
414
- this.inputPath = data?.inputPath;
415
- }
416
- };
417
- // src/entities/protocolConfig.ts
418
- var ProtocolConfig = class {
419
- request;
420
- capabilities;
421
- constructor(data) {
422
- this.request = new RequestConfig(data?.request);
423
- this.capabilities = new ProtocolCapabilities(data?.capabilities);
424
- }
425
- };
426
- // src/entities/webSocketConfig.ts
427
- var WebSocketConfig = class {
428
- endpoint;
429
- connection;
430
- protocol;
431
- constructor(data) {
432
- this.endpoint = data?.endpoint ?? "";
433
- this.connection = new ConnectionConfig(data?.connection);
434
- this.protocol = new ProtocolConfig(data?.protocol);
435
- }
436
- };
437
439
  // src/entities/evalConfig.ts
438
440
  var EvalConfig = class {
439
441
  version;
440
442
  configPath;
443
+ parametersPath;
444
+ variables;
441
445
  websocket;
442
446
  execution;
443
447
  evaluator;
@@ -446,6 +450,8 @@ var EvalConfig = class {
446
450
  constructor(data) {
447
451
  this.version = data?.version ?? 1;
448
452
  this.configPath = data?.configPath ?? "";
453
+ this.parametersPath = data?.parametersPath ?? "";
454
+ this.variables = { ...data?.variables ?? {} };
449
455
  this.websocket = new WebSocketConfig(data?.websocket);
450
456
  this.execution = new ExecutionConfig(data?.execution);
451
457
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -757,6 +763,11 @@ var eval_config_schema_default = {
757
763
  ],
758
764
  properties: {
759
765
  "version": { "const": 1 },
766
+ "variables": {
767
+ "type": "object",
768
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
769
+ "additionalProperties": { "type": "string" }
770
+ },
760
771
  "websocket": {
761
772
  "type": "object",
762
773
  "additionalProperties": false,
@@ -915,16 +926,80 @@ var eval_config_schema_default = {
915
926
  }
916
927
  }
917
928
  };
929
+ // src/yaml/yamlDocument.ts
930
+ var YamlDocument = class YamlDocument {
931
+ file;
932
+ document;
933
+ lineCounter;
934
+ value;
935
+ errors;
936
+ constructor(file, document, lineCounter) {
937
+ this.file = file;
938
+ this.document = document;
939
+ this.lineCounter = lineCounter;
940
+ this.value = document.toJS();
941
+ this.errors = document.errors.map((error) => new ValidationIssue({
942
+ message: error.message.split("\n")[0],
943
+ location: this.toLocation(error.pos[0], [])
944
+ }));
945
+ }
946
+ locate(path) {
947
+ return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
948
+ }
949
+ locateKey(path, key) {
950
+ const parent = this.findNode(path);
951
+ if (isMap(parent)) {
952
+ const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
953
+ if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
954
+ }
955
+ return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
956
+ }
957
+ static formatPath(path) {
958
+ return path.map((segment, index) => {
959
+ if (typeof segment === "number") return `[${segment}]`;
960
+ return index === 0 ? segment : `.${segment}`;
961
+ }).join("");
962
+ }
963
+ findNode(path) {
964
+ let node = this.document.contents ?? void 0;
965
+ for (const segment of path) {
966
+ let next;
967
+ if (isMap(node)) {
968
+ const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
969
+ next = pair?.value ?? pair?.key;
970
+ } else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
971
+ if (!next) break;
972
+ node = next;
973
+ }
974
+ return node;
975
+ }
976
+ toLocation(offset, path) {
977
+ const position = this.lineCounter.linePos(offset);
978
+ return new SourceLocation({
979
+ file: this.file,
980
+ line: position.line,
981
+ column: position.col,
982
+ path: YamlDocument.formatPath(path)
983
+ });
984
+ }
985
+ };
918
986
  // src/yaml/environmentResolver.ts
919
- const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g;
987
+ const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
920
988
  const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
921
989
  var EnvironmentResolver = class EnvironmentResolver {
922
990
  environment;
923
- constructor(environment = process.env) {
991
+ parameters;
992
+ levels;
993
+ constructor(environment = process.env, parameters = {}, levels = []) {
924
994
  this.environment = environment;
995
+ this.parameters = parameters;
996
+ this.levels = levels;
925
997
  }
926
- resolve(value, document, issues, secrets = []) {
927
- return this.resolveValue(value, [], document, issues, secrets);
998
+ withLevels(levels) {
999
+ return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
1000
+ }
1001
+ resolve(value, document, issues, secrets = [], options = {}) {
1002
+ return this.resolveValue(value, [], document, issues, secrets, options);
928
1003
  }
929
1004
  static hasReference(value) {
930
1005
  return new RegExp(REFERENCE.source).test(value);
@@ -932,21 +1007,38 @@ var EnvironmentResolver = class EnvironmentResolver {
932
1007
  static isSecretName(name) {
933
1008
  return SECRET_NAME.test(name);
934
1009
  }
935
- resolveValue(value, path, document, issues, secrets) {
1010
+ lookup(name) {
1011
+ for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
1012
+ return this.environment[name];
1013
+ }
1014
+ sources() {
1015
+ const sources = [
1016
+ ...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
1017
+ ...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
1018
+ "environment variable"
1019
+ ];
1020
+ return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
1021
+ }
1022
+ resolveValue(value, path, document, issues, secrets, options) {
936
1023
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
937
- const resolved = this.environment[name];
1024
+ options.used?.add(name);
1025
+ const resolved = this.lookup(name);
938
1026
  if (resolved === void 0) {
1027
+ if (options.optional?.(path)) {
1028
+ options.unresolved?.add(YamlDocument.formatPath(path));
1029
+ return reference;
1030
+ }
939
1031
  issues.push(new ValidationIssue({
940
- message: `environment variable '${name}' is not set`,
1032
+ message: `${this.sources()} '${name}' is not set`,
941
1033
  location: document.locate(path)
942
1034
  }));
943
1035
  return reference;
944
1036
  }
945
- if (EnvironmentResolver.isSecretName(name) && !secrets.includes(resolved)) secrets.push(resolved);
1037
+ if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
946
1038
  return resolved;
947
1039
  });
948
- if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
949
- if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
1040
+ if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
1041
+ if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
950
1042
  return value;
951
1043
  }
952
1044
  };
@@ -1030,63 +1122,6 @@ var SchemaValidator = class SchemaValidator {
1030
1122
  return path;
1031
1123
  }
1032
1124
  };
1033
- // src/yaml/yamlDocument.ts
1034
- var YamlDocument = class YamlDocument {
1035
- file;
1036
- document;
1037
- lineCounter;
1038
- value;
1039
- errors;
1040
- constructor(file, document, lineCounter) {
1041
- this.file = file;
1042
- this.document = document;
1043
- this.lineCounter = lineCounter;
1044
- this.value = document.toJS();
1045
- this.errors = document.errors.map((error) => new ValidationIssue({
1046
- message: error.message.split("\n")[0],
1047
- location: this.toLocation(error.pos[0], [])
1048
- }));
1049
- }
1050
- locate(path) {
1051
- return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
1052
- }
1053
- locateKey(path, key) {
1054
- const parent = this.findNode(path);
1055
- if (isMap(parent)) {
1056
- const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
1057
- if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
1058
- }
1059
- return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
1060
- }
1061
- static formatPath(path) {
1062
- return path.map((segment, index) => {
1063
- if (typeof segment === "number") return `[${segment}]`;
1064
- return index === 0 ? segment : `.${segment}`;
1065
- }).join("");
1066
- }
1067
- findNode(path) {
1068
- let node = this.document.contents ?? void 0;
1069
- for (const segment of path) {
1070
- let next;
1071
- if (isMap(node)) {
1072
- const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
1073
- next = pair?.value ?? pair?.key;
1074
- } else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
1075
- if (!next) break;
1076
- node = next;
1077
- }
1078
- return node;
1079
- }
1080
- toLocation(offset, path) {
1081
- const position = this.lineCounter.linePos(offset);
1082
- return new SourceLocation({
1083
- file: this.file,
1084
- line: position.line,
1085
- column: position.col,
1086
- path: YamlDocument.formatPath(path)
1087
- });
1088
- }
1089
- };
1090
1125
  // src/yaml/yamlDocumentLoader.ts
1091
1126
  var YamlDocumentLoader = class {
1092
1127
  load(filePath, displayPath) {
@@ -1105,34 +1140,184 @@ var YamlDocumentLoader = class {
1105
1140
  function isRecord(value) {
1106
1141
  return value !== null && typeof value === "object" && !Array.isArray(value);
1107
1142
  }
1143
+ // src/yaml/variablesReader.ts
1144
+ const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
1145
+ var VariablesReader = class {
1146
+ read(document, resolver, issues, secrets) {
1147
+ const raw = isRecord(document.value) ? document.value.variables : void 0;
1148
+ if (raw === void 0) return {};
1149
+ if (!isRecord(raw)) {
1150
+ issues.push(new ValidationIssue({
1151
+ message: "must be object",
1152
+ location: document.locate(["variables"])
1153
+ }));
1154
+ return {};
1155
+ }
1156
+ const variables = {};
1157
+ for (const [name, value] of Object.entries(raw)) {
1158
+ if (!VARIABLE_NAME.test(name)) {
1159
+ issues.push(new ValidationIssue({
1160
+ message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
1161
+ location: document.locateKey(["variables"], name)
1162
+ }));
1163
+ continue;
1164
+ }
1165
+ if (typeof value !== "string") {
1166
+ issues.push(new ValidationIssue({
1167
+ message: "must be string",
1168
+ location: document.locate(["variables", name])
1169
+ }));
1170
+ continue;
1171
+ }
1172
+ const before = issues.length;
1173
+ const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
1174
+ if (issues.length > before) continue;
1175
+ variables[name] = resolved.variables[name];
1176
+ if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
1177
+ }
1178
+ return variables;
1179
+ }
1180
+ };
1181
+ // src/config/webSocketResolver.ts
1182
+ const WEBSOCKET_SCHEMA = {
1183
+ ...eval_config_schema_default,
1184
+ required: ["websocket"],
1185
+ additionalProperties: true,
1186
+ properties: { websocket: eval_config_schema_default.properties.websocket }
1187
+ };
1188
+ var WebSocketResolver = class WebSocketResolver {
1189
+ schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
1190
+ resolve(document, resolver, issues, secrets, used, defer) {
1191
+ const raw = isRecord(document.value) ? document.value.websocket : void 0;
1192
+ const unresolved = /* @__PURE__ */ new Set();
1193
+ const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
1194
+ const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
1195
+ used,
1196
+ optional,
1197
+ unresolved
1198
+ });
1199
+ const subprotocols = WebSocketResolver.subprotocols(raw);
1200
+ if (subprotocols !== void 0) {
1201
+ const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
1202
+ const set = !unresolved.has(YamlDocument.formatPath([
1203
+ "websocket",
1204
+ "connection",
1205
+ "subprotocols",
1206
+ index
1207
+ ]));
1208
+ const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
1209
+ return set && !empty;
1210
+ });
1211
+ value.websocket.connection.subprotocols = kept;
1212
+ }
1213
+ const prefix = YamlDocument.formatPath([
1214
+ "websocket",
1215
+ "connection",
1216
+ "subprotocols"
1217
+ ]);
1218
+ return {
1219
+ value,
1220
+ deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
1221
+ };
1222
+ }
1223
+ validate(value, document) {
1224
+ const issues = this.schemaValidator.validate(value, document);
1225
+ if (issues.length > 0) return issues;
1226
+ return WebSocketResolver.validateInputPath(value, document);
1227
+ }
1228
+ static validateInputPath(value, document) {
1229
+ const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
1230
+ if (typeof request.inputPath !== "string") return [];
1231
+ const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
1232
+ const last = tokens.pop();
1233
+ let parent = request.value;
1234
+ for (const token of tokens) parent = WebSocketResolver.child(parent, token);
1235
+ if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
1236
+ return [new ValidationIssue({
1237
+ message: "must point to a missing or string value whose parent exists in request.value",
1238
+ location: document.locate([
1239
+ "websocket",
1240
+ "protocol",
1241
+ "request",
1242
+ "inputPath"
1243
+ ])
1244
+ })];
1245
+ }
1246
+ static section(value, ...path) {
1247
+ let current = value;
1248
+ for (const key of path) current = WebSocketResolver.child(current, key);
1249
+ return isRecord(current) ? current : {};
1250
+ }
1251
+ static child(value, key) {
1252
+ if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
1253
+ return isRecord(value) ? value[key] : void 0;
1254
+ }
1255
+ static isSubprotocol(path) {
1256
+ return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
1257
+ }
1258
+ static subprotocols(websocket) {
1259
+ const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
1260
+ return Array.isArray(subprotocols) ? subprotocols : void 0;
1261
+ }
1262
+ };
1108
1263
  // src/config/evalConfigLoader.ts
1109
1264
  var EvalConfigLoader = class EvalConfigLoader {
1110
1265
  cwd;
1111
1266
  documentLoader = new YamlDocumentLoader();
1112
1267
  schemaValidator = new SchemaValidator(eval_config_schema_default);
1268
+ variablesReader = new VariablesReader();
1269
+ webSocketResolver = new WebSocketResolver();
1113
1270
  environmentResolver;
1114
- constructor(environment = process.env, cwd = process.cwd()) {
1271
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1115
1272
  this.cwd = cwd;
1116
- this.environmentResolver = new EnvironmentResolver(environment);
1273
+ this.environmentResolver = new EnvironmentResolver(environment, parameters);
1117
1274
  }
1118
1275
  load(configPath, override, report = new ReportOverride()) {
1119
1276
  const document = this.documentLoader.load(configPath, relative(this.cwd, configPath));
1120
1277
  const issues = [...document.errors];
1121
1278
  if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
1122
1279
  const named = [];
1123
- const value = issues.length === 0 ? this.environmentResolver.resolve(document.value, document, issues, named) : void 0;
1124
- if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
1125
- if (issues.length === 0) issues.push(...this.validateInputPath(value, document));
1280
+ const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
1281
+ let value = {};
1282
+ if (issues.length === 0) {
1283
+ const resolver = this.environmentResolver.withLevels([variables]);
1284
+ const { websocket: _websocket, variables: _variables, ...rest } = document.value;
1285
+ const resolved = resolver.resolve(rest, document, issues, named);
1286
+ const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
1287
+ value = {
1288
+ ...resolved,
1289
+ ..._variables === void 0 ? {} : { variables },
1290
+ ..._websocket === void 0 ? {} : websocket.value
1291
+ };
1292
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
1293
+ if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
1294
+ }
1126
1295
  if (issues.length > 0) throw new EvalError({
1127
1296
  category: ErrorCategory.Configuration,
1128
1297
  message: "invalid configuration",
1129
1298
  issues
1130
1299
  });
1131
- const config = this.build(value, configPath, named);
1300
+ const config = this.build(value, configPath, variables, named);
1132
1301
  this.applyRepetition(config, override, value, document);
1133
1302
  EvalConfigLoader.applyReport(config, report);
1134
1303
  return config;
1135
1304
  }
1305
+ static buildWebSocket(value) {
1306
+ const section = (...path) => WebSocketResolver.section(value, ...path);
1307
+ const connection = section("websocket", "connection");
1308
+ const request = section("websocket", "protocol", "request");
1309
+ return new WebSocketConfig({
1310
+ endpoint: WebSocketResolver.section(value, "websocket").endpoint,
1311
+ connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
1312
+ protocol: new ProtocolConfig({
1313
+ request: new RequestConfig({
1314
+ ...request,
1315
+ format: request.format ?? RequestFormat.Json
1316
+ }),
1317
+ capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
1318
+ })
1319
+ });
1320
+ }
1136
1321
  static applyReport(config, report) {
1137
1322
  if (report.formats === void 0) return;
1138
1323
  const known = Object.values(ReportFormat);
@@ -1150,49 +1335,21 @@ var EvalConfigLoader = class EvalConfigLoader {
1150
1335
  const apiKey = EvalConfigLoader.section(document.value, "evaluator").apiKey;
1151
1336
  if (typeof apiKey !== "string" || EnvironmentResolver.hasReference(apiKey)) return [];
1152
1337
  return [new ValidationIssue({
1153
- message: "must be an environment reference such as ${OPENAI_API_KEY}",
1338
+ message: "must be a reference such as ${OPENAI_API_KEY}",
1154
1339
  location: document.locate(["evaluator", "apiKey"])
1155
1340
  })];
1156
1341
  }
1157
- validateInputPath(value, document) {
1158
- const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
1159
- if (typeof request.inputPath !== "string") return [];
1160
- const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
1161
- const last = tokens.pop();
1162
- let parent = request.value;
1163
- for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
1164
- if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
1165
- return [new ValidationIssue({
1166
- message: "must point to a missing or string value whose parent exists in request.value",
1167
- location: document.locate([
1168
- "websocket",
1169
- "protocol",
1170
- "request",
1171
- "inputPath"
1172
- ])
1173
- })];
1174
- }
1175
- build(value, configPath, named) {
1342
+ build(value, configPath, variables, named) {
1176
1343
  const section = (...path) => EvalConfigLoader.section(value, ...path);
1177
1344
  const configDirectory = dirname(configPath);
1178
- const connection = section("websocket", "connection");
1179
- const request = section("websocket", "protocol", "request");
1345
+ const websocket = EvalConfigLoader.buildWebSocket(value);
1180
1346
  const evaluator = section("evaluator");
1181
1347
  const report = section("report");
1182
1348
  return new EvalConfig({
1183
1349
  version: value.version,
1184
1350
  configPath,
1185
- websocket: new WebSocketConfig({
1186
- endpoint: EvalConfigLoader.section(value, "websocket").endpoint,
1187
- connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
1188
- protocol: new ProtocolConfig({
1189
- request: new RequestConfig({
1190
- ...request,
1191
- format: request.format ?? RequestFormat.Json
1192
- }),
1193
- capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
1194
- })
1195
- }),
1351
+ variables,
1352
+ websocket,
1196
1353
  execution: new ExecutionConfig({
1197
1354
  ...DEFAULT_CONFIG.execution,
1198
1355
  ...section("execution"),
@@ -1213,7 +1370,7 @@ var EvalConfigLoader = class EvalConfigLoader {
1213
1370
  directory: resolve(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
1214
1371
  formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
1215
1372
  }),
1216
- secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols ?? [], named)
1373
+ secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
1217
1374
  });
1218
1375
  }
1219
1376
  applyRepetition(config, override, value, document) {
@@ -1266,13 +1423,41 @@ var EvalConfigLoader = class EvalConfigLoader {
1266
1423
  ])];
1267
1424
  }
1268
1425
  static section(value, ...path) {
1269
- let current = value;
1270
- for (const key of path) current = EvalConfigLoader.child(current, key);
1271
- return isRecord(current) ? current : {};
1426
+ return WebSocketResolver.section(value, ...path);
1272
1427
  }
1273
- static child(value, key) {
1274
- if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
1275
- return isRecord(value) ? value[key] : void 0;
1428
+ };
1429
+ // src/config/parametersLoader.ts
1430
+ var ParametersLoader = class ParametersLoader {
1431
+ cwd;
1432
+ constructor(cwd = process.cwd()) {
1433
+ this.cwd = cwd;
1434
+ }
1435
+ load(path) {
1436
+ const absolute = resolve(this.cwd, path);
1437
+ const shown = relative(this.cwd, absolute) || path;
1438
+ let text;
1439
+ let value;
1440
+ try {
1441
+ text = readFileSync(absolute, "utf8");
1442
+ } catch {
1443
+ throw ParametersLoader.error(`parameters file not found: ${shown}`);
1444
+ }
1445
+ try {
1446
+ value = JSON.parse(text);
1447
+ } catch {
1448
+ throw ParametersLoader.error(`parameters file ${shown} is not valid JSON`);
1449
+ }
1450
+ if (!isRecord(value)) throw ParametersLoader.error(`parameters file ${shown} must contain a JSON object`);
1451
+ return {
1452
+ path: absolute,
1453
+ parameters: Object.fromEntries(Object.entries(value).filter((entry) => typeof entry[1] === "string"))
1454
+ };
1455
+ }
1456
+ static error(message) {
1457
+ return new EvalError({
1458
+ category: ErrorCategory.Configuration,
1459
+ message
1460
+ });
1276
1461
  }
1277
1462
  };
1278
1463
  // schemas/conversation.schema.json
@@ -1290,6 +1475,11 @@ var conversation_schema_default = {
1290
1475
  "version": { "const": 1 },
1291
1476
  "name": { "$ref": "#/definitions/text" },
1292
1477
  "description": { "type": "string" },
1478
+ "variables": {
1479
+ "type": "object",
1480
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
1481
+ "additionalProperties": { "type": "string" }
1482
+ },
1293
1483
  "turns": {
1294
1484
  "type": "array",
1295
1485
  "minItems": 1,
@@ -1302,8 +1492,8 @@ var conversation_schema_default = {
1302
1492
  "properties": { "conversation": {
1303
1493
  "type": "object",
1304
1494
  "additionalProperties": false,
1305
- "required": ["criteria"],
1306
- "properties": { "criteria": { "$ref": "#/definitions/text" } }
1495
+ "required": ["should"],
1496
+ "properties": { "should": { "$ref": "#/definitions/text" } }
1307
1497
  } }
1308
1498
  }
1309
1499
  },
@@ -1346,12 +1536,12 @@ var conversation_schema_default = {
1346
1536
  }
1347
1537
  },
1348
1538
  "response": {
1349
- "description": "A string is shorthand for criteria.",
1539
+ "description": "A string is shorthand for should.",
1350
1540
  "type": ["string", "object"],
1351
1541
  "pattern": "\\S",
1352
1542
  "additionalProperties": false,
1353
1543
  "properties": {
1354
- "criteria": { "$ref": "#/definitions/text" },
1544
+ "should": { "$ref": "#/definitions/text" },
1355
1545
  "contains": { "$ref": "#/definitions/textList" },
1356
1546
  "notContains": { "$ref": "#/definitions/textList" }
1357
1547
  }
@@ -1383,7 +1573,7 @@ var conversation_schema_default = {
1383
1573
  };
1384
1574
  // src/scenarios/conversationValidator.ts
1385
1575
  const RESPONSE_KEYS = [
1386
- "criteria",
1576
+ "should",
1387
1577
  "contains",
1388
1578
  "notContains"
1389
1579
  ];
@@ -1406,7 +1596,7 @@ var ConversationValidator = class ConversationValidator {
1406
1596
  validateResponse(response, path, document, issues) {
1407
1597
  if (!RESPONSE_KEYS.some((key) => key in response)) {
1408
1598
  issues.push(new ValidationIssue({
1409
- message: "response must have at least one of: criteria, contains, notContains",
1599
+ message: "response must have at least one of: should, contains, notContains",
1410
1600
  location: document.locate(path)
1411
1601
  }));
1412
1602
  return;
@@ -1449,30 +1639,66 @@ var ConversationValidator = class ConversationValidator {
1449
1639
  };
1450
1640
  // src/scenarios/scenarioLoader.ts
1451
1641
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
1452
- var ScenarioLoader = class {
1642
+ var ScenarioLoader = class ScenarioLoader {
1453
1643
  cwd;
1454
1644
  documentLoader = new YamlDocumentLoader();
1455
1645
  schemaValidator = new SchemaValidator(conversation_schema_default);
1456
1646
  conversationValidator = new ConversationValidator();
1647
+ variablesReader = new VariablesReader();
1648
+ webSocketResolver = new WebSocketResolver();
1457
1649
  environmentResolver;
1458
- constructor(environment = process.env, cwd = process.cwd()) {
1650
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1459
1651
  this.cwd = cwd;
1460
- this.environmentResolver = new EnvironmentResolver(environment);
1652
+ this.environmentResolver = new EnvironmentResolver(environment, parameters);
1461
1653
  }
1462
1654
  load(target, config) {
1463
1655
  const files = this.discover(resolve(this.cwd, target), config);
1656
+ const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.cwd, config.configPath)) : void 0;
1464
1657
  const issues = [];
1658
+ const connectionIssues = [];
1465
1659
  const specs = [];
1466
1660
  for (const file of files) {
1467
1661
  const document = this.documentLoader.load(file, relative(this.cwd, file));
1468
1662
  const fileIssues = [...document.errors];
1469
1663
  const secrets = [];
1470
- const value = fileIssues.length === 0 ? this.environmentResolver.resolve(document.value, document, fileIssues, secrets) : void 0;
1664
+ const used = /* @__PURE__ */ new Set();
1665
+ const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
1666
+ const resolver = this.environmentResolver.withLevels([variables, config.variables]);
1667
+ let value;
1668
+ if (fileIssues.length === 0) {
1669
+ const { variables: raw, ...rest } = document.value;
1670
+ value = {
1671
+ ...resolver.resolve(rest, document, fileIssues, secrets, { used }),
1672
+ ...raw === void 0 ? {} : { variables }
1673
+ };
1674
+ }
1675
+ let websocket;
1676
+ const websocketIssues = [];
1677
+ if (fileIssues.length === 0 && configDocument !== void 0) {
1678
+ const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
1679
+ if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
1680
+ connectionIssues.push({
1681
+ file: document,
1682
+ issues: websocketIssues
1683
+ });
1684
+ if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
1685
+ }
1471
1686
  if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
1472
1687
  if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
1473
- if (fileIssues.length === 0) specs.push(this.build(value, document, secrets));
1688
+ if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
1689
+ if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
1474
1690
  else issues.push(...fileIssues);
1475
1691
  }
1692
+ const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
1693
+ if (shared.length > 0) throw new EvalError({
1694
+ category: ErrorCategory.Configuration,
1695
+ message: "invalid configuration",
1696
+ issues: shared
1697
+ });
1698
+ for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
1699
+ message: issue.toString(),
1700
+ location: ScenarioLoader.variablesLocation(entry.file)
1701
+ }));
1476
1702
  if (issues.length > 0) throw new EvalError({
1477
1703
  category: ErrorCategory.Specification,
1478
1704
  message: "invalid scenario specification",
@@ -1480,6 +1706,23 @@ var ScenarioLoader = class {
1480
1706
  });
1481
1707
  return specs;
1482
1708
  }
1709
+ static shared(lists) {
1710
+ if (lists.length === 0) return [];
1711
+ const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
1712
+ return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
1713
+ }
1714
+ static variablesLocation(document) {
1715
+ return new SourceLocation({
1716
+ ...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
1717
+ path: ""
1718
+ });
1719
+ }
1720
+ static unusedVariables(variables, used, document) {
1721
+ return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
1722
+ message: `variable '${name}' is not used by the scenario or the websocket configuration`,
1723
+ location: document.locateKey(["variables"], name)
1724
+ }));
1725
+ }
1483
1726
  discover(target, config) {
1484
1727
  if (!existsSync(target)) throw new EvalError({
1485
1728
  category: ErrorCategory.Specification,
@@ -1504,7 +1747,7 @@ var ScenarioLoader = class {
1504
1747
  }
1505
1748
  return files;
1506
1749
  }
1507
- build(value, document, secrets) {
1750
+ build(value, document, websocket, secrets) {
1508
1751
  const scenario = value;
1509
1752
  const turns = scenario.turns;
1510
1753
  const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
@@ -1514,10 +1757,11 @@ var ScenarioLoader = class {
1514
1757
  description: typeof scenario.description === "string" ? scenario.description : "",
1515
1758
  turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
1516
1759
  conversation: conversation ? new ConversationExpectation({
1517
- criteria: conversation.criteria,
1760
+ criteria: conversation.should,
1518
1761
  location: document.locate(["expect", "conversation"])
1519
1762
  }) : void 0,
1520
1763
  location: document.locate([]),
1764
+ websocket,
1521
1765
  secrets
1522
1766
  });
1523
1767
  }
@@ -1543,7 +1787,7 @@ var ScenarioLoader = class {
1543
1787
  });
1544
1788
  const fields = response;
1545
1789
  return new ResponseExpectation({
1546
- criteria: fields.criteria,
1790
+ criteria: fields.should,
1547
1791
  contains: fields.contains ?? [],
1548
1792
  notContains: fields.notContains ?? [],
1549
1793
  location
@@ -2366,7 +2610,11 @@ var ConversationRunner = class {
2366
2610
  async run(spec, signal = NEVER_ABORTED) {
2367
2611
  const startedAt = Date.now();
2368
2612
  const outcomes = [];
2369
- const assertions = new AssertionRunner(this.config.websocket.protocol.capabilities);
2613
+ const config = new EvalConfig({
2614
+ ...this.config,
2615
+ websocket: spec.websocket ?? this.config.websocket
2616
+ });
2617
+ const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
2370
2618
  const history = [];
2371
2619
  const evaluate = this.semanticEvaluator(signal);
2372
2620
  const inline = !this.config.execution.continueOnFailure;
@@ -2391,7 +2639,7 @@ var ConversationRunner = class {
2391
2639
  try {
2392
2640
  const steps = (async () => {
2393
2641
  for (let attempt = 0;; attempt++) {
2394
- connection = this.createConnection(this.config);
2642
+ connection = this.createConnection(config);
2395
2643
  connectionAttempts++;
2396
2644
  try {
2397
2645
  await connection.connect();
@@ -2885,7 +3133,7 @@ var ScenarioRunner = class {
2885
3133
  required: minPass,
2886
3134
  attempts,
2887
3135
  tests: this.results.tests(attempts, minPass),
2888
- agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
3136
+ agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
2889
3137
  evaluatorUsage: this.tokens.evaluator(attempts)
2890
3138
  });
2891
3139
  }
@@ -2976,7 +3224,9 @@ var ConsoleReporter = class ConsoleReporter {
2976
3224
  const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
2977
3225
  if (!this.verbose) return line;
2978
3226
  if (result.attempts.length === 0) return `${line}\n`;
2979
- const lines = [""];
3227
+ const websocket = spec.websocket;
3228
+ const subprotocols = websocket?.connection.subprotocols.length ?? 0;
3229
+ const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
2980
3230
  for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
2981
3231
  lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
2982
3232
  lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
@@ -3156,6 +3406,7 @@ var JsonReporter = class JsonReporter {
3156
3406
  const { websocket, execution, evaluator, report } = config;
3157
3407
  return {
3158
3408
  path: this.path(config.configPath),
3409
+ parametersPath: config.parametersPath === "" ? void 0 : this.path(config.parametersPath),
3159
3410
  websocket: {
3160
3411
  endpoint: websocket.endpoint,
3161
3412
  subprotocols: [...websocket.connection.subprotocols],
@@ -3188,11 +3439,16 @@ var JsonReporter = class JsonReporter {
3188
3439
  };
3189
3440
  }
3190
3441
  scenario(result, config) {
3191
- const capability = config.websocket.protocol.capabilities.tokenUsage;
3442
+ const websocket = result.spec.websocket ?? config.websocket;
3443
+ const capability = websocket.protocol.capabilities.tokenUsage;
3192
3444
  return {
3193
3445
  name: result.spec.name,
3194
3446
  file: result.spec.location.file,
3195
3447
  status: result.status,
3448
+ connection: {
3449
+ endpoint: websocket.endpoint,
3450
+ subprotocols: websocket.connection.subprotocols.length
3451
+ },
3196
3452
  ...JsonReporter.counts(result),
3197
3453
  agentUsage: JsonReporter.usage(result.agentUsage),
3198
3454
  evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
@@ -3338,6 +3594,6 @@ var ReportFileWriter = class {
3338
3594
  return path;
3339
3595
  }
3340
3596
  };
3341
- export { ActiveTurn, AgentEventType, ArgumentDifference, ArgumentDifferenceKind, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, CONFIG_FILE_NAME, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, DEFAULT_CONFIG, ErrorCategory, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationScope, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorProvider, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, FlakyPolicy, INTERRUPTED_EXIT_CODE, JSON_REPORT_FILE, JSON_REPORT_VERSION, JsonReporter, LlmEvaluator, ProtocolCapabilities, ProtocolConfig, REDACTED, ReportConfig, ReportFileWriter, ReportFormat, ReportOverride, RequestConfig, RequestFormat, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ResultStatus, RunStatus, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestKind, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, WebSocketConfig, applyScore };
3597
+ export { ActiveTurn, AgentEventType, ArgumentDifference, ArgumentDifferenceKind, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, CONFIG_FILE_NAME, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, DEFAULT_CONFIG, ErrorCategory, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationScope, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorProvider, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, FlakyPolicy, INTERRUPTED_EXIT_CODE, JSON_REPORT_FILE, JSON_REPORT_VERSION, JsonReporter, LlmEvaluator, ParametersLoader, ProtocolCapabilities, ProtocolConfig, REDACTED, ReportConfig, ReportFileWriter, ReportFormat, ReportOverride, RequestConfig, RequestFormat, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ResultStatus, RunStatus, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestKind, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, WebSocketConfig, applyScore };
3342
3598
 
3343
3599
  //# sourceMappingURL=index.js.map