@mappies/miniwat-eval 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -23,13 +23,13 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
23
23
  let _mappies_miniwat_agent_core = require("@mappies/miniwat-agent-core");
24
24
  let node_fs = require("node:fs");
25
25
  let node_path = require("node:path");
26
- let ajv = require("ajv");
27
26
  let yaml = require("yaml");
27
+ let ajv = require("ajv");
28
28
  let configurapi_handler_ws = require("configurapi-handler-ws");
29
- configurapi_handler_ws = __toESM(configurapi_handler_ws, 1);
30
29
  let node_util = require("node:util");
31
30
  let _mappies_miniwat_agent = require("@mappies/miniwat-agent");
32
31
  let node_crypto = require("node:crypto");
32
+ configurapi_handler_ws = __toESM(configurapi_handler_ws, 1);
33
33
  // src/entities/turnExecution.ts
34
34
  var TurnExecution = class {
35
35
  response;
@@ -296,6 +296,50 @@ var TurnSpec = class {
296
296
  this.location = new SourceLocation(data?.location);
297
297
  }
298
298
  };
299
+ // src/entities/protocolCapabilities.ts
300
+ var ProtocolCapabilities = class {
301
+ toolUses;
302
+ toolResults;
303
+ tokenUsage;
304
+ constructor(data) {
305
+ this.toolUses = data?.toolUses ?? false;
306
+ this.toolResults = data?.toolResults ?? false;
307
+ this.tokenUsage = data?.tokenUsage ?? false;
308
+ }
309
+ };
310
+ // src/interfaces/requestFormat.ts
311
+ const RequestFormat = { Json: "json" };
312
+ // src/entities/requestConfig.ts
313
+ var RequestConfig = class {
314
+ format;
315
+ value;
316
+ inputPath;
317
+ constructor(data) {
318
+ this.format = data?.format ?? RequestFormat.Json;
319
+ this.value = data?.value;
320
+ this.inputPath = data?.inputPath;
321
+ }
322
+ };
323
+ // src/entities/protocolConfig.ts
324
+ var ProtocolConfig = class {
325
+ request;
326
+ capabilities;
327
+ constructor(data) {
328
+ this.request = new RequestConfig(data?.request);
329
+ this.capabilities = new ProtocolCapabilities(data?.capabilities);
330
+ }
331
+ };
332
+ // src/entities/webSocketConfig.ts
333
+ var WebSocketConfig = class {
334
+ endpoint;
335
+ connection;
336
+ protocol;
337
+ constructor(data) {
338
+ this.endpoint = data?.endpoint ?? "";
339
+ this.connection = new ConnectionConfig(data?.connection);
340
+ this.protocol = new ProtocolConfig(data?.protocol);
341
+ }
342
+ };
299
343
  // src/entities/conversationSpec.ts
300
344
  var ConversationSpec = class {
301
345
  version;
@@ -304,6 +348,7 @@ var ConversationSpec = class {
304
348
  turns;
305
349
  conversation;
306
350
  location;
351
+ websocket;
307
352
  secrets;
308
353
  constructor(data) {
309
354
  this.version = data?.version ?? 1;
@@ -312,6 +357,7 @@ var ConversationSpec = class {
312
357
  this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
313
358
  this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
314
359
  this.location = new SourceLocation(data?.location);
360
+ this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
315
361
  this.secrets = [...data?.secrets ?? []];
316
362
  }
317
363
  };
@@ -413,54 +459,12 @@ var ReportConfig = class {
413
459
  this.formats = data?.formats ?? [];
414
460
  }
415
461
  };
416
- // src/entities/protocolCapabilities.ts
417
- var ProtocolCapabilities = class {
418
- toolUses;
419
- toolResults;
420
- tokenUsage;
421
- constructor(data) {
422
- this.toolUses = data?.toolUses ?? false;
423
- this.toolResults = data?.toolResults ?? false;
424
- this.tokenUsage = data?.tokenUsage ?? false;
425
- }
426
- };
427
- // src/interfaces/requestFormat.ts
428
- const RequestFormat = { Json: "json" };
429
- // src/entities/requestConfig.ts
430
- var RequestConfig = class {
431
- format;
432
- value;
433
- inputPath;
434
- constructor(data) {
435
- this.format = data?.format ?? RequestFormat.Json;
436
- this.value = data?.value;
437
- this.inputPath = data?.inputPath;
438
- }
439
- };
440
- // src/entities/protocolConfig.ts
441
- var ProtocolConfig = class {
442
- request;
443
- capabilities;
444
- constructor(data) {
445
- this.request = new RequestConfig(data?.request);
446
- this.capabilities = new ProtocolCapabilities(data?.capabilities);
447
- }
448
- };
449
- // src/entities/webSocketConfig.ts
450
- var WebSocketConfig = class {
451
- endpoint;
452
- connection;
453
- protocol;
454
- constructor(data) {
455
- this.endpoint = data?.endpoint ?? "";
456
- this.connection = new ConnectionConfig(data?.connection);
457
- this.protocol = new ProtocolConfig(data?.protocol);
458
- }
459
- };
460
462
  // src/entities/evalConfig.ts
461
463
  var EvalConfig = class {
462
464
  version;
463
465
  configPath;
466
+ parametersPath;
467
+ variables;
464
468
  websocket;
465
469
  execution;
466
470
  evaluator;
@@ -469,6 +473,8 @@ var EvalConfig = class {
469
473
  constructor(data) {
470
474
  this.version = data?.version ?? 1;
471
475
  this.configPath = data?.configPath ?? "";
476
+ this.parametersPath = data?.parametersPath ?? "";
477
+ this.variables = { ...data?.variables ?? {} };
472
478
  this.websocket = new WebSocketConfig(data?.websocket);
473
479
  this.execution = new ExecutionConfig(data?.execution);
474
480
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -780,6 +786,11 @@ var eval_config_schema_default = {
780
786
  ],
781
787
  properties: {
782
788
  "version": { "const": 1 },
789
+ "variables": {
790
+ "type": "object",
791
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
792
+ "additionalProperties": { "type": "string" }
793
+ },
783
794
  "websocket": {
784
795
  "type": "object",
785
796
  "additionalProperties": false,
@@ -938,16 +949,80 @@ var eval_config_schema_default = {
938
949
  }
939
950
  }
940
951
  };
952
+ // src/yaml/yamlDocument.ts
953
+ var YamlDocument = class YamlDocument {
954
+ file;
955
+ document;
956
+ lineCounter;
957
+ value;
958
+ errors;
959
+ constructor(file, document, lineCounter) {
960
+ this.file = file;
961
+ this.document = document;
962
+ this.lineCounter = lineCounter;
963
+ this.value = document.toJS();
964
+ this.errors = document.errors.map((error) => new ValidationIssue({
965
+ message: error.message.split("\n")[0],
966
+ location: this.toLocation(error.pos[0], [])
967
+ }));
968
+ }
969
+ locate(path) {
970
+ return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
971
+ }
972
+ locateKey(path, key) {
973
+ const parent = this.findNode(path);
974
+ if ((0, yaml.isMap)(parent)) {
975
+ const pair = parent.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === key);
976
+ if (pair && (0, yaml.isScalar)(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
977
+ }
978
+ return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
979
+ }
980
+ static formatPath(path) {
981
+ return path.map((segment, index) => {
982
+ if (typeof segment === "number") return `[${segment}]`;
983
+ return index === 0 ? segment : `.${segment}`;
984
+ }).join("");
985
+ }
986
+ findNode(path) {
987
+ let node = this.document.contents ?? void 0;
988
+ for (const segment of path) {
989
+ let next;
990
+ if ((0, yaml.isMap)(node)) {
991
+ const pair = node.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === String(segment));
992
+ next = pair?.value ?? pair?.key;
993
+ } else if ((0, yaml.isSeq)(node) && typeof segment === "number") next = node.items[segment];
994
+ if (!next) break;
995
+ node = next;
996
+ }
997
+ return node;
998
+ }
999
+ toLocation(offset, path) {
1000
+ const position = this.lineCounter.linePos(offset);
1001
+ return new SourceLocation({
1002
+ file: this.file,
1003
+ line: position.line,
1004
+ column: position.col,
1005
+ path: YamlDocument.formatPath(path)
1006
+ });
1007
+ }
1008
+ };
941
1009
  // src/yaml/environmentResolver.ts
942
- const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g;
1010
+ const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
943
1011
  const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
944
1012
  var EnvironmentResolver = class EnvironmentResolver {
945
1013
  environment;
946
- constructor(environment = process.env) {
1014
+ parameters;
1015
+ levels;
1016
+ constructor(environment = process.env, parameters = {}, levels = []) {
947
1017
  this.environment = environment;
1018
+ this.parameters = parameters;
1019
+ this.levels = levels;
1020
+ }
1021
+ withLevels(levels) {
1022
+ return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
948
1023
  }
949
- resolve(value, document, issues, secrets = []) {
950
- return this.resolveValue(value, [], document, issues, secrets);
1024
+ resolve(value, document, issues, secrets = [], options = {}) {
1025
+ return this.resolveValue(value, [], document, issues, secrets, options);
951
1026
  }
952
1027
  static hasReference(value) {
953
1028
  return new RegExp(REFERENCE.source).test(value);
@@ -955,21 +1030,38 @@ var EnvironmentResolver = class EnvironmentResolver {
955
1030
  static isSecretName(name) {
956
1031
  return SECRET_NAME.test(name);
957
1032
  }
958
- resolveValue(value, path, document, issues, secrets) {
1033
+ lookup(name) {
1034
+ for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
1035
+ return this.environment[name];
1036
+ }
1037
+ sources() {
1038
+ const sources = [
1039
+ ...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
1040
+ ...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
1041
+ "environment variable"
1042
+ ];
1043
+ return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
1044
+ }
1045
+ resolveValue(value, path, document, issues, secrets, options) {
959
1046
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
960
- const resolved = this.environment[name];
1047
+ options.used?.add(name);
1048
+ const resolved = this.lookup(name);
961
1049
  if (resolved === void 0) {
1050
+ if (options.optional?.(path)) {
1051
+ options.unresolved?.add(YamlDocument.formatPath(path));
1052
+ return reference;
1053
+ }
962
1054
  issues.push(new ValidationIssue({
963
- message: `environment variable '${name}' is not set`,
1055
+ message: `${this.sources()} '${name}' is not set`,
964
1056
  location: document.locate(path)
965
1057
  }));
966
1058
  return reference;
967
1059
  }
968
- if (EnvironmentResolver.isSecretName(name) && !secrets.includes(resolved)) secrets.push(resolved);
1060
+ if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
969
1061
  return resolved;
970
1062
  });
971
- if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
972
- if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
1063
+ if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
1064
+ if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
973
1065
  return value;
974
1066
  }
975
1067
  };
@@ -1053,63 +1145,6 @@ var SchemaValidator = class SchemaValidator {
1053
1145
  return path;
1054
1146
  }
1055
1147
  };
1056
- // src/yaml/yamlDocument.ts
1057
- var YamlDocument = class YamlDocument {
1058
- file;
1059
- document;
1060
- lineCounter;
1061
- value;
1062
- errors;
1063
- constructor(file, document, lineCounter) {
1064
- this.file = file;
1065
- this.document = document;
1066
- this.lineCounter = lineCounter;
1067
- this.value = document.toJS();
1068
- this.errors = document.errors.map((error) => new ValidationIssue({
1069
- message: error.message.split("\n")[0],
1070
- location: this.toLocation(error.pos[0], [])
1071
- }));
1072
- }
1073
- locate(path) {
1074
- return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
1075
- }
1076
- locateKey(path, key) {
1077
- const parent = this.findNode(path);
1078
- if ((0, yaml.isMap)(parent)) {
1079
- const pair = parent.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === key);
1080
- if (pair && (0, yaml.isScalar)(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
1081
- }
1082
- return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
1083
- }
1084
- static formatPath(path) {
1085
- return path.map((segment, index) => {
1086
- if (typeof segment === "number") return `[${segment}]`;
1087
- return index === 0 ? segment : `.${segment}`;
1088
- }).join("");
1089
- }
1090
- findNode(path) {
1091
- let node = this.document.contents ?? void 0;
1092
- for (const segment of path) {
1093
- let next;
1094
- if ((0, yaml.isMap)(node)) {
1095
- const pair = node.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === String(segment));
1096
- next = pair?.value ?? pair?.key;
1097
- } else if ((0, yaml.isSeq)(node) && typeof segment === "number") next = node.items[segment];
1098
- if (!next) break;
1099
- node = next;
1100
- }
1101
- return node;
1102
- }
1103
- toLocation(offset, path) {
1104
- const position = this.lineCounter.linePos(offset);
1105
- return new SourceLocation({
1106
- file: this.file,
1107
- line: position.line,
1108
- column: position.col,
1109
- path: YamlDocument.formatPath(path)
1110
- });
1111
- }
1112
- };
1113
1148
  // src/yaml/yamlDocumentLoader.ts
1114
1149
  var YamlDocumentLoader = class {
1115
1150
  load(filePath, displayPath) {
@@ -1128,34 +1163,184 @@ var YamlDocumentLoader = class {
1128
1163
  function isRecord(value) {
1129
1164
  return value !== null && typeof value === "object" && !Array.isArray(value);
1130
1165
  }
1166
+ // src/yaml/variablesReader.ts
1167
+ const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
1168
+ var VariablesReader = class {
1169
+ read(document, resolver, issues, secrets) {
1170
+ const raw = isRecord(document.value) ? document.value.variables : void 0;
1171
+ if (raw === void 0) return {};
1172
+ if (!isRecord(raw)) {
1173
+ issues.push(new ValidationIssue({
1174
+ message: "must be object",
1175
+ location: document.locate(["variables"])
1176
+ }));
1177
+ return {};
1178
+ }
1179
+ const variables = {};
1180
+ for (const [name, value] of Object.entries(raw)) {
1181
+ if (!VARIABLE_NAME.test(name)) {
1182
+ issues.push(new ValidationIssue({
1183
+ message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
1184
+ location: document.locateKey(["variables"], name)
1185
+ }));
1186
+ continue;
1187
+ }
1188
+ if (typeof value !== "string") {
1189
+ issues.push(new ValidationIssue({
1190
+ message: "must be string",
1191
+ location: document.locate(["variables", name])
1192
+ }));
1193
+ continue;
1194
+ }
1195
+ const before = issues.length;
1196
+ const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
1197
+ if (issues.length > before) continue;
1198
+ variables[name] = resolved.variables[name];
1199
+ if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
1200
+ }
1201
+ return variables;
1202
+ }
1203
+ };
1204
+ // src/config/webSocketResolver.ts
1205
+ const WEBSOCKET_SCHEMA = {
1206
+ ...eval_config_schema_default,
1207
+ required: ["websocket"],
1208
+ additionalProperties: true,
1209
+ properties: { websocket: eval_config_schema_default.properties.websocket }
1210
+ };
1211
+ var WebSocketResolver = class WebSocketResolver {
1212
+ schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
1213
+ resolve(document, resolver, issues, secrets, used, defer) {
1214
+ const raw = isRecord(document.value) ? document.value.websocket : void 0;
1215
+ const unresolved = /* @__PURE__ */ new Set();
1216
+ const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
1217
+ const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
1218
+ used,
1219
+ optional,
1220
+ unresolved
1221
+ });
1222
+ const subprotocols = WebSocketResolver.subprotocols(raw);
1223
+ if (subprotocols !== void 0) {
1224
+ const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
1225
+ const set = !unresolved.has(YamlDocument.formatPath([
1226
+ "websocket",
1227
+ "connection",
1228
+ "subprotocols",
1229
+ index
1230
+ ]));
1231
+ const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
1232
+ return set && !empty;
1233
+ });
1234
+ value.websocket.connection.subprotocols = kept;
1235
+ }
1236
+ const prefix = YamlDocument.formatPath([
1237
+ "websocket",
1238
+ "connection",
1239
+ "subprotocols"
1240
+ ]);
1241
+ return {
1242
+ value,
1243
+ deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
1244
+ };
1245
+ }
1246
+ validate(value, document) {
1247
+ const issues = this.schemaValidator.validate(value, document);
1248
+ if (issues.length > 0) return issues;
1249
+ return WebSocketResolver.validateInputPath(value, document);
1250
+ }
1251
+ static validateInputPath(value, document) {
1252
+ const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
1253
+ if (typeof request.inputPath !== "string") return [];
1254
+ const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
1255
+ const last = tokens.pop();
1256
+ let parent = request.value;
1257
+ for (const token of tokens) parent = WebSocketResolver.child(parent, token);
1258
+ if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
1259
+ return [new ValidationIssue({
1260
+ message: "must point to a missing or string value whose parent exists in request.value",
1261
+ location: document.locate([
1262
+ "websocket",
1263
+ "protocol",
1264
+ "request",
1265
+ "inputPath"
1266
+ ])
1267
+ })];
1268
+ }
1269
+ static section(value, ...path) {
1270
+ let current = value;
1271
+ for (const key of path) current = WebSocketResolver.child(current, key);
1272
+ return isRecord(current) ? current : {};
1273
+ }
1274
+ static child(value, key) {
1275
+ if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
1276
+ return isRecord(value) ? value[key] : void 0;
1277
+ }
1278
+ static isSubprotocol(path) {
1279
+ return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
1280
+ }
1281
+ static subprotocols(websocket) {
1282
+ const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
1283
+ return Array.isArray(subprotocols) ? subprotocols : void 0;
1284
+ }
1285
+ };
1131
1286
  // src/config/evalConfigLoader.ts
1132
1287
  var EvalConfigLoader = class EvalConfigLoader {
1133
1288
  cwd;
1134
1289
  documentLoader = new YamlDocumentLoader();
1135
1290
  schemaValidator = new SchemaValidator(eval_config_schema_default);
1291
+ variablesReader = new VariablesReader();
1292
+ webSocketResolver = new WebSocketResolver();
1136
1293
  environmentResolver;
1137
- constructor(environment = process.env, cwd = process.cwd()) {
1294
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1138
1295
  this.cwd = cwd;
1139
- this.environmentResolver = new EnvironmentResolver(environment);
1296
+ this.environmentResolver = new EnvironmentResolver(environment, parameters);
1140
1297
  }
1141
1298
  load(configPath, override, report = new ReportOverride()) {
1142
1299
  const document = this.documentLoader.load(configPath, (0, node_path.relative)(this.cwd, configPath));
1143
1300
  const issues = [...document.errors];
1144
1301
  if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
1145
1302
  const named = [];
1146
- const value = issues.length === 0 ? this.environmentResolver.resolve(document.value, document, issues, named) : void 0;
1147
- if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
1148
- if (issues.length === 0) issues.push(...this.validateInputPath(value, document));
1303
+ const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
1304
+ let value = {};
1305
+ if (issues.length === 0) {
1306
+ const resolver = this.environmentResolver.withLevels([variables]);
1307
+ const { websocket: _websocket, variables: _variables, ...rest } = document.value;
1308
+ const resolved = resolver.resolve(rest, document, issues, named);
1309
+ const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
1310
+ value = {
1311
+ ...resolved,
1312
+ ..._variables === void 0 ? {} : { variables },
1313
+ ..._websocket === void 0 ? {} : websocket.value
1314
+ };
1315
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
1316
+ if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
1317
+ }
1149
1318
  if (issues.length > 0) throw new EvalError({
1150
1319
  category: ErrorCategory.Configuration,
1151
1320
  message: "invalid configuration",
1152
1321
  issues
1153
1322
  });
1154
- const config = this.build(value, configPath, named);
1323
+ const config = this.build(value, configPath, variables, named);
1155
1324
  this.applyRepetition(config, override, value, document);
1156
1325
  EvalConfigLoader.applyReport(config, report);
1157
1326
  return config;
1158
1327
  }
1328
+ static buildWebSocket(value) {
1329
+ const section = (...path) => WebSocketResolver.section(value, ...path);
1330
+ const connection = section("websocket", "connection");
1331
+ const request = section("websocket", "protocol", "request");
1332
+ return new WebSocketConfig({
1333
+ endpoint: WebSocketResolver.section(value, "websocket").endpoint,
1334
+ connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
1335
+ protocol: new ProtocolConfig({
1336
+ request: new RequestConfig({
1337
+ ...request,
1338
+ format: request.format ?? RequestFormat.Json
1339
+ }),
1340
+ capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
1341
+ })
1342
+ });
1343
+ }
1159
1344
  static applyReport(config, report) {
1160
1345
  if (report.formats === void 0) return;
1161
1346
  const known = Object.values(ReportFormat);
@@ -1173,49 +1358,21 @@ var EvalConfigLoader = class EvalConfigLoader {
1173
1358
  const apiKey = EvalConfigLoader.section(document.value, "evaluator").apiKey;
1174
1359
  if (typeof apiKey !== "string" || EnvironmentResolver.hasReference(apiKey)) return [];
1175
1360
  return [new ValidationIssue({
1176
- message: "must be an environment reference such as ${OPENAI_API_KEY}",
1361
+ message: "must be a reference such as ${OPENAI_API_KEY}",
1177
1362
  location: document.locate(["evaluator", "apiKey"])
1178
1363
  })];
1179
1364
  }
1180
- validateInputPath(value, document) {
1181
- const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
1182
- if (typeof request.inputPath !== "string") return [];
1183
- const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
1184
- const last = tokens.pop();
1185
- let parent = request.value;
1186
- for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
1187
- if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
1188
- return [new ValidationIssue({
1189
- message: "must point to a missing or string value whose parent exists in request.value",
1190
- location: document.locate([
1191
- "websocket",
1192
- "protocol",
1193
- "request",
1194
- "inputPath"
1195
- ])
1196
- })];
1197
- }
1198
- build(value, configPath, named) {
1365
+ build(value, configPath, variables, named) {
1199
1366
  const section = (...path) => EvalConfigLoader.section(value, ...path);
1200
1367
  const configDirectory = (0, node_path.dirname)(configPath);
1201
- const connection = section("websocket", "connection");
1202
- const request = section("websocket", "protocol", "request");
1368
+ const websocket = EvalConfigLoader.buildWebSocket(value);
1203
1369
  const evaluator = section("evaluator");
1204
1370
  const report = section("report");
1205
1371
  return new EvalConfig({
1206
1372
  version: value.version,
1207
1373
  configPath,
1208
- websocket: new WebSocketConfig({
1209
- endpoint: EvalConfigLoader.section(value, "websocket").endpoint,
1210
- connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
1211
- protocol: new ProtocolConfig({
1212
- request: new RequestConfig({
1213
- ...request,
1214
- format: request.format ?? RequestFormat.Json
1215
- }),
1216
- capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
1217
- })
1218
- }),
1374
+ variables,
1375
+ websocket,
1219
1376
  execution: new ExecutionConfig({
1220
1377
  ...DEFAULT_CONFIG.execution,
1221
1378
  ...section("execution"),
@@ -1236,7 +1393,7 @@ var EvalConfigLoader = class EvalConfigLoader {
1236
1393
  directory: (0, node_path.resolve)(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
1237
1394
  formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
1238
1395
  }),
1239
- secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols ?? [], named)
1396
+ secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
1240
1397
  });
1241
1398
  }
1242
1399
  applyRepetition(config, override, value, document) {
@@ -1289,13 +1446,41 @@ var EvalConfigLoader = class EvalConfigLoader {
1289
1446
  ])];
1290
1447
  }
1291
1448
  static section(value, ...path) {
1292
- let current = value;
1293
- for (const key of path) current = EvalConfigLoader.child(current, key);
1294
- return isRecord(current) ? current : {};
1449
+ return WebSocketResolver.section(value, ...path);
1295
1450
  }
1296
- static child(value, key) {
1297
- if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
1298
- return isRecord(value) ? value[key] : void 0;
1451
+ };
1452
+ // src/config/parametersLoader.ts
1453
+ var ParametersLoader = class ParametersLoader {
1454
+ cwd;
1455
+ constructor(cwd = process.cwd()) {
1456
+ this.cwd = cwd;
1457
+ }
1458
+ load(path) {
1459
+ const absolute = (0, node_path.resolve)(this.cwd, path);
1460
+ const shown = (0, node_path.relative)(this.cwd, absolute) || path;
1461
+ let text;
1462
+ let value;
1463
+ try {
1464
+ text = (0, node_fs.readFileSync)(absolute, "utf8");
1465
+ } catch {
1466
+ throw ParametersLoader.error(`parameters file not found: ${shown}`);
1467
+ }
1468
+ try {
1469
+ value = JSON.parse(text);
1470
+ } catch {
1471
+ throw ParametersLoader.error(`parameters file ${shown} is not valid JSON`);
1472
+ }
1473
+ if (!isRecord(value)) throw ParametersLoader.error(`parameters file ${shown} must contain a JSON object`);
1474
+ return {
1475
+ path: absolute,
1476
+ parameters: Object.fromEntries(Object.entries(value).filter((entry) => typeof entry[1] === "string"))
1477
+ };
1478
+ }
1479
+ static error(message) {
1480
+ return new EvalError({
1481
+ category: ErrorCategory.Configuration,
1482
+ message
1483
+ });
1299
1484
  }
1300
1485
  };
1301
1486
  // schemas/conversation.schema.json
@@ -1313,6 +1498,11 @@ var conversation_schema_default = {
1313
1498
  "version": { "const": 1 },
1314
1499
  "name": { "$ref": "#/definitions/text" },
1315
1500
  "description": { "type": "string" },
1501
+ "variables": {
1502
+ "type": "object",
1503
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
1504
+ "additionalProperties": { "type": "string" }
1505
+ },
1316
1506
  "turns": {
1317
1507
  "type": "array",
1318
1508
  "minItems": 1,
@@ -1325,8 +1515,8 @@ var conversation_schema_default = {
1325
1515
  "properties": { "conversation": {
1326
1516
  "type": "object",
1327
1517
  "additionalProperties": false,
1328
- "required": ["criteria"],
1329
- "properties": { "criteria": { "$ref": "#/definitions/text" } }
1518
+ "required": ["should"],
1519
+ "properties": { "should": { "$ref": "#/definitions/text" } }
1330
1520
  } }
1331
1521
  }
1332
1522
  },
@@ -1369,12 +1559,12 @@ var conversation_schema_default = {
1369
1559
  }
1370
1560
  },
1371
1561
  "response": {
1372
- "description": "A string is shorthand for criteria.",
1562
+ "description": "A string is shorthand for should.",
1373
1563
  "type": ["string", "object"],
1374
1564
  "pattern": "\\S",
1375
1565
  "additionalProperties": false,
1376
1566
  "properties": {
1377
- "criteria": { "$ref": "#/definitions/text" },
1567
+ "should": { "$ref": "#/definitions/text" },
1378
1568
  "contains": { "$ref": "#/definitions/textList" },
1379
1569
  "notContains": { "$ref": "#/definitions/textList" }
1380
1570
  }
@@ -1406,7 +1596,7 @@ var conversation_schema_default = {
1406
1596
  };
1407
1597
  // src/scenarios/conversationValidator.ts
1408
1598
  const RESPONSE_KEYS = [
1409
- "criteria",
1599
+ "should",
1410
1600
  "contains",
1411
1601
  "notContains"
1412
1602
  ];
@@ -1429,7 +1619,7 @@ var ConversationValidator = class ConversationValidator {
1429
1619
  validateResponse(response, path, document, issues) {
1430
1620
  if (!RESPONSE_KEYS.some((key) => key in response)) {
1431
1621
  issues.push(new ValidationIssue({
1432
- message: "response must have at least one of: criteria, contains, notContains",
1622
+ message: "response must have at least one of: should, contains, notContains",
1433
1623
  location: document.locate(path)
1434
1624
  }));
1435
1625
  return;
@@ -1472,30 +1662,66 @@ var ConversationValidator = class ConversationValidator {
1472
1662
  };
1473
1663
  // src/scenarios/scenarioLoader.ts
1474
1664
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
1475
- var ScenarioLoader = class {
1665
+ var ScenarioLoader = class ScenarioLoader {
1476
1666
  cwd;
1477
1667
  documentLoader = new YamlDocumentLoader();
1478
1668
  schemaValidator = new SchemaValidator(conversation_schema_default);
1479
1669
  conversationValidator = new ConversationValidator();
1670
+ variablesReader = new VariablesReader();
1671
+ webSocketResolver = new WebSocketResolver();
1480
1672
  environmentResolver;
1481
- constructor(environment = process.env, cwd = process.cwd()) {
1673
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1482
1674
  this.cwd = cwd;
1483
- this.environmentResolver = new EnvironmentResolver(environment);
1675
+ this.environmentResolver = new EnvironmentResolver(environment, parameters);
1484
1676
  }
1485
1677
  load(target, config) {
1486
1678
  const files = this.discover((0, node_path.resolve)(this.cwd, target), config);
1679
+ const configDocument = (0, node_fs.existsSync)(config.configPath) ? this.documentLoader.load(config.configPath, (0, node_path.relative)(this.cwd, config.configPath)) : void 0;
1487
1680
  const issues = [];
1681
+ const connectionIssues = [];
1488
1682
  const specs = [];
1489
1683
  for (const file of files) {
1490
1684
  const document = this.documentLoader.load(file, (0, node_path.relative)(this.cwd, file));
1491
1685
  const fileIssues = [...document.errors];
1492
1686
  const secrets = [];
1493
- const value = fileIssues.length === 0 ? this.environmentResolver.resolve(document.value, document, fileIssues, secrets) : void 0;
1687
+ const used = /* @__PURE__ */ new Set();
1688
+ const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
1689
+ const resolver = this.environmentResolver.withLevels([variables, config.variables]);
1690
+ let value;
1691
+ if (fileIssues.length === 0) {
1692
+ const { variables: raw, ...rest } = document.value;
1693
+ value = {
1694
+ ...resolver.resolve(rest, document, fileIssues, secrets, { used }),
1695
+ ...raw === void 0 ? {} : { variables }
1696
+ };
1697
+ }
1698
+ let websocket;
1699
+ const websocketIssues = [];
1700
+ if (fileIssues.length === 0 && configDocument !== void 0) {
1701
+ const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
1702
+ if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
1703
+ connectionIssues.push({
1704
+ file: document,
1705
+ issues: websocketIssues
1706
+ });
1707
+ if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
1708
+ }
1494
1709
  if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
1495
1710
  if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
1496
- if (fileIssues.length === 0) specs.push(this.build(value, document, secrets));
1711
+ if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
1712
+ if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
1497
1713
  else issues.push(...fileIssues);
1498
1714
  }
1715
+ const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
1716
+ if (shared.length > 0) throw new EvalError({
1717
+ category: ErrorCategory.Configuration,
1718
+ message: "invalid configuration",
1719
+ issues: shared
1720
+ });
1721
+ for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
1722
+ message: issue.toString(),
1723
+ location: ScenarioLoader.variablesLocation(entry.file)
1724
+ }));
1499
1725
  if (issues.length > 0) throw new EvalError({
1500
1726
  category: ErrorCategory.Specification,
1501
1727
  message: "invalid scenario specification",
@@ -1503,6 +1729,23 @@ var ScenarioLoader = class {
1503
1729
  });
1504
1730
  return specs;
1505
1731
  }
1732
+ static shared(lists) {
1733
+ if (lists.length === 0) return [];
1734
+ const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
1735
+ return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
1736
+ }
1737
+ static variablesLocation(document) {
1738
+ return new SourceLocation({
1739
+ ...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
1740
+ path: ""
1741
+ });
1742
+ }
1743
+ static unusedVariables(variables, used, document) {
1744
+ return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
1745
+ message: `variable '${name}' is not used by the scenario or the websocket configuration`,
1746
+ location: document.locateKey(["variables"], name)
1747
+ }));
1748
+ }
1506
1749
  discover(target, config) {
1507
1750
  if (!(0, node_fs.existsSync)(target)) throw new EvalError({
1508
1751
  category: ErrorCategory.Specification,
@@ -1527,7 +1770,7 @@ var ScenarioLoader = class {
1527
1770
  }
1528
1771
  return files;
1529
1772
  }
1530
- build(value, document, secrets) {
1773
+ build(value, document, websocket, secrets) {
1531
1774
  const scenario = value;
1532
1775
  const turns = scenario.turns;
1533
1776
  const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
@@ -1537,10 +1780,11 @@ var ScenarioLoader = class {
1537
1780
  description: typeof scenario.description === "string" ? scenario.description : "",
1538
1781
  turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
1539
1782
  conversation: conversation ? new ConversationExpectation({
1540
- criteria: conversation.criteria,
1783
+ criteria: conversation.should,
1541
1784
  location: document.locate(["expect", "conversation"])
1542
1785
  }) : void 0,
1543
1786
  location: document.locate([]),
1787
+ websocket,
1544
1788
  secrets
1545
1789
  });
1546
1790
  }
@@ -1566,7 +1810,7 @@ var ScenarioLoader = class {
1566
1810
  });
1567
1811
  const fields = response;
1568
1812
  return new ResponseExpectation({
1569
- criteria: fields.criteria,
1813
+ criteria: fields.should,
1570
1814
  contains: fields.contains ?? [],
1571
1815
  notContains: fields.notContains ?? [],
1572
1816
  location
@@ -2389,7 +2633,11 @@ var ConversationRunner = class {
2389
2633
  async run(spec, signal = NEVER_ABORTED) {
2390
2634
  const startedAt = Date.now();
2391
2635
  const outcomes = [];
2392
- const assertions = new AssertionRunner(this.config.websocket.protocol.capabilities);
2636
+ const config = new EvalConfig({
2637
+ ...this.config,
2638
+ websocket: spec.websocket ?? this.config.websocket
2639
+ });
2640
+ const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
2393
2641
  const history = [];
2394
2642
  const evaluate = this.semanticEvaluator(signal);
2395
2643
  const inline = !this.config.execution.continueOnFailure;
@@ -2414,7 +2662,7 @@ var ConversationRunner = class {
2414
2662
  try {
2415
2663
  const steps = (async () => {
2416
2664
  for (let attempt = 0;; attempt++) {
2417
- connection = this.createConnection(this.config);
2665
+ connection = this.createConnection(config);
2418
2666
  connectionAttempts++;
2419
2667
  try {
2420
2668
  await connection.connect();
@@ -2908,7 +3156,7 @@ var ScenarioRunner = class {
2908
3156
  required: minPass,
2909
3157
  attempts,
2910
3158
  tests: this.results.tests(attempts, minPass),
2911
- agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
3159
+ agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
2912
3160
  evaluatorUsage: this.tokens.evaluator(attempts)
2913
3161
  });
2914
3162
  }
@@ -2999,7 +3247,9 @@ var ConsoleReporter = class ConsoleReporter {
2999
3247
  const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
3000
3248
  if (!this.verbose) return line;
3001
3249
  if (result.attempts.length === 0) return `${line}\n`;
3002
- const lines = [""];
3250
+ const websocket = spec.websocket;
3251
+ const subprotocols = websocket?.connection.subprotocols.length ?? 0;
3252
+ const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
3003
3253
  for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3004
3254
  lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
3005
3255
  lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
@@ -3179,6 +3429,7 @@ var JsonReporter = class JsonReporter {
3179
3429
  const { websocket, execution, evaluator, report } = config;
3180
3430
  return {
3181
3431
  path: this.path(config.configPath),
3432
+ parametersPath: config.parametersPath === "" ? void 0 : this.path(config.parametersPath),
3182
3433
  websocket: {
3183
3434
  endpoint: websocket.endpoint,
3184
3435
  subprotocols: [...websocket.connection.subprotocols],
@@ -3211,11 +3462,16 @@ var JsonReporter = class JsonReporter {
3211
3462
  };
3212
3463
  }
3213
3464
  scenario(result, config) {
3214
- const capability = config.websocket.protocol.capabilities.tokenUsage;
3465
+ const websocket = result.spec.websocket ?? config.websocket;
3466
+ const capability = websocket.protocol.capabilities.tokenUsage;
3215
3467
  return {
3216
3468
  name: result.spec.name,
3217
3469
  file: result.spec.location.file,
3218
3470
  status: result.status,
3471
+ connection: {
3472
+ endpoint: websocket.endpoint,
3473
+ subprotocols: websocket.connection.subprotocols.length
3474
+ },
3219
3475
  ...JsonReporter.counts(result),
3220
3476
  agentUsage: JsonReporter.usage(result.agentUsage),
3221
3477
  evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
@@ -3406,6 +3662,7 @@ exports.JSON_REPORT_FILE = JSON_REPORT_FILE;
3406
3662
  exports.JSON_REPORT_VERSION = JSON_REPORT_VERSION;
3407
3663
  exports.JsonReporter = JsonReporter;
3408
3664
  exports.LlmEvaluator = LlmEvaluator;
3665
+ exports.ParametersLoader = ParametersLoader;
3409
3666
  exports.ProtocolCapabilities = ProtocolCapabilities;
3410
3667
  exports.ProtocolConfig = ProtocolConfig;
3411
3668
  exports.REDACTED = REDACTED;