@mappies/miniwat-eval 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -23,8 +23,8 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
23
23
  let _mappies_miniwat_agent_core = require("@mappies/miniwat-agent-core");
24
24
  let node_fs = require("node:fs");
25
25
  let node_path = require("node:path");
26
- let ajv = require("ajv");
27
26
  let yaml = require("yaml");
27
+ let ajv = require("ajv");
28
28
  let configurapi_handler_ws = require("configurapi-handler-ws");
29
29
  let node_util = require("node:util");
30
30
  let _mappies_miniwat_agent = require("@mappies/miniwat-agent");
@@ -296,6 +296,50 @@ var TurnSpec = class {
296
296
  this.location = new SourceLocation(data?.location);
297
297
  }
298
298
  };
299
+ // src/entities/protocolCapabilities.ts
300
+ var ProtocolCapabilities = class {
301
+ toolUses;
302
+ toolResults;
303
+ tokenUsage;
304
+ constructor(data) {
305
+ this.toolUses = data?.toolUses ?? false;
306
+ this.toolResults = data?.toolResults ?? false;
307
+ this.tokenUsage = data?.tokenUsage ?? false;
308
+ }
309
+ };
310
+ // src/interfaces/requestFormat.ts
311
+ const RequestFormat = { Json: "json" };
312
+ // src/entities/requestConfig.ts
313
+ var RequestConfig = class {
314
+ format;
315
+ value;
316
+ inputPath;
317
+ constructor(data) {
318
+ this.format = data?.format ?? RequestFormat.Json;
319
+ this.value = data?.value;
320
+ this.inputPath = data?.inputPath;
321
+ }
322
+ };
323
+ // src/entities/protocolConfig.ts
324
+ var ProtocolConfig = class {
325
+ request;
326
+ capabilities;
327
+ constructor(data) {
328
+ this.request = new RequestConfig(data?.request);
329
+ this.capabilities = new ProtocolCapabilities(data?.capabilities);
330
+ }
331
+ };
332
+ // src/entities/webSocketConfig.ts
333
+ var WebSocketConfig = class {
334
+ endpoint;
335
+ connection;
336
+ protocol;
337
+ constructor(data) {
338
+ this.endpoint = data?.endpoint ?? "";
339
+ this.connection = new ConnectionConfig(data?.connection);
340
+ this.protocol = new ProtocolConfig(data?.protocol);
341
+ }
342
+ };
299
343
  // src/entities/conversationSpec.ts
300
344
  var ConversationSpec = class {
301
345
  version;
@@ -304,6 +348,7 @@ var ConversationSpec = class {
304
348
  turns;
305
349
  conversation;
306
350
  location;
351
+ websocket;
307
352
  secrets;
308
353
  constructor(data) {
309
354
  this.version = data?.version ?? 1;
@@ -312,6 +357,7 @@ var ConversationSpec = class {
312
357
  this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
313
358
  this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
314
359
  this.location = new SourceLocation(data?.location);
360
+ this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
315
361
  this.secrets = [...data?.secrets ?? []];
316
362
  }
317
363
  };
@@ -413,55 +459,12 @@ var ReportConfig = class {
413
459
  this.formats = data?.formats ?? [];
414
460
  }
415
461
  };
416
- // src/entities/protocolCapabilities.ts
417
- var ProtocolCapabilities = class {
418
- toolUses;
419
- toolResults;
420
- tokenUsage;
421
- constructor(data) {
422
- this.toolUses = data?.toolUses ?? false;
423
- this.toolResults = data?.toolResults ?? false;
424
- this.tokenUsage = data?.tokenUsage ?? false;
425
- }
426
- };
427
- // src/interfaces/requestFormat.ts
428
- const RequestFormat = { Json: "json" };
429
- // src/entities/requestConfig.ts
430
- var RequestConfig = class {
431
- format;
432
- value;
433
- inputPath;
434
- constructor(data) {
435
- this.format = data?.format ?? RequestFormat.Json;
436
- this.value = data?.value;
437
- this.inputPath = data?.inputPath;
438
- }
439
- };
440
- // src/entities/protocolConfig.ts
441
- var ProtocolConfig = class {
442
- request;
443
- capabilities;
444
- constructor(data) {
445
- this.request = new RequestConfig(data?.request);
446
- this.capabilities = new ProtocolCapabilities(data?.capabilities);
447
- }
448
- };
449
- // src/entities/webSocketConfig.ts
450
- var WebSocketConfig = class {
451
- endpoint;
452
- connection;
453
- protocol;
454
- constructor(data) {
455
- this.endpoint = data?.endpoint ?? "";
456
- this.connection = new ConnectionConfig(data?.connection);
457
- this.protocol = new ProtocolConfig(data?.protocol);
458
- }
459
- };
460
462
  // src/entities/evalConfig.ts
461
463
  var EvalConfig = class {
462
464
  version;
463
465
  configPath;
464
466
  parametersPath;
467
+ variables;
465
468
  websocket;
466
469
  execution;
467
470
  evaluator;
@@ -471,6 +474,7 @@ var EvalConfig = class {
471
474
  this.version = data?.version ?? 1;
472
475
  this.configPath = data?.configPath ?? "";
473
476
  this.parametersPath = data?.parametersPath ?? "";
477
+ this.variables = { ...data?.variables ?? {} };
474
478
  this.websocket = new WebSocketConfig(data?.websocket);
475
479
  this.execution = new ExecutionConfig(data?.execution);
476
480
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -782,6 +786,11 @@ var eval_config_schema_default = {
782
786
  ],
783
787
  properties: {
784
788
  "version": { "const": 1 },
789
+ "variables": {
790
+ "type": "object",
791
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
792
+ "additionalProperties": { "type": "string" }
793
+ },
785
794
  "websocket": {
786
795
  "type": "object",
787
796
  "additionalProperties": false,
@@ -940,18 +949,80 @@ var eval_config_schema_default = {
940
949
  }
941
950
  }
942
951
  };
952
+ // src/yaml/yamlDocument.ts
953
+ var YamlDocument = class YamlDocument {
954
+ file;
955
+ document;
956
+ lineCounter;
957
+ value;
958
+ errors;
959
+ constructor(file, document, lineCounter) {
960
+ this.file = file;
961
+ this.document = document;
962
+ this.lineCounter = lineCounter;
963
+ this.value = document.toJS();
964
+ this.errors = document.errors.map((error) => new ValidationIssue({
965
+ message: error.message.split("\n")[0],
966
+ location: this.toLocation(error.pos[0], [])
967
+ }));
968
+ }
969
+ locate(path) {
970
+ return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
971
+ }
972
+ locateKey(path, key) {
973
+ const parent = this.findNode(path);
974
+ if ((0, yaml.isMap)(parent)) {
975
+ const pair = parent.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === key);
976
+ if (pair && (0, yaml.isScalar)(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
977
+ }
978
+ return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
979
+ }
980
+ static formatPath(path) {
981
+ return path.map((segment, index) => {
982
+ if (typeof segment === "number") return `[${segment}]`;
983
+ return index === 0 ? segment : `.${segment}`;
984
+ }).join("");
985
+ }
986
+ findNode(path) {
987
+ let node = this.document.contents ?? void 0;
988
+ for (const segment of path) {
989
+ let next;
990
+ if ((0, yaml.isMap)(node)) {
991
+ const pair = node.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === String(segment));
992
+ next = pair?.value ?? pair?.key;
993
+ } else if ((0, yaml.isSeq)(node) && typeof segment === "number") next = node.items[segment];
994
+ if (!next) break;
995
+ node = next;
996
+ }
997
+ return node;
998
+ }
999
+ toLocation(offset, path) {
1000
+ const position = this.lineCounter.linePos(offset);
1001
+ return new SourceLocation({
1002
+ file: this.file,
1003
+ line: position.line,
1004
+ column: position.col,
1005
+ path: YamlDocument.formatPath(path)
1006
+ });
1007
+ }
1008
+ };
943
1009
  // src/yaml/environmentResolver.ts
944
1010
  const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
945
1011
  const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
946
1012
  var EnvironmentResolver = class EnvironmentResolver {
947
1013
  environment;
948
1014
  parameters;
949
- constructor(environment = process.env, parameters = {}) {
1015
+ levels;
1016
+ constructor(environment = process.env, parameters = {}, levels = []) {
950
1017
  this.environment = environment;
951
1018
  this.parameters = parameters;
1019
+ this.levels = levels;
1020
+ }
1021
+ withLevels(levels) {
1022
+ return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
952
1023
  }
953
- resolve(value, document, issues, secrets = []) {
954
- return this.resolveValue(value, [], document, issues, secrets);
1024
+ resolve(value, document, issues, secrets = [], options = {}) {
1025
+ return this.resolveValue(value, [], document, issues, secrets, options);
955
1026
  }
956
1027
  static hasReference(value) {
957
1028
  return new RegExp(REFERENCE.source).test(value);
@@ -959,22 +1030,38 @@ var EnvironmentResolver = class EnvironmentResolver {
959
1030
  static isSecretName(name) {
960
1031
  return SECRET_NAME.test(name);
961
1032
  }
962
- resolveValue(value, path, document, issues, secrets) {
1033
+ lookup(name) {
1034
+ for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
1035
+ return this.environment[name];
1036
+ }
1037
+ sources() {
1038
+ const sources = [
1039
+ ...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
1040
+ ...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
1041
+ "environment variable"
1042
+ ];
1043
+ return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
1044
+ }
1045
+ resolveValue(value, path, document, issues, secrets, options) {
963
1046
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
964
- const resolved = Object.hasOwn(this.parameters, name) ? this.parameters[name] : this.environment[name];
1047
+ options.used?.add(name);
1048
+ const resolved = this.lookup(name);
965
1049
  if (resolved === void 0) {
966
- const source = Object.keys(this.parameters).length > 0 ? "parameter or environment variable" : "environment variable";
1050
+ if (options.optional?.(path)) {
1051
+ options.unresolved?.add(YamlDocument.formatPath(path));
1052
+ return reference;
1053
+ }
967
1054
  issues.push(new ValidationIssue({
968
- message: `${source} '${name}' is not set`,
1055
+ message: `${this.sources()} '${name}' is not set`,
969
1056
  location: document.locate(path)
970
1057
  }));
971
1058
  return reference;
972
1059
  }
973
- if (EnvironmentResolver.isSecretName(name) && !secrets.includes(resolved)) secrets.push(resolved);
1060
+ if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
974
1061
  return resolved;
975
1062
  });
976
- if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
977
- if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
1063
+ if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
1064
+ if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
978
1065
  return value;
979
1066
  }
980
1067
  };
@@ -1058,63 +1145,6 @@ var SchemaValidator = class SchemaValidator {
1058
1145
  return path;
1059
1146
  }
1060
1147
  };
1061
- // src/yaml/yamlDocument.ts
1062
- var YamlDocument = class YamlDocument {
1063
- file;
1064
- document;
1065
- lineCounter;
1066
- value;
1067
- errors;
1068
- constructor(file, document, lineCounter) {
1069
- this.file = file;
1070
- this.document = document;
1071
- this.lineCounter = lineCounter;
1072
- this.value = document.toJS();
1073
- this.errors = document.errors.map((error) => new ValidationIssue({
1074
- message: error.message.split("\n")[0],
1075
- location: this.toLocation(error.pos[0], [])
1076
- }));
1077
- }
1078
- locate(path) {
1079
- return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
1080
- }
1081
- locateKey(path, key) {
1082
- const parent = this.findNode(path);
1083
- if ((0, yaml.isMap)(parent)) {
1084
- const pair = parent.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === key);
1085
- if (pair && (0, yaml.isScalar)(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
1086
- }
1087
- return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
1088
- }
1089
- static formatPath(path) {
1090
- return path.map((segment, index) => {
1091
- if (typeof segment === "number") return `[${segment}]`;
1092
- return index === 0 ? segment : `.${segment}`;
1093
- }).join("");
1094
- }
1095
- findNode(path) {
1096
- let node = this.document.contents ?? void 0;
1097
- for (const segment of path) {
1098
- let next;
1099
- if ((0, yaml.isMap)(node)) {
1100
- const pair = node.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === String(segment));
1101
- next = pair?.value ?? pair?.key;
1102
- } else if ((0, yaml.isSeq)(node) && typeof segment === "number") next = node.items[segment];
1103
- if (!next) break;
1104
- node = next;
1105
- }
1106
- return node;
1107
- }
1108
- toLocation(offset, path) {
1109
- const position = this.lineCounter.linePos(offset);
1110
- return new SourceLocation({
1111
- file: this.file,
1112
- line: position.line,
1113
- column: position.col,
1114
- path: YamlDocument.formatPath(path)
1115
- });
1116
- }
1117
- };
1118
1148
  // src/yaml/yamlDocumentLoader.ts
1119
1149
  var YamlDocumentLoader = class {
1120
1150
  load(filePath, displayPath) {
@@ -1133,11 +1163,133 @@ var YamlDocumentLoader = class {
1133
1163
  function isRecord(value) {
1134
1164
  return value !== null && typeof value === "object" && !Array.isArray(value);
1135
1165
  }
1166
+ // src/yaml/variablesReader.ts
1167
+ const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
1168
+ var VariablesReader = class {
1169
+ read(document, resolver, issues, secrets) {
1170
+ const raw = isRecord(document.value) ? document.value.variables : void 0;
1171
+ if (raw === void 0) return {};
1172
+ if (!isRecord(raw)) {
1173
+ issues.push(new ValidationIssue({
1174
+ message: "must be object",
1175
+ location: document.locate(["variables"])
1176
+ }));
1177
+ return {};
1178
+ }
1179
+ const variables = {};
1180
+ for (const [name, value] of Object.entries(raw)) {
1181
+ if (!VARIABLE_NAME.test(name)) {
1182
+ issues.push(new ValidationIssue({
1183
+ message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
1184
+ location: document.locateKey(["variables"], name)
1185
+ }));
1186
+ continue;
1187
+ }
1188
+ if (typeof value !== "string") {
1189
+ issues.push(new ValidationIssue({
1190
+ message: "must be string",
1191
+ location: document.locate(["variables", name])
1192
+ }));
1193
+ continue;
1194
+ }
1195
+ const before = issues.length;
1196
+ const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
1197
+ if (issues.length > before) continue;
1198
+ variables[name] = resolved.variables[name];
1199
+ if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
1200
+ }
1201
+ return variables;
1202
+ }
1203
+ };
1204
+ // src/config/webSocketResolver.ts
1205
+ const WEBSOCKET_SCHEMA = {
1206
+ ...eval_config_schema_default,
1207
+ required: ["websocket"],
1208
+ additionalProperties: true,
1209
+ properties: { websocket: eval_config_schema_default.properties.websocket }
1210
+ };
1211
+ var WebSocketResolver = class WebSocketResolver {
1212
+ schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
1213
+ resolve(document, resolver, issues, secrets, used, defer) {
1214
+ const raw = isRecord(document.value) ? document.value.websocket : void 0;
1215
+ const unresolved = /* @__PURE__ */ new Set();
1216
+ const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
1217
+ const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
1218
+ used,
1219
+ optional,
1220
+ unresolved
1221
+ });
1222
+ const subprotocols = WebSocketResolver.subprotocols(raw);
1223
+ if (subprotocols !== void 0) {
1224
+ const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
1225
+ const set = !unresolved.has(YamlDocument.formatPath([
1226
+ "websocket",
1227
+ "connection",
1228
+ "subprotocols",
1229
+ index
1230
+ ]));
1231
+ const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
1232
+ return set && !empty;
1233
+ });
1234
+ value.websocket.connection.subprotocols = kept;
1235
+ }
1236
+ const prefix = YamlDocument.formatPath([
1237
+ "websocket",
1238
+ "connection",
1239
+ "subprotocols"
1240
+ ]);
1241
+ return {
1242
+ value,
1243
+ deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
1244
+ };
1245
+ }
1246
+ validate(value, document) {
1247
+ const issues = this.schemaValidator.validate(value, document);
1248
+ if (issues.length > 0) return issues;
1249
+ return WebSocketResolver.validateInputPath(value, document);
1250
+ }
1251
+ static validateInputPath(value, document) {
1252
+ const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
1253
+ if (typeof request.inputPath !== "string") return [];
1254
+ const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
1255
+ const last = tokens.pop();
1256
+ let parent = request.value;
1257
+ for (const token of tokens) parent = WebSocketResolver.child(parent, token);
1258
+ if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
1259
+ return [new ValidationIssue({
1260
+ message: "must point to a missing or string value whose parent exists in request.value",
1261
+ location: document.locate([
1262
+ "websocket",
1263
+ "protocol",
1264
+ "request",
1265
+ "inputPath"
1266
+ ])
1267
+ })];
1268
+ }
1269
+ static section(value, ...path) {
1270
+ let current = value;
1271
+ for (const key of path) current = WebSocketResolver.child(current, key);
1272
+ return isRecord(current) ? current : {};
1273
+ }
1274
+ static child(value, key) {
1275
+ if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
1276
+ return isRecord(value) ? value[key] : void 0;
1277
+ }
1278
+ static isSubprotocol(path) {
1279
+ return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
1280
+ }
1281
+ static subprotocols(websocket) {
1282
+ const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
1283
+ return Array.isArray(subprotocols) ? subprotocols : void 0;
1284
+ }
1285
+ };
1136
1286
  // src/config/evalConfigLoader.ts
1137
1287
  var EvalConfigLoader = class EvalConfigLoader {
1138
1288
  cwd;
1139
1289
  documentLoader = new YamlDocumentLoader();
1140
1290
  schemaValidator = new SchemaValidator(eval_config_schema_default);
1291
+ variablesReader = new VariablesReader();
1292
+ webSocketResolver = new WebSocketResolver();
1141
1293
  environmentResolver;
1142
1294
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1143
1295
  this.cwd = cwd;
@@ -1148,19 +1300,47 @@ var EvalConfigLoader = class EvalConfigLoader {
1148
1300
  const issues = [...document.errors];
1149
1301
  if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
1150
1302
  const named = [];
1151
- const value = issues.length === 0 ? this.environmentResolver.resolve(document.value, document, issues, named) : void 0;
1152
- if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
1153
- if (issues.length === 0) issues.push(...this.validateInputPath(value, document));
1303
+ const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
1304
+ let value = {};
1305
+ if (issues.length === 0) {
1306
+ const resolver = this.environmentResolver.withLevels([variables]);
1307
+ const { websocket: _websocket, variables: _variables, ...rest } = document.value;
1308
+ const resolved = resolver.resolve(rest, document, issues, named);
1309
+ const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
1310
+ value = {
1311
+ ...resolved,
1312
+ ..._variables === void 0 ? {} : { variables },
1313
+ ..._websocket === void 0 ? {} : websocket.value
1314
+ };
1315
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
1316
+ if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
1317
+ }
1154
1318
  if (issues.length > 0) throw new EvalError({
1155
1319
  category: ErrorCategory.Configuration,
1156
1320
  message: "invalid configuration",
1157
1321
  issues
1158
1322
  });
1159
- const config = this.build(value, configPath, named);
1323
+ const config = this.build(value, configPath, variables, named);
1160
1324
  this.applyRepetition(config, override, value, document);
1161
1325
  EvalConfigLoader.applyReport(config, report);
1162
1326
  return config;
1163
1327
  }
1328
+ static buildWebSocket(value) {
1329
+ const section = (...path) => WebSocketResolver.section(value, ...path);
1330
+ const connection = section("websocket", "connection");
1331
+ const request = section("websocket", "protocol", "request");
1332
+ return new WebSocketConfig({
1333
+ endpoint: WebSocketResolver.section(value, "websocket").endpoint,
1334
+ connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
1335
+ protocol: new ProtocolConfig({
1336
+ request: new RequestConfig({
1337
+ ...request,
1338
+ format: request.format ?? RequestFormat.Json
1339
+ }),
1340
+ capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
1341
+ })
1342
+ });
1343
+ }
1164
1344
  static applyReport(config, report) {
1165
1345
  if (report.formats === void 0) return;
1166
1346
  const known = Object.values(ReportFormat);
@@ -1182,45 +1362,17 @@ var EvalConfigLoader = class EvalConfigLoader {
1182
1362
  location: document.locate(["evaluator", "apiKey"])
1183
1363
  })];
1184
1364
  }
1185
- validateInputPath(value, document) {
1186
- const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
1187
- if (typeof request.inputPath !== "string") return [];
1188
- const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
1189
- const last = tokens.pop();
1190
- let parent = request.value;
1191
- for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
1192
- if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
1193
- return [new ValidationIssue({
1194
- message: "must point to a missing or string value whose parent exists in request.value",
1195
- location: document.locate([
1196
- "websocket",
1197
- "protocol",
1198
- "request",
1199
- "inputPath"
1200
- ])
1201
- })];
1202
- }
1203
- build(value, configPath, named) {
1365
+ build(value, configPath, variables, named) {
1204
1366
  const section = (...path) => EvalConfigLoader.section(value, ...path);
1205
1367
  const configDirectory = (0, node_path.dirname)(configPath);
1206
- const connection = section("websocket", "connection");
1207
- const request = section("websocket", "protocol", "request");
1368
+ const websocket = EvalConfigLoader.buildWebSocket(value);
1208
1369
  const evaluator = section("evaluator");
1209
1370
  const report = section("report");
1210
1371
  return new EvalConfig({
1211
1372
  version: value.version,
1212
1373
  configPath,
1213
- websocket: new WebSocketConfig({
1214
- endpoint: EvalConfigLoader.section(value, "websocket").endpoint,
1215
- connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
1216
- protocol: new ProtocolConfig({
1217
- request: new RequestConfig({
1218
- ...request,
1219
- format: request.format ?? RequestFormat.Json
1220
- }),
1221
- capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
1222
- })
1223
- }),
1374
+ variables,
1375
+ websocket,
1224
1376
  execution: new ExecutionConfig({
1225
1377
  ...DEFAULT_CONFIG.execution,
1226
1378
  ...section("execution"),
@@ -1241,7 +1393,7 @@ var EvalConfigLoader = class EvalConfigLoader {
1241
1393
  directory: (0, node_path.resolve)(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
1242
1394
  formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
1243
1395
  }),
1244
- secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols ?? [], named)
1396
+ secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
1245
1397
  });
1246
1398
  }
1247
1399
  applyRepetition(config, override, value, document) {
@@ -1294,13 +1446,7 @@ var EvalConfigLoader = class EvalConfigLoader {
1294
1446
  ])];
1295
1447
  }
1296
1448
  static section(value, ...path) {
1297
- let current = value;
1298
- for (const key of path) current = EvalConfigLoader.child(current, key);
1299
- return isRecord(current) ? current : {};
1300
- }
1301
- static child(value, key) {
1302
- if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
1303
- return isRecord(value) ? value[key] : void 0;
1449
+ return WebSocketResolver.section(value, ...path);
1304
1450
  }
1305
1451
  };
1306
1452
  // src/config/parametersLoader.ts
@@ -1352,6 +1498,11 @@ var conversation_schema_default = {
1352
1498
  "version": { "const": 1 },
1353
1499
  "name": { "$ref": "#/definitions/text" },
1354
1500
  "description": { "type": "string" },
1501
+ "variables": {
1502
+ "type": "object",
1503
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
1504
+ "additionalProperties": { "type": "string" }
1505
+ },
1355
1506
  "turns": {
1356
1507
  "type": "array",
1357
1508
  "minItems": 1,
@@ -1364,8 +1515,8 @@ var conversation_schema_default = {
1364
1515
  "properties": { "conversation": {
1365
1516
  "type": "object",
1366
1517
  "additionalProperties": false,
1367
- "required": ["criteria"],
1368
- "properties": { "criteria": { "$ref": "#/definitions/text" } }
1518
+ "required": ["should"],
1519
+ "properties": { "should": { "$ref": "#/definitions/text" } }
1369
1520
  } }
1370
1521
  }
1371
1522
  },
@@ -1408,12 +1559,12 @@ var conversation_schema_default = {
1408
1559
  }
1409
1560
  },
1410
1561
  "response": {
1411
- "description": "A string is shorthand for criteria.",
1562
+ "description": "A string is shorthand for should.",
1412
1563
  "type": ["string", "object"],
1413
1564
  "pattern": "\\S",
1414
1565
  "additionalProperties": false,
1415
1566
  "properties": {
1416
- "criteria": { "$ref": "#/definitions/text" },
1567
+ "should": { "$ref": "#/definitions/text" },
1417
1568
  "contains": { "$ref": "#/definitions/textList" },
1418
1569
  "notContains": { "$ref": "#/definitions/textList" }
1419
1570
  }
@@ -1445,7 +1596,7 @@ var conversation_schema_default = {
1445
1596
  };
1446
1597
  // src/scenarios/conversationValidator.ts
1447
1598
  const RESPONSE_KEYS = [
1448
- "criteria",
1599
+ "should",
1449
1600
  "contains",
1450
1601
  "notContains"
1451
1602
  ];
@@ -1468,7 +1619,7 @@ var ConversationValidator = class ConversationValidator {
1468
1619
  validateResponse(response, path, document, issues) {
1469
1620
  if (!RESPONSE_KEYS.some((key) => key in response)) {
1470
1621
  issues.push(new ValidationIssue({
1471
- message: "response must have at least one of: criteria, contains, notContains",
1622
+ message: "response must have at least one of: should, contains, notContains",
1472
1623
  location: document.locate(path)
1473
1624
  }));
1474
1625
  return;
@@ -1511,11 +1662,13 @@ var ConversationValidator = class ConversationValidator {
1511
1662
  };
1512
1663
  // src/scenarios/scenarioLoader.ts
1513
1664
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
1514
- var ScenarioLoader = class {
1665
+ var ScenarioLoader = class ScenarioLoader {
1515
1666
  cwd;
1516
1667
  documentLoader = new YamlDocumentLoader();
1517
1668
  schemaValidator = new SchemaValidator(conversation_schema_default);
1518
1669
  conversationValidator = new ConversationValidator();
1670
+ variablesReader = new VariablesReader();
1671
+ webSocketResolver = new WebSocketResolver();
1519
1672
  environmentResolver;
1520
1673
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1521
1674
  this.cwd = cwd;
@@ -1523,18 +1676,52 @@ var ScenarioLoader = class {
1523
1676
  }
1524
1677
  load(target, config) {
1525
1678
  const files = this.discover((0, node_path.resolve)(this.cwd, target), config);
1679
+ const configDocument = (0, node_fs.existsSync)(config.configPath) ? this.documentLoader.load(config.configPath, (0, node_path.relative)(this.cwd, config.configPath)) : void 0;
1526
1680
  const issues = [];
1681
+ const connectionIssues = [];
1527
1682
  const specs = [];
1528
1683
  for (const file of files) {
1529
1684
  const document = this.documentLoader.load(file, (0, node_path.relative)(this.cwd, file));
1530
1685
  const fileIssues = [...document.errors];
1531
1686
  const secrets = [];
1532
- const value = fileIssues.length === 0 ? this.environmentResolver.resolve(document.value, document, fileIssues, secrets) : void 0;
1687
+ const used = /* @__PURE__ */ new Set();
1688
+ const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
1689
+ const resolver = this.environmentResolver.withLevels([variables, config.variables]);
1690
+ let value;
1691
+ if (fileIssues.length === 0) {
1692
+ const { variables: raw, ...rest } = document.value;
1693
+ value = {
1694
+ ...resolver.resolve(rest, document, fileIssues, secrets, { used }),
1695
+ ...raw === void 0 ? {} : { variables }
1696
+ };
1697
+ }
1698
+ let websocket;
1699
+ const websocketIssues = [];
1700
+ if (fileIssues.length === 0 && configDocument !== void 0) {
1701
+ const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
1702
+ if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
1703
+ connectionIssues.push({
1704
+ file: document,
1705
+ issues: websocketIssues
1706
+ });
1707
+ if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
1708
+ }
1533
1709
  if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
1534
1710
  if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
1535
- if (fileIssues.length === 0) specs.push(this.build(value, document, secrets));
1711
+ if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
1712
+ if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
1536
1713
  else issues.push(...fileIssues);
1537
1714
  }
1715
+ const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
1716
+ if (shared.length > 0) throw new EvalError({
1717
+ category: ErrorCategory.Configuration,
1718
+ message: "invalid configuration",
1719
+ issues: shared
1720
+ });
1721
+ for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
1722
+ message: issue.toString(),
1723
+ location: ScenarioLoader.variablesLocation(entry.file)
1724
+ }));
1538
1725
  if (issues.length > 0) throw new EvalError({
1539
1726
  category: ErrorCategory.Specification,
1540
1727
  message: "invalid scenario specification",
@@ -1542,6 +1729,23 @@ var ScenarioLoader = class {
1542
1729
  });
1543
1730
  return specs;
1544
1731
  }
1732
+ static shared(lists) {
1733
+ if (lists.length === 0) return [];
1734
+ const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
1735
+ return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
1736
+ }
1737
+ static variablesLocation(document) {
1738
+ return new SourceLocation({
1739
+ ...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
1740
+ path: ""
1741
+ });
1742
+ }
1743
+ static unusedVariables(variables, used, document) {
1744
+ return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
1745
+ message: `variable '${name}' is not used by the scenario or the websocket configuration`,
1746
+ location: document.locateKey(["variables"], name)
1747
+ }));
1748
+ }
1545
1749
  discover(target, config) {
1546
1750
  if (!(0, node_fs.existsSync)(target)) throw new EvalError({
1547
1751
  category: ErrorCategory.Specification,
@@ -1566,7 +1770,7 @@ var ScenarioLoader = class {
1566
1770
  }
1567
1771
  return files;
1568
1772
  }
1569
- build(value, document, secrets) {
1773
+ build(value, document, websocket, secrets) {
1570
1774
  const scenario = value;
1571
1775
  const turns = scenario.turns;
1572
1776
  const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
@@ -1576,10 +1780,11 @@ var ScenarioLoader = class {
1576
1780
  description: typeof scenario.description === "string" ? scenario.description : "",
1577
1781
  turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
1578
1782
  conversation: conversation ? new ConversationExpectation({
1579
- criteria: conversation.criteria,
1783
+ criteria: conversation.should,
1580
1784
  location: document.locate(["expect", "conversation"])
1581
1785
  }) : void 0,
1582
1786
  location: document.locate([]),
1787
+ websocket,
1583
1788
  secrets
1584
1789
  });
1585
1790
  }
@@ -1605,7 +1810,7 @@ var ScenarioLoader = class {
1605
1810
  });
1606
1811
  const fields = response;
1607
1812
  return new ResponseExpectation({
1608
- criteria: fields.criteria,
1813
+ criteria: fields.should,
1609
1814
  contains: fields.contains ?? [],
1610
1815
  notContains: fields.notContains ?? [],
1611
1816
  location
@@ -2428,7 +2633,11 @@ var ConversationRunner = class {
2428
2633
  async run(spec, signal = NEVER_ABORTED) {
2429
2634
  const startedAt = Date.now();
2430
2635
  const outcomes = [];
2431
- const assertions = new AssertionRunner(this.config.websocket.protocol.capabilities);
2636
+ const config = new EvalConfig({
2637
+ ...this.config,
2638
+ websocket: spec.websocket ?? this.config.websocket
2639
+ });
2640
+ const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
2432
2641
  const history = [];
2433
2642
  const evaluate = this.semanticEvaluator(signal);
2434
2643
  const inline = !this.config.execution.continueOnFailure;
@@ -2453,7 +2662,7 @@ var ConversationRunner = class {
2453
2662
  try {
2454
2663
  const steps = (async () => {
2455
2664
  for (let attempt = 0;; attempt++) {
2456
- connection = this.createConnection(this.config);
2665
+ connection = this.createConnection(config);
2457
2666
  connectionAttempts++;
2458
2667
  try {
2459
2668
  await connection.connect();
@@ -2947,7 +3156,7 @@ var ScenarioRunner = class {
2947
3156
  required: minPass,
2948
3157
  attempts,
2949
3158
  tests: this.results.tests(attempts, minPass),
2950
- agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
3159
+ agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
2951
3160
  evaluatorUsage: this.tokens.evaluator(attempts)
2952
3161
  });
2953
3162
  }
@@ -3038,7 +3247,9 @@ var ConsoleReporter = class ConsoleReporter {
3038
3247
  const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
3039
3248
  if (!this.verbose) return line;
3040
3249
  if (result.attempts.length === 0) return `${line}\n`;
3041
- const lines = [""];
3250
+ const websocket = spec.websocket;
3251
+ const subprotocols = websocket?.connection.subprotocols.length ?? 0;
3252
+ const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
3042
3253
  for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3043
3254
  lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
3044
3255
  lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
@@ -3251,11 +3462,16 @@ var JsonReporter = class JsonReporter {
3251
3462
  };
3252
3463
  }
3253
3464
  scenario(result, config) {
3254
- const capability = config.websocket.protocol.capabilities.tokenUsage;
3465
+ const websocket = result.spec.websocket ?? config.websocket;
3466
+ const capability = websocket.protocol.capabilities.tokenUsage;
3255
3467
  return {
3256
3468
  name: result.spec.name,
3257
3469
  file: result.spec.location.file,
3258
3470
  status: result.status,
3471
+ connection: {
3472
+ endpoint: websocket.endpoint,
3473
+ subprotocols: websocket.connection.subprotocols.length
3474
+ },
3259
3475
  ...JsonReporter.counts(result),
3260
3476
  agentUsage: JsonReporter.usage(result.agentUsage),
3261
3477
  evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),