@mappies/miniwat-eval 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,8 +1,8 @@
1
1
  import { Message, ResponseMessage, TokenUsage } from "@mappies/miniwat-agent-core";
2
2
  import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
3
3
  import { dirname, extname, join, relative, resolve } from "node:path";
4
- import { Ajv } from "ajv";
5
4
  import { LineCounter, isMap, isPair, isScalar, isSeq, parseDocument } from "yaml";
5
+ import { Ajv } from "ajv";
6
6
  import handlerWs from "configurapi-handler-ws";
7
7
  import { isDeepStrictEqual, styleText } from "node:util";
8
8
  import { Agent, InMemoryAgentStateStore } from "@mappies/miniwat-agent";
@@ -273,6 +273,50 @@ var TurnSpec = class {
273
273
  this.location = new SourceLocation(data?.location);
274
274
  }
275
275
  };
276
+ // src/entities/protocolCapabilities.ts
277
+ var ProtocolCapabilities = class {
278
+ toolUses;
279
+ toolResults;
280
+ tokenUsage;
281
+ constructor(data) {
282
+ this.toolUses = data?.toolUses ?? false;
283
+ this.toolResults = data?.toolResults ?? false;
284
+ this.tokenUsage = data?.tokenUsage ?? false;
285
+ }
286
+ };
287
+ // src/interfaces/requestFormat.ts
288
+ const RequestFormat = { Json: "json" };
289
+ // src/entities/requestConfig.ts
290
+ var RequestConfig = class {
291
+ format;
292
+ value;
293
+ inputPath;
294
+ constructor(data) {
295
+ this.format = data?.format ?? RequestFormat.Json;
296
+ this.value = data?.value;
297
+ this.inputPath = data?.inputPath;
298
+ }
299
+ };
300
+ // src/entities/protocolConfig.ts
301
+ var ProtocolConfig = class {
302
+ request;
303
+ capabilities;
304
+ constructor(data) {
305
+ this.request = new RequestConfig(data?.request);
306
+ this.capabilities = new ProtocolCapabilities(data?.capabilities);
307
+ }
308
+ };
309
+ // src/entities/webSocketConfig.ts
310
+ var WebSocketConfig = class {
311
+ endpoint;
312
+ connection;
313
+ protocol;
314
+ constructor(data) {
315
+ this.endpoint = data?.endpoint ?? "";
316
+ this.connection = new ConnectionConfig(data?.connection);
317
+ this.protocol = new ProtocolConfig(data?.protocol);
318
+ }
319
+ };
276
320
  // src/entities/conversationSpec.ts
277
321
  var ConversationSpec = class {
278
322
  version;
@@ -281,6 +325,7 @@ var ConversationSpec = class {
281
325
  turns;
282
326
  conversation;
283
327
  location;
328
+ websocket;
284
329
  secrets;
285
330
  constructor(data) {
286
331
  this.version = data?.version ?? 1;
@@ -289,6 +334,7 @@ var ConversationSpec = class {
289
334
  this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
290
335
  this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
291
336
  this.location = new SourceLocation(data?.location);
337
+ this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
292
338
  this.secrets = [...data?.secrets ?? []];
293
339
  }
294
340
  };
@@ -390,55 +436,12 @@ var ReportConfig = class {
390
436
  this.formats = data?.formats ?? [];
391
437
  }
392
438
  };
393
- // src/entities/protocolCapabilities.ts
394
- var ProtocolCapabilities = class {
395
- toolUses;
396
- toolResults;
397
- tokenUsage;
398
- constructor(data) {
399
- this.toolUses = data?.toolUses ?? false;
400
- this.toolResults = data?.toolResults ?? false;
401
- this.tokenUsage = data?.tokenUsage ?? false;
402
- }
403
- };
404
- // src/interfaces/requestFormat.ts
405
- const RequestFormat = { Json: "json" };
406
- // src/entities/requestConfig.ts
407
- var RequestConfig = class {
408
- format;
409
- value;
410
- inputPath;
411
- constructor(data) {
412
- this.format = data?.format ?? RequestFormat.Json;
413
- this.value = data?.value;
414
- this.inputPath = data?.inputPath;
415
- }
416
- };
417
- // src/entities/protocolConfig.ts
418
- var ProtocolConfig = class {
419
- request;
420
- capabilities;
421
- constructor(data) {
422
- this.request = new RequestConfig(data?.request);
423
- this.capabilities = new ProtocolCapabilities(data?.capabilities);
424
- }
425
- };
426
- // src/entities/webSocketConfig.ts
427
- var WebSocketConfig = class {
428
- endpoint;
429
- connection;
430
- protocol;
431
- constructor(data) {
432
- this.endpoint = data?.endpoint ?? "";
433
- this.connection = new ConnectionConfig(data?.connection);
434
- this.protocol = new ProtocolConfig(data?.protocol);
435
- }
436
- };
437
439
  // src/entities/evalConfig.ts
438
440
  var EvalConfig = class {
439
441
  version;
440
442
  configPath;
441
443
  parametersPath;
444
+ variables;
442
445
  websocket;
443
446
  execution;
444
447
  evaluator;
@@ -448,6 +451,7 @@ var EvalConfig = class {
448
451
  this.version = data?.version ?? 1;
449
452
  this.configPath = data?.configPath ?? "";
450
453
  this.parametersPath = data?.parametersPath ?? "";
454
+ this.variables = { ...data?.variables ?? {} };
451
455
  this.websocket = new WebSocketConfig(data?.websocket);
452
456
  this.execution = new ExecutionConfig(data?.execution);
453
457
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -759,6 +763,11 @@ var eval_config_schema_default = {
759
763
  ],
760
764
  properties: {
761
765
  "version": { "const": 1 },
766
+ "variables": {
767
+ "type": "object",
768
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
769
+ "additionalProperties": { "type": "string" }
770
+ },
762
771
  "websocket": {
763
772
  "type": "object",
764
773
  "additionalProperties": false,
@@ -917,18 +926,80 @@ var eval_config_schema_default = {
917
926
  }
918
927
  }
919
928
  };
929
+ // src/yaml/yamlDocument.ts
930
+ var YamlDocument = class YamlDocument {
931
+ file;
932
+ document;
933
+ lineCounter;
934
+ value;
935
+ errors;
936
+ constructor(file, document, lineCounter) {
937
+ this.file = file;
938
+ this.document = document;
939
+ this.lineCounter = lineCounter;
940
+ this.value = document.toJS();
941
+ this.errors = document.errors.map((error) => new ValidationIssue({
942
+ message: error.message.split("\n")[0],
943
+ location: this.toLocation(error.pos[0], [])
944
+ }));
945
+ }
946
+ locate(path) {
947
+ return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
948
+ }
949
+ locateKey(path, key) {
950
+ const parent = this.findNode(path);
951
+ if (isMap(parent)) {
952
+ const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
953
+ if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
954
+ }
955
+ return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
956
+ }
957
+ static formatPath(path) {
958
+ return path.map((segment, index) => {
959
+ if (typeof segment === "number") return `[${segment}]`;
960
+ return index === 0 ? segment : `.${segment}`;
961
+ }).join("");
962
+ }
963
+ findNode(path) {
964
+ let node = this.document.contents ?? void 0;
965
+ for (const segment of path) {
966
+ let next;
967
+ if (isMap(node)) {
968
+ const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
969
+ next = pair?.value ?? pair?.key;
970
+ } else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
971
+ if (!next) break;
972
+ node = next;
973
+ }
974
+ return node;
975
+ }
976
+ toLocation(offset, path) {
977
+ const position = this.lineCounter.linePos(offset);
978
+ return new SourceLocation({
979
+ file: this.file,
980
+ line: position.line,
981
+ column: position.col,
982
+ path: YamlDocument.formatPath(path)
983
+ });
984
+ }
985
+ };
920
986
  // src/yaml/environmentResolver.ts
921
987
  const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
922
988
  const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
923
989
  var EnvironmentResolver = class EnvironmentResolver {
924
990
  environment;
925
991
  parameters;
926
- constructor(environment = process.env, parameters = {}) {
992
+ levels;
993
+ constructor(environment = process.env, parameters = {}, levels = []) {
927
994
  this.environment = environment;
928
995
  this.parameters = parameters;
996
+ this.levels = levels;
997
+ }
998
+ withLevels(levels) {
999
+ return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
929
1000
  }
930
- resolve(value, document, issues, secrets = []) {
931
- return this.resolveValue(value, [], document, issues, secrets);
1001
+ resolve(value, document, issues, secrets = [], options = {}) {
1002
+ return this.resolveValue(value, [], document, issues, secrets, options);
932
1003
  }
933
1004
  static hasReference(value) {
934
1005
  return new RegExp(REFERENCE.source).test(value);
@@ -936,22 +1007,38 @@ var EnvironmentResolver = class EnvironmentResolver {
936
1007
  static isSecretName(name) {
937
1008
  return SECRET_NAME.test(name);
938
1009
  }
939
- resolveValue(value, path, document, issues, secrets) {
1010
+ lookup(name) {
1011
+ for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
1012
+ return this.environment[name];
1013
+ }
1014
+ sources() {
1015
+ const sources = [
1016
+ ...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
1017
+ ...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
1018
+ "environment variable"
1019
+ ];
1020
+ return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
1021
+ }
1022
+ resolveValue(value, path, document, issues, secrets, options) {
940
1023
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
941
- const resolved = Object.hasOwn(this.parameters, name) ? this.parameters[name] : this.environment[name];
1024
+ options.used?.add(name);
1025
+ const resolved = this.lookup(name);
942
1026
  if (resolved === void 0) {
943
- const source = Object.keys(this.parameters).length > 0 ? "parameter or environment variable" : "environment variable";
1027
+ if (options.optional?.(path)) {
1028
+ options.unresolved?.add(YamlDocument.formatPath(path));
1029
+ return reference;
1030
+ }
944
1031
  issues.push(new ValidationIssue({
945
- message: `${source} '${name}' is not set`,
1032
+ message: `${this.sources()} '${name}' is not set`,
946
1033
  location: document.locate(path)
947
1034
  }));
948
1035
  return reference;
949
1036
  }
950
- if (EnvironmentResolver.isSecretName(name) && !secrets.includes(resolved)) secrets.push(resolved);
1037
+ if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
951
1038
  return resolved;
952
1039
  });
953
- if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
954
- if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
1040
+ if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
1041
+ if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
955
1042
  return value;
956
1043
  }
957
1044
  };
@@ -1035,63 +1122,6 @@ var SchemaValidator = class SchemaValidator {
1035
1122
  return path;
1036
1123
  }
1037
1124
  };
1038
- // src/yaml/yamlDocument.ts
1039
- var YamlDocument = class YamlDocument {
1040
- file;
1041
- document;
1042
- lineCounter;
1043
- value;
1044
- errors;
1045
- constructor(file, document, lineCounter) {
1046
- this.file = file;
1047
- this.document = document;
1048
- this.lineCounter = lineCounter;
1049
- this.value = document.toJS();
1050
- this.errors = document.errors.map((error) => new ValidationIssue({
1051
- message: error.message.split("\n")[0],
1052
- location: this.toLocation(error.pos[0], [])
1053
- }));
1054
- }
1055
- locate(path) {
1056
- return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
1057
- }
1058
- locateKey(path, key) {
1059
- const parent = this.findNode(path);
1060
- if (isMap(parent)) {
1061
- const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
1062
- if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
1063
- }
1064
- return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
1065
- }
1066
- static formatPath(path) {
1067
- return path.map((segment, index) => {
1068
- if (typeof segment === "number") return `[${segment}]`;
1069
- return index === 0 ? segment : `.${segment}`;
1070
- }).join("");
1071
- }
1072
- findNode(path) {
1073
- let node = this.document.contents ?? void 0;
1074
- for (const segment of path) {
1075
- let next;
1076
- if (isMap(node)) {
1077
- const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
1078
- next = pair?.value ?? pair?.key;
1079
- } else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
1080
- if (!next) break;
1081
- node = next;
1082
- }
1083
- return node;
1084
- }
1085
- toLocation(offset, path) {
1086
- const position = this.lineCounter.linePos(offset);
1087
- return new SourceLocation({
1088
- file: this.file,
1089
- line: position.line,
1090
- column: position.col,
1091
- path: YamlDocument.formatPath(path)
1092
- });
1093
- }
1094
- };
1095
1125
  // src/yaml/yamlDocumentLoader.ts
1096
1126
  var YamlDocumentLoader = class {
1097
1127
  load(filePath, displayPath) {
@@ -1110,11 +1140,133 @@ var YamlDocumentLoader = class {
1110
1140
  function isRecord(value) {
1111
1141
  return value !== null && typeof value === "object" && !Array.isArray(value);
1112
1142
  }
1143
+ // src/yaml/variablesReader.ts
1144
+ const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
1145
+ var VariablesReader = class {
1146
+ read(document, resolver, issues, secrets) {
1147
+ const raw = isRecord(document.value) ? document.value.variables : void 0;
1148
+ if (raw === void 0) return {};
1149
+ if (!isRecord(raw)) {
1150
+ issues.push(new ValidationIssue({
1151
+ message: "must be object",
1152
+ location: document.locate(["variables"])
1153
+ }));
1154
+ return {};
1155
+ }
1156
+ const variables = {};
1157
+ for (const [name, value] of Object.entries(raw)) {
1158
+ if (!VARIABLE_NAME.test(name)) {
1159
+ issues.push(new ValidationIssue({
1160
+ message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
1161
+ location: document.locateKey(["variables"], name)
1162
+ }));
1163
+ continue;
1164
+ }
1165
+ if (typeof value !== "string") {
1166
+ issues.push(new ValidationIssue({
1167
+ message: "must be string",
1168
+ location: document.locate(["variables", name])
1169
+ }));
1170
+ continue;
1171
+ }
1172
+ const before = issues.length;
1173
+ const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
1174
+ if (issues.length > before) continue;
1175
+ variables[name] = resolved.variables[name];
1176
+ if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
1177
+ }
1178
+ return variables;
1179
+ }
1180
+ };
1181
+ // src/config/webSocketResolver.ts
1182
+ const WEBSOCKET_SCHEMA = {
1183
+ ...eval_config_schema_default,
1184
+ required: ["websocket"],
1185
+ additionalProperties: true,
1186
+ properties: { websocket: eval_config_schema_default.properties.websocket }
1187
+ };
1188
+ var WebSocketResolver = class WebSocketResolver {
1189
+ schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
1190
+ resolve(document, resolver, issues, secrets, used, defer) {
1191
+ const raw = isRecord(document.value) ? document.value.websocket : void 0;
1192
+ const unresolved = /* @__PURE__ */ new Set();
1193
+ const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
1194
+ const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
1195
+ used,
1196
+ optional,
1197
+ unresolved
1198
+ });
1199
+ const subprotocols = WebSocketResolver.subprotocols(raw);
1200
+ if (subprotocols !== void 0) {
1201
+ const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
1202
+ const set = !unresolved.has(YamlDocument.formatPath([
1203
+ "websocket",
1204
+ "connection",
1205
+ "subprotocols",
1206
+ index
1207
+ ]));
1208
+ const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
1209
+ return set && !empty;
1210
+ });
1211
+ value.websocket.connection.subprotocols = kept;
1212
+ }
1213
+ const prefix = YamlDocument.formatPath([
1214
+ "websocket",
1215
+ "connection",
1216
+ "subprotocols"
1217
+ ]);
1218
+ return {
1219
+ value,
1220
+ deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
1221
+ };
1222
+ }
1223
+ validate(value, document) {
1224
+ const issues = this.schemaValidator.validate(value, document);
1225
+ if (issues.length > 0) return issues;
1226
+ return WebSocketResolver.validateInputPath(value, document);
1227
+ }
1228
+ static validateInputPath(value, document) {
1229
+ const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
1230
+ if (typeof request.inputPath !== "string") return [];
1231
+ const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
1232
+ const last = tokens.pop();
1233
+ let parent = request.value;
1234
+ for (const token of tokens) parent = WebSocketResolver.child(parent, token);
1235
+ if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
1236
+ return [new ValidationIssue({
1237
+ message: "must point to a missing or string value whose parent exists in request.value",
1238
+ location: document.locate([
1239
+ "websocket",
1240
+ "protocol",
1241
+ "request",
1242
+ "inputPath"
1243
+ ])
1244
+ })];
1245
+ }
1246
+ static section(value, ...path) {
1247
+ let current = value;
1248
+ for (const key of path) current = WebSocketResolver.child(current, key);
1249
+ return isRecord(current) ? current : {};
1250
+ }
1251
+ static child(value, key) {
1252
+ if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
1253
+ return isRecord(value) ? value[key] : void 0;
1254
+ }
1255
+ static isSubprotocol(path) {
1256
+ return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
1257
+ }
1258
+ static subprotocols(websocket) {
1259
+ const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
1260
+ return Array.isArray(subprotocols) ? subprotocols : void 0;
1261
+ }
1262
+ };
1113
1263
  // src/config/evalConfigLoader.ts
1114
1264
  var EvalConfigLoader = class EvalConfigLoader {
1115
1265
  cwd;
1116
1266
  documentLoader = new YamlDocumentLoader();
1117
1267
  schemaValidator = new SchemaValidator(eval_config_schema_default);
1268
+ variablesReader = new VariablesReader();
1269
+ webSocketResolver = new WebSocketResolver();
1118
1270
  environmentResolver;
1119
1271
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1120
1272
  this.cwd = cwd;
@@ -1125,19 +1277,47 @@ var EvalConfigLoader = class EvalConfigLoader {
1125
1277
  const issues = [...document.errors];
1126
1278
  if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
1127
1279
  const named = [];
1128
- const value = issues.length === 0 ? this.environmentResolver.resolve(document.value, document, issues, named) : void 0;
1129
- if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
1130
- if (issues.length === 0) issues.push(...this.validateInputPath(value, document));
1280
+ const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
1281
+ let value = {};
1282
+ if (issues.length === 0) {
1283
+ const resolver = this.environmentResolver.withLevels([variables]);
1284
+ const { websocket: _websocket, variables: _variables, ...rest } = document.value;
1285
+ const resolved = resolver.resolve(rest, document, issues, named);
1286
+ const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
1287
+ value = {
1288
+ ...resolved,
1289
+ ..._variables === void 0 ? {} : { variables },
1290
+ ..._websocket === void 0 ? {} : websocket.value
1291
+ };
1292
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
1293
+ if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
1294
+ }
1131
1295
  if (issues.length > 0) throw new EvalError({
1132
1296
  category: ErrorCategory.Configuration,
1133
1297
  message: "invalid configuration",
1134
1298
  issues
1135
1299
  });
1136
- const config = this.build(value, configPath, named);
1300
+ const config = this.build(value, configPath, variables, named);
1137
1301
  this.applyRepetition(config, override, value, document);
1138
1302
  EvalConfigLoader.applyReport(config, report);
1139
1303
  return config;
1140
1304
  }
1305
+ static buildWebSocket(value) {
1306
+ const section = (...path) => WebSocketResolver.section(value, ...path);
1307
+ const connection = section("websocket", "connection");
1308
+ const request = section("websocket", "protocol", "request");
1309
+ return new WebSocketConfig({
1310
+ endpoint: WebSocketResolver.section(value, "websocket").endpoint,
1311
+ connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
1312
+ protocol: new ProtocolConfig({
1313
+ request: new RequestConfig({
1314
+ ...request,
1315
+ format: request.format ?? RequestFormat.Json
1316
+ }),
1317
+ capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
1318
+ })
1319
+ });
1320
+ }
1141
1321
  static applyReport(config, report) {
1142
1322
  if (report.formats === void 0) return;
1143
1323
  const known = Object.values(ReportFormat);
@@ -1159,45 +1339,17 @@ var EvalConfigLoader = class EvalConfigLoader {
1159
1339
  location: document.locate(["evaluator", "apiKey"])
1160
1340
  })];
1161
1341
  }
1162
- validateInputPath(value, document) {
1163
- const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
1164
- if (typeof request.inputPath !== "string") return [];
1165
- const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
1166
- const last = tokens.pop();
1167
- let parent = request.value;
1168
- for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
1169
- if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
1170
- return [new ValidationIssue({
1171
- message: "must point to a missing or string value whose parent exists in request.value",
1172
- location: document.locate([
1173
- "websocket",
1174
- "protocol",
1175
- "request",
1176
- "inputPath"
1177
- ])
1178
- })];
1179
- }
1180
- build(value, configPath, named) {
1342
+ build(value, configPath, variables, named) {
1181
1343
  const section = (...path) => EvalConfigLoader.section(value, ...path);
1182
1344
  const configDirectory = dirname(configPath);
1183
- const connection = section("websocket", "connection");
1184
- const request = section("websocket", "protocol", "request");
1345
+ const websocket = EvalConfigLoader.buildWebSocket(value);
1185
1346
  const evaluator = section("evaluator");
1186
1347
  const report = section("report");
1187
1348
  return new EvalConfig({
1188
1349
  version: value.version,
1189
1350
  configPath,
1190
- websocket: new WebSocketConfig({
1191
- endpoint: EvalConfigLoader.section(value, "websocket").endpoint,
1192
- connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
1193
- protocol: new ProtocolConfig({
1194
- request: new RequestConfig({
1195
- ...request,
1196
- format: request.format ?? RequestFormat.Json
1197
- }),
1198
- capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
1199
- })
1200
- }),
1351
+ variables,
1352
+ websocket,
1201
1353
  execution: new ExecutionConfig({
1202
1354
  ...DEFAULT_CONFIG.execution,
1203
1355
  ...section("execution"),
@@ -1218,7 +1370,7 @@ var EvalConfigLoader = class EvalConfigLoader {
1218
1370
  directory: resolve(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
1219
1371
  formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
1220
1372
  }),
1221
- secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols ?? [], named)
1373
+ secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
1222
1374
  });
1223
1375
  }
1224
1376
  applyRepetition(config, override, value, document) {
@@ -1271,13 +1423,7 @@ var EvalConfigLoader = class EvalConfigLoader {
1271
1423
  ])];
1272
1424
  }
1273
1425
  static section(value, ...path) {
1274
- let current = value;
1275
- for (const key of path) current = EvalConfigLoader.child(current, key);
1276
- return isRecord(current) ? current : {};
1277
- }
1278
- static child(value, key) {
1279
- if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
1280
- return isRecord(value) ? value[key] : void 0;
1426
+ return WebSocketResolver.section(value, ...path);
1281
1427
  }
1282
1428
  };
1283
1429
  // src/config/parametersLoader.ts
@@ -1329,6 +1475,11 @@ var conversation_schema_default = {
1329
1475
  "version": { "const": 1 },
1330
1476
  "name": { "$ref": "#/definitions/text" },
1331
1477
  "description": { "type": "string" },
1478
+ "variables": {
1479
+ "type": "object",
1480
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
1481
+ "additionalProperties": { "type": "string" }
1482
+ },
1332
1483
  "turns": {
1333
1484
  "type": "array",
1334
1485
  "minItems": 1,
@@ -1341,8 +1492,8 @@ var conversation_schema_default = {
1341
1492
  "properties": { "conversation": {
1342
1493
  "type": "object",
1343
1494
  "additionalProperties": false,
1344
- "required": ["criteria"],
1345
- "properties": { "criteria": { "$ref": "#/definitions/text" } }
1495
+ "required": ["should"],
1496
+ "properties": { "should": { "$ref": "#/definitions/text" } }
1346
1497
  } }
1347
1498
  }
1348
1499
  },
@@ -1385,12 +1536,12 @@ var conversation_schema_default = {
1385
1536
  }
1386
1537
  },
1387
1538
  "response": {
1388
- "description": "A string is shorthand for criteria.",
1539
+ "description": "A string is shorthand for should.",
1389
1540
  "type": ["string", "object"],
1390
1541
  "pattern": "\\S",
1391
1542
  "additionalProperties": false,
1392
1543
  "properties": {
1393
- "criteria": { "$ref": "#/definitions/text" },
1544
+ "should": { "$ref": "#/definitions/text" },
1394
1545
  "contains": { "$ref": "#/definitions/textList" },
1395
1546
  "notContains": { "$ref": "#/definitions/textList" }
1396
1547
  }
@@ -1422,7 +1573,7 @@ var conversation_schema_default = {
1422
1573
  };
1423
1574
  // src/scenarios/conversationValidator.ts
1424
1575
  const RESPONSE_KEYS = [
1425
- "criteria",
1576
+ "should",
1426
1577
  "contains",
1427
1578
  "notContains"
1428
1579
  ];
@@ -1445,7 +1596,7 @@ var ConversationValidator = class ConversationValidator {
1445
1596
  validateResponse(response, path, document, issues) {
1446
1597
  if (!RESPONSE_KEYS.some((key) => key in response)) {
1447
1598
  issues.push(new ValidationIssue({
1448
- message: "response must have at least one of: criteria, contains, notContains",
1599
+ message: "response must have at least one of: should, contains, notContains",
1449
1600
  location: document.locate(path)
1450
1601
  }));
1451
1602
  return;
@@ -1488,11 +1639,13 @@ var ConversationValidator = class ConversationValidator {
1488
1639
  };
1489
1640
  // src/scenarios/scenarioLoader.ts
1490
1641
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
1491
- var ScenarioLoader = class {
1642
+ var ScenarioLoader = class ScenarioLoader {
1492
1643
  cwd;
1493
1644
  documentLoader = new YamlDocumentLoader();
1494
1645
  schemaValidator = new SchemaValidator(conversation_schema_default);
1495
1646
  conversationValidator = new ConversationValidator();
1647
+ variablesReader = new VariablesReader();
1648
+ webSocketResolver = new WebSocketResolver();
1496
1649
  environmentResolver;
1497
1650
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1498
1651
  this.cwd = cwd;
@@ -1500,18 +1653,52 @@ var ScenarioLoader = class {
1500
1653
  }
1501
1654
  load(target, config) {
1502
1655
  const files = this.discover(resolve(this.cwd, target), config);
1656
+ const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.cwd, config.configPath)) : void 0;
1503
1657
  const issues = [];
1658
+ const connectionIssues = [];
1504
1659
  const specs = [];
1505
1660
  for (const file of files) {
1506
1661
  const document = this.documentLoader.load(file, relative(this.cwd, file));
1507
1662
  const fileIssues = [...document.errors];
1508
1663
  const secrets = [];
1509
- const value = fileIssues.length === 0 ? this.environmentResolver.resolve(document.value, document, fileIssues, secrets) : void 0;
1664
+ const used = /* @__PURE__ */ new Set();
1665
+ const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
1666
+ const resolver = this.environmentResolver.withLevels([variables, config.variables]);
1667
+ let value;
1668
+ if (fileIssues.length === 0) {
1669
+ const { variables: raw, ...rest } = document.value;
1670
+ value = {
1671
+ ...resolver.resolve(rest, document, fileIssues, secrets, { used }),
1672
+ ...raw === void 0 ? {} : { variables }
1673
+ };
1674
+ }
1675
+ let websocket;
1676
+ const websocketIssues = [];
1677
+ if (fileIssues.length === 0 && configDocument !== void 0) {
1678
+ const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
1679
+ if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
1680
+ connectionIssues.push({
1681
+ file: document,
1682
+ issues: websocketIssues
1683
+ });
1684
+ if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
1685
+ }
1510
1686
  if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
1511
1687
  if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
1512
- if (fileIssues.length === 0) specs.push(this.build(value, document, secrets));
1688
+ if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
1689
+ if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
1513
1690
  else issues.push(...fileIssues);
1514
1691
  }
1692
+ const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
1693
+ if (shared.length > 0) throw new EvalError({
1694
+ category: ErrorCategory.Configuration,
1695
+ message: "invalid configuration",
1696
+ issues: shared
1697
+ });
1698
+ for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
1699
+ message: issue.toString(),
1700
+ location: ScenarioLoader.variablesLocation(entry.file)
1701
+ }));
1515
1702
  if (issues.length > 0) throw new EvalError({
1516
1703
  category: ErrorCategory.Specification,
1517
1704
  message: "invalid scenario specification",
@@ -1519,6 +1706,23 @@ var ScenarioLoader = class {
1519
1706
  });
1520
1707
  return specs;
1521
1708
  }
1709
+ static shared(lists) {
1710
+ if (lists.length === 0) return [];
1711
+ const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
1712
+ return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
1713
+ }
1714
+ static variablesLocation(document) {
1715
+ return new SourceLocation({
1716
+ ...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
1717
+ path: ""
1718
+ });
1719
+ }
1720
+ static unusedVariables(variables, used, document) {
1721
+ return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
1722
+ message: `variable '${name}' is not used by the scenario or the websocket configuration`,
1723
+ location: document.locateKey(["variables"], name)
1724
+ }));
1725
+ }
1522
1726
  discover(target, config) {
1523
1727
  if (!existsSync(target)) throw new EvalError({
1524
1728
  category: ErrorCategory.Specification,
@@ -1543,7 +1747,7 @@ var ScenarioLoader = class {
1543
1747
  }
1544
1748
  return files;
1545
1749
  }
1546
- build(value, document, secrets) {
1750
+ build(value, document, websocket, secrets) {
1547
1751
  const scenario = value;
1548
1752
  const turns = scenario.turns;
1549
1753
  const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
@@ -1553,10 +1757,11 @@ var ScenarioLoader = class {
1553
1757
  description: typeof scenario.description === "string" ? scenario.description : "",
1554
1758
  turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
1555
1759
  conversation: conversation ? new ConversationExpectation({
1556
- criteria: conversation.criteria,
1760
+ criteria: conversation.should,
1557
1761
  location: document.locate(["expect", "conversation"])
1558
1762
  }) : void 0,
1559
1763
  location: document.locate([]),
1764
+ websocket,
1560
1765
  secrets
1561
1766
  });
1562
1767
  }
@@ -1582,7 +1787,7 @@ var ScenarioLoader = class {
1582
1787
  });
1583
1788
  const fields = response;
1584
1789
  return new ResponseExpectation({
1585
- criteria: fields.criteria,
1790
+ criteria: fields.should,
1586
1791
  contains: fields.contains ?? [],
1587
1792
  notContains: fields.notContains ?? [],
1588
1793
  location
@@ -2405,7 +2610,11 @@ var ConversationRunner = class {
2405
2610
  async run(spec, signal = NEVER_ABORTED) {
2406
2611
  const startedAt = Date.now();
2407
2612
  const outcomes = [];
2408
- const assertions = new AssertionRunner(this.config.websocket.protocol.capabilities);
2613
+ const config = new EvalConfig({
2614
+ ...this.config,
2615
+ websocket: spec.websocket ?? this.config.websocket
2616
+ });
2617
+ const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
2409
2618
  const history = [];
2410
2619
  const evaluate = this.semanticEvaluator(signal);
2411
2620
  const inline = !this.config.execution.continueOnFailure;
@@ -2430,7 +2639,7 @@ var ConversationRunner = class {
2430
2639
  try {
2431
2640
  const steps = (async () => {
2432
2641
  for (let attempt = 0;; attempt++) {
2433
- connection = this.createConnection(this.config);
2642
+ connection = this.createConnection(config);
2434
2643
  connectionAttempts++;
2435
2644
  try {
2436
2645
  await connection.connect();
@@ -2924,7 +3133,7 @@ var ScenarioRunner = class {
2924
3133
  required: minPass,
2925
3134
  attempts,
2926
3135
  tests: this.results.tests(attempts, minPass),
2927
- agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
3136
+ agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
2928
3137
  evaluatorUsage: this.tokens.evaluator(attempts)
2929
3138
  });
2930
3139
  }
@@ -3015,7 +3224,9 @@ var ConsoleReporter = class ConsoleReporter {
3015
3224
  const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
3016
3225
  if (!this.verbose) return line;
3017
3226
  if (result.attempts.length === 0) return `${line}\n`;
3018
- const lines = [""];
3227
+ const websocket = spec.websocket;
3228
+ const subprotocols = websocket?.connection.subprotocols.length ?? 0;
3229
+ const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
3019
3230
  for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3020
3231
  lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
3021
3232
  lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
@@ -3228,11 +3439,16 @@ var JsonReporter = class JsonReporter {
3228
3439
  };
3229
3440
  }
3230
3441
  scenario(result, config) {
3231
- const capability = config.websocket.protocol.capabilities.tokenUsage;
3442
+ const websocket = result.spec.websocket ?? config.websocket;
3443
+ const capability = websocket.protocol.capabilities.tokenUsage;
3232
3444
  return {
3233
3445
  name: result.spec.name,
3234
3446
  file: result.spec.location.file,
3235
3447
  status: result.status,
3448
+ connection: {
3449
+ endpoint: websocket.endpoint,
3450
+ subprotocols: websocket.connection.subprotocols.length
3451
+ },
3236
3452
  ...JsonReporter.counts(result),
3237
3453
  agentUsage: JsonReporter.usage(result.agentUsage),
3238
3454
  evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),