@mappies/miniwat-eval 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -23,8 +23,8 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
23
23
  let _mappies_miniwat_agent_core = require("@mappies/miniwat-agent-core");
24
24
  let node_fs = require("node:fs");
25
25
  let node_path = require("node:path");
26
- let ajv = require("ajv");
27
26
  let yaml = require("yaml");
27
+ let ajv = require("ajv");
28
28
  let configurapi_handler_ws = require("configurapi-handler-ws");
29
29
  let node_util = require("node:util");
30
30
  let _mappies_miniwat_agent = require("@mappies/miniwat-agent");
@@ -296,22 +296,70 @@ var TurnSpec = class {
296
296
  this.location = new SourceLocation(data?.location);
297
297
  }
298
298
  };
299
+ // src/entities/protocolCapabilities.ts
300
+ var ProtocolCapabilities = class {
301
+ toolUses;
302
+ toolResults;
303
+ tokenUsage;
304
+ constructor(data) {
305
+ this.toolUses = data?.toolUses ?? false;
306
+ this.toolResults = data?.toolResults ?? false;
307
+ this.tokenUsage = data?.tokenUsage ?? false;
308
+ }
309
+ };
310
+ // src/interfaces/requestFormat.ts
311
+ const RequestFormat = { Json: "json" };
312
+ // src/entities/requestConfig.ts
313
+ var RequestConfig = class {
314
+ format;
315
+ value;
316
+ inputPath;
317
+ constructor(data) {
318
+ this.format = data?.format ?? RequestFormat.Json;
319
+ this.value = data?.value;
320
+ this.inputPath = data?.inputPath;
321
+ }
322
+ };
323
+ // src/entities/protocolConfig.ts
324
+ var ProtocolConfig = class {
325
+ request;
326
+ capabilities;
327
+ constructor(data) {
328
+ this.request = new RequestConfig(data?.request);
329
+ this.capabilities = new ProtocolCapabilities(data?.capabilities);
330
+ }
331
+ };
332
+ // src/entities/webSocketConfig.ts
333
+ var WebSocketConfig = class {
334
+ endpoint;
335
+ connection;
336
+ protocol;
337
+ constructor(data) {
338
+ this.endpoint = data?.endpoint ?? "";
339
+ this.connection = new ConnectionConfig(data?.connection);
340
+ this.protocol = new ProtocolConfig(data?.protocol);
341
+ }
342
+ };
299
343
  // src/entities/conversationSpec.ts
300
344
  var ConversationSpec = class {
301
345
  version;
302
346
  name;
347
+ runName;
303
348
  description;
304
349
  turns;
305
350
  conversation;
306
351
  location;
352
+ websocket;
307
353
  secrets;
308
354
  constructor(data) {
309
355
  this.version = data?.version ?? 1;
310
356
  this.name = data?.name ?? "";
357
+ this.runName = data?.runName ?? "";
311
358
  this.description = data?.description ?? "";
312
359
  this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
313
360
  this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
314
361
  this.location = new SourceLocation(data?.location);
362
+ this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
315
363
  this.secrets = [...data?.secrets ?? []];
316
364
  }
317
365
  };
@@ -413,55 +461,12 @@ var ReportConfig = class {
413
461
  this.formats = data?.formats ?? [];
414
462
  }
415
463
  };
416
- // src/entities/protocolCapabilities.ts
417
- var ProtocolCapabilities = class {
418
- toolUses;
419
- toolResults;
420
- tokenUsage;
421
- constructor(data) {
422
- this.toolUses = data?.toolUses ?? false;
423
- this.toolResults = data?.toolResults ?? false;
424
- this.tokenUsage = data?.tokenUsage ?? false;
425
- }
426
- };
427
- // src/interfaces/requestFormat.ts
428
- const RequestFormat = { Json: "json" };
429
- // src/entities/requestConfig.ts
430
- var RequestConfig = class {
431
- format;
432
- value;
433
- inputPath;
434
- constructor(data) {
435
- this.format = data?.format ?? RequestFormat.Json;
436
- this.value = data?.value;
437
- this.inputPath = data?.inputPath;
438
- }
439
- };
440
- // src/entities/protocolConfig.ts
441
- var ProtocolConfig = class {
442
- request;
443
- capabilities;
444
- constructor(data) {
445
- this.request = new RequestConfig(data?.request);
446
- this.capabilities = new ProtocolCapabilities(data?.capabilities);
447
- }
448
- };
449
- // src/entities/webSocketConfig.ts
450
- var WebSocketConfig = class {
451
- endpoint;
452
- connection;
453
- protocol;
454
- constructor(data) {
455
- this.endpoint = data?.endpoint ?? "";
456
- this.connection = new ConnectionConfig(data?.connection);
457
- this.protocol = new ProtocolConfig(data?.protocol);
458
- }
459
- };
460
464
  // src/entities/evalConfig.ts
461
465
  var EvalConfig = class {
462
466
  version;
463
467
  configPath;
464
468
  parametersPath;
469
+ variables;
465
470
  websocket;
466
471
  execution;
467
472
  evaluator;
@@ -471,6 +476,7 @@ var EvalConfig = class {
471
476
  this.version = data?.version ?? 1;
472
477
  this.configPath = data?.configPath ?? "";
473
478
  this.parametersPath = data?.parametersPath ?? "";
479
+ this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
474
480
  this.websocket = new WebSocketConfig(data?.websocket);
475
481
  this.execution = new ExecutionConfig(data?.execution);
476
482
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -782,6 +788,14 @@ var eval_config_schema_default = {
782
788
  ],
783
789
  properties: {
784
790
  "version": { "const": 1 },
791
+ "variables": {
792
+ "type": "object",
793
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
794
+ "additionalProperties": { "anyOf": [{ "type": "string" }, {
795
+ "type": "array",
796
+ "items": { "type": "string" }
797
+ }] }
798
+ },
785
799
  "websocket": {
786
800
  "type": "object",
787
801
  "additionalProperties": false,
@@ -940,18 +954,80 @@ var eval_config_schema_default = {
940
954
  }
941
955
  }
942
956
  };
957
+ // src/yaml/yamlDocument.ts
958
+ var YamlDocument = class YamlDocument {
959
+ file;
960
+ document;
961
+ lineCounter;
962
+ value;
963
+ errors;
964
+ constructor(file, document, lineCounter) {
965
+ this.file = file;
966
+ this.document = document;
967
+ this.lineCounter = lineCounter;
968
+ this.value = document.toJS();
969
+ this.errors = document.errors.map((error) => new ValidationIssue({
970
+ message: error.message.split("\n")[0],
971
+ location: this.toLocation(error.pos[0], [])
972
+ }));
973
+ }
974
+ locate(path) {
975
+ return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
976
+ }
977
+ locateKey(path, key) {
978
+ const parent = this.findNode(path);
979
+ if ((0, yaml.isMap)(parent)) {
980
+ const pair = parent.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === key);
981
+ if (pair && (0, yaml.isScalar)(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
982
+ }
983
+ return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
984
+ }
985
+ static formatPath(path) {
986
+ return path.map((segment, index) => {
987
+ if (typeof segment === "number") return `[${segment}]`;
988
+ return index === 0 ? segment : `.${segment}`;
989
+ }).join("");
990
+ }
991
+ findNode(path) {
992
+ let node = this.document.contents ?? void 0;
993
+ for (const segment of path) {
994
+ let next;
995
+ if ((0, yaml.isMap)(node)) {
996
+ const pair = node.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === String(segment));
997
+ next = pair?.value ?? pair?.key;
998
+ } else if ((0, yaml.isSeq)(node) && typeof segment === "number") next = node.items[segment];
999
+ if (!next) break;
1000
+ node = next;
1001
+ }
1002
+ return node;
1003
+ }
1004
+ toLocation(offset, path) {
1005
+ const position = this.lineCounter.linePos(offset);
1006
+ return new SourceLocation({
1007
+ file: this.file,
1008
+ line: position.line,
1009
+ column: position.col,
1010
+ path: YamlDocument.formatPath(path)
1011
+ });
1012
+ }
1013
+ };
943
1014
  // src/yaml/environmentResolver.ts
944
1015
  const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
945
1016
  const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
946
1017
  var EnvironmentResolver = class EnvironmentResolver {
947
1018
  environment;
948
1019
  parameters;
949
- constructor(environment = process.env, parameters = {}) {
1020
+ levels;
1021
+ constructor(environment = process.env, parameters = {}, levels = []) {
950
1022
  this.environment = environment;
951
1023
  this.parameters = parameters;
1024
+ this.levels = levels;
1025
+ }
1026
+ withLevels(levels) {
1027
+ return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
952
1028
  }
953
- resolve(value, document, issues, secrets = []) {
954
- return this.resolveValue(value, [], document, issues, secrets);
1029
+ resolve(value, document, issues, secrets = [], options = {}) {
1030
+ return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
955
1031
  }
956
1032
  static hasReference(value) {
957
1033
  return new RegExp(REFERENCE.source).test(value);
@@ -959,22 +1035,45 @@ var EnvironmentResolver = class EnvironmentResolver {
959
1035
  static isSecretName(name) {
960
1036
  return SECRET_NAME.test(name);
961
1037
  }
962
- resolveValue(value, path, document, issues, secrets) {
1038
+ find(name) {
1039
+ for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
1040
+ return this.environment[name];
1041
+ }
1042
+ sources() {
1043
+ const sources = [
1044
+ ...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
1045
+ ...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
1046
+ "environment variable"
1047
+ ];
1048
+ return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
1049
+ }
1050
+ resolveValue(value, path, document, issues, secrets, options) {
963
1051
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
964
- const resolved = Object.hasOwn(this.parameters, name) ? this.parameters[name] : this.environment[name];
1052
+ options.used?.add(name);
1053
+ const resolved = this.find(name);
965
1054
  if (resolved === void 0) {
966
- const source = Object.keys(this.parameters).length > 0 ? "parameter or environment variable" : "environment variable";
1055
+ if (options.optional?.(path)) {
1056
+ options.unresolved?.add(YamlDocument.formatPath(path));
1057
+ return reference;
1058
+ }
967
1059
  issues.push(new ValidationIssue({
968
- message: `${source} '${name}' is not set`,
1060
+ message: `${this.sources()} '${name}' is not set`,
969
1061
  location: document.locate(path)
970
1062
  }));
971
1063
  return reference;
972
1064
  }
973
- if (EnvironmentResolver.isSecretName(name) && !secrets.includes(resolved)) secrets.push(resolved);
1065
+ if (Array.isArray(resolved)) {
1066
+ issues.push(new ValidationIssue({
1067
+ message: `variable '${name}' is a list, which can only be used as runWith.from`,
1068
+ location: document.locate(path)
1069
+ }));
1070
+ return reference;
1071
+ }
1072
+ if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
974
1073
  return resolved;
975
1074
  });
976
- if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
977
- if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
1075
+ if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
1076
+ if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
978
1077
  return value;
979
1078
  }
980
1079
  };
@@ -1058,63 +1157,6 @@ var SchemaValidator = class SchemaValidator {
1058
1157
  return path;
1059
1158
  }
1060
1159
  };
1061
- // src/yaml/yamlDocument.ts
1062
- var YamlDocument = class YamlDocument {
1063
- file;
1064
- document;
1065
- lineCounter;
1066
- value;
1067
- errors;
1068
- constructor(file, document, lineCounter) {
1069
- this.file = file;
1070
- this.document = document;
1071
- this.lineCounter = lineCounter;
1072
- this.value = document.toJS();
1073
- this.errors = document.errors.map((error) => new ValidationIssue({
1074
- message: error.message.split("\n")[0],
1075
- location: this.toLocation(error.pos[0], [])
1076
- }));
1077
- }
1078
- locate(path) {
1079
- return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
1080
- }
1081
- locateKey(path, key) {
1082
- const parent = this.findNode(path);
1083
- if ((0, yaml.isMap)(parent)) {
1084
- const pair = parent.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === key);
1085
- if (pair && (0, yaml.isScalar)(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
1086
- }
1087
- return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
1088
- }
1089
- static formatPath(path) {
1090
- return path.map((segment, index) => {
1091
- if (typeof segment === "number") return `[${segment}]`;
1092
- return index === 0 ? segment : `.${segment}`;
1093
- }).join("");
1094
- }
1095
- findNode(path) {
1096
- let node = this.document.contents ?? void 0;
1097
- for (const segment of path) {
1098
- let next;
1099
- if ((0, yaml.isMap)(node)) {
1100
- const pair = node.items.find((item) => (0, yaml.isPair)(item) && (0, yaml.isScalar)(item.key) && String(item.key.value) === String(segment));
1101
- next = pair?.value ?? pair?.key;
1102
- } else if ((0, yaml.isSeq)(node) && typeof segment === "number") next = node.items[segment];
1103
- if (!next) break;
1104
- node = next;
1105
- }
1106
- return node;
1107
- }
1108
- toLocation(offset, path) {
1109
- const position = this.lineCounter.linePos(offset);
1110
- return new SourceLocation({
1111
- file: this.file,
1112
- line: position.line,
1113
- column: position.col,
1114
- path: YamlDocument.formatPath(path)
1115
- });
1116
- }
1117
- };
1118
1160
  // src/yaml/yamlDocumentLoader.ts
1119
1161
  var YamlDocumentLoader = class {
1120
1162
  load(filePath, displayPath) {
@@ -1133,11 +1175,150 @@ var YamlDocumentLoader = class {
1133
1175
  function isRecord(value) {
1134
1176
  return value !== null && typeof value === "object" && !Array.isArray(value);
1135
1177
  }
1178
+ // src/yaml/variablesReader.ts
1179
+ const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
1180
+ var VariablesReader = class {
1181
+ read(document, resolver, issues, secrets) {
1182
+ const raw = isRecord(document.value) ? document.value.variables : void 0;
1183
+ if (raw === void 0) return {};
1184
+ if (!isRecord(raw)) {
1185
+ issues.push(new ValidationIssue({
1186
+ message: "must be object",
1187
+ location: document.locate(["variables"])
1188
+ }));
1189
+ return {};
1190
+ }
1191
+ const variables = {};
1192
+ for (const [name, value] of Object.entries(raw)) {
1193
+ if (!VARIABLE_NAME.test(name)) {
1194
+ issues.push(new ValidationIssue({
1195
+ message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
1196
+ location: document.locateKey(["variables"], name)
1197
+ }));
1198
+ continue;
1199
+ }
1200
+ if (Array.isArray(value)) {
1201
+ const bad = value.findIndex((item) => typeof item !== "string");
1202
+ if (bad >= 0) {
1203
+ issues.push(new ValidationIssue({
1204
+ message: "must be string",
1205
+ location: document.locate([
1206
+ "variables",
1207
+ name,
1208
+ bad
1209
+ ])
1210
+ }));
1211
+ continue;
1212
+ }
1213
+ variables[name] = [...value];
1214
+ continue;
1215
+ }
1216
+ if (typeof value !== "string") {
1217
+ issues.push(new ValidationIssue({
1218
+ message: "must be string or a list of strings",
1219
+ location: document.locate(["variables", name])
1220
+ }));
1221
+ continue;
1222
+ }
1223
+ const before = issues.length;
1224
+ const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
1225
+ if (issues.length > before) continue;
1226
+ const text = resolved.variables[name];
1227
+ variables[name] = text;
1228
+ if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
1229
+ }
1230
+ return variables;
1231
+ }
1232
+ };
1233
+ // src/config/webSocketResolver.ts
1234
+ const WEBSOCKET_SCHEMA = {
1235
+ ...eval_config_schema_default,
1236
+ required: ["websocket"],
1237
+ additionalProperties: true,
1238
+ properties: { websocket: eval_config_schema_default.properties.websocket }
1239
+ };
1240
+ var WebSocketResolver = class WebSocketResolver {
1241
+ schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
1242
+ resolve(document, resolver, issues, secrets, used, defer) {
1243
+ const raw = isRecord(document.value) ? document.value.websocket : void 0;
1244
+ const unresolved = /* @__PURE__ */ new Set();
1245
+ const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
1246
+ const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
1247
+ used,
1248
+ optional,
1249
+ unresolved
1250
+ });
1251
+ const subprotocols = WebSocketResolver.subprotocols(raw);
1252
+ if (subprotocols !== void 0) {
1253
+ const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
1254
+ const set = !unresolved.has(YamlDocument.formatPath([
1255
+ "websocket",
1256
+ "connection",
1257
+ "subprotocols",
1258
+ index
1259
+ ]));
1260
+ const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
1261
+ return set && !empty;
1262
+ });
1263
+ value.websocket.connection.subprotocols = kept;
1264
+ }
1265
+ const prefix = YamlDocument.formatPath([
1266
+ "websocket",
1267
+ "connection",
1268
+ "subprotocols"
1269
+ ]);
1270
+ return {
1271
+ value,
1272
+ deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
1273
+ };
1274
+ }
1275
+ validate(value, document) {
1276
+ const issues = this.schemaValidator.validate(value, document);
1277
+ if (issues.length > 0) return issues;
1278
+ return WebSocketResolver.validateInputPath(value, document);
1279
+ }
1280
+ static validateInputPath(value, document) {
1281
+ const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
1282
+ if (typeof request.inputPath !== "string") return [];
1283
+ const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
1284
+ const last = tokens.pop();
1285
+ let parent = request.value;
1286
+ for (const token of tokens) parent = WebSocketResolver.child(parent, token);
1287
+ if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
1288
+ return [new ValidationIssue({
1289
+ message: "must point to a missing or string value whose parent exists in request.value",
1290
+ location: document.locate([
1291
+ "websocket",
1292
+ "protocol",
1293
+ "request",
1294
+ "inputPath"
1295
+ ])
1296
+ })];
1297
+ }
1298
+ static section(value, ...path) {
1299
+ let current = value;
1300
+ for (const key of path) current = WebSocketResolver.child(current, key);
1301
+ return isRecord(current) ? current : {};
1302
+ }
1303
+ static child(value, key) {
1304
+ if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
1305
+ return isRecord(value) ? value[key] : void 0;
1306
+ }
1307
+ static isSubprotocol(path) {
1308
+ return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
1309
+ }
1310
+ static subprotocols(websocket) {
1311
+ const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
1312
+ return Array.isArray(subprotocols) ? subprotocols : void 0;
1313
+ }
1314
+ };
1136
1315
  // src/config/evalConfigLoader.ts
1137
1316
  var EvalConfigLoader = class EvalConfigLoader {
1138
1317
  cwd;
1139
1318
  documentLoader = new YamlDocumentLoader();
1140
1319
  schemaValidator = new SchemaValidator(eval_config_schema_default);
1320
+ variablesReader = new VariablesReader();
1321
+ webSocketResolver = new WebSocketResolver();
1141
1322
  environmentResolver;
1142
1323
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1143
1324
  this.cwd = cwd;
@@ -1148,19 +1329,47 @@ var EvalConfigLoader = class EvalConfigLoader {
1148
1329
  const issues = [...document.errors];
1149
1330
  if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
1150
1331
  const named = [];
1151
- const value = issues.length === 0 ? this.environmentResolver.resolve(document.value, document, issues, named) : void 0;
1152
- if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
1153
- if (issues.length === 0) issues.push(...this.validateInputPath(value, document));
1332
+ const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
1333
+ let value = {};
1334
+ if (issues.length === 0) {
1335
+ const resolver = this.environmentResolver.withLevels([variables]);
1336
+ const { websocket: _websocket, variables: _variables, ...rest } = document.value;
1337
+ const resolved = resolver.resolve(rest, document, issues, named);
1338
+ const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
1339
+ value = {
1340
+ ...resolved,
1341
+ ..._variables === void 0 ? {} : { variables },
1342
+ ..._websocket === void 0 ? {} : websocket.value
1343
+ };
1344
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
1345
+ if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
1346
+ }
1154
1347
  if (issues.length > 0) throw new EvalError({
1155
1348
  category: ErrorCategory.Configuration,
1156
1349
  message: "invalid configuration",
1157
1350
  issues
1158
1351
  });
1159
- const config = this.build(value, configPath, named);
1352
+ const config = this.build(value, configPath, variables, named);
1160
1353
  this.applyRepetition(config, override, value, document);
1161
1354
  EvalConfigLoader.applyReport(config, report);
1162
1355
  return config;
1163
1356
  }
1357
+ static buildWebSocket(value) {
1358
+ const section = (...path) => WebSocketResolver.section(value, ...path);
1359
+ const connection = section("websocket", "connection");
1360
+ const request = section("websocket", "protocol", "request");
1361
+ return new WebSocketConfig({
1362
+ endpoint: WebSocketResolver.section(value, "websocket").endpoint,
1363
+ connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
1364
+ protocol: new ProtocolConfig({
1365
+ request: new RequestConfig({
1366
+ ...request,
1367
+ format: request.format ?? RequestFormat.Json
1368
+ }),
1369
+ capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
1370
+ })
1371
+ });
1372
+ }
1164
1373
  static applyReport(config, report) {
1165
1374
  if (report.formats === void 0) return;
1166
1375
  const known = Object.values(ReportFormat);
@@ -1182,45 +1391,17 @@ var EvalConfigLoader = class EvalConfigLoader {
1182
1391
  location: document.locate(["evaluator", "apiKey"])
1183
1392
  })];
1184
1393
  }
1185
- validateInputPath(value, document) {
1186
- const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
1187
- if (typeof request.inputPath !== "string") return [];
1188
- const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
1189
- const last = tokens.pop();
1190
- let parent = request.value;
1191
- for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
1192
- if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
1193
- return [new ValidationIssue({
1194
- message: "must point to a missing or string value whose parent exists in request.value",
1195
- location: document.locate([
1196
- "websocket",
1197
- "protocol",
1198
- "request",
1199
- "inputPath"
1200
- ])
1201
- })];
1202
- }
1203
- build(value, configPath, named) {
1394
+ build(value, configPath, variables, named) {
1204
1395
  const section = (...path) => EvalConfigLoader.section(value, ...path);
1205
1396
  const configDirectory = (0, node_path.dirname)(configPath);
1206
- const connection = section("websocket", "connection");
1207
- const request = section("websocket", "protocol", "request");
1397
+ const websocket = EvalConfigLoader.buildWebSocket(value);
1208
1398
  const evaluator = section("evaluator");
1209
1399
  const report = section("report");
1210
1400
  return new EvalConfig({
1211
1401
  version: value.version,
1212
1402
  configPath,
1213
- websocket: new WebSocketConfig({
1214
- endpoint: EvalConfigLoader.section(value, "websocket").endpoint,
1215
- connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
1216
- protocol: new ProtocolConfig({
1217
- request: new RequestConfig({
1218
- ...request,
1219
- format: request.format ?? RequestFormat.Json
1220
- }),
1221
- capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
1222
- })
1223
- }),
1403
+ variables,
1404
+ websocket,
1224
1405
  execution: new ExecutionConfig({
1225
1406
  ...DEFAULT_CONFIG.execution,
1226
1407
  ...section("execution"),
@@ -1241,7 +1422,7 @@ var EvalConfigLoader = class EvalConfigLoader {
1241
1422
  directory: (0, node_path.resolve)(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
1242
1423
  formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
1243
1424
  }),
1244
- secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols ?? [], named)
1425
+ secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
1245
1426
  });
1246
1427
  }
1247
1428
  applyRepetition(config, override, value, document) {
@@ -1294,13 +1475,7 @@ var EvalConfigLoader = class EvalConfigLoader {
1294
1475
  ])];
1295
1476
  }
1296
1477
  static section(value, ...path) {
1297
- let current = value;
1298
- for (const key of path) current = EvalConfigLoader.child(current, key);
1299
- return isRecord(current) ? current : {};
1300
- }
1301
- static child(value, key) {
1302
- if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
1303
- return isRecord(value) ? value[key] : void 0;
1478
+ return WebSocketResolver.section(value, ...path);
1304
1479
  }
1305
1480
  };
1306
1481
  // src/config/parametersLoader.ts
@@ -1352,6 +1527,48 @@ var conversation_schema_default = {
1352
1527
  "version": { "const": 1 },
1353
1528
  "name": { "$ref": "#/definitions/text" },
1354
1529
  "description": { "type": "string" },
1530
+ "variables": {
1531
+ "type": "object",
1532
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
1533
+ "additionalProperties": { "anyOf": [{ "type": "string" }, {
1534
+ "type": "array",
1535
+ "items": { "type": "string" }
1536
+ }] }
1537
+ },
1538
+ "runWith": {
1539
+ "description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
1540
+ "anyOf": [{
1541
+ "type": "array",
1542
+ "minItems": 1,
1543
+ "items": {
1544
+ "type": "object",
1545
+ "required": ["name"],
1546
+ "properties": { "name": {
1547
+ "type": "string",
1548
+ "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
1549
+ } },
1550
+ "additionalProperties": { "type": "string" }
1551
+ }
1552
+ }, {
1553
+ "type": "object",
1554
+ "additionalProperties": false,
1555
+ "required": ["variable", "from"],
1556
+ "properties": {
1557
+ "variable": {
1558
+ "type": "string",
1559
+ "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
1560
+ },
1561
+ "from": { "anyOf": [{
1562
+ "type": "array",
1563
+ "minItems": 1,
1564
+ "items": { "type": "string" }
1565
+ }, {
1566
+ "type": "string",
1567
+ "pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
1568
+ }] }
1569
+ }
1570
+ }]
1571
+ },
1355
1572
  "turns": {
1356
1573
  "type": "array",
1357
1574
  "minItems": 1,
@@ -1364,8 +1581,8 @@ var conversation_schema_default = {
1364
1581
  "properties": { "conversation": {
1365
1582
  "type": "object",
1366
1583
  "additionalProperties": false,
1367
- "required": ["criteria"],
1368
- "properties": { "criteria": { "$ref": "#/definitions/text" } }
1584
+ "required": ["should"],
1585
+ "properties": { "should": { "$ref": "#/definitions/text" } }
1369
1586
  } }
1370
1587
  }
1371
1588
  },
@@ -1408,12 +1625,12 @@ var conversation_schema_default = {
1408
1625
  }
1409
1626
  },
1410
1627
  "response": {
1411
- "description": "A string is shorthand for criteria.",
1628
+ "description": "A string is shorthand for should.",
1412
1629
  "type": ["string", "object"],
1413
1630
  "pattern": "\\S",
1414
1631
  "additionalProperties": false,
1415
1632
  "properties": {
1416
- "criteria": { "$ref": "#/definitions/text" },
1633
+ "should": { "$ref": "#/definitions/text" },
1417
1634
  "contains": { "$ref": "#/definitions/textList" },
1418
1635
  "notContains": { "$ref": "#/definitions/textList" }
1419
1636
  }
@@ -1445,7 +1662,7 @@ var conversation_schema_default = {
1445
1662
  };
1446
1663
  // src/scenarios/conversationValidator.ts
1447
1664
  const RESPONSE_KEYS = [
1448
- "criteria",
1665
+ "should",
1449
1666
  "contains",
1450
1667
  "notContains"
1451
1668
  ];
@@ -1468,7 +1685,7 @@ var ConversationValidator = class ConversationValidator {
1468
1685
  validateResponse(response, path, document, issues) {
1469
1686
  if (!RESPONSE_KEYS.some((key) => key in response)) {
1470
1687
  issues.push(new ValidationIssue({
1471
- message: "response must have at least one of: criteria, contains, notContains",
1688
+ message: "response must have at least one of: should, contains, notContains",
1472
1689
  location: document.locate(path)
1473
1690
  }));
1474
1691
  return;
@@ -1509,13 +1726,231 @@ var ConversationValidator = class ConversationValidator {
1509
1726
  return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
1510
1727
  }
1511
1728
  };
1729
+ // src/scenarios/runWithReader.ts
1730
+ const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
1731
+ const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
1732
+ const FORMS = "must be a list of rows, or an object with variable and from";
1733
+ const SINGLE_RUN = {
1734
+ name: "",
1735
+ variables: {},
1736
+ keys: [],
1737
+ location: [],
1738
+ secrets: []
1739
+ };
1740
+ var RunWithReader = class RunWithReader {
1741
+ read(document, resolver, issues, used) {
1742
+ const raw = isRecord(document.value) ? document.value.runWith : void 0;
1743
+ if (raw === void 0) return void 0;
1744
+ if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
1745
+ if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
1746
+ issues.push(new ValidationIssue({
1747
+ message: FORMS,
1748
+ location: document.locate(["runWith"])
1749
+ }));
1750
+ return [];
1751
+ }
1752
+ rows(rows, document, resolver, issues, used) {
1753
+ if (rows.length === 0) {
1754
+ issues.push(new ValidationIssue({
1755
+ message: "must not be empty",
1756
+ location: document.locate(["runWith"])
1757
+ }));
1758
+ return [];
1759
+ }
1760
+ const names = /* @__PURE__ */ new Set();
1761
+ const runs = [];
1762
+ rows.forEach((row, index) => {
1763
+ const path = ["runWith", index];
1764
+ if (!isRecord(row)) {
1765
+ issues.push(new ValidationIssue({
1766
+ message: "must be object",
1767
+ location: document.locate(path)
1768
+ }));
1769
+ return;
1770
+ }
1771
+ const name = row.name;
1772
+ if (name === void 0) issues.push(new ValidationIssue({
1773
+ message: "missing required key 'name'",
1774
+ location: document.locate(path)
1775
+ }));
1776
+ else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
1777
+ message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
1778
+ location: document.locate([...path, "name"])
1779
+ }));
1780
+ else if (names.has(name)) issues.push(new ValidationIssue({
1781
+ message: `duplicate run name '${name}'`,
1782
+ location: document.locate([...path, "name"])
1783
+ }));
1784
+ else names.add(name);
1785
+ const run = {
1786
+ name: typeof name === "string" ? name : "",
1787
+ variables: {},
1788
+ keys: [],
1789
+ location: path,
1790
+ secrets: []
1791
+ };
1792
+ for (const [key, value] of Object.entries(row)) {
1793
+ if (key === "name") continue;
1794
+ if (!VARIABLE_NAME.test(key)) {
1795
+ issues.push(new ValidationIssue({
1796
+ message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
1797
+ location: document.locateKey(path, key)
1798
+ }));
1799
+ continue;
1800
+ }
1801
+ if (typeof value !== "string") {
1802
+ issues.push(new ValidationIssue({
1803
+ message: "must be string",
1804
+ location: document.locate([...path, key])
1805
+ }));
1806
+ continue;
1807
+ }
1808
+ const resolved = resolver.resolve(value, document, issues, run.secrets, {
1809
+ used,
1810
+ at: [...path, key]
1811
+ });
1812
+ RunWithReader.set(run, key, resolved, document.locateKey(path, key));
1813
+ }
1814
+ runs.push(run);
1815
+ });
1816
+ return runs;
1817
+ }
1818
+ list(form, document, resolver, issues, used) {
1819
+ const before = issues.length;
1820
+ for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
1821
+ message: `unknown key '${key}'`,
1822
+ location: document.locateKey(["runWith"], key)
1823
+ }));
1824
+ const variable = form.variable;
1825
+ if (variable === void 0) issues.push(new ValidationIssue({
1826
+ message: "missing required key 'variable'",
1827
+ location: document.locate(["runWith"])
1828
+ }));
1829
+ else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
1830
+ message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
1831
+ location: document.locate(["runWith", "variable"])
1832
+ }));
1833
+ const entries = this.entries(form.from, document, resolver, issues, used);
1834
+ if (issues.length > before || entries === void 0) return [];
1835
+ const name = variable;
1836
+ const keyLocation = document.locate(["runWith", "variable"]);
1837
+ const names = /* @__PURE__ */ new Set();
1838
+ const runs = [];
1839
+ for (const { entry, path } of entries) {
1840
+ if (!VARIABLE_NAME.test(entry)) {
1841
+ issues.push(new ValidationIssue({
1842
+ message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
1843
+ location: document.locate(path)
1844
+ }));
1845
+ continue;
1846
+ }
1847
+ if (names.has(entry)) {
1848
+ issues.push(new ValidationIssue({
1849
+ message: `duplicate run name '${entry}'`,
1850
+ location: document.locate(path)
1851
+ }));
1852
+ continue;
1853
+ }
1854
+ names.add(entry);
1855
+ const run = {
1856
+ name: entry,
1857
+ variables: {},
1858
+ keys: [],
1859
+ location: path,
1860
+ secrets: []
1861
+ };
1862
+ const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
1863
+ used,
1864
+ at: path
1865
+ });
1866
+ RunWithReader.set(run, name, value, keyLocation);
1867
+ runs.push(run);
1868
+ }
1869
+ return runs;
1870
+ }
1871
+ entries(from, document, resolver, issues, used) {
1872
+ const path = ["runWith", "from"];
1873
+ const location = document.locate(path);
1874
+ if (from === void 0) {
1875
+ issues.push(new ValidationIssue({
1876
+ message: "missing required key 'from'",
1877
+ location: document.locate(["runWith"])
1878
+ }));
1879
+ return;
1880
+ }
1881
+ let list;
1882
+ let at;
1883
+ if (Array.isArray(from)) {
1884
+ list = from;
1885
+ at = (index) => [...path, index];
1886
+ } else {
1887
+ const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
1888
+ if (reference === null) {
1889
+ issues.push(new ValidationIssue({
1890
+ message: "must be a list, or a list variable such as ${tokens}",
1891
+ location
1892
+ }));
1893
+ return;
1894
+ }
1895
+ const name = reference[1];
1896
+ const found = resolver.find(name);
1897
+ used.add(name);
1898
+ if (found === void 0) {
1899
+ issues.push(new ValidationIssue({
1900
+ message: `list variable '${name}' is not set`,
1901
+ location
1902
+ }));
1903
+ return;
1904
+ }
1905
+ if (!Array.isArray(found)) {
1906
+ issues.push(new ValidationIssue({
1907
+ message: `variable '${name}' is not a list`,
1908
+ location
1909
+ }));
1910
+ return;
1911
+ }
1912
+ list = found;
1913
+ at = () => path;
1914
+ }
1915
+ if (list.length === 0) {
1916
+ issues.push(new ValidationIssue({
1917
+ message: "must not be empty",
1918
+ location
1919
+ }));
1920
+ return;
1921
+ }
1922
+ const bad = list.findIndex((item) => typeof item !== "string");
1923
+ if (bad >= 0) {
1924
+ issues.push(new ValidationIssue({
1925
+ message: "must be string",
1926
+ location: document.locate(at(bad))
1927
+ }));
1928
+ return;
1929
+ }
1930
+ return list.map((entry, index) => ({
1931
+ entry,
1932
+ path: at(index)
1933
+ }));
1934
+ }
1935
+ static set(run, name, value, location) {
1936
+ run.variables[name] = value;
1937
+ run.keys.push({
1938
+ name,
1939
+ location
1940
+ });
1941
+ if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
1942
+ }
1943
+ };
1512
1944
  // src/scenarios/scenarioLoader.ts
1513
1945
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
1514
- var ScenarioLoader = class {
1946
+ var ScenarioLoader = class ScenarioLoader {
1515
1947
  cwd;
1516
1948
  documentLoader = new YamlDocumentLoader();
1517
1949
  schemaValidator = new SchemaValidator(conversation_schema_default);
1518
1950
  conversationValidator = new ConversationValidator();
1951
+ variablesReader = new VariablesReader();
1952
+ runWithReader = new RunWithReader();
1953
+ webSocketResolver = new WebSocketResolver();
1519
1954
  environmentResolver;
1520
1955
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1521
1956
  this.cwd = cwd;
@@ -1523,18 +1958,66 @@ var ScenarioLoader = class {
1523
1958
  }
1524
1959
  load(target, config) {
1525
1960
  const files = this.discover((0, node_path.resolve)(this.cwd, target), config);
1961
+ const configDocument = (0, node_fs.existsSync)(config.configPath) ? this.documentLoader.load(config.configPath, (0, node_path.relative)(this.cwd, config.configPath)) : void 0;
1526
1962
  const issues = [];
1963
+ const connectionIssues = [];
1527
1964
  const specs = [];
1528
1965
  for (const file of files) {
1529
1966
  const document = this.documentLoader.load(file, (0, node_path.relative)(this.cwd, file));
1530
1967
  const fileIssues = [...document.errors];
1531
- const secrets = [];
1532
- const value = fileIssues.length === 0 ? this.environmentResolver.resolve(document.value, document, fileIssues, secrets) : void 0;
1533
- if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
1534
- if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
1535
- if (fileIssues.length === 0) specs.push(this.build(value, document, secrets));
1968
+ const fileSecrets = [];
1969
+ const used = /* @__PURE__ */ new Set();
1970
+ const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
1971
+ const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
1972
+ const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
1973
+ if (fileIssues.length > 0) {
1974
+ issues.push(...fileIssues);
1975
+ continue;
1976
+ }
1977
+ const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
1978
+ const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
1979
+ for (const result of results) for (const name of result.used) used.add(name);
1980
+ fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
1981
+ run: result.run,
1982
+ issues: result.issues
1983
+ })), (issue, run) => run === void 0 ? issue : new ValidationIssue({
1984
+ message: `run '${run.name}': ${issue.message}`,
1985
+ location: issue.location
1986
+ })));
1987
+ const attempted = results.filter((result) => result.connectionChecked);
1988
+ if (attempted.length > 0) connectionIssues.push({
1989
+ file: document,
1990
+ runs: attempted.map((result) => ({
1991
+ run: result.run,
1992
+ issues: result.websocketIssues
1993
+ }))
1994
+ });
1995
+ if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
1996
+ if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
1997
+ const websocket = result.websocket;
1998
+ specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
1999
+ }
1536
2000
  else issues.push(...fileIssues);
1537
2001
  }
2002
+ const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
2003
+ if (shared.length > 0) throw new EvalError({
2004
+ category: ErrorCategory.Configuration,
2005
+ message: "invalid configuration",
2006
+ issues: shared
2007
+ });
2008
+ for (const entry of connectionIssues) {
2009
+ const fileLocation = ScenarioLoader.variablesLocation(entry.file);
2010
+ issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
2011
+ message: issue.toString(),
2012
+ location: fileLocation
2013
+ }) : new ValidationIssue({
2014
+ message: `run '${run.name}': ${issue.toString()}`,
2015
+ location: new SourceLocation({
2016
+ ...entry.file.locate(run.location),
2017
+ path: ""
2018
+ })
2019
+ })));
2020
+ }
1538
2021
  if (issues.length > 0) throw new EvalError({
1539
2022
  category: ErrorCategory.Specification,
1540
2023
  message: "invalid scenario specification",
@@ -1542,6 +2025,71 @@ var ScenarioLoader = class {
1542
2025
  });
1543
2026
  return specs;
1544
2027
  }
2028
+ resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
2029
+ const issues = [];
2030
+ const used = /* @__PURE__ */ new Set();
2031
+ const resolver = scenarioResolver.withLevels([run.variables]);
2032
+ const value = {
2033
+ ...resolver.resolve(rest, document, issues, secrets, { used }),
2034
+ ...variables === void 0 ? {} : { variables }
2035
+ };
2036
+ const websocketIssues = [];
2037
+ let websocket;
2038
+ let connectionChecked = false;
2039
+ if (issues.length === 0 && configDocument !== void 0) {
2040
+ const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
2041
+ if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
2042
+ if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
2043
+ connectionChecked = true;
2044
+ }
2045
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
2046
+ if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
2047
+ return {
2048
+ run,
2049
+ value,
2050
+ used,
2051
+ secrets,
2052
+ issues,
2053
+ websocket,
2054
+ websocketIssues,
2055
+ connectionChecked
2056
+ };
2057
+ }
2058
+ static byRun(runs, place) {
2059
+ const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
2060
+ const sharedTexts = new Set(shared.map((issue) => issue.toString()));
2061
+ const result = shared.map((issue) => place(issue, void 0));
2062
+ for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
2063
+ return result;
2064
+ }
2065
+ static shared(lists) {
2066
+ if (lists.length === 0) return [];
2067
+ const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
2068
+ return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
2069
+ }
2070
+ static variablesLocation(document) {
2071
+ return new SourceLocation({
2072
+ ...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
2073
+ path: ""
2074
+ });
2075
+ }
2076
+ static unusedVariables(variables, used, results, document) {
2077
+ const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
2078
+ const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
2079
+ message: message(name),
2080
+ location: document.locateKey(["variables"], name)
2081
+ }));
2082
+ const seen = /* @__PURE__ */ new Set();
2083
+ for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
2084
+ const issue = new ValidationIssue({
2085
+ message: message(key.name),
2086
+ location: key.location
2087
+ });
2088
+ if (!seen.has(issue.toString())) issues.push(issue);
2089
+ seen.add(issue.toString());
2090
+ }
2091
+ return issues;
2092
+ }
1545
2093
  discover(target, config) {
1546
2094
  if (!(0, node_fs.existsSync)(target)) throw new EvalError({
1547
2095
  category: ErrorCategory.Specification,
@@ -1566,20 +2114,22 @@ var ScenarioLoader = class {
1566
2114
  }
1567
2115
  return files;
1568
2116
  }
1569
- build(value, document, secrets) {
2117
+ build(value, document, runName, websocket, secrets) {
1570
2118
  const scenario = value;
1571
2119
  const turns = scenario.turns;
1572
2120
  const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
1573
2121
  return new ConversationSpec({
1574
2122
  version: scenario.version,
1575
2123
  name: scenario.name,
2124
+ runName,
1576
2125
  description: typeof scenario.description === "string" ? scenario.description : "",
1577
2126
  turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
1578
2127
  conversation: conversation ? new ConversationExpectation({
1579
- criteria: conversation.criteria,
2128
+ criteria: conversation.should,
1580
2129
  location: document.locate(["expect", "conversation"])
1581
2130
  }) : void 0,
1582
2131
  location: document.locate([]),
2132
+ websocket,
1583
2133
  secrets
1584
2134
  });
1585
2135
  }
@@ -1605,7 +2155,7 @@ var ScenarioLoader = class {
1605
2155
  });
1606
2156
  const fields = response;
1607
2157
  return new ResponseExpectation({
1608
- criteria: fields.criteria,
2158
+ criteria: fields.should,
1609
2159
  contains: fields.contains ?? [],
1610
2160
  notContains: fields.notContains ?? [],
1611
2161
  location
@@ -2428,7 +2978,11 @@ var ConversationRunner = class {
2428
2978
  async run(spec, signal = NEVER_ABORTED) {
2429
2979
  const startedAt = Date.now();
2430
2980
  const outcomes = [];
2431
- const assertions = new AssertionRunner(this.config.websocket.protocol.capabilities);
2981
+ const config = new EvalConfig({
2982
+ ...this.config,
2983
+ websocket: spec.websocket ?? this.config.websocket
2984
+ });
2985
+ const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
2432
2986
  const history = [];
2433
2987
  const evaluate = this.semanticEvaluator(signal);
2434
2988
  const inline = !this.config.execution.continueOnFailure;
@@ -2453,7 +3007,7 @@ var ConversationRunner = class {
2453
3007
  try {
2454
3008
  const steps = (async () => {
2455
3009
  for (let attempt = 0;; attempt++) {
2456
- connection = this.createConnection(this.config);
3010
+ connection = this.createConnection(config);
2457
3011
  connectionAttempts++;
2458
3012
  try {
2459
3013
  await connection.connect();
@@ -2947,7 +3501,7 @@ var ScenarioRunner = class {
2947
3501
  required: minPass,
2948
3502
  attempts,
2949
3503
  tests: this.results.tests(attempts, minPass),
2950
- agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
3504
+ agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
2951
3505
  evaluatorUsage: this.tokens.evaluator(attempts)
2952
3506
  });
2953
3507
  }
@@ -3035,10 +3589,12 @@ var ConsoleReporter = class ConsoleReporter {
3035
3589
  scenario(result, repeat, tokenUsageCapability) {
3036
3590
  const spec = result.spec;
3037
3591
  const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
3038
- const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
3592
+ const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
3039
3593
  if (!this.verbose) return line;
3040
3594
  if (result.attempts.length === 0) return `${line}\n`;
3041
- const lines = [""];
3595
+ const websocket = spec.websocket;
3596
+ const subprotocols = websocket?.connection.subprotocols.length ?? 0;
3597
+ const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
3042
3598
  for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3043
3599
  lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
3044
3600
  lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
@@ -3046,6 +3602,9 @@ var ConsoleReporter = class ConsoleReporter {
3046
3602
  lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3047
3603
  return line + lines.join("\n");
3048
3604
  }
3605
+ static title(spec) {
3606
+ return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
3607
+ }
3049
3608
  summary(run, exitCode, reportPaths = []) {
3050
3609
  const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
3051
3610
  const lines = this.verbose ? [] : [""];
@@ -3077,7 +3636,7 @@ var ConsoleReporter = class ConsoleReporter {
3077
3636
  }
3078
3637
  problems(result) {
3079
3638
  const spec = result.spec;
3080
- const lines = [`${this.status(result.status, 6)} ${spec.name} (${spec.location.file})`];
3639
+ const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
3081
3640
  for (const attempt of result.attempts) {
3082
3641
  const execution = attempt.execution;
3083
3642
  if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
@@ -3251,11 +3810,17 @@ var JsonReporter = class JsonReporter {
3251
3810
  };
3252
3811
  }
3253
3812
  scenario(result, config) {
3254
- const capability = config.websocket.protocol.capabilities.tokenUsage;
3813
+ const websocket = result.spec.websocket ?? config.websocket;
3814
+ const capability = websocket.protocol.capabilities.tokenUsage;
3255
3815
  return {
3256
3816
  name: result.spec.name,
3817
+ runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
3257
3818
  file: result.spec.location.file,
3258
3819
  status: result.status,
3820
+ connection: {
3821
+ endpoint: websocket.endpoint,
3822
+ subprotocols: websocket.connection.subprotocols.length
3823
+ },
3259
3824
  ...JsonReporter.counts(result),
3260
3825
  agentUsage: JsonReporter.usage(result.agentUsage),
3261
3826
  evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),