@mappies/miniwat-eval 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,8 +1,8 @@
1
1
  import { Message, ResponseMessage, TokenUsage } from "@mappies/miniwat-agent-core";
2
2
  import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
3
3
  import { dirname, extname, join, relative, resolve } from "node:path";
4
- import { Ajv } from "ajv";
5
4
  import { LineCounter, isMap, isPair, isScalar, isSeq, parseDocument } from "yaml";
5
+ import { Ajv } from "ajv";
6
6
  import handlerWs from "configurapi-handler-ws";
7
7
  import { isDeepStrictEqual, styleText } from "node:util";
8
8
  import { Agent, InMemoryAgentStateStore } from "@mappies/miniwat-agent";
@@ -273,22 +273,70 @@ var TurnSpec = class {
273
273
  this.location = new SourceLocation(data?.location);
274
274
  }
275
275
  };
276
+ // src/entities/protocolCapabilities.ts
277
+ var ProtocolCapabilities = class {
278
+ toolUses;
279
+ toolResults;
280
+ tokenUsage;
281
+ constructor(data) {
282
+ this.toolUses = data?.toolUses ?? false;
283
+ this.toolResults = data?.toolResults ?? false;
284
+ this.tokenUsage = data?.tokenUsage ?? false;
285
+ }
286
+ };
287
+ // src/interfaces/requestFormat.ts
288
+ const RequestFormat = { Json: "json" };
289
+ // src/entities/requestConfig.ts
290
+ var RequestConfig = class {
291
+ format;
292
+ value;
293
+ inputPath;
294
+ constructor(data) {
295
+ this.format = data?.format ?? RequestFormat.Json;
296
+ this.value = data?.value;
297
+ this.inputPath = data?.inputPath;
298
+ }
299
+ };
300
+ // src/entities/protocolConfig.ts
301
+ var ProtocolConfig = class {
302
+ request;
303
+ capabilities;
304
+ constructor(data) {
305
+ this.request = new RequestConfig(data?.request);
306
+ this.capabilities = new ProtocolCapabilities(data?.capabilities);
307
+ }
308
+ };
309
+ // src/entities/webSocketConfig.ts
310
+ var WebSocketConfig = class {
311
+ endpoint;
312
+ connection;
313
+ protocol;
314
+ constructor(data) {
315
+ this.endpoint = data?.endpoint ?? "";
316
+ this.connection = new ConnectionConfig(data?.connection);
317
+ this.protocol = new ProtocolConfig(data?.protocol);
318
+ }
319
+ };
276
320
  // src/entities/conversationSpec.ts
277
321
  var ConversationSpec = class {
278
322
  version;
279
323
  name;
324
+ runName;
280
325
  description;
281
326
  turns;
282
327
  conversation;
283
328
  location;
329
+ websocket;
284
330
  secrets;
285
331
  constructor(data) {
286
332
  this.version = data?.version ?? 1;
287
333
  this.name = data?.name ?? "";
334
+ this.runName = data?.runName ?? "";
288
335
  this.description = data?.description ?? "";
289
336
  this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
290
337
  this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
291
338
  this.location = new SourceLocation(data?.location);
339
+ this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
292
340
  this.secrets = [...data?.secrets ?? []];
293
341
  }
294
342
  };
@@ -390,55 +438,12 @@ var ReportConfig = class {
390
438
  this.formats = data?.formats ?? [];
391
439
  }
392
440
  };
393
- // src/entities/protocolCapabilities.ts
394
- var ProtocolCapabilities = class {
395
- toolUses;
396
- toolResults;
397
- tokenUsage;
398
- constructor(data) {
399
- this.toolUses = data?.toolUses ?? false;
400
- this.toolResults = data?.toolResults ?? false;
401
- this.tokenUsage = data?.tokenUsage ?? false;
402
- }
403
- };
404
- // src/interfaces/requestFormat.ts
405
- const RequestFormat = { Json: "json" };
406
- // src/entities/requestConfig.ts
407
- var RequestConfig = class {
408
- format;
409
- value;
410
- inputPath;
411
- constructor(data) {
412
- this.format = data?.format ?? RequestFormat.Json;
413
- this.value = data?.value;
414
- this.inputPath = data?.inputPath;
415
- }
416
- };
417
- // src/entities/protocolConfig.ts
418
- var ProtocolConfig = class {
419
- request;
420
- capabilities;
421
- constructor(data) {
422
- this.request = new RequestConfig(data?.request);
423
- this.capabilities = new ProtocolCapabilities(data?.capabilities);
424
- }
425
- };
426
- // src/entities/webSocketConfig.ts
427
- var WebSocketConfig = class {
428
- endpoint;
429
- connection;
430
- protocol;
431
- constructor(data) {
432
- this.endpoint = data?.endpoint ?? "";
433
- this.connection = new ConnectionConfig(data?.connection);
434
- this.protocol = new ProtocolConfig(data?.protocol);
435
- }
436
- };
437
441
  // src/entities/evalConfig.ts
438
442
  var EvalConfig = class {
439
443
  version;
440
444
  configPath;
441
445
  parametersPath;
446
+ variables;
442
447
  websocket;
443
448
  execution;
444
449
  evaluator;
@@ -448,6 +453,7 @@ var EvalConfig = class {
448
453
  this.version = data?.version ?? 1;
449
454
  this.configPath = data?.configPath ?? "";
450
455
  this.parametersPath = data?.parametersPath ?? "";
456
+ this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
451
457
  this.websocket = new WebSocketConfig(data?.websocket);
452
458
  this.execution = new ExecutionConfig(data?.execution);
453
459
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -759,6 +765,14 @@ var eval_config_schema_default = {
759
765
  ],
760
766
  properties: {
761
767
  "version": { "const": 1 },
768
+ "variables": {
769
+ "type": "object",
770
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
771
+ "additionalProperties": { "anyOf": [{ "type": "string" }, {
772
+ "type": "array",
773
+ "items": { "type": "string" }
774
+ }] }
775
+ },
762
776
  "websocket": {
763
777
  "type": "object",
764
778
  "additionalProperties": false,
@@ -917,18 +931,80 @@ var eval_config_schema_default = {
917
931
  }
918
932
  }
919
933
  };
934
+ // src/yaml/yamlDocument.ts
935
+ var YamlDocument = class YamlDocument {
936
+ file;
937
+ document;
938
+ lineCounter;
939
+ value;
940
+ errors;
941
+ constructor(file, document, lineCounter) {
942
+ this.file = file;
943
+ this.document = document;
944
+ this.lineCounter = lineCounter;
945
+ this.value = document.toJS();
946
+ this.errors = document.errors.map((error) => new ValidationIssue({
947
+ message: error.message.split("\n")[0],
948
+ location: this.toLocation(error.pos[0], [])
949
+ }));
950
+ }
951
+ locate(path) {
952
+ return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
953
+ }
954
+ locateKey(path, key) {
955
+ const parent = this.findNode(path);
956
+ if (isMap(parent)) {
957
+ const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
958
+ if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
959
+ }
960
+ return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
961
+ }
962
+ static formatPath(path) {
963
+ return path.map((segment, index) => {
964
+ if (typeof segment === "number") return `[${segment}]`;
965
+ return index === 0 ? segment : `.${segment}`;
966
+ }).join("");
967
+ }
968
+ findNode(path) {
969
+ let node = this.document.contents ?? void 0;
970
+ for (const segment of path) {
971
+ let next;
972
+ if (isMap(node)) {
973
+ const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
974
+ next = pair?.value ?? pair?.key;
975
+ } else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
976
+ if (!next) break;
977
+ node = next;
978
+ }
979
+ return node;
980
+ }
981
+ toLocation(offset, path) {
982
+ const position = this.lineCounter.linePos(offset);
983
+ return new SourceLocation({
984
+ file: this.file,
985
+ line: position.line,
986
+ column: position.col,
987
+ path: YamlDocument.formatPath(path)
988
+ });
989
+ }
990
+ };
920
991
  // src/yaml/environmentResolver.ts
921
992
  const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
922
993
  const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
923
994
  var EnvironmentResolver = class EnvironmentResolver {
924
995
  environment;
925
996
  parameters;
926
- constructor(environment = process.env, parameters = {}) {
997
+ levels;
998
+ constructor(environment = process.env, parameters = {}, levels = []) {
927
999
  this.environment = environment;
928
1000
  this.parameters = parameters;
1001
+ this.levels = levels;
1002
+ }
1003
+ withLevels(levels) {
1004
+ return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
929
1005
  }
930
- resolve(value, document, issues, secrets = []) {
931
- return this.resolveValue(value, [], document, issues, secrets);
1006
+ resolve(value, document, issues, secrets = [], options = {}) {
1007
+ return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
932
1008
  }
933
1009
  static hasReference(value) {
934
1010
  return new RegExp(REFERENCE.source).test(value);
@@ -936,22 +1012,45 @@ var EnvironmentResolver = class EnvironmentResolver {
936
1012
  static isSecretName(name) {
937
1013
  return SECRET_NAME.test(name);
938
1014
  }
939
- resolveValue(value, path, document, issues, secrets) {
1015
+ find(name) {
1016
+ for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
1017
+ return this.environment[name];
1018
+ }
1019
+ sources() {
1020
+ const sources = [
1021
+ ...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
1022
+ ...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
1023
+ "environment variable"
1024
+ ];
1025
+ return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
1026
+ }
1027
+ resolveValue(value, path, document, issues, secrets, options) {
940
1028
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
941
- const resolved = Object.hasOwn(this.parameters, name) ? this.parameters[name] : this.environment[name];
1029
+ options.used?.add(name);
1030
+ const resolved = this.find(name);
942
1031
  if (resolved === void 0) {
943
- const source = Object.keys(this.parameters).length > 0 ? "parameter or environment variable" : "environment variable";
1032
+ if (options.optional?.(path)) {
1033
+ options.unresolved?.add(YamlDocument.formatPath(path));
1034
+ return reference;
1035
+ }
944
1036
  issues.push(new ValidationIssue({
945
- message: `${source} '${name}' is not set`,
1037
+ message: `${this.sources()} '${name}' is not set`,
946
1038
  location: document.locate(path)
947
1039
  }));
948
1040
  return reference;
949
1041
  }
950
- if (EnvironmentResolver.isSecretName(name) && !secrets.includes(resolved)) secrets.push(resolved);
1042
+ if (Array.isArray(resolved)) {
1043
+ issues.push(new ValidationIssue({
1044
+ message: `variable '${name}' is a list, which can only be used as runWith.from`,
1045
+ location: document.locate(path)
1046
+ }));
1047
+ return reference;
1048
+ }
1049
+ if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
951
1050
  return resolved;
952
1051
  });
953
- if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
954
- if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
1052
+ if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
1053
+ if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
955
1054
  return value;
956
1055
  }
957
1056
  };
@@ -1035,63 +1134,6 @@ var SchemaValidator = class SchemaValidator {
1035
1134
  return path;
1036
1135
  }
1037
1136
  };
1038
- // src/yaml/yamlDocument.ts
1039
- var YamlDocument = class YamlDocument {
1040
- file;
1041
- document;
1042
- lineCounter;
1043
- value;
1044
- errors;
1045
- constructor(file, document, lineCounter) {
1046
- this.file = file;
1047
- this.document = document;
1048
- this.lineCounter = lineCounter;
1049
- this.value = document.toJS();
1050
- this.errors = document.errors.map((error) => new ValidationIssue({
1051
- message: error.message.split("\n")[0],
1052
- location: this.toLocation(error.pos[0], [])
1053
- }));
1054
- }
1055
- locate(path) {
1056
- return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
1057
- }
1058
- locateKey(path, key) {
1059
- const parent = this.findNode(path);
1060
- if (isMap(parent)) {
1061
- const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
1062
- if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
1063
- }
1064
- return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
1065
- }
1066
- static formatPath(path) {
1067
- return path.map((segment, index) => {
1068
- if (typeof segment === "number") return `[${segment}]`;
1069
- return index === 0 ? segment : `.${segment}`;
1070
- }).join("");
1071
- }
1072
- findNode(path) {
1073
- let node = this.document.contents ?? void 0;
1074
- for (const segment of path) {
1075
- let next;
1076
- if (isMap(node)) {
1077
- const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
1078
- next = pair?.value ?? pair?.key;
1079
- } else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
1080
- if (!next) break;
1081
- node = next;
1082
- }
1083
- return node;
1084
- }
1085
- toLocation(offset, path) {
1086
- const position = this.lineCounter.linePos(offset);
1087
- return new SourceLocation({
1088
- file: this.file,
1089
- line: position.line,
1090
- column: position.col,
1091
- path: YamlDocument.formatPath(path)
1092
- });
1093
- }
1094
- };
1095
1137
  // src/yaml/yamlDocumentLoader.ts
1096
1138
  var YamlDocumentLoader = class {
1097
1139
  load(filePath, displayPath) {
@@ -1110,11 +1152,150 @@ var YamlDocumentLoader = class {
1110
1152
  function isRecord(value) {
1111
1153
  return value !== null && typeof value === "object" && !Array.isArray(value);
1112
1154
  }
1155
+ // src/yaml/variablesReader.ts
1156
+ const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
1157
+ var VariablesReader = class {
1158
+ read(document, resolver, issues, secrets) {
1159
+ const raw = isRecord(document.value) ? document.value.variables : void 0;
1160
+ if (raw === void 0) return {};
1161
+ if (!isRecord(raw)) {
1162
+ issues.push(new ValidationIssue({
1163
+ message: "must be object",
1164
+ location: document.locate(["variables"])
1165
+ }));
1166
+ return {};
1167
+ }
1168
+ const variables = {};
1169
+ for (const [name, value] of Object.entries(raw)) {
1170
+ if (!VARIABLE_NAME.test(name)) {
1171
+ issues.push(new ValidationIssue({
1172
+ message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
1173
+ location: document.locateKey(["variables"], name)
1174
+ }));
1175
+ continue;
1176
+ }
1177
+ if (Array.isArray(value)) {
1178
+ const bad = value.findIndex((item) => typeof item !== "string");
1179
+ if (bad >= 0) {
1180
+ issues.push(new ValidationIssue({
1181
+ message: "must be string",
1182
+ location: document.locate([
1183
+ "variables",
1184
+ name,
1185
+ bad
1186
+ ])
1187
+ }));
1188
+ continue;
1189
+ }
1190
+ variables[name] = [...value];
1191
+ continue;
1192
+ }
1193
+ if (typeof value !== "string") {
1194
+ issues.push(new ValidationIssue({
1195
+ message: "must be string or a list of strings",
1196
+ location: document.locate(["variables", name])
1197
+ }));
1198
+ continue;
1199
+ }
1200
+ const before = issues.length;
1201
+ const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
1202
+ if (issues.length > before) continue;
1203
+ const text = resolved.variables[name];
1204
+ variables[name] = text;
1205
+ if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
1206
+ }
1207
+ return variables;
1208
+ }
1209
+ };
1210
+ // src/config/webSocketResolver.ts
1211
+ const WEBSOCKET_SCHEMA = {
1212
+ ...eval_config_schema_default,
1213
+ required: ["websocket"],
1214
+ additionalProperties: true,
1215
+ properties: { websocket: eval_config_schema_default.properties.websocket }
1216
+ };
1217
+ var WebSocketResolver = class WebSocketResolver {
1218
+ schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
1219
+ resolve(document, resolver, issues, secrets, used, defer) {
1220
+ const raw = isRecord(document.value) ? document.value.websocket : void 0;
1221
+ const unresolved = /* @__PURE__ */ new Set();
1222
+ const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
1223
+ const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
1224
+ used,
1225
+ optional,
1226
+ unresolved
1227
+ });
1228
+ const subprotocols = WebSocketResolver.subprotocols(raw);
1229
+ if (subprotocols !== void 0) {
1230
+ const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
1231
+ const set = !unresolved.has(YamlDocument.formatPath([
1232
+ "websocket",
1233
+ "connection",
1234
+ "subprotocols",
1235
+ index
1236
+ ]));
1237
+ const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
1238
+ return set && !empty;
1239
+ });
1240
+ value.websocket.connection.subprotocols = kept;
1241
+ }
1242
+ const prefix = YamlDocument.formatPath([
1243
+ "websocket",
1244
+ "connection",
1245
+ "subprotocols"
1246
+ ]);
1247
+ return {
1248
+ value,
1249
+ deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
1250
+ };
1251
+ }
1252
+ validate(value, document) {
1253
+ const issues = this.schemaValidator.validate(value, document);
1254
+ if (issues.length > 0) return issues;
1255
+ return WebSocketResolver.validateInputPath(value, document);
1256
+ }
1257
+ static validateInputPath(value, document) {
1258
+ const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
1259
+ if (typeof request.inputPath !== "string") return [];
1260
+ const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
1261
+ const last = tokens.pop();
1262
+ let parent = request.value;
1263
+ for (const token of tokens) parent = WebSocketResolver.child(parent, token);
1264
+ if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
1265
+ return [new ValidationIssue({
1266
+ message: "must point to a missing or string value whose parent exists in request.value",
1267
+ location: document.locate([
1268
+ "websocket",
1269
+ "protocol",
1270
+ "request",
1271
+ "inputPath"
1272
+ ])
1273
+ })];
1274
+ }
1275
+ static section(value, ...path) {
1276
+ let current = value;
1277
+ for (const key of path) current = WebSocketResolver.child(current, key);
1278
+ return isRecord(current) ? current : {};
1279
+ }
1280
+ static child(value, key) {
1281
+ if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
1282
+ return isRecord(value) ? value[key] : void 0;
1283
+ }
1284
+ static isSubprotocol(path) {
1285
+ return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
1286
+ }
1287
+ static subprotocols(websocket) {
1288
+ const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
1289
+ return Array.isArray(subprotocols) ? subprotocols : void 0;
1290
+ }
1291
+ };
1113
1292
  // src/config/evalConfigLoader.ts
1114
1293
  var EvalConfigLoader = class EvalConfigLoader {
1115
1294
  cwd;
1116
1295
  documentLoader = new YamlDocumentLoader();
1117
1296
  schemaValidator = new SchemaValidator(eval_config_schema_default);
1297
+ variablesReader = new VariablesReader();
1298
+ webSocketResolver = new WebSocketResolver();
1118
1299
  environmentResolver;
1119
1300
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1120
1301
  this.cwd = cwd;
@@ -1125,19 +1306,47 @@ var EvalConfigLoader = class EvalConfigLoader {
1125
1306
  const issues = [...document.errors];
1126
1307
  if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
1127
1308
  const named = [];
1128
- const value = issues.length === 0 ? this.environmentResolver.resolve(document.value, document, issues, named) : void 0;
1129
- if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
1130
- if (issues.length === 0) issues.push(...this.validateInputPath(value, document));
1309
+ const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
1310
+ let value = {};
1311
+ if (issues.length === 0) {
1312
+ const resolver = this.environmentResolver.withLevels([variables]);
1313
+ const { websocket: _websocket, variables: _variables, ...rest } = document.value;
1314
+ const resolved = resolver.resolve(rest, document, issues, named);
1315
+ const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
1316
+ value = {
1317
+ ...resolved,
1318
+ ..._variables === void 0 ? {} : { variables },
1319
+ ..._websocket === void 0 ? {} : websocket.value
1320
+ };
1321
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
1322
+ if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
1323
+ }
1131
1324
  if (issues.length > 0) throw new EvalError({
1132
1325
  category: ErrorCategory.Configuration,
1133
1326
  message: "invalid configuration",
1134
1327
  issues
1135
1328
  });
1136
- const config = this.build(value, configPath, named);
1329
+ const config = this.build(value, configPath, variables, named);
1137
1330
  this.applyRepetition(config, override, value, document);
1138
1331
  EvalConfigLoader.applyReport(config, report);
1139
1332
  return config;
1140
1333
  }
1334
+ static buildWebSocket(value) {
1335
+ const section = (...path) => WebSocketResolver.section(value, ...path);
1336
+ const connection = section("websocket", "connection");
1337
+ const request = section("websocket", "protocol", "request");
1338
+ return new WebSocketConfig({
1339
+ endpoint: WebSocketResolver.section(value, "websocket").endpoint,
1340
+ connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
1341
+ protocol: new ProtocolConfig({
1342
+ request: new RequestConfig({
1343
+ ...request,
1344
+ format: request.format ?? RequestFormat.Json
1345
+ }),
1346
+ capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
1347
+ })
1348
+ });
1349
+ }
1141
1350
  static applyReport(config, report) {
1142
1351
  if (report.formats === void 0) return;
1143
1352
  const known = Object.values(ReportFormat);
@@ -1159,45 +1368,17 @@ var EvalConfigLoader = class EvalConfigLoader {
1159
1368
  location: document.locate(["evaluator", "apiKey"])
1160
1369
  })];
1161
1370
  }
1162
- validateInputPath(value, document) {
1163
- const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
1164
- if (typeof request.inputPath !== "string") return [];
1165
- const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
1166
- const last = tokens.pop();
1167
- let parent = request.value;
1168
- for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
1169
- if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
1170
- return [new ValidationIssue({
1171
- message: "must point to a missing or string value whose parent exists in request.value",
1172
- location: document.locate([
1173
- "websocket",
1174
- "protocol",
1175
- "request",
1176
- "inputPath"
1177
- ])
1178
- })];
1179
- }
1180
- build(value, configPath, named) {
1371
+ build(value, configPath, variables, named) {
1181
1372
  const section = (...path) => EvalConfigLoader.section(value, ...path);
1182
1373
  const configDirectory = dirname(configPath);
1183
- const connection = section("websocket", "connection");
1184
- const request = section("websocket", "protocol", "request");
1374
+ const websocket = EvalConfigLoader.buildWebSocket(value);
1185
1375
  const evaluator = section("evaluator");
1186
1376
  const report = section("report");
1187
1377
  return new EvalConfig({
1188
1378
  version: value.version,
1189
1379
  configPath,
1190
- websocket: new WebSocketConfig({
1191
- endpoint: EvalConfigLoader.section(value, "websocket").endpoint,
1192
- connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
1193
- protocol: new ProtocolConfig({
1194
- request: new RequestConfig({
1195
- ...request,
1196
- format: request.format ?? RequestFormat.Json
1197
- }),
1198
- capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
1199
- })
1200
- }),
1380
+ variables,
1381
+ websocket,
1201
1382
  execution: new ExecutionConfig({
1202
1383
  ...DEFAULT_CONFIG.execution,
1203
1384
  ...section("execution"),
@@ -1218,7 +1399,7 @@ var EvalConfigLoader = class EvalConfigLoader {
1218
1399
  directory: resolve(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
1219
1400
  formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
1220
1401
  }),
1221
- secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols ?? [], named)
1402
+ secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
1222
1403
  });
1223
1404
  }
1224
1405
  applyRepetition(config, override, value, document) {
@@ -1271,13 +1452,7 @@ var EvalConfigLoader = class EvalConfigLoader {
1271
1452
  ])];
1272
1453
  }
1273
1454
  static section(value, ...path) {
1274
- let current = value;
1275
- for (const key of path) current = EvalConfigLoader.child(current, key);
1276
- return isRecord(current) ? current : {};
1277
- }
1278
- static child(value, key) {
1279
- if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
1280
- return isRecord(value) ? value[key] : void 0;
1455
+ return WebSocketResolver.section(value, ...path);
1281
1456
  }
1282
1457
  };
1283
1458
  // src/config/parametersLoader.ts
@@ -1329,6 +1504,48 @@ var conversation_schema_default = {
1329
1504
  "version": { "const": 1 },
1330
1505
  "name": { "$ref": "#/definitions/text" },
1331
1506
  "description": { "type": "string" },
1507
+ "variables": {
1508
+ "type": "object",
1509
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
1510
+ "additionalProperties": { "anyOf": [{ "type": "string" }, {
1511
+ "type": "array",
1512
+ "items": { "type": "string" }
1513
+ }] }
1514
+ },
1515
+ "runWith": {
1516
+ "description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
1517
+ "anyOf": [{
1518
+ "type": "array",
1519
+ "minItems": 1,
1520
+ "items": {
1521
+ "type": "object",
1522
+ "required": ["name"],
1523
+ "properties": { "name": {
1524
+ "type": "string",
1525
+ "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
1526
+ } },
1527
+ "additionalProperties": { "type": "string" }
1528
+ }
1529
+ }, {
1530
+ "type": "object",
1531
+ "additionalProperties": false,
1532
+ "required": ["variable", "from"],
1533
+ "properties": {
1534
+ "variable": {
1535
+ "type": "string",
1536
+ "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
1537
+ },
1538
+ "from": { "anyOf": [{
1539
+ "type": "array",
1540
+ "minItems": 1,
1541
+ "items": { "type": "string" }
1542
+ }, {
1543
+ "type": "string",
1544
+ "pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
1545
+ }] }
1546
+ }
1547
+ }]
1548
+ },
1332
1549
  "turns": {
1333
1550
  "type": "array",
1334
1551
  "minItems": 1,
@@ -1341,8 +1558,8 @@ var conversation_schema_default = {
1341
1558
  "properties": { "conversation": {
1342
1559
  "type": "object",
1343
1560
  "additionalProperties": false,
1344
- "required": ["criteria"],
1345
- "properties": { "criteria": { "$ref": "#/definitions/text" } }
1561
+ "required": ["should"],
1562
+ "properties": { "should": { "$ref": "#/definitions/text" } }
1346
1563
  } }
1347
1564
  }
1348
1565
  },
@@ -1385,12 +1602,12 @@ var conversation_schema_default = {
1385
1602
  }
1386
1603
  },
1387
1604
  "response": {
1388
- "description": "A string is shorthand for criteria.",
1605
+ "description": "A string is shorthand for should.",
1389
1606
  "type": ["string", "object"],
1390
1607
  "pattern": "\\S",
1391
1608
  "additionalProperties": false,
1392
1609
  "properties": {
1393
- "criteria": { "$ref": "#/definitions/text" },
1610
+ "should": { "$ref": "#/definitions/text" },
1394
1611
  "contains": { "$ref": "#/definitions/textList" },
1395
1612
  "notContains": { "$ref": "#/definitions/textList" }
1396
1613
  }
@@ -1422,7 +1639,7 @@ var conversation_schema_default = {
1422
1639
  };
1423
1640
  // src/scenarios/conversationValidator.ts
1424
1641
  const RESPONSE_KEYS = [
1425
- "criteria",
1642
+ "should",
1426
1643
  "contains",
1427
1644
  "notContains"
1428
1645
  ];
@@ -1445,7 +1662,7 @@ var ConversationValidator = class ConversationValidator {
1445
1662
  validateResponse(response, path, document, issues) {
1446
1663
  if (!RESPONSE_KEYS.some((key) => key in response)) {
1447
1664
  issues.push(new ValidationIssue({
1448
- message: "response must have at least one of: criteria, contains, notContains",
1665
+ message: "response must have at least one of: should, contains, notContains",
1449
1666
  location: document.locate(path)
1450
1667
  }));
1451
1668
  return;
@@ -1486,13 +1703,231 @@ var ConversationValidator = class ConversationValidator {
1486
1703
  return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
1487
1704
  }
1488
1705
  };
1706
+ // src/scenarios/runWithReader.ts
1707
+ const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
1708
+ const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
1709
+ const FORMS = "must be a list of rows, or an object with variable and from";
1710
+ const SINGLE_RUN = {
1711
+ name: "",
1712
+ variables: {},
1713
+ keys: [],
1714
+ location: [],
1715
+ secrets: []
1716
+ };
1717
+ var RunWithReader = class RunWithReader {
1718
+ read(document, resolver, issues, used) {
1719
+ const raw = isRecord(document.value) ? document.value.runWith : void 0;
1720
+ if (raw === void 0) return void 0;
1721
+ if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
1722
+ if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
1723
+ issues.push(new ValidationIssue({
1724
+ message: FORMS,
1725
+ location: document.locate(["runWith"])
1726
+ }));
1727
+ return [];
1728
+ }
1729
+ rows(rows, document, resolver, issues, used) {
1730
+ if (rows.length === 0) {
1731
+ issues.push(new ValidationIssue({
1732
+ message: "must not be empty",
1733
+ location: document.locate(["runWith"])
1734
+ }));
1735
+ return [];
1736
+ }
1737
+ const names = /* @__PURE__ */ new Set();
1738
+ const runs = [];
1739
+ rows.forEach((row, index) => {
1740
+ const path = ["runWith", index];
1741
+ if (!isRecord(row)) {
1742
+ issues.push(new ValidationIssue({
1743
+ message: "must be object",
1744
+ location: document.locate(path)
1745
+ }));
1746
+ return;
1747
+ }
1748
+ const name = row.name;
1749
+ if (name === void 0) issues.push(new ValidationIssue({
1750
+ message: "missing required key 'name'",
1751
+ location: document.locate(path)
1752
+ }));
1753
+ else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
1754
+ message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
1755
+ location: document.locate([...path, "name"])
1756
+ }));
1757
+ else if (names.has(name)) issues.push(new ValidationIssue({
1758
+ message: `duplicate run name '${name}'`,
1759
+ location: document.locate([...path, "name"])
1760
+ }));
1761
+ else names.add(name);
1762
+ const run = {
1763
+ name: typeof name === "string" ? name : "",
1764
+ variables: {},
1765
+ keys: [],
1766
+ location: path,
1767
+ secrets: []
1768
+ };
1769
+ for (const [key, value] of Object.entries(row)) {
1770
+ if (key === "name") continue;
1771
+ if (!VARIABLE_NAME.test(key)) {
1772
+ issues.push(new ValidationIssue({
1773
+ message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
1774
+ location: document.locateKey(path, key)
1775
+ }));
1776
+ continue;
1777
+ }
1778
+ if (typeof value !== "string") {
1779
+ issues.push(new ValidationIssue({
1780
+ message: "must be string",
1781
+ location: document.locate([...path, key])
1782
+ }));
1783
+ continue;
1784
+ }
1785
+ const resolved = resolver.resolve(value, document, issues, run.secrets, {
1786
+ used,
1787
+ at: [...path, key]
1788
+ });
1789
+ RunWithReader.set(run, key, resolved, document.locateKey(path, key));
1790
+ }
1791
+ runs.push(run);
1792
+ });
1793
+ return runs;
1794
+ }
1795
+ list(form, document, resolver, issues, used) {
1796
+ const before = issues.length;
1797
+ for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
1798
+ message: `unknown key '${key}'`,
1799
+ location: document.locateKey(["runWith"], key)
1800
+ }));
1801
+ const variable = form.variable;
1802
+ if (variable === void 0) issues.push(new ValidationIssue({
1803
+ message: "missing required key 'variable'",
1804
+ location: document.locate(["runWith"])
1805
+ }));
1806
+ else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
1807
+ message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
1808
+ location: document.locate(["runWith", "variable"])
1809
+ }));
1810
+ const entries = this.entries(form.from, document, resolver, issues, used);
1811
+ if (issues.length > before || entries === void 0) return [];
1812
+ const name = variable;
1813
+ const keyLocation = document.locate(["runWith", "variable"]);
1814
+ const names = /* @__PURE__ */ new Set();
1815
+ const runs = [];
1816
+ for (const { entry, path } of entries) {
1817
+ if (!VARIABLE_NAME.test(entry)) {
1818
+ issues.push(new ValidationIssue({
1819
+ message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
1820
+ location: document.locate(path)
1821
+ }));
1822
+ continue;
1823
+ }
1824
+ if (names.has(entry)) {
1825
+ issues.push(new ValidationIssue({
1826
+ message: `duplicate run name '${entry}'`,
1827
+ location: document.locate(path)
1828
+ }));
1829
+ continue;
1830
+ }
1831
+ names.add(entry);
1832
+ const run = {
1833
+ name: entry,
1834
+ variables: {},
1835
+ keys: [],
1836
+ location: path,
1837
+ secrets: []
1838
+ };
1839
+ const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
1840
+ used,
1841
+ at: path
1842
+ });
1843
+ RunWithReader.set(run, name, value, keyLocation);
1844
+ runs.push(run);
1845
+ }
1846
+ return runs;
1847
+ }
1848
+ entries(from, document, resolver, issues, used) {
1849
+ const path = ["runWith", "from"];
1850
+ const location = document.locate(path);
1851
+ if (from === void 0) {
1852
+ issues.push(new ValidationIssue({
1853
+ message: "missing required key 'from'",
1854
+ location: document.locate(["runWith"])
1855
+ }));
1856
+ return;
1857
+ }
1858
+ let list;
1859
+ let at;
1860
+ if (Array.isArray(from)) {
1861
+ list = from;
1862
+ at = (index) => [...path, index];
1863
+ } else {
1864
+ const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
1865
+ if (reference === null) {
1866
+ issues.push(new ValidationIssue({
1867
+ message: "must be a list, or a list variable such as ${tokens}",
1868
+ location
1869
+ }));
1870
+ return;
1871
+ }
1872
+ const name = reference[1];
1873
+ const found = resolver.find(name);
1874
+ used.add(name);
1875
+ if (found === void 0) {
1876
+ issues.push(new ValidationIssue({
1877
+ message: `list variable '${name}' is not set`,
1878
+ location
1879
+ }));
1880
+ return;
1881
+ }
1882
+ if (!Array.isArray(found)) {
1883
+ issues.push(new ValidationIssue({
1884
+ message: `variable '${name}' is not a list`,
1885
+ location
1886
+ }));
1887
+ return;
1888
+ }
1889
+ list = found;
1890
+ at = () => path;
1891
+ }
1892
+ if (list.length === 0) {
1893
+ issues.push(new ValidationIssue({
1894
+ message: "must not be empty",
1895
+ location
1896
+ }));
1897
+ return;
1898
+ }
1899
+ const bad = list.findIndex((item) => typeof item !== "string");
1900
+ if (bad >= 0) {
1901
+ issues.push(new ValidationIssue({
1902
+ message: "must be string",
1903
+ location: document.locate(at(bad))
1904
+ }));
1905
+ return;
1906
+ }
1907
+ return list.map((entry, index) => ({
1908
+ entry,
1909
+ path: at(index)
1910
+ }));
1911
+ }
1912
+ static set(run, name, value, location) {
1913
+ run.variables[name] = value;
1914
+ run.keys.push({
1915
+ name,
1916
+ location
1917
+ });
1918
+ if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
1919
+ }
1920
+ };
1489
1921
  // src/scenarios/scenarioLoader.ts
1490
1922
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
1491
- var ScenarioLoader = class {
1923
+ var ScenarioLoader = class ScenarioLoader {
1492
1924
  cwd;
1493
1925
  documentLoader = new YamlDocumentLoader();
1494
1926
  schemaValidator = new SchemaValidator(conversation_schema_default);
1495
1927
  conversationValidator = new ConversationValidator();
1928
+ variablesReader = new VariablesReader();
1929
+ runWithReader = new RunWithReader();
1930
+ webSocketResolver = new WebSocketResolver();
1496
1931
  environmentResolver;
1497
1932
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1498
1933
  this.cwd = cwd;
@@ -1500,18 +1935,66 @@ var ScenarioLoader = class {
1500
1935
  }
1501
1936
  load(target, config) {
1502
1937
  const files = this.discover(resolve(this.cwd, target), config);
1938
+ const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.cwd, config.configPath)) : void 0;
1503
1939
  const issues = [];
1940
+ const connectionIssues = [];
1504
1941
  const specs = [];
1505
1942
  for (const file of files) {
1506
1943
  const document = this.documentLoader.load(file, relative(this.cwd, file));
1507
1944
  const fileIssues = [...document.errors];
1508
- const secrets = [];
1509
- const value = fileIssues.length === 0 ? this.environmentResolver.resolve(document.value, document, fileIssues, secrets) : void 0;
1510
- if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
1511
- if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
1512
- if (fileIssues.length === 0) specs.push(this.build(value, document, secrets));
1945
+ const fileSecrets = [];
1946
+ const used = /* @__PURE__ */ new Set();
1947
+ const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
1948
+ const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
1949
+ const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
1950
+ if (fileIssues.length > 0) {
1951
+ issues.push(...fileIssues);
1952
+ continue;
1953
+ }
1954
+ const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
1955
+ const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
1956
+ for (const result of results) for (const name of result.used) used.add(name);
1957
+ fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
1958
+ run: result.run,
1959
+ issues: result.issues
1960
+ })), (issue, run) => run === void 0 ? issue : new ValidationIssue({
1961
+ message: `run '${run.name}': ${issue.message}`,
1962
+ location: issue.location
1963
+ })));
1964
+ const attempted = results.filter((result) => result.connectionChecked);
1965
+ if (attempted.length > 0) connectionIssues.push({
1966
+ file: document,
1967
+ runs: attempted.map((result) => ({
1968
+ run: result.run,
1969
+ issues: result.websocketIssues
1970
+ }))
1971
+ });
1972
+ if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
1973
+ if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
1974
+ const websocket = result.websocket;
1975
+ specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
1976
+ }
1513
1977
  else issues.push(...fileIssues);
1514
1978
  }
1979
+ const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
1980
+ if (shared.length > 0) throw new EvalError({
1981
+ category: ErrorCategory.Configuration,
1982
+ message: "invalid configuration",
1983
+ issues: shared
1984
+ });
1985
+ for (const entry of connectionIssues) {
1986
+ const fileLocation = ScenarioLoader.variablesLocation(entry.file);
1987
+ issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
1988
+ message: issue.toString(),
1989
+ location: fileLocation
1990
+ }) : new ValidationIssue({
1991
+ message: `run '${run.name}': ${issue.toString()}`,
1992
+ location: new SourceLocation({
1993
+ ...entry.file.locate(run.location),
1994
+ path: ""
1995
+ })
1996
+ })));
1997
+ }
1515
1998
  if (issues.length > 0) throw new EvalError({
1516
1999
  category: ErrorCategory.Specification,
1517
2000
  message: "invalid scenario specification",
@@ -1519,6 +2002,71 @@ var ScenarioLoader = class {
1519
2002
  });
1520
2003
  return specs;
1521
2004
  }
2005
+ resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
2006
+ const issues = [];
2007
+ const used = /* @__PURE__ */ new Set();
2008
+ const resolver = scenarioResolver.withLevels([run.variables]);
2009
+ const value = {
2010
+ ...resolver.resolve(rest, document, issues, secrets, { used }),
2011
+ ...variables === void 0 ? {} : { variables }
2012
+ };
2013
+ const websocketIssues = [];
2014
+ let websocket;
2015
+ let connectionChecked = false;
2016
+ if (issues.length === 0 && configDocument !== void 0) {
2017
+ const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
2018
+ if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
2019
+ if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
2020
+ connectionChecked = true;
2021
+ }
2022
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
2023
+ if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
2024
+ return {
2025
+ run,
2026
+ value,
2027
+ used,
2028
+ secrets,
2029
+ issues,
2030
+ websocket,
2031
+ websocketIssues,
2032
+ connectionChecked
2033
+ };
2034
+ }
2035
+ static byRun(runs, place) {
2036
+ const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
2037
+ const sharedTexts = new Set(shared.map((issue) => issue.toString()));
2038
+ const result = shared.map((issue) => place(issue, void 0));
2039
+ for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
2040
+ return result;
2041
+ }
2042
+ static shared(lists) {
2043
+ if (lists.length === 0) return [];
2044
+ const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
2045
+ return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
2046
+ }
2047
+ static variablesLocation(document) {
2048
+ return new SourceLocation({
2049
+ ...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
2050
+ path: ""
2051
+ });
2052
+ }
2053
+ static unusedVariables(variables, used, results, document) {
2054
+ const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
2055
+ const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
2056
+ message: message(name),
2057
+ location: document.locateKey(["variables"], name)
2058
+ }));
2059
+ const seen = /* @__PURE__ */ new Set();
2060
+ for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
2061
+ const issue = new ValidationIssue({
2062
+ message: message(key.name),
2063
+ location: key.location
2064
+ });
2065
+ if (!seen.has(issue.toString())) issues.push(issue);
2066
+ seen.add(issue.toString());
2067
+ }
2068
+ return issues;
2069
+ }
1522
2070
  discover(target, config) {
1523
2071
  if (!existsSync(target)) throw new EvalError({
1524
2072
  category: ErrorCategory.Specification,
@@ -1543,20 +2091,22 @@ var ScenarioLoader = class {
1543
2091
  }
1544
2092
  return files;
1545
2093
  }
1546
- build(value, document, secrets) {
2094
+ build(value, document, runName, websocket, secrets) {
1547
2095
  const scenario = value;
1548
2096
  const turns = scenario.turns;
1549
2097
  const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
1550
2098
  return new ConversationSpec({
1551
2099
  version: scenario.version,
1552
2100
  name: scenario.name,
2101
+ runName,
1553
2102
  description: typeof scenario.description === "string" ? scenario.description : "",
1554
2103
  turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
1555
2104
  conversation: conversation ? new ConversationExpectation({
1556
- criteria: conversation.criteria,
2105
+ criteria: conversation.should,
1557
2106
  location: document.locate(["expect", "conversation"])
1558
2107
  }) : void 0,
1559
2108
  location: document.locate([]),
2109
+ websocket,
1560
2110
  secrets
1561
2111
  });
1562
2112
  }
@@ -1582,7 +2132,7 @@ var ScenarioLoader = class {
1582
2132
  });
1583
2133
  const fields = response;
1584
2134
  return new ResponseExpectation({
1585
- criteria: fields.criteria,
2135
+ criteria: fields.should,
1586
2136
  contains: fields.contains ?? [],
1587
2137
  notContains: fields.notContains ?? [],
1588
2138
  location
@@ -2405,7 +2955,11 @@ var ConversationRunner = class {
2405
2955
  async run(spec, signal = NEVER_ABORTED) {
2406
2956
  const startedAt = Date.now();
2407
2957
  const outcomes = [];
2408
- const assertions = new AssertionRunner(this.config.websocket.protocol.capabilities);
2958
+ const config = new EvalConfig({
2959
+ ...this.config,
2960
+ websocket: spec.websocket ?? this.config.websocket
2961
+ });
2962
+ const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
2409
2963
  const history = [];
2410
2964
  const evaluate = this.semanticEvaluator(signal);
2411
2965
  const inline = !this.config.execution.continueOnFailure;
@@ -2430,7 +2984,7 @@ var ConversationRunner = class {
2430
2984
  try {
2431
2985
  const steps = (async () => {
2432
2986
  for (let attempt = 0;; attempt++) {
2433
- connection = this.createConnection(this.config);
2987
+ connection = this.createConnection(config);
2434
2988
  connectionAttempts++;
2435
2989
  try {
2436
2990
  await connection.connect();
@@ -2924,7 +3478,7 @@ var ScenarioRunner = class {
2924
3478
  required: minPass,
2925
3479
  attempts,
2926
3480
  tests: this.results.tests(attempts, minPass),
2927
- agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
3481
+ agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
2928
3482
  evaluatorUsage: this.tokens.evaluator(attempts)
2929
3483
  });
2930
3484
  }
@@ -3012,10 +3566,12 @@ var ConsoleReporter = class ConsoleReporter {
3012
3566
  scenario(result, repeat, tokenUsageCapability) {
3013
3567
  const spec = result.spec;
3014
3568
  const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
3015
- const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
3569
+ const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
3016
3570
  if (!this.verbose) return line;
3017
3571
  if (result.attempts.length === 0) return `${line}\n`;
3018
- const lines = [""];
3572
+ const websocket = spec.websocket;
3573
+ const subprotocols = websocket?.connection.subprotocols.length ?? 0;
3574
+ const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
3019
3575
  for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
3020
3576
  lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
3021
3577
  lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
@@ -3023,6 +3579,9 @@ var ConsoleReporter = class ConsoleReporter {
3023
3579
  lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
3024
3580
  return line + lines.join("\n");
3025
3581
  }
3582
+ static title(spec) {
3583
+ return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
3584
+ }
3026
3585
  summary(run, exitCode, reportPaths = []) {
3027
3586
  const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
3028
3587
  const lines = this.verbose ? [] : [""];
@@ -3054,7 +3613,7 @@ var ConsoleReporter = class ConsoleReporter {
3054
3613
  }
3055
3614
  problems(result) {
3056
3615
  const spec = result.spec;
3057
- const lines = [`${this.status(result.status, 6)} ${spec.name} (${spec.location.file})`];
3616
+ const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
3058
3617
  for (const attempt of result.attempts) {
3059
3618
  const execution = attempt.execution;
3060
3619
  if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
@@ -3228,11 +3787,17 @@ var JsonReporter = class JsonReporter {
3228
3787
  };
3229
3788
  }
3230
3789
  scenario(result, config) {
3231
- const capability = config.websocket.protocol.capabilities.tokenUsage;
3790
+ const websocket = result.spec.websocket ?? config.websocket;
3791
+ const capability = websocket.protocol.capabilities.tokenUsage;
3232
3792
  return {
3233
3793
  name: result.spec.name,
3794
+ runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
3234
3795
  file: result.spec.location.file,
3235
3796
  status: result.status,
3797
+ connection: {
3798
+ endpoint: websocket.endpoint,
3799
+ subprotocols: websocket.connection.subprotocols.length
3800
+ },
3236
3801
  ...JsonReporter.counts(result),
3237
3802
  agentUsage: JsonReporter.usage(result.agentUsage),
3238
3803
  evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),