@mappies/miniwat-eval 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/main.js CHANGED
@@ -2,8 +2,8 @@
2
2
  import { Command, CommanderError } from "commander";
3
3
  import { dirname, extname, join, relative, resolve } from "node:path";
4
4
  import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
5
- import { Ajv } from "ajv";
6
5
  import { LineCounter, isMap, isPair, isScalar, isSeq, parseDocument } from "yaml";
6
+ import { Ajv } from "ajv";
7
7
  import { Message, ResponseMessage, TokenUsage } from "@mappies/miniwat-agent-core";
8
8
  import { Agent, InMemoryAgentStateStore } from "@mappies/miniwat-agent";
9
9
  import { randomUUID } from "node:crypto";
@@ -215,6 +215,14 @@ var eval_config_schema_default = {
215
215
  ],
216
216
  properties: {
217
217
  "version": { "const": 1 },
218
+ "variables": {
219
+ "type": "object",
220
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
221
+ "additionalProperties": { "anyOf": [{ "type": "string" }, {
222
+ "type": "array",
223
+ "items": { "type": "string" }
224
+ }] }
225
+ },
218
226
  "websocket": {
219
227
  "type": "object",
220
228
  "additionalProperties": false,
@@ -510,6 +518,7 @@ var EvalConfig = class {
510
518
  version;
511
519
  configPath;
512
520
  parametersPath;
521
+ variables;
513
522
  websocket;
514
523
  execution;
515
524
  evaluator;
@@ -519,6 +528,7 @@ var EvalConfig = class {
519
528
  this.version = data?.version ?? 1;
520
529
  this.configPath = data?.configPath ?? "";
521
530
  this.parametersPath = data?.parametersPath ?? "";
531
+ this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
522
532
  this.websocket = new WebSocketConfig(data?.websocket);
523
533
  this.execution = new ExecutionConfig(data?.execution);
524
534
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -531,18 +541,80 @@ const ReportFormat = {
531
541
  Console: "console",
532
542
  Json: "json"
533
543
  };
544
+ // src/yaml/yamlDocument.ts
545
+ var YamlDocument = class YamlDocument {
546
+ file;
547
+ document;
548
+ lineCounter;
549
+ value;
550
+ errors;
551
+ constructor(file, document, lineCounter) {
552
+ this.file = file;
553
+ this.document = document;
554
+ this.lineCounter = lineCounter;
555
+ this.value = document.toJS();
556
+ this.errors = document.errors.map((error) => new ValidationIssue({
557
+ message: error.message.split("\n")[0],
558
+ location: this.toLocation(error.pos[0], [])
559
+ }));
560
+ }
561
+ locate(path) {
562
+ return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
563
+ }
564
+ locateKey(path, key) {
565
+ const parent = this.findNode(path);
566
+ if (isMap(parent)) {
567
+ const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
568
+ if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
569
+ }
570
+ return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
571
+ }
572
+ static formatPath(path) {
573
+ return path.map((segment, index) => {
574
+ if (typeof segment === "number") return `[${segment}]`;
575
+ return index === 0 ? segment : `.${segment}`;
576
+ }).join("");
577
+ }
578
+ findNode(path) {
579
+ let node = this.document.contents ?? void 0;
580
+ for (const segment of path) {
581
+ let next;
582
+ if (isMap(node)) {
583
+ const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
584
+ next = pair?.value ?? pair?.key;
585
+ } else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
586
+ if (!next) break;
587
+ node = next;
588
+ }
589
+ return node;
590
+ }
591
+ toLocation(offset, path) {
592
+ const position = this.lineCounter.linePos(offset);
593
+ return new SourceLocation({
594
+ file: this.file,
595
+ line: position.line,
596
+ column: position.col,
597
+ path: YamlDocument.formatPath(path)
598
+ });
599
+ }
600
+ };
534
601
  // src/yaml/environmentResolver.ts
535
602
  const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
536
603
  const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
537
604
  var EnvironmentResolver = class EnvironmentResolver {
538
605
  environment;
539
606
  parameters;
540
- constructor(environment = process.env, parameters = {}) {
607
+ levels;
608
+ constructor(environment = process.env, parameters = {}, levels = []) {
541
609
  this.environment = environment;
542
610
  this.parameters = parameters;
611
+ this.levels = levels;
612
+ }
613
+ withLevels(levels) {
614
+ return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
543
615
  }
544
- resolve(value, document, issues, secrets = []) {
545
- return this.resolveValue(value, [], document, issues, secrets);
616
+ resolve(value, document, issues, secrets = [], options = {}) {
617
+ return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
546
618
  }
547
619
  static hasReference(value) {
548
620
  return new RegExp(REFERENCE.source).test(value);
@@ -550,22 +622,45 @@ var EnvironmentResolver = class EnvironmentResolver {
550
622
  static isSecretName(name) {
551
623
  return SECRET_NAME.test(name);
552
624
  }
553
- resolveValue(value, path, document, issues, secrets) {
625
+ find(name) {
626
+ for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
627
+ return this.environment[name];
628
+ }
629
+ sources() {
630
+ const sources = [
631
+ ...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
632
+ ...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
633
+ "environment variable"
634
+ ];
635
+ return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
636
+ }
637
+ resolveValue(value, path, document, issues, secrets, options) {
554
638
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
555
- const resolved = Object.hasOwn(this.parameters, name) ? this.parameters[name] : this.environment[name];
639
+ options.used?.add(name);
640
+ const resolved = this.find(name);
556
641
  if (resolved === void 0) {
557
- const source = Object.keys(this.parameters).length > 0 ? "parameter or environment variable" : "environment variable";
642
+ if (options.optional?.(path)) {
643
+ options.unresolved?.add(YamlDocument.formatPath(path));
644
+ return reference;
645
+ }
646
+ issues.push(new ValidationIssue({
647
+ message: `${this.sources()} '${name}' is not set`,
648
+ location: document.locate(path)
649
+ }));
650
+ return reference;
651
+ }
652
+ if (Array.isArray(resolved)) {
558
653
  issues.push(new ValidationIssue({
559
- message: `${source} '${name}' is not set`,
654
+ message: `variable '${name}' is a list, which can only be used as runWith.from`,
560
655
  location: document.locate(path)
561
656
  }));
562
657
  return reference;
563
658
  }
564
- if (EnvironmentResolver.isSecretName(name) && !secrets.includes(resolved)) secrets.push(resolved);
659
+ if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
565
660
  return resolved;
566
661
  });
567
- if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
568
- if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
662
+ if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
663
+ if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
569
664
  return value;
570
665
  }
571
666
  };
@@ -649,63 +744,6 @@ var SchemaValidator = class SchemaValidator {
649
744
  return path;
650
745
  }
651
746
  };
652
- // src/yaml/yamlDocument.ts
653
- var YamlDocument = class YamlDocument {
654
- file;
655
- document;
656
- lineCounter;
657
- value;
658
- errors;
659
- constructor(file, document, lineCounter) {
660
- this.file = file;
661
- this.document = document;
662
- this.lineCounter = lineCounter;
663
- this.value = document.toJS();
664
- this.errors = document.errors.map((error) => new ValidationIssue({
665
- message: error.message.split("\n")[0],
666
- location: this.toLocation(error.pos[0], [])
667
- }));
668
- }
669
- locate(path) {
670
- return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
671
- }
672
- locateKey(path, key) {
673
- const parent = this.findNode(path);
674
- if (isMap(parent)) {
675
- const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
676
- if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
677
- }
678
- return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
679
- }
680
- static formatPath(path) {
681
- return path.map((segment, index) => {
682
- if (typeof segment === "number") return `[${segment}]`;
683
- return index === 0 ? segment : `.${segment}`;
684
- }).join("");
685
- }
686
- findNode(path) {
687
- let node = this.document.contents ?? void 0;
688
- for (const segment of path) {
689
- let next;
690
- if (isMap(node)) {
691
- const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
692
- next = pair?.value ?? pair?.key;
693
- } else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
694
- if (!next) break;
695
- node = next;
696
- }
697
- return node;
698
- }
699
- toLocation(offset, path) {
700
- const position = this.lineCounter.linePos(offset);
701
- return new SourceLocation({
702
- file: this.file,
703
- line: position.line,
704
- column: position.col,
705
- path: YamlDocument.formatPath(path)
706
- });
707
- }
708
- };
709
747
  // src/yaml/yamlDocumentLoader.ts
710
748
  var YamlDocumentLoader = class {
711
749
  load(filePath, displayPath) {
@@ -720,6 +758,143 @@ var YamlDocumentLoader = class {
720
758
  }), lineCounter);
721
759
  }
722
760
  };
761
+ // src/yaml/variablesReader.ts
762
+ const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
763
+ var VariablesReader = class {
764
+ read(document, resolver, issues, secrets) {
765
+ const raw = isRecord(document.value) ? document.value.variables : void 0;
766
+ if (raw === void 0) return {};
767
+ if (!isRecord(raw)) {
768
+ issues.push(new ValidationIssue({
769
+ message: "must be object",
770
+ location: document.locate(["variables"])
771
+ }));
772
+ return {};
773
+ }
774
+ const variables = {};
775
+ for (const [name, value] of Object.entries(raw)) {
776
+ if (!VARIABLE_NAME.test(name)) {
777
+ issues.push(new ValidationIssue({
778
+ message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
779
+ location: document.locateKey(["variables"], name)
780
+ }));
781
+ continue;
782
+ }
783
+ if (Array.isArray(value)) {
784
+ const bad = value.findIndex((item) => typeof item !== "string");
785
+ if (bad >= 0) {
786
+ issues.push(new ValidationIssue({
787
+ message: "must be string",
788
+ location: document.locate([
789
+ "variables",
790
+ name,
791
+ bad
792
+ ])
793
+ }));
794
+ continue;
795
+ }
796
+ variables[name] = [...value];
797
+ continue;
798
+ }
799
+ if (typeof value !== "string") {
800
+ issues.push(new ValidationIssue({
801
+ message: "must be string or a list of strings",
802
+ location: document.locate(["variables", name])
803
+ }));
804
+ continue;
805
+ }
806
+ const before = issues.length;
807
+ const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
808
+ if (issues.length > before) continue;
809
+ const text = resolved.variables[name];
810
+ variables[name] = text;
811
+ if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
812
+ }
813
+ return variables;
814
+ }
815
+ };
816
+ // src/config/webSocketResolver.ts
817
+ const WEBSOCKET_SCHEMA = {
818
+ ...eval_config_schema_default,
819
+ required: ["websocket"],
820
+ additionalProperties: true,
821
+ properties: { websocket: eval_config_schema_default.properties.websocket }
822
+ };
823
+ var WebSocketResolver = class WebSocketResolver {
824
+ schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
825
+ resolve(document, resolver, issues, secrets, used, defer) {
826
+ const raw = isRecord(document.value) ? document.value.websocket : void 0;
827
+ const unresolved = /* @__PURE__ */ new Set();
828
+ const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
829
+ const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
830
+ used,
831
+ optional,
832
+ unresolved
833
+ });
834
+ const subprotocols = WebSocketResolver.subprotocols(raw);
835
+ if (subprotocols !== void 0) {
836
+ const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
837
+ const set = !unresolved.has(YamlDocument.formatPath([
838
+ "websocket",
839
+ "connection",
840
+ "subprotocols",
841
+ index
842
+ ]));
843
+ const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
844
+ return set && !empty;
845
+ });
846
+ value.websocket.connection.subprotocols = kept;
847
+ }
848
+ const prefix = YamlDocument.formatPath([
849
+ "websocket",
850
+ "connection",
851
+ "subprotocols"
852
+ ]);
853
+ return {
854
+ value,
855
+ deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
856
+ };
857
+ }
858
+ validate(value, document) {
859
+ const issues = this.schemaValidator.validate(value, document);
860
+ if (issues.length > 0) return issues;
861
+ return WebSocketResolver.validateInputPath(value, document);
862
+ }
863
+ static validateInputPath(value, document) {
864
+ const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
865
+ if (typeof request.inputPath !== "string") return [];
866
+ const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
867
+ const last = tokens.pop();
868
+ let parent = request.value;
869
+ for (const token of tokens) parent = WebSocketResolver.child(parent, token);
870
+ if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
871
+ return [new ValidationIssue({
872
+ message: "must point to a missing or string value whose parent exists in request.value",
873
+ location: document.locate([
874
+ "websocket",
875
+ "protocol",
876
+ "request",
877
+ "inputPath"
878
+ ])
879
+ })];
880
+ }
881
+ static section(value, ...path) {
882
+ let current = value;
883
+ for (const key of path) current = WebSocketResolver.child(current, key);
884
+ return isRecord(current) ? current : {};
885
+ }
886
+ static child(value, key) {
887
+ if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
888
+ return isRecord(value) ? value[key] : void 0;
889
+ }
890
+ static isSubprotocol(path) {
891
+ return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
892
+ }
893
+ static subprotocols(websocket) {
894
+ const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
895
+ return Array.isArray(subprotocols) ? subprotocols : void 0;
896
+ }
897
+ };
723
898
  // src/config/defaultConfig.ts
724
899
  const DEFAULT_CONFIG = new EvalConfig({
725
900
  execution: new ExecutionConfig({
@@ -755,6 +930,8 @@ var EvalConfigLoader = class EvalConfigLoader {
755
930
  cwd;
756
931
  documentLoader = new YamlDocumentLoader();
757
932
  schemaValidator = new SchemaValidator(eval_config_schema_default);
933
+ variablesReader = new VariablesReader();
934
+ webSocketResolver = new WebSocketResolver();
758
935
  environmentResolver;
759
936
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
760
937
  this.cwd = cwd;
@@ -765,19 +942,47 @@ var EvalConfigLoader = class EvalConfigLoader {
765
942
  const issues = [...document.errors];
766
943
  if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
767
944
  const named = [];
768
- const value = issues.length === 0 ? this.environmentResolver.resolve(document.value, document, issues, named) : void 0;
769
- if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
770
- if (issues.length === 0) issues.push(...this.validateInputPath(value, document));
945
+ const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
946
+ let value = {};
947
+ if (issues.length === 0) {
948
+ const resolver = this.environmentResolver.withLevels([variables]);
949
+ const { websocket: _websocket, variables: _variables, ...rest } = document.value;
950
+ const resolved = resolver.resolve(rest, document, issues, named);
951
+ const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
952
+ value = {
953
+ ...resolved,
954
+ ..._variables === void 0 ? {} : { variables },
955
+ ..._websocket === void 0 ? {} : websocket.value
956
+ };
957
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
958
+ if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
959
+ }
771
960
  if (issues.length > 0) throw new EvalError({
772
961
  category: ErrorCategory.Configuration,
773
962
  message: "invalid configuration",
774
963
  issues
775
964
  });
776
- const config = this.build(value, configPath, named);
965
+ const config = this.build(value, configPath, variables, named);
777
966
  this.applyRepetition(config, override, value, document);
778
967
  EvalConfigLoader.applyReport(config, report);
779
968
  return config;
780
969
  }
970
+ static buildWebSocket(value) {
971
+ const section = (...path) => WebSocketResolver.section(value, ...path);
972
+ const connection = section("websocket", "connection");
973
+ const request = section("websocket", "protocol", "request");
974
+ return new WebSocketConfig({
975
+ endpoint: WebSocketResolver.section(value, "websocket").endpoint,
976
+ connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
977
+ protocol: new ProtocolConfig({
978
+ request: new RequestConfig({
979
+ ...request,
980
+ format: request.format ?? RequestFormat.Json
981
+ }),
982
+ capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
983
+ })
984
+ });
985
+ }
781
986
  static applyReport(config, report) {
782
987
  if (report.formats === void 0) return;
783
988
  const known = Object.values(ReportFormat);
@@ -799,45 +1004,17 @@ var EvalConfigLoader = class EvalConfigLoader {
799
1004
  location: document.locate(["evaluator", "apiKey"])
800
1005
  })];
801
1006
  }
802
- validateInputPath(value, document) {
803
- const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
804
- if (typeof request.inputPath !== "string") return [];
805
- const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
806
- const last = tokens.pop();
807
- let parent = request.value;
808
- for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
809
- if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
810
- return [new ValidationIssue({
811
- message: "must point to a missing or string value whose parent exists in request.value",
812
- location: document.locate([
813
- "websocket",
814
- "protocol",
815
- "request",
816
- "inputPath"
817
- ])
818
- })];
819
- }
820
- build(value, configPath, named) {
1007
+ build(value, configPath, variables, named) {
821
1008
  const section = (...path) => EvalConfigLoader.section(value, ...path);
822
1009
  const configDirectory = dirname(configPath);
823
- const connection = section("websocket", "connection");
824
- const request = section("websocket", "protocol", "request");
1010
+ const websocket = EvalConfigLoader.buildWebSocket(value);
825
1011
  const evaluator = section("evaluator");
826
1012
  const report = section("report");
827
1013
  return new EvalConfig({
828
1014
  version: value.version,
829
1015
  configPath,
830
- websocket: new WebSocketConfig({
831
- endpoint: EvalConfigLoader.section(value, "websocket").endpoint,
832
- connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
833
- protocol: new ProtocolConfig({
834
- request: new RequestConfig({
835
- ...request,
836
- format: request.format ?? RequestFormat.Json
837
- }),
838
- capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
839
- })
840
- }),
1016
+ variables,
1017
+ websocket,
841
1018
  execution: new ExecutionConfig({
842
1019
  ...DEFAULT_CONFIG.execution,
843
1020
  ...section("execution"),
@@ -858,7 +1035,7 @@ var EvalConfigLoader = class EvalConfigLoader {
858
1035
  directory: resolve(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
859
1036
  formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
860
1037
  }),
861
- secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols ?? [], named)
1038
+ secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
862
1039
  });
863
1040
  }
864
1041
  applyRepetition(config, override, value, document) {
@@ -911,13 +1088,7 @@ var EvalConfigLoader = class EvalConfigLoader {
911
1088
  ])];
912
1089
  }
913
1090
  static section(value, ...path) {
914
- let current = value;
915
- for (const key of path) current = EvalConfigLoader.child(current, key);
916
- return isRecord(current) ? current : {};
917
- }
918
- static child(value, key) {
919
- if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
920
- return isRecord(value) ? value[key] : void 0;
1091
+ return WebSocketResolver.section(value, ...path);
921
1092
  }
922
1093
  };
923
1094
  // src/interfaces/runStatus.ts
@@ -1109,18 +1280,22 @@ var TurnSpec = class {
1109
1280
  var ConversationSpec = class {
1110
1281
  version;
1111
1282
  name;
1283
+ runName;
1112
1284
  description;
1113
1285
  turns;
1114
1286
  conversation;
1115
1287
  location;
1288
+ websocket;
1116
1289
  secrets;
1117
1290
  constructor(data) {
1118
1291
  this.version = data?.version ?? 1;
1119
1292
  this.name = data?.name ?? "";
1293
+ this.runName = data?.runName ?? "";
1120
1294
  this.description = data?.description ?? "";
1121
1295
  this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
1122
1296
  this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
1123
1297
  this.location = new SourceLocation(data?.location);
1298
+ this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
1124
1299
  this.secrets = [...data?.secrets ?? []];
1125
1300
  }
1126
1301
  };
@@ -2420,7 +2595,11 @@ var ConversationRunner = class {
2420
2595
  async run(spec, signal = NEVER_ABORTED) {
2421
2596
  const startedAt = Date.now();
2422
2597
  const outcomes = [];
2423
- const assertions = new AssertionRunner(this.config.websocket.protocol.capabilities);
2598
+ const config = new EvalConfig({
2599
+ ...this.config,
2600
+ websocket: spec.websocket ?? this.config.websocket
2601
+ });
2602
+ const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
2424
2603
  const history = [];
2425
2604
  const evaluate = this.semanticEvaluator(signal);
2426
2605
  const inline = !this.config.execution.continueOnFailure;
@@ -2445,7 +2624,7 @@ var ConversationRunner = class {
2445
2624
  try {
2446
2625
  const steps = (async () => {
2447
2626
  for (let attempt = 0;; attempt++) {
2448
- connection = this.createConnection(this.config);
2627
+ connection = this.createConnection(config);
2449
2628
  connectionAttempts++;
2450
2629
  try {
2451
2630
  await connection.connect();
@@ -2626,7 +2805,7 @@ var ScenarioRunner = class {
2626
2805
  required: minPass,
2627
2806
  attempts,
2628
2807
  tests: this.results.tests(attempts, minPass),
2629
- agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
2808
+ agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
2630
2809
  evaluatorUsage: this.tokens.evaluator(attempts)
2631
2810
  });
2632
2811
  }
@@ -2785,10 +2964,12 @@ var ConsoleReporter = class ConsoleReporter {
2785
2964
  scenario(result, repeat, tokenUsageCapability) {
2786
2965
  const spec = result.spec;
2787
2966
  const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
2788
- const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
2967
+ const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
2789
2968
  if (!this.verbose) return line;
2790
2969
  if (result.attempts.length === 0) return `${line}\n`;
2791
- const lines = [""];
2970
+ const websocket = spec.websocket;
2971
+ const subprotocols = websocket?.connection.subprotocols.length ?? 0;
2972
+ const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
2792
2973
  for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
2793
2974
  lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
2794
2975
  lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
@@ -2796,6 +2977,9 @@ var ConsoleReporter = class ConsoleReporter {
2796
2977
  lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
2797
2978
  return line + lines.join("\n");
2798
2979
  }
2980
+ static title(spec) {
2981
+ return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
2982
+ }
2799
2983
  summary(run, exitCode, reportPaths = []) {
2800
2984
  const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
2801
2985
  const lines = this.verbose ? [] : [""];
@@ -2827,7 +3011,7 @@ var ConsoleReporter = class ConsoleReporter {
2827
3011
  }
2828
3012
  problems(result) {
2829
3013
  const spec = result.spec;
2830
- const lines = [`${this.status(result.status, 6)} ${spec.name} (${spec.location.file})`];
3014
+ const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
2831
3015
  for (const attempt of result.attempts) {
2832
3016
  const execution = attempt.execution;
2833
3017
  if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
@@ -3000,11 +3184,17 @@ var JsonReporter = class JsonReporter {
3000
3184
  };
3001
3185
  }
3002
3186
  scenario(result, config) {
3003
- const capability = config.websocket.protocol.capabilities.tokenUsage;
3187
+ const websocket = result.spec.websocket ?? config.websocket;
3188
+ const capability = websocket.protocol.capabilities.tokenUsage;
3004
3189
  return {
3005
3190
  name: result.spec.name,
3191
+ runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
3006
3192
  file: result.spec.location.file,
3007
3193
  status: result.status,
3194
+ connection: {
3195
+ endpoint: websocket.endpoint,
3196
+ subprotocols: websocket.connection.subprotocols.length
3197
+ },
3008
3198
  ...JsonReporter.counts(result),
3009
3199
  agentUsage: JsonReporter.usage(result.agentUsage),
3010
3200
  evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
@@ -3165,6 +3355,48 @@ var conversation_schema_default = {
3165
3355
  "version": { "const": 1 },
3166
3356
  "name": { "$ref": "#/definitions/text" },
3167
3357
  "description": { "type": "string" },
3358
+ "variables": {
3359
+ "type": "object",
3360
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
3361
+ "additionalProperties": { "anyOf": [{ "type": "string" }, {
3362
+ "type": "array",
3363
+ "items": { "type": "string" }
3364
+ }] }
3365
+ },
3366
+ "runWith": {
3367
+ "description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
3368
+ "anyOf": [{
3369
+ "type": "array",
3370
+ "minItems": 1,
3371
+ "items": {
3372
+ "type": "object",
3373
+ "required": ["name"],
3374
+ "properties": { "name": {
3375
+ "type": "string",
3376
+ "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
3377
+ } },
3378
+ "additionalProperties": { "type": "string" }
3379
+ }
3380
+ }, {
3381
+ "type": "object",
3382
+ "additionalProperties": false,
3383
+ "required": ["variable", "from"],
3384
+ "properties": {
3385
+ "variable": {
3386
+ "type": "string",
3387
+ "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
3388
+ },
3389
+ "from": { "anyOf": [{
3390
+ "type": "array",
3391
+ "minItems": 1,
3392
+ "items": { "type": "string" }
3393
+ }, {
3394
+ "type": "string",
3395
+ "pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
3396
+ }] }
3397
+ }
3398
+ }]
3399
+ },
3168
3400
  "turns": {
3169
3401
  "type": "array",
3170
3402
  "minItems": 1,
@@ -3177,8 +3409,8 @@ var conversation_schema_default = {
3177
3409
  "properties": { "conversation": {
3178
3410
  "type": "object",
3179
3411
  "additionalProperties": false,
3180
- "required": ["criteria"],
3181
- "properties": { "criteria": { "$ref": "#/definitions/text" } }
3412
+ "required": ["should"],
3413
+ "properties": { "should": { "$ref": "#/definitions/text" } }
3182
3414
  } }
3183
3415
  }
3184
3416
  },
@@ -3221,12 +3453,12 @@ var conversation_schema_default = {
3221
3453
  }
3222
3454
  },
3223
3455
  "response": {
3224
- "description": "A string is shorthand for criteria.",
3456
+ "description": "A string is shorthand for should.",
3225
3457
  "type": ["string", "object"],
3226
3458
  "pattern": "\\S",
3227
3459
  "additionalProperties": false,
3228
3460
  "properties": {
3229
- "criteria": { "$ref": "#/definitions/text" },
3461
+ "should": { "$ref": "#/definitions/text" },
3230
3462
  "contains": { "$ref": "#/definitions/textList" },
3231
3463
  "notContains": { "$ref": "#/definitions/textList" }
3232
3464
  }
@@ -3258,7 +3490,7 @@ var conversation_schema_default = {
3258
3490
  };
3259
3491
  // src/scenarios/conversationValidator.ts
3260
3492
  const RESPONSE_KEYS = [
3261
- "criteria",
3493
+ "should",
3262
3494
  "contains",
3263
3495
  "notContains"
3264
3496
  ];
@@ -3281,7 +3513,7 @@ var ConversationValidator = class ConversationValidator {
3281
3513
  validateResponse(response, path, document, issues) {
3282
3514
  if (!RESPONSE_KEYS.some((key) => key in response)) {
3283
3515
  issues.push(new ValidationIssue({
3284
- message: "response must have at least one of: criteria, contains, notContains",
3516
+ message: "response must have at least one of: should, contains, notContains",
3285
3517
  location: document.locate(path)
3286
3518
  }));
3287
3519
  return;
@@ -3322,13 +3554,231 @@ var ConversationValidator = class ConversationValidator {
3322
3554
  return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
3323
3555
  }
3324
3556
  };
3557
+ // src/scenarios/runWithReader.ts
3558
+ const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
3559
+ const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
3560
+ const FORMS = "must be a list of rows, or an object with variable and from";
3561
+ const SINGLE_RUN = {
3562
+ name: "",
3563
+ variables: {},
3564
+ keys: [],
3565
+ location: [],
3566
+ secrets: []
3567
+ };
3568
+ var RunWithReader = class RunWithReader {
3569
+ read(document, resolver, issues, used) {
3570
+ const raw = isRecord(document.value) ? document.value.runWith : void 0;
3571
+ if (raw === void 0) return void 0;
3572
+ if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
3573
+ if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
3574
+ issues.push(new ValidationIssue({
3575
+ message: FORMS,
3576
+ location: document.locate(["runWith"])
3577
+ }));
3578
+ return [];
3579
+ }
3580
+ rows(rows, document, resolver, issues, used) {
3581
+ if (rows.length === 0) {
3582
+ issues.push(new ValidationIssue({
3583
+ message: "must not be empty",
3584
+ location: document.locate(["runWith"])
3585
+ }));
3586
+ return [];
3587
+ }
3588
+ const names = /* @__PURE__ */ new Set();
3589
+ const runs = [];
3590
+ rows.forEach((row, index) => {
3591
+ const path = ["runWith", index];
3592
+ if (!isRecord(row)) {
3593
+ issues.push(new ValidationIssue({
3594
+ message: "must be object",
3595
+ location: document.locate(path)
3596
+ }));
3597
+ return;
3598
+ }
3599
+ const name = row.name;
3600
+ if (name === void 0) issues.push(new ValidationIssue({
3601
+ message: "missing required key 'name'",
3602
+ location: document.locate(path)
3603
+ }));
3604
+ else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
3605
+ message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
3606
+ location: document.locate([...path, "name"])
3607
+ }));
3608
+ else if (names.has(name)) issues.push(new ValidationIssue({
3609
+ message: `duplicate run name '${name}'`,
3610
+ location: document.locate([...path, "name"])
3611
+ }));
3612
+ else names.add(name);
3613
+ const run = {
3614
+ name: typeof name === "string" ? name : "",
3615
+ variables: {},
3616
+ keys: [],
3617
+ location: path,
3618
+ secrets: []
3619
+ };
3620
+ for (const [key, value] of Object.entries(row)) {
3621
+ if (key === "name") continue;
3622
+ if (!VARIABLE_NAME.test(key)) {
3623
+ issues.push(new ValidationIssue({
3624
+ message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
3625
+ location: document.locateKey(path, key)
3626
+ }));
3627
+ continue;
3628
+ }
3629
+ if (typeof value !== "string") {
3630
+ issues.push(new ValidationIssue({
3631
+ message: "must be string",
3632
+ location: document.locate([...path, key])
3633
+ }));
3634
+ continue;
3635
+ }
3636
+ const resolved = resolver.resolve(value, document, issues, run.secrets, {
3637
+ used,
3638
+ at: [...path, key]
3639
+ });
3640
+ RunWithReader.set(run, key, resolved, document.locateKey(path, key));
3641
+ }
3642
+ runs.push(run);
3643
+ });
3644
+ return runs;
3645
+ }
3646
+ list(form, document, resolver, issues, used) {
3647
+ const before = issues.length;
3648
+ for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
3649
+ message: `unknown key '${key}'`,
3650
+ location: document.locateKey(["runWith"], key)
3651
+ }));
3652
+ const variable = form.variable;
3653
+ if (variable === void 0) issues.push(new ValidationIssue({
3654
+ message: "missing required key 'variable'",
3655
+ location: document.locate(["runWith"])
3656
+ }));
3657
+ else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
3658
+ message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
3659
+ location: document.locate(["runWith", "variable"])
3660
+ }));
3661
+ const entries = this.entries(form.from, document, resolver, issues, used);
3662
+ if (issues.length > before || entries === void 0) return [];
3663
+ const name = variable;
3664
+ const keyLocation = document.locate(["runWith", "variable"]);
3665
+ const names = /* @__PURE__ */ new Set();
3666
+ const runs = [];
3667
+ for (const { entry, path } of entries) {
3668
+ if (!VARIABLE_NAME.test(entry)) {
3669
+ issues.push(new ValidationIssue({
3670
+ message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
3671
+ location: document.locate(path)
3672
+ }));
3673
+ continue;
3674
+ }
3675
+ if (names.has(entry)) {
3676
+ issues.push(new ValidationIssue({
3677
+ message: `duplicate run name '${entry}'`,
3678
+ location: document.locate(path)
3679
+ }));
3680
+ continue;
3681
+ }
3682
+ names.add(entry);
3683
+ const run = {
3684
+ name: entry,
3685
+ variables: {},
3686
+ keys: [],
3687
+ location: path,
3688
+ secrets: []
3689
+ };
3690
+ const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
3691
+ used,
3692
+ at: path
3693
+ });
3694
+ RunWithReader.set(run, name, value, keyLocation);
3695
+ runs.push(run);
3696
+ }
3697
+ return runs;
3698
+ }
3699
+ entries(from, document, resolver, issues, used) {
3700
+ const path = ["runWith", "from"];
3701
+ const location = document.locate(path);
3702
+ if (from === void 0) {
3703
+ issues.push(new ValidationIssue({
3704
+ message: "missing required key 'from'",
3705
+ location: document.locate(["runWith"])
3706
+ }));
3707
+ return;
3708
+ }
3709
+ let list;
3710
+ let at;
3711
+ if (Array.isArray(from)) {
3712
+ list = from;
3713
+ at = (index) => [...path, index];
3714
+ } else {
3715
+ const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
3716
+ if (reference === null) {
3717
+ issues.push(new ValidationIssue({
3718
+ message: "must be a list, or a list variable such as ${tokens}",
3719
+ location
3720
+ }));
3721
+ return;
3722
+ }
3723
+ const name = reference[1];
3724
+ const found = resolver.find(name);
3725
+ used.add(name);
3726
+ if (found === void 0) {
3727
+ issues.push(new ValidationIssue({
3728
+ message: `list variable '${name}' is not set`,
3729
+ location
3730
+ }));
3731
+ return;
3732
+ }
3733
+ if (!Array.isArray(found)) {
3734
+ issues.push(new ValidationIssue({
3735
+ message: `variable '${name}' is not a list`,
3736
+ location
3737
+ }));
3738
+ return;
3739
+ }
3740
+ list = found;
3741
+ at = () => path;
3742
+ }
3743
+ if (list.length === 0) {
3744
+ issues.push(new ValidationIssue({
3745
+ message: "must not be empty",
3746
+ location
3747
+ }));
3748
+ return;
3749
+ }
3750
+ const bad = list.findIndex((item) => typeof item !== "string");
3751
+ if (bad >= 0) {
3752
+ issues.push(new ValidationIssue({
3753
+ message: "must be string",
3754
+ location: document.locate(at(bad))
3755
+ }));
3756
+ return;
3757
+ }
3758
+ return list.map((entry, index) => ({
3759
+ entry,
3760
+ path: at(index)
3761
+ }));
3762
+ }
3763
+ static set(run, name, value, location) {
3764
+ run.variables[name] = value;
3765
+ run.keys.push({
3766
+ name,
3767
+ location
3768
+ });
3769
+ if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
3770
+ }
3771
+ };
3325
3772
  // src/scenarios/scenarioLoader.ts
3326
3773
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
3327
- var ScenarioLoader = class {
3774
+ var ScenarioLoader = class ScenarioLoader {
3328
3775
  cwd;
3329
3776
  documentLoader = new YamlDocumentLoader();
3330
3777
  schemaValidator = new SchemaValidator(conversation_schema_default);
3331
3778
  conversationValidator = new ConversationValidator();
3779
+ variablesReader = new VariablesReader();
3780
+ runWithReader = new RunWithReader();
3781
+ webSocketResolver = new WebSocketResolver();
3332
3782
  environmentResolver;
3333
3783
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
3334
3784
  this.cwd = cwd;
@@ -3336,18 +3786,66 @@ var ScenarioLoader = class {
3336
3786
  }
3337
3787
  load(target, config) {
3338
3788
  const files = this.discover(resolve(this.cwd, target), config);
3789
+ const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.cwd, config.configPath)) : void 0;
3339
3790
  const issues = [];
3791
+ const connectionIssues = [];
3340
3792
  const specs = [];
3341
3793
  for (const file of files) {
3342
3794
  const document = this.documentLoader.load(file, relative(this.cwd, file));
3343
3795
  const fileIssues = [...document.errors];
3344
- const secrets = [];
3345
- const value = fileIssues.length === 0 ? this.environmentResolver.resolve(document.value, document, fileIssues, secrets) : void 0;
3346
- if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
3347
- if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
3348
- if (fileIssues.length === 0) specs.push(this.build(value, document, secrets));
3796
+ const fileSecrets = [];
3797
+ const used = /* @__PURE__ */ new Set();
3798
+ const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
3799
+ const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
3800
+ const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
3801
+ if (fileIssues.length > 0) {
3802
+ issues.push(...fileIssues);
3803
+ continue;
3804
+ }
3805
+ const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
3806
+ const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
3807
+ for (const result of results) for (const name of result.used) used.add(name);
3808
+ fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
3809
+ run: result.run,
3810
+ issues: result.issues
3811
+ })), (issue, run) => run === void 0 ? issue : new ValidationIssue({
3812
+ message: `run '${run.name}': ${issue.message}`,
3813
+ location: issue.location
3814
+ })));
3815
+ const attempted = results.filter((result) => result.connectionChecked);
3816
+ if (attempted.length > 0) connectionIssues.push({
3817
+ file: document,
3818
+ runs: attempted.map((result) => ({
3819
+ run: result.run,
3820
+ issues: result.websocketIssues
3821
+ }))
3822
+ });
3823
+ if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
3824
+ if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
3825
+ const websocket = result.websocket;
3826
+ specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
3827
+ }
3349
3828
  else issues.push(...fileIssues);
3350
3829
  }
3830
+ const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
3831
+ if (shared.length > 0) throw new EvalError({
3832
+ category: ErrorCategory.Configuration,
3833
+ message: "invalid configuration",
3834
+ issues: shared
3835
+ });
3836
+ for (const entry of connectionIssues) {
3837
+ const fileLocation = ScenarioLoader.variablesLocation(entry.file);
3838
+ issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
3839
+ message: issue.toString(),
3840
+ location: fileLocation
3841
+ }) : new ValidationIssue({
3842
+ message: `run '${run.name}': ${issue.toString()}`,
3843
+ location: new SourceLocation({
3844
+ ...entry.file.locate(run.location),
3845
+ path: ""
3846
+ })
3847
+ })));
3848
+ }
3351
3849
  if (issues.length > 0) throw new EvalError({
3352
3850
  category: ErrorCategory.Specification,
3353
3851
  message: "invalid scenario specification",
@@ -3355,6 +3853,71 @@ var ScenarioLoader = class {
3355
3853
  });
3356
3854
  return specs;
3357
3855
  }
3856
+ resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
3857
+ const issues = [];
3858
+ const used = /* @__PURE__ */ new Set();
3859
+ const resolver = scenarioResolver.withLevels([run.variables]);
3860
+ const value = {
3861
+ ...resolver.resolve(rest, document, issues, secrets, { used }),
3862
+ ...variables === void 0 ? {} : { variables }
3863
+ };
3864
+ const websocketIssues = [];
3865
+ let websocket;
3866
+ let connectionChecked = false;
3867
+ if (issues.length === 0 && configDocument !== void 0) {
3868
+ const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
3869
+ if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
3870
+ if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
3871
+ connectionChecked = true;
3872
+ }
3873
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
3874
+ if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
3875
+ return {
3876
+ run,
3877
+ value,
3878
+ used,
3879
+ secrets,
3880
+ issues,
3881
+ websocket,
3882
+ websocketIssues,
3883
+ connectionChecked
3884
+ };
3885
+ }
3886
+ static byRun(runs, place) {
3887
+ const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
3888
+ const sharedTexts = new Set(shared.map((issue) => issue.toString()));
3889
+ const result = shared.map((issue) => place(issue, void 0));
3890
+ for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
3891
+ return result;
3892
+ }
3893
+ static shared(lists) {
3894
+ if (lists.length === 0) return [];
3895
+ const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
3896
+ return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
3897
+ }
3898
+ static variablesLocation(document) {
3899
+ return new SourceLocation({
3900
+ ...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
3901
+ path: ""
3902
+ });
3903
+ }
3904
+ static unusedVariables(variables, used, results, document) {
3905
+ const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
3906
+ const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
3907
+ message: message(name),
3908
+ location: document.locateKey(["variables"], name)
3909
+ }));
3910
+ const seen = /* @__PURE__ */ new Set();
3911
+ for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
3912
+ const issue = new ValidationIssue({
3913
+ message: message(key.name),
3914
+ location: key.location
3915
+ });
3916
+ if (!seen.has(issue.toString())) issues.push(issue);
3917
+ seen.add(issue.toString());
3918
+ }
3919
+ return issues;
3920
+ }
3358
3921
  discover(target, config) {
3359
3922
  if (!existsSync(target)) throw new EvalError({
3360
3923
  category: ErrorCategory.Specification,
@@ -3379,20 +3942,22 @@ var ScenarioLoader = class {
3379
3942
  }
3380
3943
  return files;
3381
3944
  }
3382
- build(value, document, secrets) {
3945
+ build(value, document, runName, websocket, secrets) {
3383
3946
  const scenario = value;
3384
3947
  const turns = scenario.turns;
3385
3948
  const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
3386
3949
  return new ConversationSpec({
3387
3950
  version: scenario.version,
3388
3951
  name: scenario.name,
3952
+ runName,
3389
3953
  description: typeof scenario.description === "string" ? scenario.description : "",
3390
3954
  turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
3391
3955
  conversation: conversation ? new ConversationExpectation({
3392
- criteria: conversation.criteria,
3956
+ criteria: conversation.should,
3393
3957
  location: document.locate(["expect", "conversation"])
3394
3958
  }) : void 0,
3395
3959
  location: document.locate([]),
3960
+ websocket,
3396
3961
  secrets
3397
3962
  });
3398
3963
  }
@@ -3418,7 +3983,7 @@ var ScenarioLoader = class {
3418
3983
  });
3419
3984
  const fields = response;
3420
3985
  return new ResponseExpectation({
3421
- criteria: fields.criteria,
3986
+ criteria: fields.should,
3422
3987
  contains: fields.contains ?? [],
3423
3988
  notContains: fields.notContains ?? [],
3424
3989
  location