@mappies/miniwat-eval 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/main.js CHANGED
@@ -2,8 +2,8 @@
2
2
  import { Command, CommanderError } from "commander";
3
3
  import { dirname, extname, join, relative, resolve } from "node:path";
4
4
  import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
5
- import { Ajv } from "ajv";
6
5
  import { LineCounter, isMap, isPair, isScalar, isSeq, parseDocument } from "yaml";
6
+ import { Ajv } from "ajv";
7
7
  import { Message, ResponseMessage, TokenUsage } from "@mappies/miniwat-agent-core";
8
8
  import { Agent, InMemoryAgentStateStore } from "@mappies/miniwat-agent";
9
9
  import { randomUUID } from "node:crypto";
@@ -215,6 +215,11 @@ var eval_config_schema_default = {
215
215
  ],
216
216
  properties: {
217
217
  "version": { "const": 1 },
218
+ "variables": {
219
+ "type": "object",
220
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
221
+ "additionalProperties": { "type": "string" }
222
+ },
218
223
  "websocket": {
219
224
  "type": "object",
220
225
  "additionalProperties": false,
@@ -510,6 +515,7 @@ var EvalConfig = class {
510
515
  version;
511
516
  configPath;
512
517
  parametersPath;
518
+ variables;
513
519
  websocket;
514
520
  execution;
515
521
  evaluator;
@@ -519,6 +525,7 @@ var EvalConfig = class {
519
525
  this.version = data?.version ?? 1;
520
526
  this.configPath = data?.configPath ?? "";
521
527
  this.parametersPath = data?.parametersPath ?? "";
528
+ this.variables = { ...data?.variables ?? {} };
522
529
  this.websocket = new WebSocketConfig(data?.websocket);
523
530
  this.execution = new ExecutionConfig(data?.execution);
524
531
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -531,18 +538,80 @@ const ReportFormat = {
531
538
  Console: "console",
532
539
  Json: "json"
533
540
  };
541
+ // src/yaml/yamlDocument.ts
542
+ var YamlDocument = class YamlDocument {
543
+ file;
544
+ document;
545
+ lineCounter;
546
+ value;
547
+ errors;
548
+ constructor(file, document, lineCounter) {
549
+ this.file = file;
550
+ this.document = document;
551
+ this.lineCounter = lineCounter;
552
+ this.value = document.toJS();
553
+ this.errors = document.errors.map((error) => new ValidationIssue({
554
+ message: error.message.split("\n")[0],
555
+ location: this.toLocation(error.pos[0], [])
556
+ }));
557
+ }
558
+ locate(path) {
559
+ return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
560
+ }
561
+ locateKey(path, key) {
562
+ const parent = this.findNode(path);
563
+ if (isMap(parent)) {
564
+ const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
565
+ if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
566
+ }
567
+ return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
568
+ }
569
+ static formatPath(path) {
570
+ return path.map((segment, index) => {
571
+ if (typeof segment === "number") return `[${segment}]`;
572
+ return index === 0 ? segment : `.${segment}`;
573
+ }).join("");
574
+ }
575
+ findNode(path) {
576
+ let node = this.document.contents ?? void 0;
577
+ for (const segment of path) {
578
+ let next;
579
+ if (isMap(node)) {
580
+ const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
581
+ next = pair?.value ?? pair?.key;
582
+ } else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
583
+ if (!next) break;
584
+ node = next;
585
+ }
586
+ return node;
587
+ }
588
+ toLocation(offset, path) {
589
+ const position = this.lineCounter.linePos(offset);
590
+ return new SourceLocation({
591
+ file: this.file,
592
+ line: position.line,
593
+ column: position.col,
594
+ path: YamlDocument.formatPath(path)
595
+ });
596
+ }
597
+ };
534
598
  // src/yaml/environmentResolver.ts
535
599
  const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
536
600
  const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
537
601
  var EnvironmentResolver = class EnvironmentResolver {
538
602
  environment;
539
603
  parameters;
540
- constructor(environment = process.env, parameters = {}) {
604
+ levels;
605
+ constructor(environment = process.env, parameters = {}, levels = []) {
541
606
  this.environment = environment;
542
607
  this.parameters = parameters;
608
+ this.levels = levels;
609
+ }
610
+ withLevels(levels) {
611
+ return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
543
612
  }
544
- resolve(value, document, issues, secrets = []) {
545
- return this.resolveValue(value, [], document, issues, secrets);
613
+ resolve(value, document, issues, secrets = [], options = {}) {
614
+ return this.resolveValue(value, [], document, issues, secrets, options);
546
615
  }
547
616
  static hasReference(value) {
548
617
  return new RegExp(REFERENCE.source).test(value);
@@ -550,22 +619,38 @@ var EnvironmentResolver = class EnvironmentResolver {
550
619
  static isSecretName(name) {
551
620
  return SECRET_NAME.test(name);
552
621
  }
553
- resolveValue(value, path, document, issues, secrets) {
622
+ lookup(name) {
623
+ for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
624
+ return this.environment[name];
625
+ }
626
+ sources() {
627
+ const sources = [
628
+ ...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
629
+ ...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
630
+ "environment variable"
631
+ ];
632
+ return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
633
+ }
634
+ resolveValue(value, path, document, issues, secrets, options) {
554
635
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
555
- const resolved = Object.hasOwn(this.parameters, name) ? this.parameters[name] : this.environment[name];
636
+ options.used?.add(name);
637
+ const resolved = this.lookup(name);
556
638
  if (resolved === void 0) {
557
- const source = Object.keys(this.parameters).length > 0 ? "parameter or environment variable" : "environment variable";
639
+ if (options.optional?.(path)) {
640
+ options.unresolved?.add(YamlDocument.formatPath(path));
641
+ return reference;
642
+ }
558
643
  issues.push(new ValidationIssue({
559
- message: `${source} '${name}' is not set`,
644
+ message: `${this.sources()} '${name}' is not set`,
560
645
  location: document.locate(path)
561
646
  }));
562
647
  return reference;
563
648
  }
564
- if (EnvironmentResolver.isSecretName(name) && !secrets.includes(resolved)) secrets.push(resolved);
649
+ if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
565
650
  return resolved;
566
651
  });
567
- if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
568
- if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
652
+ if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
653
+ if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
569
654
  return value;
570
655
  }
571
656
  };
@@ -649,63 +734,6 @@ var SchemaValidator = class SchemaValidator {
649
734
  return path;
650
735
  }
651
736
  };
652
- // src/yaml/yamlDocument.ts
653
- var YamlDocument = class YamlDocument {
654
- file;
655
- document;
656
- lineCounter;
657
- value;
658
- errors;
659
- constructor(file, document, lineCounter) {
660
- this.file = file;
661
- this.document = document;
662
- this.lineCounter = lineCounter;
663
- this.value = document.toJS();
664
- this.errors = document.errors.map((error) => new ValidationIssue({
665
- message: error.message.split("\n")[0],
666
- location: this.toLocation(error.pos[0], [])
667
- }));
668
- }
669
- locate(path) {
670
- return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
671
- }
672
- locateKey(path, key) {
673
- const parent = this.findNode(path);
674
- if (isMap(parent)) {
675
- const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
676
- if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
677
- }
678
- return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
679
- }
680
- static formatPath(path) {
681
- return path.map((segment, index) => {
682
- if (typeof segment === "number") return `[${segment}]`;
683
- return index === 0 ? segment : `.${segment}`;
684
- }).join("");
685
- }
686
- findNode(path) {
687
- let node = this.document.contents ?? void 0;
688
- for (const segment of path) {
689
- let next;
690
- if (isMap(node)) {
691
- const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
692
- next = pair?.value ?? pair?.key;
693
- } else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
694
- if (!next) break;
695
- node = next;
696
- }
697
- return node;
698
- }
699
- toLocation(offset, path) {
700
- const position = this.lineCounter.linePos(offset);
701
- return new SourceLocation({
702
- file: this.file,
703
- line: position.line,
704
- column: position.col,
705
- path: YamlDocument.formatPath(path)
706
- });
707
- }
708
- };
709
737
  // src/yaml/yamlDocumentLoader.ts
710
738
  var YamlDocumentLoader = class {
711
739
  load(filePath, displayPath) {
@@ -720,6 +748,126 @@ var YamlDocumentLoader = class {
720
748
  }), lineCounter);
721
749
  }
722
750
  };
751
+ // src/yaml/variablesReader.ts
752
+ const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
753
+ var VariablesReader = class {
754
+ read(document, resolver, issues, secrets) {
755
+ const raw = isRecord(document.value) ? document.value.variables : void 0;
756
+ if (raw === void 0) return {};
757
+ if (!isRecord(raw)) {
758
+ issues.push(new ValidationIssue({
759
+ message: "must be object",
760
+ location: document.locate(["variables"])
761
+ }));
762
+ return {};
763
+ }
764
+ const variables = {};
765
+ for (const [name, value] of Object.entries(raw)) {
766
+ if (!VARIABLE_NAME.test(name)) {
767
+ issues.push(new ValidationIssue({
768
+ message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
769
+ location: document.locateKey(["variables"], name)
770
+ }));
771
+ continue;
772
+ }
773
+ if (typeof value !== "string") {
774
+ issues.push(new ValidationIssue({
775
+ message: "must be string",
776
+ location: document.locate(["variables", name])
777
+ }));
778
+ continue;
779
+ }
780
+ const before = issues.length;
781
+ const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
782
+ if (issues.length > before) continue;
783
+ variables[name] = resolved.variables[name];
784
+ if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
785
+ }
786
+ return variables;
787
+ }
788
+ };
789
+ // src/config/webSocketResolver.ts
790
+ const WEBSOCKET_SCHEMA = {
791
+ ...eval_config_schema_default,
792
+ required: ["websocket"],
793
+ additionalProperties: true,
794
+ properties: { websocket: eval_config_schema_default.properties.websocket }
795
+ };
796
+ var WebSocketResolver = class WebSocketResolver {
797
+ schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
798
+ resolve(document, resolver, issues, secrets, used, defer) {
799
+ const raw = isRecord(document.value) ? document.value.websocket : void 0;
800
+ const unresolved = /* @__PURE__ */ new Set();
801
+ const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
802
+ const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
803
+ used,
804
+ optional,
805
+ unresolved
806
+ });
807
+ const subprotocols = WebSocketResolver.subprotocols(raw);
808
+ if (subprotocols !== void 0) {
809
+ const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
810
+ const set = !unresolved.has(YamlDocument.formatPath([
811
+ "websocket",
812
+ "connection",
813
+ "subprotocols",
814
+ index
815
+ ]));
816
+ const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
817
+ return set && !empty;
818
+ });
819
+ value.websocket.connection.subprotocols = kept;
820
+ }
821
+ const prefix = YamlDocument.formatPath([
822
+ "websocket",
823
+ "connection",
824
+ "subprotocols"
825
+ ]);
826
+ return {
827
+ value,
828
+ deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
829
+ };
830
+ }
831
+ validate(value, document) {
832
+ const issues = this.schemaValidator.validate(value, document);
833
+ if (issues.length > 0) return issues;
834
+ return WebSocketResolver.validateInputPath(value, document);
835
+ }
836
+ static validateInputPath(value, document) {
837
+ const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
838
+ if (typeof request.inputPath !== "string") return [];
839
+ const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
840
+ const last = tokens.pop();
841
+ let parent = request.value;
842
+ for (const token of tokens) parent = WebSocketResolver.child(parent, token);
843
+ if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
844
+ return [new ValidationIssue({
845
+ message: "must point to a missing or string value whose parent exists in request.value",
846
+ location: document.locate([
847
+ "websocket",
848
+ "protocol",
849
+ "request",
850
+ "inputPath"
851
+ ])
852
+ })];
853
+ }
854
+ static section(value, ...path) {
855
+ let current = value;
856
+ for (const key of path) current = WebSocketResolver.child(current, key);
857
+ return isRecord(current) ? current : {};
858
+ }
859
+ static child(value, key) {
860
+ if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
861
+ return isRecord(value) ? value[key] : void 0;
862
+ }
863
+ static isSubprotocol(path) {
864
+ return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
865
+ }
866
+ static subprotocols(websocket) {
867
+ const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
868
+ return Array.isArray(subprotocols) ? subprotocols : void 0;
869
+ }
870
+ };
723
871
  // src/config/defaultConfig.ts
724
872
  const DEFAULT_CONFIG = new EvalConfig({
725
873
  execution: new ExecutionConfig({
@@ -755,6 +903,8 @@ var EvalConfigLoader = class EvalConfigLoader {
755
903
  cwd;
756
904
  documentLoader = new YamlDocumentLoader();
757
905
  schemaValidator = new SchemaValidator(eval_config_schema_default);
906
+ variablesReader = new VariablesReader();
907
+ webSocketResolver = new WebSocketResolver();
758
908
  environmentResolver;
759
909
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
760
910
  this.cwd = cwd;
@@ -765,19 +915,47 @@ var EvalConfigLoader = class EvalConfigLoader {
765
915
  const issues = [...document.errors];
766
916
  if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
767
917
  const named = [];
768
- const value = issues.length === 0 ? this.environmentResolver.resolve(document.value, document, issues, named) : void 0;
769
- if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
770
- if (issues.length === 0) issues.push(...this.validateInputPath(value, document));
918
+ const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
919
+ let value = {};
920
+ if (issues.length === 0) {
921
+ const resolver = this.environmentResolver.withLevels([variables]);
922
+ const { websocket: _websocket, variables: _variables, ...rest } = document.value;
923
+ const resolved = resolver.resolve(rest, document, issues, named);
924
+ const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
925
+ value = {
926
+ ...resolved,
927
+ ..._variables === void 0 ? {} : { variables },
928
+ ..._websocket === void 0 ? {} : websocket.value
929
+ };
930
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
931
+ if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
932
+ }
771
933
  if (issues.length > 0) throw new EvalError({
772
934
  category: ErrorCategory.Configuration,
773
935
  message: "invalid configuration",
774
936
  issues
775
937
  });
776
- const config = this.build(value, configPath, named);
938
+ const config = this.build(value, configPath, variables, named);
777
939
  this.applyRepetition(config, override, value, document);
778
940
  EvalConfigLoader.applyReport(config, report);
779
941
  return config;
780
942
  }
943
+ static buildWebSocket(value) {
944
+ const section = (...path) => WebSocketResolver.section(value, ...path);
945
+ const connection = section("websocket", "connection");
946
+ const request = section("websocket", "protocol", "request");
947
+ return new WebSocketConfig({
948
+ endpoint: WebSocketResolver.section(value, "websocket").endpoint,
949
+ connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
950
+ protocol: new ProtocolConfig({
951
+ request: new RequestConfig({
952
+ ...request,
953
+ format: request.format ?? RequestFormat.Json
954
+ }),
955
+ capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
956
+ })
957
+ });
958
+ }
781
959
  static applyReport(config, report) {
782
960
  if (report.formats === void 0) return;
783
961
  const known = Object.values(ReportFormat);
@@ -799,45 +977,17 @@ var EvalConfigLoader = class EvalConfigLoader {
799
977
  location: document.locate(["evaluator", "apiKey"])
800
978
  })];
801
979
  }
802
- validateInputPath(value, document) {
803
- const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
804
- if (typeof request.inputPath !== "string") return [];
805
- const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
806
- const last = tokens.pop();
807
- let parent = request.value;
808
- for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
809
- if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
810
- return [new ValidationIssue({
811
- message: "must point to a missing or string value whose parent exists in request.value",
812
- location: document.locate([
813
- "websocket",
814
- "protocol",
815
- "request",
816
- "inputPath"
817
- ])
818
- })];
819
- }
820
- build(value, configPath, named) {
980
+ build(value, configPath, variables, named) {
821
981
  const section = (...path) => EvalConfigLoader.section(value, ...path);
822
982
  const configDirectory = dirname(configPath);
823
- const connection = section("websocket", "connection");
824
- const request = section("websocket", "protocol", "request");
983
+ const websocket = EvalConfigLoader.buildWebSocket(value);
825
984
  const evaluator = section("evaluator");
826
985
  const report = section("report");
827
986
  return new EvalConfig({
828
987
  version: value.version,
829
988
  configPath,
830
- websocket: new WebSocketConfig({
831
- endpoint: EvalConfigLoader.section(value, "websocket").endpoint,
832
- connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
833
- protocol: new ProtocolConfig({
834
- request: new RequestConfig({
835
- ...request,
836
- format: request.format ?? RequestFormat.Json
837
- }),
838
- capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
839
- })
840
- }),
989
+ variables,
990
+ websocket,
841
991
  execution: new ExecutionConfig({
842
992
  ...DEFAULT_CONFIG.execution,
843
993
  ...section("execution"),
@@ -858,7 +1008,7 @@ var EvalConfigLoader = class EvalConfigLoader {
858
1008
  directory: resolve(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
859
1009
  formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
860
1010
  }),
861
- secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols ?? [], named)
1011
+ secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
862
1012
  });
863
1013
  }
864
1014
  applyRepetition(config, override, value, document) {
@@ -911,13 +1061,7 @@ var EvalConfigLoader = class EvalConfigLoader {
911
1061
  ])];
912
1062
  }
913
1063
  static section(value, ...path) {
914
- let current = value;
915
- for (const key of path) current = EvalConfigLoader.child(current, key);
916
- return isRecord(current) ? current : {};
917
- }
918
- static child(value, key) {
919
- if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
920
- return isRecord(value) ? value[key] : void 0;
1064
+ return WebSocketResolver.section(value, ...path);
921
1065
  }
922
1066
  };
923
1067
  // src/interfaces/runStatus.ts
@@ -1113,6 +1257,7 @@ var ConversationSpec = class {
1113
1257
  turns;
1114
1258
  conversation;
1115
1259
  location;
1260
+ websocket;
1116
1261
  secrets;
1117
1262
  constructor(data) {
1118
1263
  this.version = data?.version ?? 1;
@@ -1121,6 +1266,7 @@ var ConversationSpec = class {
1121
1266
  this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
1122
1267
  this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
1123
1268
  this.location = new SourceLocation(data?.location);
1269
+ this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
1124
1270
  this.secrets = [...data?.secrets ?? []];
1125
1271
  }
1126
1272
  };
@@ -2420,7 +2566,11 @@ var ConversationRunner = class {
2420
2566
  async run(spec, signal = NEVER_ABORTED) {
2421
2567
  const startedAt = Date.now();
2422
2568
  const outcomes = [];
2423
- const assertions = new AssertionRunner(this.config.websocket.protocol.capabilities);
2569
+ const config = new EvalConfig({
2570
+ ...this.config,
2571
+ websocket: spec.websocket ?? this.config.websocket
2572
+ });
2573
+ const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
2424
2574
  const history = [];
2425
2575
  const evaluate = this.semanticEvaluator(signal);
2426
2576
  const inline = !this.config.execution.continueOnFailure;
@@ -2445,7 +2595,7 @@ var ConversationRunner = class {
2445
2595
  try {
2446
2596
  const steps = (async () => {
2447
2597
  for (let attempt = 0;; attempt++) {
2448
- connection = this.createConnection(this.config);
2598
+ connection = this.createConnection(config);
2449
2599
  connectionAttempts++;
2450
2600
  try {
2451
2601
  await connection.connect();
@@ -2626,7 +2776,7 @@ var ScenarioRunner = class {
2626
2776
  required: minPass,
2627
2777
  attempts,
2628
2778
  tests: this.results.tests(attempts, minPass),
2629
- agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
2779
+ agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
2630
2780
  evaluatorUsage: this.tokens.evaluator(attempts)
2631
2781
  });
2632
2782
  }
@@ -2788,7 +2938,9 @@ var ConsoleReporter = class ConsoleReporter {
2788
2938
  const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
2789
2939
  if (!this.verbose) return line;
2790
2940
  if (result.attempts.length === 0) return `${line}\n`;
2791
- const lines = [""];
2941
+ const websocket = spec.websocket;
2942
+ const subprotocols = websocket?.connection.subprotocols.length ?? 0;
2943
+ const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
2792
2944
  for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
2793
2945
  lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
2794
2946
  lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
@@ -3000,11 +3152,16 @@ var JsonReporter = class JsonReporter {
3000
3152
  };
3001
3153
  }
3002
3154
  scenario(result, config) {
3003
- const capability = config.websocket.protocol.capabilities.tokenUsage;
3155
+ const websocket = result.spec.websocket ?? config.websocket;
3156
+ const capability = websocket.protocol.capabilities.tokenUsage;
3004
3157
  return {
3005
3158
  name: result.spec.name,
3006
3159
  file: result.spec.location.file,
3007
3160
  status: result.status,
3161
+ connection: {
3162
+ endpoint: websocket.endpoint,
3163
+ subprotocols: websocket.connection.subprotocols.length
3164
+ },
3008
3165
  ...JsonReporter.counts(result),
3009
3166
  agentUsage: JsonReporter.usage(result.agentUsage),
3010
3167
  evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
@@ -3165,6 +3322,11 @@ var conversation_schema_default = {
3165
3322
  "version": { "const": 1 },
3166
3323
  "name": { "$ref": "#/definitions/text" },
3167
3324
  "description": { "type": "string" },
3325
+ "variables": {
3326
+ "type": "object",
3327
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
3328
+ "additionalProperties": { "type": "string" }
3329
+ },
3168
3330
  "turns": {
3169
3331
  "type": "array",
3170
3332
  "minItems": 1,
@@ -3177,8 +3339,8 @@ var conversation_schema_default = {
3177
3339
  "properties": { "conversation": {
3178
3340
  "type": "object",
3179
3341
  "additionalProperties": false,
3180
- "required": ["criteria"],
3181
- "properties": { "criteria": { "$ref": "#/definitions/text" } }
3342
+ "required": ["should"],
3343
+ "properties": { "should": { "$ref": "#/definitions/text" } }
3182
3344
  } }
3183
3345
  }
3184
3346
  },
@@ -3221,12 +3383,12 @@ var conversation_schema_default = {
3221
3383
  }
3222
3384
  },
3223
3385
  "response": {
3224
- "description": "A string is shorthand for criteria.",
3386
+ "description": "A string is shorthand for should.",
3225
3387
  "type": ["string", "object"],
3226
3388
  "pattern": "\\S",
3227
3389
  "additionalProperties": false,
3228
3390
  "properties": {
3229
- "criteria": { "$ref": "#/definitions/text" },
3391
+ "should": { "$ref": "#/definitions/text" },
3230
3392
  "contains": { "$ref": "#/definitions/textList" },
3231
3393
  "notContains": { "$ref": "#/definitions/textList" }
3232
3394
  }
@@ -3258,7 +3420,7 @@ var conversation_schema_default = {
3258
3420
  };
3259
3421
  // src/scenarios/conversationValidator.ts
3260
3422
  const RESPONSE_KEYS = [
3261
- "criteria",
3423
+ "should",
3262
3424
  "contains",
3263
3425
  "notContains"
3264
3426
  ];
@@ -3281,7 +3443,7 @@ var ConversationValidator = class ConversationValidator {
3281
3443
  validateResponse(response, path, document, issues) {
3282
3444
  if (!RESPONSE_KEYS.some((key) => key in response)) {
3283
3445
  issues.push(new ValidationIssue({
3284
- message: "response must have at least one of: criteria, contains, notContains",
3446
+ message: "response must have at least one of: should, contains, notContains",
3285
3447
  location: document.locate(path)
3286
3448
  }));
3287
3449
  return;
@@ -3324,11 +3486,13 @@ var ConversationValidator = class ConversationValidator {
3324
3486
  };
3325
3487
  // src/scenarios/scenarioLoader.ts
3326
3488
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
3327
- var ScenarioLoader = class {
3489
+ var ScenarioLoader = class ScenarioLoader {
3328
3490
  cwd;
3329
3491
  documentLoader = new YamlDocumentLoader();
3330
3492
  schemaValidator = new SchemaValidator(conversation_schema_default);
3331
3493
  conversationValidator = new ConversationValidator();
3494
+ variablesReader = new VariablesReader();
3495
+ webSocketResolver = new WebSocketResolver();
3332
3496
  environmentResolver;
3333
3497
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
3334
3498
  this.cwd = cwd;
@@ -3336,18 +3500,52 @@ var ScenarioLoader = class {
3336
3500
  }
3337
3501
  load(target, config) {
3338
3502
  const files = this.discover(resolve(this.cwd, target), config);
3503
+ const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.cwd, config.configPath)) : void 0;
3339
3504
  const issues = [];
3505
+ const connectionIssues = [];
3340
3506
  const specs = [];
3341
3507
  for (const file of files) {
3342
3508
  const document = this.documentLoader.load(file, relative(this.cwd, file));
3343
3509
  const fileIssues = [...document.errors];
3344
3510
  const secrets = [];
3345
- const value = fileIssues.length === 0 ? this.environmentResolver.resolve(document.value, document, fileIssues, secrets) : void 0;
3511
+ const used = /* @__PURE__ */ new Set();
3512
+ const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
3513
+ const resolver = this.environmentResolver.withLevels([variables, config.variables]);
3514
+ let value;
3515
+ if (fileIssues.length === 0) {
3516
+ const { variables: raw, ...rest } = document.value;
3517
+ value = {
3518
+ ...resolver.resolve(rest, document, fileIssues, secrets, { used }),
3519
+ ...raw === void 0 ? {} : { variables }
3520
+ };
3521
+ }
3522
+ let websocket;
3523
+ const websocketIssues = [];
3524
+ if (fileIssues.length === 0 && configDocument !== void 0) {
3525
+ const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
3526
+ if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
3527
+ connectionIssues.push({
3528
+ file: document,
3529
+ issues: websocketIssues
3530
+ });
3531
+ if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
3532
+ }
3346
3533
  if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
3347
3534
  if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
3348
- if (fileIssues.length === 0) specs.push(this.build(value, document, secrets));
3535
+ if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
3536
+ if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
3349
3537
  else issues.push(...fileIssues);
3350
3538
  }
3539
+ const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
3540
+ if (shared.length > 0) throw new EvalError({
3541
+ category: ErrorCategory.Configuration,
3542
+ message: "invalid configuration",
3543
+ issues: shared
3544
+ });
3545
+ for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
3546
+ message: issue.toString(),
3547
+ location: ScenarioLoader.variablesLocation(entry.file)
3548
+ }));
3351
3549
  if (issues.length > 0) throw new EvalError({
3352
3550
  category: ErrorCategory.Specification,
3353
3551
  message: "invalid scenario specification",
@@ -3355,6 +3553,23 @@ var ScenarioLoader = class {
3355
3553
  });
3356
3554
  return specs;
3357
3555
  }
3556
+ static shared(lists) {
3557
+ if (lists.length === 0) return [];
3558
+ const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
3559
+ return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
3560
+ }
3561
+ static variablesLocation(document) {
3562
+ return new SourceLocation({
3563
+ ...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
3564
+ path: ""
3565
+ });
3566
+ }
3567
+ static unusedVariables(variables, used, document) {
3568
+ return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
3569
+ message: `variable '${name}' is not used by the scenario or the websocket configuration`,
3570
+ location: document.locateKey(["variables"], name)
3571
+ }));
3572
+ }
3358
3573
  discover(target, config) {
3359
3574
  if (!existsSync(target)) throw new EvalError({
3360
3575
  category: ErrorCategory.Specification,
@@ -3379,7 +3594,7 @@ var ScenarioLoader = class {
3379
3594
  }
3380
3595
  return files;
3381
3596
  }
3382
- build(value, document, secrets) {
3597
+ build(value, document, websocket, secrets) {
3383
3598
  const scenario = value;
3384
3599
  const turns = scenario.turns;
3385
3600
  const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
@@ -3389,10 +3604,11 @@ var ScenarioLoader = class {
3389
3604
  description: typeof scenario.description === "string" ? scenario.description : "",
3390
3605
  turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
3391
3606
  conversation: conversation ? new ConversationExpectation({
3392
- criteria: conversation.criteria,
3607
+ criteria: conversation.should,
3393
3608
  location: document.locate(["expect", "conversation"])
3394
3609
  }) : void 0,
3395
3610
  location: document.locate([]),
3611
+ websocket,
3396
3612
  secrets
3397
3613
  });
3398
3614
  }
@@ -3418,7 +3634,7 @@ var ScenarioLoader = class {
3418
3634
  });
3419
3635
  const fields = response;
3420
3636
  return new ResponseExpectation({
3421
- criteria: fields.criteria,
3637
+ criteria: fields.should,
3422
3638
  contains: fields.contains ?? [],
3423
3639
  notContains: fields.notContains ?? [],
3424
3640
  location