@mappies/miniwat-eval 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/main.js CHANGED
@@ -2,8 +2,8 @@
2
2
  import { Command, CommanderError } from "commander";
3
3
  import { dirname, extname, join, relative, resolve } from "node:path";
4
4
  import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
5
- import { Ajv } from "ajv";
6
5
  import { LineCounter, isMap, isPair, isScalar, isSeq, parseDocument } from "yaml";
6
+ import { Ajv } from "ajv";
7
7
  import { Message, ResponseMessage, TokenUsage } from "@mappies/miniwat-agent-core";
8
8
  import { Agent, InMemoryAgentStateStore } from "@mappies/miniwat-agent";
9
9
  import { randomUUID } from "node:crypto";
@@ -164,6 +164,44 @@ var ConfigLocator = class ConfigLocator {
164
164
  return existsSync(path) && statSync(path).isFile();
165
165
  }
166
166
  };
167
+ // src/utils/isRecord.ts
168
+ function isRecord(value) {
169
+ return value !== null && typeof value === "object" && !Array.isArray(value);
170
+ }
171
+ // src/config/parametersLoader.ts
172
+ var ParametersLoader = class ParametersLoader {
173
+ cwd;
174
+ constructor(cwd = process.cwd()) {
175
+ this.cwd = cwd;
176
+ }
177
+ load(path) {
178
+ const absolute = resolve(this.cwd, path);
179
+ const shown = relative(this.cwd, absolute) || path;
180
+ let text;
181
+ let value;
182
+ try {
183
+ text = readFileSync(absolute, "utf8");
184
+ } catch {
185
+ throw ParametersLoader.error(`parameters file not found: ${shown}`);
186
+ }
187
+ try {
188
+ value = JSON.parse(text);
189
+ } catch {
190
+ throw ParametersLoader.error(`parameters file ${shown} is not valid JSON`);
191
+ }
192
+ if (!isRecord(value)) throw ParametersLoader.error(`parameters file ${shown} must contain a JSON object`);
193
+ return {
194
+ path: absolute,
195
+ parameters: Object.fromEntries(Object.entries(value).filter((entry) => typeof entry[1] === "string"))
196
+ };
197
+ }
198
+ static error(message) {
199
+ return new EvalError({
200
+ category: ErrorCategory.Configuration,
201
+ message
202
+ });
203
+ }
204
+ };
167
205
  // schemas/eval-config.schema.json
168
206
  var eval_config_schema_default = {
169
207
  $schema: "http://json-schema.org/draft-07/schema#",
@@ -177,6 +215,11 @@ var eval_config_schema_default = {
177
215
  ],
178
216
  properties: {
179
217
  "version": { "const": 1 },
218
+ "variables": {
219
+ "type": "object",
220
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
221
+ "additionalProperties": { "type": "string" }
222
+ },
180
223
  "websocket": {
181
224
  "type": "object",
182
225
  "additionalProperties": false,
@@ -471,6 +514,8 @@ var WebSocketConfig = class {
471
514
  var EvalConfig = class {
472
515
  version;
473
516
  configPath;
517
+ parametersPath;
518
+ variables;
474
519
  websocket;
475
520
  execution;
476
521
  evaluator;
@@ -479,6 +524,8 @@ var EvalConfig = class {
479
524
  constructor(data) {
480
525
  this.version = data?.version ?? 1;
481
526
  this.configPath = data?.configPath ?? "";
527
+ this.parametersPath = data?.parametersPath ?? "";
528
+ this.variables = { ...data?.variables ?? {} };
482
529
  this.websocket = new WebSocketConfig(data?.websocket);
483
530
  this.execution = new ExecutionConfig(data?.execution);
484
531
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -491,16 +538,80 @@ const ReportFormat = {
491
538
  Console: "console",
492
539
  Json: "json"
493
540
  };
541
+ // src/yaml/yamlDocument.ts
542
+ var YamlDocument = class YamlDocument {
543
+ file;
544
+ document;
545
+ lineCounter;
546
+ value;
547
+ errors;
548
+ constructor(file, document, lineCounter) {
549
+ this.file = file;
550
+ this.document = document;
551
+ this.lineCounter = lineCounter;
552
+ this.value = document.toJS();
553
+ this.errors = document.errors.map((error) => new ValidationIssue({
554
+ message: error.message.split("\n")[0],
555
+ location: this.toLocation(error.pos[0], [])
556
+ }));
557
+ }
558
+ locate(path) {
559
+ return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
560
+ }
561
+ locateKey(path, key) {
562
+ const parent = this.findNode(path);
563
+ if (isMap(parent)) {
564
+ const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
565
+ if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
566
+ }
567
+ return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
568
+ }
569
+ static formatPath(path) {
570
+ return path.map((segment, index) => {
571
+ if (typeof segment === "number") return `[${segment}]`;
572
+ return index === 0 ? segment : `.${segment}`;
573
+ }).join("");
574
+ }
575
+ findNode(path) {
576
+ let node = this.document.contents ?? void 0;
577
+ for (const segment of path) {
578
+ let next;
579
+ if (isMap(node)) {
580
+ const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
581
+ next = pair?.value ?? pair?.key;
582
+ } else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
583
+ if (!next) break;
584
+ node = next;
585
+ }
586
+ return node;
587
+ }
588
+ toLocation(offset, path) {
589
+ const position = this.lineCounter.linePos(offset);
590
+ return new SourceLocation({
591
+ file: this.file,
592
+ line: position.line,
593
+ column: position.col,
594
+ path: YamlDocument.formatPath(path)
595
+ });
596
+ }
597
+ };
494
598
  // src/yaml/environmentResolver.ts
495
- const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g;
599
+ const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
496
600
  const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
497
601
  var EnvironmentResolver = class EnvironmentResolver {
498
602
  environment;
499
- constructor(environment = process.env) {
603
+ parameters;
604
+ levels;
605
+ constructor(environment = process.env, parameters = {}, levels = []) {
500
606
  this.environment = environment;
607
+ this.parameters = parameters;
608
+ this.levels = levels;
609
+ }
610
+ withLevels(levels) {
611
+ return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
501
612
  }
502
- resolve(value, document, issues, secrets = []) {
503
- return this.resolveValue(value, [], document, issues, secrets);
613
+ resolve(value, document, issues, secrets = [], options = {}) {
614
+ return this.resolveValue(value, [], document, issues, secrets, options);
504
615
  }
505
616
  static hasReference(value) {
506
617
  return new RegExp(REFERENCE.source).test(value);
@@ -508,21 +619,38 @@ var EnvironmentResolver = class EnvironmentResolver {
508
619
  static isSecretName(name) {
509
620
  return SECRET_NAME.test(name);
510
621
  }
511
- resolveValue(value, path, document, issues, secrets) {
622
+ lookup(name) {
623
+ for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
624
+ return this.environment[name];
625
+ }
626
+ sources() {
627
+ const sources = [
628
+ ...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
629
+ ...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
630
+ "environment variable"
631
+ ];
632
+ return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
633
+ }
634
+ resolveValue(value, path, document, issues, secrets, options) {
512
635
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
513
- const resolved = this.environment[name];
636
+ options.used?.add(name);
637
+ const resolved = this.lookup(name);
514
638
  if (resolved === void 0) {
639
+ if (options.optional?.(path)) {
640
+ options.unresolved?.add(YamlDocument.formatPath(path));
641
+ return reference;
642
+ }
515
643
  issues.push(new ValidationIssue({
516
- message: `environment variable '${name}' is not set`,
644
+ message: `${this.sources()} '${name}' is not set`,
517
645
  location: document.locate(path)
518
646
  }));
519
647
  return reference;
520
648
  }
521
- if (EnvironmentResolver.isSecretName(name) && !secrets.includes(resolved)) secrets.push(resolved);
649
+ if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
522
650
  return resolved;
523
651
  });
524
- if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
525
- if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
652
+ if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
653
+ if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
526
654
  return value;
527
655
  }
528
656
  };
@@ -606,63 +734,6 @@ var SchemaValidator = class SchemaValidator {
606
734
  return path;
607
735
  }
608
736
  };
609
- // src/yaml/yamlDocument.ts
610
- var YamlDocument = class YamlDocument {
611
- file;
612
- document;
613
- lineCounter;
614
- value;
615
- errors;
616
- constructor(file, document, lineCounter) {
617
- this.file = file;
618
- this.document = document;
619
- this.lineCounter = lineCounter;
620
- this.value = document.toJS();
621
- this.errors = document.errors.map((error) => new ValidationIssue({
622
- message: error.message.split("\n")[0],
623
- location: this.toLocation(error.pos[0], [])
624
- }));
625
- }
626
- locate(path) {
627
- return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
628
- }
629
- locateKey(path, key) {
630
- const parent = this.findNode(path);
631
- if (isMap(parent)) {
632
- const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
633
- if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
634
- }
635
- return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
636
- }
637
- static formatPath(path) {
638
- return path.map((segment, index) => {
639
- if (typeof segment === "number") return `[${segment}]`;
640
- return index === 0 ? segment : `.${segment}`;
641
- }).join("");
642
- }
643
- findNode(path) {
644
- let node = this.document.contents ?? void 0;
645
- for (const segment of path) {
646
- let next;
647
- if (isMap(node)) {
648
- const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
649
- next = pair?.value ?? pair?.key;
650
- } else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
651
- if (!next) break;
652
- node = next;
653
- }
654
- return node;
655
- }
656
- toLocation(offset, path) {
657
- const position = this.lineCounter.linePos(offset);
658
- return new SourceLocation({
659
- file: this.file,
660
- line: position.line,
661
- column: position.col,
662
- path: YamlDocument.formatPath(path)
663
- });
664
- }
665
- };
666
737
  // src/yaml/yamlDocumentLoader.ts
667
738
  var YamlDocumentLoader = class {
668
739
  load(filePath, displayPath) {
@@ -677,10 +748,126 @@ var YamlDocumentLoader = class {
677
748
  }), lineCounter);
678
749
  }
679
750
  };
680
- // src/utils/isRecord.ts
681
- function isRecord(value) {
682
- return value !== null && typeof value === "object" && !Array.isArray(value);
683
- }
751
+ // src/yaml/variablesReader.ts
752
+ const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
753
+ var VariablesReader = class {
754
+ read(document, resolver, issues, secrets) {
755
+ const raw = isRecord(document.value) ? document.value.variables : void 0;
756
+ if (raw === void 0) return {};
757
+ if (!isRecord(raw)) {
758
+ issues.push(new ValidationIssue({
759
+ message: "must be object",
760
+ location: document.locate(["variables"])
761
+ }));
762
+ return {};
763
+ }
764
+ const variables = {};
765
+ for (const [name, value] of Object.entries(raw)) {
766
+ if (!VARIABLE_NAME.test(name)) {
767
+ issues.push(new ValidationIssue({
768
+ message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
769
+ location: document.locateKey(["variables"], name)
770
+ }));
771
+ continue;
772
+ }
773
+ if (typeof value !== "string") {
774
+ issues.push(new ValidationIssue({
775
+ message: "must be string",
776
+ location: document.locate(["variables", name])
777
+ }));
778
+ continue;
779
+ }
780
+ const before = issues.length;
781
+ const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
782
+ if (issues.length > before) continue;
783
+ variables[name] = resolved.variables[name];
784
+ if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
785
+ }
786
+ return variables;
787
+ }
788
+ };
789
+ // src/config/webSocketResolver.ts
790
+ const WEBSOCKET_SCHEMA = {
791
+ ...eval_config_schema_default,
792
+ required: ["websocket"],
793
+ additionalProperties: true,
794
+ properties: { websocket: eval_config_schema_default.properties.websocket }
795
+ };
796
+ var WebSocketResolver = class WebSocketResolver {
797
+ schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
798
+ resolve(document, resolver, issues, secrets, used, defer) {
799
+ const raw = isRecord(document.value) ? document.value.websocket : void 0;
800
+ const unresolved = /* @__PURE__ */ new Set();
801
+ const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
802
+ const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
803
+ used,
804
+ optional,
805
+ unresolved
806
+ });
807
+ const subprotocols = WebSocketResolver.subprotocols(raw);
808
+ if (subprotocols !== void 0) {
809
+ const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
810
+ const set = !unresolved.has(YamlDocument.formatPath([
811
+ "websocket",
812
+ "connection",
813
+ "subprotocols",
814
+ index
815
+ ]));
816
+ const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
817
+ return set && !empty;
818
+ });
819
+ value.websocket.connection.subprotocols = kept;
820
+ }
821
+ const prefix = YamlDocument.formatPath([
822
+ "websocket",
823
+ "connection",
824
+ "subprotocols"
825
+ ]);
826
+ return {
827
+ value,
828
+ deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
829
+ };
830
+ }
831
+ validate(value, document) {
832
+ const issues = this.schemaValidator.validate(value, document);
833
+ if (issues.length > 0) return issues;
834
+ return WebSocketResolver.validateInputPath(value, document);
835
+ }
836
+ static validateInputPath(value, document) {
837
+ const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
838
+ if (typeof request.inputPath !== "string") return [];
839
+ const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
840
+ const last = tokens.pop();
841
+ let parent = request.value;
842
+ for (const token of tokens) parent = WebSocketResolver.child(parent, token);
843
+ if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
844
+ return [new ValidationIssue({
845
+ message: "must point to a missing or string value whose parent exists in request.value",
846
+ location: document.locate([
847
+ "websocket",
848
+ "protocol",
849
+ "request",
850
+ "inputPath"
851
+ ])
852
+ })];
853
+ }
854
+ static section(value, ...path) {
855
+ let current = value;
856
+ for (const key of path) current = WebSocketResolver.child(current, key);
857
+ return isRecord(current) ? current : {};
858
+ }
859
+ static child(value, key) {
860
+ if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
861
+ return isRecord(value) ? value[key] : void 0;
862
+ }
863
+ static isSubprotocol(path) {
864
+ return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
865
+ }
866
+ static subprotocols(websocket) {
867
+ const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
868
+ return Array.isArray(subprotocols) ? subprotocols : void 0;
869
+ }
870
+ };
684
871
  // src/config/defaultConfig.ts
685
872
  const DEFAULT_CONFIG = new EvalConfig({
686
873
  execution: new ExecutionConfig({
@@ -716,29 +903,59 @@ var EvalConfigLoader = class EvalConfigLoader {
716
903
  cwd;
717
904
  documentLoader = new YamlDocumentLoader();
718
905
  schemaValidator = new SchemaValidator(eval_config_schema_default);
906
+ variablesReader = new VariablesReader();
907
+ webSocketResolver = new WebSocketResolver();
719
908
  environmentResolver;
720
- constructor(environment = process.env, cwd = process.cwd()) {
909
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
721
910
  this.cwd = cwd;
722
- this.environmentResolver = new EnvironmentResolver(environment);
911
+ this.environmentResolver = new EnvironmentResolver(environment, parameters);
723
912
  }
724
913
  load(configPath, override, report = new ReportOverride()) {
725
914
  const document = this.documentLoader.load(configPath, relative(this.cwd, configPath));
726
915
  const issues = [...document.errors];
727
916
  if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
728
917
  const named = [];
729
- const value = issues.length === 0 ? this.environmentResolver.resolve(document.value, document, issues, named) : void 0;
730
- if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
731
- if (issues.length === 0) issues.push(...this.validateInputPath(value, document));
918
+ const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
919
+ let value = {};
920
+ if (issues.length === 0) {
921
+ const resolver = this.environmentResolver.withLevels([variables]);
922
+ const { websocket: _websocket, variables: _variables, ...rest } = document.value;
923
+ const resolved = resolver.resolve(rest, document, issues, named);
924
+ const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
925
+ value = {
926
+ ...resolved,
927
+ ..._variables === void 0 ? {} : { variables },
928
+ ..._websocket === void 0 ? {} : websocket.value
929
+ };
930
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
931
+ if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
932
+ }
732
933
  if (issues.length > 0) throw new EvalError({
733
934
  category: ErrorCategory.Configuration,
734
935
  message: "invalid configuration",
735
936
  issues
736
937
  });
737
- const config = this.build(value, configPath, named);
938
+ const config = this.build(value, configPath, variables, named);
738
939
  this.applyRepetition(config, override, value, document);
739
940
  EvalConfigLoader.applyReport(config, report);
740
941
  return config;
741
942
  }
943
+ static buildWebSocket(value) {
944
+ const section = (...path) => WebSocketResolver.section(value, ...path);
945
+ const connection = section("websocket", "connection");
946
+ const request = section("websocket", "protocol", "request");
947
+ return new WebSocketConfig({
948
+ endpoint: WebSocketResolver.section(value, "websocket").endpoint,
949
+ connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
950
+ protocol: new ProtocolConfig({
951
+ request: new RequestConfig({
952
+ ...request,
953
+ format: request.format ?? RequestFormat.Json
954
+ }),
955
+ capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
956
+ })
957
+ });
958
+ }
742
959
  static applyReport(config, report) {
743
960
  if (report.formats === void 0) return;
744
961
  const known = Object.values(ReportFormat);
@@ -756,49 +973,21 @@ var EvalConfigLoader = class EvalConfigLoader {
756
973
  const apiKey = EvalConfigLoader.section(document.value, "evaluator").apiKey;
757
974
  if (typeof apiKey !== "string" || EnvironmentResolver.hasReference(apiKey)) return [];
758
975
  return [new ValidationIssue({
759
- message: "must be an environment reference such as ${OPENAI_API_KEY}",
976
+ message: "must be a reference such as ${OPENAI_API_KEY}",
760
977
  location: document.locate(["evaluator", "apiKey"])
761
978
  })];
762
979
  }
763
- validateInputPath(value, document) {
764
- const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
765
- if (typeof request.inputPath !== "string") return [];
766
- const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
767
- const last = tokens.pop();
768
- let parent = request.value;
769
- for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
770
- if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
771
- return [new ValidationIssue({
772
- message: "must point to a missing or string value whose parent exists in request.value",
773
- location: document.locate([
774
- "websocket",
775
- "protocol",
776
- "request",
777
- "inputPath"
778
- ])
779
- })];
780
- }
781
- build(value, configPath, named) {
980
+ build(value, configPath, variables, named) {
782
981
  const section = (...path) => EvalConfigLoader.section(value, ...path);
783
982
  const configDirectory = dirname(configPath);
784
- const connection = section("websocket", "connection");
785
- const request = section("websocket", "protocol", "request");
983
+ const websocket = EvalConfigLoader.buildWebSocket(value);
786
984
  const evaluator = section("evaluator");
787
985
  const report = section("report");
788
986
  return new EvalConfig({
789
987
  version: value.version,
790
988
  configPath,
791
- websocket: new WebSocketConfig({
792
- endpoint: EvalConfigLoader.section(value, "websocket").endpoint,
793
- connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
794
- protocol: new ProtocolConfig({
795
- request: new RequestConfig({
796
- ...request,
797
- format: request.format ?? RequestFormat.Json
798
- }),
799
- capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
800
- })
801
- }),
989
+ variables,
990
+ websocket,
802
991
  execution: new ExecutionConfig({
803
992
  ...DEFAULT_CONFIG.execution,
804
993
  ...section("execution"),
@@ -819,7 +1008,7 @@ var EvalConfigLoader = class EvalConfigLoader {
819
1008
  directory: resolve(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
820
1009
  formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
821
1010
  }),
822
- secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols ?? [], named)
1011
+ secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
823
1012
  });
824
1013
  }
825
1014
  applyRepetition(config, override, value, document) {
@@ -872,13 +1061,7 @@ var EvalConfigLoader = class EvalConfigLoader {
872
1061
  ])];
873
1062
  }
874
1063
  static section(value, ...path) {
875
- let current = value;
876
- for (const key of path) current = EvalConfigLoader.child(current, key);
877
- return isRecord(current) ? current : {};
878
- }
879
- static child(value, key) {
880
- if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
881
- return isRecord(value) ? value[key] : void 0;
1064
+ return WebSocketResolver.section(value, ...path);
882
1065
  }
883
1066
  };
884
1067
  // src/interfaces/runStatus.ts
@@ -1074,6 +1257,7 @@ var ConversationSpec = class {
1074
1257
  turns;
1075
1258
  conversation;
1076
1259
  location;
1260
+ websocket;
1077
1261
  secrets;
1078
1262
  constructor(data) {
1079
1263
  this.version = data?.version ?? 1;
@@ -1082,6 +1266,7 @@ var ConversationSpec = class {
1082
1266
  this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
1083
1267
  this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
1084
1268
  this.location = new SourceLocation(data?.location);
1269
+ this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
1085
1270
  this.secrets = [...data?.secrets ?? []];
1086
1271
  }
1087
1272
  };
@@ -2381,7 +2566,11 @@ var ConversationRunner = class {
2381
2566
  async run(spec, signal = NEVER_ABORTED) {
2382
2567
  const startedAt = Date.now();
2383
2568
  const outcomes = [];
2384
- const assertions = new AssertionRunner(this.config.websocket.protocol.capabilities);
2569
+ const config = new EvalConfig({
2570
+ ...this.config,
2571
+ websocket: spec.websocket ?? this.config.websocket
2572
+ });
2573
+ const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
2385
2574
  const history = [];
2386
2575
  const evaluate = this.semanticEvaluator(signal);
2387
2576
  const inline = !this.config.execution.continueOnFailure;
@@ -2406,7 +2595,7 @@ var ConversationRunner = class {
2406
2595
  try {
2407
2596
  const steps = (async () => {
2408
2597
  for (let attempt = 0;; attempt++) {
2409
- connection = this.createConnection(this.config);
2598
+ connection = this.createConnection(config);
2410
2599
  connectionAttempts++;
2411
2600
  try {
2412
2601
  await connection.connect();
@@ -2587,7 +2776,7 @@ var ScenarioRunner = class {
2587
2776
  required: minPass,
2588
2777
  attempts,
2589
2778
  tests: this.results.tests(attempts, minPass),
2590
- agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
2779
+ agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
2591
2780
  evaluatorUsage: this.tokens.evaluator(attempts)
2592
2781
  });
2593
2782
  }
@@ -2749,7 +2938,9 @@ var ConsoleReporter = class ConsoleReporter {
2749
2938
  const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
2750
2939
  if (!this.verbose) return line;
2751
2940
  if (result.attempts.length === 0) return `${line}\n`;
2752
- const lines = [""];
2941
+ const websocket = spec.websocket;
2942
+ const subprotocols = websocket?.connection.subprotocols.length ?? 0;
2943
+ const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
2753
2944
  for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
2754
2945
  lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
2755
2946
  lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
@@ -2928,6 +3119,7 @@ var JsonReporter = class JsonReporter {
2928
3119
  const { websocket, execution, evaluator, report } = config;
2929
3120
  return {
2930
3121
  path: this.path(config.configPath),
3122
+ parametersPath: config.parametersPath === "" ? void 0 : this.path(config.parametersPath),
2931
3123
  websocket: {
2932
3124
  endpoint: websocket.endpoint,
2933
3125
  subprotocols: [...websocket.connection.subprotocols],
@@ -2960,11 +3152,16 @@ var JsonReporter = class JsonReporter {
2960
3152
  };
2961
3153
  }
2962
3154
  scenario(result, config) {
2963
- const capability = config.websocket.protocol.capabilities.tokenUsage;
3155
+ const websocket = result.spec.websocket ?? config.websocket;
3156
+ const capability = websocket.protocol.capabilities.tokenUsage;
2964
3157
  return {
2965
3158
  name: result.spec.name,
2966
3159
  file: result.spec.location.file,
2967
3160
  status: result.status,
3161
+ connection: {
3162
+ endpoint: websocket.endpoint,
3163
+ subprotocols: websocket.connection.subprotocols.length
3164
+ },
2968
3165
  ...JsonReporter.counts(result),
2969
3166
  agentUsage: JsonReporter.usage(result.agentUsage),
2970
3167
  evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
@@ -3125,6 +3322,11 @@ var conversation_schema_default = {
3125
3322
  "version": { "const": 1 },
3126
3323
  "name": { "$ref": "#/definitions/text" },
3127
3324
  "description": { "type": "string" },
3325
+ "variables": {
3326
+ "type": "object",
3327
+ "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
3328
+ "additionalProperties": { "type": "string" }
3329
+ },
3128
3330
  "turns": {
3129
3331
  "type": "array",
3130
3332
  "minItems": 1,
@@ -3137,8 +3339,8 @@ var conversation_schema_default = {
3137
3339
  "properties": { "conversation": {
3138
3340
  "type": "object",
3139
3341
  "additionalProperties": false,
3140
- "required": ["criteria"],
3141
- "properties": { "criteria": { "$ref": "#/definitions/text" } }
3342
+ "required": ["should"],
3343
+ "properties": { "should": { "$ref": "#/definitions/text" } }
3142
3344
  } }
3143
3345
  }
3144
3346
  },
@@ -3181,12 +3383,12 @@ var conversation_schema_default = {
3181
3383
  }
3182
3384
  },
3183
3385
  "response": {
3184
- "description": "A string is shorthand for criteria.",
3386
+ "description": "A string is shorthand for should.",
3185
3387
  "type": ["string", "object"],
3186
3388
  "pattern": "\\S",
3187
3389
  "additionalProperties": false,
3188
3390
  "properties": {
3189
- "criteria": { "$ref": "#/definitions/text" },
3391
+ "should": { "$ref": "#/definitions/text" },
3190
3392
  "contains": { "$ref": "#/definitions/textList" },
3191
3393
  "notContains": { "$ref": "#/definitions/textList" }
3192
3394
  }
@@ -3218,7 +3420,7 @@ var conversation_schema_default = {
3218
3420
  };
3219
3421
  // src/scenarios/conversationValidator.ts
3220
3422
  const RESPONSE_KEYS = [
3221
- "criteria",
3423
+ "should",
3222
3424
  "contains",
3223
3425
  "notContains"
3224
3426
  ];
@@ -3241,7 +3443,7 @@ var ConversationValidator = class ConversationValidator {
3241
3443
  validateResponse(response, path, document, issues) {
3242
3444
  if (!RESPONSE_KEYS.some((key) => key in response)) {
3243
3445
  issues.push(new ValidationIssue({
3244
- message: "response must have at least one of: criteria, contains, notContains",
3446
+ message: "response must have at least one of: should, contains, notContains",
3245
3447
  location: document.locate(path)
3246
3448
  }));
3247
3449
  return;
@@ -3284,30 +3486,66 @@ var ConversationValidator = class ConversationValidator {
3284
3486
  };
3285
3487
  // src/scenarios/scenarioLoader.ts
3286
3488
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
3287
- var ScenarioLoader = class {
3489
+ var ScenarioLoader = class ScenarioLoader {
3288
3490
  cwd;
3289
3491
  documentLoader = new YamlDocumentLoader();
3290
3492
  schemaValidator = new SchemaValidator(conversation_schema_default);
3291
3493
  conversationValidator = new ConversationValidator();
3494
+ variablesReader = new VariablesReader();
3495
+ webSocketResolver = new WebSocketResolver();
3292
3496
  environmentResolver;
3293
- constructor(environment = process.env, cwd = process.cwd()) {
3497
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
3294
3498
  this.cwd = cwd;
3295
- this.environmentResolver = new EnvironmentResolver(environment);
3499
+ this.environmentResolver = new EnvironmentResolver(environment, parameters);
3296
3500
  }
3297
3501
  load(target, config) {
3298
3502
  const files = this.discover(resolve(this.cwd, target), config);
3503
+ const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.cwd, config.configPath)) : void 0;
3299
3504
  const issues = [];
3505
+ const connectionIssues = [];
3300
3506
  const specs = [];
3301
3507
  for (const file of files) {
3302
3508
  const document = this.documentLoader.load(file, relative(this.cwd, file));
3303
3509
  const fileIssues = [...document.errors];
3304
3510
  const secrets = [];
3305
- const value = fileIssues.length === 0 ? this.environmentResolver.resolve(document.value, document, fileIssues, secrets) : void 0;
3511
+ const used = /* @__PURE__ */ new Set();
3512
+ const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
3513
+ const resolver = this.environmentResolver.withLevels([variables, config.variables]);
3514
+ let value;
3515
+ if (fileIssues.length === 0) {
3516
+ const { variables: raw, ...rest } = document.value;
3517
+ value = {
3518
+ ...resolver.resolve(rest, document, fileIssues, secrets, { used }),
3519
+ ...raw === void 0 ? {} : { variables }
3520
+ };
3521
+ }
3522
+ let websocket;
3523
+ const websocketIssues = [];
3524
+ if (fileIssues.length === 0 && configDocument !== void 0) {
3525
+ const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
3526
+ if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
3527
+ connectionIssues.push({
3528
+ file: document,
3529
+ issues: websocketIssues
3530
+ });
3531
+ if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
3532
+ }
3306
3533
  if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
3307
3534
  if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
3308
- if (fileIssues.length === 0) specs.push(this.build(value, document, secrets));
3535
+ if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
3536
+ if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
3309
3537
  else issues.push(...fileIssues);
3310
3538
  }
3539
+ const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
3540
+ if (shared.length > 0) throw new EvalError({
3541
+ category: ErrorCategory.Configuration,
3542
+ message: "invalid configuration",
3543
+ issues: shared
3544
+ });
3545
+ for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
3546
+ message: issue.toString(),
3547
+ location: ScenarioLoader.variablesLocation(entry.file)
3548
+ }));
3311
3549
  if (issues.length > 0) throw new EvalError({
3312
3550
  category: ErrorCategory.Specification,
3313
3551
  message: "invalid scenario specification",
@@ -3315,6 +3553,23 @@ var ScenarioLoader = class {
3315
3553
  });
3316
3554
  return specs;
3317
3555
  }
3556
+ static shared(lists) {
3557
+ if (lists.length === 0) return [];
3558
+ const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
3559
+ return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
3560
+ }
3561
+ static variablesLocation(document) {
3562
+ return new SourceLocation({
3563
+ ...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
3564
+ path: ""
3565
+ });
3566
+ }
3567
+ static unusedVariables(variables, used, document) {
3568
+ return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
3569
+ message: `variable '${name}' is not used by the scenario or the websocket configuration`,
3570
+ location: document.locateKey(["variables"], name)
3571
+ }));
3572
+ }
3318
3573
  discover(target, config) {
3319
3574
  if (!existsSync(target)) throw new EvalError({
3320
3575
  category: ErrorCategory.Specification,
@@ -3339,7 +3594,7 @@ var ScenarioLoader = class {
3339
3594
  }
3340
3595
  return files;
3341
3596
  }
3342
- build(value, document, secrets) {
3597
+ build(value, document, websocket, secrets) {
3343
3598
  const scenario = value;
3344
3599
  const turns = scenario.turns;
3345
3600
  const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
@@ -3349,10 +3604,11 @@ var ScenarioLoader = class {
3349
3604
  description: typeof scenario.description === "string" ? scenario.description : "",
3350
3605
  turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
3351
3606
  conversation: conversation ? new ConversationExpectation({
3352
- criteria: conversation.criteria,
3607
+ criteria: conversation.should,
3353
3608
  location: document.locate(["expect", "conversation"])
3354
3609
  }) : void 0,
3355
3610
  location: document.locate([]),
3611
+ websocket,
3356
3612
  secrets
3357
3613
  });
3358
3614
  }
@@ -3378,7 +3634,7 @@ var ScenarioLoader = class {
3378
3634
  });
3379
3635
  const fields = response;
3380
3636
  return new ResponseExpectation({
3381
- criteria: fields.criteria,
3637
+ criteria: fields.should,
3382
3638
  contains: fields.contains ?? [],
3383
3639
  notContains: fields.notContains ?? [],
3384
3640
  location
@@ -3427,15 +3683,20 @@ var RunCommand = class RunCommand {
3427
3683
  this.signals = signals;
3428
3684
  this.colour = colour;
3429
3685
  }
3430
- async execute(target, config, execution = new ExecutionOverride(), report = new ReportOverride()) {
3686
+ async execute(target, config, execution = new ExecutionOverride(), report = new ReportOverride(), parameters = "") {
3431
3687
  let evalConfig;
3432
3688
  let specs;
3433
3689
  let evaluator;
3434
3690
  let redactor = new SecretRedactor([]);
3435
3691
  try {
3436
3692
  const configPath = new ConfigLocator(this.cwd).locate(target, config);
3437
- evalConfig = new EvalConfigLoader(this.environment, this.cwd).load(configPath, execution, report);
3438
- specs = new ScenarioLoader(this.environment, this.cwd).load(target, evalConfig);
3693
+ const parameterFile = parameters === "" ? {
3694
+ path: "",
3695
+ parameters: {}
3696
+ } : new ParametersLoader(this.cwd).load(parameters);
3697
+ evalConfig = new EvalConfigLoader(this.environment, this.cwd, parameterFile.parameters).load(configPath, execution, report);
3698
+ evalConfig.parametersPath = parameterFile.path;
3699
+ specs = new ScenarioLoader(this.environment, this.cwd, parameterFile.parameters).load(target, evalConfig);
3439
3700
  redactor = new SecretRedactor([...evalConfig.secrets, ...specs.flatMap((spec) => spec.secrets)]);
3440
3701
  evaluator = RunCommand.needsEvaluator(specs) ? await this.createEvaluator(evalConfig) : new UnavailableEvaluator();
3441
3702
  } catch (error) {
@@ -3524,11 +3785,12 @@ var ValidateCommand = class {
3524
3785
  this.environment = environment;
3525
3786
  this.cwd = cwd;
3526
3787
  }
3527
- execute(target, config) {
3788
+ execute(target, config, parameters = "") {
3528
3789
  try {
3529
3790
  const configPath = new ConfigLocator(this.cwd).locate(target, config);
3530
- const evalConfig = new EvalConfigLoader(this.environment, this.cwd).load(configPath, new ExecutionOverride());
3531
- const specs = new ScenarioLoader(this.environment, this.cwd).load(target, evalConfig);
3791
+ const values = parameters === "" ? {} : new ParametersLoader(this.cwd).load(parameters).parameters;
3792
+ const evalConfig = new EvalConfigLoader(this.environment, this.cwd, values).load(configPath, new ExecutionOverride());
3793
+ const specs = new ScenarioLoader(this.environment, this.cwd, values).load(target, evalConfig);
3532
3794
  this.out(`Valid: ${specs.length} scenario${specs.length === 1 ? "" : "s"}\n`);
3533
3795
  return 0;
3534
3796
  } catch (error) {
@@ -3545,10 +3807,10 @@ function createProgram(setExitCode, out, err, colour = false) {
3545
3807
  writeOut: out,
3546
3808
  writeErr: err
3547
3809
  }).exitOverride();
3548
- program.command("validate").description("Parse, interpolate and validate configuration and scenarios with no network calls").argument("<target>", "scenario file or directory").option("--config <path>", "configuration file (default: search upward for eval-config.yml)").action((target, options) => {
3549
- setExitCode(new ValidateCommand(out, err).execute(target, options.config ?? ""));
3810
+ program.command("validate").description("Parse, interpolate and validate configuration and scenarios with no network calls").argument("<target>", "scenario file or directory").option("--config <path>", "configuration file (default: search upward for eval-config.yml)").option("--parameters <path>", "JSON file whose string values resolve ${name} references before the environment").action((target, options) => {
3811
+ setExitCode(new ValidateCommand(out, err).execute(target, options.config ?? "", options.parameters ?? ""));
3550
3812
  });
3551
- program.command("run").description("Run every scenario against the agent, repeat times each, and report the results").argument("<target>", "scenario file or directory").option("--config <path>", "configuration file (default: search upward for eval-config.yml)").option("--repeat <n>", "attempts per scenario (overrides execution.repeat)").option("--min-pass <n>", "passing attempts a scenario needs (overrides execution.minPass)").option("--quick", "shorthand for --repeat 1 --min-pass 1; wins over both").option("--concurrency <n>", "scenarios run at once (overrides execution.scenarioConcurrency)").option("--report <formats>", "comma-separated console, json (overrides report.formats)").option("--verbose", "include every attempt's transcript in the console report").action(async (target, options) => {
3813
+ program.command("run").description("Run every scenario against the agent, repeat times each, and report the results").argument("<target>", "scenario file or directory").option("--config <path>", "configuration file (default: search upward for eval-config.yml)").option("--parameters <path>", "JSON file whose string values resolve ${name} references before the environment").option("--repeat <n>", "attempts per scenario (overrides execution.repeat)").option("--min-pass <n>", "passing attempts a scenario needs (overrides execution.minPass)").option("--quick", "shorthand for --repeat 1 --min-pass 1; wins over both").option("--concurrency <n>", "scenarios run at once (overrides execution.scenarioConcurrency)").option("--report <formats>", "comma-separated console, json (overrides report.formats)").option("--verbose", "include every attempt's transcript in the console report").action(async (target, options) => {
3552
3814
  let execution;
3553
3815
  let report;
3554
3816
  try {
@@ -3560,7 +3822,7 @@ function createProgram(setExitCode, out, err, colour = false) {
3560
3822
  setExitCode(1);
3561
3823
  return;
3562
3824
  }
3563
- setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour).execute(target, options.config ?? "", execution, report));
3825
+ setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour).execute(target, options.config ?? "", execution, report, options.parameters ?? ""));
3564
3826
  });
3565
3827
  return program;
3566
3828
  }