@mappies/miniwat-eval 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -11
- package/dist/index.cjs +389 -173
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +58 -48
- package/dist/index.d.ts +58 -48
- package/dist/index.js +389 -173
- package/dist/index.js.map +1 -1
- package/dist/main.js +345 -129
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +9 -4
- package/schemas/eval-config.schema.json +5 -0
- package/schemas/report.schema.json +11 -1
package/dist/main.js
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
import { Command, CommanderError } from "commander";
|
|
3
3
|
import { dirname, extname, join, relative, resolve } from "node:path";
|
|
4
4
|
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
5
|
-
import { Ajv } from "ajv";
|
|
6
5
|
import { LineCounter, isMap, isPair, isScalar, isSeq, parseDocument } from "yaml";
|
|
6
|
+
import { Ajv } from "ajv";
|
|
7
7
|
import { Message, ResponseMessage, TokenUsage } from "@mappies/miniwat-agent-core";
|
|
8
8
|
import { Agent, InMemoryAgentStateStore } from "@mappies/miniwat-agent";
|
|
9
9
|
import { randomUUID } from "node:crypto";
|
|
@@ -215,6 +215,11 @@ var eval_config_schema_default = {
|
|
|
215
215
|
],
|
|
216
216
|
properties: {
|
|
217
217
|
"version": { "const": 1 },
|
|
218
|
+
"variables": {
|
|
219
|
+
"type": "object",
|
|
220
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
221
|
+
"additionalProperties": { "type": "string" }
|
|
222
|
+
},
|
|
218
223
|
"websocket": {
|
|
219
224
|
"type": "object",
|
|
220
225
|
"additionalProperties": false,
|
|
@@ -510,6 +515,7 @@ var EvalConfig = class {
|
|
|
510
515
|
version;
|
|
511
516
|
configPath;
|
|
512
517
|
parametersPath;
|
|
518
|
+
variables;
|
|
513
519
|
websocket;
|
|
514
520
|
execution;
|
|
515
521
|
evaluator;
|
|
@@ -519,6 +525,7 @@ var EvalConfig = class {
|
|
|
519
525
|
this.version = data?.version ?? 1;
|
|
520
526
|
this.configPath = data?.configPath ?? "";
|
|
521
527
|
this.parametersPath = data?.parametersPath ?? "";
|
|
528
|
+
this.variables = { ...data?.variables ?? {} };
|
|
522
529
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
523
530
|
this.execution = new ExecutionConfig(data?.execution);
|
|
524
531
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -531,18 +538,80 @@ const ReportFormat = {
|
|
|
531
538
|
Console: "console",
|
|
532
539
|
Json: "json"
|
|
533
540
|
};
|
|
541
|
+
// src/yaml/yamlDocument.ts
|
|
542
|
+
var YamlDocument = class YamlDocument {
|
|
543
|
+
file;
|
|
544
|
+
document;
|
|
545
|
+
lineCounter;
|
|
546
|
+
value;
|
|
547
|
+
errors;
|
|
548
|
+
constructor(file, document, lineCounter) {
|
|
549
|
+
this.file = file;
|
|
550
|
+
this.document = document;
|
|
551
|
+
this.lineCounter = lineCounter;
|
|
552
|
+
this.value = document.toJS();
|
|
553
|
+
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
554
|
+
message: error.message.split("\n")[0],
|
|
555
|
+
location: this.toLocation(error.pos[0], [])
|
|
556
|
+
}));
|
|
557
|
+
}
|
|
558
|
+
locate(path) {
|
|
559
|
+
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
560
|
+
}
|
|
561
|
+
locateKey(path, key) {
|
|
562
|
+
const parent = this.findNode(path);
|
|
563
|
+
if (isMap(parent)) {
|
|
564
|
+
const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
|
|
565
|
+
if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
566
|
+
}
|
|
567
|
+
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
568
|
+
}
|
|
569
|
+
static formatPath(path) {
|
|
570
|
+
return path.map((segment, index) => {
|
|
571
|
+
if (typeof segment === "number") return `[${segment}]`;
|
|
572
|
+
return index === 0 ? segment : `.${segment}`;
|
|
573
|
+
}).join("");
|
|
574
|
+
}
|
|
575
|
+
findNode(path) {
|
|
576
|
+
let node = this.document.contents ?? void 0;
|
|
577
|
+
for (const segment of path) {
|
|
578
|
+
let next;
|
|
579
|
+
if (isMap(node)) {
|
|
580
|
+
const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
|
|
581
|
+
next = pair?.value ?? pair?.key;
|
|
582
|
+
} else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
|
|
583
|
+
if (!next) break;
|
|
584
|
+
node = next;
|
|
585
|
+
}
|
|
586
|
+
return node;
|
|
587
|
+
}
|
|
588
|
+
toLocation(offset, path) {
|
|
589
|
+
const position = this.lineCounter.linePos(offset);
|
|
590
|
+
return new SourceLocation({
|
|
591
|
+
file: this.file,
|
|
592
|
+
line: position.line,
|
|
593
|
+
column: position.col,
|
|
594
|
+
path: YamlDocument.formatPath(path)
|
|
595
|
+
});
|
|
596
|
+
}
|
|
597
|
+
};
|
|
534
598
|
// src/yaml/environmentResolver.ts
|
|
535
599
|
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
|
|
536
600
|
const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
|
|
537
601
|
var EnvironmentResolver = class EnvironmentResolver {
|
|
538
602
|
environment;
|
|
539
603
|
parameters;
|
|
540
|
-
|
|
604
|
+
levels;
|
|
605
|
+
constructor(environment = process.env, parameters = {}, levels = []) {
|
|
541
606
|
this.environment = environment;
|
|
542
607
|
this.parameters = parameters;
|
|
608
|
+
this.levels = levels;
|
|
609
|
+
}
|
|
610
|
+
withLevels(levels) {
|
|
611
|
+
return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
|
|
543
612
|
}
|
|
544
|
-
resolve(value, document, issues, secrets = []) {
|
|
545
|
-
return this.resolveValue(value, [], document, issues, secrets);
|
|
613
|
+
resolve(value, document, issues, secrets = [], options = {}) {
|
|
614
|
+
return this.resolveValue(value, [], document, issues, secrets, options);
|
|
546
615
|
}
|
|
547
616
|
static hasReference(value) {
|
|
548
617
|
return new RegExp(REFERENCE.source).test(value);
|
|
@@ -550,22 +619,38 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
550
619
|
static isSecretName(name) {
|
|
551
620
|
return SECRET_NAME.test(name);
|
|
552
621
|
}
|
|
553
|
-
|
|
622
|
+
lookup(name) {
|
|
623
|
+
for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
|
|
624
|
+
return this.environment[name];
|
|
625
|
+
}
|
|
626
|
+
sources() {
|
|
627
|
+
const sources = [
|
|
628
|
+
...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
|
|
629
|
+
...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
|
|
630
|
+
"environment variable"
|
|
631
|
+
];
|
|
632
|
+
return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
|
|
633
|
+
}
|
|
634
|
+
resolveValue(value, path, document, issues, secrets, options) {
|
|
554
635
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
555
|
-
|
|
636
|
+
options.used?.add(name);
|
|
637
|
+
const resolved = this.lookup(name);
|
|
556
638
|
if (resolved === void 0) {
|
|
557
|
-
|
|
639
|
+
if (options.optional?.(path)) {
|
|
640
|
+
options.unresolved?.add(YamlDocument.formatPath(path));
|
|
641
|
+
return reference;
|
|
642
|
+
}
|
|
558
643
|
issues.push(new ValidationIssue({
|
|
559
|
-
message: `${
|
|
644
|
+
message: `${this.sources()} '${name}' is not set`,
|
|
560
645
|
location: document.locate(path)
|
|
561
646
|
}));
|
|
562
647
|
return reference;
|
|
563
648
|
}
|
|
564
|
-
if (EnvironmentResolver.isSecretName(name) && !secrets.includes(resolved)) secrets.push(resolved);
|
|
649
|
+
if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
|
|
565
650
|
return resolved;
|
|
566
651
|
});
|
|
567
|
-
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
|
|
568
|
-
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
|
|
652
|
+
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
|
|
653
|
+
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
|
|
569
654
|
return value;
|
|
570
655
|
}
|
|
571
656
|
};
|
|
@@ -649,63 +734,6 @@ var SchemaValidator = class SchemaValidator {
|
|
|
649
734
|
return path;
|
|
650
735
|
}
|
|
651
736
|
};
|
|
652
|
-
// src/yaml/yamlDocument.ts
|
|
653
|
-
var YamlDocument = class YamlDocument {
|
|
654
|
-
file;
|
|
655
|
-
document;
|
|
656
|
-
lineCounter;
|
|
657
|
-
value;
|
|
658
|
-
errors;
|
|
659
|
-
constructor(file, document, lineCounter) {
|
|
660
|
-
this.file = file;
|
|
661
|
-
this.document = document;
|
|
662
|
-
this.lineCounter = lineCounter;
|
|
663
|
-
this.value = document.toJS();
|
|
664
|
-
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
665
|
-
message: error.message.split("\n")[0],
|
|
666
|
-
location: this.toLocation(error.pos[0], [])
|
|
667
|
-
}));
|
|
668
|
-
}
|
|
669
|
-
locate(path) {
|
|
670
|
-
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
671
|
-
}
|
|
672
|
-
locateKey(path, key) {
|
|
673
|
-
const parent = this.findNode(path);
|
|
674
|
-
if (isMap(parent)) {
|
|
675
|
-
const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
|
|
676
|
-
if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
677
|
-
}
|
|
678
|
-
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
679
|
-
}
|
|
680
|
-
static formatPath(path) {
|
|
681
|
-
return path.map((segment, index) => {
|
|
682
|
-
if (typeof segment === "number") return `[${segment}]`;
|
|
683
|
-
return index === 0 ? segment : `.${segment}`;
|
|
684
|
-
}).join("");
|
|
685
|
-
}
|
|
686
|
-
findNode(path) {
|
|
687
|
-
let node = this.document.contents ?? void 0;
|
|
688
|
-
for (const segment of path) {
|
|
689
|
-
let next;
|
|
690
|
-
if (isMap(node)) {
|
|
691
|
-
const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
|
|
692
|
-
next = pair?.value ?? pair?.key;
|
|
693
|
-
} else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
|
|
694
|
-
if (!next) break;
|
|
695
|
-
node = next;
|
|
696
|
-
}
|
|
697
|
-
return node;
|
|
698
|
-
}
|
|
699
|
-
toLocation(offset, path) {
|
|
700
|
-
const position = this.lineCounter.linePos(offset);
|
|
701
|
-
return new SourceLocation({
|
|
702
|
-
file: this.file,
|
|
703
|
-
line: position.line,
|
|
704
|
-
column: position.col,
|
|
705
|
-
path: YamlDocument.formatPath(path)
|
|
706
|
-
});
|
|
707
|
-
}
|
|
708
|
-
};
|
|
709
737
|
// src/yaml/yamlDocumentLoader.ts
|
|
710
738
|
var YamlDocumentLoader = class {
|
|
711
739
|
load(filePath, displayPath) {
|
|
@@ -720,6 +748,126 @@ var YamlDocumentLoader = class {
|
|
|
720
748
|
}), lineCounter);
|
|
721
749
|
}
|
|
722
750
|
};
|
|
751
|
+
// src/yaml/variablesReader.ts
|
|
752
|
+
const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
753
|
+
var VariablesReader = class {
|
|
754
|
+
read(document, resolver, issues, secrets) {
|
|
755
|
+
const raw = isRecord(document.value) ? document.value.variables : void 0;
|
|
756
|
+
if (raw === void 0) return {};
|
|
757
|
+
if (!isRecord(raw)) {
|
|
758
|
+
issues.push(new ValidationIssue({
|
|
759
|
+
message: "must be object",
|
|
760
|
+
location: document.locate(["variables"])
|
|
761
|
+
}));
|
|
762
|
+
return {};
|
|
763
|
+
}
|
|
764
|
+
const variables = {};
|
|
765
|
+
for (const [name, value] of Object.entries(raw)) {
|
|
766
|
+
if (!VARIABLE_NAME.test(name)) {
|
|
767
|
+
issues.push(new ValidationIssue({
|
|
768
|
+
message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
769
|
+
location: document.locateKey(["variables"], name)
|
|
770
|
+
}));
|
|
771
|
+
continue;
|
|
772
|
+
}
|
|
773
|
+
if (typeof value !== "string") {
|
|
774
|
+
issues.push(new ValidationIssue({
|
|
775
|
+
message: "must be string",
|
|
776
|
+
location: document.locate(["variables", name])
|
|
777
|
+
}));
|
|
778
|
+
continue;
|
|
779
|
+
}
|
|
780
|
+
const before = issues.length;
|
|
781
|
+
const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
|
|
782
|
+
if (issues.length > before) continue;
|
|
783
|
+
variables[name] = resolved.variables[name];
|
|
784
|
+
if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
|
|
785
|
+
}
|
|
786
|
+
return variables;
|
|
787
|
+
}
|
|
788
|
+
};
|
|
789
|
+
// src/config/webSocketResolver.ts
|
|
790
|
+
const WEBSOCKET_SCHEMA = {
|
|
791
|
+
...eval_config_schema_default,
|
|
792
|
+
required: ["websocket"],
|
|
793
|
+
additionalProperties: true,
|
|
794
|
+
properties: { websocket: eval_config_schema_default.properties.websocket }
|
|
795
|
+
};
|
|
796
|
+
var WebSocketResolver = class WebSocketResolver {
|
|
797
|
+
schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
|
|
798
|
+
resolve(document, resolver, issues, secrets, used, defer) {
|
|
799
|
+
const raw = isRecord(document.value) ? document.value.websocket : void 0;
|
|
800
|
+
const unresolved = /* @__PURE__ */ new Set();
|
|
801
|
+
const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
|
|
802
|
+
const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
|
|
803
|
+
used,
|
|
804
|
+
optional,
|
|
805
|
+
unresolved
|
|
806
|
+
});
|
|
807
|
+
const subprotocols = WebSocketResolver.subprotocols(raw);
|
|
808
|
+
if (subprotocols !== void 0) {
|
|
809
|
+
const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
|
|
810
|
+
const set = !unresolved.has(YamlDocument.formatPath([
|
|
811
|
+
"websocket",
|
|
812
|
+
"connection",
|
|
813
|
+
"subprotocols",
|
|
814
|
+
index
|
|
815
|
+
]));
|
|
816
|
+
const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
|
|
817
|
+
return set && !empty;
|
|
818
|
+
});
|
|
819
|
+
value.websocket.connection.subprotocols = kept;
|
|
820
|
+
}
|
|
821
|
+
const prefix = YamlDocument.formatPath([
|
|
822
|
+
"websocket",
|
|
823
|
+
"connection",
|
|
824
|
+
"subprotocols"
|
|
825
|
+
]);
|
|
826
|
+
return {
|
|
827
|
+
value,
|
|
828
|
+
deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
|
|
829
|
+
};
|
|
830
|
+
}
|
|
831
|
+
validate(value, document) {
|
|
832
|
+
const issues = this.schemaValidator.validate(value, document);
|
|
833
|
+
if (issues.length > 0) return issues;
|
|
834
|
+
return WebSocketResolver.validateInputPath(value, document);
|
|
835
|
+
}
|
|
836
|
+
static validateInputPath(value, document) {
|
|
837
|
+
const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
|
|
838
|
+
if (typeof request.inputPath !== "string") return [];
|
|
839
|
+
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
840
|
+
const last = tokens.pop();
|
|
841
|
+
let parent = request.value;
|
|
842
|
+
for (const token of tokens) parent = WebSocketResolver.child(parent, token);
|
|
843
|
+
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
844
|
+
return [new ValidationIssue({
|
|
845
|
+
message: "must point to a missing or string value whose parent exists in request.value",
|
|
846
|
+
location: document.locate([
|
|
847
|
+
"websocket",
|
|
848
|
+
"protocol",
|
|
849
|
+
"request",
|
|
850
|
+
"inputPath"
|
|
851
|
+
])
|
|
852
|
+
})];
|
|
853
|
+
}
|
|
854
|
+
static section(value, ...path) {
|
|
855
|
+
let current = value;
|
|
856
|
+
for (const key of path) current = WebSocketResolver.child(current, key);
|
|
857
|
+
return isRecord(current) ? current : {};
|
|
858
|
+
}
|
|
859
|
+
static child(value, key) {
|
|
860
|
+
if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
|
|
861
|
+
return isRecord(value) ? value[key] : void 0;
|
|
862
|
+
}
|
|
863
|
+
static isSubprotocol(path) {
|
|
864
|
+
return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
|
|
865
|
+
}
|
|
866
|
+
static subprotocols(websocket) {
|
|
867
|
+
const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
|
|
868
|
+
return Array.isArray(subprotocols) ? subprotocols : void 0;
|
|
869
|
+
}
|
|
870
|
+
};
|
|
723
871
|
// src/config/defaultConfig.ts
|
|
724
872
|
const DEFAULT_CONFIG = new EvalConfig({
|
|
725
873
|
execution: new ExecutionConfig({
|
|
@@ -755,6 +903,8 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
755
903
|
cwd;
|
|
756
904
|
documentLoader = new YamlDocumentLoader();
|
|
757
905
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
906
|
+
variablesReader = new VariablesReader();
|
|
907
|
+
webSocketResolver = new WebSocketResolver();
|
|
758
908
|
environmentResolver;
|
|
759
909
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
760
910
|
this.cwd = cwd;
|
|
@@ -765,19 +915,47 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
765
915
|
const issues = [...document.errors];
|
|
766
916
|
if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
|
|
767
917
|
const named = [];
|
|
768
|
-
const
|
|
769
|
-
|
|
770
|
-
if (issues.length === 0)
|
|
918
|
+
const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
|
|
919
|
+
let value = {};
|
|
920
|
+
if (issues.length === 0) {
|
|
921
|
+
const resolver = this.environmentResolver.withLevels([variables]);
|
|
922
|
+
const { websocket: _websocket, variables: _variables, ...rest } = document.value;
|
|
923
|
+
const resolved = resolver.resolve(rest, document, issues, named);
|
|
924
|
+
const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
|
|
925
|
+
value = {
|
|
926
|
+
...resolved,
|
|
927
|
+
..._variables === void 0 ? {} : { variables },
|
|
928
|
+
..._websocket === void 0 ? {} : websocket.value
|
|
929
|
+
};
|
|
930
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
|
|
931
|
+
if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
|
|
932
|
+
}
|
|
771
933
|
if (issues.length > 0) throw new EvalError({
|
|
772
934
|
category: ErrorCategory.Configuration,
|
|
773
935
|
message: "invalid configuration",
|
|
774
936
|
issues
|
|
775
937
|
});
|
|
776
|
-
const config = this.build(value, configPath, named);
|
|
938
|
+
const config = this.build(value, configPath, variables, named);
|
|
777
939
|
this.applyRepetition(config, override, value, document);
|
|
778
940
|
EvalConfigLoader.applyReport(config, report);
|
|
779
941
|
return config;
|
|
780
942
|
}
|
|
943
|
+
static buildWebSocket(value) {
|
|
944
|
+
const section = (...path) => WebSocketResolver.section(value, ...path);
|
|
945
|
+
const connection = section("websocket", "connection");
|
|
946
|
+
const request = section("websocket", "protocol", "request");
|
|
947
|
+
return new WebSocketConfig({
|
|
948
|
+
endpoint: WebSocketResolver.section(value, "websocket").endpoint,
|
|
949
|
+
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
950
|
+
protocol: new ProtocolConfig({
|
|
951
|
+
request: new RequestConfig({
|
|
952
|
+
...request,
|
|
953
|
+
format: request.format ?? RequestFormat.Json
|
|
954
|
+
}),
|
|
955
|
+
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
956
|
+
})
|
|
957
|
+
});
|
|
958
|
+
}
|
|
781
959
|
static applyReport(config, report) {
|
|
782
960
|
if (report.formats === void 0) return;
|
|
783
961
|
const known = Object.values(ReportFormat);
|
|
@@ -799,45 +977,17 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
799
977
|
location: document.locate(["evaluator", "apiKey"])
|
|
800
978
|
})];
|
|
801
979
|
}
|
|
802
|
-
|
|
803
|
-
const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
|
|
804
|
-
if (typeof request.inputPath !== "string") return [];
|
|
805
|
-
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
806
|
-
const last = tokens.pop();
|
|
807
|
-
let parent = request.value;
|
|
808
|
-
for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
|
|
809
|
-
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
810
|
-
return [new ValidationIssue({
|
|
811
|
-
message: "must point to a missing or string value whose parent exists in request.value",
|
|
812
|
-
location: document.locate([
|
|
813
|
-
"websocket",
|
|
814
|
-
"protocol",
|
|
815
|
-
"request",
|
|
816
|
-
"inputPath"
|
|
817
|
-
])
|
|
818
|
-
})];
|
|
819
|
-
}
|
|
820
|
-
build(value, configPath, named) {
|
|
980
|
+
build(value, configPath, variables, named) {
|
|
821
981
|
const section = (...path) => EvalConfigLoader.section(value, ...path);
|
|
822
982
|
const configDirectory = dirname(configPath);
|
|
823
|
-
const
|
|
824
|
-
const request = section("websocket", "protocol", "request");
|
|
983
|
+
const websocket = EvalConfigLoader.buildWebSocket(value);
|
|
825
984
|
const evaluator = section("evaluator");
|
|
826
985
|
const report = section("report");
|
|
827
986
|
return new EvalConfig({
|
|
828
987
|
version: value.version,
|
|
829
988
|
configPath,
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
833
|
-
protocol: new ProtocolConfig({
|
|
834
|
-
request: new RequestConfig({
|
|
835
|
-
...request,
|
|
836
|
-
format: request.format ?? RequestFormat.Json
|
|
837
|
-
}),
|
|
838
|
-
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
839
|
-
})
|
|
840
|
-
}),
|
|
989
|
+
variables,
|
|
990
|
+
websocket,
|
|
841
991
|
execution: new ExecutionConfig({
|
|
842
992
|
...DEFAULT_CONFIG.execution,
|
|
843
993
|
...section("execution"),
|
|
@@ -858,7 +1008,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
858
1008
|
directory: resolve(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
|
|
859
1009
|
formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
|
|
860
1010
|
}),
|
|
861
|
-
secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols
|
|
1011
|
+
secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
|
|
862
1012
|
});
|
|
863
1013
|
}
|
|
864
1014
|
applyRepetition(config, override, value, document) {
|
|
@@ -911,13 +1061,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
911
1061
|
])];
|
|
912
1062
|
}
|
|
913
1063
|
static section(value, ...path) {
|
|
914
|
-
|
|
915
|
-
for (const key of path) current = EvalConfigLoader.child(current, key);
|
|
916
|
-
return isRecord(current) ? current : {};
|
|
917
|
-
}
|
|
918
|
-
static child(value, key) {
|
|
919
|
-
if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
|
|
920
|
-
return isRecord(value) ? value[key] : void 0;
|
|
1064
|
+
return WebSocketResolver.section(value, ...path);
|
|
921
1065
|
}
|
|
922
1066
|
};
|
|
923
1067
|
// src/interfaces/runStatus.ts
|
|
@@ -1113,6 +1257,7 @@ var ConversationSpec = class {
|
|
|
1113
1257
|
turns;
|
|
1114
1258
|
conversation;
|
|
1115
1259
|
location;
|
|
1260
|
+
websocket;
|
|
1116
1261
|
secrets;
|
|
1117
1262
|
constructor(data) {
|
|
1118
1263
|
this.version = data?.version ?? 1;
|
|
@@ -1121,6 +1266,7 @@ var ConversationSpec = class {
|
|
|
1121
1266
|
this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
|
|
1122
1267
|
this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
|
|
1123
1268
|
this.location = new SourceLocation(data?.location);
|
|
1269
|
+
this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
|
|
1124
1270
|
this.secrets = [...data?.secrets ?? []];
|
|
1125
1271
|
}
|
|
1126
1272
|
};
|
|
@@ -2420,7 +2566,11 @@ var ConversationRunner = class {
|
|
|
2420
2566
|
async run(spec, signal = NEVER_ABORTED) {
|
|
2421
2567
|
const startedAt = Date.now();
|
|
2422
2568
|
const outcomes = [];
|
|
2423
|
-
const
|
|
2569
|
+
const config = new EvalConfig({
|
|
2570
|
+
...this.config,
|
|
2571
|
+
websocket: spec.websocket ?? this.config.websocket
|
|
2572
|
+
});
|
|
2573
|
+
const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
|
|
2424
2574
|
const history = [];
|
|
2425
2575
|
const evaluate = this.semanticEvaluator(signal);
|
|
2426
2576
|
const inline = !this.config.execution.continueOnFailure;
|
|
@@ -2445,7 +2595,7 @@ var ConversationRunner = class {
|
|
|
2445
2595
|
try {
|
|
2446
2596
|
const steps = (async () => {
|
|
2447
2597
|
for (let attempt = 0;; attempt++) {
|
|
2448
|
-
connection = this.createConnection(
|
|
2598
|
+
connection = this.createConnection(config);
|
|
2449
2599
|
connectionAttempts++;
|
|
2450
2600
|
try {
|
|
2451
2601
|
await connection.connect();
|
|
@@ -2626,7 +2776,7 @@ var ScenarioRunner = class {
|
|
|
2626
2776
|
required: minPass,
|
|
2627
2777
|
attempts,
|
|
2628
2778
|
tests: this.results.tests(attempts, minPass),
|
|
2629
|
-
agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
|
|
2779
|
+
agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
|
|
2630
2780
|
evaluatorUsage: this.tokens.evaluator(attempts)
|
|
2631
2781
|
});
|
|
2632
2782
|
}
|
|
@@ -2788,7 +2938,9 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
2788
2938
|
const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
|
|
2789
2939
|
if (!this.verbose) return line;
|
|
2790
2940
|
if (result.attempts.length === 0) return `${line}\n`;
|
|
2791
|
-
const
|
|
2941
|
+
const websocket = spec.websocket;
|
|
2942
|
+
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
2943
|
+
const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
|
|
2792
2944
|
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
2793
2945
|
lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
|
|
2794
2946
|
lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
|
|
@@ -3000,11 +3152,16 @@ var JsonReporter = class JsonReporter {
|
|
|
3000
3152
|
};
|
|
3001
3153
|
}
|
|
3002
3154
|
scenario(result, config) {
|
|
3003
|
-
const
|
|
3155
|
+
const websocket = result.spec.websocket ?? config.websocket;
|
|
3156
|
+
const capability = websocket.protocol.capabilities.tokenUsage;
|
|
3004
3157
|
return {
|
|
3005
3158
|
name: result.spec.name,
|
|
3006
3159
|
file: result.spec.location.file,
|
|
3007
3160
|
status: result.status,
|
|
3161
|
+
connection: {
|
|
3162
|
+
endpoint: websocket.endpoint,
|
|
3163
|
+
subprotocols: websocket.connection.subprotocols.length
|
|
3164
|
+
},
|
|
3008
3165
|
...JsonReporter.counts(result),
|
|
3009
3166
|
agentUsage: JsonReporter.usage(result.agentUsage),
|
|
3010
3167
|
evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
|
|
@@ -3165,6 +3322,11 @@ var conversation_schema_default = {
|
|
|
3165
3322
|
"version": { "const": 1 },
|
|
3166
3323
|
"name": { "$ref": "#/definitions/text" },
|
|
3167
3324
|
"description": { "type": "string" },
|
|
3325
|
+
"variables": {
|
|
3326
|
+
"type": "object",
|
|
3327
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
3328
|
+
"additionalProperties": { "type": "string" }
|
|
3329
|
+
},
|
|
3168
3330
|
"turns": {
|
|
3169
3331
|
"type": "array",
|
|
3170
3332
|
"minItems": 1,
|
|
@@ -3177,8 +3339,8 @@ var conversation_schema_default = {
|
|
|
3177
3339
|
"properties": { "conversation": {
|
|
3178
3340
|
"type": "object",
|
|
3179
3341
|
"additionalProperties": false,
|
|
3180
|
-
"required": ["
|
|
3181
|
-
"properties": { "
|
|
3342
|
+
"required": ["should"],
|
|
3343
|
+
"properties": { "should": { "$ref": "#/definitions/text" } }
|
|
3182
3344
|
} }
|
|
3183
3345
|
}
|
|
3184
3346
|
},
|
|
@@ -3221,12 +3383,12 @@ var conversation_schema_default = {
|
|
|
3221
3383
|
}
|
|
3222
3384
|
},
|
|
3223
3385
|
"response": {
|
|
3224
|
-
"description": "A string is shorthand for
|
|
3386
|
+
"description": "A string is shorthand for should.",
|
|
3225
3387
|
"type": ["string", "object"],
|
|
3226
3388
|
"pattern": "\\S",
|
|
3227
3389
|
"additionalProperties": false,
|
|
3228
3390
|
"properties": {
|
|
3229
|
-
"
|
|
3391
|
+
"should": { "$ref": "#/definitions/text" },
|
|
3230
3392
|
"contains": { "$ref": "#/definitions/textList" },
|
|
3231
3393
|
"notContains": { "$ref": "#/definitions/textList" }
|
|
3232
3394
|
}
|
|
@@ -3258,7 +3420,7 @@ var conversation_schema_default = {
|
|
|
3258
3420
|
};
|
|
3259
3421
|
// src/scenarios/conversationValidator.ts
|
|
3260
3422
|
const RESPONSE_KEYS = [
|
|
3261
|
-
"
|
|
3423
|
+
"should",
|
|
3262
3424
|
"contains",
|
|
3263
3425
|
"notContains"
|
|
3264
3426
|
];
|
|
@@ -3281,7 +3443,7 @@ var ConversationValidator = class ConversationValidator {
|
|
|
3281
3443
|
validateResponse(response, path, document, issues) {
|
|
3282
3444
|
if (!RESPONSE_KEYS.some((key) => key in response)) {
|
|
3283
3445
|
issues.push(new ValidationIssue({
|
|
3284
|
-
message: "response must have at least one of:
|
|
3446
|
+
message: "response must have at least one of: should, contains, notContains",
|
|
3285
3447
|
location: document.locate(path)
|
|
3286
3448
|
}));
|
|
3287
3449
|
return;
|
|
@@ -3324,11 +3486,13 @@ var ConversationValidator = class ConversationValidator {
|
|
|
3324
3486
|
};
|
|
3325
3487
|
// src/scenarios/scenarioLoader.ts
|
|
3326
3488
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
3327
|
-
var ScenarioLoader = class {
|
|
3489
|
+
var ScenarioLoader = class ScenarioLoader {
|
|
3328
3490
|
cwd;
|
|
3329
3491
|
documentLoader = new YamlDocumentLoader();
|
|
3330
3492
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
3331
3493
|
conversationValidator = new ConversationValidator();
|
|
3494
|
+
variablesReader = new VariablesReader();
|
|
3495
|
+
webSocketResolver = new WebSocketResolver();
|
|
3332
3496
|
environmentResolver;
|
|
3333
3497
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
3334
3498
|
this.cwd = cwd;
|
|
@@ -3336,18 +3500,52 @@ var ScenarioLoader = class {
|
|
|
3336
3500
|
}
|
|
3337
3501
|
load(target, config) {
|
|
3338
3502
|
const files = this.discover(resolve(this.cwd, target), config);
|
|
3503
|
+
const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.cwd, config.configPath)) : void 0;
|
|
3339
3504
|
const issues = [];
|
|
3505
|
+
const connectionIssues = [];
|
|
3340
3506
|
const specs = [];
|
|
3341
3507
|
for (const file of files) {
|
|
3342
3508
|
const document = this.documentLoader.load(file, relative(this.cwd, file));
|
|
3343
3509
|
const fileIssues = [...document.errors];
|
|
3344
3510
|
const secrets = [];
|
|
3345
|
-
const
|
|
3511
|
+
const used = /* @__PURE__ */ new Set();
|
|
3512
|
+
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
|
|
3513
|
+
const resolver = this.environmentResolver.withLevels([variables, config.variables]);
|
|
3514
|
+
let value;
|
|
3515
|
+
if (fileIssues.length === 0) {
|
|
3516
|
+
const { variables: raw, ...rest } = document.value;
|
|
3517
|
+
value = {
|
|
3518
|
+
...resolver.resolve(rest, document, fileIssues, secrets, { used }),
|
|
3519
|
+
...raw === void 0 ? {} : { variables }
|
|
3520
|
+
};
|
|
3521
|
+
}
|
|
3522
|
+
let websocket;
|
|
3523
|
+
const websocketIssues = [];
|
|
3524
|
+
if (fileIssues.length === 0 && configDocument !== void 0) {
|
|
3525
|
+
const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
|
|
3526
|
+
if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
|
|
3527
|
+
connectionIssues.push({
|
|
3528
|
+
file: document,
|
|
3529
|
+
issues: websocketIssues
|
|
3530
|
+
});
|
|
3531
|
+
if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
|
|
3532
|
+
}
|
|
3346
3533
|
if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
|
|
3347
3534
|
if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
|
|
3348
|
-
if (fileIssues.length === 0)
|
|
3535
|
+
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
|
|
3536
|
+
if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
3349
3537
|
else issues.push(...fileIssues);
|
|
3350
3538
|
}
|
|
3539
|
+
const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
|
|
3540
|
+
if (shared.length > 0) throw new EvalError({
|
|
3541
|
+
category: ErrorCategory.Configuration,
|
|
3542
|
+
message: "invalid configuration",
|
|
3543
|
+
issues: shared
|
|
3544
|
+
});
|
|
3545
|
+
for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
|
|
3546
|
+
message: issue.toString(),
|
|
3547
|
+
location: ScenarioLoader.variablesLocation(entry.file)
|
|
3548
|
+
}));
|
|
3351
3549
|
if (issues.length > 0) throw new EvalError({
|
|
3352
3550
|
category: ErrorCategory.Specification,
|
|
3353
3551
|
message: "invalid scenario specification",
|
|
@@ -3355,6 +3553,23 @@ var ScenarioLoader = class {
|
|
|
3355
3553
|
});
|
|
3356
3554
|
return specs;
|
|
3357
3555
|
}
|
|
3556
|
+
static shared(lists) {
|
|
3557
|
+
if (lists.length === 0) return [];
|
|
3558
|
+
const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
|
|
3559
|
+
return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
|
|
3560
|
+
}
|
|
3561
|
+
static variablesLocation(document) {
|
|
3562
|
+
return new SourceLocation({
|
|
3563
|
+
...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
|
|
3564
|
+
path: ""
|
|
3565
|
+
});
|
|
3566
|
+
}
|
|
3567
|
+
static unusedVariables(variables, used, document) {
|
|
3568
|
+
return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
|
|
3569
|
+
message: `variable '${name}' is not used by the scenario or the websocket configuration`,
|
|
3570
|
+
location: document.locateKey(["variables"], name)
|
|
3571
|
+
}));
|
|
3572
|
+
}
|
|
3358
3573
|
discover(target, config) {
|
|
3359
3574
|
if (!existsSync(target)) throw new EvalError({
|
|
3360
3575
|
category: ErrorCategory.Specification,
|
|
@@ -3379,7 +3594,7 @@ var ScenarioLoader = class {
|
|
|
3379
3594
|
}
|
|
3380
3595
|
return files;
|
|
3381
3596
|
}
|
|
3382
|
-
build(value, document, secrets) {
|
|
3597
|
+
build(value, document, websocket, secrets) {
|
|
3383
3598
|
const scenario = value;
|
|
3384
3599
|
const turns = scenario.turns;
|
|
3385
3600
|
const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
|
|
@@ -3389,10 +3604,11 @@ var ScenarioLoader = class {
|
|
|
3389
3604
|
description: typeof scenario.description === "string" ? scenario.description : "",
|
|
3390
3605
|
turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
|
|
3391
3606
|
conversation: conversation ? new ConversationExpectation({
|
|
3392
|
-
criteria: conversation.
|
|
3607
|
+
criteria: conversation.should,
|
|
3393
3608
|
location: document.locate(["expect", "conversation"])
|
|
3394
3609
|
}) : void 0,
|
|
3395
3610
|
location: document.locate([]),
|
|
3611
|
+
websocket,
|
|
3396
3612
|
secrets
|
|
3397
3613
|
});
|
|
3398
3614
|
}
|
|
@@ -3418,7 +3634,7 @@ var ScenarioLoader = class {
|
|
|
3418
3634
|
});
|
|
3419
3635
|
const fields = response;
|
|
3420
3636
|
return new ResponseExpectation({
|
|
3421
|
-
criteria: fields.
|
|
3637
|
+
criteria: fields.should,
|
|
3422
3638
|
contains: fields.contains ?? [],
|
|
3423
3639
|
notContains: fields.notContains ?? [],
|
|
3424
3640
|
location
|