@mappies/miniwat-eval 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -11
- package/dist/index.cjs +743 -178
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +65 -48
- package/dist/index.d.ts +66 -48
- package/dist/index.js +743 -178
- package/dist/index.js.map +1 -1
- package/dist/main.js +699 -134
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +43 -4
- package/schemas/eval-config.schema.json +10 -0
- package/schemas/report.schema.json +12 -1
package/dist/main.js
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
import { Command, CommanderError } from "commander";
|
|
3
3
|
import { dirname, extname, join, relative, resolve } from "node:path";
|
|
4
4
|
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
5
|
-
import { Ajv } from "ajv";
|
|
6
5
|
import { LineCounter, isMap, isPair, isScalar, isSeq, parseDocument } from "yaml";
|
|
6
|
+
import { Ajv } from "ajv";
|
|
7
7
|
import { Message, ResponseMessage, TokenUsage } from "@mappies/miniwat-agent-core";
|
|
8
8
|
import { Agent, InMemoryAgentStateStore } from "@mappies/miniwat-agent";
|
|
9
9
|
import { randomUUID } from "node:crypto";
|
|
@@ -215,6 +215,14 @@ var eval_config_schema_default = {
|
|
|
215
215
|
],
|
|
216
216
|
properties: {
|
|
217
217
|
"version": { "const": 1 },
|
|
218
|
+
"variables": {
|
|
219
|
+
"type": "object",
|
|
220
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
221
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
222
|
+
"type": "array",
|
|
223
|
+
"items": { "type": "string" }
|
|
224
|
+
}] }
|
|
225
|
+
},
|
|
218
226
|
"websocket": {
|
|
219
227
|
"type": "object",
|
|
220
228
|
"additionalProperties": false,
|
|
@@ -510,6 +518,7 @@ var EvalConfig = class {
|
|
|
510
518
|
version;
|
|
511
519
|
configPath;
|
|
512
520
|
parametersPath;
|
|
521
|
+
variables;
|
|
513
522
|
websocket;
|
|
514
523
|
execution;
|
|
515
524
|
evaluator;
|
|
@@ -519,6 +528,7 @@ var EvalConfig = class {
|
|
|
519
528
|
this.version = data?.version ?? 1;
|
|
520
529
|
this.configPath = data?.configPath ?? "";
|
|
521
530
|
this.parametersPath = data?.parametersPath ?? "";
|
|
531
|
+
this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
|
|
522
532
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
523
533
|
this.execution = new ExecutionConfig(data?.execution);
|
|
524
534
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -531,18 +541,80 @@ const ReportFormat = {
|
|
|
531
541
|
Console: "console",
|
|
532
542
|
Json: "json"
|
|
533
543
|
};
|
|
544
|
+
// src/yaml/yamlDocument.ts
|
|
545
|
+
var YamlDocument = class YamlDocument {
|
|
546
|
+
file;
|
|
547
|
+
document;
|
|
548
|
+
lineCounter;
|
|
549
|
+
value;
|
|
550
|
+
errors;
|
|
551
|
+
constructor(file, document, lineCounter) {
|
|
552
|
+
this.file = file;
|
|
553
|
+
this.document = document;
|
|
554
|
+
this.lineCounter = lineCounter;
|
|
555
|
+
this.value = document.toJS();
|
|
556
|
+
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
557
|
+
message: error.message.split("\n")[0],
|
|
558
|
+
location: this.toLocation(error.pos[0], [])
|
|
559
|
+
}));
|
|
560
|
+
}
|
|
561
|
+
locate(path) {
|
|
562
|
+
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
563
|
+
}
|
|
564
|
+
locateKey(path, key) {
|
|
565
|
+
const parent = this.findNode(path);
|
|
566
|
+
if (isMap(parent)) {
|
|
567
|
+
const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
|
|
568
|
+
if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
569
|
+
}
|
|
570
|
+
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
571
|
+
}
|
|
572
|
+
static formatPath(path) {
|
|
573
|
+
return path.map((segment, index) => {
|
|
574
|
+
if (typeof segment === "number") return `[${segment}]`;
|
|
575
|
+
return index === 0 ? segment : `.${segment}`;
|
|
576
|
+
}).join("");
|
|
577
|
+
}
|
|
578
|
+
findNode(path) {
|
|
579
|
+
let node = this.document.contents ?? void 0;
|
|
580
|
+
for (const segment of path) {
|
|
581
|
+
let next;
|
|
582
|
+
if (isMap(node)) {
|
|
583
|
+
const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
|
|
584
|
+
next = pair?.value ?? pair?.key;
|
|
585
|
+
} else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
|
|
586
|
+
if (!next) break;
|
|
587
|
+
node = next;
|
|
588
|
+
}
|
|
589
|
+
return node;
|
|
590
|
+
}
|
|
591
|
+
toLocation(offset, path) {
|
|
592
|
+
const position = this.lineCounter.linePos(offset);
|
|
593
|
+
return new SourceLocation({
|
|
594
|
+
file: this.file,
|
|
595
|
+
line: position.line,
|
|
596
|
+
column: position.col,
|
|
597
|
+
path: YamlDocument.formatPath(path)
|
|
598
|
+
});
|
|
599
|
+
}
|
|
600
|
+
};
|
|
534
601
|
// src/yaml/environmentResolver.ts
|
|
535
602
|
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
|
|
536
603
|
const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
|
|
537
604
|
var EnvironmentResolver = class EnvironmentResolver {
|
|
538
605
|
environment;
|
|
539
606
|
parameters;
|
|
540
|
-
|
|
607
|
+
levels;
|
|
608
|
+
constructor(environment = process.env, parameters = {}, levels = []) {
|
|
541
609
|
this.environment = environment;
|
|
542
610
|
this.parameters = parameters;
|
|
611
|
+
this.levels = levels;
|
|
612
|
+
}
|
|
613
|
+
withLevels(levels) {
|
|
614
|
+
return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
|
|
543
615
|
}
|
|
544
|
-
resolve(value, document, issues, secrets = []) {
|
|
545
|
-
return this.resolveValue(value, [], document, issues, secrets);
|
|
616
|
+
resolve(value, document, issues, secrets = [], options = {}) {
|
|
617
|
+
return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
|
|
546
618
|
}
|
|
547
619
|
static hasReference(value) {
|
|
548
620
|
return new RegExp(REFERENCE.source).test(value);
|
|
@@ -550,22 +622,45 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
550
622
|
static isSecretName(name) {
|
|
551
623
|
return SECRET_NAME.test(name);
|
|
552
624
|
}
|
|
553
|
-
|
|
625
|
+
find(name) {
|
|
626
|
+
for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
|
|
627
|
+
return this.environment[name];
|
|
628
|
+
}
|
|
629
|
+
sources() {
|
|
630
|
+
const sources = [
|
|
631
|
+
...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
|
|
632
|
+
...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
|
|
633
|
+
"environment variable"
|
|
634
|
+
];
|
|
635
|
+
return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
|
|
636
|
+
}
|
|
637
|
+
resolveValue(value, path, document, issues, secrets, options) {
|
|
554
638
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
555
|
-
|
|
639
|
+
options.used?.add(name);
|
|
640
|
+
const resolved = this.find(name);
|
|
556
641
|
if (resolved === void 0) {
|
|
557
|
-
|
|
642
|
+
if (options.optional?.(path)) {
|
|
643
|
+
options.unresolved?.add(YamlDocument.formatPath(path));
|
|
644
|
+
return reference;
|
|
645
|
+
}
|
|
646
|
+
issues.push(new ValidationIssue({
|
|
647
|
+
message: `${this.sources()} '${name}' is not set`,
|
|
648
|
+
location: document.locate(path)
|
|
649
|
+
}));
|
|
650
|
+
return reference;
|
|
651
|
+
}
|
|
652
|
+
if (Array.isArray(resolved)) {
|
|
558
653
|
issues.push(new ValidationIssue({
|
|
559
|
-
message:
|
|
654
|
+
message: `variable '${name}' is a list, which can only be used as runWith.from`,
|
|
560
655
|
location: document.locate(path)
|
|
561
656
|
}));
|
|
562
657
|
return reference;
|
|
563
658
|
}
|
|
564
|
-
if (EnvironmentResolver.isSecretName(name) && !secrets.includes(resolved)) secrets.push(resolved);
|
|
659
|
+
if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
|
|
565
660
|
return resolved;
|
|
566
661
|
});
|
|
567
|
-
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
|
|
568
|
-
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
|
|
662
|
+
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
|
|
663
|
+
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
|
|
569
664
|
return value;
|
|
570
665
|
}
|
|
571
666
|
};
|
|
@@ -649,63 +744,6 @@ var SchemaValidator = class SchemaValidator {
|
|
|
649
744
|
return path;
|
|
650
745
|
}
|
|
651
746
|
};
|
|
652
|
-
// src/yaml/yamlDocument.ts
|
|
653
|
-
var YamlDocument = class YamlDocument {
|
|
654
|
-
file;
|
|
655
|
-
document;
|
|
656
|
-
lineCounter;
|
|
657
|
-
value;
|
|
658
|
-
errors;
|
|
659
|
-
constructor(file, document, lineCounter) {
|
|
660
|
-
this.file = file;
|
|
661
|
-
this.document = document;
|
|
662
|
-
this.lineCounter = lineCounter;
|
|
663
|
-
this.value = document.toJS();
|
|
664
|
-
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
665
|
-
message: error.message.split("\n")[0],
|
|
666
|
-
location: this.toLocation(error.pos[0], [])
|
|
667
|
-
}));
|
|
668
|
-
}
|
|
669
|
-
locate(path) {
|
|
670
|
-
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
671
|
-
}
|
|
672
|
-
locateKey(path, key) {
|
|
673
|
-
const parent = this.findNode(path);
|
|
674
|
-
if (isMap(parent)) {
|
|
675
|
-
const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
|
|
676
|
-
if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
677
|
-
}
|
|
678
|
-
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
679
|
-
}
|
|
680
|
-
static formatPath(path) {
|
|
681
|
-
return path.map((segment, index) => {
|
|
682
|
-
if (typeof segment === "number") return `[${segment}]`;
|
|
683
|
-
return index === 0 ? segment : `.${segment}`;
|
|
684
|
-
}).join("");
|
|
685
|
-
}
|
|
686
|
-
findNode(path) {
|
|
687
|
-
let node = this.document.contents ?? void 0;
|
|
688
|
-
for (const segment of path) {
|
|
689
|
-
let next;
|
|
690
|
-
if (isMap(node)) {
|
|
691
|
-
const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
|
|
692
|
-
next = pair?.value ?? pair?.key;
|
|
693
|
-
} else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
|
|
694
|
-
if (!next) break;
|
|
695
|
-
node = next;
|
|
696
|
-
}
|
|
697
|
-
return node;
|
|
698
|
-
}
|
|
699
|
-
toLocation(offset, path) {
|
|
700
|
-
const position = this.lineCounter.linePos(offset);
|
|
701
|
-
return new SourceLocation({
|
|
702
|
-
file: this.file,
|
|
703
|
-
line: position.line,
|
|
704
|
-
column: position.col,
|
|
705
|
-
path: YamlDocument.formatPath(path)
|
|
706
|
-
});
|
|
707
|
-
}
|
|
708
|
-
};
|
|
709
747
|
// src/yaml/yamlDocumentLoader.ts
|
|
710
748
|
var YamlDocumentLoader = class {
|
|
711
749
|
load(filePath, displayPath) {
|
|
@@ -720,6 +758,143 @@ var YamlDocumentLoader = class {
|
|
|
720
758
|
}), lineCounter);
|
|
721
759
|
}
|
|
722
760
|
};
|
|
761
|
+
// src/yaml/variablesReader.ts
|
|
762
|
+
const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
763
|
+
var VariablesReader = class {
|
|
764
|
+
read(document, resolver, issues, secrets) {
|
|
765
|
+
const raw = isRecord(document.value) ? document.value.variables : void 0;
|
|
766
|
+
if (raw === void 0) return {};
|
|
767
|
+
if (!isRecord(raw)) {
|
|
768
|
+
issues.push(new ValidationIssue({
|
|
769
|
+
message: "must be object",
|
|
770
|
+
location: document.locate(["variables"])
|
|
771
|
+
}));
|
|
772
|
+
return {};
|
|
773
|
+
}
|
|
774
|
+
const variables = {};
|
|
775
|
+
for (const [name, value] of Object.entries(raw)) {
|
|
776
|
+
if (!VARIABLE_NAME.test(name)) {
|
|
777
|
+
issues.push(new ValidationIssue({
|
|
778
|
+
message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
779
|
+
location: document.locateKey(["variables"], name)
|
|
780
|
+
}));
|
|
781
|
+
continue;
|
|
782
|
+
}
|
|
783
|
+
if (Array.isArray(value)) {
|
|
784
|
+
const bad = value.findIndex((item) => typeof item !== "string");
|
|
785
|
+
if (bad >= 0) {
|
|
786
|
+
issues.push(new ValidationIssue({
|
|
787
|
+
message: "must be string",
|
|
788
|
+
location: document.locate([
|
|
789
|
+
"variables",
|
|
790
|
+
name,
|
|
791
|
+
bad
|
|
792
|
+
])
|
|
793
|
+
}));
|
|
794
|
+
continue;
|
|
795
|
+
}
|
|
796
|
+
variables[name] = [...value];
|
|
797
|
+
continue;
|
|
798
|
+
}
|
|
799
|
+
if (typeof value !== "string") {
|
|
800
|
+
issues.push(new ValidationIssue({
|
|
801
|
+
message: "must be string or a list of strings",
|
|
802
|
+
location: document.locate(["variables", name])
|
|
803
|
+
}));
|
|
804
|
+
continue;
|
|
805
|
+
}
|
|
806
|
+
const before = issues.length;
|
|
807
|
+
const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
|
|
808
|
+
if (issues.length > before) continue;
|
|
809
|
+
const text = resolved.variables[name];
|
|
810
|
+
variables[name] = text;
|
|
811
|
+
if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
|
|
812
|
+
}
|
|
813
|
+
return variables;
|
|
814
|
+
}
|
|
815
|
+
};
|
|
816
|
+
// src/config/webSocketResolver.ts
|
|
817
|
+
const WEBSOCKET_SCHEMA = {
|
|
818
|
+
...eval_config_schema_default,
|
|
819
|
+
required: ["websocket"],
|
|
820
|
+
additionalProperties: true,
|
|
821
|
+
properties: { websocket: eval_config_schema_default.properties.websocket }
|
|
822
|
+
};
|
|
823
|
+
var WebSocketResolver = class WebSocketResolver {
|
|
824
|
+
schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
|
|
825
|
+
resolve(document, resolver, issues, secrets, used, defer) {
|
|
826
|
+
const raw = isRecord(document.value) ? document.value.websocket : void 0;
|
|
827
|
+
const unresolved = /* @__PURE__ */ new Set();
|
|
828
|
+
const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
|
|
829
|
+
const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
|
|
830
|
+
used,
|
|
831
|
+
optional,
|
|
832
|
+
unresolved
|
|
833
|
+
});
|
|
834
|
+
const subprotocols = WebSocketResolver.subprotocols(raw);
|
|
835
|
+
if (subprotocols !== void 0) {
|
|
836
|
+
const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
|
|
837
|
+
const set = !unresolved.has(YamlDocument.formatPath([
|
|
838
|
+
"websocket",
|
|
839
|
+
"connection",
|
|
840
|
+
"subprotocols",
|
|
841
|
+
index
|
|
842
|
+
]));
|
|
843
|
+
const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
|
|
844
|
+
return set && !empty;
|
|
845
|
+
});
|
|
846
|
+
value.websocket.connection.subprotocols = kept;
|
|
847
|
+
}
|
|
848
|
+
const prefix = YamlDocument.formatPath([
|
|
849
|
+
"websocket",
|
|
850
|
+
"connection",
|
|
851
|
+
"subprotocols"
|
|
852
|
+
]);
|
|
853
|
+
return {
|
|
854
|
+
value,
|
|
855
|
+
deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
|
|
856
|
+
};
|
|
857
|
+
}
|
|
858
|
+
validate(value, document) {
|
|
859
|
+
const issues = this.schemaValidator.validate(value, document);
|
|
860
|
+
if (issues.length > 0) return issues;
|
|
861
|
+
return WebSocketResolver.validateInputPath(value, document);
|
|
862
|
+
}
|
|
863
|
+
static validateInputPath(value, document) {
|
|
864
|
+
const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
|
|
865
|
+
if (typeof request.inputPath !== "string") return [];
|
|
866
|
+
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
867
|
+
const last = tokens.pop();
|
|
868
|
+
let parent = request.value;
|
|
869
|
+
for (const token of tokens) parent = WebSocketResolver.child(parent, token);
|
|
870
|
+
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
871
|
+
return [new ValidationIssue({
|
|
872
|
+
message: "must point to a missing or string value whose parent exists in request.value",
|
|
873
|
+
location: document.locate([
|
|
874
|
+
"websocket",
|
|
875
|
+
"protocol",
|
|
876
|
+
"request",
|
|
877
|
+
"inputPath"
|
|
878
|
+
])
|
|
879
|
+
})];
|
|
880
|
+
}
|
|
881
|
+
static section(value, ...path) {
|
|
882
|
+
let current = value;
|
|
883
|
+
for (const key of path) current = WebSocketResolver.child(current, key);
|
|
884
|
+
return isRecord(current) ? current : {};
|
|
885
|
+
}
|
|
886
|
+
static child(value, key) {
|
|
887
|
+
if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
|
|
888
|
+
return isRecord(value) ? value[key] : void 0;
|
|
889
|
+
}
|
|
890
|
+
static isSubprotocol(path) {
|
|
891
|
+
return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
|
|
892
|
+
}
|
|
893
|
+
static subprotocols(websocket) {
|
|
894
|
+
const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
|
|
895
|
+
return Array.isArray(subprotocols) ? subprotocols : void 0;
|
|
896
|
+
}
|
|
897
|
+
};
|
|
723
898
|
// src/config/defaultConfig.ts
|
|
724
899
|
const DEFAULT_CONFIG = new EvalConfig({
|
|
725
900
|
execution: new ExecutionConfig({
|
|
@@ -755,6 +930,8 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
755
930
|
cwd;
|
|
756
931
|
documentLoader = new YamlDocumentLoader();
|
|
757
932
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
933
|
+
variablesReader = new VariablesReader();
|
|
934
|
+
webSocketResolver = new WebSocketResolver();
|
|
758
935
|
environmentResolver;
|
|
759
936
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
760
937
|
this.cwd = cwd;
|
|
@@ -765,19 +942,47 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
765
942
|
const issues = [...document.errors];
|
|
766
943
|
if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
|
|
767
944
|
const named = [];
|
|
768
|
-
const
|
|
769
|
-
|
|
770
|
-
if (issues.length === 0)
|
|
945
|
+
const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
|
|
946
|
+
let value = {};
|
|
947
|
+
if (issues.length === 0) {
|
|
948
|
+
const resolver = this.environmentResolver.withLevels([variables]);
|
|
949
|
+
const { websocket: _websocket, variables: _variables, ...rest } = document.value;
|
|
950
|
+
const resolved = resolver.resolve(rest, document, issues, named);
|
|
951
|
+
const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
|
|
952
|
+
value = {
|
|
953
|
+
...resolved,
|
|
954
|
+
..._variables === void 0 ? {} : { variables },
|
|
955
|
+
..._websocket === void 0 ? {} : websocket.value
|
|
956
|
+
};
|
|
957
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
|
|
958
|
+
if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
|
|
959
|
+
}
|
|
771
960
|
if (issues.length > 0) throw new EvalError({
|
|
772
961
|
category: ErrorCategory.Configuration,
|
|
773
962
|
message: "invalid configuration",
|
|
774
963
|
issues
|
|
775
964
|
});
|
|
776
|
-
const config = this.build(value, configPath, named);
|
|
965
|
+
const config = this.build(value, configPath, variables, named);
|
|
777
966
|
this.applyRepetition(config, override, value, document);
|
|
778
967
|
EvalConfigLoader.applyReport(config, report);
|
|
779
968
|
return config;
|
|
780
969
|
}
|
|
970
|
+
static buildWebSocket(value) {
|
|
971
|
+
const section = (...path) => WebSocketResolver.section(value, ...path);
|
|
972
|
+
const connection = section("websocket", "connection");
|
|
973
|
+
const request = section("websocket", "protocol", "request");
|
|
974
|
+
return new WebSocketConfig({
|
|
975
|
+
endpoint: WebSocketResolver.section(value, "websocket").endpoint,
|
|
976
|
+
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
977
|
+
protocol: new ProtocolConfig({
|
|
978
|
+
request: new RequestConfig({
|
|
979
|
+
...request,
|
|
980
|
+
format: request.format ?? RequestFormat.Json
|
|
981
|
+
}),
|
|
982
|
+
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
983
|
+
})
|
|
984
|
+
});
|
|
985
|
+
}
|
|
781
986
|
static applyReport(config, report) {
|
|
782
987
|
if (report.formats === void 0) return;
|
|
783
988
|
const known = Object.values(ReportFormat);
|
|
@@ -799,45 +1004,17 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
799
1004
|
location: document.locate(["evaluator", "apiKey"])
|
|
800
1005
|
})];
|
|
801
1006
|
}
|
|
802
|
-
|
|
803
|
-
const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
|
|
804
|
-
if (typeof request.inputPath !== "string") return [];
|
|
805
|
-
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
806
|
-
const last = tokens.pop();
|
|
807
|
-
let parent = request.value;
|
|
808
|
-
for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
|
|
809
|
-
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
810
|
-
return [new ValidationIssue({
|
|
811
|
-
message: "must point to a missing or string value whose parent exists in request.value",
|
|
812
|
-
location: document.locate([
|
|
813
|
-
"websocket",
|
|
814
|
-
"protocol",
|
|
815
|
-
"request",
|
|
816
|
-
"inputPath"
|
|
817
|
-
])
|
|
818
|
-
})];
|
|
819
|
-
}
|
|
820
|
-
build(value, configPath, named) {
|
|
1007
|
+
build(value, configPath, variables, named) {
|
|
821
1008
|
const section = (...path) => EvalConfigLoader.section(value, ...path);
|
|
822
1009
|
const configDirectory = dirname(configPath);
|
|
823
|
-
const
|
|
824
|
-
const request = section("websocket", "protocol", "request");
|
|
1010
|
+
const websocket = EvalConfigLoader.buildWebSocket(value);
|
|
825
1011
|
const evaluator = section("evaluator");
|
|
826
1012
|
const report = section("report");
|
|
827
1013
|
return new EvalConfig({
|
|
828
1014
|
version: value.version,
|
|
829
1015
|
configPath,
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
833
|
-
protocol: new ProtocolConfig({
|
|
834
|
-
request: new RequestConfig({
|
|
835
|
-
...request,
|
|
836
|
-
format: request.format ?? RequestFormat.Json
|
|
837
|
-
}),
|
|
838
|
-
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
839
|
-
})
|
|
840
|
-
}),
|
|
1016
|
+
variables,
|
|
1017
|
+
websocket,
|
|
841
1018
|
execution: new ExecutionConfig({
|
|
842
1019
|
...DEFAULT_CONFIG.execution,
|
|
843
1020
|
...section("execution"),
|
|
@@ -858,7 +1035,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
858
1035
|
directory: resolve(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
|
|
859
1036
|
formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
|
|
860
1037
|
}),
|
|
861
|
-
secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols
|
|
1038
|
+
secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
|
|
862
1039
|
});
|
|
863
1040
|
}
|
|
864
1041
|
applyRepetition(config, override, value, document) {
|
|
@@ -911,13 +1088,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
911
1088
|
])];
|
|
912
1089
|
}
|
|
913
1090
|
static section(value, ...path) {
|
|
914
|
-
|
|
915
|
-
for (const key of path) current = EvalConfigLoader.child(current, key);
|
|
916
|
-
return isRecord(current) ? current : {};
|
|
917
|
-
}
|
|
918
|
-
static child(value, key) {
|
|
919
|
-
if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
|
|
920
|
-
return isRecord(value) ? value[key] : void 0;
|
|
1091
|
+
return WebSocketResolver.section(value, ...path);
|
|
921
1092
|
}
|
|
922
1093
|
};
|
|
923
1094
|
// src/interfaces/runStatus.ts
|
|
@@ -1109,18 +1280,22 @@ var TurnSpec = class {
|
|
|
1109
1280
|
var ConversationSpec = class {
|
|
1110
1281
|
version;
|
|
1111
1282
|
name;
|
|
1283
|
+
runName;
|
|
1112
1284
|
description;
|
|
1113
1285
|
turns;
|
|
1114
1286
|
conversation;
|
|
1115
1287
|
location;
|
|
1288
|
+
websocket;
|
|
1116
1289
|
secrets;
|
|
1117
1290
|
constructor(data) {
|
|
1118
1291
|
this.version = data?.version ?? 1;
|
|
1119
1292
|
this.name = data?.name ?? "";
|
|
1293
|
+
this.runName = data?.runName ?? "";
|
|
1120
1294
|
this.description = data?.description ?? "";
|
|
1121
1295
|
this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
|
|
1122
1296
|
this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
|
|
1123
1297
|
this.location = new SourceLocation(data?.location);
|
|
1298
|
+
this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
|
|
1124
1299
|
this.secrets = [...data?.secrets ?? []];
|
|
1125
1300
|
}
|
|
1126
1301
|
};
|
|
@@ -2420,7 +2595,11 @@ var ConversationRunner = class {
|
|
|
2420
2595
|
async run(spec, signal = NEVER_ABORTED) {
|
|
2421
2596
|
const startedAt = Date.now();
|
|
2422
2597
|
const outcomes = [];
|
|
2423
|
-
const
|
|
2598
|
+
const config = new EvalConfig({
|
|
2599
|
+
...this.config,
|
|
2600
|
+
websocket: spec.websocket ?? this.config.websocket
|
|
2601
|
+
});
|
|
2602
|
+
const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
|
|
2424
2603
|
const history = [];
|
|
2425
2604
|
const evaluate = this.semanticEvaluator(signal);
|
|
2426
2605
|
const inline = !this.config.execution.continueOnFailure;
|
|
@@ -2445,7 +2624,7 @@ var ConversationRunner = class {
|
|
|
2445
2624
|
try {
|
|
2446
2625
|
const steps = (async () => {
|
|
2447
2626
|
for (let attempt = 0;; attempt++) {
|
|
2448
|
-
connection = this.createConnection(
|
|
2627
|
+
connection = this.createConnection(config);
|
|
2449
2628
|
connectionAttempts++;
|
|
2450
2629
|
try {
|
|
2451
2630
|
await connection.connect();
|
|
@@ -2626,7 +2805,7 @@ var ScenarioRunner = class {
|
|
|
2626
2805
|
required: minPass,
|
|
2627
2806
|
attempts,
|
|
2628
2807
|
tests: this.results.tests(attempts, minPass),
|
|
2629
|
-
agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
|
|
2808
|
+
agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
|
|
2630
2809
|
evaluatorUsage: this.tokens.evaluator(attempts)
|
|
2631
2810
|
});
|
|
2632
2811
|
}
|
|
@@ -2785,10 +2964,12 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
2785
2964
|
scenario(result, repeat, tokenUsageCapability) {
|
|
2786
2965
|
const spec = result.spec;
|
|
2787
2966
|
const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
|
|
2788
|
-
const line = `${this.status(result.status, STATUS_WIDTH)} ${spec
|
|
2967
|
+
const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
|
|
2789
2968
|
if (!this.verbose) return line;
|
|
2790
2969
|
if (result.attempts.length === 0) return `${line}\n`;
|
|
2791
|
-
const
|
|
2970
|
+
const websocket = spec.websocket;
|
|
2971
|
+
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
2972
|
+
const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
|
|
2792
2973
|
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
2793
2974
|
lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
|
|
2794
2975
|
lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
|
|
@@ -2796,6 +2977,9 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
2796
2977
|
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
2797
2978
|
return line + lines.join("\n");
|
|
2798
2979
|
}
|
|
2980
|
+
static title(spec) {
|
|
2981
|
+
return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
|
|
2982
|
+
}
|
|
2799
2983
|
summary(run, exitCode, reportPaths = []) {
|
|
2800
2984
|
const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
|
|
2801
2985
|
const lines = this.verbose ? [] : [""];
|
|
@@ -2827,7 +3011,7 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
2827
3011
|
}
|
|
2828
3012
|
problems(result) {
|
|
2829
3013
|
const spec = result.spec;
|
|
2830
|
-
const lines = [`${this.status(result.status, 6)} ${spec
|
|
3014
|
+
const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
|
|
2831
3015
|
for (const attempt of result.attempts) {
|
|
2832
3016
|
const execution = attempt.execution;
|
|
2833
3017
|
if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
|
|
@@ -3000,11 +3184,17 @@ var JsonReporter = class JsonReporter {
|
|
|
3000
3184
|
};
|
|
3001
3185
|
}
|
|
3002
3186
|
scenario(result, config) {
|
|
3003
|
-
const
|
|
3187
|
+
const websocket = result.spec.websocket ?? config.websocket;
|
|
3188
|
+
const capability = websocket.protocol.capabilities.tokenUsage;
|
|
3004
3189
|
return {
|
|
3005
3190
|
name: result.spec.name,
|
|
3191
|
+
runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
|
|
3006
3192
|
file: result.spec.location.file,
|
|
3007
3193
|
status: result.status,
|
|
3194
|
+
connection: {
|
|
3195
|
+
endpoint: websocket.endpoint,
|
|
3196
|
+
subprotocols: websocket.connection.subprotocols.length
|
|
3197
|
+
},
|
|
3008
3198
|
...JsonReporter.counts(result),
|
|
3009
3199
|
agentUsage: JsonReporter.usage(result.agentUsage),
|
|
3010
3200
|
evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
|
|
@@ -3165,6 +3355,48 @@ var conversation_schema_default = {
|
|
|
3165
3355
|
"version": { "const": 1 },
|
|
3166
3356
|
"name": { "$ref": "#/definitions/text" },
|
|
3167
3357
|
"description": { "type": "string" },
|
|
3358
|
+
"variables": {
|
|
3359
|
+
"type": "object",
|
|
3360
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
3361
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
3362
|
+
"type": "array",
|
|
3363
|
+
"items": { "type": "string" }
|
|
3364
|
+
}] }
|
|
3365
|
+
},
|
|
3366
|
+
"runWith": {
|
|
3367
|
+
"description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
|
|
3368
|
+
"anyOf": [{
|
|
3369
|
+
"type": "array",
|
|
3370
|
+
"minItems": 1,
|
|
3371
|
+
"items": {
|
|
3372
|
+
"type": "object",
|
|
3373
|
+
"required": ["name"],
|
|
3374
|
+
"properties": { "name": {
|
|
3375
|
+
"type": "string",
|
|
3376
|
+
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
|
|
3377
|
+
} },
|
|
3378
|
+
"additionalProperties": { "type": "string" }
|
|
3379
|
+
}
|
|
3380
|
+
}, {
|
|
3381
|
+
"type": "object",
|
|
3382
|
+
"additionalProperties": false,
|
|
3383
|
+
"required": ["variable", "from"],
|
|
3384
|
+
"properties": {
|
|
3385
|
+
"variable": {
|
|
3386
|
+
"type": "string",
|
|
3387
|
+
"pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
|
|
3388
|
+
},
|
|
3389
|
+
"from": { "anyOf": [{
|
|
3390
|
+
"type": "array",
|
|
3391
|
+
"minItems": 1,
|
|
3392
|
+
"items": { "type": "string" }
|
|
3393
|
+
}, {
|
|
3394
|
+
"type": "string",
|
|
3395
|
+
"pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
|
|
3396
|
+
}] }
|
|
3397
|
+
}
|
|
3398
|
+
}]
|
|
3399
|
+
},
|
|
3168
3400
|
"turns": {
|
|
3169
3401
|
"type": "array",
|
|
3170
3402
|
"minItems": 1,
|
|
@@ -3177,8 +3409,8 @@ var conversation_schema_default = {
|
|
|
3177
3409
|
"properties": { "conversation": {
|
|
3178
3410
|
"type": "object",
|
|
3179
3411
|
"additionalProperties": false,
|
|
3180
|
-
"required": ["
|
|
3181
|
-
"properties": { "
|
|
3412
|
+
"required": ["should"],
|
|
3413
|
+
"properties": { "should": { "$ref": "#/definitions/text" } }
|
|
3182
3414
|
} }
|
|
3183
3415
|
}
|
|
3184
3416
|
},
|
|
@@ -3221,12 +3453,12 @@ var conversation_schema_default = {
|
|
|
3221
3453
|
}
|
|
3222
3454
|
},
|
|
3223
3455
|
"response": {
|
|
3224
|
-
"description": "A string is shorthand for
|
|
3456
|
+
"description": "A string is shorthand for should.",
|
|
3225
3457
|
"type": ["string", "object"],
|
|
3226
3458
|
"pattern": "\\S",
|
|
3227
3459
|
"additionalProperties": false,
|
|
3228
3460
|
"properties": {
|
|
3229
|
-
"
|
|
3461
|
+
"should": { "$ref": "#/definitions/text" },
|
|
3230
3462
|
"contains": { "$ref": "#/definitions/textList" },
|
|
3231
3463
|
"notContains": { "$ref": "#/definitions/textList" }
|
|
3232
3464
|
}
|
|
@@ -3258,7 +3490,7 @@ var conversation_schema_default = {
|
|
|
3258
3490
|
};
|
|
3259
3491
|
// src/scenarios/conversationValidator.ts
|
|
3260
3492
|
const RESPONSE_KEYS = [
|
|
3261
|
-
"
|
|
3493
|
+
"should",
|
|
3262
3494
|
"contains",
|
|
3263
3495
|
"notContains"
|
|
3264
3496
|
];
|
|
@@ -3281,7 +3513,7 @@ var ConversationValidator = class ConversationValidator {
|
|
|
3281
3513
|
validateResponse(response, path, document, issues) {
|
|
3282
3514
|
if (!RESPONSE_KEYS.some((key) => key in response)) {
|
|
3283
3515
|
issues.push(new ValidationIssue({
|
|
3284
|
-
message: "response must have at least one of:
|
|
3516
|
+
message: "response must have at least one of: should, contains, notContains",
|
|
3285
3517
|
location: document.locate(path)
|
|
3286
3518
|
}));
|
|
3287
3519
|
return;
|
|
@@ -3322,13 +3554,231 @@ var ConversationValidator = class ConversationValidator {
|
|
|
3322
3554
|
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
3323
3555
|
}
|
|
3324
3556
|
};
|
|
3557
|
+
// src/scenarios/runWithReader.ts
|
|
3558
|
+
const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
|
|
3559
|
+
const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
|
|
3560
|
+
const FORMS = "must be a list of rows, or an object with variable and from";
|
|
3561
|
+
const SINGLE_RUN = {
|
|
3562
|
+
name: "",
|
|
3563
|
+
variables: {},
|
|
3564
|
+
keys: [],
|
|
3565
|
+
location: [],
|
|
3566
|
+
secrets: []
|
|
3567
|
+
};
|
|
3568
|
+
var RunWithReader = class RunWithReader {
|
|
3569
|
+
read(document, resolver, issues, used) {
|
|
3570
|
+
const raw = isRecord(document.value) ? document.value.runWith : void 0;
|
|
3571
|
+
if (raw === void 0) return void 0;
|
|
3572
|
+
if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
|
|
3573
|
+
if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
|
|
3574
|
+
issues.push(new ValidationIssue({
|
|
3575
|
+
message: FORMS,
|
|
3576
|
+
location: document.locate(["runWith"])
|
|
3577
|
+
}));
|
|
3578
|
+
return [];
|
|
3579
|
+
}
|
|
3580
|
+
rows(rows, document, resolver, issues, used) {
|
|
3581
|
+
if (rows.length === 0) {
|
|
3582
|
+
issues.push(new ValidationIssue({
|
|
3583
|
+
message: "must not be empty",
|
|
3584
|
+
location: document.locate(["runWith"])
|
|
3585
|
+
}));
|
|
3586
|
+
return [];
|
|
3587
|
+
}
|
|
3588
|
+
const names = /* @__PURE__ */ new Set();
|
|
3589
|
+
const runs = [];
|
|
3590
|
+
rows.forEach((row, index) => {
|
|
3591
|
+
const path = ["runWith", index];
|
|
3592
|
+
if (!isRecord(row)) {
|
|
3593
|
+
issues.push(new ValidationIssue({
|
|
3594
|
+
message: "must be object",
|
|
3595
|
+
location: document.locate(path)
|
|
3596
|
+
}));
|
|
3597
|
+
return;
|
|
3598
|
+
}
|
|
3599
|
+
const name = row.name;
|
|
3600
|
+
if (name === void 0) issues.push(new ValidationIssue({
|
|
3601
|
+
message: "missing required key 'name'",
|
|
3602
|
+
location: document.locate(path)
|
|
3603
|
+
}));
|
|
3604
|
+
else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
|
|
3605
|
+
message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
|
|
3606
|
+
location: document.locate([...path, "name"])
|
|
3607
|
+
}));
|
|
3608
|
+
else if (names.has(name)) issues.push(new ValidationIssue({
|
|
3609
|
+
message: `duplicate run name '${name}'`,
|
|
3610
|
+
location: document.locate([...path, "name"])
|
|
3611
|
+
}));
|
|
3612
|
+
else names.add(name);
|
|
3613
|
+
const run = {
|
|
3614
|
+
name: typeof name === "string" ? name : "",
|
|
3615
|
+
variables: {},
|
|
3616
|
+
keys: [],
|
|
3617
|
+
location: path,
|
|
3618
|
+
secrets: []
|
|
3619
|
+
};
|
|
3620
|
+
for (const [key, value] of Object.entries(row)) {
|
|
3621
|
+
if (key === "name") continue;
|
|
3622
|
+
if (!VARIABLE_NAME.test(key)) {
|
|
3623
|
+
issues.push(new ValidationIssue({
|
|
3624
|
+
message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
3625
|
+
location: document.locateKey(path, key)
|
|
3626
|
+
}));
|
|
3627
|
+
continue;
|
|
3628
|
+
}
|
|
3629
|
+
if (typeof value !== "string") {
|
|
3630
|
+
issues.push(new ValidationIssue({
|
|
3631
|
+
message: "must be string",
|
|
3632
|
+
location: document.locate([...path, key])
|
|
3633
|
+
}));
|
|
3634
|
+
continue;
|
|
3635
|
+
}
|
|
3636
|
+
const resolved = resolver.resolve(value, document, issues, run.secrets, {
|
|
3637
|
+
used,
|
|
3638
|
+
at: [...path, key]
|
|
3639
|
+
});
|
|
3640
|
+
RunWithReader.set(run, key, resolved, document.locateKey(path, key));
|
|
3641
|
+
}
|
|
3642
|
+
runs.push(run);
|
|
3643
|
+
});
|
|
3644
|
+
return runs;
|
|
3645
|
+
}
|
|
3646
|
+
list(form, document, resolver, issues, used) {
|
|
3647
|
+
const before = issues.length;
|
|
3648
|
+
for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
|
|
3649
|
+
message: `unknown key '${key}'`,
|
|
3650
|
+
location: document.locateKey(["runWith"], key)
|
|
3651
|
+
}));
|
|
3652
|
+
const variable = form.variable;
|
|
3653
|
+
if (variable === void 0) issues.push(new ValidationIssue({
|
|
3654
|
+
message: "missing required key 'variable'",
|
|
3655
|
+
location: document.locate(["runWith"])
|
|
3656
|
+
}));
|
|
3657
|
+
else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
|
|
3658
|
+
message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
|
|
3659
|
+
location: document.locate(["runWith", "variable"])
|
|
3660
|
+
}));
|
|
3661
|
+
const entries = this.entries(form.from, document, resolver, issues, used);
|
|
3662
|
+
if (issues.length > before || entries === void 0) return [];
|
|
3663
|
+
const name = variable;
|
|
3664
|
+
const keyLocation = document.locate(["runWith", "variable"]);
|
|
3665
|
+
const names = /* @__PURE__ */ new Set();
|
|
3666
|
+
const runs = [];
|
|
3667
|
+
for (const { entry, path } of entries) {
|
|
3668
|
+
if (!VARIABLE_NAME.test(entry)) {
|
|
3669
|
+
issues.push(new ValidationIssue({
|
|
3670
|
+
message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
|
|
3671
|
+
location: document.locate(path)
|
|
3672
|
+
}));
|
|
3673
|
+
continue;
|
|
3674
|
+
}
|
|
3675
|
+
if (names.has(entry)) {
|
|
3676
|
+
issues.push(new ValidationIssue({
|
|
3677
|
+
message: `duplicate run name '${entry}'`,
|
|
3678
|
+
location: document.locate(path)
|
|
3679
|
+
}));
|
|
3680
|
+
continue;
|
|
3681
|
+
}
|
|
3682
|
+
names.add(entry);
|
|
3683
|
+
const run = {
|
|
3684
|
+
name: entry,
|
|
3685
|
+
variables: {},
|
|
3686
|
+
keys: [],
|
|
3687
|
+
location: path,
|
|
3688
|
+
secrets: []
|
|
3689
|
+
};
|
|
3690
|
+
const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
|
|
3691
|
+
used,
|
|
3692
|
+
at: path
|
|
3693
|
+
});
|
|
3694
|
+
RunWithReader.set(run, name, value, keyLocation);
|
|
3695
|
+
runs.push(run);
|
|
3696
|
+
}
|
|
3697
|
+
return runs;
|
|
3698
|
+
}
|
|
3699
|
+
entries(from, document, resolver, issues, used) {
|
|
3700
|
+
const path = ["runWith", "from"];
|
|
3701
|
+
const location = document.locate(path);
|
|
3702
|
+
if (from === void 0) {
|
|
3703
|
+
issues.push(new ValidationIssue({
|
|
3704
|
+
message: "missing required key 'from'",
|
|
3705
|
+
location: document.locate(["runWith"])
|
|
3706
|
+
}));
|
|
3707
|
+
return;
|
|
3708
|
+
}
|
|
3709
|
+
let list;
|
|
3710
|
+
let at;
|
|
3711
|
+
if (Array.isArray(from)) {
|
|
3712
|
+
list = from;
|
|
3713
|
+
at = (index) => [...path, index];
|
|
3714
|
+
} else {
|
|
3715
|
+
const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
|
|
3716
|
+
if (reference === null) {
|
|
3717
|
+
issues.push(new ValidationIssue({
|
|
3718
|
+
message: "must be a list, or a list variable such as ${tokens}",
|
|
3719
|
+
location
|
|
3720
|
+
}));
|
|
3721
|
+
return;
|
|
3722
|
+
}
|
|
3723
|
+
const name = reference[1];
|
|
3724
|
+
const found = resolver.find(name);
|
|
3725
|
+
used.add(name);
|
|
3726
|
+
if (found === void 0) {
|
|
3727
|
+
issues.push(new ValidationIssue({
|
|
3728
|
+
message: `list variable '${name}' is not set`,
|
|
3729
|
+
location
|
|
3730
|
+
}));
|
|
3731
|
+
return;
|
|
3732
|
+
}
|
|
3733
|
+
if (!Array.isArray(found)) {
|
|
3734
|
+
issues.push(new ValidationIssue({
|
|
3735
|
+
message: `variable '${name}' is not a list`,
|
|
3736
|
+
location
|
|
3737
|
+
}));
|
|
3738
|
+
return;
|
|
3739
|
+
}
|
|
3740
|
+
list = found;
|
|
3741
|
+
at = () => path;
|
|
3742
|
+
}
|
|
3743
|
+
if (list.length === 0) {
|
|
3744
|
+
issues.push(new ValidationIssue({
|
|
3745
|
+
message: "must not be empty",
|
|
3746
|
+
location
|
|
3747
|
+
}));
|
|
3748
|
+
return;
|
|
3749
|
+
}
|
|
3750
|
+
const bad = list.findIndex((item) => typeof item !== "string");
|
|
3751
|
+
if (bad >= 0) {
|
|
3752
|
+
issues.push(new ValidationIssue({
|
|
3753
|
+
message: "must be string",
|
|
3754
|
+
location: document.locate(at(bad))
|
|
3755
|
+
}));
|
|
3756
|
+
return;
|
|
3757
|
+
}
|
|
3758
|
+
return list.map((entry, index) => ({
|
|
3759
|
+
entry,
|
|
3760
|
+
path: at(index)
|
|
3761
|
+
}));
|
|
3762
|
+
}
|
|
3763
|
+
static set(run, name, value, location) {
|
|
3764
|
+
run.variables[name] = value;
|
|
3765
|
+
run.keys.push({
|
|
3766
|
+
name,
|
|
3767
|
+
location
|
|
3768
|
+
});
|
|
3769
|
+
if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
|
|
3770
|
+
}
|
|
3771
|
+
};
|
|
3325
3772
|
// src/scenarios/scenarioLoader.ts
|
|
3326
3773
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
3327
|
-
var ScenarioLoader = class {
|
|
3774
|
+
var ScenarioLoader = class ScenarioLoader {
|
|
3328
3775
|
cwd;
|
|
3329
3776
|
documentLoader = new YamlDocumentLoader();
|
|
3330
3777
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
3331
3778
|
conversationValidator = new ConversationValidator();
|
|
3779
|
+
variablesReader = new VariablesReader();
|
|
3780
|
+
runWithReader = new RunWithReader();
|
|
3781
|
+
webSocketResolver = new WebSocketResolver();
|
|
3332
3782
|
environmentResolver;
|
|
3333
3783
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
3334
3784
|
this.cwd = cwd;
|
|
@@ -3336,18 +3786,66 @@ var ScenarioLoader = class {
|
|
|
3336
3786
|
}
|
|
3337
3787
|
load(target, config) {
|
|
3338
3788
|
const files = this.discover(resolve(this.cwd, target), config);
|
|
3789
|
+
const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.cwd, config.configPath)) : void 0;
|
|
3339
3790
|
const issues = [];
|
|
3791
|
+
const connectionIssues = [];
|
|
3340
3792
|
const specs = [];
|
|
3341
3793
|
for (const file of files) {
|
|
3342
3794
|
const document = this.documentLoader.load(file, relative(this.cwd, file));
|
|
3343
3795
|
const fileIssues = [...document.errors];
|
|
3344
|
-
const
|
|
3345
|
-
const
|
|
3346
|
-
|
|
3347
|
-
|
|
3348
|
-
|
|
3796
|
+
const fileSecrets = [];
|
|
3797
|
+
const used = /* @__PURE__ */ new Set();
|
|
3798
|
+
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
|
|
3799
|
+
const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
|
|
3800
|
+
const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
|
|
3801
|
+
if (fileIssues.length > 0) {
|
|
3802
|
+
issues.push(...fileIssues);
|
|
3803
|
+
continue;
|
|
3804
|
+
}
|
|
3805
|
+
const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
|
|
3806
|
+
const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
|
|
3807
|
+
for (const result of results) for (const name of result.used) used.add(name);
|
|
3808
|
+
fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
|
|
3809
|
+
run: result.run,
|
|
3810
|
+
issues: result.issues
|
|
3811
|
+
})), (issue, run) => run === void 0 ? issue : new ValidationIssue({
|
|
3812
|
+
message: `run '${run.name}': ${issue.message}`,
|
|
3813
|
+
location: issue.location
|
|
3814
|
+
})));
|
|
3815
|
+
const attempted = results.filter((result) => result.connectionChecked);
|
|
3816
|
+
if (attempted.length > 0) connectionIssues.push({
|
|
3817
|
+
file: document,
|
|
3818
|
+
runs: attempted.map((result) => ({
|
|
3819
|
+
run: result.run,
|
|
3820
|
+
issues: result.websocketIssues
|
|
3821
|
+
}))
|
|
3822
|
+
});
|
|
3823
|
+
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
|
|
3824
|
+
if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
|
|
3825
|
+
const websocket = result.websocket;
|
|
3826
|
+
specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
3827
|
+
}
|
|
3349
3828
|
else issues.push(...fileIssues);
|
|
3350
3829
|
}
|
|
3830
|
+
const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
|
|
3831
|
+
if (shared.length > 0) throw new EvalError({
|
|
3832
|
+
category: ErrorCategory.Configuration,
|
|
3833
|
+
message: "invalid configuration",
|
|
3834
|
+
issues: shared
|
|
3835
|
+
});
|
|
3836
|
+
for (const entry of connectionIssues) {
|
|
3837
|
+
const fileLocation = ScenarioLoader.variablesLocation(entry.file);
|
|
3838
|
+
issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
|
|
3839
|
+
message: issue.toString(),
|
|
3840
|
+
location: fileLocation
|
|
3841
|
+
}) : new ValidationIssue({
|
|
3842
|
+
message: `run '${run.name}': ${issue.toString()}`,
|
|
3843
|
+
location: new SourceLocation({
|
|
3844
|
+
...entry.file.locate(run.location),
|
|
3845
|
+
path: ""
|
|
3846
|
+
})
|
|
3847
|
+
})));
|
|
3848
|
+
}
|
|
3351
3849
|
if (issues.length > 0) throw new EvalError({
|
|
3352
3850
|
category: ErrorCategory.Specification,
|
|
3353
3851
|
message: "invalid scenario specification",
|
|
@@ -3355,6 +3853,71 @@ var ScenarioLoader = class {
|
|
|
3355
3853
|
});
|
|
3356
3854
|
return specs;
|
|
3357
3855
|
}
|
|
3856
|
+
resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
|
|
3857
|
+
const issues = [];
|
|
3858
|
+
const used = /* @__PURE__ */ new Set();
|
|
3859
|
+
const resolver = scenarioResolver.withLevels([run.variables]);
|
|
3860
|
+
const value = {
|
|
3861
|
+
...resolver.resolve(rest, document, issues, secrets, { used }),
|
|
3862
|
+
...variables === void 0 ? {} : { variables }
|
|
3863
|
+
};
|
|
3864
|
+
const websocketIssues = [];
|
|
3865
|
+
let websocket;
|
|
3866
|
+
let connectionChecked = false;
|
|
3867
|
+
if (issues.length === 0 && configDocument !== void 0) {
|
|
3868
|
+
const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
|
|
3869
|
+
if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
|
|
3870
|
+
if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
|
|
3871
|
+
connectionChecked = true;
|
|
3872
|
+
}
|
|
3873
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
|
|
3874
|
+
if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
|
|
3875
|
+
return {
|
|
3876
|
+
run,
|
|
3877
|
+
value,
|
|
3878
|
+
used,
|
|
3879
|
+
secrets,
|
|
3880
|
+
issues,
|
|
3881
|
+
websocket,
|
|
3882
|
+
websocketIssues,
|
|
3883
|
+
connectionChecked
|
|
3884
|
+
};
|
|
3885
|
+
}
|
|
3886
|
+
static byRun(runs, place) {
|
|
3887
|
+
const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
|
|
3888
|
+
const sharedTexts = new Set(shared.map((issue) => issue.toString()));
|
|
3889
|
+
const result = shared.map((issue) => place(issue, void 0));
|
|
3890
|
+
for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
|
|
3891
|
+
return result;
|
|
3892
|
+
}
|
|
3893
|
+
static shared(lists) {
|
|
3894
|
+
if (lists.length === 0) return [];
|
|
3895
|
+
const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
|
|
3896
|
+
return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
|
|
3897
|
+
}
|
|
3898
|
+
static variablesLocation(document) {
|
|
3899
|
+
return new SourceLocation({
|
|
3900
|
+
...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
|
|
3901
|
+
path: ""
|
|
3902
|
+
});
|
|
3903
|
+
}
|
|
3904
|
+
static unusedVariables(variables, used, results, document) {
|
|
3905
|
+
const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
|
|
3906
|
+
const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
|
|
3907
|
+
message: message(name),
|
|
3908
|
+
location: document.locateKey(["variables"], name)
|
|
3909
|
+
}));
|
|
3910
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3911
|
+
for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
|
|
3912
|
+
const issue = new ValidationIssue({
|
|
3913
|
+
message: message(key.name),
|
|
3914
|
+
location: key.location
|
|
3915
|
+
});
|
|
3916
|
+
if (!seen.has(issue.toString())) issues.push(issue);
|
|
3917
|
+
seen.add(issue.toString());
|
|
3918
|
+
}
|
|
3919
|
+
return issues;
|
|
3920
|
+
}
|
|
3358
3921
|
discover(target, config) {
|
|
3359
3922
|
if (!existsSync(target)) throw new EvalError({
|
|
3360
3923
|
category: ErrorCategory.Specification,
|
|
@@ -3379,20 +3942,22 @@ var ScenarioLoader = class {
|
|
|
3379
3942
|
}
|
|
3380
3943
|
return files;
|
|
3381
3944
|
}
|
|
3382
|
-
build(value, document, secrets) {
|
|
3945
|
+
build(value, document, runName, websocket, secrets) {
|
|
3383
3946
|
const scenario = value;
|
|
3384
3947
|
const turns = scenario.turns;
|
|
3385
3948
|
const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
|
|
3386
3949
|
return new ConversationSpec({
|
|
3387
3950
|
version: scenario.version,
|
|
3388
3951
|
name: scenario.name,
|
|
3952
|
+
runName,
|
|
3389
3953
|
description: typeof scenario.description === "string" ? scenario.description : "",
|
|
3390
3954
|
turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
|
|
3391
3955
|
conversation: conversation ? new ConversationExpectation({
|
|
3392
|
-
criteria: conversation.
|
|
3956
|
+
criteria: conversation.should,
|
|
3393
3957
|
location: document.locate(["expect", "conversation"])
|
|
3394
3958
|
}) : void 0,
|
|
3395
3959
|
location: document.locate([]),
|
|
3960
|
+
websocket,
|
|
3396
3961
|
secrets
|
|
3397
3962
|
});
|
|
3398
3963
|
}
|
|
@@ -3418,7 +3983,7 @@ var ScenarioLoader = class {
|
|
|
3418
3983
|
});
|
|
3419
3984
|
const fields = response;
|
|
3420
3985
|
return new ResponseExpectation({
|
|
3421
|
-
criteria: fields.
|
|
3986
|
+
criteria: fields.should,
|
|
3422
3987
|
contains: fields.contains ?? [],
|
|
3423
3988
|
notContains: fields.notContains ?? [],
|
|
3424
3989
|
location
|