@mappies/miniwat-eval 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -13
- package/dist/index.cjs +435 -178
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +72 -51
- package/dist/index.d.ts +72 -51
- package/dist/index.js +434 -178
- package/dist/index.js.map +1 -1
- package/dist/main.js +410 -148
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +9 -4
- package/schemas/eval-config.schema.json +5 -0
- package/schemas/report.schema.json +12 -1
package/dist/main.js
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
import { Command, CommanderError } from "commander";
|
|
3
3
|
import { dirname, extname, join, relative, resolve } from "node:path";
|
|
4
4
|
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
5
|
-
import { Ajv } from "ajv";
|
|
6
5
|
import { LineCounter, isMap, isPair, isScalar, isSeq, parseDocument } from "yaml";
|
|
6
|
+
import { Ajv } from "ajv";
|
|
7
7
|
import { Message, ResponseMessage, TokenUsage } from "@mappies/miniwat-agent-core";
|
|
8
8
|
import { Agent, InMemoryAgentStateStore } from "@mappies/miniwat-agent";
|
|
9
9
|
import { randomUUID } from "node:crypto";
|
|
@@ -164,6 +164,44 @@ var ConfigLocator = class ConfigLocator {
|
|
|
164
164
|
return existsSync(path) && statSync(path).isFile();
|
|
165
165
|
}
|
|
166
166
|
};
|
|
167
|
+
// src/utils/isRecord.ts
|
|
168
|
+
function isRecord(value) {
|
|
169
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
170
|
+
}
|
|
171
|
+
// src/config/parametersLoader.ts
|
|
172
|
+
var ParametersLoader = class ParametersLoader {
|
|
173
|
+
cwd;
|
|
174
|
+
constructor(cwd = process.cwd()) {
|
|
175
|
+
this.cwd = cwd;
|
|
176
|
+
}
|
|
177
|
+
load(path) {
|
|
178
|
+
const absolute = resolve(this.cwd, path);
|
|
179
|
+
const shown = relative(this.cwd, absolute) || path;
|
|
180
|
+
let text;
|
|
181
|
+
let value;
|
|
182
|
+
try {
|
|
183
|
+
text = readFileSync(absolute, "utf8");
|
|
184
|
+
} catch {
|
|
185
|
+
throw ParametersLoader.error(`parameters file not found: ${shown}`);
|
|
186
|
+
}
|
|
187
|
+
try {
|
|
188
|
+
value = JSON.parse(text);
|
|
189
|
+
} catch {
|
|
190
|
+
throw ParametersLoader.error(`parameters file ${shown} is not valid JSON`);
|
|
191
|
+
}
|
|
192
|
+
if (!isRecord(value)) throw ParametersLoader.error(`parameters file ${shown} must contain a JSON object`);
|
|
193
|
+
return {
|
|
194
|
+
path: absolute,
|
|
195
|
+
parameters: Object.fromEntries(Object.entries(value).filter((entry) => typeof entry[1] === "string"))
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
static error(message) {
|
|
199
|
+
return new EvalError({
|
|
200
|
+
category: ErrorCategory.Configuration,
|
|
201
|
+
message
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
};
|
|
167
205
|
// schemas/eval-config.schema.json
|
|
168
206
|
var eval_config_schema_default = {
|
|
169
207
|
$schema: "http://json-schema.org/draft-07/schema#",
|
|
@@ -177,6 +215,11 @@ var eval_config_schema_default = {
|
|
|
177
215
|
],
|
|
178
216
|
properties: {
|
|
179
217
|
"version": { "const": 1 },
|
|
218
|
+
"variables": {
|
|
219
|
+
"type": "object",
|
|
220
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
221
|
+
"additionalProperties": { "type": "string" }
|
|
222
|
+
},
|
|
180
223
|
"websocket": {
|
|
181
224
|
"type": "object",
|
|
182
225
|
"additionalProperties": false,
|
|
@@ -471,6 +514,8 @@ var WebSocketConfig = class {
|
|
|
471
514
|
var EvalConfig = class {
|
|
472
515
|
version;
|
|
473
516
|
configPath;
|
|
517
|
+
parametersPath;
|
|
518
|
+
variables;
|
|
474
519
|
websocket;
|
|
475
520
|
execution;
|
|
476
521
|
evaluator;
|
|
@@ -479,6 +524,8 @@ var EvalConfig = class {
|
|
|
479
524
|
constructor(data) {
|
|
480
525
|
this.version = data?.version ?? 1;
|
|
481
526
|
this.configPath = data?.configPath ?? "";
|
|
527
|
+
this.parametersPath = data?.parametersPath ?? "";
|
|
528
|
+
this.variables = { ...data?.variables ?? {} };
|
|
482
529
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
483
530
|
this.execution = new ExecutionConfig(data?.execution);
|
|
484
531
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -491,16 +538,80 @@ const ReportFormat = {
|
|
|
491
538
|
Console: "console",
|
|
492
539
|
Json: "json"
|
|
493
540
|
};
|
|
541
|
+
// src/yaml/yamlDocument.ts
|
|
542
|
+
var YamlDocument = class YamlDocument {
|
|
543
|
+
file;
|
|
544
|
+
document;
|
|
545
|
+
lineCounter;
|
|
546
|
+
value;
|
|
547
|
+
errors;
|
|
548
|
+
constructor(file, document, lineCounter) {
|
|
549
|
+
this.file = file;
|
|
550
|
+
this.document = document;
|
|
551
|
+
this.lineCounter = lineCounter;
|
|
552
|
+
this.value = document.toJS();
|
|
553
|
+
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
554
|
+
message: error.message.split("\n")[0],
|
|
555
|
+
location: this.toLocation(error.pos[0], [])
|
|
556
|
+
}));
|
|
557
|
+
}
|
|
558
|
+
locate(path) {
|
|
559
|
+
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
560
|
+
}
|
|
561
|
+
locateKey(path, key) {
|
|
562
|
+
const parent = this.findNode(path);
|
|
563
|
+
if (isMap(parent)) {
|
|
564
|
+
const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
|
|
565
|
+
if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
566
|
+
}
|
|
567
|
+
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
568
|
+
}
|
|
569
|
+
static formatPath(path) {
|
|
570
|
+
return path.map((segment, index) => {
|
|
571
|
+
if (typeof segment === "number") return `[${segment}]`;
|
|
572
|
+
return index === 0 ? segment : `.${segment}`;
|
|
573
|
+
}).join("");
|
|
574
|
+
}
|
|
575
|
+
findNode(path) {
|
|
576
|
+
let node = this.document.contents ?? void 0;
|
|
577
|
+
for (const segment of path) {
|
|
578
|
+
let next;
|
|
579
|
+
if (isMap(node)) {
|
|
580
|
+
const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
|
|
581
|
+
next = pair?.value ?? pair?.key;
|
|
582
|
+
} else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
|
|
583
|
+
if (!next) break;
|
|
584
|
+
node = next;
|
|
585
|
+
}
|
|
586
|
+
return node;
|
|
587
|
+
}
|
|
588
|
+
toLocation(offset, path) {
|
|
589
|
+
const position = this.lineCounter.linePos(offset);
|
|
590
|
+
return new SourceLocation({
|
|
591
|
+
file: this.file,
|
|
592
|
+
line: position.line,
|
|
593
|
+
column: position.col,
|
|
594
|
+
path: YamlDocument.formatPath(path)
|
|
595
|
+
});
|
|
596
|
+
}
|
|
597
|
+
};
|
|
494
598
|
// src/yaml/environmentResolver.ts
|
|
495
|
-
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g;
|
|
599
|
+
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
|
|
496
600
|
const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
|
|
497
601
|
var EnvironmentResolver = class EnvironmentResolver {
|
|
498
602
|
environment;
|
|
499
|
-
|
|
603
|
+
parameters;
|
|
604
|
+
levels;
|
|
605
|
+
constructor(environment = process.env, parameters = {}, levels = []) {
|
|
500
606
|
this.environment = environment;
|
|
607
|
+
this.parameters = parameters;
|
|
608
|
+
this.levels = levels;
|
|
609
|
+
}
|
|
610
|
+
withLevels(levels) {
|
|
611
|
+
return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
|
|
501
612
|
}
|
|
502
|
-
resolve(value, document, issues, secrets = []) {
|
|
503
|
-
return this.resolveValue(value, [], document, issues, secrets);
|
|
613
|
+
resolve(value, document, issues, secrets = [], options = {}) {
|
|
614
|
+
return this.resolveValue(value, [], document, issues, secrets, options);
|
|
504
615
|
}
|
|
505
616
|
static hasReference(value) {
|
|
506
617
|
return new RegExp(REFERENCE.source).test(value);
|
|
@@ -508,21 +619,38 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
508
619
|
static isSecretName(name) {
|
|
509
620
|
return SECRET_NAME.test(name);
|
|
510
621
|
}
|
|
511
|
-
|
|
622
|
+
lookup(name) {
|
|
623
|
+
for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
|
|
624
|
+
return this.environment[name];
|
|
625
|
+
}
|
|
626
|
+
sources() {
|
|
627
|
+
const sources = [
|
|
628
|
+
...this.levels.some((level) => Object.keys(level).length > 0) ? ["variable"] : [],
|
|
629
|
+
...Object.keys(this.parameters).length > 0 ? ["parameter"] : [],
|
|
630
|
+
"environment variable"
|
|
631
|
+
];
|
|
632
|
+
return sources.length === 1 ? sources[0] : `${sources.slice(0, -1).join(", ")} or ${sources[sources.length - 1]}`;
|
|
633
|
+
}
|
|
634
|
+
resolveValue(value, path, document, issues, secrets, options) {
|
|
512
635
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
513
|
-
|
|
636
|
+
options.used?.add(name);
|
|
637
|
+
const resolved = this.lookup(name);
|
|
514
638
|
if (resolved === void 0) {
|
|
639
|
+
if (options.optional?.(path)) {
|
|
640
|
+
options.unresolved?.add(YamlDocument.formatPath(path));
|
|
641
|
+
return reference;
|
|
642
|
+
}
|
|
515
643
|
issues.push(new ValidationIssue({
|
|
516
|
-
message:
|
|
644
|
+
message: `${this.sources()} '${name}' is not set`,
|
|
517
645
|
location: document.locate(path)
|
|
518
646
|
}));
|
|
519
647
|
return reference;
|
|
520
648
|
}
|
|
521
|
-
if (EnvironmentResolver.isSecretName(name) && !secrets.includes(resolved)) secrets.push(resolved);
|
|
649
|
+
if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
|
|
522
650
|
return resolved;
|
|
523
651
|
});
|
|
524
|
-
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets));
|
|
525
|
-
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets)]));
|
|
652
|
+
if (Array.isArray(value)) return value.map((item, index) => this.resolveValue(item, [...path, index], document, issues, secrets, options));
|
|
653
|
+
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, [...path, key], document, issues, secrets, options)]));
|
|
526
654
|
return value;
|
|
527
655
|
}
|
|
528
656
|
};
|
|
@@ -606,63 +734,6 @@ var SchemaValidator = class SchemaValidator {
|
|
|
606
734
|
return path;
|
|
607
735
|
}
|
|
608
736
|
};
|
|
609
|
-
// src/yaml/yamlDocument.ts
|
|
610
|
-
var YamlDocument = class YamlDocument {
|
|
611
|
-
file;
|
|
612
|
-
document;
|
|
613
|
-
lineCounter;
|
|
614
|
-
value;
|
|
615
|
-
errors;
|
|
616
|
-
constructor(file, document, lineCounter) {
|
|
617
|
-
this.file = file;
|
|
618
|
-
this.document = document;
|
|
619
|
-
this.lineCounter = lineCounter;
|
|
620
|
-
this.value = document.toJS();
|
|
621
|
-
this.errors = document.errors.map((error) => new ValidationIssue({
|
|
622
|
-
message: error.message.split("\n")[0],
|
|
623
|
-
location: this.toLocation(error.pos[0], [])
|
|
624
|
-
}));
|
|
625
|
-
}
|
|
626
|
-
locate(path) {
|
|
627
|
-
return this.toLocation(this.findNode(path)?.range?.[0] ?? 0, path);
|
|
628
|
-
}
|
|
629
|
-
locateKey(path, key) {
|
|
630
|
-
const parent = this.findNode(path);
|
|
631
|
-
if (isMap(parent)) {
|
|
632
|
-
const pair = parent.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === key);
|
|
633
|
-
if (pair && isScalar(pair.key)) return this.toLocation(pair.key.range?.[0] ?? 0, [...path, key]);
|
|
634
|
-
}
|
|
635
|
-
return this.toLocation(parent?.range?.[0] ?? 0, [...path, key]);
|
|
636
|
-
}
|
|
637
|
-
static formatPath(path) {
|
|
638
|
-
return path.map((segment, index) => {
|
|
639
|
-
if (typeof segment === "number") return `[${segment}]`;
|
|
640
|
-
return index === 0 ? segment : `.${segment}`;
|
|
641
|
-
}).join("");
|
|
642
|
-
}
|
|
643
|
-
findNode(path) {
|
|
644
|
-
let node = this.document.contents ?? void 0;
|
|
645
|
-
for (const segment of path) {
|
|
646
|
-
let next;
|
|
647
|
-
if (isMap(node)) {
|
|
648
|
-
const pair = node.items.find((item) => isPair(item) && isScalar(item.key) && String(item.key.value) === String(segment));
|
|
649
|
-
next = pair?.value ?? pair?.key;
|
|
650
|
-
} else if (isSeq(node) && typeof segment === "number") next = node.items[segment];
|
|
651
|
-
if (!next) break;
|
|
652
|
-
node = next;
|
|
653
|
-
}
|
|
654
|
-
return node;
|
|
655
|
-
}
|
|
656
|
-
toLocation(offset, path) {
|
|
657
|
-
const position = this.lineCounter.linePos(offset);
|
|
658
|
-
return new SourceLocation({
|
|
659
|
-
file: this.file,
|
|
660
|
-
line: position.line,
|
|
661
|
-
column: position.col,
|
|
662
|
-
path: YamlDocument.formatPath(path)
|
|
663
|
-
});
|
|
664
|
-
}
|
|
665
|
-
};
|
|
666
737
|
// src/yaml/yamlDocumentLoader.ts
|
|
667
738
|
var YamlDocumentLoader = class {
|
|
668
739
|
load(filePath, displayPath) {
|
|
@@ -677,10 +748,126 @@ var YamlDocumentLoader = class {
|
|
|
677
748
|
}), lineCounter);
|
|
678
749
|
}
|
|
679
750
|
};
|
|
680
|
-
// src/
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
751
|
+
// src/yaml/variablesReader.ts
|
|
752
|
+
const VARIABLE_NAME = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
753
|
+
var VariablesReader = class {
|
|
754
|
+
read(document, resolver, issues, secrets) {
|
|
755
|
+
const raw = isRecord(document.value) ? document.value.variables : void 0;
|
|
756
|
+
if (raw === void 0) return {};
|
|
757
|
+
if (!isRecord(raw)) {
|
|
758
|
+
issues.push(new ValidationIssue({
|
|
759
|
+
message: "must be object",
|
|
760
|
+
location: document.locate(["variables"])
|
|
761
|
+
}));
|
|
762
|
+
return {};
|
|
763
|
+
}
|
|
764
|
+
const variables = {};
|
|
765
|
+
for (const [name, value] of Object.entries(raw)) {
|
|
766
|
+
if (!VARIABLE_NAME.test(name)) {
|
|
767
|
+
issues.push(new ValidationIssue({
|
|
768
|
+
message: `variable name '${name}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
769
|
+
location: document.locateKey(["variables"], name)
|
|
770
|
+
}));
|
|
771
|
+
continue;
|
|
772
|
+
}
|
|
773
|
+
if (typeof value !== "string") {
|
|
774
|
+
issues.push(new ValidationIssue({
|
|
775
|
+
message: "must be string",
|
|
776
|
+
location: document.locate(["variables", name])
|
|
777
|
+
}));
|
|
778
|
+
continue;
|
|
779
|
+
}
|
|
780
|
+
const before = issues.length;
|
|
781
|
+
const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
|
|
782
|
+
if (issues.length > before) continue;
|
|
783
|
+
variables[name] = resolved.variables[name];
|
|
784
|
+
if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
|
|
785
|
+
}
|
|
786
|
+
return variables;
|
|
787
|
+
}
|
|
788
|
+
};
|
|
789
|
+
// src/config/webSocketResolver.ts
|
|
790
|
+
const WEBSOCKET_SCHEMA = {
|
|
791
|
+
...eval_config_schema_default,
|
|
792
|
+
required: ["websocket"],
|
|
793
|
+
additionalProperties: true,
|
|
794
|
+
properties: { websocket: eval_config_schema_default.properties.websocket }
|
|
795
|
+
};
|
|
796
|
+
var WebSocketResolver = class WebSocketResolver {
|
|
797
|
+
schemaValidator = new SchemaValidator(WEBSOCKET_SCHEMA);
|
|
798
|
+
resolve(document, resolver, issues, secrets, used, defer) {
|
|
799
|
+
const raw = isRecord(document.value) ? document.value.websocket : void 0;
|
|
800
|
+
const unresolved = /* @__PURE__ */ new Set();
|
|
801
|
+
const optional = (path) => WebSocketResolver.isSubprotocol(path) || defer;
|
|
802
|
+
const value = resolver.resolve({ websocket: raw }, document, issues, secrets, {
|
|
803
|
+
used,
|
|
804
|
+
optional,
|
|
805
|
+
unresolved
|
|
806
|
+
});
|
|
807
|
+
const subprotocols = WebSocketResolver.subprotocols(raw);
|
|
808
|
+
if (subprotocols !== void 0) {
|
|
809
|
+
const kept = WebSocketResolver.subprotocols(value.websocket).filter((entry, index) => {
|
|
810
|
+
const set = !unresolved.has(YamlDocument.formatPath([
|
|
811
|
+
"websocket",
|
|
812
|
+
"connection",
|
|
813
|
+
"subprotocols",
|
|
814
|
+
index
|
|
815
|
+
]));
|
|
816
|
+
const empty = entry === "" && typeof subprotocols[index] === "string" && EnvironmentResolver.hasReference(subprotocols[index]);
|
|
817
|
+
return set && !empty;
|
|
818
|
+
});
|
|
819
|
+
value.websocket.connection.subprotocols = kept;
|
|
820
|
+
}
|
|
821
|
+
const prefix = YamlDocument.formatPath([
|
|
822
|
+
"websocket",
|
|
823
|
+
"connection",
|
|
824
|
+
"subprotocols"
|
|
825
|
+
]);
|
|
826
|
+
return {
|
|
827
|
+
value,
|
|
828
|
+
deferred: new Set([...unresolved].filter((path) => !path.startsWith(`${prefix}[`)))
|
|
829
|
+
};
|
|
830
|
+
}
|
|
831
|
+
validate(value, document) {
|
|
832
|
+
const issues = this.schemaValidator.validate(value, document);
|
|
833
|
+
if (issues.length > 0) return issues;
|
|
834
|
+
return WebSocketResolver.validateInputPath(value, document);
|
|
835
|
+
}
|
|
836
|
+
static validateInputPath(value, document) {
|
|
837
|
+
const request = WebSocketResolver.section(value, "websocket", "protocol", "request");
|
|
838
|
+
if (typeof request.inputPath !== "string") return [];
|
|
839
|
+
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
840
|
+
const last = tokens.pop();
|
|
841
|
+
let parent = request.value;
|
|
842
|
+
for (const token of tokens) parent = WebSocketResolver.child(parent, token);
|
|
843
|
+
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
844
|
+
return [new ValidationIssue({
|
|
845
|
+
message: "must point to a missing or string value whose parent exists in request.value",
|
|
846
|
+
location: document.locate([
|
|
847
|
+
"websocket",
|
|
848
|
+
"protocol",
|
|
849
|
+
"request",
|
|
850
|
+
"inputPath"
|
|
851
|
+
])
|
|
852
|
+
})];
|
|
853
|
+
}
|
|
854
|
+
static section(value, ...path) {
|
|
855
|
+
let current = value;
|
|
856
|
+
for (const key of path) current = WebSocketResolver.child(current, key);
|
|
857
|
+
return isRecord(current) ? current : {};
|
|
858
|
+
}
|
|
859
|
+
static child(value, key) {
|
|
860
|
+
if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
|
|
861
|
+
return isRecord(value) ? value[key] : void 0;
|
|
862
|
+
}
|
|
863
|
+
static isSubprotocol(path) {
|
|
864
|
+
return path.length === 4 && path[0] === "websocket" && path[1] === "connection" && path[2] === "subprotocols" && typeof path[3] === "number";
|
|
865
|
+
}
|
|
866
|
+
static subprotocols(websocket) {
|
|
867
|
+
const subprotocols = WebSocketResolver.section(websocket, "connection").subprotocols;
|
|
868
|
+
return Array.isArray(subprotocols) ? subprotocols : void 0;
|
|
869
|
+
}
|
|
870
|
+
};
|
|
684
871
|
// src/config/defaultConfig.ts
|
|
685
872
|
const DEFAULT_CONFIG = new EvalConfig({
|
|
686
873
|
execution: new ExecutionConfig({
|
|
@@ -716,29 +903,59 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
716
903
|
cwd;
|
|
717
904
|
documentLoader = new YamlDocumentLoader();
|
|
718
905
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
906
|
+
variablesReader = new VariablesReader();
|
|
907
|
+
webSocketResolver = new WebSocketResolver();
|
|
719
908
|
environmentResolver;
|
|
720
|
-
constructor(environment = process.env, cwd = process.cwd()) {
|
|
909
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
721
910
|
this.cwd = cwd;
|
|
722
|
-
this.environmentResolver = new EnvironmentResolver(environment);
|
|
911
|
+
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
723
912
|
}
|
|
724
913
|
load(configPath, override, report = new ReportOverride()) {
|
|
725
914
|
const document = this.documentLoader.load(configPath, relative(this.cwd, configPath));
|
|
726
915
|
const issues = [...document.errors];
|
|
727
916
|
if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
|
|
728
917
|
const named = [];
|
|
729
|
-
const
|
|
730
|
-
|
|
731
|
-
if (issues.length === 0)
|
|
918
|
+
const variables = issues.length === 0 ? this.variablesReader.read(document, this.environmentResolver, issues, named) : {};
|
|
919
|
+
let value = {};
|
|
920
|
+
if (issues.length === 0) {
|
|
921
|
+
const resolver = this.environmentResolver.withLevels([variables]);
|
|
922
|
+
const { websocket: _websocket, variables: _variables, ...rest } = document.value;
|
|
923
|
+
const resolved = resolver.resolve(rest, document, issues, named);
|
|
924
|
+
const websocket = this.webSocketResolver.resolve(document, resolver, issues, named, /* @__PURE__ */ new Set(), true);
|
|
925
|
+
value = {
|
|
926
|
+
...resolved,
|
|
927
|
+
..._variables === void 0 ? {} : { variables },
|
|
928
|
+
..._websocket === void 0 ? {} : websocket.value
|
|
929
|
+
};
|
|
930
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document).filter((issue) => !websocket.deferred.has(issue.location.path)));
|
|
931
|
+
if (issues.length === 0 && websocket.deferred.size === 0) issues.push(...WebSocketResolver.validateInputPath(value, document));
|
|
932
|
+
}
|
|
732
933
|
if (issues.length > 0) throw new EvalError({
|
|
733
934
|
category: ErrorCategory.Configuration,
|
|
734
935
|
message: "invalid configuration",
|
|
735
936
|
issues
|
|
736
937
|
});
|
|
737
|
-
const config = this.build(value, configPath, named);
|
|
938
|
+
const config = this.build(value, configPath, variables, named);
|
|
738
939
|
this.applyRepetition(config, override, value, document);
|
|
739
940
|
EvalConfigLoader.applyReport(config, report);
|
|
740
941
|
return config;
|
|
741
942
|
}
|
|
943
|
+
static buildWebSocket(value) {
|
|
944
|
+
const section = (...path) => WebSocketResolver.section(value, ...path);
|
|
945
|
+
const connection = section("websocket", "connection");
|
|
946
|
+
const request = section("websocket", "protocol", "request");
|
|
947
|
+
return new WebSocketConfig({
|
|
948
|
+
endpoint: WebSocketResolver.section(value, "websocket").endpoint,
|
|
949
|
+
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
950
|
+
protocol: new ProtocolConfig({
|
|
951
|
+
request: new RequestConfig({
|
|
952
|
+
...request,
|
|
953
|
+
format: request.format ?? RequestFormat.Json
|
|
954
|
+
}),
|
|
955
|
+
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
956
|
+
})
|
|
957
|
+
});
|
|
958
|
+
}
|
|
742
959
|
static applyReport(config, report) {
|
|
743
960
|
if (report.formats === void 0) return;
|
|
744
961
|
const known = Object.values(ReportFormat);
|
|
@@ -756,49 +973,21 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
756
973
|
const apiKey = EvalConfigLoader.section(document.value, "evaluator").apiKey;
|
|
757
974
|
if (typeof apiKey !== "string" || EnvironmentResolver.hasReference(apiKey)) return [];
|
|
758
975
|
return [new ValidationIssue({
|
|
759
|
-
message: "must be
|
|
976
|
+
message: "must be a reference such as ${OPENAI_API_KEY}",
|
|
760
977
|
location: document.locate(["evaluator", "apiKey"])
|
|
761
978
|
})];
|
|
762
979
|
}
|
|
763
|
-
|
|
764
|
-
const request = EvalConfigLoader.section(value, "websocket", "protocol", "request");
|
|
765
|
-
if (typeof request.inputPath !== "string") return [];
|
|
766
|
-
const tokens = request.inputPath.split("/").slice(1).map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
767
|
-
const last = tokens.pop();
|
|
768
|
-
let parent = request.value;
|
|
769
|
-
for (const token of tokens) parent = EvalConfigLoader.child(parent, token);
|
|
770
|
-
if (Array.isArray(parent) ? last === "-" || /^(0|[1-9][0-9]*)$/.test(last) && Number(last) <= parent.length && (Number(last) === parent.length || typeof parent[Number(last)] === "string") : isRecord(parent) && (parent[last] === void 0 || typeof parent[last] === "string")) return [];
|
|
771
|
-
return [new ValidationIssue({
|
|
772
|
-
message: "must point to a missing or string value whose parent exists in request.value",
|
|
773
|
-
location: document.locate([
|
|
774
|
-
"websocket",
|
|
775
|
-
"protocol",
|
|
776
|
-
"request",
|
|
777
|
-
"inputPath"
|
|
778
|
-
])
|
|
779
|
-
})];
|
|
780
|
-
}
|
|
781
|
-
build(value, configPath, named) {
|
|
980
|
+
build(value, configPath, variables, named) {
|
|
782
981
|
const section = (...path) => EvalConfigLoader.section(value, ...path);
|
|
783
982
|
const configDirectory = dirname(configPath);
|
|
784
|
-
const
|
|
785
|
-
const request = section("websocket", "protocol", "request");
|
|
983
|
+
const websocket = EvalConfigLoader.buildWebSocket(value);
|
|
786
984
|
const evaluator = section("evaluator");
|
|
787
985
|
const report = section("report");
|
|
788
986
|
return new EvalConfig({
|
|
789
987
|
version: value.version,
|
|
790
988
|
configPath,
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
connection: new ConnectionConfig({ subprotocols: [...connection.subprotocols ?? []] }),
|
|
794
|
-
protocol: new ProtocolConfig({
|
|
795
|
-
request: new RequestConfig({
|
|
796
|
-
...request,
|
|
797
|
-
format: request.format ?? RequestFormat.Json
|
|
798
|
-
}),
|
|
799
|
-
capabilities: new ProtocolCapabilities(section("websocket", "protocol", "capabilities"))
|
|
800
|
-
})
|
|
801
|
-
}),
|
|
989
|
+
variables,
|
|
990
|
+
websocket,
|
|
802
991
|
execution: new ExecutionConfig({
|
|
803
992
|
...DEFAULT_CONFIG.execution,
|
|
804
993
|
...section("execution"),
|
|
@@ -819,7 +1008,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
819
1008
|
directory: resolve(configDirectory, report.directory ?? DEFAULT_CONFIG.report.directory),
|
|
820
1009
|
formats: [...report.formats ?? DEFAULT_CONFIG.report.formats]
|
|
821
1010
|
}),
|
|
822
|
-
secrets: EvalConfigLoader.secrets(evaluator.apiKey, connection.subprotocols
|
|
1011
|
+
secrets: EvalConfigLoader.secrets(evaluator.apiKey, websocket.connection.subprotocols, named)
|
|
823
1012
|
});
|
|
824
1013
|
}
|
|
825
1014
|
applyRepetition(config, override, value, document) {
|
|
@@ -872,13 +1061,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
872
1061
|
])];
|
|
873
1062
|
}
|
|
874
1063
|
static section(value, ...path) {
|
|
875
|
-
|
|
876
|
-
for (const key of path) current = EvalConfigLoader.child(current, key);
|
|
877
|
-
return isRecord(current) ? current : {};
|
|
878
|
-
}
|
|
879
|
-
static child(value, key) {
|
|
880
|
-
if (Array.isArray(value)) return /^(0|[1-9][0-9]*)$/.test(key) ? value[Number(key)] : void 0;
|
|
881
|
-
return isRecord(value) ? value[key] : void 0;
|
|
1064
|
+
return WebSocketResolver.section(value, ...path);
|
|
882
1065
|
}
|
|
883
1066
|
};
|
|
884
1067
|
// src/interfaces/runStatus.ts
|
|
@@ -1074,6 +1257,7 @@ var ConversationSpec = class {
|
|
|
1074
1257
|
turns;
|
|
1075
1258
|
conversation;
|
|
1076
1259
|
location;
|
|
1260
|
+
websocket;
|
|
1077
1261
|
secrets;
|
|
1078
1262
|
constructor(data) {
|
|
1079
1263
|
this.version = data?.version ?? 1;
|
|
@@ -1082,6 +1266,7 @@ var ConversationSpec = class {
|
|
|
1082
1266
|
this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
|
|
1083
1267
|
this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
|
|
1084
1268
|
this.location = new SourceLocation(data?.location);
|
|
1269
|
+
this.websocket = data?.websocket ? new WebSocketConfig(data.websocket) : void 0;
|
|
1085
1270
|
this.secrets = [...data?.secrets ?? []];
|
|
1086
1271
|
}
|
|
1087
1272
|
};
|
|
@@ -2381,7 +2566,11 @@ var ConversationRunner = class {
|
|
|
2381
2566
|
async run(spec, signal = NEVER_ABORTED) {
|
|
2382
2567
|
const startedAt = Date.now();
|
|
2383
2568
|
const outcomes = [];
|
|
2384
|
-
const
|
|
2569
|
+
const config = new EvalConfig({
|
|
2570
|
+
...this.config,
|
|
2571
|
+
websocket: spec.websocket ?? this.config.websocket
|
|
2572
|
+
});
|
|
2573
|
+
const assertions = new AssertionRunner(config.websocket.protocol.capabilities);
|
|
2385
2574
|
const history = [];
|
|
2386
2575
|
const evaluate = this.semanticEvaluator(signal);
|
|
2387
2576
|
const inline = !this.config.execution.continueOnFailure;
|
|
@@ -2406,7 +2595,7 @@ var ConversationRunner = class {
|
|
|
2406
2595
|
try {
|
|
2407
2596
|
const steps = (async () => {
|
|
2408
2597
|
for (let attempt = 0;; attempt++) {
|
|
2409
|
-
connection = this.createConnection(
|
|
2598
|
+
connection = this.createConnection(config);
|
|
2410
2599
|
connectionAttempts++;
|
|
2411
2600
|
try {
|
|
2412
2601
|
await connection.connect();
|
|
@@ -2587,7 +2776,7 @@ var ScenarioRunner = class {
|
|
|
2587
2776
|
required: minPass,
|
|
2588
2777
|
attempts,
|
|
2589
2778
|
tests: this.results.tests(attempts, minPass),
|
|
2590
|
-
agentUsage: this.tokens.agent(attempts, this.config.websocket.protocol.capabilities.tokenUsage),
|
|
2779
|
+
agentUsage: this.tokens.agent(attempts, (spec.websocket ?? this.config.websocket).protocol.capabilities.tokenUsage),
|
|
2591
2780
|
evaluatorUsage: this.tokens.evaluator(attempts)
|
|
2592
2781
|
});
|
|
2593
2782
|
}
|
|
@@ -2749,7 +2938,9 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
2749
2938
|
const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
|
|
2750
2939
|
if (!this.verbose) return line;
|
|
2751
2940
|
if (result.attempts.length === 0) return `${line}\n`;
|
|
2752
|
-
const
|
|
2941
|
+
const websocket = spec.websocket;
|
|
2942
|
+
const subprotocols = websocket?.connection.subprotocols.length ?? 0;
|
|
2943
|
+
const lines = ["", ...websocket === void 0 ? [] : [` Connection: ${websocket.endpoint}, ${subprotocols} ${subprotocols === 1 ? "subprotocol" : "subprotocols"}`, ""]];
|
|
2753
2944
|
for (const attempt of result.attempts) lines.push(` Attempt ${attempt.attempt} of ${repeat}: ${this.status(attempt.status)}`, ...this.formatAttempt(spec, attempt), "");
|
|
2754
2945
|
lines.push(` Result: ${this.status(result.status)} ${ConsoleReporter.counts(result)}`);
|
|
2755
2946
|
lines.push(...result.tests.map((test) => ` ${this.status(test.status, 8)} ${test.title} ${test.location.toString()} ${ConsoleReporter.counts(test)}`));
|
|
@@ -2928,6 +3119,7 @@ var JsonReporter = class JsonReporter {
|
|
|
2928
3119
|
const { websocket, execution, evaluator, report } = config;
|
|
2929
3120
|
return {
|
|
2930
3121
|
path: this.path(config.configPath),
|
|
3122
|
+
parametersPath: config.parametersPath === "" ? void 0 : this.path(config.parametersPath),
|
|
2931
3123
|
websocket: {
|
|
2932
3124
|
endpoint: websocket.endpoint,
|
|
2933
3125
|
subprotocols: [...websocket.connection.subprotocols],
|
|
@@ -2960,11 +3152,16 @@ var JsonReporter = class JsonReporter {
|
|
|
2960
3152
|
};
|
|
2961
3153
|
}
|
|
2962
3154
|
scenario(result, config) {
|
|
2963
|
-
const
|
|
3155
|
+
const websocket = result.spec.websocket ?? config.websocket;
|
|
3156
|
+
const capability = websocket.protocol.capabilities.tokenUsage;
|
|
2964
3157
|
return {
|
|
2965
3158
|
name: result.spec.name,
|
|
2966
3159
|
file: result.spec.location.file,
|
|
2967
3160
|
status: result.status,
|
|
3161
|
+
connection: {
|
|
3162
|
+
endpoint: websocket.endpoint,
|
|
3163
|
+
subprotocols: websocket.connection.subprotocols.length
|
|
3164
|
+
},
|
|
2968
3165
|
...JsonReporter.counts(result),
|
|
2969
3166
|
agentUsage: JsonReporter.usage(result.agentUsage),
|
|
2970
3167
|
evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
|
|
@@ -3125,6 +3322,11 @@ var conversation_schema_default = {
|
|
|
3125
3322
|
"version": { "const": 1 },
|
|
3126
3323
|
"name": { "$ref": "#/definitions/text" },
|
|
3127
3324
|
"description": { "type": "string" },
|
|
3325
|
+
"variables": {
|
|
3326
|
+
"type": "object",
|
|
3327
|
+
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
3328
|
+
"additionalProperties": { "type": "string" }
|
|
3329
|
+
},
|
|
3128
3330
|
"turns": {
|
|
3129
3331
|
"type": "array",
|
|
3130
3332
|
"minItems": 1,
|
|
@@ -3137,8 +3339,8 @@ var conversation_schema_default = {
|
|
|
3137
3339
|
"properties": { "conversation": {
|
|
3138
3340
|
"type": "object",
|
|
3139
3341
|
"additionalProperties": false,
|
|
3140
|
-
"required": ["
|
|
3141
|
-
"properties": { "
|
|
3342
|
+
"required": ["should"],
|
|
3343
|
+
"properties": { "should": { "$ref": "#/definitions/text" } }
|
|
3142
3344
|
} }
|
|
3143
3345
|
}
|
|
3144
3346
|
},
|
|
@@ -3181,12 +3383,12 @@ var conversation_schema_default = {
|
|
|
3181
3383
|
}
|
|
3182
3384
|
},
|
|
3183
3385
|
"response": {
|
|
3184
|
-
"description": "A string is shorthand for
|
|
3386
|
+
"description": "A string is shorthand for should.",
|
|
3185
3387
|
"type": ["string", "object"],
|
|
3186
3388
|
"pattern": "\\S",
|
|
3187
3389
|
"additionalProperties": false,
|
|
3188
3390
|
"properties": {
|
|
3189
|
-
"
|
|
3391
|
+
"should": { "$ref": "#/definitions/text" },
|
|
3190
3392
|
"contains": { "$ref": "#/definitions/textList" },
|
|
3191
3393
|
"notContains": { "$ref": "#/definitions/textList" }
|
|
3192
3394
|
}
|
|
@@ -3218,7 +3420,7 @@ var conversation_schema_default = {
|
|
|
3218
3420
|
};
|
|
3219
3421
|
// src/scenarios/conversationValidator.ts
|
|
3220
3422
|
const RESPONSE_KEYS = [
|
|
3221
|
-
"
|
|
3423
|
+
"should",
|
|
3222
3424
|
"contains",
|
|
3223
3425
|
"notContains"
|
|
3224
3426
|
];
|
|
@@ -3241,7 +3443,7 @@ var ConversationValidator = class ConversationValidator {
|
|
|
3241
3443
|
validateResponse(response, path, document, issues) {
|
|
3242
3444
|
if (!RESPONSE_KEYS.some((key) => key in response)) {
|
|
3243
3445
|
issues.push(new ValidationIssue({
|
|
3244
|
-
message: "response must have at least one of:
|
|
3446
|
+
message: "response must have at least one of: should, contains, notContains",
|
|
3245
3447
|
location: document.locate(path)
|
|
3246
3448
|
}));
|
|
3247
3449
|
return;
|
|
@@ -3284,30 +3486,66 @@ var ConversationValidator = class ConversationValidator {
|
|
|
3284
3486
|
};
|
|
3285
3487
|
// src/scenarios/scenarioLoader.ts
|
|
3286
3488
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
3287
|
-
var ScenarioLoader = class {
|
|
3489
|
+
var ScenarioLoader = class ScenarioLoader {
|
|
3288
3490
|
cwd;
|
|
3289
3491
|
documentLoader = new YamlDocumentLoader();
|
|
3290
3492
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
3291
3493
|
conversationValidator = new ConversationValidator();
|
|
3494
|
+
variablesReader = new VariablesReader();
|
|
3495
|
+
webSocketResolver = new WebSocketResolver();
|
|
3292
3496
|
environmentResolver;
|
|
3293
|
-
constructor(environment = process.env, cwd = process.cwd()) {
|
|
3497
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
3294
3498
|
this.cwd = cwd;
|
|
3295
|
-
this.environmentResolver = new EnvironmentResolver(environment);
|
|
3499
|
+
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
3296
3500
|
}
|
|
3297
3501
|
load(target, config) {
|
|
3298
3502
|
const files = this.discover(resolve(this.cwd, target), config);
|
|
3503
|
+
const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.cwd, config.configPath)) : void 0;
|
|
3299
3504
|
const issues = [];
|
|
3505
|
+
const connectionIssues = [];
|
|
3300
3506
|
const specs = [];
|
|
3301
3507
|
for (const file of files) {
|
|
3302
3508
|
const document = this.documentLoader.load(file, relative(this.cwd, file));
|
|
3303
3509
|
const fileIssues = [...document.errors];
|
|
3304
3510
|
const secrets = [];
|
|
3305
|
-
const
|
|
3511
|
+
const used = /* @__PURE__ */ new Set();
|
|
3512
|
+
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
|
|
3513
|
+
const resolver = this.environmentResolver.withLevels([variables, config.variables]);
|
|
3514
|
+
let value;
|
|
3515
|
+
if (fileIssues.length === 0) {
|
|
3516
|
+
const { variables: raw, ...rest } = document.value;
|
|
3517
|
+
value = {
|
|
3518
|
+
...resolver.resolve(rest, document, fileIssues, secrets, { used }),
|
|
3519
|
+
...raw === void 0 ? {} : { variables }
|
|
3520
|
+
};
|
|
3521
|
+
}
|
|
3522
|
+
let websocket;
|
|
3523
|
+
const websocketIssues = [];
|
|
3524
|
+
if (fileIssues.length === 0 && configDocument !== void 0) {
|
|
3525
|
+
const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
|
|
3526
|
+
if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
|
|
3527
|
+
connectionIssues.push({
|
|
3528
|
+
file: document,
|
|
3529
|
+
issues: websocketIssues
|
|
3530
|
+
});
|
|
3531
|
+
if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
|
|
3532
|
+
}
|
|
3306
3533
|
if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
|
|
3307
3534
|
if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
|
|
3308
|
-
if (fileIssues.length === 0)
|
|
3535
|
+
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
|
|
3536
|
+
if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
3309
3537
|
else issues.push(...fileIssues);
|
|
3310
3538
|
}
|
|
3539
|
+
const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
|
|
3540
|
+
if (shared.length > 0) throw new EvalError({
|
|
3541
|
+
category: ErrorCategory.Configuration,
|
|
3542
|
+
message: "invalid configuration",
|
|
3543
|
+
issues: shared
|
|
3544
|
+
});
|
|
3545
|
+
for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
|
|
3546
|
+
message: issue.toString(),
|
|
3547
|
+
location: ScenarioLoader.variablesLocation(entry.file)
|
|
3548
|
+
}));
|
|
3311
3549
|
if (issues.length > 0) throw new EvalError({
|
|
3312
3550
|
category: ErrorCategory.Specification,
|
|
3313
3551
|
message: "invalid scenario specification",
|
|
@@ -3315,6 +3553,23 @@ var ScenarioLoader = class {
|
|
|
3315
3553
|
});
|
|
3316
3554
|
return specs;
|
|
3317
3555
|
}
|
|
3556
|
+
static shared(lists) {
|
|
3557
|
+
if (lists.length === 0) return [];
|
|
3558
|
+
const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
|
|
3559
|
+
return lists[0].filter((issue) => texts.every((set) => set.has(issue.toString())));
|
|
3560
|
+
}
|
|
3561
|
+
static variablesLocation(document) {
|
|
3562
|
+
return new SourceLocation({
|
|
3563
|
+
...isRecord(document.value) && document.value.variables !== void 0 ? document.locateKey([], "variables") : document.locate([]),
|
|
3564
|
+
path: ""
|
|
3565
|
+
});
|
|
3566
|
+
}
|
|
3567
|
+
static unusedVariables(variables, used, document) {
|
|
3568
|
+
return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
|
|
3569
|
+
message: `variable '${name}' is not used by the scenario or the websocket configuration`,
|
|
3570
|
+
location: document.locateKey(["variables"], name)
|
|
3571
|
+
}));
|
|
3572
|
+
}
|
|
3318
3573
|
discover(target, config) {
|
|
3319
3574
|
if (!existsSync(target)) throw new EvalError({
|
|
3320
3575
|
category: ErrorCategory.Specification,
|
|
@@ -3339,7 +3594,7 @@ var ScenarioLoader = class {
|
|
|
3339
3594
|
}
|
|
3340
3595
|
return files;
|
|
3341
3596
|
}
|
|
3342
|
-
build(value, document, secrets) {
|
|
3597
|
+
build(value, document, websocket, secrets) {
|
|
3343
3598
|
const scenario = value;
|
|
3344
3599
|
const turns = scenario.turns;
|
|
3345
3600
|
const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
|
|
@@ -3349,10 +3604,11 @@ var ScenarioLoader = class {
|
|
|
3349
3604
|
description: typeof scenario.description === "string" ? scenario.description : "",
|
|
3350
3605
|
turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
|
|
3351
3606
|
conversation: conversation ? new ConversationExpectation({
|
|
3352
|
-
criteria: conversation.
|
|
3607
|
+
criteria: conversation.should,
|
|
3353
3608
|
location: document.locate(["expect", "conversation"])
|
|
3354
3609
|
}) : void 0,
|
|
3355
3610
|
location: document.locate([]),
|
|
3611
|
+
websocket,
|
|
3356
3612
|
secrets
|
|
3357
3613
|
});
|
|
3358
3614
|
}
|
|
@@ -3378,7 +3634,7 @@ var ScenarioLoader = class {
|
|
|
3378
3634
|
});
|
|
3379
3635
|
const fields = response;
|
|
3380
3636
|
return new ResponseExpectation({
|
|
3381
|
-
criteria: fields.
|
|
3637
|
+
criteria: fields.should,
|
|
3382
3638
|
contains: fields.contains ?? [],
|
|
3383
3639
|
notContains: fields.notContains ?? [],
|
|
3384
3640
|
location
|
|
@@ -3427,15 +3683,20 @@ var RunCommand = class RunCommand {
|
|
|
3427
3683
|
this.signals = signals;
|
|
3428
3684
|
this.colour = colour;
|
|
3429
3685
|
}
|
|
3430
|
-
async execute(target, config, execution = new ExecutionOverride(), report = new ReportOverride()) {
|
|
3686
|
+
async execute(target, config, execution = new ExecutionOverride(), report = new ReportOverride(), parameters = "") {
|
|
3431
3687
|
let evalConfig;
|
|
3432
3688
|
let specs;
|
|
3433
3689
|
let evaluator;
|
|
3434
3690
|
let redactor = new SecretRedactor([]);
|
|
3435
3691
|
try {
|
|
3436
3692
|
const configPath = new ConfigLocator(this.cwd).locate(target, config);
|
|
3437
|
-
|
|
3438
|
-
|
|
3693
|
+
const parameterFile = parameters === "" ? {
|
|
3694
|
+
path: "",
|
|
3695
|
+
parameters: {}
|
|
3696
|
+
} : new ParametersLoader(this.cwd).load(parameters);
|
|
3697
|
+
evalConfig = new EvalConfigLoader(this.environment, this.cwd, parameterFile.parameters).load(configPath, execution, report);
|
|
3698
|
+
evalConfig.parametersPath = parameterFile.path;
|
|
3699
|
+
specs = new ScenarioLoader(this.environment, this.cwd, parameterFile.parameters).load(target, evalConfig);
|
|
3439
3700
|
redactor = new SecretRedactor([...evalConfig.secrets, ...specs.flatMap((spec) => spec.secrets)]);
|
|
3440
3701
|
evaluator = RunCommand.needsEvaluator(specs) ? await this.createEvaluator(evalConfig) : new UnavailableEvaluator();
|
|
3441
3702
|
} catch (error) {
|
|
@@ -3524,11 +3785,12 @@ var ValidateCommand = class {
|
|
|
3524
3785
|
this.environment = environment;
|
|
3525
3786
|
this.cwd = cwd;
|
|
3526
3787
|
}
|
|
3527
|
-
execute(target, config) {
|
|
3788
|
+
execute(target, config, parameters = "") {
|
|
3528
3789
|
try {
|
|
3529
3790
|
const configPath = new ConfigLocator(this.cwd).locate(target, config);
|
|
3530
|
-
const
|
|
3531
|
-
const
|
|
3791
|
+
const values = parameters === "" ? {} : new ParametersLoader(this.cwd).load(parameters).parameters;
|
|
3792
|
+
const evalConfig = new EvalConfigLoader(this.environment, this.cwd, values).load(configPath, new ExecutionOverride());
|
|
3793
|
+
const specs = new ScenarioLoader(this.environment, this.cwd, values).load(target, evalConfig);
|
|
3532
3794
|
this.out(`Valid: ${specs.length} scenario${specs.length === 1 ? "" : "s"}\n`);
|
|
3533
3795
|
return 0;
|
|
3534
3796
|
} catch (error) {
|
|
@@ -3545,10 +3807,10 @@ function createProgram(setExitCode, out, err, colour = false) {
|
|
|
3545
3807
|
writeOut: out,
|
|
3546
3808
|
writeErr: err
|
|
3547
3809
|
}).exitOverride();
|
|
3548
|
-
program.command("validate").description("Parse, interpolate and validate configuration and scenarios with no network calls").argument("<target>", "scenario file or directory").option("--config <path>", "configuration file (default: search upward for eval-config.yml)").action((target, options) => {
|
|
3549
|
-
setExitCode(new ValidateCommand(out, err).execute(target, options.config ?? ""));
|
|
3810
|
+
program.command("validate").description("Parse, interpolate and validate configuration and scenarios with no network calls").argument("<target>", "scenario file or directory").option("--config <path>", "configuration file (default: search upward for eval-config.yml)").option("--parameters <path>", "JSON file whose string values resolve ${name} references before the environment").action((target, options) => {
|
|
3811
|
+
setExitCode(new ValidateCommand(out, err).execute(target, options.config ?? "", options.parameters ?? ""));
|
|
3550
3812
|
});
|
|
3551
|
-
program.command("run").description("Run every scenario against the agent, repeat times each, and report the results").argument("<target>", "scenario file or directory").option("--config <path>", "configuration file (default: search upward for eval-config.yml)").option("--repeat <n>", "attempts per scenario (overrides execution.repeat)").option("--min-pass <n>", "passing attempts a scenario needs (overrides execution.minPass)").option("--quick", "shorthand for --repeat 1 --min-pass 1; wins over both").option("--concurrency <n>", "scenarios run at once (overrides execution.scenarioConcurrency)").option("--report <formats>", "comma-separated console, json (overrides report.formats)").option("--verbose", "include every attempt's transcript in the console report").action(async (target, options) => {
|
|
3813
|
+
program.command("run").description("Run every scenario against the agent, repeat times each, and report the results").argument("<target>", "scenario file or directory").option("--config <path>", "configuration file (default: search upward for eval-config.yml)").option("--parameters <path>", "JSON file whose string values resolve ${name} references before the environment").option("--repeat <n>", "attempts per scenario (overrides execution.repeat)").option("--min-pass <n>", "passing attempts a scenario needs (overrides execution.minPass)").option("--quick", "shorthand for --repeat 1 --min-pass 1; wins over both").option("--concurrency <n>", "scenarios run at once (overrides execution.scenarioConcurrency)").option("--report <formats>", "comma-separated console, json (overrides report.formats)").option("--verbose", "include every attempt's transcript in the console report").action(async (target, options) => {
|
|
3552
3814
|
let execution;
|
|
3553
3815
|
let report;
|
|
3554
3816
|
try {
|
|
@@ -3560,7 +3822,7 @@ function createProgram(setExitCode, out, err, colour = false) {
|
|
|
3560
3822
|
setExitCode(1);
|
|
3561
3823
|
return;
|
|
3562
3824
|
}
|
|
3563
|
-
setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour).execute(target, options.config ?? "", execution, report));
|
|
3825
|
+
setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour).execute(target, options.config ?? "", execution, report, options.parameters ?? ""));
|
|
3564
3826
|
});
|
|
3565
3827
|
return program;
|
|
3566
3828
|
}
|