@mappies/miniwat-eval 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -5
- package/dist/index.cjs +393 -44
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +9 -2
- package/dist/index.d.ts +10 -2
- package/dist/index.js +393 -44
- package/dist/index.js.map +1 -1
- package/dist/main.js +393 -44
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +35 -1
- package/schemas/eval-config.schema.json +6 -1
- package/schemas/report.schema.json +1 -0
package/dist/main.js
CHANGED
|
@@ -218,7 +218,10 @@ var eval_config_schema_default = {
|
|
|
218
218
|
"variables": {
|
|
219
219
|
"type": "object",
|
|
220
220
|
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
221
|
-
"additionalProperties": { "type": "string" }
|
|
221
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
222
|
+
"type": "array",
|
|
223
|
+
"items": { "type": "string" }
|
|
224
|
+
}] }
|
|
222
225
|
},
|
|
223
226
|
"websocket": {
|
|
224
227
|
"type": "object",
|
|
@@ -525,7 +528,7 @@ var EvalConfig = class {
|
|
|
525
528
|
this.version = data?.version ?? 1;
|
|
526
529
|
this.configPath = data?.configPath ?? "";
|
|
527
530
|
this.parametersPath = data?.parametersPath ?? "";
|
|
528
|
-
this.variables =
|
|
531
|
+
this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
|
|
529
532
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
530
533
|
this.execution = new ExecutionConfig(data?.execution);
|
|
531
534
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -611,7 +614,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
611
614
|
return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
|
|
612
615
|
}
|
|
613
616
|
resolve(value, document, issues, secrets = [], options = {}) {
|
|
614
|
-
return this.resolveValue(value, [], document, issues, secrets, options);
|
|
617
|
+
return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
|
|
615
618
|
}
|
|
616
619
|
static hasReference(value) {
|
|
617
620
|
return new RegExp(REFERENCE.source).test(value);
|
|
@@ -619,7 +622,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
619
622
|
static isSecretName(name) {
|
|
620
623
|
return SECRET_NAME.test(name);
|
|
621
624
|
}
|
|
622
|
-
|
|
625
|
+
find(name) {
|
|
623
626
|
for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
|
|
624
627
|
return this.environment[name];
|
|
625
628
|
}
|
|
@@ -634,7 +637,7 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
634
637
|
resolveValue(value, path, document, issues, secrets, options) {
|
|
635
638
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
636
639
|
options.used?.add(name);
|
|
637
|
-
const resolved = this.
|
|
640
|
+
const resolved = this.find(name);
|
|
638
641
|
if (resolved === void 0) {
|
|
639
642
|
if (options.optional?.(path)) {
|
|
640
643
|
options.unresolved?.add(YamlDocument.formatPath(path));
|
|
@@ -646,6 +649,13 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
646
649
|
}));
|
|
647
650
|
return reference;
|
|
648
651
|
}
|
|
652
|
+
if (Array.isArray(resolved)) {
|
|
653
|
+
issues.push(new ValidationIssue({
|
|
654
|
+
message: `variable '${name}' is a list, which can only be used as runWith.from`,
|
|
655
|
+
location: document.locate(path)
|
|
656
|
+
}));
|
|
657
|
+
return reference;
|
|
658
|
+
}
|
|
649
659
|
if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
|
|
650
660
|
return resolved;
|
|
651
661
|
});
|
|
@@ -770,9 +780,25 @@ var VariablesReader = class {
|
|
|
770
780
|
}));
|
|
771
781
|
continue;
|
|
772
782
|
}
|
|
783
|
+
if (Array.isArray(value)) {
|
|
784
|
+
const bad = value.findIndex((item) => typeof item !== "string");
|
|
785
|
+
if (bad >= 0) {
|
|
786
|
+
issues.push(new ValidationIssue({
|
|
787
|
+
message: "must be string",
|
|
788
|
+
location: document.locate([
|
|
789
|
+
"variables",
|
|
790
|
+
name,
|
|
791
|
+
bad
|
|
792
|
+
])
|
|
793
|
+
}));
|
|
794
|
+
continue;
|
|
795
|
+
}
|
|
796
|
+
variables[name] = [...value];
|
|
797
|
+
continue;
|
|
798
|
+
}
|
|
773
799
|
if (typeof value !== "string") {
|
|
774
800
|
issues.push(new ValidationIssue({
|
|
775
|
-
message: "must be string",
|
|
801
|
+
message: "must be string or a list of strings",
|
|
776
802
|
location: document.locate(["variables", name])
|
|
777
803
|
}));
|
|
778
804
|
continue;
|
|
@@ -780,8 +806,9 @@ var VariablesReader = class {
|
|
|
780
806
|
const before = issues.length;
|
|
781
807
|
const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
|
|
782
808
|
if (issues.length > before) continue;
|
|
783
|
-
|
|
784
|
-
|
|
809
|
+
const text = resolved.variables[name];
|
|
810
|
+
variables[name] = text;
|
|
811
|
+
if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
|
|
785
812
|
}
|
|
786
813
|
return variables;
|
|
787
814
|
}
|
|
@@ -1253,6 +1280,7 @@ var TurnSpec = class {
|
|
|
1253
1280
|
var ConversationSpec = class {
|
|
1254
1281
|
version;
|
|
1255
1282
|
name;
|
|
1283
|
+
runName;
|
|
1256
1284
|
description;
|
|
1257
1285
|
turns;
|
|
1258
1286
|
conversation;
|
|
@@ -1262,6 +1290,7 @@ var ConversationSpec = class {
|
|
|
1262
1290
|
constructor(data) {
|
|
1263
1291
|
this.version = data?.version ?? 1;
|
|
1264
1292
|
this.name = data?.name ?? "";
|
|
1293
|
+
this.runName = data?.runName ?? "";
|
|
1265
1294
|
this.description = data?.description ?? "";
|
|
1266
1295
|
this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
|
|
1267
1296
|
this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
|
|
@@ -2935,7 +2964,7 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
2935
2964
|
scenario(result, repeat, tokenUsageCapability) {
|
|
2936
2965
|
const spec = result.spec;
|
|
2937
2966
|
const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
|
|
2938
|
-
const line = `${this.status(result.status, STATUS_WIDTH)} ${spec
|
|
2967
|
+
const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
|
|
2939
2968
|
if (!this.verbose) return line;
|
|
2940
2969
|
if (result.attempts.length === 0) return `${line}\n`;
|
|
2941
2970
|
const websocket = spec.websocket;
|
|
@@ -2948,6 +2977,9 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
2948
2977
|
lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
|
|
2949
2978
|
return line + lines.join("\n");
|
|
2950
2979
|
}
|
|
2980
|
+
static title(spec) {
|
|
2981
|
+
return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
|
|
2982
|
+
}
|
|
2951
2983
|
summary(run, exitCode, reportPaths = []) {
|
|
2952
2984
|
const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
|
|
2953
2985
|
const lines = this.verbose ? [] : [""];
|
|
@@ -2979,7 +3011,7 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
2979
3011
|
}
|
|
2980
3012
|
problems(result) {
|
|
2981
3013
|
const spec = result.spec;
|
|
2982
|
-
const lines = [`${this.status(result.status, 6)} ${spec
|
|
3014
|
+
const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
|
|
2983
3015
|
for (const attempt of result.attempts) {
|
|
2984
3016
|
const execution = attempt.execution;
|
|
2985
3017
|
if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
|
|
@@ -3156,6 +3188,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3156
3188
|
const capability = websocket.protocol.capabilities.tokenUsage;
|
|
3157
3189
|
return {
|
|
3158
3190
|
name: result.spec.name,
|
|
3191
|
+
runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
|
|
3159
3192
|
file: result.spec.location.file,
|
|
3160
3193
|
status: result.status,
|
|
3161
3194
|
connection: {
|
|
@@ -3325,7 +3358,44 @@ var conversation_schema_default = {
|
|
|
3325
3358
|
"variables": {
|
|
3326
3359
|
"type": "object",
|
|
3327
3360
|
"propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
|
|
3328
|
-
"additionalProperties": { "type": "string" }
|
|
3361
|
+
"additionalProperties": { "anyOf": [{ "type": "string" }, {
|
|
3362
|
+
"type": "array",
|
|
3363
|
+
"items": { "type": "string" }
|
|
3364
|
+
}] }
|
|
3365
|
+
},
|
|
3366
|
+
"runWith": {
|
|
3367
|
+
"description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
|
|
3368
|
+
"anyOf": [{
|
|
3369
|
+
"type": "array",
|
|
3370
|
+
"minItems": 1,
|
|
3371
|
+
"items": {
|
|
3372
|
+
"type": "object",
|
|
3373
|
+
"required": ["name"],
|
|
3374
|
+
"properties": { "name": {
|
|
3375
|
+
"type": "string",
|
|
3376
|
+
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
|
|
3377
|
+
} },
|
|
3378
|
+
"additionalProperties": { "type": "string" }
|
|
3379
|
+
}
|
|
3380
|
+
}, {
|
|
3381
|
+
"type": "object",
|
|
3382
|
+
"additionalProperties": false,
|
|
3383
|
+
"required": ["variable", "from"],
|
|
3384
|
+
"properties": {
|
|
3385
|
+
"variable": {
|
|
3386
|
+
"type": "string",
|
|
3387
|
+
"pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
|
|
3388
|
+
},
|
|
3389
|
+
"from": { "anyOf": [{
|
|
3390
|
+
"type": "array",
|
|
3391
|
+
"minItems": 1,
|
|
3392
|
+
"items": { "type": "string" }
|
|
3393
|
+
}, {
|
|
3394
|
+
"type": "string",
|
|
3395
|
+
"pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
|
|
3396
|
+
}] }
|
|
3397
|
+
}
|
|
3398
|
+
}]
|
|
3329
3399
|
},
|
|
3330
3400
|
"turns": {
|
|
3331
3401
|
"type": "array",
|
|
@@ -3484,6 +3554,221 @@ var ConversationValidator = class ConversationValidator {
|
|
|
3484
3554
|
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
3485
3555
|
}
|
|
3486
3556
|
};
|
|
3557
|
+
// src/scenarios/runWithReader.ts
|
|
3558
|
+
const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
|
|
3559
|
+
const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
|
|
3560
|
+
const FORMS = "must be a list of rows, or an object with variable and from";
|
|
3561
|
+
const SINGLE_RUN = {
|
|
3562
|
+
name: "",
|
|
3563
|
+
variables: {},
|
|
3564
|
+
keys: [],
|
|
3565
|
+
location: [],
|
|
3566
|
+
secrets: []
|
|
3567
|
+
};
|
|
3568
|
+
var RunWithReader = class RunWithReader {
|
|
3569
|
+
read(document, resolver, issues, used) {
|
|
3570
|
+
const raw = isRecord(document.value) ? document.value.runWith : void 0;
|
|
3571
|
+
if (raw === void 0) return void 0;
|
|
3572
|
+
if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
|
|
3573
|
+
if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
|
|
3574
|
+
issues.push(new ValidationIssue({
|
|
3575
|
+
message: FORMS,
|
|
3576
|
+
location: document.locate(["runWith"])
|
|
3577
|
+
}));
|
|
3578
|
+
return [];
|
|
3579
|
+
}
|
|
3580
|
+
rows(rows, document, resolver, issues, used) {
|
|
3581
|
+
if (rows.length === 0) {
|
|
3582
|
+
issues.push(new ValidationIssue({
|
|
3583
|
+
message: "must not be empty",
|
|
3584
|
+
location: document.locate(["runWith"])
|
|
3585
|
+
}));
|
|
3586
|
+
return [];
|
|
3587
|
+
}
|
|
3588
|
+
const names = /* @__PURE__ */ new Set();
|
|
3589
|
+
const runs = [];
|
|
3590
|
+
rows.forEach((row, index) => {
|
|
3591
|
+
const path = ["runWith", index];
|
|
3592
|
+
if (!isRecord(row)) {
|
|
3593
|
+
issues.push(new ValidationIssue({
|
|
3594
|
+
message: "must be object",
|
|
3595
|
+
location: document.locate(path)
|
|
3596
|
+
}));
|
|
3597
|
+
return;
|
|
3598
|
+
}
|
|
3599
|
+
const name = row.name;
|
|
3600
|
+
if (name === void 0) issues.push(new ValidationIssue({
|
|
3601
|
+
message: "missing required key 'name'",
|
|
3602
|
+
location: document.locate(path)
|
|
3603
|
+
}));
|
|
3604
|
+
else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
|
|
3605
|
+
message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
|
|
3606
|
+
location: document.locate([...path, "name"])
|
|
3607
|
+
}));
|
|
3608
|
+
else if (names.has(name)) issues.push(new ValidationIssue({
|
|
3609
|
+
message: `duplicate run name '${name}'`,
|
|
3610
|
+
location: document.locate([...path, "name"])
|
|
3611
|
+
}));
|
|
3612
|
+
else names.add(name);
|
|
3613
|
+
const run = {
|
|
3614
|
+
name: typeof name === "string" ? name : "",
|
|
3615
|
+
variables: {},
|
|
3616
|
+
keys: [],
|
|
3617
|
+
location: path,
|
|
3618
|
+
secrets: []
|
|
3619
|
+
};
|
|
3620
|
+
for (const [key, value] of Object.entries(row)) {
|
|
3621
|
+
if (key === "name") continue;
|
|
3622
|
+
if (!VARIABLE_NAME.test(key)) {
|
|
3623
|
+
issues.push(new ValidationIssue({
|
|
3624
|
+
message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
|
|
3625
|
+
location: document.locateKey(path, key)
|
|
3626
|
+
}));
|
|
3627
|
+
continue;
|
|
3628
|
+
}
|
|
3629
|
+
if (typeof value !== "string") {
|
|
3630
|
+
issues.push(new ValidationIssue({
|
|
3631
|
+
message: "must be string",
|
|
3632
|
+
location: document.locate([...path, key])
|
|
3633
|
+
}));
|
|
3634
|
+
continue;
|
|
3635
|
+
}
|
|
3636
|
+
const resolved = resolver.resolve(value, document, issues, run.secrets, {
|
|
3637
|
+
used,
|
|
3638
|
+
at: [...path, key]
|
|
3639
|
+
});
|
|
3640
|
+
RunWithReader.set(run, key, resolved, document.locateKey(path, key));
|
|
3641
|
+
}
|
|
3642
|
+
runs.push(run);
|
|
3643
|
+
});
|
|
3644
|
+
return runs;
|
|
3645
|
+
}
|
|
3646
|
+
list(form, document, resolver, issues, used) {
|
|
3647
|
+
const before = issues.length;
|
|
3648
|
+
for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
|
|
3649
|
+
message: `unknown key '${key}'`,
|
|
3650
|
+
location: document.locateKey(["runWith"], key)
|
|
3651
|
+
}));
|
|
3652
|
+
const variable = form.variable;
|
|
3653
|
+
if (variable === void 0) issues.push(new ValidationIssue({
|
|
3654
|
+
message: "missing required key 'variable'",
|
|
3655
|
+
location: document.locate(["runWith"])
|
|
3656
|
+
}));
|
|
3657
|
+
else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
|
|
3658
|
+
message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
|
|
3659
|
+
location: document.locate(["runWith", "variable"])
|
|
3660
|
+
}));
|
|
3661
|
+
const entries = this.entries(form.from, document, resolver, issues, used);
|
|
3662
|
+
if (issues.length > before || entries === void 0) return [];
|
|
3663
|
+
const name = variable;
|
|
3664
|
+
const keyLocation = document.locate(["runWith", "variable"]);
|
|
3665
|
+
const names = /* @__PURE__ */ new Set();
|
|
3666
|
+
const runs = [];
|
|
3667
|
+
for (const { entry, path } of entries) {
|
|
3668
|
+
if (!VARIABLE_NAME.test(entry)) {
|
|
3669
|
+
issues.push(new ValidationIssue({
|
|
3670
|
+
message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
|
|
3671
|
+
location: document.locate(path)
|
|
3672
|
+
}));
|
|
3673
|
+
continue;
|
|
3674
|
+
}
|
|
3675
|
+
if (names.has(entry)) {
|
|
3676
|
+
issues.push(new ValidationIssue({
|
|
3677
|
+
message: `duplicate run name '${entry}'`,
|
|
3678
|
+
location: document.locate(path)
|
|
3679
|
+
}));
|
|
3680
|
+
continue;
|
|
3681
|
+
}
|
|
3682
|
+
names.add(entry);
|
|
3683
|
+
const run = {
|
|
3684
|
+
name: entry,
|
|
3685
|
+
variables: {},
|
|
3686
|
+
keys: [],
|
|
3687
|
+
location: path,
|
|
3688
|
+
secrets: []
|
|
3689
|
+
};
|
|
3690
|
+
const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
|
|
3691
|
+
used,
|
|
3692
|
+
at: path
|
|
3693
|
+
});
|
|
3694
|
+
RunWithReader.set(run, name, value, keyLocation);
|
|
3695
|
+
runs.push(run);
|
|
3696
|
+
}
|
|
3697
|
+
return runs;
|
|
3698
|
+
}
|
|
3699
|
+
entries(from, document, resolver, issues, used) {
|
|
3700
|
+
const path = ["runWith", "from"];
|
|
3701
|
+
const location = document.locate(path);
|
|
3702
|
+
if (from === void 0) {
|
|
3703
|
+
issues.push(new ValidationIssue({
|
|
3704
|
+
message: "missing required key 'from'",
|
|
3705
|
+
location: document.locate(["runWith"])
|
|
3706
|
+
}));
|
|
3707
|
+
return;
|
|
3708
|
+
}
|
|
3709
|
+
let list;
|
|
3710
|
+
let at;
|
|
3711
|
+
if (Array.isArray(from)) {
|
|
3712
|
+
list = from;
|
|
3713
|
+
at = (index) => [...path, index];
|
|
3714
|
+
} else {
|
|
3715
|
+
const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
|
|
3716
|
+
if (reference === null) {
|
|
3717
|
+
issues.push(new ValidationIssue({
|
|
3718
|
+
message: "must be a list, or a list variable such as ${tokens}",
|
|
3719
|
+
location
|
|
3720
|
+
}));
|
|
3721
|
+
return;
|
|
3722
|
+
}
|
|
3723
|
+
const name = reference[1];
|
|
3724
|
+
const found = resolver.find(name);
|
|
3725
|
+
used.add(name);
|
|
3726
|
+
if (found === void 0) {
|
|
3727
|
+
issues.push(new ValidationIssue({
|
|
3728
|
+
message: `list variable '${name}' is not set`,
|
|
3729
|
+
location
|
|
3730
|
+
}));
|
|
3731
|
+
return;
|
|
3732
|
+
}
|
|
3733
|
+
if (!Array.isArray(found)) {
|
|
3734
|
+
issues.push(new ValidationIssue({
|
|
3735
|
+
message: `variable '${name}' is not a list`,
|
|
3736
|
+
location
|
|
3737
|
+
}));
|
|
3738
|
+
return;
|
|
3739
|
+
}
|
|
3740
|
+
list = found;
|
|
3741
|
+
at = () => path;
|
|
3742
|
+
}
|
|
3743
|
+
if (list.length === 0) {
|
|
3744
|
+
issues.push(new ValidationIssue({
|
|
3745
|
+
message: "must not be empty",
|
|
3746
|
+
location
|
|
3747
|
+
}));
|
|
3748
|
+
return;
|
|
3749
|
+
}
|
|
3750
|
+
const bad = list.findIndex((item) => typeof item !== "string");
|
|
3751
|
+
if (bad >= 0) {
|
|
3752
|
+
issues.push(new ValidationIssue({
|
|
3753
|
+
message: "must be string",
|
|
3754
|
+
location: document.locate(at(bad))
|
|
3755
|
+
}));
|
|
3756
|
+
return;
|
|
3757
|
+
}
|
|
3758
|
+
return list.map((entry, index) => ({
|
|
3759
|
+
entry,
|
|
3760
|
+
path: at(index)
|
|
3761
|
+
}));
|
|
3762
|
+
}
|
|
3763
|
+
static set(run, name, value, location) {
|
|
3764
|
+
run.variables[name] = value;
|
|
3765
|
+
run.keys.push({
|
|
3766
|
+
name,
|
|
3767
|
+
location
|
|
3768
|
+
});
|
|
3769
|
+
if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
|
|
3770
|
+
}
|
|
3771
|
+
};
|
|
3487
3772
|
// src/scenarios/scenarioLoader.ts
|
|
3488
3773
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
3489
3774
|
var ScenarioLoader = class ScenarioLoader {
|
|
@@ -3492,6 +3777,7 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
3492
3777
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
3493
3778
|
conversationValidator = new ConversationValidator();
|
|
3494
3779
|
variablesReader = new VariablesReader();
|
|
3780
|
+
runWithReader = new RunWithReader();
|
|
3495
3781
|
webSocketResolver = new WebSocketResolver();
|
|
3496
3782
|
environmentResolver;
|
|
3497
3783
|
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
@@ -3507,45 +3793,59 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
3507
3793
|
for (const file of files) {
|
|
3508
3794
|
const document = this.documentLoader.load(file, relative(this.cwd, file));
|
|
3509
3795
|
const fileIssues = [...document.errors];
|
|
3510
|
-
const
|
|
3796
|
+
const fileSecrets = [];
|
|
3511
3797
|
const used = /* @__PURE__ */ new Set();
|
|
3512
|
-
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues,
|
|
3513
|
-
const
|
|
3514
|
-
|
|
3515
|
-
if (fileIssues.length
|
|
3516
|
-
|
|
3517
|
-
|
|
3518
|
-
...resolver.resolve(rest, document, fileIssues, secrets, { used }),
|
|
3519
|
-
...raw === void 0 ? {} : { variables }
|
|
3520
|
-
};
|
|
3798
|
+
const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
|
|
3799
|
+
const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
|
|
3800
|
+
const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
|
|
3801
|
+
if (fileIssues.length > 0) {
|
|
3802
|
+
issues.push(...fileIssues);
|
|
3803
|
+
continue;
|
|
3521
3804
|
}
|
|
3522
|
-
|
|
3523
|
-
const
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
|
|
3527
|
-
|
|
3528
|
-
|
|
3529
|
-
|
|
3530
|
-
|
|
3531
|
-
|
|
3805
|
+
const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
|
|
3806
|
+
const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
|
|
3807
|
+
for (const result of results) for (const name of result.used) used.add(name);
|
|
3808
|
+
fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
|
|
3809
|
+
run: result.run,
|
|
3810
|
+
issues: result.issues
|
|
3811
|
+
})), (issue, run) => run === void 0 ? issue : new ValidationIssue({
|
|
3812
|
+
message: `run '${run.name}': ${issue.message}`,
|
|
3813
|
+
location: issue.location
|
|
3814
|
+
})));
|
|
3815
|
+
const attempted = results.filter((result) => result.connectionChecked);
|
|
3816
|
+
if (attempted.length > 0) connectionIssues.push({
|
|
3817
|
+
file: document,
|
|
3818
|
+
runs: attempted.map((result) => ({
|
|
3819
|
+
run: result.run,
|
|
3820
|
+
issues: result.websocketIssues
|
|
3821
|
+
}))
|
|
3822
|
+
});
|
|
3823
|
+
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
|
|
3824
|
+
if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
|
|
3825
|
+
const websocket = result.websocket;
|
|
3826
|
+
specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
3532
3827
|
}
|
|
3533
|
-
if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
|
|
3534
|
-
if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
|
|
3535
|
-
if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
|
|
3536
|
-
if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
|
|
3537
3828
|
else issues.push(...fileIssues);
|
|
3538
3829
|
}
|
|
3539
|
-
const shared = ScenarioLoader.shared(connectionIssues.
|
|
3830
|
+
const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
|
|
3540
3831
|
if (shared.length > 0) throw new EvalError({
|
|
3541
3832
|
category: ErrorCategory.Configuration,
|
|
3542
3833
|
message: "invalid configuration",
|
|
3543
3834
|
issues: shared
|
|
3544
3835
|
});
|
|
3545
|
-
for (const entry of connectionIssues)
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
|
|
3836
|
+
for (const entry of connectionIssues) {
|
|
3837
|
+
const fileLocation = ScenarioLoader.variablesLocation(entry.file);
|
|
3838
|
+
issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
|
|
3839
|
+
message: issue.toString(),
|
|
3840
|
+
location: fileLocation
|
|
3841
|
+
}) : new ValidationIssue({
|
|
3842
|
+
message: `run '${run.name}': ${issue.toString()}`,
|
|
3843
|
+
location: new SourceLocation({
|
|
3844
|
+
...entry.file.locate(run.location),
|
|
3845
|
+
path: ""
|
|
3846
|
+
})
|
|
3847
|
+
})));
|
|
3848
|
+
}
|
|
3549
3849
|
if (issues.length > 0) throw new EvalError({
|
|
3550
3850
|
category: ErrorCategory.Specification,
|
|
3551
3851
|
message: "invalid scenario specification",
|
|
@@ -3553,6 +3853,43 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
3553
3853
|
});
|
|
3554
3854
|
return specs;
|
|
3555
3855
|
}
|
|
3856
|
+
resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
|
|
3857
|
+
const issues = [];
|
|
3858
|
+
const used = /* @__PURE__ */ new Set();
|
|
3859
|
+
const resolver = scenarioResolver.withLevels([run.variables]);
|
|
3860
|
+
const value = {
|
|
3861
|
+
...resolver.resolve(rest, document, issues, secrets, { used }),
|
|
3862
|
+
...variables === void 0 ? {} : { variables }
|
|
3863
|
+
};
|
|
3864
|
+
const websocketIssues = [];
|
|
3865
|
+
let websocket;
|
|
3866
|
+
let connectionChecked = false;
|
|
3867
|
+
if (issues.length === 0 && configDocument !== void 0) {
|
|
3868
|
+
const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
|
|
3869
|
+
if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
|
|
3870
|
+
if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
|
|
3871
|
+
connectionChecked = true;
|
|
3872
|
+
}
|
|
3873
|
+
if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
|
|
3874
|
+
if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
|
|
3875
|
+
return {
|
|
3876
|
+
run,
|
|
3877
|
+
value,
|
|
3878
|
+
used,
|
|
3879
|
+
secrets,
|
|
3880
|
+
issues,
|
|
3881
|
+
websocket,
|
|
3882
|
+
websocketIssues,
|
|
3883
|
+
connectionChecked
|
|
3884
|
+
};
|
|
3885
|
+
}
|
|
3886
|
+
static byRun(runs, place) {
|
|
3887
|
+
const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
|
|
3888
|
+
const sharedTexts = new Set(shared.map((issue) => issue.toString()));
|
|
3889
|
+
const result = shared.map((issue) => place(issue, void 0));
|
|
3890
|
+
for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
|
|
3891
|
+
return result;
|
|
3892
|
+
}
|
|
3556
3893
|
static shared(lists) {
|
|
3557
3894
|
if (lists.length === 0) return [];
|
|
3558
3895
|
const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
|
|
@@ -3564,11 +3901,22 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
3564
3901
|
path: ""
|
|
3565
3902
|
});
|
|
3566
3903
|
}
|
|
3567
|
-
static unusedVariables(variables, used, document) {
|
|
3568
|
-
|
|
3569
|
-
|
|
3904
|
+
static unusedVariables(variables, used, results, document) {
|
|
3905
|
+
const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
|
|
3906
|
+
const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
|
|
3907
|
+
message: message(name),
|
|
3570
3908
|
location: document.locateKey(["variables"], name)
|
|
3571
3909
|
}));
|
|
3910
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3911
|
+
for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
|
|
3912
|
+
const issue = new ValidationIssue({
|
|
3913
|
+
message: message(key.name),
|
|
3914
|
+
location: key.location
|
|
3915
|
+
});
|
|
3916
|
+
if (!seen.has(issue.toString())) issues.push(issue);
|
|
3917
|
+
seen.add(issue.toString());
|
|
3918
|
+
}
|
|
3919
|
+
return issues;
|
|
3572
3920
|
}
|
|
3573
3921
|
discover(target, config) {
|
|
3574
3922
|
if (!existsSync(target)) throw new EvalError({
|
|
@@ -3594,13 +3942,14 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
3594
3942
|
}
|
|
3595
3943
|
return files;
|
|
3596
3944
|
}
|
|
3597
|
-
build(value, document, websocket, secrets) {
|
|
3945
|
+
build(value, document, runName, websocket, secrets) {
|
|
3598
3946
|
const scenario = value;
|
|
3599
3947
|
const turns = scenario.turns;
|
|
3600
3948
|
const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
|
|
3601
3949
|
return new ConversationSpec({
|
|
3602
3950
|
version: scenario.version,
|
|
3603
3951
|
name: scenario.name,
|
|
3952
|
+
runName,
|
|
3604
3953
|
description: typeof scenario.description === "string" ? scenario.description : "",
|
|
3605
3954
|
turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
|
|
3606
3955
|
conversation: conversation ? new ConversationExpectation({
|