@mappies/miniwat-eval 0.1.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -6
- package/dist/index.cjs +51 -10
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +14 -3
- package/dist/index.d.ts +14 -3
- package/dist/index.js +50 -10
- package/dist/index.js.map +1 -1
- package/dist/main.js +69 -23
- package/dist/main.js.map +1 -1
- package/package.json +2 -5
- package/schemas/report.schema.json +1 -0
package/dist/main.js
CHANGED
|
@@ -164,6 +164,44 @@ var ConfigLocator = class ConfigLocator {
|
|
|
164
164
|
return existsSync(path) && statSync(path).isFile();
|
|
165
165
|
}
|
|
166
166
|
};
|
|
167
|
+
// src/utils/isRecord.ts
|
|
168
|
+
function isRecord(value) {
|
|
169
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
170
|
+
}
|
|
171
|
+
// src/config/parametersLoader.ts
|
|
172
|
+
var ParametersLoader = class ParametersLoader {
|
|
173
|
+
cwd;
|
|
174
|
+
constructor(cwd = process.cwd()) {
|
|
175
|
+
this.cwd = cwd;
|
|
176
|
+
}
|
|
177
|
+
load(path) {
|
|
178
|
+
const absolute = resolve(this.cwd, path);
|
|
179
|
+
const shown = relative(this.cwd, absolute) || path;
|
|
180
|
+
let text;
|
|
181
|
+
let value;
|
|
182
|
+
try {
|
|
183
|
+
text = readFileSync(absolute, "utf8");
|
|
184
|
+
} catch {
|
|
185
|
+
throw ParametersLoader.error(`parameters file not found: ${shown}`);
|
|
186
|
+
}
|
|
187
|
+
try {
|
|
188
|
+
value = JSON.parse(text);
|
|
189
|
+
} catch {
|
|
190
|
+
throw ParametersLoader.error(`parameters file ${shown} is not valid JSON`);
|
|
191
|
+
}
|
|
192
|
+
if (!isRecord(value)) throw ParametersLoader.error(`parameters file ${shown} must contain a JSON object`);
|
|
193
|
+
return {
|
|
194
|
+
path: absolute,
|
|
195
|
+
parameters: Object.fromEntries(Object.entries(value).filter((entry) => typeof entry[1] === "string"))
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
static error(message) {
|
|
199
|
+
return new EvalError({
|
|
200
|
+
category: ErrorCategory.Configuration,
|
|
201
|
+
message
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
};
|
|
167
205
|
// schemas/eval-config.schema.json
|
|
168
206
|
var eval_config_schema_default = {
|
|
169
207
|
$schema: "http://json-schema.org/draft-07/schema#",
|
|
@@ -471,6 +509,7 @@ var WebSocketConfig = class {
|
|
|
471
509
|
var EvalConfig = class {
|
|
472
510
|
version;
|
|
473
511
|
configPath;
|
|
512
|
+
parametersPath;
|
|
474
513
|
websocket;
|
|
475
514
|
execution;
|
|
476
515
|
evaluator;
|
|
@@ -479,6 +518,7 @@ var EvalConfig = class {
|
|
|
479
518
|
constructor(data) {
|
|
480
519
|
this.version = data?.version ?? 1;
|
|
481
520
|
this.configPath = data?.configPath ?? "";
|
|
521
|
+
this.parametersPath = data?.parametersPath ?? "";
|
|
482
522
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
483
523
|
this.execution = new ExecutionConfig(data?.execution);
|
|
484
524
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -492,12 +532,14 @@ const ReportFormat = {
|
|
|
492
532
|
Json: "json"
|
|
493
533
|
};
|
|
494
534
|
// src/yaml/environmentResolver.ts
|
|
495
|
-
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g;
|
|
535
|
+
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
|
|
496
536
|
const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
|
|
497
537
|
var EnvironmentResolver = class EnvironmentResolver {
|
|
498
538
|
environment;
|
|
499
|
-
|
|
539
|
+
parameters;
|
|
540
|
+
constructor(environment = process.env, parameters = {}) {
|
|
500
541
|
this.environment = environment;
|
|
542
|
+
this.parameters = parameters;
|
|
501
543
|
}
|
|
502
544
|
resolve(value, document, issues, secrets = []) {
|
|
503
545
|
return this.resolveValue(value, [], document, issues, secrets);
|
|
@@ -510,10 +552,11 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
510
552
|
}
|
|
511
553
|
resolveValue(value, path, document, issues, secrets) {
|
|
512
554
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
513
|
-
const resolved = this.environment[name];
|
|
555
|
+
const resolved = Object.hasOwn(this.parameters, name) ? this.parameters[name] : this.environment[name];
|
|
514
556
|
if (resolved === void 0) {
|
|
557
|
+
const source = Object.keys(this.parameters).length > 0 ? "parameter or environment variable" : "environment variable";
|
|
515
558
|
issues.push(new ValidationIssue({
|
|
516
|
-
message:
|
|
559
|
+
message: `${source} '${name}' is not set`,
|
|
517
560
|
location: document.locate(path)
|
|
518
561
|
}));
|
|
519
562
|
return reference;
|
|
@@ -677,10 +720,6 @@ var YamlDocumentLoader = class {
|
|
|
677
720
|
}), lineCounter);
|
|
678
721
|
}
|
|
679
722
|
};
|
|
680
|
-
// src/utils/isRecord.ts
|
|
681
|
-
function isRecord(value) {
|
|
682
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
683
|
-
}
|
|
684
723
|
// src/config/defaultConfig.ts
|
|
685
724
|
const DEFAULT_CONFIG = new EvalConfig({
|
|
686
725
|
execution: new ExecutionConfig({
|
|
@@ -717,9 +756,9 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
717
756
|
documentLoader = new YamlDocumentLoader();
|
|
718
757
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
719
758
|
environmentResolver;
|
|
720
|
-
constructor(environment = process.env, cwd = process.cwd()) {
|
|
759
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
721
760
|
this.cwd = cwd;
|
|
722
|
-
this.environmentResolver = new EnvironmentResolver(environment);
|
|
761
|
+
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
723
762
|
}
|
|
724
763
|
load(configPath, override, report = new ReportOverride()) {
|
|
725
764
|
const document = this.documentLoader.load(configPath, relative(this.cwd, configPath));
|
|
@@ -756,7 +795,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
756
795
|
const apiKey = EvalConfigLoader.section(document.value, "evaluator").apiKey;
|
|
757
796
|
if (typeof apiKey !== "string" || EnvironmentResolver.hasReference(apiKey)) return [];
|
|
758
797
|
return [new ValidationIssue({
|
|
759
|
-
message: "must be
|
|
798
|
+
message: "must be a reference such as ${OPENAI_API_KEY}",
|
|
760
799
|
location: document.locate(["evaluator", "apiKey"])
|
|
761
800
|
})];
|
|
762
801
|
}
|
|
@@ -2928,6 +2967,7 @@ var JsonReporter = class JsonReporter {
|
|
|
2928
2967
|
const { websocket, execution, evaluator, report } = config;
|
|
2929
2968
|
return {
|
|
2930
2969
|
path: this.path(config.configPath),
|
|
2970
|
+
parametersPath: config.parametersPath === "" ? void 0 : this.path(config.parametersPath),
|
|
2931
2971
|
websocket: {
|
|
2932
2972
|
endpoint: websocket.endpoint,
|
|
2933
2973
|
subprotocols: [...websocket.connection.subprotocols],
|
|
@@ -3290,9 +3330,9 @@ var ScenarioLoader = class {
|
|
|
3290
3330
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
3291
3331
|
conversationValidator = new ConversationValidator();
|
|
3292
3332
|
environmentResolver;
|
|
3293
|
-
constructor(environment = process.env, cwd = process.cwd()) {
|
|
3333
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
3294
3334
|
this.cwd = cwd;
|
|
3295
|
-
this.environmentResolver = new EnvironmentResolver(environment);
|
|
3335
|
+
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
3296
3336
|
}
|
|
3297
3337
|
load(target, config) {
|
|
3298
3338
|
const files = this.discover(resolve(this.cwd, target), config);
|
|
@@ -3427,15 +3467,20 @@ var RunCommand = class RunCommand {
|
|
|
3427
3467
|
this.signals = signals;
|
|
3428
3468
|
this.colour = colour;
|
|
3429
3469
|
}
|
|
3430
|
-
async execute(target, config, execution = new ExecutionOverride(), report = new ReportOverride()) {
|
|
3470
|
+
async execute(target, config, execution = new ExecutionOverride(), report = new ReportOverride(), parameters = "") {
|
|
3431
3471
|
let evalConfig;
|
|
3432
3472
|
let specs;
|
|
3433
3473
|
let evaluator;
|
|
3434
3474
|
let redactor = new SecretRedactor([]);
|
|
3435
3475
|
try {
|
|
3436
3476
|
const configPath = new ConfigLocator(this.cwd).locate(target, config);
|
|
3437
|
-
|
|
3438
|
-
|
|
3477
|
+
const parameterFile = parameters === "" ? {
|
|
3478
|
+
path: "",
|
|
3479
|
+
parameters: {}
|
|
3480
|
+
} : new ParametersLoader(this.cwd).load(parameters);
|
|
3481
|
+
evalConfig = new EvalConfigLoader(this.environment, this.cwd, parameterFile.parameters).load(configPath, execution, report);
|
|
3482
|
+
evalConfig.parametersPath = parameterFile.path;
|
|
3483
|
+
specs = new ScenarioLoader(this.environment, this.cwd, parameterFile.parameters).load(target, evalConfig);
|
|
3439
3484
|
redactor = new SecretRedactor([...evalConfig.secrets, ...specs.flatMap((spec) => spec.secrets)]);
|
|
3440
3485
|
evaluator = RunCommand.needsEvaluator(specs) ? await this.createEvaluator(evalConfig) : new UnavailableEvaluator();
|
|
3441
3486
|
} catch (error) {
|
|
@@ -3524,11 +3569,12 @@ var ValidateCommand = class {
|
|
|
3524
3569
|
this.environment = environment;
|
|
3525
3570
|
this.cwd = cwd;
|
|
3526
3571
|
}
|
|
3527
|
-
execute(target, config) {
|
|
3572
|
+
execute(target, config, parameters = "") {
|
|
3528
3573
|
try {
|
|
3529
3574
|
const configPath = new ConfigLocator(this.cwd).locate(target, config);
|
|
3530
|
-
const
|
|
3531
|
-
const
|
|
3575
|
+
const values = parameters === "" ? {} : new ParametersLoader(this.cwd).load(parameters).parameters;
|
|
3576
|
+
const evalConfig = new EvalConfigLoader(this.environment, this.cwd, values).load(configPath, new ExecutionOverride());
|
|
3577
|
+
const specs = new ScenarioLoader(this.environment, this.cwd, values).load(target, evalConfig);
|
|
3532
3578
|
this.out(`Valid: ${specs.length} scenario${specs.length === 1 ? "" : "s"}\n`);
|
|
3533
3579
|
return 0;
|
|
3534
3580
|
} catch (error) {
|
|
@@ -3545,10 +3591,10 @@ function createProgram(setExitCode, out, err, colour = false) {
|
|
|
3545
3591
|
writeOut: out,
|
|
3546
3592
|
writeErr: err
|
|
3547
3593
|
}).exitOverride();
|
|
3548
|
-
program.command("validate").description("Parse, interpolate and validate configuration and scenarios with no network calls").argument("<target>", "scenario file or directory").option("--config <path>", "configuration file (default: search upward for eval-config.yml)").action((target, options) => {
|
|
3549
|
-
setExitCode(new ValidateCommand(out, err).execute(target, options.config ?? ""));
|
|
3594
|
+
program.command("validate").description("Parse, interpolate and validate configuration and scenarios with no network calls").argument("<target>", "scenario file or directory").option("--config <path>", "configuration file (default: search upward for eval-config.yml)").option("--parameters <path>", "JSON file whose string values resolve ${name} references before the environment").action((target, options) => {
|
|
3595
|
+
setExitCode(new ValidateCommand(out, err).execute(target, options.config ?? "", options.parameters ?? ""));
|
|
3550
3596
|
});
|
|
3551
|
-
program.command("run").description("Run every scenario against the agent, repeat times each, and report the results").argument("<target>", "scenario file or directory").option("--config <path>", "configuration file (default: search upward for eval-config.yml)").option("--repeat <n>", "attempts per scenario (overrides execution.repeat)").option("--min-pass <n>", "passing attempts a scenario needs (overrides execution.minPass)").option("--quick", "shorthand for --repeat 1 --min-pass 1; wins over both").option("--concurrency <n>", "scenarios run at once (overrides execution.scenarioConcurrency)").option("--report <formats>", "comma-separated console, json (overrides report.formats)").option("--verbose", "include every attempt's transcript in the console report").action(async (target, options) => {
|
|
3597
|
+
program.command("run").description("Run every scenario against the agent, repeat times each, and report the results").argument("<target>", "scenario file or directory").option("--config <path>", "configuration file (default: search upward for eval-config.yml)").option("--parameters <path>", "JSON file whose string values resolve ${name} references before the environment").option("--repeat <n>", "attempts per scenario (overrides execution.repeat)").option("--min-pass <n>", "passing attempts a scenario needs (overrides execution.minPass)").option("--quick", "shorthand for --repeat 1 --min-pass 1; wins over both").option("--concurrency <n>", "scenarios run at once (overrides execution.scenarioConcurrency)").option("--report <formats>", "comma-separated console, json (overrides report.formats)").option("--verbose", "include every attempt's transcript in the console report").action(async (target, options) => {
|
|
3552
3598
|
let execution;
|
|
3553
3599
|
let report;
|
|
3554
3600
|
try {
|
|
@@ -3560,7 +3606,7 @@ function createProgram(setExitCode, out, err, colour = false) {
|
|
|
3560
3606
|
setExitCode(1);
|
|
3561
3607
|
return;
|
|
3562
3608
|
}
|
|
3563
|
-
setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour).execute(target, options.config ?? "", execution, report));
|
|
3609
|
+
setExitCode(await new RunCommand(out, err, process.env, process.cwd(), void 0, void 0, process, colour).execute(target, options.config ?? "", execution, report, options.parameters ?? ""));
|
|
3564
3610
|
});
|
|
3565
3611
|
return program;
|
|
3566
3612
|
}
|