@mappies/miniwat-eval 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/main.js CHANGED
@@ -218,7 +218,10 @@ var eval_config_schema_default = {
218
218
  "variables": {
219
219
  "type": "object",
220
220
  "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
221
- "additionalProperties": { "type": "string" }
221
+ "additionalProperties": { "anyOf": [{ "type": "string" }, {
222
+ "type": "array",
223
+ "items": { "type": "string" }
224
+ }] }
222
225
  },
223
226
  "websocket": {
224
227
  "type": "object",
@@ -525,7 +528,7 @@ var EvalConfig = class {
525
528
  this.version = data?.version ?? 1;
526
529
  this.configPath = data?.configPath ?? "";
527
530
  this.parametersPath = data?.parametersPath ?? "";
528
- this.variables = { ...data?.variables ?? {} };
531
+ this.variables = Object.fromEntries(Object.entries(data?.variables ?? {}).map(([name, value]) => [name, Array.isArray(value) ? [...value] : value]));
529
532
  this.websocket = new WebSocketConfig(data?.websocket);
530
533
  this.execution = new ExecutionConfig(data?.execution);
531
534
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -611,7 +614,7 @@ var EnvironmentResolver = class EnvironmentResolver {
611
614
  return new EnvironmentResolver(this.environment, this.parameters, [...levels, ...this.levels]);
612
615
  }
613
616
  resolve(value, document, issues, secrets = [], options = {}) {
614
- return this.resolveValue(value, [], document, issues, secrets, options);
617
+ return this.resolveValue(value, options.at ?? [], document, issues, secrets, options);
615
618
  }
616
619
  static hasReference(value) {
617
620
  return new RegExp(REFERENCE.source).test(value);
@@ -619,7 +622,7 @@ var EnvironmentResolver = class EnvironmentResolver {
619
622
  static isSecretName(name) {
620
623
  return SECRET_NAME.test(name);
621
624
  }
622
- lookup(name) {
625
+ find(name) {
623
626
  for (const level of [...this.levels, this.parameters]) if (Object.hasOwn(level, name)) return level[name];
624
627
  return this.environment[name];
625
628
  }
@@ -634,7 +637,7 @@ var EnvironmentResolver = class EnvironmentResolver {
634
637
  resolveValue(value, path, document, issues, secrets, options) {
635
638
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
636
639
  options.used?.add(name);
637
- const resolved = this.lookup(name);
640
+ const resolved = this.find(name);
638
641
  if (resolved === void 0) {
639
642
  if (options.optional?.(path)) {
640
643
  options.unresolved?.add(YamlDocument.formatPath(path));
@@ -646,6 +649,13 @@ var EnvironmentResolver = class EnvironmentResolver {
646
649
  }));
647
650
  return reference;
648
651
  }
652
+ if (Array.isArray(resolved)) {
653
+ issues.push(new ValidationIssue({
654
+ message: `variable '${name}' is a list, which can only be used as runWith.from`,
655
+ location: document.locate(path)
656
+ }));
657
+ return reference;
658
+ }
649
659
  if (EnvironmentResolver.isSecretName(name) && resolved !== "" && !secrets.includes(resolved)) secrets.push(resolved);
650
660
  return resolved;
651
661
  });
@@ -770,9 +780,25 @@ var VariablesReader = class {
770
780
  }));
771
781
  continue;
772
782
  }
783
+ if (Array.isArray(value)) {
784
+ const bad = value.findIndex((item) => typeof item !== "string");
785
+ if (bad >= 0) {
786
+ issues.push(new ValidationIssue({
787
+ message: "must be string",
788
+ location: document.locate([
789
+ "variables",
790
+ name,
791
+ bad
792
+ ])
793
+ }));
794
+ continue;
795
+ }
796
+ variables[name] = [...value];
797
+ continue;
798
+ }
773
799
  if (typeof value !== "string") {
774
800
  issues.push(new ValidationIssue({
775
- message: "must be string",
801
+ message: "must be string or a list of strings",
776
802
  location: document.locate(["variables", name])
777
803
  }));
778
804
  continue;
@@ -780,8 +806,9 @@ var VariablesReader = class {
780
806
  const before = issues.length;
781
807
  const resolved = resolver.resolve({ variables: { [name]: value } }, document, issues, secrets);
782
808
  if (issues.length > before) continue;
783
- variables[name] = resolved.variables[name];
784
- if (EnvironmentResolver.isSecretName(name) && variables[name] !== "" && !secrets.includes(variables[name])) secrets.push(variables[name]);
809
+ const text = resolved.variables[name];
810
+ variables[name] = text;
811
+ if (EnvironmentResolver.isSecretName(name) && text !== "" && !secrets.includes(text)) secrets.push(text);
785
812
  }
786
813
  return variables;
787
814
  }
@@ -1253,6 +1280,7 @@ var TurnSpec = class {
1253
1280
  var ConversationSpec = class {
1254
1281
  version;
1255
1282
  name;
1283
+ runName;
1256
1284
  description;
1257
1285
  turns;
1258
1286
  conversation;
@@ -1262,6 +1290,7 @@ var ConversationSpec = class {
1262
1290
  constructor(data) {
1263
1291
  this.version = data?.version ?? 1;
1264
1292
  this.name = data?.name ?? "";
1293
+ this.runName = data?.runName ?? "";
1265
1294
  this.description = data?.description ?? "";
1266
1295
  this.turns = (data?.turns ?? []).map((turn) => new TurnSpec(turn));
1267
1296
  this.conversation = data?.conversation ? new ConversationExpectation(data.conversation) : void 0;
@@ -2935,7 +2964,7 @@ var ConsoleReporter = class ConsoleReporter {
2935
2964
  scenario(result, repeat, tokenUsageCapability) {
2936
2965
  const spec = result.spec;
2937
2966
  const outcome = result.attempts.length === 0 ? "not started" : `${result.passed}/${repeat} passed`;
2938
- const line = `${this.status(result.status, STATUS_WIDTH)} ${spec.name} (${spec.location.file}) ${outcome}\n`;
2967
+ const line = `${this.status(result.status, STATUS_WIDTH)} ${ConsoleReporter.title(spec)} (${spec.location.file}) ${outcome}\n`;
2939
2968
  if (!this.verbose) return line;
2940
2969
  if (result.attempts.length === 0) return `${line}\n`;
2941
2970
  const websocket = spec.websocket;
@@ -2948,6 +2977,9 @@ var ConsoleReporter = class ConsoleReporter {
2948
2977
  lines.push(` ${ConsoleReporter.usage("Evaluator tokens", result.evaluatorUsage, true)}`, "", "");
2949
2978
  return line + lines.join("\n");
2950
2979
  }
2980
+ static title(spec) {
2981
+ return spec.runName === "" ? spec.name : `${spec.name} [${spec.runName}]`;
2982
+ }
2951
2983
  summary(run, exitCode, reportPaths = []) {
2952
2984
  const problems = run.scenarios.filter((result) => result.status !== ResultStatus.Pass && result.attempts.length > 0);
2953
2985
  const lines = this.verbose ? [] : [""];
@@ -2979,7 +3011,7 @@ var ConsoleReporter = class ConsoleReporter {
2979
3011
  }
2980
3012
  problems(result) {
2981
3013
  const spec = result.spec;
2982
- const lines = [`${this.status(result.status, 6)} ${spec.name} (${spec.location.file})`];
3014
+ const lines = [`${this.status(result.status, 6)} ${ConsoleReporter.title(spec)} (${spec.location.file})`];
2983
3015
  for (const attempt of result.attempts) {
2984
3016
  const execution = attempt.execution;
2985
3017
  if (execution.failure !== void 0) lines.push(` Attempt ${attempt.attempt} stopped at turn ${execution.turns.length + 1}: ${execution.failure.category} error: ${execution.failure.message}`);
@@ -3156,6 +3188,7 @@ var JsonReporter = class JsonReporter {
3156
3188
  const capability = websocket.protocol.capabilities.tokenUsage;
3157
3189
  return {
3158
3190
  name: result.spec.name,
3191
+ runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
3159
3192
  file: result.spec.location.file,
3160
3193
  status: result.status,
3161
3194
  connection: {
@@ -3325,7 +3358,44 @@ var conversation_schema_default = {
3325
3358
  "variables": {
3326
3359
  "type": "object",
3327
3360
  "propertyNames": { "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$" },
3328
- "additionalProperties": { "type": "string" }
3361
+ "additionalProperties": { "anyOf": [{ "type": "string" }, {
3362
+ "type": "array",
3363
+ "items": { "type": "string" }
3364
+ }] }
3365
+ },
3366
+ "runWith": {
3367
+ "description": "Runs the scenario once per entry. Either rows, each with a name and variables, or { variable, from }, where from is a list of variable or parameter names or a list variable such as ${tokens}.",
3368
+ "anyOf": [{
3369
+ "type": "array",
3370
+ "minItems": 1,
3371
+ "items": {
3372
+ "type": "object",
3373
+ "required": ["name"],
3374
+ "properties": { "name": {
3375
+ "type": "string",
3376
+ "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"
3377
+ } },
3378
+ "additionalProperties": { "type": "string" }
3379
+ }
3380
+ }, {
3381
+ "type": "object",
3382
+ "additionalProperties": false,
3383
+ "required": ["variable", "from"],
3384
+ "properties": {
3385
+ "variable": {
3386
+ "type": "string",
3387
+ "pattern": "^[A-Za-z_][A-Za-z0-9_-]*$"
3388
+ },
3389
+ "from": { "anyOf": [{
3390
+ "type": "array",
3391
+ "minItems": 1,
3392
+ "items": { "type": "string" }
3393
+ }, {
3394
+ "type": "string",
3395
+ "pattern": "^\\$\\{[A-Za-z_][A-Za-z0-9_-]*\\}$"
3396
+ }] }
3397
+ }
3398
+ }]
3329
3399
  },
3330
3400
  "turns": {
3331
3401
  "type": "array",
@@ -3484,6 +3554,221 @@ var ConversationValidator = class ConversationValidator {
3484
3554
  return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
3485
3555
  }
3486
3556
  };
3557
+ // src/scenarios/runWithReader.ts
3558
+ const RUN_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
3559
+ const LIST_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_-]*)\}$/;
3560
+ const FORMS = "must be a list of rows, or an object with variable and from";
3561
+ const SINGLE_RUN = {
3562
+ name: "",
3563
+ variables: {},
3564
+ keys: [],
3565
+ location: [],
3566
+ secrets: []
3567
+ };
3568
+ var RunWithReader = class RunWithReader {
3569
+ read(document, resolver, issues, used) {
3570
+ const raw = isRecord(document.value) ? document.value.runWith : void 0;
3571
+ if (raw === void 0) return void 0;
3572
+ if (Array.isArray(raw)) return this.rows(raw, document, resolver, issues, used);
3573
+ if (isRecord(raw)) return this.list(raw, document, resolver, issues, used);
3574
+ issues.push(new ValidationIssue({
3575
+ message: FORMS,
3576
+ location: document.locate(["runWith"])
3577
+ }));
3578
+ return [];
3579
+ }
3580
+ rows(rows, document, resolver, issues, used) {
3581
+ if (rows.length === 0) {
3582
+ issues.push(new ValidationIssue({
3583
+ message: "must not be empty",
3584
+ location: document.locate(["runWith"])
3585
+ }));
3586
+ return [];
3587
+ }
3588
+ const names = /* @__PURE__ */ new Set();
3589
+ const runs = [];
3590
+ rows.forEach((row, index) => {
3591
+ const path = ["runWith", index];
3592
+ if (!isRecord(row)) {
3593
+ issues.push(new ValidationIssue({
3594
+ message: "must be object",
3595
+ location: document.locate(path)
3596
+ }));
3597
+ return;
3598
+ }
3599
+ const name = row.name;
3600
+ if (name === void 0) issues.push(new ValidationIssue({
3601
+ message: "missing required key 'name'",
3602
+ location: document.locate(path)
3603
+ }));
3604
+ else if (typeof name !== "string" || !RUN_NAME.test(name)) issues.push(new ValidationIssue({
3605
+ message: "must be a run name matching [A-Za-z0-9][A-Za-z0-9_.-]*",
3606
+ location: document.locate([...path, "name"])
3607
+ }));
3608
+ else if (names.has(name)) issues.push(new ValidationIssue({
3609
+ message: `duplicate run name '${name}'`,
3610
+ location: document.locate([...path, "name"])
3611
+ }));
3612
+ else names.add(name);
3613
+ const run = {
3614
+ name: typeof name === "string" ? name : "",
3615
+ variables: {},
3616
+ keys: [],
3617
+ location: path,
3618
+ secrets: []
3619
+ };
3620
+ for (const [key, value] of Object.entries(row)) {
3621
+ if (key === "name") continue;
3622
+ if (!VARIABLE_NAME.test(key)) {
3623
+ issues.push(new ValidationIssue({
3624
+ message: `variable name '${key}' must match [A-Za-z_][A-Za-z0-9_-]*`,
3625
+ location: document.locateKey(path, key)
3626
+ }));
3627
+ continue;
3628
+ }
3629
+ if (typeof value !== "string") {
3630
+ issues.push(new ValidationIssue({
3631
+ message: "must be string",
3632
+ location: document.locate([...path, key])
3633
+ }));
3634
+ continue;
3635
+ }
3636
+ const resolved = resolver.resolve(value, document, issues, run.secrets, {
3637
+ used,
3638
+ at: [...path, key]
3639
+ });
3640
+ RunWithReader.set(run, key, resolved, document.locateKey(path, key));
3641
+ }
3642
+ runs.push(run);
3643
+ });
3644
+ return runs;
3645
+ }
3646
+ list(form, document, resolver, issues, used) {
3647
+ const before = issues.length;
3648
+ for (const key of Object.keys(form).filter((key) => key !== "variable" && key !== "from")) issues.push(new ValidationIssue({
3649
+ message: `unknown key '${key}'`,
3650
+ location: document.locateKey(["runWith"], key)
3651
+ }));
3652
+ const variable = form.variable;
3653
+ if (variable === void 0) issues.push(new ValidationIssue({
3654
+ message: "missing required key 'variable'",
3655
+ location: document.locate(["runWith"])
3656
+ }));
3657
+ else if (typeof variable !== "string" || !VARIABLE_NAME.test(variable)) issues.push(new ValidationIssue({
3658
+ message: "must be a variable name matching [A-Za-z_][A-Za-z0-9_-]*",
3659
+ location: document.locate(["runWith", "variable"])
3660
+ }));
3661
+ const entries = this.entries(form.from, document, resolver, issues, used);
3662
+ if (issues.length > before || entries === void 0) return [];
3663
+ const name = variable;
3664
+ const keyLocation = document.locate(["runWith", "variable"]);
3665
+ const names = /* @__PURE__ */ new Set();
3666
+ const runs = [];
3667
+ for (const { entry, path } of entries) {
3668
+ if (!VARIABLE_NAME.test(entry)) {
3669
+ issues.push(new ValidationIssue({
3670
+ message: `entry '${entry}' must be a variable or parameter name matching [A-Za-z_][A-Za-z0-9_-]*`,
3671
+ location: document.locate(path)
3672
+ }));
3673
+ continue;
3674
+ }
3675
+ if (names.has(entry)) {
3676
+ issues.push(new ValidationIssue({
3677
+ message: `duplicate run name '${entry}'`,
3678
+ location: document.locate(path)
3679
+ }));
3680
+ continue;
3681
+ }
3682
+ names.add(entry);
3683
+ const run = {
3684
+ name: entry,
3685
+ variables: {},
3686
+ keys: [],
3687
+ location: path,
3688
+ secrets: []
3689
+ };
3690
+ const value = resolver.resolve(`\${${entry}}`, document, issues, run.secrets, {
3691
+ used,
3692
+ at: path
3693
+ });
3694
+ RunWithReader.set(run, name, value, keyLocation);
3695
+ runs.push(run);
3696
+ }
3697
+ return runs;
3698
+ }
3699
+ entries(from, document, resolver, issues, used) {
3700
+ const path = ["runWith", "from"];
3701
+ const location = document.locate(path);
3702
+ if (from === void 0) {
3703
+ issues.push(new ValidationIssue({
3704
+ message: "missing required key 'from'",
3705
+ location: document.locate(["runWith"])
3706
+ }));
3707
+ return;
3708
+ }
3709
+ let list;
3710
+ let at;
3711
+ if (Array.isArray(from)) {
3712
+ list = from;
3713
+ at = (index) => [...path, index];
3714
+ } else {
3715
+ const reference = typeof from === "string" ? LIST_REFERENCE.exec(from) : null;
3716
+ if (reference === null) {
3717
+ issues.push(new ValidationIssue({
3718
+ message: "must be a list, or a list variable such as ${tokens}",
3719
+ location
3720
+ }));
3721
+ return;
3722
+ }
3723
+ const name = reference[1];
3724
+ const found = resolver.find(name);
3725
+ used.add(name);
3726
+ if (found === void 0) {
3727
+ issues.push(new ValidationIssue({
3728
+ message: `list variable '${name}' is not set`,
3729
+ location
3730
+ }));
3731
+ return;
3732
+ }
3733
+ if (!Array.isArray(found)) {
3734
+ issues.push(new ValidationIssue({
3735
+ message: `variable '${name}' is not a list`,
3736
+ location
3737
+ }));
3738
+ return;
3739
+ }
3740
+ list = found;
3741
+ at = () => path;
3742
+ }
3743
+ if (list.length === 0) {
3744
+ issues.push(new ValidationIssue({
3745
+ message: "must not be empty",
3746
+ location
3747
+ }));
3748
+ return;
3749
+ }
3750
+ const bad = list.findIndex((item) => typeof item !== "string");
3751
+ if (bad >= 0) {
3752
+ issues.push(new ValidationIssue({
3753
+ message: "must be string",
3754
+ location: document.locate(at(bad))
3755
+ }));
3756
+ return;
3757
+ }
3758
+ return list.map((entry, index) => ({
3759
+ entry,
3760
+ path: at(index)
3761
+ }));
3762
+ }
3763
+ static set(run, name, value, location) {
3764
+ run.variables[name] = value;
3765
+ run.keys.push({
3766
+ name,
3767
+ location
3768
+ });
3769
+ if (EnvironmentResolver.isSecretName(name) && value !== "" && !run.secrets.includes(value)) run.secrets.push(value);
3770
+ }
3771
+ };
3487
3772
  // src/scenarios/scenarioLoader.ts
3488
3773
  const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
3489
3774
  var ScenarioLoader = class ScenarioLoader {
@@ -3492,6 +3777,7 @@ var ScenarioLoader = class ScenarioLoader {
3492
3777
  schemaValidator = new SchemaValidator(conversation_schema_default);
3493
3778
  conversationValidator = new ConversationValidator();
3494
3779
  variablesReader = new VariablesReader();
3780
+ runWithReader = new RunWithReader();
3495
3781
  webSocketResolver = new WebSocketResolver();
3496
3782
  environmentResolver;
3497
3783
  constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
@@ -3507,45 +3793,59 @@ var ScenarioLoader = class ScenarioLoader {
3507
3793
  for (const file of files) {
3508
3794
  const document = this.documentLoader.load(file, relative(this.cwd, file));
3509
3795
  const fileIssues = [...document.errors];
3510
- const secrets = [];
3796
+ const fileSecrets = [];
3511
3797
  const used = /* @__PURE__ */ new Set();
3512
- const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, secrets) : {};
3513
- const resolver = this.environmentResolver.withLevels([variables, config.variables]);
3514
- let value;
3515
- if (fileIssues.length === 0) {
3516
- const { variables: raw, ...rest } = document.value;
3517
- value = {
3518
- ...resolver.resolve(rest, document, fileIssues, secrets, { used }),
3519
- ...raw === void 0 ? {} : { variables }
3520
- };
3798
+ const variables = fileIssues.length === 0 ? this.variablesReader.read(document, this.environmentResolver.withLevels([config.variables]), fileIssues, fileSecrets) : {};
3799
+ const scenarioResolver = this.environmentResolver.withLevels([variables, config.variables]);
3800
+ const runs = fileIssues.length === 0 ? this.runWithReader.read(document, scenarioResolver, fileIssues, used) ?? [SINGLE_RUN] : [];
3801
+ if (fileIssues.length > 0) {
3802
+ issues.push(...fileIssues);
3803
+ continue;
3521
3804
  }
3522
- let websocket;
3523
- const websocketIssues = [];
3524
- if (fileIssues.length === 0 && configDocument !== void 0) {
3525
- const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
3526
- if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
3527
- connectionIssues.push({
3528
- file: document,
3529
- issues: websocketIssues
3530
- });
3531
- if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
3805
+ const { variables: rawVariables, runWith: _runWith, ...rest } = document.value;
3806
+ const results = runs.map((run) => this.resolveRun(run, rest, rawVariables === void 0 ? void 0 : variables, document, configDocument, scenarioResolver, [...fileSecrets, ...run.secrets]));
3807
+ for (const result of results) for (const name of result.used) used.add(name);
3808
+ fileIssues.push(...ScenarioLoader.byRun(results.map((result) => ({
3809
+ run: result.run,
3810
+ issues: result.issues
3811
+ })), (issue, run) => run === void 0 ? issue : new ValidationIssue({
3812
+ message: `run '${run.name}': ${issue.message}`,
3813
+ location: issue.location
3814
+ })));
3815
+ const attempted = results.filter((result) => result.connectionChecked);
3816
+ if (attempted.length > 0) connectionIssues.push({
3817
+ file: document,
3818
+ runs: attempted.map((result) => ({
3819
+ run: result.run,
3820
+ issues: result.websocketIssues
3821
+ }))
3822
+ });
3823
+ if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, results, document));
3824
+ if (fileIssues.length === 0 && results.every((result) => result.websocketIssues.length === 0)) for (const result of results) {
3825
+ const websocket = result.websocket;
3826
+ specs.push(this.build(result.value, document, result.run.name, websocket, [.../* @__PURE__ */ new Set([...result.secrets, ...websocket?.connection.subprotocols ?? []])]));
3532
3827
  }
3533
- if (fileIssues.length === 0) fileIssues.push(...this.schemaValidator.validate(value, document));
3534
- if (fileIssues.length === 0) fileIssues.push(...this.conversationValidator.validate(value, document));
3535
- if (fileIssues.length === 0) fileIssues.push(...ScenarioLoader.unusedVariables(variables, used, document));
3536
- if (fileIssues.length === 0 && websocketIssues.length === 0) specs.push(this.build(value, document, websocket, [.../* @__PURE__ */ new Set([...secrets, ...websocket?.connection.subprotocols ?? []])]));
3537
3828
  else issues.push(...fileIssues);
3538
3829
  }
3539
- const shared = ScenarioLoader.shared(connectionIssues.map((entry) => entry.issues));
3830
+ const shared = ScenarioLoader.shared(connectionIssues.flatMap((entry) => entry.runs.map((run) => run.issues)));
3540
3831
  if (shared.length > 0) throw new EvalError({
3541
3832
  category: ErrorCategory.Configuration,
3542
3833
  message: "invalid configuration",
3543
3834
  issues: shared
3544
3835
  });
3545
- for (const entry of connectionIssues) for (const issue of entry.issues) issues.push(new ValidationIssue({
3546
- message: issue.toString(),
3547
- location: ScenarioLoader.variablesLocation(entry.file)
3548
- }));
3836
+ for (const entry of connectionIssues) {
3837
+ const fileLocation = ScenarioLoader.variablesLocation(entry.file);
3838
+ issues.push(...ScenarioLoader.byRun(entry.runs, (issue, run) => run === void 0 ? new ValidationIssue({
3839
+ message: issue.toString(),
3840
+ location: fileLocation
3841
+ }) : new ValidationIssue({
3842
+ message: `run '${run.name}': ${issue.toString()}`,
3843
+ location: new SourceLocation({
3844
+ ...entry.file.locate(run.location),
3845
+ path: ""
3846
+ })
3847
+ })));
3848
+ }
3549
3849
  if (issues.length > 0) throw new EvalError({
3550
3850
  category: ErrorCategory.Specification,
3551
3851
  message: "invalid scenario specification",
@@ -3553,6 +3853,43 @@ var ScenarioLoader = class ScenarioLoader {
3553
3853
  });
3554
3854
  return specs;
3555
3855
  }
3856
+ resolveRun(run, rest, variables, document, configDocument, scenarioResolver, secrets) {
3857
+ const issues = [];
3858
+ const used = /* @__PURE__ */ new Set();
3859
+ const resolver = scenarioResolver.withLevels([run.variables]);
3860
+ const value = {
3861
+ ...resolver.resolve(rest, document, issues, secrets, { used }),
3862
+ ...variables === void 0 ? {} : { variables }
3863
+ };
3864
+ const websocketIssues = [];
3865
+ let websocket;
3866
+ let connectionChecked = false;
3867
+ if (issues.length === 0 && configDocument !== void 0) {
3868
+ const resolved = this.webSocketResolver.resolve(configDocument, resolver, websocketIssues, secrets, used, false);
3869
+ if (websocketIssues.length === 0) websocketIssues.push(...this.webSocketResolver.validate(resolved.value, configDocument));
3870
+ if (websocketIssues.length === 0) websocket = EvalConfigLoader.buildWebSocket(resolved.value);
3871
+ connectionChecked = true;
3872
+ }
3873
+ if (issues.length === 0) issues.push(...this.schemaValidator.validate(value, document));
3874
+ if (issues.length === 0) issues.push(...this.conversationValidator.validate(value, document));
3875
+ return {
3876
+ run,
3877
+ value,
3878
+ used,
3879
+ secrets,
3880
+ issues,
3881
+ websocket,
3882
+ websocketIssues,
3883
+ connectionChecked
3884
+ };
3885
+ }
3886
+ static byRun(runs, place) {
3887
+ const shared = ScenarioLoader.shared(runs.map((entry) => entry.issues));
3888
+ const sharedTexts = new Set(shared.map((issue) => issue.toString()));
3889
+ const result = shared.map((issue) => place(issue, void 0));
3890
+ for (const { run, issues } of runs) for (const issue of issues.filter((entry) => !sharedTexts.has(entry.toString()))) result.push(place(issue, run));
3891
+ return result;
3892
+ }
3556
3893
  static shared(lists) {
3557
3894
  if (lists.length === 0) return [];
3558
3895
  const texts = lists.slice(1).map((list) => new Set(list.map((issue) => issue.toString())));
@@ -3564,11 +3901,22 @@ var ScenarioLoader = class ScenarioLoader {
3564
3901
  path: ""
3565
3902
  });
3566
3903
  }
3567
- static unusedVariables(variables, used, document) {
3568
- return Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
3569
- message: `variable '${name}' is not used by the scenario or the websocket configuration`,
3904
+ static unusedVariables(variables, used, results, document) {
3905
+ const message = (name) => `variable '${name}' is not used by the scenario or the websocket configuration`;
3906
+ const issues = Object.keys(variables).filter((name) => !used.has(name)).map((name) => new ValidationIssue({
3907
+ message: message(name),
3570
3908
  location: document.locateKey(["variables"], name)
3571
3909
  }));
3910
+ const seen = /* @__PURE__ */ new Set();
3911
+ for (const { run, used: runUsed } of results) for (const key of run.keys.filter((entry) => !runUsed.has(entry.name))) {
3912
+ const issue = new ValidationIssue({
3913
+ message: message(key.name),
3914
+ location: key.location
3915
+ });
3916
+ if (!seen.has(issue.toString())) issues.push(issue);
3917
+ seen.add(issue.toString());
3918
+ }
3919
+ return issues;
3572
3920
  }
3573
3921
  discover(target, config) {
3574
3922
  if (!existsSync(target)) throw new EvalError({
@@ -3594,13 +3942,14 @@ var ScenarioLoader = class ScenarioLoader {
3594
3942
  }
3595
3943
  return files;
3596
3944
  }
3597
- build(value, document, websocket, secrets) {
3945
+ build(value, document, runName, websocket, secrets) {
3598
3946
  const scenario = value;
3599
3947
  const turns = scenario.turns;
3600
3948
  const conversation = isRecord(scenario.expect) ? scenario.expect.conversation : void 0;
3601
3949
  return new ConversationSpec({
3602
3950
  version: scenario.version,
3603
3951
  name: scenario.name,
3952
+ runName,
3604
3953
  description: typeof scenario.description === "string" ? scenario.description : "",
3605
3954
  turns: turns.map((turn, index) => this.buildTurn(turn, ["turns", index], document)),
3606
3955
  conversation: conversation ? new ConversationExpectation({