@artemiskit/core 0.2.3 → 0.2.4

package/CHANGELOG.md

@@ -1,5 +1,42 @@
 # @artemiskit/core
 
+## 0.2.4
+
+### Patch Changes
+
+- 16604a6: ## New Features
+
+  ### Validate Command
+
+  New `artemiskit validate` command for validating scenario files without running them:
+
+  - **YAML syntax validation** - Catches formatting errors
+  - **Schema validation** - Validates against ArtemisKit schema using Zod
+  - **Semantic validation** - Detects duplicate case IDs, undefined variables
+  - **Warnings** - Identifies deprecated fields, missing descriptions, performance hints
+
+  Options:
+
+  - `--json` - Output results as JSON
+  - `--strict` - Treat warnings as errors
+  - `--quiet` - Only show errors
+  - `--export junit` - Export to JUnit XML for CI integration
+
+  ### JUnit XML Export
+
+  Added JUnit XML export support for CI/CD integration with Jenkins, GitHub Actions, GitLab CI, and other systems:
+
+  - `akit run scenarios/ --export junit` - Export run results
+  - `akit redteam scenarios/chatbot.yaml --export junit` - Export security test results
+  - `akit validate scenarios/ --export junit` - Export validation results
+
+  JUnit reports include:
+
+  - Test suite metadata (run ID, provider, model, success rate)
+  - Individual test cases with pass/fail status
+  - Failure details with matcher type and expected values
+  - Timing information for each test
+
 ## 0.2.3
 
 ### Patch Changes

package/dist/index.d.ts

@@ -12,4 +12,5 @@ export * from './provenance';
 export * from './utils';
 export * from './redaction';
 export * from './cost';
+export * from './validator';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,cAAc,YAAY,CAAC;AAG3B,cAAc,YAAY,CAAC;AAG3B,cAAc,cAAc,CAAC;AAG7B,cAAc,UAAU,CAAC;AAGzB,cAAc,WAAW,CAAC;AAG1B,cAAc,aAAa,CAAC;AAG5B,cAAc,cAAc,CAAC;AAG7B,cAAc,SAAS,CAAC;AAGxB,cAAc,aAAa,CAAC;AAG5B,cAAc,QAAQ,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,cAAc,YAAY,CAAC;AAG3B,cAAc,YAAY,CAAC;AAG3B,cAAc,cAAc,CAAC;AAG7B,cAAc,UAAU,CAAC;AAGzB,cAAc,WAAW,CAAC;AAG1B,cAAc,aAAa,CAAC;AAG5B,cAAc,cAAc,CAAC;AAG7B,cAAc,SAAS,CAAC;AAGxB,cAAc,aAAa,CAAC;AAG5B,cAAc,QAAQ,CAAC;AAGvB,cAAc,aAAa,CAAC"}

package/dist/index.js

@@ -10896,6 +10896,55 @@ var require_public_api = __commonJS((exports) => {
   exports.stringify = stringify;
 });
 
+// ../../node_modules/.bun/yaml@2.8.2/node_modules/yaml/dist/index.js
+var require_dist = __commonJS((exports) => {
+  var composer = require_composer();
+  var Document = require_Document();
+  var Schema = require_Schema();
+  var errors2 = require_errors();
+  var Alias = require_Alias();
+  var identity = require_identity();
+  var Pair = require_Pair();
+  var Scalar = require_Scalar();
+  var YAMLMap = require_YAMLMap();
+  var YAMLSeq = require_YAMLSeq();
+  var cst = require_cst();
+  var lexer = require_lexer();
+  var lineCounter = require_line_counter();
+  var parser = require_parser();
+  var publicApi = require_public_api();
+  var visit = require_visit();
+  exports.Composer = composer.Composer;
+  exports.Document = Document.Document;
+  exports.Schema = Schema.Schema;
+  exports.YAMLError = errors2.YAMLError;
+  exports.YAMLParseError = errors2.YAMLParseError;
+  exports.YAMLWarning = errors2.YAMLWarning;
+  exports.Alias = Alias.Alias;
+  exports.isAlias = identity.isAlias;
+  exports.isCollection = identity.isCollection;
+  exports.isDocument = identity.isDocument;
+  exports.isMap = identity.isMap;
+  exports.isNode = identity.isNode;
+  exports.isPair = identity.isPair;
+  exports.isScalar = identity.isScalar;
+  exports.isSeq = identity.isSeq;
+  exports.Pair = Pair.Pair;
+  exports.Scalar = Scalar.Scalar;
+  exports.YAMLMap = YAMLMap.YAMLMap;
+  exports.YAMLSeq = YAMLSeq.YAMLSeq;
+  exports.CST = cst;
+  exports.Lexer = lexer.Lexer;
+  exports.LineCounter = lineCounter.LineCounter;
+  exports.Parser = parser.Parser;
+  exports.parse = publicApi.parse;
+  exports.parseAllDocuments = publicApi.parseAllDocuments;
+  exports.parseDocument = publicApi.parseDocument;
+  exports.stringify = publicApi.stringify;
+  exports.visit = visit.visit;
+  exports.visitAsync = visit.visitAsync;
+});
+
 // src/evaluators/combined.ts
 async function getEvaluatorForType(type) {
   const { getEvaluator } = await Promise.resolve().then(() => (init_evaluators(), exports_evaluators));
@@ -13616,55 +13665,8 @@ var ScenarioSchema = exports_external.object({
   }).optional()
 });
 // src/scenario/parser.ts
+var import_yaml = __toESM(require_dist(), 1);
 import { readFile } from "fs/promises";
-
-// ../../node_modules/.bun/yaml@2.8.2/node_modules/yaml/dist/index.js
-var composer = require_composer();
-var Document = require_Document();
-var Schema = require_Schema();
-var errors2 = require_errors();
-var Alias = require_Alias();
-var identity = require_identity();
-var Pair = require_Pair();
-var Scalar = require_Scalar();
-var YAMLMap = require_YAMLMap();
-var YAMLSeq = require_YAMLSeq();
-var cst = require_cst();
-var lexer = require_lexer();
-var lineCounter = require_line_counter();
-var parser = require_parser();
-var publicApi = require_public_api();
-var visit = require_visit();
-var $Composer = composer.Composer;
-var $Document = Document.Document;
-var $Schema = Schema.Schema;
-var $YAMLError = errors2.YAMLError;
-var $YAMLParseError = errors2.YAMLParseError;
-var $YAMLWarning = errors2.YAMLWarning;
-var $Alias = Alias.Alias;
-var $isAlias = identity.isAlias;
-var $isCollection = identity.isCollection;
-var $isDocument = identity.isDocument;
-var $isMap = identity.isMap;
-var $isNode = identity.isNode;
-var $isPair = identity.isPair;
-var $isScalar = identity.isScalar;
-var $isSeq = identity.isSeq;
-var $Pair = Pair.Pair;
-var $Scalar = Scalar.Scalar;
-var $YAMLMap = YAMLMap.YAMLMap;
-var $YAMLSeq = YAMLSeq.YAMLSeq;
-var $Lexer = lexer.Lexer;
-var $LineCounter = lineCounter.LineCounter;
-var $Parser = parser.Parser;
-var $parse = publicApi.parse;
-var $parseAllDocuments = publicApi.parseAllDocuments;
-var $parseDocument = publicApi.parseDocument;
-var $stringify = publicApi.stringify;
-var $visit = visit.visit;
-var $visitAsync = visit.visitAsync;
-
-// src/scenario/parser.ts
 function expandEnvVars(obj) {
   if (typeof obj === "string") {
     return obj.replace(/\$\{([^}]+)\}/g, (_, expr) => {
@@ -13699,7 +13701,7 @@ async function parseScenarioFile(filePath) {
 }
 function parseScenarioString(content, source) {
   try {
-    const raw = $parse(content);
+    const raw = import_yaml.parse(content);
     const expanded = expandEnvVars(raw);
     const result = ScenarioSchema.safeParse(expanded);
     if (!result.success) {
@@ -16929,7 +16931,7 @@ class RealtimeChannel {
       }).map((bind) => {
         if (typeof handledPayload === "object" && "ids" in handledPayload) {
           const postgresChanges = handledPayload.data;
-          const { schema: schema2, table, commit_timestamp, type: type2, errors: errors3 } = postgresChanges;
+          const { schema: schema2, table, commit_timestamp, type: type2, errors: errors2 } = postgresChanges;
           const enrichedPayload = {
             schema: schema2,
             table,
@@ -16937,7 +16939,7 @@ class RealtimeChannel {
             eventType: type2,
             new: {},
             old: {},
-            errors: errors3
+            errors: errors2
           };
           handledPayload = Object.assign(Object.assign({}, enrichedPayload), this._getPayloadRecords(postgresChanges));
         }
@@ -22508,7 +22510,7 @@ class GoTrueClient {
       }
     });
   }
-  async unlinkIdentity(identity2) {
+  async unlinkIdentity(identity) {
     try {
       return await this._useSession(async (result) => {
         var _a, _b;
@@ -22516,7 +22518,7 @@ class GoTrueClient {
         if (error) {
           throw error;
         }
-        return await _request(this.fetch, "DELETE", `${this.url}/user/identities/${identity2.identity_id}`, {
+        return await _request(this.fetch, "DELETE", `${this.url}/user/identities/${identity.identity_id}`, {
           headers: this.headers,
           jwt: (_b = (_a = data.session) === null || _a === undefined ? undefined : _a.access_token) !== null && _b !== undefined ? _b : undefined
         });
@@ -22690,20 +22692,20 @@ class GoTrueClient {
       if (this.broadcastChannel && broadcast) {
         this.broadcastChannel.postMessage({ event, session });
       }
-      const errors3 = [];
+      const errors2 = [];
       const promises = Array.from(this.stateChangeEmitters.values()).map(async (x) => {
         try {
           await x.callback(event, session);
         } catch (e) {
-          errors3.push(e);
+          errors2.push(e);
         }
       });
       await Promise.all(promises);
-      if (errors3.length > 0) {
-        for (let i = 0;i < errors3.length; i += 1) {
-          console.error(errors3[i]);
+      if (errors2.length > 0) {
+        for (let i = 0;i < errors2.length; i += 1) {
+          console.error(errors2[i]);
         }
-        throw errors3[0];
+        throw errors2[0];
       }
     } finally {
       this._debug(debugName, "end");
@@ -24751,6 +24753,251 @@ class Logger {
   }
 }
 var logger = new Logger("artemis");
+// src/validator/validator.ts
+var import_yaml2 = __toESM(require_dist(), 1);
+import { readFileSync } from "fs";
+class ScenarioValidator {
+  _options;
+  constructor(options = {}) {
+    this._options = options;
+  }
+  get options() {
+    return this._options;
+  }
+  validate(filePath) {
+    const errors4 = [];
+    const warnings = [];
+    let content;
+    try {
+      content = readFileSync(filePath, "utf-8");
+    } catch (err) {
+      const error = err;
+      errors4.push({
+        line: 1,
+        message: `Failed to read file: ${error.message}`,
+        rule: "file-read",
+        severity: "error"
+      });
+      return { file: filePath, valid: false, errors: errors4, warnings };
+    }
+    let parsed;
+    try {
+      parsed = import_yaml2.default.parse(content, {
+        prettyErrors: true,
+        strict: true
+      });
+    } catch (err) {
+      if (err instanceof import_yaml2.default.YAMLError) {
+        const linePos = err.linePos?.[0];
+        errors4.push({
+          line: linePos?.line || 1,
+          column: linePos?.col,
+          message: `Invalid YAML syntax: ${err.message}`,
+          rule: "yaml-syntax",
+          severity: "error"
+        });
+      } else {
+        errors4.push({
+          line: 1,
+          message: `YAML parse error: ${err.message}`,
+          rule: "yaml-syntax",
+          severity: "error"
+        });
+      }
+      return { file: filePath, valid: false, errors: errors4, warnings };
+    }
+    if (parsed === null || typeof parsed !== "object") {
+      errors4.push({
+        line: 1,
+        message: "Scenario must be a YAML object",
+        rule: "schema-type",
+        severity: "error"
+      });
+      return { file: filePath, valid: false, errors: errors4, warnings };
+    }
+    const schemaResult = ScenarioSchema.safeParse(parsed);
+    if (!schemaResult.success) {
+      const zodErrors = this.formatZodErrors(schemaResult.error, content);
+      errors4.push(...zodErrors);
+    }
+    if (schemaResult.success) {
+      const semanticErrors = this.validateSemantics(schemaResult.data, content);
+      errors4.push(...semanticErrors);
+    }
+    const detectedWarnings = this.detectWarnings(parsed, content);
+    warnings.push(...detectedWarnings);
+    return {
+      file: filePath,
+      valid: errors4.length === 0,
+      errors: errors4,
+      warnings
+    };
+  }
+  formatZodErrors(error, content) {
+    const issues = [];
+    const lines = content.split(`
+`);
+    for (const issue of error.issues) {
+      const path = issue.path.join(".");
+      const line = this.findLineForPath(lines, issue.path);
+      let message;
+      switch (issue.code) {
+        case "invalid_type":
+          message = `'${path}' expected ${issue.expected}, received ${issue.received}`;
+          break;
+        case "invalid_enum_value":
+          message = `'${path}' must be one of: ${issue.options.join(", ")}`;
+          break;
+        case "too_small":
+          if (issue.type === "array") {
+            message = `'${path}' must have at least ${issue.minimum} item(s)`;
+          } else {
+            message = `'${path}' is too small`;
+          }
+          break;
+        case "unrecognized_keys":
+          message = `Unrecognized field(s): ${issue.keys.join(", ")}`;
+          break;
+        default:
+          message = issue.message;
+      }
+      issues.push({
+        line,
+        message,
+        rule: `schema-${issue.code}`,
+        severity: "error"
+      });
+    }
+    return issues;
+  }
+  findLineForPath(lines, path) {
+    if (path.length === 0)
+      return 1;
+    const searchKey = String(path[path.length - 1]);
+    for (let i2 = 0;i2 < lines.length; i2++) {
+      const line = lines[i2];
+      if (line.includes(`${searchKey}:`) || line.includes(`- ${searchKey}:`)) {
+        return i2 + 1;
+      }
+      if (typeof path[path.length - 1] === "number" && path.includes("cases")) {
+        if (line.trim().startsWith("- id:")) {
+          return i2 + 1;
+        }
+      }
+    }
+    return 1;
+  }
+  validateSemantics(scenario, content) {
+    const errors4 = [];
+    const lines = content.split(`
+`);
+    const caseIds = new Set;
+    for (const testCase of scenario.cases) {
+      if (caseIds.has(testCase.id)) {
+        const line = this.findLineForCaseId(lines, testCase.id);
+        errors4.push({
+          line,
+          message: `Duplicate case ID: '${testCase.id}'`,
+          rule: "duplicate-case-id",
+          severity: "error"
+        });
+      }
+      caseIds.add(testCase.id);
+    }
+    const globalVars = scenario.variables || {};
+    for (const testCase of scenario.cases) {
+      const caseVars = testCase.variables || {};
+      const allVars = { ...globalVars, ...caseVars };
+      const prompt2 = typeof testCase.prompt === "string" ? testCase.prompt : JSON.stringify(testCase.prompt);
+      const refs = this.extractVariableRefs(prompt2);
+      for (const ref of refs) {
+        if (!(ref in allVars)) {
+          const line = this.findLineForCaseId(lines, testCase.id);
+          errors4.push({
+            line,
+            message: `Undefined variable '{{${ref}}}' in case '${testCase.id}'`,
+            rule: "undefined-variable",
+            severity: "error",
+            suggestion: `Define '${ref}' in scenario.variables or case.variables`
+          });
+        }
+      }
+    }
+    return errors4;
+  }
+  findLineForCaseId(lines, caseId) {
+    for (let i2 = 0;i2 < lines.length; i2++) {
+      if (lines[i2].includes(`id: ${caseId}`) || lines[i2].includes(`id: "${caseId}"`) || lines[i2].includes(`id: '${caseId}'`)) {
+        return i2 + 1;
+      }
+    }
+    return 1;
+  }
+  extractVariableRefs(text) {
+    const regex2 = /\{\{(\w+)\}\}/g;
+    const refs = [];
+    const matches = text.matchAll(regex2);
+    for (const match of matches) {
+      refs.push(match[1]);
+    }
+    return refs;
+  }
+  detectWarnings(parsed, content) {
+    const warnings = [];
+    const lines = content.split(`
+`);
+    if (parsed && typeof parsed === "object") {
+      const obj = parsed;
+      if (this.hasDeepKey(obj, "criteria")) {
+        const line = this.findLineForKey(lines, "criteria");
+        warnings.push({
+          line,
+          message: "'criteria' is deprecated, use 'rubric' instead (llm_grader)",
+          rule: "deprecated-field",
+          severity: "warning",
+          suggestion: "Replace 'criteria' with 'rubric'"
+        });
+      }
+      const cases = obj.cases;
+      if (Array.isArray(cases) && cases.length > 20) {
+        warnings.push({
+          line: 1,
+          message: `Scenario has ${cases.length} cases. Consider using --parallel for faster execution.`,
+          rule: "performance-hint",
+          severity: "warning"
+        });
+      }
+      if (!obj.description) {
+        warnings.push({
+          line: 1,
+          message: "Scenario is missing 'description' field. Adding a description improves documentation.",
+          rule: "missing-description",
+          severity: "warning"
+        });
+      }
+    }
+    return warnings;
+  }
+  hasDeepKey(obj, key) {
+    if (obj === null || typeof obj !== "object")
+      return false;
+    if (key in obj)
+      return true;
+    for (const value of Object.values(obj)) {
+      if (this.hasDeepKey(value, key))
+        return true;
+    }
+    return false;
+  }
+  findLineForKey(lines, key) {
+    for (let i2 = 0;i2 < lines.length; i2++) {
+      if (lines[i2].includes(`${key}:`)) {
+        return i2 + 1;
+      }
+    }
+    return 1;
+  }
+}
 export {
   wrapError,
   validateScenario,
@@ -24798,6 +25045,7 @@ export {
   TestCaseSchema,
   SupabaseStorageAdapter,
   SimilarityEvaluator,
+  ScenarioValidator,
   ScenarioSchema,
   SUPPORTED_EXPRESSIONS,
   RegexEvaluator,

package/dist/validator/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Validator module exports
+ */
+export * from './types';
+export { ScenarioValidator } from './validator';
+//# sourceMappingURL=index.d.ts.map

package/dist/validator/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/validator/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC"}

package/dist/validator/types.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Validator types
+ */
+/**
+ * Validation error severity
+ */
+export type ValidationSeverity = 'error' | 'warning';
+/**
+ * Validation error/warning
+ */
+export interface ValidationIssue {
+    /** Line number in the file (1-indexed) */
+    line: number;
+    /** Column number (optional) */
+    column?: number;
+    /** Error/warning message */
+    message: string;
+    /** Rule that triggered this issue */
+    rule: string;
+    /** Severity level */
+    severity: ValidationSeverity;
+    /** Suggested fix (optional) */
+    suggestion?: string;
+}
+/**
+ * Result for a single file validation
+ */
+export interface ValidationResult {
+    /** File path that was validated */
+    file: string;
+    /** Whether the file is valid (no errors) */
+    valid: boolean;
+    /** List of errors found */
+    errors: ValidationIssue[];
+    /** List of warnings found */
+    warnings: ValidationIssue[];
+}
+/**
+ * Summary of validation across multiple files
+ */
+export interface ValidationSummary {
+    /** Total files validated */
+    total: number;
+    /** Files that passed validation */
+    passed: number;
+    /** Files that failed validation */
+    failed: number;
+    /** Files with warnings only */
+    withWarnings: number;
+}
+/**
+ * Options for the validator
+ */
+export interface ValidatorOptions {
+    /** Treat warnings as errors */
+    strict?: boolean;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/validator/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/validator/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,OAAO,GAAG,SAAS,CAAC;AAErD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb,+BAA+B;IAC/B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4BAA4B;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,qBAAqB;IACrB,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,+BAA+B;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,4CAA4C;IAC5C,KAAK,EAAE,OAAO,CAAC;IACf,2BAA2B;IAC3B,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,6BAA6B;IAC7B,QAAQ,EAAE,eAAe,EAAE,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,mCAAmC;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,+BAA+B;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB"}

package/dist/validator/validator.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Scenario Validator
+ *
+ * Validates scenario files for:
+ * 1. YAML syntax errors
+ * 2. Schema violations (required fields, types)
+ * 3. Semantic errors (duplicate IDs, undefined variables)
+ * 4. Warnings (deprecated patterns)
+ */
+import type { ValidationResult, ValidatorOptions } from './types';
+/**
+ * Scenario validator class
+ */
+export declare class ScenarioValidator {
+    private _options;
+    constructor(options?: ValidatorOptions);
+    get options(): ValidatorOptions;
+    /**
+     * Validate a scenario file
+     */
+    validate(filePath: string): ValidationResult;
+    /**
+     * Format Zod errors into ValidationIssues
+     */
+    private formatZodErrors;
+    /**
+     * Find approximate line number for a YAML path
+     */
+    private findLineForPath;
+    /**
+     * Validate semantic rules
+     */
+    private validateSemantics;
+    /**
+     * Find line number for a case ID
+     */
+    private findLineForCaseId;
+    /**
+     * Extract variable references from a string ({{varName}} format)
+     */
+    private extractVariableRefs;
+    /**
+     * Detect warnings (non-blocking issues)
+     */
+    private detectWarnings;
+    /**
+     * Check if object has a key at any depth
+     */
+    private hasDeepKey;
+    /**
+     * Find line number for a key
+     */
+    private findLineForKey;
+}
+//# sourceMappingURL=validator.d.ts.map

package/dist/validator/validator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validator.d.ts","sourceRoot":"","sources":["../../src/validator/validator.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,OAAO,KAAK,EAAmB,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAEnF;;GAEG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAmB;gBAEvB,OAAO,GAAE,gBAAqB;IAI1C,IAAI,OAAO,IAAI,gBAAgB,CAE9B;IAED;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,gBAAgB;IAmF5C;;OAEG;IACH,OAAO,CAAC,eAAe;IAyCvB;;OAEG;IACH,OAAO,CAAC,eAAe;IAuBvB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAoDzB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAazB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAU3B;;OAEG;IACH,OAAO,CAAC,cAAc;IA6CtB;;OAEG;IACH,OAAO,CAAC,UAAU;IAYlB;;OAEG;IACH,OAAO,CAAC,cAAc;CAQvB"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@artemiskit/core",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "description": "Core runner, evaluators, and storage for ArtemisKit LLM evaluation toolkit",
   "type": "module",
   "license": "Apache-2.0",

package/src/index.ts

@@ -32,3 +32,6 @@ export * from './redaction';
 
 // Cost estimation
 export * from './cost';
+
+// Validator
+export * from './validator';

package/src/validator/index.ts

@@ -0,0 +1,6 @@
+/**
+ * Validator module exports
+ */
+
+export * from './types';
+export { ScenarioValidator } from './validator';

package/src/validator/types.ts

@@ -0,0 +1,62 @@
+/**
+ * Validator types
+ */
+
+/**
+ * Validation error severity
+ */
+export type ValidationSeverity = 'error' | 'warning';
+
+/**
+ * Validation error/warning
+ */
+export interface ValidationIssue {
+  /** Line number in the file (1-indexed) */
+  line: number;
+  /** Column number (optional) */
+  column?: number;
+  /** Error/warning message */
+  message: string;
+  /** Rule that triggered this issue */
+  rule: string;
+  /** Severity level */
+  severity: ValidationSeverity;
+  /** Suggested fix (optional) */
+  suggestion?: string;
+}
+
+/**
+ * Result for a single file validation
+ */
+export interface ValidationResult {
+  /** File path that was validated */
+  file: string;
+  /** Whether the file is valid (no errors) */
+  valid: boolean;
+  /** List of errors found */
+  errors: ValidationIssue[];
+  /** List of warnings found */
+  warnings: ValidationIssue[];
+}
+
+/**
+ * Summary of validation across multiple files
+ */
+export interface ValidationSummary {
+  /** Total files validated */
+  total: number;
+  /** Files that passed validation */
+  passed: number;
+  /** Files that failed validation */
+  failed: number;
+  /** Files with warnings only */
+  withWarnings: number;
+}
+
+/**
+ * Options for the validator
+ */
+export interface ValidatorOptions {
+  /** Treat warnings as errors */
+  strict?: boolean;
+}

package/src/validator/validator.ts

@@ -0,0 +1,345 @@
+/**
+ * Scenario Validator
+ *
+ * Validates scenario files for:
+ * 1. YAML syntax errors
+ * 2. Schema violations (required fields, types)
+ * 3. Semantic errors (duplicate IDs, undefined variables)
+ * 4. Warnings (deprecated patterns)
+ */
+
+import { readFileSync } from 'node:fs';
+import yaml from 'yaml';
+import type { ZodError } from 'zod';
+import { ScenarioSchema } from '../scenario/schema';
+import type { ValidationIssue, ValidationResult, ValidatorOptions } from './types';
+
+/**
+ * Scenario validator class
+ */
+export class ScenarioValidator {
+  private _options: ValidatorOptions;
+
+  constructor(options: ValidatorOptions = {}) {
+    this._options = options;
+  }
+
+  get options(): ValidatorOptions {
+    return this._options;
+  }
+
+  /**
+   * Validate a scenario file
+   */
+  validate(filePath: string): ValidationResult {
+    const errors: ValidationIssue[] = [];
+    const warnings: ValidationIssue[] = [];
+
+    // Read file content
+    let content: string;
+    try {
+      content = readFileSync(filePath, 'utf-8');
+    } catch (err) {
+      const error = err as NodeJS.ErrnoException;
+      errors.push({
+        line: 1,
+        message: `Failed to read file: ${error.message}`,
+        rule: 'file-read',
+        severity: 'error',
+      });
+      return { file: filePath, valid: false, errors, warnings };
+    }
+
+    // Level 1: YAML Syntax validation
+    let parsed: unknown;
+    try {
+      parsed = yaml.parse(content, {
+        prettyErrors: true,
+        strict: true,
+      });
+    } catch (err) {
+      if (err instanceof yaml.YAMLError) {
+        const linePos = err.linePos?.[0];
+        errors.push({
+          line: linePos?.line || 1,
+          column: linePos?.col,
+          message: `Invalid YAML syntax: ${err.message}`,
+          rule: 'yaml-syntax',
+          severity: 'error',
+        });
+      } else {
+        errors.push({
+          line: 1,
+          message: `YAML parse error: ${(err as Error).message}`,
+          rule: 'yaml-syntax',
+          severity: 'error',
+        });
+      }
+      return { file: filePath, valid: false, errors, warnings };
+    }
+
+    // Check if parsed result is null or not an object
+    if (parsed === null || typeof parsed !== 'object') {
+      errors.push({
+        line: 1,
+        message: 'Scenario must be a YAML object',
+        rule: 'schema-type',
+        severity: 'error',
+      });
+      return { file: filePath, valid: false, errors, warnings };
+    }
+
+    // Level 2: Schema validation using Zod
+    const schemaResult = ScenarioSchema.safeParse(parsed);
+    if (!schemaResult.success) {
+      const zodErrors = this.formatZodErrors(schemaResult.error, content);
+      errors.push(...zodErrors);
+    }
+
+    // Level 3: Semantic validation (only if schema passed)
+    if (schemaResult.success) {
+      const semanticErrors = this.validateSemantics(schemaResult.data, content);
+      errors.push(...semanticErrors);
+    }
+
+    // Level 4: Warnings detection
+    const detectedWarnings = this.detectWarnings(parsed, content);
+    warnings.push(...detectedWarnings);
+
+    return {
+      file: filePath,
+      valid: errors.length === 0,
+      errors,
+      warnings,
+    };
+  }
+
+  /**
+   * Format Zod errors into ValidationIssues
+   */
+  private formatZodErrors(error: ZodError, content: string): ValidationIssue[] {
+    const issues: ValidationIssue[] = [];
+    const lines = content.split('\n');
+
+    for (const issue of error.issues) {
+      const path = issue.path.join('.');
+      const line = this.findLineForPath(lines, issue.path);
+
+      let message: string;
+      switch (issue.code) {
+        case 'invalid_type':
+          message = `'${path}' expected ${issue.expected}, received ${issue.received}`;
+          break;
+        case 'invalid_enum_value':
+          message = `'${path}' must be one of: ${(issue as { options: string[] }).options.join(', ')}`;
+          break;
+        case 'too_small':
+          if ((issue as { type: string }).type === 'array') {
+            message = `'${path}' must have at least ${(issue as { minimum: number }).minimum} item(s)`;
+          } else {
+            message = `'${path}' is too small`;
+          }
+          break;
+        case 'unrecognized_keys':
+          message = `Unrecognized field(s): ${(issue as { keys: string[] }).keys.join(', ')}`;
+          break;
+        default:
+          message = issue.message;
+      }
+
+      issues.push({
+        line,
+        message,
+        rule: `schema-${issue.code}`,
+        severity: 'error',
+      });
+    }
+
+    return issues;
+  }
+
+  /**
+   * Find approximate line number for a YAML path
+   */
+  private findLineForPath(lines: string[], path: (string | number)[]): number {
+    if (path.length === 0) return 1;
+
+    // Simple heuristic: search for the key in the file
+    const searchKey = String(path[path.length - 1]);
+
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      // Check if line contains the key (accounting for YAML formatting)
+      if (line.includes(`${searchKey}:`) || line.includes(`- ${searchKey}:`)) {
+        return i + 1; // 1-indexed
+      }
+      // For array indices, look for "- id:" pattern
+      if (typeof path[path.length - 1] === 'number' && path.includes('cases')) {
+        if (line.trim().startsWith('- id:')) {
+          return i + 1;
+        }
+      }
+    }
+
+    return 1; // Default to first line
+  }
+
+  /**
+   * Validate semantic rules
+   */
+  private validateSemantics(
+    scenario: {
+      cases: Array<{ id: string; prompt: string | unknown; variables?: Record<string, unknown> }>;
+      variables?: Record<string, unknown>;
+    },
+    content: string
+  ): ValidationIssue[] {
+    const errors: ValidationIssue[] = [];
+    const lines = content.split('\n');
+
+    // Check for duplicate case IDs
+    const caseIds = new Set<string>();
+    for (const testCase of scenario.cases) {
+      if (caseIds.has(testCase.id)) {
+        const line = this.findLineForCaseId(lines, testCase.id);
+        errors.push({
+          line,
+          message: `Duplicate case ID: '${testCase.id}'`,
+          rule: 'duplicate-case-id',
+          severity: 'error',
+        });
+      }
+      caseIds.add(testCase.id);
+    }
+
+    // Check variable references
+    const globalVars = scenario.variables || {};
+    for (const testCase of scenario.cases) {
+      const caseVars = testCase.variables || {};
+      const allVars = { ...globalVars, ...caseVars };
+
+      const prompt =
+        typeof testCase.prompt === 'string' ? testCase.prompt : JSON.stringify(testCase.prompt);
+
+      const refs = this.extractVariableRefs(prompt);
+      for (const ref of refs) {
+        if (!(ref in allVars)) {
+          const line = this.findLineForCaseId(lines, testCase.id);
+          errors.push({
+            line,
+            message: `Undefined variable '{{${ref}}}' in case '${testCase.id}'`,
+            rule: 'undefined-variable',
+            severity: 'error',
+            suggestion: `Define '${ref}' in scenario.variables or case.variables`,
+          });
+        }
+      }
+    }
+
+    return errors;
+  }
+
+  /**
+   * Find line number for a case ID
+   */
+  private findLineForCaseId(lines: string[], caseId: string): number {
+    for (let i = 0; i < lines.length; i++) {
+      if (
+        lines[i].includes(`id: ${caseId}`) ||
+        lines[i].includes(`id: "${caseId}"`) ||
+        lines[i].includes(`id: '${caseId}'`)
+      ) {
+        return i + 1;
+      }
+    }
+    return 1;
+  }
+
+  /**
+   * Extract variable references from a string ({{varName}} format)
+   */
+  private extractVariableRefs(text: string): string[] {
+    const regex = /\{\{(\w+)\}\}/g;
+    const refs: string[] = [];
+    const matches = text.matchAll(regex);
+    for (const match of matches) {
+      refs.push(match[1]);
+    }
+    return refs;
+  }
+
+  /**
+   * Detect warnings (non-blocking issues)
+   */
+  private detectWarnings(parsed: unknown, content: string): ValidationIssue[] {
+    const warnings: ValidationIssue[] = [];
+    const lines = content.split('\n');
+
+    if (parsed && typeof parsed === 'object') {
+      const obj = parsed as Record<string, unknown>;
+
+      // Check for deprecated 'criteria' field (should be 'rubric' for llm_grader)
+      if (this.hasDeepKey(obj, 'criteria')) {
+        const line = this.findLineForKey(lines, 'criteria');
+        warnings.push({
+          line,
+          message: "'criteria' is deprecated, use 'rubric' instead (llm_grader)",
+          rule: 'deprecated-field',
+          severity: 'warning',
+          suggestion: "Replace 'criteria' with 'rubric'",
+        });
+      }
+
+      // Check for very large number of cases without parallel recommendation
+      const cases = obj.cases as unknown[] | undefined;
+      if (Array.isArray(cases) && cases.length > 20) {
+        warnings.push({
+          line: 1,
+          message: `Scenario has ${cases.length} cases. Consider using --parallel for faster execution.`,
+          rule: 'performance-hint',
+          severity: 'warning',
+        });
+      }
+
+      // Check for missing description
+      if (!obj.description) {
+        warnings.push({
+          line: 1,
+          message:
+            "Scenario is missing 'description' field. Adding a description improves documentation.",
+          rule: 'missing-description',
+          severity: 'warning',
+        });
+      }
+    }
+
+    return warnings;
+  }
+
+  /**
+   * Check if object has a key at any depth
+   */
+  private hasDeepKey(obj: unknown, key: string): boolean {
+    if (obj === null || typeof obj !== 'object') return false;
+
+    if (key in (obj as Record<string, unknown>)) return true;
+
+    for (const value of Object.values(obj as Record<string, unknown>)) {
+      if (this.hasDeepKey(value, key)) return true;
+    }
+
+    return false;
+  }
+
+  /**
+   * Find line number for a key
+   */
+  private findLineForKey(lines: string[], key: string): number {
+    for (let i = 0; i < lines.length; i++) {
+      if (lines[i].includes(`${key}:`)) {
+        return i + 1;
+      }
+    }
+    return 1;
+  }
+}