@artemiskit/core 0.2.3 → 0.3.0

package/src/storage/types.ts

@@ -173,3 +173,199 @@ export interface BaselineStorageAdapter extends StorageAdapter {
     regressionThreshold: number;
   } | null>;
 }
+
+// ============================================================================
+// Case Results Types (for granular analytics)
+// ============================================================================
+
+/**
+ * Status of an individual case result
+ */
+export type CaseResultStatus = 'passed' | 'failed' | 'error';
+
+/**
+ * Individual case result record for storage
+ */
+export interface CaseResultRecord {
+  /** Unique ID (auto-generated if not provided) */
+  id?: string;
+  /** Run ID this case belongs to */
+  runId: string;
+  /** Case ID from the test */
+  caseId: string;
+  /** Optional case name */
+  caseName?: string;
+  /** Result status */
+  status: CaseResultStatus;
+  /** Score from 0.0 to 1.0 */
+  score: number;
+  /** Type of matcher used */
+  matcherType: string;
+  /** Reason for the status */
+  reason?: string;
+  /** Model response */
+  response: string;
+  /** Latency in milliseconds */
+  latencyMs: number;
+  /** Prompt tokens used */
+  promptTokens: number;
+  /** Completion tokens used */
+  completionTokens: number;
+  /** Total tokens used */
+  totalTokens: number;
+  /** Error message if status is 'error' */
+  error?: string;
+  /** Tags for categorization */
+  tags?: string[];
+  /** ISO timestamp when created */
+  createdAt?: string;
+}
+
+/**
+ * Options for querying case results
+ */
+export interface CaseResultQueryOptions {
+  /** Filter by run ID */
+  runId?: string;
+  /** Filter by case ID */
+  caseId?: string;
+  /** Filter by status */
+  status?: CaseResultStatus;
+  /** Filter by tags (any match) */
+  tags?: string[];
+  /** Maximum results to return */
+  limit?: number;
+  /** Offset for pagination */
+  offset?: number;
+}
+
+// ============================================================================
+// Metrics History Types (for trending)
+// ============================================================================
+
+/**
+ * Daily metrics snapshot for a project/scenario
+ */
+export interface MetricsSnapshot {
+  /** Unique ID (auto-generated if not provided) */
+  id?: string;
+  /** Date of the snapshot (YYYY-MM-DD) */
+  date: string;
+  /** Project name */
+  project: string;
+  /** Optional scenario name (null for project-wide) */
+  scenario?: string;
+  /** Total runs on this date */
+  totalRuns: number;
+  /** Total cases across all runs */
+  totalCases: number;
+  /** Total passed cases */
+  passedCases: number;
+  /** Total failed cases */
+  failedCases: number;
+  /** Average success rate */
+  avgSuccessRate: number;
+  /** Average latency in ms */
+  avgLatencyMs: number;
+  /** Average tokens per run */
+  avgTokensPerRun: number;
+  /** Minimum success rate */
+  minSuccessRate?: number;
+  /** Maximum success rate */
+  maxSuccessRate?: number;
+  /** Minimum latency in ms */
+  minLatencyMs?: number;
+  /** Maximum latency in ms */
+  maxLatencyMs?: number;
+  /** Total tokens consumed */
+  totalTokens: number;
+  /** ISO timestamp when created */
+  createdAt?: string;
+  /** ISO timestamp when last updated */
+  updatedAt?: string;
+}
+
+/**
+ * Options for querying metrics history
+ */
+export interface MetricsTrendOptions {
+  /** Project to query */
+  project: string;
+  /** Optional scenario filter */
+  scenario?: string;
+  /** Start date (YYYY-MM-DD) */
+  startDate?: string;
+  /** End date (YYYY-MM-DD) */
+  endDate?: string;
+  /** Maximum results to return */
+  limit?: number;
+}
+
+/**
+ * Trend data point for visualization
+ */
+export interface TrendDataPoint {
+  date: string;
+  successRate: number;
+  latencyMs: number;
+  totalRuns: number;
+  totalTokens: number;
+}
+
+// ============================================================================
+// Enhanced Storage Adapter with Analytics
+// ============================================================================
+
+/**
+ * Extended storage adapter with analytics capabilities
+ */
+export interface AnalyticsStorageAdapter extends BaselineStorageAdapter {
+  /**
+   * Save an individual case result
+   */
+  saveCaseResult(result: CaseResultRecord): Promise<string>;
+
+  /**
+   * Save multiple case results in batch
+   */
+  saveCaseResults(results: CaseResultRecord[]): Promise<string[]>;
+
+  /**
+   * Get case results for a run
+   */
+  getCaseResults(runId: string): Promise<CaseResultRecord[]>;
+
+  /**
+   * Query case results with filters
+   */
+  queryCaseResults(options: CaseResultQueryOptions): Promise<CaseResultRecord[]>;
+
+  /**
+   * Save a metrics snapshot
+   */
+  saveMetricsSnapshot(snapshot: MetricsSnapshot): Promise<string>;
+
+  /**
+   * Get metrics trend data
+   */
+  getMetricsTrend(options: MetricsTrendOptions): Promise<TrendDataPoint[]>;
+
+  /**
+   * Get a specific metrics snapshot
+   */
+  getMetricsSnapshot(
+    date: string,
+    project: string,
+    scenario?: string
+  ): Promise<MetricsSnapshot | null>;
+
+  /**
+   * Aggregate and save daily metrics from runs
+   * This can be called to build/update metrics_history from existing runs
+   */
+  aggregateDailyMetrics?(
+    date: string,
+    project: string,
+    scenario?: string
+  ): Promise<MetricsSnapshot>;
+}

package/src/validator/index.ts

@@ -0,0 +1,6 @@
+/**
+ * Validator module exports
+ */
+
+export * from './types';
+export { ScenarioValidator } from './validator';

package/src/validator/types.ts

@@ -0,0 +1,62 @@
+/**
+ * Validator types
+ */
+
+/**
+ * Validation error severity
+ */
+export type ValidationSeverity = 'error' | 'warning';
+
+/**
+ * Validation error/warning
+ */
+export interface ValidationIssue {
+  /** Line number in the file (1-indexed) */
+  line: number;
+  /** Column number (optional) */
+  column?: number;
+  /** Error/warning message */
+  message: string;
+  /** Rule that triggered this issue */
+  rule: string;
+  /** Severity level */
+  severity: ValidationSeverity;
+  /** Suggested fix (optional) */
+  suggestion?: string;
+}
+
+/**
+ * Result for a single file validation
+ */
+export interface ValidationResult {
+  /** File path that was validated */
+  file: string;
+  /** Whether the file is valid (no errors) */
+  valid: boolean;
+  /** List of errors found */
+  errors: ValidationIssue[];
+  /** List of warnings found */
+  warnings: ValidationIssue[];
+}
+
+/**
+ * Summary of validation across multiple files
+ */
+export interface ValidationSummary {
+  /** Total files validated */
+  total: number;
+  /** Files that passed validation */
+  passed: number;
+  /** Files that failed validation */
+  failed: number;
+  /** Files with warnings only */
+  withWarnings: number;
+}
+
+/**
+ * Options for the validator
+ */
+export interface ValidatorOptions {
+  /** Treat warnings as errors */
+  strict?: boolean;
+}

package/src/validator/validator.ts

@@ -0,0 +1,345 @@
+/**
+ * Scenario Validator
+ *
+ * Validates scenario files for:
+ * 1. YAML syntax errors
+ * 2. Schema violations (required fields, types)
+ * 3. Semantic errors (duplicate IDs, undefined variables)
+ * 4. Warnings (deprecated patterns)
+ */
+
+import { readFileSync } from 'node:fs';
+import yaml from 'yaml';
+import type { ZodError } from 'zod';
+import { ScenarioSchema } from '../scenario/schema';
+import type { ValidationIssue, ValidationResult, ValidatorOptions } from './types';
+
+/**
+ * Scenario validator class
+ */
+export class ScenarioValidator {
+  private _options: ValidatorOptions;
+
+  constructor(options: ValidatorOptions = {}) {
+    this._options = options;
+  }
+
+  get options(): ValidatorOptions {
+    return this._options;
+  }
+
+  /**
+   * Validate a scenario file
+   */
+  validate(filePath: string): ValidationResult {
+    const errors: ValidationIssue[] = [];
+    const warnings: ValidationIssue[] = [];
+
+    // Read file content
+    let content: string;
+    try {
+      content = readFileSync(filePath, 'utf-8');
+    } catch (err) {
+      const error = err as NodeJS.ErrnoException;
+      errors.push({
+        line: 1,
+        message: `Failed to read file: ${error.message}`,
+        rule: 'file-read',
+        severity: 'error',
+      });
+      return { file: filePath, valid: false, errors, warnings };
+    }
+
+    // Level 1: YAML Syntax validation
+    let parsed: unknown;
+    try {
+      parsed = yaml.parse(content, {
+        prettyErrors: true,
+        strict: true,
+      });
+    } catch (err) {
+      if (err instanceof yaml.YAMLError) {
+        const linePos = err.linePos?.[0];
+        errors.push({
+          line: linePos?.line || 1,
+          column: linePos?.col,
+          message: `Invalid YAML syntax: ${err.message}`,
+          rule: 'yaml-syntax',
+          severity: 'error',
+        });
+      } else {
+        errors.push({
+          line: 1,
+          message: `YAML parse error: ${(err as Error).message}`,
+          rule: 'yaml-syntax',
+          severity: 'error',
+        });
+      }
+      return { file: filePath, valid: false, errors, warnings };
+    }
+
+    // Check if parsed result is null or not an object
+    if (parsed === null || typeof parsed !== 'object') {
+      errors.push({
+        line: 1,
+        message: 'Scenario must be a YAML object',
+        rule: 'schema-type',
+        severity: 'error',
+      });
+      return { file: filePath, valid: false, errors, warnings };
+    }
+
+    // Level 2: Schema validation using Zod
+    const schemaResult = ScenarioSchema.safeParse(parsed);
+    if (!schemaResult.success) {
+      const zodErrors = this.formatZodErrors(schemaResult.error, content);
+      errors.push(...zodErrors);
+    }
+
+    // Level 3: Semantic validation (only if schema passed)
+    if (schemaResult.success) {
+      const semanticErrors = this.validateSemantics(schemaResult.data, content);
+      errors.push(...semanticErrors);
+    }
+
+    // Level 4: Warnings detection
+    const detectedWarnings = this.detectWarnings(parsed, content);
+    warnings.push(...detectedWarnings);
+
+    return {
+      file: filePath,
+      valid: errors.length === 0,
+      errors,
+      warnings,
+    };
+  }
+
+  /**
+   * Format Zod errors into ValidationIssues
+   */
+  private formatZodErrors(error: ZodError, content: string): ValidationIssue[] {
+    const issues: ValidationIssue[] = [];
+    const lines = content.split('\n');
+
+    for (const issue of error.issues) {
+      const path = issue.path.join('.');
+      const line = this.findLineForPath(lines, issue.path);
+
+      let message: string;
+      switch (issue.code) {
+        case 'invalid_type':
+          message = `'${path}' expected ${issue.expected}, received ${issue.received}`;
+          break;
+        case 'invalid_enum_value':
+          message = `'${path}' must be one of: ${(issue as { options: string[] }).options.join(', ')}`;
+          break;
+        case 'too_small':
+          if ((issue as { type: string }).type === 'array') {
+            message = `'${path}' must have at least ${(issue as { minimum: number }).minimum} item(s)`;
+          } else {
+            message = `'${path}' is too small`;
+          }
+          break;
+        case 'unrecognized_keys':
+          message = `Unrecognized field(s): ${(issue as { keys: string[] }).keys.join(', ')}`;
+          break;
+        default:
+          message = issue.message;
+      }
+
+      issues.push({
+        line,
+        message,
+        rule: `schema-${issue.code}`,
+        severity: 'error',
+      });
+    }
+
+    return issues;
+  }
+
+  /**
+   * Find approximate line number for a YAML path
+   */
+  private findLineForPath(lines: string[], path: (string | number)[]): number {
+    if (path.length === 0) return 1;
+
+    // Simple heuristic: search for the key in the file
+    const searchKey = String(path[path.length - 1]);
+
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      // Check if line contains the key (accounting for YAML formatting)
+      if (line.includes(`${searchKey}:`) || line.includes(`- ${searchKey}:`)) {
+        return i + 1; // 1-indexed
+      }
+      // For array indices, look for "- id:" pattern
+      if (typeof path[path.length - 1] === 'number' && path.includes('cases')) {
+        if (line.trim().startsWith('- id:')) {
+          return i + 1;
+        }
+      }
+    }
+
+    return 1; // Default to first line
+  }
+
+  /**
+   * Validate semantic rules
+   */
+  private validateSemantics(
+    scenario: {
+      cases: Array<{ id: string; prompt: string | unknown; variables?: Record<string, unknown> }>;
+      variables?: Record<string, unknown>;
+    },
+    content: string
+  ): ValidationIssue[] {
+    const errors: ValidationIssue[] = [];
+    const lines = content.split('\n');
+
+    // Check for duplicate case IDs
+    const caseIds = new Set<string>();
+    for (const testCase of scenario.cases) {
+      if (caseIds.has(testCase.id)) {
+        const line = this.findLineForCaseId(lines, testCase.id);
+        errors.push({
+          line,
+          message: `Duplicate case ID: '${testCase.id}'`,
+          rule: 'duplicate-case-id',
+          severity: 'error',
+        });
+      }
+      caseIds.add(testCase.id);
+    }
+
+    // Check variable references
+    const globalVars = scenario.variables || {};
+    for (const testCase of scenario.cases) {
+      const caseVars = testCase.variables || {};
+      const allVars = { ...globalVars, ...caseVars };
+
+      const prompt =
+        typeof testCase.prompt === 'string' ? testCase.prompt : JSON.stringify(testCase.prompt);
+
+      const refs = this.extractVariableRefs(prompt);
+      for (const ref of refs) {
+        if (!(ref in allVars)) {
+          const line = this.findLineForCaseId(lines, testCase.id);
+          errors.push({
+            line,
+            message: `Undefined variable '{{${ref}}}' in case '${testCase.id}'`,
+            rule: 'undefined-variable',
+            severity: 'error',
+            suggestion: `Define '${ref}' in scenario.variables or case.variables`,
+          });
+        }
+      }
+    }
+
+    return errors;
+  }
+
+  /**
+   * Find line number for a case ID
+   */
+  private findLineForCaseId(lines: string[], caseId: string): number {
+    for (let i = 0; i < lines.length; i++) {
+      if (
+        lines[i].includes(`id: ${caseId}`) ||
+        lines[i].includes(`id: "${caseId}"`) ||
+        lines[i].includes(`id: '${caseId}'`)
+      ) {
+        return i + 1;
+      }
+    }
+    return 1;
+  }
+
+  /**
+   * Extract variable references from a string ({{varName}} format)
+   */
+  private extractVariableRefs(text: string): string[] {
+    const regex = /\{\{(\w+)\}\}/g;
+    const refs: string[] = [];
+    const matches = text.matchAll(regex);
+    for (const match of matches) {
+      refs.push(match[1]);
+    }
+    return refs;
+  }
+
+  /**
+   * Detect warnings (non-blocking issues)
+   */
+  private detectWarnings(parsed: unknown, content: string): ValidationIssue[] {
+    const warnings: ValidationIssue[] = [];
+    const lines = content.split('\n');
+
+    if (parsed && typeof parsed === 'object') {
+      const obj = parsed as Record<string, unknown>;
+
+      // Check for deprecated 'criteria' field (should be 'rubric' for llm_grader)
+      if (this.hasDeepKey(obj, 'criteria')) {
+        const line = this.findLineForKey(lines, 'criteria');
+        warnings.push({
+          line,
+          message: "'criteria' is deprecated, use 'rubric' instead (llm_grader)",
+          rule: 'deprecated-field',
+          severity: 'warning',
+          suggestion: "Replace 'criteria' with 'rubric'",
+        });
+      }
+
+      // Check for very large number of cases without parallel recommendation
+      const cases = obj.cases as unknown[] | undefined;
+      if (Array.isArray(cases) && cases.length > 20) {
+        warnings.push({
+          line: 1,
+          message: `Scenario has ${cases.length} cases. Consider using --parallel for faster execution.`,
+          rule: 'performance-hint',
+          severity: 'warning',
+        });
+      }
+
+      // Check for missing description
+      if (!obj.description) {
+        warnings.push({
+          line: 1,
+          message:
+            "Scenario is missing 'description' field. Adding a description improves documentation.",
+          rule: 'missing-description',
+          severity: 'warning',
+        });
+      }
+    }
+
+    return warnings;
+  }
+
+  /**
+   * Check if object has a key at any depth
+   */
+  private hasDeepKey(obj: unknown, key: string): boolean {
+    if (obj === null || typeof obj !== 'object') return false;
+
+    if (key in (obj as Record<string, unknown>)) return true;
+
+    for (const value of Object.values(obj as Record<string, unknown>)) {
+      if (this.hasDeepKey(value, key)) return true;
+    }
+
+    return false;
+  }
+
+  /**
+   * Find line number for a key
+   */
+  private findLineForKey(lines: string[], key: string): number {
+    for (let i = 0; i < lines.length; i++) {
+      if (lines[i].includes(`${key}:`)) {
+        return i + 1;
+      }
+    }
+    return 1;
+  }
+}