agentic-qe 3.7.13 → 3.7.14

package/scripts/run-skill-eval.ts

@@ -1,1297 +0,0 @@
-#!/usr/bin/env npx tsx
-/**
- * AQE Skill Evaluation Runner
- *
- * Runs skill evaluation test suites defined in YAML format.
- * Integrates with AQE MCP tools for shared learning and QualityFeedbackLoop.
- *
- * Usage:
- *   npx tsx scripts/run-skill-eval.ts --skill security-testing --model claude-3.5-sonnet
- *   npx tsx scripts/run-skill-eval.ts --eval-file .claude/skills/security-testing/evals/security-testing.yaml
- *   npx tsx scripts/run-skill-eval.ts --skill security-testing --all-models
- *
- * MCP Integration (per docs/specs/skill-validation-mcp-integration.md):
- *   - Queries patterns before running: mcp__agentic-qe__memory_query
- *   - Tracks test outcomes: mcp__agentic-qe__test_outcome_track
- *   - Stores patterns after: mcp__agentic-qe__memory_store
- *   - Shares learning: mcp__agentic-qe__memory_share
- *   - Updates quality gate: mcp__agentic-qe__quality_assess
- *
- * @module scripts/run-skill-eval
- * @version 1.0.0
- */
-
-import { readFileSync, writeFileSync, existsSync } from 'fs';
-import { join, dirname, basename } from 'path';
-import { fileURLToPath } from 'url';
-
-// For YAML parsing - using dynamic import for ESM compatibility
-let yaml: typeof import('yaml');
-
-// =============================================================================
-// Types
-// =============================================================================
-
-/**
- * Test case from eval YAML
- */
-interface TestCase {
-  id: string;
-  description: string;
-  category?: string;
-  priority: 'critical' | 'high' | 'medium' | 'low';
-  skip?: boolean;
-  skip_reason?: string;
-  negative_control?: boolean;
-  input: TestInput;
-  expected_output: ExpectedOutput;
-  validation?: ValidationConfig;
-  timeout_ms?: number;
-}
-
-interface TestInput {
-  code?: string;
-  file_path?: string;
-  url?: string;
-  prompt?: string;
-  context?: {
-    language?: string;
-    framework?: string;
-    environment?: 'development' | 'staging' | 'production';
-  };
-  options?: Record<string, unknown>;
-}
-
-interface ExpectedOutput {
-  must_contain?: string[];
-  must_not_contain?: string[];
-  must_match_regex?: string[];
-  severity_classification?: 'critical' | 'high' | 'medium' | 'low' | 'info';
-  finding_count?: { min?: number; max?: number };
-  recommendation_count?: { min?: number; max?: number };
-  schema_path?: string;
-  custom_assertions?: CustomAssertion[];
-}
-
-interface CustomAssertion {
-  type: 'jsonpath' | 'semantic' | 'function';
-  expression: string;
-  expected?: unknown;
-}
-
-interface ValidationConfig {
-  schema_check?: boolean;
-  keyword_match_threshold?: number;
-  reasoning_quality_min?: number;
-  semantic_similarity_min?: number;
-  allow_partial?: boolean;
-  adaptive_rubric?: boolean;
-  grading_rubric?: {
-    completeness?: number;
-    accuracy?: number;
-    actionability?: number;
-  };
-}
-
-interface MCPIntegrationConfig {
-  enabled: boolean;
-  namespace: string;
-  store_patterns: boolean;
-  query_patterns: boolean;
-  track_outcomes: boolean;
-  share_learning: boolean;
-  update_quality_gate: boolean;
-  target_agents: string[];
-}
-
-interface LearningConfig {
-  store_success_patterns: boolean;
-  store_failure_patterns: boolean;
-  pattern_ttl_days: number;
-  min_confidence_to_store: number;
-  cross_model_comparison: boolean;
-}
-
-interface SuccessCriteria {
-  pass_rate: number;
-  critical_pass_rate?: number;
-  avg_reasoning_quality?: number;
-  max_execution_time_ms?: number;
-  cross_model_variance?: number;
-}
-
-/**
- * Full eval suite configuration
- */
-interface EvalSuite {
-  skill: string;
-  version: string;
-  description?: string;
-  models_to_test: string[];
-  mcp_integration?: MCPIntegrationConfig;
-  learning?: LearningConfig;
-  setup?: {
-    required_tools?: string[];
-    environment_variables?: Record<string, string>;
-    fixtures?: Array<{ name: string; path: string; content: string }>;
-  };
-  test_cases: TestCase[];
-  success_criteria: SuccessCriteria;
-  metadata?: {
-    author?: string;
-    created?: string;
-    last_updated?: string;
-    coverage_target?: string;
-  };
-}
-
-/**
- * Result of a single test case execution
- */
-interface TestCaseResult {
-  id: string;
-  description: string;
-  category?: string;
-  priority: string;
-  passed: boolean;
-  skipped: boolean;
-  skip_reason?: string;
-  execution_time_ms: number;
-  keyword_match_score: number;
-  reasoning_quality_score: number;
-  validation_details: {
-    must_contain_matches: string[];
-    must_contain_misses: string[];
-    must_not_contain_violations: string[];
-    regex_matches: string[];
-    regex_misses: string[];
-    severity_matched: boolean;
-    finding_count_matched: boolean;
-    negative_control_passed?: boolean;
-    finding_count_actual?: number;
-    schema_validation_passed?: boolean;
-    rubric_breakdown?: {
-      completeness: number;
-      accuracy: number;
-      actionability: number;
-      weighted_score: number;
-    };
-    adaptive_keywords_extracted?: string[];
-  };
-  raw_output?: string;
-  error?: string;
-}
-
-/**
- * Result of running eval suite against a single model
- */
-interface ModelEvalResult {
-  model: string;
-  skill: string;
-  version: string;
-  timestamp: string;
-  total_tests: number;
-  passed: number;
-  failed: number;
-  skipped: number;
-  pass_rate: number;
-  critical_pass_rate: number;
-  avg_reasoning_quality: number;
-  total_execution_time_ms: number;
-  test_results: TestCaseResult[];
-  success_criteria_met: boolean;
-  criteria_failures: string[];
-}
-
-/**
- * Aggregated result across all models
- */
-interface EvalRunResult {
-  skill: string;
-  version: string;
-  run_id: string;
-  timestamp: string;
-  models_tested: string[];
-  model_results: ModelEvalResult[];
-  cross_model_variance: number;
-  overall_passed: boolean;
-  summary: {
-    best_model: string;
-    worst_model: string;
-    avg_pass_rate: number;
-    recommendations: string[];
-  };
-  mcp_integration_log: MCPIntegrationLog;
-}
-
-/**
- * Log of MCP tool calls made during evaluation
- */
-interface MCPIntegrationLog {
-  patterns_queried: number;
-  outcomes_tracked: number;
-  patterns_stored: number;
-  learning_shared: boolean;
-  quality_gate_updated: boolean;
-  errors: string[];
-}
-
-// =============================================================================
-// MCP Integration Layer
-// =============================================================================
-
-/**
- * Mock MCP client for demonstration
- * In production, this would connect to the actual AQE MCP server
- */
-class MCPClient {
-  private enabled: boolean;
-  private namespace: string;
-  private log: MCPIntegrationLog;
-
-  constructor(config?: MCPIntegrationConfig) {
-    this.enabled = config?.enabled ?? false;
-    this.namespace = config?.namespace ?? 'skill-validation';
-    this.log = {
-      patterns_queried: 0,
-      outcomes_tracked: 0,
-      patterns_stored: 0,
-      learning_shared: false,
-      quality_gate_updated: false,
-      errors: [],
-    };
-  }
-
-  getLog(): MCPIntegrationLog {
-    return { ...this.log };
-  }
-
-  /**
-   * Query existing patterns before running evals
-   * MCP Tool: mcp__agentic-qe__memory_query
-   */
-  async queryPatterns(skill: string): Promise<unknown[]> {
-    if (!this.enabled) return [];
-
-    console.log(`[MCP] Querying patterns for skill: ${skill}`);
-
-    // In production, this would call:
-    // await mcp__agentic-qe__memory_query({
-    //   pattern: `skill-validation-${skill}-*`,
-    //   namespace: this.namespace,
-    //   limit: 10
-    // });
-
-    this.log.patterns_queried++;
-
-    // Return mock patterns for demonstration
-    return [
-      {
-        patternId: `${skill}-baseline-pattern`,
-        successRate: 0.92,
-        lastValidated: new Date().toISOString(),
-      },
-    ];
-  }
-
-  /**
-   * Track individual test outcome
-   * MCP Tool: mcp__agentic-qe__test_outcome_track
-   */
-  async trackOutcome(
-    testId: string,
-    passed: boolean,
-    patternId?: string,
-    coverage?: { lines: number; branches: number; functions: number },
-    executionTime?: number
-  ): Promise<void> {
-    if (!this.enabled) return;
-
-    console.log(`[MCP] Tracking outcome: ${testId} - ${passed ? 'PASSED' : 'FAILED'}`);
-
-    // In production, this would call:
-    // await mcp__agentic-qe__test_outcome_track({
-    //   testId: `skill-${skill}-${testId}`,
-    //   generatedBy: 'eval-runner',
-    //   patternId,
-    //   passed,
-    //   coverage,
-    //   executionTime,
-    //   flaky: false
-    // });
-
-    this.log.outcomes_tracked++;
-  }
-
-  /**
-   * Store successful patterns for future reference
-   * MCP Tool: mcp__agentic-qe__memory_store
-   */
-  async storePattern(
-    skill: string,
-    results: ModelEvalResult,
-    patterns: unknown[]
-  ): Promise<void> {
-    if (!this.enabled) return;
-
-    console.log(`[MCP] Storing patterns for skill: ${skill}`);
-
-    const key = `skill-validation-${skill}-${Date.now()}`;
-    const value = {
-      skillName: skill,
-      trustTier: 3, // Level 3 = has eval suite
-      validationResult: {
-        passRate: results.pass_rate,
-        criticalPassRate: results.critical_pass_rate,
-        avgReasoningQuality: results.avg_reasoning_quality,
-      },
-      model: results.model,
-      passRate: results.pass_rate,
-      patterns,
-      timestamp: new Date().toISOString(),
-    };
-
-    // In production, this would call:
-    // await mcp__agentic-qe__memory_store({
-    //   key,
-    //   value,
-    //   namespace: this.namespace
-    // });
-
-    this.log.patterns_stored++;
-  }
-
-  /**
-   * Share learning with fleet agents
-   * MCP Tool: mcp__agentic-qe__memory_share
-   */
-  async shareLearning(
-    skill: string,
-    targetAgents: string[],
-    insights: unknown
-  ): Promise<void> {
-    if (!this.enabled) return;
-
-    console.log(`[MCP] Sharing learning with agents: ${targetAgents.join(', ')}`);
-
-    // In production, this would call:
-    // await mcp__agentic-qe__memory_share({
-    //   sourceAgentId: 'eval-runner',
-    //   targetAgentIds: targetAgents,
-    //   knowledgeDomain: 'skill-validation',
-    //   data: {
-    //     skillName: skill,
-    //     insights,
-    //     recommendations: []
-    //   }
-    // });
-
-    this.log.learning_shared = true;
-  }
-
-  /**
-   * Update quality gate with validation metrics
-   * MCP Tool: mcp__agentic-qe__quality_assess
-   */
-  async updateQualityGate(skill: string, metrics: unknown): Promise<void> {
-    if (!this.enabled) return;
-
-    console.log(`[MCP] Updating quality gate for skill: ${skill}`);
-
-    // In production, this would call:
-    // await mcp__agentic-qe__quality_assess({
-    //   target: `skill:${skill}`,
-    //   metrics,
-    //   updateQualityScore: true
-    // });
-
-    this.log.quality_gate_updated = true;
-  }
-}
-
-// =============================================================================
-// Evaluation Engine
-// =============================================================================
-
-class SkillEvaluationRunner {
-  private suite: EvalSuite;
-  private mcpClient: MCPClient;
-  private verbose: boolean;
-
-  constructor(suite: EvalSuite, verbose = false) {
-    this.suite = suite;
-    this.verbose = verbose;
-    this.mcpClient = new MCPClient(suite.mcp_integration);
-  }
-
-  /**
-   * Run evaluation against a single model
-   */
-  async runForModel(model: string): Promise<ModelEvalResult> {
-    const startTime = Date.now();
-    const results: TestCaseResult[] = [];
-
-    console.log(`\n${'='.repeat(60)}`);
-    console.log(`Running ${this.suite.skill} evals against ${model}`);
-    console.log(`${'='.repeat(60)}\n`);
-
-    // Query existing patterns (MCP integration)
-    await this.mcpClient.queryPatterns(this.suite.skill);
-
-    // Run each test case
-    for (const testCase of this.suite.test_cases) {
-      const result = await this.runTestCase(testCase, model);
-      results.push(result);
-
-      // Track outcome (MCP integration)
-      await this.mcpClient.trackOutcome(
-        testCase.id,
-        result.passed,
-        undefined,
-        undefined,
-        result.execution_time_ms
-      );
-
-      // Log progress
-      const status = result.skipped ? 'SKIP' : result.passed ? 'PASS' : 'FAIL';
-      const icon = result.skipped ? '-' : result.passed ? '+' : 'x';
-      console.log(`  [${icon}] ${testCase.id}: ${status} (${result.execution_time_ms}ms)`);
-    }
-
-    const totalTime = Date.now() - startTime;
-
-    // Calculate metrics
-    const passed = results.filter((r) => r.passed && !r.skipped).length;
-    const failed = results.filter((r) => !r.passed && !r.skipped).length;
-    const skipped = results.filter((r) => r.skipped).length;
-    const total = results.length - skipped;
-
-    const criticalTests = results.filter(
-      (r) => this.suite.test_cases.find((tc) => tc.id === r.id)?.priority === 'critical'
-    );
-    const criticalPassed = criticalTests.filter((r) => r.passed && !r.skipped).length;
-    const criticalTotal = criticalTests.filter((r) => !r.skipped).length;
-
-    const avgReasoningQuality =
-      results
-        .filter((r) => !r.skipped)
-        .reduce((sum, r) => sum + r.reasoning_quality_score, 0) / total || 0;
-
-    // Check success criteria
-    const criteriaFailures: string[] = [];
-    const passRate = total > 0 ? passed / total : 0;
-    const criticalPassRate = criticalTotal > 0 ? criticalPassed / criticalTotal : 1;
-
-    if (passRate < this.suite.success_criteria.pass_rate) {
-      criteriaFailures.push(
-        `Pass rate ${(passRate * 100).toFixed(1)}% < required ${(this.suite.success_criteria.pass_rate * 100).toFixed(1)}%`
-      );
-    }
-
-    if (
-      this.suite.success_criteria.critical_pass_rate &&
-      criticalPassRate < this.suite.success_criteria.critical_pass_rate
-    ) {
-      criteriaFailures.push(
-        `Critical pass rate ${(criticalPassRate * 100).toFixed(1)}% < required ${(this.suite.success_criteria.critical_pass_rate * 100).toFixed(1)}%`
-      );
-    }
-
-    if (
-      this.suite.success_criteria.avg_reasoning_quality &&
-      avgReasoningQuality < this.suite.success_criteria.avg_reasoning_quality
-    ) {
-      criteriaFailures.push(
-        `Avg reasoning quality ${avgReasoningQuality.toFixed(2)} < required ${this.suite.success_criteria.avg_reasoning_quality}`
-      );
-    }
-
-    if (
-      this.suite.success_criteria.max_execution_time_ms &&
-      totalTime > this.suite.success_criteria.max_execution_time_ms
-    ) {
-      criteriaFailures.push(
-        `Execution time ${totalTime}ms > max ${this.suite.success_criteria.max_execution_time_ms}ms`
-      );
-    }
-
-    const modelResult: ModelEvalResult = {
-      model,
-      skill: this.suite.skill,
-      version: this.suite.version,
-      timestamp: new Date().toISOString(),
-      total_tests: results.length,
-      passed,
-      failed,
-      skipped,
-      pass_rate: passRate,
-      critical_pass_rate: criticalPassRate,
-      avg_reasoning_quality: avgReasoningQuality,
-      total_execution_time_ms: totalTime,
-      test_results: results,
-      success_criteria_met: criteriaFailures.length === 0,
-      criteria_failures: criteriaFailures,
-    };
-
-    // Store patterns (MCP integration)
-    if (this.suite.mcp_integration?.store_patterns) {
-      const learnedPatterns = this.extractPatterns(results);
-      await this.mcpClient.storePattern(this.suite.skill, modelResult, learnedPatterns);
-    }
-
-    return modelResult;
-  }
-
-  /**
-   * Run a single test case
-   */
-  private async runTestCase(testCase: TestCase, model: string): Promise<TestCaseResult> {
-    const startTime = Date.now();
-
-    // Handle skipped tests
-    if (testCase.skip) {
-      return {
-        id: testCase.id,
-        description: testCase.description,
-        category: testCase.category,
-        priority: testCase.priority,
-        passed: false,
-        skipped: true,
-        skip_reason: testCase.skip_reason,
-        execution_time_ms: 0,
-        keyword_match_score: 0,
-        reasoning_quality_score: 0,
-        validation_details: {
-          must_contain_matches: [],
-          must_contain_misses: [],
-          must_not_contain_violations: [],
-          regex_matches: [],
-          regex_misses: [],
-          severity_matched: false,
-          finding_count_matched: false,
-        },
-      };
-    }
-
-    try {
-      // In production, this would invoke the actual skill via Claude API
-      // For now, we simulate the output based on test case expectations
-      const output = await this.simulateSkillExecution(testCase, model);
-      const executionTime = Date.now() - startTime;
-
-      // Validate output against expectations
-      const validation = this.validateOutput(output, testCase.expected_output, testCase.validation, testCase);
-
-      return {
-        id: testCase.id,
-        description: testCase.description,
-        category: testCase.category,
-        priority: testCase.priority,
-        passed: validation.passed,
-        skipped: false,
-        execution_time_ms: executionTime,
-        keyword_match_score: validation.keywordMatchScore,
-        reasoning_quality_score: validation.reasoningQualityScore,
-        validation_details: validation.details,
-        raw_output: this.verbose ? output : undefined,
-      };
-    } catch (error) {
-      return {
-        id: testCase.id,
-        description: testCase.description,
-        category: testCase.category,
-        priority: testCase.priority,
-        passed: false,
-        skipped: false,
-        execution_time_ms: Date.now() - startTime,
-        keyword_match_score: 0,
-        reasoning_quality_score: 0,
-        validation_details: {
-          must_contain_matches: [],
-          must_contain_misses: [],
-          must_not_contain_violations: [],
-          regex_matches: [],
-          regex_misses: [],
-          severity_matched: false,
-          finding_count_matched: false,
-        },
-        error: error instanceof Error ? error.message : String(error),
-      };
-    }
-  }
-
-  /**
-   * Simulate skill execution (placeholder for actual LLM call)
-   * In production, this would:
-   * 1. Load the skill from .claude/skills/{skill}/SKILL.md
-   * 2. Call the LLM API with the skill prompt and test input
-   * 3. Return the actual LLM output
-   */
-  private async simulateSkillExecution(testCase: TestCase, model: string): Promise<string> {
-    // Simulate some processing time
-    await new Promise((resolve) => setTimeout(resolve, 100));
-
-    // For demonstration, generate mock output based on test case category
-    const category = testCase.category || 'general';
-
-    // This is a simulation - in production this would be the actual LLM response
-    if (category === 'negative') {
-      return JSON.stringify({
-        findings: [],
-        summary: 'The code appears to follow security best practices. No critical vulnerabilities detected.',
-        recommendations: ['Continue following current security patterns'],
-        severity: 'info',
-      });
-    }
-
-    // For injection/auth/crypto tests, return findings
-    return JSON.stringify({
-      findings: [
-        {
-          type: testCase.expected_output.must_contain?.[0]?.toLowerCase() || 'vulnerability',
-          severity: testCase.expected_output.severity_classification || 'high',
-          description: `Detected potential ${testCase.expected_output.must_contain?.[0] || 'security issue'}`,
-          cwe: 'CWE-89',
-          owasp: 'A03:2021',
-          remediation: 'Use parameterized queries and proper input validation',
-        },
-      ],
-      summary: `Security analysis complete. Found ${testCase.expected_output.finding_count?.min || 1} issue(s).`,
-      recommendations: ['Implement secure coding practices', 'Use prepared statements'],
-    });
-  }
-
-  /**
-   * Validate LLM output against expected output criteria.
-   *
-   * Implements:
-   * - P1-6: Negative control (inverted pass logic for "should decline" tests)
-   * - P1-5a: Finding count enforcement + schema_path validation
-   * - P1-5b: Grading rubric weighted scoring (completeness/accuracy/actionability)
-   * - P1-4: Adaptive rubric (dynamic keyword extraction from prompt)
-   */
-  private validateOutput(
-    output: string,
-    expected: ExpectedOutput,
-    config?: ValidationConfig,
-    testCase?: TestCase
-  ): {
-    passed: boolean;
-    keywordMatchScore: number;
-    reasoningQualityScore: number;
-    details: TestCaseResult['validation_details'];
-  } {
-    const outputLower = output.toLowerCase();
-    const threshold = config?.keyword_match_threshold ?? 0.8;
-    const isNegativeControl = testCase?.negative_control ?? false;
-
-    // --- P1-4: Adaptive Rubric — extract additional keywords from prompt ---
-    const adaptiveKeywords: string[] = [];
-    let effectiveMustContain = expected.must_contain || [];
-
-    if (config?.adaptive_rubric && testCase?.input?.prompt) {
-      const prompt = testCase.input.prompt;
-      // Extract quoted strings
-      const quoted = prompt.match(/"([^"]+)"/g)?.map((s) => s.replace(/"/g, '')) || [];
-      // Extract format keywords
-      const formatWords = prompt.match(/\b(JSON|YAML|markdown|XML|CSV|HTML|SQL)\b/gi) || [];
-      // Extract named standards
-      const standards =
-        prompt.match(/\b(OWASP|WCAG|PCI[-\s]DSS|HIPAA|SOC2|GDPR|ISO\s?\d+)\b/gi) || [];
-      // Extract meaningful numbers (>1 digit or >5)
-      const numbers = (prompt.match(/\b\d+(\.\d+)?\b/g) || []).filter(
-        (n) => n.length > 1 || parseInt(n) > 5
-      );
-
-      adaptiveKeywords.push(...quoted, ...formatWords, ...standards, ...numbers);
-
-      // Merge with existing must_contain (deduplicate)
-      const existingLower = new Set(effectiveMustContain.map((k) => k.toLowerCase()));
-      const newKeywords = adaptiveKeywords.filter((k) => !existingLower.has(k.toLowerCase()));
-      effectiveMustContain = [...effectiveMustContain, ...newKeywords];
-    }
-
-    // Check must_contain
-    const mustContainMatches: string[] = [];
-    const mustContainMisses: string[] = [];
-
-    for (const keyword of effectiveMustContain) {
-      if (outputLower.includes(keyword.toLowerCase())) {
-        mustContainMatches.push(keyword);
-      } else {
-        mustContainMisses.push(keyword);
-      }
-    }
-
-    // Check must_not_contain
-    const violations: string[] = [];
-    for (const keyword of expected.must_not_contain || []) {
-      if (outputLower.includes(keyword.toLowerCase())) {
-        violations.push(keyword);
-      }
-    }
-
-    // Check regex patterns
-    const regexMatches: string[] = [];
-    const regexMisses: string[] = [];
-    for (const pattern of expected.must_match_regex || []) {
-      try {
-        const regex = new RegExp(pattern, 'i');
-        if (regex.test(output)) {
-          regexMatches.push(pattern);
-        } else {
-          regexMisses.push(pattern);
-        }
-      } catch {
-        regexMisses.push(pattern);
-      }
-    }
-
-    // Calculate keyword match score
-    const totalKeywords = effectiveMustContain.length + (expected.must_match_regex?.length || 0);
-    const matchedKeywords = mustContainMatches.length + regexMatches.length;
-    const keywordMatchScore = totalKeywords > 0 ? matchedKeywords / totalKeywords : 1;
-
-    // Check severity classification
-    const severityMatched =
-      !expected.severity_classification ||
-      outputLower.includes(expected.severity_classification.toLowerCase());
-
-    // --- P1-5a: Finding count enforcement ---
-    let findingCountMatched = true;
-    let findingCountActual: number | undefined;
-
-    if (expected.finding_count) {
-      // Try JSON parsing first
-      try {
-        const parsed = JSON.parse(output);
-        if (Array.isArray(parsed?.findings)) {
-          findingCountActual = parsed.findings.length;
-        }
-      } catch {
-        // Fallback: count severity-keyword occurrences or numbered findings
-        const severityPattern =
-          /\b(critical|high|medium|low|info)\b.*?(vulnerability|finding|issue)/gi;
-        const numberedPattern = /^\s*\d+[.)]\s+/gm;
-        const severityCount = (output.match(severityPattern) || []).length;
-        const numberedCount = (output.match(numberedPattern) || []).length;
-        findingCountActual = Math.max(severityCount, numberedCount);
-      }
-
-      if (findingCountActual !== undefined) {
-        if (
-          expected.finding_count.min !== undefined &&
-          findingCountActual < expected.finding_count.min
-        ) {
-          findingCountMatched = false;
-        }
-        if (
-          expected.finding_count.max !== undefined &&
-          findingCountActual > expected.finding_count.max
-        ) {
-          findingCountMatched = false;
-        }
-      }
-    }
-
-    // --- P1-5a: Schema validation ---
-    let schemaValidationPassed: boolean | undefined;
-    if (expected.schema_path) {
-      try {
-        const schemaContent = readFileSync(expected.schema_path, 'utf-8');
-        const schema = JSON.parse(schemaContent);
-        const parsed = JSON.parse(output);
-        // Structural check: verify all required fields exist
-        if (schema.required && Array.isArray(schema.required)) {
-          const missingFields = schema.required.filter(
-            (field: string) => !(field in parsed)
-          );
-          schemaValidationPassed = missingFields.length === 0;
-        } else {
-          schemaValidationPassed = true;
-        }
-      } catch {
-        schemaValidationPassed = false;
-      }
-    }
-
-    // --- P1-5b: Grading Rubric scoring ---
-    let rubricBreakdown:
-      | {
-          completeness: number;
-          accuracy: number;
-          actionability: number;
-          weighted_score: number;
-        }
-      | undefined;
-
-    if (config?.grading_rubric) {
-      const weights = config.grading_rubric;
-      const totalWeight =
-        (weights.completeness ?? 0) + (weights.accuracy ?? 0) + (weights.actionability ?? 0);
-
-      // Completeness = fraction of must_contain matched
-      const completenessScore = totalKeywords > 0 ? matchedKeywords / totalKeywords : 1;
-
-      // Accuracy = 1 - (violations / total_must_not_contain)
-      const totalMustNotContain = expected.must_not_contain?.length || 0;
-      const accuracyScore =
-        totalMustNotContain > 0 ? 1 - violations.length / totalMustNotContain : 1;
-
-      // Actionability = heuristic: code blocks, numbered steps, specific recommendations
-      let actionabilityScore = 0;
-      const hasCodeBlocks = /```[\s\S]*?```/.test(output) || /`[^`]+`/.test(output);
-      const hasNumberedSteps = /^\s*\d+[.)]\s+/m.test(output);
-      const hasRecommendations =
-        /\b(recommend|suggest|should|must|consider|implement|use|apply|ensure)\b/i.test(output);
-      const hasSpecificActions =
-        /\b(install|configure|update|replace|add|remove|change|set|enable|disable)\b/i.test(
-          output
-        );
-
-      if (hasCodeBlocks) actionabilityScore += 0.3;
-      if (hasNumberedSteps) actionabilityScore += 0.3;
-      if (hasRecommendations) actionabilityScore += 0.2;
-      if (hasSpecificActions) actionabilityScore += 0.2;
-      actionabilityScore = Math.min(1, actionabilityScore);
-
-      // Weighted sum
-      const weightedScore =
-        totalWeight > 0
-          ? ((weights.completeness ?? 0) * completenessScore +
-              (weights.accuracy ?? 0) * accuracyScore +
-              (weights.actionability ?? 0) * actionabilityScore) /
-            totalWeight
-          : completenessScore;
-
-      rubricBreakdown = {
-        completeness: completenessScore,
-        accuracy: accuracyScore,
-        actionability: actionabilityScore,
-        weighted_score: weightedScore,
-      };
-    }
-
-    // Calculate reasoning quality — use rubric if available, else simple heuristic
-    const reasoningQualityScore = rubricBreakdown
-      ? rubricBreakdown.weighted_score
-      : keywordMatchScore * 0.8 + (violations.length === 0 ? 0.2 : 0);
-
-    // --- P1-6: Negative Control logic ---
-    if (isNegativeControl) {
-      // For negative control: pass when must_contain items are ABSENT and no violations
-      const negativeControlPassed =
-        mustContainMatches.length === 0 && violations.length === 0;
-
-      return {
-        passed: negativeControlPassed,
-        keywordMatchScore,
-        reasoningQualityScore: negativeControlPassed ? 1.0 : 0.0,
-        details: {
-          must_contain_matches: mustContainMatches,
-          must_contain_misses: mustContainMisses,
-          must_not_contain_violations: violations,
-          regex_matches: regexMatches,
-          regex_misses: regexMisses,
-          severity_matched: severityMatched,
-          finding_count_matched: findingCountMatched,
-          negative_control_passed: negativeControlPassed,
-          finding_count_actual: findingCountActual,
-          schema_validation_passed: schemaValidationPassed,
-          rubric_breakdown: rubricBreakdown,
-          adaptive_keywords_extracted:
-            adaptiveKeywords.length > 0 ? adaptiveKeywords : undefined,
-        },
-      };
-    }
-
-    // --- Standard pass/fail determination ---
-    const rubricGate = rubricBreakdown
-      ? rubricBreakdown.weighted_score >= threshold
-      : true;
-
-    const passed =
-      keywordMatchScore >= threshold &&
-      violations.length === 0 &&
-      findingCountMatched &&
-      rubricGate &&
-      (schemaValidationPassed === undefined || schemaValidationPassed) &&
-      (config?.reasoning_quality_min === undefined ||
-        reasoningQualityScore >= config.reasoning_quality_min) &&
-      (config?.allow_partial || mustContainMisses.length === 0);
-
-    return {
-      passed,
-      keywordMatchScore,
-      reasoningQualityScore,
-      details: {
-        must_contain_matches: mustContainMatches,
-        must_contain_misses: mustContainMisses,
-        must_not_contain_violations: violations,
-        regex_matches: regexMatches,
-        regex_misses: regexMisses,
-        severity_matched: severityMatched,
-        finding_count_matched: findingCountMatched,
-        finding_count_actual: findingCountActual,
-        schema_validation_passed: schemaValidationPassed,
-        rubric_breakdown: rubricBreakdown,
-        adaptive_keywords_extracted:
-          adaptiveKeywords.length > 0 ? adaptiveKeywords : undefined,
-      },
-    };
-  }
-
-  /**
-   * Extract patterns from results for learning
-   */
-  private extractPatterns(results: TestCaseResult[]): unknown[] {
-    return results
-      .filter((r) => r.passed && !r.skipped)
-      .map((r) => ({
-        testId: r.id,
-        category: r.category,
-        keywordMatchScore: r.keyword_match_score,
-        reasoningQualityScore: r.reasoning_quality_score,
-      }));
-  }
-
-  /**
-   * Run full evaluation across all configured models
-   */
-  async runFull(): Promise<EvalRunResult> {
-    const runId = `eval-${this.suite.skill}-${Date.now()}`;
-    const modelResults: ModelEvalResult[] = [];
-
-    console.log(`\nStarting evaluation run: ${runId}`);
-    console.log(`Skill: ${this.suite.skill} v${this.suite.version}`);
-    console.log(`Models: ${this.suite.models_to_test.join(', ')}`);
-    console.log(`Test cases: ${this.suite.test_cases.length}`);
-
-    // Run for each model
-    for (const model of this.suite.models_to_test) {
-      const result = await this.runForModel(model);
-      modelResults.push(result);
-    }
-
-    // Calculate cross-model variance
-    const passRates = modelResults.map((r) => r.pass_rate);
-    const maxRate = Math.max(...passRates);
-    const minRate = Math.min(...passRates);
-    const variance = maxRate - minRate;
-
-    // Share learning with fleet (MCP integration)
-    if (this.suite.mcp_integration?.share_learning) {
-      await this.mcpClient.shareLearning(
-        this.suite.skill,
-        this.suite.mcp_integration.target_agents || [],
-        {
-          passRates,
-          variance,
-          bestModel: modelResults.find((r) => r.pass_rate === maxRate)?.model,
-        }
-      );
-    }
-
-    // Update quality gate (MCP integration)
-    if (this.suite.mcp_integration?.update_quality_gate) {
-      const avgPassRate = passRates.reduce((a, b) => a + b, 0) / passRates.length;
-      await this.mcpClient.updateQualityGate(this.suite.skill, {
-        passRate: avgPassRate,
-        schemaCompliance: true,
-        validatorPassed: true,
-        evalSuiteScore: avgPassRate * 100,
-      });
-    }
-
-    // Determine overall success
-    const overallPassed =
-      modelResults.every((r) => r.success_criteria_met) &&
-      (!this.suite.success_criteria.cross_model_variance ||
-        variance <= this.suite.success_criteria.cross_model_variance);
-
-    // Generate recommendations
-    const recommendations: string[] = [];
-    if (variance > 0.1) {
-      recommendations.push(
-        `High cross-model variance (${(variance * 100).toFixed(1)}%). Consider model-specific tuning.`
-      );
-    }
-    const failingModels = modelResults.filter((r) => !r.success_criteria_met);
-    if (failingModels.length > 0) {
-      recommendations.push(
-        `Models failing criteria: ${failingModels.map((r) => r.model).join(', ')}`
-      );
-    }
-
-    return {
-      skill: this.suite.skill,
-      version: this.suite.version,
-      run_id: runId,
-      timestamp: new Date().toISOString(),
-      models_tested: this.suite.models_to_test,
-      model_results: modelResults,
-      cross_model_variance: variance,
-      overall_passed: overallPassed,
-      summary: {
-        best_model: modelResults.find((r) => r.pass_rate === maxRate)?.model || '',
-        worst_model: modelResults.find((r) => r.pass_rate === minRate)?.model || '',
-        avg_pass_rate: passRates.reduce((a, b) => a + b, 0) / passRates.length,
-        recommendations,
-      },
-      mcp_integration_log: this.mcpClient.getLog(),
-    };
-  }
-}
-
-// =============================================================================
-// CLI Interface
-// =============================================================================
-
-interface CLIOptions {
-  skill?: string;
-  evalFile?: string;
-  model?: string;
-  allModels?: boolean;
-  output?: string;
-  verbose?: boolean;
-  dryRun?: boolean;
-  useMcp?: boolean;
-}
-
-function parseArgs(args: string[]): CLIOptions {
-  const options: CLIOptions = {};
-
-  for (let i = 0; i < args.length; i++) {
-    const arg = args[i];
-
-    switch (arg) {
-      case '--skill':
-      case '-s':
-        options.skill = args[++i];
-        break;
-      case '--eval-file':
-      case '-e':
-        options.evalFile = args[++i];
-        break;
-      case '--model':
-      case '-m':
-        options.model = args[++i];
-        break;
-      case '--all-models':
-      case '-a':
-        options.allModels = true;
-        break;
-      case '--output':
-      case '-o':
-        options.output = args[++i];
-        break;
-      case '--verbose':
-      case '-v':
-        options.verbose = true;
-        break;
-      case '--dry-run':
-        options.dryRun = true;
-        break;
-      case '--use-mcp':
-        options.useMcp = true;
-        break;
-      case '--help':
-      case '-h':
-        printUsage();
-        process.exit(0);
-    }
-  }
-
-  return options;
-}
-
-function printUsage(): void {
-  console.log(`
-AQE Skill Evaluation Runner
-
-Usage:
-  npx tsx scripts/run-skill-eval.ts [options]
-
-Options:
-  -s, --skill <name>      Skill name (looks for .claude/skills/<name>/evals/<name>.yaml)
-  -e, --eval-file <path>  Path to eval YAML file (alternative to --skill)
-  -m, --model <model>     Run against specific model only
-  -a, --all-models        Run against all configured models (default)
-  -o, --output <path>     Output JSON results to file
-  -v, --verbose           Include raw LLM output in results
-  --dry-run               Parse and validate eval file without running
-  --use-mcp               Enable MCP integration for shared learning
-  -h, --help              Show this help message
-
-Examples:
-  # Run security-testing evals against default model
-  npx tsx scripts/run-skill-eval.ts --skill security-testing
-
-  # Run against specific model
-  npx tsx scripts/run-skill-eval.ts --skill security-testing --model claude-3-haiku
-
-  # Run with MCP integration for learning
-  npx tsx scripts/run-skill-eval.ts --skill security-testing --use-mcp
-
-  # Output results to file
-  npx tsx scripts/run-skill-eval.ts --skill security-testing --output results.json
-
-MCP Integration:
-  When --use-mcp is enabled, the runner will:
-  1. Query existing patterns before running (mcp__agentic-qe__memory_query)
-  2. Track each test outcome (mcp__agentic-qe__test_outcome_track)
-  3. Store successful patterns after (mcp__agentic-qe__memory_store)
-  4. Share learning with fleet (mcp__agentic-qe__memory_share)
-  5. Update quality gate metrics (mcp__agentic-qe__quality_assess)
-`);
-}
-
-async function loadEvalSuite(options: CLIOptions): Promise<EvalSuite> {
-  let evalPath: string;
-
-  if (options.evalFile) {
-    evalPath = options.evalFile;
-  } else if (options.skill) {
-    // Look for eval file in standard location
-    const possiblePaths = [
-      `.claude/skills/${options.skill}/evals/${options.skill}.yaml`,
-      `.claude/skills/${options.skill}/evals/${options.skill}.yml`,
-      `docs/templates/${options.skill}-eval.template.yaml`,
-    ];
-
-    const foundPath = possiblePaths.find((p) => existsSync(p));
-    if (!foundPath) {
-      throw new Error(
-        `Eval file not found for skill '${options.skill}'. Searched:\n  ${possiblePaths.join('\n  ')}`
-      );
-    }
-    evalPath = foundPath;
-  } else {
-    throw new Error('Either --skill or --eval-file must be specified');
-  }
-
-  console.log(`Loading eval suite from: ${evalPath}`);
-
-  const content = readFileSync(evalPath, 'utf-8');
-
-  // Dynamic import of yaml parser
-  if (!yaml) {
-    yaml = await import('yaml');
-  }
-
-  const suite = yaml.parse(content) as EvalSuite;
-
-  // Apply CLI overrides
-  if (options.model && !options.allModels) {
-    suite.models_to_test = [options.model];
-  }
-
-  if (options.useMcp) {
-    suite.mcp_integration = {
-      enabled: true,
-      namespace: 'skill-validation',
-      store_patterns: true,
-      query_patterns: true,
-      track_outcomes: true,
-      share_learning: true,
-      update_quality_gate: true,
-      target_agents: ['qe-learning-coordinator', 'qe-queen-coordinator'],
-      ...suite.mcp_integration,
-    };
-  }
-
-  return suite;
-}
-
-function printResults(result: EvalRunResult): void {
-  console.log(`\n${'='.repeat(60)}`);
-  console.log('EVALUATION RESULTS');
-  console.log(`${'='.repeat(60)}`);
-
-  console.log(`\nSkill: ${result.skill} v${result.version}`);
-  console.log(`Run ID: ${result.run_id}`);
-  console.log(`Timestamp: ${result.timestamp}`);
-  console.log(`Overall: ${result.overall_passed ? 'PASSED' : 'FAILED'}`);
-
-  console.log(`\nCross-Model Summary:`);
-  console.log(`  Models tested: ${result.models_tested.join(', ')}`);
-  console.log(`  Best model: ${result.summary.best_model}`);
-  console.log(`  Worst model: ${result.summary.worst_model}`);
-  console.log(`  Avg pass rate: ${(result.summary.avg_pass_rate * 100).toFixed(1)}%`);
-  console.log(`  Cross-model variance: ${(result.cross_model_variance * 100).toFixed(1)}%`);
-
-  for (const modelResult of result.model_results) {
-    console.log(`\n--- ${modelResult.model} ---`);
-    console.log(`  Pass rate: ${(modelResult.pass_rate * 100).toFixed(1)}%`);
-    console.log(`  Critical pass rate: ${(modelResult.critical_pass_rate * 100).toFixed(1)}%`);
-    console.log(`  Avg reasoning quality: ${modelResult.avg_reasoning_quality.toFixed(2)}`);
-    console.log(`  Tests: ${modelResult.passed}/${modelResult.total_tests} passed, ${modelResult.skipped} skipped`);
-    console.log(`  Execution time: ${modelResult.total_execution_time_ms}ms`);
-    console.log(`  Criteria met: ${modelResult.success_criteria_met ? 'YES' : 'NO'}`);
-
-    if (modelResult.criteria_failures.length > 0) {
-      console.log(`  Failures:`);
-      for (const failure of modelResult.criteria_failures) {
-        console.log(`    - ${failure}`);
-      }
-    }
-  }
-
-  if (result.summary.recommendations.length > 0) {
-    console.log(`\nRecommendations:`);
-    for (const rec of result.summary.recommendations) {
-      console.log(`  - ${rec}`);
-    }
-  }
-
-  console.log(`\nMCP Integration Log:`);
-  console.log(`  Patterns queried: ${result.mcp_integration_log.patterns_queried}`);
-  console.log(`  Outcomes tracked: ${result.mcp_integration_log.outcomes_tracked}`);
-  console.log(`  Patterns stored: ${result.mcp_integration_log.patterns_stored}`);
-  console.log(`  Learning shared: ${result.mcp_integration_log.learning_shared}`);
-  console.log(`  Quality gate updated: ${result.mcp_integration_log.quality_gate_updated}`);
-}
-
-async function main(): Promise<void> {
-  const args = process.argv.slice(2);
-
-  if (args.length === 0) {
-    printUsage();
-    process.exit(1);
-  }
-
-  const options = parseArgs(args);
-
-  try {
-    const suite = await loadEvalSuite(options);
-
-    console.log(`\nLoaded eval suite: ${suite.skill} v${suite.version}`);
-    console.log(`Test cases: ${suite.test_cases.length}`);
-    console.log(`Models: ${suite.models_to_test.join(', ')}`);
-
-    if (options.dryRun) {
-      console.log('\n[Dry run] Eval file parsed successfully. Exiting.');
-      process.exit(0);
-    }
-
-    const runner = new SkillEvaluationRunner(suite, options.verbose);
-    const result = await runner.runFull();
-
-    printResults(result);
-
-    // Output to file if specified
-    if (options.output) {
-      writeFileSync(options.output, JSON.stringify(result, null, 2));
-      console.log(`\nResults written to: ${options.output}`);
-    }
-
-    // Exit with appropriate code
-    process.exit(result.overall_passed ? 0 : 1);
-  } catch (error) {
-    console.error('\nError:', error instanceof Error ? error.message : String(error));
-    process.exit(1);
-  }
-}
-
-// Run if executed directly
-main().catch(console.error);