agentic-qe 3.4.1 → 3.4.2

package/v3/dist/validation/parallel-eval-runner.d.ts

@@ -0,0 +1,307 @@
+/**
+ * Parallel Evaluation Runner for Skill Validation
+ * ADR-056 Phase 5: Distributed test execution across worker pool
+ *
+ * This module provides parallel execution of skill evaluation test cases
+ * using a worker pool pattern for faster skill validation.
+ *
+ * Features:
+ * - Worker pool with configurable concurrency
+ * - Batch-based test case distribution
+ * - Automatic retry of failed tests
+ * - Progress tracking and heartbeat monitoring
+ * - Integration with SkillValidationLearner for outcome recording
+ * - Parallel speedup measurement vs sequential execution
+ *
+ * @module validation/parallel-eval-runner
+ * @see .claude/skills/.validation/schemas/skill-eval.schema.json
+ */
+import { SkillValidationLearner, TestCaseResult } from '../learning/skill-validation-learner.js';
+/**
+ * Configuration for the parallel evaluation runner
+ */
+export interface ParallelEvalConfig {
+    /** Maximum number of concurrent workers (default: 5) */
+    maxWorkers: number;
+    /** Number of test cases per worker batch */
+    batchSize: number;
+    /** Whether to retry failed tests once */
+    retryFailedTests: boolean;
+    /** Timeout per test case in milliseconds */
+    timeout: number;
+    /** Directory containing skill eval suites */
+    skillsDir: string;
+    /** Progress reporting interval in milliseconds */
+    progressIntervalMs: number;
+}
+/**
+ * Default configuration values
+ */
+export declare const DEFAULT_PARALLEL_EVAL_CONFIG: ParallelEvalConfig;
+/**
+ * Test case input structure from eval YAML
+ */
+export interface EvalTestCaseInput {
+    code: string;
+    context: {
+        language?: string;
+        wcagLevel?: string;
+        description?: string;
+        environment?: string;
+        options?: Record<string, unknown>;
+    };
+}
+/**
+ * Expected output structure from eval YAML
+ */
+export interface EvalTestCaseExpectedOutput {
+    must_contain?: string[];
+    must_not_contain?: string[];
+    must_match_regex?: string[];
+    finding_count?: {
+        min?: number;
+        max?: number;
+    };
+    severity_classification?: string;
+}
+/**
+ * Validation criteria from eval YAML
+ */
+export interface EvalTestCaseValidation {
+    schema_check?: boolean;
+    keyword_match_threshold?: number;
+    reasoning_quality_min?: number;
+    grading_rubric?: {
+        completeness?: number;
+        accuracy?: number;
+        actionability?: number;
+    };
+}
+/**
+ * Single test case from eval suite YAML
+ */
+export interface EvalTestCase {
+    id: string;
+    description: string;
+    category: string;
+    priority: 'critical' | 'high' | 'medium' | 'low';
+    input: EvalTestCaseInput;
+    expected_output: EvalTestCaseExpectedOutput;
+    validation?: EvalTestCaseValidation;
+}
+/**
+ * Complete eval suite loaded from YAML
+ */
+export interface EvalSuite {
+    skill: string;
+    version: string;
+    description: string;
+    models_to_test: string[];
+    mcp_integration?: {
+        enabled: boolean;
+        namespace?: string;
+        query_patterns?: boolean;
+        track_outcomes?: boolean;
+        store_patterns?: boolean;
+        share_learning?: boolean;
+        update_quality_gate?: boolean;
+        target_agents?: string[];
+    };
+    learning?: {
+        store_success_patterns?: boolean;
+        store_failure_patterns?: boolean;
+        pattern_ttl_days?: number;
+        min_confidence_to_store?: number;
+        cross_model_comparison?: boolean;
+    };
+    setup?: {
+        required_tools?: string[];
+        environment_variables?: Record<string, string>;
+        fixtures?: Array<{
+            name: string;
+            content: string;
+        }>;
+    };
+    test_cases: EvalTestCase[];
+    success_criteria: {
+        pass_rate: number;
+        critical_pass_rate: number;
+        avg_reasoning_quality?: number;
+        max_execution_time_ms?: number;
+        cross_model_variance?: number;
+    };
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Task unit sent to a worker
+ */
+export interface TestCaseTask {
+    /** Name of the skill being validated */
+    skillName: string;
+    /** Unique test case identifier */
+    testCaseId: string;
+    /** Full test case definition */
+    testCase: EvalTestCase;
+    /** Model to use for this test */
+    model: string;
+    /** Worker batch ID */
+    batchId: number;
+    /** Index within the batch */
+    indexInBatch: number;
+}
+/**
+ * Result of parallel evaluation for a skill
+ */
+export interface ParallelEvalResult {
+    /** Name of the skill evaluated */
+    skill: string;
+    /** Model used for evaluation */
+    model: string;
+    /** Total number of test cases */
+    totalTests: number;
+    /** Number of tests that passed */
+    passedTests: number;
+    /** Number of tests that failed */
+    failedTests: number;
+    /** Number of tests that were skipped */
+    skippedTests: number;
+    /** Pass rate as a decimal (0-1) */
+    passRate: number;
+    /** Individual test case results */
+    testResults: TestCaseResult[];
+    /** Total duration in milliseconds */
+    totalDurationMs: number;
+    /** Speedup factor vs sequential (e.g., 3.5x faster) */
+    parallelSpeedup: number;
+    /** Average reasoning quality score */
+    avgReasoningQuality: number;
+    /** Whether the eval suite passed overall */
+    passed: boolean;
+    /** Workers used in this run */
+    workersUsed: number;
+    /** Timestamp of evaluation */
+    timestamp: Date;
+}
+/**
+ * Worker communication message types
+ */
+export interface WorkerMessage {
+    type: 'task' | 'result' | 'error' | 'heartbeat' | 'progress';
+    workerId: number;
+    payload: unknown;
+}
+/**
+ * Progress report from a worker
+ */
+export interface WorkerProgress {
+    workerId: number;
+    tasksCompleted: number;
+    tasksTotal: number;
+    currentTask?: string;
+    elapsedMs: number;
+}
+/**
+ * Aggregated progress across all workers
+ */
+export interface EvalProgress {
+    skill: string;
+    model: string;
+    totalTasks: number;
+    completedTasks: number;
+    failedTasks: number;
+    activeWorkers: number;
+    elapsedMs: number;
+    estimatedRemainingMs: number;
+    workerProgress: WorkerProgress[];
+}
+/**
+ * LLM executor interface for test case execution
+ */
+export interface LLMExecutor {
+    execute(prompt: string, model: string, options?: {
+        timeout?: number;
+    }): Promise<{
+        output: string;
+        tokensUsed: number;
+        durationMs: number;
+    }>;
+}
+/**
+ * Mock LLM executor for testing
+ */
+export declare class MockLLMExecutor implements LLMExecutor {
+    execute(prompt: string, model: string, options?: {
+        timeout?: number;
+    }): Promise<{
+        output: string;
+        tokensUsed: number;
+        durationMs: number;
+    }>;
+    private generateMockResponse;
+}
+/**
+ * Parallel evaluation runner using worker pool pattern
+ * Distributes eval test cases across agents for faster execution
+ */
+export declare class ParallelEvalRunner {
+    private readonly config;
+    private readonly skillValidationLearner;
+    private readonly executor;
+    private readonly workers;
+    private progressCallback?;
+    constructor(config: ParallelEvalConfig | undefined, skillValidationLearner: SkillValidationLearner, executor?: LLMExecutor);
+    /**
+     * Set progress callback for reporting
+     */
+    onProgress(callback: (progress: EvalProgress) => void): void;
+    /**
+     * Run eval suite for a skill in parallel
+     */
+    runEvalParallel(skill: string, model: string): Promise<ParallelEvalResult>;
+    /**
+     * Run multiple skill evals in parallel
+     */
+    runMultipleEvalsParallel(skills: string[], models: string[]): Promise<Map<string, ParallelEvalResult[]>>;
+    /**
+     * Load eval suite from skill's evals/ directory
+     */
+    loadEvalSuite(skill: string): EvalSuite | null;
+    /**
+     * Create test case tasks from suite
+     */
+    private createTasks;
+    /**
+     * Partition test cases into batches for workers
+     */
+    private partitionTestCases;
+    /**
+     * Determine trust tier based on suite configuration
+     */
+    private determineTrustTier;
+    /**
+     * Start progress reporting interval
+     */
+    private startProgressReporting;
+    /**
+     * Record validation outcome to learner
+     */
+    private recordOutcome;
+    /**
+     * Get worker pool status
+     */
+    getWorkerStatus(): Array<{
+        id: number;
+        status: string;
+        tasksCompleted: number;
+        tasksFailed: number;
+    }>;
+    /**
+     * Reset worker statistics
+     */
+    resetWorkers(): void;
+}
+/**
+ * Create a ParallelEvalRunner instance
+ */
+export declare function createParallelEvalRunner(skillValidationLearner: SkillValidationLearner, config?: Partial<ParallelEvalConfig>, executor?: LLMExecutor): ParallelEvalRunner;
+//# sourceMappingURL=parallel-eval-runner.d.ts.map

package/v3/dist/validation/parallel-eval-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parallel-eval-runner.d.ts","sourceRoot":"","sources":["../../src/validation/parallel-eval-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAKH,OAAO,EACL,sBAAsB,EACtB,cAAc,EAIf,MAAM,yCAAyC,CAAC;AAMjD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,wDAAwD;IACxD,UAAU,EAAE,MAAM,CAAC;IAEnB,4CAA4C;IAC5C,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,gBAAgB,EAAE,OAAO,CAAC;IAE1B,4CAA4C;IAC5C,OAAO,EAAE,MAAM,CAAC;IAEhB,6CAA6C;IAC7C,SAAS,EAAE,MAAM,CAAC;IAElB,kDAAkD;IAClD,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED;;GAEG;AACH,eAAO,MAAM,4BAA4B,EAAE,kBAO1C,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE;QACP,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,aAAa,CAAC,EAAE;QACd,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,GAAG,CAAC,EAAE,MAAM,CAAC;KACd,CAAC;IACF,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,cAAc,CAAC,EAAE;QACf,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,UAAU,GAAG,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACjD,KAAK,EAAE,iBAAiB,CAAC;IACzB,eAAe,EAAE,0BAA0B,CAAC;IAC5C,UAAU,CAAC,EAAE,sBAAsB,CAAC;CACrC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,mBAAmB,CAAC,EAAE,OAAO,CAAC;QAC9B,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;KAC1B,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,sBAAsB,CAAC,EAAE,OAAO,CAAC;QACjC,sBAAsB,CAAC,EAAE,OAAO,CAAC;QACjC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,uBAAuB,CAAC,EAAE,MAAM,CAAC;QACjC,sBAAsB,CAAC,EAAE,OAAO,CAAC;KAClC,CAAC;IACF,KAAK,CAAC,EAAE;QACN,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;QAC1B,qBAAqB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC/C,QAAQ,CAAC,EAAE,KAAK,CAAC;YACf,IAAI,EAAE,MAAM,CAAC;YACb,OAAO,EAAE,MAAM,CAAC;SACjB,CAAC,CAAC;KACJ,CAAC;IACF,UAAU,EAAE,YAAY,EAAE,CAAC;IAC3B,gBAAgB,EAAE;QAChB,SAAS,EAAE,MAAM,CAAC;QAClB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAC;QAC/B,qBAAqB,CAAC,EAAE,MAAM,CAAC;QAC/B,oBAAoB,CAAC,EAAE,MAAM,CAAC;KAC/B,CAAC;IACF,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAC;IAElB,kCAAkC;IAClC,UAAU,EAAE,MAAM,CAAC;IAEnB,gCAAgC;IAChC,QAAQ,EAAE,YAAY,CAAC;IAEvB,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAC;IAEd,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAEhB,6BAA6B;IAC7B,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,kCAAkC;IAClC,KAAK,EAAE,MAAM,CAAC;IAEd,gCAAgC;IAChC,KAAK,EAAE,MAAM,CAAC;IAEd,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IAEnB,kCAAkC;IAClC,WAAW,EAAE,MAAM,CAAC;IAEpB,kCAAkC;IAClC,WAAW,EAAE,MAAM,CAAC;IAEpB,wCAAwC;IACxC,YAAY,EAAE,MAAM,CAAC;IAErB,mCAAmC;IACnC,QAAQ,EAAE,MAAM,CAAC;IAEjB,mCAAmC;IACnC,WAAW,EAAE,cAAc,EAAE,CAAC;IAE9B,qCAAqC;IACrC,eAAe,EAAE,MAAM,CAAC;IAExB,uDAAuD;IACvD,eAAe,EAAE,MAAM,CAAC;IAExB,sCAAsC;IACtC,mBAAmB,EAAE,MAAM,CAAC;IAE5B,4CAA4C;IAC5C,MAAM,EAAE,OAAO,CAAC;IAEhB,+BAA+B;IAC/B,WAAW,EAAE,MAAM,CAAC;IAEpB,8BAA8B;IAC9B,SAAS,EAAE,IAAI,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,WAAW,GAAG,UAAU,CAAC;IAC7D,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,cAAc,EAAE,cAAc,EAAE,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,CACL,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAC7B,OAAO,CAAC;QACT,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,qBAAa,eAAgB,YAAW,WAAW;IAC3C,OAAO,CACX,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAC7B,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAetE,OAAO,CAAC,oBAAoB;CA4B7B;AA6QD;;;GAGG;AACH,qBAAa,kBAAkB;IAK3B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,sBAAsB;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAN3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoB;IAC5C,OAAO,CAAC,gBAAgB,CAAC,CAAmC;gBAGzC,MAAM,EAAE,kBAAkB,YAA+B,EACzD,sBAAsB,EAAE,sBAAsB,EAC9C,QAAQ,GAAE,WAAmC;IAQhE;;OAEG;IACH,UAAU,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,GAAG,IAAI;IAI5D;;OAEG;IACG,eAAe,CACnB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,kBAAkB,CAAC;IA+G9B;;OAEG;IACG,wBAAwB,CAC5B,MAAM,EAAE,MAAM,EAAE,EAChB,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;IA4B7C;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAwB9C;;OAEG;IACH,OAAO,CAAC,WAAW;IAWnB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAa1B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAc1B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IA4C9B;;OAEG;YACW,aAAa;IAM3B;;OAEG;IACH,eAAe,IAAI,KAAK,CAAC;QACvB,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,CAAC;QACf,cAAc,EAAE,MAAM,CAAC;QACvB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;IASF;;OAEG;IACH,YAAY,IAAI,IAAI;CAOrB;AAMD;;GAEG;AACH,wBAAgB,wBAAwB,CACtC,sBAAsB,EAAE,sBAAsB,EAC9C,MAAM,GAAE,OAAO,CAAC,kBAAkB,CAAM,EACxC,QAAQ,CAAC,EAAE,WAAW,GACrB,kBAAkB,CAOpB"}