agentic-qe 3.4.0 → 3.4.2

package/v3/dist/validation/validation-result-aggregator.d.ts

@@ -0,0 +1,232 @@
+/**
+ * Validation Result Aggregator
+ * ADR-056 Phase 5: Aggregates validation results from parallel runs
+ *
+ * This module provides:
+ * - Aggregation of multiple skill validation results
+ * - Cross-model anomaly detection
+ * - Regression detection against historical data
+ * - Unified markdown and JSON report generation
+ * - Trust tier manifest auto-updates
+ *
+ * @module validation/validation-result-aggregator
+ * @see docs/adrs/ADR-056-trust-but-verify.md
+ */
+import type { SkillValidationLearner, SkillValidationOutcome, SkillTrustTier } from '../learning/skill-validation-learner.js';
+import type { SwarmValidationResult as BaseSwarmValidationResult } from './swarm-skill-validator.js';
+/**
+ * Issue severity level
+ */
+export type IssueSeverity = 'critical' | 'high' | 'medium' | 'low';
+/**
+ * Issue found during validation
+ */
+export interface ValidationIssue {
+    skill: string;
+    model: string;
+    severity: IssueSeverity;
+    type: 'schema_failure' | 'validator_failure' | 'eval_failure' | 'timeout' | 'error';
+    message: string;
+    testId?: string;
+    details?: Record<string, unknown>;
+}
+/**
+ * Model-specific anomaly detected during cross-model analysis
+ */
+export interface ModelAnomaly {
+    model: string;
+    type: 'high_variance' | 'low_performance' | 'inconsistent' | 'regression';
+    description: string;
+    passRate: number;
+    avgPassRate: number;
+    deviation: number;
+}
+/**
+ * Validation summary for a single skill
+ */
+export interface SkillValidationSummary {
+    skill: string;
+    trustTier: SkillTrustTier;
+    passRateByModel: Map<string, number>;
+    avgPassRate: number;
+    schemaValid: boolean;
+    validatorPassed: boolean;
+    evalPassed: boolean;
+    issues: ValidationIssue[];
+    executionTimeMs: number;
+    testCount: number;
+    passedTests: number;
+    failedTests: number;
+}
+/**
+ * Cross-model analysis report
+ */
+export interface CrossModelReport {
+    variance: number;
+    stdDeviation: number;
+    anomalies: ModelAnomaly[];
+    consistentSkills: string[];
+    inconsistentSkills: string[];
+    modelPerformance: Map<string, {
+        avgPassRate: number;
+        skillCount: number;
+        totalTests: number;
+    }>;
+}
+/**
+ * Regression detected against historical baseline
+ */
+export interface RegressionReport {
+    skill: string;
+    model: string;
+    previousPassRate: number;
+    currentPassRate: number;
+    regressionAmount: number;
+    trend: 'improving' | 'stable' | 'declining';
+    possibleCauses: string[];
+    severity: IssueSeverity;
+}
+/**
+ * Aggregated result from a parallel validation run (input to aggregator)
+ * Contains multiple outcomes from a single model run
+ */
+export interface ParallelValidationRunResult {
+    runId: string;
+    model: string;
+    outcomes: SkillValidationOutcome[];
+    timestamp: Date;
+    durationMs: number;
+    metadata?: {
+        environment?: string;
+        version?: string;
+        [key: string]: unknown;
+    };
+}
+/**
+ * Re-export the base swarm validation result for convenience
+ */
+export type { BaseSwarmValidationResult as SwarmValidationResult };
+/**
+ * Complete aggregated validation report
+ */
+export interface AggregatedValidationReport {
+    timestamp: Date;
+    runId: string;
+    summary: {
+        totalSkills: number;
+        passedSkills: number;
+        failedSkills: number;
+        avgPassRate: number;
+        totalDurationMs: number;
+        totalTests: number;
+        passedTests: number;
+        failedTests: number;
+        modelsUsed: string[];
+    };
+    skillResults: Map<string, SkillValidationSummary>;
+    crossModelAnalysis: CrossModelReport;
+    regressions: RegressionReport[];
+    recommendations: string[];
+    metadata: {
+        version: string;
+        environment?: string;
+        generatedBy: string;
+        inputs: {
+            runIds: string[];
+            models: string[];
+        };
+    };
+}
+/**
+ * Configuration for the aggregator
+ */
+export interface AggregatorConfig {
+    /** Variance threshold for cross-model anomaly detection (0-1) */
+    varianceThreshold: number;
+    /** Pass rate drop threshold for regression detection (0-1) */
+    regressionThreshold: number;
+    /** Minimum samples required for statistical analysis */
+    minSamples: number;
+    /** Whether to auto-update manifest after aggregation */
+    autoUpdateManifest: boolean;
+}
+/**
+ * Aggregates validation results from parallel runs, detects anomalies,
+ * and generates unified reports.
+ *
+ * @example
+ * ```typescript
+ * const aggregator = new ValidationResultAggregator(learner, manifestPath);
+ *
+ * // Aggregate results from multiple parallel runs
+ * const report = await aggregator.aggregateResults([
+ *   { runId: 'run-1', model: 'sonnet', outcomes: [...], ... },
+ *   { runId: 'run-2', model: 'opus', outcomes: [...], ... },
+ * ]);
+ *
+ * // Generate markdown report
+ * const markdown = aggregator.generateMarkdownReport(report);
+ *
+ * // Update manifest with new pass rates
+ * await aggregator.updateManifest(report);
+ * ```
+ */
+export declare class ValidationResultAggregator {
+    private readonly learner;
+    private readonly manifestPath;
+    private config;
+    constructor(learner: SkillValidationLearner, manifestPath: string, config?: Partial<AggregatorConfig>);
+    /**
+     * Aggregate results from parallel validation runs
+     */
+    aggregateResults(results: ParallelValidationRunResult[]): Promise<AggregatedValidationReport>;
+    /**
+     * Build skill results map from multiple validation runs
+     */
+    private buildSkillResultsMap;
+    /**
+     * Categorize issue type based on validation level and error message
+     */
+    private categorizeIssueType;
+    /**
+     * Calculate summary statistics
+     */
+    private calculateSummary;
+    /**
+     * Detect cross-model anomalies
+     */
+    detectCrossModelAnomalies(results: ParallelValidationRunResult[]): Promise<CrossModelReport>;
+    /**
+     * Detect regressions against historical data
+     */
+    detectRegressions(results: ParallelValidationRunResult[], threshold: number): Promise<RegressionReport[]>;
+    /**
+     * Analyze possible causes for regression
+     */
+    private analyzePossibleCauses;
+    /**
+     * Categorize regression severity
+     */
+    private categorizeRegressionSeverity;
+    /**
+     * Generate recommendations based on analysis
+     */
+    private generateRecommendations;
+    /**
+     * Update trust tier manifest with new pass rates
+     */
+    updateManifest(report: AggregatedValidationReport): Promise<void>;
+    /**
+     * Generate markdown report
+     */
+    generateMarkdownReport(report: AggregatedValidationReport): string;
+    /**
+     * Generate JSON report for CI integration
+     */
+    generateJsonReport(report: AggregatedValidationReport): string;
+}
+/**
+ * Create a ValidationResultAggregator instance
+ */
+export declare function createValidationResultAggregator(learner: SkillValidationLearner, manifestPath: string, config?: Partial<AggregatorConfig>): ValidationResultAggregator;
+//# sourceMappingURL=validation-result-aggregator.d.ts.map

package/v3/dist/validation/validation-result-aggregator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validation-result-aggregator.d.ts","sourceRoot":"","sources":["../../src/validation/validation-result-aggregator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,KAAK,EACV,sBAAsB,EACtB,sBAAsB,EACtB,cAAc,EACf,MAAM,yCAAyC,CAAC;AACjD,OAAO,KAAK,EAAE,qBAAqB,IAAI,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAMrG;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,aAAa,CAAC;IACxB,IAAI,EAAE,gBAAgB,GAAG,mBAAmB,GAAG,cAAc,GAAG,SAAS,GAAG,OAAO,CAAC;IACpF,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,eAAe,GAAG,iBAAiB,GAAG,cAAc,GAAG,YAAY,CAAC;IAC1E,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,cAAc,CAAC;IAC1B,eAAe,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,OAAO,CAAC;IACzB,UAAU,EAAE,OAAO,CAAC;IACpB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,gBAAgB,EAAE,GAAG,CAAC,MAAM,EAAE;QAC5B,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;IAC5C,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,aAAa,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,2BAA2B;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,sBAAsB,EAAE,CAAC;IACnC,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE;QACT,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACxB,CAAC;CACH;AAED;;GAEG;AACH,YAAY,EAAE,yBAAyB,IAAI,qBAAqB,EAAE,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,SAAS,EAAE,IAAI,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE;QACP,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;IACF,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;IAClD,kBAAkB,EAAE,gBAAgB,CAAC;IACrC,WAAW,EAAE,gBAAgB,EAAE,CAAC;IAChC,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,QAAQ,EAAE;QACR,OAAO,EAAE,MAAM,CAAC;QAChB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE;YACN,MAAM,EAAE,MAAM,EAAE,CAAC;YACjB,MAAM,EAAE,MAAM,EAAE,CAAC;SAClB,CAAC;KACH,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,iEAAiE;IACjE,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8DAA8D;IAC9D,mBAAmB,EAAE,MAAM,CAAC;IAC5B,wDAAwD;IACxD,UAAU,EAAE,MAAM,CAAC;IACnB,wDAAwD;IACxD,kBAAkB,EAAE,OAAO,CAAC;CAC7B;AAiBD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,0BAA0B;IAInC,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAJ/B,OAAO,CAAC,MAAM,CAAmB;gBAGd,OAAO,EAAE,sBAAsB,EAC/B,YAAY,EAAE,MAAM,EACrC,MAAM,GAAE,OAAO,CAAC,gBAAgB,CAAM;IAKxC;;OAEG;IACG,gBAAgB,CACpB,OAAO,EAAE,2BAA2B,EAAE,GACrC,OAAO,CAAC,0BAA0B,CAAC;IAkDtC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAiF5B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAW3B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAuBxB;;OAEG;IACG,yBAAyB,CAC7B,OAAO,EAAE,2BAA2B,EAAE,GACrC,OAAO,CAAC,gBAAgB,CAAC;IAsG5B;;OAEG;IACG,iBAAiB,CACrB,OAAO,EAAE,2BAA2B,EAAE,EACtC,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,gBAAgB,EAAE,CAAC;IA4C9B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAwC7B;;OAEG;IACH,OAAO,CAAC,4BAA4B;IAOpC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAqE/B;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC;IA+CvE;;OAEG;IACH,sBAAsB,CAAC,MAAM,EAAE,0BAA0B,GAAG,MAAM;IA0JlE;;OAEG;IACH,kBAAkB,CAAC,MAAM,EAAE,0BAA0B,GAAG,MAAM;CAqB/D;AAMD;;GAEG;AACH,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,sBAAsB,EAC/B,YAAY,EAAE,MAAM,EACpB,MAAM,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GACjC,0BAA0B,CAE5B"}