@yasserkhanorg/e2e-agents 0.3.2

package/dist/esm/agent/model-router.js

@@ -0,0 +1,150 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+const MODEL_RATES = {
+    'claude-haiku-4-0-20250430': 0.25 / 1000000,
+    'claude-sonnet-4-5-20250929': 3 / 1000000,
+    'claude-opus-4-6-20250820': 15 / 1000000,
+};
+export class ModelRouter {
+    constructor(config = {}) {
+        this.modelConfig = {
+            simpleModel: config.simpleModel || 'claude-haiku-4-0-20250430',
+            moderateModel: config.moderateModel || 'claude-sonnet-4-5-20250929',
+            complexModel: config.complexModel || 'claude-opus-4-6-20250820',
+            criticalModel: config.criticalModel || 'claude-opus-4-6-20250820',
+        };
+    }
+    /**
+     * Classify task complexity based on operation, attempt number, and context
+     */
+    classifyTask(context) {
+        // Healing: Haiku for attempts 1-2, Sonnet for attempt 3
+        if (context.operation === 'heal') {
+            if (context.attemptNumber && context.attemptNumber <= 2) {
+                // First two healing attempts use simple classification
+                return {
+                    type: 'simple',
+                    confidence: 95,
+                    reasoning: `Healing attempt ${context.attemptNumber}/3 - re-exploration with targeted fixes`,
+                };
+            }
+            // Final healing attempt is more complex
+            return {
+                type: 'moderate',
+                confidence: 90,
+                reasoning: 'Final healing attempt - may need comprehensive refactoring',
+            };
+        }
+        // Validation: Always simple (use Haiku)
+        if (context.operation === 'validate') {
+            return {
+                type: 'simple',
+                confidence: 100,
+                reasoning: 'Selector/code validation is lightweight',
+            };
+        }
+        // Scoring: Simple (quick analysis)
+        if (context.operation === 'score') {
+            return {
+                type: 'simple',
+                confidence: 95,
+                reasoning: 'Test quality scoring via static analysis',
+            };
+        }
+        // Exploration: Simple (just navigation and snapshot)
+        if (context.operation === 'explore') {
+            return {
+                type: 'simple',
+                confidence: 90,
+                reasoning: 'UI exploration is mostly navigation',
+            };
+        }
+        // Generation: Varies by UI map coverage
+        if (context.operation === 'generate') {
+            // Strong signal: Use Sonnet but can optimize
+            if (context.uiMapCoverage && context.uiMapCoverage >= 75) {
+                return {
+                    type: 'moderate',
+                    confidence: 90,
+                    reasoning: `Strong UI signal (${context.uiMapCoverage}% coverage) - moderate complexity generation`,
+                };
+            }
+            // Moderate signal: Use Sonnet
+            if (context.uiMapCoverage && context.uiMapCoverage >= 50) {
+                return {
+                    type: 'moderate',
+                    confidence: 75,
+                    reasoning: `Moderate UI signal (${context.uiMapCoverage}% coverage) - standard generation`,
+                };
+            }
+            // Weak signal: Complex task (need better reasoning)
+            if (context.uiMapCoverage && context.uiMapCoverage < 50) {
+                return {
+                    type: 'complex',
+                    confidence: 70,
+                    reasoning: `Weak UI signal (${context.uiMapCoverage}% coverage) - requires advanced reasoning`,
+                };
+            }
+            // Unknown coverage: Assume moderate
+            return {
+                type: 'moderate',
+                confidence: 50,
+                reasoning: 'Generation with unknown UI coverage - use standard model',
+            };
+        }
+        // Default to moderate
+        return {
+            type: 'moderate',
+            confidence: 50,
+            reasoning: 'Unknown operation - defaulting to moderate complexity',
+        };
+    }
+    /**
+     * Select appropriate model based on task complexity
+     */
+    selectModel(complexity) {
+        switch (complexity.type) {
+            case 'simple':
+                return this.modelConfig.simpleModel;
+            case 'moderate':
+                return this.modelConfig.moderateModel;
+            case 'complex':
+                return this.modelConfig.complexModel;
+            case 'critical':
+                return this.modelConfig.criticalModel;
+        }
+    }
+    /**
+     * Get estimated cost for a task
+     */
+    estimateCost(complexity, estimatedTokens = 5000) {
+        const model = this.selectModel(complexity);
+        const rate = MODEL_RATES[model] || 0.003 / 1000000;
+        return estimatedTokens * rate;
+    }
+    /**
+     * Get cost savings vs always using Sonnet
+     */
+    estimateSavings(complexity, estimatedTokens = 5000) {
+        const selectedModel = this.selectModel(complexity);
+        const selectedRate = MODEL_RATES[selectedModel] || 0.003 / 1000000;
+        const sonnetRate = MODEL_RATES['claude-sonnet-4-5-20250929'];
+        const selectedCost = estimatedTokens * selectedRate;
+        const sonnetCost = estimatedTokens * sonnetRate;
+        const savedCost = sonnetCost - selectedCost;
+        const savingsPercent = (savedCost / sonnetCost) * 100;
+        return { savedCost, savingsPercent };
+    }
+    /**
+     * Format complexity for logging
+     */
+    formatComplexity(complexity, tokensUsed) {
+        const model = this.selectModel(complexity);
+        const modelShort = model.includes('haiku') ? 'Haiku' : model.includes('sonnet') ? 'Sonnet' : 'Opus';
+        const confidence = `${complexity.confidence}%`;
+        const cost = tokensUsed
+            ? ` ($${(tokensUsed * (MODEL_RATES[model] || 0)).toFixed(4)})`
+            : '';
+        return `${complexity.type.toUpperCase()}/${modelShort}/${confidence}${cost} - ${complexity.reasoning}`;
+    }
+}

package/dist/esm/agent/operational_insights.js

@@ -0,0 +1,123 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { existsSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { inferSubsystemFromTestPath } from './test_path.js';
+function readJson(path) {
+    if (!existsSync(path)) {
+        return null;
+    }
+    try {
+        return JSON.parse(readFileSync(path, 'utf-8'));
+    }
+    catch {
+        return null;
+    }
+}
+function normalizeTestName(test) {
+    return test.replace(/ \(flags:.*\)$/, '').trim();
+}
+function subsystemForTest(test) {
+    return inferSubsystemFromTestPath(test);
+}
+function riskyRate(entry) {
+    if (entry.flakeRate30d !== undefined) {
+        return entry.flakeRate30d;
+    }
+    return entry.flakeRate;
+}
+function loadFlakyManifest(appRoot) {
+    const path = join(appRoot, '.e2e-ai-agents', 'flaky-tests.json');
+    return readJson(path);
+}
+function loadQualityGates(appRoot) {
+    const path = join(appRoot, '.e2e-ai-agents', 'quality-gates.json');
+    return readJson(path);
+}
+function loadCalibration(appRoot) {
+    const path = join(appRoot, '.e2e-ai-agents', 'calibration.json');
+    return readJson(path);
+}
+export function applyOperationalInsights(plan, appRoot) {
+    const enhanced = { ...plan };
+    const insights = {};
+    const flaky = loadFlakyManifest(appRoot);
+    if (flaky && Array.isArray(flaky.tests)) {
+        const recommended = new Set(plan.recommendedTests.map(normalizeTestName));
+        const risky = flaky.tests
+            .filter((entry) => recommended.has(normalizeTestName(entry.test)) && riskyRate(entry) >= 0.2)
+            .sort((a, b) => riskyRate(b) - riskyRate(a))
+            .slice(0, 10);
+        const quarantined = risky.filter((entry) => entry.quarantine).map((entry) => entry.test);
+        const owners = Array.from(new Set(risky
+            .flatMap((entry) => entry.owners || [])
+            .filter(Boolean)));
+        insights.flaky = {
+            highRiskRecommendedTests: risky,
+            quarantinedRecommendedTests: quarantined,
+            ownerMentions: owners,
+        };
+        if (quarantined.length > 0) {
+            enhanced.reasons = [...enhanced.reasons, `Quarantined flaky tests in recommendation: ${quarantined.join(', ')}`];
+        }
+        if (owners.length > 0) {
+            enhanced.reasons = [...enhanced.reasons, `Subsystem owners to notify for flaky risk: ${owners.join(', ')}`];
+        }
+    }
+    const gates = loadQualityGates(appRoot);
+    if (gates && Array.isArray(gates.gates)) {
+        const failed = gates.gates.filter((gate) => gate.status === 'fail');
+        const warnings = gates.gates.filter((gate) => gate.status === 'warn');
+        insights.qualityGates = { failed, warnings };
+        if (failed.length > 0 && enhanced.runSet !== 'full') {
+            enhanced.runSet = 'full';
+            enhanced.reasons = [...enhanced.reasons, `Quality gates failed: ${failed.map((gate) => gate.name).join(', ')}`];
+            enhanced.policy.triggeredRules = [...new Set([...enhanced.policy.triggeredRules, 'quality-gate-failed'])];
+            enhanced.decision = {
+                action: 'run-now',
+                title: 'Run now',
+                summary: 'Quality gate failures detected. Full suite is required before merge.',
+            };
+        }
+    }
+    const calibration = loadCalibration(appRoot);
+    if (calibration) {
+        insights.calibration = calibration.overall;
+        if (calibration.overall.falseNegativeRate >= 0.2 && enhanced.runSet !== 'full') {
+            enhanced.runSet = 'full';
+            enhanced.reasons = [...enhanced.reasons, 'Historical false-negative rate is high; escalating to full suite.'];
+            enhanced.policy.triggeredRules = [...new Set([...enhanced.policy.triggeredRules, 'historical-fnr-high'])];
+        }
+        const recommendedSubsystems = Array.from(new Set(plan.recommendedTests.map(subsystemForTest)));
+        const highRiskSubsystems = recommendedSubsystems
+            .map((subsystem) => {
+            const metric = calibration.bySubsystem[subsystem];
+            if (!metric) {
+                return null;
+            }
+            if (metric.samples < 5) {
+                return null;
+            }
+            if (metric.recent30d.falseNegativeRate >= 0.2 || metric.falseNegativeRate >= 0.25) {
+                return { subsystem, fnr: metric.recent30d.falseNegativeRate || metric.falseNegativeRate };
+            }
+            return null;
+        })
+            .filter(Boolean);
+        if (highRiskSubsystems.length > 0 && enhanced.runSet !== 'full') {
+            enhanced.runSet = 'full';
+            enhanced.reasons = [
+                ...enhanced.reasons,
+                `Historical subsystem false-negative risk is high: ${highRiskSubsystems.map((entry) => `${entry.subsystem}(${entry.fnr})`).join(', ')}`,
+            ];
+            enhanced.policy.triggeredRules = [...new Set([...enhanced.policy.triggeredRules, 'subsystem-fnr-high'])];
+            enhanced.decision = {
+                action: 'run-now',
+                title: 'Run now',
+                summary: 'Subsystem calibration risk is high. Full suite is required before merge.',
+            };
+        }
+    }
+    enhanced.insights = insights;
+    return enhanced;
+}