@yasserkhanorg/e2e-agents 0.3.2

package/dist/agent/model-router.js

@@ -0,0 +1,154 @@
+"use strict";
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ModelRouter = void 0;
+const MODEL_RATES = {
+    'claude-haiku-4-0-20250430': 0.25 / 1000000,
+    'claude-sonnet-4-5-20250929': 3 / 1000000,
+    'claude-opus-4-6-20250820': 15 / 1000000,
+};
+class ModelRouter {
+    constructor(config = {}) {
+        this.modelConfig = {
+            simpleModel: config.simpleModel || 'claude-haiku-4-0-20250430',
+            moderateModel: config.moderateModel || 'claude-sonnet-4-5-20250929',
+            complexModel: config.complexModel || 'claude-opus-4-6-20250820',
+            criticalModel: config.criticalModel || 'claude-opus-4-6-20250820',
+        };
+    }
+    /**
+     * Classify task complexity based on operation, attempt number, and context
+     */
+    classifyTask(context) {
+        // Healing: Haiku for attempts 1-2, Sonnet for attempt 3
+        if (context.operation === 'heal') {
+            if (context.attemptNumber && context.attemptNumber <= 2) {
+                // First two healing attempts use simple classification
+                return {
+                    type: 'simple',
+                    confidence: 95,
+                    reasoning: `Healing attempt ${context.attemptNumber}/3 - re-exploration with targeted fixes`,
+                };
+            }
+            // Final healing attempt is more complex
+            return {
+                type: 'moderate',
+                confidence: 90,
+                reasoning: 'Final healing attempt - may need comprehensive refactoring',
+            };
+        }
+        // Validation: Always simple (use Haiku)
+        if (context.operation === 'validate') {
+            return {
+                type: 'simple',
+                confidence: 100,
+                reasoning: 'Selector/code validation is lightweight',
+            };
+        }
+        // Scoring: Simple (quick analysis)
+        if (context.operation === 'score') {
+            return {
+                type: 'simple',
+                confidence: 95,
+                reasoning: 'Test quality scoring via static analysis',
+            };
+        }
+        // Exploration: Simple (just navigation and snapshot)
+        if (context.operation === 'explore') {
+            return {
+                type: 'simple',
+                confidence: 90,
+                reasoning: 'UI exploration is mostly navigation',
+            };
+        }
+        // Generation: Varies by UI map coverage
+        if (context.operation === 'generate') {
+            // Strong signal: Use Sonnet but can optimize
+            if (context.uiMapCoverage && context.uiMapCoverage >= 75) {
+                return {
+                    type: 'moderate',
+                    confidence: 90,
+                    reasoning: `Strong UI signal (${context.uiMapCoverage}% coverage) - moderate complexity generation`,
+                };
+            }
+            // Moderate signal: Use Sonnet
+            if (context.uiMapCoverage && context.uiMapCoverage >= 50) {
+                return {
+                    type: 'moderate',
+                    confidence: 75,
+                    reasoning: `Moderate UI signal (${context.uiMapCoverage}% coverage) - standard generation`,
+                };
+            }
+            // Weak signal: Complex task (need better reasoning)
+            if (context.uiMapCoverage && context.uiMapCoverage < 50) {
+                return {
+                    type: 'complex',
+                    confidence: 70,
+                    reasoning: `Weak UI signal (${context.uiMapCoverage}% coverage) - requires advanced reasoning`,
+                };
+            }
+            // Unknown coverage: Assume moderate
+            return {
+                type: 'moderate',
+                confidence: 50,
+                reasoning: 'Generation with unknown UI coverage - use standard model',
+            };
+        }
+        // Default to moderate
+        return {
+            type: 'moderate',
+            confidence: 50,
+            reasoning: 'Unknown operation - defaulting to moderate complexity',
+        };
+    }
+    /**
+     * Select appropriate model based on task complexity
+     */
+    selectModel(complexity) {
+        switch (complexity.type) {
+            case 'simple':
+                return this.modelConfig.simpleModel;
+            case 'moderate':
+                return this.modelConfig.moderateModel;
+            case 'complex':
+                return this.modelConfig.complexModel;
+            case 'critical':
+                return this.modelConfig.criticalModel;
+        }
+    }
+    /**
+     * Get estimated cost for a task
+     */
+    estimateCost(complexity, estimatedTokens = 5000) {
+        const model = this.selectModel(complexity);
+        const rate = MODEL_RATES[model] || 0.003 / 1000000;
+        return estimatedTokens * rate;
+    }
+    /**
+     * Get cost savings vs always using Sonnet
+     */
+    estimateSavings(complexity, estimatedTokens = 5000) {
+        const selectedModel = this.selectModel(complexity);
+        const selectedRate = MODEL_RATES[selectedModel] || 0.003 / 1000000;
+        const sonnetRate = MODEL_RATES['claude-sonnet-4-5-20250929'];
+        const selectedCost = estimatedTokens * selectedRate;
+        const sonnetCost = estimatedTokens * sonnetRate;
+        const savedCost = sonnetCost - selectedCost;
+        const savingsPercent = (savedCost / sonnetCost) * 100;
+        return { savedCost, savingsPercent };
+    }
+    /**
+     * Format complexity for logging
+     */
+    formatComplexity(complexity, tokensUsed) {
+        const model = this.selectModel(complexity);
+        const modelShort = model.includes('haiku') ? 'Haiku' : model.includes('sonnet') ? 'Sonnet' : 'Opus';
+        const confidence = `${complexity.confidence}%`;
+        const cost = tokensUsed
+            ? ` ($${(tokensUsed * (MODEL_RATES[model] || 0)).toFixed(4)})`
+            : '';
+        return `${complexity.type.toUpperCase()}/${modelShort}/${confidence}${cost} - ${complexity.reasoning}`;
+    }
+}
+exports.ModelRouter = ModelRouter;

package/dist/agent/operational_insights.d.ts

@@ -0,0 +1,41 @@
+import type { PlanReport } from './plan.js';
+import type { CalibrationSummary } from './feedback.js';
+export interface FlakyTestRecord {
+    test: string;
+    flakeRate: number;
+    flakeRate7d?: number;
+    flakeRate30d?: number;
+    trend?: 'up' | 'down' | 'stable';
+    subsystem?: string;
+    owners?: string[];
+    quarantine?: boolean;
+    quarantineState?: 'none' | 'active' | 'retire-candidate';
+    lastFailureAt?: string;
+}
+export interface FlakyManifest {
+    schemaVersion?: string;
+    tests: FlakyTestRecord[];
+}
+export interface QualityGateRecord {
+    name: string;
+    status: 'pass' | 'warn' | 'fail';
+    details?: string;
+}
+export interface QualityGateManifest {
+    schemaVersion?: string;
+    gates: QualityGateRecord[];
+}
+export interface OperationalInsights {
+    flaky?: {
+        highRiskRecommendedTests: FlakyTestRecord[];
+        quarantinedRecommendedTests: string[];
+        ownerMentions?: string[];
+    };
+    qualityGates?: {
+        failed: QualityGateRecord[];
+        warnings: QualityGateRecord[];
+    };
+    calibration?: CalibrationSummary['overall'];
+}
+export declare function applyOperationalInsights(plan: PlanReport, appRoot: string): PlanReport;
+//# sourceMappingURL=operational_insights.d.ts.map

package/dist/agent/operational_insights.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"operational_insights.d.ts","sourceRoot":"","sources":["../../src/agent/operational_insights.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,eAAe,CAAC;AAGtD,MAAM,WAAW,eAAe;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,IAAI,GAAG,MAAM,GAAG,QAAQ,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,kBAAkB,CAAC;IACzD,aAAa,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,eAAe,EAAE,CAAC;CAC5B;AAED,MAAM,WAAW,iBAAiB;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAChC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,iBAAiB,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,mBAAmB;IAChC,KAAK,CAAC,EAAE;QACJ,wBAAwB,EAAE,eAAe,EAAE,CAAC;QAC5C,2BAA2B,EAAE,MAAM,EAAE,CAAC;QACtC,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;KAC5B,CAAC;IACF,YAAY,CAAC,EAAE;QACX,MAAM,EAAE,iBAAiB,EAAE,CAAC;QAC5B,QAAQ,EAAE,iBAAiB,EAAE,CAAC;KACjC,CAAC;IACF,WAAW,CAAC,EAAE,kBAAkB,CAAC,SAAS,CAAC,CAAC;CAC/C;AA2CD,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,GAAG,UAAU,CA4FtF"}

package/dist/agent/operational_insights.js

@@ -0,0 +1,126 @@
+"use strict";
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.applyOperationalInsights = applyOperationalInsights;
+const fs_1 = require("fs");
+const path_1 = require("path");
+const test_path_js_1 = require("./test_path.js");
+function readJson(path) {
+    if (!(0, fs_1.existsSync)(path)) {
+        return null;
+    }
+    try {
+        return JSON.parse((0, fs_1.readFileSync)(path, 'utf-8'));
+    }
+    catch {
+        return null;
+    }
+}
+function normalizeTestName(test) {
+    return test.replace(/ \(flags:.*\)$/, '').trim();
+}
+function subsystemForTest(test) {
+    return (0, test_path_js_1.inferSubsystemFromTestPath)(test);
+}
+function riskyRate(entry) {
+    if (entry.flakeRate30d !== undefined) {
+        return entry.flakeRate30d;
+    }
+    return entry.flakeRate;
+}
+function loadFlakyManifest(appRoot) {
+    const path = (0, path_1.join)(appRoot, '.e2e-ai-agents', 'flaky-tests.json');
+    return readJson(path);
+}
+function loadQualityGates(appRoot) {
+    const path = (0, path_1.join)(appRoot, '.e2e-ai-agents', 'quality-gates.json');
+    return readJson(path);
+}
+function loadCalibration(appRoot) {
+    const path = (0, path_1.join)(appRoot, '.e2e-ai-agents', 'calibration.json');
+    return readJson(path);
+}
+function applyOperationalInsights(plan, appRoot) {
+    const enhanced = { ...plan };
+    const insights = {};
+    const flaky = loadFlakyManifest(appRoot);
+    if (flaky && Array.isArray(flaky.tests)) {
+        const recommended = new Set(plan.recommendedTests.map(normalizeTestName));
+        const risky = flaky.tests
+            .filter((entry) => recommended.has(normalizeTestName(entry.test)) && riskyRate(entry) >= 0.2)
+            .sort((a, b) => riskyRate(b) - riskyRate(a))
+            .slice(0, 10);
+        const quarantined = risky.filter((entry) => entry.quarantine).map((entry) => entry.test);
+        const owners = Array.from(new Set(risky
+            .flatMap((entry) => entry.owners || [])
+            .filter(Boolean)));
+        insights.flaky = {
+            highRiskRecommendedTests: risky,
+            quarantinedRecommendedTests: quarantined,
+            ownerMentions: owners,
+        };
+        if (quarantined.length > 0) {
+            enhanced.reasons = [...enhanced.reasons, `Quarantined flaky tests in recommendation: ${quarantined.join(', ')}`];
+        }
+        if (owners.length > 0) {
+            enhanced.reasons = [...enhanced.reasons, `Subsystem owners to notify for flaky risk: ${owners.join(', ')}`];
+        }
+    }
+    const gates = loadQualityGates(appRoot);
+    if (gates && Array.isArray(gates.gates)) {
+        const failed = gates.gates.filter((gate) => gate.status === 'fail');
+        const warnings = gates.gates.filter((gate) => gate.status === 'warn');
+        insights.qualityGates = { failed, warnings };
+        if (failed.length > 0 && enhanced.runSet !== 'full') {
+            enhanced.runSet = 'full';
+            enhanced.reasons = [...enhanced.reasons, `Quality gates failed: ${failed.map((gate) => gate.name).join(', ')}`];
+            enhanced.policy.triggeredRules = [...new Set([...enhanced.policy.triggeredRules, 'quality-gate-failed'])];
+            enhanced.decision = {
+                action: 'run-now',
+                title: 'Run now',
+                summary: 'Quality gate failures detected. Full suite is required before merge.',
+            };
+        }
+    }
+    const calibration = loadCalibration(appRoot);
+    if (calibration) {
+        insights.calibration = calibration.overall;
+        if (calibration.overall.falseNegativeRate >= 0.2 && enhanced.runSet !== 'full') {
+            enhanced.runSet = 'full';
+            enhanced.reasons = [...enhanced.reasons, 'Historical false-negative rate is high; escalating to full suite.'];
+            enhanced.policy.triggeredRules = [...new Set([...enhanced.policy.triggeredRules, 'historical-fnr-high'])];
+        }
+        const recommendedSubsystems = Array.from(new Set(plan.recommendedTests.map(subsystemForTest)));
+        const highRiskSubsystems = recommendedSubsystems
+            .map((subsystem) => {
+            const metric = calibration.bySubsystem[subsystem];
+            if (!metric) {
+                return null;
+            }
+            if (metric.samples < 5) {
+                return null;
+            }
+            if (metric.recent30d.falseNegativeRate >= 0.2 || metric.falseNegativeRate >= 0.25) {
+                return { subsystem, fnr: metric.recent30d.falseNegativeRate || metric.falseNegativeRate };
+            }
+            return null;
+        })
+            .filter(Boolean);
+        if (highRiskSubsystems.length > 0 && enhanced.runSet !== 'full') {
+            enhanced.runSet = 'full';
+            enhanced.reasons = [
+                ...enhanced.reasons,
+                `Historical subsystem false-negative risk is high: ${highRiskSubsystems.map((entry) => `${entry.subsystem}(${entry.fnr})`).join(', ')}`,
+            ];
+            enhanced.policy.triggeredRules = [...new Set([...enhanced.policy.triggeredRules, 'subsystem-fnr-high'])];
+            enhanced.decision = {
+                action: 'run-now',
+                title: 'Run now',
+                summary: 'Subsystem calibration risk is high. Full suite is required before merge.',
+            };
+        }
+    }
+    enhanced.insights = insights;
+    return enhanced;
+}

package/dist/agent/pipeline.d.ts

@@ -0,0 +1,23 @@
+import type { PipelineConfig } from './config.js';
+import type { FlowImpact } from './analysis.js';
+export interface PipelineResult {
+    flowId: string;
+    flowName: string;
+    generatedDir: string;
+    generateStatus: 'success' | 'skipped' | 'failed';
+    healStatus?: 'success' | 'skipped' | 'failed';
+    error?: string;
+}
+export interface PipelineSummary {
+    runner: 'e2e-test-gen' | 'package-native' | 'unknown';
+    results: PipelineResult[];
+    warnings: string[];
+}
+export interface SpecHealTarget {
+    specPath: string;
+    status?: 'failed' | 'flaky';
+    reason?: string;
+}
+export declare function runTargetedSpecHeal(testsRoot: string, targets: SpecHealTarget[], pipeline: PipelineConfig): PipelineSummary;
+export declare function runPlaywrightPipeline(testsRoot: string, flows: FlowImpact[], pipeline: PipelineConfig): PipelineSummary;
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/agent/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/agent/pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,eAAe,CAAC;AAG9C,MAAM,WAAW,cAAc;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,SAAS,GAAG,SAAS,GAAG,QAAQ,CAAC;IACjD,UAAU,CAAC,EAAE,SAAS,GAAG,SAAS,GAAG,QAAQ,CAAC;IAC9C,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC5B,MAAM,EAAE,cAAc,GAAG,gBAAgB,GAAG,SAAS,CAAC;IACtD,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,QAAQ,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAmeD,wBAAgB,mBAAmB,CAC/B,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,cAAc,EAAE,EACzB,QAAQ,EAAE,cAAc,GACzB,eAAe,CAsFjB;AAwBD,wBAAgB,qBAAqB,CACjC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,UAAU,EAAE,EACnB,QAAQ,EAAE,cAAc,GACzB,eAAe,CAyHjB"}