@yasserkhanorg/e2e-agents 0.3.2

package/dist/esm/agent/subsystem_risk.js

@@ -0,0 +1,204 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { existsSync, readFileSync } from 'fs';
+import { matchGlob, normalizePath } from './utils.js';
+const PRIORITY_RANK = {
+    P0: 0,
+    P1: 1,
+    P2: 2,
+};
+function coerceNumber(value) {
+    if (typeof value === 'number' && Number.isFinite(value)) {
+        return value;
+    }
+    if (typeof value === 'string') {
+        const parsed = Number(value);
+        if (Number.isFinite(parsed)) {
+            return parsed;
+        }
+    }
+    return undefined;
+}
+function normalizePriority(value) {
+    if (value === 'P0' || value === 'P1' || value === 'P2') {
+        return value;
+    }
+    return undefined;
+}
+function parseStringArray(value) {
+    if (!Array.isArray(value)) {
+        return [];
+    }
+    return value
+        .filter((item) => typeof item === 'string')
+        .map((item) => item.trim())
+        .filter(Boolean);
+}
+function parsePathArray(value) {
+    return parseStringArray(value).map((item) => normalizePath(item));
+}
+function parseKeywords(value) {
+    if (!Array.isArray(value)) {
+        return [];
+    }
+    return Array.from(new Set(value
+        .filter((item) => typeof item === 'string')
+        .map((item) => item.trim().toLowerCase())
+        .filter(Boolean)));
+}
+function parseRules(rawRules, warnings) {
+    if (!Array.isArray(rawRules)) {
+        warnings.push('Subsystem risk map has no "rules" array.');
+        return [];
+    }
+    const parsed = [];
+    for (let i = 0; i < rawRules.length; i += 1) {
+        const rawRule = rawRules[i];
+        if (!rawRule || typeof rawRule !== 'object') {
+            continue;
+        }
+        const patterns = parsePathArray(rawRule.patterns);
+        if (patterns.length === 0) {
+            warnings.push(`Subsystem risk rule at index ${i} has no valid patterns and was skipped.`);
+            continue;
+        }
+        const id = typeof rawRule.id === 'string' && rawRule.id.trim()
+            ? rawRule.id.trim()
+            : `rule-${i + 1}`;
+        const description = typeof rawRule.description === 'string' ? rawRule.description.trim() : undefined;
+        const reasons = parseStringArray(rawRule.reasons);
+        const keywords = parseKeywords(rawRule.keywords);
+        const scoreDelta = coerceNumber(rawRule.scoreDelta) ?? 0;
+        const priorityFloor = normalizePriority(rawRule.priorityFloor);
+        const normalizedReasons = reasons.length > 0
+            ? reasons
+            : (description ? [description] : []);
+        parsed.push({
+            id,
+            description,
+            patterns,
+            scoreDelta,
+            priorityFloor,
+            reasons: normalizedReasons,
+            keywords,
+        });
+    }
+    return parsed;
+}
+function comparePriority(a, b) {
+    return PRIORITY_RANK[a] <= PRIORITY_RANK[b] ? a : b;
+}
+export function loadSubsystemRiskResolver(config) {
+    const mapPath = normalizePath(config.mapPath);
+    const warnings = [];
+    if (!config.enabled) {
+        return {
+            info: {
+                source: 'map',
+                enabled: false,
+                mapPath,
+                mapFound: false,
+                rulesLoaded: 0,
+            },
+            warnings,
+            matchFile: () => [],
+        };
+    }
+    if (!existsSync(config.mapPath)) {
+        warnings.push(`Subsystem risk map file not found: ${config.mapPath}`);
+        return {
+            info: {
+                source: 'map',
+                enabled: true,
+                mapPath,
+                mapFound: false,
+                rulesLoaded: 0,
+            },
+            warnings,
+            matchFile: () => [],
+        };
+    }
+    let raw;
+    try {
+        raw = JSON.parse(readFileSync(config.mapPath, 'utf-8'));
+    }
+    catch {
+        warnings.push(`Subsystem risk map is invalid JSON: ${config.mapPath}`);
+        return {
+            info: {
+                source: 'map',
+                enabled: true,
+                mapPath,
+                mapFound: true,
+                rulesLoaded: 0,
+            },
+            warnings,
+            matchFile: () => [],
+        };
+    }
+    const rules = parseRules(raw.rules, warnings);
+    if (rules.length === 0) {
+        warnings.push(`Subsystem risk map loaded but no valid rules were found: ${config.mapPath}`);
+    }
+    const maxRules = Math.max(1, Math.round(config.maxRulesPerFile));
+    return {
+        info: {
+            source: 'map',
+            enabled: true,
+            mapPath,
+            mapFound: true,
+            rulesLoaded: rules.length,
+        },
+        warnings,
+        matchFile: (relativePath) => {
+            const normalizedPath = normalizePath(relativePath);
+            const matches = [];
+            for (const rule of rules) {
+                const matched = rule.patterns.some((pattern) => matchGlob(normalizedPath, pattern));
+                if (!matched) {
+                    continue;
+                }
+                const reasons = rule.reasons.length > 0
+                    ? rule.reasons
+                    : [`Subsystem risk rule matched: ${rule.id}`];
+                matches.push({
+                    ruleId: rule.id,
+                    scoreDelta: rule.scoreDelta,
+                    priorityFloor: rule.priorityFloor,
+                    reasons,
+                    keywords: rule.keywords,
+                    priorityRank: rule.priorityFloor ? PRIORITY_RANK[rule.priorityFloor] : 99,
+                });
+            }
+            if (matches.length <= maxRules) {
+                return matches.map(({ priorityRank, ...rest }) => rest);
+            }
+            matches.sort((a, b) => {
+                const deltaDiff = Math.abs(b.scoreDelta) - Math.abs(a.scoreDelta);
+                if (deltaDiff !== 0) {
+                    return deltaDiff;
+                }
+                const priorityDiff = a.priorityRank - b.priorityRank;
+                if (priorityDiff !== 0) {
+                    return priorityDiff;
+                }
+                return a.ruleId.localeCompare(b.ruleId);
+            });
+            const capped = matches.slice(0, maxRules).map(({ priorityRank, ...rest }) => rest);
+            let floor;
+            for (const match of capped) {
+                if (match.priorityFloor) {
+                    floor = floor ? comparePriority(floor, match.priorityFloor) : match.priorityFloor;
+                }
+            }
+            if (floor) {
+                for (const match of capped) {
+                    if (!match.priorityFloor) {
+                        match.priorityFloor = floor;
+                    }
+                }
+            }
+            return capped;
+        },
+    };
+}

package/dist/esm/agent/telemetry.js

@@ -0,0 +1,216 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+/**
+ * Telemetry Collection System (Phase A2)
+ *
+ * Tracks costs, performance, and success metrics for test generation operations.
+ * Provides visibility into:
+ * - Cost per operation (input/output tokens * model rate)
+ * - Model usage breakdown (Haiku, Sonnet, Opus)
+ * - Success rate by operation
+ * - Performance metrics (duration, tokens used)
+ *
+ * Data stored in: `.e2e-ai-agents/metrics/YYYY-MM-DD.json`
+ */
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
+import { join } from 'path';
+import { randomUUID } from 'crypto';
+const MODEL_RATES = {
+    'claude-haiku-4-0-20250430': 0.25 / 1000000, // per input token
+    'claude-sonnet-4-5-20250929': 3 / 1000000,
+    'claude-opus-4-6-20250820': 15 / 1000000,
+};
+export class TelemetryCollector {
+    constructor(metricsDir = '.e2e-ai-agents/metrics') {
+        this.metrics = new Map();
+        this.metricsDir = metricsDir;
+        this.ensureMetricsDir();
+        this.loadTodayMetrics();
+    }
+    /**
+     * Ensure metrics directory exists
+     */
+    ensureMetricsDir() {
+        if (!existsSync(this.metricsDir)) {
+            mkdirSync(this.metricsDir, { recursive: true });
+        }
+    }
+    /**
+     * Get today's metrics file path
+     */
+    getTodayPath() {
+        const now = new Date();
+        const date = now.toISOString().split('T')[0]; // YYYY-MM-DD
+        return join(this.metricsDir, `${date}.json`);
+    }
+    /**
+     * Load metrics from disk
+     */
+    loadTodayMetrics() {
+        const path = this.getTodayPath();
+        if (existsSync(path)) {
+            try {
+                const data = JSON.parse(readFileSync(path, 'utf-8'));
+                this.metrics.set(path, data);
+            }
+            catch (error) {
+                console.error(`Failed to load metrics from ${path}:`, error);
+            }
+        }
+    }
+    /**
+     * Track a metric
+     */
+    track(metric) {
+        const fullMetric = {
+            id: randomUUID().substring(0, 8),
+            ...metric,
+        };
+        const path = this.getTodayPath();
+        const metrics = this.metrics.get(path) || [];
+        metrics.push(fullMetric);
+        this.metrics.set(path, metrics);
+        // Persist to disk
+        this.saveMetrics();
+    }
+    /**
+     * Save metrics to disk
+     */
+    saveMetrics() {
+        this.metrics.forEach((metrics, path) => {
+            writeFileSync(path, JSON.stringify(metrics, null, 2), 'utf-8');
+        });
+    }
+    /**
+     * Calculate cost for a metric
+     */
+    static calculateCost(model, tokensInput, tokensOutput) {
+        const inputRate = MODEL_RATES[model] || 0.003 / 1000000; // Default estimate
+        const outputRate = inputRate * 3; // Output usually 3x input cost
+        return tokensInput * inputRate + tokensOutput * outputRate;
+    }
+    /**
+     * Generate report for a date range
+     */
+    generateReport(since, until) {
+        const start = since || new Date(new Date().setDate(new Date().getDate() - 7)); // Default: last 7 days
+        const end = until || new Date();
+        // Collect all metrics in date range
+        const allMetrics = [];
+        this.metrics.forEach((metrics) => {
+            metrics.forEach((m) => {
+                const metricDate = new Date(m.timestamp);
+                if (metricDate >= start && metricDate <= end) {
+                    allMetrics.push(m);
+                }
+            });
+        });
+        // Calculate summary
+        const successCount = allMetrics.filter((m) => m.success).length;
+        const failureCount = allMetrics.length - successCount;
+        const totalCost = allMetrics.reduce((sum, m) => sum + m.costUsd, 0);
+        const avgCost = allMetrics.length > 0 ? totalCost / allMetrics.length : 0;
+        const totalTokens = allMetrics.reduce((sum, m) => sum + (m.tokensInput + m.tokensOutput), 0);
+        const avgDuration = allMetrics.length > 0 ? allMetrics.reduce((sum, m) => sum + m.durationMs, 0) / allMetrics.length / 1000 : 0;
+        // By model
+        const byModel = {};
+        allMetrics.forEach((m) => {
+            if (!byModel[m.model]) {
+                byModel[m.model] = { count: 0, totalCost: 0, successCount: 0 };
+            }
+            byModel[m.model].count += 1;
+            byModel[m.model].totalCost += m.costUsd;
+            if (m.success)
+                byModel[m.model].successCount += 1;
+        });
+        Object.keys(byModel).forEach((model) => {
+            const data = byModel[model];
+            byModel[model] = {
+                count: data.count,
+                totalCost: data.totalCost,
+                avgCost: data.totalCost / data.count,
+                successRate: (data.successCount / data.count) * 100,
+            };
+        });
+        // By operation
+        const byOperation = {};
+        allMetrics.forEach((m) => {
+            if (!byOperation[m.operation]) {
+                byOperation[m.operation] = { count: 0, totalCost: 0, totalDuration: 0, successCount: 0 };
+            }
+            byOperation[m.operation].count += 1;
+            byOperation[m.operation].totalCost += m.costUsd;
+            byOperation[m.operation].totalDuration += m.durationMs;
+            if (m.success)
+                byOperation[m.operation].successCount += 1;
+        });
+        Object.keys(byOperation).forEach((op) => {
+            const data = byOperation[op];
+            byOperation[op] = {
+                count: data.count,
+                totalCost: data.totalCost,
+                avgDuration: data.totalDuration / data.count / 1000,
+                successRate: (data.successCount / data.count) * 100,
+            };
+        });
+        return {
+            period: {
+                start: start.toISOString().split('T')[0],
+                end: end.toISOString().split('T')[0],
+            },
+            summary: {
+                totalOperations: allMetrics.length,
+                successCount,
+                failureCount,
+                successRate: allMetrics.length > 0 ? (successCount / allMetrics.length) * 100 : 0,
+                totalCost,
+                avgCost,
+                totalTokens,
+                avgDuration,
+            },
+            byModel,
+            byOperation,
+        };
+    }
+    /**
+     * Format report for console output
+     */
+    static formatReport(report) {
+        const lines = [
+            '',
+            '📊 Test Generation Metrics',
+            `Period: ${report.period.start} to ${report.period.end}`,
+            '═'.repeat(50),
+            '',
+            `Total Operations: ${report.summary.totalOperations}`,
+            `Success Rate: ${report.summary.successRate.toFixed(1)}% (${report.summary.successCount}/${report.summary.totalOperations})`,
+            `Total Cost: $${report.summary.totalCost.toFixed(2)}`,
+            `Avg Cost/Op: $${report.summary.avgCost.toFixed(4)}`,
+            `Avg Duration: ${report.summary.avgDuration.toFixed(1)}s`,
+            `Total Tokens: ${report.summary.totalTokens.toLocaleString()}`,
+            '',
+            'Model Usage:',
+        ];
+        Object.entries(report.byModel).forEach(([model, data]) => {
+            const shortName = model.includes('haiku') ? 'Haiku' : model.includes('sonnet') ? 'Sonnet' : 'Opus';
+            lines.push(`  ${shortName}: ${data.count} ops - $${data.totalCost.toFixed(2)} (avg $${data.avgCost.toFixed(4)}, ${data.successRate.toFixed(0)}% success)`);
+        });
+        lines.push('', 'By Operation:');
+        Object.entries(report.byOperation).forEach(([op, data]) => {
+            lines.push(`  ${op}: ${data.count} ops - $${data.totalCost.toFixed(2)} (${data.avgDuration.toFixed(1)}s avg, ${data.successRate.toFixed(0)}% success)`);
+        });
+        lines.push('');
+        return lines.join('\n');
+    }
+    /**
+     * Export metrics as JSON
+     */
+    exportJson(filepath) {
+        const allMetrics = [];
+        this.metrics.forEach((metrics) => {
+            allMetrics.push(...metrics);
+        });
+        writeFileSync(filepath, JSON.stringify(allMetrics, null, 2), 'utf-8');
+        console.log(`  ✓ Exported ${allMetrics.length} metrics to ${filepath}`);
+    }
+}

package/dist/esm/agent/test_path.js

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+function normalizeTestName(test) {
+    return test.replace(/ \(flags:.*\)$/, '').trim();
+}
+export function inferSubsystemFromTestPath(test) {
+    const normalized = normalizeTestName(test).replace(/^\/+/, '');
+    const parts = normalized.split('/').filter(Boolean);
+    if (parts.length === 0) {
+        return 'unknown';
+    }
+    const specsIdx = parts.findIndex((part) => part === 'specs' || part === 'tests');
+    if (specsIdx >= 0 && specsIdx + 1 < parts.length) {
+        return parts[specsIdx + 1] || 'unknown';
+    }
+    if (parts.length >= 2 && (parts[0] === 'e2e-tests' || parts[0] === 'playwright')) {
+        return parts[1] || 'unknown';
+    }
+    return parts[0] || 'unknown';
+}

package/dist/esm/agent/tests.js

@@ -0,0 +1,101 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { existsSync } from 'fs';
+import { globSync } from 'glob';
+import { join } from 'path';
+import { normalizePath, safeReadTextFile, tokenize, uniqueTokens } from './utils.js';
+export function discoverTests(appRoot, patterns) {
+    const files = new Set();
+    for (const pattern of patterns) {
+        const matches = globSync(pattern, {
+            cwd: appRoot,
+            ignore: ['**/node_modules/**', '**/.git/**'],
+            nodir: true,
+        });
+        for (const match of matches) {
+            files.add(normalizePath(match));
+        }
+    }
+    return Array.from(files).map((relativePath) => {
+        const fullPath = join(appRoot, relativePath);
+        const content = safeReadTextFile(fullPath);
+        return { path: relativePath, content };
+    });
+}
+function buildFlowKeywords(flow) {
+    const tokens = [];
+    tokens.push(...tokenize(flow.id));
+    tokens.push(...tokenize(flow.name));
+    tokens.push(...flow.keywords);
+    return uniqueTokens(tokens);
+}
+export function mapTestsToFlows(flows, tests) {
+    const coverage = [];
+    for (const flow of flows) {
+        const keywords = buildFlowKeywords(flow);
+        const matched = [];
+        let score = 0;
+        for (const test of tests) {
+            const haystack = `${test.path} ${test.content || ''}`.toLowerCase();
+            let localScore = 0;
+            for (const keyword of keywords) {
+                if (keyword && haystack.includes(keyword.toLowerCase())) {
+                    localScore += 1;
+                }
+            }
+            if (localScore > 0) {
+                matched.push(test.path);
+                score += localScore;
+            }
+        }
+        coverage.push({
+            flowId: flow.id,
+            flowName: flow.name,
+            priority: flow.priority,
+            coveredBy: matched,
+            score,
+            source: 'heuristic',
+        });
+    }
+    return coverage;
+}
+function resolveExpectedTests(testsRoot, expectedTests) {
+    const resolved = [];
+    for (const entry of expectedTests) {
+        if (!entry) {
+            continue;
+        }
+        resolved.push(normalizePath(entry));
+    }
+    return resolved;
+}
+export function mapCatalogTestsToFlows(flows, testsRoot, testsByFlow) {
+    return flows.map((flow) => {
+        const expectedTests = resolveExpectedTests(testsRoot, testsByFlow.get(flow.id) || []);
+        const coveredBy = [];
+        for (const expected of expectedTests) {
+            const isAbsolute = expected.startsWith('/');
+            const globTarget = isAbsolute ? expected : expected;
+            if (expected.includes('*') || expected.includes('?') || expected.includes('{')) {
+                const matches = globSync(globTarget, { cwd: isAbsolute ? undefined : testsRoot, nodir: true });
+                if (matches.length > 0) {
+                    coveredBy.push(...matches.map((match) => normalizePath(match)));
+                }
+                continue;
+            }
+            const fullPath = isAbsolute ? expected : join(testsRoot, expected);
+            if (existsSync(fullPath)) {
+                coveredBy.push(isAbsolute ? normalizePath(expected) : expected);
+            }
+        }
+        return {
+            flowId: flow.id,
+            flowName: flow.name,
+            priority: flow.priority,
+            coveredBy: Array.from(new Set(coveredBy)),
+            score: coveredBy.length,
+            expectedTests,
+            source: 'catalog',
+        };
+    });
+}