@yasserkhanorg/e2e-agents 0.3.2

package/dist/esm/agent/dependency_graph.js

@@ -0,0 +1,224 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { globSync } from 'glob';
+import { dirname, join, normalize } from 'path';
+import { normalizePath, safeReadTextFile } from './utils.js';
+const IMPORT_REGEXES = [
+    /import\s+[\s\S]*?\s+from\s+['"]([^'"]+)['"]/g,
+    /import\s+['"]([^'"]+)['"]/g,
+    /export\s+[\s\S]*?\s+from\s+['"]([^'"]+)['"]/g,
+    /require\(\s*['"]([^'"]+)['"]\s*\)/g,
+    /import\(\s*['"]([^'"]+)['"]\s*\)/g,
+];
+const RESOLVABLE_EXTENSIONS = ['ts', 'tsx', 'js', 'jsx', 'mjs', 'cjs'];
+function extractImportSpecifiers(content) {
+    const imports = [];
+    for (const regex of IMPORT_REGEXES) {
+        regex.lastIndex = 0;
+        let match = regex.exec(content);
+        while (match) {
+            const specifier = match[1];
+            if (specifier) {
+                imports.push(specifier);
+            }
+            match = regex.exec(content);
+        }
+    }
+    return Array.from(new Set(imports));
+}
+function listCandidateFiles(appRoot, cfg) {
+    const files = new Set();
+    for (const pattern of cfg.filePatterns) {
+        const matches = globSync(pattern, {
+            cwd: appRoot,
+            nodir: true,
+            ignore: cfg.excludePatterns,
+        });
+        for (const match of matches) {
+            files.add(normalizePath(match));
+        }
+    }
+    return Array.from(files);
+}
+function resolveWithCandidates(candidateBase, fileSet) {
+    if (fileSet.has(candidateBase)) {
+        return candidateBase;
+    }
+    for (const ext of RESOLVABLE_EXTENSIONS) {
+        const withExt = `${candidateBase}.${ext}`;
+        if (fileSet.has(withExt)) {
+            return withExt;
+        }
+    }
+    for (const ext of RESOLVABLE_EXTENSIONS) {
+        const indexPath = `${candidateBase}/index.${ext}`;
+        if (fileSet.has(indexPath)) {
+            return indexPath;
+        }
+    }
+    return null;
+}
+function expandAliasTarget(pattern, target, specifier) {
+    const normalizedPattern = normalizePath(pattern);
+    const normalizedTarget = normalizePath(target);
+    const normalizedSpecifier = normalizePath(specifier);
+    if (normalizedPattern.endsWith('/*')) {
+        const prefix = normalizedPattern.slice(0, -2);
+        if (normalizedSpecifier === prefix || normalizedSpecifier.startsWith(`${prefix}/`)) {
+            const suffix = normalizedSpecifier === prefix ? '' : normalizedSpecifier.slice(prefix.length + 1);
+            if (normalizedTarget.endsWith('/*')) {
+                const targetPrefix = normalizedTarget.slice(0, -2);
+                return suffix ? normalizePath(`${targetPrefix}/${suffix}`) : normalizePath(targetPrefix);
+            }
+            return normalizePath(normalizedTarget);
+        }
+        return null;
+    }
+    if (normalizedPattern === normalizedSpecifier) {
+        if (normalizedTarget.endsWith('/*')) {
+            return normalizePath(normalizedTarget.slice(0, -2));
+        }
+        return normalizePath(normalizedTarget);
+    }
+    return null;
+}
+function resolvePathAliasImport(specifier, fileSet, cfg) {
+    for (const [pattern, targets] of Object.entries(cfg.pathAliases)) {
+        for (const target of targets) {
+            const aliasPath = expandAliasTarget(pattern, target, specifier);
+            if (!aliasPath) {
+                continue;
+            }
+            const resolved = resolveWithCandidates(aliasPath, fileSet);
+            if (resolved) {
+                return resolved;
+            }
+        }
+    }
+    return null;
+}
+function resolveImport(fromFile, specifier, fileSet, cfg) {
+    if (specifier.startsWith('.')) {
+        const fromDir = dirname(fromFile);
+        const relativeCandidate = normalizePath(normalize(join(fromDir, specifier)));
+        return resolveWithCandidates(relativeCandidate, fileSet);
+    }
+    const aliasResolved = resolvePathAliasImport(specifier, fileSet, cfg);
+    if (aliasResolved) {
+        return aliasResolved;
+    }
+    for (const root of cfg.aliasRoots) {
+        const rootedCandidate = normalizePath(normalize(join(root, specifier)));
+        const resolved = resolveWithCandidates(rootedCandidate, fileSet);
+        if (resolved) {
+            return resolved;
+        }
+    }
+    const directCandidate = normalizePath(specifier);
+    return resolveWithCandidates(directCandidate, fileSet);
+}
+export function expandByDependencyGraph(appRoot, changedFiles, cfg) {
+    const warnings = [];
+    if (!cfg.enabled) {
+        return {
+            source: 'static-dependency-graph',
+            seedFiles: [],
+            impactedFiles: [],
+            expandedFiles: [],
+            analyzedFiles: 0,
+            analyzedEdges: 0,
+            maxDepth: 0,
+            truncated: false,
+            warnings,
+        };
+    }
+    const candidates = listCandidateFiles(appRoot, cfg);
+    const fileSet = new Set(candidates);
+    if (candidates.length === 0) {
+        warnings.push('Dependency graph found no candidate source files.');
+    }
+    const reverse = new Map();
+    let analyzedEdges = 0;
+    for (const file of candidates) {
+        const fullPath = join(appRoot, file);
+        const content = safeReadTextFile(fullPath);
+        if (!content) {
+            continue;
+        }
+        const imports = extractImportSpecifiers(content);
+        for (const specifier of imports) {
+            const resolved = resolveImport(file, specifier, fileSet, cfg);
+            if (!resolved) {
+                continue;
+            }
+            if (!reverse.has(resolved)) {
+                reverse.set(resolved, new Set());
+            }
+            reverse.get(resolved)?.add(file);
+            analyzedEdges += 1;
+        }
+    }
+    const seeds = Array.from(new Set(changedFiles
+        .map((file) => normalizePath(file))
+        .filter((file) => fileSet.has(file))));
+    if (seeds.length === 0) {
+        warnings.push('No changed files were found in dependency graph candidates.');
+        return {
+            source: 'static-dependency-graph',
+            seedFiles: [],
+            impactedFiles: [],
+            expandedFiles: [],
+            analyzedFiles: candidates.length,
+            analyzedEdges,
+            maxDepth: cfg.maxDepth,
+            truncated: false,
+            warnings,
+        };
+    }
+    const impacted = new Set(seeds);
+    const queue = seeds.map((file) => ({ file, depth: 0 }));
+    let truncated = false;
+    while (queue.length > 0) {
+        const next = queue.shift();
+        if (!next) {
+            continue;
+        }
+        if (next.depth >= cfg.maxDepth) {
+            continue;
+        }
+        const dependents = reverse.get(next.file);
+        if (!dependents) {
+            continue;
+        }
+        for (const dependent of dependents) {
+            if (impacted.has(dependent)) {
+                continue;
+            }
+            if (impacted.size - seeds.length >= cfg.maxExpandedFiles) {
+                truncated = true;
+                break;
+            }
+            impacted.add(dependent);
+            queue.push({ file: dependent, depth: next.depth + 1 });
+        }
+        if (truncated) {
+            break;
+        }
+    }
+    const impactedFiles = Array.from(impacted);
+    const expandedFiles = impactedFiles.filter((file) => !seeds.includes(file));
+    if (truncated) {
+        warnings.push(`Dependency expansion was truncated at maxExpandedFiles=${cfg.maxExpandedFiles}.`);
+    }
+    return {
+        source: 'static-dependency-graph',
+        seedFiles: seeds,
+        impactedFiles,
+        expandedFiles,
+        analyzedFiles: candidates.length,
+        analyzedEdges,
+        maxDepth: cfg.maxDepth,
+        truncated,
+        warnings,
+    };
+}

package/dist/esm/agent/feedback.js

@@ -0,0 +1,253 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { join } from 'path';
+import { inferSubsystemFromTestPath } from './test_path.js';
+function readJson(path) {
+    if (!existsSync(path)) {
+        return null;
+    }
+    try {
+        return JSON.parse(readFileSync(path, 'utf-8'));
+    }
+    catch {
+        return null;
+    }
+}
+function normalizeTestName(test) {
+    return test.replace(/ \(flags:.*\)$/, '').trim();
+}
+function asSet(values) {
+    return new Set(values.map(normalizeTestName).filter(Boolean));
+}
+function ratio(numerator, denominator) {
+    if (denominator <= 0) {
+        return 0;
+    }
+    return Number((numerator / denominator).toFixed(4));
+}
+function subsystemForTest(test) {
+    return inferSubsystemFromTestPath(test);
+}
+function parseTimestamp(value) {
+    const time = Date.parse(value);
+    if (Number.isNaN(time)) {
+        return null;
+    }
+    return time;
+}
+function filterRecent(entries, days) {
+    const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
+    return entries.filter((entry) => {
+        const time = parseTimestamp(entry.timestamp);
+        return time !== null && time >= cutoff;
+    });
+}
+function aggregateMetrics(entries) {
+    let truePositives = 0;
+    let recommendedTotal = 0;
+    let failuresTotal = 0;
+    let escapedTotal = 0;
+    for (const entry of entries) {
+        const recommended = asSet(entry.recommendedTests || []);
+        const failed = asSet(entry.failedTests || []);
+        const escaped = asSet(entry.escapedFailures || []);
+        const tp = Array.from(recommended).filter((test) => failed.has(test)).length;
+        truePositives += tp;
+        recommendedTotal += recommended.size;
+        failuresTotal += failed.size;
+        escapedTotal += escaped.size;
+    }
+    return {
+        precision: ratio(truePositives, recommendedTotal),
+        recall: ratio(truePositives, failuresTotal),
+        falseNegativeRate: ratio(escapedTotal, failuresTotal + escapedTotal),
+        samples: entries.length,
+    };
+}
+function aggregate(entries) {
+    const subsystemAcc = new Map();
+    for (const entry of entries) {
+        const recommended = asSet(entry.recommendedTests || []);
+        const failed = asSet(entry.failedTests || []);
+        const escaped = asSet(entry.escapedFailures || []);
+        const allSubsystems = new Set([
+            ...Array.from(recommended).map(subsystemForTest),
+            ...Array.from(failed).map(subsystemForTest),
+            ...Array.from(escaped).map(subsystemForTest),
+        ]);
+        for (const subsystem of allSubsystems) {
+            const bucket = subsystemAcc.get(subsystem) || { entries: [] };
+            bucket.entries.push({
+                ...entry,
+                recommendedTests: Array.from(recommended).filter((test) => subsystemForTest(test) === subsystem),
+                executedTests: (entry.executedTests || []).filter((test) => subsystemForTest(test) === subsystem),
+                failedTests: Array.from(failed).filter((test) => subsystemForTest(test) === subsystem),
+                escapedFailures: Array.from(escaped).filter((test) => subsystemForTest(test) === subsystem),
+            });
+            subsystemAcc.set(subsystem, bucket);
+        }
+    }
+    const bySubsystem = {};
+    for (const [subsystem, bucket] of subsystemAcc.entries()) {
+        const all = aggregateMetrics(bucket.entries);
+        const recent7d = aggregateMetrics(filterRecent(bucket.entries, 7));
+        const recent30d = aggregateMetrics(filterRecent(bucket.entries, 30));
+        bySubsystem[subsystem] = {
+            precision: all.precision,
+            recall: all.recall,
+            falseNegativeRate: all.falseNegativeRate,
+            samples: all.samples,
+            recent7d,
+            recent30d,
+        };
+    }
+    const overall = aggregateMetrics(entries);
+    const recent7d = aggregateMetrics(filterRecent(entries, 7));
+    const recent30d = aggregateMetrics(filterRecent(entries, 30));
+    return {
+        schemaVersion: '1.1.0',
+        generatedAt: new Date().toISOString(),
+        samples: entries.length,
+        overall,
+        recent7d,
+        recent30d,
+        bySubsystem,
+    };
+}
+function rateFor(runs, failed) {
+    return runs > 0 ? Number((failed / runs).toFixed(4)) : 0;
+}
+function trendFor(rate7d, rate30d) {
+    if (rate7d - rate30d >= 0.08) {
+        return 'up';
+    }
+    if (rate30d - rate7d >= 0.08) {
+        return 'down';
+    }
+    return 'stable';
+}
+function loadOwners(appRoot) {
+    const path = join(appRoot, '.e2e-ai-agents', 'subsystem-owners.json');
+    const manifest = readJson(path);
+    if (!manifest) {
+        return {};
+    }
+    if (manifest.ownersBySubsystem) {
+        return manifest.ownersBySubsystem;
+    }
+    if (manifest.subsystems) {
+        return manifest.subsystems;
+    }
+    return {};
+}
+function daysSince(value) {
+    if (!value) {
+        return null;
+    }
+    const time = parseTimestamp(value);
+    if (time === null) {
+        return null;
+    }
+    return (Date.now() - time) / (24 * 60 * 60 * 1000);
+}
+function aggregateFlaky(entries, appRoot) {
+    const acc = new Map();
+    const cutoff7 = Date.now() - 7 * 24 * 60 * 60 * 1000;
+    const cutoff30 = Date.now() - 30 * 24 * 60 * 60 * 1000;
+    const ownersBySubsystem = loadOwners(appRoot);
+    for (const entry of entries) {
+        const executed = asSet(entry.executedTests || []);
+        const failed = asSet(entry.failedTests || []);
+        const time = parseTimestamp(entry.timestamp);
+        for (const test of executed) {
+            const bucket = acc.get(test) || {
+                runs: 0,
+                failed: 0,
+                runs7d: 0,
+                failed7d: 0,
+                runs30d: 0,
+                failed30d: 0,
+                lastFailureAt: undefined,
+            };
+            bucket.runs += 1;
+            if (time !== null && time >= cutoff7) {
+                bucket.runs7d += 1;
+            }
+            if (time !== null && time >= cutoff30) {
+                bucket.runs30d += 1;
+            }
+            if (failed.has(test)) {
+                bucket.failed += 1;
+                bucket.lastFailureAt = entry.timestamp;
+                if (time !== null && time >= cutoff7) {
+                    bucket.failed7d += 1;
+                }
+                if (time !== null && time >= cutoff30) {
+                    bucket.failed30d += 1;
+                }
+            }
+            acc.set(test, bucket);
+        }
+    }
+    const tests = Array.from(acc.entries())
+        .map(([test, bucket]) => {
+        const flakeRate = rateFor(bucket.runs, bucket.failed);
+        const flakeRate7d = rateFor(bucket.runs7d, bucket.failed7d);
+        const flakeRate30d = rateFor(bucket.runs30d, bucket.failed30d);
+        const trend = trendFor(flakeRate7d, flakeRate30d);
+        const quarantine = (bucket.runs30d >= 5 && flakeRate30d >= 0.35) || (bucket.runs >= 8 && flakeRate >= 0.4);
+        const daysFromLastFailure = daysSince(bucket.lastFailureAt);
+        const quarantineState = quarantine
+            ? (daysFromLastFailure !== null && daysFromLastFailure >= 14 && flakeRate7d <= 0.05 ? 'retire-candidate' : 'active')
+            : 'none';
+        const subsystem = subsystemForTest(test);
+        const owners = ownersBySubsystem[subsystem] || [];
+        return {
+            test,
+            subsystem,
+            owners,
+            flakeRate,
+            flakeRate7d,
+            flakeRate30d,
+            trend,
+            quarantine,
+            quarantineState,
+            lastFailureAt: bucket.lastFailureAt,
+            samples: bucket.runs,
+            samples7d: bucket.runs7d,
+            samples30d: bucket.runs30d,
+        };
+    })
+        .filter((entry) => entry.flakeRate > 0)
+        .sort((a, b) => (b.flakeRate30d || b.flakeRate) - (a.flakeRate30d || a.flakeRate));
+    return {
+        schemaVersion: '1.1.0',
+        generatedAt: new Date().toISOString(),
+        tests,
+    };
+}
+export function appendFeedbackAndRecompute(appRoot, input) {
+    const baseDir = join(appRoot, '.e2e-ai-agents');
+    mkdirSync(baseDir, { recursive: true });
+    const feedbackPath = join(baseDir, 'feedback.json');
+    const existing = readJson(feedbackPath) || { schemaVersion: '1.0.0', entries: [] };
+    existing.entries.push({
+        ...input,
+        recommendedTests: input.recommendedTests || [],
+        executedTests: input.executedTests || [],
+        failedTests: input.failedTests || [],
+        escapedFailures: input.escapedFailures || [],
+    });
+    writeFileSync(feedbackPath, JSON.stringify(existing, null, 2), 'utf-8');
+    const calibration = aggregate(existing.entries);
+    const calibrationPath = join(baseDir, 'calibration.json');
+    writeFileSync(calibrationPath, JSON.stringify(calibration, null, 2), 'utf-8');
+    const flaky = aggregateFlaky(existing.entries, appRoot);
+    const flakyPath = join(baseDir, 'flaky-tests.json');
+    writeFileSync(flakyPath, JSON.stringify(flaky, null, 2), 'utf-8');
+    return { feedbackPath, calibrationPath, calibration };
+}
+export function readCalibration(appRoot) {
+    return readJson(join(appRoot, '.e2e-ai-agents', 'calibration.json'));
+}

package/dist/esm/agent/flags.js

@@ -0,0 +1,160 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+const ROLE_ORDER = [
+    'system_admin',
+    'team_admin',
+    'channel_admin',
+    'member',
+    'guest',
+    'deactivated',
+];
+const FEATURE_FLAG_REGEX = /\bFeatureFlags?\.(\w+)\b/g;
+const FEATURE_FLAG_STRING_REGEX = /\b(?:isFeatureEnabled|getFeatureFlag|featureFlag)\s*\(\s*['"]([^'"]+)['"]\s*\)/g;
+const SERVICE_SETTINGS_REGEX = /\bServiceSettings\.(\w+)\b/g;
+const TEST_GATE_REGEX = /\bskipIfFeatureFlagNotSet\s*\(\s*['"]([^'"]+)['"]\s*\)/g;
+const ROLE_ALIASES = {
+    'system admin': 'system_admin',
+    'system_admin': 'system_admin',
+    sysadmin: 'system_admin',
+    'team admin': 'team_admin',
+    'team_admin': 'team_admin',
+    'channel admin': 'channel_admin',
+    'channel_admin': 'channel_admin',
+    member: 'member',
+    members: 'member',
+    guest: 'guest',
+    guests: 'guest',
+    deactivated: 'deactivated',
+    inactive: 'deactivated',
+    disabled: 'deactivated',
+};
+export function normalizeRole(role) {
+    const key = role.trim().toLowerCase();
+    return ROLE_ALIASES[key] ?? null;
+}
+export function normalizeRoles(roles, fallback) {
+    const normalized = roles
+        .map((role) => normalizeRole(role))
+        .filter((role) => Boolean(role));
+    const combined = normalized.length > 0 ? normalized : fallback;
+    const unique = new Set(combined);
+    return ROLE_ORDER.filter((role) => unique.has(role));
+}
+export function normalizeFlagSource(source) {
+    if (!source) {
+        return 'featureFlag';
+    }
+    const value = source.trim().toLowerCase();
+    if (['feature', 'featureflag', 'feature_flag', 'flag'].includes(value)) {
+        return 'featureFlag';
+    }
+    if (['config', 'service', 'servicesettings', 'server'].includes(value)) {
+        return 'configFlag';
+    }
+    if (['test', 'gate', 'testgate'].includes(value)) {
+        return 'testGate';
+    }
+    return 'featureFlag';
+}
+export function normalizeFlagState(value, fallback) {
+    if (!value) {
+        return fallback;
+    }
+    const lowered = value.trim().toLowerCase();
+    if (lowered === 'on' || lowered === 'off' || lowered === 'unknown') {
+        return lowered;
+    }
+    return fallback;
+}
+export function mergeFlags(flags, defaultState) {
+    const map = new Map();
+    for (const flag of flags) {
+        const key = `${flag.source}:${flag.name.toLowerCase()}`;
+        if (!map.has(key)) {
+            map.set(key, {
+                ...flag,
+                defaultState: flag.defaultState ?? defaultState,
+            });
+        }
+    }
+    return Array.from(map.values());
+}
+export function extractFlagHits(content, config) {
+    if (!content) {
+        return [];
+    }
+    const hits = [];
+    const defaultState = config.flags.defaultState;
+    for (const match of content.matchAll(FEATURE_FLAG_REGEX)) {
+        if (match[1]) {
+            hits.push({ name: match[1], source: 'featureFlag', defaultState });
+        }
+    }
+    for (const match of content.matchAll(FEATURE_FLAG_STRING_REGEX)) {
+        if (match[1]) {
+            hits.push({ name: match[1], source: 'featureFlag', defaultState });
+        }
+    }
+    for (const match of content.matchAll(SERVICE_SETTINGS_REGEX)) {
+        if (match[1]) {
+            hits.push({ name: match[1], source: 'configFlag', defaultState });
+        }
+    }
+    for (const match of content.matchAll(TEST_GATE_REGEX)) {
+        if (match[1]) {
+            hits.push({ name: match[1], source: 'testGate', defaultState });
+        }
+    }
+    return mergeFlags(hits, defaultState);
+}
+export function inferAudienceFromPath(relativePath, config) {
+    const normalized = relativePath.toLowerCase();
+    if (normalized.includes('admin_console') || normalized.includes('system_console')) {
+        return normalizeRoles(['system_admin'], config.audience.defaultRoles);
+    }
+    if (normalized.includes('team') && normalized.includes('admin')) {
+        return normalizeRoles(['team_admin'], config.audience.defaultRoles);
+    }
+    if (normalized.includes('channel') && normalized.includes('admin')) {
+        return normalizeRoles(['channel_admin'], config.audience.defaultRoles);
+    }
+    return normalizeRoles(config.audience.defaultRoles, config.audience.defaultRoles);
+}
+export function formatFlags(flags) {
+    if (flags.length === 0) {
+        return 'none';
+    }
+    return flags.map((flag) => `${flag.name} (${flag.defaultState})`).join(', ');
+}
+export function computeBlastRadius(audience, flags, config) {
+    const normalizedAudience = normalizeRoles(audience, config.audience.defaultRoles);
+    const normalizedFlags = mergeFlags(flags, config.flags.defaultState);
+    const hasMember = normalizedAudience.includes('member');
+    const hasGuest = normalizedAudience.includes('guest');
+    const hasAdmin = normalizedAudience.some((role) => role === 'system_admin' || role === 'team_admin' || role === 'channel_admin');
+    const scope = hasMember || hasGuest ? 'broad' : hasAdmin ? 'admin-only' : 'unknown';
+    const flagState = normalizedFlags.length === 0
+        ? 'unflagged'
+        : normalizedFlags.some((flag) => flag.defaultState === 'off')
+            ? 'flagged-off'
+            : 'flagged-on';
+    let scoreDelta = 0;
+    if (hasMember) {
+        scoreDelta += config.blastRadius.memberBonus;
+    }
+    if (hasGuest) {
+        scoreDelta += config.blastRadius.guestBonus;
+    }
+    if (!hasMember && !hasGuest) {
+        scoreDelta += config.blastRadius.adminOnlyPenalty;
+    }
+    if (normalizedFlags.some((flag) => flag.defaultState === 'off')) {
+        scoreDelta += config.blastRadius.flagOffPenalty;
+    }
+    return {
+        audience: normalizedAudience,
+        flags: normalizedFlags,
+        summary: `${scope}; ${flagState}`,
+        scoreDelta,
+    };
+}