@yasserkhanorg/impact-gate 2.0.0

package/dist/esm/qa-agent/cli.js

@@ -0,0 +1,205 @@
+#!/usr/bin/env node
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { resolve, sep } from 'path';
+import { runQAAgent } from './orchestrator.js';
+const MODES = new Set(['pr', 'hunt', 'fix', 'release']);
+const KNOWN_FLAGS = new Set([
+    '--base-url', '--since', '--phase', '--time', '--budget',
+    '--headed', '--tests-root', '--project', '--output', '--help', '-h',
+]);
+function printUsage() {
+    console.log(`
+Usage: impact-gate-qa <mode> [options]
+
+Modes:
+  pr        Test changed features from a PR
+  hunt      Deep-dive into a specific area
+  fix       Verify healed tests and side effects
+  release   Full regression + release readiness verdict
+
+Options:
+  --base-url <url>      Application URL (required)
+  --since <ref>         Git ref for diff (default: origin/main)
+  --phase <1|2|3>       Run only up to this phase
+  --time <minutes>      Time limit (default: 15)
+  --budget <usd>        LLM budget in USD (default: 2.00)
+  --headed              Run browser in headed mode
+  --tests-root <path>   Path to tests directory
+  --project <name>      Playwright project name
+  --output <dir>        Output directory (default: .e2e-ai-agents)
+  --help                Show this help
+
+Examples:
+  impact-gate-qa pr --since origin/main --base-url http://localhost:8065
+  impact-gate-qa hunt "channel settings" --base-url http://localhost:8065
+  impact-gate-qa release --base-url http://localhost:8065 --time 30
+  impact-gate-qa fix --base-url http://localhost:8065
+`);
+}
+function parseCliArgs(argv) {
+    if (argv.length === 0 || argv.includes('--help') || argv.includes('-h')) {
+        printUsage();
+        return null;
+    }
+    const modeArg = argv[0];
+    if (!MODES.has(modeArg)) {
+        console.error(`Unknown mode: ${modeArg}`);
+        printUsage();
+        return null;
+    }
+    const mode = modeArg;
+    let baseUrl = '';
+    let since;
+    let huntTarget;
+    let phase;
+    let timeLimitMinutes = mode === 'release' ? 30 : 15;
+    let budgetUSD = 2.0;
+    let headed = false;
+    let testsRoot;
+    let project;
+    let outputDir;
+    // For hunt mode, the second positional arg is the target
+    let startFlags = 1;
+    if (mode === 'hunt' && argv[1] && !argv[1].startsWith('--')) {
+        huntTarget = argv[1];
+        startFlags = 2;
+    }
+    for (let i = startFlags; i < argv.length; i++) {
+        const arg = argv[i];
+        const next = argv[i + 1];
+        switch (arg) {
+            case '--base-url':
+                baseUrl = next || '';
+                i++;
+                break;
+            case '--since':
+                since = next;
+                i++;
+                break;
+            case '--phase': {
+                const parsed = parseInt(next || '0', 10);
+                if (parsed !== 1 && parsed !== 2 && parsed !== 3) {
+                    console.error(`Error: --phase must be 1, 2, or 3 (got "${next}")`);
+                    process.exit(1);
+                }
+                phase = parsed;
+                i++;
+                break;
+            }
+            case '--time': {
+                const parsed = parseInt(next || '15', 10);
+                if (!Number.isFinite(parsed) || parsed <= 0) {
+                    console.error(`Error: --time must be a positive number (got "${next}")`);
+                    process.exit(1);
+                }
+                timeLimitMinutes = parsed;
+                i++;
+                break;
+            }
+            case '--budget': {
+                const parsed = parseFloat(next || '2.0');
+                if (!Number.isFinite(parsed) || parsed <= 0) {
+                    console.error(`Error: --budget must be a positive number (got "${next}")`);
+                    process.exit(1);
+                }
+                budgetUSD = parsed;
+                i++;
+                break;
+            }
+            case '--headed':
+                headed = true;
+                break;
+            case '--tests-root':
+                testsRoot = next;
+                i++;
+                break;
+            case '--project':
+                project = next;
+                i++;
+                break;
+            case '--output':
+                outputDir = next;
+                i++;
+                break;
+            default:
+                if (arg.startsWith('--') && !KNOWN_FLAGS.has(arg)) {
+                    console.error(`Warning: unknown flag "${arg}" (ignored)`);
+                }
+                break;
+        }
+    }
+    // Validate --since and hunt target against flag injection (must not start with -)
+    if (since && since.startsWith('-')) {
+        console.error(`Error: --since value "${since}" looks like a flag, not a git ref`);
+        process.exit(1);
+    }
+    if (huntTarget && huntTarget.startsWith('-')) {
+        console.error(`Error: hunt target "${huntTarget}" looks like a flag`);
+        process.exit(1);
+    }
+    if (!baseUrl) {
+        console.error('Error: --base-url is required');
+        process.exit(1);
+    }
+    // Validate baseUrl is a proper HTTP(S) URL
+    try {
+        const parsed = new URL(baseUrl);
+        if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+            console.error(`Error: --base-url must use http or https (got "${parsed.protocol}")`);
+            process.exit(1);
+        }
+        // Normalize: remove trailing slash
+        baseUrl = parsed.origin + parsed.pathname.replace(/\/+$/, '');
+    }
+    catch {
+        console.error(`Error: --base-url is not a valid URL ("${baseUrl}")`);
+        process.exit(1);
+    }
+    // Validate --output stays within project directory
+    if (outputDir) {
+        const resolved = resolve(outputDir);
+        const cwd = process.cwd();
+        const normalizedCwd = cwd.endsWith(sep) ? cwd : cwd + sep;
+        if (resolved !== cwd && !resolved.startsWith(normalizedCwd)) {
+            console.error(`Error: --output "${outputDir}" resolves outside the project directory`);
+            process.exit(1);
+        }
+    }
+    return {
+        mode,
+        baseUrl,
+        since: since || 'origin/main',
+        huntTarget,
+        phase,
+        timeLimitMinutes,
+        budgetUSD,
+        headed,
+        testsRoot,
+        project,
+        outputDir,
+    };
+}
+async function main() {
+    const config = parseCliArgs(process.argv.slice(2));
+    if (!config) {
+        process.exit(0);
+    }
+    const report = await runQAAgent(config);
+    // Exit code based on verdict
+    switch (report.verdict.decision) {
+        case 'go':
+            process.exit(0);
+            break;
+        case 'conditional':
+            process.exit(1);
+            break;
+        case 'no-go':
+            process.exit(2);
+            break;
+    }
+}
+main().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exit(1);
+});

package/dist/esm/qa-agent/orchestrator.js

@@ -0,0 +1,120 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { execFileSync } from 'child_process';
+import { mkdirSync } from 'fs';
+import { logger } from '../logger.js';
+import { runPhase1 } from './phase1/runner.js';
+import { runAgentLoop } from './phase2/agent_loop.js';
+import { computeVerdict } from './phase3/verdict.js';
+import { generateReport } from './phase3/reporter.js';
+import { generateSpecsForFindings } from './phase3/spec_generator.js';
+import { submitFeedback } from './phase3/feedback.js';
+function emptyPhase2Result() {
+    return { findings: [], flowsExplored: [], actionsCount: 0, tokensUsed: 0, costUSD: 0, durationMs: 0 };
+}
+export async function runQAAgent(inputConfig) {
+    const outputDir = inputConfig.outputDir || '.e2e-ai-agents';
+    const screenshotDir = inputConfig.screenshotDir || `${outputDir}/qa-screenshots`;
+    mkdirSync(screenshotDir, { recursive: true });
+    const config = { ...inputConfig, outputDir, screenshotDir };
+    // -----------------------------------------------------------------------
+    // Phase 1: Scripted (scope resolution + run matched specs)
+    // -----------------------------------------------------------------------
+    logger.info('=== Phase 1: Scope & Scripted Tests ===');
+    let phase1;
+    if (config.phase && config.phase > 1) {
+        // Skip Phase 1 — provide empty results
+        phase1 = { flows: [], specResults: [] };
+    }
+    else {
+        phase1 = runPhase1(config);
+    }
+    if (phase1.flows.length === 0 && phase1.specResults.length === 0 && !(config.phase && config.phase > 1)) {
+        logger.warn('Phase 1 produced no flows and no spec results — scoping may have failed. Check that route-families.json and plan.json are available.');
+    }
+    logger.info('Phase 1 complete', {
+        flows: phase1.flows.length,
+        specResults: phase1.specResults.length,
+    });
+    if (config.phase === 1) {
+        return earlyReturn(config, phase1);
+    }
+    // -----------------------------------------------------------------------
+    // Phase 2: Autonomous exploration (LLM + agent-browser)
+    // -----------------------------------------------------------------------
+    logger.info('=== Phase 2: Autonomous Exploration ===');
+    // Verify agent-browser is available before starting the exploration loop
+    if (!(config.phase && config.phase > 2)) {
+        try {
+            execFileSync('agent-browser', ['--version'], { encoding: 'utf-8', timeout: 5000 });
+        }
+        catch {
+            logger.error('agent-browser CLI not found. Install it (>= 0.18.0) or skip Phase 2 with --phase 1.');
+            return earlyReturn(config, phase1);
+        }
+    }
+    let phase2;
+    if (config.phase && config.phase > 2) {
+        phase2 = emptyPhase2Result();
+    }
+    else {
+        const flows = phase1.flows.length > 0
+            ? phase1.flows
+            : [{ id: 'main', name: 'Main application', priority: 'P1' }];
+        // In fix mode, limit Phase 2 to verification only
+        const phase2Config = config.mode === 'fix'
+            ? { ...config, timeLimitMinutes: Math.min(config.timeLimitMinutes ?? 15, 5) }
+            : config;
+        phase2 = await runAgentLoop(phase2Config, flows);
+    }
+    logger.info('Phase 2 complete', {
+        findings: phase2.findings.length,
+        flowsExplored: phase2.flowsExplored.length,
+        cost: `$${phase2.costUSD.toFixed(4)}`,
+    });
+    if (config.phase === 2) {
+        return earlyReturn(config, phase1, phase2);
+    }
+    // -----------------------------------------------------------------------
+    // Phase 3: Report + Spec Generation + Verdict
+    // -----------------------------------------------------------------------
+    logger.info('=== Phase 3: Report & Verdict ===');
+    // Generate specs for discovered bugs
+    const generatedSpecs = generateSpecsForFindings(phase2.findings, config);
+    // Compute verdict
+    const verdict = computeVerdict(phase1, phase2);
+    // Generate report
+    const phase3 = generateReport(config, phase1, phase2, verdict, generatedSpecs);
+    // Submit feedback
+    try {
+        submitFeedback(config);
+    }
+    catch (err) {
+        logger.warn('Feedback submission failed', { error: String(err) });
+    }
+    logger.info(`=== QA Agent Complete: ${verdict.decision.toUpperCase()} ===`);
+    logger.info(verdict.reason);
+    return buildQAReport(config, phase1, phase2, phase3, verdict);
+}
+function earlyReturn(config, phase1, phase2) {
+    const p2 = phase2 || emptyPhase2Result();
+    const verdict = computeVerdict(phase1, p2);
+    const phase3 = generateReport(config, phase1, p2, verdict, []);
+    return buildQAReport(config, phase1, p2, phase3, verdict);
+}
+function buildQAReport(config, phase1, phase2, phase3, verdict) {
+    return {
+        schemaVersion: '1.0.0',
+        generatedAt: new Date().toISOString(),
+        mode: config.mode,
+        config: {
+            baseUrl: config.baseUrl,
+            timeLimitMinutes: config.timeLimitMinutes,
+            budgetUSD: config.budgetUSD,
+        },
+        phase1,
+        phase2,
+        phase3,
+        verdict,
+    };
+}

package/dist/esm/qa-agent/phase1/runner.js

@@ -0,0 +1,139 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { spawnSync } from 'child_process';
+import { existsSync, readdirSync } from 'fs';
+import { join } from 'path';
+import { logger } from '../../logger.js';
+import { safeEnv } from '../safe_env.js';
+import { resolveScope } from './scope.js';
+export function runPhase1(config) {
+    const { flows, specPaths } = resolveScope(config);
+    logger.info('Phase 1: Scope resolved', {
+        flows: flows.length,
+        specDirs: specPaths.length,
+        mode: config.mode,
+    });
+    // Run impact-gate CLI for impact/plan if we have a since ref
+    if (config.since && config.mode !== 'release') {
+        runE2eAgentsCli(config);
+    }
+    // Run matched Playwright specs
+    const specResults = runMatchedSpecs(specPaths, config);
+    return {
+        flows,
+        specResults,
+        planPath: config.testsRoot
+            ? join(config.testsRoot, '.e2e-ai-agents', 'plan.json')
+            : undefined,
+    };
+}
+function runE2eAgentsCli(config) {
+    const args = ['impact-gate'];
+    switch (config.mode) {
+        case 'pr':
+            args.push('plan');
+            if (config.since)
+                args.push('--since', config.since);
+            break;
+        case 'hunt':
+            args.push('impact');
+            if (config.huntTarget)
+                args.push('--flow-patterns', config.huntTarget);
+            if (config.since)
+                args.push('--since', config.since);
+            break;
+        case 'fix':
+            args.push('heal');
+            break;
+        default:
+            return;
+    }
+    if (config.testsRoot) {
+        args.push('--tests-root', config.testsRoot);
+    }
+    logger.info('Running impact-gate', { args: args.slice(1) });
+    const result = spawnSync('npx', args, {
+        cwd: config.testsRoot || process.cwd(),
+        encoding: 'utf-8',
+        timeout: 120000,
+        maxBuffer: 2 * 1024 * 1024,
+        env: safeEnv(),
+    });
+    // Exit code 2 = "no changes detected" from impact-gate CLI, not an error
+    if (result.status !== 0 && result.status !== 2) {
+        logger.warn('impact-gate exited with non-zero status', {
+            status: result.status,
+            stderr: (result.stderr || '').slice(0, 500),
+        });
+    }
+}
+function runMatchedSpecs(specPaths, config) {
+    const results = [];
+    const specFiles = collectSpecFiles(specPaths);
+    if (specFiles.length === 0) {
+        logger.info('No spec files found to run');
+        return results;
+    }
+    logger.info('Running matched specs', { count: specFiles.length });
+    for (const specFile of specFiles) {
+        const result = runSingleSpec(specFile, config);
+        results.push(result);
+    }
+    return results;
+}
+function collectSpecFiles(specPaths) {
+    const files = [];
+    for (const p of specPaths) {
+        if (!existsSync(p))
+            continue;
+        try {
+            const entries = readdirSync(p, { recursive: true, encoding: 'utf-8' });
+            for (const entry of entries) {
+                if (typeof entry === 'string' && (entry.endsWith('.spec.ts') || entry.endsWith('.test.ts'))) {
+                    files.push(join(p, entry));
+                }
+            }
+        }
+        catch {
+            // Skip unreadable directories
+        }
+    }
+    return files;
+}
+function runSingleSpec(specPath, config) {
+    const args = [
+        'playwright', 'test',
+        specPath,
+        '--reporter', 'json',
+    ];
+    if (config.project) {
+        args.push('--project', config.project);
+    }
+    const result = spawnSync('npx', args, {
+        cwd: config.testsRoot || process.cwd(),
+        encoding: 'utf-8',
+        timeout: 120000,
+        maxBuffer: 2 * 1024 * 1024,
+        env: safeEnv(config.baseUrl ? { BASE_URL: config.baseUrl } : {}),
+    });
+    // Try to parse JSON output
+    try {
+        const report = JSON.parse(result.stdout || '{}');
+        return {
+            specPath,
+            passed: report.stats?.expected || 0,
+            failed: report.stats?.unexpected || 0,
+            flaky: report.stats?.flaky || 0,
+            skipped: report.stats?.skipped || 0,
+        };
+    }
+    catch {
+        return {
+            specPath,
+            passed: result.status === 0 ? 1 : 0,
+            failed: result.status === 0 ? 0 : 1,
+            flaky: 0,
+            skipped: 0,
+        };
+    }
+}

package/dist/esm/qa-agent/phase1/scope.js

@@ -0,0 +1,126 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { existsSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { loadRouteFamilyManifest } from '../../knowledge/route_families.js';
+export function resolveScope(config) {
+    const testsRoot = config.testsRoot || process.cwd();
+    const planPath = join(testsRoot, '.e2e-ai-agents', 'plan.json');
+    // Try to read plan.json (written by impact-gate plan command)
+    const plan = readPlan(planPath);
+    const manifest = loadRouteFamilyManifest(testsRoot, {});
+    const flows = [];
+    const specPaths = [];
+    if (config.mode === 'hunt' && config.huntTarget) {
+        return resolveHuntScope(config.huntTarget, manifest, testsRoot);
+    }
+    if (config.mode === 'release') {
+        return resolveReleaseScope(manifest, testsRoot);
+    }
+    // PR / fix mode: use plan.json flows
+    if (plan) {
+        const allFlows = [
+            ...(plan.flows || []),
+            ...(plan.gaps || []).map((g) => ({ id: g.flowId, name: g.flowName, priority: g.priority })),
+        ];
+        for (const f of allFlows) {
+            const family = manifest?.families.find((fam) => fam.id === f.id);
+            const url = resolveUrlForFamily(family);
+            flows.push({
+                id: f.id,
+                name: f.name,
+                priority: f.priority || 'P1',
+                url,
+            });
+        }
+        // Collect spec paths from covered flows
+        for (const c of plan.coveredFlows || []) {
+            if (c.specDirs) {
+                for (const dir of c.specDirs) {
+                    const fullDir = join(testsRoot, dir);
+                    if (existsSync(fullDir)) {
+                        specPaths.push(fullDir);
+                    }
+                }
+            }
+        }
+    }
+    // Sort by priority: P0 first
+    flows.sort((a, b) => a.priority.localeCompare(b.priority));
+    return { flows, specPaths };
+}
+function resolveHuntScope(target, manifest, testsRoot) {
+    const flows = [];
+    const specPaths = [];
+    const targetLower = target.toLowerCase();
+    if (manifest) {
+        for (const family of manifest.families) {
+            const matches = family.id.toLowerCase().includes(targetLower) ||
+                (family.userFlows || []).some((uf) => uf.toLowerCase().includes(targetLower));
+            if (matches) {
+                flows.push({
+                    id: family.id,
+                    name: family.id,
+                    priority: family.priority || 'P1',
+                    url: resolveUrlForFamily(family),
+                });
+                for (const dir of family.specDirs || []) {
+                    const fullDir = join(testsRoot, dir);
+                    if (existsSync(fullDir)) {
+                        specPaths.push(fullDir);
+                    }
+                }
+            }
+        }
+    }
+    // If no manifest matches, create a generic flow
+    if (flows.length === 0) {
+        flows.push({ id: target, name: target, priority: 'P1' });
+    }
+    return { flows, specPaths };
+}
+function resolveReleaseScope(manifest, testsRoot) {
+    const flows = [];
+    const specPaths = [];
+    if (manifest) {
+        for (const family of manifest.families) {
+            if (family.priority === 'P0' || family.priority === 'P1') {
+                flows.push({
+                    id: family.id,
+                    name: family.id,
+                    priority: family.priority,
+                    url: resolveUrlForFamily(family),
+                });
+                for (const dir of family.specDirs || []) {
+                    const fullDir = join(testsRoot, dir);
+                    if (existsSync(fullDir)) {
+                        specPaths.push(fullDir);
+                    }
+                }
+            }
+        }
+    }
+    flows.sort((a, b) => a.priority.localeCompare(b.priority));
+    return { flows, specPaths };
+}
+function resolveUrlForFamily(family) {
+    if (!family || !family.routes || family.routes.length === 0)
+        return undefined;
+    // Take the first route pattern and substitute common placeholders
+    const route = family.routes[0];
+    return route
+        .replace(/\{team\}/g, 'default')
+        .replace(/\{channel\}/g, 'town-square')
+        .replace(/\{user_id\}/g, 'me')
+        .replace(/\{[^}]+\}/g, 'test');
+}
+function readPlan(path) {
+    if (!existsSync(path))
+        return null;
+    try {
+        return JSON.parse(readFileSync(path, 'utf-8'));
+    }
+    catch {
+        return null;
+    }
+}

package/dist/esm/qa-agent/phase2/agent_browser.js

@@ -0,0 +1,95 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { execFileSync } from 'child_process';
+const COMMAND = 'agent-browser';
+const TIMEOUT_MS = 30000;
+const MAX_OUTPUT = 512 * 1024; // 512 KB
+function run(args, timeoutMs = TIMEOUT_MS) {
+    const result = execFileSync(COMMAND, args, {
+        encoding: 'utf-8',
+        timeout: timeoutMs,
+        maxBuffer: MAX_OUTPUT,
+    });
+    return result.trim();
+}
+/**
+ * Thin wrapper around the `agent-browser` CLI.
+ *
+ * Every method calls execFileSync (array form — no shell injection) and
+ * returns the stdout string.  Session persistence is handled by
+ * agent-browser's daemon; the browser stays open between calls.
+ */
+export class AgentBrowser {
+    constructor(options) {
+        this.session = options?.session;
+    }
+    args(base) {
+        if (this.session) {
+            return [...base, '--session', this.session];
+        }
+        return base;
+    }
+    open(url) {
+        return run(this.args(['open', url]));
+    }
+    click(ref) {
+        return run(this.args(['click', ref]));
+    }
+    fill(ref, value) {
+        return run(this.args(['fill', ref, value]));
+    }
+    type(ref, value) {
+        return run(this.args(['type', ref, value]));
+    }
+    press(key) {
+        return run(this.args(['press', key]));
+    }
+    scroll(direction, ref) {
+        const scrollArgs = ['scroll', direction];
+        if (ref)
+            scrollArgs.push(ref);
+        return run(this.args(scrollArgs));
+    }
+    snapshot() {
+        return run(this.args(['snapshot', '-i']));
+    }
+    screenshot(path) {
+        const screenshotArgs = ['screenshot'];
+        if (path) {
+            screenshotArgs.push(path);
+        }
+        screenshotArgs.push('--annotate');
+        return run(this.args(screenshotArgs));
+    }
+    getUrl() {
+        return run(this.args(['get', 'url']));
+    }
+    getTitle() {
+        return run(this.args(['get', 'title']));
+    }
+    getText(ref) {
+        return run(this.args(['get', 'text', ref]));
+    }
+    /**
+     * Run a JS expression in the browser via agent-browser's evaluate command.
+     * SECURITY: Only used internally for console error capture. Do NOT expose to LLM tools.
+     * Uses execFileSync array form — expression is a CLI arg, NOT JS eval().
+     */
+    evaluateInternal(expression) {
+        return run(this.args(['evaluate', expression]));
+    }
+    back() {
+        return run(this.args(['back']));
+    }
+    forward() {
+        return run(this.args(['forward']));
+    }
+    close() {
+        try {
+            run(this.args(['close']), 5000);
+        }
+        catch {
+            // Ignore close errors — daemon may already be gone
+        }
+    }
+}