@yasserkhanorg/e2e-agents 1.2.2 → 1.3.0

package/dist/agent/feedback.d.ts

@@ -46,10 +46,30 @@ export interface CalibrationSummary {
         };
     }>;
 }
+export interface FlakySummary {
+    schemaVersion: '1.1.0';
+    generatedAt: string;
+    tests: Array<{
+        test: string;
+        subsystem: string;
+        owners: string[];
+        flakeRate: number;
+        flakeRate7d: number;
+        flakeRate30d: number;
+        trend: 'up' | 'down' | 'stable';
+        quarantine: boolean;
+        quarantineState: 'none' | 'active' | 'retire-candidate';
+        lastFailureAt?: string;
+        samples: number;
+        samples7d: number;
+        samples30d: number;
+    }>;
+}
 export declare function appendFeedbackAndRecompute(appRoot: string, input: RecommendationFeedbackEntry): {
     feedbackPath: string;
     calibrationPath: string;
     calibration: CalibrationSummary;
 };
 export declare function readCalibration(appRoot: string): CalibrationSummary | null;
+export declare function readFlakyTests(appRoot: string): FlakySummary | null;
 //# sourceMappingURL=feedback.d.ts.map

package/dist/agent/feedback.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"feedback.d.ts","sourceRoot":"","sources":["../../src/agent/feedback.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,2BAA2B;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,CAAC;IACtC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,kBAAkB;IAC/B,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE;QACL,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,EAAE,MAAM,CAAC;KAC7B,CAAC;IACF,QAAQ,EAAE;QACN,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,EAAE,MAAM,CAAC;QAC1B,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,SAAS,EAAE;QACP,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,EAAE,MAAM,CAAC;QAC1B,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,WAAW,EAAE,MAAM,CACnB,MAAM,EACN;QACI,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,EAAE,MAAM,CAAC;QAC1B,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE;YACN,SAAS,EAAE,MAAM,CAAC;YAClB,MAAM,EAAE,MAAM,CAAC;YACf,iBAAiB,EAAE,MAAM,CAAC;YAC1B,OAAO,EAAE,MAAM,CAAC;SACnB,CAAC;QACF,SAAS,EAAE;YACP,SAAS,EAAE,MAAM,CAAC;YAClB,MAAM,EAAE,MAAM,CAAC;YACf,iBAAiB,EAAE,MAAM,CAAC;YAC1B,OAAO,EAAE,MAAM,CAAC;SACnB,CAAC;KACL,CACA,CAAC;CACL;AAkSD,wBAAgB,0BAA0B,CACtC,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,2BAA2B,GACnC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,eAAe,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,kBAAkB,CAAA;CAAC,CAwBlF;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CAE1E"}
+{"version":3,"file":"feedback.d.ts","sourceRoot":"","sources":["../../src/agent/feedback.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,2BAA2B;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,CAAC;IACtC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,kBAAkB;IAC/B,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE;QACL,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,EAAE,MAAM,CAAC;KAC7B,CAAC;IACF,QAAQ,EAAE;QACN,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,EAAE,MAAM,CAAC;QAC1B,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,SAAS,EAAE;QACP,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,EAAE,MAAM,CAAC;QAC1B,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,WAAW,EAAE,MAAM,CACnB,MAAM,EACN;QACI,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,EAAE,MAAM,CAAC;QAC1B,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE;YACN,SAAS,EAAE,MAAM,CAAC;YAClB,MAAM,EAAE,MAAM,CAAC;YACf,iBAAiB,EAAE,MAAM,CAAC;YAC1B,OAAO,EAAE,MAAM,CAAC;SACnB,CAAC;QACF,SAAS,EAAE;YACP,SAAS,EAAE,MAAM,CAAC;YAClB,MAAM,EAAE,MAAM,CAAC;YACf,iBAAiB,EAAE,MAAM,CAAC;YAC1B,OAAO,EAAE,MAAM,CAAC;SACnB,CAAC;KACL,CACA,CAAC;CACL;AAOD,MAAM,WAAW,YAAY;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,KAAK,CAAC;QACT,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,IAAI,GAAG,MAAM,GAAG,QAAQ,CAAC;QAChC,UAAU,EAAE,OAAO,CAAC;QACpB,eAAe,EAAE,MAAM,GAAG,QAAQ,GAAG,kBAAkB,CAAC;QACxD,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;CACN;AAyQD,wBAAgB,0BAA0B,CACtC,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,2BAA2B,GACnC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,eAAe,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,kBAAkB,CAAA;CAAC,CAwBlF;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CAE1E;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CAEnE"}

package/dist/agent/feedback.js

@@ -4,6 +4,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.appendFeedbackAndRecompute = appendFeedbackAndRecompute;
 exports.readCalibration = readCalibration;
+exports.readFlakyTests = readFlakyTests;
 const fs_1 = require("fs");
 const path_1 = require("path");
 const test_path_js_1 = require("./test_path.js");
@@ -255,3 +256,6 @@ function appendFeedbackAndRecompute(appRoot, input) {
 function readCalibration(appRoot) {
     return readJson((0, path_1.join)(appRoot, '.e2e-ai-agents', 'calibration.json'));
 }
+function readFlakyTests(appRoot) {
+    return readJson((0, path_1.join)(appRoot, '.e2e-ai-agents', 'flaky-tests.json'));
+}

package/dist/esm/agent/feedback.js

@@ -251,3 +251,6 @@ export function appendFeedbackAndRecompute(appRoot, input) {
 export function readCalibration(appRoot) {
     return readJson(join(appRoot, '.e2e-ai-agents', 'calibration.json'));
 }
+export function readFlakyTests(appRoot) {
+    return readJson(join(appRoot, '.e2e-ai-agents', 'flaky-tests.json'));
+}

package/dist/esm/index.js

@@ -14,7 +14,7 @@ export { analyzeImpactDeterministic, recommendTestsDeterministic, handoffGenerat
 export { analyzeImpact as analyzeImpactV2, getGaps, getPartialGaps } from './engine/impact_engine.js';
 export { extractScenarios } from './engine/impact_engine.js';
 export { buildPlanFromImpact } from './engine/plan_builder.js';
-export { appendFeedbackAndRecompute, readCalibration } from './agent/feedback.js';
+export { appendFeedbackAndRecompute, readCalibration, readFlakyTests } from './agent/feedback.js';
 export { finalizeGeneratedTests } from './agent/handoff.js';
 export { ingestTraceabilityInput } from './agent/traceability_ingest.js';
 export { captureTraceabilityInput } from './agent/traceability_capture.js';

package/dist/esm/qa-agent/cli.js

@@ -0,0 +1,205 @@
+#!/usr/bin/env node
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { resolve, sep } from 'path';
+import { runQAAgent } from './orchestrator.js';
+const MODES = new Set(['pr', 'hunt', 'fix', 'release']);
+const KNOWN_FLAGS = new Set([
+    '--base-url', '--since', '--phase', '--time', '--budget',
+    '--headed', '--tests-root', '--project', '--output', '--help', '-h',
+]);
+function printUsage() {
+    console.log(`
+Usage: e2e-qa-agent <mode> [options]
+
+Modes:
+  pr        Test changed features from a PR
+  hunt      Deep-dive into a specific area
+  fix       Verify healed tests and side effects
+  release   Full regression + release readiness verdict
+
+Options:
+  --base-url <url>      Application URL (required)
+  --since <ref>         Git ref for diff (default: origin/main)
+  --phase <1|2|3>       Run only up to this phase
+  --time <minutes>      Time limit (default: 15)
+  --budget <usd>        LLM budget in USD (default: 2.00)
+  --headed              Run browser in headed mode
+  --tests-root <path>   Path to tests directory
+  --project <name>      Playwright project name
+  --output <dir>        Output directory (default: .e2e-ai-agents)
+  --help                Show this help
+
+Examples:
+  e2e-qa-agent pr --since origin/main --base-url http://localhost:8065
+  e2e-qa-agent hunt "channel settings" --base-url http://localhost:8065
+  e2e-qa-agent release --base-url http://localhost:8065 --time 30
+  e2e-qa-agent fix --base-url http://localhost:8065
+`);
+}
+function parseCliArgs(argv) {
+    if (argv.length === 0 || argv.includes('--help') || argv.includes('-h')) {
+        printUsage();
+        return null;
+    }
+    const modeArg = argv[0];
+    if (!MODES.has(modeArg)) {
+        console.error(`Unknown mode: ${modeArg}`);
+        printUsage();
+        return null;
+    }
+    const mode = modeArg;
+    let baseUrl = '';
+    let since;
+    let huntTarget;
+    let phase;
+    let timeLimitMinutes = mode === 'release' ? 30 : 15;
+    let budgetUSD = 2.0;
+    let headed = false;
+    let testsRoot;
+    let project;
+    let outputDir;
+    // For hunt mode, the second positional arg is the target
+    let startFlags = 1;
+    if (mode === 'hunt' && argv[1] && !argv[1].startsWith('--')) {
+        huntTarget = argv[1];
+        startFlags = 2;
+    }
+    for (let i = startFlags; i < argv.length; i++) {
+        const arg = argv[i];
+        const next = argv[i + 1];
+        switch (arg) {
+            case '--base-url':
+                baseUrl = next || '';
+                i++;
+                break;
+            case '--since':
+                since = next;
+                i++;
+                break;
+            case '--phase': {
+                const parsed = parseInt(next || '0', 10);
+                if (parsed !== 1 && parsed !== 2 && parsed !== 3) {
+                    console.error(`Error: --phase must be 1, 2, or 3 (got "${next}")`);
+                    process.exit(1);
+                }
+                phase = parsed;
+                i++;
+                break;
+            }
+            case '--time': {
+                const parsed = parseInt(next || '15', 10);
+                if (!Number.isFinite(parsed) || parsed <= 0) {
+                    console.error(`Error: --time must be a positive number (got "${next}")`);
+                    process.exit(1);
+                }
+                timeLimitMinutes = parsed;
+                i++;
+                break;
+            }
+            case '--budget': {
+                const parsed = parseFloat(next || '2.0');
+                if (!Number.isFinite(parsed) || parsed <= 0) {
+                    console.error(`Error: --budget must be a positive number (got "${next}")`);
+                    process.exit(1);
+                }
+                budgetUSD = parsed;
+                i++;
+                break;
+            }
+            case '--headed':
+                headed = true;
+                break;
+            case '--tests-root':
+                testsRoot = next;
+                i++;
+                break;
+            case '--project':
+                project = next;
+                i++;
+                break;
+            case '--output':
+                outputDir = next;
+                i++;
+                break;
+            default:
+                if (arg.startsWith('--') && !KNOWN_FLAGS.has(arg)) {
+                    console.error(`Warning: unknown flag "${arg}" (ignored)`);
+                }
+                break;
+        }
+    }
+    // Validate --since and hunt target against flag injection (must not start with -)
+    if (since && since.startsWith('-')) {
+        console.error(`Error: --since value "${since}" looks like a flag, not a git ref`);
+        process.exit(1);
+    }
+    if (huntTarget && huntTarget.startsWith('-')) {
+        console.error(`Error: hunt target "${huntTarget}" looks like a flag`);
+        process.exit(1);
+    }
+    if (!baseUrl) {
+        console.error('Error: --base-url is required');
+        process.exit(1);
+    }
+    // Validate baseUrl is a proper HTTP(S) URL
+    try {
+        const parsed = new URL(baseUrl);
+        if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+            console.error(`Error: --base-url must use http or https (got "${parsed.protocol}")`);
+            process.exit(1);
+        }
+        // Normalize: remove trailing slash
+        baseUrl = parsed.origin + parsed.pathname.replace(/\/+$/, '');
+    }
+    catch {
+        console.error(`Error: --base-url is not a valid URL ("${baseUrl}")`);
+        process.exit(1);
+    }
+    // Validate --output stays within project directory
+    if (outputDir) {
+        const resolved = resolve(outputDir);
+        const cwd = process.cwd();
+        const normalizedCwd = cwd.endsWith(sep) ? cwd : cwd + sep;
+        if (resolved !== cwd && !resolved.startsWith(normalizedCwd)) {
+            console.error(`Error: --output "${outputDir}" resolves outside the project directory`);
+            process.exit(1);
+        }
+    }
+    return {
+        mode,
+        baseUrl,
+        since: since || 'origin/main',
+        huntTarget,
+        phase,
+        timeLimitMinutes,
+        budgetUSD,
+        headed,
+        testsRoot,
+        project,
+        outputDir,
+    };
+}
+async function main() {
+    const config = parseCliArgs(process.argv.slice(2));
+    if (!config) {
+        process.exit(0);
+    }
+    const report = await runQAAgent(config);
+    // Exit code based on verdict
+    switch (report.verdict.decision) {
+        case 'go':
+            process.exit(0);
+            break;
+        case 'conditional':
+            process.exit(1);
+            break;
+        case 'no-go':
+            process.exit(2);
+            break;
+    }
+}
+main().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exit(1);
+});

package/dist/esm/qa-agent/orchestrator.js

@@ -0,0 +1,120 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { execFileSync } from 'child_process';
+import { mkdirSync } from 'fs';
+import { logger } from '../logger.js';
+import { runPhase1 } from './phase1/runner.js';
+import { runAgentLoop } from './phase2/agent_loop.js';
+import { computeVerdict } from './phase3/verdict.js';
+import { generateReport } from './phase3/reporter.js';
+import { generateSpecsForFindings } from './phase3/spec_generator.js';
+import { submitFeedback } from './phase3/feedback.js';
+function emptyPhase2Result() {
+    return { findings: [], flowsExplored: [], actionsCount: 0, tokensUsed: 0, costUSD: 0, durationMs: 0 };
+}
+export async function runQAAgent(inputConfig) {
+    const outputDir = inputConfig.outputDir || '.e2e-ai-agents';
+    const screenshotDir = inputConfig.screenshotDir || `${outputDir}/qa-screenshots`;
+    mkdirSync(screenshotDir, { recursive: true });
+    const config = { ...inputConfig, outputDir, screenshotDir };
+    // -----------------------------------------------------------------------
+    // Phase 1: Scripted (scope resolution + run matched specs)
+    // -----------------------------------------------------------------------
+    logger.info('=== Phase 1: Scope & Scripted Tests ===');
+    let phase1;
+    if (config.phase && config.phase > 1) {
+        // Skip Phase 1 — provide empty results
+        phase1 = { flows: [], specResults: [] };
+    }
+    else {
+        phase1 = runPhase1(config);
+    }
+    if (phase1.flows.length === 0 && phase1.specResults.length === 0 && !(config.phase && config.phase > 1)) {
+        logger.warn('Phase 1 produced no flows and no spec results — scoping may have failed. Check that route-families.json and plan.json are available.');
+    }
+    logger.info('Phase 1 complete', {
+        flows: phase1.flows.length,
+        specResults: phase1.specResults.length,
+    });
+    if (config.phase === 1) {
+        return earlyReturn(config, phase1);
+    }
+    // -----------------------------------------------------------------------
+    // Phase 2: Autonomous exploration (LLM + agent-browser)
+    // -----------------------------------------------------------------------
+    logger.info('=== Phase 2: Autonomous Exploration ===');
+    // Verify agent-browser is available before starting the exploration loop
+    if (!(config.phase && config.phase > 2)) {
+        try {
+            execFileSync('agent-browser', ['--version'], { encoding: 'utf-8', timeout: 5000 });
+        }
+        catch {
+            logger.error('agent-browser CLI not found. Install it (>= 0.18.0) or skip Phase 2 with --phase 1.');
+            return earlyReturn(config, phase1);
+        }
+    }
+    let phase2;
+    if (config.phase && config.phase > 2) {
+        phase2 = emptyPhase2Result();
+    }
+    else {
+        const flows = phase1.flows.length > 0
+            ? phase1.flows
+            : [{ id: 'main', name: 'Main application', priority: 'P1' }];
+        // In fix mode, limit Phase 2 to verification only
+        const phase2Config = config.mode === 'fix'
+            ? { ...config, timeLimitMinutes: Math.min(config.timeLimitMinutes ?? 15, 5) }
+            : config;
+        phase2 = await runAgentLoop(phase2Config, flows);
+    }
+    logger.info('Phase 2 complete', {
+        findings: phase2.findings.length,
+        flowsExplored: phase2.flowsExplored.length,
+        cost: `$${phase2.costUSD.toFixed(4)}`,
+    });
+    if (config.phase === 2) {
+        return earlyReturn(config, phase1, phase2);
+    }
+    // -----------------------------------------------------------------------
+    // Phase 3: Report + Spec Generation + Verdict
+    // -----------------------------------------------------------------------
+    logger.info('=== Phase 3: Report & Verdict ===');
+    // Generate specs for discovered bugs
+    const generatedSpecs = generateSpecsForFindings(phase2.findings, config);
+    // Compute verdict
+    const verdict = computeVerdict(phase1, phase2);
+    // Generate report
+    const phase3 = generateReport(config, phase1, phase2, verdict, generatedSpecs);
+    // Submit feedback
+    try {
+        submitFeedback(config);
+    }
+    catch (err) {
+        logger.warn('Feedback submission failed', { error: String(err) });
+    }
+    logger.info(`=== QA Agent Complete: ${verdict.decision.toUpperCase()} ===`);
+    logger.info(verdict.reason);
+    return buildQAReport(config, phase1, phase2, phase3, verdict);
+}
+function earlyReturn(config, phase1, phase2) {
+    const p2 = phase2 || emptyPhase2Result();
+    const verdict = computeVerdict(phase1, p2);
+    const phase3 = generateReport(config, phase1, p2, verdict, []);
+    return buildQAReport(config, phase1, p2, phase3, verdict);
+}
+function buildQAReport(config, phase1, phase2, phase3, verdict) {
+    return {
+        schemaVersion: '1.0.0',
+        generatedAt: new Date().toISOString(),
+        mode: config.mode,
+        config: {
+            baseUrl: config.baseUrl,
+            timeLimitMinutes: config.timeLimitMinutes,
+            budgetUSD: config.budgetUSD,
+        },
+        phase1,
+        phase2,
+        phase3,
+        verdict,
+    };
+}

package/dist/esm/qa-agent/phase1/runner.js

@@ -0,0 +1,139 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { spawnSync } from 'child_process';
+import { existsSync, readdirSync } from 'fs';
+import { join } from 'path';
+import { logger } from '../../logger.js';
+import { safeEnv } from '../safe_env.js';
+import { resolveScope } from './scope.js';
+export function runPhase1(config) {
+    const { flows, specPaths } = resolveScope(config);
+    logger.info('Phase 1: Scope resolved', {
+        flows: flows.length,
+        specDirs: specPaths.length,
+        mode: config.mode,
+    });
+    // Run e2e-agents CLI for impact/plan if we have a since ref
+    if (config.since && config.mode !== 'release') {
+        runE2eAgentsCli(config);
+    }
+    // Run matched Playwright specs
+    const specResults = runMatchedSpecs(specPaths, config);
+    return {
+        flows,
+        specResults,
+        planPath: config.testsRoot
+            ? join(config.testsRoot, '.e2e-ai-agents', 'plan.json')
+            : undefined,
+    };
+}
+function runE2eAgentsCli(config) {
+    const args = ['e2e-ai-agents'];
+    switch (config.mode) {
+        case 'pr':
+            args.push('plan');
+            if (config.since)
+                args.push('--since', config.since);
+            break;
+        case 'hunt':
+            args.push('impact');
+            if (config.huntTarget)
+                args.push('--flow-patterns', config.huntTarget);
+            if (config.since)
+                args.push('--since', config.since);
+            break;
+        case 'fix':
+            args.push('heal');
+            break;
+        default:
+            return;
+    }
+    if (config.testsRoot) {
+        args.push('--tests-root', config.testsRoot);
+    }
+    logger.info('Running e2e-ai-agents', { args: args.slice(1) });
+    const result = spawnSync('npx', args, {
+        cwd: config.testsRoot || process.cwd(),
+        encoding: 'utf-8',
+        timeout: 120000,
+        maxBuffer: 2 * 1024 * 1024,
+        env: safeEnv(),
+    });
+    // Exit code 2 = "no changes detected" from e2e-agents CLI, not an error
+    if (result.status !== 0 && result.status !== 2) {
+        logger.warn('e2e-agents exited with non-zero status', {
+            status: result.status,
+            stderr: (result.stderr || '').slice(0, 500),
+        });
+    }
+}
+function runMatchedSpecs(specPaths, config) {
+    const results = [];
+    const specFiles = collectSpecFiles(specPaths);
+    if (specFiles.length === 0) {
+        logger.info('No spec files found to run');
+        return results;
+    }
+    logger.info('Running matched specs', { count: specFiles.length });
+    for (const specFile of specFiles) {
+        const result = runSingleSpec(specFile, config);
+        results.push(result);
+    }
+    return results;
+}
+function collectSpecFiles(specPaths) {
+    const files = [];
+    for (const p of specPaths) {
+        if (!existsSync(p))
+            continue;
+        try {
+            const entries = readdirSync(p, { recursive: true, encoding: 'utf-8' });
+            for (const entry of entries) {
+                if (typeof entry === 'string' && (entry.endsWith('.spec.ts') || entry.endsWith('.test.ts'))) {
+                    files.push(join(p, entry));
+                }
+            }
+        }
+        catch {
+            // Skip unreadable directories
+        }
+    }
+    return files;
+}
+function runSingleSpec(specPath, config) {
+    const args = [
+        'playwright', 'test',
+        specPath,
+        '--reporter', 'json',
+    ];
+    if (config.project) {
+        args.push('--project', config.project);
+    }
+    const result = spawnSync('npx', args, {
+        cwd: config.testsRoot || process.cwd(),
+        encoding: 'utf-8',
+        timeout: 120000,
+        maxBuffer: 2 * 1024 * 1024,
+        env: safeEnv(config.baseUrl ? { BASE_URL: config.baseUrl } : {}),
+    });
+    // Try to parse JSON output
+    try {
+        const report = JSON.parse(result.stdout || '{}');
+        return {
+            specPath,
+            passed: report.stats?.expected || 0,
+            failed: report.stats?.unexpected || 0,
+            flaky: report.stats?.flaky || 0,
+            skipped: report.stats?.skipped || 0,
+        };
+    }
+    catch {
+        return {
+            specPath,
+            passed: result.status === 0 ? 1 : 0,
+            failed: result.status === 0 ? 0 : 1,
+            flaky: 0,
+            skipped: 0,
+        };
+    }
+}

package/dist/esm/qa-agent/phase1/scope.js

@@ -0,0 +1,126 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { existsSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { loadRouteFamilyManifest } from '../../knowledge/route_families.js';
+export function resolveScope(config) {
+    const testsRoot = config.testsRoot || process.cwd();
+    const planPath = join(testsRoot, '.e2e-ai-agents', 'plan.json');
+    // Try to read plan.json (written by e2e-agents plan command)
+    const plan = readPlan(planPath);
+    const manifest = loadRouteFamilyManifest(testsRoot, {});
+    const flows = [];
+    const specPaths = [];
+    if (config.mode === 'hunt' && config.huntTarget) {
+        return resolveHuntScope(config.huntTarget, manifest, testsRoot);
+    }
+    if (config.mode === 'release') {
+        return resolveReleaseScope(manifest, testsRoot);
+    }
+    // PR / fix mode: use plan.json flows
+    if (plan) {
+        const allFlows = [
+            ...(plan.flows || []),
+            ...(plan.gaps || []).map((g) => ({ id: g.flowId, name: g.flowName, priority: g.priority })),
+        ];
+        for (const f of allFlows) {
+            const family = manifest?.families.find((fam) => fam.id === f.id);
+            const url = resolveUrlForFamily(family);
+            flows.push({
+                id: f.id,
+                name: f.name,
+                priority: f.priority || 'P1',
+                url,
+            });
+        }
+        // Collect spec paths from covered flows
+        for (const c of plan.coveredFlows || []) {
+            if (c.specDirs) {
+                for (const dir of c.specDirs) {
+                    const fullDir = join(testsRoot, dir);
+                    if (existsSync(fullDir)) {
+                        specPaths.push(fullDir);
+                    }
+                }
+            }
+        }
+    }
+    // Sort by priority: P0 first
+    flows.sort((a, b) => a.priority.localeCompare(b.priority));
+    return { flows, specPaths };
+}
+function resolveHuntScope(target, manifest, testsRoot) {
+    const flows = [];
+    const specPaths = [];
+    const targetLower = target.toLowerCase();
+    if (manifest) {
+        for (const family of manifest.families) {
+            const matches = family.id.toLowerCase().includes(targetLower) ||
+                (family.userFlows || []).some((uf) => uf.toLowerCase().includes(targetLower));
+            if (matches) {
+                flows.push({
+                    id: family.id,
+                    name: family.id,
+                    priority: family.priority || 'P1',
+                    url: resolveUrlForFamily(family),
+                });
+                for (const dir of family.specDirs || []) {
+                    const fullDir = join(testsRoot, dir);
+                    if (existsSync(fullDir)) {
+                        specPaths.push(fullDir);
+                    }
+                }
+            }
+        }
+    }
+    // If no manifest matches, create a generic flow
+    if (flows.length === 0) {
+        flows.push({ id: target, name: target, priority: 'P1' });
+    }
+    return { flows, specPaths };
+}
+function resolveReleaseScope(manifest, testsRoot) {
+    const flows = [];
+    const specPaths = [];
+    if (manifest) {
+        for (const family of manifest.families) {
+            if (family.priority === 'P0' || family.priority === 'P1') {
+                flows.push({
+                    id: family.id,
+                    name: family.id,
+                    priority: family.priority,
+                    url: resolveUrlForFamily(family),
+                });
+                for (const dir of family.specDirs || []) {
+                    const fullDir = join(testsRoot, dir);
+                    if (existsSync(fullDir)) {
+                        specPaths.push(fullDir);
+                    }
+                }
+            }
+        }
+    }
+    flows.sort((a, b) => a.priority.localeCompare(b.priority));
+    return { flows, specPaths };
+}
+function resolveUrlForFamily(family) {
+    if (!family || !family.routes || family.routes.length === 0)
+        return undefined;
+    // Take the first route pattern and substitute common placeholders
+    const route = family.routes[0];
+    return route
+        .replace(/\{team\}/g, 'default')
+        .replace(/\{channel\}/g, 'town-square')
+        .replace(/\{user_id\}/g, 'me')
+        .replace(/\{[^}]+\}/g, 'test');
+}
+function readPlan(path) {
+    if (!existsSync(path))
+        return null;
+    try {
+        return JSON.parse(readFileSync(path, 'utf-8'));
+    }
+    catch {
+        return null;
+    }
+}