@yasserkhanorg/impact-gate 2.0.0

package/dist/esm/agent/validation_runner.js

@@ -0,0 +1,73 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { relative } from 'path';
+import { normalizePath } from './utils.js';
+import { runCommand, summarizeCommandOutput } from './process_runner.js';
+export function runPlaywrightRuntimeValidation(testsRoot, testFile, pipeline, playwrightBinary) {
+    if (!playwrightBinary) {
+        return {
+            status: 'failed',
+            detail: 'Playwright binary not found; cannot execute runtime validation.',
+        };
+    }
+    const relativeSpecPath = normalizePath(relative(testsRoot, testFile));
+    if (relativeSpecPath.startsWith('../') || relativeSpecPath.startsWith('..\\')) {
+        return {
+            status: 'failed',
+            detail: 'Generated spec path resolved outside testsRoot during runtime validation.',
+        };
+    }
+    const args = ['test', relativeSpecPath, '--workers', '1', '--retries', '0', '--max-failures', '1', '--reporter', 'line'];
+    if (pipeline.headless === false) {
+        args.push('--headed');
+    }
+    if (pipeline.project) {
+        args.push('--project', pipeline.project);
+    }
+    const commandResult = runCommand(playwrightBinary, args, testsRoot, 10 * 60 * 1000);
+    if (commandResult.status === 0) {
+        return { status: 'passed' };
+    }
+    const summary = summarizeCommandOutput(commandResult.stdout, commandResult.stderr);
+    return {
+        status: 'failed',
+        detail: summary || commandResult.error || `playwright test failed with status ${commandResult.status}`,
+    };
+}
+export function runPlaywrightListValidation(testsRoot, testFile, pipeline, playwrightBinary) {
+    if (!playwrightBinary) {
+        return {
+            status: 'skipped',
+            detail: 'Playwright binary not found under testsRoot/node_modules/.bin; runtime compile validation skipped.',
+        };
+    }
+    const relativeSpecPath = normalizePath(relative(testsRoot, testFile));
+    if (relativeSpecPath.startsWith('../') || relativeSpecPath.startsWith('..\\')) {
+        return {
+            status: 'failed',
+            detail: 'Generated spec path resolved outside testsRoot during validation.',
+        };
+    }
+    const args = ['test', '--list', relativeSpecPath];
+    if (pipeline.headless === false) {
+        args.push('--headed');
+    }
+    if (pipeline.project) {
+        args.push('--project', pipeline.project);
+    }
+    const commandResult = runCommand(playwrightBinary, args, testsRoot);
+    if (commandResult.error && /ENOENT/.test(commandResult.error)) {
+        return {
+            status: 'skipped',
+            detail: 'Playwright binary was not executable; runtime compile validation skipped.',
+        };
+    }
+    if (commandResult.status === 0) {
+        return { status: 'passed' };
+    }
+    const summary = summarizeCommandOutput(commandResult.stdout, commandResult.stderr);
+    return {
+        status: 'failed',
+        detail: summary || commandResult.error || `playwright --list failed with status ${commandResult.status}`,
+    };
+}

package/dist/esm/agentic/fix_loop.js

@@ -0,0 +1,91 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { sanitizeForPrompt } from '../crew/sanitize.js';
+export function buildFixPrompt(ctx) {
+    const isCompileError = ctx.failures.some((f) => f.testTitle === '(compile)');
+    const failuresBlock = ctx.failures.map((f) => {
+        const lines = [`  Test: ${sanitizeForPrompt(f.testTitle)}`, `  Error: ${sanitizeForPrompt(f.error)}`];
+        if (f.stack)
+            lines.push(`  Stack: ${sanitizeForPrompt(f.stack)}`);
+        if (f.line)
+            lines.push(`  Line: ${f.line}`);
+        if (f.expected)
+            lines.push(`  Expected: ${sanitizeForPrompt(f.expected)}`);
+        if (f.actual)
+            lines.push(`  Actual: ${sanitizeForPrompt(f.actual)}`);
+        return lines.join('\n');
+    }).join('\n\n');
+    const errorType = isCompileError ? 'COMPILE ERROR' : 'TEST FAILURE';
+    const apiBlock = ctx.apiSurfaceHint
+        ? `\nAVAILABLE PAGE OBJECT API:\n${ctx.apiSurfaceHint}\n`
+        : '';
+    return [
+        `Fix this Playwright E2E test. This is attempt ${ctx.attempt} of ${ctx.maxAttempts}.`,
+        '',
+        `## ${errorType}`,
+        '',
+        failuresBlock,
+        '',
+        '## CURRENT SPEC CODE',
+        '',
+        '```typescript',
+        ctx.specCode,
+        '```',
+        apiBlock,
+        '## RULES',
+        '',
+        '1. Import ONLY from "@mattermost/playwright-lib" — no "@playwright/test" imports.',
+        '2. Every test must call `await pw.initSetup()` first.',
+        '3. Use `await pw.testBrowser.login(user)` to log in.',
+        '4. Use ONLY page object methods listed in the API above. Do NOT invent methods.',
+        '5. If a method is not available, use `page.getByRole()` or `page.getByTestId()`.',
+        '6. For flaky/timing issues: add `await expect(locator).toBeVisible()` waits before interactions.',
+        '7. Keep the same test scenarios — fix the implementation, not the intent.',
+        '8. Return the COMPLETE fixed spec file — not a diff or partial code.',
+        '',
+        isCompileError
+            ? 'The file does not compile. Fix syntax errors, missing imports, or invalid method calls.'
+            : 'The test compiles but fails at runtime. Fix selectors, waits, or assertion logic.',
+        '',
+        'Return ONLY the complete TypeScript code. No explanations, no markdown fences (except wrapping the code).',
+    ].join('\n');
+}
+/**
+ * Extract fixed spec code from an LLM response.
+ * Returns null if the response doesn't contain valid test code.
+ */
+export function applyFix(llmResponse) {
+    let code = llmResponse.trim();
+    if (!code)
+        return null;
+    // Strip markdown fences
+    const fenced = code.match(/```(?:typescript|ts)?\s*([\s\S]*?)```/i);
+    if (fenced) {
+        code = fenced[1].trim();
+    }
+    // Must contain test( to be valid
+    if (!code.includes('test('))
+        return null;
+    // Ensure it has the right import
+    if (!code.includes('@mattermost/playwright-lib')) {
+        code = `import {expect, test} from '@mattermost/playwright-lib';\n\n${code}`;
+    }
+    return code;
+}
+/**
+ * Run one fix attempt: call LLM with failure context, return fixed code.
+ */
+export async function generateFix(provider, ctx) {
+    const prompt = buildFixPrompt(ctx);
+    const response = await provider.generateText(prompt, {
+        maxTokens: 8000,
+        temperature: 0.1,
+        timeout: 60000,
+        systemPrompt: 'You are an expert Playwright test fixer for Mattermost. Return only TypeScript code.',
+    });
+    const code = applyFix(response.text);
+    return {
+        code,
+        tokensUsed: { input: response.usage?.inputTokens || 0, output: response.usage?.outputTokens || 0 },
+    };
+}

package/dist/esm/agentic/playwright_runner.js

@@ -0,0 +1,161 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { spawnSync } from 'child_process';
+import { existsSync, readFileSync, mkdirSync, rmSync } from 'fs';
+import { join, resolve } from 'path';
+const MAX_STDOUT_CHARS = 8000;
+const MAX_ERROR_CHARS = 2000;
+const MAX_STACK_CHARS = 1000;
+function extractSpecs(suites) {
+    const specs = [];
+    for (const suite of suites) {
+        specs.push(...suite.specs);
+        if (suite.suites) {
+            specs.push(...extractSpecs(suite.suites));
+        }
+    }
+    return specs;
+}
+export function parsePlaywrightJsonReport(report, specPath) {
+    const failures = [];
+    const allSpecs = extractSpecs(report.suites);
+    let passed = 0;
+    let failed = 0;
+    for (const spec of allSpecs) {
+        if (spec.ok) {
+            passed++;
+        }
+        else {
+            failed++;
+            const lastResult = spec.tests[0]?.results?.at(-1);
+            failures.push({
+                testTitle: spec.title,
+                specPath,
+                error: (lastResult?.error?.message || 'Unknown error').slice(0, MAX_ERROR_CHARS),
+                stack: (lastResult?.error?.stack || '').slice(0, MAX_STACK_CHARS),
+                line: extractLineNumber(lastResult?.error?.stack),
+            });
+        }
+    }
+    return {
+        specPath,
+        passed,
+        failed,
+        flaky: report.stats.flaky || 0,
+        skipped: report.stats.skipped || 0,
+        failures,
+        stdout: '',
+        durationMs: report.stats.duration || 0,
+        compiled: true,
+    };
+}
+function extractLineNumber(stack) {
+    if (!stack)
+        return undefined;
+    const match = stack.match(/:(\d+):\d+\)?$/m);
+    return match ? parseInt(match[1], 10) : undefined;
+}
+/**
+ * Run a single Playwright spec file and return structured results.
+ * Uses a temp JSON reporter to get machine-readable output.
+ */
+export function runPlaywrightSpec(specPath, testsRoot, options) {
+    // SECURITY: Validate spec path is within testsRoot and has valid extension
+    const resolvedSpec = resolve(specPath);
+    const resolvedRoot = resolve(testsRoot);
+    if (!resolvedSpec.startsWith(resolvedRoot + '/')) {
+        throw new Error(`Security: spec path ${specPath} is outside testsRoot`);
+    }
+    if (!resolvedSpec.endsWith('.spec.ts') && !resolvedSpec.endsWith('.test.ts')) {
+        throw new Error(`Security: spec path must end in .spec.ts or .test.ts`);
+    }
+    const reportDir = join(testsRoot, '.e2e-ai-agents', 'agentic-reports');
+    if (!existsSync(reportDir)) {
+        mkdirSync(reportDir, { recursive: true });
+    }
+    const reportPath = join(reportDir, `report-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.json`);
+    const args = [
+        'playwright', 'test',
+        specPath,
+        '--reporter', 'json',
+        '--project', options.project || 'chrome',
+    ];
+    if (options.baseUrl) {
+        args.push('--config', 'playwright.config.ts');
+    }
+    const startTime = Date.now();
+    const result = spawnSync('npx', args, {
+        cwd: testsRoot,
+        encoding: 'utf-8',
+        timeout: options.timeoutMs || 120000,
+        maxBuffer: 2 * 1024 * 1024,
+        env: {
+            ...process.env,
+            PLAYWRIGHT_JSON_OUTPUT_NAME: reportPath,
+        },
+    });
+    const durationMs = Date.now() - startTime;
+    const stdout = (result.stdout || '').slice(0, MAX_STDOUT_CHARS);
+    const stderr = (result.stderr || '').slice(0, MAX_STDOUT_CHARS);
+    // Check for compile errors
+    if (stderr.includes('SyntaxError') || stderr.includes('Cannot find module') || stderr.includes('TypeError')) {
+        return {
+            specPath,
+            passed: 0,
+            failed: 1,
+            flaky: 0,
+            skipped: 0,
+            failures: [{
+                    testTitle: '(compile)',
+                    specPath,
+                    error: stderr.slice(0, MAX_ERROR_CHARS),
+                    stack: '',
+                }],
+            stdout: `${stdout}\n${stderr}`,
+            durationMs,
+            compiled: false,
+        };
+    }
+    // Try to parse JSON report from stdout (Playwright JSON reporter writes to stdout)
+    try {
+        const jsonReport = JSON.parse(stdout);
+        const parsed = parsePlaywrightJsonReport(jsonReport, specPath);
+        parsed.durationMs = durationMs;
+        parsed.stdout = stdout;
+        return parsed;
+    }
+    catch {
+        // Fallback: try the file-based report
+    }
+    // Try file-based report
+    if (existsSync(reportPath)) {
+        try {
+            const jsonReport = JSON.parse(readFileSync(reportPath, 'utf-8'));
+            const parsed = parsePlaywrightJsonReport(jsonReport, specPath);
+            parsed.durationMs = durationMs;
+            parsed.stdout = stdout;
+            try {
+                rmSync(reportPath);
+            }
+            catch { /* ignore */ }
+            return parsed;
+        }
+        catch {
+            // Fallback to exit code
+        }
+    }
+    // Last resort: use exit code
+    return {
+        specPath,
+        passed: result.status === 0 ? 1 : 0,
+        failed: result.status === 0 ? 0 : 1,
+        flaky: 0,
+        skipped: 0,
+        failures: result.status !== 0
+            ? [{ testTitle: '(unknown)', specPath, error: stderr.slice(0, MAX_ERROR_CHARS), stack: '' }]
+            : [],
+        stdout,
+        durationMs,
+        compiled: !stderr.includes('Error'),
+    };
+}

package/dist/esm/agentic/runner.js

@@ -0,0 +1,207 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { dirname, join, resolve } from 'path';
+import { runPlaywrightSpec } from './playwright_runner.js';
+import { generateFix } from './fix_loop.js';
+import { parseGenerationResponse } from '../prompts/generation.js';
+import { formatApiSurfaceForPrompt } from '../knowledge/api_surface.js';
+import { sanitizeForPrompt } from '../crew/sanitize.js';
+function buildGeneratePrompt(scenario, apiSurfaceHint, profile) {
+    const projectName = profile?.projectName || 'Mattermost';
+    const importSource = profile?.importStatement || '@mattermost/playwright-lib';
+    const scenariosBlock = scenario.scenarios
+        .map((s, i) => `  ${i + 1}. ${sanitizeForPrompt(s)}`)
+        .join('\n');
+    return [
+        `Generate a ${projectName} Playwright E2E test file.`,
+        '',
+        `FLOW: ${sanitizeForPrompt(scenario.name)}`,
+        `Route Family: ${scenario.routeFamily}`,
+        `Priority: ${scenario.priority}`,
+        scenario.evidence ? `Evidence: ${sanitizeForPrompt(scenario.evidence)}` : '',
+        '',
+        'SCENARIOS TO IMPLEMENT:',
+        scenariosBlock,
+        '',
+        'AVAILABLE PAGE OBJECTS AND METHODS:',
+        apiSurfaceHint || 'Use page.getByRole() or page.getByTestId() for selectors.',
+        '',
+        'MANDATORY RULES:',
+        `1. Import ONLY from "${importSource}" — no other test framework imports.`,
+        '2. Every test must call `await pw.initSetup()` first.',
+        '3. Use `await pw.testBrowser.login(user)` to log in — never hardcode credentials.',
+        '4. Use ONLY page object methods listed above. Do NOT invent methods.',
+        '5. If a method is not available, use `page.getByRole()` or `page.getByTestId()`.',
+        `6. Tag every test: {tag: '@${scenario.routeFamily}'}`,
+        '7. Write one test per scenario with a descriptive name.',
+        `8. Use \`expect\` from "${importSource}".`,
+        '9. Include the copyright header.',
+        '10. NEVER fabricate test IDs (MM-TXXXX). Use descriptive names only.',
+        '',
+        'EXAMPLE STRUCTURE:',
+        '```typescript',
+        '// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.',
+        '// See LICENSE.txt for license information.',
+        '',
+        `import {expect, test} from '${importSource}';`,
+        '',
+        'test(',
+        "    'user can post a message in channel',",
+        `    {tag: '@${scenario.routeFamily}'},`,
+        '    async ({pw}) => {',
+        '        const {user} = await pw.initSetup();',
+        '        const {channelsPage} = await pw.testBrowser.login(user);',
+        '        await channelsPage.goto();',
+        '        await channelsPage.toBeVisible();',
+        '        // test steps...',
+        '    },',
+        ');',
+        '```',
+        '',
+        'Return ONLY the TypeScript code. No explanations.',
+    ].filter(Boolean).join('\n');
+}
+function resolveSpecPath(scenario, testsRoot) {
+    let specPath;
+    if (scenario.targetSpec) {
+        specPath = join(testsRoot, scenario.targetSpec);
+    }
+    else {
+        const safeName = scenario.id.replace(/[^a-zA-Z0-9_-]/g, '_').toLowerCase();
+        const outputDir = join(testsRoot, 'specs', 'functional', 'ai-assisted');
+        specPath = join(outputDir, `${safeName}.spec.ts`);
+    }
+    // SECURITY: Prevent path traversal
+    const resolved = resolve(specPath);
+    const resolvedRoot = resolve(testsRoot);
+    if (!resolved.startsWith(resolvedRoot + '/') && resolved !== resolvedRoot) {
+        throw new Error(`Path traversal blocked: ${specPath} resolves outside testsRoot`);
+    }
+    if (!resolved.endsWith('.spec.ts') && !resolved.endsWith('.test.ts')) {
+        throw new Error(`Invalid spec path: must end in .spec.ts or .test.ts`);
+    }
+    return specPath;
+}
+async function generateInitialSpec(provider, scenario, specPath, apiSurfaceHint, profile) {
+    const prompt = buildGeneratePrompt(scenario, apiSurfaceHint, profile);
+    const response = await provider.generateText(prompt, {
+        maxTokens: 8000,
+        temperature: 0.1,
+        timeout: 60000,
+        systemPrompt: `You are an expert Playwright test writer for ${profile?.projectName || 'Mattermost'}. Return only TypeScript code.`,
+    });
+    // Reuse existing parsing logic from prompts/generation.ts
+    const parsed = parseGenerationResponse(response.text, specPath, 'create_spec', scenario.id);
+    return parsed?.code ?? null;
+}
+async function runSingleScenario(scenario, options) {
+    const { config, provider } = options;
+    const warnings = [];
+    const specPath = resolveSpecPath(scenario, config.testsRoot);
+    // Build API surface hint
+    let apiHint = options.apiSurfaceHint || '';
+    if (!apiHint && options.apiSurface) {
+        const allClassNames = options.apiSurface.pageObjects.map((po) => po.className);
+        apiHint = formatApiSurfaceForPrompt(options.apiSurface, allClassNames);
+    }
+    // Step 1: Generate initial spec
+    let specCode;
+    try {
+        specCode = await generateInitialSpec(provider, scenario, specPath, apiHint, options.generationProfile);
+    }
+    catch (error) {
+        const msg = error instanceof Error ? error.message : String(error);
+        warnings.push(`Generation failed for ${scenario.id}: ${msg}`);
+        return { specPath, scenarioSource: scenario.id, status: 'failed', attempts: 0, warnings };
+    }
+    if (!specCode) {
+        warnings.push(`LLM returned invalid code for ${scenario.id}`);
+        return { specPath, scenarioSource: scenario.id, status: 'failed', attempts: 0, warnings };
+    }
+    // Write the spec file
+    const dir = dirname(specPath);
+    if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+    }
+    writeFileSync(specPath, specCode, 'utf-8');
+    // Dry run: skip execution
+    if (config.dryRun) {
+        return { specPath, scenarioSource: scenario.id, status: 'skipped', attempts: 0, warnings };
+    }
+    // Step 2: Run -> Fix loop
+    let lastRun;
+    for (let attempt = 1; attempt <= config.maxAttempts; attempt++) {
+        lastRun = runPlaywrightSpec(specPath, config.testsRoot, {
+            project: config.project,
+            baseUrl: config.baseUrl,
+            timeoutMs: config.testTimeoutMs,
+        });
+        // All passed!
+        if (lastRun.failed === 0 && lastRun.compiled) {
+            return {
+                specPath,
+                scenarioSource: scenario.id,
+                status: 'passed',
+                attempts: attempt,
+                finalRun: lastRun,
+                warnings,
+            };
+        }
+        // If this is the last attempt, don't try to fix
+        if (attempt >= config.maxAttempts) {
+            break;
+        }
+        // Step 3: Fix
+        const currentCode = readFileSync(specPath, 'utf-8');
+        try {
+            const fixResult = await generateFix(provider, {
+                specCode: currentCode,
+                failures: lastRun.failures,
+                attempt,
+                maxAttempts: config.maxAttempts,
+                apiSurfaceHint: apiHint,
+            });
+            if (fixResult.code) {
+                writeFileSync(specPath, fixResult.code, 'utf-8');
+            }
+            else {
+                warnings.push(`Fix attempt ${attempt} returned invalid code for ${scenario.id}`);
+            }
+        }
+        catch (error) {
+            const msg = error instanceof Error ? error.message : String(error);
+            warnings.push(`Fix attempt ${attempt} failed for ${scenario.id}: ${msg}`);
+        }
+    }
+    return {
+        specPath,
+        scenarioSource: scenario.id,
+        status: lastRun?.compiled === false ? 'compile-error' : 'max-attempts',
+        attempts: config.maxAttempts,
+        finalRun: lastRun,
+        warnings,
+    };
+}
+export async function runAgenticGeneration(options) {
+    const startTime = Date.now();
+    const results = [];
+    const warnings = [];
+    for (const scenario of options.scenarios) {
+        const result = await runSingleScenario(scenario, options);
+        results.push(result);
+        warnings.push(...result.warnings);
+    }
+    const totalPassed = results.filter((r) => r.status === 'passed').length;
+    const totalFailed = results.filter((r) => r.status !== 'passed' && r.status !== 'skipped').length;
+    const totalAttempts = results.reduce((sum, r) => sum + r.attempts, 0);
+    return {
+        results,
+        totalGenerated: results.length,
+        totalPassed,
+        totalFailed,
+        totalAttempts,
+        durationMs: Date.now() - startTime,
+        warnings,
+    };
+}

package/dist/esm/agentic/types.js

@@ -0,0 +1,3 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+export {};

package/dist/esm/agents/coverage-evaluator.js

@@ -0,0 +1,37 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+/**
+ * Coverage Evaluator Agent — wraps pipeline stage2 (coverage evaluation) in the Agent interface.
+ */
+import { runCoverageStage } from '../pipeline/stage2_coverage.js';
+export class CoverageEvaluatorAgent {
+    constructor() {
+        this.role = 'coverage-evaluator';
+    }
+    async execute(_task, ctx) {
+        const warnings = [];
+        if (ctx.impactedFlows.length === 0) {
+            warnings.push('Coverage evaluator: no impacted flows to evaluate.');
+            return { role: this.role, status: 'partial', output: [], warnings };
+        }
+        try {
+            const result = await runCoverageStage(ctx.impactedFlows, ctx.specIndex, ctx.context, ctx.testsRoot, { provider: ctx.providerOverride });
+            // Replace impacted flows with coverage-enriched versions.
+            // This is intentionally a full replace (not push) because coverage evaluation
+            // returns the same flow IDs with updated coverage fields.
+            ctx.impactedFlows = result.decisions;
+            warnings.push(...result.warnings);
+            return {
+                role: this.role,
+                status: 'success',
+                output: result.decisions,
+                warnings,
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            warnings.push(`Coverage evaluator failed: ${message}`);
+            return { role: this.role, status: 'failed', output: null, warnings };
+        }
+    }
+}