@yasserkhanorg/impact-gate 2.0.0

package/dist/esm/agent/playwright_report.js

@@ -0,0 +1,123 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { readFileSync } from 'fs';
+import { resolve } from 'path';
+import { normalizePath } from './utils.js';
+function asRecord(value) {
+    if (!value || typeof value !== 'object' || Array.isArray(value)) {
+        return null;
+    }
+    return value;
+}
+function asArray(value) {
+    return Array.isArray(value) ? value : [];
+}
+function isFailureStatus(value) {
+    return value === 'failed' || value === 'timedOut' || value === 'interrupted';
+}
+function classifyTestInstability(testNode) {
+    const outcome = typeof testNode.outcome === 'string' ? testNode.outcome : '';
+    if (outcome === 'unexpected') {
+        return 'failed';
+    }
+    if (outcome === 'flaky') {
+        return 'flaky';
+    }
+    const status = typeof testNode.status === 'string' ? testNode.status : '';
+    if (status === 'flaky') {
+        return 'flaky';
+    }
+    if (isFailureStatus(status)) {
+        return 'failed';
+    }
+    let sawFailure = false;
+    let sawPass = false;
+    for (const resultValue of asArray(testNode.results)) {
+        const resultNode = asRecord(resultValue);
+        if (!resultNode) {
+            continue;
+        }
+        const resultStatus = typeof resultNode.status === 'string' ? resultNode.status : '';
+        if (isFailureStatus(resultStatus)) {
+            sawFailure = true;
+        }
+        else if (resultStatus === 'passed') {
+            sawPass = true;
+        }
+    }
+    if (sawFailure && sawPass) {
+        return 'flaky';
+    }
+    if (sawFailure) {
+        return 'failed';
+    }
+    return 'stable';
+}
+function relativize(pathValue, roots) {
+    const normalized = normalizePath(pathValue);
+    for (const root of roots) {
+        const normalizedRoot = normalizePath(resolve(root));
+        if (normalized === normalizedRoot) {
+            return '.';
+        }
+        if (normalized.startsWith(`${normalizedRoot}/`)) {
+            return normalized.slice(normalizedRoot.length + 1);
+        }
+    }
+    return normalized;
+}
+function collectUnstableSpecs(value, roots, unstableMap) {
+    const node = asRecord(value);
+    if (!node) {
+        return;
+    }
+    for (const specValue of asArray(node.specs)) {
+        const specNode = asRecord(specValue);
+        if (!specNode || typeof specNode.file !== 'string') {
+            continue;
+        }
+        let failingTests = 0;
+        let flakyTests = 0;
+        for (const testValue of asArray(specNode.tests)) {
+            const testNode = asRecord(testValue);
+            if (!testNode) {
+                continue;
+            }
+            const instability = classifyTestInstability(testNode);
+            if (instability === 'failed') {
+                failingTests += 1;
+            }
+            else if (instability === 'flaky') {
+                flakyTests += 1;
+            }
+        }
+        if (failingTests === 0 && flakyTests === 0) {
+            continue;
+        }
+        const specPath = relativize(specNode.file, roots);
+        const existing = unstableMap.get(specPath);
+        const mergedFailing = (existing?.failingTests || 0) + failingTests;
+        const mergedFlaky = (existing?.flakyTests || 0) + flakyTests;
+        unstableMap.set(specPath, {
+            specPath,
+            status: mergedFailing > 0 ? 'failed' : 'flaky',
+            failingTests: mergedFailing,
+            flakyTests: mergedFlaky,
+        });
+    }
+    for (const suite of asArray(node.suites)) {
+        collectUnstableSpecs(suite, roots, unstableMap);
+    }
+}
+export function extractPlaywrightUnstableSpecs(reportPath, roots) {
+    const fullPath = resolve(reportPath);
+    const raw = JSON.parse(readFileSync(fullPath, 'utf-8'));
+    const unstableMap = new Map();
+    collectUnstableSpecs(raw, roots, unstableMap);
+    return Array.from(unstableMap.values()).sort((a, b) => {
+        if (a.status !== b.status) {
+            return a.status === 'failed' ? -1 : 1;
+        }
+        return a.specPath.localeCompare(b.specPath);
+    });
+}

package/dist/esm/agent/process_runner.js

@@ -0,0 +1,83 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { existsSync } from 'fs';
+import { join } from 'path';
+import { spawnSync } from 'child_process';
+export function resolvePlaywrightBinary(testsRoot) {
+    const unixPath = join(testsRoot, 'node_modules', '.bin', 'playwright');
+    const windowsPath = join(testsRoot, 'node_modules', '.bin', 'playwright.cmd');
+    if (existsSync(unixPath)) {
+        return unixPath;
+    }
+    if (existsSync(windowsPath)) {
+        return windowsPath;
+    }
+    return null;
+}
+export function summarizeCommandOutput(stdout, stderr) {
+    const combined = [stdout, stderr].filter(Boolean).join('\n').trim();
+    if (!combined) {
+        return '';
+    }
+    const lines = combined.split('\n').slice(-20);
+    return lines.join('\n').slice(0, 2000);
+}
+export function runCommand(command, args, cwd, timeoutMs = 60 * 60 * 1000, envOverride) {
+    // When spawning `claude`, unset CLAUDECODE so nested invocations are allowed.
+    // Claude Code sets this variable to block nested sessions; child processes
+    // that spawn their own claude instance must run without it.
+    let env = envOverride;
+    if (!env && command === 'claude') {
+        const { CLAUDECODE: _, ...rest } = process.env;
+        env = rest;
+    }
+    const result = spawnSync(command, args, {
+        cwd,
+        encoding: 'utf-8',
+        timeout: timeoutMs,
+        stdio: 'pipe',
+        ...(env ? { env } : {}),
+    });
+    return {
+        status: result.status ?? 1,
+        stdout: result.stdout || '',
+        stderr: result.stderr || '',
+        error: result.error ? result.error.message : undefined,
+    };
+}
+export function resolveMcpCommandTimeoutMs(pipeline) {
+    const value = pipeline.mcpCommandTimeoutMs;
+    if (typeof value !== 'number' || !Number.isFinite(value)) {
+        return 180000;
+    }
+    return Math.max(60000, Math.min(15 * 60 * 1000, Math.round(value)));
+}
+export function resolveMcpRetries(pipeline) {
+    const value = pipeline.mcpRetries;
+    if (typeof value !== 'number' || !Number.isFinite(value)) {
+        return 1;
+    }
+    return Math.max(0, Math.min(5, Math.round(value)));
+}
+export function isRetryableMcpFailure(result) {
+    const haystack = [result.error || '', result.stderr || '', result.stdout || ''].join('\n').toLowerCase();
+    return haystack.includes('etimedout') ||
+        haystack.includes('timed out') ||
+        haystack.includes('econnreset') ||
+        haystack.includes('429') ||
+        haystack.includes('rate limit') ||
+        haystack.includes('temporar');
+}
+export function runCommandWithRetries(command, args, cwd, timeoutMs, retries) {
+    let result = runCommand(command, args, cwd, timeoutMs);
+    for (let attempt = 1; attempt <= retries; attempt += 1) {
+        if (result.status === 0) {
+            return result;
+        }
+        if (!isRetryableMcpFailure(result)) {
+            return result;
+        }
+        result = runCommand(command, args, cwd, timeoutMs);
+    }
+    return result;
+}

package/dist/esm/agent/spec_generator.js

@@ -0,0 +1,249 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { collectMatches, escapeRegExp, parseInitSetupBindings } from './api_catalog.js';
+import { firstFlowFiles } from './pipeline_utils.js';
+export function validateGeneratedSpecContent(content, apiSurface) {
+    const issues = [];
+    if (/\btest\.describe\s*\(/.test(content)) {
+        issues.push({
+            code: 'disallowed-describe',
+            message: 'Generated tests must not use test.describe.',
+        });
+    }
+    if (/\btest\.only\s*\(/.test(content)) {
+        issues.push({
+            code: 'disallowed-only',
+            message: 'Generated tests must not use test.only.',
+        });
+    }
+    if (!/\btest\s*\(/.test(content)) {
+        issues.push({
+            code: 'missing-test',
+            message: 'Generated file does not include a test() declaration.',
+        });
+    }
+    if (/\btag\s*:\s*\[/.test(content)) {
+        issues.push({
+            code: 'tag-array-disallowed',
+            message: 'Generated tests must use a single tag string, not a tag array.',
+        });
+    }
+    const hasTagOption = /\btag\s*:\s*['"][^'"]+['"]/.test(content);
+    const hasTagInTitle = /\btest(?:\.\w+)?\s*\(\s*['"][^'"]*@ai-assisted[^'"]*['"]/.test(content);
+    if (!(hasTagOption || hasTagInTitle) || !/@ai-assisted/.test(content)) {
+        issues.push({
+            code: 'missing-tag',
+            message: "Generated tests must include '@ai-assisted' either as tag option or in test title.",
+        });
+    }
+    if (/\bsystemConsolePage\.toBeVisible\s*\(/.test(content)) {
+        issues.push({
+            code: 'fragile-system-console-visibility',
+            message: 'Avoid systemConsolePage.toBeVisible(); it relies on legacy backstage navigation that may be absent.',
+        });
+    }
+    const fragileSelectors = [
+        '.backstage-navbar',
+        '.admin-console__wrapper',
+        '.left-panel',
+        '.panel-card',
+    ].filter((selector) => content.includes(selector));
+    if (fragileSelectors.length > 0) {
+        issues.push({
+            code: 'fragile-selector',
+            message: `Avoid brittle class selectors in generated tests: ${Array.from(new Set(fragileSelectors)).join(', ')}`,
+        });
+    }
+    if (apiSurface) {
+        const unknownPwProps = Array.from(collectMatches(content, /\bpw\.([A-Za-z_][A-Za-z0-9_]*)\b/g)).filter((prop) => !apiSurface.pwProps.has(prop));
+        const unknownBrowserMethods = Array.from(collectMatches(content, /\bpw\.testBrowser\.([A-Za-z_][A-Za-z0-9_]*)\b/g)).filter((method) => !apiSurface.testBrowserMethods.has(method));
+        const unknownNestedPwMembers = [];
+        for (const match of content.matchAll(/\bpw\.([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*)\b/g)) {
+            const objectName = match[1];
+            const methodName = match[2];
+            if (!objectName || !methodName || objectName === 'testBrowser') {
+                continue;
+            }
+            const knownMethods = apiSurface.pwNestedMethods.get(objectName);
+            if (!knownMethods || !knownMethods.has(methodName)) {
+                unknownNestedPwMembers.push(`pw.${objectName}.${methodName}`);
+            }
+        }
+        const unknownChannelMembers = Array.from(collectMatches(content, /\bchannelsPage\.([A-Za-z_][A-Za-z0-9_]*)\b/g)).filter((member) => !apiSurface.channelsPageMembers.has(member));
+        const unknownSidebarMembers = Array.from(collectMatches(content, /\bchannelsPage\.sidebarRight\.([A-Za-z_][A-Za-z0-9_]*)\b/g)).filter((member) => !apiSurface.sidebarRightMembers.has(member));
+        const initSetupBindings = parseInitSetupBindings(content);
+        const unknownInitSetupKeys = initSetupBindings
+            .map((binding) => binding.key)
+            .filter((key) => !apiSurface.initSetupKeys.has(key));
+        const unknownInitSetupVariableMethods = [];
+        for (const binding of initSetupBindings) {
+            const knownMethods = apiSurface.initSetupVariableMethods.get(binding.variable);
+            if (!knownMethods || knownMethods.size === 0) {
+                continue;
+            }
+            const methodPattern = new RegExp(`\\b${escapeRegExp(binding.variable)}\\.([A-Za-z_][A-Za-z0-9_]*)\\b`, 'g');
+            for (const method of collectMatches(content, methodPattern)) {
+                if (!knownMethods.has(method)) {
+                    unknownInitSetupVariableMethods.push(`${binding.variable}.${method}`);
+                }
+            }
+        }
+        const unknown = [
+            ...unknownPwProps.map((value) => `pw.${value}`),
+            ...unknownBrowserMethods.map((value) => `pw.testBrowser.${value}`),
+            ...unknownNestedPwMembers,
+            ...unknownChannelMembers.map((value) => `channelsPage.${value}`),
+            ...unknownSidebarMembers.map((value) => `channelsPage.sidebarRight.${value}`),
+            ...unknownInitSetupKeys.map((value) => `pw.initSetup.{${value}}`),
+            ...unknownInitSetupVariableMethods,
+        ];
+        if (unknown.length > 0) {
+            issues.push({
+                code: 'unknown-api-surface',
+                message: `Generated test uses unknown API/page-object members: ${Array.from(new Set(unknown)).join(', ')}`,
+            });
+        }
+    }
+    return issues;
+}
+export function createNativePlaywrightSpec(flow, slug, strategy) {
+    const linkedFiles = firstFlowFiles(flow).join(', ') || 'N/A';
+    const header = [
+        "import {test, expect} from '@mattermost/playwright-lib';",
+        '',
+        '/**',
+        ` * Auto-generated by @yasserkhanorg/impact-gate`,
+        ` * Flow: ${flow.id} (${flow.name})`,
+        ` * Strategy: ${strategy}`,
+        ` * Linked files: ${linkedFiles}`,
+        ' */',
+    ];
+    const start = [
+        `test('${flow.priority}: ${flow.name} generated coverage', {tag: '@ai-assisted'}, async ({pw}) => {`,
+        '  const {user, team} = await pw.initSetup();',
+        '  const {channelsPage} = await pw.testBrowser.login(user);',
+        '  await channelsPage.goto(team.name);',
+    ];
+    const end = [
+        '});',
+        '',
+    ];
+    if (strategy === 'thread-reply') {
+        return [
+            ...header,
+            ...start,
+            `  const parentMessage = \`ai-${slug}-parent-\${Date.now()}\`;`,
+            '  await channelsPage.postMessage(parentMessage);',
+            '  const rootPost = await channelsPage.getLastPost();',
+            '  await rootPost.openAThread();',
+            `  const replyMessage = \`ai-${slug}-reply-\${Date.now()}\`;`,
+            '  await channelsPage.sidebarRight.postMessage(replyMessage);',
+            '  const lastReply = await channelsPage.sidebarRight.getLastPost();',
+            '  await expect(lastReply.container).toContainText(replyMessage);',
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'lifecycle-channel') {
+        return [
+            ...header,
+            ...start,
+            `  const channelName = \`ai-${slug}-\${Date.now().toString().slice(-6)}\`;`,
+            "  await channelsPage.newChannel(channelName, 'O');",
+            '  await expect(channelsPage.page).toHaveURL(new RegExp(`/channels/${channelName}$`));',
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'channel-settings') {
+        return [
+            ...header,
+            ...start,
+            '  await channelsPage.openChannelSettings();',
+            "  await expect(channelsPage.page.getByRole('dialog', {name: 'Channel Settings'})).toBeVisible();",
+            "  await channelsPage.page.keyboard.press('Escape');",
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'channel-switch') {
+        return [
+            ...header,
+            ...start,
+            "  await channelsPage.goto(team.name, 'off-topic');",
+            "  await expect(channelsPage.page).toHaveURL(/\\/channels\\/off-topic$/);",
+            "  await expect(channelsPage.page.locator('#channelHeaderTitle')).toContainText(/off-topic/i);",
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'markdown-post') {
+        return [
+            ...header,
+            ...start,
+            `  const message = '**ai-${slug}-bold** _italic_';`,
+            '  await channelsPage.postMessage(message);',
+            '  const lastPost = await channelsPage.getLastPost();',
+            "  await expect(lastPost.container.locator('strong')).toBeVisible();",
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'mentions-post') {
+        return [
+            ...header,
+            ...start,
+            '  const mention = `@${user.username}`;',
+            '  await channelsPage.postMessage(`Ping ${mention}`);',
+            '  const lastPost = await channelsPage.getLastPost();',
+            '  await expect(lastPost.container).toContainText(mention);',
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'realtime-post') {
+        return [
+            ...header,
+            ...start,
+            `  const message = \`ai-${slug}-realtime-\${Date.now()}\`;`,
+            '  await channelsPage.postMessage(message);',
+            '  const lastPost = await channelsPage.getLastPost();',
+            '  await expect(lastPost.container).toContainText(message);',
+            "  await expect(channelsPage.page.locator('#channel_view')).toBeVisible();",
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'message-post') {
+        return [
+            ...header,
+            ...start,
+            `  const message = \`ai-${slug}-message-\${Date.now()}\`;`,
+            '  await channelsPage.postMessage(message);',
+            '  await expect(channelsPage.getLastPost()).toContainText(message);',
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'channel-baseline') {
+        return [
+            ...header,
+            ...start,
+            "  await expect(channelsPage.page.locator('#channelHeaderTitle')).toBeVisible();",
+            "  await expect(channelsPage.page.locator('#SidebarContainer')).toBeVisible();",
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'search-baseline') {
+        return [
+            ...header,
+            ...start,
+            `  const searchTerm = \`ai-${slug}-\${Date.now().toString().slice(-6)}\`;`,
+            '  await channelsPage.postMessage(searchTerm);',
+            '  await channelsPage.globalHeader.openSearch();',
+            '  await channelsPage.searchBox.searchInput.fill(searchTerm);',
+            "  await channelsPage.page.keyboard.press('Enter');",
+            "  await expect(channelsPage.page.locator('#searchContainer')).toBeVisible();",
+            ...end,
+        ].join('\n');
+    }
+    return [
+        ...header,
+        ...start,
+        '  await expect(channelsPage.page).toHaveURL(/\\/channels\\//);',
+        "  await expect(channelsPage.page.locator('#channelHeaderTitle')).toBeVisible();",
+        ...end,
+    ].join('\n');
+}

package/dist/esm/agent/test_path.js

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+function normalizeTestName(test) {
+    return test.replace(/ \(flags:.*\)$/, '').trim();
+}
+export function inferSubsystemFromTestPath(test) {
+    const normalized = normalizeTestName(test).replace(/^\/+/, '');
+    const parts = normalized.split('/').filter(Boolean);
+    if (parts.length === 0) {
+        return 'unknown';
+    }
+    const specsIdx = parts.findIndex((part) => part === 'specs' || part === 'tests');
+    if (specsIdx >= 0 && specsIdx + 1 < parts.length) {
+        return parts[specsIdx + 1] || 'unknown';
+    }
+    if (parts.length >= 2 && (parts[0] === 'e2e-tests' || parts[0] === 'playwright')) {
+        return parts[1] || 'unknown';
+    }
+    return parts[0] || 'unknown';
+}