@yasserkhanorg/e2e-agents 0.9.0 → 0.11.0

package/dist/esm/agent/pipeline_types.js

@@ -0,0 +1,3 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+export {};

package/dist/esm/agent/pipeline_utils.js

@@ -0,0 +1,146 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { basename } from 'path';
+import { baseNameWithoutExt, normalizePath, titleCase, tokenize, uniqueTokens } from './utils.js';
+export function createMcpStatus(backend, requested) {
+    return {
+        requested,
+        active: requested && (backend === 'e2e-test-gen' || backend === 'playwright-agents'),
+        backend,
+    };
+}
+export function classifyPipelineFailure(result) {
+    if (result.failureCategory || result.failureCode) {
+        return result;
+    }
+    if (!result.error) {
+        return result;
+    }
+    const errorText = result.error.toLowerCase();
+    if (errorText.includes('etimedout') || errorText.includes('timed out')) {
+        return { ...result, failureCategory: 'environment', failureCode: 'mcp_timeout' };
+    }
+    if (errorText.includes('outside testsroot')) {
+        return { ...result, failureCategory: 'path-safety', failureCode: 'path_outside_tests_root' };
+    }
+    if (errorText.includes('playwright binary') || errorText.includes('not found')) {
+        return { ...result, failureCategory: 'environment', failureCode: 'dependency_missing' };
+    }
+    if (errorText.includes('compile validation')) {
+        return { ...result, failureCategory: 'validation', failureCode: 'compile_validation_failed' };
+    }
+    if (errorText.includes('runtime validation') || errorText.includes('playwright test failed')) {
+        return { ...result, failureCategory: 'runtime', failureCode: 'runtime_validation_failed' };
+    }
+    if (errorText.includes('quality checks failed') || errorText.includes('invalid test content')) {
+        return { ...result, failureCategory: 'quality', failureCode: 'quality_guard_failed' };
+    }
+    if (errorText.includes('generate failed') || errorText.includes('did not produce expected test file')) {
+        return { ...result, failureCategory: 'generation', failureCode: 'generation_failed' };
+    }
+    return { ...result, failureCategory: 'unknown', failureCode: 'unknown' };
+}
+export function finalizePipelineSummary(summary) {
+    return {
+        ...summary,
+        results: summary.results.map(classifyPipelineFailure),
+    };
+}
+export function toSafeSlug(value) {
+    return value.replace(/[^a-zA-Z0-9._-]+/g, '-').replace(/^-+|-+$/g, '') || 'flow';
+}
+export function stripSpecSuffix(value) {
+    return value.replace(/\.(spec|test)\.[^.]+$/i, '').replace(/\.[^.]+$/, '');
+}
+export function buildSyntheticFlowFromSpecTarget(relativeSpecPath, target) {
+    const normalizedSpecPath = normalizePath(relativeSpecPath);
+    const noSuffix = stripSpecSuffix(normalizedSpecPath);
+    const flowId = toSafeSlug(noSuffix.replace(/\//g, '.'));
+    const base = baseNameWithoutExt(stripSpecSuffix(basename(normalizedSpecPath)));
+    const flowName = titleCase(base.replace(/[._-]+/g, ' ')) || 'Recovered Spec';
+    const keywords = uniqueTokens(tokenize(noSuffix.replace(/[/.]/g, ' ')));
+    const reasons = [
+        `Playwright report marked this spec as ${target.status || 'unstable'}.`,
+        target.reason || `Auto-heal target: ${normalizedSpecPath}`,
+    ];
+    return {
+        id: flowId,
+        name: flowName,
+        kind: 'flow',
+        score: target.status === 'failed' ? 12 : 9,
+        priority: target.status === 'failed' ? 'P0' : 'P1',
+        reasons,
+        keywords,
+        files: [normalizedSpecPath],
+    };
+}
+export function firstFlowFiles(flow) {
+    return (flow.files || []).filter(Boolean).slice(0, 5);
+}
+export function buildNativeStrategyOrder(flow) {
+    const flowId = (flow.id || '').toLowerCase();
+    const haystack = [
+        flow.id,
+        flow.name,
+        ...(flow.files || []),
+        ...(flow.reasons || []),
+        ...(flow.keywords || []),
+    ].join(' ').toLowerCase();
+    const strategies = [];
+    if (flowId.includes('search')) {
+        strategies.push('search-baseline');
+    }
+    if (flowId.includes('threads') || flowId.includes('thread')) {
+        strategies.push('thread-reply');
+    }
+    if (flowId.includes('channels.lifecycle')) {
+        strategies.push('lifecycle-channel');
+    }
+    if (flowId.includes('channels.settings')) {
+        strategies.push('channel-settings');
+    }
+    if (flowId.includes('channels.switch')) {
+        strategies.push('channel-switch');
+    }
+    if (flowId.includes('messaging.markdown')) {
+        strategies.push('markdown-post');
+    }
+    if (flowId.includes('messaging.mentions')) {
+        strategies.push('mentions-post');
+    }
+    if (flowId.includes('messaging.realtime')) {
+        strategies.push('realtime-post');
+    }
+    if (/(thread|reply|rhs|sidebar[_-]?right)/.test(haystack)) {
+        strategies.push('thread-reply');
+    }
+    if (/(create|join|leave|invite)/.test(haystack)) {
+        strategies.push('lifecycle-channel');
+    }
+    if (/(settings|preferences)/.test(haystack)) {
+        strategies.push('channel-settings');
+    }
+    if (/(switch|quick\\s*switch)/.test(haystack)) {
+        strategies.push('channel-switch');
+    }
+    if (/(markdown|format)/.test(haystack)) {
+        strategies.push('markdown-post');
+    }
+    if (/(mention|@)/.test(haystack)) {
+        strategies.push('mentions-post');
+    }
+    if (/(realtime|websocket|presence)/.test(haystack)) {
+        strategies.push('realtime-post');
+    }
+    if (/(search|find|spotlight)/.test(haystack)) {
+        strategies.push('search-baseline');
+    }
+    if (/(message|post|realtime|websocket|chat)/.test(haystack)) {
+        strategies.push('message-post');
+    }
+    if (/(channel|navigation|sidebar|switch)/.test(haystack)) {
+        strategies.push('channel-baseline');
+    }
+    strategies.push('generic-baseline');
+    return Array.from(new Set(strategies));
+}

package/dist/esm/agent/process_runner.js

@@ -0,0 +1,83 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { existsSync } from 'fs';
+import { join } from 'path';
+import { spawnSync } from 'child_process';
+export function resolvePlaywrightBinary(testsRoot) {
+    const unixPath = join(testsRoot, 'node_modules', '.bin', 'playwright');
+    const windowsPath = join(testsRoot, 'node_modules', '.bin', 'playwright.cmd');
+    if (existsSync(unixPath)) {
+        return unixPath;
+    }
+    if (existsSync(windowsPath)) {
+        return windowsPath;
+    }
+    return null;
+}
+export function summarizeCommandOutput(stdout, stderr) {
+    const combined = [stdout, stderr].filter(Boolean).join('\n').trim();
+    if (!combined) {
+        return '';
+    }
+    const lines = combined.split('\n').slice(-20);
+    return lines.join('\n').slice(0, 2000);
+}
+export function runCommand(command, args, cwd, timeoutMs = 60 * 60 * 1000) {
+    // When spawning `claude`, unset CLAUDECODE so nested invocations are allowed.
+    // Claude Code sets this variable to block nested sessions; child processes
+    // that spawn their own claude instance must run without it.
+    let env;
+    if (command === 'claude') {
+        const { CLAUDECODE: _, ...rest } = process.env;
+        env = rest;
+    }
+    const result = spawnSync(command, args, {
+        cwd,
+        encoding: 'utf-8',
+        timeout: timeoutMs,
+        stdio: 'pipe',
+        ...(env ? { env } : {}),
+    });
+    return {
+        status: result.status ?? 1,
+        stdout: result.stdout || '',
+        stderr: result.stderr || '',
+        error: result.error ? result.error.message : undefined,
+    };
+}
+export function resolveMcpCommandTimeoutMs(pipeline) {
+    const value = pipeline.mcpCommandTimeoutMs;
+    if (typeof value !== 'number' || !Number.isFinite(value)) {
+        return 180000;
+    }
+    return Math.max(60000, Math.min(15 * 60 * 1000, Math.round(value)));
+}
+export function resolveMcpRetries(pipeline) {
+    const value = pipeline.mcpRetries;
+    if (typeof value !== 'number' || !Number.isFinite(value)) {
+        return 1;
+    }
+    return Math.max(0, Math.min(5, Math.round(value)));
+}
+export function isRetryableMcpFailure(result) {
+    const haystack = [result.error || '', result.stderr || '', result.stdout || ''].join('\n').toLowerCase();
+    return haystack.includes('etimedout') ||
+        haystack.includes('timed out') ||
+        haystack.includes('econnreset') ||
+        haystack.includes('429') ||
+        haystack.includes('rate limit') ||
+        haystack.includes('temporar');
+}
+export function runCommandWithRetries(command, args, cwd, timeoutMs, retries) {
+    let result = runCommand(command, args, cwd, timeoutMs);
+    for (let attempt = 1; attempt <= retries; attempt += 1) {
+        if (result.status === 0) {
+            return result;
+        }
+        if (!isRetryableMcpFailure(result)) {
+            return result;
+        }
+        result = runCommand(command, args, cwd, timeoutMs);
+    }
+    return result;
+}

package/dist/esm/agent/spec_generator.js

@@ -0,0 +1,249 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { collectMatches, escapeRegExp, parseInitSetupBindings } from './api_catalog.js';
+import { firstFlowFiles } from './pipeline_utils.js';
+export function validateGeneratedSpecContent(content, apiSurface) {
+    const issues = [];
+    if (/\btest\.describe\s*\(/.test(content)) {
+        issues.push({
+            code: 'disallowed-describe',
+            message: 'Generated tests must not use test.describe.',
+        });
+    }
+    if (/\btest\.only\s*\(/.test(content)) {
+        issues.push({
+            code: 'disallowed-only',
+            message: 'Generated tests must not use test.only.',
+        });
+    }
+    if (!/\btest\s*\(/.test(content)) {
+        issues.push({
+            code: 'missing-test',
+            message: 'Generated file does not include a test() declaration.',
+        });
+    }
+    if (/\btag\s*:\s*\[/.test(content)) {
+        issues.push({
+            code: 'tag-array-disallowed',
+            message: 'Generated tests must use a single tag string, not a tag array.',
+        });
+    }
+    const hasTagOption = /\btag\s*:\s*['"][^'"]+['"]/.test(content);
+    const hasTagInTitle = /\btest(?:\.\w+)?\s*\(\s*['"][^'"]*@ai-assisted[^'"]*['"]/.test(content);
+    if (!(hasTagOption || hasTagInTitle) || !/@ai-assisted/.test(content)) {
+        issues.push({
+            code: 'missing-tag',
+            message: "Generated tests must include '@ai-assisted' either as tag option or in test title.",
+        });
+    }
+    if (/\bsystemConsolePage\.toBeVisible\s*\(/.test(content)) {
+        issues.push({
+            code: 'fragile-system-console-visibility',
+            message: 'Avoid systemConsolePage.toBeVisible(); it relies on legacy backstage navigation that may be absent.',
+        });
+    }
+    const fragileSelectors = [
+        '.backstage-navbar',
+        '.admin-console__wrapper',
+        '.left-panel',
+        '.panel-card',
+    ].filter((selector) => content.includes(selector));
+    if (fragileSelectors.length > 0) {
+        issues.push({
+            code: 'fragile-selector',
+            message: `Avoid brittle class selectors in generated tests: ${Array.from(new Set(fragileSelectors)).join(', ')}`,
+        });
+    }
+    if (apiSurface) {
+        const unknownPwProps = Array.from(collectMatches(content, /\bpw\.([A-Za-z_][A-Za-z0-9_]*)\b/g)).filter((prop) => !apiSurface.pwProps.has(prop));
+        const unknownBrowserMethods = Array.from(collectMatches(content, /\bpw\.testBrowser\.([A-Za-z_][A-Za-z0-9_]*)\b/g)).filter((method) => !apiSurface.testBrowserMethods.has(method));
+        const unknownNestedPwMembers = [];
+        for (const match of content.matchAll(/\bpw\.([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*)\b/g)) {
+            const objectName = match[1];
+            const methodName = match[2];
+            if (!objectName || !methodName || objectName === 'testBrowser') {
+                continue;
+            }
+            const knownMethods = apiSurface.pwNestedMethods.get(objectName);
+            if (!knownMethods || !knownMethods.has(methodName)) {
+                unknownNestedPwMembers.push(`pw.${objectName}.${methodName}`);
+            }
+        }
+        const unknownChannelMembers = Array.from(collectMatches(content, /\bchannelsPage\.([A-Za-z_][A-Za-z0-9_]*)\b/g)).filter((member) => !apiSurface.channelsPageMembers.has(member));
+        const unknownSidebarMembers = Array.from(collectMatches(content, /\bchannelsPage\.sidebarRight\.([A-Za-z_][A-Za-z0-9_]*)\b/g)).filter((member) => !apiSurface.sidebarRightMembers.has(member));
+        const initSetupBindings = parseInitSetupBindings(content);
+        const unknownInitSetupKeys = initSetupBindings
+            .map((binding) => binding.key)
+            .filter((key) => !apiSurface.initSetupKeys.has(key));
+        const unknownInitSetupVariableMethods = [];
+        for (const binding of initSetupBindings) {
+            const knownMethods = apiSurface.initSetupVariableMethods.get(binding.variable);
+            if (!knownMethods || knownMethods.size === 0) {
+                continue;
+            }
+            const methodPattern = new RegExp(`\\b${escapeRegExp(binding.variable)}\\.([A-Za-z_][A-Za-z0-9_]*)\\b`, 'g');
+            for (const method of collectMatches(content, methodPattern)) {
+                if (!knownMethods.has(method)) {
+                    unknownInitSetupVariableMethods.push(`${binding.variable}.${method}`);
+                }
+            }
+        }
+        const unknown = [
+            ...unknownPwProps.map((value) => `pw.${value}`),
+            ...unknownBrowserMethods.map((value) => `pw.testBrowser.${value}`),
+            ...unknownNestedPwMembers,
+            ...unknownChannelMembers.map((value) => `channelsPage.${value}`),
+            ...unknownSidebarMembers.map((value) => `channelsPage.sidebarRight.${value}`),
+            ...unknownInitSetupKeys.map((value) => `pw.initSetup.{${value}}`),
+            ...unknownInitSetupVariableMethods,
+        ];
+        if (unknown.length > 0) {
+            issues.push({
+                code: 'unknown-api-surface',
+                message: `Generated test uses unknown API/page-object members: ${Array.from(new Set(unknown)).join(', ')}`,
+            });
+        }
+    }
+    return issues;
+}
+export function createNativePlaywrightSpec(flow, slug, strategy) {
+    const linkedFiles = firstFlowFiles(flow).join(', ') || 'N/A';
+    const header = [
+        "import {test, expect} from '@mattermost/playwright-lib';",
+        '',
+        '/**',
+        ` * Auto-generated by @yasserkhanorg/e2e-agents`,
+        ` * Flow: ${flow.id} (${flow.name})`,
+        ` * Strategy: ${strategy}`,
+        ` * Linked files: ${linkedFiles}`,
+        ' */',
+    ];
+    const start = [
+        `test('${flow.priority}: ${flow.name} generated coverage', {tag: '@ai-assisted'}, async ({pw}) => {`,
+        '  const {user, team} = await pw.initSetup();',
+        '  const {channelsPage} = await pw.testBrowser.login(user);',
+        '  await channelsPage.goto(team.name);',
+    ];
+    const end = [
+        '});',
+        '',
+    ];
+    if (strategy === 'thread-reply') {
+        return [
+            ...header,
+            ...start,
+            `  const parentMessage = \`ai-${slug}-parent-\${Date.now()}\`;`,
+            '  await channelsPage.postMessage(parentMessage);',
+            '  const rootPost = await channelsPage.getLastPost();',
+            '  await rootPost.openAThread();',
+            `  const replyMessage = \`ai-${slug}-reply-\${Date.now()}\`;`,
+            '  await channelsPage.sidebarRight.postMessage(replyMessage);',
+            '  const lastReply = await channelsPage.sidebarRight.getLastPost();',
+            '  await expect(lastReply.container).toContainText(replyMessage);',
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'lifecycle-channel') {
+        return [
+            ...header,
+            ...start,
+            `  const channelName = \`ai-${slug}-\${Date.now().toString().slice(-6)}\`;`,
+            "  await channelsPage.newChannel(channelName, 'O');",
+            '  await expect(channelsPage.page).toHaveURL(new RegExp(`/channels/${channelName}$`));',
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'channel-settings') {
+        return [
+            ...header,
+            ...start,
+            '  await channelsPage.openChannelSettings();',
+            "  await expect(channelsPage.page.getByRole('dialog', {name: 'Channel Settings'})).toBeVisible();",
+            "  await channelsPage.page.keyboard.press('Escape');",
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'channel-switch') {
+        return [
+            ...header,
+            ...start,
+            "  await channelsPage.goto(team.name, 'off-topic');",
+            "  await expect(channelsPage.page).toHaveURL(/\\/channels\\/off-topic$/);",
+            "  await expect(channelsPage.page.locator('#channelHeaderTitle')).toContainText(/off-topic/i);",
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'markdown-post') {
+        return [
+            ...header,
+            ...start,
+            `  const message = '**ai-${slug}-bold** _italic_';`,
+            '  await channelsPage.postMessage(message);',
+            '  const lastPost = await channelsPage.getLastPost();',
+            "  await expect(lastPost.container.locator('strong')).toBeVisible();",
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'mentions-post') {
+        return [
+            ...header,
+            ...start,
+            '  const mention = `@${user.username}`;',
+            '  await channelsPage.postMessage(`Ping ${mention}`);',
+            '  const lastPost = await channelsPage.getLastPost();',
+            '  await expect(lastPost.container).toContainText(mention);',
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'realtime-post') {
+        return [
+            ...header,
+            ...start,
+            `  const message = \`ai-${slug}-realtime-\${Date.now()}\`;`,
+            '  await channelsPage.postMessage(message);',
+            '  const lastPost = await channelsPage.getLastPost();',
+            '  await expect(lastPost.container).toContainText(message);',
+            "  await expect(channelsPage.page.locator('#channel_view')).toBeVisible();",
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'message-post') {
+        return [
+            ...header,
+            ...start,
+            `  const message = \`ai-${slug}-message-\${Date.now()}\`;`,
+            '  await channelsPage.postMessage(message);',
+            '  await expect(channelsPage.getLastPost()).toContainText(message);',
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'channel-baseline') {
+        return [
+            ...header,
+            ...start,
+            "  await expect(channelsPage.page.locator('#channelHeaderTitle')).toBeVisible();",
+            "  await expect(channelsPage.page.locator('#SidebarContainer')).toBeVisible();",
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'search-baseline') {
+        return [
+            ...header,
+            ...start,
+            `  const searchTerm = \`ai-${slug}-\${Date.now().toString().slice(-6)}\`;`,
+            '  await channelsPage.postMessage(searchTerm);',
+            '  await channelsPage.globalHeader.openSearch();',
+            '  await channelsPage.searchBox.searchInput.fill(searchTerm);',
+            "  await channelsPage.page.keyboard.press('Enter');",
+            "  await expect(channelsPage.page.locator('#searchContainer')).toBeVisible();",
+            ...end,
+        ].join('\n');
+    }
+    return [
+        ...header,
+        ...start,
+        '  await expect(channelsPage.page).toHaveURL(/\\/channels\\//);',
+        "  await expect(channelsPage.page.locator('#channelHeaderTitle')).toBeVisible();",
+        ...end,
+    ].join('\n');
+}

package/dist/esm/agent/validation_runner.js

@@ -0,0 +1,73 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { relative } from 'path';
+import { normalizePath } from './utils.js';
+import { runCommand, summarizeCommandOutput } from './process_runner.js';
+export function runPlaywrightRuntimeValidation(testsRoot, testFile, pipeline, playwrightBinary) {
+    if (!playwrightBinary) {
+        return {
+            status: 'failed',
+            detail: 'Playwright binary not found; cannot execute runtime validation.',
+        };
+    }
+    const relativeSpecPath = normalizePath(relative(testsRoot, testFile));
+    if (relativeSpecPath.startsWith('../') || relativeSpecPath.startsWith('..\\')) {
+        return {
+            status: 'failed',
+            detail: 'Generated spec path resolved outside testsRoot during runtime validation.',
+        };
+    }
+    const args = ['test', relativeSpecPath, '--workers', '1', '--retries', '0', '--max-failures', '1', '--reporter', 'line'];
+    if (pipeline.headless === false) {
+        args.push('--headed');
+    }
+    if (pipeline.project) {
+        args.push('--project', pipeline.project);
+    }
+    const commandResult = runCommand(playwrightBinary, args, testsRoot, 10 * 60 * 1000);
+    if (commandResult.status === 0) {
+        return { status: 'passed' };
+    }
+    const summary = summarizeCommandOutput(commandResult.stdout, commandResult.stderr);
+    return {
+        status: 'failed',
+        detail: summary || commandResult.error || `playwright test failed with status ${commandResult.status}`,
+    };
+}
+export function runPlaywrightListValidation(testsRoot, testFile, pipeline, playwrightBinary) {
+    if (!playwrightBinary) {
+        return {
+            status: 'skipped',
+            detail: 'Playwright binary not found under testsRoot/node_modules/.bin; runtime compile validation skipped.',
+        };
+    }
+    const relativeSpecPath = normalizePath(relative(testsRoot, testFile));
+    if (relativeSpecPath.startsWith('../') || relativeSpecPath.startsWith('..\\')) {
+        return {
+            status: 'failed',
+            detail: 'Generated spec path resolved outside testsRoot during validation.',
+        };
+    }
+    const args = ['test', '--list', relativeSpecPath];
+    if (pipeline.headless === false) {
+        args.push('--headed');
+    }
+    if (pipeline.project) {
+        args.push('--project', pipeline.project);
+    }
+    const commandResult = runCommand(playwrightBinary, args, testsRoot);
+    if (commandResult.error && /ENOENT/.test(commandResult.error)) {
+        return {
+            status: 'skipped',
+            detail: 'Playwright binary was not executable; runtime compile validation skipped.',
+        };
+    }
+    if (commandResult.status === 0) {
+        return { status: 'passed' };
+    }
+    const summary = summarizeCommandOutput(commandResult.stdout, commandResult.stderr);
+    return {
+        status: 'failed',
+        detail: summary || commandResult.error || `playwright --list failed with status ${commandResult.status}`,
+    };
+}

package/dist/esm/agentic/playwright_runner.js

@@ -73,7 +73,7 @@ export function runPlaywrightSpec(specPath, testsRoot, options) {
     if (!existsSync(reportDir)) {
         mkdirSync(reportDir, { recursive: true });
     }
-    const reportPath = join(reportDir, `report-${Date.now()}.json`);
+    const reportPath = join(reportDir, `report-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.json`);
     const args = [
         'playwright', 'test',
         specPath,

package/dist/esm/cli/commands/analyze.js

@@ -0,0 +1,74 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { resolveConfig } from '../../agent/config.js';
+import { runPipeline } from '../../pipeline/orchestrator.js';
+export async function runAnalyzeCommand(args, autoConfig) {
+    if (!args.path && !autoConfig) {
+        console.error('Error: --path is required for analyze command');
+        process.exit(1);
+    }
+    const { config } = resolveConfig(process.cwd(), autoConfig, {
+        path: args.path,
+        profile: args.profile,
+        testsRoot: args.testsRoot,
+        mode: 'impact',
+        gitSince: args.gitSince,
+        llmProvider: args.llmProvider,
+    });
+    const testsRoot = config.testsRoot || config.path;
+    const analyzeStages = [
+        'preprocess', 'impact', 'coverage',
+    ];
+    if (args.analyzeGenerate) {
+        analyzeStages.push('generation');
+    }
+    if (args.analyzeHeal || args.analyzeHealReport) {
+        analyzeStages.push('heal');
+    }
+    const result = await runPipeline({
+        appPath: config.path,
+        testsRoot,
+        gitSince: args.gitSince || config.git.since,
+        routeFamilies: config.routeFamilies,
+        apiSurface: config.apiSurface,
+        stages: analyzeStages,
+        generation: args.analyzeGenerate
+            ? {
+                defaultOutputDir: args.analyzeGenerateOutputDir || 'specs/functional/ai-assisted',
+                dryRun: args.dryRun,
+            }
+            : undefined,
+        heal: (args.analyzeHeal || args.analyzeHealReport)
+            ? {
+                mcp: args.pipelineMcp ?? true,
+                mcpAllowFallback: args.pipelineMcpAllowFallback ?? false,
+                mcpOnly: args.pipelineMcpOnly ?? false,
+                mcpCommandTimeoutMs: args.pipelineMcpTimeoutMs,
+                mcpRetries: args.pipelineMcpRetries ?? 1,
+                dryRun: args.dryRun,
+            }
+            : undefined,
+        playwrightReportPath: args.analyzeHealReport,
+    });
+    console.log(`Analyze report: ${result.reportPath}`);
+    console.log(`Analyze flows identified: ${result.report.summary.flowsIdentified}`);
+    console.log(`Analyze flows covered: ${result.report.summary.flowsCovered}`);
+    console.log(`Analyze flows uncovered: ${result.report.summary.flowsUncovered}`);
+    console.log(`Analyze overall confidence: ${result.report.summary.overallConfidence}`);
+    console.log(`Analyze route families: ${result.report.summary.routeFamiliesImpacted.join(', ') || 'none'}`);
+    if (result.generated && result.generated.length > 0) {
+        const written = result.generated.filter((g) => g.written).length;
+        console.log(`Analyze generated specs: ${result.generated.length} (written=${written})`);
+        for (const g of result.generated) {
+            console.log(`  ${g.mode}: ${g.specPath}`);
+        }
+    }
+    if (result.healResult) {
+        const healed = result.healResult.summary.results.filter((r) => r.healStatus === 'success').length;
+        const healFailed = result.healResult.summary.results.filter((r) => r.healStatus === 'failed').length;
+        console.log(`Analyze heal targets: ${result.healResult.targets.length} (healed=${healed}, failed=${healFailed})`);
+    }
+    if (result.warnings.length > 0) {
+        console.log(`Analyze warnings: ${result.warnings.join(' | ')}`);
+    }
+}

package/dist/esm/cli/commands/feedback.js

@@ -0,0 +1,36 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { readFileSync } from 'fs';
+import { resolveConfig } from '../../agent/config.js';
+import { appendFeedbackAndRecompute } from '../../agent/feedback.js';
+export function runFeedbackCommand(args, autoConfig) {
+    if (!args.path && !autoConfig) {
+        console.error('Error: --path is required for feedback command');
+        process.exit(1);
+    }
+    if (!args.feedbackInputPath) {
+        console.error('Error: --feedback-input <path> is required for feedback command');
+        process.exit(1);
+    }
+    const { config } = resolveConfig(process.cwd(), autoConfig, {
+        path: args.path,
+        profile: args.profile,
+        testsRoot: args.testsRoot,
+        mode: 'impact',
+        llmProvider: args.llmProvider,
+    });
+    const reportRoot = config.testsRoot || config.path;
+    const raw = JSON.parse(readFileSync(args.feedbackInputPath, 'utf-8'));
+    const payload = {
+        timestamp: raw.timestamp || new Date().toISOString(),
+        runSet: raw.runSet || 'targeted',
+        recommendedTests: raw.recommendedTests || [],
+        executedTests: raw.executedTests || [],
+        failedTests: raw.failedTests || [],
+        escapedFailures: raw.escapedFailures || [],
+    };
+    const output = appendFeedbackAndRecompute(reportRoot, payload);
+    console.log(`Feedback data: ${output.feedbackPath}`);
+    console.log(`Calibration data: ${output.calibrationPath}`);
+    console.log(`Calibration overall: precision=${output.calibration.overall.precision}, recall=${output.calibration.overall.recall}, fnr=${output.calibration.overall.falseNegativeRate}`);
+}