@yasserkhanorg/e2e-agents 0.3.2 → 0.3.4

package/dist/agent/pipeline.js

@@ -8,6 +8,47 @@ const fs_1 = require("fs");
 const path_1 = require("path");
 const child_process_1 = require("child_process");
 const utils_js_1 = require("./utils.js");
+function createMcpStatus(backend, requested) {
+    return {
+        requested,
+        active: requested && (backend === 'e2e-test-gen' || backend === 'playwright-agents'),
+        backend,
+    };
+}
+function classifyPipelineFailure(result) {
+    if (result.failureCategory || result.failureCode) {
+        return result;
+    }
+    if (!result.error) {
+        return result;
+    }
+    const errorText = result.error.toLowerCase();
+    if (errorText.includes('outside testsroot')) {
+        return { ...result, failureCategory: 'path-safety', failureCode: 'path_outside_tests_root' };
+    }
+    if (errorText.includes('playwright binary') || errorText.includes('not found')) {
+        return { ...result, failureCategory: 'environment', failureCode: 'dependency_missing' };
+    }
+    if (errorText.includes('compile validation')) {
+        return { ...result, failureCategory: 'validation', failureCode: 'compile_validation_failed' };
+    }
+    if (errorText.includes('runtime validation') || errorText.includes('playwright test failed')) {
+        return { ...result, failureCategory: 'runtime', failureCode: 'runtime_validation_failed' };
+    }
+    if (errorText.includes('quality checks failed') || errorText.includes('invalid test content')) {
+        return { ...result, failureCategory: 'quality', failureCode: 'quality_guard_failed' };
+    }
+    if (errorText.includes('generate failed') || errorText.includes('did not produce expected test file')) {
+        return { ...result, failureCategory: 'generation', failureCode: 'generation_failed' };
+    }
+    return { ...result, failureCategory: 'unknown', failureCode: 'unknown' };
+}
+function finalizePipelineSummary(summary) {
+    return {
+        ...summary,
+        results: summary.results.map(classifyPipelineFailure),
+    };
+}
 function hasE2eTestGenCLI(testsRoot) {
     const cliPath = (0, path_1.join)(testsRoot, 'e2e-test-gen-cli.ts');
     return (0, fs_1.existsSync)(cliPath) ? cliPath : null;
@@ -44,6 +85,7 @@ function firstFlowFiles(flow) {
     return (flow.files || []).filter(Boolean).slice(0, 5);
 }
 function buildNativeStrategyOrder(flow) {
+    const flowId = (flow.id || '').toLowerCase();
     const haystack = [
         flow.id,
         flow.name,
@@ -52,22 +94,259 @@ function buildNativeStrategyOrder(flow) {
         ...(flow.keywords || []),
     ].join(' ').toLowerCase();
     const strategies = [];
+    if (flowId.includes('search')) {
+        strategies.push('search-baseline');
+    }
+    if (flowId.includes('threads') || flowId.includes('thread')) {
+        strategies.push('thread-reply');
+    }
+    if (flowId.includes('channels.lifecycle')) {
+        strategies.push('lifecycle-channel');
+    }
+    if (flowId.includes('channels.settings')) {
+        strategies.push('channel-settings');
+    }
+    if (flowId.includes('channels.switch')) {
+        strategies.push('channel-switch');
+    }
+    if (flowId.includes('messaging.markdown')) {
+        strategies.push('markdown-post');
+    }
+    if (flowId.includes('messaging.mentions')) {
+        strategies.push('mentions-post');
+    }
+    if (flowId.includes('messaging.realtime')) {
+        strategies.push('realtime-post');
+    }
     if (/(thread|reply|rhs|sidebar[_-]?right)/.test(haystack)) {
         strategies.push('thread-reply');
     }
+    if (/(create|join|leave|invite)/.test(haystack)) {
+        strategies.push('lifecycle-channel');
+    }
+    if (/(settings|preferences)/.test(haystack)) {
+        strategies.push('channel-settings');
+    }
+    if (/(switch|quick\\s*switch)/.test(haystack)) {
+        strategies.push('channel-switch');
+    }
+    if (/(markdown|format)/.test(haystack)) {
+        strategies.push('markdown-post');
+    }
+    if (/(mention|@)/.test(haystack)) {
+        strategies.push('mentions-post');
+    }
+    if (/(realtime|websocket|presence)/.test(haystack)) {
+        strategies.push('realtime-post');
+    }
+    if (/(search|find|spotlight)/.test(haystack)) {
+        strategies.push('search-baseline');
+    }
     if (/(message|post|realtime|websocket|chat)/.test(haystack)) {
         strategies.push('message-post');
     }
     if (/(channel|navigation|sidebar|switch)/.test(haystack)) {
         strategies.push('channel-baseline');
     }
-    if (/(search|find|spotlight)/.test(haystack)) {
-        strategies.push('search-baseline');
-    }
     strategies.push('generic-baseline');
     return Array.from(new Set(strategies));
 }
-function validateGeneratedSpecContent(content) {
+function createDefaultApiSurfaceCatalog() {
+    const pwNestedMethods = new Map();
+    pwNestedMethods.set('apiClient', new Set([
+        'createPost',
+        'createDirectChannel',
+        'createChannel',
+        'getChannels',
+        'getChannelByName',
+        'getPostsSince',
+    ]));
+    return {
+        pwProps: new Set([
+            'initSetup',
+            'testBrowser',
+            'apiInitSetup',
+            'apiAdminSetup',
+            'apiCreateChannel',
+            'apiCreateUser',
+            'apiLogin',
+            'apiClient',
+        ]),
+        pwNestedMethods,
+        initSetupKeys: new Set([
+            'user',
+            'team',
+            'adminClient',
+            'adminUser',
+            'adminConfig',
+            'userClient',
+            'offTopicUrl',
+            'townSquareUrl',
+        ]),
+        initSetupVariableMethods: new Map(),
+        testBrowserMethods: new Set([
+            'login',
+            'openNewBrowserContext',
+            'newContext',
+        ]),
+        channelsPageMembers: new Set([
+            'goto',
+            'page',
+            'postMessage',
+            'getLastPost',
+            'sidebarRight',
+            'openChannelSettings',
+            'newChannel',
+            'globalHeader',
+            'searchBox',
+        ]),
+        sidebarRightMembers: new Set([
+            'openThreadForPost',
+            'postMessage',
+            'getLastPost',
+        ]),
+    };
+}
+function collectMatches(content, pattern) {
+    const out = new Set();
+    for (const match of content.matchAll(pattern)) {
+        const value = match[1];
+        if (value) {
+            out.add(value);
+        }
+    }
+    return out;
+}
+function addNestedMethod(catalog, objectName, methodName) {
+    const methods = catalog.pwNestedMethods.get(objectName) || new Set();
+    methods.add(methodName);
+    catalog.pwNestedMethods.set(objectName, methods);
+}
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+function parseInitSetupBindings(content) {
+    const bindings = [];
+    for (const match of content.matchAll(/(?:const|let|var)\s*\{\s*([^}]+)\s*\}\s*=\s*await\s+pw\.initSetup\s*\(/g)) {
+        const raw = match[1];
+        if (!raw) {
+            continue;
+        }
+        for (const part of raw.split(',')) {
+            const cleaned = part.trim();
+            if (!cleaned) {
+                continue;
+            }
+            const [leftRaw, rightRaw] = cleaned.split(':');
+            const key = (leftRaw || '').trim();
+            const variableCandidate = (rightRaw || leftRaw || '').trim().split('=')[0]?.trim();
+            if (!key || !variableCandidate) {
+                continue;
+            }
+            bindings.push({ key, variable: variableCandidate });
+        }
+    }
+    return bindings;
+}
+function collectDestructuredInitSetupKeys(content) {
+    return new Set(parseInitSetupBindings(content).map((binding) => binding.key));
+}
+function addInitSetupVariableMethod(catalog, variable, methodName) {
+    const methods = catalog.initSetupVariableMethods.get(variable) || new Set();
+    methods.add(methodName);
+    catalog.initSetupVariableMethods.set(variable, methods);
+}
+function collectApiSurfaceFromContent(content, catalog) {
+    for (const prop of collectMatches(content, /\bpw\.([A-Za-z_][A-Za-z0-9_]*)\b/g)) {
+        catalog.pwProps.add(prop);
+    }
+    for (const match of content.matchAll(/\bpw\.([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*)\b/g)) {
+        const objectName = match[1];
+        const methodName = match[2];
+        if (!objectName || !methodName) {
+            continue;
+        }
+        addNestedMethod(catalog, objectName, methodName);
+    }
+    for (const method of collectMatches(content, /\bpw\.testBrowser\.([A-Za-z_][A-Za-z0-9_]*)\b/g)) {
+        catalog.testBrowserMethods.add(method);
+    }
+    for (const member of collectMatches(content, /\bchannelsPage\.([A-Za-z_][A-Za-z0-9_]*)\b/g)) {
+        catalog.channelsPageMembers.add(member);
+    }
+    for (const member of collectMatches(content, /\bchannelsPage\.sidebarRight\.([A-Za-z_][A-Za-z0-9_]*)\b/g)) {
+        catalog.sidebarRightMembers.add(member);
+    }
+    for (const binding of parseInitSetupBindings(content)) {
+        catalog.initSetupKeys.add(binding.key);
+        const methodPattern = new RegExp(`\\b${escapeRegExp(binding.variable)}\\.([A-Za-z_][A-Za-z0-9_]*)\\b`, 'g');
+        for (const method of collectMatches(content, methodPattern)) {
+            addInitSetupVariableMethod(catalog, binding.variable, method);
+        }
+    }
+}
+function buildApiSurfaceCatalog(testsRoot, seedFile) {
+    const catalog = createDefaultApiSurfaceCatalog();
+    const candidateRoots = [
+        (0, path_1.join)(testsRoot, 'specs'),
+        (0, path_1.join)(testsRoot, 'tests'),
+    ];
+    const files = [];
+    for (const root of candidateRoots) {
+        if (!(0, fs_1.existsSync)(root)) {
+            continue;
+        }
+        const stack = [root];
+        while (stack.length > 0) {
+            const current = stack.pop();
+            let entries;
+            try {
+                entries = (0, fs_1.readdirSync)(current, { withFileTypes: true });
+            }
+            catch {
+                continue;
+            }
+            for (const entry of entries) {
+                const full = (0, path_1.join)(current, entry.name);
+                if (entry.isDirectory()) {
+                    if (entry.name === 'node_modules' || entry.name === '.git' || entry.name === 'dist') {
+                        continue;
+                    }
+                    stack.push(full);
+                    continue;
+                }
+                if (!entry.isFile()) {
+                    continue;
+                }
+                if (!/\.(spec|test)\.[jt]sx?$/.test(entry.name)) {
+                    continue;
+                }
+                files.push(full);
+            }
+        }
+    }
+    const uniqueFiles = Array.from(new Set(files)).slice(0, 2500);
+    for (const filePath of uniqueFiles) {
+        try {
+            const content = (0, fs_1.readFileSync)(filePath, 'utf-8');
+            collectApiSurfaceFromContent(content, catalog);
+        }
+        catch {
+            continue;
+        }
+    }
+    const absoluteSeed = (0, path_1.join)(testsRoot, seedFile);
+    if ((0, fs_1.existsSync)(absoluteSeed)) {
+        try {
+            collectApiSurfaceFromContent((0, fs_1.readFileSync)(absoluteSeed, 'utf-8'), catalog);
+        }
+        catch {
+            // ignore seed read failures; defaults + catalog scan still apply
+        }
+    }
+    return catalog;
+}
+function validateGeneratedSpecContent(content, apiSurface) {
     const issues = [];
     if (/\btest\.describe\s*\(/.test(content)) {
         issues.push({
@@ -93,13 +372,64 @@ function validateGeneratedSpecContent(content) {
             message: 'Generated tests must use a single tag string, not a tag array.',
         });
     }
-    const hasTagString = /\btag\s*:\s*['"][^'"]+['"]/.test(content);
-    if (!hasTagString || !/@ai-assisted/.test(content)) {
+    const hasTagOption = /\btag\s*:\s*['"][^'"]+['"]/.test(content);
+    const hasTagInTitle = /\btest(?:\.\w+)?\s*\(\s*['"][^'"]*@ai-assisted[^'"]*['"]/.test(content);
+    if (!(hasTagOption || hasTagInTitle) || !/@ai-assisted/.test(content)) {
         issues.push({
             code: 'missing-tag',
-            message: "Generated tests must include a single '@ai-assisted' tag.",
+            message: "Generated tests must include '@ai-assisted' either as tag option or in test title.",
         });
     }
+    if (apiSurface) {
+        const unknownPwProps = Array.from(collectMatches(content, /\bpw\.([A-Za-z_][A-Za-z0-9_]*)\b/g)).filter((prop) => !apiSurface.pwProps.has(prop));
+        const unknownBrowserMethods = Array.from(collectMatches(content, /\bpw\.testBrowser\.([A-Za-z_][A-Za-z0-9_]*)\b/g)).filter((method) => !apiSurface.testBrowserMethods.has(method));
+        const unknownNestedPwMembers = [];
+        for (const match of content.matchAll(/\bpw\.([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*)\b/g)) {
+            const objectName = match[1];
+            const methodName = match[2];
+            if (!objectName || !methodName || objectName === 'testBrowser') {
+                continue;
+            }
+            const knownMethods = apiSurface.pwNestedMethods.get(objectName);
+            if (!knownMethods || !knownMethods.has(methodName)) {
+                unknownNestedPwMembers.push(`pw.${objectName}.${methodName}`);
+            }
+        }
+        const unknownChannelMembers = Array.from(collectMatches(content, /\bchannelsPage\.([A-Za-z_][A-Za-z0-9_]*)\b/g)).filter((member) => !apiSurface.channelsPageMembers.has(member));
+        const unknownSidebarMembers = Array.from(collectMatches(content, /\bchannelsPage\.sidebarRight\.([A-Za-z_][A-Za-z0-9_]*)\b/g)).filter((member) => !apiSurface.sidebarRightMembers.has(member));
+        const initSetupBindings = parseInitSetupBindings(content);
+        const unknownInitSetupKeys = initSetupBindings
+            .map((binding) => binding.key)
+            .filter((key) => !apiSurface.initSetupKeys.has(key));
+        const unknownInitSetupVariableMethods = [];
+        for (const binding of initSetupBindings) {
+            const knownMethods = apiSurface.initSetupVariableMethods.get(binding.variable);
+            if (!knownMethods || knownMethods.size === 0) {
+                continue;
+            }
+            const methodPattern = new RegExp(`\\b${escapeRegExp(binding.variable)}\\.([A-Za-z_][A-Za-z0-9_]*)\\b`, 'g');
+            for (const method of collectMatches(content, methodPattern)) {
+                if (!knownMethods.has(method)) {
+                    unknownInitSetupVariableMethods.push(`${binding.variable}.${method}`);
+                }
+            }
+        }
+        const unknown = [
+            ...unknownPwProps.map((value) => `pw.${value}`),
+            ...unknownBrowserMethods.map((value) => `pw.testBrowser.${value}`),
+            ...unknownNestedPwMembers,
+            ...unknownChannelMembers.map((value) => `channelsPage.${value}`),
+            ...unknownSidebarMembers.map((value) => `channelsPage.sidebarRight.${value}`),
+            ...unknownInitSetupKeys.map((value) => `pw.initSetup.{${value}}`),
+            ...unknownInitSetupVariableMethods,
+        ];
+        if (unknown.length > 0) {
+            issues.push({
+                code: 'unknown-api-surface',
+                message: `Generated test uses unknown API/page-object members: ${Array.from(new Set(unknown)).join(', ')}`,
+            });
+        }
+    }
     return issues;
 }
 function createNativePlaywrightSpec(flow, slug, strategy) {
@@ -130,10 +460,76 @@ function createNativePlaywrightSpec(flow, slug, strategy) {
             ...start,
             `  const parentMessage = \`ai-${slug}-parent-\${Date.now()}\`;`,
             '  await channelsPage.postMessage(parentMessage);',
-            '  await channelsPage.openAThread(parentMessage);',
+            '  const rootPost = await channelsPage.getLastPost();',
+            '  await rootPost.openAThread();',
             `  const replyMessage = \`ai-${slug}-reply-\${Date.now()}\`;`,
             '  await channelsPage.sidebarRight.postMessage(replyMessage);',
-            '  await expect(channelsPage.sidebarRight.getLastPost()).toContainText(replyMessage);',
+            '  const lastReply = await channelsPage.sidebarRight.getLastPost();',
+            '  await expect(lastReply.container).toContainText(replyMessage);',
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'lifecycle-channel') {
+        return [
+            ...header,
+            ...start,
+            `  const channelName = \`ai-${slug}-\${Date.now().toString().slice(-6)}\`;`,
+            "  await channelsPage.newChannel(channelName, 'O');",
+            '  await expect(channelsPage.page).toHaveURL(new RegExp(`/channels/${channelName}$`));',
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'channel-settings') {
+        return [
+            ...header,
+            ...start,
+            '  await channelsPage.openChannelSettings();',
+            "  await expect(channelsPage.page.getByRole('dialog', {name: 'Channel Settings'})).toBeVisible();",
+            "  await channelsPage.page.keyboard.press('Escape');",
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'channel-switch') {
+        return [
+            ...header,
+            ...start,
+            "  await channelsPage.goto(team.name, 'off-topic');",
+            "  await expect(channelsPage.page).toHaveURL(/\\/channels\\/off-topic$/);",
+            "  await expect(channelsPage.page.locator('#channelHeaderTitle')).toContainText(/off-topic/i);",
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'markdown-post') {
+        return [
+            ...header,
+            ...start,
+            `  const message = '**ai-${slug}-bold** _italic_';`,
+            '  await channelsPage.postMessage(message);',
+            '  const lastPost = await channelsPage.getLastPost();',
+            "  await expect(lastPost.container.locator('strong')).toBeVisible();",
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'mentions-post') {
+        return [
+            ...header,
+            ...start,
+            '  const mention = `@${user.username}`;',
+            '  await channelsPage.postMessage(`Ping ${mention}`);',
+            '  const lastPost = await channelsPage.getLastPost();',
+            '  await expect(lastPost.container).toContainText(mention);',
+            ...end,
+        ].join('\n');
+    }
+    if (strategy === 'realtime-post') {
+        return [
+            ...header,
+            ...start,
+            `  const message = \`ai-${slug}-realtime-\${Date.now()}\`;`,
+            '  await channelsPage.postMessage(message);',
+            '  const lastPost = await channelsPage.getLastPost();',
+            '  await expect(lastPost.container).toContainText(message);',
+            "  await expect(channelsPage.page.locator('#channel_view')).toBeVisible();",
             ...end,
         ].join('\n');
     }
@@ -160,11 +556,12 @@ function createNativePlaywrightSpec(flow, slug, strategy) {
         return [
             ...header,
             ...start,
-            `  const searchTerm = '${slug}'.slice(0, 20);`,
-            "  await channelsPage.page.keyboard.press('ControlOrMeta+K');",
-            '  await channelsPage.page.keyboard.type(searchTerm);',
-            "  await channelsPage.page.keyboard.press('Escape');",
-            '  await expect(channelsPage.page).toHaveURL(/\\/channels\\//);',
+            `  const searchTerm = \`ai-${slug}-\${Date.now().toString().slice(-6)}\`;`,
+            '  await channelsPage.postMessage(searchTerm);',
+            '  await channelsPage.globalHeader.openSearch();',
+            '  await channelsPage.searchBox.searchInput.fill(searchTerm);',
+            "  await channelsPage.page.keyboard.press('Enter');",
+            "  await expect(channelsPage.page.locator('#searchContainer')).toBeVisible();",
             ...end,
         ].join('\n');
     }
@@ -195,11 +592,11 @@ function summarizeCommandOutput(stdout, stderr) {
     const lines = combined.split('\n').slice(-20);
     return lines.join('\n').slice(0, 2000);
 }
-function runCommand(command, args, cwd) {
+function runCommand(command, args, cwd, timeoutMs = 60 * 60 * 1000) {
     const result = (0, child_process_1.spawnSync)(command, args, {
         cwd,
         encoding: 'utf-8',
-        timeout: 60 * 60 * 1000,
+        timeout: timeoutMs,
         stdio: 'pipe',
     });
     return {
@@ -209,6 +606,37 @@ function runCommand(command, args, cwd) {
         error: result.error ? result.error.message : undefined,
     };
 }
+function runPlaywrightRuntimeValidation(testsRoot, testFile, pipeline, playwrightBinary) {
+    if (!playwrightBinary) {
+        return {
+            status: 'failed',
+            detail: 'Playwright binary not found; cannot execute runtime validation.',
+        };
+    }
+    const relativeSpecPath = (0, utils_js_1.normalizePath)((0, path_1.relative)(testsRoot, testFile));
+    if (relativeSpecPath.startsWith('../') || relativeSpecPath.startsWith('..\\')) {
+        return {
+            status: 'failed',
+            detail: 'Generated spec path resolved outside testsRoot during runtime validation.',
+        };
+    }
+    const args = ['test', relativeSpecPath, '--workers', '1', '--retries', '0', '--max-failures', '1', '--reporter', 'line'];
+    if (pipeline.headless === false) {
+        args.push('--headed');
+    }
+    if (pipeline.project) {
+        args.push('--project', pipeline.project);
+    }
+    const commandResult = runCommand(playwrightBinary, args, testsRoot, 10 * 60 * 1000);
+    if (commandResult.status === 0) {
+        return { status: 'passed' };
+    }
+    const summary = summarizeCommandOutput(commandResult.stdout, commandResult.stderr);
+    return {
+        status: 'failed',
+        detail: summary || commandResult.error || `playwright test failed with status ${commandResult.status}`,
+    };
+}
 function runPlaywrightListValidation(testsRoot, testFile, pipeline, playwrightBinary) {
     if (!playwrightBinary) {
         return {
@@ -224,6 +652,9 @@ function runPlaywrightListValidation(testsRoot, testFile, pipeline, playwrightBi
         };
     }
     const args = ['test', '--list', relativeSpecPath];
+    if (pipeline.headless === false) {
+        args.push('--headed');
+    }
     if (pipeline.project) {
         args.push('--project', pipeline.project);
     }
@@ -243,7 +674,7 @@ function runPlaywrightListValidation(testsRoot, testFile, pipeline, playwrightBi
         detail: summary || commandResult.error || `playwright --list failed with status ${commandResult.status}`,
     };
 }
-function runPackageNativeFlow(testsRoot, flow, pipeline, outputDir, testFile, playwrightBinary) {
+function runPackageNativeFlow(testsRoot, flow, pipeline, outputDir, testFile, playwrightBinary, apiSurface) {
     const flowId = flow.id;
     const flowName = flow.name;
     const existingFile = (0, fs_1.existsSync)(testFile);
@@ -284,7 +715,7 @@ function runPackageNativeFlow(testsRoot, flow, pipeline, outputDir, testFile, pl
             wroteNewFile = true;
         }
         const currentContent = candidate.write ? candidate.content : (originalContent || '');
-        const qualityIssues = validateGeneratedSpecContent(currentContent);
+        const qualityIssues = validateGeneratedSpecContent(currentContent, apiSurface);
         if (qualityIssues.length > 0) {
             attempts.push(`${candidate.label}: ${qualityIssues.map((issue) => issue.message).join(' ')}`);
             if (pipeline.heal && i < candidates.length - 1) {
@@ -353,10 +784,10 @@ function runPackageNativeFlow(testsRoot, flow, pipeline, outputDir, testFile, pl
 }
 function runPackageNativePipeline(testsRoot, flows, pipeline, baseWarnings = []) {
     const warningSet = new Set(baseWarnings);
-    if (pipeline.mcp) {
-        warningSet.add('Package-native pipeline does not run Playwright MCP directly. Use follow-up heal workflows if MCP is required.');
-    }
+    const mcp = createMcpStatus('package-native', Boolean(pipeline.mcp));
     const playwrightBinary = pipeline.heal ? resolvePlaywrightBinary(testsRoot) : null;
+    const seedFile = resolveAgentSeedSpec(testsRoot) || 'specs/seed.spec.ts';
+    const apiSurface = buildApiSurfaceCatalog(testsRoot, seedFile);
     if (pipeline.heal && !playwrightBinary) {
         warningSet.add('Playwright binary was not found. Heal uses static quality checks without runtime compile validation.');
     }
@@ -364,7 +795,7 @@ function runPackageNativePipeline(testsRoot, flows, pipeline, baseWarnings = [])
     const outputBase = (0, path_1.resolve)(testsRoot, pipeline.outputDir || 'specs/functional/ai-assisted');
     if (!(0, utils_js_1.isPathWithinRoot)(testsRoot, outputBase)) {
         warningSet.add(`Pipeline outputDir resolves outside testsRoot and was blocked: ${pipeline.outputDir}`);
-        return { runner: 'unknown', results, warnings: Array.from(warningSet) };
+        return { runner: 'unknown', results, warnings: Array.from(warningSet), mcp: createMcpStatus('unknown', Boolean(pipeline.mcp)) };
     }
     for (const flow of flows) {
         if (flow.priority !== 'P0' && flow.priority !== 'P1') {
@@ -403,22 +834,26 @@ function runPackageNativePipeline(testsRoot, flows, pipeline, baseWarnings = [])
             });
             continue;
         }
-        results.push(runPackageNativeFlow(testsRoot, flow, pipeline, outputDir, testFile, playwrightBinary));
+        results.push(runPackageNativeFlow(testsRoot, flow, pipeline, outputDir, testFile, playwrightBinary, apiSurface));
     }
-    return { runner: 'package-native', results, warnings: Array.from(warningSet) };
+    return { runner: 'package-native', results, warnings: Array.from(warningSet), mcp };
 }
 function runTargetedSpecHeal(testsRoot, targets, pipeline) {
     const warnings = new Set();
     const results = [];
+    const mcp = createMcpStatus('package-native', Boolean(pipeline.mcp));
     if (targets.length === 0) {
         warnings.add('No targeted specs provided for heal.');
-        return {
+        return finalizePipelineSummary({
             runner: 'package-native',
             results,
             warnings: Array.from(warnings),
-        };
+            mcp,
+        });
     }
     const playwrightBinary = pipeline.heal ? resolvePlaywrightBinary(testsRoot) : null;
+    const seedFile = resolveAgentSeedSpec(testsRoot) || 'specs/seed.spec.ts';
+    const apiSurface = buildApiSurfaceCatalog(testsRoot, seedFile);
     if (pipeline.heal && !playwrightBinary) {
         warnings.add('Playwright binary was not found. Targeted heal uses static quality checks without runtime compile validation.');
     }
@@ -470,13 +905,14 @@ function runTargetedSpecHeal(testsRoot, targets, pipeline) {
             continue;
         }
         const syntheticFlow = buildSyntheticFlowFromSpecTarget(relativeSpecPath, target);
-        results.push(runPackageNativeFlow(testsRoot, syntheticFlow, pipeline, (0, utils_js_1.normalizePath)((0, path_1.dirname)(absoluteSpecPath)), absoluteSpecPath, playwrightBinary));
+        results.push(runPackageNativeFlow(testsRoot, syntheticFlow, pipeline, (0, utils_js_1.normalizePath)((0, path_1.dirname)(absoluteSpecPath)), absoluteSpecPath, playwrightBinary, apiSurface));
     }
-    return {
+    return finalizePipelineSummary({
         runner: 'package-native',
         results,
         warnings: Array.from(warnings),
-    };
+        mcp,
+    });
 }
 function findSpecFiles(root) {
     if (!(0, fs_1.existsSync)(root)) {
@@ -499,17 +935,387 @@ function findDisallowedDescribeFiles(root) {
     const files = findSpecFiles(root);
     return files.filter((file) => /\btest\.describe\s*\(/.test((0, fs_1.readFileSync)(file, 'utf-8')));
 }
+function hasCommand(command, cwd) {
+    const result = runCommand(command, ['--version'], cwd);
+    return result.status === 0;
+}
+function hasPlaywrightAgentDefinitions(testsRoot) {
+    const required = [
+        '.mcp.json',
+        '.claude/agents/playwright-test-planner.md',
+        '.claude/agents/playwright-test-generator.md',
+        '.claude/agents/playwright-test-healer.md',
+    ];
+    return required.every((path) => (0, fs_1.existsSync)((0, path_1.join)(testsRoot, path)));
+}
+function hasPlaywrightConfig(testsRoot) {
+    const candidates = [
+        'playwright.config.ts',
+        'playwright.config.js',
+        'playwright.config.mts',
+        'playwright.config.mjs',
+        'playwright.config.cts',
+        'playwright.config.cjs',
+    ];
+    return candidates.some((candidate) => (0, fs_1.existsSync)((0, path_1.join)(testsRoot, candidate)));
+}
+function bootstrapPlaywrightAgentDefinitions(testsRoot, pipeline) {
+    const args = ['playwright', 'init-agents', '--loop=claude', '--prompts'];
+    if (pipeline.project) {
+        args.push('--project', pipeline.project);
+    }
+    return runCommand('npx', args, testsRoot);
+}
+function resolveAgentSeedSpec(testsRoot) {
+    const preferred = (0, path_1.join)(testsRoot, 'specs', 'seed.spec.ts');
+    const specsRoot = (0, path_1.join)(testsRoot, 'specs');
+    const specFiles = findSpecFiles(specsRoot).filter((file) => !(0, utils_js_1.normalizePath)(file).includes('/functional/ai-assisted/'));
+    const scored = specFiles
+        .map((file) => {
+        const rel = (0, utils_js_1.normalizePath)((0, path_1.relative)(testsRoot, file));
+        const content = (0, fs_1.readFileSync)(file, 'utf-8');
+        let score = 0;
+        if (rel.endsWith('/seed.spec.ts')) {
+            // Generated default seed from init-agents is often a placeholder; prefer real tests.
+            if (!/generate code here/i.test(content)) {
+                score += 2;
+            }
+        }
+        if (content.includes('@mattermost/playwright-lib')) {
+            score += 8;
+        }
+        if (content.includes('pw.initSetup(')) {
+            score += 6;
+        }
+        if (content.includes('testBrowser.login(')) {
+            score += 4;
+        }
+        if (content.includes('channelsPage')) {
+            score += 2;
+        }
+        if (rel.includes('/functional/channels/')) {
+            score += 1;
+        }
+        return { rel, score };
+    })
+        .sort((a, b) => b.score - a.score);
+    if (scored.length > 0 && scored[0].score > 0) {
+        return scored[0].rel;
+    }
+    if ((0, fs_1.existsSync)(preferred)) {
+        return (0, utils_js_1.normalizePath)((0, path_1.relative)(testsRoot, preferred));
+    }
+    return null;
+}
+function buildPlaywrightAgentsPrompt(flow, seedFile, planFile, testFile, includeHealer) {
+    const linkedFiles = firstFlowFiles(flow).join(', ') || 'N/A';
+    const reasons = (flow.reasons || []).slice(0, 5).join(' | ') || 'N/A';
+    return [
+        'Use official Playwright Test agents (planner, generator, healer) to implement exactly one high-quality test for this flow.',
+        '',
+        `Flow ID: ${flow.id}`,
+        `Flow Name: ${flow.name}`,
+        `Priority: ${flow.priority}`,
+        `Linked files: ${linkedFiles}`,
+        `Risk reasons: ${reasons}`,
+        '',
+        'Workflow requirements:',
+        '1) Use #playwright-test-planner to explore and save a focused test plan.',
+        '2) Use #playwright-test-generator to generate one test from that plan.',
+        includeHealer
+            ? '3) Use #playwright-test-healer to run and fix that generated test.'
+            : '3) Skip runtime healing and focus on producing compile-ready test code.',
+        '',
+        `Seed file: ${seedFile}`,
+        `Plan file to save: ${planFile}`,
+        `Generated test file path (must be exact): ${testFile}`,
+        '',
+        'Quality constraints (must follow):',
+        '- The generated file must contain a standalone test() and must not use test.describe or test.only.',
+        '- Do not mark the test with test.fixme unless user explicitly requests skipping.',
+        "- The generated test must include a single tag string '@ai-assisted'.",
+        '- Match fixture/import style from the seed file. Prefer existing page-object APIs over raw brittle selectors.',
+        '- Only use `pw` and page-object methods that already exist in the seed/current specs (for example, do not invent APIs like `pw.mainClient.*`).',
+        '- Keep the scenario strictly aligned to the flow and linked files, not broad unrelated flows.',
+        '',
+        'At the end, return a short summary that includes the generated test file path and whether healing succeeded.',
+    ].join('\n');
+}
+function buildPlaywrightHealerPrompt(testFile, extra) {
+    const lines = [
+        'Heal this specific Playwright test file and keep edits minimal.',
+        `Target test file: ${testFile}`,
+        'Constraints:',
+        '- Do not use test.describe or test.only.',
+        "- Keep a single tag string '@ai-assisted'.",
+        '- Use only existing Mattermost Playwright fixture/page-object APIs; do not invent new `pw.*` clients or methods.',
+        '- Keep the test intent unchanged and focused.',
+        '',
+        'Run and fix this test until it compiles/passes, or mark test.fixme with a clear comment when behavior is truly broken.',
+    ];
+    if (extra) {
+        lines.push('', `Context: ${extra}`);
+    }
+    return lines.join('\n');
+}
+function runPlaywrightAgentsFlow(testsRoot, flow, pipeline, outputDir, preferredTestFile, seedFile, apiSurface, playwrightBinary) {
+    (0, fs_1.mkdirSync)(outputDir, { recursive: true });
+    const slug = toSafeSlug(flow.id);
+    const planFile = (0, utils_js_1.normalizePath)((0, path_1.relative)(testsRoot, (0, path_1.join)(outputDir, `${slug}.plan.md`)));
+    const targetTestFile = (0, utils_js_1.normalizePath)((0, path_1.relative)(testsRoot, preferredTestFile));
+    if (pipeline.dryRun) {
+        return {
+            flowId: flow.id,
+            flowName: flow.name,
+            generatedDir: outputDir,
+            generateStatus: 'skipped',
+            healStatus: pipeline.heal ? 'skipped' : undefined,
+        };
+    }
+    const prompt = buildPlaywrightAgentsPrompt(flow, seedFile, planFile, targetTestFile, Boolean(pipeline.heal));
+    const runArgs = [
+        '-p',
+        '--permission-mode',
+        'bypassPermissions',
+        '--mcp-config',
+        '.mcp.json',
+        '--add-dir',
+        testsRoot,
+        '--',
+        prompt,
+    ];
+    const runResult = runCommand('claude', runArgs, testsRoot);
+    if (runResult.status !== 0) {
+        return {
+            flowId: flow.id,
+            flowName: flow.name,
+            generatedDir: outputDir,
+            generateStatus: 'failed',
+            healStatus: pipeline.heal ? 'failed' : undefined,
+            error: summarizeCommandOutput(runResult.stdout, runResult.stderr) || runResult.error || 'Playwright agents run failed',
+        };
+    }
+    let actualTestFile = preferredTestFile;
+    if (!(0, fs_1.existsSync)(actualTestFile)) {
+        const candidates = findSpecFiles(outputDir);
+        if (candidates.length === 1) {
+            actualTestFile = candidates[0];
+        }
+    }
+    if (!(0, fs_1.existsSync)(actualTestFile)) {
+        return {
+            flowId: flow.id,
+            flowName: flow.name,
+            generatedDir: outputDir,
+            generateStatus: 'failed',
+            healStatus: pipeline.heal ? 'failed' : undefined,
+            error: `Playwright agents did not produce expected test file: ${targetTestFile}`,
+        };
+    }
+    const relativeActualTestFile = (0, utils_js_1.normalizePath)((0, path_1.relative)(testsRoot, actualTestFile));
+    let qualityIssues = validateGeneratedSpecContent((0, fs_1.readFileSync)(actualTestFile, 'utf-8'), apiSurface);
+    if (qualityIssues.length > 0 && pipeline.heal) {
+        const healResult = runCommand('claude', [
+            '-p',
+            '--permission-mode',
+            'bypassPermissions',
+            '--agent',
+            'playwright-test-healer',
+            '--mcp-config',
+            '.mcp.json',
+            '--add-dir',
+            testsRoot,
+            '--',
+            buildPlaywrightHealerPrompt(relativeActualTestFile, qualityIssues.map((issue) => issue.message).join(' | ')),
+        ], testsRoot);
+        if (healResult.status === 0 && (0, fs_1.existsSync)(actualTestFile)) {
+            qualityIssues = validateGeneratedSpecContent((0, fs_1.readFileSync)(actualTestFile, 'utf-8'), apiSurface);
+        }
+    }
+    if (qualityIssues.length > 0) {
+        return {
+            flowId: flow.id,
+            flowName: flow.name,
+            generatedDir: outputDir,
+            generateStatus: 'failed',
+            healStatus: pipeline.heal ? 'failed' : undefined,
+            error: `Playwright agents produced invalid test content: ${qualityIssues.map((issue) => issue.message).join(' | ')}`,
+        };
+    }
+    if (pipeline.heal) {
+        let compileValidation = runPlaywrightListValidation(testsRoot, actualTestFile, pipeline, playwrightBinary);
+        if (compileValidation.status === 'failed') {
+            const healResult = runCommand('claude', [
+                '-p',
+                '--permission-mode',
+                'bypassPermissions',
+                '--agent',
+                'playwright-test-healer',
+                '--mcp-config',
+                '.mcp.json',
+                '--add-dir',
+                testsRoot,
+                '--',
+                buildPlaywrightHealerPrompt(relativeActualTestFile, compileValidation.detail || 'playwright --list failed'),
+            ], testsRoot);
+            if (healResult.status === 0 && (0, fs_1.existsSync)(actualTestFile)) {
+                compileValidation = runPlaywrightListValidation(testsRoot, actualTestFile, pipeline, playwrightBinary);
+            }
+            if (compileValidation.status === 'failed') {
+                return {
+                    flowId: flow.id,
+                    flowName: flow.name,
+                    generatedDir: outputDir,
+                    generateStatus: 'failed',
+                    healStatus: 'failed',
+                    error: `Playwright agents compile validation failed: ${compileValidation.detail || 'playwright --list failed'}`,
+                };
+            }
+        }
+        let runtimeValidation = runPlaywrightRuntimeValidation(testsRoot, actualTestFile, pipeline, playwrightBinary);
+        if (runtimeValidation.status === 'failed') {
+            const healResult = runCommand('claude', [
+                '-p',
+                '--permission-mode',
+                'bypassPermissions',
+                '--agent',
+                'playwright-test-healer',
+                '--mcp-config',
+                '.mcp.json',
+                '--add-dir',
+                testsRoot,
+                '--',
+                buildPlaywrightHealerPrompt(relativeActualTestFile, runtimeValidation.detail || 'playwright runtime failed'),
+            ], testsRoot);
+            if (healResult.status === 0 && (0, fs_1.existsSync)(actualTestFile)) {
+                runtimeValidation = runPlaywrightRuntimeValidation(testsRoot, actualTestFile, pipeline, playwrightBinary);
+            }
+            if (runtimeValidation.status === 'failed') {
+                return {
+                    flowId: flow.id,
+                    flowName: flow.name,
+                    generatedDir: outputDir,
+                    generateStatus: 'failed',
+                    healStatus: 'failed',
+                    error: `Playwright agents runtime validation failed: ${runtimeValidation.detail || 'playwright test failed'}`,
+                };
+            }
+        }
+    }
+    return {
+        flowId: flow.id,
+        flowName: flow.name,
+        generatedDir: outputDir,
+        generateStatus: 'success',
+        healStatus: pipeline.heal ? 'success' : undefined,
+    };
+}
+function runPlaywrightAgentsPipeline(testsRoot, flows, pipeline) {
+    const warnings = [];
+    const results = [];
+    if (!hasCommand('claude', testsRoot)) {
+        warnings.push('Claude CLI is required for official Playwright planner/generator/healer execution but was not found.');
+        return { runner: 'unknown', results, warnings, mcp: createMcpStatus('unknown', true) };
+    }
+    if (!hasPlaywrightConfig(testsRoot)) {
+        warnings.push('Playwright config file not found in testsRoot; skipping official Playwright agents backend.');
+        return { runner: 'unknown', results, warnings, mcp: createMcpStatus('unknown', true) };
+    }
+    if (!hasPlaywrightAgentDefinitions(testsRoot)) {
+        const bootstrap = bootstrapPlaywrightAgentDefinitions(testsRoot, pipeline);
+        if (bootstrap.status !== 0) {
+            warnings.push(summarizeCommandOutput(bootstrap.stdout, bootstrap.stderr) ||
+                bootstrap.error ||
+                'Failed to initialize Playwright agents via `npx playwright init-agents`.');
+            return { runner: 'unknown', results, warnings, mcp: createMcpStatus('unknown', true) };
+        }
+    }
+    if (!hasPlaywrightAgentDefinitions(testsRoot)) {
+        warnings.push('Playwright agent definitions are missing after bootstrap.');
+        return { runner: 'unknown', results, warnings, mcp: createMcpStatus('unknown', true) };
+    }
+    const seedFile = resolveAgentSeedSpec(testsRoot);
+    if (!seedFile) {
+        warnings.push('No seed spec file found under specs/. Playwright planner cannot be initialized.');
+        return { runner: 'unknown', results, warnings, mcp: createMcpStatus('unknown', true) };
+    }
+    const playwrightBinary = pipeline.heal ? resolvePlaywrightBinary(testsRoot) : null;
+    const apiSurface = buildApiSurfaceCatalog(testsRoot, seedFile);
+    if (pipeline.heal && !playwrightBinary) {
+        warnings.push('Playwright binary was not found. Healer runtime validation may be limited.');
+    }
+    const outputBase = (0, path_1.resolve)(testsRoot, pipeline.outputDir || 'specs/functional/ai-assisted');
+    if (!(0, utils_js_1.isPathWithinRoot)(testsRoot, outputBase)) {
+        warnings.push(`Pipeline outputDir resolves outside testsRoot and was blocked: ${pipeline.outputDir}`);
+        return { runner: 'unknown', results, warnings, mcp: createMcpStatus('unknown', true) };
+    }
+    for (const flow of flows) {
+        if (flow.priority !== 'P0' && flow.priority !== 'P1') {
+            continue;
+        }
+        const slug = toSafeSlug(flow.id);
+        const outputDir = (0, utils_js_1.normalizePath)((0, path_1.join)(outputBase, slug));
+        if (!(0, utils_js_1.isPathWithinRoot)(testsRoot, outputDir)) {
+            results.push({
+                flowId: flow.id,
+                flowName: flow.name,
+                generatedDir: outputDir,
+                generateStatus: 'failed',
+                error: 'output directory resolves outside testsRoot',
+            });
+            continue;
+        }
+        const testFile = (0, utils_js_1.normalizePath)((0, path_1.join)(outputDir, `${slug}.spec.ts`));
+        if (!(0, utils_js_1.isPathWithinRoot)(testsRoot, testFile)) {
+            results.push({
+                flowId: flow.id,
+                flowName: flow.name,
+                generatedDir: outputDir,
+                generateStatus: 'failed',
+                error: 'generated test path resolves outside testsRoot',
+            });
+            continue;
+        }
+        results.push(runPlaywrightAgentsFlow(testsRoot, flow, pipeline, outputDir, testFile, seedFile, apiSurface, playwrightBinary));
+    }
+    return { runner: 'playwright-agents', results, warnings, mcp: createMcpStatus('playwright-agents', true) };
+}
 function runPlaywrightPipeline(testsRoot, flows, pipeline) {
+    const mcpFallbackWarnings = [];
+    if (pipeline.mcp) {
+        const agentsSummary = runPlaywrightAgentsPipeline(testsRoot, flows, pipeline);
+        if (agentsSummary.runner !== 'unknown' || agentsSummary.results.length > 0) {
+            return finalizePipelineSummary(agentsSummary);
+        }
+        if (!pipeline.mcpAllowFallback) {
+            const warnings = [
+                ...agentsSummary.warnings,
+                'Official Playwright MCP mode is strict; fallback generation is disabled unless pipeline.mcpAllowFallback=true.',
+            ];
+            return finalizePipelineSummary({
+                runner: 'unknown',
+                results: agentsSummary.results,
+                warnings,
+                mcp: createMcpStatus('unknown', true),
+            });
+        }
+        mcpFallbackWarnings.push(...agentsSummary.warnings);
+    }
     const cliPath = hasE2eTestGenCLI(testsRoot);
     if (!cliPath) {
-        return runPackageNativePipeline(testsRoot, flows, pipeline, ['e2e-test-gen-cli.ts not found; using package-native pipeline fallback.']);
+        return finalizePipelineSummary(runPackageNativePipeline(testsRoot, flows, pipeline, mcpFallbackWarnings));
     }
-    const warnings = [];
+    const warnings = [...mcpFallbackWarnings];
     const results = [];
     const outputBase = (0, path_1.resolve)(testsRoot, pipeline.outputDir || 'specs/functional/ai-assisted');
     if (!(0, utils_js_1.isPathWithinRoot)(testsRoot, outputBase)) {
         warnings.push(`Pipeline outputDir resolves outside testsRoot and was blocked: ${pipeline.outputDir}`);
-        return { runner: 'unknown', results, warnings };
+        return finalizePipelineSummary({
+            runner: 'unknown',
+            results,
+            warnings,
+            mcp: createMcpStatus('unknown', Boolean(pipeline.mcp)),
+        });
     }
     for (const flow of flows) {
         if (flow.priority !== 'P0' && flow.priority !== 'P1') {
@@ -605,5 +1411,10 @@ function runPlaywrightPipeline(testsRoot, flows, pipeline) {
             healStatus,
         });
     }
-    return { runner: 'e2e-test-gen', results, warnings };
+    return finalizePipelineSummary({
+        runner: 'e2e-test-gen',
+        results,
+        warnings,
+        mcp: createMcpStatus('e2e-test-gen', Boolean(pipeline.mcp)),
+    });
 }