@yasserkhanorg/e2e-agents 0.3.3 → 0.3.5

package/dist/esm/agent/pipeline.js

@@ -11,6 +11,43 @@ function createMcpStatus(backend, requested) {
         backend,
     };
 }
+function classifyPipelineFailure(result) {
+    if (result.failureCategory || result.failureCode) {
+        return result;
+    }
+    if (!result.error) {
+        return result;
+    }
+    const errorText = result.error.toLowerCase();
+    if (errorText.includes('etimedout') || errorText.includes('timed out')) {
+        return { ...result, failureCategory: 'environment', failureCode: 'mcp_timeout' };
+    }
+    if (errorText.includes('outside testsroot')) {
+        return { ...result, failureCategory: 'path-safety', failureCode: 'path_outside_tests_root' };
+    }
+    if (errorText.includes('playwright binary') || errorText.includes('not found')) {
+        return { ...result, failureCategory: 'environment', failureCode: 'dependency_missing' };
+    }
+    if (errorText.includes('compile validation')) {
+        return { ...result, failureCategory: 'validation', failureCode: 'compile_validation_failed' };
+    }
+    if (errorText.includes('runtime validation') || errorText.includes('playwright test failed')) {
+        return { ...result, failureCategory: 'runtime', failureCode: 'runtime_validation_failed' };
+    }
+    if (errorText.includes('quality checks failed') || errorText.includes('invalid test content')) {
+        return { ...result, failureCategory: 'quality', failureCode: 'quality_guard_failed' };
+    }
+    if (errorText.includes('generate failed') || errorText.includes('did not produce expected test file')) {
+        return { ...result, failureCategory: 'generation', failureCode: 'generation_failed' };
+    }
+    return { ...result, failureCategory: 'unknown', failureCode: 'unknown' };
+}
+function finalizePipelineSummary(summary) {
+    return {
+        ...summary,
+        results: summary.results.map(classifyPipelineFailure),
+    };
+}
 function hasE2eTestGenCLI(testsRoot) {
     const cliPath = join(testsRoot, 'e2e-test-gen-cli.ts');
     return existsSync(cliPath) ? cliPath : null;
@@ -114,6 +151,15 @@ function buildNativeStrategyOrder(flow) {
     return Array.from(new Set(strategies));
 }
 function createDefaultApiSurfaceCatalog() {
+    const pwNestedMethods = new Map();
+    pwNestedMethods.set('apiClient', new Set([
+        'createPost',
+        'createDirectChannel',
+        'createChannel',
+        'getChannels',
+        'getChannelByName',
+        'getPostsSince',
+    ]));
     return {
         pwProps: new Set([
             'initSetup',
@@ -123,7 +169,20 @@ function createDefaultApiSurfaceCatalog() {
             'apiCreateChannel',
             'apiCreateUser',
             'apiLogin',
+            'apiClient',
+        ]),
+        pwNestedMethods,
+        initSetupKeys: new Set([
+            'user',
+            'team',
+            'adminClient',
+            'adminUser',
+            'adminConfig',
+            'userClient',
+            'offTopicUrl',
+            'townSquareUrl',
         ]),
+        initSetupVariableMethods: new Map(),
         testBrowserMethods: new Set([
             'login',
             'openNewBrowserContext',
@@ -157,10 +216,57 @@ function collectMatches(content, pattern) {
     }
     return out;
 }
+function addNestedMethod(catalog, objectName, methodName) {
+    const methods = catalog.pwNestedMethods.get(objectName) || new Set();
+    methods.add(methodName);
+    catalog.pwNestedMethods.set(objectName, methods);
+}
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+function parseInitSetupBindings(content) {
+    const bindings = [];
+    for (const match of content.matchAll(/(?:const|let|var)\s*\{\s*([^}]+)\s*\}\s*=\s*await\s+pw\.initSetup\s*\(/g)) {
+        const raw = match[1];
+        if (!raw) {
+            continue;
+        }
+        for (const part of raw.split(',')) {
+            const cleaned = part.trim();
+            if (!cleaned) {
+                continue;
+            }
+            const [leftRaw, rightRaw] = cleaned.split(':');
+            const key = (leftRaw || '').trim();
+            const variableCandidate = (rightRaw || leftRaw || '').trim().split('=')[0]?.trim();
+            if (!key || !variableCandidate) {
+                continue;
+            }
+            bindings.push({ key, variable: variableCandidate });
+        }
+    }
+    return bindings;
+}
+function collectDestructuredInitSetupKeys(content) {
+    return new Set(parseInitSetupBindings(content).map((binding) => binding.key));
+}
+function addInitSetupVariableMethod(catalog, variable, methodName) {
+    const methods = catalog.initSetupVariableMethods.get(variable) || new Set();
+    methods.add(methodName);
+    catalog.initSetupVariableMethods.set(variable, methods);
+}
 function collectApiSurfaceFromContent(content, catalog) {
     for (const prop of collectMatches(content, /\bpw\.([A-Za-z_][A-Za-z0-9_]*)\b/g)) {
         catalog.pwProps.add(prop);
     }
+    for (const match of content.matchAll(/\bpw\.([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*)\b/g)) {
+        const objectName = match[1];
+        const methodName = match[2];
+        if (!objectName || !methodName) {
+            continue;
+        }
+        addNestedMethod(catalog, objectName, methodName);
+    }
     for (const method of collectMatches(content, /\bpw\.testBrowser\.([A-Za-z_][A-Za-z0-9_]*)\b/g)) {
         catalog.testBrowserMethods.add(method);
     }
@@ -170,6 +276,13 @@ function collectApiSurfaceFromContent(content, catalog) {
     for (const member of collectMatches(content, /\bchannelsPage\.sidebarRight\.([A-Za-z_][A-Za-z0-9_]*)\b/g)) {
         catalog.sidebarRightMembers.add(member);
     }
+    for (const binding of parseInitSetupBindings(content)) {
+        catalog.initSetupKeys.add(binding.key);
+        const methodPattern = new RegExp(`\\b${escapeRegExp(binding.variable)}\\.([A-Za-z_][A-Za-z0-9_]*)\\b`, 'g');
+        for (const method of collectMatches(content, methodPattern)) {
+            addInitSetupVariableMethod(catalog, binding.variable, method);
+        }
+    }
 }
 function buildApiSurfaceCatalog(testsRoot, seedFile) {
     const catalog = createDefaultApiSurfaceCatalog();
@@ -266,16 +379,66 @@ function validateGeneratedSpecContent(content, apiSurface) {
             message: "Generated tests must include '@ai-assisted' either as tag option or in test title.",
         });
     }
+    if (/\bsystemConsolePage\.toBeVisible\s*\(/.test(content)) {
+        issues.push({
+            code: 'fragile-system-console-visibility',
+            message: 'Avoid systemConsolePage.toBeVisible(); it relies on legacy backstage navigation that may be absent.',
+        });
+    }
+    const fragileSelectors = [
+        '.backstage-navbar',
+        '.admin-console__wrapper',
+        '.left-panel',
+        '.panel-card',
+    ].filter((selector) => content.includes(selector));
+    if (fragileSelectors.length > 0) {
+        issues.push({
+            code: 'fragile-selector',
+            message: `Avoid brittle class selectors in generated tests: ${Array.from(new Set(fragileSelectors)).join(', ')}`,
+        });
+    }
     if (apiSurface) {
         const unknownPwProps = Array.from(collectMatches(content, /\bpw\.([A-Za-z_][A-Za-z0-9_]*)\b/g)).filter((prop) => !apiSurface.pwProps.has(prop));
         const unknownBrowserMethods = Array.from(collectMatches(content, /\bpw\.testBrowser\.([A-Za-z_][A-Za-z0-9_]*)\b/g)).filter((method) => !apiSurface.testBrowserMethods.has(method));
+        const unknownNestedPwMembers = [];
+        for (const match of content.matchAll(/\bpw\.([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*)\b/g)) {
+            const objectName = match[1];
+            const methodName = match[2];
+            if (!objectName || !methodName || objectName === 'testBrowser') {
+                continue;
+            }
+            const knownMethods = apiSurface.pwNestedMethods.get(objectName);
+            if (!knownMethods || !knownMethods.has(methodName)) {
+                unknownNestedPwMembers.push(`pw.${objectName}.${methodName}`);
+            }
+        }
         const unknownChannelMembers = Array.from(collectMatches(content, /\bchannelsPage\.([A-Za-z_][A-Za-z0-9_]*)\b/g)).filter((member) => !apiSurface.channelsPageMembers.has(member));
         const unknownSidebarMembers = Array.from(collectMatches(content, /\bchannelsPage\.sidebarRight\.([A-Za-z_][A-Za-z0-9_]*)\b/g)).filter((member) => !apiSurface.sidebarRightMembers.has(member));
+        const initSetupBindings = parseInitSetupBindings(content);
+        const unknownInitSetupKeys = initSetupBindings
+            .map((binding) => binding.key)
+            .filter((key) => !apiSurface.initSetupKeys.has(key));
+        const unknownInitSetupVariableMethods = [];
+        for (const binding of initSetupBindings) {
+            const knownMethods = apiSurface.initSetupVariableMethods.get(binding.variable);
+            if (!knownMethods || knownMethods.size === 0) {
+                continue;
+            }
+            const methodPattern = new RegExp(`\\b${escapeRegExp(binding.variable)}\\.([A-Za-z_][A-Za-z0-9_]*)\\b`, 'g');
+            for (const method of collectMatches(content, methodPattern)) {
+                if (!knownMethods.has(method)) {
+                    unknownInitSetupVariableMethods.push(`${binding.variable}.${method}`);
+                }
+            }
+        }
         const unknown = [
             ...unknownPwProps.map((value) => `pw.${value}`),
             ...unknownBrowserMethods.map((value) => `pw.testBrowser.${value}`),
+            ...unknownNestedPwMembers,
             ...unknownChannelMembers.map((value) => `channelsPage.${value}`),
             ...unknownSidebarMembers.map((value) => `channelsPage.sidebarRight.${value}`),
+            ...unknownInitSetupKeys.map((value) => `pw.initSetup.{${value}}`),
+            ...unknownInitSetupVariableMethods,
         ];
         if (unknown.length > 0) {
             issues.push({
@@ -446,11 +609,11 @@ function summarizeCommandOutput(stdout, stderr) {
     const lines = combined.split('\n').slice(-20);
     return lines.join('\n').slice(0, 2000);
 }
-function runCommand(command, args, cwd) {
+function runCommand(command, args, cwd, timeoutMs = 60 * 60 * 1000) {
     const result = spawnSync(command, args, {
         cwd,
         encoding: 'utf-8',
-        timeout: 60 * 60 * 1000,
+        timeout: timeoutMs,
         stdio: 'pipe',
     });
     return {
@@ -460,6 +623,73 @@ function runCommand(command, args, cwd) {
         error: result.error ? result.error.message : undefined,
     };
 }
+function resolveMcpCommandTimeoutMs(pipeline) {
+    const value = pipeline.mcpCommandTimeoutMs;
+    if (typeof value !== 'number' || !Number.isFinite(value)) {
+        return 180000;
+    }
+    return Math.max(60000, Math.min(15 * 60 * 1000, Math.round(value)));
+}
+function resolveMcpRetries(pipeline) {
+    const value = pipeline.mcpRetries;
+    if (typeof value !== 'number' || !Number.isFinite(value)) {
+        return 1;
+    }
+    return Math.max(0, Math.min(5, Math.round(value)));
+}
+function isRetryableMcpFailure(result) {
+    const haystack = [result.error || '', result.stderr || '', result.stdout || ''].join('\n').toLowerCase();
+    return haystack.includes('etimedout') ||
+        haystack.includes('timed out') ||
+        haystack.includes('econnreset') ||
+        haystack.includes('429') ||
+        haystack.includes('rate limit') ||
+        haystack.includes('temporar');
+}
+function runCommandWithRetries(command, args, cwd, timeoutMs, retries) {
+    let result = runCommand(command, args, cwd, timeoutMs);
+    for (let attempt = 1; attempt <= retries; attempt += 1) {
+        if (result.status === 0) {
+            return result;
+        }
+        if (!isRetryableMcpFailure(result)) {
+            return result;
+        }
+        result = runCommand(command, args, cwd, timeoutMs);
+    }
+    return result;
+}
+function runPlaywrightRuntimeValidation(testsRoot, testFile, pipeline, playwrightBinary) {
+    if (!playwrightBinary) {
+        return {
+            status: 'failed',
+            detail: 'Playwright binary not found; cannot execute runtime validation.',
+        };
+    }
+    const relativeSpecPath = normalizePath(relative(testsRoot, testFile));
+    if (relativeSpecPath.startsWith('../') || relativeSpecPath.startsWith('..\\')) {
+        return {
+            status: 'failed',
+            detail: 'Generated spec path resolved outside testsRoot during runtime validation.',
+        };
+    }
+    const args = ['test', relativeSpecPath, '--workers', '1', '--retries', '0', '--max-failures', '1', '--reporter', 'line'];
+    if (pipeline.headless === false) {
+        args.push('--headed');
+    }
+    if (pipeline.project) {
+        args.push('--project', pipeline.project);
+    }
+    const commandResult = runCommand(playwrightBinary, args, testsRoot, 10 * 60 * 1000);
+    if (commandResult.status === 0) {
+        return { status: 'passed' };
+    }
+    const summary = summarizeCommandOutput(commandResult.stdout, commandResult.stderr);
+    return {
+        status: 'failed',
+        detail: summary || commandResult.error || `playwright test failed with status ${commandResult.status}`,
+    };
+}
 function runPlaywrightListValidation(testsRoot, testFile, pipeline, playwrightBinary) {
     if (!playwrightBinary) {
         return {
@@ -475,6 +705,9 @@ function runPlaywrightListValidation(testsRoot, testFile, pipeline, playwrightBi
         };
     }
     const args = ['test', '--list', relativeSpecPath];
+    if (pipeline.headless === false) {
+        args.push('--headed');
+    }
     if (pipeline.project) {
         args.push('--project', pipeline.project);
     }
@@ -664,12 +897,12 @@ export function runTargetedSpecHeal(testsRoot, targets, pipeline) {
     const mcp = createMcpStatus('package-native', Boolean(pipeline.mcp));
     if (targets.length === 0) {
         warnings.add('No targeted specs provided for heal.');
-        return {
+        return finalizePipelineSummary({
             runner: 'package-native',
             results,
             warnings: Array.from(warnings),
             mcp,
-        };
+        });
     }
     const playwrightBinary = pipeline.heal ? resolvePlaywrightBinary(testsRoot) : null;
     const seedFile = resolveAgentSeedSpec(testsRoot) || 'specs/seed.spec.ts';
@@ -727,12 +960,12 @@ export function runTargetedSpecHeal(testsRoot, targets, pipeline) {
         const syntheticFlow = buildSyntheticFlowFromSpecTarget(relativeSpecPath, target);
         results.push(runPackageNativeFlow(testsRoot, syntheticFlow, pipeline, normalizePath(dirname(absoluteSpecPath)), absoluteSpecPath, playwrightBinary, apiSurface));
     }
-    return {
+    return finalizePipelineSummary({
         runner: 'package-native',
         results,
         warnings: Array.from(warnings),
         mcp,
-    };
+    });
 }
 function findSpecFiles(root) {
     if (!existsSync(root)) {
@@ -779,12 +1012,12 @@ function hasPlaywrightConfig(testsRoot) {
     ];
     return candidates.some((candidate) => existsSync(join(testsRoot, candidate)));
 }
-function bootstrapPlaywrightAgentDefinitions(testsRoot, pipeline) {
+function bootstrapPlaywrightAgentDefinitions(testsRoot, pipeline, timeoutMs) {
     const args = ['playwright', 'init-agents', '--loop=claude', '--prompts'];
     if (pipeline.project) {
         args.push('--project', pipeline.project);
     }
-    return runCommand('npx', args, testsRoot);
+    return runCommand('npx', args, testsRoot, timeoutMs);
 }
 function resolveAgentSeedSpec(testsRoot) {
     const preferred = join(testsRoot, 'specs', 'seed.spec.ts');
@@ -856,6 +1089,8 @@ function buildPlaywrightAgentsPrompt(flow, seedFile, planFile, testFile, include
         "- The generated test must include a single tag string '@ai-assisted'.",
         '- Match fixture/import style from the seed file. Prefer existing page-object APIs over raw brittle selectors.',
         '- Only use `pw` and page-object methods that already exist in the seed/current specs (for example, do not invent APIs like `pw.mainClient.*`).',
+        '- For system-console/admin flows, avoid `systemConsolePage.toBeVisible()` and brittle class selectors (`.backstage-navbar`, `.admin-console__wrapper`, `.left-panel`, `.panel-card`).',
+        '- Prefer stable assertions using URL patterns, test IDs, roles, labels, and established page-object methods.',
         '- Keep the scenario strictly aligned to the flow and linked files, not broad unrelated flows.',
         '',
         'At the end, return a short summary that includes the generated test file path and whether healing succeeded.',
@@ -869,6 +1104,8 @@ function buildPlaywrightHealerPrompt(testFile, extra) {
         '- Do not use test.describe or test.only.',
         "- Keep a single tag string '@ai-assisted'.",
         '- Use only existing Mattermost Playwright fixture/page-object APIs; do not invent new `pw.*` clients or methods.',
+        '- Avoid `systemConsolePage.toBeVisible()` and brittle class selectors (`.backstage-navbar`, `.admin-console__wrapper`, `.left-panel`, `.panel-card`).',
+        '- Prefer stable checks with URL/test IDs/roles/page-object methods.',
         '- Keep the test intent unchanged and focused.',
         '',
         'Run and fix this test until it compiles/passes, or mark test.fixme with a clear comment when behavior is truly broken.',
@@ -878,11 +1115,69 @@ function buildPlaywrightHealerPrompt(testFile, extra) {
     }
     return lines.join('\n');
 }
-function runPlaywrightAgentsFlow(testsRoot, flow, pipeline, outputDir, preferredTestFile, seedFile, apiSurface, playwrightBinary, allowRuntimeHeal) {
+function runPlaywrightAgentsFlow(testsRoot, flow, pipeline, outputDir, preferredTestFile, seedFile, apiSurface, playwrightBinary, mcpTimeoutMs, mcpRetries) {
     mkdirSync(outputDir, { recursive: true });
     const slug = toSafeSlug(flow.id);
     const planFile = normalizePath(relative(testsRoot, join(outputDir, `${slug}.plan.md`)));
+    const absolutePlanFile = join(testsRoot, planFile);
     const targetTestFile = normalizePath(relative(testsRoot, preferredTestFile));
+    const existingSpecFiles = findSpecFiles(outputDir);
+    const existingSpecSnapshots = new Map();
+    for (const specFile of existingSpecFiles) {
+        try {
+            existingSpecSnapshots.set(specFile, readFileSync(specFile, 'utf-8'));
+        }
+        catch {
+            continue;
+        }
+    }
+    const originalPlanContent = existsSync(absolutePlanFile) ? readFileSync(absolutePlanFile, 'utf-8') : null;
+    const restoreArtifactsOnFailure = () => {
+        for (const currentSpecFile of findSpecFiles(outputDir)) {
+            const originalSpecContent = existingSpecSnapshots.get(currentSpecFile);
+            if (originalSpecContent === undefined) {
+                rmSync(currentSpecFile, { force: true });
+                continue;
+            }
+            try {
+                if (readFileSync(currentSpecFile, 'utf-8') !== originalSpecContent) {
+                    writeFileSync(currentSpecFile, originalSpecContent, 'utf-8');
+                }
+            }
+            catch {
+                // best-effort restore only
+            }
+        }
+        for (const [specFile, originalSpecContent] of existingSpecSnapshots.entries()) {
+            if (!existsSync(specFile)) {
+                writeFileSync(specFile, originalSpecContent, 'utf-8');
+            }
+        }
+        if (originalPlanContent === null) {
+            rmSync(absolutePlanFile, { force: true });
+        }
+        else {
+            try {
+                if (!existsSync(absolutePlanFile) || readFileSync(absolutePlanFile, 'utf-8') !== originalPlanContent) {
+                    writeFileSync(absolutePlanFile, originalPlanContent, 'utf-8');
+                }
+            }
+            catch {
+                // best-effort restore only
+            }
+        }
+    };
+    const failFlow = (error) => {
+        restoreArtifactsOnFailure();
+        return {
+            flowId: flow.id,
+            flowName: flow.name,
+            generatedDir: outputDir,
+            generateStatus: 'failed',
+            healStatus: pipeline.heal ? 'failed' : undefined,
+            error,
+        };
+    };
     if (pipeline.dryRun) {
         return {
             flowId: flow.id,
@@ -892,11 +1187,14 @@ function runPlaywrightAgentsFlow(testsRoot, flow, pipeline, outputDir, preferred
             healStatus: pipeline.heal ? 'skipped' : undefined,
         };
     }
-    const prompt = buildPlaywrightAgentsPrompt(flow, seedFile, planFile, targetTestFile, allowRuntimeHeal);
+    const prompt = buildPlaywrightAgentsPrompt(flow, seedFile, planFile, targetTestFile, Boolean(pipeline.heal));
     const runArgs = [
         '-p',
         '--permission-mode',
         'bypassPermissions',
+        '--setting-sources',
+        'project,local',
+        '--strict-mcp-config',
         '--mcp-config',
         '.mcp.json',
         '--add-dir',
@@ -904,16 +1202,9 @@ function runPlaywrightAgentsFlow(testsRoot, flow, pipeline, outputDir, preferred
         '--',
         prompt,
     ];
-    const runResult = runCommand('claude', runArgs, testsRoot);
+    const runResult = runCommandWithRetries('claude', runArgs, testsRoot, mcpTimeoutMs, mcpRetries);
     if (runResult.status !== 0) {
-        return {
-            flowId: flow.id,
-            flowName: flow.name,
-            generatedDir: outputDir,
-            generateStatus: 'failed',
-            healStatus: pipeline.heal ? 'failed' : undefined,
-            error: summarizeCommandOutput(runResult.stdout, runResult.stderr) || runResult.error || 'Playwright agents run failed',
-        };
+        return failFlow(summarizeCommandOutput(runResult.stdout, runResult.stderr) || runResult.error || 'Playwright agents run failed');
     }
     let actualTestFile = preferredTestFile;
     if (!existsSync(actualTestFile)) {
@@ -923,22 +1214,18 @@ function runPlaywrightAgentsFlow(testsRoot, flow, pipeline, outputDir, preferred
         }
     }
     if (!existsSync(actualTestFile)) {
-        return {
-            flowId: flow.id,
-            flowName: flow.name,
-            generatedDir: outputDir,
-            generateStatus: 'failed',
-            healStatus: pipeline.heal ? 'failed' : undefined,
-            error: `Playwright agents did not produce expected test file: ${targetTestFile}`,
-        };
+        return failFlow(`Playwright agents did not produce expected test file: ${targetTestFile}`);
     }
     const relativeActualTestFile = normalizePath(relative(testsRoot, actualTestFile));
     let qualityIssues = validateGeneratedSpecContent(readFileSync(actualTestFile, 'utf-8'), apiSurface);
-    if (qualityIssues.length > 0 && allowRuntimeHeal) {
-        const healResult = runCommand('claude', [
+    if (qualityIssues.length > 0 && pipeline.heal) {
+        const healResult = runCommandWithRetries('claude', [
             '-p',
             '--permission-mode',
             'bypassPermissions',
+            '--setting-sources',
+            'project,local',
+            '--strict-mcp-config',
             '--agent',
             'playwright-test-healer',
             '--mcp-config',
@@ -947,28 +1234,24 @@ function runPlaywrightAgentsFlow(testsRoot, flow, pipeline, outputDir, preferred
             testsRoot,
             '--',
             buildPlaywrightHealerPrompt(relativeActualTestFile, qualityIssues.map((issue) => issue.message).join(' | ')),
-        ], testsRoot);
+        ], testsRoot, mcpTimeoutMs, mcpRetries);
         if (healResult.status === 0 && existsSync(actualTestFile)) {
             qualityIssues = validateGeneratedSpecContent(readFileSync(actualTestFile, 'utf-8'), apiSurface);
         }
     }
     if (qualityIssues.length > 0) {
-        return {
-            flowId: flow.id,
-            flowName: flow.name,
-            generatedDir: outputDir,
-            generateStatus: 'failed',
-            healStatus: pipeline.heal ? 'failed' : undefined,
-            error: `Playwright agents produced invalid test content: ${qualityIssues.map((issue) => issue.message).join(' | ')}`,
-        };
+        return failFlow(`Playwright agents produced invalid test content: ${qualityIssues.map((issue) => issue.message).join(' | ')}`);
     }
-    if (allowRuntimeHeal) {
-        let validation = runPlaywrightListValidation(testsRoot, actualTestFile, pipeline, playwrightBinary);
-        if (validation.status === 'failed') {
-            const healResult = runCommand('claude', [
+    if (pipeline.heal) {
+        let compileValidation = runPlaywrightListValidation(testsRoot, actualTestFile, pipeline, playwrightBinary);
+        if (compileValidation.status === 'failed') {
+            const healResult = runCommandWithRetries('claude', [
                 '-p',
                 '--permission-mode',
                 'bypassPermissions',
+                '--setting-sources',
+                'project,local',
+                '--strict-mcp-config',
                 '--agent',
                 'playwright-test-healer',
                 '--mcp-config',
@@ -976,20 +1259,38 @@ function runPlaywrightAgentsFlow(testsRoot, flow, pipeline, outputDir, preferred
                 '--add-dir',
                 testsRoot,
                 '--',
-                buildPlaywrightHealerPrompt(relativeActualTestFile, validation.detail || 'playwright --list failed'),
-            ], testsRoot);
+                buildPlaywrightHealerPrompt(relativeActualTestFile, compileValidation.detail || 'playwright --list failed'),
+            ], testsRoot, mcpTimeoutMs, mcpRetries);
             if (healResult.status === 0 && existsSync(actualTestFile)) {
-                validation = runPlaywrightListValidation(testsRoot, actualTestFile, pipeline, playwrightBinary);
+                compileValidation = runPlaywrightListValidation(testsRoot, actualTestFile, pipeline, playwrightBinary);
             }
-            if (validation.status === 'failed') {
-                return {
-                    flowId: flow.id,
-                    flowName: flow.name,
-                    generatedDir: outputDir,
-                    generateStatus: 'failed',
-                    healStatus: 'failed',
-                    error: `Playwright agents heal failed: ${validation.detail || 'playwright validation failed'}`,
-                };
+            if (compileValidation.status === 'failed') {
+                return failFlow(`Playwright agents compile validation failed: ${compileValidation.detail || 'playwright --list failed'}`);
+            }
+        }
+        let runtimeValidation = runPlaywrightRuntimeValidation(testsRoot, actualTestFile, pipeline, playwrightBinary);
+        if (runtimeValidation.status === 'failed') {
+            const healResult = runCommandWithRetries('claude', [
+                '-p',
+                '--permission-mode',
+                'bypassPermissions',
+                '--setting-sources',
+                'project,local',
+                '--strict-mcp-config',
+                '--agent',
+                'playwright-test-healer',
+                '--mcp-config',
+                '.mcp.json',
+                '--add-dir',
+                testsRoot,
+                '--',
+                buildPlaywrightHealerPrompt(relativeActualTestFile, runtimeValidation.detail || 'playwright runtime failed'),
+            ], testsRoot, mcpTimeoutMs, mcpRetries);
+            if (healResult.status === 0 && existsSync(actualTestFile)) {
+                runtimeValidation = runPlaywrightRuntimeValidation(testsRoot, actualTestFile, pipeline, playwrightBinary);
+            }
+            if (runtimeValidation.status === 'failed') {
+                return failFlow(`Playwright agents runtime validation failed: ${runtimeValidation.detail || 'playwright test failed'}`);
             }
         }
     }
@@ -1004,6 +1305,8 @@ function runPlaywrightAgentsFlow(testsRoot, flow, pipeline, outputDir, preferred
 function runPlaywrightAgentsPipeline(testsRoot, flows, pipeline) {
     const warnings = [];
     const results = [];
+    const mcpTimeoutMs = resolveMcpCommandTimeoutMs(pipeline);
+    const mcpRetries = resolveMcpRetries(pipeline);
     if (!hasCommand('claude', testsRoot)) {
         warnings.push('Claude CLI is required for official Playwright planner/generator/healer execution but was not found.');
         return { runner: 'unknown', results, warnings, mcp: createMcpStatus('unknown', true) };
@@ -1013,7 +1316,7 @@ function runPlaywrightAgentsPipeline(testsRoot, flows, pipeline) {
         return { runner: 'unknown', results, warnings, mcp: createMcpStatus('unknown', true) };
     }
     if (!hasPlaywrightAgentDefinitions(testsRoot)) {
-        const bootstrap = bootstrapPlaywrightAgentDefinitions(testsRoot, pipeline);
+        const bootstrap = bootstrapPlaywrightAgentDefinitions(testsRoot, pipeline, mcpTimeoutMs);
         if (bootstrap.status !== 0) {
             warnings.push(summarizeCommandOutput(bootstrap.stdout, bootstrap.stderr) ||
                 bootstrap.error ||
@@ -1030,15 +1333,11 @@ function runPlaywrightAgentsPipeline(testsRoot, flows, pipeline) {
         warnings.push('No seed spec file found under specs/. Playwright planner cannot be initialized.');
         return { runner: 'unknown', results, warnings, mcp: createMcpStatus('unknown', true) };
     }
-    const allowRuntimeHeal = Boolean(pipeline.heal && pipeline.baseUrl);
-    const playwrightBinary = allowRuntimeHeal ? resolvePlaywrightBinary(testsRoot) : null;
+    const playwrightBinary = pipeline.heal ? resolvePlaywrightBinary(testsRoot) : null;
     const apiSurface = buildApiSurfaceCatalog(testsRoot, seedFile);
-    if (allowRuntimeHeal && !playwrightBinary) {
+    if (pipeline.heal && !playwrightBinary) {
         warnings.push('Playwright binary was not found. Healer runtime validation may be limited.');
     }
-    if (pipeline.heal && !allowRuntimeHeal) {
-        warnings.push('Skipping runtime healer in official MCP mode because no --pipeline-base-url was provided.');
-    }
     const outputBase = resolve(testsRoot, pipeline.outputDir || 'specs/functional/ai-assisted');
     if (!isPathWithinRoot(testsRoot, outputBase)) {
         warnings.push(`Pipeline outputDir resolves outside testsRoot and was blocked: ${pipeline.outputDir}`);
@@ -1071,41 +1370,78 @@ function runPlaywrightAgentsPipeline(testsRoot, flows, pipeline) {
             });
             continue;
         }
-        results.push(runPlaywrightAgentsFlow(testsRoot, flow, pipeline, outputDir, testFile, seedFile, apiSurface, playwrightBinary, allowRuntimeHeal));
+        results.push(runPlaywrightAgentsFlow(testsRoot, flow, pipeline, outputDir, testFile, seedFile, apiSurface, playwrightBinary, mcpTimeoutMs, mcpRetries));
+        if (pipeline.mcpOnly && results[results.length - 1].generateStatus === 'failed') {
+            warnings.push(`MCP-only mode: stopping after first failed flow (${flow.id}).`);
+            break;
+        }
     }
     return { runner: 'playwright-agents', results, warnings, mcp: createMcpStatus('playwright-agents', true) };
 }
 export function runPlaywrightPipeline(testsRoot, flows, pipeline) {
     const mcpFallbackWarnings = [];
+    // MCP-only mode requires MCP to be enabled
+    if (pipeline.mcpOnly && !pipeline.mcp) {
+        const warnings = [
+            '❌ MCP-Only Mode Error: --pipeline-mcp-only requires --pipeline-mcp flag',
+            'Run with: npm run gen:tests -- --pipeline-mcp',
+        ];
+        return finalizePipelineSummary({
+            runner: 'unknown',
+            results: [],
+            warnings,
+            mcp: createMcpStatus('unknown', false),
+        });
+    }
     if (pipeline.mcp) {
         const agentsSummary = runPlaywrightAgentsPipeline(testsRoot, flows, pipeline);
         if (agentsSummary.runner !== 'unknown' || agentsSummary.results.length > 0) {
-            return agentsSummary;
+            return finalizePipelineSummary(agentsSummary);
+        }
+        // Handle strict MCP-only mode
+        if (pipeline.mcpOnly) {
+            const warnings = [
+                ...agentsSummary.warnings,
+                '❌ MCP-Only Mode Error: Claude Code CLI / Playwright Agents MCP is not available',
+                'Please install Claude Code CLI: brew install anthropic/tap/claude-code',
+                'Or check that the MCP server is properly configured',
+            ];
+            return finalizePipelineSummary({
+                runner: 'unknown',
+                results: agentsSummary.results,
+                warnings,
+                mcp: createMcpStatus('unknown', true),
+            });
         }
         if (!pipeline.mcpAllowFallback) {
             const warnings = [
                 ...agentsSummary.warnings,
                 'Official Playwright MCP mode is strict; fallback generation is disabled unless pipeline.mcpAllowFallback=true.',
             ];
-            return {
+            return finalizePipelineSummary({
                 runner: 'unknown',
                 results: agentsSummary.results,
                 warnings,
                 mcp: createMcpStatus('unknown', true),
-            };
+            });
         }
         mcpFallbackWarnings.push(...agentsSummary.warnings);
     }
     const cliPath = hasE2eTestGenCLI(testsRoot);
     if (!cliPath) {
-        return runPackageNativePipeline(testsRoot, flows, pipeline, mcpFallbackWarnings);
+        return finalizePipelineSummary(runPackageNativePipeline(testsRoot, flows, pipeline, mcpFallbackWarnings));
     }
     const warnings = [...mcpFallbackWarnings];
     const results = [];
     const outputBase = resolve(testsRoot, pipeline.outputDir || 'specs/functional/ai-assisted');
     if (!isPathWithinRoot(testsRoot, outputBase)) {
         warnings.push(`Pipeline outputDir resolves outside testsRoot and was blocked: ${pipeline.outputDir}`);
-        return { runner: 'unknown', results, warnings, mcp: createMcpStatus('unknown', Boolean(pipeline.mcp)) };
+        return finalizePipelineSummary({
+            runner: 'unknown',
+            results,
+            warnings,
+            mcp: createMcpStatus('unknown', Boolean(pipeline.mcp)),
+        });
     }
     for (const flow of flows) {
         if (flow.priority !== 'P0' && flow.priority !== 'P1') {
@@ -1201,5 +1537,10 @@ export function runPlaywrightPipeline(testsRoot, flows, pipeline) {
             healStatus,
         });
     }
-    return { runner: 'e2e-test-gen', results, warnings, mcp: createMcpStatus('e2e-test-gen', Boolean(pipeline.mcp)) };
+    return finalizePipelineSummary({
+        runner: 'e2e-test-gen',
+        results,
+        warnings,
+        mcp: createMcpStatus('e2e-test-gen', Boolean(pipeline.mcp)),
+    });
 }