@yasserkhanorg/e2e-agents 0.8.0 → 0.9.0

package/dist/cli.js

@@ -19,6 +19,9 @@ const traceability_capture_js_1 = require("./agent/traceability_capture.js");
 const pipeline_js_1 = require("./agent/pipeline.js");
 const playwright_report_js_1 = require("./agent/playwright_report.js");
 const orchestrator_js_1 = require("./pipeline/orchestrator.js");
+const provider_factory_js_1 = require("./provider_factory.js");
+const runner_js_1 = require("./agentic/runner.js");
+const api_surface_js_1 = require("./knowledge/api_surface.js");
 const CONFIG_CANDIDATES = ['e2e-ai-agents.config.json', '.e2e-ai-agents.config.json'];
 function findConfigUpwards(startDir) {
     if (!startDir) {
@@ -69,6 +72,7 @@ function printUsage() {
         '  e2e-ai-agents feedback --path <app-root> --feedback-input <json>',
         '  e2e-ai-agents traceability-capture --path <app-root> --traceability-report <json>',
         '  e2e-ai-agents traceability-ingest --path <app-root> --traceability-input <json>',
+        '  e2e-ai-agents generate [--scenarios <path|json>] [--max-attempts <n>] [--dry-run]',
         '  e2e-ai-agents analyze --path <app-root> [--tests-root <path>] [--since <ref>] [--generate] [--generate-output <dir>] [--heal] [--heal-report <json>]',
         '  e2e-ai-agents llm-health',
         '',
@@ -131,6 +135,8 @@ function printUsage() {
         '  --pr-base <branch>   PR base branch for finalize-generated-tests',
         '                      (auto-heal-pr defaults to base=master)',
         '  --dry-run            Preview actions without mutating git state',
+        '  --max-attempts <n>    Max fix attempts per scenario (default: 3)',
+        '  --scenarios <path|json>  Scenarios file/JSON for generate command',
         '  --apply               Apply data-testid patches and generate tests',
         '                        (legacy shortcut; prefer approve-and-generate)',
         '  --help                Show help',
@@ -146,6 +152,7 @@ function parseArgs(argv) {
         || command === 'plan'
         || command === 'heal'
         || command === 'suggest'
+        || command === 'generate'
         || command === 'finalize-generated-tests'
         || command === 'feedback'
         || command === 'traceability-capture'
@@ -161,6 +168,16 @@ function parseArgs(argv) {
             parsed.help = true;
             continue;
         }
+        if (arg === '--max-attempts' && next) {
+            parsed.maxAttempts = parseInt(next, 10);
+            i += 1;
+            continue;
+        }
+        if (arg === '--scenarios' && next) {
+            parsed.generateScenarios = next;
+            i += 1;
+            continue;
+        }
         if (arg === '--apply') {
             parsed.apply = true;
             continue;
@@ -981,6 +998,119 @@ async function main() {
         }
         return;
     }
+    if (args.command === 'generate') {
+        const reportRoot = config.testsRoot || config.path;
+        // Load scenarios from --scenarios flag or plan-report.json
+        let scenarios = [];
+        if (args.generateScenarios) {
+            let raw;
+            if ((0, fs_1.existsSync)(args.generateScenarios)) {
+                raw = JSON.parse((0, fs_1.readFileSync)(args.generateScenarios, 'utf-8'));
+            }
+            else {
+                raw = JSON.parse(args.generateScenarios);
+            }
+            if (!Array.isArray(raw)) {
+                // eslint-disable-next-line no-console
+                console.error('--scenarios must be a JSON array of ScenarioInput objects.');
+                process.exit(1);
+            }
+            for (const item of raw) {
+                if (!item.id || !item.name || !Array.isArray(item.scenarios) || !item.routeFamily || !item.priority) {
+                    // eslint-disable-next-line no-console
+                    console.error(`Invalid scenario: each must have id, name, scenarios[], routeFamily, priority.`);
+                    process.exit(1);
+                }
+            }
+            scenarios = raw;
+        }
+        else {
+            const planReportPath = (0, path_1.join)(reportRoot, '.e2e-ai-agents', 'plan-report.json');
+            if (!(0, fs_1.existsSync)(planReportPath)) {
+                // eslint-disable-next-line no-console
+                console.error('No plan report found. Run `plan` first or pass --scenarios.');
+                process.exit(1);
+            }
+            const planReport = JSON.parse((0, fs_1.readFileSync)(planReportPath, 'utf-8'));
+            scenarios = (planReport.gapDetails || []).map((gap) => ({
+                id: gap.id,
+                name: gap.id,
+                scenarios: gap.missingScenarios || gap.reasons || ['Verify core user flow'],
+                routeFamily: gap.id.split('.')[0] || gap.id,
+                priority: 'P1',
+            }));
+        }
+        if (scenarios.length === 0) {
+            // eslint-disable-next-line no-console
+            console.log('No scenarios to generate tests for.');
+            return;
+        }
+        let apiSurface;
+        try {
+            apiSurface = (0, api_surface_js_1.loadOrBuildApiSurface)(reportRoot, config.apiSurface);
+        }
+        catch {
+            // eslint-disable-next-line no-console
+            console.warn('Could not load API surface catalog. Generation will use generic selectors.');
+        }
+        const provider = await provider_factory_js_1.LLMProviderFactory.createFromEnv();
+        // eslint-disable-next-line no-console
+        console.log(`Generating tests for ${scenarios.length} scenario(s)...`);
+        const summary = await (0, runner_js_1.runAgenticGeneration)({
+            scenarios,
+            config: {
+                maxAttempts: args.maxAttempts || 3,
+                project: args.pipelineProject || 'chrome',
+                baseUrl: args.pipelineBaseUrl,
+                testTimeoutMs: 120000,
+                testsRoot: reportRoot,
+                dryRun: args.dryRun,
+            },
+            provider,
+            apiSurface,
+        });
+        // eslint-disable-next-line no-console
+        console.log(`\nAgentic Generation Summary:`);
+        // eslint-disable-next-line no-console
+        console.log(`  Generated: ${summary.totalGenerated}`);
+        // eslint-disable-next-line no-console
+        console.log(`  Passed:    ${summary.totalPassed}`);
+        // eslint-disable-next-line no-console
+        console.log(`  Failed:    ${summary.totalFailed}`);
+        // eslint-disable-next-line no-console
+        console.log(`  Attempts:  ${summary.totalAttempts}`);
+        // eslint-disable-next-line no-console
+        console.log(`  Duration:  ${(summary.durationMs / 1000).toFixed(1)}s`);
+        for (const result of summary.results) {
+            const icon = result.status === 'passed' ? 'PASS' : result.status === 'skipped' ? 'SKIP' : 'FAIL';
+            // eslint-disable-next-line no-console
+            console.log(`  [${icon}] ${result.scenarioSource} (${result.attempts} attempts)`);
+            if (result.status === 'passed' || result.status === 'skipped') {
+                // eslint-disable-next-line no-console
+                console.log(`     ${result.specPath}`);
+            }
+        }
+        if (summary.warnings.length > 0) {
+            // eslint-disable-next-line no-console
+            console.log(`\nWarnings:`);
+            for (const w of summary.warnings) {
+                // eslint-disable-next-line no-console
+                console.warn(`  - ${w}`);
+            }
+        }
+        const summaryDir = (0, path_1.join)(reportRoot, '.e2e-ai-agents');
+        if (!(0, fs_1.existsSync)(summaryDir)) {
+            (0, fs_1.mkdirSync)(summaryDir, { recursive: true });
+        }
+        const summaryPath = (0, path_1.join)(summaryDir, 'agentic-summary.json');
+        (0, fs_1.writeFileSync)(summaryPath, JSON.stringify(summary, null, 2), 'utf-8');
+        // eslint-disable-next-line no-console
+        console.log(`\nReport: ${summaryPath}`);
+        if (summary.totalFailed > 0) {
+            process.exit(1);
+        }
+        return;
+    }
     // eslint-disable-next-line no-console
     console.error(`Unknown command: ${args.command}`);
     printUsage();

package/dist/engine/plan_builder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"plan_builder.d.ts","sourceRoot":"","sources":["../../src/engine/plan_builder.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAC,YAAY,EAAkB,MAAM,oBAAoB,CAAC;AAEtE,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,oBAAoB,CAAC;AAG3D,OAAO,KAAK,EACR,UAAU,EACV,SAAS,EACT,kBAAkB,EAIrB,MAAM,kBAAkB,CAAC;AAE1B,YAAY,EAAC,UAAU,EAAE,SAAS,EAAE,kBAAkB,EAAC,CAAC;AAqOxD,wBAAgB,mBAAmB,CAC/B,MAAM,EAAE,YAAY,EACpB,cAAc,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,EACtC,YAAY,CAAC,EAAE,kBAAkB,GAClC,UAAU,CA0IZ;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,MAAM,CAMzE;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAuGhE;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,SAAiC,GAAG,MAAM,CAMvH"}
+{"version":3,"file":"plan_builder.d.ts","sourceRoot":"","sources":["../../src/engine/plan_builder.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAC,YAAY,EAAkB,MAAM,oBAAoB,CAAC;AAEtE,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,oBAAoB,CAAC;AAG3D,OAAO,KAAK,EACR,UAAU,EACV,SAAS,EACT,kBAAkB,EAIrB,MAAM,kBAAkB,CAAC;AAE1B,YAAY,EAAC,UAAU,EAAE,SAAS,EAAE,kBAAkB,EAAC,CAAC;AAqOxD,wBAAgB,mBAAmB,CAC/B,MAAM,EAAE,YAAY,EACpB,cAAc,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,EACtC,YAAY,CAAC,EAAE,kBAAkB,GAClC,UAAU,CA0IZ;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,MAAM,CAMzE;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAwGhE;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,SAAiC,GAAG,MAAM,CAMvH"}

package/dist/engine/plan_builder.js

@@ -392,13 +392,14 @@ function renderCiSummaryMarkdown(plan) {
         lines.push(`### 💡 New behavior detected in ${flowsWithAdvisory.length} covered feature${flowsWithAdvisory.length !== 1 ? 's' : ''} — consider adding tests`);
         lines.push('');
         for (const flow of flowsWithAdvisory) {
-            // Green [!TIP] box: just the name (always visible, compact)
-            lines.push(`> [!TIP]`);
-            lines.push(`> **${flow.name}** · ${flow.priority}`);
-            lines.push('');
-            // Specs + scenarios: collapsible below
-            const coverageSummary = flow.coveredBy.join(', ');
-            lines.push(`<details><summary>${coverageSummary} — click to see suggested scenarios</summary>`);
+            const specParts = [];
+            for (const s of flow.coveredBy) {
+                // Strip "N Playwright spec(s)" → "N PW" and "N Cypress spec(s)" → "N Cy"
+                specParts.push(s.replace(/ Playwright spec\(s\)/, ' PW').replace(/ Cypress spec\(s\)/, ' Cy'));
+            }
+            const specSummary = specParts.length > 0 ? ` — ${specParts.join(' · ')}` : '';
+            const scenarioCount = flow.advisoryScenarios.length;
+            lines.push(`<details><summary>💡 <strong>${flow.name}</strong> · ${flow.priority}${specSummary} · ${scenarioCount} scenario${scenarioCount !== 1 ? 's' : ''}</summary>`);
             lines.push('');
             for (const s of flow.advisoryScenarios) {
                 lines.push(`- [ ] ${s}`);

package/dist/esm/agentic/fix_loop.js

@@ -0,0 +1,90 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+export function buildFixPrompt(ctx) {
+    const isCompileError = ctx.failures.some((f) => f.testTitle === '(compile)');
+    const failuresBlock = ctx.failures.map((f) => {
+        const lines = [`  Test: ${f.testTitle}`, `  Error: ${f.error}`];
+        if (f.stack)
+            lines.push(`  Stack: ${f.stack}`);
+        if (f.line)
+            lines.push(`  Line: ${f.line}`);
+        if (f.expected)
+            lines.push(`  Expected: ${f.expected}`);
+        if (f.actual)
+            lines.push(`  Actual: ${f.actual}`);
+        return lines.join('\n');
+    }).join('\n\n');
+    const errorType = isCompileError ? 'COMPILE ERROR' : 'TEST FAILURE';
+    const apiBlock = ctx.apiSurfaceHint
+        ? `\nAVAILABLE PAGE OBJECT API:\n${ctx.apiSurfaceHint}\n`
+        : '';
+    return [
+        `Fix this Playwright E2E test. This is attempt ${ctx.attempt} of ${ctx.maxAttempts}.`,
+        '',
+        `## ${errorType}`,
+        '',
+        failuresBlock,
+        '',
+        '## CURRENT SPEC CODE',
+        '',
+        '```typescript',
+        ctx.specCode,
+        '```',
+        apiBlock,
+        '## RULES',
+        '',
+        '1. Import ONLY from "@mattermost/playwright-lib" — no "@playwright/test" imports.',
+        '2. Every test must call `await pw.initSetup()` first.',
+        '3. Use `await pw.testBrowser.login(user)` to log in.',
+        '4. Use ONLY page object methods listed in the API above. Do NOT invent methods.',
+        '5. If a method is not available, use `page.getByRole()` or `page.getByTestId()`.',
+        '6. For flaky/timing issues: add `await expect(locator).toBeVisible()` waits before interactions.',
+        '7. Keep the same test scenarios — fix the implementation, not the intent.',
+        '8. Return the COMPLETE fixed spec file — not a diff or partial code.',
+        '',
+        isCompileError
+            ? 'The file does not compile. Fix syntax errors, missing imports, or invalid method calls.'
+            : 'The test compiles but fails at runtime. Fix selectors, waits, or assertion logic.',
+        '',
+        'Return ONLY the complete TypeScript code. No explanations, no markdown fences (except wrapping the code).',
+    ].join('\n');
+}
+/**
+ * Extract fixed spec code from an LLM response.
+ * Returns null if the response doesn't contain valid test code.
+ */
+export function applyFix(llmResponse) {
+    let code = llmResponse.trim();
+    if (!code)
+        return null;
+    // Strip markdown fences
+    const fenced = code.match(/```(?:typescript|ts)?\s*([\s\S]*?)```/i);
+    if (fenced) {
+        code = fenced[1].trim();
+    }
+    // Must contain test( to be valid
+    if (!code.includes('test('))
+        return null;
+    // Ensure it has the right import
+    if (!code.includes('@mattermost/playwright-lib')) {
+        code = `import {expect, test} from '@mattermost/playwright-lib';\n\n${code}`;
+    }
+    return code;
+}
+/**
+ * Run one fix attempt: call LLM with failure context, return fixed code.
+ */
+export async function generateFix(provider, ctx) {
+    const prompt = buildFixPrompt(ctx);
+    const response = await provider.generateText(prompt, {
+        maxTokens: 8000,
+        temperature: 0.1,
+        timeout: 60000,
+        systemPrompt: 'You are an expert Playwright test fixer for Mattermost. Return only TypeScript code.',
+    });
+    const code = applyFix(response.text);
+    return {
+        code,
+        tokensUsed: { input: response.usage?.inputTokens || 0, output: response.usage?.outputTokens || 0 },
+    };
+}

package/dist/esm/agentic/playwright_runner.js

@@ -0,0 +1,161 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { spawnSync } from 'child_process';
+import { existsSync, readFileSync, mkdirSync, rmSync } from 'fs';
+import { join, resolve } from 'path';
+const MAX_STDOUT_CHARS = 8000;
+const MAX_ERROR_CHARS = 2000;
+const MAX_STACK_CHARS = 1000;
+function extractSpecs(suites) {
+    const specs = [];
+    for (const suite of suites) {
+        specs.push(...suite.specs);
+        if (suite.suites) {
+            specs.push(...extractSpecs(suite.suites));
+        }
+    }
+    return specs;
+}
+export function parsePlaywrightJsonReport(report, specPath) {
+    const failures = [];
+    const allSpecs = extractSpecs(report.suites);
+    let passed = 0;
+    let failed = 0;
+    for (const spec of allSpecs) {
+        if (spec.ok) {
+            passed++;
+        }
+        else {
+            failed++;
+            const lastResult = spec.tests[0]?.results?.at(-1);
+            failures.push({
+                testTitle: spec.title,
+                specPath,
+                error: (lastResult?.error?.message || 'Unknown error').slice(0, MAX_ERROR_CHARS),
+                stack: (lastResult?.error?.stack || '').slice(0, MAX_STACK_CHARS),
+                line: extractLineNumber(lastResult?.error?.stack),
+            });
+        }
+    }
+    return {
+        specPath,
+        passed,
+        failed,
+        flaky: report.stats.flaky || 0,
+        skipped: report.stats.skipped || 0,
+        failures,
+        stdout: '',
+        durationMs: report.stats.duration || 0,
+        compiled: true,
+    };
+}
+function extractLineNumber(stack) {
+    if (!stack)
+        return undefined;
+    const match = stack.match(/:(\d+):\d+\)?$/m);
+    return match ? parseInt(match[1], 10) : undefined;
+}
+/**
+ * Run a single Playwright spec file and return structured results.
+ * Uses a temp JSON reporter to get machine-readable output.
+ */
+export function runPlaywrightSpec(specPath, testsRoot, options) {
+    // SECURITY: Validate spec path is within testsRoot and has valid extension
+    const resolvedSpec = resolve(specPath);
+    const resolvedRoot = resolve(testsRoot);
+    if (!resolvedSpec.startsWith(resolvedRoot + '/')) {
+        throw new Error(`Security: spec path ${specPath} is outside testsRoot`);
+    }
+    if (!resolvedSpec.endsWith('.spec.ts') && !resolvedSpec.endsWith('.test.ts')) {
+        throw new Error(`Security: spec path must end in .spec.ts or .test.ts`);
+    }
+    const reportDir = join(testsRoot, '.e2e-ai-agents', 'agentic-reports');
+    if (!existsSync(reportDir)) {
+        mkdirSync(reportDir, { recursive: true });
+    }
+    const reportPath = join(reportDir, `report-${Date.now()}.json`);
+    const args = [
+        'playwright', 'test',
+        specPath,
+        '--reporter', 'json',
+        '--project', options.project || 'chrome',
+    ];
+    if (options.baseUrl) {
+        args.push('--config', 'playwright.config.ts');
+    }
+    const startTime = Date.now();
+    const result = spawnSync('npx', args, {
+        cwd: testsRoot,
+        encoding: 'utf-8',
+        timeout: options.timeoutMs || 120000,
+        maxBuffer: 2 * 1024 * 1024,
+        env: {
+            ...process.env,
+            PLAYWRIGHT_JSON_OUTPUT_NAME: reportPath,
+        },
+    });
+    const durationMs = Date.now() - startTime;
+    const stdout = (result.stdout || '').slice(0, MAX_STDOUT_CHARS);
+    const stderr = (result.stderr || '').slice(0, MAX_STDOUT_CHARS);
+    // Check for compile errors
+    if (stderr.includes('SyntaxError') || stderr.includes('Cannot find module') || stderr.includes('TypeError')) {
+        return {
+            specPath,
+            passed: 0,
+            failed: 1,
+            flaky: 0,
+            skipped: 0,
+            failures: [{
+                    testTitle: '(compile)',
+                    specPath,
+                    error: stderr.slice(0, MAX_ERROR_CHARS),
+                    stack: '',
+                }],
+            stdout: `${stdout}\n${stderr}`,
+            durationMs,
+            compiled: false,
+        };
+    }
+    // Try to parse JSON report from stdout (Playwright JSON reporter writes to stdout)
+    try {
+        const jsonReport = JSON.parse(stdout);
+        const parsed = parsePlaywrightJsonReport(jsonReport, specPath);
+        parsed.durationMs = durationMs;
+        parsed.stdout = stdout;
+        return parsed;
+    }
+    catch {
+        // Fallback: try the file-based report
+    }
+    // Try file-based report
+    if (existsSync(reportPath)) {
+        try {
+            const jsonReport = JSON.parse(readFileSync(reportPath, 'utf-8'));
+            const parsed = parsePlaywrightJsonReport(jsonReport, specPath);
+            parsed.durationMs = durationMs;
+            parsed.stdout = stdout;
+            try {
+                rmSync(reportPath);
+            }
+            catch { /* ignore */ }
+            return parsed;
+        }
+        catch {
+            // Fallback to exit code
+        }
+    }
+    // Last resort: use exit code
+    return {
+        specPath,
+        passed: result.status === 0 ? 1 : 0,
+        failed: result.status === 0 ? 0 : 1,
+        flaky: 0,
+        skipped: 0,
+        failures: result.status !== 0
+            ? [{ testTitle: '(unknown)', specPath, error: stderr.slice(0, MAX_ERROR_CHARS), stack: '' }]
+            : [],
+        stdout,
+        durationMs,
+        compiled: !stderr.includes('Error'),
+    };
+}

package/dist/esm/agentic/runner.js

@@ -0,0 +1,204 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { dirname, join, resolve } from 'path';
+import { runPlaywrightSpec } from './playwright_runner.js';
+import { generateFix } from './fix_loop.js';
+import { parseGenerationResponse } from '../prompts/generation.js';
+import { formatApiSurfaceForPrompt } from '../knowledge/api_surface.js';
+function buildGeneratePrompt(scenario, apiSurfaceHint) {
+    const scenariosBlock = scenario.scenarios
+        .map((s, i) => `  ${i + 1}. ${s}`)
+        .join('\n');
+    return [
+        'Generate a Mattermost Playwright E2E test file.',
+        '',
+        `FLOW: ${scenario.name}`,
+        `Route Family: ${scenario.routeFamily}`,
+        `Priority: ${scenario.priority}`,
+        scenario.evidence ? `Evidence: ${scenario.evidence}` : '',
+        '',
+        'SCENARIOS TO IMPLEMENT:',
+        scenariosBlock,
+        '',
+        'AVAILABLE PAGE OBJECTS AND METHODS:',
+        apiSurfaceHint || 'Use page.getByRole() or page.getByTestId() for selectors.',
+        '',
+        'MANDATORY RULES:',
+        '1. Import ONLY from "@mattermost/playwright-lib" — no other test framework imports.',
+        '2. Every test must call `await pw.initSetup()` first.',
+        '3. Use `await pw.testBrowser.login(user)` to log in — never hardcode credentials.',
+        '4. Use ONLY page object methods listed above. Do NOT invent methods.',
+        '5. If a method is not available, use `page.getByRole()` or `page.getByTestId()`.',
+        `6. Tag every test: {tag: '@${scenario.routeFamily}'}`,
+        '7. Write one test per scenario with a descriptive name.',
+        '8. Use `expect` from "@mattermost/playwright-lib".',
+        '9. Include the copyright header.',
+        '10. NEVER fabricate test IDs (MM-TXXXX). Use descriptive names only.',
+        '',
+        'EXAMPLE STRUCTURE:',
+        '```typescript',
+        '// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.',
+        '// See LICENSE.txt for license information.',
+        '',
+        "import {expect, test} from '@mattermost/playwright-lib';",
+        '',
+        'test(',
+        "    'user can post a message in channel',",
+        `    {tag: '@${scenario.routeFamily}'},`,
+        '    async ({pw}) => {',
+        '        const {user} = await pw.initSetup();',
+        '        const {channelsPage} = await pw.testBrowser.login(user);',
+        '        await channelsPage.goto();',
+        '        await channelsPage.toBeVisible();',
+        '        // test steps...',
+        '    },',
+        ');',
+        '```',
+        '',
+        'Return ONLY the TypeScript code. No explanations.',
+    ].filter(Boolean).join('\n');
+}
+function resolveSpecPath(scenario, testsRoot) {
+    let specPath;
+    if (scenario.targetSpec) {
+        specPath = join(testsRoot, scenario.targetSpec);
+    }
+    else {
+        const safeName = scenario.id.replace(/[^a-zA-Z0-9_-]/g, '_').toLowerCase();
+        const outputDir = join(testsRoot, 'specs', 'functional', 'ai-assisted');
+        specPath = join(outputDir, `${safeName}.spec.ts`);
+    }
+    // SECURITY: Prevent path traversal
+    const resolved = resolve(specPath);
+    const resolvedRoot = resolve(testsRoot);
+    if (!resolved.startsWith(resolvedRoot + '/') && resolved !== resolvedRoot) {
+        throw new Error(`Path traversal blocked: ${specPath} resolves outside testsRoot`);
+    }
+    if (!resolved.endsWith('.spec.ts') && !resolved.endsWith('.test.ts')) {
+        throw new Error(`Invalid spec path: must end in .spec.ts or .test.ts`);
+    }
+    return specPath;
+}
+async function generateInitialSpec(provider, scenario, specPath, apiSurfaceHint) {
+    const prompt = buildGeneratePrompt(scenario, apiSurfaceHint);
+    const response = await provider.generateText(prompt, {
+        maxTokens: 8000,
+        temperature: 0.1,
+        timeout: 60000,
+        systemPrompt: 'You are an expert Playwright test writer for Mattermost. Return only TypeScript code.',
+    });
+    // Reuse existing parsing logic from prompts/generation.ts
+    const parsed = parseGenerationResponse(response.text, specPath, 'create_spec', scenario.id);
+    return parsed?.code ?? null;
+}
+async function runSingleScenario(scenario, options) {
+    const { config, provider } = options;
+    const warnings = [];
+    const specPath = resolveSpecPath(scenario, config.testsRoot);
+    // Build API surface hint
+    let apiHint = options.apiSurfaceHint || '';
+    if (!apiHint && options.apiSurface) {
+        const allClassNames = options.apiSurface.pageObjects.map((po) => po.className);
+        apiHint = formatApiSurfaceForPrompt(options.apiSurface, allClassNames);
+    }
+    // Step 1: Generate initial spec
+    let specCode;
+    try {
+        specCode = await generateInitialSpec(provider, scenario, specPath, apiHint);
+    }
+    catch (error) {
+        const msg = error instanceof Error ? error.message : String(error);
+        warnings.push(`Generation failed for ${scenario.id}: ${msg}`);
+        return { specPath, scenarioSource: scenario.id, status: 'failed', attempts: 0, warnings };
+    }
+    if (!specCode) {
+        warnings.push(`LLM returned invalid code for ${scenario.id}`);
+        return { specPath, scenarioSource: scenario.id, status: 'failed', attempts: 0, warnings };
+    }
+    // Write the spec file
+    const dir = dirname(specPath);
+    if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+    }
+    writeFileSync(specPath, specCode, 'utf-8');
+    // Dry run: skip execution
+    if (config.dryRun) {
+        return { specPath, scenarioSource: scenario.id, status: 'skipped', attempts: 0, warnings };
+    }
+    // Step 2: Run -> Fix loop
+    let lastRun;
+    for (let attempt = 1; attempt <= config.maxAttempts; attempt++) {
+        lastRun = runPlaywrightSpec(specPath, config.testsRoot, {
+            project: config.project,
+            baseUrl: config.baseUrl,
+            timeoutMs: config.testTimeoutMs,
+        });
+        // All passed!
+        if (lastRun.failed === 0 && lastRun.compiled) {
+            return {
+                specPath,
+                scenarioSource: scenario.id,
+                status: 'passed',
+                attempts: attempt,
+                finalRun: lastRun,
+                warnings,
+            };
+        }
+        // If this is the last attempt, don't try to fix
+        if (attempt >= config.maxAttempts) {
+            break;
+        }
+        // Step 3: Fix
+        const currentCode = readFileSync(specPath, 'utf-8');
+        try {
+            const fixResult = await generateFix(provider, {
+                specCode: currentCode,
+                failures: lastRun.failures,
+                attempt,
+                maxAttempts: config.maxAttempts,
+                apiSurfaceHint: apiHint,
+            });
+            if (fixResult.code) {
+                writeFileSync(specPath, fixResult.code, 'utf-8');
+            }
+            else {
+                warnings.push(`Fix attempt ${attempt} returned invalid code for ${scenario.id}`);
+            }
+        }
+        catch (error) {
+            const msg = error instanceof Error ? error.message : String(error);
+            warnings.push(`Fix attempt ${attempt} failed for ${scenario.id}: ${msg}`);
+        }
+    }
+    return {
+        specPath,
+        scenarioSource: scenario.id,
+        status: lastRun?.compiled === false ? 'compile-error' : 'max-attempts',
+        attempts: config.maxAttempts,
+        finalRun: lastRun,
+        warnings,
+    };
+}
+export async function runAgenticGeneration(options) {
+    const startTime = Date.now();
+    const results = [];
+    const warnings = [];
+    for (const scenario of options.scenarios) {
+        const result = await runSingleScenario(scenario, options);
+        results.push(result);
+        warnings.push(...result.warnings);
+    }
+    const totalPassed = results.filter((r) => r.status === 'passed').length;
+    const totalFailed = results.filter((r) => r.status !== 'passed' && r.status !== 'skipped').length;
+    const totalAttempts = results.reduce((sum, r) => sum + r.attempts, 0);
+    return {
+        results,
+        totalGenerated: results.length,
+        totalPassed,
+        totalFailed,
+        totalAttempts,
+        durationMs: Date.now() - startTime,
+        warnings,
+    };
+}

package/dist/esm/agentic/types.js

@@ -0,0 +1,3 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+export {};