@yasserkhanorg/e2e-agents 0.8.0 → 0.9.0

package/dist/agentic/fix_loop.d.ts

@@ -0,0 +1,26 @@
+import type { LLMProvider } from '../provider_interface.js';
+import type { TestFailure } from './types.js';
+export interface FixPromptContext {
+    specCode: string;
+    failures: TestFailure[];
+    attempt: number;
+    maxAttempts: number;
+    apiSurfaceHint?: string;
+}
+export declare function buildFixPrompt(ctx: FixPromptContext): string;
+/**
+ * Extract fixed spec code from an LLM response.
+ * Returns null if the response doesn't contain valid test code.
+ */
+export declare function applyFix(llmResponse: string): string | null;
+/**
+ * Run one fix attempt: call LLM with failure context, return fixed code.
+ */
+export declare function generateFix(provider: LLMProvider, ctx: FixPromptContext): Promise<{
+    code: string | null;
+    tokensUsed: {
+        input: number;
+        output: number;
+    };
+}>;
+//# sourceMappingURL=fix_loop.d.ts.map

package/dist/agentic/fix_loop.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fix_loop.d.ts","sourceRoot":"","sources":["../../src/agentic/fix_loop.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,YAAY,CAAC;AAE5C,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,gBAAgB,GAAG,MAAM,CA+C5D;AAED;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAmB3D;AAED;;GAEG;AACH,wBAAsB,WAAW,CAC7B,QAAQ,EAAE,WAAW,EACrB,GAAG,EAAE,gBAAgB,GACtB,OAAO,CAAC;IAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,UAAU,EAAE;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAC,CAAA;CAAC,CAAC,CAe7E"}

package/dist/agentic/fix_loop.js

@@ -0,0 +1,95 @@
+"use strict";
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildFixPrompt = buildFixPrompt;
+exports.applyFix = applyFix;
+exports.generateFix = generateFix;
+function buildFixPrompt(ctx) {
+    const isCompileError = ctx.failures.some((f) => f.testTitle === '(compile)');
+    const failuresBlock = ctx.failures.map((f) => {
+        const lines = [`  Test: ${f.testTitle}`, `  Error: ${f.error}`];
+        if (f.stack)
+            lines.push(`  Stack: ${f.stack}`);
+        if (f.line)
+            lines.push(`  Line: ${f.line}`);
+        if (f.expected)
+            lines.push(`  Expected: ${f.expected}`);
+        if (f.actual)
+            lines.push(`  Actual: ${f.actual}`);
+        return lines.join('\n');
+    }).join('\n\n');
+    const errorType = isCompileError ? 'COMPILE ERROR' : 'TEST FAILURE';
+    const apiBlock = ctx.apiSurfaceHint
+        ? `\nAVAILABLE PAGE OBJECT API:\n${ctx.apiSurfaceHint}\n`
+        : '';
+    return [
+        `Fix this Playwright E2E test. This is attempt ${ctx.attempt} of ${ctx.maxAttempts}.`,
+        '',
+        `## ${errorType}`,
+        '',
+        failuresBlock,
+        '',
+        '## CURRENT SPEC CODE',
+        '',
+        '```typescript',
+        ctx.specCode,
+        '```',
+        apiBlock,
+        '## RULES',
+        '',
+        '1. Import ONLY from "@mattermost/playwright-lib" — no "@playwright/test" imports.',
+        '2. Every test must call `await pw.initSetup()` first.',
+        '3. Use `await pw.testBrowser.login(user)` to log in.',
+        '4. Use ONLY page object methods listed in the API above. Do NOT invent methods.',
+        '5. If a method is not available, use `page.getByRole()` or `page.getByTestId()`.',
+        '6. For flaky/timing issues: add `await expect(locator).toBeVisible()` waits before interactions.',
+        '7. Keep the same test scenarios — fix the implementation, not the intent.',
+        '8. Return the COMPLETE fixed spec file — not a diff or partial code.',
+        '',
+        isCompileError
+            ? 'The file does not compile. Fix syntax errors, missing imports, or invalid method calls.'
+            : 'The test compiles but fails at runtime. Fix selectors, waits, or assertion logic.',
+        '',
+        'Return ONLY the complete TypeScript code. No explanations, no markdown fences (except wrapping the code).',
+    ].join('\n');
+}
+/**
+ * Extract fixed spec code from an LLM response.
+ * Returns null if the response doesn't contain valid test code.
+ */
+function applyFix(llmResponse) {
+    let code = llmResponse.trim();
+    if (!code)
+        return null;
+    // Strip markdown fences
+    const fenced = code.match(/```(?:typescript|ts)?\s*([\s\S]*?)```/i);
+    if (fenced) {
+        code = fenced[1].trim();
+    }
+    // Must contain test( to be valid
+    if (!code.includes('test('))
+        return null;
+    // Ensure it has the right import
+    if (!code.includes('@mattermost/playwright-lib')) {
+        code = `import {expect, test} from '@mattermost/playwright-lib';\n\n${code}`;
+    }
+    return code;
+}
+/**
+ * Run one fix attempt: call LLM with failure context, return fixed code.
+ */
+async function generateFix(provider, ctx) {
+    const prompt = buildFixPrompt(ctx);
+    const response = await provider.generateText(prompt, {
+        maxTokens: 8000,
+        temperature: 0.1,
+        timeout: 60000,
+        systemPrompt: 'You are an expert Playwright test fixer for Mattermost. Return only TypeScript code.',
+    });
+    const code = applyFix(response.text);
+    return {
+        code,
+        tokensUsed: { input: response.usage?.inputTokens || 0, output: response.usage?.outputTokens || 0 },
+    };
+}

package/dist/agentic/playwright_runner.d.ts

@@ -0,0 +1,43 @@
+import type { PlaywrightRunResult } from './types.js';
+interface PlaywrightReportSpec {
+    title: string;
+    ok: boolean;
+    tests: Array<{
+        status: string;
+        results: Array<{
+            status: string;
+            duration: number;
+            error?: {
+                message: string;
+                stack?: string;
+            };
+        }>;
+    }>;
+}
+interface PlaywrightReportSuite {
+    title: string;
+    specs: PlaywrightReportSpec[];
+    suites?: PlaywrightReportSuite[];
+}
+interface PlaywrightReport {
+    suites: PlaywrightReportSuite[];
+    stats: {
+        expected: number;
+        unexpected: number;
+        flaky: number;
+        skipped: number;
+        duration: number;
+    };
+}
+export declare function parsePlaywrightJsonReport(report: PlaywrightReport, specPath: string): PlaywrightRunResult;
+/**
+ * Run a single Playwright spec file and return structured results.
+ * Uses a temp JSON reporter to get machine-readable output.
+ */
+export declare function runPlaywrightSpec(specPath: string, testsRoot: string, options: {
+    project?: string;
+    baseUrl?: string;
+    timeoutMs?: number;
+}): PlaywrightRunResult;
+export {};
+//# sourceMappingURL=playwright_runner.d.ts.map

package/dist/agentic/playwright_runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"playwright_runner.d.ts","sourceRoot":"","sources":["../../src/agentic/playwright_runner.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAC,mBAAmB,EAAc,MAAM,YAAY,CAAC;AAMjE,UAAU,oBAAoB;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,EAAE,KAAK,CAAC;QACT,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,KAAK,CAAC;YACX,MAAM,EAAE,MAAM,CAAC;YACf,QAAQ,EAAE,MAAM,CAAC;YACjB,KAAK,CAAC,EAAE;gBAAC,OAAO,EAAE,MAAM,CAAC;gBAAC,KAAK,CAAC,EAAE,MAAM,CAAA;aAAC,CAAC;SAC7C,CAAC,CAAC;KACN,CAAC,CAAC;CACN;AAED,UAAU,qBAAqB;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,oBAAoB,EAAE,CAAC;IAC9B,MAAM,CAAC,EAAE,qBAAqB,EAAE,CAAC;CACpC;AAED,UAAU,gBAAgB;IACtB,MAAM,EAAE,qBAAqB,EAAE,CAAC;IAChC,KAAK,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;KACpB,CAAC;CACL;AAaD,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,GAAG,mBAAmB,CAkCzG;AAQD;;;GAGG;AACH,wBAAgB,iBAAiB,CAC7B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAC,GAClE,mBAAmB,CAsGrB"}

package/dist/agentic/playwright_runner.js

@@ -0,0 +1,165 @@
+"use strict";
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parsePlaywrightJsonReport = parsePlaywrightJsonReport;
+exports.runPlaywrightSpec = runPlaywrightSpec;
+const child_process_1 = require("child_process");
+const fs_1 = require("fs");
+const path_1 = require("path");
+const MAX_STDOUT_CHARS = 8000;
+const MAX_ERROR_CHARS = 2000;
+const MAX_STACK_CHARS = 1000;
+function extractSpecs(suites) {
+    const specs = [];
+    for (const suite of suites) {
+        specs.push(...suite.specs);
+        if (suite.suites) {
+            specs.push(...extractSpecs(suite.suites));
+        }
+    }
+    return specs;
+}
+function parsePlaywrightJsonReport(report, specPath) {
+    const failures = [];
+    const allSpecs = extractSpecs(report.suites);
+    let passed = 0;
+    let failed = 0;
+    for (const spec of allSpecs) {
+        if (spec.ok) {
+            passed++;
+        }
+        else {
+            failed++;
+            const lastResult = spec.tests[0]?.results?.at(-1);
+            failures.push({
+                testTitle: spec.title,
+                specPath,
+                error: (lastResult?.error?.message || 'Unknown error').slice(0, MAX_ERROR_CHARS),
+                stack: (lastResult?.error?.stack || '').slice(0, MAX_STACK_CHARS),
+                line: extractLineNumber(lastResult?.error?.stack),
+            });
+        }
+    }
+    return {
+        specPath,
+        passed,
+        failed,
+        flaky: report.stats.flaky || 0,
+        skipped: report.stats.skipped || 0,
+        failures,
+        stdout: '',
+        durationMs: report.stats.duration || 0,
+        compiled: true,
+    };
+}
+function extractLineNumber(stack) {
+    if (!stack)
+        return undefined;
+    const match = stack.match(/:(\d+):\d+\)?$/m);
+    return match ? parseInt(match[1], 10) : undefined;
+}
+/**
+ * Run a single Playwright spec file and return structured results.
+ * Uses a temp JSON reporter to get machine-readable output.
+ */
+function runPlaywrightSpec(specPath, testsRoot, options) {
+    // SECURITY: Validate spec path is within testsRoot and has valid extension
+    const resolvedSpec = (0, path_1.resolve)(specPath);
+    const resolvedRoot = (0, path_1.resolve)(testsRoot);
+    if (!resolvedSpec.startsWith(resolvedRoot + '/')) {
+        throw new Error(`Security: spec path ${specPath} is outside testsRoot`);
+    }
+    if (!resolvedSpec.endsWith('.spec.ts') && !resolvedSpec.endsWith('.test.ts')) {
+        throw new Error(`Security: spec path must end in .spec.ts or .test.ts`);
+    }
+    const reportDir = (0, path_1.join)(testsRoot, '.e2e-ai-agents', 'agentic-reports');
+    if (!(0, fs_1.existsSync)(reportDir)) {
+        (0, fs_1.mkdirSync)(reportDir, { recursive: true });
+    }
+    const reportPath = (0, path_1.join)(reportDir, `report-${Date.now()}.json`);
+    const args = [
+        'playwright', 'test',
+        specPath,
+        '--reporter', 'json',
+        '--project', options.project || 'chrome',
+    ];
+    if (options.baseUrl) {
+        args.push('--config', 'playwright.config.ts');
+    }
+    const startTime = Date.now();
+    const result = (0, child_process_1.spawnSync)('npx', args, {
+        cwd: testsRoot,
+        encoding: 'utf-8',
+        timeout: options.timeoutMs || 120000,
+        maxBuffer: 2 * 1024 * 1024,
+        env: {
+            ...process.env,
+            PLAYWRIGHT_JSON_OUTPUT_NAME: reportPath,
+        },
+    });
+    const durationMs = Date.now() - startTime;
+    const stdout = (result.stdout || '').slice(0, MAX_STDOUT_CHARS);
+    const stderr = (result.stderr || '').slice(0, MAX_STDOUT_CHARS);
+    // Check for compile errors
+    if (stderr.includes('SyntaxError') || stderr.includes('Cannot find module') || stderr.includes('TypeError')) {
+        return {
+            specPath,
+            passed: 0,
+            failed: 1,
+            flaky: 0,
+            skipped: 0,
+            failures: [{
+                    testTitle: '(compile)',
+                    specPath,
+                    error: stderr.slice(0, MAX_ERROR_CHARS),
+                    stack: '',
+                }],
+            stdout: `${stdout}\n${stderr}`,
+            durationMs,
+            compiled: false,
+        };
+    }
+    // Try to parse JSON report from stdout (Playwright JSON reporter writes to stdout)
+    try {
+        const jsonReport = JSON.parse(stdout);
+        const parsed = parsePlaywrightJsonReport(jsonReport, specPath);
+        parsed.durationMs = durationMs;
+        parsed.stdout = stdout;
+        return parsed;
+    }
+    catch {
+        // Fallback: try the file-based report
+    }
+    // Try file-based report
+    if ((0, fs_1.existsSync)(reportPath)) {
+        try {
+            const jsonReport = JSON.parse((0, fs_1.readFileSync)(reportPath, 'utf-8'));
+            const parsed = parsePlaywrightJsonReport(jsonReport, specPath);
+            parsed.durationMs = durationMs;
+            parsed.stdout = stdout;
+            try {
+                (0, fs_1.rmSync)(reportPath);
+            }
+            catch { /* ignore */ }
+            return parsed;
+        }
+        catch {
+            // Fallback to exit code
+        }
+    }
+    // Last resort: use exit code
+    return {
+        specPath,
+        passed: result.status === 0 ? 1 : 0,
+        failed: result.status === 0 ? 0 : 1,
+        flaky: 0,
+        skipped: 0,
+        failures: result.status !== 0
+            ? [{ testTitle: '(unknown)', specPath, error: stderr.slice(0, MAX_ERROR_CHARS), stack: '' }]
+            : [],
+        stdout,
+        durationMs,
+        compiled: !stderr.includes('Error'),
+    };
+}

package/dist/agentic/runner.d.ts

@@ -0,0 +1,25 @@
+import type { LLMProvider } from '../provider_interface.js';
+import type { AgenticConfig, AgenticSummary } from './types.js';
+import type { ApiSurfaceCatalog } from '../knowledge/api_surface.js';
+export interface ScenarioInput {
+    id: string;
+    name: string;
+    scenarios: string[];
+    routeFamily: string;
+    priority: string;
+    /** Existing spec to add scenarios to */
+    targetSpec?: string;
+    /** Changed files for context */
+    changedFiles?: string[];
+    /** Evidence from impact analysis */
+    evidence?: string;
+}
+export interface AgenticRunOptions {
+    scenarios: ScenarioInput[];
+    config: AgenticConfig;
+    provider: LLMProvider;
+    apiSurfaceHint?: string;
+    apiSurface?: ApiSurfaceCatalog;
+}
+export declare function runAgenticGeneration(options: AgenticRunOptions): Promise<AgenticSummary>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/agentic/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/agentic/runner.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAC,aAAa,EAAiB,cAAc,EAAsB,MAAM,YAAY,CAAC;AAKlG,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,6BAA6B,CAAC;AAEnE,MAAM,WAAW,aAAa;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,oCAAoC;IACpC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,iBAAiB;IAC9B,SAAS,EAAE,aAAa,EAAE,CAAC;IAC3B,MAAM,EAAE,aAAa,CAAC;IACtB,QAAQ,EAAE,WAAW,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,iBAAiB,CAAC;CAClC;AAqMD,wBAAsB,oBAAoB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,cAAc,CAAC,CAwB9F"}

package/dist/agentic/runner.js

@@ -0,0 +1,207 @@
+"use strict";
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runAgenticGeneration = runAgenticGeneration;
+const fs_1 = require("fs");
+const path_1 = require("path");
+const playwright_runner_js_1 = require("./playwright_runner.js");
+const fix_loop_js_1 = require("./fix_loop.js");
+const generation_js_1 = require("../prompts/generation.js");
+const api_surface_js_1 = require("../knowledge/api_surface.js");
+function buildGeneratePrompt(scenario, apiSurfaceHint) {
+    const scenariosBlock = scenario.scenarios
+        .map((s, i) => `  ${i + 1}. ${s}`)
+        .join('\n');
+    return [
+        'Generate a Mattermost Playwright E2E test file.',
+        '',
+        `FLOW: ${scenario.name}`,
+        `Route Family: ${scenario.routeFamily}`,
+        `Priority: ${scenario.priority}`,
+        scenario.evidence ? `Evidence: ${scenario.evidence}` : '',
+        '',
+        'SCENARIOS TO IMPLEMENT:',
+        scenariosBlock,
+        '',
+        'AVAILABLE PAGE OBJECTS AND METHODS:',
+        apiSurfaceHint || 'Use page.getByRole() or page.getByTestId() for selectors.',
+        '',
+        'MANDATORY RULES:',
+        '1. Import ONLY from "@mattermost/playwright-lib" — no other test framework imports.',
+        '2. Every test must call `await pw.initSetup()` first.',
+        '3. Use `await pw.testBrowser.login(user)` to log in — never hardcode credentials.',
+        '4. Use ONLY page object methods listed above. Do NOT invent methods.',
+        '5. If a method is not available, use `page.getByRole()` or `page.getByTestId()`.',
+        `6. Tag every test: {tag: '@${scenario.routeFamily}'}`,
+        '7. Write one test per scenario with a descriptive name.',
+        '8. Use `expect` from "@mattermost/playwright-lib".',
+        '9. Include the copyright header.',
+        '10. NEVER fabricate test IDs (MM-TXXXX). Use descriptive names only.',
+        '',
+        'EXAMPLE STRUCTURE:',
+        '```typescript',
+        '// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.',
+        '// See LICENSE.txt for license information.',
+        '',
+        "import {expect, test} from '@mattermost/playwright-lib';",
+        '',
+        'test(',
+        "    'user can post a message in channel',",
+        `    {tag: '@${scenario.routeFamily}'},`,
+        '    async ({pw}) => {',
+        '        const {user} = await pw.initSetup();',
+        '        const {channelsPage} = await pw.testBrowser.login(user);',
+        '        await channelsPage.goto();',
+        '        await channelsPage.toBeVisible();',
+        '        // test steps...',
+        '    },',
+        ');',
+        '```',
+        '',
+        'Return ONLY the TypeScript code. No explanations.',
+    ].filter(Boolean).join('\n');
+}
+function resolveSpecPath(scenario, testsRoot) {
+    let specPath;
+    if (scenario.targetSpec) {
+        specPath = (0, path_1.join)(testsRoot, scenario.targetSpec);
+    }
+    else {
+        const safeName = scenario.id.replace(/[^a-zA-Z0-9_-]/g, '_').toLowerCase();
+        const outputDir = (0, path_1.join)(testsRoot, 'specs', 'functional', 'ai-assisted');
+        specPath = (0, path_1.join)(outputDir, `${safeName}.spec.ts`);
+    }
+    // SECURITY: Prevent path traversal
+    const resolved = (0, path_1.resolve)(specPath);
+    const resolvedRoot = (0, path_1.resolve)(testsRoot);
+    if (!resolved.startsWith(resolvedRoot + '/') && resolved !== resolvedRoot) {
+        throw new Error(`Path traversal blocked: ${specPath} resolves outside testsRoot`);
+    }
+    if (!resolved.endsWith('.spec.ts') && !resolved.endsWith('.test.ts')) {
+        throw new Error(`Invalid spec path: must end in .spec.ts or .test.ts`);
+    }
+    return specPath;
+}
+async function generateInitialSpec(provider, scenario, specPath, apiSurfaceHint) {
+    const prompt = buildGeneratePrompt(scenario, apiSurfaceHint);
+    const response = await provider.generateText(prompt, {
+        maxTokens: 8000,
+        temperature: 0.1,
+        timeout: 60000,
+        systemPrompt: 'You are an expert Playwright test writer for Mattermost. Return only TypeScript code.',
+    });
+    // Reuse existing parsing logic from prompts/generation.ts
+    const parsed = (0, generation_js_1.parseGenerationResponse)(response.text, specPath, 'create_spec', scenario.id);
+    return parsed?.code ?? null;
+}
+async function runSingleScenario(scenario, options) {
+    const { config, provider } = options;
+    const warnings = [];
+    const specPath = resolveSpecPath(scenario, config.testsRoot);
+    // Build API surface hint
+    let apiHint = options.apiSurfaceHint || '';
+    if (!apiHint && options.apiSurface) {
+        const allClassNames = options.apiSurface.pageObjects.map((po) => po.className);
+        apiHint = (0, api_surface_js_1.formatApiSurfaceForPrompt)(options.apiSurface, allClassNames);
+    }
+    // Step 1: Generate initial spec
+    let specCode;
+    try {
+        specCode = await generateInitialSpec(provider, scenario, specPath, apiHint);
+    }
+    catch (error) {
+        const msg = error instanceof Error ? error.message : String(error);
+        warnings.push(`Generation failed for ${scenario.id}: ${msg}`);
+        return { specPath, scenarioSource: scenario.id, status: 'failed', attempts: 0, warnings };
+    }
+    if (!specCode) {
+        warnings.push(`LLM returned invalid code for ${scenario.id}`);
+        return { specPath, scenarioSource: scenario.id, status: 'failed', attempts: 0, warnings };
+    }
+    // Write the spec file
+    const dir = (0, path_1.dirname)(specPath);
+    if (!(0, fs_1.existsSync)(dir)) {
+        (0, fs_1.mkdirSync)(dir, { recursive: true });
+    }
+    (0, fs_1.writeFileSync)(specPath, specCode, 'utf-8');
+    // Dry run: skip execution
+    if (config.dryRun) {
+        return { specPath, scenarioSource: scenario.id, status: 'skipped', attempts: 0, warnings };
+    }
+    // Step 2: Run -> Fix loop
+    let lastRun;
+    for (let attempt = 1; attempt <= config.maxAttempts; attempt++) {
+        lastRun = (0, playwright_runner_js_1.runPlaywrightSpec)(specPath, config.testsRoot, {
+            project: config.project,
+            baseUrl: config.baseUrl,
+            timeoutMs: config.testTimeoutMs,
+        });
+        // All passed!
+        if (lastRun.failed === 0 && lastRun.compiled) {
+            return {
+                specPath,
+                scenarioSource: scenario.id,
+                status: 'passed',
+                attempts: attempt,
+                finalRun: lastRun,
+                warnings,
+            };
+        }
+        // If this is the last attempt, don't try to fix
+        if (attempt >= config.maxAttempts) {
+            break;
+        }
+        // Step 3: Fix
+        const currentCode = (0, fs_1.readFileSync)(specPath, 'utf-8');
+        try {
+            const fixResult = await (0, fix_loop_js_1.generateFix)(provider, {
+                specCode: currentCode,
+                failures: lastRun.failures,
+                attempt,
+                maxAttempts: config.maxAttempts,
+                apiSurfaceHint: apiHint,
+            });
+            if (fixResult.code) {
+                (0, fs_1.writeFileSync)(specPath, fixResult.code, 'utf-8');
+            }
+            else {
+                warnings.push(`Fix attempt ${attempt} returned invalid code for ${scenario.id}`);
+            }
+        }
+        catch (error) {
+            const msg = error instanceof Error ? error.message : String(error);
+            warnings.push(`Fix attempt ${attempt} failed for ${scenario.id}: ${msg}`);
+        }
+    }
+    return {
+        specPath,
+        scenarioSource: scenario.id,
+        status: lastRun?.compiled === false ? 'compile-error' : 'max-attempts',
+        attempts: config.maxAttempts,
+        finalRun: lastRun,
+        warnings,
+    };
+}
+async function runAgenticGeneration(options) {
+    const startTime = Date.now();
+    const results = [];
+    const warnings = [];
+    for (const scenario of options.scenarios) {
+        const result = await runSingleScenario(scenario, options);
+        results.push(result);
+        warnings.push(...result.warnings);
+    }
+    const totalPassed = results.filter((r) => r.status === 'passed').length;
+    const totalFailed = results.filter((r) => r.status !== 'passed' && r.status !== 'skipped').length;
+    const totalAttempts = results.reduce((sum, r) => sum + r.attempts, 0);
+    return {
+        results,
+        totalGenerated: results.length,
+        totalPassed,
+        totalFailed,
+        totalAttempts,
+        durationMs: Date.now() - startTime,
+        warnings,
+    };
+}

package/dist/agentic/types.d.ts

@@ -0,0 +1,62 @@
+export interface TestFailure {
+    testTitle: string;
+    specPath: string;
+    error: string;
+    /** Truncated stack trace */
+    stack: string;
+    /** Expected vs actual if available */
+    expected?: string;
+    actual?: string;
+    /** Line number in spec where failure occurred */
+    line?: number;
+}
+export interface PlaywrightRunResult {
+    specPath: string;
+    passed: number;
+    failed: number;
+    flaky: number;
+    skipped: number;
+    failures: TestFailure[];
+    /** Raw stdout (truncated) */
+    stdout: string;
+    /** Duration in ms */
+    durationMs: number;
+    /** Whether the spec even compiled */
+    compiled: boolean;
+}
+export interface AgenticConfig {
+    /** Max fix attempts before giving up (default: 3) */
+    maxAttempts: number;
+    /** Playwright project to use (default: 'chrome') */
+    project: string;
+    /** Base URL for Playwright (e.g. http://localhost:8065) */
+    baseUrl?: string;
+    /** Timeout per test run in ms (default: 120000) */
+    testTimeoutMs: number;
+    /** LLM provider override */
+    provider?: string;
+    /** Whether to use Playwright MCP for browser exploration */
+    useMcp?: boolean;
+    /** Dry run — generate but don't run tests */
+    dryRun?: boolean;
+    /** Tests root directory */
+    testsRoot: string;
+}
+export interface AgenticResult {
+    specPath: string;
+    scenarioSource: string;
+    status: 'passed' | 'failed' | 'max-attempts' | 'compile-error' | 'skipped';
+    attempts: number;
+    finalRun?: PlaywrightRunResult;
+    warnings: string[];
+}
+export interface AgenticSummary {
+    results: AgenticResult[];
+    totalGenerated: number;
+    totalPassed: number;
+    totalFailed: number;
+    totalAttempts: number;
+    durationMs: number;
+    warnings: string[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/agentic/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/agentic/types.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,WAAW;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,sCAAsC;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iDAAiD;IACjD,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,mBAAmB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,6BAA6B;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,qCAAqC;IACrC,QAAQ,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,aAAa;IAC1B,qDAAqD;IACrD,WAAW,EAAE,MAAM,CAAC;IACpB,oDAAoD;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,aAAa,EAAE,MAAM,CAAC;IACtB,4BAA4B;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,4DAA4D;IAC5D,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,6CAA6C;IAC7C,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,aAAa;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,cAAc,GAAG,eAAe,GAAG,SAAS,CAAC;IAC3E,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,mBAAmB,CAAC;IAC/B,QAAQ,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC3B,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACtB"}

package/dist/agentic/types.js

@@ -0,0 +1,4 @@
+"use strict";
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+Object.defineProperty(exports, "__esModule", { value: true });