@yasserkhanorg/e2e-agents 1.4.0 → 1.6.0

package/dist/esm/qa-agent/phase2/tools.js

@@ -94,7 +94,7 @@ export const TOOL_DEFINITIONS = [
     },
     {
         name: 'report_finding',
-        description: 'Report a bug, visual issue, UX problem, or gap you discovered. Always include current URL and repro steps.',
+        description: 'Report a bug, visual issue, UX problem, or gap you discovered. Include expected/actual behavior and repro steps. Take before/after screenshots before calling this.',
         input_schema: {
             type: 'object',
             properties: {
@@ -106,6 +106,13 @@ export const TOOL_DEFINITIONS = [
                     items: { type: 'string' },
                     description: 'Steps to reproduce',
                 },
+                screenshot_refs: {
+                    type: 'array',
+                    items: { type: 'string' },
+                    description: 'Paths to before/after screenshots (from take_screenshot)',
+                },
+                expected_behavior: { type: 'string', description: 'What should have happened' },
+                actual_behavior: { type: 'string', description: 'What actually happened' },
             },
             required: ['type', 'severity', 'summary', 'repro_steps'],
         },
@@ -133,6 +140,23 @@ export const TOOL_DEFINITIONS = [
             required: ['role'],
         },
     },
+    {
+        name: 'wait_for',
+        description: 'Wait for an element condition or page state. Use after actions that trigger async changes (navigation, API calls, animations).',
+        input_schema: {
+            type: 'object',
+            properties: {
+                condition: {
+                    type: 'string',
+                    enum: ['visible', 'hidden', 'stable', 'networkidle'],
+                    description: 'What to wait for: visible/hidden (element state), stable (no DOM changes for 1s), networkidle (no pending requests)',
+                },
+                ref: { type: 'string', description: 'Accessibility ref for element conditions (visible/hidden). Not needed for stable/networkidle.' },
+                timeout_ms: { type: 'number', description: 'Max wait time in ms (default 5000, max 15000)' },
+            },
+            required: ['condition'],
+        },
+    },
 ];
 export function executeTool(ctx, name, input) {
     switch (name) {
@@ -204,6 +228,36 @@ export function executeTool(ctx, name, input) {
             if (!Array.isArray(input.repro_steps)) {
                 return { output: `Invalid repro_steps: expected an array of strings.` };
             }
+            // Auto-capture console errors at time of finding
+            let autoConsoleErrors;
+            try {
+                const raw = ctx.browser.evaluateInternal('JSON.stringify(window.__consoleErrors || [])');
+                const parsed = JSON.parse(raw);
+                if (Array.isArray(parsed) && parsed.length > 0) {
+                    autoConsoleErrors = parsed.map(String).slice(-10);
+                }
+            }
+            catch {
+                // Console error capture not available
+            }
+            // Auto-take screenshot if none provided
+            let autoScreenshot;
+            const screenshotRefs = Array.isArray(input.screenshot_refs)
+                ? input.screenshot_refs.map(String)
+                : undefined;
+            if (!screenshotRefs || screenshotRefs.length === 0) {
+                try {
+                    const nextCount = ctx.screenshotCounter + 1;
+                    const filename = `${String(nextCount).padStart(3, '0')}-finding-auto.png`;
+                    const screenshotPath = `${ctx.screenshotDir}/${filename}`;
+                    ctx.browser.screenshot(screenshotPath);
+                    ctx.screenshotCounter = nextCount;
+                    autoScreenshot = screenshotPath;
+                }
+                catch {
+                    autoScreenshot = undefined;
+                }
+            }
             const finding = {
                 id: `f-${crypto.randomUUID()}`,
                 type: rawType,
@@ -213,6 +267,11 @@ export function executeTool(ctx, name, input) {
                 evidence: {
                     url: ctx.currentUrl,
                     reproSteps: input.repro_steps.map(String),
+                    screenshotRefs: screenshotRefs || (autoScreenshot ? [autoScreenshot] : undefined),
+                    screenshotPath: autoScreenshot || (screenshotRefs ? screenshotRefs[0] : undefined),
+                    consoleErrors: autoConsoleErrors,
+                    expectedBehavior: input.expected_behavior ? String(input.expected_behavior) : undefined,
+                    actualBehavior: input.actual_behavior ? String(input.actual_behavior) : undefined,
                 },
                 timestamp: Date.now(),
             };
@@ -229,6 +288,45 @@ export function executeTool(ctx, name, input) {
                 flowDone: { flowId, status: rawStatus },
             };
         }
+        case 'wait_for': {
+            const condition = String(input.condition || '');
+            const VALID_CONDITIONS = new Set(['visible', 'hidden', 'stable', 'networkidle']);
+            if (!VALID_CONDITIONS.has(condition)) {
+                return { output: `Invalid condition "${condition}". Must be one of: ${[...VALID_CONDITIONS].join(', ')}.` };
+            }
+            const timeoutMs = Math.min(Math.max(Number(input.timeout_ms) || 5000, 500), 15000);
+            try {
+                if (condition === 'stable' || condition === 'networkidle') {
+                    const waitMs = condition === 'networkidle' ? Math.min(timeoutMs, 3000) : 1000;
+                    ctx.browser.evaluateInternal(`new Promise(r => setTimeout(r, ${waitMs}))`);
+                    return { output: `Waited ${waitMs}ms for ${condition} (heuristic delay)` };
+                }
+                // Element-level wait: poll snapshot for ref presence/absence
+                const ref = input.ref ? String(input.ref) : undefined;
+                if (!ref) {
+                    return { output: `Element condition "${condition}" requires a ref parameter.` };
+                }
+                const start = Date.now();
+                const wantVisible = condition === 'visible';
+                // Use word-boundary regex to avoid false positives (@e1 matching @e10)
+                const refPattern = new RegExp(`(?<![\\w@])${ref.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\w])`);
+                const pollIntervalMs = 300;
+                while (Date.now() - start < timeoutMs) {
+                    const snap = ctx.browser.snapshot();
+                    const found = refPattern.test(snap);
+                    if ((wantVisible && found) || (!wantVisible && !found)) {
+                        return { output: `Element ${ref} is now ${condition} (took ${Date.now() - start}ms)` };
+                    }
+                    // Synchronous in-process sleep via Atomics.wait (available in Node.js 8.10+)
+                    const buf = new SharedArrayBuffer(4);
+                    Atomics.wait(new Int32Array(buf), 0, 0, pollIntervalMs);
+                }
+                return { output: `Timeout: element ${ref} did not become ${condition} within ${timeoutMs}ms` };
+            }
+            catch (err) {
+                return { output: `wait_for error: ${String(err)}` };
+            }
+        }
         case 'switch_user': {
             const role = String(input.role);
             const user = ctx.users?.find((u) => u.role === role);

package/dist/esm/qa-agent/phase3/reporter.js

@@ -80,16 +80,43 @@ function renderMarkdown(report) {
     if (report.phase2.findings.length > 0) {
         lines.push(`## Findings`, '');
         for (const f of report.phase2.findings) {
-            lines.push(`### [${f.severity.toUpperCase()}] ${f.summary}`);
+            const dupNote = f.duplicateCount && f.duplicateCount > 1
+                ? ` (seen ${f.duplicateCount} times)`
+                : '';
+            lines.push(`### [${f.severity.toUpperCase()}] ${f.summary}${dupNote}`);
             lines.push('');
             lines.push(`- **Type:** ${f.type}`);
             lines.push(`- **Flow:** ${f.flow}`);
             lines.push(`- **URL:** ${f.evidence.url}`);
-            if (f.evidence.screenshotPath) {
-                lines.push(`- **Screenshot:** ${f.evidence.screenshotPath}`);
+            // Expected vs actual behavior
+            if (f.evidence.expectedBehavior || f.evidence.actualBehavior) {
+                const escapePipe = (s) => s.replace(/\|/g, '\\|');
+                lines.push('');
+                lines.push(`| Expected | Actual |`);
+                lines.push(`|----------|--------|`);
+                lines.push(`| ${escapePipe(f.evidence.expectedBehavior || '—')} | ${escapePipe(f.evidence.actualBehavior || '—')} |`);
+                lines.push('');
+            }
+            // Screenshot evidence (inline images)
+            if (f.evidence.screenshotRefs && f.evidence.screenshotRefs.length > 0) {
+                for (const ref of f.evidence.screenshotRefs) {
+                    lines.push(`![Evidence](${ref})`);
+                }
+            }
+            else if (f.evidence.screenshotPath) {
+                lines.push(`![Evidence](${f.evidence.screenshotPath})`);
+            }
+            // Console errors
+            if (f.evidence.consoleErrors && f.evidence.consoleErrors.length > 0) {
+                lines.push('');
+                lines.push('**Console errors:**');
+                for (const err of f.evidence.consoleErrors.slice(0, 5)) {
+                    lines.push(`- \`${err.replace(/`/g, '\\`')}\``);
+                }
             }
             if (f.evidence.reproSteps.length > 0) {
-                lines.push('- **Repro steps:**');
+                lines.push('');
+                lines.push('**Repro steps:**');
                 for (const step of f.evidence.reproSteps) {
                     lines.push(`  1. ${step}`);
                 }

package/dist/esm/training/enricher.js

@@ -59,9 +59,47 @@ function sampleFiles(dir, maxFiles) {
     walk(dir);
     return files;
 }
-function buildEnrichPrompt(families, projectRoot) {
+/**
+ * Build a shallow directory listing of the source tree (depth 2-3) so the LLM
+ * can suggest accurate webappPaths / serverPaths for test-derived families.
+ */
+function getSourceTreeListing(projectRoot, maxDepth = 3) {
+    const lines = [];
+    function walk(dir, depth, prefix) {
+        if (depth > maxDepth || lines.length > 200)
+            return;
+        let entries;
+        try {
+            entries = readdirSync(dir).sort();
+        }
+        catch {
+            return;
+        }
+        const dirs = entries.filter((e) => {
+            if (e.startsWith('.') || SKIP_DIRS.has(e))
+                return false;
+            try {
+                const stat = lstatSync(join(dir, e));
+                return !stat.isSymbolicLink() && stat.isDirectory();
+            }
+            catch {
+                return false;
+            }
+        });
+        for (const d of dirs) {
+            lines.push(`${prefix}${d}/`);
+            walk(join(dir, d), depth + 1, prefix + '  ');
+        }
+    }
+    walk(resolve(projectRoot), 0, '');
+    return lines.join('\n');
+}
+function buildEnrichPrompt(families, projectRoot, testsRoot) {
     const sections = [];
+    const hasTestOnlyFamilies = families.some((f) => f.webappPaths.length === 0 && f.serverPaths.length === 0);
+    const resolvedTestsRoot = testsRoot ? resolve(testsRoot) : resolve(projectRoot);
     for (const family of families) {
+        const isTestOnly = family.webappPaths.length === 0 && family.serverPaths.length === 0;
         const allDirs = [
             ...family.webappPaths.map((p) => p.replace(/\/?\*.*$/, '')),
             ...family.serverPaths.map((p) => p.replace(/\/?\*.*$/, '')),
@@ -75,10 +113,19 @@ function buildEnrichPrompt(families, projectRoot) {
             if (samples.length >= MAX_FILES_PER_FAMILY)
                 break;
         }
+        // For test-only families, sample the test files themselves for richer context
+        if (isTestOnly) {
+            for (const specDir of family.specDirs) {
+                if (samples.length >= MAX_FILES_PER_FAMILY)
+                    break;
+                const fullDir = join(resolvedTestsRoot, specDir);
+                samples.push(...sampleFiles(fullDir, MAX_FILES_PER_FAMILY - samples.length));
+            }
+        }
         // Sample spec descriptions
         const specSamples = [];
         for (const specDir of family.specDirs) {
-            const fullDir = join(resolve(projectRoot), specDir);
+            const fullDir = join(resolvedTestsRoot, specDir);
             const specFiles = sampleFiles(fullDir, 5);
             for (const sf of specFiles) {
                 const matches = sf.content.match(/(?:test|it|describe)\s*\(\s*['"`]([^'"`]+)/g);
@@ -87,7 +134,7 @@ function buildEnrichPrompt(families, projectRoot) {
                 }
             }
         }
-        sections.push(`## Family: ${family.id}
+        sections.push(`## Family: ${family.id}${isTestOnly ? ' [TEST-ONLY — needs webappPaths/serverPaths]' : ''}
 Routes (guessed): ${JSON.stringify(family.routes)}
 Webapp paths: ${JSON.stringify(family.webappPaths)}
 Server paths: ${JSON.stringify(family.serverPaths)}
@@ -102,6 +149,10 @@ Test descriptions:
 ${specSamples.length > 0 ? specSamples.map((d) => `- ${d}`).join('\n') : '(none found)'}
 `);
     }
+    // Include source tree listing when we have test-only families
+    const sourceTreeSection = hasTestOnlyFamilies
+        ? `\n## Source Directory Structure\nUse this to suggest accurate webappPaths and serverPaths for test-only families:\n\`\`\`\n${getSourceTreeListing(projectRoot)}\n\`\`\`\n`
+        : '';
     return `You are analyzing a codebase to enrich route-family definitions for an E2E test impact analysis tool.
 
 For each family below, provide:
@@ -110,6 +161,8 @@ For each family below, provide:
 3. **routes**: Improved URL patterns (e.g., "/{team}/channels/{channel}" instead of "/channels")
 4. **pageObjects**: Array of page object class names found in the code
 5. **components**: Array of UI component names relevant to this family
+6. **webappPaths**: Array of glob patterns for frontend source directories (e.g., "src/components/drafts/**"). REQUIRED for families marked [TEST-ONLY].
+7. **serverPaths**: Array of glob patterns for backend source directories. REQUIRED for families marked [TEST-ONLY].
 
 Respond in JSON format:
 \`\`\`json
@@ -120,11 +173,13 @@ Respond in JSON format:
     "userFlows": ["Flow name 1", "Flow name 2"],
     "routes": ["/improved/route/{param}"],
     "pageObjects": ["PageName"],
-    "components": ["ComponentName"]
+    "components": ["ComponentName"],
+    "webappPaths": ["src/components/feature_name/**"],
+    "serverPaths": ["server/channels/api4/feature.go"]
   }
 ]
 \`\`\`
-
+${sourceTreeSection}
 ${sections.join('\n---\n')}`;
 }
 export function validateEntries(parsed) {
@@ -143,6 +198,8 @@ export function validateEntries(parsed) {
         userFlows: filterStrings(entry.userFlows, 500),
         pageObjects: filterStrings(entry.pageObjects, 200),
         components: filterStrings(entry.components, 200),
+        webappPaths: filterStrings(entry.webappPaths, 300),
+        serverPaths: filterStrings(entry.serverPaths, 300),
     }));
 }
 export function parseEnrichResponse(response) {
@@ -192,9 +249,16 @@ function applyEnrichment(family, enriched) {
     if (enriched.components && (!family.components || family.components.length === 0)) {
         result.components = enriched.components;
     }
+    // Only fill source paths when the family has none (test-derived families)
+    if (enriched.webappPaths && (!family.webappPaths || family.webappPaths.length === 0)) {
+        result.webappPaths = enriched.webappPaths;
+    }
+    if (enriched.serverPaths && (!family.serverPaths || family.serverPaths.length === 0)) {
+        result.serverPaths = enriched.serverPaths;
+    }
     return result;
 }
-export async function enrichFamilies(families, scanned, projectRoot, provider, budgetUSD) {
+export async function enrichFamilies(families, scanned, projectRoot, provider, budgetUSD, testsRoot) {
     const scannedMap = new Map(scanned.map((s) => [s.id, s]));
     const enriched = [];
     let totalTokens = 0;
@@ -218,7 +282,7 @@ export async function enrichFamilies(families, scanned, projectRoot, provider, b
             enriched.push(...chunk);
             continue;
         }
-        let prompt = buildEnrichPrompt(scannedChunk, projectRoot);
+        let prompt = buildEnrichPrompt(scannedChunk, projectRoot, testsRoot);
         if (prompt.length > MAX_PROMPT_CHARS) {
             // Truncate at the last complete section boundary to avoid malformed input
             const lastSectionEnd = prompt.lastIndexOf('\n---\n', MAX_PROMPT_CHARS);

package/dist/esm/training/merger.js

@@ -1,5 +1,6 @@
 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
 // See LICENSE.txt for license information.
+import { execFileSync } from 'child_process';
 import { existsSync } from 'fs';
 import { join, resolve } from 'path';
 import { isGuessedRoute } from './types.js';
@@ -67,6 +68,21 @@ function scannedToRouteFamily(scanned) {
     }
     return family;
 }
+/**
+ * Try to find a matching family ID with singular/plural normalization.
+ * "team" matches "teams", "emoji" matches "emoji", etc.
+ */
+function findFuzzyMatch(id, idMap) {
+    if (idMap.has(id))
+        return id;
+    // Try adding 's'
+    if (!id.endsWith('s') && idMap.has(id + 's'))
+        return id + 's';
+    // Try removing 's'
+    if (id.endsWith('s') && idMap.has(id.slice(0, -1)))
+        return id.slice(0, -1);
+    return undefined;
+}
 export function mergeFamilies(existing, scanned) {
     const existingFamilies = existing?.families || [];
     const existingMap = new Map(existingFamilies.map((f) => [f.id, f]));
@@ -74,9 +90,15 @@ export function mergeFamilies(existing, scanned) {
     const newFamilies = [];
     const updatedFamilies = [];
     const mergedFamilies = [];
-    // Process existing families
+    // Process existing families — match scanned by exact or fuzzy ID
     for (const ef of existingFamilies) {
-        const sf = scannedMap.get(ef.id);
+        let sf = scannedMap.get(ef.id);
+        // Try singular/plural match if exact match failed
+        if (!sf) {
+            const fuzzyId = findFuzzyMatch(ef.id, scannedMap);
+            if (fuzzyId)
+                sf = scannedMap.get(fuzzyId);
+        }
         if (sf) {
             mergedFamilies.push(mergeFamily(ef, sf));
             updatedFamilies.push(ef.id);
@@ -86,9 +108,10 @@ export function mergeFamilies(existing, scanned) {
             mergedFamilies.push({ ...ef });
         }
     }
-    // Add new families from scanner
+    // Add new families from scanner (if no existing family matched)
     for (const sf of scanned) {
-        if (!existingMap.has(sf.id)) {
+        const matchedExisting = findFuzzyMatch(sf.id, existingMap);
+        if (!matchedExisting) {
             mergedFamilies.push(scannedToRouteFamily(sf));
             newFamilies.push(sf.id);
         }
@@ -108,8 +131,33 @@ export function mergeFamilies(existing, scanned) {
         summary: parts.join(', '),
     };
 }
-export function detectStaleFamilies(manifest, projectRoot) {
-    const resolved = resolve(projectRoot);
+/**
+ * Detect families whose paths no longer exist on disk.
+ *
+ * Paths in the manifest may be relative to different roots:
+ * - webappPaths / serverPaths are typically relative to the repo root
+ * - specDirs may be relative to the tests root
+ *
+ * We try each pattern against all provided roots (and the git repo root
+ * if discoverable) to avoid false positives from path-prefix mismatches.
+ */
+export function detectStaleFamilies(manifest, projectRoot, testsRoot) {
+    const roots = new Set([resolve(projectRoot)]);
+    if (testsRoot)
+        roots.add(resolve(testsRoot));
+    // Also try to discover the git repo root — manifest paths may be repo-relative
+    try {
+        const gitRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], {
+            cwd: projectRoot,
+            encoding: 'utf-8',
+            stdio: ['pipe', 'pipe', 'pipe'],
+        }).trim();
+        if (gitRoot)
+            roots.add(resolve(gitRoot));
+    }
+    catch {
+        // Not a git repo or git not available — that's fine
+    }
     const stale = [];
     for (const family of manifest.families) {
         const allPatterns = [
@@ -119,15 +167,34 @@ export function detectStaleFamilies(manifest, projectRoot) {
         ];
         if (allPatterns.length === 0)
             continue;
-        // Check if any pattern resolves to existing files/dirs
+        // Check if any pattern resolves to existing files/dirs in any root
         let hasAny = false;
         for (const pattern of allPatterns) {
             // Strip trailing glob (* or **) to get the directory
             const dirPart = pattern.replace(/\/?\*.*$/, '');
-            if (dirPart && existsSync(join(resolved, dirPart))) {
-                hasAny = true;
-                break;
+            if (!dirPart)
+                continue;
+            // For file-level patterns like "server/channels/api4/draft*.go",
+            // dirPart is "server/channels/api4/draft" — check the parent dir instead
+            const isFileGlob = /\.\w+$/.test(pattern);
+            const pathsToCheck = [dirPart];
+            if (isFileGlob) {
+                const parentDir = dirPart.split('/').slice(0, -1).join('/');
+                if (parentDir)
+                    pathsToCheck.push(parentDir);
+            }
+            for (const checkPath of pathsToCheck) {
+                for (const root of roots) {
+                    if (existsSync(join(root, checkPath))) {
+                        hasAny = true;
+                        break;
+                    }
+                }
+                if (hasAny)
+                    break;
             }
+            if (hasAny)
+                break;
         }
         if (!hasAny) {
             stale.push(family.id);