@yasserkhanorg/e2e-agents 1.5.0 → 1.7.0

package/dist/esm/logger.js

@@ -48,6 +48,7 @@ function logLevelToString(level) {
 export class Logger {
     constructor(minLevel) {
         this.level = minLevel ?? getLogLevelFromEnv();
+        this.jsonMode = process.env.LOG_FORMAT?.toLowerCase() === 'json';
     }
     error(message, context) {
         if (this.level >= LogLevel.ERROR) {
@@ -72,11 +73,37 @@ export class Logger {
     setLevel(level) {
         this.level = level;
     }
+    setJsonMode(enabled) {
+        this.jsonMode = enabled;
+    }
+    /**
+     * Start a timer for measuring duration of an operation.
+     * Returns an object with `end()` that logs at DEBUG level and returns elapsed ms.
+     */
+    timer(label) {
+        const start = performance.now();
+        return {
+            end: () => {
+                const elapsed = Math.round(performance.now() - start);
+                this.debug(`${label} completed`, { durationMs: elapsed });
+                return elapsed;
+            },
+        };
+    }
     log(level, message, context) {
         const timestamp = new Date().toISOString();
         const levelStr = logLevelToString(level);
-        const contextStr = context ? ` ${JSON.stringify(context)}` : '';
-        const output = `[${timestamp}] [${levelStr}] ${message}${contextStr}`;
+        let output;
+        if (this.jsonMode) {
+            const entry = { ts: timestamp, level: levelStr, msg: message };
+            if (context)
+                entry.ctx = context;
+            output = JSON.stringify(entry);
+        }
+        else {
+            const contextStr = context ? ` ${JSON.stringify(context)}` : '';
+            output = `[${timestamp}] [${levelStr}] ${message}${contextStr}`;
+        }
         if (level <= LogLevel.WARN) {
             console.error(output);
         }

package/dist/esm/pipeline/orchestrator.js

@@ -3,6 +3,7 @@
 import { existsSync, mkdirSync, writeFileSync } from 'fs';
 import { join } from 'path';
 import { getChangedFiles } from '../agent/git.js';
+import { logger } from '../logger.js';
 import { preprocess } from './stage0_preprocess.js';
 import { runImpactStage } from './stage1_impact.js';
 import { runCoverageStage } from './stage2_coverage.js';
@@ -58,20 +59,25 @@ export async function runPipeline(config) {
         const reportPath = writeReport(config.testsRoot, emptyReport);
         return { report: emptyReport, reportPath, warnings: allWarnings };
     }
+    const timings = {};
     // Step 2: Preprocess — deterministic file classification + route family binding
+    const preprocessTimer = logger.timer('preprocess');
     const preprocessResult = preprocess(changedFiles, {
         appPath: config.appPath,
         testsRoot: config.testsRoot,
         routeFamilies: config.routeFamilies,
         apiSurface: config.apiSurface,
     });
+    timings.preprocess = preprocessTimer.end();
     allWarnings.push(...preprocessResult.warnings);
     let decisions = [];
     // Step 3: Impact stage — AI-powered flow identification per family
     if (stages.includes('impact')) {
+        const impactTimer = logger.timer('impact');
         const impactResult = await runImpactStage(preprocessResult.familyGroups, preprocessResult.manifest, preprocessResult.specIndex, preprocessResult.apiSurface, preprocessResult.context, config.impact || {});
         decisions = impactResult.decisions;
         allWarnings.push(...impactResult.warnings);
+        timings.impact = impactTimer.end();
         // Check cannot_determine ratio
         const cannotDetermineRatio = computeCannotDetermineRatio(decisions);
         if (cannotDetermineRatio > 0.3) {
@@ -80,18 +86,23 @@ export async function runPipeline(config) {
     }
     // Step 4: Coverage stage — AI-powered spec coverage evaluation
     if (stages.includes('coverage') && decisions.length > 0) {
+        const coverageTimer = logger.timer('coverage');
         const coverageResult = await runCoverageStage(decisions, preprocessResult.specIndex, preprocessResult.context, config.testsRoot, config.coverage || {});
         decisions = coverageResult.decisions;
+        timings.coverage = coverageTimer.end();
         allWarnings.push(...coverageResult.warnings);
     }
     // Step 5: Generation stage — AI-powered spec generation for create_spec / add_scenarios
     if (stages.includes('generation') && decisions.length > 0) {
+        const generationTimer = logger.timer('generation');
         const generationResult = await runGenerationStage(decisions, preprocessResult.apiSurface, config.testsRoot, config.generation || {});
         generatedSpecs = generationResult.generated;
+        timings.generation = generationTimer.end();
         allWarnings.push(...generationResult.warnings);
     }
     // Step 6: Heal stage — MCP-backed playwright-test-healer for failing/flaky specs
     if (stages.includes('heal')) {
+        const healTimer = logger.timer('heal');
         const healTargets = resolveHealTargets(config.testsRoot, {
             playwrightReportPath: config.playwrightReportPath,
             generatedSpecs,
@@ -103,6 +114,7 @@ export async function runPipeline(config) {
         else {
             allWarnings.push('Heal stage: no targets found (no failing specs in report, no generated specs).');
         }
+        timings.heal = healTimer.end();
     }
     // Build report
     const report = {
@@ -118,16 +130,18 @@ export async function runPipeline(config) {
             generationAgent: stages.includes('generation') ? (config.generation?.provider || 'auto') : undefined,
         },
     };
-    const reportPath = writeReport(config.testsRoot, report, healResult);
+    const reportPath = writeReport(config.testsRoot, report, healResult, timings);
     return { report, reportPath, warnings: allWarnings, generated: generatedSpecs, healResult };
 }
-function writeReport(testsRoot, report, healResult) {
+function writeReport(testsRoot, report, healResult, timings) {
     const outputDir = join(testsRoot, '.e2e-ai-agents');
     if (!existsSync(outputDir)) {
         mkdirSync(outputDir, { recursive: true });
     }
+    // Include timings in the JSON report if available
+    const reportWithTimings = timings ? { ...report, timings } : report;
     const jsonPath = join(outputDir, 'pipeline-report.json');
-    writeFileSync(jsonPath, JSON.stringify(report, null, 2), 'utf-8');
+    writeFileSync(jsonPath, JSON.stringify(reportWithTimings, null, 2), 'utf-8');
     const mdPath = join(outputDir, 'pipeline-report.md');
     writeFileSync(mdPath, renderMarkdown(report, healResult), 'utf-8');
     return jsonPath;

package/dist/esm/training/enricher.js

@@ -59,9 +59,47 @@ function sampleFiles(dir, maxFiles) {
     walk(dir);
     return files;
 }
-function buildEnrichPrompt(families, projectRoot) {
+/**
+ * Build a shallow directory listing of the source tree (depth 2-3) so the LLM
+ * can suggest accurate webappPaths / serverPaths for test-derived families.
+ */
+function getSourceTreeListing(projectRoot, maxDepth = 3) {
+    const lines = [];
+    function walk(dir, depth, prefix) {
+        if (depth > maxDepth || lines.length > 200)
+            return;
+        let entries;
+        try {
+            entries = readdirSync(dir).sort();
+        }
+        catch {
+            return;
+        }
+        const dirs = entries.filter((e) => {
+            if (e.startsWith('.') || SKIP_DIRS.has(e))
+                return false;
+            try {
+                const stat = lstatSync(join(dir, e));
+                return !stat.isSymbolicLink() && stat.isDirectory();
+            }
+            catch {
+                return false;
+            }
+        });
+        for (const d of dirs) {
+            lines.push(`${prefix}${d}/`);
+            walk(join(dir, d), depth + 1, prefix + '  ');
+        }
+    }
+    walk(resolve(projectRoot), 0, '');
+    return lines.join('\n');
+}
+function buildEnrichPrompt(families, projectRoot, testsRoot) {
     const sections = [];
+    const hasTestOnlyFamilies = families.some((f) => f.webappPaths.length === 0 && f.serverPaths.length === 0);
+    const resolvedTestsRoot = testsRoot ? resolve(testsRoot) : resolve(projectRoot);
     for (const family of families) {
+        const isTestOnly = family.webappPaths.length === 0 && family.serverPaths.length === 0;
         const allDirs = [
             ...family.webappPaths.map((p) => p.replace(/\/?\*.*$/, '')),
             ...family.serverPaths.map((p) => p.replace(/\/?\*.*$/, '')),
@@ -75,10 +113,19 @@ function buildEnrichPrompt(families, projectRoot) {
             if (samples.length >= MAX_FILES_PER_FAMILY)
                 break;
         }
+        // For test-only families, sample the test files themselves for richer context
+        if (isTestOnly) {
+            for (const specDir of family.specDirs) {
+                if (samples.length >= MAX_FILES_PER_FAMILY)
+                    break;
+                const fullDir = join(resolvedTestsRoot, specDir);
+                samples.push(...sampleFiles(fullDir, MAX_FILES_PER_FAMILY - samples.length));
+            }
+        }
         // Sample spec descriptions
         const specSamples = [];
         for (const specDir of family.specDirs) {
-            const fullDir = join(resolve(projectRoot), specDir);
+            const fullDir = join(resolvedTestsRoot, specDir);
             const specFiles = sampleFiles(fullDir, 5);
             for (const sf of specFiles) {
                 const matches = sf.content.match(/(?:test|it|describe)\s*\(\s*['"`]([^'"`]+)/g);
@@ -87,7 +134,7 @@ function buildEnrichPrompt(families, projectRoot) {
                 }
             }
         }
-        sections.push(`## Family: ${family.id}
+        sections.push(`## Family: ${family.id}${isTestOnly ? ' [TEST-ONLY — needs webappPaths/serverPaths]' : ''}
 Routes (guessed): ${JSON.stringify(family.routes)}
 Webapp paths: ${JSON.stringify(family.webappPaths)}
 Server paths: ${JSON.stringify(family.serverPaths)}
@@ -102,6 +149,10 @@ Test descriptions:
 ${specSamples.length > 0 ? specSamples.map((d) => `- ${d}`).join('\n') : '(none found)'}
 `);
     }
+    // Include source tree listing when we have test-only families
+    const sourceTreeSection = hasTestOnlyFamilies
+        ? `\n## Source Directory Structure\nUse this to suggest accurate webappPaths and serverPaths for test-only families:\n\`\`\`\n${getSourceTreeListing(projectRoot)}\n\`\`\`\n`
+        : '';
     return `You are analyzing a codebase to enrich route-family definitions for an E2E test impact analysis tool.
 
 For each family below, provide:
@@ -110,6 +161,8 @@ For each family below, provide:
 3. **routes**: Improved URL patterns (e.g., "/{team}/channels/{channel}" instead of "/channels")
 4. **pageObjects**: Array of page object class names found in the code
 5. **components**: Array of UI component names relevant to this family
+6. **webappPaths**: Array of glob patterns for frontend source directories (e.g., "src/components/drafts/**"). REQUIRED for families marked [TEST-ONLY].
+7. **serverPaths**: Array of glob patterns for backend source directories. REQUIRED for families marked [TEST-ONLY].
 
 Respond in JSON format:
 \`\`\`json
@@ -120,11 +173,13 @@ Respond in JSON format:
     "userFlows": ["Flow name 1", "Flow name 2"],
     "routes": ["/improved/route/{param}"],
     "pageObjects": ["PageName"],
-    "components": ["ComponentName"]
+    "components": ["ComponentName"],
+    "webappPaths": ["src/components/feature_name/**"],
+    "serverPaths": ["server/channels/api4/feature.go"]
   }
 ]
 \`\`\`
-
+${sourceTreeSection}
 ${sections.join('\n---\n')}`;
 }
 export function validateEntries(parsed) {
@@ -143,6 +198,8 @@ export function validateEntries(parsed) {
         userFlows: filterStrings(entry.userFlows, 500),
         pageObjects: filterStrings(entry.pageObjects, 200),
         components: filterStrings(entry.components, 200),
+        webappPaths: filterStrings(entry.webappPaths, 300),
+        serverPaths: filterStrings(entry.serverPaths, 300),
     }));
 }
 export function parseEnrichResponse(response) {
@@ -192,13 +249,22 @@ function applyEnrichment(family, enriched) {
     if (enriched.components && (!family.components || family.components.length === 0)) {
         result.components = enriched.components;
     }
+    // Only fill source paths when the family has none (test-derived families)
+    if (enriched.webappPaths && (!family.webappPaths || family.webappPaths.length === 0)) {
+        result.webappPaths = enriched.webappPaths;
+    }
+    if (enriched.serverPaths && (!family.serverPaths || family.serverPaths.length === 0)) {
+        result.serverPaths = enriched.serverPaths;
+    }
     return result;
 }
-export async function enrichFamilies(families, scanned, projectRoot, provider, budgetUSD) {
+export async function enrichFamilies(families, scanned, projectRoot, provider, budgetUSD, testsRoot) {
     const scannedMap = new Map(scanned.map((s) => [s.id, s]));
     const enriched = [];
     let totalTokens = 0;
     let totalCost = 0;
+    let requestCount = 0;
+    let totalResponseMs = 0;
     const skipped = [];
     // Process in chunks of 4 families
     const chunkSize = 4;
@@ -218,7 +284,7 @@ export async function enrichFamilies(families, scanned, projectRoot, provider, b
             enriched.push(...chunk);
             continue;
         }
-        let prompt = buildEnrichPrompt(scannedChunk, projectRoot);
+        let prompt = buildEnrichPrompt(scannedChunk, projectRoot, testsRoot);
         if (prompt.length > MAX_PROMPT_CHARS) {
             // Truncate at the last complete section boundary to avoid malformed input
             const lastSectionEnd = prompt.lastIndexOf('\n---\n', MAX_PROMPT_CHARS);
@@ -231,15 +297,18 @@ export async function enrichFamilies(families, scanned, projectRoot, provider, b
                 prompt = prompt.slice(0, MAX_PROMPT_CHARS);
             }
         }
-        let timer;
+        let timeoutTimer;
         try {
             const timeoutPromise = new Promise((_, reject) => {
-                timer = setTimeout(() => reject(new Error('LLM request timed out')), LLM_TIMEOUT_MS);
+                timeoutTimer = setTimeout(() => reject(new Error('LLM request timed out')), LLM_TIMEOUT_MS);
             });
+            const reqStart = performance.now();
             const response = await Promise.race([
                 provider.generateText(prompt, { maxTokens: 4096, temperature: 0.3 }),
                 timeoutPromise,
             ]);
+            totalResponseMs += performance.now() - reqStart;
+            requestCount++;
             totalTokens += (response.usage?.inputTokens ?? 0) + (response.usage?.outputTokens ?? 0);
             totalCost += response.cost ?? 0;
             const entries = parseEnrichResponse(response.text);
@@ -260,8 +329,8 @@ export async function enrichFamilies(families, scanned, projectRoot, provider, b
             enriched.push(...chunk);
         }
         finally {
-            if (timer)
-                clearTimeout(timer);
+            if (timeoutTimer)
+                clearTimeout(timeoutTimer);
         }
     }
     return {
@@ -269,5 +338,7 @@ export async function enrichFamilies(families, scanned, projectRoot, provider, b
         tokensUsed: totalTokens,
         costUSD: Math.round(totalCost * 100) / 100,
         skippedFamilies: skipped,
+        requestCount,
+        avgResponseMs: requestCount > 0 ? Math.round(totalResponseMs / requestCount) : 0,
     };
 }

package/dist/esm/training/merger.js

@@ -1,5 +1,6 @@
 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
 // See LICENSE.txt for license information.
+import { execFileSync } from 'child_process';
 import { existsSync } from 'fs';
 import { join, resolve } from 'path';
 import { isGuessedRoute } from './types.js';
@@ -67,6 +68,21 @@ function scannedToRouteFamily(scanned) {
     }
     return family;
 }
+/**
+ * Try to find a matching family ID with singular/plural normalization.
+ * "team" matches "teams", "emoji" matches "emoji", etc.
+ */
+function findFuzzyMatch(id, idMap) {
+    if (idMap.has(id))
+        return id;
+    // Try adding 's'
+    if (!id.endsWith('s') && idMap.has(id + 's'))
+        return id + 's';
+    // Try removing 's'
+    if (id.endsWith('s') && idMap.has(id.slice(0, -1)))
+        return id.slice(0, -1);
+    return undefined;
+}
 export function mergeFamilies(existing, scanned) {
     const existingFamilies = existing?.families || [];
     const existingMap = new Map(existingFamilies.map((f) => [f.id, f]));
@@ -74,9 +90,15 @@ export function mergeFamilies(existing, scanned) {
     const newFamilies = [];
     const updatedFamilies = [];
     const mergedFamilies = [];
-    // Process existing families
+    // Process existing families — match scanned by exact or fuzzy ID
     for (const ef of existingFamilies) {
-        const sf = scannedMap.get(ef.id);
+        let sf = scannedMap.get(ef.id);
+        // Try singular/plural match if exact match failed
+        if (!sf) {
+            const fuzzyId = findFuzzyMatch(ef.id, scannedMap);
+            if (fuzzyId)
+                sf = scannedMap.get(fuzzyId);
+        }
         if (sf) {
             mergedFamilies.push(mergeFamily(ef, sf));
             updatedFamilies.push(ef.id);
@@ -86,9 +108,10 @@ export function mergeFamilies(existing, scanned) {
             mergedFamilies.push({ ...ef });
         }
     }
-    // Add new families from scanner
+    // Add new families from scanner (if no existing family matched)
     for (const sf of scanned) {
-        if (!existingMap.has(sf.id)) {
+        const matchedExisting = findFuzzyMatch(sf.id, existingMap);
+        if (!matchedExisting) {
             mergedFamilies.push(scannedToRouteFamily(sf));
             newFamilies.push(sf.id);
         }
@@ -108,8 +131,33 @@ export function mergeFamilies(existing, scanned) {
         summary: parts.join(', '),
     };
 }
-export function detectStaleFamilies(manifest, projectRoot) {
-    const resolved = resolve(projectRoot);
+/**
+ * Detect families whose paths no longer exist on disk.
+ *
+ * Paths in the manifest may be relative to different roots:
+ * - webappPaths / serverPaths are typically relative to the repo root
+ * - specDirs may be relative to the tests root
+ *
+ * We try each pattern against all provided roots (and the git repo root
+ * if discoverable) to avoid false positives from path-prefix mismatches.
+ */
+export function detectStaleFamilies(manifest, projectRoot, testsRoot) {
+    const roots = new Set([resolve(projectRoot)]);
+    if (testsRoot)
+        roots.add(resolve(testsRoot));
+    // Also try to discover the git repo root — manifest paths may be repo-relative
+    try {
+        const gitRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], {
+            cwd: projectRoot,
+            encoding: 'utf-8',
+            stdio: ['pipe', 'pipe', 'pipe'],
+        }).trim();
+        if (gitRoot)
+            roots.add(resolve(gitRoot));
+    }
+    catch {
+        // Not a git repo or git not available — that's fine
+    }
     const stale = [];
     for (const family of manifest.families) {
         const allPatterns = [
@@ -119,15 +167,34 @@ export function detectStaleFamilies(manifest, projectRoot) {
         ];
         if (allPatterns.length === 0)
             continue;
-        // Check if any pattern resolves to existing files/dirs
+        // Check if any pattern resolves to existing files/dirs in any root
         let hasAny = false;
         for (const pattern of allPatterns) {
             // Strip trailing glob (* or **) to get the directory
             const dirPart = pattern.replace(/\/?\*.*$/, '');
-            if (dirPart && existsSync(join(resolved, dirPart))) {
-                hasAny = true;
-                break;
+            if (!dirPart)
+                continue;
+            // For file-level patterns like "server/channels/api4/draft*.go",
+            // dirPart is "server/channels/api4/draft" — check the parent dir instead
+            const isFileGlob = /\.\w+$/.test(pattern);
+            const pathsToCheck = [dirPart];
+            if (isFileGlob) {
+                const parentDir = dirPart.split('/').slice(0, -1).join('/');
+                if (parentDir)
+                    pathsToCheck.push(parentDir);
+            }
+            for (const checkPath of pathsToCheck) {
+                for (const root of roots) {
+                    if (existsSync(join(root, checkPath))) {
+                        hasAny = true;
+                        break;
+                    }
+                }
+                if (hasAny)
+                    break;
             }
+            if (hasAny)
+                break;
         }
         if (!hasAny) {
             stale.push(family.id);