@yasserkhanorg/e2e-agents 0.5.11 → 0.5.13

package/dist/agent/ai_mapping.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ai_mapping.d.ts","sourceRoot":"","sources":["../../src/agent/ai_mapping.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAC,YAAY,EAAE,QAAQ,EAAC,MAAM,YAAY,CAAC;AA4BvD,MAAM,WAAW,eAAe;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACtB;AAuTD,wBAAsB,iBAAiB,CACnC,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,qBAAqB,EAC7B,KAAK,EAAE,UAAU,EAAE,EACnB,KAAK,EAAE,QAAQ,EAAE,GAClB,OAAO,CAAC,eAAe,CAAC,CA0O1B"}
+{"version":3,"file":"ai_mapping.d.ts","sourceRoot":"","sources":["../../src/agent/ai_mapping.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAC,YAAY,EAAE,QAAQ,EAAC,MAAM,YAAY,CAAC;AA4BvD,MAAM,WAAW,eAAe;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACtB;AAyWD,wBAAsB,iBAAiB,CACnC,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,qBAAqB,EAC7B,KAAK,EAAE,UAAU,EAAE,EACnB,KAAK,EAAE,QAAQ,EAAE,GAClB,OAAO,CAAC,eAAe,CAAC,CA4O1B"}

package/dist/agent/ai_mapping.js

@@ -139,6 +139,21 @@ function isStrongCandidateMatch(flow, matchedKeywords) {
     const keywords = flowKeywords(flow);
     return keywords.length === 1 && matchedKeywords[0].length >= MIN_SINGLE_KEYWORD_LENGTH;
 }
+// Stop-words excluded from content-fallback keyword matching.
+const CONTENT_FALLBACK_STOP_WORDS = new Set(['and', 'for', 'the', 'to', 'of', 'on', 'at', 'with', 'in', 'a', 'an']);
+// Returns raw (unfiltered) tokens for flows where flowKeywords() returns nothing.
+// Used exclusively for content-title matching when all standard keywords are low-signal.
+// Empty array is returned when the flow already has effective path keywords.
+function contentFallbackKeywords(flow) {
+    if (flowKeywords(flow).length > 0) {
+        return [];
+    }
+    return (0, utils_js_1.uniqueTokens)([
+        ...(0, utils_js_1.tokenize)(flow.id || ''),
+        ...(0, utils_js_1.tokenize)(flow.name || ''),
+        ...(flow.keywords || []),
+    ]).filter((k) => k.length >= 3 && !CONTENT_FALLBACK_STOP_WORDS.has(k));
+}
 // Extract test/describe/it title strings from file content for semantic matching.
 function extractTestTitles(content) {
     const titles = [];
@@ -214,7 +229,39 @@ function selectCandidateTests(flows, tests, maxCandidateTests) {
             }
             contentCandidates.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path));
         }
-        const allCandidates = [...strongCandidates, ...contentCandidates.slice(0, perFlowLimit)];
+        // Pass 2b: comprehensive fallback for all-low-signal flows.
+        // When flowKeywords() is empty (all tokens are low-signal), flowKeywords-based
+        // matching in both Pass 1 and Pass 2 yields nothing. As a last resort, search
+        // test titles using the raw unfiltered tokens from the flow ID/name, but require
+        // ALL tokens to match simultaneously — they are individually weak signals so the
+        // full conjunction is needed to establish behavioral coverage evidence.
+        const fallbackCandidates = [];
+        if (strongCandidates.length === 0 && contentCandidates.length === 0) {
+            const fallbackKws = contentFallbackKeywords(flow);
+            if (fallbackKws.length > 0) {
+                for (const testPath of normalizedTests) {
+                    const testFile = testByNormalizedPath.get(testPath);
+                    if (!testFile?.content) {
+                        continue;
+                    }
+                    const haystack = extractTestTitles(testFile.content).toLowerCase();
+                    if (!haystack) {
+                        continue;
+                    }
+                    const matched = fallbackKws.filter((k) => haystack.includes(k));
+                    if (matched.length < fallbackKws.length) {
+                        continue; // all tokens must appear in at least one test title
+                    }
+                    fallbackCandidates.push({ path: testPath, score: matched.length, matchedKeywords: matched });
+                }
+                fallbackCandidates.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path));
+            }
+        }
+        const allCandidates = [
+            ...strongCandidates,
+            ...contentCandidates.slice(0, perFlowLimit),
+            ...fallbackCandidates.slice(0, perFlowLimit),
+        ];
         if (allCandidates.length === 0) {
             // Exact-name fallback: if the flow ID has no effective keywords (all tokens are
             // low-signal, e.g. view_user_group_modal), look for a test whose path contains
@@ -371,13 +418,12 @@ async function mapAITestsToFlows(appRoot, testsRoot, config, flows, tests) {
         'Rules:',
         '- Keep at most 5 tests per flow.',
         '- Use exact flowId values from FLOWS.',
-        '- Only map a test when its path clearly matches the flow scenario. Generic subsystem similarity is not enough.',
+        '- Map a test when you have clear evidence it covers the flow — from the file path OR from test titles in the content. Behavioral coverage via test titles is sufficient even when the filename does not exactly match the flow (e.g. search_user_post_spec.js covers search_messages if its titles assert searching for messages). Generic subsystem similarity without behavioral evidence is not enough.',
         '- A flow may only map to tests listed under FLOW_CANDIDATE_SIGNALS for that flow.',
         '- Treat single-keyword or broad subsystem overlap as insufficient evidence.',
         '- If the candidate path overlap is weak or ambiguous, return tests: [].',
         '- If unsure for a flow, return tests: [].',
-        '- For every flow you map to tests, read CANDIDATE_TEST_CONTENT and list up to 5 specific test scenarios NOT yet covered by those tests. Write each as a short imperative statement (e.g. "Search messages with date filter"). Only include missingScenarios you can clearly identify; return [] if unsure.',
-        '- If tests: [], set missingScenarios: [] as well — do not invent scenarios for unmapped flows.',
+        '- For EVERY flow (whether or not tests were found), return missingScenarios with 3-5 key user-facing test scenarios that must be covered. Write each as a short imperative statement starting with a verb (e.g. "Search for a message by keyword and verify results appear"). For mapped flows, focus on what the existing tests do NOT cover; for unmapped flows, describe the core scenarios a new test should include.',
         '',
         `FLOWS (${prioritizedFlows.length}):`,
         JSON.stringify(prioritizedFlows.map((flow) => ({
@@ -444,6 +490,16 @@ async function mapAITestsToFlows(appRoot, testsRoot, config, flows, tests) {
         if (!entry || !allowedFlowIds.has(entry.flowId) || !Array.isArray(entry.tests)) {
             continue;
         }
+        // Capture scenario suggestions for ALL flows up-front — before any early returns —
+        // so unmapped flows (tests: []) still get their suggested scenarios in the gap report.
+        if (Array.isArray(entry.missingScenarios) && entry.missingScenarios.length > 0) {
+            const scenarios = entry.missingScenarios
+                .filter((s) => typeof s === 'string' && s.trim().length > 0)
+                .slice(0, 5);
+            if (scenarios.length > 0) {
+                scenarioGaps.set(entry.flowId, scenarios);
+            }
+        }
         const flow = prioritizedFlowsById.get(entry.flowId);
         const confidence = typeof entry.confidence === 'number' ? entry.confidence : undefined;
         const allowedTestsForFlow = candidateSelection.byFlow.get(entry.flowId);
@@ -462,15 +518,6 @@ async function mapAITestsToFlows(appRoot, testsRoot, config, flows, tests) {
         for (const testPath of valid) {
             matchedTests.add(testPath);
         }
-        // Store missing scenarios identified by the AI for this flow.
-        if (Array.isArray(entry.missingScenarios) && entry.missingScenarios.length > 0) {
-            const scenarios = entry.missingScenarios
-                .filter((s) => typeof s === 'string' && s.trim().length > 0)
-                .slice(0, 5);
-            if (scenarios.length > 0) {
-                scenarioGaps.set(entry.flowId, scenarios);
-            }
-        }
     }
     // Post-AI exact-name fallback: for any flow still uncovered, search all test paths
     // for a file or directory whose name exactly matches the flow ID. This handles flows

package/dist/esm/agent/ai_mapping.js

@@ -136,6 +136,21 @@ function isStrongCandidateMatch(flow, matchedKeywords) {
     const keywords = flowKeywords(flow);
     return keywords.length === 1 && matchedKeywords[0].length >= MIN_SINGLE_KEYWORD_LENGTH;
 }
+// Stop-words excluded from content-fallback keyword matching.
+const CONTENT_FALLBACK_STOP_WORDS = new Set(['and', 'for', 'the', 'to', 'of', 'on', 'at', 'with', 'in', 'a', 'an']);
+// Returns raw (unfiltered) tokens for flows where flowKeywords() returns nothing.
+// Used exclusively for content-title matching when all standard keywords are low-signal.
+// Empty array is returned when the flow already has effective path keywords.
+function contentFallbackKeywords(flow) {
+    if (flowKeywords(flow).length > 0) {
+        return [];
+    }
+    return uniqueTokens([
+        ...tokenize(flow.id || ''),
+        ...tokenize(flow.name || ''),
+        ...(flow.keywords || []),
+    ]).filter((k) => k.length >= 3 && !CONTENT_FALLBACK_STOP_WORDS.has(k));
+}
 // Extract test/describe/it title strings from file content for semantic matching.
 function extractTestTitles(content) {
     const titles = [];
@@ -211,7 +226,39 @@ function selectCandidateTests(flows, tests, maxCandidateTests) {
             }
             contentCandidates.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path));
         }
-        const allCandidates = [...strongCandidates, ...contentCandidates.slice(0, perFlowLimit)];
+        // Pass 2b: comprehensive fallback for all-low-signal flows.
+        // When flowKeywords() is empty (all tokens are low-signal), flowKeywords-based
+        // matching in both Pass 1 and Pass 2 yields nothing. As a last resort, search
+        // test titles using the raw unfiltered tokens from the flow ID/name, but require
+        // ALL tokens to match simultaneously — they are individually weak signals so the
+        // full conjunction is needed to establish behavioral coverage evidence.
+        const fallbackCandidates = [];
+        if (strongCandidates.length === 0 && contentCandidates.length === 0) {
+            const fallbackKws = contentFallbackKeywords(flow);
+            if (fallbackKws.length > 0) {
+                for (const testPath of normalizedTests) {
+                    const testFile = testByNormalizedPath.get(testPath);
+                    if (!testFile?.content) {
+                        continue;
+                    }
+                    const haystack = extractTestTitles(testFile.content).toLowerCase();
+                    if (!haystack) {
+                        continue;
+                    }
+                    const matched = fallbackKws.filter((k) => haystack.includes(k));
+                    if (matched.length < fallbackKws.length) {
+                        continue; // all tokens must appear in at least one test title
+                    }
+                    fallbackCandidates.push({ path: testPath, score: matched.length, matchedKeywords: matched });
+                }
+                fallbackCandidates.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path));
+            }
+        }
+        const allCandidates = [
+            ...strongCandidates,
+            ...contentCandidates.slice(0, perFlowLimit),
+            ...fallbackCandidates.slice(0, perFlowLimit),
+        ];
         if (allCandidates.length === 0) {
             // Exact-name fallback: if the flow ID has no effective keywords (all tokens are
             // low-signal, e.g. view_user_group_modal), look for a test whose path contains
@@ -368,13 +415,12 @@ export async function mapAITestsToFlows(appRoot, testsRoot, config, flows, tests
         'Rules:',
         '- Keep at most 5 tests per flow.',
         '- Use exact flowId values from FLOWS.',
-        '- Only map a test when its path clearly matches the flow scenario. Generic subsystem similarity is not enough.',
+        '- Map a test when you have clear evidence it covers the flow — from the file path OR from test titles in the content. Behavioral coverage via test titles is sufficient even when the filename does not exactly match the flow (e.g. search_user_post_spec.js covers search_messages if its titles assert searching for messages). Generic subsystem similarity without behavioral evidence is not enough.',
         '- A flow may only map to tests listed under FLOW_CANDIDATE_SIGNALS for that flow.',
         '- Treat single-keyword or broad subsystem overlap as insufficient evidence.',
         '- If the candidate path overlap is weak or ambiguous, return tests: [].',
         '- If unsure for a flow, return tests: [].',
-        '- For every flow you map to tests, read CANDIDATE_TEST_CONTENT and list up to 5 specific test scenarios NOT yet covered by those tests. Write each as a short imperative statement (e.g. "Search messages with date filter"). Only include missingScenarios you can clearly identify; return [] if unsure.',
-        '- If tests: [], set missingScenarios: [] as well — do not invent scenarios for unmapped flows.',
+        '- For EVERY flow (whether or not tests were found), return missingScenarios with 3-5 key user-facing test scenarios that must be covered. Write each as a short imperative statement starting with a verb (e.g. "Search for a message by keyword and verify results appear"). For mapped flows, focus on what the existing tests do NOT cover; for unmapped flows, describe the core scenarios a new test should include.',
         '',
         `FLOWS (${prioritizedFlows.length}):`,
         JSON.stringify(prioritizedFlows.map((flow) => ({
@@ -441,6 +487,16 @@ export async function mapAITestsToFlows(appRoot, testsRoot, config, flows, tests
         if (!entry || !allowedFlowIds.has(entry.flowId) || !Array.isArray(entry.tests)) {
             continue;
         }
+        // Capture scenario suggestions for ALL flows up-front — before any early returns —
+        // so unmapped flows (tests: []) still get their suggested scenarios in the gap report.
+        if (Array.isArray(entry.missingScenarios) && entry.missingScenarios.length > 0) {
+            const scenarios = entry.missingScenarios
+                .filter((s) => typeof s === 'string' && s.trim().length > 0)
+                .slice(0, 5);
+            if (scenarios.length > 0) {
+                scenarioGaps.set(entry.flowId, scenarios);
+            }
+        }
         const flow = prioritizedFlowsById.get(entry.flowId);
         const confidence = typeof entry.confidence === 'number' ? entry.confidence : undefined;
         const allowedTestsForFlow = candidateSelection.byFlow.get(entry.flowId);
@@ -459,15 +515,6 @@ export async function mapAITestsToFlows(appRoot, testsRoot, config, flows, tests
         for (const testPath of valid) {
             matchedTests.add(testPath);
         }
-        // Store missing scenarios identified by the AI for this flow.
-        if (Array.isArray(entry.missingScenarios) && entry.missingScenarios.length > 0) {
-            const scenarios = entry.missingScenarios
-                .filter((s) => typeof s === 'string' && s.trim().length > 0)
-                .slice(0, 5);
-            if (scenarios.length > 0) {
-                scenarioGaps.set(entry.flowId, scenarios);
-            }
-        }
     }
     // Post-AI exact-name fallback: for any flow still uncovered, search all test paths
     // for a file or directory whose name exactly matches the flow ID. This handles flows

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yasserkhanorg/e2e-agents",
-  "version": "0.5.11",
+  "version": "0.5.13",
   "description": "Pluggable LLM provider library for AI-powered test automation. Use Claude, Ollama, or your own LLM. Integrate with Playwright, Jest, or any test framework. MCP server for test agents, cost tracking, and hybrid provider mode.",
   "main": "dist/index.js",
   "module": "dist/esm/index.js",