@ash-mallick/browserstack-sync 1.1.3 → 1.1.4

package/lib/ai-analyzer.js

@@ -14,74 +14,78 @@ const OLLAMA_HOST = process.env.OLLAMA_HOST || 'http://localhost:11434';
 /**
  * System prompt for the AI to understand what we want.
  */
-const SYSTEM_PROMPT = `You are a QA expert creating detailed manual test steps from automated test code. Your goal is to write steps so clear and detailed that ANY tester can execute the test manually by just reading them.
-
-IMPORTANT: Output ONLY a valid JSON array. No markdown, no explanation, no extra text.
-
-Each step must have:
-- "step": A detailed, numbered action description. Include:
-  - Exact UI element to interact with (button name, field label, section title)
-  - Exact text/values to enter or look for
-  - Location hints (e.g., "in the header", "in the Solutions section", "at the top of the page")
-- "result": The specific expected outcome that can be visually verified. Include:
-  - Exact text that should be displayed
-  - UI state changes (visible, enabled, selected, etc.)
-  - URL changes if applicable
-
-RULES FOR DETAILED STEPS:
-1. NAVIGATION: "Open the browser and navigate to [exact URL]. Wait for the page to fully load."
-2. VISIBILITY CHECK: "Locate the [section name] section on the page. Verify it is visible and contains the text '[exact text]'."
-3. ELEMENT VERIFICATION: "In the [section name], find the [element type] with text '[text]'. Verify it is displayed."
-4. LINK VALIDATION: "Find the '[link text]' link in the [location]. Verify it has href pointing to '[URL pattern]'."
-5. BUTTON CHECK: "Locate the '[button text]' button in the [section]. Verify it is visible and clickable."
-6. CARD/TILE VALIDATION: "In the [section], locate the first card/tile. Verify it displays: title, image, and description."
-7. SCROLL ACTION: "Scroll down to the [section name] section to bring it into view."
-
-EXAMPLE OUTPUT:
+const SYSTEM_PROMPT = `You are a QA expert creating manual test steps from automated test code.
+
+CRITICAL RULES:
+1. Generate steps ONLY for what is explicitly in the test code provided
+2. DO NOT add steps for things not in the code
+3. DO NOT hallucinate or assume content - use ONLY text/values from the code
+4. Each expect() or assertion = one verification step
+5. Output ONLY valid JSON array, nothing else
+
+STEP FORMAT:
+{
+  "step": "Numbered action description with exact element and location",
+  "result": "Expected outcome using exact text from the code"
+}
+
+MAPPING CODE TO STEPS:
+
+1. getByTestId('section-name') → "Locate the section with test ID 'section-name'"
+2. locator('.class-name') → "Find the element with class 'class-name'"
+3. getByRole('link', { name: /text/i }) → "Find the link labeled 'text'"
+4. expect(element).toBeVisible() → Result: "The element is visible"
+5. expect(element).toContainText('exact text') → Result: "The element displays 'exact text'"
+6. expect(element).toHaveText('exact text') → Result: "The element shows exactly 'exact text'"
+7. expect(link).toHaveAttribute('href', '/path') → Result: "The link href is '/path'"
+8. scrollIntoViewIfNeeded() → "Scroll to bring the element into view"
+
+EXAMPLE - For this code:
+  const scope = page.getByTestId("success-stories");
+  await expect(scope.locator(".band__headline")).toHaveText("Success stories");
+  
+OUTPUT:
 [
-  {"step": "1. Open the browser and navigate to the application homepage (base URL + '/'). Wait for the page to fully load.", "result": "The homepage loads successfully. The masthead banner is visible at the top of the page."},
-  {"step": "2. Locate the masthead section at the top of the page. Verify the main heading is displayed.", "result": "The heading 'Accelerate AI with an open ecosystem' is visible in the masthead."},
-  {"step": "3. In the masthead section, find the 'Explore our AI ecosystem' call-to-action link.", "result": "The 'Explore our AI ecosystem' link is visible and has an href containing '/ai'."},
-  {"step": "4. Scroll down to the 'Solutions' section. Verify the section headline is displayed.", "result": "The Solutions section is visible with headline 'Find the solution you're looking for, in less time'."},
-  {"step": "5. In the Solutions section, locate the product slider/carousel. Verify the first product card is displayed.", "result": "The first product card shows: product title, product image, and product description."}
+  {"step": "1. Locate the section with test ID 'success-stories' on the page.", "result": "The success-stories section is found."},
+  {"step": "2. In the success-stories section, find the element with class 'band__headline'. Verify its text content.", "result": "The headline displays exactly 'Success stories'."}
 ]
 
-IMPORTANT: 
-- Number each step sequentially (1, 2, 3...)
-- Be specific about WHAT to look for and WHERE
-- Include exact text strings from the code in quotes
-- Make expected results measurable and verifiable
-- Output ONLY the JSON array, nothing else`;
+REMEMBER:
+- ONLY use text/values that appear in the test code
+- DO NOT make up text or values
+- Number steps 1, 2, 3...
+- Be precise and specific`;
 
 /**
  * Build the user prompt for AI analysis.
  */
 function buildUserPrompt(testTitle, testCode) {
-  return `Create detailed manual test steps from this automated test. A QA tester should be able to execute this test manually by reading your steps.
+  return `Convert this automated test into manual test steps.
 
-TEST NAME: "${testTitle}"
+TEST: "${testTitle}"
 
-AUTOMATED TEST CODE:
-\`\`\`javascript
+CODE:
 ${testCode}
-\`\`\`
-
-ANALYZE THE CODE CAREFULLY:
-1. Look at page.goto() calls to identify which page/URL to navigate to
-2. Look at getByTestId(), locator(), getByRole(), getByLabel() to identify UI elements
-3. Look at expect() assertions to understand what needs to be verified
-4. Look at .toContainText(), .toHaveText(), .toBeVisible() for expected text/state
-5. Look at .toHaveAttribute('href', ...) for link URL validations
-6. Identify ALL assertions and convert them to expected results
-
-CREATE COMPREHENSIVE STEPS:
-- Include EVERY action from the test code
-- Include EVERY assertion/verification from the test code
-- Use exact text strings from .toContainText() and .toHaveText() in your expected results
-- Number steps sequentially (1, 2, 3...)
-- Make each step detailed enough for a manual tester
-
-OUTPUT: Return ONLY a valid JSON array, no other text.`;
+
+INSTRUCTIONS:
+1. Create one step for EACH expect() assertion in the code
+2. Use ONLY the exact text strings that appear in the code (in quotes or regex)
+3. For each getByTestId(), locator(), or getByRole() - describe THAT element
+4. For each .toContainText('X') or .toHaveText('X') - the expected result is "displays 'X'"
+5. For each .toHaveAttribute('href', 'Y') - the expected result is "href is 'Y'"
+6. DO NOT add extra text or values that are not in the code
+
+Example mapping:
+- expect(page.getByTestId("masthead")).toBeVisible() 
+  → Step: "Locate the element with test ID 'masthead'." Result: "The masthead element is visible."
+
+- expect(el).toContainText("Hello World")
+  → Step: "Verify the element's text content." Result: "The element contains 'Hello World'."
+
+- expect(link).toHaveAttribute("href", "/contact")
+  → Step: "Check the link's href attribute." Result: "The link href is '/contact'."
+
+OUTPUT: Return ONLY a JSON array of {step, result} objects. No other text.`;
 }
 
 /**

package/lib/browserstack.js

@@ -112,32 +112,91 @@ export async function updateTestCase(projectId, testCaseId, testCase, auth) {
 
 /** Match existing BS test case to our local case: by title or by our TC-xxx tag */
 function findExisting(existingList, localCase) {
-  const byTitle = existingList.find((t) => (t.title || '').trim() === (localCase.title || '').trim());
+  const byTitle = existingList.find((t) => (t.name || t.title || '').trim() === (localCase.title || '').trim());
   if (byTitle) return byTitle;
   const byTag = existingList.find((t) => (t.tags || []).includes(localCase.id));
   return byTag;
 }
 
+/**
+ * Compare local test case with existing BrowserStack test case.
+ * Returns true if content has changed and update is needed.
+ */
+function hasContentChanged(existing, localCase) {
+  // Compare steps - this is the main content that changes
+  const existingSteps = existing.test_case_steps || [];
+  const localSteps = Array.isArray(localCase.steps) ? localCase.steps : [];
+  
+  // Different number of steps = changed
+  if (existingSteps.length !== localSteps.length) {
+    return true;
+  }
+  
+  // Compare each step
+  for (let i = 0; i < localSteps.length; i++) {
+    const existingStep = existingSteps[i] || {};
+    const localStep = localSteps[i] || {};
+    
+    const existingStepText = (existingStep.step || '').trim();
+    const localStepText = (localStep.step || '').trim();
+    const existingResult = (existingStep.result || '').trim();
+    const localResult = (localStep.result || '').trim();
+    
+    if (existingStepText !== localStepText || existingResult !== localResult) {
+      return true;
+    }
+  }
+  
+  // Compare title
+  const existingTitle = (existing.name || existing.title || '').trim();
+  const localTitle = (localCase.title || '').trim();
+  if (existingTitle !== localTitle) {
+    return true;
+  }
+  
+  // No changes detected
+  return false;
+}
+
 export async function syncToBrowserStack(specsMap, projectId, auth) {
   console.log('\nSyncing to BrowserStack project', projectId);
+  
+  let totalCreated = 0;
+  let totalUpdated = 0;
+  let totalSkipped = 0;
+  
   for (const [baseName, { specFile, cases }] of specsMap) {
     const folderName = baseName;
     const folderId = await ensureFolder(projectId, folderName, null, auth);
     const existing = await getTestCasesInFolder(projectId, folderId, auth);
-    console.log('Folder:', folderName, '(id:', folderId, ')');
+    console.log('Folder:', folderName, '(id:', folderId, ') -', cases.length, 'test(s)');
+    
     for (const tc of cases) {
       try {
         const found = findExisting(existing, tc);
         if (found) {
-          await updateTestCase(projectId, found.identifier, tc, auth);
-          console.log('  Updated', tc.id, tc.title, '->', found.identifier);
+          // Check if content has actually changed before updating
+          if (hasContentChanged(found, tc)) {
+            await updateTestCase(projectId, found.identifier, tc, auth);
+            console.log('  ✓ Updated', tc.id, tc.title, '->', found.identifier);
+            totalUpdated++;
+          } else {
+            console.log('  ○ Skipped', tc.id, tc.title, '(no changes)');
+            totalSkipped++;
+          }
         } else {
           const id = await createTestCase(projectId, folderId, tc, auth);
-          console.log('  Created', tc.id, tc.title, '->', id);
+          console.log('  + Created', tc.id, tc.title, '->', id);
+          totalCreated++;
         }
       } catch (e) {
-        console.error('  Failed', tc.id, tc.title, e.message);
+        console.error('  ✗ Failed', tc.id, tc.title, e.message);
       }
     }
   }
+  
+  console.log('\nSync summary:');
+  console.log(`  Created: ${totalCreated} test case(s)`);
+  console.log(`  Updated: ${totalUpdated} test case(s)`);
+  console.log(`  Skipped: ${totalSkipped} test case(s) (no changes)`);
 }

package/lib/enrich.js

@@ -40,6 +40,46 @@ function inferTags(specBaseName, title, isCypress) {
   return tags;
 }
 
+/**
+ * Count the number of expect() assertions in test code.
+ */
+function countAssertions(code) {
+  const expectMatches = code.match(/expect\s*\(/g) || [];
+  return expectMatches.length;
+}
+
+/**
+ * Merge AI steps with regex steps to ensure completeness.
+ * If AI generated fewer steps than assertions, supplement with unique regex steps.
+ */
+function mergeStepsForCompleteness(aiSteps, regexSteps, assertionCount) {
+  if (!aiSteps || aiSteps.length === 0) {
+    return regexSteps;
+  }
+  
+  // If AI generated enough steps, use AI steps
+  if (aiSteps.length >= assertionCount * 0.7) { // 70% coverage threshold
+    return aiSteps;
+  }
+  
+  // If AI generated too few steps, supplement with unique regex steps
+  const aiStepTexts = new Set(aiSteps.map(s => s.step.toLowerCase()));
+  const supplementalSteps = regexSteps.filter(rs => {
+    // Only add regex steps that aren't similar to existing AI steps
+    const rsLower = rs.step.toLowerCase();
+    return !Array.from(aiStepTexts).some(aiText => 
+      aiText.includes(rsLower.slice(0, 30)) || rsLower.includes(aiText.slice(0, 30))
+    );
+  });
+  
+  // Combine and renumber
+  const combined = [...aiSteps, ...supplementalSteps];
+  return combined.map((step, idx) => ({
+    step: step.step.replace(/^\d+\.\s*/, `${idx + 1}. `),
+    result: step.result,
+  }));
+}
+
 /**
  * Enrich a single case with state, case_type, steps, expected_results, jira_issues, automation_status, tags, description.
  * @param {Object} case_ - Test case with id, title, code
@@ -54,13 +94,19 @@ export function enrichCase(case_, specBaseName, isCypress = false, aiSteps = nul
   const fw = isCypress ? FRAMEWORK.cypress : FRAMEWORK.playwright;
   const tags = inferTags(specBaseName, title, isCypress);
 
-  // Determine steps: AI steps > regex-extracted steps > default
+  // Count assertions in code for completeness check
+  const assertionCount = code ? countAssertions(code) : 0;
+  
+  // Get regex-based steps as fallback/supplement
+  const regexSteps = code ? extractStepsWithRegex(code, isCypress) : [];
+
+  // Determine steps: merge AI + regex for completeness, or use regex only
   let steps;
   if (aiSteps && aiSteps.length > 0) {
-    steps = aiSteps;
-  } else if (code) {
-    // Try regex-based extraction from code
-    steps = extractStepsWithRegex(code, isCypress);
+    // Merge AI steps with regex steps to ensure we don't miss assertions
+    steps = mergeStepsForCompleteness(aiSteps, regexSteps, assertionCount);
+  } else if (regexSteps.length > 0) {
+    steps = regexSteps;
   } else {
     // Fallback to generic step
     steps = [{ step: fw.step, result: DEFAULT_EXPECTED_RESULT }];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ash-mallick/browserstack-sync",
-  "version": "1.1.3",
+  "version": "1.1.4",
   "description": "Sync Playwright & Cypress e2e specs to CSV and BrowserStack Test Management with FREE AI-powered test step extraction using Ollama (local)",
   "author": "Ashutosh Mallick",
   "type": "module",
@@ -42,6 +42,7 @@
     "url": ""
   },
   "dependencies": {
+    "@ash-mallick/browserstack-sync": "^1.1.3",
     "dotenv": "^16.4.5",
     "ollama": "^0.6.3"
   },