@ash-mallick/browserstack-sync 1.1.2 → 1.1.3

package/lib/ai-analyzer.js

@@ -14,44 +14,133 @@ const OLLAMA_HOST = process.env.OLLAMA_HOST || 'http://localhost:11434';
 /**
  * System prompt for the AI to understand what we want.
  */
-const SYSTEM_PROMPT = `You are a test automation expert. Your job is to analyze Playwright or Cypress test code and extract human-readable test steps.
-
-For each test, output a JSON array of steps. Each step should have:
-- "step": A clear, human-readable action (e.g., "Navigate to /login page", "Enter 'test@example.com' in the email field")
-- "result": The expected result or assertion for this step (e.g., "Login form is displayed", "Error message 'Invalid credentials' appears")
-
-Rules:
-1. Convert code actions to plain English that a manual tester could follow
-2. For page.goto(url) -> "Navigate to <url>"
-3. For page.fill(selector, value) or getByLabel().fill() -> "Enter '<value>' in the <field name> field"
-4. For page.click() or getByRole().click() -> "Click the '<button/link name>' button/link"
-5. For expect() assertions -> This becomes the "result" of the previous step
-6. If there's no explicit assertion, the result should describe the expected state
-7. Be concise but clear
-8. Group related actions if they form one logical step
-
-Output ONLY valid JSON array, no markdown, no explanation. Example:
+const SYSTEM_PROMPT = `You are a QA expert creating detailed manual test steps from automated test code. Your goal is to write steps so clear and detailed that ANY tester can execute the test manually by just reading them.
+
+IMPORTANT: Output ONLY a valid JSON array. No markdown, no explanation, no extra text.
+
+Each step must have:
+- "step": A detailed, numbered action description. Include:
+  - Exact UI element to interact with (button name, field label, section title)
+  - Exact text/values to enter or look for
+  - Location hints (e.g., "in the header", "in the Solutions section", "at the top of the page")
+- "result": The specific expected outcome that can be visually verified. Include:
+  - Exact text that should be displayed
+  - UI state changes (visible, enabled, selected, etc.)
+  - URL changes if applicable
+
+RULES FOR DETAILED STEPS:
+1. NAVIGATION: "Open the browser and navigate to [exact URL]. Wait for the page to fully load."
+2. VISIBILITY CHECK: "Locate the [section name] section on the page. Verify it is visible and contains the text '[exact text]'."
+3. ELEMENT VERIFICATION: "In the [section name], find the [element type] with text '[text]'. Verify it is displayed."
+4. LINK VALIDATION: "Find the '[link text]' link in the [location]. Verify it has href pointing to '[URL pattern]'."
+5. BUTTON CHECK: "Locate the '[button text]' button in the [section]. Verify it is visible and clickable."
+6. CARD/TILE VALIDATION: "In the [section], locate the first card/tile. Verify it displays: title, image, and description."
+7. SCROLL ACTION: "Scroll down to the [section name] section to bring it into view."
+
+EXAMPLE OUTPUT:
 [
-  {"step": "Navigate to /login page", "result": "Login page loads successfully"},
-  {"step": "Enter 'user@test.com' in the Email field", "result": "Email is entered"},
-  {"step": "Enter 'password123' in the Password field", "result": "Password is masked and entered"},
-  {"step": "Click the 'Sign In' button", "result": "User is redirected to /dashboard"}
-]`;
+  {"step": "1. Open the browser and navigate to the application homepage (base URL + '/'). Wait for the page to fully load.", "result": "The homepage loads successfully. The masthead banner is visible at the top of the page."},
+  {"step": "2. Locate the masthead section at the top of the page. Verify the main heading is displayed.", "result": "The heading 'Accelerate AI with an open ecosystem' is visible in the masthead."},
+  {"step": "3. In the masthead section, find the 'Explore our AI ecosystem' call-to-action link.", "result": "The 'Explore our AI ecosystem' link is visible and has an href containing '/ai'."},
+  {"step": "4. Scroll down to the 'Solutions' section. Verify the section headline is displayed.", "result": "The Solutions section is visible with headline 'Find the solution you're looking for, in less time'."},
+  {"step": "5. In the Solutions section, locate the product slider/carousel. Verify the first product card is displayed.", "result": "The first product card shows: product title, product image, and product description."}
+]
+
+IMPORTANT: 
+- Number each step sequentially (1, 2, 3...)
+- Be specific about WHAT to look for and WHERE
+- Include exact text strings from the code in quotes
+- Make expected results measurable and verifiable
+- Output ONLY the JSON array, nothing else`;
 
 /**
  * Build the user prompt for AI analysis.
  */
 function buildUserPrompt(testTitle, testCode) {
-  return `Analyze this Playwright/Cypress test and extract the test steps:
+  return `Create detailed manual test steps from this automated test. A QA tester should be able to execute this test manually by reading your steps.
 
-Test Title: ${testTitle}
+TEST NAME: "${testTitle}"
 
-Test Code:
-\`\`\`
+AUTOMATED TEST CODE:
+\`\`\`javascript
 ${testCode}
 \`\`\`
 
-Return a JSON array of steps with "step" and "result" fields.`;
+ANALYZE THE CODE CAREFULLY:
+1. Look at page.goto() calls to identify which page/URL to navigate to
+2. Look at getByTestId(), locator(), getByRole(), getByLabel() to identify UI elements
+3. Look at expect() assertions to understand what needs to be verified
+4. Look at .toContainText(), .toHaveText(), .toBeVisible() for expected text/state
+5. Look at .toHaveAttribute('href', ...) for link URL validations
+6. Identify ALL assertions and convert them to expected results
+
+CREATE COMPREHENSIVE STEPS:
+- Include EVERY action from the test code
+- Include EVERY assertion/verification from the test code
+- Use exact text strings from .toContainText() and .toHaveText() in your expected results
+- Number steps sequentially (1, 2, 3...)
+- Make each step detailed enough for a manual tester
+
+OUTPUT: Return ONLY a valid JSON array, no other text.`;
+}
+
+/**
+ * Attempt to repair truncated JSON array.
+ * Handles cases where the response was cut off mid-array.
+ */
+function repairTruncatedJSON(jsonStr) {
+  // First, try parsing as-is
+  try {
+    return JSON.parse(jsonStr);
+  } catch (_) {
+    // Continue with repair attempts
+  }
+
+  // Try to find the last complete object in the array
+  // Pattern: find all complete {"step": "...", "result": "..."} objects
+  const objectPattern = /\{\s*"step"\s*:\s*"(?:[^"\\]|\\.)*"\s*,\s*"result"\s*:\s*"(?:[^"\\]|\\.)*"\s*\}/g;
+  const matches = jsonStr.match(objectPattern);
+  
+  if (matches && matches.length > 0) {
+    // Reconstruct a valid JSON array from complete objects
+    const reconstructed = '[' + matches.join(',') + ']';
+    try {
+      return JSON.parse(reconstructed);
+    } catch (_) {
+      // Fall through
+    }
+  }
+
+  // Try removing the last incomplete object and close the array
+  let trimmed = jsonStr.trim();
+  
+  // If it starts with [, try to close it properly
+  if (trimmed.startsWith('[')) {
+    // Find the last complete }, before truncation
+    const lastCompleteObj = trimmed.lastIndexOf('},');
+    if (lastCompleteObj > 0) {
+      const fixedJson = trimmed.slice(0, lastCompleteObj + 1) + ']';
+      try {
+        return JSON.parse(fixedJson);
+      } catch (_) {
+        // Fall through
+      }
+    }
+    
+    // Try finding the last }
+    const lastBrace = trimmed.lastIndexOf('}');
+    if (lastBrace > 0) {
+      const fixedJson = trimmed.slice(0, lastBrace + 1) + ']';
+      try {
+        return JSON.parse(fixedJson);
+      } catch (_) {
+        // Fall through
+      }
+    }
+  }
+
+  // Last resort: throw the original error
+  return JSON.parse(jsonStr);
 }
 
 /**
@@ -71,15 +160,22 @@ function parseAIResponse(content) {
 
   // Also try to extract JSON array from anywhere in the response
   if (!jsonStr.startsWith('[')) {
-    const arrayMatch = jsonStr.match(/\[[\s\S]*\]/);
+    const arrayMatch = jsonStr.match(/\[[\s\S]*?\](?=[^}\]]*$)?/);
     if (arrayMatch) {
       jsonStr = arrayMatch[0];
     }
   }
 
+  // Try to find just the array portion if there's extra text
+  const arrayStart = jsonStr.indexOf('[');
+  if (arrayStart > 0) {
+    jsonStr = jsonStr.slice(arrayStart);
+  }
+
   let steps;
   try {
-    steps = JSON.parse(jsonStr);
+    // Use repair function to handle truncated responses
+    steps = repairTruncatedJSON(jsonStr);
   } catch (parseError) {
     throw new Error(`Failed to parse AI response as JSON: ${parseError.message}. Response: ${content.slice(0, 200)}`);
   }
@@ -88,10 +184,13 @@ function parseAIResponse(content) {
     throw new Error('AI response is not an array');
   }
 
-  return steps.map((s) => ({
-    step: String(s.step || ''),
-    result: String(s.result || ''),
-  }));
+  // Filter out any empty or invalid steps
+  return steps
+    .filter((s) => s && (s.step || s.result))
+    .map((s) => ({
+      step: String(s.step || '').trim(),
+      result: String(s.result || '').trim(),
+    }));
 }
 
 /**
@@ -136,7 +235,9 @@ async function analyzeWithOllama(testTitle, testCode, model) {
       { role: 'user', content: buildUserPrompt(testTitle, testCode) },
     ],
     options: {
-      temperature: 0.2,
+      temperature: 0.1,      // Lower temperature for more consistent, focused output
+      num_predict: 8192,     // Increased max tokens for detailed step descriptions
+      num_ctx: 8192,         // Context window for longer test code input
     },
   });
 
@@ -176,47 +277,131 @@ export async function analyzeTestWithAI(testTitle, testCode, apiKey, options = {
  */
 export function extractStepsWithRegex(testCode, isCypress = false) {
   const steps = [];
+  let stepNum = 1;
 
-  // Common Playwright patterns
+  // Helper to clean up regex patterns from text
+  const cleanText = (text) => text?.replace(/\\?\//gi, '').replace(/\^|\$/g, '').trim() || '';
+
+  // Common Playwright patterns - order matters for sequential extraction
   const patterns = [
     // page.goto or cy.visit
     {
-      regex: /(?:page\.goto|cy\.visit)\s*\(\s*['"`]([^'"`]+)['"`]/gi,
-      format: (match) => ({ step: `Navigate to ${match[1]}`, result: 'Page loads successfully' }),
+      regex: /(?:page\.goto|cy\.visit)\s*\(\s*(?:process\.env\.[A-Z_]+\s*\+\s*)?['"`]([^'"`]+)['"`]/gi,
+      format: (match) => ({
+        step: `${stepNum++}. Open the browser and navigate to the URL: ${match[1] || '/'}. Wait for the page to fully load.`,
+        result: 'The page loads successfully and all elements are visible.',
+      }),
     },
-    // page.fill or cy.get().type() with getByLabel
+    // getByTestId - identify section
     {
-      regex: /(?:getByLabel|getByPlaceholder)\s*\([^)]+\)\.fill\s*\(\s*['"`]([^'"`]+)['"`]/gi,
-      format: (match) => {
-        const labelMatch = match.input.match(/(?:getByLabel|getByPlaceholder)\s*\(\s*['"`/]([^'"`/]+)/i);
-        const label = labelMatch ? labelMatch[1].replace(/\\?\/i$/, '') : 'field';
-        return { step: `Enter '${match[1]}' in the ${label} field`, result: 'Value is entered' };
-      },
+      regex: /(?:page\.|await\s+)getByTestId\s*\(\s*['"`]([^'"`]+)['"`]\s*\)/gi,
+      format: (match) => ({
+        step: `${stepNum++}. Locate the section/component with test ID "${match[1]}" on the page.`,
+        result: `The "${match[1]}" section is found and visible on the page.`,
+      }),
     },
-    // page.click or getByRole().click()
+    // locator with class or selector
     {
-      regex: /getByRole\s*\(\s*['"`]([^'"`]+)['"`]\s*,\s*\{[^}]*name:\s*['"`/]([^'"`/]+)/gi,
-      format: (match) => ({ step: `Click the '${match[2].replace(/\\?\/i$/, '')}' ${match[1]}`, result: `${match[1]} is clicked` }),
+      regex: /\.locator\s*\(\s*['"`]\.([^'"`]+)['"`]\s*\)(?:\.first\(\))?/gi,
+      format: (match) => ({
+        step: `${stepNum++}. Within the current section, find the element with class "${match[1]}".`,
+        result: `Element with class "${match[1]}" is located.`,
+      }),
+    },
+    // getByRole with name - link
+    {
+      regex: /getByRole\s*\(\s*['"`]link['"`]\s*,\s*\{[^}]*name:\s*['"`/]([^'"`/]+)/gi,
+      format: (match) => ({
+        step: `${stepNum++}. Find the link labeled "${cleanText(match[1])}" on the page.`,
+        result: `The "${cleanText(match[1])}" link is visible and clickable.`,
+      }),
+    },
+    // getByRole with name - button
+    {
+      regex: /getByRole\s*\(\s*['"`]button['"`]\s*,\s*\{[^}]*name:\s*['"`/]([^'"`/]+)/gi,
+      format: (match) => ({
+        step: `${stepNum++}. Locate the button labeled "${cleanText(match[1])}" on the page.`,
+        result: `The "${cleanText(match[1])}" button is displayed and enabled.`,
+      }),
+    },
+    // scrollIntoViewIfNeeded
+    {
+      regex: /scrollIntoViewIfNeeded\s*\(\s*\)/gi,
+      format: () => ({
+        step: `${stepNum++}. Scroll the page until the target element is visible in the viewport.`,
+        result: 'The element scrolls into view successfully.',
+      }),
     },
     // expect toBeVisible
     {
-      regex: /expect\s*\([^)]+\)\.toBeVisible\s*\(\s*\)/gi,
-      format: () => ({ step: 'Verify element is visible', result: 'Element is visible on the page' }),
+      regex: /expect\s*\([^)]+\)\.toBeVisible\s*\(/gi,
+      format: () => ({
+        step: `${stepNum++}. Verify that the element is visible on the page.`,
+        result: 'The element is confirmed to be visible and displayed.',
+      }),
+    },
+    // expect toContainText with text
+    {
+      regex: /expect\s*\([^)]+\)\.toContainText\s*\(\s*['"`]([^'"`]+)['"`]/gi,
+      format: (match) => ({
+        step: `${stepNum++}. Verify the element contains the text "${match[1]}".`,
+        result: `The text "${match[1]}" is displayed within the element.`,
+      }),
+    },
+    // expect toHaveText with text
+    {
+      regex: /expect\s*\([^)]+\)\.toHaveText\s*\(\s*['"`]([^'"`]+)['"`]/gi,
+      format: (match) => ({
+        step: `${stepNum++}. Verify the element has the exact text "${match[1]}".`,
+        result: `The element displays exactly: "${match[1]}".`,
+      }),
+    },
+    // expect toHaveAttribute href - capture full URL including slashes
+    {
+      regex: /expect\s*\([^)]+\)\.toHaveAttribute\s*\(\s*['"`]href['"`]\s*,\s*(?:\/)?['"`]?([^'"`\)]+?)['"`]?\s*\)/gi,
+      format: (match) => {
+        const url = match[1]?.trim().replace(/^\/|\/$/g, '') || '';
+        return {
+          step: `${stepNum++}. Verify the link has an href attribute pointing to "${url}".`,
+          result: `The link's href contains or matches "${url}".`,
+        };
+      },
     },
     // expect toHaveURL
     {
       regex: /expect\s*\([^)]+\)\.toHaveURL\s*\(\s*['"`/]([^'"`/]+)/gi,
-      format: (match) => ({ step: 'Verify URL', result: `URL matches ${match[1]}` }),
+      format: (match) => ({
+        step: `${stepNum++}. Verify the browser URL matches "${cleanText(match[1])}".`,
+        result: `The URL is confirmed to contain/match "${cleanText(match[1])}".`,
+      }),
+    },
+    // page.fill or getByLabel().fill()
+    {
+      regex: /(?:getByLabel|getByPlaceholder)\s*\([^)]+\)\.fill\s*\(\s*['"`]([^'"`]+)['"`]/gi,
+      format: (match) => {
+        const labelMatch = match.input.match(/(?:getByLabel|getByPlaceholder)\s*\(\s*['"`/]([^'"`/]+)/i);
+        const label = cleanText(labelMatch?.[1]) || 'input field';
+        return {
+          step: `${stepNum++}. Locate the "${label}" input field and enter the value "${match[1]}".`,
+          result: `The value "${match[1]}" is entered in the "${label}" field.`,
+        };
+      },
     },
-    // expect toHaveText or toContainText
+    // .click()
     {
-      regex: /expect\s*\([^)]+\)\.(?:toHaveText|toContainText)\s*\(\s*['"`/]([^'"`/]+)/gi,
-      format: (match) => ({ step: 'Verify text content', result: `Text '${match[1]}' is displayed` }),
+      regex: /\.click\s*\(\s*\)/gi,
+      format: () => ({
+        step: `${stepNum++}. Click on the identified element.`,
+        result: 'The click action is performed successfully.',
+      }),
     },
     // cy.get().should()
     {
       regex: /cy\.get\s*\([^)]+\)\.should\s*\(\s*['"`]([^'"`]+)['"`]/gi,
-      format: (match) => ({ step: 'Verify element state', result: `Element should ${match[1]}` }),
+      format: (match) => ({
+        step: `${stepNum++}. Verify the element state using Cypress assertion.`,
+        result: `Element assertion "${match[1]}" passes.`,
+      }),
     },
   ];
 
@@ -239,8 +424,8 @@ export function extractStepsWithRegex(testCode, isCypress = false) {
   if (steps.length === 0) {
     const fw = isCypress ? 'Cypress' : 'Playwright';
     steps.push({
-      step: `Execute ${fw} automated test`,
-      result: 'Test passes with all assertions verified',
+      step: `1. Execute the "${testTitle}" automated test using ${fw}.`,
+      result: 'Test completes successfully with all assertions verified.',
     });
   }
 

package/lib/index.js

@@ -38,35 +38,47 @@ export async function runSync(options = {}) {
     return;
   }
 
-  // Enrich with AI-powered step analysis (falls back to regex if no API key)
+  // Filter specs BEFORE AI analysis (to avoid analyzing specs we don't need)
+  if (options.spec && options.spec.length > 0) {
+    const set = new Set(options.spec);
+    // Match by exact name, base name, or partial match
+    specsMap = new Map([...specsMap].filter(([name]) => {
+      // Check exact match
+      if (set.has(name)) return true;
+      // Check if any spec pattern matches this name
+      for (const pattern of options.spec) {
+        if (name.includes(pattern) || pattern.includes(name)) return true;
+        // Also try without extension
+        const patternBase = pattern.replace(/\.(spec|test|cy)\.(js|ts)$/i, '');
+        const nameBase = name.replace(/\.(spec|test|cy)$/i, '');
+        if (nameBase.includes(patternBase) || patternBase.includes(nameBase)) return true;
+      }
+      return false;
+    }));
+    if (specsMap.size === 0) {
+      console.log('No matching spec files for:', options.spec.join(', '));
+      console.log('Available specs:', [...analyzeSpecs(e2eDir).keys()].join(', '));
+      return;
+    }
+  } else if (!options.all && !csvOnly && options.interactive !== false && process.stdin.isTTY) {
+    // Interactive selection BEFORE AI analysis
+    const selected = await promptSpecSelection(specsMap);
+    if (selected === null) {
+      // user chose "all"
+    } else if (selected.size === 0) {
+      console.log('No spec files selected. Exiting.');
+      return;
+    } else {
+      specsMap = new Map([...specsMap].filter(([name]) => selected.has(name)));
+    }
+  }
+
+  // Enrich with AI-powered step analysis (only for filtered specs)
   await enrichSpecsMapWithAI(specsMap, {
     useAI,
     model: options.model,
   });
 
-  if (!csvOnly) {
-    if (options.all) {
-      // sync all, no prompt
-    } else if (options.spec && options.spec.length > 0) {
-      const set = new Set(options.spec);
-      specsMap = new Map([...specsMap].filter(([name]) => set.has(name)));
-      if (specsMap.size === 0) {
-        console.log('No matching spec files for:', options.spec.join(', '));
-        return;
-      }
-    } else if (options.interactive !== false && process.stdin.isTTY) {
-      const selected = await promptSpecSelection(specsMap);
-      if (selected === null) {
-        // user chose "all"
-      } else if (selected.size === 0) {
-        console.log('No spec files selected. Exiting.');
-        return;
-      } else {
-        specsMap = new Map([...specsMap].filter(([name]) => selected.has(name)));
-      }
-    }
-  }
-
   writeCsvFiles(specsMap, csvOutputDir);
 
   if (csvOnly) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ash-mallick/browserstack-sync",
-  "version": "1.1.2",
+  "version": "1.1.3",
   "description": "Sync Playwright & Cypress e2e specs to CSV and BrowserStack Test Management with FREE AI-powered test step extraction using Ollama (local)",
   "author": "Ashutosh Mallick",
   "type": "module",