@merlean/analyzer 1.2.0 → 2.1.0

package/lib/analyzer.js

@@ -1,23 +1,32 @@
 /**
- * Codebase Scanner
+ * Frontend-focused Codebase Scanner
  * 
- * Scans codebase and prepares file contents for backend analysis.
- * NO LLM calls here - that happens on the backend.
+ * Scans FRONTEND code to learn how it communicates with the backend.
+ * Extracts: fetch(), axios, $.ajax, API calls, form submissions
+ * 
+ * This is what the bot needs - it runs in the browser and should know
+ * what API calls the frontend already makes.
  */
 
 const fs = require('fs');
 const path = require('path');
 const { glob } = require('glob');
 
-// File patterns to scan
-const SCAN_PATTERNS = [
+// Frontend file patterns (prioritize frontend code)
+const FRONTEND_PATTERNS = [
   '**/*.js',
-  '**/*.ts',
   '**/*.jsx',
+  '**/*.ts',
   '**/*.tsx',
-  '**/*.php',
-  '**/*.py',
-  '**/*.rb'
+  '**/*.vue',
+  '**/*.svelte',
+  '**/app.js',
+  '**/main.js',
+  '**/index.js',
+  '**/*api*.js',
+  '**/*service*.js',
+  '**/*fetch*.js',
+  '**/*http*.js'
 ];
 
 // Directories to ignore
@@ -30,23 +39,28 @@ const IGNORE_PATTERNS = [
   '**/__pycache__/**',
   '**/venv/**',
   '**/*.min.js',
-  '**/*.map'
+  '**/*.map',
+  '**/server.js',      // Skip backend files
+  '**/server/**',
+  '**/backend/**',
+  '**/api/**',         // Skip backend API folders
+  '**/controllers/**'
 ];
 
 // Keywords to prioritize files
 const PRIORITY_KEYWORDS = [
-  'route', 'router', 'controller', 'api', 'endpoint',
-  'form', 'submit', 'action', 'handler', 'service'
+  'fetch', 'axios', 'api', 'service', 'http', 'request',
+  'ajax', 'client', 'frontend', 'app', 'main', 'store'
 ];
 
 /**
- * Scan codebase and collect file contents
+ * Scan codebase and collect frontend API patterns
  */
 async function scanCodebase(codebasePath) {
-  console.log('   Scanning files...');
+  console.log('   Scanning frontend files...');
 
   // Get files to scan
-  const files = await glob(SCAN_PATTERNS, {
+  const files = await glob(FRONTEND_PATTERNS, {
     cwd: codebasePath,
     ignore: IGNORE_PATTERNS,
     absolute: true
@@ -54,116 +68,162 @@ async function scanCodebase(codebasePath) {
 
   console.log(`   Found ${files.length} files`);
 
-  // Prioritize and limit files
+  // Prioritize frontend-focused files
   const prioritizedFiles = prioritizeFiles(files, codebasePath);
-  const filesToAnalyze = prioritizedFiles.slice(0, 50); // Limit for performance
+  const filesToAnalyze = prioritizedFiles.slice(0, 30); // Fewer files, but more content
 
-  console.log(`   Preparing ${filesToAnalyze.length} priority files...`);
+  console.log(`   Analyzing ${filesToAnalyze.length} frontend files...`);
 
-  // Read and prepare file contents
+  // Read and extract API patterns from files
   const fileContents = [];
   for (const file of filesToAnalyze) {
     try {
       const content = fs.readFileSync(file, 'utf-8');
       const relativePath = path.relative(codebasePath, file);
       
-      // Smart extraction: if file is large, extract route-like patterns
-      let extractedContent;
-      if (content.length > 8000) {
-        // For large files, extract route definitions and API patterns
-        extractedContent = extractRoutePatterns(content, relativePath);
-      } else {
-        // For smaller files, include more content
-        extractedContent = content.slice(0, 8000);
-      }
+      // Extract API calls from the file
+      const extracted = extractApiPatterns(content, relativePath);
       
-      fileContents.push({
-        path: relativePath,
-        content: extractedContent
-      });
+      if (extracted.hasApiCalls) {
+        fileContents.push({
+          path: relativePath,
+          content: extracted.content
+        });
+      }
     } catch (error) {
       // Skip files that can't be read
     }
   }
 
+  console.log(`   Found API patterns in ${fileContents.length} files`);
+
   return fileContents;
 }
 
 /**
- * Extract route patterns from large files
+ * Extract API call patterns from file content
  */
-function extractRoutePatterns(content, filePath) {
+function extractApiPatterns(content, filePath) {
+  const apiPatterns = [];
   const lines = content.split('\n');
-  const relevantLines = [];
   
-  // Patterns that indicate API routes/endpoints
-  const routePatterns = [
-    /app\.(get|post|put|patch|delete|use)\s*\(/i,
-    /router\.(get|post|put|patch|delete|use)\s*\(/i,
-    /Route::(get|post|put|patch|delete)\s*\(/i,
-    /@(Get|Post|Put|Patch|Delete|RequestMapping)/i,
-    /def\s+(get|post|put|patch|delete|index|create|update|destroy)/i,
-    /function\s+\w+\s*\(\s*(req|request)/i,
-    /fetch\s*\(/i,
-    /axios\./i,
-    /api['"]\s*:/i,
-    /endpoint/i,
-    /\/api\//i
+  // Patterns that indicate API calls
+  const patterns = [
+    // fetch() calls
+    { regex: /fetch\s*\(\s*[`'"](.*?)[`'"]/g, type: 'fetch' },
+    { regex: /fetch\s*\(\s*`([^`]*)`/g, type: 'fetch-template' },
+    { regex: /fetch\s*\(\s*(['"])?\/api\//g, type: 'fetch-api' },
+    
+    // axios calls
+    { regex: /axios\.(get|post|put|patch|delete)\s*\(\s*[`'"](.*?)[`'"]/g, type: 'axios' },
+    { regex: /axios\s*\(\s*\{[^}]*url\s*:\s*[`'"](.*?)[`'"]/g, type: 'axios-config' },
+    
+    // jQuery ajax
+    { regex: /\$\.(ajax|get|post)\s*\(\s*[`'"](.*?)[`'"]/g, type: 'jquery' },
+    
+    // Generic API URLs
+    { regex: /['"`](\/api\/[^'"`\s]+)['"`]/g, type: 'api-url' },
+    { regex: /['"`](https?:\/\/[^'"`\s]*\/api[^'"`\s]*)['"`]/g, type: 'full-url' },
+    
+    // Method + URL patterns
+    { regex: /(GET|POST|PUT|PATCH|DELETE)\s*[,:]?\s*['"`](\/[^'"`]+)['"`]/gi, type: 'method-url' },
   ];
+
+  let hasApiCalls = false;
+  const extractedBlocks = [];
   
-  let inRouteBlock = false;
-  let braceCount = 0;
+  // FIRST: Extract API base URL definitions (critical for resolving paths)
+  const baseUrlDefinitions = [];
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    // Match: const API_BASE = ..., const baseURL = ..., const apiUrl = ..., etc.
+    if (/^\s*(const|let|var)\s+(API_BASE|API_URL|BASE_URL|baseURL|apiUrl|apiBase|API_ENDPOINT)/i.test(line)) {
+      baseUrlDefinitions.push(`${i + 1}: ${line}`);
+    }
+    // Also match: axios.defaults.baseURL = ...
+    if (/baseURL\s*[:=]/i.test(line)) {
+      baseUrlDefinitions.push(`${i + 1}: ${line}`);
+    }
+  }
   
+  if (baseUrlDefinitions.length > 0) {
+    extractedBlocks.push('// API BASE URL DEFINITIONS (use these to resolve full paths):\n' + baseUrlDefinitions.join('\n'));
+    hasApiCalls = true;
+  }
+
+  // Line-by-line extraction with context
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i];
     
-    // Check if line matches any route pattern
-    const isRouteLine = routePatterns.some(pattern => pattern.test(line));
+    // Check for API patterns
+    const hasPattern = patterns.some(p => p.regex.test(line));
+    // Reset regex lastIndex
+    patterns.forEach(p => p.regex.lastIndex = 0);
     
-    if (isRouteLine) {
-      // Include context: 2 lines before
-      for (let j = Math.max(0, i - 2); j < i; j++) {
-        if (!relevantLines.includes(lines[j])) {
-          relevantLines.push(`// Line ${j + 1}: ${lines[j]}`);
-        }
-      }
-      relevantLines.push(`// Line ${i + 1}: ${line}`);
-      inRouteBlock = true;
-      braceCount = (line.match(/{/g) || []).length - (line.match(/}/g) || []).length;
-    } else if (inRouteBlock) {
-      // Continue capturing the route handler
-      relevantLines.push(line);
-      braceCount += (line.match(/{/g) || []).length - (line.match(/}/g) || []).length;
+    // Also check for common API keywords
+    const hasKeyword = /fetch|axios|\.ajax|\.get\(|\.post\(|\.put\(|\.delete\(|\/api\/|endpoint/i.test(line);
+    
+    // Check if this is a POST/PUT request - need more context for body structure
+    const isPostRequest = /method:\s*['"]POST|\.post\(|method:\s*['"]PUT|\.put\(/i.test(line);
+    
+    if (hasPattern || hasKeyword) {
+      hasApiCalls = true;
       
-      // End of route block
-      if (braceCount <= 0) {
-        inRouteBlock = false;
-        relevantLines.push('// ---');
-      }
+      // Get MORE context for POST/PUT requests to capture body structure definitions
+      // Body objects are often defined 15-25 lines before the fetch call
+      const contextBefore = isPostRequest ? 25 : 3;  // More context for POST
+      const contextAfter = isPostRequest ? 10 : 5;
       
-      // Safety limit per block
-      if (relevantLines.length > 200) {
-        inRouteBlock = false;
-      }
+      const startLine = Math.max(0, i - contextBefore);
+      const endLine = Math.min(lines.length - 1, i + contextAfter);
+      
+      const block = lines.slice(startLine, endLine + 1)
+        .map((l, idx) => `${startLine + idx + 1}: ${l}`)
+        .join('\n');
+      
+      extractedBlocks.push(block);
+      
+      // Skip ahead to avoid duplicates
+      i = endLine;
     }
   }
-  
-  // If we found relevant lines, return them; otherwise return truncated content
-  if (relevantLines.length > 10) {
-    return `// Extracted route patterns from ${filePath} (${lines.length} lines total)\n\n${relevantLines.join('\n')}`;
+
+  // If we found API patterns, return extracted content
+  if (hasApiCalls && extractedBlocks.length > 0) {
+    return {
+      hasApiCalls: true,
+      content: `// File: ${filePath}\n// API patterns found:\n\n${extractedBlocks.join('\n\n// ---\n\n')}`
+    };
   }
-  
-  return content.slice(0, 8000);
+
+  // Fallback: include first 5000 chars if file looks relevant
+  if (/api|fetch|axios|service|http/i.test(filePath)) {
+    return {
+      hasApiCalls: true,
+      content: `// File: ${filePath}\n${content.slice(0, 5000)}`
+    };
+  }
+
+  return { hasApiCalls: false, content: '' };
 }
 
 /**
- * Prioritize files based on keywords in path/name
+ * Prioritize frontend files based on keywords
  */
 function prioritizeFiles(files, basePath) {
   return files.sort((a, b) => {
     const aPath = path.relative(basePath, a).toLowerCase();
     const bPath = path.relative(basePath, b).toLowerCase();
     
+    // Deprioritize test files
+    if (aPath.includes('test') || aPath.includes('spec')) return 1;
+    if (bPath.includes('test') || bPath.includes('spec')) return -1;
+    
+    // Prioritize src/frontend folders
+    if (aPath.includes('src/') || aPath.includes('frontend/')) {
+      if (!bPath.includes('src/') && !bPath.includes('frontend/')) return -1;
+    }
+    
     const aScore = PRIORITY_KEYWORDS.reduce((score, kw) => 
       aPath.includes(kw) ? score + 1 : score, 0);
     const bScore = PRIORITY_KEYWORDS.reduce((score, kw) => 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@merlean/analyzer",
-  "version": "1.2.0",
+  "version": "2.1.0",
   "description": "AI Bot codebase analyzer - generates site maps for AI assistant integration",
   "keywords": ["ai", "bot", "analyzer", "claude", "anthropic", "widget"],
   "author": "zmaren",