@merlean/analyzer 1.1.1 → 2.0.0

package/lib/analyzer.js

@@ -1,23 +1,32 @@
 /**
- * Codebase Scanner
+ * Frontend-focused Codebase Scanner
  * 
- * Scans codebase and prepares file contents for backend analysis.
- * NO LLM calls here - that happens on the backend.
+ * Scans FRONTEND code to learn how it communicates with the backend.
+ * Extracts: fetch(), axios, $.ajax, API calls, form submissions
+ * 
+ * This is what the bot needs - it runs in the browser and should know
+ * what API calls the frontend already makes.
  */
 
 const fs = require('fs');
 const path = require('path');
 const { glob } = require('glob');
 
-// File patterns to scan
-const SCAN_PATTERNS = [
+// Frontend file patterns (prioritize frontend code)
+const FRONTEND_PATTERNS = [
   '**/*.js',
-  '**/*.ts',
   '**/*.jsx',
+  '**/*.ts',
   '**/*.tsx',
-  '**/*.php',
-  '**/*.py',
-  '**/*.rb'
+  '**/*.vue',
+  '**/*.svelte',
+  '**/app.js',
+  '**/main.js',
+  '**/index.js',
+  '**/*api*.js',
+  '**/*service*.js',
+  '**/*fetch*.js',
+  '**/*http*.js'
 ];
 
 // Directories to ignore
@@ -30,23 +39,28 @@ const IGNORE_PATTERNS = [
   '**/__pycache__/**',
   '**/venv/**',
   '**/*.min.js',
-  '**/*.map'
+  '**/*.map',
+  '**/server.js',      // Skip backend files
+  '**/server/**',
+  '**/backend/**',
+  '**/api/**',         // Skip backend API folders
+  '**/controllers/**'
 ];
 
 // Keywords to prioritize files
 const PRIORITY_KEYWORDS = [
-  'route', 'router', 'controller', 'api', 'endpoint',
-  'form', 'submit', 'action', 'handler', 'service'
+  'fetch', 'axios', 'api', 'service', 'http', 'request',
+  'ajax', 'client', 'frontend', 'app', 'main', 'store'
 ];
 
 /**
- * Scan codebase and collect file contents
+ * Scan codebase and collect frontend API patterns
  */
 async function scanCodebase(codebasePath) {
-  console.log('   Scanning files...');
+  console.log('   Scanning frontend files...');
 
   // Get files to scan
-  const files = await glob(SCAN_PATTERNS, {
+  const files = await glob(FRONTEND_PATTERNS, {
     cwd: codebasePath,
     ignore: IGNORE_PATTERNS,
     absolute: true
@@ -54,42 +68,136 @@ async function scanCodebase(codebasePath) {
 
   console.log(`   Found ${files.length} files`);
 
-  // Prioritize and limit files
+  // Prioritize frontend-focused files
   const prioritizedFiles = prioritizeFiles(files, codebasePath);
-  const filesToAnalyze = prioritizedFiles.slice(0, 50); // Limit for performance
+  const filesToAnalyze = prioritizedFiles.slice(0, 30); // Fewer files, but more content
 
-  console.log(`   Preparing ${filesToAnalyze.length} priority files...`);
+  console.log(`   Analyzing ${filesToAnalyze.length} frontend files...`);
 
-  // Read and prepare file contents
+  // Read and extract API patterns from files
   const fileContents = [];
   for (const file of filesToAnalyze) {
     try {
       const content = fs.readFileSync(file, 'utf-8');
       const relativePath = path.relative(codebasePath, file);
       
-      // Limit content per file
-      const truncatedContent = content.slice(0, 3000);
+      // Extract API calls from the file
+      const extracted = extractApiPatterns(content, relativePath);
       
-      fileContents.push({
-        path: relativePath,
-        content: truncatedContent
-      });
+      if (extracted.hasApiCalls) {
+        fileContents.push({
+          path: relativePath,
+          content: extracted.content
+        });
+      }
     } catch (error) {
       // Skip files that can't be read
     }
   }
 
+  console.log(`   Found API patterns in ${fileContents.length} files`);
+
   return fileContents;
 }
 
 /**
- * Prioritize files based on keywords in path/name
+ * Extract API call patterns from file content
+ */
+function extractApiPatterns(content, filePath) {
+  const apiPatterns = [];
+  const lines = content.split('\n');
+  
+  // Patterns that indicate API calls
+  const patterns = [
+    // fetch() calls
+    { regex: /fetch\s*\(\s*[`'"](.*?)[`'"]/g, type: 'fetch' },
+    { regex: /fetch\s*\(\s*`([^`]*)`/g, type: 'fetch-template' },
+    { regex: /fetch\s*\(\s*(['"])?\/api\//g, type: 'fetch-api' },
+    
+    // axios calls
+    { regex: /axios\.(get|post|put|patch|delete)\s*\(\s*[`'"](.*?)[`'"]/g, type: 'axios' },
+    { regex: /axios\s*\(\s*\{[^}]*url\s*:\s*[`'"](.*?)[`'"]/g, type: 'axios-config' },
+    
+    // jQuery ajax
+    { regex: /\$\.(ajax|get|post)\s*\(\s*[`'"](.*?)[`'"]/g, type: 'jquery' },
+    
+    // Generic API URLs
+    { regex: /['"`](\/api\/[^'"`\s]+)['"`]/g, type: 'api-url' },
+    { regex: /['"`](https?:\/\/[^'"`\s]*\/api[^'"`\s]*)['"`]/g, type: 'full-url' },
+    
+    // Method + URL patterns
+    { regex: /(GET|POST|PUT|PATCH|DELETE)\s*[,:]?\s*['"`](\/[^'"`]+)['"`]/gi, type: 'method-url' },
+  ];
+
+  let hasApiCalls = false;
+  const extractedBlocks = [];
+
+  // Line-by-line extraction with context
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    
+    // Check for API patterns
+    const hasPattern = patterns.some(p => p.regex.test(line));
+    // Reset regex lastIndex
+    patterns.forEach(p => p.regex.lastIndex = 0);
+    
+    // Also check for common API keywords
+    const hasKeyword = /fetch|axios|\.ajax|\.get\(|\.post\(|\.put\(|\.delete\(|\/api\/|endpoint|baseURL/i.test(line);
+    
+    if (hasPattern || hasKeyword) {
+      hasApiCalls = true;
+      
+      // Get context: 3 lines before and 5 lines after
+      const startLine = Math.max(0, i - 3);
+      const endLine = Math.min(lines.length - 1, i + 5);
+      
+      const block = lines.slice(startLine, endLine + 1)
+        .map((l, idx) => `${startLine + idx + 1}: ${l}`)
+        .join('\n');
+      
+      extractedBlocks.push(block);
+      
+      // Skip ahead to avoid duplicates
+      i = endLine;
+    }
+  }
+
+  // If we found API patterns, return extracted content
+  if (hasApiCalls && extractedBlocks.length > 0) {
+    return {
+      hasApiCalls: true,
+      content: `// File: ${filePath}\n// API patterns found:\n\n${extractedBlocks.join('\n\n// ---\n\n')}`
+    };
+  }
+
+  // Fallback: include first 5000 chars if file looks relevant
+  if (/api|fetch|axios|service|http/i.test(filePath)) {
+    return {
+      hasApiCalls: true,
+      content: `// File: ${filePath}\n${content.slice(0, 5000)}`
+    };
+  }
+
+  return { hasApiCalls: false, content: '' };
+}
+
+/**
+ * Prioritize frontend files based on keywords
  */
 function prioritizeFiles(files, basePath) {
   return files.sort((a, b) => {
     const aPath = path.relative(basePath, a).toLowerCase();
     const bPath = path.relative(basePath, b).toLowerCase();
     
+    // Deprioritize test files
+    if (aPath.includes('test') || aPath.includes('spec')) return 1;
+    if (bPath.includes('test') || bPath.includes('spec')) return -1;
+    
+    // Prioritize src/frontend folders
+    if (aPath.includes('src/') || aPath.includes('frontend/')) {
+      if (!bPath.includes('src/') && !bPath.includes('frontend/')) return -1;
+    }
+    
     const aScore = PRIORITY_KEYWORDS.reduce((score, kw) => 
       aPath.includes(kw) ? score + 1 : score, 0);
     const bScore = PRIORITY_KEYWORDS.reduce((score, kw) => 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@merlean/analyzer",
-  "version": "1.1.1",
+  "version": "2.0.0",
   "description": "AI Bot codebase analyzer - generates site maps for AI assistant integration",
   "keywords": ["ai", "bot", "analyzer", "claude", "anthropic", "widget"],
   "author": "zmaren",