@merlean/analyzer 1.0.0

package/bin/cli.js

@@ -0,0 +1,156 @@
+#!/usr/bin/env node
+
+/**
+ * AI Bot Analyzer CLI (LLM-powered)
+ * 
+ * Usage:
+ *   npx @ai-bot/analyzer /path/to/codebase --name "My App" --upload https://backend.fly.dev
+ * 
+ * Options:
+ *   --name, -n     Site name (required)
+ *   --upload, -u   Backend URL to upload map (required)
+ *   --output, -o   Output file path (optional, default: ./site-map.json)
+ *   --token, -t    Auth token for upload (optional)
+ * 
+ * Requires ANTHROPIC_API_KEY environment variable
+ */
+
+// Load env from parent directory (.env in bot-smart/)
+require('dotenv').config({ path: require('path').join(__dirname, '../../.env') });
+require('dotenv').config();
+
+const { analyzeCodebase } = require('../lib/analyzer');
+const { uploadMap } = require('../lib/uploader');
+const path = require('path');
+const fs = require('fs');
+
+// Parse CLI arguments
+function parseArgs() {
+  const args = process.argv.slice(2);
+  const options = {
+    path: null,
+    name: null,
+    upload: null,
+    output: './site-map.json',
+    token: null
+  };
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    
+    if (arg === '--name' || arg === '-n') {
+      options.name = args[++i];
+    } else if (arg === '--upload' || arg === '-u') {
+      options.upload = args[++i];
+    } else if (arg === '--output' || arg === '-o') {
+      options.output = args[++i];
+    } else if (arg === '--token' || arg === '-t') {
+      options.token = args[++i];
+    } else if (arg === '--help' || arg === '-h') {
+      printHelp();
+      process.exit(0);
+    } else if (!arg.startsWith('-') && !options.path) {
+      options.path = arg;
+    }
+  }
+
+  return options;
+}
+
+function printHelp() {
+  console.log(`
+AI Bot Analyzer - Generate site maps for AI Bot integration
+
+Usage:
+  ai-bot-analyze <path> --name <name> --upload <url>
+
+Arguments:
+  <path>              Path to codebase to analyze
+
+Options:
+  --name, -n <name>   Site name (required)
+  --upload, -u <url>  Backend URL to upload map (required)
+  --output, -o <file> Output file (default: ./site-map.json)
+  --token, -t <token> Auth token for upload
+  --help, -h          Show this help
+
+Examples:
+  # Analyze and upload to production
+  ai-bot-analyze ./my-app --name "My App" --upload https://ai-bot-backend.fly.dev
+
+  # Analyze with auth token
+  ai-bot-analyze ./my-app --name "My App" --upload https://backend.fly.dev --token secret123
+
+  # Just generate local file
+  ai-bot-analyze ./my-app --name "My App" --output ./map.json
+`);
+}
+
+async function main() {
+  console.log('\n🔍 AI Bot Analyzer\n');
+
+  const options = parseArgs();
+
+  // Validate required args
+  if (!options.path) {
+    console.error('❌ Error: Path to codebase is required');
+    console.log('   Run with --help for usage');
+    process.exit(1);
+  }
+
+  if (!options.name) {
+    console.error('❌ Error: --name is required');
+    process.exit(1);
+  }
+
+  const codebasePath = path.resolve(options.path);
+  
+  if (!fs.existsSync(codebasePath)) {
+    console.error(`❌ Error: Path does not exist: ${codebasePath}`);
+    process.exit(1);
+  }
+
+  console.log(`📁 Analyzing: ${codebasePath}`);
+  console.log(`📛 Site name: ${options.name}`);
+
+  try {
+    // Analyze codebase
+    const siteMap = await analyzeCodebase(codebasePath, options.name);
+    
+    console.log('\n📊 Analysis Summary:');
+    console.log(`   Routes:     ${siteMap.routes?.length || 0}`);
+    console.log(`   Forms:      ${siteMap.forms?.length || 0}`);
+    console.log(`   Actions:    ${siteMap.actions?.length || 0}`);
+    console.log(`   Site ID:    ${siteMap.siteId}`);
+
+    // Save locally
+    const outputPath = path.resolve(options.output);
+    fs.writeFileSync(outputPath, JSON.stringify(siteMap, null, 2));
+    console.log(`\n💾 Saved to: ${outputPath}`);
+
+    // Upload if URL provided
+    if (options.upload) {
+      console.log(`\n📤 Uploading to: ${options.upload}`);
+      const result = await uploadMap(options.upload, siteMap, options.token);
+      
+      if (result.success) {
+        console.log('✅ Upload successful!');
+        console.log(`\n🎉 Integration Ready!\n`);
+        console.log(`   Add this to your website:\n`);
+        console.log(`   <script src="${options.upload}/bot.js" data-site-id="${siteMap.siteId}"></script>\n`);
+      } else {
+        console.error(`❌ Upload failed: ${result.error}`);
+        process.exit(1);
+      }
+    } else {
+      console.log('\n⚠️  No --upload URL provided. Map saved locally only.');
+    }
+
+  } catch (error) {
+    console.error(`\n❌ Error: ${error.message}`);
+    process.exit(1);
+  }
+}
+
+main();
+

package/bin/wrapper.js

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+
+/**
+ * AI Bot Analyzer - Wrapper Script
+ * 
+ * Attempts to run the compiled binary, falls back to source if not available.
+ * This allows the package to work even if binary download failed.
+ */
+
+const { spawn, execFileSync } = require('child_process');
+const path = require('path');
+const fs = require('fs');
+
+const binDir = __dirname;
+const binaryName = process.platform === 'win32' ? 'ai-bot-analyze.exe' : 'ai-bot-analyze';
+const binaryPath = path.join(binDir, binaryName);
+const sourcePath = path.join(binDir, 'cli.js');
+const sourceModeFlagPath = path.join(binDir, '..', '.source-mode');
+
+// Check if we should use source mode
+const useSourceMode = fs.existsSync(sourceModeFlagPath) || !fs.existsSync(binaryPath);
+
+if (useSourceMode) {
+  // Run source directly with Node.js
+  require('./cli.js');
+} else {
+  // Run compiled binary
+  const args = process.argv.slice(2);
+  
+  try {
+    const result = spawn(binaryPath, args, {
+      stdio: 'inherit',
+      env: process.env
+    });
+
+    result.on('error', (err) => {
+      // If binary fails, fall back to source
+      console.error('Binary execution failed, falling back to source mode...');
+      require('./cli.js');
+    });
+
+    result.on('exit', (code) => {
+      process.exit(code || 0);
+    });
+
+  } catch (err) {
+    // Fall back to source mode
+    require('./cli.js');
+  }
+}
+

package/lib/analyzer.js

@@ -0,0 +1,195 @@
+/**
+ * LLM-based Codebase Analyzer
+ * 
+ * Uses Claude to analyze codebase and extract:
+ * - API routes/endpoints
+ * - Forms and their fields
+ * - Actions/mutations
+ */
+
+require('dotenv').config({ path: require('path').join(__dirname, '../../.env') });
+require('dotenv').config();
+
+const Anthropic = require('@anthropic-ai/sdk').default;
+const fs = require('fs');
+const path = require('path');
+const { glob } = require('glob');
+const crypto = require('crypto');
+
+// Initialize Anthropic
+const anthropic = new Anthropic({
+  apiKey: process.env.ANTHROPIC_API_KEY
+});
+
+// File patterns to scan
+const SCAN_PATTERNS = [
+  '**/*.js',
+  '**/*.ts',
+  '**/*.jsx',
+  '**/*.tsx',
+  '**/*.php',
+  '**/*.py',
+  '**/*.rb'
+];
+
+// Directories to ignore
+const IGNORE_PATTERNS = [
+  '**/node_modules/**',
+  '**/vendor/**',
+  '**/.git/**',
+  '**/dist/**',
+  '**/build/**',
+  '**/__pycache__/**',
+  '**/venv/**',
+  '**/*.min.js',
+  '**/*.map'
+];
+
+// Keywords to prioritize files
+const PRIORITY_KEYWORDS = [
+  'route', 'router', 'controller', 'api', 'endpoint',
+  'form', 'submit', 'action', 'handler', 'service'
+];
+
+/**
+ * Analyze a codebase using LLM
+ */
+async function analyzeCodebase(codebasePath, siteName) {
+  const siteId = generateSiteId();
+  
+  console.log('   Scanning files...');
+
+  // Get files to scan
+  const files = await glob(SCAN_PATTERNS, {
+    cwd: codebasePath,
+    ignore: IGNORE_PATTERNS,
+    absolute: true
+  });
+
+  console.log(`   Found ${files.length} files`);
+
+  // Prioritize and limit files
+  const prioritizedFiles = prioritizeFiles(files, codebasePath);
+  const filesToAnalyze = prioritizedFiles.slice(0, 50); // Limit to avoid rate limits
+
+  console.log(`   Analyzing ${filesToAnalyze.length} priority files...`);
+
+  // Read and prepare file contents
+  const fileContents = [];
+  for (const file of filesToAnalyze) {
+    try {
+      const content = fs.readFileSync(file, 'utf-8');
+      const relativePath = path.relative(codebasePath, file);
+      
+      // Limit content per file to avoid token limits
+      const truncatedContent = content.slice(0, 3000);
+      
+      fileContents.push({
+        path: relativePath,
+        content: truncatedContent
+      });
+    } catch (error) {
+      // Skip files that can't be read
+    }
+  }
+
+  // Send to LLM for analysis
+  console.log('   Calling LLM for analysis...');
+  const analysis = await analyzeWithLLM(fileContents, siteName);
+
+  return {
+    siteId,
+    siteName,
+    analyzedAt: new Date().toISOString(),
+    framework: analysis.framework,
+    routes: analysis.routes || [],
+    forms: analysis.forms || [],
+    actions: analysis.actions || []
+  };
+}
+
+/**
+ * Prioritize files based on keywords in path/name
+ */
+function prioritizeFiles(files, basePath) {
+  return files.sort((a, b) => {
+    const aPath = path.relative(basePath, a).toLowerCase();
+    const bPath = path.relative(basePath, b).toLowerCase();
+    
+    const aScore = PRIORITY_KEYWORDS.reduce((score, kw) => 
+      aPath.includes(kw) ? score + 1 : score, 0);
+    const bScore = PRIORITY_KEYWORDS.reduce((score, kw) => 
+      bPath.includes(kw) ? score + 1 : score, 0);
+    
+    return bScore - aScore;
+  });
+}
+
+/**
+ * Analyze files using Claude
+ */
+async function analyzeWithLLM(fileContents, siteName) {
+  const filesText = fileContents.map(f => 
+    `=== ${f.path} ===\n${f.content}`
+  ).join('\n\n');
+
+  const prompt = `Analyze this codebase and extract API information.
+
+CODEBASE FILES:
+${filesText}
+
+Extract and return as JSON:
+1. framework: detected framework (express, laravel, django, fastapi, nextjs, wordpress, etc.) or null
+2. routes: array of {method, path, description} - API endpoints found
+3. forms: array of {action, method, fields: [{name, type}]} - forms found
+4. actions: array of {name, endpoint, method, description} - API calls/actions found
+
+Focus on:
+- REST API routes
+- Form submissions
+- AJAX/fetch calls
+- Controller methods
+
+Return ONLY valid JSON, no markdown or explanation:
+{"framework": "...", "routes": [...], "forms": [...], "actions": [...]}`;
+
+  try {
+    const response = await anthropic.messages.create({
+      model: 'claude-sonnet-4-20250514',
+      max_tokens: 4096,
+      messages: [{
+        role: 'user',
+        content: prompt
+      }]
+    });
+
+    const text = response.content[0].type === 'text' ? response.content[0].text : '';
+    
+    // Extract JSON from response
+    const jsonMatch = text.match(/\{[\s\S]*\}/);
+    if (jsonMatch) {
+      return JSON.parse(jsonMatch[0]);
+    }
+    
+    return { framework: null, routes: [], forms: [], actions: [] };
+
+  } catch (error) {
+    if (error.status === 429) {
+      console.log('   ⚠️ Rate limited, waiting 30s...');
+      await new Promise(r => setTimeout(r, 30000));
+      return analyzeWithLLM(fileContents, siteName);
+    }
+    console.error('   LLM error:', error.message);
+    return { framework: null, routes: [], forms: [], actions: [] };
+  }
+}
+
+/**
+ * Generate a unique site ID
+ */
+function generateSiteId() {
+  const random = crypto.randomBytes(4).toString('hex');
+  return `site_${random}`;
+}
+
+module.exports = { analyzeCodebase };

package/lib/uploader.js

@@ -0,0 +1,37 @@
+/**
+ * Upload site map to AI Bot backend
+ */
+
+async function uploadMap(backendUrl, siteMap, token) {
+  try {
+    const url = `${backendUrl.replace(/\/$/, '')}/api/sites`;
+    
+    const headers = {
+      'Content-Type': 'application/json'
+    };
+    
+    if (token) {
+      headers['Authorization'] = `Bearer ${token}`;
+    }
+
+    const response = await fetch(url, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify(siteMap)
+    });
+
+    if (!response.ok) {
+      const error = await response.text();
+      return { success: false, error: `HTTP ${response.status}: ${error}` };
+    }
+
+    const data = await response.json();
+    return { success: true, data };
+
+  } catch (error) {
+    return { success: false, error: error.message };
+  }
+}
+
+module.exports = { uploadMap };
+

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@merlean/analyzer",
+  "version": "1.0.0",
+  "description": "AI Bot codebase analyzer - generates site maps for AI assistant integration",
+  "keywords": ["ai", "bot", "analyzer", "claude", "anthropic", "widget"],
+  "author": "zmaren",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/zmaren/merlean.git",
+    "directory": "bot-smart/analyzer"
+  },
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org",
+    "access": "public"
+  },
+  "bin": {
+    "ai-bot-analyze": "./bin/wrapper.js"
+  },
+  "main": "lib/analyzer.js",
+  "files": [
+    "bin/",
+    "lib/",
+    "scripts/"
+  ],
+  "scripts": {
+    "postinstall": "node scripts/postinstall.js",
+    "build": "pkg . --targets node18-linux-x64,node18-macos-x64,node18-macos-arm64,node18-win-x64 --output dist/ai-bot-analyze",
+    "build:local": "pkg bin/cli.js --targets node18-macos-arm64 --output dist/ai-bot-analyze-macos-arm64"
+  },
+  "pkg": {
+    "scripts": ["lib/**/*.js", "bin/cli.js"],
+    "assets": [],
+    "targets": ["node18-linux-x64", "node18-macos-x64", "node18-macos-arm64", "node18-win-x64"],
+    "outputPath": "dist"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.39.0",
+    "dotenv": "^16.3.1",
+    "glob": "^10.3.10"
+  },
+  "devDependencies": {
+    "pkg": "^5.8.1"
+  }
+}

package/scripts/postinstall.js

@@ -0,0 +1,156 @@
+#!/usr/bin/env node
+
+/**
+ * Postinstall script - Downloads the correct binary for the user's platform
+ * from GitHub Releases
+ */
+
+const https = require('https');
+const fs = require('fs');
+const path = require('path');
+const { execSync } = require('child_process');
+
+const REPO = 'zmaren/merlean';
+const BINARY_NAME = 'ai-bot-analyze';
+const VERSION = require('../package.json').version;
+
+// Map Node.js platform/arch to binary names
+function getBinaryName() {
+  const platform = process.platform;
+  const arch = process.arch;
+
+  const platformMap = {
+    'darwin-arm64': 'ai-bot-analyze-macos-arm64',
+    'darwin-x64': 'ai-bot-analyze-macos-x64',
+    'linux-x64': 'ai-bot-analyze-linux-x64',
+    'win32-x64': 'ai-bot-analyze-win-x64.exe'
+  };
+
+  const key = `${platform}-${arch}`;
+  const binaryName = platformMap[key];
+
+  if (!binaryName) {
+    console.error(`❌ Unsupported platform: ${platform}-${arch}`);
+    console.error('   Supported: darwin-arm64, darwin-x64, linux-x64, win32-x64');
+    console.error('   Falling back to source mode (requires Node.js)');
+    return null;
+  }
+
+  return binaryName;
+}
+
+// Download binary from GitHub Releases
+async function downloadBinary(binaryName) {
+  const binDir = path.join(__dirname, '..', 'bin');
+  const binaryPath = path.join(binDir, process.platform === 'win32' ? `${BINARY_NAME}.exe` : BINARY_NAME);
+
+  // Skip if binary already exists
+  if (fs.existsSync(binaryPath)) {
+    console.log('✓ Binary already exists');
+    return true;
+  }
+
+  const url = `https://github.com/${REPO}/releases/download/v${VERSION}/${binaryName}`;
+  
+  console.log(`📦 Downloading ${binaryName}...`);
+  console.log(`   From: ${url}`);
+
+  return new Promise((resolve) => {
+    const download = (downloadUrl, redirectCount = 0) => {
+      if (redirectCount > 5) {
+        console.error('❌ Too many redirects');
+        resolve(false);
+        return;
+      }
+
+      const protocol = downloadUrl.startsWith('https') ? https : require('http');
+      
+      protocol.get(downloadUrl, (response) => {
+        // Handle redirects (GitHub releases redirect to S3)
+        if (response.statusCode === 301 || response.statusCode === 302) {
+          download(response.headers.location, redirectCount + 1);
+          return;
+        }
+
+        if (response.statusCode === 404) {
+          console.log(`⚠️  Binary not found for v${VERSION}`);
+          console.log('   Falling back to source mode');
+          resolve(false);
+          return;
+        }
+
+        if (response.statusCode !== 200) {
+          console.error(`❌ Download failed: HTTP ${response.statusCode}`);
+          resolve(false);
+          return;
+        }
+
+        // Ensure bin directory exists
+        if (!fs.existsSync(binDir)) {
+          fs.mkdirSync(binDir, { recursive: true });
+        }
+
+        const file = fs.createWriteStream(binaryPath);
+        response.pipe(file);
+
+        file.on('finish', () => {
+          file.close();
+          
+          // Make executable on Unix
+          if (process.platform !== 'win32') {
+            fs.chmodSync(binaryPath, 0o755);
+          }
+          
+          console.log('✓ Binary downloaded successfully');
+          resolve(true);
+        });
+
+        file.on('error', (err) => {
+          fs.unlink(binaryPath, () => {}); // Clean up
+          console.error(`❌ Write error: ${err.message}`);
+          resolve(false);
+        });
+
+      }).on('error', (err) => {
+        console.error(`❌ Download error: ${err.message}`);
+        resolve(false);
+      });
+    };
+
+    download(url);
+  });
+}
+
+// Mark that we're using source mode
+function setSourceMode() {
+  const flagPath = path.join(__dirname, '..', '.source-mode');
+  fs.writeFileSync(flagPath, 'true');
+}
+
+// Main
+async function main() {
+  // Skip in CI environments during package build
+  if (process.env.CI || process.env.PKG_EXECPATH) {
+    console.log('⏭️  Skipping binary download (CI/build environment)');
+    return;
+  }
+
+  const binaryName = getBinaryName();
+  
+  if (!binaryName) {
+    setSourceMode();
+    return;
+  }
+
+  const success = await downloadBinary(binaryName);
+  
+  if (!success) {
+    setSourceMode();
+  }
+}
+
+main().catch((err) => {
+  console.error('Postinstall error:', err.message);
+  // Don't fail install, fall back to source mode
+});
+