@tarunpahade/asl-parser 1.0.1

package/lib/commands/rag-command.js

@@ -0,0 +1,204 @@
+// lib/rag-command.js
+// Simple RAG search command for asl-parser
+// Usage: asl rag "your search query"
+
+const fs = require('fs');
+const path = require('path');
+const { glob } = require('glob');
+const chalk = require('chalk');
+
+// Same ignore patterns as parser.js
+const IGNORE_DIRS = [
+  'node_modules', '.git', '.next', '.nuxt', 'dist', 'build',
+  '.cache', 'coverage', '.vscode', '.idea', 'vendor',
+  '__pycache__', '.pytest_cache', 'target', 'bin', 'obj'
+];
+
+const CODE_EXTENSIONS = [
+  '.js', '.jsx', '.ts', '.tsx', '.py', '.java', '.cpp', '.c', '.h',
+  '.cs', '.go', '.rs', '.rb', '.php', '.swift', '.kt', '.scala',
+  '.vue', '.svelte', '.html', '.css', '.scss', '.less',
+  '.json', '.yaml', '.yml', '.toml', '.xml'
+];
+
+/**
+ * Simple RAG Search - finds relevant code sections
+ */
+class RagSearch {
+  constructor(basePath) {
+    this.basePath = basePath;
+    this.chunks = [];
+  }
+
+  /**
+   * Index all code files by creating searchable chunks
+   */
+  async index() {
+    console.log(chalk.blue('📚 Indexing codebase...'));
+    
+    const patterns = CODE_EXTENSIONS.map(ext => `**/*${ext}`);
+    
+    for (const pattern of patterns) {
+      try {
+        const files = await glob(pattern, {
+          cwd: this.basePath,
+          absolute: true,
+          ignore: IGNORE_DIRS.map(dir => `**/${dir}/**`)
+        });
+
+        for (const filePath of files) {
+          this.indexFile(filePath);
+        }
+      } catch (error) {
+        // Silently skip glob errors
+      }
+    }
+
+    console.log(chalk.green(`✅ Indexed ${this.chunks.length} code sections\n`));
+  }
+
+  /**
+   * Break a file into searchable chunks
+   */
+  indexFile(filePath) {
+    try {
+      const content = fs.readFileSync(filePath, 'utf-8');
+      const relativePath = path.relative(this.basePath, filePath);
+      const lines = content.split('\n');
+      const chunkSize = 20;
+
+      // Create overlapping chunks
+      for (let i = 0; i < lines.length; i += chunkSize - 5) {
+        const chunk = lines.slice(i, i + chunkSize).join('\n');
+        
+        if (chunk.trim().length > 0) {
+          this.chunks.push({
+            file: relativePath,
+            lineStart: i + 1,
+            lineEnd: Math.min(i + chunkSize, lines.length),
+            content: chunk,
+            keywords: this.extractKeywords(chunk, relativePath)
+          });
+        }
+      }
+    } catch (error) {
+      // Skip unreadable files
+    }
+  }
+
+  /**
+   * Extract keywords from code chunk
+   */
+  extractKeywords(content, filePath) {
+    const keywords = [];
+    
+    // Extract function/class names
+    const funcMatches = content.match(/(?:function|const|async|def|class)\s+(\w+)/g);
+    if (funcMatches) {
+      keywords.push(...funcMatches.map(f => f.split(/\s+/)[1]));
+    }
+
+    // Extract from file path
+    const pathParts = filePath.split('/').filter(p => p.length > 2);
+    keywords.push(...pathParts);
+
+    return keywords.map(k => k.toLowerCase()).filter(Boolean);
+  }
+
+  /**
+   * Search for relevant chunks
+   */
+  search(query, topK = 10) {
+    const queryTokens = query.toLowerCase().split(/\s+/);
+    const results = [];
+
+    for (const chunk of this.chunks) {
+      let score = 0;
+      const contentLower = chunk.content.toLowerCase();
+
+      // Score based on token matches
+      for (const token of queryTokens) {
+        // Content matches (high weight)
+        const contentMatches = (contentLower.match(new RegExp(`\\b${token}\\w*`, 'g')) || []).length;
+        score += contentMatches * 2;
+
+        // Keyword matches (medium weight)
+        const keywordMatches = chunk.keywords.filter(k => k.includes(token)).length;
+        score += keywordMatches * 1.5;
+      }
+
+      if (score > 0) {
+        results.push({ ...chunk, score });
+      }
+    }
+
+    return results
+      .sort((a, b) => b.score - a.score)
+      .slice(0, topK);
+  }
+
+  /**
+   * Format results for display
+   */
+  displayResults(results, query) {
+    if (results.length === 0) {
+      console.log(chalk.yellow(`❌ No relevant code found for: "${query}"\n`));
+      return;
+    }
+
+    console.log(chalk.green(`✅ Found ${results.length} relevant sections:\n`));
+    console.log(chalk.cyan(`📍 Query: "${query}"\n`));
+
+    results.forEach((result, idx) => {
+      const relevance = Math.min(100, Math.round((result.score / 5) * 10));
+      const fileIcon = result.file.includes('test') ? '🧪' : '📄';
+      
+      console.log(chalk.yellow(`${idx + 1}. ${fileIcon} ${result.file}`));
+      console.log(chalk.gray(`   Lines ${result.lineStart}-${result.lineEnd} | Relevance: ${relevance}%`));
+      
+      // Show code snippet
+      const snippet = result.content
+        .split('\n')
+        .slice(0, 2)
+        .join('\n')
+        .substring(0, 70);
+      
+      console.log(chalk.white(`   ${snippet}...`));
+      console.log('');
+    });
+  }
+}
+
+/**
+ * Main RAG command function
+ */
+async function ragCommand(query, targetPath = './') {
+  try {
+    const absolutePath = path.resolve(targetPath);
+    
+    if (!fs.existsSync(absolutePath)) {
+      console.error(chalk.red(`Error: Path "${targetPath}" does not exist`));
+      process.exit(1);
+    }
+
+    if (!fs.statSync(absolutePath).isDirectory()) {
+      console.error(chalk.red(`Error: "${targetPath}" is not a directory`));
+      process.exit(1);
+    }
+
+    console.log(chalk.blue(`\n🔍 RAG Search: "${query}"\n`));
+
+    // Initialize and search
+    const rag = new RagSearch(absolutePath);
+    await rag.index();
+
+    const results = rag.search(query, 10);
+    rag.displayResults(results, query);
+
+  } catch (error) {
+    console.error(chalk.red(`\nError: ${error.message}\n`));
+    process.exit(1);
+  }
+}
+
+module.exports = ragCommand;

package/lib/parser.js

@@ -0,0 +1,335 @@
+const fs = require('fs');
+const path = require('path');
+const { glob } = require('glob');
+
+// File extensions to analyze
+const CODE_EXTENSIONS = [
+  '.js', '.jsx', '.ts', '.tsx', '.py', '.java', '.cpp', '.c', '.h',
+  '.cs', '.go', '.rs', '.rb', '.php', '.swift', '.kt', '.scala',
+  '.vue', '.svelte', '.html', '.css', '.scss', '.less',
+  '.json', '.yaml', '.yml', '.toml', '.xml'
+];
+
+// Directories to ignore
+const IGNORE_DIRS = [
+  'node_modules',
+  '.git',
+  '.next',
+  '.nuxt',
+  'dist',
+  'build',
+  '.cache',
+  'coverage',
+  '.vscode',
+  '.idea',
+  'vendor',
+  '__pycache__',
+  '.pytest_cache',
+  'target',
+  'bin',
+  'obj'
+];
+
+// Files to ignore
+const IGNORE_FILES = [
+  'package-lock.json',
+  'yarn.lock',
+  'pnpm-lock.yaml',
+  '.DS_Store',
+  'Thumbs.db'
+];
+
+/**
+ * Check if a file should be ignored
+ */
+function shouldIgnore(filePath, basePath) {
+  const relativePath = path.relative(basePath, filePath);
+  const parts = relativePath.split(path.sep);
+
+  // Check if any part matches ignore directories
+  for (const part of parts) {
+    if (IGNORE_DIRS.includes(part) || part.startsWith('.')) {
+      return true;
+    }
+  }
+
+  // Check if file matches ignore patterns
+  const fileName = path.basename(filePath);
+  if (IGNORE_FILES.includes(fileName)) {
+    return true;
+  }
+
+  return false;
+}
+
+/**
+ * Get file extension
+ */
+function getFileExtension(filePath) {
+  return path.extname(filePath).toLowerCase();
+}
+
+/**
+ * Analyze a single file
+ */
+function analyzeFile(filePath) {
+  try {
+    const content = fs.readFileSync(filePath, 'utf-8');
+    const ext = getFileExtension(filePath);
+    const fileName = path.basename(filePath);
+    const relativePath = path.relative(process.cwd(), filePath);
+
+    // Basic analysis
+    const lines = content.split('\n');
+    const totalLines = lines.length;
+    const codeLines = lines.filter(line => line.trim() && !line.trim().startsWith('//') && !line.trim().startsWith('/*') && !line.trim().startsWith('*')).length;
+    const commentLines = totalLines - codeLines;
+
+    // Detect functions/classes (basic heuristics)
+    const functions = [];
+    const classes = [];
+
+    // JavaScript/TypeScript patterns
+    if (['.js', '.jsx', '.ts', '.tsx'].includes(ext)) {
+      // Match: function name(
+      const funcDeclMatches = content.matchAll(/function\s+(\w+)\s*\(/g);
+      for (const match of funcDeclMatches) {
+        functions.push(match[1]);
+      }
+      
+      // Match: const name = (async)? (
+      const arrowFuncMatches = content.matchAll(/const\s+(\w+)\s*=\s*(?:async\s+)?\(/g);
+      for (const match of arrowFuncMatches) {
+        functions.push(match[1]);
+      }
+      
+      // Match: name: (async)? (
+      const methodMatches = content.matchAll(/(\w+)\s*:\s*(?:async\s+)?\(/g);
+      for (const match of methodMatches) {
+        functions.push(match[1]);
+      }
+
+      const classMatches = content.matchAll(/class\s+(\w+)/g);
+      for (const match of classMatches) {
+        classes.push(match[1]);
+      }
+    }
+
+    // Python patterns
+    if (ext === '.py') {
+      const functionMatches = content.matchAll(/def\s+(\w+)/g);
+      for (const match of functionMatches) {
+        functions.push(match[1]);
+      }
+
+      const classMatches = content.matchAll(/class\s+(\w+)/g);
+      for (const match of classMatches) {
+        classes.push(match[1]);
+      }
+    }
+
+    // Java patterns
+    if (ext === '.java') {
+      const methodMatches = content.matchAll(/(?:public|private|protected)?\s*(?:static)?\s*\w+\s+(\w+)\s*\(/g);
+      for (const match of methodMatches) {
+        functions.push(match[1]);
+      }
+
+      const classMatches = content.matchAll(/class\s+(\w+)/g);
+      for (const match of classMatches) {
+        classes.push(match[1]);
+      }
+    }
+
+    return {
+      path: relativePath,
+      fileName,
+      extension: ext,
+      totalLines,
+      codeLines,
+      commentLines,
+      functions: [...new Set(functions)],
+      classes: [...new Set(classes)],
+      size: content.length
+    };
+  } catch (error) {
+    return {
+      path: path.relative(process.cwd(), filePath),
+      fileName: path.basename(filePath),
+      error: error.message
+    };
+  }
+}
+
+
+/**
+ * Parse entire codebase
+ */
+async function parseCodebase(basePath, outputPath) {
+  const files = [];
+  const stats = {
+    totalFiles: 0,
+    totalLines: 0,
+    totalSize: 0,
+    byExtension: {},
+    errors: []
+  };
+
+  // Find all code files
+  const patterns = CODE_EXTENSIONS.map(ext => `**/*${ext}`);
+  
+  for (const pattern of patterns) {
+    try {
+      const foundFiles = await glob(pattern, {
+        cwd: basePath,
+        absolute: true,
+        ignore: IGNORE_DIRS.map(dir => `**/${dir}/**`)
+      });
+
+      for (const filePath of foundFiles) {
+        if (!shouldIgnore(filePath, basePath)) {
+          files.push(filePath);
+        }
+      }
+    } catch (error) {
+      stats.errors.push(`Error globbing ${pattern}: ${error.message}`);
+    }
+  }
+
+  // Analyze each file
+  const analyzedFiles = files.map(filePath => {
+    const analysis = analyzeFile(filePath);
+    stats.totalFiles++;
+    
+    if (!analysis.error) {
+      stats.totalLines += analysis.totalLines;
+      stats.totalSize += analysis.size;
+      
+      const ext = analysis.extension;
+      if (!stats.byExtension[ext]) {
+        stats.byExtension[ext] = { count: 0, lines: 0 };
+      }
+      stats.byExtension[ext].count++;
+      stats.byExtension[ext].lines += analysis.totalLines;
+    } else {
+      stats.errors.push(`Error analyzing ${analysis.path}: ${analysis.error}`);
+    }
+    
+    return analysis;
+  });
+
+  // Generate markdown content
+  const markdown = generateMarkdown(analyzedFiles, stats, basePath);
+
+  // Write to file
+  fs.writeFileSync(outputPath, markdown, 'utf-8');
+}
+
+/**
+ * Generate markdown documentation
+ */
+function generateMarkdown(files, stats, basePath) {
+  const timestamp = new Date().toISOString();
+  const projectName = path.basename(basePath);
+
+  let md = `# ASL Parser Report: ${projectName}\n\n`;
+  md += `**Generated:** ${timestamp}\n\n`;
+  md += `---\n\n`;
+
+  // Summary Statistics
+  md += `## 📊 Summary Statistics\n\n`;
+  md += `- **Total Files Analyzed:** ${stats.totalFiles}\n`;
+  md += `- **Total Lines of Code:** ${stats.totalLines.toLocaleString()}\n`;
+  md += `- **Total Size:** ${(stats.totalSize / 1024).toFixed(2)} KB\n\n`;
+
+  // Files by Extension
+  md += `### Files by Extension\n\n`;
+  md += `| Extension | Count | Lines |\n`;
+  md += `|-----------|-------|-------|\n`;
+  const sortedExtensions = Object.entries(stats.byExtension)
+    .sort((a, b) => b[1].count - a[1].count);
+  for (const [ext, data] of sortedExtensions) {
+    md += `| ${ext || '(no extension)'} | ${data.count} | ${data.lines.toLocaleString()} |\n`;
+  }
+  md += `\n`;
+
+  // File Details
+  md += `## 📁 File Analysis\n\n`;
+
+  // Group files by directory
+  const filesByDir = {};
+  for (const file of files) {
+    if (file.error) continue;
+    
+    const dir = path.dirname(file.path);
+    if (!filesByDir[dir]) {
+      filesByDir[dir] = [];
+    }
+    filesByDir[dir].push(file);
+  }
+
+  const sortedDirs = Object.keys(filesByDir).sort();
+
+  for (const dir of sortedDirs) {
+    const dirFiles = filesByDir[dir];
+    md += `### ${dir || '(root)'}\n\n`;
+
+    for (const file of dirFiles) {
+      md += `#### \`${file.fileName}\`\n\n`;
+      md += `- **Path:** \`${file.path}\`\n`;
+      md += `- **Extension:** ${file.extension || '(none)'}\n`;
+      md += `- **Lines:** ${file.totalLines} (${file.codeLines} code, ${file.commentLines} comments)\n`;
+      md += `- **Size:** ${(file.size / 1024).toFixed(2)} KB\n`;
+
+      if (file.classes.length > 0) {
+        md += `- **Classes:** ${file.classes.join(', ')}\n`;
+      }
+
+      if (file.functions.length > 0) {
+        md += `- **Functions:** ${file.functions.slice(0, 10).join(', ')}${file.functions.length > 10 ? ` (and ${file.functions.length - 10} more)` : ''}\n`;
+      }
+
+      md += `\n`;
+    }
+  }
+
+  // Errors
+  if (stats.errors.length > 0) {
+    md += `## ⚠️ Errors\n\n`;
+    for (const error of stats.errors) {
+      md += `- ${error}\n`;
+    }
+    md += `\n`;
+  }
+
+  // Agent Specifications
+  md += `---\n\n`;
+  md += `## 🤖 Agent Specifications\n\n`;
+  md += `This section contains structured information about the codebase that can be used by AI agents.\n\n`;
+
+  // Generate agent-friendly structure
+  md += `### Codebase Structure\n\n`;
+  md += `\`\`\`json\n`;
+  md += JSON.stringify({
+    projectName,
+    totalFiles: stats.totalFiles,
+    totalLines: stats.totalLines,
+    extensions: stats.byExtension,
+    structure: sortedDirs.map(dir => ({
+      directory: dir,
+      files: filesByDir[dir].map(f => ({
+        name: f.fileName,
+        path: f.path,
+        extension: f.extension,
+        lines: f.totalLines,
+        classes: f.classes,
+        functions: f.functions
+      }))
+    }))
+  }, null, 2);
+  md += `\n\`\`\`\n\n`;
+
+  return md;
+}
+
+module.exports = { parseCodebase, analyzeFile };