onbuzz 3.3.0

package/src/analyzers/codeCloneDetector/index.js

@@ -0,0 +1,192 @@
+// import { Command } from 'commander'; // Not needed for programmatic use
+import fs from 'fs';
+import path from 'path';
+import { FileScanner } from './scanner.js';
+import { CodeParser } from './parser.js';
+import { CloneDetector } from './detector.js';
+import { RefactoringAnalyzer } from './analyzer.js';
+import { Reporter } from './reporter.js';
+
+/**
+ * Main clone detection orchestrator
+ */
+class CloneDetectionTool {
+  constructor(config) {
+    this.config = config;
+    this.scanner = new FileScanner(config);
+    this.parser = new CodeParser(config);
+    this.detector = new CloneDetector(config, this.parser);
+    this.analyzer = new RefactoringAnalyzer(config);
+    this.reporter = new Reporter();
+    this.parseBatchSize = config.parseBatchSize || 50; // Parse files in batches
+  }
+
+  /**
+   * Parse files in batches to avoid blocking the event loop
+   * @param {Array} files - Array of file objects to parse
+   * @returns {Promise<Array>} Array of parsed file objects
+   */
+  async parseFilesInBatches(files) {
+    const parsedFiles = [];
+
+    for (let i = 0; i < files.length; i += this.parseBatchSize) {
+      const batch = files.slice(i, i + this.parseBatchSize);
+
+      // Parse batch
+      for (const file of batch) {
+        parsedFiles.push(this.parser.parseFile(file));
+      }
+
+      // Yield to event loop between batches
+      if (i + this.parseBatchSize < files.length) {
+        await new Promise(resolve => setImmediate(resolve));
+      }
+
+      // Log progress for large projects
+      if (files.length > 100 && (i + this.parseBatchSize) % 100 === 0) {
+        const progress = Math.min(100, ((i + this.parseBatchSize) / files.length * 100)).toFixed(0);
+        console.log(`  Parsing progress: ${progress}%`);
+      }
+    }
+
+    return parsedFiles;
+  }
+
+  /**
+   * Run the complete clone detection pipeline
+   */
+  async run(projectPath, outputPath) {
+    console.log('Starting Code Clone Detection...\n');
+    console.log(`Project: ${projectPath}`);
+    console.log(`Config: minTokens=${this.config.minTokens}, minLines=${this.config.minLines}\n`);
+    
+    try {
+      // Step 1: Scan files
+      console.log('[1/5] Scanning project files...');
+      const files = await this.scanner.scanProject(projectPath);
+      
+      if (files.length === 0) {
+        console.log('No files found to analyze.');
+        return null;
+      }
+
+      // Step 2: Parse and tokenize (with batching to avoid blocking)
+      console.log('[2/5] Parsing and tokenizing code...');
+      const parsedFiles = await this.parseFilesInBatches(files);
+
+      // Step 3: Detect clones (async with yielding)
+      console.log('[3/5] Detecting code clones...');
+      const clones = await this.detector.detectClones(parsedFiles);
+      
+      if (clones.length === 0) {
+        console.log('No significant clones detected.');
+        return this.reporter.generateReport([], parsedFiles);
+      }
+
+      // Step 4: Analyze and generate refactoring advice
+      console.log('[4/5] Analyzing clones and generating refactoring advice...');
+      const analyzedClones = this.analyzer.analyzeClones(clones);
+
+      // Step 5: Generate report
+      console.log('[5/5] Generating report...');
+      const report = this.reporter.generateReport(analyzedClones, parsedFiles);
+
+      // Save report
+      if (outputPath) {
+        this.reporter.saveReport(report, outputPath);
+      }
+
+      // Print summary
+      this.reporter.printSummary(report);
+
+      // Print AI summary
+      console.log('\n📋 AI Agent Summary:');
+      console.log(this.reporter.generateAISummary(report));
+      console.log();
+
+      return report;
+    } catch (error) {
+      console.error('\n❌ Error during clone detection:', error.message);
+      console.error(error.stack);
+      throw error;
+    }
+  }
+}
+
+/**
+ * CLI Interface (disabled for now)
+ */
+async function main() {
+  // CLI functionality disabled - use CloneDetectionTool programmatically
+  console.error('CLI not available in this version. Use CloneDetectionTool programmatically.');
+  process.exit(1);
+  // const program = new Command();
+
+  program
+    .name('code-clone-detector')
+    .description('AI-powered code clone detection and refactoring advisor')
+    .version('1.0.0')
+    .argument('<project-path>', 'Path to the project directory to analyze')
+    .option('-o, --output <path>', 'Output file path for JSON report', 'clone-report.json')
+    .option('-c, --config <path>', 'Path to config file', 'config.json')
+    .option('--min-tokens <number>', 'Minimum token count for clones', parseInt)
+    .option('--min-lines <number>', 'Minimum line count for clones', parseInt)
+    .action(async (projectPath, options) => {
+      try {
+        // Load config
+        let config;
+        const configPath = path.resolve(options.config);
+        
+        if (fs.existsSync(configPath)) {
+          config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+          console.log(`Loaded config from: ${configPath}`);
+        } else {
+          console.log('Using default configuration');
+          config = {
+            minTokens: 50,
+            minLines: 5,
+            include: ['**/*.js', '**/*.jsx', '**/*.ts', '**/*.tsx'],
+            exclude: ['**/node_modules/**', '**/dist/**', '**/build/**'],
+            similarityThreshold: 0.85,
+            maxFileSize: 500000
+          };
+        }
+
+        // Override with CLI options
+        if (options.minTokens) config.minTokens = options.minTokens;
+        if (options.minLines) config.minLines = options.minLines;
+
+        // Resolve paths
+        const absProjectPath = path.resolve(projectPath);
+        const absOutputPath = path.resolve(options.output);
+
+        if (!fs.existsSync(absProjectPath)) {
+          console.error(`Error: Project path does not exist: ${absProjectPath}`);
+          process.exit(1);
+        }
+
+        // Run detection
+        const tool = new CloneDetectionTool(config);
+        const report = await tool.run(absProjectPath, absOutputPath);
+
+        if (report) {
+          console.log(`✅ Analysis complete! Review the report at: ${absOutputPath}`);
+          process.exit(0);
+        } else {
+          process.exit(1);
+        }
+      } catch (error) {
+        console.error('Fatal error:', error.message);
+        process.exit(1);
+      }
+    });
+
+  program.parse();
+}
+
+// Run CLI if executed directly
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main();
+}
+
+export { CloneDetectionTool };

package/src/analyzers/codeCloneDetector/parser.js

@@ -0,0 +1,199 @@
+import * as parser from '@babel/parser';
+import traverse from '@babel/traverse';
+import crypto from 'crypto';
+
+/**
+ * Parses and tokenizes source code
+ */
+export class CodeParser {
+  constructor(config) {
+    this.config = config;
+  }
+
+  /**
+   * Parse file and extract tokens
+   * @param {Object} file - File object with content
+   * @returns {Object} Parsed file with tokens and AST info
+   */
+  parseFile(file) {
+    const extension = file.extension;
+    
+    // Handle JavaScript/TypeScript files
+    if (['.js', '.jsx', '.ts', '.tsx', '.vue'].includes(extension)) {
+      return this.parseJavaScript(file);
+    }
+    
+    // Fallback to simple tokenization
+    return this.simpleTokenize(file);
+  }
+
+  /**
+   * Parse JavaScript/TypeScript with Babel
+   */
+  parseJavaScript(file) {
+    try {
+      const ast = parser.parse(file.content, {
+        sourceType: 'module',
+        plugins: [
+          'jsx',
+          'typescript',
+          'decorators-legacy',
+          'classProperties',
+          'optionalChaining',
+          'nullishCoalescingOperator'
+        ],
+        errorRecovery: true
+      });
+
+      const tokens = [];
+      const blocks = [];
+      let blockId = 0;
+
+      // Extract meaningful code blocks (functions, classes, etc.)
+      traverse.default(ast, {
+        FunctionDeclaration: (path) => this.extractBlock(path, file, blocks, tokens, blockId++),
+        FunctionExpression: (path) => this.extractBlock(path, file, blocks, tokens, blockId++),
+        ArrowFunctionExpression: (path) => this.extractBlock(path, file, blocks, tokens, blockId++),
+        ClassMethod: (path) => this.extractBlock(path, file, blocks, tokens, blockId++),
+        ClassDeclaration: (path) => this.extractBlock(path, file, blocks, tokens, blockId++),
+      });
+
+      // Also create a flat token sequence for the entire file
+      const fileTokens = this.extractTokenSequence(ast);
+
+      return {
+        ...file,
+        ast,
+        tokens: fileTokens,
+        blocks,
+        parsed: true
+      };
+    } catch (error) {
+      console.error(`Parse error in ${file.path}:`, error.message);
+      return this.simpleTokenize(file);
+    }
+  }
+
+  /**
+   * Extract a code block with token sequence
+   */
+  extractBlock(path, file, blocks, tokens, blockId) {
+    const node = path.node;
+    const loc = node.loc;
+    
+    if (!loc) return;
+
+    // Get the source code for this block
+    const lines = file.content.split('\n');
+    const blockCode = lines.slice(loc.start.line - 1, loc.end.line).join('\n');
+    
+    // Create token sequence for this block
+    const blockTokens = this.tokenizeCode(blockCode);
+    
+    if (blockTokens.length < this.config.minTokens) return;
+
+    blocks.push({
+      id: `${file.path}:block${blockId}`,
+      file: file.path,
+      startLine: loc.start.line,
+      endLine: loc.end.line,
+      code: blockCode,
+      tokens: blockTokens,
+      hash: this.hashTokens(blockTokens),
+      type: node.type
+    });
+  }
+
+  /**
+   * Extract token sequence from AST
+   */
+  extractTokenSequence(ast) {
+    const tokens = [];
+    
+    traverse.default(ast, {
+      enter(path) {
+        const node = path.node;
+        
+        // Normalize identifiers but keep structure
+        if (node.type === 'Identifier') {
+          tokens.push('IDENT');
+        } else if (node.type === 'Literal' || node.type === 'StringLiteral' || 
+                   node.type === 'NumericLiteral' || node.type === 'BooleanLiteral') {
+          tokens.push('LIT');
+        } else {
+          tokens.push(node.type);
+        }
+      }
+    });
+    
+    return tokens;
+  }
+
+  /**
+   * Simple tokenization for non-JS files
+   */
+  simpleTokenize(file) {
+    const tokens = this.tokenizeCode(file.content);
+    
+    return {
+      ...file,
+      tokens,
+      blocks: [{
+        id: `${file.path}:full`,
+        file: file.path,
+        startLine: 1,
+        endLine: file.content.split('\n').length,
+        code: file.content,
+        tokens,
+        hash: this.hashTokens(tokens),
+        type: 'File'
+      }],
+      parsed: false
+    };
+  }
+
+  /**
+   * Tokenize code string (language-agnostic)
+   */
+  tokenizeCode(code) {
+    // Remove comments and normalize
+    const cleaned = code
+      .replace(/\/\*[\s\S]*?\*\//g, '') // Block comments
+      .replace(/\/\/.*/g, '') // Line comments
+      .replace(/\s+/g, ' '); // Normalize whitespace
+
+    // Simple tokenization
+    const tokens = cleaned.match(/[a-zA-Z_$][a-zA-Z0-9_$]*|[{}()\[\];,.]|[+\-*/%=<>!&|]+|"[^"]*"|'[^']*'|`[^`]*`|\d+/g) || [];
+    
+    return tokens;
+  }
+
+  /**
+   * Create hash of token sequence
+   */
+  hashTokens(tokens) {
+    return crypto
+      .createHash('md5')
+      .update(tokens.join(','))
+      .digest('hex');
+  }
+
+  /**
+   * Calculate similarity between token sequences
+   */
+  calculateSimilarity(tokens1, tokens2) {
+    const len1 = tokens1.length;
+    const len2 = tokens2.length;
+    
+    if (len1 === 0 || len2 === 0) return 0;
+    
+    // Use Jaccard similarity for token sets
+    const set1 = new Set(tokens1);
+    const set2 = new Set(tokens2);
+    
+    const intersection = new Set([...set1].filter(x => set2.has(x)));
+    const union = new Set([...set1, ...set2]);
+    
+    return intersection.size / union.size;
+  }
+}

package/src/analyzers/codeCloneDetector/reporter.js

@@ -0,0 +1,148 @@
+import fs from 'fs';
+
+/**
+ * Formats and outputs clone detection results
+ */
+export class Reporter {
+  /**
+   * Generate comprehensive report
+   * @param {Array} clones - Analyzed clones with refactoring advice
+   * @param {Array} files - Parsed files
+   * @returns {Object} Report object
+   */
+  generateReport(clones, files) {
+    const summary = this.generateSummary(clones, files);
+    
+    return {
+      summary,
+      clones: clones.map(clone => this.formatClone(clone)),
+      metadata: {
+        generatedAt: new Date().toISOString(),
+        tool: 'code-clone-detector',
+        version: '1.0.0'
+      }
+    };
+  }
+
+  /**
+   * Generate summary statistics
+   */
+  generateSummary(clones, files) {
+    const totalFiles = files.length;
+    const totalClones = clones.length;
+    
+    // Calculate total duplicated lines
+    const totalDuplicatedLines = clones.reduce((sum, clone) => 
+      sum + clone.metrics.duplicatedLines, 0
+    );
+    
+    // Calculate total lines of code
+    const totalLines = files.reduce((sum, file) => 
+      sum + file.content.split('\n').length, 0
+    );
+    
+    const duplicationPercentage = totalLines > 0 
+      ? ((totalDuplicatedLines / totalLines) * 100).toFixed(2)
+      : 0;
+    
+    // Priority breakdown
+    const priorityCounts = {
+      high: clones.filter(c => c.refactoringAdvice.priority === 'high').length,
+      medium: clones.filter(c => c.refactoringAdvice.priority === 'medium').length,
+      low: clones.filter(c => c.refactoringAdvice.priority === 'low').length
+    };
+    
+    return {
+      totalFiles,
+      totalClones,
+      totalDuplicatedLines,
+      duplicationPercentage: parseFloat(duplicationPercentage),
+      priorityCounts,
+      topRefactoringOpportunities: clones.slice(0, 5).map(c => c.id)
+    };
+  }
+
+  /**
+   * Format clone for output
+   */
+  formatClone(clone) {
+    return {
+      id: clone.id,
+      type: clone.type,
+      confidence: clone.confidence,
+      instances: clone.instances,
+      metrics: clone.metrics,
+      refactoringAdvice: clone.refactoringAdvice
+    };
+  }
+
+  /**
+   * Save report to file
+   */
+  saveReport(report, outputPath) {
+    try {
+      fs.writeFileSync(outputPath, JSON.stringify(report, null, 2));
+      console.log(`\nReport saved to: ${outputPath}`);
+      return true;
+    } catch (error) {
+      console.error('Error saving report:', error.message);
+      return false;
+    }
+  }
+
+  /**
+   * Print summary to console
+   */
+  printSummary(report) {
+    const { summary } = report;
+    
+    console.log('\n' + '='.repeat(60));
+    console.log('CODE CLONE DETECTION SUMMARY');
+    console.log('='.repeat(60));
+    console.log(`Total Files Analyzed: ${summary.totalFiles}`);
+    console.log(`Total Clone Groups Found: ${summary.totalClones}`);
+    console.log(`Total Duplicated Lines: ${summary.totalDuplicatedLines}`);
+    console.log(`Duplication Percentage: ${summary.duplicationPercentage}%`);
+    console.log('\nPriority Breakdown:');
+    console.log(`  High Priority: ${summary.priorityCounts.high}`);
+    console.log(`  Medium Priority: ${summary.priorityCounts.medium}`);
+    console.log(`  Low Priority: ${summary.priorityCounts.low}`);
+    
+    if (report.clones.length > 0) {
+      console.log('\n' + '-'.repeat(60));
+      console.log('TOP REFACTORING OPPORTUNITIES');
+      console.log('-'.repeat(60));
+      
+      report.clones.slice(0, 3).forEach((clone, idx) => {
+        console.log(`\n${idx + 1}. ${clone.id} [${clone.refactoringAdvice.priority.toUpperCase()} PRIORITY]`);
+        console.log(`   Type: ${clone.type} (${(clone.confidence * 100).toFixed(0)}% confidence)`);
+        console.log(`   Instances: ${clone.metrics.instanceCount} copies across ${clone.metrics.filesCovered} files`);
+        console.log(`   Size: ${clone.metrics.lineCount} lines, ${clone.metrics.tokenCount} tokens`);
+        console.log(`   Impact Score: ${clone.metrics.impactScore}`);
+        console.log(`   Strategy: ${clone.refactoringAdvice.strategy}`);
+        console.log(`   Suggested Name: ${clone.refactoringAdvice.suggestedName}`);
+        console.log(`   Reasoning: ${clone.refactoringAdvice.reasoning}`);
+      });
+    }
+    
+    console.log('\n' + '='.repeat(60));
+  }
+
+  /**
+   * Generate AI-friendly summary
+   */
+  generateAISummary(report) {
+    const topClone = report.clones[0];
+    
+    if (!topClone) {
+      return 'No significant code duplication found in the project.';
+    }
+    
+    let summary = `Found ${report.summary.totalClones} code clone groups with ${report.summary.duplicationPercentage}% duplication. `;
+    summary += `Top priority: ${topClone.refactoringAdvice.suggestedName} appears ${topClone.metrics.instanceCount} times. `;
+    summary += `Recommended action: ${topClone.refactoringAdvice.strategy}. `;
+    summary += topClone.refactoringAdvice.reasoning;
+    
+    return summary;
+  }
+}

package/src/analyzers/codeCloneDetector/scanner.js

@@ -0,0 +1,88 @@
+import { glob } from 'glob';
+import fs from 'fs/promises';
+import path from 'path';
+
+/**
+ * Scans project directory for source files (async, non-blocking)
+ */
+export class FileScanner {
+  constructor(config) {
+    this.config = config;
+    this.batchSize = config.batchSize || 20; // Read files in parallel batches
+  }
+
+  /**
+   * Find all files matching the configuration
+   * @param {string} projectPath - Root directory to scan
+   * @returns {Promise<Array>} Array of file objects with path and content
+   */
+  async scanProject(projectPath) {
+    const files = [];
+    const allFilePaths = [];
+
+    // Find all matching files
+    for (const pattern of this.config.include) {
+      const matches = await glob(pattern, {
+        cwd: projectPath,
+        ignore: this.config.exclude,
+        absolute: true,
+        nodir: true
+      });
+      allFilePaths.push(...matches);
+    }
+
+    // Remove duplicates
+    const uniqueFilePaths = [...new Set(allFilePaths)];
+    console.log(`Found ${uniqueFilePaths.length} files to analyze`);
+
+    // Process files in batches to avoid blocking and memory issues
+    for (let i = 0; i < uniqueFilePaths.length; i += this.batchSize) {
+      const batch = uniqueFilePaths.slice(i, i + this.batchSize);
+
+      // Process batch in parallel
+      const batchResults = await Promise.all(
+        batch.map(filePath => this.readFile(filePath, projectPath))
+      );
+
+      // Filter out null results (skipped/failed files)
+      files.push(...batchResults.filter(Boolean));
+
+      // Yield to event loop between batches to prevent blocking
+      await new Promise(resolve => setImmediate(resolve));
+    }
+
+    return files;
+  }
+
+  /**
+   * Read a single file asynchronously
+   * @param {string} filePath - Absolute path to file
+   * @param {string} projectPath - Root project path for relative path calculation
+   * @returns {Promise<Object|null>} File object or null if skipped/failed
+   */
+  async readFile(filePath, projectPath) {
+    try {
+      const stats = await fs.stat(filePath);
+
+      // Skip files that are too large
+      if (stats.size > this.config.maxFileSize) {
+        console.warn(`Skipping large file: ${filePath}`);
+        return null;
+      }
+
+      const content = await fs.readFile(filePath, 'utf-8');
+      const relativePath = path.relative(projectPath, filePath);
+
+      return {
+        path: relativePath,
+        absolutePath: filePath,
+        content,
+        size: stats.size,
+        extension: path.extname(filePath)
+      };
+    } catch (error) {
+      console.error(`Error reading file ${filePath}:`, error.message);
+      return null;
+    }
+  }
+}